Optimize asm matmul dma software path

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

.gitattributes

@@ -0,0 +1,24 @@
+* text=auto eol=lf
+
+# Binary artifacts
+*.bin -text
+*.elf -text
+*.o -text
+*.a -text
+*.so -text
+*.dll -text
+*.exe -text
+
+# Documents and images
+*.pdf -text
+*.png -text
+*.jpg -text
+*.jpeg -text
+*.gif -text
+*.bmp -text
+*.ico -text
+
+# FPGA packaged IP/checkpoints can be binary archives.
+*.xcix -text
+*.dcp -text
+*.bit -text

rtl/ip/Bus_interconnects/axi2sram_sp_external.v

@@ -95,7 +95,6 @@ module axi2sram_sp_external #(
     localparam              WRITE       = 3'h2;
     localparam              SEND_B      = 3'h3;
     localparam              WAIT_WVALID = 3'h4;
-    localparam              WRITE_NOP   = 3'h5;
     localparam              READ_ADDR   = 3'h6;
 
     localparam              FIXED       = 2'b00;
@@ -158,6 +157,7 @@ module axi2sram_sp_external #(
         s_arready = 1'b0;
         // read response channel
         s_rvalid  = 1'b0;
+        s_rdata   = 'h0;
         s_rresp   = 'h0;
         s_rlast   = 'h0;
         s_rid     = ax_req_q_id;
@@ -208,7 +208,7 @@ module axi2sram_sp_external #(
                     if (s_wvalid) begin
                         req_o          = 1'b1;
                         we_o           = 1'b1;
-                        state_d        = (s_wlast) ? SEND_B : WRITE_NOP;
+                        state_d        = (s_wlast) ? SEND_B : WRITE;
                         cnt_d          = 1;
                     // we still have to wait for the first w_valid to arrive
                     end else
@@ -283,17 +283,11 @@ module axi2sram_sp_external #(
                 state_d = READ;
             end
 
-            //ext SRAM need nop between continuous write operations
-            WRITE_NOP: begin
-                s_wready = 1'b0;
-                state_d  = WRITE;
-            end
-
             // ~> we already wrote the first word here
             WRITE: begin
 
                 s_wready = 1'b1;
-                state_d  = WRITE_NOP;
+                state_d  = WRITE;
 
                 // consume a word here
                 if (s_wvalid) begin
@@ -377,4 +371,3 @@ module axi2sram_sp_external #(
     // end
 
 endmodule
-

rtl/ip/cdma/snix_axil_cdma_mux.sv

@@ -94,6 +94,23 @@ module snix_axil_cdma_mux #(
     input  logic                             mm2mm_rvalid,
     output logic                             mm2mm_rready,
 
+    // ==========================================
+    // Local endpoint interface
+    // ==========================================
+    output logic                             ep_wr_valid,
+    output logic [3:0]                       ep_wr_id,
+    output logic [3:0]                       ep_wr_bank,
+    output logic [7:0]                       ep_wr_word_addr,
+    output logic [31:0]                      ep_wr_data,
+    input  logic                             ep_wr_ready,
+
+    output logic                             ep_rd_valid,
+    output logic [3:0]                       ep_rd_id,
+    output logic [3:0]                       ep_rd_bank,
+    output logic [7:0]                       ep_rd_word_addr,
+    input  logic [31:0]                      ep_rd_data,
+    input  logic                             ep_rd_ready,
+    
     // Global Interrupt (OR'd from all channels)
     output logic                             dma_finish 
 );
@@ -102,6 +119,33 @@ module snix_axil_cdma_mux #(
     // Local Parameters & Utilities
     // ==========================================
     localparam int CH_BITS = $clog2(PORTS);
+    localparam logic [1:0] MODE_MEM2MEM = 2'd0;
+    localparam logic [1:0] MODE_MEM2EP  = 2'd1;
+    localparam logic [1:0] MODE_EP2MEM  = 2'd2;
+
+    localparam logic [7:0] SEQ_REG_CTRL       = 8'h00;
+    localparam logic [7:0] SEQ_REG_STATUS     = 8'h04;
+    localparam logic [7:0] SEQ_REG_SRC_BASE   = 8'h08;
+    localparam logic [7:0] SEQ_REG_DST_BASE   = 8'h0C;
+    localparam logic [7:0] SEQ_REG_GROUPS     = 8'h10;
+    localparam logic [7:0] SEQ_REG_MAT_BASE   = 8'h14;
+    localparam logic [7:0] SEQ_REG_SRC_STRIDE = 8'h18;
+    localparam logic [7:0] SEQ_REG_DST_STRIDE = 8'h1C;
+
+    localparam logic [11:0] MAT_CTRL_OFF   = 12'h000;
+    localparam logic [11:0] MAT_STATUS_OFF = 12'h004;
+    localparam logic [11:0] MAT_CRC_OFF    = 12'h010;
+    localparam logic [11:0] MAT_B0_AB_OFF  = 12'h100;
+    localparam logic [11:0] MAT_B0_C_OFF   = 12'h180;
+    localparam logic [11:0] MAT_B1_AB_OFF  = 12'h300;
+    localparam logic [11:0] MAT_B1_C_OFF   = 12'h380;
+
+    localparam logic [31:0] MAT_STATUS_DONE_CLR = 32'h0000_0006;
+    localparam logic [31:0] MAT_STATUS_DONE_BIT = 32'h0000_0002;
+    localparam logic [31:0] MAT_CTRL_START_B0   = 32'h0000_0001;
+    localparam logic [31:0] MAT_CTRL_START_B1   = 32'h0000_0002;
+    localparam logic [31:0] MAT_AB_BYTES        = 32'd128;
+    localparam logic [31:0] MAT_C_BYTES         = 32'd192;
     
     // Function to safely apply WSTRB to 32-bit registers
     function automatic logic [31:0] apply_wstrb(
@@ -125,12 +169,25 @@ module snix_axil_cdma_mux #(
     logic [31:0]           ch_len      [PORTS];
     logic [31:0]           ch_tag      [PORTS];
     logic [31:0]           ch_ctrl     [PORTS];
+    logic [31:0]           ch_mode     [PORTS];
+    logic [31:0]           ch_ep_cfg   [PORTS];
     
     logic [PORTS-1:0]      ch_req;   // Pending requests (Busy)
     logic [PORTS-1:0]      ch_done;  // Completion flags
 
     logic [PORTS-1:0]      arb_set_done; // From Arbiter to CSR
 
+    logic [31:0]           seq_src_base;
+    logic [31:0]           seq_dst_base;
+    logic [31:0]           seq_groups;
+    logic [31:0]           seq_mat_base;
+    logic [31:0]           seq_src_stride;
+    logic [31:0]           seq_dst_stride;
+    logic                  seq_busy;
+    logic                  seq_done_flag;
+    wire                   seq_start_req;
+    wire                   seq_done_clear_req;
+
     // ==========================================
     // Address Decoding (0x40 offset per channel)
     // ==========================================
@@ -138,6 +195,14 @@ module snix_axil_cdma_mux #(
     wire [5:0]         wr_reg = s_axil_awaddr[5:0];
     wire [CH_BITS-1:0] rd_ch  = s_axil_araddr[6 +: CH_BITS];
     wire [5:0]         rd_reg = s_axil_araddr[5:0];
+    wire               wr_seq_reg = (s_axil_awaddr[11:8] == 4'h2);
+    wire               rd_seq_reg = (s_axil_araddr[11:8] == 4'h2);
+    wire [7:0]         wr_seq_off = s_axil_awaddr[7:0];
+    wire [7:0]         rd_seq_off = s_axil_araddr[7:0];
+    assign seq_start_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_CTRL) &&
+                           s_axil_wstrb[0] && s_axil_wdata[0];
+    assign seq_done_clear_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_STATUS) &&
+                                s_axil_wstrb[0] && s_axil_wdata[1];
 
     // ==========================================
     // AXI-Lite Slave Logic (Robust Backpressure)
@@ -197,7 +262,19 @@ module snix_axil_cdma_mux #(
         end else begin
             if (do_read) begin
                 s_axil_rvalid <= 1'b1;
-                if (rd_ch < PORTS) begin
+                if (rd_seq_reg) begin
+                    case (rd_seq_off)
+                        SEQ_REG_CTRL:       s_axil_rdata <= {31'd0, seq_start_req};
+                        SEQ_REG_STATUS:     s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
+                        SEQ_REG_SRC_BASE:   s_axil_rdata <= seq_src_base;
+                        SEQ_REG_DST_BASE:   s_axil_rdata <= seq_dst_base;
+                        SEQ_REG_GROUPS:     s_axil_rdata <= seq_groups;
+                        SEQ_REG_MAT_BASE:   s_axil_rdata <= seq_mat_base;
+                        SEQ_REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
+                        SEQ_REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
+                        default:            s_axil_rdata <= 32'd0;
+                    endcase
+                end else if (rd_ch < PORTS) begin
                     case (rd_reg)
                         6'h00: s_axil_rdata <= ch_src_addr[rd_ch];
                         6'h04: s_axil_rdata <= ch_dst_addr[rd_ch];
@@ -205,6 +282,8 @@ module snix_axil_cdma_mux #(
                         6'h0C: s_axil_rdata <= ch_tag[rd_ch];
                         6'h10: s_axil_rdata <= ch_ctrl[rd_ch];
                         6'h14: s_axil_rdata <= {30'd0, ch_done[rd_ch], ch_req[rd_ch]};
+                        6'h18: s_axil_rdata <= ch_mode[rd_ch];
+                        6'h1C: s_axil_rdata <= ch_ep_cfg[rd_ch];
                         default: s_axil_rdata <= 32'd0;
                     endcase
                 end else begin
@@ -225,9 +304,16 @@ module snix_axil_cdma_mux #(
         if (!rst_n) begin
             ch_req  <= '0;
             ch_done <= '0;
+            seq_src_base   <= 32'h0;
+            seq_dst_base   <= 32'h0;
+            seq_groups     <= 32'h0;
+            seq_mat_base   <= 32'h1f50_0000;
+            seq_src_stride <= 32'd128;
+            seq_dst_stride <= 32'd192;
             for (int i=0; i<PORTS; i++) begin
                 ch_src_addr[i] <= '0; ch_dst_addr[i] <= '0;
                 ch_len[i] <= '0; ch_tag[i] <= '0; ch_ctrl[i] <= '0;
+                ch_mode[i] <= '0; ch_ep_cfg[i] <= '0;
             end
         end else begin
             // 1. Process Hardware Status Updates (Highest Priority for Done flag)
@@ -239,12 +325,23 @@ module snix_axil_cdma_mux #(
             end
 
             // 2. Process CPU Writes
-            if (do_write && wr_ch < PORTS) begin
+            if (do_write && wr_seq_reg) begin
+                case (wr_seq_off)
+                    SEQ_REG_SRC_BASE:   if (!seq_busy) seq_src_base   <= apply_wstrb(seq_src_base,   s_axil_wdata, s_axil_wstrb);
+                    SEQ_REG_DST_BASE:   if (!seq_busy) seq_dst_base   <= apply_wstrb(seq_dst_base,   s_axil_wdata, s_axil_wstrb);
+                    SEQ_REG_GROUPS:     if (!seq_busy) seq_groups     <= apply_wstrb(seq_groups,     s_axil_wdata, s_axil_wstrb);
+                    SEQ_REG_MAT_BASE:   if (!seq_busy) seq_mat_base   <= apply_wstrb(seq_mat_base,   s_axil_wdata, s_axil_wstrb);
+                    SEQ_REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
+                    SEQ_REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
+                endcase
+            end else if (do_write && wr_ch < PORTS) begin
                 case (wr_reg)
                     6'h00: if (!ch_req[wr_ch]) ch_src_addr[wr_ch] <= apply_wstrb(ch_src_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
                     6'h04: if (!ch_req[wr_ch]) ch_dst_addr[wr_ch] <= apply_wstrb(ch_dst_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
                     6'h08: if (!ch_req[wr_ch]) ch_len[wr_ch]      <= apply_wstrb(ch_len[wr_ch],      s_axil_wdata, s_axil_wstrb);
                     6'h0C: if (!ch_req[wr_ch]) ch_tag[wr_ch]      <= apply_wstrb(ch_tag[wr_ch],      s_axil_wdata, s_axil_wstrb);
+                    6'h18: if (!ch_req[wr_ch]) ch_mode[wr_ch]     <= apply_wstrb(ch_mode[wr_ch],     s_axil_wdata, s_axil_wstrb);
+                    6'h1C: if (!ch_req[wr_ch]) ch_ep_cfg[wr_ch]   <= apply_wstrb(ch_ep_cfg[wr_ch],   s_axil_wdata, s_axil_wstrb);
                     6'h10: begin
                         ch_ctrl[wr_ch] <= apply_wstrb(ch_ctrl[wr_ch], s_axil_wdata, s_axil_wstrb);
                         // Trigger Bit Processing
@@ -269,7 +366,7 @@ module snix_axil_cdma_mux #(
     // ==========================================
     // Round-Robin Arbiter & Engine Driver
     // ==========================================
-    typedef enum logic [1:0] {IDLE, RUN} state_t;
+    typedef enum logic [1:0] {IDLE, START, RUN} state_t;
     state_t state;
     
     logic [CH_BITS-1:0] cur_ch;
@@ -283,6 +380,175 @@ module snix_axil_cdma_mux #(
     logic [7:0]            engine_len;
     logic [2:0]            engine_size;
     logic                  engine_done;
+    logic [1:0]            engine_mode;
+    logic [3:0]            engine_ep_id;
+    logic [3:0]            engine_ep_bank;
+    logic [7:0]            engine_ep_word_off;
+
+    logic                  mm2mm_start;
+    logic                  mm2mm_done;
+    logic                  ep_start;
+    logic                  ep_done;
+
+    logic [ID_WIDTH-1:0]    core_awid;
+    logic [ADDR_WIDTH-1:0]  core_awaddr;
+    logic [7:0]             core_awlen;
+    logic [2:0]             core_awsize;
+    logic [1:0]             core_awburst;
+    logic                   core_awlock;
+    logic [3:0]             core_awcache;
+    logic [2:0]             core_awprot;
+    logic [3:0]             core_awqos;
+    logic [USER_WIDTH-1:0]  core_awuser;
+    logic                   core_awvalid;
+    logic                   core_awready;
+    logic [DATA_WIDTH-1:0]  core_wdata;
+    logic [DATA_WIDTH/8-1:0] core_wstrb;
+    logic                   core_wlast;
+    logic [USER_WIDTH-1:0]  core_wuser;
+    logic                   core_wvalid;
+    logic                   core_wready;
+    logic                   core_bready;
+    logic [ID_WIDTH-1:0]    core_arid;
+    logic [ADDR_WIDTH-1:0]  core_araddr;
+    logic [7:0]             core_arlen;
+    logic [2:0]             core_arsize;
+    logic [1:0]             core_arburst;
+    logic                   core_arlock;
+    logic [3:0]             core_arcache;
+    logic [2:0]             core_arprot;
+    logic [3:0]             core_arqos;
+    logic [USER_WIDTH-1:0]  core_aruser;
+    logic                   core_arvalid;
+    logic                   core_arready;
+    logic                   core_rready;
+
+    logic [ID_WIDTH-1:0]    ep_awid;
+    logic [ADDR_WIDTH-1:0]  ep_awaddr;
+    logic [7:0]             ep_awlen;
+    logic [2:0]             ep_awsize;
+    logic [1:0]             ep_awburst;
+    logic                   ep_awlock;
+    logic [3:0]             ep_awcache;
+    logic [2:0]             ep_awprot;
+    logic [3:0]             ep_awqos;
+    logic [USER_WIDTH-1:0]  ep_awuser;
+    logic                   ep_awvalid;
+    logic                   ep_awready;
+    logic [DATA_WIDTH-1:0]  ep_wdata;
+    logic [DATA_WIDTH/8-1:0] ep_wstrb;
+    logic                   ep_wlast;
+    logic [USER_WIDTH-1:0]  ep_wuser;
+    logic                   ep_wvalid;
+    logic                   ep_wready;
+    logic                   ep_bready;
+    logic [ID_WIDTH-1:0]    ep_arid;
+    logic [ADDR_WIDTH-1:0]  ep_araddr;
+    logic [7:0]             ep_arlen;
+    logic [2:0]             ep_arsize;
+    logic [1:0]             ep_arburst;
+    logic                   ep_arlock;
+    logic [3:0]             ep_arcache;
+    logic [2:0]             ep_arprot;
+    logic [3:0]             ep_arqos;
+    logic [USER_WIDTH-1:0]  ep_aruser;
+    logic                   ep_arvalid;
+    logic                   ep_arready;
+    logic                   ep_rready;
+
+    logic [ID_WIDTH-1:0]    seq_awid;
+    logic [ADDR_WIDTH-1:0]  seq_awaddr;
+    logic [7:0]             seq_awlen;
+    logic [2:0]             seq_awsize;
+    logic [1:0]             seq_awburst;
+    logic                   seq_awlock;
+    logic [3:0]             seq_awcache;
+    logic [2:0]             seq_awprot;
+    logic [3:0]             seq_awqos;
+    logic [USER_WIDTH-1:0]  seq_awuser;
+    logic                   seq_awvalid;
+    logic                   seq_awready;
+    logic [DATA_WIDTH-1:0]  seq_wdata;
+    logic [DATA_WIDTH/8-1:0] seq_wstrb;
+    logic                   seq_wlast;
+    logic [USER_WIDTH-1:0]  seq_wuser;
+    logic                   seq_wvalid;
+    logic                   seq_wready;
+    logic                   seq_bready;
+    logic [ID_WIDTH-1:0]    seq_arid;
+    logic [ADDR_WIDTH-1:0]  seq_araddr;
+    logic [7:0]             seq_arlen;
+    logic [2:0]             seq_arsize;
+    logic [1:0]             seq_arburst;
+    logic                   seq_arlock;
+    logic [3:0]             seq_arcache;
+    logic [2:0]             seq_arprot;
+    logic [3:0]             seq_arqos;
+    logic [USER_WIDTH-1:0]  seq_aruser;
+    logic                   seq_arvalid;
+    logic                   seq_arready;
+    logic                   seq_rready;
+
+    // Hardware matmul loop sequencer state. Keep these declarations before the
+    // DMA routing logic because Vivado does not allow forward enum references.
+    typedef enum logic [4:0] {
+        SQ_IDLE,
+        SQ_CLEAR_CRC,
+        SQ_LOAD0,
+        SQ_START0,
+        SQ_LOAD1,
+        SQ_WAIT_DONE,
+        SQ_STORE,
+        SQ_CLEAR_DONE,
+        SQ_START_NEXT,
+        SQ_LOAD_NEXT,
+        SQ_DONE
+    } seq_state_t;
+
+    typedef enum logic [3:0] {
+        OP_IDLE,
+        OP_READ_ADDR,
+        OP_READ_DATA,
+        OP_WRITE_ADDR,
+        OP_WRITE_DATA,
+        OP_WRITE_RESP,
+        OP_REG_READ_ADDR,
+        OP_REG_READ_DATA,
+        OP_COPY_START,
+        OP_COPY_WAIT
+    } op_state_t;
+
+    seq_state_t seq_state;
+    op_state_t  op_state;
+
+    logic        op_start_copy;
+    logic        op_start_write;
+    logic        op_start_read;
+    logic        op_done;
+    logic [31:0] op_src_addr;
+    logic [31:0] op_dst_addr;
+    logic [31:0] op_bytes;
+    logic [31:0] op_write_addr;
+    logic [31:0] op_write_data;
+    logic [31:0] op_read_addr;
+    logic [31:0] op_read_data;
+    logic [31:0] op_src_cur;
+    logic [31:0] op_dst_cur;
+    logic [31:0] op_bytes_left;
+    logic [31:0] op_data_buf;
+    logic        op_buf_valid;
+    logic        op_copy_store;
+
+    logic [31:0] seq_group_idx;
+    logic [31:0] seq_load_idx;
+    logic [31:0] seq_store_idx;
+    logic        seq_compute_bank;
+    logic [31:0] seq_status_data;
+
+    wire use_ep_engine = (state == RUN) && (engine_mode != MODE_MEM2MEM);
+    wire use_seq_engine = seq_busy && (op_state != OP_IDLE) &&
+                          (op_state != OP_COPY_START) && (op_state != OP_COPY_WAIT);
+    wire seq_dma_active = seq_busy && ((op_state == OP_COPY_START) || (op_state == OP_COPY_WAIT));
 
     // Dynamic routing to the engine based on current active channel
     assign engine_src   = ch_src_addr[cur_ch];
@@ -290,6 +556,55 @@ module snix_axil_cdma_mux #(
     assign engine_bytes = ch_len[cur_ch];
     assign engine_len   = ch_ctrl[cur_ch][13:6]; 
     assign engine_size  = ch_ctrl[cur_ch][5:3];
+    assign engine_mode  = ch_mode[cur_ch][1:0];
+    assign engine_ep_bank     = ch_ep_cfg[cur_ch][3:0];
+    assign engine_ep_word_off = ch_ep_cfg[cur_ch][15:8];
+    assign engine_ep_id       = ch_ep_cfg[cur_ch][19:16];
+    assign engine_done  = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done;
+    assign mm2mm_start  = engine_start && (engine_mode == MODE_MEM2MEM);
+    assign ep_start     = engine_start && (engine_mode != MODE_MEM2MEM);
+
+    assign core_awready = (!use_seq_engine && !use_ep_engine) && mm2mm_awready;
+    assign core_wready  = (!use_seq_engine && !use_ep_engine) && mm2mm_wready;
+    assign core_arready = (!use_seq_engine && !use_ep_engine) && mm2mm_arready;
+    assign ep_awready   = (!use_seq_engine && use_ep_engine) && mm2mm_awready;
+    assign ep_wready    = (!use_seq_engine && use_ep_engine) && mm2mm_wready;
+    assign ep_arready   = (!use_seq_engine && use_ep_engine) && mm2mm_arready;
+    assign seq_awready  = use_seq_engine && mm2mm_awready;
+    assign seq_wready   = use_seq_engine && mm2mm_wready;
+    assign seq_arready  = use_seq_engine && mm2mm_arready;
+
+    assign mm2mm_awid    = use_seq_engine ? seq_awid    : (use_ep_engine ? ep_awid    : core_awid);
+    assign mm2mm_awaddr  = use_seq_engine ? seq_awaddr  : (use_ep_engine ? ep_awaddr  : core_awaddr);
+    assign mm2mm_awlen   = use_seq_engine ? seq_awlen   : (use_ep_engine ? ep_awlen   : core_awlen);
+    assign mm2mm_awsize  = use_seq_engine ? seq_awsize  : (use_ep_engine ? ep_awsize  : core_awsize);
+    assign mm2mm_awburst = use_seq_engine ? seq_awburst : (use_ep_engine ? ep_awburst : core_awburst);
+    assign mm2mm_awlock  = use_seq_engine ? seq_awlock  : (use_ep_engine ? ep_awlock  : core_awlock);
+    assign mm2mm_awcache = use_seq_engine ? seq_awcache : (use_ep_engine ? ep_awcache : core_awcache);
+    assign mm2mm_awprot  = use_seq_engine ? seq_awprot  : (use_ep_engine ? ep_awprot  : core_awprot);
+    assign mm2mm_awqos   = use_seq_engine ? seq_awqos   : (use_ep_engine ? ep_awqos   : core_awqos);
+    assign mm2mm_awuser  = use_seq_engine ? seq_awuser  : (use_ep_engine ? ep_awuser  : core_awuser);
+    assign mm2mm_awvalid = use_seq_engine ? seq_awvalid : (use_ep_engine ? ep_awvalid : core_awvalid);
+
+    assign mm2mm_wdata   = use_seq_engine ? seq_wdata   : (use_ep_engine ? ep_wdata   : core_wdata);
+    assign mm2mm_wstrb   = use_seq_engine ? seq_wstrb   : (use_ep_engine ? ep_wstrb   : core_wstrb);
+    assign mm2mm_wlast   = use_seq_engine ? seq_wlast   : (use_ep_engine ? ep_wlast   : core_wlast);
+    assign mm2mm_wuser   = use_seq_engine ? seq_wuser   : (use_ep_engine ? ep_wuser   : core_wuser);
+    assign mm2mm_wvalid  = use_seq_engine ? seq_wvalid  : (use_ep_engine ? ep_wvalid  : core_wvalid);
+    assign mm2mm_bready  = use_seq_engine ? seq_bready  : (use_ep_engine ? ep_bready  : core_bready);
+
+    assign mm2mm_arid    = use_seq_engine ? seq_arid    : (use_ep_engine ? ep_arid    : core_arid);
+    assign mm2mm_araddr  = use_seq_engine ? seq_araddr  : (use_ep_engine ? ep_araddr  : core_araddr);
+    assign mm2mm_arlen   = use_seq_engine ? seq_arlen   : (use_ep_engine ? ep_arlen   : core_arlen);
+    assign mm2mm_arsize  = use_seq_engine ? seq_arsize  : (use_ep_engine ? ep_arsize  : core_arsize);
+    assign mm2mm_arburst = use_seq_engine ? seq_arburst : (use_ep_engine ? ep_arburst : core_arburst);
+    assign mm2mm_arlock  = use_seq_engine ? seq_arlock  : (use_ep_engine ? ep_arlock  : core_arlock);
+    assign mm2mm_arcache = use_seq_engine ? seq_arcache : (use_ep_engine ? ep_arcache : core_arcache);
+    assign mm2mm_arprot  = use_seq_engine ? seq_arprot  : (use_ep_engine ? ep_arprot  : core_arprot);
+    assign mm2mm_arqos   = use_seq_engine ? seq_arqos   : (use_ep_engine ? ep_arqos   : core_arqos);
+    assign mm2mm_aruser  = use_seq_engine ? seq_aruser  : (use_ep_engine ? ep_aruser  : core_aruser);
+    assign mm2mm_arvalid = use_seq_engine ? seq_arvalid : (use_ep_engine ? ep_arvalid : core_arvalid);
+    assign mm2mm_rready  = use_seq_engine ? seq_rready  : (use_ep_engine ? ep_rready  : core_rready);
 
     always_ff @(posedge clk) begin
         if (!rst_n) begin
@@ -304,7 +619,9 @@ module snix_axil_cdma_mux #(
 
             case (state)
                 IDLE: begin
-                    for (int i = 0; i < PORTS; i++) begin
+                    if (seq_busy) begin
+                        state <= IDLE;
+                    end else for (int i = 0; i < PORTS; i++) begin
                         logic [CH_BITS:0] check_ch_ext;
                         logic [CH_BITS-1:0] check_ch;
 
@@ -315,13 +632,17 @@ module snix_axil_cdma_mux #(
                         if (ch_req[check_ch] && !arb_set_done[check_ch]) begin
                             cur_ch       <= check_ch;
                             rr_ptr       <= (check_ch == (PORTS - 1)) ? '0 : (check_ch + 1);
-                            engine_start <= 1'b1;
+                            state        <= START;
-                            state        <= RUN;
                             break;
                         end
                     end
                 end
 
+                START: begin
+                    engine_start <= 1'b1;
+                    state        <= RUN;
+                end
+
                 RUN: begin
                     if (engine_done) begin
                         arb_set_done[cur_ch] <= 1'b1;
@@ -345,37 +666,474 @@ module snix_axil_cdma_mux #(
     ) u_core_engine (
         .clk               (clk),
         .rst_n             (rst_n),
-        .ctrl_start        (engine_start),
+        .ctrl_start        ((op_state == OP_COPY_START) ? 1'b1 : mm2mm_start),
         .ctrl_stop         (1'b0), // Tied off; can be wired if global abort is needed
-        .ctrl_src_addr     (engine_src),
+        .ctrl_src_addr     (seq_dma_active ? op_src_cur : engine_src),
-        .ctrl_dst_addr     (engine_dst),
+        .ctrl_dst_addr     (seq_dma_active ? op_dst_cur : engine_dst),
-        .ctrl_len          (engine_len),
+        .ctrl_len          (seq_dma_active ? 8'd63 : engine_len),
-        .ctrl_size         (engine_size),
+        .ctrl_size         (seq_dma_active ? 3'd2 : engine_size),
-        .ctrl_transfer_len (engine_bytes),
+        .ctrl_transfer_len (seq_dma_active ? op_bytes_left : engine_bytes),
-        .ctrl_done         (engine_done),
+        .ctrl_done         (mm2mm_done),
         // AXI4 Port Connections
-        .mm2mm_awid   (mm2mm_awid),   .mm2mm_awaddr (mm2mm_awaddr),
+        .mm2mm_awid   (core_awid),   .mm2mm_awaddr (core_awaddr),
-        .mm2mm_awlen  (mm2mm_awlen),  .mm2mm_awsize (mm2mm_awsize),
+        .mm2mm_awlen  (core_awlen),  .mm2mm_awsize (core_awsize),
-        .mm2mm_awburst(mm2mm_awburst),.mm2mm_awlock (mm2mm_awlock),
+        .mm2mm_awburst(core_awburst),.mm2mm_awlock (core_awlock),
-        .mm2mm_awcache(mm2mm_awcache),.mm2mm_awprot (mm2mm_awprot),
+        .mm2mm_awcache(core_awcache),.mm2mm_awprot (core_awprot),
-        .mm2mm_awqos  (mm2mm_awqos),  .mm2mm_awuser (mm2mm_awuser),
+        .mm2mm_awqos  (core_awqos),  .mm2mm_awuser (core_awuser),
-        .mm2mm_awvalid(mm2mm_awvalid),.mm2mm_awready(mm2mm_awready),
+        .mm2mm_awvalid(core_awvalid),.mm2mm_awready(core_awready),
-        .mm2mm_wdata  (mm2mm_wdata),  .mm2mm_wstrb  (mm2mm_wstrb),
+        .mm2mm_wdata  (core_wdata),  .mm2mm_wstrb  (core_wstrb),
-        .mm2mm_wlast  (mm2mm_wlast),  .mm2mm_wuser  (mm2mm_wuser),
+        .mm2mm_wlast  (core_wlast),  .mm2mm_wuser  (core_wuser),
-        .mm2mm_wvalid (mm2mm_wvalid), .mm2mm_wready (mm2mm_wready),
+        .mm2mm_wvalid (core_wvalid), .mm2mm_wready (core_wready),
         .mm2mm_bid    (mm2mm_bid),    .mm2mm_bresp  (mm2mm_bresp),
         .mm2mm_buser  (mm2mm_buser),  .mm2mm_bvalid (mm2mm_bvalid),
-        .mm2mm_bready (mm2mm_bready),
+        .mm2mm_bready (core_bready),
-        .mm2mm_arid   (mm2mm_arid),   .mm2mm_araddr (mm2mm_araddr),
+        .mm2mm_arid   (core_arid),   .mm2mm_araddr (core_araddr),
-        .mm2mm_arlen  (mm2mm_arlen),  .mm2mm_arsize (mm2mm_arsize),
+        .mm2mm_arlen  (core_arlen),  .mm2mm_arsize (core_arsize),
-        .mm2mm_arburst(mm2mm_arburst),.mm2mm_arlock (mm2mm_arlock),
+        .mm2mm_arburst(core_arburst),.mm2mm_arlock (core_arlock),
-        .mm2mm_arcache(mm2mm_arcache),.mm2mm_arprot (mm2mm_arprot),
+        .mm2mm_arcache(core_arcache),.mm2mm_arprot (core_arprot),
-        .mm2mm_arqos  (mm2mm_arqos),  .mm2mm_aruser (mm2mm_aruser),
+        .mm2mm_arqos  (core_arqos),  .mm2mm_aruser (core_aruser),
-        .mm2mm_arvalid(mm2mm_arvalid),.mm2mm_arready(mm2mm_arready),
+        .mm2mm_arvalid(core_arvalid),.mm2mm_arready(core_arready),
         .mm2mm_rid    (mm2mm_rid),    .mm2mm_rdata  (mm2mm_rdata),
         .mm2mm_rresp  (mm2mm_rresp),  .mm2mm_rlast  (mm2mm_rlast),
         .mm2mm_ruser  (mm2mm_ruser),  .mm2mm_rvalid (mm2mm_rvalid),
-        .mm2mm_rready (mm2mm_rready)
+        .mm2mm_rready (core_rready)
     );
 
+    // ==========================================
+    // Hardware matmul loop sequencer
+    // ==========================================
+    wire seq_has_more_load = (seq_load_idx < seq_groups);
+    wire [31:0] seq_src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride);
+    wire [31:0] seq_dst_addr_for_load = seq_mat_base + (seq_load_idx[0] ? MAT_B1_AB_OFF : MAT_B0_AB_OFF);
+    wire [31:0] seq_src_addr_for_store = seq_mat_base + (seq_store_idx[0] ? MAT_B1_C_OFF : MAT_B0_C_OFF);
+    wire [31:0] seq_dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride);
+    wire [31:0] seq_start_value_for_bank = seq_compute_bank ? MAT_CTRL_START_B1 : MAT_CTRL_START_B0;
+    wire [31:0] seq_start_value_for_loaded_bank =
+        (seq_load_idx[0] == 1'b0) ? MAT_CTRL_START_B1 : MAT_CTRL_START_B0;
+
+    assign seq_awid    = '0;
+    assign seq_awsize  = 3'd2;
+    assign seq_awburst = 2'b01;
+    assign seq_awlock  = 1'b0;
+    assign seq_awcache = 4'b0;
+    assign seq_awprot  = 3'b0;
+    assign seq_awqos   = 4'b0;
+    assign seq_awuser  = '0;
+    assign seq_wstrb   = {{(DATA_WIDTH/8){1'b1}}};
+    assign seq_wuser   = '0;
+    assign seq_arid    = '0;
+    assign seq_arsize  = 3'd2;
+    assign seq_arburst = 2'b01;
+    assign seq_arlock  = 1'b0;
+    assign seq_arcache = 4'b0;
+    assign seq_arprot  = 3'b0;
+    assign seq_arqos   = 4'b0;
+    assign seq_aruser  = '0;
+
+    assign seq_araddr  = (op_state == OP_REG_READ_ADDR) ? op_read_addr : op_src_cur;
+    assign seq_arlen   = (op_state == OP_REG_READ_ADDR) ? 8'd0 : 8'd0;
+    assign seq_arvalid = (op_state == OP_READ_ADDR) || (op_state == OP_REG_READ_ADDR);
+    assign seq_rready  = (op_state == OP_REG_READ_DATA) ||
+                         ((op_state == OP_READ_DATA) && !op_buf_valid);
+
+    assign seq_awaddr  = (op_state == OP_WRITE_ADDR && op_copy_store) ? op_dst_cur : op_write_addr;
+    assign seq_awlen   = 8'd0;
+    assign seq_awvalid = (op_state == OP_WRITE_ADDR);
+    assign seq_wdata   = op_copy_store ? op_data_buf : op_write_data;
+    assign seq_wlast   = 1'b1;
+    assign seq_wvalid  = (op_state == OP_WRITE_DATA) && (!op_copy_store || op_buf_valid);
+    assign seq_bready  = (op_state == OP_WRITE_RESP);
+
+    always_ff @(posedge clk) begin
+        if (!rst_n) begin
+            op_state      <= OP_IDLE;
+            op_done       <= 1'b0;
+            op_src_cur    <= '0;
+            op_dst_cur    <= '0;
+            op_bytes_left <= '0;
+            op_data_buf   <= '0;
+            op_buf_valid  <= 1'b0;
+            op_copy_store <= 1'b0;
+            op_read_data  <= '0;
+        end else begin
+            op_done <= 1'b0;
+
+            case (op_state)
+                OP_IDLE: begin
+                    op_buf_valid <= 1'b0;
+                    if (op_start_copy) begin
+                        op_src_cur    <= op_src_addr;
+                        op_dst_cur    <= op_dst_addr;
+                        op_bytes_left <= op_bytes;
+                        op_copy_store <= 1'b1;
+                        op_state      <= OP_COPY_START;
+                    end else if (op_start_write) begin
+                        op_copy_store <= 1'b0;
+                        op_state      <= OP_WRITE_ADDR;
+                    end else if (op_start_read) begin
+                        op_copy_store <= 1'b0;
+                        op_state      <= OP_REG_READ_ADDR;
+                    end
+                end
+
+                OP_READ_ADDR: begin
+                    if (seq_arvalid && seq_arready)
+                        op_state <= OP_READ_DATA;
+                end
+
+                OP_READ_DATA: begin
+                    if (mm2mm_rvalid && seq_rready) begin
+                        op_data_buf  <= mm2mm_rdata;
+                        op_buf_valid <= 1'b1;
+                        op_state     <= OP_WRITE_ADDR;
+                    end
+                end
+
+                OP_WRITE_ADDR: begin
+                    if (seq_awvalid && seq_awready)
+                        op_state <= OP_WRITE_DATA;
+                end
+
+                OP_WRITE_DATA: begin
+                    if (seq_wvalid && seq_wready) begin
+                        op_buf_valid <= 1'b0;
+                        op_state     <= OP_WRITE_RESP;
+                    end
+                end
+
+                OP_WRITE_RESP: begin
+                    if (mm2mm_bvalid && seq_bready) begin
+                        if (op_copy_store && (op_bytes_left > 32'd4)) begin
+                            op_src_cur    <= op_src_cur + 32'd4;
+                            op_dst_cur    <= op_dst_cur + 32'd4;
+                            op_bytes_left <= op_bytes_left - 32'd4;
+                            op_state      <= OP_READ_ADDR;
+                        end else begin
+                            op_done  <= 1'b1;
+                            op_state <= OP_IDLE;
+                        end
+                    end
+                end
+
+                OP_REG_READ_ADDR: begin
+                    if (seq_arvalid && seq_arready)
+                        op_state <= OP_REG_READ_DATA;
+                end
+
+                OP_REG_READ_DATA: begin
+                    if (mm2mm_rvalid && seq_rready) begin
+                        op_read_data <= mm2mm_rdata;
+                        op_done      <= 1'b1;
+                        op_state     <= OP_IDLE;
+                    end
+                end
+
+                OP_COPY_START: begin
+                    op_state <= OP_COPY_WAIT;
+                end
+
+                OP_COPY_WAIT: begin
+                    if (mm2mm_done) begin
+                        op_done  <= 1'b1;
+                        op_state <= OP_IDLE;
+                    end
+                end
+
+                default: op_state <= OP_IDLE;
+            endcase
+        end
+    end
+
+    always_ff @(posedge clk) begin
+        if (!rst_n) begin
+            seq_state        <= SQ_IDLE;
+            seq_busy         <= 1'b0;
+            seq_done_flag    <= 1'b0;
+            op_start_copy    <= 1'b0;
+            op_start_write   <= 1'b0;
+            op_start_read    <= 1'b0;
+            op_src_addr      <= '0;
+            op_dst_addr      <= '0;
+            op_bytes         <= '0;
+            op_write_addr    <= '0;
+            op_write_data    <= '0;
+            op_read_addr     <= '0;
+            seq_group_idx    <= '0;
+            seq_load_idx     <= '0;
+            seq_store_idx    <= '0;
+            seq_compute_bank <= 1'b0;
+            seq_status_data  <= '0;
+        end else begin
+            op_start_copy  <= 1'b0;
+            op_start_write <= 1'b0;
+            op_start_read  <= 1'b0;
+
+            if (seq_done_clear_req)
+                seq_done_flag <= 1'b0;
+
+            case (seq_state)
+                SQ_IDLE: begin
+                    seq_busy <= 1'b0;
+                    if (seq_start_req && (seq_groups != 32'd0)) begin
+                        seq_busy         <= 1'b1;
+                        seq_done_flag    <= 1'b0;
+                        seq_group_idx    <= 32'd0;
+                        seq_load_idx     <= 32'd0;
+                        seq_store_idx    <= 32'd0;
+                        seq_compute_bank <= 1'b0;
+                        op_write_addr    <= seq_mat_base + MAT_CRC_OFF;
+                        op_write_data    <= 32'd0;
+                        op_start_write   <= 1'b1;
+                        seq_state        <= SQ_CLEAR_CRC;
+                    end
+                end
+
+                SQ_CLEAR_CRC: begin
+                    if (op_done) begin
+                        op_src_addr    <= seq_src_base;
+                        op_dst_addr    <= seq_mat_base + MAT_B0_AB_OFF;
+                        op_bytes       <= MAT_AB_BYTES;
+                        op_start_copy  <= 1'b1;
+                        seq_load_idx   <= 32'd1;
+                        seq_state      <= SQ_LOAD0;
+                    end
+                end
+
+                SQ_LOAD0: begin
+                    if (op_done) begin
+                        op_write_addr  <= seq_mat_base + MAT_CTRL_OFF;
+                        op_write_data  <= MAT_CTRL_START_B0;
+                        op_start_write <= 1'b1;
+                        seq_state      <= SQ_START0;
+                    end
+                end
+
+                SQ_START0: begin
+                    if (op_done) begin
+                        if (seq_load_idx < seq_groups) begin
+                            op_src_addr    <= seq_src_base + (seq_load_idx * seq_src_stride);
+                            op_dst_addr    <= seq_mat_base + MAT_B1_AB_OFF;
+                            op_bytes       <= MAT_AB_BYTES;
+                            op_start_copy  <= 1'b1;
+                            seq_load_idx   <= seq_load_idx + 32'd1;
+                            seq_state      <= SQ_LOAD1;
+                        end else begin
+                            op_read_addr   <= seq_mat_base + MAT_STATUS_OFF;
+                            op_start_read  <= 1'b1;
+                            seq_state      <= SQ_WAIT_DONE;
+                        end
+                    end
+                end
+
+                SQ_LOAD1: begin
+                    if (op_done) begin
+                        op_write_addr  <= seq_mat_base + MAT_CTRL_OFF;
+                        op_write_data  <= seq_start_value_for_loaded_bank;
+                        op_start_write <= 1'b1;
+                        seq_state      <= SQ_START_NEXT;
+                    end
+                end
+
+                SQ_WAIT_DONE: begin
+                    if (op_done) begin
+                        seq_status_data <= op_read_data;
+                        if ((op_read_data & MAT_STATUS_DONE_BIT) != 32'd0) begin
+                            seq_store_idx <= seq_group_idx;
+                            if ((seq_group_idx + 32'd1) >= seq_groups) begin
+                                op_src_addr   <= seq_mat_base + (seq_compute_bank ? MAT_B1_C_OFF : MAT_B0_C_OFF);
+                                op_dst_addr   <= seq_dst_base + (seq_group_idx * seq_dst_stride);
+                                op_bytes      <= MAT_C_BYTES;
+                                op_start_copy <= 1'b1;
+                                seq_state     <= SQ_STORE;
+                            end else begin
+                                op_write_addr  <= seq_mat_base + MAT_STATUS_OFF;
+                                op_write_data  <= MAT_STATUS_DONE_CLR;
+                                op_start_write <= 1'b1;
+                                seq_state      <= SQ_CLEAR_DONE;
+                            end
+                        end else begin
+                            op_read_addr   <= seq_mat_base + MAT_STATUS_OFF;
+                            op_start_read  <= 1'b1;
+                        end
+                    end
+                end
+
+                SQ_STORE: begin
+                    if (op_done) begin
+                        if ((seq_store_idx + 32'd1) >= seq_groups) begin
+                            seq_state <= SQ_DONE;
+                        end else if (seq_has_more_load) begin
+                            op_src_addr    <= seq_src_addr_for_load;
+                            op_dst_addr    <= seq_dst_addr_for_load;
+                            op_bytes       <= MAT_AB_BYTES;
+                            op_start_copy  <= 1'b1;
+                            seq_load_idx   <= seq_load_idx + 32'd1;
+                            seq_state      <= SQ_LOAD_NEXT;
+                        end else begin
+                            op_read_addr   <= seq_mat_base + MAT_STATUS_OFF;
+                            op_start_read  <= 1'b1;
+                            seq_state      <= SQ_WAIT_DONE;
+                        end
+                    end
+                end
+
+                SQ_CLEAR_DONE: begin
+                    if (op_done) begin
+                        seq_group_idx    <= seq_group_idx + 32'd1;
+                        seq_compute_bank <= ~seq_compute_bank;
+                        op_src_addr      <= seq_src_addr_for_store;
+                        op_dst_addr      <= seq_dst_addr_for_store;
+                        op_bytes         <= MAT_C_BYTES;
+                        op_start_copy    <= 1'b1;
+                        seq_state        <= SQ_STORE;
+                    end
+                end
+
+                SQ_START_NEXT: begin
+                    if (op_done) begin
+                        op_read_addr  <= seq_mat_base + MAT_STATUS_OFF;
+                        op_start_read <= 1'b1;
+                        seq_state     <= SQ_WAIT_DONE;
+                    end
+                end
+
+                SQ_LOAD_NEXT: begin
+                    if (op_done) begin
+                        op_write_addr  <= seq_mat_base + MAT_CTRL_OFF;
+                        op_write_data  <= seq_start_value_for_loaded_bank;
+                        op_start_write <= 1'b1;
+                        seq_state      <= SQ_START_NEXT;
+                    end
+                end
+
+                SQ_DONE: begin
+                    seq_busy      <= 1'b0;
+                    seq_done_flag <= 1'b1;
+                    seq_state     <= SQ_IDLE;
+                end
+
+                default: seq_state <= SQ_IDLE;
+            endcase
+        end
+    end
+
+    // ==========================================
+    // Endpoint transfer engine (32-bit only in this SoC integration)
+    // ==========================================
+    typedef enum logic [2:0] {EP_IDLE, EP_AR, EP_READ, EP_AW, EP_WRITE, EP_BRESP, EP_DONE} ep_state_t;
+    ep_state_t ep_state;
+    logic [31:0] ep_axi_addr;
+    logic [31:0] ep_bytes_left;
+    logic [7:0]  ep_word_addr_r;
+
+    wire [7:0] ep_beats_total = (engine_bytes[31:2] == 30'd0) ? 8'd1 : engine_bytes[9:2];
+    wire [7:0] ep_axi_len     = ep_beats_total - 8'd1;
+    wire       ep_mode_mem2ep = (engine_mode == MODE_MEM2EP);
+    wire       ep_mode_ep2mem = (engine_mode == MODE_EP2MEM);
+
+    always_ff @(posedge clk) begin
+        if (!rst_n) begin
+            ep_state       <= EP_IDLE;
+            ep_done        <= 1'b0;
+            ep_axi_addr    <= '0;
+            ep_bytes_left  <= '0;
+            ep_word_addr_r <= '0;
+        end else begin
+            ep_done <= 1'b0;
+
+            case (ep_state)
+                EP_IDLE: begin
+                    if (ep_start) begin
+                        ep_axi_addr    <= ep_mode_mem2ep ? engine_src : engine_dst;
+                        ep_bytes_left  <= engine_bytes;
+                        ep_word_addr_r <= engine_ep_word_off;
+                        ep_state       <= ep_mode_mem2ep ? EP_AR : EP_AW;
+                    end
+                end
+
+                EP_AR: begin
+                    if (ep_arvalid && ep_arready)
+                        ep_state <= EP_READ;
+                end
+
+                EP_READ: begin
+                    if (mm2mm_rvalid && ep_rready) begin
+                        ep_word_addr_r <= ep_word_addr_r + 8'd1;
+                        ep_bytes_left  <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
+                        if (mm2mm_rlast)
+                            ep_state <= EP_DONE;
+                    end
+                end
+
+                EP_AW: begin
+                    if (ep_awvalid && ep_awready)
+                        ep_state <= EP_WRITE;
+                end
+
+                EP_WRITE: begin
+                    if (ep_wvalid && ep_wready) begin
+                        ep_word_addr_r <= ep_word_addr_r + 8'd1;
+                        ep_bytes_left  <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
+                        if (ep_wlast)
+                            ep_state <= EP_BRESP;
+                    end
+                end
+
+                EP_BRESP: begin
+                    if (mm2mm_bvalid && ep_bready)
+                        ep_state <= EP_DONE;
+                end
+
+                EP_DONE: begin
+                    ep_done  <= 1'b1;
+                    ep_state <= EP_IDLE;
+                end
+
+                default: ep_state <= EP_IDLE;
+            endcase
+        end
+    end
+
+    assign ep_arid    = '0;
+    assign ep_araddr  = ep_axi_addr;
+    assign ep_arlen   = ep_axi_len;
+    assign ep_arsize  = 3'd2;
+    assign ep_arburst = 2'b01;
+    assign ep_arlock  = 1'b0;
+    assign ep_arcache = 4'b0;
+    assign ep_arprot  = 3'b0;
+    assign ep_arqos   = 4'b0;
+    assign ep_aruser  = '0;
+    assign ep_arvalid = (ep_state == EP_AR);
+    assign ep_rready  = (ep_state == EP_READ) && ep_wr_ready;
+
+    assign ep_wr_valid     = (ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep;
+    assign ep_wr_id        = engine_ep_id;
+    assign ep_wr_bank      = engine_ep_bank;
+    assign ep_wr_word_addr = ep_word_addr_r;
+    assign ep_wr_data      = mm2mm_rdata[31:0];
+
+    assign ep_awid    = '0;
+    assign ep_awaddr  = ep_axi_addr;
+    assign ep_awlen   = ep_axi_len;
+    assign ep_awsize  = 3'd2;
+    assign ep_awburst = 2'b01;
+    assign ep_awlock  = 1'b0;
+    assign ep_awcache = 4'b0;
+    assign ep_awprot  = 3'b0;
+    assign ep_awqos   = 4'b0;
+    assign ep_awuser  = '0;
+    assign ep_awvalid = (ep_state == EP_AW);
+    assign ep_wdata   = ep_rd_data;
+    assign ep_wstrb   = {{(DATA_WIDTH/8){1'b1}}};
+    assign ep_wlast   = (ep_bytes_left <= 32'd4) && ep_wvalid;
+    assign ep_wuser   = '0;
+    assign ep_wvalid  = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready;
+    assign ep_bready  = (ep_state == EP_BRESP);
+
+    assign ep_rd_valid     = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready;
+    assign ep_rd_id        = engine_ep_id;
+    assign ep_rd_bank      = engine_ep_bank;
+    assign ep_rd_word_addr = ep_word_addr_r;
+
 endmodule

rtl/ip/matmul/matmul_axi_slave.v

@@ -2,7 +2,7 @@
 //  3-stage pipelined multiplier (LUT-based, no DSP)
 //  latency=3, throughput=1 result/cycle
 // ═══════════════════════════════════════════════════════
-(* use_dsp = "no" *)
+(* use_dsp = "no", use_dsp48 = "no", syn_multstyle = "logic" *)
 module pipelined_mult_32x32 (
     input             clk,
     input             rstn,
@@ -11,8 +11,24 @@ module pipelined_mult_32x32 (
     output reg [63:0] p
 );
     reg [31:0] a_r, b_r;
+    (* use_dsp = "no", use_dsp48 = "no", syn_multstyle = "logic" *)
     reg [63:0] prod;
 
+    function automatic [63:0] lut_mul32;
+        input [31:0] lhs;
+        input [31:0] rhs;
+        integer bit_idx;
+        reg [63:0] acc;
+        begin
+            acc = 64'b0;
+            for (bit_idx = 0; bit_idx < 32; bit_idx = bit_idx + 1) begin
+                if (rhs[bit_idx])
+                    acc = acc + ({32'b0, lhs} << bit_idx);
+            end
+            lut_mul32 = acc;
+        end
+    endfunction
+    
     always @(posedge clk) begin
         if (!rstn) begin
             a_r  <= 32'b0; b_r  <= 32'b0;
@@ -20,7 +36,7 @@ module pipelined_mult_32x32 (
         end else begin
             a_r  <= a;         // Stage 1: register inputs
             b_r  <= b;
-            prod <= a_r * b_r;  // Stage 2: LUT-based multiply
+            prod <= lut_mul32(a_r, b_r);  // Stage 2: LUT-based multiply
             p    <= prod;       // Stage 3: register output
         end
     end
@@ -80,6 +96,62 @@ module matmul_axi_slave (
     output reg        s_axi_rvalid,
     input             s_axi_rready,
 
+    // CDMA local endpoint port
+    input             ep_wr_valid,
+    input      [3:0]  ep_wr_id,
+    input      [3:0]  ep_wr_bank,
+    input      [7:0]  ep_wr_word_addr,
+    input      [31:0] ep_wr_data,
+    output            ep_wr_ready,
+
+    input             ep_rd_valid,
+    input      [3:0]  ep_rd_id,
+    input      [3:0]  ep_rd_bank,
+    input      [7:0]  ep_rd_word_addr,
+    output reg [31:0] ep_rd_data,
+    output            ep_rd_ready,
+
+    // AXI master port for autonomous matrix stream load/store
+    output reg [3:0]  m_axi_awid,
+    output reg [31:0] m_axi_awaddr,
+    output reg [7:0]  m_axi_awlen,
+    output reg [2:0]  m_axi_awsize,
+    output reg [1:0]  m_axi_awburst,
+    output reg        m_axi_awlock,
+    output reg [3:0]  m_axi_awcache,
+    output reg [2:0]  m_axi_awprot,
+    output reg        m_axi_awvalid,
+    input             m_axi_awready,
+
+    output reg [31:0] m_axi_wdata,
+    output reg [3:0]  m_axi_wstrb,
+    output reg        m_axi_wlast,
+    output reg        m_axi_wvalid,
+    input             m_axi_wready,
+
+    input      [3:0]  m_axi_bid,
+    input      [1:0]  m_axi_bresp,
+    input             m_axi_bvalid,
+    output reg        m_axi_bready,
+
+    output reg [3:0]  m_axi_arid,
+    output reg [31:0] m_axi_araddr,
+    output reg [7:0]  m_axi_arlen,
+    output reg [2:0]  m_axi_arsize,
+    output reg [1:0]  m_axi_arburst,
+    output reg        m_axi_arlock,
+    output reg [3:0]  m_axi_arcache,
+    output reg [2:0]  m_axi_arprot,
+    output reg        m_axi_arvalid,
+    input             m_axi_arready,
+
+    input      [3:0]  m_axi_rid,
+    input      [31:0] m_axi_rdata,
+    input      [1:0]  m_axi_rresp,
+    input             m_axi_rlast,
+    input             m_axi_rvalid,
+    output reg        m_axi_rready,
+
     // Interrupt output (active high level, pulsed by calc_done)
     output            intr
 );
@@ -90,6 +162,11 @@ module matmul_axi_slave (
     localparam ADDR_CTRL    = 12'h000;
     localparam ADDR_STATUS  = 12'h004;
     localparam ADDR_CRC     = 12'h010;
+    localparam ADDR_DMA_SRC_BASE   = 12'h020;
+    localparam ADDR_DMA_DST_BASE   = 12'h024;
+    localparam ADDR_DMA_GROUPS     = 12'h028;
+    localparam ADDR_DMA_SRC_STRIDE = 12'h02C;
+    localparam ADDR_DMA_DST_STRIDE = 12'h030;
     
     localparam ADDR_B0_A    = 12'h100; 
     localparam ADDR_B0_B    = 12'h140; 
@@ -104,8 +181,9 @@ module matmul_axi_slave (
     // ==========================================
     reg        req_0, req_1;
     reg        busy;
-    reg        done;
+    reg [1:0]  done_count;
     reg        error;
+    wire       done = (done_count != 2'd0);
     
     reg [31:0] b0_a [0:15];
     reg [31:0] b0_b [0:15];
@@ -115,6 +193,57 @@ module matmul_axi_slave (
     reg [31:0] b1_b [0:15];
     reg [65:0] b1_c [0:15];
 
+    reg [31:0] dma_src_base;
+    reg [31:0] dma_dst_base;
+    reg [31:0] dma_groups;
+    reg [31:0] dma_src_stride;
+    reg [31:0] dma_dst_stride;
+    reg        dma_busy;
+    reg        dma_done;
+    reg [31:0] load_group;
+    reg [31:0] store_group;
+    reg [31:0] dma_next_rd_addr;
+    reg [31:0] dma_next_wr_addr;
+    reg [31:0] completed_groups;
+    reg [1:0]  bank_loaded;
+    reg [1:0]  bank_store_pending;
+    reg [1:0]  bank_storing;
+    reg [1:0]  bank_compute_pending;
+    reg        dma_compute_bank;
+    reg        dma_load_bank;
+    reg        dma_store_bank;
+    reg        dma_start_pulse;
+    reg        dma_done_clear_pulse;
+    reg        dma_req0_pulse;
+    reg        dma_req1_pulse;
+
+    assign ep_wr_ready = 1'b1;
+    assign ep_rd_ready = 1'b1;
+
+    wire [3:0] ep_wr_lo_idx = ep_wr_word_addr[3:0];
+    wire [3:0] ep_wr_hi_idx = ep_wr_word_addr[3:0];
+    wire [7:0] ep_rd_elem_raw = ep_rd_word_addr / 8'd3;
+    wire [7:0] ep_rd_part_raw = ep_rd_word_addr % 8'd3;
+    wire [3:0] ep_rd_elem = ep_rd_elem_raw[3:0];
+    wire [1:0] ep_rd_part = ep_rd_part_raw[1:0];
+    wire [65:0] ep_rd_c0 = b0_c[ep_rd_elem];
+    wire [65:0] ep_rd_c1 = b1_c[ep_rd_elem];
+
+    always @(*) begin
+        ep_rd_data = 32'b0;
+        if (ep_rd_word_addr < 8'd48) begin
+            if (ep_rd_bank[0] == 1'b0) begin
+                if (ep_rd_part == 2'd0)      ep_rd_data = ep_rd_c0[31:0];
+                else if (ep_rd_part == 2'd1) ep_rd_data = ep_rd_c0[63:32];
+                else                         ep_rd_data = {30'b0, ep_rd_c0[65:64]};
+            end else begin
+                if (ep_rd_part == 2'd0)      ep_rd_data = ep_rd_c1[31:0];
+                else if (ep_rd_part == 2'd1) ep_rd_data = ep_rd_c1[63:32];
+                else                         ep_rd_data = {30'b0, ep_rd_c1[65:64]};
+            end
+        end
+    end
+
     function [31:0] apply_wstrb;
         input [31:0] old_value;
         input [31:0] new_value;
@@ -134,10 +263,15 @@ module matmul_axi_slave (
     reg [31:0] crc32_table [0:255];
     reg [31:0] crc_reg;
 
-    wire [31:0] crc_stream_data = s_axi_rdata;
+    wire        crc_dma_valid;
-    wire        crc_stream_valid = (r_state == R_DATA) && s_axi_rvalid && s_axi_rready &&
+    wire [31:0] crc_dma_data;
-                                    ((r_offset >= ADDR_B0_C && r_offset < (ADDR_B0_C + 12'h0C0)) ||
+    wire [31:0] crc_stream_data = crc_dma_valid ? crc_dma_data :
-                                     (r_offset >= ADDR_B1_C && r_offset < (ADDR_B1_C + 12'h0C0)));
+                                  (ep_rd_valid ? ep_rd_data : s_axi_rdata);
+    wire        crc_axi_valid = (r_state == R_DATA) && s_axi_rvalid && s_axi_rready &&
+                                ((r_offset >= ADDR_B0_C && r_offset < (ADDR_B0_C + 12'h0C0)) ||
+                                 (r_offset >= ADDR_B1_C && r_offset < (ADDR_B1_C + 12'h0C0)));
+    wire        crc_ep_valid = ep_rd_valid && ep_rd_ready && (ep_rd_word_addr < 8'd48);
+    wire        crc_stream_valid = crc_axi_valid || crc_ep_valid || crc_dma_valid;
 
     wire [31:0] crc_t0, crc_c0, crc_t1, crc_c1, crc_t2, crc_c2, crc_t3, crc_next;
     assign crc_t0 = crc32_table[(crc_reg[7:0]   ^ crc_stream_data[7:0])];
@@ -304,8 +438,50 @@ module matmul_axi_slave (
     reg       cur_bank;
     reg       calc_done_pulse;
 
+    localparam DMA_RD_IDLE = 2'd0;
+    localparam DMA_RD_ADDR = 2'd1;
+    localparam DMA_RD_DATA = 2'd2;
+    reg [1:0] dma_rd_state;
+    reg [4:0] dma_rd_count;
+    reg       dma_rd_bank;
+    wire [3:0] dma_rd_word_idx = dma_rd_count[3:0];
+    wire       dma_rd_commit = (dma_rd_state == DMA_RD_DATA) && m_axi_rvalid && m_axi_rready;
+
+    localparam DMA_WR_IDLE = 2'd0;
+    localparam DMA_WR_ADDR = 2'd1;
+    localparam DMA_WR_DATA = 2'd2;
+    localparam DMA_WR_RESP = 2'd3;
+    reg [1:0] dma_wr_state;
+    reg [5:0] dma_wr_count;
+
+    wire [7:0] dma_wr_elem_raw = dma_wr_count / 6'd3;
+    wire [7:0] dma_wr_part_raw = dma_wr_count % 6'd3;
+    wire [3:0] dma_wr_elem = dma_wr_elem_raw[3:0];
+    wire [1:0] dma_wr_part = dma_wr_part_raw[1:0];
+    wire [65:0] dma_wr_c0 = b0_c[dma_wr_elem];
+    wire [65:0] dma_wr_c1 = b1_c[dma_wr_elem];
+    wire [65:0] dma_wr_c = dma_store_bank ? dma_wr_c1 : dma_wr_c0;
+    wire [31:0] dma_wr_word =
+        (dma_wr_part == 2'd0) ? dma_wr_c[31:0] :
+        (dma_wr_part == 2'd1) ? dma_wr_c[63:32] :
+                                {30'b0, dma_wr_c[65:64]};
+    wire [5:0] dma_wr_next_count = dma_wr_count + 6'd1;
+    wire [7:0] dma_wr_next_elem_raw = dma_wr_next_count / 6'd3;
+    wire [7:0] dma_wr_next_part_raw = dma_wr_next_count % 6'd3;
+    wire [3:0] dma_wr_next_elem = dma_wr_next_elem_raw[3:0];
+    wire [1:0] dma_wr_next_part = dma_wr_next_part_raw[1:0];
+    wire [65:0] dma_wr_next_c0 = b0_c[dma_wr_next_elem];
+    wire [65:0] dma_wr_next_c1 = b1_c[dma_wr_next_elem];
+    wire [65:0] dma_wr_next_c = dma_store_bank ? dma_wr_next_c1 : dma_wr_next_c0;
+    wire [31:0] dma_wr_word_next =
+        (dma_wr_next_part == 2'd0) ? dma_wr_next_c[31:0] :
+        (dma_wr_next_part == 2'd1) ? dma_wr_next_c[63:32] :
+                                     {30'b0, dma_wr_next_c[65:64]};
+    
     // 【修改点 1】单独引出一个写脉冲信号给 CRC 清零用
     wire crc_clear_pulse = (w_state == W_DATA) && s_axi_wvalid && s_axi_wready && (w_offset == ADDR_CRC);
+    wire done_clear_pulse = (w_state == W_DATA) && s_axi_wvalid && s_axi_wready &&
+                            (w_offset == ADDR_STATUS) && s_axi_wdata[1];
 
     always @(posedge clk) begin
         if (!resetn) begin
@@ -315,15 +491,55 @@ module matmul_axi_slave (
             s_axi_bvalid  <= 1'b0;
             req_0         <= 1'b0;
             req_1         <= 1'b0;
-            done          <= 1'b0;
+            done_count    <= 2'd0;
             error         <= 1'b0;
+            dma_src_base   <= 32'd0;
+            dma_dst_base   <= 32'd0;
+            dma_groups     <= 32'd0;
+            dma_src_stride <= 32'd128;
+            dma_dst_stride <= 32'd192;
+            dma_start_pulse <= 1'b0;
+            dma_done_clear_pulse <= 1'b0;
         end else begin
+            dma_start_pulse <= 1'b0;
+            dma_done_clear_pulse <= 1'b0;
+
+            if (ep_wr_valid && ep_wr_ready) begin
+                if (ep_wr_bank[0] == 1'b0) begin
+                    if (ep_wr_word_addr < 8'd16)
+                        b0_a[ep_wr_lo_idx] <= ep_wr_data;
+                    else if (ep_wr_word_addr < 8'd32)
+                        b0_b[ep_wr_hi_idx] <= ep_wr_data;
+                    else
+                        error <= 1'b1;
+                end else begin
+                    if (ep_wr_word_addr < 8'd16)
+                        b1_a[ep_wr_lo_idx] <= ep_wr_data;
+                    else if (ep_wr_word_addr < 8'd32)
+                        b1_b[ep_wr_hi_idx] <= ep_wr_data;
+                    else
+                        error <= 1'b1;
+                end
+            end
+
+            if (dma_rd_commit) begin
+                if (dma_rd_bank == 1'b0) begin
+                    if (dma_rd_count < 5'd16)
+                        b0_a[dma_rd_word_idx] <= m_axi_rdata;
+                    else
+                        b0_b[dma_rd_word_idx] <= m_axi_rdata;
+                end else begin
+                    if (dma_rd_count < 5'd16)
+                        b1_a[dma_rd_word_idx] <= m_axi_rdata;
+                    else
+                        b1_b[dma_rd_word_idx] <= m_axi_rdata;
+                end
+            end
+
             if (req_0 && state == C_RUN && cur_bank == 1'b0) req_0 <= 1'b0;
             if (req_1 && state == C_RUN && cur_bank == 1'b1) req_1 <= 1'b0;
-            
+            if (dma_req0_pulse) req_0 <= 1'b1;
-            if (calc_done_pulse) begin
+            if (dma_req1_pulse) req_1 <= 1'b1;
-                done <= 1'b1;
-            end
             
             case (w_state)
                 W_IDLE: begin
@@ -342,11 +558,22 @@ module matmul_axi_slave (
                         if (w_offset == ADDR_CTRL) begin
                             if (s_axi_wdata[0]) req_0 <= 1'b1; 
                             if (s_axi_wdata[1]) req_1 <= 1'b1; 
+                            if (s_axi_wdata[8]) dma_start_pulse <= 1'b1;
                         end
                         else if (w_offset == ADDR_STATUS) begin
-                            if (s_axi_wdata[1]) done  <= 1'b0;
                             if (s_axi_wdata[2]) error <= 1'b0;
+                            if (s_axi_wdata[4]) dma_done_clear_pulse <= 1'b1;
                         end
+                        else if (w_offset == ADDR_DMA_SRC_BASE)
+                            dma_src_base <= apply_wstrb(dma_src_base, s_axi_wdata, s_axi_wstrb);
+                        else if (w_offset == ADDR_DMA_DST_BASE)
+                            dma_dst_base <= apply_wstrb(dma_dst_base, s_axi_wdata, s_axi_wstrb);
+                        else if (w_offset == ADDR_DMA_GROUPS)
+                            dma_groups <= apply_wstrb(dma_groups, s_axi_wdata, s_axi_wstrb);
+                        else if (w_offset == ADDR_DMA_SRC_STRIDE)
+                            dma_src_stride <= apply_wstrb(dma_src_stride, s_axi_wdata, s_axi_wstrb);
+                        else if (w_offset == ADDR_DMA_DST_STRIDE)
+                            dma_dst_stride <= apply_wstrb(dma_dst_stride, s_axi_wdata, s_axi_wstrb);
                         // 【修改点 2】删除 else if (w_offset == ADDR_CRC) { crc_reg <= ... } 防止多重驱动
                         else if (w_offset >= ADDR_B0_A && w_offset < ADDR_B0_B)
                             b0_a[(w_offset - ADDR_B0_A) >> 2] <= apply_wstrb(b0_a[(w_offset - ADDR_B0_A) >> 2], s_axi_wdata, s_axi_wstrb);
@@ -378,6 +605,12 @@ module matmul_axi_slave (
                 end
                 default: w_state <= W_IDLE;
             endcase
+
+            case ({calc_done_pulse, done_clear_pulse})
+                2'b10: if (done_count != 2'd3) done_count <= done_count + 2'd1;
+                2'b01: if (done_count != 2'd0) done_count <= done_count - 2'd1;
+                default: done_count <= done_count;
+            endcase
         end
     end
 
@@ -462,7 +695,17 @@ module matmul_axi_slave (
             else if (current_r_offset == ADDR_CRC)
                 s_axi_rdata = crc_reg ^ 32'hFFFFFFFF;
             else if (current_r_offset == ADDR_STATUS)
-                s_axi_rdata = {29'b0, error, done, busy};
+                s_axi_rdata = {27'b0, dma_done, dma_busy, error, done, busy};
+            else if (current_r_offset == ADDR_DMA_SRC_BASE)
+                s_axi_rdata = dma_src_base;
+            else if (current_r_offset == ADDR_DMA_DST_BASE)
+                s_axi_rdata = dma_dst_base;
+            else if (current_r_offset == ADDR_DMA_GROUPS)
+                s_axi_rdata = dma_groups;
+            else if (current_r_offset == ADDR_DMA_SRC_STRIDE)
+                s_axi_rdata = dma_src_stride;
+            else if (current_r_offset == ADDR_DMA_DST_STRIDE)
+                s_axi_rdata = dma_dst_stride;
             else if (current_r_offset >= ADDR_B0_A && current_r_offset < ADDR_B0_B)
                 s_axi_rdata = b0_a[(current_r_offset - ADDR_B0_A) >> 2];
             else if (current_r_offset >= ADDR_B0_B && current_r_offset < ADDR_B0_C)
@@ -518,8 +761,33 @@ module matmul_axi_slave (
     // Pipeline delay shift registers for accumulation (4 stages = 3 multiplier pipe + 1)
     reg [3:0] acc_idx_d1, acc_idx_d2, acc_idx_d3, acc_idx_d4;
     reg       acc_bank_d1, acc_bank_d2, acc_bank_d3, acc_bank_d4;
+    reg       acc_valid_d1, acc_valid_d2, acc_valid_d3, acc_valid_d4;
     reg [4:0] pipe_cnt;
 
+    task start_calc_bank;
+        input bank;
+        begin
+            cur_bank    <= bank;
+            busy        <= 1'b1;
+            state       <= C_RUN;
+            pipe_cnt    <= 5'd1;
+            acc_idx_d1  <= 4'd0;
+            acc_bank_d1 <= bank;
+            acc_valid_d1 <= 1'b1;
+            if (bank == 1'b0) begin
+                m_in_a[0] <= b0_a[4'd0]; m_in_b[0] <= b0_b[4'd0];
+                m_in_a[1] <= b0_a[4'd1]; m_in_b[1] <= b0_b[4'd4];
+                m_in_a[2] <= b0_a[4'd2]; m_in_b[2] <= b0_b[4'd8];
+                m_in_a[3] <= b0_a[4'd3]; m_in_b[3] <= b0_b[4'd12];
+            end else begin
+                m_in_a[0] <= b1_a[4'd0]; m_in_b[0] <= b1_b[4'd0];
+                m_in_a[1] <= b1_a[4'd1]; m_in_b[1] <= b1_b[4'd4];
+                m_in_a[2] <= b1_a[4'd2]; m_in_b[2] <= b1_b[4'd8];
+                m_in_a[3] <= b1_a[4'd3]; m_in_b[3] <= b1_b[4'd12];
+            end
+        end
+    endtask
+
     always @(posedge clk) begin
         if (!resetn) begin
             state           <= C_IDLE;
@@ -535,6 +803,10 @@ module matmul_axi_slave (
             acc_bank_d2     <= 1'b0;
             acc_bank_d3     <= 1'b0;
             acc_bank_d4     <= 1'b0;
+            acc_valid_d1    <= 1'b0;
+            acc_valid_d2    <= 1'b0;
+            acc_valid_d3    <= 1'b0;
+            acc_valid_d4    <= 1'b0;
         end else begin
             calc_done_pulse <= 1'b0;
 
@@ -542,24 +814,24 @@ module matmul_axi_slave (
             acc_idx_d4  <= acc_idx_d3;
             acc_idx_d3  <= acc_idx_d2;
             acc_idx_d2  <= acc_idx_d1;
-            acc_idx_d1  <= pipe_cnt[3:0];
+            acc_idx_d1  <= 4'd0;
             acc_bank_d4 <= acc_bank_d3;
             acc_bank_d3 <= acc_bank_d2;
             acc_bank_d2 <= acc_bank_d1;
-            acc_bank_d1 <= cur_bank;
+            acc_bank_d1 <= 1'b0;
+            acc_valid_d4 <= acc_valid_d3;
+            acc_valid_d3 <= acc_valid_d2;
+            acc_valid_d2 <= acc_valid_d1;
+            acc_valid_d1 <= 1'b0;
 
             case (state)
                 C_IDLE: begin
-                    pipe_cnt <= 5'd0;
                     if (req_0) begin
-                        cur_bank <= 1'b0;
+                        start_calc_bank(1'b0);
-                        busy     <= 1'b1;
-                        state    <= C_RUN;
                     end else if (req_1) begin
-                        cur_bank <= 1'b1;
+                        start_calc_bank(1'b1);
-                        busy     <= 1'b1;
-                        state    <= C_RUN;
                     end else begin
+                        pipe_cnt <= 5'd0;
                         busy     <= 1'b0;
                     end
                 end
@@ -567,6 +839,9 @@ module matmul_axi_slave (
                 C_RUN: begin
                     // Issue 4 new operands every cycle (pipe_cnt 0-15, 16 rounds)
                     if (pipe_cnt < 5'd16) begin
+                        acc_idx_d1   <= pipe_cnt[3:0];
+                        acc_bank_d1  <= cur_bank;
+                        acc_valid_d1 <= 1'b1;
                         if (cur_bank == 1'b0) begin
                             m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}];
                             m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}];
@@ -580,8 +855,8 @@ module matmul_axi_slave (
                         end
                     end
 
-                    // Accumulate results from 4 cycles ago (pipe_cnt 4-19)
+                    // Accumulate results from 4 cycles ago.
-                    if (pipe_cnt >= 5'd4 && pipe_cnt < 5'd20) begin
+                    if (acc_valid_d4) begin
                         if (acc_bank_d4 == 1'b0)
                             b0_c[acc_idx_d4] <= ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
                                                 ({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
@@ -590,10 +865,19 @@ module matmul_axi_slave (
                                                 ({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
                     end
 
-                    if (pipe_cnt == 5'd20) begin
+                    if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin
                         calc_done_pulse <= 1'b1;
-                        busy            <= 1'b0;
+                    end
-                        state           <= C_IDLE;
+
+                    if (pipe_cnt < 5'd16) begin
+                        pipe_cnt <= pipe_cnt + 5'd1;
+                    end else if (cur_bank == 1'b0 && req_1) begin
+                        start_calc_bank(1'b1);
+                    end else if (cur_bank == 1'b1 && req_0) begin
+                        start_calc_bank(1'b0);
+                    end else if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin
+                        busy  <= 1'b0;
+                        state <= C_IDLE;
                     end else begin
                         pipe_cnt <= pipe_cnt + 5'd1;
                     end
@@ -602,6 +886,212 @@ module matmul_axi_slave (
         end
     end
 
+    // ==========================================
+    // Autonomous AXI master loader/storer
+    // ==========================================
+    assign crc_dma_valid = (dma_wr_state == DMA_WR_DATA) && m_axi_wvalid && m_axi_wready;
+    assign crc_dma_data  = m_axi_wdata;
+
+    always @(posedge clk) begin
+        if (!resetn) begin
+            dma_busy           <= 1'b0;
+            dma_done           <= 1'b0;
+            load_group         <= 32'd0;
+            store_group        <= 32'd0;
+            dma_next_rd_addr   <= 32'd0;
+            dma_next_wr_addr   <= 32'd0;
+            completed_groups   <= 32'd0;
+            bank_loaded        <= 2'b00;
+            bank_store_pending <= 2'b00;
+            bank_storing       <= 2'b00;
+            bank_compute_pending <= 2'b00;
+            dma_compute_bank   <= 1'b0;
+            dma_load_bank      <= 1'b0;
+            dma_store_bank     <= 1'b0;
+            dma_rd_state       <= DMA_RD_IDLE;
+            dma_rd_count       <= 5'd0;
+            dma_rd_bank        <= 1'b0;
+            dma_wr_state       <= DMA_WR_IDLE;
+            dma_wr_count       <= 6'd0;
+
+            m_axi_arid         <= 4'd0;
+            m_axi_araddr       <= 32'd0;
+            m_axi_arlen        <= 8'd31;
+            m_axi_arsize       <= 3'd2;
+            m_axi_arburst      <= 2'b01;
+            m_axi_arlock       <= 1'b0;
+            m_axi_arcache      <= 4'b0;
+            m_axi_arprot       <= 3'b0;
+            m_axi_arvalid      <= 1'b0;
+            m_axi_rready       <= 1'b0;
+
+            m_axi_awid         <= 4'd0;
+            m_axi_awaddr       <= 32'd0;
+            m_axi_awlen        <= 8'd47;
+            m_axi_awsize       <= 3'd2;
+            m_axi_awburst      <= 2'b01;
+            m_axi_awlock       <= 1'b0;
+            m_axi_awcache      <= 4'b0;
+            m_axi_awprot       <= 3'b0;
+            m_axi_awvalid      <= 1'b0;
+            m_axi_wdata        <= 32'd0;
+            m_axi_wstrb        <= 4'hF;
+            m_axi_wlast        <= 1'b0;
+            m_axi_wvalid       <= 1'b0;
+            m_axi_bready       <= 1'b0;
+            dma_req0_pulse     <= 1'b0;
+            dma_req1_pulse     <= 1'b0;
+        end else begin
+            dma_req0_pulse <= 1'b0;
+            dma_req1_pulse <= 1'b0;
+
+            if (dma_done_clear_pulse)
+                dma_done <= 1'b0;
+
+            if (dma_start_pulse && !dma_busy && dma_groups != 32'd0) begin
+                dma_busy           <= 1'b1;
+                dma_done           <= 1'b0;
+                load_group         <= 32'd0;
+                store_group        <= 32'd0;
+                dma_next_rd_addr   <= dma_src_base;
+                dma_next_wr_addr   <= dma_dst_base;
+                completed_groups   <= 32'd0;
+                bank_loaded        <= 2'b00;
+                bank_store_pending <= 2'b00;
+                bank_storing       <= 2'b00;
+                bank_compute_pending <= 2'b00;
+                dma_compute_bank   <= 1'b0;
+                dma_load_bank      <= 1'b0;
+                dma_store_bank     <= 1'b0;
+                dma_rd_state       <= DMA_RD_IDLE;
+                dma_wr_state       <= DMA_WR_IDLE;
+            end
+
+            if (dma_busy) begin
+                if (state == C_RUN)
+                    bank_compute_pending[cur_bank] <= 1'b0;
+
+                if (bank_loaded[0] && !req_0 && !bank_store_pending[0] && !bank_storing[0] &&
+                    !bank_compute_pending[0] && !(state == C_RUN && cur_bank == 1'b0)) begin
+                    dma_req0_pulse <= 1'b1;
+                    bank_loaded[0] <= 1'b0;
+                    bank_compute_pending[0] <= 1'b1;
+                    dma_compute_bank <= 1'b0;
+                end else if (bank_loaded[1] && !req_1 && !bank_store_pending[1] && !bank_storing[1] &&
+                             !bank_compute_pending[1] && !(state == C_RUN && cur_bank == 1'b1)) begin
+                    dma_req1_pulse <= 1'b1;
+                    bank_loaded[1] <= 1'b0;
+                    bank_compute_pending[1] <= 1'b1;
+                    dma_compute_bank <= 1'b1;
+                end
+            end
+
+            if (calc_done_pulse && dma_busy) begin
+                bank_store_pending[cur_bank] <= 1'b1;
+            end
+
+            case (dma_rd_state)
+                DMA_RD_IDLE: begin
+                    m_axi_arvalid <= 1'b0;
+                    m_axi_rready  <= 1'b0;
+                    if (dma_busy && load_group < dma_groups && !bank_loaded[dma_load_bank] &&
+                        !bank_compute_pending[dma_load_bank] &&
+                        !((dma_load_bank == 1'b0) ? req_0 : req_1) &&
+                        !(state == C_RUN && cur_bank == dma_load_bank)) begin
+                        dma_rd_bank   <= dma_load_bank;
+                        dma_rd_count  <= 5'd0;
+                        m_axi_araddr  <= dma_next_rd_addr;
+                        m_axi_arlen   <= 8'd31;
+                        m_axi_arsize  <= 3'd2;
+                        m_axi_arburst <= 2'b01;
+                        m_axi_arvalid <= 1'b1;
+                        dma_rd_state  <= DMA_RD_ADDR;
+                    end
+                end
+                DMA_RD_ADDR: begin
+                    if (m_axi_arvalid && m_axi_arready) begin
+                        m_axi_arvalid <= 1'b0;
+                        m_axi_rready  <= 1'b1;
+                        dma_rd_state  <= DMA_RD_DATA;
+                    end
+                end
+                DMA_RD_DATA: begin
+                    if (m_axi_rvalid && m_axi_rready) begin
+                        if (m_axi_rlast) begin
+                            m_axi_rready <= 1'b0;
+                            bank_loaded[dma_rd_bank] <= 1'b1;
+                            load_group <= load_group + 32'd1;
+                            dma_next_rd_addr <= dma_next_rd_addr + dma_src_stride;
+                            dma_load_bank <= ~dma_load_bank;
+                            dma_rd_state <= DMA_RD_IDLE;
+                        end else begin
+                            dma_rd_count <= dma_rd_count + 5'd1;
+                        end
+                    end
+                end
+                default: dma_rd_state <= DMA_RD_IDLE;
+            endcase
+
+            case (dma_wr_state)
+                DMA_WR_IDLE: begin
+                    m_axi_awvalid <= 1'b0;
+                    m_axi_wvalid  <= 1'b0;
+                    m_axi_bready  <= 1'b0;
+                    if (dma_busy && bank_store_pending[dma_store_bank]) begin
+                        dma_wr_count <= 6'd0;
+                        bank_store_pending[dma_store_bank] <= 1'b0;
+                        bank_storing[dma_store_bank] <= 1'b1;
+                        m_axi_awaddr  <= dma_next_wr_addr;
+                        m_axi_awlen   <= 8'd47;
+                        m_axi_awsize  <= 3'd2;
+                        m_axi_awburst <= 2'b01;
+                        m_axi_awvalid <= 1'b1;
+                        dma_wr_state  <= DMA_WR_ADDR;
+                    end
+                end
+                DMA_WR_ADDR: begin
+                    if (m_axi_awvalid && m_axi_awready) begin
+                        m_axi_awvalid <= 1'b0;
+                        m_axi_wvalid  <= 1'b1;
+                        m_axi_wdata   <= dma_wr_word;
+                        m_axi_wlast   <= 1'b0;
+                        dma_wr_state  <= DMA_WR_DATA;
+                    end
+                end
+                DMA_WR_DATA: begin
+                    if (m_axi_wvalid && m_axi_wready) begin
+                        if (dma_wr_count == 6'd47) begin
+                            m_axi_wvalid <= 1'b0;
+                            m_axi_wlast  <= 1'b0;
+                            m_axi_bready <= 1'b1;
+                            dma_wr_state <= DMA_WR_RESP;
+                        end else begin
+                            dma_wr_count <= dma_wr_count + 6'd1;
+                            m_axi_wdata  <= dma_wr_word_next;
+                            m_axi_wlast  <= (dma_wr_count == 6'd46);
+                        end
+                    end
+                end
+                DMA_WR_RESP: begin
+                    if (m_axi_bvalid && m_axi_bready) begin
+                        m_axi_bready <= 1'b0;
+                        bank_storing[dma_store_bank] <= 1'b0;
+                        dma_store_bank <= ~dma_store_bank;
+                        store_group <= store_group + 32'd1;
+                        dma_next_wr_addr <= dma_next_wr_addr + dma_dst_stride;
+                        completed_groups <= completed_groups + 32'd1;
+                        if ((completed_groups + 32'd1) >= dma_groups) begin
+                            dma_busy <= 1'b0;
+                            dma_done <= 1'b1;
+                        end
+                        dma_wr_state <= DMA_WR_IDLE;
+                    end
+                end
+                default: dma_wr_state <= DMA_WR_IDLE;
+            endcase
+        end
+    end
+
     assign intr = done;
 
 endmodule

rtl/ip/mlkem/top/mlkem_top.v

@@ -264,8 +264,6 @@ module mlkem_top #(
                  : em_we ? em_wa[PA_AW-1:0]
                  : dec_v_we ? dec_v_wr[PA_AW-1:0]
                          : (a_slot*256 + a_widx) & ((1<<PA_AW)-1);
-    assign ba_we = ...;
-    assign ba_wa = ...;
     assign ba_wd = td_we ? td_wd :
                    em_we ? bt_rd_data :     // 从 bank_t 相对槽位 0 读取 e2
                    dec_v_we ? cd_out :      // v' 解压缩后的系数

rtl/soc_top.v

@@ -361,6 +361,43 @@ wire [1 :0] dma_m_bresp  ;
 wire        dma_m_bvalid ;
 wire        dma_m_bready ;
 
+wire [3 :0] cdma_m_arid   ;
+wire [31:0] cdma_m_araddr ;
+wire [7 :0] cdma_m_arlen  ;
+wire [2 :0] cdma_m_arsize ;
+wire [1 :0] cdma_m_arburst;
+wire        cdma_m_arlock ;
+wire [3 :0] cdma_m_arcache;
+wire [2 :0] cdma_m_arprot ;
+wire        cdma_m_arvalid;
+wire        cdma_m_arready = 1'b0;
+wire [3 :0] cdma_m_rid    = 4'd0;
+wire [31:0] cdma_m_rdata  = 32'd0;
+wire [1 :0] cdma_m_rresp  = 2'd0;
+wire        cdma_m_rlast  = 1'b0;
+wire        cdma_m_rvalid = 1'b0;
+wire        cdma_m_rready ;
+wire [3 :0] cdma_m_awid   ;
+wire [31:0] cdma_m_awaddr ;
+wire [7 :0] cdma_m_awlen  ;
+wire [2 :0] cdma_m_awsize ;
+wire [1 :0] cdma_m_awburst;
+wire        cdma_m_awlock ;
+wire [3 :0] cdma_m_awcache;
+wire [2 :0] cdma_m_awprot ;
+wire        cdma_m_awvalid;
+wire        cdma_m_awready = 1'b0;
+wire [3 :0] cdma_m_wid    ;
+wire [31:0] cdma_m_wdata  ;
+wire [3 :0] cdma_m_wstrb  ;
+wire        cdma_m_wlast  ;
+wire        cdma_m_wvalid ;
+wire        cdma_m_wready = 1'b0;
+wire [3 :0] cdma_m_bid    = 4'd0;
+wire [1 :0] cdma_m_bresp  = 2'd0;
+wire        cdma_m_bvalid = 1'b0;
+wire        cdma_m_bready ;
+
 wire [4 :0] dma_s_arid   ;
 wire [31:0] dma_s_araddr ;
 wire [7 :0] dma_s_arlen  ;
@@ -1475,6 +1512,19 @@ mlkem_axi_slave u_mlkem_axi_slave (
 );
 
 //slave 7 - matmul accelerator
+wire        dma_ep_wr_valid;
+wire [3:0]  dma_ep_wr_id;
+wire [3:0]  dma_ep_wr_bank;
+wire [7:0]  dma_ep_wr_word_addr;
+wire [31:0] dma_ep_wr_data;
+wire        dma_ep_wr_ready;
+wire        dma_ep_rd_valid;
+wire [3:0]  dma_ep_rd_id;
+wire [3:0]  dma_ep_rd_bank;
+wire [7:0]  dma_ep_rd_word_addr;
+wire [31:0] dma_ep_rd_data;
+wire        dma_ep_rd_ready;
+
 matmul_axi_slave u_matmul_axi_slave (
     .clk            (sys_clk),
     .resetn         (sys_resetn),
@@ -1519,6 +1569,56 @@ matmul_axi_slave u_matmul_axi_slave (
     .s_axi_rlast    (axiOut_7_rlast ),
     .s_axi_rvalid   (axiOut_7_rvalid),
     .s_axi_rready   (axiOut_7_rready),
+
+    .ep_wr_valid    (dma_ep_wr_valid),
+    .ep_wr_id       (dma_ep_wr_id),
+    .ep_wr_bank     (dma_ep_wr_bank),
+    .ep_wr_word_addr(dma_ep_wr_word_addr),
+    .ep_wr_data     (dma_ep_wr_data),
+    .ep_wr_ready    (dma_ep_wr_ready),
+    .ep_rd_valid    (dma_ep_rd_valid),
+    .ep_rd_id       (dma_ep_rd_id),
+    .ep_rd_bank     (dma_ep_rd_bank),
+    .ep_rd_word_addr(dma_ep_rd_word_addr),
+    .ep_rd_data     (dma_ep_rd_data),
+    .ep_rd_ready    (dma_ep_rd_ready),
+
+    .m_axi_awid     (dma_m_awid),
+    .m_axi_awaddr   (dma_m_awaddr),
+    .m_axi_awlen    (dma_m_awlen),
+    .m_axi_awsize   (dma_m_awsize),
+    .m_axi_awburst  (dma_m_awburst),
+    .m_axi_awlock   (dma_m_awlock),
+    .m_axi_awcache  (dma_m_awcache),
+    .m_axi_awprot   (dma_m_awprot),
+    .m_axi_awvalid  (dma_m_awvalid),
+    .m_axi_awready  (dma_m_awready),
+    .m_axi_wdata    (dma_m_wdata),
+    .m_axi_wstrb    (dma_m_wstrb),
+    .m_axi_wlast    (dma_m_wlast),
+    .m_axi_wvalid   (dma_m_wvalid),
+    .m_axi_wready   (dma_m_wready),
+    .m_axi_bid      (dma_m_bid),
+    .m_axi_bresp    (dma_m_bresp),
+    .m_axi_bvalid   (dma_m_bvalid),
+    .m_axi_bready   (dma_m_bready),
+    .m_axi_arid     (dma_m_arid),
+    .m_axi_araddr   (dma_m_araddr),
+    .m_axi_arlen    (dma_m_arlen),
+    .m_axi_arsize   (dma_m_arsize),
+    .m_axi_arburst  (dma_m_arburst),
+    .m_axi_arlock   (dma_m_arlock),
+    .m_axi_arcache  (dma_m_arcache),
+    .m_axi_arprot   (dma_m_arprot),
+    .m_axi_arvalid  (dma_m_arvalid),
+    .m_axi_arready  (dma_m_arready),
+    .m_axi_rid      (dma_m_rid),
+    .m_axi_rdata    (dma_m_rdata),
+    .m_axi_rresp    (dma_m_rresp),
+    .m_axi_rlast    (dma_m_rlast),
+    .m_axi_rvalid   (dma_m_rvalid),
+    .m_axi_rready   (dma_m_rready),
+
     .intr           (matmul_intr)
 );
 
@@ -1644,48 +1744,61 @@ snix_axil_cdma_mux #(
     .s_axil_rready   (cdma_s_rready),
 
     // AXI4 主机接口 (接 Crossbar 去搬运数据)
-    .mm2mm_awid      (dma_m_awid),
+    .mm2mm_awid      (cdma_m_awid),
-    .mm2mm_awaddr    (dma_m_awaddr),
+    .mm2mm_awaddr    (cdma_m_awaddr),
-    .mm2mm_awlen     (dma_m_awlen),
+    .mm2mm_awlen     (cdma_m_awlen),
-    .mm2mm_awsize    (dma_m_awsize),
+    .mm2mm_awsize    (cdma_m_awsize),
-    .mm2mm_awburst   (dma_m_awburst),
+    .mm2mm_awburst   (cdma_m_awburst),
-    .mm2mm_awlock    (dma_m_awlock),
+    .mm2mm_awlock    (cdma_m_awlock),
-    .mm2mm_awcache   (dma_m_awcache),
+    .mm2mm_awcache   (cdma_m_awcache),
-    .mm2mm_awprot    (dma_m_awprot),
+    .mm2mm_awprot    (cdma_m_awprot),
     .mm2mm_awqos     (),             // 悬空即可
     .mm2mm_awuser    (),             // 悬空即可
-    .mm2mm_awvalid   (dma_m_awvalid),
+    .mm2mm_awvalid   (cdma_m_awvalid),
-    .mm2mm_awready   (dma_m_awready),
+    .mm2mm_awready   (cdma_m_awready),
-    .mm2mm_wdata     (dma_m_wdata),
+    .mm2mm_wdata     (cdma_m_wdata),
-    .mm2mm_wstrb     (dma_m_wstrb),
+    .mm2mm_wstrb     (cdma_m_wstrb),
-    .mm2mm_wlast     (dma_m_wlast),
+    .mm2mm_wlast     (cdma_m_wlast),
     .mm2mm_wuser     (),             // 悬空即可
-    .mm2mm_wvalid    (dma_m_wvalid),
+    .mm2mm_wvalid    (cdma_m_wvalid),
-    .mm2mm_wready    (dma_m_wready),
+    .mm2mm_wready    (cdma_m_wready),
-    .mm2mm_bid       (dma_m_bid),
+    .mm2mm_bid       (cdma_m_bid),
-    .mm2mm_bresp     (dma_m_bresp),
+    .mm2mm_bresp     (cdma_m_bresp),
     .mm2mm_buser     (1'b0),
-    .mm2mm_bvalid    (dma_m_bvalid),
+    .mm2mm_bvalid    (cdma_m_bvalid),
-    .mm2mm_bready    (dma_m_bready),
+    .mm2mm_bready    (cdma_m_bready),
-    .mm2mm_arid      (dma_m_arid),
+    .mm2mm_arid      (cdma_m_arid),
-    .mm2mm_araddr    (dma_m_araddr),
+    .mm2mm_araddr    (cdma_m_araddr),
-    .mm2mm_arlen     (dma_m_arlen),
+    .mm2mm_arlen     (cdma_m_arlen),
-    .mm2mm_arsize    (dma_m_arsize),
+    .mm2mm_arsize    (cdma_m_arsize),
-    .mm2mm_arburst   (dma_m_arburst),
+    .mm2mm_arburst   (cdma_m_arburst),
-    .mm2mm_arlock    (dma_m_arlock),
+    .mm2mm_arlock    (cdma_m_arlock),
-    .mm2mm_arcache   (dma_m_arcache),
+    .mm2mm_arcache   (cdma_m_arcache),
-    .mm2mm_arprot    (dma_m_arprot),
+    .mm2mm_arprot    (cdma_m_arprot),
     .mm2mm_arqos     (),             // 悬空即可
     .mm2mm_aruser    (),             // 悬空即可
-    .mm2mm_arvalid   (dma_m_arvalid),
+    .mm2mm_arvalid   (cdma_m_arvalid),
-    .mm2mm_arready   (dma_m_arready),
+    .mm2mm_arready   (cdma_m_arready),
-    .mm2mm_rid       (dma_m_rid),
+    .mm2mm_rid       (cdma_m_rid),
-    .mm2mm_rdata     (dma_m_rdata),
+    .mm2mm_rdata     (cdma_m_rdata),
-    .mm2mm_rresp     (dma_m_rresp),
+    .mm2mm_rresp     (cdma_m_rresp),
-    .mm2mm_rlast     (dma_m_rlast),
+    .mm2mm_rlast     (cdma_m_rlast),
     .mm2mm_ruser     (1'b0),
-    .mm2mm_rvalid    (dma_m_rvalid),
+    .mm2mm_rvalid    (cdma_m_rvalid),
-    .mm2mm_rready    (dma_m_rready),
+    .mm2mm_rready    (cdma_m_rready),
+
+    .ep_wr_valid     (dma_ep_wr_valid),
+    .ep_wr_id        (dma_ep_wr_id),
+    .ep_wr_bank      (dma_ep_wr_bank),
+    .ep_wr_word_addr (dma_ep_wr_word_addr),
+    .ep_wr_data      (dma_ep_wr_data),
+    .ep_wr_ready     (dma_ep_wr_ready),
+    .ep_rd_valid     (dma_ep_rd_valid),
+    .ep_rd_id        (dma_ep_rd_id),
+    .ep_rd_bank      (dma_ep_rd_bank),
+    .ep_rd_word_addr (dma_ep_rd_word_addr),
+    .ep_rd_data      (dma_ep_rd_data),
+    .ep_rd_ready     (dma_ep_rd_ready),
 
     // 全局中断输出
     .dma_finish      (dma_finish)

sdk/software/bsp/common.mk

@@ -19,27 +19,34 @@ CFLAGS += -nostartfiles -nostdlib -nostdinc -static -fno-builtin
 CFLAGS += -DCLOCKS_PER_SEC=CORE_CLOCKS_PER_SEC -D_CLOCKS_PER_SEC_=CORE_CLOCKS_PER_SEC
 CFLAGS += -DUSE_CPU_CLOCK_COUNT
 
+LINKER_SCRIPT ?= $(COMMON_DIR)/env/script.lds
+BSP_STARTUP ?= $(COMMON_DIR)/env/start.S
+BSP_USE_TRAP ?= 1
+BSP_USE_DRIVERS ?= 1
+
 #若使用 newlib , 将下面的 -lsemihost 替换为 -lgloss
 LDFLAGS +=  	-T $(LINKER_SCRIPT) \
 				-Wl,--gc-sections -Wl,--check-sections \
 				-lc -lm -lg -lsemihost -lgcc -L$(PICOLIBC_DIR)/lib
 
-LINKER_SCRIPT := $(COMMON_DIR)/env/script.lds
+ASM_SRCS += $(BSP_STARTUP)
-
-ASM_SRCS += $(COMMON_DIR)/env/start.S 
 
+ifeq ($(BSP_USE_TRAP),1)
 ifeq ($(TARGET), RTThread_Nano)
 
 else
 ASM_SRCS += $(COMMON_DIR)/env/trap_handler.S 
 endif
+endif
 
+ifeq ($(BSP_USE_DRIVERS),1)
 C_SRCS   += $(COMMON_DIR)/drivers/confreg_time.c
 C_SRCS   += $(COMMON_DIR)/drivers/core_time.c
 C_SRCS   += $(COMMON_DIR)/drivers/common_func.c
 C_SRCS   += $(COMMON_DIR)/drivers/dvi.c \
 			$(COMMON_DIR)/drivers/led.c \
 			$(COMMON_DIR)/drivers/seg7.c
+endif
 
 INCLUDES += -I./ \
 			-I$(COMMON_DIR)/include \

sdk/software/examples/asm/Makefile

@@ -1,7 +1,8 @@
 TARGET = user-sample
 
 MATMUL_GROUP_NUM ?= 5000
-CFLAGS += -O3 -g -DMATMUL_GROUP_NUM=$(MATMUL_GROUP_NUM)
+MATMUL_FAST_START ?= 1
+CFLAGS += -O3 -g -DMATMUL_GROUP_NUM=$(MATMUL_GROUP_NUM) -DMATMUL_FAST_START=$(MATMUL_FAST_START)
 
 C_SRCS := $(wildcard ./*.c )
 
@@ -9,6 +10,14 @@ OBJDIR = obj
 COMMON_DIR = ../../bsp
 GCC_DIR=../../../toolchains/loongson-gnu-toolchain-8.3-x86_64-loongarch32r-linux-gnusf-v2.0
 PICOLIBC_DIR=../../../toolchains/picolibc
+
+ifeq ($(MATMUL_FAST_START),1)
+LINKER_SCRIPT := script_matmul_fast.lds
+BSP_STARTUP := start_matmul_fast.S
+BSP_USE_TRAP := 0
+BSP_USE_DRIVERS := 0
+endif
+
 include ../../bsp/common.mk
 
 # Strip picolibc: no -lc -lm -lg -lsemihost, keep -lgcc only

sdk/software/examples/asm/dma.h

@@ -9,21 +9,46 @@ typedef struct {
     volatile u32 TAG;       // 0x0C: software tag (ignored by HW)
     volatile u32 CTRL;      // 0x10: control (burst config + start)
     volatile u32 STATUS;    // 0x14: status (busy, done)
-    volatile u32 RESV[10];  // 0x18 ~ 0x3C: padding to 0x40 bytes
+    volatile u32 MODE;      // 0x18: 0=mem2mem, 1=mem2ep, 2=ep2mem
+    volatile u32 EP_CFG;    // 0x1C: [3:0]=bank, [15:8]=word offset, [19:16]=endpoint id
+    volatile u32 RESV[8];   // 0x20 ~ 0x3C: padding to 0x40 bytes
 } dma_ch_regs_t;
 
 // DMA 控制器基地址
 #define CDMA_MUX_BASE 0xbf300000
 #define DMA_CH(i)     ((volatile dma_ch_regs_t*)(CDMA_MUX_BASE + (i) * 0x40))
 
+typedef struct {
+    volatile u32 CTRL;       // 0x200: bit0 start
+    volatile u32 STATUS;     // 0x204: bit0 busy, bit1 done (W1C)
+    volatile u32 SRC_BASE;   // 0x208
+    volatile u32 DST_BASE;   // 0x20C
+    volatile u32 GROUPS;     // 0x210
+    volatile u32 MAT_BASE;   // 0x214
+    volatile u32 SRC_STRIDE; // 0x218
+    volatile u32 DST_STRIDE; // 0x21C
+} dma_seq_regs_t;
+
+#define DMA_SEQ ((volatile dma_seq_regs_t*)(CDMA_MUX_BASE + 0x200))
+
 // CTRL 寄存器位定义
 #define CTRL_START_BIT       (1 << 0)
 #define CTRL_BURST_SIZE(x)   (((x) & 0x7) << 3)  // Bits [5:3]: AXI AxSIZE (0=1B, 1=2B, 2=4B, 3=8B)
 #define CTRL_BURST_LEN(x)    (((x) & 0xFF) << 6) // Bits [13:6]: AXI AxLEN (0=1 beat, 15=16 beats)
 
+#define DMA_MODE_MEM2MEM     0
+#define DMA_MODE_MEM2EP      1
+#define DMA_MODE_EP2MEM      2
+#define DMA_EP_CFG(id, bank, word_off) \
+    ((((id) & 0xF) << 16) | (((word_off) & 0xFF) << 8) | ((bank) & 0xF))
+
 // STATUS 寄存器位定义
 #define STATUS_BUSY_BIT      (1 << 0)
 #define STATUS_DONE_BIT      (1 << 1)
 
+#define SEQ_START_BIT        (1 << 0)
+#define SEQ_BUSY_BIT         (1 << 0)
+#define SEQ_DONE_BIT         (1 << 1)
+
 void dma_start_transfer(int ch_id, u32 src, u32 dst, u32 bytes, u32 tag);
 void dma_wait_polling(int ch_id);

sdk/software/examples/asm/script_matmul_fast.lds

@@ -0,0 +1,89 @@
+OUTPUT_ARCH(loongarch)
+ENTRY(_start)
+
+MEMORY
+{
+  isram (rwxai) : ORIGIN = 0x1c000000, LENGTH = 512K
+  dsram (rwxai) : ORIGIN = 0x1c080000, LENGTH = 512K
+}
+
+SECTIONS
+{
+  __stack_size = DEFINED(__stack_size) ? __stack_size : 4K;
+
+  .init :
+  {
+    KEEP(*(SORT_NONE(.init)))
+  } >isram AT>isram
+
+  . = ALIGN(16);
+  .text :
+  {
+    _ftext = .;
+    *(.text.unlikely .text.unlikely.*)
+    *(.text.startup .text.startup.*)
+    *(.text .text.*)
+    *(.gnu.linkonce.t.*)
+  } >isram AT>isram
+
+  . = ALIGN(16);
+  .rodata :
+  {
+    *(.rdata)
+    *(.rodata .rodata.*)
+    *(.gnu.linkonce.r.*)
+    *(.srodata.cst16)
+    *(.srodata.cst8)
+    *(.srodata.cst4)
+    *(.srodata.cst2)
+    *(.srodata .srodata.*)
+  } >isram AT>isram
+
+  . = ALIGN(16);
+  PROVIDE(__etext = .);
+  PROVIDE(_etext = .);
+  PROVIDE(etext = .);
+  PROVIDE(_data_lma = .);
+
+  .data ORIGIN(dsram) :
+  {
+    PROVIDE(_data = .);
+    *(.data .data.*)
+    *(.gnu.linkonce.d.*)
+    . = ALIGN(16);
+    PROVIDE(__global_pointer$ = . + 0x800);
+    *(.sdata .sdata.*)
+    *(.gnu.linkonce.s.*)
+    . = ALIGN(16);
+    PROVIDE(_edata = .);
+    PROVIDE(edata = .);
+  } >dsram AT>isram
+
+  .bss :
+  {
+    PROVIDE(__bss_start = .);
+    *(.sbss*)
+    *(.sbss.*)
+    *(.dynsbss)
+    *(.scommon)
+    *(.bss)
+    *(.bss.*)
+    *(.dynbss)
+    *(.gnu.linkonce.sb.*)
+    *(COMMON)
+    . = ALIGN(16);
+    PROVIDE(_end = .);
+    PROVIDE(end = .);
+  } >dsram AT>dsram
+
+  PROVIDE(__heap_start = _end);
+
+  .stack ORIGIN(dsram) + LENGTH(dsram) - __stack_size :
+  {
+    PROVIDE(_heap_end = .);
+    PROVIDE(__heap_end = _heap_end);
+    PROVIDE(__heap_size = __heap_end - __heap_start);
+    . = __stack_size;
+    PROVIDE(_stack = .);
+  } >dsram AT>dsram
+}

sdk/software/examples/asm/start_matmul_fast.S

@@ -0,0 +1,54 @@
+#include "regdef.h"
+
+    .section .init
+    .globl _start
+    .type _start, @function
+
+_start:
+    /* Enable direct mapped windows used by SRAM and uncached peripherals. */
+    li.w    t0, 0x8
+    li.w    t1, 0x18
+    csrxchg t0, t1, csr_crmd
+
+    csrwr   zero, csr_dmw0
+    csrwr   zero, csr_dmw1
+    li.w    t0, 0x19
+    csrwr   t0, csr_dmw0
+    li.w    t0, 0xa0000009
+    csrwr   t0, csr_dmw1
+
+    li.w    t0, 0x10
+    li.w    t1, 0x18
+    csrxchg t0, t1, csr_crmd
+
+    /* Polling-only benchmark: keep interrupts disabled. */
+    li.w    t0, 0x4
+    csrxchg zero, t0, csr_crmd
+    csrwr   zero, csr_prmd
+    csrwr   zero, csr_ecfg
+
+    /* UART0 init, 115200 baud with the same divisor selection as BSP start.S. */
+    li.w    t0, 0xbf000000
+    li.w    t1, 0x07
+    st.b    t1, t0, 2
+    li.w    t1, 0x80
+    st.b    t1, t0, 3
+    st.b    zero, t0, 1
+    li.w    t2, 0xbf20f500
+    ld.w    t2, t2, 0
+    li.w    t1, 0x1b
+    beq     zero, t2, 1f
+    li.w    t1, 0x1
+1:
+    st.b    t1, t0, 0
+    li.w    t1, 0x03
+    st.b    t1, t0, 3
+    st.b    zero, t0, 4
+
+    la.local sp, _stack
+    li.w    a0, 0
+    li.w    a1, 0
+    bl      main
+
+1:
+    b       1b

sdk/software/examples/asm/user-sample.c

@@ -1,51 +1,98 @@
 /**
  * user-sample - CIC-IEC 2026 Stage 3 Evaluation Program
  *
- * (ULTIMATE PERFORMANCE DMA + PING-PONG BUFFER ARCHITECTURE v2)
+ * Matmul AXI-master path with fast startup and low-overhead UART output.
- *
- * Optimizations over v1:
- *   1. Pre-fire DMA Ch0 load at loop tail → fully overlaps with IP compute
- *   2. Inlined wait loops   → zero function-call overhead in hot path
- *   3. Inlined DMA start    → skips redundant STATUS_BUSY_BIT check
- *   4. XOR bank toggle      → avoids g%2 recomputation per iteration
  */
 
 #include <dma.h>
 
-/* ─── UART 16550 direct access (bypasses slow printf/picolibc) ──────── */
 #define UART_THR (*(volatile unsigned char *)0xbf000000)
 #define UART_LSR (*(volatile unsigned char *)0xbf000005)
 #define LSR_THRE  0x20
 
-static inline void uart_tx(unsigned char c) {
+static inline void uart_tx(unsigned char c)
-    while (!(UART_LSR & LSR_THRE)) {}
+{
+    while ((UART_LSR & LSR_THRE) == 0) {
+    }
     UART_THR = c;
 }
 
-static void uart_puts(const char *s) {
+static void uart_put_matmul_start_fifo(void)
-    while (*s) uart_tx(*s++);
+{
+    UART_THR = 'M';
+    UART_THR = 'A';
+    UART_THR = 'T';
+    UART_THR = 'M';
+    UART_THR = 'U';
+    UART_THR = 'L';
+    UART_THR = '_';
+    UART_THR = 'S';
+    UART_THR = 'T';
+    UART_THR = 'A';
+    UART_THR = 'R';
+    UART_THR = 'T';
 }
 
-static void uart_puthex(unsigned int val) {
+static inline void uart_wait_tx_empty(void)
-    int i;
+{
-    for (i = 28; i >= 0; i -= 4) {
+    while ((UART_LSR & LSR_THRE) == 0) {
-        unsigned char nibble = (val >> i) & 0xF;
-        uart_tx(nibble < 10 ? '0' + nibble : 'a' + nibble - 10);
     }
 }
 
-/* ─── Configurable group count (default 5000) ──────────────────────── */
+static inline unsigned char hex_digit(unsigned int val, int shift)
+{
+    unsigned char nibble = (unsigned char)((val >> shift) & 0xf);
+
+    return (unsigned char)(nibble < 10 ? ('0' + nibble) : ('a' + nibble - 10));
+}
+
+static void uart_put_result_fifo(unsigned int crc)
+{
+    uart_wait_tx_empty();
+    UART_THR = 'M';
+    UART_THR = 'A';
+    UART_THR = 'T';
+    UART_THR = 'M';
+    UART_THR = 'U';
+    UART_THR = 'L';
+    UART_THR = '_';
+    UART_THR = 'C';
+    UART_THR = 'R';
+    UART_THR = 'C';
+    UART_THR = '3';
+    UART_THR = '2';
+    UART_THR = '=';
+    UART_THR = hex_digit(crc, 28);
+    UART_THR = hex_digit(crc, 24);
+    UART_THR = hex_digit(crc, 20);
+
+    uart_wait_tx_empty();
+    UART_THR = hex_digit(crc, 16);
+    UART_THR = hex_digit(crc, 12);
+    UART_THR = hex_digit(crc, 8);
+    UART_THR = hex_digit(crc, 4);
+    UART_THR = hex_digit(crc, 0);
+    UART_THR = 'M';
+    UART_THR = 'A';
+    UART_THR = 'T';
+    UART_THR = 'M';
+    UART_THR = 'U';
+    UART_THR = 'L';
+    UART_THR = '_';
+    UART_THR = 'D';
+    UART_THR = 'O';
+    UART_THR = 'N';
+    UART_THR = 'E';
+}
+
 #ifndef MATMUL_GROUP_NUM
 #define MATMUL_GROUP_NUM 5000
 #endif
 
-/* ─── BSP globals ─────────────────────────────────────────────────── */
+#if MATMUL_FAST_START == 0
 unsigned long UART_BASE              = 0xbf000000;
-unsigned long CONFREG_TIMER_BASE     = 0xbf20f100;
+#endif
-unsigned long CONFREG_CLOCKS_PER_SEC = 50000000L;
-unsigned long CORE_CLOCKS_PER_SEC    = 33000000L;
 
-/* ─── Memory layout ────────────────────────────────────────────────── */
 #define EXTRAM_BASE_PHYS  0x1c400000
 #define EXTRAM_BASE_UNCACHED (0xA0000000 | (EXTRAM_BASE_PHYS & 0x1FFFFFFF))
 
@@ -53,161 +100,61 @@ unsigned long CORE_CLOCKS_PER_SEC    = 33000000L;
 #define DST_BASE_WORDS    (SRC_BASE_WORDS + MATMUL_GROUP_NUM * 32)
 #define RESULT_WORDS      (MATMUL_GROUP_NUM * 48)
 
-/* ExtRAM Physical Addresses for DMA */
 #define EXTRAM_PHYS_AB(g) (EXTRAM_BASE_PHYS + (SRC_BASE_WORDS * 4) + (g) * 128)
 #define EXTRAM_PHYS_C(g)  (EXTRAM_BASE_PHYS + (DST_BASE_WORDS * 4) + (g) * 192)
 
-/* ─── MATMUL IP registers (Ping-Pong Memory Map) ──────────────────── */
 #define MATMUL_BASE_PHYS     0x1f500000
 #define MATMUL_BASE_UNCACHED 0xbf500000
 
 #define MATMUL_CTRL       (MATMUL_BASE_UNCACHED + 0x000)
 #define MATMUL_STATUS     (MATMUL_BASE_UNCACHED + 0x004)
-#define MATMUL_PHYS_B0_AB (MATMUL_BASE_PHYS + 0x100)
+#define MATMUL_DMA_SRC_BASE   (MATMUL_BASE_UNCACHED + 0x020)
-#define MATMUL_PHYS_B0_C  (MATMUL_BASE_PHYS + 0x180)
+#define MATMUL_DMA_DST_BASE   (MATMUL_BASE_UNCACHED + 0x024)
-#define MATMUL_PHYS_B1_AB (MATMUL_BASE_PHYS + 0x300)
+#define MATMUL_DMA_GROUPS     (MATMUL_BASE_UNCACHED + 0x028)
-#define MATMUL_PHYS_B1_C  (MATMUL_BASE_PHYS + 0x380)
+#define MATMUL_DMA_SRC_STRIDE (MATMUL_BASE_UNCACHED + 0x02C)
+#define MATMUL_DMA_DST_STRIDE (MATMUL_BASE_UNCACHED + 0x030)
 
 #define CTRL_START_B0     0x1
 #define CTRL_START_B1     0x2
+#define CTRL_DMA_START    (1 << 8)
 #define STATUS_DONE       0x2
 #define STATUS_ERROR      0x4
+#define STATUS_DMA_BUSY   (1 << 3)
+#define STATUS_DMA_DONE   (1 << 4)
 
 #define MATMUL_CRC (MATMUL_BASE_UNCACHED + 0x010)
 
-/* ─── Pre-computed DMA burst control word ──────────────────────────── */
 #define DMA_CTRL_VAL (CTRL_BURST_SIZE(2) | CTRL_BURST_LEN(63) | CTRL_START_BIT)
+#define MATMUL_EP_ID 0
 
-/* ═══════════════════════════════════════════════════════════════════ */
 int main(void)
 {
-    if (MATMUL_GROUP_NUM <= 0) return 0;
+    if (MATMUL_GROUP_NUM <= 0) {
+        return 0;
+    }
 
-    /* ── Hot-path local volatile pointers (one-time addr calc) ──────── */
+    volatile unsigned int *mat_ctrl = (volatile unsigned int *)MATMUL_CTRL;
     volatile unsigned int *mat_status = (volatile unsigned int *)MATMUL_STATUS;
-    volatile unsigned int *mat_ctrl   = (volatile unsigned int *)MATMUL_CTRL;
+    volatile unsigned int *mat_crc = (volatile unsigned int *)MATMUL_CRC;
-    volatile unsigned int *mat_crc    = (volatile unsigned int *)MATMUL_CRC;
-    volatile dma_ch_regs_t *ch0   = DMA_CH(0);
-    volatile dma_ch_regs_t *ch1   = DMA_CH(1);
 
     unsigned int crc;
-    int g, bank;
 
-    // Ping-Pong Register Address Arrays (Physical for DMA)
+    uart_put_matmul_start_fifo();
-    unsigned int phys_bank_AB[2] = {MATMUL_PHYS_B0_AB, MATMUL_PHYS_B1_AB};
-    unsigned int phys_bank_C[2]  = {MATMUL_PHYS_B0_C,  MATMUL_PHYS_B1_C};
-    unsigned int ctrl_start[2]   = {CTRL_START_B0, CTRL_START_B1};
 
-    /* ── Start marker (must be first output) ──────────────────── */
+    *(volatile unsigned int *)MATMUL_CRC = 0;
-    uart_puts("MATMUL_START\r\n");
+    *mat_status = STATUS_DMA_DONE | STATUS_DONE | STATUS_ERROR;
+    *(volatile unsigned int *)MATMUL_DMA_SRC_BASE = EXTRAM_PHYS_AB(0);
+    *(volatile unsigned int *)MATMUL_DMA_DST_BASE = EXTRAM_PHYS_C(0);
+    *(volatile unsigned int *)MATMUL_DMA_GROUPS = MATMUL_GROUP_NUM;
+    *(volatile unsigned int *)MATMUL_DMA_SRC_STRIDE = 128;
+    *(volatile unsigned int *)MATMUL_DMA_DST_STRIDE = 192;
+    *mat_ctrl = CTRL_DMA_START;
+    while (!(*mat_status & STATUS_DMA_DONE)) {}
+    *mat_status = STATUS_DMA_DONE;
 
-    // Reset IP state
-    *mat_status = STATUS_DONE | STATUS_ERROR;
-    *mat_crc    = 0;
-
-    /* ═══════════════════════════════════════════════════════════════
-     * PROLOGUE: Load group 0 → Bank 0, start IP, pre-fire group 1
-     * ═══════════════════════════════════════════════════════════════ */
-
-    // DMA Ch0: Load group 0 A+B (128 bytes) → IP Bank 0
-    ch0->SRC_ADDR = EXTRAM_PHYS_AB(0);
-    ch0->DST_ADDR = phys_bank_AB[0];
-    ch0->LENGTH   = 128;
-    ch0->CTRL     = DMA_CTRL_VAL;
-    while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
-    ch0->STATUS   = STATUS_DONE_BIT;
-
-    // Start IP Bank 0 (group 0)
-    *mat_ctrl = ctrl_start[0];
-
-    // Pre-fire: DMA Ch0 load group 1 → Bank 1 (overlaps with IP compute)
-    if (MATMUL_GROUP_NUM > 1) {
-        ch0->SRC_ADDR = EXTRAM_PHYS_AB(1);
-        ch0->DST_ADDR = phys_bank_AB[1];
-        ch0->LENGTH   = 128;
-        ch0->CTRL     = DMA_CTRL_VAL;
-    }
-
-    /* ═══════════════════════════════════════════════════════════════
-     * MAIN LOOP: Fully-pipelined — pre-fire DMA load at loop tail
-     *
-     * At iteration g:
-     *   IP is computing group (g-1) on prev_bank
-     *   Ch0 loaded group g to cur_bank (pre-fired in iteration g-1)
-     *
-     * Pipeline:
-     *   Ch0:  | load g+1 |              | load g+2 |
-     *   Ch1:         | store g-1 |              | store g |
-     *   IP :         | compute g |              | compute g+1 |
-     * ═══════════════════════════════════════════════════════════════ */
-    bank = 1;  // cur_bank for iteration g=1; prev_bank = bank ^ 1
-
-    for (g = 1; g < MATMUL_GROUP_NUM; g++) {
-        int cur_bank  = bank;
-        int prev_bank = bank ^ 1;
-
-        // 1. Wait for previous IP computation (group g-1 on prev_bank)
-        while (!(*mat_status & STATUS_DONE)) {}
-        *mat_status = STATUS_DONE;
-
-        // 2. Ensure previous DMA Ch1 store is done (group g-2)
-        if (g > 1) {
-            while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
-            ch1->STATUS = STATUS_DONE_BIT;
-        }
-
-        // 3. Start DMA Ch1 store: group (g-1) result → ExtRAM
-        ch1->SRC_ADDR = phys_bank_C[prev_bank];
-        ch1->DST_ADDR = EXTRAM_PHYS_C(g - 1);
-        ch1->LENGTH   = 192;
-        ch1->CTRL     = DMA_CTRL_VAL;
-
-        // 4. Wait for current DMA Ch0 load (group g → cur_bank, pre-fired in iteration g-1)
-        while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
-        ch0->STATUS   = STATUS_DONE_BIT;
-
-        // 5. Start IP computation for current group on cur_bank
-        *mat_ctrl = ctrl_start[cur_bank];
-
-        // ★ 6. PRE-FIRE: DMA Ch0 load next group (g+1) → next_bank
-        //    This runs CONCURRENTLY with IP computation started in step 5!
-        if (g < MATMUL_GROUP_NUM - 1) {
-            int next_bank = bank ^ 1;
-            ch0->SRC_ADDR = EXTRAM_PHYS_AB(g + 1);
-            ch0->DST_ADDR = phys_bank_AB[next_bank];
-            ch0->LENGTH   = 128;
-            ch0->CTRL     = DMA_CTRL_VAL;
-        }
-
-        // Toggle bank for next iteration
-        bank ^= 1;
-    }
-
-    /* ═══════════════════════════════════════════════════════════════
-     * EPILOGUE: Flush the final group
-     * ═══════════════════════════════════════════════════════════════ */
-
-    // Wait for last IP computation
-    while (!(*mat_status & STATUS_DONE)) {}
-    *mat_status = STATUS_DONE;
-
-    // Wait for previous DMA store and start/complete final store
-    while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
-    ch1->STATUS = STATUS_DONE_BIT;
-
-    ch1->SRC_ADDR = phys_bank_C[(MATMUL_GROUP_NUM - 1) % 2];
-    ch1->DST_ADDR = EXTRAM_PHYS_C(MATMUL_GROUP_NUM - 1);
-    ch1->LENGTH   = 192;
-    ch1->CTRL     = DMA_CTRL_VAL;
-    while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
-    ch1->STATUS = STATUS_DONE_BIT;
-
-    /* ── Read hardware CRC32 ──────────────────────────────────── */
     crc = *mat_crc;
 
-    uart_puts("MATMUL_CRC32=");
+    uart_put_result_fifo(crc);
-    uart_puthex(crc);
-    uart_puts("\r\nMATMUL_DONE\r\n");
 
     return 0;
 }

sdk/software/examples/mlkem_test/main.c

@@ -150,16 +150,16 @@ static void readback(uint32_t sel, uint8_t *buf, int n)
 
 /* Arbitrary but fixed test seeds / message (32 bytes each). */
 static const uint8_t SEED_D[32] = {
-    0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F,
+    0x6d,0xbb,0xc4,0x37,0x51,0x36,0xdf,0x3b,0x07,0xf7,0xc7,0x0e,0x63,0x9e,0x22,0x3e,
-    0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F
+    0x17,0x7e,0x7f,0xd5,0x3b,0x16,0x1b,0x3f,0x4d,0x57,0x79,0x17,0x94,0xf1,0x26,0x24
 };
 static const uint8_t SEED_Z[32] = {
-    0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2A,0x2B,0x2C,0x2D,0x2E,0x2F,
+    0xf6,0x96,0x48,0x40,0x48,0xec,0x21,0xf9,0x6c,0xf5,0x0a,0x56,0xd0,0x75,0x9c,0x44,
-    0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x3A,0x3B,0x3C,0x3D,0x3E,0x3F
+    0x8f,0x37,0x79,0x75,0x2f,0x03,0x83,0xd3,0x74,0x49,0x69,0x06,0x94,0xcf,0x7a,0x68
 };
 static const uint8_t MSG_M[32] = {
-    0xA0,0xA1,0xA2,0xA3,0xA4,0xA5,0xA6,0xA7,0xA8,0xA9,0xAA,0xAB,0xAC,0xAD,0xAE,0xAF,
+    0x20,0xa7,0xb7,0xe1,0x0f,0x70,0x49,0x6c,0xc3,0x82,0x20,0xb9,0x44,0xde,0xf6,0x99,
-    0xB0,0xB1,0xB2,0xB3,0xB4,0xB5,0xB6,0xB7,0xB8,0xB9,0xBA,0xBB,0xBC,0xBD,0xBE,0xBF
+    0xbf,0x14,0xd1,0x4e,0x55,0xcf,0x4c,0x90,0xa1,0x2c,0x1b,0x33,0xfc,0x80,0xff,0xff
 };
 
 static uint8_t ek[EK_BYTES];