Reduce CDMA matmul bridge request latency

rtl/ip/cdma/cdma_matmul_bridge.sv

@@ -128,6 +128,35 @@ module cdma_matmul_bridge #(
         end
     endfunction
 
+    task issue_load_req;
+        input [31:0] load_idx;
+        begin
+            cdma_req_mode        <= MODE_MEM2EP;
+            cdma_req_src         <= seq_src_base + (load_idx * seq_src_stride);
+            cdma_req_dst         <= '0;
+            cdma_req_bytes       <= MAT_AB_BYTES;
+            cdma_req_ep_id       <= 4'd0;
+            cdma_req_ep_bank     <= {3'b000, load_idx[0]};
+            cdma_req_ep_word_off <= 8'd0;
+            cdma_req_valid       <= 1'b1;
+            seq_state            <= SQ_LOAD_REQ;
+        end
+    endtask
+
+    task issue_store_req;
+        begin
+            cdma_req_mode        <= MODE_EP2MEM;
+            cdma_req_src         <= '0;
+            cdma_req_dst         <= dst_addr_for_store;
+            cdma_req_bytes       <= MAT_C_BYTES;
+            cdma_req_ep_id       <= 4'd0;
+            cdma_req_ep_bank     <= {3'b000, seq_compute_bank};
+            cdma_req_ep_word_off <= 8'd0;
+            cdma_req_valid       <= 1'b1;
+            seq_state            <= SQ_STORE_REQ;
+        end
+    endtask
+
     assign s_axil_bresp = 2'b00;
     assign s_axil_rresp = 2'b00;
     assign seq_done_irq = seq_done_flag;
@@ -270,19 +299,11 @@ module cdma_matmul_bridge #(
                 end
 
                 SQ_CLEAR: begin
-                    seq_state <= SQ_LOAD_START;
+                    issue_load_req(32'd0);
                 end
 
                 SQ_LOAD_START: begin
-                    cdma_req_mode        <= MODE_MEM2EP;
+                    issue_load_req(seq_load_idx);
-                    cdma_req_src         <= src_addr_for_load;
-                    cdma_req_dst         <= '0;
-                    cdma_req_bytes       <= MAT_AB_BYTES;
-                    cdma_req_ep_id       <= 4'd0;
-                    cdma_req_ep_bank     <= {3'b000, loaded_bank};
-                    cdma_req_ep_word_off <= 8'd0;
-                    cdma_req_valid       <= 1'b1;
-                    seq_state            <= SQ_LOAD_REQ;
                 end
 
                 SQ_LOAD_REQ: begin
@@ -299,12 +320,12 @@ module cdma_matmul_bridge #(
                             seq_defer_load_start     <= 1'b0;
                             seq_deferred_start_valid <= 1'b1;
                             seq_deferred_start_bank  <= cdma_req_ep_bank[0];
-                            seq_state                <= SQ_STORE_START;
+                            issue_store_req();
                         end else begin
                             mat_start_valid <= 1'b1;
                             mat_start_bank  <= cdma_req_ep_bank[0];
                             if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin
-                                seq_state <= SQ_LOAD_START;
+                                issue_load_req(seq_load_idx + 32'd1);
                             end else begin
                                 seq_state <= SQ_WAIT_DONE;
                             end
@@ -321,29 +342,21 @@ module cdma_matmul_bridge #(
                             if ((seq_store_idx + 32'd1) >= seq_groups) begin
                                 seq_state <= SQ_DONE;
                             end else if (seq_load_idx < seq_groups) begin
-                                seq_state <= SQ_LOAD_START;
+                                issue_load_req(seq_load_idx);
                             end else begin
                                 seq_state <= SQ_WAIT_DONE;
                             end
                         end else if (seq_load_idx < seq_groups) begin
                             seq_defer_load_start <= 1'b1;
-                            seq_state            <= SQ_LOAD_START;
+                            issue_load_req(seq_load_idx);
                         end else begin
-                            seq_state <= SQ_STORE_START;
+                            issue_store_req();
                         end
                     end
                 end
 
                 SQ_STORE_START: begin
-                    cdma_req_mode        <= MODE_EP2MEM;
+                    issue_store_req();
-                    cdma_req_src         <= '0;
-                    cdma_req_dst         <= dst_addr_for_store;
-                    cdma_req_bytes       <= MAT_C_BYTES;
-                    cdma_req_ep_id       <= 4'd0;
-                    cdma_req_ep_bank     <= {3'b000, seq_compute_bank};
-                    cdma_req_ep_word_off <= 8'd0;
-                    cdma_req_valid       <= 1'b1;
-                    seq_state            <= SQ_STORE_REQ;
                 end
 
                 SQ_STORE_REQ: begin

rtl/ip/cdma/snix_axil_cdma_mux.sv

@@ -400,6 +400,12 @@ module snix_axil_cdma_mux #(
     logic                   ep_arready;
     logic                   ep_rready;
 
+    typedef enum logic [2:0] {EP_IDLE, EP_AR, EP_READ, EP_AW, EP_WRITE, EP_BRESP, EP_DONE} ep_state_t;
+    ep_state_t ep_state;
+    logic [31:0] ep_axi_addr;
+    logic [31:0] ep_bytes_left;
+    logic [7:0]  ep_word_addr_r;
+
     logic        ext_active;
     logic [1:0]  ext_mode_r;
     logic [ADDR_WIDTH-1:0] ext_src_r;
@@ -559,15 +565,23 @@ module snix_axil_cdma_mux #(
         .mm2mm_rready (core_rready)
     );
 
+    wire [7:0] ep_beats_total = (engine_bytes[31:2] == 30'd0) ? 8'd1 : engine_bytes[9:2];
+    wire [7:0] ep_axi_len     = ep_beats_total - 8'd1;
+    wire       ep_mode_mem2ep = (engine_mode == MODE_MEM2EP);
+    wire       ep_mode_ep2mem = (engine_mode == MODE_EP2MEM);
+    wire       ep_rd_done_fire = (ep_state == EP_READ) && mm2mm_rvalid && ep_rready && mm2mm_rlast;
+    wire       ep_wr_done_fire = (ep_state == EP_BRESP) && mm2mm_bvalid && ep_bready;
+    wire       ep_done_fast    = ep_rd_done_fire || ep_wr_done_fire;
+
     // ==========================================
     // Generic external endpoint requester
     // ==========================================
     assign ext_req_ready = !ext_active && (state == IDLE) && (ep_state == EP_IDLE);
+    assign ext_req_done  = ext_active && ep_done_fast;
 
     always_ff @(posedge clk) begin
         if (!rst_n) begin
             ext_active       <= 1'b0;
-            ext_req_done     <= 1'b0;
             ext_mode_r       <= MODE_MEM2EP;
             ext_src_r        <= '0;
             ext_dst_r        <= '0;
@@ -576,8 +590,6 @@ module snix_axil_cdma_mux #(
             ext_ep_bank_r    <= '0;
             ext_ep_word_off_r <= '0;
         end else begin
-            ext_req_done    <= 1'b0;
-
             if (ext_accept) begin
                 ext_active        <= 1'b1;
                 ext_mode_r        <= ext_req_mode;
@@ -587,9 +599,8 @@ module snix_axil_cdma_mux #(
                 ext_ep_id_r       <= ext_req_ep_id;
                 ext_ep_bank_r     <= ext_req_ep_bank;
                 ext_ep_word_off_r <= ext_req_ep_word_off;
-            end else if (ext_active && ep_done) begin
+            end else if (ext_active && ep_done_fast) begin
-                ext_active    <= 1'b0;
+                ext_active <= 1'b0;
-                ext_req_done  <= 1'b1;
             end
         end
     end
@@ -597,17 +608,6 @@ module snix_axil_cdma_mux #(
     // ==========================================
     // Endpoint transfer engine (32-bit only in this SoC integration)
     // ==========================================
-    typedef enum logic [2:0] {EP_IDLE, EP_AR, EP_READ, EP_AW, EP_WRITE, EP_BRESP, EP_DONE} ep_state_t;
-    ep_state_t ep_state;
-    logic [31:0] ep_axi_addr;
-    logic [31:0] ep_bytes_left;
-    logic [7:0]  ep_word_addr_r;
-
-    wire [7:0] ep_beats_total = (engine_bytes[31:2] == 30'd0) ? 8'd1 : engine_bytes[9:2];
-    wire [7:0] ep_axi_len     = ep_beats_total - 8'd1;
-    wire       ep_mode_mem2ep = (engine_mode == MODE_MEM2EP);
-    wire       ep_mode_ep2mem = (engine_mode == MODE_EP2MEM);
-
     always_ff @(posedge clk) begin
         if (!rst_n) begin
             ep_state       <= EP_IDLE;