Add direct CDMA matmul path
This commit is contained in:
@@ -94,6 +94,32 @@ module snix_axil_cdma_mux #(
|
||||
input logic mm2mm_rvalid,
|
||||
output logic mm2mm_rready,
|
||||
|
||||
// ==========================================
|
||||
// Local endpoint interface
|
||||
// ==========================================
|
||||
output logic ep_wr_valid,
|
||||
output logic [3:0] ep_wr_id,
|
||||
output logic [3:0] ep_wr_bank,
|
||||
output logic [7:0] ep_wr_word_addr,
|
||||
output logic [31:0] ep_wr_data,
|
||||
input logic ep_wr_ready,
|
||||
|
||||
output logic ep_rd_valid,
|
||||
output logic [3:0] ep_rd_id,
|
||||
output logic [3:0] ep_rd_bank,
|
||||
output logic [7:0] ep_rd_word_addr,
|
||||
input logic [31:0] ep_rd_data,
|
||||
input logic ep_rd_ready,
|
||||
|
||||
// ==========================================
|
||||
// Matmul direct control sideband
|
||||
// ==========================================
|
||||
output logic mat_start_valid,
|
||||
output logic mat_start_bank,
|
||||
output logic mat_done_clear,
|
||||
output logic mat_crc_clear,
|
||||
input logic mat_done,
|
||||
|
||||
// Global Interrupt (OR'd from all channels)
|
||||
output logic dma_finish
|
||||
);
|
||||
@@ -102,6 +128,29 @@ module snix_axil_cdma_mux #(
|
||||
// Local Parameters & Utilities
|
||||
// ==========================================
|
||||
localparam int CH_BITS = $clog2(PORTS);
|
||||
localparam logic [1:0] MODE_MEM2MEM = 2'd0;
|
||||
localparam logic [1:0] MODE_MEM2EP = 2'd1;
|
||||
localparam logic [1:0] MODE_EP2MEM = 2'd2;
|
||||
|
||||
localparam logic [7:0] SEQ_REG_CTRL = 8'h00;
|
||||
localparam logic [7:0] SEQ_REG_STATUS = 8'h04;
|
||||
localparam logic [7:0] SEQ_REG_SRC_BASE = 8'h08;
|
||||
localparam logic [7:0] SEQ_REG_DST_BASE = 8'h0C;
|
||||
localparam logic [7:0] SEQ_REG_GROUPS = 8'h10;
|
||||
localparam logic [7:0] SEQ_REG_MAT_BASE = 8'h14;
|
||||
localparam logic [7:0] SEQ_REG_SRC_STRIDE = 8'h18;
|
||||
localparam logic [7:0] SEQ_REG_DST_STRIDE = 8'h1C;
|
||||
|
||||
localparam logic [11:0] MAT_CTRL_OFF = 12'h000;
|
||||
localparam logic [11:0] MAT_STATUS_OFF = 12'h004;
|
||||
localparam logic [11:0] MAT_CRC_OFF = 12'h010;
|
||||
localparam logic [11:0] MAT_B0_AB_OFF = 12'h100;
|
||||
localparam logic [11:0] MAT_B0_C_OFF = 12'h180;
|
||||
localparam logic [11:0] MAT_B1_AB_OFF = 12'h300;
|
||||
localparam logic [11:0] MAT_B1_C_OFF = 12'h380;
|
||||
|
||||
localparam logic [31:0] MAT_AB_BYTES = 32'd128;
|
||||
localparam logic [31:0] MAT_C_BYTES = 32'd192;
|
||||
|
||||
// Function to safely apply WSTRB to 32-bit registers
|
||||
function automatic logic [31:0] apply_wstrb(
|
||||
@@ -125,12 +174,25 @@ module snix_axil_cdma_mux #(
|
||||
logic [31:0] ch_len [PORTS];
|
||||
logic [31:0] ch_tag [PORTS];
|
||||
logic [31:0] ch_ctrl [PORTS];
|
||||
logic [31:0] ch_mode [PORTS];
|
||||
logic [31:0] ch_ep_cfg [PORTS];
|
||||
|
||||
logic [PORTS-1:0] ch_req; // Pending requests (Busy)
|
||||
logic [PORTS-1:0] ch_done; // Completion flags
|
||||
|
||||
logic [PORTS-1:0] arb_set_done; // From Arbiter to CSR
|
||||
|
||||
logic [31:0] seq_src_base;
|
||||
logic [31:0] seq_dst_base;
|
||||
logic [31:0] seq_groups;
|
||||
logic [31:0] seq_mat_base;
|
||||
logic [31:0] seq_src_stride;
|
||||
logic [31:0] seq_dst_stride;
|
||||
logic seq_busy;
|
||||
logic seq_done_flag;
|
||||
wire seq_start_req;
|
||||
wire seq_done_clear_req;
|
||||
|
||||
// ==========================================
|
||||
// Address Decoding (0x40 offset per channel)
|
||||
// ==========================================
|
||||
@@ -138,6 +200,14 @@ module snix_axil_cdma_mux #(
|
||||
wire [5:0] wr_reg = s_axil_awaddr[5:0];
|
||||
wire [CH_BITS-1:0] rd_ch = s_axil_araddr[6 +: CH_BITS];
|
||||
wire [5:0] rd_reg = s_axil_araddr[5:0];
|
||||
wire wr_seq_reg = (s_axil_awaddr[11:8] == 4'h2);
|
||||
wire rd_seq_reg = (s_axil_araddr[11:8] == 4'h2);
|
||||
wire [7:0] wr_seq_off = s_axil_awaddr[7:0];
|
||||
wire [7:0] rd_seq_off = s_axil_araddr[7:0];
|
||||
assign seq_start_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_CTRL) &&
|
||||
s_axil_wstrb[0] && s_axil_wdata[0];
|
||||
assign seq_done_clear_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_STATUS) &&
|
||||
s_axil_wstrb[0] && s_axil_wdata[1];
|
||||
|
||||
// ==========================================
|
||||
// AXI-Lite Slave Logic (Robust Backpressure)
|
||||
@@ -197,7 +267,19 @@ module snix_axil_cdma_mux #(
|
||||
end else begin
|
||||
if (do_read) begin
|
||||
s_axil_rvalid <= 1'b1;
|
||||
if (rd_ch < PORTS) begin
|
||||
if (rd_seq_reg) begin
|
||||
case (rd_seq_off)
|
||||
SEQ_REG_CTRL: s_axil_rdata <= {31'd0, seq_start_req};
|
||||
SEQ_REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
|
||||
SEQ_REG_SRC_BASE: s_axil_rdata <= seq_src_base;
|
||||
SEQ_REG_DST_BASE: s_axil_rdata <= seq_dst_base;
|
||||
SEQ_REG_GROUPS: s_axil_rdata <= seq_groups;
|
||||
SEQ_REG_MAT_BASE: s_axil_rdata <= seq_mat_base;
|
||||
SEQ_REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
|
||||
SEQ_REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
|
||||
default: s_axil_rdata <= 32'd0;
|
||||
endcase
|
||||
end else if (rd_ch < PORTS) begin
|
||||
case (rd_reg)
|
||||
6'h00: s_axil_rdata <= ch_src_addr[rd_ch];
|
||||
6'h04: s_axil_rdata <= ch_dst_addr[rd_ch];
|
||||
@@ -205,6 +287,8 @@ module snix_axil_cdma_mux #(
|
||||
6'h0C: s_axil_rdata <= ch_tag[rd_ch];
|
||||
6'h10: s_axil_rdata <= ch_ctrl[rd_ch];
|
||||
6'h14: s_axil_rdata <= {30'd0, ch_done[rd_ch], ch_req[rd_ch]};
|
||||
6'h18: s_axil_rdata <= ch_mode[rd_ch];
|
||||
6'h1C: s_axil_rdata <= ch_ep_cfg[rd_ch];
|
||||
default: s_axil_rdata <= 32'd0;
|
||||
endcase
|
||||
end else begin
|
||||
@@ -219,15 +303,22 @@ module snix_axil_cdma_mux #(
|
||||
// ==========================================
|
||||
// Register File Write Logic
|
||||
// ==========================================
|
||||
assign dma_finish = |ch_done;
|
||||
assign dma_finish = (|ch_done) | seq_done_flag;
|
||||
|
||||
always_ff @(posedge clk) begin
|
||||
if (!rst_n) begin
|
||||
ch_req <= '0;
|
||||
ch_done <= '0;
|
||||
seq_src_base <= 32'h0;
|
||||
seq_dst_base <= 32'h0;
|
||||
seq_groups <= 32'h0;
|
||||
seq_mat_base <= 32'h1f50_0000;
|
||||
seq_src_stride <= 32'd128;
|
||||
seq_dst_stride <= 32'd192;
|
||||
for (int i=0; i<PORTS; i++) begin
|
||||
ch_src_addr[i] <= '0; ch_dst_addr[i] <= '0;
|
||||
ch_len[i] <= '0; ch_tag[i] <= '0; ch_ctrl[i] <= '0;
|
||||
ch_mode[i] <= '0; ch_ep_cfg[i] <= '0;
|
||||
end
|
||||
end else begin
|
||||
// 1. Process Hardware Status Updates (Highest Priority for Done flag)
|
||||
@@ -239,12 +330,23 @@ module snix_axil_cdma_mux #(
|
||||
end
|
||||
|
||||
// 2. Process CPU Writes
|
||||
if (do_write && wr_ch < PORTS) begin
|
||||
if (do_write && wr_seq_reg) begin
|
||||
case (wr_seq_off)
|
||||
SEQ_REG_SRC_BASE: if (!seq_busy) seq_src_base <= apply_wstrb(seq_src_base, s_axil_wdata, s_axil_wstrb);
|
||||
SEQ_REG_DST_BASE: if (!seq_busy) seq_dst_base <= apply_wstrb(seq_dst_base, s_axil_wdata, s_axil_wstrb);
|
||||
SEQ_REG_GROUPS: if (!seq_busy) seq_groups <= apply_wstrb(seq_groups, s_axil_wdata, s_axil_wstrb);
|
||||
SEQ_REG_MAT_BASE: if (!seq_busy) seq_mat_base <= apply_wstrb(seq_mat_base, s_axil_wdata, s_axil_wstrb);
|
||||
SEQ_REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
|
||||
SEQ_REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
|
||||
endcase
|
||||
end else if (do_write && wr_ch < PORTS) begin
|
||||
case (wr_reg)
|
||||
6'h00: if (!ch_req[wr_ch]) ch_src_addr[wr_ch] <= apply_wstrb(ch_src_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
6'h04: if (!ch_req[wr_ch]) ch_dst_addr[wr_ch] <= apply_wstrb(ch_dst_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
6'h08: if (!ch_req[wr_ch]) ch_len[wr_ch] <= apply_wstrb(ch_len[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
6'h0C: if (!ch_req[wr_ch]) ch_tag[wr_ch] <= apply_wstrb(ch_tag[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
6'h18: if (!ch_req[wr_ch]) ch_mode[wr_ch] <= apply_wstrb(ch_mode[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
6'h1C: if (!ch_req[wr_ch]) ch_ep_cfg[wr_ch] <= apply_wstrb(ch_ep_cfg[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
6'h10: begin
|
||||
ch_ctrl[wr_ch] <= apply_wstrb(ch_ctrl[wr_ch], s_axil_wdata, s_axil_wstrb);
|
||||
// Trigger Bit Processing
|
||||
@@ -269,7 +371,7 @@ module snix_axil_cdma_mux #(
|
||||
// ==========================================
|
||||
// Round-Robin Arbiter & Engine Driver
|
||||
// ==========================================
|
||||
typedef enum logic [1:0] {IDLE, RUN} state_t;
|
||||
typedef enum logic [1:0] {IDLE, START, RUN} state_t;
|
||||
state_t state;
|
||||
|
||||
logic [CH_BITS-1:0] cur_ch;
|
||||
@@ -283,13 +385,165 @@ module snix_axil_cdma_mux #(
|
||||
logic [7:0] engine_len;
|
||||
logic [2:0] engine_size;
|
||||
logic engine_done;
|
||||
logic [1:0] engine_mode;
|
||||
logic [3:0] engine_ep_id;
|
||||
logic [3:0] engine_ep_bank;
|
||||
logic [7:0] engine_ep_word_off;
|
||||
|
||||
logic mm2mm_start;
|
||||
logic mm2mm_done;
|
||||
logic ep_start;
|
||||
logic ep_done;
|
||||
|
||||
logic [ID_WIDTH-1:0] core_awid;
|
||||
logic [ADDR_WIDTH-1:0] core_awaddr;
|
||||
logic [7:0] core_awlen;
|
||||
logic [2:0] core_awsize;
|
||||
logic [1:0] core_awburst;
|
||||
logic core_awlock;
|
||||
logic [3:0] core_awcache;
|
||||
logic [2:0] core_awprot;
|
||||
logic [3:0] core_awqos;
|
||||
logic [USER_WIDTH-1:0] core_awuser;
|
||||
logic core_awvalid;
|
||||
logic core_awready;
|
||||
logic [DATA_WIDTH-1:0] core_wdata;
|
||||
logic [DATA_WIDTH/8-1:0] core_wstrb;
|
||||
logic core_wlast;
|
||||
logic [USER_WIDTH-1:0] core_wuser;
|
||||
logic core_wvalid;
|
||||
logic core_wready;
|
||||
logic core_bready;
|
||||
logic [ID_WIDTH-1:0] core_arid;
|
||||
logic [ADDR_WIDTH-1:0] core_araddr;
|
||||
logic [7:0] core_arlen;
|
||||
logic [2:0] core_arsize;
|
||||
logic [1:0] core_arburst;
|
||||
logic core_arlock;
|
||||
logic [3:0] core_arcache;
|
||||
logic [2:0] core_arprot;
|
||||
logic [3:0] core_arqos;
|
||||
logic [USER_WIDTH-1:0] core_aruser;
|
||||
logic core_arvalid;
|
||||
logic core_arready;
|
||||
logic core_rready;
|
||||
|
||||
logic [ID_WIDTH-1:0] ep_awid;
|
||||
logic [ADDR_WIDTH-1:0] ep_awaddr;
|
||||
logic [7:0] ep_awlen;
|
||||
logic [2:0] ep_awsize;
|
||||
logic [1:0] ep_awburst;
|
||||
logic ep_awlock;
|
||||
logic [3:0] ep_awcache;
|
||||
logic [2:0] ep_awprot;
|
||||
logic [3:0] ep_awqos;
|
||||
logic [USER_WIDTH-1:0] ep_awuser;
|
||||
logic ep_awvalid;
|
||||
logic ep_awready;
|
||||
logic [DATA_WIDTH-1:0] ep_wdata;
|
||||
logic [DATA_WIDTH/8-1:0] ep_wstrb;
|
||||
logic ep_wlast;
|
||||
logic [USER_WIDTH-1:0] ep_wuser;
|
||||
logic ep_wvalid;
|
||||
logic ep_wready;
|
||||
logic ep_bready;
|
||||
logic [ID_WIDTH-1:0] ep_arid;
|
||||
logic [ADDR_WIDTH-1:0] ep_araddr;
|
||||
logic [7:0] ep_arlen;
|
||||
logic [2:0] ep_arsize;
|
||||
logic [1:0] ep_arburst;
|
||||
logic ep_arlock;
|
||||
logic [3:0] ep_arcache;
|
||||
logic [2:0] ep_arprot;
|
||||
logic [3:0] ep_arqos;
|
||||
logic [USER_WIDTH-1:0] ep_aruser;
|
||||
logic ep_arvalid;
|
||||
logic ep_arready;
|
||||
logic ep_rready;
|
||||
|
||||
// Hardware matmul loop sequencer state. The sequencer uses the endpoint
|
||||
// path below, so matmul register/memory traffic does not enter the AXI bus.
|
||||
typedef enum logic [4:0] {
|
||||
SQ_IDLE,
|
||||
SQ_CLEAR,
|
||||
SQ_LOAD,
|
||||
SQ_LOAD_WAIT,
|
||||
SQ_WAIT_DONE,
|
||||
SQ_STORE_START,
|
||||
SQ_STORE,
|
||||
SQ_DONE
|
||||
} seq_state_t;
|
||||
|
||||
seq_state_t seq_state;
|
||||
|
||||
logic [31:0] seq_group_idx;
|
||||
logic [31:0] seq_load_idx;
|
||||
logic [31:0] seq_store_idx;
|
||||
logic seq_compute_bank;
|
||||
logic seq_ep_owner;
|
||||
logic seq_ep_start;
|
||||
logic [1:0] seq_ep_mode;
|
||||
logic [31:0] seq_ep_src;
|
||||
logic [31:0] seq_ep_dst;
|
||||
logic [31:0] seq_ep_bytes;
|
||||
logic seq_ep_bank;
|
||||
|
||||
wire use_seq_engine = 1'b0;
|
||||
wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || seq_ep_owner;
|
||||
wire select_seq_ep = seq_ep_owner || seq_ep_start;
|
||||
|
||||
// Dynamic routing to the engine based on current active channel
|
||||
assign engine_src = ch_src_addr[cur_ch];
|
||||
assign engine_dst = ch_dst_addr[cur_ch];
|
||||
assign engine_bytes = ch_len[cur_ch];
|
||||
assign engine_src = select_seq_ep ? seq_ep_src : ch_src_addr[cur_ch];
|
||||
assign engine_dst = select_seq_ep ? seq_ep_dst : ch_dst_addr[cur_ch];
|
||||
assign engine_bytes = select_seq_ep ? seq_ep_bytes : ch_len[cur_ch];
|
||||
assign engine_len = ch_ctrl[cur_ch][13:6];
|
||||
assign engine_size = ch_ctrl[cur_ch][5:3];
|
||||
assign engine_mode = select_seq_ep ? seq_ep_mode : ch_mode[cur_ch][1:0];
|
||||
assign engine_ep_bank = select_seq_ep ? {3'b000, seq_ep_bank} : ch_ep_cfg[cur_ch][3:0];
|
||||
assign engine_ep_word_off = select_seq_ep ? 8'd0 : ch_ep_cfg[cur_ch][15:8];
|
||||
assign engine_ep_id = select_seq_ep ? 4'd0 : ch_ep_cfg[cur_ch][19:16];
|
||||
assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done;
|
||||
assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM);
|
||||
assign ep_start = seq_ep_start || (engine_start && (engine_mode != MODE_MEM2MEM));
|
||||
|
||||
assign core_awready = (!use_seq_engine && !use_ep_engine) && mm2mm_awready;
|
||||
assign core_wready = (!use_seq_engine && !use_ep_engine) && mm2mm_wready;
|
||||
assign core_arready = (!use_seq_engine && !use_ep_engine) && mm2mm_arready;
|
||||
assign ep_awready = (!use_seq_engine && use_ep_engine) && mm2mm_awready;
|
||||
assign ep_wready = (!use_seq_engine && use_ep_engine) && mm2mm_wready;
|
||||
assign ep_arready = (!use_seq_engine && use_ep_engine) && mm2mm_arready;
|
||||
|
||||
assign mm2mm_awid = use_ep_engine ? ep_awid : core_awid;
|
||||
assign mm2mm_awaddr = use_ep_engine ? ep_awaddr : core_awaddr;
|
||||
assign mm2mm_awlen = use_ep_engine ? ep_awlen : core_awlen;
|
||||
assign mm2mm_awsize = use_ep_engine ? ep_awsize : core_awsize;
|
||||
assign mm2mm_awburst = use_ep_engine ? ep_awburst : core_awburst;
|
||||
assign mm2mm_awlock = use_ep_engine ? ep_awlock : core_awlock;
|
||||
assign mm2mm_awcache = use_ep_engine ? ep_awcache : core_awcache;
|
||||
assign mm2mm_awprot = use_ep_engine ? ep_awprot : core_awprot;
|
||||
assign mm2mm_awqos = use_ep_engine ? ep_awqos : core_awqos;
|
||||
assign mm2mm_awuser = use_ep_engine ? ep_awuser : core_awuser;
|
||||
assign mm2mm_awvalid = use_ep_engine ? ep_awvalid : core_awvalid;
|
||||
|
||||
assign mm2mm_wdata = use_ep_engine ? ep_wdata : core_wdata;
|
||||
assign mm2mm_wstrb = use_ep_engine ? ep_wstrb : core_wstrb;
|
||||
assign mm2mm_wlast = use_ep_engine ? ep_wlast : core_wlast;
|
||||
assign mm2mm_wuser = use_ep_engine ? ep_wuser : core_wuser;
|
||||
assign mm2mm_wvalid = use_ep_engine ? ep_wvalid : core_wvalid;
|
||||
assign mm2mm_bready = use_ep_engine ? ep_bready : core_bready;
|
||||
|
||||
assign mm2mm_arid = use_ep_engine ? ep_arid : core_arid;
|
||||
assign mm2mm_araddr = use_ep_engine ? ep_araddr : core_araddr;
|
||||
assign mm2mm_arlen = use_ep_engine ? ep_arlen : core_arlen;
|
||||
assign mm2mm_arsize = use_ep_engine ? ep_arsize : core_arsize;
|
||||
assign mm2mm_arburst = use_ep_engine ? ep_arburst : core_arburst;
|
||||
assign mm2mm_arlock = use_ep_engine ? ep_arlock : core_arlock;
|
||||
assign mm2mm_arcache = use_ep_engine ? ep_arcache : core_arcache;
|
||||
assign mm2mm_arprot = use_ep_engine ? ep_arprot : core_arprot;
|
||||
assign mm2mm_arqos = use_ep_engine ? ep_arqos : core_arqos;
|
||||
assign mm2mm_aruser = use_ep_engine ? ep_aruser : core_aruser;
|
||||
assign mm2mm_arvalid = use_ep_engine ? ep_arvalid : core_arvalid;
|
||||
assign mm2mm_rready = use_ep_engine ? ep_rready : core_rready;
|
||||
|
||||
always_ff @(posedge clk) begin
|
||||
if (!rst_n) begin
|
||||
@@ -304,7 +558,9 @@ module snix_axil_cdma_mux #(
|
||||
|
||||
case (state)
|
||||
IDLE: begin
|
||||
for (int i = 0; i < PORTS; i++) begin
|
||||
if (seq_busy) begin
|
||||
state <= IDLE;
|
||||
end else for (int i = 0; i < PORTS; i++) begin
|
||||
logic [CH_BITS:0] check_ch_ext;
|
||||
logic [CH_BITS-1:0] check_ch;
|
||||
|
||||
@@ -315,13 +571,17 @@ module snix_axil_cdma_mux #(
|
||||
if (ch_req[check_ch] && !arb_set_done[check_ch]) begin
|
||||
cur_ch <= check_ch;
|
||||
rr_ptr <= (check_ch == (PORTS - 1)) ? '0 : (check_ch + 1);
|
||||
engine_start <= 1'b1;
|
||||
state <= RUN;
|
||||
state <= START;
|
||||
break;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
START: begin
|
||||
engine_start <= 1'b1;
|
||||
state <= RUN;
|
||||
end
|
||||
|
||||
RUN: begin
|
||||
if (engine_done) begin
|
||||
arb_set_done[cur_ch] <= 1'b1;
|
||||
@@ -345,37 +605,287 @@ module snix_axil_cdma_mux #(
|
||||
) u_core_engine (
|
||||
.clk (clk),
|
||||
.rst_n (rst_n),
|
||||
.ctrl_start (engine_start),
|
||||
.ctrl_start (mm2mm_start),
|
||||
.ctrl_stop (1'b0), // Tied off; can be wired if global abort is needed
|
||||
.ctrl_src_addr (engine_src),
|
||||
.ctrl_dst_addr (engine_dst),
|
||||
.ctrl_len (engine_len),
|
||||
.ctrl_size (engine_size),
|
||||
.ctrl_transfer_len (engine_bytes),
|
||||
.ctrl_done (engine_done),
|
||||
.ctrl_done (mm2mm_done),
|
||||
// AXI4 Port Connections
|
||||
.mm2mm_awid (mm2mm_awid), .mm2mm_awaddr (mm2mm_awaddr),
|
||||
.mm2mm_awlen (mm2mm_awlen), .mm2mm_awsize (mm2mm_awsize),
|
||||
.mm2mm_awburst(mm2mm_awburst),.mm2mm_awlock (mm2mm_awlock),
|
||||
.mm2mm_awcache(mm2mm_awcache),.mm2mm_awprot (mm2mm_awprot),
|
||||
.mm2mm_awqos (mm2mm_awqos), .mm2mm_awuser (mm2mm_awuser),
|
||||
.mm2mm_awvalid(mm2mm_awvalid),.mm2mm_awready(mm2mm_awready),
|
||||
.mm2mm_wdata (mm2mm_wdata), .mm2mm_wstrb (mm2mm_wstrb),
|
||||
.mm2mm_wlast (mm2mm_wlast), .mm2mm_wuser (mm2mm_wuser),
|
||||
.mm2mm_wvalid (mm2mm_wvalid), .mm2mm_wready (mm2mm_wready),
|
||||
.mm2mm_awid (core_awid), .mm2mm_awaddr (core_awaddr),
|
||||
.mm2mm_awlen (core_awlen), .mm2mm_awsize (core_awsize),
|
||||
.mm2mm_awburst(core_awburst),.mm2mm_awlock (core_awlock),
|
||||
.mm2mm_awcache(core_awcache),.mm2mm_awprot (core_awprot),
|
||||
.mm2mm_awqos (core_awqos), .mm2mm_awuser (core_awuser),
|
||||
.mm2mm_awvalid(core_awvalid),.mm2mm_awready(core_awready),
|
||||
.mm2mm_wdata (core_wdata), .mm2mm_wstrb (core_wstrb),
|
||||
.mm2mm_wlast (core_wlast), .mm2mm_wuser (core_wuser),
|
||||
.mm2mm_wvalid (core_wvalid), .mm2mm_wready (core_wready),
|
||||
.mm2mm_bid (mm2mm_bid), .mm2mm_bresp (mm2mm_bresp),
|
||||
.mm2mm_buser (mm2mm_buser), .mm2mm_bvalid (mm2mm_bvalid),
|
||||
.mm2mm_bready (mm2mm_bready),
|
||||
.mm2mm_arid (mm2mm_arid), .mm2mm_araddr (mm2mm_araddr),
|
||||
.mm2mm_arlen (mm2mm_arlen), .mm2mm_arsize (mm2mm_arsize),
|
||||
.mm2mm_arburst(mm2mm_arburst),.mm2mm_arlock (mm2mm_arlock),
|
||||
.mm2mm_arcache(mm2mm_arcache),.mm2mm_arprot (mm2mm_arprot),
|
||||
.mm2mm_arqos (mm2mm_arqos), .mm2mm_aruser (mm2mm_aruser),
|
||||
.mm2mm_arvalid(mm2mm_arvalid),.mm2mm_arready(mm2mm_arready),
|
||||
.mm2mm_bready (core_bready),
|
||||
.mm2mm_arid (core_arid), .mm2mm_araddr (core_araddr),
|
||||
.mm2mm_arlen (core_arlen), .mm2mm_arsize (core_arsize),
|
||||
.mm2mm_arburst(core_arburst),.mm2mm_arlock (core_arlock),
|
||||
.mm2mm_arcache(core_arcache),.mm2mm_arprot (core_arprot),
|
||||
.mm2mm_arqos (core_arqos), .mm2mm_aruser (core_aruser),
|
||||
.mm2mm_arvalid(core_arvalid),.mm2mm_arready(core_arready),
|
||||
.mm2mm_rid (mm2mm_rid), .mm2mm_rdata (mm2mm_rdata),
|
||||
.mm2mm_rresp (mm2mm_rresp), .mm2mm_rlast (mm2mm_rlast),
|
||||
.mm2mm_ruser (mm2mm_ruser), .mm2mm_rvalid (mm2mm_rvalid),
|
||||
.mm2mm_rready (mm2mm_rready)
|
||||
.mm2mm_rready (core_rready)
|
||||
);
|
||||
|
||||
// ==========================================
|
||||
// Direct matmul endpoint sequencer
|
||||
// ==========================================
|
||||
wire [31:0] seq_src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride);
|
||||
wire [31:0] seq_dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride);
|
||||
wire seq_loaded_bank = seq_load_idx[0];
|
||||
|
||||
always_ff @(posedge clk) begin
|
||||
if (!rst_n) begin
|
||||
seq_state <= SQ_IDLE;
|
||||
seq_busy <= 1'b0;
|
||||
seq_done_flag <= 1'b0;
|
||||
seq_group_idx <= '0;
|
||||
seq_load_idx <= '0;
|
||||
seq_store_idx <= '0;
|
||||
seq_compute_bank <= 1'b0;
|
||||
seq_ep_owner <= 1'b0;
|
||||
seq_ep_start <= 1'b0;
|
||||
seq_ep_mode <= MODE_MEM2EP;
|
||||
seq_ep_src <= '0;
|
||||
seq_ep_dst <= '0;
|
||||
seq_ep_bytes <= '0;
|
||||
seq_ep_bank <= 1'b0;
|
||||
mat_start_valid <= 1'b0;
|
||||
mat_start_bank <= 1'b0;
|
||||
mat_done_clear <= 1'b0;
|
||||
mat_crc_clear <= 1'b0;
|
||||
end else begin
|
||||
seq_ep_start <= 1'b0;
|
||||
mat_start_valid <= 1'b0;
|
||||
mat_done_clear <= 1'b0;
|
||||
mat_crc_clear <= 1'b0;
|
||||
|
||||
if (seq_ep_start)
|
||||
seq_ep_owner <= 1'b1;
|
||||
else if (seq_ep_owner && ep_done)
|
||||
seq_ep_owner <= 1'b0;
|
||||
|
||||
if (seq_done_clear_req)
|
||||
seq_done_flag <= 1'b0;
|
||||
|
||||
case (seq_state)
|
||||
SQ_IDLE: begin
|
||||
seq_busy <= 1'b0;
|
||||
if (seq_start_req && (seq_groups != 32'd0)) begin
|
||||
seq_busy <= 1'b1;
|
||||
seq_done_flag <= 1'b0;
|
||||
seq_group_idx <= 32'd0;
|
||||
seq_load_idx <= 32'd0;
|
||||
seq_store_idx <= 32'd0;
|
||||
seq_compute_bank <= 1'b0;
|
||||
mat_crc_clear <= 1'b1;
|
||||
mat_done_clear <= 1'b1;
|
||||
seq_state <= SQ_CLEAR;
|
||||
end
|
||||
end
|
||||
|
||||
SQ_CLEAR: begin
|
||||
seq_state <= SQ_LOAD;
|
||||
end
|
||||
|
||||
SQ_LOAD: begin
|
||||
if (!seq_ep_owner && !seq_ep_start) begin
|
||||
seq_ep_mode <= MODE_MEM2EP;
|
||||
seq_ep_src <= seq_src_addr_for_load;
|
||||
seq_ep_dst <= '0;
|
||||
seq_ep_bytes <= MAT_AB_BYTES;
|
||||
seq_ep_bank <= seq_loaded_bank;
|
||||
seq_ep_start <= 1'b1;
|
||||
seq_state <= SQ_LOAD_WAIT;
|
||||
end
|
||||
end
|
||||
|
||||
SQ_LOAD_WAIT: begin
|
||||
if (seq_ep_owner && ep_done) begin
|
||||
mat_start_valid <= 1'b1;
|
||||
mat_start_bank <= seq_ep_bank;
|
||||
seq_load_idx <= seq_load_idx + 32'd1;
|
||||
if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin
|
||||
seq_state <= SQ_LOAD;
|
||||
end else begin
|
||||
seq_state <= SQ_WAIT_DONE;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
SQ_WAIT_DONE: begin
|
||||
if (mat_done) begin
|
||||
mat_done_clear <= 1'b1;
|
||||
seq_state <= SQ_STORE_START;
|
||||
end
|
||||
end
|
||||
|
||||
SQ_STORE_START: begin
|
||||
if (!seq_ep_owner && !seq_ep_start) begin
|
||||
seq_ep_mode <= MODE_EP2MEM;
|
||||
seq_ep_src <= '0;
|
||||
seq_ep_dst <= seq_dst_addr_for_store;
|
||||
seq_ep_bytes <= MAT_C_BYTES;
|
||||
seq_ep_bank <= seq_compute_bank;
|
||||
seq_ep_start <= 1'b1;
|
||||
seq_state <= SQ_STORE;
|
||||
end
|
||||
end
|
||||
|
||||
SQ_STORE: begin
|
||||
if (seq_ep_owner && ep_done) begin
|
||||
seq_store_idx <= seq_store_idx + 32'd1;
|
||||
seq_group_idx <= seq_group_idx + 32'd1;
|
||||
seq_compute_bank <= ~seq_compute_bank;
|
||||
if ((seq_store_idx + 32'd1) >= seq_groups) begin
|
||||
seq_state <= SQ_DONE;
|
||||
end else if (seq_load_idx < seq_groups) begin
|
||||
seq_state <= SQ_LOAD;
|
||||
end else begin
|
||||
seq_state <= SQ_WAIT_DONE;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
SQ_DONE: begin
|
||||
seq_busy <= 1'b0;
|
||||
seq_done_flag <= 1'b1;
|
||||
seq_state <= SQ_IDLE;
|
||||
end
|
||||
|
||||
default: seq_state <= SQ_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
// ==========================================
|
||||
// Endpoint transfer engine (32-bit only in this SoC integration)
|
||||
// ==========================================
|
||||
typedef enum logic [2:0] {EP_IDLE, EP_AR, EP_READ, EP_AW, EP_WRITE, EP_BRESP, EP_DONE} ep_state_t;
|
||||
ep_state_t ep_state;
|
||||
logic [31:0] ep_axi_addr;
|
||||
logic [31:0] ep_bytes_left;
|
||||
logic [7:0] ep_word_addr_r;
|
||||
|
||||
wire [7:0] ep_beats_total = (engine_bytes[31:2] == 30'd0) ? 8'd1 : engine_bytes[9:2];
|
||||
wire [7:0] ep_axi_len = ep_beats_total - 8'd1;
|
||||
wire ep_mode_mem2ep = (engine_mode == MODE_MEM2EP);
|
||||
wire ep_mode_ep2mem = (engine_mode == MODE_EP2MEM);
|
||||
|
||||
always_ff @(posedge clk) begin
|
||||
if (!rst_n) begin
|
||||
ep_state <= EP_IDLE;
|
||||
ep_done <= 1'b0;
|
||||
ep_axi_addr <= '0;
|
||||
ep_bytes_left <= '0;
|
||||
ep_word_addr_r <= '0;
|
||||
end else begin
|
||||
ep_done <= 1'b0;
|
||||
|
||||
case (ep_state)
|
||||
EP_IDLE: begin
|
||||
if (ep_start) begin
|
||||
ep_axi_addr <= ep_mode_mem2ep ? engine_src : engine_dst;
|
||||
ep_bytes_left <= engine_bytes;
|
||||
ep_word_addr_r <= engine_ep_word_off;
|
||||
ep_state <= ep_mode_mem2ep ? EP_AR : EP_AW;
|
||||
end
|
||||
end
|
||||
|
||||
EP_AR: begin
|
||||
if (ep_arvalid && ep_arready)
|
||||
ep_state <= EP_READ;
|
||||
end
|
||||
|
||||
EP_READ: begin
|
||||
if (mm2mm_rvalid && ep_rready) begin
|
||||
ep_word_addr_r <= ep_word_addr_r + 8'd1;
|
||||
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
|
||||
if (mm2mm_rlast)
|
||||
ep_state <= EP_DONE;
|
||||
end
|
||||
end
|
||||
|
||||
EP_AW: begin
|
||||
if (ep_awvalid && ep_awready)
|
||||
ep_state <= EP_WRITE;
|
||||
end
|
||||
|
||||
EP_WRITE: begin
|
||||
if (ep_wvalid && ep_wready) begin
|
||||
ep_word_addr_r <= ep_word_addr_r + 8'd1;
|
||||
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
|
||||
if (ep_wlast)
|
||||
ep_state <= EP_BRESP;
|
||||
end
|
||||
end
|
||||
|
||||
EP_BRESP: begin
|
||||
if (mm2mm_bvalid && ep_bready)
|
||||
ep_state <= EP_DONE;
|
||||
end
|
||||
|
||||
EP_DONE: begin
|
||||
ep_done <= 1'b1;
|
||||
ep_state <= EP_IDLE;
|
||||
end
|
||||
|
||||
default: ep_state <= EP_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
assign ep_arid = '0;
|
||||
assign ep_araddr = ep_axi_addr;
|
||||
assign ep_arlen = ep_axi_len;
|
||||
assign ep_arsize = 3'd2;
|
||||
assign ep_arburst = 2'b01;
|
||||
assign ep_arlock = 1'b0;
|
||||
assign ep_arcache = 4'b0;
|
||||
assign ep_arprot = 3'b0;
|
||||
assign ep_arqos = 4'b0;
|
||||
assign ep_aruser = '0;
|
||||
assign ep_arvalid = (ep_state == EP_AR);
|
||||
assign ep_rready = (ep_state == EP_READ) && ep_wr_ready;
|
||||
|
||||
assign ep_wr_valid = (ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep;
|
||||
assign ep_wr_id = engine_ep_id;
|
||||
assign ep_wr_bank = engine_ep_bank;
|
||||
assign ep_wr_word_addr = ep_word_addr_r;
|
||||
assign ep_wr_data = mm2mm_rdata[31:0];
|
||||
|
||||
assign ep_awid = '0;
|
||||
assign ep_awaddr = ep_axi_addr;
|
||||
assign ep_awlen = ep_axi_len;
|
||||
assign ep_awsize = 3'd2;
|
||||
assign ep_awburst = 2'b01;
|
||||
assign ep_awlock = 1'b0;
|
||||
assign ep_awcache = 4'b0;
|
||||
assign ep_awprot = 3'b0;
|
||||
assign ep_awqos = 4'b0;
|
||||
assign ep_awuser = '0;
|
||||
assign ep_awvalid = (ep_state == EP_AW);
|
||||
assign ep_wdata = ep_rd_data;
|
||||
assign ep_wstrb = {{(DATA_WIDTH/8){1'b1}}};
|
||||
assign ep_wlast = (ep_bytes_left <= 32'd4) && ep_wvalid;
|
||||
assign ep_wuser = '0;
|
||||
assign ep_wvalid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready;
|
||||
assign ep_bready = (ep_state == EP_BRESP);
|
||||
|
||||
assign ep_rd_valid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready;
|
||||
assign ep_rd_id = engine_ep_id;
|
||||
assign ep_rd_bank = engine_ep_bank;
|
||||
assign ep_rd_word_addr = ep_word_addr_r;
|
||||
|
||||
endmodule
|
||||
@@ -80,6 +80,28 @@ module matmul_axi_slave (
|
||||
output reg s_axi_rvalid,
|
||||
input s_axi_rready,
|
||||
|
||||
// CDMA local endpoint port
|
||||
input ep_wr_valid,
|
||||
input [3:0] ep_wr_id,
|
||||
input [3:0] ep_wr_bank,
|
||||
input [7:0] ep_wr_word_addr,
|
||||
input [31:0] ep_wr_data,
|
||||
output ep_wr_ready,
|
||||
|
||||
input ep_rd_valid,
|
||||
input [3:0] ep_rd_id,
|
||||
input [3:0] ep_rd_bank,
|
||||
input [7:0] ep_rd_word_addr,
|
||||
output reg [31:0] ep_rd_data,
|
||||
output ep_rd_ready,
|
||||
|
||||
// Direct control from the CDMA/matmul bridge
|
||||
input mat_start_valid,
|
||||
input mat_start_bank,
|
||||
input mat_done_clear,
|
||||
input mat_crc_clear,
|
||||
output mat_done,
|
||||
|
||||
// Interrupt output (active high level, pulsed by calc_done)
|
||||
output intr
|
||||
);
|
||||
@@ -115,6 +137,33 @@ module matmul_axi_slave (
|
||||
reg [31:0] b1_b [0:15];
|
||||
reg [65:0] b1_c [0:15];
|
||||
|
||||
assign ep_wr_ready = 1'b1;
|
||||
assign ep_rd_ready = 1'b1;
|
||||
assign mat_done = done;
|
||||
|
||||
wire [3:0] ep_wr_idx = ep_wr_word_addr[3:0];
|
||||
wire [7:0] ep_rd_elem_raw = ep_rd_word_addr / 8'd3;
|
||||
wire [7:0] ep_rd_part_raw = ep_rd_word_addr % 8'd3;
|
||||
wire [3:0] ep_rd_elem = ep_rd_elem_raw[3:0];
|
||||
wire [1:0] ep_rd_part = ep_rd_part_raw[1:0];
|
||||
wire [65:0] ep_rd_c0 = b0_c[ep_rd_elem];
|
||||
wire [65:0] ep_rd_c1 = b1_c[ep_rd_elem];
|
||||
|
||||
always @(*) begin
|
||||
ep_rd_data = 32'b0;
|
||||
if (ep_rd_word_addr < 8'd48) begin
|
||||
if (ep_rd_bank[0] == 1'b0) begin
|
||||
if (ep_rd_part == 2'd0) ep_rd_data = ep_rd_c0[31:0];
|
||||
else if (ep_rd_part == 2'd1) ep_rd_data = ep_rd_c0[63:32];
|
||||
else ep_rd_data = {30'b0, ep_rd_c0[65:64]};
|
||||
end else begin
|
||||
if (ep_rd_part == 2'd0) ep_rd_data = ep_rd_c1[31:0];
|
||||
else if (ep_rd_part == 2'd1) ep_rd_data = ep_rd_c1[63:32];
|
||||
else ep_rd_data = {30'b0, ep_rd_c1[65:64]};
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function [31:0] apply_wstrb;
|
||||
input [31:0] old_value;
|
||||
input [31:0] new_value;
|
||||
@@ -134,10 +183,12 @@ module matmul_axi_slave (
|
||||
reg [31:0] crc32_table [0:255];
|
||||
reg [31:0] crc_reg;
|
||||
|
||||
wire [31:0] crc_stream_data = s_axi_rdata;
|
||||
wire crc_stream_valid = (r_state == R_DATA) && s_axi_rvalid && s_axi_rready &&
|
||||
wire [31:0] crc_stream_data = (ep_rd_valid && ep_rd_ready) ? ep_rd_data : s_axi_rdata;
|
||||
wire crc_axi_valid = (r_state == R_DATA) && s_axi_rvalid && s_axi_rready &&
|
||||
((r_offset >= ADDR_B0_C && r_offset < (ADDR_B0_C + 12'h0C0)) ||
|
||||
(r_offset >= ADDR_B1_C && r_offset < (ADDR_B1_C + 12'h0C0)));
|
||||
wire crc_ep_valid = ep_rd_valid && ep_rd_ready && (ep_rd_word_addr < 8'd48);
|
||||
wire crc_stream_valid = crc_axi_valid || crc_ep_valid;
|
||||
|
||||
wire [31:0] crc_t0, crc_c0, crc_t1, crc_c1, crc_t2, crc_c2, crc_t3, crc_next;
|
||||
assign crc_t0 = crc32_table[(crc_reg[7:0] ^ crc_stream_data[7:0])];
|
||||
@@ -305,7 +356,8 @@ module matmul_axi_slave (
|
||||
reg calc_done_pulse;
|
||||
|
||||
// 【修改点 1】单独引出一个写脉冲信号给 CRC 清零用
|
||||
wire crc_clear_pulse = (w_state == W_DATA) && s_axi_wvalid && s_axi_wready && (w_offset == ADDR_CRC);
|
||||
wire crc_clear_pulse = ((w_state == W_DATA) && s_axi_wvalid && s_axi_wready && (w_offset == ADDR_CRC)) ||
|
||||
mat_crc_clear;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (!resetn) begin
|
||||
@@ -321,10 +373,34 @@ module matmul_axi_slave (
|
||||
if (req_0 && state == C_RUN && cur_bank == 1'b0) req_0 <= 1'b0;
|
||||
if (req_1 && state == C_RUN && cur_bank == 1'b1) req_1 <= 1'b0;
|
||||
|
||||
if (mat_start_valid) begin
|
||||
if (mat_start_bank) req_1 <= 1'b1;
|
||||
else req_0 <= 1'b1;
|
||||
end
|
||||
|
||||
if (calc_done_pulse) begin
|
||||
done <= 1'b1;
|
||||
end
|
||||
|
||||
if (mat_done_clear) begin
|
||||
done <= 1'b0;
|
||||
error <= 1'b0;
|
||||
end
|
||||
|
||||
if (ep_wr_valid && ep_wr_ready) begin
|
||||
if (ep_wr_word_addr < 8'd16) begin
|
||||
if (ep_wr_bank[0] == 1'b0)
|
||||
b0_a[ep_wr_idx] <= ep_wr_data;
|
||||
else
|
||||
b1_a[ep_wr_idx] <= ep_wr_data;
|
||||
end else if (ep_wr_word_addr < 8'd32) begin
|
||||
if (ep_wr_bank[0] == 1'b0)
|
||||
b0_b[ep_wr_idx] <= ep_wr_data;
|
||||
else
|
||||
b1_b[ep_wr_idx] <= ep_wr_data;
|
||||
end
|
||||
end
|
||||
|
||||
case (w_state)
|
||||
W_IDLE: begin
|
||||
s_axi_awready <= 1'b1;
|
||||
|
||||
@@ -604,6 +604,23 @@ wire axiOut_7_bready ;
|
||||
|
||||
wire confreg_int;
|
||||
wire matmul_intr;
|
||||
wire matmul_ep_wr_valid;
|
||||
wire [3:0] matmul_ep_wr_id;
|
||||
wire [3:0] matmul_ep_wr_bank;
|
||||
wire [7:0] matmul_ep_wr_word_addr;
|
||||
wire [31:0] matmul_ep_wr_data;
|
||||
wire matmul_ep_wr_ready;
|
||||
wire matmul_ep_rd_valid;
|
||||
wire [3:0] matmul_ep_rd_id;
|
||||
wire [3:0] matmul_ep_rd_bank;
|
||||
wire [7:0] matmul_ep_rd_word_addr;
|
||||
wire [31:0] matmul_ep_rd_data;
|
||||
wire matmul_ep_rd_ready;
|
||||
wire matmul_direct_start_valid;
|
||||
wire matmul_direct_start_bank;
|
||||
wire matmul_direct_done_clear;
|
||||
wire matmul_direct_crc_clear;
|
||||
wire matmul_direct_done;
|
||||
wire mlkem_intr;
|
||||
|
||||
AxiCrossbar_2x8 u_AxiCrossbar_2x8 (
|
||||
@@ -1519,6 +1536,27 @@ matmul_axi_slave u_matmul_axi_slave (
|
||||
.s_axi_rlast (axiOut_7_rlast ),
|
||||
.s_axi_rvalid (axiOut_7_rvalid),
|
||||
.s_axi_rready (axiOut_7_rready),
|
||||
|
||||
.ep_wr_valid (matmul_ep_wr_valid),
|
||||
.ep_wr_id (matmul_ep_wr_id),
|
||||
.ep_wr_bank (matmul_ep_wr_bank),
|
||||
.ep_wr_word_addr(matmul_ep_wr_word_addr),
|
||||
.ep_wr_data (matmul_ep_wr_data),
|
||||
.ep_wr_ready (matmul_ep_wr_ready),
|
||||
|
||||
.ep_rd_valid (matmul_ep_rd_valid),
|
||||
.ep_rd_id (matmul_ep_rd_id),
|
||||
.ep_rd_bank (matmul_ep_rd_bank),
|
||||
.ep_rd_word_addr(matmul_ep_rd_word_addr),
|
||||
.ep_rd_data (matmul_ep_rd_data),
|
||||
.ep_rd_ready (matmul_ep_rd_ready),
|
||||
|
||||
.mat_start_valid(matmul_direct_start_valid),
|
||||
.mat_start_bank (matmul_direct_start_bank),
|
||||
.mat_done_clear (matmul_direct_done_clear),
|
||||
.mat_crc_clear (matmul_direct_crc_clear),
|
||||
.mat_done (matmul_direct_done),
|
||||
|
||||
.intr (matmul_intr)
|
||||
);
|
||||
|
||||
@@ -1687,6 +1725,26 @@ snix_axil_cdma_mux #(
|
||||
.mm2mm_rvalid (dma_m_rvalid),
|
||||
.mm2mm_rready (dma_m_rready),
|
||||
|
||||
// 本地直连到 matmul,不经过 AXI crossbar
|
||||
.ep_wr_valid (matmul_ep_wr_valid),
|
||||
.ep_wr_id (matmul_ep_wr_id),
|
||||
.ep_wr_bank (matmul_ep_wr_bank),
|
||||
.ep_wr_word_addr (matmul_ep_wr_word_addr),
|
||||
.ep_wr_data (matmul_ep_wr_data),
|
||||
.ep_wr_ready (matmul_ep_wr_ready),
|
||||
.ep_rd_valid (matmul_ep_rd_valid),
|
||||
.ep_rd_id (matmul_ep_rd_id),
|
||||
.ep_rd_bank (matmul_ep_rd_bank),
|
||||
.ep_rd_word_addr (matmul_ep_rd_word_addr),
|
||||
.ep_rd_data (matmul_ep_rd_data),
|
||||
.ep_rd_ready (matmul_ep_rd_ready),
|
||||
|
||||
.mat_start_valid (matmul_direct_start_valid),
|
||||
.mat_start_bank (matmul_direct_start_bank),
|
||||
.mat_done_clear (matmul_direct_done_clear),
|
||||
.mat_crc_clear (matmul_direct_crc_clear),
|
||||
.mat_done (matmul_direct_done),
|
||||
|
||||
// 全局中断输出
|
||||
.dma_finish (dma_finish)
|
||||
);
|
||||
|
||||
9811
sdk/axi_ram.mif
9811
sdk/axi_ram.mif
File diff suppressed because it is too large
Load Diff
@@ -10,21 +10,46 @@ typedef struct {
|
||||
volatile uint32_t TAG; // 0x0C: 软件 Tag (硬件不处理,留给软件标记任务用)
|
||||
volatile uint32_t CTRL; // 0x10: 控制寄存器 (配置 Burst 属性 + 启动)
|
||||
volatile uint32_t STATUS; // 0x14: 状态寄存器 (Busy, Done)
|
||||
volatile uint32_t RESV[10]; // 0x18 ~ 0x3C: 保留空间,凑齐 0x40 字节
|
||||
volatile uint32_t MODE; // 0x18: 0=mem2mem, 1=mem2ep, 2=ep2mem
|
||||
volatile uint32_t EP_CFG; // 0x1C: [3:0]=bank, [15:8]=word offset, [19:16]=endpoint id
|
||||
volatile uint32_t RESV[8]; // 0x20 ~ 0x3C: 保留空间,凑齐 0x40 字节
|
||||
} dma_ch_regs_t;
|
||||
|
||||
// DMA 控制器基地址
|
||||
#define CDMA_MUX_BASE 0xbf300000
|
||||
#define DMA_CH(i) ((volatile dma_ch_regs_t*)(CDMA_MUX_BASE + (i) * 0x40))
|
||||
|
||||
typedef struct {
|
||||
volatile uint32_t CTRL; // 0x200: bit0 start
|
||||
volatile uint32_t STATUS; // 0x204: bit0 busy, bit1 done (W1C)
|
||||
volatile uint32_t SRC_BASE; // 0x208
|
||||
volatile uint32_t DST_BASE; // 0x20C
|
||||
volatile uint32_t GROUPS; // 0x210
|
||||
volatile uint32_t MAT_BASE; // 0x214: 保留,兼容旧软件
|
||||
volatile uint32_t SRC_STRIDE; // 0x218
|
||||
volatile uint32_t DST_STRIDE; // 0x21C
|
||||
} dma_seq_regs_t;
|
||||
|
||||
#define DMA_SEQ ((volatile dma_seq_regs_t*)(CDMA_MUX_BASE + 0x200))
|
||||
|
||||
// CTRL 寄存器位定义
|
||||
#define CTRL_START_BIT (1 << 0)
|
||||
#define CTRL_BURST_SIZE(x) (((x) & 0x7) << 3) // Bits [5:3]: AXI AxSIZE (0=1B, 1=2B, 2=4B, 3=8B)
|
||||
#define CTRL_BURST_LEN(x) (((x) & 0xFF) << 6) // Bits [13:6]: AXI AxLEN (0=1 beat, 15=16 beats)
|
||||
|
||||
#define DMA_MODE_MEM2MEM 0
|
||||
#define DMA_MODE_MEM2EP 1
|
||||
#define DMA_MODE_EP2MEM 2
|
||||
#define DMA_EP_CFG(id, bank, word_off) \
|
||||
((((id) & 0xF) << 16) | (((word_off) & 0xFF) << 8) | ((bank) & 0xF))
|
||||
|
||||
// STATUS 寄存器位定义
|
||||
#define STATUS_BUSY_BIT (1 << 0)
|
||||
#define STATUS_DONE_BIT (1 << 1)
|
||||
|
||||
#define SEQ_START_BIT (1 << 0)
|
||||
#define SEQ_BUSY_BIT (1 << 0)
|
||||
#define SEQ_DONE_BIT (1 << 1)
|
||||
|
||||
void dma_start_transfer(int ch_id, uint32_t src, uint32_t dst, uint32_t bytes, uint32_t tag);
|
||||
void dma_wait_polling(int ch_id);
|
||||
@@ -9,21 +9,46 @@ typedef struct {
|
||||
volatile u32 TAG; // 0x0C: software tag (ignored by HW)
|
||||
volatile u32 CTRL; // 0x10: control (burst config + start)
|
||||
volatile u32 STATUS; // 0x14: status (busy, done)
|
||||
volatile u32 RESV[10]; // 0x18 ~ 0x3C: padding to 0x40 bytes
|
||||
volatile u32 MODE; // 0x18: 0=mem2mem, 1=mem2ep, 2=ep2mem
|
||||
volatile u32 EP_CFG; // 0x1C: [3:0]=bank, [15:8]=word offset, [19:16]=endpoint id
|
||||
volatile u32 RESV[8]; // 0x20 ~ 0x3C: padding to 0x40 bytes
|
||||
} dma_ch_regs_t;
|
||||
|
||||
// DMA 控制器基地址
|
||||
#define CDMA_MUX_BASE 0xbf300000
|
||||
#define DMA_CH(i) ((volatile dma_ch_regs_t*)(CDMA_MUX_BASE + (i) * 0x40))
|
||||
|
||||
typedef struct {
|
||||
volatile u32 CTRL; // 0x200: bit0 start
|
||||
volatile u32 STATUS; // 0x204: bit0 busy, bit1 done (W1C)
|
||||
volatile u32 SRC_BASE; // 0x208
|
||||
volatile u32 DST_BASE; // 0x20C
|
||||
volatile u32 GROUPS; // 0x210
|
||||
volatile u32 MAT_BASE; // 0x214: reserved for compatibility
|
||||
volatile u32 SRC_STRIDE; // 0x218
|
||||
volatile u32 DST_STRIDE; // 0x21C
|
||||
} dma_seq_regs_t;
|
||||
|
||||
#define DMA_SEQ ((volatile dma_seq_regs_t*)(CDMA_MUX_BASE + 0x200))
|
||||
|
||||
// CTRL 寄存器位定义
|
||||
#define CTRL_START_BIT (1 << 0)
|
||||
#define CTRL_BURST_SIZE(x) (((x) & 0x7) << 3) // Bits [5:3]: AXI AxSIZE (0=1B, 1=2B, 2=4B, 3=8B)
|
||||
#define CTRL_BURST_LEN(x) (((x) & 0xFF) << 6) // Bits [13:6]: AXI AxLEN (0=1 beat, 15=16 beats)
|
||||
|
||||
#define DMA_MODE_MEM2MEM 0
|
||||
#define DMA_MODE_MEM2EP 1
|
||||
#define DMA_MODE_EP2MEM 2
|
||||
#define DMA_EP_CFG(id, bank, word_off) \
|
||||
((((id) & 0xF) << 16) | (((word_off) & 0xFF) << 8) | ((bank) & 0xF))
|
||||
|
||||
// STATUS 寄存器位定义
|
||||
#define STATUS_BUSY_BIT (1 << 0)
|
||||
#define STATUS_DONE_BIT (1 << 1)
|
||||
|
||||
#define SEQ_START_BIT (1 << 0)
|
||||
#define SEQ_BUSY_BIT (1 << 0)
|
||||
#define SEQ_DONE_BIT (1 << 1)
|
||||
|
||||
void dma_start_transfer(int ch_id, u32 src, u32 dst, u32 bytes, u32 tag);
|
||||
void dma_wait_polling(int ch_id);
|
||||
@@ -1,213 +1,85 @@
|
||||
/**
|
||||
* user-sample - CIC-IEC 2026 Stage 3 Evaluation Program
|
||||
* user-sample - CICIEC 2026 Stage 3 evaluation program
|
||||
*
|
||||
* (ULTIMATE PERFORMANCE DMA + PING-PONG BUFFER ARCHITECTURE v2)
|
||||
*
|
||||
* Optimizations over v1:
|
||||
* 1. Pre-fire DMA Ch0 load at loop tail → fully overlaps with IP compute
|
||||
* 2. Inlined wait loops → zero function-call overhead in hot path
|
||||
* 3. Inlined DMA start → skips redundant STATUS_BUSY_BIT check
|
||||
* 4. XOR bank toggle → avoids g%2 recomputation per iteration
|
||||
* The CDMA/matmul direct sequencer reads 5000 A/B groups from ExtRAM, writes
|
||||
* the complete C result region, and accumulates the required IEEE CRC32 in
|
||||
* hardware while the C words are streamed out of matmul.
|
||||
*/
|
||||
|
||||
#include <dma.h>
|
||||
|
||||
/* ─── UART 16550 direct access (bypasses slow printf/picolibc) ──────── */
|
||||
#define UART_THR (*(volatile unsigned char *)0xbf000000)
|
||||
#define UART_LSR (*(volatile unsigned char *)0xbf000005)
|
||||
#define LSR_THRE 0x20
|
||||
|
||||
static inline void uart_tx(unsigned char c) {
|
||||
while (!(UART_LSR & LSR_THRE)) {}
|
||||
static inline void uart_tx(unsigned char c)
|
||||
{
|
||||
while ((UART_LSR & LSR_THRE) == 0) {
|
||||
}
|
||||
UART_THR = c;
|
||||
}
|
||||
|
||||
static void uart_puts(const char *s) {
|
||||
while (*s) uart_tx(*s++);
|
||||
}
|
||||
|
||||
static void uart_puthex(unsigned int val) {
|
||||
int i;
|
||||
for (i = 28; i >= 0; i -= 4) {
|
||||
unsigned char nibble = (val >> i) & 0xF;
|
||||
uart_tx(nibble < 10 ? '0' + nibble : 'a' + nibble - 10);
|
||||
static void uart_puts(const char *s)
|
||||
{
|
||||
while (*s) {
|
||||
uart_tx((unsigned char)*s++);
|
||||
}
|
||||
}
|
||||
|
||||
static void uart_puthex(unsigned int val)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 28; i >= 0; i -= 4) {
|
||||
unsigned char nibble = (unsigned char)((val >> i) & 0xf);
|
||||
uart_tx((unsigned char)(nibble < 10 ? ('0' + nibble) : ('a' + nibble - 10)));
|
||||
}
|
||||
}
|
||||
|
||||
/* ─── Configurable group count (default 5000) ──────────────────────── */
|
||||
#ifndef MATMUL_GROUP_NUM
|
||||
#define MATMUL_GROUP_NUM 5000
|
||||
#endif
|
||||
|
||||
/* ─── BSP globals ─────────────────────────────────────────────────── */
|
||||
unsigned long UART_BASE = 0xbf000000;
|
||||
unsigned long CONFREG_TIMER_BASE = 0xbf20f100;
|
||||
unsigned long CONFREG_CLOCKS_PER_SEC = 50000000L;
|
||||
unsigned long CORE_CLOCKS_PER_SEC = 33000000L;
|
||||
|
||||
/* ─── Memory layout ────────────────────────────────────────────────── */
|
||||
#define EXTRAM_BASE_PHYS 0x1c400000
|
||||
#define EXTRAM_BASE_UNCACHED (0xA0000000 | (EXTRAM_BASE_PHYS & 0x1FFFFFFF))
|
||||
#define EXTRAM_BASE_PHYS 0x1c400000u
|
||||
#define SRC_BASE_WORDS 0u
|
||||
#define DST_BASE_WORDS (SRC_BASE_WORDS + MATMUL_GROUP_NUM * 32u)
|
||||
|
||||
#define SRC_BASE_WORDS 0
|
||||
#define DST_BASE_WORDS (SRC_BASE_WORDS + MATMUL_GROUP_NUM * 32)
|
||||
#define RESULT_WORDS (MATMUL_GROUP_NUM * 48)
|
||||
#define EXTRAM_PHYS_AB(g) (EXTRAM_BASE_PHYS + (SRC_BASE_WORDS * 4u) + (unsigned int)(g) * 128u)
|
||||
#define EXTRAM_PHYS_C(g) (EXTRAM_BASE_PHYS + (DST_BASE_WORDS * 4u) + (unsigned int)(g) * 192u)
|
||||
|
||||
/* ExtRAM Physical Addresses for DMA */
|
||||
#define EXTRAM_PHYS_AB(g) (EXTRAM_BASE_PHYS + (SRC_BASE_WORDS * 4) + (g) * 128)
|
||||
#define EXTRAM_PHYS_C(g) (EXTRAM_BASE_PHYS + (DST_BASE_WORDS * 4) + (g) * 192)
|
||||
#define MATMUL_BASE_UNCACHED 0xbf500000u
|
||||
#define MATMUL_CRC (*(volatile unsigned int *)(MATMUL_BASE_UNCACHED + 0x010u))
|
||||
|
||||
/* ─── MATMUL IP registers (Ping-Pong Memory Map) ──────────────────── */
|
||||
#define MATMUL_BASE_PHYS 0x1f500000
|
||||
#define MATMUL_BASE_UNCACHED 0xbf500000
|
||||
|
||||
#define MATMUL_CTRL (MATMUL_BASE_UNCACHED + 0x000)
|
||||
#define MATMUL_STATUS (MATMUL_BASE_UNCACHED + 0x004)
|
||||
#define MATMUL_PHYS_B0_AB (MATMUL_BASE_PHYS + 0x100)
|
||||
#define MATMUL_PHYS_B0_C (MATMUL_BASE_PHYS + 0x180)
|
||||
#define MATMUL_PHYS_B1_AB (MATMUL_BASE_PHYS + 0x300)
|
||||
#define MATMUL_PHYS_B1_C (MATMUL_BASE_PHYS + 0x380)
|
||||
|
||||
#define CTRL_START_B0 0x1
|
||||
#define CTRL_START_B1 0x2
|
||||
#define STATUS_DONE 0x2
|
||||
#define STATUS_ERROR 0x4
|
||||
|
||||
#define MATMUL_CRC (MATMUL_BASE_UNCACHED + 0x010)
|
||||
|
||||
/* ─── Pre-computed DMA burst control word ──────────────────────────── */
|
||||
#define DMA_CTRL_VAL (CTRL_BURST_SIZE(2) | CTRL_BURST_LEN(63) | CTRL_START_BIT)
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════ */
|
||||
int main(void)
|
||||
{
|
||||
if (MATMUL_GROUP_NUM <= 0) return 0;
|
||||
|
||||
/* ── Hot-path local volatile pointers (one-time addr calc) ──────── */
|
||||
volatile unsigned int *mat_status = (volatile unsigned int *)MATMUL_STATUS;
|
||||
volatile unsigned int *mat_ctrl = (volatile unsigned int *)MATMUL_CTRL;
|
||||
volatile unsigned int *mat_crc = (volatile unsigned int *)MATMUL_CRC;
|
||||
volatile dma_ch_regs_t *ch0 = DMA_CH(0);
|
||||
volatile dma_ch_regs_t *ch1 = DMA_CH(1);
|
||||
|
||||
unsigned int crc;
|
||||
int g, bank;
|
||||
|
||||
// Ping-Pong Register Address Arrays (Physical for DMA)
|
||||
unsigned int phys_bank_AB[2] = {MATMUL_PHYS_B0_AB, MATMUL_PHYS_B1_AB};
|
||||
unsigned int phys_bank_C[2] = {MATMUL_PHYS_B0_C, MATMUL_PHYS_B1_C};
|
||||
unsigned int ctrl_start[2] = {CTRL_START_B0, CTRL_START_B1};
|
||||
|
||||
/* ── Start marker (must be first output) ──────────────────── */
|
||||
uart_puts("MATMUL_START\r\n");
|
||||
|
||||
// Reset IP state
|
||||
*mat_status = STATUS_DONE | STATUS_ERROR;
|
||||
*mat_crc = 0;
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
* PROLOGUE: Load group 0 → Bank 0, start IP, pre-fire group 1
|
||||
* ═══════════════════════════════════════════════════════════════ */
|
||||
|
||||
// DMA Ch0: Load group 0 A+B (128 bytes) → IP Bank 0
|
||||
ch0->SRC_ADDR = EXTRAM_PHYS_AB(0);
|
||||
ch0->DST_ADDR = phys_bank_AB[0];
|
||||
ch0->LENGTH = 128;
|
||||
ch0->CTRL = DMA_CTRL_VAL;
|
||||
while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch0->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
// Start IP Bank 0 (group 0)
|
||||
*mat_ctrl = ctrl_start[0];
|
||||
|
||||
// Pre-fire: DMA Ch0 load group 1 → Bank 1 (overlaps with IP compute)
|
||||
if (MATMUL_GROUP_NUM > 1) {
|
||||
ch0->SRC_ADDR = EXTRAM_PHYS_AB(1);
|
||||
ch0->DST_ADDR = phys_bank_AB[1];
|
||||
ch0->LENGTH = 128;
|
||||
ch0->CTRL = DMA_CTRL_VAL;
|
||||
if (MATMUL_GROUP_NUM <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
* MAIN LOOP: Fully-pipelined — pre-fire DMA load at loop tail
|
||||
*
|
||||
* At iteration g:
|
||||
* IP is computing group (g-1) on prev_bank
|
||||
* Ch0 loaded group g to cur_bank (pre-fired in iteration g-1)
|
||||
*
|
||||
* Pipeline:
|
||||
* Ch0: | load g+1 | | load g+2 |
|
||||
* Ch1: | store g-1 | | store g |
|
||||
* IP : | compute g | | compute g+1 |
|
||||
* ═══════════════════════════════════════════════════════════════ */
|
||||
bank = 1; // cur_bank for iteration g=1; prev_bank = bank ^ 1
|
||||
uart_puts("MATMUL_START\n");
|
||||
|
||||
for (g = 1; g < MATMUL_GROUP_NUM; g++) {
|
||||
int cur_bank = bank;
|
||||
int prev_bank = bank ^ 1;
|
||||
DMA_SEQ->STATUS = SEQ_DONE_BIT;
|
||||
DMA_SEQ->SRC_BASE = EXTRAM_PHYS_AB(0);
|
||||
DMA_SEQ->DST_BASE = EXTRAM_PHYS_C(0);
|
||||
DMA_SEQ->GROUPS = MATMUL_GROUP_NUM;
|
||||
DMA_SEQ->MAT_BASE = 0x1f500000u;
|
||||
DMA_SEQ->SRC_STRIDE = 128u;
|
||||
DMA_SEQ->DST_STRIDE = 192u;
|
||||
DMA_SEQ->CTRL = SEQ_START_BIT;
|
||||
|
||||
// 1. Wait for previous IP computation (group g-1 on prev_bank)
|
||||
while (!(*mat_status & STATUS_DONE)) {}
|
||||
*mat_status = STATUS_DONE;
|
||||
|
||||
// 2. Ensure previous DMA Ch1 store is done (group g-2)
|
||||
if (g > 1) {
|
||||
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch1->STATUS = STATUS_DONE_BIT;
|
||||
while ((DMA_SEQ->STATUS & SEQ_DONE_BIT) == 0u) {
|
||||
}
|
||||
DMA_SEQ->STATUS = SEQ_DONE_BIT;
|
||||
|
||||
// 3. Start DMA Ch1 store: group (g-1) result → ExtRAM
|
||||
ch1->SRC_ADDR = phys_bank_C[prev_bank];
|
||||
ch1->DST_ADDR = EXTRAM_PHYS_C(g - 1);
|
||||
ch1->LENGTH = 192;
|
||||
ch1->CTRL = DMA_CTRL_VAL;
|
||||
|
||||
// 4. Wait for current DMA Ch0 load (group g → cur_bank, pre-fired in iteration g-1)
|
||||
while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch0->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
// 5. Start IP computation for current group on cur_bank
|
||||
*mat_ctrl = ctrl_start[cur_bank];
|
||||
|
||||
// ★ 6. PRE-FIRE: DMA Ch0 load next group (g+1) → next_bank
|
||||
// This runs CONCURRENTLY with IP computation started in step 5!
|
||||
if (g < MATMUL_GROUP_NUM - 1) {
|
||||
int next_bank = bank ^ 1;
|
||||
ch0->SRC_ADDR = EXTRAM_PHYS_AB(g + 1);
|
||||
ch0->DST_ADDR = phys_bank_AB[next_bank];
|
||||
ch0->LENGTH = 128;
|
||||
ch0->CTRL = DMA_CTRL_VAL;
|
||||
}
|
||||
|
||||
// Toggle bank for next iteration
|
||||
bank ^= 1;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
* EPILOGUE: Flush the final group
|
||||
* ═══════════════════════════════════════════════════════════════ */
|
||||
|
||||
// Wait for last IP computation
|
||||
while (!(*mat_status & STATUS_DONE)) {}
|
||||
*mat_status = STATUS_DONE;
|
||||
|
||||
// Wait for previous DMA store and start/complete final store
|
||||
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch1->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
ch1->SRC_ADDR = phys_bank_C[(MATMUL_GROUP_NUM - 1) % 2];
|
||||
ch1->DST_ADDR = EXTRAM_PHYS_C(MATMUL_GROUP_NUM - 1);
|
||||
ch1->LENGTH = 192;
|
||||
ch1->CTRL = DMA_CTRL_VAL;
|
||||
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch1->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
/* ── Read hardware CRC32 ──────────────────────────────────── */
|
||||
crc = *mat_crc;
|
||||
crc = MATMUL_CRC;
|
||||
|
||||
uart_puts("MATMUL_CRC32=");
|
||||
uart_puthex(crc);
|
||||
uart_puts("\r\nMATMUL_DONE\r\n");
|
||||
uart_puts("\nMATMUL_DONE\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1,197 +1,88 @@
|
||||
/**
|
||||
* matmul_test - CIC-IEC 2026 Stage 3 Evaluation Program
|
||||
*
|
||||
* Reads 5000 groups of 4x4 unsigned matrix multiplications from ExtRAM,
|
||||
* computes C = A x B via the hardware matmul IP, writes results back,
|
||||
* computes CRC32 over the result region, and outputs the required identifiers.
|
||||
*
|
||||
* Data layout:
|
||||
* Input: ExtRAM word offset 0x00000000, 5000 * 32 = 160000 words
|
||||
* Result: ExtRAM word offset 0x0009C400, 5000 * 48 = 240000 words
|
||||
* CRC32 over result region byte stream (960000 bytes, little-endian)
|
||||
* Uses the CDMA-to-matmul direct endpoint sequencer. The CDMA still owns the
|
||||
* external-memory traffic, while matmul A/B/C transfers use local wires instead
|
||||
* of the AXI crossbar.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <dma.h>
|
||||
|
||||
/* ─── BSP globals ─────────────────────────────────────────────────── */
|
||||
unsigned long UART_BASE = 0xbf000000;
|
||||
unsigned long CONFREG_TIMER_BASE = 0xbf20f100;
|
||||
unsigned long CONFREG_CLOCKS_PER_SEC = 50000000L;
|
||||
unsigned long CORE_CLOCKS_PER_SEC = 33000000L;
|
||||
/* UART 16550 direct access avoids the printf/newlib overhead on the hot path. */
|
||||
#define UART_THR (*(volatile unsigned char *)0xbf000000)
|
||||
#define UART_LSR (*(volatile unsigned char *)0xbf000005)
|
||||
#define LSR_THRE 0x20
|
||||
|
||||
/* ─── Memory layout ────────────────────────────────────────────────── */
|
||||
#define EXTRAM_BASE_PHYS 0x1c400000
|
||||
#define EXTRAM_BASE (0xA0000000 | (EXTRAM_BASE_PHYS & 0x1FFFFFFF))
|
||||
|
||||
#define SRC_BASE 0
|
||||
#define N_GROUPS 5000
|
||||
#define DST_BASE (SRC_BASE + N_GROUPS * 32) /* word offset 160000 */
|
||||
#define RESULT_WORDS (N_GROUPS * 48) /* 240000 words */
|
||||
#define RESULT_BYTES (RESULT_WORDS * 4) /* 960000 bytes */
|
||||
|
||||
/* ─── MATMUL IP registers ─────────────────────────────────────────── */
|
||||
#define MATMUL_BASE 0xbf500000
|
||||
#define MATMUL_CTRL (MATMUL_BASE + 0x000)
|
||||
#define MATMUL_STATUS (MATMUL_BASE + 0x004)
|
||||
#define MATMUL_A_BASE (MATMUL_BASE + 0x020)
|
||||
#define MATMUL_B_BASE (MATMUL_BASE + 0x060)
|
||||
#define MATMUL_C_BASE (MATMUL_BASE + 0x0a0)
|
||||
|
||||
#define CTRL_START 0x1
|
||||
#define STATUS_BUSY 0x1
|
||||
#define STATUS_DONE 0x2
|
||||
#define STATUS_ERROR 0x4
|
||||
|
||||
/* ─── Convenience macros ──────────────────────────────────────────── */
|
||||
#define REG_WR(addr, val) (*(volatile uint32_t *)(addr) = (uint32_t)(val))
|
||||
#define REG_RD(addr) (*(volatile uint32_t *)(addr))
|
||||
|
||||
/* ─── CRC32 table (IEEE 802.3, polynomial 0xEDB88320) ─────────────── */
|
||||
static const uint32_t crc32_table[256] = {
|
||||
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
|
||||
0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
|
||||
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
|
||||
0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
|
||||
0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
|
||||
0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
|
||||
0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
|
||||
0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
|
||||
0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
|
||||
0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
|
||||
0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940,
|
||||
0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
|
||||
0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
|
||||
0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
|
||||
0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
|
||||
0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
|
||||
0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
|
||||
0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
|
||||
0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
|
||||
0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
|
||||
0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
|
||||
0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
|
||||
0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
|
||||
0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
|
||||
0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
|
||||
0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
|
||||
0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
|
||||
0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
|
||||
0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
|
||||
0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
|
||||
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
|
||||
0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
|
||||
0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
|
||||
0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
|
||||
0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
|
||||
0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
|
||||
0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
|
||||
0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
|
||||
0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
|
||||
0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
|
||||
0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
|
||||
0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
|
||||
0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
|
||||
0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
|
||||
0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
|
||||
0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
|
||||
0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB30A04,
|
||||
0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
|
||||
0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
|
||||
0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
|
||||
0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
|
||||
0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
|
||||
0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
|
||||
0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
|
||||
0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
|
||||
0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
|
||||
0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
|
||||
0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
|
||||
0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
|
||||
0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
|
||||
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
|
||||
0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
|
||||
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
|
||||
0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
|
||||
};
|
||||
|
||||
/* ─── Read N words from IP register space ─────────────────────────── */
|
||||
static void read_ip(uint32_t base, volatile uint32_t *dst, int n)
|
||||
static inline void uart_tx(unsigned char c)
|
||||
{
|
||||
for (int i = 0; i < n; i++)
|
||||
dst[i] = REG_RD(base + i * 4);
|
||||
}
|
||||
|
||||
/* ─── Write N words to IP register space ──────────────────────────── */
|
||||
static void write_ip(uint32_t base, const volatile uint32_t *src, int n)
|
||||
{
|
||||
for (int i = 0; i < n; i++)
|
||||
REG_WR(base + i * 4, src[i]);
|
||||
}
|
||||
|
||||
/* ─── Poll IP STATUS until done or error ──────────────────────────── */
|
||||
static void wait_done(void)
|
||||
{
|
||||
uint32_t status;
|
||||
do {
|
||||
status = REG_RD(MATMUL_STATUS);
|
||||
} while (!(status & (STATUS_DONE | STATUS_ERROR)));
|
||||
}
|
||||
|
||||
/* ─── CRC32 over ExtRAM result region (little-endian word → byte stream) ─ */
|
||||
static uint32_t compute_crc32(volatile uint32_t *extram)
|
||||
{
|
||||
uint32_t crc = 0xFFFFFFFF;
|
||||
uint32_t i, w;
|
||||
|
||||
for (i = 0; i < RESULT_WORDS; i++) {
|
||||
w = extram[DST_BASE + i];
|
||||
crc = crc32_table[(crc ^ (w & 0xFF)) & 0xFF] ^ (crc >> 8);
|
||||
crc = crc32_table[(crc ^ ((w >> 8) & 0xFF)) & 0xFF] ^ (crc >> 8);
|
||||
crc = crc32_table[(crc ^ ((w >> 16) & 0xFF)) & 0xFF] ^ (crc >> 8);
|
||||
crc = crc32_table[(crc ^ (w >> 24) ) & 0xFF] ^ (crc >> 8);
|
||||
while (!(UART_LSR & LSR_THRE)) {
|
||||
}
|
||||
return crc ^ 0xFFFFFFFF;
|
||||
UART_THR = c;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════ */
|
||||
static void uart_puts(const char *s)
|
||||
{
|
||||
while (*s) {
|
||||
uart_tx((unsigned char)*s++);
|
||||
}
|
||||
}
|
||||
|
||||
static void uart_puthex(uint32_t val)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 28; i >= 0; i -= 4) {
|
||||
unsigned char nibble = (unsigned char)((val >> i) & 0xF);
|
||||
uart_tx((unsigned char)(nibble < 10 ? ('0' + nibble) : ('a' + nibble - 10)));
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef MATMUL_GROUP_NUM
|
||||
#define MATMUL_GROUP_NUM 5000
|
||||
#endif
|
||||
|
||||
unsigned long UART_BASE = 0xbf000000;
|
||||
|
||||
#define EXTRAM_BASE_PHYS 0x1c400000u
|
||||
|
||||
#define SRC_BASE_WORDS 0u
|
||||
#define DST_BASE_WORDS (SRC_BASE_WORDS + MATMUL_GROUP_NUM * 32u)
|
||||
|
||||
#define EXTRAM_PHYS_AB(g) (EXTRAM_BASE_PHYS + (SRC_BASE_WORDS * 4u) + (uint32_t)(g) * 128u)
|
||||
#define EXTRAM_PHYS_C(g) (EXTRAM_BASE_PHYS + (DST_BASE_WORDS * 4u) + (uint32_t)(g) * 192u)
|
||||
|
||||
#define MATMUL_BASE_UNCACHED 0xbf500000u
|
||||
#define MATMUL_CRC (*(volatile uint32_t *)(MATMUL_BASE_UNCACHED + 0x010u))
|
||||
|
||||
int main(void)
|
||||
{
|
||||
volatile uint32_t *extram = (volatile uint32_t *)EXTRAM_BASE;
|
||||
uint32_t a_buf[16], b_buf[16], c_buf[48];
|
||||
uint32_t crc;
|
||||
int g, i;
|
||||
|
||||
/* ── Start marker (must be first output) ──────────────────── */
|
||||
printf("MATMUL_START\n");
|
||||
|
||||
/* ── Process all 5000 groups ──────────────────────────────── */
|
||||
for (g = 0; g < N_GROUPS; g++) {
|
||||
/* Read A[16] + B[16] from ExtRAM */
|
||||
for (i = 0; i < 16; i++) {
|
||||
a_buf[i] = extram[SRC_BASE + g * 32 + i];
|
||||
b_buf[i] = extram[SRC_BASE + g * 32 + 16 + i];
|
||||
}
|
||||
/* Load into IP registers */
|
||||
write_ip(MATMUL_A_BASE, a_buf, 16);
|
||||
write_ip(MATMUL_B_BASE, b_buf, 16);
|
||||
/* Clear status flags, start computation */
|
||||
REG_WR(MATMUL_STATUS, STATUS_DONE | STATUS_ERROR);
|
||||
REG_WR(MATMUL_CTRL, CTRL_START);
|
||||
/* Wait for hardware to finish */
|
||||
wait_done();
|
||||
/* Read C[48] from IP, write to ExtRAM result region */
|
||||
read_ip(MATMUL_C_BASE, c_buf, 48);
|
||||
for (i = 0; i < 48; i++)
|
||||
extram[DST_BASE + g * 48 + i] = c_buf[i];
|
||||
if (MATMUL_GROUP_NUM <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── CRC32 over result region ─────────────────────────────── */
|
||||
crc = compute_crc32(extram);
|
||||
uart_puts("MATMUL_START\n");
|
||||
|
||||
/* ── Final markers ────────────────────────────────────────── */
|
||||
printf("MATMUL_CRC32=%08lx\n", (unsigned long)crc);
|
||||
printf("MATMUL_DONE\n");
|
||||
DMA_SEQ->STATUS = SEQ_DONE_BIT;
|
||||
DMA_SEQ->SRC_BASE = EXTRAM_PHYS_AB(0);
|
||||
DMA_SEQ->DST_BASE = EXTRAM_PHYS_C(0);
|
||||
DMA_SEQ->GROUPS = MATMUL_GROUP_NUM;
|
||||
DMA_SEQ->MAT_BASE = 0x1f500000u;
|
||||
DMA_SEQ->SRC_STRIDE = 128u;
|
||||
DMA_SEQ->DST_STRIDE = 192u;
|
||||
DMA_SEQ->CTRL = SEQ_START_BIT;
|
||||
|
||||
while ((DMA_SEQ->STATUS & SEQ_DONE_BIT) == 0u) {
|
||||
}
|
||||
DMA_SEQ->STATUS = SEQ_DONE_BIT;
|
||||
|
||||
crc = MATMUL_CRC;
|
||||
|
||||
uart_puts("MATMUL_CRC32=");
|
||||
uart_puthex(crc);
|
||||
uart_puts("\nMATMUL_DONE\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user