Add direct CDMA matmul path
This commit is contained in:
@@ -1,213 +1,85 @@
|
||||
/**
|
||||
* user-sample - CIC-IEC 2026 Stage 3 Evaluation Program
|
||||
* user-sample - CICIEC 2026 Stage 3 evaluation program
|
||||
*
|
||||
* (ULTIMATE PERFORMANCE DMA + PING-PONG BUFFER ARCHITECTURE v2)
|
||||
*
|
||||
* Optimizations over v1:
|
||||
* 1. Pre-fire DMA Ch0 load at loop tail → fully overlaps with IP compute
|
||||
* 2. Inlined wait loops → zero function-call overhead in hot path
|
||||
* 3. Inlined DMA start → skips redundant STATUS_BUSY_BIT check
|
||||
* 4. XOR bank toggle → avoids g%2 recomputation per iteration
|
||||
* The CDMA/matmul direct sequencer reads 5000 A/B groups from ExtRAM, writes
|
||||
* the complete C result region, and accumulates the required IEEE CRC32 in
|
||||
* hardware while the C words are streamed out of matmul.
|
||||
*/
|
||||
|
||||
#include <dma.h>
|
||||
|
||||
/* ─── UART 16550 direct access (bypasses slow printf/picolibc) ──────── */
|
||||
#define UART_THR (*(volatile unsigned char *)0xbf000000)
|
||||
#define UART_LSR (*(volatile unsigned char *)0xbf000005)
|
||||
#define LSR_THRE 0x20
|
||||
|
||||
static inline void uart_tx(unsigned char c) {
|
||||
while (!(UART_LSR & LSR_THRE)) {}
|
||||
static inline void uart_tx(unsigned char c)
|
||||
{
|
||||
while ((UART_LSR & LSR_THRE) == 0) {
|
||||
}
|
||||
UART_THR = c;
|
||||
}
|
||||
|
||||
static void uart_puts(const char *s) {
|
||||
while (*s) uart_tx(*s++);
|
||||
}
|
||||
|
||||
static void uart_puthex(unsigned int val) {
|
||||
int i;
|
||||
for (i = 28; i >= 0; i -= 4) {
|
||||
unsigned char nibble = (val >> i) & 0xF;
|
||||
uart_tx(nibble < 10 ? '0' + nibble : 'a' + nibble - 10);
|
||||
static void uart_puts(const char *s)
|
||||
{
|
||||
while (*s) {
|
||||
uart_tx((unsigned char)*s++);
|
||||
}
|
||||
}
|
||||
|
||||
static void uart_puthex(unsigned int val)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 28; i >= 0; i -= 4) {
|
||||
unsigned char nibble = (unsigned char)((val >> i) & 0xf);
|
||||
uart_tx((unsigned char)(nibble < 10 ? ('0' + nibble) : ('a' + nibble - 10)));
|
||||
}
|
||||
}
|
||||
|
||||
/* ─── Configurable group count (default 5000) ──────────────────────── */
|
||||
#ifndef MATMUL_GROUP_NUM
|
||||
#define MATMUL_GROUP_NUM 5000
|
||||
#endif
|
||||
|
||||
/* ─── BSP globals ─────────────────────────────────────────────────── */
|
||||
unsigned long UART_BASE = 0xbf000000;
|
||||
unsigned long CONFREG_TIMER_BASE = 0xbf20f100;
|
||||
unsigned long CONFREG_CLOCKS_PER_SEC = 50000000L;
|
||||
unsigned long CORE_CLOCKS_PER_SEC = 33000000L;
|
||||
unsigned long UART_BASE = 0xbf000000;
|
||||
|
||||
/* ─── Memory layout ────────────────────────────────────────────────── */
|
||||
#define EXTRAM_BASE_PHYS 0x1c400000
|
||||
#define EXTRAM_BASE_UNCACHED (0xA0000000 | (EXTRAM_BASE_PHYS & 0x1FFFFFFF))
|
||||
#define EXTRAM_BASE_PHYS 0x1c400000u
|
||||
#define SRC_BASE_WORDS 0u
|
||||
#define DST_BASE_WORDS (SRC_BASE_WORDS + MATMUL_GROUP_NUM * 32u)
|
||||
|
||||
#define SRC_BASE_WORDS 0
|
||||
#define DST_BASE_WORDS (SRC_BASE_WORDS + MATMUL_GROUP_NUM * 32)
|
||||
#define RESULT_WORDS (MATMUL_GROUP_NUM * 48)
|
||||
#define EXTRAM_PHYS_AB(g) (EXTRAM_BASE_PHYS + (SRC_BASE_WORDS * 4u) + (unsigned int)(g) * 128u)
|
||||
#define EXTRAM_PHYS_C(g) (EXTRAM_BASE_PHYS + (DST_BASE_WORDS * 4u) + (unsigned int)(g) * 192u)
|
||||
|
||||
/* ExtRAM Physical Addresses for DMA */
|
||||
#define EXTRAM_PHYS_AB(g) (EXTRAM_BASE_PHYS + (SRC_BASE_WORDS * 4) + (g) * 128)
|
||||
#define EXTRAM_PHYS_C(g) (EXTRAM_BASE_PHYS + (DST_BASE_WORDS * 4) + (g) * 192)
|
||||
#define MATMUL_BASE_UNCACHED 0xbf500000u
|
||||
#define MATMUL_CRC (*(volatile unsigned int *)(MATMUL_BASE_UNCACHED + 0x010u))
|
||||
|
||||
/* ─── MATMUL IP registers (Ping-Pong Memory Map) ──────────────────── */
|
||||
#define MATMUL_BASE_PHYS 0x1f500000
|
||||
#define MATMUL_BASE_UNCACHED 0xbf500000
|
||||
|
||||
#define MATMUL_CTRL (MATMUL_BASE_UNCACHED + 0x000)
|
||||
#define MATMUL_STATUS (MATMUL_BASE_UNCACHED + 0x004)
|
||||
#define MATMUL_PHYS_B0_AB (MATMUL_BASE_PHYS + 0x100)
|
||||
#define MATMUL_PHYS_B0_C (MATMUL_BASE_PHYS + 0x180)
|
||||
#define MATMUL_PHYS_B1_AB (MATMUL_BASE_PHYS + 0x300)
|
||||
#define MATMUL_PHYS_B1_C (MATMUL_BASE_PHYS + 0x380)
|
||||
|
||||
#define CTRL_START_B0 0x1
|
||||
#define CTRL_START_B1 0x2
|
||||
#define STATUS_DONE 0x2
|
||||
#define STATUS_ERROR 0x4
|
||||
|
||||
#define MATMUL_CRC (MATMUL_BASE_UNCACHED + 0x010)
|
||||
|
||||
/* ─── Pre-computed DMA burst control word ──────────────────────────── */
|
||||
#define DMA_CTRL_VAL (CTRL_BURST_SIZE(2) | CTRL_BURST_LEN(63) | CTRL_START_BIT)
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════ */
|
||||
int main(void)
|
||||
{
|
||||
if (MATMUL_GROUP_NUM <= 0) return 0;
|
||||
|
||||
/* ── Hot-path local volatile pointers (one-time addr calc) ──────── */
|
||||
volatile unsigned int *mat_status = (volatile unsigned int *)MATMUL_STATUS;
|
||||
volatile unsigned int *mat_ctrl = (volatile unsigned int *)MATMUL_CTRL;
|
||||
volatile unsigned int *mat_crc = (volatile unsigned int *)MATMUL_CRC;
|
||||
volatile dma_ch_regs_t *ch0 = DMA_CH(0);
|
||||
volatile dma_ch_regs_t *ch1 = DMA_CH(1);
|
||||
|
||||
unsigned int crc;
|
||||
int g, bank;
|
||||
|
||||
// Ping-Pong Register Address Arrays (Physical for DMA)
|
||||
unsigned int phys_bank_AB[2] = {MATMUL_PHYS_B0_AB, MATMUL_PHYS_B1_AB};
|
||||
unsigned int phys_bank_C[2] = {MATMUL_PHYS_B0_C, MATMUL_PHYS_B1_C};
|
||||
unsigned int ctrl_start[2] = {CTRL_START_B0, CTRL_START_B1};
|
||||
|
||||
/* ── Start marker (must be first output) ──────────────────── */
|
||||
uart_puts("MATMUL_START\r\n");
|
||||
|
||||
// Reset IP state
|
||||
*mat_status = STATUS_DONE | STATUS_ERROR;
|
||||
*mat_crc = 0;
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
* PROLOGUE: Load group 0 → Bank 0, start IP, pre-fire group 1
|
||||
* ═══════════════════════════════════════════════════════════════ */
|
||||
|
||||
// DMA Ch0: Load group 0 A+B (128 bytes) → IP Bank 0
|
||||
ch0->SRC_ADDR = EXTRAM_PHYS_AB(0);
|
||||
ch0->DST_ADDR = phys_bank_AB[0];
|
||||
ch0->LENGTH = 128;
|
||||
ch0->CTRL = DMA_CTRL_VAL;
|
||||
while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch0->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
// Start IP Bank 0 (group 0)
|
||||
*mat_ctrl = ctrl_start[0];
|
||||
|
||||
// Pre-fire: DMA Ch0 load group 1 → Bank 1 (overlaps with IP compute)
|
||||
if (MATMUL_GROUP_NUM > 1) {
|
||||
ch0->SRC_ADDR = EXTRAM_PHYS_AB(1);
|
||||
ch0->DST_ADDR = phys_bank_AB[1];
|
||||
ch0->LENGTH = 128;
|
||||
ch0->CTRL = DMA_CTRL_VAL;
|
||||
if (MATMUL_GROUP_NUM <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
* MAIN LOOP: Fully-pipelined — pre-fire DMA load at loop tail
|
||||
*
|
||||
* At iteration g:
|
||||
* IP is computing group (g-1) on prev_bank
|
||||
* Ch0 loaded group g to cur_bank (pre-fired in iteration g-1)
|
||||
*
|
||||
* Pipeline:
|
||||
* Ch0: | load g+1 | | load g+2 |
|
||||
* Ch1: | store g-1 | | store g |
|
||||
* IP : | compute g | | compute g+1 |
|
||||
* ═══════════════════════════════════════════════════════════════ */
|
||||
bank = 1; // cur_bank for iteration g=1; prev_bank = bank ^ 1
|
||||
uart_puts("MATMUL_START\n");
|
||||
|
||||
for (g = 1; g < MATMUL_GROUP_NUM; g++) {
|
||||
int cur_bank = bank;
|
||||
int prev_bank = bank ^ 1;
|
||||
DMA_SEQ->STATUS = SEQ_DONE_BIT;
|
||||
DMA_SEQ->SRC_BASE = EXTRAM_PHYS_AB(0);
|
||||
DMA_SEQ->DST_BASE = EXTRAM_PHYS_C(0);
|
||||
DMA_SEQ->GROUPS = MATMUL_GROUP_NUM;
|
||||
DMA_SEQ->MAT_BASE = 0x1f500000u;
|
||||
DMA_SEQ->SRC_STRIDE = 128u;
|
||||
DMA_SEQ->DST_STRIDE = 192u;
|
||||
DMA_SEQ->CTRL = SEQ_START_BIT;
|
||||
|
||||
// 1. Wait for previous IP computation (group g-1 on prev_bank)
|
||||
while (!(*mat_status & STATUS_DONE)) {}
|
||||
*mat_status = STATUS_DONE;
|
||||
|
||||
// 2. Ensure previous DMA Ch1 store is done (group g-2)
|
||||
if (g > 1) {
|
||||
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch1->STATUS = STATUS_DONE_BIT;
|
||||
}
|
||||
|
||||
// 3. Start DMA Ch1 store: group (g-1) result → ExtRAM
|
||||
ch1->SRC_ADDR = phys_bank_C[prev_bank];
|
||||
ch1->DST_ADDR = EXTRAM_PHYS_C(g - 1);
|
||||
ch1->LENGTH = 192;
|
||||
ch1->CTRL = DMA_CTRL_VAL;
|
||||
|
||||
// 4. Wait for current DMA Ch0 load (group g → cur_bank, pre-fired in iteration g-1)
|
||||
while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch0->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
// 5. Start IP computation for current group on cur_bank
|
||||
*mat_ctrl = ctrl_start[cur_bank];
|
||||
|
||||
// ★ 6. PRE-FIRE: DMA Ch0 load next group (g+1) → next_bank
|
||||
// This runs CONCURRENTLY with IP computation started in step 5!
|
||||
if (g < MATMUL_GROUP_NUM - 1) {
|
||||
int next_bank = bank ^ 1;
|
||||
ch0->SRC_ADDR = EXTRAM_PHYS_AB(g + 1);
|
||||
ch0->DST_ADDR = phys_bank_AB[next_bank];
|
||||
ch0->LENGTH = 128;
|
||||
ch0->CTRL = DMA_CTRL_VAL;
|
||||
}
|
||||
|
||||
// Toggle bank for next iteration
|
||||
bank ^= 1;
|
||||
while ((DMA_SEQ->STATUS & SEQ_DONE_BIT) == 0u) {
|
||||
}
|
||||
DMA_SEQ->STATUS = SEQ_DONE_BIT;
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
* EPILOGUE: Flush the final group
|
||||
* ═══════════════════════════════════════════════════════════════ */
|
||||
|
||||
// Wait for last IP computation
|
||||
while (!(*mat_status & STATUS_DONE)) {}
|
||||
*mat_status = STATUS_DONE;
|
||||
|
||||
// Wait for previous DMA store and start/complete final store
|
||||
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch1->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
ch1->SRC_ADDR = phys_bank_C[(MATMUL_GROUP_NUM - 1) % 2];
|
||||
ch1->DST_ADDR = EXTRAM_PHYS_C(MATMUL_GROUP_NUM - 1);
|
||||
ch1->LENGTH = 192;
|
||||
ch1->CTRL = DMA_CTRL_VAL;
|
||||
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
|
||||
ch1->STATUS = STATUS_DONE_BIT;
|
||||
|
||||
/* ── Read hardware CRC32 ──────────────────────────────────── */
|
||||
crc = *mat_crc;
|
||||
crc = MATMUL_CRC;
|
||||
|
||||
uart_puts("MATMUL_CRC32=");
|
||||
uart_puthex(crc);
|
||||
uart_puts("\r\nMATMUL_DONE\r\n");
|
||||
uart_puts("\nMATMUL_DONE\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user