diff --git a/rtl/ip/fft/axi_fft_wrapper.v b/rtl/ip/fft/axi_fft_wrapper.v index d67f2f9..a1ed573 100644 --- a/rtl/ip/fft/axi_fft_wrapper.v +++ b/rtl/ip/fft/axi_fft_wrapper.v @@ -1,8 +1,8 @@ -`define FFT_CSR_ADDR 16'hf000 -`define FFT_IN_RE_BASE 16'h1000 -`define FFT_IN_IM_BASE 16'h2000 -`define FFT_OUT_RE_BASE 16'h3000 -`define FFT_OUT_IM_BASE 16'h4000 +`define FFT_CSR_ADDR 16'hc000 +`define FFT_IN_RE_BASE 16'h8000 +`define FFT_IN_IM_BASE 16'h9000 +`define FFT_OUT_RE_BASE 16'ha000 +`define FFT_OUT_IM_BASE 16'hb000 module fft_controller #( parameter FFT_LEN = 1024, // FFT处理长度 @@ -201,10 +201,10 @@ module fft_controller #( assign s_rresp = 2'b00; // 地址译码 - wire hit_in_re = (aw_addr_reg[15:12] == 4'h1); - wire hit_in_im = (aw_addr_reg[15:12] == 4'h2); - wire hit_out_re = (ar_addr_reg[15:12] == 4'h3); - wire hit_out_im = (ar_addr_reg[15:12] == 4'h4); + wire hit_in_re = (aw_addr_reg[15:12] == 4'h8); + wire hit_in_im = (aw_addr_reg[15:12] == 4'h9); + wire hit_out_re = (ar_addr_reg[15:12] == 4'ha); + wire hit_out_im = (ar_addr_reg[15:12] == 4'hb); // 寄存器控制 reg [3:0] fft_ctrl; @@ -380,4 +380,4 @@ module fft_controller #( s_rdata = rdata_d; end -endmodule \ No newline at end of file +endmodule diff --git a/rtl/ip/mlkem/top/fft_mlkem_axi_slave.v b/rtl/ip/mlkem/top/fft_mlkem_axi_slave.v index bc72d8e..60526de 100644 --- a/rtl/ip/mlkem/top/fft_mlkem_axi_slave.v +++ b/rtl/ip/mlkem/top/fft_mlkem_axi_slave.v @@ -2,9 +2,9 @@ // // Both software stacks use the same base address, 0xbf400000. Their internal // maps do not overlap: -// ML-KEM: 0x0000..0x0114 -// FFT: 0x1000, 0x2000, 0x3000, 0x4000, 0xf000 windows -// Route 0x0xxx to ML-KEM and all other offsets to FFT. +// ML-KEM: 0x0000..0x3fff +// FFT: 0x8000..0xcfff +// Route the low ML-KEM block to ML-KEM and the high FFT block to FFT. module fft_mlkem_axi_slave ( input clk, input resetn, @@ -54,8 +54,8 @@ module fft_mlkem_axi_slave ( output mlkem_intr ); - wire aw_to_mlkem = (s_axi_awaddr[15:12] == 4'h0); - wire ar_to_mlkem = (s_axi_araddr[15:12] == 4'h0); + wire aw_to_mlkem = (s_axi_awaddr[15:12] <= 4'h3); + wire ar_to_mlkem = (s_axi_araddr[15:12] <= 4'h3); reg write_sel_mlkem; reg write_active; diff --git a/rtl/ip/mlkem/top/mlkem_axi_slave.v b/rtl/ip/mlkem/top/mlkem_axi_slave.v index e7cd112..635b2da 100644 --- a/rtl/ip/mlkem/top/mlkem_axi_slave.v +++ b/rtl/ip/mlkem/top/mlkem_axi_slave.v @@ -24,6 +24,9 @@ // 0x10C RDBK_SEL RW 0=ek 1=dk_pke 2=full-dk 3=ciphertext // 0x110 RDBK_ADDR RW output byte index // 0x114 RDBK_DATA R selected readback-tap byte (low 8 bits) +// 0x1000..0x1FFF EK W: preload ek_in, R: read generated ek +// 0x2000..0x2FFF DK W: preload dk_in, R: read generated full dk +// 0x3000..0x3FFF CT W: preload c_in, R: read generated ciphertext module mlkem_axi_slave ( input clk, // sys_clk (AXI bus clock, 50 MHz) input resetn, // sys_resetn (active low, already synchronized) @@ -94,6 +97,9 @@ module mlkem_axi_slave ( localparam ADDR_RDB_SEL = 12'h10C; localparam ADDR_RDB_ADR = 12'h110; localparam ADDR_RDB_DAT = 12'h114; + localparam WIN_EK = 4'h1; + localparam WIN_DK = 4'h2; + localparam WIN_CT = 4'h3; assign s_axi_bresp = 2'b00; assign s_axi_rresp = 2'b00; @@ -111,6 +117,41 @@ module mlkem_axi_slave ( end endfunction + function [7:0] select_byte; + input [31:0] word; + input [1:0] lane; + begin + case (lane) + 2'd0: select_byte = word[7:0]; + 2'd1: select_byte = word[15:8]; + 2'd2: select_byte = word[23:16]; + default: select_byte = word[31:24]; + endcase + end + endfunction + + function [1:0] window_preload_sel; + input [3:0] window; + begin + case (window) + WIN_DK: window_preload_sel = 2'd1; + WIN_CT: window_preload_sel = 2'd2; + default: window_preload_sel = 2'd0; + endcase + end + endfunction + + function [1:0] window_readback_sel; + input [3:0] window; + begin + case (window) + WIN_DK: window_readback_sel = 2'd2; // full dk + WIN_CT: window_readback_sel = 2'd3; // ciphertext + default: window_readback_sel = 2'd0; // ek + endcase + end + endfunction + // ========================================== // mlkem_clk = sys_clk / 2 (25 MHz) via toggle FF. // ========================================== @@ -184,19 +225,25 @@ module mlkem_axi_slave ( reg [1:0] pre_sel_l; reg [11:0] pre_addr_l; reg [7:0] pre_byte_l; + reg pre_auto_inc_l; // ========================================== // mlkem_top instance // ========================================== wire [7:0] dbg_byte_o, dbg_dk_o, dbg_ct_o; wire [255:0] ss_o; - wire dbg_byte_sel = (rdb_sel == 2'd1); // 0=ek, 1=dk_pke + wire bulk_read_active; + wire [1:0] bulk_rdb_sel; + wire [11:0] bulk_rdb_addr; + wire [1:0] eff_rdb_sel = bulk_read_active ? bulk_rdb_sel : rdb_sel; + wire [11:0] eff_rdb_addr = bulk_read_active ? bulk_rdb_addr : rdb_addr; + wire dbg_byte_sel = (eff_rdb_sel == 2'd1); // 0=ek, 1=dk_pke // The core's ek/dkp read-address muxes give dbg_dk_idx_i priority over // dbg_byte_idx_i whenever it lands in the ek (768..1567) or dk_pke (<768) // region (mlkem_top: ek_rd_addr / dkp_rd_addr). So only drive dbg_dk_idx_i // with the real index during a full-dk readback (rdb_sel==2); otherwise hold // it out of range (0xFFF) so an ek/dk_pke readback isn't corrupted. - wire [11:0] dbg_dk_idx = (rdb_sel == 2'd2) ? rdb_addr : 12'hFFF; + wire [11:0] dbg_dk_idx = (eff_rdb_sel == 2'd2) ? eff_rdb_addr : 12'hFFF; // one-cycle write-enable pulses into the core (combinational off the synced // edge; pre_sel_l/pre_addr_l/pre_byte_l are stable in sys, sampled by core). @@ -230,7 +277,7 @@ module mlkem_axi_slave ( // shared secret out .ss_o (ss_o), // ciphertext readback tap - .dbg_ct_idx_i (rdb_addr[10:0]), + .dbg_ct_idx_i (eff_rdb_addr[10:0]), .dbg_ct_o (dbg_ct_o), // coefficient debug tap (unused by CPU path; tie inputs off) .dbg_slot_i (6'd0), @@ -238,7 +285,7 @@ module mlkem_axi_slave ( .dbg_coeff_o (/* unused */), // ek / dk_pke byte readback .dbg_byte_sel_i (dbg_byte_sel), - .dbg_byte_idx_i (rdb_addr[10:0]), + .dbg_byte_idx_i (eff_rdb_addr[10:0]), .dbg_byte_o (dbg_byte_o), // full-dk byte readback .dbg_dk_idx_i (dbg_dk_idx), @@ -304,17 +351,27 @@ module mlkem_axi_slave ( // AXI Write channel (INCR burst). Preload-data beats are wait-stated until // the cross-domain byte handshake completes (W_PRELOAD). // ========================================== - localparam W_IDLE = 2'd0; - localparam W_DATA = 2'd1; - localparam W_PRELOAD = 2'd2; - localparam W_RESP = 2'd3; + localparam W_IDLE = 3'd0; + localparam W_DATA = 3'd1; + localparam W_PRELOAD = 3'd2; + localparam W_STREAM = 3'd3; + localparam W_RESP = 3'd4; - reg [1:0] w_state; + reg [2:0] w_state; reg [31:0] w_addr; reg [4:0] w_id; reg [1:0] w_burst; reg w_last_l; // saved wlast across a preload wait wire [11:0] w_offset = w_addr[11:0]; + wire w_stream_window = (w_addr[15:12] == WIN_EK) || + (w_addr[15:12] == WIN_DK) || + (w_addr[15:12] == WIN_CT); + + reg [31:0] stream_wdata; + reg [3:0] stream_wstrb; + reg [11:0] stream_byte_addr; + reg [2:0] stream_lane; + reg [1:0] stream_sel; assign s_axi_bid = w_id; @@ -338,7 +395,13 @@ module mlkem_axi_slave ( pre_sel_l <= 2'd0; pre_addr_l <= 12'd0; pre_byte_l <= 8'd0; + pre_auto_inc_l <= 1'b0; w_last_l <= 1'b0; + stream_wdata <= 32'd0; + stream_wstrb <= 4'd0; + stream_byte_addr <= 12'd0; + stream_lane <= 3'd0; + stream_sel <= 2'd0; for (wi = 0; wi < 8; wi = wi + 1) begin d_word[wi] = 32'd0; z_word[wi] = 32'd0; @@ -348,7 +411,8 @@ module mlkem_axi_slave ( // clear the preload-busy flag when the mlkem domain acks the byte if (pre_ack_edge) begin pre_busy <= 1'b0; - pre_addr <= pre_addr + 12'd1; // auto-increment for next byte + if (pre_auto_inc_l) + pre_addr <= pre_addr + 12'd1; // legacy PRE_DAT auto-increment end case (w_state) @@ -366,11 +430,21 @@ module mlkem_axi_slave ( W_DATA: begin if (s_axi_wvalid && s_axi_wready) begin - if (w_offset == ADDR_PRE_DAT) begin + if (w_stream_window) begin + stream_wdata <= s_axi_wdata; + stream_wstrb <= s_axi_wstrb; + stream_byte_addr <= w_offset; + stream_lane <= 3'd0; + stream_sel <= window_preload_sel(w_addr[15:12]); + w_last_l <= s_axi_wlast; + s_axi_wready <= 1'b0; + w_state <= W_STREAM; + end else if (w_offset == ADDR_PRE_DAT) begin // launch cross-domain byte write; stall the beat. pre_sel_l <= pre_sel; pre_addr_l <= pre_addr; pre_byte_l <= s_axi_wdata[7:0]; + pre_auto_inc_l <= 1'b1; pre_req_tg <= ~pre_req_tg; pre_busy <= 1'b1; w_last_l <= s_axi_wlast; @@ -424,6 +498,31 @@ module mlkem_axi_slave ( end end + W_STREAM: begin + if (!pre_busy) begin + if (stream_lane < 3'd4 && !stream_wstrb[stream_lane[1:0]]) begin + stream_lane <= stream_lane + 3'd1; + end else if (stream_lane < 3'd4) begin + pre_sel_l <= stream_sel; + pre_addr_l <= stream_byte_addr + {10'd0, stream_lane[1:0]}; + pre_byte_l <= select_byte(stream_wdata, stream_lane[1:0]); + pre_auto_inc_l <= 1'b0; + pre_req_tg <= ~pre_req_tg; + pre_busy <= 1'b1; + stream_lane <= stream_lane + 3'd1; + end else begin + if (w_burst == 2'b01) w_addr <= w_addr + 32'd4; + if (w_last_l) begin + s_axi_bvalid <= 1'b1; + w_state <= W_RESP; + end else begin + s_axi_wready <= 1'b1; + w_state <= W_DATA; + end + end + end + end + W_RESP: begin if (s_axi_bvalid && s_axi_bready) begin s_axi_bvalid <= 1'b0; @@ -439,15 +538,31 @@ module mlkem_axi_slave ( // ========================================== // AXI Read channel (INCR burst). // ========================================== - localparam R_IDLE = 1'b0; - localparam R_DATA = 1'b1; + localparam R_IDLE = 3'd0; + localparam R_DATA = 3'd1; + localparam R_BULK_WAIT = 3'd2; + localparam RDB_WAIT_CYCLES = 3'd4; - reg r_state; + reg [2:0] r_state; reg [31:0] r_addr; reg [7:0] r_cnt; reg [7:0] r_len; reg [1:0] r_burst; wire [11:0] r_offset = r_addr[11:0]; + wire ar_stream_window = (s_axi_araddr[15:12] == WIN_EK) || + (s_axi_araddr[15:12] == WIN_DK) || + (s_axi_araddr[15:12] == WIN_CT); + + reg r_bulk; + reg [1:0] bulk_sel_r; + reg [11:0] bulk_addr_r; + reg [1:0] bulk_lane; + reg [2:0] bulk_wait; + reg [31:0] bulk_word; + + assign bulk_read_active = r_bulk && (r_state == R_BULK_WAIT); + assign bulk_rdb_sel = bulk_sel_r; + assign bulk_rdb_addr = bulk_addr_r; always @(posedge clk or negedge resetn) begin if (!resetn) begin @@ -460,6 +575,12 @@ module mlkem_axi_slave ( r_cnt <= 8'd0; r_len <= 8'd0; r_burst <= 2'd0; + r_bulk <= 1'b0; + bulk_sel_r <= 2'd0; + bulk_addr_r <= 12'd0; + bulk_lane <= 2'd0; + bulk_wait <= 3'd0; + bulk_word <= 32'd0; end else begin case (r_state) R_IDLE: begin @@ -470,10 +591,44 @@ module mlkem_axi_slave ( r_len <= s_axi_arlen; r_burst <= s_axi_arburst; r_cnt <= 8'd0; - s_axi_rvalid <= 1'b1; - s_axi_rlast <= (s_axi_arlen == 8'd0); s_axi_rid <= s_axi_arid; - r_state <= R_DATA; + r_bulk <= ar_stream_window; + if (ar_stream_window) begin + bulk_sel_r <= window_readback_sel(s_axi_araddr[15:12]); + bulk_addr_r <= s_axi_araddr[11:0]; + bulk_lane <= 2'd0; + bulk_wait <= RDB_WAIT_CYCLES; + bulk_word <= 32'd0; + s_axi_rvalid <= 1'b0; + s_axi_rlast <= 1'b0; + r_state <= R_BULK_WAIT; + end else begin + s_axi_rvalid <= 1'b1; + s_axi_rlast <= (s_axi_arlen == 8'd0); + r_state <= R_DATA; + end + end + end + R_BULK_WAIT: begin + if (bulk_wait != 3'd0) begin + bulk_wait <= bulk_wait - 3'd1; + end else begin + case (bulk_lane) + 2'd0: bulk_word[7:0] <= rdb_byte; + 2'd1: bulk_word[15:8] <= rdb_byte; + 2'd2: bulk_word[23:16] <= rdb_byte; + default: bulk_word[31:24] <= rdb_byte; + endcase + + if (bulk_lane == 2'd3) begin + s_axi_rvalid <= 1'b1; + s_axi_rlast <= (r_cnt == r_len); + r_state <= R_DATA; + end else begin + bulk_lane <= bulk_lane + 2'd1; + bulk_addr_r <= bulk_addr_r + 12'd1; + bulk_wait <= RDB_WAIT_CYCLES; + end end end R_DATA: begin @@ -482,11 +637,22 @@ module mlkem_axi_slave ( s_axi_rvalid <= 1'b0; s_axi_rlast <= 1'b0; s_axi_arready <= 1'b1; + r_bulk <= 1'b0; r_state <= R_IDLE; end else begin r_cnt <= r_cnt + 8'd1; - s_axi_rlast <= (r_cnt + 8'd1 == r_len); if (r_burst == 2'b01) r_addr <= r_addr + 32'd4; + if (r_bulk) begin + bulk_addr_r <= (r_burst == 2'b01) ? (r_offset + 12'd4) : r_offset; + bulk_lane <= 2'd0; + bulk_wait <= RDB_WAIT_CYCLES; + bulk_word <= 32'd0; + s_axi_rvalid <= 1'b0; + s_axi_rlast <= 1'b0; + r_state <= R_BULK_WAIT; + end else begin + s_axi_rlast <= (r_cnt + 8'd1 == r_len); + end end end end @@ -499,12 +665,14 @@ module mlkem_axi_slave ( // first beat is valid the cycle rvalid asserts (matches matmul style). wire [11:0] cur_off = (r_state == R_DATA) ? r_offset : s_axi_araddr[11:0]; // Selected readback byte (taps are combinational over the core's BRAMs). - wire [7:0] rdb_byte = (rdb_sel == 2'd3) ? dbg_ct_o : - (rdb_sel == 2'd2) ? dbg_dk_o : dbg_byte_o; + wire [7:0] rdb_byte = (eff_rdb_sel == 2'd3) ? dbg_ct_o : + (eff_rdb_sel == 2'd2) ? dbg_dk_o : dbg_byte_o; always @(*) begin s_axi_rdata = 32'd0; - if (r_state == R_DATA || (r_state == R_IDLE && s_axi_arvalid)) begin + if (r_bulk && r_state == R_DATA) begin + s_axi_rdata = bulk_word; + end else if (r_state == R_DATA || (r_state == R_IDLE && s_axi_arvalid)) begin case (1'b1) (cur_off == ADDR_CTRL): s_axi_rdata = 32'd0; (cur_off == ADDR_STATUS): s_axi_rdata = {29'd0, pre_busy, status_done, status_busy}; diff --git a/sdk/software/examples/fft/fft.h b/sdk/software/examples/fft/fft.h index dca7dec..cf86335 100644 --- a/sdk/software/examples/fft/fft.h +++ b/sdk/software/examples/fft/fft.h @@ -1,11 +1,11 @@ #pragma once #define FFT_BASE 0xbf400000 -#define FFT_IN_RE_BASE (FFT_BASE + 0x1000) -#define FFT_IN_IM_BASE (FFT_BASE + 0x2000) -#define FFT_OUT_RE_BASE (FFT_BASE + 0x3000) -#define FFT_OUT_IM_BASE (FFT_BASE + 0x4000) -#define FFT_CSR_REG (FFT_BASE + 0xF000) +#define FFT_IN_RE_BASE (FFT_BASE + 0x8000) +#define FFT_IN_IM_BASE (FFT_BASE + 0x9000) +#define FFT_OUT_RE_BASE (FFT_BASE + 0xA000) +#define FFT_OUT_IM_BASE (FFT_BASE + 0xB000) +#define FFT_CSR_REG (FFT_BASE + 0xC000) #define FFT_CTRL_START (1 << 4) #define FFT_STAT_DONE (1 << 1) #define FFT_STAT_BUSY (1 << 0) @@ -13,4 +13,4 @@ void fft_start(); void fft_wait(); -unsigned int fft_get_csr(); \ No newline at end of file +unsigned int fft_get_csr(); diff --git a/sdk/software/examples/fft_dma/fft.h b/sdk/software/examples/fft_dma/fft.h index dca7dec..cf86335 100644 --- a/sdk/software/examples/fft_dma/fft.h +++ b/sdk/software/examples/fft_dma/fft.h @@ -1,11 +1,11 @@ #pragma once #define FFT_BASE 0xbf400000 -#define FFT_IN_RE_BASE (FFT_BASE + 0x1000) -#define FFT_IN_IM_BASE (FFT_BASE + 0x2000) -#define FFT_OUT_RE_BASE (FFT_BASE + 0x3000) -#define FFT_OUT_IM_BASE (FFT_BASE + 0x4000) -#define FFT_CSR_REG (FFT_BASE + 0xF000) +#define FFT_IN_RE_BASE (FFT_BASE + 0x8000) +#define FFT_IN_IM_BASE (FFT_BASE + 0x9000) +#define FFT_OUT_RE_BASE (FFT_BASE + 0xA000) +#define FFT_OUT_IM_BASE (FFT_BASE + 0xB000) +#define FFT_CSR_REG (FFT_BASE + 0xC000) #define FFT_CTRL_START (1 << 4) #define FFT_STAT_DONE (1 << 1) #define FFT_STAT_BUSY (1 << 0) @@ -13,4 +13,4 @@ void fft_start(); void fft_wait(); -unsigned int fft_get_csr(); \ No newline at end of file +unsigned int fft_get_csr(); diff --git a/sdk/software/examples/mlkem_test/dma.c b/sdk/software/examples/mlkem_test/dma.c index 312d33c..37a1777 100644 --- a/sdk/software/examples/mlkem_test/dma.c +++ b/sdk/software/examples/mlkem_test/dma.c @@ -14,6 +14,7 @@ void dma_start_transfer(int ch_id, u32 src, u32 dst, u32 bytes, u32 tag) { ch->DST_ADDR = dst; ch->LENGTH = bytes; ch->TAG = tag; // 可选，填入任务ID + ch->MODE = DMA_MODE_MEM2MEM; // 置 Burst 属性并触发传输 // AXI SIZE: 2 (代表 2^2 = 4 Bytes，与 32-bit 数据线匹配) @@ -34,4 +35,4 @@ void dma_wait_polling(int ch_id) { // 清除 Done 标志位 (Write 1 to Clear Bit 1) // 注意：写 1 清 0 的设计，所以我们对 bit 1 写入 1 ch->STATUS = STATUS_DONE_BIT; -} \ No newline at end of file +} diff --git a/sdk/software/examples/mlkem_test/main.c b/sdk/software/examples/mlkem_test/main.c index eac9527..c8e1908 100644 --- a/sdk/software/examples/mlkem_test/main.c +++ b/sdk/software/examples/mlkem_test/main.c @@ -20,6 +20,8 @@ #include #include +#include "dma.h" + /* ─── BSP globals ─────────────────────────────────────────────────── */ unsigned long UART_BASE = 0xbf000000; unsigned long CONFREG_TIMER_BASE = 0xbf20f100; @@ -41,6 +43,9 @@ unsigned long CORE_CLOCKS_PER_SEC = 33000000L; #define MLKEM_RDB_SEL (MLKEM_BASE + 0x10C) /* RW 0=ek 1=dk_pke 2=dk 3=ct */ #define MLKEM_RDB_ADR (MLKEM_BASE + 0x110) /* RW output byte index */ #define MLKEM_RDB_DAT (MLKEM_BASE + 0x114) /* R selected byte (low 8b) */ +#define MLKEM_EK_WINDOW (MLKEM_BASE + 0x1000) /* W preload ek, R read ek */ +#define MLKEM_DK_WINDOW (MLKEM_BASE + 0x2000) /* W preload dk, R read dk */ +#define MLKEM_CT_WINDOW (MLKEM_BASE + 0x3000) /* W preload c, R read ct */ #define CTRL_START 0x1 #define STATUS_BUSY 0x1 @@ -70,6 +75,8 @@ unsigned long CORE_CLOCKS_PER_SEC = 33000000L; /* ─── Convenience macros ──────────────────────────────────────────── */ #define REG_WR(addr, val) (*(volatile uint32_t *)(addr) = (uint32_t)(val)) #define REG_RD(addr) (*(volatile uint32_t *)(addr)) +#define PHYS_ADDR(addr) ((uint32_t)(addr) & 0x1FFFFFFFu) +#define UNCACHED_ADDR(addr) ((PHYS_ADDR(addr)) | 0xA0000000u) /* PLACEHOLDER */ @@ -149,6 +156,39 @@ static void readback(uint32_t sel, uint8_t *buf, int n) /* PLACEHOLDER2 */ +static volatile uint8_t *uncached_u8(uint8_t *buf) +{ + return (volatile uint8_t *)UNCACHED_ADDR(buf); +} + +static void dma_to_mlkem(uint32_t dst_window, volatile uint8_t *src, int n) +{ + dma_start_transfer(0, PHYS_ADDR(src), PHYS_ADDR(dst_window), (uint32_t)n, 0); + dma_wait_polling(0); +} + +static void dma_from_mlkem(volatile uint8_t *dst, uint32_t src_window, int n) +{ + dma_start_transfer(0, PHYS_ADDR(src_window), PHYS_ADDR(dst), (uint32_t)n, 0); + dma_wait_polling(0); +} + +static void print_hex_uc(const char *label, volatile uint8_t *b, int n) +{ + int i; + printf("%s", label); + for (i = 0; i < n; i++) + printf("%02x", b[i]); + printf("\n"); +} + +static void clear_uc(volatile uint8_t *b, int n) +{ + int i; + for (i = 0; i < n; i++) + b[i] = 0; +} + /* Arbitrary but fixed test seeds / message (32 bytes each). */ static uint8_t SEED_D[32] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F, @@ -163,9 +203,9 @@ static uint8_t MSG_M[32] = { 0xB0,0xB1,0xB2,0xB3,0xB4,0xB5,0xB6,0xB7,0xB8,0xB9,0xBA,0xBB,0xBC,0xBD,0xBE,0xBF }; -static uint8_t ek[EK_BYTES]; -static uint8_t dk[DK_BYTES]; -static uint8_t ct[CT_BYTES]; +static uint8_t ek[EK_BYTES] __attribute__((aligned(64))); +static uint8_t dk[DK_BYTES] __attribute__((aligned(64))); +static uint8_t ct[CT_BYTES] __attribute__((aligned(64))); static uint8_t ss_enc[32]; static uint8_t ss_dec[32]; @@ -181,6 +221,9 @@ static void print_hex(const char *label, const uint8_t *b, int n) int main(void) { int i, mismatch; + volatile uint8_t *ek_uc = uncached_u8(ek); + volatile uint8_t *dk_uc = uncached_u8(dk); + volatile uint8_t *ct_uc = uncached_u8(ct); printf("MLKEM_START\n"); @@ -188,21 +231,21 @@ int main(void) write_block32(MLKEM_D, SEED_D); write_block32(MLKEM_Z, SEED_Z); run_op(OP_KEYGEN); - readback(RDB_EK, ek, EK_BYTES); - readback(RDB_DK, dk, DK_BYTES); - print_hex("EK32=", ek, 32); /* first 32 bytes as a fingerprint */ + dma_from_mlkem(ek_uc, MLKEM_EK_WINDOW, EK_BYTES); + dma_from_mlkem(dk_uc, MLKEM_DK_WINDOW, DK_BYTES); + print_hex_uc("EK32=", ek_uc, 32); /* first 32 bytes as a fingerprint */ /* ── 2. Encaps: ek + msg -> ss, ct ────────────────────────────── */ - preload(PRE_EK, ek, EK_BYTES); + dma_to_mlkem(MLKEM_EK_WINDOW, ek_uc, EK_BYTES); write_block32(MLKEM_MSG, MSG_M); run_op(OP_ENCAPS); read_block32(MLKEM_SS, ss_enc); - readback(RDB_CT, ct, CT_BYTES); + dma_from_mlkem(ct_uc, MLKEM_CT_WINDOW, CT_BYTES); print_hex("SS_ENC=", ss_enc, 32); /* ── 3. Decaps: dk + ct -> ss' ────────────────────────────────── */ - preload(PRE_DK, dk, DK_BYTES); - preload(PRE_C, ct, CT_BYTES); + dma_to_mlkem(MLKEM_DK_WINDOW, dk_uc, DK_BYTES); + dma_to_mlkem(MLKEM_CT_WINDOW, ct_uc, CT_BYTES); run_op(OP_DECAPS); read_block32(MLKEM_SS, ss_dec); print_hex("SS_DEC=", ss_dec, 32); @@ -223,9 +266,9 @@ int main(void) SEED_D[2] = 0; SEED_Z[2] = 0; MSG_M[2] = 0; - memset(ek, 0, sizeof(ek)); - memset(dk, 0, sizeof(dk)); - memset(ct, 0, sizeof(ct)); + clear_uc(ek_uc, sizeof(ek)); + clear_uc(dk_uc, sizeof(dk)); + clear_uc(ct_uc, sizeof(ct)); memset(ss_enc, 0, sizeof(ss_enc)); memset(ss_dec, 0, sizeof(ss_dec)); @@ -233,21 +276,21 @@ int main(void) write_block32(MLKEM_D, SEED_D); write_block32(MLKEM_Z, SEED_Z); run_op(OP_KEYGEN); - readback(RDB_EK, ek, EK_BYTES); - readback(RDB_DK, dk, DK_BYTES); - print_hex("EK32=", ek, 32); /* first 32 bytes as a fingerprint */ + dma_from_mlkem(ek_uc, MLKEM_EK_WINDOW, EK_BYTES); + dma_from_mlkem(dk_uc, MLKEM_DK_WINDOW, DK_BYTES); + print_hex_uc("EK32=", ek_uc, 32); /* first 32 bytes as a fingerprint */ /* ── 2. Encaps: ek + msg -> ss, ct ────────────────────────────── */ - preload(PRE_EK, ek, EK_BYTES); + dma_to_mlkem(MLKEM_EK_WINDOW, ek_uc, EK_BYTES); write_block32(MLKEM_MSG, MSG_M); run_op(OP_ENCAPS); read_block32(MLKEM_SS, ss_enc); - readback(RDB_CT, ct, CT_BYTES); + dma_from_mlkem(ct_uc, MLKEM_CT_WINDOW, CT_BYTES); print_hex("SS_ENC=", ss_enc, 32); /* ── 3. Decaps: dk + ct -> ss' ────────────────────────────────── */ - preload(PRE_DK, dk, DK_BYTES); - preload(PRE_C, ct, CT_BYTES); + dma_to_mlkem(MLKEM_DK_WINDOW, dk_uc, DK_BYTES); + dma_to_mlkem(MLKEM_CT_WINDOW, ct_uc, CT_BYTES); run_op(OP_DECAPS); read_block32(MLKEM_SS, ss_dec); print_hex("SS_DEC=", ss_dec, 32);