748 lines
32 KiB
Verilog
748 lines
32 KiB
Verilog
// mlkem_axi_slave.v — AXI4 (burst) slave wrapper for the ML-KEM core.
|
|
// Makes the ML-KEM accelerator (KeyGen / Encaps / Decaps) CPU-controllable.
|
|
//
|
|
// This slave is placed behind an AXI clock-domain converter. Its AXI side and
|
|
// mlkem_top both run in mlkem_clk; sys_clk masters reach it through Axi_CDC in
|
|
// fft_mlkem_axi_slave.
|
|
//
|
|
// Register map (byte offsets from base, e.g. virtual 0xbf400000):
|
|
// 0x000 CTRL W bit0 = start (launches op selected in CONFIG)
|
|
// bit1 = soft_reset (resets ML-KEM core state)
|
|
// 0x004 STATUS R bit0 busy, bit1 done, bit2 preload_busy
|
|
// W1C write bit1=1 to clear the latched done flag
|
|
// 0x008 CONFIG RW [2:0] k (2/3/4), [5:4] op (0=KeyGen 1=Encaps 2=Decaps)
|
|
// 0x010..0x02C D RW KeyGen seed d (8 words; d byte0 in word0[7:0])
|
|
// 0x030..0x04C Z RW implicit-reject seed z (8 words)
|
|
// 0x050..0x06C MSG RW Encaps message m (8 words)
|
|
// 0x070..0x08C SS R shared secret out (8 words)
|
|
// 0x100 PRELOAD_SEL RW 0=ek_in 1=dk_in 2=c_in
|
|
// 0x104 PRELOAD_ADDR RW byte address (auto-increments after each data write)
|
|
// 0x108 PRELOAD_DATA W write low byte -> streams to selected port (wait-stated)
|
|
// 0x10C RDBK_SEL RW 0=ek 1=dk_pke 2=full-dk 3=ciphertext
|
|
// 0x110 RDBK_ADDR RW output byte index
|
|
// 0x114 RDBK_DATA R selected readback-tap byte (low 8 bits)
|
|
// 0x1000..0x1FFF EK W: preload ek_in, R: read generated ek
|
|
// 0x2000..0x2FFF DK W: preload dk_in, R: read generated full dk
|
|
// 0x3000..0x3FFF CT W: preload c_in, R: read generated ciphertext
|
|
|
|
(* use_dsp = "no" *)
|
|
module mlkem_axi_slave (
|
|
input clk, // mlkem_clk AXI-side clock after Axi_CDC
|
|
input resetn, // mlkem_resetn, active low
|
|
|
|
// AXI Write Address Channel
|
|
input [4:0] s_axi_awid,
|
|
input [31:0] s_axi_awaddr,
|
|
input [7:0] s_axi_awlen,
|
|
input [2:0] s_axi_awsize,
|
|
input [1:0] s_axi_awburst,
|
|
input s_axi_awlock,
|
|
input [3:0] s_axi_awcache,
|
|
input [2:0] s_axi_awprot,
|
|
input s_axi_awvalid,
|
|
output reg s_axi_awready,
|
|
|
|
// AXI Write Data Channel
|
|
input [4:0] s_axi_wid,
|
|
input [31:0] s_axi_wdata,
|
|
input [3:0] s_axi_wstrb,
|
|
input s_axi_wlast,
|
|
input s_axi_wvalid,
|
|
output reg s_axi_wready,
|
|
|
|
// AXI Write Response Channel
|
|
output [4:0] s_axi_bid,
|
|
output [1:0] s_axi_bresp,
|
|
output reg s_axi_bvalid,
|
|
input s_axi_bready,
|
|
|
|
// AXI Read Address Channel
|
|
input [4:0] s_axi_arid,
|
|
input [31:0] s_axi_araddr,
|
|
input [7:0] s_axi_arlen,
|
|
input [2:0] s_axi_arsize,
|
|
input [1:0] s_axi_arburst,
|
|
input s_axi_arlock,
|
|
input [3:0] s_axi_arcache,
|
|
input [2:0] s_axi_arprot,
|
|
input s_axi_arvalid,
|
|
output reg s_axi_arready,
|
|
|
|
// AXI Read Data Channel
|
|
output reg [4:0] s_axi_rid,
|
|
output reg [31:0] s_axi_rdata,
|
|
output [1:0] s_axi_rresp,
|
|
output reg s_axi_rlast,
|
|
output reg s_axi_rvalid,
|
|
input s_axi_rready,
|
|
|
|
// Interrupt output (active high level, asserted while done latched)
|
|
output intr
|
|
);
|
|
|
|
// ==========================================
|
|
// Address map (byte offsets within the 1 MB slave window)
|
|
// ==========================================
|
|
localparam ADDR_CTRL = 12'h000;
|
|
localparam ADDR_STATUS = 12'h004;
|
|
localparam ADDR_CONFIG = 12'h008;
|
|
localparam ADDR_D = 12'h010; // 0x010..0x02C (8 words)
|
|
localparam ADDR_Z = 12'h030; // 0x030..0x04C
|
|
localparam ADDR_MSG = 12'h050; // 0x050..0x06C
|
|
localparam ADDR_SS = 12'h070; // 0x070..0x08C (read-only)
|
|
localparam ADDR_PRE_SEL = 12'h100;
|
|
localparam ADDR_PRE_ADR = 12'h104;
|
|
localparam ADDR_PRE_DAT = 12'h108;
|
|
localparam ADDR_RDB_SEL = 12'h10C;
|
|
localparam ADDR_RDB_ADR = 12'h110;
|
|
localparam ADDR_RDB_DAT = 12'h114;
|
|
localparam WIN_EK = 4'h1;
|
|
localparam WIN_DK = 4'h2;
|
|
localparam WIN_CT = 4'h3;
|
|
|
|
assign s_axi_bresp = 2'b00;
|
|
assign s_axi_rresp = 2'b00;
|
|
|
|
function [31:0] apply_wstrb;
|
|
input [31:0] old_value;
|
|
input [31:0] new_value;
|
|
input [3:0] strobe;
|
|
begin
|
|
apply_wstrb = old_value;
|
|
if (strobe[0]) apply_wstrb[7:0] = new_value[7:0];
|
|
if (strobe[1]) apply_wstrb[15:8] = new_value[15:8];
|
|
if (strobe[2]) apply_wstrb[23:16] = new_value[23:16];
|
|
if (strobe[3]) apply_wstrb[31:24] = new_value[31:24];
|
|
end
|
|
endfunction
|
|
|
|
function [7:0] select_byte;
|
|
input [31:0] word;
|
|
input [1:0] lane;
|
|
begin
|
|
case (lane)
|
|
2'd0: select_byte = word[7:0];
|
|
2'd1: select_byte = word[15:8];
|
|
2'd2: select_byte = word[23:16];
|
|
default: select_byte = word[31:24];
|
|
endcase
|
|
end
|
|
endfunction
|
|
|
|
function [1:0] window_preload_sel;
|
|
input [3:0] window;
|
|
begin
|
|
case (window)
|
|
WIN_DK: window_preload_sel = 2'd1;
|
|
WIN_CT: window_preload_sel = 2'd2;
|
|
default: window_preload_sel = 2'd0;
|
|
endcase
|
|
end
|
|
endfunction
|
|
|
|
function [1:0] window_readback_sel;
|
|
input [3:0] window;
|
|
begin
|
|
case (window)
|
|
WIN_DK: window_readback_sel = 2'd2; // full dk
|
|
WIN_CT: window_readback_sel = 2'd3; // ciphertext
|
|
default: window_readback_sel = 2'd0; // ek
|
|
endcase
|
|
end
|
|
endfunction
|
|
|
|
// ==========================================
|
|
// AXI/core clock-domain config / data registers.
|
|
// ==========================================
|
|
reg [2:0] cfg_k; // 2/3/4
|
|
reg [1:0] cfg_op; // 0=KeyGen 1=Encaps 2=Decaps
|
|
reg [31:0] d_word [0:7];
|
|
reg [31:0] z_word [0:7];
|
|
reg [31:0] msg_word [0:7];
|
|
|
|
wire [255:0] d_bus = {d_word[7], d_word[6], d_word[5], d_word[4],
|
|
d_word[3], d_word[2], d_word[1], d_word[0]};
|
|
wire [255:0] z_bus = {z_word[7], z_word[6], z_word[5], z_word[4],
|
|
z_word[3], z_word[2], z_word[1], z_word[0]};
|
|
wire [255:0] msg_bus = {msg_word[7],msg_word[6],msg_word[5],msg_word[4],
|
|
msg_word[3],msg_word[2],msg_word[1],msg_word[0]};
|
|
|
|
// Preload / readback index registers (sys domain).
|
|
reg [1:0] pre_sel; // 0=ek 1=dk 2=c
|
|
reg [11:0] pre_addr; // byte address into the selected port
|
|
reg [1:0] rdb_sel; // 0=ek 1=dk_pke 2=full-dk 3=ciphertext
|
|
reg [11:0] rdb_addr; // output byte index
|
|
|
|
// ==========================================
|
|
// Start pulse: CTRL.start flips a toggle, edge-detected into start_i.
|
|
// ==========================================
|
|
reg start_tg; // sys-domain toggle, flips on CTRL.start write
|
|
reg [2:0] start_sync; // mlkem-domain 2FF sync (+1 for edge)
|
|
wire mlkem_start = start_sync[2] ^ start_sync[1]; // 1 clk pulse
|
|
|
|
// ==========================================
|
|
// Software reset pulse: CTRL.soft_reset flips a toggle and stretches reset.
|
|
// ==========================================
|
|
reg srst_tg; // flips on CTRL.soft_reset
|
|
reg [2:0] srst_sync; // edge detector shift register
|
|
reg [3:0] srst_cnt; // reset stretcher in clk domain
|
|
wire mlkem_srst_edge = srst_sync[2] ^ srst_sync[1];
|
|
wire mlkem_srst_active = (srst_cnt != 4'd0);
|
|
wire core_resetn = resetn && !mlkem_srst_active;
|
|
|
|
// ==========================================
|
|
// Busy / done status.
|
|
// ==========================================
|
|
wire core_busy;
|
|
wire core_done;
|
|
reg done_latch;
|
|
reg [1:0] busy_sync;
|
|
reg [1:0] done_sync;
|
|
// done-clear: STATUS W1C flips a toggle, edge-detected to clear done_latch.
|
|
reg dclr_tg;
|
|
reg [2:0] dclr_sync;
|
|
wire mlkem_dclr = dclr_sync[2] ^ dclr_sync[1];
|
|
|
|
// ==========================================
|
|
// Preload byte write handshake.
|
|
// Latch {sel,addr,byte}, flip pre_req_tg, pulse the selected *_in_we for
|
|
// one cycle, then flip pre_ack_tg to release the AXI write beat.
|
|
// ==========================================
|
|
reg pre_req_tg; // a byte is pending
|
|
reg [2:0] pre_req_sync; // edge detector shift register
|
|
wire mlkem_pre_we = pre_req_sync[2] ^ pre_req_sync[1];
|
|
reg pre_ack_tg; // byte consumed
|
|
reg [2:0] pre_ack_sync; // edge detector shift register
|
|
wire pre_ack_edge = pre_ack_sync[2] ^ pre_ack_sync[1];
|
|
reg pre_busy; // a preload handshake is in flight
|
|
// latched byte/port for the in-flight preload, stable while pre_busy
|
|
reg [1:0] pre_sel_l;
|
|
reg [11:0] pre_addr_l;
|
|
reg [7:0] pre_byte_l;
|
|
reg pre_auto_inc_l;
|
|
|
|
// ==========================================
|
|
// Bulk readback byte handshake.
|
|
// Hold {sel,addr}, flip rdb_req_tg, let the BRAM read settle, latch one
|
|
// byte, then flip rdb_ack_tg.
|
|
// ==========================================
|
|
reg rdb_req_tg; // one readback byte request
|
|
reg [2:0] rdb_req_sync; // edge detector shift register
|
|
wire mlkem_rdb_req = rdb_req_sync[2] ^ rdb_req_sync[1];
|
|
reg rdb_ack_tg; // readback byte ready
|
|
reg [2:0] rdb_ack_sync; // edge detector shift register
|
|
wire rdb_ack_edge = rdb_ack_sync[2] ^ rdb_ack_sync[1];
|
|
reg [1:0] rdb_req_sel_l; // stable while request in flight
|
|
reg [11:0] rdb_req_addr_l; // stable while request in flight
|
|
reg [1:0] rdb_sel_m; // captured request
|
|
reg [11:0] rdb_addr_m; // captured request
|
|
reg [1:0] rdb_wait_m; // waits for BRAM registered read
|
|
reg rdb_busy_m;
|
|
reg [7:0] rdb_data_m; // held stable until next ack
|
|
|
|
// ==========================================
|
|
// mlkem_top instance
|
|
// ==========================================
|
|
wire [7:0] dbg_byte_o, dbg_dk_o, dbg_ct_o;
|
|
wire [255:0] ss_o;
|
|
wire [1:0] eff_rdb_sel = rdb_busy_m ? rdb_sel_m : rdb_sel;
|
|
wire [11:0] eff_rdb_addr = rdb_busy_m ? rdb_addr_m : rdb_addr;
|
|
wire dbg_byte_sel = (eff_rdb_sel == 2'd1); // 0=ek, 1=dk_pke
|
|
// The core's ek/dkp read-address muxes give dbg_dk_idx_i priority over
|
|
// dbg_byte_idx_i whenever it lands in the ek (768..1567) or dk_pke (<768)
|
|
// region (mlkem_top: ek_rd_addr / dkp_rd_addr). So only drive dbg_dk_idx_i
|
|
// with the real index during a full-dk readback (rdb_sel==2); otherwise hold
|
|
// it out of range (0xFFF) so an ek/dk_pke readback isn't corrupted.
|
|
wire [11:0] dbg_dk_idx = (eff_rdb_sel == 2'd2) ? eff_rdb_addr : 12'hFFF;
|
|
|
|
// one-cycle write-enable pulses into the core.
|
|
wire ek_in_we = core_resetn && mlkem_pre_we && (pre_sel_l == 2'd0);
|
|
wire dk_in_we = core_resetn && mlkem_pre_we && (pre_sel_l == 2'd1);
|
|
wire c_in_we = core_resetn && mlkem_pre_we && (pre_sel_l == 2'd2);
|
|
|
|
mlkem_top #(.KMAX(4)) u_mlkem (
|
|
.clk (clk),
|
|
.rst_n (core_resetn),
|
|
.k_i (cfg_k),
|
|
.op_i (cfg_op),
|
|
.d_i (d_bus),
|
|
.z_i (z_bus),
|
|
.msg_i (msg_bus),
|
|
.start_i (core_resetn ? mlkem_start : 1'b0),
|
|
.busy_o (core_busy),
|
|
.done_o (core_done),
|
|
// ek preload
|
|
.ek_in_we (ek_in_we),
|
|
.ek_in_addr (pre_addr_l[10:0]),
|
|
.ek_in_byte (pre_byte_l),
|
|
// dk preload
|
|
.dk_in_we (dk_in_we),
|
|
.dk_in_addr (pre_addr_l),
|
|
.dk_in_byte (pre_byte_l),
|
|
// ciphertext preload
|
|
.c_in_we (c_in_we),
|
|
.c_in_addr (pre_addr_l[10:0]),
|
|
.c_in_byte (pre_byte_l),
|
|
// shared secret out
|
|
.ss_o (ss_o),
|
|
// ciphertext readback tap
|
|
.dbg_ct_idx_i (eff_rdb_addr[10:0]),
|
|
.dbg_ct_o (dbg_ct_o),
|
|
// coefficient debug tap (unused by CPU path; tie inputs off)
|
|
.dbg_slot_i (6'd0),
|
|
.dbg_idx_i (8'd0),
|
|
.dbg_coeff_o (/* unused */),
|
|
// ek / dk_pke byte readback
|
|
.dbg_byte_sel_i (dbg_byte_sel),
|
|
.dbg_byte_idx_i (eff_rdb_addr[10:0]),
|
|
.dbg_byte_o (dbg_byte_o),
|
|
// full-dk byte readback
|
|
.dbg_dk_idx_i (dbg_dk_idx),
|
|
.dbg_dk_o (dbg_dk_o),
|
|
// remaining debug taps unused
|
|
.dbg_rho_o (/* unused */),
|
|
.dbg_sigma_o (/* unused */),
|
|
.dbg_r_o (/* unused */),
|
|
.dbg_hek_o (/* unused */),
|
|
.dbg_mprime_o (/* unused */),
|
|
.dbg_kbar_o (/* unused */),
|
|
.dbg_decz_o (/* unused */),
|
|
.dbg_dech_o (/* unused */)
|
|
);
|
|
|
|
// ==========================================
|
|
// Core/AXI clock domain: edge detectors, done latch, preload/readback acks.
|
|
// ==========================================
|
|
always @(posedge clk or negedge resetn) begin
|
|
if (!resetn) begin
|
|
start_sync <= 3'd0;
|
|
srst_sync <= 3'd0;
|
|
srst_cnt <= 4'd0;
|
|
dclr_sync <= 3'd0;
|
|
pre_req_sync <= 3'd0;
|
|
pre_ack_tg <= 1'b0;
|
|
rdb_req_sync <= 3'd0;
|
|
rdb_ack_tg <= 1'b0;
|
|
rdb_sel_m <= 2'd0;
|
|
rdb_addr_m <= 12'd0;
|
|
rdb_wait_m <= 2'd0;
|
|
rdb_busy_m <= 1'b0;
|
|
rdb_data_m <= 8'd0;
|
|
done_latch <= 1'b0;
|
|
end else begin
|
|
// Edge-detect request toggles.
|
|
start_sync <= {start_sync[1:0], start_tg};
|
|
srst_sync <= {srst_sync[1:0], srst_tg};
|
|
dclr_sync <= {dclr_sync[1:0], dclr_tg};
|
|
pre_req_sync <= {pre_req_sync[1:0], pre_req_tg};
|
|
rdb_req_sync <= {rdb_req_sync[1:0], rdb_req_tg};
|
|
|
|
if (mlkem_srst_edge)
|
|
srst_cnt <= 4'd8;
|
|
else if (srst_cnt != 4'd0)
|
|
srst_cnt <= srst_cnt - 4'd1;
|
|
|
|
// acknowledge a consumed preload byte (write-enable pulsed this cycle)
|
|
if (core_resetn && mlkem_pre_we) pre_ack_tg <= ~pre_ack_tg;
|
|
|
|
if (!core_resetn) begin
|
|
rdb_wait_m <= 2'd0;
|
|
rdb_busy_m <= 1'b0;
|
|
end else if (mlkem_rdb_req) begin
|
|
rdb_sel_m <= rdb_req_sel_l;
|
|
rdb_addr_m <= rdb_req_addr_l;
|
|
rdb_wait_m <= 2'd2;
|
|
rdb_busy_m <= 1'b1;
|
|
end else if (rdb_wait_m != 2'd0) begin
|
|
if (rdb_wait_m == 2'd1) begin
|
|
rdb_data_m <= rdb_byte;
|
|
rdb_ack_tg <= ~rdb_ack_tg;
|
|
rdb_wait_m <= 2'd0;
|
|
rdb_busy_m <= 1'b0;
|
|
end else begin
|
|
rdb_wait_m <= rdb_wait_m - 2'd1;
|
|
end
|
|
end
|
|
|
|
// sticky done: set on the core's done_o pulse, cleared by synced W1C.
|
|
// (mlkem_start also clears it so a new run starts with done low.)
|
|
if (mlkem_srst_active) done_latch <= 1'b0;
|
|
else if (core_done) done_latch <= 1'b1;
|
|
else if (mlkem_dclr) done_latch <= 1'b0;
|
|
else if (mlkem_start) done_latch <= 1'b0;
|
|
end
|
|
end
|
|
|
|
// ==========================================
|
|
// AXI clock domain: register status and acknowledge toggles.
|
|
// ==========================================
|
|
always @(posedge clk or negedge resetn) begin
|
|
if (!resetn) begin
|
|
busy_sync <= 2'd0;
|
|
done_sync <= 2'd0;
|
|
pre_ack_sync <= 3'd0;
|
|
rdb_ack_sync <= 3'd0;
|
|
end else begin
|
|
busy_sync <= {busy_sync[0], core_busy | mlkem_srst_active};
|
|
done_sync <= {done_sync[0], done_latch};
|
|
pre_ack_sync <= {pre_ack_sync[1:0], pre_ack_tg};
|
|
rdb_ack_sync <= {rdb_ack_sync[1:0], rdb_ack_tg};
|
|
end
|
|
end
|
|
wire status_busy = busy_sync[1];
|
|
wire status_done = done_sync[1];
|
|
|
|
assign intr = status_done;
|
|
|
|
// ==========================================
|
|
// AXI Write channel (INCR burst). Preload-data beats are wait-stated until
|
|
// the cross-domain byte handshake completes (W_PRELOAD).
|
|
// ==========================================
|
|
localparam W_IDLE = 3'd0;
|
|
localparam W_DATA = 3'd1;
|
|
localparam W_PRELOAD = 3'd2;
|
|
localparam W_STREAM = 3'd3;
|
|
localparam W_RESP = 3'd4;
|
|
|
|
reg [2:0] w_state;
|
|
reg [31:0] w_addr;
|
|
reg [4:0] w_id;
|
|
reg [1:0] w_burst;
|
|
reg w_last_l; // saved wlast across a preload wait
|
|
wire [11:0] w_offset = w_addr[11:0];
|
|
wire w_stream_window = (w_addr[15:12] == WIN_EK) ||
|
|
(w_addr[15:12] == WIN_DK) ||
|
|
(w_addr[15:12] == WIN_CT);
|
|
|
|
reg [31:0] stream_wdata;
|
|
reg [3:0] stream_wstrb;
|
|
reg [11:0] stream_byte_addr;
|
|
reg [2:0] stream_lane;
|
|
reg [1:0] stream_sel;
|
|
|
|
assign s_axi_bid = w_id;
|
|
|
|
integer wi;
|
|
always @(posedge clk or negedge resetn) begin
|
|
if (!resetn) begin
|
|
w_state <= W_IDLE;
|
|
s_axi_awready <= 1'b1;
|
|
s_axi_wready <= 1'b0;
|
|
s_axi_bvalid <= 1'b0;
|
|
cfg_k <= 3'd4; // default ML-KEM-1024
|
|
cfg_op <= 2'd0;
|
|
pre_sel <= 2'd0;
|
|
pre_addr <= 12'd0;
|
|
rdb_sel <= 2'd0;
|
|
rdb_addr <= 12'd0;
|
|
start_tg <= 1'b0;
|
|
srst_tg <= 1'b0;
|
|
dclr_tg <= 1'b0;
|
|
pre_req_tg <= 1'b0;
|
|
pre_busy <= 1'b0;
|
|
pre_sel_l <= 2'd0;
|
|
pre_addr_l <= 12'd0;
|
|
pre_byte_l <= 8'd0;
|
|
pre_auto_inc_l <= 1'b0;
|
|
w_last_l <= 1'b0;
|
|
stream_wdata <= 32'd0;
|
|
stream_wstrb <= 4'd0;
|
|
stream_byte_addr <= 12'd0;
|
|
stream_lane <= 3'd0;
|
|
stream_sel <= 2'd0;
|
|
for (wi = 0; wi < 8; wi = wi + 1) begin
|
|
d_word[wi] = 32'd0;
|
|
z_word[wi] = 32'd0;
|
|
msg_word[wi] = 32'd0;
|
|
end
|
|
end else begin
|
|
// clear the preload-busy flag when the mlkem domain acks the byte
|
|
if (pre_ack_edge) begin
|
|
pre_busy <= 1'b0;
|
|
if (pre_auto_inc_l)
|
|
pre_addr <= pre_addr + 12'd1; // legacy PRE_DAT auto-increment
|
|
end
|
|
|
|
case (w_state)
|
|
W_IDLE: begin
|
|
s_axi_awready <= 1'b1;
|
|
if (s_axi_awvalid && s_axi_awready) begin
|
|
s_axi_awready <= 1'b0;
|
|
s_axi_wready <= 1'b1;
|
|
w_addr <= s_axi_awaddr;
|
|
w_id <= s_axi_awid;
|
|
w_burst <= s_axi_awburst;
|
|
w_state <= W_DATA;
|
|
end
|
|
end
|
|
|
|
W_DATA: begin
|
|
if (s_axi_wvalid && s_axi_wready) begin
|
|
if (w_stream_window) begin
|
|
stream_wdata <= s_axi_wdata;
|
|
stream_wstrb <= s_axi_wstrb;
|
|
stream_byte_addr <= w_offset;
|
|
stream_lane <= 3'd0;
|
|
stream_sel <= window_preload_sel(w_addr[15:12]);
|
|
w_last_l <= s_axi_wlast;
|
|
s_axi_wready <= 1'b0;
|
|
w_state <= W_STREAM;
|
|
end else if (w_offset == ADDR_PRE_DAT) begin
|
|
// launch cross-domain byte write; stall the beat.
|
|
pre_sel_l <= pre_sel;
|
|
pre_addr_l <= pre_addr;
|
|
pre_byte_l <= s_axi_wdata[7:0];
|
|
pre_auto_inc_l <= 1'b1;
|
|
pre_req_tg <= ~pre_req_tg;
|
|
pre_busy <= 1'b1;
|
|
w_last_l <= s_axi_wlast;
|
|
s_axi_wready <= 1'b0;
|
|
w_state <= W_PRELOAD;
|
|
end else begin
|
|
// normal single-cycle register write
|
|
case (1'b1)
|
|
(w_offset == ADDR_CTRL): begin
|
|
if (s_axi_wdata[0]) start_tg <= ~start_tg;
|
|
if (s_axi_wdata[1]) srst_tg <= ~srst_tg;
|
|
end
|
|
(w_offset == ADDR_STATUS):
|
|
if (s_axi_wdata[1]) dclr_tg <= ~dclr_tg; // clear done
|
|
(w_offset == ADDR_CONFIG): begin
|
|
cfg_k <= s_axi_wdata[2:0];
|
|
cfg_op <= s_axi_wdata[5:4];
|
|
end
|
|
(w_offset == ADDR_PRE_SEL): pre_sel <= s_axi_wdata[1:0];
|
|
(w_offset == ADDR_PRE_ADR): pre_addr <= s_axi_wdata[11:0];
|
|
(w_offset == ADDR_RDB_SEL): rdb_sel <= s_axi_wdata[1:0];
|
|
(w_offset == ADDR_RDB_ADR): rdb_addr <= s_axi_wdata[11:0];
|
|
(w_offset >= ADDR_D && w_offset < ADDR_D + 12'h20):
|
|
d_word[(w_offset-ADDR_D)>>2] <= apply_wstrb(d_word[(w_offset-ADDR_D)>>2], s_axi_wdata, s_axi_wstrb);
|
|
(w_offset >= ADDR_Z && w_offset < ADDR_Z + 12'h20):
|
|
z_word[(w_offset-ADDR_Z)>>2] <= apply_wstrb(z_word[(w_offset-ADDR_Z)>>2], s_axi_wdata, s_axi_wstrb);
|
|
(w_offset >= ADDR_MSG && w_offset < ADDR_MSG + 12'h20):
|
|
msg_word[(w_offset-ADDR_MSG)>>2] <= apply_wstrb(msg_word[(w_offset-ADDR_MSG)>>2], s_axi_wdata, s_axi_wstrb);
|
|
default: ; // SS / RDB_DAT / unmapped: ignore
|
|
endcase
|
|
|
|
if (w_burst == 2'b01) w_addr <= w_addr + 32'd4;
|
|
if (s_axi_wlast) begin
|
|
s_axi_wready <= 1'b0;
|
|
s_axi_bvalid <= 1'b1;
|
|
w_state <= W_RESP;
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
W_PRELOAD: begin
|
|
// wait for the synced ack (pre_ack_edge clears pre_busy above)
|
|
if (!pre_busy) begin
|
|
if (w_burst == 2'b01) w_addr <= w_addr + 32'd4;
|
|
if (w_last_l) begin
|
|
s_axi_bvalid <= 1'b1;
|
|
w_state <= W_RESP;
|
|
end else begin
|
|
s_axi_wready <= 1'b1;
|
|
w_state <= W_DATA;
|
|
end
|
|
end
|
|
end
|
|
|
|
W_STREAM: begin
|
|
if (!pre_busy) begin
|
|
if (stream_lane < 3'd4 && !stream_wstrb[stream_lane[1:0]]) begin
|
|
stream_lane <= stream_lane + 3'd1;
|
|
end else if (stream_lane < 3'd4) begin
|
|
pre_sel_l <= stream_sel;
|
|
pre_addr_l <= stream_byte_addr + {10'd0, stream_lane[1:0]};
|
|
pre_byte_l <= select_byte(stream_wdata, stream_lane[1:0]);
|
|
pre_auto_inc_l <= 1'b0;
|
|
pre_req_tg <= ~pre_req_tg;
|
|
pre_busy <= 1'b1;
|
|
stream_lane <= stream_lane + 3'd1;
|
|
end else begin
|
|
if (w_burst == 2'b01) w_addr <= w_addr + 32'd4;
|
|
if (w_last_l) begin
|
|
s_axi_bvalid <= 1'b1;
|
|
w_state <= W_RESP;
|
|
end else begin
|
|
s_axi_wready <= 1'b1;
|
|
w_state <= W_DATA;
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
W_RESP: begin
|
|
if (s_axi_bvalid && s_axi_bready) begin
|
|
s_axi_bvalid <= 1'b0;
|
|
s_axi_awready <= 1'b1;
|
|
w_state <= W_IDLE;
|
|
end
|
|
end
|
|
default: w_state <= W_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
// ==========================================
|
|
// AXI Read channel (INCR burst).
|
|
// ==========================================
|
|
localparam R_IDLE = 3'd0;
|
|
localparam R_DATA = 3'd1;
|
|
localparam R_BULK_WAIT = 3'd2;
|
|
|
|
reg [2:0] r_state;
|
|
reg [31:0] r_addr;
|
|
reg [7:0] r_cnt;
|
|
reg [7:0] r_len;
|
|
reg [1:0] r_burst;
|
|
wire [11:0] r_offset = r_addr[11:0];
|
|
wire ar_stream_window = (s_axi_araddr[15:12] == WIN_EK) ||
|
|
(s_axi_araddr[15:12] == WIN_DK) ||
|
|
(s_axi_araddr[15:12] == WIN_CT);
|
|
|
|
reg r_bulk;
|
|
reg [1:0] bulk_sel_r;
|
|
reg [11:0] bulk_addr_r;
|
|
reg [1:0] bulk_lane;
|
|
reg [31:0] bulk_word;
|
|
|
|
always @(posedge clk or negedge resetn) begin
|
|
if (!resetn) begin
|
|
r_state <= R_IDLE;
|
|
s_axi_arready <= 1'b1;
|
|
s_axi_rvalid <= 1'b0;
|
|
s_axi_rlast <= 1'b0;
|
|
s_axi_rid <= 5'd0;
|
|
r_addr <= 32'd0;
|
|
r_cnt <= 8'd0;
|
|
r_len <= 8'd0;
|
|
r_burst <= 2'd0;
|
|
r_bulk <= 1'b0;
|
|
bulk_sel_r <= 2'd0;
|
|
bulk_addr_r <= 12'd0;
|
|
bulk_lane <= 2'd0;
|
|
bulk_word <= 32'd0;
|
|
rdb_req_tg <= 1'b0;
|
|
rdb_req_sel_l <= 2'd0;
|
|
rdb_req_addr_l <= 12'd0;
|
|
end else begin
|
|
case (r_state)
|
|
R_IDLE: begin
|
|
s_axi_arready <= 1'b1;
|
|
if (s_axi_arvalid && s_axi_arready) begin
|
|
s_axi_arready <= 1'b0;
|
|
r_addr <= s_axi_araddr;
|
|
r_len <= s_axi_arlen;
|
|
r_burst <= s_axi_arburst;
|
|
r_cnt <= 8'd0;
|
|
s_axi_rid <= s_axi_arid;
|
|
r_bulk <= ar_stream_window;
|
|
if (ar_stream_window) begin
|
|
bulk_sel_r <= window_readback_sel(s_axi_araddr[15:12]);
|
|
bulk_addr_r <= s_axi_araddr[11:0];
|
|
bulk_lane <= 2'd0;
|
|
bulk_word <= 32'd0;
|
|
rdb_req_sel_l <= window_readback_sel(s_axi_araddr[15:12]);
|
|
rdb_req_addr_l <= s_axi_araddr[11:0];
|
|
rdb_req_tg <= ~rdb_req_tg;
|
|
s_axi_rvalid <= 1'b0;
|
|
s_axi_rlast <= 1'b0;
|
|
r_state <= R_BULK_WAIT;
|
|
end else begin
|
|
s_axi_rvalid <= 1'b1;
|
|
s_axi_rlast <= (s_axi_arlen == 8'd0);
|
|
r_state <= R_DATA;
|
|
end
|
|
end
|
|
end
|
|
R_BULK_WAIT: begin
|
|
if (rdb_ack_edge) begin
|
|
case (bulk_lane)
|
|
2'd0: bulk_word[7:0] <= rdb_data_m;
|
|
2'd1: bulk_word[15:8] <= rdb_data_m;
|
|
2'd2: bulk_word[23:16] <= rdb_data_m;
|
|
default: bulk_word[31:24] <= rdb_data_m;
|
|
endcase
|
|
|
|
if (bulk_lane == 2'd3) begin
|
|
s_axi_rvalid <= 1'b1;
|
|
s_axi_rlast <= (r_cnt == r_len);
|
|
r_state <= R_DATA;
|
|
end else begin
|
|
bulk_lane <= bulk_lane + 2'd1;
|
|
bulk_addr_r <= bulk_addr_r + 12'd1;
|
|
rdb_req_addr_l <= bulk_addr_r + 12'd1;
|
|
rdb_req_tg <= ~rdb_req_tg;
|
|
end
|
|
end
|
|
end
|
|
R_DATA: begin
|
|
if (s_axi_rvalid && s_axi_rready) begin
|
|
if (r_cnt == r_len) begin
|
|
s_axi_rvalid <= 1'b0;
|
|
s_axi_rlast <= 1'b0;
|
|
s_axi_arready <= 1'b1;
|
|
r_bulk <= 1'b0;
|
|
r_state <= R_IDLE;
|
|
end else begin
|
|
r_cnt <= r_cnt + 8'd1;
|
|
if (r_burst == 2'b01) r_addr <= r_addr + 32'd4;
|
|
if (r_bulk) begin
|
|
bulk_addr_r <= (r_burst == 2'b01) ? (r_offset + 12'd4) : r_offset;
|
|
bulk_lane <= 2'd0;
|
|
bulk_word <= 32'd0;
|
|
rdb_req_sel_l <= bulk_sel_r;
|
|
rdb_req_addr_l <= (r_burst == 2'b01) ? (r_offset + 12'd4) : r_offset;
|
|
rdb_req_tg <= ~rdb_req_tg;
|
|
s_axi_rvalid <= 1'b0;
|
|
s_axi_rlast <= 1'b0;
|
|
r_state <= R_BULK_WAIT;
|
|
end else begin
|
|
s_axi_rlast <= (r_cnt + 8'd1 == r_len);
|
|
end
|
|
end
|
|
end
|
|
end
|
|
default: r_state <= R_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
// Read-data mux. Use the live address while in R_IDLE accepting AR so the
|
|
// first beat is valid the cycle rvalid asserts (matches matmul style).
|
|
wire [11:0] cur_off = (r_state == R_DATA) ? r_offset : s_axi_araddr[11:0];
|
|
// Selected readback byte (taps are combinational over the core's BRAMs).
|
|
wire [7:0] rdb_byte = (eff_rdb_sel == 2'd3) ? dbg_ct_o :
|
|
(eff_rdb_sel == 2'd2) ? dbg_dk_o : dbg_byte_o;
|
|
|
|
always @(*) begin
|
|
s_axi_rdata = 32'd0;
|
|
if (r_bulk && r_state == R_DATA) begin
|
|
s_axi_rdata = bulk_word;
|
|
end else if (r_state == R_DATA || (r_state == R_IDLE && s_axi_arvalid)) begin
|
|
case (1'b1)
|
|
(cur_off == ADDR_CTRL): s_axi_rdata = 32'd0;
|
|
(cur_off == ADDR_STATUS): s_axi_rdata = {29'd0, pre_busy, status_done, status_busy};
|
|
(cur_off == ADDR_CONFIG): s_axi_rdata = {26'd0, cfg_op, 1'b0, cfg_k};
|
|
(cur_off == ADDR_PRE_SEL): s_axi_rdata = {30'd0, pre_sel};
|
|
(cur_off == ADDR_PRE_ADR): s_axi_rdata = {20'd0, pre_addr};
|
|
(cur_off == ADDR_RDB_SEL): s_axi_rdata = {30'd0, rdb_sel};
|
|
(cur_off == ADDR_RDB_ADR): s_axi_rdata = {20'd0, rdb_addr};
|
|
(cur_off == ADDR_RDB_DAT): s_axi_rdata = {24'd0, rdb_byte};
|
|
(cur_off >= ADDR_D && cur_off < ADDR_D + 12'h20):
|
|
s_axi_rdata = d_word[(cur_off-ADDR_D)>>2];
|
|
(cur_off >= ADDR_Z && cur_off < ADDR_Z + 12'h20):
|
|
s_axi_rdata = z_word[(cur_off-ADDR_Z)>>2];
|
|
(cur_off >= ADDR_MSG && cur_off < ADDR_MSG + 12'h20):
|
|
s_axi_rdata = msg_word[(cur_off-ADDR_MSG)>>2];
|
|
(cur_off >= ADDR_SS && cur_off < ADDR_SS + 12'h20):
|
|
s_axi_rdata = ss_o[((cur_off-ADDR_SS)>>2)*32 +: 32];
|
|
default: s_axi_rdata = 32'd0;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
endmodule
|