From cd039e2e2e938bad2a41bb097516df8033437e68 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sun, 5 Jul 2026 21:26:19 +0800 Subject: [PATCH] mlkem_test: add mlkem-native --- .../examples/mlkem_test/mlkem_native/LICENSE | 294 ++ .../mlkem_test/mlkem_native/README.md | 23 + .../mlkem_test/mlkem_native/mlkem_native.c | 699 ++++ .../mlkem_test/mlkem_native/mlkem_native.h | 527 +++ .../mlkem_native/mlkem_native_asm.S | 723 ++++ .../mlkem_native/mlkem_native_config.h | 645 ++++ .../mlkem_test/mlkem_native/src/cbmc.h | 197 + .../mlkem_test/mlkem_native/src/common.h | 282 ++ .../mlkem_test/mlkem_native/src/compress.c | 743 ++++ .../mlkem_test/mlkem_native/src/compress.h | 599 +++ .../mlkem_test/mlkem_native/src/debug.c | 64 + .../mlkem_test/mlkem_native/src/debug.h | 121 + .../mlkem_native/src/fips202/fips202.c | 249 ++ .../mlkem_native/src/fips202/fips202.h | 144 + .../mlkem_native/src/fips202/fips202x4.c | 207 ++ .../mlkem_native/src/fips202/fips202x4.h | 81 + .../mlkem_native/src/fips202/keccakf1600.c | 499 +++ .../mlkem_native/src/fips202/keccakf1600.h | 98 + .../src/fips202/native/aarch64/auto.h | 70 + .../aarch64/src/fips202_native_aarch64.h | 80 + .../src/keccak_f1600_x1_scalar_aarch64_asm.S | 377 ++ .../src/keccak_f1600_x1_v84a_aarch64_asm.S | 206 ++ .../src/keccak_f1600_x2_v84a_aarch64_asm.S | 261 ++ ...k_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S | 1078 ++++++ ...00_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S | 989 +++++ .../aarch64/src/keccakf1600_round_constants.c | 47 + .../src/fips202/native/aarch64/x1_scalar.h | 26 + .../src/fips202/native/aarch64/x1_v84a.h | 34 + .../src/fips202/native/aarch64/x2_v84a.h | 37 + .../fips202/native/aarch64/x4_v8a_scalar.h | 26 + .../native/aarch64/x4_v8a_v84a_scalar.h | 35 + .../mlkem_native/src/fips202/native/api.h | 117 + .../src/fips202/native/armv81m/README.md | 10 + .../src/fips202/native/armv81m/mve.h | 65 + .../armv81m/src/fips202_native_armv81m.h | 36 + .../native/armv81m/src/keccak_f1600_x4_mve.S | 716 ++++ .../native/armv81m/src/keccak_f1600_x4_mve.c | 40 + .../armv81m/src/keccakf1600_round_constants.c | 52 + .../armv81m/src/state_extract_bytes_x4_mve.S | 334 ++ .../armv81m/src/state_xor_bytes_x4_mve.S | 355 ++ .../mlkem_native/src/fips202/native/auto.h | 29 + .../native/x86_64/keccak_f1600_x4_avx2.h | 33 + .../native/x86_64/src/fips202_native_x86_64.h | 44 + .../x86_64/src/keccak_f1600_x4_avx2_asm.S | 487 +++ .../native/x86_64/src/keccakf1600_constants.c | 52 + .../mlkem_test/mlkem_native/src/indcpa.c | 653 ++++ .../mlkem_test/mlkem_native/src/indcpa.h | 163 + .../mlkem_test/mlkem_native/src/kem.c | 446 +++ .../mlkem_test/mlkem_native/src/kem.h | 304 ++ .../mlkem_native/src/native/aarch64/README.md | 16 + .../mlkem_native/src/native/aarch64/meta.h | 124 + .../src/native/aarch64/src/aarch64_zetas.c | 182 + .../native/aarch64/src/arith_native_aarch64.h | 183 + .../src/native/aarch64/src/intt_aarch64_asm.S | 630 ++++ .../src/native/aarch64/src/ntt_aarch64_asm.S | 564 +++ .../src/poly_mulcache_compute_aarch64_asm.S | 129 + .../aarch64/src/poly_reduce_aarch64_asm.S | 152 + .../aarch64/src/poly_tobytes_aarch64_asm.S | 119 + .../aarch64/src/poly_tomont_aarch64_asm.S | 98 + ...mul_acc_montgomery_cached_k2_aarch64_asm.S | 263 ++ ...mul_acc_montgomery_cached_k3_aarch64_asm.S | 316 ++ ...mul_acc_montgomery_cached_k4_aarch64_asm.S | 370 ++ .../aarch64/src/rej_uniform_aarch64_asm.S | 225 ++ .../native/aarch64/src/rej_uniform_table.c | 543 +++ .../mlkem_test/mlkem_native/src/native/api.h | 638 ++++ .../mlkem_test/mlkem_native/src/native/meta.h | 30 + .../mlkem_native/src/native/ppc64le/README.md | 6 + .../mlkem_native/src/native/ppc64le/meta.h | 77 + .../native/ppc64le/src/arith_native_ppc64le.h | 24 + .../src/native/ppc64le/src/consts.c | 299 ++ .../src/native/ppc64le/src/consts.h | 34 + .../src/native/ppc64le/src/intt_ppc_asm.S | 3243 +++++++++++++++++ .../src/native/ppc64le/src/ntt_ppc_asm.S | 1672 +++++++++ .../native/ppc64le/src/poly_tomont_ppc_asm.S | 315 ++ .../src/native/ppc64le/src/reduce_ppc_asm.S | 731 ++++ .../mlkem_native/src/native/riscv64/README.md | 11 + .../mlkem_native/src/native/riscv64/meta.h | 133 + .../native/riscv64/src/arith_native_riscv64.h | 45 + .../src/native/riscv64/src/rv64v_debug.c | 83 + .../src/native/riscv64/src/rv64v_debug.h | 140 + .../src/native/riscv64/src/rv64v_izetas.inc | 27 + .../src/native/riscv64/src/rv64v_poly.c | 793 ++++ .../src/native/riscv64/src/rv64v_zetas.inc | 27 + .../riscv64/src/rv64v_zetas_basemul.inc | 39 + .../mlkem_native/src/native/x86_64/README.md | 4 + .../mlkem_native/src/native/x86_64/meta.h | 304 ++ .../native/x86_64/src/arith_native_x86_64.h | 327 ++ .../src/native/x86_64/src/compress_consts.c | 104 + .../src/native/x86_64/src/compress_consts.h | 45 + .../src/native/x86_64/src/consts.c | 102 + .../src/native/x86_64/src/consts.h | 25 + .../src/native/x86_64/src/intt_avx2_asm.S | 741 ++++ .../src/native/x86_64/src/ntt_avx2_asm.S | 661 ++++ .../native/x86_64/src/nttfrombytes_avx2_asm.S | 215 ++ .../native/x86_64/src/ntttobytes_avx2_asm.S | 203 ++ .../native/x86_64/src/nttunpack_avx2_asm.S | 190 + .../x86_64/src/poly_compress_d10_avx2_asm.S | 410 +++ .../x86_64/src/poly_compress_d11_avx2_asm.S | 476 +++ .../x86_64/src/poly_compress_d4_avx2_asm.S | 191 + .../x86_64/src/poly_compress_d5_avx2_asm.S | 248 ++ .../x86_64/src/poly_decompress_d10_avx2_asm.S | 256 ++ .../x86_64/src/poly_decompress_d11_avx2_asm.S | 305 ++ .../x86_64/src/poly_decompress_d4_avx2_asm.S | 208 ++ .../x86_64/src/poly_decompress_d5_avx2_asm.S | 220 ++ .../src/poly_mulcache_compute_avx2_asm.S | 118 + ...asemul_acc_montgomery_cached_k2_avx2_asm.S | 536 +++ ...asemul_acc_montgomery_cached_k3_avx2_asm.S | 784 ++++ ...asemul_acc_montgomery_cached_k4_avx2_asm.S | 1032 ++++++ .../src/native/x86_64/src/reduce_avx2_asm.S | 234 ++ .../native/x86_64/src/rej_uniform_avx2_asm.S | 136 + .../src/native/x86_64/src/rej_uniform_table.c | 545 +++ .../src/native/x86_64/src/tomont_avx2_asm.S | 171 + .../mlkem_test/mlkem_native/src/params.h | 76 + .../mlkem_test/mlkem_native/src/poly.c | 576 +++ .../mlkem_test/mlkem_native/src/poly.h | 290 ++ .../mlkem_test/mlkem_native/src/poly_k.c | 501 +++ .../mlkem_test/mlkem_native/src/poly_k.h | 593 +++ .../mlkem_test/mlkem_native/src/randombytes.h | 52 + .../mlkem_test/mlkem_native/src/sampling.c | 367 ++ .../mlkem_test/mlkem_native/src/sampling.h | 107 + .../mlkem_test/mlkem_native/src/symmetric.h | 70 + .../mlkem_test/mlkem_native/src/sys.h | 304 ++ .../mlkem_test/mlkem_native/src/verify.c | 20 + .../mlkem_test/mlkem_native/src/verify.h | 446 +++ .../mlkem_test/mlkem_native/src/zetas.inc | 30 + .../examples/mlkem_test/sw_mlkem_config.h | 28 + 126 files changed, 37929 insertions(+) create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/LICENSE create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/README.md create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_config.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/cbmc.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/common.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/compress.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/compress.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/debug.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/debug.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/auto.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/fips202_native_aarch64.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccakf1600_round_constants.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_scalar.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_v84a.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x2_v84a.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_scalar.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/api.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/README.md create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/mve.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/fips202_native_armv81m.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccakf1600_round_constants.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/auto.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/fips202_native_x86_64.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccakf1600_constants.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/kem.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/kem.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/README.md create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/meta.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/aarch64_zetas.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/arith_native_aarch64.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/intt_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/ntt_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_mulcache_compute_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_reduce_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tobytes_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tomont_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_aarch64_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_table.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/api.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/meta.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/README.md create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/meta.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/arith_native_ppc64le.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/intt_ppc_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/ntt_ppc_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/poly_tomont_ppc_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/reduce_ppc_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/README.md create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/meta.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/arith_native_riscv64.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_izetas.inc create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_poly.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas.inc create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas_basemul.inc create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/README.md create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/meta.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/arith_native_x86_64.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/intt_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntt_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttfrombytes_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntttobytes_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttunpack_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d10_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d11_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d4_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d5_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d10_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d11_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d4_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d5_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_mulcache_compute_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/reduce_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_table.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/tomont_avx2_asm.S create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/params.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/poly.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/poly.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/randombytes.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/sampling.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/sampling.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/symmetric.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/sys.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/verify.c create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/verify.h create mode 100644 sdk/software/examples/mlkem_test/mlkem_native/src/zetas.inc create mode 100644 sdk/software/examples/mlkem_test/sw_mlkem_config.h diff --git a/sdk/software/examples/mlkem_test/mlkem_native/LICENSE b/sdk/software/examples/mlkem_test/mlkem_native/LICENSE new file mode 100644 index 0000000..0b5aad5 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/LICENSE @@ -0,0 +1,294 @@ +mlkem-native is a fork of the public domain Kyber reference +implementation, available on https://github.com/pq-crystals/kyber. + +All source code in mlkem/* and dev/* is licensed under your choice +of the Apache-2.0 license OR the ISC license OR the MIT license. +These licenses are reproduced at the bottom of this file. +The copyright holders are indicated at the top of each file. + +The code in test/notrandombytes/* is derived from +https://cr.yp.to/papers.html#surf and licensed under +LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT. +It is only used for testing purposes. + +The benchmarking code in test/hal/hal.c carries the +MIT license. It is only used for testing purposes. + +``` +Copyright (c) The mlkem-native project authors +Copyright (c) 2020 Dougall Johnson +Copyright (c) 2022 Arm Limited +SPDX-License-Identifier: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +The tiny_sha3 implementation in examples/custom_backend/mlkem_native/mlkem/fips202/native/custom/src/ +carries the MIT license. It is only used for testing purposes. + +``` +SPDX-License-Identifier: MIT + +19-Nov-11 Markku-Juhani O. Saarinen +``` + +ISC license +----------- + +Copyright + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +MIT license +----------- + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Apache-2.0 license +------------------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sdk/software/examples/mlkem_test/mlkem_native/README.md b/sdk/software/examples/mlkem_test/mlkem_native/README.md new file mode 100644 index 0000000..473326c --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/README.md @@ -0,0 +1,23 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# mlkem-native source tree + +This is the main source tree of mlkem-native. + +## Building + +To build a mlkem-native for a fixed parameter set (ML-KEM-512/768/1024), build the compilation in units in `src/*` separately, and link to an RNG and your application. See [examples/basic](../examples/basic) for a simple example. + +Alternatively, you can use the auto-geneated helper files [mlkem_native.c](mlkem_native.c) and [mlkem_native_asm.S](mlkem_native_asm.S), which bundle all *.c and *.S files together. See [examples/monolithic_build](../examples/monolithic_build) and [examples/monolithic_build_native](../examples/monolithic_build_native) for examples with and without native code. + +## Configuration + +The build is configured by [mlkem_native_config.h](mlkem_native_config.h), or by the file pointed to by `MLK_CONFIG_FILE`. Note in particular `MLK_CONFIG_PARAMETER_SET` and `MLK_CONFIG_NAMESPACE_PREFIX`, which set the parameter set and namespace prefix, respectively. + +## API + +The public API is defined in [mlkem_native.h](mlkem_native.h). + +## Supporting multiple parameter sets + +If you want to support multiple parameter sets, build the library once per parameter set you want to support. Set `MLK_CONFIG_MULTILEVEL_WITH_SHARED` for one of the builds, and `MLK_CONFIG_MULTILEVEL_NO_SHARED` for the others, to avoid duplicating shared functionality. Finally, link with RNG and your application as before. This is demonstrated in the examples [examples/multilevel_build](../examples/multilevel_build), [examples/multilevel_build_native](../examples/multilevel_build_native), [examples/monolithic_build_multilevel](../examples/monolithic_build_multilevel) and [examples/monolithic_build_multilevel_native](../examples/monolithic_build_multilevel_native). diff --git a/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.c b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.c new file mode 100644 index 0000000..e2c49cf --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.c @@ -0,0 +1,699 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +/****************************************************************************** + * + * Single compilation unit (SCU) for fixed-level build of mlkem-native + * + * This compilation unit bundles together all source files for a build + * of mlkem-native for a fixed security level (MLKEM-512/768/1024). + * + * # API + * + * The API exposed by this file is described in mlkem_native.h. + * + * # Multi-level build + * + * If you want an SCU build of mlkem-native with support for multiple security + * levels, you need to include this file multiple times, and set + * MLK_CONFIG_MULTILEVEL_WITH_SHARED and MLK_CONFIG_MULTILEVEL_NO_SHARED + * appropriately. This is exemplified in examples/monolithic_build_multilevel + * and examples/monolithic_build_multilevel_native. + * + * # Configuration + * + * The following options from the mlkem-native configuration are relevant: + * + * - MLK_CONFIG_FIPS202_CUSTOM_HEADER + * Set this option if you use a custom FIPS202 implementation. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * Set this option if you want to include the native arithmetic backends + * in your build. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * Set this option if you want to include the native FIPS202 backends + * in your build. + * + * - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * Set this option if you want to keep the directives defined in + * level-independent headers. This is needed for a multi-level build. + */ + +/* If parts of the mlkem-native source tree are not used, + * consider reducing this header via `unifdef`. + * + * Example: + * ```bash + * unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native.c + * ``` + */ + +#include "src/common.h" + +#include "src/compress.c" +#include "src/debug.c" +#include "src/indcpa.c" +#include "src/kem.c" +#include "src/poly.c" +#include "src/poly_k.c" +#include "src/sampling.c" +#include "src/verify.c" + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +#include "src/fips202/fips202.c" +#include "src/fips202/fips202x4.c" +#include "src/fips202/keccakf1600.c" +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#if defined(MLK_SYS_AARCH64) +#include "src/native/aarch64/src/aarch64_zetas.c" +#include "src/native/aarch64/src/rej_uniform_table.c" +#endif +#if defined(MLK_SYS_X86_64) +#include "src/native/x86_64/src/compress_consts.c" +#include "src/native/x86_64/src/consts.c" +#include "src/native/x86_64/src/rej_uniform_table.c" +#endif +#if defined(MLK_SYS_RISCV64) +#include "src/native/riscv64/src/rv64v_debug.c" +#include "src/native/riscv64/src/rv64v_poly.c" +#endif +#if defined(MLK_SYS_PPC64LE) +#include "src/native/ppc64le/src/consts.c" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#if defined(MLK_SYS_AARCH64) +#include "src/fips202/native/aarch64/src/keccakf1600_round_constants.c" +#endif +#if defined(MLK_SYS_X86_64) +#include "src/fips202/native/x86_64/src/keccakf1600_constants.c" +#endif +#if defined(MLK_SYS_ARMV81M_MVE) +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c" +#include "src/fips202/native/armv81m/src/keccakf1600_round_constants.c" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + +/* Macro #undef's + * + * The following undefines macros from headers + * included by the source files imported above. + * + * This is to allow building and linking multiple builds + * of mlkem-native for varying parameter sets through concatenation + * of this file, as if the files had been compiled separately. + * If this is not relevant to you, you may remove the following. + */ + +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files + */ +/* mlkem/mlkem_native.h */ +#undef CRYPTO_BYTES +#undef CRYPTO_CIPHERTEXTBYTES +#undef CRYPTO_PUBLICKEYBYTES +#undef CRYPTO_SECRETKEYBYTES +#undef CRYPTO_SYMBYTES +#undef MLKEM1024_BYTES +#undef MLKEM1024_CIPHERTEXTBYTES +#undef MLKEM1024_PUBLICKEYBYTES +#undef MLKEM1024_SECRETKEYBYTES +#undef MLKEM1024_SYMBYTES +#undef MLKEM512_BYTES +#undef MLKEM512_CIPHERTEXTBYTES +#undef MLKEM512_PUBLICKEYBYTES +#undef MLKEM512_SECRETKEYBYTES +#undef MLKEM512_SYMBYTES +#undef MLKEM768_BYTES +#undef MLKEM768_CIPHERTEXTBYTES +#undef MLKEM768_PUBLICKEYBYTES +#undef MLKEM768_SECRETKEYBYTES +#undef MLKEM768_SYMBYTES +#undef MLKEM_BYTES +#undef MLKEM_CIPHERTEXTBYTES +#undef MLKEM_CIPHERTEXTBYTES_ +#undef MLKEM_PUBLICKEYBYTES +#undef MLKEM_PUBLICKEYBYTES_ +#undef MLKEM_SECRETKEYBYTES +#undef MLKEM_SECRETKEYBYTES_ +#undef MLKEM_SYMBYTES +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_LEGACY_CONFIG +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_NAMESPACE +#undef MLK_API_QUALIFIER +#undef MLK_CONFIG_API_CONSTANTS_ONLY +#undef MLK_CONFIG_API_NAMESPACE_PREFIX +#undef MLK_CONFIG_API_NO_SUPERCOP +#undef MLK_CONFIG_API_PARAMETER_SET +#undef MLK_CONFIG_API_QUALIFIER +#undef MLK_ERR_FAIL +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_RNG_FAIL +#undef MLK_H +#undef MLK_MAX3_ +#undef MLK_TOTAL_ALLOC_1024 +#undef MLK_TOTAL_ALLOC_1024_DECAPS +#undef MLK_TOTAL_ALLOC_1024_ENCAPS +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_512 +#undef MLK_TOTAL_ALLOC_512_DECAPS +#undef MLK_TOTAL_ALLOC_512_ENCAPS +#undef MLK_TOTAL_ALLOC_512_KEYPAIR +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_768 +#undef MLK_TOTAL_ALLOC_768_DECAPS +#undef MLK_TOTAL_ALLOC_768_ENCAPS +#undef MLK_TOTAL_ALLOC_768_KEYPAIR +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +#undef crypto_kem_check_pk +#undef crypto_kem_check_sk +#undef crypto_kem_dec +#undef crypto_kem_enc +#undef crypto_kem_enc_derand +#undef crypto_kem_keypair +#undef crypto_kem_keypair_derand +/* mlkem/src/common.h */ +#undef MLK_ADD_PARAM_SET +#undef MLK_ALLOC +#undef MLK_APPLY +#undef MLK_ASM_FN_SIZE +#undef MLK_ASM_FN_SYMBOL +#undef MLK_ASM_NAMESPACE +#undef MLK_BUILD_INTERNAL +#undef MLK_COMMON_H +#undef MLK_CONCAT +#undef MLK_CONCAT_ +#undef MLK_CONTEXT_PARAMETERS_0 +#undef MLK_CONTEXT_PARAMETERS_1 +#undef MLK_CONTEXT_PARAMETERS_2 +#undef MLK_CONTEXT_PARAMETERS_3 +#undef MLK_CONTEXT_PARAMETERS_4 +#undef MLK_EMPTY_CU +#undef MLK_ERR_FAIL +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_RNG_FAIL +#undef MLK_EXTERNAL_API +#undef MLK_FIPS202X4_HEADER_FILE +#undef MLK_FIPS202_HEADER_FILE +#undef MLK_FREE +#undef MLK_INTERNAL_API +#undef MLK_INTERNAL_DATA_DECLARATION +#undef MLK_INTERNAL_DATA_DEFINITION +#undef MLK_NAMESPACE +#undef MLK_NAMESPACE_K +#undef MLK_NAMESPACE_PREFIX +#undef MLK_NAMESPACE_PREFIX_K +#undef mlk_memcpy +#undef mlk_memset +/* mlkem/src/indcpa.h */ +#undef MLK_INDCPA_H +#undef mlk_gen_matrix +#undef mlk_indcpa_dec +#undef mlk_indcpa_enc +#undef mlk_indcpa_keypair_derand +/* mlkem/src/kem.h */ +#undef MLK_KEM_H +#undef mlk_kem_check_pk +#undef mlk_kem_check_sk +#undef mlk_kem_dec +#undef mlk_kem_enc +#undef mlk_kem_enc_derand +#undef mlk_kem_keypair +#undef mlk_kem_keypair_derand +/* mlkem/src/params.h */ +#undef MLKEM_DU +#undef MLKEM_DV +#undef MLKEM_ETA1 +#undef MLKEM_ETA2 +#undef MLKEM_INDCCA_CIPHERTEXTBYTES +#undef MLKEM_INDCCA_PUBLICKEYBYTES +#undef MLKEM_INDCCA_SECRETKEYBYTES +#undef MLKEM_INDCPA_BYTES +#undef MLKEM_INDCPA_MSGBYTES +#undef MLKEM_INDCPA_PUBLICKEYBYTES +#undef MLKEM_INDCPA_SECRETKEYBYTES +#undef MLKEM_K +#undef MLKEM_N +#undef MLKEM_POLYBYTES +#undef MLKEM_POLYCOMPRESSEDBYTES_D10 +#undef MLKEM_POLYCOMPRESSEDBYTES_D11 +#undef MLKEM_POLYCOMPRESSEDBYTES_D4 +#undef MLKEM_POLYCOMPRESSEDBYTES_D5 +#undef MLKEM_POLYCOMPRESSEDBYTES_DU +#undef MLKEM_POLYCOMPRESSEDBYTES_DV +#undef MLKEM_POLYVECBYTES +#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU +#undef MLKEM_Q +#undef MLKEM_Q_HALF +#undef MLKEM_SSBYTES +#undef MLKEM_SYMBYTES +#undef MLKEM_UINT12_LIMIT +#undef MLK_PARAMS_H +/* mlkem/src/poly_k.h */ +#undef MLK_POLY_K_H +#undef mlk_poly_compress_du +#undef mlk_poly_compress_dv +#undef mlk_poly_decompress_du +#undef mlk_poly_decompress_dv +#undef mlk_poly_getnoise_eta1122_4x +#undef mlk_poly_getnoise_eta1_4x +#undef mlk_poly_getnoise_eta2 +#undef mlk_poly_getnoise_eta2_4x +#undef mlk_polymat +#undef mlk_polyvec +#undef mlk_polyvec_add +#undef mlk_polyvec_basemul_acc_montgomery_cached +#undef mlk_polyvec_compress_du +#undef mlk_polyvec_decompress_du +#undef mlk_polyvec_frombytes +#undef mlk_polyvec_invntt_tomont +#undef mlk_polyvec_mulcache +#undef mlk_polyvec_mulcache_compute +#undef mlk_polyvec_ntt +#undef mlk_polyvec_reduce +#undef mlk_polyvec_tobytes +#undef mlk_polyvec_tomont + +#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS) +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files + */ +/* mlkem/src/compress.h */ +#undef MLK_COMPRESS_H +#undef mlk_poly_compress_d10 +#undef mlk_poly_compress_d11 +#undef mlk_poly_compress_d4 +#undef mlk_poly_compress_d5 +#undef mlk_poly_decompress_d10 +#undef mlk_poly_decompress_d11 +#undef mlk_poly_decompress_d4 +#undef mlk_poly_decompress_d5 +#undef mlk_poly_frombytes +#undef mlk_poly_frommsg +#undef mlk_poly_tobytes +#undef mlk_poly_tomsg +/* mlkem/src/debug.h */ +#undef MLK_DEBUG_H +#undef mlk_assert +#undef mlk_assert_abs_bound +#undef mlk_assert_abs_bound_2d +#undef mlk_assert_bound +#undef mlk_assert_bound_2d +#undef mlk_debug_check_assert +#undef mlk_debug_check_bounds +/* mlkem/src/poly.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NTT_BOUND +#undef MLK_POLY_H +#undef mlk_poly_add +#undef mlk_poly_invntt_tomont +#undef mlk_poly_mulcache_compute +#undef mlk_poly_ntt +#undef mlk_poly_reduce +#undef mlk_poly_sub +#undef mlk_poly_tomont +/* mlkem/src/randombytes.h */ +#undef MLK_RANDOMBYTES_H +/* mlkem/src/sampling.h */ +#undef MLK_SAMPLING_H +#undef mlk_poly_cbd2 +#undef mlk_poly_cbd3 +#undef mlk_poly_rej_uniform +#undef mlk_poly_rej_uniform_x4 +/* mlkem/src/symmetric.h */ +#undef MLK_SYMMETRIC_H +#undef MLK_XOF_RATE +#undef mlk_hash_g +#undef mlk_hash_h +#undef mlk_hash_j +#undef mlk_prf_eta +#undef mlk_prf_eta1 +#undef mlk_prf_eta1_x4 +#undef mlk_prf_eta2 +#undef mlk_xof_absorb +#undef mlk_xof_ctx +#undef mlk_xof_init +#undef mlk_xof_release +#undef mlk_xof_squeezeblocks +#undef mlk_xof_x4_absorb +#undef mlk_xof_x4_ctx +#undef mlk_xof_x4_init +#undef mlk_xof_x4_release +#undef mlk_xof_x4_squeezeblocks +/* mlkem/src/sys.h */ +#undef MLK_ALIGN +#undef MLK_ALIGN_UP +#undef MLK_ALWAYS_INLINE +#undef MLK_CET_ENDBR +#undef MLK_CT_TESTING_DECLASSIFY +#undef MLK_CT_TESTING_SECRET +#undef MLK_DEFAULT_ALIGN +#undef MLK_HAVE_INLINE_ASM +#undef MLK_INLINE +#undef MLK_MUST_CHECK_RETURN_VALUE +#undef MLK_NOINLINE +#undef MLK_RESTRICT +#undef MLK_STATIC_TESTABLE +#undef MLK_SYSV_ABI +#undef MLK_SYSV_ABI_SUPPORTED +#undef MLK_SYS_AARCH64 +#undef MLK_SYS_AARCH64_EB +#undef MLK_SYS_APPLE +#undef MLK_SYS_ARMV81M_MVE +#undef MLK_SYS_BIG_ENDIAN +#undef MLK_SYS_H +#undef MLK_SYS_LINUX +#undef MLK_SYS_LITTLE_ENDIAN +#undef MLK_SYS_PPC64LE +#undef MLK_SYS_RISCV32 +#undef MLK_SYS_RISCV64 +#undef MLK_SYS_RISCV64_RVV +#undef MLK_SYS_WINDOWS +#undef MLK_SYS_X86_64 +#undef MLK_SYS_X86_64_AVX2 +/* mlkem/src/verify.h */ +#undef MLK_USE_ASM_VALUE_BARRIER +#undef MLK_VERIFY_H +#undef mlk_ct_opt_blocker_u64 +/* mlkem/src/cbmc.h */ +#undef MLK_CBMC_H +#undef __contract__ +#undef __loop__ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +/* + * Undefine macros from FIPS-202 files + */ +/* mlkem/src/fips202/fips202.h */ +#undef FIPS202_X4_DEFAULT_IMPLEMENTATION +#undef MLK_FIPS202_FIPS202_H +#undef SHA3_256_HASHBYTES +#undef SHA3_256_RATE +#undef SHA3_384_RATE +#undef SHA3_512_HASHBYTES +#undef SHA3_512_RATE +#undef SHAKE128_RATE +#undef SHAKE256_RATE +#undef mlk_sha3_256 +#undef mlk_sha3_512 +#undef mlk_shake128_absorb_once +#undef mlk_shake128_init +#undef mlk_shake128_release +#undef mlk_shake128_squeezeblocks +#undef mlk_shake256 +/* mlkem/src/fips202/fips202x4.h */ +#undef MLK_FIPS202_FIPS202X4_H +#undef mlk_shake128x4_absorb_once +#undef mlk_shake128x4_init +#undef mlk_shake128x4_release +#undef mlk_shake128x4_squeezeblocks +#undef mlk_shake256x4 +/* mlkem/src/fips202/keccakf1600.h */ +#undef MLK_FIPS202_KECCAKF1600_H +#undef MLK_KECCAK_LANES +#undef MLK_KECCAK_WAY +#undef mlk_keccakf1600_extract_bytes +#undef mlk_keccakf1600_permute +#undef mlk_keccakf1600_xor_bytes +#undef mlk_keccakf1600x4_extract_bytes +#undef mlk_keccakf1600x4_permute +#undef mlk_keccakf1600x4_xor_bytes +#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* mlkem/src/fips202/native/api.h */ +#undef MLK_FIPS202_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +/* mlkem/src/fips202/native/auto.h */ +#undef MLK_FIPS202_NATIVE_AUTO_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (FIPS202, AArch64) + */ +/* mlkem/src/fips202/native/aarch64/auto.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#undef mlk_keccak_f1600_x1_scalar_aarch64_asm +#undef mlk_keccak_f1600_x1_v84a_aarch64_asm +#undef mlk_keccak_f1600_x2_v84a_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm +#undef mlk_keccakf1600_round_constants +/* mlkem/src/fips202/native/aarch64/x1_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#undef MLK_USE_FIPS202_X1_NATIVE +/* mlkem/src/fips202/native/aarch64/x1_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#undef MLK_USE_FIPS202_X1_NATIVE +/* mlkem/src/fips202/native/aarch64/x2_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X2_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#undef MLK_USE_FIPS202_X4_NATIVE +/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#undef MLK_USE_FIPS202_X4_NATIVE +/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#undef MLK_USE_FIPS202_X4_NATIVE +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (FIPS202, x86_64) + */ +/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */ +#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#undef MLK_FIPS202_X86_64_NEED_X4_AVX2 +#undef MLK_USE_FIPS202_X4_NATIVE +/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */ +#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#undef mlk_keccak_f1600_x4_avx2_asm +#undef mlk_keccak_rho56 +#undef mlk_keccak_rho8 +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_ARMV81M_MVE) +/* + * Undefine macros from native code (FIPS202, Armv8.1-M) + */ +/* mlkem/src/fips202/native/armv81m/mve.h */ +#undef MLK_FIPS202_ARMV81M_NEED_X4 +#undef MLK_FIPS202_NATIVE_ARMV81M +#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#undef MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE +#undef MLK_USE_FIPS202_X4_NATIVE +#undef MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE +#undef mlk_keccak_f1600_x4_native_impl +/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */ +#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#undef mlk_keccak_f1600_x4_mve_asm +#undef mlk_keccak_f1600_x4_state_extract_bytes_asm +#undef mlk_keccak_f1600_x4_state_xor_bytes_asm +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_ARMV81M_MVE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* mlkem/src/native/api.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +#undef MLK_NTT_BOUND +/* mlkem/src/native/meta.h */ +#undef MLK_NATIVE_META_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (Arith, AArch64) + */ +/* mlkem/src/native/aarch64/meta.h */ +#undef MLK_ARITH_BACKEND_AARCH64 +#undef MLK_NATIVE_AARCH64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */ +#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#undef mlk_aarch64_invntt_zetas_layer12345 +#undef mlk_aarch64_invntt_zetas_layer67 +#undef mlk_aarch64_ntt_zetas_layer12345 +#undef mlk_aarch64_ntt_zetas_layer67 +#undef mlk_aarch64_zetas_mulcache_native +#undef mlk_aarch64_zetas_mulcache_twisted_native +#undef mlk_intt_aarch64_asm +#undef mlk_ntt_aarch64_asm +#undef mlk_poly_mulcache_compute_aarch64_asm +#undef mlk_poly_reduce_aarch64_asm +#undef mlk_poly_tobytes_aarch64_asm +#undef mlk_poly_tomont_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm +#undef mlk_rej_uniform_aarch64_asm +#undef mlk_rej_uniform_table +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (Arith, X86_64) + */ +/* mlkem/src/native/x86_64/meta.h */ +#undef MLK_ARITH_BACKEND_X86_64_DEFAULT +#undef MLK_NATIVE_X86_64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_COMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_FROMBYTES +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */ +#undef MLK_AVX2_REJ_UNIFORM_BUFLEN +#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#undef mlk_invntt_avx2_asm +#undef mlk_ntt_avx2_asm +#undef mlk_nttfrombytes_avx2_asm +#undef mlk_ntttobytes_avx2_asm +#undef mlk_nttunpack_avx2_asm +#undef mlk_poly_compress_d10_avx2_asm +#undef mlk_poly_compress_d11_avx2_asm +#undef mlk_poly_compress_d4_avx2_asm +#undef mlk_poly_compress_d5_avx2_asm +#undef mlk_poly_decompress_d10_avx2_asm +#undef mlk_poly_decompress_d11_avx2_asm +#undef mlk_poly_decompress_d4_avx2_asm +#undef mlk_poly_decompress_d5_avx2_asm +#undef mlk_poly_mulcache_compute_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm +#undef mlk_reduce_avx2_asm +#undef mlk_rej_uniform_avx2_asm +#undef mlk_rej_uniform_table +#undef mlk_tomont_avx2_asm +/* mlkem/src/native/x86_64/src/compress_consts.h */ +#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#undef mlk_compress_d10_data +#undef mlk_compress_d11_data +#undef mlk_compress_d4_data +#undef mlk_compress_d5_data +#undef mlk_decompress_d10_data +#undef mlk_decompress_d11_data +#undef mlk_decompress_d4_data +#undef mlk_decompress_d5_data +/* mlkem/src/native/x86_64/src/consts.h */ +#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD +#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP +#undef MLK_NATIVE_X86_64_SRC_CONSTS_H +#undef mlk_qdata +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mlkem/src/native/riscv64/meta.h */ +#undef MLK_ARITH_BACKEND_RISCV64 +#undef MLK_NATIVE_RISCV64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mlk_rv64v_poly_add +#undef mlk_rv64v_poly_basemul_mont_add_k2 +#undef mlk_rv64v_poly_basemul_mont_add_k3 +#undef mlk_rv64v_poly_basemul_mont_add_k4 +#undef mlk_rv64v_poly_invntt_tomont +#undef mlk_rv64v_poly_ntt +#undef mlk_rv64v_poly_reduce +#undef mlk_rv64v_poly_sub +#undef mlk_rv64v_poly_tomont +#undef mlk_rv64v_rej_uniform +/* mlkem/src/native/riscv64/src/rv64v_debug.h */ +#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#undef mlk_assert_abs_bound_int16m1 +#undef mlk_assert_abs_bound_int16m2 +#undef mlk_assert_bound_int16m1 +#undef mlk_assert_bound_int16m2 +#undef mlk_debug_check_bounds_int16m1 +#undef mlk_debug_check_bounds_int16m2 +#endif /* MLK_SYS_RISCV64 */ +#if defined(MLK_SYS_PPC64LE) +/* + * Undefine macros from native code (Arith, PPC64LE) + */ +/* mlkem/src/native/ppc64le/meta.h */ +#undef MLK_ARITH_BACKEND_NAME +#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT +#undef MLK_NATIVE_PPC64LE_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */ +#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#undef mlk_intt_ppc_asm +#undef mlk_ntt_ppc_asm +#undef mlk_poly_tomont_ppc_asm +#undef mlk_reduce_ppc_asm +/* mlkem/src/native/ppc64le/src/consts.h */ +#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#undef MLK_PPC_C20159_OFFSET +#undef MLK_PPC_NQ_OFFSET +#undef MLK_PPC_N_INV_OFFSET +#undef MLK_PPC_N_INV_TW_OFFSET +#undef MLK_PPC_Q_OFFSET +#undef MLK_PPC_TOMONT_OFFSET +#undef MLK_PPC_TOMONT_TW_OFFSET +#undef MLK_PPC_ZETA_INTT_OFFSET +#undef MLK_PPC_ZETA_INTT_TW_OFFSET +#undef MLK_PPC_ZETA_NTT_OFFSET +#undef MLK_PPC_ZETA_NTT_TW_OFFSET +#undef mlk_ppc_qdata +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.h b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.h new file mode 100644 index 0000000..9bbb3e2 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.h @@ -0,0 +1,527 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_H +#define MLK_H + +/* + * Public API for mlkem-native. + * + * This header defines the public API of a single build of mlkem-native. + * + * Make sure the configuration file is in the include path + * (this is "mlkem_native_config.h" by default, or MLK_CONFIG_FILE if defined). + * + * # Multi-level builds + * + * This header specifies a build of mlkem-native for a fixed security level. + * If you need multiple security levels, leave the security level unspecified + * in the configuration file and include this header multiple times, setting + * MLK_CONFIG_PARAMETER_SET accordingly for each, and #undef'ing the MLK_H + * guard to allow multiple inclusions. + * + * # Legacy configuration (deprecated) + * + * Instead of providing the config file used for the build, you can + * alternatively set the following configuration options prior to + * including this header. + * + * This method of configuration is deprecated. + * It will be removed in mlkem-native-v2. + * + * - MLK_CONFIG_API_PARAMETER_SET [required] + * + * The parameter set used for the build; 512, 768, or 1024. + * + * - MLK_CONFIG_API_NAMESPACE_PREFIX [required] + * + * The namespace prefix used for the build. + * + * NOTE: + * For a multi-level build, you must include the 512/768/1024 suffixes + * in MLK_CONFIG_API_NAMESPACE_PREFIX. + * + * - MLK_CONFIG_API_NO_SUPERCOP [optional] + * + * By default, this header will also expose the mlkem-native API in the + * SUPERCOP naming convention crypto_kem_xxx. If you don't want/need this, + * set MLK_CONFIG_API_NO_SUPERCOP. You must set this for a multi-level build. + * + * - MLK_CONFIG_API_CONSTANTS_ONLY [optional] + * + * If you don't want this header to expose any function declarations, + * but only constants for the sizes of key material, set + * MLK_CONFIG_API_CONSTANTS_ONLY. In this case, you don't need to set + * MLK_CONFIG_API_PARAMETER_SET or MLK_CONFIG_API_NAMESPACE_PREFIX, + * nor include a configuration. + * + * - MLK_CONFIG_API_QUALIFIER [optional] + * + * Qualifier to apply to external API. + * + ******************************************************************************/ + +/******************************* Key sizes ************************************/ + +/* Sizes of cryptographic material, per parameter set */ +/* See mlkem/common.h for the arithmetic expressions giving rise to these */ +/* check-magic: off */ +#define MLKEM512_SECRETKEYBYTES 1632 +#define MLKEM512_PUBLICKEYBYTES 800 +#define MLKEM512_CIPHERTEXTBYTES 768 + +#define MLKEM768_SECRETKEYBYTES 2400 +#define MLKEM768_PUBLICKEYBYTES 1184 +#define MLKEM768_CIPHERTEXTBYTES 1088 + +#define MLKEM1024_SECRETKEYBYTES 3168 +#define MLKEM1024_PUBLICKEYBYTES 1568 +#define MLKEM1024_CIPHERTEXTBYTES 1568 +/* check-magic: on */ + +/* Size of randomness coins in bytes (level-independent) */ +#define MLKEM_SYMBYTES 32 +#define MLKEM512_SYMBYTES MLKEM_SYMBYTES +#define MLKEM768_SYMBYTES MLKEM_SYMBYTES +#define MLKEM1024_SYMBYTES MLKEM_SYMBYTES +/* Size of shared secret in bytes (level-independent) */ +#define MLKEM_BYTES 32 +#define MLKEM512_BYTES MLKEM_BYTES +#define MLKEM768_BYTES MLKEM_BYTES +#define MLKEM1024_BYTES MLKEM_BYTES + +/* Sizes of cryptographic material, as a function of LVL=512,768,1024 */ +#define MLKEM_SECRETKEYBYTES_(LVL) MLKEM##LVL##_SECRETKEYBYTES +#define MLKEM_PUBLICKEYBYTES_(LVL) MLKEM##LVL##_PUBLICKEYBYTES +#define MLKEM_CIPHERTEXTBYTES_(LVL) MLKEM##LVL##_CIPHERTEXTBYTES +#define MLKEM_SECRETKEYBYTES(LVL) MLKEM_SECRETKEYBYTES_(LVL) +#define MLKEM_PUBLICKEYBYTES(LVL) MLKEM_PUBLICKEYBYTES_(LVL) +#define MLKEM_CIPHERTEXTBYTES(LVL) MLKEM_CIPHERTEXTBYTES_(LVL) + +/****************************** Error codes ***********************************/ + +/* Generic failure condition */ +#define MLK_ERR_FAIL (-1) +/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is defined and the provided MLK_CUSTOM_ALLOC can fail. */ +#define MLK_ERR_OUT_OF_MEMORY (-2) +/* An rng failure occured. Might be due to insufficient entropy or + * system misconfiguration. */ +#define MLK_ERR_RNG_FAIL (-3) + +/****************************** Function API **********************************/ + +#define MLK_API_CONCAT_(x, y) x##y +#define MLK_API_CONCAT(x, y) MLK_API_CONCAT_(x, y) +#define MLK_API_CONCAT_UNDERSCORE(x, y) MLK_API_CONCAT(MLK_API_CONCAT(x, _), y) + +#if !defined(MLK_CONFIG_API_PARAMETER_SET) +/* Recommended configuration via same config file as used for the build. */ + +/* For now, we derive the legacy API configuration MLK_CONFIG_API_XXX from + * the config file. In mlkem-native-v2, this will be removed and we will + * exclusively work with MLK_CONFIG_XXX. */ + +/* You need to make sure the config file is in the include path. */ +#if defined(MLK_CONFIG_FILE) +#include MLK_CONFIG_FILE +#else +#include "mlkem_native_config.h" +#endif + +#define MLK_CONFIG_API_PARAMETER_SET MLK_CONFIG_PARAMETER_SET + +#if defined(MLK_CONFIG_MULTILEVEL_BUILD) +#define MLK_CONFIG_API_NAMESPACE_PREFIX \ + MLK_API_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, MLK_CONFIG_PARAMETER_SET) +#else +#define MLK_CONFIG_API_NAMESPACE_PREFIX MLK_CONFIG_NAMESPACE_PREFIX +#endif + +#if defined(MLK_CONFIG_NO_SUPERCOP) +#define MLK_CONFIG_API_NO_SUPERCOP +#endif + +#if defined(MLK_CONFIG_CONSTANTS_ONLY) +#define MLK_CONFIG_API_CONSTANTS_ONLY +#endif + +#if defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLK_CONFIG_API_QUALIFIER MLK_CONFIG_EXTERNAL_API_QUALIFIER +#endif + +#else /* !MLK_CONFIG_API_PARAMETER_SET */ + +#define MLK_API_LEGACY_CONFIG + +#endif /* MLK_CONFIG_API_PARAMETER_SET */ + +#define MLK_API_NAMESPACE(sym) \ + MLK_API_CONCAT_UNDERSCORE(MLK_CONFIG_API_NAMESPACE_PREFIX, sym) + +#if defined(__GNUC__) || defined(__clang__) +#define MLK_API_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) +#else +#define MLK_API_MUST_CHECK_RETURN_VALUE +#endif + +#if defined(MLK_CONFIG_API_QUALIFIER) +#define MLK_API_QUALIFIER MLK_CONFIG_API_QUALIFIER +#else +#define MLK_API_QUALIFIER +#endif + +#if !defined(MLK_CONFIG_API_CONSTANTS_ONLY) + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].} + * + * @param[out] pk Output public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[out] sk Output private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param[in] coins Input randomness, an array of 2*MLKEM_SYMBYTES uniformly + * random bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(keypair_derand)( + uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)], + uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)], + const uint8_t coins[2 * MLKEM_SYMBYTES] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].} + * + * @param[out] pk Output public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[out] sk Output private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(keypair)( + uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)], + uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ + +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].} + * + * @param[out] ct Output ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[in] coins Input randomness, an array of MLKEM_SYMBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(enc_derand)( + uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)], + uint8_t ss[MLKEM_BYTES], + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)], + const uint8_t coins[MLKEM_SYMBYTES] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].} + * + * @param[out] ct Output ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(enc)( + uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)], + uint8_t ss[MLKEM_BYTES], + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ + +/** + * Generate shared secret for a given ciphertext and private key. + * + * @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].} + * + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] ct Input ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[in] sk Input private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL The 'hash check' @[FIPS203, Section 7.3] + * for the secret key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(dec)( + uint8_t ss[MLKEM_BYTES], + const uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)], + const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + + +/** + * Implements modulus check mandated by FIPS 203, i.e., ensures that + * coefficients are in [0,q-1]. + * + * @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].} + * + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL Modulus check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(check_pk)( + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + +/** + * Implements public key hash check mandated by FIPS 203, i.e., ensures that + * sk[768𝑘+32 ∶ 768𝑘+64] = H(pk) = H(sk[384𝑘 : 768𝑘+32]). + * + * @spec{Implements @[FIPS203, Section 7.3, 'hash check'].} + * + * @param[in] sk Input private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL Public key hash check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(check_sk)( + const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + +#ifdef __cplusplus +} +#endif + +/****************************** SUPERCOP API *********************************/ + +#if !defined(MLK_CONFIG_API_NO_SUPERCOP) +/* Export API in SUPERCOP naming scheme CRYPTO_xxx / crypto_kem_xxx */ +#define CRYPTO_SECRETKEYBYTES MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET) +#define CRYPTO_PUBLICKEYBYTES MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET) +#define CRYPTO_CIPHERTEXTBYTES \ + MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET) +#define CRYPTO_SYMBYTES MLKEM_SYMBYTES +#define CRYPTO_BYTES MLKEM_BYTES + +#define crypto_kem_keypair_derand MLK_API_NAMESPACE(keypair_derand) +#define crypto_kem_keypair MLK_API_NAMESPACE(keypair) +#define crypto_kem_enc_derand MLK_API_NAMESPACE(enc_derand) +#define crypto_kem_enc MLK_API_NAMESPACE(enc) +#define crypto_kem_dec MLK_API_NAMESPACE(dec) +#define crypto_kem_check_pk MLK_API_NAMESPACE(check_pk) +#define crypto_kem_check_sk MLK_API_NAMESPACE(check_sk) + +#else /* !MLK_CONFIG_API_NO_SUPERCOP */ + +/* If the SUPERCOP API is not needed, we can undefine the various helper macros + * above. Otherwise, they are needed for lazy evaluation of crypto_kem_xxx. */ +#if !defined(MLK_API_LEGACY_CONFIG) +#undef MLK_CONFIG_API_PARAMETER_SET +#undef MLK_CONFIG_API_NAMESPACE_PREFIX +#undef MLK_CONFIG_API_NO_SUPERCOP +#undef MLK_CONFIG_API_CONSTANTS_ONLY +#undef MLK_CONFIG_API_QUALIFIER +#endif /* !MLK_API_LEGACY_CONFIG */ + +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_NAMESPACE +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_QUALIFIER + +#endif /* MLK_CONFIG_API_NO_SUPERCOP */ +#endif /* !MLK_CONFIG_API_CONSTANTS_ONLY */ + + +/***************************** Memory Usage **********************************/ + +/* + * By default mlkem-native performs all memory allocations on the stack. + * Alternatively, mlkem-native supports custom allocation of large structures + * through the `MLK_CONFIG_CUSTOM_ALLOC_FREE` configuration option. + * See mlkem_native_config.h for details. + * + * `MLK_TOTAL_ALLOC_{512,768,1024}_{KEYPAIR,ENCAPS,DECAPS}` indicates the + * maximum (accumulative) allocation via MLK_ALLOC for each parameter set and + * operation. Note that some stack allocation remains even when using custom + * allocators, so these values are lower than total stack usage with the default + * stack-only allocation. + * + * These constants may be used to implement custom allocations using a + * fixed-sized buffer and a simple allocator (e.g., bump allocator). + */ +/* check-magic: off */ +#define MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT 5824 +#define MLK_TOTAL_ALLOC_512_KEYPAIR_PCT 10048 +#define MLK_TOTAL_ALLOC_512_ENCAPS 8384 +#define MLK_TOTAL_ALLOC_512_DECAPS 9152 +#define MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT 10176 +#define MLK_TOTAL_ALLOC_768_KEYPAIR_PCT 15552 +#define MLK_TOTAL_ALLOC_768_ENCAPS 13248 +#define MLK_TOTAL_ALLOC_768_DECAPS 14336 +#define MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT 15552 +#define MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT 22400 +#define MLK_TOTAL_ALLOC_1024_ENCAPS 19136 +#define MLK_TOTAL_ALLOC_1024_DECAPS 20704 +/* check-magic: on */ + +/* + * MLK_TOTAL_ALLOC_*_KEYPAIR adapts based on MLK_CONFIG_KEYGEN_PCT. + * For legacy config, we don't know which options are used, so assume + * the worst case (PCT enabled). + */ +#if defined(MLK_API_LEGACY_CONFIG) || defined(MLK_CONFIG_KEYGEN_PCT) +#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#else +#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#endif + +#define MLK_MAX3_(a, b, c) \ + ((a) > (b) ? ((a) > (c) ? (a) : (c)) : ((b) > (c) ? (b) : (c))) + +/* + * `MLK_TOTAL_ALLOC_{512,768,1024}` is the maximum across all operations for + * each parameter set. + */ +#define MLK_TOTAL_ALLOC_512 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_512_KEYPAIR, MLK_TOTAL_ALLOC_512_ENCAPS, \ + MLK_TOTAL_ALLOC_512_DECAPS) +#define MLK_TOTAL_ALLOC_768 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_768_KEYPAIR, MLK_TOTAL_ALLOC_768_ENCAPS, \ + MLK_TOTAL_ALLOC_768_DECAPS) +#define MLK_TOTAL_ALLOC_1024 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_1024_KEYPAIR, MLK_TOTAL_ALLOC_1024_ENCAPS, \ + MLK_TOTAL_ALLOC_1024_DECAPS) + +#undef MLK_API_LEGACY_CONFIG + +#endif /* !MLK_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_asm.S new file mode 100644 index 0000000..18adf0b --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_asm.S @@ -0,0 +1,723 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +/****************************************************************************** + * + * Single assembly unit for fixed-level build of mlkem-native + * + * This assembly unit bundles together all assembly files for a build + * of mlkem-native for a fixed security level (MLKEM-512/768/1024). + * + * # Multi-level build + * + * If you want an SCU build of mlkem-native with support for multiple security + * levels, you should include this file once with + * MLK_CONFIG_MULTILEVEL_WITH_SHARED set. + * + * (You could also follow the same pattern as for mlkem_native_monobuild.c + * and include it for every level, setting MLK_CONFIG_MULTILEVEL_NO_SHARED + * for all but one. For builds with MLK_CONFIG_MULTILEVEL_NO_SHARED, this + * file will then be ignored.) + * + * # Configuration + * + * The following options from the mlkem-native configuration are relevant: + * + * - MLK_CONFIG_FIPS202_CUSTOM_HEADER + * Set this option if you use a custom FIPS202 implementation. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * Set this option if you want to include the native arithmetic backends + * in your build. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * Set this option if you want to include the native FIPS202 backends + * in your build. + * + * - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * Set this option if you want to keep the directives defined in + * level-independent headers. This is needed for a multi-level build. + */ + +/* If parts of the mlkem-native source tree are not used, + * consider reducing this header via `unifdef`. + * + * Example: + * ```bash + * unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native_asm.S + * ``` + */ + +#include "src/common.h" + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#if defined(MLK_SYS_AARCH64) +#include "src/native/aarch64/src/intt_aarch64_asm.S" +#include "src/native/aarch64/src/ntt_aarch64_asm.S" +#include "src/native/aarch64/src/poly_mulcache_compute_aarch64_asm.S" +#include "src/native/aarch64/src/poly_reduce_aarch64_asm.S" +#include "src/native/aarch64/src/poly_tobytes_aarch64_asm.S" +#include "src/native/aarch64/src/poly_tomont_aarch64_asm.S" +#include "src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S" +#include "src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S" +#include "src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S" +#include "src/native/aarch64/src/rej_uniform_aarch64_asm.S" +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +#include "src/native/x86_64/src/intt_avx2_asm.S" +#include "src/native/x86_64/src/ntt_avx2_asm.S" +#include "src/native/x86_64/src/nttfrombytes_avx2_asm.S" +#include "src/native/x86_64/src/ntttobytes_avx2_asm.S" +#include "src/native/x86_64/src/nttunpack_avx2_asm.S" +#include "src/native/x86_64/src/poly_compress_d10_avx2_asm.S" +#include "src/native/x86_64/src/poly_compress_d11_avx2_asm.S" +#include "src/native/x86_64/src/poly_compress_d4_avx2_asm.S" +#include "src/native/x86_64/src/poly_compress_d5_avx2_asm.S" +#include "src/native/x86_64/src/poly_decompress_d10_avx2_asm.S" +#include "src/native/x86_64/src/poly_decompress_d11_avx2_asm.S" +#include "src/native/x86_64/src/poly_decompress_d4_avx2_asm.S" +#include "src/native/x86_64/src/poly_decompress_d5_avx2_asm.S" +#include "src/native/x86_64/src/poly_mulcache_compute_avx2_asm.S" +#include "src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S" +#include "src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S" +#include "src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S" +#include "src/native/x86_64/src/reduce_avx2_asm.S" +#include "src/native/x86_64/src/rej_uniform_avx2_asm.S" +#include "src/native/x86_64/src/tomont_avx2_asm.S" +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +#endif +#if defined(MLK_SYS_PPC64LE) +#include "src/native/ppc64le/src/intt_ppc_asm.S" +#include "src/native/ppc64le/src/ntt_ppc_asm.S" +#include "src/native/ppc64le/src/poly_tomont_ppc_asm.S" +#include "src/native/ppc64le/src/reduce_ppc_asm.S" +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#if defined(MLK_SYS_AARCH64) +#include "src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S" +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +#include "src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S" +#endif +#if defined(MLK_SYS_ARMV81M_MVE) +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S" +#include "src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S" +#include "src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + + +/* Macro #undef's + * + * The following undefines macros from headers + * included by the source files imported above. + * + * This is to allow building and linking multiple builds + * of mlkem-native for varying parameter sets through concatenation + * of this file, as if the files had been compiled separately. + * If this is not relevant to you, you may remove the following. + * + * NOTE: This is not needed for the assembly SCU since, at present, + * there is no need to include it multiple times. + * We keep it for uniformity with mlkem_native.c only. + * + * NOTE: To avoid having to distinguish between which headers are included + * from the assembly files, we #undef the same set of directives + * as in mlkem_native.c + */ + +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files + */ +/* mlkem/mlkem_native.h */ +#undef CRYPTO_BYTES +#undef CRYPTO_CIPHERTEXTBYTES +#undef CRYPTO_PUBLICKEYBYTES +#undef CRYPTO_SECRETKEYBYTES +#undef CRYPTO_SYMBYTES +#undef MLKEM1024_BYTES +#undef MLKEM1024_CIPHERTEXTBYTES +#undef MLKEM1024_PUBLICKEYBYTES +#undef MLKEM1024_SECRETKEYBYTES +#undef MLKEM1024_SYMBYTES +#undef MLKEM512_BYTES +#undef MLKEM512_CIPHERTEXTBYTES +#undef MLKEM512_PUBLICKEYBYTES +#undef MLKEM512_SECRETKEYBYTES +#undef MLKEM512_SYMBYTES +#undef MLKEM768_BYTES +#undef MLKEM768_CIPHERTEXTBYTES +#undef MLKEM768_PUBLICKEYBYTES +#undef MLKEM768_SECRETKEYBYTES +#undef MLKEM768_SYMBYTES +#undef MLKEM_BYTES +#undef MLKEM_CIPHERTEXTBYTES +#undef MLKEM_CIPHERTEXTBYTES_ +#undef MLKEM_PUBLICKEYBYTES +#undef MLKEM_PUBLICKEYBYTES_ +#undef MLKEM_SECRETKEYBYTES +#undef MLKEM_SECRETKEYBYTES_ +#undef MLKEM_SYMBYTES +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_LEGACY_CONFIG +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_NAMESPACE +#undef MLK_API_QUALIFIER +#undef MLK_CONFIG_API_CONSTANTS_ONLY +#undef MLK_CONFIG_API_NAMESPACE_PREFIX +#undef MLK_CONFIG_API_NO_SUPERCOP +#undef MLK_CONFIG_API_PARAMETER_SET +#undef MLK_CONFIG_API_QUALIFIER +#undef MLK_ERR_FAIL +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_RNG_FAIL +#undef MLK_H +#undef MLK_MAX3_ +#undef MLK_TOTAL_ALLOC_1024 +#undef MLK_TOTAL_ALLOC_1024_DECAPS +#undef MLK_TOTAL_ALLOC_1024_ENCAPS +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_512 +#undef MLK_TOTAL_ALLOC_512_DECAPS +#undef MLK_TOTAL_ALLOC_512_ENCAPS +#undef MLK_TOTAL_ALLOC_512_KEYPAIR +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_768 +#undef MLK_TOTAL_ALLOC_768_DECAPS +#undef MLK_TOTAL_ALLOC_768_ENCAPS +#undef MLK_TOTAL_ALLOC_768_KEYPAIR +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +#undef crypto_kem_check_pk +#undef crypto_kem_check_sk +#undef crypto_kem_dec +#undef crypto_kem_enc +#undef crypto_kem_enc_derand +#undef crypto_kem_keypair +#undef crypto_kem_keypair_derand +/* mlkem/src/common.h */ +#undef MLK_ADD_PARAM_SET +#undef MLK_ALLOC +#undef MLK_APPLY +#undef MLK_ASM_FN_SIZE +#undef MLK_ASM_FN_SYMBOL +#undef MLK_ASM_NAMESPACE +#undef MLK_BUILD_INTERNAL +#undef MLK_COMMON_H +#undef MLK_CONCAT +#undef MLK_CONCAT_ +#undef MLK_CONTEXT_PARAMETERS_0 +#undef MLK_CONTEXT_PARAMETERS_1 +#undef MLK_CONTEXT_PARAMETERS_2 +#undef MLK_CONTEXT_PARAMETERS_3 +#undef MLK_CONTEXT_PARAMETERS_4 +#undef MLK_EMPTY_CU +#undef MLK_ERR_FAIL +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_RNG_FAIL +#undef MLK_EXTERNAL_API +#undef MLK_FIPS202X4_HEADER_FILE +#undef MLK_FIPS202_HEADER_FILE +#undef MLK_FREE +#undef MLK_INTERNAL_API +#undef MLK_INTERNAL_DATA_DECLARATION +#undef MLK_INTERNAL_DATA_DEFINITION +#undef MLK_NAMESPACE +#undef MLK_NAMESPACE_K +#undef MLK_NAMESPACE_PREFIX +#undef MLK_NAMESPACE_PREFIX_K +#undef mlk_memcpy +#undef mlk_memset +/* mlkem/src/indcpa.h */ +#undef MLK_INDCPA_H +#undef mlk_gen_matrix +#undef mlk_indcpa_dec +#undef mlk_indcpa_enc +#undef mlk_indcpa_keypair_derand +/* mlkem/src/kem.h */ +#undef MLK_KEM_H +#undef mlk_kem_check_pk +#undef mlk_kem_check_sk +#undef mlk_kem_dec +#undef mlk_kem_enc +#undef mlk_kem_enc_derand +#undef mlk_kem_keypair +#undef mlk_kem_keypair_derand +/* mlkem/src/params.h */ +#undef MLKEM_DU +#undef MLKEM_DV +#undef MLKEM_ETA1 +#undef MLKEM_ETA2 +#undef MLKEM_INDCCA_CIPHERTEXTBYTES +#undef MLKEM_INDCCA_PUBLICKEYBYTES +#undef MLKEM_INDCCA_SECRETKEYBYTES +#undef MLKEM_INDCPA_BYTES +#undef MLKEM_INDCPA_MSGBYTES +#undef MLKEM_INDCPA_PUBLICKEYBYTES +#undef MLKEM_INDCPA_SECRETKEYBYTES +#undef MLKEM_K +#undef MLKEM_N +#undef MLKEM_POLYBYTES +#undef MLKEM_POLYCOMPRESSEDBYTES_D10 +#undef MLKEM_POLYCOMPRESSEDBYTES_D11 +#undef MLKEM_POLYCOMPRESSEDBYTES_D4 +#undef MLKEM_POLYCOMPRESSEDBYTES_D5 +#undef MLKEM_POLYCOMPRESSEDBYTES_DU +#undef MLKEM_POLYCOMPRESSEDBYTES_DV +#undef MLKEM_POLYVECBYTES +#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU +#undef MLKEM_Q +#undef MLKEM_Q_HALF +#undef MLKEM_SSBYTES +#undef MLKEM_SYMBYTES +#undef MLKEM_UINT12_LIMIT +#undef MLK_PARAMS_H +/* mlkem/src/poly_k.h */ +#undef MLK_POLY_K_H +#undef mlk_poly_compress_du +#undef mlk_poly_compress_dv +#undef mlk_poly_decompress_du +#undef mlk_poly_decompress_dv +#undef mlk_poly_getnoise_eta1122_4x +#undef mlk_poly_getnoise_eta1_4x +#undef mlk_poly_getnoise_eta2 +#undef mlk_poly_getnoise_eta2_4x +#undef mlk_polymat +#undef mlk_polyvec +#undef mlk_polyvec_add +#undef mlk_polyvec_basemul_acc_montgomery_cached +#undef mlk_polyvec_compress_du +#undef mlk_polyvec_decompress_du +#undef mlk_polyvec_frombytes +#undef mlk_polyvec_invntt_tomont +#undef mlk_polyvec_mulcache +#undef mlk_polyvec_mulcache_compute +#undef mlk_polyvec_ntt +#undef mlk_polyvec_reduce +#undef mlk_polyvec_tobytes +#undef mlk_polyvec_tomont + +#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS) +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files + */ +/* mlkem/src/compress.h */ +#undef MLK_COMPRESS_H +#undef mlk_poly_compress_d10 +#undef mlk_poly_compress_d11 +#undef mlk_poly_compress_d4 +#undef mlk_poly_compress_d5 +#undef mlk_poly_decompress_d10 +#undef mlk_poly_decompress_d11 +#undef mlk_poly_decompress_d4 +#undef mlk_poly_decompress_d5 +#undef mlk_poly_frombytes +#undef mlk_poly_frommsg +#undef mlk_poly_tobytes +#undef mlk_poly_tomsg +/* mlkem/src/debug.h */ +#undef MLK_DEBUG_H +#undef mlk_assert +#undef mlk_assert_abs_bound +#undef mlk_assert_abs_bound_2d +#undef mlk_assert_bound +#undef mlk_assert_bound_2d +#undef mlk_debug_check_assert +#undef mlk_debug_check_bounds +/* mlkem/src/poly.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NTT_BOUND +#undef MLK_POLY_H +#undef mlk_poly_add +#undef mlk_poly_invntt_tomont +#undef mlk_poly_mulcache_compute +#undef mlk_poly_ntt +#undef mlk_poly_reduce +#undef mlk_poly_sub +#undef mlk_poly_tomont +/* mlkem/src/randombytes.h */ +#undef MLK_RANDOMBYTES_H +/* mlkem/src/sampling.h */ +#undef MLK_SAMPLING_H +#undef mlk_poly_cbd2 +#undef mlk_poly_cbd3 +#undef mlk_poly_rej_uniform +#undef mlk_poly_rej_uniform_x4 +/* mlkem/src/symmetric.h */ +#undef MLK_SYMMETRIC_H +#undef MLK_XOF_RATE +#undef mlk_hash_g +#undef mlk_hash_h +#undef mlk_hash_j +#undef mlk_prf_eta +#undef mlk_prf_eta1 +#undef mlk_prf_eta1_x4 +#undef mlk_prf_eta2 +#undef mlk_xof_absorb +#undef mlk_xof_ctx +#undef mlk_xof_init +#undef mlk_xof_release +#undef mlk_xof_squeezeblocks +#undef mlk_xof_x4_absorb +#undef mlk_xof_x4_ctx +#undef mlk_xof_x4_init +#undef mlk_xof_x4_release +#undef mlk_xof_x4_squeezeblocks +/* mlkem/src/sys.h */ +#undef MLK_ALIGN +#undef MLK_ALIGN_UP +#undef MLK_ALWAYS_INLINE +#undef MLK_CET_ENDBR +#undef MLK_CT_TESTING_DECLASSIFY +#undef MLK_CT_TESTING_SECRET +#undef MLK_DEFAULT_ALIGN +#undef MLK_HAVE_INLINE_ASM +#undef MLK_INLINE +#undef MLK_MUST_CHECK_RETURN_VALUE +#undef MLK_NOINLINE +#undef MLK_RESTRICT +#undef MLK_STATIC_TESTABLE +#undef MLK_SYSV_ABI +#undef MLK_SYSV_ABI_SUPPORTED +#undef MLK_SYS_AARCH64 +#undef MLK_SYS_AARCH64_EB +#undef MLK_SYS_APPLE +#undef MLK_SYS_ARMV81M_MVE +#undef MLK_SYS_BIG_ENDIAN +#undef MLK_SYS_H +#undef MLK_SYS_LINUX +#undef MLK_SYS_LITTLE_ENDIAN +#undef MLK_SYS_PPC64LE +#undef MLK_SYS_RISCV32 +#undef MLK_SYS_RISCV64 +#undef MLK_SYS_RISCV64_RVV +#undef MLK_SYS_WINDOWS +#undef MLK_SYS_X86_64 +#undef MLK_SYS_X86_64_AVX2 +/* mlkem/src/verify.h */ +#undef MLK_USE_ASM_VALUE_BARRIER +#undef MLK_VERIFY_H +#undef mlk_ct_opt_blocker_u64 +/* mlkem/src/cbmc.h */ +#undef MLK_CBMC_H +#undef __contract__ +#undef __loop__ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +/* + * Undefine macros from FIPS-202 files + */ +/* mlkem/src/fips202/fips202.h */ +#undef FIPS202_X4_DEFAULT_IMPLEMENTATION +#undef MLK_FIPS202_FIPS202_H +#undef SHA3_256_HASHBYTES +#undef SHA3_256_RATE +#undef SHA3_384_RATE +#undef SHA3_512_HASHBYTES +#undef SHA3_512_RATE +#undef SHAKE128_RATE +#undef SHAKE256_RATE +#undef mlk_sha3_256 +#undef mlk_sha3_512 +#undef mlk_shake128_absorb_once +#undef mlk_shake128_init +#undef mlk_shake128_release +#undef mlk_shake128_squeezeblocks +#undef mlk_shake256 +/* mlkem/src/fips202/fips202x4.h */ +#undef MLK_FIPS202_FIPS202X4_H +#undef mlk_shake128x4_absorb_once +#undef mlk_shake128x4_init +#undef mlk_shake128x4_release +#undef mlk_shake128x4_squeezeblocks +#undef mlk_shake256x4 +/* mlkem/src/fips202/keccakf1600.h */ +#undef MLK_FIPS202_KECCAKF1600_H +#undef MLK_KECCAK_LANES +#undef MLK_KECCAK_WAY +#undef mlk_keccakf1600_extract_bytes +#undef mlk_keccakf1600_permute +#undef mlk_keccakf1600_xor_bytes +#undef mlk_keccakf1600x4_extract_bytes +#undef mlk_keccakf1600x4_permute +#undef mlk_keccakf1600x4_xor_bytes +#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* mlkem/src/fips202/native/api.h */ +#undef MLK_FIPS202_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +/* mlkem/src/fips202/native/auto.h */ +#undef MLK_FIPS202_NATIVE_AUTO_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (FIPS202, AArch64) + */ +/* mlkem/src/fips202/native/aarch64/auto.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#undef mlk_keccak_f1600_x1_scalar_aarch64_asm +#undef mlk_keccak_f1600_x1_v84a_aarch64_asm +#undef mlk_keccak_f1600_x2_v84a_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm +#undef mlk_keccakf1600_round_constants +/* mlkem/src/fips202/native/aarch64/x1_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#undef MLK_USE_FIPS202_X1_NATIVE +/* mlkem/src/fips202/native/aarch64/x1_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#undef MLK_USE_FIPS202_X1_NATIVE +/* mlkem/src/fips202/native/aarch64/x2_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X2_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#undef MLK_USE_FIPS202_X4_NATIVE +/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#undef MLK_USE_FIPS202_X4_NATIVE +/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#undef MLK_USE_FIPS202_X4_NATIVE +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (FIPS202, x86_64) + */ +/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */ +#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#undef MLK_FIPS202_X86_64_NEED_X4_AVX2 +#undef MLK_USE_FIPS202_X4_NATIVE +/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */ +#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#undef mlk_keccak_f1600_x4_avx2_asm +#undef mlk_keccak_rho56 +#undef mlk_keccak_rho8 +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_ARMV81M_MVE) +/* + * Undefine macros from native code (FIPS202, Armv8.1-M) + */ +/* mlkem/src/fips202/native/armv81m/mve.h */ +#undef MLK_FIPS202_ARMV81M_NEED_X4 +#undef MLK_FIPS202_NATIVE_ARMV81M +#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#undef MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE +#undef MLK_USE_FIPS202_X4_NATIVE +#undef MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE +#undef mlk_keccak_f1600_x4_native_impl +/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */ +#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#undef mlk_keccak_f1600_x4_mve_asm +#undef mlk_keccak_f1600_x4_state_extract_bytes_asm +#undef mlk_keccak_f1600_x4_state_xor_bytes_asm +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_ARMV81M_MVE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* mlkem/src/native/api.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +#undef MLK_NTT_BOUND +/* mlkem/src/native/meta.h */ +#undef MLK_NATIVE_META_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (Arith, AArch64) + */ +/* mlkem/src/native/aarch64/meta.h */ +#undef MLK_ARITH_BACKEND_AARCH64 +#undef MLK_NATIVE_AARCH64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */ +#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#undef mlk_aarch64_invntt_zetas_layer12345 +#undef mlk_aarch64_invntt_zetas_layer67 +#undef mlk_aarch64_ntt_zetas_layer12345 +#undef mlk_aarch64_ntt_zetas_layer67 +#undef mlk_aarch64_zetas_mulcache_native +#undef mlk_aarch64_zetas_mulcache_twisted_native +#undef mlk_intt_aarch64_asm +#undef mlk_ntt_aarch64_asm +#undef mlk_poly_mulcache_compute_aarch64_asm +#undef mlk_poly_reduce_aarch64_asm +#undef mlk_poly_tobytes_aarch64_asm +#undef mlk_poly_tomont_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm +#undef mlk_rej_uniform_aarch64_asm +#undef mlk_rej_uniform_table +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (Arith, X86_64) + */ +/* mlkem/src/native/x86_64/meta.h */ +#undef MLK_ARITH_BACKEND_X86_64_DEFAULT +#undef MLK_NATIVE_X86_64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_COMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_FROMBYTES +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */ +#undef MLK_AVX2_REJ_UNIFORM_BUFLEN +#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#undef mlk_invntt_avx2_asm +#undef mlk_ntt_avx2_asm +#undef mlk_nttfrombytes_avx2_asm +#undef mlk_ntttobytes_avx2_asm +#undef mlk_nttunpack_avx2_asm +#undef mlk_poly_compress_d10_avx2_asm +#undef mlk_poly_compress_d11_avx2_asm +#undef mlk_poly_compress_d4_avx2_asm +#undef mlk_poly_compress_d5_avx2_asm +#undef mlk_poly_decompress_d10_avx2_asm +#undef mlk_poly_decompress_d11_avx2_asm +#undef mlk_poly_decompress_d4_avx2_asm +#undef mlk_poly_decompress_d5_avx2_asm +#undef mlk_poly_mulcache_compute_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm +#undef mlk_reduce_avx2_asm +#undef mlk_rej_uniform_avx2_asm +#undef mlk_rej_uniform_table +#undef mlk_tomont_avx2_asm +/* mlkem/src/native/x86_64/src/compress_consts.h */ +#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#undef mlk_compress_d10_data +#undef mlk_compress_d11_data +#undef mlk_compress_d4_data +#undef mlk_compress_d5_data +#undef mlk_decompress_d10_data +#undef mlk_decompress_d11_data +#undef mlk_decompress_d4_data +#undef mlk_decompress_d5_data +/* mlkem/src/native/x86_64/src/consts.h */ +#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD +#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP +#undef MLK_NATIVE_X86_64_SRC_CONSTS_H +#undef mlk_qdata +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mlkem/src/native/riscv64/meta.h */ +#undef MLK_ARITH_BACKEND_RISCV64 +#undef MLK_NATIVE_RISCV64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mlk_rv64v_poly_add +#undef mlk_rv64v_poly_basemul_mont_add_k2 +#undef mlk_rv64v_poly_basemul_mont_add_k3 +#undef mlk_rv64v_poly_basemul_mont_add_k4 +#undef mlk_rv64v_poly_invntt_tomont +#undef mlk_rv64v_poly_ntt +#undef mlk_rv64v_poly_reduce +#undef mlk_rv64v_poly_sub +#undef mlk_rv64v_poly_tomont +#undef mlk_rv64v_rej_uniform +/* mlkem/src/native/riscv64/src/rv64v_debug.h */ +#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#undef mlk_assert_abs_bound_int16m1 +#undef mlk_assert_abs_bound_int16m2 +#undef mlk_assert_bound_int16m1 +#undef mlk_assert_bound_int16m2 +#undef mlk_debug_check_bounds_int16m1 +#undef mlk_debug_check_bounds_int16m2 +#endif /* MLK_SYS_RISCV64 */ +#if defined(MLK_SYS_PPC64LE) +/* + * Undefine macros from native code (Arith, PPC64LE) + */ +/* mlkem/src/native/ppc64le/meta.h */ +#undef MLK_ARITH_BACKEND_NAME +#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT +#undef MLK_NATIVE_PPC64LE_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */ +#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#undef mlk_intt_ppc_asm +#undef mlk_ntt_ppc_asm +#undef mlk_poly_tomont_ppc_asm +#undef mlk_reduce_ppc_asm +/* mlkem/src/native/ppc64le/src/consts.h */ +#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#undef MLK_PPC_C20159_OFFSET +#undef MLK_PPC_NQ_OFFSET +#undef MLK_PPC_N_INV_OFFSET +#undef MLK_PPC_N_INV_TW_OFFSET +#undef MLK_PPC_Q_OFFSET +#undef MLK_PPC_TOMONT_OFFSET +#undef MLK_PPC_TOMONT_TW_OFFSET +#undef MLK_PPC_ZETA_INTT_OFFSET +#undef MLK_PPC_ZETA_INTT_TW_OFFSET +#undef MLK_PPC_ZETA_NTT_OFFSET +#undef MLK_PPC_ZETA_NTT_TW_OFFSET +#undef mlk_ppc_qdata +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_config.h b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_config.h new file mode 100644 index 0000000..e9e5b12 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_config.h @@ -0,0 +1,645 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_CONFIG_H +#define MLK_CONFIG_H + +/** + * Specifies the parameter set for ML-KEM: + * - MLK_CONFIG_PARAMETER_SET=512 corresponds to ML-KEM-512 + * - MLK_CONFIG_PARAMETER_SET=768 corresponds to ML-KEM-768 + * - MLK_CONFIG_PARAMETER_SET=1024 corresponds to ML-KEM-1024 + * + * If you want to support multiple parameter sets, build the library multiple + * times and set MLK_CONFIG_MULTILEVEL_BUILD. See MLK_CONFIG_MULTILEVEL_BUILD + * for how to do this while minimizing code duplication. + * + * This can also be set using CFLAGS. + */ +#ifndef MLK_CONFIG_PARAMETER_SET +#define MLK_CONFIG_PARAMETER_SET \ + 768 /* Change this for different security strengths */ +#endif + +/** + * MLK_CONFIG_FILE + * + * If defined, this is a header that will be included instead of the default + * configuration file mlkem/mlkem_native_config.h. + * + * When you need to build mlkem-native in multiple configurations, using + * varying MLK_CONFIG_FILE can be more convenient than configuring everything + * through CFLAGS. + * + * To use, MLK_CONFIG_FILE _must_ be defined prior to the inclusion of any + * mlkem-native headers. For example, it can be set by passing + * `-DMLK_CONFIG_FILE="..."` on the command line. + */ +/* #define MLK_CONFIG_FILE "mlkem_native_config.h" */ + +/** + * The prefix to use to namespace global symbols from mlkem/. + * + * In a multi-level build, level-dependent symbols will additionally be + * prefixed with the parameter set (512/768/1024). + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_NAMESPACE_PREFIX) +#define MLK_CONFIG_NAMESPACE_PREFIX MLK_DEFAULT_NAMESPACE_PREFIX +#endif + +/** + * MLK_CONFIG_MULTILEVEL_BUILD + * + * Set this if the build is part of a multi-level build supporting multiple + * parameter sets. + * + * If you need only a single parameter set, keep this unset. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_BUILD */ + +/** + * MLK_CONFIG_EXTERNAL_API_QUALIFIER + * + * If set, this option provides an additional function qualifier to be added + * to declarations of mlkem-native's public API. + * + * The primary use case for this option are single-CU builds where the public + * API exposed by mlkem-native is wrapped by another API in the consuming + * application. In this case, even mlkem-native's public API can be marked + * `static`. + */ +/* #define MLK_CONFIG_EXTERNAL_API_QUALIFIER */ + +/** + * MLK_CONFIG_NO_RANDOMIZED_API + * + * If this option is set, mlkem-native will be built without the randomized + * API functions (crypto_kem_keypair and crypto_kem_enc). This allows users + * to build mlkem-native without providing a randombytes() implementation + * if they only need the deterministic API (crypto_kem_keypair_derand, + * crypto_kem_enc_derand, crypto_kem_dec). + * + * @note This option is incompatible with MLK_CONFIG_KEYGEN_PCT as the + * current PCT implementation requires crypto_kem_enc(). + */ +/* #define MLK_CONFIG_NO_RANDOMIZED_API */ + +/** + * MLK_CONFIG_NO_SUPERCOP + * + * By default, mlkem_native.h exposes the mlkem-native API in the SUPERCOP + * naming convention (crypto_kem_xxx). If you don't need this, set + * MLK_CONFIG_NO_SUPERCOP. + * + * @note You must set this for a multi-level build as the SUPERCOP naming + * does not disambiguate between the parameter sets. + */ +/* #define MLK_CONFIG_NO_SUPERCOP */ + +/** + * MLK_CONFIG_CONSTANTS_ONLY + * + * If you only need the size constants (MLKEM_PUBLICKEYBYTES, etc.) but no + * function declarations, set MLK_CONFIG_CONSTANTS_ONLY. + * + * This only affects the public header mlkem_native.h, not the + * implementation. + */ +/* #define MLK_CONFIG_CONSTANTS_ONLY */ + +/****************************************************************************** + * + * Build-only configuration options + * + * The remaining configurations are build-options only. + * They do not affect the API described in mlkem_native.h. + * + *****************************************************************************/ + +#if defined(MLK_BUILD_INTERNAL) +/** + * MLK_CONFIG_MULTILEVEL_WITH_SHARED + * + * This is for multi-level builds of mlkem-native only. If you need only a + * single parameter set, keep this unset. + * + * If this is set, all MLK_CONFIG_PARAMETER_SET-independent code will be + * included in the build, including code needed only for other parameter + * sets. + * + * Example: mlk_poly_cbd3 is only needed for MLK_CONFIG_PARAMETER_SET == 512. + * Yet, if this option is set for a build with + * MLK_CONFIG_PARAMETER_SET == 768/1024, it would be included. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_WITH_SHARED */ + +/** + * MLK_CONFIG_MULTILEVEL_NO_SHARED + * + * This is for multi-level builds of mlkem-native only. If you need only a + * single parameter set, keep this unset. + * + * If this is set, no MLK_CONFIG_PARAMETER_SET-independent code will be + * included in the build. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/** + * MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * + * This is only relevant for single compilation unit (SCU) builds of + * mlkem-native. In this case, it determines whether directives defined in + * parameter-set-independent headers should be #undef'ined or not at the + * end of the SCU file. This is needed in multilevel builds. + * + * See examples/multilevel_build_native for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ + +/** + * MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * + * Determines whether a native arithmetic backend should be used. + * + * The arithmetic backend covers performance-critical functions such as the + * number-theoretic transform (NTT). + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the arithmetic backend to be used is determined + * by MLK_CONFIG_ARITH_BACKEND_FILE: if the latter is unset, the default + * backend for the target architecture will be used. If set, it must be the + * name of a backend metadata file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* #define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif + +/** + * MLK_CONFIG_ARITH_BACKEND_FILE + * + * The arithmetic backend to use. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is unset, this option is ignored. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, this option must either + * be undefined or the filename of an arithmetic backend. If unset, the + * default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLK_CONFIG_ARITH_BACKEND_FILE) +#define MLK_CONFIG_ARITH_BACKEND_FILE "native/meta.h" +#endif + +/** + * MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * + * Determines whether a native FIPS202 backend should be used. + * + * The FIPS202 backend covers 1x/2x/4x-fold Keccak-f1600, which is the + * performance bottleneck of SHA3 and SHAKE. + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the FIPS202 backend to be used is determined by + * MLK_CONFIG_FIPS202_BACKEND_FILE: if the latter is unset, the default + * backend for the target architecture will be used. If set, it must be + * the name of a backend metadata file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* #define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#endif + +/** + * MLK_CONFIG_FIPS202_BACKEND_FILE + * + * The FIPS-202 backend to use. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, this option must either + * be undefined or the filename of a FIPS202 backend. If unset, the default + * backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLK_CONFIG_FIPS202_BACKEND_FILE) +#define MLK_CONFIG_FIPS202_BACKEND_FILE "fips202/native/auto.h" +#endif + +/** + * MLK_CONFIG_FIPS202_CUSTOM_HEADER + * + * Custom header to use for FIPS-202. + * + * This should only be set if you intend to use a custom FIPS-202 + * implementation, different from the one shipped with mlkem-native. + * + * If set, it must be the name of a file serving as the replacement for + * mlkem/fips202/fips202.h, and exposing the same API (see FIPS202.md). + */ +/* #define MLK_CONFIG_FIPS202_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLK_CONFIG_FIPS202X4_CUSTOM_HEADER + * + * Custom header to use for FIPS-202-X4. + * + * This should only be set if you intend to use a custom FIPS-202 + * implementation, different from the one shipped with mlkem-native. + * + * If set, it must be the name of a file serving as the replacement for + * mlkem/fips202/fips202x4.h, and exposing the same API (see FIPS202.md). + */ +/* #define MLK_CONFIG_FIPS202X4_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLK_CONFIG_CUSTOM_ZEROIZE + * + * In compliance with @[FIPS203, Section 3.3], mlkem-native zeroizes + * intermediate stack buffers before returning from function calls. + * + * Set this option and define `mlk_zeroize` if you want to use a custom + * method to zeroize intermediate stack buffers. The default implementation + * uses SecureZeroMemory on Windows and a memset + compiler barrier + * otherwise. If neither of those is available on the target platform, + * compilation will fail, and you will need to use MLK_CONFIG_CUSTOM_ZEROIZE + * to provide a custom implementation of `mlk_zeroize()`. + * + * @warning The explicit stack zeroization conducted by mlkem-native reduces + * the likelihood of data leaking on the stack, but does not + * eliminate it. The C standard makes no guarantee about where a + * compiler allocates structures and whether/where it makes copies + * of them. Also, in addition to entire structures, there may also + * be potentially exploitable leakage of individual values on the + * stack. If you need bullet-proof zeroization of the stack, you + * need to consider additional measures instead of what this + * feature provides. In this case, you can set mlk_zeroize to a + * no-op. + */ +/* #define MLK_CONFIG_CUSTOM_ZEROIZE + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_RANDOMBYTES + * + * mlkem-native does not provide a secure randombytes implementation. Such + * an implementation has to be provided by the consumer. + * + * If this option is not set, mlkem-native expects a function + * int randombytes(uint8_t *out, size_t outlen). It is expected to return + * zero on success, and non-zero on failure. In case of failure, the + * top-level APIs will return an MLK_ERR_RNG_FAIL error code. + * + * Set this option and define `mlk_randombytes` (with the same signature + * and behaviour) if you want to use a custom method to sample randombytes + * with a different name or signature. + */ +/* #define MLK_CONFIG_CUSTOM_RANDOMBYTES + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE int mlk_randombytes(uint8_t *ptr, size_t len) + { + ... your implementation ... + return 0; + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + * + * mlkem-native backends may rely on specific hardware features. Those + * backends will only be included in an mlkem-native build if support for + * the respective features is enabled at compile-time. However, when + * building for a heterogeneous set of CPUs to run the resulting + * binary/library on, feature detection at _runtime_ is needed to decide + * whether a backend can be used or not. + * + * Set this option and define `mlk_sys_check_capability` if you want to + * use a custom method to dispatch between implementations. + * + * If this option is not set, mlkem-native uses compile-time feature + * detection only to decide which backend to use. + * + * If you compile mlkem-native on a system with different capabilities + * than the system that the resulting binary/library will be run on, you + * must use this option. + */ +/* #define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) + __contract__( + ensures(return_value == 0 || return_value == 1) + ) + { + ... your implementation ... + } +*/ + +/** + * MLK_CONFIG_CUSTOM_ALLOC_FREE [EXPERIMENTAL] + * + * Set this option and define `MLK_CUSTOM_ALLOC` and `MLK_CUSTOM_FREE` if + * you want to use custom allocation for large local structures or buffers. + * + * By default, all buffers/structures are allocated on the stack. If this + * option is set, most of them will be allocated via MLK_CUSTOM_ALLOC. + * + * Parameters to MLK_CUSTOM_ALLOC: + * - T* v: Target pointer to declare. + * - T: Type of structure to be allocated. + * - N: Number of elements to be allocated. + * + * Parameters to MLK_CUSTOM_FREE: + * - T* v: Target pointer to free. May be NULL. + * - T: Type of structure to be freed. + * - N: Number of elements to be freed. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. We expect a + * stable API for v2. + * + * @note Even if this option is set, some allocations further down the call + * stack will still be made from the stack, consuming up to 3KB of + * stack space. Those will likely be added to the scope of this + * option in the future. + * + * @note MLK_CUSTOM_ALLOC need not guarantee a successful allocation nor + * include error handling. Upon failure, the target pointer should + * simply be set to NULL. The calling code will handle this case and + * invoke MLK_CUSTOM_FREE. + */ +/* #define MLK_CONFIG_CUSTOM_ALLOC_FREE + #if !defined(__ASSEMBLER__) + #include + #define MLK_CUSTOM_ALLOC(v, T, N) \ + T* (v) = (T *)aligned_alloc(MLK_DEFAULT_ALIGN, \ + MLK_ALIGN_UP(sizeof(T) * (N))) + #define MLK_CUSTOM_FREE(v, T, N) free(v) + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_MEMCPY + * + * Set this option and define `mlk_memcpy` if you want to use a custom + * method to copy memory instead of the standard library memcpy function. + * + * The custom implementation must have the same signature and behavior as + * the standard memcpy function: + * void *mlk_memcpy(void *dest, const void *src, size_t n) + */ +/* #define MLK_CONFIG_CUSTOM_MEMCPY + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void *mlk_memcpy(void *dest, const void *src, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_MEMSET + * + * Set this option and define `mlk_memset` if you want to use a custom + * method to set memory instead of the standard library memset function. + * + * The custom implementation must have the same signature and behavior as + * the standard memset function: + * void *mlk_memset(void *s, int c, size_t n) + */ +/* #define MLK_CONFIG_CUSTOM_MEMSET + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void *mlk_memset(void *s, int c, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_INTERNAL_API_QUALIFIER + * + * If set, this option provides an additional qualifier to be added to + * declarations of internal API functions and data. + * + * The primary use case for this option are single-CU builds, in which case + * this option can be set to `static`. + */ +/* #define MLK_CONFIG_INTERNAL_API_QUALIFIER */ + +/** + * MLK_CONFIG_CT_TESTING_ENABLED + * + * If set, mlkem-native annotates data as secret/public using valgrind's + * annotations VALGRIND_MAKE_MEM_UNDEFINED and VALGRIND_MAKE_MEM_DEFINED, + * enabling various checks for secret-dependent control flow or + * variable-time execution (depending on the exact version of valgrind + * installed). + */ +/* #define MLK_CONFIG_CT_TESTING_ENABLED */ + +/** + * MLK_CONFIG_NO_ASM + * + * If this option is set, mlkem-native will be built without use of native + * code or inline assembly. + * + * By default, inline assembly is used to implement value barriers. Without + * inline assembly, mlkem-native will use a global volatile 'opt blocker' + * instead; see verify.h. + * + * Inline assembly is also used to implement a secure zeroization function + * on non-Windows platforms. If this option is set and the target platform + * is not Windows, you MUST set MLK_CONFIG_CUSTOM_ZEROIZE and provide a + * custom zeroization function. + * + * If this option is set, MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 and + * MLK_CONFIG_USE_NATIVE_BACKEND_ARITH will be ignored, and no native + * backends will be used. + */ +/* #define MLK_CONFIG_NO_ASM */ + +/** + * MLK_CONFIG_NO_ASM_VALUE_BARRIER + * + * If this option is set, mlkem-native will be built without use of native + * code or inline assembly for value barriers. + * + * By default, inline assembly (if available) is used to implement value + * barriers. Without inline assembly, mlkem-native will use a global + * volatile 'opt blocker' instead; see verify.h. + */ +/* #define MLK_CONFIG_NO_ASM_VALUE_BARRIER */ + +/** + * MLK_CONFIG_KEYGEN_PCT + * + * Compliance with @[FIPS140_3_IG, p.87] requires a Pairwise Consistency + * Test (PCT) to be carried out on a freshly generated keypair before it + * can be exported. + * + * Set this option if such a check should be implemented. In this case, + * crypto_kem_keypair_derand and crypto_kem_keypair will return a non-zero + * error code if the PCT failed. + * + * @note This feature will drastically lower the performance of key + * generation. + */ +/* #define MLK_CONFIG_KEYGEN_PCT */ + +/** + * MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + * + * If this option is set, the user must provide a runtime function + * `static inline int mlk_break_pct() { ... }` to indicate whether the PCT + * should be made to fail. + * + * This option only has an effect if MLK_CONFIG_KEYGEN_PCT is set. + */ +/* #define MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + #if !defined(__ASSEMBLER__) + #include "src/sys.h" + static MLK_INLINE int mlk_break_pct(void) + { + ... return 0/1 depending on whether PCT should be broken ... + } + #endif +*/ + +/** + * MLK_CONFIG_SERIAL_FIPS202_ONLY + * + * Set this to use a FIPS202 implementation with global state that supports + * only one active Keccak computation at a time (e.g. some hardware + * accelerators). + * + * If this option is set, batched Keccak operations are disabled for + * rejection sampling during matrix generation. Instead, matrix entries + * will be generated one at a time. + * + * This allows offloading Keccak computations to a hardware accelerator + * that holds only a single Keccak state locally, rather than requiring + * support for batched (4x) Keccak states. + * + * @note Depending on the target CPU, disabling batched Keccak may reduce + * performance when using software FIPS202 implementations. Only + * enable this when you have to. + */ +/* #define MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +/** + * MLK_CONFIG_CONTEXT_PARAMETER + * + * Set this to add a context parameter that is provided to public API + * functions and is then available in custom callbacks. + * + * The type of the context parameter is configured via + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + */ +/* #define MLK_CONFIG_CONTEXT_PARAMETER */ + +/** + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE + * + * Set this to define the type for the context parameter used by + * MLK_CONFIG_CONTEXT_PARAMETER. + * + * This is only relevant if MLK_CONFIG_CONTEXT_PARAMETER is set. + */ +/* #define MLK_CONFIG_CONTEXT_PARAMETER_TYPE void* */ + +/************************* Config internals ********************************/ + +#endif /* MLK_BUILD_INTERNAL */ + +/* Default namespace + * + * Don't change this. If you need a different namespace, re-define + * MLK_CONFIG_NAMESPACE_PREFIX above instead, and remove the following. + * + * The default MLKEM namespace is + * + * PQCP_MLKEM_NATIVE_MLKEM_ + * + * e.g., PQCP_MLKEM_NATIVE_MLKEM512_ + */ + +#if MLK_CONFIG_PARAMETER_SET == 512 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM512 +#elif MLK_CONFIG_PARAMETER_SET == 768 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM768 +#elif MLK_CONFIG_PARAMETER_SET == 1024 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM1024 +#endif + +#endif /* !MLK_CONFIG_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/cbmc.h b/sdk/software/examples/mlkem_test/mlkem_native/src/cbmc.h new file mode 100644 index 0000000..07ec633 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/cbmc.h @@ -0,0 +1,197 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_CBMC_H +#define MLK_CBMC_H +/*************************************************** + * Basic replacements for __CPROVER_XXX contracts + ***************************************************/ +/* + * The `__contract__` / `__loop__` annotation macros use a + * leading-double-underscore spelling in line with other CBMC macros. + * clang-tidy flags these as reserved identifiers; we suppress the diagnostic + * at each definition site (NOLINT) rather than disabling the check globally, + * so it stays active for the rest of the tree. + */ +#ifndef CBMC + +/* clang-format off */ +#define __contract__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ + +#else /* !CBMC */ + + +/* clang-format off */ +#define __contract__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ + +/* https://diffblue.github.io/cbmc/contracts-assigns.html */ +#define assigns(...) __CPROVER_assigns(__VA_ARGS__) + +/* https://diffblue.github.io/cbmc/contracts-requires-ensures.html */ +#define requires(...) __CPROVER_requires(__VA_ARGS__) +#define ensures(...) __CPROVER_ensures(__VA_ARGS__) +/* https://diffblue.github.io/cbmc/contracts-loops.html */ +#define invariant(...) __CPROVER_loop_invariant(__VA_ARGS__) +#define decreases(...) __CPROVER_decreases(__VA_ARGS__) +/* cassert to avoid confusion with in-built assert */ +#define cassert(x) __CPROVER_assert(x, "cbmc assertion failed") +#define assume(...) __CPROVER_assume(__VA_ARGS__) + +/*************************************************** + * Macros for "expression" forms that may appear + * _inside_ top-level contracts. + ***************************************************/ + +/* + * function return value - useful inside ensures + * https://diffblue.github.io/cbmc/contracts-functions.html + */ +#define return_value (__CPROVER_return_value) + +/* + * assigns l-value targets + * https://diffblue.github.io/cbmc/contracts-assigns.html + */ +#define object_whole(...) __CPROVER_object_whole(__VA_ARGS__) +#define memory_slice(...) __CPROVER_object_upto(__VA_ARGS__) + +/* + * Pointer-related predicates + * https://diffblue.github.io/cbmc/contracts-memory-predicates.html + */ +#define memory_no_alias(...) __CPROVER_is_fresh(__VA_ARGS__) +#define readable(...) __CPROVER_r_ok(__VA_ARGS__) +#define writeable(...) __CPROVER_w_ok(__VA_ARGS__) + +/* Maximum supported buffer size + * + * Larger buffers may be supported, but due to internal modeling constraints + * in CBMC, the proofs of memory- and type-safety won't be able to run. + * + * If you find yourself in need for a buffer size larger than this, + * please contact the maintainers, so we can prioritize work to relax + * this somewhat artificial bound. + */ +#define MLK_MAX_BUFFER_SIZE (SIZE_MAX >> 12) + +/* + * History variables + * https://diffblue.github.io/cbmc/contracts-history-variables.html + */ +#define old(...) __CPROVER_old(__VA_ARGS__) +#define loop_entry(...) __CPROVER_loop_entry(__VA_ARGS__) + +/* + * Quantifiers + * Note that the range on qvar is _exclusive_ between qvar_lb .. qvar_ub + * https://diffblue.github.io/cbmc/contracts-quantifiers.html + * + * The quantified variable is declared as uint32_t, so these macros + * quantify only over indices in [0, UINT32_MAX). Bounds larger than + * UINT32_MAX (4 GiB) are NOT supported: the explicit (uint32_t) casts + * on the bounds will trigger CBMC's conversion check if a wider bound + * (e.g. a size_t > UINT32_MAX) is passed. + * + * Quantifying over size_t (64-bit) was found to blow up SMT proof + * times, so we deliberately keep the index width at 32 bits. Callers + * dealing with size_t-typed buffers must add an explicit + * requires(len <= UINT32_MAX) + * precondition. + */ + +/* + * Prevent clang-format from corrupting CBMC's special ==> operator + */ +/* clang-format off */ +#define forall(qvar, qvar_lb, qvar_ub, predicate) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> (predicate) \ + } + +#define exists(qvar, qvar_lb, qvar_ub, predicate) \ + __CPROVER_exists \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) && (predicate) \ + } +/* clang-format on */ + +/*************************************************** + * Convenience macros for common contract patterns + ***************************************************/ + +/* + * Boolean-value predidate that asserts that "all values of array_var are in + * range value_lb (inclusive) .. value_ub (exclusive)" + * Example: + * array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q) + * expands to + * __CPROVER_forall { int k; (0 <= k && k <= MLKEM_N-1) ==> ( + * 0 <= a->coeffs[k]) && a->coeffs[k] < MLKEM_Q)) } + */ + +/* + * Prevent clang-format from corrupting CBMC's special ==> operator + */ +/* clang-format off */ +#define CBMC_CONCAT_(left, right) left##right +#define CBMC_CONCAT(left, right) CBMC_CONCAT_(left, right) + +#define array_bound_core(qvar, qvar_lb, qvar_ub, array_var, \ + value_lb, value_ub) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + (((int)(value_lb) <= ((array_var)[(qvar)])) && \ + (((array_var)[(qvar)]) < (int)(value_ub))) \ + } + +#define array_bound(array_var, qvar_lb, qvar_ub, value_lb, value_ub) \ + array_bound_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (qvar_lb), \ + (qvar_ub), (array_var), (value_lb), (value_ub)) + +#define array_unchanged_core(qvar, qvar_lb, qvar_ub, array_var) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + ((array_var)[(qvar)]) == (old(* (int16_t (*)[(qvar_ub)])(array_var)))[(qvar)] \ + } + +#define array_unchanged(array_var, N) \ + array_unchanged_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var)) + +#define array_unchanged_u64_core(qvar, qvar_lb, qvar_ub, array_var) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + ((array_var)[(qvar)]) == (old(* (uint64_t (*)[(qvar_ub)])(array_var)))[(qvar)] \ + } + +#define array_unchanged_u64(array_var, N) \ + array_unchanged_u64_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var)) +/* clang-format on */ + +/* Wrapper around array_bound operating on absolute values. + * + * The absolute value bound `k` is exclusive. + * + * Note that since the lower bound in array_bound is inclusive, we have to + * raise it by 1 here. + */ +#define array_abs_bound(arr, lb, ub, k) \ + array_bound((arr), (lb), (ub), -((int)(k)) + 1, (k)) + +#endif /* CBMC */ + +#endif /* !MLK_CBMC_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/common.h b/sdk/software/examples/mlkem_test/mlkem_native/src/common.h new file mode 100644 index 0000000..463029b --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/common.h @@ -0,0 +1,282 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_COMMON_H +#define MLK_COMMON_H + +#ifndef __ASSEMBLER__ +#include +#endif + +#define MLK_BUILD_INTERNAL + +#if defined(MLK_CONFIG_FILE) +#include MLK_CONFIG_FILE +#else +#include "mlkem_native_config.h" +#endif + +#include "params.h" +#include "sys.h" + +/* Internal and public API have external linkage by default, but + * this can be overwritten by the user, e.g. for single-CU builds. */ +#if !defined(MLK_CONFIG_INTERNAL_API_QUALIFIER) +#define MLK_INTERNAL_API +#define MLK_INTERNAL_DATA_DECLARATION extern +#define MLK_INTERNAL_DATA_DEFINITION +#else +#define MLK_INTERNAL_API MLK_CONFIG_INTERNAL_API_QUALIFIER +#define MLK_INTERNAL_DATA_DECLARATION MLK_CONFIG_INTERNAL_API_QUALIFIER +#define MLK_INTERNAL_DATA_DEFINITION MLK_CONFIG_INTERNAL_API_QUALIFIER +#endif + +#if !defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLK_EXTERNAL_API +#else +#define MLK_EXTERNAL_API MLK_CONFIG_EXTERNAL_API_QUALIFIER +#endif + +#define MLK_CONCAT_(x1, x2) x1##x2 +#define MLK_CONCAT(x1, x2) MLK_CONCAT_(x1, x2) + +#if (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || \ + defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)) +#define MLK_ADD_PARAM_SET(s) MLK_CONCAT(s, MLK_CONFIG_PARAMETER_SET) +#else +#define MLK_ADD_PARAM_SET(s) s +#endif + +#define MLK_NAMESPACE_PREFIX MLK_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, _) +#define MLK_NAMESPACE_PREFIX_K \ + MLK_CONCAT(MLK_ADD_PARAM_SET(MLK_CONFIG_NAMESPACE_PREFIX), _) + +/* Functions are prefixed by MLK_CONFIG_NAMESPACE_PREFIX. + * + * If multiple parameter sets are used, functions depending on the parameter + * set are additionally prefixed with 512/768/1024. See mlkem_native_config.h. + * + * Example: If MLK_CONFIG_NAMESPACE_PREFIX is mlkem, then + * MLK_NAMESPACE_K(enc) becomes mlkem512_enc/mlkem768_enc/mlkem1024_enc. + */ +#define MLK_NAMESPACE(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX, s) +#define MLK_NAMESPACE_K(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX_K, s) + +/* On Apple platforms, we need to emit leading underscore + * in front of assembly symbols. We thus introducee a separate + * namespace wrapper for ASM symbols. */ +#if !defined(__APPLE__) +#define MLK_ASM_NAMESPACE(sym) MLK_NAMESPACE(sym) +#else +#define MLK_ASM_NAMESPACE(sym) MLK_CONCAT(_, MLK_NAMESPACE(sym)) +#endif + +/* + * On X86_64 if control-flow protections (CET) are enabled (through + * -fcf-protection=), we add an endbr64 instruction at every global function + * label. See sys.h for more details + */ +#if defined(MLK_SYS_X86_64) +#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : MLK_CET_ENDBR +#elif defined(MLK_SYS_ARMV81M_MVE) +/* clang-format off */ +#define MLK_ASM_FN_SYMBOL(sym) \ + .type MLK_ASM_NAMESPACE(sym), %function; \ + MLK_ASM_NAMESPACE(sym) : +/* clang-format on */ +#else /* !MLK_SYS_X86_64 && MLK_SYS_ARMV81M_MVE */ +#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : +#endif /* !MLK_SYS_X86_64 && !MLK_SYS_ARMV81M_MVE */ + +/* + * Output the size of an assembly function. + */ +#if defined(__ELF__) +#define MLK_ASM_FN_SIZE(sym) \ + .size MLK_ASM_NAMESPACE(sym), .- MLK_ASM_NAMESPACE(sym) +#else +#define MLK_ASM_FN_SIZE(sym) +#endif + +/* We aim to simplify the user's life by supporting builds where + * all source files are included, even those that are not needed. + * Those files are appropriately guarded and will be empty when unneeded. + * The following is to avoid compilers complaining about this. */ +#define MLK_EMPTY_CU(s) extern int MLK_NAMESPACE_K(empty_cu_##s); + +/* MLK_CONFIG_NO_ASM takes precedence over MLK_USE_NATIVE_XXX */ +#if defined(MLK_CONFIG_NO_ASM) +#undef MLK_CONFIG_USE_NATIVE_BACKEND_ARITH +#undef MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLK_CONFIG_ARITH_BACKEND_FILE) +#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, but MLK_CONFIG_ARITH_BACKEND_FILE is not. +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLK_CONFIG_FIPS202_BACKEND_FILE) +#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, but MLK_CONFIG_FIPS202_BACKEND_FILE is not. +#endif + +#if defined(MLK_CONFIG_NO_RANDOMIZED_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_RANDOMIZED_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires crypto_kem_enc() +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#include MLK_CONFIG_ARITH_BACKEND_FILE +/* Include to enforce consistency of API and implementation, + * and conduct sanity checks on the backend. + * + * Keep this _after_ the inclusion of the backend; otherwise, + * the sanity checks won't have an effect. */ +#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__) +#include "native/api.h" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#include MLK_CONFIG_FIPS202_BACKEND_FILE +/* Include to enforce consistency of API and implementation, + * and conduct sanity checks on the backend. + * + * Keep this _after_ the inclusion of the backend; otherwise, + * the sanity checks won't have an effect. */ +#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__) +#include "fips202/native/api.h" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +#define MLK_FIPS202_HEADER_FILE "fips202/fips202.h" +#else +#define MLK_FIPS202_HEADER_FILE MLK_CONFIG_FIPS202_CUSTOM_HEADER +#endif + +#if !defined(MLK_CONFIG_FIPS202X4_CUSTOM_HEADER) +#define MLK_FIPS202X4_HEADER_FILE "fips202/fips202x4.h" +#else +#define MLK_FIPS202X4_HEADER_FILE MLK_CONFIG_FIPS202X4_CUSTOM_HEADER +#endif + +/* Standard library function replacements */ +#if !defined(__ASSEMBLER__) +#if !defined(MLK_CONFIG_CUSTOM_MEMCPY) +#include +#define mlk_memcpy memcpy +#endif + +#if !defined(MLK_CONFIG_CUSTOM_MEMSET) +#include +#define mlk_memset memset +#endif + + +/* Allocation macros for large local structures + * + * MLK_ALLOC(v, T, N) declares T *v and attempts to point it to an T[N] + * MLK_FREE(v, T, N) zeroizes and frees the allocation + * + * Default implementation uses stack allocation. + * Can be overridden by setting the config option MLK_CONFIG_CUSTOM_ALLOC_FREE + * and defining MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE. + */ +#if defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) != \ + (defined(MLK_CUSTOM_ALLOC) && defined(MLK_CUSTOM_FREE)) +#error Bad configuration: MLK_CONFIG_CUSTOM_ALLOC_FREE must be set together with MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE +#endif + +/* + * If the integration wants to provide a context parameter for use in + * platform-specific hooks, then it should define this parameter. + * + * The MLK_CONTEXT_PARAMETERS_n macros are intended to be used with macros + * defining the function names and expand to either pass or discard the context + * argument as required by the current build. If there is no context parameter + * requested then these are removed from the prototypes and from all calls. + */ +#ifdef MLK_CONFIG_CONTEXT_PARAMETER +#define MLK_CONTEXT_PARAMETERS_0(context) (context) +#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0, context) +#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1, context) +#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) \ + (arg0, arg1, arg2, context) +#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3, context) +#else /* MLK_CONFIG_CONTEXT_PARAMETER */ +#define MLK_CONTEXT_PARAMETERS_0(context) () +#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0) +#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1) +#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) (arg0, arg1, arg2) +#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3) +#endif /* !MLK_CONFIG_CONTEXT_PARAMETER */ + +#if defined(MLK_CONFIG_CONTEXT_PARAMETER_TYPE) != \ + defined(MLK_CONFIG_CONTEXT_PARAMETER) +#error MLK_CONFIG_CONTEXT_PARAMETER_TYPE must be defined if and only if MLK_CONFIG_CONTEXT_PARAMETER is defined +#endif + +#if !defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) +/* Default: stack allocation */ + +/* This is a declaration macro, not an expression macro: T is a type and v is + * a declarator, neither of which can be wrapped in parentheses. The + * bugprone-macro-parentheses diagnostic is therefore a false positive here. */ +#define MLK_ALLOC(v, T, N, context) \ + MLK_ALIGN T mlk_alloc_##v[N]; \ + T *v = mlk_alloc_##v /* NOLINT(bugprone-macro-parentheses) */ + +/* The MLK_FREE macro body references mlk_zeroize(), which is declared in + * verify.h. We deliberately do NOT include verify.h here: doing so would + * create a circular dependency (verify.h includes common.h), and common.h + * itself never calls mlk_zeroize() -- only the macro expansion does. Each + * translation unit that uses MLK_FREE therefore includes verify.h directly. */ +#define MLK_FREE(v, T, N, context) \ + do \ + { \ + mlk_zeroize(mlk_alloc_##v, sizeof(mlk_alloc_##v)); \ + (v) = NULL; \ + } while (0) + +#else /* !MLK_CONFIG_CUSTOM_ALLOC_FREE */ + +/* Custom allocation */ + +/* + * The indirection here is necessary to use MLK_CONTEXT_PARAMETERS_3 here. + */ +#define MLK_APPLY(f, args) f args + +#define MLK_ALLOC(v, T, N, context) \ + MLK_APPLY(MLK_CUSTOM_ALLOC, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)) + +#define MLK_FREE(v, T, N, context) \ + do \ + { \ + if (v != NULL) \ + { \ + mlk_zeroize(v, sizeof(T) * (N)); \ + MLK_APPLY(MLK_CUSTOM_FREE, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)); \ + v = NULL; \ + } \ + } while (0) + +#endif /* MLK_CONFIG_CUSTOM_ALLOC_FREE */ + +/****************************** Error codes ***********************************/ + +/* Generic failure condition */ +#define MLK_ERR_FAIL (-1) +/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is defined and the provided MLK_CUSTOM_ALLOC can fail. */ +#define MLK_ERR_OUT_OF_MEMORY (-2) +/* An rng failure occured. Might be due to insufficient entropy or + * system misconfiguration. */ +#define MLK_ERR_RNG_FAIL (-3) + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_COMMON_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/compress.c b/sdk/software/examples/mlkem_test/mlkem_native/src/compress.c new file mode 100644 index 0000000..874ea1e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/compress.c @@ -0,0 +1,743 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "cbmc.h" +#include "compress.h" +#include "debug.h" +#include "verify.h" + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +/* Reference: `poly_compress()` in the reference implementation @[REF], + * for ML-KEM-{512,768}. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d4_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8] = {0}; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_bound(t, 0, j, 0, 16)) + decreases(8 - j)) + { + t[j] = mlk_scalar_compress_d4(a->coeffs[8 * i + j]); + } + + /* All t[i] are 4-bit wide, so the truncations don't alter the value. */ + r[i * 4] = (uint8_t)(t[0] | (t[1] << 4)); + r[i * 4 + 1] = (uint8_t)(t[2] | (t[3] << 4)); + r[i * 4 + 2] = (uint8_t)(t[4] | (t[5] << 4)); + r[i * 4 + 3] = (uint8_t)(t[6] | (t[7] << 4)); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d4_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */ + + mlk_poly_compress_d4_c(r, a); +} + +/* Reference: Embedded into `polyvec_compress()` in the + * reference implementation, for ML-KEM-{512,768}. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d10_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +) +{ + unsigned j; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + for (j = 0; j < MLKEM_N / 4; j++) + __loop__(invariant(j <= MLKEM_N / 4) + decreases(MLKEM_N / 4 - j)) + { + unsigned k; + uint16_t t[4]; + for (k = 0; k < 4; k++) + __loop__( + invariant(k <= 4) + invariant(forall(r, 0, k, t[r] < (1u << 10))) + decreases(4 - k)) + { + t[k] = mlk_scalar_compress_d10(a->coeffs[4 * j + k]); + } + + /* + * Make all implicit truncation explicit. No data is being + * truncated for the LHS's since each t[i] is 10-bit in size. + */ + r[5 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF); + r[5 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 2) & 0xFF)); + r[5 * j + 2] = (uint8_t)((t[1] >> 6) | ((t[2] << 4) & 0xFF)); + r[5 * j + 3] = (uint8_t)((t[2] >> 4) | ((t[3] << 6) & 0xFF)); + r[5 * j + 4] = (uint8_t)(t[3] >> 2); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d10_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */ + + mlk_poly_compress_d10_c(r, a); +} + +/* Reference: `poly_decompress()` in the reference implementation @[REF], + * for ML-KEM-{512,768}. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d4_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 2; i++) + __loop__( + invariant(i <= MLKEM_N / 2) + invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 2 - i)) + { + r->coeffs[2 * i + 0] = mlk_scalar_decompress_d4((a[i] >> 0) & 0xF); + r->coeffs[2 * i + 1] = mlk_scalar_decompress_d4((a[i] >> 4) & 0xF); + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d4(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4) + int ret; + ret = mlk_poly_decompress_d4_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */ + + mlk_poly_decompress_d4_c(r, a); +} + +/* Reference: Embedded into `polyvec_decompress()` in the + * reference implementation, for ML-KEM-{512,768}. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d10_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned j; + for (j = 0; j < MLKEM_N / 4; j++) + __loop__( + invariant(j <= MLKEM_N / 4) + invariant(array_bound(r->coeffs, 0, 4 * j, 0, MLKEM_Q)) + decreases(MLKEM_N / 4 - j)) + { + unsigned k; + uint16_t t[4]; + uint8_t const *base = &a[5 * j]; + + t[0] = 0x3FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8)); + t[1] = 0x3FF & ((base[1] >> 2) | ((uint16_t)base[2] << 6)); + t[2] = 0x3FF & ((base[2] >> 4) | ((uint16_t)base[3] << 4)); + t[3] = 0x3FF & ((base[3] >> 6) | ((uint16_t)base[4] << 2)); + + for (k = 0; k < 4; k++) + __loop__( + invariant(k <= 4) + invariant(array_bound(r->coeffs, 0, 4 * j + k, 0, MLKEM_Q)) + decreases(4 - k)) + { + r->coeffs[4 * j + k] = mlk_scalar_decompress_d10(t[k]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d10(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10) + int ret; + ret = mlk_poly_decompress_d10_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */ + + mlk_poly_decompress_d10_c(r, a); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +/* Reference: `poly_compress()` in the reference implementation @[REF], + * for ML-KEM-1024. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d5_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8] = {0}; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_bound(t, 0, j, 0, 32)) + decreases(8 - j)) + { + t[j] = mlk_scalar_compress_d5(a->coeffs[8 * i + j]); + } + + r[i * 5] = (uint8_t)(0xFF & ((t[0] >> 0) | (t[1] << 5))); + r[i * 5 + 1] = (uint8_t)(0xFF & ((t[1] >> 3) | (t[2] << 2) | (t[3] << 7))); + r[i * 5 + 2] = (uint8_t)(0xFF & ((t[3] >> 1) | (t[4] << 4))); + r[i * 5 + 3] = (uint8_t)(0xFF & ((t[4] >> 4) | (t[5] << 1) | (t[6] << 6))); + r[i * 5 + 4] = (uint8_t)(0xFF & ((t[6] >> 2) | (t[7] << 3))); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d5_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */ + + mlk_poly_compress_d5_c(r, a); +} + +/* Reference: Embedded into `polyvec_compress()` in the + * reference implementation, for ML-KEM-1024. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d11_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +) +{ + unsigned j; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (j = 0; j < MLKEM_N / 8; j++) + __loop__(invariant(j <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - j)) + { + unsigned k; + uint16_t t[8]; + for (k = 0; k < 8; k++) + __loop__( + invariant(k <= 8) + invariant(forall(r, 0, k, t[r] < (1u << 11))) + decreases(8 - k)) + { + t[k] = mlk_scalar_compress_d11(a->coeffs[8 * j + k]); + } + + /* + * Make all implicit truncation explicit. No data is being + * truncated for the LHS's since each t[i] is 11-bit in size. + */ + r[11 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF); + r[11 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 3) & 0xFF)); + r[11 * j + 2] = (uint8_t)((t[1] >> 5) | ((t[2] << 6) & 0xFF)); + r[11 * j + 3] = (uint8_t)((t[2] >> 2) & 0xFF); + r[11 * j + 4] = (uint8_t)((t[2] >> 10) | ((t[3] << 1) & 0xFF)); + r[11 * j + 5] = (uint8_t)((t[3] >> 7) | ((t[4] << 4) & 0xFF)); + r[11 * j + 6] = (uint8_t)((t[4] >> 4) | ((t[5] << 7) & 0xFF)); + r[11 * j + 7] = (uint8_t)((t[5] >> 1) & 0xFF); + r[11 * j + 8] = (uint8_t)((t[5] >> 9) | ((t[6] << 2) & 0xFF)); + r[11 * j + 9] = (uint8_t)((t[6] >> 6) | ((t[7] << 5) & 0xFF)); + r[11 * j + 10] = (uint8_t)(t[7] >> 3); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d11_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */ + + mlk_poly_compress_d11_c(r, a); +} + +/* Reference: `poly_decompress()` in the reference implementation @[REF], + * for ML-KEM-1024. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d5_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8]; + const unsigned offset = i * 5; + /* + * Explicitly truncate to avoid warning about + * implicit truncation in CBMC and unwind loop for ease + * of proof. + */ + + /* + * Decompress 5 8-bit bytes (so 40 bits) into + * 8 5-bit values stored in t[] + */ + t[0] = 0x1F & (a[offset + 0] >> 0); + t[1] = 0x1F & ((a[offset + 0] >> 5) | (a[offset + 1] << 3)); + t[2] = 0x1F & (a[offset + 1] >> 2); + t[3] = 0x1F & ((a[offset + 1] >> 7) | (a[offset + 2] << 1)); + t[4] = 0x1F & ((a[offset + 2] >> 4) | (a[offset + 3] << 4)); + t[5] = 0x1F & (a[offset + 3] >> 1); + t[6] = 0x1F & ((a[offset + 3] >> 6) | (a[offset + 4] << 2)); + t[7] = 0x1F & (a[offset + 4] >> 3); + + /* and copy to the correct slice in r[] */ + for (j = 0; j < 8; j++) + __loop__( + invariant(j <= 8 && i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q)) + decreases(8 - j)) + { + r->coeffs[8 * i + j] = mlk_scalar_decompress_d5(t[j]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d5(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5) + int ret; + ret = mlk_poly_decompress_d5_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */ + + mlk_poly_decompress_d5_c(r, a); +} + +/* Reference: Embedded into `polyvec_decompress()` in the + * reference implementation, for ML-KEM-1024. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d11_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned j; + for (j = 0; j < MLKEM_N / 8; j++) + __loop__( + invariant(j <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * j, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - j)) + { + unsigned k; + uint16_t t[8]; + uint8_t const *base = &a[11 * j]; + t[0] = 0x7FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8)); + t[1] = 0x7FF & ((base[1] >> 3) | ((uint16_t)base[2] << 5)); + t[2] = 0x7FF & ((base[2] >> 6) | ((uint16_t)base[3] << 2) | + ((uint16_t)base[4] << 10)); + t[3] = 0x7FF & ((base[4] >> 1) | ((uint16_t)base[5] << 7)); + t[4] = 0x7FF & ((base[5] >> 4) | ((uint16_t)base[6] << 4)); + t[5] = 0x7FF & ((base[6] >> 7) | ((uint16_t)base[7] << 1) | + ((uint16_t)base[8] << 9)); + t[6] = 0x7FF & ((base[8] >> 2) | ((uint16_t)base[9] << 6)); + t[7] = 0x7FF & ((base[9] >> 5) | ((uint16_t)base[10] << 3)); + + for (k = 0; k < 8; k++) + __loop__( + invariant(k <= 8) + invariant(array_bound(r->coeffs, 0, 8 * j + k, 0, MLKEM_Q)) + decreases(8 - k)) + { + r->coeffs[8 * j + k] = mlk_scalar_decompress_d11(t[k]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d11(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11) + int ret; + ret = mlk_poly_decompress_d11_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */ + + mlk_poly_decompress_d11_c(r, a); +} + +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +/* Reference: `poly_tobytes()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_tobytes_c(uint8_t r[MLKEM_POLYBYTES], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 2; i++) + __loop__(invariant(i <= MLKEM_N / 2) + decreases(MLKEM_N / 2 - i)) + { + /* The conversion to uint16_t is safe since we assume that + * the coefficients of `a` are non-negative. */ + const uint16_t t0 = (uint16_t)a->coeffs[2 * i]; + const uint16_t t1 = (uint16_t)a->coeffs[2 * i + 1]; + /* + * t0 and t1 are both < MLKEM_Q, so contain at most 12 bits each of + * significant data, so these can be packed into 24 bits or exactly + * 3 bytes, as follows. + */ + + /* Least significant bits 0 - 7 of t0. */ + r[3 * i + 0] = (uint8_t)(t0 & 0xFF); + + /* + * Most significant bits 8 - 11 of t0 become the least significant + * nibble of the second byte. The least significant 4 bits + * of t1 become the upper nibble of the second byte. + * + * The conversion to uint8_t does not alter the value. + */ + r[3 * i + 1] = (uint8_t)((t0 >> 8) | ((t1 << 4) & 0xF0)); + + /* Bits 4 - 11 of t1 become the third byte. The conversion to uint8_t + * does not alter the value because t1 is 12-bit wide. */ + r[3 * i + 2] = (uint8_t)(t1 >> 4); + } +} + +MLK_INTERNAL_API +void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a) +{ +#if defined(MLK_USE_NATIVE_POLY_TOBYTES) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_tobytes_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_TOBYTES */ + + mlk_poly_tobytes_c(r, a); +} + +/* Reference: `poly_frombytes()` in the reference implementation @[REF]. */ +MLK_STATIC_TESTABLE void mlk_poly_frombytes_c(mlk_poly *r, + const uint8_t a[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 2; i++) + __loop__( + invariant(i <= MLKEM_N / 2) + invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_UINT12_LIMIT)) + decreases(MLKEM_N / 2 - i)) + { + const uint8_t t0 = a[3 * i + 0]; + const uint8_t t1 = a[3 * i + 1]; + const uint8_t t2 = a[3 * i + 2]; + /* Safety: + * - The explicit cast to uint16_t ensures that << 8 does + * not signed-overflow even on a 16-bit system. + * - The cast to int16_t is safe due to the explicit 0xFFF truncation. + */ + r->coeffs[2 * i + 0] = (int16_t)(t0 | (((uint16_t)t1 << 8) & 0xFFF)); + r->coeffs[2 * i + 1] = (int16_t)((t1 >> 4) | (t2 << 4)); + } + + /* Note that the coefficients are not canonical */ + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +} + +MLK_INTERNAL_API +void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES]) +{ +#if defined(MLK_USE_NATIVE_POLY_FROMBYTES) + int ret; + ret = mlk_poly_frombytes_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_FROMBYTES */ + + mlk_poly_frombytes_c(r, a); +} + +/* Reference: `poly_frommsg()` in the reference implementation @[REF]. + * - We use a value barrier around the bit-selection mask to + * reduce the risk of compiler-introduced branches. + * The reference implementation contains the expression + * `(msg[i] >> j) & 1` which the compiler can reason must + * be either 0 or 1. */ +MLK_INTERNAL_API +void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES]) +{ + unsigned i; +#if (MLKEM_INDCPA_MSGBYTES != MLKEM_N / 8) +#error "MLKEM_INDCPA_MSGBYTES must be equal to MLKEM_N/8 bytes!" +#endif + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + for (j = 0; j < 8; j++) + __loop__( + invariant(i < MLKEM_N / 8 && j <= 8) + invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q)) + decreases(8 - j)) + { + /* mlk_ct_sel_int16(MLKEM_Q_HALF, 0, b) is `Decompress_1(b != 0)` + * as per @[FIPS203, Eq (4.8)]. */ + + /* Prevent the compiler from recognizing this as a bit selection */ + uint8_t mask = mlk_value_barrier_u8((uint8_t)(1u << j)); + r->coeffs[8 * i + j] = mlk_ct_sel_int16(MLKEM_Q_HALF, 0, msg[i] & mask); + } + } + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); +} + +/* Reference: `poly_tomsg()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r) +{ + unsigned i; + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + msg[i] = 0; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + decreases(8 - j)) + { + uint32_t t = mlk_scalar_compress_d1(r->coeffs[8 * i + j]); + msg[i] |= (uint8_t)(t << j); + } + } +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(compress) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/compress.h b/sdk/software/examples/mlkem_test/mlkem_native/src/compress.h new file mode 100644 index 0000000..f9df8e8 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/compress.h @@ -0,0 +1,599 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#ifndef MLK_COMPRESS_H +#define MLK_COMPRESS_H + + +#include "cbmc.h" +#include "common.h" +#include "debug.h" +#include "poly.h" +#include "verify.h" + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2 / MLKEM_Q). + * + * @spec{Compress_1 from @[FIPS203, Eq (4.7)].} + * + * @reference{Part of poly_tomsg() in the reference implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d1(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 2) + ensures(return_value == (((uint32_t)u * 2 + MLKEM_Q / 2) / MLKEM_Q) % 2) ) +{ + /* Compute as follows: + * ``` + * round(u * 2 / MLKEM_Q) + * = round(u * 2 * (2^31 / MLKEM_Q) / 2^31) + * ~= round(u * 2 * round(2^31 / MLKEM_Q) / 2^31) + * ``` + */ + /* check-magic: 1290168 == 2*round(2^31 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290168; + /* Unsigned shifting by 31 positions leaves only the top bit. */ + return (uint8_t)((d0 + ((uint32_t)1u << 30)) >> 31); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 16 / MLKEM_Q) % 16. + * + * @spec{Compress_4 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `poly_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d4(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 16) + ensures(return_value == (((uint32_t)u * 16 + MLKEM_Q / 2) / MLKEM_Q) % 16)) +{ + /* Compute as follows: + * ``` + * round(u * 16 / MLKEM_Q) + * = round(u * 16 * (2^28 / MLKEM_Q) / 2^28) + * ~= round(u * 16 * round(2^28 / MLKEM_Q) / 2^28) + * ``` + */ + /* check-magic: 1290160 == 16 * round(2^28 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290160; + /* The return value is < 16, so not altered by the conversion to uint8_t. */ + return (uint8_t)((d0 + ((uint32_t)1u << 27)) >> 28); /* round(d0/2^28) */ +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 16). + * + * @spec{Decompress_4 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `poly_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 16 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d4(uint8_t u) +__contract__( + requires(0 <= u && u < 16) + ensures(return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 8) >> 4); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 32 / MLKEM_Q) % 32. + * + * @spec{Compress_5 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `poly_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d5(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 32) + ensures(return_value == (((uint32_t)u * 32 + MLKEM_Q / 2) / MLKEM_Q) % 32) ) +{ + /* Compute as follows: + * ``` + * round(u * 32 / MLKEM_Q) + * = round(u * 32 * (2^27 / MLKEM_Q) / 2^27) + * ~= round(u * 32 * round(2^27 / MLKEM_Q) / 2^27) + * ``` + */ + /* check-magic: 1290176 == 2^5 * round(2^27 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290176; + /* The return value is < 32, so not altered by the conversion to uint8_t. */ + return (uint8_t)((d0 + ((uint32_t)1u << 26)) >> 27); /* round(d0/2^27) */ +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 32). + * + * @spec{Decompress_5 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `poly_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 32 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d5(uint8_t u) +__contract__( + requires(0 <= u && u < 32) + ensures(0 <= return_value && return_value <= MLKEM_Q - 1) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 16) >> 5); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2**10 / MLKEM_Q) % 2**10. + * + * @spec{Compress_10 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `polyvec_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint16_t mlk_scalar_compress_d10(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < (1u << 10)) + ensures(return_value == (((uint32_t)u * (1u << 10) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 10))) +{ + /* Compute as follows: + * ``` + * round(u * 1024 / MLKEM_Q) + * = round(u * 1024 * (2^33 / MLKEM_Q) / 2^33) + * ~= round(u * 1024 * round(2^33 / MLKEM_Q) / 2^33) + * ``` + */ + /* check-magic: 2642263040 == 2^10 * round(2^33 / MLKEM_Q) */ + uint64_t d0 = (uint64_t)u * 2642263040; + d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */ + return (d0 & 0x3FF); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 1024). + * + * @spec{Decompress_10 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `polyvec_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 1024 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d10(uint16_t u) +__contract__( + requires(0 <= u && u < 1024) + ensures(0 <= return_value && return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 512) >> 10); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2**11 / MLKEM_Q) % 2**11. + * + * @spec{Compress_11 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `polyvec_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint16_t mlk_scalar_compress_d11(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < (1u << 11)) + ensures(return_value == (((uint32_t)u * (1u << 11) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 11))) +{ + /* Compute as follows: + * ``` + * round(u * 2048 / MLKEM_Q) + * = round(u * 2048 * (2^33 / MLKEM_Q) / 2^33) + * ~= round(u * 2048 * round(2^33 / MLKEM_Q) / 2^33) + * ``` + */ + /* check-magic: 5284526080 == 2^11 * round(2^33 / MLKEM_Q) */ + uint64_t d0 = (uint64_t)u * 5284526080; + d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */ + return (d0 & 0x7FF); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 2048). + * + * @spec{Decompress_11 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `polyvec_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 2048 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d11(uint16_t u) +__contract__( + requires(0 <= u && u < 2048) + ensures(0 <= return_value && return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 1024) >> 11); +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +#define mlk_poly_compress_d4 MLK_NAMESPACE(poly_compress_d4) +/** + * Compression (4 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_4 (Compress_4 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L23], where `d_v=4` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const mlk_poly *a); + +#define mlk_poly_compress_d10 MLK_NAMESPACE(poly_compress_d10) +/** + * Compression (10 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_10 (Compress_10 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L22], where `d_u=10` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const mlk_poly *a); + +#define mlk_poly_decompress_d4 MLK_NAMESPACE(poly_decompress_d4) +/** + * De-serialization and subsequent decompression (4 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d4. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_4 (ByteDecode_4 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L4], where `d_v=4` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d4(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]); + +#define mlk_poly_decompress_d10 MLK_NAMESPACE(poly_decompress_d10) +/** + * De-serialization and subsequent decompression (10 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d10. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_10 (ByteDecode_10 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L3], where `d_u=10` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d10(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +#define mlk_poly_compress_d5 MLK_NAMESPACE(poly_compress_d5) +/** + * Compression (5 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_5 (Compress_5 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L23], where `d_v=5` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const mlk_poly *a); + +#define mlk_poly_compress_d11 MLK_NAMESPACE(poly_compress_d11) +/** + * Compression (11 bits) and subsequent serialization of a polynomial. + * + * @spec{`ByteEncode_11 (Compress_11 (a))`: ByteEncode_d @[FIPS203, + * Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to vectors as + * per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L22], where `d_u=11` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const mlk_poly *a); + +#define mlk_poly_decompress_d5 MLK_NAMESPACE(poly_decompress_d5) +/** + * De-serialization and subsequent decompression (5 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d5. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_5 (ByteDecode_5 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L4], where `d_v=5` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d5(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]); + +#define mlk_poly_decompress_d11 MLK_NAMESPACE(poly_decompress_d11) +/** + * De-serialization and subsequent decompression (11 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d11. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_11 (ByteDecode_11 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L3], where `d_u=11` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d11(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#define mlk_poly_tobytes MLK_NAMESPACE(poly_tobytes) +/** + * Serialization of a polynomial. Signed coefficients are converted to + * unsigned form before serialization. + * + * @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].} + * + * @param[out] r Output byte array (of MLKEM_POLYBYTES bytes). + * @param[in] a Input polynomial, with each coefficient in the range + * [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + + +#define mlk_poly_frombytes MLK_NAMESPACE(poly_frombytes) +/** + * De-serialization of a polynomial. + * + * @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].} + * + * @param[out] r Output polynomial, with each coefficient unsigned and in + * the range 0..4095. + * @param[in] a Input byte array (of MLKEM_POLYBYTES bytes). + */ +MLK_INTERNAL_API +void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); + + +#define mlk_poly_frommsg MLK_NAMESPACE(poly_frommsg) +/** + * Convert a 32-byte message to a polynomial. + * + * @spec{Implements `Decompress_1 (ByteDecode_1 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_1 (ByteDecode_1 (w))` appears in @[FIPS203, Algorithm 15 + * (K-PKE.Encrypt), L20].} + * + * @param[out] r Output polynomial. + * @param[in] msg Input message. + */ +MLK_INTERNAL_API +void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES]) +__contract__( + requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_tomsg MLK_NAMESPACE(poly_tomsg) +/** + * Convert a polynomial to a 32-byte message. + * + * @spec{Implements `ByteEncode_1 (Compress_1 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_1 (Compress_1 (w))` appears in @[FIPS203, Algorithm 14 + * (K-PKE.Decrypt), L7].} + * + * @param[out] msg Output message. + * @param[in] r Input polynomial. Coefficients must be unsigned canonical. + */ +MLK_INTERNAL_API +void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r) +__contract__( + requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(msg, MLKEM_INDCPA_MSGBYTES)) +); + +#endif /* !MLK_COMPRESS_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/debug.c b/sdk/software/examples/mlkem_test/mlkem_native/src/debug.c new file mode 100644 index 0000000..386f526 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/debug.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */ + +#include "common.h" + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG) + + +#include +#include +#include "debug.h" + +#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] " + +void mlk_debug_check_assert(const char *file, int line, const int val) +{ + if (val == 0) + { + fprintf(stderr, MLK_DEBUG_ERROR_HEADER "Assertion failed (value %d)\n", + file, line, val); + exit(1); + } +} + +void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr, + unsigned len, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + int err = 0; + unsigned i; + for (i = 0; i < len; i++) + { + int16_t val = ptr[i]; + if (!(val > lower_bound_exclusive && val < upper_bound_exclusive)) + { + fprintf( + stderr, + MLK_DEBUG_ERROR_HEADER + "Bounds assertion failed: Index %u, value %d out of bounds (%d,%d)\n", + file, line, i, (int)val, lower_bound_exclusive, + upper_bound_exclusive); + err = 1; + } + } + + if (err == 1) + { + exit(1); + } +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG */ + +MLK_EMPTY_CU(debug) + +#endif /* !(!MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_DEBUG_ERROR_HEADER diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/debug.h b/sdk/software/examples/mlkem_test/mlkem_native/src/debug.h new file mode 100644 index 0000000..ce4a33a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/debug.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_DEBUG_H +#define MLK_DEBUG_H +#include "common.h" + +#if defined(MLKEM_DEBUG) + +/** + * Check debug assertion. + * + * Prints an error message to stderr and calls exit(1) on failure. + * + * @param[in] file Filename. + * @param line Line number. + * @param val Value asserted to be non-zero. + */ +#define mlk_debug_check_assert MLK_NAMESPACE(mlkem_debug_assert) +void mlk_debug_check_assert(const char *file, int line, const int val); + +/** + * Check whether values in an array of int16_t are within specified bounds. + * + * Prints an error message to stderr and calls exit(1) on failure. + * + * @param[in] file Filename. + * @param line Line number. + * @param[in] ptr Base of array to be checked. + * @param len Number of int16_t in @p ptr. + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds MLK_NAMESPACE(mlkem_debug_check_bounds) +void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr, + unsigned len, int lower_bound_exclusive, + int upper_bound_exclusive); + +/* Check assertion, calling exit() upon failure + * + * val: Value that's asserted to be non-zero + */ +#define mlk_assert(val) mlk_debug_check_assert(__FILE__, __LINE__, (val)) + +/* Check bounds in array of int16_t's + * ptr: Base of int16_t array; will be explicitly cast to int16_t*, + * so you may pass a byte-compatible type such as mlk_poly or mlk_polyvec. + * len: Number of int16_t in array + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + mlk_debug_check_bounds(__FILE__, __LINE__, (const int16_t *)(ptr), (len), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in array of int16_t's + * ptr: Base of array, expression of type int16_t* + * len: Number of int16_t in array + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + mlk_assert_bound((ptr), (len), (-(value_abs_bd) + 1), (value_abs_bd)) + +/* Version of bounds assertions for 2-dimensional arrays */ +#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \ + mlk_assert_bound((ptr), ((len0) * (len1)), (value_lb), (value_ub)) + +#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \ + mlk_assert_abs_bound((ptr), ((len0) * (len1)), (value_abs_bd)) + +/* When running CBMC, convert debug assertions into proof obligations */ +#elif defined(CBMC) +#include "cbmc.h" + +#define mlk_assert(val) cassert(val) + +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + cassert(array_bound(((int16_t *)(ptr)), 0, (len), (value_lb), (value_ub))) + +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + cassert(array_abs_bound(((int16_t *)(ptr)), 0, (len), (value_abs_bd))) + +/* Because of https://github.com/diffblue/cbmc/issues/8570, we can't + * just use a single flattened array_bound(...) here. */ +#define mlk_assert_bound_2d(ptr, M, N, value_lb, value_ub) \ + cassert(forall(kN, 0, (M), \ + array_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \ + (value_lb), (value_ub)))) + +#define mlk_assert_abs_bound_2d(ptr, M, N, value_abs_bd) \ + cassert(forall(kN, 0, (M), \ + array_abs_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \ + (value_abs_bd)))) + +#else /* !MLKEM_DEBUG && CBMC */ + +#define mlk_assert(val) \ + do \ + { \ + } while (0) +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + do \ + { \ + } while (0) +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \ + do \ + { \ + } while (0) + + +#endif /* !MLKEM_DEBUG && !CBMC */ +#endif /* !MLK_DEBUG_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.c new file mode 100644 index 0000000..33883aa --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [mupq] + * Common files for pqm4, pqm3, pqriscv + * Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen + * https://github.com/mupq/mupq + * + * - [supercop] + * SUPERCOP benchmarking framework + * Daniel J. Bernstein + * http://bench.cr.yp.to/supercop.html + * + * - [tweetfips] + * 'tweetfips202' FIPS202 implementation + * Van Assche, Bernstein, Schwabe + * https://keccak.team/2015/tweetfips202.html + */ + +/* Based on the CC0 implementation from @[mupq] and the public domain + * implementation @[supercop, crypto_hash/keccakc512/simple/] + * by Ronny Van Keer, and the public domain @[tweetfips] implementation. */ + +#include "../common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "../verify.h" +#include "fips202.h" +#include "keccakf1600.h" + +/** + * Absorb step of Keccak; non-incremental, starts by zeroeing the state. + * + * @warning Must only be called once. + * + * @param[out] s Pointer to (uninitialized) output Keccak state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + * @param[in] m Input to be absorbed into @p s. + * @param mlen Length of input in bytes. + * @param p Domain-separation byte for different Keccak-derived + * functions. + */ +static void mlk_keccak_absorb_once(uint64_t *s, unsigned r, const uint8_t *m, + size_t mlen, uint8_t p) +__contract__( + requires(mlen <= MLK_MAX_BUFFER_SIZE) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(m, mlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES))) +{ + /* Initialize state */ + size_t i; + for (i = 0; i < 25; ++i) + __loop__(invariant(i <= 25) + decreases(25 - i)) + { + s[i] = 0; + } + + while (mlen >= r) + __loop__( + assigns(mlen, m, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + invariant(mlen <= loop_entry(mlen)) + invariant(m == loop_entry(m) + (loop_entry(mlen) - mlen)) + decreases(mlen)) + { + mlk_keccakf1600_xor_bytes(s, m, 0, r); + mlk_keccakf1600_permute(s); + mlen -= r; + m += r; + } + + /* At this point, mlen < r, so the truncations to unsigned are safe below. */ + + if (mlen > 0) + { + mlk_keccakf1600_xor_bytes(s, m, 0, (unsigned int)mlen); + } + + if (mlen == r - 1) + { + p |= 128; + mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1); + } + else + { + mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1); + p = 128; + mlk_keccakf1600_xor_bytes(s, &p, r - 1, 1); + } +} + +/** + * Block-level Keccak squeeze. + * + * @param[out] h Output bytes. + * @param nblocks Number of blocks to be squeezed. + * @param[in,out] s Input/output state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + */ +static void mlk_keccak_squeezeblocks(uint8_t *h, size_t nblocks, uint64_t *s, + unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(h, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(h, nblocks * r))) +{ + while (nblocks > 0) + __loop__( + assigns(h, nblocks, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES), + memory_slice(h, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks) && + h == loop_entry(h) + r * (loop_entry(nblocks) - nblocks)) + decreases(nblocks)) + { + mlk_keccakf1600_permute(s); + mlk_keccakf1600_extract_bytes(s, h, 0, r); + h += r; + nblocks--; + } +} + +/** + * Keccak squeeze; can be called on byte-level. + * + * @warning Must only be called once. + * + * @param[out] h Output bytes. + * @param outlen Number of bytes to be squeezed. + * @param[in,out] s Keccak state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + */ +static void mlk_keccak_squeeze_once(uint8_t *h, size_t outlen, uint64_t *s, + unsigned r) +__contract__( + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(h, outlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(h, outlen))) +{ + size_t len; + while (outlen > 0) + __loop__( + assigns(len, h, outlen, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES), + memory_slice(h, outlen)) + invariant(outlen <= loop_entry(outlen) && + h == loop_entry(h) + (loop_entry(outlen) - outlen)) + decreases(outlen)) + { + mlk_keccakf1600_permute(s); + + if (outlen < r) + { + len = outlen; + } + else + { + len = r; + } + mlk_keccakf1600_extract_bytes(s, h, 0, (unsigned int)len); + h += len; + outlen -= len; + } +} + +void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input, + size_t inlen) +{ + mlk_keccak_absorb_once(state->ctx, SHAKE128_RATE, input, inlen, 0x1F); +} + +void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks, + mlk_shake128ctx *state) +{ + mlk_keccak_squeezeblocks(output, nblocks, state->ctx, SHAKE128_RATE); +} + +void mlk_shake128_init(mlk_shake128ctx *state) { (void)state; } +void mlk_shake128_release(mlk_shake128ctx *state) +{ + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(state, sizeof(mlk_shake128ctx)); +} + +typedef mlk_shake128ctx mlk_shake256ctx; +void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input, + size_t inlen) +{ + mlk_shake256ctx state; + /* Absorb input */ + mlk_keccak_absorb_once(state.ctx, SHAKE256_RATE, input, inlen, 0x1F); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, outlen, state.ctx, SHAKE256_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(&state, sizeof(state)); +} + +void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen) +{ + uint64_t ctx[25]; + /* Absorb input */ + mlk_keccak_absorb_once(ctx, SHA3_256_RATE, input, inlen, 0x06); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, 32, ctx, SHA3_256_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(ctx, sizeof(ctx)); +} + +void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen) +{ + uint64_t ctx[25]; + /* Absorb input */ + mlk_keccak_absorb_once(ctx, SHA3_512_RATE, input, inlen, 0x06); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, 64, ctx, SHA3_512_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(ctx, sizeof(ctx)); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.h new file mode 100644 index 0000000..6349ae1 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_FIPS202_H +#define MLK_FIPS202_FIPS202_H + +#include "../cbmc.h" +#include "../common.h" + +#define SHAKE128_RATE 168 +#define SHAKE256_RATE 136 +#define SHA3_256_RATE 136 +#define SHA3_384_RATE 104 +#define SHA3_512_RATE 72 + +/** Context for the non-incremental SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[25]; /**< Keccak state. */ +} MLK_ALIGN mlk_shake128ctx; + +#define mlk_shake128_absorb_once MLK_NAMESPACE(shake128_absorb_once) +/** + * One-shot absorb step of the SHAKE128 XOF. + * + * For call-sites (in mlkem-native): + * - This function MUST ONLY be called straight after mlk_shake128_init(). + * - This function MUST ONLY be called once. + * + * Consequently, for providers of custom FIPS202 code to be used with + * mlkem-native: + * - You may assume that the input context is freshly initialized via + * mlk_shake128_init(). + * - You may assume that this function is called exactly once. + * + * @param[in,out] state SHAKE128 context. + * @param[in] input Input to be absorbed into the state. + * @param inlen Length of input in bytes. + */ +void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input, + size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mlk_shake128ctx))) + requires(memory_no_alias(input, inlen)) + assigns(memory_slice(state, sizeof(mlk_shake128ctx))) +); + +#define mlk_shake128_squeezeblocks MLK_NAMESPACE(shake128_squeezeblocks) +/** + * Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes + * each. Modifies the state. Can be called multiple times to keep squeezing, + * i.e., is incremental. + * + * @param[out] output Output blocks. + * @param nblocks Number of blocks to be squeezed (written to output). + * @param[in,out] state Keccak state. + */ +void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks, + mlk_shake128ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mlk_shake128ctx))) + requires(memory_no_alias(output, nblocks * SHAKE128_RATE)) + assigns(memory_slice(output, nblocks * SHAKE128_RATE), memory_slice(state, sizeof(mlk_shake128ctx))) +); + +#define mlk_shake128_init MLK_NAMESPACE(shake128_init) +void mlk_shake128_init(mlk_shake128ctx *state); + +#define mlk_shake128_release MLK_NAMESPACE(shake128_release) +void mlk_shake128_release(mlk_shake128ctx *state); + +/* One-stop SHAKE256 call. Aliasing between input and + * output is not permitted */ +#define mlk_shake256 MLK_NAMESPACE(shake256) +/** + * SHAKE256 XOF with non-incremental API. + * + * @param[out] output Output buffer. + * @param outlen Requested output length in bytes. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input, + size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, outlen)) + assigns(memory_slice(output, outlen)) +); + +/* One-stop SHA3_256 call. Aliasing between input and + * output is not permitted */ +#define SHA3_256_HASHBYTES 32 +#define mlk_sha3_256 MLK_NAMESPACE(sha3_256) +/** + * SHA3-256 with non-incremental API. + * + * @param[out] output Output buffer. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, SHA3_256_HASHBYTES)) + assigns(memory_slice(output, SHA3_256_HASHBYTES)) +); + +/* One-stop SHA3_512 call. Aliasing between input and + * output is not permitted */ +#define SHA3_512_HASHBYTES 64 +#define mlk_sha3_512 MLK_NAMESPACE(sha3_512) +/** + * SHA3-512 with non-incremental API. + * + * @param[out] output Output buffer. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, SHA3_512_HASHBYTES)) + assigns(memory_slice(output, SHA3_512_HASHBYTES)) +); + +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) || \ + !defined(MLK_USE_FIPS202_X4_NATIVE) +/* If you provide your own FIPS-202 implementation where the x4- + * Keccak-f1600-x4 implementation falls back to 4-fold Keccak-f1600, + * set this to gain a small speedup. */ +#define FIPS202_X4_DEFAULT_IMPLEMENTATION +#endif /* !MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 || !MLK_USE_FIPS202_X4_NATIVE \ + */ + + +#endif /* !MLK_FIPS202_FIPS202_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.c new file mode 100644 index 0000000..400ca07 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#include "../common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "../verify.h" +#include "fips202.h" +#include "fips202x4.h" +#include "keccakf1600.h" + +typedef mlk_shake128x4ctx mlk_shake256x4_ctx; + +static void mlk_keccak_absorb_once_x4(uint64_t *s, unsigned r, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, + size_t inlen, uint8_t p) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))) +{ + while (inlen >= r) + __loop__( + assigns(inlen, in0, in1, in2, in3, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + invariant(inlen <= loop_entry(inlen)) + invariant(in0 == loop_entry(in0) + (loop_entry(inlen) - inlen)) + invariant(in1 == loop_entry(in1) + (loop_entry(inlen) - inlen)) + invariant(in2 == loop_entry(in2) + (loop_entry(inlen) - inlen)) + invariant(in3 == loop_entry(in3) + (loop_entry(inlen) - inlen)) + decreases(inlen)) + { + mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, r); + mlk_keccakf1600x4_permute(s); + + in0 += r; + in1 += r; + in2 += r; + in3 += r; + inlen -= r; + } + + /* At this point, inlen < r, so the truncations to unsigned are safe below. */ + + if (inlen > 0) + { + mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, (unsigned int)inlen); + } + + if (inlen == r - 1) + { + p |= 128; + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1); + } + else + { + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1); + p = 128; + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, r - 1, 1); + } +} + +static void mlk_keccak_squeezeblocks_x4(uint8_t *out0, uint8_t *out1, + uint8_t *out2, uint8_t *out3, + size_t nblocks, uint64_t *s, unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(r == SHAKE128_RATE || r == SHAKE256_RATE) + requires(nblocks <= (MLK_MAX_BUFFER_SIZE / SHAKE256_RATE)) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(out0, nblocks * r)) + requires(memory_no_alias(out1, nblocks * r)) + requires(memory_no_alias(out2, nblocks * r)) + requires(memory_no_alias(out3, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + assigns(memory_slice(out0, nblocks * r)) + assigns(memory_slice(out1, nblocks * r)) + assigns(memory_slice(out2, nblocks * r)) + assigns(memory_slice(out3, nblocks * r))) +{ + size_t current_offset = 0; + while (nblocks > 0) + __loop__( + assigns(nblocks, current_offset, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY), + memory_slice(out0, nblocks * r), + memory_slice(out1, nblocks * r), + memory_slice(out2, nblocks * r), + memory_slice(out3, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks)) + invariant(current_offset == (loop_entry(nblocks) - nblocks) * r) + decreases(nblocks)) + { + mlk_keccakf1600x4_permute(s); + mlk_keccakf1600x4_extract_bytes( + s, &out0[current_offset], &out1[current_offset], &out2[current_offset], + &out3[current_offset], 0, r); + current_offset += r; + nblocks--; + } +} + +void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + const uint8_t *in3, size_t inlen) +{ + mlk_memset(state, 0, sizeof(mlk_shake128x4ctx)); + mlk_keccak_absorb_once_x4(state->ctx, SHAKE128_RATE, in0, in1, in2, in3, + inlen, 0x1F); +} + +void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + uint8_t *out3, size_t nblocks, + mlk_shake128x4ctx *state) +{ + mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx, + SHAKE128_RATE); +} + +void mlk_shake128x4_init(mlk_shake128x4ctx *state) { (void)state; } +void mlk_shake128x4_release(mlk_shake128x4ctx *state) +{ + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(state, sizeof(mlk_shake128x4ctx)); +} + +static void mlk_shake256x4_absorb_once(mlk_shake256x4_ctx *state, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, + size_t inlen) +{ + mlk_memset(state, 0, sizeof(mlk_shake128x4ctx)); + mlk_keccak_absorb_once_x4(state->ctx, SHAKE256_RATE, in0, in1, in2, in3, + inlen, 0x1F); +} + +static void mlk_shake256x4_squeezeblocks(uint8_t *out0, uint8_t *out1, + uint8_t *out2, uint8_t *out3, + size_t nblocks, + mlk_shake256x4_ctx *state) +{ + mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx, + SHAKE256_RATE); +} + +void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3, + size_t outlen, const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, size_t inlen) +{ + mlk_shake256x4_ctx statex; + size_t nblocks = outlen / SHAKE256_RATE; + uint8_t tmp0[SHAKE256_RATE]; + uint8_t tmp1[SHAKE256_RATE]; + uint8_t tmp2[SHAKE256_RATE]; + uint8_t tmp3[SHAKE256_RATE]; + + mlk_shake256x4_absorb_once(&statex, in0, in1, in2, in3, inlen); + mlk_shake256x4_squeezeblocks(out0, out1, out2, out3, nblocks, &statex); + + out0 += nblocks * SHAKE256_RATE; + out1 += nblocks * SHAKE256_RATE; + out2 += nblocks * SHAKE256_RATE; + out3 += nblocks * SHAKE256_RATE; + + outlen -= nblocks * SHAKE256_RATE; + + if (outlen) + { + mlk_shake256x4_squeezeblocks(tmp0, tmp1, tmp2, tmp3, 1, &statex); + mlk_memcpy(out0, tmp0, outlen); + mlk_memcpy(out1, tmp1, outlen); + mlk_memcpy(out2, tmp2, outlen); + mlk_memcpy(out3, tmp3, outlen); + } + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(&statex, sizeof(statex)); + mlk_zeroize(tmp0, sizeof(tmp0)); + mlk_zeroize(tmp1, sizeof(tmp1)); + mlk_zeroize(tmp2, sizeof(tmp2)); + mlk_zeroize(tmp3, sizeof(tmp3)); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202x4) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.h new file mode 100644 index 0000000..a0490a0 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/fips202x4.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_FIPS202X4_H +#define MLK_FIPS202_FIPS202X4_H + + +#include "../cbmc.h" +#include "../common.h" + +#include "fips202.h" +#include "keccakf1600.h" + +/** Context for the non-incremental 4-way SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[MLK_KECCAK_LANES * + MLK_KECCAK_WAY]; /**< 4-way Keccak state, stored sequentially. */ +} MLK_ALIGN mlk_shake128x4ctx; + +#define mlk_shake128x4_absorb_once MLK_NAMESPACE(shake128x4_absorb_once) +void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + const uint8_t *in3, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + assigns(memory_slice(state, sizeof(mlk_shake128x4ctx))) +); + +#define mlk_shake128x4_squeezeblocks MLK_NAMESPACE(shake128x4_squeezeblocks) +void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + uint8_t *out3, size_t nblocks, + mlk_shake128x4ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out3, nblocks * SHAKE128_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE128_RATE), + memory_slice(out1, nblocks * SHAKE128_RATE), + memory_slice(out2, nblocks * SHAKE128_RATE), + memory_slice(out3, nblocks * SHAKE128_RATE), + memory_slice(state, sizeof(mlk_shake128x4ctx))) +); + +#define mlk_shake128x4_init MLK_NAMESPACE(shake128x4_init) +void mlk_shake128x4_init(mlk_shake128x4ctx *state); + +#define mlk_shake128x4_release MLK_NAMESPACE(shake128x4_release) +void mlk_shake128x4_release(mlk_shake128x4ctx *state); + +#define mlk_shake256x4 MLK_NAMESPACE(shake256x4) +void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3, + size_t outlen, const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + requires(memory_no_alias(out0, outlen)) + requires(memory_no_alias(out1, outlen)) + requires(memory_no_alias(out2, outlen)) + requires(memory_no_alias(out3, outlen)) + assigns(memory_slice(out0, outlen)) + assigns(memory_slice(out1, outlen)) + assigns(memory_slice(out2, outlen)) + assigns(memory_slice(out3, outlen)) +); + +#endif /* !MLK_FIPS202_FIPS202X4_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.c new file mode 100644 index 0000000..586b283 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.c @@ -0,0 +1,499 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [mupq] + * Common files for pqm4, pqm3, pqriscv + * Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen + * https://github.com/mupq/mupq + * + * - [supercop] + * SUPERCOP benchmarking framework + * Daniel J. Bernstein + * http://bench.cr.yp.to/supercop.html + * + * - [tweetfips] + * 'tweetfips202' FIPS202 implementation + * Van Assche, Bernstein, Schwabe + * https://keccak.team/2015/tweetfips202.html + */ + +/* Based on the CC0 implementation from @[mupq] and the public domain + * implementation @[supercop, crypto_hash/keccakc512/simple/] + * by Ronny Van Keer, and the public domain @[tweetfips] implementation. */ + + +#include "keccakf1600.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#define MLK_KECCAK_NROUNDS 24 +#define MLK_KECCAK_ROL(a, offset) (((a) << (offset)) ^ ((a) >> (64 - (offset)))) + +void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data, + unsigned offset, unsigned length) +{ + unsigned i; +#if defined(MLK_SYS_LITTLE_ENDIAN) + uint8_t *state_ptr = (uint8_t *)state + offset; + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + data[i] = state_ptr[i]; + } +#else /* MLK_SYS_LITTLE_ENDIAN */ + /* Portable version */ + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + data[i] = (state[(offset + i) >> 3] >> (8 * ((offset + i) & 0x07))) & 0xFF; + } +#endif /* !MLK_SYS_LITTLE_ENDIAN */ +} + +void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, + unsigned offset, unsigned length) +{ + unsigned i; +#if defined(MLK_SYS_LITTLE_ENDIAN) + uint8_t *state_ptr = (uint8_t *)state + offset; + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + state_ptr[i] ^= data[i]; + } +#else /* MLK_SYS_LITTLE_ENDIAN */ + /* Portable version */ + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + state[(offset + i) >> 3] ^= (uint64_t)data[i] + << (8 * ((offset + i) & 0x07)); + } +#endif /* !MLK_SYS_LITTLE_ENDIAN */ +} + +static void mlk_keccakf1600x4_extract_bytes_c(uint64_t *state, + unsigned char *data0, + unsigned char *data1, + unsigned char *data2, + unsigned char *data3, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) +) +{ + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 0, data0, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 1, data1, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 2, data2, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 3, data3, offset, + length); +} + +void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned char *data3, unsigned offset, + unsigned length) +{ +#if defined(MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE) + if (mlk_keccakf1600_extract_bytes_x4_native(state, data0, data1, data2, data3, + offset, length) == + MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE */ + mlk_keccakf1600x4_extract_bytes_c(state, data0, data1, data2, data3, offset, + length); +} + +static void mlk_keccakf1600x4_xor_bytes_c(uint64_t *state, + const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +) +{ + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 0, data0, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 1, data1, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 2, data2, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 3, data3, offset, + length); +} + +void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, unsigned offset, + unsigned length) +{ +#if defined(MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE) + if (mlk_keccakf1600_xor_bytes_x4_native(state, data0, data1, data2, data3, + offset, + length) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE */ + mlk_keccakf1600x4_xor_bytes_c(state, data0, data1, data2, data3, offset, + length); +} + +void mlk_keccakf1600x4_permute(uint64_t *state) +{ +#if defined(MLK_USE_FIPS202_X4_NATIVE) + if (mlk_keccak_f1600_x4_native(state) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_FIPS202_X4_NATIVE */ + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 0); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 1); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 2); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 3); +} + +static const uint64_t mlk_KeccakF_RoundConstants[MLK_KECCAK_NROUNDS] = { + (uint64_t)0x0000000000000001ULL, (uint64_t)0x0000000000008082ULL, + (uint64_t)0x800000000000808aULL, (uint64_t)0x8000000080008000ULL, + (uint64_t)0x000000000000808bULL, (uint64_t)0x0000000080000001ULL, + (uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008009ULL, + (uint64_t)0x000000000000008aULL, (uint64_t)0x0000000000000088ULL, + (uint64_t)0x0000000080008009ULL, (uint64_t)0x000000008000000aULL, + (uint64_t)0x000000008000808bULL, (uint64_t)0x800000000000008bULL, + (uint64_t)0x8000000000008089ULL, (uint64_t)0x8000000000008003ULL, + (uint64_t)0x8000000000008002ULL, (uint64_t)0x8000000000000080ULL, + (uint64_t)0x000000000000800aULL, (uint64_t)0x800000008000000aULL, + (uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008080ULL, + (uint64_t)0x0000000080000001ULL, (uint64_t)0x8000000080008008ULL}; + +MLK_STATIC_TESTABLE +void mlk_keccakf1600_permute_c(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +) +{ + unsigned round; + + uint64_t Aba, Abe, Abi, Abo, Abu; + uint64_t Aga, Age, Agi, Ago, Agu; + uint64_t Aka, Ake, Aki, Ako, Aku; + uint64_t Ama, Ame, Ami, Amo, Amu; + uint64_t Asa, Ase, Asi, Aso, Asu; + uint64_t BCa, BCe, BCi, BCo, BCu; + uint64_t Da, De, Di, Do, Du; + uint64_t Eba, Ebe, Ebi, Ebo, Ebu; + uint64_t Ega, Ege, Egi, Ego, Egu; + uint64_t Eka, Eke, Eki, Eko, Eku; + uint64_t Ema, Eme, Emi, Emo, Emu; + uint64_t Esa, Ese, Esi, Eso, Esu; + + /* copyFromState(A, state) */ + Aba = state[0]; + Abe = state[1]; + Abi = state[2]; + Abo = state[3]; + Abu = state[4]; + Aga = state[5]; + Age = state[6]; + Agi = state[7]; + Ago = state[8]; + Agu = state[9]; + Aka = state[10]; + Ake = state[11]; + Aki = state[12]; + Ako = state[13]; + Aku = state[14]; + Ama = state[15]; + Ame = state[16]; + Ami = state[17]; + Amo = state[18]; + Amu = state[19]; + Asa = state[20]; + Ase = state[21]; + Asi = state[22]; + Aso = state[23]; + Asu = state[24]; + + for (round = 0; round < MLK_KECCAK_NROUNDS; round += 2) + __loop__(invariant(round <= MLK_KECCAK_NROUNDS && round % 2 == 0) + decreases(MLK_KECCAK_NROUNDS - round)) + { + /* prepareTheta */ + BCa = Aba ^ Aga ^ Aka ^ Ama ^ Asa; + BCe = Abe ^ Age ^ Ake ^ Ame ^ Ase; + BCi = Abi ^ Agi ^ Aki ^ Ami ^ Asi; + BCo = Abo ^ Ago ^ Ako ^ Amo ^ Aso; + BCu = Abu ^ Agu ^ Aku ^ Amu ^ Asu; + + /* thetaRhoPiChiIotaPrepareTheta(round, A, E) */ + Da = BCu ^ MLK_KECCAK_ROL(BCe, 1); + De = BCa ^ MLK_KECCAK_ROL(BCi, 1); + Di = BCe ^ MLK_KECCAK_ROL(BCo, 1); + Do = BCi ^ MLK_KECCAK_ROL(BCu, 1); + Du = BCo ^ MLK_KECCAK_ROL(BCa, 1); + + Aba ^= Da; + BCa = Aba; + Age ^= De; + BCe = MLK_KECCAK_ROL(Age, 44); + Aki ^= Di; + BCi = MLK_KECCAK_ROL(Aki, 43); + Amo ^= Do; + BCo = MLK_KECCAK_ROL(Amo, 21); + Asu ^= Du; + BCu = MLK_KECCAK_ROL(Asu, 14); + Eba = BCa ^ ((~BCe) & BCi); + Eba ^= (uint64_t)mlk_KeccakF_RoundConstants[round]; + Ebe = BCe ^ ((~BCi) & BCo); + Ebi = BCi ^ ((~BCo) & BCu); + Ebo = BCo ^ ((~BCu) & BCa); + Ebu = BCu ^ ((~BCa) & BCe); + + Abo ^= Do; + BCa = MLK_KECCAK_ROL(Abo, 28); + Agu ^= Du; + BCe = MLK_KECCAK_ROL(Agu, 20); + Aka ^= Da; + BCi = MLK_KECCAK_ROL(Aka, 3); + Ame ^= De; + BCo = MLK_KECCAK_ROL(Ame, 45); + Asi ^= Di; + BCu = MLK_KECCAK_ROL(Asi, 61); + Ega = BCa ^ ((~BCe) & BCi); + Ege = BCe ^ ((~BCi) & BCo); + Egi = BCi ^ ((~BCo) & BCu); + Ego = BCo ^ ((~BCu) & BCa); + Egu = BCu ^ ((~BCa) & BCe); + + Abe ^= De; + BCa = MLK_KECCAK_ROL(Abe, 1); + Agi ^= Di; + BCe = MLK_KECCAK_ROL(Agi, 6); + Ako ^= Do; + BCi = MLK_KECCAK_ROL(Ako, 25); + Amu ^= Du; + BCo = MLK_KECCAK_ROL(Amu, 8); + Asa ^= Da; + BCu = MLK_KECCAK_ROL(Asa, 18); + Eka = BCa ^ ((~BCe) & BCi); + Eke = BCe ^ ((~BCi) & BCo); + Eki = BCi ^ ((~BCo) & BCu); + Eko = BCo ^ ((~BCu) & BCa); + Eku = BCu ^ ((~BCa) & BCe); + + Abu ^= Du; + BCa = MLK_KECCAK_ROL(Abu, 27); + Aga ^= Da; + BCe = MLK_KECCAK_ROL(Aga, 36); + Ake ^= De; + BCi = MLK_KECCAK_ROL(Ake, 10); + Ami ^= Di; + BCo = MLK_KECCAK_ROL(Ami, 15); + Aso ^= Do; + BCu = MLK_KECCAK_ROL(Aso, 56); + Ema = BCa ^ ((~BCe) & BCi); + Eme = BCe ^ ((~BCi) & BCo); + Emi = BCi ^ ((~BCo) & BCu); + Emo = BCo ^ ((~BCu) & BCa); + Emu = BCu ^ ((~BCa) & BCe); + + Abi ^= Di; + BCa = MLK_KECCAK_ROL(Abi, 62); + Ago ^= Do; + BCe = MLK_KECCAK_ROL(Ago, 55); + Aku ^= Du; + BCi = MLK_KECCAK_ROL(Aku, 39); + Ama ^= Da; + BCo = MLK_KECCAK_ROL(Ama, 41); + Ase ^= De; + BCu = MLK_KECCAK_ROL(Ase, 2); + Esa = BCa ^ ((~BCe) & BCi); + Ese = BCe ^ ((~BCi) & BCo); + Esi = BCi ^ ((~BCo) & BCu); + Eso = BCo ^ ((~BCu) & BCa); + Esu = BCu ^ ((~BCa) & BCe); + + /* prepareTheta */ + BCa = Eba ^ Ega ^ Eka ^ Ema ^ Esa; + BCe = Ebe ^ Ege ^ Eke ^ Eme ^ Ese; + BCi = Ebi ^ Egi ^ Eki ^ Emi ^ Esi; + BCo = Ebo ^ Ego ^ Eko ^ Emo ^ Eso; + BCu = Ebu ^ Egu ^ Eku ^ Emu ^ Esu; + + /* thetaRhoPiChiIotaPrepareTheta(round+1, E, A) */ + Da = BCu ^ MLK_KECCAK_ROL(BCe, 1); + De = BCa ^ MLK_KECCAK_ROL(BCi, 1); + Di = BCe ^ MLK_KECCAK_ROL(BCo, 1); + Do = BCi ^ MLK_KECCAK_ROL(BCu, 1); + Du = BCo ^ MLK_KECCAK_ROL(BCa, 1); + + Eba ^= Da; + BCa = Eba; + Ege ^= De; + BCe = MLK_KECCAK_ROL(Ege, 44); + Eki ^= Di; + BCi = MLK_KECCAK_ROL(Eki, 43); + Emo ^= Do; + BCo = MLK_KECCAK_ROL(Emo, 21); + Esu ^= Du; + BCu = MLK_KECCAK_ROL(Esu, 14); + Aba = BCa ^ ((~BCe) & BCi); + Aba ^= (uint64_t)mlk_KeccakF_RoundConstants[round + 1]; + Abe = BCe ^ ((~BCi) & BCo); + Abi = BCi ^ ((~BCo) & BCu); + Abo = BCo ^ ((~BCu) & BCa); + Abu = BCu ^ ((~BCa) & BCe); + + Ebo ^= Do; + BCa = MLK_KECCAK_ROL(Ebo, 28); + Egu ^= Du; + BCe = MLK_KECCAK_ROL(Egu, 20); + Eka ^= Da; + BCi = MLK_KECCAK_ROL(Eka, 3); + Eme ^= De; + BCo = MLK_KECCAK_ROL(Eme, 45); + Esi ^= Di; + BCu = MLK_KECCAK_ROL(Esi, 61); + Aga = BCa ^ ((~BCe) & BCi); + Age = BCe ^ ((~BCi) & BCo); + Agi = BCi ^ ((~BCo) & BCu); + Ago = BCo ^ ((~BCu) & BCa); + Agu = BCu ^ ((~BCa) & BCe); + + Ebe ^= De; + BCa = MLK_KECCAK_ROL(Ebe, 1); + Egi ^= Di; + BCe = MLK_KECCAK_ROL(Egi, 6); + Eko ^= Do; + BCi = MLK_KECCAK_ROL(Eko, 25); + Emu ^= Du; + BCo = MLK_KECCAK_ROL(Emu, 8); + Esa ^= Da; + BCu = MLK_KECCAK_ROL(Esa, 18); + Aka = BCa ^ ((~BCe) & BCi); + Ake = BCe ^ ((~BCi) & BCo); + Aki = BCi ^ ((~BCo) & BCu); + Ako = BCo ^ ((~BCu) & BCa); + Aku = BCu ^ ((~BCa) & BCe); + + Ebu ^= Du; + BCa = MLK_KECCAK_ROL(Ebu, 27); + Ega ^= Da; + BCe = MLK_KECCAK_ROL(Ega, 36); + Eke ^= De; + BCi = MLK_KECCAK_ROL(Eke, 10); + Emi ^= Di; + BCo = MLK_KECCAK_ROL(Emi, 15); + Eso ^= Do; + BCu = MLK_KECCAK_ROL(Eso, 56); + Ama = BCa ^ ((~BCe) & BCi); + Ame = BCe ^ ((~BCi) & BCo); + Ami = BCi ^ ((~BCo) & BCu); + Amo = BCo ^ ((~BCu) & BCa); + Amu = BCu ^ ((~BCa) & BCe); + + Ebi ^= Di; + BCa = MLK_KECCAK_ROL(Ebi, 62); + Ego ^= Do; + BCe = MLK_KECCAK_ROL(Ego, 55); + Eku ^= Du; + BCi = MLK_KECCAK_ROL(Eku, 39); + Ema ^= Da; + BCo = MLK_KECCAK_ROL(Ema, 41); + Ese ^= De; + BCu = MLK_KECCAK_ROL(Ese, 2); + Asa = BCa ^ ((~BCe) & BCi); + Ase = BCe ^ ((~BCi) & BCo); + Asi = BCi ^ ((~BCo) & BCu); + Aso = BCo ^ ((~BCu) & BCa); + Asu = BCu ^ ((~BCa) & BCe); + } + + /* copyToState(state, A) */ + state[0] = Aba; + state[1] = Abe; + state[2] = Abi; + state[3] = Abo; + state[4] = Abu; + state[5] = Aga; + state[6] = Age; + state[7] = Agi; + state[8] = Ago; + state[9] = Agu; + state[10] = Aka; + state[11] = Ake; + state[12] = Aki; + state[13] = Ako; + state[14] = Aku; + state[15] = Ama; + state[16] = Ame; + state[17] = Ami; + state[18] = Amo; + state[19] = Amu; + state[20] = Asa; + state[21] = Ase; + state[22] = Asi; + state[23] = Aso; + state[24] = Asu; +} + +void mlk_keccakf1600_permute(uint64_t *state) +{ +#if defined(MLK_USE_FIPS202_X1_NATIVE) + if (mlk_keccak_f1600_x1_native(state) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_FIPS202_X1_NATIVE */ + mlk_keccakf1600_permute_c(state); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(keccakf1600) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_KECCAK_NROUNDS +#undef MLK_KECCAK_ROL diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.h new file mode 100644 index 0000000..c26c36a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/keccakf1600.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_KECCAKF1600_H +#define MLK_FIPS202_KECCAKF1600_H +#include "../cbmc.h" +#include "../common.h" + +#define MLK_KECCAK_LANES 25 +#define MLK_KECCAK_WAY 4 + +/* + * WARNING: + * The contents of this structure, including the placement + * and interleaving of Keccak lanes, are IMPLEMENTATION-DEFINED. + * The struct is only exposed here to allow its construction on the stack. + */ + +#define mlk_keccakf1600_extract_bytes MLK_NAMESPACE(keccakf1600_extract_bytes) +void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(data, length)) + assigns(memory_slice(data, length)) +); + +#define mlk_keccakf1600_xor_bytes MLK_NAMESPACE(keccakf1600_xor_bytes) +void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(data, length)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +); + +#define mlk_keccakf1600x4_extract_bytes \ + MLK_NAMESPACE(keccakf1600x4_extract_bytes) +void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) +); + +#define mlk_keccakf1600x4_xor_bytes MLK_NAMESPACE(keccakf1600x4_xor_bytes) +void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +); + + +#define mlk_keccakf1600x4_permute MLK_NAMESPACE(keccakf1600x4_permute) +void mlk_keccakf1600x4_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +); + +#define mlk_keccakf1600_permute MLK_NAMESPACE(keccakf1600_permute) +void mlk_keccakf1600_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +); + +#endif /* !MLK_FIPS202_KECCAKF1600_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/auto.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/auto.h new file mode 100644 index 0000000..f29dfca --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/auto.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +#define MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* Default FIPS202 assembly profile for AArch64 systems */ + +/* + * Default logic to decide which implementation to use. + * + */ + +/* + * Keccak-f1600 + * + * - On Arm-based Apple CPUs, we pick a pure Neon implementation. + * - Otherwise, unless MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, + * we use lazy-rotation scalar assembly from @[HYBRID]. + * - Otherwise, if MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, we + * fall back to the standard C implementation. + */ +#if defined(__ARM_FEATURE_SHA3) && defined(__APPLE__) +#include "x1_v84a.h" +#elif !defined(MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER) +#include "x1_scalar.h" +#endif + +/* + * Keccak-f1600x2/x4 + * + * The optimal implementation is highly CPU-specific; see @[HYBRID]. + * + * For now, if v8.4-A is not implemented, we fall back to Keccak-f1600. + * If v8.4-A is implemented and we are on an Apple CPU, we use a plain + * Neon-based implementation. + * If v8.4-A is implemented and we are not on an Apple CPU, we use a + * scalar/Neon/Neon hybrid. + * The reason for this distinction is that Apple CPUs appear to implement + * the SHA3 instructions on all SIMD units, while Arm CPUs prior to Cortex-X4 + * don't, and ordinary Neon instructions are still needed. + */ +#if defined(__ARM_FEATURE_SHA3) +/* + * For Apple-M cores, we use a plain implementation leveraging SHA3 + * instructions only. + */ +#if defined(__APPLE__) +#include "x2_v84a.h" +#else +#include "x4_v8a_v84a_scalar.h" +#endif + +#else /* __ARM_FEATURE_SHA3 */ + +#include "x4_v8a_scalar.h" + +#endif /* !__ARM_FEATURE_SHA3 */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_AUTO_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/fips202_native_aarch64.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/fips202_native_aarch64.h new file mode 100644 index 0000000..08b105b --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/fips202_native_aarch64.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#define MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint64_t + mlk_keccakf1600_round_constants[24]; + +#define mlk_keccak_f1600_x1_scalar_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) +void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_scalar_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#define mlk_keccak_f1600_x1_v84a_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm) +void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_v84a_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#define mlk_keccak_f1600_x2_v84a_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm) +void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x2_v84a_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 2)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 2)) +); + +#define mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) +void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in + * proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) +); + +#define mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) +void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm( + uint64_t state[100], const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in + * proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) +); + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S new file mode 100644 index 0000000..53e7dcc --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S @@ -0,0 +1,377 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x1_scalar_aarch64_asm + Description: AArch64 scalar implementation of Keccak-f[1600] permutation for single state + Signature: void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 128 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x1_scalar_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x80 + .cfi_adjust_cfa_offset 0x80 + stp x19, x20, [sp, #0x20] + .cfi_rel_offset x19, 0x20 + .cfi_rel_offset x20, 0x28 + stp x21, x22, [sp, #0x30] + .cfi_rel_offset x21, 0x30 + .cfi_rel_offset x22, 0x38 + stp x23, x24, [sp, #0x40] + .cfi_rel_offset x23, 0x40 + .cfi_rel_offset x24, 0x48 + stp x25, x26, [sp, #0x50] + .cfi_rel_offset x25, 0x50 + .cfi_rel_offset x26, 0x58 + stp x27, x28, [sp, #0x60] + .cfi_rel_offset x27, 0x60 + .cfi_rel_offset x28, 0x68 + stp x29, x30, [sp, #0x70] + .cfi_rel_offset x29, 0x70 + .cfi_rel_offset x30, 0x78 + +Lkeccak_f1600_x1_scalar_initial: + mov x26, x1 + str x1, [sp, #0x8] + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + str x0, [sp] + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #0x18] + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #0x18] + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #0x1 // =1 + bic x9, x0, x16, ror #9 + str x24, [sp, #0x10] + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 + +Lkeccak_f1600_x1_scalar_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #0x18] + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x10] + eor x17, x10, x9, ror #47 + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0x18] + add x25, x25, #0x1 + str x25, [sp, #0x10] + cmp x25, #0x17 + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + b.le Lkeccak_f1600_x1_scalar_loop + ror x6, x6, #0x2b + ror x11, x11, #0x32 + ror x21, x21, #0x14 + ror x2, x2, #0x3d + ror x7, x7, #0x13 + ror x12, x12, #0x3 + ror x17, x17, #0x24 + ror x22, x22, #0x2c + ror x3, x3, #0x27 + ror x8, x8, #0x38 + ror x13, x13, #0x2e + ror x28, x28, #0x3f + ror x23, x23, #0x3a + ror x4, x4, #0x36 + ror x9, x9, #0x31 + ror x14, x14, #0x8 + ror x19, x19, #0x25 + ror x24, x24, #0x1c + ror x5, x5, #0x19 + ror x10, x10, #0x17 + ror x15, x15, #0x3e + ror x20, x20, #0x2 + ror x25, x25, #0x9 + ldr x0, [sp] + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + ldp x19, x20, [sp, #0x20] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x30] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x40] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x50] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x60] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x70] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0x80 + .cfi_adjust_cfa_offset -0x80 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x1_scalar_aarch64_asm) + +#endif /* MLK_FIPS202_AARCH64_NEED_X1_SCALAR && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S new file mode 100644 index 0000000..6f7f750 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S @@ -0,0 +1,206 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +/*yaml + Name: keccak_f1600_x1_v84a_aarch64_asm + Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for single state + Signature: void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 64 + description: register preservation +*/ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// +// This implementation is essentially from the paper @[HYBRID]. +// The only difference is interleaving/deinterleaving of Keccak state +// during load and store, so that the caller need not do this. +// + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x1_v84a_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldp d0, d1, [x0] + ldp d2, d3, [x0, #0x10] + ldp d4, d5, [x0, #0x20] + ldp d6, d7, [x0, #0x30] + ldp d8, d9, [x0, #0x40] + ldp d10, d11, [x0, #0x50] + ldp d12, d13, [x0, #0x60] + ldp d14, d15, [x0, #0x70] + ldp d16, d17, [x0, #0x80] + ldp d18, d19, [x0, #0x90] + ldp d20, d21, [x0, #0xa0] + ldp d22, d23, [x0, #0xb0] + ldr d24, [x0, #0xc0] + mov x2, #0x18 // =24 + +Lkeccak_f1600_x1_v84a_loop: + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor3 v26.16b, v26.16b, v19.16b, v24.16b + rax1 v25.2d, v30.2d, v28.2d + rax1 v28.2d, v28.2d, v26.2d + rax1 v26.2d, v26.2d, v29.2d + rax1 v29.2d, v29.2d, v27.2d + rax1 v27.2d, v27.2d, v30.2d + eor v30.16b, v0.16b, v26.16b + xar v0.2d, v2.2d, v29.2d, #0x2 + xar v2.2d, v12.2d, v29.2d, #0x15 + xar v12.2d, v13.2d, v28.2d, #0x27 + xar v13.2d, v19.2d, v27.2d, #0x38 + xar v19.2d, v23.2d, v28.2d, #0x8 + xar v23.2d, v15.2d, v26.2d, #0x17 + xar v15.2d, v1.2d, v25.2d, #0x3f + xar v1.2d, v8.2d, v28.2d, #0x9 + xar v8.2d, v16.2d, v25.2d, #0x13 + xar v16.2d, v7.2d, v29.2d, #0x3a + xar v7.2d, v10.2d, v26.2d, #0x3d + xar v10.2d, v3.2d, v28.2d, #0x24 + xar v3.2d, v18.2d, v28.2d, #0x2b + xar v18.2d, v17.2d, v29.2d, #0x31 + xar v17.2d, v11.2d, v25.2d, #0x36 + xar v11.2d, v9.2d, v27.2d, #0x2c + xar v9.2d, v22.2d, v29.2d, #0x3 + xar v22.2d, v14.2d, v27.2d, #0x19 + xar v14.2d, v20.2d, v26.2d, #0x2e + xar v20.2d, v4.2d, v27.2d, #0x25 + xar v4.2d, v24.2d, v27.2d, #0x32 + xar v24.2d, v21.2d, v25.2d, #0x3e + xar v21.2d, v5.2d, v26.2d, #0x1c + xar v27.2d, v6.2d, v25.2d, #0x14 + ld1r { v31.2d }, [x1], #8 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bcax v6.16b, v11.16b, v8.16b, v7.16b + bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax v8.16b, v8.16b, v10.16b, v9.16b + bcax v9.16b, v9.16b, v11.16b, v10.16b + bcax v10.16b, v15.16b, v12.16b, v16.16b + bcax v11.16b, v16.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v15.16b, v14.16b + bcax v14.16b, v14.16b, v16.16b, v15.16b + bcax v15.16b, v20.16b, v17.16b, v21.16b + bcax v16.16b, v21.16b, v18.16b, v17.16b + bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax v18.16b, v18.16b, v20.16b, v19.16b + bcax v19.16b, v19.16b, v21.16b, v20.16b + bcax v20.16b, v0.16b, v22.16b, v1.16b + bcax v21.16b, v1.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v0.16b, v24.16b + bcax v24.16b, v24.16b, v1.16b, v0.16b + bcax v0.16b, v30.16b, v2.16b, v27.16b + bcax v1.16b, v27.16b, v3.16b, v2.16b + bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax v3.16b, v3.16b, v30.16b, v4.16b + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor v0.16b, v0.16b, v31.16b + sub x2, x2, #0x1 + cbnz x2, Lkeccak_f1600_x1_v84a_loop + stp d0, d1, [x0] + stp d2, d3, [x0, #0x10] + stp d4, d5, [x0, #0x20] + stp d6, d7, [x0, #0x30] + stp d8, d9, [x0, #0x40] + stp d10, d11, [x0, #0x50] + stp d12, d13, [x0, #0x60] + stp d14, d15, [x0, #0x70] + stp d16, d17, [x0, #0x80] + stp d18, d19, [x0, #0x90] + stp d20, d21, [x0, #0xa0] + stp d22, d23, [x0, #0xb0] + str d24, [x0, #0xc0] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x1_v84a_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X1_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S new file mode 100644 index 0000000..9505c33 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S @@ -0,0 +1,261 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +/*yaml + Name: keccak_f1600_x2_v84a_aarch64_asm + Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for two sequential states + Signature: void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 400 + permissions: read/write + c_parameter: uint64_t state[50] + description: Two sequential Keccak states (state0[25], state1[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 64 + description: register preservation +*/ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// +// This implementation is essentially from the paper @[HYBRID]. +// The only difference is interleaving/deinterleaving of Keccak state +// during load and store, so that the caller need not do this. +// + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x2_v84a_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + add x2, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x2] + trn1 v24.2d, v25.2d, v27.2d + mov x2, #0x18 // =24 + +Lkeccak_f1600_x2_v84a_loop: + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor3 v26.16b, v26.16b, v19.16b, v24.16b + rax1 v25.2d, v30.2d, v28.2d + rax1 v28.2d, v28.2d, v26.2d + rax1 v26.2d, v26.2d, v29.2d + rax1 v29.2d, v29.2d, v27.2d + rax1 v27.2d, v27.2d, v30.2d + eor v30.16b, v0.16b, v26.16b + xar v0.2d, v2.2d, v29.2d, #0x2 + xar v2.2d, v12.2d, v29.2d, #0x15 + xar v12.2d, v13.2d, v28.2d, #0x27 + xar v13.2d, v19.2d, v27.2d, #0x38 + xar v19.2d, v23.2d, v28.2d, #0x8 + xar v23.2d, v15.2d, v26.2d, #0x17 + xar v15.2d, v1.2d, v25.2d, #0x3f + xar v1.2d, v8.2d, v28.2d, #0x9 + xar v8.2d, v16.2d, v25.2d, #0x13 + xar v16.2d, v7.2d, v29.2d, #0x3a + xar v7.2d, v10.2d, v26.2d, #0x3d + xar v10.2d, v3.2d, v28.2d, #0x24 + xar v3.2d, v18.2d, v28.2d, #0x2b + xar v18.2d, v17.2d, v29.2d, #0x31 + xar v17.2d, v11.2d, v25.2d, #0x36 + xar v11.2d, v9.2d, v27.2d, #0x2c + xar v9.2d, v22.2d, v29.2d, #0x3 + xar v22.2d, v14.2d, v27.2d, #0x19 + xar v14.2d, v20.2d, v26.2d, #0x2e + xar v20.2d, v4.2d, v27.2d, #0x25 + xar v4.2d, v24.2d, v27.2d, #0x32 + xar v24.2d, v21.2d, v25.2d, #0x3e + xar v21.2d, v5.2d, v26.2d, #0x1c + xar v27.2d, v6.2d, v25.2d, #0x14 + ld1r { v31.2d }, [x1], #8 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bcax v6.16b, v11.16b, v8.16b, v7.16b + bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax v8.16b, v8.16b, v10.16b, v9.16b + bcax v9.16b, v9.16b, v11.16b, v10.16b + bcax v10.16b, v15.16b, v12.16b, v16.16b + bcax v11.16b, v16.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v15.16b, v14.16b + bcax v14.16b, v14.16b, v16.16b, v15.16b + bcax v15.16b, v20.16b, v17.16b, v21.16b + bcax v16.16b, v21.16b, v18.16b, v17.16b + bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax v18.16b, v18.16b, v20.16b, v19.16b + bcax v19.16b, v19.16b, v21.16b, v20.16b + bcax v20.16b, v0.16b, v22.16b, v1.16b + bcax v21.16b, v1.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v0.16b, v24.16b + bcax v24.16b, v24.16b, v1.16b, v0.16b + bcax v0.16b, v30.16b, v2.16b, v27.16b + bcax v1.16b, v27.16b, v3.16b, v2.16b + bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax v3.16b, v3.16b, v30.16b, v4.16b + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor v0.16b, v0.16b, v31.16b + sub x2, x2, #0x1 + cbnz x2, Lkeccak_f1600_x2_v84a_loop + sub x0, x0, #0xc0 + add x2, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x2], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x2] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x2_v84a_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X2_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000..3d543b4 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,1078 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states + Signature: void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t state[100] + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + mov x30, #0x0 // =0 + str x30, [sp, #0x20] + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lkeccak_f1600_x4_v8a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor v30.16b, v0.16b, v5.16b + eor v30.16b, v30.16b, v10.16b + eor x0, x30, x21 + eor v30.16b, v30.16b, v15.16b + eor x26, x27, x6 + eor x27, x26, x7 + eor v30.16b, v30.16b, v20.16b + eor x29, x0, x22 + eor v29.16b, v1.16b, v6.16b + eor x26, x29, x23 + eor v29.16b, v29.16b, v11.16b + eor x29, x4, x5 + eor x30, x29, x1 + eor v29.16b, v29.16b, v16.16b + eor x0, x27, x8 + eor v29.16b, v29.16b, v21.16b + eor x29, x30, x2 + eor v28.16b, v2.16b, v7.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor v28.16b, v28.16b, v12.16b + eor x27, x26, x0, ror #63 + eor v28.16b, v28.16b, v17.16b + eor x4, x4, x27 + eor v28.16b, v28.16b, v22.16b + eor x30, x30, x17 + eor x30, x30, x28 + eor v27.16b, v3.16b, v8.16b + eor x29, x29, x3 + eor v27.16b, v27.16b, v13.16b + eor x0, x0, x30, ror #63 + eor v27.16b, v27.16b, v18.16b + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor v27.16b, v27.16b, v23.16b + eor x23, x23, x30 + eor v26.16b, v4.16b, v9.16b + str x23, [sp, #0xd0] + eor v26.16b, v26.16b, v14.16b + eor x23, x14, x15 + eor x14, x14, x0 + eor v26.16b, v26.16b, v19.16b + eor x23, x23, x11 + eor v26.16b, v26.16b, v24.16b + eor x15, x15, x0 + eor x1, x1, x27 + add v31.2d, v28.2d, v28.2d + eor x23, x23, x12 + sri v31.2d, v28.2d, #0x3f + eor x23, x23, x13 + eor v25.16b, v31.16b, v30.16b + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + add v31.2d, v26.2d, v26.2d + eor x23, x23, x26, ror #63 + sri v31.2d, v26.2d, #0x3f + eor x26, x13, x0 + eor v28.16b, v31.16b, v28.16b + eor x13, x28, x23 + eor x28, x24, x30 + add v31.2d, v29.2d, v29.2d + eor x24, x16, x23 + sri v31.2d, v29.2d, #0x3f + eor x16, x21, x30 + eor v26.16b, v31.16b, v26.16b + eor x21, x25, x30 + eor x30, x19, x23 + add v31.2d, v27.2d, v27.2d + eor x19, x20, x23 + sri v31.2d, v27.2d, #0x3f + eor x20, x17, x23 + eor v29.16b, v31.16b, v29.16b + eor x17, x12, x0 + eor x0, x2, x27 + add v31.2d, v30.2d, v30.2d + eor x2, x6, x29 + sri v31.2d, v30.2d, #0x3f + eor x6, x8, x29 + eor v27.16b, v31.16b, v27.16b + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + eor v30.16b, v0.16b, v26.16b + bic x3, x13, x17, ror #19 + eor v31.16b, v2.16b, v29.16b + eor x5, x5, x27 + ldr x27, [sp, #0xd0] + shl v0.2d, v31.2d, #0x3e + bic x25, x17, x2, ror #5 + sri v0.2d, v31.2d, #0x2 + eor x9, x9, x29 + eor v31.16b, v12.16b, v29.16b + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + shl v2.2d, v31.2d, #0x2b + eor x8, x8, x17, ror #2 + sri v2.2d, v31.2d, #0x15 + eor x17, x10, x29 + eor v31.16b, v13.16b, v28.16b + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + shl v12.2d, v31.2d, #0x19 + bic x10, x4, x27, ror #2 + sri v12.2d, v31.2d, #0x27 + bic x7, x5, x28, ror #10 + eor v31.16b, v19.16b, v27.16b + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + shl v13.2d, v31.2d, #0x8 + bic x7, x2, x5, ror #47 + sri v13.2d, v31.2d, #0x38 + eor x2, x25, x24, ror #39 + eor v31.16b, v23.16b, v28.16b + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + shl v19.2d, v31.2d, #0x38 + eor x25, x25, x17, ror #53 + sri v19.2d, v31.2d, #0x8 + bic x17, x11, x17, ror #60 + eor v31.16b, v15.16b, v26.16b + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + shl v23.2d, v31.2d, #0x29 + eor x7, x7, x22, ror #25 + sri v23.2d, v31.2d, #0x17 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor v31.16b, v1.16b, v25.16b + eor x22, x22, x15, ror #23 + shl v15.2d, v31.2d, #0x1 + bic x20, x27, x20, ror #48 + sri v15.2d, v31.2d, #0x3f + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor v31.16b, v8.16b, v28.16b + eor x15, x5, x27, ror #27 + shl v1.2d, v31.2d, #0x37 + eor x5, x20, x11, ror #41 + sri v1.2d, v31.2d, #0x9 + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + eor v31.16b, v16.16b, v25.16b + eor x17, x24, x9, ror #47 + shl v8.2d, v31.2d, #0x2d + mov x24, #0x1 // =1 + sri v8.2d, v31.2d, #0x13 + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + eor v31.16b, v7.16b, v29.16b + bic x24, x29, x1, ror #44 + shl v16.2d, v31.2d, #0x6 + bic x27, x1, x21, ror #50 + sri v16.2d, v31.2d, #0x3a + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + eor v31.16b, v10.16b, v26.16b + ldr x11, [x11] + shl v7.2d, v31.2d, #0x3 + bic x4, x21, x30, ror #57 + sri v7.2d, v31.2d, #0x3d + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + eor v31.16b, v3.16b, v28.16b + bic x9, x14, x6, ror #5 + shl v10.2d, v31.2d, #0x1c + eor x9, x9, x0, ror #43 + sri v10.2d, v31.2d, #0x24 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor v31.16b, v18.16b, v28.16b + eor x11, x4, x26, ror #35 + shl v3.2d, v31.2d, #0x15 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + sri v3.2d, v31.2d, #0x2b + eor x16, x27, x30, ror #43 + eor v31.16b, v17.16b, v29.16b + bic x27, x30, x26, ror #42 + shl v18.2d, v31.2d, #0xf + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + sri v18.2d, v31.2d, #0x31 + eor x14, x26, x6, ror #46 + eor v31.16b, v11.16b, v25.16b + eor x6, x27, x29, ror #41 + shl v17.2d, v31.2d, #0xa + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + sri v17.2d, v31.2d, #0x36 + eor x26, x8, x9, ror #57 + eor v31.16b, v9.16b, v27.16b + eor x27, x0, x14, ror #10 + shl v11.2d, v31.2d, #0x14 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + sri v11.2d, v31.2d, #0x2c + eor x30, x23, x22, ror #50 + eor v31.16b, v22.16b, v29.16b + eor x0, x26, x10, ror #31 + shl v9.2d, v31.2d, #0x3d + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + sri v9.2d, v31.2d, #0x3 + eor x30, x30, x24, ror #34 + eor v31.16b, v14.16b, v27.16b + eor x0, x0, x7, ror #27 + shl v22.2d, v31.2d, #0x27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + sri v22.2d, v31.2d, #0x19 + ror x30, x27, #0x3e + eor v31.16b, v20.16b, v26.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + shl v14.2d, v31.2d, #0x12 + eor x16, x30, x16 + sri v14.2d, v31.2d, #0x2e + eor x28, x30, x28, ror #63 + eor v31.16b, v4.16b, v27.16b + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + shl v20.2d, v31.2d, #0x1b + eor x28, x1, x2, ror #61 + sri v20.2d, v31.2d, #0x25 + eor x19, x30, x19, ror #37 + eor v31.16b, v24.16b, v27.16b + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + shl v4.2d, v31.2d, #0xe + eor x26, x26, x0, ror #55 + sri v4.2d, v31.2d, #0x32 + eor x28, x28, x3, ror #39 + eor v31.16b, v21.16b, v25.16b + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + shl v24.2d, v31.2d, #0x2 + eor x0, x0, x29, ror #63 + sri v24.2d, v31.2d, #0x3e + eor x27, x28, x27, ror #61 + eor v31.16b, v5.16b, v26.16b + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + shl v21.2d, v31.2d, #0x24 + eor x29, x30, x20, ror #2 + sri v21.2d, v31.2d, #0x1c + eor x20, x26, x3, ror #39 + eor v31.16b, v6.16b, v25.16b + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + shl v27.2d, v31.2d, #0x2c + eor x3, x28, x21, ror #20 + sri v27.2d, v31.2d, #0x14 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + bic v31.16b, v7.16b, v11.16b + eor x24, x28, x24, ror #28 + eor v5.16b, v31.16b, v10.16b + eor x1, x30, x17, ror #36 + bic v31.16b, v8.16b, v7.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor v6.16b, v31.16b, v11.16b + eor x8, x27, x8, ror #56 + bic v31.16b, v9.16b, v8.16b + eor x17, x27, x7, ror #19 + eor v7.16b, v31.16b, v7.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bic v31.16b, v10.16b, v9.16b + eor x4, x26, x4, ror #54 + eor v8.16b, v31.16b, v8.16b + eor x0, x0, x12, ror #3 + bic v31.16b, v11.16b, v10.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor v9.16b, v31.16b, v9.16b + eor x26, x26, x5, ror #25 + bic v31.16b, v12.16b, v16.16b + eor x2, x7, x16, ror #39 + eor v10.16b, v31.16b, v15.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + bic v31.16b, v13.16b, v12.16b + eor x7, x7, x22, ror #25 + eor v11.16b, v31.16b, v16.16b + eor x12, x30, x20, ror #58 + bic v31.16b, v14.16b, v13.16b + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor v12.16b, v31.16b, v12.16b + eor x22, x20, x15, ror #23 + bic v31.16b, v15.16b, v14.16b + bic x6, x19, x13, ror #42 + eor v13.16b, v31.16b, v13.16b + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + bic v31.16b, v16.16b, v15.16b + eor x5, x21, x5, ror #21 + eor v14.16b, v31.16b, v14.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic v31.16b, v17.16b, v21.16b + bic x21, x21, x25, ror #50 + eor v15.16b, v31.16b, v20.16b + bic x20, x27, x4, ror #25 + bic v31.16b, v18.16b, v17.16b + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor v16.16b, v31.16b, v21.16b + eor x21, x17, x25, ror #30 + bic v31.16b, v19.16b, v18.16b + bic x19, x25, x19, ror #57 + eor v17.16b, v31.16b, v17.16b + ldr x25, [sp, #0x18] + eor x17, x10, x9, ror #47 + bic v31.16b, v20.16b, v19.16b + ldr x9, [sp, #0x8] + eor v18.16b, v31.16b, v18.16b + eor x15, x20, x28, ror #27 + bic v31.16b, v21.16b, v20.16b + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + eor v19.16b, v31.16b, v19.16b + bic x20, x11, x27, ror #60 + bic v31.16b, v22.16b, v1.16b + eor x20, x20, x4, ror #21 + eor v20.16b, v31.16b, v0.16b + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + bic v31.16b, v23.16b, v22.16b + ldr x28, [x9, x25, lsl #3] + eor v21.16b, v31.16b, v1.16b + ldr x9, [sp, #0xd0] + bic v31.16b, v24.16b, v23.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + eor v22.16b, v31.16b, v22.16b + cmp x25, #0x17 + bic v31.16b, v0.16b, v24.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor v23.16b, v31.16b, v23.16b + eor x1, x5, x28 + bic v31.16b, v1.16b, v0.16b + eor x5, x4, x11, ror #41 + eor v24.16b, v31.16b, v24.16b + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + bic v31.16b, v2.16b, v27.16b + eor x28, x27, x24, ror #57 + eor v0.16b, v31.16b, v30.16b + bic x27, x24, x9, ror #47 + bic v31.16b, v3.16b, v2.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor v1.16b, v31.16b, v27.16b + eor x24, x19, x29, ror #44 + bic v31.16b, v4.16b, v3.16b + bic x29, x3, x29, ror #35 + eor v2.16b, v31.16b, v2.16b + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic v31.16b, v30.16b, v4.16b + bic x29, x9, x0, ror #19 + eor v3.16b, v31.16b, v3.16b + bic x14, x14, x8, ror #5 + bic v31.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + eor v4.16b, v31.16b, v4.16b + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lkeccak_f1600_x4_v8a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor v30.16b, v0.16b, v5.16b + eor v30.16b, v30.16b, v10.16b + eor x26, x8, x9, ror #57 + eor v30.16b, v30.16b, v15.16b + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor v30.16b, v30.16b, v20.16b + eor x26, x26, x6, ror #51 + eor v29.16b, v1.16b, v6.16b + eor x30, x23, x22, ror #50 + eor v29.16b, v29.16b, v11.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor v29.16b, v29.16b, v16.16b + eor x27, x27, x12, ror #5 + eor v29.16b, v29.16b, v21.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor v28.16b, v2.16b, v7.16b + eor x26, x30, x21, ror #26 + eor v28.16b, v28.16b, v12.16b + eor x26, x26, x25, ror #15 + eor v28.16b, v28.16b, v17.16b + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + eor v28.16b, v28.16b, v22.16b + ror x26, x26, #0x3a + eor v27.16b, v3.16b, v8.16b + eor x16, x30, x16 + eor v27.16b, v27.16b, v13.16b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + eor v27.16b, v27.16b, v18.16b + eor x29, x29, x17, ror #36 + eor v27.16b, v27.16b, v23.16b + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor v26.16b, v4.16b, v9.16b + eor x29, x29, x20, ror #2 + eor v26.16b, v26.16b, v14.16b + eor x28, x28, x4, ror #54 + eor v26.16b, v26.16b, v19.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor v26.16b, v26.16b, v24.16b + eor x28, x28, x5, ror #25 + add v31.2d, v28.2d, v28.2d + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + sri v31.2d, v28.2d, #0x3f + eor x27, x28, x27, ror #61 + eor v25.16b, v31.16b, v30.16b + eor x13, x0, x13, ror #46 + add v31.2d, v26.2d, v26.2d + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + sri v31.2d, v26.2d, #0x3f + eor x20, x26, x3, ror #39 + eor v28.16b, v31.16b, v28.16b + eor x11, x0, x11, ror #50 + add v31.2d, v29.2d, v29.2d + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + sri v31.2d, v29.2d, #0x3f + eor x21, x26, x1 + eor v26.16b, v31.16b, v26.16b + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + add v31.2d, v27.2d, v27.2d + eor x1, x30, x17, ror #36 + sri v31.2d, v27.2d, #0x3f + eor x14, x0, x14, ror #8 + eor v29.16b, v31.16b, v29.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + add v31.2d, v30.2d, v30.2d + eor x17, x27, x7, ror #19 + sri v31.2d, v30.2d, #0x3f + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor v27.16b, v31.16b, v27.16b + eor x4, x26, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x0, x0, x12, ror #3 + eor v31.16b, v2.16b, v29.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + shl v0.2d, v31.2d, #0x3e + eor x26, x26, x5, ror #25 + sri v0.2d, v31.2d, #0x2 + eor x2, x7, x16, ror #39 + eor v31.16b, v12.16b, v29.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + shl v2.2d, v31.2d, #0x2b + eor x7, x7, x22, ror #25 + sri v2.2d, v31.2d, #0x15 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor v31.16b, v13.16b, v28.16b + eor x30, x27, x6, ror #43 + shl v12.2d, v31.2d, #0x19 + eor x22, x20, x15, ror #23 + sri v12.2d, v31.2d, #0x27 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + eor v31.16b, v19.16b, v27.16b + bic x5, x13, x17, ror #63 + shl v13.2d, v31.2d, #0x8 + eor x5, x21, x5, ror #21 + sri v13.2d, v31.2d, #0x38 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v31.16b, v23.16b, v28.16b + bic x21, x21, x25, ror #50 + shl v19.2d, v31.2d, #0x38 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + sri v19.2d, v31.2d, #0x8 + eor x16, x21, x19, ror #43 + eor v31.16b, v15.16b, v26.16b + eor x21, x17, x25, ror #30 + shl v23.2d, v31.2d, #0x29 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + sri v23.2d, v31.2d, #0x17 + eor x17, x10, x9, ror #47 + eor v31.16b, v1.16b, v25.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + shl v15.2d, v31.2d, #0x1 + bic x20, x4, x28, ror #2 + sri v15.2d, v31.2d, #0x3f + eor x10, x20, x1, ror #50 + eor v31.16b, v8.16b, v28.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + shl v1.2d, v31.2d, #0x37 + bic x4, x28, x1, ror #48 + sri v1.2d, v31.2d, #0x9 + bic x1, x1, x11, ror #57 + eor v31.16b, v16.16b, v25.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + shl v8.2d, v31.2d, #0x2d + add x25, x25, #0x1 + sri v8.2d, v31.2d, #0x13 + str x25, [sp, #0x18] + cmp x25, #0x17 + eor v31.16b, v7.16b, v29.16b + eor x25, x1, x27, ror #53 + shl v16.2d, v31.2d, #0x6 + bic x27, x30, x26, ror #47 + sri v16.2d, v31.2d, #0x3a + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor v31.16b, v10.16b, v26.16b + eor x11, x19, x13, ror #35 + shl v7.2d, v31.2d, #0x3 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + sri v7.2d, v31.2d, #0x3d + bic x27, x24, x9, ror #47 + eor v31.16b, v3.16b, v28.16b + bic x19, x23, x3, ror #9 + shl v10.2d, v31.2d, #0x1c + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + sri v10.2d, v31.2d, #0x24 + bic x29, x3, x29, ror #35 + eor v31.16b, v18.16b, v28.16b + eor x13, x13, x9, ror #57 + shl v3.2d, v31.2d, #0x15 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + sri v3.2d, v31.2d, #0x2b + bic x14, x14, x8, ror #5 + eor v31.16b, v17.16b, v29.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + shl v18.2d, v31.2d, #0xf + bic x23, x8, x23, ror #38 + sri v18.2d, v31.2d, #0x31 + eor x8, x27, x0, ror #2 + eor v31.16b, v11.16b, v25.16b + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + shl v17.2d, v31.2d, #0xa + eor x23, x3, x26, ror #52 + sri v17.2d, v31.2d, #0x36 + eor x3, x29, x30, ror #24 + eor x0, x15, x11, ror #52 + eor v31.16b, v9.16b, v27.16b + eor x0, x0, x13, ror #48 + shl v11.2d, v31.2d, #0x14 + eor x26, x8, x9, ror #57 + sri v11.2d, v31.2d, #0x2c + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor v31.16b, v22.16b, v29.16b + eor x26, x26, x6, ror #51 + shl v9.2d, v31.2d, #0x3d + eor x30, x23, x22, ror #50 + sri v9.2d, v31.2d, #0x3 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor v31.16b, v14.16b, v27.16b + eor x27, x27, x12, ror #5 + shl v22.2d, v31.2d, #0x27 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + sri v22.2d, v31.2d, #0x19 + eor x26, x30, x21, ror #26 + eor v31.16b, v20.16b, v26.16b + eor x26, x26, x25, ror #15 + shl v14.2d, v31.2d, #0x12 + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + sri v14.2d, v31.2d, #0x2e + ror x26, x26, #0x3a + eor v31.16b, v4.16b, v27.16b + eor x16, x30, x16 + shl v20.2d, v31.2d, #0x1b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + sri v20.2d, v31.2d, #0x25 + eor x29, x29, x17, ror #36 + eor v31.16b, v24.16b, v27.16b + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + shl v4.2d, v31.2d, #0xe + eor x29, x29, x20, ror #2 + sri v4.2d, v31.2d, #0x32 + eor x28, x28, x4, ror #54 + eor v31.16b, v21.16b, v25.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + shl v24.2d, v31.2d, #0x2 + eor x28, x28, x5, ror #25 + sri v24.2d, v31.2d, #0x3e + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + eor v31.16b, v5.16b, v26.16b + eor x27, x28, x27, ror #61 + shl v21.2d, v31.2d, #0x24 + eor x13, x0, x13, ror #46 + sri v21.2d, v31.2d, #0x1c + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor v31.16b, v6.16b, v25.16b + eor x20, x26, x3, ror #39 + shl v27.2d, v31.2d, #0x2c + eor x11, x0, x11, ror #50 + sri v27.2d, v31.2d, #0x14 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + bic v31.16b, v7.16b, v11.16b + eor x21, x26, x1 + eor v5.16b, v31.16b, v10.16b + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + bic v31.16b, v8.16b, v7.16b + eor x1, x30, x17, ror #36 + eor v6.16b, v31.16b, v11.16b + eor x14, x0, x14, ror #8 + bic v31.16b, v9.16b, v8.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor v7.16b, v31.16b, v7.16b + eor x17, x27, x7, ror #19 + bic v31.16b, v10.16b, v9.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor v8.16b, v31.16b, v8.16b + eor x4, x26, x4, ror #54 + bic v31.16b, v11.16b, v10.16b + eor x0, x0, x12, ror #3 + eor v9.16b, v31.16b, v9.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + bic v31.16b, v12.16b, v16.16b + eor x26, x26, x5, ror #25 + eor v10.16b, v31.16b, v15.16b + eor x2, x7, x16, ror #39 + bic v31.16b, v13.16b, v12.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor v11.16b, v31.16b, v16.16b + eor x7, x7, x22, ror #25 + bic v31.16b, v14.16b, v13.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor v12.16b, v31.16b, v12.16b + eor x30, x27, x6, ror #43 + bic v31.16b, v15.16b, v14.16b + eor x22, x20, x15, ror #23 + eor v13.16b, v31.16b, v13.16b + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic v31.16b, v16.16b, v15.16b + bic x5, x13, x17, ror #63 + eor v14.16b, v31.16b, v14.16b + eor x5, x21, x5, ror #21 + bic v31.16b, v17.16b, v21.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v15.16b, v31.16b, v20.16b + bic x21, x21, x25, ror #50 + bic v31.16b, v18.16b, v17.16b + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor v16.16b, v31.16b, v21.16b + eor x16, x21, x19, ror #43 + bic v31.16b, v19.16b, v18.16b + eor x21, x17, x25, ror #30 + eor v17.16b, v31.16b, v17.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bic v31.16b, v20.16b, v19.16b + eor x17, x10, x9, ror #47 + eor v18.16b, v31.16b, v18.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bic v31.16b, v21.16b, v20.16b + bic x20, x4, x28, ror #2 + eor v19.16b, v31.16b, v19.16b + eor x10, x20, x1, ror #50 + bic v31.16b, v22.16b, v1.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + eor v20.16b, v31.16b, v0.16b + bic x4, x28, x1, ror #48 + bic v31.16b, v23.16b, v22.16b + bic x1, x1, x11, ror #57 + eor v21.16b, v31.16b, v1.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bic v31.16b, v24.16b, v23.16b + add x25, x25, #0x1 + eor v22.16b, v31.16b, v22.16b + str x25, [sp, #0x18] + cmp x25, #0x17 + bic v31.16b, v0.16b, v24.16b + eor x25, x1, x27, ror #53 + eor v23.16b, v31.16b, v23.16b + bic x27, x30, x26, ror #47 + bic v31.16b, v1.16b, v0.16b + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor v24.16b, v31.16b, v24.16b + eor x11, x19, x13, ror #35 + bic v31.16b, v2.16b, v27.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v0.16b, v31.16b, v30.16b + bic x27, x24, x9, ror #47 + bic v31.16b, v3.16b, v2.16b + bic x19, x23, x3, ror #9 + eor v1.16b, v31.16b, v27.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v4.16b, v3.16b + bic x29, x3, x29, ror #35 + eor v2.16b, v31.16b, v2.16b + eor x13, x13, x9, ror #57 + bic v31.16b, v30.16b, v4.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + eor v3.16b, v31.16b, v3.16b + bic x14, x14, x8, ror #5 + bic v31.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + eor v4.16b, v31.16b, v4.16b + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lkeccak_f1600_x4_v8a_scalar_hybrid_loop_end: + b.le Lkeccak_f1600_x4_v8a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x30, [sp, #0x20] + cmp x30, #0x1 + b.eq Lkeccak_f1600_x4_v8a_scalar_hybrid_done + mov x30, #0x1 // =1 + str x30, [sp, #0x20] + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x0, x0, #0x258 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x258 + b Lkeccak_f1600_x4_v8a_scalar_hybrid_initial + +Lkeccak_f1600_x4_v8a_scalar_hybrid_done: + ldr x0, [sp] + add x0, x0, #0x258 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x258 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + +#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000..82f8aa8 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,989 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states with ARMv8.4-A optimizations + Signature: void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [SHA3] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t state[100] + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + mov x30, #0x0 // =0 + str x30, [sp, #0x20] + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor v30.16b, v30.16b, v15.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor v30.16b, v30.16b, v20.16b + eor x27, x26, x7 + eor x29, x0, x22 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x26, x29, x23 + eor x29, x4, x5 + eor v29.16b, v29.16b, v16.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor v29.16b, v29.16b, v21.16b + eor x29, x30, x2 + eor x30, x19, x20 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor v28.16b, v28.16b, v17.16b + eor x4, x4, x27 + eor x30, x30, x17 + eor v28.16b, v28.16b, v22.16b + eor x30, x30, x28 + eor x29, x29, x3 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor v27.16b, v27.16b, v18.16b + eor x22, x22, x30 + eor v27.16b, v27.16b, v23.16b + eor x23, x23, x30 + str x23, [sp, #0xd0] + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x23, x14, x15 + eor x14, x14, x0 + eor v26.16b, v26.16b, v19.16b + eor x23, x23, x11 + eor x15, x15, x0 + eor v26.16b, v26.16b, v24.16b + eor x1, x1, x27 + eor x23, x23, x12 + rax1 v25.2d, v30.2d, v28.2d + eor x23, x23, x13 + eor x11, x11, x0 + add v31.2d, v26.2d, v26.2d + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + sri v31.2d, v26.2d, #0x3f + eor x26, x13, x0 + eor x13, x28, x23 + eor v28.16b, v31.16b, v28.16b + eor x28, x24, x30 + eor x24, x16, x23 + rax1 v26.2d, v26.2d, v29.2d + eor x16, x21, x30 + eor x21, x25, x30 + add v31.2d, v27.2d, v27.2d + eor x30, x19, x23 + sri v31.2d, v27.2d, #0x3f + eor x19, x20, x23 + eor x20, x17, x23 + eor v29.16b, v31.16b, v29.16b + eor x17, x12, x0 + eor x0, x2, x27 + rax1 v27.2d, v27.2d, v30.2d + eor x2, x6, x29 + eor x6, x8, x29 + eor v30.16b, v0.16b, v26.16b + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + eor v31.16b, v2.16b, v29.16b + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + shl v0.2d, v31.2d, #0x3e + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + sri v0.2d, v31.2d, #0x2 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor v31.16b, v13.16b, v28.16b + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + shl v12.2d, v31.2d, #0x19 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + sri v12.2d, v31.2d, #0x27 + bic x7, x5, x28, ror #10 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + eor v31.16b, v23.16b, v28.16b + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + shl v19.2d, v31.2d, #0x38 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + sri v19.2d, v31.2d, #0x8 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor v31.16b, v1.16b, v25.16b + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + shl v15.2d, v31.2d, #0x1 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + sri v15.2d, v31.2d, #0x3f + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor v31.16b, v16.16b, v25.16b + eor x5, x20, x11, ror #41 + shl v8.2d, v31.2d, #0x2d + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + sri v8.2d, v31.2d, #0x13 + eor x17, x24, x9, ror #47 + mov x24, #0x1 // =1 + xar v16.2d, v7.2d, v29.2d, #0x3a + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + eor v31.16b, v10.16b, v26.16b + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + shl v7.2d, v31.2d, #0x3 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + sri v7.2d, v31.2d, #0x3d + ldr x11, [x11] + bic x4, x21, x30, ror #57 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + eor v31.16b, v18.16b, v28.16b + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + shl v3.2d, v31.2d, #0x15 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + sri v3.2d, v31.2d, #0x2b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + bic x0, x16, x19, ror #35 + eor v31.16b, v11.16b, v25.16b + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + shl v17.2d, v31.2d, #0xa + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + sri v17.2d, v31.2d, #0x36 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor v31.16b, v22.16b, v29.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + shl v9.2d, v31.2d, #0x3d + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + sri v9.2d, v31.2d, #0x3 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor v31.16b, v20.16b, v26.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + shl v14.2d, v31.2d, #0x12 + eor x26, x30, x21, ror #26 + sri v14.2d, v31.2d, #0x2e + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + eor v31.16b, v24.16b, v27.16b + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + shl v4.2d, v31.2d, #0xe + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + sri v4.2d, v31.2d, #0x32 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v31.16b, v5.16b, v26.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + shl v21.2d, v31.2d, #0x24 + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + sri v21.2d, v31.2d, #0x1c + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + bic v31.16b, v7.16b, v11.16b + eor x29, x30, x20, ror #2 + eor v5.16b, v31.16b, v10.16b + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + bic v31.16b, v9.16b, v8.16b + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor v7.16b, v31.16b, v7.16b + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + bic v31.16b, v11.16b, v10.16b + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor v9.16b, v31.16b, v9.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + bic v31.16b, v13.16b, v12.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor v11.16b, v31.16b, v16.16b + eor x26, x26, x5, ror #25 + bcax v12.16b, v12.16b, v14.16b, v13.16b + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic v31.16b, v15.16b, v14.16b + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor v13.16b, v31.16b, v13.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + bic v31.16b, v16.16b, v15.16b + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + eor v14.16b, v31.16b, v14.16b + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic v31.16b, v18.16b, v17.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v16.16b, v31.16b, v21.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v17.16b, v17.16b, v19.16b, v18.16b + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + bic v31.16b, v20.16b, v19.16b + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + eor v18.16b, v31.16b, v18.16b + ldr x25, [sp, #0x18] + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x17, x10, x9, ror #47 + ldr x9, [sp, #0x8] + bic v31.16b, v22.16b, v1.16b + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor v20.16b, v31.16b, v0.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic v31.16b, v24.16b, v23.16b + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + eor v22.16b, v31.16b, v22.16b + ldr x9, [sp, #0xd0] + add x25, x25, #0x1 + bcax v23.16b, v23.16b, v0.16b, v24.16b + str x25, [sp, #0x18] + cmp x25, #0x17 + bic v31.16b, v1.16b, v0.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor v24.16b, v31.16b, v24.16b + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x11, x19, x13, ror #35 + bic v31.16b, v3.16b, v2.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v1.16b, v31.16b, v27.16b + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bcax v2.16b, v2.16b, v4.16b, v3.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v30.16b, v4.16b + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor v3.16b, v31.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor v30.16b, v30.16b, v15.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor v30.16b, v30.16b, v20.16b + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor v29.16b, v29.16b, v16.16b + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor v29.16b, v29.16b, v21.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + eor v28.16b, v28.16b, v17.16b + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + eor v28.16b, v28.16b, v22.16b + ror x26, x26, #0x3a + eor x16, x30, x16 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + eor v27.16b, v27.16b, v18.16b + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor v27.16b, v27.16b, v23.16b + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor v26.16b, v26.16b, v19.16b + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + eor v26.16b, v26.16b, v24.16b + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + rax1 v25.2d, v30.2d, v28.2d + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + add v31.2d, v26.2d, v26.2d + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + sri v31.2d, v26.2d, #0x3f + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor v28.16b, v31.16b, v28.16b + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + rax1 v26.2d, v26.2d, v29.2d + eor x21, x26, x1 + add v31.2d, v27.2d, v27.2d + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + sri v31.2d, v27.2d, #0x3f + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor v29.16b, v31.16b, v29.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + rax1 v27.2d, v27.2d, v30.2d + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + eor v30.16b, v0.16b, v26.16b + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor v31.16b, v2.16b, v29.16b + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + shl v0.2d, v31.2d, #0x3e + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + sri v0.2d, v31.2d, #0x2 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + xar v2.2d, v12.2d, v29.2d, #0x15 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor v31.16b, v13.16b, v28.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + shl v12.2d, v31.2d, #0x19 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + sri v12.2d, v31.2d, #0x27 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + xar v13.2d, v19.2d, v27.2d, #0x38 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + eor v31.16b, v23.16b, v28.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + shl v19.2d, v31.2d, #0x38 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + sri v19.2d, v31.2d, #0x8 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + eor v31.16b, v1.16b, v25.16b + ldr x25, [sp, #0x18] + eor x17, x10, x9, ror #47 + shl v15.2d, v31.2d, #0x1 + ldr x9, [sp, #0x8] + sri v15.2d, v31.2d, #0x3f + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor v31.16b, v16.16b, v25.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + shl v8.2d, v31.2d, #0x2d + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + sri v8.2d, v31.2d, #0x13 + ldr x9, [sp, #0xd0] + add x25, x25, #0x1 + xar v16.2d, v7.2d, v29.2d, #0x3a + str x25, [sp, #0x18] + cmp x25, #0x17 + eor v31.16b, v10.16b, v26.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + shl v7.2d, v31.2d, #0x3 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + sri v7.2d, v31.2d, #0x3d + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + eor v31.16b, v18.16b, v28.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + shl v3.2d, v31.2d, #0x15 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + sri v3.2d, v31.2d, #0x2b + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + xar v18.2d, v17.2d, v29.2d, #0x31 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor v31.16b, v11.16b, v25.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + shl v17.2d, v31.2d, #0xa + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + sri v17.2d, v31.2d, #0x36 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + eor v31.16b, v22.16b, v29.16b + eor x0, x15, x11, ror #52 + shl v9.2d, v31.2d, #0x3d + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + sri v9.2d, v31.2d, #0x3 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor v31.16b, v20.16b, v26.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + shl v14.2d, v31.2d, #0x12 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + sri v14.2d, v31.2d, #0x2e + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor v31.16b, v24.16b, v27.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + shl v4.2d, v31.2d, #0xe + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + sri v4.2d, v31.2d, #0x32 + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor v31.16b, v5.16b, v26.16b + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + shl v21.2d, v31.2d, #0x24 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + sri v21.2d, v31.2d, #0x1c + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + bic v31.16b, v7.16b, v11.16b + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor v5.16b, v31.16b, v10.16b + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + bic v31.16b, v9.16b, v8.16b + eor x3, x28, x21, ror #20 + eor v7.16b, v31.16b, v7.16b + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + bic v31.16b, v11.16b, v10.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor v9.16b, v31.16b, v9.16b + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bic v31.16b, v13.16b, v12.16b + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor v11.16b, v31.16b, v16.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + bcax v12.16b, v12.16b, v14.16b, v13.16b + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic v31.16b, v15.16b, v14.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor v13.16b, v31.16b, v13.16b + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic v31.16b, v16.16b, v15.16b + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor v14.16b, v31.16b, v14.16b + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + bic v31.16b, v18.16b, v17.16b + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor v16.16b, v31.16b, v21.16b + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bcax v17.16b, v17.16b, v19.16b, v18.16b + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + bic v31.16b, v20.16b, v19.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + eor v18.16b, v31.16b, v18.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x17, x10, x9, ror #47 + bic v31.16b, v22.16b, v1.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + eor v20.16b, v31.16b, v0.16b + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic v31.16b, v24.16b, v23.16b + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + eor v22.16b, v31.16b, v22.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v23.16b, v23.16b, v0.16b, v24.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bic v31.16b, v1.16b, v0.16b + cmp x25, #0x17 + eor x25, x1, x27, ror #53 + eor v24.16b, v31.16b, v24.16b + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic v31.16b, v3.16b, v2.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v1.16b, v31.16b, v27.16b + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bcax v2.16b, v2.16b, v4.16b, v3.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v30.16b, v4.16b + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor v3.16b, v31.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end: + b.le Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x30, [sp, #0x20] + cmp x30, #0x1 + b.eq Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_done + mov x30, #0x1 // =1 + str x30, [sp, #0x20] + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x0, x0, #0x258 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x258 + b Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_initial + +Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_done: + ldr x0, [sp] + add x0, x0, #0x258 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x258 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccakf1600_round_constants.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccakf1600_round_constants.c new file mode 100644 index 0000000..12a897f --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/src/keccakf1600_round_constants.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" + +#if (defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) || \ + defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) || \ + defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ + defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_aarch64.h" + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t + mlk_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +#else /* (MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \ + MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \ + || MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202_aarch64_round_constants) + +#endif /* !((MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \ + MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \ + || MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_scalar.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_scalar.h new file mode 100644 index 0000000..bd6b376 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_scalar.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H + +/* Part of backend API */ +#define MLK_USE_FIPS202_X1_NATIVE +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X1_SCALAR + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +{ + mlk_keccak_f1600_x1_scalar_aarch64_asm(state, + mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_v84a.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_v84a.h new file mode 100644 index 0000000..c7d374a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x1_v84a.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#define MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_FIPS202_X1_NATIVE +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X1_V84A + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x1_v84a_aarch64_asm(state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x2_v84a.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x2_v84a.h new file mode 100644 index 0000000..3dabec4 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x2_v84a.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#define MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_FIPS202_X4_NATIVE +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X2_V84A + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 0 * 25, + mlk_keccakf1600_round_constants); + mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 2 * 25, + mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_scalar.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_scalar.h new file mode 100644 index 0000000..fb8b99c --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_scalar.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H + +/* Part of backend API */ +#define MLK_USE_FIPS202_X4_NATIVE +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm( + state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h new file mode 100644 index 0000000..c215198 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_FIPS202_X4_NATIVE +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm( + state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/api.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/api.h new file mode 100644 index 0000000..0414808 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/api.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_API_H +#define MLK_FIPS202_NATIVE_API_H +/* + * FIPS-202 native interface + * + * This header is primarily for documentation purposes. + * It should not be included by backend implementations. + */ + +#include "../../cbmc.h" + +/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */ +#define MLK_NATIVE_FUNC_SUCCESS (0) +/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that + * the target/parameters are unsupported; typically, this would be because of + * dependencies on CPU features not detected on the host CPU. In this case, + * the frontend falls back to the default C implementation. */ +#define MLK_NATIVE_FUNC_FALLBACK (-1) + +/* + * This is the C<->native interface allowing for the drop-in + * of custom Keccak-F1600 implementations. + * + * A _backend_ is a specific implementation of parts of this interface. + * + * You can replace 1-fold or 4-fold batched Keccak-F1600. + * To enable, set MLK_USE_FIPS202_X1_NATIVE or MLK_USE_FIPS202_X4_NATIVE + * in your backend, and define the inline wrappers mlk_keccak_f1600_x1_native() + * and/or mlk_keccak_f1600_x4_native(), respectively, to forward to your + * implementation. + */ + +#if defined(MLK_USE_FIPS202_X1_NATIVE) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 1))); +#endif /* MLK_USE_FIPS202_X1_NATIVE */ +#if defined(MLK_USE_FIPS202_X4_NATIVE) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4))); +#endif /* MLK_USE_FIPS202_X4_NATIVE */ + +/* + * Native x4 XOR bytes and extract bytes interface. + * + * These functions allow backends to provide optimized implementations for + * XORing input data into the state and extracting output data from the state. + * This is particularly useful for backends that use a different internal state + * representation (e.g., bit-interleaved), as conversion can happen during + * XOR/extract rather than before/after each permutation. + * + * NOTE: We assume that the custom representation of the zero state is the + * all-zero state. + * + * MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE: Backend provides native XOR bytes + * MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE: Backend provides native extract + * bytes + */ + +#if defined(MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native( + uint64_t *state, const unsigned char *data0, const unsigned char *data1, + const unsigned char *data2, const unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= 25 * sizeof(uint64_t) && + 0 <= length && length <= 25 * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(memory_no_alias(data0, length)) + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4))); +#endif /* MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE */ + +#if defined(MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native( + uint64_t *state, unsigned char *data0, unsigned char *data1, + unsigned char *data2, unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= 25 * sizeof(uint64_t) && + 0 <= length && length <= 25 * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)); +#endif /* MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE */ + +#endif /* !MLK_FIPS202_NATIVE_API_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/README.md b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/README.md new file mode 100644 index 0000000..9c2c0d2 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/README.md @@ -0,0 +1,10 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# FIPS202 backend for Armv8.1-M + MVE + +This directory contains the source code for a FIPS202 backend targeting +the Armv8.1-M + MVE/Helium architecture. It is automatically derived from +the respective development source in [dev/fips202/armv81m](../../../../../dev/fips202/armv81m). + +**Warning:** This backend is still in active development and has not yet undergone +the same level of review as the rest of the code. Use at your own risk! diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/mve.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/mve.h new file mode 100644 index 0000000..3b595e4 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/mve.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#define MLK_FIPS202_NATIVE_ARMV81M_MVE_H + +#define MLK_FIPS202_NATIVE_ARMV81M + +/* Part of backend API */ +#define MLK_USE_FIPS202_X4_NATIVE +#define MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE +#define MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE +/* Guard for assembly file */ +#define MLK_FIPS202_ARMV81M_NEED_X4 + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_armv81m.h" + +/* + * Native x4 permutation + * State is kept in bit-interleaved format. + */ +#define mlk_keccak_f1600_x4_native_impl \ + MLK_NAMESPACE(keccak_f1600_x4_native_impl) +int mlk_keccak_f1600_x4_native_impl(uint64_t *state); + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + return mlk_keccak_f1600_x4_native_impl(state); +} + +/* + * Native x4 XOR bytes (with on-the-fly bit interleaving) + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native( + uint64_t *state, const uint8_t *data0, const uint8_t *data1, + const uint8_t *data2, const uint8_t *data3, unsigned offset, + unsigned length) +{ + mlk_keccak_f1600_x4_state_xor_bytes_asm(state, data0, data1, data2, data3, + offset, length); + return MLK_NATIVE_FUNC_SUCCESS; +} + +/* + * Native x4 extract bytes (with on-the-fly bit de-interleaving) + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native( + uint64_t *state, uint8_t *data0, uint8_t *data1, uint8_t *data2, + uint8_t *data3, unsigned offset, unsigned length) +{ + mlk_keccak_f1600_x4_state_extract_bytes_asm(state, data0, data1, data2, data3, + offset, length); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_ARMV81M_MVE_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/fips202_native_armv81m.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/fips202_native_armv81m.h new file mode 100644 index 0000000..7888f33 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/fips202_native_armv81m.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#define MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H + +#include "../../../../common.h" + +/* Keccak round constants in bit-interleaved form */ +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint32_t + mlk_keccakf1600_round_constants[48]; + +#define mlk_keccak_f1600_x4_mve_asm MLK_NAMESPACE(keccak_f1600_x4_mve_asm) +void mlk_keccak_f1600_x4_mve_asm(uint64_t state[100], uint64_t tmpstate[100], + const uint32_t rc[48]); + +#define mlk_keccak_f1600_x4_state_xor_bytes_asm \ + MLK_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm) +void mlk_keccak_f1600_x4_state_xor_bytes_asm(void *state, const uint8_t *d0, + const uint8_t *d1, + const uint8_t *d2, + const uint8_t *d3, unsigned offset, + unsigned length); + +#define mlk_keccak_f1600_x4_state_extract_bytes_asm \ + MLK_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm) +void mlk_keccak_f1600_x4_state_extract_bytes_asm(void *state, uint8_t *data0, + uint8_t *data1, uint8_t *data2, + uint8_t *data3, + unsigned offset, + unsigned length); + +#endif /* !MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S new file mode 100644 index 0000000..ab4f7a1 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S @@ -0,0 +1,716 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2025 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: keccak_f1600_x4_mve_asm + Description: Armv8.1-M MVE implementation of batched (x4) Keccak-f[1600] permutation using bit-interleaved state + Signature: void mlk_keccak_f1600_x4_mve_asm(void *state, void *tmpstate, const uint32_t *rc) + ABI: + Architecture: armv81m + CallingConvention: AAPCS32 + Features: [MVE] + r0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: void *state + description: Bit-interleaved state for 4 Keccak instances (even halves followed by odd halves) + r1: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: void *tmpstate + description: Temporary storage for intermediate state + r2: + type: buffer + size_bytes: 192 + permissions: read + c_parameter: const uint32_t *rc + description: Keccak round constants in bit-interleaved form (24 pairs of 32-bit words) + Stack: + bytes: 228 + description: register preservation (36) + SIMD registers (64) + temporary storage (128) +*/ + +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_mve_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_mve_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} + .cfi_adjust_cfa_offset 0x24 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x20 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + sub sp, #0x80 + .cfi_adjust_cfa_offset 0x80 + mov r6, r2 + mov.w lr, #0x18 + mov r2, r0 + mov r4, r1 + add.w r3, r2, #0x190 + vldrw.u32 q0, [r3] + vldrw.u32 q1, [r2] + vldrw.u32 q2, [r2, #32] + wls lr, lr, Lkeccak_f1600_x4_mve_asm_roundend @ imm = #0x8c0 + +Lkeccak_f1600_x4_mve_asm_roundstart: + vldrw.u32 q6, [r2, #112] + veor q7, q6, q2 + vldrw.u32 q2, [r2, #80] + veor q1, q2, q1 + add.w r5, r2, #0x190 + vldrw.u32 q5, [r5, #80] + veor q4, q5, q0 + vldrw.u32 q0, [r2, #192] + veor q3, q7, q0 + vldrw.u32 q0, [r2, #160] + veor q1, q1, q0 + vldrw.u32 q0, [r5, #160] + veor q0, q4, q0 + vldrw.u32 q6, [r2, #272] + veor q2, q3, q6 + vldrw.u32 q7, [r2, #240] + veor q5, q1, q7 + vldrw.u32 q4, [r5, #240] + veor q4, q0, q4 + vldrw.u32 q6, [r2, #352] + veor q3, q2, q6 + vldrw.u32 q0, [r2, #320] + veor q2, q5, q0 + vldrw.u32 q1, [r5, #320] + veor q5, q4, q1 + vldrw.u32 q4, [r5, #32] + veor q0, q3, q5 + vldrw.u32 q1, [r5, #16] + veor q6, q1, q0 + vstrw.32 q5, [sp] + vshr.u32 q7, q6, #0x1f + add.w r10, r4, #0x190 + vsli.32 q7, q6, #0x1 + vldrw.u32 q6, [r5, #112] + veor q6, q4, q6 + vldrw.u32 q4, [r5, #192] + veor q4, q6, q4 + vldrw.u32 q6, [r5, #272] + veor q4, q4, q6 + vldrw.u32 q6, [r5, #352] + veor q5, q4, q6 + vstrw.32 q7, [r4, #160] + vshr.u32 q4, q5, #0x1f + vsli.32 q4, q5, #0x1 + vldrw.u32 q6, [r2, #16] + veor q7, q4, q2 + veor q1, q6, q7 + vldrw.u32 q6, [r5, #96] + veor q6, q6, q0 + vstrw.32 q1, [r10, #160] + vshr.u32 q1, q6, #0xa + vsli.32 q1, q6, #0x16 + vldrw.u32 q6, [r2, #96] + veor q4, q6, q7 + vstrw.32 q1, [r10, #16] + vshr.u32 q6, q4, #0xa + vsli.32 q6, q4, #0x16 + vldrw.u32 q1, [r5, #336] + veor q4, q1, q0 + vldrw.u32 q1, [r2, #176] + veor q1, q1, q7 + vstrw.32 q6, [r4, #16] + vshr.u32 q6, q1, #0x1b + vsli.32 q6, q1, #0x5 + vldrw.u32 q1, [r2, #256] + veor q1, q1, q7 + vstrw.32 q6, [r4, #272] + vshr.u32 q6, q1, #0xa + vsli.32 q6, q1, #0x16 + vldrw.u32 q1, [r2, #336] + veor q1, q1, q7 + vstrw.32 q6, [r10, #128] + vshr.u32 q6, q1, #0x1f + vsli.32 q6, q1, #0x1 + vldrw.u32 q7, [r5, #176] + veor q7, q7, q0 + vstrw.32 q6, [r4, #384] + vshr.u32 q1, q7, #0x1b + vsli.32 q1, q7, #0x5 + vldrw.u32 q6, [r5, #256] + veor q0, q6, q0 + vstrw.32 q1, [r10, #272] + vshr.u32 q1, q4, #0x1f + vldrw.u32 q7, [r5, #64] + vsli.32 q1, q4, #0x1 + vldrw.u32 q4, [r5, #144] + vshr.u32 q6, q0, #0x9 + vstrw.32 q1, [r10, #384] + vsli.32 q6, q0, #0x17 + veor q7, q7, q4 + vldrw.u32 q1, [r5, #224] + veor q4, q7, q1 + vldrw.u32 q7, [r5, #304] + veor q1, q4, q7 + vldrw.u32 q0, [r5, #384] + veor q7, q1, q0 + vstrw.32 q6, [r4, #128] + vshr.u32 q1, q7, #0x1f + vsli.32 q1, q7, #0x1 + vldrw.u32 q6, [r2, #144] + veor q0, q1, q3 + vldrw.u32 q3, [r2, #64] + veor q1, q3, q6 + vldrw.u32 q6, [r2, #224] + veor q1, q1, q6 + vldrw.u32 q3, [r2, #304] + veor q6, q1, q3 + vldrw.u32 q4, [r2, #384] + veor q3, q6, q4 + vldrw.u32 q4, [r2, #48] + veor q5, q3, q5 + vldrw.u32 q1, [r5, #48] + veor q1, q1, q5 + vshr.u32 q6, q1, #0x12 + vsli.32 q6, q1, #0xe + vldrw.u32 q1, [r2, #128] + veor q1, q1, q0 + vstrw.32 q6, [r10, #80] + vshr.u32 q6, q1, #0x5 + vsli.32 q6, q1, #0x1b + vldrw.u32 q1, [r5, #128] + veor q1, q1, q5 + vstrw.32 q6, [r10, #336] + vshr.u32 q6, q1, #0x4 + vsli.32 q6, q1, #0x1c + veor q1, q4, q0 + vstrw.32 q6, [r4, #336] + vshr.u32 q4, q1, #0x12 + vsli.32 q4, q1, #0xe + vldrw.u32 q6, [r2, #208] + veor q6, q6, q0 + vstrw.32 q4, [r4, #80] + vshr.u32 q1, q6, #0x14 + vsli.32 q1, q6, #0xc + vldrw.u32 q4, [r2, #288] + veor q4, q4, q0 + vldrw.u32 q6, [r2, #368] + veor q0, q6, q0 + vshr.u32 q6, q0, #0x4 + vstrw.32 q1, [r10, #192] + vsli.32 q6, q0, #0x1c + vshr.u32 q0, q4, #0x16 + vldrw.u32 q1, [r5, #368] + vsli.32 q0, q4, #0xa + vstrw.32 q6, [r4, #304] + veor q4, q1, q5 + vstrw.32 q0, [r10, #48] + vshr.u32 q1, q4, #0x4 + vsli.32 q1, q4, #0x1c + vldrw.u32 q6, [r5, #208] + veor q6, q6, q5 + vldrw.u32 q0, [r5, #288] + veor q5, q0, q5 + vstrw.32 q1, [r10, #304] + vshr.u32 q0, q6, #0x13 + vsli.32 q0, q6, #0xd + vldrw.u32 q1, [r5, #96] + vshr.u32 q6, q5, #0x15 + vldrw.u32 q4, [r5, #16] + vsli.32 q6, q5, #0xb + vldrw.u32 q5, [r5, #176] + veor q1, q4, q1 + vldrw.u32 q4, [r5, #256] + veor q5, q1, q5 + vldrw.u32 q1, [r5, #336] + veor q5, q5, q4 + vstrw.32 q0, [r4, #192] + veor q0, q5, q1 + vstrw.32 q6, [r4, #48] + vshr.u32 q5, q0, #0x1f + vsli.32 q5, q0, #0x1 + vldrw.u32 q4, [r2, #16] + veor q3, q5, q3 + vldrw.u32 q6, [r2, #96] + veor q4, q4, q6 + vldrw.u32 q1, [r2, #176] + veor q5, q4, q1 + vldrw.u32 q6, [r2, #256] + veor q6, q5, q6 + vldrw.u32 q4, [r2, #336] + veor q5, q6, q4 + vldrw.u32 q1, [r5] + veor q7, q5, q7 + vldrw.u32 q4, [r2] + veor q1, q1, q7 + veor q4, q4, q3 + vshr.u32 q6, q1, #0x20 + vsli.32 q6, q1, #0x0 + vldrw.u32 q1, [r2, #80] + veor q1, q1, q3 + vstrw.32 q6, [r10] + vshr.u32 q6, q4, #0x20 + vsli.32 q6, q4, #0x0 + vldrw.u32 q4, [r5, #80] + veor q4, q4, q7 + vstrw.32 q6, [r4] + vshr.u32 q6, q1, #0xe + vsli.32 q6, q1, #0x12 + vldrw.u32 q1, [r2, #160] + veor q1, q1, q3 + vstrw.32 q6, [r4, #256] + vshr.u32 q6, q4, #0xe + vsli.32 q6, q4, #0x12 + vldrw.u32 q4, [r2, #240] + veor q4, q4, q3 + vstrw.32 q6, [r10, #256] + vshr.u32 q6, q1, #0x1f + vsli.32 q6, q1, #0x1 + vldrw.u32 q1, [r2, #320] + veor q1, q1, q3 + vstrw.32 q6, [r10, #112] + vshr.u32 q6, q4, #0xc + vsli.32 q6, q4, #0x14 + vldrw.u32 q3, [r5, #240] + veor q3, q3, q7 + vstrw.32 q6, [r10, #368] + vshr.u32 q4, q3, #0xb + vsli.32 q4, q3, #0x15 + vldrw.u32 q3, [r5, #160] + veor q6, q3, q7 + vstrw.32 q4, [r4, #368] + vshr.u32 q3, q6, #0x1e + vsli.32 q3, q6, #0x2 + vldrw.u32 q6, [r5, #320] + veor q7, q6, q7 + vldrw.u32 q4, [r2, #368] + vshr.u32 q6, q1, #0x17 + vstrw.32 q3, [r4, #112] + vsli.32 q6, q1, #0x9 + vshr.u32 q1, q7, #0x17 + vldrw.u32 q3, [r2, #48] + vsli.32 q1, q7, #0x9 + vldrw.u32 q7, [r2, #128] + veor q3, q3, q7 + vldrw.u32 q7, [r2, #208] + veor q7, q3, q7 + vldrw.u32 q3, [r2, #288] + veor q3, q7, q3 + vldrw.u32 q7, [r5, #128] + veor q3, q3, q4 + vldrw.u32 q4, [r5, #48] + veor q0, q3, q0 + veor q4, q4, q7 + vldrw.u32 q7, [r5, #208] + veor q4, q4, q7 + vldrw.u32 q7, [r5, #288] + veor q4, q4, q7 + vldrw.u32 q7, [r5, #368] + veor q7, q4, q7 + vstrw.32 q6, [r4, #224] + vshr.u32 q4, q7, #0x1f + vstrw.32 q1, [r10, #224] + vsli.32 q4, q7, #0x1 + veor q5, q4, q5 + vldrw.u32 q6, [r2, #192] + veor q1, q6, q5 + vldrw.u32 q4, [r5, #112] + veor q7, q2, q7 + vldrw.u32 q6, [r5, #32] + vshr.u32 q2, q1, #0xb + vsli.32 q2, q1, #0x15 + veor q1, q6, q0 + vstrw.32 q2, [r10, #32] + vshr.u32 q6, q1, #0x1 + vsli.32 q6, q1, #0x1f + vldrw.u32 q2, [r2, #112] + veor q2, q2, q5 + vstrw.32 q6, [r10, #320] + vshr.u32 q1, q2, #0x1d + vsli.32 q1, q2, #0x3 + vldrw.u32 q6, [r2, #32] + veor q4, q4, q0 + vstrw.32 q1, [r4, #176] + veor q2, q6, q5 + vshr.u32 q6, q2, #0x1 + vldrw.u32 q1, [r5, #352] + vsli.32 q6, q2, #0x1f + veor q1, q1, q0 + vstrw.32 q6, [r4, #320] + vshr.u32 q6, q1, #0x1 + vsli.32 q6, q1, #0x1f + vldrw.u32 q2, [r5, #192] + vshr.u32 q1, q4, #0x1d + vstrw.32 q6, [r4, #144] + vsli.32 q1, q4, #0x3 + veor q2, q2, q0 + vldrw.u32 q6, [r5, #272] + veor q0, q6, q0 + vldrw.u32 q4, [r2, #352] + veor q6, q4, q5 + vldrw.u32 q4, [r2, #272] + veor q4, q4, q5 + vstrw.32 q1, [r10, #176] + vshr.u32 q1, q2, #0xa + vsli.32 q1, q2, #0x16 + vldrw.u32 q5, [sp] + vshr.u32 q2, q0, #0x18 + vstrw.32 q1, [r4, #32] + vsli.32 q2, q0, #0x8 + vshr.u32 q1, q6, #0x2 + vstrw.32 q2, [r4, #288] + vsli.32 q1, q6, #0x1e + vshr.u32 q6, q4, #0x19 + vstrw.32 q1, [r10, #144] + vsli.32 q6, q4, #0x7 + vshr.u32 q0, q5, #0x1f + vstrw.32 q6, [r10, #288] + vsli.32 q0, q5, #0x1 + veor q5, q0, q3 + vldrw.u32 q6, [r2, #64] + veor q3, q6, q5 + vldrw.u32 q1, [r5, #64] + vshr.u32 q4, q3, #0x13 + vldrw.u32 q2, [r2, #384] + vsli.32 q4, q3, #0xd + vldrw.u32 q0, [r5, #224] + veor q6, q1, q7 + vstrw.32 q4, [r10, #240] + veor q2, q2, q5 + veor q3, q0, q7 + vldrw.u32 q0, [r2, #224] + vshr.u32 q4, q6, #0x12 + vldrw.u32 q1, [r5, #384] + vsli.32 q4, q6, #0xe + vshr.u32 q6, q2, #0x19 + vstrw.32 q4, [r4, #240] + vsli.32 q6, q2, #0x7 + vshr.u32 q2, q3, #0xc + vstrw.32 q6, [r4, #64] + vsli.32 q2, q3, #0x14 + veor q0, q0, q5 + vldrw.u32 q6, [r2, #144] + veor q4, q1, q7 + veor q6, q6, q5 + vstrw.32 q2, [r4, #352] + vshr.u32 q2, q4, #0x19 + vsli.32 q2, q4, #0x7 + vldrw.u32 q1, [r2, #304] + veor q5, q1, q5 + vldrw.u32 q1, [r5, #144] + veor q4, q1, q7 + vldrw.u32 q3, [r5, #304] + veor q1, q3, q7 + vstrw.32 q2, [r10, #64] + vshr.u32 q3, q0, #0xd + vsli.32 q3, q0, #0x13 + vldrw.u32 q7, [r4, #80] + vshr.u32 q0, q6, #0x16 + vstrw.32 q3, [r10, #352] + vsli.32 q0, q6, #0xa + vshr.u32 q2, q5, #0x1c + vsli.32 q2, q5, #0x4 + vldrw.u32 q5, [r4, #112] + vshr.u32 q3, q1, #0x1c + vsli.32 q3, q1, #0x4 + vldrw.u32 q1, [r4, #128] + vbic q6, q5, q0 + vstrw.32 q3, [r10, #208] + vbic q3, q1, q5 + veor q3, q0, q3 + vstrw.32 q3, [r2, #96] + vbic q3, q0, q7 + veor q0, q7, q6 + vldrw.u32 q6, [r4, #144] + vbic q7, q7, q6 + vstrw.32 q0, [r2, #80] + veor q3, q6, q3 + vstrw.32 q3, [r2, #144] + veor q0, q1, q7 + vstrw.32 q0, [r2, #128] + vbic q1, q6, q1 + vshr.u32 q6, q4, #0x16 + vldrw.u32 q3, [r10, #112] + vsli.32 q6, q4, #0xa + vldrw.u32 q4, [r10, #80] + veor q1, q5, q1 + vldrw.u32 q0, [r10, #144] + vbic q7, q4, q0 + vldrw.u32 q5, [r10, #128] + veor q7, q5, q7 + vstrw.32 q1, [r2, #112] + vbic q1, q0, q5 + vstrw.32 q7, [r5, #128] + veor q7, q3, q1 + vstrw.32 q7, [r5, #112] + vbic q7, q5, q3 + vbic q1, q3, q6 + vldrw.u32 q3, [r4, #176] + veor q5, q4, q1 + vbic q4, q6, q4 + vldrw.u32 q1, [r4, #160] + veor q0, q0, q4 + vldrw.u32 q4, [r4, #224] + veor q7, q6, q7 + vstrw.32 q0, [r5, #144] + vbic q0, q1, q4 + vstrw.32 q7, [r5, #96] + veor q0, q2, q0 + vstrw.32 q0, [r2, #208] + vbic q6, q3, q1 + vstrw.32 q5, [r5, #80] + vbic q7, q4, q2 + vldrw.u32 q0, [r10, #160] + veor q6, q4, q6 + vldrw.u32 q5, [r4, #192] + vbic q4, q2, q5 + vldrw.u32 q2, [r10, #224] + veor q4, q3, q4 + vstrw.32 q4, [r2, #176] + vbic q4, q5, q3 + vstrw.32 q6, [r2, #224] + veor q4, q1, q4 + vldrw.u32 q1, [r10, #208] + veor q3, q5, q7 + vldrw.u32 q5, [r10, #192] + vbic q6, q1, q5 + vldrw.u32 q7, [r10, #176] + veor q6, q7, q6 + vstrw.32 q3, [r2, #192] + vbic q3, q0, q2 + vstrw.32 q6, [r5, #176] + veor q3, q1, q3 + vstrw.32 q3, [r5, #208] + vbic q3, q5, q7 + vstrw.32 q4, [r2, #160] + veor q3, q0, q3 + vstrw.32 q3, [r5, #160] + vbic q6, q2, q1 + vldrw.u32 q1, [r4, #288] + vbic q7, q7, q0 + vldrw.u32 q3, [r4, #272] + veor q0, q5, q6 + vldrw.u32 q4, [r4, #304] + veor q6, q2, q7 + vldrw.u32 q7, [r4, #256] + vbic q5, q4, q1 + vstrw.32 q0, [r5, #192] + veor q5, q3, q5 + vstrw.32 q6, [r5, #224] + vbic q0, q3, q7 + vstrw.32 q5, [r2, #272] + vbic q6, q1, q3 + veor q5, q7, q6 + vldrw.u32 q3, [r4, #240] + veor q6, q3, q0 + vldrw.u32 q2, [r10, #288] + vbic q0, q3, q4 + vstrw.32 q6, [r2, #240] + vbic q7, q7, q3 + vstrw.32 q5, [r2, #256] + veor q7, q4, q7 + vstrw.32 q7, [r2, #304] + veor q7, q1, q0 + vstrw.32 q7, [r2, #288] + vldrw.u32 q5, [r10, #304] + vbic q7, q5, q2 + vldrw.u32 q3, [r10, #272] + veor q1, q3, q7 + vldrw.u32 q7, [r4, #336] + vbic q4, q2, q3 + vldrw.u32 q6, [r10, #256] + vbic q3, q3, q6 + vldrw.u32 q0, [r10, #240] + veor q3, q0, q3 + vstrw.32 q1, [r5, #272] + vbic q1, q0, q5 + vstrw.32 q3, [r5, #240] + veor q1, q2, q1 + vldrw.u32 q3, [r4, #384] + vbic q2, q6, q0 + vldrw.u32 q0, [r4, #320] + veor q2, q5, q2 + vldrw.u32 q5, [r4, #352] + veor q4, q6, q4 + vstrw.32 q2, [r5, #304] + vbic q2, q7, q0 + vstrw.32 q1, [r5, #288] + veor q1, q3, q2 + vstrw.32 q1, [r2, #384] + vbic q2, q5, q7 + vstrw.32 q4, [r5, #256] + veor q4, q0, q2 + vstrw.32 q4, [r2, #320] + vbic q2, q0, q3 + vldrw.u32 q4, [r4, #368] + vbic q3, q3, q4 + vldrw.u32 q0, [r10, #320] + veor q1, q5, q3 + vldrw.u32 q6, [r10, #336] + vbic q5, q4, q5 + vstrw.32 q1, [r2, #352] + veor q5, q7, q5 + vstrw.32 q5, [r2, #336] + veor q3, q4, q2 + vstrw.32 q3, [r2, #368] + vbic q7, q6, q0 + vldrw.u32 q5, [r10, #352] + vbic q3, q5, q6 + vldrw.u32 q1, [r10, #368] + vbic q4, q1, q5 + vldrw.u32 q2, [r4, #16] + veor q6, q6, q4 + vldrw.u32 q4, [r10, #384] + veor q3, q0, q3 + vstrw.32 q3, [r5, #320] + veor q3, q4, q7 + vstrw.32 q3, [r5, #384] + vbic q0, q0, q4 + vstrw.32 q6, [r5, #336] + veor q3, q1, q0 + vstrw.32 q3, [r5, #368] + vbic q7, q4, q1 + veor q5, q5, q7 + vldrw.u32 q6, [r4, #32] + vbic q3, q6, q2 + vldrw.u32 q4, [r4, #48] + vbic q0, q4, q6 + vldrw.u32 q1, [r4] + veor q0, q2, q0 + vldrw.u32 q7, [r4, #64] + veor q3, q1, q3 + vstrw.32 q5, [r5, #352] + vbic q5, q1, q7 + vstrw.32 q0, [r2, #16] + veor q0, q4, q5 + vstrw.32 q0, [r2, #48] + vbic q5, q2, q1 + veor q2, q7, q5 + vldrw.u32 q0, [r10, #16] + vbic q5, q7, q4 + vldrw.u32 q4, [r10] + vbic q1, q0, q4 + vldrw.u32 q7, [r10, #64] + veor q1, q7, q1 + vstrw.32 q2, [r2, #64] + veor q2, q6, q5 + vbic q6, q4, q7 + vldrw.u32 q5, [r10, #48] + veor q6, q5, q6 + ldrd r7, r8, [r6] + vbic q7, q7, q5 + vstrw.32 q1, [r5, #64] + vdup.32 q1, r7 + veor q1, q3, q1 + vldrw.u32 q3, [r10, #32] + veor q7, q3, q7 + add.w r6, r6, #0x8 + vbic q5, q5, q3 + vstrw.32 q6, [r5, #48] + vbic q6, q3, q0 + vstrw.32 q1, [r2] + veor q5, q0, q5 + vstrw.32 q7, [r5, #32] + veor q4, q4, q6 + vstrw.32 q5, [r5, #16] + vdup.32 q6, r8 + vstrw.32 q2, [r2, #32] + veor q0, q4, q6 + vstrw.32 q0, [r5] + +Lkeccak_f1600_x4_mve_asm_roundend_pre: + le lr, Lkeccak_f1600_x4_mve_asm_roundstart @ imm = #-0x8c0 + +Lkeccak_f1600_x4_mve_asm_roundend: + add sp, #0x80 + .cfi_adjust_cfa_offset -0x80 + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x24 + .cfi_endproc + nop + +MLK_ASM_FN_SIZE(keccak_f1600_x4_mve_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c new file mode 100644 index 0000000..d0c06b5 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../../common.h" +#include "../../../../verify.h" + +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_armv81m.h" + + +/* + * Keccak-f1600 x4 permutation (on bit-interleaved state) + * State is expected to already be in bit-interleaved format. + */ +#define mlk_keccak_f1600_x4_native_impl \ + MLK_NAMESPACE(keccak_f1600_x4_native_impl) +int mlk_keccak_f1600_x4_native_impl(uint64_t *state) +{ + MLK_ALIGN uint64_t state_tmp[100]; + mlk_keccak_f1600_x4_mve_asm(state, state_tmp, + mlk_keccakf1600_round_constants); + mlk_zeroize(state_tmp, sizeof(state_tmp)); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(keccak_f1600_x4_mve) + +#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \ + */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +/* Some macros are kept because they are also defined in a header. */ +/* Keep: mlk_keccak_f1600_x4_native_impl (mve.h) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccakf1600_round_constants.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccakf1600_round_constants.c new file mode 100644 index 0000000..b19bf33 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/keccakf1600_round_constants.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../../common.h" + +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_armv81m.h" + +/* + * Keccak round constants in bit-interleaved form. + * Each 64-bit constant is split into two 32-bit words: + * - low word contains even-indexed bits + * - high word contains odd-indexed bits + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint32_t + mlk_keccakf1600_round_constants[48] = { + 0x00000001, 0x00000000, /* RC0 */ + 0x00000000, 0x00000089, /* RC1 */ + 0x00000000, 0x8000008b, /* RC2 */ + 0x00000000, 0x80008080, /* RC3 */ + 0x00000001, 0x0000008b, /* RC4 */ + 0x00000001, 0x00008000, /* RC5 */ + 0x00000001, 0x80008088, /* RC6 */ + 0x00000001, 0x80000082, /* RC7 */ + 0x00000000, 0x0000000b, /* RC8 */ + 0x00000000, 0x0000000a, /* RC9 */ + 0x00000001, 0x00008082, /* RC10 */ + 0x00000000, 0x00008003, /* RC11 */ + 0x00000001, 0x0000808b, /* RC12 */ + 0x00000001, 0x8000000b, /* RC13 */ + 0x00000001, 0x8000008a, /* RC14 */ + 0x00000001, 0x80000081, /* RC15 */ + 0x00000000, 0x80000081, /* RC16 */ + 0x00000000, 0x80000008, /* RC17 */ + 0x00000000, 0x00000083, /* RC18 */ + 0x00000000, 0x80008003, /* RC19 */ + 0x00000001, 0x80008088, /* RC20 */ + 0x00000000, 0x80000088, /* RC21 */ + 0x00000001, 0x00008000, /* RC22 */ + 0x00000000, 0x80008082, /* RC23 */ +}; + +#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202_armv81m_round_constants) + +#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \ + */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S new file mode 100644 index 0000000..daa832a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S @@ -0,0 +1,334 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2026 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// --------------------------------------------------------------------------- +// Overview +// --------------------------------------------------------------------------- +// MVE/Helium implementation of KeccakF1600x4_StateExtractBytes +// (inverse of state_xor_bytes_x4_mve.S). +// +// void KeccakF1600x4_StateExtractBytes(state, d0, d1, d2, d3, offset, length) +// +// Reads 'length' bytes from the bit-interleaved Keccak state starting at +// byte 'offset', recombines the even and odd halves of each lane back +// into plain bytes, and writes them to four output buffers (d0..d3). +// +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd +// +// --------------------------------------------------------------------------- +// Three-phase structure +// --------------------------------------------------------------------------- +// Prologue -- if offset is not 8-byte aligned, extract +// min(length, 8-(offset%8)) bytes via predicated byte stores. +// Main -- process full 8-byte groups: load even/odd lane pair, +// de-interleave, scatter-store to output buffers. +// Tail -- extract remaining <8 bytes via predicated byte stores. + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/state_extract_bytes_x4_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_extract_bytes_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr} + .cfi_adjust_cfa_offset 0x28 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x24 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldr r4, [sp, #0x68] + ldr.w r10, [sp, #0x6c] + ldr r6, [sp, #0x70] + cmp r6, #0x0 + beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x2ea + and r5, r10, #0x7 + bic r9, r10, #0x7 + add.w r8, r0, r9, lsl #1 + add.w r7, r8, #0x190 + cmp r5, #0x0 + beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_pre_main @ imm = #0x112 + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov.f64 d4, d1 + vmov.f64 d6, d3 + rsb.w lr, r5, #0x8 + cmp r6, lr + it ls + movls lr, r6 + vctp.8 lr + vmrs r11, p0 + lsl.w r11, r11, r5 + vmsr p0, r11 + subs r1, r1, r5 + subs r2, r2, r5 + subs r3, r3, r5 + subs r4, r4, r5 + vpstttt + vstrbt.8 q0, [r1], #4 + vstrbt.8 q1, [r2], #4 + vstrbt.8 q2, [r3], #4 + vstrbt.8 q3, [r4], #4 + subs.w r6, r6, lr + cmp r6, #0x0 + beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x1cc + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + b Lkeccak_f1600_x4_state_extract_bytes_asm_main_body @ imm = #0xe + +Lkeccak_f1600_x4_state_extract_bytes_asm_pre_main: + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + mov.w r12, #0x4 + vsub.i32 q7, q7, r12 + +Lkeccak_f1600_x4_state_extract_bytes_asm_main_body: + lsr.w lr, r6, #0x3 + wls lr, lr, Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_end @ imm = #0xb4 + +Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_start: + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vstrw.32 q0, [q7, #4]! + vstrw.32 q1, [q7, #4]! + le lr, Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_start @ imm = #-0xb4 + +Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_end: + ands r6, r6, #0x7 + beq Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0xee + mov.w r12, #0x4 + vadd.i32 q7, q7, r12 + vmov r1, r3, q7[2], q7[0] + vmov r2, r4, q7[3], q7[1] + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov.f64 d4, d1 + vmov.f64 d6, d3 + vctp.8 r6 + vpstttt + vstrbt.8 q0, [r1], #4 + vstrbt.8 q1, [r2], #4 + vstrbt.8 q2, [r3], #4 + vstrbt.8 q3, [r4], #4 + +Lkeccak_f1600_x4_state_extract_bytes_asm_exit: + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x28 + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_state_extract_bytes_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S new file mode 100644 index 0000000..c14ee76 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S @@ -0,0 +1,355 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2026 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// --------------------------------------------------------------------------- +// Overview +// --------------------------------------------------------------------------- +// MVE/Helium implementation of KeccakF1600x4_StateXORBytes. +// +// void KeccakF1600x4_StateXORBytes(state, d0, d1, d2, d3, offset, length) +// +// Reads 'length' plain bytes from each of four input buffers (d0..d3), +// splits every byte into its even and odd bits (bit-interleaving), and +// XORs the result into the Keccak state starting at byte 'offset'. +// +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd +// +// --------------------------------------------------------------------------- +// Three-phase structure +// --------------------------------------------------------------------------- +// Prologue -- if offset is not 8-byte aligned, absorb +// min(length, 8-(offset%8)) bytes via predicated byte loads. +// Main -- process full 8-byte groups via word-level gather loads, +// bit-interleave, then VEOR into even/odd state halves. +// Tail -- absorb remaining <8 bytes via predicated byte loads. + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr} + .cfi_adjust_cfa_offset 0x28 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x24 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldr r4, [sp, #0x68] + ldr.w r10, [sp, #0x6c] + ldr r6, [sp, #0x70] + cmp r6, #0x0 + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x346 + and r5, r10, #0x7 + bic r9, r10, #0x7 + add.w r8, r0, r9, lsl #1 + add.w r7, r8, #0x190 + cmp r5, #0x0 + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x12c + subs r1, r1, r5 + subs r2, r2, r5 + subs r3, r3, r5 + subs r4, r4, r5 + rsb.w lr, r5, #0x8 + cmp r6, lr + it ls + movls lr, r6 + subs.w r6, r6, lr + vctp.8 lr + vmrs r11, p0 + lsl.w r11, r11, r5 + vmsr p0, r11 + vpstttt + vldrbt.u8 q0, [r1], #4 + vldrbt.u8 q1, [r2], #4 + vldrbt.u8 q2, [r3], #4 + vldrbt.u8 q3, [r4], #4 + vmov.f64 d1, d4 + vmov.f64 d3, d6 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + b Lkeccak_f1600_x4_state_xor_bytes_asm_main_body @ imm = #0xe + +Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main: + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + mov.w r0, #0x4 + vsub.i32 q7, q7, r0 + +Lkeccak_f1600_x4_state_xor_bytes_asm_main_body: + lsr.w lr, r6, #0x3 + wls lr, lr, Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end @ imm = #0xd4 + +Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_start: + vldrw.u32 q0, [q7, #4]! + vldrw.u32 q1, [q7, #4]! + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + le lr, Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_start @ imm = #-0xd4 + +Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end: + ands r6, r6, #0x7 + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x110 + mov.w r0, #0x4 + vadd.i32 q7, q7, r0 + vmov r1, r3, q7[2], q7[0] + vmov r2, r4, q7[3], q7[1] + vctp.8 r6 + vpstttt + vldrbt.u8 q0, [r1] + vldrbt.u8 q1, [r2] + vldrbt.u8 q2, [r3] + vldrbt.u8 q3, [r4] + vmov.f64 d1, d4 + vmov.f64 d3, d6 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + +Lkeccak_f1600_x4_state_xor_bytes_asm_exit: + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x28 + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_state_xor_bytes_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/auto.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/auto.h new file mode 100644 index 0000000..e5a6819 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/auto.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AUTO_H +#define MLK_FIPS202_NATIVE_AUTO_H + +/* + * Default FIPS202 backend + */ +#include "../../sys.h" + +#if defined(MLK_SYS_AARCH64) +#include "aarch64/auto.h" +#endif + +/* The x86_64 backend requires toolchain support for the SysV ABI */ +#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED) +#include "x86_64/keccak_f1600_x4_avx2.h" +#endif + +/* We do not yet include the FIPS202 backend for Armv8.1-M+MVE by default + * as it is still experimental and undergoing review. */ +/* #if defined(MLK_SYS_ARMV81M_MVE) */ +/* #include "armv81m/mve.h" */ +/* #endif */ + +#endif /* !MLK_FIPS202_NATIVE_AUTO_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h new file mode 100644 index 0000000..1c37f31 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#define MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H + +#include "../../../common.h" + +#define MLK_FIPS202_X86_64_NEED_X4_AVX2 + +/* Part of backend API */ +#define MLK_USE_FIPS202_X4_NATIVE + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_x86_64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_avx2_asm(state, mlk_keccakf1600_round_constants, + mlk_keccak_rho8, mlk_keccak_rho56); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/fips202_native_x86_64.h b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/fips202_native_x86_64.h new file mode 100644 index 0000000..69ce174 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/fips202_native_x86_64.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#define MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +/* TODO: Reconsider whether this check is needed -- x86_64 is always + * little-endian, so the backend selection already implies this. */ +#ifndef MLK_SYS_LITTLE_ENDIAN +#error Expecting a little-endian platform +#endif + +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint64_t + mlk_keccakf1600_round_constants[24]; + +#define mlk_keccak_rho8 MLK_NAMESPACE(keccak_rho8) +MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho8[4]; + +#define mlk_keccak_rho56 MLK_NAMESPACE(keccak_rho56) +MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho56[4]; + +#define mlk_keccak_f1600_x4_avx2_asm MLK_NAMESPACE(keccak_f1600_x4_avx2_asm) +MLK_SYSV_ABI +void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], + const uint64_t rho8[4], + const uint64_t rho56[4]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/keccak_f1600_x4_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(states, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + requires(rho8 == mlk_keccak_rho8) + requires(rho56 == mlk_keccak_rho56) + assigns(memory_slice(states, sizeof(uint64_t) * 25 * 4)) +); + +#endif /* !MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S new file mode 100644 index 0000000..23d7eed --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S @@ -0,0 +1,487 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#include "../../../../common.h" + +#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: keccak_f1600_x4_avx2_asm + Description: x86_64 AVX2 Keccak-f[1600] permutation for four sequential states + Signature: void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], const uint64_t rho8[4], const uint64_t rho56[4]) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t states[100] + description: Four sequential Keccak states (4 x 25 x uint64_t) + rsi: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint64_t rho8[4] + description: Rotation constant rho8 (4 x uint64_t) + rcx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint64_t rho56[4] + description: Rotation constant rho56 (4 x uint64_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/x86_64/src/keccak_f1600_x4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_avx2_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_avx2_asm) + + .cfi_startproc + movq %rsp, %r11 + .cfi_def_cfa_register %r11 + andq $-0x20, %rsp + subq $0x300, %rsp # imm = 0x300 + vmovdqu (%rdi), %ymm0 + vmovdqu 0xc8(%rdi), %ymm3 + vmovdqu 0x190(%rdi), %ymm1 + vmovdqu 0x258(%rdi), %ymm4 + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x278(%rdi), %ymm4 + vmovdqu %ymm3, 0x40(%rsp) + vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, (%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0x20(%rdi), %ymm0 + vmovdqu 0x1b0(%rdi), %ymm1 + vmovdqu %ymm3, 0x60(%rsp) + vmovdqu 0xe8(%rdi), %ymm3 + vmovdqu %ymm7, 0x20(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x298(%rdi), %ymm4 + vperm2i128 $0x31, %ymm1, %ymm0, %ymm14 # ymm14 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, 0x80(%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0x40(%rdi), %ymm0 + vmovdqu 0x1d0(%rdi), %ymm1 + vmovdqu %ymm3, 0xc0(%rsp) + vmovdqu 0x108(%rdi), %ymm3 + vmovdqu %ymm14, %ymm10 + vmovdqu %ymm7, 0xa0(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm11 # ymm11 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu %ymm3, 0x100(%rsp) + vperm2i128 $0x31, %ymm1, %ymm0, %ymm8 # ymm8 = ymm0[2,3],ymm1[2,3] + vmovdqu 0x128(%rdi), %ymm3 + vmovdqu 0x60(%rdi), %ymm0 + vmovdqu 0x1f0(%rdi), %ymm1 + vmovdqu %ymm7, 0xe0(%rsp) + vmovdqu %ymm11, %ymm14 + vmovdqu 0x2b8(%rdi), %ymm4 + vmovdqu 0x2f8(%rdi), %ymm5 + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vmovdqu 0x2d8(%rdi), %ymm4 + vperm2i128 $0x20, %ymm3, %ymm2, %ymm15 # ymm15 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm9 # ymm9 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm3, 0x140(%rsp) + vmovdqu 0x80(%rdi), %ymm0 + vmovdqu 0x148(%rdi), %ymm3 + vmovdqu 0x210(%rdi), %ymm1 + vmovdqu %ymm7, 0x120(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm13 # ymm13 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, 0x160(%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0xa0(%rdi), %ymm0 + vmovdqu 0x230(%rdi), %ymm1 + vmovdqu %ymm3, 0x1a0(%rsp) + vmovdqu 0x168(%rdi), %ymm3 + vpunpcklqdq %ymm5, %ymm1, %ymm4 # ymm4 = ymm1[0],ymm5[0],ymm1[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm5[1],ymm1[3],ymm5[3] + vmovdqu %ymm7, 0x180(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm12 # ymm12 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm7 # ymm7 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[2,3],ymm1[2,3] + vmovq 0x250(%rdi), %xmm0 + vmovq 0xc0(%rdi), %xmm1 + vmovdqu %ymm12, 0x1c0(%rsp) + vmovdqu %ymm4, 0x1e0(%rsp) + vpinsrq $0x1, 0x318(%rdi), %xmm0, %xmm0 + vpinsrq $0x1, 0x188(%rdi), %xmm1, %xmm1 + vinserti128 $0x1, %xmm0, %ymm1, %ymm2 + movq $0x0, %r10 + +LLkeccak_f1600_x4_avx2: + vmovdqu 0xa0(%rsp), %ymm4 + vpxor 0x1c0(%rsp), %ymm9, %ymm0 + vmovdqu %ymm9, 0x200(%rsp) + vmovdqu %ymm10, %ymm9 + vmovdqu 0xc0(%rsp), %ymm11 + vmovdqu 0x160(%rsp), %ymm12 + vmovdqu %ymm3, 0x240(%rsp) + vpxor 0x100(%rsp), %ymm4, %ymm1 + vmovdqu 0x40(%rsp), %ymm10 + vmovdqu %ymm4, 0x220(%rsp) + vpxor %ymm3, %ymm12, %ymm12 + vmovdqu 0x20(%rsp), %ymm6 + vmovdqu 0x140(%rsp), %ymm4 + vmovdqu %ymm14, 0x2a0(%rsp) + vpxor %ymm1, %ymm0, %ymm0 + vpxor %ymm8, %ymm11, %ymm1 + vpxor 0x180(%rsp), %ymm7, %ymm11 + vmovdqu %ymm10, 0x280(%rsp) + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm15, %ymm9, %ymm1 + vmovdqu 0xe0(%rsp), %ymm3 + vmovdqu %ymm8, 0x260(%rsp) + vpxor %ymm1, %ymm11, %ymm11 + vpxor 0x120(%rsp), %ymm14, %ymm1 + vpxor %ymm6, %ymm12, %ymm12 + vmovdqu 0x60(%rsp), %ymm8 + vpxor %ymm10, %ymm11, %ymm11 + vpxor 0x1e0(%rsp), %ymm13, %ymm10 + vpxor %ymm4, %ymm3, %ymm3 + vmovdqu %ymm4, 0x2c0(%rsp) + vpsrlq $0x3f, %ymm12, %ymm4 + vpsrlq $0x3f, %ymm11, %ymm5 + vpxor (%rsp), %ymm0, %ymm0 + vpxor %ymm1, %ymm10, %ymm10 + vmovdqu 0x80(%rsp), %ymm1 + vpxor %ymm8, %ymm10, %ymm10 + vmovdqu %ymm1, %ymm14 + vpxor 0x1a0(%rsp), %ymm2, %ymm1 + vmovdqu %ymm14, 0x2e0(%rsp) + vpxor %ymm3, %ymm1, %ymm1 + vpsllq $0x1, %ymm12, %ymm3 + vpor %ymm4, %ymm3, %ymm3 + vpsllq $0x1, %ymm11, %ymm4 + vpxor %ymm14, %ymm1, %ymm1 + vpor %ymm5, %ymm4, %ymm4 + vpsrlq $0x3f, %ymm10, %ymm14 + vpxor %ymm1, %ymm3, %ymm3 + vpsllq $0x1, %ymm10, %ymm5 + vpxor %ymm0, %ymm4, %ymm4 + vpor %ymm14, %ymm5, %ymm5 + vpxor %ymm6, %ymm4, %ymm6 + vpxor %ymm12, %ymm5, %ymm5 + vpsrlq $0x3f, %ymm1, %ymm12 + vpsllq $0x1, %ymm1, %ymm1 + vpxor %ymm7, %ymm5, %ymm7 + vpxor %ymm9, %ymm5, %ymm9 + vpor %ymm12, %ymm1, %ymm1 + vpxor (%rsp), %ymm3, %ymm12 + vpxor %ymm11, %ymm1, %ymm1 + vpsrlq $0x3f, %ymm0, %ymm11 + vpsllq $0x1, %ymm0, %ymm0 + vpxor %ymm13, %ymm1, %ymm13 + vpxor %ymm8, %ymm1, %ymm8 + vpor %ymm11, %ymm0, %ymm0 + vpxor %ymm10, %ymm0, %ymm0 + vpxor 0xc0(%rsp), %ymm4, %ymm10 + vpxor %ymm2, %ymm0, %ymm2 + vpsrlq $0x14, %ymm10, %ymm11 + vpsllq $0x2c, %ymm10, %ymm10 + vpor %ymm11, %ymm10, %ymm10 + vpxor %ymm15, %ymm5, %ymm11 + vpbroadcastq (%rsi), %ymm15 + vpsrlq $0x15, %ymm11, %ymm14 + vpsllq $0x2b, %ymm11, %ymm11 + vpor %ymm14, %ymm11, %ymm11 + vpandn %ymm11, %ymm10, %ymm14 + vpxor %ymm15, %ymm14, %ymm14 + vpxor %ymm12, %ymm14, %ymm15 + vpsrlq $0x2b, %ymm13, %ymm14 + vpsllq $0x15, %ymm13, %ymm13 + vmovdqu %ymm15, (%rsp) + vpor %ymm14, %ymm13, %ymm13 + vpandn %ymm13, %ymm11, %ymm14 + vpxor %ymm10, %ymm14, %ymm15 + vpsrlq $0x32, %ymm2, %ymm14 + vpsllq $0xe, %ymm2, %ymm2 + vmovdqu %ymm15, 0x20(%rsp) + vpor %ymm14, %ymm2, %ymm2 + vpandn %ymm2, %ymm13, %ymm14 + vpxor %ymm11, %ymm14, %ymm11 + vmovdqu %ymm11, 0x40(%rsp) + vpandn %ymm12, %ymm2, %ymm11 + vpandn %ymm10, %ymm12, %ymm12 + vpxor %ymm13, %ymm11, %ymm11 + vmovdqu %ymm11, 0x60(%rsp) + vpxor %ymm2, %ymm12, %ymm11 + vpsrlq $0x24, %ymm8, %ymm2 + vpsllq $0x1c, %ymm8, %ymm8 + vmovdqu %ymm11, 0x80(%rsp) + vpor %ymm2, %ymm8, %ymm8 + vpxor 0xe0(%rsp), %ymm0, %ymm2 + vpsrlq $0x2c, %ymm2, %ymm10 + vpsllq $0x14, %ymm2, %ymm2 + vpor %ymm10, %ymm2, %ymm2 + vpxor 0x100(%rsp), %ymm3, %ymm10 + vpsrlq $0x3d, %ymm10, %ymm11 + vpsllq $0x3, %ymm10, %ymm10 + vpor %ymm11, %ymm10, %ymm10 + vpandn %ymm10, %ymm2, %ymm11 + vpxor %ymm8, %ymm11, %ymm11 + vmovdqu %ymm11, 0xa0(%rsp) + vpxor 0x160(%rsp), %ymm4, %ymm11 + vpsrlq $0x13, %ymm11, %ymm12 + vpsllq $0x2d, %ymm11, %ymm11 + vpor %ymm12, %ymm11, %ymm11 + vpandn %ymm11, %ymm10, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vmovdqu %ymm12, 0xc0(%rsp) + vpsrlq $0x3, %ymm7, %ymm12 + vpsllq $0x3d, %ymm7, %ymm7 + vpor %ymm12, %ymm7, %ymm7 + vpandn %ymm7, %ymm11, %ymm12 + vpxor %ymm10, %ymm12, %ymm10 + vpandn %ymm8, %ymm7, %ymm12 + vpandn %ymm2, %ymm8, %ymm8 + vpsrlq $0x3f, %ymm6, %ymm2 + vpsllq $0x1, %ymm6, %ymm6 + vpxor %ymm11, %ymm12, %ymm14 + vpor %ymm2, %ymm6, %ymm6 + vpsrlq $0x3a, %ymm9, %ymm2 + vpxor %ymm7, %ymm8, %ymm12 + vpsllq $0x6, %ymm9, %ymm9 + vmovdqu %ymm12, 0xe0(%rsp) + vpxor 0x1a0(%rsp), %ymm0, %ymm7 + vpor %ymm2, %ymm9, %ymm9 + vpxor 0x120(%rsp), %ymm1, %ymm2 + vpshufb (%rdx), %ymm7, %ymm7 + vpsrlq $0x27, %ymm2, %ymm11 + vpsllq $0x19, %ymm2, %ymm2 + vpor %ymm2, %ymm11, %ymm11 + vpandn %ymm11, %ymm9, %ymm2 + vpandn %ymm7, %ymm11, %ymm8 + vpxor %ymm6, %ymm2, %ymm12 + vpxor 0x1c0(%rsp), %ymm3, %ymm2 + vpxor %ymm9, %ymm8, %ymm8 + vmovdqu %ymm12, 0x100(%rsp) + vpsrlq $0x2e, %ymm2, %ymm12 + vpsllq $0x12, %ymm2, %ymm2 + vpor %ymm2, %ymm12, %ymm2 + vpandn %ymm2, %ymm7, %ymm12 + vpxor %ymm11, %ymm12, %ymm15 + vpandn %ymm6, %ymm2, %ymm11 + vpandn %ymm9, %ymm6, %ymm6 + vpxor %ymm7, %ymm11, %ymm12 + vmovdqu %ymm12, 0x120(%rsp) + vpxor %ymm2, %ymm6, %ymm12 + vpxor 0x2e0(%rsp), %ymm0, %ymm6 + vpxor 0x2c0(%rsp), %ymm0, %ymm0 + vmovdqu %ymm12, 0x140(%rsp) + vpsrlq $0x25, %ymm6, %ymm2 + vpsllq $0x1b, %ymm6, %ymm6 + vpor %ymm6, %ymm2, %ymm2 + vpxor 0x220(%rsp), %ymm3, %ymm6 + vpxor 0x200(%rsp), %ymm3, %ymm3 + vpsrlq $0x1c, %ymm6, %ymm7 + vpsllq $0x24, %ymm6, %ymm6 + vpor %ymm6, %ymm7, %ymm7 + vpxor 0x260(%rsp), %ymm4, %ymm6 + vpxor 0x240(%rsp), %ymm4, %ymm4 + vpsrlq $0x36, %ymm6, %ymm12 + vpsllq $0xa, %ymm6, %ymm6 + vpor %ymm6, %ymm12, %ymm12 + vpxor 0x180(%rsp), %ymm5, %ymm6 + vpxor 0x280(%rsp), %ymm5, %ymm5 + vpandn %ymm12, %ymm7, %ymm9 + vpsrlq $0x31, %ymm6, %ymm11 + vpsllq $0xf, %ymm6, %ymm6 + vpxor %ymm2, %ymm9, %ymm9 + vpor %ymm6, %ymm11, %ymm11 + vpandn %ymm11, %ymm12, %ymm6 + vpxor %ymm7, %ymm6, %ymm6 + vmovdqu %ymm6, 0x160(%rsp) + vpxor 0x1e0(%rsp), %ymm1, %ymm6 + vpxor 0x2a0(%rsp), %ymm1, %ymm1 + vpshufb (%rcx), %ymm6, %ymm6 + vpandn %ymm6, %ymm11, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vmovdqu %ymm13, 0x180(%rsp) + vpandn %ymm2, %ymm6, %ymm13 + vpandn %ymm7, %ymm2, %ymm2 + vpxor %ymm6, %ymm2, %ymm2 + vpsrlq $0x3e, %ymm4, %ymm6 + vpxor %ymm11, %ymm13, %ymm13 + vmovdqu %ymm2, 0x1a0(%rsp) + vpsrlq $0x2, %ymm5, %ymm2 + vpsllq $0x3e, %ymm5, %ymm5 + vpor %ymm5, %ymm2, %ymm2 + vpsrlq $0x9, %ymm1, %ymm5 + vpsllq $0x37, %ymm1, %ymm1 + vpsllq $0x2, %ymm4, %ymm4 + vpor %ymm1, %ymm5, %ymm1 + vpsrlq $0x19, %ymm0, %ymm5 + vpor %ymm4, %ymm6, %ymm4 + vpsllq $0x27, %ymm0, %ymm0 + vpor %ymm0, %ymm5, %ymm5 + vpandn %ymm5, %ymm1, %ymm0 + vpxor %ymm2, %ymm0, %ymm0 + vmovdqu %ymm0, 0x1c0(%rsp) + vpsrlq $0x17, %ymm3, %ymm0 + vpsllq $0x29, %ymm3, %ymm3 + vpor %ymm3, %ymm0, %ymm0 + vpandn %ymm4, %ymm0, %ymm7 + vpandn %ymm0, %ymm5, %ymm3 + vpxor %ymm5, %ymm7, %ymm7 + vpandn %ymm2, %ymm4, %ymm5 + vpandn %ymm1, %ymm2, %ymm2 + vpxor %ymm0, %ymm5, %ymm5 + vpxor %ymm1, %ymm3, %ymm3 + vpxor %ymm4, %ymm2, %ymm2 + vmovdqu %ymm5, 0x1e0(%rsp) + addq $0x8, %rsi + addq $0x1, %r10 + cmpq $0x18, %r10 + jne LLkeccak_f1600_x4_avx2 + vmovdqu (%rsp), %ymm4 + vmovdqu 0x40(%rsp), %ymm5 + vmovdqu 0x20(%rsp), %ymm0 + vmovdqu 0x60(%rsp), %ymm1 + vmovdqu 0x1c0(%rsp), %ymm12 + vmovdqu %ymm2, 0x1c0(%rsp) + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vpunpcklqdq %ymm1, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm1[0],ymm5[2],ymm1[2] + vpunpckhqdq %ymm1, %ymm5, %ymm1 # ymm1 = ymm5[1],ymm1[1],ymm5[3],ymm1[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vmovdqu 0x80(%rsp), %ymm4 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm6, (%rdi) + vmovdqu %ymm5, 0xc8(%rdi) + vmovdqu %ymm2, 0x190(%rdi) + vmovdqu %ymm0, 0x258(%rdi) + vmovdqu 0xa0(%rsp), %ymm0 + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm1 # ymm1 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vmovdqu 0xc0(%rsp), %ymm0 + vpunpcklqdq %ymm10, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm10[0],ymm0[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm10[1],ymm0[3],ymm10[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vmovdqu 0xe0(%rsp), %ymm4 + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x100(%rsp), %ymm0 + vmovdqu %ymm2, 0x1b0(%rdi) + vmovdqu %ymm1, 0x278(%rdi) + vpunpcklqdq %ymm4, %ymm14, %ymm2 # ymm2 = ymm14[0],ymm4[0],ymm14[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm14, %ymm1 # ymm1 = ymm14[1],ymm4[1],ymm14[3],ymm4[3] + vpunpcklqdq %ymm8, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm8[0],ymm0[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3] + vmovdqu %ymm6, 0x20(%rdi) + vmovdqu %ymm5, 0xe8(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x120(%rsp), %ymm4 + vmovdqu 0x140(%rsp), %ymm0 + vmovdqu %ymm2, 0x1d0(%rdi) + vmovdqu %ymm1, 0x298(%rdi) + vpunpcklqdq %ymm4, %ymm15, %ymm2 # ymm2 = ymm15[0],ymm4[0],ymm15[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm15, %ymm1 # ymm1 = ymm15[1],ymm4[1],ymm15[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm9[0],ymm0[2],ymm9[2] + vmovdqu %ymm5, 0x108(%rdi) + vpunpckhqdq %ymm9, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm9[1],ymm0[3],ymm9[3] + vmovdqu %ymm6, 0x40(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vmovdqu 0x160(%rsp), %ymm4 + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x180(%rsp), %ymm0 + vmovdqu %ymm5, 0x128(%rdi) + vmovdqu 0x1a0(%rsp), %ymm5 + vmovdqu %ymm2, 0x1f0(%rdi) + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vpunpcklqdq %ymm5, %ymm13, %ymm4 # ymm4 = ymm13[0],ymm5[0],ymm13[2],ymm5[2] + vmovdqu %ymm6, 0x60(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vmovdqu %ymm1, 0x2b8(%rdi) + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vpunpckhqdq %ymm5, %ymm13, %ymm1 # ymm1 = ymm13[1],ymm5[1],ymm13[3],ymm5[3] + vmovdqu %ymm6, 0x80(%rdi) + vmovdqu 0x1e0(%rsp), %ymm4 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm2, 0x210(%rdi) + vpunpcklqdq %ymm3, %ymm12, %ymm2 # ymm2 = ymm12[0],ymm3[0],ymm12[2],ymm3[2] + vmovdqu %ymm0, 0x2d8(%rdi) + vpunpckhqdq %ymm3, %ymm12, %ymm0 # ymm0 = ymm12[1],ymm3[1],ymm12[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm4[0],ymm7[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm7, %ymm1 # ymm1 = ymm7[1],ymm4[1],ymm7[3],ymm4[3] + vmovdqu %ymm5, 0x148(%rdi) + vperm2i128 $0x20, %ymm3, %ymm2, %ymm5 # ymm5 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x1c0(%rsp), %ymm3 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm5, 0xa0(%rdi) + vextracti128 $0x1, %ymm3, %xmm15 + vmovdqu %ymm4, 0x168(%rdi) + vmovdqu %ymm2, 0x230(%rdi) + vmovdqu %ymm0, 0x2f8(%rdi) + vmovq %xmm3, 0xc0(%rdi) + vmovhpd %xmm3, 0x188(%rdi) + vmovq %xmm15, 0x250(%rdi) + vmovhpd %xmm15, 0x318(%rdi) + movq %r11, %rsp + .cfi_def_cfa_register %rsp + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_avx2_asm) + +#endif /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccakf1600_constants.c b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccakf1600_constants.c new file mode 100644 index 0000000..1fb5891 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/fips202/native/x86_64/src/keccakf1600_constants.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include + +#include "fips202_native_x86_64.h" + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t + mlk_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho8[4] = { + 0x0605040302010007, + 0x0e0d0c0b0a09080f, + 0x1615141312111017, + 0x1e1d1c1b1a19181f, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho56[4] = { + 0x0007060504030201, + 0x080f0e0d0c0b0a09, + 0x1017161514131211, + 0x181f1e1d1c1b1a19, +}; + +#else /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(fips202_x86_64_constants) + +#endif /* !(MLK_FIPS202_X86_64_NEED_X4_AVX2 && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.c b/sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.c new file mode 100644 index 0000000..50d2b92 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.c @@ -0,0 +1,653 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "indcpa.h" + +#include "debug.h" +#include "randombytes.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_pack_pk MLK_ADD_PARAM_SET(mlk_pack_pk) +#define mlk_unpack_pk MLK_ADD_PARAM_SET(mlk_unpack_pk) +#define mlk_pack_sk MLK_ADD_PARAM_SET(mlk_pack_sk) +#define mlk_unpack_sk MLK_ADD_PARAM_SET(mlk_unpack_sk) +#define mlk_pack_ciphertext MLK_ADD_PARAM_SET(mlk_pack_ciphertext) +#define mlk_unpack_ciphertext MLK_ADD_PARAM_SET(mlk_unpack_ciphertext) +#define mlk_matvec_mul MLK_ADD_PARAM_SET(mlk_matvec_mul) +#define mlk_polyvec_permute_bitrev_to_custom \ + MLK_ADD_PARAM_SET(mlk_polyvec_permute_bitrev_to_custom) +#define mlk_polymat_permute_bitrev_to_custom \ + MLK_ADD_PARAM_SET(mlk_polymat_permute_bitrev_to_custom) +#define mlk_keypair_getnoise_eta1 MLK_ADD_PARAM_SET(mlk_keypair_getnoise_eta1) +#define mlk_enc_getnoise_eta1_eta2 MLK_ADD_PARAM_SET(mlk_enc_getnoise_eta1_eta2) +/* End of parameter set namespacing */ + +/** + * Serialize the public key as the concatenation of the serialized vector of + * polynomials pk and the public seed used to generate the matrix A. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L19].} + * + * @param[out] r Output serialized public key. + * @param[in] pk Input public-key polyvec. Must have coefficients within + * [0,..,MLKEM_Q-1]. + * @param[in] seed Input public seed. + */ +static void mlk_pack_pk(uint8_t r[MLKEM_INDCPA_PUBLICKEYBYTES], + const mlk_polyvec *pk, + const uint8_t seed[MLKEM_SYMBYTES]) +{ + mlk_assert_bound_2d(pk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + mlk_polyvec_tobytes(r, pk); + mlk_memcpy(r + MLKEM_POLYVECBYTES, seed, MLKEM_SYMBYTES); +} + +/** + * De-serialize public key from a byte array; approximate inverse of + * mlk_pack_pk. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L2-3].} + * + * @param[out] pk Output public-key polynomial vector. Coefficients + * will be normalized to [0,1,..,MLKEM_Q-1]. + * @param[out] seed Output seed to generate matrix A. + * @param[in] packedpk Input serialized public key. + */ +static void mlk_unpack_pk(mlk_polyvec *pk, uint8_t seed[MLKEM_SYMBYTES], + const uint8_t packedpk[MLKEM_INDCPA_PUBLICKEYBYTES]) +{ + mlk_polyvec_frombytes(pk, packedpk); + mlk_memcpy(seed, packedpk + MLKEM_POLYVECBYTES, MLKEM_SYMBYTES); + + /* NOTE: If a modulus check was conducted on the PK, we know at this + * point that the coefficients of `pk` are unsigned canonical. The + * specifications and proofs, however, do _not_ assume this, and instead + * work with the easily provable bound by MLKEM_UINT12_LIMIT. */ +} + +/** + * Serialize the secret key. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L20].} + * + * @param[out] r Output serialized secret key. + * @param[in] sk Input vector of polynomials (secret key). + */ +static void mlk_pack_sk(uint8_t r[MLKEM_INDCPA_SECRETKEYBYTES], + const mlk_polyvec *sk) +{ + mlk_assert_bound_2d(sk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + mlk_polyvec_tobytes(r, sk); +} + +/** + * De-serialize the secret key; inverse of mlk_pack_sk. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L5].} + * + * @param[out] sk Output vector of polynomials (secret key). + * @param[in] packedsk Input serialized secret key. + */ +static void mlk_unpack_sk(mlk_polyvec *sk, + const uint8_t packedsk[MLKEM_INDCPA_SECRETKEYBYTES]) +{ + mlk_polyvec_frombytes(sk, packedsk); +} + +/** + * Serialize the ciphertext as the concatenation of the compressed and + * serialized vector of polynomials b and the compressed and serialized + * polynomial v. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L22-23].} + * + * @param[out] r Output serialized ciphertext. + * @param[in] b Input vector of polynomials b. + * @param[in] v Input polynomial v. + */ +static void mlk_pack_ciphertext(uint8_t r[MLKEM_INDCPA_BYTES], + const mlk_polyvec *b, mlk_poly *v) +{ + mlk_polyvec_compress_du(r, b); + mlk_poly_compress_dv(r + MLKEM_POLYVECCOMPRESSEDBYTES_DU, v); +} + +/** + * De-serialize and decompress ciphertext from a byte array; approximate + * inverse of mlk_pack_ciphertext. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L1-4].} + * + * @param[out] b Output vector of polynomials b. + * @param[out] v Output polynomial v. + * @param[in] c Input serialized ciphertext. + */ +static void mlk_unpack_ciphertext(mlk_polyvec *b, mlk_poly *v, + const uint8_t c[MLKEM_INDCPA_BYTES]) +{ + mlk_polyvec_decompress_du(b, c); + mlk_poly_decompress_dv(v, c + MLKEM_POLYVECCOMPRESSEDBYTES_DU); +} + +/* Helper function to ensure that the polynomial entries in the output + * of gen_matrix use the standard (bitreversed) ordering of coefficients. + * No-op unless a native backend with a custom ordering is used. + * + * We don't inline this into gen_matrix to avoid having to split the CBMC + * proof for gen_matrix based on MLK_USE_NATIVE_NTT_CUSTOM_ORDER. */ +static void mlk_polyvec_permute_bitrev_to_custom(mlk_polyvec *v) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(v, sizeof(mlk_polyvec))) + requires(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(v, sizeof(mlk_polyvec))) + ensures(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) +{ +#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER) + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(v, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + invariant(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + decreases(MLKEM_K - i)) + { + mlk_poly_permute_bitrev_to_custom(v->vec[i].coeffs); + } +#else /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ + /* Nothing to do */ + (void)v; +#endif /* !MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ +} + +static void mlk_polymat_permute_bitrev_to_custom(mlk_polymat *a) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) + assigns(memory_slice(a, sizeof(mlk_polymat))) + ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(a, sizeof(mlk_polymat))) + invariant(i <= MLKEM_K) + invariant(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) + decreases(MLKEM_K - i)) + { + mlk_polyvec_permute_bitrev_to_custom(&a->vec[i]); + } +} + +/* Reference: `gen_matrix()` in the reference implementation @[REF]. + * - We use a special subroutine to generate 4 polynomials + * at a time, to be able to leverage batched Keccak-f1600 + * implementations. The reference implementation generates + * one matrix entry a time. + * + * Not static for benchmarking */ +MLK_INTERNAL_API +void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES], + int transposed) +{ + unsigned i, j; + MLK_ALIGN uint8_t seed_ext[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]; + + for (j = 0; j < 4; j++) + { + mlk_memcpy(seed_ext[j], seed, MLKEM_SYMBYTES); + } + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + /* Sample 4 matrix entries a time. */ + for (i = 0; i < (MLKEM_K * MLKEM_K / 4) * 4; i += 4) + { + for (j = 0; j < 4; j++) + { + uint8_t x, y; + /* MLKEM_K <= 4, so the values fit in uint8_t. */ + x = (uint8_t)((i + j) / MLKEM_K); + y = (uint8_t)((i + j) % MLKEM_K); + if (transposed) + { + seed_ext[j][MLKEM_SYMBYTES + 0] = x; + seed_ext[j][MLKEM_SYMBYTES + 1] = y; + } + else + { + seed_ext[j][MLKEM_SYMBYTES + 0] = y; + seed_ext[j][MLKEM_SYMBYTES + 1] = x; + } + } + + mlk_poly_rej_uniform_x4(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], + &a->vec[(i + 1) / MLKEM_K].vec[(i + 1) % MLKEM_K], + &a->vec[(i + 2) / MLKEM_K].vec[(i + 2) % MLKEM_K], + &a->vec[(i + 3) / MLKEM_K].vec[(i + 3) % MLKEM_K], + seed_ext); + } +#else /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + /* When using serial FIPS202, sample all entries individually. */ + i = 0; +#endif /* MLK_CONFIG_SERIAL_FIPS202_ONLY */ + + /* For MLKEM_K == 3, sample the last entry individually. + * When MLK_CONFIG_SERIAL_FIPS202_ONLY is set, sample all entries + * individually. */ + for (; i < MLKEM_K * MLKEM_K; i++) + { + uint8_t x, y; + /* MLKEM_K <= 4, so the values fit in uint8_t. */ + x = (uint8_t)(i / MLKEM_K); + y = (uint8_t)(i % MLKEM_K); + + if (transposed) + { + seed_ext[0][MLKEM_SYMBYTES + 0] = x; + seed_ext[0][MLKEM_SYMBYTES + 1] = y; + } + else + { + seed_ext[0][MLKEM_SYMBYTES + 0] = y; + seed_ext[0][MLKEM_SYMBYTES + 1] = x; + } + + mlk_poly_rej_uniform(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], seed_ext[0]); + } + + mlk_assert(i == MLKEM_K * MLKEM_K); + + /* + * The public matrix is generated in NTT domain. If the native backend + * uses a custom order in NTT domain, permute A accordingly. + */ + mlk_polymat_permute_bitrev_to_custom(a); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(seed_ext, sizeof(seed_ext)); +} + +/** + * Compute matrix-vector product in NTT domain, via Montgomery multiplication. + * + * @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.12), (2.13)].} + * + * @param[out] out Output polynomial vector. + * @param[in] a Input matrix. Must be in NTT domain and have coefficients + * of absolute value < 4096. + * @param[in] v Input polynomial vector. Must be in NTT domain. + * @param[in] vc Mulcache for @p v, computed via + * mlk_polyvec_mulcache_compute(). + */ +static void mlk_matvec_mul(mlk_polyvec *out, const mlk_polymat *a, + const mlk_polyvec *v, const mlk_polyvec_mulcache *vc) +__contract__( + requires(memory_no_alias(out, sizeof(mlk_polyvec))) + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(memory_no_alias(v, sizeof(mlk_polyvec))) + requires(memory_no_alias(vc, sizeof(mlk_polyvec_mulcache))) + requires(forall(k0, 0, MLKEM_K, + forall(k1, 0, MLKEM_K, + array_bound(a->vec[k0].vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)))) + assigns(memory_slice(out, sizeof(mlk_polyvec)))) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(out, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + decreases(MLKEM_K - i)) + { + mlk_polyvec_basemul_acc_montgomery_cached(&out->vec[i], &a->vec[i], v, vc); + } +} + +/** + * Compute and fill the pv and e polyvec structures needed by + * mlk_keypair_derand(). Uses x4-batched versions of `poly_getnoise` to + * leverage batched Keccak-f1600. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)] steps 8-15.} + * + * @param[out] pv Output polynomial vector. + * @param[out] e Output polynomial vector. + * @param[in] seed Seed bytes for sampling. + */ +static void mlk_keypair_getnoise_eta1(mlk_polyvec *pv, mlk_polyvec *e, + const uint8_t seed[MLKEM_SYMBYTES]) +__contract__( + requires(memory_no_alias(pv, sizeof(mlk_polyvec))) + requires(memory_no_alias(e, sizeof(mlk_polyvec))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(pv, sizeof(mlk_polyvec))) + assigns(memory_slice(e, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, array_abs_bound(pv->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) + ensures(forall(k1, 0, MLKEM_K, array_abs_bound(e->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) +) +{ +#if MLKEM_K == 2 + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], /* Fill elements of pv */ + &e->vec[0], &e->vec[1], /* and two elements of e */ + seed, 0, 1, 2, 3); +#elif MLKEM_K == 3 + /* + * Only the first three output buffers are needed, so we pass NULL as + * the fourth parameter, and 0xFF as its dummy nonce. + */ + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], NULL, seed, + 0, 1, 2, 0xFF); + /* Same here */ + mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], NULL, seed, 3, + 4, 5, 0xFF); +#elif MLKEM_K == 4 + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], &pv->vec[3], + seed, 0, 1, 2, 3); + mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], &e->vec[3], + seed, 4, 5, 6, 7); +#endif /* MLKEM_K == 4 */ +} + +/** + * Compute and fill the sp, ep, and epp polynomial structures needed by + * mlk_indcpa_enc(). Uses x4-batched versions of `poly_getnoise` to leverage + * batched Keccak-f1600. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)] steps 9-16.} + * + * @param[out] sp Output polynomial vector. + * @param[out] ep Output polynomial vector. + * @param[out] epp Output polynomial. + * @param[in] coins Seed bytes for sampling. + */ +static void mlk_enc_getnoise_eta1_eta2(mlk_polyvec *sp, mlk_polyvec *ep, + mlk_poly *epp, + const uint8_t coins[MLKEM_SYMBYTES]) +__contract__( + requires(memory_no_alias(sp, sizeof(mlk_polyvec))) + requires(memory_no_alias(ep, sizeof(mlk_polyvec))) + requires(memory_no_alias(epp, sizeof(mlk_poly))) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(sp, sizeof(mlk_polyvec))) + assigns(memory_slice(ep, sizeof(mlk_polyvec))) + assigns(memory_slice(epp, sizeof(mlk_poly))) + ensures(forall(k0, 0, MLKEM_K, array_abs_bound(sp->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) + ensures(forall(k1, 0, MLKEM_K, array_abs_bound(ep->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))) + ensures(array_abs_bound(epp->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)) +) +{ +#if MLKEM_K == 2 + mlk_poly_getnoise_eta1122_4x(&sp->vec[0], &sp->vec[1], &ep->vec[0], + &ep->vec[1], coins, 0, 1, 2, 3); + mlk_poly_getnoise_eta2(epp, coins, 4); +#elif MLKEM_K == 3 + /* + * In this call, only the first three output buffers are needed, + * so we pass NULL as the fourth parameter, and 0xFF as its dummy nonce. + */ + mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], NULL, coins, + 0, 1, 2, 0xFF /* irrelevant */); + /* The fourth output buffer in this call _is_ used. */ + mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], epp, coins, + 3, 4, 5, 6); +#elif MLKEM_K == 4 + mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], &sp->vec[3], + coins, 0, 1, 2, 3); + mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], &ep->vec[3], + coins, 4, 5, 6, 7); + mlk_poly_getnoise_eta2(epp, coins, 8); +#endif /* MLKEM_K == 4 */ +} + + +/* Reference: `indcpa_keypair_derand()` in the reference implementation @[REF]. + * - We use a different implementation of `gen_matrix()` which + * uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above). + * - We use a mulcache to speed up matrix-vector multiplication. + * - We include buffer zeroization. + */ +MLK_INTERNAL_API +int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + const uint8_t *publicseed; + const uint8_t *noiseseed; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context); + MLK_ALLOC(a, mlk_polymat, 1, context); + MLK_ALLOC(e, mlk_polyvec, 1, context); + MLK_ALLOC(pkpv, mlk_polyvec, 1, context); + MLK_ALLOC(skpv, mlk_polyvec, 1, context); + MLK_ALLOC(skpv_cache, mlk_polyvec_mulcache, 1, context); + + if (buf == NULL || coins_with_domain_separator == NULL || a == NULL || + e == NULL || pkpv == NULL || skpv == NULL || skpv_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + publicseed = buf; + noiseseed = buf + MLKEM_SYMBYTES; + + /* Concatenate coins with MLKEM_K for domain separation of security levels */ + mlk_memcpy(coins_with_domain_separator, coins, MLKEM_SYMBYTES); + coins_with_domain_separator[MLKEM_SYMBYTES] = MLKEM_K; + + mlk_hash_g(buf, coins_with_domain_separator, MLKEM_SYMBYTES + 1); + + /* + * Declassify the public seed. + * Required to use it in conditional-branches in rejection sampling. + * This is needed because all output of randombytes is marked as secret + * (=undefined) + */ + MLK_CT_TESTING_DECLASSIFY(publicseed, MLKEM_SYMBYTES); + + mlk_gen_matrix(a, publicseed, 0 /* no transpose */); + + mlk_keypair_getnoise_eta1(skpv, e, noiseseed); + + mlk_polyvec_ntt(skpv); + mlk_polyvec_ntt(e); + + mlk_polyvec_mulcache_compute(skpv_cache, skpv); + mlk_matvec_mul(pkpv, a, skpv, skpv_cache); + mlk_polyvec_tomont(pkpv); + + mlk_polyvec_add(pkpv, e); + mlk_polyvec_reduce(pkpv); + mlk_polyvec_reduce(skpv); + + mlk_pack_sk(sk, skpv); + mlk_pack_pk(pk, pkpv, publicseed); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(skpv_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(skpv, mlk_polyvec, 1, context); + MLK_FREE(pkpv, mlk_polyvec, 1, context); + MLK_FREE(e, mlk_polyvec, 1, context); + MLK_FREE(a, mlk_polymat, 1, context); + MLK_FREE(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} + +/* Reference: `indcpa_enc()` in the reference implementation @[REF]. + * - We use x4-batched versions of `poly_getnoise` to leverage + * batched x4-batched Keccak-f1600. + * - We use a different implementation of `gen_matrix()` which + * uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above). + * - We use a mulcache to speed up matrix-vector multiplication. + * - We include buffer zeroization. + */ +MLK_INTERNAL_API +int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(seed, uint8_t, MLKEM_SYMBYTES, context); + MLK_ALLOC(at, mlk_polymat, 1, context); + MLK_ALLOC(sp, mlk_polyvec, 1, context); + MLK_ALLOC(pkpv, mlk_polyvec, 1, context); + MLK_ALLOC(ep, mlk_polyvec, 1, context); + MLK_ALLOC(b, mlk_polyvec, 1, context); + MLK_ALLOC(v, mlk_poly, 1, context); + MLK_ALLOC(k, mlk_poly, 1, context); + MLK_ALLOC(epp, mlk_poly, 1, context); + MLK_ALLOC(sp_cache, mlk_polyvec_mulcache, 1, context); + + if (seed == NULL || at == NULL || sp == NULL || pkpv == NULL || ep == NULL || + b == NULL || v == NULL || k == NULL || epp == NULL || sp_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_unpack_pk(pkpv, seed, pk); + mlk_poly_frommsg(k, m); + + /* + * Declassify the public seed. + * Required to use it in conditional-branches in rejection sampling. + * This is needed because in re-encryption the publicseed originated from sk + * which is marked undefined. + */ + MLK_CT_TESTING_DECLASSIFY(seed, MLKEM_SYMBYTES); + + mlk_gen_matrix(at, seed, 1 /* transpose */); + + mlk_enc_getnoise_eta1_eta2(sp, ep, epp, coins); + + mlk_polyvec_ntt(sp); + + mlk_polyvec_mulcache_compute(sp_cache, sp); + mlk_matvec_mul(b, at, sp, sp_cache); + mlk_polyvec_basemul_acc_montgomery_cached(v, pkpv, sp, sp_cache); + + mlk_polyvec_invntt_tomont(b); + mlk_poly_invntt_tomont(v); + + mlk_polyvec_add(b, ep); + mlk_poly_add(v, epp); + mlk_poly_add(v, k); + + mlk_polyvec_reduce(b); + mlk_poly_reduce(v); + + mlk_pack_ciphertext(c, b, v); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(sp_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(epp, mlk_poly, 1, context); + MLK_FREE(k, mlk_poly, 1, context); + MLK_FREE(v, mlk_poly, 1, context); + MLK_FREE(b, mlk_polyvec, 1, context); + MLK_FREE(ep, mlk_polyvec, 1, context); + MLK_FREE(pkpv, mlk_polyvec, 1, context); + MLK_FREE(sp, mlk_polyvec, 1, context); + MLK_FREE(at, mlk_polymat, 1, context); + MLK_FREE(seed, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} + +/* Reference: `indcpa_dec()` in the reference implementation @[REF]. + * - We use a mulcache for the scalar product. + * - We include buffer zeroization. */ +MLK_INTERNAL_API +int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(b, mlk_polyvec, 1, context); + MLK_ALLOC(skpv, mlk_polyvec, 1, context); + MLK_ALLOC(v, mlk_poly, 1, context); + MLK_ALLOC(sb, mlk_poly, 1, context); + MLK_ALLOC(b_cache, mlk_polyvec_mulcache, 1, context); + + if (b == NULL || skpv == NULL || v == NULL || sb == NULL || b_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_unpack_ciphertext(b, v, c); + mlk_unpack_sk(skpv, sk); + + mlk_polyvec_ntt(b); + mlk_polyvec_mulcache_compute(b_cache, b); + mlk_polyvec_basemul_acc_montgomery_cached(sb, skpv, b, b_cache); + mlk_poly_invntt_tomont(sb); + + mlk_poly_sub(v, sb); + mlk_poly_reduce(v); + + mlk_poly_tomsg(m, v); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(b_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(sb, mlk_poly, 1, context); + MLK_FREE(v, mlk_poly, 1, context); + MLK_FREE(skpv, mlk_polyvec, 1, context); + MLK_FREE(b, mlk_polyvec, 1, context); + return ret; +} + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_pack_pk +#undef mlk_unpack_pk +#undef mlk_pack_sk +#undef mlk_unpack_sk +#undef mlk_pack_ciphertext +#undef mlk_unpack_ciphertext +#undef mlk_matvec_mul +#undef mlk_polyvec_permute_bitrev_to_custom +#undef mlk_polymat_permute_bitrev_to_custom +#undef mlk_keypair_getnoise_eta1 +#undef mlk_enc_getnoise_eta1_eta2 diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.h b/sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.h new file mode 100644 index 0000000..5fe5e22 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_INDCPA_H +#define MLK_INDCPA_H + +#include "cbmc.h" +#include "common.h" +#include "poly_k.h" + +#define mlk_gen_matrix MLK_NAMESPACE_K(gen_matrix) +/** + * Deterministically generate matrix A (or the transpose of A) from a seed. + * Entries of the matrix are polynomials that look uniformly random. + * Performs rejection sampling on the output of an XOF. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L3-7] and + * @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L4-8]. The @p transposed + * parameter only affects internal presentation.} + * + * @param[out] a Output matrix A. + * @param[in] seed Input seed. + * @param transposed Boolean deciding whether A or A^T is generated. + */ +MLK_INTERNAL_API +void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES], + int transposed) +__contract__( + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + requires(transposed == 0 || transposed == 1) + assigns(memory_slice(a, sizeof(mlk_polymat))) + ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) +); + +#define mlk_indcpa_keypair_derand \ + MLK_NAMESPACE_K(indcpa_keypair_derand) MLK_CONTEXT_PARAMETERS_3 +/** + * Generate public and private key for the CPA-secure public-key encryption + * scheme underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)].} + * + * @param[out] pk Output public key + * (length MLKEM_INDCPA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key + * (length MLKEM_INDCPA_SECRETKEYBYTES bytes). + * @param[in] coins Input randomness (length MLKEM_SYMBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) +); + +#define mlk_indcpa_enc MLK_NAMESPACE_K(indcpa_enc) MLK_CONTEXT_PARAMETERS_4 +/** + * Encryption function of the CPA-secure public-key encryption scheme + * underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)].} + * + * @param[out] c Output ciphertext (length MLKEM_INDCPA_BYTES bytes). + * @param[in] m Input message (length MLKEM_INDCPA_MSGBYTES bytes). + * @param[in] pk Input public key + * (length MLKEM_INDCPA_PUBLICKEYBYTES bytes). + * @param[in] coins Input random coins used as seed (length MLKEM_SYMBYTES + * bytes) to deterministically generate all randomness. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL Operation failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(c, MLKEM_INDCPA_BYTES)) + requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(c, MLKEM_INDCPA_BYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY) +); + +#define mlk_indcpa_dec MLK_NAMESPACE_K(indcpa_dec) MLK_CONTEXT_PARAMETERS_3 +/** + * Decryption function of the CPA-secure public-key encryption scheme + * underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt)].} + * + * @param[out] m Output decrypted message + * (length MLKEM_INDCPA_MSGBYTES bytes). + * @param[in] c Input ciphertext (length MLKEM_INDCPA_BYTES bytes). + * @param[in] sk Input secret key + * (length MLKEM_INDCPA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL Operation failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(c, MLKEM_INDCPA_BYTES)) + requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + assigns(memory_slice(m, MLKEM_INDCPA_MSGBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY) +); + +#endif /* !MLK_INDCPA_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/kem.c b/sdk/software/examples/mlkem_test/mlkem_native/src/kem.c new file mode 100644 index 0000000..3c82d6d --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/kem.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "kem.h" + +#include "indcpa.h" +#include "randombytes.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying security levels) + * within a single compilation unit. */ +#define mlk_check_pct MLK_ADD_PARAM_SET(mlk_check_pct) MLK_CONTEXT_PARAMETERS_2 +/* End of parameter set namespacing */ + +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(p, mlk_polyvec, 1, context); + MLK_ALLOC(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context); + + if (p == NULL || p_reencoded == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_polyvec_frombytes(p, pk); + mlk_polyvec_reduce(p); + mlk_polyvec_tobytes(p_reencoded, p); + + /* We use a constant-time memcmp here to avoid having to + * declassify the PK before the PCT has succeeded. */ + ret = mlk_ct_memcmp(pk, p_reencoded, MLKEM_POLYVECBYTES) ? MLK_ERR_FAIL : 0; + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context); + MLK_FREE(p, mlk_polyvec, 1, context); + return ret; +} + + +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(test, uint8_t, MLKEM_SYMBYTES, context); + + if (test == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* + * The parts of `sk` being hashed and compared here are public, so + * no public information is leaked through the runtime or the return value + * of this function. + */ + + /* Declassify the public part of the secret key */ + MLK_CT_TESTING_DECLASSIFY(sk + MLKEM_INDCPA_SECRETKEYBYTES, + MLKEM_INDCCA_PUBLICKEYBYTES); + MLK_CT_TESTING_DECLASSIFY( + sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, MLKEM_SYMBYTES); + + mlk_hash_h(test, sk + MLKEM_INDCPA_SECRETKEYBYTES, + MLKEM_INDCCA_PUBLICKEYBYTES); + /* This doesn't have to be a constant-time memcmp, but it's the only place + * in the library where a normal memcmp would be used otherwise, so for sake + * of minimizing stdlib dependency, we use our constant-time one anyway. */ + ret = mlk_ct_memcmp(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, + test, MLKEM_SYMBYTES) + ? MLK_ERR_FAIL + : 0; + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(test, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} + +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) +); + +#if defined(MLK_CONFIG_KEYGEN_PCT) +/* Specification: + * Partially implements 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency]. */ + +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context); + MLK_ALLOC(ss_enc, uint8_t, MLKEM_SSBYTES, context); + MLK_ALLOC(ss_dec, uint8_t, MLKEM_SSBYTES, context); + + if (ct == NULL || ss_enc == NULL || ss_dec == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + ret = mlk_kem_enc(ct, ss_enc, pk, context); + if (ret != 0) + { + goto cleanup; + } + + ret = mlk_kem_dec(ss_dec, ct, sk, context); + if (ret != 0) + { + goto cleanup; + } + +#if defined(MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST) + /* Deliberately break PCT for testing purposes */ + if (mlk_break_pct()) + { + ss_enc[0] = ~ss_enc[0]; + } +#endif /* MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST */ + + ret = mlk_ct_memcmp(ss_enc, ss_dec, MLKEM_SSBYTES); + /* The result of the PCT is public. */ + MLK_CT_TESTING_DECLASSIFY(&ret, sizeof(ret)); + + if (ret != 0) + { + ret = MLK_ERR_FAIL; + } + +cleanup: + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(ss_dec, uint8_t, MLKEM_SSBYTES, context); + MLK_FREE(ss_enc, uint8_t, MLKEM_SSBYTES, context); + MLK_FREE(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context); + return ret; +} +#else /* MLK_CONFIG_KEYGEN_PCT */ +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + /* Skip PCT */ + ((void)pk); + ((void)sk); +#if defined(MLK_CONFIG_CONTEXT_PARAMETER) + ((void)context); +#endif + return 0; +} +#endif /* !MLK_CONFIG_KEYGEN_PCT */ + +/* Reference: `crypto_kem_keypair_derand()` in the reference implementation + * @[REF]. + * - We optionally include PCT which is not present in + * the reference code. */ +MLK_EXTERNAL_API +int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + const uint8_t coins[2 * MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret; + + ret = mlk_indcpa_keypair_derand(pk, sk, coins, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(sk + MLKEM_INDCPA_SECRETKEYBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_hash_h(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, pk, + MLKEM_INDCCA_PUBLICKEYBYTES); + /* Value z for pseudo-random output on reject */ + mlk_memcpy(sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES, + coins + MLKEM_SYMBYTES, MLKEM_SYMBYTES); + + /* Declassify public key */ + MLK_CT_TESTING_DECLASSIFY(pk, MLKEM_INDCCA_PUBLICKEYBYTES); + + /* Pairwise Consistency Test (PCT) @[FIPS140_3_IG, p.87] */ + ret = mlk_check_pct(pk, sk, context); + if (ret != 0) + { + goto cleanup; + } + +cleanup: + if (ret != 0) + { + mlk_zeroize(pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_zeroize(sk, MLKEM_INDCCA_SECRETKEYBYTES); + } + + return ret; +} + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/* Reference: `crypto_kem_keypair()` in the reference implementation @[REF] + * - We zeroize the stack buffer */ +MLK_EXTERNAL_API +int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(coins, uint8_t, 2 * MLKEM_SYMBYTES, context); + + if (coins == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Acquire necessary randomness, and mark it as secret. */ + if (mlk_randombytes(coins, 2 * MLKEM_SYMBYTES) != 0) + { + ret = MLK_ERR_RNG_FAIL; + goto cleanup; + } + + MLK_CT_TESTING_SECRET(coins, 2 * MLKEM_SYMBYTES); + + ret = mlk_kem_keypair_derand(pk, sk, coins, context); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(coins, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ + +/* Reference: `crypto_kem_enc_derand()` in the reference implementation @[REF] + * - We include public key check + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + + if (buf == NULL || kr == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Specification: Implements @[FIPS203, Section 7.2, Modulus check] */ + ret = mlk_kem_check_pk(pk, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(buf, coins, MLKEM_SYMBYTES); + + /* Multitarget countermeasure for coins + contributory KEM */ + mlk_hash_h(buf + MLKEM_SYMBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES); + + /* coins are in kr+MLKEM_SYMBYTES */ + ret = mlk_indcpa_enc(ct, buf, pk, kr + MLKEM_SYMBYTES, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(ss, kr, MLKEM_SYMBYTES); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/* Reference: `crypto_kem_enc()` in the reference implementation @[REF] + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(coins, uint8_t, MLKEM_SYMBYTES, context); + + if (coins == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + if (mlk_randombytes(coins, MLKEM_SYMBYTES) != 0) + { + ret = MLK_ERR_RNG_FAIL; + goto cleanup; + } + + MLK_CT_TESTING_SECRET(coins, MLKEM_SYMBYTES); + + ret = mlk_kem_enc_derand(ct, ss, pk, coins, context); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(coins, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ + +/* Reference: `crypto_kem_dec()` in the reference implementation @[REF] + * - We include secret key check + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES], + const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + uint8_t fail; + const uint8_t *pk = sk + MLKEM_INDCPA_SECRETKEYBYTES; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES, + context); + + if (buf == NULL || kr == NULL || tmp == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Specification: Implements @[FIPS203, Section 7.3, Hash check] */ + ret = mlk_kem_check_sk(sk, context); + if (ret != 0) + { + goto cleanup; + } + + ret = mlk_indcpa_dec(buf, ct, sk, context); + if (ret != 0) + { + goto cleanup; + } + + /* Multitarget countermeasure for coins + contributory KEM */ + mlk_memcpy(buf + MLKEM_SYMBYTES, + sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, + MLKEM_SYMBYTES); + mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES); + + /* Recompute and compare ciphertext */ + /* coins are in kr+MLKEM_SYMBYTES */ + ret = mlk_indcpa_enc(tmp, buf, pk, kr + MLKEM_SYMBYTES, context); + if (ret != 0) + { + goto cleanup; + } + + fail = mlk_ct_memcmp(ct, tmp, MLKEM_INDCCA_CIPHERTEXTBYTES); + + /* Compute rejection key */ + mlk_memcpy(tmp, sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES, + MLKEM_SYMBYTES); + mlk_memcpy(tmp + MLKEM_SYMBYTES, ct, MLKEM_INDCCA_CIPHERTEXTBYTES); + mlk_hash_j(ss, tmp, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES); + + /* Copy true key to return buffer if fail is 0 */ + mlk_ct_cmov_zero(ss, kr, MLKEM_SYMBYTES, fail); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES, + context); + MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + + return ret; +} + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_check_pct diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/kem.h b/sdk/software/examples/mlkem_test/mlkem_native/src/kem.h new file mode 100644 index 0000000..5310dc8 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/kem.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#ifndef MLK_KEM_H +#define MLK_KEM_H + +#include "cbmc.h" +#include "common.h" +#include "sys.h" + +#if defined(MLK_CHECK_APIS) +/* Include to ensure consistency between internal kem.h + * and external mlkem_native.h. */ +#include "mlkem_native.h" + +#if MLKEM_INDCCA_SECRETKEYBYTES != \ + MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for SECRETKEYBYTES between kem.h and mlkem_native.h +#endif + +#if MLKEM_INDCCA_PUBLICKEYBYTES != \ + MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for PUBLICKEYBYTES between kem.h and mlkem_native.h +#endif + +#if MLKEM_INDCCA_CIPHERTEXTBYTES != \ + MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for CIPHERTEXTBYTES between kem.h and mlkem_native.h +#endif + +#endif /* MLK_CHECK_APIS */ + +#define mlk_kem_keypair_derand \ + MLK_NAMESPACE_K(keypair_derand) MLK_CONTEXT_PARAMETERS_3 +#define mlk_kem_keypair MLK_NAMESPACE_K(keypair) MLK_CONTEXT_PARAMETERS_2 +#define mlk_kem_enc_derand MLK_NAMESPACE_K(enc_derand) MLK_CONTEXT_PARAMETERS_4 +#define mlk_kem_enc MLK_NAMESPACE_K(enc) MLK_CONTEXT_PARAMETERS_3 +#define mlk_kem_dec MLK_NAMESPACE_K(dec) MLK_CONTEXT_PARAMETERS_3 +#define mlk_kem_check_pk MLK_NAMESPACE_K(check_pk) MLK_CONTEXT_PARAMETERS_1 +#define mlk_kem_check_sk MLK_NAMESPACE_K(check_sk) MLK_CONTEXT_PARAMETERS_1 + +/** + * Implements modulus check mandated by FIPS 203, i.e., ensures that + * coefficients are in [0,q-1]. + * + * @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].} + * + * @reference{Not implemented in the reference implementation @[REF].} + * + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL Modulus check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY) +); + + +/** + * Implements public key hash check mandated by FIPS 203, i.e., ensures that + * sk[768𝑘+32 ∶ 768𝑘+64] = H(pk) = H(sk[384𝑘 : 768𝑘+32]). + * + * @spec{Implements @[FIPS203, Section 7.3, 'hash check'].} + * + * @reference{Not implemented in the reference implementation @[REF].} + * + * @param[in] sk Input private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL Public key hash check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY) +); + +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].} + * + * @param[out] pk Output public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param[in] coins Input randomness (an already allocated array filled + * with 2*MLKEM_SYMBYTES random bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + const uint8_t coins[2 * MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + requires(memory_no_alias(coins, 2 * MLKEM_SYMBYTES)) + assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) +); + +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].} + * + * @param[out] pk Output public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) +); + +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].} + * + * @param[out] ct Output ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[in] coins Input randomness (an already allocated array filled + * with MLKEM_SYMBYTES random bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY) +); + +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].} + * + * @param[out] ct Output ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) +); + +/** + * Generate shared secret for a given ciphertext and private key. + * + * @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].} + * + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] ct Input ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[in] sk Input private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_FAIL The 'hash check' @[FIPS203, Section 7.3] + * for the secret key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES], + const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY) +); + +#endif /* !MLK_KEM_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/README.md b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/README.md new file mode 100644 index 0000000..0d56871 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/README.md @@ -0,0 +1,16 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# AArch64 backend (little endian) + +This directory contains a native backend for little endian AArch64 systems. It is derived from [^NeonNTT] [^SLOTHY_Paper]. + +The code in this directory is auto-generated from the 'clean' assembly in [dev/aarch64_clean](../../../../dev/aarch64_clean) +in a two-step fashion: First, it is superoptimized using the [SLOTHY](https://github.com/slothy-optimizer/slothy) superoptimizer, +giving the assembly in [dev/aarch64_opt](../../../../dev/aarch64_opt). Then, it is stripped of remaining register aliases, macros +and most preprocessor directives by [`scripts/simpasm`](../../../../scripts/simpasm). + +If you want to understand how the assembly works, and/or make changes to it, consult [dev/](../../../../dev). + + +[^NeonNTT]: Becker, Hwang, Kannwischer, Yang, Yang: Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1, [https://eprint.iacr.org/2021/986](https://eprint.iacr.org/2021/986) +[^SLOTHY_Paper]: Abdulrahman, Becker, Kannwischer, Klein: Fast and Clean: Auditable high-performance assembly via constraint solving, [https://eprint.iacr.org/2022/1303](https://eprint.iacr.org/2022/1303) diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/meta.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/meta.h new file mode 100644 index 0000000..a2ef66c --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/meta.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_AARCH64_META_H +#define MLK_NATIVE_AARCH64_META_H + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#define MLK_USE_NATIVE_POLY_TOBYTES +#define MLK_USE_NATIVE_REJ_UNIFORM + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_AARCH64 + + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/arith_native_aarch64.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + mlk_ntt_aarch64_asm(data, mlk_aarch64_ntt_zetas_layer12345, + mlk_aarch64_ntt_zetas_layer67); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + mlk_intt_aarch64_asm(data, mlk_aarch64_invntt_zetas_layer12345, + mlk_aarch64_invntt_zetas_layer67); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + mlk_poly_reduce_aarch64_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + mlk_poly_tomont_aarch64_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + mlk_poly_mulcache_compute_aarch64_asm( + x, y, mlk_aarch64_zetas_mulcache_native, + mlk_aarch64_zetas_mulcache_twisted_native); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +{ + mlk_poly_tobytes_aarch64_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + if (len != MLKEM_N || + buflen % 24 != 0) /* NEON support is mandatory for AArch64 */ + { + return MLK_NATIVE_FUNC_FALLBACK; + } + return (int)mlk_rej_uniform_aarch64_asm(r, buf, buflen, + mlk_rej_uniform_table); +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_AARCH64_META_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/aarch64_zetas.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/aarch64_zetas.c new file mode 100644 index 0000000..cc3a48d --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/aarch64_zetas.c @@ -0,0 +1,182 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_aarch64.h" + +/* + * Table of zeta values used in the AArch64 forward NTT + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_ntt_zetas_layer12345[80] = { + -1600, -15749, -749, -7373, -40, -394, -687, -6762, 630, 6201, + -1432, -14095, 848, 8347, 0, 0, 1062, 10453, 296, 2914, + -882, -8682, 0, 0, -1410, -13879, 1339, 13180, 1476, 14529, + 0, 0, 193, 1900, -283, -2786, 56, 551, 0, 0, + 797, 7845, -1089, -10719, 1333, 13121, 0, 0, -543, -5345, + 1426, 14036, -1235, -12156, 0, 0, -69, -679, 535, 5266, + -447, -4400, 0, 0, 569, 5601, -936, -9213, -450, -4429, + 0, 0, -1583, -15582, -1355, -13338, 821, 8081, 0, 0, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_ntt_zetas_layer67[384] = { + 289, 289, 331, 331, -76, -76, -1573, -1573, 2845, + 2845, 3258, 3258, -748, -748, -15483, -15483, 17, 17, + 583, 583, 1637, 1637, -1041, -1041, 167, 167, 5739, + 5739, 16113, 16113, -10247, -10247, -568, -568, -680, -680, + 723, 723, 1100, 1100, -5591, -5591, -6693, -6693, 7117, + 7117, 10828, 10828, 1197, 1197, -1025, -1025, -1052, -1052, + -1274, -1274, 11782, 11782, -10089, -10089, -10355, -10355, -12540, + -12540, 1409, 1409, -48, -48, 756, 756, -314, -314, + 13869, 13869, -472, -472, 7441, 7441, -3091, -3091, -667, + -667, 233, 233, -1173, -1173, -279, -279, -6565, -6565, + 2293, 2293, -11546, -11546, -2746, -2746, 650, 650, -1352, + -1352, -816, -816, 632, 632, 6398, 6398, -13308, -13308, + -8032, -8032, 6221, 6221, -1626, -1626, -540, -540, -1482, + -1482, 1461, 1461, -16005, -16005, -5315, -5315, -14588, -14588, + 14381, 14381, 1651, 1651, -1540, -1540, 952, 952, -642, + -642, 16251, 16251, -15159, -15159, 9371, 9371, -6319, -6319, + -464, -464, 33, 33, 1320, 1320, -1414, -1414, -4567, + -4567, 325, 325, 12993, 12993, -13918, -13918, 939, 939, + -892, -892, 733, 733, 268, 268, 9243, 9243, -8780, + -8780, 7215, 7215, 2638, 2638, -1021, -1021, -941, -941, + -992, -992, 641, 641, -10050, -10050, -9262, -9262, -9764, + -9764, 6309, 6309, -1010, -1010, 1435, 1435, 807, 807, + 452, 452, -9942, -9942, 14125, 14125, 7943, 7943, 4449, + 4449, 1584, 1584, -1292, -1292, 375, 375, -1239, -1239, + 15592, 15592, -12717, -12717, 3691, 3691, -12196, -12196, -1031, + -1031, -109, -109, -780, -780, 1645, 1645, -10148, -10148, + -1073, -1073, -7678, -7678, 16192, 16192, 1438, 1438, -461, + -461, 1534, 1534, -927, -927, 14155, 14155, -4538, -4538, + 15099, 15099, -9125, -9125, 1063, 1063, -556, -556, -1230, + -1230, -863, -863, 10463, 10463, -5473, -5473, -12107, -12107, + -8495, -8495, 319, 319, 757, 757, 561, 561, -735, + -735, 3140, 3140, 7451, 7451, 5522, 5522, -7235, -7235, + -682, -682, -712, -712, 1481, 1481, 648, 648, -6713, + -6713, -7008, -7008, 14578, 14578, 6378, 6378, -525, -525, + 403, 403, 1143, 1143, -554, -554, -5168, -5168, 3967, + 3967, 11251, 11251, -5453, -5453, 1092, 1092, 1026, 1026, + -1179, -1179, 886, 886, 10749, 10749, 10099, 10099, -11605, + -11605, 8721, 8721, -855, -855, -219, -219, 1227, 1227, + 910, 910, -8416, -8416, -2156, -2156, 12078, 12078, 8957, + 8957, -1607, -1607, -1455, -1455, -1219, -1219, 885, 885, + -15818, -15818, -14322, -14322, -11999, -11999, 8711, 8711, 1212, + 1212, 1029, 1029, -394, -394, -1175, -1175, 11930, 11930, + 10129, 10129, -3878, -3878, -11566, -11566, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_invntt_zetas_layer12345[80] = { + 1583, 15582, -821, -8081, 1355, 13338, 0, 0, -569, + -5601, 450, 4429, 936, 9213, 0, 0, 69, 679, + 447, 4400, -535, -5266, 0, 0, 543, 5345, 1235, + 12156, -1426, -14036, 0, 0, -797, -7845, -1333, -13121, + 1089, 10719, 0, 0, -193, -1900, -56, -551, 283, + 2786, 0, 0, 1410, 13879, -1476, -14529, -1339, -13180, + 0, 0, -1062, -10453, 882, 8682, -296, -2914, 0, + 0, 1600, 15749, 40, 394, 749, 7373, -848, -8347, + 1432, 14095, -630, -6201, 687, 6762, 0, 0, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_invntt_zetas_layer67[384] = { + -910, -910, -1227, -1227, 219, 219, 855, 855, -8957, + -8957, -12078, -12078, 2156, 2156, 8416, 8416, 1175, 1175, + 394, 394, -1029, -1029, -1212, -1212, 11566, 11566, 3878, + 3878, -10129, -10129, -11930, -11930, -885, -885, 1219, 1219, + 1455, 1455, 1607, 1607, -8711, -8711, 11999, 11999, 14322, + 14322, 15818, 15818, -648, -648, -1481, -1481, 712, 712, + 682, 682, -6378, -6378, -14578, -14578, 7008, 7008, 6713, + 6713, -886, -886, 1179, 1179, -1026, -1026, -1092, -1092, + -8721, -8721, 11605, 11605, -10099, -10099, -10749, -10749, 554, + 554, -1143, -1143, -403, -403, 525, 525, 5453, 5453, + -11251, -11251, -3967, -3967, 5168, 5168, 927, 927, -1534, + -1534, 461, 461, -1438, -1438, 9125, 9125, -15099, -15099, + 4538, 4538, -14155, -14155, 735, 735, -561, -561, -757, + -757, -319, -319, 7235, 7235, -5522, -5522, -7451, -7451, + -3140, -3140, 863, 863, 1230, 1230, 556, 556, -1063, + -1063, 8495, 8495, 12107, 12107, 5473, 5473, -10463, -10463, + -452, -452, -807, -807, -1435, -1435, 1010, 1010, -4449, + -4449, -7943, -7943, -14125, -14125, 9942, 9942, -1645, -1645, + 780, 780, 109, 109, 1031, 1031, -16192, -16192, 7678, + 7678, 1073, 1073, 10148, 10148, 1239, 1239, -375, -375, + 1292, 1292, -1584, -1584, 12196, 12196, -3691, -3691, 12717, + 12717, -15592, -15592, 1414, 1414, -1320, -1320, -33, -33, + 464, 464, 13918, 13918, -12993, -12993, -325, -325, 4567, + 4567, -641, -641, 992, 992, 941, 941, 1021, 1021, + -6309, -6309, 9764, 9764, 9262, 9262, 10050, 10050, -268, + -268, -733, -733, 892, 892, -939, -939, -2638, -2638, + -7215, -7215, 8780, 8780, -9243, -9243, -632, -632, 816, + 816, 1352, 1352, -650, -650, -6221, -6221, 8032, 8032, + 13308, 13308, -6398, -6398, 642, 642, -952, -952, 1540, + 1540, -1651, -1651, 6319, 6319, -9371, -9371, 15159, 15159, + -16251, -16251, -1461, -1461, 1482, 1482, 540, 540, 1626, + 1626, -14381, -14381, 14588, 14588, 5315, 5315, 16005, 16005, + 1274, 1274, 1052, 1052, 1025, 1025, -1197, -1197, 12540, + 12540, 10355, 10355, 10089, 10089, -11782, -11782, 279, 279, + 1173, 1173, -233, -233, 667, 667, 2746, 2746, 11546, + 11546, -2293, -2293, 6565, 6565, 314, 314, -756, -756, + 48, 48, -1409, -1409, 3091, 3091, -7441, -7441, 472, + 472, -13869, -13869, 1573, 1573, 76, 76, -331, -331, + -289, -289, 15483, 15483, 748, 748, -3258, -3258, -2845, + -2845, -1100, -1100, -723, -723, 680, 680, 568, 568, + -10828, -10828, -7117, -7117, 6693, 6693, 5591, 5591, 1041, + 1041, -1637, -1637, -583, -583, -17, -17, 10247, 10247, + -16113, -16113, -5739, -5739, -167, -167, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_zetas_mulcache_native[128] = { + 17, -17, -568, 568, 583, -583, -680, 680, 1637, -1637, + 723, -723, -1041, 1041, 1100, -1100, 1409, -1409, -667, 667, + -48, 48, 233, -233, 756, -756, -1173, 1173, -314, 314, + -279, 279, -1626, 1626, 1651, -1651, -540, 540, -1540, 1540, + -1482, 1482, 952, -952, 1461, -1461, -642, 642, 939, -939, + -1021, 1021, -892, 892, -941, 941, 733, -733, -992, 992, + 268, -268, 641, -641, 1584, -1584, -1031, 1031, -1292, 1292, + -109, 109, 375, -375, -780, 780, -1239, 1239, 1645, -1645, + 1063, -1063, 319, -319, -556, 556, 757, -757, -1230, 1230, + 561, -561, -863, 863, -735, 735, -525, 525, 1092, -1092, + 403, -403, 1026, -1026, 1143, -1143, -1179, 1179, -554, 554, + 886, -886, -1607, 1607, 1212, -1212, -1455, 1455, 1029, -1029, + -1219, 1219, -394, 394, 885, -885, -1175, 1175, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_zetas_mulcache_twisted_native[128] = { + 167, -167, -5591, 5591, 5739, -5739, -6693, 6693, 16113, + -16113, 7117, -7117, -10247, 10247, 10828, -10828, 13869, -13869, + -6565, 6565, -472, 472, 2293, -2293, 7441, -7441, -11546, + 11546, -3091, 3091, -2746, 2746, -16005, 16005, 16251, -16251, + -5315, 5315, -15159, 15159, -14588, 14588, 9371, -9371, 14381, + -14381, -6319, 6319, 9243, -9243, -10050, 10050, -8780, 8780, + -9262, 9262, 7215, -7215, -9764, 9764, 2638, -2638, 6309, + -6309, 15592, -15592, -10148, 10148, -12717, 12717, -1073, 1073, + 3691, -3691, -7678, 7678, -12196, 12196, 16192, -16192, 10463, + -10463, 3140, -3140, -5473, 5473, 7451, -7451, -12107, 12107, + 5522, -5522, -8495, 8495, -7235, 7235, -5168, 5168, 10749, + -10749, 3967, -3967, 10099, -10099, 11251, -11251, -11605, 11605, + -5453, 5453, 8721, -8721, -15818, 15818, 11930, -11930, -14322, + 14322, 10129, -10129, -11999, 11999, -3878, 3878, 8711, -8711, + -11566, 11566, +}; + +#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(aarch64_zetas) + +#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/arith_native_aarch64.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/arith_native_aarch64.h new file mode 100644 index 0000000..006c60c --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/arith_native_aarch64.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#define MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H + +#include "../../../cbmc.h" +#include "../../../common.h" + +#define mlk_aarch64_ntt_zetas_layer12345 \ + MLK_NAMESPACE(aarch64_ntt_zetas_layer12345) +#define mlk_aarch64_ntt_zetas_layer67 MLK_NAMESPACE(aarch64_ntt_zetas_layer67) +#define mlk_aarch64_invntt_zetas_layer12345 \ + MLK_NAMESPACE(aarch64_invntt_zetas_layer12345) +#define mlk_aarch64_invntt_zetas_layer67 \ + MLK_NAMESPACE(aarch64_invntt_zetas_layer67) +#define mlk_aarch64_zetas_mulcache_native \ + MLK_NAMESPACE(aarch64_zetas_mulcache_native) +#define mlk_aarch64_zetas_mulcache_twisted_native \ + MLK_NAMESPACE(aarch64_zetas_mulcache_twisted_native) +#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table) + +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_ntt_zetas_layer12345[80]; +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_aarch64_ntt_zetas_layer67[384]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_invntt_zetas_layer12345[80]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_invntt_zetas_layer67[384]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_zetas_mulcache_native[128]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_zetas_mulcache_twisted_native[128]; +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096]; + +#define mlk_ntt_aarch64_asm MLK_NAMESPACE(ntt_aarch64_asm) +void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], + const int16_t twiddles56[384]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/ntt_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(p, 0, MLKEM_N, 8192)) + requires(twiddles12345 == mlk_aarch64_ntt_zetas_layer12345) + requires(twiddles56 == mlk_aarch64_ntt_zetas_layer67) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(p, 0, MLKEM_N, 23595)) + /* check-magic: on */ +); + +#define mlk_intt_aarch64_asm MLK_NAMESPACE(intt_aarch64_asm) +void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], + const int16_t twiddles56[384]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/intt_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(twiddles12345 == mlk_aarch64_invntt_zetas_layer12345) + requires(twiddles56 == mlk_aarch64_invntt_zetas_layer67) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(p, 0, MLKEM_N, 26625)) + /* check-magic: on */ +); + +#define mlk_poly_reduce_aarch64_asm MLK_NAMESPACE(poly_reduce_aarch64_asm) +void mlk_poly_reduce_aarch64_asm(int16_t p[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/poly_reduce_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_tomont_aarch64_asm MLK_NAMESPACE(poly_tomont_aarch64_asm) +void mlk_poly_tomont_aarch64_asm(int16_t p[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/poly_tomont_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute_aarch64_asm \ + MLK_NAMESPACE(poly_mulcache_compute_aarch64_asm) +void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], + const int16_t mlk_poly[256], + const int16_t zetas[128], + const int16_t zetas_twisted[128]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/poly_mulcache_compute_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N)) + requires(zetas == mlk_aarch64_zetas_mulcache_native) + requires(zetas_twisted == mlk_aarch64_zetas_mulcache_twisted_native) + assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q)) +); + +#define mlk_poly_tobytes_aarch64_asm MLK_NAMESPACE(poly_tobytes_aarch64_asm) +void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/poly_tobytes_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm( + int16_t r[256], const int16_t a[512], const int16_t b[512], + const int16_t b_cache[256]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm( + int16_t r[256], const int16_t a[768], const int16_t b[768], + const int16_t b_cache[384]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm( + int16_t r[256], const int16_t a[1024], const int16_t b[1024], + const int16_t b_cache[512]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_rej_uniform_aarch64_asm MLK_NAMESPACE(rej_uniform_aarch64_asm) +MLK_MUST_CHECK_RETURN_VALUE +uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, + unsigned buflen, const uint8_t table[4096]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/rej_uniform_aarch64_asm.ml. */ +__contract__( + requires(buflen % 24 == 0) + requires(memory_no_alias(buf, buflen)) + requires(table == mlk_rej_uniform_table) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value <= MLKEM_N) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); + +#endif /* !MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/intt_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/intt_aarch64_asm.S new file mode 100644 index 0000000..af44dbd --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/intt_aarch64_asm.S @@ -0,0 +1,630 @@ +/* Copyright (c) 2022 Arm Limited + * Copyright (c) 2022 Hanno Becker + * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [SLOTHY_Paper] + * Fast and Clean: Auditable high-performance assembly via constraint solving + * Abdulrahman, Becker, Kannwischer, Klein + * https://eprint.iacr.org/2022/1303 + */ + +/*yaml + Name: intt_aarch64_asm + Description: AArch64 ML-KEM inverse NTT following @[NeonNTT] and @[SLOTHY_Paper] + Signature: void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + x1: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const int16_t twiddles12345[80] + description: Twiddle factors for layers 1-5 + x2: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t twiddles56[384] + description: Twiddle factors for layers 6-7 + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/intt_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(intt_aarch64_asm) +MLK_ASM_FN_SYMBOL(intt_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w5, #0xd01 // =3329 + mov v7.h[0], w5 + mov w5, #0x4ebf // =20159 + mov v7.h[1], w5 + mov w5, #0x200 // =512 + dup v29.8h, w5 + mov w5, #0x13b0 // =5040 + dup v30.8h, w5 + mov x3, x0 + mov x4, #0x8 // =8 + ldr q13, [x3, #0x20] + ldr q8, [x3, #0x30] + ldr q6, [x3] + ldr q16, [x3, #0x10] + ldr q4, [x3, #0x50] + ldr q11, [x3, #0x40] + ldr q3, [x3, #0x70] + trn1 v23.4s, v13.4s, v8.4s + ldr q0, [x3, #0x60] + trn2 v19.4s, v6.4s, v16.4s + trn2 v21.4s, v13.4s, v8.4s + trn1 v6.4s, v6.4s, v16.4s + ldr q24, [x2, #0x20] + trn1 v10.2d, v19.2d, v21.2d + ldr q16, [x2], #0x60 + trn1 v5.2d, v6.2d, v23.2d + trn1 v28.4s, v0.4s, v3.4s + trn2 v18.2d, v6.2d, v23.2d + mul v31.8h, v10.8h, v29.8h + trn2 v13.4s, v0.4s, v3.4s + ldur q14, [x2, #-0x50] + sqrdmulh v26.8h, v18.8h, v30.8h + ldur q20, [x2, #-0x20] + mul v17.8h, v18.8h, v29.8h + trn2 v18.2d, v19.2d, v21.2d + mul v9.8h, v18.8h, v29.8h + trn1 v12.4s, v11.4s, v4.4s + sqrdmulh v22.8h, v18.8h, v30.8h + sqrdmulh v3.8h, v10.8h, v30.8h + sqrdmulh v25.8h, v5.8h, v30.8h + mls v9.8h, v22.8h, v7.h[0] + mls v17.8h, v26.8h, v7.h[0] + trn2 v26.4s, v11.4s, v4.4s + mul v8.8h, v5.8h, v29.8h + trn1 v10.2d, v26.2d, v13.2d + ldur q11, [x2, #-0x10] + mls v31.8h, v3.8h, v7.h[0] + trn1 v6.2d, v12.2d, v28.2d + trn2 v3.2d, v26.2d, v13.2d + ldur q4, [x2, #-0x30] + mls v8.8h, v25.8h, v7.h[0] + sub v19.8h, v17.8h, v9.8h + trn2 v13.2d, v12.2d, v28.2d + sqrdmulh v1.8h, v3.8h, v30.8h + add v9.8h, v17.8h, v9.8h + mul v18.8h, v19.8h, v20.8h + add v28.8h, v8.8h, v31.8h + sqrdmulh v20.8h, v19.8h, v11.8h + sub v12.8h, v28.8h, v9.8h + sub v23.8h, v8.8h, v31.8h + sqrdmulh v11.8h, v13.8h, v30.8h + sqrdmulh v5.8h, v23.8h, v4.8h + mul v0.8h, v23.8h, v24.8h + mul v2.8h, v13.8h, v29.8h + mls v0.8h, v5.8h, v7.h[0] + add v24.8h, v28.8h, v9.8h + mls v18.8h, v20.8h, v7.h[0] + sqrdmulh v15.8h, v6.8h, v30.8h + sqrdmulh v25.8h, v12.8h, v14.8h + mul v21.8h, v12.8h, v16.8h + sub v23.8h, v0.8h, v18.8h + sqrdmulh v8.8h, v23.8h, v14.8h + mul v23.8h, v23.8h, v16.8h + mls v21.8h, v25.8h, v7.h[0] + mls v23.8h, v8.8h, v7.h[0] + mul v14.8h, v3.8h, v29.8h + add v3.8h, v0.8h, v18.8h + trn2 v4.4s, v24.4s, v3.4s + mls v14.8h, v1.8h, v7.h[0] + trn1 v9.4s, v24.4s, v3.4s + trn2 v12.4s, v21.4s, v23.4s + mls v2.8h, v11.8h, v7.h[0] + trn1 v28.4s, v21.4s, v23.4s + ldr q11, [x1], #0x10 + mul v31.8h, v10.8h, v29.8h + trn1 v25.2d, v4.2d, v12.2d + trn1 v20.2d, v9.2d, v28.2d + ldr q23, [x2, #0x50] + trn2 v13.2d, v4.2d, v12.2d + sqrdmulh v21.8h, v10.8h, v30.8h + trn2 v4.2d, v9.2d, v28.2d + ldr q9, [x2, #0x40] + mul v27.8h, v6.8h, v29.8h + add v26.8h, v20.8h, v25.8h + sub v3.8h, v2.8h, v14.8h + sqdmulh v12.8h, v26.8h, v7.h[1] + add v5.8h, v4.8h, v13.8h + sub v8.8h, v4.8h, v13.8h + add v10.8h, v2.8h, v14.8h + sqdmulh v6.8h, v5.8h, v7.h[1] + ldr q2, [x2, #0x10] + mls v27.8h, v15.8h, v7.h[0] + ldr q15, [x2, #0x20] + srshr v12.8h, v12.8h, #0xb + mls v31.8h, v21.8h, v7.h[0] + srshr v6.8h, v6.8h, #0xb + sqrdmulh v23.8h, v3.8h, v23.8h + mls v26.8h, v12.8h, v7.h[0] + add v21.8h, v27.8h, v31.8h + mls v5.8h, v6.8h, v7.h[0] + sub v6.8h, v27.8h, v31.8h + sub v14.8h, v21.8h, v10.8h + ldr q27, [x2], #0x60 + mul v3.8h, v3.8h, v9.8h + mls v3.8h, v23.8h, v7.h[0] + ldur q13, [x2, #-0x30] + sub v12.8h, v26.8h, v5.8h + add v5.8h, v26.8h, v5.8h + sqrdmulh v31.8h, v8.8h, v11.h[5] + sqrdmulh v19.8h, v12.8h, v11.h[1] + mul v24.8h, v12.8h, v11.h[0] + sqrdmulh v13.8h, v6.8h, v13.8h + mls v24.8h, v19.8h, v7.h[0] + sub x4, x4, #0x2 + +Lintt_layer4567_start: + add v16.8h, v21.8h, v10.8h + mul v18.8h, v6.8h, v15.8h + sub v19.8h, v20.8h, v25.8h + ldr q21, [x3, #0xa0] + str q5, [x3], #0x40 + mls v18.8h, v13.8h, v7.h[0] + sqrdmulh v15.8h, v14.8h, v2.8h + ldr q10, [x3, #0x50] + ldr q12, [x3, #0x40] + stur q24, [x3, #-0x20] + mul v5.8h, v8.8h, v11.h[4] + sub v0.8h, v18.8h, v3.8h + ldr q24, [x3, #0x70] + mls v5.8h, v31.8h, v7.h[0] + ldr q26, [x2, #0x50] + trn2 v1.4s, v12.4s, v10.4s + add v6.8h, v18.8h, v3.8h + sqrdmulh v20.8h, v0.8h, v2.8h + trn1 v13.4s, v12.4s, v10.4s + trn1 v18.4s, v16.4s, v6.4s + mul v22.8h, v0.8h, v27.8h + trn1 v17.4s, v21.4s, v24.4s + sqrdmulh v0.8h, v19.8h, v11.h[3] + trn1 v25.2d, v13.2d, v17.2d + mls v22.8h, v20.8h, v7.h[0] + trn2 v21.4s, v21.4s, v24.4s + mul v24.8h, v25.8h, v29.8h + trn2 v28.2d, v13.2d, v17.2d + sqrdmulh v4.8h, v25.8h, v30.8h + trn2 v3.2d, v1.2d, v21.2d + mul v17.8h, v28.8h, v29.8h + sqrdmulh v31.8h, v28.8h, v30.8h + ldr q2, [x2, #0x10] + mls v24.8h, v4.8h, v7.h[0] + mul v4.8h, v19.8h, v11.h[2] + ldr q19, [x2, #0x40] + mls v4.8h, v0.8h, v7.h[0] + mul v0.8h, v14.8h, v27.8h + mls v0.8h, v15.8h, v7.h[0] + sub v8.8h, v4.8h, v5.8h + mul v12.8h, v3.8h, v29.8h + mul v23.8h, v8.8h, v11.h[0] + trn2 v28.4s, v16.4s, v6.4s + sqrdmulh v10.8h, v8.8h, v11.h[1] + trn1 v9.4s, v0.4s, v22.4s + trn2 v22.4s, v0.4s, v22.4s + ldr q11, [x1], #0x10 + mls v17.8h, v31.8h, v7.h[0] + trn1 v20.2d, v18.2d, v9.2d + trn2 v14.2d, v18.2d, v9.2d + ldr q15, [x2, #0x20] + trn1 v6.2d, v1.2d, v21.2d + sqrdmulh v9.8h, v3.8h, v30.8h + trn1 v25.2d, v28.2d, v22.2d + trn2 v16.2d, v28.2d, v22.2d + mls v23.8h, v10.8h, v7.h[0] + add v1.8h, v20.8h, v25.8h + sqrdmulh v21.8h, v6.8h, v30.8h + add v8.8h, v14.8h, v16.8h + ldr q27, [x2], #0x60 + sqdmulh v28.8h, v8.8h, v7.h[1] + mls v12.8h, v9.8h, v7.h[0] + sqdmulh v31.8h, v1.8h, v7.h[1] + mul v0.8h, v6.8h, v29.8h + sub v10.8h, v17.8h, v12.8h + mls v0.8h, v21.8h, v7.h[0] + srshr v21.8h, v28.8h, #0xb + srshr v13.8h, v31.8h, #0xb + sqrdmulh v22.8h, v10.8h, v26.8h + mls v8.8h, v21.8h, v7.h[0] + mls v1.8h, v13.8h, v7.h[0] + add v21.8h, v24.8h, v0.8h + stur q23, [x3, #-0x10] + sub v6.8h, v24.8h, v0.8h + mul v3.8h, v10.8h, v19.8h + add v0.8h, v4.8h, v5.8h + sqdmulh v13.8h, v0.8h, v7.h[1] + ldur q10, [x2, #-0x30] + add v5.8h, v1.8h, v8.8h + mls v3.8h, v22.8h, v7.h[0] + sub v8.8h, v1.8h, v8.8h + mul v24.8h, v8.8h, v11.h[0] + sqrdmulh v8.8h, v8.8h, v11.h[1] + srshr v1.8h, v13.8h, #0xb + sqrdmulh v13.8h, v6.8h, v10.8h + mls v0.8h, v1.8h, v7.h[0] + add v10.8h, v17.8h, v12.8h + mls v24.8h, v8.8h, v7.h[0] + sub v8.8h, v14.8h, v16.8h + sqrdmulh v31.8h, v8.8h, v11.h[5] + sub v14.8h, v21.8h, v10.8h + stur q0, [x3, #-0x30] + sub x4, x4, #0x1 + cbnz x4, Lintt_layer4567_start + mul v15.8h, v6.8h, v15.8h + sub v22.8h, v20.8h, v25.8h + add v4.8h, v21.8h, v10.8h + str q24, [x3, #0x20] + mls v15.8h, v13.8h, v7.h[0] + str q5, [x3], #0x40 + ldr q9, [x1], #0x10 + sqrdmulh v28.8h, v14.8h, v2.8h + mul v16.8h, v14.8h, v27.8h + sub v18.8h, v15.8h, v3.8h + add v15.8h, v15.8h, v3.8h + sqrdmulh v0.8h, v18.8h, v2.8h + trn2 v24.4s, v4.4s, v15.4s + trn1 v2.4s, v4.4s, v15.4s + mul v18.8h, v18.8h, v27.8h + mls v16.8h, v28.8h, v7.h[0] + mls v18.8h, v0.8h, v7.h[0] + mul v23.8h, v8.8h, v11.h[4] + sqrdmulh v12.8h, v22.8h, v11.h[3] + trn1 v17.4s, v16.4s, v18.4s + trn2 v4.4s, v16.4s, v18.4s + mls v23.8h, v31.8h, v7.h[0] + trn2 v3.2d, v2.2d, v17.2d + trn2 v6.2d, v24.2d, v4.2d + mul v26.8h, v22.8h, v11.h[2] + trn1 v28.2d, v2.2d, v17.2d + mls v26.8h, v12.8h, v7.h[0] + add v25.8h, v3.8h, v6.8h + sub v18.8h, v3.8h, v6.8h + trn1 v24.2d, v24.2d, v4.2d + sqdmulh v1.8h, v25.8h, v7.h[1] + sub v27.8h, v28.8h, v24.8h + sqrdmulh v2.8h, v18.8h, v9.h[5] + add v28.8h, v28.8h, v24.8h + mul v24.8h, v27.8h, v9.h[2] + sqdmulh v12.8h, v28.8h, v7.h[1] + mul v20.8h, v18.8h, v9.h[4] + mls v20.8h, v2.8h, v7.h[0] + srshr v1.8h, v1.8h, #0xb + sqrdmulh v19.8h, v27.8h, v9.h[3] + srshr v15.8h, v12.8h, #0xb + mls v25.8h, v1.8h, v7.h[0] + add v8.8h, v26.8h, v23.8h + sub v4.8h, v26.8h, v23.8h + mls v28.8h, v15.8h, v7.h[0] + mls v24.8h, v19.8h, v7.h[0] + mul v2.8h, v4.8h, v11.h[0] + sub v19.8h, v28.8h, v25.8h + sqrdmulh v15.8h, v4.8h, v11.h[1] + add v25.8h, v28.8h, v25.8h + sub v10.8h, v24.8h, v20.8h + str q25, [x3], #0x40 + sqrdmulh v22.8h, v19.8h, v9.h[1] + add v28.8h, v24.8h, v20.8h + sqrdmulh v25.8h, v10.8h, v9.h[1] + mul v27.8h, v19.8h, v9.h[0] + mul v26.8h, v10.8h, v9.h[0] + sqdmulh v20.8h, v28.8h, v7.h[1] + sqdmulh v16.8h, v8.8h, v7.h[1] + mls v26.8h, v25.8h, v7.h[0] + mls v2.8h, v15.8h, v7.h[0] + srshr v15.8h, v20.8h, #0xb + srshr v1.8h, v16.8h, #0xb + mls v27.8h, v22.8h, v7.h[0] + mls v28.8h, v15.8h, v7.h[0] + mls v8.8h, v1.8h, v7.h[0] + stur q27, [x3, #-0x20] + stur q2, [x3, #-0x50] + stur q28, [x3, #-0x30] + stur q26, [x3, #-0x10] + stur q8, [x3, #-0x70] + mov x4, #0x4 // =4 + ldr q0, [x1], #0x20 + ldur q1, [x1, #-0x10] + ldr q26, [x0] + ldr q13, [x0, #0x40] + ldr q28, [x0, #0xc0] + ldr q2, [x0, #0x140] + ldr q6, [x0, #0x80] + ldr q9, [x0, #0x100] + ldr q29, [x0, #0x1c0] + ldr q23, [x0, #0x180] + sub v17.8h, v26.8h, v13.8h + add v4.8h, v26.8h, v13.8h + ldr q25, [x0, #0xd0] + ldr q24, [x0, #0x50] + add v5.8h, v6.8h, v28.8h + mul v19.8h, v17.8h, v0.h[6] + sub v10.8h, v6.8h, v28.8h + ldr q30, [x0, #0x150] + sqrdmulh v12.8h, v17.8h, v0.h[7] + add v17.8h, v9.8h, v2.8h + sub v28.8h, v9.8h, v2.8h + ldr q2, [x0, #0x90] + sub v26.8h, v23.8h, v29.8h + sqrdmulh v31.8h, v10.8h, v1.h[1] + add v22.8h, v23.8h, v29.8h + ldr q3, [x0, #0x110] + sqrdmulh v9.8h, v28.8h, v1.h[3] + sub v20.8h, v4.8h, v5.8h + sub v27.8h, v17.8h, v22.8h + ldr q29, [x0, #0x10] + add v16.8h, v4.8h, v5.8h + sqrdmulh v4.8h, v26.8h, v1.h[5] + add v6.8h, v17.8h, v22.8h + ldr q22, [x0, #0x1d0] + mul v8.8h, v28.8h, v1.h[2] + sub v21.8h, v2.8h, v25.8h + sub v5.8h, v16.8h, v6.8h + mul v17.8h, v26.8h, v1.h[4] + mul v26.8h, v10.8h, v1.h[0] + mls v26.8h, v31.8h, v7.h[0] + mls v17.8h, v4.8h, v7.h[0] + mls v19.8h, v12.8h, v7.h[0] + mls v8.8h, v9.8h, v7.h[0] + sqrdmulh v10.8h, v27.8h, v0.h[5] + sub v12.8h, v19.8h, v26.8h + add v9.8h, v19.8h, v26.8h + sqrdmulh v26.8h, v20.8h, v0.h[3] + sub v11.8h, v8.8h, v17.8h + add v14.8h, v8.8h, v17.8h + sqrdmulh v13.8h, v12.8h, v0.h[3] + add v23.8h, v9.8h, v14.8h + sqrdmulh v28.8h, v11.8h, v0.h[5] + sub v19.8h, v9.8h, v14.8h + mul v17.8h, v27.8h, v0.h[4] + str q23, [x0, #0x40] + mul v14.8h, v20.8h, v0.h[2] + mul v8.8h, v11.8h, v0.h[4] + mul v4.8h, v12.8h, v0.h[2] + mls v14.8h, v26.8h, v7.h[0] + mls v17.8h, v10.8h, v7.h[0] + mls v8.8h, v28.8h, v7.h[0] + mls v4.8h, v13.8h, v7.h[0] + sub v10.8h, v14.8h, v17.8h + add v20.8h, v14.8h, v17.8h + sqrdmulh v28.8h, v5.8h, v0.h[1] + mul v18.8h, v5.8h, v0.h[0] + str q20, [x0, #0x80] + sub v13.8h, v4.8h, v8.8h + mul v23.8h, v10.8h, v0.h[0] + mul v17.8h, v19.8h, v0.h[0] + sqrdmulh v9.8h, v13.8h, v0.h[1] + mls v18.8h, v28.8h, v7.h[0] + sqrdmulh v10.8h, v10.8h, v0.h[1] + sub x4, x4, #0x2 + +Lintt_layer123_start: + sub v12.8h, v3.8h, v30.8h + mul v11.8h, v21.8h, v1.h[0] + add v28.8h, v4.8h, v8.8h + ldr q20, [x0, #0x190] + add v27.8h, v16.8h, v6.8h + sqrdmulh v8.8h, v12.8h, v1.h[3] + add v16.8h, v29.8h, v24.8h + str q28, [x0, #0xc0] + mls v23.8h, v10.8h, v7.h[0] + str q27, [x0], #0x10 + add v15.8h, v20.8h, v22.8h + str q18, [x0, #0xf0] + mul v14.8h, v13.8h, v0.h[0] + add v2.8h, v2.8h, v25.8h + sub v26.8h, v20.8h, v22.8h + mul v4.8h, v12.8h, v1.h[2] + sub v5.8h, v16.8h, v2.8h + str q23, [x0, #0x170] + add v20.8h, v3.8h, v30.8h + sqrdmulh v27.8h, v26.8h, v1.h[5] + add v16.8h, v16.8h, v2.8h + mul v18.8h, v26.8h, v1.h[4] + sub v31.8h, v20.8h, v15.8h + mls v4.8h, v8.8h, v7.h[0] + sub v28.8h, v29.8h, v24.8h + mls v18.8h, v27.8h, v7.h[0] + ldr q22, [x0, #0x1d0] + mul v26.8h, v28.8h, v0.h[6] + mul v2.8h, v5.8h, v0.h[2] + sub v12.8h, v4.8h, v18.8h + sqrdmulh v24.8h, v28.8h, v0.h[7] + mls v14.8h, v9.8h, v7.h[0] + sqrdmulh v10.8h, v12.8h, v0.h[5] + mls v26.8h, v24.8h, v7.h[0] + ldr q24, [x0, #0x50] + mul v8.8h, v12.8h, v0.h[4] + str q14, [x0, #0x1b0] + add v28.8h, v4.8h, v18.8h + sqrdmulh v5.8h, v5.8h, v0.h[3] + add v6.8h, v20.8h, v15.8h + sqrdmulh v3.8h, v19.8h, v0.h[1] + sub v13.8h, v16.8h, v6.8h + sqrdmulh v12.8h, v21.8h, v1.h[1] + sqrdmulh v21.8h, v13.8h, v0.h[1] + sqrdmulh v27.8h, v31.8h, v0.h[5] + ldr q25, [x0, #0xd0] + mls v11.8h, v12.8h, v7.h[0] + mul v23.8h, v31.8h, v0.h[4] + mul v18.8h, v13.8h, v0.h[0] + add v30.8h, v26.8h, v11.8h + sub v13.8h, v26.8h, v11.8h + mls v23.8h, v27.8h, v7.h[0] + add v12.8h, v30.8h, v28.8h + sub v19.8h, v30.8h, v28.8h + mls v2.8h, v5.8h, v7.h[0] + str q12, [x0, #0x40] + sqrdmulh v26.8h, v13.8h, v0.h[3] + mls v8.8h, v10.8h, v7.h[0] + ldr q30, [x0, #0x150] + sub v20.8h, v2.8h, v23.8h + mul v4.8h, v13.8h, v0.h[2] + add v13.8h, v2.8h, v23.8h + mls v4.8h, v26.8h, v7.h[0] + ldr q2, [x0, #0x90] + mul v23.8h, v20.8h, v0.h[0] + ldr q29, [x0, #0x10] + sqrdmulh v10.8h, v20.8h, v0.h[1] + str q13, [x0, #0x80] + sub v13.8h, v4.8h, v8.8h + mls v17.8h, v3.8h, v7.h[0] + ldr q3, [x0, #0x110] + mls v18.8h, v21.8h, v7.h[0] + sub v21.8h, v2.8h, v25.8h + sqrdmulh v9.8h, v13.8h, v0.h[1] + str q17, [x0, #0x130] + mul v17.8h, v19.8h, v0.h[0] + sub x4, x4, #0x1 + cbnz x4, Lintt_layer123_start + mls v23.8h, v10.8h, v7.h[0] + ldr q11, [x0, #0x190] + str q18, [x0, #0x100] + add v27.8h, v3.8h, v30.8h + mul v13.8h, v13.8h, v0.h[0] + sub v5.8h, v29.8h, v24.8h + add v14.8h, v16.8h, v6.8h + mls v13.8h, v9.8h, v7.h[0] + add v10.8h, v11.8h, v22.8h + str q23, [x0, #0x180] + sub v20.8h, v11.8h, v22.8h + sub v23.8h, v27.8h, v10.8h + sqrdmulh v16.8h, v21.8h, v1.h[1] + sqrdmulh v31.8h, v23.8h, v0.h[5] + str q13, [x0, #0x1c0] + add v13.8h, v4.8h, v8.8h + mul v18.8h, v21.8h, v1.h[0] + str q13, [x0, #0xc0] + sqrdmulh v13.8h, v19.8h, v0.h[1] + sqrdmulh v28.8h, v20.8h, v1.h[5] + str q14, [x0], #0x10 + mul v4.8h, v20.8h, v1.h[4] + mls v17.8h, v13.8h, v7.h[0] + sub v13.8h, v3.8h, v30.8h + sqrdmulh v8.8h, v13.8h, v1.h[3] + mul v12.8h, v13.8h, v1.h[2] + mls v4.8h, v28.8h, v7.h[0] + mls v12.8h, v8.8h, v7.h[0] + mls v18.8h, v16.8h, v7.h[0] + str q17, [x0, #0x130] + sqrdmulh v15.8h, v5.8h, v0.h[7] + add v11.8h, v27.8h, v10.8h + mul v16.8h, v5.8h, v0.h[6] + sub v8.8h, v12.8h, v4.8h + sqrdmulh v28.8h, v8.8h, v0.h[5] + add v13.8h, v2.8h, v25.8h + mls v16.8h, v15.8h, v7.h[0] + add v26.8h, v12.8h, v4.8h + mul v8.8h, v8.8h, v0.h[4] + add v4.8h, v29.8h, v24.8h + mls v8.8h, v28.8h, v7.h[0] + sub v20.8h, v4.8h, v13.8h + add v14.8h, v4.8h, v13.8h + add v12.8h, v16.8h, v18.8h + sqrdmulh v22.8h, v20.8h, v0.h[3] + add v27.8h, v14.8h, v11.8h + sub v13.8h, v16.8h, v18.8h + mul v4.8h, v20.8h, v0.h[2] + str q27, [x0], #0x10 + sub v24.8h, v12.8h, v26.8h + sqrdmulh v3.8h, v13.8h, v0.h[3] + mul v13.8h, v13.8h, v0.h[2] + sqrdmulh v27.8h, v24.8h, v0.h[1] + mls v13.8h, v3.8h, v7.h[0] + mul v9.8h, v24.8h, v0.h[0] + mls v9.8h, v27.8h, v7.h[0] + add v30.8h, v13.8h, v8.8h + sub v13.8h, v13.8h, v8.8h + mls v4.8h, v22.8h, v7.h[0] + str q30, [x0, #0xb0] + sqrdmulh v16.8h, v13.8h, v0.h[1] + str q9, [x0, #0x130] + mul v9.8h, v13.8h, v0.h[0] + add v13.8h, v12.8h, v26.8h + str q13, [x0, #0x30] + mul v13.8h, v23.8h, v0.h[4] + sub v23.8h, v14.8h, v11.8h + mls v13.8h, v31.8h, v7.h[0] + mls v9.8h, v16.8h, v7.h[0] + mul v30.8h, v23.8h, v0.h[0] + sub v24.8h, v4.8h, v13.8h + add v13.8h, v4.8h, v13.8h + sqrdmulh v23.8h, v23.8h, v0.h[1] + str q9, [x0, #0x1b0] + str q13, [x0, #0x70] + sqrdmulh v13.8h, v24.8h, v0.h[1] + mul v21.8h, v24.8h, v0.h[0] + mls v30.8h, v23.8h, v7.h[0] + mls v21.8h, v13.8h, v7.h[0] + str q30, [x0, #0xf0] + str q21, [x0, #0x170] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(intt_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/ntt_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/ntt_aarch64_asm.S new file mode 100644 index 0000000..7174799 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/ntt_aarch64_asm.S @@ -0,0 +1,564 @@ +/* Copyright (c) 2022 Arm Limited + * Copyright (c) 2022 Hanno Becker + * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [SLOTHY_Paper] + * Fast and Clean: Auditable high-performance assembly via constraint solving + * Abdulrahman, Becker, Kannwischer, Klein + * https://eprint.iacr.org/2022/1303 + */ + +/*yaml + Name: ntt_aarch64_asm + Description: AArch64 ML-KEM forward NTT following @[NeonNTT] and @[SLOTHY_Paper] + Signature: void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + x1: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const int16_t twiddles12345[80] + description: Twiddle factors for layers 1-5 + x2: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t twiddles56[384] + description: Twiddle factors for layers 6-7 + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/ntt_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_aarch64_asm) +MLK_ASM_FN_SYMBOL(ntt_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w5, #0xd01 // =3329 + mov v7.h[0], w5 + mov w5, #0x4ebf // =20159 + mov v7.h[1], w5 + mov x3, x0 + mov x4, #0x4 // =4 + ldr q0, [x1], #0x20 + ldur q1, [x1, #-0x10] + ldr q21, [x0, #0x40] + ldr q5, [x0, #0x1c0] + ldr q30, [x0, #0x110] + ldr q24, [x0, #0x140] + ldr q12, [x0, #0x80] + sqrdmulh v9.8h, v5.8h, v0.h[1] + mul v23.8h, v5.8h, v0.h[0] + sqrdmulh v17.8h, v24.8h, v0.h[1] + ldr q13, [x0, #0xc0] + mls v23.8h, v9.8h, v7.h[0] + mul v8.8h, v24.8h, v0.h[0] + mls v8.8h, v17.8h, v7.h[0] + add v9.8h, v13.8h, v23.8h + sub v10.8h, v13.8h, v23.8h + mul v11.8h, v30.8h, v0.h[0] + ldr q13, [x0, #0x180] + sqrdmulh v28.8h, v9.8h, v0.h[3] + sub v29.8h, v21.8h, v8.8h + mul v26.8h, v9.8h, v0.h[2] + add v8.8h, v21.8h, v8.8h + mul v2.8h, v13.8h, v0.h[0] + mls v26.8h, v28.8h, v7.h[0] + mul v28.8h, v10.8h, v0.h[4] + sqrdmulh v23.8h, v10.8h, v0.h[5] + add v22.8h, v8.8h, v26.8h + sqrdmulh v10.8h, v13.8h, v0.h[1] + sqrdmulh v21.8h, v22.8h, v0.h[7] + ldr q13, [x0, #0x100] + mul v16.8h, v22.8h, v0.h[6] + mls v28.8h, v23.8h, v7.h[0] + mls v2.8h, v10.8h, v7.h[0] + sqrdmulh v23.8h, v13.8h, v0.h[1] + sub v10.8h, v29.8h, v28.8h + add v17.8h, v29.8h, v28.8h + mls v16.8h, v21.8h, v7.h[0] + sub v18.8h, v12.8h, v2.8h + ldr q29, [x0] + sqrdmulh v14.8h, v17.8h, v1.h[3] + add v22.8h, v12.8h, v2.8h + sqrdmulh v9.8h, v18.8h, v0.h[5] + mul v21.8h, v13.8h, v0.h[0] + ldr q13, [x0, #0x150] + mul v5.8h, v18.8h, v0.h[4] + mls v5.8h, v9.8h, v7.h[0] + mul v18.8h, v13.8h, v0.h[0] + mls v21.8h, v23.8h, v7.h[0] + sqrdmulh v2.8h, v13.8h, v0.h[1] + mul v13.8h, v17.8h, v1.h[2] + sub v4.8h, v29.8h, v21.8h + mls v13.8h, v14.8h, v7.h[0] + add v25.8h, v29.8h, v21.8h + add v6.8h, v4.8h, v5.8h + sqrdmulh v15.8h, v22.8h, v0.h[3] + sub v21.8h, v4.8h, v5.8h + sub v5.8h, v8.8h, v26.8h + mul v23.8h, v22.8h, v0.h[2] + add v28.8h, v6.8h, v13.8h + sub v13.8h, v6.8h, v13.8h + mul v4.8h, v5.8h, v1.h[0] + sub x4, x4, #0x2 + +Lntt_layer123_start: + mls v23.8h, v15.8h, v7.h[0] + ldr q6, [x0, #0x190] + ldr q15, [x0, #0x90] + ldr q19, [x0, #0x10] + mul v22.8h, v10.8h, v1.h[4] + ldr q24, [x0, #0x50] + str q13, [x0, #0x140] + sqrdmulh v13.8h, v6.8h, v0.h[1] + sub v20.8h, v25.8h, v23.8h + sqrdmulh v3.8h, v30.8h, v0.h[1] + str q28, [x0, #0x100] + ldr q30, [x0, #0x120] + mul v8.8h, v6.8h, v0.h[0] + sqrdmulh v27.8h, v10.8h, v1.h[5] + mls v11.8h, v3.8h, v7.h[0] + mls v18.8h, v2.8h, v7.h[0] + ldr q31, [x0, #0x160] + sqrdmulh v10.8h, v5.8h, v1.h[1] + mls v8.8h, v13.8h, v7.h[0] + ldr q13, [x0, #0x1d0] + sub v14.8h, v24.8h, v18.8h + add v9.8h, v24.8h, v18.8h + sqrdmulh v2.8h, v31.8h, v0.h[1] + mls v4.8h, v10.8h, v7.h[0] + add v10.8h, v25.8h, v23.8h + sub v24.8h, v19.8h, v11.8h + add v25.8h, v19.8h, v11.8h + sqrdmulh v28.8h, v13.8h, v0.h[1] + mul v11.8h, v30.8h, v0.h[0] + mul v17.8h, v13.8h, v0.h[0] + sub v13.8h, v10.8h, v16.8h + sub v6.8h, v15.8h, v8.8h + mls v17.8h, v28.8h, v7.h[0] + str q13, [x0, #0x40] + mls v22.8h, v27.8h, v7.h[0] + ldr q13, [x0, #0xd0] + add v26.8h, v20.8h, v4.8h + mul v18.8h, v31.8h, v0.h[0] + add v27.8h, v10.8h, v16.8h + str q26, [x0, #0x80] + sqrdmulh v31.8h, v6.8h, v0.h[5] + add v3.8h, v21.8h, v22.8h + str q27, [x0], #0x10 + mul v26.8h, v6.8h, v0.h[4] + add v6.8h, v13.8h, v17.8h + sub v5.8h, v13.8h, v17.8h + str q3, [x0, #0x170] + sub v17.8h, v21.8h, v22.8h + sqrdmulh v10.8h, v6.8h, v0.h[3] + sub v13.8h, v20.8h, v4.8h + add v20.8h, v15.8h, v8.8h + sqrdmulh v12.8h, v5.8h, v0.h[5] + str q13, [x0, #0xb0] + mul v8.8h, v6.8h, v0.h[2] + str q17, [x0, #0x1b0] + mls v8.8h, v10.8h, v7.h[0] + mul v29.8h, v5.8h, v0.h[4] + mls v29.8h, v12.8h, v7.h[0] + sub v5.8h, v9.8h, v8.8h + add v3.8h, v9.8h, v8.8h + sqrdmulh v15.8h, v20.8h, v0.h[3] + mul v4.8h, v5.8h, v1.h[0] + add v6.8h, v14.8h, v29.8h + sqrdmulh v9.8h, v3.8h, v0.h[7] + sqrdmulh v12.8h, v6.8h, v1.h[3] + sub v10.8h, v14.8h, v29.8h + mul v23.8h, v6.8h, v1.h[2] + mls v26.8h, v31.8h, v7.h[0] + mls v23.8h, v12.8h, v7.h[0] + mul v16.8h, v3.8h, v0.h[6] + add v13.8h, v24.8h, v26.8h + sub v21.8h, v24.8h, v26.8h + mls v16.8h, v9.8h, v7.h[0] + add v28.8h, v13.8h, v23.8h + sub v13.8h, v13.8h, v23.8h + mul v23.8h, v20.8h, v0.h[2] + sub x4, x4, #0x1 + cbnz x4, Lntt_layer123_start + sqrdmulh v3.8h, v5.8h, v1.h[1] + mls v23.8h, v15.8h, v7.h[0] + ldr q5, [x0, #0x190] + mul v29.8h, v10.8h, v1.h[4] + mls v4.8h, v3.8h, v7.h[0] + sub v19.8h, v25.8h, v23.8h + sqrdmulh v31.8h, v5.8h, v0.h[1] + sqrdmulh v6.8h, v30.8h, v0.h[1] + sub v3.8h, v19.8h, v4.8h + mul v5.8h, v5.8h, v0.h[0] + str q3, [x0, #0xc0] + sqrdmulh v12.8h, v10.8h, v1.h[5] + mls v18.8h, v2.8h, v7.h[0] + ldr q3, [x0, #0x1d0] + mls v5.8h, v31.8h, v7.h[0] + sqrdmulh v10.8h, v3.8h, v0.h[1] + mls v11.8h, v6.8h, v7.h[0] + ldr q31, [x0, #0x90] + mul v30.8h, v3.8h, v0.h[0] + mls v30.8h, v10.8h, v7.h[0] + sub v10.8h, v31.8h, v5.8h + mls v29.8h, v12.8h, v7.h[0] + ldr q6, [x0, #0xd0] + sqrdmulh v15.8h, v10.8h, v0.h[5] + mul v17.8h, v10.8h, v0.h[4] + add v10.8h, v6.8h, v30.8h + sub v6.8h, v6.8h, v30.8h + sqrdmulh v12.8h, v10.8h, v0.h[3] + sub v27.8h, v21.8h, v29.8h + sqrdmulh v3.8h, v6.8h, v0.h[5] + mul v10.8h, v10.8h, v0.h[2] + ldr q20, [x0, #0x50] + mls v10.8h, v12.8h, v7.h[0] + mul v2.8h, v6.8h, v0.h[4] + add v6.8h, v20.8h, v18.8h + add v5.8h, v31.8h, v5.8h + mls v2.8h, v3.8h, v7.h[0] + sub v31.8h, v6.8h, v10.8h + sqrdmulh v12.8h, v5.8h, v0.h[3] + sub v22.8h, v20.8h, v18.8h + add v6.8h, v6.8h, v10.8h + mul v20.8h, v31.8h, v1.h[0] + add v30.8h, v22.8h, v2.8h + sqrdmulh v3.8h, v6.8h, v0.h[7] + sqrdmulh v10.8h, v30.8h, v1.h[3] + mul v9.8h, v30.8h, v1.h[2] + ldr q30, [x0, #0x10] + mls v17.8h, v15.8h, v7.h[0] + mls v9.8h, v10.8h, v7.h[0] + mul v15.8h, v6.8h, v0.h[6] + add v24.8h, v30.8h, v11.8h + sub v10.8h, v22.8h, v2.8h + mls v15.8h, v3.8h, v7.h[0] + add v6.8h, v19.8h, v4.8h + add v22.8h, v25.8h, v23.8h + sqrdmulh v3.8h, v10.8h, v1.h[5] + str q13, [x0, #0x140] + sub v19.8h, v30.8h, v11.8h + add v25.8h, v22.8h, v16.8h + mul v5.8h, v5.8h, v0.h[2] + sub v13.8h, v22.8h, v16.8h + str q28, [x0, #0x100] + mls v5.8h, v12.8h, v7.h[0] + str q13, [x0, #0x40] + str q6, [x0, #0x80] + add v21.8h, v21.8h, v29.8h + sqrdmulh v13.8h, v31.8h, v1.h[1] + str q25, [x0], #0x10 + add v12.8h, v19.8h, v17.8h + sub v31.8h, v19.8h, v17.8h + mul v30.8h, v10.8h, v1.h[4] + str q21, [x0, #0x170] + add v21.8h, v24.8h, v5.8h + add v6.8h, v12.8h, v9.8h + mls v30.8h, v3.8h, v7.h[0] + str q27, [x0, #0x1b0] + sub v10.8h, v21.8h, v15.8h + sub v12.8h, v12.8h, v9.8h + mls v20.8h, v13.8h, v7.h[0] + str q6, [x0, #0x100] + str q10, [x0, #0x40] + sub v13.8h, v24.8h, v5.8h + add v3.8h, v21.8h, v15.8h + str q12, [x0, #0x140] + sub v10.8h, v31.8h, v30.8h + add v21.8h, v31.8h, v30.8h + str q3, [x0], #0x10 + add v12.8h, v13.8h, v20.8h + sub v13.8h, v13.8h, v20.8h + str q21, [x0, #0x170] + str q10, [x0, #0x1b0] + str q12, [x0, #0x70] + str q13, [x0, #0xb0] + mov x0, x3 + mov x4, #0x8 // =8 + ldr q2, [x0, #0x20] + ldr q13, [x1], #0x10 + ldr q30, [x0, #0x30] + ldr q25, [x2, #0x40] + ldr q5, [x0] + ldr q18, [x0, #0x60] + ldr q12, [x0, #0x70] + sqrdmulh v17.8h, v2.8h, v13.h[1] + ldr q4, [x1], #0x10 + ldr q23, [x0, #0x10] + sqrdmulh v21.8h, v30.8h, v13.h[1] + ldr q24, [x2, #0x20] + ldr q9, [x2], #0x60 + mul v10.8h, v30.8h, v13.h[0] + mul v11.8h, v2.8h, v13.h[0] + mls v10.8h, v21.8h, v7.h[0] + sqrdmulh v29.8h, v12.8h, v4.h[1] + mul v1.8h, v12.8h, v4.h[0] + add v21.8h, v23.8h, v10.8h + sub v10.8h, v23.8h, v10.8h + mul v8.8h, v18.8h, v4.h[0] + sqrdmulh v23.8h, v21.8h, v13.h[3] + mul v2.8h, v21.8h, v13.h[2] + mls v1.8h, v29.8h, v7.h[0] + mls v2.8h, v23.8h, v7.h[0] + ldur q15, [x2, #-0x50] + sqrdmulh v0.8h, v10.8h, v13.h[5] + mls v11.8h, v17.8h, v7.h[0] + ldr q29, [x0, #0x50] + mul v23.8h, v10.8h, v13.h[4] + mls v23.8h, v0.8h, v7.h[0] + sub v16.8h, v29.8h, v1.8h + add v3.8h, v5.8h, v11.8h + sub v31.8h, v5.8h, v11.8h + sqrdmulh v22.8h, v16.8h, v4.h[5] + add v30.8h, v3.8h, v2.8h + sub v0.8h, v3.8h, v2.8h + sqrdmulh v28.8h, v18.8h, v4.h[1] + add v21.8h, v31.8h, v23.8h + sub v19.8h, v31.8h, v23.8h + mul v26.8h, v16.8h, v4.h[4] + trn2 v3.4s, v30.4s, v0.4s + ldur q23, [x2, #-0x10] + trn2 v18.4s, v21.4s, v19.4s + mls v26.8h, v22.8h, v7.h[0] + trn1 v13.4s, v30.4s, v0.4s + mls v8.8h, v28.8h, v7.h[0] + trn2 v31.2d, v3.2d, v18.2d + trn1 v11.4s, v21.4s, v19.4s + add v27.8h, v29.8h, v1.8h + sqrdmulh v6.8h, v31.8h, v15.8h + trn1 v2.2d, v13.2d, v11.2d + trn2 v13.2d, v13.2d, v11.2d + mul v1.8h, v31.8h, v9.8h + ldr q11, [x0, #0x40] + sqrdmulh v29.8h, v13.8h, v15.8h + mls v1.8h, v6.8h, v7.h[0] + trn1 v6.2d, v3.2d, v18.2d + mul v17.8h, v13.8h, v9.8h + sub v13.8h, v11.8h, v8.8h + sqrdmulh v10.8h, v27.8h, v4.h[3] + sub v12.8h, v13.8h, v26.8h + sub v18.8h, v6.8h, v1.8h + mls v17.8h, v29.8h, v7.h[0] + add v30.8h, v6.8h, v1.8h + add v6.8h, v13.8h, v26.8h + ldur q13, [x2, #-0x30] + sqrdmulh v16.8h, v18.8h, v23.8h + trn1 v28.4s, v6.4s, v12.4s + mul v23.8h, v18.8h, v25.8h + ldr q25, [x2, #0x10] + add v20.8h, v2.8h, v17.8h + mul v0.8h, v30.8h, v24.8h + sqrdmulh v29.8h, v30.8h, v13.8h + sub v30.8h, v2.8h, v17.8h + mls v23.8h, v16.8h, v7.h[0] + sub x4, x4, #0x2 + +Lntt_layer4567_start: + ldr q19, [x2, #0x50] + sub v31.8h, v30.8h, v23.8h + mls v0.8h, v29.8h, v7.h[0] + add v16.8h, v11.8h, v8.8h + ldr q18, [x0, #0xa0] + trn2 v14.4s, v6.4s, v12.4s + mul v26.8h, v27.8h, v4.h[2] + ldr q4, [x1], #0x10 + ldr q24, [x2, #0x40] + ldr q21, [x0, #0xb0] + mls v26.8h, v10.8h, v7.h[0] + add v23.8h, v30.8h, v23.8h + sub v15.8h, v20.8h, v0.8h + ldr q9, [x0, #0x90] + add v10.8h, v20.8h, v0.8h + mul v8.8h, v18.8h, v4.h[0] + ldr q1, [x2], #0x60 + trn1 v27.4s, v23.4s, v31.4s + sqrdmulh v12.8h, v18.8h, v4.h[1] + trn1 v5.4s, v10.4s, v15.4s + sub v30.8h, v16.8h, v26.8h + trn2 v13.2d, v5.2d, v27.2d + sqrdmulh v2.8h, v21.8h, v4.h[1] + add v29.8h, v16.8h, v26.8h + mul v0.8h, v21.8h, v4.h[0] + str q13, [x0, #0x20] + trn1 v11.4s, v29.4s, v30.4s + mls v8.8h, v12.8h, v7.h[0] + trn2 v26.4s, v29.4s, v30.4s + trn2 v6.2d, v11.2d, v28.2d + mls v0.8h, v2.8h, v7.h[0] + trn2 v16.2d, v26.2d, v14.2d + trn1 v26.2d, v26.2d, v14.2d + trn1 v20.2d, v5.2d, v27.2d + sqrdmulh v29.8h, v6.8h, v25.8h + trn2 v15.4s, v10.4s, v15.4s + sqrdmulh v13.8h, v16.8h, v25.8h + str q20, [x0], #0x40 + sub v30.8h, v9.8h, v0.8h + add v27.8h, v9.8h, v0.8h + mul v17.8h, v6.8h, v1.8h + sqrdmulh v22.8h, v30.8h, v4.h[5] + mul v18.8h, v16.8h, v1.8h + mls v18.8h, v13.8h, v7.h[0] + mul v2.8h, v30.8h, v4.h[4] + mls v2.8h, v22.8h, v7.h[0] + trn2 v22.4s, v23.4s, v31.4s + sub v3.8h, v26.8h, v18.8h + ldur q25, [x2, #-0x30] + mls v17.8h, v29.8h, v7.h[0] + trn2 v31.2d, v15.2d, v22.2d + trn1 v20.2d, v15.2d, v22.2d + add v16.8h, v26.8h, v18.8h + sqrdmulh v26.8h, v3.8h, v19.8h + trn1 v21.2d, v11.2d, v28.2d + ldr q11, [x0, #0x40] + sqrdmulh v29.8h, v16.8h, v25.8h + stur q20, [x0, #-0x30] + add v20.8h, v21.8h, v17.8h + stur q31, [x0, #-0x10] + mul v23.8h, v3.8h, v24.8h + ldr q25, [x2, #0x10] + sub v13.8h, v11.8h, v8.8h + mls v23.8h, v26.8h, v7.h[0] + ldur q1, [x2, #-0x40] + sub v12.8h, v13.8h, v2.8h + add v6.8h, v13.8h, v2.8h + sqrdmulh v10.8h, v27.8h, v4.h[3] + sub v30.8h, v21.8h, v17.8h + mul v0.8h, v16.8h, v1.8h + trn1 v28.4s, v6.4s, v12.4s + sub x4, x4, #0x1 + cbnz x4, Lntt_layer4567_start + add v22.8h, v11.8h, v8.8h + mul v27.8h, v27.8h, v4.h[2] + trn2 v17.4s, v6.4s, v12.4s + ldr q15, [x2], #0x60 + mls v27.8h, v10.8h, v7.h[0] + add v4.8h, v30.8h, v23.8h + sub v18.8h, v30.8h, v23.8h + ldur q6, [x2, #-0x30] + mls v0.8h, v29.8h, v7.h[0] + ldur q12, [x2, #-0x40] + ldur q24, [x2, #-0x20] + ldur q2, [x2, #-0x10] + trn1 v9.4s, v4.4s, v18.4s + add v10.8h, v22.8h, v27.8h + sub v13.8h, v22.8h, v27.8h + sub v1.8h, v20.8h, v0.8h + trn2 v21.4s, v10.4s, v13.4s + add v27.8h, v20.8h, v0.8h + trn2 v3.2d, v21.2d, v17.2d + trn1 v13.4s, v10.4s, v13.4s + trn1 v31.4s, v27.4s, v1.4s + sqrdmulh v10.8h, v3.8h, v25.8h + trn2 v5.2d, v13.2d, v28.2d + trn1 v13.2d, v13.2d, v28.2d + trn1 v21.2d, v21.2d, v17.2d + sqrdmulh v17.8h, v5.8h, v25.8h + trn2 v30.2d, v31.2d, v9.2d + mul v25.8h, v3.8h, v15.8h + str q30, [x0, #0x20] + trn2 v30.4s, v4.4s, v18.4s + mls v25.8h, v10.8h, v7.h[0] + trn2 v3.4s, v27.4s, v1.4s + mul v20.8h, v5.8h, v15.8h + trn2 v10.2d, v3.2d, v30.2d + mls v20.8h, v17.8h, v7.h[0] + str q10, [x0, #0x30] + sub v18.8h, v21.8h, v25.8h + add v10.8h, v21.8h, v25.8h + trn1 v3.2d, v3.2d, v30.2d + sqrdmulh v30.8h, v18.8h, v2.8h + mul v12.8h, v10.8h, v12.8h + sqrdmulh v6.8h, v10.8h, v6.8h + str q3, [x0, #0x10] + add v21.8h, v13.8h, v20.8h + mul v10.8h, v18.8h, v24.8h + sub v13.8h, v13.8h, v20.8h + mls v10.8h, v30.8h, v7.h[0] + mls v12.8h, v6.8h, v7.h[0] + trn1 v30.2d, v31.2d, v9.2d + sub v3.8h, v13.8h, v10.8h + add v6.8h, v13.8h, v10.8h + add v10.8h, v21.8h, v12.8h + sub v21.8h, v21.8h, v12.8h + trn2 v13.4s, v6.4s, v3.4s + trn1 v12.4s, v10.4s, v21.4s + trn2 v21.4s, v10.4s, v21.4s + trn1 v3.4s, v6.4s, v3.4s + str q30, [x0], #0x40 + trn2 v10.2d, v21.2d, v13.2d + trn1 v13.2d, v21.2d, v13.2d + trn2 v21.2d, v12.2d, v3.2d + trn1 v3.2d, v12.2d, v3.2d + str q10, [x0, #0x30] + str q13, [x0, #0x10] + str q3, [x0], #0x40 + stur q21, [x0, #-0x20] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(ntt_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_mulcache_compute_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_mulcache_compute_aarch64_asm.S new file mode 100644 index 0000000..376cbcd --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_mulcache_compute_aarch64_asm.S @@ -0,0 +1,129 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_mulcache_compute_aarch64_asm + Description: Compute multiplication cache for polynomial + Signature: void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], const int16_t mlk_poly[256], const int16_t zetas[128], const int16_t zetas_twisted[128]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 256 + permissions: write-only + c_parameter: int16_t cache[128] + description: Output cache + x1: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t mlk_poly[256] + description: Input polynomial + x2: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const int16_t zetas[128] + description: Zeta values + x3: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const int16_t zetas_twisted[128] + description: Twisted zeta values + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/poly_mulcache_compute_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_mulcache_compute_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_mulcache_compute_aarch64_asm) + + .cfi_startproc + mov w5, #0xd01 // =3329 + dup v6.8h, w5 + mov w5, #0x4ebf // =20159 + dup v7.8h, w5 + mov x4, #0x10 // =16 + ldr q0, [x1], #0x20 + ldur q2, [x1, #-0x10] + ldr q19, [x1], #0x20 + ldr q29, [x3], #0x10 + ldur q16, [x1, #-0x10] + ldr q18, [x2], #0x10 + ldr q26, [x1], #0x20 + ldr q25, [x2], #0x10 + uzp2 v5.8h, v0.8h, v2.8h + ldr q28, [x3], #0x10 + ldur q7, [x1, #-0x10] + ldr q2, [x1], #0x20 + uzp2 v27.8h, v19.8h, v16.8h + sqrdmulh v16.8h, v5.8h, v29.8h + ldr q17, [x3], #0x10 + ldr q19, [x3], #0x10 + mul v5.8h, v5.8h, v18.8h + uzp2 v29.8h, v26.8h, v7.8h + mul v26.8h, v27.8h, v25.8h + sqrdmulh v4.8h, v27.8h, v28.8h + mls v5.8h, v16.8h, v6.h[0] + lsr x4, x4, #1 + sub x4, x4, #0x2 + +Lpoly_mulcache_compute_loop_start: + str q5, [x0], #0x10 + sqrdmulh v22.8h, v29.8h, v17.8h + ldr q28, [x2], #0x10 + ldur q24, [x1, #-0x10] + ldr q0, [x1], #0x20 + mls v26.8h, v4.8h, v6.h[0] + ldur q16, [x1, #-0x10] + ldr q17, [x3], #0x10 + mul v5.8h, v29.8h, v28.8h + uzp2 v23.8h, v2.8h, v24.8h + ldr q18, [x2], #0x10 + mls v5.8h, v22.8h, v6.h[0] + uzp2 v29.8h, v0.8h, v16.8h + sqrdmulh v4.8h, v23.8h, v19.8h + ldr q2, [x1], #0x20 + ldr q19, [x3], #0x10 + str q26, [x0], #0x10 + mul v26.8h, v23.8h, v18.8h + subs x4, x4, #0x1 + cbnz x4, Lpoly_mulcache_compute_loop_start + mls v26.8h, v4.8h, v6.h[0] + str q5, [x0], #0x10 + ldr q5, [x2], #0x10 + ldur q4, [x1, #-0x10] + sqrdmulh v16.8h, v29.8h, v17.8h + ldr q0, [x2], #0x10 + mul v29.8h, v29.8h, v5.8h + uzp2 v18.8h, v2.8h, v4.8h + str q26, [x0], #0x10 + sqrdmulh v17.8h, v18.8h, v19.8h + mls v29.8h, v16.8h, v6.h[0] + mul v26.8h, v18.8h, v0.8h + mls v26.8h, v17.8h, v6.h[0] + str q29, [x0], #0x10 + str q26, [x0], #0x10 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_mulcache_compute_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_reduce_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_reduce_aarch64_asm.S new file mode 100644 index 0000000..cd62a18 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_reduce_aarch64_asm.S @@ -0,0 +1,152 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_reduce_aarch64_asm + Description: Barrett reduction of polynomial coefficients + Signature: void mlk_poly_reduce_aarch64_asm(int16_t p[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/poly_reduce_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_reduce_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_reduce_aarch64_asm) + + .cfi_startproc + mov w2, #0xd01 // =3329 + dup v3.8h, w2 + mov w2, #0x4ebf // =20159 + dup v4.8h, w2 + mov x1, #0x8 // =8 + ldr q21, [x0], #0x40 + ldur q18, [x0, #-0x20] + ldur q0, [x0, #-0x30] + ldur q5, [x0, #-0x10] + ldr q26, [x0], #0x40 + sqdmulh v17.8h, v21.8h, v4.h[0] + sqdmulh v27.8h, v18.8h, v4.h[0] + sqdmulh v22.8h, v0.8h, v4.h[0] + srshr v17.8h, v17.8h, #0xb + sqdmulh v23.8h, v5.8h, v4.h[0] + srshr v29.8h, v27.8h, #0xb + mls v21.8h, v17.8h, v3.h[0] + srshr v17.8h, v22.8h, #0xb + mls v18.8h, v29.8h, v3.h[0] + srshr v22.8h, v23.8h, #0xb + mls v0.8h, v17.8h, v3.h[0] + sshr v2.8h, v21.8h, #0xf + mls v5.8h, v22.8h, v3.h[0] + sshr v29.8h, v18.8h, #0xf + and v19.16b, v3.16b, v2.16b + sqdmulh v2.8h, v26.8h, v4.h[0] + sshr v31.8h, v0.8h, #0xf + add v17.8h, v21.8h, v19.8h + and v21.16b, v3.16b, v29.16b + and v31.16b, v3.16b, v31.16b + sub x1, x1, #0x2 + +Lpoly_reduce_loop_start: + add v21.8h, v18.8h, v21.8h + ldur q18, [x0, #-0x20] + add v25.8h, v0.8h, v31.8h + ldur q0, [x0, #-0x30] + stur q21, [x0, #-0x60] + sshr v28.8h, v5.8h, #0xf + stur q17, [x0, #-0x80] + srshr v23.8h, v2.8h, #0xb + sqdmulh v30.8h, v18.8h, v4.h[0] + stur q25, [x0, #-0x70] + and v22.16b, v3.16b, v28.16b + sqdmulh v7.8h, v0.8h, v4.h[0] + add v16.8h, v5.8h, v22.8h + ldur q5, [x0, #-0x10] + mls v26.8h, v23.8h, v3.h[0] + stur q16, [x0, #-0x50] + srshr v6.8h, v30.8h, #0xb + srshr v1.8h, v7.8h, #0xb + sqdmulh v19.8h, v5.8h, v4.h[0] + mls v18.8h, v6.8h, v3.h[0] + sshr v24.8h, v26.8h, #0xf + mls v0.8h, v1.8h, v3.h[0] + and v27.16b, v3.16b, v24.16b + srshr v29.8h, v19.8h, #0xb + add v17.8h, v26.8h, v27.8h + ldr q26, [x0], #0x40 + sshr v1.8h, v18.8h, #0xf + mls v5.8h, v29.8h, v3.h[0] + sshr v20.8h, v0.8h, #0xf + and v21.16b, v3.16b, v1.16b + and v31.16b, v3.16b, v20.16b + sqdmulh v2.8h, v26.8h, v4.h[0] + subs x1, x1, #0x1 + cbnz x1, Lpoly_reduce_loop_start + add v28.8h, v0.8h, v31.8h + ldur q29, [x0, #-0x10] + add v21.8h, v18.8h, v21.8h + srshr v18.8h, v2.8h, #0xb + sshr v2.8h, v5.8h, #0xf + ldur q16, [x0, #-0x20] + stur q17, [x0, #-0x80] + ldur q0, [x0, #-0x30] + and v2.16b, v3.16b, v2.16b + sqdmulh v24.8h, v29.8h, v4.h[0] + stur q28, [x0, #-0x70] + stur q21, [x0, #-0x60] + add v31.8h, v5.8h, v2.8h + sqdmulh v6.8h, v16.8h, v4.h[0] + stur q31, [x0, #-0x50] + sqdmulh v17.8h, v0.8h, v4.h[0] + srshr v22.8h, v24.8h, #0xb + mls v26.8h, v18.8h, v3.h[0] + srshr v31.8h, v6.8h, #0xb + mls v29.8h, v22.8h, v3.h[0] + srshr v19.8h, v17.8h, #0xb + mls v16.8h, v31.8h, v3.h[0] + sshr v7.8h, v26.8h, #0xf + mls v0.8h, v19.8h, v3.h[0] + and v5.16b, v3.16b, v7.16b + sshr v22.8h, v29.8h, #0xf + add v27.8h, v26.8h, v5.8h + and v26.16b, v3.16b, v22.16b + sshr v20.8h, v16.8h, #0xf + stur q27, [x0, #-0x40] + and v2.16b, v3.16b, v20.16b + sshr v23.8h, v0.8h, #0xf + add v18.8h, v29.8h, v26.8h + add v31.8h, v16.8h, v2.8h + and v29.16b, v3.16b, v23.16b + stur q18, [x0, #-0x10] + add v25.8h, v0.8h, v29.8h + stur q31, [x0, #-0x20] + stur q25, [x0, #-0x30] + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_reduce_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tobytes_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tobytes_aarch64_asm.S new file mode 100644 index 0000000..fddbc28 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tobytes_aarch64_asm.S @@ -0,0 +1,119 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_tobytes_aarch64_asm + Description: Convert polynomial to byte representation + Signature: void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 384 + permissions: write-only + c_parameter: uint8_t r[384] + description: Output byte array + x1: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t a[256] + description: Input polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/poly_tobytes_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tobytes_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_tobytes_aarch64_asm) + + .cfi_startproc + mov x2, #0x10 // =16 + ldr q5, [x1, #0x10] + ldr q3, [x1], #0x20 + ldr q29, [x1], #0x20 + ldur q2, [x1, #-0x10] + ldr q27, [x1, #0x10] + ldr q23, [x1, #0x30] + ldr q17, [x1], #0x20 + ldr q16, [x1], #0x20 + uzp2 v26.8h, v3.8h, v5.8h + uzp1 v19.8h, v3.8h, v5.8h + uzp2 v0.8h, v29.8h, v2.8h + uzp1 v1.8h, v29.8h, v2.8h + xtn v5.8b, v26.8h + shrn v3.8b, v19.8h, #0x8 + shrn v4.8b, v26.8h, #0x4 + xtn v18.8b, v0.8h + shrn v30.8b, v0.8h, #0x4 + xtn v28.8b, v1.8h + shrn v29.8b, v1.8h, #0x8 + sli v3.8b, v5.8b, #0x4 + xtn v2.8b, v19.8h + sli v29.8b, v18.8b, #0x4 + lsr x2, x2, #1 + sub x2, x2, #0x2 + +Lpoly_tobytes_loop_start: + uzp1 v25.8h, v17.8h, v27.8h + uzp2 v31.8h, v17.8h, v27.8h + uzp1 v24.8h, v16.8h, v23.8h + uzp2 v6.8h, v16.8h, v23.8h + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + shrn v3.8b, v25.8h, #0x8 + ldr q17, [x1], #0x20 + shrn v4.8b, v31.8h, #0x4 + xtn v21.8b, v6.8h + ldr q23, [x1, #0x10] + st3 { v28.8b, v29.8b, v30.8b }, [x0], #24 + shrn v29.8b, v24.8h, #0x8 + ldur q27, [x1, #-0x10] + xtn v20.8b, v31.8h + ldr q16, [x1], #0x20 + sli v29.8b, v21.8b, #0x4 + xtn v2.8b, v25.8h + sli v3.8b, v20.8b, #0x4 + xtn v28.8b, v24.8h + shrn v30.8b, v6.8h, #0x4 + subs x2, x2, #0x1 + cbnz x2, Lpoly_tobytes_loop_start + uzp2 v7.8h, v17.8h, v27.8h + uzp1 v25.8h, v17.8h, v27.8h + uzp2 v0.8h, v16.8h, v23.8h + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + st3 { v28.8b, v29.8b, v30.8b }, [x0], #24 + shrn v21.8b, v25.8h, #0x8 + uzp1 v2.8h, v16.8h, v23.8h + shrn v22.8b, v7.8h, #0x4 + shrn v4.8b, v0.8h, #0x4 + xtn v28.8b, v7.8h + xtn v27.8b, v0.8h + shrn v3.8b, v2.8h, #0x8 + sli v21.8b, v28.8b, #0x4 + xtn v2.8b, v2.8h + sli v3.8b, v27.8b, #0x4 + xtn v20.8b, v25.8h + st3 { v20.8b, v21.8b, v22.8b }, [x0], #24 + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_tobytes_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tomont_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tomont_aarch64_asm.S new file mode 100644 index 0000000..f311517 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/poly_tomont_aarch64_asm.S @@ -0,0 +1,98 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_tomont_aarch64_asm + Description: Convert polynomial to Montgomery domain + Signature: void mlk_poly_tomont_aarch64_asm(int16_t p[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/poly_tomont_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tomont_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_tomont_aarch64_asm) + + .cfi_startproc + mov w2, #0xd01 // =3329 + dup v4.8h, w2 + mov w2, #-0x414 // =-1044 + dup v2.8h, w2 + mov w2, #-0x2824 // =-10276 + dup v3.8h, w2 + mov x1, #0x8 // =8 + ldr q18, [x0, #0x20] + ldr q0, [x0, #0x10] + ldr q16, [x0], #0x40 + sqrdmulh v23.8h, v0.8h, v3.8h + mul v26.8h, v0.8h, v2.8h + sqrdmulh v19.8h, v16.8h, v3.8h + mls v26.8h, v23.8h, v4.h[0] + mul v29.8h, v16.8h, v2.8h + ldur q16, [x0, #-0x10] + mls v29.8h, v19.8h, v4.h[0] + stur q26, [x0, #-0x30] + sqrdmulh v26.8h, v18.8h, v3.8h + mul v18.8h, v18.8h, v2.8h + stur q29, [x0, #-0x40] + sqrdmulh v29.8h, v16.8h, v3.8h + mls v18.8h, v26.8h, v4.h[0] + sub x1, x1, #0x1 + +Lpoly_tomont_loop: + ldr q19, [x0, #0x10] + mul v26.8h, v16.8h, v2.8h + ldr q23, [x0, #0x20] + ldr q17, [x0], #0x40 + mls v26.8h, v29.8h, v4.h[0] + ldur q16, [x0, #-0x10] + sqrdmulh v28.8h, v19.8h, v3.8h + stur q18, [x0, #-0x60] + mul v0.8h, v19.8h, v2.8h + stur q26, [x0, #-0x50] + sqrdmulh v24.8h, v23.8h, v3.8h + mul v18.8h, v23.8h, v2.8h + sqrdmulh v22.8h, v17.8h, v3.8h + mul v26.8h, v17.8h, v2.8h + mls v0.8h, v28.8h, v4.h[0] + mls v26.8h, v22.8h, v4.h[0] + sqrdmulh v29.8h, v16.8h, v3.8h + stur q0, [x0, #-0x30] + mls v18.8h, v24.8h, v4.h[0] + stur q26, [x0, #-0x40] + sub x1, x1, #0x1 + cbnz x1, Lpoly_tomont_loop + mul v16.8h, v16.8h, v2.8h + stur q18, [x0, #-0x20] + mls v16.8h, v29.8h, v4.h[0] + stur q16, [x0, #-0x10] + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_tomont_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S new file mode 100644 index 0000000..9ba56fc --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S @@ -0,0 +1,263 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=2 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(int16_t r[256], const int16_t a[512], const int16_t b[512], const int16_t b_cache[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t a[512] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t b[512] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t b_cache[256] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + mov x13, #0x10 // =16 + ldr q12, [x1], #0x20 + ldur q9, [x1, #-0x10] + ldr q22, [x2], #0x20 + ldur q30, [x2, #-0x10] + ldr q6, [x5], #0x20 + ldr q7, [x4, #0x10] + ldr q8, [x4], #0x20 + ldur q23, [x5, #-0x10] + uzp1 v16.8h, v12.8h, v9.8h + uzp2 v14.8h, v12.8h, v9.8h + uzp2 v13.8h, v22.8h, v30.8h + uzp1 v18.8h, v22.8h, v30.8h + ld1 { v27.8h }, [x3], #16 + ld1 { v17.8h }, [x6], #16 + smull2 v4.4s, v16.8h, v18.8h + ldr q31, [x1, #0x10] + smull v19.4s, v16.4h, v13.4h + ldr q24, [x1], #0x20 + smlal v19.4s, v14.4h, v18.4h + ldr q22, [x2], #0x20 + smlal2 v4.4s, v14.8h, v27.8h + uzp2 v5.8h, v6.8h, v23.8h + smull2 v29.4s, v16.8h, v13.8h + uzp2 v26.8h, v8.8h, v7.8h + smlal2 v29.4s, v14.8h, v18.8h + uzp1 v30.8h, v24.8h, v31.8h + uzp1 v8.8h, v8.8h, v7.8h + smull v11.4s, v16.4h, v18.4h + smlal v11.4s, v14.4h, v27.4h + ldur q1, [x2, #-0x10] + uzp1 v28.8h, v6.8h, v23.8h + smlal2 v29.4s, v8.8h, v5.8h + ldr q25, [x5], #0x20 + smlal v19.4s, v8.4h, v5.4h + ldr q3, [x4, #0x10] + smlal2 v29.4s, v26.8h, v28.8h + uzp1 v27.8h, v22.8h, v1.8h + smlal v19.4s, v26.4h, v28.4h + ldr q12, [x4], #0x20 + smlal2 v4.4s, v8.8h, v28.8h + ldur q21, [x5, #-0x10] + smlal2 v4.4s, v26.8h, v17.8h + smlal v11.4s, v8.4h, v28.4h + ld1 { v15.8h }, [x6], #16 + smlal v11.4s, v26.4h, v17.4h + ld1 { v20.8h }, [x3], #16 + uzp1 v28.8h, v19.8h, v29.8h + smull2 v23.4s, v30.8h, v27.8h + smull v26.4s, v30.4h, v27.4h + uzp2 v16.8h, v22.8h, v1.8h + mul v28.8h, v28.8h, v2.8h + uzp1 v10.8h, v11.8h, v4.8h + smull2 v8.4s, v30.8h, v16.8h + mul v13.8h, v10.8h, v2.8h + smlal v19.4s, v28.4h, v0.4h + smlal2 v29.4s, v28.8h, v0.8h + smull v18.4s, v30.4h, v16.4h + uzp1 v30.8h, v25.8h, v21.8h + smlal v11.4s, v13.4h, v0.4h + uzp2 v6.8h, v24.8h, v31.8h + uzp1 v16.8h, v12.8h, v3.8h + smlal2 v4.4s, v13.8h, v0.8h + uzp2 v17.8h, v25.8h, v21.8h + smlal2 v8.4s, v6.8h, v27.8h + uzp2 v12.8h, v12.8h, v3.8h + smlal v18.4s, v6.4h, v27.4h + uzp2 v9.8h, v19.8h, v29.8h + smlal2 v8.4s, v16.8h, v17.8h + smlal2 v8.4s, v12.8h, v30.8h + uzp2 v19.8h, v11.8h, v4.8h + sub x13, x13, #0x2 + +Lpolyvec_basemul_acc_montgomery_cached_k2_loop_start: + smlal v18.4s, v16.4h, v17.4h + ldr q7, [x4], #0x20 + ldr q10, [x2, #0x10] + smlal v18.4s, v12.4h, v30.4h + smlal2 v23.4s, v6.8h, v20.8h + ldr q14, [x2], #0x20 + smlal2 v23.4s, v16.8h, v30.8h + zip1 v25.8h, v19.8h, v9.8h + zip2 v3.8h, v19.8h, v9.8h + smlal2 v23.4s, v12.8h, v15.8h + smlal v26.4s, v6.4h, v20.4h + uzp1 v5.8h, v18.8h, v8.8h + uzp2 v21.8h, v14.8h, v10.8h + smlal v26.4s, v16.4h, v30.4h + str q25, [x0], #0x20 + mul v29.8h, v5.8h, v2.8h + uzp1 v24.8h, v14.8h, v10.8h + stur q3, [x0, #-0x10] + smlal v26.4s, v12.4h, v15.4h + ld1 { v15.8h }, [x6], #16 + ldr q28, [x1, #0x10] + ldr q11, [x1], #0x20 + ldr q13, [x5], #0x20 + ldur q27, [x4, #-0x10] + smlal2 v8.4s, v29.8h, v0.8h + ldur q22, [x5, #-0x10] + smlal v18.4s, v29.4h, v0.4h + uzp1 v4.8h, v26.8h, v23.8h + uzp1 v1.8h, v11.8h, v28.8h + uzp2 v6.8h, v11.8h, v28.8h + uzp1 v16.8h, v7.8h, v27.8h + mul v31.8h, v4.8h, v2.8h + uzp2 v17.8h, v13.8h, v22.8h + ld1 { v20.8h }, [x3], #16 + uzp2 v9.8h, v18.8h, v8.8h + smull2 v8.4s, v1.8h, v21.8h + uzp1 v30.8h, v13.8h, v22.8h + smlal2 v8.4s, v6.8h, v24.8h + smlal2 v8.4s, v16.8h, v17.8h + uzp2 v12.8h, v7.8h, v27.8h + smlal v26.4s, v31.4h, v0.4h + smlal2 v23.4s, v31.8h, v0.8h + smull v18.4s, v1.4h, v21.4h + smlal v18.4s, v6.4h, v24.4h + smlal2 v8.4s, v12.8h, v30.8h + uzp2 v19.8h, v26.8h, v23.8h + smull2 v23.4s, v1.8h, v24.8h + smull v26.4s, v1.4h, v24.4h + subs x13, x13, #0x1 + cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k2_loop_start + smlal v26.4s, v6.4h, v20.4h + smlal2 v23.4s, v6.8h, v20.8h + smlal v26.4s, v16.4h, v30.4h + smlal2 v23.4s, v16.8h, v30.8h + smlal v26.4s, v12.4h, v15.4h + smlal2 v23.4s, v12.8h, v15.8h + smlal v18.4s, v16.4h, v17.4h + smlal v18.4s, v12.4h, v30.4h + zip1 v12.8h, v19.8h, v9.8h + str q12, [x0], #0x20 + uzp1 v12.8h, v26.8h, v23.8h + mul v6.8h, v12.8h, v2.8h + uzp1 v12.8h, v18.8h, v8.8h + mul v12.8h, v12.8h, v2.8h + smlal v26.4s, v6.4h, v0.4h + smlal2 v23.4s, v6.8h, v0.8h + smlal2 v8.4s, v12.8h, v0.8h + smlal v18.4s, v12.4h, v0.4h + zip2 v12.8h, v19.8h, v9.8h + uzp2 v6.8h, v26.8h, v23.8h + stur q12, [x0, #-0x10] + uzp2 v12.8h, v18.8h, v8.8h + zip2 v1.8h, v6.8h, v12.8h + zip1 v12.8h, v6.8h, v12.8h + str q1, [x0, #0x10] + str q12, [x0], #0x20 + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) + + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S new file mode 100644 index 0000000..10b76bc --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S @@ -0,0 +1,316 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=3 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(int16_t r[256], const int16_t a[768], const int16_t b[768], const int16_t b_cache[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t a[768] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t b[768] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t b_cache[384] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + add x7, x1, #0x400 + add x8, x2, #0x400 + add x9, x3, #0x200 + mov x13, #0x10 // =16 + ldr q6, [x7], #0x20 + ldr q19, [x2, #0x10] + ldr q23, [x1], #0x20 + ldur q14, [x1, #-0x10] + ldr q17, [x2], #0x20 + ldr q11, [x4, #0x10] + ldur q28, [x7, #-0x10] + ld1 { v30.8h }, [x3], #16 + ldr q26, [x4], #0x20 + ldr q16, [x8, #0x10] + uzp1 v8.8h, v23.8h, v14.8h + ldr q22, [x5, #0x10] + ldr q18, [x5], #0x20 + uzp1 v20.8h, v17.8h, v19.8h + uzp2 v24.8h, v23.8h, v14.8h + ldr q31, [x8], #0x20 + smull2 v4.4s, v8.8h, v20.8h + uzp1 v25.8h, v26.8h, v11.8h + smull v13.4s, v8.4h, v20.4h + ld1 { v23.8h }, [x6], #16 + uzp1 v1.8h, v18.8h, v22.8h + smlal v13.4s, v24.4h, v30.4h + smlal2 v4.4s, v24.8h, v30.8h + uzp2 v5.8h, v26.8h, v11.8h + smlal2 v4.4s, v25.8h, v1.8h + uzp1 v29.8h, v6.8h, v28.8h + smlal2 v4.4s, v5.8h, v23.8h + ld1 { v7.8h }, [x9], #16 + smlal v13.4s, v25.4h, v1.4h + uzp2 v17.8h, v17.8h, v19.8h + uzp1 v27.8h, v31.8h, v16.8h + smlal v13.4s, v5.4h, v23.4h + uzp2 v22.8h, v18.8h, v22.8h + smull v18.4s, v8.4h, v17.4h + uzp2 v28.8h, v6.8h, v28.8h + smlal v13.4s, v29.4h, v27.4h + smlal2 v4.4s, v29.8h, v27.8h + uzp2 v26.8h, v31.8h, v16.8h + smlal2 v4.4s, v28.8h, v7.8h + ldr q3, [x7, #0x10] + smlal v13.4s, v28.4h, v7.4h + ldr q7, [x1], #0x20 + smlal v18.4s, v24.4h, v20.4h + ldr q15, [x2], #0x20 + smlal v18.4s, v25.4h, v22.4h + smull2 v8.4s, v8.8h, v17.8h + ldur q17, [x1, #-0x10] + uzp1 v23.8h, v13.8h, v4.8h + smlal v18.4s, v5.4h, v1.4h + smlal2 v8.4s, v24.8h, v20.8h + ld1 { v16.8h }, [x3], #16 + mul v23.8h, v23.8h, v2.8h + ldr q19, [x5, #0x10] + ldr q14, [x4, #0x10] + ldr q11, [x4], #0x20 + ldur q20, [x2, #-0x10] + smlal2 v8.4s, v25.8h, v22.8h + smlal2 v8.4s, v5.8h, v1.8h + ldr q22, [x5], #0x20 + uzp1 v1.8h, v7.8h, v17.8h + smlal v18.4s, v29.4h, v26.4h + smlal v13.4s, v23.4h, v0.4h + uzp2 v31.8h, v11.8h, v14.8h + uzp1 v21.8h, v15.8h, v20.8h + smlal2 v4.4s, v23.8h, v0.8h + ld1 { v9.8h }, [x6], #16 + smlal v18.4s, v28.4h, v27.4h + smlal2 v8.4s, v29.8h, v26.8h + ldr q25, [x7], #0x20 + smull v26.4s, v1.4h, v21.4h + uzp1 v24.8h, v22.8h, v19.8h + smlal2 v8.4s, v28.8h, v27.8h + uzp2 v28.8h, v7.8h, v17.8h + uzp1 v29.8h, v11.8h, v14.8h + smull2 v23.4s, v1.8h, v21.8h + ldr q27, [x8], #0x20 + smlal2 v23.4s, v28.8h, v16.8h + ldur q11, [x8, #-0x10] + smlal2 v23.4s, v29.8h, v24.8h + uzp2 v7.8h, v13.8h, v4.8h + uzp2 v19.8h, v22.8h, v19.8h + ld1 { v4.8h }, [x9], #16 + smlal2 v23.4s, v31.8h, v9.8h + uzp1 v13.8h, v25.8h, v3.8h + uzp1 v14.8h, v18.8h, v8.8h + smlal v26.4s, v28.4h, v16.4h + uzp2 v17.8h, v27.8h, v11.8h + uzp2 v20.8h, v15.8h, v20.8h + mul v14.8h, v14.8h, v2.8h + sub x13, x13, #0x2 + +Lpolyvec_basemul_acc_montgomery_cached_k3_loop_start: + uzp1 v6.8h, v27.8h, v11.8h + smlal v26.4s, v29.4h, v24.4h + uzp2 v16.8h, v25.8h, v3.8h + smlal v26.4s, v31.4h, v9.4h + ldr q3, [x7, #0x10] + smlal v26.4s, v13.4h, v6.4h + smlal2 v8.4s, v14.8h, v0.8h + ldr q27, [x8], #0x20 + smlal v18.4s, v14.4h, v0.4h + ldr q25, [x7], #0x20 + smlal2 v23.4s, v13.8h, v6.8h + ldr q11, [x1], #0x20 + smlal2 v23.4s, v16.8h, v4.8h + smlal v26.4s, v16.4h, v4.4h + ldur q22, [x1, #-0x10] + uzp2 v30.8h, v18.8h, v8.8h + smull v18.4s, v1.4h, v20.4h + smlal v18.4s, v28.4h, v21.4h + ldr q14, [x2], #0x20 + smlal v18.4s, v29.4h, v19.4h + zip1 v5.8h, v7.8h, v30.8h + uzp1 v4.8h, v26.8h, v23.8h + smull2 v8.4s, v1.8h, v20.8h + zip2 v10.8h, v7.8h, v30.8h + smlal v18.4s, v31.4h, v24.4h + mul v12.8h, v4.8h, v2.8h + ldr q4, [x5, #0x10] + ldr q20, [x4, #0x10] + ldr q1, [x4], #0x20 + ldur q30, [x2, #-0x10] + smlal2 v8.4s, v28.8h, v21.8h + smlal2 v8.4s, v29.8h, v19.8h + ldr q19, [x5], #0x20 + smlal2 v8.4s, v31.8h, v24.8h + ld1 { v15.8h }, [x3], #16 + uzp2 v31.8h, v1.8h, v20.8h + smlal v26.4s, v12.4h, v0.4h + smlal2 v23.4s, v12.8h, v0.8h + uzp1 v21.8h, v14.8h, v30.8h + uzp1 v29.8h, v1.8h, v20.8h + uzp1 v1.8h, v11.8h, v22.8h + smlal2 v8.4s, v13.8h, v17.8h + ld1 { v9.8h }, [x6], #16 + smlal v18.4s, v13.4h, v17.4h + uzp1 v24.8h, v19.8h, v4.8h + uzp2 v7.8h, v26.8h, v23.8h + smull v26.4s, v1.4h, v21.4h + smlal v18.4s, v16.4h, v6.4h + uzp2 v19.8h, v19.8h, v4.8h + smlal2 v8.4s, v16.8h, v6.8h + uzp2 v28.8h, v11.8h, v22.8h + smull2 v23.4s, v1.8h, v21.8h + uzp1 v13.8h, v25.8h, v3.8h + smlal2 v23.4s, v28.8h, v15.8h + ldur q11, [x8, #-0x10] + smlal2 v23.4s, v29.8h, v24.8h + ld1 { v4.8h }, [x9], #16 + smlal2 v23.4s, v31.8h, v9.8h + uzp1 v12.8h, v18.8h, v8.8h + uzp2 v20.8h, v14.8h, v30.8h + smlal v26.4s, v28.4h, v15.4h + str q5, [x0], #0x20 + mul v14.8h, v12.8h, v2.8h + stur q10, [x0, #-0x10] + uzp2 v17.8h, v27.8h, v11.8h + subs x13, x13, #0x1 + cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k3_loop_start + uzp2 v3.8h, v25.8h, v3.8h + smull2 v16.4s, v1.8h, v20.8h + smull v25.4s, v1.4h, v20.4h + uzp1 v22.8h, v27.8h, v11.8h + smlal2 v16.4s, v28.8h, v21.8h + smlal v25.4s, v28.4h, v21.4h + smlal2 v16.4s, v29.8h, v19.8h + smlal v25.4s, v29.4h, v19.4h + smlal2 v16.4s, v31.8h, v24.8h + smlal v25.4s, v31.4h, v24.4h + smlal v25.4s, v13.4h, v17.4h + smlal2 v16.4s, v13.8h, v17.8h + smlal2 v16.4s, v3.8h, v22.8h + smlal v25.4s, v3.4h, v22.4h + smlal2 v23.4s, v13.8h, v22.8h + smlal v26.4s, v29.4h, v24.4h + smlal v26.4s, v31.4h, v9.4h + smlal v26.4s, v13.4h, v22.4h + uzp1 v10.8h, v25.8h, v16.8h + smlal2 v23.4s, v3.8h, v4.8h + smlal v26.4s, v3.4h, v4.4h + mul v13.8h, v10.8h, v2.8h + smlal v18.4s, v14.4h, v0.4h + smlal2 v8.4s, v14.8h, v0.8h + uzp1 v3.8h, v26.8h, v23.8h + mul v24.8h, v3.8h, v2.8h + uzp2 v17.8h, v18.8h, v8.8h + smlal v25.4s, v13.4h, v0.4h + smlal2 v16.4s, v13.8h, v0.8h + zip1 v21.8h, v7.8h, v17.8h + zip2 v20.8h, v7.8h, v17.8h + smlal2 v23.4s, v24.8h, v0.8h + str q21, [x0], #0x20 + smlal v26.4s, v24.4h, v0.4h + uzp2 v13.8h, v25.8h, v16.8h + stur q20, [x0, #-0x10] + uzp2 v23.8h, v26.8h, v23.8h + zip1 v18.8h, v23.8h, v13.8h + zip2 v13.8h, v23.8h, v13.8h + str q18, [x0], #0x20 + stur q13, [x0, #-0x10] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S new file mode 100644 index 0000000..1f5128a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S @@ -0,0 +1,370 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=4 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(int16_t r[256], const int16_t a[1024], const int16_t b[1024], const int16_t b_cache[512]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t a[1024] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t b[1024] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t b_cache[512] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + add x7, x1, #0x400 + add x8, x2, #0x400 + add x9, x3, #0x200 + add x10, x1, #0x600 + add x11, x2, #0x600 + add x12, x3, #0x300 + mov x13, #0x10 // =16 + ldr q28, [x1], #0x20 + ldur q5, [x1, #-0x10] + ldr q31, [x2], #0x20 + ldur q27, [x2, #-0x10] + ldr q7, [x5], #0x20 + ldr q10, [x4], #0x20 + ldur q18, [x5, #-0x10] + ldur q9, [x4, #-0x10] + uzp1 v11.8h, v28.8h, v5.8h + uzp2 v19.8h, v28.8h, v5.8h + uzp2 v4.8h, v31.8h, v27.8h + uzp1 v1.8h, v31.8h, v27.8h + ldr q29, [x7], #0x20 + ldr q28, [x8, #0x10] + uzp1 v24.8h, v10.8h, v9.8h + uzp1 v17.8h, v7.8h, v18.8h + uzp2 v7.8h, v7.8h, v18.8h + ldr q21, [x8], #0x20 + uzp2 v27.8h, v10.8h, v9.8h + ldur q6, [x7, #-0x10] + smull v18.4s, v11.4h, v4.4h + ld1 { v9.8h }, [x3], #16 + smull2 v8.4s, v11.8h, v4.8h + ldr q16, [x11], #0x20 + smlal2 v8.4s, v19.8h, v1.8h + ldur q14, [x11, #-0x10] + smlal v18.4s, v19.4h, v1.4h + uzp1 v10.8h, v21.8h, v28.8h + smlal v18.4s, v24.4h, v7.4h + ldr q4, [x10], #0x20 + smlal2 v8.4s, v24.8h, v7.8h + ld1 { v12.8h }, [x6], #16 + smull2 v23.4s, v11.8h, v1.8h + uzp2 v13.8h, v29.8h, v6.8h + smull v26.4s, v11.4h, v1.4h + uzp1 v29.8h, v29.8h, v6.8h + smlal v26.4s, v19.4h, v9.4h + ldur q15, [x10, #-0x10] + smlal2 v23.4s, v19.8h, v9.8h + uzp2 v9.8h, v21.8h, v28.8h + smlal v18.4s, v27.4h, v17.4h + uzp2 v6.8h, v16.8h, v14.8h + uzp1 v21.8h, v16.8h, v14.8h + smlal2 v8.4s, v27.8h, v17.8h + smlal2 v8.4s, v29.8h, v9.8h + uzp1 v30.8h, v4.8h, v15.8h + uzp2 v16.8h, v4.8h, v15.8h + smlal v18.4s, v29.4h, v9.4h + smlal2 v8.4s, v13.8h, v10.8h + ld1 { v15.8h }, [x9], #16 + smlal v18.4s, v13.4h, v10.4h + ldr q11, [x4], #0x20 + smlal v18.4s, v30.4h, v6.4h + ldr q7, [x2], #0x20 + smlal2 v8.4s, v30.8h, v6.8h + ld1 { v9.8h }, [x12], #16 + smlal2 v23.4s, v24.8h, v17.8h + ldur q4, [x2, #-0x10] + smlal v26.4s, v24.4h, v17.4h + ldur q25, [x4, #-0x10] + smlal2 v8.4s, v16.8h, v21.8h + ldr q5, [x5], #0x20 + smlal v18.4s, v16.4h, v21.4h + ldur q22, [x5, #-0x10] + smlal v26.4s, v27.4h, v12.4h + ldr q19, [x1, #0x10] + smlal v26.4s, v29.4h, v10.4h + ld1 { v20.8h }, [x3], #16 + smlal v26.4s, v13.4h, v15.4h + uzp1 v24.8h, v7.8h, v4.8h + smlal2 v23.4s, v27.8h, v12.8h + uzp1 v28.8h, v18.8h, v8.8h + smlal v26.4s, v30.4h, v21.4h + uzp2 v27.8h, v11.8h, v25.8h + smlal2 v23.4s, v29.8h, v10.8h + uzp2 v31.8h, v7.8h, v4.8h + smlal2 v23.4s, v13.8h, v15.8h + uzp1 v14.8h, v5.8h, v22.8h + uzp1 v17.8h, v11.8h, v25.8h + smlal v26.4s, v16.4h, v9.4h + mul v29.8h, v28.8h, v2.8h + sub x13, x13, #0x2 + +Lpolyvec_basemul_acc_montgomery_cached_k4_loop_start: + smlal2 v23.4s, v30.8h, v21.8h + ldr q11, [x1], #0x20 + uzp2 v15.8h, v5.8h, v22.8h + smlal v18.4s, v29.4h, v0.4h + ldr q12, [x7], #0x20 + smlal2 v8.4s, v29.8h, v0.8h + ldur q3, [x7, #-0x10] + ldr q21, [x8], #0x20 + uzp1 v29.8h, v11.8h, v19.8h + ldur q13, [x8, #-0x10] + uzp2 v5.8h, v11.8h, v19.8h + smlal2 v23.4s, v16.8h, v9.8h + uzp2 v28.8h, v18.8h, v8.8h + smull2 v8.4s, v29.8h, v31.8h + smlal2 v8.4s, v5.8h, v24.8h + uzp1 v7.8h, v12.8h, v3.8h + smlal2 v8.4s, v17.8h, v15.8h + uzp2 v11.8h, v21.8h, v13.8h + uzp1 v4.8h, v26.8h, v23.8h + smlal2 v8.4s, v27.8h, v14.8h + smlal2 v8.4s, v7.8h, v11.8h + mul v6.8h, v4.8h, v2.8h + ldr q19, [x11], #0x20 + uzp2 v25.8h, v12.8h, v3.8h + ldr q12, [x10], #0x20 + smull v18.4s, v29.4h, v31.4h + ldur q3, [x10, #-0x10] + smlal v18.4s, v5.4h, v24.4h + uzp1 v4.8h, v21.8h, v13.8h + smlal v18.4s, v17.4h, v15.4h + ldur q13, [x11, #-0x10] + ld1 { v1.8h }, [x6], #16 + smlal v26.4s, v6.4h, v0.4h + smlal2 v23.4s, v6.8h, v0.8h + ld1 { v10.8h }, [x9], #16 + smlal v18.4s, v27.4h, v14.4h + uzp1 v30.8h, v12.8h, v3.8h + smlal2 v8.4s, v25.8h, v4.8h + uzp2 v31.8h, v19.8h, v13.8h + smlal v18.4s, v7.4h, v11.4h + ld1 { v9.8h }, [x12], #16 + smlal v18.4s, v25.4h, v4.4h + uzp1 v21.8h, v19.8h, v13.8h + uzp2 v16.8h, v12.8h, v3.8h + smlal v18.4s, v30.4h, v31.4h + smlal2 v8.4s, v30.8h, v31.8h + uzp2 v31.8h, v26.8h, v23.8h + smlal2 v8.4s, v16.8h, v21.8h + smlal v18.4s, v16.4h, v21.4h + zip1 v15.8h, v31.8h, v28.8h + ldr q19, [x1, #0x10] + smull2 v23.4s, v29.8h, v24.8h + smull v26.4s, v29.4h, v24.4h + ldr q3, [x2, #0x10] + smlal v26.4s, v5.4h, v20.4h + ldr q11, [x2], #0x20 + uzp1 v6.8h, v18.8h, v8.8h + smlal v26.4s, v17.4h, v14.4h + smlal v26.4s, v27.4h, v1.4h + zip2 v13.8h, v31.8h, v28.8h + smlal v26.4s, v7.4h, v4.4h + str q15, [x0], #0x20 + smlal v26.4s, v25.4h, v10.4h + stur q13, [x0, #-0x10] + mul v29.8h, v6.8h, v2.8h + uzp1 v24.8h, v11.8h, v3.8h + uzp2 v31.8h, v11.8h, v3.8h + ldr q11, [x4], #0x20 + smlal2 v23.4s, v5.8h, v20.8h + ldur q28, [x4, #-0x10] + smlal2 v23.4s, v17.8h, v14.8h + ldr q5, [x5], #0x20 + smlal2 v23.4s, v27.8h, v1.8h + ldur q22, [x5, #-0x10] + smlal v26.4s, v30.4h, v21.4h + ld1 { v20.8h }, [x3], #16 + smlal v26.4s, v16.4h, v9.4h + uzp1 v17.8h, v11.8h, v28.8h + smlal2 v23.4s, v7.8h, v4.8h + uzp2 v27.8h, v11.8h, v28.8h + smlal2 v23.4s, v25.8h, v10.8h + uzp1 v14.8h, v5.8h, v22.8h + subs x13, x13, #0x1 + cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k4_loop_start + smlal v18.4s, v29.4h, v0.4h + ldr q11, [x1], #0x20 + uzp2 v28.8h, v5.8h, v22.8h + smlal2 v23.4s, v30.8h, v21.8h + smlal2 v8.4s, v29.8h, v0.8h + ldr q15, [x8, #0x10] + smlal2 v23.4s, v16.8h, v9.8h + ldr q21, [x8], #0x20 + uzp1 v22.8h, v11.8h, v19.8h + uzp2 v12.8h, v11.8h, v19.8h + ldr q1, [x7, #0x10] + ld1 { v6.8h }, [x6], #16 + uzp2 v3.8h, v18.8h, v8.8h + smull v9.4s, v22.4h, v31.4h + smull2 v18.4s, v22.8h, v31.8h + ldr q16, [x7], #0x20 + smull v19.4s, v22.4h, v24.4h + uzp1 v30.8h, v21.8h, v15.8h + uzp2 v25.8h, v21.8h, v15.8h + smull2 v8.4s, v22.8h, v24.8h + smlal v19.4s, v12.4h, v20.4h + ldr q13, [x10, #0x10] + smlal2 v8.4s, v12.8h, v20.8h + uzp1 v29.8h, v16.8h, v1.8h + smlal2 v18.4s, v12.8h, v24.8h + ldr q5, [x10], #0x20 + smlal v9.4s, v12.4h, v24.4h + ldr q4, [x11], #0x20 + smlal v9.4s, v17.4h, v28.4h + ldur q22, [x11, #-0x10] + smlal2 v18.4s, v17.8h, v28.8h + uzp2 v16.8h, v16.8h, v1.8h + smlal v19.4s, v17.4h, v14.4h + ld1 { v28.8h }, [x9], #16 + smlal2 v8.4s, v17.8h, v14.8h + uzp1 v7.8h, v5.8h, v13.8h + smlal v9.4s, v27.4h, v14.4h + uzp1 v17.8h, v4.8h, v22.8h + smlal2 v18.4s, v27.8h, v14.8h + uzp2 v12.8h, v5.8h, v13.8h + uzp2 v21.8h, v4.8h, v22.8h + smlal v19.4s, v27.4h, v6.4h + smlal2 v8.4s, v27.8h, v6.8h + ld1 { v15.8h }, [x12], #16 + smlal v19.4s, v29.4h, v30.4h + uzp1 v20.8h, v26.8h, v23.8h + smlal v9.4s, v29.4h, v25.4h + smlal2 v18.4s, v29.8h, v25.8h + smlal2 v8.4s, v29.8h, v30.8h + smlal v19.4s, v16.4h, v28.4h + smlal2 v8.4s, v16.8h, v28.8h + smlal2 v18.4s, v16.8h, v30.8h + smlal v9.4s, v16.4h, v30.4h + smlal v9.4s, v7.4h, v21.4h + smlal2 v18.4s, v7.8h, v21.8h + smlal2 v8.4s, v7.8h, v17.8h + smlal v19.4s, v7.4h, v17.4h + smlal v19.4s, v12.4h, v15.4h + smlal2 v8.4s, v12.8h, v15.8h + smlal2 v18.4s, v12.8h, v17.8h + smlal v9.4s, v12.4h, v17.4h + mul v6.8h, v20.8h, v2.8h + uzp1 v4.8h, v19.8h, v8.8h + mul v17.8h, v4.8h, v2.8h + uzp1 v12.8h, v9.8h, v18.8h + smlal v26.4s, v6.4h, v0.4h + mul v21.8h, v12.8h, v2.8h + smlal2 v23.4s, v6.8h, v0.8h + smlal2 v8.4s, v17.8h, v0.8h + smlal v19.4s, v17.4h, v0.4h + smlal2 v18.4s, v21.8h, v0.8h + uzp2 v23.8h, v26.8h, v23.8h + smlal v9.4s, v21.4h, v0.4h + zip2 v12.8h, v23.8h, v3.8h + zip1 v22.8h, v23.8h, v3.8h + uzp2 v14.8h, v19.8h, v8.8h + uzp2 v18.8h, v9.8h, v18.8h + str q12, [x0, #0x10] + str q22, [x0], #0x20 + zip2 v24.8h, v14.8h, v18.8h + zip1 v21.8h, v14.8h, v18.8h + str q24, [x0, #0x10] + str q21, [x0], #0x20 + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_aarch64_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_aarch64_asm.S new file mode 100644 index 0000000..3918338 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_aarch64_asm.S @@ -0,0 +1,225 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_aarch64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q + Signature: uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output buffer + x1: + type: buffer + size_bytes: x2 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + x2: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be multiple of 24) + test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE + x3: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t table[4096] + description: Lookup table + Stack: + bytes: 576 + description: register preservation and temporary storage +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/rej_uniform_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(rej_uniform_aarch64_asm) +MLK_ASM_FN_SYMBOL(rej_uniform_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x240 + .cfi_adjust_cfa_offset 0x240 + mov x7, #0x1 // =1 + movk x7, #0x2, lsl #16 + movk x7, #0x4, lsl #32 + movk x7, #0x8, lsl #48 + mov v31.d[0], x7 + mov x7, #0x10 // =16 + movk x7, #0x20, lsl #16 + movk x7, #0x40, lsl #32 + movk x7, #0x80, lsl #48 + mov v31.d[1], x7 + mov w11, #0xd01 // =3329 + dup v30.8h, w11 + mov x8, sp + mov x7, x8 + mov x11, #0x0 // =0 + eor v16.16b, v16.16b, v16.16b + +Lrej_uniform_initial_zero: + str q16, [x7], #0x40 + stur q16, [x7, #-0x30] + stur q16, [x7, #-0x20] + stur q16, [x7, #-0x10] + add x11, x11, #0x20 + cmp x11, #0x100 + b.lt Lrej_uniform_initial_zero + mov x7, x8 + mov x9, #0x0 // =0 + mov x4, #0x100 // =256 + cmp x2, #0x30 + b.lo Lrej_uniform_loop48_end + +Lrej_uniform_loop48: + cmp x9, x4 + b.hs Lrej_uniform_memory_copy + sub x2, x2, #0x30 + ld3 { v0.16b, v1.16b, v2.16b }, [x1], #48 + zip1 v4.16b, v0.16b, v1.16b + zip2 v5.16b, v0.16b, v1.16b + zip1 v6.16b, v1.16b, v2.16b + zip2 v7.16b, v1.16b, v2.16b + bic v4.8h, #0xf0, lsl #8 + bic v5.8h, #0xf0, lsl #8 + ushr v6.8h, v6.8h, #0x4 + ushr v7.8h, v7.8h, #0x4 + zip1 v16.8h, v4.8h, v6.8h + zip2 v17.8h, v4.8h, v6.8h + zip1 v18.8h, v5.8h, v7.8h + zip2 v19.8h, v5.8h, v7.8h + cmhi v4.8h, v30.8h, v16.8h + cmhi v5.8h, v30.8h, v17.8h + cmhi v6.8h, v30.8h, v18.8h + cmhi v7.8h, v30.8h, v19.8h + and v4.16b, v4.16b, v31.16b + and v5.16b, v5.16b, v31.16b + and v6.16b, v6.16b, v31.16b + and v7.16b, v7.16b, v31.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + uaddlv s22, v6.8h + uaddlv s23, v7.8h + fmov w12, s20 + fmov w13, s21 + fmov w14, s22 + fmov w15, s23 + ldr q24, [x3, x12, lsl #4] + ldr q25, [x3, x13, lsl #4] + ldr q26, [x3, x14, lsl #4] + ldr q27, [x3, x15, lsl #4] + cnt v4.16b, v4.16b + cnt v5.16b, v5.16b + cnt v6.16b, v6.16b + cnt v7.16b, v7.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + uaddlv s22, v6.8h + uaddlv s23, v7.8h + fmov w12, s20 + fmov w13, s21 + fmov w14, s22 + fmov w15, s23 + tbl v16.16b, { v16.16b }, v24.16b + tbl v17.16b, { v17.16b }, v25.16b + tbl v18.16b, { v18.16b }, v26.16b + tbl v19.16b, { v19.16b }, v27.16b + st1 { v16.8h }, [x7] + add x7, x7, x12, lsl #1 + st1 { v17.8h }, [x7] + add x7, x7, x13, lsl #1 + st1 { v18.8h }, [x7] + add x7, x7, x14, lsl #1 + st1 { v19.8h }, [x7] + add x7, x7, x15, lsl #1 + add x12, x12, x13 + add x14, x14, x15 + add x9, x9, x12 + add x9, x9, x14 + cmp x2, #0x30 + b.hs Lrej_uniform_loop48 + +Lrej_uniform_loop48_end: + cmp x9, x4 + b.hs Lrej_uniform_memory_copy + cmp x2, #0x18 + b.lo Lrej_uniform_memory_copy + ld3 { v0.8b, v1.8b, v2.8b }, [x1], #24 + zip1 v4.16b, v0.16b, v1.16b + zip1 v5.16b, v1.16b, v2.16b + bic v4.8h, #0xf0, lsl #8 + ushr v5.8h, v5.8h, #0x4 + zip1 v16.8h, v4.8h, v5.8h + zip2 v17.8h, v4.8h, v5.8h + cmhi v4.8h, v30.8h, v16.8h + cmhi v5.8h, v30.8h, v17.8h + and v4.16b, v4.16b, v31.16b + and v5.16b, v5.16b, v31.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + fmov w12, s20 + fmov w13, s21 + ldr q24, [x3, x12, lsl #4] + ldr q25, [x3, x13, lsl #4] + cnt v4.16b, v4.16b + cnt v5.16b, v5.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + fmov w12, s20 + fmov w13, s21 + tbl v16.16b, { v16.16b }, v24.16b + tbl v17.16b, { v17.16b }, v25.16b + st1 { v16.8h }, [x7] + add x7, x7, x12, lsl #1 + st1 { v17.8h }, [x7] + add x9, x9, x12 + add x9, x9, x13 + +Lrej_uniform_memory_copy: + cmp x9, x4 + csel x9, x9, x4, lo + mov x11, #0x0 // =0 + mov x7, x8 + +Lrej_uniform_final_copy: + ldr q16, [x7], #0x40 + ldur q17, [x7, #-0x30] + ldur q18, [x7, #-0x20] + ldur q19, [x7, #-0x10] + str q16, [x0], #0x40 + stur q17, [x0, #-0x30] + stur q18, [x0, #-0x20] + stur q19, [x0, #-0x10] + add x11, x11, #0x20 + cmp x11, #0x100 + b.lt Lrej_uniform_final_copy + mov x0, x9 + +Lrej_uniform_return: + add sp, sp, #0x240 + .cfi_adjust_cfa_offset -0x240 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(rej_uniform_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_table.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_table.c new file mode 100644 index 0000000..e1f48dc --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/aarch64/src/rej_uniform_table.c @@ -0,0 +1,543 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_aarch64.h" + +/* + * Lookup table used by rejection sampling of the public matrix. + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_rej_uniform_table[4096] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 0 */, + 0, 1, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 1 */, + 2, 3, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 2 */, + 0, 1, 2, 3, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 3 */, + 4, 5, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 4 */, + 0, 1, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 5 */, + 2, 3, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 6 */, + 0, 1, 2, 3, 4, 5, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 7 */, + 6, 7, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 8 */, + 0, 1, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 9 */, + 2, 3, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 10 */, + 0, 1, 2, 3, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 11 */, + 4, 5, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 12 */, + 0, 1, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 13 */, + 2, 3, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 14 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 255, 255, 255, 255, 255, 255, 255, 255 /* 15 */, + 8, 9, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 16 */, + 0, 1, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 17 */, + 2, 3, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 18 */, + 0, 1, 2, 3, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 19 */, + 4, 5, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 20 */, + 0, 1, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 21 */, + 2, 3, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 22 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 23 */, + 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 24 */, + 0, 1, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 25 */, + 2, 3, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 26 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 27 */, + 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 28 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 29 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 30 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 255, 255, 255, 255, 255, 255 /* 31 */, + 10, 11, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 32 */, + 0, 1, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 33 */, + 2, 3, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 34 */, + 0, 1, 2, 3, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 35 */, + 4, 5, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 36 */, + 0, 1, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 37 */, + 2, 3, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 38 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 39 */, + 6, 7, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 40 */, + 0, 1, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 41 */, + 2, 3, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 42 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 43 */, + 4, 5, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 44 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 45 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 46 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 255, 255, 255, 255, 255, 255 /* 47 */, + 8, 9, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 48 */, + 0, 1, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 49 */, + 2, 3, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 50 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 51 */, + 4, 5, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 52 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 53 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 54 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 55 */, + 6, 7, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 56 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 57 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 58 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 59 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 60 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 61 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 62 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 255, 255, 255, 255 /* 63 */, + 12, 13, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 64 */, + 0, 1, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 65 */, + 2, 3, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 66 */, + 0, 1, 2, 3, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 67 */, + 4, 5, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 68 */, + 0, 1, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 69 */, + 2, 3, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 70 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 71 */, + 6, 7, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 72 */, + 0, 1, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 73 */, + 2, 3, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 74 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 75 */, + 4, 5, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 76 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 77 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 78 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 255, 255, 255, 255, 255, 255 /* 79 */, + 8, 9, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 80 */, + 0, 1, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 81 */, + 2, 3, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 82 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 83 */, + 4, 5, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 84 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 85 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 86 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 87 */, + 6, 7, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 88 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 89 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 90 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 91 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 92 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 93 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 94 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 255, 255, 255, 255 /* 95 */, + 10, 11, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 96 */, + 0, 1, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 97 */, + 2, 3, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 98 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 99 */, + 4, 5, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 100 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 101 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 102 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 103 */, + 6, 7, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 104 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 105 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 106 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 107 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 108 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 109 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 110 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 255, 255, 255, 255 /* 111 */, + 8, 9, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 112 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 113 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 114 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 115 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 116 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 117 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 118 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 119 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 120 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 121 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 122 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 123 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 124 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 125 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 126 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 255, 255 /* 127 */, + 14, 15, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 128 */, + 0, 1, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 129 */, + 2, 3, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 130 */, + 0, 1, 2, 3, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 131 */, + 4, 5, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 132 */, + 0, 1, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 133 */, + 2, 3, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 134 */, + 0, 1, 2, 3, 4, 5, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 135 */, + 6, 7, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 136 */, + 0, 1, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 137 */, + 2, 3, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 138 */, + 0, 1, 2, 3, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 139 */, + 4, 5, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 140 */, + 0, 1, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 141 */, + 2, 3, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 142 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 14, 15, 255, 255, 255, 255, 255, 255 /* 143 */, + 8, 9, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 144 */, + 0, 1, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 145 */, + 2, 3, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 146 */, + 0, 1, 2, 3, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 147 */, + 4, 5, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 148 */, + 0, 1, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 149 */, + 2, 3, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 150 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 151 */, + 6, 7, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 152 */, + 0, 1, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 153 */, + 2, 3, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 154 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 155 */, + 4, 5, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 156 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 157 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 158 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 14, 15, 255, 255, 255, 255 /* 159 */, + 10, 11, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 160 */, + 0, 1, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 161 */, + 2, 3, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 162 */, + 0, 1, 2, 3, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 163 */, + 4, 5, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 164 */, + 0, 1, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 165 */, + 2, 3, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 166 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 167 */, + 6, 7, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 168 */, + 0, 1, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 169 */, + 2, 3, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 170 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 171 */, + 4, 5, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 172 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 173 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 174 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 255, 255, 255, 255 /* 175 */, + 8, 9, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 176 */, + 0, 1, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 177 */, + 2, 3, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 178 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 179 */, + 4, 5, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 180 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 181 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 182 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 183 */, + 6, 7, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 184 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 185 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 186 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 187 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 188 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 189 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 190 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 255, 255 /* 191 */, + 12, 13, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 192 */, + 0, 1, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 193 */, + 2, 3, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 194 */, + 0, 1, 2, 3, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 195 */, + 4, 5, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 196 */, + 0, 1, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 197 */, + 2, 3, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 198 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 199 */, + 6, 7, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 200 */, + 0, 1, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 201 */, + 2, 3, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 202 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 203 */, + 4, 5, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 204 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 205 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 206 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 255, 255, 255, 255 /* 207 */, + 8, 9, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 208 */, + 0, 1, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 209 */, + 2, 3, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 210 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 211 */, + 4, 5, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 212 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 213 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 214 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 215 */, + 6, 7, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 216 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 217 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 218 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 219 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 220 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 221 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 222 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 255, 255 /* 223 */, + 10, 11, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 224 */, + 0, 1, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 225 */, + 2, 3, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 226 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 227 */, + 4, 5, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 228 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 229 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 230 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 231 */, + 6, 7, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 232 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 233 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 234 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 235 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 236 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 237 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 238 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 255, 255 /* 239 */, + 8, 9, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 240 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 241 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 242 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 243 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 244 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 245 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 246 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 247 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 248 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 249 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 250 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 251 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 252 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 253 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 254 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 /* 255 */, +}; + +#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(aarch64_rej_uniform_table) + +#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/api.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/api.h new file mode 100644 index 0000000..45f267e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/api.h @@ -0,0 +1,638 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_NATIVE_API_H +#define MLK_NATIVE_API_H +/* + * Native arithmetic interface + * + * This header is primarily for documentation purposes. + * It should not be included by backend implementations. + * + * To ensure consistency with backends, the header will be + * included automatically after inclusion of the active + * backend, to ensure consistency of function signatures, + * and run sanity checks. + */ + +#include "../cbmc.h" +#include "../common.h" + +/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */ +#define MLK_NATIVE_FUNC_SUCCESS (0) +/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that + * the target/parameters are unsupported; typically, this would be because of + * dependencies on CPU features not detected on the host CPU. In this case, + * the frontend falls back to the default C implementation. */ +#define MLK_NATIVE_FUNC_FALLBACK (-1) + + +/* Absolute exclusive upper bound for the output of the inverse NTT + * + * NOTE: This is the same bound as in poly.h and has to be kept + * in sync. */ +#define MLK_INVNTT_BOUND (8 * MLKEM_Q) + +/* Absolute exclusive upper bound for the output of the forward NTT + * + * NOTE: This is the same bound as in poly.h and has to be kept + * in sync. */ +#define MLK_NTT_BOUND (8 * MLKEM_Q) + +/* + * This is the C<->native interface allowing for the drop-in of + * native code for performance critical arithmetic components of ML-KEM. + * + * A _backend_ is a specific implementation of (part of) this interface. + * + * To add a function to a backend, define MLK_USE_NATIVE_XXX and + * implement `static inline xxx(...)` in the profile header. + * + * The only exception is MLK_USE_NATIVE_NTT_CUSTOM_ORDER. This option can + * be set if there are native implementations for all of NTT, invNTT, and + * base multiplication, and allows the native implementation to use a + * custom order of polynomial coefficients in NTT domain -- the use of such + * custom order is not an implementation-detail since the public matrix + * is generated in NTT domain. In this case, a permutation function + * mlk_poly_permute_bitrev_to_custom() needs to be provided that permutes + * polynomials in NTT domain from bitreversed to the custom order. + */ + +/* + * Those functions are meant to be trivial wrappers around the chosen native + * implementation. The are static inline to avoid unnecessary calls. + * The macro before each declaration controls whether a native + * implementation is present. + */ + +#if defined(MLK_USE_NATIVE_NTT) +/** + * Compute the negacyclic number-theoretic transform (NTT) of a polynomial + * in place. + * + * The input polynomial is assumed to be in normal order. The output + * polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_NTT_BOUND)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_NTT */ + +#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER) +/* + * This must only be set if NTT, invNTT, basemul, mulcache, and + * to/from byte stream conversions all have native implementations + * that are adapted to the custom order. + */ +#if !defined(MLK_USE_NATIVE_NTT) || !defined(MLK_USE_NATIVE_INTT) || \ + !defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) || \ + !defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) || \ + !defined(MLK_USE_NATIVE_POLY_TOBYTES) || \ + !defined(MLK_USE_NATIVE_POLY_FROMBYTES) +#error \ + "Invalid native profile: MLK_USE_NATIVE_NTT_CUSTOM_ORDER can only be \ +set if there are native implementations for NTT, invNTT, mulcache, basemul, \ +and to/from bytes conversions." +#endif /* !MLK_USE_NATIVE_NTT || !MLK_USE_NATIVE_INTT || \ + !MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE || \ + !MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED || \ + !MLK_USE_NATIVE_POLY_TOBYTES || !MLK_USE_NATIVE_POLY_FROMBYTES */ + +/** + * When MLK_USE_NATIVE_NTT_CUSTOM_ORDER is defined, convert a polynomial in + * NTT domain from bitreversed order to the custom order output by the + * native NTT. + * + * This must only be defined if there is native code for all of (a) NTT, + * (b) invNTT, (c) basemul, (d) mulcache. + * + * @param[in,out] p Input/output polynomial. + */ +static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t p[MLKEM_N]) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ + +#if defined(MLK_USE_NATIVE_INTT) +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place. + * + * The input polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. The output + * polynomial is assumed to be in normal order. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_INVNTT_BOUND)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_INTT */ + +#if defined(MLK_USE_NATIVE_POLY_REDUCE) +/** + * Apply modular reduction to all coefficients of a polynomial, mapping them + * to unsigned canonical representatives in [0,..,MLKEM_Q-1]. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_POLY_REDUCE */ + +#if defined(MLK_USE_NATIVE_POLY_TOMONT) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_POLY_TOMONT */ + +#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) +/** + * Compute multiplication cache for a polynomial in NTT domain. + * + * The purpose of the multiplication cache is to cache repeated computations + * required during a base multiplication of polynomials in NTT domain. The + * structure of the multiplication-cache is implementation defined. + * + * @param[out] cache Multiplication cache. + * @param[in] mlk_poly Input polynomial. Must be in NTT domain and in + * bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native( + int16_t cache[MLKEM_N / 2], const int16_t mlk_poly[MLKEM_N]) +__contract__( + requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q)) +); +#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */ + +#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +/** + * Compute scalar product of length-2 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 2 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +/** + * Compute scalar product of length-3 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 3 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +/** + * Compute scalar product of length-4 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 4 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */ + +#if defined(MLK_USE_NATIVE_POLY_TOBYTES) +/** + * Serialization of a polynomial with unsigned canonical coefficients. + * + * @spec{Implements ByteEncode_12 from @[FIPS203, Algorithm 5].} + * + * @param[out] r Output byte array (of MLKEM_POLYBYTES bytes). + * @param[in] a Input polynomial, with each coefficient in [0,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) +); +#endif /* MLK_USE_NATIVE_POLY_TOBYTES */ + +#if defined(MLK_USE_NATIVE_POLY_FROMBYTES) +/** + * Deserialization of a polynomial. + * + * @spec{Implements ByteDecode_12 from @[FIPS203, Algorithm 6].} + * + * @param[out] a Output polynomial in NTT domain. + * @param[in] r Input byte array (of MLKEM_POLYBYTES bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_frombytes_native( + int16_t a[MLKEM_N], const uint8_t r[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(a, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); +#endif /* MLK_USE_NATIVE_POLY_FROMBYTES */ + +#if defined(MLK_USE_NATIVE_REJ_UNIFORM) +/** + * Run rejection sampling on uniformly random bytes to generate uniformly random + * integers mod MLKEM_Q, represented in [0,..,MLKEM_Q-1]. + * + * @param[out] r Output buffer. + * @param len Requested number of 16-bit integers (uniform mod MLKEM_Q). + * @param[in] buf Input buffer (assumed to be uniform random bytes). + * @param buflen Length of input buffer in bytes. + * + * @retval MLK_NATIVE_FUNC_FALLBACK Native implementation does not support + * the input lengths. + * @retval other Non-negative number of sampled 16-bit + * integers (at most @p len). + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +__contract__( + requires(len <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * len)) + requires(memory_no_alias(buf, buflen)) + assigns(memory_slice(r, sizeof(int16_t) * len)) + ensures(return_value != MLK_NATIVE_FUNC_FALLBACK + ==> (0 <= return_value && return_value <= len)) + ensures(return_value != MLK_NATIVE_FUNC_FALLBACK + ==> array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); +#endif /* MLK_USE_NATIVE_REJ_UNIFORM */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4) +/** + * Compression (4 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_4 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d4_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */ + +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10) +/** + * Compression (10 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_10 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d10_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4) +/** + * De-serialization and subsequent decompression (4 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d4. + * + * @spec{Decompress_4 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d4_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10) +/** + * De-serialization and subsequent decompression (10 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d10. + * + * @spec{Decompress_10 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d10_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5) +/** + * Compression (5 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_5 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d5_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */ + +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11) +/** + * Compression (11 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_11 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d11_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5) +/** + * De-serialization and subsequent decompression (5 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d5. + * + * @spec{Decompress_5 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d5_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11) +/** + * De-serialization and subsequent decompression (11 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d11. + * + * @spec{Decompress_11 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d11_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#endif /* !MLK_NATIVE_API_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/meta.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/meta.h new file mode 100644 index 0000000..97db18a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/meta.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_META_H +#define MLK_NATIVE_META_H + +/* + * Default arithmetic backend + */ +#include "../sys.h" + +#ifdef MLK_SYS_AARCH64 +#include "aarch64/meta.h" +#endif + +/* The x86_64 backend requires toolchain support for the SysV ABI */ +#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED) +#include "x86_64/meta.h" +#endif + +#if defined(MLK_SYS_RISCV64_RVV) +#include "riscv64/meta.h" +#endif + +#ifdef MLK_SYS_PPC64LE +#include "ppc64le/meta.h" +#endif + +#endif /* !MLK_NATIVE_META_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/README.md b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/README.md new file mode 100644 index 0000000..733e32e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/README.md @@ -0,0 +1,6 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# ppc64le backend (little endian) + +This directory contains a native backend for little endian POWER 9 (ppc64le) and above systems. +Or, Power systems supporting ISA 2.07 and above. diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/meta.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/meta.h new file mode 100644 index 0000000..9cd3b66 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/meta.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_PPC64LE_META_H +#define MLK_NATIVE_PPC64LE_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_PPC64LE_DEFAULT + +#define MLK_ARITH_BACKEND_NAME PPC64LE_DEFAULT + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT + +#if !defined(__ASSEMBLER__) +#include +#include "../../common.h" +#include "../../params.h" +#include "../api.h" +#include "src/arith_native_ppc64le.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_ntt_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_intt_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_reduce_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_poly_tomont_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_PPC64LE_META_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/arith_native_ppc64le.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/arith_native_ppc64le.h new file mode 100644 index 0000000..3bd47eb --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/arith_native_ppc64le.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#define MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H + +#include +#include "../../../common.h" +#include "consts.h" + +#define mlk_ntt_ppc_asm MLK_NAMESPACE(ntt_ppc_asm) +void mlk_ntt_ppc_asm(int16_t *, const int16_t *); + +#define mlk_intt_ppc_asm MLK_NAMESPACE(intt_ppc_asm) +void mlk_intt_ppc_asm(int16_t *, const int16_t *); + +#define mlk_reduce_ppc_asm MLK_NAMESPACE(reduce_ppc_asm) +void mlk_reduce_ppc_asm(int16_t *r, const int16_t *); + +#define mlk_poly_tomont_ppc_asm MLK_NAMESPACE(poly_tomont_ppc_asm) +void mlk_poly_tomont_ppc_asm(int16_t *, const int16_t *); + +#endif /* !MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.c new file mode 100644 index 0000000..f6e1abd --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.c @@ -0,0 +1,299 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) + +#include "consts.h" + +/* + * Constant table for the PPC64LE NTT/invNTT. + * For the layout, see the offset macros in consts.h. + */ +/* clang-format off */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_ppc_qdata[2072] = { + /* -Q */ + -3329, -3329, -3329, -3329, -3329, -3329, -3329, -3329, + /* Q */ + 3329, 3329, 3329, 3329, 3329, 3329, 3329, 3329, + /* round(2^26 / MLKEM_Q), Barrett reduction constant */ + 20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159, + /* N^-1 in Montgomery form */ + 512, 512, 512, 512, 512, 512, 512, 512, + /* Barrett twist of N^-1 */ + 5040, 5040, 5040, 5040, 5040, 5040, 5040, 5040, + /* 2^16 mod MLKEM_Q (R), Barrett tomont factor */ + -1044, -1044, -1044, -1044, -1044, -1044, -1044, -1044, + /* Barrett twist of R for poly_tomont */ + -10276, -10276, -10276, -10276, -10276, -10276, -10276, -10276, + /* Twiddle factors for the forward NTT */ + -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600, + -749, -749, -749, -749, -749, -749, -749, -749, + -40, -40, -40, -40, -40, -40, -40, -40, + -687, -687, -687, -687, -687, -687, -687, -687, + 630, 630, 630, 630, 630, 630, 630, 630, + -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432, + 848, 848, 848, 848, 848, 848, 848, 848, + 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410, + 193, 193, 193, 193, 193, 193, 193, 193, + 797, 797, 797, 797, 797, 797, 797, 797, + -543, -543, -543, -543, -543, -543, -543, -543, + -69, -69, -69, -69, -69, -69, -69, -69, + 569, 569, 569, 569, 569, 569, 569, 569, + -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583, + 296, 296, 296, 296, 296, 296, 296, 296, + -882, -882, -882, -882, -882, -882, -882, -882, + 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339, + 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, + -283, -283, -283, -283, -283, -283, -283, -283, + 56, 56, 56, 56, 56, 56, 56, 56, + -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089, + 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, + 1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426, + -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235, + 535, 535, 535, 535, 535, 535, 535, 535, + -447, -447, -447, -447, -447, -447, -447, -447, + -936, -936, -936, -936, -936, -936, -936, -936, + -450, -450, -450, -450, -450, -450, -450, -450, + -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355, + 821, 821, 821, 821, 821, 821, 821, 821, + 331, 331, 331, 331, 289, 289, 289, 289, + -1573, -1573, -1573, -1573, -76, -76, -76, -76, + -1025, -1025, -1025, -1025, 1197, 1197, 1197, 1197, + -1274, -1274, -1274, -1274, -1052, -1052, -1052, -1052, + -1352, -1352, -1352, -1352, 650, 650, 650, 650, + 632, 632, 632, 632, -816, -816, -816, -816, + 33, 33, 33, 33, -464, -464, -464, -464, + -1414, -1414, -1414, -1414, 1320, 1320, 1320, 1320, + 1435, 1435, 1435, 1435, -1010, -1010, -1010, -1010, + 452, 452, 452, 452, 807, 807, 807, 807, + -461, -461, -461, -461, 1438, 1438, 1438, 1438, + -927, -927, -927, -927, 1534, 1534, 1534, 1534, + -712, -712, -712, -712, -682, -682, -682, -682, + 648, 648, 648, 648, 1481, 1481, 1481, 1481, + -219, -219, -219, -219, -855, -855, -855, -855, + 910, 910, 910, 910, 1227, 1227, 1227, 1227, + 583, 583, 17, 17, -680, -680, -568, -568, + -1041, -1041, 1637, 1637, 1100, 1100, 723, 723, + -48, -48, 1409, 1409, 233, 233, -667, -667, + -314, -314, 756, 756, -279, -279, -1173, -1173, + -540, -540, -1626, -1626, -1540, -1540, 1651, 1651, + 1461, 1461, -1482, -1482, -642, -642, 952, 952, + -892, -892, 939, 939, -941, -941, -1021, -1021, + 268, 268, 733, 733, 641, 641, -992, -992, + -1292, -1292, 1584, 1584, -109, -109, -1031, -1031, + -1239, -1239, 375, 375, 1645, 1645, -780, -780, + -556, -556, 1063, 1063, 757, 757, 319, 319, + -863, -863, -1230, -1230, -735, -735, 561, 561, + 403, 403, -525, -525, 1026, 1026, 1092, 1092, + -554, -554, 1143, 1143, 886, 886, -1179, -1179, + -1455, -1455, -1607, -1607, 1029, 1029, 1212, 1212, + 885, 885, -1219, -1219, -1175, -1175, -394, -394, + /* Twiddle factors for the inverse NTT */ + -394, -394, -1175, -1175, -1219, -1219, 885, 885, + 1212, 1212, 1029, 1029, -1607, -1607, -1455, -1455, + -1179, -1179, 886, 886, 1143, 1143, -554, -554, + 1092, 1092, 1026, 1026, -525, -525, 403, 403, + 561, 561, -735, -735, -1230, -1230, -863, -863, + 319, 319, 757, 757, 1063, 1063, -556, -556, + -780, -780, 1645, 1645, 375, 375, -1239, -1239, + -1031, -1031, -109, -109, 1584, 1584, -1292, -1292, + -992, -992, 641, 641, 733, 733, 268, 268, + -1021, -1021, -941, -941, 939, 939, -892, -892, + 952, 952, -642, -642, -1482, -1482, 1461, 1461, + 1651, 1651, -1540, -1540, -1626, -1626, -540, -540, + -1173, -1173, -279, -279, 756, 756, -314, -314, + -667, -667, 233, 233, 1409, 1409, -48, -48, + 723, 723, 1100, 1100, 1637, 1637, -1041, -1041, + -568, -568, -680, -680, 17, 17, 583, 583, + 1227, 1227, 1227, 1227, 910, 910, 910, 910, + -855, -855, -855, -855, -219, -219, -219, -219, + 1481, 1481, 1481, 1481, 648, 648, 648, 648, + -682, -682, -682, -682, -712, -712, -712, -712, + 1534, 1534, 1534, 1534, -927, -927, -927, -927, + 1438, 1438, 1438, 1438, -461, -461, -461, -461, + 807, 807, 807, 807, 452, 452, 452, 452, + -1010, -1010, -1010, -1010, 1435, 1435, 1435, 1435, + 1320, 1320, 1320, 1320, -1414, -1414, -1414, -1414, + -464, -464, -464, -464, 33, 33, 33, 33, + -816, -816, -816, -816, 632, 632, 632, 632, + 650, 650, 650, 650, -1352, -1352, -1352, -1352, + -1052, -1052, -1052, -1052, -1274, -1274, -1274, -1274, + 1197, 1197, 1197, 1197, -1025, -1025, -1025, -1025, + -76, -76, -76, -76, -1573, -1573, -1573, -1573, + 289, 289, 289, 289, 331, 331, 331, 331, + 821, 821, 821, 821, 821, 821, 821, 821, + -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355, + -450, -450, -450, -450, -450, -450, -450, -450, + -936, -936, -936, -936, -936, -936, -936, -936, + -447, -447, -447, -447, -447, -447, -447, -447, + 535, 535, 535, 535, 535, 535, 535, 535, + -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235, + 1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426, + 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, + -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089, + 56, 56, 56, 56, 56, 56, 56, 56, + -283, -283, -283, -283, -283, -283, -283, -283, + 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, + 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339, + -882, -882, -882, -882, -882, -882, -882, -882, + 296, 296, 296, 296, 296, 296, 296, 296, + -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583, + 569, 569, 569, 569, 569, 569, 569, 569, + -69, -69, -69, -69, -69, -69, -69, -69, + -543, -543, -543, -543, -543, -543, -543, -543, + 797, 797, 797, 797, 797, 797, 797, 797, + 193, 193, 193, 193, 193, 193, 193, 193, + -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410, + 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + 848, 848, 848, 848, 848, 848, 848, 848, + -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432, + 630, 630, 630, 630, 630, 630, 630, 630, + -687, -687, -687, -687, -687, -687, -687, -687, + -40, -40, -40, -40, -40, -40, -40, -40, + -749, -749, -749, -749, -749, -749, -749, -749, + -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600, + /* Twisted twiddle factors for the forward NTT */ + -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749, + -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373, + -394, -394, -394, -394, -394, -394, -394, -394, + -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762, + 6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201, + -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095, + 8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347, + 10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453, + -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879, + 1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900, + 7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845, + -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345, + -679, -679, -679, -679, -679, -679, -679, -679, + 5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601, + -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582, + 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, + -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682, + 13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180, + 14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529, + -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786, + 551, 551, 551, 551, 551, 551, 551, 551, + -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719, + 13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121, + 14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036, + -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156, + 5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266, + -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400, + -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213, + -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429, + -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338, + 8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081, + 3258, 3258, 3258, 3258, 2845, 2845, 2845, 2845, + -15483, -15483, -15483, -15483, -748, -748, -748, -748, + -10089, -10089, -10089, -10089, 11782, 11782, 11782, 11782, + -12540, -12540, -12540, -12540, -10355, -10355, -10355, -10355, + -13308, -13308, -13308, -13308, 6398, 6398, 6398, 6398, + 6221, 6221, 6221, 6221, -8032, -8032, -8032, -8032, + 325, 325, 325, 325, -4567, -4567, -4567, -4567, + -13918, -13918, -13918, -13918, 12993, 12993, 12993, 12993, + 14125, 14125, 14125, 14125, -9942, -9942, -9942, -9942, + 4449, 4449, 4449, 4449, 7943, 7943, 7943, 7943, + -4538, -4538, -4538, -4538, 14155, 14155, 14155, 14155, + -9125, -9125, -9125, -9125, 15099, 15099, 15099, 15099, + -7008, -7008, -7008, -7008, -6713, -6713, -6713, -6713, + 6378, 6378, 6378, 6378, 14578, 14578, 14578, 14578, + -2156, -2156, -2156, -2156, -8416, -8416, -8416, -8416, + 8957, 8957, 8957, 8957, 12078, 12078, 12078, 12078, + 5739, 5739, 167, 167, -6693, -6693, -5591, -5591, + -10247, -10247, 16113, 16113, 10828, 10828, 7117, 7117, + -472, -472, 13869, 13869, 2293, 2293, -6565, -6565, + -3091, -3091, 7441, 7441, -2746, -2746, -11546, -11546, + -5315, -5315, -16005, -16005, -15159, -15159, 16251, 16251, + 14381, 14381, -14588, -14588, -6319, -6319, 9371, 9371, + -8780, -8780, 9243, 9243, -9262, -9262, -10050, -10050, + 2638, 2638, 7215, 7215, 6309, 6309, -9764, -9764, + -12717, -12717, 15592, 15592, -1073, -1073, -10148, -10148, + -12196, -12196, 3691, 3691, 16192, 16192, -7678, -7678, + -5473, -5473, 10463, 10463, 7451, 7451, 3140, 3140, + -8495, -8495, -12107, -12107, -7235, -7235, 5522, 5522, + 3967, 3967, -5168, -5168, 10099, 10099, 10749, 10749, + -5453, -5453, 11251, 11251, 8721, 8721, -11605, -11605, + -14322, -14322, -15818, -15818, 10129, 10129, 11930, 11930, + 8711, 8711, -11999, -11999, -11566, -11566, -3878, -3878, + /* Twisted twiddle factors for the inverse NTT */ + -3878, -3878, -11566, -11566, -11999, -11999, 8711, 8711, + 11930, 11930, 10129, 10129, -15818, -15818, -14322, -14322, + -11605, -11605, 8721, 8721, 11251, 11251, -5453, -5453, + 10749, 10749, 10099, 10099, -5168, -5168, 3967, 3967, + 5522, 5522, -7235, -7235, -12107, -12107, -8495, -8495, + 3140, 3140, 7451, 7451, 10463, 10463, -5473, -5473, + -7678, -7678, 16192, 16192, 3691, 3691, -12196, -12196, + -10148, -10148, -1073, -1073, 15592, 15592, -12717, -12717, + -9764, -9764, 6309, 6309, 7215, 7215, 2638, 2638, + -10050, -10050, -9262, -9262, 9243, 9243, -8780, -8780, + 9371, 9371, -6319, -6319, -14588, -14588, 14381, 14381, + 16251, 16251, -15159, -15159, -16005, -16005, -5315, -5315, + -11546, -11546, -2746, -2746, 7441, 7441, -3091, -3091, + -6565, -6565, 2293, 2293, 13869, 13869, -472, -472, + 7117, 7117, 10828, 10828, 16113, 16113, -10247, -10247, + -5591, -5591, -6693, -6693, 167, 167, 5739, 5739, + 12078, 12078, 12078, 12078, 8957, 8957, 8957, 8957, + -8416, -8416, -8416, -8416, -2156, -2156, -2156, -2156, + 14578, 14578, 14578, 14578, 6378, 6378, 6378, 6378, + -6713, -6713, -6713, -6713, -7008, -7008, -7008, -7008, + 15099, 15099, 15099, 15099, -9125, -9125, -9125, -9125, + 14155, 14155, 14155, 14155, -4538, -4538, -4538, -4538, + 7943, 7943, 7943, 7943, 4449, 4449, 4449, 4449, + -9942, -9942, -9942, -9942, 14125, 14125, 14125, 14125, + 12993, 12993, 12993, 12993, -13918, -13918, -13918, -13918, + -4567, -4567, -4567, -4567, 325, 325, 325, 325, + -8032, -8032, -8032, -8032, 6221, 6221, 6221, 6221, + 6398, 6398, 6398, 6398, -13308, -13308, -13308, -13308, + -10355, -10355, -10355, -10355, -12540, -12540, -12540, -12540, + 11782, 11782, 11782, 11782, -10089, -10089, -10089, -10089, + -748, -748, -748, -748, -15483, -15483, -15483, -15483, + 2845, 2845, 2845, 2845, 3258, 3258, 3258, 3258, + 8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081, + -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338, + -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429, + -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213, + -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400, + 5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266, + -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156, + 14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036, + 13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121, + -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719, + 551, 551, 551, 551, 551, 551, 551, 551, + -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786, + 14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529, + 13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180, + -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682, + 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, + -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582, + 5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601, + -679, -679, -679, -679, -679, -679, -679, -679, + -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345, + 7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845, + 1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900, + -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879, + 10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453, + 8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347, + -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095, + 6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201, + -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762, + -394, -394, -394, -394, -394, -394, -394, -394, + -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373, + -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749, +}; +/* clang-format on */ + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED && __POWER8_VECTOR__ */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.h new file mode 100644 index 0000000..acac168 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/consts.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#define MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#include "../../../common.h" + +/* Byte offsets into the constant table */ +#define MLK_PPC_NQ_OFFSET 0 +#define MLK_PPC_Q_OFFSET 16 +#define MLK_PPC_C20159_OFFSET 32 +#define MLK_PPC_N_INV_OFFSET 48 +#define MLK_PPC_N_INV_TW_OFFSET 64 +#define MLK_PPC_TOMONT_OFFSET 80 +#define MLK_PPC_TOMONT_TW_OFFSET 96 +#define MLK_PPC_ZETA_NTT_OFFSET 112 +#define MLK_PPC_ZETA_INTT_OFFSET 1120 +#define MLK_PPC_ZETA_NTT_TW_OFFSET 2128 +#define MLK_PPC_ZETA_INTT_TW_OFFSET 3136 + +#ifndef __ASSEMBLER__ +#define mlk_ppc_qdata MLK_NAMESPACE(ppc_qdata) +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_ppc_qdata[2072]; +#endif + +#endif /* !MLK_NATIVE_PPC64LE_SRC_CONSTS_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/intt_ppc_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/intt_ppc_asm.S new file mode 100644 index 0000000..6b56be3 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/intt_ppc_asm.S @@ -0,0 +1,3243 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: intt_ppc_asm + Description: PowerPC64 VSX inverse NTT + Signature: void mlk_intt_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/intt_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(intt_ppc_asm) +MLK_ASM_FN_SYMBOL(intt_ppc_asm) + + stdu 1, -352(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + stxvd2x 52, 10, 1 + stxvd2x 53, 11, 1 + stxvd2x 54, 12, 1 + stxvd2x 55, 14, 1 + stxvd2x 56, 15, 1 + stxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + stxvd2x 58, 10, 1 + stxvd2x 59, 11, 1 + stxvd2x 60, 12, 1 + stxvd2x 61, 14, 1 + stxvd2x 62, 15, 1 + stxvd2x 63, 16, 1 + lxvd2x 0, 0, 4 + xxlxor 35, 35, 35 + xxlor 3, 35, 35 + li 10, 16 + li 11, 32 + lxvd2x 6, 10, 4 + lxvd2x 32, 11, 4 + vspltisw 8, 13 + vadduwm 8, 8, 8 + xxlor 8, 40, 40 + vspltisw 9, 1 + vsubuwm 10, 8, 9 + vslw 9, 9, 10 + xxlor 7, 41, 41 + li 10, 16 + li 11, 32 + li 12, 48 + li 15, 64 + li 16, 80 + li 17, 96 + li 18, 112 + addi 14, 4, 48 + lvx 10, 0, 14 + addi 14, 4, 64 + lvx 11, 0, 14 + li 8, 4 + mtctr 8 + xxlor 37, 0, 0 + +intt_ppc_asm_Loopf: + lxvd2x 57, 0, 3 + lxvd2x 58, 10, 3 + lxvd2x 62, 11, 3 + lxvd2x 63, 12, 3 + addi 3, 3, 64 + vmhraddshs 14, 25, 11, 3 + vmhraddshs 19, 26, 11, 3 + vmhraddshs 24, 30, 11, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 6, 25, 10, 3 + vmladduhm 7, 26, 10, 3 + vmladduhm 8, 30, 10, 3 + vmladduhm 9, 31, 10, 3 + vmladduhm 6, 14, 5, 6 + vmladduhm 7, 19, 5, 7 + vmladduhm 8, 24, 5, 8 + vmladduhm 9, 29, 5, 9 + lxvd2x 57, 0, 3 + lxvd2x 58, 10, 3 + lxvd2x 62, 11, 3 + lxvd2x 63, 12, 3 + addi 3, 3, 64 + vmhraddshs 14, 25, 11, 3 + vmhraddshs 19, 26, 11, 3 + vmhraddshs 24, 30, 11, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + addi 3, 3, -128 + stxvd2x 38, 0, 3 + stxvd2x 39, 10, 3 + stxvd2x 40, 11, 3 + stxvd2x 41, 12, 3 + stxvd2x 45, 15, 3 + stxvd2x 50, 16, 3 + stxvd2x 55, 17, 3 + stxvd2x 60, 18, 3 + addi 3, 3, 128 + bdnz intt_ppc_asm_Loopf + addi 3, 3, -512 + nop + nop + ori 2, 2, 0 + addi 14, 4, 1120 + addi 22, 4, 3136 + li 7, 4 + mr 5, 3 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + mr 5, 3 + li 7, 8 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + nop + ori 2, 2, 0 + li 7, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 32 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, -64 + addi 22, 22, -64 + li 9, 16 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, -64 + addi 22, 22, -64 + li 9, 272 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + nop + nop + ori 2, 2, 0 + li 7, 128 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 320 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + nop + nop + ori 2, 2, 0 + li 7, 256 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 192 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + lxvd2x 52, 10, 1 + lxvd2x 53, 11, 1 + lxvd2x 54, 12, 1 + lxvd2x 55, 14, 1 + lxvd2x 56, 15, 1 + lxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + lxvd2x 58, 10, 1 + lxvd2x 59, 11, 1 + lxvd2x 60, 12, 1 + lxvd2x 61, 14, 1 + lxvd2x 62, 15, 1 + lxvd2x 63, 16, 1 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + mtlr 0 + addi 1, 1, 352 + blr + +MLK_ASM_FN_SIZE(intt_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/ntt_ppc_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/ntt_ppc_asm.S new file mode 100644 index 0000000..b68d380 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/ntt_ppc_asm.S @@ -0,0 +1,1672 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: ntt_ppc_asm + Description: PowerPC64 VSX forward NTT + Signature: void mlk_ntt_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/ntt_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_ppc_asm) +MLK_ASM_FN_SYMBOL(ntt_ppc_asm) + + stdu 1, -352(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + stxvd2x 52, 10, 1 + stxvd2x 53, 11, 1 + stxvd2x 54, 12, 1 + stxvd2x 55, 14, 1 + stxvd2x 56, 15, 1 + stxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + stxvd2x 58, 10, 1 + stxvd2x 59, 11, 1 + stxvd2x 60, 12, 1 + stxvd2x 61, 14, 1 + stxvd2x 62, 15, 1 + stxvd2x 63, 16, 1 + lvx 5, 0, 4 + addi 14, 4, 112 + addi 22, 4, 2128 + vxor 3, 3, 3 + ori 2, 2, 0 + li 7, 256 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 192 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 128 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 320 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 64 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 32 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 16 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 272 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + ori 2, 2, 0 + li 7, 16 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + mr 5, 3 + li 7, 8 + li 10, 16 + li 11, 32 + li 12, 48 + li 15, 64 + li 16, 80 + li 17, 96 + li 18, 112 + nop + ori 2, 2, 0 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + mr 5, 3 + li 7, 4 + nop + ori 2, 2, 0 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + lxvd2x 52, 10, 1 + lxvd2x 53, 11, 1 + lxvd2x 54, 12, 1 + lxvd2x 55, 14, 1 + lxvd2x 56, 15, 1 + lxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + lxvd2x 58, 10, 1 + lxvd2x 59, 11, 1 + lxvd2x 60, 12, 1 + lxvd2x 61, 14, 1 + lxvd2x 62, 15, 1 + lxvd2x 63, 16, 1 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + mtlr 0 + addi 1, 1, 352 + blr + +MLK_ASM_FN_SIZE(ntt_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/poly_tomont_ppc_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/poly_tomont_ppc_asm.S new file mode 100644 index 0000000..4f569f4 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/poly_tomont_ppc_asm.S @@ -0,0 +1,315 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +/* + * Poly_tomont: Inplace conversion of all coefficients of a polynomial + * from normal domain to Montgomery domain + * + * Arguments:*r: pointer to input/output polynomial + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: poly_tomont_ppc_asm + Description: PowerPC64 VSX conversion to Montgomery form + Signature: void mlk_poly_tomont_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/poly_tomont_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tomont_ppc_asm) +MLK_ASM_FN_SYMBOL(poly_tomont_ppc_asm) + + stdu 1, -320(1) + mflr 0 + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + li 11, 208 + li 12, 224 + stxvd2x 52, 6, 1 + stxvd2x 53, 7, 1 + stxvd2x 54, 8, 1 + stxvd2x 55, 9, 1 + stxvd2x 56, 10, 1 + stxvd2x 57, 11, 1 + stxvd2x 58, 12, 1 + li 6, 240 + li 7, 256 + li 8, 272 + li 9, 288 + stxvd2x 59, 6, 1 + stxvd2x 60, 7, 1 + stxvd2x 61, 8, 1 + stxvd2x 62, 9, 1 + li 6, 0 + li 7, 80 + li 8, 96 + lxvd2x 37, 6, 4 + lxvd2x 32, 7, 4 + lxvd2x 34, 8, 4 + vxor 3, 3, 3 + li 4, -128 + li 5, -112 + li 6, -96 + li 7, -80 + li 8, -64 + li 9, -48 + li 10, -32 + li 11, -16 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + li 11, 208 + li 12, 224 + lxvd2x 52, 6, 1 + lxvd2x 53, 7, 1 + lxvd2x 54, 8, 1 + lxvd2x 55, 9, 1 + lxvd2x 56, 10, 1 + lxvd2x 57, 11, 1 + lxvd2x 58, 12, 1 + li 6, 240 + li 7, 256 + li 8, 272 + li 9, 288 + lxvd2x 59, 6, 1 + lxvd2x 60, 7, 1 + lxvd2x 61, 8, 1 + lxvd2x 62, 9, 1 + mtlr 0 + addi 1, 1, 320 + blr + +MLK_ASM_FN_SIZE(poly_tomont_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/reduce_ppc_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/reduce_ppc_asm.S new file mode 100644 index 0000000..58fd3ac --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/ppc64le/src/reduce_ppc_asm.S @@ -0,0 +1,731 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: reduce_ppc_asm + Description: PowerPC64 VSX modular reduction + Signature: void mlk_reduce_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/reduce_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(reduce_ppc_asm) +MLK_ASM_FN_SYMBOL(reduce_ppc_asm) + + stdu 1, -224(1) + mflr 0 + std 14, 96(1) + std 15, 104(1) + std 16, 112(1) + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + stxvd2x 52, 6, 1 + stxvd2x 53, 7, 1 + stxvd2x 54, 8, 1 + stxvd2x 55, 9, 1 + stxvd2x 56, 10, 1 + vxor 7, 7, 7 + li 6, 16 + li 7, 32 + lxvd2x 35, 6, 4 + lxvd2x 32, 7, 4 + vspltisw 2, 13 + vadduwm 2, 2, 2 + vspltisw 4, 1 + vsubuwm 5, 2, 4 + vslw 1, 4, 5 + li 4, -128 + li 5, -112 + li 6, -96 + li 7, -80 + li 8, -64 + li 9, -48 + li 10, -32 + li 11, -16 + li 14, 16 + li 15, 32 + li 16, 48 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + addi 3, 3, -512 + vxor 9, 9, 9 + vspltish 10, 15 + vmr 11, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + ld 14, 96(1) + ld 15, 104(1) + ld 16, 112(1) + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + lxvd2x 52, 6, 1 + lxvd2x 53, 7, 1 + lxvd2x 54, 8, 1 + lxvd2x 55, 9, 1 + lxvd2x 56, 10, 1 + mtlr 0 + addi 1, 1, 224 + blr + +MLK_ASM_FN_SIZE(reduce_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/README.md b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/README.md new file mode 100644 index 0000000..56ebe93 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/README.md @@ -0,0 +1,11 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# RISC-V Vector Extension Backend + +This is an arithmetic backend for CPUs implementing the RISC-V Vector Extension. The backend is functional for all physical `VLEN`, but the NTT and inverse NTT are so far only implemented for VLEN=256, falling back to the default C implementations for other VLENs. + +## Requirements + +- RISC-V 64-bit architecture +- Vector extension (RVV) version 1.0 +- Standard "gc" extensions (integer and compressed instructions) diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/meta.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/meta.h new file mode 100644 index 0000000..c1d2be4 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/meta.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_RISCV64_META_H +#define MLK_NATIVE_RISCV64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_RISCV64 + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_REJ_UNIFORM +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED + +#include "../../common.h" + +#if !defined(__ASSEMBLER__) +#include + +#include "../api.h" +#include "src/arith_native_riscv64.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + /* VLEN = 256 only for now */ + if (__riscv_vsetvlmax_e16m1() != 16) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_rv64v_poly_ntt(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + /* VLEN = 256 only for now */ + if (__riscv_vsetvlmax_e16m1() != 16) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_rv64v_poly_invntt_tomont(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + mlk_rv64v_poly_tomont(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + /* mlk_rv64v_rej_uniform requires buflen to be a multiple of 12 */ + if (buflen % 12 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + /* The cast from unsigned to signed integer is safe + * because the return value is <= len, which we asssume + * to be bound by 4096 and hence <= INT_MAX. */ + return (int)mlk_rv64v_rej_uniform(r, len, buf, buflen); +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + mlk_rv64v_poly_reduce(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + (void)x; /* not using the cache at the moment */ + (void)y; + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k2(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k3(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k4(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_RISCV64_META_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/arith_native_riscv64.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/arith_native_riscv64.h new file mode 100644 index 0000000..f21b6da --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/arith_native_riscv64.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#define MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H + +#include "../../../common.h" + +#define mlk_rv64v_poly_ntt MLK_NAMESPACE(ntt_riscv64) +void mlk_rv64v_poly_ntt(int16_t *); + +#define mlk_rv64v_poly_invntt_tomont MLK_NAMESPACE(intt_riscv64) +void mlk_rv64v_poly_invntt_tomont(int16_t *r); + +#define mlk_rv64v_poly_basemul_mont_add_k2 MLK_NAMESPACE(basemul_add_k2_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_basemul_mont_add_k3 MLK_NAMESPACE(basemul_add_k3_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_basemul_mont_add_k4 MLK_NAMESPACE(basemul_add_k4_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_tomont MLK_NAMESPACE(tomont_riscv64) +void mlk_rv64v_poly_tomont(int16_t *r); + +#define mlk_rv64v_poly_reduce MLK_NAMESPACE(reduce_riscv64) +void mlk_rv64v_poly_reduce(int16_t *r); + +#define mlk_rv64v_poly_add MLK_NAMESPACE(poly_add_riscv64) +void mlk_rv64v_poly_add(int16_t *r, const int16_t *a, const int16_t *b); + +#define mlk_rv64v_poly_sub MLK_NAMESPACE(poly_sub_riscv64) +void mlk_rv64v_poly_sub(int16_t *r, const int16_t *a, const int16_t *b); + +#define mlk_rv64v_rej_uniform MLK_NAMESPACE(rj_uniform_riscv64) +MLK_MUST_CHECK_RETURN_VALUE +unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len, + const uint8_t *buf, unsigned int buflen); + +#endif /* !MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.c new file mode 100644 index 0000000..46baa07 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_RISCV64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG) + +#include +#include + +#include "../../../debug.h" +#include "rv64v_debug.h" + +#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] " + +/** + * Check whether values in a vint16m1_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (number of active elements). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + /* Allocate temporary array to store vector elements + * We use the maximum possible vector length to be safe */ + int16_t temp_array[64]; + + /* Store vector elements to temporary array for inspection */ + __riscv_vse16_v_i16m1(temp_array, vec, vl); + + /* Reuse existing array bounds checking function */ + mlk_debug_check_bounds(file, line, temp_array, (unsigned)vl, + lower_bound_exclusive, upper_bound_exclusive); +} + +/** + * Check whether values in a vint16m2_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (active elements per m1 half). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + /* Allocate temporary array to store vector elements + * m2 vectors hold 2x the elements of m1 vectors */ + int16_t temp_array[2 * 64]; + + /* Store vector elements to temporary array for inspection */ + __riscv_vse16_v_i16m2(temp_array, vec, 2 * vl); + + /* Reuse existing array bounds checking function for all elements */ + mlk_debug_check_bounds(file, line, temp_array, (unsigned)(2 * vl), + lower_bound_exclusive, upper_bound_exclusive); +} + +#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + MLKEM_DEBUG */ + +MLK_EMPTY_CU(rv64v_debug) + +#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + MLKEM_DEBUG) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_DEBUG_ERROR_HEADER diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.h new file mode 100644 index 0000000..18d4b12 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_debug.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#define MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H + +#include "../../../debug.h" + +#include + +/************************************************* + * RISC-V Vector Bounds Assertion Macros + * + * These macros provide runtime bounds checking for RISC-V vector types + * vint16m1_t and vint16m2_t, following the same pattern as the scalar + * bounds assertions in debug.h + * + * The macros are only active when MLKEM_DEBUG is defined, otherwise they + * compile to no-ops for zero runtime overhead in release builds. + **************************************************/ + +#if defined(MLKEM_DEBUG) + +/** + * Check whether values in a vint16m1_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (number of active elements). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds_int16m1 \ + MLK_NAMESPACE(mlkem_debug_check_bounds_int16m1) +void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive); + +/** + * Check whether values in a vint16m2_t vector are within specified bounds + * by splitting into m1 vectors. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (active elements per m1 half). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds_int16m2 \ + MLK_NAMESPACE(mlkem_debug_check_bounds_int16m2) +void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive); + +/* Check bounds in vint16m1_t vector + * vec: RISC-V vector of type vint16m1_t + * vl: Vector length (number of active elements) + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + mlk_debug_check_bounds_int16m1(__FILE__, __LINE__, (vec), (vl), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in vint16m1_t vector + * vec: RISC-V vector of type vint16m1_t + * vl: Vector length (number of active elements) + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + mlk_assert_bound_int16m1((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd)) + +/* Check bounds in vint16m2_t vector + * vec: RISC-V vector of type vint16m2_t + * vl: Vector length (number of active elements per m1 half) + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + mlk_debug_check_bounds_int16m2(__FILE__, __LINE__, (vec), (vl), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in vint16m2_t vector + * vec: RISC-V vector of type vint16m2_t + * vl: Vector length (number of active elements per m1 half) + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + mlk_assert_bound_int16m2((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd)) + +#elif defined(CBMC) + +/* For CBMC, we would need to implement vector bounds checking using CBMC + * primitives This is complex and would require extracting vector elements, so + * for now we provide empty implementations that could be extended later */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#else /* !MLKEM_DEBUG && CBMC */ + +/* When debugging is disabled, all assertions become no-ops */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#endif /* !MLKEM_DEBUG && !CBMC */ + +#endif /* !MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_izetas.inc b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_izetas.inc new file mode 100644 index 0000000..e24a9ab --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_izetas.inc @@ -0,0 +1,27 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t izeta[] = { + -1044, 758, 1571, 205, 1275, -677, 1065, -448, -1628, -1522, 1460, + -958, -991, -996, 308, 108, 1517, 359, -411, 1542, 725, 1508, + -961, 398, -478, 870, 854, 1510, -794, 1278, 1530, 1185, -202, + -287, -608, -732, 951, 247, 1421, -107, 1659, 1187, -220, 874, + 1335, -1218, 136, 1215, -1422, -1493, -1017, 681, -830, 271, 90, + 853, -384, 1465, 1285, -1322, -610, -603, -1097, -817, -1468, 1474, + 130, 1602, -1469, -126, 1162, 1618, 75, 156, -329, -418, -349, + 872, -644, 1590, 1202, -962, -1458, 829, 666, 320, 8, -516, + -1119, 602, -1483, 777, 147, -1159, -778, 246, -182, -1577, -383, + -264, 1544, 282, -1491, 1293, -1653, -1574, 460, 291, 235, -177, + -587, -422, -622, 171, 1325, -573, -1015, 552, -652, -1223, -105, + -1550, -871, 1251, -843, -555, -430, 1103, +}; diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_poly.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_poly.c new file mode 100644 index 0000000..af529ee --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_poly.c @@ -0,0 +1,793 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* === ML-KEM NTT using RISC-V Vector intrinstics */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_RISCV64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include + +#include "arith_native_riscv64.h" +#include "rv64v_debug.h" + +/* Montgomery reduction constants */ +/* check-magic: -3327 == signed_mod(pow(MLKEM_Q,-1,2^16), 2^16) */ +#define MLK_RVV_QI -3327 + +/* check-magic: 2285 == unsigned_mod(2^16, MLKEM_Q) */ +#define MLK_RVV_MONT_R1 2285 + +/* check-magic: 1353 == pow(2, 32, MLKEM_Q) */ +#define MLK_RVV_MONT_R2 1353 + +/* check-magic: 1441 == pow(2,32-7,MLKEM_Q) */ +#define MLK_RVV_MONT_NR 1441 + +static MLK_INLINE vint16m1_t fq_redc(vint16m1_t rh, vint16m1_t rl, size_t vl) +{ + vint16m1_t t; + + t = __riscv_vmul_vx_i16m1(rl, MLK_RVV_QI, vl); /* t = l * Q^-1 */ + t = __riscv_vmulh_vx_i16m1(t, MLKEM_Q, vl); /* t = (t*Q) / R */ + t = __riscv_vsub_vv_i16m1(rh, t, vl); /* t = h - t */ + + return t; +} + +/* Narrowing reduction */ + +static MLK_INLINE vint16m1_t fq_redc2(vint32m2_t z, size_t vl) +{ + vint16m1_t t; + + t = __riscv_vmul_vx_i16m1(__riscv_vncvt_x_x_w_i16m1(z, vl), MLK_RVV_QI, + vl); /* t = l * Q^-1 */ + z = __riscv_vsub_vv_i32m2(z, __riscv_vwmul_vx_i32m2(t, MLKEM_Q, vl), + vl); /* x = (x - (t*Q)) */ + t = __riscv_vnsra_wx_i16m1(z, 16, vl); + + return t; +} + +/* Narrowing Barrett */ + +static MLK_INLINE vint16m1_t fq_barrett(vint16m1_t a, size_t vl) +{ + vint16m1_t t; + const int16_t v = ((1 << 26) + MLKEM_Q / 2) / MLKEM_Q; + + t = __riscv_vmulh_vx_i16m1(a, v, vl); + t = __riscv_vadd_vx_i16m1(t, 1 << (25 - 16), vl); + t = __riscv_vsra_vx_i16m1(t, 26 - 16, vl); + t = __riscv_vmul_vx_i16m1(t, MLKEM_Q, vl); + t = __riscv_vsub_vv_i16m1(a, t, vl); + + mlk_assert_abs_bound_int16m1(t, vl, MLKEM_Q_HALF); + return t; +} + +/* Conditionally add Q (if negative). + * + * Constant-time note: We deliberately avoid a `vmslt` + masked `vadd_mu` + * here. RVV 1.0 does not mandate mask-population-independent latency for + * masked ops, and Zvkt's DIEL guarantee does not cover the v0 mask + * register. Using an arithmetic-shift sign mask keeps all operands in the + * Zvkt-covered set and matches the idiom used in the portable C + * (mlk_ct_cmask_neg_i16, verify.h) and AArch64 (poly_reduce_aarch64_asm.S) + * implementations. */ + +static MLK_INLINE vint16m1_t fq_cadd(vint16m1_t rx, size_t vl) +{ + vint16m1_t m; + + m = __riscv_vsra_vx_i16m1(rx, 15, vl); /* m = (x < 0) ? -1 : 0 */ + m = __riscv_vand_vx_i16m1(m, MLKEM_Q, vl); /* m = (x < 0) ? Q : 0 */ + rx = __riscv_vadd_vv_i16m1(rx, m, vl); /* x += m */ + return rx; +} + +/* Montgomery multiply: vector-vector */ + +static MLK_INLINE vint16m1_t fq_mul_vv(vint16m1_t rx, vint16m1_t ry, size_t vl) +{ + vint16m1_t rl, rh; + + rh = __riscv_vmulh_vv_i16m1(rx, ry, vl); /* h = (x * y) / R */ + rl = __riscv_vmul_vv_i16m1(rx, ry, vl); /* l = (x * y) % R */ + return fq_redc(rh, rl, vl); +} + +/* Montgomery multiply: vector-scalar */ + +static MLK_INLINE vint16m1_t fq_mul_vx(vint16m1_t rx, int16_t ry, size_t vl) +{ + vint16m1_t rl, rh; + + rh = __riscv_vmulh_vx_i16m1(rx, ry, vl); /* h = (x * y) / R */ + rl = __riscv_vmul_vx_i16m1(rx, ry, vl); /* l = (x * y) % R */ + return fq_redc(rh, rl, vl); +} + +/* full normalization */ + +static MLK_INLINE vint16m1_t fq_mulq_vx(vint16m1_t rx, int16_t ry, size_t vl) +{ + vint16m1_t result; + + result = fq_mul_vx(rx, ry, vl); + + mlk_assert_abs_bound_int16m1(result, vl, MLKEM_Q); + return result; +} + +/* create a permutation for swapping index bits a and b, a < b */ + +static vuint16m2_t bitswap_perm(unsigned a, unsigned b, size_t vl) +{ + const vuint16m2_t v2id = __riscv_vid_v_u16m2(vl); + + vuint16m2_t xa, xb; + xa = __riscv_vsrl_vx_u16m2(v2id, b - a, vl); + xa = __riscv_vxor_vv_u16m2(xa, v2id, vl); + xa = __riscv_vand_vx_u16m2(xa, (1 << a), vl); + xb = __riscv_vsll_vx_u16m2(xa, b - a, vl); + xa = __riscv_vxor_vv_u16m2(xa, xb, vl); + xa = __riscv_vxor_vv_u16m2(v2id, xa, vl); + return xa; +} + +/** + * Compute negacyclic number-theoretic transform (NTT) of a polynomial in + * place; input assumed to be in normal order, output in bitreversed order. + * + * @param[in,out] r Input/output polynomial. + */ + +/* Forward / Cooley-Tukey butterfly operation */ + +#define MLK_RVV_CT_BFLY_FX(u0, u1, ut, uc, vl, layer) \ + { \ + mlk_assert_abs_bound(&uc, 1, MLKEM_Q_HALF); \ + \ + ut = fq_mul_vx(u1, uc, vl); \ + mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \ + \ + u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \ + mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \ + } + +#define MLK_RVV_CT_BFLY_FV(u0, u1, ut, uc, vl, layer) \ + { \ + mlk_assert_abs_bound_int16m1(uc, vl, MLKEM_Q_HALF); \ + \ + ut = fq_mul_vv(u1, uc, vl); \ + mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \ + \ + u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \ + mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \ + } + +static vint16m2_t mlk_rv64v_ntt2(vint16m2_t vp, vint16m1_t cz) +{ + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2); + const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2); + const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2); + + /* p1 = p8(p4(p2)) */ + const vuint16m2_t v2p1 = __riscv_vrgather_vv_u16m2( + __riscv_vrgather_vv_u16m2(v2p2, v2p4, vl2), v2p8, vl2); + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t cs8 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl); + const vuint16m1_t cs4 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl); + const vuint16m1_t cs2 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl); + + vint16m1_t vt, c0, t0, t1; + + /* swap 8 */ + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 5); + + /* swap 4 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 6); + + /* swap 2 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 7); + + /* reorganize */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p1, vl2); + + return vp; +} + +/* Only for VLEN=256 for now */ +void mlk_rv64v_poly_ntt(int16_t *r) +{ +/* zetas can be compiled into vector constants; don't pass as a pointer */ +#include "rv64v_zetas.inc" + + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + vint16m1_t vt; + vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf; + + const vint16m1_t z0 = __riscv_vle16_v_i16m1(&zeta[0x00], vl); + const vint16m1_t z2 = __riscv_vle16_v_i16m1(&zeta[0x10], vl); + const vint16m1_t z4 = __riscv_vle16_v_i16m1(&zeta[0x20], vl); + const vint16m1_t z6 = __riscv_vle16_v_i16m1(&zeta[0x30], vl); + const vint16m1_t z8 = __riscv_vle16_v_i16m1(&zeta[0x40], vl); + const vint16m1_t za = __riscv_vle16_v_i16m1(&zeta[0x50], vl); + const vint16m1_t zc = __riscv_vle16_v_i16m1(&zeta[0x60], vl); + const vint16m1_t ze = __riscv_vle16_v_i16m1(&zeta[0x70], vl); + + v0 = __riscv_vle16_v_i16m1(&r[0x00], vl); + v1 = __riscv_vle16_v_i16m1(&r[0x10], vl); + v2 = __riscv_vle16_v_i16m1(&r[0x20], vl); + v3 = __riscv_vle16_v_i16m1(&r[0x30], vl); + v4 = __riscv_vle16_v_i16m1(&r[0x40], vl); + v5 = __riscv_vle16_v_i16m1(&r[0x50], vl); + v6 = __riscv_vle16_v_i16m1(&r[0x60], vl); + v7 = __riscv_vle16_v_i16m1(&r[0x70], vl); + v8 = __riscv_vle16_v_i16m1(&r[0x80], vl); + v9 = __riscv_vle16_v_i16m1(&r[0x90], vl); + va = __riscv_vle16_v_i16m1(&r[0xa0], vl); + vb = __riscv_vle16_v_i16m1(&r[0xb0], vl); + vc = __riscv_vle16_v_i16m1(&r[0xc0], vl); + vd = __riscv_vle16_v_i16m1(&r[0xd0], vl); + ve = __riscv_vle16_v_i16m1(&r[0xe0], vl); + vf = __riscv_vle16_v_i16m1(&r[0xf0], vl); + + MLK_RVV_CT_BFLY_FX(v0, v8, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v1, v9, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v2, va, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v3, vb, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v4, vc, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v5, vd, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v6, ve, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v7, vf, vt, zeta[0x01], vl, 1); + + MLK_RVV_CT_BFLY_FX(v0, v4, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v1, v5, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v2, v6, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v3, v7, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v8, vc, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(v9, vd, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(va, ve, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(vb, vf, vt, zeta[0x11], vl, 2); + + MLK_RVV_CT_BFLY_FX(v0, v2, vt, zeta[0x20], vl, 3); + MLK_RVV_CT_BFLY_FX(v1, v3, vt, zeta[0x20], vl, 3); + MLK_RVV_CT_BFLY_FX(v4, v6, vt, zeta[0x21], vl, 3); + MLK_RVV_CT_BFLY_FX(v5, v7, vt, zeta[0x21], vl, 3); + MLK_RVV_CT_BFLY_FX(v8, va, vt, zeta[0x30], vl, 3); + MLK_RVV_CT_BFLY_FX(v9, vb, vt, zeta[0x30], vl, 3); + MLK_RVV_CT_BFLY_FX(vc, ve, vt, zeta[0x31], vl, 3); + MLK_RVV_CT_BFLY_FX(vd, vf, vt, zeta[0x31], vl, 3); + + MLK_RVV_CT_BFLY_FX(v0, v1, vt, zeta[0x40], vl, 4); + MLK_RVV_CT_BFLY_FX(v2, v3, vt, zeta[0x41], vl, 4); + MLK_RVV_CT_BFLY_FX(v4, v5, vt, zeta[0x50], vl, 4); + MLK_RVV_CT_BFLY_FX(v6, v7, vt, zeta[0x51], vl, 4); + MLK_RVV_CT_BFLY_FX(v8, v9, vt, zeta[0x60], vl, 4); + MLK_RVV_CT_BFLY_FX(va, vb, vt, zeta[0x61], vl, 4); + MLK_RVV_CT_BFLY_FX(vc, vd, vt, zeta[0x70], vl, 4); + MLK_RVV_CT_BFLY_FX(ve, vf, vt, zeta[0x71], vl, 4); + + __riscv_vse16_v_i16m2( + &r[0x00], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v0, v1), z0), vl2); + __riscv_vse16_v_i16m2( + &r[0x20], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v2, v3), z2), vl2); + __riscv_vse16_v_i16m2( + &r[0x40], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v4, v5), z4), vl2); + __riscv_vse16_v_i16m2( + &r[0x60], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v6, v7), z6), vl2); + __riscv_vse16_v_i16m2( + &r[0x80], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v8, v9), z8), vl2); + __riscv_vse16_v_i16m2( + &r[0xa0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(va, vb), za), vl2); + __riscv_vse16_v_i16m2( + &r[0xc0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(vc, vd), zc), vl2); + __riscv_vse16_v_i16m2( + &r[0xe0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(ve, vf), ze), vl2); +} + +/* Reverse / Gentleman-Sande butterfly operation */ + +#define MLK_RVV_GS_BFLY_RX(u0, u1, ut, uc, vl) \ + { \ + ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \ + u1 = fq_mul_vx(ut, uc, vl); \ + } + +#define MLK_RVV_GS_BFLY_RV(u0, u1, ut, uc, vl) \ + { \ + ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \ + u1 = fq_mul_vv(ut, uc, vl); \ + } + +static vint16m2_t mlk_rv64v_intt2(vint16m2_t vp, vint16m1_t cz) +{ + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2); + const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2); + const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2); + + /* p0 = p2(p4(p8)) */ + const vuint16m2_t v2p0 = __riscv_vrgather_vv_u16m2( + __riscv_vrgather_vv_u16m2(v2p8, v2p4, vl2), v2p2, vl2); + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t cs8 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl); + const vuint16m1_t cs4 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl); + const vuint16m1_t cs2 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl); + + vint16m1_t t0, t1, c0, vt; + + /* initial permute */ + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p0, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + /* pre-scale */ + t0 = fq_mulq_vx(t0, MLK_RVV_MONT_NR, vl); + t1 = fq_mulq_vx(t1, MLK_RVV_MONT_NR, vl); + + /* absolute bounds: < t0 < q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: < t0 < 2*q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, 2 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 2 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: t0 < 4*q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, 4 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 4 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: < 8*q */ + mlk_assert_abs_bound_int16m1(t0, vl, 8 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + t0 = fq_mulq_vx(t0, MLK_RVV_MONT_R1, vl); + + /* absolute bounds: < q */ + mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 8 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2); + + return vp; +} + +#define MLK_RV64V_ABS_BOUNDS16(vl, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, \ + vb, vc, vd, ve, vf, b0, b1, b2, b3, b4, b5, b6, \ + b7, b8, b9, ba, bb, bc, bd, be, bf) \ + do \ + { \ + mlk_assert_abs_bound_int16m1(v0, vl, (b0) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v1, vl, (b1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v2, vl, (b2) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v3, vl, (b3) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v4, vl, (b4) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v5, vl, (b5) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v6, vl, (b6) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v7, vl, (b7) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v8, vl, (b8) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v9, vl, (b9) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(va, vl, (ba) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vb, vl, (bb) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vc, vl, (bc) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vd, vl, (bd) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(ve, vl, (be) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vf, vl, (bf) * MLKEM_Q); \ + } while (0) + + +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place; input assumed to be in bitreversed order, output in + * normal order. + * + * @param[in,out] r Input/output polynomial. + */ +/* Only for VLEN=256 for now */ +void mlk_rv64v_poly_invntt_tomont(int16_t *r) +{ +/* zetas can be compiled into vector constants; don't pass as a pointer */ +#include "rv64v_izetas.inc" + + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vint16m1_t z0 = __riscv_vle16_v_i16m1(&izeta[0x00], vl); + const vint16m1_t z2 = __riscv_vle16_v_i16m1(&izeta[0x10], vl); + const vint16m1_t z4 = __riscv_vle16_v_i16m1(&izeta[0x20], vl); + const vint16m1_t z6 = __riscv_vle16_v_i16m1(&izeta[0x30], vl); + const vint16m1_t z8 = __riscv_vle16_v_i16m1(&izeta[0x40], vl); + const vint16m1_t za = __riscv_vle16_v_i16m1(&izeta[0x50], vl); + const vint16m1_t zc = __riscv_vle16_v_i16m1(&izeta[0x60], vl); + const vint16m1_t ze = __riscv_vle16_v_i16m1(&izeta[0x70], vl); + + vint16m1_t vt; + vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf; + vint16m2_t vp; + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x00], vl2), z0); + v0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x20], vl2), z2); + v2 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v3 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x40], vl2), z4); + v4 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v5 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x60], vl2), z6); + v6 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v7 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x80], vl2), z8); + v8 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v9 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xa0], vl2), za); + va = __riscv_vget_v_i16m2_i16m1(vp, 0); + vb = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xc0], vl2), zc); + vc = __riscv_vget_v_i16m2_i16m1(vp, 0); + vd = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xe0], vl2), ze); + ve = __riscv_vget_v_i16m2_i16m1(vp, 0); + vf = __riscv_vget_v_i16m2_i16m1(vp, 1); + + /* absolute bounds < q (see mlk_rv64v_intt2) */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + + MLK_RVV_GS_BFLY_RX(v0, v1, vt, izeta[0x40], vl); + MLK_RVV_GS_BFLY_RX(v2, v3, vt, izeta[0x41], vl); + MLK_RVV_GS_BFLY_RX(v4, v5, vt, izeta[0x50], vl); + MLK_RVV_GS_BFLY_RX(v6, v7, vt, izeta[0x51], vl); + MLK_RVV_GS_BFLY_RX(v8, v9, vt, izeta[0x60], vl); + MLK_RVV_GS_BFLY_RX(va, vb, vt, izeta[0x61], vl); + MLK_RVV_GS_BFLY_RX(vc, vd, vt, izeta[0x70], vl); + MLK_RVV_GS_BFLY_RX(ve, vf, vt, izeta[0x71], vl); + + /* absolute bounds: + * - v{0,2,4,6,8,a,c,e}: < 2*q + * - v{1,3,5,7,9,b,d,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1); + + MLK_RVV_GS_BFLY_RX(v0, v2, vt, izeta[0x20], vl); + MLK_RVV_GS_BFLY_RX(v1, v3, vt, izeta[0x20], vl); + MLK_RVV_GS_BFLY_RX(v4, v6, vt, izeta[0x21], vl); + MLK_RVV_GS_BFLY_RX(v5, v7, vt, izeta[0x21], vl); + MLK_RVV_GS_BFLY_RX(v8, va, vt, izeta[0x30], vl); + MLK_RVV_GS_BFLY_RX(v9, vb, vt, izeta[0x30], vl); + MLK_RVV_GS_BFLY_RX(vc, ve, vt, izeta[0x31], vl); + MLK_RVV_GS_BFLY_RX(vd, vf, vt, izeta[0x31], vl); + + /* absolute bounds: + * - v{0,4,8,c}: < 4*q + * - v{1,5,9,d}: < 2*q + * - v{2,3,6,7,a,b,e,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1); + + MLK_RVV_GS_BFLY_RX(v0, v4, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v1, v5, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v2, v6, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v3, v7, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v8, vc, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(v9, vd, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(va, ve, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(vb, vf, vt, izeta[0x11], vl); + + /* absolute bounds: + * - v{0,8}: < 8*q + * - v{1,9}: < 4*q + * - v{2,3,a,b}: < 2*q + * - v{4,5,6,7,c,d,e,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 8, 4, 2, 2, 1, 1, 1, 1, 8, 4, 2, 2, 1, 1, 1, 1); + + /* Reduce v0, v8 to avoid overflow */ + v0 = fq_mulq_vx(v0, MLK_RVV_MONT_R1, vl); + v8 = fq_mulq_vx(v8, MLK_RVV_MONT_R1, vl); + + /* absolute bounds: < 4*q */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4); + + MLK_RVV_GS_BFLY_RX(v0, v8, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v1, v9, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v2, va, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v3, vb, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v4, vc, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v5, vd, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v6, ve, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v7, vf, vt, izeta[0x01], vl); + + /* absolute bounds: < 8*q */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8); + + __riscv_vse16_v_i16m1(&r[0x00], v0, vl); + __riscv_vse16_v_i16m1(&r[0x10], v1, vl); + __riscv_vse16_v_i16m1(&r[0x20], v2, vl); + __riscv_vse16_v_i16m1(&r[0x30], v3, vl); + __riscv_vse16_v_i16m1(&r[0x40], v4, vl); + __riscv_vse16_v_i16m1(&r[0x50], v5, vl); + __riscv_vse16_v_i16m1(&r[0x60], v6, vl); + __riscv_vse16_v_i16m1(&r[0x70], v7, vl); + __riscv_vse16_v_i16m1(&r[0x80], v8, vl); + __riscv_vse16_v_i16m1(&r[0x90], v9, vl); + __riscv_vse16_v_i16m1(&r[0xa0], va, vl); + __riscv_vse16_v_i16m1(&r[0xb0], vb, vl); + __riscv_vse16_v_i16m1(&r[0xc0], vc, vl); + __riscv_vse16_v_i16m1(&r[0xd0], vd, vl); + __riscv_vse16_v_i16m1(&r[0xe0], ve, vl); + __riscv_vse16_v_i16m1(&r[0xf0], vf, vl); +} + +/* ML-KEM's middle field GF(3329)[X]/(X^2) multiplication */ + +static MLK_INLINE void mlk_rv64v_poly_basemul_mont_add_k(int16_t *r, + const int16_t *a, + const int16_t *b, + unsigned kn) +{ +#include "rv64v_zetas_basemul.inc" + + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + size_t i, j; + + const vuint16m1_t sw0 = __riscv_vxor_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl); + const vbool16_t sb0 = __riscv_vmseq_vx_u16m1_b16( + __riscv_vand_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl), 0, vl); + + vint16m1_t vt, vu; + vint32m2_t wa, wb, ws; + + for (i = 0; i < MLKEM_N; i += vl) + { + const vint16m1_t vz = __riscv_vle16_v_i16m1(&roots[i], vl); + + for (j = 0; j < kn; j += MLKEM_N) + { + vt = __riscv_vle16_v_i16m1(&a[i + j], vl); + vu = __riscv_vle16_v_i16m1(&b[i + j], vl); + + wa = __riscv_vwmul_vv_i32m2(vz, fq_mul_vv(vt, vu, vl), vl); + wb = __riscv_vwmul_vv_i32m2(vt, __riscv_vrgather_vv_i16m1(vu, sw0, vl), + vl); + + wa = + __riscv_vadd_vv_i32m2(wa, __riscv_vslidedown_vx_i32m2(wa, 1, vl), vl); + wb = __riscv_vadd_vv_i32m2(wb, __riscv_vslideup_vx_i32m2(wb, wb, 1, vl), + vl); + + wa = __riscv_vmerge_vvm_i32m2(wb, wa, sb0, vl); + + if (j == 0) + { + ws = wa; + } + else + { + ws = __riscv_vadd_vv_i32m2(ws, wa, vl); + } + } + /* the idea is to keep 32-bit intermediate result, reduce in the end */ + __riscv_vse16_v_i16m1(&r[i], fq_redc2(ws, vl), vl); + } +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 2 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 3 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 4 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * @param[in,out] r Input/output polynomial. + */ +void mlk_rv64v_poly_tomont(int16_t *r) +{ + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + size_t i; + + for (i = 0; i < MLKEM_N; i += vl) + { + vint16m1_t vec = __riscv_vle16_v_i16m1(&r[i], vl); + vec = fq_mul_vx(vec, MLK_RVV_MONT_R2, vl); + __riscv_vse16_v_i16m1(&r[i], vec, vl); + } +} + +/** + * Apply Barrett reduction to all coefficients of a polynomial. For details + * of the Barrett reduction see the comments in poly.c. + * + * @param[in,out] r Input/output polynomial. + */ +void mlk_rv64v_poly_reduce(int16_t *r) +{ + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + vint16m1_t vt; + size_t i; + + for (i = 0; i < MLKEM_N; i += vl) + { + vt = __riscv_vle16_v_i16m1(&r[i], vl); + vt = fq_barrett(vt, vl); + vt = fq_cadd(vt, vl); + __riscv_vse16_v_i16m1(&r[i], vt, vl); + } +} + +/* Run rejection sampling to get uniform random integers mod q. + * buflen must be a multiple of 12. */ + +unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len, + const uint8_t *buf, unsigned int buflen) +{ + unsigned n, ctr, pos; + vuint16m1_t x, y; + vbool16_t lt; + + pos = 0; + ctr = 0; + + while (ctr < len && pos < buflen) + { + const unsigned vl = (unsigned)__riscv_vsetvl_e16m1((buflen - pos) * 8 / 12); + const unsigned vl23 = (vl * 24) / 32; + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t srl12v = __riscv_vmul_vx_u16m1(vid, 12, vl); + const vuint16m1_t sel12v = __riscv_vsrl_vx_u16m1(srl12v, 4, vl); + const vuint16m1_t sll12v = __riscv_vsll_vx_u16m1(vid, 2, vl); + + /* Functionally, this loop is not necessary, but it avoids re-evaluating + * the VL too many times. In particular, in the first outer iteration, + * the inner loop will process the bulk of the data with fixed VL. */ + while (ctr < len && vl23 * 2 <= buflen - pos) + { + x = __riscv_vle16_v_u16m1((uint16_t *)&buf[pos], vl23); + pos += vl23 * 2; + x = __riscv_vrgather_vv_u16m1(x, sel12v, vl); + x = __riscv_vor_vv_u16m1( + __riscv_vsrl_vv_u16m1(x, srl12v, vl), + __riscv_vsll_vv_u16m1(__riscv_vslidedown(x, 1, vl), sll12v, vl), vl); + x = __riscv_vand_vx_u16m1(x, 0xFFF, vl); + + lt = __riscv_vmsltu_vx_u16m1_b16(x, MLKEM_Q, vl); + y = __riscv_vcompress_vm_u16m1(x, lt, vl); + n = (unsigned)__riscv_vcpop_m_b16(lt, vl); + + if (ctr + n > len) + { + n = len - ctr; + } + __riscv_vse16_v_u16m1((uint16_t *)&r[ctr], y, n); + ctr += n; + } + } + + return ctr; +} + +#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(rv64v_poly) + +#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_RVV_QI +#undef MLK_RVV_MONT_R1 +#undef MLK_RVV_MONT_R2 +#undef MLK_RVV_MONT_NR +#undef MLK_RVV_CT_BFLY_FX +#undef MLK_RVV_CT_BFLY_FV +#undef MLK_RVV_GS_BFLY_RX +#undef MLK_RVV_GS_BFLY_RV +#undef MLK_RV64V_ABS_BOUNDS16 diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas.inc b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas.inc new file mode 100644 index 0000000..9deb959 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas.inc @@ -0,0 +1,27 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t zeta[] = { + -1044, -758, 573, -1325, 1223, 652, -552, 1015, -1103, 430, 555, + 843, -1251, 871, 1550, 105, -359, -1517, 264, 383, -1293, 1491, + -282, -1544, 422, 587, 177, -235, -291, -460, 1574, 1653, 1493, + 1422, -829, 1458, 516, -8, -320, -666, -246, 778, 1159, -147, + -777, 1483, -602, 1119, 287, 202, -1602, -130, -1618, -1162, 126, + 1469, -1590, 644, -872, 349, 418, 329, -156, -75, -171, 622, + -681, 1017, -853, -90, -271, 830, 817, 1097, 603, 610, 1322, + -1285, -1465, 384, 1577, 182, 732, 608, 107, -1421, -247, -951, + -1215, -136, 1218, -1335, -874, 220, -1187, -1659, 962, -1202, -1542, + 411, -398, 961, -1508, -725, -1185, -1530, -1278, 794, -1510, -854, + -870, 478, -1474, 1468, -205, -1571, 448, -1065, 677, -1275, -108, + -308, 996, 991, 958, -1460, 1522, 1628, +}; diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas_basemul.inc b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas_basemul.inc new file mode 100644 index 0000000..ee4033c --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/riscv64/src/rv64v_zetas_basemul.inc @@ -0,0 +1,39 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t roots[] = { + -1044, -1103, -1044, 1103, -1044, 430, -1044, -430, -1044, 555, -1044, + -555, -1044, 843, -1044, -843, -1044, -1251, -1044, 1251, -1044, 871, + -1044, -871, -1044, 1550, -1044, -1550, -1044, 105, -1044, -105, -1044, + 422, -1044, -422, -1044, 587, -1044, -587, -1044, 177, -1044, -177, + -1044, -235, -1044, 235, -1044, -291, -1044, 291, -1044, -460, -1044, + 460, -1044, 1574, -1044, -1574, -1044, 1653, -1044, -1653, -1044, -246, + -1044, 246, -1044, 778, -1044, -778, -1044, 1159, -1044, -1159, -1044, + -147, -1044, 147, -1044, -777, -1044, 777, -1044, 1483, -1044, -1483, + -1044, -602, -1044, 602, -1044, 1119, -1044, -1119, -1044, -1590, -1044, + 1590, -1044, 644, -1044, -644, -1044, -872, -1044, 872, -1044, 349, + -1044, -349, -1044, 418, -1044, -418, -1044, 329, -1044, -329, -1044, + -156, -1044, 156, -1044, -75, -1044, 75, -1044, 817, -1044, -817, + -1044, 1097, -1044, -1097, -1044, 603, -1044, -603, -1044, 610, -1044, + -610, -1044, 1322, -1044, -1322, -1044, -1285, -1044, 1285, -1044, -1465, + -1044, 1465, -1044, 384, -1044, -384, -1044, -1215, -1044, 1215, -1044, + -136, -1044, 136, -1044, 1218, -1044, -1218, -1044, -1335, -1044, 1335, + -1044, -874, -1044, 874, -1044, 220, -1044, -220, -1044, -1187, -1044, + 1187, -1044, -1659, -1044, 1659, -1044, -1185, -1044, 1185, -1044, -1530, + -1044, 1530, -1044, -1278, -1044, 1278, -1044, 794, -1044, -794, -1044, + -1510, -1044, 1510, -1044, -854, -1044, 854, -1044, -870, -1044, 870, + -1044, 478, -1044, -478, -1044, -108, -1044, 108, -1044, -308, -1044, + 308, -1044, 996, -1044, -996, -1044, 991, -1044, -991, -1044, 958, + -1044, -958, -1044, -1460, -1044, 1460, -1044, 1522, -1044, -1522, -1044, + 1628, -1044, -1628, +}; diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/README.md b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/README.md new file mode 100644 index 0000000..2073425 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/README.md @@ -0,0 +1,4 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +This directory contains the native x86_64 arithmetic backend for ML-KEM provided by the official [AVX2 +implementation](https://github.com/pq-crystals/kyber/tree/main/avx2) of the Kyber team. diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/meta.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/meta.h new file mode 100644 index 0000000..e59849e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/meta.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_X86_64_META_H +#define MLK_NATIVE_X86_64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_X86_64_DEFAULT + +#define MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#define MLK_USE_NATIVE_REJ_UNIFORM +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLY_TOBYTES +#define MLK_USE_NATIVE_POLY_FROMBYTES +#define MLK_USE_NATIVE_POLY_COMPRESS_D4 +#define MLK_USE_NATIVE_POLY_COMPRESS_D5 +#define MLK_USE_NATIVE_POLY_COMPRESS_D10 +#define MLK_USE_NATIVE_POLY_COMPRESS_D11 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D11 + +#if !defined(__ASSEMBLER__) +#include "../../common.h" +#include "../api.h" +#include "src/arith_native_x86_64.h" +#include "src/compress_consts.h" + +static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t data[MLKEM_N]) +{ + if (mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + mlk_nttunpack_avx2_asm(data); + } +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2) || len != MLKEM_N || + buflen % 12 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + return (int)mlk_rej_uniform_avx2_asm(r, buf, buflen, mlk_rej_uniform_table); +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_ntt_avx2_asm(data, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_invntt_avx2_asm(data, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_reduce_avx2_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_tomont_avx2_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_mulcache_compute_avx2_asm(x, y, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_ntttobytes_avx2_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_frombytes_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYBYTES]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_nttfrombytes_avx2_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d4_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d4_avx2_asm(r, a, mlk_compress_d4_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d10_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d10_avx2_asm(r, a, mlk_compress_d10_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d4_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d4_avx2_asm(r, a, mlk_decompress_d4_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d10_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d10_avx2_asm(r, a, mlk_decompress_d10_data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d5_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d5_avx2_asm(r, a, mlk_compress_d5_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d11_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d11_avx2_asm(r, a, mlk_compress_d11_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d5_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d5_avx2_asm(r, a, mlk_decompress_d5_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d11_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d11_avx2_asm(r, a, mlk_decompress_d11_data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_X86_64_META_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/arith_native_x86_64.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/arith_native_x86_64.h new file mode 100644 index 0000000..6d97c8e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/arith_native_x86_64.h @@ -0,0 +1,327 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#define MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H + +#include "../../../common.h" + +#include +#include "compress_consts.h" +#include "consts.h" + +#define MLK_AVX2_REJ_UNIFORM_BUFLEN \ + (3 * 168) /* REJ_UNIFORM_NBLOCKS * SHAKE128_RATE */ + +#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096]; + +#define mlk_rej_uniform_avx2_asm MLK_NAMESPACE(rej_uniform_avx2_asm) +MLK_MUST_CHECK_RETURN_VALUE MLK_SYSV_ABI +uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf, + unsigned buflen, const uint8_t *table) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/rej_uniform_avx2_asm.ml. */ +__contract__( + requires(buflen % 12 == 0) + requires(memory_no_alias(buf, buflen)) + requires(table == mlk_rej_uniform_table) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value <= MLKEM_N) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); + +#define mlk_ntt_avx2_asm MLK_NAMESPACE(ntt_avx2_asm) +MLK_SYSV_ABI +void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/ntt_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(r, 0, MLKEM_N, 8192)) + requires(qdata == mlk_qdata) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(r, 0, MLKEM_N, 23595)) + /* check-magic: on */ +); + +#define mlk_invntt_avx2_asm MLK_NAMESPACE(invntt_avx2_asm) +MLK_SYSV_ABI +void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/intt_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(qdata == mlk_qdata) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(r, 0, MLKEM_N, 26632)) + /* check-magic: on */ +); + +#define mlk_nttunpack_avx2_asm MLK_NAMESPACE(nttunpack_avx2_asm) +MLK_SYSV_ABI +void mlk_nttunpack_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/nttunpack_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* Output is a permutation of input: every output coefficient + * is some input coefficient */ + ensures(forall(i, 0, MLKEM_N, exists(j, 0, MLKEM_N, + r[i] == old(*(int16_t (*)[MLKEM_N])r)[j]))) +); + +#define mlk_reduce_avx2_asm MLK_NAMESPACE(reduce_avx2_asm) +MLK_SYSV_ABI +void mlk_reduce_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/reduce_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute_avx2_asm \ + MLK_NAMESPACE(poly_mulcache_compute_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in, + const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/poly_mulcache_compute_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(out, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(in, sizeof(int16_t) * MLKEM_N)) + requires(qdata == mlk_qdata) + assigns(memory_slice(out, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(array_abs_bound(out, 0, MLKEM_N/2, MLKEM_Q)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_ntttobytes_avx2_asm MLK_NAMESPACE(ntttobytes_avx2_asm) +MLK_SYSV_ABI +void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/ntttobytes_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + +#define mlk_nttfrombytes_avx2_asm MLK_NAMESPACE(nttfrombytes_avx2_asm) +MLK_SYSV_ABI +void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/nttfrombytes_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); + +#define mlk_tomont_avx2_asm MLK_NAMESPACE(tomont_avx2_asm) +MLK_SYSV_ABI +void mlk_tomont_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/tomont_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_compress_d4_avx2_asm MLK_NAMESPACE(poly_compress_d4_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d4_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_compress_d4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d4_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +); + +#define mlk_poly_decompress_d4_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d4_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d4_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_decompress_d4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(data == mlk_decompress_d4_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d10_avx2_asm MLK_NAMESPACE(poly_compress_d10_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d10_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_compress_d10_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d10_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +); + +#define mlk_poly_decompress_d10_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d10_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d10_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_decompress_d10_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(data == mlk_decompress_d10_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d5_avx2_asm MLK_NAMESPACE(poly_compress_d5_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d5_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_compress_d5_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d5_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +); + +#define mlk_poly_decompress_d5_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d5_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d5_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_decompress_d5_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(data == mlk_decompress_d5_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d11_avx2_asm MLK_NAMESPACE(poly_compress_d11_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d11_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_compress_d11_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d11_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +); + +#define mlk_poly_decompress_d11_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d11_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d11_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/poly_decompress_d11_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(data == mlk_decompress_d11_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#endif /* !MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.c new file mode 100644 index 0000000..91fd4d1 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) + +#include "compress_consts.h" + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d4_data[32] = { + 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, + 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, /* permdidx */ +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d4_data[32] = { + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, /* shufbidx */ +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d10_data[32] = { + 0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 255, + 255, 255, 255, 255, 255, 9, 10, 11, 12, 255, 255, + 255, 255, 255, 255, 0, 1, 2, 3, 4, 8, /* shufbidx */ +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d10_data[32] = { + 0, 1, 1, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 9, + 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, /* shufbidx */ +}; + +#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d5_data[32] = { + 0, 1, 2, 3, 4, 255, 255, 255, 255, 255, 8, + 9, 10, 11, 12, 255, 9, 10, 11, 12, 255, 0, + 1, 2, 3, 4, 255, 255, 255, 255, 255, 8, /* shufbidx */ +}; + +/* shufbidx[0:32], mask[32:64], shift[64:96] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d5_data[96] = { + 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, + 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, /* shufbidx */ + 31, 0, 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, 31, 0, + 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, /* mask */ + 0, 4, 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, 0, 4, + 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, /* shift */ +}; + +/* srlvqidx[0:32], shufbidx[32:64] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d11_data[64] = + { + 10, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, + 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, + 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 255, 255, 255, 255, 255, 5, 6, 7, 8, 9, 10, + 255, 255, 255, 255, 0, 0, 1, 2, 3, 4, /* shufbidx */ +}; + +/* shufbidx[0:32], srlvdidx[32:64], srlvqidx[64:96], shift[96:128] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d11_data[128] = { + 0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10, + 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 11, 12, 12, 13, /* shufbidx */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* srlvdidx */ + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */ + 32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, + 32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, /* shift */ +}; + +#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT */ + +MLK_EMPTY_CU(avx2_compress_consts) + +#endif /* !MLK_ARITH_BACKEND_X86_64_DEFAULT */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.h new file mode 100644 index 0000000..d8da95f --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/compress_consts.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#define MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H + +#include "../../../common.h" + +#ifndef __ASSEMBLER__ + +#define mlk_compress_d4_data MLK_NAMESPACE(compress_d4_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d4_data[32]; + +#define mlk_decompress_d4_data MLK_NAMESPACE(decompress_d4_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d4_data[32]; + +#define mlk_compress_d10_data MLK_NAMESPACE(compress_d10_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d10_data[32]; + +#define mlk_decompress_d10_data MLK_NAMESPACE(decompress_d10_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d10_data[32]; + +#define mlk_compress_d5_data MLK_NAMESPACE(compress_d5_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d5_data[32]; + +#define mlk_decompress_d5_data MLK_NAMESPACE(decompress_d5_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d5_data[96]; + +#define mlk_compress_d11_data MLK_NAMESPACE(compress_d11_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d11_data[64]; + +#define mlk_decompress_d11_data MLK_NAMESPACE(decompress_d11_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d11_data[128]; + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.c new file mode 100644 index 0000000..d0acfe0 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "consts.h" + +/* + * Table of zeta values used in the AVX2 NTTs + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_qdata[624] = { + 3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 3854, + 3340, 2826, 2312, 1798, 1284, 770, 256, 7, 0, + 6, 0, 5, 0, 4, 0, 3, 0, 2, + 0, 1, 0, 0, 0, 31498, 31498, 31498, 31498, + -758, -758, -758, -758, 0, 0, 0, 0, 0, + 0, 0, 0, 14745, 14745, 14745, 14745, 14745, 14745, + 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, + 14745, -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, 13525, + 13525, 13525, 13525, 13525, 13525, 13525, 13525, -12402, -12402, + -12402, -12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493, + 1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422, + 1422, 1422, 1422, 1422, -20907, -20907, -20907, -20907, 27758, + 27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690, + -15690, -15690, -171, -171, -171, -171, 622, 622, 622, + 622, 1577, 1577, 1577, 1577, 182, 182, 182, 182, + -5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057, 5571, + 5571, -1102, -1102, 21438, 21438, -26242, -26242, 573, 573, + -1325, -1325, 264, 264, 383, 383, -829, -829, 1458, + 1458, -1602, -1602, -130, -130, -5689, -6516, 1496, 30967, + -23565, 20179, 20710, 25080, -12796, 26616, 16064, -12442, 9134, + -650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491, + -282, -1544, 516, -8, -320, -666, -1618, -1162, 126, + 1469, -335, -11477, -32227, 20494, -27738, 945, -14883, 6182, + 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103, + 555, -1251, 1550, 422, 177, -291, 1574, -246, 1159, + -777, -602, -1590, -872, 418, -156, 11182, 13387, -14233, + -21655, 13131, -4587, 23092, 5493, -32502, 30317, -18741, 12639, + 20100, 18525, 19529, -12619, 430, 843, 871, 105, 587, + -235, -460, 1653, 778, -147, 1483, 1119, 644, 349, + 329, -75, 787, 787, 787, 787, 787, 787, 787, + 787, 787, 787, 787, 787, 787, 787, 787, 787, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, 28191, 28191, + 28191, 28191, 28191, 28191, 28191, 28191, -16694, -16694, -16694, + -16694, -16694, -16694, -16694, -16694, 287, 287, 287, 287, + 287, 287, 287, 287, 202, 202, 202, 202, 202, + 202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358, + 1358, 1358, -11202, -11202, -11202, -11202, 31164, 31164, 31164, + 31164, 962, 962, 962, 962, -1202, -1202, -1202, -1202, + -1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073, + -28073, 24313, 24313, -10532, -10532, 8800, 8800, 18426, 18426, + 8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017, + 1017, 732, 732, 608, 608, -1542, -1542, 411, 411, + -205, -205, -1571, -1571, 19883, -28250, -15887, -8898, -28309, + 9075, -30199, 18249, 13426, 14017, -29156, -12757, 16832, 4311, + -24155, -17915, -853, -90, -271, 830, 107, -1421, -247, + -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, + -31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, + -7934, -22502, 10906, 31636, 28644, 23998, -17422, 817, 603, + 1322, -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, + -870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384, + -6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860, + -20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335, + 220, -1659, -1530, 794, -854, 478, -308, 991, -1460, + 1628, -1103, 555, -1251, 1550, 422, 177, -291, 1574, + -246, 1159, -777, -602, -1590, -872, 418, -156, 430, + 843, 871, 105, 587, -235, -460, 1653, 778, -147, + 1483, 1119, 644, 349, 329, -75, 817, 603, 1322, + -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, -870, + -108, 996, 958, 1522, 1097, 610, -1285, 384, -136, + -1335, 220, -1659, -1530, 794, -854, 478, -308, 991, + -1460, 1628, -335, -11477, -32227, 20494, -27738, 945, -14883, + 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, + 11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493, -32502, + 30317, -18741, 12639, 20100, 18525, 19529, -12619, -31183, 25435, + -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, + 10906, 31636, 28644, 23998, -17422, 20297, 2146, 15355, -32384, + -6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860, + -20257, 7756, 23132, +}; + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(avx2_consts) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.h b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.h new file mode 100644 index 0000000..591529d --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/consts.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_X86_64_SRC_CONSTS_H +#define MLK_NATIVE_X86_64_SRC_CONSTS_H +#include "../../../common.h" +#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB 0 +#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD 16 +#define MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP 32 +#define MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES 496 + +#ifndef __ASSEMBLER__ +#define mlk_qdata MLK_NAMESPACE(qdata) +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_qdata[624]; +#endif + +#endif /* !MLK_NATIVE_X86_64_SRC_CONSTS_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/intt_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/intt_avx2_asm.S new file mode 100644 index 0000000..9c572f5 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/intt_avx2_asm.S @@ -0,0 +1,741 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [AVX2_NTT] + * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography. + * Gregor Seiler + * https://eprint.iacr.org/2018/039 + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * The core ideas behind the implementation are described in @[AVX2_NTT]. + * + * Changes: + * - Different placement of modular reductions to simplify + * reasoning of non-overflow + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: invntt_avx2_asm + Description: x86_64 AVX2 inverse NTT + Signature: void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/intt_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(invntt_avx2_asm) +MLK_ASM_FN_SYMBOL(invntt_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x5a105a1, %eax # imm = 0x5A105A1 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x60(%rdi), %ymm7 + vpmullw %ymm2, %ymm4, %ymm12 + vpmulhw %ymm3, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm2, %ymm6, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmullw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmullw %ymm2, %ymm7, %ymm12 + vpmulhw %ymm3, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xe0(%rdi), %ymm11 + vpmullw %ymm2, %ymm8, %ymm12 + vpmulhw %ymm3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmullw %ymm2, %ymm10, %ymm12 + vpmulhw %ymm3, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpmullw %ymm2, %ymm9, %ymm12 + vpmulhw %ymm3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm2, %ymm11, %ymm12 + vpmulhw %ymm3, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm11, %ymm11 + vpermq $0x4e, 0x3a0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1] + vpermq $0x4e, 0x360(%rsi), %ymm1 # ymm1 = mem[2,3,0,1] + vpermq $0x4e, 0x3c0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x380(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm12 + vpshufb %ymm12, %ymm15, %ymm15 + vpshufb %ymm12, %ymm1, %ymm1 + vpshufb %ymm12, %ymm2, %ymm2 + vpshufb %ymm12, %ymm3, %ymm3 + vpsubw %ymm4, %ymm6, %ymm12 + vpaddw %ymm6, %ymm4, %ymm4 + vpsubw %ymm5, %ymm7, %ymm13 + vpmullw %ymm15, %ymm12, %ymm6 + vpaddw %ymm7, %ymm5, %ymm5 + vpsubw %ymm8, %ymm10, %ymm14 + vpmullw %ymm15, %ymm13, %ymm7 + vpaddw %ymm10, %ymm8, %ymm8 + vpsubw %ymm9, %ymm11, %ymm15 + vpmullw %ymm1, %ymm14, %ymm10 + vpaddw %ymm11, %ymm9, %ymm9 + vpmullw %ymm1, %ymm15, %ymm11 + vpmulhw %ymm2, %ymm12, %ymm12 + vpmulhw %ymm2, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm6, %ymm12, %ymm6 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpermq $0x4e, 0x320(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x340(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm1 + vpshufb %ymm1, %ymm2, %ymm2 + vpshufb %ymm1, %ymm3, %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpslld $0x10, %ymm11, %ymm8 + vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15] + vpsrld $0x10, %ymm10, %ymm10 + vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15] + vmovdqa 0x20(%rsi), %ymm12 + vpermd 0x2e0(%rsi), %ymm12, %ymm2 + vpermd 0x300(%rsi), %ymm12, %ymm10 + vpsubw %ymm3, %ymm5, %ymm12 + vpaddw %ymm5, %ymm3, %ymm3 + vpsubw %ymm4, %ymm7, %ymm13 + vpmullw %ymm2, %ymm12, %ymm5 + vpaddw %ymm7, %ymm4, %ymm4 + vpsubw %ymm6, %ymm9, %ymm14 + vpmullw %ymm2, %ymm13, %ymm7 + vpaddw %ymm9, %ymm6, %ymm6 + vpsubw %ymm8, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm9 + vpaddw %ymm11, %ymm8, %ymm8 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm10, %ymm12, %ymm12 + vpmulhw %ymm10, %ymm13, %ymm13 + vpmulhw %ymm10, %ymm14, %ymm14 + vpmulhw %ymm10, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm5, %ymm12, %ymm5 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm9, %ymm14, %ymm9 + vpsubw %ymm11, %ymm15, %ymm11 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7] + vpsrlq $0x20, %ymm9, %ymm9 + vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7] + vpermq $0x1b, 0x2a0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0] + vpermq $0x1b, 0x2c0(%rsi), %ymm9 # ymm9 = mem[3,2,1,0] + vpsubw %ymm10, %ymm4, %ymm12 + vpaddw %ymm4, %ymm10, %ymm10 + vpsubw %ymm3, %ymm8, %ymm13 + vpmullw %ymm2, %ymm12, %ymm4 + vpaddw %ymm8, %ymm3, %ymm3 + vpsubw %ymm6, %ymm7, %ymm14 + vpmullw %ymm2, %ymm13, %ymm8 + vpaddw %ymm7, %ymm6, %ymm6 + vpsubw %ymm5, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm7 + vpaddw %ymm11, %ymm5, %ymm5 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm9, %ymm12, %ymm12 + vpmulhw %ymm9, %ymm13, %ymm13 + vpmulhw %ymm9, %ymm14, %ymm14 + vpmulhw %ymm9, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm4, %ymm12, %ymm4 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm7, %ymm14, %ymm7 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm10, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3] + vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] + vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3] + vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3] + vpermq $0x4e, 0x260(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x280(%rsi), %ymm7 # ymm7 = mem[2,3,0,1] + vpsubw %ymm9, %ymm3, %ymm12 + vpaddw %ymm3, %ymm9, %ymm9 + vpsubw %ymm10, %ymm5, %ymm13 + vpmullw %ymm2, %ymm12, %ymm3 + vpaddw %ymm5, %ymm10, %ymm10 + vpsubw %ymm6, %ymm8, %ymm14 + vpmullw %ymm2, %ymm13, %ymm5 + vpaddw %ymm8, %ymm6, %ymm6 + vpsubw %ymm4, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm8 + vpaddw %ymm11, %ymm4, %ymm4 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm7, %ymm12, %ymm12 + vpmulhw %ymm7, %ymm13, %ymm13 + vpmulhw %ymm7, %ymm14, %ymm14 + vpmulhw %ymm7, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm3, %ymm12, %ymm3 + vpsubw %ymm5, %ymm13, %ymm5 + vpsubw %ymm8, %ymm14, %ymm8 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1] + vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3] + vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3] + vmovdqa 0x220(%rsi), %ymm2 + vmovdqa 0x240(%rsi), %ymm8 + vpsubw %ymm7, %ymm10, %ymm12 + vpaddw %ymm10, %ymm7, %ymm7 + vpsubw %ymm9, %ymm4, %ymm13 + vpmullw %ymm2, %ymm12, %ymm10 + vpaddw %ymm4, %ymm9, %ymm9 + vpsubw %ymm6, %ymm5, %ymm14 + vpmullw %ymm2, %ymm13, %ymm4 + vpaddw %ymm5, %ymm6, %ymm6 + vpsubw %ymm3, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm5 + vpaddw %ymm11, %ymm3, %ymm3 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm8, %ymm12, %ymm12 + vpmulhw %ymm8, %ymm13, %ymm13 + vpmulhw %ymm8, %ymm14, %ymm14 + vpmulhw %ymm8, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm10, %ymm12, %ymm10 + vpsubw %ymm4, %ymm13, %ymm4 + vpsubw %ymm5, %ymm14, %ymm5 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa %ymm6, 0x40(%rdi) + vmovdqa %ymm3, 0x60(%rdi) + vmovdqa %ymm10, 0x80(%rdi) + vmovdqa %ymm4, 0xa0(%rdi) + vmovdqa %ymm5, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x5a105a1, %eax # imm = 0x5A105A1 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x160(%rdi), %ymm7 + vpmullw %ymm2, %ymm4, %ymm12 + vpmulhw %ymm3, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm2, %ymm6, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmullw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmullw %ymm2, %ymm7, %ymm12 + vpmulhw %ymm3, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1e0(%rdi), %ymm11 + vpmullw %ymm2, %ymm8, %ymm12 + vpmulhw %ymm3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmullw %ymm2, %ymm10, %ymm12 + vpmulhw %ymm3, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpmullw %ymm2, %ymm9, %ymm12 + vpmulhw %ymm3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm2, %ymm11, %ymm12 + vpmulhw %ymm3, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm11, %ymm11 + vpermq $0x4e, 0x1e0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1] + vpermq $0x4e, 0x1a0(%rsi), %ymm1 # ymm1 = mem[2,3,0,1] + vpermq $0x4e, 0x200(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x1c0(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm12 + vpshufb %ymm12, %ymm15, %ymm15 + vpshufb %ymm12, %ymm1, %ymm1 + vpshufb %ymm12, %ymm2, %ymm2 + vpshufb %ymm12, %ymm3, %ymm3 + vpsubw %ymm4, %ymm6, %ymm12 + vpaddw %ymm6, %ymm4, %ymm4 + vpsubw %ymm5, %ymm7, %ymm13 + vpmullw %ymm15, %ymm12, %ymm6 + vpaddw %ymm7, %ymm5, %ymm5 + vpsubw %ymm8, %ymm10, %ymm14 + vpmullw %ymm15, %ymm13, %ymm7 + vpaddw %ymm10, %ymm8, %ymm8 + vpsubw %ymm9, %ymm11, %ymm15 + vpmullw %ymm1, %ymm14, %ymm10 + vpaddw %ymm11, %ymm9, %ymm9 + vpmullw %ymm1, %ymm15, %ymm11 + vpmulhw %ymm2, %ymm12, %ymm12 + vpmulhw %ymm2, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm6, %ymm12, %ymm6 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpermq $0x4e, 0x160(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x180(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm1 + vpshufb %ymm1, %ymm2, %ymm2 + vpshufb %ymm1, %ymm3, %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpslld $0x10, %ymm11, %ymm8 + vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15] + vpsrld $0x10, %ymm10, %ymm10 + vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15] + vmovdqa 0x20(%rsi), %ymm12 + vpermd 0x120(%rsi), %ymm12, %ymm2 + vpermd 0x140(%rsi), %ymm12, %ymm10 + vpsubw %ymm3, %ymm5, %ymm12 + vpaddw %ymm5, %ymm3, %ymm3 + vpsubw %ymm4, %ymm7, %ymm13 + vpmullw %ymm2, %ymm12, %ymm5 + vpaddw %ymm7, %ymm4, %ymm4 + vpsubw %ymm6, %ymm9, %ymm14 + vpmullw %ymm2, %ymm13, %ymm7 + vpaddw %ymm9, %ymm6, %ymm6 + vpsubw %ymm8, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm9 + vpaddw %ymm11, %ymm8, %ymm8 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm10, %ymm12, %ymm12 + vpmulhw %ymm10, %ymm13, %ymm13 + vpmulhw %ymm10, %ymm14, %ymm14 + vpmulhw %ymm10, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm5, %ymm12, %ymm5 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm9, %ymm14, %ymm9 + vpsubw %ymm11, %ymm15, %ymm11 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7] + vpsrlq $0x20, %ymm9, %ymm9 + vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7] + vpermq $0x1b, 0xe0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0] + vpermq $0x1b, 0x100(%rsi), %ymm9 # ymm9 = mem[3,2,1,0] + vpsubw %ymm10, %ymm4, %ymm12 + vpaddw %ymm4, %ymm10, %ymm10 + vpsubw %ymm3, %ymm8, %ymm13 + vpmullw %ymm2, %ymm12, %ymm4 + vpaddw %ymm8, %ymm3, %ymm3 + vpsubw %ymm6, %ymm7, %ymm14 + vpmullw %ymm2, %ymm13, %ymm8 + vpaddw %ymm7, %ymm6, %ymm6 + vpsubw %ymm5, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm7 + vpaddw %ymm11, %ymm5, %ymm5 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm9, %ymm12, %ymm12 + vpmulhw %ymm9, %ymm13, %ymm13 + vpmulhw %ymm9, %ymm14, %ymm14 + vpmulhw %ymm9, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm4, %ymm12, %ymm4 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm7, %ymm14, %ymm7 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm10, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3] + vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] + vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3] + vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3] + vpermq $0x4e, 0xa0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0xc0(%rsi), %ymm7 # ymm7 = mem[2,3,0,1] + vpsubw %ymm9, %ymm3, %ymm12 + vpaddw %ymm3, %ymm9, %ymm9 + vpsubw %ymm10, %ymm5, %ymm13 + vpmullw %ymm2, %ymm12, %ymm3 + vpaddw %ymm5, %ymm10, %ymm10 + vpsubw %ymm6, %ymm8, %ymm14 + vpmullw %ymm2, %ymm13, %ymm5 + vpaddw %ymm8, %ymm6, %ymm6 + vpsubw %ymm4, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm8 + vpaddw %ymm11, %ymm4, %ymm4 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm7, %ymm12, %ymm12 + vpmulhw %ymm7, %ymm13, %ymm13 + vpmulhw %ymm7, %ymm14, %ymm14 + vpmulhw %ymm7, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm3, %ymm12, %ymm3 + vpsubw %ymm5, %ymm13, %ymm5 + vpsubw %ymm8, %ymm14, %ymm8 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1] + vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3] + vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3] + vmovdqa 0x60(%rsi), %ymm2 + vmovdqa 0x80(%rsi), %ymm8 + vpsubw %ymm7, %ymm10, %ymm12 + vpaddw %ymm10, %ymm7, %ymm7 + vpsubw %ymm9, %ymm4, %ymm13 + vpmullw %ymm2, %ymm12, %ymm10 + vpaddw %ymm4, %ymm9, %ymm9 + vpsubw %ymm6, %ymm5, %ymm14 + vpmullw %ymm2, %ymm13, %ymm4 + vpaddw %ymm5, %ymm6, %ymm6 + vpsubw %ymm3, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm5 + vpaddw %ymm11, %ymm3, %ymm3 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm8, %ymm12, %ymm12 + vpmulhw %ymm8, %ymm13, %ymm13 + vpmulhw %ymm8, %ymm14, %ymm14 + vpmulhw %ymm8, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm10, %ymm12, %ymm10 + vpsubw %ymm4, %ymm13, %ymm4 + vpsubw %ymm5, %ymm14, %ymm5 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm6, 0x140(%rdi) + vmovdqa %ymm3, 0x160(%rdi) + vmovdqa %ymm10, 0x180(%rdi) + vmovdqa %ymm4, 0x1a0(%rdi) + vmovdqa %ymm5, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + vmovdqa (%rdi), %ymm4 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x120(%rdi), %ymm9 + vpbroadcastq 0x40(%rsi), %ymm2 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x140(%rdi), %ymm10 + vmovdqa 0x60(%rdi), %ymm7 + vmovdqa 0x160(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vmovdqa %ymm4, (%rdi) + vmovdqa %ymm5, 0x20(%rdi) + vmovdqa %ymm6, 0x40(%rdi) + vmovdqa %ymm7, 0x60(%rdi) + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm11, 0x160(%rdi) + vmovdqa 0x80(%rdi), %ymm4 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm5 + vmovdqa 0x1a0(%rdi), %ymm9 + vpbroadcastq 0x40(%rsi), %ymm2 + vmovdqa 0xc0(%rdi), %ymm6 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm7 + vmovdqa 0x1e0(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vmovdqa %ymm4, 0x80(%rdi) + vmovdqa %ymm5, 0xa0(%rdi) + vmovdqa %ymm6, 0xc0(%rdi) + vmovdqa %ymm7, 0xe0(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa %ymm10, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(invntt_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntt_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntt_avx2_asm.S new file mode 100644 index 0000000..d51ccfb --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntt_avx2_asm.S @@ -0,0 +1,661 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [AVX2_NTT] + * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography. + * Gregor Seiler + * https://eprint.iacr.org/2018/039 + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * The core ideas behind the implementation are described in @[AVX2_NTT]. + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: ntt_avx2_asm + Description: x86_64 AVX2 forward NTT + Signature: void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/ntt_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_avx2_asm) +MLK_ASM_FN_SYMBOL(ntt_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpbroadcastq 0x40(%rsi), %ymm15 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm9 + vmovdqa 0x140(%rdi), %ymm10 + vmovdqa 0x160(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm3, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm5, 0x40(%rdi) + vmovdqa %ymm6, 0x60(%rdi) + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm11, 0x160(%rdi) + vpbroadcastq 0x40(%rsi), %ymm15 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa 0x80(%rdi), %ymm4 + vmovdqa 0xa0(%rdi), %ymm5 + vmovdqa 0xc0(%rdi), %ymm6 + vmovdqa 0xe0(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm3, 0x80(%rdi) + vmovdqa %ymm4, 0xa0(%rdi) + vmovdqa %ymm5, 0xc0(%rdi) + vmovdqa %ymm6, 0xe0(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa %ymm10, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + vmovdqa 0x60(%rsi), %ymm15 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm11 + vmovdqa 0x80(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3] + vmovdqa 0xa0(%rsi), %ymm15 + vmovdqa 0xc0(%rsi), %ymm2 + vpmullw %ymm15, %ymm7, %ymm12 + vpmullw %ymm15, %ymm10, %ymm13 + vpmullw %ymm15, %ymm5, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm7, %ymm6, %ymm4 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm10, %ymm8, %ymm6 + vpsubw %ymm10, %ymm8, %ymm10 + vpaddw %ymm5, %ymm3, %ymm8 + vpsubw %ymm5, %ymm3, %ymm5 + vpaddw %ymm11, %ymm9, %ymm3 + vpsubw %ymm11, %ymm9, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm10, %ymm10 + vpsubw %ymm14, %ymm8, %ymm8 + vpaddw %ymm14, %ymm5, %ymm5 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3] + vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3] + vmovdqa 0xe0(%rsi), %ymm15 + vmovdqa 0x100(%rsi), %ymm2 + vpmullw %ymm15, %ymm9, %ymm12 + vpmullw %ymm15, %ymm5, %ymm13 + vpmullw %ymm15, %ymm8, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm11, %ymm11 + vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2] + vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3] + vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm9, %ymm3, %ymm6 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm5, %ymm7, %ymm3 + vpsubw %ymm5, %ymm7, %ymm5 + vpaddw %ymm8, %ymm4, %ymm7 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm11, %ymm10, %ymm4 + vpsubw %ymm11, %ymm10, %ymm11 + vpsubw %ymm12, %ymm6, %ymm6 + vpaddw %ymm12, %ymm9, %ymm9 + vpsubw %ymm13, %ymm3, %ymm3 + vpaddw %ymm13, %ymm5, %ymm5 + vpsubw %ymm14, %ymm7, %ymm7 + vpaddw %ymm14, %ymm8, %ymm8 + vpsubw %ymm15, %ymm4, %ymm4 + vpaddw %ymm15, %ymm11, %ymm11 + vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7] + vpsrlq $0x20, %ymm4, %ymm4 + vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7] + vmovdqa 0x120(%rsi), %ymm15 + vmovdqa 0x140(%rsi), %ymm2 + vpmullw %ymm15, %ymm10, %ymm12 + vpmullw %ymm15, %ymm8, %ymm13 + vpmullw %ymm15, %ymm7, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7] + vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm3 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm8, %ymm9, %ymm4 + vpsubw %ymm8, %ymm9, %ymm8 + vpaddw %ymm7, %ymm6, %ymm9 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm11, %ymm5, %ymm6 + vpsubw %ymm11, %ymm5, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm8, %ymm8 + vpsubw %ymm14, %ymm9, %ymm9 + vpaddw %ymm14, %ymm7, %ymm7 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vpslld $0x10, %ymm7, %ymm5 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15] + vpslld $0x10, %ymm11, %ymm9 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa 0x160(%rsi), %ymm15 + vmovdqa 0x180(%rsi), %ymm2 + vpmullw %ymm15, %ymm5, %ymm12 + vpmullw %ymm15, %ymm7, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpslld $0x10, %ymm10, %ymm6 + vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15] + vpsrld $0x10, %ymm3, %ymm3 + vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15] + vpslld $0x10, %ymm8, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm5, %ymm6, %ymm4 + vpsubw %ymm5, %ymm6, %ymm5 + vpaddw %ymm7, %ymm10, %ymm6 + vpsubw %ymm7, %ymm10, %ymm7 + vpaddw %ymm9, %ymm3, %ymm10 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm11, %ymm8, %ymm3 + vpsubw %ymm11, %ymm8, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm7, %ymm7 + vpsubw %ymm14, %ymm10, %ymm10 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa 0x1a0(%rsi), %ymm14 + vmovdqa 0x1e0(%rsi), %ymm15 + vmovdqa 0x1c0(%rsi), %ymm8 + vmovdqa 0x200(%rsi), %ymm2 + vpmullw %ymm14, %ymm10, %ymm12 + vpmullw %ymm14, %ymm3, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm8, %ymm10, %ymm10 + vpmulhw %ymm8, %ymm3, %ymm3 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm8 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm3, %ymm6, %ymm4 + vpsubw %ymm3, %ymm6, %ymm3 + vpaddw %ymm9, %ymm5, %ymm6 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm11, %ymm7, %ymm5 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm8, %ymm8 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm3, %ymm3 + vpsubw %ymm14, %ymm6, %ymm6 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm5, %ymm5 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm8, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm10, 0x40(%rdi) + vmovdqa %ymm3, 0x60(%rdi) + vmovdqa %ymm6, 0x80(%rdi) + vmovdqa %ymm5, 0xa0(%rdi) + vmovdqa %ymm9, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + vmovdqa 0x220(%rsi), %ymm15 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vmovdqa 0x240(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x160(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3] + vmovdqa 0x260(%rsi), %ymm15 + vmovdqa 0x280(%rsi), %ymm2 + vpmullw %ymm15, %ymm7, %ymm12 + vpmullw %ymm15, %ymm10, %ymm13 + vpmullw %ymm15, %ymm5, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm7, %ymm6, %ymm4 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm10, %ymm8, %ymm6 + vpsubw %ymm10, %ymm8, %ymm10 + vpaddw %ymm5, %ymm3, %ymm8 + vpsubw %ymm5, %ymm3, %ymm5 + vpaddw %ymm11, %ymm9, %ymm3 + vpsubw %ymm11, %ymm9, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm10, %ymm10 + vpsubw %ymm14, %ymm8, %ymm8 + vpaddw %ymm14, %ymm5, %ymm5 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3] + vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3] + vmovdqa 0x2a0(%rsi), %ymm15 + vmovdqa 0x2c0(%rsi), %ymm2 + vpmullw %ymm15, %ymm9, %ymm12 + vpmullw %ymm15, %ymm5, %ymm13 + vpmullw %ymm15, %ymm8, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm11, %ymm11 + vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2] + vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3] + vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm9, %ymm3, %ymm6 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm5, %ymm7, %ymm3 + vpsubw %ymm5, %ymm7, %ymm5 + vpaddw %ymm8, %ymm4, %ymm7 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm11, %ymm10, %ymm4 + vpsubw %ymm11, %ymm10, %ymm11 + vpsubw %ymm12, %ymm6, %ymm6 + vpaddw %ymm12, %ymm9, %ymm9 + vpsubw %ymm13, %ymm3, %ymm3 + vpaddw %ymm13, %ymm5, %ymm5 + vpsubw %ymm14, %ymm7, %ymm7 + vpaddw %ymm14, %ymm8, %ymm8 + vpsubw %ymm15, %ymm4, %ymm4 + vpaddw %ymm15, %ymm11, %ymm11 + vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7] + vpsrlq $0x20, %ymm4, %ymm4 + vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7] + vmovdqa 0x2e0(%rsi), %ymm15 + vmovdqa 0x300(%rsi), %ymm2 + vpmullw %ymm15, %ymm10, %ymm12 + vpmullw %ymm15, %ymm8, %ymm13 + vpmullw %ymm15, %ymm7, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7] + vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm3 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm8, %ymm9, %ymm4 + vpsubw %ymm8, %ymm9, %ymm8 + vpaddw %ymm7, %ymm6, %ymm9 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm11, %ymm5, %ymm6 + vpsubw %ymm11, %ymm5, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm8, %ymm8 + vpsubw %ymm14, %ymm9, %ymm9 + vpaddw %ymm14, %ymm7, %ymm7 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vpslld $0x10, %ymm7, %ymm5 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15] + vpslld $0x10, %ymm11, %ymm9 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa 0x320(%rsi), %ymm15 + vmovdqa 0x340(%rsi), %ymm2 + vpmullw %ymm15, %ymm5, %ymm12 + vpmullw %ymm15, %ymm7, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpslld $0x10, %ymm10, %ymm6 + vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15] + vpsrld $0x10, %ymm3, %ymm3 + vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15] + vpslld $0x10, %ymm8, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm5, %ymm6, %ymm4 + vpsubw %ymm5, %ymm6, %ymm5 + vpaddw %ymm7, %ymm10, %ymm6 + vpsubw %ymm7, %ymm10, %ymm7 + vpaddw %ymm9, %ymm3, %ymm10 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm11, %ymm8, %ymm3 + vpsubw %ymm11, %ymm8, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm7, %ymm7 + vpsubw %ymm14, %ymm10, %ymm10 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa 0x360(%rsi), %ymm14 + vmovdqa 0x3a0(%rsi), %ymm15 + vmovdqa 0x380(%rsi), %ymm8 + vmovdqa 0x3c0(%rsi), %ymm2 + vpmullw %ymm14, %ymm10, %ymm12 + vpmullw %ymm14, %ymm3, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm8, %ymm10, %ymm10 + vpmulhw %ymm8, %ymm3, %ymm3 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm8 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm3, %ymm6, %ymm4 + vpsubw %ymm3, %ymm6, %ymm3 + vpaddw %ymm9, %ymm5, %ymm6 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm11, %ymm7, %ymm5 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm8, %ymm8 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm3, %ymm3 + vpsubw %ymm14, %ymm6, %ymm6 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm5, %ymm5 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm4, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm3, 0x160(%rdi) + vmovdqa %ymm6, 0x180(%rdi) + vmovdqa %ymm5, 0x1a0(%rdi) + vmovdqa %ymm9, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(ntt_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttfrombytes_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttfrombytes_avx2_asm.S new file mode 100644 index 0000000..3d83397 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttfrombytes_avx2_asm.S @@ -0,0 +1,215 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: nttfrombytes_avx2_asm + Description: x86_64 AVX2 polynomial deserialization in NTT domain + Signature: void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 384 + permissions: read-only + c_parameter: const uint8_t *a + description: Input byte array (MLKEM_POLYBYTES = 384) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/nttfrombytes_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(nttfrombytes_avx2_asm) +MLK_ASM_FN_SYMBOL(nttfrombytes_avx2_asm) + + .cfi_startproc + movl $0xfff0fff, %eax # imm = 0xFFF0FFF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqu (%rsi), %ymm4 + vmovdqu 0x20(%rsi), %ymm5 + vmovdqu 0x40(%rsi), %ymm6 + vmovdqu 0x60(%rsi), %ymm7 + vmovdqu 0x80(%rsi), %ymm8 + vmovdqu 0xa0(%rsi), %ymm9 + vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1] + vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3] + vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3] + vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3] + vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] + vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3] + vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7] + vpsrlq $0x20, %ymm8, %ymm8 + vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7] + vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7] + vpslld $0x10, %ymm7, %ymm10 + vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15] + vpslld $0x10, %ymm8, %ymm4 + vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15] + vpsrld $0x10, %ymm5, %ymm5 + vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15] + vpslld $0x10, %ymm9, %ymm5 + vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrlw $0xc, %ymm10, %ymm11 + vpsllw $0x4, %ymm7, %ymm12 + vpor %ymm11, %ymm12, %ymm11 + vpand %ymm0, %ymm10, %ymm10 + vpand %ymm0, %ymm11, %ymm11 + vpsrlw $0x8, %ymm7, %ymm12 + vpsllw $0x8, %ymm4, %ymm13 + vpor %ymm12, %ymm13, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpsrlw $0x4, %ymm4, %ymm13 + vpand %ymm0, %ymm13, %ymm13 + vpsrlw $0xc, %ymm8, %ymm14 + vpsllw $0x4, %ymm5, %ymm15 + vpor %ymm14, %ymm15, %ymm14 + vpand %ymm0, %ymm8, %ymm8 + vpand %ymm0, %ymm14, %ymm14 + vpsrlw $0x8, %ymm5, %ymm15 + vpsllw $0x8, %ymm9, %ymm1 + vpor %ymm15, %ymm1, %ymm15 + vpand %ymm0, %ymm15, %ymm15 + vpsrlw $0x4, %ymm9, %ymm1 + vpand %ymm0, %ymm1, %ymm1 + vmovdqa %ymm10, (%rdi) + vmovdqa %ymm11, 0x20(%rdi) + vmovdqa %ymm12, 0x40(%rdi) + vmovdqa %ymm13, 0x60(%rdi) + vmovdqa %ymm8, 0x80(%rdi) + vmovdqa %ymm14, 0xa0(%rdi) + vmovdqa %ymm15, 0xc0(%rdi) + vmovdqa %ymm1, 0xe0(%rdi) + vmovdqu 0xc0(%rsi), %ymm4 + vmovdqu 0xe0(%rsi), %ymm5 + vmovdqu 0x100(%rsi), %ymm6 + vmovdqu 0x120(%rsi), %ymm7 + vmovdqu 0x140(%rsi), %ymm8 + vmovdqu 0x160(%rsi), %ymm9 + vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1] + vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3] + vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3] + vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3] + vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] + vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3] + vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7] + vpsrlq $0x20, %ymm8, %ymm8 + vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7] + vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7] + vpslld $0x10, %ymm7, %ymm10 + vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15] + vpslld $0x10, %ymm8, %ymm4 + vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15] + vpsrld $0x10, %ymm5, %ymm5 + vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15] + vpslld $0x10, %ymm9, %ymm5 + vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrlw $0xc, %ymm10, %ymm11 + vpsllw $0x4, %ymm7, %ymm12 + vpor %ymm11, %ymm12, %ymm11 + vpand %ymm0, %ymm10, %ymm10 + vpand %ymm0, %ymm11, %ymm11 + vpsrlw $0x8, %ymm7, %ymm12 + vpsllw $0x8, %ymm4, %ymm13 + vpor %ymm12, %ymm13, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpsrlw $0x4, %ymm4, %ymm13 + vpand %ymm0, %ymm13, %ymm13 + vpsrlw $0xc, %ymm8, %ymm14 + vpsllw $0x4, %ymm5, %ymm15 + vpor %ymm14, %ymm15, %ymm14 + vpand %ymm0, %ymm8, %ymm8 + vpand %ymm0, %ymm14, %ymm14 + vpsrlw $0x8, %ymm5, %ymm15 + vpsllw $0x8, %ymm9, %ymm1 + vpor %ymm15, %ymm1, %ymm15 + vpand %ymm0, %ymm15, %ymm15 + vpsrlw $0x4, %ymm9, %ymm1 + vpand %ymm0, %ymm1, %ymm1 + vmovdqa %ymm10, 0x100(%rdi) + vmovdqa %ymm11, 0x120(%rdi) + vmovdqa %ymm12, 0x140(%rdi) + vmovdqa %ymm13, 0x160(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm14, 0x1a0(%rdi) + vmovdqa %ymm15, 0x1c0(%rdi) + vmovdqa %ymm1, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(nttfrombytes_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntttobytes_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntttobytes_avx2_asm.S new file mode 100644 index 0000000..1415d44 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/ntttobytes_avx2_asm.S @@ -0,0 +1,203 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: ntttobytes_avx2_asm + Description: x86_64 AVX2 polynomial serialization in NTT domain + Signature: void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 384 + permissions: write-only + c_parameter: uint8_t *r + description: Output byte array (MLKEM_POLYBYTES = 384) + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/ntttobytes_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntttobytes_avx2_asm) +MLK_ASM_FN_SYMBOL(ntttobytes_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa (%rsi), %ymm5 + vmovdqa 0x20(%rsi), %ymm6 + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vmovdqa 0x80(%rsi), %ymm9 + vmovdqa 0xa0(%rsi), %ymm10 + vmovdqa 0xc0(%rsi), %ymm11 + vmovdqa 0xe0(%rsi), %ymm12 + vpsllw $0xc, %ymm6, %ymm4 + vpor %ymm4, %ymm5, %ymm4 + vpsrlw $0x4, %ymm6, %ymm5 + vpsllw $0x8, %ymm7, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + vpsrlw $0x8, %ymm7, %ymm6 + vpsllw $0x4, %ymm8, %ymm7 + vpor %ymm6, %ymm7, %ymm6 + vpsllw $0xc, %ymm10, %ymm7 + vpor %ymm7, %ymm9, %ymm7 + vpsrlw $0x4, %ymm10, %ymm8 + vpsllw $0x8, %ymm11, %ymm9 + vpor %ymm8, %ymm9, %ymm8 + vpsrlw $0x8, %ymm11, %ymm9 + vpsllw $0x4, %ymm12, %ymm10 + vpor %ymm9, %ymm10, %ymm9 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] + vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vmovdqu %ymm5, (%rdi) + vmovdqu %ymm7, 0x20(%rdi) + vmovdqu %ymm6, 0x40(%rdi) + vmovdqu %ymm8, 0x60(%rdi) + vmovdqu %ymm3, 0x80(%rdi) + vmovdqu %ymm9, 0xa0(%rdi) + vmovdqa 0x100(%rsi), %ymm5 + vmovdqa 0x120(%rsi), %ymm6 + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vmovdqa 0x180(%rsi), %ymm9 + vmovdqa 0x1a0(%rsi), %ymm10 + vmovdqa 0x1c0(%rsi), %ymm11 + vmovdqa 0x1e0(%rsi), %ymm12 + vpsllw $0xc, %ymm6, %ymm4 + vpor %ymm4, %ymm5, %ymm4 + vpsrlw $0x4, %ymm6, %ymm5 + vpsllw $0x8, %ymm7, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + vpsrlw $0x8, %ymm7, %ymm6 + vpsllw $0x4, %ymm8, %ymm7 + vpor %ymm6, %ymm7, %ymm6 + vpsllw $0xc, %ymm10, %ymm7 + vpor %ymm7, %ymm9, %ymm7 + vpsrlw $0x4, %ymm10, %ymm8 + vpsllw $0x8, %ymm11, %ymm9 + vpor %ymm8, %ymm9, %ymm8 + vpsrlw $0x8, %ymm11, %ymm9 + vpsllw $0x4, %ymm12, %ymm10 + vpor %ymm9, %ymm10, %ymm9 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] + vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vmovdqu %ymm5, 0xc0(%rdi) + vmovdqu %ymm7, 0xe0(%rdi) + vmovdqu %ymm6, 0x100(%rdi) + vmovdqu %ymm8, 0x120(%rdi) + vmovdqu %ymm3, 0x140(%rdi) + vmovdqu %ymm9, 0x160(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(ntttobytes_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttunpack_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttunpack_avx2_asm.S new file mode 100644 index 0000000..97b04fa --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/nttunpack_avx2_asm.S @@ -0,0 +1,190 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: nttunpack_avx2_asm + Description: x86_64 AVX2 NTT unpack from custom to bitreversed order + Signature: void mlk_nttunpack_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/nttunpack_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(nttunpack_avx2_asm) +MLK_ASM_FN_SYMBOL(nttunpack_avx2_asm) + + .cfi_startproc + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm11 + vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3] + vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3] + vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] + vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3] + vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2] + vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] + vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3] + vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7] + vpsrlq $0x20, %ymm10, %ymm10 + vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7] + vpslld $0x10, %ymm5, %ymm10 + vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpslld $0x10, %ymm4, %ymm9 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] + vpslld $0x10, %ymm3, %ymm8 + vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15] + vpsrld $0x10, %ymm7, %ymm7 + vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15] + vpslld $0x10, %ymm11, %ymm7 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa %ymm10, (%rdi) + vmovdqa %ymm5, 0x20(%rdi) + vmovdqa %ymm9, 0x40(%rdi) + vmovdqa %ymm4, 0x60(%rdi) + vmovdqa %ymm8, 0x80(%rdi) + vmovdqa %ymm3, 0xa0(%rdi) + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x160(%rdi), %ymm7 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3] + vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3] + vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] + vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3] + vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2] + vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] + vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3] + vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7] + vpsrlq $0x20, %ymm10, %ymm10 + vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7] + vpslld $0x10, %ymm5, %ymm10 + vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpslld $0x10, %ymm4, %ymm9 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] + vpslld $0x10, %ymm3, %ymm8 + vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15] + vpsrld $0x10, %ymm7, %ymm7 + vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15] + vpslld $0x10, %ymm11, %ymm7 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa %ymm10, 0x100(%rdi) + vmovdqa %ymm5, 0x120(%rdi) + vmovdqa %ymm9, 0x140(%rdi) + vmovdqa %ymm4, 0x160(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm3, 0x1a0(%rdi) + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(nttunpack_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d10_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d10_avx2_asm.S new file mode 100644 index 0000000..4994ecc --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d10_avx2_asm.S @@ -0,0 +1,410 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d10_avx2_asm + * + * Description: Compression of a polynomial to 10 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D10) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_compress_d10_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=10) + Signature: void mlk_poly_compress_d10_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 320 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_compress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d10_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d10_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpsllw $0x3, %ymm0, %ymm1 + movl $0xf000f, %eax # imm = 0xF000F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x10001000, %eax # imm = 0x10001000 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x3ff03ff, %eax # imm = 0x3FF03FF + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movabsq $0x400000104000001, %rax # imm = 0x400000104000001 + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + movl $0xc, %eax + vmovq %rax, %xmm6 + vpbroadcastq %xmm6, %ymm6 + vmovdqa (%rdx), %ymm7 + vmovdqa (%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, (%rdi) + vmovd %xmm9, 0x10(%rdi) + vmovdqa 0x20(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x14(%rdi) + vmovd %xmm9, 0x24(%rdi) + vmovdqa 0x40(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x28(%rdi) + vmovd %xmm9, 0x38(%rdi) + vmovdqa 0x60(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x3c(%rdi) + vmovd %xmm9, 0x4c(%rdi) + vmovdqa 0x80(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x50(%rdi) + vmovd %xmm9, 0x60(%rdi) + vmovdqa 0xa0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x64(%rdi) + vmovd %xmm9, 0x74(%rdi) + vmovdqa 0xc0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x78(%rdi) + vmovd %xmm9, 0x88(%rdi) + vmovdqa 0xe0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x8c(%rdi) + vmovd %xmm9, 0x9c(%rdi) + vmovdqa 0x100(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xa0(%rdi) + vmovd %xmm9, 0xb0(%rdi) + vmovdqa 0x120(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xb4(%rdi) + vmovd %xmm9, 0xc4(%rdi) + vmovdqa 0x140(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xc8(%rdi) + vmovd %xmm9, 0xd8(%rdi) + vmovdqa 0x160(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xdc(%rdi) + vmovd %xmm9, 0xec(%rdi) + vmovdqa 0x180(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xf0(%rdi) + vmovd %xmm9, 0x100(%rdi) + vmovdqa 0x1a0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x104(%rdi) + vmovd %xmm9, 0x114(%rdi) + vmovdqa 0x1c0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x118(%rdi) + vmovd %xmm9, 0x128(%rdi) + vmovdqa 0x1e0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x12c(%rdi) + vmovd %xmm9, 0x13c(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d10_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == \ + 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d11_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d11_avx2_asm.S new file mode 100644 index 0000000..adcab50 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d11_avx2_asm.S @@ -0,0 +1,476 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d11_avx2_asm + * + * Description: Compression of a polynomial to 11 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D11) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to constants + * (srlvqidx[0:32], shufbidx[32:64]) + **************************************************/ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_compress_d11_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=11) + Signature: void mlk_poly_compress_d11_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 352 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 64 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_compress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d11_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d11_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpsllw $0x3, %ymm0, %ymm1 + movl $0x240024, %eax # imm = 0x240024 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x20002000, %eax # imm = 0x20002000 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x7ff07ff, %eax # imm = 0x7FF07FF + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movabsq $0x800000108000001, %rax # imm = 0x800000108000001 + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + movl $0xa, %eax + vmovq %rax, %xmm6 + vpbroadcastq %xmm6, %ymm6 + vmovdqa (%rdx), %ymm7 + vmovdqa 0x20(%rdx), %ymm8 + vmovdqa (%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, (%rdi) + vmovd %xmm10, 0x10(%rdi) + vpextrw $0x2, %xmm10, 0x14(%rdi) + vmovdqa 0x20(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x16(%rdi) + vmovd %xmm10, 0x26(%rdi) + vpextrw $0x2, %xmm10, 0x2a(%rdi) + vmovdqa 0x40(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x2c(%rdi) + vmovd %xmm10, 0x3c(%rdi) + vpextrw $0x2, %xmm10, 0x40(%rdi) + vmovdqa 0x60(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x42(%rdi) + vmovd %xmm10, 0x52(%rdi) + vpextrw $0x2, %xmm10, 0x56(%rdi) + vmovdqa 0x80(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x58(%rdi) + vmovd %xmm10, 0x68(%rdi) + vpextrw $0x2, %xmm10, 0x6c(%rdi) + vmovdqa 0xa0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x6e(%rdi) + vmovd %xmm10, 0x7e(%rdi) + vpextrw $0x2, %xmm10, 0x82(%rdi) + vmovdqa 0xc0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x84(%rdi) + vmovd %xmm10, 0x94(%rdi) + vpextrw $0x2, %xmm10, 0x98(%rdi) + vmovdqa 0xe0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x9a(%rdi) + vmovd %xmm10, 0xaa(%rdi) + vpextrw $0x2, %xmm10, 0xae(%rdi) + vmovdqa 0x100(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xb0(%rdi) + vmovd %xmm10, 0xc0(%rdi) + vpextrw $0x2, %xmm10, 0xc4(%rdi) + vmovdqa 0x120(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xc6(%rdi) + vmovd %xmm10, 0xd6(%rdi) + vpextrw $0x2, %xmm10, 0xda(%rdi) + vmovdqa 0x140(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xdc(%rdi) + vmovd %xmm10, 0xec(%rdi) + vpextrw $0x2, %xmm10, 0xf0(%rdi) + vmovdqa 0x160(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xf2(%rdi) + vmovd %xmm10, 0x102(%rdi) + vpextrw $0x2, %xmm10, 0x106(%rdi) + vmovdqa 0x180(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x108(%rdi) + vmovd %xmm10, 0x118(%rdi) + vpextrw $0x2, %xmm10, 0x11c(%rdi) + vmovdqa 0x1a0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x11e(%rdi) + vmovd %xmm10, 0x12e(%rdi) + vpextrw $0x2, %xmm10, 0x132(%rdi) + vmovdqa 0x1c0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x134(%rdi) + vmovd %xmm10, 0x144(%rdi) + vpextrw $0x2, %xmm10, 0x148(%rdi) + vmovdqa 0x1e0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x14a(%rdi) + vmovd %xmm10, 0x15a(%rdi) + vpextrw $0x2, %xmm10, 0x15e(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d11_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d4_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d4_avx2_asm.S new file mode 100644 index 0000000..3105ad0 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d4_avx2_asm.S @@ -0,0 +1,191 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d4_avx2_asm + * + * Description: Compression of a polynomial to 4 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D4) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to permdidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_compress_d4_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=4) + Signature: void mlk_poly_compress_d4_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 128 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_compress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d4_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d4_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x2000200, %eax # imm = 0x2000200 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0xf000f, %eax # imm = 0xF000F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x10011001, %eax # imm = 0x10011001 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa (%rsi), %ymm5 + vmovdqa 0x20(%rsi), %ymm6 + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, (%rdi) + vmovdqa 0x80(%rsi), %ymm5 + vmovdqa 0xa0(%rsi), %ymm6 + vmovdqa 0xc0(%rsi), %ymm7 + vmovdqa 0xe0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x20(%rdi) + vmovdqa 0x100(%rsi), %ymm5 + vmovdqa 0x120(%rsi), %ymm6 + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x40(%rdi) + vmovdqa 0x180(%rsi), %ymm5 + vmovdqa 0x1a0(%rsi), %ymm6 + vmovdqa 0x1c0(%rsi), %ymm7 + vmovdqa 0x1e0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x60(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == \ + 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d5_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d5_avx2_asm.S new file mode 100644 index 0000000..87fdf39 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_compress_d5_avx2_asm.S @@ -0,0 +1,248 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d5_avx2_asm + * + * Description: Compression of a polynomial to 5 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D5) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_compress_d5_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=5) + Signature: void mlk_poly_compress_d5_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 160 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_compress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d5_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d5_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4000400, %eax # imm = 0x4000400 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x1f001f, %eax # imm = 0x1F001F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x20012001, %eax # imm = 0x20012001 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x4000001, %eax # imm = 0x4000001 + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movl $0xc, %eax + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + vmovdqa (%rdx), %ymm6 + vmovdqa (%rsi), %ymm7 + vmovdqa 0x20(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, (%rdi) + vmovd %xmm8, 0x10(%rdi) + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x14(%rdi) + vmovd %xmm8, 0x24(%rdi) + vmovdqa 0x80(%rsi), %ymm7 + vmovdqa 0xa0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x28(%rdi) + vmovd %xmm8, 0x38(%rdi) + vmovdqa 0xc0(%rsi), %ymm7 + vmovdqa 0xe0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x3c(%rdi) + vmovd %xmm8, 0x4c(%rdi) + vmovdqa 0x100(%rsi), %ymm7 + vmovdqa 0x120(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x50(%rdi) + vmovd %xmm8, 0x60(%rdi) + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x64(%rdi) + vmovd %xmm8, 0x74(%rdi) + vmovdqa 0x180(%rsi), %ymm7 + vmovdqa 0x1a0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x78(%rdi) + vmovd %xmm8, 0x88(%rdi) + vmovdqa 0x1c0(%rsi), %ymm7 + vmovdqa 0x1e0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x8c(%rdi) + vmovd %xmm8, 0x9c(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d5_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d10_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d10_avx2_asm.S new file mode 100644 index 0000000..3974a22 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d10_avx2_asm.S @@ -0,0 +1,256 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d10_avx2_asm + * + * Description: Decompression of a polynomial from 10 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D10) + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_decompress_d10_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=10) + Signature: void mlk_poly_decompress_d10_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 320 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_decompress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d10_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d10_avx2_asm) + + .cfi_startproc + movl $0xd013404, %eax # imm = 0xD013404 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4, %eax + vmovq %rax, %xmm1 + vpbroadcastq %xmm1, %ymm1 + movl $0x7fe01ff8, %eax # imm = 0x7FE01FF8 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdx), %ymm3 + vmovdqu (%rsi), %xmm4 + vmovd 0x10(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovdqu 0x14(%rsi), %xmm4 + vmovd 0x24(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovdqu 0x28(%rsi), %xmm4 + vmovd 0x38(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovdqu 0x3c(%rsi), %xmm4 + vmovd 0x4c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovdqu 0x50(%rsi), %xmm4 + vmovd 0x60(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovdqu 0x64(%rsi), %xmm4 + vmovd 0x74(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovdqu 0x78(%rsi), %xmm4 + vmovd 0x88(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovdqu 0x8c(%rsi), %xmm4 + vmovd 0x9c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovdqu 0xa0(%rsi), %xmm4 + vmovd 0xb0(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovdqu 0xb4(%rsi), %xmm4 + vmovd 0xc4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovdqu 0xc8(%rsi), %xmm4 + vmovd 0xd8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovdqu 0xdc(%rsi), %xmm4 + vmovd 0xec(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovdqu 0xf0(%rsi), %xmm4 + vmovd 0x100(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovdqu 0x104(%rsi), %xmm4 + vmovd 0x114(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovdqu 0x118(%rsi), %xmm4 + vmovd 0x128(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovdqu 0x12c(%rsi), %xmm4 + vmovd 0x13c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d10_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == \ + 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d11_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d11_avx2_asm.S new file mode 100644 index 0000000..48cb02e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d11_avx2_asm.S @@ -0,0 +1,305 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d11_avx2_asm + * + * Description: Decompression of a polynomial from 11 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D11) + * - const uint8_t *data: pointer to constants + * (shufbidx[0:32], srlvdidx[32:64], + * srlvqidx[64:96], shift[96:128]) + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_decompress_d11_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=11) + Signature: void mlk_poly_decompress_d11_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 352 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 128 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_decompress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d11_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d11_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x7ff07ff0, %eax # imm = 0x7FF07FF0 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rdx), %ymm2 + vmovdqa 0x20(%rdx), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqu (%rsi), %xmm6 + vmovd 0x10(%rsi), %xmm7 + vpinsrw $0x2, 0x14(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, (%rdi) + vmovdqu 0x16(%rsi), %xmm6 + vmovd 0x26(%rsi), %xmm7 + vpinsrw $0x2, 0x2a(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x20(%rdi) + vmovdqu 0x2c(%rsi), %xmm6 + vmovd 0x3c(%rsi), %xmm7 + vpinsrw $0x2, 0x40(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x40(%rdi) + vmovdqu 0x42(%rsi), %xmm6 + vmovd 0x52(%rsi), %xmm7 + vpinsrw $0x2, 0x56(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x60(%rdi) + vmovdqu 0x58(%rsi), %xmm6 + vmovd 0x68(%rsi), %xmm7 + vpinsrw $0x2, 0x6c(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x80(%rdi) + vmovdqu 0x6e(%rsi), %xmm6 + vmovd 0x7e(%rsi), %xmm7 + vpinsrw $0x2, 0x82(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xa0(%rdi) + vmovdqu 0x84(%rsi), %xmm6 + vmovd 0x94(%rsi), %xmm7 + vpinsrw $0x2, 0x98(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xc0(%rdi) + vmovdqu 0x9a(%rsi), %xmm6 + vmovd 0xaa(%rsi), %xmm7 + vpinsrw $0x2, 0xae(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xe0(%rdi) + vmovdqu 0xb0(%rsi), %xmm6 + vmovd 0xc0(%rsi), %xmm7 + vpinsrw $0x2, 0xc4(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x100(%rdi) + vmovdqu 0xc6(%rsi), %xmm6 + vmovd 0xd6(%rsi), %xmm7 + vpinsrw $0x2, 0xda(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x120(%rdi) + vmovdqu 0xdc(%rsi), %xmm6 + vmovd 0xec(%rsi), %xmm7 + vpinsrw $0x2, 0xf0(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x140(%rdi) + vmovdqu 0xf2(%rsi), %xmm6 + vmovd 0x102(%rsi), %xmm7 + vpinsrw $0x2, 0x106(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x160(%rdi) + vmovdqu 0x108(%rsi), %xmm6 + vmovd 0x118(%rsi), %xmm7 + vpinsrw $0x2, 0x11c(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x180(%rdi) + vmovdqu 0x11e(%rsi), %xmm6 + vmovd 0x12e(%rsi), %xmm7 + vpinsrw $0x2, 0x132(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1a0(%rdi) + vmovdqu 0x134(%rsi), %xmm6 + vmovd 0x144(%rsi), %xmm7 + vpinsrw $0x2, 0x148(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1c0(%rdi) + vmovdqu 0x14a(%rsi), %xmm6 + vmovd 0x15a(%rsi), %xmm7 + vpinsrw $0x2, 0x15e(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d11_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d4_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d4_avx2_asm.S new file mode 100644 index 0000000..0aeec7d --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d4_avx2_asm.S @@ -0,0 +1,208 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d4_avx2_asm + * + * Description: Decompression of a polynomial from 4 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D4) + * - const int8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_decompress_d4_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=4) + Signature: void mlk_poly_decompress_d4_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 128 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_decompress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d4_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d4_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf0000f, %eax # imm = 0xF0000F + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x800800, %eax # imm = 0x800800 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdx), %ymm3 + vmovq (%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovq 0x8(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovq 0x10(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovq 0x18(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovq 0x20(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovq 0x28(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovq 0x30(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovq 0x38(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovq 0x40(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovq 0x48(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovq 0x50(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovq 0x58(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovq 0x60(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovq 0x68(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovq 0x70(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovq 0x78(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == \ + 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d5_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d5_avx2_asm.S new file mode 100644 index 0000000..7f4406e --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_decompress_d5_avx2_asm.S @@ -0,0 +1,220 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d5_avx2_asm + * + * Description: Decompression of a polynomial from 5 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D5) + * - const uint8_t *data: pointer to constants + * (shufbidx[0:32], mask[32:64], shift[64:96]) + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_decompress_d5_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=5) + Signature: void mlk_poly_decompress_d5_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 96 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_decompress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d5_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d5_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa (%rdx), %ymm1 + vmovdqa 0x20(%rdx), %ymm2 + vmovdqa 0x40(%rdx), %ymm3 + vmovq (%rsi), %xmm4 + vpinsrw $0x4, 0x8(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovq 0xa(%rsi), %xmm4 + vpinsrw $0x4, 0x12(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovq 0x14(%rsi), %xmm4 + vpinsrw $0x4, 0x1c(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovq 0x1e(%rsi), %xmm4 + vpinsrw $0x4, 0x26(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovq 0x28(%rsi), %xmm4 + vpinsrw $0x4, 0x30(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovq 0x32(%rsi), %xmm4 + vpinsrw $0x4, 0x3a(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovq 0x3c(%rsi), %xmm4 + vpinsrw $0x4, 0x44(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovq 0x46(%rsi), %xmm4 + vpinsrw $0x4, 0x4e(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovq 0x50(%rsi), %xmm4 + vpinsrw $0x4, 0x58(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovq 0x5a(%rsi), %xmm4 + vpinsrw $0x4, 0x62(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovq 0x64(%rsi), %xmm4 + vpinsrw $0x4, 0x6c(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovq 0x6e(%rsi), %xmm4 + vpinsrw $0x4, 0x76(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovq 0x78(%rsi), %xmm4 + vpinsrw $0x4, 0x80(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovq 0x82(%rsi), %xmm4 + vpinsrw $0x4, 0x8a(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovq 0x8c(%rsi), %xmm4 + vpinsrw $0x4, 0x94(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovq 0x96(%rsi), %xmm4 + vpinsrw $0x4, 0x9e(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d5_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_mulcache_compute_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_mulcache_compute_avx2_asm.S new file mode 100644 index 0000000..3fc9e64 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/poly_mulcache_compute_avx2_asm.S @@ -0,0 +1,118 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: poly_mulcache_compute_avx2_asm + Description: x86_64 AVX2 mulcache computation + Signature: void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 256 + permissions: write-only + c_parameter: int16_t *out + description: Output mulcache (128 x int16_t) + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *in + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/poly_mulcache_compute_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_mulcache_compute_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_mulcache_compute_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa 0x20(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x3e0(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm8, 0x20(%rdi) + vmovdqa 0xa0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x480(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm8, 0x60(%rdi) + vmovdqa 0x120(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x420(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm8, 0xa0(%rdi) + vmovdqa 0x1a0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x4c0(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm8, 0xe0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_mulcache_compute_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S new file mode 100644 index 0000000..8efc8ae --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S @@ -0,0 +1,536 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k2_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=2) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (2 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (2 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (2 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S new file mode 100644 index 0000000..915cad1 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S @@ -0,0 +1,784 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k3_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=3) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (3 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (3 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (3 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x400(%rsi), %ymm2 + vmovdqa 0x420(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x420(%rdx), %ymm5 + vmovdqa 0x200(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x440(%rsi), %ymm2 + vmovdqa 0x460(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm5 + vmovdqa 0x220(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x480(%rsi), %ymm2 + vmovdqa 0x4a0(%rsi), %ymm3 + vmovdqa 0x480(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm5 + vmovdqa 0x240(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x4c0(%rsi), %ymm2 + vmovdqa 0x4e0(%rsi), %ymm3 + vmovdqa 0x4c0(%rdx), %ymm4 + vmovdqa 0x4e0(%rdx), %ymm5 + vmovdqa 0x260(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x500(%rsi), %ymm2 + vmovdqa 0x520(%rsi), %ymm3 + vmovdqa 0x500(%rdx), %ymm4 + vmovdqa 0x520(%rdx), %ymm5 + vmovdqa 0x280(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x540(%rsi), %ymm2 + vmovdqa 0x560(%rsi), %ymm3 + vmovdqa 0x540(%rdx), %ymm4 + vmovdqa 0x560(%rdx), %ymm5 + vmovdqa 0x2a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x580(%rsi), %ymm2 + vmovdqa 0x5a0(%rsi), %ymm3 + vmovdqa 0x580(%rdx), %ymm4 + vmovdqa 0x5a0(%rdx), %ymm5 + vmovdqa 0x2c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x5c0(%rsi), %ymm2 + vmovdqa 0x5e0(%rsi), %ymm3 + vmovdqa 0x5c0(%rdx), %ymm4 + vmovdqa 0x5e0(%rdx), %ymm5 + vmovdqa 0x2e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S new file mode 100644 index 0000000..c350edd --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S @@ -0,0 +1,1032 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k4_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=4) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (4 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (4 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (4 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x400(%rsi), %ymm2 + vmovdqa 0x420(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x420(%rdx), %ymm5 + vmovdqa 0x200(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x440(%rsi), %ymm2 + vmovdqa 0x460(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm5 + vmovdqa 0x220(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x480(%rsi), %ymm2 + vmovdqa 0x4a0(%rsi), %ymm3 + vmovdqa 0x480(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm5 + vmovdqa 0x240(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x4c0(%rsi), %ymm2 + vmovdqa 0x4e0(%rsi), %ymm3 + vmovdqa 0x4c0(%rdx), %ymm4 + vmovdqa 0x4e0(%rdx), %ymm5 + vmovdqa 0x260(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x500(%rsi), %ymm2 + vmovdqa 0x520(%rsi), %ymm3 + vmovdqa 0x500(%rdx), %ymm4 + vmovdqa 0x520(%rdx), %ymm5 + vmovdqa 0x280(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x540(%rsi), %ymm2 + vmovdqa 0x560(%rsi), %ymm3 + vmovdqa 0x540(%rdx), %ymm4 + vmovdqa 0x560(%rdx), %ymm5 + vmovdqa 0x2a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x580(%rsi), %ymm2 + vmovdqa 0x5a0(%rsi), %ymm3 + vmovdqa 0x580(%rdx), %ymm4 + vmovdqa 0x5a0(%rdx), %ymm5 + vmovdqa 0x2c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x5c0(%rsi), %ymm2 + vmovdqa 0x5e0(%rsi), %ymm3 + vmovdqa 0x5c0(%rdx), %ymm4 + vmovdqa 0x5e0(%rdx), %ymm5 + vmovdqa 0x2e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x600(%rsi), %ymm2 + vmovdqa 0x620(%rsi), %ymm3 + vmovdqa 0x600(%rdx), %ymm4 + vmovdqa 0x620(%rdx), %ymm5 + vmovdqa 0x300(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x640(%rsi), %ymm2 + vmovdqa 0x660(%rsi), %ymm3 + vmovdqa 0x640(%rdx), %ymm4 + vmovdqa 0x660(%rdx), %ymm5 + vmovdqa 0x320(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x680(%rsi), %ymm2 + vmovdqa 0x6a0(%rsi), %ymm3 + vmovdqa 0x680(%rdx), %ymm4 + vmovdqa 0x6a0(%rdx), %ymm5 + vmovdqa 0x340(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x6c0(%rsi), %ymm2 + vmovdqa 0x6e0(%rsi), %ymm3 + vmovdqa 0x6c0(%rdx), %ymm4 + vmovdqa 0x6e0(%rdx), %ymm5 + vmovdqa 0x360(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x700(%rsi), %ymm2 + vmovdqa 0x720(%rsi), %ymm3 + vmovdqa 0x700(%rdx), %ymm4 + vmovdqa 0x720(%rdx), %ymm5 + vmovdqa 0x380(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x740(%rsi), %ymm2 + vmovdqa 0x760(%rsi), %ymm3 + vmovdqa 0x740(%rdx), %ymm4 + vmovdqa 0x760(%rdx), %ymm5 + vmovdqa 0x3a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x780(%rsi), %ymm2 + vmovdqa 0x7a0(%rsi), %ymm3 + vmovdqa 0x780(%rdx), %ymm4 + vmovdqa 0x7a0(%rdx), %ymm5 + vmovdqa 0x3c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x7c0(%rsi), %ymm2 + vmovdqa 0x7e0(%rsi), %ymm3 + vmovdqa 0x7c0(%rdx), %ymm4 + vmovdqa 0x7e0(%rdx), %ymm5 + vmovdqa 0x3e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/reduce_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/reduce_avx2_asm.S new file mode 100644 index 0000000..ea869b4 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/reduce_avx2_asm.S @@ -0,0 +1,234 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * Changes: + * - Add call to csub in reduce128_avx to produce outputs + * in [0,1,...,q-1] rather than [0,1,...,q], matching the + * semantics of mlk_poly_reduce(), + * - Use a macro instead of a local function call. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: reduce_avx2_asm + Description: x86_64 AVX2 modular reduction + Signature: void mlk_reduce_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/reduce_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(reduce_avx2_asm) +MLK_ASM_FN_SYMBOL(reduce_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rdi), %ymm2 + vmovdqa 0x20(%rdi), %ymm3 + vmovdqa 0x40(%rdi), %ymm4 + vmovdqa 0x60(%rdi), %ymm5 + vmovdqa 0x80(%rdi), %ymm6 + vmovdqa 0xa0(%rdi), %ymm7 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm9 + vpmulhw %ymm1, %ymm2, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm2, %ymm2 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vpmulhw %ymm1, %ymm4, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmulhw %ymm1, %ymm5, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmulhw %ymm1, %ymm6, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vpmulhw %ymm1, %ymm8, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpsubw %ymm0, %ymm2, %ymm2 + vpsraw $0xf, %ymm2, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm2, %ymm2 + vpsubw %ymm0, %ymm3, %ymm3 + vpsraw $0xf, %ymm3, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm3, %ymm3 + vpsubw %ymm0, %ymm4, %ymm4 + vpsraw $0xf, %ymm4, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm4, %ymm4 + vpsubw %ymm0, %ymm5, %ymm5 + vpsraw $0xf, %ymm5, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm0, %ymm6, %ymm6 + vpsraw $0xf, %ymm6, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm6, %ymm6 + vpsubw %ymm0, %ymm7, %ymm7 + vpsraw $0xf, %ymm7, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm0, %ymm8, %ymm8 + vpsraw $0xf, %ymm8, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm0, %ymm9, %ymm9 + vpsraw $0xf, %ymm9, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm9, %ymm9 + vmovdqa %ymm2, (%rdi) + vmovdqa %ymm3, 0x20(%rdi) + vmovdqa %ymm4, 0x40(%rdi) + vmovdqa %ymm5, 0x60(%rdi) + vmovdqa %ymm6, 0x80(%rdi) + vmovdqa %ymm7, 0xa0(%rdi) + vmovdqa %ymm8, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm2 + vmovdqa 0x120(%rdi), %ymm3 + vmovdqa 0x140(%rdi), %ymm4 + vmovdqa 0x160(%rdi), %ymm5 + vmovdqa 0x180(%rdi), %ymm6 + vmovdqa 0x1a0(%rdi), %ymm7 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm9 + vpmulhw %ymm1, %ymm2, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm2, %ymm2 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vpmulhw %ymm1, %ymm4, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmulhw %ymm1, %ymm5, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmulhw %ymm1, %ymm6, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vpmulhw %ymm1, %ymm8, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpsubw %ymm0, %ymm2, %ymm2 + vpsraw $0xf, %ymm2, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm2, %ymm2 + vpsubw %ymm0, %ymm3, %ymm3 + vpsraw $0xf, %ymm3, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm3, %ymm3 + vpsubw %ymm0, %ymm4, %ymm4 + vpsraw $0xf, %ymm4, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm4, %ymm4 + vpsubw %ymm0, %ymm5, %ymm5 + vpsraw $0xf, %ymm5, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm0, %ymm6, %ymm6 + vpsraw $0xf, %ymm6, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm6, %ymm6 + vpsubw %ymm0, %ymm7, %ymm7 + vpsraw $0xf, %ymm7, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm0, %ymm8, %ymm8 + vpsraw $0xf, %ymm8, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm0, %ymm9, %ymm9 + vpsraw $0xf, %ymm9, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm9, %ymm9 + vmovdqa %ymm2, 0x100(%rdi) + vmovdqa %ymm3, 0x120(%rdi) + vmovdqa %ymm4, 0x140(%rdi) + vmovdqa %ymm5, 0x160(%rdi) + vmovdqa %ymm6, 0x180(%rdi) + vmovdqa %ymm7, 0x1a0(%rdi) + vmovdqa %ymm8, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(reduce_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_avx2_asm.S new file mode 100644 index 0000000..8ecde7a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_avx2_asm.S @@ -0,0 +1,136 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/************************************************* + * Name: mlk_rej_uniform_avx2_asm + * + * Description: Run rejection sampling on uniform random bytes to generate + * uniform random integers mod q + * + * Arguments: - int16_t *r: pointer to output buffer of MLKEM_N + * 16-bit coefficients. + * - const uint8_t *buf: pointer to input buffer + * (assumed to be uniform random bytes) + * - unsigned buflen: length of input buffer in bytes. + * Must be a multiple of 12. + * + * Returns number of sampled 16-bit integers (at most MLKEM_N). + **************************************************/ +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: rej_uniform_avx2_asm + Description: x86_64 AVX2 rejection sampling + Signature: uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf, unsigned buflen, const uint8_t *table) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output buffer (256 x int16_t) + rsi: + type: buffer + size_bytes: rdx + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + rdx: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be multiple of 12) + test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE + rcx: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t *table + description: Lookup table +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/rej_uniform_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(rej_uniform_avx2_asm) +MLK_ASM_FN_SYMBOL(rej_uniform_avx2_asm) + + .cfi_startproc + subq $0x210, %rsp # imm = 0x210 + .cfi_adjust_cfa_offset 0x210 + xorl %eax, %eax + testq %rdx, %rdx + je Lrej_uniform_asm_end + movabsq $0xd010d010d010d01, %rax # imm = 0xD010D010D010D01 + movq %rax, %xmm0 + pinsrq $0x1, %rax, %xmm0 + movabsq $0xfff0fff0fff0fff, %rax # imm = 0xFFF0FFF0FFF0FFF + movq %rax, %xmm5 + pinsrq $0x1, %rax, %xmm5 + movabsq $0x504040302010100, %rax # imm = 0x504040302010100 + movq %rax, %xmm4 + movabsq $0xb0a0a0908070706, %rax # imm = 0xB0A0A0908070706 + pinsrq $0x1, %rax, %xmm4 + movq $0x0, %rax + movq $0x0, %r8 + movq $0x5555, %r9 # imm = 0x5555 + +Lrej_uniform_asm_loop_start: + movq (%rsi,%r8), %xmm2 + pinsrd $0x2, 0x8(%rsi,%r8), %xmm2 + pshufb %xmm4, %xmm2 + movdqa %xmm2, %xmm3 + psrlw $0x4, %xmm3 + pblendw $0xaa, %xmm3, %xmm2 # xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7] + pand %xmm5, %xmm2 + movdqa %xmm0, %xmm1 + pcmpgtw %xmm2, %xmm1 + pmovmskb %xmm1, %r11d + pextq %r9, %r11, %r11 + movq %r11, %r10 + shlq $0x4, %r10 + movdqu (%rcx,%r10), %xmm3 + pshufb %xmm3, %xmm2 + movdqu %xmm2, (%rsp,%rax,2) + popcntq %r11, %r11 + addq %r11, %rax + cmpq $0x100, %rax # imm = 0x100 + jae Lrej_uniform_asm_final_copy + addq $0xc, %r8 + cmpq %r8, %rdx + ja Lrej_uniform_asm_loop_start + +Lrej_uniform_asm_final_copy: + movq $0x100, %rcx # imm = 0x100 + cmpq $0x100, %rax # imm = 0x100 + cmovaq %rcx, %rax + movq %rsp, %rsi + movq %rax, %rcx + shlq %rcx + rep movsb (%rsi), %es:(%rdi) + +Lrej_uniform_asm_end: + addq $0x210, %rsp # imm = 0x210 + .cfi_adjust_cfa_offset -0x210 + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(rej_uniform_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_table.c b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_table.c new file mode 100644 index 0000000..d62256a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/rej_uniform_table.c @@ -0,0 +1,545 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_x86_64.h" + +/* + * Lookup table used by rejection sampling of the public matrix. + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_rej_uniform_table[4096] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 0 */, + 0, 1, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 1 */, + 2, 3, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 2 */, + 0, 1, 2, 3, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 3 */, + 4, 5, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 4 */, + 0, 1, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 5 */, + 2, 3, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 6 */, + 0, 1, 2, 3, 4, 5, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 7 */, + 6, 7, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 8 */, + 0, 1, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 9 */, + 2, 3, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 10 */, + 0, 1, 2, 3, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 11 */, + 4, 5, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 12 */, + 0, 1, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 13 */, + 2, 3, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 14 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 255, 255, 255, 255, 255, 255, 255, 255 /* 15 */, + 8, 9, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 16 */, + 0, 1, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 17 */, + 2, 3, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 18 */, + 0, 1, 2, 3, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 19 */, + 4, 5, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 20 */, + 0, 1, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 21 */, + 2, 3, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 22 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 23 */, + 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 24 */, + 0, 1, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 25 */, + 2, 3, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 26 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 27 */, + 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 28 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 29 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 30 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 255, 255, 255, 255, 255, 255 /* 31 */, + 10, 11, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 32 */, + 0, 1, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 33 */, + 2, 3, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 34 */, + 0, 1, 2, 3, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 35 */, + 4, 5, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 36 */, + 0, 1, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 37 */, + 2, 3, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 38 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 39 */, + 6, 7, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 40 */, + 0, 1, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 41 */, + 2, 3, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 42 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 43 */, + 4, 5, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 44 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 45 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 46 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 255, 255, 255, 255, 255, 255 /* 47 */, + 8, 9, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 48 */, + 0, 1, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 49 */, + 2, 3, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 50 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 51 */, + 4, 5, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 52 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 53 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 54 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 55 */, + 6, 7, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 56 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 57 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 58 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 59 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 60 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 61 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 62 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 255, 255, 255, 255 /* 63 */, + 12, 13, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 64 */, + 0, 1, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 65 */, + 2, 3, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 66 */, + 0, 1, 2, 3, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 67 */, + 4, 5, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 68 */, + 0, 1, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 69 */, + 2, 3, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 70 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 71 */, + 6, 7, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 72 */, + 0, 1, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 73 */, + 2, 3, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 74 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 75 */, + 4, 5, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 76 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 77 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 78 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 255, 255, 255, 255, 255, 255 /* 79 */, + 8, 9, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 80 */, + 0, 1, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 81 */, + 2, 3, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 82 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 83 */, + 4, 5, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 84 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 85 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 86 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 87 */, + 6, 7, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 88 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 89 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 90 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 91 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 92 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 93 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 94 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 255, 255, 255, 255 /* 95 */, + 10, 11, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 96 */, + 0, 1, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 97 */, + 2, 3, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 98 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 99 */, + 4, 5, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 100 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 101 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 102 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 103 */, + 6, 7, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 104 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 105 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 106 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 107 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 108 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 109 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 110 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 255, 255, 255, 255 /* 111 */, + 8, 9, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 112 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 113 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 114 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 115 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 116 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 117 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 118 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 119 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 120 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 121 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 122 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 123 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 124 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 125 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 126 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 255, 255 /* 127 */, + 14, 15, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 128 */, + 0, 1, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 129 */, + 2, 3, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 130 */, + 0, 1, 2, 3, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 131 */, + 4, 5, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 132 */, + 0, 1, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 133 */, + 2, 3, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 134 */, + 0, 1, 2, 3, 4, 5, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 135 */, + 6, 7, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 136 */, + 0, 1, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 137 */, + 2, 3, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 138 */, + 0, 1, 2, 3, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 139 */, + 4, 5, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 140 */, + 0, 1, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 141 */, + 2, 3, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 142 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 14, 15, 255, 255, 255, 255, 255, 255 /* 143 */, + 8, 9, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 144 */, + 0, 1, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 145 */, + 2, 3, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 146 */, + 0, 1, 2, 3, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 147 */, + 4, 5, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 148 */, + 0, 1, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 149 */, + 2, 3, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 150 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 151 */, + 6, 7, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 152 */, + 0, 1, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 153 */, + 2, 3, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 154 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 155 */, + 4, 5, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 156 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 157 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 158 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 14, 15, 255, 255, 255, 255 /* 159 */, + 10, 11, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 160 */, + 0, 1, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 161 */, + 2, 3, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 162 */, + 0, 1, 2, 3, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 163 */, + 4, 5, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 164 */, + 0, 1, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 165 */, + 2, 3, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 166 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 167 */, + 6, 7, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 168 */, + 0, 1, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 169 */, + 2, 3, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 170 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 171 */, + 4, 5, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 172 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 173 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 174 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 255, 255, 255, 255 /* 175 */, + 8, 9, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 176 */, + 0, 1, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 177 */, + 2, 3, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 178 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 179 */, + 4, 5, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 180 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 181 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 182 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 183 */, + 6, 7, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 184 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 185 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 186 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 187 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 188 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 189 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 190 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 255, 255 /* 191 */, + 12, 13, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 192 */, + 0, 1, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 193 */, + 2, 3, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 194 */, + 0, 1, 2, 3, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 195 */, + 4, 5, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 196 */, + 0, 1, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 197 */, + 2, 3, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 198 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 199 */, + 6, 7, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 200 */, + 0, 1, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 201 */, + 2, 3, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 202 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 203 */, + 4, 5, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 204 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 205 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 206 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 255, 255, 255, 255 /* 207 */, + 8, 9, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 208 */, + 0, 1, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 209 */, + 2, 3, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 210 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 211 */, + 4, 5, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 212 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 213 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 214 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 215 */, + 6, 7, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 216 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 217 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 218 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 219 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 220 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 221 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 222 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 255, 255 /* 223 */, + 10, 11, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 224 */, + 0, 1, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 225 */, + 2, 3, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 226 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 227 */, + 4, 5, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 228 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 229 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 230 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 231 */, + 6, 7, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 232 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 233 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 234 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 235 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 236 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 237 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 238 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 255, 255 /* 239 */, + 8, 9, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 240 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 241 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 242 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 243 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 244 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 245 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 246 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 247 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 248 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 249 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 250 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 251 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 252 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 253 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 254 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 /* 255 */, +}; + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(avx2_rej_uniform_table) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/tomont_avx2_asm.S b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/tomont_avx2_asm.S new file mode 100644 index 0000000..04d090c --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/native/x86_64/src/tomont_avx2_asm.S @@ -0,0 +1,171 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * Implementation from Kyber reference repository @[REF_AVX2] + * + * Changes: + * - Add call to csub in reduce128_avx to produce outputs + * in [0,1,...,q-1] rather than [0,1,...,q], matching the + * semantics of mlk_poly_reduce(), + * - Use a macro instead of a local function call. + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: tomont_avx2_asm + Description: x86_64 AVX2 Montgomery conversion + Signature: void mlk_tomont_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/tomont_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(tomont_avx2_asm) +MLK_ASM_FN_SYMBOL(tomont_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x50495049, %eax # imm = 0x50495049 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x5490549, %eax # imm = 0x5490549 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdi), %ymm3 + vmovdqa 0x20(%rdi), %ymm4 + vmovdqa 0x40(%rdi), %ymm5 + vmovdqa 0x60(%rdi), %ymm6 + vmovdqa 0x80(%rdi), %ymm7 + vmovdqa 0xa0(%rdi), %ymm8 + vmovdqa 0xc0(%rdi), %ymm9 + vmovdqa 0xe0(%rdi), %ymm10 + vpmullw %ymm1, %ymm3, %ymm11 + vpmulhw %ymm2, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm3, %ymm3 + vpmullw %ymm1, %ymm4, %ymm12 + vpmulhw %ymm2, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm1, %ymm5, %ymm13 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm5, %ymm5 + vpmullw %ymm1, %ymm6, %ymm14 + vpmulhw %ymm2, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm14, %ymm14 + vpsubw %ymm14, %ymm6, %ymm6 + vpmullw %ymm1, %ymm7, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm15, %ymm15 + vpsubw %ymm15, %ymm7, %ymm7 + vpmullw %ymm1, %ymm8, %ymm11 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm8, %ymm8 + vpmullw %ymm1, %ymm9, %ymm12 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm1, %ymm10, %ymm13 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm10, %ymm10 + vmovdqa %ymm3, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm5, 0x40(%rdi) + vmovdqa %ymm6, 0x60(%rdi) + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm8, 0xa0(%rdi) + vmovdqa %ymm9, 0xc0(%rdi) + vmovdqa %ymm10, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm3 + vmovdqa 0x120(%rdi), %ymm4 + vmovdqa 0x140(%rdi), %ymm5 + vmovdqa 0x160(%rdi), %ymm6 + vmovdqa 0x180(%rdi), %ymm7 + vmovdqa 0x1a0(%rdi), %ymm8 + vmovdqa 0x1c0(%rdi), %ymm9 + vmovdqa 0x1e0(%rdi), %ymm10 + vpmullw %ymm1, %ymm3, %ymm11 + vpmulhw %ymm2, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm3, %ymm3 + vpmullw %ymm1, %ymm4, %ymm12 + vpmulhw %ymm2, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm1, %ymm5, %ymm13 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm5, %ymm5 + vpmullw %ymm1, %ymm6, %ymm14 + vpmulhw %ymm2, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm14, %ymm14 + vpsubw %ymm14, %ymm6, %ymm6 + vpmullw %ymm1, %ymm7, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm15, %ymm15 + vpsubw %ymm15, %ymm7, %ymm7 + vpmullw %ymm1, %ymm8, %ymm11 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm8, %ymm8 + vpmullw %ymm1, %ymm9, %ymm12 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm1, %ymm10, %ymm13 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm10, %ymm10 + vmovdqa %ymm3, 0x100(%rdi) + vmovdqa %ymm4, 0x120(%rdi) + vmovdqa %ymm5, 0x140(%rdi) + vmovdqa %ymm6, 0x160(%rdi) + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm8, 0x1a0(%rdi) + vmovdqa %ymm9, 0x1c0(%rdi) + vmovdqa %ymm10, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(tomont_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/params.h b/sdk/software/examples/mlkem_test/mlkem_native/src/params.h new file mode 100644 index 0000000..0459853 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/params.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_PARAMS_H +#define MLK_PARAMS_H + +#if !defined(MLK_CONFIG_PARAMETER_SET) +#error MLK_CONFIG_PARAMETER_SET is not defined +#endif + +#if MLK_CONFIG_PARAMETER_SET == 512 +#define MLKEM_K 2 +#elif MLK_CONFIG_PARAMETER_SET == 768 +#define MLKEM_K 3 +#elif MLK_CONFIG_PARAMETER_SET == 1024 +#define MLKEM_K 4 +#else +#error Invalid value for MLK_CONFIG_PARAMETER_SET. Must be 512, 768, or 1024. +#endif + +#define MLKEM_N 256 +#define MLKEM_Q 3329 +#define MLKEM_Q_HALF ((MLKEM_Q + 1) / 2) /* 1665 */ +#define MLKEM_UINT12_LIMIT 4096 + +#define MLKEM_SYMBYTES 32 /* size in bytes of hashes, and seeds */ +#define MLKEM_SSBYTES 32 /* size in bytes of shared key */ + +#define MLKEM_POLYBYTES 384 +#define MLKEM_POLYVECBYTES (MLKEM_K * MLKEM_POLYBYTES) + +#define MLKEM_POLYCOMPRESSEDBYTES_D4 128 +#define MLKEM_POLYCOMPRESSEDBYTES_D5 160 +#define MLKEM_POLYCOMPRESSEDBYTES_D10 320 +#define MLKEM_POLYCOMPRESSEDBYTES_D11 352 + +#if MLKEM_K == 2 +#define MLKEM_ETA1 3 +#define MLKEM_DU 10 +#define MLKEM_DV 4 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D4 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D10 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#elif MLKEM_K == 3 +#define MLKEM_ETA1 2 +#define MLKEM_DU 10 +#define MLKEM_DV 4 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D4 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D10 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#elif MLKEM_K == 4 +#define MLKEM_ETA1 2 +#define MLKEM_DU 11 +#define MLKEM_DV 5 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D5 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D11 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#endif /* MLKEM_K == 4 */ + +#define MLKEM_ETA2 2 + +#define MLKEM_INDCPA_MSGBYTES (MLKEM_SYMBYTES) +#define MLKEM_INDCPA_PUBLICKEYBYTES (MLKEM_POLYVECBYTES + MLKEM_SYMBYTES) +#define MLKEM_INDCPA_SECRETKEYBYTES (MLKEM_POLYVECBYTES) +#define MLKEM_INDCPA_BYTES \ + (MLKEM_POLYVECCOMPRESSEDBYTES_DU + MLKEM_POLYCOMPRESSEDBYTES_DV) + +#define MLKEM_INDCCA_PUBLICKEYBYTES (MLKEM_INDCPA_PUBLICKEYBYTES) +/* 32 bytes of additional space to save H(pk) */ +#define MLKEM_INDCCA_SECRETKEYBYTES \ + (MLKEM_INDCPA_SECRETKEYBYTES + MLKEM_INDCPA_PUBLICKEYBYTES + \ + 2 * MLKEM_SYMBYTES) +#define MLKEM_INDCCA_CIPHERTEXTBYTES (MLKEM_INDCPA_BYTES) + +#endif /* !MLK_PARAMS_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/poly.c b/sdk/software/examples/mlkem_test/mlkem_native/src/poly.c new file mode 100644 index 0000000..3a25fdf --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/poly.c @@ -0,0 +1,576 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "cbmc.h" +#include "debug.h" +#include "poly.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/** + * Montgomery multiplication modulo MLKEM_Q. + * + * @reference{`fqmul()` in the reference implementation @[REF].} + * + * @param a First factor. Can be any int16_t. + * @param b Second factor. Must be signed canonical + * (abs value < (MLKEM_Q+1)/2). + * + * @return 16-bit integer congruent to a*b*R^{-1} mod MLKEM_Q, and + * smaller than MLKEM_Q in absolute value. + */ +static MLK_INLINE int16_t mlk_fqmul(int16_t a, int16_t b) +__contract__( + requires(b > -MLKEM_Q_HALF && b < MLKEM_Q_HALF) + ensures(return_value > -MLKEM_Q && return_value < MLKEM_Q) +) +{ + int16_t res; + mlk_assert_abs_bound(&b, 1, MLKEM_Q_HALF); + + res = mlk_montgomery_reduce((int32_t)a * (int32_t)b); + /* Bounds: + * |res| <= ceil(|a| * |b| / 2^16) + (MLKEM_Q + 1) / 2 + * <= ceil(2^15 * ((MLKEM_Q - 1)/2) / 2^16) + (MLKEM_Q + 1) / 2 + * <= ceil((MLKEM_Q - 1) / 4) + (MLKEM_Q + 1) / 2 + * < MLKEM_Q + */ + + mlk_assert_abs_bound(&res, 1, MLKEM_Q); + return res; +} + +/** + * Barrett reduction; given a 16-bit integer a, computes the centered + * representative congruent to a mod MLKEM_Q in [-(MLKEM_Q-1)/2, (MLKEM_Q-1)/2]. + * + * @reference{`barrett_reduce()` in the reference implementation @[REF].} + * + * @param a Input integer to be reduced. + * + * @return Integer in [-(MLKEM_Q-1)/2, (MLKEM_Q-1)/2] congruent to @p a modulo + * MLKEM_Q. + */ +static MLK_INLINE int16_t mlk_barrett_reduce(int16_t a) +__contract__( + ensures(return_value > -MLKEM_Q_HALF && return_value < MLKEM_Q_HALF) +) +{ + /* Barrett reduction approximates + * ``` + * round(a/MLKEM_Q) + * = round(a*(2^N/MLKEM_Q))/2^N) + * ~= round(a*round(2^N/MLKEM_Q)/2^N) + * ``` + * Here, we pick N=26. + */ + const int32_t magic = 20159; /* check-magic: 20159 == round(2^26 / MLKEM_Q) */ + + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + const int32_t t = (magic * a + ((int32_t)1 << 25)) >> 26; + + /* + * t is in -10 .. +10, so we need 32-bit math to + * evaluate t * MLKEM_Q and the subsequent subtraction + */ + int16_t res = (int16_t)(a - t * MLKEM_Q); + + mlk_assert_abs_bound(&res, 1, MLKEM_Q_HALF); + return res; +} + +/* Reference: `poly_tomont()` in the reference implementation @[REF]. */ +MLK_STATIC_TESTABLE void mlk_poly_tomont_c(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) +) +{ + unsigned i; + const int16_t f = 1353; /* check-magic: 1353 == signed_mod(2^32, MLKEM_Q) */ + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(array_abs_bound(r->coeffs, 0, i, MLKEM_Q)) + decreases(MLKEM_N - i)) + { + r->coeffs[i] = mlk_fqmul(r->coeffs[i], f); + } + + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_tomont(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_POLY_TOMONT) + int ret; + ret = mlk_poly_tomont_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_TOMONT */ + + mlk_poly_tomont_c(r); +} + +/** + * Constant-time conversion of signed representatives modulo MLKEM_Q within + * range [-(MLKEM_Q-1), MLKEM_Q-1] into unsigned representatives within + * range [0, MLKEM_Q-1]. + * + * @reference{Not present in the reference implementation @[REF]. Used here + * to implement different semantics of `poly_reduce()`; see below. In the + * reference implementation @[REF] this logic is part of all compression + * functions (see `compress.c`).} + * + * @param c Signed coefficient to be converted. + * + * @return Unsigned representative in [0, MLKEM_Q). + */ +static MLK_INLINE int16_t mlk_scalar_signed_to_unsigned_q(int16_t c) +__contract__( + requires(c > -MLKEM_Q && c < MLKEM_Q) + ensures(return_value >= 0 && return_value < MLKEM_Q) + ensures(return_value == (int32_t)c + (((int32_t)c < 0) * MLKEM_Q))) +{ + mlk_assert_abs_bound(&c, 1, MLKEM_Q); + + /* Add MLKEM_Q if c is negative, but in constant time. + * + * Note that c + MLKEM_Q does not overflow in int16_t, + * so the cast to uint16_t is safe. */ + c = mlk_ct_sel_int16((int16_t)(c + MLKEM_Q), c, mlk_ct_cmask_neg_i16(c)); + + mlk_assert_bound(&c, 1, 0, MLKEM_Q); + return c; +} + +/* Reference: `poly_reduce()` in the reference implementation @[REF] + * - We use _unsigned_ canonical outputs, while the reference + * implementation uses _signed_ canonical outputs. + * Accordingly, we need a conditional addition of MLKEM_Q + * here to go from signed to unsigned representatives. + * This conditional addition is then dropped from all + * polynomial compression functions instead (see `compress.c`). */ +MLK_STATIC_TESTABLE void mlk_poly_reduce_c(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(array_bound(r->coeffs, 0, i, 0, MLKEM_Q)) + decreases(MLKEM_N - i)) + { + /* Barrett reduction, giving signed canonical representative */ + int16_t t = mlk_barrett_reduce(r->coeffs[i]); + /* Conditional addition to get unsigned canonical representative */ + r->coeffs[i] = mlk_scalar_signed_to_unsigned_q(t); + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_reduce(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_POLY_REDUCE) + int ret; + ret = mlk_poly_reduce_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_REDUCE */ + + mlk_poly_reduce_c(r); +} + +/* Reference: `poly_add()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_poly_add(mlk_poly *r, const mlk_poly *b) +{ + unsigned i; + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(forall(k0, i, MLKEM_N, r->coeffs[k0] == loop_entry(*r).coeffs[k0])) + invariant(forall(k1, 0, i, r->coeffs[k1] == loop_entry(*r).coeffs[k1] + b->coeffs[k1])) + decreases(MLKEM_N - i)) + { + /* The preconditions imply that the addition stays within int16_t. */ + r->coeffs[i] = (int16_t)(r->coeffs[i] + b->coeffs[i]); + } +} + +/* Reference: `poly_sub()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_poly_sub(mlk_poly *r, const mlk_poly *b) +{ + unsigned i; + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(forall(k0, i, MLKEM_N, r->coeffs[k0] == loop_entry(*r).coeffs[k0])) + invariant(forall(k1, 0, i, r->coeffs[k1] == loop_entry(*r).coeffs[k1] - b->coeffs[k1])) + decreases(MLKEM_N - i)) + { + /* The preconditions imply that the subtraction stays within int16_t. */ + r->coeffs[i] = (int16_t)(r->coeffs[i] - b->coeffs[i]); + } +} + +#include "zetas.inc" + +/* Reference: Does not exist in the reference implementation @[REF]. + * - The reference implementation does not use a + * multiplication cache ('mulcache'). This idea originates + * from @[NeonNTT] and is used at the C level here. */ +MLK_STATIC_TESTABLE void mlk_poly_mulcache_compute_c(mlk_poly_mulcache *x, + const mlk_poly *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_poly_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_poly))) + assigns(memory_slice(x, sizeof(mlk_poly_mulcache))) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 4; i++) + __loop__( + invariant(i <= MLKEM_N / 4) + invariant(array_abs_bound(x->coeffs, 0, 2 * i, MLKEM_Q)) + decreases(MLKEM_N / 4 - i)) + { + x->coeffs[2 * i + 0] = mlk_fqmul(a->coeffs[4 * i + 1], mlk_zetas[64 + i]); + /* The values in zeta table are <= MLKEM_Q in absolute value, + * so the negation in int16_t is safe. */ + x->coeffs[2 * i + 1] = + mlk_fqmul(a->coeffs[4 * i + 3], (int16_t)(-mlk_zetas[64 + i])); + } + + /* + * This bound is true for the C implementation, but not needed + * in the higher level bounds reasoning. It is thus omitted + * from the spec to not unnecessarily constrain native + * implementations, but checked here nonetheless. + */ + mlk_assert_abs_bound(x, MLKEM_N / 2, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_mulcache_compute(mlk_poly_mulcache *x, const mlk_poly *a) +{ +#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) + int ret; + ret = mlk_poly_mulcache_compute_native(x->coeffs, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */ + + mlk_poly_mulcache_compute_c(x, a); +} + +/* + * Computes a block CT butterflies with a fixed twiddle factor, + * using Montgomery multiplication. + * Parameters: + * - r: Pointer to base of polynomial (_not_ the base of butterfly block) + * - root: Twiddle factor to use for the butterfly. This must be in + * Montgomery form and signed canonical. + * - start: Offset to the beginning of the butterfly block + * - len: Index difference between coefficients subject to a butterfly + * - bound: Ghost variable describing coefficient bound: Prior to `start`, + * coefficients must be bound by `bound + MLKEM_Q`. Post `start`, + * they must be bound by `bound`. + * When this function returns, output coefficients in the index range + * [start, start+2*len) have bound bumped to `bound + MLKEM_Q`. + * Example: + * - start=8, len=4 + * This would compute the following four butterflies + * 8 -- 12 + * 9 -- 13 + * 10 -- 14 + * 11 -- 15 + * - start=4, len=2 + * This would compute the following two butterflies + * 4 -- 6 + * 5 -- 7 + */ + +/* Reference: Embedded in `ntt()` in the reference implementation @[REF]. */ +static void mlk_ntt_butterfly_block(int16_t r[MLKEM_N], int16_t zeta, + unsigned start, unsigned len, + unsigned bound) +__contract__( + requires(start < MLKEM_N) + requires(1 <= len && len <= MLKEM_N / 2 && start + 2 * len <= MLKEM_N) + requires(0 <= bound && bound < INT16_MAX - MLKEM_Q) + requires(-MLKEM_Q_HALF < zeta && zeta < MLKEM_Q_HALF) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(r, 0, start, bound + MLKEM_Q)) + requires(array_abs_bound(r, start, MLKEM_N, bound)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, start + 2*len, bound + MLKEM_Q)) + ensures(array_abs_bound(r, start + 2 * len, MLKEM_N, bound))) +{ + /* `bound` is a ghost variable only needed in the CBMC specification */ + unsigned j; + ((void)bound); + for (j = start; j < start + len; j++) + __loop__( + invariant(start <= j && j <= start + len) + /* + * Coefficients are updated in strided pairs, so the bounds for the + * intermediate states alternate twice between the old and new bound + */ + invariant(array_abs_bound(r, 0, j, bound + MLKEM_Q)) + invariant(array_abs_bound(r, j, start + len, bound)) + invariant(array_abs_bound(r, start + len, j + len, bound + MLKEM_Q)) + invariant(array_abs_bound(r, j + len, MLKEM_N, bound)) + decreases(start + len - j)) + { + int16_t t; + t = mlk_fqmul(r[j + len], zeta); + /* The precondition implies that the arithmetic does not overflow. */ + r[j + len] = (int16_t)(r[j] - t); + r[j] = (int16_t)(r[j] + t); + } +} + +/* + * Compute one layer of forward NTT + * Parameters: + * - r: Pointer to base of polynomial + * - layer: Variable indicating which layer is being applied. + */ + +/* Reference: Embedded in `ntt()` in the reference implementation @[REF]. */ +static void mlk_ntt_layer(int16_t r[MLKEM_N], unsigned layer) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(1 <= layer && layer <= 7) + requires(array_abs_bound(r, 0, MLKEM_N, layer * MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, (layer + 1) * MLKEM_Q))) +{ + unsigned start, k, len; + /* Twiddle factors for layer n are at indices 2^(n-1)..2^n-1. */ + k = 1u << (layer - 1); + len = (unsigned)MLKEM_N >> layer; + for (start = 0; start < MLKEM_N; start += 2 * len) + __loop__( + invariant(start < MLKEM_N + 2 * len) + invariant(k <= MLKEM_N / 2 && 2 * len * k == start + MLKEM_N) + invariant(array_abs_bound(r, 0, start, layer * MLKEM_Q + MLKEM_Q)) + invariant(array_abs_bound(r, start, MLKEM_N, layer * MLKEM_Q)) + decreases(MLKEM_N - start)) + { + int16_t zeta = mlk_zetas[k++]; + mlk_ntt_butterfly_block(r, zeta, start, len, layer * MLKEM_Q); + } +} + +/* + * Compute full forward NTT + * NOTE: This particular implementation satisfies a much tighter + * bound on the output coefficients (5*q) than the contractual one (8*q), + * but this is not needed in the calling code. Should we change the + * base multiplication strategy to require smaller NTT output bounds, + * the proof may need strengthening. + */ + +/* Reference: `ntt()` in the reference implementation @[REF]. + * - Iterate over `layer` instead of `len` in the outer loop + * to simplify computation of zeta index. */ +MLK_STATIC_TESTABLE void mlk_poly_ntt_c(mlk_poly *p) +__contract__( + requires(memory_no_alias(p, sizeof(mlk_poly))) + requires(array_abs_bound(p->coeffs, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(p, sizeof(mlk_poly))) + ensures(array_abs_bound(p->coeffs, 0, MLKEM_N, MLK_NTT_BOUND)) +) +{ + unsigned layer; + int16_t *r; + + mlk_assert_abs_bound(p, MLKEM_N, MLKEM_Q); + + r = p->coeffs; + + for (layer = 1; layer <= 7; layer++) + __loop__( + invariant(1 <= layer && layer <= 8) + invariant(array_abs_bound(r, 0, MLKEM_N, layer * MLKEM_Q)) + decreases(8 - layer)) + { + mlk_ntt_layer(r, layer); + } + + /* Check the stronger bound */ + mlk_assert_abs_bound(p, MLKEM_N, MLK_NTT_BOUND); +} + +MLK_INTERNAL_API +void mlk_poly_ntt(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_NTT) + int ret; + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); + ret = mlk_ntt_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLK_NTT_BOUND); + return; + } +#endif /* MLK_USE_NATIVE_NTT */ + + mlk_poly_ntt_c(r); +} + + +/* Compute one layer of inverse NTT */ + +/* Reference: Embedded into `invntt()` in the reference implementation @[REF] */ +static void mlk_invntt_layer(int16_t *r, unsigned layer) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(1 <= layer && layer <= 7) + requires(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q))) +{ + unsigned start, k, len; + len = (unsigned)MLKEM_N >> layer; + k = (1u << layer) - 1; + for (start = 0; start < MLKEM_N; start += 2 * len) + __loop__( + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + invariant(start <= MLKEM_N && k <= 127) + /* Normalised form of k == MLKEM_N / len - 1 - start / (2 * len) */ + invariant(2 * len * k + start == 2 * MLKEM_N - 2 * len) + decreases(MLKEM_N - start)) + { + unsigned j; + int16_t zeta = mlk_zetas[k--]; + for (j = start; j < start + len; j++) + __loop__( + invariant(start <= j && j <= start + len) + invariant(start <= MLKEM_N && k <= 127) + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + decreases(start + len - j)) + { + int16_t t = r[j]; + /* The preconditions imply that the arithmetic does not overflow. */ + r[j] = mlk_barrett_reduce((int16_t)(t + r[j + len])); + r[j + len] = (int16_t)(r[j + len] - t); + r[j + len] = mlk_fqmul(r[j + len], zeta); + } + } +} + +/* Reference: `invntt()` in the reference implementation @[REF] + * - We normalize at the beginning of the inverse NTT, + * while the reference implementation normalizes at + * the end. This allows us to drop a call to `poly_reduce()` + * from the base multiplication. */ +MLK_STATIC_TESTABLE void mlk_poly_invntt_tomont_c(mlk_poly *p) +__contract__( + requires(memory_no_alias(p, sizeof(mlk_poly))) + assigns(memory_slice(p, sizeof(mlk_poly))) + ensures(array_abs_bound(p->coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND)) +) +{ + unsigned j, layer; + const int16_t f = 1441; /* check-magic: 1441 == pow(2,32 - 7,MLKEM_Q) */ + int16_t *r = p->coeffs; + + /* + * Scale input polynomial to account for Montgomery factor + * and NTT twist. This also brings coefficients down to + * absolute value < MLKEM_Q. + */ + for (j = 0; j < MLKEM_N; j++) + __loop__( + invariant(j <= MLKEM_N) + invariant(array_abs_bound(r, 0, j, MLKEM_Q)) + decreases(MLKEM_N - j)) + { + r[j] = mlk_fqmul(r[j], f); + } + + /* Run the invNTT layers */ + for (layer = 7; layer > 0; layer--) + __loop__( + invariant(0 <= layer && layer < 8) + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + decreases(layer)) + { + mlk_invntt_layer(r, layer); + } + + mlk_assert_abs_bound(p, MLKEM_N, MLK_INVNTT_BOUND); +} + +MLK_INTERNAL_API +void mlk_poly_invntt_tomont(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_INTT) + int ret; + ret = mlk_intt_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLK_INVNTT_BOUND); + return; + } +#endif /* MLK_USE_NATIVE_INTT */ + + mlk_poly_invntt_tomont_c(r); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(mlk_poly) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/poly.h b/sdk/software/examples/mlkem_test/mlkem_native/src/poly.h new file mode 100644 index 0000000..4d9e0f8 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/poly.h @@ -0,0 +1,290 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_POLY_H +#define MLK_POLY_H + + +#include "cbmc.h" +#include "common.h" +#include "debug.h" +#include "verify.h" + +/* Absolute exclusive upper bound for the output of the inverse NTT */ +#define MLK_INVNTT_BOUND (8 * MLKEM_Q) + +/* Absolute exclusive upper bound for the output of the forward NTT */ +#define MLK_NTT_BOUND (8 * MLKEM_Q) + +/** + * Element of R_q = Z_q[X]/(X^n + 1). Represents polynomial + * coeffs[0] + X*coeffs[1] + X^2*coeffs[2] + ... + X^{n-1}*coeffs[n-1]. + */ +typedef struct +{ + int16_t coeffs[MLKEM_N]; /**< Polynomial coefficients. */ +} MLK_ALIGN mlk_poly; + +/** + * INTERNAL representation of precomputed data speeding up + * the base multiplication of two polynomials in NTT domain. + */ +typedef struct +{ + int16_t coeffs[MLKEM_N >> 1]; /**< Cached coefficients. */ +} MLK_ALIGN mlk_poly_mulcache; + +/** + * Generic Montgomery reduction; given a 32-bit integer a, computes a 16-bit + * integer congruent to a * R^-1 mod MLKEM_Q, where R=2^16. + * + * @param a Input integer to be reduced, of absolute value smaller or equal + * to INT32_MAX - 2^15 * MLKEM_Q. + * + * @return Integer congruent to a * R^-1 modulo MLKEM_Q, with absolute value + * <= ceil(|a| / 2^16) + (MLKEM_Q + 1)/2. + */ +static MLK_ALWAYS_INLINE int16_t mlk_montgomery_reduce(int32_t a) +__contract__( + requires(a < +(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q)) && + a > -(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q))) + /* We don't attempt to express an input-dependent output bound + * as the post-condition here. There are two call-sites for this + * function: + * - The base multiplication: Here, we need no output bound. + * - mlk_fqmul: Here, we inline this function and prove another spec + * for mlk_fqmul which does have a post-condition bound. */ +) +{ + /* check-magic: 62209 == unsigned_mod(pow(MLKEM_Q, -1, 2^16), 2^16) */ + const uint32_t QINV = 62209; + + /* Compute a*q^{-1} mod 2^16 in unsigned representatives. */ + const uint16_t a_reduced = mlk_cast_int32_to_uint16(a); + const uint16_t a_inverted = (a_reduced * QINV) & UINT16_MAX; + + /* Lift to signed canonical representative mod 2^16. */ + const int16_t t = mlk_cast_uint16_to_int16(a_inverted); + + int32_t r; + + mlk_assert(a < +(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q)) && + a > -(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q))); + + r = a - ((int32_t)t * MLKEM_Q); + + /* + * PORTABILITY: Right-shift on a signed integer is, strictly-speaking, + * implementation-defined for negative left argument. Here, + * we assume it's sign-preserving "arithmetic" shift right. (C99 6.5.7 (5)) + */ + r = r >> 16; + /* Bounds: |r >> 16| <= ceil(|r| / 2^16) + * <= ceil(|a| / 2^16 + MLKEM_Q / 2) + * <= ceil(|a| / 2^16) + (MLKEM_Q + 1) / 2 + * + * (Note that |a >> n| = ceil(|a| / 2^16) for negative a) + */ + return (int16_t)r; +} + +#define mlk_poly_tomont MLK_NAMESPACE(poly_tomont) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * Bounds: output < MLKEM_Q in absolute value. + * + * @spec{Internal normalization required in `mlk_indcpa_keypair_derand` as + * part of matrix-vector multiplication @[FIPS203, Algorithm 13, K-PKE.KeyGen, + * L18].} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_tomont(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute MLK_NAMESPACE(poly_mulcache_compute) +/** + * Compute the mulcache for a polynomial in NTT domain. + * + * The mulcache of a degree-2 polynomial b := b0 + b1*X in Fq[X]/(X^2-zeta) + * is the value b1*zeta, needed when computing products of b in + * Fq[X]/(X^2-zeta). + * + * The mulcache of a polynomial in NTT domain -- which is a 128-tuple of + * degree-2 polynomials in Fq[X]/(X^2-zeta), for varying zeta, is the + * 128-tuple of mulcaches of those polynomials. + * + * @spec{Caches `b_1 * \gamma` in @[FIPS203, Algorithm 12, BaseCaseMultiply, + * L1].} + * + * @param[out] x Mulcache to be populated. + * @param[in] a Input polynomial. + */ +/* + * NOTE: The default C implementation of this function populates + * the mulcache with values in (-q,q), but this is not needed for the + * higher level safety proofs, and thus not part of the spec. + */ +MLK_INTERNAL_API +void mlk_poly_mulcache_compute(mlk_poly_mulcache *x, const mlk_poly *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_poly_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_poly))) + assigns(memory_slice(x, sizeof(mlk_poly_mulcache))) +); + +#define mlk_poly_reduce MLK_NAMESPACE(poly_reduce) +/** + * Convert a polynomial to unsigned canonical representatives. + * + * The input coefficients can be arbitrary integers in int16_t. The output + * coefficients are in [0,1,..,MLKEM_Q-1]. + * + * @spec{Normalizes on unsigned canonical representatives ahead of calling + * @[FIPS203, Compress_d, Eq (4.7)]. This is not made explicit in FIPS 203.} + * + * @param[in,out] r Input/output polynomial. + */ +/* + * NOTE: The semantics of mlk_poly_reduce() is different in + * the reference implementation, which requires + * signed canonical output data. Unsigned canonical + * outputs are better suited to the only remaining + * use of mlk_poly_reduce() in the context of (de)serialization. + */ +MLK_INTERNAL_API +void mlk_poly_reduce(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_add MLK_NAMESPACE(poly_add) +/** + * Add two polynomials in place. + * + * The coefficients of @p r and @p b must be such that the addition does not + * overflow. Otherwise, the behaviour of this function is undefined. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L21].} + * + * @param[in,out] r Input-output polynomial to be added to. + * @param[in] b Input polynomial that should be added to @p r. Must be + * disjoint from @p r. + */ +/* + * NOTE: The reference implementation uses a 3-argument mlk_poly_add. + * We specialize to the accumulator form to avoid reasoning about aliasing. + */ +MLK_INTERNAL_API +void mlk_poly_add(mlk_poly *r, const mlk_poly *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(b, sizeof(mlk_poly))) + requires(forall(k0, 0, MLKEM_N, (int32_t) r->coeffs[k0] + b->coeffs[k0] <= INT16_MAX)) + requires(forall(k1, 0, MLKEM_N, (int32_t) r->coeffs[k1] + b->coeffs[k1] >= INT16_MIN)) + ensures(forall(k, 0, MLKEM_N, r->coeffs[k] == old(*r).coeffs[k] + b->coeffs[k])) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#define mlk_poly_sub MLK_NAMESPACE(poly_sub) +/** + * Subtract two polynomials; no modular reduction is performed. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 15, K-PKE.Decrypt, L6].} + * + * @param[in,out] r Input-output polynomial to be subtracted from. + * @param[in] b Second input polynomial. + */ +/* + * NOTE: The reference implementation uses a 3-argument mlk_poly_sub. + * We specialize to the accumulator form to avoid reasoning about aliasing. + */ +MLK_INTERNAL_API +void mlk_poly_sub(mlk_poly *r, const mlk_poly *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(b, sizeof(mlk_poly))) + requires(forall(k0, 0, MLKEM_N, (int32_t) r->coeffs[k0] - b->coeffs[k0] <= INT16_MAX)) + requires(forall(k1, 0, MLKEM_N, (int32_t) r->coeffs[k1] - b->coeffs[k1] >= INT16_MIN)) + ensures(forall(k, 0, MLKEM_N, r->coeffs[k] == old(*r).coeffs[k] - b->coeffs[k])) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#define mlk_poly_ntt MLK_NAMESPACE(poly_ntt) +/** + * Compute the negacyclic number-theoretic transform (NTT) of a polynomial + * in place. + * + * The input is assumed to be in normal order and coefficient-wise bound by + * MLKEM_Q in absolute value. + * + * The output polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set, and coefficient-wise bound + * by MLK_NTT_BOUND in absolute value. + * + * (NOTE: Sometimes the input to the NTT is actually smaller, which gives + * better bounds.) + * + * @spec{Implements @[FIPS203, Algorithm 9, NTT].} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_ntt(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLK_NTT_BOUND)) +); + +#define mlk_poly_invntt_tomont MLK_NAMESPACE(poly_invntt_tomont) +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place; input assumed to be in bitreversed order, output in + * normal order. + * + * The input is assumed to be in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set, and can have arbitrary + * coefficients in int16_t. + * + * The output polynomial is in normal order, and coefficient-wise bound by + * MLK_INVNTT_BOUND in absolute value. + * + * @spec{Implements composition of @[FIPS203, Algorithm 10, NTT^{-1}] and + * elementwise modular multiplication with a suitable Montgomery factor + * introduced during the base multiplication.} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_invntt_tomont(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND)) +); + +#endif /* !MLK_POLY_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.c b/sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.c new file mode 100644 index 0000000..99636b8 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.c @@ -0,0 +1,501 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "poly_k.h" + +#include "debug.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_poly_cbd_eta1 MLK_ADD_PARAM_SET(mlk_poly_cbd_eta1) +#define mlk_poly_cbd_eta2 MLK_ADD_PARAM_SET(mlk_poly_cbd_eta2) +#define mlk_polyvec_basemul_acc_montgomery_cached_c \ + MLK_ADD_PARAM_SET(mlk_polyvec_basemul_acc_montgomery_cached_c) +/* End of parameter set namespacing */ + +/* Reference: `polyvec_compress()` in the reference implementation @[REF] + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_INTERNAL_API +void mlk_polyvec_compress_du(uint8_t r[MLKEM_POLYVECCOMPRESSEDBYTES_DU], + const mlk_polyvec *a) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_compress_du(r + i * MLKEM_POLYCOMPRESSEDBYTES_DU, &a->vec[i]); + } +} + +/* Reference: `polyvec_decompress()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_decompress_du(mlk_polyvec *r, + const uint8_t a[MLKEM_POLYVECCOMPRESSEDBYTES_DU]) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_decompress_du(&r->vec[i], a + i * MLKEM_POLYCOMPRESSEDBYTES_DU); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); +} + +/* Reference: `polyvec_tobytes()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_INTERNAL_API +void mlk_polyvec_tobytes(uint8_t r[MLKEM_POLYVECBYTES], const mlk_polyvec *a) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(r, MLKEM_POLYVECBYTES)) + invariant(i <= MLKEM_K) + decreases(MLKEM_K - i) + ) + { + mlk_poly_tobytes(&r[i * MLKEM_POLYBYTES], &a->vec[i]); + } +} + +/* Reference: `polyvec_frombytes()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_frombytes(mlk_polyvec *r, const uint8_t a[MLKEM_POLYVECBYTES]) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_frombytes(&r->vec[i], a + i * MLKEM_POLYBYTES); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +} + +/* Reference: `polyvec_ntt()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_ntt(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_ntt(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLK_NTT_BOUND); +} + +/* Reference: `polyvec_invntt_tomont()` in the reference implementation @[REF]. + * - We normalize at the beginning of the inverse NTT, + * while the reference implementation normalizes at + * the end. This allows us to drop a call to `poly_reduce()` + * from the base multiplication. */ +MLK_INTERNAL_API +void mlk_polyvec_invntt_tomont(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_invntt_tomont(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLK_INVNTT_BOUND); +} + +/* Reference: `polyvec_basemul_acc_montgomery()` in the + * reference implementation @[REF]. + * - We use a multiplication cache ('mulcache') here + * which is not present in the reference implementation @[REF]. + * This idea originates from @[NeonNTT] and is used + * at the C level here. + * - We compute the coefficients of the scalar product in 32-bit + * coefficients and perform only a single modular reduction + * at the end. The reference implementation uses 2 * MLKEM_K + * more modular reductions since it reduces after every modular + * multiplication. */ +MLK_STATIC_TESTABLE void mlk_polyvec_basemul_acc_montgomery_cached_c( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(memory_no_alias(b_cache, sizeof(mlk_polyvec_mulcache))) + requires(forall(k1, 0, MLKEM_K, + array_bound(a->vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) + assigns(memory_slice(r, sizeof(mlk_poly))) +) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); + + for (i = 0; i < MLKEM_N / 2; i++) + __loop__(invariant(i <= MLKEM_N / 2) + decreases(MLKEM_N / 2 - i)) + { + unsigned k; + int32_t t[2] = {0}; + for (k = 0; k < MLKEM_K; k++) + __loop__( + invariant(k <= MLKEM_K && + t[0] <= (int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768 && + t[0] >= - ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768) && + t[1] <= ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768) && + t[1] >= - ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768)) + decreases(MLKEM_K - k)) + { + t[0] += (int32_t)a->vec[k].coeffs[2 * i + 1] * b_cache->vec[k].coeffs[i]; + t[0] += (int32_t)a->vec[k].coeffs[2 * i] * b->vec[k].coeffs[2 * i]; + t[1] += (int32_t)a->vec[k].coeffs[2 * i] * b->vec[k].coeffs[2 * i + 1]; + t[1] += (int32_t)a->vec[k].coeffs[2 * i + 1] * b->vec[k].coeffs[2 * i]; + } + r->coeffs[2 * i + 0] = mlk_montgomery_reduce(t[0]); + r->coeffs[2 * i + 1] = mlk_montgomery_reduce(t[1]); + } +} + +MLK_INTERNAL_API +void mlk_polyvec_basemul_acc_montgomery_cached( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +{ +#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) + { + int ret; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +#if MLKEM_K == 2 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#elif MLKEM_K == 3 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#elif MLKEM_K == 4 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#endif + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } + } +#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */ + + mlk_polyvec_basemul_acc_montgomery_cached_c(r, a, b, b_cache); +} + +/* Reference: Does not exist in the reference implementation @[REF]. + * - The reference implementation does not use a + * multiplication cache ('mulcache'). This idea originates + * from @[NeonNTT] and is used at the C level here. */ +MLK_INTERNAL_API +void mlk_polyvec_mulcache_compute(mlk_polyvec_mulcache *x, const mlk_polyvec *a) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_mulcache_compute(&x->vec[i], &a->vec[i]); + } +} + +/* Reference: `polyvec_reduce()` in the reference implementation @[REF]. + * - We use _unsigned_ canonical outputs, while the reference + * implementation uses _signed_ canonical outputs. + * Accordingly, we need a conditional addition of MLKEM_Q + * here to go from signed to unsigned representatives. + * This conditional addition is then dropped from all + * polynomial compression functions instead (see `compress.c`). */ +MLK_INTERNAL_API +void mlk_polyvec_reduce(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_reduce(&r->vec[i]); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); +} + +/* Reference: `polyvec_add()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_polyvec_add(mlk_polyvec *r, const mlk_polyvec *b) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(r, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + invariant(forall(j0, i, MLKEM_K, + forall(k0, 0, MLKEM_N, + ((int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] <= INT16_MAX) && + ((int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] >= INT16_MIN)))) + invariant(forall(j2, 0, i, + forall(k2, 0, MLKEM_N, + (r->vec[j2].coeffs[k2] <= INT16_MAX) && + (r->vec[j2].coeffs[k2] >= INT16_MIN)))) + decreases(MLKEM_K - i) + ) + { + mlk_poly_add(&r->vec[i], &b->vec[i]); + } +} + +/* Reference: `polyvec_tomont()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_tomont(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_tomont(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLKEM_Q); +} + + +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter MLKEM_ETA1. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_eta1], where eta1 + * is specified per parameter set in @[FIPS203, Table 2] and represented as + * MLKEM_ETA1 here.} + * + * @reference{`poly_cbd_eta1` in the reference implementation @[REF].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +static MLK_INLINE void mlk_poly_cbd_eta1( + mlk_poly *r, const uint8_t buf[MLKEM_ETA1 * MLKEM_N / 4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(buf, MLKEM_ETA1 * MLKEM_N / 4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1)) +) +{ +#if MLKEM_ETA1 == 2 + mlk_poly_cbd2(r, buf); +#elif MLKEM_ETA1 == 3 + mlk_poly_cbd3(r, buf); +#else +#error "Invalid value of MLKEM_ETA1" +#endif +} + +/* Reference: Does not exist in the reference implementation @[REF]. + * - This implements a x4-batched version of `poly_getnoise_eta1()` + * from the reference implementation, to leverage + * batched Keccak-f1600.*/ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, uint8_t nonce2, + uint8_t nonce3) +{ + MLK_ALIGN uint8_t buf[4][MLK_ALIGN_UP(MLKEM_ETA1 * MLKEM_N / 4)]; + MLK_ALIGN uint8_t extkey[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 1)]; + mlk_memcpy(extkey[0], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[1], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[2], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[3], seed, MLKEM_SYMBYTES); + extkey[0][MLKEM_SYMBYTES] = nonce0; + extkey[1][MLKEM_SYMBYTES] = nonce1; + extkey[2][MLKEM_SYMBYTES] = nonce2; + extkey[3][MLKEM_SYMBYTES] = nonce3; + +#if !defined(FIPS202_X4_DEFAULT_IMPLEMENTATION) && \ + !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + mlk_prf_eta1_x4(buf, extkey); +#else + mlk_prf_eta1(buf[0], extkey[0]); + mlk_prf_eta1(buf[1], extkey[1]); + mlk_prf_eta1(buf[2], extkey[2]); + if (r3 != NULL) + { + mlk_prf_eta1(buf[3], extkey[3]); + } +#endif /* !(!FIPS202_X4_DEFAULT_IMPLEMENTATION && \ + !MLK_CONFIG_SERIAL_FIPS202_ONLY) */ + + mlk_poly_cbd_eta1(r0, buf[0]); + mlk_poly_cbd_eta1(r1, buf[1]); + mlk_poly_cbd_eta1(r2, buf[2]); + if (r3 != NULL) + { + mlk_poly_cbd_eta1(r3, buf[3]); + mlk_assert_abs_bound(r3, MLKEM_N, MLKEM_ETA1 + 1); + } + + mlk_assert_abs_bound(r0, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r1, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r2, MLKEM_N, MLKEM_ETA1 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} + +#if MLKEM_K == 2 || MLKEM_K == 4 +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter MLKEM_ETA2. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_eta2], where eta2 + * is specified per parameter set in @[FIPS203, Table 2] and represented as + * MLKEM_ETA2 here.} + * + * @reference{`poly_cbd_eta2` in the reference implementation @[REF].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +static MLK_INLINE void mlk_poly_cbd_eta2( + mlk_poly *r, const uint8_t buf[MLKEM_ETA2 * MLKEM_N / 4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(buf, MLKEM_ETA2 * MLKEM_N / 4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))) +{ +#if MLKEM_ETA2 == 2 + mlk_poly_cbd2(r, buf); +#else +#error "Invalid value of MLKEM_ETA2" +#endif +} + +/* Reference: `poly_getnoise_eta2()` in the reference implementation @[REF]. + * - We include buffer zeroization. */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta2(mlk_poly *r, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce) +{ + MLK_ALIGN uint8_t buf[MLKEM_ETA2 * MLKEM_N / 4]; + MLK_ALIGN uint8_t extkey[MLKEM_SYMBYTES + 1]; + + mlk_memcpy(extkey, seed, MLKEM_SYMBYTES); + extkey[MLKEM_SYMBYTES] = nonce; + mlk_prf_eta2(buf, extkey); + + mlk_poly_cbd_eta2(r, buf); + + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_ETA2 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* MLKEM_K == 2 || MLKEM_K == 4 */ + +#if MLKEM_K == 2 +/* Reference: Does not exist in the reference implementation @[REF]. + * - This implements a x4-batched version of `poly_getnoise_eta1()` + * and `poly_getnoise_eta2()` from the reference implementation, + * leveraging batched Keccak-f1600. + * - If a x4-batched Keccak-f1600 is available, we squeeze + * more random data than needed for the eta2 calls, to be + * be able to use a x4-batched Keccak-f1600. */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1122_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, + const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, + uint8_t nonce2, uint8_t nonce3) +{ +#if MLKEM_ETA2 >= MLKEM_ETA1 +#error mlk_poly_getnoise_eta1122_4x assumes MLKEM_ETA1 > MLKEM_ETA2 +#endif + MLK_ALIGN uint8_t buf[4][MLK_ALIGN_UP(MLKEM_ETA1 * MLKEM_N / 4)]; + MLK_ALIGN uint8_t extkey[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 1)]; + + mlk_memcpy(extkey[0], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[1], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[2], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[3], seed, MLKEM_SYMBYTES); + extkey[0][MLKEM_SYMBYTES] = nonce0; + extkey[1][MLKEM_SYMBYTES] = nonce1; + extkey[2][MLKEM_SYMBYTES] = nonce2; + extkey[3][MLKEM_SYMBYTES] = nonce3; + + /* On systems with fast batched Keccak, we use 4-fold batched PRF, + * even though that means generating more random data in buf[2] and buf[3] + * than necessary. */ +#if !defined(FIPS202_X4_DEFAULT_IMPLEMENTATION) && \ + !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + mlk_prf_eta1_x4(buf, extkey); +#else + mlk_prf_eta1(buf[0], extkey[0]); + mlk_prf_eta1(buf[1], extkey[1]); + mlk_prf_eta2(buf[2], extkey[2]); + mlk_prf_eta2(buf[3], extkey[3]); +#endif /* !(!FIPS202_X4_DEFAULT_IMPLEMENTATION && \ + !MLK_CONFIG_SERIAL_FIPS202_ONLY) */ + + mlk_poly_cbd_eta1(r0, buf[0]); + mlk_poly_cbd_eta1(r1, buf[1]); + mlk_poly_cbd_eta2(r2, buf[2]); + mlk_poly_cbd_eta2(r3, buf[3]); + + mlk_assert_abs_bound(r0, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r1, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r2, MLKEM_N, MLKEM_ETA2 + 1); + mlk_assert_abs_bound(r3, MLKEM_N, MLKEM_ETA2 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* MLKEM_K == 2 */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_poly_cbd_eta1 +#undef mlk_poly_cbd_eta2 +#undef mlk_polyvec_basemul_acc_montgomery_cached_c diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.h b/sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.h new file mode 100644 index 0000000..e72432a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/poly_k.h @@ -0,0 +1,593 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_POLY_K_H +#define MLK_POLY_K_H + +#include "common.h" +#include "compress.h" +#include "poly.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_polyvec MLK_ADD_PARAM_SET(mlk_polyvec) +#define mlk_polymat MLK_ADD_PARAM_SET(mlk_polymat) +#define mlk_polyvec_mulcache MLK_ADD_PARAM_SET(mlk_polyvec_mulcache) +/* End of parameter set namespacing */ + +/** Vector of MLKEM_K polynomials. */ +typedef struct +{ + mlk_poly vec[MLKEM_K]; /**< Component polynomials. */ +} MLK_ALIGN mlk_polyvec; + +/** MLKEM_K x MLKEM_K matrix of polynomials. */ +typedef struct +{ + mlk_polyvec vec[MLKEM_K]; /**< Rows of the matrix. */ +} MLK_ALIGN mlk_polymat; + +/** Vector of MLKEM_K mlk_poly_mulcache entries. */ +typedef struct +{ + mlk_poly_mulcache vec[MLKEM_K]; /**< Per-component caches. */ +} MLK_ALIGN mlk_polyvec_mulcache; + +#define mlk_poly_compress_du MLK_NAMESPACE_K(poly_compress_du) +/** + * Compression (du bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_{d_u} (Compress_{d_u} (u))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L22], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DU + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +static MLK_INLINE void mlk_poly_compress_du( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DU], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_DU))) +{ +#if MLKEM_DU == 10 + mlk_poly_compress_d10(r, a); +#elif MLKEM_DU == 11 + mlk_poly_compress_d11(r, a); +#else +#error "Invalid value of MLKEM_DU" +#endif +} + +#define mlk_poly_decompress_du MLK_NAMESPACE_K(poly_decompress_du) +/** + * De-serialization and subsequent decompression (du bits) of a polynomial; + * approximate inverse of mlk_poly_compress_du. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_{d_u} (ByteDecode_{d_u} (u))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L3], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DU + * bytes). + */ +static MLK_INLINE void mlk_poly_decompress_du( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_DU]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +{ +#if MLKEM_DU == 10 + mlk_poly_decompress_d10(r, a); +#elif MLKEM_DU == 11 + mlk_poly_decompress_d11(r, a); +#else +#error "Invalid value of MLKEM_DU" +#endif +} + +#define mlk_poly_compress_dv MLK_NAMESPACE_K(poly_compress_dv) +/** + * Compression (dv bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_{d_v} (Compress_{d_v} (v))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L23], with level-specific d_v defined in + * @[FIPS203, Table 2], and given by MLKEM_DV here.} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DV + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +static MLK_INLINE void mlk_poly_compress_dv( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DV], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_DV)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_DV))) +{ +#if MLKEM_DV == 4 + mlk_poly_compress_d4(r, a); +#elif MLKEM_DV == 5 + mlk_poly_compress_d5(r, a); +#else +#error "Invalid value of MLKEM_DV" +#endif +} + + +#define mlk_poly_decompress_dv MLK_NAMESPACE_K(poly_decompress_dv) +/** + * De-serialization and subsequent decompression (dv bits) of a polynomial; + * approximate inverse of mlk_poly_compress_dv. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_{d_v} (ByteDecode_{d_v} (v))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L4], with level-specific d_v defined in + * @[FIPS203, Table 2], and given by MLKEM_DV here.} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DV + * bytes). + */ +static MLK_INLINE void mlk_poly_decompress_dv( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_DV]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_DV)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +{ +#if MLKEM_DV == 4 + mlk_poly_decompress_d4(r, a); +#elif MLKEM_DV == 5 + mlk_poly_decompress_d5(r, a); +#else +#error "Invalid value of MLKEM_DV" +#endif +} + +#define mlk_polyvec_compress_du MLK_NAMESPACE_K(polyvec_compress_du) +/** + * Compress and serialize a vector of polynomials. + * + * @spec{Implements `ByteEncode_{d_u} (Compress_{d_u} (u))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L22], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output byte array (needs space for + * MLKEM_POLYVECCOMPRESSEDBYTES_DU bytes). + * @param[in] a Input vector of polynomials. Coefficients must be unsigned + * canonical, i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_polyvec_compress_du(uint8_t r[MLKEM_POLYVECCOMPRESSEDBYTES_DU], + const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(forall(k0, 0, MLKEM_K, + array_bound(a->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(r, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) +); + +#define mlk_polyvec_decompress_du MLK_NAMESPACE_K(polyvec_decompress_du) +/** + * De-serialize and decompress a vector of polynomials; approximate inverse + * of mlk_polyvec_compress_du. + * + * @spec{Implements `Decompress_{d_u} (ByteDecode_{d_u} (u))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L3], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output vector of polynomials. Coefficients are normalized + * to [0,1,..,MLKEM_Q-1]. + * @param[in] a Input byte array (of length MLKEM_POLYVECCOMPRESSEDBYTES_DU + * bytes). + */ +MLK_INTERNAL_API +void mlk_polyvec_decompress_du(mlk_polyvec *r, + const uint8_t a[MLKEM_POLYVECCOMPRESSEDBYTES_DU]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +); + +#define mlk_polyvec_tobytes MLK_NAMESPACE_K(polyvec_tobytes) +/** + * Serialize a vector of polynomials. + * + * @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays] and + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[out] r Output byte array (needs space for MLKEM_POLYVECBYTES bytes). + * @param[in] a Input vector of polynomials. Each polynomial must have + * coefficients in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_polyvec_tobytes(uint8_t r[MLKEM_POLYVECBYTES], const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(r, MLKEM_POLYVECBYTES)) + requires(forall(k0, 0, MLKEM_K, + array_bound(a->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(r, MLKEM_POLYVECBYTES)) +); + +#define mlk_polyvec_frombytes MLK_NAMESPACE_K(polyvec_frombytes) +/** + * De-serialize a vector of polynomials; inverse of mlk_polyvec_tobytes. + * + * @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays] and + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[out] r Output vector of polynomials. Coefficients will be + * normalized in [0,1,..,4095]. + * @param[in] a Input byte array (of length MLKEM_POLYVECBYTES bytes). + */ +MLK_INTERNAL_API +void mlk_polyvec_frombytes(mlk_polyvec *r, const uint8_t a[MLKEM_POLYVECBYTES]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(memory_no_alias(a, MLKEM_POLYVECBYTES)) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) +); + +#define mlk_polyvec_ntt MLK_NAMESPACE_K(polyvec_ntt) +/** + * Apply forward NTT to all elements of a vector of polynomials. + * + * The input is assumed to be in normal order and coefficient-wise bound by + * MLKEM_Q in absolute value. + * + * The output polynomial is in bitreversed order, and coefficient-wise bound + * by MLK_NTT_BOUND in absolute value. + * + * @spec{Implements @[FIPS203, Algorithm 9, NTT]. Extended to vectors as per + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[in,out] r Input/output vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_ntt(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLKEM_Q))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLK_NTT_BOUND))) +); + +#define mlk_polyvec_invntt_tomont MLK_NAMESPACE_K(polyvec_invntt_tomont) +/** + * Apply inverse NTT to all elements of a vector of polynomials and multiply + * by Montgomery factor 2^16. + * + * The input is assumed to be in bitreversed order, and can have arbitrary + * coefficients in int16_t. + * + * The output polynomial is in normal order, and coefficient-wise bound by + * MLK_INVNTT_BOUND in absolute value. + * + * @spec{Implements @[FIPS203, Algorithm 10, NTT^{-1}]. Extended to vectors + * as per @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[in,out] r Input/output vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_invntt_tomont(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND))) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached \ + MLK_NAMESPACE_K(polyvec_basemul_acc_montgomery_cached) +/** + * Scalar product of two vectors of polynomials in NTT domain, using + * mulcache for the second operand. + * + * Bounds: every coefficient of @p a is assumed to be in [0,1,..,4095]. No + * bounds guarantees for the coefficients in the result. + * + * @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.14)], @[FIPS203, + * Algorithm 11, MultiplyNTTs], and @[FIPS203, Algorithm 12, + * BaseCaseMultiply].} + * + * @param[out] r Output polynomial. + * @param[in] a First input polynomial vector. + * @param[in] b Second input polynomial vector. + * @param[in] b_cache Mulcache for the second input polynomial vector. Can + * be computed via mlk_polyvec_mulcache_compute(). + */ +MLK_INTERNAL_API +void mlk_polyvec_basemul_acc_montgomery_cached( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(memory_no_alias(b_cache, sizeof(mlk_polyvec_mulcache))) + requires(forall(k1, 0, MLKEM_K, + array_bound(a->vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#define mlk_polyvec_mulcache_compute MLK_NAMESPACE_K(polyvec_mulcache_compute) +/** + * Compute the mulcache for a vector of polynomials in NTT domain. + * + * The mulcache of a degree-2 polynomial b := b0 + b1*X in Fq[X]/(X^2-zeta) + * is the value b1*zeta, needed when computing products of b in + * Fq[X]/(X^2-zeta). + * + * The mulcache of a polynomial in NTT domain -- which is a 128-tuple of + * degree-2 polynomials in Fq[X]/(X^2-zeta), for varying zeta, is the + * 128-tuple of mulcaches of those polynomials. + * + * The mulcache of a vector of polynomials is the vector of mulcaches of + * its entries. + * + * @spec{Caches `b_1 * \gamma` in @[FIPS203, Algorithm 12, BaseCaseMultiply, + * L1].} + * + * @param[out] x Mulcache to be populated. + * @param[in] a Input polynomial vector. + */ +/* + * NOTE: The default C implementation of this function populates + * the mulcache with values in (-q,q), but this is not needed for the + * higher level safety proofs, and thus not part of the spec. + */ +MLK_INTERNAL_API +void mlk_polyvec_mulcache_compute(mlk_polyvec_mulcache *x, const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_polyvec_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + assigns(memory_slice(x, sizeof(mlk_polyvec_mulcache))) +); + +#define mlk_polyvec_reduce MLK_NAMESPACE_K(polyvec_reduce) +/** + * Apply Barrett reduction to each coefficient of each element of a vector + * of polynomials. For details of the Barrett reduction see comments in + * poly.c. + * + * @spec{Normalizes on unsigned canonical representatives ahead of calling + * @[FIPS203, Compress_d, Eq (4.7)]. This is not made explicit in FIPS 203.} + * + * @param[in,out] r Input/output polynomial vector. + */ +/* + * NOTE: The semantics of mlk_polyvec_reduce() is different in + * the reference implementation, which requires + * signed canonical output data. Unsigned canonical + * outputs are better suited to the only remaining + * use of mlk_poly_reduce() in the context of (de)serialization. + */ +MLK_INTERNAL_API +void mlk_polyvec_reduce(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +); + +#define mlk_polyvec_add MLK_NAMESPACE_K(polyvec_add) +/** + * Add vectors of polynomials. + * + * The coefficients of @p r and @p b must be such that the addition does + * not overflow. Otherwise, the behaviour of this function is undefined. + * + * The coefficients returned in @p *r are in int16_t which is sufficient to + * prove type-safety of calling units. Therefore, no stronger ensures clause + * is required on this function. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L19].} + * + * @param[in,out] r Input-output vector of polynomials to be added to. + * @param[in] b Second input vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_add(mlk_polyvec *r, const mlk_polyvec *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(forall(j0, 0, MLKEM_K, + forall(k0, 0, MLKEM_N, + (int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] <= INT16_MAX))) + requires(forall(j1, 0, MLKEM_K, + forall(k1, 0, MLKEM_N, + (int32_t)r->vec[j1].coeffs[k1] + b->vec[j1].coeffs[k1] >= INT16_MIN))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) +); + +#define mlk_polyvec_tomont MLK_NAMESPACE_K(polyvec_tomont) +/** + * In-place conversion of all coefficients of a polynomial vector from the + * normal domain to the Montgomery domain. + * + * Bounds: output < MLKEM_Q in absolute value. + * + * @spec{Internal normalization required in `mlk_indcpa_keypair_derand` as + * part of matrix-vector multiplication @[FIPS203, Algorithm 13, K-PKE.KeyGen, + * L18].} + * + * @param[in,out] r Input/output polynomial vector. + */ +MLK_INTERNAL_API +void mlk_polyvec_tomont(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLKEM_Q))) +); + +#define mlk_poly_getnoise_eta1_4x MLK_NAMESPACE_K(poly_getnoise_eta1_4x) +/** + * Batch sample four polynomials deterministically from a seed and nonces, + * with output polynomials close to centered binomial distribution with + * parameter MLKEM_ETA1. + * + * @spec{Implements 4x `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))` appears in + * @[FIPS203, Algorithm 13, K-PKE.KeyGen, L{9, 13}] and @[FIPS203, + * Algorithm 14, K-PKE.Encrypt, L10].} + * + * @param[out] r0 Output polynomial. + * @param[out] r1 Output polynomial. + * @param[out] r2 Output polynomial. + * @param[out] r3 Output polynomial. May be NULL. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce0 One-byte input nonce. + * @param nonce1 One-byte input nonce. + * @param nonce2 One-byte input nonce. + * @param nonce3 One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, uint8_t nonce2, + uint8_t nonce3) +__contract__( + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + requires(memory_no_alias(r0, sizeof(mlk_poly))) + requires(memory_no_alias(r1, sizeof(mlk_poly))) + requires(memory_no_alias(r2, sizeof(mlk_poly))) + requires(r3 == NULL || memory_no_alias(r3, sizeof(mlk_poly))) + assigns(memory_slice(r0, sizeof(mlk_poly))) + assigns(memory_slice(r1, sizeof(mlk_poly))) + assigns(memory_slice(r2, sizeof(mlk_poly))) + assigns(r3 != NULL: memory_slice(r3, sizeof(mlk_poly))) + ensures(array_abs_bound(r0->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(array_abs_bound(r1->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(array_abs_bound(r2->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(r3 != NULL ==> array_abs_bound(r3->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) +); + +#if MLKEM_ETA1 == MLKEM_ETA2 +/* + * We only require mlk_poly_getnoise_eta2_4x for ml-kem-768 and ml-kem-1024 + * where MLKEM_ETA2 = MLKEM_ETA1 = 2. + * For ml-kem-512, mlk_poly_getnoise_eta1122_4x is used instead. + */ +#define mlk_poly_getnoise_eta2_4x mlk_poly_getnoise_eta1_4x +#endif /* MLKEM_ETA1 == MLKEM_ETA2 */ + +#if MLKEM_K == 2 || MLKEM_K == 4 +#define mlk_poly_getnoise_eta2 MLK_NAMESPACE_K(poly_getnoise_eta2) +/** + * Sample a polynomial deterministically from a seed and a nonce, with + * output polynomial close to centered binomial distribution with parameter + * MLKEM_ETA2. + * + * @spec{Implements `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))` appears in + * @[FIPS203, Algorithm 14, K-PKE.Encrypt, L14].} + * + * @param[out] r Output polynomial. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta2(mlk_poly *r, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)) +); +#endif /* MLKEM_K == 2 || MLKEM_K == 4 */ + +#if MLKEM_K == 2 +#define mlk_poly_getnoise_eta1122_4x MLK_NAMESPACE_K(poly_getnoise_eta1122_4x) +/** + * Batch sample four polynomials deterministically from a seed and nonces, + * with output polynomials close to centered binomial distribution with + * parameter MLKEM_ETA1 and MLKEM_ETA2. + * + * @spec{Implements two instances each of + * `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))` and + * `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))` appears in + * @[FIPS203, Algorithm 14, K-PKE.Encrypt, L14].} + * + * @param[out] r0 Output polynomial. + * @param[out] r1 Output polynomial. + * @param[out] r2 Output polynomial. + * @param[out] r3 Output polynomial. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce0 One-byte input nonce. + * @param nonce1 One-byte input nonce. + * @param nonce2 One-byte input nonce. + * @param nonce3 One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1122_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, + const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, + uint8_t nonce2, uint8_t nonce3) +__contract__( + requires(memory_no_alias(r0, sizeof(mlk_poly))) + requires(memory_no_alias(r1, sizeof(mlk_poly))) + requires(memory_no_alias(r2, sizeof(mlk_poly))) + requires(memory_no_alias(r3, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(r0, sizeof(mlk_poly))) + assigns(memory_slice(r1, sizeof(mlk_poly))) + assigns(memory_slice(r2, sizeof(mlk_poly))) + assigns(memory_slice(r3, sizeof(mlk_poly))) + ensures(array_abs_bound(r0->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1) + && array_abs_bound(r1->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1) + && array_abs_bound(r2->coeffs,0, MLKEM_N, MLKEM_ETA2 + 1) + && array_abs_bound(r3->coeffs,0, MLKEM_N, MLKEM_ETA2 + 1)) +); +#endif /* MLKEM_K == 2 */ + +#endif /* !MLK_POLY_K_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/randombytes.h b/sdk/software/examples/mlkem_test/mlkem_native/src/randombytes.h new file mode 100644 index 0000000..4627442 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/randombytes.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_RANDOMBYTES_H +#define MLK_RANDOMBYTES_H + + +#include "cbmc.h" +#include "common.h" + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#if !defined(MLK_CONFIG_CUSTOM_RANDOMBYTES) +/** + * Fill a buffer with cryptographically secure random bytes. + * + * mlkem-native does not provide an implementation of this function. + * It must be provided by the consumer. + * + * To use a custom random byte source with a different name or signature, + * set MLK_CONFIG_CUSTOM_RANDOMBYTES and define mlk_randombytes directly. + * + * @param[out] out Output buffer. + * @param outlen Number of random bytes to write. + * + * @retval 0 Success. + * @retval other Failure; top-level APIs propagate this as MLK_ERR_RNG_FAIL. + */ +int randombytes(uint8_t *out, size_t outlen); + +/** + * Internal wrapper around randombytes(). + * + * Fills a buffer with cryptographically secure random bytes. + * + * This function can be replaced by setting MLK_CONFIG_CUSTOM_RANDOMBYTES + * and defining mlk_randombytes directly. + * + * @param[out] out Output buffer. + * @param outlen Number of random bytes to write. + * + * @retval 0 Success. + * @retval other Failure; top-level APIs propagate this as MLK_ERR_RNG_FAIL. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_randombytes(uint8_t *out, size_t outlen) +__contract__( + requires(memory_no_alias(out, outlen)) + assigns(memory_slice(out, outlen))) { return randombytes(out, outlen); } +#endif /* !MLK_CONFIG_CUSTOM_RANDOMBYTES */ +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_RANDOMBYTES_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/sampling.c b/sdk/software/examples/mlkem_test/mlkem_native/src/sampling.c new file mode 100644 index 0000000..b6b9ff0 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/sampling.c @@ -0,0 +1,367 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "debug.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Reference: `rej_uniform()` in the reference implementation @[REF]. + * - Our signature differs from the reference implementation + * in that it adds the offset and always expects the base of the + * target buffer. This avoids shifting the buffer base in the + * caller, which appears tricky to reason about. */ +MLK_STATIC_TESTABLE unsigned mlk_rej_uniform_c(int16_t *r, unsigned target, + unsigned offset, + const uint8_t *buf, + unsigned buflen) +__contract__( + requires(offset <= target && target <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * target)) + requires(memory_no_alias(buf, buflen)) + requires(array_bound(r, 0, offset, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * target)) + ensures(offset <= return_value && return_value <= target) + ensures(array_bound(r, 0, return_value, 0, MLKEM_Q))) +{ + unsigned ctr, pos; + int16_t val0, val1; + + mlk_assert_bound(r, offset, 0, MLKEM_Q); + + ctr = offset; + pos = 0; + /* pos + 3 cannot overflow due to the assumption buflen <= 4096 */ + while (ctr < target && pos + 3 <= buflen) + __loop__( + invariant(offset <= ctr && ctr <= target && pos <= buflen) + invariant(array_bound(r, 0, ctr, 0, MLKEM_Q)) + decreases(buflen - pos)) + { + /* Safety: + * - The explicit cast to uint16_t ensures that << 8 does + * not signed-overflow even on a 16-bit system. + * - The conversion to int16_t is safe due to the explicit 0xFFF + * truncation. + */ + val0 = (int16_t)(((buf[pos + 0] >> 0) | ((uint16_t)buf[pos + 1] << 8)) & + 0xFFF); + val1 = (int16_t)(((buf[pos + 1] >> 4) | (buf[pos + 2] << 4)) & 0xFFF); + pos += 3; + + if (val0 < MLKEM_Q) + { + r[ctr++] = val0; + } + if (ctr < target && val1 < MLKEM_Q) + { + r[ctr++] = val1; + } + } + + mlk_assert_bound(r, ctr, 0, MLKEM_Q); + return ctr; +} + +/** + * Run rejection sampling on uniform random bytes to generate uniform random + * integers mod MLKEM_Q. + * + * @reference{`rej_uniform()` in the reference implementation @[REF]. Our + * signature differs from the reference in that it adds the offset and always + * expects the base of the target buffer; this avoids shifting the buffer + * base in the caller, which is tricky to reason about. Has an optional + * fallback to a native implementation.} + * + * @param[out] r Output buffer. + * @param target Requested number of 16-bit integers (uniform mod MLKEM_Q). + * Must be <= 4096. + * @param offset Number of 16-bit integers that have already been + * sampled. Must be <= @p target. + * @param[in] buf Input buffer (assumed to be uniform random bytes). + * @param buflen Length of input buffer in bytes. Must be <= 4096 and a + * multiple of 3. + * + * @note Strictly speaking, only a few values of @p buflen near UINT_MAX need + * excluding. The limit of 4096 is somewhat arbitrary but sufficient + * for all uses of this function. Similarly, the actual limit for + * @p target is UINT_MAX/2. + * + * @return New offset of sampled 16-bit integers, at most @p target and at + * least the initial @p offset. If the new offset is strictly less + * than @p target, the entire input buffer is guaranteed to have been + * consumed; otherwise no information is provided on how many bytes + * of the input buffer have been consumed. + */ +static unsigned mlk_rej_uniform(int16_t *r, unsigned target, unsigned offset, + const uint8_t *buf, unsigned buflen) +__contract__( + requires(offset <= target && target <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * target)) + requires(memory_no_alias(buf, buflen)) + requires(array_bound(r, 0, offset, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * target)) + ensures(offset <= return_value && return_value <= target) + ensures(array_bound(r, 0, return_value, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_REJ_UNIFORM) + if (offset == 0) + { + int ret; + ret = mlk_rej_uniform_native(r, target, buf, buflen); + if (ret != MLK_NATIVE_FUNC_FALLBACK) + { + unsigned res = (unsigned)ret; + mlk_assert_bound(r, res, 0, MLKEM_Q); + return res; + } + } +#endif /* MLK_USE_NATIVE_REJ_UNIFORM */ + + return mlk_rej_uniform_c(r, target, offset, buf, buflen); +} + +#ifndef MLKEM_GEN_MATRIX_NBLOCKS +#define MLKEM_GEN_MATRIX_NBLOCKS \ + ((12 * MLKEM_N / 8 * ((uint32_t)1 << 12) / MLKEM_Q + MLK_XOF_RATE) / \ + MLK_XOF_RATE) +#endif + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +/* Reference: Does not exist in the reference implementation @[REF]. + * - x4-batched version of `rej_uniform()` from the + * reference implementation, leveraging x4-batched Keccak-f1600. */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform_x4(mlk_poly *vec0, mlk_poly *vec1, mlk_poly *vec2, + mlk_poly *vec3, + uint8_t seed[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLK_ALIGN uint8_t + buf[4][MLK_ALIGN_UP(MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[4]; + mlk_xof_x4_ctx statex; + unsigned buflen; + + mlk_xof_x4_init(&statex); + mlk_xof_x4_absorb(&statex, seed, MLKEM_SYMBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLKEM_GEN_MATRIX_NBLOCKS. + * This should generate the matrix entries with high probability. + */ + mlk_xof_x4_squeezeblocks(buf, MLKEM_GEN_MATRIX_NBLOCKS, &statex); + buflen = MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE; + ctr[0] = mlk_rej_uniform(vec0->coeffs, MLKEM_N, 0, buf[0], buflen); + ctr[1] = mlk_rej_uniform(vec1->coeffs, MLKEM_N, 0, buf[1], buflen); + ctr[2] = mlk_rej_uniform(vec2->coeffs, MLKEM_N, 0, buf[2], buflen); + ctr[3] = mlk_rej_uniform(vec3->coeffs, MLKEM_N, 0, buf[3], buflen); + + /* + * So long as not all matrix entries have been generated, squeeze + * one more block a time until we're done. + */ + buflen = MLK_XOF_RATE; + while (ctr[0] < MLKEM_N || ctr[1] < MLKEM_N || ctr[2] < MLKEM_N || + ctr[3] < MLKEM_N) + __loop__( + assigns(ctr, statex, + memory_slice(vec0, sizeof(mlk_poly)), + memory_slice(vec1, sizeof(mlk_poly)), + memory_slice(vec2, sizeof(mlk_poly)), + memory_slice(vec3, sizeof(mlk_poly)), + object_whole(buf)) + invariant(ctr[0] <= MLKEM_N && ctr[1] <= MLKEM_N) + invariant(ctr[2] <= MLKEM_N && ctr[3] <= MLKEM_N) + invariant(array_bound(vec0->coeffs, 0, ctr[0], 0, MLKEM_Q)) + invariant(array_bound(vec1->coeffs, 0, ctr[1], 0, MLKEM_Q)) + invariant(array_bound(vec2->coeffs, 0, ctr[2], 0, MLKEM_Q)) + invariant(array_bound(vec3->coeffs, 0, ctr[3], 0, MLKEM_Q))) + { + mlk_xof_x4_squeezeblocks(buf, 1, &statex); + ctr[0] = mlk_rej_uniform(vec0->coeffs, MLKEM_N, ctr[0], buf[0], buflen); + ctr[1] = mlk_rej_uniform(vec1->coeffs, MLKEM_N, ctr[1], buf[1], buflen); + ctr[2] = mlk_rej_uniform(vec2->coeffs, MLKEM_N, ctr[2], buf[2], buflen); + ctr[3] = mlk_rej_uniform(vec3->coeffs, MLKEM_N, ctr[3], buf[3], buflen); + } + + mlk_xof_x4_release(&statex); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); +} +#endif /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +MLK_INTERNAL_API +void mlk_poly_rej_uniform(mlk_poly *entry, uint8_t seed[MLKEM_SYMBYTES + 2]) +{ + mlk_xof_ctx state; + MLK_ALIGN uint8_t buf[MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE]; + unsigned ctr, buflen; + + mlk_xof_init(&state); + mlk_xof_absorb(&state, seed, MLKEM_SYMBYTES + 2); + + /* Initially, squeeze + sample heuristic number of MLKEM_GEN_MATRIX_NBLOCKS. + */ + /* This should generate the matrix entry with high probability. */ + mlk_xof_squeezeblocks(buf, MLKEM_GEN_MATRIX_NBLOCKS, &state); + buflen = MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE; + ctr = mlk_rej_uniform(entry->coeffs, MLKEM_N, 0, buf, buflen); + + /* Squeeze + sample one more block a time until we're done */ + buflen = MLK_XOF_RATE; + while (ctr < MLKEM_N) + __loop__( + assigns(ctr, state, memory_slice(entry, sizeof(mlk_poly)), object_whole(buf)) + invariant(ctr <= MLKEM_N) + invariant(array_bound(entry->coeffs, 0, ctr, 0, MLKEM_Q))) + { + mlk_xof_squeezeblocks(buf, 1, &state); + ctr = mlk_rej_uniform(entry->coeffs, MLKEM_N, ctr, buf, buflen); + } + + mlk_xof_release(&state); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); +} + +/** + * Load 4 bytes into a 32-bit integer in little-endian order. + * + * @reference{`load32_littleendian()` in the reference implementation @[REF].} + * + * @param[in] x Input byte array. + * + * @return 32-bit unsigned integer loaded from @p x. + */ +static uint32_t mlk_load32_littleendian(const uint8_t x[4]) +{ + uint32_t r; + r = (uint32_t)x[0]; + r |= (uint32_t)x[1] << 8; + r |= (uint32_t)x[2] << 16; + r |= (uint32_t)x[3] << 24; + return r; +} + +/* Reference: `cbd2()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_poly_cbd2(mlk_poly *r, const uint8_t buf[2 * MLKEM_N / 4]) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_abs_bound(r->coeffs, 0, 8 * i, 3)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint32_t t = mlk_load32_littleendian(buf + 4 * i); + uint32_t d = t & 0x55555555; + d += (t >> 1) & 0x55555555; + + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_abs_bound(r->coeffs, 0, 8 * i + j, 3)) + decreases(8 - j)) + { + /* Safety: The & 0x3 masks each value to 2 bits (range [0, 3]), so the + * truncation and subsequent subtraction in int16_t is lossless. */ + const int16_t a = (int16_t)((d >> (4 * j + 0)) & 0x3); + const int16_t b = (int16_t)((d >> (4 * j + 2)) & 0x3); + r->coeffs[8 * i + j] = (int16_t)(a - b); + } + } +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 3 +/** + * Load 3 bytes into a 32-bit integer in little-endian order. + * + * This function is only needed for ML-KEM-512. + * + * @reference{`load24_littleendian()` in the reference implementation @[REF].} + * + * @param[in] x Input byte array. + * + * @return 32-bit unsigned integer loaded from @p x (most significant byte + * is zero). + */ +static uint32_t mlk_load24_littleendian(const uint8_t x[3]) +{ + uint32_t r; + r = (uint32_t)x[0]; + r |= (uint32_t)x[1] << 8; + r |= (uint32_t)x[2] << 16; + return r; +} + +/* Reference: `cbd3()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_poly_cbd3(mlk_poly *r, const uint8_t buf[3 * MLKEM_N / 4]) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 4; i++) + __loop__( + invariant(i <= MLKEM_N / 4) + invariant(array_abs_bound(r->coeffs, 0, 4 * i, 4)) + decreases(MLKEM_N / 4 - i)) + { + unsigned j; + const uint32_t t = mlk_load24_littleendian(buf + 3 * i); + uint32_t d = t & 0x00249249; + d += (t >> 1) & 0x00249249; + d += (t >> 2) & 0x00249249; + + for (j = 0; j < 4; j++) + __loop__( + invariant(i <= MLKEM_N / 4 && j <= 4) + invariant(array_abs_bound(r->coeffs, 0, 4 * i + j, 4)) + decreases(4 - j)) + { + /* Safety: The & 0x7 masks each value to 3 bits (range [0, 7]), so the + * truncation and subsequent subtraction in int16_t is lossless. */ + const int16_t a = (int16_t)((d >> (6 * j + 0)) & 0x7); + const int16_t b = (int16_t)((d >> (6 * j + 3)) & 0x7); + r->coeffs[4 * i + j] = (int16_t)(a - b); + } + } +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 3 */ + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(sampling) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLKEM_GEN_MATRIX_NBLOCKS diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/sampling.h b/sdk/software/examples/mlkem_test/mlkem_native/src/sampling.h new file mode 100644 index 0000000..be3b351 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/sampling.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_SAMPLING_H +#define MLK_SAMPLING_H + +#include "cbmc.h" +#include "common.h" +#include "poly.h" + +#define mlk_poly_cbd2 MLK_NAMESPACE(poly_cbd2) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter eta=2. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_2].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +MLK_INTERNAL_API +void mlk_poly_cbd2(mlk_poly *r, const uint8_t buf[2 * MLKEM_N / 4]); + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 3 +#define mlk_poly_cbd3 MLK_NAMESPACE(poly_cbd3) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter eta=3. + * + * This function is only needed for ML-KEM-512. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_3].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +MLK_INTERNAL_API +void mlk_poly_cbd3(mlk_poly *r, const uint8_t buf[3 * MLKEM_N / 4]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 3 */ + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +#define mlk_poly_rej_uniform_x4 MLK_NAMESPACE(poly_rej_uniform_x4) +/** + * Generate four polynomials using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. + * + * @spec{Implements @[FIPS203, Algorithm 7, SampleNTT].} + * + * @param[out] vec0 Polynomial to be sampled. + * @param[out] vec1 Polynomial to be sampled. + * @param[out] vec2 Polynomial to be sampled. + * @param[out] vec3 Polynomial to be sampled. + * @param[in] seed Consecutive array of 4 seed buffers of size + * MLKEM_SYMBYTES + 2 each, plus padding for alignment. + */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform_x4(mlk_poly *vec0, mlk_poly *vec1, mlk_poly *vec2, + mlk_poly *vec3, + uint8_t seed[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]) +__contract__( + requires(memory_no_alias(vec0, sizeof(mlk_poly))) + requires(memory_no_alias(vec1, sizeof(mlk_poly))) + requires(memory_no_alias(vec2, sizeof(mlk_poly))) + requires(memory_no_alias(vec3, sizeof(mlk_poly))) + requires(memory_no_alias(seed, 4 * MLK_ALIGN_UP(MLKEM_SYMBYTES + 2))) + assigns(memory_slice(vec0, sizeof(mlk_poly))) + assigns(memory_slice(vec1, sizeof(mlk_poly))) + assigns(memory_slice(vec2, sizeof(mlk_poly))) + assigns(memory_slice(vec3, sizeof(mlk_poly))) + ensures(array_bound(vec0->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec1->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec2->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec3->coeffs, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +#define mlk_poly_rej_uniform MLK_NAMESPACE(poly_rej_uniform) +/** + * Generate a polynomial using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. + * + * @spec{Implements @[FIPS203, Algorithm 7, SampleNTT].} + * + * @param[out] entry Polynomial to be sampled. + * @param[in] seed Seed buffer of size MLKEM_SYMBYTES + 2. + */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform(mlk_poly *entry, uint8_t seed[MLKEM_SYMBYTES + 2]) +__contract__( + requires(memory_no_alias(entry, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES + 2)) + assigns(memory_slice(entry, sizeof(mlk_poly))) + ensures(array_bound(entry->coeffs, 0, MLKEM_N, 0, MLKEM_Q))); + +#endif /* !MLK_SAMPLING_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/symmetric.h b/sdk/software/examples/mlkem_test/mlkem_native/src/symmetric.h new file mode 100644 index 0000000..68d7e1a --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/symmetric.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_SYMMETRIC_H +#define MLK_SYMMETRIC_H + + +#include "cbmc.h" +#include "common.h" +#include MLK_FIPS202_HEADER_FILE +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +#include MLK_FIPS202X4_HEADER_FILE +#endif + +/* Macros denoting FIPS 203 specific Hash functions */ + +/* Hash function H, @[FIPS203, Section 4.1, Eq (4.4)] */ +#define mlk_hash_h(OUT, IN, INBYTES) mlk_sha3_256(OUT, IN, INBYTES) + +/* Hash function G, @[FIPS203, Section 4.1, Eq (4.5)] */ +#define mlk_hash_g(OUT, IN, INBYTES) mlk_sha3_512(OUT, IN, INBYTES) + +/* Hash function J, @[FIPS203, Section 4.1, Eq (4.4)] */ +#define mlk_hash_j(OUT, IN, INBYTES) \ + mlk_shake256(OUT, MLKEM_SYMBYTES, IN, INBYTES) + +/* PRF function, @[FIPS203, Section 4.1, Eq (4.3)] + * Referring to (eq 4.3), `OUT` is assumed to contain `s || b`. */ +#define mlk_prf_eta(ETA, OUT, IN) \ + mlk_shake256(OUT, (ETA) * MLKEM_N / 4, IN, MLKEM_SYMBYTES + 1) +#define mlk_prf_eta1(OUT, IN) mlk_prf_eta(MLKEM_ETA1, OUT, IN) +#define mlk_prf_eta2(OUT, IN) mlk_prf_eta(MLKEM_ETA2, OUT, IN) +#define mlk_prf_eta1_x4(OUT, IN) \ + mlk_shake256x4((OUT)[0], (OUT)[1], (OUT)[2], (OUT)[3], \ + (MLKEM_ETA1 * MLKEM_N / 4), (IN)[0], (IN)[1], (IN)[2], \ + (IN)[3], MLKEM_SYMBYTES + 1) + +/* XOF function, FIPS 203 4.1 */ +#define mlk_xof_ctx mlk_shake128ctx +#define mlk_xof_x4_ctx mlk_shake128x4ctx +#define mlk_xof_init(CTX) mlk_shake128_init((CTX)) +#define mlk_xof_absorb(CTX, IN, INBYTES) \ + mlk_shake128_absorb_once((CTX), (IN), (INBYTES)) +#define mlk_xof_squeezeblocks(BUF, NBLOCKS, CTX) \ + mlk_shake128_squeezeblocks((BUF), (NBLOCKS), (CTX)) +#define mlk_xof_release(CTX) mlk_shake128_release((CTX)) + +#define mlk_xof_x4_init(CTX) mlk_shake128x4_init((CTX)) +#define mlk_xof_x4_absorb(CTX, IN, INBYTES) \ + mlk_shake128x4_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (IN)[3], \ + (INBYTES)) +#define mlk_xof_x4_squeezeblocks(BUF, NBLOCKS, CTX) \ + mlk_shake128x4_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (BUF)[3], \ + (NBLOCKS), (CTX)) +#define mlk_xof_x4_release(CTX) mlk_shake128x4_release((CTX)) + +#define MLK_XOF_RATE SHAKE128_RATE + +#endif /* !MLK_SYMMETRIC_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/sys.h b/sdk/software/examples/mlkem_test/mlkem_native/src/sys.h new file mode 100644 index 0000000..1c48d81 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/sys.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_SYS_H +#define MLK_SYS_H + +#if !defined(MLK_CONFIG_NO_ASM) && (defined(__GNUC__) || defined(__clang__)) +#define MLK_HAVE_INLINE_ASM +#endif + +/* Try to find endianness, if not forced through CFLAGS already */ +#if !defined(MLK_SYS_LITTLE_ENDIAN) && !defined(MLK_SYS_BIG_ENDIAN) +#if defined(__BYTE_ORDER__) +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define MLK_SYS_LITTLE_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MLK_SYS_BIG_ENDIAN +#else +#error "__BYTE_ORDER__ defined, but don't recognize value." +#endif +#endif /* __BYTE_ORDER__ */ + +/* MSVC does not define __BYTE_ORDER__. However, MSVC only supports + * little endian x86, x86_64, and AArch64. It is, hence, safe to assume + * little endian. */ +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_AMD64) || \ + defined(_M_IX86) || defined(_M_ARM64)) +#define MLK_SYS_LITTLE_ENDIAN +#endif + +#endif /* !MLK_SYS_LITTLE_ENDIAN && !MLK_SYS_BIG_ENDIAN */ + +/* Check if we're running on an AArch64 little endian system. _M_ARM64 is set by + * MSVC. */ +#if defined(__AARCH64EL__) || defined(_M_ARM64) +#define MLK_SYS_AARCH64 +#endif + +/* Check if we're running on an AArch64 big endian system. */ +#if defined(__AARCH64EB__) +#define MLK_SYS_AARCH64_EB +#endif + +/* Check if we're running on an Armv8.1-M system with MVE */ +#if defined(__ARM_ARCH_8_1M_MAIN__) || defined(__ARM_FEATURE_MVE) +#define MLK_SYS_ARMV81M_MVE +#endif + +/* Check if we're running on an x86_64 system. */ +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) +#define MLK_SYS_X86_64 +#if defined(__AVX2__) +#define MLK_SYS_X86_64_AVX2 +#endif +#endif /* __x86_64__ || _M_X64 || _M_AMD64 */ + +#if defined(MLK_SYS_LITTLE_ENDIAN) && defined(__powerpc64__) +#define MLK_SYS_PPC64LE +#endif + +#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 64 +#define MLK_SYS_RISCV64 +#endif + +#if defined(MLK_SYS_RISCV64) && defined(__riscv_vector) && \ + defined(__riscv_v_intrinsic) +#define MLK_SYS_RISCV64_RVV +#endif + +#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 32 +#define MLK_SYS_RISCV32 +#endif + +#if defined(_WIN32) +#define MLK_SYS_WINDOWS +#endif + +#if defined(__linux__) +#define MLK_SYS_LINUX +#endif + +#if defined(__APPLE__) +#define MLK_SYS_APPLE +#endif + +#if defined(MLK_FORCE_AARCH64) && !defined(MLK_SYS_AARCH64) +#error "MLK_FORCE_AARCH64 is set, but we don't seem to be on an AArch64 system." +#endif + +#if defined(MLK_FORCE_AARCH64_EB) && !defined(MLK_SYS_AARCH64_EB) +#error \ + "MLK_FORCE_AARCH64_EB is set, but we don't seem to be on an AArch64 system." +#endif + +#if defined(MLK_FORCE_X86_64) && !defined(MLK_SYS_X86_64) +#error "MLK_FORCE_X86_64 is set, but we don't seem to be on an X86_64 system." +#endif + +#if defined(MLK_FORCE_PPC64LE) && !defined(MLK_SYS_PPC64LE) +#error "MLK_FORCE_PPC64LE is set, but we don't seem to be on a PPC64LE system." +#endif + +#if defined(MLK_FORCE_RISCV64) && !defined(MLK_SYS_RISCV64) +#error "MLK_FORCE_RISCV64 is set, but we don't seem to be on a RISCV64 system." +#endif + +#if defined(MLK_FORCE_RISCV32) && !defined(MLK_SYS_RISCV32) +#error "MLK_FORCE_RISCV32 is set, but we don't seem to be on a RISCV32 system." +#endif + +/* + * MLK_INLINE: Hint for inlining. + * - MSVC: __inline + * - C99+: inline + * - GCC/Clang C90: __attribute__((unused)) to silence warnings + * - Other C90: empty + */ +#if !defined(MLK_INLINE) +#if defined(_MSC_VER) +#define MLK_INLINE __inline +#elif defined(inline) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +#define MLK_INLINE inline +#elif defined(__GNUC__) || defined(__clang__) +#define MLK_INLINE __attribute__((unused)) +#else +#define MLK_INLINE +#endif +#endif /* !MLK_INLINE */ + +/* + * MLK_ALWAYS_INLINE: Force inlining. + * - MSVC: __forceinline + * - GCC/Clang C99+: MLK_INLINE __attribute__((always_inline)) + * - Other: MLK_INLINE (no forced inlining) + */ +#if !defined(MLK_ALWAYS_INLINE) +#if defined(_MSC_VER) +#define MLK_ALWAYS_INLINE __forceinline +#elif (defined(__GNUC__) || defined(__clang__)) && \ + (defined(inline) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L)) +#define MLK_ALWAYS_INLINE MLK_INLINE __attribute__((always_inline)) +#else +#define MLK_ALWAYS_INLINE MLK_INLINE +#endif +#endif /* !MLK_ALWAYS_INLINE */ + +/* + * MLK_NOINLINE: Prevent inlining. + * - MSVC: __declspec(noinline) + * - GCC/Clang: __attribute__((noinline)) + * - Other: empty + */ +#if !defined(MLK_NOINLINE) +#if defined(_MSC_VER) +#define MLK_NOINLINE __declspec(noinline) +#elif defined(__GNUC__) || defined(__clang__) +#define MLK_NOINLINE __attribute__((noinline)) +#else +#define MLK_NOINLINE +#endif +#endif /* !MLK_NOINLINE */ + +#ifndef MLK_STATIC_TESTABLE +#define MLK_STATIC_TESTABLE static +#endif + +/* + * C90 does not have the restrict compiler directive yet. + * We don't use it in C90 builds. + */ +#if !defined(restrict) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define MLK_RESTRICT restrict +#else +#define MLK_RESTRICT +#endif + +#else /* !restrict */ + +#define MLK_RESTRICT restrict +#endif /* restrict */ + +#define MLK_DEFAULT_ALIGN 32 +#define MLK_ALIGN_UP(N) \ + ((((N) + (MLK_DEFAULT_ALIGN - 1)) / MLK_DEFAULT_ALIGN) * MLK_DEFAULT_ALIGN) +#if defined(__GNUC__) +#define MLK_ALIGN __attribute__((aligned(MLK_DEFAULT_ALIGN))) +#elif defined(_MSC_VER) +#define MLK_ALIGN __declspec(align(MLK_DEFAULT_ALIGN)) +#else +#define MLK_ALIGN /* No known support for alignment constraints */ +#endif + + +/* New X86_64 CPUs support Conflow-flow protection using the CET instructions. + * When enabled (through -fcf-protection=), all compilation units (including + * empty ones) need to support CET for this to work. + * For assembly, this means that source files need to signal support for + * CET by setting the appropriate note.gnu.property section. + * This can be achieved by including the header in all assembly file. + * This file also provides the _CET_ENDBR macro which needs to be placed at + * every potential target of an indirect branch. + * If CET is enabled _CET_ENDBR maps to the endbr64 instruction, otherwise + * it is empty. + * In case the compiler does not support CET (e.g., +#define MLK_CET_ENDBR _CET_ENDBR +#else +#define MLK_CET_ENDBR +#endif +#endif /* MLK_SYS_X86_64 */ + +#if defined(MLK_CONFIG_CT_TESTING_ENABLED) && !defined(__ASSEMBLER__) +#include +#define MLK_CT_TESTING_SECRET(ptr, len) \ + VALGRIND_MAKE_MEM_UNDEFINED((ptr), (len)) +#define MLK_CT_TESTING_DECLASSIFY(ptr, len) \ + VALGRIND_MAKE_MEM_DEFINED((ptr), (len)) +#else /* MLK_CONFIG_CT_TESTING_ENABLED && !__ASSEMBLER__ */ +#define MLK_CT_TESTING_SECRET(ptr, len) \ + do \ + { \ + } while (0) +#define MLK_CT_TESTING_DECLASSIFY(ptr, len) \ + do \ + { \ + } while (0) +#endif /* !(MLK_CONFIG_CT_TESTING_ENABLED && !__ASSEMBLER__) */ + +#if defined(__GNUC__) || defined(__clang__) +#define MLK_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) +#else +#define MLK_MUST_CHECK_RETURN_VALUE +#endif + +/* The x86_64 assembly backend uses the SysV calling convention. On Windows, + * where the Microsoft x64 calling convention is the default, it can still be + * used with compilers that allow choosing the calling convention per + * function: GCC and Clang support __attribute__((sysv_abi)), which makes + * calls to the annotated function follow the SysV calling convention. + * + * MLK_SYSV_ABI_SUPPORTED signals that the toolchain can call SysV assembly + * routines; the x86_64 assembly backend is only enabled if it is defined. + * MLK_SYSV_ABI is the attribute carried by declarations of x86_64 assembly + * routines. Both macros can be set externally for toolchains offering an + * equivalent mechanism that is not recognized here. */ +#if defined(MLK_SYS_X86_64) && !defined(MLK_SYSV_ABI_SUPPORTED) +#if !defined(MLK_SYS_WINDOWS) || defined(__GNUC__) || defined(__clang__) +#define MLK_SYSV_ABI_SUPPORTED +#endif +#endif + +#if !defined(MLK_SYSV_ABI) +#if defined(MLK_SYS_WINDOWS) && defined(MLK_SYSV_ABI_SUPPORTED) +#define MLK_SYSV_ABI __attribute__((sysv_abi)) +#else +#define MLK_SYSV_ABI +#endif +#endif /* !MLK_SYSV_ABI */ + +#if !defined(__ASSEMBLER__) +/* System capability enumeration */ +typedef enum +{ + /* x86_64 */ + MLK_SYS_CAP_AVX2, + /* AArch64 */ + MLK_SYS_CAP_SHA3, + /* Armv8.1-M */ + MLK_SYS_CAP_MVE +} mlk_sys_cap; + +#if !defined(MLK_CONFIG_CUSTOM_CAPABILITY_FUNC) +#include "cbmc.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) +__contract__( + ensures(return_value == 0 || return_value == 1) +) +{ + /* By default, we rely on compile-time feature detection/specification: + * If a feature is enabled at compile-time, we assume it is supported by + * the host that the resulting library/binary will be built on. + * If this assumption is not true, you MUST overwrite this function. + * See the documentation of MLK_CONFIG_CUSTOM_CAPABILITY_FUNC in + * mlkem_native_config.h for more information. */ + (void)cap; + return 1; +} +#endif /* !MLK_CONFIG_CUSTOM_CAPABILITY_FUNC */ +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_SYS_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/verify.c b/sdk/software/examples/mlkem_test/mlkem_native/src/verify.c new file mode 100644 index 0000000..db760f7 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/verify.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#include "verify.h" + +#if !defined(MLK_USE_ASM_VALUE_BARRIER) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/* + * Masking value used in constant-time functions from + * verify.h to block the compiler's range analysis and + * thereby reduce the risk of compiler-introduced branches. + */ +volatile uint64_t mlk_ct_opt_blocker_u64 = 0; + +#else /* !MLK_USE_ASM_VALUE_BARRIER && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(verify) + +#endif /* !(!MLK_USE_ASM_VALUE_BARRIER && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/verify.h b/sdk/software/examples/mlkem_test/mlkem_native/src/verify.h new file mode 100644 index 0000000..b5f33b7 --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/verify.h @@ -0,0 +1,446 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + * + * - [libmceliece] + * libmceliece implementation of Classic McEliece + * Bernstein, Chou + * https://lib.mceliece.org/ + * + * - [optblocker] + * PQC forum post on opt-blockers using volatile globals + * Daniel J. Bernstein + * https://groups.google.com/a/list.nist.gov/g/pqc-forum/c/hqbtIGFKIpU/m/H14H0wOlBgAJ + */ + +#ifndef MLK_VERIFY_H +#define MLK_VERIFY_H + + +#include "cbmc.h" +#include "common.h" + +/* Constant-time comparisons and conditional operations + + We reduce the risk for compilation into variable-time code + through the use of 'value barriers'. + + Functionally, a value barrier is a no-op. To the compiler, however, + it constitutes an arbitrary modification of its input, and therefore + harden's value propagation and range analysis. + + We consider two approaches to implement a value barrier: + - An empty inline asm block which marks the target value as clobbered. + - XOR'ing with the value of a volatile global that's set to 0; + see @[optblocker] for a discussion of this idea, and + @[libmceliece, inttypes/crypto_intN.h] for an implementation. + + The first approach is cheap because it only prevents the compiler + from reasoning about the value of the variable past the barrier, + but does not directly generate additional instructions. + + The second approach generates redundant loads and XOR operations + and therefore comes at a higher runtime cost. However, it appears + more robust towards optimization, as compilers should never drop + a volatile load. + + We use the empty-ASM value barrier for GCC and clang, and fall + back to the global volatile barrier otherwise. + + The global value barrier can be forced by setting + MLK_CONFIG_NO_ASM_VALUE_BARRIER. + +*/ + +#if defined(MLK_HAVE_INLINE_ASM) && !defined(MLK_CONFIG_NO_ASM_VALUE_BARRIER) +#define MLK_USE_ASM_VALUE_BARRIER +#endif + +#if !defined(MLK_USE_ASM_VALUE_BARRIER) + +/* + * Declaration of global volatile that the global value barrier + * is loading from and masking with. + */ +#define mlk_ct_opt_blocker_u64 MLK_NAMESPACE(ct_opt_blocker_u64) +extern volatile uint64_t mlk_ct_opt_blocker_u64; + +/* Helper functions for obtaining global masks of various sizes */ + +/* This contract is not proved but treated as an axiom. + * + * Its validity relies on the assumption that the global opt-blocker + * constant mlk_ct_opt_blocker_u64 is not modified. + */ +static MLK_INLINE uint64_t mlk_ct_get_optblocker_u64(void) +__contract__(ensures(return_value == 0)) { return mlk_ct_opt_blocker_u64; } + +static MLK_INLINE uint8_t mlk_ct_get_optblocker_u8(void) +__contract__(ensures(return_value == 0)) { return (uint8_t)mlk_ct_get_optblocker_u64(); } + +static MLK_INLINE uint32_t mlk_ct_get_optblocker_u32(void) +__contract__(ensures(return_value == 0)) { return (uint32_t)mlk_ct_get_optblocker_u64(); } + +static MLK_INLINE int32_t mlk_ct_get_optblocker_i32(void) +__contract__(ensures(return_value == 0)) { return (int32_t)mlk_ct_get_optblocker_u64(); } + +/* Opt-blocker based implementation of value barriers */ +static MLK_INLINE uint32_t mlk_value_barrier_u32(uint32_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_u32()); } + +static MLK_INLINE int32_t mlk_value_barrier_i32(int32_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_i32()); } + +static MLK_INLINE uint8_t mlk_value_barrier_u8(uint8_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_u8()); } + +#else /* !MLK_USE_ASM_VALUE_BARRIER */ + +static MLK_INLINE uint32_t mlk_value_barrier_u32(uint32_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +static MLK_INLINE int32_t mlk_value_barrier_i32(int32_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +static MLK_INLINE uint8_t mlk_value_barrier_u8(uint8_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +#endif /* MLK_USE_ASM_VALUE_BARRIER */ + +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "conversion" +#endif +/** + * Cast uint16 value to int16. + * + * @param x Input value. + * + * @return For uint16_t x, the unique y in int16_t so that x == y mod 2^16. + * Concretely: + * - x < 32768: returns x + * - x >= 32768: returns x - 65536 + */ +static MLK_ALWAYS_INLINE int16_t mlk_cast_uint16_to_int16(uint16_t x) +{ + /* + * PORTABILITY: This relies on uint16_t -> int16_t + * being implemented as the inverse of int16_t -> uint16_t, + * which is implementation-defined (C99 6.3.1.3 (3)) + * CBMC (correctly) fails to prove this conversion is OK, + * so we have to suppress that check here + */ + return (int16_t)x; +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Cast int32 value to uint16 as per C standard. + * + * @param x Input value. + * + * @return For int32_t x, the unique y in uint16_t so that x == y mod 2^16. + */ +static MLK_ALWAYS_INLINE uint16_t mlk_cast_int32_to_uint16(int32_t x) +{ + return (uint16_t)(x & (int32_t)UINT16_MAX); +} + +/** + * Cast int16 value to uint16 as per C standard. + * + * @param x Input value. + * + * @return For int16_t x, the unique y in uint16_t so that x == y mod 2^16. + */ +static MLK_ALWAYS_INLINE uint16_t mlk_cast_int16_to_uint16(int32_t x) +{ + return mlk_cast_int32_to_uint16(x); +} + +/** + * Return 0 if input is non-negative, and -1 otherwise. + * + * @reference{Embedded in the polynomial compression function in the + * reference implementation @[REF]. Used as part of signed->unsigned + * conversion for modular representatives to detect whether the input is + * negative. This happens in `mlk_poly_reduce()` here, and as part of + * polynomial compression functions in the reference implementation. See + * `mlk_poly_reduce()`. We use value barriers to reduce the risk of + * compiler-introduced branches.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFFFF). + */ +static MLK_INLINE uint16_t mlk_ct_cmask_neg_i16(int16_t x) +__contract__(ensures(return_value == ((x < 0) ? 0xFFFF : 0))) +{ + int32_t tmp = mlk_value_barrier_i32((int32_t)x); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + tmp >>= 16; + return mlk_cast_int32_to_uint16(tmp); +} + +/** + * Return 0 if input is zero, and -1 otherwise. + * + * @reference{Embedded in `cmov_int16()` in the reference implementation + * @[REF]. Uses a value barrier and shift instead of `b = -b` to convert + * condition into mask.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFFFF). + */ +static MLK_INLINE uint16_t mlk_ct_cmask_nonzero_u16(uint16_t x) +__contract__(ensures(return_value == ((x == 0) ? 0 : 0xFFFF))) +{ + int32_t tmp = mlk_value_barrier_i32(-((int32_t)x)); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + tmp >>= 16; + return mlk_cast_int32_to_uint16(tmp); +} + +/** + * Return 0 if input is zero, and -1 otherwise. + * + * @reference{Embedded in `verify()` and `cmov()` in the reference + * implementation @[REF]. We include a value barrier not present in the + * reference implementation, to prevent the compiler from realizing that + * this function returns a mask.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFF). + */ +static MLK_INLINE uint8_t mlk_ct_cmask_nonzero_u8(uint8_t x) +__contract__(ensures(return_value == ((x == 0) ? 0 : 0xFF))) +{ + uint16_t mask = mlk_ct_cmask_nonzero_u16((uint16_t)x); + return (uint8_t)(mask & 0xFF); +} + +/** + * Functionally equivalent to cond ? a : b, but implemented with guards + * against compiler-introduced branches. + * + * @spec{With `a = MLKEM_Q_HALF` and `b=0`, this essentially implements + * `Decompress_1` @[FIPS203, Eq (4.8)] in `mlk_poly_frommsg()`. With + * `a = x + MLKEM_Q`, `b = x`, and `cond` indicating whether `x` is negative, + * implements signed->unsigned conversion of modular representatives. + * Questions of representation are not considered in the specification + * @[FIPS203, Section 2.4.1, "The pseudocode is agnostic regarding how an + * integer modulo 𝑚 is represented in actual implementations"].} + * + * @reference{Embedded in the polynomial compression function in the + * reference implementation @[REF]. Used as part of signed->unsigned + * conversion for modular representatives. This happens in `mlk_poly_reduce()` + * here, and as part of polynomial compression functions in @[REF]. See + * `mlk_poly_reduce()`. Barrier to reduce the risk of compiler-introduced + * branches. For `a = MLKEM_Q_HALF` and `b=0`, also embedded in + * `poly_frommsg()` from the reference implementation, which uses + * `cmov_int16()` instead.} + * + * @param a First alternative. + * @param b Second alternative. + * @param cond Condition variable. + * + * @return @p a if @p cond != 0, else @p b. + */ +static MLK_INLINE int16_t mlk_ct_sel_int16(int16_t a, int16_t b, uint16_t cond) +__contract__(ensures(return_value == (cond ? a : b))) +{ + uint16_t au = mlk_cast_int16_to_uint16(a); + uint16_t bu = mlk_cast_int16_to_uint16(b); + uint16_t res = bu ^ (mlk_ct_cmask_nonzero_u16(cond) & (au ^ bu)); + return mlk_cast_uint16_to_int16(res); +} + +/** + * Functionally equivalent to cond ? a : b, but implemented with guards + * against compiler-introduced branches. + * + * @reference{Embedded into `cmov()` in the reference implementation @[REF]. + * Uses a value barrier to get mask from condition value.} + * + * @param a First alternative. + * @param b Second alternative. + * @param cond Condition variable. + * + * @return @p a if @p cond != 0, else @p b. + */ +static MLK_INLINE uint8_t mlk_ct_sel_uint8(uint8_t a, uint8_t b, uint8_t cond) +__contract__(ensures(return_value == (cond ? a : b))) +{ + return b ^ (mlk_ct_cmask_nonzero_u8(cond) & (a ^ b)); +} + +/** + * Compare two arrays for equality in constant time. + * + * @spec{Used to securely compute conditional move in @[FIPS203, Algorithm + * 18 (ML-KEM.Decaps_Internal, L9-11].} + * + * @reference{`cmov()` in the reference implementation @[REF]. We return + * `uint8_t`, not `int`. We use an additional XOR-accumulator in the + * comparison loop which prevents early abort if the OR-accumulator is 0xFF. + * We use a value barrier to convert the OR-accumulator into a mask; the + * reference implementation uses a shift which the compiler can argue to + * result in either 0 or 0xFF..FF.} + * + * @param[in] a First byte array. + * @param[in] b Second byte array. + * @param len Length of the byte arrays, upper-bounded to UINT16_MAX to + * control proof complexity only. + * + * @retval 0 The byte arrays are equal. + * @retval 0xFF The byte arrays are not equal. + */ +static MLK_INLINE uint8_t mlk_ct_memcmp(const uint8_t *a, const uint8_t *b, + const size_t len) +__contract__( + requires(len <= UINT16_MAX) + requires(memory_no_alias(a, len)) + requires(memory_no_alias(b, len)) + ensures((return_value == 0) || (return_value == 0xFF)) + ensures((return_value == 0) == forall(i, 0, len, (a[i] == b[i])))) +{ + uint8_t r = 0, s = 0; + unsigned i; + + for (i = 0; i < len; i++) + __loop__( + invariant(i <= len) + invariant((r == 0) == (forall(k, 0, i, (a[k] == b[k])))) + decreases(len - i)) + { + r |= a[i] ^ b[i]; + /* s is useless, but prevents the loop from being aborted once r=0xff. */ + s ^= a[i] ^ b[i]; + } + + /* + * - Convert r into a mask; this may not be necessary, but is an additional + * safeguard + * towards leaking information about a and b. + * - XOR twice with s, separated by a value barrier, to prevent the compile + * from dropping the s computation in the loop. + */ + return (mlk_value_barrier_u8(mlk_ct_cmask_nonzero_u8(r) ^ s) ^ s); +} + +/** + * Copy len bytes from x to r if b is zero; don't modify x if b is non-zero. + * Assumes two's complement representation of negative integers. Runs in + * constant time. + * + * @spec{Used to securely compute conditional move in @[FIPS203, Algorithm + * 18 (ML-KEM.Decaps_Internal, L9-11].} + * + * @reference{`cmov()` in the reference implementation @[REF]. We move if + * condition value is `0`, not `1`. We use `mlk_ct_sel_uint8` for + * constant-time selection.} + * + * @param[out] r Output byte array. + * @param[in] x Input byte array. + * @param len Number of bytes to be copied. + * @param b Condition value. + */ +static MLK_INLINE void mlk_ct_cmov_zero(uint8_t *r, const uint8_t *x, + size_t len, uint8_t b) +__contract__( + requires(len <= UINT32_MAX) + requires(memory_no_alias(r, len)) + requires(memory_no_alias(x, len)) + assigns(memory_slice(r, len)) + ensures(forall(i, 0, len, (r[i] == (b == 0 ? x[i] : old(r)[i]))))) +{ + size_t i; + for (i = 0; i < len; i++) + __loop__( + invariant(i <= len) + invariant(forall(k, 0, i, r[k] == (b == 0 ? x[k] : loop_entry(r)[k]))) + decreases(len - i)) + { + r[i] = mlk_ct_sel_uint8(r[i], x[i], b); + } +} + +/** + * Force-zeroize a buffer. + * + * @spec{Used to implement @[FIPS203, Section 3.3, Destruction of + * intermediate values].} + * + * @reference{Not present in the reference implementation @[REF].} + * + * @param[out] ptr Buffer to be zeroed. + * @param len Number of bytes to be zeroed. + */ +#if !defined(MLK_CONFIG_CUSTOM_ZEROIZE) +#if defined(MLK_SYS_WINDOWS) +#include +static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) +__contract__( + requires(memory_no_alias(ptr, len)) + assigns(memory_slice(ptr, len))) { SecureZeroMemory(ptr, len); } +#elif defined(MLK_HAVE_INLINE_ASM) +#include +static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) +__contract__( + requires(memory_no_alias(ptr, len)) + assigns(memory_slice(ptr, len))) +{ + mlk_memset(ptr, 0, len); + /* This follows OpenSSL and seems sufficient to prevent the compiler + * from optimizing away the memset. + * + * If there was a reliable way to detect availability of memset_s(), + * that would be preferred. */ + __asm__ volatile("" : : "r"(ptr) : "memory"); +} +#else /* !MLK_SYS_WINDOWS && MLK_HAVE_INLINE_ASM */ +#error No plausibly-secure implementation of mlk_zeroize available. Please provide your own using MLK_CONFIG_CUSTOM_ZEROIZE. +#endif /* !MLK_SYS_WINDOWS && !MLK_HAVE_INLINE_ASM */ +#endif /* !MLK_CONFIG_CUSTOM_ZEROIZE */ + +#endif /* !MLK_VERIFY_H */ diff --git a/sdk/software/examples/mlkem_test/mlkem_native/src/zetas.inc b/sdk/software/examples/mlkem_test/mlkem_native/src/zetas.inc new file mode 100644 index 0000000..00316da --- /dev/null +++ b/sdk/software/examples/mlkem_test/mlkem_native/src/zetas.inc @@ -0,0 +1,30 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + + +/* + * Table of zeta values used in the reference NTT and inverse NTT. + * See autogen for details. + */ +static MLK_ALIGN const int16_t mlk_zetas[128] = { + -1044, -758, -359, -1517, 1493, 1422, 287, 202, -171, 622, 1577, + 182, 962, -1202, -1474, 1468, 573, -1325, 264, 383, -829, 1458, + -1602, -130, -681, 1017, 732, 608, -1542, 411, -205, -1571, 1223, + 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, + -1618, -1162, 126, 1469, -853, -90, -271, 830, 107, -1421, -247, + -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, -1103, 430, + 555, 843, -1251, 871, 1550, 105, 422, 587, 177, -235, -291, + -460, 1574, 1653, -246, 778, 1159, -147, -777, 1483, -602, 1119, + -1590, 644, -872, 349, 418, 329, -156, -75, 817, 1097, 603, + 610, 1322, -1285, -1465, 384, -1215, -136, 1218, -1335, -874, 220, + -1187, -1659, -1185, -1530, -1278, 794, -1510, -854, -870, 478, -108, + -308, 996, 991, 958, -1460, 1522, 1628, +}; diff --git a/sdk/software/examples/mlkem_test/sw_mlkem_config.h b/sdk/software/examples/mlkem_test/sw_mlkem_config.h new file mode 100644 index 0000000..9a56cfd --- /dev/null +++ b/sdk/software/examples/mlkem_test/sw_mlkem_config.h @@ -0,0 +1,28 @@ +/* + * Local mlkem-native configuration for the ML-KEM hardware comparison test. + * + * The target bare-metal toolchain does not expose one of mlkem-native's + * built-in secure zeroization backends once assembly is disabled, so provide a + * small volatile-byte zeroizer here. + */ +#ifndef SW_MLKEM_CONFIG_H +#define SW_MLKEM_CONFIG_H + +#include "mlkem_native_config.h" + +#define MLK_CONFIG_CUSTOM_ZEROIZE + +#if !defined(__ASSEMBLER__) +#include +#include +#include "src/sys.h" + +static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) +{ + volatile uint8_t *p = (volatile uint8_t *)ptr; + while (len-- != 0u) + *p++ = 0u; +} +#endif + +#endif