mlkem_test: add mlkem-native
This commit is contained in:
294
sdk/software/examples/mlkem_test/mlkem_native/LICENSE
Normal file
294
sdk/software/examples/mlkem_test/mlkem_native/LICENSE
Normal file
@@ -0,0 +1,294 @@
|
||||
mlkem-native is a fork of the public domain Kyber reference
|
||||
implementation, available on https://github.com/pq-crystals/kyber.
|
||||
|
||||
All source code in mlkem/* and dev/* is licensed under your choice
|
||||
of the Apache-2.0 license OR the ISC license OR the MIT license.
|
||||
These licenses are reproduced at the bottom of this file.
|
||||
The copyright holders are indicated at the top of each file.
|
||||
|
||||
The code in test/notrandombytes/* is derived from
|
||||
https://cr.yp.to/papers.html#surf and licensed under
|
||||
LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT.
|
||||
It is only used for testing purposes.
|
||||
|
||||
The benchmarking code in test/hal/hal.c carries the
|
||||
MIT license. It is only used for testing purposes.
|
||||
|
||||
```
|
||||
Copyright (c) The mlkem-native project authors
|
||||
Copyright (c) 2020 Dougall Johnson
|
||||
Copyright (c) 2022 Arm Limited
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
```
|
||||
|
||||
The tiny_sha3 implementation in examples/custom_backend/mlkem_native/mlkem/fips202/native/custom/src/
|
||||
carries the MIT license. It is only used for testing purposes.
|
||||
|
||||
```
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
19-Nov-11 Markku-Juhani O. Saarinen <mjos@iki.fi>
|
||||
```
|
||||
|
||||
ISC license
|
||||
-----------
|
||||
|
||||
Copyright <YEAR> <OWNER>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any purpose
|
||||
with or without fee is hereby granted, provided that the above copyright notice
|
||||
and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
|
||||
OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
|
||||
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
|
||||
THIS SOFTWARE.
|
||||
|
||||
|
||||
MIT license
|
||||
-----------
|
||||
|
||||
Copyright <YEAR> <COPYRIGHT HOLDER>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the “Software”), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
Apache-2.0 license
|
||||
------------------
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
23
sdk/software/examples/mlkem_test/mlkem_native/README.md
Normal file
23
sdk/software/examples/mlkem_test/mlkem_native/README.md
Normal file
@@ -0,0 +1,23 @@
|
||||
[//]: # (SPDX-License-Identifier: CC-BY-4.0)
|
||||
|
||||
# mlkem-native source tree
|
||||
|
||||
This is the main source tree of mlkem-native.
|
||||
|
||||
## Building
|
||||
|
||||
To build a mlkem-native for a fixed parameter set (ML-KEM-512/768/1024), build the compilation in units in `src/*` separately, and link to an RNG and your application. See [examples/basic](../examples/basic) for a simple example.
|
||||
|
||||
Alternatively, you can use the auto-geneated helper files [mlkem_native.c](mlkem_native.c) and [mlkem_native_asm.S](mlkem_native_asm.S), which bundle all *.c and *.S files together. See [examples/monolithic_build](../examples/monolithic_build) and [examples/monolithic_build_native](../examples/monolithic_build_native) for examples with and without native code.
|
||||
|
||||
## Configuration
|
||||
|
||||
The build is configured by [mlkem_native_config.h](mlkem_native_config.h), or by the file pointed to by `MLK_CONFIG_FILE`. Note in particular `MLK_CONFIG_PARAMETER_SET` and `MLK_CONFIG_NAMESPACE_PREFIX`, which set the parameter set and namespace prefix, respectively.
|
||||
|
||||
## API
|
||||
|
||||
The public API is defined in [mlkem_native.h](mlkem_native.h).
|
||||
|
||||
## Supporting multiple parameter sets
|
||||
|
||||
If you want to support multiple parameter sets, build the library once per parameter set you want to support. Set `MLK_CONFIG_MULTILEVEL_WITH_SHARED` for one of the builds, and `MLK_CONFIG_MULTILEVEL_NO_SHARED` for the others, to avoid duplicating shared functionality. Finally, link with RNG and your application as before. This is demonstrated in the examples [examples/multilevel_build](../examples/multilevel_build), [examples/multilevel_build_native](../examples/multilevel_build_native), [examples/monolithic_build_multilevel](../examples/monolithic_build_multilevel) and [examples/monolithic_build_multilevel_native](../examples/monolithic_build_multilevel_native).
|
||||
699
sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.c
Normal file
699
sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.c
Normal file
@@ -0,0 +1,699 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* Single compilation unit (SCU) for fixed-level build of mlkem-native
|
||||
*
|
||||
* This compilation unit bundles together all source files for a build
|
||||
* of mlkem-native for a fixed security level (MLKEM-512/768/1024).
|
||||
*
|
||||
* # API
|
||||
*
|
||||
* The API exposed by this file is described in mlkem_native.h.
|
||||
*
|
||||
* # Multi-level build
|
||||
*
|
||||
* If you want an SCU build of mlkem-native with support for multiple security
|
||||
* levels, you need to include this file multiple times, and set
|
||||
* MLK_CONFIG_MULTILEVEL_WITH_SHARED and MLK_CONFIG_MULTILEVEL_NO_SHARED
|
||||
* appropriately. This is exemplified in examples/monolithic_build_multilevel
|
||||
* and examples/monolithic_build_multilevel_native.
|
||||
*
|
||||
* # Configuration
|
||||
*
|
||||
* The following options from the mlkem-native configuration are relevant:
|
||||
*
|
||||
* - MLK_CONFIG_FIPS202_CUSTOM_HEADER
|
||||
* Set this option if you use a custom FIPS202 implementation.
|
||||
*
|
||||
* - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH
|
||||
* Set this option if you want to include the native arithmetic backends
|
||||
* in your build.
|
||||
*
|
||||
* - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202
|
||||
* Set this option if you want to include the native FIPS202 backends
|
||||
* in your build.
|
||||
*
|
||||
* - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS
|
||||
* Set this option if you want to keep the directives defined in
|
||||
* level-independent headers. This is needed for a multi-level build.
|
||||
*/
|
||||
|
||||
/* If parts of the mlkem-native source tree are not used,
|
||||
* consider reducing this header via `unifdef`.
|
||||
*
|
||||
* Example:
|
||||
* ```bash
|
||||
* unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native.c
|
||||
* ```
|
||||
*/
|
||||
|
||||
#include "src/common.h"
|
||||
|
||||
#include "src/compress.c"
|
||||
#include "src/debug.c"
|
||||
#include "src/indcpa.c"
|
||||
#include "src/kem.c"
|
||||
#include "src/poly.c"
|
||||
#include "src/poly_k.c"
|
||||
#include "src/sampling.c"
|
||||
#include "src/verify.c"
|
||||
|
||||
#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER)
|
||||
#include "src/fips202/fips202.c"
|
||||
#include "src/fips202/fips202x4.c"
|
||||
#include "src/fips202/keccakf1600.c"
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH)
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
#include "src/native/aarch64/src/aarch64_zetas.c"
|
||||
#include "src/native/aarch64/src/rej_uniform_table.c"
|
||||
#endif
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
#include "src/native/x86_64/src/compress_consts.c"
|
||||
#include "src/native/x86_64/src/consts.c"
|
||||
#include "src/native/x86_64/src/rej_uniform_table.c"
|
||||
#endif
|
||||
#if defined(MLK_SYS_RISCV64)
|
||||
#include "src/native/riscv64/src/rv64v_debug.c"
|
||||
#include "src/native/riscv64/src/rv64v_poly.c"
|
||||
#endif
|
||||
#if defined(MLK_SYS_PPC64LE)
|
||||
#include "src/native/ppc64le/src/consts.c"
|
||||
#endif
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202)
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
#include "src/fips202/native/aarch64/src/keccakf1600_round_constants.c"
|
||||
#endif
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
#include "src/fips202/native/x86_64/src/keccakf1600_constants.c"
|
||||
#endif
|
||||
#if defined(MLK_SYS_ARMV81M_MVE)
|
||||
#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c"
|
||||
#include "src/fips202/native/armv81m/src/keccakf1600_round_constants.c"
|
||||
#endif
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */
|
||||
|
||||
/* Macro #undef's
|
||||
*
|
||||
* The following undefines macros from headers
|
||||
* included by the source files imported above.
|
||||
*
|
||||
* This is to allow building and linking multiple builds
|
||||
* of mlkem-native for varying parameter sets through concatenation
|
||||
* of this file, as if the files had been compiled separately.
|
||||
* If this is not relevant to you, you may remove the following.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files
|
||||
*/
|
||||
/* mlkem/mlkem_native.h */
|
||||
#undef CRYPTO_BYTES
|
||||
#undef CRYPTO_CIPHERTEXTBYTES
|
||||
#undef CRYPTO_PUBLICKEYBYTES
|
||||
#undef CRYPTO_SECRETKEYBYTES
|
||||
#undef CRYPTO_SYMBYTES
|
||||
#undef MLKEM1024_BYTES
|
||||
#undef MLKEM1024_CIPHERTEXTBYTES
|
||||
#undef MLKEM1024_PUBLICKEYBYTES
|
||||
#undef MLKEM1024_SECRETKEYBYTES
|
||||
#undef MLKEM1024_SYMBYTES
|
||||
#undef MLKEM512_BYTES
|
||||
#undef MLKEM512_CIPHERTEXTBYTES
|
||||
#undef MLKEM512_PUBLICKEYBYTES
|
||||
#undef MLKEM512_SECRETKEYBYTES
|
||||
#undef MLKEM512_SYMBYTES
|
||||
#undef MLKEM768_BYTES
|
||||
#undef MLKEM768_CIPHERTEXTBYTES
|
||||
#undef MLKEM768_PUBLICKEYBYTES
|
||||
#undef MLKEM768_SECRETKEYBYTES
|
||||
#undef MLKEM768_SYMBYTES
|
||||
#undef MLKEM_BYTES
|
||||
#undef MLKEM_CIPHERTEXTBYTES
|
||||
#undef MLKEM_CIPHERTEXTBYTES_
|
||||
#undef MLKEM_PUBLICKEYBYTES
|
||||
#undef MLKEM_PUBLICKEYBYTES_
|
||||
#undef MLKEM_SECRETKEYBYTES
|
||||
#undef MLKEM_SECRETKEYBYTES_
|
||||
#undef MLKEM_SYMBYTES
|
||||
#undef MLK_API_CONCAT
|
||||
#undef MLK_API_CONCAT_
|
||||
#undef MLK_API_CONCAT_UNDERSCORE
|
||||
#undef MLK_API_LEGACY_CONFIG
|
||||
#undef MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
#undef MLK_API_NAMESPACE
|
||||
#undef MLK_API_QUALIFIER
|
||||
#undef MLK_CONFIG_API_CONSTANTS_ONLY
|
||||
#undef MLK_CONFIG_API_NAMESPACE_PREFIX
|
||||
#undef MLK_CONFIG_API_NO_SUPERCOP
|
||||
#undef MLK_CONFIG_API_PARAMETER_SET
|
||||
#undef MLK_CONFIG_API_QUALIFIER
|
||||
#undef MLK_ERR_FAIL
|
||||
#undef MLK_ERR_OUT_OF_MEMORY
|
||||
#undef MLK_ERR_RNG_FAIL
|
||||
#undef MLK_H
|
||||
#undef MLK_MAX3_
|
||||
#undef MLK_TOTAL_ALLOC_1024
|
||||
#undef MLK_TOTAL_ALLOC_1024_DECAPS
|
||||
#undef MLK_TOTAL_ALLOC_1024_ENCAPS
|
||||
#undef MLK_TOTAL_ALLOC_1024_KEYPAIR
|
||||
#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT
|
||||
#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT
|
||||
#undef MLK_TOTAL_ALLOC_512
|
||||
#undef MLK_TOTAL_ALLOC_512_DECAPS
|
||||
#undef MLK_TOTAL_ALLOC_512_ENCAPS
|
||||
#undef MLK_TOTAL_ALLOC_512_KEYPAIR
|
||||
#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT
|
||||
#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT
|
||||
#undef MLK_TOTAL_ALLOC_768
|
||||
#undef MLK_TOTAL_ALLOC_768_DECAPS
|
||||
#undef MLK_TOTAL_ALLOC_768_ENCAPS
|
||||
#undef MLK_TOTAL_ALLOC_768_KEYPAIR
|
||||
#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT
|
||||
#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT
|
||||
#undef crypto_kem_check_pk
|
||||
#undef crypto_kem_check_sk
|
||||
#undef crypto_kem_dec
|
||||
#undef crypto_kem_enc
|
||||
#undef crypto_kem_enc_derand
|
||||
#undef crypto_kem_keypair
|
||||
#undef crypto_kem_keypair_derand
|
||||
/* mlkem/src/common.h */
|
||||
#undef MLK_ADD_PARAM_SET
|
||||
#undef MLK_ALLOC
|
||||
#undef MLK_APPLY
|
||||
#undef MLK_ASM_FN_SIZE
|
||||
#undef MLK_ASM_FN_SYMBOL
|
||||
#undef MLK_ASM_NAMESPACE
|
||||
#undef MLK_BUILD_INTERNAL
|
||||
#undef MLK_COMMON_H
|
||||
#undef MLK_CONCAT
|
||||
#undef MLK_CONCAT_
|
||||
#undef MLK_CONTEXT_PARAMETERS_0
|
||||
#undef MLK_CONTEXT_PARAMETERS_1
|
||||
#undef MLK_CONTEXT_PARAMETERS_2
|
||||
#undef MLK_CONTEXT_PARAMETERS_3
|
||||
#undef MLK_CONTEXT_PARAMETERS_4
|
||||
#undef MLK_EMPTY_CU
|
||||
#undef MLK_ERR_FAIL
|
||||
#undef MLK_ERR_OUT_OF_MEMORY
|
||||
#undef MLK_ERR_RNG_FAIL
|
||||
#undef MLK_EXTERNAL_API
|
||||
#undef MLK_FIPS202X4_HEADER_FILE
|
||||
#undef MLK_FIPS202_HEADER_FILE
|
||||
#undef MLK_FREE
|
||||
#undef MLK_INTERNAL_API
|
||||
#undef MLK_INTERNAL_DATA_DECLARATION
|
||||
#undef MLK_INTERNAL_DATA_DEFINITION
|
||||
#undef MLK_NAMESPACE
|
||||
#undef MLK_NAMESPACE_K
|
||||
#undef MLK_NAMESPACE_PREFIX
|
||||
#undef MLK_NAMESPACE_PREFIX_K
|
||||
#undef mlk_memcpy
|
||||
#undef mlk_memset
|
||||
/* mlkem/src/indcpa.h */
|
||||
#undef MLK_INDCPA_H
|
||||
#undef mlk_gen_matrix
|
||||
#undef mlk_indcpa_dec
|
||||
#undef mlk_indcpa_enc
|
||||
#undef mlk_indcpa_keypair_derand
|
||||
/* mlkem/src/kem.h */
|
||||
#undef MLK_KEM_H
|
||||
#undef mlk_kem_check_pk
|
||||
#undef mlk_kem_check_sk
|
||||
#undef mlk_kem_dec
|
||||
#undef mlk_kem_enc
|
||||
#undef mlk_kem_enc_derand
|
||||
#undef mlk_kem_keypair
|
||||
#undef mlk_kem_keypair_derand
|
||||
/* mlkem/src/params.h */
|
||||
#undef MLKEM_DU
|
||||
#undef MLKEM_DV
|
||||
#undef MLKEM_ETA1
|
||||
#undef MLKEM_ETA2
|
||||
#undef MLKEM_INDCCA_CIPHERTEXTBYTES
|
||||
#undef MLKEM_INDCCA_PUBLICKEYBYTES
|
||||
#undef MLKEM_INDCCA_SECRETKEYBYTES
|
||||
#undef MLKEM_INDCPA_BYTES
|
||||
#undef MLKEM_INDCPA_MSGBYTES
|
||||
#undef MLKEM_INDCPA_PUBLICKEYBYTES
|
||||
#undef MLKEM_INDCPA_SECRETKEYBYTES
|
||||
#undef MLKEM_K
|
||||
#undef MLKEM_N
|
||||
#undef MLKEM_POLYBYTES
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D10
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D11
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D4
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D5
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_DU
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_DV
|
||||
#undef MLKEM_POLYVECBYTES
|
||||
#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU
|
||||
#undef MLKEM_Q
|
||||
#undef MLKEM_Q_HALF
|
||||
#undef MLKEM_SSBYTES
|
||||
#undef MLKEM_SYMBYTES
|
||||
#undef MLKEM_UINT12_LIMIT
|
||||
#undef MLK_PARAMS_H
|
||||
/* mlkem/src/poly_k.h */
|
||||
#undef MLK_POLY_K_H
|
||||
#undef mlk_poly_compress_du
|
||||
#undef mlk_poly_compress_dv
|
||||
#undef mlk_poly_decompress_du
|
||||
#undef mlk_poly_decompress_dv
|
||||
#undef mlk_poly_getnoise_eta1122_4x
|
||||
#undef mlk_poly_getnoise_eta1_4x
|
||||
#undef mlk_poly_getnoise_eta2
|
||||
#undef mlk_poly_getnoise_eta2_4x
|
||||
#undef mlk_polymat
|
||||
#undef mlk_polyvec
|
||||
#undef mlk_polyvec_add
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached
|
||||
#undef mlk_polyvec_compress_du
|
||||
#undef mlk_polyvec_decompress_du
|
||||
#undef mlk_polyvec_frombytes
|
||||
#undef mlk_polyvec_invntt_tomont
|
||||
#undef mlk_polyvec_mulcache
|
||||
#undef mlk_polyvec_mulcache_compute
|
||||
#undef mlk_polyvec_ntt
|
||||
#undef mlk_polyvec_reduce
|
||||
#undef mlk_polyvec_tobytes
|
||||
#undef mlk_polyvec_tomont
|
||||
|
||||
#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS)
|
||||
/*
|
||||
* Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files
|
||||
*/
|
||||
/* mlkem/src/compress.h */
|
||||
#undef MLK_COMPRESS_H
|
||||
#undef mlk_poly_compress_d10
|
||||
#undef mlk_poly_compress_d11
|
||||
#undef mlk_poly_compress_d4
|
||||
#undef mlk_poly_compress_d5
|
||||
#undef mlk_poly_decompress_d10
|
||||
#undef mlk_poly_decompress_d11
|
||||
#undef mlk_poly_decompress_d4
|
||||
#undef mlk_poly_decompress_d5
|
||||
#undef mlk_poly_frombytes
|
||||
#undef mlk_poly_frommsg
|
||||
#undef mlk_poly_tobytes
|
||||
#undef mlk_poly_tomsg
|
||||
/* mlkem/src/debug.h */
|
||||
#undef MLK_DEBUG_H
|
||||
#undef mlk_assert
|
||||
#undef mlk_assert_abs_bound
|
||||
#undef mlk_assert_abs_bound_2d
|
||||
#undef mlk_assert_bound
|
||||
#undef mlk_assert_bound_2d
|
||||
#undef mlk_debug_check_assert
|
||||
#undef mlk_debug_check_bounds
|
||||
/* mlkem/src/poly.h */
|
||||
#undef MLK_INVNTT_BOUND
|
||||
#undef MLK_NTT_BOUND
|
||||
#undef MLK_POLY_H
|
||||
#undef mlk_poly_add
|
||||
#undef mlk_poly_invntt_tomont
|
||||
#undef mlk_poly_mulcache_compute
|
||||
#undef mlk_poly_ntt
|
||||
#undef mlk_poly_reduce
|
||||
#undef mlk_poly_sub
|
||||
#undef mlk_poly_tomont
|
||||
/* mlkem/src/randombytes.h */
|
||||
#undef MLK_RANDOMBYTES_H
|
||||
/* mlkem/src/sampling.h */
|
||||
#undef MLK_SAMPLING_H
|
||||
#undef mlk_poly_cbd2
|
||||
#undef mlk_poly_cbd3
|
||||
#undef mlk_poly_rej_uniform
|
||||
#undef mlk_poly_rej_uniform_x4
|
||||
/* mlkem/src/symmetric.h */
|
||||
#undef MLK_SYMMETRIC_H
|
||||
#undef MLK_XOF_RATE
|
||||
#undef mlk_hash_g
|
||||
#undef mlk_hash_h
|
||||
#undef mlk_hash_j
|
||||
#undef mlk_prf_eta
|
||||
#undef mlk_prf_eta1
|
||||
#undef mlk_prf_eta1_x4
|
||||
#undef mlk_prf_eta2
|
||||
#undef mlk_xof_absorb
|
||||
#undef mlk_xof_ctx
|
||||
#undef mlk_xof_init
|
||||
#undef mlk_xof_release
|
||||
#undef mlk_xof_squeezeblocks
|
||||
#undef mlk_xof_x4_absorb
|
||||
#undef mlk_xof_x4_ctx
|
||||
#undef mlk_xof_x4_init
|
||||
#undef mlk_xof_x4_release
|
||||
#undef mlk_xof_x4_squeezeblocks
|
||||
/* mlkem/src/sys.h */
|
||||
#undef MLK_ALIGN
|
||||
#undef MLK_ALIGN_UP
|
||||
#undef MLK_ALWAYS_INLINE
|
||||
#undef MLK_CET_ENDBR
|
||||
#undef MLK_CT_TESTING_DECLASSIFY
|
||||
#undef MLK_CT_TESTING_SECRET
|
||||
#undef MLK_DEFAULT_ALIGN
|
||||
#undef MLK_HAVE_INLINE_ASM
|
||||
#undef MLK_INLINE
|
||||
#undef MLK_MUST_CHECK_RETURN_VALUE
|
||||
#undef MLK_NOINLINE
|
||||
#undef MLK_RESTRICT
|
||||
#undef MLK_STATIC_TESTABLE
|
||||
#undef MLK_SYSV_ABI
|
||||
#undef MLK_SYSV_ABI_SUPPORTED
|
||||
#undef MLK_SYS_AARCH64
|
||||
#undef MLK_SYS_AARCH64_EB
|
||||
#undef MLK_SYS_APPLE
|
||||
#undef MLK_SYS_ARMV81M_MVE
|
||||
#undef MLK_SYS_BIG_ENDIAN
|
||||
#undef MLK_SYS_H
|
||||
#undef MLK_SYS_LINUX
|
||||
#undef MLK_SYS_LITTLE_ENDIAN
|
||||
#undef MLK_SYS_PPC64LE
|
||||
#undef MLK_SYS_RISCV32
|
||||
#undef MLK_SYS_RISCV64
|
||||
#undef MLK_SYS_RISCV64_RVV
|
||||
#undef MLK_SYS_WINDOWS
|
||||
#undef MLK_SYS_X86_64
|
||||
#undef MLK_SYS_X86_64_AVX2
|
||||
/* mlkem/src/verify.h */
|
||||
#undef MLK_USE_ASM_VALUE_BARRIER
|
||||
#undef MLK_VERIFY_H
|
||||
#undef mlk_ct_opt_blocker_u64
|
||||
/* mlkem/src/cbmc.h */
|
||||
#undef MLK_CBMC_H
|
||||
#undef __contract__
|
||||
#undef __loop__
|
||||
|
||||
#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER)
|
||||
/*
|
||||
* Undefine macros from FIPS-202 files
|
||||
*/
|
||||
/* mlkem/src/fips202/fips202.h */
|
||||
#undef FIPS202_X4_DEFAULT_IMPLEMENTATION
|
||||
#undef MLK_FIPS202_FIPS202_H
|
||||
#undef SHA3_256_HASHBYTES
|
||||
#undef SHA3_256_RATE
|
||||
#undef SHA3_384_RATE
|
||||
#undef SHA3_512_HASHBYTES
|
||||
#undef SHA3_512_RATE
|
||||
#undef SHAKE128_RATE
|
||||
#undef SHAKE256_RATE
|
||||
#undef mlk_sha3_256
|
||||
#undef mlk_sha3_512
|
||||
#undef mlk_shake128_absorb_once
|
||||
#undef mlk_shake128_init
|
||||
#undef mlk_shake128_release
|
||||
#undef mlk_shake128_squeezeblocks
|
||||
#undef mlk_shake256
|
||||
/* mlkem/src/fips202/fips202x4.h */
|
||||
#undef MLK_FIPS202_FIPS202X4_H
|
||||
#undef mlk_shake128x4_absorb_once
|
||||
#undef mlk_shake128x4_init
|
||||
#undef mlk_shake128x4_release
|
||||
#undef mlk_shake128x4_squeezeblocks
|
||||
#undef mlk_shake256x4
|
||||
/* mlkem/src/fips202/keccakf1600.h */
|
||||
#undef MLK_FIPS202_KECCAKF1600_H
|
||||
#undef MLK_KECCAK_LANES
|
||||
#undef MLK_KECCAK_WAY
|
||||
#undef mlk_keccakf1600_extract_bytes
|
||||
#undef mlk_keccakf1600_permute
|
||||
#undef mlk_keccakf1600_xor_bytes
|
||||
#undef mlk_keccakf1600x4_extract_bytes
|
||||
#undef mlk_keccakf1600x4_permute
|
||||
#undef mlk_keccakf1600x4_xor_bytes
|
||||
#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202)
|
||||
/* mlkem/src/fips202/native/api.h */
|
||||
#undef MLK_FIPS202_NATIVE_API_H
|
||||
#undef MLK_NATIVE_FUNC_FALLBACK
|
||||
#undef MLK_NATIVE_FUNC_SUCCESS
|
||||
/* mlkem/src/fips202/native/auto.h */
|
||||
#undef MLK_FIPS202_NATIVE_AUTO_H
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
/*
|
||||
* Undefine macros from native code (FIPS202, AArch64)
|
||||
*/
|
||||
/* mlkem/src/fips202/native/aarch64/auto.h */
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H
|
||||
/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H
|
||||
#undef mlk_keccak_f1600_x1_scalar_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x1_v84a_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x2_v84a_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm
|
||||
#undef mlk_keccakf1600_round_constants
|
||||
/* mlkem/src/fips202/native/aarch64/x1_scalar.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H
|
||||
#undef MLK_USE_FIPS202_X1_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x1_v84a.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X1_V84A
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H
|
||||
#undef MLK_USE_FIPS202_X1_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x2_v84a.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X2_V84A
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
#endif /* MLK_SYS_AARCH64 */
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
/*
|
||||
* Undefine macros from native code (FIPS202, x86_64)
|
||||
*/
|
||||
/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */
|
||||
#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H
|
||||
#undef MLK_FIPS202_X86_64_NEED_X4_AVX2
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */
|
||||
#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H
|
||||
#undef mlk_keccak_f1600_x4_avx2_asm
|
||||
#undef mlk_keccak_rho56
|
||||
#undef mlk_keccak_rho8
|
||||
#undef mlk_keccakf1600_round_constants
|
||||
#endif /* MLK_SYS_X86_64 */
|
||||
#if defined(MLK_SYS_ARMV81M_MVE)
|
||||
/*
|
||||
* Undefine macros from native code (FIPS202, Armv8.1-M)
|
||||
*/
|
||||
/* mlkem/src/fips202/native/armv81m/mve.h */
|
||||
#undef MLK_FIPS202_ARMV81M_NEED_X4
|
||||
#undef MLK_FIPS202_NATIVE_ARMV81M
|
||||
#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H
|
||||
#undef MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
#undef MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE
|
||||
#undef mlk_keccak_f1600_x4_native_impl
|
||||
/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */
|
||||
#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H
|
||||
#undef mlk_keccak_f1600_x4_mve_asm
|
||||
#undef mlk_keccak_f1600_x4_state_extract_bytes_asm
|
||||
#undef mlk_keccak_f1600_x4_state_xor_bytes_asm
|
||||
#undef mlk_keccakf1600_round_constants
|
||||
#endif /* MLK_SYS_ARMV81M_MVE */
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH)
|
||||
/* mlkem/src/native/api.h */
|
||||
#undef MLK_INVNTT_BOUND
|
||||
#undef MLK_NATIVE_API_H
|
||||
#undef MLK_NATIVE_FUNC_FALLBACK
|
||||
#undef MLK_NATIVE_FUNC_SUCCESS
|
||||
#undef MLK_NTT_BOUND
|
||||
/* mlkem/src/native/meta.h */
|
||||
#undef MLK_NATIVE_META_H
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, AArch64)
|
||||
*/
|
||||
/* mlkem/src/native/aarch64/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_AARCH64
|
||||
#undef MLK_NATIVE_AARCH64_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOBYTES
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
#undef MLK_USE_NATIVE_REJ_UNIFORM
|
||||
/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */
|
||||
#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H
|
||||
#undef mlk_aarch64_invntt_zetas_layer12345
|
||||
#undef mlk_aarch64_invntt_zetas_layer67
|
||||
#undef mlk_aarch64_ntt_zetas_layer12345
|
||||
#undef mlk_aarch64_ntt_zetas_layer67
|
||||
#undef mlk_aarch64_zetas_mulcache_native
|
||||
#undef mlk_aarch64_zetas_mulcache_twisted_native
|
||||
#undef mlk_intt_aarch64_asm
|
||||
#undef mlk_ntt_aarch64_asm
|
||||
#undef mlk_poly_mulcache_compute_aarch64_asm
|
||||
#undef mlk_poly_reduce_aarch64_asm
|
||||
#undef mlk_poly_tobytes_aarch64_asm
|
||||
#undef mlk_poly_tomont_aarch64_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm
|
||||
#undef mlk_rej_uniform_aarch64_asm
|
||||
#undef mlk_rej_uniform_table
|
||||
#endif /* MLK_SYS_AARCH64 */
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, X86_64)
|
||||
*/
|
||||
/* mlkem/src/native/x86_64/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_X86_64_DEFAULT
|
||||
#undef MLK_NATIVE_X86_64_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER
|
||||
#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D10
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D11
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D4
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D5
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5
|
||||
#undef MLK_USE_NATIVE_POLY_FROMBYTES
|
||||
#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOBYTES
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
#undef MLK_USE_NATIVE_REJ_UNIFORM
|
||||
/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */
|
||||
#undef MLK_AVX2_REJ_UNIFORM_BUFLEN
|
||||
#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H
|
||||
#undef mlk_invntt_avx2_asm
|
||||
#undef mlk_ntt_avx2_asm
|
||||
#undef mlk_nttfrombytes_avx2_asm
|
||||
#undef mlk_ntttobytes_avx2_asm
|
||||
#undef mlk_nttunpack_avx2_asm
|
||||
#undef mlk_poly_compress_d10_avx2_asm
|
||||
#undef mlk_poly_compress_d11_avx2_asm
|
||||
#undef mlk_poly_compress_d4_avx2_asm
|
||||
#undef mlk_poly_compress_d5_avx2_asm
|
||||
#undef mlk_poly_decompress_d10_avx2_asm
|
||||
#undef mlk_poly_decompress_d11_avx2_asm
|
||||
#undef mlk_poly_decompress_d4_avx2_asm
|
||||
#undef mlk_poly_decompress_d5_avx2_asm
|
||||
#undef mlk_poly_mulcache_compute_avx2_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm
|
||||
#undef mlk_reduce_avx2_asm
|
||||
#undef mlk_rej_uniform_avx2_asm
|
||||
#undef mlk_rej_uniform_table
|
||||
#undef mlk_tomont_avx2_asm
|
||||
/* mlkem/src/native/x86_64/src/compress_consts.h */
|
||||
#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H
|
||||
#undef mlk_compress_d10_data
|
||||
#undef mlk_compress_d11_data
|
||||
#undef mlk_compress_d4_data
|
||||
#undef mlk_compress_d5_data
|
||||
#undef mlk_decompress_d10_data
|
||||
#undef mlk_decompress_d11_data
|
||||
#undef mlk_decompress_d4_data
|
||||
#undef mlk_decompress_d5_data
|
||||
/* mlkem/src/native/x86_64/src/consts.h */
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP
|
||||
#undef MLK_NATIVE_X86_64_SRC_CONSTS_H
|
||||
#undef mlk_qdata
|
||||
#endif /* MLK_SYS_X86_64 */
|
||||
#if defined(MLK_SYS_RISCV64)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, RISC-V 64)
|
||||
*/
|
||||
/* mlkem/src/native/riscv64/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_RISCV64
|
||||
#undef MLK_NATIVE_RISCV64_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
#undef MLK_USE_NATIVE_REJ_UNIFORM
|
||||
/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */
|
||||
#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H
|
||||
#undef mlk_rv64v_poly_add
|
||||
#undef mlk_rv64v_poly_basemul_mont_add_k2
|
||||
#undef mlk_rv64v_poly_basemul_mont_add_k3
|
||||
#undef mlk_rv64v_poly_basemul_mont_add_k4
|
||||
#undef mlk_rv64v_poly_invntt_tomont
|
||||
#undef mlk_rv64v_poly_ntt
|
||||
#undef mlk_rv64v_poly_reduce
|
||||
#undef mlk_rv64v_poly_sub
|
||||
#undef mlk_rv64v_poly_tomont
|
||||
#undef mlk_rv64v_rej_uniform
|
||||
/* mlkem/src/native/riscv64/src/rv64v_debug.h */
|
||||
#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H
|
||||
#undef mlk_assert_abs_bound_int16m1
|
||||
#undef mlk_assert_abs_bound_int16m2
|
||||
#undef mlk_assert_bound_int16m1
|
||||
#undef mlk_assert_bound_int16m2
|
||||
#undef mlk_debug_check_bounds_int16m1
|
||||
#undef mlk_debug_check_bounds_int16m2
|
||||
#endif /* MLK_SYS_RISCV64 */
|
||||
#if defined(MLK_SYS_PPC64LE)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, PPC64LE)
|
||||
*/
|
||||
/* mlkem/src/native/ppc64le/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_NAME
|
||||
#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT
|
||||
#undef MLK_NATIVE_PPC64LE_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */
|
||||
#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H
|
||||
#undef mlk_intt_ppc_asm
|
||||
#undef mlk_ntt_ppc_asm
|
||||
#undef mlk_poly_tomont_ppc_asm
|
||||
#undef mlk_reduce_ppc_asm
|
||||
/* mlkem/src/native/ppc64le/src/consts.h */
|
||||
#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H
|
||||
#undef MLK_PPC_C20159_OFFSET
|
||||
#undef MLK_PPC_NQ_OFFSET
|
||||
#undef MLK_PPC_N_INV_OFFSET
|
||||
#undef MLK_PPC_N_INV_TW_OFFSET
|
||||
#undef MLK_PPC_Q_OFFSET
|
||||
#undef MLK_PPC_TOMONT_OFFSET
|
||||
#undef MLK_PPC_TOMONT_TW_OFFSET
|
||||
#undef MLK_PPC_ZETA_INTT_OFFSET
|
||||
#undef MLK_PPC_ZETA_INTT_TW_OFFSET
|
||||
#undef MLK_PPC_ZETA_NTT_OFFSET
|
||||
#undef MLK_PPC_ZETA_NTT_TW_OFFSET
|
||||
#undef mlk_ppc_qdata
|
||||
#endif /* MLK_SYS_PPC64LE */
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */
|
||||
#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */
|
||||
527
sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.h
Normal file
527
sdk/software/examples/mlkem_test/mlkem_native/mlkem_native.h
Normal file
@@ -0,0 +1,527 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*/
|
||||
|
||||
#ifndef MLK_H
|
||||
#define MLK_H
|
||||
|
||||
/*
|
||||
* Public API for mlkem-native.
|
||||
*
|
||||
* This header defines the public API of a single build of mlkem-native.
|
||||
*
|
||||
* Make sure the configuration file is in the include path
|
||||
* (this is "mlkem_native_config.h" by default, or MLK_CONFIG_FILE if defined).
|
||||
*
|
||||
* # Multi-level builds
|
||||
*
|
||||
* This header specifies a build of mlkem-native for a fixed security level.
|
||||
* If you need multiple security levels, leave the security level unspecified
|
||||
* in the configuration file and include this header multiple times, setting
|
||||
* MLK_CONFIG_PARAMETER_SET accordingly for each, and #undef'ing the MLK_H
|
||||
* guard to allow multiple inclusions.
|
||||
*
|
||||
* # Legacy configuration (deprecated)
|
||||
*
|
||||
* Instead of providing the config file used for the build, you can
|
||||
* alternatively set the following configuration options prior to
|
||||
* including this header.
|
||||
*
|
||||
* This method of configuration is deprecated.
|
||||
* It will be removed in mlkem-native-v2.
|
||||
*
|
||||
* - MLK_CONFIG_API_PARAMETER_SET [required]
|
||||
*
|
||||
* The parameter set used for the build; 512, 768, or 1024.
|
||||
*
|
||||
* - MLK_CONFIG_API_NAMESPACE_PREFIX [required]
|
||||
*
|
||||
* The namespace prefix used for the build.
|
||||
*
|
||||
* NOTE:
|
||||
* For a multi-level build, you must include the 512/768/1024 suffixes
|
||||
* in MLK_CONFIG_API_NAMESPACE_PREFIX.
|
||||
*
|
||||
* - MLK_CONFIG_API_NO_SUPERCOP [optional]
|
||||
*
|
||||
* By default, this header will also expose the mlkem-native API in the
|
||||
* SUPERCOP naming convention crypto_kem_xxx. If you don't want/need this,
|
||||
* set MLK_CONFIG_API_NO_SUPERCOP. You must set this for a multi-level build.
|
||||
*
|
||||
* - MLK_CONFIG_API_CONSTANTS_ONLY [optional]
|
||||
*
|
||||
* If you don't want this header to expose any function declarations,
|
||||
* but only constants for the sizes of key material, set
|
||||
* MLK_CONFIG_API_CONSTANTS_ONLY. In this case, you don't need to set
|
||||
* MLK_CONFIG_API_PARAMETER_SET or MLK_CONFIG_API_NAMESPACE_PREFIX,
|
||||
* nor include a configuration.
|
||||
*
|
||||
* - MLK_CONFIG_API_QUALIFIER [optional]
|
||||
*
|
||||
* Qualifier to apply to external API.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
/******************************* Key sizes ************************************/
|
||||
|
||||
/* Sizes of cryptographic material, per parameter set */
|
||||
/* See mlkem/common.h for the arithmetic expressions giving rise to these */
|
||||
/* check-magic: off */
|
||||
#define MLKEM512_SECRETKEYBYTES 1632
|
||||
#define MLKEM512_PUBLICKEYBYTES 800
|
||||
#define MLKEM512_CIPHERTEXTBYTES 768
|
||||
|
||||
#define MLKEM768_SECRETKEYBYTES 2400
|
||||
#define MLKEM768_PUBLICKEYBYTES 1184
|
||||
#define MLKEM768_CIPHERTEXTBYTES 1088
|
||||
|
||||
#define MLKEM1024_SECRETKEYBYTES 3168
|
||||
#define MLKEM1024_PUBLICKEYBYTES 1568
|
||||
#define MLKEM1024_CIPHERTEXTBYTES 1568
|
||||
/* check-magic: on */
|
||||
|
||||
/* Size of randomness coins in bytes (level-independent) */
|
||||
#define MLKEM_SYMBYTES 32
|
||||
#define MLKEM512_SYMBYTES MLKEM_SYMBYTES
|
||||
#define MLKEM768_SYMBYTES MLKEM_SYMBYTES
|
||||
#define MLKEM1024_SYMBYTES MLKEM_SYMBYTES
|
||||
/* Size of shared secret in bytes (level-independent) */
|
||||
#define MLKEM_BYTES 32
|
||||
#define MLKEM512_BYTES MLKEM_BYTES
|
||||
#define MLKEM768_BYTES MLKEM_BYTES
|
||||
#define MLKEM1024_BYTES MLKEM_BYTES
|
||||
|
||||
/* Sizes of cryptographic material, as a function of LVL=512,768,1024 */
|
||||
#define MLKEM_SECRETKEYBYTES_(LVL) MLKEM##LVL##_SECRETKEYBYTES
|
||||
#define MLKEM_PUBLICKEYBYTES_(LVL) MLKEM##LVL##_PUBLICKEYBYTES
|
||||
#define MLKEM_CIPHERTEXTBYTES_(LVL) MLKEM##LVL##_CIPHERTEXTBYTES
|
||||
#define MLKEM_SECRETKEYBYTES(LVL) MLKEM_SECRETKEYBYTES_(LVL)
|
||||
#define MLKEM_PUBLICKEYBYTES(LVL) MLKEM_PUBLICKEYBYTES_(LVL)
|
||||
#define MLKEM_CIPHERTEXTBYTES(LVL) MLKEM_CIPHERTEXTBYTES_(LVL)
|
||||
|
||||
/****************************** Error codes ***********************************/
|
||||
|
||||
/* Generic failure condition */
|
||||
#define MLK_ERR_FAIL (-1)
|
||||
/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE
|
||||
* is defined and the provided MLK_CUSTOM_ALLOC can fail. */
|
||||
#define MLK_ERR_OUT_OF_MEMORY (-2)
|
||||
/* An rng failure occured. Might be due to insufficient entropy or
|
||||
* system misconfiguration. */
|
||||
#define MLK_ERR_RNG_FAIL (-3)
|
||||
|
||||
/****************************** Function API **********************************/
|
||||
|
||||
#define MLK_API_CONCAT_(x, y) x##y
|
||||
#define MLK_API_CONCAT(x, y) MLK_API_CONCAT_(x, y)
|
||||
#define MLK_API_CONCAT_UNDERSCORE(x, y) MLK_API_CONCAT(MLK_API_CONCAT(x, _), y)
|
||||
|
||||
#if !defined(MLK_CONFIG_API_PARAMETER_SET)
|
||||
/* Recommended configuration via same config file as used for the build. */
|
||||
|
||||
/* For now, we derive the legacy API configuration MLK_CONFIG_API_XXX from
|
||||
* the config file. In mlkem-native-v2, this will be removed and we will
|
||||
* exclusively work with MLK_CONFIG_XXX. */
|
||||
|
||||
/* You need to make sure the config file is in the include path. */
|
||||
#if defined(MLK_CONFIG_FILE)
|
||||
#include MLK_CONFIG_FILE
|
||||
#else
|
||||
#include "mlkem_native_config.h"
|
||||
#endif
|
||||
|
||||
#define MLK_CONFIG_API_PARAMETER_SET MLK_CONFIG_PARAMETER_SET
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_BUILD)
|
||||
#define MLK_CONFIG_API_NAMESPACE_PREFIX \
|
||||
MLK_API_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, MLK_CONFIG_PARAMETER_SET)
|
||||
#else
|
||||
#define MLK_CONFIG_API_NAMESPACE_PREFIX MLK_CONFIG_NAMESPACE_PREFIX
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_NO_SUPERCOP)
|
||||
#define MLK_CONFIG_API_NO_SUPERCOP
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_CONSTANTS_ONLY)
|
||||
#define MLK_CONFIG_API_CONSTANTS_ONLY
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER)
|
||||
#define MLK_CONFIG_API_QUALIFIER MLK_CONFIG_EXTERNAL_API_QUALIFIER
|
||||
#endif
|
||||
|
||||
#else /* !MLK_CONFIG_API_PARAMETER_SET */
|
||||
|
||||
#define MLK_API_LEGACY_CONFIG
|
||||
|
||||
#endif /* MLK_CONFIG_API_PARAMETER_SET */
|
||||
|
||||
#define MLK_API_NAMESPACE(sym) \
|
||||
MLK_API_CONCAT_UNDERSCORE(MLK_CONFIG_API_NAMESPACE_PREFIX, sym)
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define MLK_API_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result))
|
||||
#else
|
||||
#define MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_API_QUALIFIER)
|
||||
#define MLK_API_QUALIFIER MLK_CONFIG_API_QUALIFIER
|
||||
#else
|
||||
#define MLK_API_QUALIFIER
|
||||
#endif
|
||||
|
||||
#if !defined(MLK_CONFIG_API_CONSTANTS_ONLY)
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Generate a public/private keypair for the ML-KEM key encapsulation mechanism.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].}
|
||||
*
|
||||
* @param[out] pk Output public key, an array of
|
||||
* MLKEM{512,768,1024}_PUBLICKEYBYTES bytes.
|
||||
* @param[out] sk Output private key, an array of
|
||||
* MLKEM{512,768,1024}_SECRETKEYBYTES bytes.
|
||||
* @param[in] coins Input randomness, an array of 2*MLKEM_SYMBYTES uniformly
|
||||
* random bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(keypair_derand)(
|
||||
uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
const uint8_t coins[2 * MLKEM_SYMBYTES]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
|
||||
|
||||
#if !defined(MLK_CONFIG_NO_RANDOMIZED_API)
|
||||
/**
|
||||
* Generate a public/private keypair for the ML-KEM key encapsulation mechanism.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].}
|
||||
*
|
||||
* @param[out] pk Output public key, an array of
|
||||
* MLKEM{512,768,1024}_PUBLICKEYBYTES bytes.
|
||||
* @param[out] sk Output private key, an array of
|
||||
* MLKEM{512,768,1024}_SECRETKEYBYTES bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
* @retval MLK_ERR_RNG_FAIL Random number generation failed.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(keypair)(
|
||||
uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */
|
||||
|
||||
/**
|
||||
* Generate ciphertext and shared secret for a given public key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].}
|
||||
*
|
||||
* @param[out] ct Output ciphertext, an array of
|
||||
* MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes.
|
||||
* @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes.
|
||||
* @param[in] pk Input public key, an array of
|
||||
* MLKEM{512,768,1024}_PUBLICKEYBYTES bytes.
|
||||
* @param[in] coins Input randomness, an array of MLKEM_SYMBYTES bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2]
|
||||
* for the public key failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(enc_derand)(
|
||||
uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
uint8_t ss[MLKEM_BYTES],
|
||||
const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
const uint8_t coins[MLKEM_SYMBYTES]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
|
||||
#if !defined(MLK_CONFIG_NO_RANDOMIZED_API)
|
||||
/**
|
||||
* Generate ciphertext and shared secret for a given public key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].}
|
||||
*
|
||||
* @param[out] ct Output ciphertext, an array of
|
||||
* MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes.
|
||||
* @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes.
|
||||
* @param[in] pk Input public key, an array of
|
||||
* MLKEM{512,768,1024}_PUBLICKEYBYTES bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2]
|
||||
* for the public key failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
* @retval MLK_ERR_RNG_FAIL Random number generation failed.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(enc)(
|
||||
uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
uint8_t ss[MLKEM_BYTES],
|
||||
const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */
|
||||
|
||||
/**
|
||||
* Generate shared secret for a given ciphertext and private key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].}
|
||||
*
|
||||
* @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes.
|
||||
* @param[in] ct Input ciphertext, an array of
|
||||
* MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes.
|
||||
* @param[in] sk Input private key, an array of
|
||||
* MLKEM{512,768,1024}_SECRETKEYBYTES bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL The 'hash check' @[FIPS203, Section 7.3]
|
||||
* for the secret key failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(dec)(
|
||||
uint8_t ss[MLKEM_BYTES],
|
||||
const uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)],
|
||||
const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
|
||||
|
||||
/**
|
||||
* Implements modulus check mandated by FIPS 203, i.e., ensures that
|
||||
* coefficients are in [0,q-1].
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].}
|
||||
*
|
||||
* @param[in] pk Input public key, an array of
|
||||
* MLKEM{512,768,1024}_PUBLICKEYBYTES bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL Modulus check failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(check_pk)(
|
||||
const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
|
||||
/**
|
||||
* Implements public key hash check mandated by FIPS 203, i.e., ensures that
|
||||
* sk[768𝑘+32 ∶ 768𝑘+64] = H(pk) = H(sk[384𝑘 : 768𝑘+32]).
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Section 7.3, 'hash check'].}
|
||||
*
|
||||
* @param[in] sk Input private key, an array of
|
||||
* MLKEM{512,768,1024}_SECRETKEYBYTES bytes.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL Public key hash check failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_API_QUALIFIER
|
||||
MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
int MLK_API_NAMESPACE(check_sk)(
|
||||
const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)]
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
,
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context
|
||||
#endif
|
||||
);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
/****************************** SUPERCOP API *********************************/
|
||||
|
||||
#if !defined(MLK_CONFIG_API_NO_SUPERCOP)
|
||||
/* Export API in SUPERCOP naming scheme CRYPTO_xxx / crypto_kem_xxx */
|
||||
#define CRYPTO_SECRETKEYBYTES MLKEM_SECRETKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)
|
||||
#define CRYPTO_PUBLICKEYBYTES MLKEM_PUBLICKEYBYTES(MLK_CONFIG_API_PARAMETER_SET)
|
||||
#define CRYPTO_CIPHERTEXTBYTES \
|
||||
MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_API_PARAMETER_SET)
|
||||
#define CRYPTO_SYMBYTES MLKEM_SYMBYTES
|
||||
#define CRYPTO_BYTES MLKEM_BYTES
|
||||
|
||||
#define crypto_kem_keypair_derand MLK_API_NAMESPACE(keypair_derand)
|
||||
#define crypto_kem_keypair MLK_API_NAMESPACE(keypair)
|
||||
#define crypto_kem_enc_derand MLK_API_NAMESPACE(enc_derand)
|
||||
#define crypto_kem_enc MLK_API_NAMESPACE(enc)
|
||||
#define crypto_kem_dec MLK_API_NAMESPACE(dec)
|
||||
#define crypto_kem_check_pk MLK_API_NAMESPACE(check_pk)
|
||||
#define crypto_kem_check_sk MLK_API_NAMESPACE(check_sk)
|
||||
|
||||
#else /* !MLK_CONFIG_API_NO_SUPERCOP */
|
||||
|
||||
/* If the SUPERCOP API is not needed, we can undefine the various helper macros
|
||||
* above. Otherwise, they are needed for lazy evaluation of crypto_kem_xxx. */
|
||||
#if !defined(MLK_API_LEGACY_CONFIG)
|
||||
#undef MLK_CONFIG_API_PARAMETER_SET
|
||||
#undef MLK_CONFIG_API_NAMESPACE_PREFIX
|
||||
#undef MLK_CONFIG_API_NO_SUPERCOP
|
||||
#undef MLK_CONFIG_API_CONSTANTS_ONLY
|
||||
#undef MLK_CONFIG_API_QUALIFIER
|
||||
#endif /* !MLK_API_LEGACY_CONFIG */
|
||||
|
||||
#undef MLK_API_CONCAT
|
||||
#undef MLK_API_CONCAT_
|
||||
#undef MLK_API_CONCAT_UNDERSCORE
|
||||
#undef MLK_API_NAMESPACE
|
||||
#undef MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
#undef MLK_API_QUALIFIER
|
||||
|
||||
#endif /* MLK_CONFIG_API_NO_SUPERCOP */
|
||||
#endif /* !MLK_CONFIG_API_CONSTANTS_ONLY */
|
||||
|
||||
|
||||
/***************************** Memory Usage **********************************/
|
||||
|
||||
/*
|
||||
* By default mlkem-native performs all memory allocations on the stack.
|
||||
* Alternatively, mlkem-native supports custom allocation of large structures
|
||||
* through the `MLK_CONFIG_CUSTOM_ALLOC_FREE` configuration option.
|
||||
* See mlkem_native_config.h for details.
|
||||
*
|
||||
* `MLK_TOTAL_ALLOC_{512,768,1024}_{KEYPAIR,ENCAPS,DECAPS}` indicates the
|
||||
* maximum (accumulative) allocation via MLK_ALLOC for each parameter set and
|
||||
* operation. Note that some stack allocation remains even when using custom
|
||||
* allocators, so these values are lower than total stack usage with the default
|
||||
* stack-only allocation.
|
||||
*
|
||||
* These constants may be used to implement custom allocations using a
|
||||
* fixed-sized buffer and a simple allocator (e.g., bump allocator).
|
||||
*/
|
||||
/* check-magic: off */
|
||||
#define MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT 5824
|
||||
#define MLK_TOTAL_ALLOC_512_KEYPAIR_PCT 10048
|
||||
#define MLK_TOTAL_ALLOC_512_ENCAPS 8384
|
||||
#define MLK_TOTAL_ALLOC_512_DECAPS 9152
|
||||
#define MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT 10176
|
||||
#define MLK_TOTAL_ALLOC_768_KEYPAIR_PCT 15552
|
||||
#define MLK_TOTAL_ALLOC_768_ENCAPS 13248
|
||||
#define MLK_TOTAL_ALLOC_768_DECAPS 14336
|
||||
#define MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT 15552
|
||||
#define MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT 22400
|
||||
#define MLK_TOTAL_ALLOC_1024_ENCAPS 19136
|
||||
#define MLK_TOTAL_ALLOC_1024_DECAPS 20704
|
||||
/* check-magic: on */
|
||||
|
||||
/*
|
||||
* MLK_TOTAL_ALLOC_*_KEYPAIR adapts based on MLK_CONFIG_KEYGEN_PCT.
|
||||
* For legacy config, we don't know which options are used, so assume
|
||||
* the worst case (PCT enabled).
|
||||
*/
|
||||
#if defined(MLK_API_LEGACY_CONFIG) || defined(MLK_CONFIG_KEYGEN_PCT)
|
||||
#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_PCT
|
||||
#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_PCT
|
||||
#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT
|
||||
#else
|
||||
#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT
|
||||
#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT
|
||||
#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT
|
||||
#endif
|
||||
|
||||
#define MLK_MAX3_(a, b, c) \
|
||||
((a) > (b) ? ((a) > (c) ? (a) : (c)) : ((b) > (c) ? (b) : (c)))
|
||||
|
||||
/*
|
||||
* `MLK_TOTAL_ALLOC_{512,768,1024}` is the maximum across all operations for
|
||||
* each parameter set.
|
||||
*/
|
||||
#define MLK_TOTAL_ALLOC_512 \
|
||||
MLK_MAX3_(MLK_TOTAL_ALLOC_512_KEYPAIR, MLK_TOTAL_ALLOC_512_ENCAPS, \
|
||||
MLK_TOTAL_ALLOC_512_DECAPS)
|
||||
#define MLK_TOTAL_ALLOC_768 \
|
||||
MLK_MAX3_(MLK_TOTAL_ALLOC_768_KEYPAIR, MLK_TOTAL_ALLOC_768_ENCAPS, \
|
||||
MLK_TOTAL_ALLOC_768_DECAPS)
|
||||
#define MLK_TOTAL_ALLOC_1024 \
|
||||
MLK_MAX3_(MLK_TOTAL_ALLOC_1024_KEYPAIR, MLK_TOTAL_ALLOC_1024_ENCAPS, \
|
||||
MLK_TOTAL_ALLOC_1024_DECAPS)
|
||||
|
||||
#undef MLK_API_LEGACY_CONFIG
|
||||
|
||||
#endif /* !MLK_H */
|
||||
723
sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_asm.S
Normal file
723
sdk/software/examples/mlkem_test/mlkem_native/mlkem_native_asm.S
Normal file
@@ -0,0 +1,723 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* Single assembly unit for fixed-level build of mlkem-native
|
||||
*
|
||||
* This assembly unit bundles together all assembly files for a build
|
||||
* of mlkem-native for a fixed security level (MLKEM-512/768/1024).
|
||||
*
|
||||
* # Multi-level build
|
||||
*
|
||||
* If you want an SCU build of mlkem-native with support for multiple security
|
||||
* levels, you should include this file once with
|
||||
* MLK_CONFIG_MULTILEVEL_WITH_SHARED set.
|
||||
*
|
||||
* (You could also follow the same pattern as for mlkem_native_monobuild.c
|
||||
* and include it for every level, setting MLK_CONFIG_MULTILEVEL_NO_SHARED
|
||||
* for all but one. For builds with MLK_CONFIG_MULTILEVEL_NO_SHARED, this
|
||||
* file will then be ignored.)
|
||||
*
|
||||
* # Configuration
|
||||
*
|
||||
* The following options from the mlkem-native configuration are relevant:
|
||||
*
|
||||
* - MLK_CONFIG_FIPS202_CUSTOM_HEADER
|
||||
* Set this option if you use a custom FIPS202 implementation.
|
||||
*
|
||||
* - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH
|
||||
* Set this option if you want to include the native arithmetic backends
|
||||
* in your build.
|
||||
*
|
||||
* - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202
|
||||
* Set this option if you want to include the native FIPS202 backends
|
||||
* in your build.
|
||||
*
|
||||
* - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS
|
||||
* Set this option if you want to keep the directives defined in
|
||||
* level-independent headers. This is needed for a multi-level build.
|
||||
*/
|
||||
|
||||
/* If parts of the mlkem-native source tree are not used,
|
||||
* consider reducing this header via `unifdef`.
|
||||
*
|
||||
* Example:
|
||||
* ```bash
|
||||
* unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native_asm.S
|
||||
* ```
|
||||
*/
|
||||
|
||||
#include "src/common.h"
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH)
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
#include "src/native/aarch64/src/intt_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/ntt_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/poly_mulcache_compute_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/poly_reduce_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/poly_tobytes_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/poly_tomont_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S"
|
||||
#include "src/native/aarch64/src/rej_uniform_aarch64_asm.S"
|
||||
#endif /* MLK_SYS_AARCH64 */
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
#include "src/native/x86_64/src/intt_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/ntt_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/nttfrombytes_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/ntttobytes_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/nttunpack_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_compress_d10_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_compress_d11_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_compress_d4_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_compress_d5_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_decompress_d10_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_decompress_d11_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_decompress_d4_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_decompress_d5_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/poly_mulcache_compute_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/reduce_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/rej_uniform_avx2_asm.S"
|
||||
#include "src/native/x86_64/src/tomont_avx2_asm.S"
|
||||
#endif /* MLK_SYS_X86_64 */
|
||||
#if defined(MLK_SYS_RISCV64)
|
||||
#endif
|
||||
#if defined(MLK_SYS_PPC64LE)
|
||||
#include "src/native/ppc64le/src/intt_ppc_asm.S"
|
||||
#include "src/native/ppc64le/src/ntt_ppc_asm.S"
|
||||
#include "src/native/ppc64le/src/poly_tomont_ppc_asm.S"
|
||||
#include "src/native/ppc64le/src/reduce_ppc_asm.S"
|
||||
#endif /* MLK_SYS_PPC64LE */
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202)
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
#include "src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S"
|
||||
#include "src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S"
|
||||
#include "src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S"
|
||||
#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S"
|
||||
#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S"
|
||||
#endif /* MLK_SYS_AARCH64 */
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
#include "src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S"
|
||||
#endif
|
||||
#if defined(MLK_SYS_ARMV81M_MVE)
|
||||
#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S"
|
||||
#include "src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S"
|
||||
#include "src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S"
|
||||
#endif
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */
|
||||
|
||||
|
||||
/* Macro #undef's
|
||||
*
|
||||
* The following undefines macros from headers
|
||||
* included by the source files imported above.
|
||||
*
|
||||
* This is to allow building and linking multiple builds
|
||||
* of mlkem-native for varying parameter sets through concatenation
|
||||
* of this file, as if the files had been compiled separately.
|
||||
* If this is not relevant to you, you may remove the following.
|
||||
*
|
||||
* NOTE: This is not needed for the assembly SCU since, at present,
|
||||
* there is no need to include it multiple times.
|
||||
* We keep it for uniformity with mlkem_native.c only.
|
||||
*
|
||||
* NOTE: To avoid having to distinguish between which headers are included
|
||||
* from the assembly files, we #undef the same set of directives
|
||||
* as in mlkem_native.c
|
||||
*/
|
||||
|
||||
/*
|
||||
* Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files
|
||||
*/
|
||||
/* mlkem/mlkem_native.h */
|
||||
#undef CRYPTO_BYTES
|
||||
#undef CRYPTO_CIPHERTEXTBYTES
|
||||
#undef CRYPTO_PUBLICKEYBYTES
|
||||
#undef CRYPTO_SECRETKEYBYTES
|
||||
#undef CRYPTO_SYMBYTES
|
||||
#undef MLKEM1024_BYTES
|
||||
#undef MLKEM1024_CIPHERTEXTBYTES
|
||||
#undef MLKEM1024_PUBLICKEYBYTES
|
||||
#undef MLKEM1024_SECRETKEYBYTES
|
||||
#undef MLKEM1024_SYMBYTES
|
||||
#undef MLKEM512_BYTES
|
||||
#undef MLKEM512_CIPHERTEXTBYTES
|
||||
#undef MLKEM512_PUBLICKEYBYTES
|
||||
#undef MLKEM512_SECRETKEYBYTES
|
||||
#undef MLKEM512_SYMBYTES
|
||||
#undef MLKEM768_BYTES
|
||||
#undef MLKEM768_CIPHERTEXTBYTES
|
||||
#undef MLKEM768_PUBLICKEYBYTES
|
||||
#undef MLKEM768_SECRETKEYBYTES
|
||||
#undef MLKEM768_SYMBYTES
|
||||
#undef MLKEM_BYTES
|
||||
#undef MLKEM_CIPHERTEXTBYTES
|
||||
#undef MLKEM_CIPHERTEXTBYTES_
|
||||
#undef MLKEM_PUBLICKEYBYTES
|
||||
#undef MLKEM_PUBLICKEYBYTES_
|
||||
#undef MLKEM_SECRETKEYBYTES
|
||||
#undef MLKEM_SECRETKEYBYTES_
|
||||
#undef MLKEM_SYMBYTES
|
||||
#undef MLK_API_CONCAT
|
||||
#undef MLK_API_CONCAT_
|
||||
#undef MLK_API_CONCAT_UNDERSCORE
|
||||
#undef MLK_API_LEGACY_CONFIG
|
||||
#undef MLK_API_MUST_CHECK_RETURN_VALUE
|
||||
#undef MLK_API_NAMESPACE
|
||||
#undef MLK_API_QUALIFIER
|
||||
#undef MLK_CONFIG_API_CONSTANTS_ONLY
|
||||
#undef MLK_CONFIG_API_NAMESPACE_PREFIX
|
||||
#undef MLK_CONFIG_API_NO_SUPERCOP
|
||||
#undef MLK_CONFIG_API_PARAMETER_SET
|
||||
#undef MLK_CONFIG_API_QUALIFIER
|
||||
#undef MLK_ERR_FAIL
|
||||
#undef MLK_ERR_OUT_OF_MEMORY
|
||||
#undef MLK_ERR_RNG_FAIL
|
||||
#undef MLK_H
|
||||
#undef MLK_MAX3_
|
||||
#undef MLK_TOTAL_ALLOC_1024
|
||||
#undef MLK_TOTAL_ALLOC_1024_DECAPS
|
||||
#undef MLK_TOTAL_ALLOC_1024_ENCAPS
|
||||
#undef MLK_TOTAL_ALLOC_1024_KEYPAIR
|
||||
#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT
|
||||
#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT
|
||||
#undef MLK_TOTAL_ALLOC_512
|
||||
#undef MLK_TOTAL_ALLOC_512_DECAPS
|
||||
#undef MLK_TOTAL_ALLOC_512_ENCAPS
|
||||
#undef MLK_TOTAL_ALLOC_512_KEYPAIR
|
||||
#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT
|
||||
#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT
|
||||
#undef MLK_TOTAL_ALLOC_768
|
||||
#undef MLK_TOTAL_ALLOC_768_DECAPS
|
||||
#undef MLK_TOTAL_ALLOC_768_ENCAPS
|
||||
#undef MLK_TOTAL_ALLOC_768_KEYPAIR
|
||||
#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT
|
||||
#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT
|
||||
#undef crypto_kem_check_pk
|
||||
#undef crypto_kem_check_sk
|
||||
#undef crypto_kem_dec
|
||||
#undef crypto_kem_enc
|
||||
#undef crypto_kem_enc_derand
|
||||
#undef crypto_kem_keypair
|
||||
#undef crypto_kem_keypair_derand
|
||||
/* mlkem/src/common.h */
|
||||
#undef MLK_ADD_PARAM_SET
|
||||
#undef MLK_ALLOC
|
||||
#undef MLK_APPLY
|
||||
#undef MLK_ASM_FN_SIZE
|
||||
#undef MLK_ASM_FN_SYMBOL
|
||||
#undef MLK_ASM_NAMESPACE
|
||||
#undef MLK_BUILD_INTERNAL
|
||||
#undef MLK_COMMON_H
|
||||
#undef MLK_CONCAT
|
||||
#undef MLK_CONCAT_
|
||||
#undef MLK_CONTEXT_PARAMETERS_0
|
||||
#undef MLK_CONTEXT_PARAMETERS_1
|
||||
#undef MLK_CONTEXT_PARAMETERS_2
|
||||
#undef MLK_CONTEXT_PARAMETERS_3
|
||||
#undef MLK_CONTEXT_PARAMETERS_4
|
||||
#undef MLK_EMPTY_CU
|
||||
#undef MLK_ERR_FAIL
|
||||
#undef MLK_ERR_OUT_OF_MEMORY
|
||||
#undef MLK_ERR_RNG_FAIL
|
||||
#undef MLK_EXTERNAL_API
|
||||
#undef MLK_FIPS202X4_HEADER_FILE
|
||||
#undef MLK_FIPS202_HEADER_FILE
|
||||
#undef MLK_FREE
|
||||
#undef MLK_INTERNAL_API
|
||||
#undef MLK_INTERNAL_DATA_DECLARATION
|
||||
#undef MLK_INTERNAL_DATA_DEFINITION
|
||||
#undef MLK_NAMESPACE
|
||||
#undef MLK_NAMESPACE_K
|
||||
#undef MLK_NAMESPACE_PREFIX
|
||||
#undef MLK_NAMESPACE_PREFIX_K
|
||||
#undef mlk_memcpy
|
||||
#undef mlk_memset
|
||||
/* mlkem/src/indcpa.h */
|
||||
#undef MLK_INDCPA_H
|
||||
#undef mlk_gen_matrix
|
||||
#undef mlk_indcpa_dec
|
||||
#undef mlk_indcpa_enc
|
||||
#undef mlk_indcpa_keypair_derand
|
||||
/* mlkem/src/kem.h */
|
||||
#undef MLK_KEM_H
|
||||
#undef mlk_kem_check_pk
|
||||
#undef mlk_kem_check_sk
|
||||
#undef mlk_kem_dec
|
||||
#undef mlk_kem_enc
|
||||
#undef mlk_kem_enc_derand
|
||||
#undef mlk_kem_keypair
|
||||
#undef mlk_kem_keypair_derand
|
||||
/* mlkem/src/params.h */
|
||||
#undef MLKEM_DU
|
||||
#undef MLKEM_DV
|
||||
#undef MLKEM_ETA1
|
||||
#undef MLKEM_ETA2
|
||||
#undef MLKEM_INDCCA_CIPHERTEXTBYTES
|
||||
#undef MLKEM_INDCCA_PUBLICKEYBYTES
|
||||
#undef MLKEM_INDCCA_SECRETKEYBYTES
|
||||
#undef MLKEM_INDCPA_BYTES
|
||||
#undef MLKEM_INDCPA_MSGBYTES
|
||||
#undef MLKEM_INDCPA_PUBLICKEYBYTES
|
||||
#undef MLKEM_INDCPA_SECRETKEYBYTES
|
||||
#undef MLKEM_K
|
||||
#undef MLKEM_N
|
||||
#undef MLKEM_POLYBYTES
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D10
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D11
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D4
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_D5
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_DU
|
||||
#undef MLKEM_POLYCOMPRESSEDBYTES_DV
|
||||
#undef MLKEM_POLYVECBYTES
|
||||
#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU
|
||||
#undef MLKEM_Q
|
||||
#undef MLKEM_Q_HALF
|
||||
#undef MLKEM_SSBYTES
|
||||
#undef MLKEM_SYMBYTES
|
||||
#undef MLKEM_UINT12_LIMIT
|
||||
#undef MLK_PARAMS_H
|
||||
/* mlkem/src/poly_k.h */
|
||||
#undef MLK_POLY_K_H
|
||||
#undef mlk_poly_compress_du
|
||||
#undef mlk_poly_compress_dv
|
||||
#undef mlk_poly_decompress_du
|
||||
#undef mlk_poly_decompress_dv
|
||||
#undef mlk_poly_getnoise_eta1122_4x
|
||||
#undef mlk_poly_getnoise_eta1_4x
|
||||
#undef mlk_poly_getnoise_eta2
|
||||
#undef mlk_poly_getnoise_eta2_4x
|
||||
#undef mlk_polymat
|
||||
#undef mlk_polyvec
|
||||
#undef mlk_polyvec_add
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached
|
||||
#undef mlk_polyvec_compress_du
|
||||
#undef mlk_polyvec_decompress_du
|
||||
#undef mlk_polyvec_frombytes
|
||||
#undef mlk_polyvec_invntt_tomont
|
||||
#undef mlk_polyvec_mulcache
|
||||
#undef mlk_polyvec_mulcache_compute
|
||||
#undef mlk_polyvec_ntt
|
||||
#undef mlk_polyvec_reduce
|
||||
#undef mlk_polyvec_tobytes
|
||||
#undef mlk_polyvec_tomont
|
||||
|
||||
#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS)
|
||||
/*
|
||||
* Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files
|
||||
*/
|
||||
/* mlkem/src/compress.h */
|
||||
#undef MLK_COMPRESS_H
|
||||
#undef mlk_poly_compress_d10
|
||||
#undef mlk_poly_compress_d11
|
||||
#undef mlk_poly_compress_d4
|
||||
#undef mlk_poly_compress_d5
|
||||
#undef mlk_poly_decompress_d10
|
||||
#undef mlk_poly_decompress_d11
|
||||
#undef mlk_poly_decompress_d4
|
||||
#undef mlk_poly_decompress_d5
|
||||
#undef mlk_poly_frombytes
|
||||
#undef mlk_poly_frommsg
|
||||
#undef mlk_poly_tobytes
|
||||
#undef mlk_poly_tomsg
|
||||
/* mlkem/src/debug.h */
|
||||
#undef MLK_DEBUG_H
|
||||
#undef mlk_assert
|
||||
#undef mlk_assert_abs_bound
|
||||
#undef mlk_assert_abs_bound_2d
|
||||
#undef mlk_assert_bound
|
||||
#undef mlk_assert_bound_2d
|
||||
#undef mlk_debug_check_assert
|
||||
#undef mlk_debug_check_bounds
|
||||
/* mlkem/src/poly.h */
|
||||
#undef MLK_INVNTT_BOUND
|
||||
#undef MLK_NTT_BOUND
|
||||
#undef MLK_POLY_H
|
||||
#undef mlk_poly_add
|
||||
#undef mlk_poly_invntt_tomont
|
||||
#undef mlk_poly_mulcache_compute
|
||||
#undef mlk_poly_ntt
|
||||
#undef mlk_poly_reduce
|
||||
#undef mlk_poly_sub
|
||||
#undef mlk_poly_tomont
|
||||
/* mlkem/src/randombytes.h */
|
||||
#undef MLK_RANDOMBYTES_H
|
||||
/* mlkem/src/sampling.h */
|
||||
#undef MLK_SAMPLING_H
|
||||
#undef mlk_poly_cbd2
|
||||
#undef mlk_poly_cbd3
|
||||
#undef mlk_poly_rej_uniform
|
||||
#undef mlk_poly_rej_uniform_x4
|
||||
/* mlkem/src/symmetric.h */
|
||||
#undef MLK_SYMMETRIC_H
|
||||
#undef MLK_XOF_RATE
|
||||
#undef mlk_hash_g
|
||||
#undef mlk_hash_h
|
||||
#undef mlk_hash_j
|
||||
#undef mlk_prf_eta
|
||||
#undef mlk_prf_eta1
|
||||
#undef mlk_prf_eta1_x4
|
||||
#undef mlk_prf_eta2
|
||||
#undef mlk_xof_absorb
|
||||
#undef mlk_xof_ctx
|
||||
#undef mlk_xof_init
|
||||
#undef mlk_xof_release
|
||||
#undef mlk_xof_squeezeblocks
|
||||
#undef mlk_xof_x4_absorb
|
||||
#undef mlk_xof_x4_ctx
|
||||
#undef mlk_xof_x4_init
|
||||
#undef mlk_xof_x4_release
|
||||
#undef mlk_xof_x4_squeezeblocks
|
||||
/* mlkem/src/sys.h */
|
||||
#undef MLK_ALIGN
|
||||
#undef MLK_ALIGN_UP
|
||||
#undef MLK_ALWAYS_INLINE
|
||||
#undef MLK_CET_ENDBR
|
||||
#undef MLK_CT_TESTING_DECLASSIFY
|
||||
#undef MLK_CT_TESTING_SECRET
|
||||
#undef MLK_DEFAULT_ALIGN
|
||||
#undef MLK_HAVE_INLINE_ASM
|
||||
#undef MLK_INLINE
|
||||
#undef MLK_MUST_CHECK_RETURN_VALUE
|
||||
#undef MLK_NOINLINE
|
||||
#undef MLK_RESTRICT
|
||||
#undef MLK_STATIC_TESTABLE
|
||||
#undef MLK_SYSV_ABI
|
||||
#undef MLK_SYSV_ABI_SUPPORTED
|
||||
#undef MLK_SYS_AARCH64
|
||||
#undef MLK_SYS_AARCH64_EB
|
||||
#undef MLK_SYS_APPLE
|
||||
#undef MLK_SYS_ARMV81M_MVE
|
||||
#undef MLK_SYS_BIG_ENDIAN
|
||||
#undef MLK_SYS_H
|
||||
#undef MLK_SYS_LINUX
|
||||
#undef MLK_SYS_LITTLE_ENDIAN
|
||||
#undef MLK_SYS_PPC64LE
|
||||
#undef MLK_SYS_RISCV32
|
||||
#undef MLK_SYS_RISCV64
|
||||
#undef MLK_SYS_RISCV64_RVV
|
||||
#undef MLK_SYS_WINDOWS
|
||||
#undef MLK_SYS_X86_64
|
||||
#undef MLK_SYS_X86_64_AVX2
|
||||
/* mlkem/src/verify.h */
|
||||
#undef MLK_USE_ASM_VALUE_BARRIER
|
||||
#undef MLK_VERIFY_H
|
||||
#undef mlk_ct_opt_blocker_u64
|
||||
/* mlkem/src/cbmc.h */
|
||||
#undef MLK_CBMC_H
|
||||
#undef __contract__
|
||||
#undef __loop__
|
||||
|
||||
#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER)
|
||||
/*
|
||||
* Undefine macros from FIPS-202 files
|
||||
*/
|
||||
/* mlkem/src/fips202/fips202.h */
|
||||
#undef FIPS202_X4_DEFAULT_IMPLEMENTATION
|
||||
#undef MLK_FIPS202_FIPS202_H
|
||||
#undef SHA3_256_HASHBYTES
|
||||
#undef SHA3_256_RATE
|
||||
#undef SHA3_384_RATE
|
||||
#undef SHA3_512_HASHBYTES
|
||||
#undef SHA3_512_RATE
|
||||
#undef SHAKE128_RATE
|
||||
#undef SHAKE256_RATE
|
||||
#undef mlk_sha3_256
|
||||
#undef mlk_sha3_512
|
||||
#undef mlk_shake128_absorb_once
|
||||
#undef mlk_shake128_init
|
||||
#undef mlk_shake128_release
|
||||
#undef mlk_shake128_squeezeblocks
|
||||
#undef mlk_shake256
|
||||
/* mlkem/src/fips202/fips202x4.h */
|
||||
#undef MLK_FIPS202_FIPS202X4_H
|
||||
#undef mlk_shake128x4_absorb_once
|
||||
#undef mlk_shake128x4_init
|
||||
#undef mlk_shake128x4_release
|
||||
#undef mlk_shake128x4_squeezeblocks
|
||||
#undef mlk_shake256x4
|
||||
/* mlkem/src/fips202/keccakf1600.h */
|
||||
#undef MLK_FIPS202_KECCAKF1600_H
|
||||
#undef MLK_KECCAK_LANES
|
||||
#undef MLK_KECCAK_WAY
|
||||
#undef mlk_keccakf1600_extract_bytes
|
||||
#undef mlk_keccakf1600_permute
|
||||
#undef mlk_keccakf1600_xor_bytes
|
||||
#undef mlk_keccakf1600x4_extract_bytes
|
||||
#undef mlk_keccakf1600x4_permute
|
||||
#undef mlk_keccakf1600x4_xor_bytes
|
||||
#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202)
|
||||
/* mlkem/src/fips202/native/api.h */
|
||||
#undef MLK_FIPS202_NATIVE_API_H
|
||||
#undef MLK_NATIVE_FUNC_FALLBACK
|
||||
#undef MLK_NATIVE_FUNC_SUCCESS
|
||||
/* mlkem/src/fips202/native/auto.h */
|
||||
#undef MLK_FIPS202_NATIVE_AUTO_H
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
/*
|
||||
* Undefine macros from native code (FIPS202, AArch64)
|
||||
*/
|
||||
/* mlkem/src/fips202/native/aarch64/auto.h */
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H
|
||||
/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H
|
||||
#undef mlk_keccak_f1600_x1_scalar_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x1_v84a_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x2_v84a_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm
|
||||
#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm
|
||||
#undef mlk_keccakf1600_round_constants
|
||||
/* mlkem/src/fips202/native/aarch64/x1_scalar.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H
|
||||
#undef MLK_USE_FIPS202_X1_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x1_v84a.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X1_V84A
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H
|
||||
#undef MLK_USE_FIPS202_X1_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x2_v84a.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X2_V84A
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */
|
||||
#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID
|
||||
#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
#endif /* MLK_SYS_AARCH64 */
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
/*
|
||||
* Undefine macros from native code (FIPS202, x86_64)
|
||||
*/
|
||||
/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */
|
||||
#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H
|
||||
#undef MLK_FIPS202_X86_64_NEED_X4_AVX2
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */
|
||||
#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H
|
||||
#undef mlk_keccak_f1600_x4_avx2_asm
|
||||
#undef mlk_keccak_rho56
|
||||
#undef mlk_keccak_rho8
|
||||
#undef mlk_keccakf1600_round_constants
|
||||
#endif /* MLK_SYS_X86_64 */
|
||||
#if defined(MLK_SYS_ARMV81M_MVE)
|
||||
/*
|
||||
* Undefine macros from native code (FIPS202, Armv8.1-M)
|
||||
*/
|
||||
/* mlkem/src/fips202/native/armv81m/mve.h */
|
||||
#undef MLK_FIPS202_ARMV81M_NEED_X4
|
||||
#undef MLK_FIPS202_NATIVE_ARMV81M
|
||||
#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H
|
||||
#undef MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE
|
||||
#undef MLK_USE_FIPS202_X4_NATIVE
|
||||
#undef MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE
|
||||
#undef mlk_keccak_f1600_x4_native_impl
|
||||
/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */
|
||||
#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H
|
||||
#undef mlk_keccak_f1600_x4_mve_asm
|
||||
#undef mlk_keccak_f1600_x4_state_extract_bytes_asm
|
||||
#undef mlk_keccak_f1600_x4_state_xor_bytes_asm
|
||||
#undef mlk_keccakf1600_round_constants
|
||||
#endif /* MLK_SYS_ARMV81M_MVE */
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH)
|
||||
/* mlkem/src/native/api.h */
|
||||
#undef MLK_INVNTT_BOUND
|
||||
#undef MLK_NATIVE_API_H
|
||||
#undef MLK_NATIVE_FUNC_FALLBACK
|
||||
#undef MLK_NATIVE_FUNC_SUCCESS
|
||||
#undef MLK_NTT_BOUND
|
||||
/* mlkem/src/native/meta.h */
|
||||
#undef MLK_NATIVE_META_H
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, AArch64)
|
||||
*/
|
||||
/* mlkem/src/native/aarch64/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_AARCH64
|
||||
#undef MLK_NATIVE_AARCH64_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOBYTES
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
#undef MLK_USE_NATIVE_REJ_UNIFORM
|
||||
/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */
|
||||
#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H
|
||||
#undef mlk_aarch64_invntt_zetas_layer12345
|
||||
#undef mlk_aarch64_invntt_zetas_layer67
|
||||
#undef mlk_aarch64_ntt_zetas_layer12345
|
||||
#undef mlk_aarch64_ntt_zetas_layer67
|
||||
#undef mlk_aarch64_zetas_mulcache_native
|
||||
#undef mlk_aarch64_zetas_mulcache_twisted_native
|
||||
#undef mlk_intt_aarch64_asm
|
||||
#undef mlk_ntt_aarch64_asm
|
||||
#undef mlk_poly_mulcache_compute_aarch64_asm
|
||||
#undef mlk_poly_reduce_aarch64_asm
|
||||
#undef mlk_poly_tobytes_aarch64_asm
|
||||
#undef mlk_poly_tomont_aarch64_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm
|
||||
#undef mlk_rej_uniform_aarch64_asm
|
||||
#undef mlk_rej_uniform_table
|
||||
#endif /* MLK_SYS_AARCH64 */
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, X86_64)
|
||||
*/
|
||||
/* mlkem/src/native/x86_64/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_X86_64_DEFAULT
|
||||
#undef MLK_NATIVE_X86_64_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER
|
||||
#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D10
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D11
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D4
|
||||
#undef MLK_USE_NATIVE_POLY_COMPRESS_D5
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4
|
||||
#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5
|
||||
#undef MLK_USE_NATIVE_POLY_FROMBYTES
|
||||
#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOBYTES
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
#undef MLK_USE_NATIVE_REJ_UNIFORM
|
||||
/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */
|
||||
#undef MLK_AVX2_REJ_UNIFORM_BUFLEN
|
||||
#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H
|
||||
#undef mlk_invntt_avx2_asm
|
||||
#undef mlk_ntt_avx2_asm
|
||||
#undef mlk_nttfrombytes_avx2_asm
|
||||
#undef mlk_ntttobytes_avx2_asm
|
||||
#undef mlk_nttunpack_avx2_asm
|
||||
#undef mlk_poly_compress_d10_avx2_asm
|
||||
#undef mlk_poly_compress_d11_avx2_asm
|
||||
#undef mlk_poly_compress_d4_avx2_asm
|
||||
#undef mlk_poly_compress_d5_avx2_asm
|
||||
#undef mlk_poly_decompress_d10_avx2_asm
|
||||
#undef mlk_poly_decompress_d11_avx2_asm
|
||||
#undef mlk_poly_decompress_d4_avx2_asm
|
||||
#undef mlk_poly_decompress_d5_avx2_asm
|
||||
#undef mlk_poly_mulcache_compute_avx2_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm
|
||||
#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm
|
||||
#undef mlk_reduce_avx2_asm
|
||||
#undef mlk_rej_uniform_avx2_asm
|
||||
#undef mlk_rej_uniform_table
|
||||
#undef mlk_tomont_avx2_asm
|
||||
/* mlkem/src/native/x86_64/src/compress_consts.h */
|
||||
#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H
|
||||
#undef mlk_compress_d10_data
|
||||
#undef mlk_compress_d11_data
|
||||
#undef mlk_compress_d4_data
|
||||
#undef mlk_compress_d5_data
|
||||
#undef mlk_decompress_d10_data
|
||||
#undef mlk_decompress_d11_data
|
||||
#undef mlk_decompress_d4_data
|
||||
#undef mlk_decompress_d5_data
|
||||
/* mlkem/src/native/x86_64/src/consts.h */
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD
|
||||
#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP
|
||||
#undef MLK_NATIVE_X86_64_SRC_CONSTS_H
|
||||
#undef mlk_qdata
|
||||
#endif /* MLK_SYS_X86_64 */
|
||||
#if defined(MLK_SYS_RISCV64)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, RISC-V 64)
|
||||
*/
|
||||
/* mlkem/src/native/riscv64/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_RISCV64
|
||||
#undef MLK_NATIVE_RISCV64_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
#undef MLK_USE_NATIVE_REJ_UNIFORM
|
||||
/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */
|
||||
#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H
|
||||
#undef mlk_rv64v_poly_add
|
||||
#undef mlk_rv64v_poly_basemul_mont_add_k2
|
||||
#undef mlk_rv64v_poly_basemul_mont_add_k3
|
||||
#undef mlk_rv64v_poly_basemul_mont_add_k4
|
||||
#undef mlk_rv64v_poly_invntt_tomont
|
||||
#undef mlk_rv64v_poly_ntt
|
||||
#undef mlk_rv64v_poly_reduce
|
||||
#undef mlk_rv64v_poly_sub
|
||||
#undef mlk_rv64v_poly_tomont
|
||||
#undef mlk_rv64v_rej_uniform
|
||||
/* mlkem/src/native/riscv64/src/rv64v_debug.h */
|
||||
#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H
|
||||
#undef mlk_assert_abs_bound_int16m1
|
||||
#undef mlk_assert_abs_bound_int16m2
|
||||
#undef mlk_assert_bound_int16m1
|
||||
#undef mlk_assert_bound_int16m2
|
||||
#undef mlk_debug_check_bounds_int16m1
|
||||
#undef mlk_debug_check_bounds_int16m2
|
||||
#endif /* MLK_SYS_RISCV64 */
|
||||
#if defined(MLK_SYS_PPC64LE)
|
||||
/*
|
||||
* Undefine macros from native code (Arith, PPC64LE)
|
||||
*/
|
||||
/* mlkem/src/native/ppc64le/meta.h */
|
||||
#undef MLK_ARITH_BACKEND_NAME
|
||||
#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT
|
||||
#undef MLK_NATIVE_PPC64LE_META_H
|
||||
#undef MLK_USE_NATIVE_INTT
|
||||
#undef MLK_USE_NATIVE_NTT
|
||||
#undef MLK_USE_NATIVE_POLY_REDUCE
|
||||
#undef MLK_USE_NATIVE_POLY_TOMONT
|
||||
/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */
|
||||
#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H
|
||||
#undef mlk_intt_ppc_asm
|
||||
#undef mlk_ntt_ppc_asm
|
||||
#undef mlk_poly_tomont_ppc_asm
|
||||
#undef mlk_reduce_ppc_asm
|
||||
/* mlkem/src/native/ppc64le/src/consts.h */
|
||||
#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H
|
||||
#undef MLK_PPC_C20159_OFFSET
|
||||
#undef MLK_PPC_NQ_OFFSET
|
||||
#undef MLK_PPC_N_INV_OFFSET
|
||||
#undef MLK_PPC_N_INV_TW_OFFSET
|
||||
#undef MLK_PPC_Q_OFFSET
|
||||
#undef MLK_PPC_TOMONT_OFFSET
|
||||
#undef MLK_PPC_TOMONT_TW_OFFSET
|
||||
#undef MLK_PPC_ZETA_INTT_OFFSET
|
||||
#undef MLK_PPC_ZETA_INTT_TW_OFFSET
|
||||
#undef MLK_PPC_ZETA_NTT_OFFSET
|
||||
#undef MLK_PPC_ZETA_NTT_TW_OFFSET
|
||||
#undef mlk_ppc_qdata
|
||||
#endif /* MLK_SYS_PPC64LE */
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */
|
||||
#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */
|
||||
@@ -0,0 +1,645 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS140_3_IG]
|
||||
* Implementation Guidance for FIPS 140-3 and the Cryptographic Module
|
||||
* Validation Program
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*/
|
||||
|
||||
#ifndef MLK_CONFIG_H
|
||||
#define MLK_CONFIG_H
|
||||
|
||||
/**
|
||||
* Specifies the parameter set for ML-KEM:
|
||||
* - MLK_CONFIG_PARAMETER_SET=512 corresponds to ML-KEM-512
|
||||
* - MLK_CONFIG_PARAMETER_SET=768 corresponds to ML-KEM-768
|
||||
* - MLK_CONFIG_PARAMETER_SET=1024 corresponds to ML-KEM-1024
|
||||
*
|
||||
* If you want to support multiple parameter sets, build the library multiple
|
||||
* times and set MLK_CONFIG_MULTILEVEL_BUILD. See MLK_CONFIG_MULTILEVEL_BUILD
|
||||
* for how to do this while minimizing code duplication.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
#ifndef MLK_CONFIG_PARAMETER_SET
|
||||
#define MLK_CONFIG_PARAMETER_SET \
|
||||
768 /* Change this for different security strengths */
|
||||
#endif
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_FILE
|
||||
*
|
||||
* If defined, this is a header that will be included instead of the default
|
||||
* configuration file mlkem/mlkem_native_config.h.
|
||||
*
|
||||
* When you need to build mlkem-native in multiple configurations, using
|
||||
* varying MLK_CONFIG_FILE can be more convenient than configuring everything
|
||||
* through CFLAGS.
|
||||
*
|
||||
* To use, MLK_CONFIG_FILE _must_ be defined prior to the inclusion of any
|
||||
* mlkem-native headers. For example, it can be set by passing
|
||||
* `-DMLK_CONFIG_FILE="..."` on the command line.
|
||||
*/
|
||||
/* #define MLK_CONFIG_FILE "mlkem_native_config.h" */
|
||||
|
||||
/**
|
||||
* The prefix to use to namespace global symbols from mlkem/.
|
||||
*
|
||||
* In a multi-level build, level-dependent symbols will additionally be
|
||||
* prefixed with the parameter set (512/768/1024).
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
#if !defined(MLK_CONFIG_NAMESPACE_PREFIX)
|
||||
#define MLK_CONFIG_NAMESPACE_PREFIX MLK_DEFAULT_NAMESPACE_PREFIX
|
||||
#endif
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_MULTILEVEL_BUILD
|
||||
*
|
||||
* Set this if the build is part of a multi-level build supporting multiple
|
||||
* parameter sets.
|
||||
*
|
||||
* If you need only a single parameter set, keep this unset.
|
||||
*
|
||||
* To build mlkem-native with support for all parameter sets, build it three
|
||||
* times -- once per parameter set -- and set the option
|
||||
* MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and
|
||||
* MLK_CONFIG_MULTILEVEL_NO_SHARED for the others.
|
||||
* MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them.
|
||||
*
|
||||
* See examples/multilevel_build for an example.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
/* #define MLK_CONFIG_MULTILEVEL_BUILD */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_EXTERNAL_API_QUALIFIER
|
||||
*
|
||||
* If set, this option provides an additional function qualifier to be added
|
||||
* to declarations of mlkem-native's public API.
|
||||
*
|
||||
* The primary use case for this option are single-CU builds where the public
|
||||
* API exposed by mlkem-native is wrapped by another API in the consuming
|
||||
* application. In this case, even mlkem-native's public API can be marked
|
||||
* `static`.
|
||||
*/
|
||||
/* #define MLK_CONFIG_EXTERNAL_API_QUALIFIER */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_NO_RANDOMIZED_API
|
||||
*
|
||||
* If this option is set, mlkem-native will be built without the randomized
|
||||
* API functions (crypto_kem_keypair and crypto_kem_enc). This allows users
|
||||
* to build mlkem-native without providing a randombytes() implementation
|
||||
* if they only need the deterministic API (crypto_kem_keypair_derand,
|
||||
* crypto_kem_enc_derand, crypto_kem_dec).
|
||||
*
|
||||
* @note This option is incompatible with MLK_CONFIG_KEYGEN_PCT as the
|
||||
* current PCT implementation requires crypto_kem_enc().
|
||||
*/
|
||||
/* #define MLK_CONFIG_NO_RANDOMIZED_API */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_NO_SUPERCOP
|
||||
*
|
||||
* By default, mlkem_native.h exposes the mlkem-native API in the SUPERCOP
|
||||
* naming convention (crypto_kem_xxx). If you don't need this, set
|
||||
* MLK_CONFIG_NO_SUPERCOP.
|
||||
*
|
||||
* @note You must set this for a multi-level build as the SUPERCOP naming
|
||||
* does not disambiguate between the parameter sets.
|
||||
*/
|
||||
/* #define MLK_CONFIG_NO_SUPERCOP */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CONSTANTS_ONLY
|
||||
*
|
||||
* If you only need the size constants (MLKEM_PUBLICKEYBYTES, etc.) but no
|
||||
* function declarations, set MLK_CONFIG_CONSTANTS_ONLY.
|
||||
*
|
||||
* This only affects the public header mlkem_native.h, not the
|
||||
* implementation.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CONSTANTS_ONLY */
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* Build-only configuration options
|
||||
*
|
||||
* The remaining configurations are build-options only.
|
||||
* They do not affect the API described in mlkem_native.h.
|
||||
*
|
||||
*****************************************************************************/
|
||||
|
||||
#if defined(MLK_BUILD_INTERNAL)
|
||||
/**
|
||||
* MLK_CONFIG_MULTILEVEL_WITH_SHARED
|
||||
*
|
||||
* This is for multi-level builds of mlkem-native only. If you need only a
|
||||
* single parameter set, keep this unset.
|
||||
*
|
||||
* If this is set, all MLK_CONFIG_PARAMETER_SET-independent code will be
|
||||
* included in the build, including code needed only for other parameter
|
||||
* sets.
|
||||
*
|
||||
* Example: mlk_poly_cbd3 is only needed for MLK_CONFIG_PARAMETER_SET == 512.
|
||||
* Yet, if this option is set for a build with
|
||||
* MLK_CONFIG_PARAMETER_SET == 768/1024, it would be included.
|
||||
*
|
||||
* To build mlkem-native with support for all parameter sets, build it three
|
||||
* times -- once per parameter set -- and set the option
|
||||
* MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and
|
||||
* MLK_CONFIG_MULTILEVEL_NO_SHARED for the others.
|
||||
* MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them.
|
||||
*
|
||||
* See examples/multilevel_build for an example.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
/* #define MLK_CONFIG_MULTILEVEL_WITH_SHARED */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_MULTILEVEL_NO_SHARED
|
||||
*
|
||||
* This is for multi-level builds of mlkem-native only. If you need only a
|
||||
* single parameter set, keep this unset.
|
||||
*
|
||||
* If this is set, no MLK_CONFIG_PARAMETER_SET-independent code will be
|
||||
* included in the build.
|
||||
*
|
||||
* To build mlkem-native with support for all parameter sets, build it three
|
||||
* times -- once per parameter set -- and set the option
|
||||
* MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and
|
||||
* MLK_CONFIG_MULTILEVEL_NO_SHARED for the others.
|
||||
* MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them.
|
||||
*
|
||||
* See examples/multilevel_build for an example.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
/* #define MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS
|
||||
*
|
||||
* This is only relevant for single compilation unit (SCU) builds of
|
||||
* mlkem-native. In this case, it determines whether directives defined in
|
||||
* parameter-set-independent headers should be #undef'ined or not at the
|
||||
* end of the SCU file. This is needed in multilevel builds.
|
||||
*
|
||||
* See examples/multilevel_build_native for an example.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
/* #define MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH
|
||||
*
|
||||
* Determines whether a native arithmetic backend should be used.
|
||||
*
|
||||
* The arithmetic backend covers performance-critical functions such as the
|
||||
* number-theoretic transform (NTT).
|
||||
*
|
||||
* If this option is unset, the C backend will be used.
|
||||
*
|
||||
* If this option is set, the arithmetic backend to be used is determined
|
||||
* by MLK_CONFIG_ARITH_BACKEND_FILE: if the latter is unset, the default
|
||||
* backend for the target architecture will be used. If set, it must be the
|
||||
* name of a backend metadata file.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH)
|
||||
/* #define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */
|
||||
#endif
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_ARITH_BACKEND_FILE
|
||||
*
|
||||
* The arithmetic backend to use.
|
||||
*
|
||||
* If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is unset, this option is ignored.
|
||||
*
|
||||
* If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, this option must either
|
||||
* be undefined or the filename of an arithmetic backend. If unset, the
|
||||
* default backend will be used.
|
||||
*
|
||||
* This can be set using CFLAGS.
|
||||
*/
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \
|
||||
!defined(MLK_CONFIG_ARITH_BACKEND_FILE)
|
||||
#define MLK_CONFIG_ARITH_BACKEND_FILE "native/meta.h"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202
|
||||
*
|
||||
* Determines whether a native FIPS202 backend should be used.
|
||||
*
|
||||
* The FIPS202 backend covers 1x/2x/4x-fold Keccak-f1600, which is the
|
||||
* performance bottleneck of SHA3 and SHAKE.
|
||||
*
|
||||
* If this option is unset, the C backend will be used.
|
||||
*
|
||||
* If this option is set, the FIPS202 backend to be used is determined by
|
||||
* MLK_CONFIG_FIPS202_BACKEND_FILE: if the latter is unset, the default
|
||||
* backend for the target architecture will be used. If set, it must be
|
||||
* the name of a backend metadata file.
|
||||
*
|
||||
* This can also be set using CFLAGS.
|
||||
*/
|
||||
#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202)
|
||||
/* #define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */
|
||||
#endif
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_FIPS202_BACKEND_FILE
|
||||
*
|
||||
* The FIPS-202 backend to use.
|
||||
*
|
||||
* If MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, this option must either
|
||||
* be undefined or the filename of a FIPS202 backend. If unset, the default
|
||||
* backend will be used.
|
||||
*
|
||||
* This can be set using CFLAGS.
|
||||
*/
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \
|
||||
!defined(MLK_CONFIG_FIPS202_BACKEND_FILE)
|
||||
#define MLK_CONFIG_FIPS202_BACKEND_FILE "fips202/native/auto.h"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_FIPS202_CUSTOM_HEADER
|
||||
*
|
||||
* Custom header to use for FIPS-202.
|
||||
*
|
||||
* This should only be set if you intend to use a custom FIPS-202
|
||||
* implementation, different from the one shipped with mlkem-native.
|
||||
*
|
||||
* If set, it must be the name of a file serving as the replacement for
|
||||
* mlkem/fips202/fips202.h, and exposing the same API (see FIPS202.md).
|
||||
*/
|
||||
/* #define MLK_CONFIG_FIPS202_CUSTOM_HEADER "SOME_FILE.h" */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_FIPS202X4_CUSTOM_HEADER
|
||||
*
|
||||
* Custom header to use for FIPS-202-X4.
|
||||
*
|
||||
* This should only be set if you intend to use a custom FIPS-202
|
||||
* implementation, different from the one shipped with mlkem-native.
|
||||
*
|
||||
* If set, it must be the name of a file serving as the replacement for
|
||||
* mlkem/fips202/fips202x4.h, and exposing the same API (see FIPS202.md).
|
||||
*/
|
||||
/* #define MLK_CONFIG_FIPS202X4_CUSTOM_HEADER "SOME_FILE.h" */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CUSTOM_ZEROIZE
|
||||
*
|
||||
* In compliance with @[FIPS203, Section 3.3], mlkem-native zeroizes
|
||||
* intermediate stack buffers before returning from function calls.
|
||||
*
|
||||
* Set this option and define `mlk_zeroize` if you want to use a custom
|
||||
* method to zeroize intermediate stack buffers. The default implementation
|
||||
* uses SecureZeroMemory on Windows and a memset + compiler barrier
|
||||
* otherwise. If neither of those is available on the target platform,
|
||||
* compilation will fail, and you will need to use MLK_CONFIG_CUSTOM_ZEROIZE
|
||||
* to provide a custom implementation of `mlk_zeroize()`.
|
||||
*
|
||||
* @warning The explicit stack zeroization conducted by mlkem-native reduces
|
||||
* the likelihood of data leaking on the stack, but does not
|
||||
* eliminate it. The C standard makes no guarantee about where a
|
||||
* compiler allocates structures and whether/where it makes copies
|
||||
* of them. Also, in addition to entire structures, there may also
|
||||
* be potentially exploitable leakage of individual values on the
|
||||
* stack. If you need bullet-proof zeroization of the stack, you
|
||||
* need to consider additional measures instead of what this
|
||||
* feature provides. In this case, you can set mlk_zeroize to a
|
||||
* no-op.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CUSTOM_ZEROIZE
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <stdint.h>
|
||||
#include "src/sys.h"
|
||||
static MLK_INLINE void mlk_zeroize(void *ptr, size_t len)
|
||||
{
|
||||
... your implementation ...
|
||||
}
|
||||
#endif
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CUSTOM_RANDOMBYTES
|
||||
*
|
||||
* mlkem-native does not provide a secure randombytes implementation. Such
|
||||
* an implementation has to be provided by the consumer.
|
||||
*
|
||||
* If this option is not set, mlkem-native expects a function
|
||||
* int randombytes(uint8_t *out, size_t outlen). It is expected to return
|
||||
* zero on success, and non-zero on failure. In case of failure, the
|
||||
* top-level APIs will return an MLK_ERR_RNG_FAIL error code.
|
||||
*
|
||||
* Set this option and define `mlk_randombytes` (with the same signature
|
||||
* and behaviour) if you want to use a custom method to sample randombytes
|
||||
* with a different name or signature.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CUSTOM_RANDOMBYTES
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <stdint.h>
|
||||
#include "src/sys.h"
|
||||
static MLK_INLINE int mlk_randombytes(uint8_t *ptr, size_t len)
|
||||
{
|
||||
... your implementation ...
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CUSTOM_CAPABILITY_FUNC
|
||||
*
|
||||
* mlkem-native backends may rely on specific hardware features. Those
|
||||
* backends will only be included in an mlkem-native build if support for
|
||||
* the respective features is enabled at compile-time. However, when
|
||||
* building for a heterogeneous set of CPUs to run the resulting
|
||||
* binary/library on, feature detection at _runtime_ is needed to decide
|
||||
* whether a backend can be used or not.
|
||||
*
|
||||
* Set this option and define `mlk_sys_check_capability` if you want to
|
||||
* use a custom method to dispatch between implementations.
|
||||
*
|
||||
* If this option is not set, mlkem-native uses compile-time feature
|
||||
* detection only to decide which backend to use.
|
||||
*
|
||||
* If you compile mlkem-native on a system with different capabilities
|
||||
* than the system that the resulting binary/library will be run on, you
|
||||
* must use this option.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC
|
||||
static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap)
|
||||
__contract__(
|
||||
ensures(return_value == 0 || return_value == 1)
|
||||
)
|
||||
{
|
||||
... your implementation ...
|
||||
}
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CUSTOM_ALLOC_FREE [EXPERIMENTAL]
|
||||
*
|
||||
* Set this option and define `MLK_CUSTOM_ALLOC` and `MLK_CUSTOM_FREE` if
|
||||
* you want to use custom allocation for large local structures or buffers.
|
||||
*
|
||||
* By default, all buffers/structures are allocated on the stack. If this
|
||||
* option is set, most of them will be allocated via MLK_CUSTOM_ALLOC.
|
||||
*
|
||||
* Parameters to MLK_CUSTOM_ALLOC:
|
||||
* - T* v: Target pointer to declare.
|
||||
* - T: Type of structure to be allocated.
|
||||
* - N: Number of elements to be allocated.
|
||||
*
|
||||
* Parameters to MLK_CUSTOM_FREE:
|
||||
* - T* v: Target pointer to free. May be NULL.
|
||||
* - T: Type of structure to be freed.
|
||||
* - N: Number of elements to be freed.
|
||||
*
|
||||
* @warning This option is experimental. Its scope, configuration and
|
||||
* function/macro signatures may change at any time. We expect a
|
||||
* stable API for v2.
|
||||
*
|
||||
* @note Even if this option is set, some allocations further down the call
|
||||
* stack will still be made from the stack, consuming up to 3KB of
|
||||
* stack space. Those will likely be added to the scope of this
|
||||
* option in the future.
|
||||
*
|
||||
* @note MLK_CUSTOM_ALLOC need not guarantee a successful allocation nor
|
||||
* include error handling. Upon failure, the target pointer should
|
||||
* simply be set to NULL. The calling code will handle this case and
|
||||
* invoke MLK_CUSTOM_FREE.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CUSTOM_ALLOC_FREE
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <stdlib.h>
|
||||
#define MLK_CUSTOM_ALLOC(v, T, N) \
|
||||
T* (v) = (T *)aligned_alloc(MLK_DEFAULT_ALIGN, \
|
||||
MLK_ALIGN_UP(sizeof(T) * (N)))
|
||||
#define MLK_CUSTOM_FREE(v, T, N) free(v)
|
||||
#endif
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CUSTOM_MEMCPY
|
||||
*
|
||||
* Set this option and define `mlk_memcpy` if you want to use a custom
|
||||
* method to copy memory instead of the standard library memcpy function.
|
||||
*
|
||||
* The custom implementation must have the same signature and behavior as
|
||||
* the standard memcpy function:
|
||||
* void *mlk_memcpy(void *dest, const void *src, size_t n)
|
||||
*/
|
||||
/* #define MLK_CONFIG_CUSTOM_MEMCPY
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <stdint.h>
|
||||
#include "src/sys.h"
|
||||
static MLK_INLINE void *mlk_memcpy(void *dest, const void *src, size_t n)
|
||||
{
|
||||
... your implementation ...
|
||||
}
|
||||
#endif
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CUSTOM_MEMSET
|
||||
*
|
||||
* Set this option and define `mlk_memset` if you want to use a custom
|
||||
* method to set memory instead of the standard library memset function.
|
||||
*
|
||||
* The custom implementation must have the same signature and behavior as
|
||||
* the standard memset function:
|
||||
* void *mlk_memset(void *s, int c, size_t n)
|
||||
*/
|
||||
/* #define MLK_CONFIG_CUSTOM_MEMSET
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <stdint.h>
|
||||
#include "src/sys.h"
|
||||
static MLK_INLINE void *mlk_memset(void *s, int c, size_t n)
|
||||
{
|
||||
... your implementation ...
|
||||
}
|
||||
#endif
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_INTERNAL_API_QUALIFIER
|
||||
*
|
||||
* If set, this option provides an additional qualifier to be added to
|
||||
* declarations of internal API functions and data.
|
||||
*
|
||||
* The primary use case for this option are single-CU builds, in which case
|
||||
* this option can be set to `static`.
|
||||
*/
|
||||
/* #define MLK_CONFIG_INTERNAL_API_QUALIFIER */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CT_TESTING_ENABLED
|
||||
*
|
||||
* If set, mlkem-native annotates data as secret/public using valgrind's
|
||||
* annotations VALGRIND_MAKE_MEM_UNDEFINED and VALGRIND_MAKE_MEM_DEFINED,
|
||||
* enabling various checks for secret-dependent control flow or
|
||||
* variable-time execution (depending on the exact version of valgrind
|
||||
* installed).
|
||||
*/
|
||||
/* #define MLK_CONFIG_CT_TESTING_ENABLED */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_NO_ASM
|
||||
*
|
||||
* If this option is set, mlkem-native will be built without use of native
|
||||
* code or inline assembly.
|
||||
*
|
||||
* By default, inline assembly is used to implement value barriers. Without
|
||||
* inline assembly, mlkem-native will use a global volatile 'opt blocker'
|
||||
* instead; see verify.h.
|
||||
*
|
||||
* Inline assembly is also used to implement a secure zeroization function
|
||||
* on non-Windows platforms. If this option is set and the target platform
|
||||
* is not Windows, you MUST set MLK_CONFIG_CUSTOM_ZEROIZE and provide a
|
||||
* custom zeroization function.
|
||||
*
|
||||
* If this option is set, MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 and
|
||||
* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH will be ignored, and no native
|
||||
* backends will be used.
|
||||
*/
|
||||
/* #define MLK_CONFIG_NO_ASM */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_NO_ASM_VALUE_BARRIER
|
||||
*
|
||||
* If this option is set, mlkem-native will be built without use of native
|
||||
* code or inline assembly for value barriers.
|
||||
*
|
||||
* By default, inline assembly (if available) is used to implement value
|
||||
* barriers. Without inline assembly, mlkem-native will use a global
|
||||
* volatile 'opt blocker' instead; see verify.h.
|
||||
*/
|
||||
/* #define MLK_CONFIG_NO_ASM_VALUE_BARRIER */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_KEYGEN_PCT
|
||||
*
|
||||
* Compliance with @[FIPS140_3_IG, p.87] requires a Pairwise Consistency
|
||||
* Test (PCT) to be carried out on a freshly generated keypair before it
|
||||
* can be exported.
|
||||
*
|
||||
* Set this option if such a check should be implemented. In this case,
|
||||
* crypto_kem_keypair_derand and crypto_kem_keypair will return a non-zero
|
||||
* error code if the PCT failed.
|
||||
*
|
||||
* @note This feature will drastically lower the performance of key
|
||||
* generation.
|
||||
*/
|
||||
/* #define MLK_CONFIG_KEYGEN_PCT */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST
|
||||
*
|
||||
* If this option is set, the user must provide a runtime function
|
||||
* `static inline int mlk_break_pct() { ... }` to indicate whether the PCT
|
||||
* should be made to fail.
|
||||
*
|
||||
* This option only has an effect if MLK_CONFIG_KEYGEN_PCT is set.
|
||||
*/
|
||||
/* #define MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "src/sys.h"
|
||||
static MLK_INLINE int mlk_break_pct(void)
|
||||
{
|
||||
... return 0/1 depending on whether PCT should be broken ...
|
||||
}
|
||||
#endif
|
||||
*/
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_SERIAL_FIPS202_ONLY
|
||||
*
|
||||
* Set this to use a FIPS202 implementation with global state that supports
|
||||
* only one active Keccak computation at a time (e.g. some hardware
|
||||
* accelerators).
|
||||
*
|
||||
* If this option is set, batched Keccak operations are disabled for
|
||||
* rejection sampling during matrix generation. Instead, matrix entries
|
||||
* will be generated one at a time.
|
||||
*
|
||||
* This allows offloading Keccak computations to a hardware accelerator
|
||||
* that holds only a single Keccak state locally, rather than requiring
|
||||
* support for batched (4x) Keccak states.
|
||||
*
|
||||
* @note Depending on the target CPU, disabling batched Keccak may reduce
|
||||
* performance when using software FIPS202 implementations. Only
|
||||
* enable this when you have to.
|
||||
*/
|
||||
/* #define MLK_CONFIG_SERIAL_FIPS202_ONLY */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER
|
||||
*
|
||||
* Set this to add a context parameter that is provided to public API
|
||||
* functions and is then available in custom callbacks.
|
||||
*
|
||||
* The type of the context parameter is configured via
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CONTEXT_PARAMETER */
|
||||
|
||||
/**
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE
|
||||
*
|
||||
* Set this to define the type for the context parameter used by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER.
|
||||
*
|
||||
* This is only relevant if MLK_CONFIG_CONTEXT_PARAMETER is set.
|
||||
*/
|
||||
/* #define MLK_CONFIG_CONTEXT_PARAMETER_TYPE void* */
|
||||
|
||||
/************************* Config internals ********************************/
|
||||
|
||||
#endif /* MLK_BUILD_INTERNAL */
|
||||
|
||||
/* Default namespace
|
||||
*
|
||||
* Don't change this. If you need a different namespace, re-define
|
||||
* MLK_CONFIG_NAMESPACE_PREFIX above instead, and remove the following.
|
||||
*
|
||||
* The default MLKEM namespace is
|
||||
*
|
||||
* PQCP_MLKEM_NATIVE_MLKEM<LEVEL>_
|
||||
*
|
||||
* e.g., PQCP_MLKEM_NATIVE_MLKEM512_
|
||||
*/
|
||||
|
||||
#if MLK_CONFIG_PARAMETER_SET == 512
|
||||
#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM512
|
||||
#elif MLK_CONFIG_PARAMETER_SET == 768
|
||||
#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM768
|
||||
#elif MLK_CONFIG_PARAMETER_SET == 1024
|
||||
#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM1024
|
||||
#endif
|
||||
|
||||
#endif /* !MLK_CONFIG_H */
|
||||
197
sdk/software/examples/mlkem_test/mlkem_native/src/cbmc.h
Normal file
197
sdk/software/examples/mlkem_test/mlkem_native/src/cbmc.h
Normal file
@@ -0,0 +1,197 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_CBMC_H
|
||||
#define MLK_CBMC_H
|
||||
/***************************************************
|
||||
* Basic replacements for __CPROVER_XXX contracts
|
||||
***************************************************/
|
||||
/*
|
||||
* The `__contract__` / `__loop__` annotation macros use a
|
||||
* leading-double-underscore spelling in line with other CBMC macros.
|
||||
* clang-tidy flags these as reserved identifiers; we suppress the diagnostic
|
||||
* at each definition site (NOLINT) rather than disabling the check globally,
|
||||
* so it stays active for the rest of the tree.
|
||||
*/
|
||||
#ifndef CBMC
|
||||
|
||||
/* clang-format off */
|
||||
#define __contract__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */
|
||||
#define __loop__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */
|
||||
/* clang-format on */
|
||||
|
||||
#else /* !CBMC */
|
||||
|
||||
|
||||
/* clang-format off */
|
||||
#define __contract__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */
|
||||
#define __loop__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */
|
||||
/* clang-format on */
|
||||
|
||||
/* https://diffblue.github.io/cbmc/contracts-assigns.html */
|
||||
#define assigns(...) __CPROVER_assigns(__VA_ARGS__)
|
||||
|
||||
/* https://diffblue.github.io/cbmc/contracts-requires-ensures.html */
|
||||
#define requires(...) __CPROVER_requires(__VA_ARGS__)
|
||||
#define ensures(...) __CPROVER_ensures(__VA_ARGS__)
|
||||
/* https://diffblue.github.io/cbmc/contracts-loops.html */
|
||||
#define invariant(...) __CPROVER_loop_invariant(__VA_ARGS__)
|
||||
#define decreases(...) __CPROVER_decreases(__VA_ARGS__)
|
||||
/* cassert to avoid confusion with in-built assert */
|
||||
#define cassert(x) __CPROVER_assert(x, "cbmc assertion failed")
|
||||
#define assume(...) __CPROVER_assume(__VA_ARGS__)
|
||||
|
||||
/***************************************************
|
||||
* Macros for "expression" forms that may appear
|
||||
* _inside_ top-level contracts.
|
||||
***************************************************/
|
||||
|
||||
/*
|
||||
* function return value - useful inside ensures
|
||||
* https://diffblue.github.io/cbmc/contracts-functions.html
|
||||
*/
|
||||
#define return_value (__CPROVER_return_value)
|
||||
|
||||
/*
|
||||
* assigns l-value targets
|
||||
* https://diffblue.github.io/cbmc/contracts-assigns.html
|
||||
*/
|
||||
#define object_whole(...) __CPROVER_object_whole(__VA_ARGS__)
|
||||
#define memory_slice(...) __CPROVER_object_upto(__VA_ARGS__)
|
||||
|
||||
/*
|
||||
* Pointer-related predicates
|
||||
* https://diffblue.github.io/cbmc/contracts-memory-predicates.html
|
||||
*/
|
||||
#define memory_no_alias(...) __CPROVER_is_fresh(__VA_ARGS__)
|
||||
#define readable(...) __CPROVER_r_ok(__VA_ARGS__)
|
||||
#define writeable(...) __CPROVER_w_ok(__VA_ARGS__)
|
||||
|
||||
/* Maximum supported buffer size
|
||||
*
|
||||
* Larger buffers may be supported, but due to internal modeling constraints
|
||||
* in CBMC, the proofs of memory- and type-safety won't be able to run.
|
||||
*
|
||||
* If you find yourself in need for a buffer size larger than this,
|
||||
* please contact the maintainers, so we can prioritize work to relax
|
||||
* this somewhat artificial bound.
|
||||
*/
|
||||
#define MLK_MAX_BUFFER_SIZE (SIZE_MAX >> 12)
|
||||
|
||||
/*
|
||||
* History variables
|
||||
* https://diffblue.github.io/cbmc/contracts-history-variables.html
|
||||
*/
|
||||
#define old(...) __CPROVER_old(__VA_ARGS__)
|
||||
#define loop_entry(...) __CPROVER_loop_entry(__VA_ARGS__)
|
||||
|
||||
/*
|
||||
* Quantifiers
|
||||
* Note that the range on qvar is _exclusive_ between qvar_lb .. qvar_ub
|
||||
* https://diffblue.github.io/cbmc/contracts-quantifiers.html
|
||||
*
|
||||
* The quantified variable is declared as uint32_t, so these macros
|
||||
* quantify only over indices in [0, UINT32_MAX). Bounds larger than
|
||||
* UINT32_MAX (4 GiB) are NOT supported: the explicit (uint32_t) casts
|
||||
* on the bounds will trigger CBMC's conversion check if a wider bound
|
||||
* (e.g. a size_t > UINT32_MAX) is passed.
|
||||
*
|
||||
* Quantifying over size_t (64-bit) was found to blow up SMT proof
|
||||
* times, so we deliberately keep the index width at 32 bits. Callers
|
||||
* dealing with size_t-typed buffers must add an explicit
|
||||
* requires(len <= UINT32_MAX)
|
||||
* precondition.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Prevent clang-format from corrupting CBMC's special ==> operator
|
||||
*/
|
||||
/* clang-format off */
|
||||
#define forall(qvar, qvar_lb, qvar_ub, predicate) \
|
||||
__CPROVER_forall \
|
||||
{ \
|
||||
uint32_t qvar; \
|
||||
((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> (predicate) \
|
||||
}
|
||||
|
||||
#define exists(qvar, qvar_lb, qvar_ub, predicate) \
|
||||
__CPROVER_exists \
|
||||
{ \
|
||||
uint32_t qvar; \
|
||||
((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) && (predicate) \
|
||||
}
|
||||
/* clang-format on */
|
||||
|
||||
/***************************************************
|
||||
* Convenience macros for common contract patterns
|
||||
***************************************************/
|
||||
|
||||
/*
|
||||
* Boolean-value predidate that asserts that "all values of array_var are in
|
||||
* range value_lb (inclusive) .. value_ub (exclusive)"
|
||||
* Example:
|
||||
* array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)
|
||||
* expands to
|
||||
* __CPROVER_forall { int k; (0 <= k && k <= MLKEM_N-1) ==> (
|
||||
* 0 <= a->coeffs[k]) && a->coeffs[k] < MLKEM_Q)) }
|
||||
*/
|
||||
|
||||
/*
|
||||
* Prevent clang-format from corrupting CBMC's special ==> operator
|
||||
*/
|
||||
/* clang-format off */
|
||||
#define CBMC_CONCAT_(left, right) left##right
|
||||
#define CBMC_CONCAT(left, right) CBMC_CONCAT_(left, right)
|
||||
|
||||
#define array_bound_core(qvar, qvar_lb, qvar_ub, array_var, \
|
||||
value_lb, value_ub) \
|
||||
__CPROVER_forall \
|
||||
{ \
|
||||
uint32_t qvar; \
|
||||
((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \
|
||||
(((int)(value_lb) <= ((array_var)[(qvar)])) && \
|
||||
(((array_var)[(qvar)]) < (int)(value_ub))) \
|
||||
}
|
||||
|
||||
#define array_bound(array_var, qvar_lb, qvar_ub, value_lb, value_ub) \
|
||||
array_bound_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (qvar_lb), \
|
||||
(qvar_ub), (array_var), (value_lb), (value_ub))
|
||||
|
||||
#define array_unchanged_core(qvar, qvar_lb, qvar_ub, array_var) \
|
||||
__CPROVER_forall \
|
||||
{ \
|
||||
uint32_t qvar; \
|
||||
((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \
|
||||
((array_var)[(qvar)]) == (old(* (int16_t (*)[(qvar_ub)])(array_var)))[(qvar)] \
|
||||
}
|
||||
|
||||
#define array_unchanged(array_var, N) \
|
||||
array_unchanged_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var))
|
||||
|
||||
#define array_unchanged_u64_core(qvar, qvar_lb, qvar_ub, array_var) \
|
||||
__CPROVER_forall \
|
||||
{ \
|
||||
uint32_t qvar; \
|
||||
((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \
|
||||
((array_var)[(qvar)]) == (old(* (uint64_t (*)[(qvar_ub)])(array_var)))[(qvar)] \
|
||||
}
|
||||
|
||||
#define array_unchanged_u64(array_var, N) \
|
||||
array_unchanged_u64_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var))
|
||||
/* clang-format on */
|
||||
|
||||
/* Wrapper around array_bound operating on absolute values.
|
||||
*
|
||||
* The absolute value bound `k` is exclusive.
|
||||
*
|
||||
* Note that since the lower bound in array_bound is inclusive, we have to
|
||||
* raise it by 1 here.
|
||||
*/
|
||||
#define array_abs_bound(arr, lb, ub, k) \
|
||||
array_bound((arr), (lb), (ub), -((int)(k)) + 1, (k))
|
||||
|
||||
#endif /* CBMC */
|
||||
|
||||
#endif /* !MLK_CBMC_H */
|
||||
282
sdk/software/examples/mlkem_test/mlkem_native/src/common.h
Normal file
282
sdk/software/examples/mlkem_test/mlkem_native/src/common.h
Normal file
@@ -0,0 +1,282 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_COMMON_H
|
||||
#define MLK_COMMON_H
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
#include <stdint.h>
|
||||
#endif
|
||||
|
||||
#define MLK_BUILD_INTERNAL
|
||||
|
||||
#if defined(MLK_CONFIG_FILE)
|
||||
#include MLK_CONFIG_FILE
|
||||
#else
|
||||
#include "mlkem_native_config.h"
|
||||
#endif
|
||||
|
||||
#include "params.h"
|
||||
#include "sys.h"
|
||||
|
||||
/* Internal and public API have external linkage by default, but
|
||||
* this can be overwritten by the user, e.g. for single-CU builds. */
|
||||
#if !defined(MLK_CONFIG_INTERNAL_API_QUALIFIER)
|
||||
#define MLK_INTERNAL_API
|
||||
#define MLK_INTERNAL_DATA_DECLARATION extern
|
||||
#define MLK_INTERNAL_DATA_DEFINITION
|
||||
#else
|
||||
#define MLK_INTERNAL_API MLK_CONFIG_INTERNAL_API_QUALIFIER
|
||||
#define MLK_INTERNAL_DATA_DECLARATION MLK_CONFIG_INTERNAL_API_QUALIFIER
|
||||
#define MLK_INTERNAL_DATA_DEFINITION MLK_CONFIG_INTERNAL_API_QUALIFIER
|
||||
#endif
|
||||
|
||||
#if !defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER)
|
||||
#define MLK_EXTERNAL_API
|
||||
#else
|
||||
#define MLK_EXTERNAL_API MLK_CONFIG_EXTERNAL_API_QUALIFIER
|
||||
#endif
|
||||
|
||||
#define MLK_CONCAT_(x1, x2) x1##x2
|
||||
#define MLK_CONCAT(x1, x2) MLK_CONCAT_(x1, x2)
|
||||
|
||||
#if (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || \
|
||||
defined(MLK_CONFIG_MULTILEVEL_NO_SHARED))
|
||||
#define MLK_ADD_PARAM_SET(s) MLK_CONCAT(s, MLK_CONFIG_PARAMETER_SET)
|
||||
#else
|
||||
#define MLK_ADD_PARAM_SET(s) s
|
||||
#endif
|
||||
|
||||
#define MLK_NAMESPACE_PREFIX MLK_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, _)
|
||||
#define MLK_NAMESPACE_PREFIX_K \
|
||||
MLK_CONCAT(MLK_ADD_PARAM_SET(MLK_CONFIG_NAMESPACE_PREFIX), _)
|
||||
|
||||
/* Functions are prefixed by MLK_CONFIG_NAMESPACE_PREFIX.
|
||||
*
|
||||
* If multiple parameter sets are used, functions depending on the parameter
|
||||
* set are additionally prefixed with 512/768/1024. See mlkem_native_config.h.
|
||||
*
|
||||
* Example: If MLK_CONFIG_NAMESPACE_PREFIX is mlkem, then
|
||||
* MLK_NAMESPACE_K(enc) becomes mlkem512_enc/mlkem768_enc/mlkem1024_enc.
|
||||
*/
|
||||
#define MLK_NAMESPACE(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX, s)
|
||||
#define MLK_NAMESPACE_K(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX_K, s)
|
||||
|
||||
/* On Apple platforms, we need to emit leading underscore
|
||||
* in front of assembly symbols. We thus introducee a separate
|
||||
* namespace wrapper for ASM symbols. */
|
||||
#if !defined(__APPLE__)
|
||||
#define MLK_ASM_NAMESPACE(sym) MLK_NAMESPACE(sym)
|
||||
#else
|
||||
#define MLK_ASM_NAMESPACE(sym) MLK_CONCAT(_, MLK_NAMESPACE(sym))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* On X86_64 if control-flow protections (CET) are enabled (through
|
||||
* -fcf-protection=), we add an endbr64 instruction at every global function
|
||||
* label. See sys.h for more details
|
||||
*/
|
||||
#if defined(MLK_SYS_X86_64)
|
||||
#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : MLK_CET_ENDBR
|
||||
#elif defined(MLK_SYS_ARMV81M_MVE)
|
||||
/* clang-format off */
|
||||
#define MLK_ASM_FN_SYMBOL(sym) \
|
||||
.type MLK_ASM_NAMESPACE(sym), %function; \
|
||||
MLK_ASM_NAMESPACE(sym) :
|
||||
/* clang-format on */
|
||||
#else /* !MLK_SYS_X86_64 && MLK_SYS_ARMV81M_MVE */
|
||||
#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) :
|
||||
#endif /* !MLK_SYS_X86_64 && !MLK_SYS_ARMV81M_MVE */
|
||||
|
||||
/*
|
||||
* Output the size of an assembly function.
|
||||
*/
|
||||
#if defined(__ELF__)
|
||||
#define MLK_ASM_FN_SIZE(sym) \
|
||||
.size MLK_ASM_NAMESPACE(sym), .- MLK_ASM_NAMESPACE(sym)
|
||||
#else
|
||||
#define MLK_ASM_FN_SIZE(sym)
|
||||
#endif
|
||||
|
||||
/* We aim to simplify the user's life by supporting builds where
|
||||
* all source files are included, even those that are not needed.
|
||||
* Those files are appropriately guarded and will be empty when unneeded.
|
||||
* The following is to avoid compilers complaining about this. */
|
||||
#define MLK_EMPTY_CU(s) extern int MLK_NAMESPACE_K(empty_cu_##s);
|
||||
|
||||
/* MLK_CONFIG_NO_ASM takes precedence over MLK_USE_NATIVE_XXX */
|
||||
#if defined(MLK_CONFIG_NO_ASM)
|
||||
#undef MLK_CONFIG_USE_NATIVE_BACKEND_ARITH
|
||||
#undef MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \
|
||||
!defined(MLK_CONFIG_ARITH_BACKEND_FILE)
|
||||
#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, but MLK_CONFIG_ARITH_BACKEND_FILE is not.
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \
|
||||
!defined(MLK_CONFIG_FIPS202_BACKEND_FILE)
|
||||
#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, but MLK_CONFIG_FIPS202_BACKEND_FILE is not.
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_NO_RANDOMIZED_API) && defined(MLK_CONFIG_KEYGEN_PCT)
|
||||
#error Bad configuration: MLK_CONFIG_NO_RANDOMIZED_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires crypto_kem_enc()
|
||||
#endif
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH)
|
||||
#include MLK_CONFIG_ARITH_BACKEND_FILE
|
||||
/* Include to enforce consistency of API and implementation,
|
||||
* and conduct sanity checks on the backend.
|
||||
*
|
||||
* Keep this _after_ the inclusion of the backend; otherwise,
|
||||
* the sanity checks won't have an effect. */
|
||||
#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__)
|
||||
#include "native/api.h"
|
||||
#endif
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */
|
||||
|
||||
#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202)
|
||||
#include MLK_CONFIG_FIPS202_BACKEND_FILE
|
||||
/* Include to enforce consistency of API and implementation,
|
||||
* and conduct sanity checks on the backend.
|
||||
*
|
||||
* Keep this _after_ the inclusion of the backend; otherwise,
|
||||
* the sanity checks won't have an effect. */
|
||||
#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__)
|
||||
#include "fips202/native/api.h"
|
||||
#endif
|
||||
#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */
|
||||
|
||||
#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER)
|
||||
#define MLK_FIPS202_HEADER_FILE "fips202/fips202.h"
|
||||
#else
|
||||
#define MLK_FIPS202_HEADER_FILE MLK_CONFIG_FIPS202_CUSTOM_HEADER
|
||||
#endif
|
||||
|
||||
#if !defined(MLK_CONFIG_FIPS202X4_CUSTOM_HEADER)
|
||||
#define MLK_FIPS202X4_HEADER_FILE "fips202/fips202x4.h"
|
||||
#else
|
||||
#define MLK_FIPS202X4_HEADER_FILE MLK_CONFIG_FIPS202X4_CUSTOM_HEADER
|
||||
#endif
|
||||
|
||||
/* Standard library function replacements */
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#if !defined(MLK_CONFIG_CUSTOM_MEMCPY)
|
||||
#include <string.h>
|
||||
#define mlk_memcpy memcpy
|
||||
#endif
|
||||
|
||||
#if !defined(MLK_CONFIG_CUSTOM_MEMSET)
|
||||
#include <string.h>
|
||||
#define mlk_memset memset
|
||||
#endif
|
||||
|
||||
|
||||
/* Allocation macros for large local structures
|
||||
*
|
||||
* MLK_ALLOC(v, T, N) declares T *v and attempts to point it to an T[N]
|
||||
* MLK_FREE(v, T, N) zeroizes and frees the allocation
|
||||
*
|
||||
* Default implementation uses stack allocation.
|
||||
* Can be overridden by setting the config option MLK_CONFIG_CUSTOM_ALLOC_FREE
|
||||
* and defining MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE.
|
||||
*/
|
||||
#if defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) != \
|
||||
(defined(MLK_CUSTOM_ALLOC) && defined(MLK_CUSTOM_FREE))
|
||||
#error Bad configuration: MLK_CONFIG_CUSTOM_ALLOC_FREE must be set together with MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE
|
||||
#endif
|
||||
|
||||
/*
|
||||
* If the integration wants to provide a context parameter for use in
|
||||
* platform-specific hooks, then it should define this parameter.
|
||||
*
|
||||
* The MLK_CONTEXT_PARAMETERS_n macros are intended to be used with macros
|
||||
* defining the function names and expand to either pass or discard the context
|
||||
* argument as required by the current build. If there is no context parameter
|
||||
* requested then these are removed from the prototypes and from all calls.
|
||||
*/
|
||||
#ifdef MLK_CONFIG_CONTEXT_PARAMETER
|
||||
#define MLK_CONTEXT_PARAMETERS_0(context) (context)
|
||||
#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0, context)
|
||||
#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1, context)
|
||||
#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) \
|
||||
(arg0, arg1, arg2, context)
|
||||
#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \
|
||||
(arg0, arg1, arg2, arg3, context)
|
||||
#else /* MLK_CONFIG_CONTEXT_PARAMETER */
|
||||
#define MLK_CONTEXT_PARAMETERS_0(context) ()
|
||||
#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0)
|
||||
#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1)
|
||||
#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) (arg0, arg1, arg2)
|
||||
#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \
|
||||
(arg0, arg1, arg2, arg3)
|
||||
#endif /* !MLK_CONFIG_CONTEXT_PARAMETER */
|
||||
|
||||
#if defined(MLK_CONFIG_CONTEXT_PARAMETER_TYPE) != \
|
||||
defined(MLK_CONFIG_CONTEXT_PARAMETER)
|
||||
#error MLK_CONFIG_CONTEXT_PARAMETER_TYPE must be defined if and only if MLK_CONFIG_CONTEXT_PARAMETER is defined
|
||||
#endif
|
||||
|
||||
#if !defined(MLK_CONFIG_CUSTOM_ALLOC_FREE)
|
||||
/* Default: stack allocation */
|
||||
|
||||
/* This is a declaration macro, not an expression macro: T is a type and v is
|
||||
* a declarator, neither of which can be wrapped in parentheses. The
|
||||
* bugprone-macro-parentheses diagnostic is therefore a false positive here. */
|
||||
#define MLK_ALLOC(v, T, N, context) \
|
||||
MLK_ALIGN T mlk_alloc_##v[N]; \
|
||||
T *v = mlk_alloc_##v /* NOLINT(bugprone-macro-parentheses) */
|
||||
|
||||
/* The MLK_FREE macro body references mlk_zeroize(), which is declared in
|
||||
* verify.h. We deliberately do NOT include verify.h here: doing so would
|
||||
* create a circular dependency (verify.h includes common.h), and common.h
|
||||
* itself never calls mlk_zeroize() -- only the macro expansion does. Each
|
||||
* translation unit that uses MLK_FREE therefore includes verify.h directly. */
|
||||
#define MLK_FREE(v, T, N, context) \
|
||||
do \
|
||||
{ \
|
||||
mlk_zeroize(mlk_alloc_##v, sizeof(mlk_alloc_##v)); \
|
||||
(v) = NULL; \
|
||||
} while (0)
|
||||
|
||||
#else /* !MLK_CONFIG_CUSTOM_ALLOC_FREE */
|
||||
|
||||
/* Custom allocation */
|
||||
|
||||
/*
|
||||
* The indirection here is necessary to use MLK_CONTEXT_PARAMETERS_3 here.
|
||||
*/
|
||||
#define MLK_APPLY(f, args) f args
|
||||
|
||||
#define MLK_ALLOC(v, T, N, context) \
|
||||
MLK_APPLY(MLK_CUSTOM_ALLOC, MLK_CONTEXT_PARAMETERS_3(v, T, N, context))
|
||||
|
||||
#define MLK_FREE(v, T, N, context) \
|
||||
do \
|
||||
{ \
|
||||
if (v != NULL) \
|
||||
{ \
|
||||
mlk_zeroize(v, sizeof(T) * (N)); \
|
||||
MLK_APPLY(MLK_CUSTOM_FREE, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)); \
|
||||
v = NULL; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#endif /* MLK_CONFIG_CUSTOM_ALLOC_FREE */
|
||||
|
||||
/****************************** Error codes ***********************************/
|
||||
|
||||
/* Generic failure condition */
|
||||
#define MLK_ERR_FAIL (-1)
|
||||
/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE
|
||||
* is defined and the provided MLK_CUSTOM_ALLOC can fail. */
|
||||
#define MLK_ERR_OUT_OF_MEMORY (-2)
|
||||
/* An rng failure occured. Might be due to insufficient entropy or
|
||||
* system misconfiguration. */
|
||||
#define MLK_ERR_RNG_FAIL (-3)
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_COMMON_H */
|
||||
743
sdk/software/examples/mlkem_test/mlkem_native/src/compress.c
Normal file
743
sdk/software/examples/mlkem_test/mlkem_native/src/compress.c
Normal file
@@ -0,0 +1,743 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*
|
||||
* - [REF]
|
||||
* CRYSTALS-Kyber C reference implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/ref
|
||||
*/
|
||||
|
||||
#include "common.h"
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
|
||||
#include "cbmc.h"
|
||||
#include "compress.h"
|
||||
#include "debug.h"
|
||||
#include "verify.h"
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3)
|
||||
/* Reference: `poly_compress()` in the reference implementation @[REF],
|
||||
* for ML-KEM-{512,768}.
|
||||
* - In contrast to the reference implementation, we assume
|
||||
* unsigned canonical coefficients here.
|
||||
* The reference implementation works with coefficients
|
||||
* in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_compress_d4_c(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
)
|
||||
{
|
||||
unsigned i;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
|
||||
for (i = 0; i < MLKEM_N / 8; i++)
|
||||
__loop__(invariant(i <= MLKEM_N / 8)
|
||||
decreases(MLKEM_N / 8 - i))
|
||||
{
|
||||
unsigned j;
|
||||
uint8_t t[8] = {0};
|
||||
for (j = 0; j < 8; j++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 8 && j <= 8)
|
||||
invariant(array_bound(t, 0, j, 0, 16))
|
||||
decreases(8 - j))
|
||||
{
|
||||
t[j] = mlk_scalar_compress_d4(a->coeffs[8 * i + j]);
|
||||
}
|
||||
|
||||
/* All t[i] are 4-bit wide, so the truncations don't alter the value. */
|
||||
r[i * 4] = (uint8_t)(t[0] | (t[1] << 4));
|
||||
r[i * 4 + 1] = (uint8_t)(t[2] | (t[3] << 4));
|
||||
r[i * 4 + 2] = (uint8_t)(t[4] | (t[5] << 4));
|
||||
r[i * 4 + 3] = (uint8_t)(t[6] | (t[7] << 4));
|
||||
}
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4],
|
||||
const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4)
|
||||
int ret;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
ret = mlk_poly_compress_d4_native(r, a->coeffs);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */
|
||||
|
||||
mlk_poly_compress_d4_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: Embedded into `polyvec_compress()` in the
|
||||
* reference implementation, for ML-KEM-{512,768}.
|
||||
* - In contrast to the reference implementation, we assume
|
||||
* unsigned canonical coefficients here.
|
||||
* The reference implementation works with coefficients
|
||||
* in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_compress_d10_c(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
)
|
||||
{
|
||||
unsigned j;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
for (j = 0; j < MLKEM_N / 4; j++)
|
||||
__loop__(invariant(j <= MLKEM_N / 4)
|
||||
decreases(MLKEM_N / 4 - j))
|
||||
{
|
||||
unsigned k;
|
||||
uint16_t t[4];
|
||||
for (k = 0; k < 4; k++)
|
||||
__loop__(
|
||||
invariant(k <= 4)
|
||||
invariant(forall(r, 0, k, t[r] < (1u << 10)))
|
||||
decreases(4 - k))
|
||||
{
|
||||
t[k] = mlk_scalar_compress_d10(a->coeffs[4 * j + k]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Make all implicit truncation explicit. No data is being
|
||||
* truncated for the LHS's since each t[i] is 10-bit in size.
|
||||
*/
|
||||
r[5 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF);
|
||||
r[5 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 2) & 0xFF));
|
||||
r[5 * j + 2] = (uint8_t)((t[1] >> 6) | ((t[2] << 4) & 0xFF));
|
||||
r[5 * j + 3] = (uint8_t)((t[2] >> 4) | ((t[3] << 6) & 0xFF));
|
||||
r[5 * j + 4] = (uint8_t)(t[3] >> 2);
|
||||
}
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10],
|
||||
const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10)
|
||||
int ret;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
ret = mlk_poly_compress_d10_native(r, a->coeffs);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */
|
||||
|
||||
mlk_poly_compress_d10_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: `poly_decompress()` in the reference implementation @[REF],
|
||||
* for ML-KEM-{512,768}. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_decompress_d4_c(
|
||||
mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < MLKEM_N / 2; i++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 2)
|
||||
invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_Q))
|
||||
decreases(MLKEM_N / 2 - i))
|
||||
{
|
||||
r->coeffs[2 * i + 0] = mlk_scalar_decompress_d4((a[i] >> 0) & 0xF);
|
||||
r->coeffs[2 * i + 1] = mlk_scalar_decompress_d4((a[i] >> 4) & 0xF);
|
||||
}
|
||||
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d4(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4)
|
||||
int ret;
|
||||
ret = mlk_poly_decompress_d4_native(r->coeffs, a);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */
|
||||
|
||||
mlk_poly_decompress_d4_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: Embedded into `polyvec_decompress()` in the
|
||||
* reference implementation, for ML-KEM-{512,768}. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_decompress_d10_c(
|
||||
mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
unsigned j;
|
||||
for (j = 0; j < MLKEM_N / 4; j++)
|
||||
__loop__(
|
||||
invariant(j <= MLKEM_N / 4)
|
||||
invariant(array_bound(r->coeffs, 0, 4 * j, 0, MLKEM_Q))
|
||||
decreases(MLKEM_N / 4 - j))
|
||||
{
|
||||
unsigned k;
|
||||
uint16_t t[4];
|
||||
uint8_t const *base = &a[5 * j];
|
||||
|
||||
t[0] = 0x3FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8));
|
||||
t[1] = 0x3FF & ((base[1] >> 2) | ((uint16_t)base[2] << 6));
|
||||
t[2] = 0x3FF & ((base[2] >> 4) | ((uint16_t)base[3] << 4));
|
||||
t[3] = 0x3FF & ((base[3] >> 6) | ((uint16_t)base[4] << 2));
|
||||
|
||||
for (k = 0; k < 4; k++)
|
||||
__loop__(
|
||||
invariant(k <= 4)
|
||||
invariant(array_bound(r->coeffs, 0, 4 * j + k, 0, MLKEM_Q))
|
||||
decreases(4 - k))
|
||||
{
|
||||
r->coeffs[4 * j + k] = mlk_scalar_decompress_d10(t[k]);
|
||||
}
|
||||
}
|
||||
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d10(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10)
|
||||
int ret;
|
||||
ret = mlk_poly_decompress_d10_native(r->coeffs, a);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */
|
||||
|
||||
mlk_poly_decompress_d10_c(r, a);
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
/* Reference: `poly_compress()` in the reference implementation @[REF],
|
||||
* for ML-KEM-1024.
|
||||
* - In contrast to the reference implementation, we assume
|
||||
* unsigned canonical coefficients here.
|
||||
* The reference implementation works with coefficients
|
||||
* in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_compress_d5_c(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
)
|
||||
{
|
||||
unsigned i;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
|
||||
for (i = 0; i < MLKEM_N / 8; i++)
|
||||
__loop__(invariant(i <= MLKEM_N / 8)
|
||||
decreases(MLKEM_N / 8 - i))
|
||||
{
|
||||
unsigned j;
|
||||
uint8_t t[8] = {0};
|
||||
for (j = 0; j < 8; j++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 8 && j <= 8)
|
||||
invariant(array_bound(t, 0, j, 0, 32))
|
||||
decreases(8 - j))
|
||||
{
|
||||
t[j] = mlk_scalar_compress_d5(a->coeffs[8 * i + j]);
|
||||
}
|
||||
|
||||
r[i * 5] = (uint8_t)(0xFF & ((t[0] >> 0) | (t[1] << 5)));
|
||||
r[i * 5 + 1] = (uint8_t)(0xFF & ((t[1] >> 3) | (t[2] << 2) | (t[3] << 7)));
|
||||
r[i * 5 + 2] = (uint8_t)(0xFF & ((t[3] >> 1) | (t[4] << 4)));
|
||||
r[i * 5 + 3] = (uint8_t)(0xFF & ((t[4] >> 4) | (t[5] << 1) | (t[6] << 6)));
|
||||
r[i * 5 + 4] = (uint8_t)(0xFF & ((t[6] >> 2) | (t[7] << 3)));
|
||||
}
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5],
|
||||
const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5)
|
||||
int ret;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
ret = mlk_poly_compress_d5_native(r, a->coeffs);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */
|
||||
|
||||
mlk_poly_compress_d5_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: Embedded into `polyvec_compress()` in the
|
||||
* reference implementation, for ML-KEM-1024.
|
||||
* - In contrast to the reference implementation, we assume
|
||||
* unsigned canonical coefficients here.
|
||||
* The reference implementation works with coefficients
|
||||
* in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_compress_d11_c(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
)
|
||||
{
|
||||
unsigned j;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
|
||||
for (j = 0; j < MLKEM_N / 8; j++)
|
||||
__loop__(invariant(j <= MLKEM_N / 8)
|
||||
decreases(MLKEM_N / 8 - j))
|
||||
{
|
||||
unsigned k;
|
||||
uint16_t t[8];
|
||||
for (k = 0; k < 8; k++)
|
||||
__loop__(
|
||||
invariant(k <= 8)
|
||||
invariant(forall(r, 0, k, t[r] < (1u << 11)))
|
||||
decreases(8 - k))
|
||||
{
|
||||
t[k] = mlk_scalar_compress_d11(a->coeffs[8 * j + k]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Make all implicit truncation explicit. No data is being
|
||||
* truncated for the LHS's since each t[i] is 11-bit in size.
|
||||
*/
|
||||
r[11 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF);
|
||||
r[11 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 3) & 0xFF));
|
||||
r[11 * j + 2] = (uint8_t)((t[1] >> 5) | ((t[2] << 6) & 0xFF));
|
||||
r[11 * j + 3] = (uint8_t)((t[2] >> 2) & 0xFF);
|
||||
r[11 * j + 4] = (uint8_t)((t[2] >> 10) | ((t[3] << 1) & 0xFF));
|
||||
r[11 * j + 5] = (uint8_t)((t[3] >> 7) | ((t[4] << 4) & 0xFF));
|
||||
r[11 * j + 6] = (uint8_t)((t[4] >> 4) | ((t[5] << 7) & 0xFF));
|
||||
r[11 * j + 7] = (uint8_t)((t[5] >> 1) & 0xFF);
|
||||
r[11 * j + 8] = (uint8_t)((t[5] >> 9) | ((t[6] << 2) & 0xFF));
|
||||
r[11 * j + 9] = (uint8_t)((t[6] >> 6) | ((t[7] << 5) & 0xFF));
|
||||
r[11 * j + 10] = (uint8_t)(t[7] >> 3);
|
||||
}
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11],
|
||||
const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11)
|
||||
int ret;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
ret = mlk_poly_compress_d11_native(r, a->coeffs);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */
|
||||
|
||||
mlk_poly_compress_d11_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: `poly_decompress()` in the reference implementation @[REF],
|
||||
* for ML-KEM-1024. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_decompress_d5_c(
|
||||
mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < MLKEM_N / 8; i++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 8)
|
||||
invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q))
|
||||
decreases(MLKEM_N / 8 - i))
|
||||
{
|
||||
unsigned j;
|
||||
uint8_t t[8];
|
||||
const unsigned offset = i * 5;
|
||||
/*
|
||||
* Explicitly truncate to avoid warning about
|
||||
* implicit truncation in CBMC and unwind loop for ease
|
||||
* of proof.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Decompress 5 8-bit bytes (so 40 bits) into
|
||||
* 8 5-bit values stored in t[]
|
||||
*/
|
||||
t[0] = 0x1F & (a[offset + 0] >> 0);
|
||||
t[1] = 0x1F & ((a[offset + 0] >> 5) | (a[offset + 1] << 3));
|
||||
t[2] = 0x1F & (a[offset + 1] >> 2);
|
||||
t[3] = 0x1F & ((a[offset + 1] >> 7) | (a[offset + 2] << 1));
|
||||
t[4] = 0x1F & ((a[offset + 2] >> 4) | (a[offset + 3] << 4));
|
||||
t[5] = 0x1F & (a[offset + 3] >> 1);
|
||||
t[6] = 0x1F & ((a[offset + 3] >> 6) | (a[offset + 4] << 2));
|
||||
t[7] = 0x1F & (a[offset + 4] >> 3);
|
||||
|
||||
/* and copy to the correct slice in r[] */
|
||||
for (j = 0; j < 8; j++)
|
||||
__loop__(
|
||||
invariant(j <= 8 && i <= MLKEM_N / 8)
|
||||
invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q))
|
||||
decreases(8 - j))
|
||||
{
|
||||
r->coeffs[8 * i + j] = mlk_scalar_decompress_d5(t[j]);
|
||||
}
|
||||
}
|
||||
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d5(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5)
|
||||
int ret;
|
||||
ret = mlk_poly_decompress_d5_native(r->coeffs, a);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */
|
||||
|
||||
mlk_poly_decompress_d5_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: Embedded into `polyvec_decompress()` in the
|
||||
* reference implementation, for ML-KEM-1024. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_decompress_d11_c(
|
||||
mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
unsigned j;
|
||||
for (j = 0; j < MLKEM_N / 8; j++)
|
||||
__loop__(
|
||||
invariant(j <= MLKEM_N / 8)
|
||||
invariant(array_bound(r->coeffs, 0, 8 * j, 0, MLKEM_Q))
|
||||
decreases(MLKEM_N / 8 - j))
|
||||
{
|
||||
unsigned k;
|
||||
uint16_t t[8];
|
||||
uint8_t const *base = &a[11 * j];
|
||||
t[0] = 0x7FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8));
|
||||
t[1] = 0x7FF & ((base[1] >> 3) | ((uint16_t)base[2] << 5));
|
||||
t[2] = 0x7FF & ((base[2] >> 6) | ((uint16_t)base[3] << 2) |
|
||||
((uint16_t)base[4] << 10));
|
||||
t[3] = 0x7FF & ((base[4] >> 1) | ((uint16_t)base[5] << 7));
|
||||
t[4] = 0x7FF & ((base[5] >> 4) | ((uint16_t)base[6] << 4));
|
||||
t[5] = 0x7FF & ((base[6] >> 7) | ((uint16_t)base[7] << 1) |
|
||||
((uint16_t)base[8] << 9));
|
||||
t[6] = 0x7FF & ((base[8] >> 2) | ((uint16_t)base[9] << 6));
|
||||
t[7] = 0x7FF & ((base[9] >> 5) | ((uint16_t)base[10] << 3));
|
||||
|
||||
for (k = 0; k < 8; k++)
|
||||
__loop__(
|
||||
invariant(k <= 8)
|
||||
invariant(array_bound(r->coeffs, 0, 8 * j + k, 0, MLKEM_Q))
|
||||
decreases(8 - k))
|
||||
{
|
||||
r->coeffs[8 * j + k] = mlk_scalar_decompress_d11(t[k]);
|
||||
}
|
||||
}
|
||||
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d11(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11)
|
||||
int ret;
|
||||
ret = mlk_poly_decompress_d11_native(r->coeffs, a);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */
|
||||
|
||||
mlk_poly_decompress_d11_c(r, a);
|
||||
}
|
||||
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
/* Reference: `poly_tobytes()` in the reference implementation @[REF].
|
||||
* - In contrast to the reference implementation, we assume
|
||||
* unsigned canonical coefficients here.
|
||||
* The reference implementation works with coefficients
|
||||
* in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_tobytes_c(uint8_t r[MLKEM_POLYBYTES],
|
||||
const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYBYTES))
|
||||
)
|
||||
{
|
||||
unsigned i;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
|
||||
for (i = 0; i < MLKEM_N / 2; i++)
|
||||
__loop__(invariant(i <= MLKEM_N / 2)
|
||||
decreases(MLKEM_N / 2 - i))
|
||||
{
|
||||
/* The conversion to uint16_t is safe since we assume that
|
||||
* the coefficients of `a` are non-negative. */
|
||||
const uint16_t t0 = (uint16_t)a->coeffs[2 * i];
|
||||
const uint16_t t1 = (uint16_t)a->coeffs[2 * i + 1];
|
||||
/*
|
||||
* t0 and t1 are both < MLKEM_Q, so contain at most 12 bits each of
|
||||
* significant data, so these can be packed into 24 bits or exactly
|
||||
* 3 bytes, as follows.
|
||||
*/
|
||||
|
||||
/* Least significant bits 0 - 7 of t0. */
|
||||
r[3 * i + 0] = (uint8_t)(t0 & 0xFF);
|
||||
|
||||
/*
|
||||
* Most significant bits 8 - 11 of t0 become the least significant
|
||||
* nibble of the second byte. The least significant 4 bits
|
||||
* of t1 become the upper nibble of the second byte.
|
||||
*
|
||||
* The conversion to uint8_t does not alter the value.
|
||||
*/
|
||||
r[3 * i + 1] = (uint8_t)((t0 >> 8) | ((t1 << 4) & 0xF0));
|
||||
|
||||
/* Bits 4 - 11 of t1 become the third byte. The conversion to uint8_t
|
||||
* does not alter the value because t1 is 12-bit wide. */
|
||||
r[3 * i + 2] = (uint8_t)(t1 >> 4);
|
||||
}
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a)
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_TOBYTES)
|
||||
int ret;
|
||||
mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q);
|
||||
ret = mlk_poly_tobytes_native(r, a->coeffs);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_TOBYTES */
|
||||
|
||||
mlk_poly_tobytes_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: `poly_frombytes()` in the reference implementation @[REF]. */
|
||||
MLK_STATIC_TESTABLE void mlk_poly_frombytes_c(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYBYTES])
|
||||
__contract__(
|
||||
requires(memory_no_alias(a, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
)
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < MLKEM_N / 2; i++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 2)
|
||||
invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_UINT12_LIMIT))
|
||||
decreases(MLKEM_N / 2 - i))
|
||||
{
|
||||
const uint8_t t0 = a[3 * i + 0];
|
||||
const uint8_t t1 = a[3 * i + 1];
|
||||
const uint8_t t2 = a[3 * i + 2];
|
||||
/* Safety:
|
||||
* - The explicit cast to uint16_t ensures that << 8 does
|
||||
* not signed-overflow even on a 16-bit system.
|
||||
* - The cast to int16_t is safe due to the explicit 0xFFF truncation.
|
||||
*/
|
||||
r->coeffs[2 * i + 0] = (int16_t)(t0 | (((uint16_t)t1 << 8) & 0xFFF));
|
||||
r->coeffs[2 * i + 1] = (int16_t)((t1 >> 4) | (t2 << 4));
|
||||
}
|
||||
|
||||
/* Note that the coefficients are not canonical */
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_UINT12_LIMIT);
|
||||
}
|
||||
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES])
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_POLY_FROMBYTES)
|
||||
int ret;
|
||||
ret = mlk_poly_frombytes_native(r->coeffs, a);
|
||||
if (ret == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_NATIVE_POLY_FROMBYTES */
|
||||
|
||||
mlk_poly_frombytes_c(r, a);
|
||||
}
|
||||
|
||||
/* Reference: `poly_frommsg()` in the reference implementation @[REF].
|
||||
* - We use a value barrier around the bit-selection mask to
|
||||
* reduce the risk of compiler-introduced branches.
|
||||
* The reference implementation contains the expression
|
||||
* `(msg[i] >> j) & 1` which the compiler can reason must
|
||||
* be either 0 or 1. */
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES])
|
||||
{
|
||||
unsigned i;
|
||||
#if (MLKEM_INDCPA_MSGBYTES != MLKEM_N / 8)
|
||||
#error "MLKEM_INDCPA_MSGBYTES must be equal to MLKEM_N/8 bytes!"
|
||||
#endif
|
||||
|
||||
for (i = 0; i < MLKEM_N / 8; i++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 8)
|
||||
invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q))
|
||||
decreases(MLKEM_N / 8 - i))
|
||||
{
|
||||
unsigned j;
|
||||
for (j = 0; j < 8; j++)
|
||||
__loop__(
|
||||
invariant(i < MLKEM_N / 8 && j <= 8)
|
||||
invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q))
|
||||
decreases(8 - j))
|
||||
{
|
||||
/* mlk_ct_sel_int16(MLKEM_Q_HALF, 0, b) is `Decompress_1(b != 0)`
|
||||
* as per @[FIPS203, Eq (4.8)]. */
|
||||
|
||||
/* Prevent the compiler from recognizing this as a bit selection */
|
||||
uint8_t mask = mlk_value_barrier_u8((uint8_t)(1u << j));
|
||||
r->coeffs[8 * i + j] = mlk_ct_sel_int16(MLKEM_Q_HALF, 0, msg[i] & mask);
|
||||
}
|
||||
}
|
||||
mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q);
|
||||
}
|
||||
|
||||
/* Reference: `poly_tomsg()` in the reference implementation @[REF].
|
||||
* - In contrast to the reference implementation, we assume
|
||||
* unsigned canonical coefficients here.
|
||||
* The reference implementation works with coefficients
|
||||
* in the range [-(MLKEM_Q-1), MLKEM_Q-1].
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r)
|
||||
{
|
||||
unsigned i;
|
||||
mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q);
|
||||
|
||||
for (i = 0; i < MLKEM_N / 8; i++)
|
||||
__loop__(invariant(i <= MLKEM_N / 8)
|
||||
decreases(MLKEM_N / 8 - i))
|
||||
{
|
||||
unsigned j;
|
||||
msg[i] = 0;
|
||||
for (j = 0; j < 8; j++)
|
||||
__loop__(
|
||||
invariant(i <= MLKEM_N / 8 && j <= 8)
|
||||
decreases(8 - j))
|
||||
{
|
||||
uint32_t t = mlk_scalar_compress_d1(r->coeffs[8 * i + j]);
|
||||
msg[i] |= (uint8_t)(t << j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(compress)
|
||||
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
599
sdk/software/examples/mlkem_test/mlkem_native/src/compress.h
Normal file
599
sdk/software/examples/mlkem_test/mlkem_native/src/compress.h
Normal file
@@ -0,0 +1,599 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*
|
||||
* - [REF]
|
||||
* CRYSTALS-Kyber C reference implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/ref
|
||||
*/
|
||||
|
||||
#ifndef MLK_COMPRESS_H
|
||||
#define MLK_COMPRESS_H
|
||||
|
||||
|
||||
#include "cbmc.h"
|
||||
#include "common.h"
|
||||
#include "debug.h"
|
||||
#include "poly.h"
|
||||
#include "verify.h"
|
||||
|
||||
/*
|
||||
* The multiplication in this routine will exceed UINT32_MAX
|
||||
* and wrap around for large values of u. This is expected and required.
|
||||
*/
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check push
|
||||
#pragma CPROVER check disable "unsigned-overflow"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * 2 / MLKEM_Q).
|
||||
*
|
||||
* @spec{Compress_1 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @reference{Part of poly_tomsg() in the reference implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed.
|
||||
*
|
||||
* @return Compressed value.
|
||||
*/
|
||||
static MLK_INLINE uint8_t mlk_scalar_compress_d1(int16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u <= MLKEM_Q - 1)
|
||||
ensures(return_value < 2)
|
||||
ensures(return_value == (((uint32_t)u * 2 + MLKEM_Q / 2) / MLKEM_Q) % 2) )
|
||||
{
|
||||
/* Compute as follows:
|
||||
* ```
|
||||
* round(u * 2 / MLKEM_Q)
|
||||
* = round(u * 2 * (2^31 / MLKEM_Q) / 2^31)
|
||||
* ~= round(u * 2 * round(2^31 / MLKEM_Q) / 2^31)
|
||||
* ```
|
||||
*/
|
||||
/* check-magic: 1290168 == 2*round(2^31 / MLKEM_Q) */
|
||||
uint32_t d0 = (uint32_t)u * 1290168;
|
||||
/* Unsigned shifting by 31 positions leaves only the top bit. */
|
||||
return (uint8_t)((d0 + ((uint32_t)1u << 30)) >> 31);
|
||||
}
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check pop
|
||||
#endif
|
||||
|
||||
/*
|
||||
* The multiplication in this routine will exceed UINT32_MAX
|
||||
* and wrap around for large values of u. This is expected and required.
|
||||
*/
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check push
|
||||
#pragma CPROVER check disable "unsigned-overflow"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * 16 / MLKEM_Q) % 16.
|
||||
*
|
||||
* @spec{Compress_4 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @reference{Embedded into `poly_compress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed.
|
||||
*
|
||||
* @return Compressed value.
|
||||
*/
|
||||
static MLK_INLINE uint8_t mlk_scalar_compress_d4(int16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u <= MLKEM_Q - 1)
|
||||
ensures(return_value < 16)
|
||||
ensures(return_value == (((uint32_t)u * 16 + MLKEM_Q / 2) / MLKEM_Q) % 16))
|
||||
{
|
||||
/* Compute as follows:
|
||||
* ```
|
||||
* round(u * 16 / MLKEM_Q)
|
||||
* = round(u * 16 * (2^28 / MLKEM_Q) / 2^28)
|
||||
* ~= round(u * 16 * round(2^28 / MLKEM_Q) / 2^28)
|
||||
* ```
|
||||
*/
|
||||
/* check-magic: 1290160 == 16 * round(2^28 / MLKEM_Q) */
|
||||
uint32_t d0 = (uint32_t)u * 1290160;
|
||||
/* The return value is < 16, so not altered by the conversion to uint8_t. */
|
||||
return (uint8_t)((d0 + ((uint32_t)1u << 27)) >> 28); /* round(d0/2^28) */
|
||||
}
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check pop
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * MLKEM_Q / 16).
|
||||
*
|
||||
* @spec{Decompress_4 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* @reference{Embedded into `poly_decompress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo 16 to be decompressed.
|
||||
*
|
||||
* @return Decompressed value.
|
||||
*/
|
||||
static MLK_INLINE int16_t mlk_scalar_decompress_d4(uint8_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u < 16)
|
||||
ensures(return_value <= (MLKEM_Q - 1))
|
||||
)
|
||||
{
|
||||
/* The return value is in 0..MLKEM_Q-1, hence not altered by the
|
||||
* conversion to int16_t. */
|
||||
return (int16_t)((((uint32_t)u * MLKEM_Q) + 8) >> 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* The multiplication in this routine will exceed UINT32_MAX
|
||||
* and wrap around for large values of u. This is expected and required.
|
||||
*/
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check push
|
||||
#pragma CPROVER check disable "unsigned-overflow"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * 32 / MLKEM_Q) % 32.
|
||||
*
|
||||
* @spec{Compress_5 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @reference{Embedded into `poly_compress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed.
|
||||
*
|
||||
* @return Compressed value.
|
||||
*/
|
||||
static MLK_INLINE uint8_t mlk_scalar_compress_d5(int16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u <= MLKEM_Q - 1)
|
||||
ensures(return_value < 32)
|
||||
ensures(return_value == (((uint32_t)u * 32 + MLKEM_Q / 2) / MLKEM_Q) % 32) )
|
||||
{
|
||||
/* Compute as follows:
|
||||
* ```
|
||||
* round(u * 32 / MLKEM_Q)
|
||||
* = round(u * 32 * (2^27 / MLKEM_Q) / 2^27)
|
||||
* ~= round(u * 32 * round(2^27 / MLKEM_Q) / 2^27)
|
||||
* ```
|
||||
*/
|
||||
/* check-magic: 1290176 == 2^5 * round(2^27 / MLKEM_Q) */
|
||||
uint32_t d0 = (uint32_t)u * 1290176;
|
||||
/* The return value is < 32, so not altered by the conversion to uint8_t. */
|
||||
return (uint8_t)((d0 + ((uint32_t)1u << 26)) >> 27); /* round(d0/2^27) */
|
||||
}
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check pop
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * MLKEM_Q / 32).
|
||||
*
|
||||
* @spec{Decompress_5 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* @reference{Embedded into `poly_decompress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo 32 to be decompressed.
|
||||
*
|
||||
* @return Decompressed value.
|
||||
*/
|
||||
static MLK_INLINE int16_t mlk_scalar_decompress_d5(uint8_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u < 32)
|
||||
ensures(0 <= return_value && return_value <= MLKEM_Q - 1)
|
||||
)
|
||||
{
|
||||
/* The return value is in 0..MLKEM_Q-1, hence not altered by the
|
||||
* conversion to int16_t. */
|
||||
return (int16_t)((((uint32_t)u * MLKEM_Q) + 16) >> 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* The multiplication in this routine will exceed UINT32_MAX
|
||||
* and wrap around for large values of u. This is expected and required.
|
||||
*/
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check push
|
||||
#pragma CPROVER check disable "unsigned-overflow"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * 2**10 / MLKEM_Q) % 2**10.
|
||||
*
|
||||
* @spec{Compress_10 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @reference{Embedded into `polyvec_compress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed.
|
||||
*
|
||||
* @return Compressed value.
|
||||
*/
|
||||
static MLK_INLINE uint16_t mlk_scalar_compress_d10(int16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u <= MLKEM_Q - 1)
|
||||
ensures(return_value < (1u << 10))
|
||||
ensures(return_value == (((uint32_t)u * (1u << 10) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 10)))
|
||||
{
|
||||
/* Compute as follows:
|
||||
* ```
|
||||
* round(u * 1024 / MLKEM_Q)
|
||||
* = round(u * 1024 * (2^33 / MLKEM_Q) / 2^33)
|
||||
* ~= round(u * 1024 * round(2^33 / MLKEM_Q) / 2^33)
|
||||
* ```
|
||||
*/
|
||||
/* check-magic: 2642263040 == 2^10 * round(2^33 / MLKEM_Q) */
|
||||
uint64_t d0 = (uint64_t)u * 2642263040;
|
||||
d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */
|
||||
return (d0 & 0x3FF);
|
||||
}
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check pop
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * MLKEM_Q / 1024).
|
||||
*
|
||||
* @spec{Decompress_10 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* @reference{Embedded into `polyvec_decompress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo 1024 to be decompressed.
|
||||
*
|
||||
* @return Decompressed value.
|
||||
*/
|
||||
static MLK_INLINE int16_t mlk_scalar_decompress_d10(uint16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u < 1024)
|
||||
ensures(0 <= return_value && return_value <= (MLKEM_Q - 1))
|
||||
)
|
||||
{
|
||||
/* The return value is in 0..MLKEM_Q-1, hence not altered by the
|
||||
* conversion to int16_t. */
|
||||
return (int16_t)((((uint32_t)u * MLKEM_Q) + 512) >> 10);
|
||||
}
|
||||
|
||||
/*
|
||||
* The multiplication in this routine will exceed UINT32_MAX
|
||||
* and wrap around for large values of u. This is expected and required.
|
||||
*/
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check push
|
||||
#pragma CPROVER check disable "unsigned-overflow"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * 2**11 / MLKEM_Q) % 2**11.
|
||||
*
|
||||
* @spec{Compress_11 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @reference{Embedded into `polyvec_compress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed.
|
||||
*
|
||||
* @return Compressed value.
|
||||
*/
|
||||
static MLK_INLINE uint16_t mlk_scalar_compress_d11(int16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u <= MLKEM_Q - 1)
|
||||
ensures(return_value < (1u << 11))
|
||||
ensures(return_value == (((uint32_t)u * (1u << 11) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 11)))
|
||||
{
|
||||
/* Compute as follows:
|
||||
* ```
|
||||
* round(u * 2048 / MLKEM_Q)
|
||||
* = round(u * 2048 * (2^33 / MLKEM_Q) / 2^33)
|
||||
* ~= round(u * 2048 * round(2^33 / MLKEM_Q) / 2^33)
|
||||
* ```
|
||||
*/
|
||||
/* check-magic: 5284526080 == 2^11 * round(2^33 / MLKEM_Q) */
|
||||
uint64_t d0 = (uint64_t)u * 5284526080;
|
||||
d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */
|
||||
return (d0 & 0x7FF);
|
||||
}
|
||||
#ifdef CBMC
|
||||
#pragma CPROVER check pop
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute round(u * MLKEM_Q / 2048).
|
||||
*
|
||||
* @spec{Decompress_11 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* @reference{Embedded into `polyvec_decompress()` in the reference
|
||||
* implementation @[REF].}
|
||||
*
|
||||
* @param u Unsigned canonical modulus modulo 2048 to be decompressed.
|
||||
*
|
||||
* @return Decompressed value.
|
||||
*/
|
||||
static MLK_INLINE int16_t mlk_scalar_decompress_d11(uint16_t u)
|
||||
__contract__(
|
||||
requires(0 <= u && u < 2048)
|
||||
ensures(0 <= return_value && return_value <= (MLKEM_Q - 1))
|
||||
)
|
||||
{
|
||||
/* The return value is in 0..MLKEM_Q-1, hence not altered by the
|
||||
* conversion to int16_t. */
|
||||
return (int16_t)((((uint32_t)u * MLKEM_Q) + 1024) >> 11);
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3)
|
||||
#define mlk_poly_compress_d4 MLK_NAMESPACE(poly_compress_d4)
|
||||
/**
|
||||
* Compression (4 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Implements `ByteEncode_4 (Compress_4 (a))`: ByteEncode_d
|
||||
* @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm
|
||||
* 14 (K-PKE.Encrypt), L23], where `d_v=4` for ML-KEM-{512,768} @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4],
|
||||
const mlk_poly *a);
|
||||
|
||||
#define mlk_poly_compress_d10 MLK_NAMESPACE(poly_compress_d10)
|
||||
/**
|
||||
* Compression (10 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Implements `ByteEncode_10 (Compress_10 (a))`: ByteEncode_d
|
||||
* @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm
|
||||
* 14 (K-PKE.Encrypt), L22], where `d_u=10` for ML-KEM-{512,768} @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10],
|
||||
const mlk_poly *a);
|
||||
|
||||
#define mlk_poly_decompress_d4 MLK_NAMESPACE(poly_decompress_d4)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (4 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d4.
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @spec{Implements `Decompress_4 (ByteDecode_4 (a))`: ByteDecode_d
|
||||
* @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm
|
||||
* 15 (K-PKE.Decrypt), L4], where `d_v=4` for ML-KEM-{512,768} @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4
|
||||
* bytes).
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d4(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]);
|
||||
|
||||
#define mlk_poly_decompress_d10 MLK_NAMESPACE(poly_decompress_d10)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (10 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d10.
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @spec{Implements `Decompress_10 (ByteDecode_10 (a))`: ByteDecode_d
|
||||
* @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm
|
||||
* 15 (K-PKE.Decrypt), L3], where `d_u=10` for ML-KEM-{512,768} @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10
|
||||
* bytes).
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d10(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]);
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
#define mlk_poly_compress_d5 MLK_NAMESPACE(poly_compress_d5)
|
||||
/**
|
||||
* Compression (5 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Implements `ByteEncode_5 (Compress_5 (a))`: ByteEncode_d
|
||||
* @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm
|
||||
* 14 (K-PKE.Encrypt), L23], where `d_v=5` for ML-KEM-1024 @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5],
|
||||
const mlk_poly *a);
|
||||
|
||||
#define mlk_poly_compress_d11 MLK_NAMESPACE(poly_compress_d11)
|
||||
/**
|
||||
* Compression (11 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{`ByteEncode_11 (Compress_11 (a))`: ByteEncode_d @[FIPS203,
|
||||
* Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to vectors as
|
||||
* per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm
|
||||
* 14 (K-PKE.Encrypt), L22], where `d_u=11` for ML-KEM-1024 @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11],
|
||||
const mlk_poly *a);
|
||||
|
||||
#define mlk_poly_decompress_d5 MLK_NAMESPACE(poly_decompress_d5)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (5 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d5.
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @spec{Implements `Decompress_5 (ByteDecode_5 (a))`: ByteDecode_d
|
||||
* @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm
|
||||
* 15 (K-PKE.Decrypt), L4], where `d_v=5` for ML-KEM-1024 @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5
|
||||
* bytes).
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d5(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]);
|
||||
|
||||
#define mlk_poly_decompress_d11 MLK_NAMESPACE(poly_decompress_d11)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (11 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d11.
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @spec{Implements `Decompress_11 (ByteDecode_11 (a))`: ByteDecode_d
|
||||
* @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm
|
||||
* 15 (K-PKE.Decrypt), L3], where `d_u=11` for ML-KEM-1024 @[FIPS203,
|
||||
* Table 2].}
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11
|
||||
* bytes).
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_decompress_d11(mlk_poly *r,
|
||||
const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]);
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
#define mlk_poly_tobytes MLK_NAMESPACE(poly_tobytes)
|
||||
/**
|
||||
* Serialization of a polynomial. Signed coefficients are converted to
|
||||
* unsigned form before serialization.
|
||||
*
|
||||
* @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].}
|
||||
*
|
||||
* @param[out] r Output byte array (of MLKEM_POLYBYTES bytes).
|
||||
* @param[in] a Input polynomial, with each coefficient in the range
|
||||
* [0,1,..,MLKEM_Q-1].
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a)
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(a, sizeof(mlk_poly)))
|
||||
requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYBYTES))
|
||||
);
|
||||
|
||||
|
||||
#define mlk_poly_frombytes MLK_NAMESPACE(poly_frombytes)
|
||||
/**
|
||||
* De-serialization of a polynomial.
|
||||
*
|
||||
* @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].}
|
||||
*
|
||||
* @param[out] r Output polynomial, with each coefficient unsigned and in
|
||||
* the range 0..4095.
|
||||
* @param[in] a Input byte array (of MLKEM_POLYBYTES bytes).
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES])
|
||||
__contract__(
|
||||
requires(memory_no_alias(a, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
);
|
||||
|
||||
|
||||
#define mlk_poly_frommsg MLK_NAMESPACE(poly_frommsg)
|
||||
/**
|
||||
* Convert a 32-byte message to a polynomial.
|
||||
*
|
||||
* @spec{Implements `Decompress_1 (ByteDecode_1 (a))`: ByteDecode_d
|
||||
* @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `Decompress_1 (ByteDecode_1 (w))` appears in @[FIPS203, Algorithm 15
|
||||
* (K-PKE.Encrypt), L20].}
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] msg Input message.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES])
|
||||
__contract__(
|
||||
requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES))
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
assigns(memory_slice(r, sizeof(mlk_poly)))
|
||||
ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_tomsg MLK_NAMESPACE(poly_tomsg)
|
||||
/**
|
||||
* Convert a polynomial to a 32-byte message.
|
||||
*
|
||||
* @spec{Implements `ByteEncode_1 (Compress_1 (a))`: ByteEncode_d
|
||||
* @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to
|
||||
* vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].
|
||||
* `ByteEncode_1 (Compress_1 (w))` appears in @[FIPS203, Algorithm 14
|
||||
* (K-PKE.Decrypt), L7].}
|
||||
*
|
||||
* @param[out] msg Output message.
|
||||
* @param[in] r Input polynomial. Coefficients must be unsigned canonical.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r)
|
||||
__contract__(
|
||||
requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES))
|
||||
requires(memory_no_alias(r, sizeof(mlk_poly)))
|
||||
requires(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(msg, MLKEM_INDCPA_MSGBYTES))
|
||||
);
|
||||
|
||||
#endif /* !MLK_COMPRESS_H */
|
||||
64
sdk/software/examples/mlkem_test/mlkem_native/src/debug.c
Normal file
64
sdk/software/examples/mlkem_test/mlkem_native/src/debug.c
Normal file
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG)
|
||||
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "debug.h"
|
||||
|
||||
#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] "
|
||||
|
||||
void mlk_debug_check_assert(const char *file, int line, const int val)
|
||||
{
|
||||
if (val == 0)
|
||||
{
|
||||
fprintf(stderr, MLK_DEBUG_ERROR_HEADER "Assertion failed (value %d)\n",
|
||||
file, line, val);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr,
|
||||
unsigned len, int lower_bound_exclusive,
|
||||
int upper_bound_exclusive)
|
||||
{
|
||||
int err = 0;
|
||||
unsigned i;
|
||||
for (i = 0; i < len; i++)
|
||||
{
|
||||
int16_t val = ptr[i];
|
||||
if (!(val > lower_bound_exclusive && val < upper_bound_exclusive))
|
||||
{
|
||||
fprintf(
|
||||
stderr,
|
||||
MLK_DEBUG_ERROR_HEADER
|
||||
"Bounds assertion failed: Index %u, value %d out of bounds (%d,%d)\n",
|
||||
file, line, i, (int)val, lower_bound_exclusive,
|
||||
upper_bound_exclusive);
|
||||
err = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (err == 1)
|
||||
{
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG */
|
||||
|
||||
MLK_EMPTY_CU(debug)
|
||||
|
||||
#endif /* !(!MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG) */
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
#undef MLK_DEBUG_ERROR_HEADER
|
||||
121
sdk/software/examples/mlkem_test/mlkem_native/src/debug.h
Normal file
121
sdk/software/examples/mlkem_test/mlkem_native/src/debug.h
Normal file
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_DEBUG_H
|
||||
#define MLK_DEBUG_H
|
||||
#include "common.h"
|
||||
|
||||
#if defined(MLKEM_DEBUG)
|
||||
|
||||
/**
|
||||
* Check debug assertion.
|
||||
*
|
||||
* Prints an error message to stderr and calls exit(1) on failure.
|
||||
*
|
||||
* @param[in] file Filename.
|
||||
* @param line Line number.
|
||||
* @param val Value asserted to be non-zero.
|
||||
*/
|
||||
#define mlk_debug_check_assert MLK_NAMESPACE(mlkem_debug_assert)
|
||||
void mlk_debug_check_assert(const char *file, int line, const int val);
|
||||
|
||||
/**
|
||||
* Check whether values in an array of int16_t are within specified bounds.
|
||||
*
|
||||
* Prints an error message to stderr and calls exit(1) on failure.
|
||||
*
|
||||
* @param[in] file Filename.
|
||||
* @param line Line number.
|
||||
* @param[in] ptr Base of array to be checked.
|
||||
* @param len Number of int16_t in @p ptr.
|
||||
* @param lower_bound_exclusive Exclusive lower bound.
|
||||
* @param upper_bound_exclusive Exclusive upper bound.
|
||||
*/
|
||||
#define mlk_debug_check_bounds MLK_NAMESPACE(mlkem_debug_check_bounds)
|
||||
void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr,
|
||||
unsigned len, int lower_bound_exclusive,
|
||||
int upper_bound_exclusive);
|
||||
|
||||
/* Check assertion, calling exit() upon failure
|
||||
*
|
||||
* val: Value that's asserted to be non-zero
|
||||
*/
|
||||
#define mlk_assert(val) mlk_debug_check_assert(__FILE__, __LINE__, (val))
|
||||
|
||||
/* Check bounds in array of int16_t's
|
||||
* ptr: Base of int16_t array; will be explicitly cast to int16_t*,
|
||||
* so you may pass a byte-compatible type such as mlk_poly or mlk_polyvec.
|
||||
* len: Number of int16_t in array
|
||||
* value_lb: Inclusive lower value bound
|
||||
* value_ub: Exclusive upper value bound */
|
||||
#define mlk_assert_bound(ptr, len, value_lb, value_ub) \
|
||||
mlk_debug_check_bounds(__FILE__, __LINE__, (const int16_t *)(ptr), (len), \
|
||||
(value_lb) - 1, (value_ub))
|
||||
|
||||
/* Check absolute bounds in array of int16_t's
|
||||
* ptr: Base of array, expression of type int16_t*
|
||||
* len: Number of int16_t in array
|
||||
* value_abs_bd: Exclusive absolute upper bound */
|
||||
#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \
|
||||
mlk_assert_bound((ptr), (len), (-(value_abs_bd) + 1), (value_abs_bd))
|
||||
|
||||
/* Version of bounds assertions for 2-dimensional arrays */
|
||||
#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \
|
||||
mlk_assert_bound((ptr), ((len0) * (len1)), (value_lb), (value_ub))
|
||||
|
||||
#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \
|
||||
mlk_assert_abs_bound((ptr), ((len0) * (len1)), (value_abs_bd))
|
||||
|
||||
/* When running CBMC, convert debug assertions into proof obligations */
|
||||
#elif defined(CBMC)
|
||||
#include "cbmc.h"
|
||||
|
||||
#define mlk_assert(val) cassert(val)
|
||||
|
||||
#define mlk_assert_bound(ptr, len, value_lb, value_ub) \
|
||||
cassert(array_bound(((int16_t *)(ptr)), 0, (len), (value_lb), (value_ub)))
|
||||
|
||||
#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \
|
||||
cassert(array_abs_bound(((int16_t *)(ptr)), 0, (len), (value_abs_bd)))
|
||||
|
||||
/* Because of https://github.com/diffblue/cbmc/issues/8570, we can't
|
||||
* just use a single flattened array_bound(...) here. */
|
||||
#define mlk_assert_bound_2d(ptr, M, N, value_lb, value_ub) \
|
||||
cassert(forall(kN, 0, (M), \
|
||||
array_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \
|
||||
(value_lb), (value_ub))))
|
||||
|
||||
#define mlk_assert_abs_bound_2d(ptr, M, N, value_abs_bd) \
|
||||
cassert(forall(kN, 0, (M), \
|
||||
array_abs_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \
|
||||
(value_abs_bd))))
|
||||
|
||||
#else /* !MLKEM_DEBUG && CBMC */
|
||||
|
||||
#define mlk_assert(val) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
#define mlk_assert_bound(ptr, len, value_lb, value_ub) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
|
||||
#endif /* !MLKEM_DEBUG && !CBMC */
|
||||
#endif /* !MLK_DEBUG_H */
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*
|
||||
* - [mupq]
|
||||
* Common files for pqm4, pqm3, pqriscv
|
||||
* Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen
|
||||
* https://github.com/mupq/mupq
|
||||
*
|
||||
* - [supercop]
|
||||
* SUPERCOP benchmarking framework
|
||||
* Daniel J. Bernstein
|
||||
* http://bench.cr.yp.to/supercop.html
|
||||
*
|
||||
* - [tweetfips]
|
||||
* 'tweetfips202' FIPS202 implementation
|
||||
* Van Assche, Bernstein, Schwabe
|
||||
* https://keccak.team/2015/tweetfips202.html
|
||||
*/
|
||||
|
||||
/* Based on the CC0 implementation from @[mupq] and the public domain
|
||||
* implementation @[supercop, crypto_hash/keccakc512/simple/]
|
||||
* by Ronny Van Keer, and the public domain @[tweetfips] implementation. */
|
||||
|
||||
#include "../common.h"
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
|
||||
#include "../verify.h"
|
||||
#include "fips202.h"
|
||||
#include "keccakf1600.h"
|
||||
|
||||
/**
|
||||
* Absorb step of Keccak; non-incremental, starts by zeroeing the state.
|
||||
*
|
||||
* @warning Must only be called once.
|
||||
*
|
||||
* @param[out] s Pointer to (uninitialized) output Keccak state.
|
||||
* @param r Rate in bytes (e.g., 168 for SHAKE128).
|
||||
* @param[in] m Input to be absorbed into @p s.
|
||||
* @param mlen Length of input in bytes.
|
||||
* @param p Domain-separation byte for different Keccak-derived
|
||||
* functions.
|
||||
*/
|
||||
static void mlk_keccak_absorb_once(uint64_t *s, unsigned r, const uint8_t *m,
|
||||
size_t mlen, uint8_t p)
|
||||
__contract__(
|
||||
requires(mlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(r > 0)
|
||||
requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES)
|
||||
requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
requires(memory_no_alias(m, mlen))
|
||||
assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)))
|
||||
{
|
||||
/* Initialize state */
|
||||
size_t i;
|
||||
for (i = 0; i < 25; ++i)
|
||||
__loop__(invariant(i <= 25)
|
||||
decreases(25 - i))
|
||||
{
|
||||
s[i] = 0;
|
||||
}
|
||||
|
||||
while (mlen >= r)
|
||||
__loop__(
|
||||
assigns(mlen, m, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
invariant(mlen <= loop_entry(mlen))
|
||||
invariant(m == loop_entry(m) + (loop_entry(mlen) - mlen))
|
||||
decreases(mlen))
|
||||
{
|
||||
mlk_keccakf1600_xor_bytes(s, m, 0, r);
|
||||
mlk_keccakf1600_permute(s);
|
||||
mlen -= r;
|
||||
m += r;
|
||||
}
|
||||
|
||||
/* At this point, mlen < r, so the truncations to unsigned are safe below. */
|
||||
|
||||
if (mlen > 0)
|
||||
{
|
||||
mlk_keccakf1600_xor_bytes(s, m, 0, (unsigned int)mlen);
|
||||
}
|
||||
|
||||
if (mlen == r - 1)
|
||||
{
|
||||
p |= 128;
|
||||
mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1);
|
||||
p = 128;
|
||||
mlk_keccakf1600_xor_bytes(s, &p, r - 1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Block-level Keccak squeeze.
|
||||
*
|
||||
* @param[out] h Output bytes.
|
||||
* @param nblocks Number of blocks to be squeezed.
|
||||
* @param[in,out] s Input/output state.
|
||||
* @param r Rate in bytes (e.g., 168 for SHAKE128).
|
||||
*/
|
||||
static void mlk_keccak_squeezeblocks(uint8_t *h, size_t nblocks, uint64_t *s,
|
||||
unsigned r)
|
||||
__contract__(
|
||||
requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES)
|
||||
requires(nblocks <= 8 /* somewhat arbitrary bound */)
|
||||
requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
requires(memory_no_alias(h, nblocks * r))
|
||||
assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
assigns(memory_slice(h, nblocks * r)))
|
||||
{
|
||||
while (nblocks > 0)
|
||||
__loop__(
|
||||
assigns(h, nblocks,
|
||||
memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES),
|
||||
memory_slice(h, nblocks * r))
|
||||
invariant(nblocks <= loop_entry(nblocks) &&
|
||||
h == loop_entry(h) + r * (loop_entry(nblocks) - nblocks))
|
||||
decreases(nblocks))
|
||||
{
|
||||
mlk_keccakf1600_permute(s);
|
||||
mlk_keccakf1600_extract_bytes(s, h, 0, r);
|
||||
h += r;
|
||||
nblocks--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keccak squeeze; can be called on byte-level.
|
||||
*
|
||||
* @warning Must only be called once.
|
||||
*
|
||||
* @param[out] h Output bytes.
|
||||
* @param outlen Number of bytes to be squeezed.
|
||||
* @param[in,out] s Keccak state.
|
||||
* @param r Rate in bytes (e.g., 168 for SHAKE128).
|
||||
*/
|
||||
static void mlk_keccak_squeeze_once(uint8_t *h, size_t outlen, uint64_t *s,
|
||||
unsigned r)
|
||||
__contract__(
|
||||
requires(outlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(r > 0)
|
||||
requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES)
|
||||
requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
requires(memory_no_alias(h, outlen))
|
||||
assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
assigns(memory_slice(h, outlen)))
|
||||
{
|
||||
size_t len;
|
||||
while (outlen > 0)
|
||||
__loop__(
|
||||
assigns(len, h, outlen,
|
||||
memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES),
|
||||
memory_slice(h, outlen))
|
||||
invariant(outlen <= loop_entry(outlen) &&
|
||||
h == loop_entry(h) + (loop_entry(outlen) - outlen))
|
||||
decreases(outlen))
|
||||
{
|
||||
mlk_keccakf1600_permute(s);
|
||||
|
||||
if (outlen < r)
|
||||
{
|
||||
len = outlen;
|
||||
}
|
||||
else
|
||||
{
|
||||
len = r;
|
||||
}
|
||||
mlk_keccakf1600_extract_bytes(s, h, 0, (unsigned int)len);
|
||||
h += len;
|
||||
outlen -= len;
|
||||
}
|
||||
}
|
||||
|
||||
void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input,
|
||||
size_t inlen)
|
||||
{
|
||||
mlk_keccak_absorb_once(state->ctx, SHAKE128_RATE, input, inlen, 0x1F);
|
||||
}
|
||||
|
||||
void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks,
|
||||
mlk_shake128ctx *state)
|
||||
{
|
||||
mlk_keccak_squeezeblocks(output, nblocks, state->ctx, SHAKE128_RATE);
|
||||
}
|
||||
|
||||
void mlk_shake128_init(mlk_shake128ctx *state) { (void)state; }
|
||||
void mlk_shake128_release(mlk_shake128ctx *state)
|
||||
{
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(state, sizeof(mlk_shake128ctx));
|
||||
}
|
||||
|
||||
typedef mlk_shake128ctx mlk_shake256ctx;
|
||||
void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input,
|
||||
size_t inlen)
|
||||
{
|
||||
mlk_shake256ctx state;
|
||||
/* Absorb input */
|
||||
mlk_keccak_absorb_once(state.ctx, SHAKE256_RATE, input, inlen, 0x1F);
|
||||
/* Squeeze output */
|
||||
mlk_keccak_squeeze_once(output, outlen, state.ctx, SHAKE256_RATE);
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(&state, sizeof(state));
|
||||
}
|
||||
|
||||
void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen)
|
||||
{
|
||||
uint64_t ctx[25];
|
||||
/* Absorb input */
|
||||
mlk_keccak_absorb_once(ctx, SHA3_256_RATE, input, inlen, 0x06);
|
||||
/* Squeeze output */
|
||||
mlk_keccak_squeeze_once(output, 32, ctx, SHA3_256_RATE);
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(ctx, sizeof(ctx));
|
||||
}
|
||||
|
||||
void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen)
|
||||
{
|
||||
uint64_t ctx[25];
|
||||
/* Absorb input */
|
||||
mlk_keccak_absorb_once(ctx, SHA3_512_RATE, input, inlen, 0x06);
|
||||
/* Squeeze output */
|
||||
mlk_keccak_squeeze_once(output, 64, ctx, SHA3_512_RATE);
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(ctx, sizeof(ctx));
|
||||
}
|
||||
|
||||
#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(fips202)
|
||||
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_FIPS202_FIPS202_H
|
||||
#define MLK_FIPS202_FIPS202_H
|
||||
|
||||
#include "../cbmc.h"
|
||||
#include "../common.h"
|
||||
|
||||
#define SHAKE128_RATE 168
|
||||
#define SHAKE256_RATE 136
|
||||
#define SHA3_256_RATE 136
|
||||
#define SHA3_384_RATE 104
|
||||
#define SHA3_512_RATE 72
|
||||
|
||||
/** Context for the non-incremental SHAKE128 API. */
|
||||
typedef struct
|
||||
{
|
||||
uint64_t ctx[25]; /**< Keccak state. */
|
||||
} MLK_ALIGN mlk_shake128ctx;
|
||||
|
||||
#define mlk_shake128_absorb_once MLK_NAMESPACE(shake128_absorb_once)
|
||||
/**
|
||||
* One-shot absorb step of the SHAKE128 XOF.
|
||||
*
|
||||
* For call-sites (in mlkem-native):
|
||||
* - This function MUST ONLY be called straight after mlk_shake128_init().
|
||||
* - This function MUST ONLY be called once.
|
||||
*
|
||||
* Consequently, for providers of custom FIPS202 code to be used with
|
||||
* mlkem-native:
|
||||
* - You may assume that the input context is freshly initialized via
|
||||
* mlk_shake128_init().
|
||||
* - You may assume that this function is called exactly once.
|
||||
*
|
||||
* @param[in,out] state SHAKE128 context.
|
||||
* @param[in] input Input to be absorbed into the state.
|
||||
* @param inlen Length of input in bytes.
|
||||
*/
|
||||
void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input,
|
||||
size_t inlen)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(state, sizeof(mlk_shake128ctx)))
|
||||
requires(memory_no_alias(input, inlen))
|
||||
assigns(memory_slice(state, sizeof(mlk_shake128ctx)))
|
||||
);
|
||||
|
||||
#define mlk_shake128_squeezeblocks MLK_NAMESPACE(shake128_squeezeblocks)
|
||||
/**
|
||||
* Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes
|
||||
* each. Modifies the state. Can be called multiple times to keep squeezing,
|
||||
* i.e., is incremental.
|
||||
*
|
||||
* @param[out] output Output blocks.
|
||||
* @param nblocks Number of blocks to be squeezed (written to output).
|
||||
* @param[in,out] state Keccak state.
|
||||
*/
|
||||
void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks,
|
||||
mlk_shake128ctx *state)
|
||||
__contract__(
|
||||
requires(nblocks <= 8 /* somewhat arbitrary bound */)
|
||||
requires(memory_no_alias(state, sizeof(mlk_shake128ctx)))
|
||||
requires(memory_no_alias(output, nblocks * SHAKE128_RATE))
|
||||
assigns(memory_slice(output, nblocks * SHAKE128_RATE), memory_slice(state, sizeof(mlk_shake128ctx)))
|
||||
);
|
||||
|
||||
#define mlk_shake128_init MLK_NAMESPACE(shake128_init)
|
||||
void mlk_shake128_init(mlk_shake128ctx *state);
|
||||
|
||||
#define mlk_shake128_release MLK_NAMESPACE(shake128_release)
|
||||
void mlk_shake128_release(mlk_shake128ctx *state);
|
||||
|
||||
/* One-stop SHAKE256 call. Aliasing between input and
|
||||
* output is not permitted */
|
||||
#define mlk_shake256 MLK_NAMESPACE(shake256)
|
||||
/**
|
||||
* SHAKE256 XOF with non-incremental API.
|
||||
*
|
||||
* @param[out] output Output buffer.
|
||||
* @param outlen Requested output length in bytes.
|
||||
* @param[in] input Input buffer.
|
||||
* @param inlen Length of input in bytes.
|
||||
*/
|
||||
void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input,
|
||||
size_t inlen)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(outlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(input, inlen))
|
||||
requires(memory_no_alias(output, outlen))
|
||||
assigns(memory_slice(output, outlen))
|
||||
);
|
||||
|
||||
/* One-stop SHA3_256 call. Aliasing between input and
|
||||
* output is not permitted */
|
||||
#define SHA3_256_HASHBYTES 32
|
||||
#define mlk_sha3_256 MLK_NAMESPACE(sha3_256)
|
||||
/**
|
||||
* SHA3-256 with non-incremental API.
|
||||
*
|
||||
* @param[out] output Output buffer.
|
||||
* @param[in] input Input buffer.
|
||||
* @param inlen Length of input in bytes.
|
||||
*/
|
||||
void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(input, inlen))
|
||||
requires(memory_no_alias(output, SHA3_256_HASHBYTES))
|
||||
assigns(memory_slice(output, SHA3_256_HASHBYTES))
|
||||
);
|
||||
|
||||
/* One-stop SHA3_512 call. Aliasing between input and
|
||||
* output is not permitted */
|
||||
#define SHA3_512_HASHBYTES 64
|
||||
#define mlk_sha3_512 MLK_NAMESPACE(sha3_512)
|
||||
/**
|
||||
* SHA3-512 with non-incremental API.
|
||||
*
|
||||
* @param[out] output Output buffer.
|
||||
* @param[in] input Input buffer.
|
||||
* @param inlen Length of input in bytes.
|
||||
*/
|
||||
void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(input, inlen))
|
||||
requires(memory_no_alias(output, SHA3_512_HASHBYTES))
|
||||
assigns(memory_slice(output, SHA3_512_HASHBYTES))
|
||||
);
|
||||
|
||||
#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) || \
|
||||
!defined(MLK_USE_FIPS202_X4_NATIVE)
|
||||
/* If you provide your own FIPS-202 implementation where the x4-
|
||||
* Keccak-f1600-x4 implementation falls back to 4-fold Keccak-f1600,
|
||||
* set this to gain a small speedup. */
|
||||
#define FIPS202_X4_DEFAULT_IMPLEMENTATION
|
||||
#endif /* !MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 || !MLK_USE_FIPS202_X4_NATIVE \
|
||||
*/
|
||||
|
||||
|
||||
#endif /* !MLK_FIPS202_FIPS202_H */
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*/
|
||||
|
||||
#include "../common.h"
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "../verify.h"
|
||||
#include "fips202.h"
|
||||
#include "fips202x4.h"
|
||||
#include "keccakf1600.h"
|
||||
|
||||
typedef mlk_shake128x4ctx mlk_shake256x4_ctx;
|
||||
|
||||
static void mlk_keccak_absorb_once_x4(uint64_t *s, unsigned r,
|
||||
const uint8_t *in0, const uint8_t *in1,
|
||||
const uint8_t *in2, const uint8_t *in3,
|
||||
size_t inlen, uint8_t p)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
requires(r > 0)
|
||||
requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES)
|
||||
requires(memory_no_alias(in0, inlen))
|
||||
requires(memory_no_alias(in1, inlen))
|
||||
requires(memory_no_alias(in2, inlen))
|
||||
requires(memory_no_alias(in3, inlen))
|
||||
assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)))
|
||||
{
|
||||
while (inlen >= r)
|
||||
__loop__(
|
||||
assigns(inlen, in0, in1, in2, in3, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
invariant(inlen <= loop_entry(inlen))
|
||||
invariant(in0 == loop_entry(in0) + (loop_entry(inlen) - inlen))
|
||||
invariant(in1 == loop_entry(in1) + (loop_entry(inlen) - inlen))
|
||||
invariant(in2 == loop_entry(in2) + (loop_entry(inlen) - inlen))
|
||||
invariant(in3 == loop_entry(in3) + (loop_entry(inlen) - inlen))
|
||||
decreases(inlen))
|
||||
{
|
||||
mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, r);
|
||||
mlk_keccakf1600x4_permute(s);
|
||||
|
||||
in0 += r;
|
||||
in1 += r;
|
||||
in2 += r;
|
||||
in3 += r;
|
||||
inlen -= r;
|
||||
}
|
||||
|
||||
/* At this point, inlen < r, so the truncations to unsigned are safe below. */
|
||||
|
||||
if (inlen > 0)
|
||||
{
|
||||
mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, (unsigned int)inlen);
|
||||
}
|
||||
|
||||
if (inlen == r - 1)
|
||||
{
|
||||
p |= 128;
|
||||
mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1);
|
||||
p = 128;
|
||||
mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, r - 1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
static void mlk_keccak_squeezeblocks_x4(uint8_t *out0, uint8_t *out1,
|
||||
uint8_t *out2, uint8_t *out3,
|
||||
size_t nblocks, uint64_t *s, unsigned r)
|
||||
__contract__(
|
||||
requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES)
|
||||
requires(r == SHAKE128_RATE || r == SHAKE256_RATE)
|
||||
requires(nblocks <= (MLK_MAX_BUFFER_SIZE / SHAKE256_RATE))
|
||||
requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
requires(memory_no_alias(out0, nblocks * r))
|
||||
requires(memory_no_alias(out1, nblocks * r))
|
||||
requires(memory_no_alias(out2, nblocks * r))
|
||||
requires(memory_no_alias(out3, nblocks * r))
|
||||
assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
assigns(memory_slice(out0, nblocks * r))
|
||||
assigns(memory_slice(out1, nblocks * r))
|
||||
assigns(memory_slice(out2, nblocks * r))
|
||||
assigns(memory_slice(out3, nblocks * r)))
|
||||
{
|
||||
size_t current_offset = 0;
|
||||
while (nblocks > 0)
|
||||
__loop__(
|
||||
assigns(nblocks, current_offset,
|
||||
memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY),
|
||||
memory_slice(out0, nblocks * r),
|
||||
memory_slice(out1, nblocks * r),
|
||||
memory_slice(out2, nblocks * r),
|
||||
memory_slice(out3, nblocks * r))
|
||||
invariant(nblocks <= loop_entry(nblocks))
|
||||
invariant(current_offset == (loop_entry(nblocks) - nblocks) * r)
|
||||
decreases(nblocks))
|
||||
{
|
||||
mlk_keccakf1600x4_permute(s);
|
||||
mlk_keccakf1600x4_extract_bytes(
|
||||
s, &out0[current_offset], &out1[current_offset], &out2[current_offset],
|
||||
&out3[current_offset], 0, r);
|
||||
current_offset += r;
|
||||
nblocks--;
|
||||
}
|
||||
}
|
||||
|
||||
void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0,
|
||||
const uint8_t *in1, const uint8_t *in2,
|
||||
const uint8_t *in3, size_t inlen)
|
||||
{
|
||||
mlk_memset(state, 0, sizeof(mlk_shake128x4ctx));
|
||||
mlk_keccak_absorb_once_x4(state->ctx, SHAKE128_RATE, in0, in1, in2, in3,
|
||||
inlen, 0x1F);
|
||||
}
|
||||
|
||||
void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2,
|
||||
uint8_t *out3, size_t nblocks,
|
||||
mlk_shake128x4ctx *state)
|
||||
{
|
||||
mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx,
|
||||
SHAKE128_RATE);
|
||||
}
|
||||
|
||||
void mlk_shake128x4_init(mlk_shake128x4ctx *state) { (void)state; }
|
||||
void mlk_shake128x4_release(mlk_shake128x4ctx *state)
|
||||
{
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(state, sizeof(mlk_shake128x4ctx));
|
||||
}
|
||||
|
||||
static void mlk_shake256x4_absorb_once(mlk_shake256x4_ctx *state,
|
||||
const uint8_t *in0, const uint8_t *in1,
|
||||
const uint8_t *in2, const uint8_t *in3,
|
||||
size_t inlen)
|
||||
{
|
||||
mlk_memset(state, 0, sizeof(mlk_shake128x4ctx));
|
||||
mlk_keccak_absorb_once_x4(state->ctx, SHAKE256_RATE, in0, in1, in2, in3,
|
||||
inlen, 0x1F);
|
||||
}
|
||||
|
||||
static void mlk_shake256x4_squeezeblocks(uint8_t *out0, uint8_t *out1,
|
||||
uint8_t *out2, uint8_t *out3,
|
||||
size_t nblocks,
|
||||
mlk_shake256x4_ctx *state)
|
||||
{
|
||||
mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx,
|
||||
SHAKE256_RATE);
|
||||
}
|
||||
|
||||
void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3,
|
||||
size_t outlen, const uint8_t *in0, const uint8_t *in1,
|
||||
const uint8_t *in2, const uint8_t *in3, size_t inlen)
|
||||
{
|
||||
mlk_shake256x4_ctx statex;
|
||||
size_t nblocks = outlen / SHAKE256_RATE;
|
||||
uint8_t tmp0[SHAKE256_RATE];
|
||||
uint8_t tmp1[SHAKE256_RATE];
|
||||
uint8_t tmp2[SHAKE256_RATE];
|
||||
uint8_t tmp3[SHAKE256_RATE];
|
||||
|
||||
mlk_shake256x4_absorb_once(&statex, in0, in1, in2, in3, inlen);
|
||||
mlk_shake256x4_squeezeblocks(out0, out1, out2, out3, nblocks, &statex);
|
||||
|
||||
out0 += nblocks * SHAKE256_RATE;
|
||||
out1 += nblocks * SHAKE256_RATE;
|
||||
out2 += nblocks * SHAKE256_RATE;
|
||||
out3 += nblocks * SHAKE256_RATE;
|
||||
|
||||
outlen -= nblocks * SHAKE256_RATE;
|
||||
|
||||
if (outlen)
|
||||
{
|
||||
mlk_shake256x4_squeezeblocks(tmp0, tmp1, tmp2, tmp3, 1, &statex);
|
||||
mlk_memcpy(out0, tmp0, outlen);
|
||||
mlk_memcpy(out1, tmp1, outlen);
|
||||
mlk_memcpy(out2, tmp2, outlen);
|
||||
mlk_memcpy(out3, tmp3, outlen);
|
||||
}
|
||||
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(&statex, sizeof(statex));
|
||||
mlk_zeroize(tmp0, sizeof(tmp0));
|
||||
mlk_zeroize(tmp1, sizeof(tmp1));
|
||||
mlk_zeroize(tmp2, sizeof(tmp2));
|
||||
mlk_zeroize(tmp3, sizeof(tmp3));
|
||||
}
|
||||
|
||||
#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(fips202x4)
|
||||
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_FIPS202_FIPS202X4_H
|
||||
#define MLK_FIPS202_FIPS202X4_H
|
||||
|
||||
|
||||
#include "../cbmc.h"
|
||||
#include "../common.h"
|
||||
|
||||
#include "fips202.h"
|
||||
#include "keccakf1600.h"
|
||||
|
||||
/** Context for the non-incremental 4-way SHAKE128 API. */
|
||||
typedef struct
|
||||
{
|
||||
uint64_t ctx[MLK_KECCAK_LANES *
|
||||
MLK_KECCAK_WAY]; /**< 4-way Keccak state, stored sequentially. */
|
||||
} MLK_ALIGN mlk_shake128x4ctx;
|
||||
|
||||
#define mlk_shake128x4_absorb_once MLK_NAMESPACE(shake128x4_absorb_once)
|
||||
void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0,
|
||||
const uint8_t *in1, const uint8_t *in2,
|
||||
const uint8_t *in3, size_t inlen)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx)))
|
||||
requires(memory_no_alias(in0, inlen))
|
||||
requires(memory_no_alias(in1, inlen))
|
||||
requires(memory_no_alias(in2, inlen))
|
||||
requires(memory_no_alias(in3, inlen))
|
||||
assigns(memory_slice(state, sizeof(mlk_shake128x4ctx)))
|
||||
);
|
||||
|
||||
#define mlk_shake128x4_squeezeblocks MLK_NAMESPACE(shake128x4_squeezeblocks)
|
||||
void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2,
|
||||
uint8_t *out3, size_t nblocks,
|
||||
mlk_shake128x4ctx *state)
|
||||
__contract__(
|
||||
requires(nblocks <= 8 /* somewhat arbitrary bound */)
|
||||
requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx)))
|
||||
requires(memory_no_alias(out0, nblocks * SHAKE128_RATE))
|
||||
requires(memory_no_alias(out1, nblocks * SHAKE128_RATE))
|
||||
requires(memory_no_alias(out2, nblocks * SHAKE128_RATE))
|
||||
requires(memory_no_alias(out3, nblocks * SHAKE128_RATE))
|
||||
assigns(memory_slice(out0, nblocks * SHAKE128_RATE),
|
||||
memory_slice(out1, nblocks * SHAKE128_RATE),
|
||||
memory_slice(out2, nblocks * SHAKE128_RATE),
|
||||
memory_slice(out3, nblocks * SHAKE128_RATE),
|
||||
memory_slice(state, sizeof(mlk_shake128x4ctx)))
|
||||
);
|
||||
|
||||
#define mlk_shake128x4_init MLK_NAMESPACE(shake128x4_init)
|
||||
void mlk_shake128x4_init(mlk_shake128x4ctx *state);
|
||||
|
||||
#define mlk_shake128x4_release MLK_NAMESPACE(shake128x4_release)
|
||||
void mlk_shake128x4_release(mlk_shake128x4ctx *state);
|
||||
|
||||
#define mlk_shake256x4 MLK_NAMESPACE(shake256x4)
|
||||
void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3,
|
||||
size_t outlen, const uint8_t *in0, const uint8_t *in1,
|
||||
const uint8_t *in2, const uint8_t *in3, size_t inlen)
|
||||
__contract__(
|
||||
requires(inlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(outlen <= MLK_MAX_BUFFER_SIZE)
|
||||
requires(memory_no_alias(in0, inlen))
|
||||
requires(memory_no_alias(in1, inlen))
|
||||
requires(memory_no_alias(in2, inlen))
|
||||
requires(memory_no_alias(in3, inlen))
|
||||
requires(memory_no_alias(out0, outlen))
|
||||
requires(memory_no_alias(out1, outlen))
|
||||
requires(memory_no_alias(out2, outlen))
|
||||
requires(memory_no_alias(out3, outlen))
|
||||
assigns(memory_slice(out0, outlen))
|
||||
assigns(memory_slice(out1, outlen))
|
||||
assigns(memory_slice(out2, outlen))
|
||||
assigns(memory_slice(out3, outlen))
|
||||
);
|
||||
|
||||
#endif /* !MLK_FIPS202_FIPS202X4_H */
|
||||
@@ -0,0 +1,499 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [mupq]
|
||||
* Common files for pqm4, pqm3, pqriscv
|
||||
* Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen
|
||||
* https://github.com/mupq/mupq
|
||||
*
|
||||
* - [supercop]
|
||||
* SUPERCOP benchmarking framework
|
||||
* Daniel J. Bernstein
|
||||
* http://bench.cr.yp.to/supercop.html
|
||||
*
|
||||
* - [tweetfips]
|
||||
* 'tweetfips202' FIPS202 implementation
|
||||
* Van Assche, Bernstein, Schwabe
|
||||
* https://keccak.team/2015/tweetfips202.html
|
||||
*/
|
||||
|
||||
/* Based on the CC0 implementation from @[mupq] and the public domain
|
||||
* implementation @[supercop, crypto_hash/keccakc512/simple/]
|
||||
* by Ronny Van Keer, and the public domain @[tweetfips] implementation. */
|
||||
|
||||
|
||||
#include "keccakf1600.h"
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#define MLK_KECCAK_NROUNDS 24
|
||||
#define MLK_KECCAK_ROL(a, offset) (((a) << (offset)) ^ ((a) >> (64 - (offset))))
|
||||
|
||||
void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data,
|
||||
unsigned offset, unsigned length)
|
||||
{
|
||||
unsigned i;
|
||||
#if defined(MLK_SYS_LITTLE_ENDIAN)
|
||||
uint8_t *state_ptr = (uint8_t *)state + offset;
|
||||
for (i = 0; i < length; i++)
|
||||
__loop__(invariant(i <= length)
|
||||
decreases(length - i))
|
||||
{
|
||||
data[i] = state_ptr[i];
|
||||
}
|
||||
#else /* MLK_SYS_LITTLE_ENDIAN */
|
||||
/* Portable version */
|
||||
for (i = 0; i < length; i++)
|
||||
__loop__(invariant(i <= length)
|
||||
decreases(length - i))
|
||||
{
|
||||
data[i] = (state[(offset + i) >> 3] >> (8 * ((offset + i) & 0x07))) & 0xFF;
|
||||
}
|
||||
#endif /* !MLK_SYS_LITTLE_ENDIAN */
|
||||
}
|
||||
|
||||
void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data,
|
||||
unsigned offset, unsigned length)
|
||||
{
|
||||
unsigned i;
|
||||
#if defined(MLK_SYS_LITTLE_ENDIAN)
|
||||
uint8_t *state_ptr = (uint8_t *)state + offset;
|
||||
for (i = 0; i < length; i++)
|
||||
__loop__(invariant(i <= length)
|
||||
decreases(length - i))
|
||||
{
|
||||
state_ptr[i] ^= data[i];
|
||||
}
|
||||
#else /* MLK_SYS_LITTLE_ENDIAN */
|
||||
/* Portable version */
|
||||
for (i = 0; i < length; i++)
|
||||
__loop__(invariant(i <= length)
|
||||
decreases(length - i))
|
||||
{
|
||||
state[(offset + i) >> 3] ^= (uint64_t)data[i]
|
||||
<< (8 * ((offset + i) & 0x07));
|
||||
}
|
||||
#endif /* !MLK_SYS_LITTLE_ENDIAN */
|
||||
}
|
||||
|
||||
static void mlk_keccakf1600x4_extract_bytes_c(uint64_t *state,
|
||||
unsigned char *data0,
|
||||
unsigned char *data1,
|
||||
unsigned char *data2,
|
||||
unsigned char *data3,
|
||||
unsigned offset, unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) &&
|
||||
0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
requires(memory_no_alias(data0, length))
|
||||
requires(memory_no_alias(data1, length))
|
||||
requires(memory_no_alias(data2, length))
|
||||
requires(memory_no_alias(data3, length))
|
||||
assigns(memory_slice(data0, length))
|
||||
assigns(memory_slice(data1, length))
|
||||
assigns(memory_slice(data2, length))
|
||||
assigns(memory_slice(data3, length))
|
||||
)
|
||||
{
|
||||
mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 0, data0, offset,
|
||||
length);
|
||||
mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 1, data1, offset,
|
||||
length);
|
||||
mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 2, data2, offset,
|
||||
length);
|
||||
mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 3, data3, offset,
|
||||
length);
|
||||
}
|
||||
|
||||
void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0,
|
||||
unsigned char *data1, unsigned char *data2,
|
||||
unsigned char *data3, unsigned offset,
|
||||
unsigned length)
|
||||
{
|
||||
#if defined(MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE)
|
||||
if (mlk_keccakf1600_extract_bytes_x4_native(state, data0, data1, data2, data3,
|
||||
offset, length) ==
|
||||
MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE */
|
||||
mlk_keccakf1600x4_extract_bytes_c(state, data0, data1, data2, data3, offset,
|
||||
length);
|
||||
}
|
||||
|
||||
static void mlk_keccakf1600x4_xor_bytes_c(uint64_t *state,
|
||||
const unsigned char *data0,
|
||||
const unsigned char *data1,
|
||||
const unsigned char *data2,
|
||||
const unsigned char *data3,
|
||||
unsigned offset, unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) &&
|
||||
0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
requires(memory_no_alias(data0, length))
|
||||
/* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */
|
||||
requires((data0 == data1 &&
|
||||
data0 == data2 &&
|
||||
data0 == data3) ||
|
||||
(memory_no_alias(data1, length) &&
|
||||
memory_no_alias(data2, length) &&
|
||||
memory_no_alias(data3, length)))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
)
|
||||
{
|
||||
mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 0, data0, offset,
|
||||
length);
|
||||
mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 1, data1, offset,
|
||||
length);
|
||||
mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 2, data2, offset,
|
||||
length);
|
||||
mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 3, data3, offset,
|
||||
length);
|
||||
}
|
||||
|
||||
void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0,
|
||||
const unsigned char *data1,
|
||||
const unsigned char *data2,
|
||||
const unsigned char *data3, unsigned offset,
|
||||
unsigned length)
|
||||
{
|
||||
#if defined(MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE)
|
||||
if (mlk_keccakf1600_xor_bytes_x4_native(state, data0, data1, data2, data3,
|
||||
offset,
|
||||
length) == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE */
|
||||
mlk_keccakf1600x4_xor_bytes_c(state, data0, data1, data2, data3, offset,
|
||||
length);
|
||||
}
|
||||
|
||||
void mlk_keccakf1600x4_permute(uint64_t *state)
|
||||
{
|
||||
#if defined(MLK_USE_FIPS202_X4_NATIVE)
|
||||
if (mlk_keccak_f1600_x4_native(state) == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_FIPS202_X4_NATIVE */
|
||||
mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 0);
|
||||
mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 1);
|
||||
mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 2);
|
||||
mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 3);
|
||||
}
|
||||
|
||||
static const uint64_t mlk_KeccakF_RoundConstants[MLK_KECCAK_NROUNDS] = {
|
||||
(uint64_t)0x0000000000000001ULL, (uint64_t)0x0000000000008082ULL,
|
||||
(uint64_t)0x800000000000808aULL, (uint64_t)0x8000000080008000ULL,
|
||||
(uint64_t)0x000000000000808bULL, (uint64_t)0x0000000080000001ULL,
|
||||
(uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008009ULL,
|
||||
(uint64_t)0x000000000000008aULL, (uint64_t)0x0000000000000088ULL,
|
||||
(uint64_t)0x0000000080008009ULL, (uint64_t)0x000000008000000aULL,
|
||||
(uint64_t)0x000000008000808bULL, (uint64_t)0x800000000000008bULL,
|
||||
(uint64_t)0x8000000000008089ULL, (uint64_t)0x8000000000008003ULL,
|
||||
(uint64_t)0x8000000000008002ULL, (uint64_t)0x8000000000000080ULL,
|
||||
(uint64_t)0x000000000000800aULL, (uint64_t)0x800000008000000aULL,
|
||||
(uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008080ULL,
|
||||
(uint64_t)0x0000000080000001ULL, (uint64_t)0x8000000080008008ULL};
|
||||
|
||||
MLK_STATIC_TESTABLE
|
||||
void mlk_keccakf1600_permute_c(uint64_t *state)
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
)
|
||||
{
|
||||
unsigned round;
|
||||
|
||||
uint64_t Aba, Abe, Abi, Abo, Abu;
|
||||
uint64_t Aga, Age, Agi, Ago, Agu;
|
||||
uint64_t Aka, Ake, Aki, Ako, Aku;
|
||||
uint64_t Ama, Ame, Ami, Amo, Amu;
|
||||
uint64_t Asa, Ase, Asi, Aso, Asu;
|
||||
uint64_t BCa, BCe, BCi, BCo, BCu;
|
||||
uint64_t Da, De, Di, Do, Du;
|
||||
uint64_t Eba, Ebe, Ebi, Ebo, Ebu;
|
||||
uint64_t Ega, Ege, Egi, Ego, Egu;
|
||||
uint64_t Eka, Eke, Eki, Eko, Eku;
|
||||
uint64_t Ema, Eme, Emi, Emo, Emu;
|
||||
uint64_t Esa, Ese, Esi, Eso, Esu;
|
||||
|
||||
/* copyFromState(A, state) */
|
||||
Aba = state[0];
|
||||
Abe = state[1];
|
||||
Abi = state[2];
|
||||
Abo = state[3];
|
||||
Abu = state[4];
|
||||
Aga = state[5];
|
||||
Age = state[6];
|
||||
Agi = state[7];
|
||||
Ago = state[8];
|
||||
Agu = state[9];
|
||||
Aka = state[10];
|
||||
Ake = state[11];
|
||||
Aki = state[12];
|
||||
Ako = state[13];
|
||||
Aku = state[14];
|
||||
Ama = state[15];
|
||||
Ame = state[16];
|
||||
Ami = state[17];
|
||||
Amo = state[18];
|
||||
Amu = state[19];
|
||||
Asa = state[20];
|
||||
Ase = state[21];
|
||||
Asi = state[22];
|
||||
Aso = state[23];
|
||||
Asu = state[24];
|
||||
|
||||
for (round = 0; round < MLK_KECCAK_NROUNDS; round += 2)
|
||||
__loop__(invariant(round <= MLK_KECCAK_NROUNDS && round % 2 == 0)
|
||||
decreases(MLK_KECCAK_NROUNDS - round))
|
||||
{
|
||||
/* prepareTheta */
|
||||
BCa = Aba ^ Aga ^ Aka ^ Ama ^ Asa;
|
||||
BCe = Abe ^ Age ^ Ake ^ Ame ^ Ase;
|
||||
BCi = Abi ^ Agi ^ Aki ^ Ami ^ Asi;
|
||||
BCo = Abo ^ Ago ^ Ako ^ Amo ^ Aso;
|
||||
BCu = Abu ^ Agu ^ Aku ^ Amu ^ Asu;
|
||||
|
||||
/* thetaRhoPiChiIotaPrepareTheta(round, A, E) */
|
||||
Da = BCu ^ MLK_KECCAK_ROL(BCe, 1);
|
||||
De = BCa ^ MLK_KECCAK_ROL(BCi, 1);
|
||||
Di = BCe ^ MLK_KECCAK_ROL(BCo, 1);
|
||||
Do = BCi ^ MLK_KECCAK_ROL(BCu, 1);
|
||||
Du = BCo ^ MLK_KECCAK_ROL(BCa, 1);
|
||||
|
||||
Aba ^= Da;
|
||||
BCa = Aba;
|
||||
Age ^= De;
|
||||
BCe = MLK_KECCAK_ROL(Age, 44);
|
||||
Aki ^= Di;
|
||||
BCi = MLK_KECCAK_ROL(Aki, 43);
|
||||
Amo ^= Do;
|
||||
BCo = MLK_KECCAK_ROL(Amo, 21);
|
||||
Asu ^= Du;
|
||||
BCu = MLK_KECCAK_ROL(Asu, 14);
|
||||
Eba = BCa ^ ((~BCe) & BCi);
|
||||
Eba ^= (uint64_t)mlk_KeccakF_RoundConstants[round];
|
||||
Ebe = BCe ^ ((~BCi) & BCo);
|
||||
Ebi = BCi ^ ((~BCo) & BCu);
|
||||
Ebo = BCo ^ ((~BCu) & BCa);
|
||||
Ebu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Abo ^= Do;
|
||||
BCa = MLK_KECCAK_ROL(Abo, 28);
|
||||
Agu ^= Du;
|
||||
BCe = MLK_KECCAK_ROL(Agu, 20);
|
||||
Aka ^= Da;
|
||||
BCi = MLK_KECCAK_ROL(Aka, 3);
|
||||
Ame ^= De;
|
||||
BCo = MLK_KECCAK_ROL(Ame, 45);
|
||||
Asi ^= Di;
|
||||
BCu = MLK_KECCAK_ROL(Asi, 61);
|
||||
Ega = BCa ^ ((~BCe) & BCi);
|
||||
Ege = BCe ^ ((~BCi) & BCo);
|
||||
Egi = BCi ^ ((~BCo) & BCu);
|
||||
Ego = BCo ^ ((~BCu) & BCa);
|
||||
Egu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Abe ^= De;
|
||||
BCa = MLK_KECCAK_ROL(Abe, 1);
|
||||
Agi ^= Di;
|
||||
BCe = MLK_KECCAK_ROL(Agi, 6);
|
||||
Ako ^= Do;
|
||||
BCi = MLK_KECCAK_ROL(Ako, 25);
|
||||
Amu ^= Du;
|
||||
BCo = MLK_KECCAK_ROL(Amu, 8);
|
||||
Asa ^= Da;
|
||||
BCu = MLK_KECCAK_ROL(Asa, 18);
|
||||
Eka = BCa ^ ((~BCe) & BCi);
|
||||
Eke = BCe ^ ((~BCi) & BCo);
|
||||
Eki = BCi ^ ((~BCo) & BCu);
|
||||
Eko = BCo ^ ((~BCu) & BCa);
|
||||
Eku = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Abu ^= Du;
|
||||
BCa = MLK_KECCAK_ROL(Abu, 27);
|
||||
Aga ^= Da;
|
||||
BCe = MLK_KECCAK_ROL(Aga, 36);
|
||||
Ake ^= De;
|
||||
BCi = MLK_KECCAK_ROL(Ake, 10);
|
||||
Ami ^= Di;
|
||||
BCo = MLK_KECCAK_ROL(Ami, 15);
|
||||
Aso ^= Do;
|
||||
BCu = MLK_KECCAK_ROL(Aso, 56);
|
||||
Ema = BCa ^ ((~BCe) & BCi);
|
||||
Eme = BCe ^ ((~BCi) & BCo);
|
||||
Emi = BCi ^ ((~BCo) & BCu);
|
||||
Emo = BCo ^ ((~BCu) & BCa);
|
||||
Emu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Abi ^= Di;
|
||||
BCa = MLK_KECCAK_ROL(Abi, 62);
|
||||
Ago ^= Do;
|
||||
BCe = MLK_KECCAK_ROL(Ago, 55);
|
||||
Aku ^= Du;
|
||||
BCi = MLK_KECCAK_ROL(Aku, 39);
|
||||
Ama ^= Da;
|
||||
BCo = MLK_KECCAK_ROL(Ama, 41);
|
||||
Ase ^= De;
|
||||
BCu = MLK_KECCAK_ROL(Ase, 2);
|
||||
Esa = BCa ^ ((~BCe) & BCi);
|
||||
Ese = BCe ^ ((~BCi) & BCo);
|
||||
Esi = BCi ^ ((~BCo) & BCu);
|
||||
Eso = BCo ^ ((~BCu) & BCa);
|
||||
Esu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
/* prepareTheta */
|
||||
BCa = Eba ^ Ega ^ Eka ^ Ema ^ Esa;
|
||||
BCe = Ebe ^ Ege ^ Eke ^ Eme ^ Ese;
|
||||
BCi = Ebi ^ Egi ^ Eki ^ Emi ^ Esi;
|
||||
BCo = Ebo ^ Ego ^ Eko ^ Emo ^ Eso;
|
||||
BCu = Ebu ^ Egu ^ Eku ^ Emu ^ Esu;
|
||||
|
||||
/* thetaRhoPiChiIotaPrepareTheta(round+1, E, A) */
|
||||
Da = BCu ^ MLK_KECCAK_ROL(BCe, 1);
|
||||
De = BCa ^ MLK_KECCAK_ROL(BCi, 1);
|
||||
Di = BCe ^ MLK_KECCAK_ROL(BCo, 1);
|
||||
Do = BCi ^ MLK_KECCAK_ROL(BCu, 1);
|
||||
Du = BCo ^ MLK_KECCAK_ROL(BCa, 1);
|
||||
|
||||
Eba ^= Da;
|
||||
BCa = Eba;
|
||||
Ege ^= De;
|
||||
BCe = MLK_KECCAK_ROL(Ege, 44);
|
||||
Eki ^= Di;
|
||||
BCi = MLK_KECCAK_ROL(Eki, 43);
|
||||
Emo ^= Do;
|
||||
BCo = MLK_KECCAK_ROL(Emo, 21);
|
||||
Esu ^= Du;
|
||||
BCu = MLK_KECCAK_ROL(Esu, 14);
|
||||
Aba = BCa ^ ((~BCe) & BCi);
|
||||
Aba ^= (uint64_t)mlk_KeccakF_RoundConstants[round + 1];
|
||||
Abe = BCe ^ ((~BCi) & BCo);
|
||||
Abi = BCi ^ ((~BCo) & BCu);
|
||||
Abo = BCo ^ ((~BCu) & BCa);
|
||||
Abu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Ebo ^= Do;
|
||||
BCa = MLK_KECCAK_ROL(Ebo, 28);
|
||||
Egu ^= Du;
|
||||
BCe = MLK_KECCAK_ROL(Egu, 20);
|
||||
Eka ^= Da;
|
||||
BCi = MLK_KECCAK_ROL(Eka, 3);
|
||||
Eme ^= De;
|
||||
BCo = MLK_KECCAK_ROL(Eme, 45);
|
||||
Esi ^= Di;
|
||||
BCu = MLK_KECCAK_ROL(Esi, 61);
|
||||
Aga = BCa ^ ((~BCe) & BCi);
|
||||
Age = BCe ^ ((~BCi) & BCo);
|
||||
Agi = BCi ^ ((~BCo) & BCu);
|
||||
Ago = BCo ^ ((~BCu) & BCa);
|
||||
Agu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Ebe ^= De;
|
||||
BCa = MLK_KECCAK_ROL(Ebe, 1);
|
||||
Egi ^= Di;
|
||||
BCe = MLK_KECCAK_ROL(Egi, 6);
|
||||
Eko ^= Do;
|
||||
BCi = MLK_KECCAK_ROL(Eko, 25);
|
||||
Emu ^= Du;
|
||||
BCo = MLK_KECCAK_ROL(Emu, 8);
|
||||
Esa ^= Da;
|
||||
BCu = MLK_KECCAK_ROL(Esa, 18);
|
||||
Aka = BCa ^ ((~BCe) & BCi);
|
||||
Ake = BCe ^ ((~BCi) & BCo);
|
||||
Aki = BCi ^ ((~BCo) & BCu);
|
||||
Ako = BCo ^ ((~BCu) & BCa);
|
||||
Aku = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Ebu ^= Du;
|
||||
BCa = MLK_KECCAK_ROL(Ebu, 27);
|
||||
Ega ^= Da;
|
||||
BCe = MLK_KECCAK_ROL(Ega, 36);
|
||||
Eke ^= De;
|
||||
BCi = MLK_KECCAK_ROL(Eke, 10);
|
||||
Emi ^= Di;
|
||||
BCo = MLK_KECCAK_ROL(Emi, 15);
|
||||
Eso ^= Do;
|
||||
BCu = MLK_KECCAK_ROL(Eso, 56);
|
||||
Ama = BCa ^ ((~BCe) & BCi);
|
||||
Ame = BCe ^ ((~BCi) & BCo);
|
||||
Ami = BCi ^ ((~BCo) & BCu);
|
||||
Amo = BCo ^ ((~BCu) & BCa);
|
||||
Amu = BCu ^ ((~BCa) & BCe);
|
||||
|
||||
Ebi ^= Di;
|
||||
BCa = MLK_KECCAK_ROL(Ebi, 62);
|
||||
Ego ^= Do;
|
||||
BCe = MLK_KECCAK_ROL(Ego, 55);
|
||||
Eku ^= Du;
|
||||
BCi = MLK_KECCAK_ROL(Eku, 39);
|
||||
Ema ^= Da;
|
||||
BCo = MLK_KECCAK_ROL(Ema, 41);
|
||||
Ese ^= De;
|
||||
BCu = MLK_KECCAK_ROL(Ese, 2);
|
||||
Asa = BCa ^ ((~BCe) & BCi);
|
||||
Ase = BCe ^ ((~BCi) & BCo);
|
||||
Asi = BCi ^ ((~BCo) & BCu);
|
||||
Aso = BCo ^ ((~BCu) & BCa);
|
||||
Asu = BCu ^ ((~BCa) & BCe);
|
||||
}
|
||||
|
||||
/* copyToState(state, A) */
|
||||
state[0] = Aba;
|
||||
state[1] = Abe;
|
||||
state[2] = Abi;
|
||||
state[3] = Abo;
|
||||
state[4] = Abu;
|
||||
state[5] = Aga;
|
||||
state[6] = Age;
|
||||
state[7] = Agi;
|
||||
state[8] = Ago;
|
||||
state[9] = Agu;
|
||||
state[10] = Aka;
|
||||
state[11] = Ake;
|
||||
state[12] = Aki;
|
||||
state[13] = Ako;
|
||||
state[14] = Aku;
|
||||
state[15] = Ama;
|
||||
state[16] = Ame;
|
||||
state[17] = Ami;
|
||||
state[18] = Amo;
|
||||
state[19] = Amu;
|
||||
state[20] = Asa;
|
||||
state[21] = Ase;
|
||||
state[22] = Asi;
|
||||
state[23] = Aso;
|
||||
state[24] = Asu;
|
||||
}
|
||||
|
||||
void mlk_keccakf1600_permute(uint64_t *state)
|
||||
{
|
||||
#if defined(MLK_USE_FIPS202_X1_NATIVE)
|
||||
if (mlk_keccak_f1600_x1_native(state) == MLK_NATIVE_FUNC_SUCCESS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
#endif /* MLK_USE_FIPS202_X1_NATIVE */
|
||||
mlk_keccakf1600_permute_c(state);
|
||||
}
|
||||
|
||||
#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(keccakf1600)
|
||||
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
#undef MLK_KECCAK_NROUNDS
|
||||
#undef MLK_KECCAK_ROL
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_FIPS202_KECCAKF1600_H
|
||||
#define MLK_FIPS202_KECCAKF1600_H
|
||||
#include "../cbmc.h"
|
||||
#include "../common.h"
|
||||
|
||||
#define MLK_KECCAK_LANES 25
|
||||
#define MLK_KECCAK_WAY 4
|
||||
|
||||
/*
|
||||
* WARNING:
|
||||
* The contents of this structure, including the placement
|
||||
* and interleaving of Keccak lanes, are IMPLEMENTATION-DEFINED.
|
||||
* The struct is only exposed here to allow its construction on the stack.
|
||||
*/
|
||||
|
||||
#define mlk_keccakf1600_extract_bytes MLK_NAMESPACE(keccakf1600_extract_bytes)
|
||||
void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data,
|
||||
unsigned offset, unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) &&
|
||||
0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
requires(memory_no_alias(data, length))
|
||||
assigns(memory_slice(data, length))
|
||||
);
|
||||
|
||||
#define mlk_keccakf1600_xor_bytes MLK_NAMESPACE(keccakf1600_xor_bytes)
|
||||
void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data,
|
||||
unsigned offset, unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) &&
|
||||
0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
requires(memory_no_alias(data, length))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
);
|
||||
|
||||
#define mlk_keccakf1600x4_extract_bytes \
|
||||
MLK_NAMESPACE(keccakf1600x4_extract_bytes)
|
||||
void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0,
|
||||
unsigned char *data1, unsigned char *data2,
|
||||
unsigned char *data3, unsigned offset,
|
||||
unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) &&
|
||||
0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
requires(memory_no_alias(data0, length))
|
||||
requires(memory_no_alias(data1, length))
|
||||
requires(memory_no_alias(data2, length))
|
||||
requires(memory_no_alias(data3, length))
|
||||
assigns(memory_slice(data0, length))
|
||||
assigns(memory_slice(data1, length))
|
||||
assigns(memory_slice(data2, length))
|
||||
assigns(memory_slice(data3, length))
|
||||
);
|
||||
|
||||
#define mlk_keccakf1600x4_xor_bytes MLK_NAMESPACE(keccakf1600x4_xor_bytes)
|
||||
void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0,
|
||||
const unsigned char *data1,
|
||||
const unsigned char *data2,
|
||||
const unsigned char *data3, unsigned offset,
|
||||
unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) &&
|
||||
0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
requires(memory_no_alias(data0, length))
|
||||
/* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */
|
||||
requires((data0 == data1 &&
|
||||
data0 == data2 &&
|
||||
data0 == data3) ||
|
||||
(memory_no_alias(data1, length) &&
|
||||
memory_no_alias(data2, length) &&
|
||||
memory_no_alias(data3, length)))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
);
|
||||
|
||||
|
||||
#define mlk_keccakf1600x4_permute MLK_NAMESPACE(keccakf1600x4_permute)
|
||||
void mlk_keccakf1600x4_permute(uint64_t *state)
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))
|
||||
);
|
||||
|
||||
#define mlk_keccakf1600_permute MLK_NAMESPACE(keccakf1600_permute)
|
||||
void mlk_keccakf1600_permute(uint64_t *state)
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES))
|
||||
);
|
||||
|
||||
#endif /* !MLK_FIPS202_KECCAKF1600_H */
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [HYBRID]
|
||||
* Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64
|
||||
* Becker, Kannwischer
|
||||
* https://eprint.iacr.org/2022/1243
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_AUTO_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_AUTO_H
|
||||
/* Default FIPS202 assembly profile for AArch64 systems */
|
||||
|
||||
/*
|
||||
* Default logic to decide which implementation to use.
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* Keccak-f1600
|
||||
*
|
||||
* - On Arm-based Apple CPUs, we pick a pure Neon implementation.
|
||||
* - Otherwise, unless MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set,
|
||||
* we use lazy-rotation scalar assembly from @[HYBRID].
|
||||
* - Otherwise, if MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, we
|
||||
* fall back to the standard C implementation.
|
||||
*/
|
||||
#if defined(__ARM_FEATURE_SHA3) && defined(__APPLE__)
|
||||
#include "x1_v84a.h"
|
||||
#elif !defined(MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER)
|
||||
#include "x1_scalar.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Keccak-f1600x2/x4
|
||||
*
|
||||
* The optimal implementation is highly CPU-specific; see @[HYBRID].
|
||||
*
|
||||
* For now, if v8.4-A is not implemented, we fall back to Keccak-f1600.
|
||||
* If v8.4-A is implemented and we are on an Apple CPU, we use a plain
|
||||
* Neon-based implementation.
|
||||
* If v8.4-A is implemented and we are not on an Apple CPU, we use a
|
||||
* scalar/Neon/Neon hybrid.
|
||||
* The reason for this distinction is that Apple CPUs appear to implement
|
||||
* the SHA3 instructions on all SIMD units, while Arm CPUs prior to Cortex-X4
|
||||
* don't, and ordinary Neon instructions are still needed.
|
||||
*/
|
||||
#if defined(__ARM_FEATURE_SHA3)
|
||||
/*
|
||||
* For Apple-M cores, we use a plain implementation leveraging SHA3
|
||||
* instructions only.
|
||||
*/
|
||||
#if defined(__APPLE__)
|
||||
#include "x2_v84a.h"
|
||||
#else
|
||||
#include "x4_v8a_v84a_scalar.h"
|
||||
#endif
|
||||
|
||||
#else /* __ARM_FEATURE_SHA3 */
|
||||
|
||||
#include "x4_v8a_scalar.h"
|
||||
|
||||
#endif /* !__ARM_FEATURE_SHA3 */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_AUTO_H */
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H
|
||||
|
||||
#include "../../../../cbmc.h"
|
||||
#include "../../../../common.h"
|
||||
|
||||
#define mlk_keccakf1600_round_constants \
|
||||
MLK_NAMESPACE(keccakf1600_round_constants)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint64_t
|
||||
mlk_keccakf1600_round_constants[24];
|
||||
|
||||
#define mlk_keccak_f1600_x1_scalar_aarch64_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm)
|
||||
void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25],
|
||||
const uint64_t rc[24])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_scalar_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1))
|
||||
requires(rc == mlk_keccakf1600_round_constants)
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1))
|
||||
);
|
||||
|
||||
#define mlk_keccak_f1600_x1_v84a_aarch64_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm)
|
||||
void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25],
|
||||
const uint64_t rc[24])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_v84a_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1))
|
||||
requires(rc == mlk_keccakf1600_round_constants)
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1))
|
||||
);
|
||||
|
||||
#define mlk_keccak_f1600_x2_v84a_aarch64_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm)
|
||||
void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50],
|
||||
const uint64_t rc[24])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/keccak_f1600_x2_v84a_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 2))
|
||||
requires(rc == mlk_keccakf1600_round_constants)
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 2))
|
||||
);
|
||||
|
||||
#define mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm)
|
||||
void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100],
|
||||
const uint64_t rc[24])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in
|
||||
* proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.ml
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4))
|
||||
requires(rc == mlk_keccakf1600_round_constants)
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4))
|
||||
);
|
||||
|
||||
#define mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm)
|
||||
void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(
|
||||
uint64_t state[100], const uint64_t rc[24])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in
|
||||
* proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.ml
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4))
|
||||
requires(rc == mlk_keccakf1600_round_constants)
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4))
|
||||
);
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */
|
||||
@@ -0,0 +1,377 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) 2021-2022 Arm Limited
|
||||
* Copyright (c) 2022 Matthias Kannwischer
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
// Author: Hanno Becker <hanno.becker@arm.com>
|
||||
// Author: Matthias Kannwischer <matthias@kannwischer.eu>
|
||||
|
||||
/*yaml
|
||||
Name: keccak_f1600_x1_scalar_aarch64_asm
|
||||
Description: AArch64 scalar implementation of Keccak-f[1600] permutation for single state
|
||||
Signature: void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], const uint64_t rc[24])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 200
|
||||
permissions: read/write
|
||||
c_parameter: uint64_t state[25]
|
||||
description: Keccak state (25 x uint64_t)
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 192
|
||||
permissions: read-only
|
||||
c_parameter: uint64_t const *rc
|
||||
description: Round constants (24 x uint64_t)
|
||||
Stack:
|
||||
bytes: 128
|
||||
description: register preservation and temporary storage
|
||||
*/
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x1_scalar_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x80
|
||||
.cfi_adjust_cfa_offset 0x80
|
||||
stp x19, x20, [sp, #0x20]
|
||||
.cfi_rel_offset x19, 0x20
|
||||
.cfi_rel_offset x20, 0x28
|
||||
stp x21, x22, [sp, #0x30]
|
||||
.cfi_rel_offset x21, 0x30
|
||||
.cfi_rel_offset x22, 0x38
|
||||
stp x23, x24, [sp, #0x40]
|
||||
.cfi_rel_offset x23, 0x40
|
||||
.cfi_rel_offset x24, 0x48
|
||||
stp x25, x26, [sp, #0x50]
|
||||
.cfi_rel_offset x25, 0x50
|
||||
.cfi_rel_offset x26, 0x58
|
||||
stp x27, x28, [sp, #0x60]
|
||||
.cfi_rel_offset x27, 0x60
|
||||
.cfi_rel_offset x28, 0x68
|
||||
stp x29, x30, [sp, #0x70]
|
||||
.cfi_rel_offset x29, 0x70
|
||||
.cfi_rel_offset x30, 0x78
|
||||
|
||||
Lkeccak_f1600_x1_scalar_initial:
|
||||
mov x26, x1
|
||||
str x1, [sp, #0x8]
|
||||
ldp x1, x6, [x0]
|
||||
ldp x11, x16, [x0, #0x10]
|
||||
ldp x21, x2, [x0, #0x20]
|
||||
ldp x7, x12, [x0, #0x30]
|
||||
ldp x17, x22, [x0, #0x40]
|
||||
ldp x3, x8, [x0, #0x50]
|
||||
ldp x13, x28, [x0, #0x60]
|
||||
ldp x23, x4, [x0, #0x70]
|
||||
ldp x9, x14, [x0, #0x80]
|
||||
ldp x19, x24, [x0, #0x90]
|
||||
ldp x5, x10, [x0, #0xa0]
|
||||
ldp x15, x20, [x0, #0xb0]
|
||||
ldr x25, [x0, #0xc0]
|
||||
str x0, [sp]
|
||||
eor x30, x24, x25
|
||||
eor x27, x9, x10
|
||||
eor x0, x30, x21
|
||||
eor x26, x27, x6
|
||||
eor x27, x26, x7
|
||||
eor x29, x0, x22
|
||||
eor x26, x29, x23
|
||||
eor x29, x4, x5
|
||||
eor x30, x29, x1
|
||||
eor x0, x27, x8
|
||||
eor x29, x30, x2
|
||||
eor x30, x19, x20
|
||||
eor x30, x30, x16
|
||||
eor x27, x26, x0, ror #63
|
||||
eor x4, x4, x27
|
||||
eor x30, x30, x17
|
||||
eor x30, x30, x28
|
||||
eor x29, x29, x3
|
||||
eor x0, x0, x30, ror #63
|
||||
eor x30, x30, x29, ror #63
|
||||
eor x22, x22, x30
|
||||
eor x23, x23, x30
|
||||
str x23, [sp, #0x18]
|
||||
eor x23, x14, x15
|
||||
eor x14, x14, x0
|
||||
eor x23, x23, x11
|
||||
eor x15, x15, x0
|
||||
eor x1, x1, x27
|
||||
eor x23, x23, x12
|
||||
eor x23, x23, x13
|
||||
eor x11, x11, x0
|
||||
eor x29, x29, x23, ror #63
|
||||
eor x23, x23, x26, ror #63
|
||||
eor x26, x13, x0
|
||||
eor x13, x28, x23
|
||||
eor x28, x24, x30
|
||||
eor x24, x16, x23
|
||||
eor x16, x21, x30
|
||||
eor x21, x25, x30
|
||||
eor x30, x19, x23
|
||||
eor x19, x20, x23
|
||||
eor x20, x17, x23
|
||||
eor x17, x12, x0
|
||||
eor x0, x2, x27
|
||||
eor x2, x6, x29
|
||||
eor x6, x8, x29
|
||||
bic x8, x28, x13, ror #47
|
||||
eor x12, x3, x27
|
||||
bic x3, x13, x17, ror #19
|
||||
eor x5, x5, x27
|
||||
ldr x27, [sp, #0x18]
|
||||
bic x25, x17, x2, ror #5
|
||||
eor x9, x9, x29
|
||||
eor x23, x25, x5, ror #52
|
||||
eor x3, x3, x2, ror #24
|
||||
eor x8, x8, x17, ror #2
|
||||
eor x17, x10, x29
|
||||
bic x25, x12, x22, ror #47
|
||||
eor x29, x7, x29
|
||||
bic x10, x4, x27, ror #2
|
||||
bic x7, x5, x28, ror #10
|
||||
eor x10, x10, x20, ror #50
|
||||
eor x13, x7, x13, ror #57
|
||||
bic x7, x2, x5, ror #47
|
||||
eor x2, x25, x24, ror #39
|
||||
bic x25, x20, x11, ror #57
|
||||
bic x5, x17, x4, ror #25
|
||||
eor x25, x25, x17, ror #53
|
||||
bic x17, x11, x17, ror #60
|
||||
eor x28, x7, x28, ror #57
|
||||
bic x7, x9, x12, ror #42
|
||||
eor x7, x7, x22, ror #25
|
||||
bic x22, x22, x24, ror #56
|
||||
bic x24, x24, x15, ror #31
|
||||
eor x22, x22, x15, ror #23
|
||||
bic x20, x27, x20, ror #48
|
||||
bic x15, x15, x9, ror #16
|
||||
eor x12, x15, x12, ror #58
|
||||
eor x15, x5, x27, ror #27
|
||||
eor x5, x20, x11, ror #41
|
||||
ldr x11, [sp, #0x8]
|
||||
eor x20, x17, x4, ror #21
|
||||
eor x17, x24, x9, ror #47
|
||||
mov x24, #0x1 // =1
|
||||
bic x9, x0, x16, ror #9
|
||||
str x24, [sp, #0x10]
|
||||
bic x24, x29, x1, ror #44
|
||||
bic x27, x1, x21, ror #50
|
||||
bic x4, x26, x29, ror #63
|
||||
eor x1, x1, x4, ror #21
|
||||
ldr x11, [x11]
|
||||
bic x4, x21, x30, ror #57
|
||||
eor x21, x24, x21, ror #30
|
||||
eor x24, x9, x19, ror #44
|
||||
bic x9, x14, x6, ror #5
|
||||
eor x9, x9, x0, ror #43
|
||||
bic x0, x6, x0, ror #38
|
||||
eor x1, x1, x11
|
||||
eor x11, x4, x26, ror #35
|
||||
eor x4, x0, x16, ror #47
|
||||
bic x0, x16, x19, ror #35
|
||||
eor x16, x27, x30, ror #43
|
||||
bic x27, x30, x26, ror #42
|
||||
bic x26, x19, x14, ror #41
|
||||
eor x19, x0, x14, ror #12
|
||||
eor x14, x26, x6, ror #46
|
||||
eor x6, x27, x29, ror #41
|
||||
|
||||
Lkeccak_f1600_x1_scalar_loop:
|
||||
eor x0, x15, x11, ror #52
|
||||
eor x0, x0, x13, ror #48
|
||||
eor x26, x8, x9, ror #57
|
||||
eor x27, x0, x14, ror #10
|
||||
eor x29, x16, x28, ror #63
|
||||
eor x26, x26, x6, ror #51
|
||||
eor x30, x23, x22, ror #50
|
||||
eor x0, x26, x10, ror #31
|
||||
eor x29, x29, x19, ror #37
|
||||
eor x27, x27, x12, ror #5
|
||||
eor x30, x30, x24, ror #34
|
||||
eor x0, x0, x7, ror #27
|
||||
eor x26, x30, x21, ror #26
|
||||
eor x26, x26, x25, ror #15
|
||||
ror x30, x27, #0x3e
|
||||
eor x30, x30, x26, ror #57
|
||||
ror x26, x26, #0x3a
|
||||
eor x16, x30, x16
|
||||
eor x28, x30, x28, ror #63
|
||||
str x28, [sp, #0x18]
|
||||
eor x29, x29, x17, ror #36
|
||||
eor x28, x1, x2, ror #61
|
||||
eor x19, x30, x19, ror #37
|
||||
eor x29, x29, x20, ror #2
|
||||
eor x28, x28, x4, ror #54
|
||||
eor x26, x26, x0, ror #55
|
||||
eor x28, x28, x3, ror #39
|
||||
eor x28, x28, x5, ror #25
|
||||
ror x0, x0, #0x38
|
||||
eor x0, x0, x29, ror #63
|
||||
eor x27, x28, x27, ror #61
|
||||
eor x13, x0, x13, ror #46
|
||||
eor x28, x29, x28, ror #63
|
||||
eor x29, x30, x20, ror #2
|
||||
eor x20, x26, x3, ror #39
|
||||
eor x11, x0, x11, ror #50
|
||||
eor x25, x28, x25, ror #9
|
||||
eor x3, x28, x21, ror #20
|
||||
eor x21, x26, x1
|
||||
eor x9, x27, x9, ror #49
|
||||
eor x24, x28, x24, ror #28
|
||||
eor x1, x30, x17, ror #36
|
||||
eor x14, x0, x14, ror #8
|
||||
eor x22, x28, x22, ror #44
|
||||
eor x8, x27, x8, ror #56
|
||||
eor x17, x27, x7, ror #19
|
||||
eor x15, x0, x15, ror #62
|
||||
bic x7, x20, x22, ror #47
|
||||
eor x4, x26, x4, ror #54
|
||||
eor x0, x0, x12, ror #3
|
||||
eor x28, x28, x23, ror #58
|
||||
eor x23, x26, x2, ror #61
|
||||
eor x26, x26, x5, ror #25
|
||||
eor x2, x7, x16, ror #39
|
||||
bic x7, x9, x20, ror #42
|
||||
bic x30, x15, x9, ror #16
|
||||
eor x7, x7, x22, ror #25
|
||||
eor x12, x30, x20, ror #58
|
||||
bic x20, x22, x16, ror #56
|
||||
eor x30, x27, x6, ror #43
|
||||
eor x22, x20, x15, ror #23
|
||||
bic x6, x19, x13, ror #42
|
||||
eor x6, x6, x17, ror #41
|
||||
bic x5, x13, x17, ror #63
|
||||
eor x5, x21, x5, ror #21
|
||||
bic x17, x17, x21, ror #44
|
||||
eor x27, x27, x10, ror #23
|
||||
bic x21, x21, x25, ror #50
|
||||
bic x20, x27, x4, ror #25
|
||||
bic x10, x16, x15, ror #31
|
||||
eor x16, x21, x19, ror #43
|
||||
eor x21, x17, x25, ror #30
|
||||
bic x19, x25, x19, ror #57
|
||||
ldr x25, [sp, #0x10]
|
||||
eor x17, x10, x9, ror #47
|
||||
ldr x9, [sp, #0x8]
|
||||
eor x15, x20, x28, ror #27
|
||||
bic x20, x4, x28, ror #2
|
||||
eor x10, x20, x1, ror #50
|
||||
bic x20, x11, x27, ror #60
|
||||
eor x20, x20, x4, ror #21
|
||||
bic x4, x28, x1, ror #48
|
||||
bic x1, x1, x11, ror #57
|
||||
ldr x28, [x9, x25, lsl #3]
|
||||
ldr x9, [sp, #0x18]
|
||||
add x25, x25, #0x1
|
||||
str x25, [sp, #0x10]
|
||||
cmp x25, #0x17
|
||||
eor x25, x1, x27, ror #53
|
||||
bic x27, x30, x26, ror #47
|
||||
eor x1, x5, x28
|
||||
eor x5, x4, x11, ror #41
|
||||
eor x11, x19, x13, ror #35
|
||||
bic x13, x26, x24, ror #10
|
||||
eor x28, x27, x24, ror #57
|
||||
bic x27, x24, x9, ror #47
|
||||
bic x19, x23, x3, ror #9
|
||||
bic x4, x29, x14, ror #41
|
||||
eor x24, x19, x29, ror #44
|
||||
bic x29, x3, x29, ror #35
|
||||
eor x13, x13, x9, ror #57
|
||||
eor x19, x29, x14, ror #12
|
||||
bic x29, x9, x0, ror #19
|
||||
bic x14, x14, x8, ror #5
|
||||
eor x9, x14, x23, ror #43
|
||||
eor x14, x4, x8, ror #46
|
||||
bic x23, x8, x23, ror #38
|
||||
eor x8, x27, x0, ror #2
|
||||
eor x4, x23, x3, ror #47
|
||||
bic x3, x0, x30, ror #5
|
||||
eor x23, x3, x26, ror #52
|
||||
eor x3, x29, x30, ror #24
|
||||
b.le Lkeccak_f1600_x1_scalar_loop
|
||||
ror x6, x6, #0x2b
|
||||
ror x11, x11, #0x32
|
||||
ror x21, x21, #0x14
|
||||
ror x2, x2, #0x3d
|
||||
ror x7, x7, #0x13
|
||||
ror x12, x12, #0x3
|
||||
ror x17, x17, #0x24
|
||||
ror x22, x22, #0x2c
|
||||
ror x3, x3, #0x27
|
||||
ror x8, x8, #0x38
|
||||
ror x13, x13, #0x2e
|
||||
ror x28, x28, #0x3f
|
||||
ror x23, x23, #0x3a
|
||||
ror x4, x4, #0x36
|
||||
ror x9, x9, #0x31
|
||||
ror x14, x14, #0x8
|
||||
ror x19, x19, #0x25
|
||||
ror x24, x24, #0x1c
|
||||
ror x5, x5, #0x19
|
||||
ror x10, x10, #0x17
|
||||
ror x15, x15, #0x3e
|
||||
ror x20, x20, #0x2
|
||||
ror x25, x25, #0x9
|
||||
ldr x0, [sp]
|
||||
stp x1, x6, [x0]
|
||||
stp x11, x16, [x0, #0x10]
|
||||
stp x21, x2, [x0, #0x20]
|
||||
stp x7, x12, [x0, #0x30]
|
||||
stp x17, x22, [x0, #0x40]
|
||||
stp x3, x8, [x0, #0x50]
|
||||
stp x13, x28, [x0, #0x60]
|
||||
stp x23, x4, [x0, #0x70]
|
||||
stp x9, x14, [x0, #0x80]
|
||||
stp x19, x24, [x0, #0x90]
|
||||
stp x5, x10, [x0, #0xa0]
|
||||
stp x15, x20, [x0, #0xb0]
|
||||
str x25, [x0, #0xc0]
|
||||
ldp x19, x20, [sp, #0x20]
|
||||
.cfi_restore x19
|
||||
.cfi_restore x20
|
||||
ldp x21, x22, [sp, #0x30]
|
||||
.cfi_restore x21
|
||||
.cfi_restore x22
|
||||
ldp x23, x24, [sp, #0x40]
|
||||
.cfi_restore x23
|
||||
.cfi_restore x24
|
||||
ldp x25, x26, [sp, #0x50]
|
||||
.cfi_restore x25
|
||||
.cfi_restore x26
|
||||
ldp x27, x28, [sp, #0x60]
|
||||
.cfi_restore x27
|
||||
.cfi_restore x28
|
||||
ldp x29, x30, [sp, #0x70]
|
||||
.cfi_restore x29
|
||||
.cfi_restore x30
|
||||
add sp, sp, #0x80
|
||||
.cfi_adjust_cfa_offset -0x80
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x1_scalar_aarch64_asm)
|
||||
|
||||
#endif /* MLK_FIPS202_AARCH64_NEED_X1_SCALAR && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,206 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) 2021-2022 Arm Limited
|
||||
* Copyright (c) 2022 Matthias Kannwischer
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [HYBRID]
|
||||
* Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64
|
||||
* Becker, Kannwischer
|
||||
* https://eprint.iacr.org/2022/1243
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: keccak_f1600_x1_v84a_aarch64_asm
|
||||
Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for single state
|
||||
Signature: void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], const uint64_t rc[24])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
Features: [SHA3]
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 200
|
||||
permissions: read/write
|
||||
c_parameter: uint64_t state[25]
|
||||
description: Keccak state (25 x uint64_t)
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 192
|
||||
permissions: read-only
|
||||
c_parameter: const uint64_t rc[24]
|
||||
description: Round constants (24 x uint64_t)
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: register preservation
|
||||
*/
|
||||
|
||||
//
|
||||
// Author: Hanno Becker <hanno.becker@arm.com>
|
||||
// Author: Matthias Kannwischer <matthias@kannwischer.eu>
|
||||
//
|
||||
// This implementation is essentially from the paper @[HYBRID].
|
||||
// The only difference is interleaving/deinterleaving of Keccak state
|
||||
// during load and store, so that the caller need not do this.
|
||||
//
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#if defined(__ARM_FEATURE_SHA3)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x1_v84a_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
ldp d0, d1, [x0]
|
||||
ldp d2, d3, [x0, #0x10]
|
||||
ldp d4, d5, [x0, #0x20]
|
||||
ldp d6, d7, [x0, #0x30]
|
||||
ldp d8, d9, [x0, #0x40]
|
||||
ldp d10, d11, [x0, #0x50]
|
||||
ldp d12, d13, [x0, #0x60]
|
||||
ldp d14, d15, [x0, #0x70]
|
||||
ldp d16, d17, [x0, #0x80]
|
||||
ldp d18, d19, [x0, #0x90]
|
||||
ldp d20, d21, [x0, #0xa0]
|
||||
ldp d22, d23, [x0, #0xb0]
|
||||
ldr d24, [x0, #0xc0]
|
||||
mov x2, #0x18 // =24
|
||||
|
||||
Lkeccak_f1600_x1_v84a_loop:
|
||||
eor3 v30.16b, v0.16b, v5.16b, v10.16b
|
||||
eor3 v29.16b, v1.16b, v6.16b, v11.16b
|
||||
eor3 v28.16b, v2.16b, v7.16b, v12.16b
|
||||
eor3 v27.16b, v3.16b, v8.16b, v13.16b
|
||||
eor3 v26.16b, v4.16b, v9.16b, v14.16b
|
||||
eor3 v30.16b, v30.16b, v15.16b, v20.16b
|
||||
eor3 v29.16b, v29.16b, v16.16b, v21.16b
|
||||
eor3 v28.16b, v28.16b, v17.16b, v22.16b
|
||||
eor3 v27.16b, v27.16b, v18.16b, v23.16b
|
||||
eor3 v26.16b, v26.16b, v19.16b, v24.16b
|
||||
rax1 v25.2d, v30.2d, v28.2d
|
||||
rax1 v28.2d, v28.2d, v26.2d
|
||||
rax1 v26.2d, v26.2d, v29.2d
|
||||
rax1 v29.2d, v29.2d, v27.2d
|
||||
rax1 v27.2d, v27.2d, v30.2d
|
||||
eor v30.16b, v0.16b, v26.16b
|
||||
xar v0.2d, v2.2d, v29.2d, #0x2
|
||||
xar v2.2d, v12.2d, v29.2d, #0x15
|
||||
xar v12.2d, v13.2d, v28.2d, #0x27
|
||||
xar v13.2d, v19.2d, v27.2d, #0x38
|
||||
xar v19.2d, v23.2d, v28.2d, #0x8
|
||||
xar v23.2d, v15.2d, v26.2d, #0x17
|
||||
xar v15.2d, v1.2d, v25.2d, #0x3f
|
||||
xar v1.2d, v8.2d, v28.2d, #0x9
|
||||
xar v8.2d, v16.2d, v25.2d, #0x13
|
||||
xar v16.2d, v7.2d, v29.2d, #0x3a
|
||||
xar v7.2d, v10.2d, v26.2d, #0x3d
|
||||
xar v10.2d, v3.2d, v28.2d, #0x24
|
||||
xar v3.2d, v18.2d, v28.2d, #0x2b
|
||||
xar v18.2d, v17.2d, v29.2d, #0x31
|
||||
xar v17.2d, v11.2d, v25.2d, #0x36
|
||||
xar v11.2d, v9.2d, v27.2d, #0x2c
|
||||
xar v9.2d, v22.2d, v29.2d, #0x3
|
||||
xar v22.2d, v14.2d, v27.2d, #0x19
|
||||
xar v14.2d, v20.2d, v26.2d, #0x2e
|
||||
xar v20.2d, v4.2d, v27.2d, #0x25
|
||||
xar v4.2d, v24.2d, v27.2d, #0x32
|
||||
xar v24.2d, v21.2d, v25.2d, #0x3e
|
||||
xar v21.2d, v5.2d, v26.2d, #0x1c
|
||||
xar v27.2d, v6.2d, v25.2d, #0x14
|
||||
ld1r { v31.2d }, [x1], #8
|
||||
bcax v5.16b, v10.16b, v7.16b, v11.16b
|
||||
bcax v6.16b, v11.16b, v8.16b, v7.16b
|
||||
bcax v7.16b, v7.16b, v9.16b, v8.16b
|
||||
bcax v8.16b, v8.16b, v10.16b, v9.16b
|
||||
bcax v9.16b, v9.16b, v11.16b, v10.16b
|
||||
bcax v10.16b, v15.16b, v12.16b, v16.16b
|
||||
bcax v11.16b, v16.16b, v13.16b, v12.16b
|
||||
bcax v12.16b, v12.16b, v14.16b, v13.16b
|
||||
bcax v13.16b, v13.16b, v15.16b, v14.16b
|
||||
bcax v14.16b, v14.16b, v16.16b, v15.16b
|
||||
bcax v15.16b, v20.16b, v17.16b, v21.16b
|
||||
bcax v16.16b, v21.16b, v18.16b, v17.16b
|
||||
bcax v17.16b, v17.16b, v19.16b, v18.16b
|
||||
bcax v18.16b, v18.16b, v20.16b, v19.16b
|
||||
bcax v19.16b, v19.16b, v21.16b, v20.16b
|
||||
bcax v20.16b, v0.16b, v22.16b, v1.16b
|
||||
bcax v21.16b, v1.16b, v23.16b, v22.16b
|
||||
bcax v22.16b, v22.16b, v24.16b, v23.16b
|
||||
bcax v23.16b, v23.16b, v0.16b, v24.16b
|
||||
bcax v24.16b, v24.16b, v1.16b, v0.16b
|
||||
bcax v0.16b, v30.16b, v2.16b, v27.16b
|
||||
bcax v1.16b, v27.16b, v3.16b, v2.16b
|
||||
bcax v2.16b, v2.16b, v4.16b, v3.16b
|
||||
bcax v3.16b, v3.16b, v30.16b, v4.16b
|
||||
bcax v4.16b, v4.16b, v27.16b, v30.16b
|
||||
eor v0.16b, v0.16b, v31.16b
|
||||
sub x2, x2, #0x1
|
||||
cbnz x2, Lkeccak_f1600_x1_v84a_loop
|
||||
stp d0, d1, [x0]
|
||||
stp d2, d3, [x0, #0x10]
|
||||
stp d4, d5, [x0, #0x20]
|
||||
stp d6, d7, [x0, #0x30]
|
||||
stp d8, d9, [x0, #0x40]
|
||||
stp d10, d11, [x0, #0x50]
|
||||
stp d12, d13, [x0, #0x60]
|
||||
stp d14, d15, [x0, #0x70]
|
||||
stp d16, d17, [x0, #0x80]
|
||||
stp d18, d19, [x0, #0x90]
|
||||
stp d20, d21, [x0, #0xa0]
|
||||
stp d22, d23, [x0, #0xb0]
|
||||
str d24, [x0, #0xc0]
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x1_v84a_aarch64_asm)
|
||||
|
||||
#endif /* __ARM_FEATURE_SHA3 */
|
||||
|
||||
#endif /* MLK_FIPS202_AARCH64_NEED_X1_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,261 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) 2021-2022 Arm Limited
|
||||
* Copyright (c) 2022 Matthias Kannwischer
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [HYBRID]
|
||||
* Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64
|
||||
* Becker, Kannwischer
|
||||
* https://eprint.iacr.org/2022/1243
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: keccak_f1600_x2_v84a_aarch64_asm
|
||||
Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for two sequential states
|
||||
Signature: void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], const uint64_t rc[24])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
Features: [SHA3]
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 400
|
||||
permissions: read/write
|
||||
c_parameter: uint64_t state[50]
|
||||
description: Two sequential Keccak states (state0[25], state1[25])
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 192
|
||||
permissions: read-only
|
||||
c_parameter: const uint64_t rc[24]
|
||||
description: Round constants (24 x uint64_t)
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: register preservation
|
||||
*/
|
||||
|
||||
//
|
||||
// Author: Hanno Becker <hanno.becker@arm.com>
|
||||
// Author: Matthias Kannwischer <matthias@kannwischer.eu>
|
||||
//
|
||||
// This implementation is essentially from the paper @[HYBRID].
|
||||
// The only difference is interleaving/deinterleaving of Keccak state
|
||||
// during load and store, so that the caller need not do this.
|
||||
//
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#if defined(__ARM_FEATURE_SHA3)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x2_v84a_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
add x2, x0, #0xc8
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v0.2d, v25.2d, v27.2d
|
||||
trn2 v1.2d, v25.2d, v27.2d
|
||||
trn1 v2.2d, v26.2d, v28.2d
|
||||
trn2 v3.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v4.2d, v25.2d, v27.2d
|
||||
trn2 v5.2d, v25.2d, v27.2d
|
||||
trn1 v6.2d, v26.2d, v28.2d
|
||||
trn2 v7.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v8.2d, v25.2d, v27.2d
|
||||
trn2 v9.2d, v25.2d, v27.2d
|
||||
trn1 v10.2d, v26.2d, v28.2d
|
||||
trn2 v11.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v12.2d, v25.2d, v27.2d
|
||||
trn2 v13.2d, v25.2d, v27.2d
|
||||
trn1 v14.2d, v26.2d, v28.2d
|
||||
trn2 v15.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v16.2d, v25.2d, v27.2d
|
||||
trn2 v17.2d, v25.2d, v27.2d
|
||||
trn1 v18.2d, v26.2d, v28.2d
|
||||
trn2 v19.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v20.2d, v25.2d, v27.2d
|
||||
trn2 v21.2d, v25.2d, v27.2d
|
||||
trn1 v22.2d, v26.2d, v28.2d
|
||||
trn2 v23.2d, v26.2d, v28.2d
|
||||
ldr d25, [x0]
|
||||
ldr d27, [x2]
|
||||
trn1 v24.2d, v25.2d, v27.2d
|
||||
mov x2, #0x18 // =24
|
||||
|
||||
Lkeccak_f1600_x2_v84a_loop:
|
||||
eor3 v30.16b, v0.16b, v5.16b, v10.16b
|
||||
eor3 v29.16b, v1.16b, v6.16b, v11.16b
|
||||
eor3 v28.16b, v2.16b, v7.16b, v12.16b
|
||||
eor3 v27.16b, v3.16b, v8.16b, v13.16b
|
||||
eor3 v26.16b, v4.16b, v9.16b, v14.16b
|
||||
eor3 v30.16b, v30.16b, v15.16b, v20.16b
|
||||
eor3 v29.16b, v29.16b, v16.16b, v21.16b
|
||||
eor3 v28.16b, v28.16b, v17.16b, v22.16b
|
||||
eor3 v27.16b, v27.16b, v18.16b, v23.16b
|
||||
eor3 v26.16b, v26.16b, v19.16b, v24.16b
|
||||
rax1 v25.2d, v30.2d, v28.2d
|
||||
rax1 v28.2d, v28.2d, v26.2d
|
||||
rax1 v26.2d, v26.2d, v29.2d
|
||||
rax1 v29.2d, v29.2d, v27.2d
|
||||
rax1 v27.2d, v27.2d, v30.2d
|
||||
eor v30.16b, v0.16b, v26.16b
|
||||
xar v0.2d, v2.2d, v29.2d, #0x2
|
||||
xar v2.2d, v12.2d, v29.2d, #0x15
|
||||
xar v12.2d, v13.2d, v28.2d, #0x27
|
||||
xar v13.2d, v19.2d, v27.2d, #0x38
|
||||
xar v19.2d, v23.2d, v28.2d, #0x8
|
||||
xar v23.2d, v15.2d, v26.2d, #0x17
|
||||
xar v15.2d, v1.2d, v25.2d, #0x3f
|
||||
xar v1.2d, v8.2d, v28.2d, #0x9
|
||||
xar v8.2d, v16.2d, v25.2d, #0x13
|
||||
xar v16.2d, v7.2d, v29.2d, #0x3a
|
||||
xar v7.2d, v10.2d, v26.2d, #0x3d
|
||||
xar v10.2d, v3.2d, v28.2d, #0x24
|
||||
xar v3.2d, v18.2d, v28.2d, #0x2b
|
||||
xar v18.2d, v17.2d, v29.2d, #0x31
|
||||
xar v17.2d, v11.2d, v25.2d, #0x36
|
||||
xar v11.2d, v9.2d, v27.2d, #0x2c
|
||||
xar v9.2d, v22.2d, v29.2d, #0x3
|
||||
xar v22.2d, v14.2d, v27.2d, #0x19
|
||||
xar v14.2d, v20.2d, v26.2d, #0x2e
|
||||
xar v20.2d, v4.2d, v27.2d, #0x25
|
||||
xar v4.2d, v24.2d, v27.2d, #0x32
|
||||
xar v24.2d, v21.2d, v25.2d, #0x3e
|
||||
xar v21.2d, v5.2d, v26.2d, #0x1c
|
||||
xar v27.2d, v6.2d, v25.2d, #0x14
|
||||
ld1r { v31.2d }, [x1], #8
|
||||
bcax v5.16b, v10.16b, v7.16b, v11.16b
|
||||
bcax v6.16b, v11.16b, v8.16b, v7.16b
|
||||
bcax v7.16b, v7.16b, v9.16b, v8.16b
|
||||
bcax v8.16b, v8.16b, v10.16b, v9.16b
|
||||
bcax v9.16b, v9.16b, v11.16b, v10.16b
|
||||
bcax v10.16b, v15.16b, v12.16b, v16.16b
|
||||
bcax v11.16b, v16.16b, v13.16b, v12.16b
|
||||
bcax v12.16b, v12.16b, v14.16b, v13.16b
|
||||
bcax v13.16b, v13.16b, v15.16b, v14.16b
|
||||
bcax v14.16b, v14.16b, v16.16b, v15.16b
|
||||
bcax v15.16b, v20.16b, v17.16b, v21.16b
|
||||
bcax v16.16b, v21.16b, v18.16b, v17.16b
|
||||
bcax v17.16b, v17.16b, v19.16b, v18.16b
|
||||
bcax v18.16b, v18.16b, v20.16b, v19.16b
|
||||
bcax v19.16b, v19.16b, v21.16b, v20.16b
|
||||
bcax v20.16b, v0.16b, v22.16b, v1.16b
|
||||
bcax v21.16b, v1.16b, v23.16b, v22.16b
|
||||
bcax v22.16b, v22.16b, v24.16b, v23.16b
|
||||
bcax v23.16b, v23.16b, v0.16b, v24.16b
|
||||
bcax v24.16b, v24.16b, v1.16b, v0.16b
|
||||
bcax v0.16b, v30.16b, v2.16b, v27.16b
|
||||
bcax v1.16b, v27.16b, v3.16b, v2.16b
|
||||
bcax v2.16b, v2.16b, v4.16b, v3.16b
|
||||
bcax v3.16b, v3.16b, v30.16b, v4.16b
|
||||
bcax v4.16b, v4.16b, v27.16b, v30.16b
|
||||
eor v0.16b, v0.16b, v31.16b
|
||||
sub x2, x2, #0x1
|
||||
cbnz x2, Lkeccak_f1600_x2_v84a_loop
|
||||
sub x0, x0, #0xc0
|
||||
add x2, x0, #0xc8
|
||||
trn1 v25.2d, v0.2d, v1.2d
|
||||
trn1 v26.2d, v2.2d, v3.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v0.2d, v1.2d
|
||||
trn2 v28.2d, v2.2d, v3.2d
|
||||
st1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v25.2d, v4.2d, v5.2d
|
||||
trn1 v26.2d, v6.2d, v7.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v4.2d, v5.2d
|
||||
trn2 v28.2d, v6.2d, v7.2d
|
||||
st1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v25.2d, v8.2d, v9.2d
|
||||
trn1 v26.2d, v10.2d, v11.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v8.2d, v9.2d
|
||||
trn2 v28.2d, v10.2d, v11.2d
|
||||
st1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v25.2d, v12.2d, v13.2d
|
||||
trn1 v26.2d, v14.2d, v15.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v12.2d, v13.2d
|
||||
trn2 v28.2d, v14.2d, v15.2d
|
||||
st1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v25.2d, v16.2d, v17.2d
|
||||
trn1 v26.2d, v18.2d, v19.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v16.2d, v17.2d
|
||||
trn2 v28.2d, v18.2d, v19.2d
|
||||
st1 { v27.2d, v28.2d }, [x2], #32
|
||||
trn1 v25.2d, v20.2d, v21.2d
|
||||
trn1 v26.2d, v22.2d, v23.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v20.2d, v21.2d
|
||||
trn2 v28.2d, v22.2d, v23.2d
|
||||
st1 { v27.2d, v28.2d }, [x2], #32
|
||||
str d24, [x0]
|
||||
trn2 v25.2d, v24.2d, v24.2d
|
||||
str d25, [x2]
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x2_v84a_aarch64_asm)
|
||||
|
||||
#endif /* __ARM_FEATURE_SHA3 */
|
||||
|
||||
#endif /* MLK_FIPS202_AARCH64_NEED_X2_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,989 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) 2021-2022 Arm Limited
|
||||
* Copyright (c) 2022 Matthias Kannwischer
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
// Author: Hanno Becker <hannobecker@posteo.de>
|
||||
// Author: Matthias Kannwischer <matthias@kannwischer.eu>
|
||||
|
||||
/*yaml
|
||||
Name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm
|
||||
Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states with ARMv8.4-A optimizations
|
||||
Signature: void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
Features: [SHA3]
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 800
|
||||
permissions: read/write
|
||||
c_parameter: uint64_t state[100]
|
||||
description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25])
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 192
|
||||
permissions: read-only
|
||||
c_parameter: const uint64_t rc[24]
|
||||
description: Round constants (24 x uint64_t)
|
||||
Stack:
|
||||
bytes: 224
|
||||
description: register preservation and temporary storage
|
||||
*/
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#if defined(__ARM_FEATURE_SHA3)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0xe0
|
||||
.cfi_adjust_cfa_offset 0xe0
|
||||
stp x19, x20, [sp, #0x30]
|
||||
.cfi_rel_offset x19, 0x30
|
||||
.cfi_rel_offset x20, 0x38
|
||||
stp x21, x22, [sp, #0x40]
|
||||
.cfi_rel_offset x21, 0x40
|
||||
.cfi_rel_offset x22, 0x48
|
||||
stp x23, x24, [sp, #0x50]
|
||||
.cfi_rel_offset x23, 0x50
|
||||
.cfi_rel_offset x24, 0x58
|
||||
stp x25, x26, [sp, #0x60]
|
||||
.cfi_rel_offset x25, 0x60
|
||||
.cfi_rel_offset x26, 0x68
|
||||
stp x27, x28, [sp, #0x70]
|
||||
.cfi_rel_offset x27, 0x70
|
||||
.cfi_rel_offset x28, 0x78
|
||||
stp x29, x30, [sp, #0x80]
|
||||
.cfi_rel_offset x29, 0x80
|
||||
.cfi_rel_offset x30, 0x88
|
||||
stp d8, d9, [sp, #0x90]
|
||||
.cfi_rel_offset d8, 0x90
|
||||
.cfi_rel_offset d9, 0x98
|
||||
stp d10, d11, [sp, #0xa0]
|
||||
.cfi_rel_offset d10, 0xa0
|
||||
.cfi_rel_offset d11, 0xa8
|
||||
stp d12, d13, [sp, #0xb0]
|
||||
.cfi_rel_offset d12, 0xb0
|
||||
.cfi_rel_offset d13, 0xb8
|
||||
stp d14, d15, [sp, #0xc0]
|
||||
.cfi_rel_offset d14, 0xc0
|
||||
.cfi_rel_offset d15, 0xc8
|
||||
mov x29, x1
|
||||
mov x30, #0x0 // =0
|
||||
str x30, [sp, #0x20]
|
||||
str x29, [sp, #0x8]
|
||||
str x29, [sp, #0x10]
|
||||
str x0, [sp]
|
||||
add x4, x0, #0xc8
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v0.2d, v25.2d, v27.2d
|
||||
trn2 v1.2d, v25.2d, v27.2d
|
||||
trn1 v2.2d, v26.2d, v28.2d
|
||||
trn2 v3.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v4.2d, v25.2d, v27.2d
|
||||
trn2 v5.2d, v25.2d, v27.2d
|
||||
trn1 v6.2d, v26.2d, v28.2d
|
||||
trn2 v7.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v8.2d, v25.2d, v27.2d
|
||||
trn2 v9.2d, v25.2d, v27.2d
|
||||
trn1 v10.2d, v26.2d, v28.2d
|
||||
trn2 v11.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v12.2d, v25.2d, v27.2d
|
||||
trn2 v13.2d, v25.2d, v27.2d
|
||||
trn1 v14.2d, v26.2d, v28.2d
|
||||
trn2 v15.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v16.2d, v25.2d, v27.2d
|
||||
trn2 v17.2d, v25.2d, v27.2d
|
||||
trn1 v18.2d, v26.2d, v28.2d
|
||||
trn2 v19.2d, v26.2d, v28.2d
|
||||
ldp q25, q26, [x0], #0x20
|
||||
ld1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v20.2d, v25.2d, v27.2d
|
||||
trn2 v21.2d, v25.2d, v27.2d
|
||||
trn1 v22.2d, v26.2d, v28.2d
|
||||
trn2 v23.2d, v26.2d, v28.2d
|
||||
ldr d25, [x0]
|
||||
ldr d27, [x4]
|
||||
trn1 v24.2d, v25.2d, v27.2d
|
||||
sub x0, x0, #0xc0
|
||||
add x0, x0, #0x190
|
||||
ldp x1, x6, [x0]
|
||||
ldp x11, x16, [x0, #0x10]
|
||||
ldp x21, x2, [x0, #0x20]
|
||||
ldp x7, x12, [x0, #0x30]
|
||||
ldp x17, x22, [x0, #0x40]
|
||||
ldp x3, x8, [x0, #0x50]
|
||||
ldp x13, x28, [x0, #0x60]
|
||||
ldp x23, x4, [x0, #0x70]
|
||||
ldp x9, x14, [x0, #0x80]
|
||||
ldp x19, x24, [x0, #0x90]
|
||||
ldp x5, x10, [x0, #0xa0]
|
||||
ldp x15, x20, [x0, #0xb0]
|
||||
ldr x25, [x0, #0xc0]
|
||||
sub x0, x0, #0x190
|
||||
|
||||
Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_initial:
|
||||
eor x30, x24, x25
|
||||
eor x27, x9, x10
|
||||
eor3 v30.16b, v0.16b, v5.16b, v10.16b
|
||||
eor v30.16b, v30.16b, v15.16b
|
||||
eor x0, x30, x21
|
||||
eor x26, x27, x6
|
||||
eor v30.16b, v30.16b, v20.16b
|
||||
eor x27, x26, x7
|
||||
eor x29, x0, x22
|
||||
eor3 v29.16b, v1.16b, v6.16b, v11.16b
|
||||
eor x26, x29, x23
|
||||
eor x29, x4, x5
|
||||
eor v29.16b, v29.16b, v16.16b
|
||||
eor x30, x29, x1
|
||||
eor x0, x27, x8
|
||||
eor v29.16b, v29.16b, v21.16b
|
||||
eor x29, x30, x2
|
||||
eor x30, x19, x20
|
||||
eor3 v28.16b, v2.16b, v7.16b, v12.16b
|
||||
eor x30, x30, x16
|
||||
eor x27, x26, x0, ror #63
|
||||
eor v28.16b, v28.16b, v17.16b
|
||||
eor x4, x4, x27
|
||||
eor x30, x30, x17
|
||||
eor v28.16b, v28.16b, v22.16b
|
||||
eor x30, x30, x28
|
||||
eor x29, x29, x3
|
||||
eor3 v27.16b, v3.16b, v8.16b, v13.16b
|
||||
eor x0, x0, x30, ror #63
|
||||
eor x30, x30, x29, ror #63
|
||||
eor v27.16b, v27.16b, v18.16b
|
||||
eor x22, x22, x30
|
||||
eor v27.16b, v27.16b, v23.16b
|
||||
eor x23, x23, x30
|
||||
str x23, [sp, #0xd0]
|
||||
eor3 v26.16b, v4.16b, v9.16b, v14.16b
|
||||
eor x23, x14, x15
|
||||
eor x14, x14, x0
|
||||
eor v26.16b, v26.16b, v19.16b
|
||||
eor x23, x23, x11
|
||||
eor x15, x15, x0
|
||||
eor v26.16b, v26.16b, v24.16b
|
||||
eor x1, x1, x27
|
||||
eor x23, x23, x12
|
||||
rax1 v25.2d, v30.2d, v28.2d
|
||||
eor x23, x23, x13
|
||||
eor x11, x11, x0
|
||||
add v31.2d, v26.2d, v26.2d
|
||||
eor x29, x29, x23, ror #63
|
||||
eor x23, x23, x26, ror #63
|
||||
sri v31.2d, v26.2d, #0x3f
|
||||
eor x26, x13, x0
|
||||
eor x13, x28, x23
|
||||
eor v28.16b, v31.16b, v28.16b
|
||||
eor x28, x24, x30
|
||||
eor x24, x16, x23
|
||||
rax1 v26.2d, v26.2d, v29.2d
|
||||
eor x16, x21, x30
|
||||
eor x21, x25, x30
|
||||
add v31.2d, v27.2d, v27.2d
|
||||
eor x30, x19, x23
|
||||
sri v31.2d, v27.2d, #0x3f
|
||||
eor x19, x20, x23
|
||||
eor x20, x17, x23
|
||||
eor v29.16b, v31.16b, v29.16b
|
||||
eor x17, x12, x0
|
||||
eor x0, x2, x27
|
||||
rax1 v27.2d, v27.2d, v30.2d
|
||||
eor x2, x6, x29
|
||||
eor x6, x8, x29
|
||||
eor v30.16b, v0.16b, v26.16b
|
||||
bic x8, x28, x13, ror #47
|
||||
eor x12, x3, x27
|
||||
eor v31.16b, v2.16b, v29.16b
|
||||
bic x3, x13, x17, ror #19
|
||||
eor x5, x5, x27
|
||||
shl v0.2d, v31.2d, #0x3e
|
||||
ldr x27, [sp, #0xd0]
|
||||
bic x25, x17, x2, ror #5
|
||||
sri v0.2d, v31.2d, #0x2
|
||||
eor x9, x9, x29
|
||||
eor x23, x25, x5, ror #52
|
||||
xar v2.2d, v12.2d, v29.2d, #0x15
|
||||
eor x3, x3, x2, ror #24
|
||||
eor x8, x8, x17, ror #2
|
||||
eor v31.16b, v13.16b, v28.16b
|
||||
eor x17, x10, x29
|
||||
bic x25, x12, x22, ror #47
|
||||
shl v12.2d, v31.2d, #0x19
|
||||
eor x29, x7, x29
|
||||
bic x10, x4, x27, ror #2
|
||||
sri v12.2d, v31.2d, #0x27
|
||||
bic x7, x5, x28, ror #10
|
||||
xar v13.2d, v19.2d, v27.2d, #0x38
|
||||
eor x10, x10, x20, ror #50
|
||||
eor x13, x7, x13, ror #57
|
||||
eor v31.16b, v23.16b, v28.16b
|
||||
bic x7, x2, x5, ror #47
|
||||
eor x2, x25, x24, ror #39
|
||||
shl v19.2d, v31.2d, #0x38
|
||||
bic x25, x20, x11, ror #57
|
||||
bic x5, x17, x4, ror #25
|
||||
sri v19.2d, v31.2d, #0x8
|
||||
eor x25, x25, x17, ror #53
|
||||
bic x17, x11, x17, ror #60
|
||||
xar v23.2d, v15.2d, v26.2d, #0x17
|
||||
eor x28, x7, x28, ror #57
|
||||
bic x7, x9, x12, ror #42
|
||||
eor v31.16b, v1.16b, v25.16b
|
||||
eor x7, x7, x22, ror #25
|
||||
bic x22, x22, x24, ror #56
|
||||
shl v15.2d, v31.2d, #0x1
|
||||
bic x24, x24, x15, ror #31
|
||||
eor x22, x22, x15, ror #23
|
||||
sri v15.2d, v31.2d, #0x3f
|
||||
bic x20, x27, x20, ror #48
|
||||
bic x15, x15, x9, ror #16
|
||||
xar v1.2d, v8.2d, v28.2d, #0x9
|
||||
eor x12, x15, x12, ror #58
|
||||
eor x15, x5, x27, ror #27
|
||||
eor v31.16b, v16.16b, v25.16b
|
||||
eor x5, x20, x11, ror #41
|
||||
shl v8.2d, v31.2d, #0x2d
|
||||
ldr x11, [sp, #0x8]
|
||||
eor x20, x17, x4, ror #21
|
||||
sri v8.2d, v31.2d, #0x13
|
||||
eor x17, x24, x9, ror #47
|
||||
mov x24, #0x1 // =1
|
||||
xar v16.2d, v7.2d, v29.2d, #0x3a
|
||||
bic x9, x0, x16, ror #9
|
||||
str x24, [sp, #0x18]
|
||||
eor v31.16b, v10.16b, v26.16b
|
||||
bic x24, x29, x1, ror #44
|
||||
bic x27, x1, x21, ror #50
|
||||
shl v7.2d, v31.2d, #0x3
|
||||
bic x4, x26, x29, ror #63
|
||||
eor x1, x1, x4, ror #21
|
||||
sri v7.2d, v31.2d, #0x3d
|
||||
ldr x11, [x11]
|
||||
bic x4, x21, x30, ror #57
|
||||
xar v10.2d, v3.2d, v28.2d, #0x24
|
||||
eor x21, x24, x21, ror #30
|
||||
eor x24, x9, x19, ror #44
|
||||
eor v31.16b, v18.16b, v28.16b
|
||||
bic x9, x14, x6, ror #5
|
||||
eor x9, x9, x0, ror #43
|
||||
shl v3.2d, v31.2d, #0x15
|
||||
bic x0, x6, x0, ror #38
|
||||
eor x1, x1, x11
|
||||
sri v3.2d, v31.2d, #0x2b
|
||||
eor x11, x4, x26, ror #35
|
||||
eor x4, x0, x16, ror #47
|
||||
xar v18.2d, v17.2d, v29.2d, #0x31
|
||||
bic x0, x16, x19, ror #35
|
||||
eor v31.16b, v11.16b, v25.16b
|
||||
eor x16, x27, x30, ror #43
|
||||
bic x27, x30, x26, ror #42
|
||||
shl v17.2d, v31.2d, #0xa
|
||||
bic x26, x19, x14, ror #41
|
||||
eor x19, x0, x14, ror #12
|
||||
sri v17.2d, v31.2d, #0x36
|
||||
eor x14, x26, x6, ror #46
|
||||
eor x6, x27, x29, ror #41
|
||||
xar v11.2d, v9.2d, v27.2d, #0x2c
|
||||
eor x0, x15, x11, ror #52
|
||||
eor x0, x0, x13, ror #48
|
||||
eor v31.16b, v22.16b, v29.16b
|
||||
eor x26, x8, x9, ror #57
|
||||
eor x27, x0, x14, ror #10
|
||||
shl v9.2d, v31.2d, #0x3d
|
||||
eor x29, x16, x28, ror #63
|
||||
eor x26, x26, x6, ror #51
|
||||
sri v9.2d, v31.2d, #0x3
|
||||
eor x30, x23, x22, ror #50
|
||||
eor x0, x26, x10, ror #31
|
||||
xar v22.2d, v14.2d, v27.2d, #0x19
|
||||
eor x29, x29, x19, ror #37
|
||||
eor x27, x27, x12, ror #5
|
||||
eor v31.16b, v20.16b, v26.16b
|
||||
eor x30, x30, x24, ror #34
|
||||
eor x0, x0, x7, ror #27
|
||||
shl v14.2d, v31.2d, #0x12
|
||||
eor x26, x30, x21, ror #26
|
||||
sri v14.2d, v31.2d, #0x2e
|
||||
eor x26, x26, x25, ror #15
|
||||
ror x30, x27, #0x3e
|
||||
xar v20.2d, v4.2d, v27.2d, #0x25
|
||||
eor x30, x30, x26, ror #57
|
||||
ror x26, x26, #0x3a
|
||||
eor v31.16b, v24.16b, v27.16b
|
||||
eor x16, x30, x16
|
||||
eor x28, x30, x28, ror #63
|
||||
shl v4.2d, v31.2d, #0xe
|
||||
str x28, [sp, #0xd0]
|
||||
eor x29, x29, x17, ror #36
|
||||
sri v4.2d, v31.2d, #0x32
|
||||
eor x28, x1, x2, ror #61
|
||||
eor x19, x30, x19, ror #37
|
||||
xar v24.2d, v21.2d, v25.2d, #0x3e
|
||||
eor x29, x29, x20, ror #2
|
||||
eor x28, x28, x4, ror #54
|
||||
eor v31.16b, v5.16b, v26.16b
|
||||
eor x26, x26, x0, ror #55
|
||||
eor x28, x28, x3, ror #39
|
||||
shl v21.2d, v31.2d, #0x24
|
||||
eor x28, x28, x5, ror #25
|
||||
ror x0, x0, #0x38
|
||||
sri v21.2d, v31.2d, #0x1c
|
||||
eor x0, x0, x29, ror #63
|
||||
eor x27, x28, x27, ror #61
|
||||
xar v27.2d, v6.2d, v25.2d, #0x14
|
||||
eor x13, x0, x13, ror #46
|
||||
eor x28, x29, x28, ror #63
|
||||
bic v31.16b, v7.16b, v11.16b
|
||||
eor x29, x30, x20, ror #2
|
||||
eor v5.16b, v31.16b, v10.16b
|
||||
eor x20, x26, x3, ror #39
|
||||
eor x11, x0, x11, ror #50
|
||||
bcax v6.16b, v11.16b, v8.16b, v7.16b
|
||||
eor x25, x28, x25, ror #9
|
||||
eor x3, x28, x21, ror #20
|
||||
bic v31.16b, v9.16b, v8.16b
|
||||
eor x21, x26, x1
|
||||
eor x9, x27, x9, ror #49
|
||||
eor v7.16b, v31.16b, v7.16b
|
||||
eor x24, x28, x24, ror #28
|
||||
eor x1, x30, x17, ror #36
|
||||
bcax v8.16b, v8.16b, v10.16b, v9.16b
|
||||
eor x14, x0, x14, ror #8
|
||||
eor x22, x28, x22, ror #44
|
||||
bic v31.16b, v11.16b, v10.16b
|
||||
eor x8, x27, x8, ror #56
|
||||
eor x17, x27, x7, ror #19
|
||||
eor v9.16b, v31.16b, v9.16b
|
||||
eor x15, x0, x15, ror #62
|
||||
bic x7, x20, x22, ror #47
|
||||
bcax v10.16b, v15.16b, v12.16b, v16.16b
|
||||
eor x4, x26, x4, ror #54
|
||||
eor x0, x0, x12, ror #3
|
||||
bic v31.16b, v13.16b, v12.16b
|
||||
eor x28, x28, x23, ror #58
|
||||
eor x23, x26, x2, ror #61
|
||||
eor v11.16b, v31.16b, v16.16b
|
||||
eor x26, x26, x5, ror #25
|
||||
bcax v12.16b, v12.16b, v14.16b, v13.16b
|
||||
eor x2, x7, x16, ror #39
|
||||
bic x7, x9, x20, ror #42
|
||||
bic v31.16b, v15.16b, v14.16b
|
||||
bic x30, x15, x9, ror #16
|
||||
eor x7, x7, x22, ror #25
|
||||
eor v13.16b, v31.16b, v13.16b
|
||||
eor x12, x30, x20, ror #58
|
||||
bic x20, x22, x16, ror #56
|
||||
bic v31.16b, v16.16b, v15.16b
|
||||
eor x30, x27, x6, ror #43
|
||||
eor x22, x20, x15, ror #23
|
||||
eor v14.16b, v31.16b, v14.16b
|
||||
bic x6, x19, x13, ror #42
|
||||
eor x6, x6, x17, ror #41
|
||||
bcax v15.16b, v20.16b, v17.16b, v21.16b
|
||||
bic x5, x13, x17, ror #63
|
||||
eor x5, x21, x5, ror #21
|
||||
bic v31.16b, v18.16b, v17.16b
|
||||
bic x17, x17, x21, ror #44
|
||||
eor x27, x27, x10, ror #23
|
||||
eor v16.16b, v31.16b, v21.16b
|
||||
bic x21, x21, x25, ror #50
|
||||
bic x20, x27, x4, ror #25
|
||||
bcax v17.16b, v17.16b, v19.16b, v18.16b
|
||||
bic x10, x16, x15, ror #31
|
||||
eor x16, x21, x19, ror #43
|
||||
bic v31.16b, v20.16b, v19.16b
|
||||
eor x21, x17, x25, ror #30
|
||||
bic x19, x25, x19, ror #57
|
||||
eor v18.16b, v31.16b, v18.16b
|
||||
ldr x25, [sp, #0x18]
|
||||
bcax v19.16b, v19.16b, v21.16b, v20.16b
|
||||
eor x17, x10, x9, ror #47
|
||||
ldr x9, [sp, #0x8]
|
||||
bic v31.16b, v22.16b, v1.16b
|
||||
eor x15, x20, x28, ror #27
|
||||
bic x20, x4, x28, ror #2
|
||||
eor v20.16b, v31.16b, v0.16b
|
||||
eor x10, x20, x1, ror #50
|
||||
bic x20, x11, x27, ror #60
|
||||
bcax v21.16b, v1.16b, v23.16b, v22.16b
|
||||
eor x20, x20, x4, ror #21
|
||||
bic x4, x28, x1, ror #48
|
||||
bic v31.16b, v24.16b, v23.16b
|
||||
bic x1, x1, x11, ror #57
|
||||
ldr x28, [x9, x25, lsl #3]
|
||||
eor v22.16b, v31.16b, v22.16b
|
||||
ldr x9, [sp, #0xd0]
|
||||
add x25, x25, #0x1
|
||||
bcax v23.16b, v23.16b, v0.16b, v24.16b
|
||||
str x25, [sp, #0x18]
|
||||
cmp x25, #0x17
|
||||
bic v31.16b, v1.16b, v0.16b
|
||||
eor x25, x1, x27, ror #53
|
||||
bic x27, x30, x26, ror #47
|
||||
eor v24.16b, v31.16b, v24.16b
|
||||
eor x1, x5, x28
|
||||
eor x5, x4, x11, ror #41
|
||||
bcax v0.16b, v30.16b, v2.16b, v27.16b
|
||||
eor x11, x19, x13, ror #35
|
||||
bic v31.16b, v3.16b, v2.16b
|
||||
bic x13, x26, x24, ror #10
|
||||
eor x28, x27, x24, ror #57
|
||||
eor v1.16b, v31.16b, v27.16b
|
||||
bic x27, x24, x9, ror #47
|
||||
bic x19, x23, x3, ror #9
|
||||
bcax v2.16b, v2.16b, v4.16b, v3.16b
|
||||
bic x4, x29, x14, ror #41
|
||||
eor x24, x19, x29, ror #44
|
||||
bic v31.16b, v30.16b, v4.16b
|
||||
bic x29, x3, x29, ror #35
|
||||
eor x13, x13, x9, ror #57
|
||||
eor v3.16b, v31.16b, v3.16b
|
||||
eor x19, x29, x14, ror #12
|
||||
bic x29, x9, x0, ror #19
|
||||
bcax v4.16b, v4.16b, v27.16b, v30.16b
|
||||
bic x14, x14, x8, ror #5
|
||||
eor x9, x14, x23, ror #43
|
||||
eor x14, x4, x8, ror #46
|
||||
bic x23, x8, x23, ror #38
|
||||
eor x8, x27, x0, ror #2
|
||||
eor x4, x23, x3, ror #47
|
||||
bic x3, x0, x30, ror #5
|
||||
eor x23, x3, x26, ror #52
|
||||
eor x3, x29, x30, ror #24
|
||||
ldr x30, [sp, #0x10]
|
||||
ld1r { v28.2d }, [x30], #8
|
||||
str x30, [sp, #0x10]
|
||||
eor v0.16b, v0.16b, v28.16b
|
||||
|
||||
Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_loop:
|
||||
eor x0, x15, x11, ror #52
|
||||
eor x0, x0, x13, ror #48
|
||||
eor3 v30.16b, v0.16b, v5.16b, v10.16b
|
||||
eor v30.16b, v30.16b, v15.16b
|
||||
eor x26, x8, x9, ror #57
|
||||
eor x27, x0, x14, ror #10
|
||||
eor v30.16b, v30.16b, v20.16b
|
||||
eor x29, x16, x28, ror #63
|
||||
eor x26, x26, x6, ror #51
|
||||
eor3 v29.16b, v1.16b, v6.16b, v11.16b
|
||||
eor x30, x23, x22, ror #50
|
||||
eor x0, x26, x10, ror #31
|
||||
eor v29.16b, v29.16b, v16.16b
|
||||
eor x29, x29, x19, ror #37
|
||||
eor x27, x27, x12, ror #5
|
||||
eor v29.16b, v29.16b, v21.16b
|
||||
eor x30, x30, x24, ror #34
|
||||
eor x0, x0, x7, ror #27
|
||||
eor3 v28.16b, v2.16b, v7.16b, v12.16b
|
||||
eor x26, x30, x21, ror #26
|
||||
eor x26, x26, x25, ror #15
|
||||
eor v28.16b, v28.16b, v17.16b
|
||||
ror x30, x27, #0x3e
|
||||
eor x30, x30, x26, ror #57
|
||||
eor v28.16b, v28.16b, v22.16b
|
||||
ror x26, x26, #0x3a
|
||||
eor x16, x30, x16
|
||||
eor3 v27.16b, v3.16b, v8.16b, v13.16b
|
||||
eor x28, x30, x28, ror #63
|
||||
str x28, [sp, #0xd0]
|
||||
eor v27.16b, v27.16b, v18.16b
|
||||
eor x29, x29, x17, ror #36
|
||||
eor x28, x1, x2, ror #61
|
||||
eor v27.16b, v27.16b, v23.16b
|
||||
eor x19, x30, x19, ror #37
|
||||
eor x29, x29, x20, ror #2
|
||||
eor3 v26.16b, v4.16b, v9.16b, v14.16b
|
||||
eor x28, x28, x4, ror #54
|
||||
eor x26, x26, x0, ror #55
|
||||
eor v26.16b, v26.16b, v19.16b
|
||||
eor x28, x28, x3, ror #39
|
||||
eor x28, x28, x5, ror #25
|
||||
eor v26.16b, v26.16b, v24.16b
|
||||
ror x0, x0, #0x38
|
||||
eor x0, x0, x29, ror #63
|
||||
rax1 v25.2d, v30.2d, v28.2d
|
||||
eor x27, x28, x27, ror #61
|
||||
eor x13, x0, x13, ror #46
|
||||
add v31.2d, v26.2d, v26.2d
|
||||
eor x28, x29, x28, ror #63
|
||||
eor x29, x30, x20, ror #2
|
||||
sri v31.2d, v26.2d, #0x3f
|
||||
eor x20, x26, x3, ror #39
|
||||
eor x11, x0, x11, ror #50
|
||||
eor v28.16b, v31.16b, v28.16b
|
||||
eor x25, x28, x25, ror #9
|
||||
eor x3, x28, x21, ror #20
|
||||
rax1 v26.2d, v26.2d, v29.2d
|
||||
eor x21, x26, x1
|
||||
add v31.2d, v27.2d, v27.2d
|
||||
eor x9, x27, x9, ror #49
|
||||
eor x24, x28, x24, ror #28
|
||||
sri v31.2d, v27.2d, #0x3f
|
||||
eor x1, x30, x17, ror #36
|
||||
eor x14, x0, x14, ror #8
|
||||
eor v29.16b, v31.16b, v29.16b
|
||||
eor x22, x28, x22, ror #44
|
||||
eor x8, x27, x8, ror #56
|
||||
rax1 v27.2d, v27.2d, v30.2d
|
||||
eor x17, x27, x7, ror #19
|
||||
eor x15, x0, x15, ror #62
|
||||
eor v30.16b, v0.16b, v26.16b
|
||||
bic x7, x20, x22, ror #47
|
||||
eor x4, x26, x4, ror #54
|
||||
eor v31.16b, v2.16b, v29.16b
|
||||
eor x0, x0, x12, ror #3
|
||||
eor x28, x28, x23, ror #58
|
||||
shl v0.2d, v31.2d, #0x3e
|
||||
eor x23, x26, x2, ror #61
|
||||
eor x26, x26, x5, ror #25
|
||||
sri v0.2d, v31.2d, #0x2
|
||||
eor x2, x7, x16, ror #39
|
||||
bic x7, x9, x20, ror #42
|
||||
xar v2.2d, v12.2d, v29.2d, #0x15
|
||||
bic x30, x15, x9, ror #16
|
||||
eor x7, x7, x22, ror #25
|
||||
eor v31.16b, v13.16b, v28.16b
|
||||
eor x12, x30, x20, ror #58
|
||||
bic x20, x22, x16, ror #56
|
||||
shl v12.2d, v31.2d, #0x19
|
||||
eor x30, x27, x6, ror #43
|
||||
eor x22, x20, x15, ror #23
|
||||
sri v12.2d, v31.2d, #0x27
|
||||
bic x6, x19, x13, ror #42
|
||||
eor x6, x6, x17, ror #41
|
||||
xar v13.2d, v19.2d, v27.2d, #0x38
|
||||
bic x5, x13, x17, ror #63
|
||||
eor x5, x21, x5, ror #21
|
||||
eor v31.16b, v23.16b, v28.16b
|
||||
bic x17, x17, x21, ror #44
|
||||
eor x27, x27, x10, ror #23
|
||||
shl v19.2d, v31.2d, #0x38
|
||||
bic x21, x21, x25, ror #50
|
||||
bic x20, x27, x4, ror #25
|
||||
sri v19.2d, v31.2d, #0x8
|
||||
bic x10, x16, x15, ror #31
|
||||
eor x16, x21, x19, ror #43
|
||||
xar v23.2d, v15.2d, v26.2d, #0x17
|
||||
eor x21, x17, x25, ror #30
|
||||
bic x19, x25, x19, ror #57
|
||||
eor v31.16b, v1.16b, v25.16b
|
||||
ldr x25, [sp, #0x18]
|
||||
eor x17, x10, x9, ror #47
|
||||
shl v15.2d, v31.2d, #0x1
|
||||
ldr x9, [sp, #0x8]
|
||||
sri v15.2d, v31.2d, #0x3f
|
||||
eor x15, x20, x28, ror #27
|
||||
bic x20, x4, x28, ror #2
|
||||
xar v1.2d, v8.2d, v28.2d, #0x9
|
||||
eor x10, x20, x1, ror #50
|
||||
bic x20, x11, x27, ror #60
|
||||
eor v31.16b, v16.16b, v25.16b
|
||||
eor x20, x20, x4, ror #21
|
||||
bic x4, x28, x1, ror #48
|
||||
shl v8.2d, v31.2d, #0x2d
|
||||
bic x1, x1, x11, ror #57
|
||||
ldr x28, [x9, x25, lsl #3]
|
||||
sri v8.2d, v31.2d, #0x13
|
||||
ldr x9, [sp, #0xd0]
|
||||
add x25, x25, #0x1
|
||||
xar v16.2d, v7.2d, v29.2d, #0x3a
|
||||
str x25, [sp, #0x18]
|
||||
cmp x25, #0x17
|
||||
eor v31.16b, v10.16b, v26.16b
|
||||
eor x25, x1, x27, ror #53
|
||||
bic x27, x30, x26, ror #47
|
||||
shl v7.2d, v31.2d, #0x3
|
||||
eor x1, x5, x28
|
||||
eor x5, x4, x11, ror #41
|
||||
sri v7.2d, v31.2d, #0x3d
|
||||
eor x11, x19, x13, ror #35
|
||||
bic x13, x26, x24, ror #10
|
||||
xar v10.2d, v3.2d, v28.2d, #0x24
|
||||
eor x28, x27, x24, ror #57
|
||||
bic x27, x24, x9, ror #47
|
||||
eor v31.16b, v18.16b, v28.16b
|
||||
bic x19, x23, x3, ror #9
|
||||
bic x4, x29, x14, ror #41
|
||||
shl v3.2d, v31.2d, #0x15
|
||||
eor x24, x19, x29, ror #44
|
||||
bic x29, x3, x29, ror #35
|
||||
sri v3.2d, v31.2d, #0x2b
|
||||
eor x13, x13, x9, ror #57
|
||||
eor x19, x29, x14, ror #12
|
||||
xar v18.2d, v17.2d, v29.2d, #0x31
|
||||
bic x29, x9, x0, ror #19
|
||||
bic x14, x14, x8, ror #5
|
||||
eor v31.16b, v11.16b, v25.16b
|
||||
eor x9, x14, x23, ror #43
|
||||
eor x14, x4, x8, ror #46
|
||||
shl v17.2d, v31.2d, #0xa
|
||||
bic x23, x8, x23, ror #38
|
||||
eor x8, x27, x0, ror #2
|
||||
sri v17.2d, v31.2d, #0x36
|
||||
eor x4, x23, x3, ror #47
|
||||
bic x3, x0, x30, ror #5
|
||||
xar v11.2d, v9.2d, v27.2d, #0x2c
|
||||
eor x23, x3, x26, ror #52
|
||||
eor x3, x29, x30, ror #24
|
||||
eor v31.16b, v22.16b, v29.16b
|
||||
eor x0, x15, x11, ror #52
|
||||
shl v9.2d, v31.2d, #0x3d
|
||||
eor x0, x0, x13, ror #48
|
||||
eor x26, x8, x9, ror #57
|
||||
sri v9.2d, v31.2d, #0x3
|
||||
eor x27, x0, x14, ror #10
|
||||
eor x29, x16, x28, ror #63
|
||||
xar v22.2d, v14.2d, v27.2d, #0x19
|
||||
eor x26, x26, x6, ror #51
|
||||
eor x30, x23, x22, ror #50
|
||||
eor v31.16b, v20.16b, v26.16b
|
||||
eor x0, x26, x10, ror #31
|
||||
eor x29, x29, x19, ror #37
|
||||
shl v14.2d, v31.2d, #0x12
|
||||
eor x27, x27, x12, ror #5
|
||||
eor x30, x30, x24, ror #34
|
||||
sri v14.2d, v31.2d, #0x2e
|
||||
eor x0, x0, x7, ror #27
|
||||
eor x26, x30, x21, ror #26
|
||||
xar v20.2d, v4.2d, v27.2d, #0x25
|
||||
eor x26, x26, x25, ror #15
|
||||
ror x30, x27, #0x3e
|
||||
eor v31.16b, v24.16b, v27.16b
|
||||
eor x30, x30, x26, ror #57
|
||||
ror x26, x26, #0x3a
|
||||
shl v4.2d, v31.2d, #0xe
|
||||
eor x16, x30, x16
|
||||
eor x28, x30, x28, ror #63
|
||||
sri v4.2d, v31.2d, #0x32
|
||||
str x28, [sp, #0xd0]
|
||||
eor x29, x29, x17, ror #36
|
||||
xar v24.2d, v21.2d, v25.2d, #0x3e
|
||||
eor x28, x1, x2, ror #61
|
||||
eor x19, x30, x19, ror #37
|
||||
eor v31.16b, v5.16b, v26.16b
|
||||
eor x29, x29, x20, ror #2
|
||||
eor x28, x28, x4, ror #54
|
||||
shl v21.2d, v31.2d, #0x24
|
||||
eor x26, x26, x0, ror #55
|
||||
eor x28, x28, x3, ror #39
|
||||
sri v21.2d, v31.2d, #0x1c
|
||||
eor x28, x28, x5, ror #25
|
||||
ror x0, x0, #0x38
|
||||
xar v27.2d, v6.2d, v25.2d, #0x14
|
||||
eor x0, x0, x29, ror #63
|
||||
eor x27, x28, x27, ror #61
|
||||
bic v31.16b, v7.16b, v11.16b
|
||||
eor x13, x0, x13, ror #46
|
||||
eor x28, x29, x28, ror #63
|
||||
eor v5.16b, v31.16b, v10.16b
|
||||
eor x29, x30, x20, ror #2
|
||||
eor x20, x26, x3, ror #39
|
||||
bcax v6.16b, v11.16b, v8.16b, v7.16b
|
||||
eor x11, x0, x11, ror #50
|
||||
eor x25, x28, x25, ror #9
|
||||
bic v31.16b, v9.16b, v8.16b
|
||||
eor x3, x28, x21, ror #20
|
||||
eor v7.16b, v31.16b, v7.16b
|
||||
eor x21, x26, x1
|
||||
eor x9, x27, x9, ror #49
|
||||
bcax v8.16b, v8.16b, v10.16b, v9.16b
|
||||
eor x24, x28, x24, ror #28
|
||||
eor x1, x30, x17, ror #36
|
||||
bic v31.16b, v11.16b, v10.16b
|
||||
eor x14, x0, x14, ror #8
|
||||
eor x22, x28, x22, ror #44
|
||||
eor v9.16b, v31.16b, v9.16b
|
||||
eor x8, x27, x8, ror #56
|
||||
eor x17, x27, x7, ror #19
|
||||
bcax v10.16b, v15.16b, v12.16b, v16.16b
|
||||
eor x15, x0, x15, ror #62
|
||||
bic x7, x20, x22, ror #47
|
||||
bic v31.16b, v13.16b, v12.16b
|
||||
eor x4, x26, x4, ror #54
|
||||
eor x0, x0, x12, ror #3
|
||||
eor v11.16b, v31.16b, v16.16b
|
||||
eor x28, x28, x23, ror #58
|
||||
eor x23, x26, x2, ror #61
|
||||
bcax v12.16b, v12.16b, v14.16b, v13.16b
|
||||
eor x26, x26, x5, ror #25
|
||||
eor x2, x7, x16, ror #39
|
||||
bic v31.16b, v15.16b, v14.16b
|
||||
bic x7, x9, x20, ror #42
|
||||
bic x30, x15, x9, ror #16
|
||||
eor v13.16b, v31.16b, v13.16b
|
||||
eor x7, x7, x22, ror #25
|
||||
eor x12, x30, x20, ror #58
|
||||
bic v31.16b, v16.16b, v15.16b
|
||||
bic x20, x22, x16, ror #56
|
||||
eor x30, x27, x6, ror #43
|
||||
eor v14.16b, v31.16b, v14.16b
|
||||
eor x22, x20, x15, ror #23
|
||||
bic x6, x19, x13, ror #42
|
||||
bcax v15.16b, v20.16b, v17.16b, v21.16b
|
||||
eor x6, x6, x17, ror #41
|
||||
bic x5, x13, x17, ror #63
|
||||
bic v31.16b, v18.16b, v17.16b
|
||||
eor x5, x21, x5, ror #21
|
||||
bic x17, x17, x21, ror #44
|
||||
eor v16.16b, v31.16b, v21.16b
|
||||
eor x27, x27, x10, ror #23
|
||||
bic x21, x21, x25, ror #50
|
||||
bcax v17.16b, v17.16b, v19.16b, v18.16b
|
||||
bic x20, x27, x4, ror #25
|
||||
bic x10, x16, x15, ror #31
|
||||
bic v31.16b, v20.16b, v19.16b
|
||||
eor x16, x21, x19, ror #43
|
||||
eor x21, x17, x25, ror #30
|
||||
eor v18.16b, v31.16b, v18.16b
|
||||
bic x19, x25, x19, ror #57
|
||||
ldr x25, [sp, #0x18]
|
||||
bcax v19.16b, v19.16b, v21.16b, v20.16b
|
||||
eor x17, x10, x9, ror #47
|
||||
bic v31.16b, v22.16b, v1.16b
|
||||
ldr x9, [sp, #0x8]
|
||||
eor x15, x20, x28, ror #27
|
||||
eor v20.16b, v31.16b, v0.16b
|
||||
bic x20, x4, x28, ror #2
|
||||
eor x10, x20, x1, ror #50
|
||||
bcax v21.16b, v1.16b, v23.16b, v22.16b
|
||||
bic x20, x11, x27, ror #60
|
||||
eor x20, x20, x4, ror #21
|
||||
bic v31.16b, v24.16b, v23.16b
|
||||
bic x4, x28, x1, ror #48
|
||||
bic x1, x1, x11, ror #57
|
||||
eor v22.16b, v31.16b, v22.16b
|
||||
ldr x28, [x9, x25, lsl #3]
|
||||
ldr x9, [sp, #0xd0]
|
||||
bcax v23.16b, v23.16b, v0.16b, v24.16b
|
||||
add x25, x25, #0x1
|
||||
str x25, [sp, #0x18]
|
||||
bic v31.16b, v1.16b, v0.16b
|
||||
cmp x25, #0x17
|
||||
eor x25, x1, x27, ror #53
|
||||
eor v24.16b, v31.16b, v24.16b
|
||||
bic x27, x30, x26, ror #47
|
||||
eor x1, x5, x28
|
||||
bcax v0.16b, v30.16b, v2.16b, v27.16b
|
||||
eor x5, x4, x11, ror #41
|
||||
eor x11, x19, x13, ror #35
|
||||
bic v31.16b, v3.16b, v2.16b
|
||||
bic x13, x26, x24, ror #10
|
||||
eor x28, x27, x24, ror #57
|
||||
eor v1.16b, v31.16b, v27.16b
|
||||
bic x27, x24, x9, ror #47
|
||||
bic x19, x23, x3, ror #9
|
||||
bcax v2.16b, v2.16b, v4.16b, v3.16b
|
||||
bic x4, x29, x14, ror #41
|
||||
eor x24, x19, x29, ror #44
|
||||
bic v31.16b, v30.16b, v4.16b
|
||||
bic x29, x3, x29, ror #35
|
||||
eor x13, x13, x9, ror #57
|
||||
eor v3.16b, v31.16b, v3.16b
|
||||
eor x19, x29, x14, ror #12
|
||||
bic x29, x9, x0, ror #19
|
||||
bcax v4.16b, v4.16b, v27.16b, v30.16b
|
||||
bic x14, x14, x8, ror #5
|
||||
eor x9, x14, x23, ror #43
|
||||
eor x14, x4, x8, ror #46
|
||||
bic x23, x8, x23, ror #38
|
||||
eor x8, x27, x0, ror #2
|
||||
eor x4, x23, x3, ror #47
|
||||
bic x3, x0, x30, ror #5
|
||||
eor x23, x3, x26, ror #52
|
||||
eor x3, x29, x30, ror #24
|
||||
ldr x30, [sp, #0x10]
|
||||
ld1r { v28.2d }, [x30], #8
|
||||
str x30, [sp, #0x10]
|
||||
eor v0.16b, v0.16b, v28.16b
|
||||
|
||||
Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end:
|
||||
b.le Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_loop
|
||||
ror x2, x2, #0x3d
|
||||
ror x3, x3, #0x27
|
||||
ror x4, x4, #0x36
|
||||
ror x5, x5, #0x19
|
||||
ror x6, x6, #0x2b
|
||||
ror x7, x7, #0x13
|
||||
ror x8, x8, #0x38
|
||||
ror x9, x9, #0x31
|
||||
ror x10, x10, #0x17
|
||||
ror x11, x11, #0x32
|
||||
ror x12, x12, #0x3
|
||||
ror x13, x13, #0x2e
|
||||
ror x14, x14, #0x8
|
||||
ror x15, x15, #0x3e
|
||||
ror x17, x17, #0x24
|
||||
ror x28, x28, #0x3f
|
||||
ror x19, x19, #0x25
|
||||
ror x20, x20, #0x2
|
||||
ror x21, x21, #0x14
|
||||
ror x22, x22, #0x2c
|
||||
ror x23, x23, #0x3a
|
||||
ror x24, x24, #0x1c
|
||||
ror x25, x25, #0x9
|
||||
ldr x30, [sp, #0x20]
|
||||
cmp x30, #0x1
|
||||
b.eq Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_done
|
||||
mov x30, #0x1 // =1
|
||||
str x30, [sp, #0x20]
|
||||
ldr x0, [sp]
|
||||
add x0, x0, #0x190
|
||||
stp x1, x6, [x0]
|
||||
stp x11, x16, [x0, #0x10]
|
||||
stp x21, x2, [x0, #0x20]
|
||||
stp x7, x12, [x0, #0x30]
|
||||
stp x17, x22, [x0, #0x40]
|
||||
stp x3, x8, [x0, #0x50]
|
||||
stp x13, x28, [x0, #0x60]
|
||||
stp x23, x4, [x0, #0x70]
|
||||
stp x9, x14, [x0, #0x80]
|
||||
stp x19, x24, [x0, #0x90]
|
||||
stp x5, x10, [x0, #0xa0]
|
||||
stp x15, x20, [x0, #0xb0]
|
||||
str x25, [x0, #0xc0]
|
||||
sub x0, x0, #0x190
|
||||
add x0, x0, #0x258
|
||||
ldp x1, x6, [x0]
|
||||
ldp x11, x16, [x0, #0x10]
|
||||
ldp x21, x2, [x0, #0x20]
|
||||
ldp x7, x12, [x0, #0x30]
|
||||
ldp x17, x22, [x0, #0x40]
|
||||
ldp x3, x8, [x0, #0x50]
|
||||
ldp x13, x28, [x0, #0x60]
|
||||
ldp x23, x4, [x0, #0x70]
|
||||
ldp x9, x14, [x0, #0x80]
|
||||
ldp x19, x24, [x0, #0x90]
|
||||
ldp x5, x10, [x0, #0xa0]
|
||||
ldp x15, x20, [x0, #0xb0]
|
||||
ldr x25, [x0, #0xc0]
|
||||
sub x0, x0, #0x258
|
||||
b Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_initial
|
||||
|
||||
Lkeccak_f1600_x4_v8a_v84a_scalar_hybrid_done:
|
||||
ldr x0, [sp]
|
||||
add x0, x0, #0x258
|
||||
stp x1, x6, [x0]
|
||||
stp x11, x16, [x0, #0x10]
|
||||
stp x21, x2, [x0, #0x20]
|
||||
stp x7, x12, [x0, #0x30]
|
||||
stp x17, x22, [x0, #0x40]
|
||||
stp x3, x8, [x0, #0x50]
|
||||
stp x13, x28, [x0, #0x60]
|
||||
stp x23, x4, [x0, #0x70]
|
||||
stp x9, x14, [x0, #0x80]
|
||||
stp x19, x24, [x0, #0x90]
|
||||
stp x5, x10, [x0, #0xa0]
|
||||
stp x15, x20, [x0, #0xb0]
|
||||
str x25, [x0, #0xc0]
|
||||
sub x0, x0, #0x258
|
||||
add x4, x0, #0xc8
|
||||
trn1 v25.2d, v0.2d, v1.2d
|
||||
trn1 v26.2d, v2.2d, v3.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v0.2d, v1.2d
|
||||
trn2 v28.2d, v2.2d, v3.2d
|
||||
st1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v25.2d, v4.2d, v5.2d
|
||||
trn1 v26.2d, v6.2d, v7.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v4.2d, v5.2d
|
||||
trn2 v28.2d, v6.2d, v7.2d
|
||||
st1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v25.2d, v8.2d, v9.2d
|
||||
trn1 v26.2d, v10.2d, v11.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v8.2d, v9.2d
|
||||
trn2 v28.2d, v10.2d, v11.2d
|
||||
st1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v25.2d, v12.2d, v13.2d
|
||||
trn1 v26.2d, v14.2d, v15.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v12.2d, v13.2d
|
||||
trn2 v28.2d, v14.2d, v15.2d
|
||||
st1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v25.2d, v16.2d, v17.2d
|
||||
trn1 v26.2d, v18.2d, v19.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v16.2d, v17.2d
|
||||
trn2 v28.2d, v18.2d, v19.2d
|
||||
st1 { v27.2d, v28.2d }, [x4], #32
|
||||
trn1 v25.2d, v20.2d, v21.2d
|
||||
trn1 v26.2d, v22.2d, v23.2d
|
||||
stp q25, q26, [x0], #0x20
|
||||
trn2 v27.2d, v20.2d, v21.2d
|
||||
trn2 v28.2d, v22.2d, v23.2d
|
||||
st1 { v27.2d, v28.2d }, [x4], #32
|
||||
str d24, [x0]
|
||||
trn2 v25.2d, v24.2d, v24.2d
|
||||
str d25, [x4]
|
||||
ldp d8, d9, [sp, #0x90]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0xa0]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0xb0]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0xc0]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
ldp x19, x20, [sp, #0x30]
|
||||
.cfi_restore x19
|
||||
.cfi_restore x20
|
||||
ldp x21, x22, [sp, #0x40]
|
||||
.cfi_restore x21
|
||||
.cfi_restore x22
|
||||
ldp x23, x24, [sp, #0x50]
|
||||
.cfi_restore x23
|
||||
.cfi_restore x24
|
||||
ldp x25, x26, [sp, #0x60]
|
||||
.cfi_restore x25
|
||||
.cfi_restore x26
|
||||
ldp x27, x28, [sp, #0x70]
|
||||
.cfi_restore x27
|
||||
.cfi_restore x28
|
||||
ldp x29, x30, [sp, #0x80]
|
||||
.cfi_restore x29
|
||||
.cfi_restore x30
|
||||
add sp, sp, #0xe0
|
||||
.cfi_adjust_cfa_offset -0xe0
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm)
|
||||
|
||||
#endif /* __ARM_FEATURE_SHA3 */
|
||||
|
||||
#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../../common.h"
|
||||
|
||||
#if (defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) || \
|
||||
defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) || \
|
||||
defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) || \
|
||||
defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \
|
||||
defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "fips202_native_aarch64.h"
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t
|
||||
mlk_keccakf1600_round_constants[24] = {
|
||||
0x0000000000000001, 0x0000000000008082, 0x800000000000808a,
|
||||
0x8000000080008000, 0x000000000000808b, 0x0000000080000001,
|
||||
0x8000000080008081, 0x8000000000008009, 0x000000000000008a,
|
||||
0x0000000000000088, 0x0000000080008009, 0x000000008000000a,
|
||||
0x000000008000808b, 0x800000000000008b, 0x8000000000008089,
|
||||
0x8000000000008003, 0x8000000000008002, 0x8000000000000080,
|
||||
0x000000000000800a, 0x800000008000000a, 0x8000000080008081,
|
||||
0x8000000000008080, 0x0000000080000001, 0x8000000080008008,
|
||||
};
|
||||
|
||||
#else /* (MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \
|
||||
MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \
|
||||
|| MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \
|
||||
MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(fips202_aarch64_round_constants)
|
||||
|
||||
#endif /* !((MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \
|
||||
MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \
|
||||
|| MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \
|
||||
MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED) */
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X1_NATIVE
|
||||
/* Guard for assembly file */
|
||||
#define MLK_FIPS202_AARCH64_NEED_X1_SCALAR
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_aarch64.h"
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state)
|
||||
{
|
||||
mlk_keccak_f1600_x1_scalar_aarch64_asm(state,
|
||||
mlk_keccakf1600_round_constants);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H */
|
||||
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H
|
||||
|
||||
#if !defined(__ARM_FEATURE_SHA3)
|
||||
#error This backend can only be used if SHA3 extensions are available.
|
||||
#endif
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X1_NATIVE
|
||||
/* Guard for assembly file */
|
||||
#define MLK_FIPS202_AARCH64_NEED_X1_V84A
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_aarch64.h"
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state)
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_SHA3))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_keccak_f1600_x1_v84a_aarch64_asm(state, mlk_keccakf1600_round_constants);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H */
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H
|
||||
|
||||
#if !defined(__ARM_FEATURE_SHA3)
|
||||
#error This backend can only be used if SHA3 extensions are available.
|
||||
#endif
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X4_NATIVE
|
||||
/* Guard for assembly file */
|
||||
#define MLK_FIPS202_AARCH64_NEED_X2_V84A
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_aarch64.h"
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state)
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_SHA3))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 0 * 25,
|
||||
mlk_keccakf1600_round_constants);
|
||||
mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 2 * 25,
|
||||
mlk_keccakf1600_round_constants);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H */
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X4_NATIVE
|
||||
/* Guard for assembly file */
|
||||
#define MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_aarch64.h"
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state)
|
||||
{
|
||||
mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(
|
||||
state, mlk_keccakf1600_round_constants);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H */
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H
|
||||
#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H
|
||||
|
||||
#if !defined(__ARM_FEATURE_SHA3)
|
||||
#error This backend can only be used if SHA3 extensions are available.
|
||||
#endif
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X4_NATIVE
|
||||
/* Guard for assembly file */
|
||||
#define MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_aarch64.h"
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state)
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_SHA3))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(
|
||||
state, mlk_keccakf1600_round_constants);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H */
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_API_H
|
||||
#define MLK_FIPS202_NATIVE_API_H
|
||||
/*
|
||||
* FIPS-202 native interface
|
||||
*
|
||||
* This header is primarily for documentation purposes.
|
||||
* It should not be included by backend implementations.
|
||||
*/
|
||||
|
||||
#include "../../cbmc.h"
|
||||
|
||||
/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */
|
||||
#define MLK_NATIVE_FUNC_SUCCESS (0)
|
||||
/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that
|
||||
* the target/parameters are unsupported; typically, this would be because of
|
||||
* dependencies on CPU features not detected on the host CPU. In this case,
|
||||
* the frontend falls back to the default C implementation. */
|
||||
#define MLK_NATIVE_FUNC_FALLBACK (-1)
|
||||
|
||||
/*
|
||||
* This is the C<->native interface allowing for the drop-in
|
||||
* of custom Keccak-F1600 implementations.
|
||||
*
|
||||
* A _backend_ is a specific implementation of parts of this interface.
|
||||
*
|
||||
* You can replace 1-fold or 4-fold batched Keccak-F1600.
|
||||
* To enable, set MLK_USE_FIPS202_X1_NATIVE or MLK_USE_FIPS202_X4_NATIVE
|
||||
* in your backend, and define the inline wrappers mlk_keccak_f1600_x1_native()
|
||||
* and/or mlk_keccak_f1600_x4_native(), respectively, to forward to your
|
||||
* implementation.
|
||||
*/
|
||||
|
||||
#if defined(MLK_USE_FIPS202_X1_NATIVE)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state)
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 1)));
|
||||
#endif /* MLK_USE_FIPS202_X1_NATIVE */
|
||||
#if defined(MLK_USE_FIPS202_X4_NATIVE)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state)
|
||||
__contract__(
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4)));
|
||||
#endif /* MLK_USE_FIPS202_X4_NATIVE */
|
||||
|
||||
/*
|
||||
* Native x4 XOR bytes and extract bytes interface.
|
||||
*
|
||||
* These functions allow backends to provide optimized implementations for
|
||||
* XORing input data into the state and extracting output data from the state.
|
||||
* This is particularly useful for backends that use a different internal state
|
||||
* representation (e.g., bit-interleaved), as conversion can happen during
|
||||
* XOR/extract rather than before/after each permutation.
|
||||
*
|
||||
* NOTE: We assume that the custom representation of the zero state is the
|
||||
* all-zero state.
|
||||
*
|
||||
* MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE: Backend provides native XOR bytes
|
||||
* MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE: Backend provides native extract
|
||||
* bytes
|
||||
*/
|
||||
|
||||
#if defined(MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native(
|
||||
uint64_t *state, const unsigned char *data0, const unsigned char *data1,
|
||||
const unsigned char *data2, const unsigned char *data3, unsigned offset,
|
||||
unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= 25 * sizeof(uint64_t) &&
|
||||
0 <= length && length <= 25 * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4))
|
||||
requires(memory_no_alias(data0, length))
|
||||
requires((data0 == data1 &&
|
||||
data0 == data2 &&
|
||||
data0 == data3) ||
|
||||
(memory_no_alias(data1, length) &&
|
||||
memory_no_alias(data2, length) &&
|
||||
memory_no_alias(data3, length)))
|
||||
assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4)));
|
||||
#endif /* MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE */
|
||||
|
||||
#if defined(MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native(
|
||||
uint64_t *state, unsigned char *data0, unsigned char *data1,
|
||||
unsigned char *data2, unsigned char *data3, unsigned offset,
|
||||
unsigned length)
|
||||
__contract__(
|
||||
requires(0 <= offset && offset <= 25 * sizeof(uint64_t) &&
|
||||
0 <= length && length <= 25 * sizeof(uint64_t) - offset)
|
||||
requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4))
|
||||
requires(memory_no_alias(data0, length))
|
||||
requires(memory_no_alias(data1, length))
|
||||
requires(memory_no_alias(data2, length))
|
||||
requires(memory_no_alias(data3, length))
|
||||
assigns(memory_slice(data0, length))
|
||||
assigns(memory_slice(data1, length))
|
||||
assigns(memory_slice(data2, length))
|
||||
assigns(memory_slice(data3, length))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS));
|
||||
#endif /* MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_API_H */
|
||||
@@ -0,0 +1,10 @@
|
||||
[//]: # (SPDX-License-Identifier: CC-BY-4.0)
|
||||
|
||||
# FIPS202 backend for Armv8.1-M + MVE
|
||||
|
||||
This directory contains the source code for a FIPS202 backend targeting
|
||||
the Armv8.1-M + MVE/Helium architecture. It is automatically derived from
|
||||
the respective development source in [dev/fips202/armv81m](../../../../../dev/fips202/armv81m).
|
||||
|
||||
**Warning:** This backend is still in active development and has not yet undergone
|
||||
the same level of review as the rest of the code. Use at your own risk!
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_ARMV81M_MVE_H
|
||||
#define MLK_FIPS202_NATIVE_ARMV81M_MVE_H
|
||||
|
||||
#define MLK_FIPS202_NATIVE_ARMV81M
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X4_NATIVE
|
||||
#define MLK_USE_FIPS202_X4_XOR_BYTES_NATIVE
|
||||
#define MLK_USE_FIPS202_X4_EXTRACT_BYTES_NATIVE
|
||||
/* Guard for assembly file */
|
||||
#define MLK_FIPS202_ARMV81M_NEED_X4
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_armv81m.h"
|
||||
|
||||
/*
|
||||
* Native x4 permutation
|
||||
* State is kept in bit-interleaved format.
|
||||
*/
|
||||
#define mlk_keccak_f1600_x4_native_impl \
|
||||
MLK_NAMESPACE(keccak_f1600_x4_native_impl)
|
||||
int mlk_keccak_f1600_x4_native_impl(uint64_t *state);
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state)
|
||||
{
|
||||
return mlk_keccak_f1600_x4_native_impl(state);
|
||||
}
|
||||
|
||||
/*
|
||||
* Native x4 XOR bytes (with on-the-fly bit interleaving)
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native(
|
||||
uint64_t *state, const uint8_t *data0, const uint8_t *data1,
|
||||
const uint8_t *data2, const uint8_t *data3, unsigned offset,
|
||||
unsigned length)
|
||||
{
|
||||
mlk_keccak_f1600_x4_state_xor_bytes_asm(state, data0, data1, data2, data3,
|
||||
offset, length);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* Native x4 extract bytes (with on-the-fly bit de-interleaving)
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native(
|
||||
uint64_t *state, uint8_t *data0, uint8_t *data1, uint8_t *data2,
|
||||
uint8_t *data3, unsigned offset, unsigned length)
|
||||
{
|
||||
mlk_keccak_f1600_x4_state_extract_bytes_asm(state, data0, data1, data2, data3,
|
||||
offset, length);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_ARMV81M_MVE_H */
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H
|
||||
#define MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H
|
||||
|
||||
#include "../../../../common.h"
|
||||
|
||||
/* Keccak round constants in bit-interleaved form */
|
||||
#define mlk_keccakf1600_round_constants \
|
||||
MLK_NAMESPACE(keccakf1600_round_constants)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint32_t
|
||||
mlk_keccakf1600_round_constants[48];
|
||||
|
||||
#define mlk_keccak_f1600_x4_mve_asm MLK_NAMESPACE(keccak_f1600_x4_mve_asm)
|
||||
void mlk_keccak_f1600_x4_mve_asm(uint64_t state[100], uint64_t tmpstate[100],
|
||||
const uint32_t rc[48]);
|
||||
|
||||
#define mlk_keccak_f1600_x4_state_xor_bytes_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm)
|
||||
void mlk_keccak_f1600_x4_state_xor_bytes_asm(void *state, const uint8_t *d0,
|
||||
const uint8_t *d1,
|
||||
const uint8_t *d2,
|
||||
const uint8_t *d3, unsigned offset,
|
||||
unsigned length);
|
||||
|
||||
#define mlk_keccak_f1600_x4_state_extract_bytes_asm \
|
||||
MLK_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm)
|
||||
void mlk_keccak_f1600_x4_state_extract_bytes_asm(void *state, uint8_t *data0,
|
||||
uint8_t *data1, uint8_t *data2,
|
||||
uint8_t *data3,
|
||||
unsigned offset,
|
||||
unsigned length);
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H */
|
||||
@@ -0,0 +1,716 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) 2025 Arm Limited
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: keccak_f1600_x4_mve_asm
|
||||
Description: Armv8.1-M MVE implementation of batched (x4) Keccak-f[1600] permutation using bit-interleaved state
|
||||
Signature: void mlk_keccak_f1600_x4_mve_asm(void *state, void *tmpstate, const uint32_t *rc)
|
||||
ABI:
|
||||
Architecture: armv81m
|
||||
CallingConvention: AAPCS32
|
||||
Features: [MVE]
|
||||
r0:
|
||||
type: buffer
|
||||
size_bytes: 800
|
||||
permissions: read/write
|
||||
c_parameter: void *state
|
||||
description: Bit-interleaved state for 4 Keccak instances (even halves followed by odd halves)
|
||||
r1:
|
||||
type: buffer
|
||||
size_bytes: 800
|
||||
permissions: read/write
|
||||
c_parameter: void *tmpstate
|
||||
description: Temporary storage for intermediate state
|
||||
r2:
|
||||
type: buffer
|
||||
size_bytes: 192
|
||||
permissions: read
|
||||
c_parameter: const uint32_t *rc
|
||||
description: Keccak round constants in bit-interleaved form (24 pairs of 32-bit words)
|
||||
Stack:
|
||||
bytes: 228
|
||||
description: register preservation (36) + SIMD registers (64) + temporary storage (128)
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bit-interleaving background
|
||||
// ---------------------------------------------------------------------------
|
||||
// Each 64-bit Keccak lane is stored as two 32-bit words:
|
||||
// even half -- bits 0, 2, 4, ..., 62 of the lane
|
||||
// odd half -- bits 1, 3, 5, ..., 63 of the lane
|
||||
// This representation allows 64-bit lane rotations (used in the Keccak
|
||||
// round function) to be implemented as pairs of 32-bit rotations.
|
||||
//
|
||||
// Batched (x4) processing:
|
||||
// Four Keccak instances are processed as a batch. Their states are
|
||||
// stored interleaved in a single 800-byte buffer: first the even
|
||||
// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes).
|
||||
// Within each 16-byte row, the four u32 words correspond to
|
||||
// instances 0..3 of the same lane, enabling SIMD-parallel operations
|
||||
// across all four instances.
|
||||
//
|
||||
// State memory layout (25 lanes x 4 instances x 2 halves):
|
||||
// S[i][l]_even/odd = even/odd half of lane l, instance i (u32)
|
||||
// Each row is 16 bytes (one Q-register).
|
||||
// Offset Contents
|
||||
// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even
|
||||
// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even
|
||||
// ...
|
||||
// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even
|
||||
// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd
|
||||
// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd
|
||||
// ...
|
||||
// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/armv81m/src/keccak_f1600_x4_mve.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.thumb
|
||||
.syntax unified
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x4_mve_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x4_mve_asm)
|
||||
|
||||
.cfi_startproc
|
||||
push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
|
||||
.cfi_adjust_cfa_offset 0x24
|
||||
.cfi_rel_offset r4, 0x0
|
||||
.cfi_rel_offset r5, 0x4
|
||||
.cfi_rel_offset r6, 0x8
|
||||
.cfi_rel_offset r7, 0xc
|
||||
.cfi_rel_offset r8, 0x10
|
||||
.cfi_rel_offset r9, 0x14
|
||||
.cfi_rel_offset r10, 0x18
|
||||
.cfi_rel_offset r11, 0x1c
|
||||
.cfi_rel_offset lr, 0x20
|
||||
vpush {d8, d9, d10, d11, d12, d13, d14, d15}
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
sub sp, #0x80
|
||||
.cfi_adjust_cfa_offset 0x80
|
||||
mov r6, r2
|
||||
mov.w lr, #0x18
|
||||
mov r2, r0
|
||||
mov r4, r1
|
||||
add.w r3, r2, #0x190
|
||||
vldrw.u32 q0, [r3]
|
||||
vldrw.u32 q1, [r2]
|
||||
vldrw.u32 q2, [r2, #32]
|
||||
wls lr, lr, Lkeccak_f1600_x4_mve_asm_roundend @ imm = #0x8c0
|
||||
|
||||
Lkeccak_f1600_x4_mve_asm_roundstart:
|
||||
vldrw.u32 q6, [r2, #112]
|
||||
veor q7, q6, q2
|
||||
vldrw.u32 q2, [r2, #80]
|
||||
veor q1, q2, q1
|
||||
add.w r5, r2, #0x190
|
||||
vldrw.u32 q5, [r5, #80]
|
||||
veor q4, q5, q0
|
||||
vldrw.u32 q0, [r2, #192]
|
||||
veor q3, q7, q0
|
||||
vldrw.u32 q0, [r2, #160]
|
||||
veor q1, q1, q0
|
||||
vldrw.u32 q0, [r5, #160]
|
||||
veor q0, q4, q0
|
||||
vldrw.u32 q6, [r2, #272]
|
||||
veor q2, q3, q6
|
||||
vldrw.u32 q7, [r2, #240]
|
||||
veor q5, q1, q7
|
||||
vldrw.u32 q4, [r5, #240]
|
||||
veor q4, q0, q4
|
||||
vldrw.u32 q6, [r2, #352]
|
||||
veor q3, q2, q6
|
||||
vldrw.u32 q0, [r2, #320]
|
||||
veor q2, q5, q0
|
||||
vldrw.u32 q1, [r5, #320]
|
||||
veor q5, q4, q1
|
||||
vldrw.u32 q4, [r5, #32]
|
||||
veor q0, q3, q5
|
||||
vldrw.u32 q1, [r5, #16]
|
||||
veor q6, q1, q0
|
||||
vstrw.32 q5, [sp]
|
||||
vshr.u32 q7, q6, #0x1f
|
||||
add.w r10, r4, #0x190
|
||||
vsli.32 q7, q6, #0x1
|
||||
vldrw.u32 q6, [r5, #112]
|
||||
veor q6, q4, q6
|
||||
vldrw.u32 q4, [r5, #192]
|
||||
veor q4, q6, q4
|
||||
vldrw.u32 q6, [r5, #272]
|
||||
veor q4, q4, q6
|
||||
vldrw.u32 q6, [r5, #352]
|
||||
veor q5, q4, q6
|
||||
vstrw.32 q7, [r4, #160]
|
||||
vshr.u32 q4, q5, #0x1f
|
||||
vsli.32 q4, q5, #0x1
|
||||
vldrw.u32 q6, [r2, #16]
|
||||
veor q7, q4, q2
|
||||
veor q1, q6, q7
|
||||
vldrw.u32 q6, [r5, #96]
|
||||
veor q6, q6, q0
|
||||
vstrw.32 q1, [r10, #160]
|
||||
vshr.u32 q1, q6, #0xa
|
||||
vsli.32 q1, q6, #0x16
|
||||
vldrw.u32 q6, [r2, #96]
|
||||
veor q4, q6, q7
|
||||
vstrw.32 q1, [r10, #16]
|
||||
vshr.u32 q6, q4, #0xa
|
||||
vsli.32 q6, q4, #0x16
|
||||
vldrw.u32 q1, [r5, #336]
|
||||
veor q4, q1, q0
|
||||
vldrw.u32 q1, [r2, #176]
|
||||
veor q1, q1, q7
|
||||
vstrw.32 q6, [r4, #16]
|
||||
vshr.u32 q6, q1, #0x1b
|
||||
vsli.32 q6, q1, #0x5
|
||||
vldrw.u32 q1, [r2, #256]
|
||||
veor q1, q1, q7
|
||||
vstrw.32 q6, [r4, #272]
|
||||
vshr.u32 q6, q1, #0xa
|
||||
vsli.32 q6, q1, #0x16
|
||||
vldrw.u32 q1, [r2, #336]
|
||||
veor q1, q1, q7
|
||||
vstrw.32 q6, [r10, #128]
|
||||
vshr.u32 q6, q1, #0x1f
|
||||
vsli.32 q6, q1, #0x1
|
||||
vldrw.u32 q7, [r5, #176]
|
||||
veor q7, q7, q0
|
||||
vstrw.32 q6, [r4, #384]
|
||||
vshr.u32 q1, q7, #0x1b
|
||||
vsli.32 q1, q7, #0x5
|
||||
vldrw.u32 q6, [r5, #256]
|
||||
veor q0, q6, q0
|
||||
vstrw.32 q1, [r10, #272]
|
||||
vshr.u32 q1, q4, #0x1f
|
||||
vldrw.u32 q7, [r5, #64]
|
||||
vsli.32 q1, q4, #0x1
|
||||
vldrw.u32 q4, [r5, #144]
|
||||
vshr.u32 q6, q0, #0x9
|
||||
vstrw.32 q1, [r10, #384]
|
||||
vsli.32 q6, q0, #0x17
|
||||
veor q7, q7, q4
|
||||
vldrw.u32 q1, [r5, #224]
|
||||
veor q4, q7, q1
|
||||
vldrw.u32 q7, [r5, #304]
|
||||
veor q1, q4, q7
|
||||
vldrw.u32 q0, [r5, #384]
|
||||
veor q7, q1, q0
|
||||
vstrw.32 q6, [r4, #128]
|
||||
vshr.u32 q1, q7, #0x1f
|
||||
vsli.32 q1, q7, #0x1
|
||||
vldrw.u32 q6, [r2, #144]
|
||||
veor q0, q1, q3
|
||||
vldrw.u32 q3, [r2, #64]
|
||||
veor q1, q3, q6
|
||||
vldrw.u32 q6, [r2, #224]
|
||||
veor q1, q1, q6
|
||||
vldrw.u32 q3, [r2, #304]
|
||||
veor q6, q1, q3
|
||||
vldrw.u32 q4, [r2, #384]
|
||||
veor q3, q6, q4
|
||||
vldrw.u32 q4, [r2, #48]
|
||||
veor q5, q3, q5
|
||||
vldrw.u32 q1, [r5, #48]
|
||||
veor q1, q1, q5
|
||||
vshr.u32 q6, q1, #0x12
|
||||
vsli.32 q6, q1, #0xe
|
||||
vldrw.u32 q1, [r2, #128]
|
||||
veor q1, q1, q0
|
||||
vstrw.32 q6, [r10, #80]
|
||||
vshr.u32 q6, q1, #0x5
|
||||
vsli.32 q6, q1, #0x1b
|
||||
vldrw.u32 q1, [r5, #128]
|
||||
veor q1, q1, q5
|
||||
vstrw.32 q6, [r10, #336]
|
||||
vshr.u32 q6, q1, #0x4
|
||||
vsli.32 q6, q1, #0x1c
|
||||
veor q1, q4, q0
|
||||
vstrw.32 q6, [r4, #336]
|
||||
vshr.u32 q4, q1, #0x12
|
||||
vsli.32 q4, q1, #0xe
|
||||
vldrw.u32 q6, [r2, #208]
|
||||
veor q6, q6, q0
|
||||
vstrw.32 q4, [r4, #80]
|
||||
vshr.u32 q1, q6, #0x14
|
||||
vsli.32 q1, q6, #0xc
|
||||
vldrw.u32 q4, [r2, #288]
|
||||
veor q4, q4, q0
|
||||
vldrw.u32 q6, [r2, #368]
|
||||
veor q0, q6, q0
|
||||
vshr.u32 q6, q0, #0x4
|
||||
vstrw.32 q1, [r10, #192]
|
||||
vsli.32 q6, q0, #0x1c
|
||||
vshr.u32 q0, q4, #0x16
|
||||
vldrw.u32 q1, [r5, #368]
|
||||
vsli.32 q0, q4, #0xa
|
||||
vstrw.32 q6, [r4, #304]
|
||||
veor q4, q1, q5
|
||||
vstrw.32 q0, [r10, #48]
|
||||
vshr.u32 q1, q4, #0x4
|
||||
vsli.32 q1, q4, #0x1c
|
||||
vldrw.u32 q6, [r5, #208]
|
||||
veor q6, q6, q5
|
||||
vldrw.u32 q0, [r5, #288]
|
||||
veor q5, q0, q5
|
||||
vstrw.32 q1, [r10, #304]
|
||||
vshr.u32 q0, q6, #0x13
|
||||
vsli.32 q0, q6, #0xd
|
||||
vldrw.u32 q1, [r5, #96]
|
||||
vshr.u32 q6, q5, #0x15
|
||||
vldrw.u32 q4, [r5, #16]
|
||||
vsli.32 q6, q5, #0xb
|
||||
vldrw.u32 q5, [r5, #176]
|
||||
veor q1, q4, q1
|
||||
vldrw.u32 q4, [r5, #256]
|
||||
veor q5, q1, q5
|
||||
vldrw.u32 q1, [r5, #336]
|
||||
veor q5, q5, q4
|
||||
vstrw.32 q0, [r4, #192]
|
||||
veor q0, q5, q1
|
||||
vstrw.32 q6, [r4, #48]
|
||||
vshr.u32 q5, q0, #0x1f
|
||||
vsli.32 q5, q0, #0x1
|
||||
vldrw.u32 q4, [r2, #16]
|
||||
veor q3, q5, q3
|
||||
vldrw.u32 q6, [r2, #96]
|
||||
veor q4, q4, q6
|
||||
vldrw.u32 q1, [r2, #176]
|
||||
veor q5, q4, q1
|
||||
vldrw.u32 q6, [r2, #256]
|
||||
veor q6, q5, q6
|
||||
vldrw.u32 q4, [r2, #336]
|
||||
veor q5, q6, q4
|
||||
vldrw.u32 q1, [r5]
|
||||
veor q7, q5, q7
|
||||
vldrw.u32 q4, [r2]
|
||||
veor q1, q1, q7
|
||||
veor q4, q4, q3
|
||||
vshr.u32 q6, q1, #0x20
|
||||
vsli.32 q6, q1, #0x0
|
||||
vldrw.u32 q1, [r2, #80]
|
||||
veor q1, q1, q3
|
||||
vstrw.32 q6, [r10]
|
||||
vshr.u32 q6, q4, #0x20
|
||||
vsli.32 q6, q4, #0x0
|
||||
vldrw.u32 q4, [r5, #80]
|
||||
veor q4, q4, q7
|
||||
vstrw.32 q6, [r4]
|
||||
vshr.u32 q6, q1, #0xe
|
||||
vsli.32 q6, q1, #0x12
|
||||
vldrw.u32 q1, [r2, #160]
|
||||
veor q1, q1, q3
|
||||
vstrw.32 q6, [r4, #256]
|
||||
vshr.u32 q6, q4, #0xe
|
||||
vsli.32 q6, q4, #0x12
|
||||
vldrw.u32 q4, [r2, #240]
|
||||
veor q4, q4, q3
|
||||
vstrw.32 q6, [r10, #256]
|
||||
vshr.u32 q6, q1, #0x1f
|
||||
vsli.32 q6, q1, #0x1
|
||||
vldrw.u32 q1, [r2, #320]
|
||||
veor q1, q1, q3
|
||||
vstrw.32 q6, [r10, #112]
|
||||
vshr.u32 q6, q4, #0xc
|
||||
vsli.32 q6, q4, #0x14
|
||||
vldrw.u32 q3, [r5, #240]
|
||||
veor q3, q3, q7
|
||||
vstrw.32 q6, [r10, #368]
|
||||
vshr.u32 q4, q3, #0xb
|
||||
vsli.32 q4, q3, #0x15
|
||||
vldrw.u32 q3, [r5, #160]
|
||||
veor q6, q3, q7
|
||||
vstrw.32 q4, [r4, #368]
|
||||
vshr.u32 q3, q6, #0x1e
|
||||
vsli.32 q3, q6, #0x2
|
||||
vldrw.u32 q6, [r5, #320]
|
||||
veor q7, q6, q7
|
||||
vldrw.u32 q4, [r2, #368]
|
||||
vshr.u32 q6, q1, #0x17
|
||||
vstrw.32 q3, [r4, #112]
|
||||
vsli.32 q6, q1, #0x9
|
||||
vshr.u32 q1, q7, #0x17
|
||||
vldrw.u32 q3, [r2, #48]
|
||||
vsli.32 q1, q7, #0x9
|
||||
vldrw.u32 q7, [r2, #128]
|
||||
veor q3, q3, q7
|
||||
vldrw.u32 q7, [r2, #208]
|
||||
veor q7, q3, q7
|
||||
vldrw.u32 q3, [r2, #288]
|
||||
veor q3, q7, q3
|
||||
vldrw.u32 q7, [r5, #128]
|
||||
veor q3, q3, q4
|
||||
vldrw.u32 q4, [r5, #48]
|
||||
veor q0, q3, q0
|
||||
veor q4, q4, q7
|
||||
vldrw.u32 q7, [r5, #208]
|
||||
veor q4, q4, q7
|
||||
vldrw.u32 q7, [r5, #288]
|
||||
veor q4, q4, q7
|
||||
vldrw.u32 q7, [r5, #368]
|
||||
veor q7, q4, q7
|
||||
vstrw.32 q6, [r4, #224]
|
||||
vshr.u32 q4, q7, #0x1f
|
||||
vstrw.32 q1, [r10, #224]
|
||||
vsli.32 q4, q7, #0x1
|
||||
veor q5, q4, q5
|
||||
vldrw.u32 q6, [r2, #192]
|
||||
veor q1, q6, q5
|
||||
vldrw.u32 q4, [r5, #112]
|
||||
veor q7, q2, q7
|
||||
vldrw.u32 q6, [r5, #32]
|
||||
vshr.u32 q2, q1, #0xb
|
||||
vsli.32 q2, q1, #0x15
|
||||
veor q1, q6, q0
|
||||
vstrw.32 q2, [r10, #32]
|
||||
vshr.u32 q6, q1, #0x1
|
||||
vsli.32 q6, q1, #0x1f
|
||||
vldrw.u32 q2, [r2, #112]
|
||||
veor q2, q2, q5
|
||||
vstrw.32 q6, [r10, #320]
|
||||
vshr.u32 q1, q2, #0x1d
|
||||
vsli.32 q1, q2, #0x3
|
||||
vldrw.u32 q6, [r2, #32]
|
||||
veor q4, q4, q0
|
||||
vstrw.32 q1, [r4, #176]
|
||||
veor q2, q6, q5
|
||||
vshr.u32 q6, q2, #0x1
|
||||
vldrw.u32 q1, [r5, #352]
|
||||
vsli.32 q6, q2, #0x1f
|
||||
veor q1, q1, q0
|
||||
vstrw.32 q6, [r4, #320]
|
||||
vshr.u32 q6, q1, #0x1
|
||||
vsli.32 q6, q1, #0x1f
|
||||
vldrw.u32 q2, [r5, #192]
|
||||
vshr.u32 q1, q4, #0x1d
|
||||
vstrw.32 q6, [r4, #144]
|
||||
vsli.32 q1, q4, #0x3
|
||||
veor q2, q2, q0
|
||||
vldrw.u32 q6, [r5, #272]
|
||||
veor q0, q6, q0
|
||||
vldrw.u32 q4, [r2, #352]
|
||||
veor q6, q4, q5
|
||||
vldrw.u32 q4, [r2, #272]
|
||||
veor q4, q4, q5
|
||||
vstrw.32 q1, [r10, #176]
|
||||
vshr.u32 q1, q2, #0xa
|
||||
vsli.32 q1, q2, #0x16
|
||||
vldrw.u32 q5, [sp]
|
||||
vshr.u32 q2, q0, #0x18
|
||||
vstrw.32 q1, [r4, #32]
|
||||
vsli.32 q2, q0, #0x8
|
||||
vshr.u32 q1, q6, #0x2
|
||||
vstrw.32 q2, [r4, #288]
|
||||
vsli.32 q1, q6, #0x1e
|
||||
vshr.u32 q6, q4, #0x19
|
||||
vstrw.32 q1, [r10, #144]
|
||||
vsli.32 q6, q4, #0x7
|
||||
vshr.u32 q0, q5, #0x1f
|
||||
vstrw.32 q6, [r10, #288]
|
||||
vsli.32 q0, q5, #0x1
|
||||
veor q5, q0, q3
|
||||
vldrw.u32 q6, [r2, #64]
|
||||
veor q3, q6, q5
|
||||
vldrw.u32 q1, [r5, #64]
|
||||
vshr.u32 q4, q3, #0x13
|
||||
vldrw.u32 q2, [r2, #384]
|
||||
vsli.32 q4, q3, #0xd
|
||||
vldrw.u32 q0, [r5, #224]
|
||||
veor q6, q1, q7
|
||||
vstrw.32 q4, [r10, #240]
|
||||
veor q2, q2, q5
|
||||
veor q3, q0, q7
|
||||
vldrw.u32 q0, [r2, #224]
|
||||
vshr.u32 q4, q6, #0x12
|
||||
vldrw.u32 q1, [r5, #384]
|
||||
vsli.32 q4, q6, #0xe
|
||||
vshr.u32 q6, q2, #0x19
|
||||
vstrw.32 q4, [r4, #240]
|
||||
vsli.32 q6, q2, #0x7
|
||||
vshr.u32 q2, q3, #0xc
|
||||
vstrw.32 q6, [r4, #64]
|
||||
vsli.32 q2, q3, #0x14
|
||||
veor q0, q0, q5
|
||||
vldrw.u32 q6, [r2, #144]
|
||||
veor q4, q1, q7
|
||||
veor q6, q6, q5
|
||||
vstrw.32 q2, [r4, #352]
|
||||
vshr.u32 q2, q4, #0x19
|
||||
vsli.32 q2, q4, #0x7
|
||||
vldrw.u32 q1, [r2, #304]
|
||||
veor q5, q1, q5
|
||||
vldrw.u32 q1, [r5, #144]
|
||||
veor q4, q1, q7
|
||||
vldrw.u32 q3, [r5, #304]
|
||||
veor q1, q3, q7
|
||||
vstrw.32 q2, [r10, #64]
|
||||
vshr.u32 q3, q0, #0xd
|
||||
vsli.32 q3, q0, #0x13
|
||||
vldrw.u32 q7, [r4, #80]
|
||||
vshr.u32 q0, q6, #0x16
|
||||
vstrw.32 q3, [r10, #352]
|
||||
vsli.32 q0, q6, #0xa
|
||||
vshr.u32 q2, q5, #0x1c
|
||||
vsli.32 q2, q5, #0x4
|
||||
vldrw.u32 q5, [r4, #112]
|
||||
vshr.u32 q3, q1, #0x1c
|
||||
vsli.32 q3, q1, #0x4
|
||||
vldrw.u32 q1, [r4, #128]
|
||||
vbic q6, q5, q0
|
||||
vstrw.32 q3, [r10, #208]
|
||||
vbic q3, q1, q5
|
||||
veor q3, q0, q3
|
||||
vstrw.32 q3, [r2, #96]
|
||||
vbic q3, q0, q7
|
||||
veor q0, q7, q6
|
||||
vldrw.u32 q6, [r4, #144]
|
||||
vbic q7, q7, q6
|
||||
vstrw.32 q0, [r2, #80]
|
||||
veor q3, q6, q3
|
||||
vstrw.32 q3, [r2, #144]
|
||||
veor q0, q1, q7
|
||||
vstrw.32 q0, [r2, #128]
|
||||
vbic q1, q6, q1
|
||||
vshr.u32 q6, q4, #0x16
|
||||
vldrw.u32 q3, [r10, #112]
|
||||
vsli.32 q6, q4, #0xa
|
||||
vldrw.u32 q4, [r10, #80]
|
||||
veor q1, q5, q1
|
||||
vldrw.u32 q0, [r10, #144]
|
||||
vbic q7, q4, q0
|
||||
vldrw.u32 q5, [r10, #128]
|
||||
veor q7, q5, q7
|
||||
vstrw.32 q1, [r2, #112]
|
||||
vbic q1, q0, q5
|
||||
vstrw.32 q7, [r5, #128]
|
||||
veor q7, q3, q1
|
||||
vstrw.32 q7, [r5, #112]
|
||||
vbic q7, q5, q3
|
||||
vbic q1, q3, q6
|
||||
vldrw.u32 q3, [r4, #176]
|
||||
veor q5, q4, q1
|
||||
vbic q4, q6, q4
|
||||
vldrw.u32 q1, [r4, #160]
|
||||
veor q0, q0, q4
|
||||
vldrw.u32 q4, [r4, #224]
|
||||
veor q7, q6, q7
|
||||
vstrw.32 q0, [r5, #144]
|
||||
vbic q0, q1, q4
|
||||
vstrw.32 q7, [r5, #96]
|
||||
veor q0, q2, q0
|
||||
vstrw.32 q0, [r2, #208]
|
||||
vbic q6, q3, q1
|
||||
vstrw.32 q5, [r5, #80]
|
||||
vbic q7, q4, q2
|
||||
vldrw.u32 q0, [r10, #160]
|
||||
veor q6, q4, q6
|
||||
vldrw.u32 q5, [r4, #192]
|
||||
vbic q4, q2, q5
|
||||
vldrw.u32 q2, [r10, #224]
|
||||
veor q4, q3, q4
|
||||
vstrw.32 q4, [r2, #176]
|
||||
vbic q4, q5, q3
|
||||
vstrw.32 q6, [r2, #224]
|
||||
veor q4, q1, q4
|
||||
vldrw.u32 q1, [r10, #208]
|
||||
veor q3, q5, q7
|
||||
vldrw.u32 q5, [r10, #192]
|
||||
vbic q6, q1, q5
|
||||
vldrw.u32 q7, [r10, #176]
|
||||
veor q6, q7, q6
|
||||
vstrw.32 q3, [r2, #192]
|
||||
vbic q3, q0, q2
|
||||
vstrw.32 q6, [r5, #176]
|
||||
veor q3, q1, q3
|
||||
vstrw.32 q3, [r5, #208]
|
||||
vbic q3, q5, q7
|
||||
vstrw.32 q4, [r2, #160]
|
||||
veor q3, q0, q3
|
||||
vstrw.32 q3, [r5, #160]
|
||||
vbic q6, q2, q1
|
||||
vldrw.u32 q1, [r4, #288]
|
||||
vbic q7, q7, q0
|
||||
vldrw.u32 q3, [r4, #272]
|
||||
veor q0, q5, q6
|
||||
vldrw.u32 q4, [r4, #304]
|
||||
veor q6, q2, q7
|
||||
vldrw.u32 q7, [r4, #256]
|
||||
vbic q5, q4, q1
|
||||
vstrw.32 q0, [r5, #192]
|
||||
veor q5, q3, q5
|
||||
vstrw.32 q6, [r5, #224]
|
||||
vbic q0, q3, q7
|
||||
vstrw.32 q5, [r2, #272]
|
||||
vbic q6, q1, q3
|
||||
veor q5, q7, q6
|
||||
vldrw.u32 q3, [r4, #240]
|
||||
veor q6, q3, q0
|
||||
vldrw.u32 q2, [r10, #288]
|
||||
vbic q0, q3, q4
|
||||
vstrw.32 q6, [r2, #240]
|
||||
vbic q7, q7, q3
|
||||
vstrw.32 q5, [r2, #256]
|
||||
veor q7, q4, q7
|
||||
vstrw.32 q7, [r2, #304]
|
||||
veor q7, q1, q0
|
||||
vstrw.32 q7, [r2, #288]
|
||||
vldrw.u32 q5, [r10, #304]
|
||||
vbic q7, q5, q2
|
||||
vldrw.u32 q3, [r10, #272]
|
||||
veor q1, q3, q7
|
||||
vldrw.u32 q7, [r4, #336]
|
||||
vbic q4, q2, q3
|
||||
vldrw.u32 q6, [r10, #256]
|
||||
vbic q3, q3, q6
|
||||
vldrw.u32 q0, [r10, #240]
|
||||
veor q3, q0, q3
|
||||
vstrw.32 q1, [r5, #272]
|
||||
vbic q1, q0, q5
|
||||
vstrw.32 q3, [r5, #240]
|
||||
veor q1, q2, q1
|
||||
vldrw.u32 q3, [r4, #384]
|
||||
vbic q2, q6, q0
|
||||
vldrw.u32 q0, [r4, #320]
|
||||
veor q2, q5, q2
|
||||
vldrw.u32 q5, [r4, #352]
|
||||
veor q4, q6, q4
|
||||
vstrw.32 q2, [r5, #304]
|
||||
vbic q2, q7, q0
|
||||
vstrw.32 q1, [r5, #288]
|
||||
veor q1, q3, q2
|
||||
vstrw.32 q1, [r2, #384]
|
||||
vbic q2, q5, q7
|
||||
vstrw.32 q4, [r5, #256]
|
||||
veor q4, q0, q2
|
||||
vstrw.32 q4, [r2, #320]
|
||||
vbic q2, q0, q3
|
||||
vldrw.u32 q4, [r4, #368]
|
||||
vbic q3, q3, q4
|
||||
vldrw.u32 q0, [r10, #320]
|
||||
veor q1, q5, q3
|
||||
vldrw.u32 q6, [r10, #336]
|
||||
vbic q5, q4, q5
|
||||
vstrw.32 q1, [r2, #352]
|
||||
veor q5, q7, q5
|
||||
vstrw.32 q5, [r2, #336]
|
||||
veor q3, q4, q2
|
||||
vstrw.32 q3, [r2, #368]
|
||||
vbic q7, q6, q0
|
||||
vldrw.u32 q5, [r10, #352]
|
||||
vbic q3, q5, q6
|
||||
vldrw.u32 q1, [r10, #368]
|
||||
vbic q4, q1, q5
|
||||
vldrw.u32 q2, [r4, #16]
|
||||
veor q6, q6, q4
|
||||
vldrw.u32 q4, [r10, #384]
|
||||
veor q3, q0, q3
|
||||
vstrw.32 q3, [r5, #320]
|
||||
veor q3, q4, q7
|
||||
vstrw.32 q3, [r5, #384]
|
||||
vbic q0, q0, q4
|
||||
vstrw.32 q6, [r5, #336]
|
||||
veor q3, q1, q0
|
||||
vstrw.32 q3, [r5, #368]
|
||||
vbic q7, q4, q1
|
||||
veor q5, q5, q7
|
||||
vldrw.u32 q6, [r4, #32]
|
||||
vbic q3, q6, q2
|
||||
vldrw.u32 q4, [r4, #48]
|
||||
vbic q0, q4, q6
|
||||
vldrw.u32 q1, [r4]
|
||||
veor q0, q2, q0
|
||||
vldrw.u32 q7, [r4, #64]
|
||||
veor q3, q1, q3
|
||||
vstrw.32 q5, [r5, #352]
|
||||
vbic q5, q1, q7
|
||||
vstrw.32 q0, [r2, #16]
|
||||
veor q0, q4, q5
|
||||
vstrw.32 q0, [r2, #48]
|
||||
vbic q5, q2, q1
|
||||
veor q2, q7, q5
|
||||
vldrw.u32 q0, [r10, #16]
|
||||
vbic q5, q7, q4
|
||||
vldrw.u32 q4, [r10]
|
||||
vbic q1, q0, q4
|
||||
vldrw.u32 q7, [r10, #64]
|
||||
veor q1, q7, q1
|
||||
vstrw.32 q2, [r2, #64]
|
||||
veor q2, q6, q5
|
||||
vbic q6, q4, q7
|
||||
vldrw.u32 q5, [r10, #48]
|
||||
veor q6, q5, q6
|
||||
ldrd r7, r8, [r6]
|
||||
vbic q7, q7, q5
|
||||
vstrw.32 q1, [r5, #64]
|
||||
vdup.32 q1, r7
|
||||
veor q1, q3, q1
|
||||
vldrw.u32 q3, [r10, #32]
|
||||
veor q7, q3, q7
|
||||
add.w r6, r6, #0x8
|
||||
vbic q5, q5, q3
|
||||
vstrw.32 q6, [r5, #48]
|
||||
vbic q6, q3, q0
|
||||
vstrw.32 q1, [r2]
|
||||
veor q5, q0, q5
|
||||
vstrw.32 q7, [r5, #32]
|
||||
veor q4, q4, q6
|
||||
vstrw.32 q5, [r5, #16]
|
||||
vdup.32 q6, r8
|
||||
vstrw.32 q2, [r2, #32]
|
||||
veor q0, q4, q6
|
||||
vstrw.32 q0, [r5]
|
||||
|
||||
Lkeccak_f1600_x4_mve_asm_roundend_pre:
|
||||
le lr, Lkeccak_f1600_x4_mve_asm_roundstart @ imm = #-0x8c0
|
||||
|
||||
Lkeccak_f1600_x4_mve_asm_roundend:
|
||||
add sp, #0x80
|
||||
.cfi_adjust_cfa_offset -0x80
|
||||
vpop {d8, d9, d10, d11, d12, d13, d14, d15}
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
|
||||
.cfi_restore r4
|
||||
.cfi_restore r5
|
||||
.cfi_restore r6
|
||||
.cfi_restore r7
|
||||
.cfi_restore r8
|
||||
.cfi_restore r9
|
||||
.cfi_restore r10
|
||||
.cfi_restore r11
|
||||
.cfi_restore lr
|
||||
.cfi_adjust_cfa_offset -0x24
|
||||
.cfi_endproc
|
||||
nop
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x4_mve_asm)
|
||||
|
||||
#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#include "../../../../common.h"
|
||||
#include "../../../../verify.h"
|
||||
|
||||
#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "fips202_native_armv81m.h"
|
||||
|
||||
|
||||
/*
|
||||
* Keccak-f1600 x4 permutation (on bit-interleaved state)
|
||||
* State is expected to already be in bit-interleaved format.
|
||||
*/
|
||||
#define mlk_keccak_f1600_x4_native_impl \
|
||||
MLK_NAMESPACE(keccak_f1600_x4_native_impl)
|
||||
int mlk_keccak_f1600_x4_native_impl(uint64_t *state)
|
||||
{
|
||||
MLK_ALIGN uint64_t state_tmp[100];
|
||||
mlk_keccak_f1600_x4_mve_asm(state, state_tmp,
|
||||
mlk_keccakf1600_round_constants);
|
||||
mlk_zeroize(state_tmp, sizeof(state_tmp));
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(keccak_f1600_x4_mve)
|
||||
|
||||
#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \
|
||||
*/
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
/* Some macros are kept because they are also defined in a header. */
|
||||
/* Keep: mlk_keccak_f1600_x4_native_impl (mve.h) */
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#include "../../../../common.h"
|
||||
|
||||
#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "fips202_native_armv81m.h"
|
||||
|
||||
/*
|
||||
* Keccak round constants in bit-interleaved form.
|
||||
* Each 64-bit constant is split into two 32-bit words:
|
||||
* - low word contains even-indexed bits
|
||||
* - high word contains odd-indexed bits
|
||||
*/
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint32_t
|
||||
mlk_keccakf1600_round_constants[48] = {
|
||||
0x00000001, 0x00000000, /* RC0 */
|
||||
0x00000000, 0x00000089, /* RC1 */
|
||||
0x00000000, 0x8000008b, /* RC2 */
|
||||
0x00000000, 0x80008080, /* RC3 */
|
||||
0x00000001, 0x0000008b, /* RC4 */
|
||||
0x00000001, 0x00008000, /* RC5 */
|
||||
0x00000001, 0x80008088, /* RC6 */
|
||||
0x00000001, 0x80000082, /* RC7 */
|
||||
0x00000000, 0x0000000b, /* RC8 */
|
||||
0x00000000, 0x0000000a, /* RC9 */
|
||||
0x00000001, 0x00008082, /* RC10 */
|
||||
0x00000000, 0x00008003, /* RC11 */
|
||||
0x00000001, 0x0000808b, /* RC12 */
|
||||
0x00000001, 0x8000000b, /* RC13 */
|
||||
0x00000001, 0x8000008a, /* RC14 */
|
||||
0x00000001, 0x80000081, /* RC15 */
|
||||
0x00000000, 0x80000081, /* RC16 */
|
||||
0x00000000, 0x80000008, /* RC17 */
|
||||
0x00000000, 0x00000083, /* RC18 */
|
||||
0x00000000, 0x80008003, /* RC19 */
|
||||
0x00000001, 0x80008088, /* RC20 */
|
||||
0x00000000, 0x80000088, /* RC21 */
|
||||
0x00000001, 0x00008000, /* RC22 */
|
||||
0x00000000, 0x80008082, /* RC23 */
|
||||
};
|
||||
|
||||
#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(fips202_armv81m_round_constants)
|
||||
|
||||
#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \
|
||||
*/
|
||||
@@ -0,0 +1,334 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) The mldsa-native project authors
|
||||
* Copyright (c) 2026 Arm Limited
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overview
|
||||
// ---------------------------------------------------------------------------
|
||||
// MVE/Helium implementation of KeccakF1600x4_StateExtractBytes
|
||||
// (inverse of state_xor_bytes_x4_mve.S).
|
||||
//
|
||||
// void KeccakF1600x4_StateExtractBytes(state, d0, d1, d2, d3, offset, length)
|
||||
//
|
||||
// Reads 'length' bytes from the bit-interleaved Keccak state starting at
|
||||
// byte 'offset', recombines the even and odd halves of each lane back
|
||||
// into plain bytes, and writes them to four output buffers (d0..d3).
|
||||
//
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bit-interleaving background
|
||||
// ---------------------------------------------------------------------------
|
||||
// Each 64-bit Keccak lane is stored as two 32-bit words:
|
||||
// even half -- bits 0, 2, 4, ..., 62 of the lane
|
||||
// odd half -- bits 1, 3, 5, ..., 63 of the lane
|
||||
// This representation allows 64-bit lane rotations (used in the Keccak
|
||||
// round function) to be implemented as pairs of 32-bit rotations.
|
||||
//
|
||||
// Batched (x4) processing:
|
||||
// Four Keccak instances are processed as a batch. Their states are
|
||||
// stored interleaved in a single 800-byte buffer: first the even
|
||||
// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes).
|
||||
// Within each 16-byte row, the four u32 words correspond to
|
||||
// instances 0..3 of the same lane, enabling SIMD-parallel operations
|
||||
// across all four instances.
|
||||
//
|
||||
// State memory layout (25 lanes x 4 instances x 2 halves):
|
||||
// S[i][l]_even/odd = even/odd half of lane l, instance i (u32)
|
||||
// Each row is 16 bytes (one Q-register).
|
||||
// Offset Contents
|
||||
// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even
|
||||
// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even
|
||||
// ...
|
||||
// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even
|
||||
// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd
|
||||
// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd
|
||||
// ...
|
||||
// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd
|
||||
//
|
||||
// ---------------------------------------------------------------------------
|
||||
// Three-phase structure
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prologue -- if offset is not 8-byte aligned, extract
|
||||
// min(length, 8-(offset%8)) bytes via predicated byte stores.
|
||||
// Main -- process full 8-byte groups: load even/odd lane pair,
|
||||
// de-interleave, scatter-store to output buffers.
|
||||
// Tail -- extract remaining <8 bytes via predicated byte stores.
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/armv81m/src/state_extract_bytes_x4_mve.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.thumb
|
||||
.syntax unified
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_extract_bytes_asm)
|
||||
|
||||
.cfi_startproc
|
||||
push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr}
|
||||
.cfi_adjust_cfa_offset 0x28
|
||||
.cfi_rel_offset r4, 0x0
|
||||
.cfi_rel_offset r5, 0x4
|
||||
.cfi_rel_offset r6, 0x8
|
||||
.cfi_rel_offset r7, 0xc
|
||||
.cfi_rel_offset r8, 0x10
|
||||
.cfi_rel_offset r9, 0x14
|
||||
.cfi_rel_offset r10, 0x18
|
||||
.cfi_rel_offset r11, 0x1c
|
||||
.cfi_rel_offset lr, 0x24
|
||||
vpush {d8, d9, d10, d11, d12, d13, d14, d15}
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
ldr r4, [sp, #0x68]
|
||||
ldr.w r10, [sp, #0x6c]
|
||||
ldr r6, [sp, #0x70]
|
||||
cmp r6, #0x0
|
||||
beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x2ea
|
||||
and r5, r10, #0x7
|
||||
bic r9, r10, #0x7
|
||||
add.w r8, r0, r9, lsl #1
|
||||
add.w r7, r8, #0x190
|
||||
cmp r5, #0x0
|
||||
beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_pre_main @ imm = #0x112
|
||||
vldrw.u32 q0, [r8], #16
|
||||
vldrw.u32 q1, [r7], #16
|
||||
vrev32.16 q2, q0
|
||||
vrev32.16 q3, q1
|
||||
vsli.32 q0, q0, #0x8
|
||||
vsli.16 q0, q0, #0x4
|
||||
vsli.8 q0, q0, #0x1
|
||||
vshr.u8 q4, q0, #0x3
|
||||
vsli.8 q0, q4, #0x4
|
||||
vshr.u8 q4, q0, #0x5
|
||||
vsli.8 q0, q4, #0x6
|
||||
vsli.32 q1, q1, #0x8
|
||||
vsli.16 q1, q1, #0x4
|
||||
vsli.8 q1, q1, #0x1
|
||||
vshr.u8 q4, q1, #0x3
|
||||
vsli.8 q1, q4, #0x4
|
||||
vshr.u8 q4, q1, #0x5
|
||||
vsli.8 q1, q4, #0x6
|
||||
mov.w r0, #0x55
|
||||
vdup.8 q4, r0
|
||||
vand q0, q0, q4
|
||||
vand q1, q1, q4
|
||||
vshl.i32 q1, q1, #0x1
|
||||
vorr q0, q0, q1
|
||||
vsli.32 q2, q2, #0x8
|
||||
vsli.16 q2, q2, #0x4
|
||||
vsli.8 q2, q2, #0x1
|
||||
vshr.u8 q1, q2, #0x3
|
||||
vsli.8 q2, q1, #0x4
|
||||
vshr.u8 q1, q2, #0x5
|
||||
vsli.8 q2, q1, #0x6
|
||||
vsli.32 q3, q3, #0x8
|
||||
vsli.16 q3, q3, #0x4
|
||||
vsli.8 q3, q3, #0x1
|
||||
vshr.u8 q1, q3, #0x3
|
||||
vsli.8 q3, q1, #0x4
|
||||
vshr.u8 q1, q3, #0x5
|
||||
vsli.8 q3, q1, #0x6
|
||||
vand q1, q2, q4
|
||||
vand q3, q3, q4
|
||||
vshl.i32 q3, q3, #0x1
|
||||
vorr q1, q1, q3
|
||||
vrev64.32 q2, q0
|
||||
vrev64.32 q3, q1
|
||||
movw r0, #0xf0f
|
||||
vmsr p0, r0
|
||||
vpsel q0, q0, q3
|
||||
vpsel q1, q2, q1
|
||||
vmov.f64 d4, d1
|
||||
vmov.f64 d6, d3
|
||||
rsb.w lr, r5, #0x8
|
||||
cmp r6, lr
|
||||
it ls
|
||||
movls lr, r6
|
||||
vctp.8 lr
|
||||
vmrs r11, p0
|
||||
lsl.w r11, r11, r5
|
||||
vmsr p0, r11
|
||||
subs r1, r1, r5
|
||||
subs r2, r2, r5
|
||||
subs r3, r3, r5
|
||||
subs r4, r4, r5
|
||||
vpstttt
|
||||
vstrbt.8 q0, [r1], #4
|
||||
vstrbt.8 q1, [r2], #4
|
||||
vstrbt.8 q2, [r3], #4
|
||||
vstrbt.8 q3, [r4], #4
|
||||
subs.w r6, r6, lr
|
||||
cmp r6, #0x0
|
||||
beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x1cc
|
||||
vmov q7[2], q7[0], r1, r3
|
||||
vmov q7[3], q7[1], r2, r4
|
||||
b Lkeccak_f1600_x4_state_extract_bytes_asm_main_body @ imm = #0xe
|
||||
|
||||
Lkeccak_f1600_x4_state_extract_bytes_asm_pre_main:
|
||||
vmov q7[2], q7[0], r1, r3
|
||||
vmov q7[3], q7[1], r2, r4
|
||||
mov.w r12, #0x4
|
||||
vsub.i32 q7, q7, r12
|
||||
|
||||
Lkeccak_f1600_x4_state_extract_bytes_asm_main_body:
|
||||
lsr.w lr, r6, #0x3
|
||||
wls lr, lr, Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_end @ imm = #0xb4
|
||||
|
||||
Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_start:
|
||||
vldrw.u32 q0, [r8], #16
|
||||
vldrw.u32 q1, [r7], #16
|
||||
vrev32.16 q2, q0
|
||||
vrev32.16 q3, q1
|
||||
vsli.32 q0, q0, #0x8
|
||||
vsli.16 q0, q0, #0x4
|
||||
vsli.8 q0, q0, #0x1
|
||||
vshr.u8 q4, q0, #0x3
|
||||
vsli.8 q0, q4, #0x4
|
||||
vshr.u8 q4, q0, #0x5
|
||||
vsli.8 q0, q4, #0x6
|
||||
vsli.32 q1, q1, #0x8
|
||||
vsli.16 q1, q1, #0x4
|
||||
vsli.8 q1, q1, #0x1
|
||||
vshr.u8 q4, q1, #0x3
|
||||
vsli.8 q1, q4, #0x4
|
||||
vshr.u8 q4, q1, #0x5
|
||||
vsli.8 q1, q4, #0x6
|
||||
mov.w r0, #0x55
|
||||
vdup.8 q4, r0
|
||||
vand q0, q0, q4
|
||||
vand q1, q1, q4
|
||||
vshl.i32 q1, q1, #0x1
|
||||
vorr q0, q0, q1
|
||||
vsli.32 q2, q2, #0x8
|
||||
vsli.16 q2, q2, #0x4
|
||||
vsli.8 q2, q2, #0x1
|
||||
vshr.u8 q1, q2, #0x3
|
||||
vsli.8 q2, q1, #0x4
|
||||
vshr.u8 q1, q2, #0x5
|
||||
vsli.8 q2, q1, #0x6
|
||||
vsli.32 q3, q3, #0x8
|
||||
vsli.16 q3, q3, #0x4
|
||||
vsli.8 q3, q3, #0x1
|
||||
vshr.u8 q1, q3, #0x3
|
||||
vsli.8 q3, q1, #0x4
|
||||
vshr.u8 q1, q3, #0x5
|
||||
vsli.8 q3, q1, #0x6
|
||||
vand q1, q2, q4
|
||||
vand q3, q3, q4
|
||||
vshl.i32 q3, q3, #0x1
|
||||
vorr q1, q1, q3
|
||||
vstrw.32 q0, [q7, #4]!
|
||||
vstrw.32 q1, [q7, #4]!
|
||||
le lr, Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_start @ imm = #-0xb4
|
||||
|
||||
Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_end:
|
||||
ands r6, r6, #0x7
|
||||
beq Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0xee
|
||||
mov.w r12, #0x4
|
||||
vadd.i32 q7, q7, r12
|
||||
vmov r1, r3, q7[2], q7[0]
|
||||
vmov r2, r4, q7[3], q7[1]
|
||||
vldrw.u32 q0, [r8], #16
|
||||
vldrw.u32 q1, [r7], #16
|
||||
vrev32.16 q2, q0
|
||||
vrev32.16 q3, q1
|
||||
vsli.32 q0, q0, #0x8
|
||||
vsli.16 q0, q0, #0x4
|
||||
vsli.8 q0, q0, #0x1
|
||||
vshr.u8 q4, q0, #0x3
|
||||
vsli.8 q0, q4, #0x4
|
||||
vshr.u8 q4, q0, #0x5
|
||||
vsli.8 q0, q4, #0x6
|
||||
vsli.32 q1, q1, #0x8
|
||||
vsli.16 q1, q1, #0x4
|
||||
vsli.8 q1, q1, #0x1
|
||||
vshr.u8 q4, q1, #0x3
|
||||
vsli.8 q1, q4, #0x4
|
||||
vshr.u8 q4, q1, #0x5
|
||||
vsli.8 q1, q4, #0x6
|
||||
mov.w r0, #0x55
|
||||
vdup.8 q4, r0
|
||||
vand q0, q0, q4
|
||||
vand q1, q1, q4
|
||||
vshl.i32 q1, q1, #0x1
|
||||
vorr q0, q0, q1
|
||||
vsli.32 q2, q2, #0x8
|
||||
vsli.16 q2, q2, #0x4
|
||||
vsli.8 q2, q2, #0x1
|
||||
vshr.u8 q1, q2, #0x3
|
||||
vsli.8 q2, q1, #0x4
|
||||
vshr.u8 q1, q2, #0x5
|
||||
vsli.8 q2, q1, #0x6
|
||||
vsli.32 q3, q3, #0x8
|
||||
vsli.16 q3, q3, #0x4
|
||||
vsli.8 q3, q3, #0x1
|
||||
vshr.u8 q1, q3, #0x3
|
||||
vsli.8 q3, q1, #0x4
|
||||
vshr.u8 q1, q3, #0x5
|
||||
vsli.8 q3, q1, #0x6
|
||||
vand q1, q2, q4
|
||||
vand q3, q3, q4
|
||||
vshl.i32 q3, q3, #0x1
|
||||
vorr q1, q1, q3
|
||||
vrev64.32 q2, q0
|
||||
vrev64.32 q3, q1
|
||||
movw r0, #0xf0f
|
||||
vmsr p0, r0
|
||||
vpsel q0, q0, q3
|
||||
vpsel q1, q2, q1
|
||||
vmov.f64 d4, d1
|
||||
vmov.f64 d6, d3
|
||||
vctp.8 r6
|
||||
vpstttt
|
||||
vstrbt.8 q0, [r1], #4
|
||||
vstrbt.8 q1, [r2], #4
|
||||
vstrbt.8 q2, [r3], #4
|
||||
vstrbt.8 q3, [r4], #4
|
||||
|
||||
Lkeccak_f1600_x4_state_extract_bytes_asm_exit:
|
||||
vpop {d8, d9, d10, d11, d12, d13, d14, d15}
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc}
|
||||
.cfi_restore r4
|
||||
.cfi_restore r5
|
||||
.cfi_restore r6
|
||||
.cfi_restore r7
|
||||
.cfi_restore r8
|
||||
.cfi_restore r9
|
||||
.cfi_restore r10
|
||||
.cfi_restore r11
|
||||
.cfi_restore lr
|
||||
.cfi_adjust_cfa_offset -0x28
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x4_state_extract_bytes_asm)
|
||||
|
||||
#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,355 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) The mldsa-native project authors
|
||||
* Copyright (c) 2026 Arm Limited
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overview
|
||||
// ---------------------------------------------------------------------------
|
||||
// MVE/Helium implementation of KeccakF1600x4_StateXORBytes.
|
||||
//
|
||||
// void KeccakF1600x4_StateXORBytes(state, d0, d1, d2, d3, offset, length)
|
||||
//
|
||||
// Reads 'length' plain bytes from each of four input buffers (d0..d3),
|
||||
// splits every byte into its even and odd bits (bit-interleaving), and
|
||||
// XORs the result into the Keccak state starting at byte 'offset'.
|
||||
//
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bit-interleaving background
|
||||
// ---------------------------------------------------------------------------
|
||||
// Each 64-bit Keccak lane is stored as two 32-bit words:
|
||||
// even half -- bits 0, 2, 4, ..., 62 of the lane
|
||||
// odd half -- bits 1, 3, 5, ..., 63 of the lane
|
||||
// This representation allows 64-bit lane rotations (used in the Keccak
|
||||
// round function) to be implemented as pairs of 32-bit rotations.
|
||||
//
|
||||
// Batched (x4) processing:
|
||||
// Four Keccak instances are processed as a batch. Their states are
|
||||
// stored interleaved in a single 800-byte buffer: first the even
|
||||
// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes).
|
||||
// Within each 16-byte row, the four u32 words correspond to
|
||||
// instances 0..3 of the same lane, enabling SIMD-parallel operations
|
||||
// across all four instances.
|
||||
//
|
||||
// State memory layout (25 lanes x 4 instances x 2 halves):
|
||||
// S[i][l]_even/odd = even/odd half of lane l, instance i (u32)
|
||||
// Each row is 16 bytes (one Q-register).
|
||||
// Offset Contents
|
||||
// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even
|
||||
// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even
|
||||
// ...
|
||||
// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even
|
||||
// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd
|
||||
// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd
|
||||
// ...
|
||||
// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd
|
||||
//
|
||||
// ---------------------------------------------------------------------------
|
||||
// Three-phase structure
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prologue -- if offset is not 8-byte aligned, absorb
|
||||
// min(length, 8-(offset%8)) bytes via predicated byte loads.
|
||||
// Main -- process full 8-byte groups via word-level gather loads,
|
||||
// bit-interleave, then VEOR into even/odd state halves.
|
||||
// Tail -- absorb remaining <8 bytes via predicated byte loads.
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.thumb
|
||||
.syntax unified
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm)
|
||||
|
||||
.cfi_startproc
|
||||
push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr}
|
||||
.cfi_adjust_cfa_offset 0x28
|
||||
.cfi_rel_offset r4, 0x0
|
||||
.cfi_rel_offset r5, 0x4
|
||||
.cfi_rel_offset r6, 0x8
|
||||
.cfi_rel_offset r7, 0xc
|
||||
.cfi_rel_offset r8, 0x10
|
||||
.cfi_rel_offset r9, 0x14
|
||||
.cfi_rel_offset r10, 0x18
|
||||
.cfi_rel_offset r11, 0x1c
|
||||
.cfi_rel_offset lr, 0x24
|
||||
vpush {d8, d9, d10, d11, d12, d13, d14, d15}
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
ldr r4, [sp, #0x68]
|
||||
ldr.w r10, [sp, #0x6c]
|
||||
ldr r6, [sp, #0x70]
|
||||
cmp r6, #0x0
|
||||
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x346
|
||||
and r5, r10, #0x7
|
||||
bic r9, r10, #0x7
|
||||
add.w r8, r0, r9, lsl #1
|
||||
add.w r7, r8, #0x190
|
||||
cmp r5, #0x0
|
||||
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x12c
|
||||
subs r1, r1, r5
|
||||
subs r2, r2, r5
|
||||
subs r3, r3, r5
|
||||
subs r4, r4, r5
|
||||
rsb.w lr, r5, #0x8
|
||||
cmp r6, lr
|
||||
it ls
|
||||
movls lr, r6
|
||||
subs.w r6, r6, lr
|
||||
vctp.8 lr
|
||||
vmrs r11, p0
|
||||
lsl.w r11, r11, r5
|
||||
vmsr p0, r11
|
||||
vpstttt
|
||||
vldrbt.u8 q0, [r1], #4
|
||||
vldrbt.u8 q1, [r2], #4
|
||||
vldrbt.u8 q2, [r3], #4
|
||||
vldrbt.u8 q3, [r4], #4
|
||||
vmov.f64 d1, d4
|
||||
vmov.f64 d3, d6
|
||||
vrev64.32 q2, q0
|
||||
vrev64.32 q3, q1
|
||||
movw r0, #0xf0f
|
||||
vmsr p0, r0
|
||||
vpsel q0, q0, q3
|
||||
vpsel q1, q2, q1
|
||||
vmov q2, q0
|
||||
vmov q3, q1
|
||||
vshr.u8 q4, q0, #0x2
|
||||
vsli.8 q0, q4, #0x1
|
||||
vshr.u8 q4, q0, #0x3
|
||||
vsli.8 q0, q4, #0x2
|
||||
vshr.u8 q4, q0, #0x4
|
||||
vsli.8 q0, q4, #0x3
|
||||
vshr.u16 q4, q0, #0x8
|
||||
vsli.8 q0, q4, #0x4
|
||||
vshr.u32 q4, q0, #0x10
|
||||
vsli.16 q0, q4, #0x8
|
||||
vshr.u8 q4, q3, #0x2
|
||||
vsli.8 q3, q4, #0x1
|
||||
vshr.u8 q4, q3, #0x3
|
||||
vsli.8 q3, q4, #0x2
|
||||
vshr.u8 q4, q3, #0x4
|
||||
vsli.8 q3, q4, #0x3
|
||||
vshr.u16 q4, q3, #0x8
|
||||
vsli.8 q3, q4, #0x4
|
||||
vshr.u32 q4, q3, #0x10
|
||||
vsli.16 q3, q4, #0x8
|
||||
vsli.32 q0, q3, #0x10
|
||||
vshl.i8 q4, q2, #0x2
|
||||
vsri.8 q2, q4, #0x1
|
||||
vshl.i8 q4, q2, #0x3
|
||||
vsri.8 q2, q4, #0x2
|
||||
vshl.i8 q4, q2, #0x4
|
||||
vsri.8 q2, q4, #0x3
|
||||
vshl.i16 q4, q2, #0x8
|
||||
vsri.8 q2, q4, #0x4
|
||||
vshl.i32 q4, q2, #0x10
|
||||
vsri.16 q2, q4, #0x8
|
||||
vshl.i8 q4, q1, #0x2
|
||||
vsri.8 q1, q4, #0x1
|
||||
vshl.i8 q4, q1, #0x3
|
||||
vsri.8 q1, q4, #0x2
|
||||
vshl.i8 q4, q1, #0x4
|
||||
vsri.8 q1, q4, #0x3
|
||||
vshl.i16 q4, q1, #0x8
|
||||
vsri.8 q1, q4, #0x4
|
||||
vshl.i32 q4, q1, #0x10
|
||||
vsri.16 q1, q4, #0x8
|
||||
vsri.32 q1, q2, #0x10
|
||||
vldrw.u32 q4, [r8]
|
||||
vldrw.u32 q5, [r7]
|
||||
veor q4, q4, q0
|
||||
veor q5, q5, q1
|
||||
vstrw.32 q4, [r8], #16
|
||||
vstrw.32 q5, [r7], #16
|
||||
vmov q7[2], q7[0], r1, r3
|
||||
vmov q7[3], q7[1], r2, r4
|
||||
b Lkeccak_f1600_x4_state_xor_bytes_asm_main_body @ imm = #0xe
|
||||
|
||||
Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main:
|
||||
vmov q7[2], q7[0], r1, r3
|
||||
vmov q7[3], q7[1], r2, r4
|
||||
mov.w r0, #0x4
|
||||
vsub.i32 q7, q7, r0
|
||||
|
||||
Lkeccak_f1600_x4_state_xor_bytes_asm_main_body:
|
||||
lsr.w lr, r6, #0x3
|
||||
wls lr, lr, Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end @ imm = #0xd4
|
||||
|
||||
Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_start:
|
||||
vldrw.u32 q0, [q7, #4]!
|
||||
vldrw.u32 q1, [q7, #4]!
|
||||
vmov q2, q0
|
||||
vmov q3, q1
|
||||
vshr.u8 q4, q0, #0x2
|
||||
vsli.8 q0, q4, #0x1
|
||||
vshr.u8 q4, q0, #0x3
|
||||
vsli.8 q0, q4, #0x2
|
||||
vshr.u8 q4, q0, #0x4
|
||||
vsli.8 q0, q4, #0x3
|
||||
vshr.u16 q4, q0, #0x8
|
||||
vsli.8 q0, q4, #0x4
|
||||
vshr.u32 q4, q0, #0x10
|
||||
vsli.16 q0, q4, #0x8
|
||||
vshr.u8 q4, q3, #0x2
|
||||
vsli.8 q3, q4, #0x1
|
||||
vshr.u8 q4, q3, #0x3
|
||||
vsli.8 q3, q4, #0x2
|
||||
vshr.u8 q4, q3, #0x4
|
||||
vsli.8 q3, q4, #0x3
|
||||
vshr.u16 q4, q3, #0x8
|
||||
vsli.8 q3, q4, #0x4
|
||||
vshr.u32 q4, q3, #0x10
|
||||
vsli.16 q3, q4, #0x8
|
||||
vsli.32 q0, q3, #0x10
|
||||
vshl.i8 q4, q2, #0x2
|
||||
vsri.8 q2, q4, #0x1
|
||||
vshl.i8 q4, q2, #0x3
|
||||
vsri.8 q2, q4, #0x2
|
||||
vshl.i8 q4, q2, #0x4
|
||||
vsri.8 q2, q4, #0x3
|
||||
vshl.i16 q4, q2, #0x8
|
||||
vsri.8 q2, q4, #0x4
|
||||
vshl.i32 q4, q2, #0x10
|
||||
vsri.16 q2, q4, #0x8
|
||||
vshl.i8 q4, q1, #0x2
|
||||
vsri.8 q1, q4, #0x1
|
||||
vshl.i8 q4, q1, #0x3
|
||||
vsri.8 q1, q4, #0x2
|
||||
vshl.i8 q4, q1, #0x4
|
||||
vsri.8 q1, q4, #0x3
|
||||
vshl.i16 q4, q1, #0x8
|
||||
vsri.8 q1, q4, #0x4
|
||||
vshl.i32 q4, q1, #0x10
|
||||
vsri.16 q1, q4, #0x8
|
||||
vsri.32 q1, q2, #0x10
|
||||
vldrw.u32 q4, [r8]
|
||||
vldrw.u32 q5, [r7]
|
||||
veor q4, q4, q0
|
||||
veor q5, q5, q1
|
||||
vstrw.32 q4, [r8], #16
|
||||
vstrw.32 q5, [r7], #16
|
||||
le lr, Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_start @ imm = #-0xd4
|
||||
|
||||
Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end:
|
||||
ands r6, r6, #0x7
|
||||
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x110
|
||||
mov.w r0, #0x4
|
||||
vadd.i32 q7, q7, r0
|
||||
vmov r1, r3, q7[2], q7[0]
|
||||
vmov r2, r4, q7[3], q7[1]
|
||||
vctp.8 r6
|
||||
vpstttt
|
||||
vldrbt.u8 q0, [r1]
|
||||
vldrbt.u8 q1, [r2]
|
||||
vldrbt.u8 q2, [r3]
|
||||
vldrbt.u8 q3, [r4]
|
||||
vmov.f64 d1, d4
|
||||
vmov.f64 d3, d6
|
||||
vrev64.32 q2, q0
|
||||
vrev64.32 q3, q1
|
||||
movw r0, #0xf0f
|
||||
vmsr p0, r0
|
||||
vpsel q0, q0, q3
|
||||
vpsel q1, q2, q1
|
||||
vmov q2, q0
|
||||
vmov q3, q1
|
||||
vshr.u8 q4, q0, #0x2
|
||||
vsli.8 q0, q4, #0x1
|
||||
vshr.u8 q4, q0, #0x3
|
||||
vsli.8 q0, q4, #0x2
|
||||
vshr.u8 q4, q0, #0x4
|
||||
vsli.8 q0, q4, #0x3
|
||||
vshr.u16 q4, q0, #0x8
|
||||
vsli.8 q0, q4, #0x4
|
||||
vshr.u32 q4, q0, #0x10
|
||||
vsli.16 q0, q4, #0x8
|
||||
vshr.u8 q4, q3, #0x2
|
||||
vsli.8 q3, q4, #0x1
|
||||
vshr.u8 q4, q3, #0x3
|
||||
vsli.8 q3, q4, #0x2
|
||||
vshr.u8 q4, q3, #0x4
|
||||
vsli.8 q3, q4, #0x3
|
||||
vshr.u16 q4, q3, #0x8
|
||||
vsli.8 q3, q4, #0x4
|
||||
vshr.u32 q4, q3, #0x10
|
||||
vsli.16 q3, q4, #0x8
|
||||
vsli.32 q0, q3, #0x10
|
||||
vshl.i8 q4, q2, #0x2
|
||||
vsri.8 q2, q4, #0x1
|
||||
vshl.i8 q4, q2, #0x3
|
||||
vsri.8 q2, q4, #0x2
|
||||
vshl.i8 q4, q2, #0x4
|
||||
vsri.8 q2, q4, #0x3
|
||||
vshl.i16 q4, q2, #0x8
|
||||
vsri.8 q2, q4, #0x4
|
||||
vshl.i32 q4, q2, #0x10
|
||||
vsri.16 q2, q4, #0x8
|
||||
vshl.i8 q4, q1, #0x2
|
||||
vsri.8 q1, q4, #0x1
|
||||
vshl.i8 q4, q1, #0x3
|
||||
vsri.8 q1, q4, #0x2
|
||||
vshl.i8 q4, q1, #0x4
|
||||
vsri.8 q1, q4, #0x3
|
||||
vshl.i16 q4, q1, #0x8
|
||||
vsri.8 q1, q4, #0x4
|
||||
vshl.i32 q4, q1, #0x10
|
||||
vsri.16 q1, q4, #0x8
|
||||
vsri.32 q1, q2, #0x10
|
||||
vldrw.u32 q4, [r8]
|
||||
vldrw.u32 q5, [r7]
|
||||
veor q4, q4, q0
|
||||
veor q5, q5, q1
|
||||
vstrw.32 q4, [r8], #16
|
||||
vstrw.32 q5, [r7], #16
|
||||
|
||||
Lkeccak_f1600_x4_state_xor_bytes_asm_exit:
|
||||
vpop {d8, d9, d10, d11, d12, d13, d14, d15}
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc}
|
||||
.cfi_restore r4
|
||||
.cfi_restore r5
|
||||
.cfi_restore r6
|
||||
.cfi_restore r7
|
||||
.cfi_restore r8
|
||||
.cfi_restore r9
|
||||
.cfi_restore r10
|
||||
.cfi_restore r11
|
||||
.cfi_restore lr
|
||||
.cfi_adjust_cfa_offset -0x28
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x4_state_xor_bytes_asm)
|
||||
|
||||
#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_AUTO_H
|
||||
#define MLK_FIPS202_NATIVE_AUTO_H
|
||||
|
||||
/*
|
||||
* Default FIPS202 backend
|
||||
*/
|
||||
#include "../../sys.h"
|
||||
|
||||
#if defined(MLK_SYS_AARCH64)
|
||||
#include "aarch64/auto.h"
|
||||
#endif
|
||||
|
||||
/* The x86_64 backend requires toolchain support for the SysV ABI */
|
||||
#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED)
|
||||
#include "x86_64/keccak_f1600_x4_avx2.h"
|
||||
#endif
|
||||
|
||||
/* We do not yet include the FIPS202 backend for Armv8.1-M+MVE by default
|
||||
* as it is still experimental and undergoing review. */
|
||||
/* #if defined(MLK_SYS_ARMV81M_MVE) */
|
||||
/* #include "armv81m/mve.h" */
|
||||
/* #endif */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_AUTO_H */
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H
|
||||
#define MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#define MLK_FIPS202_X86_64_NEED_X4_AVX2
|
||||
|
||||
/* Part of backend API */
|
||||
#define MLK_USE_FIPS202_X4_NATIVE
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/fips202_native_x86_64.h"
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state)
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_keccak_f1600_x4_avx2_asm(state, mlk_keccakf1600_round_constants,
|
||||
mlk_keccak_rho8, mlk_keccak_rho56);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H */
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H
|
||||
#define MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H
|
||||
|
||||
#include "../../../../cbmc.h"
|
||||
#include "../../../../common.h"
|
||||
|
||||
/* TODO: Reconsider whether this check is needed -- x86_64 is always
|
||||
* little-endian, so the backend selection already implies this. */
|
||||
#ifndef MLK_SYS_LITTLE_ENDIAN
|
||||
#error Expecting a little-endian platform
|
||||
#endif
|
||||
|
||||
#define mlk_keccakf1600_round_constants \
|
||||
MLK_NAMESPACE(keccakf1600_round_constants)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint64_t
|
||||
mlk_keccakf1600_round_constants[24];
|
||||
|
||||
#define mlk_keccak_rho8 MLK_NAMESPACE(keccak_rho8)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho8[4];
|
||||
|
||||
#define mlk_keccak_rho56 MLK_NAMESPACE(keccak_rho56)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho56[4];
|
||||
|
||||
#define mlk_keccak_f1600_x4_avx2_asm MLK_NAMESPACE(keccak_f1600_x4_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24],
|
||||
const uint64_t rho8[4],
|
||||
const uint64_t rho56[4])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/keccak_f1600_x4_avx2_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(states, sizeof(uint64_t) * 25 * 4))
|
||||
requires(rc == mlk_keccakf1600_round_constants)
|
||||
requires(rho8 == mlk_keccak_rho8)
|
||||
requires(rho56 == mlk_keccak_rho56)
|
||||
assigns(memory_slice(states, sizeof(uint64_t) * 25 * 4))
|
||||
);
|
||||
|
||||
#endif /* !MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H */
|
||||
@@ -0,0 +1,487 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#include "../../../../common.h"
|
||||
|
||||
#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
/*yaml
|
||||
Name: keccak_f1600_x4_avx2_asm
|
||||
Description: x86_64 AVX2 Keccak-f[1600] permutation for four sequential states
|
||||
Signature: void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], const uint64_t rho8[4], const uint64_t rho56[4])
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 800
|
||||
permissions: read/write
|
||||
c_parameter: uint64_t states[100]
|
||||
description: Four sequential Keccak states (4 x 25 x uint64_t)
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 192
|
||||
permissions: read-only
|
||||
c_parameter: const uint64_t rc[24]
|
||||
description: Round constants (24 x uint64_t)
|
||||
rdx:
|
||||
type: buffer
|
||||
size_bytes: 32
|
||||
permissions: read-only
|
||||
c_parameter: const uint64_t rho8[4]
|
||||
description: Rotation constant rho8 (4 x uint64_t)
|
||||
rcx:
|
||||
type: buffer
|
||||
size_bytes: 32
|
||||
permissions: read-only
|
||||
c_parameter: const uint64_t rho56[4]
|
||||
description: Rotation constant rho56 (4 x uint64_t)
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/fips202/x86_64/src/keccak_f1600_x4_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(keccak_f1600_x4_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(keccak_f1600_x4_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movq %rsp, %r11
|
||||
.cfi_def_cfa_register %r11
|
||||
andq $-0x20, %rsp
|
||||
subq $0x300, %rsp # imm = 0x300
|
||||
vmovdqu (%rdi), %ymm0
|
||||
vmovdqu 0xc8(%rdi), %ymm3
|
||||
vmovdqu 0x190(%rdi), %ymm1
|
||||
vmovdqu 0x258(%rdi), %ymm4
|
||||
vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
|
||||
vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1]
|
||||
vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
|
||||
vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3]
|
||||
vmovdqu 0x278(%rdi), %ymm4
|
||||
vmovdqu %ymm3, 0x40(%rsp)
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm7, (%rsp)
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1]
|
||||
vmovdqu 0x20(%rdi), %ymm0
|
||||
vmovdqu 0x1b0(%rdi), %ymm1
|
||||
vmovdqu %ymm3, 0x60(%rsp)
|
||||
vmovdqu 0xe8(%rdi), %ymm3
|
||||
vmovdqu %ymm7, 0x20(%rsp)
|
||||
vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
|
||||
vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1]
|
||||
vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
|
||||
vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3]
|
||||
vmovdqu 0x298(%rdi), %ymm4
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm14 # ymm14 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm7, 0x80(%rsp)
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1]
|
||||
vmovdqu 0x40(%rdi), %ymm0
|
||||
vmovdqu 0x1d0(%rdi), %ymm1
|
||||
vmovdqu %ymm3, 0xc0(%rsp)
|
||||
vmovdqu 0x108(%rdi), %ymm3
|
||||
vmovdqu %ymm14, %ymm10
|
||||
vmovdqu %ymm7, 0xa0(%rsp)
|
||||
vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
|
||||
vperm2i128 $0x20, %ymm3, %ymm2, %ymm11 # ymm11 = ymm2[0,1],ymm3[0,1]
|
||||
vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3]
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1]
|
||||
vmovdqu %ymm3, 0x100(%rsp)
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm8 # ymm8 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu 0x128(%rdi), %ymm3
|
||||
vmovdqu 0x60(%rdi), %ymm0
|
||||
vmovdqu 0x1f0(%rdi), %ymm1
|
||||
vmovdqu %ymm7, 0xe0(%rsp)
|
||||
vmovdqu %ymm11, %ymm14
|
||||
vmovdqu 0x2b8(%rdi), %ymm4
|
||||
vmovdqu 0x2f8(%rdi), %ymm5
|
||||
vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
|
||||
vmovdqu 0x2d8(%rdi), %ymm4
|
||||
vperm2i128 $0x20, %ymm3, %ymm2, %ymm15 # ymm15 = ymm2[0,1],ymm3[0,1]
|
||||
vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3]
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1]
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm9 # ymm9 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm3, 0x140(%rsp)
|
||||
vmovdqu 0x80(%rdi), %ymm0
|
||||
vmovdqu 0x148(%rdi), %ymm3
|
||||
vmovdqu 0x210(%rdi), %ymm1
|
||||
vmovdqu %ymm7, 0x120(%rsp)
|
||||
vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
|
||||
vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1]
|
||||
vperm2i128 $0x31, %ymm3, %ymm2, %ymm13 # ymm13 = ymm2[2,3],ymm3[2,3]
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm7, 0x160(%rsp)
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1]
|
||||
vmovdqu 0xa0(%rdi), %ymm0
|
||||
vmovdqu 0x230(%rdi), %ymm1
|
||||
vmovdqu %ymm3, 0x1a0(%rsp)
|
||||
vmovdqu 0x168(%rdi), %ymm3
|
||||
vpunpcklqdq %ymm5, %ymm1, %ymm4 # ymm4 = ymm1[0],ymm5[0],ymm1[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm5[1],ymm1[3],ymm5[3]
|
||||
vmovdqu %ymm7, 0x180(%rsp)
|
||||
vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
|
||||
vperm2i128 $0x20, %ymm4, %ymm2, %ymm12 # ymm12 = ymm2[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[0,1],ymm1[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm2, %ymm7 # ymm7 = ymm2[2,3],ymm4[2,3]
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[2,3],ymm1[2,3]
|
||||
vmovq 0x250(%rdi), %xmm0
|
||||
vmovq 0xc0(%rdi), %xmm1
|
||||
vmovdqu %ymm12, 0x1c0(%rsp)
|
||||
vmovdqu %ymm4, 0x1e0(%rsp)
|
||||
vpinsrq $0x1, 0x318(%rdi), %xmm0, %xmm0
|
||||
vpinsrq $0x1, 0x188(%rdi), %xmm1, %xmm1
|
||||
vinserti128 $0x1, %xmm0, %ymm1, %ymm2
|
||||
movq $0x0, %r10
|
||||
|
||||
LLkeccak_f1600_x4_avx2:
|
||||
vmovdqu 0xa0(%rsp), %ymm4
|
||||
vpxor 0x1c0(%rsp), %ymm9, %ymm0
|
||||
vmovdqu %ymm9, 0x200(%rsp)
|
||||
vmovdqu %ymm10, %ymm9
|
||||
vmovdqu 0xc0(%rsp), %ymm11
|
||||
vmovdqu 0x160(%rsp), %ymm12
|
||||
vmovdqu %ymm3, 0x240(%rsp)
|
||||
vpxor 0x100(%rsp), %ymm4, %ymm1
|
||||
vmovdqu 0x40(%rsp), %ymm10
|
||||
vmovdqu %ymm4, 0x220(%rsp)
|
||||
vpxor %ymm3, %ymm12, %ymm12
|
||||
vmovdqu 0x20(%rsp), %ymm6
|
||||
vmovdqu 0x140(%rsp), %ymm4
|
||||
vmovdqu %ymm14, 0x2a0(%rsp)
|
||||
vpxor %ymm1, %ymm0, %ymm0
|
||||
vpxor %ymm8, %ymm11, %ymm1
|
||||
vpxor 0x180(%rsp), %ymm7, %ymm11
|
||||
vmovdqu %ymm10, 0x280(%rsp)
|
||||
vpxor %ymm1, %ymm12, %ymm12
|
||||
vpxor %ymm15, %ymm9, %ymm1
|
||||
vmovdqu 0xe0(%rsp), %ymm3
|
||||
vmovdqu %ymm8, 0x260(%rsp)
|
||||
vpxor %ymm1, %ymm11, %ymm11
|
||||
vpxor 0x120(%rsp), %ymm14, %ymm1
|
||||
vpxor %ymm6, %ymm12, %ymm12
|
||||
vmovdqu 0x60(%rsp), %ymm8
|
||||
vpxor %ymm10, %ymm11, %ymm11
|
||||
vpxor 0x1e0(%rsp), %ymm13, %ymm10
|
||||
vpxor %ymm4, %ymm3, %ymm3
|
||||
vmovdqu %ymm4, 0x2c0(%rsp)
|
||||
vpsrlq $0x3f, %ymm12, %ymm4
|
||||
vpsrlq $0x3f, %ymm11, %ymm5
|
||||
vpxor (%rsp), %ymm0, %ymm0
|
||||
vpxor %ymm1, %ymm10, %ymm10
|
||||
vmovdqu 0x80(%rsp), %ymm1
|
||||
vpxor %ymm8, %ymm10, %ymm10
|
||||
vmovdqu %ymm1, %ymm14
|
||||
vpxor 0x1a0(%rsp), %ymm2, %ymm1
|
||||
vmovdqu %ymm14, 0x2e0(%rsp)
|
||||
vpxor %ymm3, %ymm1, %ymm1
|
||||
vpsllq $0x1, %ymm12, %ymm3
|
||||
vpor %ymm4, %ymm3, %ymm3
|
||||
vpsllq $0x1, %ymm11, %ymm4
|
||||
vpxor %ymm14, %ymm1, %ymm1
|
||||
vpor %ymm5, %ymm4, %ymm4
|
||||
vpsrlq $0x3f, %ymm10, %ymm14
|
||||
vpxor %ymm1, %ymm3, %ymm3
|
||||
vpsllq $0x1, %ymm10, %ymm5
|
||||
vpxor %ymm0, %ymm4, %ymm4
|
||||
vpor %ymm14, %ymm5, %ymm5
|
||||
vpxor %ymm6, %ymm4, %ymm6
|
||||
vpxor %ymm12, %ymm5, %ymm5
|
||||
vpsrlq $0x3f, %ymm1, %ymm12
|
||||
vpsllq $0x1, %ymm1, %ymm1
|
||||
vpxor %ymm7, %ymm5, %ymm7
|
||||
vpxor %ymm9, %ymm5, %ymm9
|
||||
vpor %ymm12, %ymm1, %ymm1
|
||||
vpxor (%rsp), %ymm3, %ymm12
|
||||
vpxor %ymm11, %ymm1, %ymm1
|
||||
vpsrlq $0x3f, %ymm0, %ymm11
|
||||
vpsllq $0x1, %ymm0, %ymm0
|
||||
vpxor %ymm13, %ymm1, %ymm13
|
||||
vpxor %ymm8, %ymm1, %ymm8
|
||||
vpor %ymm11, %ymm0, %ymm0
|
||||
vpxor %ymm10, %ymm0, %ymm0
|
||||
vpxor 0xc0(%rsp), %ymm4, %ymm10
|
||||
vpxor %ymm2, %ymm0, %ymm2
|
||||
vpsrlq $0x14, %ymm10, %ymm11
|
||||
vpsllq $0x2c, %ymm10, %ymm10
|
||||
vpor %ymm11, %ymm10, %ymm10
|
||||
vpxor %ymm15, %ymm5, %ymm11
|
||||
vpbroadcastq (%rsi), %ymm15
|
||||
vpsrlq $0x15, %ymm11, %ymm14
|
||||
vpsllq $0x2b, %ymm11, %ymm11
|
||||
vpor %ymm14, %ymm11, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm14
|
||||
vpxor %ymm15, %ymm14, %ymm14
|
||||
vpxor %ymm12, %ymm14, %ymm15
|
||||
vpsrlq $0x2b, %ymm13, %ymm14
|
||||
vpsllq $0x15, %ymm13, %ymm13
|
||||
vmovdqu %ymm15, (%rsp)
|
||||
vpor %ymm14, %ymm13, %ymm13
|
||||
vpandn %ymm13, %ymm11, %ymm14
|
||||
vpxor %ymm10, %ymm14, %ymm15
|
||||
vpsrlq $0x32, %ymm2, %ymm14
|
||||
vpsllq $0xe, %ymm2, %ymm2
|
||||
vmovdqu %ymm15, 0x20(%rsp)
|
||||
vpor %ymm14, %ymm2, %ymm2
|
||||
vpandn %ymm2, %ymm13, %ymm14
|
||||
vpxor %ymm11, %ymm14, %ymm11
|
||||
vmovdqu %ymm11, 0x40(%rsp)
|
||||
vpandn %ymm12, %ymm2, %ymm11
|
||||
vpandn %ymm10, %ymm12, %ymm12
|
||||
vpxor %ymm13, %ymm11, %ymm11
|
||||
vmovdqu %ymm11, 0x60(%rsp)
|
||||
vpxor %ymm2, %ymm12, %ymm11
|
||||
vpsrlq $0x24, %ymm8, %ymm2
|
||||
vpsllq $0x1c, %ymm8, %ymm8
|
||||
vmovdqu %ymm11, 0x80(%rsp)
|
||||
vpor %ymm2, %ymm8, %ymm8
|
||||
vpxor 0xe0(%rsp), %ymm0, %ymm2
|
||||
vpsrlq $0x2c, %ymm2, %ymm10
|
||||
vpsllq $0x14, %ymm2, %ymm2
|
||||
vpor %ymm10, %ymm2, %ymm2
|
||||
vpxor 0x100(%rsp), %ymm3, %ymm10
|
||||
vpsrlq $0x3d, %ymm10, %ymm11
|
||||
vpsllq $0x3, %ymm10, %ymm10
|
||||
vpor %ymm11, %ymm10, %ymm10
|
||||
vpandn %ymm10, %ymm2, %ymm11
|
||||
vpxor %ymm8, %ymm11, %ymm11
|
||||
vmovdqu %ymm11, 0xa0(%rsp)
|
||||
vpxor 0x160(%rsp), %ymm4, %ymm11
|
||||
vpsrlq $0x13, %ymm11, %ymm12
|
||||
vpsllq $0x2d, %ymm11, %ymm11
|
||||
vpor %ymm12, %ymm11, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm12
|
||||
vpxor %ymm2, %ymm12, %ymm12
|
||||
vmovdqu %ymm12, 0xc0(%rsp)
|
||||
vpsrlq $0x3, %ymm7, %ymm12
|
||||
vpsllq $0x3d, %ymm7, %ymm7
|
||||
vpor %ymm12, %ymm7, %ymm7
|
||||
vpandn %ymm7, %ymm11, %ymm12
|
||||
vpxor %ymm10, %ymm12, %ymm10
|
||||
vpandn %ymm8, %ymm7, %ymm12
|
||||
vpandn %ymm2, %ymm8, %ymm8
|
||||
vpsrlq $0x3f, %ymm6, %ymm2
|
||||
vpsllq $0x1, %ymm6, %ymm6
|
||||
vpxor %ymm11, %ymm12, %ymm14
|
||||
vpor %ymm2, %ymm6, %ymm6
|
||||
vpsrlq $0x3a, %ymm9, %ymm2
|
||||
vpxor %ymm7, %ymm8, %ymm12
|
||||
vpsllq $0x6, %ymm9, %ymm9
|
||||
vmovdqu %ymm12, 0xe0(%rsp)
|
||||
vpxor 0x1a0(%rsp), %ymm0, %ymm7
|
||||
vpor %ymm2, %ymm9, %ymm9
|
||||
vpxor 0x120(%rsp), %ymm1, %ymm2
|
||||
vpshufb (%rdx), %ymm7, %ymm7
|
||||
vpsrlq $0x27, %ymm2, %ymm11
|
||||
vpsllq $0x19, %ymm2, %ymm2
|
||||
vpor %ymm2, %ymm11, %ymm11
|
||||
vpandn %ymm11, %ymm9, %ymm2
|
||||
vpandn %ymm7, %ymm11, %ymm8
|
||||
vpxor %ymm6, %ymm2, %ymm12
|
||||
vpxor 0x1c0(%rsp), %ymm3, %ymm2
|
||||
vpxor %ymm9, %ymm8, %ymm8
|
||||
vmovdqu %ymm12, 0x100(%rsp)
|
||||
vpsrlq $0x2e, %ymm2, %ymm12
|
||||
vpsllq $0x12, %ymm2, %ymm2
|
||||
vpor %ymm2, %ymm12, %ymm2
|
||||
vpandn %ymm2, %ymm7, %ymm12
|
||||
vpxor %ymm11, %ymm12, %ymm15
|
||||
vpandn %ymm6, %ymm2, %ymm11
|
||||
vpandn %ymm9, %ymm6, %ymm6
|
||||
vpxor %ymm7, %ymm11, %ymm12
|
||||
vmovdqu %ymm12, 0x120(%rsp)
|
||||
vpxor %ymm2, %ymm6, %ymm12
|
||||
vpxor 0x2e0(%rsp), %ymm0, %ymm6
|
||||
vpxor 0x2c0(%rsp), %ymm0, %ymm0
|
||||
vmovdqu %ymm12, 0x140(%rsp)
|
||||
vpsrlq $0x25, %ymm6, %ymm2
|
||||
vpsllq $0x1b, %ymm6, %ymm6
|
||||
vpor %ymm6, %ymm2, %ymm2
|
||||
vpxor 0x220(%rsp), %ymm3, %ymm6
|
||||
vpxor 0x200(%rsp), %ymm3, %ymm3
|
||||
vpsrlq $0x1c, %ymm6, %ymm7
|
||||
vpsllq $0x24, %ymm6, %ymm6
|
||||
vpor %ymm6, %ymm7, %ymm7
|
||||
vpxor 0x260(%rsp), %ymm4, %ymm6
|
||||
vpxor 0x240(%rsp), %ymm4, %ymm4
|
||||
vpsrlq $0x36, %ymm6, %ymm12
|
||||
vpsllq $0xa, %ymm6, %ymm6
|
||||
vpor %ymm6, %ymm12, %ymm12
|
||||
vpxor 0x180(%rsp), %ymm5, %ymm6
|
||||
vpxor 0x280(%rsp), %ymm5, %ymm5
|
||||
vpandn %ymm12, %ymm7, %ymm9
|
||||
vpsrlq $0x31, %ymm6, %ymm11
|
||||
vpsllq $0xf, %ymm6, %ymm6
|
||||
vpxor %ymm2, %ymm9, %ymm9
|
||||
vpor %ymm6, %ymm11, %ymm11
|
||||
vpandn %ymm11, %ymm12, %ymm6
|
||||
vpxor %ymm7, %ymm6, %ymm6
|
||||
vmovdqu %ymm6, 0x160(%rsp)
|
||||
vpxor 0x1e0(%rsp), %ymm1, %ymm6
|
||||
vpxor 0x2a0(%rsp), %ymm1, %ymm1
|
||||
vpshufb (%rcx), %ymm6, %ymm6
|
||||
vpandn %ymm6, %ymm11, %ymm13
|
||||
vpxor %ymm12, %ymm13, %ymm13
|
||||
vmovdqu %ymm13, 0x180(%rsp)
|
||||
vpandn %ymm2, %ymm6, %ymm13
|
||||
vpandn %ymm7, %ymm2, %ymm2
|
||||
vpxor %ymm6, %ymm2, %ymm2
|
||||
vpsrlq $0x3e, %ymm4, %ymm6
|
||||
vpxor %ymm11, %ymm13, %ymm13
|
||||
vmovdqu %ymm2, 0x1a0(%rsp)
|
||||
vpsrlq $0x2, %ymm5, %ymm2
|
||||
vpsllq $0x3e, %ymm5, %ymm5
|
||||
vpor %ymm5, %ymm2, %ymm2
|
||||
vpsrlq $0x9, %ymm1, %ymm5
|
||||
vpsllq $0x37, %ymm1, %ymm1
|
||||
vpsllq $0x2, %ymm4, %ymm4
|
||||
vpor %ymm1, %ymm5, %ymm1
|
||||
vpsrlq $0x19, %ymm0, %ymm5
|
||||
vpor %ymm4, %ymm6, %ymm4
|
||||
vpsllq $0x27, %ymm0, %ymm0
|
||||
vpor %ymm0, %ymm5, %ymm5
|
||||
vpandn %ymm5, %ymm1, %ymm0
|
||||
vpxor %ymm2, %ymm0, %ymm0
|
||||
vmovdqu %ymm0, 0x1c0(%rsp)
|
||||
vpsrlq $0x17, %ymm3, %ymm0
|
||||
vpsllq $0x29, %ymm3, %ymm3
|
||||
vpor %ymm3, %ymm0, %ymm0
|
||||
vpandn %ymm4, %ymm0, %ymm7
|
||||
vpandn %ymm0, %ymm5, %ymm3
|
||||
vpxor %ymm5, %ymm7, %ymm7
|
||||
vpandn %ymm2, %ymm4, %ymm5
|
||||
vpandn %ymm1, %ymm2, %ymm2
|
||||
vpxor %ymm0, %ymm5, %ymm5
|
||||
vpxor %ymm1, %ymm3, %ymm3
|
||||
vpxor %ymm4, %ymm2, %ymm2
|
||||
vmovdqu %ymm5, 0x1e0(%rsp)
|
||||
addq $0x8, %rsi
|
||||
addq $0x1, %r10
|
||||
cmpq $0x18, %r10
|
||||
jne LLkeccak_f1600_x4_avx2
|
||||
vmovdqu (%rsp), %ymm4
|
||||
vmovdqu 0x40(%rsp), %ymm5
|
||||
vmovdqu 0x20(%rsp), %ymm0
|
||||
vmovdqu 0x60(%rsp), %ymm1
|
||||
vmovdqu 0x1c0(%rsp), %ymm12
|
||||
vmovdqu %ymm2, 0x1c0(%rsp)
|
||||
vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2]
|
||||
vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3]
|
||||
vpunpcklqdq %ymm1, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm1[0],ymm5[2],ymm1[2]
|
||||
vpunpckhqdq %ymm1, %ymm5, %ymm1 # ymm1 = ymm5[1],ymm1[1],ymm5[3],ymm1[3]
|
||||
vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3]
|
||||
vmovdqu 0x80(%rsp), %ymm4
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1]
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm6, (%rdi)
|
||||
vmovdqu %ymm5, 0xc8(%rdi)
|
||||
vmovdqu %ymm2, 0x190(%rdi)
|
||||
vmovdqu %ymm0, 0x258(%rdi)
|
||||
vmovdqu 0xa0(%rsp), %ymm0
|
||||
vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2]
|
||||
vpunpckhqdq %ymm0, %ymm4, %ymm1 # ymm1 = ymm4[1],ymm0[1],ymm4[3],ymm0[3]
|
||||
vmovdqu 0xc0(%rsp), %ymm0
|
||||
vpunpcklqdq %ymm10, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm10[0],ymm0[2],ymm10[2]
|
||||
vpunpckhqdq %ymm10, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm10[1],ymm0[3],ymm10[3]
|
||||
vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3]
|
||||
vmovdqu 0xe0(%rsp), %ymm4
|
||||
vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3]
|
||||
vmovdqu 0x100(%rsp), %ymm0
|
||||
vmovdqu %ymm2, 0x1b0(%rdi)
|
||||
vmovdqu %ymm1, 0x278(%rdi)
|
||||
vpunpcklqdq %ymm4, %ymm14, %ymm2 # ymm2 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm14, %ymm1 # ymm1 = ymm14[1],ymm4[1],ymm14[3],ymm4[3]
|
||||
vpunpcklqdq %ymm8, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
|
||||
vpunpckhqdq %ymm8, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3]
|
||||
vmovdqu %ymm6, 0x20(%rdi)
|
||||
vmovdqu %ymm5, 0xe8(%rdi)
|
||||
vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3]
|
||||
vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3]
|
||||
vmovdqu 0x120(%rsp), %ymm4
|
||||
vmovdqu 0x140(%rsp), %ymm0
|
||||
vmovdqu %ymm2, 0x1d0(%rdi)
|
||||
vmovdqu %ymm1, 0x298(%rdi)
|
||||
vpunpcklqdq %ymm4, %ymm15, %ymm2 # ymm2 = ymm15[0],ymm4[0],ymm15[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm15, %ymm1 # ymm1 = ymm15[1],ymm4[1],ymm15[3],ymm4[3]
|
||||
vpunpcklqdq %ymm9, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm9[0],ymm0[2],ymm9[2]
|
||||
vmovdqu %ymm5, 0x108(%rdi)
|
||||
vpunpckhqdq %ymm9, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm9[1],ymm0[3],ymm9[3]
|
||||
vmovdqu %ymm6, 0x40(%rdi)
|
||||
vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3]
|
||||
vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1]
|
||||
vmovdqu 0x160(%rsp), %ymm4
|
||||
vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3]
|
||||
vmovdqu 0x180(%rsp), %ymm0
|
||||
vmovdqu %ymm5, 0x128(%rdi)
|
||||
vmovdqu 0x1a0(%rsp), %ymm5
|
||||
vmovdqu %ymm2, 0x1f0(%rdi)
|
||||
vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2]
|
||||
vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3]
|
||||
vpunpcklqdq %ymm5, %ymm13, %ymm4 # ymm4 = ymm13[0],ymm5[0],ymm13[2],ymm5[2]
|
||||
vmovdqu %ymm6, 0x60(%rdi)
|
||||
vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1]
|
||||
vmovdqu %ymm1, 0x2b8(%rdi)
|
||||
vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3]
|
||||
vpunpckhqdq %ymm5, %ymm13, %ymm1 # ymm1 = ymm13[1],ymm5[1],ymm13[3],ymm5[3]
|
||||
vmovdqu %ymm6, 0x80(%rdi)
|
||||
vmovdqu 0x1e0(%rsp), %ymm4
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1]
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm2, 0x210(%rdi)
|
||||
vpunpcklqdq %ymm3, %ymm12, %ymm2 # ymm2 = ymm12[0],ymm3[0],ymm12[2],ymm3[2]
|
||||
vmovdqu %ymm0, 0x2d8(%rdi)
|
||||
vpunpckhqdq %ymm3, %ymm12, %ymm0 # ymm0 = ymm12[1],ymm3[1],ymm12[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm7, %ymm1 # ymm1 = ymm7[1],ymm4[1],ymm7[3],ymm4[3]
|
||||
vmovdqu %ymm5, 0x148(%rdi)
|
||||
vperm2i128 $0x20, %ymm3, %ymm2, %ymm5 # ymm5 = ymm2[0,1],ymm3[0,1]
|
||||
vperm2i128 $0x31, %ymm3, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm3[2,3]
|
||||
vmovdqu 0x1c0(%rsp), %ymm3
|
||||
vperm2i128 $0x20, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[0,1],ymm1[0,1]
|
||||
vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3]
|
||||
vmovdqu %ymm5, 0xa0(%rdi)
|
||||
vextracti128 $0x1, %ymm3, %xmm15
|
||||
vmovdqu %ymm4, 0x168(%rdi)
|
||||
vmovdqu %ymm2, 0x230(%rdi)
|
||||
vmovdqu %ymm0, 0x2f8(%rdi)
|
||||
vmovq %xmm3, 0xc0(%rdi)
|
||||
vmovhpd %xmm3, 0x188(%rdi)
|
||||
vmovq %xmm15, 0x250(%rdi)
|
||||
vmovhpd %xmm15, 0x318(%rdi)
|
||||
movq %r11, %rsp
|
||||
.cfi_def_cfa_register %rsp
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(keccak_f1600_x4_avx2_asm)
|
||||
|
||||
#endif /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../../common.h"
|
||||
#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "fips202_native_x86_64.h"
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t
|
||||
mlk_keccakf1600_round_constants[24] = {
|
||||
0x0000000000000001, 0x0000000000008082, 0x800000000000808a,
|
||||
0x8000000080008000, 0x000000000000808b, 0x0000000080000001,
|
||||
0x8000000080008081, 0x8000000000008009, 0x000000000000008a,
|
||||
0x0000000000000088, 0x0000000080008009, 0x000000008000000a,
|
||||
0x000000008000808b, 0x800000000000008b, 0x8000000000008089,
|
||||
0x8000000000008003, 0x8000000000008002, 0x8000000000000080,
|
||||
0x000000000000800a, 0x800000008000000a, 0x8000000080008081,
|
||||
0x8000000000008080, 0x0000000080000001, 0x8000000080008008,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho8[4] = {
|
||||
0x0605040302010007,
|
||||
0x0e0d0c0b0a09080f,
|
||||
0x1615141312111017,
|
||||
0x1e1d1c1b1a19181f,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho56[4] = {
|
||||
0x0007060504030201,
|
||||
0x080f0e0d0c0b0a09,
|
||||
0x1017161514131211,
|
||||
0x181f1e1d1c1b1a19,
|
||||
};
|
||||
|
||||
#else /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
MLK_EMPTY_CU(fips202_x86_64_constants)
|
||||
|
||||
#endif /* !(MLK_FIPS202_X86_64_NEED_X4_AVX2 && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED) */
|
||||
653
sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.c
Normal file
653
sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.c
Normal file
@@ -0,0 +1,653 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*
|
||||
* - [REF]
|
||||
* CRYSTALS-Kyber C reference implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/ref
|
||||
*/
|
||||
|
||||
#include "indcpa.h"
|
||||
|
||||
#include "debug.h"
|
||||
#include "randombytes.h"
|
||||
#include "sampling.h"
|
||||
#include "symmetric.h"
|
||||
#include "verify.h"
|
||||
|
||||
/* Parameter set namespacing
|
||||
* This is to facilitate building multiple instances
|
||||
* of mlkem-native (e.g. with varying parameter sets)
|
||||
* within a single compilation unit. */
|
||||
#define mlk_pack_pk MLK_ADD_PARAM_SET(mlk_pack_pk)
|
||||
#define mlk_unpack_pk MLK_ADD_PARAM_SET(mlk_unpack_pk)
|
||||
#define mlk_pack_sk MLK_ADD_PARAM_SET(mlk_pack_sk)
|
||||
#define mlk_unpack_sk MLK_ADD_PARAM_SET(mlk_unpack_sk)
|
||||
#define mlk_pack_ciphertext MLK_ADD_PARAM_SET(mlk_pack_ciphertext)
|
||||
#define mlk_unpack_ciphertext MLK_ADD_PARAM_SET(mlk_unpack_ciphertext)
|
||||
#define mlk_matvec_mul MLK_ADD_PARAM_SET(mlk_matvec_mul)
|
||||
#define mlk_polyvec_permute_bitrev_to_custom \
|
||||
MLK_ADD_PARAM_SET(mlk_polyvec_permute_bitrev_to_custom)
|
||||
#define mlk_polymat_permute_bitrev_to_custom \
|
||||
MLK_ADD_PARAM_SET(mlk_polymat_permute_bitrev_to_custom)
|
||||
#define mlk_keypair_getnoise_eta1 MLK_ADD_PARAM_SET(mlk_keypair_getnoise_eta1)
|
||||
#define mlk_enc_getnoise_eta1_eta2 MLK_ADD_PARAM_SET(mlk_enc_getnoise_eta1_eta2)
|
||||
/* End of parameter set namespacing */
|
||||
|
||||
/**
|
||||
* Serialize the public key as the concatenation of the serialized vector of
|
||||
* polynomials pk and the public seed used to generate the matrix A.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L19].}
|
||||
*
|
||||
* @param[out] r Output serialized public key.
|
||||
* @param[in] pk Input public-key polyvec. Must have coefficients within
|
||||
* [0,..,MLKEM_Q-1].
|
||||
* @param[in] seed Input public seed.
|
||||
*/
|
||||
static void mlk_pack_pk(uint8_t r[MLKEM_INDCPA_PUBLICKEYBYTES],
|
||||
const mlk_polyvec *pk,
|
||||
const uint8_t seed[MLKEM_SYMBYTES])
|
||||
{
|
||||
mlk_assert_bound_2d(pk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q);
|
||||
mlk_polyvec_tobytes(r, pk);
|
||||
mlk_memcpy(r + MLKEM_POLYVECBYTES, seed, MLKEM_SYMBYTES);
|
||||
}
|
||||
|
||||
/**
|
||||
* De-serialize public key from a byte array; approximate inverse of
|
||||
* mlk_pack_pk.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L2-3].}
|
||||
*
|
||||
* @param[out] pk Output public-key polynomial vector. Coefficients
|
||||
* will be normalized to [0,1,..,MLKEM_Q-1].
|
||||
* @param[out] seed Output seed to generate matrix A.
|
||||
* @param[in] packedpk Input serialized public key.
|
||||
*/
|
||||
static void mlk_unpack_pk(mlk_polyvec *pk, uint8_t seed[MLKEM_SYMBYTES],
|
||||
const uint8_t packedpk[MLKEM_INDCPA_PUBLICKEYBYTES])
|
||||
{
|
||||
mlk_polyvec_frombytes(pk, packedpk);
|
||||
mlk_memcpy(seed, packedpk + MLKEM_POLYVECBYTES, MLKEM_SYMBYTES);
|
||||
|
||||
/* NOTE: If a modulus check was conducted on the PK, we know at this
|
||||
* point that the coefficients of `pk` are unsigned canonical. The
|
||||
* specifications and proofs, however, do _not_ assume this, and instead
|
||||
* work with the easily provable bound by MLKEM_UINT12_LIMIT. */
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the secret key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L20].}
|
||||
*
|
||||
* @param[out] r Output serialized secret key.
|
||||
* @param[in] sk Input vector of polynomials (secret key).
|
||||
*/
|
||||
static void mlk_pack_sk(uint8_t r[MLKEM_INDCPA_SECRETKEYBYTES],
|
||||
const mlk_polyvec *sk)
|
||||
{
|
||||
mlk_assert_bound_2d(sk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q);
|
||||
mlk_polyvec_tobytes(r, sk);
|
||||
}
|
||||
|
||||
/**
|
||||
* De-serialize the secret key; inverse of mlk_pack_sk.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L5].}
|
||||
*
|
||||
* @param[out] sk Output vector of polynomials (secret key).
|
||||
* @param[in] packedsk Input serialized secret key.
|
||||
*/
|
||||
static void mlk_unpack_sk(mlk_polyvec *sk,
|
||||
const uint8_t packedsk[MLKEM_INDCPA_SECRETKEYBYTES])
|
||||
{
|
||||
mlk_polyvec_frombytes(sk, packedsk);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the ciphertext as the concatenation of the compressed and
|
||||
* serialized vector of polynomials b and the compressed and serialized
|
||||
* polynomial v.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L22-23].}
|
||||
*
|
||||
* @param[out] r Output serialized ciphertext.
|
||||
* @param[in] b Input vector of polynomials b.
|
||||
* @param[in] v Input polynomial v.
|
||||
*/
|
||||
static void mlk_pack_ciphertext(uint8_t r[MLKEM_INDCPA_BYTES],
|
||||
const mlk_polyvec *b, mlk_poly *v)
|
||||
{
|
||||
mlk_polyvec_compress_du(r, b);
|
||||
mlk_poly_compress_dv(r + MLKEM_POLYVECCOMPRESSEDBYTES_DU, v);
|
||||
}
|
||||
|
||||
/**
|
||||
* De-serialize and decompress ciphertext from a byte array; approximate
|
||||
* inverse of mlk_pack_ciphertext.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L1-4].}
|
||||
*
|
||||
* @param[out] b Output vector of polynomials b.
|
||||
* @param[out] v Output polynomial v.
|
||||
* @param[in] c Input serialized ciphertext.
|
||||
*/
|
||||
static void mlk_unpack_ciphertext(mlk_polyvec *b, mlk_poly *v,
|
||||
const uint8_t c[MLKEM_INDCPA_BYTES])
|
||||
{
|
||||
mlk_polyvec_decompress_du(b, c);
|
||||
mlk_poly_decompress_dv(v, c + MLKEM_POLYVECCOMPRESSEDBYTES_DU);
|
||||
}
|
||||
|
||||
/* Helper function to ensure that the polynomial entries in the output
|
||||
* of gen_matrix use the standard (bitreversed) ordering of coefficients.
|
||||
* No-op unless a native backend with a custom ordering is used.
|
||||
*
|
||||
* We don't inline this into gen_matrix to avoid having to split the CBMC
|
||||
* proof for gen_matrix based on MLK_USE_NATIVE_NTT_CUSTOM_ORDER. */
|
||||
static void mlk_polyvec_permute_bitrev_to_custom(mlk_polyvec *v)
|
||||
__contract__(
|
||||
/* We don't specify that this should be a permutation, but only
|
||||
* that it does not change the bound established at the end of mlk_gen_matrix. */
|
||||
requires(memory_no_alias(v, sizeof(mlk_polyvec)))
|
||||
requires(forall(x, 0, MLKEM_K,
|
||||
array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))
|
||||
assigns(memory_slice(v, sizeof(mlk_polyvec)))
|
||||
ensures(forall(x, 0, MLKEM_K,
|
||||
array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))
|
||||
{
|
||||
#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER)
|
||||
unsigned i;
|
||||
for (i = 0; i < MLKEM_K; i++)
|
||||
__loop__(
|
||||
assigns(i, memory_slice(v, sizeof(mlk_polyvec)))
|
||||
invariant(i <= MLKEM_K)
|
||||
invariant(forall(x, 0, MLKEM_K,
|
||||
array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))
|
||||
decreases(MLKEM_K - i))
|
||||
{
|
||||
mlk_poly_permute_bitrev_to_custom(v->vec[i].coeffs);
|
||||
}
|
||||
#else /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */
|
||||
/* Nothing to do */
|
||||
(void)v;
|
||||
#endif /* !MLK_USE_NATIVE_NTT_CUSTOM_ORDER */
|
||||
}
|
||||
|
||||
static void mlk_polymat_permute_bitrev_to_custom(mlk_polymat *a)
|
||||
__contract__(
|
||||
/* We don't specify that this should be a permutation, but only
|
||||
* that it does not change the bound established at the end of mlk_gen_matrix. */
|
||||
requires(memory_no_alias(a, sizeof(mlk_polymat)))
|
||||
requires(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K,
|
||||
array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))
|
||||
assigns(memory_slice(a, sizeof(mlk_polymat)))
|
||||
ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K,
|
||||
array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))))
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < MLKEM_K; i++)
|
||||
__loop__(
|
||||
assigns(i, memory_slice(a, sizeof(mlk_polymat)))
|
||||
invariant(i <= MLKEM_K)
|
||||
invariant(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K,
|
||||
array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))
|
||||
decreases(MLKEM_K - i))
|
||||
{
|
||||
mlk_polyvec_permute_bitrev_to_custom(&a->vec[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/* Reference: `gen_matrix()` in the reference implementation @[REF].
|
||||
* - We use a special subroutine to generate 4 polynomials
|
||||
* at a time, to be able to leverage batched Keccak-f1600
|
||||
* implementations. The reference implementation generates
|
||||
* one matrix entry a time.
|
||||
*
|
||||
* Not static for benchmarking */
|
||||
MLK_INTERNAL_API
|
||||
void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES],
|
||||
int transposed)
|
||||
{
|
||||
unsigned i, j;
|
||||
MLK_ALIGN uint8_t seed_ext[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)];
|
||||
|
||||
for (j = 0; j < 4; j++)
|
||||
{
|
||||
mlk_memcpy(seed_ext[j], seed, MLKEM_SYMBYTES);
|
||||
}
|
||||
|
||||
#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY)
|
||||
/* Sample 4 matrix entries a time. */
|
||||
for (i = 0; i < (MLKEM_K * MLKEM_K / 4) * 4; i += 4)
|
||||
{
|
||||
for (j = 0; j < 4; j++)
|
||||
{
|
||||
uint8_t x, y;
|
||||
/* MLKEM_K <= 4, so the values fit in uint8_t. */
|
||||
x = (uint8_t)((i + j) / MLKEM_K);
|
||||
y = (uint8_t)((i + j) % MLKEM_K);
|
||||
if (transposed)
|
||||
{
|
||||
seed_ext[j][MLKEM_SYMBYTES + 0] = x;
|
||||
seed_ext[j][MLKEM_SYMBYTES + 1] = y;
|
||||
}
|
||||
else
|
||||
{
|
||||
seed_ext[j][MLKEM_SYMBYTES + 0] = y;
|
||||
seed_ext[j][MLKEM_SYMBYTES + 1] = x;
|
||||
}
|
||||
}
|
||||
|
||||
mlk_poly_rej_uniform_x4(&a->vec[i / MLKEM_K].vec[i % MLKEM_K],
|
||||
&a->vec[(i + 1) / MLKEM_K].vec[(i + 1) % MLKEM_K],
|
||||
&a->vec[(i + 2) / MLKEM_K].vec[(i + 2) % MLKEM_K],
|
||||
&a->vec[(i + 3) / MLKEM_K].vec[(i + 3) % MLKEM_K],
|
||||
seed_ext);
|
||||
}
|
||||
#else /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */
|
||||
/* When using serial FIPS202, sample all entries individually. */
|
||||
i = 0;
|
||||
#endif /* MLK_CONFIG_SERIAL_FIPS202_ONLY */
|
||||
|
||||
/* For MLKEM_K == 3, sample the last entry individually.
|
||||
* When MLK_CONFIG_SERIAL_FIPS202_ONLY is set, sample all entries
|
||||
* individually. */
|
||||
for (; i < MLKEM_K * MLKEM_K; i++)
|
||||
{
|
||||
uint8_t x, y;
|
||||
/* MLKEM_K <= 4, so the values fit in uint8_t. */
|
||||
x = (uint8_t)(i / MLKEM_K);
|
||||
y = (uint8_t)(i % MLKEM_K);
|
||||
|
||||
if (transposed)
|
||||
{
|
||||
seed_ext[0][MLKEM_SYMBYTES + 0] = x;
|
||||
seed_ext[0][MLKEM_SYMBYTES + 1] = y;
|
||||
}
|
||||
else
|
||||
{
|
||||
seed_ext[0][MLKEM_SYMBYTES + 0] = y;
|
||||
seed_ext[0][MLKEM_SYMBYTES + 1] = x;
|
||||
}
|
||||
|
||||
mlk_poly_rej_uniform(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], seed_ext[0]);
|
||||
}
|
||||
|
||||
mlk_assert(i == MLKEM_K * MLKEM_K);
|
||||
|
||||
/*
|
||||
* The public matrix is generated in NTT domain. If the native backend
|
||||
* uses a custom order in NTT domain, permute A accordingly.
|
||||
*/
|
||||
mlk_polymat_permute_bitrev_to_custom(a);
|
||||
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
mlk_zeroize(seed_ext, sizeof(seed_ext));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute matrix-vector product in NTT domain, via Montgomery multiplication.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.12), (2.13)].}
|
||||
*
|
||||
* @param[out] out Output polynomial vector.
|
||||
* @param[in] a Input matrix. Must be in NTT domain and have coefficients
|
||||
* of absolute value < 4096.
|
||||
* @param[in] v Input polynomial vector. Must be in NTT domain.
|
||||
* @param[in] vc Mulcache for @p v, computed via
|
||||
* mlk_polyvec_mulcache_compute().
|
||||
*/
|
||||
static void mlk_matvec_mul(mlk_polyvec *out, const mlk_polymat *a,
|
||||
const mlk_polyvec *v, const mlk_polyvec_mulcache *vc)
|
||||
__contract__(
|
||||
requires(memory_no_alias(out, sizeof(mlk_polyvec)))
|
||||
requires(memory_no_alias(a, sizeof(mlk_polymat)))
|
||||
requires(memory_no_alias(v, sizeof(mlk_polyvec)))
|
||||
requires(memory_no_alias(vc, sizeof(mlk_polyvec_mulcache)))
|
||||
requires(forall(k0, 0, MLKEM_K,
|
||||
forall(k1, 0, MLKEM_K,
|
||||
array_bound(a->vec[k0].vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))))
|
||||
assigns(memory_slice(out, sizeof(mlk_polyvec))))
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < MLKEM_K; i++)
|
||||
__loop__(
|
||||
assigns(i, memory_slice(out, sizeof(mlk_polyvec)))
|
||||
invariant(i <= MLKEM_K)
|
||||
decreases(MLKEM_K - i))
|
||||
{
|
||||
mlk_polyvec_basemul_acc_montgomery_cached(&out->vec[i], &a->vec[i], v, vc);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute and fill the pv and e polyvec structures needed by
|
||||
* mlk_keypair_derand(). Uses x4-batched versions of `poly_getnoise` to
|
||||
* leverage batched Keccak-f1600.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)] steps 8-15.}
|
||||
*
|
||||
* @param[out] pv Output polynomial vector.
|
||||
* @param[out] e Output polynomial vector.
|
||||
* @param[in] seed Seed bytes for sampling.
|
||||
*/
|
||||
static void mlk_keypair_getnoise_eta1(mlk_polyvec *pv, mlk_polyvec *e,
|
||||
const uint8_t seed[MLKEM_SYMBYTES])
|
||||
__contract__(
|
||||
requires(memory_no_alias(pv, sizeof(mlk_polyvec)))
|
||||
requires(memory_no_alias(e, sizeof(mlk_polyvec)))
|
||||
requires(memory_no_alias(seed, MLKEM_SYMBYTES))
|
||||
assigns(memory_slice(pv, sizeof(mlk_polyvec)))
|
||||
assigns(memory_slice(e, sizeof(mlk_polyvec)))
|
||||
ensures(forall(k0, 0, MLKEM_K, array_abs_bound(pv->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1)))
|
||||
ensures(forall(k1, 0, MLKEM_K, array_abs_bound(e->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1)))
|
||||
)
|
||||
{
|
||||
#if MLKEM_K == 2
|
||||
mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], /* Fill elements of pv */
|
||||
&e->vec[0], &e->vec[1], /* and two elements of e */
|
||||
seed, 0, 1, 2, 3);
|
||||
#elif MLKEM_K == 3
|
||||
/*
|
||||
* Only the first three output buffers are needed, so we pass NULL as
|
||||
* the fourth parameter, and 0xFF as its dummy nonce.
|
||||
*/
|
||||
mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], NULL, seed,
|
||||
0, 1, 2, 0xFF);
|
||||
/* Same here */
|
||||
mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], NULL, seed, 3,
|
||||
4, 5, 0xFF);
|
||||
#elif MLKEM_K == 4
|
||||
mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], &pv->vec[3],
|
||||
seed, 0, 1, 2, 3);
|
||||
mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], &e->vec[3],
|
||||
seed, 4, 5, 6, 7);
|
||||
#endif /* MLKEM_K == 4 */
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute and fill the sp, ep, and epp polynomial structures needed by
|
||||
* mlk_indcpa_enc(). Uses x4-batched versions of `poly_getnoise` to leverage
|
||||
* batched Keccak-f1600.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)] steps 9-16.}
|
||||
*
|
||||
* @param[out] sp Output polynomial vector.
|
||||
* @param[out] ep Output polynomial vector.
|
||||
* @param[out] epp Output polynomial.
|
||||
* @param[in] coins Seed bytes for sampling.
|
||||
*/
|
||||
static void mlk_enc_getnoise_eta1_eta2(mlk_polyvec *sp, mlk_polyvec *ep,
|
||||
mlk_poly *epp,
|
||||
const uint8_t coins[MLKEM_SYMBYTES])
|
||||
__contract__(
|
||||
requires(memory_no_alias(sp, sizeof(mlk_polyvec)))
|
||||
requires(memory_no_alias(ep, sizeof(mlk_polyvec)))
|
||||
requires(memory_no_alias(epp, sizeof(mlk_poly)))
|
||||
requires(memory_no_alias(coins, MLKEM_SYMBYTES))
|
||||
assigns(memory_slice(sp, sizeof(mlk_polyvec)))
|
||||
assigns(memory_slice(ep, sizeof(mlk_polyvec)))
|
||||
assigns(memory_slice(epp, sizeof(mlk_poly)))
|
||||
ensures(forall(k0, 0, MLKEM_K, array_abs_bound(sp->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1)))
|
||||
ensures(forall(k1, 0, MLKEM_K, array_abs_bound(ep->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)))
|
||||
ensures(array_abs_bound(epp->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))
|
||||
)
|
||||
{
|
||||
#if MLKEM_K == 2
|
||||
mlk_poly_getnoise_eta1122_4x(&sp->vec[0], &sp->vec[1], &ep->vec[0],
|
||||
&ep->vec[1], coins, 0, 1, 2, 3);
|
||||
mlk_poly_getnoise_eta2(epp, coins, 4);
|
||||
#elif MLKEM_K == 3
|
||||
/*
|
||||
* In this call, only the first three output buffers are needed,
|
||||
* so we pass NULL as the fourth parameter, and 0xFF as its dummy nonce.
|
||||
*/
|
||||
mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], NULL, coins,
|
||||
0, 1, 2, 0xFF /* irrelevant */);
|
||||
/* The fourth output buffer in this call _is_ used. */
|
||||
mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], epp, coins,
|
||||
3, 4, 5, 6);
|
||||
#elif MLKEM_K == 4
|
||||
mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], &sp->vec[3],
|
||||
coins, 0, 1, 2, 3);
|
||||
mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], &ep->vec[3],
|
||||
coins, 4, 5, 6, 7);
|
||||
mlk_poly_getnoise_eta2(epp, coins, 8);
|
||||
#endif /* MLKEM_K == 4 */
|
||||
}
|
||||
|
||||
|
||||
/* Reference: `indcpa_keypair_derand()` in the reference implementation @[REF].
|
||||
* - We use a different implementation of `gen_matrix()` which
|
||||
* uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above).
|
||||
* - We use a mulcache to speed up matrix-vector multiplication.
|
||||
* - We include buffer zeroization.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES],
|
||||
uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES],
|
||||
const uint8_t coins[MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
const uint8_t *publicseed;
|
||||
const uint8_t *noiseseed;
|
||||
MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
MLK_ALLOC(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context);
|
||||
MLK_ALLOC(a, mlk_polymat, 1, context);
|
||||
MLK_ALLOC(e, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(pkpv, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(skpv, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(skpv_cache, mlk_polyvec_mulcache, 1, context);
|
||||
|
||||
if (buf == NULL || coins_with_domain_separator == NULL || a == NULL ||
|
||||
e == NULL || pkpv == NULL || skpv == NULL || skpv_cache == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
publicseed = buf;
|
||||
noiseseed = buf + MLKEM_SYMBYTES;
|
||||
|
||||
/* Concatenate coins with MLKEM_K for domain separation of security levels */
|
||||
mlk_memcpy(coins_with_domain_separator, coins, MLKEM_SYMBYTES);
|
||||
coins_with_domain_separator[MLKEM_SYMBYTES] = MLKEM_K;
|
||||
|
||||
mlk_hash_g(buf, coins_with_domain_separator, MLKEM_SYMBYTES + 1);
|
||||
|
||||
/*
|
||||
* Declassify the public seed.
|
||||
* Required to use it in conditional-branches in rejection sampling.
|
||||
* This is needed because all output of randombytes is marked as secret
|
||||
* (=undefined)
|
||||
*/
|
||||
MLK_CT_TESTING_DECLASSIFY(publicseed, MLKEM_SYMBYTES);
|
||||
|
||||
mlk_gen_matrix(a, publicseed, 0 /* no transpose */);
|
||||
|
||||
mlk_keypair_getnoise_eta1(skpv, e, noiseseed);
|
||||
|
||||
mlk_polyvec_ntt(skpv);
|
||||
mlk_polyvec_ntt(e);
|
||||
|
||||
mlk_polyvec_mulcache_compute(skpv_cache, skpv);
|
||||
mlk_matvec_mul(pkpv, a, skpv, skpv_cache);
|
||||
mlk_polyvec_tomont(pkpv);
|
||||
|
||||
mlk_polyvec_add(pkpv, e);
|
||||
mlk_polyvec_reduce(pkpv);
|
||||
mlk_polyvec_reduce(skpv);
|
||||
|
||||
mlk_pack_sk(sk, skpv);
|
||||
mlk_pack_pk(pk, pkpv, publicseed);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(skpv_cache, mlk_polyvec_mulcache, 1, context);
|
||||
MLK_FREE(skpv, mlk_polyvec, 1, context);
|
||||
MLK_FREE(pkpv, mlk_polyvec, 1, context);
|
||||
MLK_FREE(e, mlk_polyvec, 1, context);
|
||||
MLK_FREE(a, mlk_polymat, 1, context);
|
||||
MLK_FREE(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context);
|
||||
MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Reference: `indcpa_enc()` in the reference implementation @[REF].
|
||||
* - We use x4-batched versions of `poly_getnoise` to leverage
|
||||
* batched x4-batched Keccak-f1600.
|
||||
* - We use a different implementation of `gen_matrix()` which
|
||||
* uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above).
|
||||
* - We use a mulcache to speed up matrix-vector multiplication.
|
||||
* - We include buffer zeroization.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES],
|
||||
const uint8_t m[MLKEM_INDCPA_MSGBYTES],
|
||||
const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES],
|
||||
const uint8_t coins[MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(seed, uint8_t, MLKEM_SYMBYTES, context);
|
||||
MLK_ALLOC(at, mlk_polymat, 1, context);
|
||||
MLK_ALLOC(sp, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(pkpv, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(ep, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(b, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(v, mlk_poly, 1, context);
|
||||
MLK_ALLOC(k, mlk_poly, 1, context);
|
||||
MLK_ALLOC(epp, mlk_poly, 1, context);
|
||||
MLK_ALLOC(sp_cache, mlk_polyvec_mulcache, 1, context);
|
||||
|
||||
if (seed == NULL || at == NULL || sp == NULL || pkpv == NULL || ep == NULL ||
|
||||
b == NULL || v == NULL || k == NULL || epp == NULL || sp_cache == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
mlk_unpack_pk(pkpv, seed, pk);
|
||||
mlk_poly_frommsg(k, m);
|
||||
|
||||
/*
|
||||
* Declassify the public seed.
|
||||
* Required to use it in conditional-branches in rejection sampling.
|
||||
* This is needed because in re-encryption the publicseed originated from sk
|
||||
* which is marked undefined.
|
||||
*/
|
||||
MLK_CT_TESTING_DECLASSIFY(seed, MLKEM_SYMBYTES);
|
||||
|
||||
mlk_gen_matrix(at, seed, 1 /* transpose */);
|
||||
|
||||
mlk_enc_getnoise_eta1_eta2(sp, ep, epp, coins);
|
||||
|
||||
mlk_polyvec_ntt(sp);
|
||||
|
||||
mlk_polyvec_mulcache_compute(sp_cache, sp);
|
||||
mlk_matvec_mul(b, at, sp, sp_cache);
|
||||
mlk_polyvec_basemul_acc_montgomery_cached(v, pkpv, sp, sp_cache);
|
||||
|
||||
mlk_polyvec_invntt_tomont(b);
|
||||
mlk_poly_invntt_tomont(v);
|
||||
|
||||
mlk_polyvec_add(b, ep);
|
||||
mlk_poly_add(v, epp);
|
||||
mlk_poly_add(v, k);
|
||||
|
||||
mlk_polyvec_reduce(b);
|
||||
mlk_poly_reduce(v);
|
||||
|
||||
mlk_pack_ciphertext(c, b, v);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(sp_cache, mlk_polyvec_mulcache, 1, context);
|
||||
MLK_FREE(epp, mlk_poly, 1, context);
|
||||
MLK_FREE(k, mlk_poly, 1, context);
|
||||
MLK_FREE(v, mlk_poly, 1, context);
|
||||
MLK_FREE(b, mlk_polyvec, 1, context);
|
||||
MLK_FREE(ep, mlk_polyvec, 1, context);
|
||||
MLK_FREE(pkpv, mlk_polyvec, 1, context);
|
||||
MLK_FREE(sp, mlk_polyvec, 1, context);
|
||||
MLK_FREE(at, mlk_polymat, 1, context);
|
||||
MLK_FREE(seed, uint8_t, MLKEM_SYMBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Reference: `indcpa_dec()` in the reference implementation @[REF].
|
||||
* - We use a mulcache for the scalar product.
|
||||
* - We include buffer zeroization. */
|
||||
MLK_INTERNAL_API
|
||||
int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES],
|
||||
const uint8_t c[MLKEM_INDCPA_BYTES],
|
||||
const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(b, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(skpv, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(v, mlk_poly, 1, context);
|
||||
MLK_ALLOC(sb, mlk_poly, 1, context);
|
||||
MLK_ALLOC(b_cache, mlk_polyvec_mulcache, 1, context);
|
||||
|
||||
if (b == NULL || skpv == NULL || v == NULL || sb == NULL || b_cache == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
mlk_unpack_ciphertext(b, v, c);
|
||||
mlk_unpack_sk(skpv, sk);
|
||||
|
||||
mlk_polyvec_ntt(b);
|
||||
mlk_polyvec_mulcache_compute(b_cache, b);
|
||||
mlk_polyvec_basemul_acc_montgomery_cached(sb, skpv, b, b_cache);
|
||||
mlk_poly_invntt_tomont(sb);
|
||||
|
||||
mlk_poly_sub(v, sb);
|
||||
mlk_poly_reduce(v);
|
||||
|
||||
mlk_poly_tomsg(m, v);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(b_cache, mlk_polyvec_mulcache, 1, context);
|
||||
MLK_FREE(sb, mlk_poly, 1, context);
|
||||
MLK_FREE(v, mlk_poly, 1, context);
|
||||
MLK_FREE(skpv, mlk_polyvec, 1, context);
|
||||
MLK_FREE(b, mlk_polyvec, 1, context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
#undef mlk_pack_pk
|
||||
#undef mlk_unpack_pk
|
||||
#undef mlk_pack_sk
|
||||
#undef mlk_unpack_sk
|
||||
#undef mlk_pack_ciphertext
|
||||
#undef mlk_unpack_ciphertext
|
||||
#undef mlk_matvec_mul
|
||||
#undef mlk_polyvec_permute_bitrev_to_custom
|
||||
#undef mlk_polymat_permute_bitrev_to_custom
|
||||
#undef mlk_keypair_getnoise_eta1
|
||||
#undef mlk_enc_getnoise_eta1_eta2
|
||||
163
sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.h
Normal file
163
sdk/software/examples/mlkem_test/mlkem_native/src/indcpa.h
Normal file
@@ -0,0 +1,163 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*/
|
||||
|
||||
#ifndef MLK_INDCPA_H
|
||||
#define MLK_INDCPA_H
|
||||
|
||||
#include "cbmc.h"
|
||||
#include "common.h"
|
||||
#include "poly_k.h"
|
||||
|
||||
#define mlk_gen_matrix MLK_NAMESPACE_K(gen_matrix)
|
||||
/**
|
||||
* Deterministically generate matrix A (or the transpose of A) from a seed.
|
||||
* Entries of the matrix are polynomials that look uniformly random.
|
||||
* Performs rejection sampling on the output of an XOF.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L3-7] and
|
||||
* @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L4-8]. The @p transposed
|
||||
* parameter only affects internal presentation.}
|
||||
*
|
||||
* @param[out] a Output matrix A.
|
||||
* @param[in] seed Input seed.
|
||||
* @param transposed Boolean deciding whether A or A^T is generated.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES],
|
||||
int transposed)
|
||||
__contract__(
|
||||
requires(memory_no_alias(a, sizeof(mlk_polymat)))
|
||||
requires(memory_no_alias(seed, MLKEM_SYMBYTES))
|
||||
requires(transposed == 0 || transposed == 1)
|
||||
assigns(memory_slice(a, sizeof(mlk_polymat)))
|
||||
ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K,
|
||||
array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))
|
||||
);
|
||||
|
||||
#define mlk_indcpa_keypair_derand \
|
||||
MLK_NAMESPACE_K(indcpa_keypair_derand) MLK_CONTEXT_PARAMETERS_3
|
||||
/**
|
||||
* Generate public and private key for the CPA-secure public-key encryption
|
||||
* scheme underlying ML-KEM.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)].}
|
||||
*
|
||||
* @param[out] pk Output public key
|
||||
* (length MLKEM_INDCPA_PUBLICKEYBYTES bytes).
|
||||
* @param[out] sk Output private key
|
||||
* (length MLKEM_INDCPA_SECRETKEYBYTES bytes).
|
||||
* @param[in] coins Input randomness (length MLKEM_SYMBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
* @retval MLK_ERR_RNG_FAIL Random number generation failed.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES],
|
||||
uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES],
|
||||
const uint8_t coins[MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES))
|
||||
requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES))
|
||||
requires(memory_no_alias(coins, MLKEM_SYMBYTES))
|
||||
assigns(memory_slice(pk, MLKEM_INDCPA_PUBLICKEYBYTES))
|
||||
assigns(memory_slice(sk, MLKEM_INDCPA_SECRETKEYBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY ||
|
||||
return_value == MLK_ERR_RNG_FAIL)
|
||||
);
|
||||
|
||||
#define mlk_indcpa_enc MLK_NAMESPACE_K(indcpa_enc) MLK_CONTEXT_PARAMETERS_4
|
||||
/**
|
||||
* Encryption function of the CPA-secure public-key encryption scheme
|
||||
* underlying ML-KEM.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)].}
|
||||
*
|
||||
* @param[out] c Output ciphertext (length MLKEM_INDCPA_BYTES bytes).
|
||||
* @param[in] m Input message (length MLKEM_INDCPA_MSGBYTES bytes).
|
||||
* @param[in] pk Input public key
|
||||
* (length MLKEM_INDCPA_PUBLICKEYBYTES bytes).
|
||||
* @param[in] coins Input random coins used as seed (length MLKEM_SYMBYTES
|
||||
* bytes) to deterministically generate all randomness.
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL Operation failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES],
|
||||
const uint8_t m[MLKEM_INDCPA_MSGBYTES],
|
||||
const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES],
|
||||
const uint8_t coins[MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(c, MLKEM_INDCPA_BYTES))
|
||||
requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES))
|
||||
requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES))
|
||||
requires(memory_no_alias(coins, MLKEM_SYMBYTES))
|
||||
assigns(memory_slice(c, MLKEM_INDCPA_BYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY)
|
||||
);
|
||||
|
||||
#define mlk_indcpa_dec MLK_NAMESPACE_K(indcpa_dec) MLK_CONTEXT_PARAMETERS_3
|
||||
/**
|
||||
* Decryption function of the CPA-secure public-key encryption scheme
|
||||
* underlying ML-KEM.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt)].}
|
||||
*
|
||||
* @param[out] m Output decrypted message
|
||||
* (length MLKEM_INDCPA_MSGBYTES bytes).
|
||||
* @param[in] c Input ciphertext (length MLKEM_INDCPA_BYTES bytes).
|
||||
* @param[in] sk Input secret key
|
||||
* (length MLKEM_INDCPA_SECRETKEYBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL Operation failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_INTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES],
|
||||
const uint8_t c[MLKEM_INDCPA_BYTES],
|
||||
const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(c, MLKEM_INDCPA_BYTES))
|
||||
requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES))
|
||||
requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES))
|
||||
assigns(memory_slice(m, MLKEM_INDCPA_MSGBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY)
|
||||
);
|
||||
|
||||
#endif /* !MLK_INDCPA_H */
|
||||
446
sdk/software/examples/mlkem_test/mlkem_native/src/kem.c
Normal file
446
sdk/software/examples/mlkem_test/mlkem_native/src/kem.c
Normal file
@@ -0,0 +1,446 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS140_3_IG]
|
||||
* Implementation Guidance for FIPS 140-3 and the Cryptographic Module
|
||||
* Validation Program
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*
|
||||
* - [REF]
|
||||
* CRYSTALS-Kyber C reference implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/ref
|
||||
*/
|
||||
|
||||
#include "kem.h"
|
||||
|
||||
#include "indcpa.h"
|
||||
#include "randombytes.h"
|
||||
#include "symmetric.h"
|
||||
#include "verify.h"
|
||||
|
||||
/* Parameter set namespacing
|
||||
* This is to facilitate building multiple instances
|
||||
* of mlkem-native (e.g. with varying security levels)
|
||||
* within a single compilation unit. */
|
||||
#define mlk_check_pct MLK_ADD_PARAM_SET(mlk_check_pct) MLK_CONTEXT_PARAMETERS_2
|
||||
/* End of parameter set namespacing */
|
||||
|
||||
/* Reference: Not implemented in the reference implementation @[REF]. */
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(p, mlk_polyvec, 1, context);
|
||||
MLK_ALLOC(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context);
|
||||
|
||||
if (p == NULL || p_reencoded == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
mlk_polyvec_frombytes(p, pk);
|
||||
mlk_polyvec_reduce(p);
|
||||
mlk_polyvec_tobytes(p_reencoded, p);
|
||||
|
||||
/* We use a constant-time memcmp here to avoid having to
|
||||
* declassify the PK before the PCT has succeeded. */
|
||||
ret = mlk_ct_memcmp(pk, p_reencoded, MLKEM_POLYVECBYTES) ? MLK_ERR_FAIL : 0;
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context);
|
||||
MLK_FREE(p, mlk_polyvec, 1, context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
/* Reference: Not implemented in the reference implementation @[REF]. */
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(test, uint8_t, MLKEM_SYMBYTES, context);
|
||||
|
||||
if (test == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/*
|
||||
* The parts of `sk` being hashed and compared here are public, so
|
||||
* no public information is leaked through the runtime or the return value
|
||||
* of this function.
|
||||
*/
|
||||
|
||||
/* Declassify the public part of the secret key */
|
||||
MLK_CT_TESTING_DECLASSIFY(sk + MLKEM_INDCPA_SECRETKEYBYTES,
|
||||
MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
MLK_CT_TESTING_DECLASSIFY(
|
||||
sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, MLKEM_SYMBYTES);
|
||||
|
||||
mlk_hash_h(test, sk + MLKEM_INDCPA_SECRETKEYBYTES,
|
||||
MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
/* This doesn't have to be a constant-time memcmp, but it's the only place
|
||||
* in the library where a normal memcmp would be used otherwise, so for sake
|
||||
* of minimizing stdlib dependency, we use our constant-time one anyway. */
|
||||
ret = mlk_ct_memcmp(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES,
|
||||
test, MLKEM_SYMBYTES)
|
||||
? MLK_ERR_FAIL
|
||||
: 0;
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(test, uint8_t, MLKEM_SYMBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY ||
|
||||
return_value == MLK_ERR_RNG_FAIL)
|
||||
);
|
||||
|
||||
#if defined(MLK_CONFIG_KEYGEN_PCT)
|
||||
/* Specification:
|
||||
* Partially implements 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and
|
||||
* @[FIPS203, Section 7.1, Pairwise Consistency]. */
|
||||
|
||||
/* Reference: Not implemented in the reference implementation @[REF]. */
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context);
|
||||
MLK_ALLOC(ss_enc, uint8_t, MLKEM_SSBYTES, context);
|
||||
MLK_ALLOC(ss_dec, uint8_t, MLKEM_SSBYTES, context);
|
||||
|
||||
if (ct == NULL || ss_enc == NULL || ss_dec == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
ret = mlk_kem_enc(ct, ss_enc, pk, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
ret = mlk_kem_dec(ss_dec, ct, sk, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST)
|
||||
/* Deliberately break PCT for testing purposes */
|
||||
if (mlk_break_pct())
|
||||
{
|
||||
ss_enc[0] = ~ss_enc[0];
|
||||
}
|
||||
#endif /* MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST */
|
||||
|
||||
ret = mlk_ct_memcmp(ss_enc, ss_dec, MLKEM_SSBYTES);
|
||||
/* The result of the PCT is public. */
|
||||
MLK_CT_TESTING_DECLASSIFY(&ret, sizeof(ret));
|
||||
|
||||
if (ret != 0)
|
||||
{
|
||||
ret = MLK_ERR_FAIL;
|
||||
}
|
||||
|
||||
cleanup:
|
||||
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(ss_dec, uint8_t, MLKEM_SSBYTES, context);
|
||||
MLK_FREE(ss_enc, uint8_t, MLKEM_SSBYTES, context);
|
||||
MLK_FREE(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
#else /* MLK_CONFIG_KEYGEN_PCT */
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
/* Skip PCT */
|
||||
((void)pk);
|
||||
((void)sk);
|
||||
#if defined(MLK_CONFIG_CONTEXT_PARAMETER)
|
||||
((void)context);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
#endif /* !MLK_CONFIG_KEYGEN_PCT */
|
||||
|
||||
/* Reference: `crypto_kem_keypair_derand()` in the reference implementation
|
||||
* @[REF].
|
||||
* - We optionally include PCT which is not present in
|
||||
* the reference code. */
|
||||
MLK_EXTERNAL_API
|
||||
int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
const uint8_t coins[2 * MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = mlk_indcpa_keypair_derand(pk, sk, coins, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
mlk_memcpy(sk + MLKEM_INDCPA_SECRETKEYBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
mlk_hash_h(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, pk,
|
||||
MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
/* Value z for pseudo-random output on reject */
|
||||
mlk_memcpy(sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES,
|
||||
coins + MLKEM_SYMBYTES, MLKEM_SYMBYTES);
|
||||
|
||||
/* Declassify public key */
|
||||
MLK_CT_TESTING_DECLASSIFY(pk, MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
|
||||
/* Pairwise Consistency Test (PCT) @[FIPS140_3_IG, p.87] */
|
||||
ret = mlk_check_pct(pk, sk, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
cleanup:
|
||||
if (ret != 0)
|
||||
{
|
||||
mlk_zeroize(pk, MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
mlk_zeroize(sk, MLKEM_INDCCA_SECRETKEYBYTES);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#if !defined(MLK_CONFIG_NO_RANDOMIZED_API)
|
||||
/* Reference: `crypto_kem_keypair()` in the reference implementation @[REF]
|
||||
* - We zeroize the stack buffer */
|
||||
MLK_EXTERNAL_API
|
||||
int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(coins, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
|
||||
if (coins == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Acquire necessary randomness, and mark it as secret. */
|
||||
if (mlk_randombytes(coins, 2 * MLKEM_SYMBYTES) != 0)
|
||||
{
|
||||
ret = MLK_ERR_RNG_FAIL;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
MLK_CT_TESTING_SECRET(coins, 2 * MLKEM_SYMBYTES);
|
||||
|
||||
ret = mlk_kem_keypair_derand(pk, sk, coins, context);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(coins, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */
|
||||
|
||||
/* Reference: `crypto_kem_enc_derand()` in the reference implementation @[REF]
|
||||
* - We include public key check
|
||||
* - We include stack buffer zeroization */
|
||||
MLK_EXTERNAL_API
|
||||
int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES],
|
||||
uint8_t ss[MLKEM_SSBYTES],
|
||||
const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
const uint8_t coins[MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
|
||||
if (buf == NULL || kr == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Specification: Implements @[FIPS203, Section 7.2, Modulus check] */
|
||||
ret = mlk_kem_check_pk(pk, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
mlk_memcpy(buf, coins, MLKEM_SYMBYTES);
|
||||
|
||||
/* Multitarget countermeasure for coins + contributory KEM */
|
||||
mlk_hash_h(buf + MLKEM_SYMBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES);
|
||||
mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES);
|
||||
|
||||
/* coins are in kr+MLKEM_SYMBYTES */
|
||||
ret = mlk_indcpa_enc(ct, buf, pk, kr + MLKEM_SYMBYTES, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
mlk_memcpy(ss, kr, MLKEM_SYMBYTES);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#if !defined(MLK_CONFIG_NO_RANDOMIZED_API)
|
||||
/* Reference: `crypto_kem_enc()` in the reference implementation @[REF]
|
||||
* - We include stack buffer zeroization */
|
||||
MLK_EXTERNAL_API
|
||||
int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES],
|
||||
uint8_t ss[MLKEM_SSBYTES],
|
||||
const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
MLK_ALLOC(coins, uint8_t, MLKEM_SYMBYTES, context);
|
||||
|
||||
if (coins == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
if (mlk_randombytes(coins, MLKEM_SYMBYTES) != 0)
|
||||
{
|
||||
ret = MLK_ERR_RNG_FAIL;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
MLK_CT_TESTING_SECRET(coins, MLKEM_SYMBYTES);
|
||||
|
||||
ret = mlk_kem_enc_derand(ct, ss, pk, coins, context);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(coins, uint8_t, MLKEM_SYMBYTES, context);
|
||||
return ret;
|
||||
}
|
||||
#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */
|
||||
|
||||
/* Reference: `crypto_kem_dec()` in the reference implementation @[REF]
|
||||
* - We include secret key check
|
||||
* - We include stack buffer zeroization */
|
||||
MLK_EXTERNAL_API
|
||||
int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES],
|
||||
const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES],
|
||||
const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
{
|
||||
int ret = 0;
|
||||
uint8_t fail;
|
||||
const uint8_t *pk = sk + MLKEM_INDCPA_SECRETKEYBYTES;
|
||||
MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
MLK_ALLOC(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES,
|
||||
context);
|
||||
|
||||
if (buf == NULL || kr == NULL || tmp == NULL)
|
||||
{
|
||||
ret = MLK_ERR_OUT_OF_MEMORY;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Specification: Implements @[FIPS203, Section 7.3, Hash check] */
|
||||
ret = mlk_kem_check_sk(sk, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
ret = mlk_indcpa_dec(buf, ct, sk, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Multitarget countermeasure for coins + contributory KEM */
|
||||
mlk_memcpy(buf + MLKEM_SYMBYTES,
|
||||
sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES,
|
||||
MLKEM_SYMBYTES);
|
||||
mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES);
|
||||
|
||||
/* Recompute and compare ciphertext */
|
||||
/* coins are in kr+MLKEM_SYMBYTES */
|
||||
ret = mlk_indcpa_enc(tmp, buf, pk, kr + MLKEM_SYMBYTES, context);
|
||||
if (ret != 0)
|
||||
{
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
fail = mlk_ct_memcmp(ct, tmp, MLKEM_INDCCA_CIPHERTEXTBYTES);
|
||||
|
||||
/* Compute rejection key */
|
||||
mlk_memcpy(tmp, sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES,
|
||||
MLKEM_SYMBYTES);
|
||||
mlk_memcpy(tmp + MLKEM_SYMBYTES, ct, MLKEM_INDCCA_CIPHERTEXTBYTES);
|
||||
mlk_hash_j(ss, tmp, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES);
|
||||
|
||||
/* Copy true key to return buffer if fail is 0 */
|
||||
mlk_ct_cmov_zero(ss, kr, MLKEM_SYMBYTES, fail);
|
||||
|
||||
cleanup:
|
||||
/* Specification: Partially implements
|
||||
* @[FIPS203, Section 3.3, Destruction of intermediate values] */
|
||||
MLK_FREE(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES,
|
||||
context);
|
||||
MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
#undef mlk_check_pct
|
||||
304
sdk/software/examples/mlkem_test/mlkem_native/src/kem.h
Normal file
304
sdk/software/examples/mlkem_test/mlkem_native/src/kem.h
Normal file
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*
|
||||
* - [REF]
|
||||
* CRYSTALS-Kyber C reference implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/ref
|
||||
*/
|
||||
|
||||
#ifndef MLK_KEM_H
|
||||
#define MLK_KEM_H
|
||||
|
||||
#include "cbmc.h"
|
||||
#include "common.h"
|
||||
#include "sys.h"
|
||||
|
||||
#if defined(MLK_CHECK_APIS)
|
||||
/* Include to ensure consistency between internal kem.h
|
||||
* and external mlkem_native.h. */
|
||||
#include "mlkem_native.h"
|
||||
|
||||
#if MLKEM_INDCCA_SECRETKEYBYTES != \
|
||||
MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)
|
||||
#error Mismatch for SECRETKEYBYTES between kem.h and mlkem_native.h
|
||||
#endif
|
||||
|
||||
#if MLKEM_INDCCA_PUBLICKEYBYTES != \
|
||||
MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)
|
||||
#error Mismatch for PUBLICKEYBYTES between kem.h and mlkem_native.h
|
||||
#endif
|
||||
|
||||
#if MLKEM_INDCCA_CIPHERTEXTBYTES != \
|
||||
MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)
|
||||
#error Mismatch for CIPHERTEXTBYTES between kem.h and mlkem_native.h
|
||||
#endif
|
||||
|
||||
#endif /* MLK_CHECK_APIS */
|
||||
|
||||
#define mlk_kem_keypair_derand \
|
||||
MLK_NAMESPACE_K(keypair_derand) MLK_CONTEXT_PARAMETERS_3
|
||||
#define mlk_kem_keypair MLK_NAMESPACE_K(keypair) MLK_CONTEXT_PARAMETERS_2
|
||||
#define mlk_kem_enc_derand MLK_NAMESPACE_K(enc_derand) MLK_CONTEXT_PARAMETERS_4
|
||||
#define mlk_kem_enc MLK_NAMESPACE_K(enc) MLK_CONTEXT_PARAMETERS_3
|
||||
#define mlk_kem_dec MLK_NAMESPACE_K(dec) MLK_CONTEXT_PARAMETERS_3
|
||||
#define mlk_kem_check_pk MLK_NAMESPACE_K(check_pk) MLK_CONTEXT_PARAMETERS_1
|
||||
#define mlk_kem_check_sk MLK_NAMESPACE_K(check_sk) MLK_CONTEXT_PARAMETERS_1
|
||||
|
||||
/**
|
||||
* Implements modulus check mandated by FIPS 203, i.e., ensures that
|
||||
* coefficients are in [0,q-1].
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].}
|
||||
*
|
||||
* @reference{Not implemented in the reference implementation @[REF].}
|
||||
*
|
||||
* @param[in] pk Input public key (an already allocated array of
|
||||
* MLKEM_INDCCA_PUBLICKEYBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL Modulus check failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY)
|
||||
);
|
||||
|
||||
|
||||
/**
|
||||
* Implements public key hash check mandated by FIPS 203, i.e., ensures that
|
||||
* sk[768𝑘+32 ∶ 768𝑘+64] = H(pk) = H(sk[384𝑘 : 768𝑘+32]).
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Section 7.3, 'hash check'].}
|
||||
*
|
||||
* @reference{Not implemented in the reference implementation @[REF].}
|
||||
*
|
||||
* @param[in] sk Input private key (an already allocated array of
|
||||
* MLKEM_INDCCA_SECRETKEYBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL Public key hash check failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY)
|
||||
);
|
||||
|
||||
/**
|
||||
* Generate a public/private keypair for the ML-KEM key encapsulation mechanism.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].}
|
||||
*
|
||||
* @param[out] pk Output public key (an already allocated array of
|
||||
* MLKEM_INDCCA_PUBLICKEYBYTES bytes).
|
||||
* @param[out] sk Output private key (an already allocated array of
|
||||
* MLKEM_INDCCA_SECRETKEYBYTES bytes).
|
||||
* @param[in] coins Input randomness (an already allocated array filled
|
||||
* with 2*MLKEM_SYMBYTES random bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
const uint8_t coins[2 * MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
requires(memory_no_alias(coins, 2 * MLKEM_SYMBYTES))
|
||||
assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY ||
|
||||
return_value == MLK_ERR_RNG_FAIL)
|
||||
);
|
||||
|
||||
/**
|
||||
* Generate a public/private keypair for the ML-KEM key encapsulation mechanism.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].}
|
||||
*
|
||||
* @param[out] pk Output public key (an already allocated array of
|
||||
* MLKEM_INDCCA_PUBLICKEYBYTES bytes).
|
||||
* @param[out] sk Output private key (an already allocated array of
|
||||
* MLKEM_INDCCA_SECRETKEYBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
* @retval MLK_ERR_RNG_FAIL Random number generation failed.
|
||||
* @retval MLK_ERR_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY ||
|
||||
return_value == MLK_ERR_RNG_FAIL)
|
||||
);
|
||||
|
||||
/**
|
||||
* Generate ciphertext and shared secret for a given public key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].}
|
||||
*
|
||||
* @param[out] ct Output ciphertext (an already allocated array of
|
||||
* MLKEM_INDCCA_CIPHERTEXTBYTES bytes).
|
||||
* @param[out] ss Output shared secret (an already allocated array of
|
||||
* MLKEM_SSBYTES bytes).
|
||||
* @param[in] pk Input public key (an already allocated array of
|
||||
* MLKEM_INDCCA_PUBLICKEYBYTES bytes).
|
||||
* @param[in] coins Input randomness (an already allocated array filled
|
||||
* with MLKEM_SYMBYTES random bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2]
|
||||
* for the public key failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES],
|
||||
uint8_t ss[MLKEM_SSBYTES],
|
||||
const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
const uint8_t coins[MLKEM_SYMBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES))
|
||||
requires(memory_no_alias(ss, MLKEM_SSBYTES))
|
||||
requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
requires(memory_no_alias(coins, MLKEM_SYMBYTES))
|
||||
assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES))
|
||||
assigns(memory_slice(ss, MLKEM_SSBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY)
|
||||
);
|
||||
|
||||
/**
|
||||
* Generate ciphertext and shared secret for a given public key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].}
|
||||
*
|
||||
* @param[out] ct Output ciphertext (an already allocated array of
|
||||
* MLKEM_INDCCA_CIPHERTEXTBYTES bytes).
|
||||
* @param[out] ss Output shared secret (an already allocated array of
|
||||
* MLKEM_SSBYTES bytes).
|
||||
* @param[in] pk Input public key (an already allocated array of
|
||||
* MLKEM_INDCCA_PUBLICKEYBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
* @retval MLK_ERR_RNG_FAIL Random number generation failed.
|
||||
* @retval MLK_ERR_FAIL The 'modulus check' @[FIPS203, Section 7.2]
|
||||
* for the public key failed.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES],
|
||||
uint8_t ss[MLKEM_SSBYTES],
|
||||
const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES))
|
||||
requires(memory_no_alias(ss, MLKEM_SSBYTES))
|
||||
requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES))
|
||||
assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES))
|
||||
assigns(memory_slice(ss, MLKEM_SSBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY ||
|
||||
return_value == MLK_ERR_RNG_FAIL)
|
||||
);
|
||||
|
||||
/**
|
||||
* Generate shared secret for a given ciphertext and private key.
|
||||
*
|
||||
* @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].}
|
||||
*
|
||||
* @param[out] ss Output shared secret (an already allocated array of
|
||||
* MLKEM_SSBYTES bytes).
|
||||
* @param[in] ct Input ciphertext (an already allocated array of
|
||||
* MLKEM_INDCCA_CIPHERTEXTBYTES bytes).
|
||||
* @param[in] sk Input private key (an already allocated array of
|
||||
* MLKEM_INDCCA_SECRETKEYBYTES bytes).
|
||||
* @param context Application context. Only present when
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by
|
||||
* MLK_CONFIG_CONTEXT_PARAMETER_TYPE.
|
||||
*
|
||||
* @retval 0 Success.
|
||||
* @retval MLK_ERR_FAIL The 'hash check' @[FIPS203, Section 7.3]
|
||||
* for the secret key failed.
|
||||
* @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and
|
||||
* MLK_CUSTOM_ALLOC returned NULL.
|
||||
*/
|
||||
MLK_EXTERNAL_API
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES],
|
||||
const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES],
|
||||
const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES],
|
||||
MLK_CONFIG_CONTEXT_PARAMETER_TYPE context)
|
||||
__contract__(
|
||||
requires(memory_no_alias(ss, MLKEM_SSBYTES))
|
||||
requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES))
|
||||
requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES))
|
||||
assigns(memory_slice(ss, MLKEM_SSBYTES))
|
||||
ensures(return_value == 0 || return_value == MLK_ERR_FAIL ||
|
||||
return_value == MLK_ERR_OUT_OF_MEMORY)
|
||||
);
|
||||
|
||||
#endif /* !MLK_KEM_H */
|
||||
@@ -0,0 +1,16 @@
|
||||
[//]: # (SPDX-License-Identifier: CC-BY-4.0)
|
||||
|
||||
# AArch64 backend (little endian)
|
||||
|
||||
This directory contains a native backend for little endian AArch64 systems. It is derived from [^NeonNTT] [^SLOTHY_Paper].
|
||||
|
||||
The code in this directory is auto-generated from the 'clean' assembly in [dev/aarch64_clean](../../../../dev/aarch64_clean)
|
||||
in a two-step fashion: First, it is superoptimized using the [SLOTHY](https://github.com/slothy-optimizer/slothy) superoptimizer,
|
||||
giving the assembly in [dev/aarch64_opt](../../../../dev/aarch64_opt). Then, it is stripped of remaining register aliases, macros
|
||||
and most preprocessor directives by [`scripts/simpasm`](../../../../scripts/simpasm).
|
||||
|
||||
If you want to understand how the assembly works, and/or make changes to it, consult [dev/](../../../../dev).
|
||||
|
||||
<!--- bibliography --->
|
||||
[^NeonNTT]: Becker, Hwang, Kannwischer, Yang, Yang: Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1, [https://eprint.iacr.org/2021/986](https://eprint.iacr.org/2021/986)
|
||||
[^SLOTHY_Paper]: Abdulrahman, Becker, Kannwischer, Klein: Fast and Clean: Auditable high-performance assembly via constraint solving, [https://eprint.iacr.org/2022/1303](https://eprint.iacr.org/2022/1303)
|
||||
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_AARCH64_META_H
|
||||
#define MLK_NATIVE_AARCH64_META_H
|
||||
|
||||
/* Set of primitives that this backend replaces */
|
||||
#define MLK_USE_NATIVE_NTT
|
||||
#define MLK_USE_NATIVE_INTT
|
||||
#define MLK_USE_NATIVE_POLY_REDUCE
|
||||
#define MLK_USE_NATIVE_POLY_TOMONT
|
||||
#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#define MLK_USE_NATIVE_POLY_TOBYTES
|
||||
#define MLK_USE_NATIVE_REJ_UNIFORM
|
||||
|
||||
/* Identifier for this backend so that source and assembly files
|
||||
* in the build can be appropriately guarded. */
|
||||
#define MLK_ARITH_BACKEND_AARCH64
|
||||
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../api.h"
|
||||
#include "src/arith_native_aarch64.h"
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
mlk_ntt_aarch64_asm(data, mlk_aarch64_ntt_zetas_layer12345,
|
||||
mlk_aarch64_ntt_zetas_layer67);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
mlk_intt_aarch64_asm(data, mlk_aarch64_invntt_zetas_layer12345,
|
||||
mlk_aarch64_invntt_zetas_layer67);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
mlk_poly_reduce_aarch64_asm(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
mlk_poly_tomont_aarch64_asm(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2],
|
||||
const int16_t y[MLKEM_N])
|
||||
{
|
||||
mlk_poly_mulcache_compute_aarch64_asm(
|
||||
x, y, mlk_aarch64_zetas_mulcache_native,
|
||||
mlk_aarch64_zetas_mulcache_twisted_native);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N],
|
||||
const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)])
|
||||
{
|
||||
mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(r, a, b, b_cache);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N],
|
||||
const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)])
|
||||
{
|
||||
mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(r, a, b, b_cache);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N],
|
||||
const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)])
|
||||
{
|
||||
mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(r, a, b, b_cache);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES],
|
||||
const int16_t a[MLKEM_N])
|
||||
{
|
||||
mlk_poly_tobytes_aarch64_asm(r, a);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len,
|
||||
const uint8_t *buf,
|
||||
unsigned buflen)
|
||||
{
|
||||
if (len != MLKEM_N ||
|
||||
buflen % 24 != 0) /* NEON support is mandatory for AArch64 */
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
return (int)mlk_rej_uniform_aarch64_asm(r, buf, buflen,
|
||||
mlk_rej_uniform_table);
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_NATIVE_AARCH64_META_H */
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "arith_native_aarch64.h"
|
||||
|
||||
/*
|
||||
* Table of zeta values used in the AArch64 forward NTT
|
||||
* See autogen for details.
|
||||
*/
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t
|
||||
mlk_aarch64_ntt_zetas_layer12345[80] = {
|
||||
-1600, -15749, -749, -7373, -40, -394, -687, -6762, 630, 6201,
|
||||
-1432, -14095, 848, 8347, 0, 0, 1062, 10453, 296, 2914,
|
||||
-882, -8682, 0, 0, -1410, -13879, 1339, 13180, 1476, 14529,
|
||||
0, 0, 193, 1900, -283, -2786, 56, 551, 0, 0,
|
||||
797, 7845, -1089, -10719, 1333, 13121, 0, 0, -543, -5345,
|
||||
1426, 14036, -1235, -12156, 0, 0, -69, -679, 535, 5266,
|
||||
-447, -4400, 0, 0, 569, 5601, -936, -9213, -450, -4429,
|
||||
0, 0, -1583, -15582, -1355, -13338, 821, 8081, 0, 0,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t
|
||||
mlk_aarch64_ntt_zetas_layer67[384] = {
|
||||
289, 289, 331, 331, -76, -76, -1573, -1573, 2845,
|
||||
2845, 3258, 3258, -748, -748, -15483, -15483, 17, 17,
|
||||
583, 583, 1637, 1637, -1041, -1041, 167, 167, 5739,
|
||||
5739, 16113, 16113, -10247, -10247, -568, -568, -680, -680,
|
||||
723, 723, 1100, 1100, -5591, -5591, -6693, -6693, 7117,
|
||||
7117, 10828, 10828, 1197, 1197, -1025, -1025, -1052, -1052,
|
||||
-1274, -1274, 11782, 11782, -10089, -10089, -10355, -10355, -12540,
|
||||
-12540, 1409, 1409, -48, -48, 756, 756, -314, -314,
|
||||
13869, 13869, -472, -472, 7441, 7441, -3091, -3091, -667,
|
||||
-667, 233, 233, -1173, -1173, -279, -279, -6565, -6565,
|
||||
2293, 2293, -11546, -11546, -2746, -2746, 650, 650, -1352,
|
||||
-1352, -816, -816, 632, 632, 6398, 6398, -13308, -13308,
|
||||
-8032, -8032, 6221, 6221, -1626, -1626, -540, -540, -1482,
|
||||
-1482, 1461, 1461, -16005, -16005, -5315, -5315, -14588, -14588,
|
||||
14381, 14381, 1651, 1651, -1540, -1540, 952, 952, -642,
|
||||
-642, 16251, 16251, -15159, -15159, 9371, 9371, -6319, -6319,
|
||||
-464, -464, 33, 33, 1320, 1320, -1414, -1414, -4567,
|
||||
-4567, 325, 325, 12993, 12993, -13918, -13918, 939, 939,
|
||||
-892, -892, 733, 733, 268, 268, 9243, 9243, -8780,
|
||||
-8780, 7215, 7215, 2638, 2638, -1021, -1021, -941, -941,
|
||||
-992, -992, 641, 641, -10050, -10050, -9262, -9262, -9764,
|
||||
-9764, 6309, 6309, -1010, -1010, 1435, 1435, 807, 807,
|
||||
452, 452, -9942, -9942, 14125, 14125, 7943, 7943, 4449,
|
||||
4449, 1584, 1584, -1292, -1292, 375, 375, -1239, -1239,
|
||||
15592, 15592, -12717, -12717, 3691, 3691, -12196, -12196, -1031,
|
||||
-1031, -109, -109, -780, -780, 1645, 1645, -10148, -10148,
|
||||
-1073, -1073, -7678, -7678, 16192, 16192, 1438, 1438, -461,
|
||||
-461, 1534, 1534, -927, -927, 14155, 14155, -4538, -4538,
|
||||
15099, 15099, -9125, -9125, 1063, 1063, -556, -556, -1230,
|
||||
-1230, -863, -863, 10463, 10463, -5473, -5473, -12107, -12107,
|
||||
-8495, -8495, 319, 319, 757, 757, 561, 561, -735,
|
||||
-735, 3140, 3140, 7451, 7451, 5522, 5522, -7235, -7235,
|
||||
-682, -682, -712, -712, 1481, 1481, 648, 648, -6713,
|
||||
-6713, -7008, -7008, 14578, 14578, 6378, 6378, -525, -525,
|
||||
403, 403, 1143, 1143, -554, -554, -5168, -5168, 3967,
|
||||
3967, 11251, 11251, -5453, -5453, 1092, 1092, 1026, 1026,
|
||||
-1179, -1179, 886, 886, 10749, 10749, 10099, 10099, -11605,
|
||||
-11605, 8721, 8721, -855, -855, -219, -219, 1227, 1227,
|
||||
910, 910, -8416, -8416, -2156, -2156, 12078, 12078, 8957,
|
||||
8957, -1607, -1607, -1455, -1455, -1219, -1219, 885, 885,
|
||||
-15818, -15818, -14322, -14322, -11999, -11999, 8711, 8711, 1212,
|
||||
1212, 1029, 1029, -394, -394, -1175, -1175, 11930, 11930,
|
||||
10129, 10129, -3878, -3878, -11566, -11566,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t
|
||||
mlk_aarch64_invntt_zetas_layer12345[80] = {
|
||||
1583, 15582, -821, -8081, 1355, 13338, 0, 0, -569,
|
||||
-5601, 450, 4429, 936, 9213, 0, 0, 69, 679,
|
||||
447, 4400, -535, -5266, 0, 0, 543, 5345, 1235,
|
||||
12156, -1426, -14036, 0, 0, -797, -7845, -1333, -13121,
|
||||
1089, 10719, 0, 0, -193, -1900, -56, -551, 283,
|
||||
2786, 0, 0, 1410, 13879, -1476, -14529, -1339, -13180,
|
||||
0, 0, -1062, -10453, 882, 8682, -296, -2914, 0,
|
||||
0, 1600, 15749, 40, 394, 749, 7373, -848, -8347,
|
||||
1432, 14095, -630, -6201, 687, 6762, 0, 0,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t
|
||||
mlk_aarch64_invntt_zetas_layer67[384] = {
|
||||
-910, -910, -1227, -1227, 219, 219, 855, 855, -8957,
|
||||
-8957, -12078, -12078, 2156, 2156, 8416, 8416, 1175, 1175,
|
||||
394, 394, -1029, -1029, -1212, -1212, 11566, 11566, 3878,
|
||||
3878, -10129, -10129, -11930, -11930, -885, -885, 1219, 1219,
|
||||
1455, 1455, 1607, 1607, -8711, -8711, 11999, 11999, 14322,
|
||||
14322, 15818, 15818, -648, -648, -1481, -1481, 712, 712,
|
||||
682, 682, -6378, -6378, -14578, -14578, 7008, 7008, 6713,
|
||||
6713, -886, -886, 1179, 1179, -1026, -1026, -1092, -1092,
|
||||
-8721, -8721, 11605, 11605, -10099, -10099, -10749, -10749, 554,
|
||||
554, -1143, -1143, -403, -403, 525, 525, 5453, 5453,
|
||||
-11251, -11251, -3967, -3967, 5168, 5168, 927, 927, -1534,
|
||||
-1534, 461, 461, -1438, -1438, 9125, 9125, -15099, -15099,
|
||||
4538, 4538, -14155, -14155, 735, 735, -561, -561, -757,
|
||||
-757, -319, -319, 7235, 7235, -5522, -5522, -7451, -7451,
|
||||
-3140, -3140, 863, 863, 1230, 1230, 556, 556, -1063,
|
||||
-1063, 8495, 8495, 12107, 12107, 5473, 5473, -10463, -10463,
|
||||
-452, -452, -807, -807, -1435, -1435, 1010, 1010, -4449,
|
||||
-4449, -7943, -7943, -14125, -14125, 9942, 9942, -1645, -1645,
|
||||
780, 780, 109, 109, 1031, 1031, -16192, -16192, 7678,
|
||||
7678, 1073, 1073, 10148, 10148, 1239, 1239, -375, -375,
|
||||
1292, 1292, -1584, -1584, 12196, 12196, -3691, -3691, 12717,
|
||||
12717, -15592, -15592, 1414, 1414, -1320, -1320, -33, -33,
|
||||
464, 464, 13918, 13918, -12993, -12993, -325, -325, 4567,
|
||||
4567, -641, -641, 992, 992, 941, 941, 1021, 1021,
|
||||
-6309, -6309, 9764, 9764, 9262, 9262, 10050, 10050, -268,
|
||||
-268, -733, -733, 892, 892, -939, -939, -2638, -2638,
|
||||
-7215, -7215, 8780, 8780, -9243, -9243, -632, -632, 816,
|
||||
816, 1352, 1352, -650, -650, -6221, -6221, 8032, 8032,
|
||||
13308, 13308, -6398, -6398, 642, 642, -952, -952, 1540,
|
||||
1540, -1651, -1651, 6319, 6319, -9371, -9371, 15159, 15159,
|
||||
-16251, -16251, -1461, -1461, 1482, 1482, 540, 540, 1626,
|
||||
1626, -14381, -14381, 14588, 14588, 5315, 5315, 16005, 16005,
|
||||
1274, 1274, 1052, 1052, 1025, 1025, -1197, -1197, 12540,
|
||||
12540, 10355, 10355, 10089, 10089, -11782, -11782, 279, 279,
|
||||
1173, 1173, -233, -233, 667, 667, 2746, 2746, 11546,
|
||||
11546, -2293, -2293, 6565, 6565, 314, 314, -756, -756,
|
||||
48, 48, -1409, -1409, 3091, 3091, -7441, -7441, 472,
|
||||
472, -13869, -13869, 1573, 1573, 76, 76, -331, -331,
|
||||
-289, -289, 15483, 15483, 748, 748, -3258, -3258, -2845,
|
||||
-2845, -1100, -1100, -723, -723, 680, 680, 568, 568,
|
||||
-10828, -10828, -7117, -7117, 6693, 6693, 5591, 5591, 1041,
|
||||
1041, -1637, -1637, -583, -583, -17, -17, 10247, 10247,
|
||||
-16113, -16113, -5739, -5739, -167, -167,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t
|
||||
mlk_aarch64_zetas_mulcache_native[128] = {
|
||||
17, -17, -568, 568, 583, -583, -680, 680, 1637, -1637,
|
||||
723, -723, -1041, 1041, 1100, -1100, 1409, -1409, -667, 667,
|
||||
-48, 48, 233, -233, 756, -756, -1173, 1173, -314, 314,
|
||||
-279, 279, -1626, 1626, 1651, -1651, -540, 540, -1540, 1540,
|
||||
-1482, 1482, 952, -952, 1461, -1461, -642, 642, 939, -939,
|
||||
-1021, 1021, -892, 892, -941, 941, 733, -733, -992, 992,
|
||||
268, -268, 641, -641, 1584, -1584, -1031, 1031, -1292, 1292,
|
||||
-109, 109, 375, -375, -780, 780, -1239, 1239, 1645, -1645,
|
||||
1063, -1063, 319, -319, -556, 556, 757, -757, -1230, 1230,
|
||||
561, -561, -863, 863, -735, 735, -525, 525, 1092, -1092,
|
||||
403, -403, 1026, -1026, 1143, -1143, -1179, 1179, -554, 554,
|
||||
886, -886, -1607, 1607, 1212, -1212, -1455, 1455, 1029, -1029,
|
||||
-1219, 1219, -394, 394, 885, -885, -1175, 1175,
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t
|
||||
mlk_aarch64_zetas_mulcache_twisted_native[128] = {
|
||||
167, -167, -5591, 5591, 5739, -5739, -6693, 6693, 16113,
|
||||
-16113, 7117, -7117, -10247, 10247, 10828, -10828, 13869, -13869,
|
||||
-6565, 6565, -472, 472, 2293, -2293, 7441, -7441, -11546,
|
||||
11546, -3091, 3091, -2746, 2746, -16005, 16005, 16251, -16251,
|
||||
-5315, 5315, -15159, 15159, -14588, 14588, 9371, -9371, 14381,
|
||||
-14381, -6319, 6319, 9243, -9243, -10050, 10050, -8780, 8780,
|
||||
-9262, 9262, 7215, -7215, -9764, 9764, 2638, -2638, 6309,
|
||||
-6309, 15592, -15592, -10148, 10148, -12717, 12717, -1073, 1073,
|
||||
3691, -3691, -7678, 7678, -12196, 12196, 16192, -16192, 10463,
|
||||
-10463, 3140, -3140, -5473, 5473, 7451, -7451, -12107, 12107,
|
||||
5522, -5522, -8495, 8495, -7235, 7235, -5168, 5168, 10749,
|
||||
-10749, 3967, -3967, 10099, -10099, 11251, -11251, -11605, 11605,
|
||||
-5453, 5453, 8721, -8721, -15818, 15818, 11930, -11930, -14322,
|
||||
14322, 10129, -10129, -11999, 11999, -3878, 3878, 8711, -8711,
|
||||
-11566, 11566,
|
||||
};
|
||||
|
||||
#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(aarch64_zetas)
|
||||
|
||||
#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */
|
||||
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H
|
||||
#define MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H
|
||||
|
||||
#include "../../../cbmc.h"
|
||||
#include "../../../common.h"
|
||||
|
||||
#define mlk_aarch64_ntt_zetas_layer12345 \
|
||||
MLK_NAMESPACE(aarch64_ntt_zetas_layer12345)
|
||||
#define mlk_aarch64_ntt_zetas_layer67 MLK_NAMESPACE(aarch64_ntt_zetas_layer67)
|
||||
#define mlk_aarch64_invntt_zetas_layer12345 \
|
||||
MLK_NAMESPACE(aarch64_invntt_zetas_layer12345)
|
||||
#define mlk_aarch64_invntt_zetas_layer67 \
|
||||
MLK_NAMESPACE(aarch64_invntt_zetas_layer67)
|
||||
#define mlk_aarch64_zetas_mulcache_native \
|
||||
MLK_NAMESPACE(aarch64_zetas_mulcache_native)
|
||||
#define mlk_aarch64_zetas_mulcache_twisted_native \
|
||||
MLK_NAMESPACE(aarch64_zetas_mulcache_twisted_native)
|
||||
#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table)
|
||||
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t
|
||||
mlk_aarch64_ntt_zetas_layer12345[80];
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_aarch64_ntt_zetas_layer67[384];
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t
|
||||
mlk_aarch64_invntt_zetas_layer12345[80];
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t
|
||||
mlk_aarch64_invntt_zetas_layer67[384];
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t
|
||||
mlk_aarch64_zetas_mulcache_native[128];
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t
|
||||
mlk_aarch64_zetas_mulcache_twisted_native[128];
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096];
|
||||
|
||||
#define mlk_ntt_aarch64_asm MLK_NAMESPACE(ntt_aarch64_asm)
|
||||
void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80],
|
||||
const int16_t twiddles56[384])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/ntt_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_abs_bound(p, 0, MLKEM_N, 8192))
|
||||
requires(twiddles12345 == mlk_aarch64_ntt_zetas_layer12345)
|
||||
requires(twiddles56 == mlk_aarch64_ntt_zetas_layer67)
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
/* check-magic: off */
|
||||
ensures(array_abs_bound(p, 0, MLKEM_N, 23595))
|
||||
/* check-magic: on */
|
||||
);
|
||||
|
||||
#define mlk_intt_aarch64_asm MLK_NAMESPACE(intt_aarch64_asm)
|
||||
void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80],
|
||||
const int16_t twiddles56[384])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/intt_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
requires(twiddles12345 == mlk_aarch64_invntt_zetas_layer12345)
|
||||
requires(twiddles56 == mlk_aarch64_invntt_zetas_layer67)
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
/* check-magic: off */
|
||||
ensures(array_abs_bound(p, 0, MLKEM_N, 26625))
|
||||
/* check-magic: on */
|
||||
);
|
||||
|
||||
#define mlk_poly_reduce_aarch64_asm MLK_NAMESPACE(poly_reduce_aarch64_asm)
|
||||
void mlk_poly_reduce_aarch64_asm(int16_t p[256])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/poly_reduce_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_tomont_aarch64_asm MLK_NAMESPACE(poly_tomont_aarch64_asm)
|
||||
void mlk_poly_tomont_aarch64_asm(int16_t p[256])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/poly_tomont_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_mulcache_compute_aarch64_asm \
|
||||
MLK_NAMESPACE(poly_mulcache_compute_aarch64_asm)
|
||||
void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128],
|
||||
const int16_t mlk_poly[256],
|
||||
const int16_t zetas[128],
|
||||
const int16_t zetas_twisted[128])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/poly_mulcache_compute_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2)))
|
||||
requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N))
|
||||
requires(zetas == mlk_aarch64_zetas_mulcache_native)
|
||||
requires(zetas_twisted == mlk_aarch64_zetas_mulcache_twisted_native)
|
||||
assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2)))
|
||||
ensures(array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_tobytes_aarch64_asm MLK_NAMESPACE(poly_tobytes_aarch64_asm)
|
||||
void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/poly_tobytes_aarch64_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
assigns(memory_slice(r, MLKEM_POLYBYTES))
|
||||
);
|
||||
|
||||
#define mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm \
|
||||
MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm)
|
||||
void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(
|
||||
int16_t r[256], const int16_t a[512], const int16_t b[512],
|
||||
const int16_t b_cache[256])
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/aarch64/proofs/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2)))
|
||||
requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
);
|
||||
|
||||
#define mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm \
|
||||
MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm)
|
||||
void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(
|
||||
int16_t r[256], const int16_t a[768], const int16_t b[768],
|
||||
const int16_t b_cache[384])
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/aarch64/proofs/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2)))
|
||||
requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
);
|
||||
|
||||
#define mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm \
|
||||
MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm)
|
||||
void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(
|
||||
int16_t r[256], const int16_t a[1024], const int16_t b[1024],
|
||||
const int16_t b_cache[512])
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/aarch64/proofs/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2)))
|
||||
requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
);
|
||||
|
||||
#define mlk_rej_uniform_aarch64_asm MLK_NAMESPACE(rej_uniform_aarch64_asm)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf,
|
||||
unsigned buflen, const uint8_t table[4096])
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/aarch64/proofs/rej_uniform_aarch64_asm.ml. */
|
||||
__contract__(
|
||||
requires(buflen % 24 == 0)
|
||||
requires(memory_no_alias(buf, buflen))
|
||||
requires(table == mlk_rej_uniform_table)
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value <= MLKEM_N)
|
||||
ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#endif /* !MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H */
|
||||
@@ -0,0 +1,630 @@
|
||||
/* Copyright (c) 2022 Arm Limited
|
||||
* Copyright (c) 2022 Hanno Becker
|
||||
* Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [NeonNTT]
|
||||
* Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
|
||||
* Becker, Hwang, Kannwischer, Yang, Yang
|
||||
* https://eprint.iacr.org/2021/986
|
||||
*
|
||||
* - [SLOTHY_Paper]
|
||||
* Fast and Clean: Auditable high-performance assembly via constraint solving
|
||||
* Abdulrahman, Becker, Kannwischer, Klein
|
||||
* https://eprint.iacr.org/2022/1303
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: intt_aarch64_asm
|
||||
Description: AArch64 ML-KEM inverse NTT following @[NeonNTT] and @[SLOTHY_Paper]
|
||||
Signature: void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t p[256]
|
||||
description: Input/output polynomial
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 160
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t twiddles12345[80]
|
||||
description: Twiddle factors for layers 1-5
|
||||
x2:
|
||||
type: buffer
|
||||
size_bytes: 768
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t twiddles56[384]
|
||||
description: Twiddle factors for layers 6-7
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: saving callee-saved Neon registers
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/intt_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(intt_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(intt_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
mov w5, #0xd01 // =3329
|
||||
mov v7.h[0], w5
|
||||
mov w5, #0x4ebf // =20159
|
||||
mov v7.h[1], w5
|
||||
mov w5, #0x200 // =512
|
||||
dup v29.8h, w5
|
||||
mov w5, #0x13b0 // =5040
|
||||
dup v30.8h, w5
|
||||
mov x3, x0
|
||||
mov x4, #0x8 // =8
|
||||
ldr q13, [x3, #0x20]
|
||||
ldr q8, [x3, #0x30]
|
||||
ldr q6, [x3]
|
||||
ldr q16, [x3, #0x10]
|
||||
ldr q4, [x3, #0x50]
|
||||
ldr q11, [x3, #0x40]
|
||||
ldr q3, [x3, #0x70]
|
||||
trn1 v23.4s, v13.4s, v8.4s
|
||||
ldr q0, [x3, #0x60]
|
||||
trn2 v19.4s, v6.4s, v16.4s
|
||||
trn2 v21.4s, v13.4s, v8.4s
|
||||
trn1 v6.4s, v6.4s, v16.4s
|
||||
ldr q24, [x2, #0x20]
|
||||
trn1 v10.2d, v19.2d, v21.2d
|
||||
ldr q16, [x2], #0x60
|
||||
trn1 v5.2d, v6.2d, v23.2d
|
||||
trn1 v28.4s, v0.4s, v3.4s
|
||||
trn2 v18.2d, v6.2d, v23.2d
|
||||
mul v31.8h, v10.8h, v29.8h
|
||||
trn2 v13.4s, v0.4s, v3.4s
|
||||
ldur q14, [x2, #-0x50]
|
||||
sqrdmulh v26.8h, v18.8h, v30.8h
|
||||
ldur q20, [x2, #-0x20]
|
||||
mul v17.8h, v18.8h, v29.8h
|
||||
trn2 v18.2d, v19.2d, v21.2d
|
||||
mul v9.8h, v18.8h, v29.8h
|
||||
trn1 v12.4s, v11.4s, v4.4s
|
||||
sqrdmulh v22.8h, v18.8h, v30.8h
|
||||
sqrdmulh v3.8h, v10.8h, v30.8h
|
||||
sqrdmulh v25.8h, v5.8h, v30.8h
|
||||
mls v9.8h, v22.8h, v7.h[0]
|
||||
mls v17.8h, v26.8h, v7.h[0]
|
||||
trn2 v26.4s, v11.4s, v4.4s
|
||||
mul v8.8h, v5.8h, v29.8h
|
||||
trn1 v10.2d, v26.2d, v13.2d
|
||||
ldur q11, [x2, #-0x10]
|
||||
mls v31.8h, v3.8h, v7.h[0]
|
||||
trn1 v6.2d, v12.2d, v28.2d
|
||||
trn2 v3.2d, v26.2d, v13.2d
|
||||
ldur q4, [x2, #-0x30]
|
||||
mls v8.8h, v25.8h, v7.h[0]
|
||||
sub v19.8h, v17.8h, v9.8h
|
||||
trn2 v13.2d, v12.2d, v28.2d
|
||||
sqrdmulh v1.8h, v3.8h, v30.8h
|
||||
add v9.8h, v17.8h, v9.8h
|
||||
mul v18.8h, v19.8h, v20.8h
|
||||
add v28.8h, v8.8h, v31.8h
|
||||
sqrdmulh v20.8h, v19.8h, v11.8h
|
||||
sub v12.8h, v28.8h, v9.8h
|
||||
sub v23.8h, v8.8h, v31.8h
|
||||
sqrdmulh v11.8h, v13.8h, v30.8h
|
||||
sqrdmulh v5.8h, v23.8h, v4.8h
|
||||
mul v0.8h, v23.8h, v24.8h
|
||||
mul v2.8h, v13.8h, v29.8h
|
||||
mls v0.8h, v5.8h, v7.h[0]
|
||||
add v24.8h, v28.8h, v9.8h
|
||||
mls v18.8h, v20.8h, v7.h[0]
|
||||
sqrdmulh v15.8h, v6.8h, v30.8h
|
||||
sqrdmulh v25.8h, v12.8h, v14.8h
|
||||
mul v21.8h, v12.8h, v16.8h
|
||||
sub v23.8h, v0.8h, v18.8h
|
||||
sqrdmulh v8.8h, v23.8h, v14.8h
|
||||
mul v23.8h, v23.8h, v16.8h
|
||||
mls v21.8h, v25.8h, v7.h[0]
|
||||
mls v23.8h, v8.8h, v7.h[0]
|
||||
mul v14.8h, v3.8h, v29.8h
|
||||
add v3.8h, v0.8h, v18.8h
|
||||
trn2 v4.4s, v24.4s, v3.4s
|
||||
mls v14.8h, v1.8h, v7.h[0]
|
||||
trn1 v9.4s, v24.4s, v3.4s
|
||||
trn2 v12.4s, v21.4s, v23.4s
|
||||
mls v2.8h, v11.8h, v7.h[0]
|
||||
trn1 v28.4s, v21.4s, v23.4s
|
||||
ldr q11, [x1], #0x10
|
||||
mul v31.8h, v10.8h, v29.8h
|
||||
trn1 v25.2d, v4.2d, v12.2d
|
||||
trn1 v20.2d, v9.2d, v28.2d
|
||||
ldr q23, [x2, #0x50]
|
||||
trn2 v13.2d, v4.2d, v12.2d
|
||||
sqrdmulh v21.8h, v10.8h, v30.8h
|
||||
trn2 v4.2d, v9.2d, v28.2d
|
||||
ldr q9, [x2, #0x40]
|
||||
mul v27.8h, v6.8h, v29.8h
|
||||
add v26.8h, v20.8h, v25.8h
|
||||
sub v3.8h, v2.8h, v14.8h
|
||||
sqdmulh v12.8h, v26.8h, v7.h[1]
|
||||
add v5.8h, v4.8h, v13.8h
|
||||
sub v8.8h, v4.8h, v13.8h
|
||||
add v10.8h, v2.8h, v14.8h
|
||||
sqdmulh v6.8h, v5.8h, v7.h[1]
|
||||
ldr q2, [x2, #0x10]
|
||||
mls v27.8h, v15.8h, v7.h[0]
|
||||
ldr q15, [x2, #0x20]
|
||||
srshr v12.8h, v12.8h, #0xb
|
||||
mls v31.8h, v21.8h, v7.h[0]
|
||||
srshr v6.8h, v6.8h, #0xb
|
||||
sqrdmulh v23.8h, v3.8h, v23.8h
|
||||
mls v26.8h, v12.8h, v7.h[0]
|
||||
add v21.8h, v27.8h, v31.8h
|
||||
mls v5.8h, v6.8h, v7.h[0]
|
||||
sub v6.8h, v27.8h, v31.8h
|
||||
sub v14.8h, v21.8h, v10.8h
|
||||
ldr q27, [x2], #0x60
|
||||
mul v3.8h, v3.8h, v9.8h
|
||||
mls v3.8h, v23.8h, v7.h[0]
|
||||
ldur q13, [x2, #-0x30]
|
||||
sub v12.8h, v26.8h, v5.8h
|
||||
add v5.8h, v26.8h, v5.8h
|
||||
sqrdmulh v31.8h, v8.8h, v11.h[5]
|
||||
sqrdmulh v19.8h, v12.8h, v11.h[1]
|
||||
mul v24.8h, v12.8h, v11.h[0]
|
||||
sqrdmulh v13.8h, v6.8h, v13.8h
|
||||
mls v24.8h, v19.8h, v7.h[0]
|
||||
sub x4, x4, #0x2
|
||||
|
||||
Lintt_layer4567_start:
|
||||
add v16.8h, v21.8h, v10.8h
|
||||
mul v18.8h, v6.8h, v15.8h
|
||||
sub v19.8h, v20.8h, v25.8h
|
||||
ldr q21, [x3, #0xa0]
|
||||
str q5, [x3], #0x40
|
||||
mls v18.8h, v13.8h, v7.h[0]
|
||||
sqrdmulh v15.8h, v14.8h, v2.8h
|
||||
ldr q10, [x3, #0x50]
|
||||
ldr q12, [x3, #0x40]
|
||||
stur q24, [x3, #-0x20]
|
||||
mul v5.8h, v8.8h, v11.h[4]
|
||||
sub v0.8h, v18.8h, v3.8h
|
||||
ldr q24, [x3, #0x70]
|
||||
mls v5.8h, v31.8h, v7.h[0]
|
||||
ldr q26, [x2, #0x50]
|
||||
trn2 v1.4s, v12.4s, v10.4s
|
||||
add v6.8h, v18.8h, v3.8h
|
||||
sqrdmulh v20.8h, v0.8h, v2.8h
|
||||
trn1 v13.4s, v12.4s, v10.4s
|
||||
trn1 v18.4s, v16.4s, v6.4s
|
||||
mul v22.8h, v0.8h, v27.8h
|
||||
trn1 v17.4s, v21.4s, v24.4s
|
||||
sqrdmulh v0.8h, v19.8h, v11.h[3]
|
||||
trn1 v25.2d, v13.2d, v17.2d
|
||||
mls v22.8h, v20.8h, v7.h[0]
|
||||
trn2 v21.4s, v21.4s, v24.4s
|
||||
mul v24.8h, v25.8h, v29.8h
|
||||
trn2 v28.2d, v13.2d, v17.2d
|
||||
sqrdmulh v4.8h, v25.8h, v30.8h
|
||||
trn2 v3.2d, v1.2d, v21.2d
|
||||
mul v17.8h, v28.8h, v29.8h
|
||||
sqrdmulh v31.8h, v28.8h, v30.8h
|
||||
ldr q2, [x2, #0x10]
|
||||
mls v24.8h, v4.8h, v7.h[0]
|
||||
mul v4.8h, v19.8h, v11.h[2]
|
||||
ldr q19, [x2, #0x40]
|
||||
mls v4.8h, v0.8h, v7.h[0]
|
||||
mul v0.8h, v14.8h, v27.8h
|
||||
mls v0.8h, v15.8h, v7.h[0]
|
||||
sub v8.8h, v4.8h, v5.8h
|
||||
mul v12.8h, v3.8h, v29.8h
|
||||
mul v23.8h, v8.8h, v11.h[0]
|
||||
trn2 v28.4s, v16.4s, v6.4s
|
||||
sqrdmulh v10.8h, v8.8h, v11.h[1]
|
||||
trn1 v9.4s, v0.4s, v22.4s
|
||||
trn2 v22.4s, v0.4s, v22.4s
|
||||
ldr q11, [x1], #0x10
|
||||
mls v17.8h, v31.8h, v7.h[0]
|
||||
trn1 v20.2d, v18.2d, v9.2d
|
||||
trn2 v14.2d, v18.2d, v9.2d
|
||||
ldr q15, [x2, #0x20]
|
||||
trn1 v6.2d, v1.2d, v21.2d
|
||||
sqrdmulh v9.8h, v3.8h, v30.8h
|
||||
trn1 v25.2d, v28.2d, v22.2d
|
||||
trn2 v16.2d, v28.2d, v22.2d
|
||||
mls v23.8h, v10.8h, v7.h[0]
|
||||
add v1.8h, v20.8h, v25.8h
|
||||
sqrdmulh v21.8h, v6.8h, v30.8h
|
||||
add v8.8h, v14.8h, v16.8h
|
||||
ldr q27, [x2], #0x60
|
||||
sqdmulh v28.8h, v8.8h, v7.h[1]
|
||||
mls v12.8h, v9.8h, v7.h[0]
|
||||
sqdmulh v31.8h, v1.8h, v7.h[1]
|
||||
mul v0.8h, v6.8h, v29.8h
|
||||
sub v10.8h, v17.8h, v12.8h
|
||||
mls v0.8h, v21.8h, v7.h[0]
|
||||
srshr v21.8h, v28.8h, #0xb
|
||||
srshr v13.8h, v31.8h, #0xb
|
||||
sqrdmulh v22.8h, v10.8h, v26.8h
|
||||
mls v8.8h, v21.8h, v7.h[0]
|
||||
mls v1.8h, v13.8h, v7.h[0]
|
||||
add v21.8h, v24.8h, v0.8h
|
||||
stur q23, [x3, #-0x10]
|
||||
sub v6.8h, v24.8h, v0.8h
|
||||
mul v3.8h, v10.8h, v19.8h
|
||||
add v0.8h, v4.8h, v5.8h
|
||||
sqdmulh v13.8h, v0.8h, v7.h[1]
|
||||
ldur q10, [x2, #-0x30]
|
||||
add v5.8h, v1.8h, v8.8h
|
||||
mls v3.8h, v22.8h, v7.h[0]
|
||||
sub v8.8h, v1.8h, v8.8h
|
||||
mul v24.8h, v8.8h, v11.h[0]
|
||||
sqrdmulh v8.8h, v8.8h, v11.h[1]
|
||||
srshr v1.8h, v13.8h, #0xb
|
||||
sqrdmulh v13.8h, v6.8h, v10.8h
|
||||
mls v0.8h, v1.8h, v7.h[0]
|
||||
add v10.8h, v17.8h, v12.8h
|
||||
mls v24.8h, v8.8h, v7.h[0]
|
||||
sub v8.8h, v14.8h, v16.8h
|
||||
sqrdmulh v31.8h, v8.8h, v11.h[5]
|
||||
sub v14.8h, v21.8h, v10.8h
|
||||
stur q0, [x3, #-0x30]
|
||||
sub x4, x4, #0x1
|
||||
cbnz x4, Lintt_layer4567_start
|
||||
mul v15.8h, v6.8h, v15.8h
|
||||
sub v22.8h, v20.8h, v25.8h
|
||||
add v4.8h, v21.8h, v10.8h
|
||||
str q24, [x3, #0x20]
|
||||
mls v15.8h, v13.8h, v7.h[0]
|
||||
str q5, [x3], #0x40
|
||||
ldr q9, [x1], #0x10
|
||||
sqrdmulh v28.8h, v14.8h, v2.8h
|
||||
mul v16.8h, v14.8h, v27.8h
|
||||
sub v18.8h, v15.8h, v3.8h
|
||||
add v15.8h, v15.8h, v3.8h
|
||||
sqrdmulh v0.8h, v18.8h, v2.8h
|
||||
trn2 v24.4s, v4.4s, v15.4s
|
||||
trn1 v2.4s, v4.4s, v15.4s
|
||||
mul v18.8h, v18.8h, v27.8h
|
||||
mls v16.8h, v28.8h, v7.h[0]
|
||||
mls v18.8h, v0.8h, v7.h[0]
|
||||
mul v23.8h, v8.8h, v11.h[4]
|
||||
sqrdmulh v12.8h, v22.8h, v11.h[3]
|
||||
trn1 v17.4s, v16.4s, v18.4s
|
||||
trn2 v4.4s, v16.4s, v18.4s
|
||||
mls v23.8h, v31.8h, v7.h[0]
|
||||
trn2 v3.2d, v2.2d, v17.2d
|
||||
trn2 v6.2d, v24.2d, v4.2d
|
||||
mul v26.8h, v22.8h, v11.h[2]
|
||||
trn1 v28.2d, v2.2d, v17.2d
|
||||
mls v26.8h, v12.8h, v7.h[0]
|
||||
add v25.8h, v3.8h, v6.8h
|
||||
sub v18.8h, v3.8h, v6.8h
|
||||
trn1 v24.2d, v24.2d, v4.2d
|
||||
sqdmulh v1.8h, v25.8h, v7.h[1]
|
||||
sub v27.8h, v28.8h, v24.8h
|
||||
sqrdmulh v2.8h, v18.8h, v9.h[5]
|
||||
add v28.8h, v28.8h, v24.8h
|
||||
mul v24.8h, v27.8h, v9.h[2]
|
||||
sqdmulh v12.8h, v28.8h, v7.h[1]
|
||||
mul v20.8h, v18.8h, v9.h[4]
|
||||
mls v20.8h, v2.8h, v7.h[0]
|
||||
srshr v1.8h, v1.8h, #0xb
|
||||
sqrdmulh v19.8h, v27.8h, v9.h[3]
|
||||
srshr v15.8h, v12.8h, #0xb
|
||||
mls v25.8h, v1.8h, v7.h[0]
|
||||
add v8.8h, v26.8h, v23.8h
|
||||
sub v4.8h, v26.8h, v23.8h
|
||||
mls v28.8h, v15.8h, v7.h[0]
|
||||
mls v24.8h, v19.8h, v7.h[0]
|
||||
mul v2.8h, v4.8h, v11.h[0]
|
||||
sub v19.8h, v28.8h, v25.8h
|
||||
sqrdmulh v15.8h, v4.8h, v11.h[1]
|
||||
add v25.8h, v28.8h, v25.8h
|
||||
sub v10.8h, v24.8h, v20.8h
|
||||
str q25, [x3], #0x40
|
||||
sqrdmulh v22.8h, v19.8h, v9.h[1]
|
||||
add v28.8h, v24.8h, v20.8h
|
||||
sqrdmulh v25.8h, v10.8h, v9.h[1]
|
||||
mul v27.8h, v19.8h, v9.h[0]
|
||||
mul v26.8h, v10.8h, v9.h[0]
|
||||
sqdmulh v20.8h, v28.8h, v7.h[1]
|
||||
sqdmulh v16.8h, v8.8h, v7.h[1]
|
||||
mls v26.8h, v25.8h, v7.h[0]
|
||||
mls v2.8h, v15.8h, v7.h[0]
|
||||
srshr v15.8h, v20.8h, #0xb
|
||||
srshr v1.8h, v16.8h, #0xb
|
||||
mls v27.8h, v22.8h, v7.h[0]
|
||||
mls v28.8h, v15.8h, v7.h[0]
|
||||
mls v8.8h, v1.8h, v7.h[0]
|
||||
stur q27, [x3, #-0x20]
|
||||
stur q2, [x3, #-0x50]
|
||||
stur q28, [x3, #-0x30]
|
||||
stur q26, [x3, #-0x10]
|
||||
stur q8, [x3, #-0x70]
|
||||
mov x4, #0x4 // =4
|
||||
ldr q0, [x1], #0x20
|
||||
ldur q1, [x1, #-0x10]
|
||||
ldr q26, [x0]
|
||||
ldr q13, [x0, #0x40]
|
||||
ldr q28, [x0, #0xc0]
|
||||
ldr q2, [x0, #0x140]
|
||||
ldr q6, [x0, #0x80]
|
||||
ldr q9, [x0, #0x100]
|
||||
ldr q29, [x0, #0x1c0]
|
||||
ldr q23, [x0, #0x180]
|
||||
sub v17.8h, v26.8h, v13.8h
|
||||
add v4.8h, v26.8h, v13.8h
|
||||
ldr q25, [x0, #0xd0]
|
||||
ldr q24, [x0, #0x50]
|
||||
add v5.8h, v6.8h, v28.8h
|
||||
mul v19.8h, v17.8h, v0.h[6]
|
||||
sub v10.8h, v6.8h, v28.8h
|
||||
ldr q30, [x0, #0x150]
|
||||
sqrdmulh v12.8h, v17.8h, v0.h[7]
|
||||
add v17.8h, v9.8h, v2.8h
|
||||
sub v28.8h, v9.8h, v2.8h
|
||||
ldr q2, [x0, #0x90]
|
||||
sub v26.8h, v23.8h, v29.8h
|
||||
sqrdmulh v31.8h, v10.8h, v1.h[1]
|
||||
add v22.8h, v23.8h, v29.8h
|
||||
ldr q3, [x0, #0x110]
|
||||
sqrdmulh v9.8h, v28.8h, v1.h[3]
|
||||
sub v20.8h, v4.8h, v5.8h
|
||||
sub v27.8h, v17.8h, v22.8h
|
||||
ldr q29, [x0, #0x10]
|
||||
add v16.8h, v4.8h, v5.8h
|
||||
sqrdmulh v4.8h, v26.8h, v1.h[5]
|
||||
add v6.8h, v17.8h, v22.8h
|
||||
ldr q22, [x0, #0x1d0]
|
||||
mul v8.8h, v28.8h, v1.h[2]
|
||||
sub v21.8h, v2.8h, v25.8h
|
||||
sub v5.8h, v16.8h, v6.8h
|
||||
mul v17.8h, v26.8h, v1.h[4]
|
||||
mul v26.8h, v10.8h, v1.h[0]
|
||||
mls v26.8h, v31.8h, v7.h[0]
|
||||
mls v17.8h, v4.8h, v7.h[0]
|
||||
mls v19.8h, v12.8h, v7.h[0]
|
||||
mls v8.8h, v9.8h, v7.h[0]
|
||||
sqrdmulh v10.8h, v27.8h, v0.h[5]
|
||||
sub v12.8h, v19.8h, v26.8h
|
||||
add v9.8h, v19.8h, v26.8h
|
||||
sqrdmulh v26.8h, v20.8h, v0.h[3]
|
||||
sub v11.8h, v8.8h, v17.8h
|
||||
add v14.8h, v8.8h, v17.8h
|
||||
sqrdmulh v13.8h, v12.8h, v0.h[3]
|
||||
add v23.8h, v9.8h, v14.8h
|
||||
sqrdmulh v28.8h, v11.8h, v0.h[5]
|
||||
sub v19.8h, v9.8h, v14.8h
|
||||
mul v17.8h, v27.8h, v0.h[4]
|
||||
str q23, [x0, #0x40]
|
||||
mul v14.8h, v20.8h, v0.h[2]
|
||||
mul v8.8h, v11.8h, v0.h[4]
|
||||
mul v4.8h, v12.8h, v0.h[2]
|
||||
mls v14.8h, v26.8h, v7.h[0]
|
||||
mls v17.8h, v10.8h, v7.h[0]
|
||||
mls v8.8h, v28.8h, v7.h[0]
|
||||
mls v4.8h, v13.8h, v7.h[0]
|
||||
sub v10.8h, v14.8h, v17.8h
|
||||
add v20.8h, v14.8h, v17.8h
|
||||
sqrdmulh v28.8h, v5.8h, v0.h[1]
|
||||
mul v18.8h, v5.8h, v0.h[0]
|
||||
str q20, [x0, #0x80]
|
||||
sub v13.8h, v4.8h, v8.8h
|
||||
mul v23.8h, v10.8h, v0.h[0]
|
||||
mul v17.8h, v19.8h, v0.h[0]
|
||||
sqrdmulh v9.8h, v13.8h, v0.h[1]
|
||||
mls v18.8h, v28.8h, v7.h[0]
|
||||
sqrdmulh v10.8h, v10.8h, v0.h[1]
|
||||
sub x4, x4, #0x2
|
||||
|
||||
Lintt_layer123_start:
|
||||
sub v12.8h, v3.8h, v30.8h
|
||||
mul v11.8h, v21.8h, v1.h[0]
|
||||
add v28.8h, v4.8h, v8.8h
|
||||
ldr q20, [x0, #0x190]
|
||||
add v27.8h, v16.8h, v6.8h
|
||||
sqrdmulh v8.8h, v12.8h, v1.h[3]
|
||||
add v16.8h, v29.8h, v24.8h
|
||||
str q28, [x0, #0xc0]
|
||||
mls v23.8h, v10.8h, v7.h[0]
|
||||
str q27, [x0], #0x10
|
||||
add v15.8h, v20.8h, v22.8h
|
||||
str q18, [x0, #0xf0]
|
||||
mul v14.8h, v13.8h, v0.h[0]
|
||||
add v2.8h, v2.8h, v25.8h
|
||||
sub v26.8h, v20.8h, v22.8h
|
||||
mul v4.8h, v12.8h, v1.h[2]
|
||||
sub v5.8h, v16.8h, v2.8h
|
||||
str q23, [x0, #0x170]
|
||||
add v20.8h, v3.8h, v30.8h
|
||||
sqrdmulh v27.8h, v26.8h, v1.h[5]
|
||||
add v16.8h, v16.8h, v2.8h
|
||||
mul v18.8h, v26.8h, v1.h[4]
|
||||
sub v31.8h, v20.8h, v15.8h
|
||||
mls v4.8h, v8.8h, v7.h[0]
|
||||
sub v28.8h, v29.8h, v24.8h
|
||||
mls v18.8h, v27.8h, v7.h[0]
|
||||
ldr q22, [x0, #0x1d0]
|
||||
mul v26.8h, v28.8h, v0.h[6]
|
||||
mul v2.8h, v5.8h, v0.h[2]
|
||||
sub v12.8h, v4.8h, v18.8h
|
||||
sqrdmulh v24.8h, v28.8h, v0.h[7]
|
||||
mls v14.8h, v9.8h, v7.h[0]
|
||||
sqrdmulh v10.8h, v12.8h, v0.h[5]
|
||||
mls v26.8h, v24.8h, v7.h[0]
|
||||
ldr q24, [x0, #0x50]
|
||||
mul v8.8h, v12.8h, v0.h[4]
|
||||
str q14, [x0, #0x1b0]
|
||||
add v28.8h, v4.8h, v18.8h
|
||||
sqrdmulh v5.8h, v5.8h, v0.h[3]
|
||||
add v6.8h, v20.8h, v15.8h
|
||||
sqrdmulh v3.8h, v19.8h, v0.h[1]
|
||||
sub v13.8h, v16.8h, v6.8h
|
||||
sqrdmulh v12.8h, v21.8h, v1.h[1]
|
||||
sqrdmulh v21.8h, v13.8h, v0.h[1]
|
||||
sqrdmulh v27.8h, v31.8h, v0.h[5]
|
||||
ldr q25, [x0, #0xd0]
|
||||
mls v11.8h, v12.8h, v7.h[0]
|
||||
mul v23.8h, v31.8h, v0.h[4]
|
||||
mul v18.8h, v13.8h, v0.h[0]
|
||||
add v30.8h, v26.8h, v11.8h
|
||||
sub v13.8h, v26.8h, v11.8h
|
||||
mls v23.8h, v27.8h, v7.h[0]
|
||||
add v12.8h, v30.8h, v28.8h
|
||||
sub v19.8h, v30.8h, v28.8h
|
||||
mls v2.8h, v5.8h, v7.h[0]
|
||||
str q12, [x0, #0x40]
|
||||
sqrdmulh v26.8h, v13.8h, v0.h[3]
|
||||
mls v8.8h, v10.8h, v7.h[0]
|
||||
ldr q30, [x0, #0x150]
|
||||
sub v20.8h, v2.8h, v23.8h
|
||||
mul v4.8h, v13.8h, v0.h[2]
|
||||
add v13.8h, v2.8h, v23.8h
|
||||
mls v4.8h, v26.8h, v7.h[0]
|
||||
ldr q2, [x0, #0x90]
|
||||
mul v23.8h, v20.8h, v0.h[0]
|
||||
ldr q29, [x0, #0x10]
|
||||
sqrdmulh v10.8h, v20.8h, v0.h[1]
|
||||
str q13, [x0, #0x80]
|
||||
sub v13.8h, v4.8h, v8.8h
|
||||
mls v17.8h, v3.8h, v7.h[0]
|
||||
ldr q3, [x0, #0x110]
|
||||
mls v18.8h, v21.8h, v7.h[0]
|
||||
sub v21.8h, v2.8h, v25.8h
|
||||
sqrdmulh v9.8h, v13.8h, v0.h[1]
|
||||
str q17, [x0, #0x130]
|
||||
mul v17.8h, v19.8h, v0.h[0]
|
||||
sub x4, x4, #0x1
|
||||
cbnz x4, Lintt_layer123_start
|
||||
mls v23.8h, v10.8h, v7.h[0]
|
||||
ldr q11, [x0, #0x190]
|
||||
str q18, [x0, #0x100]
|
||||
add v27.8h, v3.8h, v30.8h
|
||||
mul v13.8h, v13.8h, v0.h[0]
|
||||
sub v5.8h, v29.8h, v24.8h
|
||||
add v14.8h, v16.8h, v6.8h
|
||||
mls v13.8h, v9.8h, v7.h[0]
|
||||
add v10.8h, v11.8h, v22.8h
|
||||
str q23, [x0, #0x180]
|
||||
sub v20.8h, v11.8h, v22.8h
|
||||
sub v23.8h, v27.8h, v10.8h
|
||||
sqrdmulh v16.8h, v21.8h, v1.h[1]
|
||||
sqrdmulh v31.8h, v23.8h, v0.h[5]
|
||||
str q13, [x0, #0x1c0]
|
||||
add v13.8h, v4.8h, v8.8h
|
||||
mul v18.8h, v21.8h, v1.h[0]
|
||||
str q13, [x0, #0xc0]
|
||||
sqrdmulh v13.8h, v19.8h, v0.h[1]
|
||||
sqrdmulh v28.8h, v20.8h, v1.h[5]
|
||||
str q14, [x0], #0x10
|
||||
mul v4.8h, v20.8h, v1.h[4]
|
||||
mls v17.8h, v13.8h, v7.h[0]
|
||||
sub v13.8h, v3.8h, v30.8h
|
||||
sqrdmulh v8.8h, v13.8h, v1.h[3]
|
||||
mul v12.8h, v13.8h, v1.h[2]
|
||||
mls v4.8h, v28.8h, v7.h[0]
|
||||
mls v12.8h, v8.8h, v7.h[0]
|
||||
mls v18.8h, v16.8h, v7.h[0]
|
||||
str q17, [x0, #0x130]
|
||||
sqrdmulh v15.8h, v5.8h, v0.h[7]
|
||||
add v11.8h, v27.8h, v10.8h
|
||||
mul v16.8h, v5.8h, v0.h[6]
|
||||
sub v8.8h, v12.8h, v4.8h
|
||||
sqrdmulh v28.8h, v8.8h, v0.h[5]
|
||||
add v13.8h, v2.8h, v25.8h
|
||||
mls v16.8h, v15.8h, v7.h[0]
|
||||
add v26.8h, v12.8h, v4.8h
|
||||
mul v8.8h, v8.8h, v0.h[4]
|
||||
add v4.8h, v29.8h, v24.8h
|
||||
mls v8.8h, v28.8h, v7.h[0]
|
||||
sub v20.8h, v4.8h, v13.8h
|
||||
add v14.8h, v4.8h, v13.8h
|
||||
add v12.8h, v16.8h, v18.8h
|
||||
sqrdmulh v22.8h, v20.8h, v0.h[3]
|
||||
add v27.8h, v14.8h, v11.8h
|
||||
sub v13.8h, v16.8h, v18.8h
|
||||
mul v4.8h, v20.8h, v0.h[2]
|
||||
str q27, [x0], #0x10
|
||||
sub v24.8h, v12.8h, v26.8h
|
||||
sqrdmulh v3.8h, v13.8h, v0.h[3]
|
||||
mul v13.8h, v13.8h, v0.h[2]
|
||||
sqrdmulh v27.8h, v24.8h, v0.h[1]
|
||||
mls v13.8h, v3.8h, v7.h[0]
|
||||
mul v9.8h, v24.8h, v0.h[0]
|
||||
mls v9.8h, v27.8h, v7.h[0]
|
||||
add v30.8h, v13.8h, v8.8h
|
||||
sub v13.8h, v13.8h, v8.8h
|
||||
mls v4.8h, v22.8h, v7.h[0]
|
||||
str q30, [x0, #0xb0]
|
||||
sqrdmulh v16.8h, v13.8h, v0.h[1]
|
||||
str q9, [x0, #0x130]
|
||||
mul v9.8h, v13.8h, v0.h[0]
|
||||
add v13.8h, v12.8h, v26.8h
|
||||
str q13, [x0, #0x30]
|
||||
mul v13.8h, v23.8h, v0.h[4]
|
||||
sub v23.8h, v14.8h, v11.8h
|
||||
mls v13.8h, v31.8h, v7.h[0]
|
||||
mls v9.8h, v16.8h, v7.h[0]
|
||||
mul v30.8h, v23.8h, v0.h[0]
|
||||
sub v24.8h, v4.8h, v13.8h
|
||||
add v13.8h, v4.8h, v13.8h
|
||||
sqrdmulh v23.8h, v23.8h, v0.h[1]
|
||||
str q9, [x0, #0x1b0]
|
||||
str q13, [x0, #0x70]
|
||||
sqrdmulh v13.8h, v24.8h, v0.h[1]
|
||||
mul v21.8h, v24.8h, v0.h[0]
|
||||
mls v30.8h, v23.8h, v7.h[0]
|
||||
mls v21.8h, v13.8h, v7.h[0]
|
||||
str q30, [x0, #0xf0]
|
||||
str q21, [x0, #0x170]
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(intt_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,564 @@
|
||||
/* Copyright (c) 2022 Arm Limited
|
||||
* Copyright (c) 2022 Hanno Becker
|
||||
* Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [NeonNTT]
|
||||
* Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
|
||||
* Becker, Hwang, Kannwischer, Yang, Yang
|
||||
* https://eprint.iacr.org/2021/986
|
||||
*
|
||||
* - [SLOTHY_Paper]
|
||||
* Fast and Clean: Auditable high-performance assembly via constraint solving
|
||||
* Abdulrahman, Becker, Kannwischer, Klein
|
||||
* https://eprint.iacr.org/2022/1303
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: ntt_aarch64_asm
|
||||
Description: AArch64 ML-KEM forward NTT following @[NeonNTT] and @[SLOTHY_Paper]
|
||||
Signature: void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t p[256]
|
||||
description: Input/output polynomial
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 160
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t twiddles12345[80]
|
||||
description: Twiddle factors for layers 1-5
|
||||
x2:
|
||||
type: buffer
|
||||
size_bytes: 768
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t twiddles56[384]
|
||||
description: Twiddle factors for layers 6-7
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: saving callee-saved Neon registers
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/ntt_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(ntt_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(ntt_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
mov w5, #0xd01 // =3329
|
||||
mov v7.h[0], w5
|
||||
mov w5, #0x4ebf // =20159
|
||||
mov v7.h[1], w5
|
||||
mov x3, x0
|
||||
mov x4, #0x4 // =4
|
||||
ldr q0, [x1], #0x20
|
||||
ldur q1, [x1, #-0x10]
|
||||
ldr q21, [x0, #0x40]
|
||||
ldr q5, [x0, #0x1c0]
|
||||
ldr q30, [x0, #0x110]
|
||||
ldr q24, [x0, #0x140]
|
||||
ldr q12, [x0, #0x80]
|
||||
sqrdmulh v9.8h, v5.8h, v0.h[1]
|
||||
mul v23.8h, v5.8h, v0.h[0]
|
||||
sqrdmulh v17.8h, v24.8h, v0.h[1]
|
||||
ldr q13, [x0, #0xc0]
|
||||
mls v23.8h, v9.8h, v7.h[0]
|
||||
mul v8.8h, v24.8h, v0.h[0]
|
||||
mls v8.8h, v17.8h, v7.h[0]
|
||||
add v9.8h, v13.8h, v23.8h
|
||||
sub v10.8h, v13.8h, v23.8h
|
||||
mul v11.8h, v30.8h, v0.h[0]
|
||||
ldr q13, [x0, #0x180]
|
||||
sqrdmulh v28.8h, v9.8h, v0.h[3]
|
||||
sub v29.8h, v21.8h, v8.8h
|
||||
mul v26.8h, v9.8h, v0.h[2]
|
||||
add v8.8h, v21.8h, v8.8h
|
||||
mul v2.8h, v13.8h, v0.h[0]
|
||||
mls v26.8h, v28.8h, v7.h[0]
|
||||
mul v28.8h, v10.8h, v0.h[4]
|
||||
sqrdmulh v23.8h, v10.8h, v0.h[5]
|
||||
add v22.8h, v8.8h, v26.8h
|
||||
sqrdmulh v10.8h, v13.8h, v0.h[1]
|
||||
sqrdmulh v21.8h, v22.8h, v0.h[7]
|
||||
ldr q13, [x0, #0x100]
|
||||
mul v16.8h, v22.8h, v0.h[6]
|
||||
mls v28.8h, v23.8h, v7.h[0]
|
||||
mls v2.8h, v10.8h, v7.h[0]
|
||||
sqrdmulh v23.8h, v13.8h, v0.h[1]
|
||||
sub v10.8h, v29.8h, v28.8h
|
||||
add v17.8h, v29.8h, v28.8h
|
||||
mls v16.8h, v21.8h, v7.h[0]
|
||||
sub v18.8h, v12.8h, v2.8h
|
||||
ldr q29, [x0]
|
||||
sqrdmulh v14.8h, v17.8h, v1.h[3]
|
||||
add v22.8h, v12.8h, v2.8h
|
||||
sqrdmulh v9.8h, v18.8h, v0.h[5]
|
||||
mul v21.8h, v13.8h, v0.h[0]
|
||||
ldr q13, [x0, #0x150]
|
||||
mul v5.8h, v18.8h, v0.h[4]
|
||||
mls v5.8h, v9.8h, v7.h[0]
|
||||
mul v18.8h, v13.8h, v0.h[0]
|
||||
mls v21.8h, v23.8h, v7.h[0]
|
||||
sqrdmulh v2.8h, v13.8h, v0.h[1]
|
||||
mul v13.8h, v17.8h, v1.h[2]
|
||||
sub v4.8h, v29.8h, v21.8h
|
||||
mls v13.8h, v14.8h, v7.h[0]
|
||||
add v25.8h, v29.8h, v21.8h
|
||||
add v6.8h, v4.8h, v5.8h
|
||||
sqrdmulh v15.8h, v22.8h, v0.h[3]
|
||||
sub v21.8h, v4.8h, v5.8h
|
||||
sub v5.8h, v8.8h, v26.8h
|
||||
mul v23.8h, v22.8h, v0.h[2]
|
||||
add v28.8h, v6.8h, v13.8h
|
||||
sub v13.8h, v6.8h, v13.8h
|
||||
mul v4.8h, v5.8h, v1.h[0]
|
||||
sub x4, x4, #0x2
|
||||
|
||||
Lntt_layer123_start:
|
||||
mls v23.8h, v15.8h, v7.h[0]
|
||||
ldr q6, [x0, #0x190]
|
||||
ldr q15, [x0, #0x90]
|
||||
ldr q19, [x0, #0x10]
|
||||
mul v22.8h, v10.8h, v1.h[4]
|
||||
ldr q24, [x0, #0x50]
|
||||
str q13, [x0, #0x140]
|
||||
sqrdmulh v13.8h, v6.8h, v0.h[1]
|
||||
sub v20.8h, v25.8h, v23.8h
|
||||
sqrdmulh v3.8h, v30.8h, v0.h[1]
|
||||
str q28, [x0, #0x100]
|
||||
ldr q30, [x0, #0x120]
|
||||
mul v8.8h, v6.8h, v0.h[0]
|
||||
sqrdmulh v27.8h, v10.8h, v1.h[5]
|
||||
mls v11.8h, v3.8h, v7.h[0]
|
||||
mls v18.8h, v2.8h, v7.h[0]
|
||||
ldr q31, [x0, #0x160]
|
||||
sqrdmulh v10.8h, v5.8h, v1.h[1]
|
||||
mls v8.8h, v13.8h, v7.h[0]
|
||||
ldr q13, [x0, #0x1d0]
|
||||
sub v14.8h, v24.8h, v18.8h
|
||||
add v9.8h, v24.8h, v18.8h
|
||||
sqrdmulh v2.8h, v31.8h, v0.h[1]
|
||||
mls v4.8h, v10.8h, v7.h[0]
|
||||
add v10.8h, v25.8h, v23.8h
|
||||
sub v24.8h, v19.8h, v11.8h
|
||||
add v25.8h, v19.8h, v11.8h
|
||||
sqrdmulh v28.8h, v13.8h, v0.h[1]
|
||||
mul v11.8h, v30.8h, v0.h[0]
|
||||
mul v17.8h, v13.8h, v0.h[0]
|
||||
sub v13.8h, v10.8h, v16.8h
|
||||
sub v6.8h, v15.8h, v8.8h
|
||||
mls v17.8h, v28.8h, v7.h[0]
|
||||
str q13, [x0, #0x40]
|
||||
mls v22.8h, v27.8h, v7.h[0]
|
||||
ldr q13, [x0, #0xd0]
|
||||
add v26.8h, v20.8h, v4.8h
|
||||
mul v18.8h, v31.8h, v0.h[0]
|
||||
add v27.8h, v10.8h, v16.8h
|
||||
str q26, [x0, #0x80]
|
||||
sqrdmulh v31.8h, v6.8h, v0.h[5]
|
||||
add v3.8h, v21.8h, v22.8h
|
||||
str q27, [x0], #0x10
|
||||
mul v26.8h, v6.8h, v0.h[4]
|
||||
add v6.8h, v13.8h, v17.8h
|
||||
sub v5.8h, v13.8h, v17.8h
|
||||
str q3, [x0, #0x170]
|
||||
sub v17.8h, v21.8h, v22.8h
|
||||
sqrdmulh v10.8h, v6.8h, v0.h[3]
|
||||
sub v13.8h, v20.8h, v4.8h
|
||||
add v20.8h, v15.8h, v8.8h
|
||||
sqrdmulh v12.8h, v5.8h, v0.h[5]
|
||||
str q13, [x0, #0xb0]
|
||||
mul v8.8h, v6.8h, v0.h[2]
|
||||
str q17, [x0, #0x1b0]
|
||||
mls v8.8h, v10.8h, v7.h[0]
|
||||
mul v29.8h, v5.8h, v0.h[4]
|
||||
mls v29.8h, v12.8h, v7.h[0]
|
||||
sub v5.8h, v9.8h, v8.8h
|
||||
add v3.8h, v9.8h, v8.8h
|
||||
sqrdmulh v15.8h, v20.8h, v0.h[3]
|
||||
mul v4.8h, v5.8h, v1.h[0]
|
||||
add v6.8h, v14.8h, v29.8h
|
||||
sqrdmulh v9.8h, v3.8h, v0.h[7]
|
||||
sqrdmulh v12.8h, v6.8h, v1.h[3]
|
||||
sub v10.8h, v14.8h, v29.8h
|
||||
mul v23.8h, v6.8h, v1.h[2]
|
||||
mls v26.8h, v31.8h, v7.h[0]
|
||||
mls v23.8h, v12.8h, v7.h[0]
|
||||
mul v16.8h, v3.8h, v0.h[6]
|
||||
add v13.8h, v24.8h, v26.8h
|
||||
sub v21.8h, v24.8h, v26.8h
|
||||
mls v16.8h, v9.8h, v7.h[0]
|
||||
add v28.8h, v13.8h, v23.8h
|
||||
sub v13.8h, v13.8h, v23.8h
|
||||
mul v23.8h, v20.8h, v0.h[2]
|
||||
sub x4, x4, #0x1
|
||||
cbnz x4, Lntt_layer123_start
|
||||
sqrdmulh v3.8h, v5.8h, v1.h[1]
|
||||
mls v23.8h, v15.8h, v7.h[0]
|
||||
ldr q5, [x0, #0x190]
|
||||
mul v29.8h, v10.8h, v1.h[4]
|
||||
mls v4.8h, v3.8h, v7.h[0]
|
||||
sub v19.8h, v25.8h, v23.8h
|
||||
sqrdmulh v31.8h, v5.8h, v0.h[1]
|
||||
sqrdmulh v6.8h, v30.8h, v0.h[1]
|
||||
sub v3.8h, v19.8h, v4.8h
|
||||
mul v5.8h, v5.8h, v0.h[0]
|
||||
str q3, [x0, #0xc0]
|
||||
sqrdmulh v12.8h, v10.8h, v1.h[5]
|
||||
mls v18.8h, v2.8h, v7.h[0]
|
||||
ldr q3, [x0, #0x1d0]
|
||||
mls v5.8h, v31.8h, v7.h[0]
|
||||
sqrdmulh v10.8h, v3.8h, v0.h[1]
|
||||
mls v11.8h, v6.8h, v7.h[0]
|
||||
ldr q31, [x0, #0x90]
|
||||
mul v30.8h, v3.8h, v0.h[0]
|
||||
mls v30.8h, v10.8h, v7.h[0]
|
||||
sub v10.8h, v31.8h, v5.8h
|
||||
mls v29.8h, v12.8h, v7.h[0]
|
||||
ldr q6, [x0, #0xd0]
|
||||
sqrdmulh v15.8h, v10.8h, v0.h[5]
|
||||
mul v17.8h, v10.8h, v0.h[4]
|
||||
add v10.8h, v6.8h, v30.8h
|
||||
sub v6.8h, v6.8h, v30.8h
|
||||
sqrdmulh v12.8h, v10.8h, v0.h[3]
|
||||
sub v27.8h, v21.8h, v29.8h
|
||||
sqrdmulh v3.8h, v6.8h, v0.h[5]
|
||||
mul v10.8h, v10.8h, v0.h[2]
|
||||
ldr q20, [x0, #0x50]
|
||||
mls v10.8h, v12.8h, v7.h[0]
|
||||
mul v2.8h, v6.8h, v0.h[4]
|
||||
add v6.8h, v20.8h, v18.8h
|
||||
add v5.8h, v31.8h, v5.8h
|
||||
mls v2.8h, v3.8h, v7.h[0]
|
||||
sub v31.8h, v6.8h, v10.8h
|
||||
sqrdmulh v12.8h, v5.8h, v0.h[3]
|
||||
sub v22.8h, v20.8h, v18.8h
|
||||
add v6.8h, v6.8h, v10.8h
|
||||
mul v20.8h, v31.8h, v1.h[0]
|
||||
add v30.8h, v22.8h, v2.8h
|
||||
sqrdmulh v3.8h, v6.8h, v0.h[7]
|
||||
sqrdmulh v10.8h, v30.8h, v1.h[3]
|
||||
mul v9.8h, v30.8h, v1.h[2]
|
||||
ldr q30, [x0, #0x10]
|
||||
mls v17.8h, v15.8h, v7.h[0]
|
||||
mls v9.8h, v10.8h, v7.h[0]
|
||||
mul v15.8h, v6.8h, v0.h[6]
|
||||
add v24.8h, v30.8h, v11.8h
|
||||
sub v10.8h, v22.8h, v2.8h
|
||||
mls v15.8h, v3.8h, v7.h[0]
|
||||
add v6.8h, v19.8h, v4.8h
|
||||
add v22.8h, v25.8h, v23.8h
|
||||
sqrdmulh v3.8h, v10.8h, v1.h[5]
|
||||
str q13, [x0, #0x140]
|
||||
sub v19.8h, v30.8h, v11.8h
|
||||
add v25.8h, v22.8h, v16.8h
|
||||
mul v5.8h, v5.8h, v0.h[2]
|
||||
sub v13.8h, v22.8h, v16.8h
|
||||
str q28, [x0, #0x100]
|
||||
mls v5.8h, v12.8h, v7.h[0]
|
||||
str q13, [x0, #0x40]
|
||||
str q6, [x0, #0x80]
|
||||
add v21.8h, v21.8h, v29.8h
|
||||
sqrdmulh v13.8h, v31.8h, v1.h[1]
|
||||
str q25, [x0], #0x10
|
||||
add v12.8h, v19.8h, v17.8h
|
||||
sub v31.8h, v19.8h, v17.8h
|
||||
mul v30.8h, v10.8h, v1.h[4]
|
||||
str q21, [x0, #0x170]
|
||||
add v21.8h, v24.8h, v5.8h
|
||||
add v6.8h, v12.8h, v9.8h
|
||||
mls v30.8h, v3.8h, v7.h[0]
|
||||
str q27, [x0, #0x1b0]
|
||||
sub v10.8h, v21.8h, v15.8h
|
||||
sub v12.8h, v12.8h, v9.8h
|
||||
mls v20.8h, v13.8h, v7.h[0]
|
||||
str q6, [x0, #0x100]
|
||||
str q10, [x0, #0x40]
|
||||
sub v13.8h, v24.8h, v5.8h
|
||||
add v3.8h, v21.8h, v15.8h
|
||||
str q12, [x0, #0x140]
|
||||
sub v10.8h, v31.8h, v30.8h
|
||||
add v21.8h, v31.8h, v30.8h
|
||||
str q3, [x0], #0x10
|
||||
add v12.8h, v13.8h, v20.8h
|
||||
sub v13.8h, v13.8h, v20.8h
|
||||
str q21, [x0, #0x170]
|
||||
str q10, [x0, #0x1b0]
|
||||
str q12, [x0, #0x70]
|
||||
str q13, [x0, #0xb0]
|
||||
mov x0, x3
|
||||
mov x4, #0x8 // =8
|
||||
ldr q2, [x0, #0x20]
|
||||
ldr q13, [x1], #0x10
|
||||
ldr q30, [x0, #0x30]
|
||||
ldr q25, [x2, #0x40]
|
||||
ldr q5, [x0]
|
||||
ldr q18, [x0, #0x60]
|
||||
ldr q12, [x0, #0x70]
|
||||
sqrdmulh v17.8h, v2.8h, v13.h[1]
|
||||
ldr q4, [x1], #0x10
|
||||
ldr q23, [x0, #0x10]
|
||||
sqrdmulh v21.8h, v30.8h, v13.h[1]
|
||||
ldr q24, [x2, #0x20]
|
||||
ldr q9, [x2], #0x60
|
||||
mul v10.8h, v30.8h, v13.h[0]
|
||||
mul v11.8h, v2.8h, v13.h[0]
|
||||
mls v10.8h, v21.8h, v7.h[0]
|
||||
sqrdmulh v29.8h, v12.8h, v4.h[1]
|
||||
mul v1.8h, v12.8h, v4.h[0]
|
||||
add v21.8h, v23.8h, v10.8h
|
||||
sub v10.8h, v23.8h, v10.8h
|
||||
mul v8.8h, v18.8h, v4.h[0]
|
||||
sqrdmulh v23.8h, v21.8h, v13.h[3]
|
||||
mul v2.8h, v21.8h, v13.h[2]
|
||||
mls v1.8h, v29.8h, v7.h[0]
|
||||
mls v2.8h, v23.8h, v7.h[0]
|
||||
ldur q15, [x2, #-0x50]
|
||||
sqrdmulh v0.8h, v10.8h, v13.h[5]
|
||||
mls v11.8h, v17.8h, v7.h[0]
|
||||
ldr q29, [x0, #0x50]
|
||||
mul v23.8h, v10.8h, v13.h[4]
|
||||
mls v23.8h, v0.8h, v7.h[0]
|
||||
sub v16.8h, v29.8h, v1.8h
|
||||
add v3.8h, v5.8h, v11.8h
|
||||
sub v31.8h, v5.8h, v11.8h
|
||||
sqrdmulh v22.8h, v16.8h, v4.h[5]
|
||||
add v30.8h, v3.8h, v2.8h
|
||||
sub v0.8h, v3.8h, v2.8h
|
||||
sqrdmulh v28.8h, v18.8h, v4.h[1]
|
||||
add v21.8h, v31.8h, v23.8h
|
||||
sub v19.8h, v31.8h, v23.8h
|
||||
mul v26.8h, v16.8h, v4.h[4]
|
||||
trn2 v3.4s, v30.4s, v0.4s
|
||||
ldur q23, [x2, #-0x10]
|
||||
trn2 v18.4s, v21.4s, v19.4s
|
||||
mls v26.8h, v22.8h, v7.h[0]
|
||||
trn1 v13.4s, v30.4s, v0.4s
|
||||
mls v8.8h, v28.8h, v7.h[0]
|
||||
trn2 v31.2d, v3.2d, v18.2d
|
||||
trn1 v11.4s, v21.4s, v19.4s
|
||||
add v27.8h, v29.8h, v1.8h
|
||||
sqrdmulh v6.8h, v31.8h, v15.8h
|
||||
trn1 v2.2d, v13.2d, v11.2d
|
||||
trn2 v13.2d, v13.2d, v11.2d
|
||||
mul v1.8h, v31.8h, v9.8h
|
||||
ldr q11, [x0, #0x40]
|
||||
sqrdmulh v29.8h, v13.8h, v15.8h
|
||||
mls v1.8h, v6.8h, v7.h[0]
|
||||
trn1 v6.2d, v3.2d, v18.2d
|
||||
mul v17.8h, v13.8h, v9.8h
|
||||
sub v13.8h, v11.8h, v8.8h
|
||||
sqrdmulh v10.8h, v27.8h, v4.h[3]
|
||||
sub v12.8h, v13.8h, v26.8h
|
||||
sub v18.8h, v6.8h, v1.8h
|
||||
mls v17.8h, v29.8h, v7.h[0]
|
||||
add v30.8h, v6.8h, v1.8h
|
||||
add v6.8h, v13.8h, v26.8h
|
||||
ldur q13, [x2, #-0x30]
|
||||
sqrdmulh v16.8h, v18.8h, v23.8h
|
||||
trn1 v28.4s, v6.4s, v12.4s
|
||||
mul v23.8h, v18.8h, v25.8h
|
||||
ldr q25, [x2, #0x10]
|
||||
add v20.8h, v2.8h, v17.8h
|
||||
mul v0.8h, v30.8h, v24.8h
|
||||
sqrdmulh v29.8h, v30.8h, v13.8h
|
||||
sub v30.8h, v2.8h, v17.8h
|
||||
mls v23.8h, v16.8h, v7.h[0]
|
||||
sub x4, x4, #0x2
|
||||
|
||||
Lntt_layer4567_start:
|
||||
ldr q19, [x2, #0x50]
|
||||
sub v31.8h, v30.8h, v23.8h
|
||||
mls v0.8h, v29.8h, v7.h[0]
|
||||
add v16.8h, v11.8h, v8.8h
|
||||
ldr q18, [x0, #0xa0]
|
||||
trn2 v14.4s, v6.4s, v12.4s
|
||||
mul v26.8h, v27.8h, v4.h[2]
|
||||
ldr q4, [x1], #0x10
|
||||
ldr q24, [x2, #0x40]
|
||||
ldr q21, [x0, #0xb0]
|
||||
mls v26.8h, v10.8h, v7.h[0]
|
||||
add v23.8h, v30.8h, v23.8h
|
||||
sub v15.8h, v20.8h, v0.8h
|
||||
ldr q9, [x0, #0x90]
|
||||
add v10.8h, v20.8h, v0.8h
|
||||
mul v8.8h, v18.8h, v4.h[0]
|
||||
ldr q1, [x2], #0x60
|
||||
trn1 v27.4s, v23.4s, v31.4s
|
||||
sqrdmulh v12.8h, v18.8h, v4.h[1]
|
||||
trn1 v5.4s, v10.4s, v15.4s
|
||||
sub v30.8h, v16.8h, v26.8h
|
||||
trn2 v13.2d, v5.2d, v27.2d
|
||||
sqrdmulh v2.8h, v21.8h, v4.h[1]
|
||||
add v29.8h, v16.8h, v26.8h
|
||||
mul v0.8h, v21.8h, v4.h[0]
|
||||
str q13, [x0, #0x20]
|
||||
trn1 v11.4s, v29.4s, v30.4s
|
||||
mls v8.8h, v12.8h, v7.h[0]
|
||||
trn2 v26.4s, v29.4s, v30.4s
|
||||
trn2 v6.2d, v11.2d, v28.2d
|
||||
mls v0.8h, v2.8h, v7.h[0]
|
||||
trn2 v16.2d, v26.2d, v14.2d
|
||||
trn1 v26.2d, v26.2d, v14.2d
|
||||
trn1 v20.2d, v5.2d, v27.2d
|
||||
sqrdmulh v29.8h, v6.8h, v25.8h
|
||||
trn2 v15.4s, v10.4s, v15.4s
|
||||
sqrdmulh v13.8h, v16.8h, v25.8h
|
||||
str q20, [x0], #0x40
|
||||
sub v30.8h, v9.8h, v0.8h
|
||||
add v27.8h, v9.8h, v0.8h
|
||||
mul v17.8h, v6.8h, v1.8h
|
||||
sqrdmulh v22.8h, v30.8h, v4.h[5]
|
||||
mul v18.8h, v16.8h, v1.8h
|
||||
mls v18.8h, v13.8h, v7.h[0]
|
||||
mul v2.8h, v30.8h, v4.h[4]
|
||||
mls v2.8h, v22.8h, v7.h[0]
|
||||
trn2 v22.4s, v23.4s, v31.4s
|
||||
sub v3.8h, v26.8h, v18.8h
|
||||
ldur q25, [x2, #-0x30]
|
||||
mls v17.8h, v29.8h, v7.h[0]
|
||||
trn2 v31.2d, v15.2d, v22.2d
|
||||
trn1 v20.2d, v15.2d, v22.2d
|
||||
add v16.8h, v26.8h, v18.8h
|
||||
sqrdmulh v26.8h, v3.8h, v19.8h
|
||||
trn1 v21.2d, v11.2d, v28.2d
|
||||
ldr q11, [x0, #0x40]
|
||||
sqrdmulh v29.8h, v16.8h, v25.8h
|
||||
stur q20, [x0, #-0x30]
|
||||
add v20.8h, v21.8h, v17.8h
|
||||
stur q31, [x0, #-0x10]
|
||||
mul v23.8h, v3.8h, v24.8h
|
||||
ldr q25, [x2, #0x10]
|
||||
sub v13.8h, v11.8h, v8.8h
|
||||
mls v23.8h, v26.8h, v7.h[0]
|
||||
ldur q1, [x2, #-0x40]
|
||||
sub v12.8h, v13.8h, v2.8h
|
||||
add v6.8h, v13.8h, v2.8h
|
||||
sqrdmulh v10.8h, v27.8h, v4.h[3]
|
||||
sub v30.8h, v21.8h, v17.8h
|
||||
mul v0.8h, v16.8h, v1.8h
|
||||
trn1 v28.4s, v6.4s, v12.4s
|
||||
sub x4, x4, #0x1
|
||||
cbnz x4, Lntt_layer4567_start
|
||||
add v22.8h, v11.8h, v8.8h
|
||||
mul v27.8h, v27.8h, v4.h[2]
|
||||
trn2 v17.4s, v6.4s, v12.4s
|
||||
ldr q15, [x2], #0x60
|
||||
mls v27.8h, v10.8h, v7.h[0]
|
||||
add v4.8h, v30.8h, v23.8h
|
||||
sub v18.8h, v30.8h, v23.8h
|
||||
ldur q6, [x2, #-0x30]
|
||||
mls v0.8h, v29.8h, v7.h[0]
|
||||
ldur q12, [x2, #-0x40]
|
||||
ldur q24, [x2, #-0x20]
|
||||
ldur q2, [x2, #-0x10]
|
||||
trn1 v9.4s, v4.4s, v18.4s
|
||||
add v10.8h, v22.8h, v27.8h
|
||||
sub v13.8h, v22.8h, v27.8h
|
||||
sub v1.8h, v20.8h, v0.8h
|
||||
trn2 v21.4s, v10.4s, v13.4s
|
||||
add v27.8h, v20.8h, v0.8h
|
||||
trn2 v3.2d, v21.2d, v17.2d
|
||||
trn1 v13.4s, v10.4s, v13.4s
|
||||
trn1 v31.4s, v27.4s, v1.4s
|
||||
sqrdmulh v10.8h, v3.8h, v25.8h
|
||||
trn2 v5.2d, v13.2d, v28.2d
|
||||
trn1 v13.2d, v13.2d, v28.2d
|
||||
trn1 v21.2d, v21.2d, v17.2d
|
||||
sqrdmulh v17.8h, v5.8h, v25.8h
|
||||
trn2 v30.2d, v31.2d, v9.2d
|
||||
mul v25.8h, v3.8h, v15.8h
|
||||
str q30, [x0, #0x20]
|
||||
trn2 v30.4s, v4.4s, v18.4s
|
||||
mls v25.8h, v10.8h, v7.h[0]
|
||||
trn2 v3.4s, v27.4s, v1.4s
|
||||
mul v20.8h, v5.8h, v15.8h
|
||||
trn2 v10.2d, v3.2d, v30.2d
|
||||
mls v20.8h, v17.8h, v7.h[0]
|
||||
str q10, [x0, #0x30]
|
||||
sub v18.8h, v21.8h, v25.8h
|
||||
add v10.8h, v21.8h, v25.8h
|
||||
trn1 v3.2d, v3.2d, v30.2d
|
||||
sqrdmulh v30.8h, v18.8h, v2.8h
|
||||
mul v12.8h, v10.8h, v12.8h
|
||||
sqrdmulh v6.8h, v10.8h, v6.8h
|
||||
str q3, [x0, #0x10]
|
||||
add v21.8h, v13.8h, v20.8h
|
||||
mul v10.8h, v18.8h, v24.8h
|
||||
sub v13.8h, v13.8h, v20.8h
|
||||
mls v10.8h, v30.8h, v7.h[0]
|
||||
mls v12.8h, v6.8h, v7.h[0]
|
||||
trn1 v30.2d, v31.2d, v9.2d
|
||||
sub v3.8h, v13.8h, v10.8h
|
||||
add v6.8h, v13.8h, v10.8h
|
||||
add v10.8h, v21.8h, v12.8h
|
||||
sub v21.8h, v21.8h, v12.8h
|
||||
trn2 v13.4s, v6.4s, v3.4s
|
||||
trn1 v12.4s, v10.4s, v21.4s
|
||||
trn2 v21.4s, v10.4s, v21.4s
|
||||
trn1 v3.4s, v6.4s, v3.4s
|
||||
str q30, [x0], #0x40
|
||||
trn2 v10.2d, v21.2d, v13.2d
|
||||
trn1 v13.2d, v21.2d, v13.2d
|
||||
trn2 v21.2d, v12.2d, v3.2d
|
||||
trn1 v3.2d, v12.2d, v3.2d
|
||||
str q10, [x0, #0x30]
|
||||
str q13, [x0, #0x10]
|
||||
str q3, [x0], #0x40
|
||||
stur q21, [x0, #-0x20]
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(ntt_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: poly_mulcache_compute_aarch64_asm
|
||||
Description: Compute multiplication cache for polynomial
|
||||
Signature: void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], const int16_t mlk_poly[256], const int16_t zetas[128], const int16_t zetas_twisted[128])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 256
|
||||
permissions: write-only
|
||||
c_parameter: int16_t cache[128]
|
||||
description: Output cache
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t mlk_poly[256]
|
||||
description: Input polynomial
|
||||
x2:
|
||||
type: buffer
|
||||
size_bytes: 256
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t zetas[128]
|
||||
description: Zeta values
|
||||
x3:
|
||||
type: buffer
|
||||
size_bytes: 256
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t zetas_twisted[128]
|
||||
description: Twisted zeta values
|
||||
Stack:
|
||||
bytes: 0
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/poly_mulcache_compute_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_mulcache_compute_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_mulcache_compute_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
mov w5, #0xd01 // =3329
|
||||
dup v6.8h, w5
|
||||
mov w5, #0x4ebf // =20159
|
||||
dup v7.8h, w5
|
||||
mov x4, #0x10 // =16
|
||||
ldr q0, [x1], #0x20
|
||||
ldur q2, [x1, #-0x10]
|
||||
ldr q19, [x1], #0x20
|
||||
ldr q29, [x3], #0x10
|
||||
ldur q16, [x1, #-0x10]
|
||||
ldr q18, [x2], #0x10
|
||||
ldr q26, [x1], #0x20
|
||||
ldr q25, [x2], #0x10
|
||||
uzp2 v5.8h, v0.8h, v2.8h
|
||||
ldr q28, [x3], #0x10
|
||||
ldur q7, [x1, #-0x10]
|
||||
ldr q2, [x1], #0x20
|
||||
uzp2 v27.8h, v19.8h, v16.8h
|
||||
sqrdmulh v16.8h, v5.8h, v29.8h
|
||||
ldr q17, [x3], #0x10
|
||||
ldr q19, [x3], #0x10
|
||||
mul v5.8h, v5.8h, v18.8h
|
||||
uzp2 v29.8h, v26.8h, v7.8h
|
||||
mul v26.8h, v27.8h, v25.8h
|
||||
sqrdmulh v4.8h, v27.8h, v28.8h
|
||||
mls v5.8h, v16.8h, v6.h[0]
|
||||
lsr x4, x4, #1
|
||||
sub x4, x4, #0x2
|
||||
|
||||
Lpoly_mulcache_compute_loop_start:
|
||||
str q5, [x0], #0x10
|
||||
sqrdmulh v22.8h, v29.8h, v17.8h
|
||||
ldr q28, [x2], #0x10
|
||||
ldur q24, [x1, #-0x10]
|
||||
ldr q0, [x1], #0x20
|
||||
mls v26.8h, v4.8h, v6.h[0]
|
||||
ldur q16, [x1, #-0x10]
|
||||
ldr q17, [x3], #0x10
|
||||
mul v5.8h, v29.8h, v28.8h
|
||||
uzp2 v23.8h, v2.8h, v24.8h
|
||||
ldr q18, [x2], #0x10
|
||||
mls v5.8h, v22.8h, v6.h[0]
|
||||
uzp2 v29.8h, v0.8h, v16.8h
|
||||
sqrdmulh v4.8h, v23.8h, v19.8h
|
||||
ldr q2, [x1], #0x20
|
||||
ldr q19, [x3], #0x10
|
||||
str q26, [x0], #0x10
|
||||
mul v26.8h, v23.8h, v18.8h
|
||||
subs x4, x4, #0x1
|
||||
cbnz x4, Lpoly_mulcache_compute_loop_start
|
||||
mls v26.8h, v4.8h, v6.h[0]
|
||||
str q5, [x0], #0x10
|
||||
ldr q5, [x2], #0x10
|
||||
ldur q4, [x1, #-0x10]
|
||||
sqrdmulh v16.8h, v29.8h, v17.8h
|
||||
ldr q0, [x2], #0x10
|
||||
mul v29.8h, v29.8h, v5.8h
|
||||
uzp2 v18.8h, v2.8h, v4.8h
|
||||
str q26, [x0], #0x10
|
||||
sqrdmulh v17.8h, v18.8h, v19.8h
|
||||
mls v29.8h, v16.8h, v6.h[0]
|
||||
mul v26.8h, v18.8h, v0.8h
|
||||
mls v26.8h, v17.8h, v6.h[0]
|
||||
str q29, [x0], #0x10
|
||||
str q26, [x0], #0x10
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_mulcache_compute_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: poly_reduce_aarch64_asm
|
||||
Description: Barrett reduction of polynomial coefficients
|
||||
Signature: void mlk_poly_reduce_aarch64_asm(int16_t p[256])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t p[256]
|
||||
description: Input/output polynomial
|
||||
Stack:
|
||||
bytes: 0
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/poly_reduce_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_reduce_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_reduce_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
mov w2, #0xd01 // =3329
|
||||
dup v3.8h, w2
|
||||
mov w2, #0x4ebf // =20159
|
||||
dup v4.8h, w2
|
||||
mov x1, #0x8 // =8
|
||||
ldr q21, [x0], #0x40
|
||||
ldur q18, [x0, #-0x20]
|
||||
ldur q0, [x0, #-0x30]
|
||||
ldur q5, [x0, #-0x10]
|
||||
ldr q26, [x0], #0x40
|
||||
sqdmulh v17.8h, v21.8h, v4.h[0]
|
||||
sqdmulh v27.8h, v18.8h, v4.h[0]
|
||||
sqdmulh v22.8h, v0.8h, v4.h[0]
|
||||
srshr v17.8h, v17.8h, #0xb
|
||||
sqdmulh v23.8h, v5.8h, v4.h[0]
|
||||
srshr v29.8h, v27.8h, #0xb
|
||||
mls v21.8h, v17.8h, v3.h[0]
|
||||
srshr v17.8h, v22.8h, #0xb
|
||||
mls v18.8h, v29.8h, v3.h[0]
|
||||
srshr v22.8h, v23.8h, #0xb
|
||||
mls v0.8h, v17.8h, v3.h[0]
|
||||
sshr v2.8h, v21.8h, #0xf
|
||||
mls v5.8h, v22.8h, v3.h[0]
|
||||
sshr v29.8h, v18.8h, #0xf
|
||||
and v19.16b, v3.16b, v2.16b
|
||||
sqdmulh v2.8h, v26.8h, v4.h[0]
|
||||
sshr v31.8h, v0.8h, #0xf
|
||||
add v17.8h, v21.8h, v19.8h
|
||||
and v21.16b, v3.16b, v29.16b
|
||||
and v31.16b, v3.16b, v31.16b
|
||||
sub x1, x1, #0x2
|
||||
|
||||
Lpoly_reduce_loop_start:
|
||||
add v21.8h, v18.8h, v21.8h
|
||||
ldur q18, [x0, #-0x20]
|
||||
add v25.8h, v0.8h, v31.8h
|
||||
ldur q0, [x0, #-0x30]
|
||||
stur q21, [x0, #-0x60]
|
||||
sshr v28.8h, v5.8h, #0xf
|
||||
stur q17, [x0, #-0x80]
|
||||
srshr v23.8h, v2.8h, #0xb
|
||||
sqdmulh v30.8h, v18.8h, v4.h[0]
|
||||
stur q25, [x0, #-0x70]
|
||||
and v22.16b, v3.16b, v28.16b
|
||||
sqdmulh v7.8h, v0.8h, v4.h[0]
|
||||
add v16.8h, v5.8h, v22.8h
|
||||
ldur q5, [x0, #-0x10]
|
||||
mls v26.8h, v23.8h, v3.h[0]
|
||||
stur q16, [x0, #-0x50]
|
||||
srshr v6.8h, v30.8h, #0xb
|
||||
srshr v1.8h, v7.8h, #0xb
|
||||
sqdmulh v19.8h, v5.8h, v4.h[0]
|
||||
mls v18.8h, v6.8h, v3.h[0]
|
||||
sshr v24.8h, v26.8h, #0xf
|
||||
mls v0.8h, v1.8h, v3.h[0]
|
||||
and v27.16b, v3.16b, v24.16b
|
||||
srshr v29.8h, v19.8h, #0xb
|
||||
add v17.8h, v26.8h, v27.8h
|
||||
ldr q26, [x0], #0x40
|
||||
sshr v1.8h, v18.8h, #0xf
|
||||
mls v5.8h, v29.8h, v3.h[0]
|
||||
sshr v20.8h, v0.8h, #0xf
|
||||
and v21.16b, v3.16b, v1.16b
|
||||
and v31.16b, v3.16b, v20.16b
|
||||
sqdmulh v2.8h, v26.8h, v4.h[0]
|
||||
subs x1, x1, #0x1
|
||||
cbnz x1, Lpoly_reduce_loop_start
|
||||
add v28.8h, v0.8h, v31.8h
|
||||
ldur q29, [x0, #-0x10]
|
||||
add v21.8h, v18.8h, v21.8h
|
||||
srshr v18.8h, v2.8h, #0xb
|
||||
sshr v2.8h, v5.8h, #0xf
|
||||
ldur q16, [x0, #-0x20]
|
||||
stur q17, [x0, #-0x80]
|
||||
ldur q0, [x0, #-0x30]
|
||||
and v2.16b, v3.16b, v2.16b
|
||||
sqdmulh v24.8h, v29.8h, v4.h[0]
|
||||
stur q28, [x0, #-0x70]
|
||||
stur q21, [x0, #-0x60]
|
||||
add v31.8h, v5.8h, v2.8h
|
||||
sqdmulh v6.8h, v16.8h, v4.h[0]
|
||||
stur q31, [x0, #-0x50]
|
||||
sqdmulh v17.8h, v0.8h, v4.h[0]
|
||||
srshr v22.8h, v24.8h, #0xb
|
||||
mls v26.8h, v18.8h, v3.h[0]
|
||||
srshr v31.8h, v6.8h, #0xb
|
||||
mls v29.8h, v22.8h, v3.h[0]
|
||||
srshr v19.8h, v17.8h, #0xb
|
||||
mls v16.8h, v31.8h, v3.h[0]
|
||||
sshr v7.8h, v26.8h, #0xf
|
||||
mls v0.8h, v19.8h, v3.h[0]
|
||||
and v5.16b, v3.16b, v7.16b
|
||||
sshr v22.8h, v29.8h, #0xf
|
||||
add v27.8h, v26.8h, v5.8h
|
||||
and v26.16b, v3.16b, v22.16b
|
||||
sshr v20.8h, v16.8h, #0xf
|
||||
stur q27, [x0, #-0x40]
|
||||
and v2.16b, v3.16b, v20.16b
|
||||
sshr v23.8h, v0.8h, #0xf
|
||||
add v18.8h, v29.8h, v26.8h
|
||||
add v31.8h, v16.8h, v2.8h
|
||||
and v29.16b, v3.16b, v23.16b
|
||||
stur q18, [x0, #-0x10]
|
||||
add v25.8h, v0.8h, v29.8h
|
||||
stur q31, [x0, #-0x20]
|
||||
stur q25, [x0, #-0x30]
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_reduce_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: poly_tobytes_aarch64_asm
|
||||
Description: Convert polynomial to byte representation
|
||||
Signature: void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 384
|
||||
permissions: write-only
|
||||
c_parameter: uint8_t r[384]
|
||||
description: Output byte array
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t a[256]
|
||||
description: Input polynomial
|
||||
Stack:
|
||||
bytes: 0
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/poly_tobytes_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_tobytes_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_tobytes_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
mov x2, #0x10 // =16
|
||||
ldr q5, [x1, #0x10]
|
||||
ldr q3, [x1], #0x20
|
||||
ldr q29, [x1], #0x20
|
||||
ldur q2, [x1, #-0x10]
|
||||
ldr q27, [x1, #0x10]
|
||||
ldr q23, [x1, #0x30]
|
||||
ldr q17, [x1], #0x20
|
||||
ldr q16, [x1], #0x20
|
||||
uzp2 v26.8h, v3.8h, v5.8h
|
||||
uzp1 v19.8h, v3.8h, v5.8h
|
||||
uzp2 v0.8h, v29.8h, v2.8h
|
||||
uzp1 v1.8h, v29.8h, v2.8h
|
||||
xtn v5.8b, v26.8h
|
||||
shrn v3.8b, v19.8h, #0x8
|
||||
shrn v4.8b, v26.8h, #0x4
|
||||
xtn v18.8b, v0.8h
|
||||
shrn v30.8b, v0.8h, #0x4
|
||||
xtn v28.8b, v1.8h
|
||||
shrn v29.8b, v1.8h, #0x8
|
||||
sli v3.8b, v5.8b, #0x4
|
||||
xtn v2.8b, v19.8h
|
||||
sli v29.8b, v18.8b, #0x4
|
||||
lsr x2, x2, #1
|
||||
sub x2, x2, #0x2
|
||||
|
||||
Lpoly_tobytes_loop_start:
|
||||
uzp1 v25.8h, v17.8h, v27.8h
|
||||
uzp2 v31.8h, v17.8h, v27.8h
|
||||
uzp1 v24.8h, v16.8h, v23.8h
|
||||
uzp2 v6.8h, v16.8h, v23.8h
|
||||
st3 { v2.8b, v3.8b, v4.8b }, [x0], #24
|
||||
shrn v3.8b, v25.8h, #0x8
|
||||
ldr q17, [x1], #0x20
|
||||
shrn v4.8b, v31.8h, #0x4
|
||||
xtn v21.8b, v6.8h
|
||||
ldr q23, [x1, #0x10]
|
||||
st3 { v28.8b, v29.8b, v30.8b }, [x0], #24
|
||||
shrn v29.8b, v24.8h, #0x8
|
||||
ldur q27, [x1, #-0x10]
|
||||
xtn v20.8b, v31.8h
|
||||
ldr q16, [x1], #0x20
|
||||
sli v29.8b, v21.8b, #0x4
|
||||
xtn v2.8b, v25.8h
|
||||
sli v3.8b, v20.8b, #0x4
|
||||
xtn v28.8b, v24.8h
|
||||
shrn v30.8b, v6.8h, #0x4
|
||||
subs x2, x2, #0x1
|
||||
cbnz x2, Lpoly_tobytes_loop_start
|
||||
uzp2 v7.8h, v17.8h, v27.8h
|
||||
uzp1 v25.8h, v17.8h, v27.8h
|
||||
uzp2 v0.8h, v16.8h, v23.8h
|
||||
st3 { v2.8b, v3.8b, v4.8b }, [x0], #24
|
||||
st3 { v28.8b, v29.8b, v30.8b }, [x0], #24
|
||||
shrn v21.8b, v25.8h, #0x8
|
||||
uzp1 v2.8h, v16.8h, v23.8h
|
||||
shrn v22.8b, v7.8h, #0x4
|
||||
shrn v4.8b, v0.8h, #0x4
|
||||
xtn v28.8b, v7.8h
|
||||
xtn v27.8b, v0.8h
|
||||
shrn v3.8b, v2.8h, #0x8
|
||||
sli v21.8b, v28.8b, #0x4
|
||||
xtn v2.8b, v2.8h
|
||||
sli v3.8b, v27.8b, #0x4
|
||||
xtn v20.8b, v25.8h
|
||||
st3 { v20.8b, v21.8b, v22.8b }, [x0], #24
|
||||
st3 { v2.8b, v3.8b, v4.8b }, [x0], #24
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_tobytes_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: poly_tomont_aarch64_asm
|
||||
Description: Convert polynomial to Montgomery domain
|
||||
Signature: void mlk_poly_tomont_aarch64_asm(int16_t p[256])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t p[256]
|
||||
description: Input/output polynomial
|
||||
Stack:
|
||||
bytes: 0
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/poly_tomont_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_tomont_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_tomont_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
mov w2, #0xd01 // =3329
|
||||
dup v4.8h, w2
|
||||
mov w2, #-0x414 // =-1044
|
||||
dup v2.8h, w2
|
||||
mov w2, #-0x2824 // =-10276
|
||||
dup v3.8h, w2
|
||||
mov x1, #0x8 // =8
|
||||
ldr q18, [x0, #0x20]
|
||||
ldr q0, [x0, #0x10]
|
||||
ldr q16, [x0], #0x40
|
||||
sqrdmulh v23.8h, v0.8h, v3.8h
|
||||
mul v26.8h, v0.8h, v2.8h
|
||||
sqrdmulh v19.8h, v16.8h, v3.8h
|
||||
mls v26.8h, v23.8h, v4.h[0]
|
||||
mul v29.8h, v16.8h, v2.8h
|
||||
ldur q16, [x0, #-0x10]
|
||||
mls v29.8h, v19.8h, v4.h[0]
|
||||
stur q26, [x0, #-0x30]
|
||||
sqrdmulh v26.8h, v18.8h, v3.8h
|
||||
mul v18.8h, v18.8h, v2.8h
|
||||
stur q29, [x0, #-0x40]
|
||||
sqrdmulh v29.8h, v16.8h, v3.8h
|
||||
mls v18.8h, v26.8h, v4.h[0]
|
||||
sub x1, x1, #0x1
|
||||
|
||||
Lpoly_tomont_loop:
|
||||
ldr q19, [x0, #0x10]
|
||||
mul v26.8h, v16.8h, v2.8h
|
||||
ldr q23, [x0, #0x20]
|
||||
ldr q17, [x0], #0x40
|
||||
mls v26.8h, v29.8h, v4.h[0]
|
||||
ldur q16, [x0, #-0x10]
|
||||
sqrdmulh v28.8h, v19.8h, v3.8h
|
||||
stur q18, [x0, #-0x60]
|
||||
mul v0.8h, v19.8h, v2.8h
|
||||
stur q26, [x0, #-0x50]
|
||||
sqrdmulh v24.8h, v23.8h, v3.8h
|
||||
mul v18.8h, v23.8h, v2.8h
|
||||
sqrdmulh v22.8h, v17.8h, v3.8h
|
||||
mul v26.8h, v17.8h, v2.8h
|
||||
mls v0.8h, v28.8h, v4.h[0]
|
||||
mls v26.8h, v22.8h, v4.h[0]
|
||||
sqrdmulh v29.8h, v16.8h, v3.8h
|
||||
stur q0, [x0, #-0x30]
|
||||
mls v18.8h, v24.8h, v4.h[0]
|
||||
stur q26, [x0, #-0x40]
|
||||
sub x1, x1, #0x1
|
||||
cbnz x1, Lpoly_tomont_loop
|
||||
mul v16.8h, v16.8h, v2.8h
|
||||
stur q18, [x0, #-0x20]
|
||||
mls v16.8h, v29.8h, v4.h[0]
|
||||
stur q16, [x0, #-0x10]
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_tomont_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,263 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [NeonNTT]
|
||||
* Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
|
||||
* Becker, Hwang, Kannwischer, Yang, Yang
|
||||
* https://eprint.iacr.org/2021/986
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm
|
||||
Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=2
|
||||
Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(int16_t r[256], const int16_t a[512], const int16_t b[512], const int16_t b_cache[256])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: write-only
|
||||
c_parameter: int16_t r[256]
|
||||
description: Output polynomial
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 1024
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t a[512]
|
||||
description: Input polynomial vector a
|
||||
x2:
|
||||
type: buffer
|
||||
size_bytes: 1024
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t b[512]
|
||||
description: Input polynomial vector b
|
||||
x3:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t b_cache[256]
|
||||
description: Cached values for b
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: saving callee-saved Neon registers
|
||||
*/
|
||||
|
||||
/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
mov w14, #0xd01 // =3329
|
||||
dup v0.8h, w14
|
||||
mov w14, #0xcff // =3327
|
||||
dup v2.8h, w14
|
||||
add x4, x1, #0x200
|
||||
add x5, x2, #0x200
|
||||
add x6, x3, #0x100
|
||||
mov x13, #0x10 // =16
|
||||
ldr q12, [x1], #0x20
|
||||
ldur q9, [x1, #-0x10]
|
||||
ldr q22, [x2], #0x20
|
||||
ldur q30, [x2, #-0x10]
|
||||
ldr q6, [x5], #0x20
|
||||
ldr q7, [x4, #0x10]
|
||||
ldr q8, [x4], #0x20
|
||||
ldur q23, [x5, #-0x10]
|
||||
uzp1 v16.8h, v12.8h, v9.8h
|
||||
uzp2 v14.8h, v12.8h, v9.8h
|
||||
uzp2 v13.8h, v22.8h, v30.8h
|
||||
uzp1 v18.8h, v22.8h, v30.8h
|
||||
ld1 { v27.8h }, [x3], #16
|
||||
ld1 { v17.8h }, [x6], #16
|
||||
smull2 v4.4s, v16.8h, v18.8h
|
||||
ldr q31, [x1, #0x10]
|
||||
smull v19.4s, v16.4h, v13.4h
|
||||
ldr q24, [x1], #0x20
|
||||
smlal v19.4s, v14.4h, v18.4h
|
||||
ldr q22, [x2], #0x20
|
||||
smlal2 v4.4s, v14.8h, v27.8h
|
||||
uzp2 v5.8h, v6.8h, v23.8h
|
||||
smull2 v29.4s, v16.8h, v13.8h
|
||||
uzp2 v26.8h, v8.8h, v7.8h
|
||||
smlal2 v29.4s, v14.8h, v18.8h
|
||||
uzp1 v30.8h, v24.8h, v31.8h
|
||||
uzp1 v8.8h, v8.8h, v7.8h
|
||||
smull v11.4s, v16.4h, v18.4h
|
||||
smlal v11.4s, v14.4h, v27.4h
|
||||
ldur q1, [x2, #-0x10]
|
||||
uzp1 v28.8h, v6.8h, v23.8h
|
||||
smlal2 v29.4s, v8.8h, v5.8h
|
||||
ldr q25, [x5], #0x20
|
||||
smlal v19.4s, v8.4h, v5.4h
|
||||
ldr q3, [x4, #0x10]
|
||||
smlal2 v29.4s, v26.8h, v28.8h
|
||||
uzp1 v27.8h, v22.8h, v1.8h
|
||||
smlal v19.4s, v26.4h, v28.4h
|
||||
ldr q12, [x4], #0x20
|
||||
smlal2 v4.4s, v8.8h, v28.8h
|
||||
ldur q21, [x5, #-0x10]
|
||||
smlal2 v4.4s, v26.8h, v17.8h
|
||||
smlal v11.4s, v8.4h, v28.4h
|
||||
ld1 { v15.8h }, [x6], #16
|
||||
smlal v11.4s, v26.4h, v17.4h
|
||||
ld1 { v20.8h }, [x3], #16
|
||||
uzp1 v28.8h, v19.8h, v29.8h
|
||||
smull2 v23.4s, v30.8h, v27.8h
|
||||
smull v26.4s, v30.4h, v27.4h
|
||||
uzp2 v16.8h, v22.8h, v1.8h
|
||||
mul v28.8h, v28.8h, v2.8h
|
||||
uzp1 v10.8h, v11.8h, v4.8h
|
||||
smull2 v8.4s, v30.8h, v16.8h
|
||||
mul v13.8h, v10.8h, v2.8h
|
||||
smlal v19.4s, v28.4h, v0.4h
|
||||
smlal2 v29.4s, v28.8h, v0.8h
|
||||
smull v18.4s, v30.4h, v16.4h
|
||||
uzp1 v30.8h, v25.8h, v21.8h
|
||||
smlal v11.4s, v13.4h, v0.4h
|
||||
uzp2 v6.8h, v24.8h, v31.8h
|
||||
uzp1 v16.8h, v12.8h, v3.8h
|
||||
smlal2 v4.4s, v13.8h, v0.8h
|
||||
uzp2 v17.8h, v25.8h, v21.8h
|
||||
smlal2 v8.4s, v6.8h, v27.8h
|
||||
uzp2 v12.8h, v12.8h, v3.8h
|
||||
smlal v18.4s, v6.4h, v27.4h
|
||||
uzp2 v9.8h, v19.8h, v29.8h
|
||||
smlal2 v8.4s, v16.8h, v17.8h
|
||||
smlal2 v8.4s, v12.8h, v30.8h
|
||||
uzp2 v19.8h, v11.8h, v4.8h
|
||||
sub x13, x13, #0x2
|
||||
|
||||
Lpolyvec_basemul_acc_montgomery_cached_k2_loop_start:
|
||||
smlal v18.4s, v16.4h, v17.4h
|
||||
ldr q7, [x4], #0x20
|
||||
ldr q10, [x2, #0x10]
|
||||
smlal v18.4s, v12.4h, v30.4h
|
||||
smlal2 v23.4s, v6.8h, v20.8h
|
||||
ldr q14, [x2], #0x20
|
||||
smlal2 v23.4s, v16.8h, v30.8h
|
||||
zip1 v25.8h, v19.8h, v9.8h
|
||||
zip2 v3.8h, v19.8h, v9.8h
|
||||
smlal2 v23.4s, v12.8h, v15.8h
|
||||
smlal v26.4s, v6.4h, v20.4h
|
||||
uzp1 v5.8h, v18.8h, v8.8h
|
||||
uzp2 v21.8h, v14.8h, v10.8h
|
||||
smlal v26.4s, v16.4h, v30.4h
|
||||
str q25, [x0], #0x20
|
||||
mul v29.8h, v5.8h, v2.8h
|
||||
uzp1 v24.8h, v14.8h, v10.8h
|
||||
stur q3, [x0, #-0x10]
|
||||
smlal v26.4s, v12.4h, v15.4h
|
||||
ld1 { v15.8h }, [x6], #16
|
||||
ldr q28, [x1, #0x10]
|
||||
ldr q11, [x1], #0x20
|
||||
ldr q13, [x5], #0x20
|
||||
ldur q27, [x4, #-0x10]
|
||||
smlal2 v8.4s, v29.8h, v0.8h
|
||||
ldur q22, [x5, #-0x10]
|
||||
smlal v18.4s, v29.4h, v0.4h
|
||||
uzp1 v4.8h, v26.8h, v23.8h
|
||||
uzp1 v1.8h, v11.8h, v28.8h
|
||||
uzp2 v6.8h, v11.8h, v28.8h
|
||||
uzp1 v16.8h, v7.8h, v27.8h
|
||||
mul v31.8h, v4.8h, v2.8h
|
||||
uzp2 v17.8h, v13.8h, v22.8h
|
||||
ld1 { v20.8h }, [x3], #16
|
||||
uzp2 v9.8h, v18.8h, v8.8h
|
||||
smull2 v8.4s, v1.8h, v21.8h
|
||||
uzp1 v30.8h, v13.8h, v22.8h
|
||||
smlal2 v8.4s, v6.8h, v24.8h
|
||||
smlal2 v8.4s, v16.8h, v17.8h
|
||||
uzp2 v12.8h, v7.8h, v27.8h
|
||||
smlal v26.4s, v31.4h, v0.4h
|
||||
smlal2 v23.4s, v31.8h, v0.8h
|
||||
smull v18.4s, v1.4h, v21.4h
|
||||
smlal v18.4s, v6.4h, v24.4h
|
||||
smlal2 v8.4s, v12.8h, v30.8h
|
||||
uzp2 v19.8h, v26.8h, v23.8h
|
||||
smull2 v23.4s, v1.8h, v24.8h
|
||||
smull v26.4s, v1.4h, v24.4h
|
||||
subs x13, x13, #0x1
|
||||
cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k2_loop_start
|
||||
smlal v26.4s, v6.4h, v20.4h
|
||||
smlal2 v23.4s, v6.8h, v20.8h
|
||||
smlal v26.4s, v16.4h, v30.4h
|
||||
smlal2 v23.4s, v16.8h, v30.8h
|
||||
smlal v26.4s, v12.4h, v15.4h
|
||||
smlal2 v23.4s, v12.8h, v15.8h
|
||||
smlal v18.4s, v16.4h, v17.4h
|
||||
smlal v18.4s, v12.4h, v30.4h
|
||||
zip1 v12.8h, v19.8h, v9.8h
|
||||
str q12, [x0], #0x20
|
||||
uzp1 v12.8h, v26.8h, v23.8h
|
||||
mul v6.8h, v12.8h, v2.8h
|
||||
uzp1 v12.8h, v18.8h, v8.8h
|
||||
mul v12.8h, v12.8h, v2.8h
|
||||
smlal v26.4s, v6.4h, v0.4h
|
||||
smlal2 v23.4s, v6.8h, v0.8h
|
||||
smlal2 v8.4s, v12.8h, v0.8h
|
||||
smlal v18.4s, v12.4h, v0.4h
|
||||
zip2 v12.8h, v19.8h, v9.8h
|
||||
uzp2 v6.8h, v26.8h, v23.8h
|
||||
stur q12, [x0, #-0x10]
|
||||
uzp2 v12.8h, v18.8h, v8.8h
|
||||
zip2 v1.8h, v6.8h, v12.8h
|
||||
zip1 v12.8h, v6.8h, v12.8h
|
||||
str q1, [x0, #0x10]
|
||||
str q12, [x0], #0x20
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm)
|
||||
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
(MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,316 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [NeonNTT]
|
||||
* Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
|
||||
* Becker, Hwang, Kannwischer, Yang, Yang
|
||||
* https://eprint.iacr.org/2021/986
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm
|
||||
Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=3
|
||||
Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(int16_t r[256], const int16_t a[768], const int16_t b[768], const int16_t b_cache[384])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: write-only
|
||||
c_parameter: int16_t r[256]
|
||||
description: Output polynomial
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 1536
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t a[768]
|
||||
description: Input polynomial vector a
|
||||
x2:
|
||||
type: buffer
|
||||
size_bytes: 1536
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t b[768]
|
||||
description: Input polynomial vector b
|
||||
x3:
|
||||
type: buffer
|
||||
size_bytes: 768
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t b_cache[384]
|
||||
description: Cached values for b
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: saving callee-saved Neon registers
|
||||
*/
|
||||
|
||||
/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
mov w14, #0xd01 // =3329
|
||||
dup v0.8h, w14
|
||||
mov w14, #0xcff // =3327
|
||||
dup v2.8h, w14
|
||||
add x4, x1, #0x200
|
||||
add x5, x2, #0x200
|
||||
add x6, x3, #0x100
|
||||
add x7, x1, #0x400
|
||||
add x8, x2, #0x400
|
||||
add x9, x3, #0x200
|
||||
mov x13, #0x10 // =16
|
||||
ldr q6, [x7], #0x20
|
||||
ldr q19, [x2, #0x10]
|
||||
ldr q23, [x1], #0x20
|
||||
ldur q14, [x1, #-0x10]
|
||||
ldr q17, [x2], #0x20
|
||||
ldr q11, [x4, #0x10]
|
||||
ldur q28, [x7, #-0x10]
|
||||
ld1 { v30.8h }, [x3], #16
|
||||
ldr q26, [x4], #0x20
|
||||
ldr q16, [x8, #0x10]
|
||||
uzp1 v8.8h, v23.8h, v14.8h
|
||||
ldr q22, [x5, #0x10]
|
||||
ldr q18, [x5], #0x20
|
||||
uzp1 v20.8h, v17.8h, v19.8h
|
||||
uzp2 v24.8h, v23.8h, v14.8h
|
||||
ldr q31, [x8], #0x20
|
||||
smull2 v4.4s, v8.8h, v20.8h
|
||||
uzp1 v25.8h, v26.8h, v11.8h
|
||||
smull v13.4s, v8.4h, v20.4h
|
||||
ld1 { v23.8h }, [x6], #16
|
||||
uzp1 v1.8h, v18.8h, v22.8h
|
||||
smlal v13.4s, v24.4h, v30.4h
|
||||
smlal2 v4.4s, v24.8h, v30.8h
|
||||
uzp2 v5.8h, v26.8h, v11.8h
|
||||
smlal2 v4.4s, v25.8h, v1.8h
|
||||
uzp1 v29.8h, v6.8h, v28.8h
|
||||
smlal2 v4.4s, v5.8h, v23.8h
|
||||
ld1 { v7.8h }, [x9], #16
|
||||
smlal v13.4s, v25.4h, v1.4h
|
||||
uzp2 v17.8h, v17.8h, v19.8h
|
||||
uzp1 v27.8h, v31.8h, v16.8h
|
||||
smlal v13.4s, v5.4h, v23.4h
|
||||
uzp2 v22.8h, v18.8h, v22.8h
|
||||
smull v18.4s, v8.4h, v17.4h
|
||||
uzp2 v28.8h, v6.8h, v28.8h
|
||||
smlal v13.4s, v29.4h, v27.4h
|
||||
smlal2 v4.4s, v29.8h, v27.8h
|
||||
uzp2 v26.8h, v31.8h, v16.8h
|
||||
smlal2 v4.4s, v28.8h, v7.8h
|
||||
ldr q3, [x7, #0x10]
|
||||
smlal v13.4s, v28.4h, v7.4h
|
||||
ldr q7, [x1], #0x20
|
||||
smlal v18.4s, v24.4h, v20.4h
|
||||
ldr q15, [x2], #0x20
|
||||
smlal v18.4s, v25.4h, v22.4h
|
||||
smull2 v8.4s, v8.8h, v17.8h
|
||||
ldur q17, [x1, #-0x10]
|
||||
uzp1 v23.8h, v13.8h, v4.8h
|
||||
smlal v18.4s, v5.4h, v1.4h
|
||||
smlal2 v8.4s, v24.8h, v20.8h
|
||||
ld1 { v16.8h }, [x3], #16
|
||||
mul v23.8h, v23.8h, v2.8h
|
||||
ldr q19, [x5, #0x10]
|
||||
ldr q14, [x4, #0x10]
|
||||
ldr q11, [x4], #0x20
|
||||
ldur q20, [x2, #-0x10]
|
||||
smlal2 v8.4s, v25.8h, v22.8h
|
||||
smlal2 v8.4s, v5.8h, v1.8h
|
||||
ldr q22, [x5], #0x20
|
||||
uzp1 v1.8h, v7.8h, v17.8h
|
||||
smlal v18.4s, v29.4h, v26.4h
|
||||
smlal v13.4s, v23.4h, v0.4h
|
||||
uzp2 v31.8h, v11.8h, v14.8h
|
||||
uzp1 v21.8h, v15.8h, v20.8h
|
||||
smlal2 v4.4s, v23.8h, v0.8h
|
||||
ld1 { v9.8h }, [x6], #16
|
||||
smlal v18.4s, v28.4h, v27.4h
|
||||
smlal2 v8.4s, v29.8h, v26.8h
|
||||
ldr q25, [x7], #0x20
|
||||
smull v26.4s, v1.4h, v21.4h
|
||||
uzp1 v24.8h, v22.8h, v19.8h
|
||||
smlal2 v8.4s, v28.8h, v27.8h
|
||||
uzp2 v28.8h, v7.8h, v17.8h
|
||||
uzp1 v29.8h, v11.8h, v14.8h
|
||||
smull2 v23.4s, v1.8h, v21.8h
|
||||
ldr q27, [x8], #0x20
|
||||
smlal2 v23.4s, v28.8h, v16.8h
|
||||
ldur q11, [x8, #-0x10]
|
||||
smlal2 v23.4s, v29.8h, v24.8h
|
||||
uzp2 v7.8h, v13.8h, v4.8h
|
||||
uzp2 v19.8h, v22.8h, v19.8h
|
||||
ld1 { v4.8h }, [x9], #16
|
||||
smlal2 v23.4s, v31.8h, v9.8h
|
||||
uzp1 v13.8h, v25.8h, v3.8h
|
||||
uzp1 v14.8h, v18.8h, v8.8h
|
||||
smlal v26.4s, v28.4h, v16.4h
|
||||
uzp2 v17.8h, v27.8h, v11.8h
|
||||
uzp2 v20.8h, v15.8h, v20.8h
|
||||
mul v14.8h, v14.8h, v2.8h
|
||||
sub x13, x13, #0x2
|
||||
|
||||
Lpolyvec_basemul_acc_montgomery_cached_k3_loop_start:
|
||||
uzp1 v6.8h, v27.8h, v11.8h
|
||||
smlal v26.4s, v29.4h, v24.4h
|
||||
uzp2 v16.8h, v25.8h, v3.8h
|
||||
smlal v26.4s, v31.4h, v9.4h
|
||||
ldr q3, [x7, #0x10]
|
||||
smlal v26.4s, v13.4h, v6.4h
|
||||
smlal2 v8.4s, v14.8h, v0.8h
|
||||
ldr q27, [x8], #0x20
|
||||
smlal v18.4s, v14.4h, v0.4h
|
||||
ldr q25, [x7], #0x20
|
||||
smlal2 v23.4s, v13.8h, v6.8h
|
||||
ldr q11, [x1], #0x20
|
||||
smlal2 v23.4s, v16.8h, v4.8h
|
||||
smlal v26.4s, v16.4h, v4.4h
|
||||
ldur q22, [x1, #-0x10]
|
||||
uzp2 v30.8h, v18.8h, v8.8h
|
||||
smull v18.4s, v1.4h, v20.4h
|
||||
smlal v18.4s, v28.4h, v21.4h
|
||||
ldr q14, [x2], #0x20
|
||||
smlal v18.4s, v29.4h, v19.4h
|
||||
zip1 v5.8h, v7.8h, v30.8h
|
||||
uzp1 v4.8h, v26.8h, v23.8h
|
||||
smull2 v8.4s, v1.8h, v20.8h
|
||||
zip2 v10.8h, v7.8h, v30.8h
|
||||
smlal v18.4s, v31.4h, v24.4h
|
||||
mul v12.8h, v4.8h, v2.8h
|
||||
ldr q4, [x5, #0x10]
|
||||
ldr q20, [x4, #0x10]
|
||||
ldr q1, [x4], #0x20
|
||||
ldur q30, [x2, #-0x10]
|
||||
smlal2 v8.4s, v28.8h, v21.8h
|
||||
smlal2 v8.4s, v29.8h, v19.8h
|
||||
ldr q19, [x5], #0x20
|
||||
smlal2 v8.4s, v31.8h, v24.8h
|
||||
ld1 { v15.8h }, [x3], #16
|
||||
uzp2 v31.8h, v1.8h, v20.8h
|
||||
smlal v26.4s, v12.4h, v0.4h
|
||||
smlal2 v23.4s, v12.8h, v0.8h
|
||||
uzp1 v21.8h, v14.8h, v30.8h
|
||||
uzp1 v29.8h, v1.8h, v20.8h
|
||||
uzp1 v1.8h, v11.8h, v22.8h
|
||||
smlal2 v8.4s, v13.8h, v17.8h
|
||||
ld1 { v9.8h }, [x6], #16
|
||||
smlal v18.4s, v13.4h, v17.4h
|
||||
uzp1 v24.8h, v19.8h, v4.8h
|
||||
uzp2 v7.8h, v26.8h, v23.8h
|
||||
smull v26.4s, v1.4h, v21.4h
|
||||
smlal v18.4s, v16.4h, v6.4h
|
||||
uzp2 v19.8h, v19.8h, v4.8h
|
||||
smlal2 v8.4s, v16.8h, v6.8h
|
||||
uzp2 v28.8h, v11.8h, v22.8h
|
||||
smull2 v23.4s, v1.8h, v21.8h
|
||||
uzp1 v13.8h, v25.8h, v3.8h
|
||||
smlal2 v23.4s, v28.8h, v15.8h
|
||||
ldur q11, [x8, #-0x10]
|
||||
smlal2 v23.4s, v29.8h, v24.8h
|
||||
ld1 { v4.8h }, [x9], #16
|
||||
smlal2 v23.4s, v31.8h, v9.8h
|
||||
uzp1 v12.8h, v18.8h, v8.8h
|
||||
uzp2 v20.8h, v14.8h, v30.8h
|
||||
smlal v26.4s, v28.4h, v15.4h
|
||||
str q5, [x0], #0x20
|
||||
mul v14.8h, v12.8h, v2.8h
|
||||
stur q10, [x0, #-0x10]
|
||||
uzp2 v17.8h, v27.8h, v11.8h
|
||||
subs x13, x13, #0x1
|
||||
cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k3_loop_start
|
||||
uzp2 v3.8h, v25.8h, v3.8h
|
||||
smull2 v16.4s, v1.8h, v20.8h
|
||||
smull v25.4s, v1.4h, v20.4h
|
||||
uzp1 v22.8h, v27.8h, v11.8h
|
||||
smlal2 v16.4s, v28.8h, v21.8h
|
||||
smlal v25.4s, v28.4h, v21.4h
|
||||
smlal2 v16.4s, v29.8h, v19.8h
|
||||
smlal v25.4s, v29.4h, v19.4h
|
||||
smlal2 v16.4s, v31.8h, v24.8h
|
||||
smlal v25.4s, v31.4h, v24.4h
|
||||
smlal v25.4s, v13.4h, v17.4h
|
||||
smlal2 v16.4s, v13.8h, v17.8h
|
||||
smlal2 v16.4s, v3.8h, v22.8h
|
||||
smlal v25.4s, v3.4h, v22.4h
|
||||
smlal2 v23.4s, v13.8h, v22.8h
|
||||
smlal v26.4s, v29.4h, v24.4h
|
||||
smlal v26.4s, v31.4h, v9.4h
|
||||
smlal v26.4s, v13.4h, v22.4h
|
||||
uzp1 v10.8h, v25.8h, v16.8h
|
||||
smlal2 v23.4s, v3.8h, v4.8h
|
||||
smlal v26.4s, v3.4h, v4.4h
|
||||
mul v13.8h, v10.8h, v2.8h
|
||||
smlal v18.4s, v14.4h, v0.4h
|
||||
smlal2 v8.4s, v14.8h, v0.8h
|
||||
uzp1 v3.8h, v26.8h, v23.8h
|
||||
mul v24.8h, v3.8h, v2.8h
|
||||
uzp2 v17.8h, v18.8h, v8.8h
|
||||
smlal v25.4s, v13.4h, v0.4h
|
||||
smlal2 v16.4s, v13.8h, v0.8h
|
||||
zip1 v21.8h, v7.8h, v17.8h
|
||||
zip2 v20.8h, v7.8h, v17.8h
|
||||
smlal2 v23.4s, v24.8h, v0.8h
|
||||
str q21, [x0], #0x20
|
||||
smlal v26.4s, v24.4h, v0.4h
|
||||
uzp2 v13.8h, v25.8h, v16.8h
|
||||
stur q20, [x0, #-0x10]
|
||||
uzp2 v23.8h, v26.8h, v23.8h
|
||||
zip1 v18.8h, v23.8h, v13.8h
|
||||
zip2 v13.8h, v23.8h, v13.8h
|
||||
str q18, [x0], #0x20
|
||||
stur q13, [x0, #-0x10]
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
(MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,370 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [NeonNTT]
|
||||
* Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
|
||||
* Becker, Hwang, Kannwischer, Yang, Yang
|
||||
* https://eprint.iacr.org/2021/986
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm
|
||||
Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=4
|
||||
Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(int16_t r[256], const int16_t a[1024], const int16_t b[1024], const int16_t b_cache[512])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: write-only
|
||||
c_parameter: int16_t r[256]
|
||||
description: Output polynomial
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: 2048
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t a[1024]
|
||||
description: Input polynomial vector a
|
||||
x2:
|
||||
type: buffer
|
||||
size_bytes: 2048
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t b[1024]
|
||||
description: Input polynomial vector b
|
||||
x3:
|
||||
type: buffer
|
||||
size_bytes: 1024
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t b_cache[512]
|
||||
description: Cached values for b
|
||||
Stack:
|
||||
bytes: 64
|
||||
description: saving callee-saved Neon registers
|
||||
*/
|
||||
|
||||
/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset 0x40
|
||||
stp d8, d9, [sp]
|
||||
.cfi_rel_offset d8, 0x0
|
||||
.cfi_rel_offset d9, 0x8
|
||||
stp d10, d11, [sp, #0x10]
|
||||
.cfi_rel_offset d10, 0x10
|
||||
.cfi_rel_offset d11, 0x18
|
||||
stp d12, d13, [sp, #0x20]
|
||||
.cfi_rel_offset d12, 0x20
|
||||
.cfi_rel_offset d13, 0x28
|
||||
stp d14, d15, [sp, #0x30]
|
||||
.cfi_rel_offset d14, 0x30
|
||||
.cfi_rel_offset d15, 0x38
|
||||
mov w14, #0xd01 // =3329
|
||||
dup v0.8h, w14
|
||||
mov w14, #0xcff // =3327
|
||||
dup v2.8h, w14
|
||||
add x4, x1, #0x200
|
||||
add x5, x2, #0x200
|
||||
add x6, x3, #0x100
|
||||
add x7, x1, #0x400
|
||||
add x8, x2, #0x400
|
||||
add x9, x3, #0x200
|
||||
add x10, x1, #0x600
|
||||
add x11, x2, #0x600
|
||||
add x12, x3, #0x300
|
||||
mov x13, #0x10 // =16
|
||||
ldr q28, [x1], #0x20
|
||||
ldur q5, [x1, #-0x10]
|
||||
ldr q31, [x2], #0x20
|
||||
ldur q27, [x2, #-0x10]
|
||||
ldr q7, [x5], #0x20
|
||||
ldr q10, [x4], #0x20
|
||||
ldur q18, [x5, #-0x10]
|
||||
ldur q9, [x4, #-0x10]
|
||||
uzp1 v11.8h, v28.8h, v5.8h
|
||||
uzp2 v19.8h, v28.8h, v5.8h
|
||||
uzp2 v4.8h, v31.8h, v27.8h
|
||||
uzp1 v1.8h, v31.8h, v27.8h
|
||||
ldr q29, [x7], #0x20
|
||||
ldr q28, [x8, #0x10]
|
||||
uzp1 v24.8h, v10.8h, v9.8h
|
||||
uzp1 v17.8h, v7.8h, v18.8h
|
||||
uzp2 v7.8h, v7.8h, v18.8h
|
||||
ldr q21, [x8], #0x20
|
||||
uzp2 v27.8h, v10.8h, v9.8h
|
||||
ldur q6, [x7, #-0x10]
|
||||
smull v18.4s, v11.4h, v4.4h
|
||||
ld1 { v9.8h }, [x3], #16
|
||||
smull2 v8.4s, v11.8h, v4.8h
|
||||
ldr q16, [x11], #0x20
|
||||
smlal2 v8.4s, v19.8h, v1.8h
|
||||
ldur q14, [x11, #-0x10]
|
||||
smlal v18.4s, v19.4h, v1.4h
|
||||
uzp1 v10.8h, v21.8h, v28.8h
|
||||
smlal v18.4s, v24.4h, v7.4h
|
||||
ldr q4, [x10], #0x20
|
||||
smlal2 v8.4s, v24.8h, v7.8h
|
||||
ld1 { v12.8h }, [x6], #16
|
||||
smull2 v23.4s, v11.8h, v1.8h
|
||||
uzp2 v13.8h, v29.8h, v6.8h
|
||||
smull v26.4s, v11.4h, v1.4h
|
||||
uzp1 v29.8h, v29.8h, v6.8h
|
||||
smlal v26.4s, v19.4h, v9.4h
|
||||
ldur q15, [x10, #-0x10]
|
||||
smlal2 v23.4s, v19.8h, v9.8h
|
||||
uzp2 v9.8h, v21.8h, v28.8h
|
||||
smlal v18.4s, v27.4h, v17.4h
|
||||
uzp2 v6.8h, v16.8h, v14.8h
|
||||
uzp1 v21.8h, v16.8h, v14.8h
|
||||
smlal2 v8.4s, v27.8h, v17.8h
|
||||
smlal2 v8.4s, v29.8h, v9.8h
|
||||
uzp1 v30.8h, v4.8h, v15.8h
|
||||
uzp2 v16.8h, v4.8h, v15.8h
|
||||
smlal v18.4s, v29.4h, v9.4h
|
||||
smlal2 v8.4s, v13.8h, v10.8h
|
||||
ld1 { v15.8h }, [x9], #16
|
||||
smlal v18.4s, v13.4h, v10.4h
|
||||
ldr q11, [x4], #0x20
|
||||
smlal v18.4s, v30.4h, v6.4h
|
||||
ldr q7, [x2], #0x20
|
||||
smlal2 v8.4s, v30.8h, v6.8h
|
||||
ld1 { v9.8h }, [x12], #16
|
||||
smlal2 v23.4s, v24.8h, v17.8h
|
||||
ldur q4, [x2, #-0x10]
|
||||
smlal v26.4s, v24.4h, v17.4h
|
||||
ldur q25, [x4, #-0x10]
|
||||
smlal2 v8.4s, v16.8h, v21.8h
|
||||
ldr q5, [x5], #0x20
|
||||
smlal v18.4s, v16.4h, v21.4h
|
||||
ldur q22, [x5, #-0x10]
|
||||
smlal v26.4s, v27.4h, v12.4h
|
||||
ldr q19, [x1, #0x10]
|
||||
smlal v26.4s, v29.4h, v10.4h
|
||||
ld1 { v20.8h }, [x3], #16
|
||||
smlal v26.4s, v13.4h, v15.4h
|
||||
uzp1 v24.8h, v7.8h, v4.8h
|
||||
smlal2 v23.4s, v27.8h, v12.8h
|
||||
uzp1 v28.8h, v18.8h, v8.8h
|
||||
smlal v26.4s, v30.4h, v21.4h
|
||||
uzp2 v27.8h, v11.8h, v25.8h
|
||||
smlal2 v23.4s, v29.8h, v10.8h
|
||||
uzp2 v31.8h, v7.8h, v4.8h
|
||||
smlal2 v23.4s, v13.8h, v15.8h
|
||||
uzp1 v14.8h, v5.8h, v22.8h
|
||||
uzp1 v17.8h, v11.8h, v25.8h
|
||||
smlal v26.4s, v16.4h, v9.4h
|
||||
mul v29.8h, v28.8h, v2.8h
|
||||
sub x13, x13, #0x2
|
||||
|
||||
Lpolyvec_basemul_acc_montgomery_cached_k4_loop_start:
|
||||
smlal2 v23.4s, v30.8h, v21.8h
|
||||
ldr q11, [x1], #0x20
|
||||
uzp2 v15.8h, v5.8h, v22.8h
|
||||
smlal v18.4s, v29.4h, v0.4h
|
||||
ldr q12, [x7], #0x20
|
||||
smlal2 v8.4s, v29.8h, v0.8h
|
||||
ldur q3, [x7, #-0x10]
|
||||
ldr q21, [x8], #0x20
|
||||
uzp1 v29.8h, v11.8h, v19.8h
|
||||
ldur q13, [x8, #-0x10]
|
||||
uzp2 v5.8h, v11.8h, v19.8h
|
||||
smlal2 v23.4s, v16.8h, v9.8h
|
||||
uzp2 v28.8h, v18.8h, v8.8h
|
||||
smull2 v8.4s, v29.8h, v31.8h
|
||||
smlal2 v8.4s, v5.8h, v24.8h
|
||||
uzp1 v7.8h, v12.8h, v3.8h
|
||||
smlal2 v8.4s, v17.8h, v15.8h
|
||||
uzp2 v11.8h, v21.8h, v13.8h
|
||||
uzp1 v4.8h, v26.8h, v23.8h
|
||||
smlal2 v8.4s, v27.8h, v14.8h
|
||||
smlal2 v8.4s, v7.8h, v11.8h
|
||||
mul v6.8h, v4.8h, v2.8h
|
||||
ldr q19, [x11], #0x20
|
||||
uzp2 v25.8h, v12.8h, v3.8h
|
||||
ldr q12, [x10], #0x20
|
||||
smull v18.4s, v29.4h, v31.4h
|
||||
ldur q3, [x10, #-0x10]
|
||||
smlal v18.4s, v5.4h, v24.4h
|
||||
uzp1 v4.8h, v21.8h, v13.8h
|
||||
smlal v18.4s, v17.4h, v15.4h
|
||||
ldur q13, [x11, #-0x10]
|
||||
ld1 { v1.8h }, [x6], #16
|
||||
smlal v26.4s, v6.4h, v0.4h
|
||||
smlal2 v23.4s, v6.8h, v0.8h
|
||||
ld1 { v10.8h }, [x9], #16
|
||||
smlal v18.4s, v27.4h, v14.4h
|
||||
uzp1 v30.8h, v12.8h, v3.8h
|
||||
smlal2 v8.4s, v25.8h, v4.8h
|
||||
uzp2 v31.8h, v19.8h, v13.8h
|
||||
smlal v18.4s, v7.4h, v11.4h
|
||||
ld1 { v9.8h }, [x12], #16
|
||||
smlal v18.4s, v25.4h, v4.4h
|
||||
uzp1 v21.8h, v19.8h, v13.8h
|
||||
uzp2 v16.8h, v12.8h, v3.8h
|
||||
smlal v18.4s, v30.4h, v31.4h
|
||||
smlal2 v8.4s, v30.8h, v31.8h
|
||||
uzp2 v31.8h, v26.8h, v23.8h
|
||||
smlal2 v8.4s, v16.8h, v21.8h
|
||||
smlal v18.4s, v16.4h, v21.4h
|
||||
zip1 v15.8h, v31.8h, v28.8h
|
||||
ldr q19, [x1, #0x10]
|
||||
smull2 v23.4s, v29.8h, v24.8h
|
||||
smull v26.4s, v29.4h, v24.4h
|
||||
ldr q3, [x2, #0x10]
|
||||
smlal v26.4s, v5.4h, v20.4h
|
||||
ldr q11, [x2], #0x20
|
||||
uzp1 v6.8h, v18.8h, v8.8h
|
||||
smlal v26.4s, v17.4h, v14.4h
|
||||
smlal v26.4s, v27.4h, v1.4h
|
||||
zip2 v13.8h, v31.8h, v28.8h
|
||||
smlal v26.4s, v7.4h, v4.4h
|
||||
str q15, [x0], #0x20
|
||||
smlal v26.4s, v25.4h, v10.4h
|
||||
stur q13, [x0, #-0x10]
|
||||
mul v29.8h, v6.8h, v2.8h
|
||||
uzp1 v24.8h, v11.8h, v3.8h
|
||||
uzp2 v31.8h, v11.8h, v3.8h
|
||||
ldr q11, [x4], #0x20
|
||||
smlal2 v23.4s, v5.8h, v20.8h
|
||||
ldur q28, [x4, #-0x10]
|
||||
smlal2 v23.4s, v17.8h, v14.8h
|
||||
ldr q5, [x5], #0x20
|
||||
smlal2 v23.4s, v27.8h, v1.8h
|
||||
ldur q22, [x5, #-0x10]
|
||||
smlal v26.4s, v30.4h, v21.4h
|
||||
ld1 { v20.8h }, [x3], #16
|
||||
smlal v26.4s, v16.4h, v9.4h
|
||||
uzp1 v17.8h, v11.8h, v28.8h
|
||||
smlal2 v23.4s, v7.8h, v4.8h
|
||||
uzp2 v27.8h, v11.8h, v28.8h
|
||||
smlal2 v23.4s, v25.8h, v10.8h
|
||||
uzp1 v14.8h, v5.8h, v22.8h
|
||||
subs x13, x13, #0x1
|
||||
cbnz x13, Lpolyvec_basemul_acc_montgomery_cached_k4_loop_start
|
||||
smlal v18.4s, v29.4h, v0.4h
|
||||
ldr q11, [x1], #0x20
|
||||
uzp2 v28.8h, v5.8h, v22.8h
|
||||
smlal2 v23.4s, v30.8h, v21.8h
|
||||
smlal2 v8.4s, v29.8h, v0.8h
|
||||
ldr q15, [x8, #0x10]
|
||||
smlal2 v23.4s, v16.8h, v9.8h
|
||||
ldr q21, [x8], #0x20
|
||||
uzp1 v22.8h, v11.8h, v19.8h
|
||||
uzp2 v12.8h, v11.8h, v19.8h
|
||||
ldr q1, [x7, #0x10]
|
||||
ld1 { v6.8h }, [x6], #16
|
||||
uzp2 v3.8h, v18.8h, v8.8h
|
||||
smull v9.4s, v22.4h, v31.4h
|
||||
smull2 v18.4s, v22.8h, v31.8h
|
||||
ldr q16, [x7], #0x20
|
||||
smull v19.4s, v22.4h, v24.4h
|
||||
uzp1 v30.8h, v21.8h, v15.8h
|
||||
uzp2 v25.8h, v21.8h, v15.8h
|
||||
smull2 v8.4s, v22.8h, v24.8h
|
||||
smlal v19.4s, v12.4h, v20.4h
|
||||
ldr q13, [x10, #0x10]
|
||||
smlal2 v8.4s, v12.8h, v20.8h
|
||||
uzp1 v29.8h, v16.8h, v1.8h
|
||||
smlal2 v18.4s, v12.8h, v24.8h
|
||||
ldr q5, [x10], #0x20
|
||||
smlal v9.4s, v12.4h, v24.4h
|
||||
ldr q4, [x11], #0x20
|
||||
smlal v9.4s, v17.4h, v28.4h
|
||||
ldur q22, [x11, #-0x10]
|
||||
smlal2 v18.4s, v17.8h, v28.8h
|
||||
uzp2 v16.8h, v16.8h, v1.8h
|
||||
smlal v19.4s, v17.4h, v14.4h
|
||||
ld1 { v28.8h }, [x9], #16
|
||||
smlal2 v8.4s, v17.8h, v14.8h
|
||||
uzp1 v7.8h, v5.8h, v13.8h
|
||||
smlal v9.4s, v27.4h, v14.4h
|
||||
uzp1 v17.8h, v4.8h, v22.8h
|
||||
smlal2 v18.4s, v27.8h, v14.8h
|
||||
uzp2 v12.8h, v5.8h, v13.8h
|
||||
uzp2 v21.8h, v4.8h, v22.8h
|
||||
smlal v19.4s, v27.4h, v6.4h
|
||||
smlal2 v8.4s, v27.8h, v6.8h
|
||||
ld1 { v15.8h }, [x12], #16
|
||||
smlal v19.4s, v29.4h, v30.4h
|
||||
uzp1 v20.8h, v26.8h, v23.8h
|
||||
smlal v9.4s, v29.4h, v25.4h
|
||||
smlal2 v18.4s, v29.8h, v25.8h
|
||||
smlal2 v8.4s, v29.8h, v30.8h
|
||||
smlal v19.4s, v16.4h, v28.4h
|
||||
smlal2 v8.4s, v16.8h, v28.8h
|
||||
smlal2 v18.4s, v16.8h, v30.8h
|
||||
smlal v9.4s, v16.4h, v30.4h
|
||||
smlal v9.4s, v7.4h, v21.4h
|
||||
smlal2 v18.4s, v7.8h, v21.8h
|
||||
smlal2 v8.4s, v7.8h, v17.8h
|
||||
smlal v19.4s, v7.4h, v17.4h
|
||||
smlal v19.4s, v12.4h, v15.4h
|
||||
smlal2 v8.4s, v12.8h, v15.8h
|
||||
smlal2 v18.4s, v12.8h, v17.8h
|
||||
smlal v9.4s, v12.4h, v17.4h
|
||||
mul v6.8h, v20.8h, v2.8h
|
||||
uzp1 v4.8h, v19.8h, v8.8h
|
||||
mul v17.8h, v4.8h, v2.8h
|
||||
uzp1 v12.8h, v9.8h, v18.8h
|
||||
smlal v26.4s, v6.4h, v0.4h
|
||||
mul v21.8h, v12.8h, v2.8h
|
||||
smlal2 v23.4s, v6.8h, v0.8h
|
||||
smlal2 v8.4s, v17.8h, v0.8h
|
||||
smlal v19.4s, v17.4h, v0.4h
|
||||
smlal2 v18.4s, v21.8h, v0.8h
|
||||
uzp2 v23.8h, v26.8h, v23.8h
|
||||
smlal v9.4s, v21.4h, v0.4h
|
||||
zip2 v12.8h, v23.8h, v3.8h
|
||||
zip1 v22.8h, v23.8h, v3.8h
|
||||
uzp2 v14.8h, v19.8h, v8.8h
|
||||
uzp2 v18.8h, v9.8h, v18.8h
|
||||
str q12, [x0, #0x10]
|
||||
str q22, [x0], #0x20
|
||||
zip2 v24.8h, v14.8h, v18.8h
|
||||
zip1 v21.8h, v14.8h, v18.8h
|
||||
str q24, [x0, #0x10]
|
||||
str q21, [x0], #0x20
|
||||
ldp d8, d9, [sp]
|
||||
.cfi_restore d8
|
||||
.cfi_restore d9
|
||||
ldp d10, d11, [sp, #0x10]
|
||||
.cfi_restore d10
|
||||
.cfi_restore d11
|
||||
ldp d12, d13, [sp, #0x20]
|
||||
.cfi_restore d12
|
||||
.cfi_restore d13
|
||||
ldp d14, d15, [sp, #0x30]
|
||||
.cfi_restore d14
|
||||
.cfi_restore d15
|
||||
add sp, sp, #0x40
|
||||
.cfi_adjust_cfa_offset -0x40
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
(MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*yaml
|
||||
Name: rej_uniform_aarch64_asm
|
||||
Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q
|
||||
Signature: uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096])
|
||||
ABI:
|
||||
Architecture: aarch64
|
||||
CallingConvention: AAPCS64
|
||||
x0:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: write-only
|
||||
c_parameter: int16_t r[256]
|
||||
description: Output buffer
|
||||
x1:
|
||||
type: buffer
|
||||
size_bytes: x2
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t *buf
|
||||
description: Input buffer
|
||||
x2:
|
||||
type: scalar
|
||||
c_parameter: unsigned buflen
|
||||
description: Length of input buffer (must be multiple of 24)
|
||||
test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE
|
||||
x3:
|
||||
type: buffer
|
||||
size_bytes: 4096
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t table[4096]
|
||||
description: Lookup table
|
||||
Stack:
|
||||
bytes: 576
|
||||
description: register preservation and temporary storage
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/aarch64_opt/src/rej_uniform_aarch64_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(rej_uniform_aarch64_asm)
|
||||
MLK_ASM_FN_SYMBOL(rej_uniform_aarch64_asm)
|
||||
|
||||
.cfi_startproc
|
||||
sub sp, sp, #0x240
|
||||
.cfi_adjust_cfa_offset 0x240
|
||||
mov x7, #0x1 // =1
|
||||
movk x7, #0x2, lsl #16
|
||||
movk x7, #0x4, lsl #32
|
||||
movk x7, #0x8, lsl #48
|
||||
mov v31.d[0], x7
|
||||
mov x7, #0x10 // =16
|
||||
movk x7, #0x20, lsl #16
|
||||
movk x7, #0x40, lsl #32
|
||||
movk x7, #0x80, lsl #48
|
||||
mov v31.d[1], x7
|
||||
mov w11, #0xd01 // =3329
|
||||
dup v30.8h, w11
|
||||
mov x8, sp
|
||||
mov x7, x8
|
||||
mov x11, #0x0 // =0
|
||||
eor v16.16b, v16.16b, v16.16b
|
||||
|
||||
Lrej_uniform_initial_zero:
|
||||
str q16, [x7], #0x40
|
||||
stur q16, [x7, #-0x30]
|
||||
stur q16, [x7, #-0x20]
|
||||
stur q16, [x7, #-0x10]
|
||||
add x11, x11, #0x20
|
||||
cmp x11, #0x100
|
||||
b.lt Lrej_uniform_initial_zero
|
||||
mov x7, x8
|
||||
mov x9, #0x0 // =0
|
||||
mov x4, #0x100 // =256
|
||||
cmp x2, #0x30
|
||||
b.lo Lrej_uniform_loop48_end
|
||||
|
||||
Lrej_uniform_loop48:
|
||||
cmp x9, x4
|
||||
b.hs Lrej_uniform_memory_copy
|
||||
sub x2, x2, #0x30
|
||||
ld3 { v0.16b, v1.16b, v2.16b }, [x1], #48
|
||||
zip1 v4.16b, v0.16b, v1.16b
|
||||
zip2 v5.16b, v0.16b, v1.16b
|
||||
zip1 v6.16b, v1.16b, v2.16b
|
||||
zip2 v7.16b, v1.16b, v2.16b
|
||||
bic v4.8h, #0xf0, lsl #8
|
||||
bic v5.8h, #0xf0, lsl #8
|
||||
ushr v6.8h, v6.8h, #0x4
|
||||
ushr v7.8h, v7.8h, #0x4
|
||||
zip1 v16.8h, v4.8h, v6.8h
|
||||
zip2 v17.8h, v4.8h, v6.8h
|
||||
zip1 v18.8h, v5.8h, v7.8h
|
||||
zip2 v19.8h, v5.8h, v7.8h
|
||||
cmhi v4.8h, v30.8h, v16.8h
|
||||
cmhi v5.8h, v30.8h, v17.8h
|
||||
cmhi v6.8h, v30.8h, v18.8h
|
||||
cmhi v7.8h, v30.8h, v19.8h
|
||||
and v4.16b, v4.16b, v31.16b
|
||||
and v5.16b, v5.16b, v31.16b
|
||||
and v6.16b, v6.16b, v31.16b
|
||||
and v7.16b, v7.16b, v31.16b
|
||||
uaddlv s20, v4.8h
|
||||
uaddlv s21, v5.8h
|
||||
uaddlv s22, v6.8h
|
||||
uaddlv s23, v7.8h
|
||||
fmov w12, s20
|
||||
fmov w13, s21
|
||||
fmov w14, s22
|
||||
fmov w15, s23
|
||||
ldr q24, [x3, x12, lsl #4]
|
||||
ldr q25, [x3, x13, lsl #4]
|
||||
ldr q26, [x3, x14, lsl #4]
|
||||
ldr q27, [x3, x15, lsl #4]
|
||||
cnt v4.16b, v4.16b
|
||||
cnt v5.16b, v5.16b
|
||||
cnt v6.16b, v6.16b
|
||||
cnt v7.16b, v7.16b
|
||||
uaddlv s20, v4.8h
|
||||
uaddlv s21, v5.8h
|
||||
uaddlv s22, v6.8h
|
||||
uaddlv s23, v7.8h
|
||||
fmov w12, s20
|
||||
fmov w13, s21
|
||||
fmov w14, s22
|
||||
fmov w15, s23
|
||||
tbl v16.16b, { v16.16b }, v24.16b
|
||||
tbl v17.16b, { v17.16b }, v25.16b
|
||||
tbl v18.16b, { v18.16b }, v26.16b
|
||||
tbl v19.16b, { v19.16b }, v27.16b
|
||||
st1 { v16.8h }, [x7]
|
||||
add x7, x7, x12, lsl #1
|
||||
st1 { v17.8h }, [x7]
|
||||
add x7, x7, x13, lsl #1
|
||||
st1 { v18.8h }, [x7]
|
||||
add x7, x7, x14, lsl #1
|
||||
st1 { v19.8h }, [x7]
|
||||
add x7, x7, x15, lsl #1
|
||||
add x12, x12, x13
|
||||
add x14, x14, x15
|
||||
add x9, x9, x12
|
||||
add x9, x9, x14
|
||||
cmp x2, #0x30
|
||||
b.hs Lrej_uniform_loop48
|
||||
|
||||
Lrej_uniform_loop48_end:
|
||||
cmp x9, x4
|
||||
b.hs Lrej_uniform_memory_copy
|
||||
cmp x2, #0x18
|
||||
b.lo Lrej_uniform_memory_copy
|
||||
ld3 { v0.8b, v1.8b, v2.8b }, [x1], #24
|
||||
zip1 v4.16b, v0.16b, v1.16b
|
||||
zip1 v5.16b, v1.16b, v2.16b
|
||||
bic v4.8h, #0xf0, lsl #8
|
||||
ushr v5.8h, v5.8h, #0x4
|
||||
zip1 v16.8h, v4.8h, v5.8h
|
||||
zip2 v17.8h, v4.8h, v5.8h
|
||||
cmhi v4.8h, v30.8h, v16.8h
|
||||
cmhi v5.8h, v30.8h, v17.8h
|
||||
and v4.16b, v4.16b, v31.16b
|
||||
and v5.16b, v5.16b, v31.16b
|
||||
uaddlv s20, v4.8h
|
||||
uaddlv s21, v5.8h
|
||||
fmov w12, s20
|
||||
fmov w13, s21
|
||||
ldr q24, [x3, x12, lsl #4]
|
||||
ldr q25, [x3, x13, lsl #4]
|
||||
cnt v4.16b, v4.16b
|
||||
cnt v5.16b, v5.16b
|
||||
uaddlv s20, v4.8h
|
||||
uaddlv s21, v5.8h
|
||||
fmov w12, s20
|
||||
fmov w13, s21
|
||||
tbl v16.16b, { v16.16b }, v24.16b
|
||||
tbl v17.16b, { v17.16b }, v25.16b
|
||||
st1 { v16.8h }, [x7]
|
||||
add x7, x7, x12, lsl #1
|
||||
st1 { v17.8h }, [x7]
|
||||
add x9, x9, x12
|
||||
add x9, x9, x13
|
||||
|
||||
Lrej_uniform_memory_copy:
|
||||
cmp x9, x4
|
||||
csel x9, x9, x4, lo
|
||||
mov x11, #0x0 // =0
|
||||
mov x7, x8
|
||||
|
||||
Lrej_uniform_final_copy:
|
||||
ldr q16, [x7], #0x40
|
||||
ldur q17, [x7, #-0x30]
|
||||
ldur q18, [x7, #-0x20]
|
||||
ldur q19, [x7, #-0x10]
|
||||
str q16, [x0], #0x40
|
||||
stur q17, [x0, #-0x30]
|
||||
stur q18, [x0, #-0x20]
|
||||
stur q19, [x0, #-0x10]
|
||||
add x11, x11, #0x20
|
||||
cmp x11, #0x100
|
||||
b.lt Lrej_uniform_final_copy
|
||||
mov x0, x9
|
||||
|
||||
Lrej_uniform_return:
|
||||
add sp, sp, #0x240
|
||||
.cfi_adjust_cfa_offset -0x240
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(rej_uniform_aarch64_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,543 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_AARCH64) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "arith_native_aarch64.h"
|
||||
|
||||
/*
|
||||
* Lookup table used by rejection sampling of the public matrix.
|
||||
* See autogen for details.
|
||||
*/
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_rej_uniform_table[4096] = {
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 0 */,
|
||||
0, 1, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 1 */,
|
||||
2, 3, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 2 */,
|
||||
0, 1, 2, 3, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 3 */,
|
||||
4, 5, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 4 */,
|
||||
0, 1, 4, 5, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 5 */,
|
||||
2, 3, 4, 5, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 6 */,
|
||||
0, 1, 2, 3, 4, 5, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 7 */,
|
||||
6, 7, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 8 */,
|
||||
0, 1, 6, 7, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 9 */,
|
||||
2, 3, 6, 7, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 10 */,
|
||||
0, 1, 2, 3, 6, 7, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 11 */,
|
||||
4, 5, 6, 7, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 12 */,
|
||||
0, 1, 4, 5, 6, 7, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 13 */,
|
||||
2, 3, 4, 5, 6, 7, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 14 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 15 */,
|
||||
8, 9, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 16 */,
|
||||
0, 1, 8, 9, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 17 */,
|
||||
2, 3, 8, 9, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 18 */,
|
||||
0, 1, 2, 3, 8, 9, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 19 */,
|
||||
4, 5, 8, 9, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 20 */,
|
||||
0, 1, 4, 5, 8, 9, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 21 */,
|
||||
2, 3, 4, 5, 8, 9, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 22 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 23 */,
|
||||
6, 7, 8, 9, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 24 */,
|
||||
0, 1, 6, 7, 8, 9, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 25 */,
|
||||
2, 3, 6, 7, 8, 9, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 26 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 27 */,
|
||||
4, 5, 6, 7, 8, 9, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 28 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 29 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 30 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 255, 255, 255, 255, 255, 255 /* 31 */,
|
||||
10, 11, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 32 */,
|
||||
0, 1, 10, 11, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 33 */,
|
||||
2, 3, 10, 11, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 34 */,
|
||||
0, 1, 2, 3, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 35 */,
|
||||
4, 5, 10, 11, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 36 */,
|
||||
0, 1, 4, 5, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 37 */,
|
||||
2, 3, 4, 5, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 38 */,
|
||||
0, 1, 2, 3, 4, 5, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 39 */,
|
||||
6, 7, 10, 11, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 40 */,
|
||||
0, 1, 6, 7, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 41 */,
|
||||
2, 3, 6, 7, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 42 */,
|
||||
0, 1, 2, 3, 6, 7, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 43 */,
|
||||
4, 5, 6, 7, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 44 */,
|
||||
0, 1, 4, 5, 6, 7, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 45 */,
|
||||
2, 3, 4, 5, 6, 7, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 46 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
10, 11, 255, 255, 255, 255, 255, 255 /* 47 */,
|
||||
8, 9, 10, 11, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 48 */,
|
||||
0, 1, 8, 9, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 49 */,
|
||||
2, 3, 8, 9, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 50 */,
|
||||
0, 1, 2, 3, 8, 9, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 51 */,
|
||||
4, 5, 8, 9, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 52 */,
|
||||
0, 1, 4, 5, 8, 9, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 53 */,
|
||||
2, 3, 4, 5, 8, 9, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 54 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
10, 11, 255, 255, 255, 255, 255, 255 /* 55 */,
|
||||
6, 7, 8, 9, 10, 11, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 56 */,
|
||||
0, 1, 6, 7, 8, 9, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 57 */,
|
||||
2, 3, 6, 7, 8, 9, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 58 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
10, 11, 255, 255, 255, 255, 255, 255 /* 59 */,
|
||||
4, 5, 6, 7, 8, 9, 10, 11,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 60 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 255, 255, 255, 255, 255, 255 /* 61 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 255, 255, 255, 255, 255, 255 /* 62 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 255, 255, 255, 255 /* 63 */,
|
||||
12, 13, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 64 */,
|
||||
0, 1, 12, 13, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 65 */,
|
||||
2, 3, 12, 13, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 66 */,
|
||||
0, 1, 2, 3, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 67 */,
|
||||
4, 5, 12, 13, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 68 */,
|
||||
0, 1, 4, 5, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 69 */,
|
||||
2, 3, 4, 5, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 70 */,
|
||||
0, 1, 2, 3, 4, 5, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 71 */,
|
||||
6, 7, 12, 13, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 72 */,
|
||||
0, 1, 6, 7, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 73 */,
|
||||
2, 3, 6, 7, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 74 */,
|
||||
0, 1, 2, 3, 6, 7, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 75 */,
|
||||
4, 5, 6, 7, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 76 */,
|
||||
0, 1, 4, 5, 6, 7, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 77 */,
|
||||
2, 3, 4, 5, 6, 7, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 78 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 79 */,
|
||||
8, 9, 12, 13, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 80 */,
|
||||
0, 1, 8, 9, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 81 */,
|
||||
2, 3, 8, 9, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 82 */,
|
||||
0, 1, 2, 3, 8, 9, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 83 */,
|
||||
4, 5, 8, 9, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 84 */,
|
||||
0, 1, 4, 5, 8, 9, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 85 */,
|
||||
2, 3, 4, 5, 8, 9, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 86 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 87 */,
|
||||
6, 7, 8, 9, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 88 */,
|
||||
0, 1, 6, 7, 8, 9, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 89 */,
|
||||
2, 3, 6, 7, 8, 9, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 90 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 91 */,
|
||||
4, 5, 6, 7, 8, 9, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 92 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 93 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 94 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 12, 13, 255, 255, 255, 255 /* 95 */,
|
||||
10, 11, 12, 13, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 96 */,
|
||||
0, 1, 10, 11, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 97 */,
|
||||
2, 3, 10, 11, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 98 */,
|
||||
0, 1, 2, 3, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 99 */,
|
||||
4, 5, 10, 11, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 100 */,
|
||||
0, 1, 4, 5, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 101 */,
|
||||
2, 3, 4, 5, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 102 */,
|
||||
0, 1, 2, 3, 4, 5, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 103 */,
|
||||
6, 7, 10, 11, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 104 */,
|
||||
0, 1, 6, 7, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 105 */,
|
||||
2, 3, 6, 7, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 106 */,
|
||||
0, 1, 2, 3, 6, 7, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 107 */,
|
||||
4, 5, 6, 7, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 108 */,
|
||||
0, 1, 4, 5, 6, 7, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 109 */,
|
||||
2, 3, 4, 5, 6, 7, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 110 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
10, 11, 12, 13, 255, 255, 255, 255 /* 111 */,
|
||||
8, 9, 10, 11, 12, 13, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 112 */,
|
||||
0, 1, 8, 9, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 113 */,
|
||||
2, 3, 8, 9, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 114 */,
|
||||
0, 1, 2, 3, 8, 9, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 115 */,
|
||||
4, 5, 8, 9, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 116 */,
|
||||
0, 1, 4, 5, 8, 9, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 117 */,
|
||||
2, 3, 4, 5, 8, 9, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 118 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
10, 11, 12, 13, 255, 255, 255, 255 /* 119 */,
|
||||
6, 7, 8, 9, 10, 11, 12, 13,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 120 */,
|
||||
0, 1, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 121 */,
|
||||
2, 3, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 122 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 255, 255, 255, 255 /* 123 */,
|
||||
4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 255, 255, 255, 255, 255, 255 /* 124 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 255, 255, 255, 255 /* 125 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 255, 255, 255, 255 /* 126 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 255, 255 /* 127 */,
|
||||
14, 15, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 128 */,
|
||||
0, 1, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 129 */,
|
||||
2, 3, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 130 */,
|
||||
0, 1, 2, 3, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 131 */,
|
||||
4, 5, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 132 */,
|
||||
0, 1, 4, 5, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 133 */,
|
||||
2, 3, 4, 5, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 134 */,
|
||||
0, 1, 2, 3, 4, 5, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 135 */,
|
||||
6, 7, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 136 */,
|
||||
0, 1, 6, 7, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 137 */,
|
||||
2, 3, 6, 7, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 138 */,
|
||||
0, 1, 2, 3, 6, 7, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 139 */,
|
||||
4, 5, 6, 7, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 140 */,
|
||||
0, 1, 4, 5, 6, 7, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 141 */,
|
||||
2, 3, 4, 5, 6, 7, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 142 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 143 */,
|
||||
8, 9, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 144 */,
|
||||
0, 1, 8, 9, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 145 */,
|
||||
2, 3, 8, 9, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 146 */,
|
||||
0, 1, 2, 3, 8, 9, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 147 */,
|
||||
4, 5, 8, 9, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 148 */,
|
||||
0, 1, 4, 5, 8, 9, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 149 */,
|
||||
2, 3, 4, 5, 8, 9, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 150 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 151 */,
|
||||
6, 7, 8, 9, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 152 */,
|
||||
0, 1, 6, 7, 8, 9, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 153 */,
|
||||
2, 3, 6, 7, 8, 9, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 154 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 155 */,
|
||||
4, 5, 6, 7, 8, 9, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 156 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 157 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 158 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 14, 15, 255, 255, 255, 255 /* 159 */,
|
||||
10, 11, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 160 */,
|
||||
0, 1, 10, 11, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 161 */,
|
||||
2, 3, 10, 11, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 162 */,
|
||||
0, 1, 2, 3, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 163 */,
|
||||
4, 5, 10, 11, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 164 */,
|
||||
0, 1, 4, 5, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 165 */,
|
||||
2, 3, 4, 5, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 166 */,
|
||||
0, 1, 2, 3, 4, 5, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 167 */,
|
||||
6, 7, 10, 11, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 168 */,
|
||||
0, 1, 6, 7, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 169 */,
|
||||
2, 3, 6, 7, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 170 */,
|
||||
0, 1, 2, 3, 6, 7, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 171 */,
|
||||
4, 5, 6, 7, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 172 */,
|
||||
0, 1, 4, 5, 6, 7, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 173 */,
|
||||
2, 3, 4, 5, 6, 7, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 174 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
10, 11, 14, 15, 255, 255, 255, 255 /* 175 */,
|
||||
8, 9, 10, 11, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 176 */,
|
||||
0, 1, 8, 9, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 177 */,
|
||||
2, 3, 8, 9, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 178 */,
|
||||
0, 1, 2, 3, 8, 9, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 179 */,
|
||||
4, 5, 8, 9, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 180 */,
|
||||
0, 1, 4, 5, 8, 9, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 181 */,
|
||||
2, 3, 4, 5, 8, 9, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 182 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
10, 11, 14, 15, 255, 255, 255, 255 /* 183 */,
|
||||
6, 7, 8, 9, 10, 11, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 184 */,
|
||||
0, 1, 6, 7, 8, 9, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 185 */,
|
||||
2, 3, 6, 7, 8, 9, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 186 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
10, 11, 14, 15, 255, 255, 255, 255 /* 187 */,
|
||||
4, 5, 6, 7, 8, 9, 10, 11,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 188 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 14, 15, 255, 255, 255, 255 /* 189 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 14, 15, 255, 255, 255, 255 /* 190 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 14, 15, 255, 255 /* 191 */,
|
||||
12, 13, 14, 15, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 192 */,
|
||||
0, 1, 12, 13, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 193 */,
|
||||
2, 3, 12, 13, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 194 */,
|
||||
0, 1, 2, 3, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 195 */,
|
||||
4, 5, 12, 13, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 196 */,
|
||||
0, 1, 4, 5, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 197 */,
|
||||
2, 3, 4, 5, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 198 */,
|
||||
0, 1, 2, 3, 4, 5, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 199 */,
|
||||
6, 7, 12, 13, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 200 */,
|
||||
0, 1, 6, 7, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 201 */,
|
||||
2, 3, 6, 7, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 202 */,
|
||||
0, 1, 2, 3, 6, 7, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 203 */,
|
||||
4, 5, 6, 7, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 204 */,
|
||||
0, 1, 4, 5, 6, 7, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 205 */,
|
||||
2, 3, 4, 5, 6, 7, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 206 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 207 */,
|
||||
8, 9, 12, 13, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 208 */,
|
||||
0, 1, 8, 9, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 209 */,
|
||||
2, 3, 8, 9, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 210 */,
|
||||
0, 1, 2, 3, 8, 9, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 211 */,
|
||||
4, 5, 8, 9, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 212 */,
|
||||
0, 1, 4, 5, 8, 9, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 213 */,
|
||||
2, 3, 4, 5, 8, 9, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 214 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 215 */,
|
||||
6, 7, 8, 9, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 216 */,
|
||||
0, 1, 6, 7, 8, 9, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 217 */,
|
||||
2, 3, 6, 7, 8, 9, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 218 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 219 */,
|
||||
4, 5, 6, 7, 8, 9, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 220 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 221 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 222 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 12, 13, 14, 15, 255, 255 /* 223 */,
|
||||
10, 11, 12, 13, 14, 15, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 224 */,
|
||||
0, 1, 10, 11, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 225 */,
|
||||
2, 3, 10, 11, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 226 */,
|
||||
0, 1, 2, 3, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 227 */,
|
||||
4, 5, 10, 11, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 228 */,
|
||||
0, 1, 4, 5, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 229 */,
|
||||
2, 3, 4, 5, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 230 */,
|
||||
0, 1, 2, 3, 4, 5, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 231 */,
|
||||
6, 7, 10, 11, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 232 */,
|
||||
0, 1, 6, 7, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 233 */,
|
||||
2, 3, 6, 7, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 234 */,
|
||||
0, 1, 2, 3, 6, 7, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 235 */,
|
||||
4, 5, 6, 7, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 236 */,
|
||||
0, 1, 4, 5, 6, 7, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 237 */,
|
||||
2, 3, 4, 5, 6, 7, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 238 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
10, 11, 12, 13, 14, 15, 255, 255 /* 239 */,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
255, 255, 255, 255, 255, 255, 255, 255 /* 240 */,
|
||||
0, 1, 8, 9, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 241 */,
|
||||
2, 3, 8, 9, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 242 */,
|
||||
0, 1, 2, 3, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 243 */,
|
||||
4, 5, 8, 9, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 244 */,
|
||||
0, 1, 4, 5, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 245 */,
|
||||
2, 3, 4, 5, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 246 */,
|
||||
0, 1, 2, 3, 4, 5, 8, 9,
|
||||
10, 11, 12, 13, 14, 15, 255, 255 /* 247 */,
|
||||
6, 7, 8, 9, 10, 11, 12, 13,
|
||||
14, 15, 255, 255, 255, 255, 255, 255 /* 248 */,
|
||||
0, 1, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 249 */,
|
||||
2, 3, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 250 */,
|
||||
0, 1, 2, 3, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 14, 15, 255, 255 /* 251 */,
|
||||
4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 255, 255, 255, 255 /* 252 */,
|
||||
0, 1, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 14, 15, 255, 255 /* 253 */,
|
||||
2, 3, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 14, 15, 255, 255 /* 254 */,
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15 /* 255 */,
|
||||
};
|
||||
|
||||
#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(aarch64_rej_uniform_table)
|
||||
|
||||
#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */
|
||||
638
sdk/software/examples/mlkem_test/mlkem_native/src/native/api.h
Normal file
638
sdk/software/examples/mlkem_test/mlkem_native/src/native/api.h
Normal file
@@ -0,0 +1,638 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [FIPS203]
|
||||
* FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard
|
||||
* National Institute of Standards and Technology
|
||||
* https://csrc.nist.gov/pubs/fips/203/final
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_API_H
|
||||
#define MLK_NATIVE_API_H
|
||||
/*
|
||||
* Native arithmetic interface
|
||||
*
|
||||
* This header is primarily for documentation purposes.
|
||||
* It should not be included by backend implementations.
|
||||
*
|
||||
* To ensure consistency with backends, the header will be
|
||||
* included automatically after inclusion of the active
|
||||
* backend, to ensure consistency of function signatures,
|
||||
* and run sanity checks.
|
||||
*/
|
||||
|
||||
#include "../cbmc.h"
|
||||
#include "../common.h"
|
||||
|
||||
/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */
|
||||
#define MLK_NATIVE_FUNC_SUCCESS (0)
|
||||
/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that
|
||||
* the target/parameters are unsupported; typically, this would be because of
|
||||
* dependencies on CPU features not detected on the host CPU. In this case,
|
||||
* the frontend falls back to the default C implementation. */
|
||||
#define MLK_NATIVE_FUNC_FALLBACK (-1)
|
||||
|
||||
|
||||
/* Absolute exclusive upper bound for the output of the inverse NTT
|
||||
*
|
||||
* NOTE: This is the same bound as in poly.h and has to be kept
|
||||
* in sync. */
|
||||
#define MLK_INVNTT_BOUND (8 * MLKEM_Q)
|
||||
|
||||
/* Absolute exclusive upper bound for the output of the forward NTT
|
||||
*
|
||||
* NOTE: This is the same bound as in poly.h and has to be kept
|
||||
* in sync. */
|
||||
#define MLK_NTT_BOUND (8 * MLKEM_Q)
|
||||
|
||||
/*
|
||||
* This is the C<->native interface allowing for the drop-in of
|
||||
* native code for performance critical arithmetic components of ML-KEM.
|
||||
*
|
||||
* A _backend_ is a specific implementation of (part of) this interface.
|
||||
*
|
||||
* To add a function to a backend, define MLK_USE_NATIVE_XXX and
|
||||
* implement `static inline xxx(...)` in the profile header.
|
||||
*
|
||||
* The only exception is MLK_USE_NATIVE_NTT_CUSTOM_ORDER. This option can
|
||||
* be set if there are native implementations for all of NTT, invNTT, and
|
||||
* base multiplication, and allows the native implementation to use a
|
||||
* custom order of polynomial coefficients in NTT domain -- the use of such
|
||||
* custom order is not an implementation-detail since the public matrix
|
||||
* is generated in NTT domain. In this case, a permutation function
|
||||
* mlk_poly_permute_bitrev_to_custom() needs to be provided that permutes
|
||||
* polynomials in NTT domain from bitreversed to the custom order.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Those functions are meant to be trivial wrappers around the chosen native
|
||||
* implementation. The are static inline to avoid unnecessary calls.
|
||||
* The macro before each declaration controls whether a native
|
||||
* implementation is present.
|
||||
*/
|
||||
|
||||
#if defined(MLK_USE_NATIVE_NTT)
|
||||
/**
|
||||
* Compute the negacyclic number-theoretic transform (NTT) of a polynomial
|
||||
* in place.
|
||||
*
|
||||
* The input polynomial is assumed to be in normal order. The output
|
||||
* polynomial is in bitreversed order, or of a custom order if
|
||||
* MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of
|
||||
* MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information.
|
||||
*
|
||||
* @param[in,out] p Input/output polynomial.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_ntt_native(int16_t p[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_NTT_BOUND))
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q))
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_NTT */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER)
|
||||
/*
|
||||
* This must only be set if NTT, invNTT, basemul, mulcache, and
|
||||
* to/from byte stream conversions all have native implementations
|
||||
* that are adapted to the custom order.
|
||||
*/
|
||||
#if !defined(MLK_USE_NATIVE_NTT) || !defined(MLK_USE_NATIVE_INTT) || \
|
||||
!defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) || \
|
||||
!defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) || \
|
||||
!defined(MLK_USE_NATIVE_POLY_TOBYTES) || \
|
||||
!defined(MLK_USE_NATIVE_POLY_FROMBYTES)
|
||||
#error \
|
||||
"Invalid native profile: MLK_USE_NATIVE_NTT_CUSTOM_ORDER can only be \
|
||||
set if there are native implementations for NTT, invNTT, mulcache, basemul, \
|
||||
and to/from bytes conversions."
|
||||
#endif /* !MLK_USE_NATIVE_NTT || !MLK_USE_NATIVE_INTT || \
|
||||
!MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE || \
|
||||
!MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED || \
|
||||
!MLK_USE_NATIVE_POLY_TOBYTES || !MLK_USE_NATIVE_POLY_FROMBYTES */
|
||||
|
||||
/**
|
||||
* When MLK_USE_NATIVE_NTT_CUSTOM_ORDER is defined, convert a polynomial in
|
||||
* NTT domain from bitreversed order to the custom order output by the
|
||||
* native NTT.
|
||||
*
|
||||
* This must only be defined if there is native code for all of (a) NTT,
|
||||
* (b) invNTT, (c) basemul, (d) mulcache.
|
||||
*
|
||||
* @param[in,out] p Input/output polynomial.
|
||||
*/
|
||||
static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t p[MLKEM_N])
|
||||
__contract__(
|
||||
/* We don't specify that this should be a permutation, but only
|
||||
* that it does not change the bound established at the end of mlk_gen_matrix. */
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)));
|
||||
#endif /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_INTT)
|
||||
/**
|
||||
* Compute the inverse negacyclic number-theoretic transform (NTT) of a
|
||||
* polynomial in place.
|
||||
*
|
||||
* The input polynomial is in bitreversed order, or of a custom order if
|
||||
* MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of
|
||||
* MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. The output
|
||||
* polynomial is assumed to be in normal order.
|
||||
*
|
||||
* @param[in,out] p Input/output polynomial.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_intt_native(int16_t p[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_INVNTT_BOUND))
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_INTT */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_REDUCE)
|
||||
/**
|
||||
* Apply modular reduction to all coefficients of a polynomial, mapping them
|
||||
* to unsigned canonical representatives in [0,..,MLKEM_Q-1].
|
||||
*
|
||||
* @param[in,out] p Input/output polynomial.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_reduce_native(int16_t p[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(p, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_POLY_REDUCE */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_TOMONT)
|
||||
/**
|
||||
* In-place conversion of all coefficients of a polynomial from the normal
|
||||
* domain to the Montgomery domain.
|
||||
*
|
||||
* @param[in,out] p Input/output polynomial.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tomont_native(int16_t p[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q))
|
||||
ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_POLY_TOMONT */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE)
|
||||
/**
|
||||
* Compute multiplication cache for a polynomial in NTT domain.
|
||||
*
|
||||
* The purpose of the multiplication cache is to cache repeated computations
|
||||
* required during a base multiplication of polynomials in NTT domain. The
|
||||
* structure of the multiplication-cache is implementation defined.
|
||||
*
|
||||
* @param[out] cache Multiplication cache.
|
||||
* @param[in] mlk_poly Input polynomial. Must be in NTT domain and in
|
||||
* bitreversed order, or of a custom order if
|
||||
* MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the
|
||||
* documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for
|
||||
* more information.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_mulcache_compute_native(
|
||||
int16_t cache[MLKEM_N / 2], const int16_t mlk_poly[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2)))
|
||||
requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2)))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED)
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2
|
||||
/**
|
||||
* Compute scalar product of length-2 polynomial vectors in NTT domain.
|
||||
*
|
||||
* @param[out] r Result of the scalar product. Again in NTT domain, of
|
||||
* the same ordering as @p a and @p b.
|
||||
* @param[in] a First polynomial vector operand. Must be in NTT
|
||||
* domain and in bitreversed order, or of a custom order
|
||||
* if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the
|
||||
* documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for
|
||||
* more information.
|
||||
* @param[in] b Second polynomial vector operand. As for @p a.
|
||||
* @param[in] b_cache Multiplication-cache for @p b.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N],
|
||||
const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2)))
|
||||
requires(array_bound(a, 0, 2 * MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
);
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3
|
||||
/**
|
||||
* Compute scalar product of length-3 polynomial vectors in NTT domain.
|
||||
*
|
||||
* @param[out] r Result of the scalar product. Again in NTT domain, of
|
||||
* the same ordering as @p a and @p b.
|
||||
* @param[in] a First polynomial vector operand. Must be in NTT
|
||||
* domain and in bitreversed order, or of a custom order
|
||||
* if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the
|
||||
* documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for
|
||||
* more information.
|
||||
* @param[in] b Second polynomial vector operand. As for @p a.
|
||||
* @param[in] b_cache Multiplication-cache for @p b.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N],
|
||||
const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2)))
|
||||
requires(array_bound(a, 0, 3 * MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
);
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
/**
|
||||
* Compute scalar product of length-4 polynomial vectors in NTT domain.
|
||||
*
|
||||
* @param[out] r Result of the scalar product. Again in NTT domain, of
|
||||
* the same ordering as @p a and @p b.
|
||||
* @param[in] a First polynomial vector operand. Must be in NTT
|
||||
* domain and in bitreversed order, or of a custom order
|
||||
* if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the
|
||||
* documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for
|
||||
* more information.
|
||||
* @param[in] b Second polynomial vector operand. As for @p a.
|
||||
* @param[in] b_cache Multiplication-cache for @p b.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N],
|
||||
const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2)))
|
||||
requires(array_bound(a, 0, 4 * MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)
|
||||
);
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_TOBYTES)
|
||||
/**
|
||||
* Serialization of a polynomial with unsigned canonical coefficients.
|
||||
*
|
||||
* @spec{Implements ByteEncode_12 from @[FIPS203, Algorithm 5].}
|
||||
*
|
||||
* @param[out] r Output byte array (of MLKEM_POLYBYTES bytes).
|
||||
* @param[in] a Input polynomial, with each coefficient in [0,..,MLKEM_Q-1].
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES],
|
||||
const int16_t a[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYBYTES))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_POLY_TOBYTES */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_FROMBYTES)
|
||||
/**
|
||||
* Deserialization of a polynomial.
|
||||
*
|
||||
* @spec{Implements ByteDecode_12 from @[FIPS203, Algorithm 6].}
|
||||
*
|
||||
* @param[out] a Output polynomial in NTT domain.
|
||||
* @param[in] r Input byte array (of MLKEM_POLYBYTES bytes).
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_frombytes_native(
|
||||
int16_t a[MLKEM_N], const uint8_t r[MLKEM_POLYBYTES])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(a, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_POLY_FROMBYTES */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_REJ_UNIFORM)
|
||||
/**
|
||||
* Run rejection sampling on uniformly random bytes to generate uniformly random
|
||||
* integers mod MLKEM_Q, represented in [0,..,MLKEM_Q-1].
|
||||
*
|
||||
* @param[out] r Output buffer.
|
||||
* @param len Requested number of 16-bit integers (uniform mod MLKEM_Q).
|
||||
* @param[in] buf Input buffer (assumed to be uniform random bytes).
|
||||
* @param buflen Length of input buffer in bytes.
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Native implementation does not support
|
||||
* the input lengths.
|
||||
* @retval other Non-negative number of sampled 16-bit
|
||||
* integers (at most @p len).
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len,
|
||||
const uint8_t *buf,
|
||||
unsigned buflen)
|
||||
__contract__(
|
||||
requires(len <= 4096 && buflen <= 4096 && buflen % 3 == 0)
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * len))
|
||||
requires(memory_no_alias(buf, buflen))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * len))
|
||||
ensures(return_value != MLK_NATIVE_FUNC_FALLBACK
|
||||
==> (0 <= return_value && return_value <= len))
|
||||
ensures(return_value != MLK_NATIVE_FUNC_FALLBACK
|
||||
==> array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q))
|
||||
);
|
||||
#endif /* MLK_USE_NATIVE_REJ_UNIFORM */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3)
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4)
|
||||
/**
|
||||
* Compression (4 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Compress_4 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d4_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK));
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10)
|
||||
/**
|
||||
* Compression (10 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Compress_10 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d10_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK));
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (4 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d4.
|
||||
*
|
||||
* @spec{Decompress_4 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4
|
||||
* bytes).
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d4_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)));
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (10 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d10.
|
||||
*
|
||||
* @spec{Decompress_10 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10
|
||||
* bytes).
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d10_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)));
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5)
|
||||
/**
|
||||
* Compression (5 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Compress_5 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d5_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK));
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11)
|
||||
/**
|
||||
* Compression (11 bits) and subsequent serialization of a polynomial.
|
||||
*
|
||||
* @spec{Compress_11 from @[FIPS203, Eq (4.7)].}
|
||||
*
|
||||
* @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11
|
||||
* bytes).
|
||||
* @param[in] a Input polynomial. Coefficients must be unsigned canonical,
|
||||
* i.e. in [0,1,..,MLKEM_Q-1].
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d11_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK));
|
||||
#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (5 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d5.
|
||||
*
|
||||
* @spec{Decompress_5 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5
|
||||
* bytes).
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d5_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)));
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */
|
||||
|
||||
#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11)
|
||||
/**
|
||||
* De-serialization and subsequent decompression (11 bits) of a polynomial;
|
||||
* approximate inverse of mlk_poly_compress_d11.
|
||||
*
|
||||
* @spec{Decompress_11 from @[FIPS203, Eq (4.8)].}
|
||||
*
|
||||
* Upon return, the coefficients of the output polynomial are
|
||||
* unsigned-canonical (non-negative and smaller than MLKEM_Q).
|
||||
*
|
||||
* @param[out] r Output polynomial.
|
||||
* @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11
|
||||
* bytes).
|
||||
*
|
||||
* @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded.
|
||||
* @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back.
|
||||
*/
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d11_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11])
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)
|
||||
ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)));
|
||||
#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
#endif /* !MLK_NATIVE_API_H */
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_NATIVE_META_H
|
||||
#define MLK_NATIVE_META_H
|
||||
|
||||
/*
|
||||
* Default arithmetic backend
|
||||
*/
|
||||
#include "../sys.h"
|
||||
|
||||
#ifdef MLK_SYS_AARCH64
|
||||
#include "aarch64/meta.h"
|
||||
#endif
|
||||
|
||||
/* The x86_64 backend requires toolchain support for the SysV ABI */
|
||||
#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED)
|
||||
#include "x86_64/meta.h"
|
||||
#endif
|
||||
|
||||
#if defined(MLK_SYS_RISCV64_RVV)
|
||||
#include "riscv64/meta.h"
|
||||
#endif
|
||||
|
||||
#ifdef MLK_SYS_PPC64LE
|
||||
#include "ppc64le/meta.h"
|
||||
#endif
|
||||
|
||||
#endif /* !MLK_NATIVE_META_H */
|
||||
@@ -0,0 +1,6 @@
|
||||
[//]: # (SPDX-License-Identifier: CC-BY-4.0)
|
||||
|
||||
# ppc64le backend (little endian)
|
||||
|
||||
This directory contains a native backend for little endian POWER 9 (ppc64le) and above systems.
|
||||
Or, Power systems supporting ISA 2.07 and above.
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_PPC64LE_META_H
|
||||
#define MLK_NATIVE_PPC64LE_META_H
|
||||
|
||||
/* Identifier for this backend so that source and assembly files
|
||||
* in the build can be appropriately guarded. */
|
||||
#define MLK_ARITH_BACKEND_PPC64LE_DEFAULT
|
||||
|
||||
#define MLK_ARITH_BACKEND_NAME PPC64LE_DEFAULT
|
||||
|
||||
/* Set of primitives that this backend replaces */
|
||||
#define MLK_USE_NATIVE_NTT
|
||||
#define MLK_USE_NATIVE_INTT
|
||||
#define MLK_USE_NATIVE_POLY_REDUCE
|
||||
#define MLK_USE_NATIVE_POLY_TOMONT
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <string.h>
|
||||
#include "../../common.h"
|
||||
#include "../../params.h"
|
||||
#include "../api.h"
|
||||
#include "src/arith_native_ppc64le.h"
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
#if defined(__POWER8_VECTOR__)
|
||||
mlk_ntt_ppc_asm(data, mlk_ppc_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
#else
|
||||
(void)data;
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
#endif
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
#if defined(__POWER8_VECTOR__)
|
||||
mlk_intt_ppc_asm(data, mlk_ppc_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
#else
|
||||
(void)data;
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
#endif
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
#if defined(__POWER8_VECTOR__)
|
||||
mlk_reduce_ppc_asm(data, mlk_ppc_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
#else
|
||||
(void)data;
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
#endif
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
#if defined(__POWER8_VECTOR__)
|
||||
mlk_poly_tomont_ppc_asm(data, mlk_ppc_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
#else
|
||||
(void)data;
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
#endif
|
||||
}
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_NATIVE_PPC64LE_META_H */
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H
|
||||
#define MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include "../../../common.h"
|
||||
#include "consts.h"
|
||||
|
||||
#define mlk_ntt_ppc_asm MLK_NAMESPACE(ntt_ppc_asm)
|
||||
void mlk_ntt_ppc_asm(int16_t *, const int16_t *);
|
||||
|
||||
#define mlk_intt_ppc_asm MLK_NAMESPACE(intt_ppc_asm)
|
||||
void mlk_intt_ppc_asm(int16_t *, const int16_t *);
|
||||
|
||||
#define mlk_reduce_ppc_asm MLK_NAMESPACE(reduce_ppc_asm)
|
||||
void mlk_reduce_ppc_asm(int16_t *r, const int16_t *);
|
||||
|
||||
#define mlk_poly_tomont_ppc_asm MLK_NAMESPACE(poly_tomont_ppc_asm)
|
||||
void mlk_poly_tomont_ppc_asm(int16_t *, const int16_t *);
|
||||
|
||||
#endif /* !MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H */
|
||||
@@ -0,0 +1,299 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__)
|
||||
|
||||
#include "consts.h"
|
||||
|
||||
/*
|
||||
* Constant table for the PPC64LE NTT/invNTT.
|
||||
* For the layout, see the offset macros in consts.h.
|
||||
*/
|
||||
/* clang-format off */
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_ppc_qdata[2072] = {
|
||||
/* -Q */
|
||||
-3329, -3329, -3329, -3329, -3329, -3329, -3329, -3329,
|
||||
/* Q */
|
||||
3329, 3329, 3329, 3329, 3329, 3329, 3329, 3329,
|
||||
/* round(2^26 / MLKEM_Q), Barrett reduction constant */
|
||||
20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159,
|
||||
/* N^-1 in Montgomery form */
|
||||
512, 512, 512, 512, 512, 512, 512, 512,
|
||||
/* Barrett twist of N^-1 */
|
||||
5040, 5040, 5040, 5040, 5040, 5040, 5040, 5040,
|
||||
/* 2^16 mod MLKEM_Q (R), Barrett tomont factor */
|
||||
-1044, -1044, -1044, -1044, -1044, -1044, -1044, -1044,
|
||||
/* Barrett twist of R for poly_tomont */
|
||||
-10276, -10276, -10276, -10276, -10276, -10276, -10276, -10276,
|
||||
/* Twiddle factors for the forward NTT */
|
||||
-1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600,
|
||||
-749, -749, -749, -749, -749, -749, -749, -749,
|
||||
-40, -40, -40, -40, -40, -40, -40, -40,
|
||||
-687, -687, -687, -687, -687, -687, -687, -687,
|
||||
630, 630, 630, 630, 630, 630, 630, 630,
|
||||
-1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432,
|
||||
848, 848, 848, 848, 848, 848, 848, 848,
|
||||
1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062,
|
||||
-1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410,
|
||||
193, 193, 193, 193, 193, 193, 193, 193,
|
||||
797, 797, 797, 797, 797, 797, 797, 797,
|
||||
-543, -543, -543, -543, -543, -543, -543, -543,
|
||||
-69, -69, -69, -69, -69, -69, -69, -69,
|
||||
569, 569, 569, 569, 569, 569, 569, 569,
|
||||
-1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583,
|
||||
296, 296, 296, 296, 296, 296, 296, 296,
|
||||
-882, -882, -882, -882, -882, -882, -882, -882,
|
||||
1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339,
|
||||
1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476,
|
||||
-283, -283, -283, -283, -283, -283, -283, -283,
|
||||
56, 56, 56, 56, 56, 56, 56, 56,
|
||||
-1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089,
|
||||
1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333,
|
||||
1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426,
|
||||
-1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235,
|
||||
535, 535, 535, 535, 535, 535, 535, 535,
|
||||
-447, -447, -447, -447, -447, -447, -447, -447,
|
||||
-936, -936, -936, -936, -936, -936, -936, -936,
|
||||
-450, -450, -450, -450, -450, -450, -450, -450,
|
||||
-1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355,
|
||||
821, 821, 821, 821, 821, 821, 821, 821,
|
||||
331, 331, 331, 331, 289, 289, 289, 289,
|
||||
-1573, -1573, -1573, -1573, -76, -76, -76, -76,
|
||||
-1025, -1025, -1025, -1025, 1197, 1197, 1197, 1197,
|
||||
-1274, -1274, -1274, -1274, -1052, -1052, -1052, -1052,
|
||||
-1352, -1352, -1352, -1352, 650, 650, 650, 650,
|
||||
632, 632, 632, 632, -816, -816, -816, -816,
|
||||
33, 33, 33, 33, -464, -464, -464, -464,
|
||||
-1414, -1414, -1414, -1414, 1320, 1320, 1320, 1320,
|
||||
1435, 1435, 1435, 1435, -1010, -1010, -1010, -1010,
|
||||
452, 452, 452, 452, 807, 807, 807, 807,
|
||||
-461, -461, -461, -461, 1438, 1438, 1438, 1438,
|
||||
-927, -927, -927, -927, 1534, 1534, 1534, 1534,
|
||||
-712, -712, -712, -712, -682, -682, -682, -682,
|
||||
648, 648, 648, 648, 1481, 1481, 1481, 1481,
|
||||
-219, -219, -219, -219, -855, -855, -855, -855,
|
||||
910, 910, 910, 910, 1227, 1227, 1227, 1227,
|
||||
583, 583, 17, 17, -680, -680, -568, -568,
|
||||
-1041, -1041, 1637, 1637, 1100, 1100, 723, 723,
|
||||
-48, -48, 1409, 1409, 233, 233, -667, -667,
|
||||
-314, -314, 756, 756, -279, -279, -1173, -1173,
|
||||
-540, -540, -1626, -1626, -1540, -1540, 1651, 1651,
|
||||
1461, 1461, -1482, -1482, -642, -642, 952, 952,
|
||||
-892, -892, 939, 939, -941, -941, -1021, -1021,
|
||||
268, 268, 733, 733, 641, 641, -992, -992,
|
||||
-1292, -1292, 1584, 1584, -109, -109, -1031, -1031,
|
||||
-1239, -1239, 375, 375, 1645, 1645, -780, -780,
|
||||
-556, -556, 1063, 1063, 757, 757, 319, 319,
|
||||
-863, -863, -1230, -1230, -735, -735, 561, 561,
|
||||
403, 403, -525, -525, 1026, 1026, 1092, 1092,
|
||||
-554, -554, 1143, 1143, 886, 886, -1179, -1179,
|
||||
-1455, -1455, -1607, -1607, 1029, 1029, 1212, 1212,
|
||||
885, 885, -1219, -1219, -1175, -1175, -394, -394,
|
||||
/* Twiddle factors for the inverse NTT */
|
||||
-394, -394, -1175, -1175, -1219, -1219, 885, 885,
|
||||
1212, 1212, 1029, 1029, -1607, -1607, -1455, -1455,
|
||||
-1179, -1179, 886, 886, 1143, 1143, -554, -554,
|
||||
1092, 1092, 1026, 1026, -525, -525, 403, 403,
|
||||
561, 561, -735, -735, -1230, -1230, -863, -863,
|
||||
319, 319, 757, 757, 1063, 1063, -556, -556,
|
||||
-780, -780, 1645, 1645, 375, 375, -1239, -1239,
|
||||
-1031, -1031, -109, -109, 1584, 1584, -1292, -1292,
|
||||
-992, -992, 641, 641, 733, 733, 268, 268,
|
||||
-1021, -1021, -941, -941, 939, 939, -892, -892,
|
||||
952, 952, -642, -642, -1482, -1482, 1461, 1461,
|
||||
1651, 1651, -1540, -1540, -1626, -1626, -540, -540,
|
||||
-1173, -1173, -279, -279, 756, 756, -314, -314,
|
||||
-667, -667, 233, 233, 1409, 1409, -48, -48,
|
||||
723, 723, 1100, 1100, 1637, 1637, -1041, -1041,
|
||||
-568, -568, -680, -680, 17, 17, 583, 583,
|
||||
1227, 1227, 1227, 1227, 910, 910, 910, 910,
|
||||
-855, -855, -855, -855, -219, -219, -219, -219,
|
||||
1481, 1481, 1481, 1481, 648, 648, 648, 648,
|
||||
-682, -682, -682, -682, -712, -712, -712, -712,
|
||||
1534, 1534, 1534, 1534, -927, -927, -927, -927,
|
||||
1438, 1438, 1438, 1438, -461, -461, -461, -461,
|
||||
807, 807, 807, 807, 452, 452, 452, 452,
|
||||
-1010, -1010, -1010, -1010, 1435, 1435, 1435, 1435,
|
||||
1320, 1320, 1320, 1320, -1414, -1414, -1414, -1414,
|
||||
-464, -464, -464, -464, 33, 33, 33, 33,
|
||||
-816, -816, -816, -816, 632, 632, 632, 632,
|
||||
650, 650, 650, 650, -1352, -1352, -1352, -1352,
|
||||
-1052, -1052, -1052, -1052, -1274, -1274, -1274, -1274,
|
||||
1197, 1197, 1197, 1197, -1025, -1025, -1025, -1025,
|
||||
-76, -76, -76, -76, -1573, -1573, -1573, -1573,
|
||||
289, 289, 289, 289, 331, 331, 331, 331,
|
||||
821, 821, 821, 821, 821, 821, 821, 821,
|
||||
-1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355,
|
||||
-450, -450, -450, -450, -450, -450, -450, -450,
|
||||
-936, -936, -936, -936, -936, -936, -936, -936,
|
||||
-447, -447, -447, -447, -447, -447, -447, -447,
|
||||
535, 535, 535, 535, 535, 535, 535, 535,
|
||||
-1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235,
|
||||
1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426,
|
||||
1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333,
|
||||
-1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089,
|
||||
56, 56, 56, 56, 56, 56, 56, 56,
|
||||
-283, -283, -283, -283, -283, -283, -283, -283,
|
||||
1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476,
|
||||
1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339,
|
||||
-882, -882, -882, -882, -882, -882, -882, -882,
|
||||
296, 296, 296, 296, 296, 296, 296, 296,
|
||||
-1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583,
|
||||
569, 569, 569, 569, 569, 569, 569, 569,
|
||||
-69, -69, -69, -69, -69, -69, -69, -69,
|
||||
-543, -543, -543, -543, -543, -543, -543, -543,
|
||||
797, 797, 797, 797, 797, 797, 797, 797,
|
||||
193, 193, 193, 193, 193, 193, 193, 193,
|
||||
-1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410,
|
||||
1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062,
|
||||
848, 848, 848, 848, 848, 848, 848, 848,
|
||||
-1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432,
|
||||
630, 630, 630, 630, 630, 630, 630, 630,
|
||||
-687, -687, -687, -687, -687, -687, -687, -687,
|
||||
-40, -40, -40, -40, -40, -40, -40, -40,
|
||||
-749, -749, -749, -749, -749, -749, -749, -749,
|
||||
-1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600,
|
||||
/* Twisted twiddle factors for the forward NTT */
|
||||
-15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749,
|
||||
-7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373,
|
||||
-394, -394, -394, -394, -394, -394, -394, -394,
|
||||
-6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762,
|
||||
6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201,
|
||||
-14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095,
|
||||
8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347,
|
||||
10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453,
|
||||
-13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879,
|
||||
1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900,
|
||||
7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845,
|
||||
-5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345,
|
||||
-679, -679, -679, -679, -679, -679, -679, -679,
|
||||
5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601,
|
||||
-15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582,
|
||||
2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914,
|
||||
-8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682,
|
||||
13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180,
|
||||
14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529,
|
||||
-2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786,
|
||||
551, 551, 551, 551, 551, 551, 551, 551,
|
||||
-10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719,
|
||||
13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121,
|
||||
14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036,
|
||||
-12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156,
|
||||
5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266,
|
||||
-4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400,
|
||||
-9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213,
|
||||
-4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429,
|
||||
-13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338,
|
||||
8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081,
|
||||
3258, 3258, 3258, 3258, 2845, 2845, 2845, 2845,
|
||||
-15483, -15483, -15483, -15483, -748, -748, -748, -748,
|
||||
-10089, -10089, -10089, -10089, 11782, 11782, 11782, 11782,
|
||||
-12540, -12540, -12540, -12540, -10355, -10355, -10355, -10355,
|
||||
-13308, -13308, -13308, -13308, 6398, 6398, 6398, 6398,
|
||||
6221, 6221, 6221, 6221, -8032, -8032, -8032, -8032,
|
||||
325, 325, 325, 325, -4567, -4567, -4567, -4567,
|
||||
-13918, -13918, -13918, -13918, 12993, 12993, 12993, 12993,
|
||||
14125, 14125, 14125, 14125, -9942, -9942, -9942, -9942,
|
||||
4449, 4449, 4449, 4449, 7943, 7943, 7943, 7943,
|
||||
-4538, -4538, -4538, -4538, 14155, 14155, 14155, 14155,
|
||||
-9125, -9125, -9125, -9125, 15099, 15099, 15099, 15099,
|
||||
-7008, -7008, -7008, -7008, -6713, -6713, -6713, -6713,
|
||||
6378, 6378, 6378, 6378, 14578, 14578, 14578, 14578,
|
||||
-2156, -2156, -2156, -2156, -8416, -8416, -8416, -8416,
|
||||
8957, 8957, 8957, 8957, 12078, 12078, 12078, 12078,
|
||||
5739, 5739, 167, 167, -6693, -6693, -5591, -5591,
|
||||
-10247, -10247, 16113, 16113, 10828, 10828, 7117, 7117,
|
||||
-472, -472, 13869, 13869, 2293, 2293, -6565, -6565,
|
||||
-3091, -3091, 7441, 7441, -2746, -2746, -11546, -11546,
|
||||
-5315, -5315, -16005, -16005, -15159, -15159, 16251, 16251,
|
||||
14381, 14381, -14588, -14588, -6319, -6319, 9371, 9371,
|
||||
-8780, -8780, 9243, 9243, -9262, -9262, -10050, -10050,
|
||||
2638, 2638, 7215, 7215, 6309, 6309, -9764, -9764,
|
||||
-12717, -12717, 15592, 15592, -1073, -1073, -10148, -10148,
|
||||
-12196, -12196, 3691, 3691, 16192, 16192, -7678, -7678,
|
||||
-5473, -5473, 10463, 10463, 7451, 7451, 3140, 3140,
|
||||
-8495, -8495, -12107, -12107, -7235, -7235, 5522, 5522,
|
||||
3967, 3967, -5168, -5168, 10099, 10099, 10749, 10749,
|
||||
-5453, -5453, 11251, 11251, 8721, 8721, -11605, -11605,
|
||||
-14322, -14322, -15818, -15818, 10129, 10129, 11930, 11930,
|
||||
8711, 8711, -11999, -11999, -11566, -11566, -3878, -3878,
|
||||
/* Twisted twiddle factors for the inverse NTT */
|
||||
-3878, -3878, -11566, -11566, -11999, -11999, 8711, 8711,
|
||||
11930, 11930, 10129, 10129, -15818, -15818, -14322, -14322,
|
||||
-11605, -11605, 8721, 8721, 11251, 11251, -5453, -5453,
|
||||
10749, 10749, 10099, 10099, -5168, -5168, 3967, 3967,
|
||||
5522, 5522, -7235, -7235, -12107, -12107, -8495, -8495,
|
||||
3140, 3140, 7451, 7451, 10463, 10463, -5473, -5473,
|
||||
-7678, -7678, 16192, 16192, 3691, 3691, -12196, -12196,
|
||||
-10148, -10148, -1073, -1073, 15592, 15592, -12717, -12717,
|
||||
-9764, -9764, 6309, 6309, 7215, 7215, 2638, 2638,
|
||||
-10050, -10050, -9262, -9262, 9243, 9243, -8780, -8780,
|
||||
9371, 9371, -6319, -6319, -14588, -14588, 14381, 14381,
|
||||
16251, 16251, -15159, -15159, -16005, -16005, -5315, -5315,
|
||||
-11546, -11546, -2746, -2746, 7441, 7441, -3091, -3091,
|
||||
-6565, -6565, 2293, 2293, 13869, 13869, -472, -472,
|
||||
7117, 7117, 10828, 10828, 16113, 16113, -10247, -10247,
|
||||
-5591, -5591, -6693, -6693, 167, 167, 5739, 5739,
|
||||
12078, 12078, 12078, 12078, 8957, 8957, 8957, 8957,
|
||||
-8416, -8416, -8416, -8416, -2156, -2156, -2156, -2156,
|
||||
14578, 14578, 14578, 14578, 6378, 6378, 6378, 6378,
|
||||
-6713, -6713, -6713, -6713, -7008, -7008, -7008, -7008,
|
||||
15099, 15099, 15099, 15099, -9125, -9125, -9125, -9125,
|
||||
14155, 14155, 14155, 14155, -4538, -4538, -4538, -4538,
|
||||
7943, 7943, 7943, 7943, 4449, 4449, 4449, 4449,
|
||||
-9942, -9942, -9942, -9942, 14125, 14125, 14125, 14125,
|
||||
12993, 12993, 12993, 12993, -13918, -13918, -13918, -13918,
|
||||
-4567, -4567, -4567, -4567, 325, 325, 325, 325,
|
||||
-8032, -8032, -8032, -8032, 6221, 6221, 6221, 6221,
|
||||
6398, 6398, 6398, 6398, -13308, -13308, -13308, -13308,
|
||||
-10355, -10355, -10355, -10355, -12540, -12540, -12540, -12540,
|
||||
11782, 11782, 11782, 11782, -10089, -10089, -10089, -10089,
|
||||
-748, -748, -748, -748, -15483, -15483, -15483, -15483,
|
||||
2845, 2845, 2845, 2845, 3258, 3258, 3258, 3258,
|
||||
8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081,
|
||||
-13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338,
|
||||
-4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429,
|
||||
-9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213,
|
||||
-4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400,
|
||||
5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266,
|
||||
-12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156,
|
||||
14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036,
|
||||
13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121,
|
||||
-10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719,
|
||||
551, 551, 551, 551, 551, 551, 551, 551,
|
||||
-2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786,
|
||||
14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529,
|
||||
13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180,
|
||||
-8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682,
|
||||
2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914,
|
||||
-15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582,
|
||||
5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601,
|
||||
-679, -679, -679, -679, -679, -679, -679, -679,
|
||||
-5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345,
|
||||
7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845,
|
||||
1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900,
|
||||
-13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879,
|
||||
10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453,
|
||||
8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347,
|
||||
-14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095,
|
||||
6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201,
|
||||
-6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762,
|
||||
-394, -394, -394, -394, -394, -394, -394, -394,
|
||||
-7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373,
|
||||
-15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749,
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED && __POWER8_VECTOR__ */
|
||||
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_PPC64LE_SRC_CONSTS_H
|
||||
#define MLK_NATIVE_PPC64LE_SRC_CONSTS_H
|
||||
#include "../../../common.h"
|
||||
|
||||
/* Byte offsets into the constant table */
|
||||
#define MLK_PPC_NQ_OFFSET 0
|
||||
#define MLK_PPC_Q_OFFSET 16
|
||||
#define MLK_PPC_C20159_OFFSET 32
|
||||
#define MLK_PPC_N_INV_OFFSET 48
|
||||
#define MLK_PPC_N_INV_TW_OFFSET 64
|
||||
#define MLK_PPC_TOMONT_OFFSET 80
|
||||
#define MLK_PPC_TOMONT_TW_OFFSET 96
|
||||
#define MLK_PPC_ZETA_NTT_OFFSET 112
|
||||
#define MLK_PPC_ZETA_INTT_OFFSET 1120
|
||||
#define MLK_PPC_ZETA_NTT_TW_OFFSET 2128
|
||||
#define MLK_PPC_ZETA_INTT_TW_OFFSET 3136
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
#define mlk_ppc_qdata MLK_NAMESPACE(ppc_qdata)
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_ppc_qdata[2072];
|
||||
#endif
|
||||
|
||||
#endif /* !MLK_NATIVE_PPC64LE_SRC_CONSTS_H */
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,315 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) IBM Corp. 2025, 2026
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*
|
||||
* Written by Danny Tsen <dtsen@us.ibm.com>
|
||||
*/
|
||||
|
||||
/*
|
||||
* Poly_tomont: Inplace conversion of all coefficients of a polynomial
|
||||
* from normal domain to Montgomery domain
|
||||
*
|
||||
* Arguments:*r: pointer to input/output polynomial
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__)
|
||||
/*yaml
|
||||
Name: poly_tomont_ppc_asm
|
||||
Description: PowerPC64 VSX conversion to Montgomery form
|
||||
Signature: void mlk_poly_tomont_ppc_asm(int16_t *r, const int16_t *qdata)
|
||||
ABI:
|
||||
Architecture: powerpc64le
|
||||
CallingConvention: ELFv2
|
||||
Features: [VSX]
|
||||
r3:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t *r
|
||||
description: Input/output polynomial (256 x int16_t)
|
||||
r4:
|
||||
type: buffer
|
||||
size_bytes: 4144
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *qdata
|
||||
description: Precomputed constants (2072 x int16_t)
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/ppc64le/src/poly_tomont_ppc_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_tomont_ppc_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_tomont_ppc_asm)
|
||||
|
||||
stdu 1, -320(1)
|
||||
mflr 0
|
||||
li 6, 128
|
||||
li 7, 144
|
||||
li 8, 160
|
||||
li 9, 176
|
||||
li 10, 192
|
||||
li 11, 208
|
||||
li 12, 224
|
||||
stxvd2x 52, 6, 1
|
||||
stxvd2x 53, 7, 1
|
||||
stxvd2x 54, 8, 1
|
||||
stxvd2x 55, 9, 1
|
||||
stxvd2x 56, 10, 1
|
||||
stxvd2x 57, 11, 1
|
||||
stxvd2x 58, 12, 1
|
||||
li 6, 240
|
||||
li 7, 256
|
||||
li 8, 272
|
||||
li 9, 288
|
||||
stxvd2x 59, 6, 1
|
||||
stxvd2x 60, 7, 1
|
||||
stxvd2x 61, 8, 1
|
||||
stxvd2x 62, 9, 1
|
||||
li 6, 0
|
||||
li 7, 80
|
||||
li 8, 96
|
||||
lxvd2x 37, 6, 4
|
||||
lxvd2x 32, 7, 4
|
||||
lxvd2x 34, 8, 4
|
||||
vxor 3, 3, 3
|
||||
li 4, -128
|
||||
li 5, -112
|
||||
li 6, -96
|
||||
li 7, -80
|
||||
li 8, -64
|
||||
li 9, -48
|
||||
li 10, -32
|
||||
li 11, -16
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 27, 14, 5, 13
|
||||
vmladduhm 28, 19, 5, 18
|
||||
vmladduhm 29, 24, 5, 23
|
||||
vmladduhm 30, 8, 5, 7
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 13, 14, 5, 13
|
||||
vmladduhm 18, 19, 5, 18
|
||||
vmladduhm 23, 24, 5, 23
|
||||
vmladduhm 7, 8, 5, 7
|
||||
stxvd2x 59, 4, 3
|
||||
stxvd2x 60, 5, 3
|
||||
stxvd2x 61, 6, 3
|
||||
stxvd2x 62, 7, 3
|
||||
stxvd2x 45, 8, 3
|
||||
stxvd2x 50, 9, 3
|
||||
stxvd2x 55, 10, 3
|
||||
stxvd2x 39, 11, 3
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 27, 14, 5, 13
|
||||
vmladduhm 28, 19, 5, 18
|
||||
vmladduhm 29, 24, 5, 23
|
||||
vmladduhm 30, 8, 5, 7
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 13, 14, 5, 13
|
||||
vmladduhm 18, 19, 5, 18
|
||||
vmladduhm 23, 24, 5, 23
|
||||
vmladduhm 7, 8, 5, 7
|
||||
stxvd2x 59, 4, 3
|
||||
stxvd2x 60, 5, 3
|
||||
stxvd2x 61, 6, 3
|
||||
stxvd2x 62, 7, 3
|
||||
stxvd2x 45, 8, 3
|
||||
stxvd2x 50, 9, 3
|
||||
stxvd2x 55, 10, 3
|
||||
stxvd2x 39, 11, 3
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 27, 14, 5, 13
|
||||
vmladduhm 28, 19, 5, 18
|
||||
vmladduhm 29, 24, 5, 23
|
||||
vmladduhm 30, 8, 5, 7
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 13, 14, 5, 13
|
||||
vmladduhm 18, 19, 5, 18
|
||||
vmladduhm 23, 24, 5, 23
|
||||
vmladduhm 7, 8, 5, 7
|
||||
stxvd2x 59, 4, 3
|
||||
stxvd2x 60, 5, 3
|
||||
stxvd2x 61, 6, 3
|
||||
stxvd2x 62, 7, 3
|
||||
stxvd2x 45, 8, 3
|
||||
stxvd2x 50, 9, 3
|
||||
stxvd2x 55, 10, 3
|
||||
stxvd2x 39, 11, 3
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 27, 14, 5, 13
|
||||
vmladduhm 28, 19, 5, 18
|
||||
vmladduhm 29, 24, 5, 23
|
||||
vmladduhm 30, 8, 5, 7
|
||||
lxvd2x 45, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 50, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 55, 0, 3
|
||||
addi 3, 3, 16
|
||||
lxvd2x 39, 0, 3
|
||||
addi 3, 3, 16
|
||||
vmhraddshs 14, 13, 2, 3
|
||||
vmhraddshs 19, 18, 2, 3
|
||||
vmhraddshs 24, 23, 2, 3
|
||||
vmhraddshs 8, 7, 2, 3
|
||||
vmladduhm 13, 13, 0, 3
|
||||
vmladduhm 18, 18, 0, 3
|
||||
vmladduhm 23, 23, 0, 3
|
||||
vmladduhm 7, 7, 0, 3
|
||||
vmladduhm 13, 14, 5, 13
|
||||
vmladduhm 18, 19, 5, 18
|
||||
vmladduhm 23, 24, 5, 23
|
||||
vmladduhm 7, 8, 5, 7
|
||||
stxvd2x 59, 4, 3
|
||||
stxvd2x 60, 5, 3
|
||||
stxvd2x 61, 6, 3
|
||||
stxvd2x 62, 7, 3
|
||||
stxvd2x 45, 8, 3
|
||||
stxvd2x 50, 9, 3
|
||||
stxvd2x 55, 10, 3
|
||||
stxvd2x 39, 11, 3
|
||||
li 6, 128
|
||||
li 7, 144
|
||||
li 8, 160
|
||||
li 9, 176
|
||||
li 10, 192
|
||||
li 11, 208
|
||||
li 12, 224
|
||||
lxvd2x 52, 6, 1
|
||||
lxvd2x 53, 7, 1
|
||||
lxvd2x 54, 8, 1
|
||||
lxvd2x 55, 9, 1
|
||||
lxvd2x 56, 10, 1
|
||||
lxvd2x 57, 11, 1
|
||||
lxvd2x 58, 12, 1
|
||||
li 6, 240
|
||||
li 7, 256
|
||||
li 8, 272
|
||||
li 9, 288
|
||||
lxvd2x 59, 6, 1
|
||||
lxvd2x 60, 7, 1
|
||||
lxvd2x 61, 8, 1
|
||||
lxvd2x 62, 9, 1
|
||||
mtlr 0
|
||||
addi 1, 1, 320
|
||||
blr
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_tomont_ppc_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
&& __POWER8_VECTOR__ */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,731 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* Copyright (c) IBM Corp. 2025, 2026
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*
|
||||
* Written by Danny Tsen <dtsen@us.ibm.com>
|
||||
*/
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__)
|
||||
/*yaml
|
||||
Name: reduce_ppc_asm
|
||||
Description: PowerPC64 VSX modular reduction
|
||||
Signature: void mlk_reduce_ppc_asm(int16_t *r, const int16_t *qdata)
|
||||
ABI:
|
||||
Architecture: powerpc64le
|
||||
CallingConvention: ELFv2
|
||||
Features: [VSX]
|
||||
r3:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t *r
|
||||
description: Input/output polynomial (256 x int16_t)
|
||||
r4:
|
||||
type: buffer
|
||||
size_bytes: 4144
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *qdata
|
||||
description: Precomputed constants (2072 x int16_t)
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/ppc64le/src/reduce_ppc_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(reduce_ppc_asm)
|
||||
MLK_ASM_FN_SYMBOL(reduce_ppc_asm)
|
||||
|
||||
stdu 1, -224(1)
|
||||
mflr 0
|
||||
std 14, 96(1)
|
||||
std 15, 104(1)
|
||||
std 16, 112(1)
|
||||
li 6, 128
|
||||
li 7, 144
|
||||
li 8, 160
|
||||
li 9, 176
|
||||
li 10, 192
|
||||
stxvd2x 52, 6, 1
|
||||
stxvd2x 53, 7, 1
|
||||
stxvd2x 54, 8, 1
|
||||
stxvd2x 55, 9, 1
|
||||
stxvd2x 56, 10, 1
|
||||
vxor 7, 7, 7
|
||||
li 6, 16
|
||||
li 7, 32
|
||||
lxvd2x 35, 6, 4
|
||||
lxvd2x 32, 7, 4
|
||||
vspltisw 2, 13
|
||||
vadduwm 2, 2, 2
|
||||
vspltisw 4, 1
|
||||
vsubuwm 5, 2, 4
|
||||
vslw 1, 4, 5
|
||||
li 4, -128
|
||||
li 5, -112
|
||||
li 6, -96
|
||||
li 7, -80
|
||||
li 8, -64
|
||||
li 9, -48
|
||||
li 10, -32
|
||||
li 11, -16
|
||||
li 14, 16
|
||||
li 15, 32
|
||||
li 16, 48
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 21, 4, 3, 8
|
||||
vmladduhm 22, 9, 3, 12
|
||||
vmladduhm 23, 13, 3, 16
|
||||
vmladduhm 24, 17, 3, 20
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 4, 4, 3, 8
|
||||
vmladduhm 9, 9, 3, 12
|
||||
vmladduhm 13, 13, 3, 16
|
||||
vmladduhm 17, 17, 3, 20
|
||||
stxvd2x 53, 4, 3
|
||||
stxvd2x 54, 5, 3
|
||||
stxvd2x 55, 6, 3
|
||||
stxvd2x 56, 7, 3
|
||||
stxvd2x 36, 8, 3
|
||||
stxvd2x 41, 9, 3
|
||||
stxvd2x 45, 10, 3
|
||||
stxvd2x 49, 11, 3
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 21, 4, 3, 8
|
||||
vmladduhm 22, 9, 3, 12
|
||||
vmladduhm 23, 13, 3, 16
|
||||
vmladduhm 24, 17, 3, 20
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 4, 4, 3, 8
|
||||
vmladduhm 9, 9, 3, 12
|
||||
vmladduhm 13, 13, 3, 16
|
||||
vmladduhm 17, 17, 3, 20
|
||||
stxvd2x 53, 4, 3
|
||||
stxvd2x 54, 5, 3
|
||||
stxvd2x 55, 6, 3
|
||||
stxvd2x 56, 7, 3
|
||||
stxvd2x 36, 8, 3
|
||||
stxvd2x 41, 9, 3
|
||||
stxvd2x 45, 10, 3
|
||||
stxvd2x 49, 11, 3
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 21, 4, 3, 8
|
||||
vmladduhm 22, 9, 3, 12
|
||||
vmladduhm 23, 13, 3, 16
|
||||
vmladduhm 24, 17, 3, 20
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 4, 4, 3, 8
|
||||
vmladduhm 9, 9, 3, 12
|
||||
vmladduhm 13, 13, 3, 16
|
||||
vmladduhm 17, 17, 3, 20
|
||||
stxvd2x 53, 4, 3
|
||||
stxvd2x 54, 5, 3
|
||||
stxvd2x 55, 6, 3
|
||||
stxvd2x 56, 7, 3
|
||||
stxvd2x 36, 8, 3
|
||||
stxvd2x 41, 9, 3
|
||||
stxvd2x 45, 10, 3
|
||||
stxvd2x 49, 11, 3
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 21, 4, 3, 8
|
||||
vmladduhm 22, 9, 3, 12
|
||||
vmladduhm 23, 13, 3, 16
|
||||
vmladduhm 24, 17, 3, 20
|
||||
lxvd2x 40, 0, 3
|
||||
lxvd2x 44, 14, 3
|
||||
lxvd2x 48, 15, 3
|
||||
lxvd2x 52, 16, 3
|
||||
addi 3, 3, 64
|
||||
vmulosh 6, 8, 0
|
||||
vmulesh 5, 8, 0
|
||||
vmulosh 11, 12, 0
|
||||
vmulesh 10, 12, 0
|
||||
vmulosh 15, 16, 0
|
||||
vmulesh 14, 16, 0
|
||||
vmulosh 19, 20, 0
|
||||
vmulesh 18, 20, 0
|
||||
xxmrglw 36, 37, 38
|
||||
xxmrghw 37, 37, 38
|
||||
xxmrglw 41, 42, 43
|
||||
xxmrghw 42, 42, 43
|
||||
xxmrglw 45, 46, 47
|
||||
xxmrghw 46, 46, 47
|
||||
xxmrglw 49, 50, 51
|
||||
xxmrghw 50, 50, 51
|
||||
vadduwm 4, 4, 1
|
||||
vadduwm 5, 5, 1
|
||||
vadduwm 9, 9, 1
|
||||
vadduwm 10, 10, 1
|
||||
vadduwm 13, 13, 1
|
||||
vadduwm 14, 14, 1
|
||||
vadduwm 17, 17, 1
|
||||
vadduwm 18, 18, 1
|
||||
vsraw 4, 4, 2
|
||||
vsraw 5, 5, 2
|
||||
vsraw 9, 9, 2
|
||||
vsraw 10, 10, 2
|
||||
vsraw 13, 13, 2
|
||||
vsraw 14, 14, 2
|
||||
vsraw 17, 17, 2
|
||||
vsraw 18, 18, 2
|
||||
vpkuwum 4, 5, 4
|
||||
vsubuhm 4, 7, 4
|
||||
vpkuwum 9, 10, 9
|
||||
vsubuhm 9, 7, 9
|
||||
vpkuwum 13, 14, 13
|
||||
vsubuhm 13, 7, 13
|
||||
vpkuwum 17, 18, 17
|
||||
vsubuhm 17, 7, 17
|
||||
vmladduhm 4, 4, 3, 8
|
||||
vmladduhm 9, 9, 3, 12
|
||||
vmladduhm 13, 13, 3, 16
|
||||
vmladduhm 17, 17, 3, 20
|
||||
stxvd2x 53, 4, 3
|
||||
stxvd2x 54, 5, 3
|
||||
stxvd2x 55, 6, 3
|
||||
stxvd2x 56, 7, 3
|
||||
stxvd2x 36, 8, 3
|
||||
stxvd2x 41, 9, 3
|
||||
stxvd2x 45, 10, 3
|
||||
stxvd2x 49, 11, 3
|
||||
addi 3, 3, -512
|
||||
vxor 9, 9, 9
|
||||
vspltish 10, 15
|
||||
vmr 11, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
lxvd2x 44, 0, 3
|
||||
lxvd2x 45, 14, 3
|
||||
lxvd2x 46, 15, 3
|
||||
lxvd2x 47, 16, 3
|
||||
addi 3, 3, 64
|
||||
vsrh 1, 12, 10
|
||||
vsrh 0, 13, 10
|
||||
vsrh 3, 14, 10
|
||||
vsrh 2, 15, 10
|
||||
vadduhm 7, 12, 11
|
||||
vadduhm 8, 13, 11
|
||||
vadduhm 5, 14, 11
|
||||
vadduhm 6, 15, 11
|
||||
vcmpequh 1, 1, 9
|
||||
vcmpequh 0, 0, 9
|
||||
vcmpequh 3, 3, 9
|
||||
vcmpequh 2, 2, 9
|
||||
xxsel 33, 39, 44, 33
|
||||
xxsel 32, 40, 45, 32
|
||||
xxsel 35, 37, 46, 35
|
||||
xxsel 34, 38, 47, 34
|
||||
stxvd2x 35, 10, 3
|
||||
stxvd2x 34, 11, 3
|
||||
stxvd2x 33, 8, 3
|
||||
stxvd2x 32, 9, 3
|
||||
ld 14, 96(1)
|
||||
ld 15, 104(1)
|
||||
ld 16, 112(1)
|
||||
li 6, 128
|
||||
li 7, 144
|
||||
li 8, 160
|
||||
li 9, 176
|
||||
li 10, 192
|
||||
lxvd2x 52, 6, 1
|
||||
lxvd2x 53, 7, 1
|
||||
lxvd2x 54, 8, 1
|
||||
lxvd2x 55, 9, 1
|
||||
lxvd2x 56, 10, 1
|
||||
mtlr 0
|
||||
addi 1, 1, 224
|
||||
blr
|
||||
|
||||
MLK_ASM_FN_SIZE(reduce_ppc_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
&& __POWER8_VECTOR__ */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,11 @@
|
||||
[//]: # (SPDX-License-Identifier: CC-BY-4.0)
|
||||
|
||||
# RISC-V Vector Extension Backend
|
||||
|
||||
This is an arithmetic backend for CPUs implementing the RISC-V Vector Extension. The backend is functional for all physical `VLEN`, but the NTT and inverse NTT are so far only implemented for VLEN=256, falling back to the default C implementations for other VLENs.
|
||||
|
||||
## Requirements
|
||||
|
||||
- RISC-V 64-bit architecture
|
||||
- Vector extension (RVV) version 1.0
|
||||
- Standard "gc" extensions (integer and compressed instructions)
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_RISCV64_META_H
|
||||
#define MLK_NATIVE_RISCV64_META_H
|
||||
|
||||
/* Identifier for this backend so that source and assembly files
|
||||
* in the build can be appropriately guarded. */
|
||||
#define MLK_ARITH_BACKEND_RISCV64
|
||||
|
||||
/* Set of primitives that this backend replaces */
|
||||
#define MLK_USE_NATIVE_NTT
|
||||
#define MLK_USE_NATIVE_INTT
|
||||
#define MLK_USE_NATIVE_POLY_TOMONT
|
||||
#define MLK_USE_NATIVE_REJ_UNIFORM
|
||||
#define MLK_USE_NATIVE_POLY_REDUCE
|
||||
#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
|
||||
#include "../../common.h"
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include <riscv_vector.h>
|
||||
|
||||
#include "../api.h"
|
||||
#include "src/arith_native_riscv64.h"
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
/* VLEN = 256 only for now */
|
||||
if (__riscv_vsetvlmax_e16m1() != 16)
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_rv64v_poly_ntt(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
/* VLEN = 256 only for now */
|
||||
if (__riscv_vsetvlmax_e16m1() != 16)
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_rv64v_poly_invntt_tomont(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
mlk_rv64v_poly_tomont(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len,
|
||||
const uint8_t *buf,
|
||||
unsigned buflen)
|
||||
{
|
||||
/* mlk_rv64v_rej_uniform requires buflen to be a multiple of 12 */
|
||||
if (buflen % 12 != 0)
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
/* The cast from unsigned to signed integer is safe
|
||||
* because the return value is <= len, which we asssume
|
||||
* to be bound by 4096 and hence <= INT_MAX. */
|
||||
return (int)mlk_rv64v_rej_uniform(r, len, buf, buflen);
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
mlk_rv64v_poly_reduce(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2],
|
||||
const int16_t y[MLKEM_N])
|
||||
{
|
||||
(void)x; /* not using the cache at the moment */
|
||||
(void)y;
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N],
|
||||
const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)])
|
||||
{
|
||||
(void)b_cache;
|
||||
mlk_rv64v_poly_basemul_mont_add_k2(r, a, b);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N],
|
||||
const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)])
|
||||
{
|
||||
(void)b_cache;
|
||||
mlk_rv64v_poly_basemul_mont_add_k3(r, a, b);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N],
|
||||
const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)])
|
||||
{
|
||||
(void)b_cache;
|
||||
mlk_rv64v_poly_basemul_mont_add_k4(r, a, b);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_NATIVE_RISCV64_META_H */
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H
|
||||
#define MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#define mlk_rv64v_poly_ntt MLK_NAMESPACE(ntt_riscv64)
|
||||
void mlk_rv64v_poly_ntt(int16_t *);
|
||||
|
||||
#define mlk_rv64v_poly_invntt_tomont MLK_NAMESPACE(intt_riscv64)
|
||||
void mlk_rv64v_poly_invntt_tomont(int16_t *r);
|
||||
|
||||
#define mlk_rv64v_poly_basemul_mont_add_k2 MLK_NAMESPACE(basemul_add_k2_riscv64)
|
||||
void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a,
|
||||
const int16_t *b);
|
||||
|
||||
#define mlk_rv64v_poly_basemul_mont_add_k3 MLK_NAMESPACE(basemul_add_k3_riscv64)
|
||||
void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a,
|
||||
const int16_t *b);
|
||||
|
||||
#define mlk_rv64v_poly_basemul_mont_add_k4 MLK_NAMESPACE(basemul_add_k4_riscv64)
|
||||
void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a,
|
||||
const int16_t *b);
|
||||
|
||||
#define mlk_rv64v_poly_tomont MLK_NAMESPACE(tomont_riscv64)
|
||||
void mlk_rv64v_poly_tomont(int16_t *r);
|
||||
|
||||
#define mlk_rv64v_poly_reduce MLK_NAMESPACE(reduce_riscv64)
|
||||
void mlk_rv64v_poly_reduce(int16_t *r);
|
||||
|
||||
#define mlk_rv64v_poly_add MLK_NAMESPACE(poly_add_riscv64)
|
||||
void mlk_rv64v_poly_add(int16_t *r, const int16_t *a, const int16_t *b);
|
||||
|
||||
#define mlk_rv64v_poly_sub MLK_NAMESPACE(poly_sub_riscv64)
|
||||
void mlk_rv64v_poly_sub(int16_t *r, const int16_t *a, const int16_t *b);
|
||||
|
||||
#define mlk_rv64v_rej_uniform MLK_NAMESPACE(rj_uniform_riscv64)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len,
|
||||
const uint8_t *buf, unsigned int buflen);
|
||||
|
||||
#endif /* !MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H */
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_RISCV64) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG)
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "../../../debug.h"
|
||||
#include "rv64v_debug.h"
|
||||
|
||||
#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] "
|
||||
|
||||
/**
|
||||
* Check whether values in a vint16m1_t vector are within specified bounds.
|
||||
*
|
||||
* @param[in] file Filename.
|
||||
* @param line Line number.
|
||||
* @param vec RISC-V vector to be checked.
|
||||
* @param vl Vector length (number of active elements).
|
||||
* @param lower_bound_exclusive Exclusive lower bound.
|
||||
* @param upper_bound_exclusive Exclusive upper bound.
|
||||
*/
|
||||
void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec,
|
||||
size_t vl, int lower_bound_exclusive,
|
||||
int upper_bound_exclusive)
|
||||
{
|
||||
/* Allocate temporary array to store vector elements
|
||||
* We use the maximum possible vector length to be safe */
|
||||
int16_t temp_array[64];
|
||||
|
||||
/* Store vector elements to temporary array for inspection */
|
||||
__riscv_vse16_v_i16m1(temp_array, vec, vl);
|
||||
|
||||
/* Reuse existing array bounds checking function */
|
||||
mlk_debug_check_bounds(file, line, temp_array, (unsigned)vl,
|
||||
lower_bound_exclusive, upper_bound_exclusive);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether values in a vint16m2_t vector are within specified bounds.
|
||||
*
|
||||
* @param[in] file Filename.
|
||||
* @param line Line number.
|
||||
* @param vec RISC-V vector to be checked.
|
||||
* @param vl Vector length (active elements per m1 half).
|
||||
* @param lower_bound_exclusive Exclusive lower bound.
|
||||
* @param upper_bound_exclusive Exclusive upper bound.
|
||||
*/
|
||||
void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec,
|
||||
size_t vl, int lower_bound_exclusive,
|
||||
int upper_bound_exclusive)
|
||||
{
|
||||
/* Allocate temporary array to store vector elements
|
||||
* m2 vectors hold 2x the elements of m1 vectors */
|
||||
int16_t temp_array[2 * 64];
|
||||
|
||||
/* Store vector elements to temporary array for inspection */
|
||||
__riscv_vse16_v_i16m2(temp_array, vec, 2 * vl);
|
||||
|
||||
/* Reuse existing array bounds checking function for all elements */
|
||||
mlk_debug_check_bounds(file, line, temp_array, (unsigned)(2 * vl),
|
||||
lower_bound_exclusive, upper_bound_exclusive);
|
||||
}
|
||||
|
||||
#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
MLKEM_DEBUG */
|
||||
|
||||
MLK_EMPTY_CU(rv64v_debug)
|
||||
|
||||
#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
MLKEM_DEBUG) */
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
#undef MLK_DEBUG_ERROR_HEADER
|
||||
@@ -0,0 +1,140 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H
|
||||
#define MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H
|
||||
|
||||
#include "../../../debug.h"
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
/*************************************************
|
||||
* RISC-V Vector Bounds Assertion Macros
|
||||
*
|
||||
* These macros provide runtime bounds checking for RISC-V vector types
|
||||
* vint16m1_t and vint16m2_t, following the same pattern as the scalar
|
||||
* bounds assertions in debug.h
|
||||
*
|
||||
* The macros are only active when MLKEM_DEBUG is defined, otherwise they
|
||||
* compile to no-ops for zero runtime overhead in release builds.
|
||||
**************************************************/
|
||||
|
||||
#if defined(MLKEM_DEBUG)
|
||||
|
||||
/**
|
||||
* Check whether values in a vint16m1_t vector are within specified bounds.
|
||||
*
|
||||
* @param[in] file Filename.
|
||||
* @param line Line number.
|
||||
* @param vec RISC-V vector to be checked.
|
||||
* @param vl Vector length (number of active elements).
|
||||
* @param lower_bound_exclusive Exclusive lower bound.
|
||||
* @param upper_bound_exclusive Exclusive upper bound.
|
||||
*/
|
||||
#define mlk_debug_check_bounds_int16m1 \
|
||||
MLK_NAMESPACE(mlkem_debug_check_bounds_int16m1)
|
||||
void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec,
|
||||
size_t vl, int lower_bound_exclusive,
|
||||
int upper_bound_exclusive);
|
||||
|
||||
/**
|
||||
* Check whether values in a vint16m2_t vector are within specified bounds
|
||||
* by splitting into m1 vectors.
|
||||
*
|
||||
* @param[in] file Filename.
|
||||
* @param line Line number.
|
||||
* @param vec RISC-V vector to be checked.
|
||||
* @param vl Vector length (active elements per m1 half).
|
||||
* @param lower_bound_exclusive Exclusive lower bound.
|
||||
* @param upper_bound_exclusive Exclusive upper bound.
|
||||
*/
|
||||
#define mlk_debug_check_bounds_int16m2 \
|
||||
MLK_NAMESPACE(mlkem_debug_check_bounds_int16m2)
|
||||
void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec,
|
||||
size_t vl, int lower_bound_exclusive,
|
||||
int upper_bound_exclusive);
|
||||
|
||||
/* Check bounds in vint16m1_t vector
|
||||
* vec: RISC-V vector of type vint16m1_t
|
||||
* vl: Vector length (number of active elements)
|
||||
* value_lb: Inclusive lower value bound
|
||||
* value_ub: Exclusive upper value bound */
|
||||
#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \
|
||||
mlk_debug_check_bounds_int16m1(__FILE__, __LINE__, (vec), (vl), \
|
||||
(value_lb) - 1, (value_ub))
|
||||
|
||||
/* Check absolute bounds in vint16m1_t vector
|
||||
* vec: RISC-V vector of type vint16m1_t
|
||||
* vl: Vector length (number of active elements)
|
||||
* value_abs_bd: Exclusive absolute upper bound */
|
||||
#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \
|
||||
mlk_assert_bound_int16m1((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd))
|
||||
|
||||
/* Check bounds in vint16m2_t vector
|
||||
* vec: RISC-V vector of type vint16m2_t
|
||||
* vl: Vector length (number of active elements per m1 half)
|
||||
* value_lb: Inclusive lower value bound
|
||||
* value_ub: Exclusive upper value bound */
|
||||
#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \
|
||||
mlk_debug_check_bounds_int16m2(__FILE__, __LINE__, (vec), (vl), \
|
||||
(value_lb) - 1, (value_ub))
|
||||
|
||||
/* Check absolute bounds in vint16m2_t vector
|
||||
* vec: RISC-V vector of type vint16m2_t
|
||||
* vl: Vector length (number of active elements per m1 half)
|
||||
* value_abs_bd: Exclusive absolute upper bound */
|
||||
#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \
|
||||
mlk_assert_bound_int16m2((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd))
|
||||
|
||||
#elif defined(CBMC)
|
||||
|
||||
/* For CBMC, we would need to implement vector bounds checking using CBMC
|
||||
* primitives This is complex and would require extracting vector elements, so
|
||||
* for now we provide empty implementations that could be extended later */
|
||||
#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#else /* !MLKEM_DEBUG && CBMC */
|
||||
|
||||
/* When debugging is disabled, all assertions become no-ops */
|
||||
#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \
|
||||
do \
|
||||
{ \
|
||||
} while (0)
|
||||
|
||||
#endif /* !MLKEM_DEBUG && !CBMC */
|
||||
|
||||
#endif /* !MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H */
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "arith_native_riscv64.h"
|
||||
|
||||
const int16_t izeta[] = {
|
||||
-1044, 758, 1571, 205, 1275, -677, 1065, -448, -1628, -1522, 1460,
|
||||
-958, -991, -996, 308, 108, 1517, 359, -411, 1542, 725, 1508,
|
||||
-961, 398, -478, 870, 854, 1510, -794, 1278, 1530, 1185, -202,
|
||||
-287, -608, -732, 951, 247, 1421, -107, 1659, 1187, -220, 874,
|
||||
1335, -1218, 136, 1215, -1422, -1493, -1017, 681, -830, 271, 90,
|
||||
853, -384, 1465, 1285, -1322, -610, -603, -1097, -817, -1468, 1474,
|
||||
130, 1602, -1469, -126, 1162, 1618, 75, 156, -329, -418, -349,
|
||||
872, -644, 1590, 1202, -962, -1458, 829, 666, 320, 8, -516,
|
||||
-1119, 602, -1483, 777, 147, -1159, -778, 246, -182, -1577, -383,
|
||||
-264, 1544, 282, -1491, 1293, -1653, -1574, 460, 291, 235, -177,
|
||||
-587, -422, -622, 171, 1325, -573, -1015, 552, -652, -1223, -105,
|
||||
-1550, -871, 1251, -843, -555, -430, 1103,
|
||||
};
|
||||
@@ -0,0 +1,793 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* === ML-KEM NTT using RISC-V Vector intrinstics */
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_RISCV64) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
#include "arith_native_riscv64.h"
|
||||
#include "rv64v_debug.h"
|
||||
|
||||
/* Montgomery reduction constants */
|
||||
/* check-magic: -3327 == signed_mod(pow(MLKEM_Q,-1,2^16), 2^16) */
|
||||
#define MLK_RVV_QI -3327
|
||||
|
||||
/* check-magic: 2285 == unsigned_mod(2^16, MLKEM_Q) */
|
||||
#define MLK_RVV_MONT_R1 2285
|
||||
|
||||
/* check-magic: 1353 == pow(2, 32, MLKEM_Q) */
|
||||
#define MLK_RVV_MONT_R2 1353
|
||||
|
||||
/* check-magic: 1441 == pow(2,32-7,MLKEM_Q) */
|
||||
#define MLK_RVV_MONT_NR 1441
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_redc(vint16m1_t rh, vint16m1_t rl, size_t vl)
|
||||
{
|
||||
vint16m1_t t;
|
||||
|
||||
t = __riscv_vmul_vx_i16m1(rl, MLK_RVV_QI, vl); /* t = l * Q^-1 */
|
||||
t = __riscv_vmulh_vx_i16m1(t, MLKEM_Q, vl); /* t = (t*Q) / R */
|
||||
t = __riscv_vsub_vv_i16m1(rh, t, vl); /* t = h - t */
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
/* Narrowing reduction */
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_redc2(vint32m2_t z, size_t vl)
|
||||
{
|
||||
vint16m1_t t;
|
||||
|
||||
t = __riscv_vmul_vx_i16m1(__riscv_vncvt_x_x_w_i16m1(z, vl), MLK_RVV_QI,
|
||||
vl); /* t = l * Q^-1 */
|
||||
z = __riscv_vsub_vv_i32m2(z, __riscv_vwmul_vx_i32m2(t, MLKEM_Q, vl),
|
||||
vl); /* x = (x - (t*Q)) */
|
||||
t = __riscv_vnsra_wx_i16m1(z, 16, vl);
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
/* Narrowing Barrett */
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_barrett(vint16m1_t a, size_t vl)
|
||||
{
|
||||
vint16m1_t t;
|
||||
const int16_t v = ((1 << 26) + MLKEM_Q / 2) / MLKEM_Q;
|
||||
|
||||
t = __riscv_vmulh_vx_i16m1(a, v, vl);
|
||||
t = __riscv_vadd_vx_i16m1(t, 1 << (25 - 16), vl);
|
||||
t = __riscv_vsra_vx_i16m1(t, 26 - 16, vl);
|
||||
t = __riscv_vmul_vx_i16m1(t, MLKEM_Q, vl);
|
||||
t = __riscv_vsub_vv_i16m1(a, t, vl);
|
||||
|
||||
mlk_assert_abs_bound_int16m1(t, vl, MLKEM_Q_HALF);
|
||||
return t;
|
||||
}
|
||||
|
||||
/* Conditionally add Q (if negative).
|
||||
*
|
||||
* Constant-time note: We deliberately avoid a `vmslt` + masked `vadd_mu`
|
||||
* here. RVV 1.0 does not mandate mask-population-independent latency for
|
||||
* masked ops, and Zvkt's DIEL guarantee does not cover the v0 mask
|
||||
* register. Using an arithmetic-shift sign mask keeps all operands in the
|
||||
* Zvkt-covered set and matches the idiom used in the portable C
|
||||
* (mlk_ct_cmask_neg_i16, verify.h) and AArch64 (poly_reduce_aarch64_asm.S)
|
||||
* implementations. */
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_cadd(vint16m1_t rx, size_t vl)
|
||||
{
|
||||
vint16m1_t m;
|
||||
|
||||
m = __riscv_vsra_vx_i16m1(rx, 15, vl); /* m = (x < 0) ? -1 : 0 */
|
||||
m = __riscv_vand_vx_i16m1(m, MLKEM_Q, vl); /* m = (x < 0) ? Q : 0 */
|
||||
rx = __riscv_vadd_vv_i16m1(rx, m, vl); /* x += m */
|
||||
return rx;
|
||||
}
|
||||
|
||||
/* Montgomery multiply: vector-vector */
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_mul_vv(vint16m1_t rx, vint16m1_t ry, size_t vl)
|
||||
{
|
||||
vint16m1_t rl, rh;
|
||||
|
||||
rh = __riscv_vmulh_vv_i16m1(rx, ry, vl); /* h = (x * y) / R */
|
||||
rl = __riscv_vmul_vv_i16m1(rx, ry, vl); /* l = (x * y) % R */
|
||||
return fq_redc(rh, rl, vl);
|
||||
}
|
||||
|
||||
/* Montgomery multiply: vector-scalar */
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_mul_vx(vint16m1_t rx, int16_t ry, size_t vl)
|
||||
{
|
||||
vint16m1_t rl, rh;
|
||||
|
||||
rh = __riscv_vmulh_vx_i16m1(rx, ry, vl); /* h = (x * y) / R */
|
||||
rl = __riscv_vmul_vx_i16m1(rx, ry, vl); /* l = (x * y) % R */
|
||||
return fq_redc(rh, rl, vl);
|
||||
}
|
||||
|
||||
/* full normalization */
|
||||
|
||||
static MLK_INLINE vint16m1_t fq_mulq_vx(vint16m1_t rx, int16_t ry, size_t vl)
|
||||
{
|
||||
vint16m1_t result;
|
||||
|
||||
result = fq_mul_vx(rx, ry, vl);
|
||||
|
||||
mlk_assert_abs_bound_int16m1(result, vl, MLKEM_Q);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* create a permutation for swapping index bits a and b, a < b */
|
||||
|
||||
static vuint16m2_t bitswap_perm(unsigned a, unsigned b, size_t vl)
|
||||
{
|
||||
const vuint16m2_t v2id = __riscv_vid_v_u16m2(vl);
|
||||
|
||||
vuint16m2_t xa, xb;
|
||||
xa = __riscv_vsrl_vx_u16m2(v2id, b - a, vl);
|
||||
xa = __riscv_vxor_vv_u16m2(xa, v2id, vl);
|
||||
xa = __riscv_vand_vx_u16m2(xa, (1 << a), vl);
|
||||
xb = __riscv_vsll_vx_u16m2(xa, b - a, vl);
|
||||
xa = __riscv_vxor_vv_u16m2(xa, xb, vl);
|
||||
xa = __riscv_vxor_vv_u16m2(v2id, xa, vl);
|
||||
return xa;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute negacyclic number-theoretic transform (NTT) of a polynomial in
|
||||
* place; input assumed to be in normal order, output in bitreversed order.
|
||||
*
|
||||
* @param[in,out] r Input/output polynomial.
|
||||
*/
|
||||
|
||||
/* Forward / Cooley-Tukey butterfly operation */
|
||||
|
||||
#define MLK_RVV_CT_BFLY_FX(u0, u1, ut, uc, vl, layer) \
|
||||
{ \
|
||||
mlk_assert_abs_bound(&uc, 1, MLKEM_Q_HALF); \
|
||||
\
|
||||
ut = fq_mul_vx(u1, uc, vl); \
|
||||
mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \
|
||||
\
|
||||
u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \
|
||||
u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \
|
||||
mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \
|
||||
}
|
||||
|
||||
#define MLK_RVV_CT_BFLY_FV(u0, u1, ut, uc, vl, layer) \
|
||||
{ \
|
||||
mlk_assert_abs_bound_int16m1(uc, vl, MLKEM_Q_HALF); \
|
||||
\
|
||||
ut = fq_mul_vv(u1, uc, vl); \
|
||||
mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \
|
||||
\
|
||||
u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \
|
||||
u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \
|
||||
mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \
|
||||
}
|
||||
|
||||
static vint16m2_t mlk_rv64v_ntt2(vint16m2_t vp, vint16m1_t cz)
|
||||
{
|
||||
size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */
|
||||
size_t vl2 = 2 * vl;
|
||||
|
||||
const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2);
|
||||
const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2);
|
||||
const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2);
|
||||
|
||||
/* p1 = p8(p4(p2)) */
|
||||
const vuint16m2_t v2p1 = __riscv_vrgather_vv_u16m2(
|
||||
__riscv_vrgather_vv_u16m2(v2p2, v2p4, vl2), v2p8, vl2);
|
||||
|
||||
const vuint16m1_t vid = __riscv_vid_v_u16m1(vl);
|
||||
const vuint16m1_t cs8 =
|
||||
__riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl);
|
||||
const vuint16m1_t cs4 =
|
||||
__riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl);
|
||||
const vuint16m1_t cs2 =
|
||||
__riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl);
|
||||
|
||||
vint16m1_t vt, c0, t0, t1;
|
||||
|
||||
/* swap 8 */
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2);
|
||||
t0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
t1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl);
|
||||
MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 5);
|
||||
|
||||
/* swap 4 */
|
||||
vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1);
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2);
|
||||
t0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
t1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl);
|
||||
MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 6);
|
||||
|
||||
/* swap 2 */
|
||||
vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1);
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2);
|
||||
t0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
t1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl);
|
||||
MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 7);
|
||||
|
||||
/* reorganize */
|
||||
vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1);
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p1, vl2);
|
||||
|
||||
return vp;
|
||||
}
|
||||
|
||||
/* Only for VLEN=256 for now */
|
||||
void mlk_rv64v_poly_ntt(int16_t *r)
|
||||
{
|
||||
/* zetas can be compiled into vector constants; don't pass as a pointer */
|
||||
#include "rv64v_zetas.inc"
|
||||
|
||||
size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */
|
||||
size_t vl2 = 2 * vl;
|
||||
|
||||
vint16m1_t vt;
|
||||
vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf;
|
||||
|
||||
const vint16m1_t z0 = __riscv_vle16_v_i16m1(&zeta[0x00], vl);
|
||||
const vint16m1_t z2 = __riscv_vle16_v_i16m1(&zeta[0x10], vl);
|
||||
const vint16m1_t z4 = __riscv_vle16_v_i16m1(&zeta[0x20], vl);
|
||||
const vint16m1_t z6 = __riscv_vle16_v_i16m1(&zeta[0x30], vl);
|
||||
const vint16m1_t z8 = __riscv_vle16_v_i16m1(&zeta[0x40], vl);
|
||||
const vint16m1_t za = __riscv_vle16_v_i16m1(&zeta[0x50], vl);
|
||||
const vint16m1_t zc = __riscv_vle16_v_i16m1(&zeta[0x60], vl);
|
||||
const vint16m1_t ze = __riscv_vle16_v_i16m1(&zeta[0x70], vl);
|
||||
|
||||
v0 = __riscv_vle16_v_i16m1(&r[0x00], vl);
|
||||
v1 = __riscv_vle16_v_i16m1(&r[0x10], vl);
|
||||
v2 = __riscv_vle16_v_i16m1(&r[0x20], vl);
|
||||
v3 = __riscv_vle16_v_i16m1(&r[0x30], vl);
|
||||
v4 = __riscv_vle16_v_i16m1(&r[0x40], vl);
|
||||
v5 = __riscv_vle16_v_i16m1(&r[0x50], vl);
|
||||
v6 = __riscv_vle16_v_i16m1(&r[0x60], vl);
|
||||
v7 = __riscv_vle16_v_i16m1(&r[0x70], vl);
|
||||
v8 = __riscv_vle16_v_i16m1(&r[0x80], vl);
|
||||
v9 = __riscv_vle16_v_i16m1(&r[0x90], vl);
|
||||
va = __riscv_vle16_v_i16m1(&r[0xa0], vl);
|
||||
vb = __riscv_vle16_v_i16m1(&r[0xb0], vl);
|
||||
vc = __riscv_vle16_v_i16m1(&r[0xc0], vl);
|
||||
vd = __riscv_vle16_v_i16m1(&r[0xd0], vl);
|
||||
ve = __riscv_vle16_v_i16m1(&r[0xe0], vl);
|
||||
vf = __riscv_vle16_v_i16m1(&r[0xf0], vl);
|
||||
|
||||
MLK_RVV_CT_BFLY_FX(v0, v8, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v1, v9, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v2, va, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v3, vb, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v4, vc, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v5, vd, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v6, ve, vt, zeta[0x01], vl, 1);
|
||||
MLK_RVV_CT_BFLY_FX(v7, vf, vt, zeta[0x01], vl, 1);
|
||||
|
||||
MLK_RVV_CT_BFLY_FX(v0, v4, vt, zeta[0x10], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(v1, v5, vt, zeta[0x10], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(v2, v6, vt, zeta[0x10], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(v3, v7, vt, zeta[0x10], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(v8, vc, vt, zeta[0x11], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(v9, vd, vt, zeta[0x11], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(va, ve, vt, zeta[0x11], vl, 2);
|
||||
MLK_RVV_CT_BFLY_FX(vb, vf, vt, zeta[0x11], vl, 2);
|
||||
|
||||
MLK_RVV_CT_BFLY_FX(v0, v2, vt, zeta[0x20], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(v1, v3, vt, zeta[0x20], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(v4, v6, vt, zeta[0x21], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(v5, v7, vt, zeta[0x21], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(v8, va, vt, zeta[0x30], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(v9, vb, vt, zeta[0x30], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(vc, ve, vt, zeta[0x31], vl, 3);
|
||||
MLK_RVV_CT_BFLY_FX(vd, vf, vt, zeta[0x31], vl, 3);
|
||||
|
||||
MLK_RVV_CT_BFLY_FX(v0, v1, vt, zeta[0x40], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(v2, v3, vt, zeta[0x41], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(v4, v5, vt, zeta[0x50], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(v6, v7, vt, zeta[0x51], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(v8, v9, vt, zeta[0x60], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(va, vb, vt, zeta[0x61], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(vc, vd, vt, zeta[0x70], vl, 4);
|
||||
MLK_RVV_CT_BFLY_FX(ve, vf, vt, zeta[0x71], vl, 4);
|
||||
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0x00], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v0, v1), z0), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0x20], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v2, v3), z2), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0x40], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v4, v5), z4), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0x60], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v6, v7), z6), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0x80], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v8, v9), z8), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0xa0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(va, vb), za), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0xc0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(vc, vd), zc), vl2);
|
||||
__riscv_vse16_v_i16m2(
|
||||
&r[0xe0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(ve, vf), ze), vl2);
|
||||
}
|
||||
|
||||
/* Reverse / Gentleman-Sande butterfly operation */
|
||||
|
||||
#define MLK_RVV_GS_BFLY_RX(u0, u1, ut, uc, vl) \
|
||||
{ \
|
||||
ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \
|
||||
u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \
|
||||
u1 = fq_mul_vx(ut, uc, vl); \
|
||||
}
|
||||
|
||||
#define MLK_RVV_GS_BFLY_RV(u0, u1, ut, uc, vl) \
|
||||
{ \
|
||||
ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \
|
||||
u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \
|
||||
u1 = fq_mul_vv(ut, uc, vl); \
|
||||
}
|
||||
|
||||
static vint16m2_t mlk_rv64v_intt2(vint16m2_t vp, vint16m1_t cz)
|
||||
{
|
||||
size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */
|
||||
size_t vl2 = 2 * vl;
|
||||
|
||||
const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2);
|
||||
const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2);
|
||||
const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2);
|
||||
|
||||
/* p0 = p2(p4(p8)) */
|
||||
const vuint16m2_t v2p0 = __riscv_vrgather_vv_u16m2(
|
||||
__riscv_vrgather_vv_u16m2(v2p8, v2p4, vl2), v2p2, vl2);
|
||||
|
||||
const vuint16m1_t vid = __riscv_vid_v_u16m1(vl);
|
||||
const vuint16m1_t cs8 =
|
||||
__riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl);
|
||||
const vuint16m1_t cs4 =
|
||||
__riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl);
|
||||
const vuint16m1_t cs2 =
|
||||
__riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl);
|
||||
|
||||
vint16m1_t t0, t1, c0, vt;
|
||||
|
||||
/* initial permute */
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p0, vl2);
|
||||
t0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
t1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
/* pre-scale */
|
||||
t0 = fq_mulq_vx(t0, MLK_RVV_MONT_NR, vl);
|
||||
t1 = fq_mulq_vx(t1, MLK_RVV_MONT_NR, vl);
|
||||
|
||||
/* absolute bounds: < t0 < q, t1 < q */
|
||||
mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q);
|
||||
mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q);
|
||||
|
||||
c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl);
|
||||
MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl);
|
||||
|
||||
/* absolute bounds: < t0 < 2*q, t1 < q */
|
||||
mlk_assert_abs_bound_int16m1(t0, vl, 2 * MLKEM_Q);
|
||||
mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q);
|
||||
|
||||
/* swap 2 */
|
||||
vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1);
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2);
|
||||
t0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
t1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl);
|
||||
MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl);
|
||||
|
||||
/* absolute bounds: t0 < 4*q, t1 < q */
|
||||
mlk_assert_abs_bound_int16m1(t0, vl, 4 * MLKEM_Q);
|
||||
mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q);
|
||||
|
||||
/* swap 4 */
|
||||
vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1);
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2);
|
||||
t0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
t1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl);
|
||||
MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl);
|
||||
|
||||
/* absolute bounds: < 8*q */
|
||||
mlk_assert_abs_bound_int16m1(t0, vl, 8 * MLKEM_Q);
|
||||
mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q);
|
||||
|
||||
t0 = fq_mulq_vx(t0, MLK_RVV_MONT_R1, vl);
|
||||
|
||||
/* absolute bounds: < q */
|
||||
mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q);
|
||||
mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q);
|
||||
|
||||
/* swap 8 */
|
||||
vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1);
|
||||
vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2);
|
||||
|
||||
return vp;
|
||||
}
|
||||
|
||||
#define MLK_RV64V_ABS_BOUNDS16(vl, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, \
|
||||
vb, vc, vd, ve, vf, b0, b1, b2, b3, b4, b5, b6, \
|
||||
b7, b8, b9, ba, bb, bc, bd, be, bf) \
|
||||
do \
|
||||
{ \
|
||||
mlk_assert_abs_bound_int16m1(v0, vl, (b0) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v1, vl, (b1) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v2, vl, (b2) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v3, vl, (b3) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v4, vl, (b4) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v5, vl, (b5) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v6, vl, (b6) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v7, vl, (b7) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v8, vl, (b8) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(v9, vl, (b9) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(va, vl, (ba) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(vb, vl, (bb) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(vc, vl, (bc) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(vd, vl, (bd) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(ve, vl, (be) * MLKEM_Q); \
|
||||
mlk_assert_abs_bound_int16m1(vf, vl, (bf) * MLKEM_Q); \
|
||||
} while (0)
|
||||
|
||||
|
||||
/**
|
||||
* Compute the inverse negacyclic number-theoretic transform (NTT) of a
|
||||
* polynomial in place; input assumed to be in bitreversed order, output in
|
||||
* normal order.
|
||||
*
|
||||
* @param[in,out] r Input/output polynomial.
|
||||
*/
|
||||
/* Only for VLEN=256 for now */
|
||||
void mlk_rv64v_poly_invntt_tomont(int16_t *r)
|
||||
{
|
||||
/* zetas can be compiled into vector constants; don't pass as a pointer */
|
||||
#include "rv64v_izetas.inc"
|
||||
|
||||
size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */
|
||||
size_t vl2 = 2 * vl;
|
||||
|
||||
const vint16m1_t z0 = __riscv_vle16_v_i16m1(&izeta[0x00], vl);
|
||||
const vint16m1_t z2 = __riscv_vle16_v_i16m1(&izeta[0x10], vl);
|
||||
const vint16m1_t z4 = __riscv_vle16_v_i16m1(&izeta[0x20], vl);
|
||||
const vint16m1_t z6 = __riscv_vle16_v_i16m1(&izeta[0x30], vl);
|
||||
const vint16m1_t z8 = __riscv_vle16_v_i16m1(&izeta[0x40], vl);
|
||||
const vint16m1_t za = __riscv_vle16_v_i16m1(&izeta[0x50], vl);
|
||||
const vint16m1_t zc = __riscv_vle16_v_i16m1(&izeta[0x60], vl);
|
||||
const vint16m1_t ze = __riscv_vle16_v_i16m1(&izeta[0x70], vl);
|
||||
|
||||
vint16m1_t vt;
|
||||
vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf;
|
||||
vint16m2_t vp;
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x00], vl2), z0);
|
||||
v0 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
v1 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x20], vl2), z2);
|
||||
v2 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
v3 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x40], vl2), z4);
|
||||
v4 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
v5 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x60], vl2), z6);
|
||||
v6 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
v7 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x80], vl2), z8);
|
||||
v8 = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
v9 = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xa0], vl2), za);
|
||||
va = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
vb = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xc0], vl2), zc);
|
||||
vc = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
vd = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xe0], vl2), ze);
|
||||
ve = __riscv_vget_v_i16m2_i16m1(vp, 0);
|
||||
vf = __riscv_vget_v_i16m2_i16m1(vp, 1);
|
||||
|
||||
/* absolute bounds < q (see mlk_rv64v_intt2) */
|
||||
MLK_RV64V_ABS_BOUNDS16(vl,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
|
||||
|
||||
MLK_RVV_GS_BFLY_RX(v0, v1, vt, izeta[0x40], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v2, v3, vt, izeta[0x41], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v4, v5, vt, izeta[0x50], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v6, v7, vt, izeta[0x51], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v8, v9, vt, izeta[0x60], vl);
|
||||
MLK_RVV_GS_BFLY_RX(va, vb, vt, izeta[0x61], vl);
|
||||
MLK_RVV_GS_BFLY_RX(vc, vd, vt, izeta[0x70], vl);
|
||||
MLK_RVV_GS_BFLY_RX(ve, vf, vt, izeta[0x71], vl);
|
||||
|
||||
/* absolute bounds:
|
||||
* - v{0,2,4,6,8,a,c,e}: < 2*q
|
||||
* - v{1,3,5,7,9,b,d,f}: < 1*q
|
||||
*/
|
||||
MLK_RV64V_ABS_BOUNDS16(vl,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf,
|
||||
2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1);
|
||||
|
||||
MLK_RVV_GS_BFLY_RX(v0, v2, vt, izeta[0x20], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v1, v3, vt, izeta[0x20], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v4, v6, vt, izeta[0x21], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v5, v7, vt, izeta[0x21], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v8, va, vt, izeta[0x30], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v9, vb, vt, izeta[0x30], vl);
|
||||
MLK_RVV_GS_BFLY_RX(vc, ve, vt, izeta[0x31], vl);
|
||||
MLK_RVV_GS_BFLY_RX(vd, vf, vt, izeta[0x31], vl);
|
||||
|
||||
/* absolute bounds:
|
||||
* - v{0,4,8,c}: < 4*q
|
||||
* - v{1,5,9,d}: < 2*q
|
||||
* - v{2,3,6,7,a,b,e,f}: < 1*q
|
||||
*/
|
||||
MLK_RV64V_ABS_BOUNDS16(vl,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf,
|
||||
4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1);
|
||||
|
||||
MLK_RVV_GS_BFLY_RX(v0, v4, vt, izeta[0x10], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v1, v5, vt, izeta[0x10], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v2, v6, vt, izeta[0x10], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v3, v7, vt, izeta[0x10], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v8, vc, vt, izeta[0x11], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v9, vd, vt, izeta[0x11], vl);
|
||||
MLK_RVV_GS_BFLY_RX(va, ve, vt, izeta[0x11], vl);
|
||||
MLK_RVV_GS_BFLY_RX(vb, vf, vt, izeta[0x11], vl);
|
||||
|
||||
/* absolute bounds:
|
||||
* - v{0,8}: < 8*q
|
||||
* - v{1,9}: < 4*q
|
||||
* - v{2,3,a,b}: < 2*q
|
||||
* - v{4,5,6,7,c,d,e,f}: < 1*q
|
||||
*/
|
||||
MLK_RV64V_ABS_BOUNDS16(vl,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf,
|
||||
8, 4, 2, 2, 1, 1, 1, 1, 8, 4, 2, 2, 1, 1, 1, 1);
|
||||
|
||||
/* Reduce v0, v8 to avoid overflow */
|
||||
v0 = fq_mulq_vx(v0, MLK_RVV_MONT_R1, vl);
|
||||
v8 = fq_mulq_vx(v8, MLK_RVV_MONT_R1, vl);
|
||||
|
||||
/* absolute bounds: < 4*q */
|
||||
MLK_RV64V_ABS_BOUNDS16(vl,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4);
|
||||
|
||||
MLK_RVV_GS_BFLY_RX(v0, v8, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v1, v9, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v2, va, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v3, vb, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v4, vc, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v5, vd, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v6, ve, vt, izeta[0x01], vl);
|
||||
MLK_RVV_GS_BFLY_RX(v7, vf, vt, izeta[0x01], vl);
|
||||
|
||||
/* absolute bounds: < 8*q */
|
||||
MLK_RV64V_ABS_BOUNDS16(vl,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8);
|
||||
|
||||
__riscv_vse16_v_i16m1(&r[0x00], v0, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x10], v1, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x20], v2, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x30], v3, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x40], v4, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x50], v5, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x60], v6, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x70], v7, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x80], v8, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0x90], v9, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0xa0], va, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0xb0], vb, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0xc0], vc, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0xd0], vd, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0xe0], ve, vl);
|
||||
__riscv_vse16_v_i16m1(&r[0xf0], vf, vl);
|
||||
}
|
||||
|
||||
/* ML-KEM's middle field GF(3329)[X]/(X^2) multiplication */
|
||||
|
||||
static MLK_INLINE void mlk_rv64v_poly_basemul_mont_add_k(int16_t *r,
|
||||
const int16_t *a,
|
||||
const int16_t *b,
|
||||
unsigned kn)
|
||||
{
|
||||
#include "rv64v_zetas_basemul.inc"
|
||||
|
||||
size_t vl = __riscv_vsetvl_e16m1(MLKEM_N);
|
||||
size_t i, j;
|
||||
|
||||
const vuint16m1_t sw0 = __riscv_vxor_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl);
|
||||
const vbool16_t sb0 = __riscv_vmseq_vx_u16m1_b16(
|
||||
__riscv_vand_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl), 0, vl);
|
||||
|
||||
vint16m1_t vt, vu;
|
||||
vint32m2_t wa, wb, ws;
|
||||
|
||||
for (i = 0; i < MLKEM_N; i += vl)
|
||||
{
|
||||
const vint16m1_t vz = __riscv_vle16_v_i16m1(&roots[i], vl);
|
||||
|
||||
for (j = 0; j < kn; j += MLKEM_N)
|
||||
{
|
||||
vt = __riscv_vle16_v_i16m1(&a[i + j], vl);
|
||||
vu = __riscv_vle16_v_i16m1(&b[i + j], vl);
|
||||
|
||||
wa = __riscv_vwmul_vv_i32m2(vz, fq_mul_vv(vt, vu, vl), vl);
|
||||
wb = __riscv_vwmul_vv_i32m2(vt, __riscv_vrgather_vv_i16m1(vu, sw0, vl),
|
||||
vl);
|
||||
|
||||
wa =
|
||||
__riscv_vadd_vv_i32m2(wa, __riscv_vslidedown_vx_i32m2(wa, 1, vl), vl);
|
||||
wb = __riscv_vadd_vv_i32m2(wb, __riscv_vslideup_vx_i32m2(wb, wb, 1, vl),
|
||||
vl);
|
||||
|
||||
wa = __riscv_vmerge_vvm_i32m2(wb, wa, sb0, vl);
|
||||
|
||||
if (j == 0)
|
||||
{
|
||||
ws = wa;
|
||||
}
|
||||
else
|
||||
{
|
||||
ws = __riscv_vadd_vv_i32m2(ws, wa, vl);
|
||||
}
|
||||
}
|
||||
/* the idea is to keep 32-bit intermediate result, reduce in the end */
|
||||
__riscv_vse16_v_i16m1(&r[i], fq_redc2(ws, vl), vl);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2
|
||||
void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a,
|
||||
const int16_t *b)
|
||||
{
|
||||
mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 2 * MLKEM_N);
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3
|
||||
void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a,
|
||||
const int16_t *b)
|
||||
{
|
||||
mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 3 * MLKEM_N);
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a,
|
||||
const int16_t *b)
|
||||
{
|
||||
mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 4 * MLKEM_N);
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
/**
|
||||
* In-place conversion of all coefficients of a polynomial from the normal
|
||||
* domain to the Montgomery domain.
|
||||
*
|
||||
* @param[in,out] r Input/output polynomial.
|
||||
*/
|
||||
void mlk_rv64v_poly_tomont(int16_t *r)
|
||||
{
|
||||
size_t vl = __riscv_vsetvl_e16m1(MLKEM_N);
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < MLKEM_N; i += vl)
|
||||
{
|
||||
vint16m1_t vec = __riscv_vle16_v_i16m1(&r[i], vl);
|
||||
vec = fq_mul_vx(vec, MLK_RVV_MONT_R2, vl);
|
||||
__riscv_vse16_v_i16m1(&r[i], vec, vl);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply Barrett reduction to all coefficients of a polynomial. For details
|
||||
* of the Barrett reduction see the comments in poly.c.
|
||||
*
|
||||
* @param[in,out] r Input/output polynomial.
|
||||
*/
|
||||
void mlk_rv64v_poly_reduce(int16_t *r)
|
||||
{
|
||||
size_t vl = __riscv_vsetvl_e16m1(MLKEM_N);
|
||||
vint16m1_t vt;
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < MLKEM_N; i += vl)
|
||||
{
|
||||
vt = __riscv_vle16_v_i16m1(&r[i], vl);
|
||||
vt = fq_barrett(vt, vl);
|
||||
vt = fq_cadd(vt, vl);
|
||||
__riscv_vse16_v_i16m1(&r[i], vt, vl);
|
||||
}
|
||||
}
|
||||
|
||||
/* Run rejection sampling to get uniform random integers mod q.
|
||||
* buflen must be a multiple of 12. */
|
||||
|
||||
unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len,
|
||||
const uint8_t *buf, unsigned int buflen)
|
||||
{
|
||||
unsigned n, ctr, pos;
|
||||
vuint16m1_t x, y;
|
||||
vbool16_t lt;
|
||||
|
||||
pos = 0;
|
||||
ctr = 0;
|
||||
|
||||
while (ctr < len && pos < buflen)
|
||||
{
|
||||
const unsigned vl = (unsigned)__riscv_vsetvl_e16m1((buflen - pos) * 8 / 12);
|
||||
const unsigned vl23 = (vl * 24) / 32;
|
||||
|
||||
const vuint16m1_t vid = __riscv_vid_v_u16m1(vl);
|
||||
const vuint16m1_t srl12v = __riscv_vmul_vx_u16m1(vid, 12, vl);
|
||||
const vuint16m1_t sel12v = __riscv_vsrl_vx_u16m1(srl12v, 4, vl);
|
||||
const vuint16m1_t sll12v = __riscv_vsll_vx_u16m1(vid, 2, vl);
|
||||
|
||||
/* Functionally, this loop is not necessary, but it avoids re-evaluating
|
||||
* the VL too many times. In particular, in the first outer iteration,
|
||||
* the inner loop will process the bulk of the data with fixed VL. */
|
||||
while (ctr < len && vl23 * 2 <= buflen - pos)
|
||||
{
|
||||
x = __riscv_vle16_v_u16m1((uint16_t *)&buf[pos], vl23);
|
||||
pos += vl23 * 2;
|
||||
x = __riscv_vrgather_vv_u16m1(x, sel12v, vl);
|
||||
x = __riscv_vor_vv_u16m1(
|
||||
__riscv_vsrl_vv_u16m1(x, srl12v, vl),
|
||||
__riscv_vsll_vv_u16m1(__riscv_vslidedown(x, 1, vl), sll12v, vl), vl);
|
||||
x = __riscv_vand_vx_u16m1(x, 0xFFF, vl);
|
||||
|
||||
lt = __riscv_vmsltu_vx_u16m1_b16(x, MLKEM_Q, vl);
|
||||
y = __riscv_vcompress_vm_u16m1(x, lt, vl);
|
||||
n = (unsigned)__riscv_vcpop_m_b16(lt, vl);
|
||||
|
||||
if (ctr + n > len)
|
||||
{
|
||||
n = len - ctr;
|
||||
}
|
||||
__riscv_vse16_v_u16m1((uint16_t *)&r[ctr], y, n);
|
||||
ctr += n;
|
||||
}
|
||||
}
|
||||
|
||||
return ctr;
|
||||
}
|
||||
|
||||
#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */
|
||||
|
||||
MLK_EMPTY_CU(rv64v_poly)
|
||||
|
||||
#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */
|
||||
|
||||
/* To facilitate single-compilation-unit (SCU) builds, undefine all macros.
|
||||
* Don't modify by hand -- this is auto-generated by scripts/autogen. */
|
||||
#undef MLK_RVV_QI
|
||||
#undef MLK_RVV_MONT_R1
|
||||
#undef MLK_RVV_MONT_R2
|
||||
#undef MLK_RVV_MONT_NR
|
||||
#undef MLK_RVV_CT_BFLY_FX
|
||||
#undef MLK_RVV_CT_BFLY_FV
|
||||
#undef MLK_RVV_GS_BFLY_RX
|
||||
#undef MLK_RVV_GS_BFLY_RV
|
||||
#undef MLK_RV64V_ABS_BOUNDS16
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "arith_native_riscv64.h"
|
||||
|
||||
const int16_t zeta[] = {
|
||||
-1044, -758, 573, -1325, 1223, 652, -552, 1015, -1103, 430, 555,
|
||||
843, -1251, 871, 1550, 105, -359, -1517, 264, 383, -1293, 1491,
|
||||
-282, -1544, 422, 587, 177, -235, -291, -460, 1574, 1653, 1493,
|
||||
1422, -829, 1458, 516, -8, -320, -666, -246, 778, 1159, -147,
|
||||
-777, 1483, -602, 1119, 287, 202, -1602, -130, -1618, -1162, 126,
|
||||
1469, -1590, 644, -872, 349, 418, 329, -156, -75, -171, 622,
|
||||
-681, 1017, -853, -90, -271, 830, 817, 1097, 603, 610, 1322,
|
||||
-1285, -1465, 384, 1577, 182, 732, 608, 107, -1421, -247, -951,
|
||||
-1215, -136, 1218, -1335, -874, 220, -1187, -1659, 962, -1202, -1542,
|
||||
411, -398, 961, -1508, -725, -1185, -1530, -1278, 794, -1510, -854,
|
||||
-870, 478, -1474, 1468, -205, -1571, 448, -1065, 677, -1275, -108,
|
||||
-308, 996, 991, 958, -1460, 1522, 1628,
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "arith_native_riscv64.h"
|
||||
|
||||
const int16_t roots[] = {
|
||||
-1044, -1103, -1044, 1103, -1044, 430, -1044, -430, -1044, 555, -1044,
|
||||
-555, -1044, 843, -1044, -843, -1044, -1251, -1044, 1251, -1044, 871,
|
||||
-1044, -871, -1044, 1550, -1044, -1550, -1044, 105, -1044, -105, -1044,
|
||||
422, -1044, -422, -1044, 587, -1044, -587, -1044, 177, -1044, -177,
|
||||
-1044, -235, -1044, 235, -1044, -291, -1044, 291, -1044, -460, -1044,
|
||||
460, -1044, 1574, -1044, -1574, -1044, 1653, -1044, -1653, -1044, -246,
|
||||
-1044, 246, -1044, 778, -1044, -778, -1044, 1159, -1044, -1159, -1044,
|
||||
-147, -1044, 147, -1044, -777, -1044, 777, -1044, 1483, -1044, -1483,
|
||||
-1044, -602, -1044, 602, -1044, 1119, -1044, -1119, -1044, -1590, -1044,
|
||||
1590, -1044, 644, -1044, -644, -1044, -872, -1044, 872, -1044, 349,
|
||||
-1044, -349, -1044, 418, -1044, -418, -1044, 329, -1044, -329, -1044,
|
||||
-156, -1044, 156, -1044, -75, -1044, 75, -1044, 817, -1044, -817,
|
||||
-1044, 1097, -1044, -1097, -1044, 603, -1044, -603, -1044, 610, -1044,
|
||||
-610, -1044, 1322, -1044, -1322, -1044, -1285, -1044, 1285, -1044, -1465,
|
||||
-1044, 1465, -1044, 384, -1044, -384, -1044, -1215, -1044, 1215, -1044,
|
||||
-136, -1044, 136, -1044, 1218, -1044, -1218, -1044, -1335, -1044, 1335,
|
||||
-1044, -874, -1044, 874, -1044, 220, -1044, -220, -1044, -1187, -1044,
|
||||
1187, -1044, -1659, -1044, 1659, -1044, -1185, -1044, 1185, -1044, -1530,
|
||||
-1044, 1530, -1044, -1278, -1044, 1278, -1044, 794, -1044, -794, -1044,
|
||||
-1510, -1044, 1510, -1044, -854, -1044, 854, -1044, -870, -1044, 870,
|
||||
-1044, 478, -1044, -478, -1044, -108, -1044, 108, -1044, -308, -1044,
|
||||
308, -1044, 996, -1044, -996, -1044, 991, -1044, -991, -1044, 958,
|
||||
-1044, -958, -1044, -1460, -1044, 1460, -1044, 1522, -1044, -1522, -1044,
|
||||
1628, -1044, -1628,
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
[//]: # (SPDX-License-Identifier: CC-BY-4.0)
|
||||
|
||||
This directory contains the native x86_64 arithmetic backend for ML-KEM provided by the official [AVX2
|
||||
implementation](https://github.com/pq-crystals/kyber/tree/main/avx2) of the Kyber team.
|
||||
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_X86_64_META_H
|
||||
#define MLK_NATIVE_X86_64_META_H
|
||||
|
||||
/* Identifier for this backend so that source and assembly files
|
||||
* in the build can be appropriately guarded. */
|
||||
#define MLK_ARITH_BACKEND_X86_64_DEFAULT
|
||||
|
||||
#define MLK_USE_NATIVE_NTT_CUSTOM_ORDER
|
||||
#define MLK_USE_NATIVE_REJ_UNIFORM
|
||||
#define MLK_USE_NATIVE_NTT
|
||||
#define MLK_USE_NATIVE_INTT
|
||||
#define MLK_USE_NATIVE_POLY_REDUCE
|
||||
#define MLK_USE_NATIVE_POLY_TOMONT
|
||||
#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED
|
||||
#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE
|
||||
#define MLK_USE_NATIVE_POLY_TOBYTES
|
||||
#define MLK_USE_NATIVE_POLY_FROMBYTES
|
||||
#define MLK_USE_NATIVE_POLY_COMPRESS_D4
|
||||
#define MLK_USE_NATIVE_POLY_COMPRESS_D5
|
||||
#define MLK_USE_NATIVE_POLY_COMPRESS_D10
|
||||
#define MLK_USE_NATIVE_POLY_COMPRESS_D11
|
||||
#define MLK_USE_NATIVE_POLY_DECOMPRESS_D4
|
||||
#define MLK_USE_NATIVE_POLY_DECOMPRESS_D5
|
||||
#define MLK_USE_NATIVE_POLY_DECOMPRESS_D10
|
||||
#define MLK_USE_NATIVE_POLY_DECOMPRESS_D11
|
||||
|
||||
#if !defined(__ASSEMBLER__)
|
||||
#include "../../common.h"
|
||||
#include "../api.h"
|
||||
#include "src/arith_native_x86_64.h"
|
||||
#include "src/compress_consts.h"
|
||||
|
||||
static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t data[MLKEM_N])
|
||||
{
|
||||
if (mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
mlk_nttunpack_avx2_asm(data);
|
||||
}
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len,
|
||||
const uint8_t *buf,
|
||||
unsigned buflen)
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2) || len != MLKEM_N ||
|
||||
buflen % 12 != 0)
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
return (int)mlk_rej_uniform_avx2_asm(r, buf, buflen, mlk_rej_uniform_table);
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_ntt_avx2_asm(data, mlk_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_invntt_avx2_asm(data, mlk_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_reduce_avx2_asm(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_tomont_avx2_asm(data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2],
|
||||
const int16_t y[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_mulcache_compute_avx2_asm(x, y, mlk_qdata);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N],
|
||||
const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(r, a, b, b_cache);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N],
|
||||
const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(r, a, b, b_cache);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native(
|
||||
int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N],
|
||||
const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(r, a, b, b_cache);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES],
|
||||
const int16_t a[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_ntttobytes_avx2_asm(r, a);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_frombytes_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYBYTES])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_nttfrombytes_avx2_asm(r, a);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3)
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d4_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_compress_d4_avx2_asm(r, a, mlk_compress_d4_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d10_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_compress_d10_avx2_asm(r, a, mlk_compress_d10_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d4_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_decompress_d4_avx2_asm(r, a, mlk_decompress_d4_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d10_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_decompress_d10_avx2_asm(r, a, mlk_decompress_d10_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */
|
||||
|
||||
#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d5_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_compress_d5_avx2_asm(r, a, mlk_compress_d5_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_compress_d11_native(
|
||||
uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_compress_d11_avx2_asm(r, a, mlk_compress_d11_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d5_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_decompress_d5_avx2_asm(r, a, mlk_decompress_d5_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
|
||||
MLK_MUST_CHECK_RETURN_VALUE
|
||||
static MLK_INLINE int mlk_poly_decompress_d11_native(
|
||||
int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11])
|
||||
{
|
||||
if (!mlk_sys_check_capability(MLK_SYS_CAP_AVX2))
|
||||
{
|
||||
return MLK_NATIVE_FUNC_FALLBACK;
|
||||
}
|
||||
|
||||
mlk_poly_decompress_d11_avx2_asm(r, a, mlk_decompress_d11_data);
|
||||
return MLK_NATIVE_FUNC_SUCCESS;
|
||||
}
|
||||
#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_NATIVE_X86_64_META_H */
|
||||
@@ -0,0 +1,327 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
#ifndef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H
|
||||
#define MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include "compress_consts.h"
|
||||
#include "consts.h"
|
||||
|
||||
#define MLK_AVX2_REJ_UNIFORM_BUFLEN \
|
||||
(3 * 168) /* REJ_UNIFORM_NBLOCKS * SHAKE128_RATE */
|
||||
|
||||
#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096];
|
||||
|
||||
#define mlk_rej_uniform_avx2_asm MLK_NAMESPACE(rej_uniform_avx2_asm)
|
||||
MLK_MUST_CHECK_RETURN_VALUE MLK_SYSV_ABI
|
||||
uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf,
|
||||
unsigned buflen, const uint8_t *table)
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/rej_uniform_avx2_asm.ml. */
|
||||
__contract__(
|
||||
requires(buflen % 12 == 0)
|
||||
requires(memory_no_alias(buf, buflen))
|
||||
requires(table == mlk_rej_uniform_table)
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(return_value <= MLKEM_N)
|
||||
ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_ntt_avx2_asm MLK_NAMESPACE(ntt_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata)
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/ntt_avx2_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_abs_bound(r, 0, MLKEM_N, 8192))
|
||||
requires(qdata == mlk_qdata)
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
/* check-magic: off */
|
||||
ensures(array_abs_bound(r, 0, MLKEM_N, 23595))
|
||||
/* check-magic: on */
|
||||
);
|
||||
|
||||
#define mlk_invntt_avx2_asm MLK_NAMESPACE(invntt_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata)
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/intt_avx2_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(qdata == mlk_qdata)
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
/* check-magic: off */
|
||||
ensures(array_abs_bound(r, 0, MLKEM_N, 26632))
|
||||
/* check-magic: on */
|
||||
);
|
||||
|
||||
#define mlk_nttunpack_avx2_asm MLK_NAMESPACE(nttunpack_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_nttunpack_avx2_asm(int16_t *r)
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/nttunpack_avx2_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
/* Output is a permutation of input: every output coefficient
|
||||
* is some input coefficient */
|
||||
ensures(forall(i, 0, MLKEM_N, exists(j, 0, MLKEM_N,
|
||||
r[i] == old(*(int16_t (*)[MLKEM_N])r)[j])))
|
||||
);
|
||||
|
||||
#define mlk_reduce_avx2_asm MLK_NAMESPACE(reduce_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_reduce_avx2_asm(int16_t *r)
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/reduce_avx2_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_mulcache_compute_avx2_asm \
|
||||
MLK_NAMESPACE(poly_mulcache_compute_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in,
|
||||
const int16_t *qdata)
|
||||
/* This must be kept in sync with the HOL-Light specification
|
||||
* in proofs/hol_light/x86_64/proofs/poly_mulcache_compute_avx2_asm.ml */
|
||||
__contract__(
|
||||
requires(memory_no_alias(out, sizeof(int16_t) * (MLKEM_N / 2)))
|
||||
requires(memory_no_alias(in, sizeof(int16_t) * MLKEM_N))
|
||||
requires(qdata == mlk_qdata)
|
||||
assigns(memory_slice(out, sizeof(int16_t) * (MLKEM_N / 2)))
|
||||
ensures(array_abs_bound(out, 0, MLKEM_N/2, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm \
|
||||
MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(
|
||||
int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2)))
|
||||
requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
);
|
||||
|
||||
#define mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm \
|
||||
MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(
|
||||
int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2)))
|
||||
requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
);
|
||||
|
||||
#define mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm \
|
||||
MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(
|
||||
int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N))
|
||||
requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N))
|
||||
requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2)))
|
||||
requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
);
|
||||
|
||||
#define mlk_ntttobytes_avx2_asm MLK_NAMESPACE(ntttobytes_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/ntttobytes_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
assigns(memory_slice(r, MLKEM_POLYBYTES))
|
||||
);
|
||||
|
||||
#define mlk_nttfrombytes_avx2_asm MLK_NAMESPACE(nttfrombytes_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/nttfrombytes_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(a, MLKEM_POLYBYTES))
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))
|
||||
);
|
||||
|
||||
#define mlk_tomont_avx2_asm MLK_NAMESPACE(tomont_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_tomont_avx2_asm(int16_t *r)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/tomont_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_compress_d4_avx2_asm MLK_NAMESPACE(poly_compress_d4_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_compress_d4_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4],
|
||||
const int16_t *MLK_RESTRICT a,
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_compress_d4_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
requires(data == mlk_compress_d4_data)
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
);
|
||||
|
||||
#define mlk_poly_decompress_d4_avx2_asm \
|
||||
MLK_NAMESPACE(poly_decompress_d4_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_decompress_d4_avx2_asm(
|
||||
int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4],
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_decompress_d4_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4))
|
||||
requires(data == mlk_decompress_d4_data)
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_compress_d10_avx2_asm MLK_NAMESPACE(poly_compress_d10_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_compress_d10_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10],
|
||||
const int16_t *MLK_RESTRICT a,
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_compress_d10_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
requires(data == mlk_compress_d10_data)
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
);
|
||||
|
||||
#define mlk_poly_decompress_d10_avx2_asm \
|
||||
MLK_NAMESPACE(poly_decompress_d10_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_decompress_d10_avx2_asm(
|
||||
int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10],
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_decompress_d10_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10))
|
||||
requires(data == mlk_decompress_d10_data)
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_compress_d5_avx2_asm MLK_NAMESPACE(poly_compress_d5_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_compress_d5_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5],
|
||||
const int16_t *MLK_RESTRICT a,
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_compress_d5_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
requires(data == mlk_compress_d5_data)
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
);
|
||||
|
||||
#define mlk_poly_decompress_d5_avx2_asm \
|
||||
MLK_NAMESPACE(poly_decompress_d5_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_decompress_d5_avx2_asm(
|
||||
int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5],
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_decompress_d5_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5))
|
||||
requires(data == mlk_decompress_d5_data)
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#define mlk_poly_compress_d11_avx2_asm MLK_NAMESPACE(poly_compress_d11_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_compress_d11_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11],
|
||||
const int16_t *MLK_RESTRICT a,
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_compress_d11_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N))
|
||||
requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
requires(data == mlk_compress_d11_data)
|
||||
assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
);
|
||||
|
||||
#define mlk_poly_decompress_d11_avx2_asm \
|
||||
MLK_NAMESPACE(poly_decompress_d11_avx2_asm)
|
||||
MLK_SYSV_ABI
|
||||
void mlk_poly_decompress_d11_avx2_asm(
|
||||
int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11],
|
||||
const uint8_t *data)
|
||||
/* This must be kept in sync with the HOL-Light specification in
|
||||
* proofs/hol_light/x86_64/proofs/poly_decompress_d11_avx2_asm.ml.
|
||||
*/
|
||||
__contract__(
|
||||
requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N))
|
||||
requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11))
|
||||
requires(data == mlk_decompress_d11_data)
|
||||
assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N))
|
||||
ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))
|
||||
);
|
||||
|
||||
#endif /* !MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H */
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT)
|
||||
|
||||
#include "compress_consts.h"
|
||||
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \
|
||||
(defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \
|
||||
MLKEM_K == 3)
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_compress_d4_data[32] = {
|
||||
0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0,
|
||||
2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, /* permdidx */
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_decompress_d4_data[32] = {
|
||||
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
|
||||
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, /* shufbidx */
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_compress_d10_data[32] = {
|
||||
0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 255,
|
||||
255, 255, 255, 255, 255, 9, 10, 11, 12, 255, 255,
|
||||
255, 255, 255, 255, 0, 1, 2, 3, 4, 8, /* shufbidx */
|
||||
};
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_decompress_d10_data[32] = {
|
||||
0, 1, 1, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 9,
|
||||
2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, /* shufbidx */
|
||||
};
|
||||
|
||||
#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
(MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \
|
||||
*/
|
||||
|
||||
#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \
|
||||
(defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4)
|
||||
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_compress_d5_data[32] = {
|
||||
0, 1, 2, 3, 4, 255, 255, 255, 255, 255, 8,
|
||||
9, 10, 11, 12, 255, 9, 10, 11, 12, 255, 0,
|
||||
1, 2, 3, 4, 255, 255, 255, 255, 255, 8, /* shufbidx */
|
||||
};
|
||||
|
||||
/* shufbidx[0:32], mask[32:64], shift[64:96] */
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_decompress_d5_data[96] = {
|
||||
0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5,
|
||||
5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, /* shufbidx */
|
||||
31, 0, 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, 31, 0,
|
||||
224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, /* mask */
|
||||
0, 4, 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, 0, 4,
|
||||
32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, /* shift */
|
||||
};
|
||||
|
||||
/* srlvqidx[0:32], shufbidx[32:64] */
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_compress_d11_data[64] =
|
||||
{
|
||||
10, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0,
|
||||
0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0,
|
||||
0, 0, 30, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
|
||||
255, 255, 255, 255, 255, 5, 6, 7, 8, 9, 10,
|
||||
255, 255, 255, 255, 0, 0, 1, 2, 3, 4, /* shufbidx */
|
||||
};
|
||||
|
||||
/* shufbidx[0:32], srlvdidx[32:64], srlvqidx[64:96], shift[96:128] */
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t
|
||||
mlk_decompress_d11_data[128] = {
|
||||
0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10,
|
||||
3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 11, 12, 12, 13, /* shufbidx */
|
||||
0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* srlvdidx */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */
|
||||
32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0,
|
||||
32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, /* shift */
|
||||
};
|
||||
|
||||
#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \
|
||||
(MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */
|
||||
|
||||
#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT */
|
||||
|
||||
MLK_EMPTY_CU(avx2_compress_consts)
|
||||
|
||||
#endif /* !MLK_ARITH_BACKEND_X86_64_DEFAULT */
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H
|
||||
#define MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
|
||||
#define mlk_compress_d4_data MLK_NAMESPACE(compress_d4_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d4_data[32];
|
||||
|
||||
#define mlk_decompress_d4_data MLK_NAMESPACE(decompress_d4_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d4_data[32];
|
||||
|
||||
#define mlk_compress_d10_data MLK_NAMESPACE(compress_d10_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d10_data[32];
|
||||
|
||||
#define mlk_decompress_d10_data MLK_NAMESPACE(decompress_d10_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d10_data[32];
|
||||
|
||||
#define mlk_compress_d5_data MLK_NAMESPACE(compress_d5_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d5_data[32];
|
||||
|
||||
#define mlk_decompress_d5_data MLK_NAMESPACE(decompress_d5_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d5_data[96];
|
||||
|
||||
#define mlk_compress_d11_data MLK_NAMESPACE(compress_d11_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d11_data[64];
|
||||
|
||||
#define mlk_decompress_d11_data MLK_NAMESPACE(decompress_d11_data)
|
||||
MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d11_data[128];
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
#endif /* !MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H */
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
|
||||
#include "consts.h"
|
||||
|
||||
/*
|
||||
* Table of zeta values used in the AVX2 NTTs
|
||||
* See autogen for details.
|
||||
*/
|
||||
MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_qdata[624] = {
|
||||
3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 3854,
|
||||
3340, 2826, 2312, 1798, 1284, 770, 256, 7, 0,
|
||||
6, 0, 5, 0, 4, 0, 3, 0, 2,
|
||||
0, 1, 0, 0, 0, 31498, 31498, 31498, 31498,
|
||||
-758, -758, -758, -758, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 14745, 14745, 14745, 14745, 14745, 14745,
|
||||
14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745,
|
||||
14745, -359, -359, -359, -359, -359, -359, -359, -359,
|
||||
-359, -359, -359, -359, -359, -359, -359, -359, 13525,
|
||||
13525, 13525, 13525, 13525, 13525, 13525, 13525, -12402, -12402,
|
||||
-12402, -12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493,
|
||||
1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422,
|
||||
1422, 1422, 1422, 1422, -20907, -20907, -20907, -20907, 27758,
|
||||
27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690,
|
||||
-15690, -15690, -171, -171, -171, -171, 622, 622, 622,
|
||||
622, 1577, 1577, 1577, 1577, 182, 182, 182, 182,
|
||||
-5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057, 5571,
|
||||
5571, -1102, -1102, 21438, 21438, -26242, -26242, 573, 573,
|
||||
-1325, -1325, 264, 264, 383, 383, -829, -829, 1458,
|
||||
1458, -1602, -1602, -130, -130, -5689, -6516, 1496, 30967,
|
||||
-23565, 20179, 20710, 25080, -12796, 26616, 16064, -12442, 9134,
|
||||
-650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491,
|
||||
-282, -1544, 516, -8, -320, -666, -1618, -1162, 126,
|
||||
1469, -335, -11477, -32227, 20494, -27738, 945, -14883, 6182,
|
||||
32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103,
|
||||
555, -1251, 1550, 422, 177, -291, 1574, -246, 1159,
|
||||
-777, -602, -1590, -872, 418, -156, 11182, 13387, -14233,
|
||||
-21655, 13131, -4587, 23092, 5493, -32502, 30317, -18741, 12639,
|
||||
20100, 18525, 19529, -12619, 430, 843, 871, 105, 587,
|
||||
-235, -460, 1653, 778, -147, 1483, 1119, 644, 349,
|
||||
329, -75, 787, 787, 787, 787, 787, 787, 787,
|
||||
787, 787, 787, 787, 787, 787, 787, 787, 787,
|
||||
-1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517,
|
||||
-1517, -1517, -1517, -1517, -1517, -1517, -1517, 28191, 28191,
|
||||
28191, 28191, 28191, 28191, 28191, 28191, -16694, -16694, -16694,
|
||||
-16694, -16694, -16694, -16694, -16694, 287, 287, 287, 287,
|
||||
287, 287, 287, 287, 202, 202, 202, 202, 202,
|
||||
202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358,
|
||||
1358, 1358, -11202, -11202, -11202, -11202, 31164, 31164, 31164,
|
||||
31164, 962, 962, 962, 962, -1202, -1202, -1202, -1202,
|
||||
-1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073,
|
||||
-28073, 24313, 24313, -10532, -10532, 8800, 8800, 18426, 18426,
|
||||
8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017,
|
||||
1017, 732, 732, 608, 608, -1542, -1542, 411, 411,
|
||||
-205, -205, -1571, -1571, 19883, -28250, -15887, -8898, -28309,
|
||||
9075, -30199, 18249, 13426, 14017, -29156, -12757, 16832, 4311,
|
||||
-24155, -17915, -853, -90, -271, 830, 107, -1421, -247,
|
||||
-951, -398, 961, -1508, -725, 448, -1065, 677, -1275,
|
||||
-31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335,
|
||||
-7934, -22502, 10906, 31636, 28644, 23998, -17422, 817, 603,
|
||||
1322, -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510,
|
||||
-870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384,
|
||||
-6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860,
|
||||
-20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335,
|
||||
220, -1659, -1530, 794, -854, 478, -308, 991, -1460,
|
||||
1628, -1103, 555, -1251, 1550, 422, 177, -291, 1574,
|
||||
-246, 1159, -777, -602, -1590, -872, 418, -156, 430,
|
||||
843, 871, 105, 587, -235, -460, 1653, 778, -147,
|
||||
1483, 1119, 644, 349, 329, -75, 817, 603, 1322,
|
||||
-1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, -870,
|
||||
-108, 996, 958, 1522, 1097, 610, -1285, 384, -136,
|
||||
-1335, 220, -1659, -1530, 794, -854, 478, -308, 991,
|
||||
-1460, 1628, -335, -11477, -32227, 20494, -27738, 945, -14883,
|
||||
6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276,
|
||||
11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493, -32502,
|
||||
30317, -18741, 12639, 20100, 18525, 19529, -12619, -31183, 25435,
|
||||
-7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502,
|
||||
10906, 31636, 28644, 23998, -17422, 20297, 2146, 15355, -32384,
|
||||
-6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860,
|
||||
-20257, 7756, 23132,
|
||||
};
|
||||
|
||||
#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
MLK_EMPTY_CU(avx2_consts)
|
||||
|
||||
#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \
|
||||
!MLK_CONFIG_MULTILEVEL_NO_SHARED) */
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-generated from scripts/autogen
|
||||
* in the mlkem-native repository.
|
||||
* Do not modify it directly.
|
||||
*/
|
||||
|
||||
#ifndef MLK_NATIVE_X86_64_SRC_CONSTS_H
|
||||
#define MLK_NATIVE_X86_64_SRC_CONSTS_H
|
||||
#include "../../../common.h"
|
||||
#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB 0
|
||||
#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD 16
|
||||
#define MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP 32
|
||||
#define MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES 496
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
#define mlk_qdata MLK_NAMESPACE(qdata)
|
||||
MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_qdata[624];
|
||||
#endif
|
||||
|
||||
#endif /* !MLK_NATIVE_X86_64_SRC_CONSTS_H */
|
||||
@@ -0,0 +1,741 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [AVX2_NTT]
|
||||
* Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography.
|
||||
* Gregor Seiler
|
||||
* https://eprint.iacr.org/2018/039
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*
|
||||
* The core ideas behind the implementation are described in @[AVX2_NTT].
|
||||
*
|
||||
* Changes:
|
||||
* - Different placement of modular reductions to simplify
|
||||
* reasoning of non-overflow
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
/*yaml
|
||||
Name: invntt_avx2_asm
|
||||
Description: x86_64 AVX2 inverse NTT
|
||||
Signature: void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t *r
|
||||
description: Input/output polynomial (256 x int16_t)
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 1248
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *qdata
|
||||
description: Precomputed constants (624 x int16_t)
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/intt_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(invntt_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(invntt_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0xd010d01, %eax # imm = 0xD010D01
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1
|
||||
vmovd %eax, %xmm2
|
||||
vpbroadcastd %xmm2, %ymm2
|
||||
movl $0x5a105a1, %eax # imm = 0x5A105A1
|
||||
vmovd %eax, %xmm3
|
||||
vpbroadcastd %xmm3, %ymm3
|
||||
vmovdqa (%rdi), %ymm4
|
||||
vmovdqa 0x40(%rdi), %ymm6
|
||||
vmovdqa 0x20(%rdi), %ymm5
|
||||
vmovdqa 0x60(%rdi), %ymm7
|
||||
vpmullw %ymm2, %ymm4, %ymm12
|
||||
vpmulhw %ymm3, %ymm4, %ymm4
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm4, %ymm4
|
||||
vpmullw %ymm2, %ymm6, %ymm12
|
||||
vpmulhw %ymm3, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm6, %ymm6
|
||||
vpmullw %ymm2, %ymm5, %ymm12
|
||||
vpmulhw %ymm3, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm5, %ymm5
|
||||
vpmullw %ymm2, %ymm7, %ymm12
|
||||
vpmulhw %ymm3, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm7, %ymm7
|
||||
vmovdqa 0x80(%rdi), %ymm8
|
||||
vmovdqa 0xc0(%rdi), %ymm10
|
||||
vmovdqa 0xa0(%rdi), %ymm9
|
||||
vmovdqa 0xe0(%rdi), %ymm11
|
||||
vpmullw %ymm2, %ymm8, %ymm12
|
||||
vpmulhw %ymm3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm8, %ymm8
|
||||
vpmullw %ymm2, %ymm10, %ymm12
|
||||
vpmulhw %ymm3, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm10, %ymm10
|
||||
vpmullw %ymm2, %ymm9, %ymm12
|
||||
vpmulhw %ymm3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm9, %ymm9
|
||||
vpmullw %ymm2, %ymm11, %ymm12
|
||||
vpmulhw %ymm3, %ymm11, %ymm11
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm11, %ymm11
|
||||
vpermq $0x4e, 0x3a0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x360(%rsi), %ymm1 # ymm1 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x3c0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x380(%rsi), %ymm3 # ymm3 = mem[2,3,0,1]
|
||||
vmovdqa (%rsi), %ymm12
|
||||
vpshufb %ymm12, %ymm15, %ymm15
|
||||
vpshufb %ymm12, %ymm1, %ymm1
|
||||
vpshufb %ymm12, %ymm2, %ymm2
|
||||
vpshufb %ymm12, %ymm3, %ymm3
|
||||
vpsubw %ymm4, %ymm6, %ymm12
|
||||
vpaddw %ymm6, %ymm4, %ymm4
|
||||
vpsubw %ymm5, %ymm7, %ymm13
|
||||
vpmullw %ymm15, %ymm12, %ymm6
|
||||
vpaddw %ymm7, %ymm5, %ymm5
|
||||
vpsubw %ymm8, %ymm10, %ymm14
|
||||
vpmullw %ymm15, %ymm13, %ymm7
|
||||
vpaddw %ymm10, %ymm8, %ymm8
|
||||
vpsubw %ymm9, %ymm11, %ymm15
|
||||
vpmullw %ymm1, %ymm14, %ymm10
|
||||
vpaddw %ymm11, %ymm9, %ymm9
|
||||
vpmullw %ymm1, %ymm15, %ymm11
|
||||
vpmulhw %ymm2, %ymm12, %ymm12
|
||||
vpmulhw %ymm2, %ymm13, %ymm13
|
||||
vpmulhw %ymm3, %ymm14, %ymm14
|
||||
vpmulhw %ymm3, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm6, %ymm12, %ymm6
|
||||
vpsubw %ymm7, %ymm13, %ymm7
|
||||
vpsubw %ymm10, %ymm14, %ymm10
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpermq $0x4e, 0x320(%rsi), %ymm2 # ymm2 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x340(%rsi), %ymm3 # ymm3 = mem[2,3,0,1]
|
||||
vmovdqa (%rsi), %ymm1
|
||||
vpshufb %ymm1, %ymm2, %ymm2
|
||||
vpshufb %ymm1, %ymm3, %ymm3
|
||||
vpsubw %ymm4, %ymm8, %ymm12
|
||||
vpaddw %ymm8, %ymm4, %ymm4
|
||||
vpsubw %ymm5, %ymm9, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm5
|
||||
vpsubw %ymm6, %ymm10, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm6
|
||||
vpsubw %ymm7, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm7
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm3, %ymm12, %ymm12
|
||||
vpmulhw %ymm3, %ymm13, %ymm13
|
||||
vpmulhw %ymm3, %ymm14, %ymm14
|
||||
vpmulhw %ymm3, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm8, %ymm12, %ymm8
|
||||
vpsubw %ymm9, %ymm13, %ymm9
|
||||
vpsubw %ymm10, %ymm14, %ymm10
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpslld $0x10, %ymm5, %ymm3
|
||||
vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
|
||||
vpslld $0x10, %ymm7, %ymm4
|
||||
vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
|
||||
vpslld $0x10, %ymm9, %ymm6
|
||||
vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15]
|
||||
vpsrld $0x10, %ymm8, %ymm8
|
||||
vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
|
||||
vpslld $0x10, %ymm11, %ymm8
|
||||
vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15]
|
||||
vpsrld $0x10, %ymm10, %ymm10
|
||||
vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
|
||||
vmovdqa 0x20(%rsi), %ymm12
|
||||
vpermd 0x2e0(%rsi), %ymm12, %ymm2
|
||||
vpermd 0x300(%rsi), %ymm12, %ymm10
|
||||
vpsubw %ymm3, %ymm5, %ymm12
|
||||
vpaddw %ymm5, %ymm3, %ymm3
|
||||
vpsubw %ymm4, %ymm7, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm5
|
||||
vpaddw %ymm7, %ymm4, %ymm4
|
||||
vpsubw %ymm6, %ymm9, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm7
|
||||
vpaddw %ymm9, %ymm6, %ymm6
|
||||
vpsubw %ymm8, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm9
|
||||
vpaddw %ymm11, %ymm8, %ymm8
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm10, %ymm12, %ymm12
|
||||
vpmulhw %ymm10, %ymm13, %ymm13
|
||||
vpmulhw %ymm10, %ymm14, %ymm14
|
||||
vpmulhw %ymm10, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm5, %ymm12, %ymm5
|
||||
vpsubw %ymm7, %ymm13, %ymm7
|
||||
vpsubw %ymm9, %ymm14, %ymm9
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF
|
||||
vmovd %eax, %xmm1
|
||||
vpbroadcastd %xmm1, %ymm1
|
||||
vpmulhw %ymm1, %ymm3, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7]
|
||||
vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7]
|
||||
vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm5, %ymm5
|
||||
vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7]
|
||||
vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7]
|
||||
vpsrlq $0x20, %ymm9, %ymm9
|
||||
vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7]
|
||||
vpermq $0x1b, 0x2a0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0]
|
||||
vpermq $0x1b, 0x2c0(%rsi), %ymm9 # ymm9 = mem[3,2,1,0]
|
||||
vpsubw %ymm10, %ymm4, %ymm12
|
||||
vpaddw %ymm4, %ymm10, %ymm10
|
||||
vpsubw %ymm3, %ymm8, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm4
|
||||
vpaddw %ymm8, %ymm3, %ymm3
|
||||
vpsubw %ymm6, %ymm7, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm8
|
||||
vpaddw %ymm7, %ymm6, %ymm6
|
||||
vpsubw %ymm5, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm7
|
||||
vpaddw %ymm11, %ymm5, %ymm5
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm9, %ymm12, %ymm12
|
||||
vpmulhw %ymm9, %ymm13, %ymm13
|
||||
vpmulhw %ymm9, %ymm14, %ymm14
|
||||
vpmulhw %ymm9, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm4, %ymm4
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm4, %ymm12, %ymm4
|
||||
vpsubw %ymm8, %ymm13, %ymm8
|
||||
vpsubw %ymm7, %ymm14, %ymm7
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpmulhw %ymm1, %ymm10, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm10, %ymm10
|
||||
vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3]
|
||||
vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
|
||||
vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2]
|
||||
vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3]
|
||||
vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2]
|
||||
vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3]
|
||||
vpermq $0x4e, 0x260(%rsi), %ymm2 # ymm2 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x280(%rsi), %ymm7 # ymm7 = mem[2,3,0,1]
|
||||
vpsubw %ymm9, %ymm3, %ymm12
|
||||
vpaddw %ymm3, %ymm9, %ymm9
|
||||
vpsubw %ymm10, %ymm5, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm3
|
||||
vpaddw %ymm5, %ymm10, %ymm10
|
||||
vpsubw %ymm6, %ymm8, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm5
|
||||
vpaddw %ymm8, %ymm6, %ymm6
|
||||
vpsubw %ymm4, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm8
|
||||
vpaddw %ymm11, %ymm4, %ymm4
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm7, %ymm12, %ymm12
|
||||
vpmulhw %ymm7, %ymm13, %ymm13
|
||||
vpmulhw %ymm7, %ymm14, %ymm14
|
||||
vpmulhw %ymm7, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm3, %ymm3
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm3, %ymm12, %ymm3
|
||||
vpsubw %ymm5, %ymm13, %ymm5
|
||||
vpsubw %ymm8, %ymm14, %ymm8
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpmulhw %ymm1, %ymm9, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm9, %ymm9
|
||||
vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1]
|
||||
vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3]
|
||||
vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3]
|
||||
vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1]
|
||||
vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3]
|
||||
vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1]
|
||||
vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3]
|
||||
vmovdqa 0x220(%rsi), %ymm2
|
||||
vmovdqa 0x240(%rsi), %ymm8
|
||||
vpsubw %ymm7, %ymm10, %ymm12
|
||||
vpaddw %ymm10, %ymm7, %ymm7
|
||||
vpsubw %ymm9, %ymm4, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm10
|
||||
vpaddw %ymm4, %ymm9, %ymm9
|
||||
vpsubw %ymm6, %ymm5, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm4
|
||||
vpaddw %ymm5, %ymm6, %ymm6
|
||||
vpsubw %ymm3, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm5
|
||||
vpaddw %ymm11, %ymm3, %ymm3
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm8, %ymm12, %ymm12
|
||||
vpmulhw %ymm8, %ymm13, %ymm13
|
||||
vpmulhw %ymm8, %ymm14, %ymm14
|
||||
vpmulhw %ymm8, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm4, %ymm4
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm10, %ymm12, %ymm10
|
||||
vpsubw %ymm4, %ymm13, %ymm4
|
||||
vpsubw %ymm5, %ymm14, %ymm5
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpmulhw %ymm1, %ymm7, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm7, %ymm7
|
||||
vmovdqa %ymm7, (%rdi)
|
||||
vmovdqa %ymm9, 0x20(%rdi)
|
||||
vmovdqa %ymm6, 0x40(%rdi)
|
||||
vmovdqa %ymm3, 0x60(%rdi)
|
||||
vmovdqa %ymm10, 0x80(%rdi)
|
||||
vmovdqa %ymm4, 0xa0(%rdi)
|
||||
vmovdqa %ymm5, 0xc0(%rdi)
|
||||
vmovdqa %ymm11, 0xe0(%rdi)
|
||||
movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1
|
||||
vmovd %eax, %xmm2
|
||||
vpbroadcastd %xmm2, %ymm2
|
||||
movl $0x5a105a1, %eax # imm = 0x5A105A1
|
||||
vmovd %eax, %xmm3
|
||||
vpbroadcastd %xmm3, %ymm3
|
||||
vmovdqa 0x100(%rdi), %ymm4
|
||||
vmovdqa 0x140(%rdi), %ymm6
|
||||
vmovdqa 0x120(%rdi), %ymm5
|
||||
vmovdqa 0x160(%rdi), %ymm7
|
||||
vpmullw %ymm2, %ymm4, %ymm12
|
||||
vpmulhw %ymm3, %ymm4, %ymm4
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm4, %ymm4
|
||||
vpmullw %ymm2, %ymm6, %ymm12
|
||||
vpmulhw %ymm3, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm6, %ymm6
|
||||
vpmullw %ymm2, %ymm5, %ymm12
|
||||
vpmulhw %ymm3, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm5, %ymm5
|
||||
vpmullw %ymm2, %ymm7, %ymm12
|
||||
vpmulhw %ymm3, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm7, %ymm7
|
||||
vmovdqa 0x180(%rdi), %ymm8
|
||||
vmovdqa 0x1c0(%rdi), %ymm10
|
||||
vmovdqa 0x1a0(%rdi), %ymm9
|
||||
vmovdqa 0x1e0(%rdi), %ymm11
|
||||
vpmullw %ymm2, %ymm8, %ymm12
|
||||
vpmulhw %ymm3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm8, %ymm8
|
||||
vpmullw %ymm2, %ymm10, %ymm12
|
||||
vpmulhw %ymm3, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm10, %ymm10
|
||||
vpmullw %ymm2, %ymm9, %ymm12
|
||||
vpmulhw %ymm3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm9, %ymm9
|
||||
vpmullw %ymm2, %ymm11, %ymm12
|
||||
vpmulhw %ymm3, %ymm11, %ymm11
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm11, %ymm11
|
||||
vpermq $0x4e, 0x1e0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x1a0(%rsi), %ymm1 # ymm1 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x200(%rsi), %ymm2 # ymm2 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x1c0(%rsi), %ymm3 # ymm3 = mem[2,3,0,1]
|
||||
vmovdqa (%rsi), %ymm12
|
||||
vpshufb %ymm12, %ymm15, %ymm15
|
||||
vpshufb %ymm12, %ymm1, %ymm1
|
||||
vpshufb %ymm12, %ymm2, %ymm2
|
||||
vpshufb %ymm12, %ymm3, %ymm3
|
||||
vpsubw %ymm4, %ymm6, %ymm12
|
||||
vpaddw %ymm6, %ymm4, %ymm4
|
||||
vpsubw %ymm5, %ymm7, %ymm13
|
||||
vpmullw %ymm15, %ymm12, %ymm6
|
||||
vpaddw %ymm7, %ymm5, %ymm5
|
||||
vpsubw %ymm8, %ymm10, %ymm14
|
||||
vpmullw %ymm15, %ymm13, %ymm7
|
||||
vpaddw %ymm10, %ymm8, %ymm8
|
||||
vpsubw %ymm9, %ymm11, %ymm15
|
||||
vpmullw %ymm1, %ymm14, %ymm10
|
||||
vpaddw %ymm11, %ymm9, %ymm9
|
||||
vpmullw %ymm1, %ymm15, %ymm11
|
||||
vpmulhw %ymm2, %ymm12, %ymm12
|
||||
vpmulhw %ymm2, %ymm13, %ymm13
|
||||
vpmulhw %ymm3, %ymm14, %ymm14
|
||||
vpmulhw %ymm3, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm6, %ymm12, %ymm6
|
||||
vpsubw %ymm7, %ymm13, %ymm7
|
||||
vpsubw %ymm10, %ymm14, %ymm10
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpermq $0x4e, 0x160(%rsi), %ymm2 # ymm2 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0x180(%rsi), %ymm3 # ymm3 = mem[2,3,0,1]
|
||||
vmovdqa (%rsi), %ymm1
|
||||
vpshufb %ymm1, %ymm2, %ymm2
|
||||
vpshufb %ymm1, %ymm3, %ymm3
|
||||
vpsubw %ymm4, %ymm8, %ymm12
|
||||
vpaddw %ymm8, %ymm4, %ymm4
|
||||
vpsubw %ymm5, %ymm9, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm5
|
||||
vpsubw %ymm6, %ymm10, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm6
|
||||
vpsubw %ymm7, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm7
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm3, %ymm12, %ymm12
|
||||
vpmulhw %ymm3, %ymm13, %ymm13
|
||||
vpmulhw %ymm3, %ymm14, %ymm14
|
||||
vpmulhw %ymm3, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm8, %ymm12, %ymm8
|
||||
vpsubw %ymm9, %ymm13, %ymm9
|
||||
vpsubw %ymm10, %ymm14, %ymm10
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpslld $0x10, %ymm5, %ymm3
|
||||
vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
|
||||
vpslld $0x10, %ymm7, %ymm4
|
||||
vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
|
||||
vpslld $0x10, %ymm9, %ymm6
|
||||
vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15]
|
||||
vpsrld $0x10, %ymm8, %ymm8
|
||||
vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
|
||||
vpslld $0x10, %ymm11, %ymm8
|
||||
vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15]
|
||||
vpsrld $0x10, %ymm10, %ymm10
|
||||
vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
|
||||
vmovdqa 0x20(%rsi), %ymm12
|
||||
vpermd 0x120(%rsi), %ymm12, %ymm2
|
||||
vpermd 0x140(%rsi), %ymm12, %ymm10
|
||||
vpsubw %ymm3, %ymm5, %ymm12
|
||||
vpaddw %ymm5, %ymm3, %ymm3
|
||||
vpsubw %ymm4, %ymm7, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm5
|
||||
vpaddw %ymm7, %ymm4, %ymm4
|
||||
vpsubw %ymm6, %ymm9, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm7
|
||||
vpaddw %ymm9, %ymm6, %ymm6
|
||||
vpsubw %ymm8, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm9
|
||||
vpaddw %ymm11, %ymm8, %ymm8
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm10, %ymm12, %ymm12
|
||||
vpmulhw %ymm10, %ymm13, %ymm13
|
||||
vpmulhw %ymm10, %ymm14, %ymm14
|
||||
vpmulhw %ymm10, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm5, %ymm12, %ymm5
|
||||
vpsubw %ymm7, %ymm13, %ymm7
|
||||
vpsubw %ymm9, %ymm14, %ymm9
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF
|
||||
vmovd %eax, %xmm1
|
||||
vpbroadcastd %xmm1, %ymm1
|
||||
vpmulhw %ymm1, %ymm3, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7]
|
||||
vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7]
|
||||
vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm5, %ymm5
|
||||
vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7]
|
||||
vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7]
|
||||
vpsrlq $0x20, %ymm9, %ymm9
|
||||
vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7]
|
||||
vpermq $0x1b, 0xe0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0]
|
||||
vpermq $0x1b, 0x100(%rsi), %ymm9 # ymm9 = mem[3,2,1,0]
|
||||
vpsubw %ymm10, %ymm4, %ymm12
|
||||
vpaddw %ymm4, %ymm10, %ymm10
|
||||
vpsubw %ymm3, %ymm8, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm4
|
||||
vpaddw %ymm8, %ymm3, %ymm3
|
||||
vpsubw %ymm6, %ymm7, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm8
|
||||
vpaddw %ymm7, %ymm6, %ymm6
|
||||
vpsubw %ymm5, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm7
|
||||
vpaddw %ymm11, %ymm5, %ymm5
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm9, %ymm12, %ymm12
|
||||
vpmulhw %ymm9, %ymm13, %ymm13
|
||||
vpmulhw %ymm9, %ymm14, %ymm14
|
||||
vpmulhw %ymm9, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm4, %ymm4
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm4, %ymm12, %ymm4
|
||||
vpsubw %ymm8, %ymm13, %ymm8
|
||||
vpsubw %ymm7, %ymm14, %ymm7
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpmulhw %ymm1, %ymm10, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm10, %ymm10
|
||||
vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3]
|
||||
vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
|
||||
vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2]
|
||||
vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3]
|
||||
vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2]
|
||||
vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3]
|
||||
vpermq $0x4e, 0xa0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1]
|
||||
vpermq $0x4e, 0xc0(%rsi), %ymm7 # ymm7 = mem[2,3,0,1]
|
||||
vpsubw %ymm9, %ymm3, %ymm12
|
||||
vpaddw %ymm3, %ymm9, %ymm9
|
||||
vpsubw %ymm10, %ymm5, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm3
|
||||
vpaddw %ymm5, %ymm10, %ymm10
|
||||
vpsubw %ymm6, %ymm8, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm5
|
||||
vpaddw %ymm8, %ymm6, %ymm6
|
||||
vpsubw %ymm4, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm8
|
||||
vpaddw %ymm11, %ymm4, %ymm4
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm7, %ymm12, %ymm12
|
||||
vpmulhw %ymm7, %ymm13, %ymm13
|
||||
vpmulhw %ymm7, %ymm14, %ymm14
|
||||
vpmulhw %ymm7, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm3, %ymm3
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm3, %ymm12, %ymm3
|
||||
vpsubw %ymm5, %ymm13, %ymm5
|
||||
vpsubw %ymm8, %ymm14, %ymm8
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpmulhw %ymm1, %ymm9, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm9, %ymm9
|
||||
vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1]
|
||||
vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3]
|
||||
vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1]
|
||||
vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3]
|
||||
vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1]
|
||||
vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3]
|
||||
vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1]
|
||||
vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3]
|
||||
vmovdqa 0x60(%rsi), %ymm2
|
||||
vmovdqa 0x80(%rsi), %ymm8
|
||||
vpsubw %ymm7, %ymm10, %ymm12
|
||||
vpaddw %ymm10, %ymm7, %ymm7
|
||||
vpsubw %ymm9, %ymm4, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm10
|
||||
vpaddw %ymm4, %ymm9, %ymm9
|
||||
vpsubw %ymm6, %ymm5, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm4
|
||||
vpaddw %ymm5, %ymm6, %ymm6
|
||||
vpsubw %ymm3, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm5
|
||||
vpaddw %ymm11, %ymm3, %ymm3
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm8, %ymm12, %ymm12
|
||||
vpmulhw %ymm8, %ymm13, %ymm13
|
||||
vpmulhw %ymm8, %ymm14, %ymm14
|
||||
vpmulhw %ymm8, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm4, %ymm4
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm10, %ymm12, %ymm10
|
||||
vpsubw %ymm4, %ymm13, %ymm4
|
||||
vpsubw %ymm5, %ymm14, %ymm5
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vpmulhw %ymm1, %ymm7, %ymm12
|
||||
vpsraw $0xa, %ymm12, %ymm12
|
||||
vpmullw %ymm0, %ymm12, %ymm12
|
||||
vpsubw %ymm12, %ymm7, %ymm7
|
||||
vmovdqa %ymm7, 0x100(%rdi)
|
||||
vmovdqa %ymm9, 0x120(%rdi)
|
||||
vmovdqa %ymm6, 0x140(%rdi)
|
||||
vmovdqa %ymm3, 0x160(%rdi)
|
||||
vmovdqa %ymm10, 0x180(%rdi)
|
||||
vmovdqa %ymm4, 0x1a0(%rdi)
|
||||
vmovdqa %ymm5, 0x1c0(%rdi)
|
||||
vmovdqa %ymm11, 0x1e0(%rdi)
|
||||
vmovdqa (%rdi), %ymm4
|
||||
vmovdqa 0x100(%rdi), %ymm8
|
||||
vmovdqa 0x20(%rdi), %ymm5
|
||||
vmovdqa 0x120(%rdi), %ymm9
|
||||
vpbroadcastq 0x40(%rsi), %ymm2
|
||||
vmovdqa 0x40(%rdi), %ymm6
|
||||
vmovdqa 0x140(%rdi), %ymm10
|
||||
vmovdqa 0x60(%rdi), %ymm7
|
||||
vmovdqa 0x160(%rdi), %ymm11
|
||||
vpbroadcastq 0x48(%rsi), %ymm3
|
||||
vpsubw %ymm4, %ymm8, %ymm12
|
||||
vpaddw %ymm8, %ymm4, %ymm4
|
||||
vpsubw %ymm5, %ymm9, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm5
|
||||
vpsubw %ymm6, %ymm10, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm6
|
||||
vpsubw %ymm7, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm7
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm3, %ymm12, %ymm12
|
||||
vpmulhw %ymm3, %ymm13, %ymm13
|
||||
vpmulhw %ymm3, %ymm14, %ymm14
|
||||
vpmulhw %ymm3, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm8, %ymm12, %ymm8
|
||||
vpsubw %ymm9, %ymm13, %ymm9
|
||||
vpsubw %ymm10, %ymm14, %ymm10
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vmovdqa %ymm4, (%rdi)
|
||||
vmovdqa %ymm5, 0x20(%rdi)
|
||||
vmovdqa %ymm6, 0x40(%rdi)
|
||||
vmovdqa %ymm7, 0x60(%rdi)
|
||||
vmovdqa %ymm8, 0x100(%rdi)
|
||||
vmovdqa %ymm9, 0x120(%rdi)
|
||||
vmovdqa %ymm10, 0x140(%rdi)
|
||||
vmovdqa %ymm11, 0x160(%rdi)
|
||||
vmovdqa 0x80(%rdi), %ymm4
|
||||
vmovdqa 0x180(%rdi), %ymm8
|
||||
vmovdqa 0xa0(%rdi), %ymm5
|
||||
vmovdqa 0x1a0(%rdi), %ymm9
|
||||
vpbroadcastq 0x40(%rsi), %ymm2
|
||||
vmovdqa 0xc0(%rdi), %ymm6
|
||||
vmovdqa 0x1c0(%rdi), %ymm10
|
||||
vmovdqa 0xe0(%rdi), %ymm7
|
||||
vmovdqa 0x1e0(%rdi), %ymm11
|
||||
vpbroadcastq 0x48(%rsi), %ymm3
|
||||
vpsubw %ymm4, %ymm8, %ymm12
|
||||
vpaddw %ymm8, %ymm4, %ymm4
|
||||
vpsubw %ymm5, %ymm9, %ymm13
|
||||
vpmullw %ymm2, %ymm12, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm5
|
||||
vpsubw %ymm6, %ymm10, %ymm14
|
||||
vpmullw %ymm2, %ymm13, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm6
|
||||
vpsubw %ymm7, %ymm11, %ymm15
|
||||
vpmullw %ymm2, %ymm14, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm7
|
||||
vpmullw %ymm2, %ymm15, %ymm11
|
||||
vpmulhw %ymm3, %ymm12, %ymm12
|
||||
vpmulhw %ymm3, %ymm13, %ymm13
|
||||
vpmulhw %ymm3, %ymm14, %ymm14
|
||||
vpmulhw %ymm3, %ymm15, %ymm15
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm10, %ymm10
|
||||
vpmulhw %ymm0, %ymm11, %ymm11
|
||||
vpsubw %ymm8, %ymm12, %ymm8
|
||||
vpsubw %ymm9, %ymm13, %ymm9
|
||||
vpsubw %ymm10, %ymm14, %ymm10
|
||||
vpsubw %ymm11, %ymm15, %ymm11
|
||||
vmovdqa %ymm4, 0x80(%rdi)
|
||||
vmovdqa %ymm5, 0xa0(%rdi)
|
||||
vmovdqa %ymm6, 0xc0(%rdi)
|
||||
vmovdqa %ymm7, 0xe0(%rdi)
|
||||
vmovdqa %ymm8, 0x180(%rdi)
|
||||
vmovdqa %ymm9, 0x1a0(%rdi)
|
||||
vmovdqa %ymm10, 0x1c0(%rdi)
|
||||
vmovdqa %ymm11, 0x1e0(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(invntt_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,661 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [AVX2_NTT]
|
||||
* Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography.
|
||||
* Gregor Seiler
|
||||
* https://eprint.iacr.org/2018/039
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*
|
||||
* The core ideas behind the implementation are described in @[AVX2_NTT].
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
/*yaml
|
||||
Name: ntt_avx2_asm
|
||||
Description: x86_64 AVX2 forward NTT
|
||||
Signature: void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t *r
|
||||
description: Input/output polynomial (256 x int16_t)
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 1248
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *qdata
|
||||
description: Precomputed constants (624 x int16_t)
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/ntt_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(ntt_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(ntt_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0xd010d01, %eax # imm = 0xD010D01
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
vpbroadcastq 0x40(%rsi), %ymm15
|
||||
vmovdqa 0x100(%rdi), %ymm8
|
||||
vmovdqa 0x120(%rdi), %ymm9
|
||||
vmovdqa 0x140(%rdi), %ymm10
|
||||
vmovdqa 0x160(%rdi), %ymm11
|
||||
vpbroadcastq 0x48(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm8, %ymm12
|
||||
vpmullw %ymm15, %ymm9, %ymm13
|
||||
vpmullw %ymm15, %ymm10, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vmovdqa (%rdi), %ymm4
|
||||
vmovdqa 0x20(%rdi), %ymm5
|
||||
vmovdqa 0x40(%rdi), %ymm6
|
||||
vmovdqa 0x60(%rdi), %ymm7
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm8, %ymm4, %ymm3
|
||||
vpsubw %ymm8, %ymm4, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm4
|
||||
vpsubw %ymm9, %ymm5, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm5
|
||||
vpsubw %ymm10, %ymm6, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm6
|
||||
vpsubw %ymm11, %ymm7, %ymm11
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vpaddw %ymm12, %ymm8, %ymm8
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm9, %ymm9
|
||||
vpsubw %ymm14, %ymm5, %ymm5
|
||||
vpaddw %ymm14, %ymm10, %ymm10
|
||||
vpsubw %ymm15, %ymm6, %ymm6
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovdqa %ymm3, (%rdi)
|
||||
vmovdqa %ymm4, 0x20(%rdi)
|
||||
vmovdqa %ymm5, 0x40(%rdi)
|
||||
vmovdqa %ymm6, 0x60(%rdi)
|
||||
vmovdqa %ymm8, 0x100(%rdi)
|
||||
vmovdqa %ymm9, 0x120(%rdi)
|
||||
vmovdqa %ymm10, 0x140(%rdi)
|
||||
vmovdqa %ymm11, 0x160(%rdi)
|
||||
vpbroadcastq 0x40(%rsi), %ymm15
|
||||
vmovdqa 0x180(%rdi), %ymm8
|
||||
vmovdqa 0x1a0(%rdi), %ymm9
|
||||
vmovdqa 0x1c0(%rdi), %ymm10
|
||||
vmovdqa 0x1e0(%rdi), %ymm11
|
||||
vpbroadcastq 0x48(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm8, %ymm12
|
||||
vpmullw %ymm15, %ymm9, %ymm13
|
||||
vpmullw %ymm15, %ymm10, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vmovdqa 0x80(%rdi), %ymm4
|
||||
vmovdqa 0xa0(%rdi), %ymm5
|
||||
vmovdqa 0xc0(%rdi), %ymm6
|
||||
vmovdqa 0xe0(%rdi), %ymm7
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm8, %ymm4, %ymm3
|
||||
vpsubw %ymm8, %ymm4, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm4
|
||||
vpsubw %ymm9, %ymm5, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm5
|
||||
vpsubw %ymm10, %ymm6, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm6
|
||||
vpsubw %ymm11, %ymm7, %ymm11
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vpaddw %ymm12, %ymm8, %ymm8
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm9, %ymm9
|
||||
vpsubw %ymm14, %ymm5, %ymm5
|
||||
vpaddw %ymm14, %ymm10, %ymm10
|
||||
vpsubw %ymm15, %ymm6, %ymm6
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovdqa %ymm3, 0x80(%rdi)
|
||||
vmovdqa %ymm4, 0xa0(%rdi)
|
||||
vmovdqa %ymm5, 0xc0(%rdi)
|
||||
vmovdqa %ymm6, 0xe0(%rdi)
|
||||
vmovdqa %ymm8, 0x180(%rdi)
|
||||
vmovdqa %ymm9, 0x1a0(%rdi)
|
||||
vmovdqa %ymm10, 0x1c0(%rdi)
|
||||
vmovdqa %ymm11, 0x1e0(%rdi)
|
||||
vmovdqa 0x60(%rsi), %ymm15
|
||||
vmovdqa 0x80(%rdi), %ymm8
|
||||
vmovdqa 0xa0(%rdi), %ymm9
|
||||
vmovdqa 0xc0(%rdi), %ymm10
|
||||
vmovdqa 0xe0(%rdi), %ymm11
|
||||
vmovdqa 0x80(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm8, %ymm12
|
||||
vpmullw %ymm15, %ymm9, %ymm13
|
||||
vpmullw %ymm15, %ymm10, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vmovdqa (%rdi), %ymm4
|
||||
vmovdqa 0x20(%rdi), %ymm5
|
||||
vmovdqa 0x40(%rdi), %ymm6
|
||||
vmovdqa 0x60(%rdi), %ymm7
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm8, %ymm4, %ymm3
|
||||
vpsubw %ymm8, %ymm4, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm4
|
||||
vpsubw %ymm9, %ymm5, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm5
|
||||
vpsubw %ymm10, %ymm6, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm6
|
||||
vpsubw %ymm11, %ymm7, %ymm11
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vpaddw %ymm12, %ymm8, %ymm8
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm9, %ymm9
|
||||
vpsubw %ymm14, %ymm5, %ymm5
|
||||
vpaddw %ymm14, %ymm10, %ymm10
|
||||
vpsubw %ymm15, %ymm6, %ymm6
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1]
|
||||
vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3]
|
||||
vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1]
|
||||
vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3]
|
||||
vmovdqa 0xa0(%rsi), %ymm15
|
||||
vmovdqa 0xc0(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm7, %ymm12
|
||||
vpmullw %ymm15, %ymm10, %ymm13
|
||||
vpmullw %ymm15, %ymm5, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm7, %ymm7
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm5, %ymm5
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm7, %ymm6, %ymm4
|
||||
vpsubw %ymm7, %ymm6, %ymm7
|
||||
vpaddw %ymm10, %ymm8, %ymm6
|
||||
vpsubw %ymm10, %ymm8, %ymm10
|
||||
vpaddw %ymm5, %ymm3, %ymm8
|
||||
vpsubw %ymm5, %ymm3, %ymm5
|
||||
vpaddw %ymm11, %ymm9, %ymm3
|
||||
vpsubw %ymm11, %ymm9, %ymm11
|
||||
vpsubw %ymm12, %ymm4, %ymm4
|
||||
vpaddw %ymm12, %ymm7, %ymm7
|
||||
vpsubw %ymm13, %ymm6, %ymm6
|
||||
vpaddw %ymm13, %ymm10, %ymm10
|
||||
vpsubw %ymm14, %ymm8, %ymm8
|
||||
vpaddw %ymm14, %ymm5, %ymm5
|
||||
vpsubw %ymm15, %ymm3, %ymm3
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3]
|
||||
vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2]
|
||||
vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3]
|
||||
vmovdqa 0xe0(%rsi), %ymm15
|
||||
vmovdqa 0x100(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm9, %ymm12
|
||||
vpmullw %ymm15, %ymm5, %ymm13
|
||||
vpmullw %ymm15, %ymm8, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm5, %ymm5
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2]
|
||||
vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3]
|
||||
vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2]
|
||||
vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm9, %ymm3, %ymm6
|
||||
vpsubw %ymm9, %ymm3, %ymm9
|
||||
vpaddw %ymm5, %ymm7, %ymm3
|
||||
vpsubw %ymm5, %ymm7, %ymm5
|
||||
vpaddw %ymm8, %ymm4, %ymm7
|
||||
vpsubw %ymm8, %ymm4, %ymm8
|
||||
vpaddw %ymm11, %ymm10, %ymm4
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpsubw %ymm12, %ymm6, %ymm6
|
||||
vpaddw %ymm12, %ymm9, %ymm9
|
||||
vpsubw %ymm13, %ymm3, %ymm3
|
||||
vpaddw %ymm13, %ymm5, %ymm5
|
||||
vpsubw %ymm14, %ymm7, %ymm7
|
||||
vpaddw %ymm14, %ymm8, %ymm8
|
||||
vpsubw %ymm15, %ymm4, %ymm4
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7]
|
||||
vpsrlq $0x20, %ymm7, %ymm7
|
||||
vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7]
|
||||
vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7]
|
||||
vpsrlq $0x20, %ymm4, %ymm4
|
||||
vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7]
|
||||
vmovdqa 0x120(%rsi), %ymm15
|
||||
vmovdqa 0x140(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm10, %ymm12
|
||||
vpmullw %ymm15, %ymm8, %ymm13
|
||||
vpmullw %ymm15, %ymm7, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm7, %ymm7
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7]
|
||||
vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm10, %ymm4, %ymm3
|
||||
vpsubw %ymm10, %ymm4, %ymm10
|
||||
vpaddw %ymm8, %ymm9, %ymm4
|
||||
vpsubw %ymm8, %ymm9, %ymm8
|
||||
vpaddw %ymm7, %ymm6, %ymm9
|
||||
vpsubw %ymm7, %ymm6, %ymm7
|
||||
vpaddw %ymm11, %ymm5, %ymm6
|
||||
vpsubw %ymm11, %ymm5, %ymm11
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vpaddw %ymm12, %ymm10, %ymm10
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm8, %ymm8
|
||||
vpsubw %ymm14, %ymm9, %ymm9
|
||||
vpaddw %ymm14, %ymm7, %ymm7
|
||||
vpsubw %ymm15, %ymm6, %ymm6
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vpslld $0x10, %ymm7, %ymm5
|
||||
vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15]
|
||||
vpsrld $0x10, %ymm9, %ymm9
|
||||
vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15]
|
||||
vpslld $0x10, %ymm11, %ymm9
|
||||
vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
|
||||
vmovdqa 0x160(%rsi), %ymm15
|
||||
vmovdqa 0x180(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm5, %ymm12
|
||||
vpmullw %ymm15, %ymm7, %ymm13
|
||||
vpmullw %ymm15, %ymm9, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm5, %ymm5
|
||||
vpmulhw %ymm2, %ymm7, %ymm7
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vpslld $0x10, %ymm10, %ymm6
|
||||
vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15]
|
||||
vpsrld $0x10, %ymm3, %ymm3
|
||||
vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15]
|
||||
vpslld $0x10, %ymm8, %ymm3
|
||||
vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm5, %ymm6, %ymm4
|
||||
vpsubw %ymm5, %ymm6, %ymm5
|
||||
vpaddw %ymm7, %ymm10, %ymm6
|
||||
vpsubw %ymm7, %ymm10, %ymm7
|
||||
vpaddw %ymm9, %ymm3, %ymm10
|
||||
vpsubw %ymm9, %ymm3, %ymm9
|
||||
vpaddw %ymm11, %ymm8, %ymm3
|
||||
vpsubw %ymm11, %ymm8, %ymm11
|
||||
vpsubw %ymm12, %ymm4, %ymm4
|
||||
vpaddw %ymm12, %ymm5, %ymm5
|
||||
vpsubw %ymm13, %ymm6, %ymm6
|
||||
vpaddw %ymm13, %ymm7, %ymm7
|
||||
vpsubw %ymm14, %ymm10, %ymm10
|
||||
vpaddw %ymm14, %ymm9, %ymm9
|
||||
vpsubw %ymm15, %ymm3, %ymm3
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovdqa 0x1a0(%rsi), %ymm14
|
||||
vmovdqa 0x1e0(%rsi), %ymm15
|
||||
vmovdqa 0x1c0(%rsi), %ymm8
|
||||
vmovdqa 0x200(%rsi), %ymm2
|
||||
vpmullw %ymm14, %ymm10, %ymm12
|
||||
vpmullw %ymm14, %ymm3, %ymm13
|
||||
vpmullw %ymm15, %ymm9, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm8, %ymm10, %ymm10
|
||||
vpmulhw %ymm8, %ymm3, %ymm3
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm10, %ymm4, %ymm8
|
||||
vpsubw %ymm10, %ymm4, %ymm10
|
||||
vpaddw %ymm3, %ymm6, %ymm4
|
||||
vpsubw %ymm3, %ymm6, %ymm3
|
||||
vpaddw %ymm9, %ymm5, %ymm6
|
||||
vpsubw %ymm9, %ymm5, %ymm9
|
||||
vpaddw %ymm11, %ymm7, %ymm5
|
||||
vpsubw %ymm11, %ymm7, %ymm11
|
||||
vpsubw %ymm12, %ymm8, %ymm8
|
||||
vpaddw %ymm12, %ymm10, %ymm10
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm3, %ymm3
|
||||
vpsubw %ymm14, %ymm6, %ymm6
|
||||
vpaddw %ymm14, %ymm9, %ymm9
|
||||
vpsubw %ymm15, %ymm5, %ymm5
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovdqa %ymm8, (%rdi)
|
||||
vmovdqa %ymm4, 0x20(%rdi)
|
||||
vmovdqa %ymm10, 0x40(%rdi)
|
||||
vmovdqa %ymm3, 0x60(%rdi)
|
||||
vmovdqa %ymm6, 0x80(%rdi)
|
||||
vmovdqa %ymm5, 0xa0(%rdi)
|
||||
vmovdqa %ymm9, 0xc0(%rdi)
|
||||
vmovdqa %ymm11, 0xe0(%rdi)
|
||||
vmovdqa 0x220(%rsi), %ymm15
|
||||
vmovdqa 0x180(%rdi), %ymm8
|
||||
vmovdqa 0x1a0(%rdi), %ymm9
|
||||
vmovdqa 0x1c0(%rdi), %ymm10
|
||||
vmovdqa 0x1e0(%rdi), %ymm11
|
||||
vmovdqa 0x240(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm8, %ymm12
|
||||
vpmullw %ymm15, %ymm9, %ymm13
|
||||
vpmullw %ymm15, %ymm10, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vmovdqa 0x100(%rdi), %ymm4
|
||||
vmovdqa 0x120(%rdi), %ymm5
|
||||
vmovdqa 0x140(%rdi), %ymm6
|
||||
vmovdqa 0x160(%rdi), %ymm7
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm8, %ymm4, %ymm3
|
||||
vpsubw %ymm8, %ymm4, %ymm8
|
||||
vpaddw %ymm9, %ymm5, %ymm4
|
||||
vpsubw %ymm9, %ymm5, %ymm9
|
||||
vpaddw %ymm10, %ymm6, %ymm5
|
||||
vpsubw %ymm10, %ymm6, %ymm10
|
||||
vpaddw %ymm11, %ymm7, %ymm6
|
||||
vpsubw %ymm11, %ymm7, %ymm11
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vpaddw %ymm12, %ymm8, %ymm8
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm9, %ymm9
|
||||
vpsubw %ymm14, %ymm5, %ymm5
|
||||
vpaddw %ymm14, %ymm10, %ymm10
|
||||
vpsubw %ymm15, %ymm6, %ymm6
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1]
|
||||
vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3]
|
||||
vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1]
|
||||
vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3]
|
||||
vmovdqa 0x260(%rsi), %ymm15
|
||||
vmovdqa 0x280(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm7, %ymm12
|
||||
vpmullw %ymm15, %ymm10, %ymm13
|
||||
vpmullw %ymm15, %ymm5, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm7, %ymm7
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm5, %ymm5
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm7, %ymm6, %ymm4
|
||||
vpsubw %ymm7, %ymm6, %ymm7
|
||||
vpaddw %ymm10, %ymm8, %ymm6
|
||||
vpsubw %ymm10, %ymm8, %ymm10
|
||||
vpaddw %ymm5, %ymm3, %ymm8
|
||||
vpsubw %ymm5, %ymm3, %ymm5
|
||||
vpaddw %ymm11, %ymm9, %ymm3
|
||||
vpsubw %ymm11, %ymm9, %ymm11
|
||||
vpsubw %ymm12, %ymm4, %ymm4
|
||||
vpaddw %ymm12, %ymm7, %ymm7
|
||||
vpsubw %ymm13, %ymm6, %ymm6
|
||||
vpaddw %ymm13, %ymm10, %ymm10
|
||||
vpsubw %ymm14, %ymm8, %ymm8
|
||||
vpaddw %ymm14, %ymm5, %ymm5
|
||||
vpsubw %ymm15, %ymm3, %ymm3
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3]
|
||||
vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2]
|
||||
vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3]
|
||||
vmovdqa 0x2a0(%rsi), %ymm15
|
||||
vmovdqa 0x2c0(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm9, %ymm12
|
||||
vpmullw %ymm15, %ymm5, %ymm13
|
||||
vpmullw %ymm15, %ymm8, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm5, %ymm5
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2]
|
||||
vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3]
|
||||
vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2]
|
||||
vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm9, %ymm3, %ymm6
|
||||
vpsubw %ymm9, %ymm3, %ymm9
|
||||
vpaddw %ymm5, %ymm7, %ymm3
|
||||
vpsubw %ymm5, %ymm7, %ymm5
|
||||
vpaddw %ymm8, %ymm4, %ymm7
|
||||
vpsubw %ymm8, %ymm4, %ymm8
|
||||
vpaddw %ymm11, %ymm10, %ymm4
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpsubw %ymm12, %ymm6, %ymm6
|
||||
vpaddw %ymm12, %ymm9, %ymm9
|
||||
vpsubw %ymm13, %ymm3, %ymm3
|
||||
vpaddw %ymm13, %ymm5, %ymm5
|
||||
vpsubw %ymm14, %ymm7, %ymm7
|
||||
vpaddw %ymm14, %ymm8, %ymm8
|
||||
vpsubw %ymm15, %ymm4, %ymm4
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7]
|
||||
vpsrlq $0x20, %ymm7, %ymm7
|
||||
vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7]
|
||||
vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7]
|
||||
vpsrlq $0x20, %ymm4, %ymm4
|
||||
vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7]
|
||||
vmovdqa 0x2e0(%rsi), %ymm15
|
||||
vmovdqa 0x300(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm10, %ymm12
|
||||
vpmullw %ymm15, %ymm8, %ymm13
|
||||
vpmullw %ymm15, %ymm7, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm10, %ymm10
|
||||
vpmulhw %ymm2, %ymm8, %ymm8
|
||||
vpmulhw %ymm2, %ymm7, %ymm7
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7]
|
||||
vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm10, %ymm4, %ymm3
|
||||
vpsubw %ymm10, %ymm4, %ymm10
|
||||
vpaddw %ymm8, %ymm9, %ymm4
|
||||
vpsubw %ymm8, %ymm9, %ymm8
|
||||
vpaddw %ymm7, %ymm6, %ymm9
|
||||
vpsubw %ymm7, %ymm6, %ymm7
|
||||
vpaddw %ymm11, %ymm5, %ymm6
|
||||
vpsubw %ymm11, %ymm5, %ymm11
|
||||
vpsubw %ymm12, %ymm3, %ymm3
|
||||
vpaddw %ymm12, %ymm10, %ymm10
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm8, %ymm8
|
||||
vpsubw %ymm14, %ymm9, %ymm9
|
||||
vpaddw %ymm14, %ymm7, %ymm7
|
||||
vpsubw %ymm15, %ymm6, %ymm6
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vpslld $0x10, %ymm7, %ymm5
|
||||
vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15]
|
||||
vpsrld $0x10, %ymm9, %ymm9
|
||||
vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15]
|
||||
vpslld $0x10, %ymm11, %ymm9
|
||||
vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
|
||||
vmovdqa 0x320(%rsi), %ymm15
|
||||
vmovdqa 0x340(%rsi), %ymm2
|
||||
vpmullw %ymm15, %ymm5, %ymm12
|
||||
vpmullw %ymm15, %ymm7, %ymm13
|
||||
vpmullw %ymm15, %ymm9, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm2, %ymm5, %ymm5
|
||||
vpmulhw %ymm2, %ymm7, %ymm7
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vpslld $0x10, %ymm10, %ymm6
|
||||
vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15]
|
||||
vpsrld $0x10, %ymm3, %ymm3
|
||||
vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15]
|
||||
vpslld $0x10, %ymm8, %ymm3
|
||||
vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm5, %ymm6, %ymm4
|
||||
vpsubw %ymm5, %ymm6, %ymm5
|
||||
vpaddw %ymm7, %ymm10, %ymm6
|
||||
vpsubw %ymm7, %ymm10, %ymm7
|
||||
vpaddw %ymm9, %ymm3, %ymm10
|
||||
vpsubw %ymm9, %ymm3, %ymm9
|
||||
vpaddw %ymm11, %ymm8, %ymm3
|
||||
vpsubw %ymm11, %ymm8, %ymm11
|
||||
vpsubw %ymm12, %ymm4, %ymm4
|
||||
vpaddw %ymm12, %ymm5, %ymm5
|
||||
vpsubw %ymm13, %ymm6, %ymm6
|
||||
vpaddw %ymm13, %ymm7, %ymm7
|
||||
vpsubw %ymm14, %ymm10, %ymm10
|
||||
vpaddw %ymm14, %ymm9, %ymm9
|
||||
vpsubw %ymm15, %ymm3, %ymm3
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovdqa 0x360(%rsi), %ymm14
|
||||
vmovdqa 0x3a0(%rsi), %ymm15
|
||||
vmovdqa 0x380(%rsi), %ymm8
|
||||
vmovdqa 0x3c0(%rsi), %ymm2
|
||||
vpmullw %ymm14, %ymm10, %ymm12
|
||||
vpmullw %ymm14, %ymm3, %ymm13
|
||||
vpmullw %ymm15, %ymm9, %ymm14
|
||||
vpmullw %ymm15, %ymm11, %ymm15
|
||||
vpmulhw %ymm8, %ymm10, %ymm10
|
||||
vpmulhw %ymm8, %ymm3, %ymm3
|
||||
vpmulhw %ymm2, %ymm9, %ymm9
|
||||
vpmulhw %ymm2, %ymm11, %ymm11
|
||||
vpmulhw %ymm0, %ymm12, %ymm12
|
||||
vpmulhw %ymm0, %ymm13, %ymm13
|
||||
vpmulhw %ymm0, %ymm14, %ymm14
|
||||
vpmulhw %ymm0, %ymm15, %ymm15
|
||||
vpaddw %ymm10, %ymm4, %ymm8
|
||||
vpsubw %ymm10, %ymm4, %ymm10
|
||||
vpaddw %ymm3, %ymm6, %ymm4
|
||||
vpsubw %ymm3, %ymm6, %ymm3
|
||||
vpaddw %ymm9, %ymm5, %ymm6
|
||||
vpsubw %ymm9, %ymm5, %ymm9
|
||||
vpaddw %ymm11, %ymm7, %ymm5
|
||||
vpsubw %ymm11, %ymm7, %ymm11
|
||||
vpsubw %ymm12, %ymm8, %ymm8
|
||||
vpaddw %ymm12, %ymm10, %ymm10
|
||||
vpsubw %ymm13, %ymm4, %ymm4
|
||||
vpaddw %ymm13, %ymm3, %ymm3
|
||||
vpsubw %ymm14, %ymm6, %ymm6
|
||||
vpaddw %ymm14, %ymm9, %ymm9
|
||||
vpsubw %ymm15, %ymm5, %ymm5
|
||||
vpaddw %ymm15, %ymm11, %ymm11
|
||||
vmovdqa %ymm8, 0x100(%rdi)
|
||||
vmovdqa %ymm4, 0x120(%rdi)
|
||||
vmovdqa %ymm10, 0x140(%rdi)
|
||||
vmovdqa %ymm3, 0x160(%rdi)
|
||||
vmovdqa %ymm6, 0x180(%rdi)
|
||||
vmovdqa %ymm5, 0x1a0(%rdi)
|
||||
vmovdqa %ymm9, 0x1c0(%rdi)
|
||||
vmovdqa %ymm11, 0x1e0(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(ntt_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
/*yaml
|
||||
Name: nttfrombytes_avx2_asm
|
||||
Description: x86_64 AVX2 polynomial deserialization in NTT domain
|
||||
Signature: void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: write-only
|
||||
c_parameter: int16_t *r
|
||||
description: Output polynomial (256 x int16_t)
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 384
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t *a
|
||||
description: Input byte array (MLKEM_POLYBYTES = 384)
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/nttfrombytes_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(nttfrombytes_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(nttfrombytes_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0xfff0fff, %eax # imm = 0xFFF0FFF
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
vmovdqu (%rsi), %ymm4
|
||||
vmovdqu 0x20(%rsi), %ymm5
|
||||
vmovdqu 0x40(%rsi), %ymm6
|
||||
vmovdqu 0x60(%rsi), %ymm7
|
||||
vmovdqu 0x80(%rsi), %ymm8
|
||||
vmovdqu 0xa0(%rsi), %ymm9
|
||||
vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1]
|
||||
vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3]
|
||||
vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3]
|
||||
vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2]
|
||||
vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3]
|
||||
vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3]
|
||||
vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2]
|
||||
vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3]
|
||||
vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
|
||||
vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm8, %ymm8
|
||||
vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]
|
||||
vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7]
|
||||
vpslld $0x10, %ymm7, %ymm10
|
||||
vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15]
|
||||
vpslld $0x10, %ymm8, %ymm4
|
||||
vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
|
||||
vpsrld $0x10, %ymm5, %ymm5
|
||||
vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15]
|
||||
vpslld $0x10, %ymm9, %ymm5
|
||||
vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15]
|
||||
vpsrlw $0xc, %ymm10, %ymm11
|
||||
vpsllw $0x4, %ymm7, %ymm12
|
||||
vpor %ymm11, %ymm12, %ymm11
|
||||
vpand %ymm0, %ymm10, %ymm10
|
||||
vpand %ymm0, %ymm11, %ymm11
|
||||
vpsrlw $0x8, %ymm7, %ymm12
|
||||
vpsllw $0x8, %ymm4, %ymm13
|
||||
vpor %ymm12, %ymm13, %ymm12
|
||||
vpand %ymm0, %ymm12, %ymm12
|
||||
vpsrlw $0x4, %ymm4, %ymm13
|
||||
vpand %ymm0, %ymm13, %ymm13
|
||||
vpsrlw $0xc, %ymm8, %ymm14
|
||||
vpsllw $0x4, %ymm5, %ymm15
|
||||
vpor %ymm14, %ymm15, %ymm14
|
||||
vpand %ymm0, %ymm8, %ymm8
|
||||
vpand %ymm0, %ymm14, %ymm14
|
||||
vpsrlw $0x8, %ymm5, %ymm15
|
||||
vpsllw $0x8, %ymm9, %ymm1
|
||||
vpor %ymm15, %ymm1, %ymm15
|
||||
vpand %ymm0, %ymm15, %ymm15
|
||||
vpsrlw $0x4, %ymm9, %ymm1
|
||||
vpand %ymm0, %ymm1, %ymm1
|
||||
vmovdqa %ymm10, (%rdi)
|
||||
vmovdqa %ymm11, 0x20(%rdi)
|
||||
vmovdqa %ymm12, 0x40(%rdi)
|
||||
vmovdqa %ymm13, 0x60(%rdi)
|
||||
vmovdqa %ymm8, 0x80(%rdi)
|
||||
vmovdqa %ymm14, 0xa0(%rdi)
|
||||
vmovdqa %ymm15, 0xc0(%rdi)
|
||||
vmovdqa %ymm1, 0xe0(%rdi)
|
||||
vmovdqu 0xc0(%rsi), %ymm4
|
||||
vmovdqu 0xe0(%rsi), %ymm5
|
||||
vmovdqu 0x100(%rsi), %ymm6
|
||||
vmovdqu 0x120(%rsi), %ymm7
|
||||
vmovdqu 0x140(%rsi), %ymm8
|
||||
vmovdqu 0x160(%rsi), %ymm9
|
||||
vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1]
|
||||
vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3]
|
||||
vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3]
|
||||
vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2]
|
||||
vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3]
|
||||
vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3]
|
||||
vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2]
|
||||
vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3]
|
||||
vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
|
||||
vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm8, %ymm8
|
||||
vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]
|
||||
vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7]
|
||||
vpslld $0x10, %ymm7, %ymm10
|
||||
vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15]
|
||||
vpslld $0x10, %ymm8, %ymm4
|
||||
vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
|
||||
vpsrld $0x10, %ymm5, %ymm5
|
||||
vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15]
|
||||
vpslld $0x10, %ymm9, %ymm5
|
||||
vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15]
|
||||
vpsrlw $0xc, %ymm10, %ymm11
|
||||
vpsllw $0x4, %ymm7, %ymm12
|
||||
vpor %ymm11, %ymm12, %ymm11
|
||||
vpand %ymm0, %ymm10, %ymm10
|
||||
vpand %ymm0, %ymm11, %ymm11
|
||||
vpsrlw $0x8, %ymm7, %ymm12
|
||||
vpsllw $0x8, %ymm4, %ymm13
|
||||
vpor %ymm12, %ymm13, %ymm12
|
||||
vpand %ymm0, %ymm12, %ymm12
|
||||
vpsrlw $0x4, %ymm4, %ymm13
|
||||
vpand %ymm0, %ymm13, %ymm13
|
||||
vpsrlw $0xc, %ymm8, %ymm14
|
||||
vpsllw $0x4, %ymm5, %ymm15
|
||||
vpor %ymm14, %ymm15, %ymm14
|
||||
vpand %ymm0, %ymm8, %ymm8
|
||||
vpand %ymm0, %ymm14, %ymm14
|
||||
vpsrlw $0x8, %ymm5, %ymm15
|
||||
vpsllw $0x8, %ymm9, %ymm1
|
||||
vpor %ymm15, %ymm1, %ymm15
|
||||
vpand %ymm0, %ymm15, %ymm15
|
||||
vpsrlw $0x4, %ymm9, %ymm1
|
||||
vpand %ymm0, %ymm1, %ymm1
|
||||
vmovdqa %ymm10, 0x100(%rdi)
|
||||
vmovdqa %ymm11, 0x120(%rdi)
|
||||
vmovdqa %ymm12, 0x140(%rdi)
|
||||
vmovdqa %ymm13, 0x160(%rdi)
|
||||
vmovdqa %ymm8, 0x180(%rdi)
|
||||
vmovdqa %ymm14, 0x1a0(%rdi)
|
||||
vmovdqa %ymm15, 0x1c0(%rdi)
|
||||
vmovdqa %ymm1, 0x1e0(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(nttfrombytes_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
/*yaml
|
||||
Name: ntttobytes_avx2_asm
|
||||
Description: x86_64 AVX2 polynomial serialization in NTT domain
|
||||
Signature: void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 384
|
||||
permissions: write-only
|
||||
c_parameter: uint8_t *r
|
||||
description: Output byte array (MLKEM_POLYBYTES = 384)
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *a
|
||||
description: Input polynomial (256 x int16_t)
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/ntttobytes_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(ntttobytes_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(ntttobytes_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0xd010d01, %eax # imm = 0xD010D01
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
vmovdqa (%rsi), %ymm5
|
||||
vmovdqa 0x20(%rsi), %ymm6
|
||||
vmovdqa 0x40(%rsi), %ymm7
|
||||
vmovdqa 0x60(%rsi), %ymm8
|
||||
vmovdqa 0x80(%rsi), %ymm9
|
||||
vmovdqa 0xa0(%rsi), %ymm10
|
||||
vmovdqa 0xc0(%rsi), %ymm11
|
||||
vmovdqa 0xe0(%rsi), %ymm12
|
||||
vpsllw $0xc, %ymm6, %ymm4
|
||||
vpor %ymm4, %ymm5, %ymm4
|
||||
vpsrlw $0x4, %ymm6, %ymm5
|
||||
vpsllw $0x8, %ymm7, %ymm6
|
||||
vpor %ymm5, %ymm6, %ymm5
|
||||
vpsrlw $0x8, %ymm7, %ymm6
|
||||
vpsllw $0x4, %ymm8, %ymm7
|
||||
vpor %ymm6, %ymm7, %ymm6
|
||||
vpsllw $0xc, %ymm10, %ymm7
|
||||
vpor %ymm7, %ymm9, %ymm7
|
||||
vpsrlw $0x4, %ymm10, %ymm8
|
||||
vpsllw $0x8, %ymm11, %ymm9
|
||||
vpor %ymm8, %ymm9, %ymm8
|
||||
vpsrlw $0x8, %ymm11, %ymm9
|
||||
vpsllw $0x4, %ymm12, %ymm10
|
||||
vpor %ymm9, %ymm10, %ymm9
|
||||
vpslld $0x10, %ymm5, %ymm3
|
||||
vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
|
||||
vpslld $0x10, %ymm7, %ymm4
|
||||
vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
|
||||
vpslld $0x10, %ymm9, %ymm6
|
||||
vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15]
|
||||
vpsrld $0x10, %ymm8, %ymm8
|
||||
vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
|
||||
vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7]
|
||||
vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
|
||||
vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm7, %ymm7
|
||||
vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7]
|
||||
vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
|
||||
vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2]
|
||||
vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3]
|
||||
vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1]
|
||||
vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3]
|
||||
vmovdqu %ymm5, (%rdi)
|
||||
vmovdqu %ymm7, 0x20(%rdi)
|
||||
vmovdqu %ymm6, 0x40(%rdi)
|
||||
vmovdqu %ymm8, 0x60(%rdi)
|
||||
vmovdqu %ymm3, 0x80(%rdi)
|
||||
vmovdqu %ymm9, 0xa0(%rdi)
|
||||
vmovdqa 0x100(%rsi), %ymm5
|
||||
vmovdqa 0x120(%rsi), %ymm6
|
||||
vmovdqa 0x140(%rsi), %ymm7
|
||||
vmovdqa 0x160(%rsi), %ymm8
|
||||
vmovdqa 0x180(%rsi), %ymm9
|
||||
vmovdqa 0x1a0(%rsi), %ymm10
|
||||
vmovdqa 0x1c0(%rsi), %ymm11
|
||||
vmovdqa 0x1e0(%rsi), %ymm12
|
||||
vpsllw $0xc, %ymm6, %ymm4
|
||||
vpor %ymm4, %ymm5, %ymm4
|
||||
vpsrlw $0x4, %ymm6, %ymm5
|
||||
vpsllw $0x8, %ymm7, %ymm6
|
||||
vpor %ymm5, %ymm6, %ymm5
|
||||
vpsrlw $0x8, %ymm7, %ymm6
|
||||
vpsllw $0x4, %ymm8, %ymm7
|
||||
vpor %ymm6, %ymm7, %ymm6
|
||||
vpsllw $0xc, %ymm10, %ymm7
|
||||
vpor %ymm7, %ymm9, %ymm7
|
||||
vpsrlw $0x4, %ymm10, %ymm8
|
||||
vpsllw $0x8, %ymm11, %ymm9
|
||||
vpor %ymm8, %ymm9, %ymm8
|
||||
vpsrlw $0x8, %ymm11, %ymm9
|
||||
vpsllw $0x4, %ymm12, %ymm10
|
||||
vpor %ymm9, %ymm10, %ymm9
|
||||
vpslld $0x10, %ymm5, %ymm3
|
||||
vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
|
||||
vpsrld $0x10, %ymm4, %ymm4
|
||||
vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
|
||||
vpslld $0x10, %ymm7, %ymm4
|
||||
vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
|
||||
vpslld $0x10, %ymm9, %ymm6
|
||||
vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15]
|
||||
vpsrld $0x10, %ymm8, %ymm8
|
||||
vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
|
||||
vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7]
|
||||
vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
|
||||
vpsrlq $0x20, %ymm6, %ymm6
|
||||
vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]
|
||||
vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7]
|
||||
vpsrlq $0x20, %ymm7, %ymm7
|
||||
vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7]
|
||||
vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2]
|
||||
vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3]
|
||||
vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
|
||||
vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
|
||||
vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2]
|
||||
vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3]
|
||||
vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1]
|
||||
vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3]
|
||||
vmovdqu %ymm5, 0xc0(%rdi)
|
||||
vmovdqu %ymm7, 0xe0(%rdi)
|
||||
vmovdqu %ymm6, 0x100(%rdi)
|
||||
vmovdqu %ymm8, 0x120(%rdi)
|
||||
vmovdqu %ymm3, 0x140(%rdi)
|
||||
vmovdqu %ymm9, 0x160(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(ntttobytes_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,190 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)
|
||||
/*yaml
|
||||
Name: nttunpack_avx2_asm
|
||||
Description: x86_64 AVX2 NTT unpack from custom to bitreversed order
|
||||
Signature: void mlk_nttunpack_avx2_asm(int16_t *r)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read/write
|
||||
c_parameter: int16_t *r
|
||||
description: Input/output polynomial (256 x int16_t)
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/nttunpack_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(nttunpack_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(nttunpack_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
vmovdqa (%rdi), %ymm4
|
||||
vmovdqa 0x20(%rdi), %ymm5
|
||||
vmovdqa 0x40(%rdi), %ymm6
|
||||
vmovdqa 0x60(%rdi), %ymm7
|
||||
vmovdqa 0x80(%rdi), %ymm8
|
||||
vmovdqa 0xa0(%rdi), %ymm9
|
||||
vmovdqa 0xc0(%rdi), %ymm10
|
||||
vmovdqa 0xe0(%rdi), %ymm11
|
||||
vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3]
|
||||
vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1]
|
||||
vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3]
|
||||
vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1]
|
||||
vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3]
|
||||
vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3]
|
||||
vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2]
|
||||
vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3]
|
||||
vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
|
||||
vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3]
|
||||
vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2]
|
||||
vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3]
|
||||
vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7]
|
||||
vpsrlq $0x20, %ymm7, %ymm7
|
||||
vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7]
|
||||
vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7]
|
||||
vpsrlq $0x20, %ymm5, %ymm5
|
||||
vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7]
|
||||
vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7]
|
||||
vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7]
|
||||
vpsrlq $0x20, %ymm10, %ymm10
|
||||
vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7]
|
||||
vpslld $0x10, %ymm5, %ymm10
|
||||
vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
|
||||
vpsrld $0x10, %ymm9, %ymm9
|
||||
vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15]
|
||||
vpslld $0x10, %ymm4, %ymm9
|
||||
vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
|
||||
vpsrld $0x10, %ymm8, %ymm8
|
||||
vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15]
|
||||
vpslld $0x10, %ymm3, %ymm8
|
||||
vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15]
|
||||
vpsrld $0x10, %ymm7, %ymm7
|
||||
vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15]
|
||||
vpslld $0x10, %ymm11, %ymm7
|
||||
vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
|
||||
vmovdqa %ymm10, (%rdi)
|
||||
vmovdqa %ymm5, 0x20(%rdi)
|
||||
vmovdqa %ymm9, 0x40(%rdi)
|
||||
vmovdqa %ymm4, 0x60(%rdi)
|
||||
vmovdqa %ymm8, 0x80(%rdi)
|
||||
vmovdqa %ymm3, 0xa0(%rdi)
|
||||
vmovdqa %ymm7, 0xc0(%rdi)
|
||||
vmovdqa %ymm11, 0xe0(%rdi)
|
||||
vmovdqa 0x100(%rdi), %ymm4
|
||||
vmovdqa 0x120(%rdi), %ymm5
|
||||
vmovdqa 0x140(%rdi), %ymm6
|
||||
vmovdqa 0x160(%rdi), %ymm7
|
||||
vmovdqa 0x180(%rdi), %ymm8
|
||||
vmovdqa 0x1a0(%rdi), %ymm9
|
||||
vmovdqa 0x1c0(%rdi), %ymm10
|
||||
vmovdqa 0x1e0(%rdi), %ymm11
|
||||
vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1]
|
||||
vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3]
|
||||
vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1]
|
||||
vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3]
|
||||
vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1]
|
||||
vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3]
|
||||
vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1]
|
||||
vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3]
|
||||
vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2]
|
||||
vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3]
|
||||
vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2]
|
||||
vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3]
|
||||
vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
|
||||
vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3]
|
||||
vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2]
|
||||
vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3]
|
||||
vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7]
|
||||
vpsrlq $0x20, %ymm7, %ymm7
|
||||
vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7]
|
||||
vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7]
|
||||
vpsrlq $0x20, %ymm5, %ymm5
|
||||
vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7]
|
||||
vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7]
|
||||
vpsrlq $0x20, %ymm3, %ymm3
|
||||
vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7]
|
||||
vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6]
|
||||
vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7]
|
||||
vpsrlq $0x20, %ymm10, %ymm10
|
||||
vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7]
|
||||
vpslld $0x10, %ymm5, %ymm10
|
||||
vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
|
||||
vpsrld $0x10, %ymm9, %ymm9
|
||||
vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15]
|
||||
vpslld $0x10, %ymm4, %ymm9
|
||||
vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
|
||||
vpsrld $0x10, %ymm8, %ymm8
|
||||
vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15]
|
||||
vpslld $0x10, %ymm3, %ymm8
|
||||
vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15]
|
||||
vpsrld $0x10, %ymm7, %ymm7
|
||||
vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15]
|
||||
vpslld $0x10, %ymm11, %ymm7
|
||||
vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
|
||||
vpsrld $0x10, %ymm6, %ymm6
|
||||
vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
|
||||
vmovdqa %ymm10, 0x100(%rdi)
|
||||
vmovdqa %ymm5, 0x120(%rdi)
|
||||
vmovdqa %ymm9, 0x140(%rdi)
|
||||
vmovdqa %ymm4, 0x160(%rdi)
|
||||
vmovdqa %ymm8, 0x180(%rdi)
|
||||
vmovdqa %ymm3, 0x1a0(%rdi)
|
||||
vmovdqa %ymm7, 0x1c0(%rdi)
|
||||
vmovdqa %ymm11, 0x1e0(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(nttunpack_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
*/
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,410 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
/*************************************************
|
||||
* Name: mlk_poly_compress_d10_avx2_asm
|
||||
*
|
||||
* Description: Compression of a polynomial to 10 bits per coefficient.
|
||||
*
|
||||
* Arguments: - uint8_t *r: pointer to output byte array
|
||||
* (of length MLKEM_POLYCOMPRESSEDBYTES_D10)
|
||||
* - const int16_t *a: pointer to input polynomial
|
||||
* - const uint8_t *data: pointer to shufbidx constant
|
||||
**************************************************/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \
|
||||
(defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3)
|
||||
/*yaml
|
||||
Name: poly_compress_d10_avx2_asm
|
||||
Description: x86_64 AVX2 polynomial compression (d=10)
|
||||
Signature: void mlk_poly_compress_d10_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 320
|
||||
permissions: write-only
|
||||
c_parameter: uint8_t *r
|
||||
description: Output compressed polynomial
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *a
|
||||
description: Input polynomial (256 x int16_t)
|
||||
rdx:
|
||||
type: buffer
|
||||
size_bytes: 32
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t *data
|
||||
description: Precomputed compression constants
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/poly_compress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_compress_d10_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_compress_d10_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
vpsllw $0x3, %ymm0, %ymm1
|
||||
movl $0xf000f, %eax # imm = 0xF000F
|
||||
vmovd %eax, %xmm2
|
||||
vpbroadcastd %xmm2, %ymm2
|
||||
movl $0x10001000, %eax # imm = 0x10001000
|
||||
vmovd %eax, %xmm3
|
||||
vpbroadcastd %xmm3, %ymm3
|
||||
movl $0x3ff03ff, %eax # imm = 0x3FF03FF
|
||||
vmovd %eax, %xmm4
|
||||
vpbroadcastd %xmm4, %ymm4
|
||||
movabsq $0x400000104000001, %rax # imm = 0x400000104000001
|
||||
vmovq %rax, %xmm5
|
||||
vpbroadcastq %xmm5, %ymm5
|
||||
movl $0xc, %eax
|
||||
vmovq %rax, %xmm6
|
||||
vpbroadcastq %xmm6, %ymm6
|
||||
vmovdqa (%rdx), %ymm7
|
||||
vmovdqa (%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, (%rdi)
|
||||
vmovd %xmm9, 0x10(%rdi)
|
||||
vmovdqa 0x20(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x14(%rdi)
|
||||
vmovd %xmm9, 0x24(%rdi)
|
||||
vmovdqa 0x40(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x28(%rdi)
|
||||
vmovd %xmm9, 0x38(%rdi)
|
||||
vmovdqa 0x60(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x3c(%rdi)
|
||||
vmovd %xmm9, 0x4c(%rdi)
|
||||
vmovdqa 0x80(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x50(%rdi)
|
||||
vmovd %xmm9, 0x60(%rdi)
|
||||
vmovdqa 0xa0(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x64(%rdi)
|
||||
vmovd %xmm9, 0x74(%rdi)
|
||||
vmovdqa 0xc0(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x78(%rdi)
|
||||
vmovd %xmm9, 0x88(%rdi)
|
||||
vmovdqa 0xe0(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x8c(%rdi)
|
||||
vmovd %xmm9, 0x9c(%rdi)
|
||||
vmovdqa 0x100(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0xa0(%rdi)
|
||||
vmovd %xmm9, 0xb0(%rdi)
|
||||
vmovdqa 0x120(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0xb4(%rdi)
|
||||
vmovd %xmm9, 0xc4(%rdi)
|
||||
vmovdqa 0x140(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0xc8(%rdi)
|
||||
vmovd %xmm9, 0xd8(%rdi)
|
||||
vmovdqa 0x160(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0xdc(%rdi)
|
||||
vmovd %xmm9, 0xec(%rdi)
|
||||
vmovdqa 0x180(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0xf0(%rdi)
|
||||
vmovd %xmm9, 0x100(%rdi)
|
||||
vmovdqa 0x1a0(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x104(%rdi)
|
||||
vmovd %xmm9, 0x114(%rdi)
|
||||
vmovdqa 0x1c0(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x118(%rdi)
|
||||
vmovd %xmm9, 0x128(%rdi)
|
||||
vmovdqa 0x1e0(%rsi), %ymm8
|
||||
vpmullw %ymm1, %ymm8, %ymm9
|
||||
vpaddw %ymm2, %ymm8, %ymm10
|
||||
vpsllw $0x3, %ymm8, %ymm8
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpsubw %ymm10, %ymm9, %ymm10
|
||||
vpandn %ymm10, %ymm9, %ymm9
|
||||
vpsrlw $0xf, %ymm9, %ymm9
|
||||
vpsubw %ymm9, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm3, %ymm8, %ymm8
|
||||
vpand %ymm4, %ymm8, %ymm8
|
||||
vpmaddwd %ymm5, %ymm8, %ymm8
|
||||
vpsllvd %ymm6, %ymm8, %ymm8
|
||||
vpsrlq $0xc, %ymm8, %ymm8
|
||||
vpshufb %ymm7, %ymm8, %ymm8
|
||||
vextracti128 $0x1, %ymm8, %xmm9
|
||||
vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7]
|
||||
vmovdqu %xmm8, 0x12c(%rdi)
|
||||
vmovd %xmm9, 0x13c(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_compress_d10_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
&& (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == \
|
||||
3) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,476 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
/*************************************************
|
||||
* Name: mlk_poly_compress_d11_avx2_asm
|
||||
*
|
||||
* Description: Compression of a polynomial to 11 bits per coefficient.
|
||||
*
|
||||
* Arguments: - uint8_t *r: pointer to output byte array
|
||||
* (of length MLKEM_POLYCOMPRESSEDBYTES_D11)
|
||||
* - const int16_t *a: pointer to input polynomial
|
||||
* - const uint8_t *data: pointer to constants
|
||||
* (srlvqidx[0:32], shufbidx[32:64])
|
||||
**************************************************/
|
||||
|
||||
#include "../../../common.h"
|
||||
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \
|
||||
(defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4)
|
||||
/*yaml
|
||||
Name: poly_compress_d11_avx2_asm
|
||||
Description: x86_64 AVX2 polynomial compression (d=11)
|
||||
Signature: void mlk_poly_compress_d11_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 352
|
||||
permissions: write-only
|
||||
c_parameter: uint8_t *r
|
||||
description: Output compressed polynomial
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *a
|
||||
description: Input polynomial (256 x int16_t)
|
||||
rdx:
|
||||
type: buffer
|
||||
size_bytes: 64
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t *data
|
||||
description: Precomputed compression constants
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/poly_compress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_compress_d11_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_compress_d11_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
vpsllw $0x3, %ymm0, %ymm1
|
||||
movl $0x240024, %eax # imm = 0x240024
|
||||
vmovd %eax, %xmm2
|
||||
vpbroadcastd %xmm2, %ymm2
|
||||
movl $0x20002000, %eax # imm = 0x20002000
|
||||
vmovd %eax, %xmm3
|
||||
vpbroadcastd %xmm3, %ymm3
|
||||
movl $0x7ff07ff, %eax # imm = 0x7FF07FF
|
||||
vmovd %eax, %xmm4
|
||||
vpbroadcastd %xmm4, %ymm4
|
||||
movabsq $0x800000108000001, %rax # imm = 0x800000108000001
|
||||
vmovq %rax, %xmm5
|
||||
vpbroadcastq %xmm5, %ymm5
|
||||
movl $0xa, %eax
|
||||
vmovq %rax, %xmm6
|
||||
vpbroadcastq %xmm6, %ymm6
|
||||
vmovdqa (%rdx), %ymm7
|
||||
vmovdqa 0x20(%rdx), %ymm8
|
||||
vmovdqa (%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, (%rdi)
|
||||
vmovd %xmm10, 0x10(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x14(%rdi)
|
||||
vmovdqa 0x20(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x16(%rdi)
|
||||
vmovd %xmm10, 0x26(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x2a(%rdi)
|
||||
vmovdqa 0x40(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x2c(%rdi)
|
||||
vmovd %xmm10, 0x3c(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x40(%rdi)
|
||||
vmovdqa 0x60(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x42(%rdi)
|
||||
vmovd %xmm10, 0x52(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x56(%rdi)
|
||||
vmovdqa 0x80(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x58(%rdi)
|
||||
vmovd %xmm10, 0x68(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x6c(%rdi)
|
||||
vmovdqa 0xa0(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x6e(%rdi)
|
||||
vmovd %xmm10, 0x7e(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x82(%rdi)
|
||||
vmovdqa 0xc0(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x84(%rdi)
|
||||
vmovd %xmm10, 0x94(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x98(%rdi)
|
||||
vmovdqa 0xe0(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x9a(%rdi)
|
||||
vmovd %xmm10, 0xaa(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0xae(%rdi)
|
||||
vmovdqa 0x100(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0xb0(%rdi)
|
||||
vmovd %xmm10, 0xc0(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0xc4(%rdi)
|
||||
vmovdqa 0x120(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0xc6(%rdi)
|
||||
vmovd %xmm10, 0xd6(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0xda(%rdi)
|
||||
vmovdqa 0x140(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0xdc(%rdi)
|
||||
vmovd %xmm10, 0xec(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0xf0(%rdi)
|
||||
vmovdqa 0x160(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0xf2(%rdi)
|
||||
vmovd %xmm10, 0x102(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x106(%rdi)
|
||||
vmovdqa 0x180(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x108(%rdi)
|
||||
vmovd %xmm10, 0x118(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x11c(%rdi)
|
||||
vmovdqa 0x1a0(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x11e(%rdi)
|
||||
vmovd %xmm10, 0x12e(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x132(%rdi)
|
||||
vmovdqa 0x1c0(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x134(%rdi)
|
||||
vmovd %xmm10, 0x144(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x148(%rdi)
|
||||
vmovdqa 0x1e0(%rsi), %ymm9
|
||||
vpmullw %ymm1, %ymm9, %ymm10
|
||||
vpaddw %ymm2, %ymm9, %ymm11
|
||||
vpsllw $0x3, %ymm9, %ymm9
|
||||
vpmulhw %ymm0, %ymm9, %ymm9
|
||||
vpsubw %ymm11, %ymm10, %ymm11
|
||||
vpandn %ymm11, %ymm10, %ymm10
|
||||
vpsrlw $0xf, %ymm10, %ymm10
|
||||
vpsubw %ymm10, %ymm9, %ymm9
|
||||
vpmulhrsw %ymm3, %ymm9, %ymm9
|
||||
vpand %ymm4, %ymm9, %ymm9
|
||||
vpmaddwd %ymm5, %ymm9, %ymm9
|
||||
vpsllvd %ymm6, %ymm9, %ymm9
|
||||
vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
|
||||
vpsrlvq %ymm7, %ymm9, %ymm9
|
||||
vpsllq $0x22, %ymm10, %ymm10
|
||||
vpaddq %ymm10, %ymm9, %ymm9
|
||||
vpshufb %ymm8, %ymm9, %ymm9
|
||||
vextracti128 $0x1, %ymm9, %xmm10
|
||||
vpblendvb %xmm8, %xmm10, %xmm9, %xmm9
|
||||
vmovdqu %xmm9, 0x14a(%rdi)
|
||||
vmovd %xmm10, 0x15a(%rdi)
|
||||
vpextrw $0x2, %xmm10, 0x15e(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_compress_d11_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
&& (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,191 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
/*************************************************
|
||||
* Name: mlk_poly_compress_d4_avx2_asm
|
||||
*
|
||||
* Description: Compression of a polynomial to 4 bits per coefficient.
|
||||
*
|
||||
* Arguments: - uint8_t *r: pointer to output byte array
|
||||
* (of length MLKEM_POLYCOMPRESSEDBYTES_D4)
|
||||
* - const int16_t *a: pointer to input polynomial
|
||||
* - const uint8_t *data: pointer to permdidx constant
|
||||
**************************************************/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \
|
||||
(defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3)
|
||||
/*yaml
|
||||
Name: poly_compress_d4_avx2_asm
|
||||
Description: x86_64 AVX2 polynomial compression (d=4)
|
||||
Signature: void mlk_poly_compress_d4_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 128
|
||||
permissions: write-only
|
||||
c_parameter: uint8_t *r
|
||||
description: Output compressed polynomial
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *a
|
||||
description: Input polynomial (256 x int16_t)
|
||||
rdx:
|
||||
type: buffer
|
||||
size_bytes: 32
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t *data
|
||||
description: Precomputed compression constants
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/poly_compress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_compress_d4_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_compress_d4_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
movl $0x2000200, %eax # imm = 0x2000200
|
||||
vmovd %eax, %xmm1
|
||||
vpbroadcastd %xmm1, %ymm1
|
||||
movl $0xf000f, %eax # imm = 0xF000F
|
||||
vmovd %eax, %xmm2
|
||||
vpbroadcastd %xmm2, %ymm2
|
||||
movl $0x10011001, %eax # imm = 0x10011001
|
||||
vmovd %eax, %xmm3
|
||||
vpbroadcastd %xmm3, %ymm3
|
||||
vmovdqa (%rdx), %ymm4
|
||||
vmovdqa (%rsi), %ymm5
|
||||
vmovdqa 0x20(%rsi), %ymm6
|
||||
vmovdqa 0x40(%rsi), %ymm7
|
||||
vmovdqa 0x60(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm5, %ymm5
|
||||
vpmulhrsw %ymm1, %ymm6, %ymm6
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm5, %ymm5
|
||||
vpand %ymm2, %ymm6, %ymm6
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm6, %ymm5, %ymm5
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm5, %ymm5
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpackuswb %ymm7, %ymm5, %ymm5
|
||||
vpermd %ymm5, %ymm4, %ymm5
|
||||
vmovdqu %ymm5, (%rdi)
|
||||
vmovdqa 0x80(%rsi), %ymm5
|
||||
vmovdqa 0xa0(%rsi), %ymm6
|
||||
vmovdqa 0xc0(%rsi), %ymm7
|
||||
vmovdqa 0xe0(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm5, %ymm5
|
||||
vpmulhrsw %ymm1, %ymm6, %ymm6
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm5, %ymm5
|
||||
vpand %ymm2, %ymm6, %ymm6
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm6, %ymm5, %ymm5
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm5, %ymm5
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpackuswb %ymm7, %ymm5, %ymm5
|
||||
vpermd %ymm5, %ymm4, %ymm5
|
||||
vmovdqu %ymm5, 0x20(%rdi)
|
||||
vmovdqa 0x100(%rsi), %ymm5
|
||||
vmovdqa 0x120(%rsi), %ymm6
|
||||
vmovdqa 0x140(%rsi), %ymm7
|
||||
vmovdqa 0x160(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm5, %ymm5
|
||||
vpmulhrsw %ymm1, %ymm6, %ymm6
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm5, %ymm5
|
||||
vpand %ymm2, %ymm6, %ymm6
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm6, %ymm5, %ymm5
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm5, %ymm5
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpackuswb %ymm7, %ymm5, %ymm5
|
||||
vpermd %ymm5, %ymm4, %ymm5
|
||||
vmovdqu %ymm5, 0x40(%rdi)
|
||||
vmovdqa 0x180(%rsi), %ymm5
|
||||
vmovdqa 0x1a0(%rsi), %ymm6
|
||||
vmovdqa 0x1c0(%rsi), %ymm7
|
||||
vmovdqa 0x1e0(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm5, %ymm5
|
||||
vpmulhw %ymm0, %ymm6, %ymm6
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm5, %ymm5
|
||||
vpmulhrsw %ymm1, %ymm6, %ymm6
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm5, %ymm5
|
||||
vpand %ymm2, %ymm6, %ymm6
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm6, %ymm5, %ymm5
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm5, %ymm5
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpackuswb %ymm7, %ymm5, %ymm5
|
||||
vpermd %ymm5, %ymm4, %ymm5
|
||||
vmovdqu %ymm5, 0x60(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_compress_d4_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
&& (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == \
|
||||
3) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
@@ -0,0 +1,248 @@
|
||||
/*
|
||||
* Copyright (c) The mlkem-native project authors
|
||||
* SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
|
||||
*/
|
||||
|
||||
/* References
|
||||
* ==========
|
||||
*
|
||||
* - [REF_AVX2]
|
||||
* CRYSTALS-Kyber optimized AVX2 implementation
|
||||
* Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé
|
||||
* https://github.com/pq-crystals/kyber/tree/main/avx2
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is derived from the public domain
|
||||
* AVX2 Kyber implementation @[REF_AVX2].
|
||||
*/
|
||||
|
||||
/*************************************************
|
||||
* Name: mlk_poly_compress_d5_avx2_asm
|
||||
*
|
||||
* Description: Compression of a polynomial to 5 bits per coefficient.
|
||||
*
|
||||
* Arguments: - uint8_t *r: pointer to output byte array
|
||||
* (of length MLKEM_POLYCOMPRESSEDBYTES_D5)
|
||||
* - const int16_t *a: pointer to input polynomial
|
||||
* - const uint8_t *data: pointer to shufbidx constant
|
||||
**************************************************/
|
||||
|
||||
#include "../../../common.h"
|
||||
#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \
|
||||
!defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \
|
||||
(defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4)
|
||||
/*yaml
|
||||
Name: poly_compress_d5_avx2_asm
|
||||
Description: x86_64 AVX2 polynomial compression (d=5)
|
||||
Signature: void mlk_poly_compress_d5_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data)
|
||||
ABI:
|
||||
Architecture: x86_64
|
||||
CallingConvention: SysV
|
||||
Features: [AVX2]
|
||||
rdi:
|
||||
type: buffer
|
||||
size_bytes: 160
|
||||
permissions: write-only
|
||||
c_parameter: uint8_t *r
|
||||
description: Output compressed polynomial
|
||||
rsi:
|
||||
type: buffer
|
||||
size_bytes: 512
|
||||
permissions: read-only
|
||||
c_parameter: const int16_t *a
|
||||
description: Input polynomial (256 x int16_t)
|
||||
rdx:
|
||||
type: buffer
|
||||
size_bytes: 32
|
||||
permissions: read-only
|
||||
c_parameter: const uint8_t *data
|
||||
description: Precomputed compression constants
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* WARNING: This file is auto-derived from the mlkem-native source file
|
||||
* dev/x86_64/src/poly_compress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly.
|
||||
*/
|
||||
|
||||
.text
|
||||
.balign 4
|
||||
.global MLK_ASM_NAMESPACE(poly_compress_d5_avx2_asm)
|
||||
MLK_ASM_FN_SYMBOL(poly_compress_d5_avx2_asm)
|
||||
|
||||
.cfi_startproc
|
||||
movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF
|
||||
vmovd %eax, %xmm0
|
||||
vpbroadcastd %xmm0, %ymm0
|
||||
movl $0x4000400, %eax # imm = 0x4000400
|
||||
vmovd %eax, %xmm1
|
||||
vpbroadcastd %xmm1, %ymm1
|
||||
movl $0x1f001f, %eax # imm = 0x1F001F
|
||||
vmovd %eax, %xmm2
|
||||
vpbroadcastd %xmm2, %ymm2
|
||||
movl $0x20012001, %eax # imm = 0x20012001
|
||||
vmovd %eax, %xmm3
|
||||
vpbroadcastd %xmm3, %ymm3
|
||||
movl $0x4000001, %eax # imm = 0x4000001
|
||||
vmovd %eax, %xmm4
|
||||
vpbroadcastd %xmm4, %ymm4
|
||||
movl $0xc, %eax
|
||||
vmovq %rax, %xmm5
|
||||
vpbroadcastq %xmm5, %ymm5
|
||||
vmovdqa (%rdx), %ymm6
|
||||
vmovdqa (%rsi), %ymm7
|
||||
vmovdqa 0x20(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, (%rdi)
|
||||
vmovd %xmm8, 0x10(%rdi)
|
||||
vmovdqa 0x40(%rsi), %ymm7
|
||||
vmovdqa 0x60(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x14(%rdi)
|
||||
vmovd %xmm8, 0x24(%rdi)
|
||||
vmovdqa 0x80(%rsi), %ymm7
|
||||
vmovdqa 0xa0(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x28(%rdi)
|
||||
vmovd %xmm8, 0x38(%rdi)
|
||||
vmovdqa 0xc0(%rsi), %ymm7
|
||||
vmovdqa 0xe0(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x3c(%rdi)
|
||||
vmovd %xmm8, 0x4c(%rdi)
|
||||
vmovdqa 0x100(%rsi), %ymm7
|
||||
vmovdqa 0x120(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x50(%rdi)
|
||||
vmovd %xmm8, 0x60(%rdi)
|
||||
vmovdqa 0x140(%rsi), %ymm7
|
||||
vmovdqa 0x160(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x64(%rdi)
|
||||
vmovd %xmm8, 0x74(%rdi)
|
||||
vmovdqa 0x180(%rsi), %ymm7
|
||||
vmovdqa 0x1a0(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x78(%rdi)
|
||||
vmovd %xmm8, 0x88(%rdi)
|
||||
vmovdqa 0x1c0(%rsi), %ymm7
|
||||
vmovdqa 0x1e0(%rsi), %ymm8
|
||||
vpmulhw %ymm0, %ymm7, %ymm7
|
||||
vpmulhw %ymm0, %ymm8, %ymm8
|
||||
vpmulhrsw %ymm1, %ymm7, %ymm7
|
||||
vpmulhrsw %ymm1, %ymm8, %ymm8
|
||||
vpand %ymm2, %ymm7, %ymm7
|
||||
vpand %ymm2, %ymm8, %ymm8
|
||||
vpackuswb %ymm8, %ymm7, %ymm7
|
||||
vpmaddubsw %ymm3, %ymm7, %ymm7
|
||||
vpmaddwd %ymm4, %ymm7, %ymm7
|
||||
vpsllvd %ymm5, %ymm7, %ymm7
|
||||
vpsrlvq %ymm5, %ymm7, %ymm7
|
||||
vpshufb %ymm6, %ymm7, %ymm7
|
||||
vextracti128 $0x1, %ymm7, %xmm8
|
||||
vpblendvb %xmm6, %xmm8, %xmm7, %xmm7
|
||||
vmovdqu %xmm7, 0x8c(%rdi)
|
||||
vmovd %xmm8, 0x9c(%rdi)
|
||||
retq
|
||||
.cfi_endproc
|
||||
|
||||
MLK_ASM_FN_SIZE(poly_compress_d5_avx2_asm)
|
||||
|
||||
#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \
|
||||
&& (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */
|
||||
|
||||
#if defined(__ELF__)
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
#endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user