blob: 9a82731f0e9bce70e1a02cb05abe12cf34b5cea5 [file] [log] [blame]
Jerry Yu49231312023-01-10 16:57:21 +08001/*
Dave Rodgmanf918d422023-03-17 17:52:23 +00002 * Armv8-A Cryptographic Extension support functions for Aarch64
Jerry Yu49231312023-01-10 16:57:21 +08003 *
4 * Copyright The Mbed TLS Contributors
5 * SPDX-License-Identifier: Apache-2.0
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License"); you may
8 * not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19
Jerry Yu48b999c2023-03-03 15:51:07 +080020#if defined(__aarch64__) && !defined(__ARM_FEATURE_CRYPTO) && \
Jerry Yu6f86c192023-03-13 11:03:40 +080021 defined(__clang__) && __clang_major__ >= 4
Jerry Yu48b999c2023-03-03 15:51:07 +080022/* TODO: Re-consider above after https://reviews.llvm.org/D131064 merged.
23 *
24 * The intrinsic declaration are guarded by predefined ACLE macros in clang:
25 * these are normally only enabled by the -march option on the command line.
26 * By defining the macros ourselves we gain access to those declarations without
27 * requiring -march on the command line.
28 *
29 * `arm_neon.h` could be included by any header file, so we put these defines
30 * at the top of this file, before any includes.
31 */
32#define __ARM_FEATURE_CRYPTO 1
Jerry Yuae129c32023-03-03 15:55:56 +080033/* See: https://arm-software.github.io/acle/main/acle.html#cryptographic-extensions
34 *
Jerry Yu490bf082023-03-06 15:21:44 +080035 * `__ARM_FEATURE_CRYPTO` is deprecated, but we need to continue to specify it
36 * for older compilers.
Jerry Yuae129c32023-03-03 15:55:56 +080037 */
38#define __ARM_FEATURE_AES 1
Dave Rodgmandb6ab242023-03-14 16:03:57 +000039#define MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG
Jerry Yu490bf082023-03-06 15:21:44 +080040#endif
Jerry Yu48b999c2023-03-03 15:51:07 +080041
Jerry Yu49231312023-01-10 16:57:21 +080042#include <string.h>
43#include "common.h"
44
45#if defined(MBEDTLS_AESCE_C)
46
47#include "aesce.h"
48
Jerry Yu72fd0bd2023-08-18 16:31:01 +080049#if defined(MBEDTLS_ARCH_IS_ARM64)
Jerry Yu49231312023-01-10 16:57:21 +080050
Jerry Yu61c4cfa2023-04-26 11:06:51 +080051/* Compiler version checks. */
Jerry Yudb368de2023-04-26 16:55:37 +080052#if defined(__clang__)
53# if __clang_major__ < 4
54# error "Minimum version of Clang for MBEDTLS_AESCE_C is 4.0."
55# endif
56#elif defined(__GNUC__)
57# if __GNUC__ < 6
58# error "Minimum version of GCC for MBEDTLS_AESCE_C is 6.0."
59# endif
60#elif defined(_MSC_VER)
Jerry Yu61c4cfa2023-04-26 11:06:51 +080061/* TODO: We haven't verified MSVC from 1920 to 1928. If someone verified that,
62 * please update this and document of `MBEDTLS_AESCE_C` in
63 * `mbedtls_config.h`. */
Jerry Yudb368de2023-04-26 16:55:37 +080064# if _MSC_VER < 1929
65# error "Minimum version of MSVC for MBEDTLS_AESCE_C is 2019 version 16.11.2."
66# endif
Jerry Yu61c4cfa2023-04-26 11:06:51 +080067#endif
68
Jerry Yu6b00f5a2023-05-04 16:30:21 +080069#ifdef __ARM_NEON
Jerry Yu08933d32023-04-27 18:28:00 +080070#include <arm_neon.h>
Jerry Yu6b00f5a2023-05-04 16:30:21 +080071#else
72#error "Target does not support NEON instructions"
73#endif
Jerry Yu08933d32023-04-27 18:28:00 +080074
Jerry Yu580e06f2023-04-28 17:42:40 +080075#if !(defined(__ARM_FEATURE_CRYPTO) || defined(__ARM_FEATURE_AES)) || \
76 defined(MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG)
Jerry Yub1d06bb2023-05-05 14:05:07 +080077# if defined(__ARMCOMPILER_VERSION)
78# if __ARMCOMPILER_VERSION <= 6090000
79# error "Must use minimum -march=armv8-a+crypto for MBEDTLS_AESCE_C"
80# else
Jerry Yu893be8d2023-07-13 17:32:11 +080081# pragma clang attribute push (__attribute__((target("aes"))), apply_to=function)
Jerry Yub1d06bb2023-05-05 14:05:07 +080082# define MBEDTLS_POP_TARGET_PRAGMA
83# endif
84# elif defined(__clang__)
Jerry Yu893be8d2023-07-13 17:32:11 +080085# pragma clang attribute push (__attribute__((target("aes"))), apply_to=function)
Jerry Yuec9be842023-03-14 10:42:47 +080086# define MBEDTLS_POP_TARGET_PRAGMA
87# elif defined(__GNUC__)
Jerry Yuec9be842023-03-14 10:42:47 +080088# pragma GCC push_options
Beniamin Sandu471a9752023-06-25 20:16:16 +030089# pragma GCC target ("+crypto")
Jerry Yuec9be842023-03-14 10:42:47 +080090# define MBEDTLS_POP_TARGET_PRAGMA
Jerry Yu07d28d82023-03-20 18:12:36 +080091# elif defined(_MSC_VER)
Jerry Yu61c4cfa2023-04-26 11:06:51 +080092# error "Required feature(__ARM_FEATURE_AES) is not enabled."
Jerry Yu49231312023-01-10 16:57:21 +080093# endif
Jerry Yu580e06f2023-04-28 17:42:40 +080094#endif /* !(__ARM_FEATURE_CRYPTO || __ARM_FEATURE_AES) ||
95 MBEDTLS_ENABLE_ARM_CRYPTO_EXTENSIONS_COMPILER_FLAG */
Jerry Yu49231312023-01-10 16:57:21 +080096
Dave Rodgman45661322023-08-04 12:31:58 +010097#if defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
98
Jerry Yub95c7762023-01-10 16:59:51 +080099#include <asm/hwcap.h>
100#include <sys/auxv.h>
Dave Rodgman45661322023-08-04 12:31:58 +0100101
Dave Rodgmanb30adce2023-08-04 12:52:51 +0100102signed char mbedtls_aesce_has_support_result = -1;
Jerry Yub95c7762023-01-10 16:59:51 +0800103
Jerry Yu36606232023-04-19 10:44:29 +0800104#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
Jerry Yub95c7762023-01-10 16:59:51 +0800105/*
106 * AES instruction support detection routine
107 */
Dave Rodgman45661322023-08-04 12:31:58 +0100108int mbedtls_aesce_has_support_impl(void)
Jerry Yub95c7762023-01-10 16:59:51 +0800109{
Dave Rodgman45661322023-08-04 12:31:58 +0100110 /* To avoid many calls to getauxval, cache the result. This is
111 * thread-safe, because we store the result in a char so cannot
112 * be vulnerable to non-atomic updates.
113 * It is possible that we could end up setting result more than
114 * once, but that is harmless.
115 */
Dave Rodgmanb30adce2023-08-04 12:52:51 +0100116 if (mbedtls_aesce_has_support_result == -1) {
Dave Rodgman45661322023-08-04 12:31:58 +0100117 unsigned long auxval = getauxval(AT_HWCAP);
118 if ((auxval & (HWCAP_ASIMD | HWCAP_AES)) ==
119 (HWCAP_ASIMD | HWCAP_AES)) {
120 mbedtls_aesce_has_support_result = 1;
121 } else {
122 mbedtls_aesce_has_support_result = 0;
123 }
124 }
125 return mbedtls_aesce_has_support_result;
Jerry Yub95c7762023-01-10 16:59:51 +0800126}
Jerry Yu0d4f4e52023-03-31 14:32:47 +0800127#endif
Jerry Yub95c7762023-01-10 16:59:51 +0800128
Dave Rodgman45661322023-08-04 12:31:58 +0100129#endif /* defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY) */
130
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100131/* Single round of AESCE encryption */
132#define AESCE_ENCRYPT_ROUND \
133 block = vaeseq_u8(block, vld1q_u8(keys)); \
134 block = vaesmcq_u8(block); \
135 keys += 16
136/* Two rounds of AESCE encryption */
137#define AESCE_ENCRYPT_ROUND_X2 AESCE_ENCRYPT_ROUND; AESCE_ENCRYPT_ROUND
138
Dave Rodgman9bb7e6f2023-06-16 09:41:21 +0100139MBEDTLS_OPTIMIZE_FOR_PERFORMANCE
Jerry Yu2bb3d812023-01-10 17:38:26 +0800140static uint8x16_t aesce_encrypt_block(uint8x16_t block,
141 unsigned char *keys,
142 int rounds)
143{
Dave Rodgman73b0c0b2023-06-16 14:48:14 +0100144 /* 10, 12 or 14 rounds. Unroll loop. */
Dave Rodgman96fdfb82023-06-15 16:21:31 +0100145 if (rounds == 10) {
146 goto rounds_10;
Jerry Yu2bb3d812023-01-10 17:38:26 +0800147 }
Dave Rodgman96fdfb82023-06-15 16:21:31 +0100148 if (rounds == 12) {
149 goto rounds_12;
150 }
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100151 AESCE_ENCRYPT_ROUND_X2;
Dave Rodgman96fdfb82023-06-15 16:21:31 +0100152rounds_12:
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100153 AESCE_ENCRYPT_ROUND_X2;
Dave Rodgman96fdfb82023-06-15 16:21:31 +0100154rounds_10:
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100155 AESCE_ENCRYPT_ROUND_X2;
156 AESCE_ENCRYPT_ROUND_X2;
157 AESCE_ENCRYPT_ROUND_X2;
158 AESCE_ENCRYPT_ROUND_X2;
159 AESCE_ENCRYPT_ROUND;
Jerry Yu2bb3d812023-01-10 17:38:26 +0800160
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800161 /* AES AddRoundKey for the previous round.
162 * SubBytes, ShiftRows for the final round. */
Dave Rodgman96fdfb82023-06-15 16:21:31 +0100163 block = vaeseq_u8(block, vld1q_u8(keys));
164 keys += 16;
Jerry Yu2bb3d812023-01-10 17:38:26 +0800165
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800166 /* Final round: no MixColumns */
Jerry Yu3304c202023-02-22 14:37:11 +0800167
168 /* Final AddRoundKey */
Dave Rodgman96fdfb82023-06-15 16:21:31 +0100169 block = veorq_u8(block, vld1q_u8(keys));
Jerry Yu2bb3d812023-01-10 17:38:26 +0800170
171 return block;
172}
173
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100174/* Single round of AESCE decryption
175 *
176 * AES AddRoundKey, SubBytes, ShiftRows
177 *
178 * block = vaesdq_u8(block, vld1q_u8(keys));
179 *
180 * AES inverse MixColumns for the next round.
181 *
182 * This means that we switch the order of the inverse AddRoundKey and
183 * inverse MixColumns operations. We have to do this as AddRoundKey is
184 * done in an atomic instruction together with the inverses of SubBytes
185 * and ShiftRows.
186 *
187 * It works because MixColumns is a linear operation over GF(2^8) and
188 * AddRoundKey is an exclusive or, which is equivalent to addition over
189 * GF(2^8). (The inverse of MixColumns needs to be applied to the
190 * affected round keys separately which has been done when the
191 * decryption round keys were calculated.)
192 *
193 * block = vaesimcq_u8(block);
194 */
195#define AESCE_DECRYPT_ROUND \
196 block = vaesdq_u8(block, vld1q_u8(keys)); \
197 block = vaesimcq_u8(block); \
198 keys += 16
199/* Two rounds of AESCE decryption */
200#define AESCE_DECRYPT_ROUND_X2 AESCE_DECRYPT_ROUND; AESCE_DECRYPT_ROUND
201
Yanray Wangb67b4742023-10-31 17:10:32 +0800202#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
Jerry Yu2bb3d812023-01-10 17:38:26 +0800203static uint8x16_t aesce_decrypt_block(uint8x16_t block,
204 unsigned char *keys,
205 int rounds)
206{
Dave Rodgman73b0c0b2023-06-16 14:48:14 +0100207 /* 10, 12 or 14 rounds. Unroll loop. */
Dave Rodgman1c4451d2023-06-15 16:28:00 +0100208 if (rounds == 10) {
209 goto rounds_10;
Jerry Yu2bb3d812023-01-10 17:38:26 +0800210 }
Dave Rodgman1c4451d2023-06-15 16:28:00 +0100211 if (rounds == 12) {
212 goto rounds_12;
213 }
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100214 AESCE_DECRYPT_ROUND_X2;
Dave Rodgman1c4451d2023-06-15 16:28:00 +0100215rounds_12:
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100216 AESCE_DECRYPT_ROUND_X2;
Dave Rodgman1c4451d2023-06-15 16:28:00 +0100217rounds_10:
Dave Rodgman48fd2ab2023-06-16 09:36:50 +0100218 AESCE_DECRYPT_ROUND_X2;
219 AESCE_DECRYPT_ROUND_X2;
220 AESCE_DECRYPT_ROUND_X2;
221 AESCE_DECRYPT_ROUND_X2;
222 AESCE_DECRYPT_ROUND;
Jerry Yu2bb3d812023-01-10 17:38:26 +0800223
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800224 /* The inverses of AES AddRoundKey, SubBytes, ShiftRows finishing up the
225 * last full round. */
Dave Rodgman1c4451d2023-06-15 16:28:00 +0100226 block = vaesdq_u8(block, vld1q_u8(keys));
227 keys += 16;
Jerry Yu2bb3d812023-01-10 17:38:26 +0800228
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800229 /* Inverse AddRoundKey for inverting the initial round key addition. */
Dave Rodgman1c4451d2023-06-15 16:28:00 +0100230 block = veorq_u8(block, vld1q_u8(keys));
Jerry Yu2bb3d812023-01-10 17:38:26 +0800231
232 return block;
233}
Yanray Wang590c9b72023-08-28 15:40:23 +0800234#endif
Jerry Yu2bb3d812023-01-10 17:38:26 +0800235
236/*
237 * AES-ECB block en(de)cryption
238 */
239int mbedtls_aesce_crypt_ecb(mbedtls_aes_context *ctx,
240 int mode,
241 const unsigned char input[16],
242 unsigned char output[16])
243{
244 uint8x16_t block = vld1q_u8(&input[0]);
245 unsigned char *keys = (unsigned char *) (ctx->buf + ctx->rk_offset);
246
Yanray Wang0d76b6e2023-11-02 11:54:39 +0800247#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
Yanray Wang111159b2023-11-10 13:41:12 +0800248 if (mode == MBEDTLS_AES_DECRYPT) {
Jerry Yu2bb3d812023-01-10 17:38:26 +0800249 block = aesce_decrypt_block(block, keys, ctx->nr);
Yanray Wang111159b2023-11-10 13:41:12 +0800250 } else
Yanray Wang0d76b6e2023-11-02 11:54:39 +0800251#endif
Yanray Wang111159b2023-11-10 13:41:12 +0800252 {
253 block = aesce_encrypt_block(block, keys, ctx->nr);
Yanray Wang0d76b6e2023-11-02 11:54:39 +0800254 }
Jerry Yu2bb3d812023-01-10 17:38:26 +0800255 vst1q_u8(&output[0], block);
256
257 return 0;
258}
259
Jerry Yue096da12023-01-10 17:07:01 +0800260/*
261 * Compute decryption round keys from encryption round keys
262 */
Yanray Wangb67b4742023-10-31 17:10:32 +0800263#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
Jerry Yue096da12023-01-10 17:07:01 +0800264void mbedtls_aesce_inverse_key(unsigned char *invkey,
265 const unsigned char *fwdkey,
266 int nr)
267{
268 int i, j;
269 j = nr;
270 vst1q_u8(invkey, vld1q_u8(fwdkey + j * 16));
271 for (i = 1, j--; j > 0; i++, j--) {
272 vst1q_u8(invkey + i * 16,
273 vaesimcq_u8(vld1q_u8(fwdkey + j * 16)));
274 }
275 vst1q_u8(invkey + i * 16, vld1q_u8(fwdkey + j * 16));
276
277}
Yanray Wang590c9b72023-08-28 15:40:23 +0800278#endif
Jerry Yue096da12023-01-10 17:07:01 +0800279
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800280static inline uint32_t aes_rot_word(uint32_t word)
Jerry Yu3f2fb712023-01-10 17:05:42 +0800281{
282 return (word << (32 - 8)) | (word >> 8);
283}
284
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800285static inline uint32_t aes_sub_word(uint32_t in)
Jerry Yu3f2fb712023-01-10 17:05:42 +0800286{
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800287 uint8x16_t v = vreinterpretq_u8_u32(vdupq_n_u32(in));
Jerry Yu3f2fb712023-01-10 17:05:42 +0800288 uint8x16_t zero = vdupq_n_u8(0);
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800289
290 /* vaeseq_u8 does both SubBytes and ShiftRows. Taking the first row yields
291 * the correct result as ShiftRows doesn't change the first row. */
292 v = vaeseq_u8(zero, v);
293 return vgetq_lane_u32(vreinterpretq_u32_u8(v), 0);
Jerry Yu3f2fb712023-01-10 17:05:42 +0800294}
295
296/*
Jerry Yubaae4012023-02-21 15:26:13 +0800297 * Key expansion function
Jerry Yu3f2fb712023-01-10 17:05:42 +0800298 */
Jerry Yubaae4012023-02-21 15:26:13 +0800299static void aesce_setkey_enc(unsigned char *rk,
300 const unsigned char *key,
301 const size_t key_bit_length)
Jerry Yu3f2fb712023-01-10 17:05:42 +0800302{
Jerry Yubaae4012023-02-21 15:26:13 +0800303 static uint8_t const rcon[] = { 0x01, 0x02, 0x04, 0x08, 0x10,
304 0x20, 0x40, 0x80, 0x1b, 0x36 };
Jerry Yu947bf962023-02-23 11:07:57 +0800305 /* See https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.197.pdf
306 * - Section 5, Nr = Nk + 6
Jerry Yu2c266512023-03-01 11:18:20 +0800307 * - Section 5.2, the length of round keys is Nb*(Nr+1)
Jerry Yu947bf962023-02-23 11:07:57 +0800308 */
309 const uint32_t key_len_in_words = key_bit_length / 32; /* Nk */
310 const size_t round_key_len_in_words = 4; /* Nb */
Jerry Yu2c266512023-03-01 11:18:20 +0800311 const size_t rounds_needed = key_len_in_words + 6; /* Nr */
312 const size_t round_keys_len_in_words =
313 round_key_len_in_words * (rounds_needed + 1); /* Nb*(Nr+1) */
314 const uint32_t *rko_end = (uint32_t *) rk + round_keys_len_in_words;
Jerry Yuc8bcdc82023-02-21 14:49:02 +0800315
Jerry Yu3304c202023-02-22 14:37:11 +0800316 memcpy(rk, key, key_len_in_words * 4);
Jerry Yu3f2fb712023-01-10 17:05:42 +0800317
Jerry Yu3304c202023-02-22 14:37:11 +0800318 for (uint32_t *rki = (uint32_t *) rk;
319 rki + key_len_in_words < rko_end;
320 rki += key_len_in_words) {
321
Jerry Yufac5a542023-02-23 10:13:40 +0800322 size_t iteration = (rki - (uint32_t *) rk) / key_len_in_words;
Jerry Yu3304c202023-02-22 14:37:11 +0800323 uint32_t *rko;
Jerry Yubaae4012023-02-21 15:26:13 +0800324 rko = rki + key_len_in_words;
325 rko[0] = aes_rot_word(aes_sub_word(rki[key_len_in_words - 1]));
Jerry Yu3304c202023-02-22 14:37:11 +0800326 rko[0] ^= rcon[iteration] ^ rki[0];
Jerry Yu3f2fb712023-01-10 17:05:42 +0800327 rko[1] = rko[0] ^ rki[1];
328 rko[2] = rko[1] ^ rki[2];
329 rko[3] = rko[2] ^ rki[3];
Jerry Yufac5a542023-02-23 10:13:40 +0800330 if (rko + key_len_in_words > rko_end) {
Jerry Yu3304c202023-02-22 14:37:11 +0800331 /* Do not write overflow words.*/
332 continue;
333 }
Yanray Wange2bc1582023-05-08 10:28:53 +0800334#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
Jerry Yubaae4012023-02-21 15:26:13 +0800335 switch (key_bit_length) {
Jerry Yu3304c202023-02-22 14:37:11 +0800336 case 128:
337 break;
Jerry Yubaae4012023-02-21 15:26:13 +0800338 case 192:
Jerry Yu3304c202023-02-22 14:37:11 +0800339 rko[4] = rko[3] ^ rki[4];
340 rko[5] = rko[4] ^ rki[5];
Jerry Yubaae4012023-02-21 15:26:13 +0800341 break;
342 case 256:
Jerry Yu3304c202023-02-22 14:37:11 +0800343 rko[4] = aes_sub_word(rko[3]) ^ rki[4];
344 rko[5] = rko[4] ^ rki[5];
345 rko[6] = rko[5] ^ rki[6];
346 rko[7] = rko[6] ^ rki[7];
Jerry Yubaae4012023-02-21 15:26:13 +0800347 break;
Jerry Yu3f2fb712023-01-10 17:05:42 +0800348 }
Yanray Wange2bc1582023-05-08 10:28:53 +0800349#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
Jerry Yu3f2fb712023-01-10 17:05:42 +0800350 }
351}
352
353/*
354 * Key expansion, wrapper
355 */
356int mbedtls_aesce_setkey_enc(unsigned char *rk,
357 const unsigned char *key,
358 size_t bits)
359{
360 switch (bits) {
Jerry Yubaae4012023-02-21 15:26:13 +0800361 case 128:
362 case 192:
363 case 256:
Jerry Yuba1e78f2023-02-24 11:18:16 +0800364 aesce_setkey_enc(rk, key, bits);
365 break;
366 default:
367 return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH;
Jerry Yu3f2fb712023-01-10 17:05:42 +0800368 }
369
370 return 0;
371}
372
Jerry Yudf87a122023-01-10 18:17:15 +0800373#if defined(MBEDTLS_GCM_C)
374
Jerry Yu132d0cb2023-03-02 17:35:53 +0800375#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 5
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800376/* Some intrinsics are not available for GCC 5.X. */
Jerry Yu132d0cb2023-03-02 17:35:53 +0800377#define vreinterpretq_p64_u8(a) ((poly64x2_t) a)
378#define vreinterpretq_u8_p128(a) ((uint8x16_t) a)
379static inline poly64_t vget_low_p64(poly64x2_t __a)
380{
381 uint64x2_t tmp = (uint64x2_t) (__a);
382 uint64x1_t lo = vcreate_u64(vgetq_lane_u64(tmp, 0));
383 return (poly64_t) (lo);
384}
385#endif /* !__clang__ && __GNUC__ && __GNUC__ == 5*/
386
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800387/* vmull_p64/vmull_high_p64 wrappers.
388 *
389 * Older compilers miss some intrinsic functions for `poly*_t`. We use
390 * uint8x16_t and uint8x16x3_t as input/output parameters.
391 */
Jerry Yu9db4b1f2023-03-21 16:56:43 +0800392#if defined(__GNUC__) && !defined(__clang__)
393/* GCC reports incompatible type error without cast. GCC think poly64_t and
394 * poly64x1_t are different, that is different with MSVC and Clang. */
395#define MBEDTLS_VMULL_P64(a, b) vmull_p64((poly64_t) a, (poly64_t) b)
396#else
397/* MSVC reports `error C2440: 'type cast'` with cast. Clang does not report
398 * error with/without cast. And I think poly64_t and poly64x1_t are same, no
399 * cast for clang also. */
400#define MBEDTLS_VMULL_P64(a, b) vmull_p64(a, b)
401#endif
Jerry Yudf87a122023-01-10 18:17:15 +0800402static inline uint8x16_t pmull_low(uint8x16_t a, uint8x16_t b)
403{
Jerry Yu9db4b1f2023-03-21 16:56:43 +0800404
Jerry Yudf87a122023-01-10 18:17:15 +0800405 return vreinterpretq_u8_p128(
Jerry Yu9db4b1f2023-03-21 16:56:43 +0800406 MBEDTLS_VMULL_P64(
407 vget_low_p64(vreinterpretq_p64_u8(a)),
408 vget_low_p64(vreinterpretq_p64_u8(b))
409 ));
Jerry Yudf87a122023-01-10 18:17:15 +0800410}
411
412static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b)
413{
414 return vreinterpretq_u8_p128(
415 vmull_high_p64(vreinterpretq_p64_u8(a),
416 vreinterpretq_p64_u8(b)));
417}
418
Jerry Yuf0526a92023-03-14 15:00:29 +0800419/* GHASH does 128b polynomial multiplication on block in GF(2^128) defined by
Jerry Yu49b43672023-03-13 10:09:34 +0800420 * `x^128 + x^7 + x^2 + x + 1`.
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800421 *
422 * Arm64 only has 64b->128b polynomial multipliers, we need to do 4 64b
423 * multiplies to generate a 128b.
424 *
425 * `poly_mult_128` executes polynomial multiplication and outputs 256b that
426 * represented by 3 128b due to code size optimization.
427 *
428 * Output layout:
429 * | | | |
430 * |------------|-------------|-------------|
431 * | ret.val[0] | h3:h2:00:00 | high 128b |
Jerry Yu8f810602023-03-14 17:28:52 +0800432 * | ret.val[1] | :m2:m1:00 | middle 128b |
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800433 * | ret.val[2] | : :l1:l0 | low 128b |
434 */
Jerry Yudf87a122023-01-10 18:17:15 +0800435static inline uint8x16x3_t poly_mult_128(uint8x16_t a, uint8x16_t b)
436{
437 uint8x16x3_t ret;
Jerry Yu8f810602023-03-14 17:28:52 +0800438 uint8x16_t h, m, l; /* retval high/middle/low */
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800439 uint8x16_t c, d, e;
440
441 h = pmull_high(a, b); /* h3:h2:00:00 = a1*b1 */
442 l = pmull_low(a, b); /* : :l1:l0 = a0*b0 */
443 c = vextq_u8(b, b, 8); /* :c1:c0 = b0:b1 */
444 d = pmull_high(a, c); /* :d2:d1:00 = a1*b0 */
445 e = pmull_low(a, c); /* :e2:e1:00 = a0*b1 */
446 m = veorq_u8(d, e); /* :m2:m1:00 = d + e */
447
448 ret.val[0] = h;
449 ret.val[1] = m;
450 ret.val[2] = l;
Jerry Yudf87a122023-01-10 18:17:15 +0800451 return ret;
452}
453
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800454/*
455 * Modulo reduction.
456 *
457 * See: https://www.researchgate.net/publication/285612706_Implementing_GCM_on_ARMv8
458 *
459 * Section 4.3
460 *
461 * Modular reduction is slightly more complex. Write the GCM modulus as f(z) =
462 * z^128 +r(z), where r(z) = z^7+z^2+z+ 1. The well known approach is to
Jerry Yube4fdef2023-03-15 14:50:42 +0800463 * consider that z^128 ≡r(z) (mod z^128 +r(z)), allowing us to write the 256-bit
464 * operand to be reduced as a(z) = h(z)z^128 +l(z)≡h(z)r(z) + l(z). That is, we
465 * simply multiply the higher part of the operand by r(z) and add it to l(z). If
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800466 * the result is still larger than 128 bits, we reduce again.
467 */
468static inline uint8x16_t poly_mult_reduce(uint8x16x3_t input)
Jerry Yudf87a122023-01-10 18:17:15 +0800469{
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800470 uint8x16_t const ZERO = vdupq_n_u8(0);
Jerry Yu8b6df3f2023-03-21 16:59:13 +0800471
Jerry Yudf87a122023-01-10 18:17:15 +0800472 uint64x2_t r = vreinterpretq_u64_u8(vdupq_n_u8(0x87));
Jerry Yu8b6df3f2023-03-21 16:59:13 +0800473#if defined(__GNUC__)
474 /* use 'asm' as an optimisation barrier to prevent loading MODULO from
475 * memory. It is for GNUC compatible compilers.
476 */
Jerry Yudf87a122023-01-10 18:17:15 +0800477 asm ("" : "+w" (r));
Jerry Yu8b6df3f2023-03-21 16:59:13 +0800478#endif
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800479 uint8x16_t const MODULO = vreinterpretq_u8_u64(vshrq_n_u64(r, 64 - 8));
Jerry Yu8f810602023-03-14 17:28:52 +0800480 uint8x16_t h, m, l; /* input high/middle/low 128b */
Jerry Yu1ac7f6b2023-03-07 15:44:59 +0800481 uint8x16_t c, d, e, f, g, n, o;
482 h = input.val[0]; /* h3:h2:00:00 */
483 m = input.val[1]; /* :m2:m1:00 */
484 l = input.val[2]; /* : :l1:l0 */
485 c = pmull_high(h, MODULO); /* :c2:c1:00 = reduction of h3 */
486 d = pmull_low(h, MODULO); /* : :d1:d0 = reduction of h2 */
487 e = veorq_u8(c, m); /* :e2:e1:00 = m2:m1:00 + c2:c1:00 */
488 f = pmull_high(e, MODULO); /* : :f1:f0 = reduction of e2 */
489 g = vextq_u8(ZERO, e, 8); /* : :g1:00 = e1:00 */
490 n = veorq_u8(d, l); /* : :n1:n0 = d1:d0 + l1:l0 */
491 o = veorq_u8(n, f); /* o1:o0 = f1:f0 + n1:n0 */
492 return veorq_u8(o, g); /* = o1:o0 + g1:00 */
Jerry Yudf87a122023-01-10 18:17:15 +0800493}
494
495/*
496 * GCM multiplication: c = a times b in GF(2^128)
497 */
498void mbedtls_aesce_gcm_mult(unsigned char c[16],
499 const unsigned char a[16],
500 const unsigned char b[16])
501{
502 uint8x16_t va, vb, vc;
503 va = vrbitq_u8(vld1q_u8(&a[0]));
504 vb = vrbitq_u8(vld1q_u8(&b[0]));
505 vc = vrbitq_u8(poly_mult_reduce(poly_mult_128(va, vb)));
506 vst1q_u8(&c[0], vc);
507}
508
509#endif /* MBEDTLS_GCM_C */
Jerry Yu48b999c2023-03-03 15:51:07 +0800510
511#if defined(MBEDTLS_POP_TARGET_PRAGMA)
512#if defined(__clang__)
513#pragma clang attribute pop
514#elif defined(__GNUC__)
515#pragma GCC pop_options
516#endif
517#undef MBEDTLS_POP_TARGET_PRAGMA
518#endif
519
Jerry Yu72fd0bd2023-08-18 16:31:01 +0800520#endif /* MBEDTLS_ARCH_IS_ARM64 */
Jerry Yu49231312023-01-10 16:57:21 +0800521
522#endif /* MBEDTLS_AESCE_C */