1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * AES block cipher, optimized for ARM64 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 * Copyright 2026 Google LLC 7 */ 8 9 #include <asm/neon.h> 10 #include <asm/simd.h> 11 #include <linux/unaligned.h> 12 #include <linux/cpufeature.h> 13 14 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes); 15 16 struct aes_block { 17 u8 b[AES_BLOCK_SIZE]; 18 }; 19 20 asmlinkage void __aes_arm64_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE], 21 const u8 in[AES_BLOCK_SIZE], int rounds); 22 asmlinkage void __aes_arm64_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], 23 const u8 in[AES_BLOCK_SIZE], int rounds); 24 asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE], 25 const u8 in[AES_BLOCK_SIZE], int rounds); 26 asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], 27 const u8 in[AES_BLOCK_SIZE], int rounds); 28 asmlinkage u32 __aes_ce_sub(u32 l); 29 asmlinkage void __aes_ce_invert(struct aes_block *out, 30 const struct aes_block *in); 31 32 /* 33 * Expand an AES key using the crypto extensions if supported and usable or 34 * generic code otherwise. The expanded key format is compatible between the 35 * two cases. The outputs are @rndkeys (required) and @inv_rndkeys (optional). 36 */ 37 static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys, 38 const u8 *in_key, int key_len, int nrounds) 39 { 40 /* 41 * The AES key schedule round constants 42 */ 43 static u8 const rcon[] = { 44 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 45 }; 46 47 u32 kwords = key_len / sizeof(u32); 48 struct aes_block *key_enc, *key_dec; 49 int i, j; 50 51 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || 52 !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { 53 aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); 54 return; 55 } 56 57 for (i = 0; i < kwords; i++) 58 rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); 59 60 scoped_ksimd() { 61 for (i = 0; i < sizeof(rcon); i++) { 62 u32 *rki = &rndkeys[i * kwords]; 63 u32 *rko = rki + kwords; 64 65 rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ 66 rcon[i] ^ rki[0]; 67 rko[1] = rko[0] ^ rki[1]; 68 rko[2] = rko[1] ^ rki[2]; 69 rko[3] = rko[2] ^ rki[3]; 70 71 if (key_len == AES_KEYSIZE_192) { 72 if (i >= 7) 73 break; 74 rko[4] = rko[3] ^ rki[4]; 75 rko[5] = rko[4] ^ rki[5]; 76 } else if (key_len == AES_KEYSIZE_256) { 77 if (i >= 6) 78 break; 79 rko[4] = __aes_ce_sub(rko[3]) ^ rki[4]; 80 rko[5] = rko[4] ^ rki[5]; 81 rko[6] = rko[5] ^ rki[6]; 82 rko[7] = rko[6] ^ rki[7]; 83 } 84 } 85 86 /* 87 * Generate the decryption keys for the Equivalent Inverse 88 * Cipher. This involves reversing the order of the round 89 * keys, and applying the Inverse Mix Columns transformation on 90 * all but the first and the last one. 91 */ 92 if (inv_rndkeys) { 93 key_enc = (struct aes_block *)rndkeys; 94 key_dec = (struct aes_block *)inv_rndkeys; 95 j = nrounds; 96 97 key_dec[0] = key_enc[j]; 98 for (i = 1, j--; j > 0; i++, j--) 99 __aes_ce_invert(key_dec + i, key_enc + j); 100 key_dec[i] = key_enc[0]; 101 } 102 } 103 } 104 105 static void aes_preparekey_arch(union aes_enckey_arch *k, 106 union aes_invkey_arch *inv_k, 107 const u8 *in_key, int key_len, int nrounds) 108 { 109 aes_expandkey_arm64(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, 110 in_key, key_len, nrounds); 111 } 112 113 /* 114 * This is here temporarily until the remaining AES mode implementations are 115 * migrated from arch/arm64/crypto/ to lib/crypto/arm64/. 116 */ 117 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 118 unsigned int key_len) 119 { 120 if (aes_check_keylen(key_len) != 0) 121 return -EINVAL; 122 ctx->key_length = key_len; 123 aes_expandkey_arm64(ctx->key_enc, ctx->key_dec, in_key, key_len, 124 6 + key_len / 4); 125 return 0; 126 } 127 EXPORT_SYMBOL(ce_aes_expandkey); 128 129 static void aes_encrypt_arch(const struct aes_enckey *key, 130 u8 out[AES_BLOCK_SIZE], 131 const u8 in[AES_BLOCK_SIZE]) 132 { 133 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 134 static_branch_likely(&have_aes) && likely(may_use_simd())) { 135 scoped_ksimd() 136 __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds); 137 } else { 138 __aes_arm64_encrypt(key->k.rndkeys, out, in, key->nrounds); 139 } 140 } 141 142 static void aes_decrypt_arch(const struct aes_key *key, 143 u8 out[AES_BLOCK_SIZE], 144 const u8 in[AES_BLOCK_SIZE]) 145 { 146 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && 147 static_branch_likely(&have_aes) && likely(may_use_simd())) { 148 scoped_ksimd() 149 __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in, 150 key->nrounds); 151 } else { 152 __aes_arm64_decrypt(key->inv_k.inv_rndkeys, out, in, 153 key->nrounds); 154 } 155 } 156 157 #ifdef CONFIG_KERNEL_MODE_NEON 158 #define aes_mod_init_arch aes_mod_init_arch 159 static void aes_mod_init_arch(void) 160 { 161 if (cpu_have_named_feature(AES)) 162 static_branch_enable(&have_aes); 163 } 164 #endif /* CONFIG_KERNEL_MODE_NEON */ 165