1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * AES block cipher using AES-NI instructions 4 * 5 * Copyright 2026 Google LLC 6 */ 7 8 #include <asm/fpu/api.h> 9 10 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni); 11 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni_avx); 12 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx2); 13 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx512); 14 15 /* The assembly code assumes the following offsets. */ 16 static_assert(offsetof(struct aes_enckey, len) == 0); 17 static_assert(offsetof(struct aes_enckey, nrounds) == 4); 18 static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16); 19 static_assert(offsetof(struct aes_key, len) == 0); 20 static_assert(offsetof(struct aes_key, nrounds) == 4); 21 static_assert(offsetof(struct aes_key, k.rndkeys) == 16); 22 static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256); 23 24 void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys, 25 const u8 in_key[AES_KEYSIZE_128]); 26 void aes256_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys, 27 const u8 in_key[AES_KEYSIZE_256]); 28 void aes_encrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE], 29 const struct aes_enckey *key); 30 void aes_decrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE], 31 const struct aes_key *key); 32 33 /* 34 * Expand an AES key using AES-NI if supported and usable or generic code 35 * otherwise. The expanded key format is compatible between the two cases. The 36 * outputs are @k->rndkeys (required) and @inv_k->inv_rndkeys (optional). 37 * 38 * We could just always use the generic key expansion code. AES key expansion 39 * is usually less performance-critical than AES en/decryption. However, 40 * there's still *some* value in speed here, as well as in non-key-dependent 41 * execution time which AES-NI provides. So, do use AES-NI to expand AES-128 42 * and AES-256 keys. (Don't bother with AES-192, as it's almost never used.) 43 */ 44 static void aes_preparekey_arch(union aes_enckey_arch *k, 45 union aes_invkey_arch *inv_k, 46 const u8 *in_key, int key_len, int nrounds) 47 { 48 u32 *rndkeys = k->rndkeys; 49 u32 *inv_rndkeys = inv_k ? inv_k->inv_rndkeys : NULL; 50 51 if (static_branch_likely(&have_aesni) && key_len != AES_KEYSIZE_192 && 52 irq_fpu_usable()) { 53 kernel_fpu_begin(); 54 if (key_len == AES_KEYSIZE_128) 55 aes128_expandkey_aesni(rndkeys, inv_rndkeys, in_key); 56 else 57 aes256_expandkey_aesni(rndkeys, inv_rndkeys, in_key); 58 kernel_fpu_end(); 59 } else { 60 aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); 61 } 62 } 63 64 static void aes_encrypt_arch(const struct aes_enckey *key, 65 u8 out[AES_BLOCK_SIZE], 66 const u8 in[AES_BLOCK_SIZE]) 67 { 68 if (static_branch_likely(&have_aesni) && irq_fpu_usable()) { 69 kernel_fpu_begin(); 70 aes_encrypt_aesni(out, in, key); 71 kernel_fpu_end(); 72 } else { 73 aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in); 74 } 75 } 76 77 static void aes_decrypt_arch(const struct aes_key *key, 78 u8 out[AES_BLOCK_SIZE], 79 const u8 in[AES_BLOCK_SIZE]) 80 { 81 if (static_branch_likely(&have_aesni) && irq_fpu_usable()) { 82 kernel_fpu_begin(); 83 aes_decrypt_aesni(out, in, key); 84 kernel_fpu_end(); 85 } else { 86 aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, 87 out, in); 88 } 89 } 90 91 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) 92 void aes_ecb_encrypt_aesni(u8 *dst, const u8 *src, long nblocks, 93 const struct aes_enckey *key); 94 void aes_ecb_decrypt_aesni(u8 *dst, const u8 *src, long nblocks, 95 const struct aes_key *key); 96 97 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 98 #define aes_ecb_encrypt_arch aes_ecb_encrypt_arch 99 static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, 100 const struct aes_enckey *key) 101 { 102 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 103 return false; 104 kernel_fpu_begin(); 105 aes_ecb_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key); 106 kernel_fpu_end(); 107 return true; 108 } 109 110 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 111 #define aes_ecb_decrypt_arch aes_ecb_decrypt_arch 112 static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, 113 const struct aes_key *key) 114 { 115 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 116 return false; 117 kernel_fpu_begin(); 118 aes_ecb_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key); 119 kernel_fpu_end(); 120 return true; 121 } 122 #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ 123 124 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) 125 void aes_cbc_encrypt_aesni(u8 *dst, const u8 *src, long nblocks, 126 u8 iv[AES_BLOCK_SIZE], const struct aes_enckey *key); 127 void aes_cbc_decrypt_aesni(u8 *dst, const u8 *src, long nblocks, 128 u8 iv[AES_BLOCK_SIZE], const struct aes_key *key); 129 void aes_cbc_cts_encrypt_aesni(u8 *dst, const u8 *src, long pn_len, 130 const u8 iv[AES_BLOCK_SIZE], 131 const struct aes_enckey *key); 132 void aes_cbc_cts_decrypt_aesni(u8 *dst, const u8 *src, long pn_len, 133 const u8 iv[AES_BLOCK_SIZE], 134 const struct aes_key *key); 135 136 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 137 #define aes_cbc_encrypt_arch aes_cbc_encrypt_arch 138 static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, 139 u8 iv[AES_BLOCK_SIZE], 140 const struct aes_enckey *key) 141 { 142 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 143 return false; 144 kernel_fpu_begin(); 145 aes_cbc_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key); 146 kernel_fpu_end(); 147 return true; 148 } 149 150 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 151 #define aes_cbc_decrypt_arch aes_cbc_decrypt_arch 152 static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, 153 u8 iv[AES_BLOCK_SIZE], 154 const struct aes_key *key) 155 { 156 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 157 return false; 158 kernel_fpu_begin(); 159 aes_cbc_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key); 160 kernel_fpu_end(); 161 return true; 162 } 163 164 /* len can be any value greater than AES_BLOCK_SIZE here. */ 165 #define aes_cbc_cts_encrypt_arch aes_cbc_cts_encrypt_arch 166 static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 167 u8 iv[AES_BLOCK_SIZE], 168 const struct aes_enckey *key) 169 { 170 const size_t cbc_blocks = (len - AES_BLOCK_SIZE - 1) / AES_BLOCK_SIZE; 171 const size_t pn_len = ((len - 1) % AES_BLOCK_SIZE) + 1; 172 173 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 174 return false; 175 176 kernel_fpu_begin(); 177 if (cbc_blocks) { 178 aes_cbc_encrypt_aesni(dst, src, cbc_blocks, iv, key); 179 dst += cbc_blocks * AES_BLOCK_SIZE; 180 src += cbc_blocks * AES_BLOCK_SIZE; 181 } 182 /* Encrypt the last 17 to 32 bytes. */ 183 aes_cbc_cts_encrypt_aesni(dst, src, pn_len, iv, key); 184 kernel_fpu_end(); 185 return true; 186 } 187 188 /* len can be any value greater than AES_BLOCK_SIZE here. */ 189 #define aes_cbc_cts_decrypt_arch aes_cbc_cts_decrypt_arch 190 static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 191 u8 iv[AES_BLOCK_SIZE], 192 const struct aes_key *key) 193 { 194 const size_t cbc_blocks = (len - AES_BLOCK_SIZE - 1) / AES_BLOCK_SIZE; 195 const size_t pn_len = ((len - 1) % AES_BLOCK_SIZE) + 1; 196 197 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 198 return false; 199 200 kernel_fpu_begin(); 201 if (cbc_blocks) { 202 aes_cbc_decrypt_aesni(dst, src, cbc_blocks, iv, key); 203 dst += cbc_blocks * AES_BLOCK_SIZE; 204 src += cbc_blocks * AES_BLOCK_SIZE; 205 } 206 /* Decrypt the last 17 to 32 bytes. */ 207 aes_cbc_cts_decrypt_aesni(dst, src, pn_len, iv, key); 208 kernel_fpu_end(); 209 return true; 210 } 211 #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ 212 213 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) && IS_ENABLED(CONFIG_X86_64) 214 void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len, const u64 le_ctr[2], 215 const struct aes_enckey *key); 216 void aes_ctr64_crypt_aesni_avx(u8 *dst, const u8 *src, s64 len, 217 const u64 le_ctr[2], 218 const struct aes_enckey *key); 219 void aes_ctr64_crypt_vaes_avx2(u8 *dst, const u8 *src, s64 len, 220 const u64 le_ctr[2], 221 const struct aes_enckey *key); 222 void aes_ctr64_crypt_vaes_avx512(u8 *dst, const u8 *src, s64 len, 223 const u64 le_ctr[2], 224 const struct aes_enckey *key); 225 void aes_xctr_crypt_aesni_avx(u8 *dst, const u8 *src, s64 len, u64 ctr, 226 const u8 iv[AES_BLOCK_SIZE], 227 const struct aes_enckey *key); 228 void aes_xctr_crypt_vaes_avx2(u8 *dst, const u8 *src, s64 len, u64 ctr, 229 const u8 iv[AES_BLOCK_SIZE], 230 const struct aes_enckey *key); 231 void aes_xctr_crypt_vaes_avx512(u8 *dst, const u8 *src, s64 len, u64 ctr, 232 const u8 iv[AES_BLOCK_SIZE], 233 const struct aes_enckey *key); 234 235 static void aes_ctr64_x86(u8 *dst, const u8 *src, size_t len, 236 const u64 le_ctr[2], const struct aes_enckey *key) 237 { 238 if (static_branch_likely(&have_vaes_avx512)) 239 aes_ctr64_crypt_vaes_avx512(dst, src, len, le_ctr, key); 240 else if (static_branch_likely(&have_vaes_avx2)) 241 aes_ctr64_crypt_vaes_avx2(dst, src, len, le_ctr, key); 242 else if (static_branch_likely(&have_aesni_avx)) 243 aes_ctr64_crypt_aesni_avx(dst, src, len, le_ctr, key); 244 else 245 aes_ctr64_crypt_aesni(dst, src, len, le_ctr, key); 246 } 247 248 #define aes_ctr_arch aes_ctr_arch 249 static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, 250 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) 251 { 252 u64 le_ctr[2]; 253 u64 ctr64; 254 size_t nblocks; 255 size_t part1_len; 256 257 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 258 return false; 259 260 ctr64 = le_ctr[0] = get_unaligned_be64(&ctr[8]); 261 le_ctr[1] = get_unaligned_be64(&ctr[0]); 262 263 kernel_fpu_begin(); 264 265 nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE); 266 ctr64 += nblocks; 267 268 if (likely(ctr64 >= nblocks)) { 269 /* The low 64 bits of the counter won't overflow. */ 270 aes_ctr64_x86(dst, src, len, le_ctr, key); 271 } else { 272 /* 273 * The low 64 bits of the counter will overflow. The 274 * assembly doesn't handle this case, so split the 275 * operation into two at the point where the overflow 276 * will occur. After the first part, add the carry bit. 277 */ 278 part1_len = min(len, (nblocks - ctr64) * AES_BLOCK_SIZE); 279 aes_ctr64_x86(dst, src, part1_len, le_ctr, key); 280 le_ctr[0] = 0; 281 le_ctr[1]++; 282 aes_ctr64_x86(dst + part1_len, src + part1_len, len - part1_len, 283 le_ctr, key); 284 } 285 kernel_fpu_end(); 286 put_unaligned_be64(ctr64, &ctr[8]); 287 put_unaligned_be64(le_ctr[1], &ctr[0]); 288 return true; 289 } 290 291 #define aes_xctr_arch aes_xctr_arch 292 static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 ctr, 293 const u8 iv[AES_BLOCK_SIZE], 294 const struct aes_enckey *key) 295 { 296 if (!static_branch_likely(&have_aesni_avx) || 297 unlikely(!irq_fpu_usable())) 298 return false; 299 kernel_fpu_begin(); 300 if (static_branch_likely(&have_vaes_avx512)) 301 aes_xctr_crypt_vaes_avx512(dst, src, len, ctr, iv, key); 302 else if (static_branch_likely(&have_vaes_avx2)) 303 aes_xctr_crypt_vaes_avx2(dst, src, len, ctr, iv, key); 304 else 305 aes_xctr_crypt_aesni_avx(dst, src, len, ctr, iv, key); 306 kernel_fpu_end(); 307 return true; 308 } 309 #endif /* CONFIG_CRYPTO_LIB_AES_CTR && CONFIG_X86_64 */ 310 311 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) 312 void aes_xts_encrypt_aesni(u8 *dst, const u8 *src, long nblocks, 313 u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key); 314 void aes_xts_decrypt_aesni(u8 *dst, const u8 *src, long nblocks, 315 u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key); 316 void aes_xts_encrypt_iv(const struct aes_enckey *tweak_key, 317 u8 iv[AES_BLOCK_SIZE]); 318 void aes_xts_encrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks, 319 u8 tweak[AES_BLOCK_SIZE], 320 const struct aes_key *key); 321 void aes_xts_decrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks, 322 u8 tweak[AES_BLOCK_SIZE], 323 const struct aes_key *key); 324 void aes_xts_encrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks, 325 u8 tweak[AES_BLOCK_SIZE], 326 const struct aes_key *key); 327 void aes_xts_decrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks, 328 u8 tweak[AES_BLOCK_SIZE], 329 const struct aes_key *key); 330 void aes_xts_encrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks, 331 u8 tweak[AES_BLOCK_SIZE], 332 const struct aes_key *key); 333 void aes_xts_decrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks, 334 u8 tweak[AES_BLOCK_SIZE], 335 const struct aes_key *key); 336 337 /* len is always a positive multiple of AES_BLOCK_SIZE here. */ 338 static __always_inline bool 339 aes_xts_crypt_x86(u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], 340 const struct aes_xts_key *key, bool cont, bool enc) 341 { 342 const long nblocks = len / AES_BLOCK_SIZE; 343 344 if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) 345 return false; 346 347 kernel_fpu_begin(); 348 if (IS_ENABLED(CONFIG_X86_64) && 349 static_branch_likely(&have_vaes_avx512)) { 350 if (!cont) 351 aes_xts_encrypt_iv(&key->tweak_key, tweak); 352 if (enc) 353 aes_xts_encrypt_vaes_avx512(dst, src, nblocks, tweak, 354 &key->main_key); 355 else 356 aes_xts_decrypt_vaes_avx512(dst, src, nblocks, tweak, 357 &key->main_key); 358 } else if (IS_ENABLED(CONFIG_X86_64) && 359 static_branch_likely(&have_vaes_avx2)) { 360 if (!cont) 361 aes_xts_encrypt_iv(&key->tweak_key, tweak); 362 if (enc) 363 aes_xts_encrypt_vaes_avx2(dst, src, nblocks, tweak, 364 &key->main_key); 365 else 366 aes_xts_decrypt_vaes_avx2(dst, src, nblocks, tweak, 367 &key->main_key); 368 } else if (IS_ENABLED(CONFIG_X86_64) && 369 static_branch_likely(&have_aesni_avx)) { 370 if (!cont) 371 aes_xts_encrypt_iv(&key->tweak_key, tweak); 372 if (enc) 373 aes_xts_encrypt_aesni_avx(dst, src, nblocks, tweak, 374 &key->main_key); 375 else 376 aes_xts_decrypt_aesni_avx(dst, src, nblocks, tweak, 377 &key->main_key); 378 } else { 379 if (!cont) 380 aes_encrypt_aesni(tweak, tweak, &key->tweak_key); 381 if (enc) 382 aes_xts_encrypt_aesni(dst, src, nblocks, tweak, 383 &key->main_key); 384 else 385 aes_xts_decrypt_aesni(dst, src, nblocks, tweak, 386 &key->main_key); 387 } 388 kernel_fpu_end(); 389 return true; 390 } 391 392 #define aes_xts_encrypt_arch aes_xts_encrypt_arch 393 static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, 394 u8 tweak[AES_BLOCK_SIZE], 395 const struct aes_xts_key *key, bool cont) 396 { 397 return aes_xts_crypt_x86(dst, src, len, tweak, key, cont, true); 398 } 399 400 #define aes_xts_decrypt_arch aes_xts_decrypt_arch 401 static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, 402 u8 tweak[AES_BLOCK_SIZE], 403 const struct aes_xts_key *key, bool cont) 404 { 405 return aes_xts_crypt_x86(dst, src, len, tweak, key, cont, false); 406 } 407 #endif /* CONFIG_CRYPTO_LIB_AES_XTS */ 408 409 #define aes_mod_init_arch aes_mod_init_arch 410 static void aes_mod_init_arch(void) 411 { 412 /* Everything below requires AES-NI. */ 413 if (!boot_cpu_has(X86_FEATURE_AES)) 414 return; 415 static_branch_enable(&have_aesni); 416 417 /* Everything below requires AVX and is also 64-bit only. */ 418 if (!boot_cpu_has(X86_FEATURE_AVX) || !IS_ENABLED(CONFIG_X86_64)) 419 return; 420 static_branch_enable(&have_aesni_avx); 421 422 /* 423 * Everything below requires VAES, and also sometimes AVX2, VPCLMULQDQ, 424 * and PCLMULQDQ. Use a single static key for all of them, since in 425 * practice every CPU with VAES also has the others. 426 */ 427 if (!boot_cpu_has(X86_FEATURE_VAES) || 428 !boot_cpu_has(X86_FEATURE_AVX2) || 429 !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) || 430 !boot_cpu_has(X86_FEATURE_PCLMULQDQ)) 431 return; 432 static_branch_enable(&have_vaes_avx2); 433 434 if (!boot_cpu_has(X86_FEATURE_AVX512BW) || 435 !boot_cpu_has(X86_FEATURE_AVX512VL) || 436 !boot_cpu_has(X86_FEATURE_BMI2) || 437 boot_cpu_has(X86_FEATURE_PREFER_YMM)) 438 return; 439 static_branch_enable(&have_vaes_avx512); 440 } 441