1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Support for AES-NI and VAES instructions. This file contains glue code. 4 * The real AES implementations are in .S files. 5 * 6 * Copyright (C) 2008, Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * 9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 10 * interface for 64-bit kernels. 11 * Authors: Adrian Hoban <adrian.hoban@intel.com> 12 * Gabriele Paoloni <gabriele.paoloni@intel.com> 13 * Tadeusz Struk (tadeusz.struk@intel.com) 14 * Aidan O'Mahony (aidan.o.mahony@intel.com) 15 * Copyright (c) 2010, Intel Corporation. 16 * 17 * Copyright 2024 Google LLC 18 */ 19 20 #include <linux/hardirq.h> 21 #include <linux/types.h> 22 #include <linux/module.h> 23 #include <linux/err.h> 24 #include <crypto/algapi.h> 25 #include <crypto/aes.h> 26 #include <crypto/b128ops.h> 27 #include <crypto/gcm.h> 28 #include <crypto/gf128mul.h> 29 #include <asm/cpu_device_id.h> 30 #include <asm/simd.h> 31 #include <crypto/scatterwalk.h> 32 #include <crypto/internal/aead.h> 33 #include <crypto/internal/simd.h> 34 #include <crypto/internal/skcipher.h> 35 #include <linux/jump_label.h> 36 #include <linux/workqueue.h> 37 #include <linux/spinlock.h> 38 #include <linux/static_call.h> 39 40 /* The common part of the x86_64 AES-GCM key struct */ 41 struct aes_gcm_key { 42 /* Expanded AES key and the AES key length in bytes */ 43 struct aes_enckey aes_key; 44 45 /* RFC4106 nonce (used only by the rfc4106 algorithms) */ 46 u32 rfc4106_nonce; 47 }; 48 49 /* Key struct used by the AES-NI implementations of AES-GCM */ 50 struct aes_gcm_key_aesni { 51 /* 52 * Common part of the key. 16-byte alignment is required by the 53 * assembly code. 54 */ 55 struct aes_gcm_key base __aligned(16); 56 57 /* 58 * Powers of the hash key H^8 through H^1. These are 128-bit values. 59 * They all have an extra factor of x^-1 and are byte-reversed. 16-byte 60 * alignment is required by the assembly code. 61 */ 62 u64 h_powers[8][2] __aligned(16); 63 64 /* 65 * h_powers_xored[i] contains the two 64-bit halves of h_powers[i] XOR'd 66 * together. It's used for Karatsuba multiplication. 16-byte alignment 67 * is required by the assembly code. 68 */ 69 u64 h_powers_xored[8] __aligned(16); 70 71 /* 72 * H^1 times x^64 (and also the usual extra factor of x^-1). 16-byte 73 * alignment is required by the assembly code. 74 */ 75 u64 h_times_x64[2] __aligned(16); 76 }; 77 #define AES_GCM_KEY_AESNI(key) \ 78 container_of((key), struct aes_gcm_key_aesni, base) 79 #define AES_GCM_KEY_AESNI_SIZE \ 80 (sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1))) 81 82 /* Key struct used by the VAES + AVX2 implementation of AES-GCM */ 83 struct aes_gcm_key_vaes_avx2 { 84 /* 85 * Common part of the key. The assembly code prefers 16-byte alignment 86 * for this. 87 */ 88 struct aes_gcm_key base __aligned(16); 89 90 /* 91 * Powers of the hash key H^8 through H^1. These are 128-bit values. 92 * They all have an extra factor of x^-1 and are byte-reversed. 93 * The assembly code prefers 32-byte alignment for this. 94 */ 95 u64 h_powers[8][2] __aligned(32); 96 97 /* 98 * Each entry in this array contains the two halves of an entry of 99 * h_powers XOR'd together, in the following order: 100 * H^8,H^6,H^7,H^5,H^4,H^2,H^3,H^1 i.e. indices 0,2,1,3,4,6,5,7. 101 * This is used for Karatsuba multiplication. 102 */ 103 u64 h_powers_xored[8]; 104 }; 105 106 #define AES_GCM_KEY_VAES_AVX2(key) \ 107 container_of((key), struct aes_gcm_key_vaes_avx2, base) 108 #define AES_GCM_KEY_VAES_AVX2_SIZE \ 109 (sizeof(struct aes_gcm_key_vaes_avx2) + (31 & ~(CRYPTO_MINALIGN - 1))) 110 111 /* Key struct used by the VAES + AVX512 implementation of AES-GCM */ 112 struct aes_gcm_key_vaes_avx512 { 113 /* 114 * Common part of the key. The assembly code prefers 16-byte alignment 115 * for this. 116 */ 117 struct aes_gcm_key base __aligned(16); 118 119 /* 120 * Powers of the hash key H^16 through H^1. These are 128-bit values. 121 * They all have an extra factor of x^-1 and are byte-reversed. This 122 * array is aligned to a 64-byte boundary to make it naturally aligned 123 * for 512-bit loads, which can improve performance. (The assembly code 124 * doesn't *need* the alignment; this is just an optimization.) 125 */ 126 u64 h_powers[16][2] __aligned(64); 127 128 /* Three padding blocks required by the assembly code */ 129 u64 padding[3][2]; 130 }; 131 #define AES_GCM_KEY_VAES_AVX512(key) \ 132 container_of((key), struct aes_gcm_key_vaes_avx512, base) 133 #define AES_GCM_KEY_VAES_AVX512_SIZE \ 134 (sizeof(struct aes_gcm_key_vaes_avx512) + (63 & ~(CRYPTO_MINALIGN - 1))) 135 136 /* 137 * These flags are passed to the AES-GCM helper functions to specify the 138 * specific version of AES-GCM (RFC4106 or not), whether it's encryption or 139 * decryption, and which assembly functions should be called. Assembly 140 * functions are selected using flags instead of function pointers to avoid 141 * indirect calls (which are very expensive on x86) regardless of inlining. 142 */ 143 #define FLAG_RFC4106 BIT(0) 144 #define FLAG_ENC BIT(1) 145 #define FLAG_AVX BIT(2) 146 #define FLAG_VAES_AVX2 BIT(3) 147 #define FLAG_VAES_AVX512 BIT(4) 148 149 static inline struct aes_gcm_key * 150 aes_gcm_key_get(struct crypto_aead *tfm, int flags) 151 { 152 if (flags & FLAG_VAES_AVX512) 153 return PTR_ALIGN(crypto_aead_ctx(tfm), 64); 154 else if (flags & FLAG_VAES_AVX2) 155 return PTR_ALIGN(crypto_aead_ctx(tfm), 32); 156 else 157 return PTR_ALIGN(crypto_aead_ctx(tfm), 16); 158 } 159 160 asmlinkage void 161 aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key); 162 asmlinkage void 163 aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key); 164 asmlinkage void 165 aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key); 166 asmlinkage void 167 aes_gcm_precompute_vaes_avx512(struct aes_gcm_key_vaes_avx512 *key); 168 169 static void aes_gcm_precompute(struct aes_gcm_key *key, int flags) 170 { 171 if (flags & FLAG_VAES_AVX512) 172 aes_gcm_precompute_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key)); 173 else if (flags & FLAG_VAES_AVX2) 174 aes_gcm_precompute_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key)); 175 else if (flags & FLAG_AVX) 176 aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key)); 177 else 178 aes_gcm_precompute_aesni(AES_GCM_KEY_AESNI(key)); 179 } 180 181 asmlinkage void 182 aes_gcm_aad_update_aesni(const struct aes_gcm_key_aesni *key, 183 u8 ghash_acc[16], const u8 *aad, int aadlen); 184 asmlinkage void 185 aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key, 186 u8 ghash_acc[16], const u8 *aad, int aadlen); 187 asmlinkage void 188 aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 189 u8 ghash_acc[16], const u8 *aad, int aadlen); 190 asmlinkage void 191 aes_gcm_aad_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 192 u8 ghash_acc[16], const u8 *aad, int aadlen); 193 194 static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16], 195 const u8 *aad, int aadlen, int flags) 196 { 197 if (flags & FLAG_VAES_AVX512) 198 aes_gcm_aad_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key), 199 ghash_acc, aad, aadlen); 200 else if (flags & FLAG_VAES_AVX2) 201 aes_gcm_aad_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key), 202 ghash_acc, aad, aadlen); 203 else if (flags & FLAG_AVX) 204 aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc, 205 aad, aadlen); 206 else 207 aes_gcm_aad_update_aesni(AES_GCM_KEY_AESNI(key), ghash_acc, 208 aad, aadlen); 209 } 210 211 asmlinkage void 212 aes_gcm_enc_update_aesni(const struct aes_gcm_key_aesni *key, 213 const u32 le_ctr[4], u8 ghash_acc[16], 214 const u8 *src, u8 *dst, int datalen); 215 asmlinkage void 216 aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key, 217 const u32 le_ctr[4], u8 ghash_acc[16], 218 const u8 *src, u8 *dst, int datalen); 219 asmlinkage void 220 aes_gcm_enc_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 221 const u32 le_ctr[4], u8 ghash_acc[16], 222 const u8 *src, u8 *dst, int datalen); 223 asmlinkage void 224 aes_gcm_enc_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 225 const u32 le_ctr[4], u8 ghash_acc[16], 226 const u8 *src, u8 *dst, int datalen); 227 228 asmlinkage void 229 aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key, 230 const u32 le_ctr[4], u8 ghash_acc[16], 231 const u8 *src, u8 *dst, int datalen); 232 asmlinkage void 233 aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key, 234 const u32 le_ctr[4], u8 ghash_acc[16], 235 const u8 *src, u8 *dst, int datalen); 236 asmlinkage void 237 aes_gcm_dec_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 238 const u32 le_ctr[4], u8 ghash_acc[16], 239 const u8 *src, u8 *dst, int datalen); 240 asmlinkage void 241 aes_gcm_dec_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 242 const u32 le_ctr[4], u8 ghash_acc[16], 243 const u8 *src, u8 *dst, int datalen); 244 245 /* __always_inline to optimize out the branches based on @flags */ 246 static __always_inline void 247 aes_gcm_update(const struct aes_gcm_key *key, 248 const u32 le_ctr[4], u8 ghash_acc[16], 249 const u8 *src, u8 *dst, int datalen, int flags) 250 { 251 if (flags & FLAG_ENC) { 252 if (flags & FLAG_VAES_AVX512) 253 aes_gcm_enc_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key), 254 le_ctr, ghash_acc, 255 src, dst, datalen); 256 else if (flags & FLAG_VAES_AVX2) 257 aes_gcm_enc_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key), 258 le_ctr, ghash_acc, 259 src, dst, datalen); 260 else if (flags & FLAG_AVX) 261 aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key), 262 le_ctr, ghash_acc, 263 src, dst, datalen); 264 else 265 aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr, 266 ghash_acc, src, dst, datalen); 267 } else { 268 if (flags & FLAG_VAES_AVX512) 269 aes_gcm_dec_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key), 270 le_ctr, ghash_acc, 271 src, dst, datalen); 272 else if (flags & FLAG_VAES_AVX2) 273 aes_gcm_dec_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key), 274 le_ctr, ghash_acc, 275 src, dst, datalen); 276 else if (flags & FLAG_AVX) 277 aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key), 278 le_ctr, ghash_acc, 279 src, dst, datalen); 280 else 281 aes_gcm_dec_update_aesni(AES_GCM_KEY_AESNI(key), 282 le_ctr, ghash_acc, 283 src, dst, datalen); 284 } 285 } 286 287 asmlinkage void 288 aes_gcm_enc_final_aesni(const struct aes_gcm_key_aesni *key, 289 const u32 le_ctr[4], u8 ghash_acc[16], 290 u64 total_aadlen, u64 total_datalen); 291 asmlinkage void 292 aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key, 293 const u32 le_ctr[4], u8 ghash_acc[16], 294 u64 total_aadlen, u64 total_datalen); 295 asmlinkage void 296 aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 297 const u32 le_ctr[4], u8 ghash_acc[16], 298 u64 total_aadlen, u64 total_datalen); 299 asmlinkage void 300 aes_gcm_enc_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 301 const u32 le_ctr[4], u8 ghash_acc[16], 302 u64 total_aadlen, u64 total_datalen); 303 304 /* __always_inline to optimize out the branches based on @flags */ 305 static __always_inline void 306 aes_gcm_enc_final(const struct aes_gcm_key *key, 307 const u32 le_ctr[4], u8 ghash_acc[16], 308 u64 total_aadlen, u64 total_datalen, int flags) 309 { 310 if (flags & FLAG_VAES_AVX512) 311 aes_gcm_enc_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key), 312 le_ctr, ghash_acc, 313 total_aadlen, total_datalen); 314 else if (flags & FLAG_VAES_AVX2) 315 aes_gcm_enc_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key), 316 le_ctr, ghash_acc, 317 total_aadlen, total_datalen); 318 else if (flags & FLAG_AVX) 319 aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key), 320 le_ctr, ghash_acc, 321 total_aadlen, total_datalen); 322 else 323 aes_gcm_enc_final_aesni(AES_GCM_KEY_AESNI(key), 324 le_ctr, ghash_acc, 325 total_aadlen, total_datalen); 326 } 327 328 asmlinkage bool __must_check 329 aes_gcm_dec_final_aesni(const struct aes_gcm_key_aesni *key, 330 const u32 le_ctr[4], const u8 ghash_acc[16], 331 u64 total_aadlen, u64 total_datalen, 332 const u8 tag[16], int taglen); 333 asmlinkage bool __must_check 334 aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key, 335 const u32 le_ctr[4], const u8 ghash_acc[16], 336 u64 total_aadlen, u64 total_datalen, 337 const u8 tag[16], int taglen); 338 asmlinkage bool __must_check 339 aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 340 const u32 le_ctr[4], const u8 ghash_acc[16], 341 u64 total_aadlen, u64 total_datalen, 342 const u8 tag[16], int taglen); 343 asmlinkage bool __must_check 344 aes_gcm_dec_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 345 const u32 le_ctr[4], const u8 ghash_acc[16], 346 u64 total_aadlen, u64 total_datalen, 347 const u8 tag[16], int taglen); 348 349 /* __always_inline to optimize out the branches based on @flags */ 350 static __always_inline bool __must_check 351 aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4], 352 u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen, 353 u8 tag[16], int taglen, int flags) 354 { 355 if (flags & FLAG_VAES_AVX512) 356 return aes_gcm_dec_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key), 357 le_ctr, ghash_acc, 358 total_aadlen, total_datalen, 359 tag, taglen); 360 else if (flags & FLAG_VAES_AVX2) 361 return aes_gcm_dec_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key), 362 le_ctr, ghash_acc, 363 total_aadlen, total_datalen, 364 tag, taglen); 365 else if (flags & FLAG_AVX) 366 return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key), 367 le_ctr, ghash_acc, 368 total_aadlen, total_datalen, 369 tag, taglen); 370 else 371 return aes_gcm_dec_final_aesni(AES_GCM_KEY_AESNI(key), 372 le_ctr, ghash_acc, 373 total_aadlen, total_datalen, 374 tag, taglen); 375 } 376 377 /* 378 * This is the Integrity Check Value (aka the authentication tag) length and can 379 * be 8, 12 or 16 bytes long. 380 */ 381 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 382 unsigned int authsize) 383 { 384 switch (authsize) { 385 case 8: 386 case 12: 387 case 16: 388 break; 389 default: 390 return -EINVAL; 391 } 392 393 return 0; 394 } 395 396 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 397 unsigned int authsize) 398 { 399 switch (authsize) { 400 case 4: 401 case 8: 402 case 12: 403 case 13: 404 case 14: 405 case 15: 406 case 16: 407 break; 408 default: 409 return -EINVAL; 410 } 411 412 return 0; 413 } 414 415 /* 416 * This is the setkey function for the x86_64 implementations of AES-GCM. It 417 * saves the RFC4106 nonce if applicable, expands the AES key, and precomputes 418 * powers of the hash key. 419 * 420 * To comply with the crypto_aead API, this has to be usable in no-SIMD context. 421 * For that reason, this function includes a portable C implementation of the 422 * needed logic. However, the portable C implementation is very slow, taking 423 * about the same time as encrypting 37 KB of data. To be ready for users that 424 * may set a key even somewhat frequently, we therefore also include a SIMD 425 * assembly implementation, expanding the AES key using AES-NI and precomputing 426 * the hash key powers using PCLMULQDQ or VPCLMULQDQ. 427 */ 428 static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key, 429 unsigned int keylen, int flags) 430 { 431 struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags); 432 int err; 433 434 if (flags & FLAG_RFC4106) { 435 if (keylen < 4) 436 return -EINVAL; 437 keylen -= 4; 438 key->rfc4106_nonce = get_unaligned_be32(raw_key + keylen); 439 } 440 441 /* The assembly code assumes the following offsets. */ 442 static_assert(offsetof(struct aes_gcm_key_aesni, base.aes_key.len) == 0); 443 static_assert(offsetof(struct aes_gcm_key_aesni, base.aes_key.k.rndkeys) == 16); 444 static_assert(offsetof(struct aes_gcm_key_aesni, h_powers) == 272); 445 static_assert(offsetof(struct aes_gcm_key_aesni, h_powers_xored) == 400); 446 static_assert(offsetof(struct aes_gcm_key_aesni, h_times_x64) == 464); 447 static_assert(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.len) == 0); 448 static_assert(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.k.rndkeys) == 16); 449 static_assert(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) == 288); 450 static_assert(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) == 416); 451 static_assert(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.len) == 0); 452 static_assert(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.k.rndkeys) == 16); 453 static_assert(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) == 320); 454 static_assert(offsetof(struct aes_gcm_key_vaes_avx512, padding) == 576); 455 456 err = aes_prepareenckey(&key->aes_key, raw_key, keylen); 457 if (err) 458 return err; 459 460 if (likely(crypto_simd_usable())) { 461 kernel_fpu_begin(); 462 aes_gcm_precompute(key, flags); 463 kernel_fpu_end(); 464 } else { 465 static const u8 x_to_the_minus1[16] __aligned(__alignof__(be128)) = { 466 [0] = 0xc2, [15] = 1 467 }; 468 static const u8 x_to_the_63[16] __aligned(__alignof__(be128)) = { 469 [7] = 1, 470 }; 471 be128 h1 = {}; 472 be128 h; 473 int i; 474 475 /* Encrypt the all-zeroes block to get the hash key H^1 */ 476 aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1); 477 478 /* Compute H^1 * x^-1 */ 479 h = h1; 480 gf128mul_lle(&h, (const be128 *)x_to_the_minus1); 481 482 /* Compute the needed key powers */ 483 if (flags & FLAG_VAES_AVX512) { 484 struct aes_gcm_key_vaes_avx512 *k = 485 AES_GCM_KEY_VAES_AVX512(key); 486 487 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 488 k->h_powers[i][0] = be64_to_cpu(h.b); 489 k->h_powers[i][1] = be64_to_cpu(h.a); 490 gf128mul_lle(&h, &h1); 491 } 492 memset(k->padding, 0, sizeof(k->padding)); 493 } else if (flags & FLAG_VAES_AVX2) { 494 struct aes_gcm_key_vaes_avx2 *k = 495 AES_GCM_KEY_VAES_AVX2(key); 496 static const u8 indices[8] = { 0, 2, 1, 3, 4, 6, 5, 7 }; 497 498 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 499 k->h_powers[i][0] = be64_to_cpu(h.b); 500 k->h_powers[i][1] = be64_to_cpu(h.a); 501 gf128mul_lle(&h, &h1); 502 } 503 for (i = 0; i < ARRAY_SIZE(k->h_powers_xored); i++) { 504 int j = indices[i]; 505 506 k->h_powers_xored[i] = k->h_powers[j][0] ^ 507 k->h_powers[j][1]; 508 } 509 } else { 510 struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key); 511 512 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 513 k->h_powers[i][0] = be64_to_cpu(h.b); 514 k->h_powers[i][1] = be64_to_cpu(h.a); 515 k->h_powers_xored[i] = k->h_powers[i][0] ^ 516 k->h_powers[i][1]; 517 gf128mul_lle(&h, &h1); 518 } 519 gf128mul_lle(&h1, (const be128 *)x_to_the_63); 520 k->h_times_x64[0] = be64_to_cpu(h1.b); 521 k->h_times_x64[1] = be64_to_cpu(h1.a); 522 } 523 } 524 return 0; 525 } 526 527 /* 528 * Initialize @ghash_acc, then pass all @assoclen bytes of associated data 529 * (a.k.a. additional authenticated data) from @sg_src through the GHASH update 530 * assembly function. kernel_fpu_begin() must have already been called. 531 */ 532 static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], 533 struct scatterlist *sg_src, unsigned int assoclen, 534 int flags) 535 { 536 struct scatter_walk walk; 537 /* 538 * The assembly function requires that the length of any non-last 539 * segment of associated data be a multiple of 16 bytes, so this 540 * function does the buffering needed to achieve that. 541 */ 542 unsigned int pos = 0; 543 u8 buf[16]; 544 545 memset(ghash_acc, 0, 16); 546 scatterwalk_start(&walk, sg_src); 547 548 while (assoclen) { 549 unsigned int orig_len_this_step = scatterwalk_next( 550 &walk, assoclen); 551 unsigned int len_this_step = orig_len_this_step; 552 unsigned int len; 553 const u8 *src = walk.addr; 554 555 assoclen -= orig_len_this_step; 556 if (unlikely(pos)) { 557 len = min(len_this_step, 16 - pos); 558 memcpy(&buf[pos], src, len); 559 pos += len; 560 src += len; 561 len_this_step -= len; 562 if (pos < 16) 563 goto next; 564 aes_gcm_aad_update(key, ghash_acc, buf, 16, flags); 565 pos = 0; 566 } 567 len = len_this_step; 568 if (unlikely(assoclen)) /* Not the last segment yet? */ 569 len = round_down(len, 16); 570 aes_gcm_aad_update(key, ghash_acc, src, len, flags); 571 src += len; 572 len_this_step -= len; 573 if (unlikely(len_this_step)) { 574 memcpy(buf, src, len_this_step); 575 pos = len_this_step; 576 } 577 next: 578 scatterwalk_done_src(&walk, orig_len_this_step); 579 if (need_resched()) { 580 kernel_fpu_end(); 581 kernel_fpu_begin(); 582 } 583 } 584 if (unlikely(pos)) 585 aes_gcm_aad_update(key, ghash_acc, buf, pos, flags); 586 } 587 588 589 /* __always_inline to optimize out the branches based on @flags */ 590 static __always_inline int 591 gcm_crypt(struct aead_request *req, int flags) 592 { 593 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 594 const struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags); 595 unsigned int assoclen = req->assoclen; 596 struct skcipher_walk walk; 597 unsigned int nbytes; 598 u8 ghash_acc[16]; /* GHASH accumulator */ 599 u32 le_ctr[4]; /* Counter in little-endian format */ 600 int taglen; 601 int err; 602 603 /* Initialize the counter and determine the associated data length. */ 604 le_ctr[0] = 2; 605 if (flags & FLAG_RFC4106) { 606 if (unlikely(assoclen != 16 && assoclen != 20)) 607 return -EINVAL; 608 assoclen -= 8; 609 le_ctr[1] = get_unaligned_be32(req->iv + 4); 610 le_ctr[2] = get_unaligned_be32(req->iv + 0); 611 le_ctr[3] = key->rfc4106_nonce; /* already byte-swapped */ 612 } else { 613 le_ctr[1] = get_unaligned_be32(req->iv + 8); 614 le_ctr[2] = get_unaligned_be32(req->iv + 4); 615 le_ctr[3] = get_unaligned_be32(req->iv + 0); 616 } 617 618 /* Begin walking through the plaintext or ciphertext. */ 619 if (flags & FLAG_ENC) 620 err = skcipher_walk_aead_encrypt(&walk, req, false); 621 else 622 err = skcipher_walk_aead_decrypt(&walk, req, false); 623 if (err) 624 return err; 625 626 /* 627 * Since the AES-GCM assembly code requires that at least three assembly 628 * functions be called to process any message (this is needed to support 629 * incremental updates cleanly), to reduce overhead we try to do all 630 * three calls in the same kernel FPU section if possible. We close the 631 * section and start a new one if there are multiple data segments or if 632 * rescheduling is needed while processing the associated data. 633 */ 634 kernel_fpu_begin(); 635 636 /* Pass the associated data through GHASH. */ 637 gcm_process_assoc(key, ghash_acc, req->src, assoclen, flags); 638 639 /* En/decrypt the data and pass the ciphertext through GHASH. */ 640 while (unlikely((nbytes = walk.nbytes) < walk.total)) { 641 /* 642 * Non-last segment. In this case, the assembly function 643 * requires that the length be a multiple of 16 (AES_BLOCK_SIZE) 644 * bytes. The needed buffering of up to 16 bytes is handled by 645 * the skcipher_walk. Here we just need to round down to a 646 * multiple of 16. 647 */ 648 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 649 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr, 650 walk.dst.virt.addr, nbytes, flags); 651 le_ctr[0] += nbytes / AES_BLOCK_SIZE; 652 kernel_fpu_end(); 653 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 654 if (err) 655 return err; 656 kernel_fpu_begin(); 657 } 658 /* Last segment: process all remaining data. */ 659 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr, 660 walk.dst.virt.addr, nbytes, flags); 661 /* 662 * The low word of the counter isn't used by the finalize, so there's no 663 * need to increment it here. 664 */ 665 666 /* Finalize */ 667 taglen = crypto_aead_authsize(tfm); 668 if (flags & FLAG_ENC) { 669 /* Finish computing the auth tag. */ 670 aes_gcm_enc_final(key, le_ctr, ghash_acc, assoclen, 671 req->cryptlen, flags); 672 673 /* Store the computed auth tag in the dst scatterlist. */ 674 scatterwalk_map_and_copy(ghash_acc, req->dst, req->assoclen + 675 req->cryptlen, taglen, 1); 676 } else { 677 unsigned int datalen = req->cryptlen - taglen; 678 u8 tag[16]; 679 680 /* Get the transmitted auth tag from the src scatterlist. */ 681 scatterwalk_map_and_copy(tag, req->src, req->assoclen + datalen, 682 taglen, 0); 683 /* 684 * Finish computing the auth tag and compare it to the 685 * transmitted one. The assembly function does the actual tag 686 * comparison. Here, just check the boolean result. 687 */ 688 if (!aes_gcm_dec_final(key, le_ctr, ghash_acc, assoclen, 689 datalen, tag, taglen, flags)) 690 err = -EBADMSG; 691 } 692 kernel_fpu_end(); 693 if (nbytes) 694 skcipher_walk_done(&walk, 0); 695 return err; 696 } 697 698 #define DEFINE_GCM_ALGS(suffix, flags, generic_driver_name, rfc_driver_name, \ 699 ctxsize, priority) \ 700 \ 701 static int gcm_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \ 702 unsigned int keylen) \ 703 { \ 704 return gcm_setkey(tfm, raw_key, keylen, (flags)); \ 705 } \ 706 \ 707 static int gcm_encrypt_##suffix(struct aead_request *req) \ 708 { \ 709 return gcm_crypt(req, (flags) | FLAG_ENC); \ 710 } \ 711 \ 712 static int gcm_decrypt_##suffix(struct aead_request *req) \ 713 { \ 714 return gcm_crypt(req, (flags)); \ 715 } \ 716 \ 717 static int rfc4106_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \ 718 unsigned int keylen) \ 719 { \ 720 return gcm_setkey(tfm, raw_key, keylen, (flags) | FLAG_RFC4106); \ 721 } \ 722 \ 723 static int rfc4106_encrypt_##suffix(struct aead_request *req) \ 724 { \ 725 return gcm_crypt(req, (flags) | FLAG_RFC4106 | FLAG_ENC); \ 726 } \ 727 \ 728 static int rfc4106_decrypt_##suffix(struct aead_request *req) \ 729 { \ 730 return gcm_crypt(req, (flags) | FLAG_RFC4106); \ 731 } \ 732 \ 733 static struct aead_alg aes_gcm_algs_##suffix[] = { { \ 734 .setkey = gcm_setkey_##suffix, \ 735 .setauthsize = generic_gcmaes_set_authsize, \ 736 .encrypt = gcm_encrypt_##suffix, \ 737 .decrypt = gcm_decrypt_##suffix, \ 738 .ivsize = GCM_AES_IV_SIZE, \ 739 .chunksize = AES_BLOCK_SIZE, \ 740 .maxauthsize = 16, \ 741 .base = { \ 742 .cra_name = "gcm(aes)", \ 743 .cra_driver_name = generic_driver_name, \ 744 .cra_priority = (priority), \ 745 .cra_blocksize = 1, \ 746 .cra_ctxsize = (ctxsize), \ 747 .cra_module = THIS_MODULE, \ 748 }, \ 749 }, { \ 750 .setkey = rfc4106_setkey_##suffix, \ 751 .setauthsize = common_rfc4106_set_authsize, \ 752 .encrypt = rfc4106_encrypt_##suffix, \ 753 .decrypt = rfc4106_decrypt_##suffix, \ 754 .ivsize = GCM_RFC4106_IV_SIZE, \ 755 .chunksize = AES_BLOCK_SIZE, \ 756 .maxauthsize = 16, \ 757 .base = { \ 758 .cra_name = "rfc4106(gcm(aes))", \ 759 .cra_driver_name = rfc_driver_name, \ 760 .cra_priority = (priority), \ 761 .cra_blocksize = 1, \ 762 .cra_ctxsize = (ctxsize), \ 763 .cra_module = THIS_MODULE, \ 764 }, \ 765 } } 766 767 /* aes_gcm_algs_aesni */ 768 DEFINE_GCM_ALGS(aesni, /* no flags */ 0, 769 "generic-gcm-aesni", "rfc4106-gcm-aesni", 770 AES_GCM_KEY_AESNI_SIZE, 400); 771 772 /* aes_gcm_algs_aesni_avx */ 773 DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX, 774 "generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx", 775 AES_GCM_KEY_AESNI_SIZE, 500); 776 777 /* aes_gcm_algs_vaes_avx2 */ 778 DEFINE_GCM_ALGS(vaes_avx2, FLAG_VAES_AVX2, 779 "generic-gcm-vaes-avx2", "rfc4106-gcm-vaes-avx2", 780 AES_GCM_KEY_VAES_AVX2_SIZE, 600); 781 782 /* aes_gcm_algs_vaes_avx512 */ 783 DEFINE_GCM_ALGS(vaes_avx512, FLAG_VAES_AVX512, 784 "generic-gcm-vaes-avx512", "rfc4106-gcm-vaes-avx512", 785 AES_GCM_KEY_VAES_AVX512_SIZE, 800); 786 787 static int __init register_avx_algs(void) 788 { 789 int err; 790 791 if (!boot_cpu_has(X86_FEATURE_AVX)) 792 return 0; 793 err = crypto_register_aeads(aes_gcm_algs_aesni_avx, 794 ARRAY_SIZE(aes_gcm_algs_aesni_avx)); 795 if (err) 796 return err; 797 /* 798 * Note: not all the algorithms registered below actually require 799 * VPCLMULQDQ. But in practice every CPU with VAES also has VPCLMULQDQ. 800 * Similarly, the assembler support was added at about the same time. 801 * For simplicity, just always check for VAES and VPCLMULQDQ together. 802 */ 803 if (!boot_cpu_has(X86_FEATURE_AVX2) || 804 !boot_cpu_has(X86_FEATURE_VAES) || 805 !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) || 806 !boot_cpu_has(X86_FEATURE_PCLMULQDQ)) 807 return 0; 808 err = crypto_register_aeads(aes_gcm_algs_vaes_avx2, 809 ARRAY_SIZE(aes_gcm_algs_vaes_avx2)); 810 if (err) 811 return err; 812 813 if (!boot_cpu_has(X86_FEATURE_AVX512BW) || 814 !boot_cpu_has(X86_FEATURE_AVX512VL) || 815 !boot_cpu_has(X86_FEATURE_BMI2)) 816 return 0; 817 818 if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) { 819 int i; 820 821 for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx512); i++) 822 aes_gcm_algs_vaes_avx512[i].base.cra_priority = 1; 823 } 824 825 err = crypto_register_aeads(aes_gcm_algs_vaes_avx512, 826 ARRAY_SIZE(aes_gcm_algs_vaes_avx512)); 827 if (err) 828 return err; 829 830 return 0; 831 } 832 833 #define unregister_aeads(A) \ 834 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ 835 crypto_unregister_aeads((A), ARRAY_SIZE(A)) 836 837 static void unregister_avx_algs(void) 838 { 839 unregister_aeads(aes_gcm_algs_aesni_avx); 840 unregister_aeads(aes_gcm_algs_vaes_avx2); 841 unregister_aeads(aes_gcm_algs_vaes_avx512); 842 } 843 844 static const struct x86_cpu_id aesni_cpu_id[] = { 845 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), 846 {} 847 }; 848 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 849 850 static int __init aesni_init(void) 851 { 852 int err; 853 854 if (!x86_match_cpu(aesni_cpu_id)) 855 return -ENODEV; 856 857 err = crypto_register_aeads(aes_gcm_algs_aesni, 858 ARRAY_SIZE(aes_gcm_algs_aesni)); 859 if (err) 860 return err; 861 862 err = register_avx_algs(); 863 if (err) 864 goto unregister_avx; 865 866 return 0; 867 868 unregister_avx: 869 unregister_avx_algs(); 870 crypto_unregister_aeads(aes_gcm_algs_aesni, 871 ARRAY_SIZE(aes_gcm_algs_aesni)); 872 return err; 873 } 874 875 static void __exit aesni_exit(void) 876 { 877 crypto_unregister_aeads(aes_gcm_algs_aesni, 878 ARRAY_SIZE(aes_gcm_algs_aesni)); 879 unregister_avx_algs(); 880 } 881 882 module_init(aesni_init); 883 module_exit(aesni_exit); 884 885 MODULE_DESCRIPTION("AES-GCM, optimized with AES-NI or VAES instructions"); 886 MODULE_LICENSE("GPL"); 887 MODULE_ALIAS_CRYPTO("gcm(aes)"); 888 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); 889