1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Support for AES-NI and VAES instructions. This file contains glue code. 4 * The real AES implementations are in aesni-intel_asm.S and other .S files. 5 * 6 * Copyright (C) 2008, Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * 9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 10 * interface for 64-bit kernels. 11 * Authors: Adrian Hoban <adrian.hoban@intel.com> 12 * Gabriele Paoloni <gabriele.paoloni@intel.com> 13 * Tadeusz Struk (tadeusz.struk@intel.com) 14 * Aidan O'Mahony (aidan.o.mahony@intel.com) 15 * Copyright (c) 2010, Intel Corporation. 16 * 17 * Copyright 2024 Google LLC 18 */ 19 20 #include <linux/hardirq.h> 21 #include <linux/types.h> 22 #include <linux/module.h> 23 #include <linux/err.h> 24 #include <crypto/algapi.h> 25 #include <crypto/aes.h> 26 #include <crypto/b128ops.h> 27 #include <crypto/gcm.h> 28 #include <crypto/xts.h> 29 #include <asm/cpu_device_id.h> 30 #include <asm/simd.h> 31 #include <crypto/scatterwalk.h> 32 #include <crypto/internal/aead.h> 33 #include <crypto/internal/simd.h> 34 #include <crypto/internal/skcipher.h> 35 #include <linux/jump_label.h> 36 #include <linux/workqueue.h> 37 #include <linux/spinlock.h> 38 #include <linux/static_call.h> 39 40 41 #define AESNI_ALIGN 16 42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 43 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 47 48 struct aesni_xts_ctx { 49 struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR; 50 struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR; 51 }; 52 53 static inline void *aes_align_addr(void *addr) 54 { 55 if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN) 56 return addr; 57 return PTR_ALIGN(addr, AESNI_ALIGN); 58 } 59 60 asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 61 unsigned int key_len); 62 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in); 63 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in); 64 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 65 const u8 *in, unsigned int len); 66 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 67 const u8 *in, unsigned int len); 68 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 69 const u8 *in, unsigned int len, u8 *iv); 70 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 71 const u8 *in, unsigned int len, u8 *iv); 72 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 73 const u8 *in, unsigned int len, u8 *iv); 74 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 75 const u8 *in, unsigned int len, u8 *iv); 76 77 asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out, 78 const u8 *in, unsigned int len, u8 *iv); 79 80 asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out, 81 const u8 *in, unsigned int len, u8 *iv); 82 83 #ifdef CONFIG_X86_64 84 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 85 const u8 *in, unsigned int len, u8 *iv); 86 #endif 87 88 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 89 { 90 return aes_align_addr(raw_ctx); 91 } 92 93 static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm) 94 { 95 return aes_align_addr(crypto_skcipher_ctx(tfm)); 96 } 97 98 static int aes_set_key_common(struct crypto_aes_ctx *ctx, 99 const u8 *in_key, unsigned int key_len) 100 { 101 int err; 102 103 if (!crypto_simd_usable()) 104 return aes_expandkey(ctx, in_key, key_len); 105 106 err = aes_check_keylen(key_len); 107 if (err) 108 return err; 109 110 kernel_fpu_begin(); 111 aesni_set_key(ctx, in_key, key_len); 112 kernel_fpu_end(); 113 return 0; 114 } 115 116 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 117 unsigned int key_len) 118 { 119 return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key, 120 key_len); 121 } 122 123 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 124 { 125 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 126 127 if (!crypto_simd_usable()) { 128 aes_encrypt(ctx, dst, src); 129 } else { 130 kernel_fpu_begin(); 131 aesni_enc(ctx, dst, src); 132 kernel_fpu_end(); 133 } 134 } 135 136 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 137 { 138 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 139 140 if (!crypto_simd_usable()) { 141 aes_decrypt(ctx, dst, src); 142 } else { 143 kernel_fpu_begin(); 144 aesni_dec(ctx, dst, src); 145 kernel_fpu_end(); 146 } 147 } 148 149 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 150 unsigned int len) 151 { 152 return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len); 153 } 154 155 static int ecb_encrypt(struct skcipher_request *req) 156 { 157 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 158 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 159 struct skcipher_walk walk; 160 unsigned int nbytes; 161 int err; 162 163 err = skcipher_walk_virt(&walk, req, false); 164 165 while ((nbytes = walk.nbytes)) { 166 kernel_fpu_begin(); 167 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 168 nbytes & AES_BLOCK_MASK); 169 kernel_fpu_end(); 170 nbytes &= AES_BLOCK_SIZE - 1; 171 err = skcipher_walk_done(&walk, nbytes); 172 } 173 174 return err; 175 } 176 177 static int ecb_decrypt(struct skcipher_request *req) 178 { 179 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 180 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 181 struct skcipher_walk walk; 182 unsigned int nbytes; 183 int err; 184 185 err = skcipher_walk_virt(&walk, req, false); 186 187 while ((nbytes = walk.nbytes)) { 188 kernel_fpu_begin(); 189 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 190 nbytes & AES_BLOCK_MASK); 191 kernel_fpu_end(); 192 nbytes &= AES_BLOCK_SIZE - 1; 193 err = skcipher_walk_done(&walk, nbytes); 194 } 195 196 return err; 197 } 198 199 static int cbc_encrypt(struct skcipher_request *req) 200 { 201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 202 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 203 struct skcipher_walk walk; 204 unsigned int nbytes; 205 int err; 206 207 err = skcipher_walk_virt(&walk, req, false); 208 209 while ((nbytes = walk.nbytes)) { 210 kernel_fpu_begin(); 211 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 212 nbytes & AES_BLOCK_MASK, walk.iv); 213 kernel_fpu_end(); 214 nbytes &= AES_BLOCK_SIZE - 1; 215 err = skcipher_walk_done(&walk, nbytes); 216 } 217 218 return err; 219 } 220 221 static int cbc_decrypt(struct skcipher_request *req) 222 { 223 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 224 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 225 struct skcipher_walk walk; 226 unsigned int nbytes; 227 int err; 228 229 err = skcipher_walk_virt(&walk, req, false); 230 231 while ((nbytes = walk.nbytes)) { 232 kernel_fpu_begin(); 233 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 234 nbytes & AES_BLOCK_MASK, walk.iv); 235 kernel_fpu_end(); 236 nbytes &= AES_BLOCK_SIZE - 1; 237 err = skcipher_walk_done(&walk, nbytes); 238 } 239 240 return err; 241 } 242 243 static int cts_cbc_encrypt(struct skcipher_request *req) 244 { 245 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 246 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 247 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 248 struct scatterlist *src = req->src, *dst = req->dst; 249 struct scatterlist sg_src[2], sg_dst[2]; 250 struct skcipher_request subreq; 251 struct skcipher_walk walk; 252 int err; 253 254 skcipher_request_set_tfm(&subreq, tfm); 255 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 256 NULL, NULL); 257 258 if (req->cryptlen <= AES_BLOCK_SIZE) { 259 if (req->cryptlen < AES_BLOCK_SIZE) 260 return -EINVAL; 261 cbc_blocks = 1; 262 } 263 264 if (cbc_blocks > 0) { 265 skcipher_request_set_crypt(&subreq, req->src, req->dst, 266 cbc_blocks * AES_BLOCK_SIZE, 267 req->iv); 268 269 err = cbc_encrypt(&subreq); 270 if (err) 271 return err; 272 273 if (req->cryptlen == AES_BLOCK_SIZE) 274 return 0; 275 276 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 277 if (req->dst != req->src) 278 dst = scatterwalk_ffwd(sg_dst, req->dst, 279 subreq.cryptlen); 280 } 281 282 /* handle ciphertext stealing */ 283 skcipher_request_set_crypt(&subreq, src, dst, 284 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 285 req->iv); 286 287 err = skcipher_walk_virt(&walk, &subreq, false); 288 if (err) 289 return err; 290 291 kernel_fpu_begin(); 292 aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 293 walk.nbytes, walk.iv); 294 kernel_fpu_end(); 295 296 return skcipher_walk_done(&walk, 0); 297 } 298 299 static int cts_cbc_decrypt(struct skcipher_request *req) 300 { 301 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 302 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 303 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 304 struct scatterlist *src = req->src, *dst = req->dst; 305 struct scatterlist sg_src[2], sg_dst[2]; 306 struct skcipher_request subreq; 307 struct skcipher_walk walk; 308 int err; 309 310 skcipher_request_set_tfm(&subreq, tfm); 311 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 312 NULL, NULL); 313 314 if (req->cryptlen <= AES_BLOCK_SIZE) { 315 if (req->cryptlen < AES_BLOCK_SIZE) 316 return -EINVAL; 317 cbc_blocks = 1; 318 } 319 320 if (cbc_blocks > 0) { 321 skcipher_request_set_crypt(&subreq, req->src, req->dst, 322 cbc_blocks * AES_BLOCK_SIZE, 323 req->iv); 324 325 err = cbc_decrypt(&subreq); 326 if (err) 327 return err; 328 329 if (req->cryptlen == AES_BLOCK_SIZE) 330 return 0; 331 332 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 333 if (req->dst != req->src) 334 dst = scatterwalk_ffwd(sg_dst, req->dst, 335 subreq.cryptlen); 336 } 337 338 /* handle ciphertext stealing */ 339 skcipher_request_set_crypt(&subreq, src, dst, 340 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 341 req->iv); 342 343 err = skcipher_walk_virt(&walk, &subreq, false); 344 if (err) 345 return err; 346 347 kernel_fpu_begin(); 348 aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 349 walk.nbytes, walk.iv); 350 kernel_fpu_end(); 351 352 return skcipher_walk_done(&walk, 0); 353 } 354 355 #ifdef CONFIG_X86_64 356 /* This is the non-AVX version. */ 357 static int ctr_crypt_aesni(struct skcipher_request *req) 358 { 359 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 361 u8 keystream[AES_BLOCK_SIZE]; 362 struct skcipher_walk walk; 363 unsigned int nbytes; 364 int err; 365 366 err = skcipher_walk_virt(&walk, req, false); 367 368 while ((nbytes = walk.nbytes) > 0) { 369 kernel_fpu_begin(); 370 if (nbytes & AES_BLOCK_MASK) 371 aesni_ctr_enc(ctx, walk.dst.virt.addr, 372 walk.src.virt.addr, 373 nbytes & AES_BLOCK_MASK, walk.iv); 374 nbytes &= ~AES_BLOCK_MASK; 375 376 if (walk.nbytes == walk.total && nbytes > 0) { 377 aesni_enc(ctx, keystream, walk.iv); 378 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes, 379 walk.src.virt.addr + walk.nbytes - nbytes, 380 keystream, nbytes); 381 crypto_inc(walk.iv, AES_BLOCK_SIZE); 382 nbytes = 0; 383 } 384 kernel_fpu_end(); 385 err = skcipher_walk_done(&walk, nbytes); 386 } 387 return err; 388 } 389 #endif 390 391 static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key, 392 unsigned int keylen) 393 { 394 struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); 395 int err; 396 397 err = xts_verify_key(tfm, key, keylen); 398 if (err) 399 return err; 400 401 keylen /= 2; 402 403 /* first half of xts-key is for crypt */ 404 err = aes_set_key_common(&ctx->crypt_ctx, key, keylen); 405 if (err) 406 return err; 407 408 /* second half of xts-key is for tweak */ 409 return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen); 410 } 411 412 typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key, 413 u8 iv[AES_BLOCK_SIZE]); 414 typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key, 415 const u8 *src, u8 *dst, int len, 416 u8 tweak[AES_BLOCK_SIZE]); 417 418 /* This handles cases where the source and/or destination span pages. */ 419 static noinline int 420 xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func) 421 { 422 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 423 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); 424 int tail = req->cryptlen % AES_BLOCK_SIZE; 425 struct scatterlist sg_src[2], sg_dst[2]; 426 struct skcipher_request subreq; 427 struct skcipher_walk walk; 428 struct scatterlist *src, *dst; 429 int err; 430 431 /* 432 * If the message length isn't divisible by the AES block size, then 433 * separate off the last full block and the partial block. This ensures 434 * that they are processed in the same call to the assembly function, 435 * which is required for ciphertext stealing. 436 */ 437 if (tail) { 438 skcipher_request_set_tfm(&subreq, tfm); 439 skcipher_request_set_callback(&subreq, 440 skcipher_request_flags(req), 441 NULL, NULL); 442 skcipher_request_set_crypt(&subreq, req->src, req->dst, 443 req->cryptlen - tail - AES_BLOCK_SIZE, 444 req->iv); 445 req = &subreq; 446 } 447 448 err = skcipher_walk_virt(&walk, req, false); 449 450 while (walk.nbytes) { 451 kernel_fpu_begin(); 452 (*crypt_func)(&ctx->crypt_ctx, 453 walk.src.virt.addr, walk.dst.virt.addr, 454 walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv); 455 kernel_fpu_end(); 456 err = skcipher_walk_done(&walk, 457 walk.nbytes & (AES_BLOCK_SIZE - 1)); 458 } 459 460 if (err || !tail) 461 return err; 462 463 /* Do ciphertext stealing with the last full block and partial block. */ 464 465 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 466 if (req->dst != req->src) 467 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 468 469 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 470 req->iv); 471 472 err = skcipher_walk_virt(&walk, req, false); 473 if (err) 474 return err; 475 476 kernel_fpu_begin(); 477 (*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr, 478 walk.nbytes, req->iv); 479 kernel_fpu_end(); 480 481 return skcipher_walk_done(&walk, 0); 482 } 483 484 /* __always_inline to avoid indirect call in fastpath */ 485 static __always_inline int 486 xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv, 487 xts_crypt_func crypt_func) 488 { 489 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 490 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); 491 492 if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) 493 return -EINVAL; 494 495 kernel_fpu_begin(); 496 (*encrypt_iv)(&ctx->tweak_ctx, req->iv); 497 498 /* 499 * In practice, virtually all XTS plaintexts and ciphertexts are either 500 * 512 or 4096 bytes and do not use multiple scatterlist elements. To 501 * optimize the performance of these cases, the below fast-path handles 502 * single-scatterlist-element messages as efficiently as possible. The 503 * code is 64-bit specific, as it assumes no page mapping is needed. 504 */ 505 if (IS_ENABLED(CONFIG_X86_64) && 506 likely(req->src->length >= req->cryptlen && 507 req->dst->length >= req->cryptlen)) { 508 (*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src), 509 sg_virt(req->dst), req->cryptlen, req->iv); 510 kernel_fpu_end(); 511 return 0; 512 } 513 kernel_fpu_end(); 514 return xts_crypt_slowpath(req, crypt_func); 515 } 516 517 static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, 518 u8 iv[AES_BLOCK_SIZE]) 519 { 520 aesni_enc(tweak_key, iv, iv); 521 } 522 523 static void aesni_xts_encrypt(const struct crypto_aes_ctx *key, 524 const u8 *src, u8 *dst, int len, 525 u8 tweak[AES_BLOCK_SIZE]) 526 { 527 aesni_xts_enc(key, dst, src, len, tweak); 528 } 529 530 static void aesni_xts_decrypt(const struct crypto_aes_ctx *key, 531 const u8 *src, u8 *dst, int len, 532 u8 tweak[AES_BLOCK_SIZE]) 533 { 534 aesni_xts_dec(key, dst, src, len, tweak); 535 } 536 537 static int xts_encrypt_aesni(struct skcipher_request *req) 538 { 539 return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_encrypt); 540 } 541 542 static int xts_decrypt_aesni(struct skcipher_request *req) 543 { 544 return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_decrypt); 545 } 546 547 static struct crypto_alg aesni_cipher_alg = { 548 .cra_name = "aes", 549 .cra_driver_name = "aes-aesni", 550 .cra_priority = 300, 551 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 552 .cra_blocksize = AES_BLOCK_SIZE, 553 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 554 .cra_module = THIS_MODULE, 555 .cra_u = { 556 .cipher = { 557 .cia_min_keysize = AES_MIN_KEY_SIZE, 558 .cia_max_keysize = AES_MAX_KEY_SIZE, 559 .cia_setkey = aes_set_key, 560 .cia_encrypt = aesni_encrypt, 561 .cia_decrypt = aesni_decrypt 562 } 563 } 564 }; 565 566 static struct skcipher_alg aesni_skciphers[] = { 567 { 568 .base = { 569 .cra_name = "ecb(aes)", 570 .cra_driver_name = "ecb-aes-aesni", 571 .cra_priority = 400, 572 .cra_blocksize = AES_BLOCK_SIZE, 573 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 574 .cra_module = THIS_MODULE, 575 }, 576 .min_keysize = AES_MIN_KEY_SIZE, 577 .max_keysize = AES_MAX_KEY_SIZE, 578 .setkey = aesni_skcipher_setkey, 579 .encrypt = ecb_encrypt, 580 .decrypt = ecb_decrypt, 581 }, { 582 .base = { 583 .cra_name = "cbc(aes)", 584 .cra_driver_name = "cbc-aes-aesni", 585 .cra_priority = 400, 586 .cra_blocksize = AES_BLOCK_SIZE, 587 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 588 .cra_module = THIS_MODULE, 589 }, 590 .min_keysize = AES_MIN_KEY_SIZE, 591 .max_keysize = AES_MAX_KEY_SIZE, 592 .ivsize = AES_BLOCK_SIZE, 593 .setkey = aesni_skcipher_setkey, 594 .encrypt = cbc_encrypt, 595 .decrypt = cbc_decrypt, 596 }, { 597 .base = { 598 .cra_name = "cts(cbc(aes))", 599 .cra_driver_name = "cts-cbc-aes-aesni", 600 .cra_priority = 400, 601 .cra_blocksize = AES_BLOCK_SIZE, 602 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 603 .cra_module = THIS_MODULE, 604 }, 605 .min_keysize = AES_MIN_KEY_SIZE, 606 .max_keysize = AES_MAX_KEY_SIZE, 607 .ivsize = AES_BLOCK_SIZE, 608 .walksize = 2 * AES_BLOCK_SIZE, 609 .setkey = aesni_skcipher_setkey, 610 .encrypt = cts_cbc_encrypt, 611 .decrypt = cts_cbc_decrypt, 612 #ifdef CONFIG_X86_64 613 }, { 614 .base = { 615 .cra_name = "ctr(aes)", 616 .cra_driver_name = "ctr-aes-aesni", 617 .cra_priority = 400, 618 .cra_blocksize = 1, 619 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 620 .cra_module = THIS_MODULE, 621 }, 622 .min_keysize = AES_MIN_KEY_SIZE, 623 .max_keysize = AES_MAX_KEY_SIZE, 624 .ivsize = AES_BLOCK_SIZE, 625 .chunksize = AES_BLOCK_SIZE, 626 .setkey = aesni_skcipher_setkey, 627 .encrypt = ctr_crypt_aesni, 628 .decrypt = ctr_crypt_aesni, 629 #endif 630 }, { 631 .base = { 632 .cra_name = "xts(aes)", 633 .cra_driver_name = "xts-aes-aesni", 634 .cra_priority = 401, 635 .cra_blocksize = AES_BLOCK_SIZE, 636 .cra_ctxsize = XTS_AES_CTX_SIZE, 637 .cra_module = THIS_MODULE, 638 }, 639 .min_keysize = 2 * AES_MIN_KEY_SIZE, 640 .max_keysize = 2 * AES_MAX_KEY_SIZE, 641 .ivsize = AES_BLOCK_SIZE, 642 .walksize = 2 * AES_BLOCK_SIZE, 643 .setkey = xts_setkey_aesni, 644 .encrypt = xts_encrypt_aesni, 645 .decrypt = xts_decrypt_aesni, 646 } 647 }; 648 649 #ifdef CONFIG_X86_64 650 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, 651 u8 iv[AES_BLOCK_SIZE]); 652 653 /* __always_inline to avoid indirect call */ 654 static __always_inline int 655 ctr_crypt(struct skcipher_request *req, 656 void (*ctr64_func)(const struct crypto_aes_ctx *key, 657 const u8 *src, u8 *dst, int len, 658 const u64 le_ctr[2])) 659 { 660 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 661 const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm)); 662 unsigned int nbytes, p1_nbytes, nblocks; 663 struct skcipher_walk walk; 664 u64 le_ctr[2]; 665 u64 ctr64; 666 int err; 667 668 ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]); 669 le_ctr[1] = get_unaligned_be64(&req->iv[0]); 670 671 err = skcipher_walk_virt(&walk, req, false); 672 673 while ((nbytes = walk.nbytes) != 0) { 674 if (nbytes < walk.total) { 675 /* Not the end yet, so keep the length block-aligned. */ 676 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 677 nblocks = nbytes / AES_BLOCK_SIZE; 678 } else { 679 /* It's the end, so include any final partial block. */ 680 nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE); 681 } 682 ctr64 += nblocks; 683 684 kernel_fpu_begin(); 685 if (likely(ctr64 >= nblocks)) { 686 /* The low 64 bits of the counter won't overflow. */ 687 (*ctr64_func)(key, walk.src.virt.addr, 688 walk.dst.virt.addr, nbytes, le_ctr); 689 } else { 690 /* 691 * The low 64 bits of the counter will overflow. The 692 * assembly doesn't handle this case, so split the 693 * operation into two at the point where the overflow 694 * will occur. After the first part, add the carry bit. 695 */ 696 p1_nbytes = min_t(unsigned int, nbytes, 697 (nblocks - ctr64) * AES_BLOCK_SIZE); 698 (*ctr64_func)(key, walk.src.virt.addr, 699 walk.dst.virt.addr, p1_nbytes, le_ctr); 700 le_ctr[0] = 0; 701 le_ctr[1]++; 702 (*ctr64_func)(key, walk.src.virt.addr + p1_nbytes, 703 walk.dst.virt.addr + p1_nbytes, 704 nbytes - p1_nbytes, le_ctr); 705 } 706 kernel_fpu_end(); 707 le_ctr[0] = ctr64; 708 709 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 710 } 711 712 put_unaligned_be64(ctr64, &req->iv[8]); 713 put_unaligned_be64(le_ctr[1], &req->iv[0]); 714 715 return err; 716 } 717 718 /* __always_inline to avoid indirect call */ 719 static __always_inline int 720 xctr_crypt(struct skcipher_request *req, 721 void (*xctr_func)(const struct crypto_aes_ctx *key, 722 const u8 *src, u8 *dst, int len, 723 const u8 iv[AES_BLOCK_SIZE], u64 ctr)) 724 { 725 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 726 const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm)); 727 struct skcipher_walk walk; 728 unsigned int nbytes; 729 u64 ctr = 1; 730 int err; 731 732 err = skcipher_walk_virt(&walk, req, false); 733 while ((nbytes = walk.nbytes) != 0) { 734 if (nbytes < walk.total) 735 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 736 737 kernel_fpu_begin(); 738 (*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr, 739 nbytes, req->iv, ctr); 740 kernel_fpu_end(); 741 742 ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE); 743 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 744 } 745 return err; 746 } 747 748 #define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \ 749 \ 750 asmlinkage void \ 751 aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ 752 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ 753 asmlinkage void \ 754 aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ 755 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ 756 \ 757 static int xts_encrypt_##suffix(struct skcipher_request *req) \ 758 { \ 759 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix); \ 760 } \ 761 \ 762 static int xts_decrypt_##suffix(struct skcipher_request *req) \ 763 { \ 764 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \ 765 } \ 766 \ 767 asmlinkage void \ 768 aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key, \ 769 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\ 770 \ 771 static int ctr_crypt_##suffix(struct skcipher_request *req) \ 772 { \ 773 return ctr_crypt(req, aes_ctr64_crypt_##suffix); \ 774 } \ 775 \ 776 asmlinkage void \ 777 aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key, \ 778 const u8 *src, u8 *dst, int len, \ 779 const u8 iv[AES_BLOCK_SIZE], u64 ctr); \ 780 \ 781 static int xctr_crypt_##suffix(struct skcipher_request *req) \ 782 { \ 783 return xctr_crypt(req, aes_xctr_crypt_##suffix); \ 784 } \ 785 \ 786 static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ 787 .base.cra_name = "xts(aes)", \ 788 .base.cra_driver_name = "xts-aes-" driver_name_suffix, \ 789 .base.cra_priority = priority, \ 790 .base.cra_blocksize = AES_BLOCK_SIZE, \ 791 .base.cra_ctxsize = XTS_AES_CTX_SIZE, \ 792 .base.cra_module = THIS_MODULE, \ 793 .min_keysize = 2 * AES_MIN_KEY_SIZE, \ 794 .max_keysize = 2 * AES_MAX_KEY_SIZE, \ 795 .ivsize = AES_BLOCK_SIZE, \ 796 .walksize = 2 * AES_BLOCK_SIZE, \ 797 .setkey = xts_setkey_aesni, \ 798 .encrypt = xts_encrypt_##suffix, \ 799 .decrypt = xts_decrypt_##suffix, \ 800 }, { \ 801 .base.cra_name = "ctr(aes)", \ 802 .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \ 803 .base.cra_priority = priority, \ 804 .base.cra_blocksize = 1, \ 805 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ 806 .base.cra_module = THIS_MODULE, \ 807 .min_keysize = AES_MIN_KEY_SIZE, \ 808 .max_keysize = AES_MAX_KEY_SIZE, \ 809 .ivsize = AES_BLOCK_SIZE, \ 810 .chunksize = AES_BLOCK_SIZE, \ 811 .setkey = aesni_skcipher_setkey, \ 812 .encrypt = ctr_crypt_##suffix, \ 813 .decrypt = ctr_crypt_##suffix, \ 814 }, { \ 815 .base.cra_name = "xctr(aes)", \ 816 .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \ 817 .base.cra_priority = priority, \ 818 .base.cra_blocksize = 1, \ 819 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ 820 .base.cra_module = THIS_MODULE, \ 821 .min_keysize = AES_MIN_KEY_SIZE, \ 822 .max_keysize = AES_MAX_KEY_SIZE, \ 823 .ivsize = AES_BLOCK_SIZE, \ 824 .chunksize = AES_BLOCK_SIZE, \ 825 .setkey = aesni_skcipher_setkey, \ 826 .encrypt = xctr_crypt_##suffix, \ 827 .decrypt = xctr_crypt_##suffix, \ 828 }} 829 830 DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500); 831 DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600); 832 DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800); 833 834 /* The common part of the x86_64 AES-GCM key struct */ 835 struct aes_gcm_key { 836 /* Expanded AES key and the AES key length in bytes */ 837 struct crypto_aes_ctx aes_key; 838 839 /* RFC4106 nonce (used only by the rfc4106 algorithms) */ 840 u32 rfc4106_nonce; 841 }; 842 843 /* Key struct used by the AES-NI implementations of AES-GCM */ 844 struct aes_gcm_key_aesni { 845 /* 846 * Common part of the key. The assembly code requires 16-byte alignment 847 * for the round keys; we get this by them being located at the start of 848 * the struct and the whole struct being 16-byte aligned. 849 */ 850 struct aes_gcm_key base; 851 852 /* 853 * Powers of the hash key H^8 through H^1. These are 128-bit values. 854 * They all have an extra factor of x^-1 and are byte-reversed. 16-byte 855 * alignment is required by the assembly code. 856 */ 857 u64 h_powers[8][2] __aligned(16); 858 859 /* 860 * h_powers_xored[i] contains the two 64-bit halves of h_powers[i] XOR'd 861 * together. It's used for Karatsuba multiplication. 16-byte alignment 862 * is required by the assembly code. 863 */ 864 u64 h_powers_xored[8] __aligned(16); 865 866 /* 867 * H^1 times x^64 (and also the usual extra factor of x^-1). 16-byte 868 * alignment is required by the assembly code. 869 */ 870 u64 h_times_x64[2] __aligned(16); 871 }; 872 #define AES_GCM_KEY_AESNI(key) \ 873 container_of((key), struct aes_gcm_key_aesni, base) 874 #define AES_GCM_KEY_AESNI_SIZE \ 875 (sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1))) 876 877 /* Key struct used by the VAES + AVX10 implementations of AES-GCM */ 878 struct aes_gcm_key_avx10 { 879 /* 880 * Common part of the key. The assembly code prefers 16-byte alignment 881 * for the round keys; we get this by them being located at the start of 882 * the struct and the whole struct being 64-byte aligned. 883 */ 884 struct aes_gcm_key base; 885 886 /* 887 * Powers of the hash key H^16 through H^1. These are 128-bit values. 888 * They all have an extra factor of x^-1 and are byte-reversed. This 889 * array is aligned to a 64-byte boundary to make it naturally aligned 890 * for 512-bit loads, which can improve performance. (The assembly code 891 * doesn't *need* the alignment; this is just an optimization.) 892 */ 893 u64 h_powers[16][2] __aligned(64); 894 895 /* Three padding blocks required by the assembly code */ 896 u64 padding[3][2]; 897 }; 898 #define AES_GCM_KEY_AVX10(key) \ 899 container_of((key), struct aes_gcm_key_avx10, base) 900 #define AES_GCM_KEY_AVX10_SIZE \ 901 (sizeof(struct aes_gcm_key_avx10) + (63 & ~(CRYPTO_MINALIGN - 1))) 902 903 /* 904 * These flags are passed to the AES-GCM helper functions to specify the 905 * specific version of AES-GCM (RFC4106 or not), whether it's encryption or 906 * decryption, and which assembly functions should be called. Assembly 907 * functions are selected using flags instead of function pointers to avoid 908 * indirect calls (which are very expensive on x86) regardless of inlining. 909 */ 910 #define FLAG_RFC4106 BIT(0) 911 #define FLAG_ENC BIT(1) 912 #define FLAG_AVX BIT(2) 913 #define FLAG_AVX10_256 BIT(3) 914 #define FLAG_AVX10_512 BIT(4) 915 916 static inline struct aes_gcm_key * 917 aes_gcm_key_get(struct crypto_aead *tfm, int flags) 918 { 919 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 920 return PTR_ALIGN(crypto_aead_ctx(tfm), 64); 921 else 922 return PTR_ALIGN(crypto_aead_ctx(tfm), 16); 923 } 924 925 asmlinkage void 926 aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key); 927 asmlinkage void 928 aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key); 929 asmlinkage void 930 aes_gcm_precompute_vaes_avx10_256(struct aes_gcm_key_avx10 *key); 931 asmlinkage void 932 aes_gcm_precompute_vaes_avx10_512(struct aes_gcm_key_avx10 *key); 933 934 static void aes_gcm_precompute(struct aes_gcm_key *key, int flags) 935 { 936 /* 937 * To make things a bit easier on the assembly side, the AVX10 938 * implementations use the same key format. Therefore, a single 939 * function using 256-bit vectors would suffice here. However, it's 940 * straightforward to provide a 512-bit one because of how the assembly 941 * code is structured, and it works nicely because the total size of the 942 * key powers is a multiple of 512 bits. So we take advantage of that. 943 * 944 * A similar situation applies to the AES-NI implementations. 945 */ 946 if (flags & FLAG_AVX10_512) 947 aes_gcm_precompute_vaes_avx10_512(AES_GCM_KEY_AVX10(key)); 948 else if (flags & FLAG_AVX10_256) 949 aes_gcm_precompute_vaes_avx10_256(AES_GCM_KEY_AVX10(key)); 950 else if (flags & FLAG_AVX) 951 aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key)); 952 else 953 aes_gcm_precompute_aesni(AES_GCM_KEY_AESNI(key)); 954 } 955 956 asmlinkage void 957 aes_gcm_aad_update_aesni(const struct aes_gcm_key_aesni *key, 958 u8 ghash_acc[16], const u8 *aad, int aadlen); 959 asmlinkage void 960 aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key, 961 u8 ghash_acc[16], const u8 *aad, int aadlen); 962 asmlinkage void 963 aes_gcm_aad_update_vaes_avx10(const struct aes_gcm_key_avx10 *key, 964 u8 ghash_acc[16], const u8 *aad, int aadlen); 965 966 static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16], 967 const u8 *aad, int aadlen, int flags) 968 { 969 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 970 aes_gcm_aad_update_vaes_avx10(AES_GCM_KEY_AVX10(key), ghash_acc, 971 aad, aadlen); 972 else if (flags & FLAG_AVX) 973 aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc, 974 aad, aadlen); 975 else 976 aes_gcm_aad_update_aesni(AES_GCM_KEY_AESNI(key), ghash_acc, 977 aad, aadlen); 978 } 979 980 asmlinkage void 981 aes_gcm_enc_update_aesni(const struct aes_gcm_key_aesni *key, 982 const u32 le_ctr[4], u8 ghash_acc[16], 983 const u8 *src, u8 *dst, int datalen); 984 asmlinkage void 985 aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key, 986 const u32 le_ctr[4], u8 ghash_acc[16], 987 const u8 *src, u8 *dst, int datalen); 988 asmlinkage void 989 aes_gcm_enc_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key, 990 const u32 le_ctr[4], u8 ghash_acc[16], 991 const u8 *src, u8 *dst, int datalen); 992 asmlinkage void 993 aes_gcm_enc_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key, 994 const u32 le_ctr[4], u8 ghash_acc[16], 995 const u8 *src, u8 *dst, int datalen); 996 997 asmlinkage void 998 aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key, 999 const u32 le_ctr[4], u8 ghash_acc[16], 1000 const u8 *src, u8 *dst, int datalen); 1001 asmlinkage void 1002 aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key, 1003 const u32 le_ctr[4], u8 ghash_acc[16], 1004 const u8 *src, u8 *dst, int datalen); 1005 asmlinkage void 1006 aes_gcm_dec_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key, 1007 const u32 le_ctr[4], u8 ghash_acc[16], 1008 const u8 *src, u8 *dst, int datalen); 1009 asmlinkage void 1010 aes_gcm_dec_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key, 1011 const u32 le_ctr[4], u8 ghash_acc[16], 1012 const u8 *src, u8 *dst, int datalen); 1013 1014 /* __always_inline to optimize out the branches based on @flags */ 1015 static __always_inline void 1016 aes_gcm_update(const struct aes_gcm_key *key, 1017 const u32 le_ctr[4], u8 ghash_acc[16], 1018 const u8 *src, u8 *dst, int datalen, int flags) 1019 { 1020 if (flags & FLAG_ENC) { 1021 if (flags & FLAG_AVX10_512) 1022 aes_gcm_enc_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key), 1023 le_ctr, ghash_acc, 1024 src, dst, datalen); 1025 else if (flags & FLAG_AVX10_256) 1026 aes_gcm_enc_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key), 1027 le_ctr, ghash_acc, 1028 src, dst, datalen); 1029 else if (flags & FLAG_AVX) 1030 aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key), 1031 le_ctr, ghash_acc, 1032 src, dst, datalen); 1033 else 1034 aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr, 1035 ghash_acc, src, dst, datalen); 1036 } else { 1037 if (flags & FLAG_AVX10_512) 1038 aes_gcm_dec_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key), 1039 le_ctr, ghash_acc, 1040 src, dst, datalen); 1041 else if (flags & FLAG_AVX10_256) 1042 aes_gcm_dec_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key), 1043 le_ctr, ghash_acc, 1044 src, dst, datalen); 1045 else if (flags & FLAG_AVX) 1046 aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key), 1047 le_ctr, ghash_acc, 1048 src, dst, datalen); 1049 else 1050 aes_gcm_dec_update_aesni(AES_GCM_KEY_AESNI(key), 1051 le_ctr, ghash_acc, 1052 src, dst, datalen); 1053 } 1054 } 1055 1056 asmlinkage void 1057 aes_gcm_enc_final_aesni(const struct aes_gcm_key_aesni *key, 1058 const u32 le_ctr[4], u8 ghash_acc[16], 1059 u64 total_aadlen, u64 total_datalen); 1060 asmlinkage void 1061 aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key, 1062 const u32 le_ctr[4], u8 ghash_acc[16], 1063 u64 total_aadlen, u64 total_datalen); 1064 asmlinkage void 1065 aes_gcm_enc_final_vaes_avx10(const struct aes_gcm_key_avx10 *key, 1066 const u32 le_ctr[4], u8 ghash_acc[16], 1067 u64 total_aadlen, u64 total_datalen); 1068 1069 /* __always_inline to optimize out the branches based on @flags */ 1070 static __always_inline void 1071 aes_gcm_enc_final(const struct aes_gcm_key *key, 1072 const u32 le_ctr[4], u8 ghash_acc[16], 1073 u64 total_aadlen, u64 total_datalen, int flags) 1074 { 1075 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 1076 aes_gcm_enc_final_vaes_avx10(AES_GCM_KEY_AVX10(key), 1077 le_ctr, ghash_acc, 1078 total_aadlen, total_datalen); 1079 else if (flags & FLAG_AVX) 1080 aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key), 1081 le_ctr, ghash_acc, 1082 total_aadlen, total_datalen); 1083 else 1084 aes_gcm_enc_final_aesni(AES_GCM_KEY_AESNI(key), 1085 le_ctr, ghash_acc, 1086 total_aadlen, total_datalen); 1087 } 1088 1089 asmlinkage bool __must_check 1090 aes_gcm_dec_final_aesni(const struct aes_gcm_key_aesni *key, 1091 const u32 le_ctr[4], const u8 ghash_acc[16], 1092 u64 total_aadlen, u64 total_datalen, 1093 const u8 tag[16], int taglen); 1094 asmlinkage bool __must_check 1095 aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key, 1096 const u32 le_ctr[4], const u8 ghash_acc[16], 1097 u64 total_aadlen, u64 total_datalen, 1098 const u8 tag[16], int taglen); 1099 asmlinkage bool __must_check 1100 aes_gcm_dec_final_vaes_avx10(const struct aes_gcm_key_avx10 *key, 1101 const u32 le_ctr[4], const u8 ghash_acc[16], 1102 u64 total_aadlen, u64 total_datalen, 1103 const u8 tag[16], int taglen); 1104 1105 /* __always_inline to optimize out the branches based on @flags */ 1106 static __always_inline bool __must_check 1107 aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4], 1108 u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen, 1109 u8 tag[16], int taglen, int flags) 1110 { 1111 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 1112 return aes_gcm_dec_final_vaes_avx10(AES_GCM_KEY_AVX10(key), 1113 le_ctr, ghash_acc, 1114 total_aadlen, total_datalen, 1115 tag, taglen); 1116 else if (flags & FLAG_AVX) 1117 return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key), 1118 le_ctr, ghash_acc, 1119 total_aadlen, total_datalen, 1120 tag, taglen); 1121 else 1122 return aes_gcm_dec_final_aesni(AES_GCM_KEY_AESNI(key), 1123 le_ctr, ghash_acc, 1124 total_aadlen, total_datalen, 1125 tag, taglen); 1126 } 1127 1128 /* 1129 * This is the Integrity Check Value (aka the authentication tag) length and can 1130 * be 8, 12 or 16 bytes long. 1131 */ 1132 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 1133 unsigned int authsize) 1134 { 1135 switch (authsize) { 1136 case 8: 1137 case 12: 1138 case 16: 1139 break; 1140 default: 1141 return -EINVAL; 1142 } 1143 1144 return 0; 1145 } 1146 1147 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 1148 unsigned int authsize) 1149 { 1150 switch (authsize) { 1151 case 4: 1152 case 8: 1153 case 12: 1154 case 13: 1155 case 14: 1156 case 15: 1157 case 16: 1158 break; 1159 default: 1160 return -EINVAL; 1161 } 1162 1163 return 0; 1164 } 1165 1166 /* 1167 * This is the setkey function for the x86_64 implementations of AES-GCM. It 1168 * saves the RFC4106 nonce if applicable, expands the AES key, and precomputes 1169 * powers of the hash key. 1170 * 1171 * To comply with the crypto_aead API, this has to be usable in no-SIMD context. 1172 * For that reason, this function includes a portable C implementation of the 1173 * needed logic. However, the portable C implementation is very slow, taking 1174 * about the same time as encrypting 37 KB of data. To be ready for users that 1175 * may set a key even somewhat frequently, we therefore also include a SIMD 1176 * assembly implementation, expanding the AES key using AES-NI and precomputing 1177 * the hash key powers using PCLMULQDQ or VPCLMULQDQ. 1178 */ 1179 static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key, 1180 unsigned int keylen, int flags) 1181 { 1182 struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags); 1183 int err; 1184 1185 if (flags & FLAG_RFC4106) { 1186 if (keylen < 4) 1187 return -EINVAL; 1188 keylen -= 4; 1189 key->rfc4106_nonce = get_unaligned_be32(raw_key + keylen); 1190 } 1191 1192 /* The assembly code assumes the following offsets. */ 1193 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0); 1194 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480); 1195 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496); 1196 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624); 1197 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688); 1198 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_enc) != 0); 1199 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_length) != 480); 1200 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, h_powers) != 512); 1201 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, padding) != 768); 1202 1203 if (likely(crypto_simd_usable())) { 1204 err = aes_check_keylen(keylen); 1205 if (err) 1206 return err; 1207 kernel_fpu_begin(); 1208 aesni_set_key(&key->aes_key, raw_key, keylen); 1209 aes_gcm_precompute(key, flags); 1210 kernel_fpu_end(); 1211 } else { 1212 static const u8 x_to_the_minus1[16] __aligned(__alignof__(be128)) = { 1213 [0] = 0xc2, [15] = 1 1214 }; 1215 static const u8 x_to_the_63[16] __aligned(__alignof__(be128)) = { 1216 [7] = 1, 1217 }; 1218 be128 h1 = {}; 1219 be128 h; 1220 int i; 1221 1222 err = aes_expandkey(&key->aes_key, raw_key, keylen); 1223 if (err) 1224 return err; 1225 1226 /* Encrypt the all-zeroes block to get the hash key H^1 */ 1227 aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1); 1228 1229 /* Compute H^1 * x^-1 */ 1230 h = h1; 1231 gf128mul_lle(&h, (const be128 *)x_to_the_minus1); 1232 1233 /* Compute the needed key powers */ 1234 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) { 1235 struct aes_gcm_key_avx10 *k = AES_GCM_KEY_AVX10(key); 1236 1237 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 1238 k->h_powers[i][0] = be64_to_cpu(h.b); 1239 k->h_powers[i][1] = be64_to_cpu(h.a); 1240 gf128mul_lle(&h, &h1); 1241 } 1242 memset(k->padding, 0, sizeof(k->padding)); 1243 } else { 1244 struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key); 1245 1246 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 1247 k->h_powers[i][0] = be64_to_cpu(h.b); 1248 k->h_powers[i][1] = be64_to_cpu(h.a); 1249 k->h_powers_xored[i] = k->h_powers[i][0] ^ 1250 k->h_powers[i][1]; 1251 gf128mul_lle(&h, &h1); 1252 } 1253 gf128mul_lle(&h1, (const be128 *)x_to_the_63); 1254 k->h_times_x64[0] = be64_to_cpu(h1.b); 1255 k->h_times_x64[1] = be64_to_cpu(h1.a); 1256 } 1257 } 1258 return 0; 1259 } 1260 1261 /* 1262 * Initialize @ghash_acc, then pass all @assoclen bytes of associated data 1263 * (a.k.a. additional authenticated data) from @sg_src through the GHASH update 1264 * assembly function. kernel_fpu_begin() must have already been called. 1265 */ 1266 static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], 1267 struct scatterlist *sg_src, unsigned int assoclen, 1268 int flags) 1269 { 1270 struct scatter_walk walk; 1271 /* 1272 * The assembly function requires that the length of any non-last 1273 * segment of associated data be a multiple of 16 bytes, so this 1274 * function does the buffering needed to achieve that. 1275 */ 1276 unsigned int pos = 0; 1277 u8 buf[16]; 1278 1279 memset(ghash_acc, 0, 16); 1280 scatterwalk_start(&walk, sg_src); 1281 1282 while (assoclen) { 1283 unsigned int orig_len_this_step = scatterwalk_next( 1284 &walk, assoclen); 1285 unsigned int len_this_step = orig_len_this_step; 1286 unsigned int len; 1287 const u8 *src = walk.addr; 1288 1289 if (unlikely(pos)) { 1290 len = min(len_this_step, 16 - pos); 1291 memcpy(&buf[pos], src, len); 1292 pos += len; 1293 src += len; 1294 len_this_step -= len; 1295 if (pos < 16) 1296 goto next; 1297 aes_gcm_aad_update(key, ghash_acc, buf, 16, flags); 1298 pos = 0; 1299 } 1300 len = len_this_step; 1301 if (unlikely(assoclen)) /* Not the last segment yet? */ 1302 len = round_down(len, 16); 1303 aes_gcm_aad_update(key, ghash_acc, src, len, flags); 1304 src += len; 1305 len_this_step -= len; 1306 if (unlikely(len_this_step)) { 1307 memcpy(buf, src, len_this_step); 1308 pos = len_this_step; 1309 } 1310 next: 1311 scatterwalk_done_src(&walk, orig_len_this_step); 1312 if (need_resched()) { 1313 kernel_fpu_end(); 1314 kernel_fpu_begin(); 1315 } 1316 assoclen -= orig_len_this_step; 1317 } 1318 if (unlikely(pos)) 1319 aes_gcm_aad_update(key, ghash_acc, buf, pos, flags); 1320 } 1321 1322 1323 /* __always_inline to optimize out the branches based on @flags */ 1324 static __always_inline int 1325 gcm_crypt(struct aead_request *req, int flags) 1326 { 1327 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1328 const struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags); 1329 unsigned int assoclen = req->assoclen; 1330 struct skcipher_walk walk; 1331 unsigned int nbytes; 1332 u8 ghash_acc[16]; /* GHASH accumulator */ 1333 u32 le_ctr[4]; /* Counter in little-endian format */ 1334 int taglen; 1335 int err; 1336 1337 /* Initialize the counter and determine the associated data length. */ 1338 le_ctr[0] = 2; 1339 if (flags & FLAG_RFC4106) { 1340 if (unlikely(assoclen != 16 && assoclen != 20)) 1341 return -EINVAL; 1342 assoclen -= 8; 1343 le_ctr[1] = get_unaligned_be32(req->iv + 4); 1344 le_ctr[2] = get_unaligned_be32(req->iv + 0); 1345 le_ctr[3] = key->rfc4106_nonce; /* already byte-swapped */ 1346 } else { 1347 le_ctr[1] = get_unaligned_be32(req->iv + 8); 1348 le_ctr[2] = get_unaligned_be32(req->iv + 4); 1349 le_ctr[3] = get_unaligned_be32(req->iv + 0); 1350 } 1351 1352 /* Begin walking through the plaintext or ciphertext. */ 1353 if (flags & FLAG_ENC) 1354 err = skcipher_walk_aead_encrypt(&walk, req, false); 1355 else 1356 err = skcipher_walk_aead_decrypt(&walk, req, false); 1357 if (err) 1358 return err; 1359 1360 /* 1361 * Since the AES-GCM assembly code requires that at least three assembly 1362 * functions be called to process any message (this is needed to support 1363 * incremental updates cleanly), to reduce overhead we try to do all 1364 * three calls in the same kernel FPU section if possible. We close the 1365 * section and start a new one if there are multiple data segments or if 1366 * rescheduling is needed while processing the associated data. 1367 */ 1368 kernel_fpu_begin(); 1369 1370 /* Pass the associated data through GHASH. */ 1371 gcm_process_assoc(key, ghash_acc, req->src, assoclen, flags); 1372 1373 /* En/decrypt the data and pass the ciphertext through GHASH. */ 1374 while (unlikely((nbytes = walk.nbytes) < walk.total)) { 1375 /* 1376 * Non-last segment. In this case, the assembly function 1377 * requires that the length be a multiple of 16 (AES_BLOCK_SIZE) 1378 * bytes. The needed buffering of up to 16 bytes is handled by 1379 * the skcipher_walk. Here we just need to round down to a 1380 * multiple of 16. 1381 */ 1382 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 1383 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr, 1384 walk.dst.virt.addr, nbytes, flags); 1385 le_ctr[0] += nbytes / AES_BLOCK_SIZE; 1386 kernel_fpu_end(); 1387 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 1388 if (err) 1389 return err; 1390 kernel_fpu_begin(); 1391 } 1392 /* Last segment: process all remaining data. */ 1393 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr, 1394 walk.dst.virt.addr, nbytes, flags); 1395 /* 1396 * The low word of the counter isn't used by the finalize, so there's no 1397 * need to increment it here. 1398 */ 1399 1400 /* Finalize */ 1401 taglen = crypto_aead_authsize(tfm); 1402 if (flags & FLAG_ENC) { 1403 /* Finish computing the auth tag. */ 1404 aes_gcm_enc_final(key, le_ctr, ghash_acc, assoclen, 1405 req->cryptlen, flags); 1406 1407 /* Store the computed auth tag in the dst scatterlist. */ 1408 scatterwalk_map_and_copy(ghash_acc, req->dst, req->assoclen + 1409 req->cryptlen, taglen, 1); 1410 } else { 1411 unsigned int datalen = req->cryptlen - taglen; 1412 u8 tag[16]; 1413 1414 /* Get the transmitted auth tag from the src scatterlist. */ 1415 scatterwalk_map_and_copy(tag, req->src, req->assoclen + datalen, 1416 taglen, 0); 1417 /* 1418 * Finish computing the auth tag and compare it to the 1419 * transmitted one. The assembly function does the actual tag 1420 * comparison. Here, just check the boolean result. 1421 */ 1422 if (!aes_gcm_dec_final(key, le_ctr, ghash_acc, assoclen, 1423 datalen, tag, taglen, flags)) 1424 err = -EBADMSG; 1425 } 1426 kernel_fpu_end(); 1427 if (nbytes) 1428 skcipher_walk_done(&walk, 0); 1429 return err; 1430 } 1431 1432 #define DEFINE_GCM_ALGS(suffix, flags, generic_driver_name, rfc_driver_name, \ 1433 ctxsize, priority) \ 1434 \ 1435 static int gcm_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \ 1436 unsigned int keylen) \ 1437 { \ 1438 return gcm_setkey(tfm, raw_key, keylen, (flags)); \ 1439 } \ 1440 \ 1441 static int gcm_encrypt_##suffix(struct aead_request *req) \ 1442 { \ 1443 return gcm_crypt(req, (flags) | FLAG_ENC); \ 1444 } \ 1445 \ 1446 static int gcm_decrypt_##suffix(struct aead_request *req) \ 1447 { \ 1448 return gcm_crypt(req, (flags)); \ 1449 } \ 1450 \ 1451 static int rfc4106_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \ 1452 unsigned int keylen) \ 1453 { \ 1454 return gcm_setkey(tfm, raw_key, keylen, (flags) | FLAG_RFC4106); \ 1455 } \ 1456 \ 1457 static int rfc4106_encrypt_##suffix(struct aead_request *req) \ 1458 { \ 1459 return gcm_crypt(req, (flags) | FLAG_RFC4106 | FLAG_ENC); \ 1460 } \ 1461 \ 1462 static int rfc4106_decrypt_##suffix(struct aead_request *req) \ 1463 { \ 1464 return gcm_crypt(req, (flags) | FLAG_RFC4106); \ 1465 } \ 1466 \ 1467 static struct aead_alg aes_gcm_algs_##suffix[] = { { \ 1468 .setkey = gcm_setkey_##suffix, \ 1469 .setauthsize = generic_gcmaes_set_authsize, \ 1470 .encrypt = gcm_encrypt_##suffix, \ 1471 .decrypt = gcm_decrypt_##suffix, \ 1472 .ivsize = GCM_AES_IV_SIZE, \ 1473 .chunksize = AES_BLOCK_SIZE, \ 1474 .maxauthsize = 16, \ 1475 .base = { \ 1476 .cra_name = "gcm(aes)", \ 1477 .cra_driver_name = generic_driver_name, \ 1478 .cra_priority = (priority), \ 1479 .cra_blocksize = 1, \ 1480 .cra_ctxsize = (ctxsize), \ 1481 .cra_module = THIS_MODULE, \ 1482 }, \ 1483 }, { \ 1484 .setkey = rfc4106_setkey_##suffix, \ 1485 .setauthsize = common_rfc4106_set_authsize, \ 1486 .encrypt = rfc4106_encrypt_##suffix, \ 1487 .decrypt = rfc4106_decrypt_##suffix, \ 1488 .ivsize = GCM_RFC4106_IV_SIZE, \ 1489 .chunksize = AES_BLOCK_SIZE, \ 1490 .maxauthsize = 16, \ 1491 .base = { \ 1492 .cra_name = "rfc4106(gcm(aes))", \ 1493 .cra_driver_name = rfc_driver_name, \ 1494 .cra_priority = (priority), \ 1495 .cra_blocksize = 1, \ 1496 .cra_ctxsize = (ctxsize), \ 1497 .cra_module = THIS_MODULE, \ 1498 }, \ 1499 } } 1500 1501 /* aes_gcm_algs_aesni */ 1502 DEFINE_GCM_ALGS(aesni, /* no flags */ 0, 1503 "generic-gcm-aesni", "rfc4106-gcm-aesni", 1504 AES_GCM_KEY_AESNI_SIZE, 400); 1505 1506 /* aes_gcm_algs_aesni_avx */ 1507 DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX, 1508 "generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx", 1509 AES_GCM_KEY_AESNI_SIZE, 500); 1510 1511 /* aes_gcm_algs_vaes_avx10_256 */ 1512 DEFINE_GCM_ALGS(vaes_avx10_256, FLAG_AVX10_256, 1513 "generic-gcm-vaes-avx10_256", "rfc4106-gcm-vaes-avx10_256", 1514 AES_GCM_KEY_AVX10_SIZE, 700); 1515 1516 /* aes_gcm_algs_vaes_avx10_512 */ 1517 DEFINE_GCM_ALGS(vaes_avx10_512, FLAG_AVX10_512, 1518 "generic-gcm-vaes-avx10_512", "rfc4106-gcm-vaes-avx10_512", 1519 AES_GCM_KEY_AVX10_SIZE, 800); 1520 1521 static int __init register_avx_algs(void) 1522 { 1523 int err; 1524 1525 if (!boot_cpu_has(X86_FEATURE_AVX)) 1526 return 0; 1527 err = crypto_register_skciphers(skcipher_algs_aesni_avx, 1528 ARRAY_SIZE(skcipher_algs_aesni_avx)); 1529 if (err) 1530 return err; 1531 err = crypto_register_aeads(aes_gcm_algs_aesni_avx, 1532 ARRAY_SIZE(aes_gcm_algs_aesni_avx)); 1533 if (err) 1534 return err; 1535 /* 1536 * Note: not all the algorithms registered below actually require 1537 * VPCLMULQDQ. But in practice every CPU with VAES also has VPCLMULQDQ. 1538 * Similarly, the assembler support was added at about the same time. 1539 * For simplicity, just always check for VAES and VPCLMULQDQ together. 1540 */ 1541 if (!boot_cpu_has(X86_FEATURE_AVX2) || 1542 !boot_cpu_has(X86_FEATURE_VAES) || 1543 !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) || 1544 !boot_cpu_has(X86_FEATURE_PCLMULQDQ) || 1545 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) 1546 return 0; 1547 err = crypto_register_skciphers(skcipher_algs_vaes_avx2, 1548 ARRAY_SIZE(skcipher_algs_vaes_avx2)); 1549 if (err) 1550 return err; 1551 1552 if (!boot_cpu_has(X86_FEATURE_AVX512BW) || 1553 !boot_cpu_has(X86_FEATURE_AVX512VL) || 1554 !boot_cpu_has(X86_FEATURE_BMI2) || 1555 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | 1556 XFEATURE_MASK_AVX512, NULL)) 1557 return 0; 1558 1559 err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_256, 1560 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256)); 1561 if (err) 1562 return err; 1563 1564 if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) { 1565 int i; 1566 1567 for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++) 1568 skcipher_algs_vaes_avx512[i].base.cra_priority = 1; 1569 for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++) 1570 aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1; 1571 } 1572 1573 err = crypto_register_skciphers(skcipher_algs_vaes_avx512, 1574 ARRAY_SIZE(skcipher_algs_vaes_avx512)); 1575 if (err) 1576 return err; 1577 err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_512, 1578 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512)); 1579 if (err) 1580 return err; 1581 1582 return 0; 1583 } 1584 1585 #define unregister_skciphers(A) \ 1586 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ 1587 crypto_unregister_skciphers((A), ARRAY_SIZE(A)) 1588 #define unregister_aeads(A) \ 1589 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ 1590 crypto_unregister_aeads((A), ARRAY_SIZE(A)) 1591 1592 static void unregister_avx_algs(void) 1593 { 1594 unregister_skciphers(skcipher_algs_aesni_avx); 1595 unregister_aeads(aes_gcm_algs_aesni_avx); 1596 unregister_skciphers(skcipher_algs_vaes_avx2); 1597 unregister_skciphers(skcipher_algs_vaes_avx512); 1598 unregister_aeads(aes_gcm_algs_vaes_avx10_256); 1599 unregister_aeads(aes_gcm_algs_vaes_avx10_512); 1600 } 1601 #else /* CONFIG_X86_64 */ 1602 static struct aead_alg aes_gcm_algs_aesni[0]; 1603 1604 static int __init register_avx_algs(void) 1605 { 1606 return 0; 1607 } 1608 1609 static void unregister_avx_algs(void) 1610 { 1611 } 1612 #endif /* !CONFIG_X86_64 */ 1613 1614 static const struct x86_cpu_id aesni_cpu_id[] = { 1615 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), 1616 {} 1617 }; 1618 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1619 1620 static int __init aesni_init(void) 1621 { 1622 int err; 1623 1624 if (!x86_match_cpu(aesni_cpu_id)) 1625 return -ENODEV; 1626 1627 err = crypto_register_alg(&aesni_cipher_alg); 1628 if (err) 1629 return err; 1630 1631 err = crypto_register_skciphers(aesni_skciphers, 1632 ARRAY_SIZE(aesni_skciphers)); 1633 if (err) 1634 goto unregister_cipher; 1635 1636 err = crypto_register_aeads(aes_gcm_algs_aesni, 1637 ARRAY_SIZE(aes_gcm_algs_aesni)); 1638 if (err) 1639 goto unregister_skciphers; 1640 1641 err = register_avx_algs(); 1642 if (err) 1643 goto unregister_avx; 1644 1645 return 0; 1646 1647 unregister_avx: 1648 unregister_avx_algs(); 1649 crypto_unregister_aeads(aes_gcm_algs_aesni, 1650 ARRAY_SIZE(aes_gcm_algs_aesni)); 1651 unregister_skciphers: 1652 crypto_unregister_skciphers(aesni_skciphers, 1653 ARRAY_SIZE(aesni_skciphers)); 1654 unregister_cipher: 1655 crypto_unregister_alg(&aesni_cipher_alg); 1656 return err; 1657 } 1658 1659 static void __exit aesni_exit(void) 1660 { 1661 crypto_unregister_aeads(aes_gcm_algs_aesni, 1662 ARRAY_SIZE(aes_gcm_algs_aesni)); 1663 crypto_unregister_skciphers(aesni_skciphers, 1664 ARRAY_SIZE(aesni_skciphers)); 1665 crypto_unregister_alg(&aesni_cipher_alg); 1666 unregister_avx_algs(); 1667 } 1668 1669 module_init(aesni_init); 1670 module_exit(aesni_exit); 1671 1672 MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions"); 1673 MODULE_LICENSE("GPL"); 1674 MODULE_ALIAS_CRYPTO("aes"); 1675