1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Support for AES-NI and VAES instructions. This file contains glue code. 4 * The real AES implementations are in aesni-intel_asm.S and other .S files. 5 * 6 * Copyright (C) 2008, Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * 9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 10 * interface for 64-bit kernels. 11 * Authors: Adrian Hoban <adrian.hoban@intel.com> 12 * Gabriele Paoloni <gabriele.paoloni@intel.com> 13 * Tadeusz Struk (tadeusz.struk@intel.com) 14 * Aidan O'Mahony (aidan.o.mahony@intel.com) 15 * Copyright (c) 2010, Intel Corporation. 16 * 17 * Copyright 2024 Google LLC 18 */ 19 20 #include <linux/hardirq.h> 21 #include <linux/types.h> 22 #include <linux/module.h> 23 #include <linux/err.h> 24 #include <crypto/algapi.h> 25 #include <crypto/aes.h> 26 #include <crypto/b128ops.h> 27 #include <crypto/gcm.h> 28 #include <crypto/xts.h> 29 #include <asm/cpu_device_id.h> 30 #include <asm/simd.h> 31 #include <crypto/scatterwalk.h> 32 #include <crypto/internal/aead.h> 33 #include <crypto/internal/simd.h> 34 #include <crypto/internal/skcipher.h> 35 #include <linux/jump_label.h> 36 #include <linux/workqueue.h> 37 #include <linux/spinlock.h> 38 #include <linux/static_call.h> 39 40 41 #define AESNI_ALIGN 16 42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 43 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 47 48 struct aesni_xts_ctx { 49 struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR; 50 struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR; 51 }; 52 53 static inline void *aes_align_addr(void *addr) 54 { 55 if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN) 56 return addr; 57 return PTR_ALIGN(addr, AESNI_ALIGN); 58 } 59 60 asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 61 unsigned int key_len); 62 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in); 63 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in); 64 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 65 const u8 *in, unsigned int len); 66 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 67 const u8 *in, unsigned int len); 68 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 69 const u8 *in, unsigned int len, u8 *iv); 70 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 71 const u8 *in, unsigned int len, u8 *iv); 72 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 73 const u8 *in, unsigned int len, u8 *iv); 74 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 75 const u8 *in, unsigned int len, u8 *iv); 76 77 asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out, 78 const u8 *in, unsigned int len, u8 *iv); 79 80 asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out, 81 const u8 *in, unsigned int len, u8 *iv); 82 83 #ifdef CONFIG_X86_64 84 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 85 const u8 *in, unsigned int len, u8 *iv); 86 #endif 87 88 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 89 { 90 return aes_align_addr(raw_ctx); 91 } 92 93 static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm) 94 { 95 return aes_align_addr(crypto_skcipher_ctx(tfm)); 96 } 97 98 static int aes_set_key_common(struct crypto_aes_ctx *ctx, 99 const u8 *in_key, unsigned int key_len) 100 { 101 int err; 102 103 if (!crypto_simd_usable()) 104 return aes_expandkey(ctx, in_key, key_len); 105 106 err = aes_check_keylen(key_len); 107 if (err) 108 return err; 109 110 kernel_fpu_begin(); 111 aesni_set_key(ctx, in_key, key_len); 112 kernel_fpu_end(); 113 return 0; 114 } 115 116 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 117 unsigned int key_len) 118 { 119 return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key, 120 key_len); 121 } 122 123 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 124 { 125 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 126 127 if (!crypto_simd_usable()) { 128 aes_encrypt(ctx, dst, src); 129 } else { 130 kernel_fpu_begin(); 131 aesni_enc(ctx, dst, src); 132 kernel_fpu_end(); 133 } 134 } 135 136 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 137 { 138 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 139 140 if (!crypto_simd_usable()) { 141 aes_decrypt(ctx, dst, src); 142 } else { 143 kernel_fpu_begin(); 144 aesni_dec(ctx, dst, src); 145 kernel_fpu_end(); 146 } 147 } 148 149 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 150 unsigned int len) 151 { 152 return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len); 153 } 154 155 static int ecb_encrypt(struct skcipher_request *req) 156 { 157 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 158 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 159 struct skcipher_walk walk; 160 unsigned int nbytes; 161 int err; 162 163 err = skcipher_walk_virt(&walk, req, false); 164 165 while ((nbytes = walk.nbytes)) { 166 kernel_fpu_begin(); 167 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 168 nbytes & AES_BLOCK_MASK); 169 kernel_fpu_end(); 170 nbytes &= AES_BLOCK_SIZE - 1; 171 err = skcipher_walk_done(&walk, nbytes); 172 } 173 174 return err; 175 } 176 177 static int ecb_decrypt(struct skcipher_request *req) 178 { 179 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 180 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 181 struct skcipher_walk walk; 182 unsigned int nbytes; 183 int err; 184 185 err = skcipher_walk_virt(&walk, req, false); 186 187 while ((nbytes = walk.nbytes)) { 188 kernel_fpu_begin(); 189 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 190 nbytes & AES_BLOCK_MASK); 191 kernel_fpu_end(); 192 nbytes &= AES_BLOCK_SIZE - 1; 193 err = skcipher_walk_done(&walk, nbytes); 194 } 195 196 return err; 197 } 198 199 static int cbc_encrypt(struct skcipher_request *req) 200 { 201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 202 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 203 struct skcipher_walk walk; 204 unsigned int nbytes; 205 int err; 206 207 err = skcipher_walk_virt(&walk, req, false); 208 209 while ((nbytes = walk.nbytes)) { 210 kernel_fpu_begin(); 211 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 212 nbytes & AES_BLOCK_MASK, walk.iv); 213 kernel_fpu_end(); 214 nbytes &= AES_BLOCK_SIZE - 1; 215 err = skcipher_walk_done(&walk, nbytes); 216 } 217 218 return err; 219 } 220 221 static int cbc_decrypt(struct skcipher_request *req) 222 { 223 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 224 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 225 struct skcipher_walk walk; 226 unsigned int nbytes; 227 int err; 228 229 err = skcipher_walk_virt(&walk, req, false); 230 231 while ((nbytes = walk.nbytes)) { 232 kernel_fpu_begin(); 233 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 234 nbytes & AES_BLOCK_MASK, walk.iv); 235 kernel_fpu_end(); 236 nbytes &= AES_BLOCK_SIZE - 1; 237 err = skcipher_walk_done(&walk, nbytes); 238 } 239 240 return err; 241 } 242 243 static int cts_cbc_encrypt(struct skcipher_request *req) 244 { 245 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 246 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 247 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 248 struct scatterlist *src = req->src, *dst = req->dst; 249 struct scatterlist sg_src[2], sg_dst[2]; 250 struct skcipher_request subreq; 251 struct skcipher_walk walk; 252 int err; 253 254 skcipher_request_set_tfm(&subreq, tfm); 255 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 256 NULL, NULL); 257 258 if (req->cryptlen <= AES_BLOCK_SIZE) { 259 if (req->cryptlen < AES_BLOCK_SIZE) 260 return -EINVAL; 261 cbc_blocks = 1; 262 } 263 264 if (cbc_blocks > 0) { 265 skcipher_request_set_crypt(&subreq, req->src, req->dst, 266 cbc_blocks * AES_BLOCK_SIZE, 267 req->iv); 268 269 err = cbc_encrypt(&subreq); 270 if (err) 271 return err; 272 273 if (req->cryptlen == AES_BLOCK_SIZE) 274 return 0; 275 276 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 277 if (req->dst != req->src) 278 dst = scatterwalk_ffwd(sg_dst, req->dst, 279 subreq.cryptlen); 280 } 281 282 /* handle ciphertext stealing */ 283 skcipher_request_set_crypt(&subreq, src, dst, 284 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 285 req->iv); 286 287 err = skcipher_walk_virt(&walk, &subreq, false); 288 if (err) 289 return err; 290 291 kernel_fpu_begin(); 292 aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 293 walk.nbytes, walk.iv); 294 kernel_fpu_end(); 295 296 return skcipher_walk_done(&walk, 0); 297 } 298 299 static int cts_cbc_decrypt(struct skcipher_request *req) 300 { 301 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 302 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 303 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 304 struct scatterlist *src = req->src, *dst = req->dst; 305 struct scatterlist sg_src[2], sg_dst[2]; 306 struct skcipher_request subreq; 307 struct skcipher_walk walk; 308 int err; 309 310 skcipher_request_set_tfm(&subreq, tfm); 311 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 312 NULL, NULL); 313 314 if (req->cryptlen <= AES_BLOCK_SIZE) { 315 if (req->cryptlen < AES_BLOCK_SIZE) 316 return -EINVAL; 317 cbc_blocks = 1; 318 } 319 320 if (cbc_blocks > 0) { 321 skcipher_request_set_crypt(&subreq, req->src, req->dst, 322 cbc_blocks * AES_BLOCK_SIZE, 323 req->iv); 324 325 err = cbc_decrypt(&subreq); 326 if (err) 327 return err; 328 329 if (req->cryptlen == AES_BLOCK_SIZE) 330 return 0; 331 332 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 333 if (req->dst != req->src) 334 dst = scatterwalk_ffwd(sg_dst, req->dst, 335 subreq.cryptlen); 336 } 337 338 /* handle ciphertext stealing */ 339 skcipher_request_set_crypt(&subreq, src, dst, 340 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 341 req->iv); 342 343 err = skcipher_walk_virt(&walk, &subreq, false); 344 if (err) 345 return err; 346 347 kernel_fpu_begin(); 348 aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 349 walk.nbytes, walk.iv); 350 kernel_fpu_end(); 351 352 return skcipher_walk_done(&walk, 0); 353 } 354 355 #ifdef CONFIG_X86_64 356 /* This is the non-AVX version. */ 357 static int ctr_crypt_aesni(struct skcipher_request *req) 358 { 359 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 361 u8 keystream[AES_BLOCK_SIZE]; 362 struct skcipher_walk walk; 363 unsigned int nbytes; 364 int err; 365 366 err = skcipher_walk_virt(&walk, req, false); 367 368 while ((nbytes = walk.nbytes) > 0) { 369 kernel_fpu_begin(); 370 if (nbytes & AES_BLOCK_MASK) 371 aesni_ctr_enc(ctx, walk.dst.virt.addr, 372 walk.src.virt.addr, 373 nbytes & AES_BLOCK_MASK, walk.iv); 374 nbytes &= ~AES_BLOCK_MASK; 375 376 if (walk.nbytes == walk.total && nbytes > 0) { 377 aesni_enc(ctx, keystream, walk.iv); 378 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes, 379 walk.src.virt.addr + walk.nbytes - nbytes, 380 keystream, nbytes); 381 crypto_inc(walk.iv, AES_BLOCK_SIZE); 382 nbytes = 0; 383 } 384 kernel_fpu_end(); 385 err = skcipher_walk_done(&walk, nbytes); 386 } 387 return err; 388 } 389 #endif 390 391 static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key, 392 unsigned int keylen) 393 { 394 struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); 395 int err; 396 397 err = xts_verify_key(tfm, key, keylen); 398 if (err) 399 return err; 400 401 keylen /= 2; 402 403 /* first half of xts-key is for crypt */ 404 err = aes_set_key_common(&ctx->crypt_ctx, key, keylen); 405 if (err) 406 return err; 407 408 /* second half of xts-key is for tweak */ 409 return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen); 410 } 411 412 typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key, 413 u8 iv[AES_BLOCK_SIZE]); 414 typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key, 415 const u8 *src, u8 *dst, int len, 416 u8 tweak[AES_BLOCK_SIZE]); 417 418 /* This handles cases where the source and/or destination span pages. */ 419 static noinline int 420 xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func) 421 { 422 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 423 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); 424 int tail = req->cryptlen % AES_BLOCK_SIZE; 425 struct scatterlist sg_src[2], sg_dst[2]; 426 struct skcipher_request subreq; 427 struct skcipher_walk walk; 428 struct scatterlist *src, *dst; 429 int err; 430 431 /* 432 * If the message length isn't divisible by the AES block size, then 433 * separate off the last full block and the partial block. This ensures 434 * that they are processed in the same call to the assembly function, 435 * which is required for ciphertext stealing. 436 */ 437 if (tail) { 438 skcipher_request_set_tfm(&subreq, tfm); 439 skcipher_request_set_callback(&subreq, 440 skcipher_request_flags(req), 441 NULL, NULL); 442 skcipher_request_set_crypt(&subreq, req->src, req->dst, 443 req->cryptlen - tail - AES_BLOCK_SIZE, 444 req->iv); 445 req = &subreq; 446 } 447 448 err = skcipher_walk_virt(&walk, req, false); 449 450 while (walk.nbytes) { 451 kernel_fpu_begin(); 452 (*crypt_func)(&ctx->crypt_ctx, 453 walk.src.virt.addr, walk.dst.virt.addr, 454 walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv); 455 kernel_fpu_end(); 456 err = skcipher_walk_done(&walk, 457 walk.nbytes & (AES_BLOCK_SIZE - 1)); 458 } 459 460 if (err || !tail) 461 return err; 462 463 /* Do ciphertext stealing with the last full block and partial block. */ 464 465 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 466 if (req->dst != req->src) 467 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 468 469 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 470 req->iv); 471 472 err = skcipher_walk_virt(&walk, req, false); 473 if (err) 474 return err; 475 476 kernel_fpu_begin(); 477 (*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr, 478 walk.nbytes, req->iv); 479 kernel_fpu_end(); 480 481 return skcipher_walk_done(&walk, 0); 482 } 483 484 /* __always_inline to avoid indirect call in fastpath */ 485 static __always_inline int 486 xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv, 487 xts_crypt_func crypt_func) 488 { 489 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 490 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); 491 492 if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) 493 return -EINVAL; 494 495 kernel_fpu_begin(); 496 (*encrypt_iv)(&ctx->tweak_ctx, req->iv); 497 498 /* 499 * In practice, virtually all XTS plaintexts and ciphertexts are either 500 * 512 or 4096 bytes and do not use multiple scatterlist elements. To 501 * optimize the performance of these cases, the below fast-path handles 502 * single-scatterlist-element messages as efficiently as possible. The 503 * code is 64-bit specific, as it assumes no page mapping is needed. 504 */ 505 if (IS_ENABLED(CONFIG_X86_64) && 506 likely(req->src->length >= req->cryptlen && 507 req->dst->length >= req->cryptlen)) { 508 (*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src), 509 sg_virt(req->dst), req->cryptlen, req->iv); 510 kernel_fpu_end(); 511 return 0; 512 } 513 kernel_fpu_end(); 514 return xts_crypt_slowpath(req, crypt_func); 515 } 516 517 static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, 518 u8 iv[AES_BLOCK_SIZE]) 519 { 520 aesni_enc(tweak_key, iv, iv); 521 } 522 523 static void aesni_xts_encrypt(const struct crypto_aes_ctx *key, 524 const u8 *src, u8 *dst, int len, 525 u8 tweak[AES_BLOCK_SIZE]) 526 { 527 aesni_xts_enc(key, dst, src, len, tweak); 528 } 529 530 static void aesni_xts_decrypt(const struct crypto_aes_ctx *key, 531 const u8 *src, u8 *dst, int len, 532 u8 tweak[AES_BLOCK_SIZE]) 533 { 534 aesni_xts_dec(key, dst, src, len, tweak); 535 } 536 537 static int xts_encrypt_aesni(struct skcipher_request *req) 538 { 539 return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_encrypt); 540 } 541 542 static int xts_decrypt_aesni(struct skcipher_request *req) 543 { 544 return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_decrypt); 545 } 546 547 static struct crypto_alg aesni_cipher_alg = { 548 .cra_name = "aes", 549 .cra_driver_name = "aes-aesni", 550 .cra_priority = 300, 551 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 552 .cra_blocksize = AES_BLOCK_SIZE, 553 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 554 .cra_module = THIS_MODULE, 555 .cra_u = { 556 .cipher = { 557 .cia_min_keysize = AES_MIN_KEY_SIZE, 558 .cia_max_keysize = AES_MAX_KEY_SIZE, 559 .cia_setkey = aes_set_key, 560 .cia_encrypt = aesni_encrypt, 561 .cia_decrypt = aesni_decrypt 562 } 563 } 564 }; 565 566 static struct skcipher_alg aesni_skciphers[] = { 567 { 568 .base = { 569 .cra_name = "ecb(aes)", 570 .cra_driver_name = "ecb-aes-aesni", 571 .cra_priority = 400, 572 .cra_blocksize = AES_BLOCK_SIZE, 573 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 574 .cra_module = THIS_MODULE, 575 }, 576 .min_keysize = AES_MIN_KEY_SIZE, 577 .max_keysize = AES_MAX_KEY_SIZE, 578 .setkey = aesni_skcipher_setkey, 579 .encrypt = ecb_encrypt, 580 .decrypt = ecb_decrypt, 581 }, { 582 .base = { 583 .cra_name = "cbc(aes)", 584 .cra_driver_name = "cbc-aes-aesni", 585 .cra_priority = 400, 586 .cra_blocksize = AES_BLOCK_SIZE, 587 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 588 .cra_module = THIS_MODULE, 589 }, 590 .min_keysize = AES_MIN_KEY_SIZE, 591 .max_keysize = AES_MAX_KEY_SIZE, 592 .ivsize = AES_BLOCK_SIZE, 593 .setkey = aesni_skcipher_setkey, 594 .encrypt = cbc_encrypt, 595 .decrypt = cbc_decrypt, 596 }, { 597 .base = { 598 .cra_name = "cts(cbc(aes))", 599 .cra_driver_name = "cts-cbc-aes-aesni", 600 .cra_priority = 400, 601 .cra_blocksize = AES_BLOCK_SIZE, 602 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 603 .cra_module = THIS_MODULE, 604 }, 605 .min_keysize = AES_MIN_KEY_SIZE, 606 .max_keysize = AES_MAX_KEY_SIZE, 607 .ivsize = AES_BLOCK_SIZE, 608 .walksize = 2 * AES_BLOCK_SIZE, 609 .setkey = aesni_skcipher_setkey, 610 .encrypt = cts_cbc_encrypt, 611 .decrypt = cts_cbc_decrypt, 612 #ifdef CONFIG_X86_64 613 }, { 614 .base = { 615 .cra_name = "ctr(aes)", 616 .cra_driver_name = "ctr-aes-aesni", 617 .cra_priority = 400, 618 .cra_blocksize = 1, 619 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 620 .cra_module = THIS_MODULE, 621 }, 622 .min_keysize = AES_MIN_KEY_SIZE, 623 .max_keysize = AES_MAX_KEY_SIZE, 624 .ivsize = AES_BLOCK_SIZE, 625 .chunksize = AES_BLOCK_SIZE, 626 .setkey = aesni_skcipher_setkey, 627 .encrypt = ctr_crypt_aesni, 628 .decrypt = ctr_crypt_aesni, 629 #endif 630 }, { 631 .base = { 632 .cra_name = "xts(aes)", 633 .cra_driver_name = "xts-aes-aesni", 634 .cra_priority = 401, 635 .cra_blocksize = AES_BLOCK_SIZE, 636 .cra_ctxsize = XTS_AES_CTX_SIZE, 637 .cra_module = THIS_MODULE, 638 }, 639 .min_keysize = 2 * AES_MIN_KEY_SIZE, 640 .max_keysize = 2 * AES_MAX_KEY_SIZE, 641 .ivsize = AES_BLOCK_SIZE, 642 .walksize = 2 * AES_BLOCK_SIZE, 643 .setkey = xts_setkey_aesni, 644 .encrypt = xts_encrypt_aesni, 645 .decrypt = xts_decrypt_aesni, 646 } 647 }; 648 649 #ifdef CONFIG_X86_64 650 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, 651 u8 iv[AES_BLOCK_SIZE]); 652 653 /* __always_inline to avoid indirect call */ 654 static __always_inline int 655 ctr_crypt(struct skcipher_request *req, 656 void (*ctr64_func)(const struct crypto_aes_ctx *key, 657 const u8 *src, u8 *dst, int len, 658 const u64 le_ctr[2])) 659 { 660 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 661 const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm)); 662 unsigned int nbytes, p1_nbytes, nblocks; 663 struct skcipher_walk walk; 664 u64 le_ctr[2]; 665 u64 ctr64; 666 int err; 667 668 ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]); 669 le_ctr[1] = get_unaligned_be64(&req->iv[0]); 670 671 err = skcipher_walk_virt(&walk, req, false); 672 673 while ((nbytes = walk.nbytes) != 0) { 674 if (nbytes < walk.total) { 675 /* Not the end yet, so keep the length block-aligned. */ 676 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 677 nblocks = nbytes / AES_BLOCK_SIZE; 678 } else { 679 /* It's the end, so include any final partial block. */ 680 nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE); 681 } 682 ctr64 += nblocks; 683 684 kernel_fpu_begin(); 685 if (likely(ctr64 >= nblocks)) { 686 /* The low 64 bits of the counter won't overflow. */ 687 (*ctr64_func)(key, walk.src.virt.addr, 688 walk.dst.virt.addr, nbytes, le_ctr); 689 } else { 690 /* 691 * The low 64 bits of the counter will overflow. The 692 * assembly doesn't handle this case, so split the 693 * operation into two at the point where the overflow 694 * will occur. After the first part, add the carry bit. 695 */ 696 p1_nbytes = min_t(unsigned int, nbytes, 697 (nblocks - ctr64) * AES_BLOCK_SIZE); 698 (*ctr64_func)(key, walk.src.virt.addr, 699 walk.dst.virt.addr, p1_nbytes, le_ctr); 700 le_ctr[0] = 0; 701 le_ctr[1]++; 702 (*ctr64_func)(key, walk.src.virt.addr + p1_nbytes, 703 walk.dst.virt.addr + p1_nbytes, 704 nbytes - p1_nbytes, le_ctr); 705 } 706 kernel_fpu_end(); 707 le_ctr[0] = ctr64; 708 709 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 710 } 711 712 put_unaligned_be64(ctr64, &req->iv[8]); 713 put_unaligned_be64(le_ctr[1], &req->iv[0]); 714 715 return err; 716 } 717 718 /* __always_inline to avoid indirect call */ 719 static __always_inline int 720 xctr_crypt(struct skcipher_request *req, 721 void (*xctr_func)(const struct crypto_aes_ctx *key, 722 const u8 *src, u8 *dst, int len, 723 const u8 iv[AES_BLOCK_SIZE], u64 ctr)) 724 { 725 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 726 const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm)); 727 struct skcipher_walk walk; 728 unsigned int nbytes; 729 u64 ctr = 1; 730 int err; 731 732 err = skcipher_walk_virt(&walk, req, false); 733 while ((nbytes = walk.nbytes) != 0) { 734 if (nbytes < walk.total) 735 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 736 737 kernel_fpu_begin(); 738 (*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr, 739 nbytes, req->iv, ctr); 740 kernel_fpu_end(); 741 742 ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE); 743 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 744 } 745 return err; 746 } 747 748 #define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \ 749 \ 750 asmlinkage void \ 751 aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ 752 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ 753 asmlinkage void \ 754 aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ 755 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ 756 \ 757 static int xts_encrypt_##suffix(struct skcipher_request *req) \ 758 { \ 759 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix); \ 760 } \ 761 \ 762 static int xts_decrypt_##suffix(struct skcipher_request *req) \ 763 { \ 764 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \ 765 } \ 766 \ 767 asmlinkage void \ 768 aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key, \ 769 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\ 770 \ 771 static int ctr_crypt_##suffix(struct skcipher_request *req) \ 772 { \ 773 return ctr_crypt(req, aes_ctr64_crypt_##suffix); \ 774 } \ 775 \ 776 asmlinkage void \ 777 aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key, \ 778 const u8 *src, u8 *dst, int len, \ 779 const u8 iv[AES_BLOCK_SIZE], u64 ctr); \ 780 \ 781 static int xctr_crypt_##suffix(struct skcipher_request *req) \ 782 { \ 783 return xctr_crypt(req, aes_xctr_crypt_##suffix); \ 784 } \ 785 \ 786 static struct skcipher_alg skcipher_algs_##suffix[] = {{ \ 787 .base.cra_name = "xts(aes)", \ 788 .base.cra_driver_name = "xts-aes-" driver_name_suffix, \ 789 .base.cra_priority = priority, \ 790 .base.cra_blocksize = AES_BLOCK_SIZE, \ 791 .base.cra_ctxsize = XTS_AES_CTX_SIZE, \ 792 .base.cra_module = THIS_MODULE, \ 793 .min_keysize = 2 * AES_MIN_KEY_SIZE, \ 794 .max_keysize = 2 * AES_MAX_KEY_SIZE, \ 795 .ivsize = AES_BLOCK_SIZE, \ 796 .walksize = 2 * AES_BLOCK_SIZE, \ 797 .setkey = xts_setkey_aesni, \ 798 .encrypt = xts_encrypt_##suffix, \ 799 .decrypt = xts_decrypt_##suffix, \ 800 }, { \ 801 .base.cra_name = "ctr(aes)", \ 802 .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \ 803 .base.cra_priority = priority, \ 804 .base.cra_blocksize = 1, \ 805 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ 806 .base.cra_module = THIS_MODULE, \ 807 .min_keysize = AES_MIN_KEY_SIZE, \ 808 .max_keysize = AES_MAX_KEY_SIZE, \ 809 .ivsize = AES_BLOCK_SIZE, \ 810 .chunksize = AES_BLOCK_SIZE, \ 811 .setkey = aesni_skcipher_setkey, \ 812 .encrypt = ctr_crypt_##suffix, \ 813 .decrypt = ctr_crypt_##suffix, \ 814 }, { \ 815 .base.cra_name = "xctr(aes)", \ 816 .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \ 817 .base.cra_priority = priority, \ 818 .base.cra_blocksize = 1, \ 819 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \ 820 .base.cra_module = THIS_MODULE, \ 821 .min_keysize = AES_MIN_KEY_SIZE, \ 822 .max_keysize = AES_MAX_KEY_SIZE, \ 823 .ivsize = AES_BLOCK_SIZE, \ 824 .chunksize = AES_BLOCK_SIZE, \ 825 .setkey = aesni_skcipher_setkey, \ 826 .encrypt = xctr_crypt_##suffix, \ 827 .decrypt = xctr_crypt_##suffix, \ 828 }} 829 830 DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500); 831 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) 832 DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600); 833 DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800); 834 #endif 835 836 /* The common part of the x86_64 AES-GCM key struct */ 837 struct aes_gcm_key { 838 /* Expanded AES key and the AES key length in bytes */ 839 struct crypto_aes_ctx aes_key; 840 841 /* RFC4106 nonce (used only by the rfc4106 algorithms) */ 842 u32 rfc4106_nonce; 843 }; 844 845 /* Key struct used by the AES-NI implementations of AES-GCM */ 846 struct aes_gcm_key_aesni { 847 /* 848 * Common part of the key. The assembly code requires 16-byte alignment 849 * for the round keys; we get this by them being located at the start of 850 * the struct and the whole struct being 16-byte aligned. 851 */ 852 struct aes_gcm_key base; 853 854 /* 855 * Powers of the hash key H^8 through H^1. These are 128-bit values. 856 * They all have an extra factor of x^-1 and are byte-reversed. 16-byte 857 * alignment is required by the assembly code. 858 */ 859 u64 h_powers[8][2] __aligned(16); 860 861 /* 862 * h_powers_xored[i] contains the two 64-bit halves of h_powers[i] XOR'd 863 * together. It's used for Karatsuba multiplication. 16-byte alignment 864 * is required by the assembly code. 865 */ 866 u64 h_powers_xored[8] __aligned(16); 867 868 /* 869 * H^1 times x^64 (and also the usual extra factor of x^-1). 16-byte 870 * alignment is required by the assembly code. 871 */ 872 u64 h_times_x64[2] __aligned(16); 873 }; 874 #define AES_GCM_KEY_AESNI(key) \ 875 container_of((key), struct aes_gcm_key_aesni, base) 876 #define AES_GCM_KEY_AESNI_SIZE \ 877 (sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1))) 878 879 /* Key struct used by the VAES + AVX10 implementations of AES-GCM */ 880 struct aes_gcm_key_avx10 { 881 /* 882 * Common part of the key. The assembly code prefers 16-byte alignment 883 * for the round keys; we get this by them being located at the start of 884 * the struct and the whole struct being 64-byte aligned. 885 */ 886 struct aes_gcm_key base; 887 888 /* 889 * Powers of the hash key H^16 through H^1. These are 128-bit values. 890 * They all have an extra factor of x^-1 and are byte-reversed. This 891 * array is aligned to a 64-byte boundary to make it naturally aligned 892 * for 512-bit loads, which can improve performance. (The assembly code 893 * doesn't *need* the alignment; this is just an optimization.) 894 */ 895 u64 h_powers[16][2] __aligned(64); 896 897 /* Three padding blocks required by the assembly code */ 898 u64 padding[3][2]; 899 }; 900 #define AES_GCM_KEY_AVX10(key) \ 901 container_of((key), struct aes_gcm_key_avx10, base) 902 #define AES_GCM_KEY_AVX10_SIZE \ 903 (sizeof(struct aes_gcm_key_avx10) + (63 & ~(CRYPTO_MINALIGN - 1))) 904 905 /* 906 * These flags are passed to the AES-GCM helper functions to specify the 907 * specific version of AES-GCM (RFC4106 or not), whether it's encryption or 908 * decryption, and which assembly functions should be called. Assembly 909 * functions are selected using flags instead of function pointers to avoid 910 * indirect calls (which are very expensive on x86) regardless of inlining. 911 */ 912 #define FLAG_RFC4106 BIT(0) 913 #define FLAG_ENC BIT(1) 914 #define FLAG_AVX BIT(2) 915 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) 916 # define FLAG_AVX10_256 BIT(3) 917 # define FLAG_AVX10_512 BIT(4) 918 #else 919 /* 920 * This should cause all calls to the AVX10 assembly functions to be 921 * optimized out, avoiding the need to ifdef each call individually. 922 */ 923 # define FLAG_AVX10_256 0 924 # define FLAG_AVX10_512 0 925 #endif 926 927 static inline struct aes_gcm_key * 928 aes_gcm_key_get(struct crypto_aead *tfm, int flags) 929 { 930 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 931 return PTR_ALIGN(crypto_aead_ctx(tfm), 64); 932 else 933 return PTR_ALIGN(crypto_aead_ctx(tfm), 16); 934 } 935 936 asmlinkage void 937 aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key); 938 asmlinkage void 939 aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key); 940 asmlinkage void 941 aes_gcm_precompute_vaes_avx10_256(struct aes_gcm_key_avx10 *key); 942 asmlinkage void 943 aes_gcm_precompute_vaes_avx10_512(struct aes_gcm_key_avx10 *key); 944 945 static void aes_gcm_precompute(struct aes_gcm_key *key, int flags) 946 { 947 /* 948 * To make things a bit easier on the assembly side, the AVX10 949 * implementations use the same key format. Therefore, a single 950 * function using 256-bit vectors would suffice here. However, it's 951 * straightforward to provide a 512-bit one because of how the assembly 952 * code is structured, and it works nicely because the total size of the 953 * key powers is a multiple of 512 bits. So we take advantage of that. 954 * 955 * A similar situation applies to the AES-NI implementations. 956 */ 957 if (flags & FLAG_AVX10_512) 958 aes_gcm_precompute_vaes_avx10_512(AES_GCM_KEY_AVX10(key)); 959 else if (flags & FLAG_AVX10_256) 960 aes_gcm_precompute_vaes_avx10_256(AES_GCM_KEY_AVX10(key)); 961 else if (flags & FLAG_AVX) 962 aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key)); 963 else 964 aes_gcm_precompute_aesni(AES_GCM_KEY_AESNI(key)); 965 } 966 967 asmlinkage void 968 aes_gcm_aad_update_aesni(const struct aes_gcm_key_aesni *key, 969 u8 ghash_acc[16], const u8 *aad, int aadlen); 970 asmlinkage void 971 aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key, 972 u8 ghash_acc[16], const u8 *aad, int aadlen); 973 asmlinkage void 974 aes_gcm_aad_update_vaes_avx10(const struct aes_gcm_key_avx10 *key, 975 u8 ghash_acc[16], const u8 *aad, int aadlen); 976 977 static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16], 978 const u8 *aad, int aadlen, int flags) 979 { 980 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 981 aes_gcm_aad_update_vaes_avx10(AES_GCM_KEY_AVX10(key), ghash_acc, 982 aad, aadlen); 983 else if (flags & FLAG_AVX) 984 aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc, 985 aad, aadlen); 986 else 987 aes_gcm_aad_update_aesni(AES_GCM_KEY_AESNI(key), ghash_acc, 988 aad, aadlen); 989 } 990 991 asmlinkage void 992 aes_gcm_enc_update_aesni(const struct aes_gcm_key_aesni *key, 993 const u32 le_ctr[4], u8 ghash_acc[16], 994 const u8 *src, u8 *dst, int datalen); 995 asmlinkage void 996 aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key, 997 const u32 le_ctr[4], u8 ghash_acc[16], 998 const u8 *src, u8 *dst, int datalen); 999 asmlinkage void 1000 aes_gcm_enc_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key, 1001 const u32 le_ctr[4], u8 ghash_acc[16], 1002 const u8 *src, u8 *dst, int datalen); 1003 asmlinkage void 1004 aes_gcm_enc_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key, 1005 const u32 le_ctr[4], u8 ghash_acc[16], 1006 const u8 *src, u8 *dst, int datalen); 1007 1008 asmlinkage void 1009 aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key, 1010 const u32 le_ctr[4], u8 ghash_acc[16], 1011 const u8 *src, u8 *dst, int datalen); 1012 asmlinkage void 1013 aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key, 1014 const u32 le_ctr[4], u8 ghash_acc[16], 1015 const u8 *src, u8 *dst, int datalen); 1016 asmlinkage void 1017 aes_gcm_dec_update_vaes_avx10_256(const struct aes_gcm_key_avx10 *key, 1018 const u32 le_ctr[4], u8 ghash_acc[16], 1019 const u8 *src, u8 *dst, int datalen); 1020 asmlinkage void 1021 aes_gcm_dec_update_vaes_avx10_512(const struct aes_gcm_key_avx10 *key, 1022 const u32 le_ctr[4], u8 ghash_acc[16], 1023 const u8 *src, u8 *dst, int datalen); 1024 1025 /* __always_inline to optimize out the branches based on @flags */ 1026 static __always_inline void 1027 aes_gcm_update(const struct aes_gcm_key *key, 1028 const u32 le_ctr[4], u8 ghash_acc[16], 1029 const u8 *src, u8 *dst, int datalen, int flags) 1030 { 1031 if (flags & FLAG_ENC) { 1032 if (flags & FLAG_AVX10_512) 1033 aes_gcm_enc_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key), 1034 le_ctr, ghash_acc, 1035 src, dst, datalen); 1036 else if (flags & FLAG_AVX10_256) 1037 aes_gcm_enc_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key), 1038 le_ctr, ghash_acc, 1039 src, dst, datalen); 1040 else if (flags & FLAG_AVX) 1041 aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key), 1042 le_ctr, ghash_acc, 1043 src, dst, datalen); 1044 else 1045 aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr, 1046 ghash_acc, src, dst, datalen); 1047 } else { 1048 if (flags & FLAG_AVX10_512) 1049 aes_gcm_dec_update_vaes_avx10_512(AES_GCM_KEY_AVX10(key), 1050 le_ctr, ghash_acc, 1051 src, dst, datalen); 1052 else if (flags & FLAG_AVX10_256) 1053 aes_gcm_dec_update_vaes_avx10_256(AES_GCM_KEY_AVX10(key), 1054 le_ctr, ghash_acc, 1055 src, dst, datalen); 1056 else if (flags & FLAG_AVX) 1057 aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key), 1058 le_ctr, ghash_acc, 1059 src, dst, datalen); 1060 else 1061 aes_gcm_dec_update_aesni(AES_GCM_KEY_AESNI(key), 1062 le_ctr, ghash_acc, 1063 src, dst, datalen); 1064 } 1065 } 1066 1067 asmlinkage void 1068 aes_gcm_enc_final_aesni(const struct aes_gcm_key_aesni *key, 1069 const u32 le_ctr[4], u8 ghash_acc[16], 1070 u64 total_aadlen, u64 total_datalen); 1071 asmlinkage void 1072 aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key, 1073 const u32 le_ctr[4], u8 ghash_acc[16], 1074 u64 total_aadlen, u64 total_datalen); 1075 asmlinkage void 1076 aes_gcm_enc_final_vaes_avx10(const struct aes_gcm_key_avx10 *key, 1077 const u32 le_ctr[4], u8 ghash_acc[16], 1078 u64 total_aadlen, u64 total_datalen); 1079 1080 /* __always_inline to optimize out the branches based on @flags */ 1081 static __always_inline void 1082 aes_gcm_enc_final(const struct aes_gcm_key *key, 1083 const u32 le_ctr[4], u8 ghash_acc[16], 1084 u64 total_aadlen, u64 total_datalen, int flags) 1085 { 1086 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 1087 aes_gcm_enc_final_vaes_avx10(AES_GCM_KEY_AVX10(key), 1088 le_ctr, ghash_acc, 1089 total_aadlen, total_datalen); 1090 else if (flags & FLAG_AVX) 1091 aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key), 1092 le_ctr, ghash_acc, 1093 total_aadlen, total_datalen); 1094 else 1095 aes_gcm_enc_final_aesni(AES_GCM_KEY_AESNI(key), 1096 le_ctr, ghash_acc, 1097 total_aadlen, total_datalen); 1098 } 1099 1100 asmlinkage bool __must_check 1101 aes_gcm_dec_final_aesni(const struct aes_gcm_key_aesni *key, 1102 const u32 le_ctr[4], const u8 ghash_acc[16], 1103 u64 total_aadlen, u64 total_datalen, 1104 const u8 tag[16], int taglen); 1105 asmlinkage bool __must_check 1106 aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key, 1107 const u32 le_ctr[4], const u8 ghash_acc[16], 1108 u64 total_aadlen, u64 total_datalen, 1109 const u8 tag[16], int taglen); 1110 asmlinkage bool __must_check 1111 aes_gcm_dec_final_vaes_avx10(const struct aes_gcm_key_avx10 *key, 1112 const u32 le_ctr[4], const u8 ghash_acc[16], 1113 u64 total_aadlen, u64 total_datalen, 1114 const u8 tag[16], int taglen); 1115 1116 /* __always_inline to optimize out the branches based on @flags */ 1117 static __always_inline bool __must_check 1118 aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4], 1119 u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen, 1120 u8 tag[16], int taglen, int flags) 1121 { 1122 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) 1123 return aes_gcm_dec_final_vaes_avx10(AES_GCM_KEY_AVX10(key), 1124 le_ctr, ghash_acc, 1125 total_aadlen, total_datalen, 1126 tag, taglen); 1127 else if (flags & FLAG_AVX) 1128 return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key), 1129 le_ctr, ghash_acc, 1130 total_aadlen, total_datalen, 1131 tag, taglen); 1132 else 1133 return aes_gcm_dec_final_aesni(AES_GCM_KEY_AESNI(key), 1134 le_ctr, ghash_acc, 1135 total_aadlen, total_datalen, 1136 tag, taglen); 1137 } 1138 1139 /* 1140 * This is the Integrity Check Value (aka the authentication tag) length and can 1141 * be 8, 12 or 16 bytes long. 1142 */ 1143 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 1144 unsigned int authsize) 1145 { 1146 switch (authsize) { 1147 case 8: 1148 case 12: 1149 case 16: 1150 break; 1151 default: 1152 return -EINVAL; 1153 } 1154 1155 return 0; 1156 } 1157 1158 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 1159 unsigned int authsize) 1160 { 1161 switch (authsize) { 1162 case 4: 1163 case 8: 1164 case 12: 1165 case 13: 1166 case 14: 1167 case 15: 1168 case 16: 1169 break; 1170 default: 1171 return -EINVAL; 1172 } 1173 1174 return 0; 1175 } 1176 1177 /* 1178 * This is the setkey function for the x86_64 implementations of AES-GCM. It 1179 * saves the RFC4106 nonce if applicable, expands the AES key, and precomputes 1180 * powers of the hash key. 1181 * 1182 * To comply with the crypto_aead API, this has to be usable in no-SIMD context. 1183 * For that reason, this function includes a portable C implementation of the 1184 * needed logic. However, the portable C implementation is very slow, taking 1185 * about the same time as encrypting 37 KB of data. To be ready for users that 1186 * may set a key even somewhat frequently, we therefore also include a SIMD 1187 * assembly implementation, expanding the AES key using AES-NI and precomputing 1188 * the hash key powers using PCLMULQDQ or VPCLMULQDQ. 1189 */ 1190 static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key, 1191 unsigned int keylen, int flags) 1192 { 1193 struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags); 1194 int err; 1195 1196 if (flags & FLAG_RFC4106) { 1197 if (keylen < 4) 1198 return -EINVAL; 1199 keylen -= 4; 1200 key->rfc4106_nonce = get_unaligned_be32(raw_key + keylen); 1201 } 1202 1203 /* The assembly code assumes the following offsets. */ 1204 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0); 1205 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480); 1206 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496); 1207 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624); 1208 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688); 1209 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_enc) != 0); 1210 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, base.aes_key.key_length) != 480); 1211 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, h_powers) != 512); 1212 BUILD_BUG_ON(offsetof(struct aes_gcm_key_avx10, padding) != 768); 1213 1214 if (likely(crypto_simd_usable())) { 1215 err = aes_check_keylen(keylen); 1216 if (err) 1217 return err; 1218 kernel_fpu_begin(); 1219 aesni_set_key(&key->aes_key, raw_key, keylen); 1220 aes_gcm_precompute(key, flags); 1221 kernel_fpu_end(); 1222 } else { 1223 static const u8 x_to_the_minus1[16] __aligned(__alignof__(be128)) = { 1224 [0] = 0xc2, [15] = 1 1225 }; 1226 static const u8 x_to_the_63[16] __aligned(__alignof__(be128)) = { 1227 [7] = 1, 1228 }; 1229 be128 h1 = {}; 1230 be128 h; 1231 int i; 1232 1233 err = aes_expandkey(&key->aes_key, raw_key, keylen); 1234 if (err) 1235 return err; 1236 1237 /* Encrypt the all-zeroes block to get the hash key H^1 */ 1238 aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1); 1239 1240 /* Compute H^1 * x^-1 */ 1241 h = h1; 1242 gf128mul_lle(&h, (const be128 *)x_to_the_minus1); 1243 1244 /* Compute the needed key powers */ 1245 if (flags & (FLAG_AVX10_256 | FLAG_AVX10_512)) { 1246 struct aes_gcm_key_avx10 *k = AES_GCM_KEY_AVX10(key); 1247 1248 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 1249 k->h_powers[i][0] = be64_to_cpu(h.b); 1250 k->h_powers[i][1] = be64_to_cpu(h.a); 1251 gf128mul_lle(&h, &h1); 1252 } 1253 memset(k->padding, 0, sizeof(k->padding)); 1254 } else { 1255 struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key); 1256 1257 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) { 1258 k->h_powers[i][0] = be64_to_cpu(h.b); 1259 k->h_powers[i][1] = be64_to_cpu(h.a); 1260 k->h_powers_xored[i] = k->h_powers[i][0] ^ 1261 k->h_powers[i][1]; 1262 gf128mul_lle(&h, &h1); 1263 } 1264 gf128mul_lle(&h1, (const be128 *)x_to_the_63); 1265 k->h_times_x64[0] = be64_to_cpu(h1.b); 1266 k->h_times_x64[1] = be64_to_cpu(h1.a); 1267 } 1268 } 1269 return 0; 1270 } 1271 1272 /* 1273 * Initialize @ghash_acc, then pass all @assoclen bytes of associated data 1274 * (a.k.a. additional authenticated data) from @sg_src through the GHASH update 1275 * assembly function. kernel_fpu_begin() must have already been called. 1276 */ 1277 static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], 1278 struct scatterlist *sg_src, unsigned int assoclen, 1279 int flags) 1280 { 1281 struct scatter_walk walk; 1282 /* 1283 * The assembly function requires that the length of any non-last 1284 * segment of associated data be a multiple of 16 bytes, so this 1285 * function does the buffering needed to achieve that. 1286 */ 1287 unsigned int pos = 0; 1288 u8 buf[16]; 1289 1290 memset(ghash_acc, 0, 16); 1291 scatterwalk_start(&walk, sg_src); 1292 1293 while (assoclen) { 1294 unsigned int orig_len_this_step = scatterwalk_next( 1295 &walk, assoclen); 1296 unsigned int len_this_step = orig_len_this_step; 1297 unsigned int len; 1298 const u8 *src = walk.addr; 1299 1300 if (unlikely(pos)) { 1301 len = min(len_this_step, 16 - pos); 1302 memcpy(&buf[pos], src, len); 1303 pos += len; 1304 src += len; 1305 len_this_step -= len; 1306 if (pos < 16) 1307 goto next; 1308 aes_gcm_aad_update(key, ghash_acc, buf, 16, flags); 1309 pos = 0; 1310 } 1311 len = len_this_step; 1312 if (unlikely(assoclen)) /* Not the last segment yet? */ 1313 len = round_down(len, 16); 1314 aes_gcm_aad_update(key, ghash_acc, src, len, flags); 1315 src += len; 1316 len_this_step -= len; 1317 if (unlikely(len_this_step)) { 1318 memcpy(buf, src, len_this_step); 1319 pos = len_this_step; 1320 } 1321 next: 1322 scatterwalk_done_src(&walk, orig_len_this_step); 1323 if (need_resched()) { 1324 kernel_fpu_end(); 1325 kernel_fpu_begin(); 1326 } 1327 assoclen -= orig_len_this_step; 1328 } 1329 if (unlikely(pos)) 1330 aes_gcm_aad_update(key, ghash_acc, buf, pos, flags); 1331 } 1332 1333 1334 /* __always_inline to optimize out the branches based on @flags */ 1335 static __always_inline int 1336 gcm_crypt(struct aead_request *req, int flags) 1337 { 1338 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1339 const struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags); 1340 unsigned int assoclen = req->assoclen; 1341 struct skcipher_walk walk; 1342 unsigned int nbytes; 1343 u8 ghash_acc[16]; /* GHASH accumulator */ 1344 u32 le_ctr[4]; /* Counter in little-endian format */ 1345 int taglen; 1346 int err; 1347 1348 /* Initialize the counter and determine the associated data length. */ 1349 le_ctr[0] = 2; 1350 if (flags & FLAG_RFC4106) { 1351 if (unlikely(assoclen != 16 && assoclen != 20)) 1352 return -EINVAL; 1353 assoclen -= 8; 1354 le_ctr[1] = get_unaligned_be32(req->iv + 4); 1355 le_ctr[2] = get_unaligned_be32(req->iv + 0); 1356 le_ctr[3] = key->rfc4106_nonce; /* already byte-swapped */ 1357 } else { 1358 le_ctr[1] = get_unaligned_be32(req->iv + 8); 1359 le_ctr[2] = get_unaligned_be32(req->iv + 4); 1360 le_ctr[3] = get_unaligned_be32(req->iv + 0); 1361 } 1362 1363 /* Begin walking through the plaintext or ciphertext. */ 1364 if (flags & FLAG_ENC) 1365 err = skcipher_walk_aead_encrypt(&walk, req, false); 1366 else 1367 err = skcipher_walk_aead_decrypt(&walk, req, false); 1368 if (err) 1369 return err; 1370 1371 /* 1372 * Since the AES-GCM assembly code requires that at least three assembly 1373 * functions be called to process any message (this is needed to support 1374 * incremental updates cleanly), to reduce overhead we try to do all 1375 * three calls in the same kernel FPU section if possible. We close the 1376 * section and start a new one if there are multiple data segments or if 1377 * rescheduling is needed while processing the associated data. 1378 */ 1379 kernel_fpu_begin(); 1380 1381 /* Pass the associated data through GHASH. */ 1382 gcm_process_assoc(key, ghash_acc, req->src, assoclen, flags); 1383 1384 /* En/decrypt the data and pass the ciphertext through GHASH. */ 1385 while (unlikely((nbytes = walk.nbytes) < walk.total)) { 1386 /* 1387 * Non-last segment. In this case, the assembly function 1388 * requires that the length be a multiple of 16 (AES_BLOCK_SIZE) 1389 * bytes. The needed buffering of up to 16 bytes is handled by 1390 * the skcipher_walk. Here we just need to round down to a 1391 * multiple of 16. 1392 */ 1393 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 1394 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr, 1395 walk.dst.virt.addr, nbytes, flags); 1396 le_ctr[0] += nbytes / AES_BLOCK_SIZE; 1397 kernel_fpu_end(); 1398 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 1399 if (err) 1400 return err; 1401 kernel_fpu_begin(); 1402 } 1403 /* Last segment: process all remaining data. */ 1404 aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr, 1405 walk.dst.virt.addr, nbytes, flags); 1406 /* 1407 * The low word of the counter isn't used by the finalize, so there's no 1408 * need to increment it here. 1409 */ 1410 1411 /* Finalize */ 1412 taglen = crypto_aead_authsize(tfm); 1413 if (flags & FLAG_ENC) { 1414 /* Finish computing the auth tag. */ 1415 aes_gcm_enc_final(key, le_ctr, ghash_acc, assoclen, 1416 req->cryptlen, flags); 1417 1418 /* Store the computed auth tag in the dst scatterlist. */ 1419 scatterwalk_map_and_copy(ghash_acc, req->dst, req->assoclen + 1420 req->cryptlen, taglen, 1); 1421 } else { 1422 unsigned int datalen = req->cryptlen - taglen; 1423 u8 tag[16]; 1424 1425 /* Get the transmitted auth tag from the src scatterlist. */ 1426 scatterwalk_map_and_copy(tag, req->src, req->assoclen + datalen, 1427 taglen, 0); 1428 /* 1429 * Finish computing the auth tag and compare it to the 1430 * transmitted one. The assembly function does the actual tag 1431 * comparison. Here, just check the boolean result. 1432 */ 1433 if (!aes_gcm_dec_final(key, le_ctr, ghash_acc, assoclen, 1434 datalen, tag, taglen, flags)) 1435 err = -EBADMSG; 1436 } 1437 kernel_fpu_end(); 1438 if (nbytes) 1439 skcipher_walk_done(&walk, 0); 1440 return err; 1441 } 1442 1443 #define DEFINE_GCM_ALGS(suffix, flags, generic_driver_name, rfc_driver_name, \ 1444 ctxsize, priority) \ 1445 \ 1446 static int gcm_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \ 1447 unsigned int keylen) \ 1448 { \ 1449 return gcm_setkey(tfm, raw_key, keylen, (flags)); \ 1450 } \ 1451 \ 1452 static int gcm_encrypt_##suffix(struct aead_request *req) \ 1453 { \ 1454 return gcm_crypt(req, (flags) | FLAG_ENC); \ 1455 } \ 1456 \ 1457 static int gcm_decrypt_##suffix(struct aead_request *req) \ 1458 { \ 1459 return gcm_crypt(req, (flags)); \ 1460 } \ 1461 \ 1462 static int rfc4106_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \ 1463 unsigned int keylen) \ 1464 { \ 1465 return gcm_setkey(tfm, raw_key, keylen, (flags) | FLAG_RFC4106); \ 1466 } \ 1467 \ 1468 static int rfc4106_encrypt_##suffix(struct aead_request *req) \ 1469 { \ 1470 return gcm_crypt(req, (flags) | FLAG_RFC4106 | FLAG_ENC); \ 1471 } \ 1472 \ 1473 static int rfc4106_decrypt_##suffix(struct aead_request *req) \ 1474 { \ 1475 return gcm_crypt(req, (flags) | FLAG_RFC4106); \ 1476 } \ 1477 \ 1478 static struct aead_alg aes_gcm_algs_##suffix[] = { { \ 1479 .setkey = gcm_setkey_##suffix, \ 1480 .setauthsize = generic_gcmaes_set_authsize, \ 1481 .encrypt = gcm_encrypt_##suffix, \ 1482 .decrypt = gcm_decrypt_##suffix, \ 1483 .ivsize = GCM_AES_IV_SIZE, \ 1484 .chunksize = AES_BLOCK_SIZE, \ 1485 .maxauthsize = 16, \ 1486 .base = { \ 1487 .cra_name = "gcm(aes)", \ 1488 .cra_driver_name = generic_driver_name, \ 1489 .cra_priority = (priority), \ 1490 .cra_blocksize = 1, \ 1491 .cra_ctxsize = (ctxsize), \ 1492 .cra_module = THIS_MODULE, \ 1493 }, \ 1494 }, { \ 1495 .setkey = rfc4106_setkey_##suffix, \ 1496 .setauthsize = common_rfc4106_set_authsize, \ 1497 .encrypt = rfc4106_encrypt_##suffix, \ 1498 .decrypt = rfc4106_decrypt_##suffix, \ 1499 .ivsize = GCM_RFC4106_IV_SIZE, \ 1500 .chunksize = AES_BLOCK_SIZE, \ 1501 .maxauthsize = 16, \ 1502 .base = { \ 1503 .cra_name = "rfc4106(gcm(aes))", \ 1504 .cra_driver_name = rfc_driver_name, \ 1505 .cra_priority = (priority), \ 1506 .cra_blocksize = 1, \ 1507 .cra_ctxsize = (ctxsize), \ 1508 .cra_module = THIS_MODULE, \ 1509 }, \ 1510 } } 1511 1512 /* aes_gcm_algs_aesni */ 1513 DEFINE_GCM_ALGS(aesni, /* no flags */ 0, 1514 "generic-gcm-aesni", "rfc4106-gcm-aesni", 1515 AES_GCM_KEY_AESNI_SIZE, 400); 1516 1517 /* aes_gcm_algs_aesni_avx */ 1518 DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX, 1519 "generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx", 1520 AES_GCM_KEY_AESNI_SIZE, 500); 1521 1522 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) 1523 /* aes_gcm_algs_vaes_avx10_256 */ 1524 DEFINE_GCM_ALGS(vaes_avx10_256, FLAG_AVX10_256, 1525 "generic-gcm-vaes-avx10_256", "rfc4106-gcm-vaes-avx10_256", 1526 AES_GCM_KEY_AVX10_SIZE, 700); 1527 1528 /* aes_gcm_algs_vaes_avx10_512 */ 1529 DEFINE_GCM_ALGS(vaes_avx10_512, FLAG_AVX10_512, 1530 "generic-gcm-vaes-avx10_512", "rfc4106-gcm-vaes-avx10_512", 1531 AES_GCM_KEY_AVX10_SIZE, 800); 1532 #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ 1533 1534 static int __init register_avx_algs(void) 1535 { 1536 int err; 1537 1538 if (!boot_cpu_has(X86_FEATURE_AVX)) 1539 return 0; 1540 err = crypto_register_skciphers(skcipher_algs_aesni_avx, 1541 ARRAY_SIZE(skcipher_algs_aesni_avx)); 1542 if (err) 1543 return err; 1544 err = crypto_register_aeads(aes_gcm_algs_aesni_avx, 1545 ARRAY_SIZE(aes_gcm_algs_aesni_avx)); 1546 if (err) 1547 return err; 1548 /* 1549 * Note: not all the algorithms registered below actually require 1550 * VPCLMULQDQ. But in practice every CPU with VAES also has VPCLMULQDQ. 1551 * Similarly, the assembler support was added at about the same time. 1552 * For simplicity, just always check for VAES and VPCLMULQDQ together. 1553 */ 1554 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) 1555 if (!boot_cpu_has(X86_FEATURE_AVX2) || 1556 !boot_cpu_has(X86_FEATURE_VAES) || 1557 !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) || 1558 !boot_cpu_has(X86_FEATURE_PCLMULQDQ) || 1559 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) 1560 return 0; 1561 err = crypto_register_skciphers(skcipher_algs_vaes_avx2, 1562 ARRAY_SIZE(skcipher_algs_vaes_avx2)); 1563 if (err) 1564 return err; 1565 1566 if (!boot_cpu_has(X86_FEATURE_AVX512BW) || 1567 !boot_cpu_has(X86_FEATURE_AVX512VL) || 1568 !boot_cpu_has(X86_FEATURE_BMI2) || 1569 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | 1570 XFEATURE_MASK_AVX512, NULL)) 1571 return 0; 1572 1573 err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_256, 1574 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256)); 1575 if (err) 1576 return err; 1577 1578 if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) { 1579 int i; 1580 1581 for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++) 1582 skcipher_algs_vaes_avx512[i].base.cra_priority = 1; 1583 for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++) 1584 aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1; 1585 } 1586 1587 err = crypto_register_skciphers(skcipher_algs_vaes_avx512, 1588 ARRAY_SIZE(skcipher_algs_vaes_avx512)); 1589 if (err) 1590 return err; 1591 err = crypto_register_aeads(aes_gcm_algs_vaes_avx10_512, 1592 ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512)); 1593 if (err) 1594 return err; 1595 #endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */ 1596 return 0; 1597 } 1598 1599 #define unregister_skciphers(A) \ 1600 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ 1601 crypto_unregister_skciphers((A), ARRAY_SIZE(A)) 1602 #define unregister_aeads(A) \ 1603 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \ 1604 crypto_unregister_aeads((A), ARRAY_SIZE(A)) 1605 1606 static void unregister_avx_algs(void) 1607 { 1608 unregister_skciphers(skcipher_algs_aesni_avx); 1609 unregister_aeads(aes_gcm_algs_aesni_avx); 1610 #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) 1611 unregister_skciphers(skcipher_algs_vaes_avx2); 1612 unregister_skciphers(skcipher_algs_vaes_avx512); 1613 unregister_aeads(aes_gcm_algs_vaes_avx10_256); 1614 unregister_aeads(aes_gcm_algs_vaes_avx10_512); 1615 #endif 1616 } 1617 #else /* CONFIG_X86_64 */ 1618 static struct aead_alg aes_gcm_algs_aesni[0]; 1619 1620 static int __init register_avx_algs(void) 1621 { 1622 return 0; 1623 } 1624 1625 static void unregister_avx_algs(void) 1626 { 1627 } 1628 #endif /* !CONFIG_X86_64 */ 1629 1630 static const struct x86_cpu_id aesni_cpu_id[] = { 1631 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), 1632 {} 1633 }; 1634 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1635 1636 static int __init aesni_init(void) 1637 { 1638 int err; 1639 1640 if (!x86_match_cpu(aesni_cpu_id)) 1641 return -ENODEV; 1642 1643 err = crypto_register_alg(&aesni_cipher_alg); 1644 if (err) 1645 return err; 1646 1647 err = crypto_register_skciphers(aesni_skciphers, 1648 ARRAY_SIZE(aesni_skciphers)); 1649 if (err) 1650 goto unregister_cipher; 1651 1652 err = crypto_register_aeads(aes_gcm_algs_aesni, 1653 ARRAY_SIZE(aes_gcm_algs_aesni)); 1654 if (err) 1655 goto unregister_skciphers; 1656 1657 err = register_avx_algs(); 1658 if (err) 1659 goto unregister_avx; 1660 1661 return 0; 1662 1663 unregister_avx: 1664 unregister_avx_algs(); 1665 crypto_unregister_aeads(aes_gcm_algs_aesni, 1666 ARRAY_SIZE(aes_gcm_algs_aesni)); 1667 unregister_skciphers: 1668 crypto_unregister_skciphers(aesni_skciphers, 1669 ARRAY_SIZE(aesni_skciphers)); 1670 unregister_cipher: 1671 crypto_unregister_alg(&aesni_cipher_alg); 1672 return err; 1673 } 1674 1675 static void __exit aesni_exit(void) 1676 { 1677 crypto_unregister_aeads(aes_gcm_algs_aesni, 1678 ARRAY_SIZE(aes_gcm_algs_aesni)); 1679 crypto_unregister_skciphers(aesni_skciphers, 1680 ARRAY_SIZE(aesni_skciphers)); 1681 crypto_unregister_alg(&aesni_cipher_alg); 1682 unregister_avx_algs(); 1683 } 1684 1685 module_init(aesni_init); 1686 module_exit(aesni_exit); 1687 1688 MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions"); 1689 MODULE_LICENSE("GPL"); 1690 MODULE_ALIAS_CRYPTO("aes"); 1691