1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <asm/i387.h> 32 #include <asm/aes.h> 33 #include <crypto/scatterwalk.h> 34 #include <crypto/internal/aead.h> 35 #include <linux/workqueue.h> 36 #include <linux/spinlock.h> 37 38 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 39 #define HAS_CTR 40 #endif 41 42 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) 43 #define HAS_LRW 44 #endif 45 46 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 47 #define HAS_PCBC 48 #endif 49 50 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE) 51 #define HAS_XTS 52 #endif 53 54 struct async_aes_ctx { 55 struct cryptd_ablkcipher *cryptd_tfm; 56 }; 57 58 /* This data is stored at the end of the crypto_tfm struct. 59 * It's a type of per "session" data storage location. 60 * This needs to be 16 byte aligned. 61 */ 62 struct aesni_rfc4106_gcm_ctx { 63 u8 hash_subkey[16]; 64 struct crypto_aes_ctx aes_key_expanded; 65 u8 nonce[4]; 66 struct cryptd_aead *cryptd_tfm; 67 }; 68 69 struct aesni_gcm_set_hash_subkey_result { 70 int err; 71 struct completion completion; 72 }; 73 74 struct aesni_hash_subkey_req_data { 75 u8 iv[16]; 76 struct aesni_gcm_set_hash_subkey_result result; 77 struct scatterlist sg; 78 }; 79 80 #define AESNI_ALIGN (16) 81 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 82 #define RFC4106_HASH_SUBKEY_SIZE 16 83 84 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 85 unsigned int key_len); 86 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 87 const u8 *in); 88 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 89 const u8 *in); 90 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 91 const u8 *in, unsigned int len); 92 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 93 const u8 *in, unsigned int len); 94 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 95 const u8 *in, unsigned int len, u8 *iv); 96 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 97 const u8 *in, unsigned int len, u8 *iv); 98 99 int crypto_fpu_init(void); 100 void crypto_fpu_exit(void); 101 102 #ifdef CONFIG_X86_64 103 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 104 const u8 *in, unsigned int len, u8 *iv); 105 106 /* asmlinkage void aesni_gcm_enc() 107 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 108 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 109 * const u8 *in, Plaintext input 110 * unsigned long plaintext_len, Length of data in bytes for encryption. 111 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 112 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 113 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 114 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 115 * const u8 *aad, Additional Authentication Data (AAD) 116 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 117 * is going to be 8 or 12 bytes 118 * u8 *auth_tag, Authenticated Tag output. 119 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 120 * Valid values are 16 (most likely), 12 or 8. 121 */ 122 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 123 const u8 *in, unsigned long plaintext_len, u8 *iv, 124 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 125 u8 *auth_tag, unsigned long auth_tag_len); 126 127 /* asmlinkage void aesni_gcm_dec() 128 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 129 * u8 *out, Plaintext output. Decrypt in-place is allowed. 130 * const u8 *in, Ciphertext input 131 * unsigned long ciphertext_len, Length of data in bytes for decryption. 132 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 133 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 134 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 135 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 136 * const u8 *aad, Additional Authentication Data (AAD) 137 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 138 * to be 8 or 12 bytes 139 * u8 *auth_tag, Authenticated Tag output. 140 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 141 * Valid values are 16 (most likely), 12 or 8. 142 */ 143 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 144 const u8 *in, unsigned long ciphertext_len, u8 *iv, 145 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 146 u8 *auth_tag, unsigned long auth_tag_len); 147 148 static inline struct 149 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 150 { 151 return 152 (struct aesni_rfc4106_gcm_ctx *) 153 PTR_ALIGN((u8 *) 154 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 155 } 156 #endif 157 158 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 159 { 160 unsigned long addr = (unsigned long)raw_ctx; 161 unsigned long align = AESNI_ALIGN; 162 163 if (align <= crypto_tfm_ctx_alignment()) 164 align = 1; 165 return (struct crypto_aes_ctx *)ALIGN(addr, align); 166 } 167 168 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 169 const u8 *in_key, unsigned int key_len) 170 { 171 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 172 u32 *flags = &tfm->crt_flags; 173 int err; 174 175 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 176 key_len != AES_KEYSIZE_256) { 177 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 178 return -EINVAL; 179 } 180 181 if (!irq_fpu_usable()) 182 err = crypto_aes_expand_key(ctx, in_key, key_len); 183 else { 184 kernel_fpu_begin(); 185 err = aesni_set_key(ctx, in_key, key_len); 186 kernel_fpu_end(); 187 } 188 189 return err; 190 } 191 192 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 193 unsigned int key_len) 194 { 195 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 196 } 197 198 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 199 { 200 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 201 202 if (!irq_fpu_usable()) 203 crypto_aes_encrypt_x86(ctx, dst, src); 204 else { 205 kernel_fpu_begin(); 206 aesni_enc(ctx, dst, src); 207 kernel_fpu_end(); 208 } 209 } 210 211 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 212 { 213 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 214 215 if (!irq_fpu_usable()) 216 crypto_aes_decrypt_x86(ctx, dst, src); 217 else { 218 kernel_fpu_begin(); 219 aesni_dec(ctx, dst, src); 220 kernel_fpu_end(); 221 } 222 } 223 224 static struct crypto_alg aesni_alg = { 225 .cra_name = "aes", 226 .cra_driver_name = "aes-aesni", 227 .cra_priority = 300, 228 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 229 .cra_blocksize = AES_BLOCK_SIZE, 230 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 231 .cra_alignmask = 0, 232 .cra_module = THIS_MODULE, 233 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list), 234 .cra_u = { 235 .cipher = { 236 .cia_min_keysize = AES_MIN_KEY_SIZE, 237 .cia_max_keysize = AES_MAX_KEY_SIZE, 238 .cia_setkey = aes_set_key, 239 .cia_encrypt = aes_encrypt, 240 .cia_decrypt = aes_decrypt 241 } 242 } 243 }; 244 245 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 246 { 247 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 248 249 aesni_enc(ctx, dst, src); 250 } 251 252 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 253 { 254 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 255 256 aesni_dec(ctx, dst, src); 257 } 258 259 static struct crypto_alg __aesni_alg = { 260 .cra_name = "__aes-aesni", 261 .cra_driver_name = "__driver-aes-aesni", 262 .cra_priority = 0, 263 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 264 .cra_blocksize = AES_BLOCK_SIZE, 265 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 266 .cra_alignmask = 0, 267 .cra_module = THIS_MODULE, 268 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list), 269 .cra_u = { 270 .cipher = { 271 .cia_min_keysize = AES_MIN_KEY_SIZE, 272 .cia_max_keysize = AES_MAX_KEY_SIZE, 273 .cia_setkey = aes_set_key, 274 .cia_encrypt = __aes_encrypt, 275 .cia_decrypt = __aes_decrypt 276 } 277 } 278 }; 279 280 static int ecb_encrypt(struct blkcipher_desc *desc, 281 struct scatterlist *dst, struct scatterlist *src, 282 unsigned int nbytes) 283 { 284 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 285 struct blkcipher_walk walk; 286 int err; 287 288 blkcipher_walk_init(&walk, dst, src, nbytes); 289 err = blkcipher_walk_virt(desc, &walk); 290 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 291 292 kernel_fpu_begin(); 293 while ((nbytes = walk.nbytes)) { 294 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 295 nbytes & AES_BLOCK_MASK); 296 nbytes &= AES_BLOCK_SIZE - 1; 297 err = blkcipher_walk_done(desc, &walk, nbytes); 298 } 299 kernel_fpu_end(); 300 301 return err; 302 } 303 304 static int ecb_decrypt(struct blkcipher_desc *desc, 305 struct scatterlist *dst, struct scatterlist *src, 306 unsigned int nbytes) 307 { 308 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 309 struct blkcipher_walk walk; 310 int err; 311 312 blkcipher_walk_init(&walk, dst, src, nbytes); 313 err = blkcipher_walk_virt(desc, &walk); 314 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 315 316 kernel_fpu_begin(); 317 while ((nbytes = walk.nbytes)) { 318 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 319 nbytes & AES_BLOCK_MASK); 320 nbytes &= AES_BLOCK_SIZE - 1; 321 err = blkcipher_walk_done(desc, &walk, nbytes); 322 } 323 kernel_fpu_end(); 324 325 return err; 326 } 327 328 static struct crypto_alg blk_ecb_alg = { 329 .cra_name = "__ecb-aes-aesni", 330 .cra_driver_name = "__driver-ecb-aes-aesni", 331 .cra_priority = 0, 332 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 333 .cra_blocksize = AES_BLOCK_SIZE, 334 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 335 .cra_alignmask = 0, 336 .cra_type = &crypto_blkcipher_type, 337 .cra_module = THIS_MODULE, 338 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), 339 .cra_u = { 340 .blkcipher = { 341 .min_keysize = AES_MIN_KEY_SIZE, 342 .max_keysize = AES_MAX_KEY_SIZE, 343 .setkey = aes_set_key, 344 .encrypt = ecb_encrypt, 345 .decrypt = ecb_decrypt, 346 }, 347 }, 348 }; 349 350 static int cbc_encrypt(struct blkcipher_desc *desc, 351 struct scatterlist *dst, struct scatterlist *src, 352 unsigned int nbytes) 353 { 354 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 355 struct blkcipher_walk walk; 356 int err; 357 358 blkcipher_walk_init(&walk, dst, src, nbytes); 359 err = blkcipher_walk_virt(desc, &walk); 360 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 361 362 kernel_fpu_begin(); 363 while ((nbytes = walk.nbytes)) { 364 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 365 nbytes & AES_BLOCK_MASK, walk.iv); 366 nbytes &= AES_BLOCK_SIZE - 1; 367 err = blkcipher_walk_done(desc, &walk, nbytes); 368 } 369 kernel_fpu_end(); 370 371 return err; 372 } 373 374 static int cbc_decrypt(struct blkcipher_desc *desc, 375 struct scatterlist *dst, struct scatterlist *src, 376 unsigned int nbytes) 377 { 378 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 379 struct blkcipher_walk walk; 380 int err; 381 382 blkcipher_walk_init(&walk, dst, src, nbytes); 383 err = blkcipher_walk_virt(desc, &walk); 384 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 385 386 kernel_fpu_begin(); 387 while ((nbytes = walk.nbytes)) { 388 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 389 nbytes & AES_BLOCK_MASK, walk.iv); 390 nbytes &= AES_BLOCK_SIZE - 1; 391 err = blkcipher_walk_done(desc, &walk, nbytes); 392 } 393 kernel_fpu_end(); 394 395 return err; 396 } 397 398 static struct crypto_alg blk_cbc_alg = { 399 .cra_name = "__cbc-aes-aesni", 400 .cra_driver_name = "__driver-cbc-aes-aesni", 401 .cra_priority = 0, 402 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 403 .cra_blocksize = AES_BLOCK_SIZE, 404 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 405 .cra_alignmask = 0, 406 .cra_type = &crypto_blkcipher_type, 407 .cra_module = THIS_MODULE, 408 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), 409 .cra_u = { 410 .blkcipher = { 411 .min_keysize = AES_MIN_KEY_SIZE, 412 .max_keysize = AES_MAX_KEY_SIZE, 413 .setkey = aes_set_key, 414 .encrypt = cbc_encrypt, 415 .decrypt = cbc_decrypt, 416 }, 417 }, 418 }; 419 420 #ifdef CONFIG_X86_64 421 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 422 struct blkcipher_walk *walk) 423 { 424 u8 *ctrblk = walk->iv; 425 u8 keystream[AES_BLOCK_SIZE]; 426 u8 *src = walk->src.virt.addr; 427 u8 *dst = walk->dst.virt.addr; 428 unsigned int nbytes = walk->nbytes; 429 430 aesni_enc(ctx, keystream, ctrblk); 431 crypto_xor(keystream, src, nbytes); 432 memcpy(dst, keystream, nbytes); 433 crypto_inc(ctrblk, AES_BLOCK_SIZE); 434 } 435 436 static int ctr_crypt(struct blkcipher_desc *desc, 437 struct scatterlist *dst, struct scatterlist *src, 438 unsigned int nbytes) 439 { 440 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 441 struct blkcipher_walk walk; 442 int err; 443 444 blkcipher_walk_init(&walk, dst, src, nbytes); 445 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 446 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 447 448 kernel_fpu_begin(); 449 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 450 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 451 nbytes & AES_BLOCK_MASK, walk.iv); 452 nbytes &= AES_BLOCK_SIZE - 1; 453 err = blkcipher_walk_done(desc, &walk, nbytes); 454 } 455 if (walk.nbytes) { 456 ctr_crypt_final(ctx, &walk); 457 err = blkcipher_walk_done(desc, &walk, 0); 458 } 459 kernel_fpu_end(); 460 461 return err; 462 } 463 464 static struct crypto_alg blk_ctr_alg = { 465 .cra_name = "__ctr-aes-aesni", 466 .cra_driver_name = "__driver-ctr-aes-aesni", 467 .cra_priority = 0, 468 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 469 .cra_blocksize = 1, 470 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 471 .cra_alignmask = 0, 472 .cra_type = &crypto_blkcipher_type, 473 .cra_module = THIS_MODULE, 474 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), 475 .cra_u = { 476 .blkcipher = { 477 .min_keysize = AES_MIN_KEY_SIZE, 478 .max_keysize = AES_MAX_KEY_SIZE, 479 .ivsize = AES_BLOCK_SIZE, 480 .setkey = aes_set_key, 481 .encrypt = ctr_crypt, 482 .decrypt = ctr_crypt, 483 }, 484 }, 485 }; 486 #endif 487 488 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 489 unsigned int key_len) 490 { 491 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 492 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; 493 int err; 494 495 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 496 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) 497 & CRYPTO_TFM_REQ_MASK); 498 err = crypto_ablkcipher_setkey(child, key, key_len); 499 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) 500 & CRYPTO_TFM_RES_MASK); 501 return err; 502 } 503 504 static int ablk_encrypt(struct ablkcipher_request *req) 505 { 506 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 507 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 508 509 if (!irq_fpu_usable()) { 510 struct ablkcipher_request *cryptd_req = 511 ablkcipher_request_ctx(req); 512 memcpy(cryptd_req, req, sizeof(*req)); 513 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 514 return crypto_ablkcipher_encrypt(cryptd_req); 515 } else { 516 struct blkcipher_desc desc; 517 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 518 desc.info = req->info; 519 desc.flags = 0; 520 return crypto_blkcipher_crt(desc.tfm)->encrypt( 521 &desc, req->dst, req->src, req->nbytes); 522 } 523 } 524 525 static int ablk_decrypt(struct ablkcipher_request *req) 526 { 527 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 528 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 529 530 if (!irq_fpu_usable()) { 531 struct ablkcipher_request *cryptd_req = 532 ablkcipher_request_ctx(req); 533 memcpy(cryptd_req, req, sizeof(*req)); 534 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 535 return crypto_ablkcipher_decrypt(cryptd_req); 536 } else { 537 struct blkcipher_desc desc; 538 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 539 desc.info = req->info; 540 desc.flags = 0; 541 return crypto_blkcipher_crt(desc.tfm)->decrypt( 542 &desc, req->dst, req->src, req->nbytes); 543 } 544 } 545 546 static void ablk_exit(struct crypto_tfm *tfm) 547 { 548 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 549 550 cryptd_free_ablkcipher(ctx->cryptd_tfm); 551 } 552 553 static void ablk_init_common(struct crypto_tfm *tfm, 554 struct cryptd_ablkcipher *cryptd_tfm) 555 { 556 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 557 558 ctx->cryptd_tfm = cryptd_tfm; 559 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 560 crypto_ablkcipher_reqsize(&cryptd_tfm->base); 561 } 562 563 static int ablk_ecb_init(struct crypto_tfm *tfm) 564 { 565 struct cryptd_ablkcipher *cryptd_tfm; 566 567 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); 568 if (IS_ERR(cryptd_tfm)) 569 return PTR_ERR(cryptd_tfm); 570 ablk_init_common(tfm, cryptd_tfm); 571 return 0; 572 } 573 574 static struct crypto_alg ablk_ecb_alg = { 575 .cra_name = "ecb(aes)", 576 .cra_driver_name = "ecb-aes-aesni", 577 .cra_priority = 400, 578 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 579 .cra_blocksize = AES_BLOCK_SIZE, 580 .cra_ctxsize = sizeof(struct async_aes_ctx), 581 .cra_alignmask = 0, 582 .cra_type = &crypto_ablkcipher_type, 583 .cra_module = THIS_MODULE, 584 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), 585 .cra_init = ablk_ecb_init, 586 .cra_exit = ablk_exit, 587 .cra_u = { 588 .ablkcipher = { 589 .min_keysize = AES_MIN_KEY_SIZE, 590 .max_keysize = AES_MAX_KEY_SIZE, 591 .setkey = ablk_set_key, 592 .encrypt = ablk_encrypt, 593 .decrypt = ablk_decrypt, 594 }, 595 }, 596 }; 597 598 static int ablk_cbc_init(struct crypto_tfm *tfm) 599 { 600 struct cryptd_ablkcipher *cryptd_tfm; 601 602 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); 603 if (IS_ERR(cryptd_tfm)) 604 return PTR_ERR(cryptd_tfm); 605 ablk_init_common(tfm, cryptd_tfm); 606 return 0; 607 } 608 609 static struct crypto_alg ablk_cbc_alg = { 610 .cra_name = "cbc(aes)", 611 .cra_driver_name = "cbc-aes-aesni", 612 .cra_priority = 400, 613 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 614 .cra_blocksize = AES_BLOCK_SIZE, 615 .cra_ctxsize = sizeof(struct async_aes_ctx), 616 .cra_alignmask = 0, 617 .cra_type = &crypto_ablkcipher_type, 618 .cra_module = THIS_MODULE, 619 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), 620 .cra_init = ablk_cbc_init, 621 .cra_exit = ablk_exit, 622 .cra_u = { 623 .ablkcipher = { 624 .min_keysize = AES_MIN_KEY_SIZE, 625 .max_keysize = AES_MAX_KEY_SIZE, 626 .ivsize = AES_BLOCK_SIZE, 627 .setkey = ablk_set_key, 628 .encrypt = ablk_encrypt, 629 .decrypt = ablk_decrypt, 630 }, 631 }, 632 }; 633 634 #ifdef CONFIG_X86_64 635 static int ablk_ctr_init(struct crypto_tfm *tfm) 636 { 637 struct cryptd_ablkcipher *cryptd_tfm; 638 639 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0); 640 if (IS_ERR(cryptd_tfm)) 641 return PTR_ERR(cryptd_tfm); 642 ablk_init_common(tfm, cryptd_tfm); 643 return 0; 644 } 645 646 static struct crypto_alg ablk_ctr_alg = { 647 .cra_name = "ctr(aes)", 648 .cra_driver_name = "ctr-aes-aesni", 649 .cra_priority = 400, 650 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 651 .cra_blocksize = 1, 652 .cra_ctxsize = sizeof(struct async_aes_ctx), 653 .cra_alignmask = 0, 654 .cra_type = &crypto_ablkcipher_type, 655 .cra_module = THIS_MODULE, 656 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list), 657 .cra_init = ablk_ctr_init, 658 .cra_exit = ablk_exit, 659 .cra_u = { 660 .ablkcipher = { 661 .min_keysize = AES_MIN_KEY_SIZE, 662 .max_keysize = AES_MAX_KEY_SIZE, 663 .ivsize = AES_BLOCK_SIZE, 664 .setkey = ablk_set_key, 665 .encrypt = ablk_encrypt, 666 .decrypt = ablk_encrypt, 667 .geniv = "chainiv", 668 }, 669 }, 670 }; 671 672 #ifdef HAS_CTR 673 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) 674 { 675 struct cryptd_ablkcipher *cryptd_tfm; 676 677 cryptd_tfm = cryptd_alloc_ablkcipher( 678 "rfc3686(__driver-ctr-aes-aesni)", 0, 0); 679 if (IS_ERR(cryptd_tfm)) 680 return PTR_ERR(cryptd_tfm); 681 ablk_init_common(tfm, cryptd_tfm); 682 return 0; 683 } 684 685 static struct crypto_alg ablk_rfc3686_ctr_alg = { 686 .cra_name = "rfc3686(ctr(aes))", 687 .cra_driver_name = "rfc3686-ctr-aes-aesni", 688 .cra_priority = 400, 689 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 690 .cra_blocksize = 1, 691 .cra_ctxsize = sizeof(struct async_aes_ctx), 692 .cra_alignmask = 0, 693 .cra_type = &crypto_ablkcipher_type, 694 .cra_module = THIS_MODULE, 695 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list), 696 .cra_init = ablk_rfc3686_ctr_init, 697 .cra_exit = ablk_exit, 698 .cra_u = { 699 .ablkcipher = { 700 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 701 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 702 .ivsize = CTR_RFC3686_IV_SIZE, 703 .setkey = ablk_set_key, 704 .encrypt = ablk_encrypt, 705 .decrypt = ablk_decrypt, 706 .geniv = "seqiv", 707 }, 708 }, 709 }; 710 #endif 711 #endif 712 713 #ifdef HAS_LRW 714 static int ablk_lrw_init(struct crypto_tfm *tfm) 715 { 716 struct cryptd_ablkcipher *cryptd_tfm; 717 718 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))", 719 0, 0); 720 if (IS_ERR(cryptd_tfm)) 721 return PTR_ERR(cryptd_tfm); 722 ablk_init_common(tfm, cryptd_tfm); 723 return 0; 724 } 725 726 static struct crypto_alg ablk_lrw_alg = { 727 .cra_name = "lrw(aes)", 728 .cra_driver_name = "lrw-aes-aesni", 729 .cra_priority = 400, 730 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 731 .cra_blocksize = AES_BLOCK_SIZE, 732 .cra_ctxsize = sizeof(struct async_aes_ctx), 733 .cra_alignmask = 0, 734 .cra_type = &crypto_ablkcipher_type, 735 .cra_module = THIS_MODULE, 736 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), 737 .cra_init = ablk_lrw_init, 738 .cra_exit = ablk_exit, 739 .cra_u = { 740 .ablkcipher = { 741 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 742 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 743 .ivsize = AES_BLOCK_SIZE, 744 .setkey = ablk_set_key, 745 .encrypt = ablk_encrypt, 746 .decrypt = ablk_decrypt, 747 }, 748 }, 749 }; 750 #endif 751 752 #ifdef HAS_PCBC 753 static int ablk_pcbc_init(struct crypto_tfm *tfm) 754 { 755 struct cryptd_ablkcipher *cryptd_tfm; 756 757 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))", 758 0, 0); 759 if (IS_ERR(cryptd_tfm)) 760 return PTR_ERR(cryptd_tfm); 761 ablk_init_common(tfm, cryptd_tfm); 762 return 0; 763 } 764 765 static struct crypto_alg ablk_pcbc_alg = { 766 .cra_name = "pcbc(aes)", 767 .cra_driver_name = "pcbc-aes-aesni", 768 .cra_priority = 400, 769 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 770 .cra_blocksize = AES_BLOCK_SIZE, 771 .cra_ctxsize = sizeof(struct async_aes_ctx), 772 .cra_alignmask = 0, 773 .cra_type = &crypto_ablkcipher_type, 774 .cra_module = THIS_MODULE, 775 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list), 776 .cra_init = ablk_pcbc_init, 777 .cra_exit = ablk_exit, 778 .cra_u = { 779 .ablkcipher = { 780 .min_keysize = AES_MIN_KEY_SIZE, 781 .max_keysize = AES_MAX_KEY_SIZE, 782 .ivsize = AES_BLOCK_SIZE, 783 .setkey = ablk_set_key, 784 .encrypt = ablk_encrypt, 785 .decrypt = ablk_decrypt, 786 }, 787 }, 788 }; 789 #endif 790 791 #ifdef HAS_XTS 792 static int ablk_xts_init(struct crypto_tfm *tfm) 793 { 794 struct cryptd_ablkcipher *cryptd_tfm; 795 796 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))", 797 0, 0); 798 if (IS_ERR(cryptd_tfm)) 799 return PTR_ERR(cryptd_tfm); 800 ablk_init_common(tfm, cryptd_tfm); 801 return 0; 802 } 803 804 static struct crypto_alg ablk_xts_alg = { 805 .cra_name = "xts(aes)", 806 .cra_driver_name = "xts-aes-aesni", 807 .cra_priority = 400, 808 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 809 .cra_blocksize = AES_BLOCK_SIZE, 810 .cra_ctxsize = sizeof(struct async_aes_ctx), 811 .cra_alignmask = 0, 812 .cra_type = &crypto_ablkcipher_type, 813 .cra_module = THIS_MODULE, 814 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list), 815 .cra_init = ablk_xts_init, 816 .cra_exit = ablk_exit, 817 .cra_u = { 818 .ablkcipher = { 819 .min_keysize = 2 * AES_MIN_KEY_SIZE, 820 .max_keysize = 2 * AES_MAX_KEY_SIZE, 821 .ivsize = AES_BLOCK_SIZE, 822 .setkey = ablk_set_key, 823 .encrypt = ablk_encrypt, 824 .decrypt = ablk_decrypt, 825 }, 826 }, 827 }; 828 #endif 829 830 #ifdef CONFIG_X86_64 831 static int rfc4106_init(struct crypto_tfm *tfm) 832 { 833 struct cryptd_aead *cryptd_tfm; 834 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 835 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 836 struct crypto_aead *cryptd_child; 837 struct aesni_rfc4106_gcm_ctx *child_ctx; 838 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 839 if (IS_ERR(cryptd_tfm)) 840 return PTR_ERR(cryptd_tfm); 841 842 cryptd_child = cryptd_aead_child(cryptd_tfm); 843 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 844 memcpy(child_ctx, ctx, sizeof(*ctx)); 845 ctx->cryptd_tfm = cryptd_tfm; 846 tfm->crt_aead.reqsize = sizeof(struct aead_request) 847 + crypto_aead_reqsize(&cryptd_tfm->base); 848 return 0; 849 } 850 851 static void rfc4106_exit(struct crypto_tfm *tfm) 852 { 853 struct aesni_rfc4106_gcm_ctx *ctx = 854 (struct aesni_rfc4106_gcm_ctx *) 855 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 856 if (!IS_ERR(ctx->cryptd_tfm)) 857 cryptd_free_aead(ctx->cryptd_tfm); 858 return; 859 } 860 861 static void 862 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 863 { 864 struct aesni_gcm_set_hash_subkey_result *result = req->data; 865 866 if (err == -EINPROGRESS) 867 return; 868 result->err = err; 869 complete(&result->completion); 870 } 871 872 static int 873 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 874 { 875 struct crypto_ablkcipher *ctr_tfm; 876 struct ablkcipher_request *req; 877 int ret = -EINVAL; 878 struct aesni_hash_subkey_req_data *req_data; 879 880 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 881 if (IS_ERR(ctr_tfm)) 882 return PTR_ERR(ctr_tfm); 883 884 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 885 886 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 887 if (ret) 888 goto out_free_ablkcipher; 889 890 ret = -ENOMEM; 891 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 892 if (!req) 893 goto out_free_ablkcipher; 894 895 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 896 if (!req_data) 897 goto out_free_request; 898 899 memset(req_data->iv, 0, sizeof(req_data->iv)); 900 901 /* Clear the data in the hash sub key container to zero.*/ 902 /* We want to cipher all zeros to create the hash sub key. */ 903 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 904 905 init_completion(&req_data->result.completion); 906 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 907 ablkcipher_request_set_tfm(req, ctr_tfm); 908 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 909 CRYPTO_TFM_REQ_MAY_BACKLOG, 910 rfc4106_set_hash_subkey_done, 911 &req_data->result); 912 913 ablkcipher_request_set_crypt(req, &req_data->sg, 914 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 915 916 ret = crypto_ablkcipher_encrypt(req); 917 if (ret == -EINPROGRESS || ret == -EBUSY) { 918 ret = wait_for_completion_interruptible 919 (&req_data->result.completion); 920 if (!ret) 921 ret = req_data->result.err; 922 } 923 kfree(req_data); 924 out_free_request: 925 ablkcipher_request_free(req); 926 out_free_ablkcipher: 927 crypto_free_ablkcipher(ctr_tfm); 928 return ret; 929 } 930 931 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 932 unsigned int key_len) 933 { 934 int ret = 0; 935 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 936 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 937 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 938 struct aesni_rfc4106_gcm_ctx *child_ctx = 939 aesni_rfc4106_gcm_ctx_get(cryptd_child); 940 u8 *new_key_mem = NULL; 941 942 if (key_len < 4) { 943 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 944 return -EINVAL; 945 } 946 /*Account for 4 byte nonce at the end.*/ 947 key_len -= 4; 948 if (key_len != AES_KEYSIZE_128) { 949 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 950 return -EINVAL; 951 } 952 953 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 954 /*This must be on a 16 byte boundary!*/ 955 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 956 return -EINVAL; 957 958 if ((unsigned long)key % AESNI_ALIGN) { 959 /*key is not aligned: use an auxuliar aligned pointer*/ 960 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 961 if (!new_key_mem) 962 return -ENOMEM; 963 964 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 965 memcpy(new_key_mem, key, key_len); 966 key = new_key_mem; 967 } 968 969 if (!irq_fpu_usable()) 970 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 971 key, key_len); 972 else { 973 kernel_fpu_begin(); 974 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 975 kernel_fpu_end(); 976 } 977 /*This must be on a 16 byte boundary!*/ 978 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 979 ret = -EINVAL; 980 goto exit; 981 } 982 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 983 memcpy(child_ctx, ctx, sizeof(*ctx)); 984 exit: 985 kfree(new_key_mem); 986 return ret; 987 } 988 989 /* This is the Integrity Check Value (aka the authentication tag length and can 990 * be 8, 12 or 16 bytes long. */ 991 static int rfc4106_set_authsize(struct crypto_aead *parent, 992 unsigned int authsize) 993 { 994 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 995 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 996 997 switch (authsize) { 998 case 8: 999 case 12: 1000 case 16: 1001 break; 1002 default: 1003 return -EINVAL; 1004 } 1005 crypto_aead_crt(parent)->authsize = authsize; 1006 crypto_aead_crt(cryptd_child)->authsize = authsize; 1007 return 0; 1008 } 1009 1010 static int rfc4106_encrypt(struct aead_request *req) 1011 { 1012 int ret; 1013 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1014 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1015 1016 if (!irq_fpu_usable()) { 1017 struct aead_request *cryptd_req = 1018 (struct aead_request *) aead_request_ctx(req); 1019 memcpy(cryptd_req, req, sizeof(*req)); 1020 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1021 return crypto_aead_encrypt(cryptd_req); 1022 } else { 1023 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1024 kernel_fpu_begin(); 1025 ret = cryptd_child->base.crt_aead.encrypt(req); 1026 kernel_fpu_end(); 1027 return ret; 1028 } 1029 } 1030 1031 static int rfc4106_decrypt(struct aead_request *req) 1032 { 1033 int ret; 1034 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1035 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1036 1037 if (!irq_fpu_usable()) { 1038 struct aead_request *cryptd_req = 1039 (struct aead_request *) aead_request_ctx(req); 1040 memcpy(cryptd_req, req, sizeof(*req)); 1041 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1042 return crypto_aead_decrypt(cryptd_req); 1043 } else { 1044 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1045 kernel_fpu_begin(); 1046 ret = cryptd_child->base.crt_aead.decrypt(req); 1047 kernel_fpu_end(); 1048 return ret; 1049 } 1050 } 1051 1052 static struct crypto_alg rfc4106_alg = { 1053 .cra_name = "rfc4106(gcm(aes))", 1054 .cra_driver_name = "rfc4106-gcm-aesni", 1055 .cra_priority = 400, 1056 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1057 .cra_blocksize = 1, 1058 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1059 .cra_alignmask = 0, 1060 .cra_type = &crypto_nivaead_type, 1061 .cra_module = THIS_MODULE, 1062 .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list), 1063 .cra_init = rfc4106_init, 1064 .cra_exit = rfc4106_exit, 1065 .cra_u = { 1066 .aead = { 1067 .setkey = rfc4106_set_key, 1068 .setauthsize = rfc4106_set_authsize, 1069 .encrypt = rfc4106_encrypt, 1070 .decrypt = rfc4106_decrypt, 1071 .geniv = "seqiv", 1072 .ivsize = 8, 1073 .maxauthsize = 16, 1074 }, 1075 }, 1076 }; 1077 1078 static int __driver_rfc4106_encrypt(struct aead_request *req) 1079 { 1080 u8 one_entry_in_sg = 0; 1081 u8 *src, *dst, *assoc; 1082 __be32 counter = cpu_to_be32(1); 1083 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1084 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1085 void *aes_ctx = &(ctx->aes_key_expanded); 1086 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1087 u8 iv_tab[16+AESNI_ALIGN]; 1088 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 1089 struct scatter_walk src_sg_walk; 1090 struct scatter_walk assoc_sg_walk; 1091 struct scatter_walk dst_sg_walk; 1092 unsigned int i; 1093 1094 /* Assuming we are supporting rfc4106 64-bit extended */ 1095 /* sequence numbers We need to have the AAD length equal */ 1096 /* to 8 or 12 bytes */ 1097 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 1098 return -EINVAL; 1099 /* IV below built */ 1100 for (i = 0; i < 4; i++) 1101 *(iv+i) = ctx->nonce[i]; 1102 for (i = 0; i < 8; i++) 1103 *(iv+4+i) = req->iv[i]; 1104 *((__be32 *)(iv+12)) = counter; 1105 1106 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1107 one_entry_in_sg = 1; 1108 scatterwalk_start(&src_sg_walk, req->src); 1109 scatterwalk_start(&assoc_sg_walk, req->assoc); 1110 src = scatterwalk_map(&src_sg_walk, 0); 1111 assoc = scatterwalk_map(&assoc_sg_walk, 0); 1112 dst = src; 1113 if (unlikely(req->src != req->dst)) { 1114 scatterwalk_start(&dst_sg_walk, req->dst); 1115 dst = scatterwalk_map(&dst_sg_walk, 0); 1116 } 1117 1118 } else { 1119 /* Allocate memory for src, dst, assoc */ 1120 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 1121 GFP_ATOMIC); 1122 if (unlikely(!src)) 1123 return -ENOMEM; 1124 assoc = (src + req->cryptlen + auth_tag_len); 1125 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1126 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1127 req->assoclen, 0); 1128 dst = src; 1129 } 1130 1131 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 1132 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 1133 + ((unsigned long)req->cryptlen), auth_tag_len); 1134 1135 /* The authTag (aka the Integrity Check Value) needs to be written 1136 * back to the packet. */ 1137 if (one_entry_in_sg) { 1138 if (unlikely(req->src != req->dst)) { 1139 scatterwalk_unmap(dst, 0); 1140 scatterwalk_done(&dst_sg_walk, 0, 0); 1141 } 1142 scatterwalk_unmap(src, 0); 1143 scatterwalk_unmap(assoc, 0); 1144 scatterwalk_done(&src_sg_walk, 0, 0); 1145 scatterwalk_done(&assoc_sg_walk, 0, 0); 1146 } else { 1147 scatterwalk_map_and_copy(dst, req->dst, 0, 1148 req->cryptlen + auth_tag_len, 1); 1149 kfree(src); 1150 } 1151 return 0; 1152 } 1153 1154 static int __driver_rfc4106_decrypt(struct aead_request *req) 1155 { 1156 u8 one_entry_in_sg = 0; 1157 u8 *src, *dst, *assoc; 1158 unsigned long tempCipherLen = 0; 1159 __be32 counter = cpu_to_be32(1); 1160 int retval = 0; 1161 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1162 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1163 void *aes_ctx = &(ctx->aes_key_expanded); 1164 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1165 u8 iv_and_authTag[32+AESNI_ALIGN]; 1166 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 1167 u8 *authTag = iv + 16; 1168 struct scatter_walk src_sg_walk; 1169 struct scatter_walk assoc_sg_walk; 1170 struct scatter_walk dst_sg_walk; 1171 unsigned int i; 1172 1173 if (unlikely((req->cryptlen < auth_tag_len) || 1174 (req->assoclen != 8 && req->assoclen != 12))) 1175 return -EINVAL; 1176 /* Assuming we are supporting rfc4106 64-bit extended */ 1177 /* sequence numbers We need to have the AAD length */ 1178 /* equal to 8 or 12 bytes */ 1179 1180 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 1181 /* IV below built */ 1182 for (i = 0; i < 4; i++) 1183 *(iv+i) = ctx->nonce[i]; 1184 for (i = 0; i < 8; i++) 1185 *(iv+4+i) = req->iv[i]; 1186 *((__be32 *)(iv+12)) = counter; 1187 1188 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1189 one_entry_in_sg = 1; 1190 scatterwalk_start(&src_sg_walk, req->src); 1191 scatterwalk_start(&assoc_sg_walk, req->assoc); 1192 src = scatterwalk_map(&src_sg_walk, 0); 1193 assoc = scatterwalk_map(&assoc_sg_walk, 0); 1194 dst = src; 1195 if (unlikely(req->src != req->dst)) { 1196 scatterwalk_start(&dst_sg_walk, req->dst); 1197 dst = scatterwalk_map(&dst_sg_walk, 0); 1198 } 1199 1200 } else { 1201 /* Allocate memory for src, dst, assoc */ 1202 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 1203 if (!src) 1204 return -ENOMEM; 1205 assoc = (src + req->cryptlen + auth_tag_len); 1206 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1207 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1208 req->assoclen, 0); 1209 dst = src; 1210 } 1211 1212 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, 1213 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 1214 authTag, auth_tag_len); 1215 1216 /* Compare generated tag with passed in tag. */ 1217 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? 1218 -EBADMSG : 0; 1219 1220 if (one_entry_in_sg) { 1221 if (unlikely(req->src != req->dst)) { 1222 scatterwalk_unmap(dst, 0); 1223 scatterwalk_done(&dst_sg_walk, 0, 0); 1224 } 1225 scatterwalk_unmap(src, 0); 1226 scatterwalk_unmap(assoc, 0); 1227 scatterwalk_done(&src_sg_walk, 0, 0); 1228 scatterwalk_done(&assoc_sg_walk, 0, 0); 1229 } else { 1230 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 1231 kfree(src); 1232 } 1233 return retval; 1234 } 1235 1236 static struct crypto_alg __rfc4106_alg = { 1237 .cra_name = "__gcm-aes-aesni", 1238 .cra_driver_name = "__driver-gcm-aes-aesni", 1239 .cra_priority = 0, 1240 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1241 .cra_blocksize = 1, 1242 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1243 .cra_alignmask = 0, 1244 .cra_type = &crypto_aead_type, 1245 .cra_module = THIS_MODULE, 1246 .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list), 1247 .cra_u = { 1248 .aead = { 1249 .encrypt = __driver_rfc4106_encrypt, 1250 .decrypt = __driver_rfc4106_decrypt, 1251 }, 1252 }, 1253 }; 1254 #endif 1255 1256 static int __init aesni_init(void) 1257 { 1258 int err; 1259 1260 if (!cpu_has_aes) { 1261 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); 1262 return -ENODEV; 1263 } 1264 1265 if ((err = crypto_fpu_init())) 1266 goto fpu_err; 1267 if ((err = crypto_register_alg(&aesni_alg))) 1268 goto aes_err; 1269 if ((err = crypto_register_alg(&__aesni_alg))) 1270 goto __aes_err; 1271 if ((err = crypto_register_alg(&blk_ecb_alg))) 1272 goto blk_ecb_err; 1273 if ((err = crypto_register_alg(&blk_cbc_alg))) 1274 goto blk_cbc_err; 1275 if ((err = crypto_register_alg(&ablk_ecb_alg))) 1276 goto ablk_ecb_err; 1277 if ((err = crypto_register_alg(&ablk_cbc_alg))) 1278 goto ablk_cbc_err; 1279 #ifdef CONFIG_X86_64 1280 if ((err = crypto_register_alg(&blk_ctr_alg))) 1281 goto blk_ctr_err; 1282 if ((err = crypto_register_alg(&ablk_ctr_alg))) 1283 goto ablk_ctr_err; 1284 if ((err = crypto_register_alg(&__rfc4106_alg))) 1285 goto __aead_gcm_err; 1286 if ((err = crypto_register_alg(&rfc4106_alg))) 1287 goto aead_gcm_err; 1288 #ifdef HAS_CTR 1289 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) 1290 goto ablk_rfc3686_ctr_err; 1291 #endif 1292 #endif 1293 #ifdef HAS_LRW 1294 if ((err = crypto_register_alg(&ablk_lrw_alg))) 1295 goto ablk_lrw_err; 1296 #endif 1297 #ifdef HAS_PCBC 1298 if ((err = crypto_register_alg(&ablk_pcbc_alg))) 1299 goto ablk_pcbc_err; 1300 #endif 1301 #ifdef HAS_XTS 1302 if ((err = crypto_register_alg(&ablk_xts_alg))) 1303 goto ablk_xts_err; 1304 #endif 1305 return err; 1306 1307 #ifdef HAS_XTS 1308 ablk_xts_err: 1309 #endif 1310 #ifdef HAS_PCBC 1311 crypto_unregister_alg(&ablk_pcbc_alg); 1312 ablk_pcbc_err: 1313 #endif 1314 #ifdef HAS_LRW 1315 crypto_unregister_alg(&ablk_lrw_alg); 1316 ablk_lrw_err: 1317 #endif 1318 #ifdef CONFIG_X86_64 1319 #ifdef HAS_CTR 1320 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1321 ablk_rfc3686_ctr_err: 1322 #endif 1323 crypto_unregister_alg(&rfc4106_alg); 1324 aead_gcm_err: 1325 crypto_unregister_alg(&__rfc4106_alg); 1326 __aead_gcm_err: 1327 crypto_unregister_alg(&ablk_ctr_alg); 1328 ablk_ctr_err: 1329 crypto_unregister_alg(&blk_ctr_alg); 1330 blk_ctr_err: 1331 #endif 1332 crypto_unregister_alg(&ablk_cbc_alg); 1333 ablk_cbc_err: 1334 crypto_unregister_alg(&ablk_ecb_alg); 1335 ablk_ecb_err: 1336 crypto_unregister_alg(&blk_cbc_alg); 1337 blk_cbc_err: 1338 crypto_unregister_alg(&blk_ecb_alg); 1339 blk_ecb_err: 1340 crypto_unregister_alg(&__aesni_alg); 1341 __aes_err: 1342 crypto_unregister_alg(&aesni_alg); 1343 aes_err: 1344 fpu_err: 1345 return err; 1346 } 1347 1348 static void __exit aesni_exit(void) 1349 { 1350 #ifdef HAS_XTS 1351 crypto_unregister_alg(&ablk_xts_alg); 1352 #endif 1353 #ifdef HAS_PCBC 1354 crypto_unregister_alg(&ablk_pcbc_alg); 1355 #endif 1356 #ifdef HAS_LRW 1357 crypto_unregister_alg(&ablk_lrw_alg); 1358 #endif 1359 #ifdef CONFIG_X86_64 1360 #ifdef HAS_CTR 1361 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1362 #endif 1363 crypto_unregister_alg(&rfc4106_alg); 1364 crypto_unregister_alg(&__rfc4106_alg); 1365 crypto_unregister_alg(&ablk_ctr_alg); 1366 crypto_unregister_alg(&blk_ctr_alg); 1367 #endif 1368 crypto_unregister_alg(&ablk_cbc_alg); 1369 crypto_unregister_alg(&ablk_ecb_alg); 1370 crypto_unregister_alg(&blk_cbc_alg); 1371 crypto_unregister_alg(&blk_ecb_alg); 1372 crypto_unregister_alg(&__aesni_alg); 1373 crypto_unregister_alg(&aesni_alg); 1374 1375 crypto_fpu_exit(); 1376 } 1377 1378 module_init(aesni_init); 1379 module_exit(aesni_exit); 1380 1381 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1382 MODULE_LICENSE("GPL"); 1383 MODULE_ALIAS("aes"); 1384