1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 */ 13 14 #include <linux/hardirq.h> 15 #include <linux/types.h> 16 #include <linux/crypto.h> 17 #include <linux/err.h> 18 #include <crypto/algapi.h> 19 #include <crypto/aes.h> 20 #include <crypto/cryptd.h> 21 #include <asm/i387.h> 22 #include <asm/aes.h> 23 24 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 25 #define HAS_CTR 26 #endif 27 28 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) 29 #define HAS_LRW 30 #endif 31 32 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 33 #define HAS_PCBC 34 #endif 35 36 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE) 37 #define HAS_XTS 38 #endif 39 40 struct async_aes_ctx { 41 struct cryptd_ablkcipher *cryptd_tfm; 42 }; 43 44 #define AESNI_ALIGN 16 45 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 46 47 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 48 unsigned int key_len); 49 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 50 const u8 *in); 51 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 52 const u8 *in); 53 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 54 const u8 *in, unsigned int len); 55 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 56 const u8 *in, unsigned int len); 57 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 58 const u8 *in, unsigned int len, u8 *iv); 59 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 60 const u8 *in, unsigned int len, u8 *iv); 61 62 static inline int kernel_fpu_using(void) 63 { 64 if (in_interrupt() && !(read_cr0() & X86_CR0_TS)) 65 return 1; 66 return 0; 67 } 68 69 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 70 { 71 unsigned long addr = (unsigned long)raw_ctx; 72 unsigned long align = AESNI_ALIGN; 73 74 if (align <= crypto_tfm_ctx_alignment()) 75 align = 1; 76 return (struct crypto_aes_ctx *)ALIGN(addr, align); 77 } 78 79 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 80 const u8 *in_key, unsigned int key_len) 81 { 82 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 83 u32 *flags = &tfm->crt_flags; 84 int err; 85 86 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 87 key_len != AES_KEYSIZE_256) { 88 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 89 return -EINVAL; 90 } 91 92 if (kernel_fpu_using()) 93 err = crypto_aes_expand_key(ctx, in_key, key_len); 94 else { 95 kernel_fpu_begin(); 96 err = aesni_set_key(ctx, in_key, key_len); 97 kernel_fpu_end(); 98 } 99 100 return err; 101 } 102 103 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 104 unsigned int key_len) 105 { 106 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 107 } 108 109 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 110 { 111 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 112 113 if (kernel_fpu_using()) 114 crypto_aes_encrypt_x86(ctx, dst, src); 115 else { 116 kernel_fpu_begin(); 117 aesni_enc(ctx, dst, src); 118 kernel_fpu_end(); 119 } 120 } 121 122 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 123 { 124 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 125 126 if (kernel_fpu_using()) 127 crypto_aes_decrypt_x86(ctx, dst, src); 128 else { 129 kernel_fpu_begin(); 130 aesni_dec(ctx, dst, src); 131 kernel_fpu_end(); 132 } 133 } 134 135 static struct crypto_alg aesni_alg = { 136 .cra_name = "aes", 137 .cra_driver_name = "aes-aesni", 138 .cra_priority = 300, 139 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 140 .cra_blocksize = AES_BLOCK_SIZE, 141 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 142 .cra_alignmask = 0, 143 .cra_module = THIS_MODULE, 144 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list), 145 .cra_u = { 146 .cipher = { 147 .cia_min_keysize = AES_MIN_KEY_SIZE, 148 .cia_max_keysize = AES_MAX_KEY_SIZE, 149 .cia_setkey = aes_set_key, 150 .cia_encrypt = aes_encrypt, 151 .cia_decrypt = aes_decrypt 152 } 153 } 154 }; 155 156 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 157 { 158 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 159 160 aesni_enc(ctx, dst, src); 161 } 162 163 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 164 { 165 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 166 167 aesni_dec(ctx, dst, src); 168 } 169 170 static struct crypto_alg __aesni_alg = { 171 .cra_name = "__aes-aesni", 172 .cra_driver_name = "__driver-aes-aesni", 173 .cra_priority = 0, 174 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 175 .cra_blocksize = AES_BLOCK_SIZE, 176 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 177 .cra_alignmask = 0, 178 .cra_module = THIS_MODULE, 179 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list), 180 .cra_u = { 181 .cipher = { 182 .cia_min_keysize = AES_MIN_KEY_SIZE, 183 .cia_max_keysize = AES_MAX_KEY_SIZE, 184 .cia_setkey = aes_set_key, 185 .cia_encrypt = __aes_encrypt, 186 .cia_decrypt = __aes_decrypt 187 } 188 } 189 }; 190 191 static int ecb_encrypt(struct blkcipher_desc *desc, 192 struct scatterlist *dst, struct scatterlist *src, 193 unsigned int nbytes) 194 { 195 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 196 struct blkcipher_walk walk; 197 int err; 198 199 blkcipher_walk_init(&walk, dst, src, nbytes); 200 err = blkcipher_walk_virt(desc, &walk); 201 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 202 203 kernel_fpu_begin(); 204 while ((nbytes = walk.nbytes)) { 205 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 206 nbytes & AES_BLOCK_MASK); 207 nbytes &= AES_BLOCK_SIZE - 1; 208 err = blkcipher_walk_done(desc, &walk, nbytes); 209 } 210 kernel_fpu_end(); 211 212 return err; 213 } 214 215 static int ecb_decrypt(struct blkcipher_desc *desc, 216 struct scatterlist *dst, struct scatterlist *src, 217 unsigned int nbytes) 218 { 219 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 220 struct blkcipher_walk walk; 221 int err; 222 223 blkcipher_walk_init(&walk, dst, src, nbytes); 224 err = blkcipher_walk_virt(desc, &walk); 225 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 226 227 kernel_fpu_begin(); 228 while ((nbytes = walk.nbytes)) { 229 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 230 nbytes & AES_BLOCK_MASK); 231 nbytes &= AES_BLOCK_SIZE - 1; 232 err = blkcipher_walk_done(desc, &walk, nbytes); 233 } 234 kernel_fpu_end(); 235 236 return err; 237 } 238 239 static struct crypto_alg blk_ecb_alg = { 240 .cra_name = "__ecb-aes-aesni", 241 .cra_driver_name = "__driver-ecb-aes-aesni", 242 .cra_priority = 0, 243 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 244 .cra_blocksize = AES_BLOCK_SIZE, 245 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 246 .cra_alignmask = 0, 247 .cra_type = &crypto_blkcipher_type, 248 .cra_module = THIS_MODULE, 249 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), 250 .cra_u = { 251 .blkcipher = { 252 .min_keysize = AES_MIN_KEY_SIZE, 253 .max_keysize = AES_MAX_KEY_SIZE, 254 .setkey = aes_set_key, 255 .encrypt = ecb_encrypt, 256 .decrypt = ecb_decrypt, 257 }, 258 }, 259 }; 260 261 static int cbc_encrypt(struct blkcipher_desc *desc, 262 struct scatterlist *dst, struct scatterlist *src, 263 unsigned int nbytes) 264 { 265 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 266 struct blkcipher_walk walk; 267 int err; 268 269 blkcipher_walk_init(&walk, dst, src, nbytes); 270 err = blkcipher_walk_virt(desc, &walk); 271 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 272 273 kernel_fpu_begin(); 274 while ((nbytes = walk.nbytes)) { 275 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 276 nbytes & AES_BLOCK_MASK, walk.iv); 277 nbytes &= AES_BLOCK_SIZE - 1; 278 err = blkcipher_walk_done(desc, &walk, nbytes); 279 } 280 kernel_fpu_end(); 281 282 return err; 283 } 284 285 static int cbc_decrypt(struct blkcipher_desc *desc, 286 struct scatterlist *dst, struct scatterlist *src, 287 unsigned int nbytes) 288 { 289 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 290 struct blkcipher_walk walk; 291 int err; 292 293 blkcipher_walk_init(&walk, dst, src, nbytes); 294 err = blkcipher_walk_virt(desc, &walk); 295 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 296 297 kernel_fpu_begin(); 298 while ((nbytes = walk.nbytes)) { 299 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 300 nbytes & AES_BLOCK_MASK, walk.iv); 301 nbytes &= AES_BLOCK_SIZE - 1; 302 err = blkcipher_walk_done(desc, &walk, nbytes); 303 } 304 kernel_fpu_end(); 305 306 return err; 307 } 308 309 static struct crypto_alg blk_cbc_alg = { 310 .cra_name = "__cbc-aes-aesni", 311 .cra_driver_name = "__driver-cbc-aes-aesni", 312 .cra_priority = 0, 313 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 314 .cra_blocksize = AES_BLOCK_SIZE, 315 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 316 .cra_alignmask = 0, 317 .cra_type = &crypto_blkcipher_type, 318 .cra_module = THIS_MODULE, 319 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), 320 .cra_u = { 321 .blkcipher = { 322 .min_keysize = AES_MIN_KEY_SIZE, 323 .max_keysize = AES_MAX_KEY_SIZE, 324 .setkey = aes_set_key, 325 .encrypt = cbc_encrypt, 326 .decrypt = cbc_decrypt, 327 }, 328 }, 329 }; 330 331 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 332 unsigned int key_len) 333 { 334 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 335 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; 336 int err; 337 338 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 339 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) 340 & CRYPTO_TFM_REQ_MASK); 341 err = crypto_ablkcipher_setkey(child, key, key_len); 342 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) 343 & CRYPTO_TFM_RES_MASK); 344 return err; 345 } 346 347 static int ablk_encrypt(struct ablkcipher_request *req) 348 { 349 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 350 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 351 352 if (kernel_fpu_using()) { 353 struct ablkcipher_request *cryptd_req = 354 ablkcipher_request_ctx(req); 355 memcpy(cryptd_req, req, sizeof(*req)); 356 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 357 return crypto_ablkcipher_encrypt(cryptd_req); 358 } else { 359 struct blkcipher_desc desc; 360 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 361 desc.info = req->info; 362 desc.flags = 0; 363 return crypto_blkcipher_crt(desc.tfm)->encrypt( 364 &desc, req->dst, req->src, req->nbytes); 365 } 366 } 367 368 static int ablk_decrypt(struct ablkcipher_request *req) 369 { 370 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 371 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 372 373 if (kernel_fpu_using()) { 374 struct ablkcipher_request *cryptd_req = 375 ablkcipher_request_ctx(req); 376 memcpy(cryptd_req, req, sizeof(*req)); 377 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 378 return crypto_ablkcipher_decrypt(cryptd_req); 379 } else { 380 struct blkcipher_desc desc; 381 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 382 desc.info = req->info; 383 desc.flags = 0; 384 return crypto_blkcipher_crt(desc.tfm)->decrypt( 385 &desc, req->dst, req->src, req->nbytes); 386 } 387 } 388 389 static void ablk_exit(struct crypto_tfm *tfm) 390 { 391 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 392 393 cryptd_free_ablkcipher(ctx->cryptd_tfm); 394 } 395 396 static void ablk_init_common(struct crypto_tfm *tfm, 397 struct cryptd_ablkcipher *cryptd_tfm) 398 { 399 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 400 401 ctx->cryptd_tfm = cryptd_tfm; 402 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 403 crypto_ablkcipher_reqsize(&cryptd_tfm->base); 404 } 405 406 static int ablk_ecb_init(struct crypto_tfm *tfm) 407 { 408 struct cryptd_ablkcipher *cryptd_tfm; 409 410 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); 411 if (IS_ERR(cryptd_tfm)) 412 return PTR_ERR(cryptd_tfm); 413 ablk_init_common(tfm, cryptd_tfm); 414 return 0; 415 } 416 417 static struct crypto_alg ablk_ecb_alg = { 418 .cra_name = "ecb(aes)", 419 .cra_driver_name = "ecb-aes-aesni", 420 .cra_priority = 400, 421 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 422 .cra_blocksize = AES_BLOCK_SIZE, 423 .cra_ctxsize = sizeof(struct async_aes_ctx), 424 .cra_alignmask = 0, 425 .cra_type = &crypto_ablkcipher_type, 426 .cra_module = THIS_MODULE, 427 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), 428 .cra_init = ablk_ecb_init, 429 .cra_exit = ablk_exit, 430 .cra_u = { 431 .ablkcipher = { 432 .min_keysize = AES_MIN_KEY_SIZE, 433 .max_keysize = AES_MAX_KEY_SIZE, 434 .setkey = ablk_set_key, 435 .encrypt = ablk_encrypt, 436 .decrypt = ablk_decrypt, 437 }, 438 }, 439 }; 440 441 static int ablk_cbc_init(struct crypto_tfm *tfm) 442 { 443 struct cryptd_ablkcipher *cryptd_tfm; 444 445 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); 446 if (IS_ERR(cryptd_tfm)) 447 return PTR_ERR(cryptd_tfm); 448 ablk_init_common(tfm, cryptd_tfm); 449 return 0; 450 } 451 452 static struct crypto_alg ablk_cbc_alg = { 453 .cra_name = "cbc(aes)", 454 .cra_driver_name = "cbc-aes-aesni", 455 .cra_priority = 400, 456 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 457 .cra_blocksize = AES_BLOCK_SIZE, 458 .cra_ctxsize = sizeof(struct async_aes_ctx), 459 .cra_alignmask = 0, 460 .cra_type = &crypto_ablkcipher_type, 461 .cra_module = THIS_MODULE, 462 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), 463 .cra_init = ablk_cbc_init, 464 .cra_exit = ablk_exit, 465 .cra_u = { 466 .ablkcipher = { 467 .min_keysize = AES_MIN_KEY_SIZE, 468 .max_keysize = AES_MAX_KEY_SIZE, 469 .ivsize = AES_BLOCK_SIZE, 470 .setkey = ablk_set_key, 471 .encrypt = ablk_encrypt, 472 .decrypt = ablk_decrypt, 473 }, 474 }, 475 }; 476 477 #ifdef HAS_CTR 478 static int ablk_ctr_init(struct crypto_tfm *tfm) 479 { 480 struct cryptd_ablkcipher *cryptd_tfm; 481 482 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(ctr(__driver-aes-aesni))", 483 0, 0); 484 if (IS_ERR(cryptd_tfm)) 485 return PTR_ERR(cryptd_tfm); 486 ablk_init_common(tfm, cryptd_tfm); 487 return 0; 488 } 489 490 static struct crypto_alg ablk_ctr_alg = { 491 .cra_name = "ctr(aes)", 492 .cra_driver_name = "ctr-aes-aesni", 493 .cra_priority = 400, 494 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 495 .cra_blocksize = 1, 496 .cra_ctxsize = sizeof(struct async_aes_ctx), 497 .cra_alignmask = 0, 498 .cra_type = &crypto_ablkcipher_type, 499 .cra_module = THIS_MODULE, 500 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list), 501 .cra_init = ablk_ctr_init, 502 .cra_exit = ablk_exit, 503 .cra_u = { 504 .ablkcipher = { 505 .min_keysize = AES_MIN_KEY_SIZE, 506 .max_keysize = AES_MAX_KEY_SIZE, 507 .ivsize = AES_BLOCK_SIZE, 508 .setkey = ablk_set_key, 509 .encrypt = ablk_encrypt, 510 .decrypt = ablk_decrypt, 511 .geniv = "chainiv", 512 }, 513 }, 514 }; 515 #endif 516 517 #ifdef HAS_LRW 518 static int ablk_lrw_init(struct crypto_tfm *tfm) 519 { 520 struct cryptd_ablkcipher *cryptd_tfm; 521 522 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))", 523 0, 0); 524 if (IS_ERR(cryptd_tfm)) 525 return PTR_ERR(cryptd_tfm); 526 ablk_init_common(tfm, cryptd_tfm); 527 return 0; 528 } 529 530 static struct crypto_alg ablk_lrw_alg = { 531 .cra_name = "lrw(aes)", 532 .cra_driver_name = "lrw-aes-aesni", 533 .cra_priority = 400, 534 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 535 .cra_blocksize = AES_BLOCK_SIZE, 536 .cra_ctxsize = sizeof(struct async_aes_ctx), 537 .cra_alignmask = 0, 538 .cra_type = &crypto_ablkcipher_type, 539 .cra_module = THIS_MODULE, 540 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), 541 .cra_init = ablk_lrw_init, 542 .cra_exit = ablk_exit, 543 .cra_u = { 544 .ablkcipher = { 545 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 546 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 547 .ivsize = AES_BLOCK_SIZE, 548 .setkey = ablk_set_key, 549 .encrypt = ablk_encrypt, 550 .decrypt = ablk_decrypt, 551 }, 552 }, 553 }; 554 #endif 555 556 #ifdef HAS_PCBC 557 static int ablk_pcbc_init(struct crypto_tfm *tfm) 558 { 559 struct cryptd_ablkcipher *cryptd_tfm; 560 561 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))", 562 0, 0); 563 if (IS_ERR(cryptd_tfm)) 564 return PTR_ERR(cryptd_tfm); 565 ablk_init_common(tfm, cryptd_tfm); 566 return 0; 567 } 568 569 static struct crypto_alg ablk_pcbc_alg = { 570 .cra_name = "pcbc(aes)", 571 .cra_driver_name = "pcbc-aes-aesni", 572 .cra_priority = 400, 573 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 574 .cra_blocksize = AES_BLOCK_SIZE, 575 .cra_ctxsize = sizeof(struct async_aes_ctx), 576 .cra_alignmask = 0, 577 .cra_type = &crypto_ablkcipher_type, 578 .cra_module = THIS_MODULE, 579 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list), 580 .cra_init = ablk_pcbc_init, 581 .cra_exit = ablk_exit, 582 .cra_u = { 583 .ablkcipher = { 584 .min_keysize = AES_MIN_KEY_SIZE, 585 .max_keysize = AES_MAX_KEY_SIZE, 586 .ivsize = AES_BLOCK_SIZE, 587 .setkey = ablk_set_key, 588 .encrypt = ablk_encrypt, 589 .decrypt = ablk_decrypt, 590 }, 591 }, 592 }; 593 #endif 594 595 #ifdef HAS_XTS 596 static int ablk_xts_init(struct crypto_tfm *tfm) 597 { 598 struct cryptd_ablkcipher *cryptd_tfm; 599 600 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))", 601 0, 0); 602 if (IS_ERR(cryptd_tfm)) 603 return PTR_ERR(cryptd_tfm); 604 ablk_init_common(tfm, cryptd_tfm); 605 return 0; 606 } 607 608 static struct crypto_alg ablk_xts_alg = { 609 .cra_name = "xts(aes)", 610 .cra_driver_name = "xts-aes-aesni", 611 .cra_priority = 400, 612 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 613 .cra_blocksize = AES_BLOCK_SIZE, 614 .cra_ctxsize = sizeof(struct async_aes_ctx), 615 .cra_alignmask = 0, 616 .cra_type = &crypto_ablkcipher_type, 617 .cra_module = THIS_MODULE, 618 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list), 619 .cra_init = ablk_xts_init, 620 .cra_exit = ablk_exit, 621 .cra_u = { 622 .ablkcipher = { 623 .min_keysize = 2 * AES_MIN_KEY_SIZE, 624 .max_keysize = 2 * AES_MAX_KEY_SIZE, 625 .ivsize = AES_BLOCK_SIZE, 626 .setkey = ablk_set_key, 627 .encrypt = ablk_encrypt, 628 .decrypt = ablk_decrypt, 629 }, 630 }, 631 }; 632 #endif 633 634 static int __init aesni_init(void) 635 { 636 int err; 637 638 if (!cpu_has_aes) { 639 printk(KERN_ERR "Intel AES-NI instructions are not detected.\n"); 640 return -ENODEV; 641 } 642 if ((err = crypto_register_alg(&aesni_alg))) 643 goto aes_err; 644 if ((err = crypto_register_alg(&__aesni_alg))) 645 goto __aes_err; 646 if ((err = crypto_register_alg(&blk_ecb_alg))) 647 goto blk_ecb_err; 648 if ((err = crypto_register_alg(&blk_cbc_alg))) 649 goto blk_cbc_err; 650 if ((err = crypto_register_alg(&ablk_ecb_alg))) 651 goto ablk_ecb_err; 652 if ((err = crypto_register_alg(&ablk_cbc_alg))) 653 goto ablk_cbc_err; 654 #ifdef HAS_CTR 655 if ((err = crypto_register_alg(&ablk_ctr_alg))) 656 goto ablk_ctr_err; 657 #endif 658 #ifdef HAS_LRW 659 if ((err = crypto_register_alg(&ablk_lrw_alg))) 660 goto ablk_lrw_err; 661 #endif 662 #ifdef HAS_PCBC 663 if ((err = crypto_register_alg(&ablk_pcbc_alg))) 664 goto ablk_pcbc_err; 665 #endif 666 #ifdef HAS_XTS 667 if ((err = crypto_register_alg(&ablk_xts_alg))) 668 goto ablk_xts_err; 669 #endif 670 671 return err; 672 673 #ifdef HAS_XTS 674 ablk_xts_err: 675 #endif 676 #ifdef HAS_PCBC 677 crypto_unregister_alg(&ablk_pcbc_alg); 678 ablk_pcbc_err: 679 #endif 680 #ifdef HAS_LRW 681 crypto_unregister_alg(&ablk_lrw_alg); 682 ablk_lrw_err: 683 #endif 684 #ifdef HAS_CTR 685 crypto_unregister_alg(&ablk_ctr_alg); 686 ablk_ctr_err: 687 #endif 688 crypto_unregister_alg(&ablk_cbc_alg); 689 ablk_cbc_err: 690 crypto_unregister_alg(&ablk_ecb_alg); 691 ablk_ecb_err: 692 crypto_unregister_alg(&blk_cbc_alg); 693 blk_cbc_err: 694 crypto_unregister_alg(&blk_ecb_alg); 695 blk_ecb_err: 696 crypto_unregister_alg(&__aesni_alg); 697 __aes_err: 698 crypto_unregister_alg(&aesni_alg); 699 aes_err: 700 return err; 701 } 702 703 static void __exit aesni_exit(void) 704 { 705 #ifdef HAS_XTS 706 crypto_unregister_alg(&ablk_xts_alg); 707 #endif 708 #ifdef HAS_PCBC 709 crypto_unregister_alg(&ablk_pcbc_alg); 710 #endif 711 #ifdef HAS_LRW 712 crypto_unregister_alg(&ablk_lrw_alg); 713 #endif 714 #ifdef HAS_CTR 715 crypto_unregister_alg(&ablk_ctr_alg); 716 #endif 717 crypto_unregister_alg(&ablk_cbc_alg); 718 crypto_unregister_alg(&ablk_ecb_alg); 719 crypto_unregister_alg(&blk_cbc_alg); 720 crypto_unregister_alg(&blk_ecb_alg); 721 crypto_unregister_alg(&__aesni_alg); 722 crypto_unregister_alg(&aesni_alg); 723 } 724 725 module_init(aesni_init); 726 module_exit(aesni_exit); 727 728 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 729 MODULE_LICENSE("GPL"); 730 MODULE_ALIAS("aes"); 731