1 /* 2 * Glue Code for 3-way parallel assembler optimized version of Twofish 3 * 4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 24 * USA 25 * 26 */ 27 28 #include <asm/processor.h> 29 #include <linux/crypto.h> 30 #include <linux/init.h> 31 #include <linux/module.h> 32 #include <linux/types.h> 33 #include <crypto/algapi.h> 34 #include <crypto/twofish.h> 35 #include <crypto/b128ops.h> 36 #include <crypto/lrw.h> 37 #include <crypto/xts.h> 38 39 /* regular block cipher functions from twofish_x86_64 module */ 40 asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst, 41 const u8 *src); 42 asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst, 43 const u8 *src); 44 45 /* 3-way parallel cipher functions */ 46 asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 47 const u8 *src, bool xor); 48 EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way); 49 asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst, 50 const u8 *src); 51 EXPORT_SYMBOL_GPL(twofish_dec_blk_3way); 52 53 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 54 const u8 *src) 55 { 56 __twofish_enc_blk_3way(ctx, dst, src, false); 57 } 58 59 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst, 60 const u8 *src) 61 { 62 __twofish_enc_blk_3way(ctx, dst, src, true); 63 } 64 65 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 66 void (*fn)(struct twofish_ctx *, u8 *, const u8 *), 67 void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *)) 68 { 69 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 70 unsigned int bsize = TF_BLOCK_SIZE; 71 unsigned int nbytes; 72 int err; 73 74 err = blkcipher_walk_virt(desc, walk); 75 76 while ((nbytes = walk->nbytes)) { 77 u8 *wsrc = walk->src.virt.addr; 78 u8 *wdst = walk->dst.virt.addr; 79 80 /* Process three block batch */ 81 if (nbytes >= bsize * 3) { 82 do { 83 fn_3way(ctx, wdst, wsrc); 84 85 wsrc += bsize * 3; 86 wdst += bsize * 3; 87 nbytes -= bsize * 3; 88 } while (nbytes >= bsize * 3); 89 90 if (nbytes < bsize) 91 goto done; 92 } 93 94 /* Handle leftovers */ 95 do { 96 fn(ctx, wdst, wsrc); 97 98 wsrc += bsize; 99 wdst += bsize; 100 nbytes -= bsize; 101 } while (nbytes >= bsize); 102 103 done: 104 err = blkcipher_walk_done(desc, walk, nbytes); 105 } 106 107 return err; 108 } 109 110 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 111 struct scatterlist *src, unsigned int nbytes) 112 { 113 struct blkcipher_walk walk; 114 115 blkcipher_walk_init(&walk, dst, src, nbytes); 116 return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way); 117 } 118 119 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 120 struct scatterlist *src, unsigned int nbytes) 121 { 122 struct blkcipher_walk walk; 123 124 blkcipher_walk_init(&walk, dst, src, nbytes); 125 return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way); 126 } 127 128 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 129 struct blkcipher_walk *walk) 130 { 131 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 132 unsigned int bsize = TF_BLOCK_SIZE; 133 unsigned int nbytes = walk->nbytes; 134 u128 *src = (u128 *)walk->src.virt.addr; 135 u128 *dst = (u128 *)walk->dst.virt.addr; 136 u128 *iv = (u128 *)walk->iv; 137 138 do { 139 u128_xor(dst, src, iv); 140 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 141 iv = dst; 142 143 src += 1; 144 dst += 1; 145 nbytes -= bsize; 146 } while (nbytes >= bsize); 147 148 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); 149 return nbytes; 150 } 151 152 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 153 struct scatterlist *src, unsigned int nbytes) 154 { 155 struct blkcipher_walk walk; 156 int err; 157 158 blkcipher_walk_init(&walk, dst, src, nbytes); 159 err = blkcipher_walk_virt(desc, &walk); 160 161 while ((nbytes = walk.nbytes)) { 162 nbytes = __cbc_encrypt(desc, &walk); 163 err = blkcipher_walk_done(desc, &walk, nbytes); 164 } 165 166 return err; 167 } 168 169 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 170 struct blkcipher_walk *walk) 171 { 172 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 173 unsigned int bsize = TF_BLOCK_SIZE; 174 unsigned int nbytes = walk->nbytes; 175 u128 *src = (u128 *)walk->src.virt.addr; 176 u128 *dst = (u128 *)walk->dst.virt.addr; 177 u128 ivs[3 - 1]; 178 u128 last_iv; 179 180 /* Start of the last block. */ 181 src += nbytes / bsize - 1; 182 dst += nbytes / bsize - 1; 183 184 last_iv = *src; 185 186 /* Process three block batch */ 187 if (nbytes >= bsize * 3) { 188 do { 189 nbytes -= bsize * (3 - 1); 190 src -= 3 - 1; 191 dst -= 3 - 1; 192 193 ivs[0] = src[0]; 194 ivs[1] = src[1]; 195 196 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); 197 198 u128_xor(dst + 1, dst + 1, ivs + 0); 199 u128_xor(dst + 2, dst + 2, ivs + 1); 200 201 nbytes -= bsize; 202 if (nbytes < bsize) 203 goto done; 204 205 u128_xor(dst, dst, src - 1); 206 src -= 1; 207 dst -= 1; 208 } while (nbytes >= bsize * 3); 209 210 if (nbytes < bsize) 211 goto done; 212 } 213 214 /* Handle leftovers */ 215 for (;;) { 216 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src); 217 218 nbytes -= bsize; 219 if (nbytes < bsize) 220 break; 221 222 u128_xor(dst, dst, src - 1); 223 src -= 1; 224 dst -= 1; 225 } 226 227 done: 228 u128_xor(dst, dst, (u128 *)walk->iv); 229 *(u128 *)walk->iv = last_iv; 230 231 return nbytes; 232 } 233 234 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 235 struct scatterlist *src, unsigned int nbytes) 236 { 237 struct blkcipher_walk walk; 238 int err; 239 240 blkcipher_walk_init(&walk, dst, src, nbytes); 241 err = blkcipher_walk_virt(desc, &walk); 242 243 while ((nbytes = walk.nbytes)) { 244 nbytes = __cbc_decrypt(desc, &walk); 245 err = blkcipher_walk_done(desc, &walk, nbytes); 246 } 247 248 return err; 249 } 250 251 static inline void u128_to_be128(be128 *dst, const u128 *src) 252 { 253 dst->a = cpu_to_be64(src->a); 254 dst->b = cpu_to_be64(src->b); 255 } 256 257 static inline void be128_to_u128(u128 *dst, const be128 *src) 258 { 259 dst->a = be64_to_cpu(src->a); 260 dst->b = be64_to_cpu(src->b); 261 } 262 263 static inline void u128_inc(u128 *i) 264 { 265 i->b++; 266 if (!i->b) 267 i->a++; 268 } 269 270 static void ctr_crypt_final(struct blkcipher_desc *desc, 271 struct blkcipher_walk *walk) 272 { 273 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 274 u8 *ctrblk = walk->iv; 275 u8 keystream[TF_BLOCK_SIZE]; 276 u8 *src = walk->src.virt.addr; 277 u8 *dst = walk->dst.virt.addr; 278 unsigned int nbytes = walk->nbytes; 279 280 twofish_enc_blk(ctx, keystream, ctrblk); 281 crypto_xor(keystream, src, nbytes); 282 memcpy(dst, keystream, nbytes); 283 284 crypto_inc(ctrblk, TF_BLOCK_SIZE); 285 } 286 287 static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 288 struct blkcipher_walk *walk) 289 { 290 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 291 unsigned int bsize = TF_BLOCK_SIZE; 292 unsigned int nbytes = walk->nbytes; 293 u128 *src = (u128 *)walk->src.virt.addr; 294 u128 *dst = (u128 *)walk->dst.virt.addr; 295 u128 ctrblk; 296 be128 ctrblocks[3]; 297 298 be128_to_u128(&ctrblk, (be128 *)walk->iv); 299 300 /* Process three block batch */ 301 if (nbytes >= bsize * 3) { 302 do { 303 if (dst != src) { 304 dst[0] = src[0]; 305 dst[1] = src[1]; 306 dst[2] = src[2]; 307 } 308 309 /* create ctrblks for parallel encrypt */ 310 u128_to_be128(&ctrblocks[0], &ctrblk); 311 u128_inc(&ctrblk); 312 u128_to_be128(&ctrblocks[1], &ctrblk); 313 u128_inc(&ctrblk); 314 u128_to_be128(&ctrblocks[2], &ctrblk); 315 u128_inc(&ctrblk); 316 317 twofish_enc_blk_xor_3way(ctx, (u8 *)dst, 318 (u8 *)ctrblocks); 319 320 src += 3; 321 dst += 3; 322 nbytes -= bsize * 3; 323 } while (nbytes >= bsize * 3); 324 325 if (nbytes < bsize) 326 goto done; 327 } 328 329 /* Handle leftovers */ 330 do { 331 if (dst != src) 332 *dst = *src; 333 334 u128_to_be128(&ctrblocks[0], &ctrblk); 335 u128_inc(&ctrblk); 336 337 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); 338 u128_xor(dst, dst, (u128 *)ctrblocks); 339 340 src += 1; 341 dst += 1; 342 nbytes -= bsize; 343 } while (nbytes >= bsize); 344 345 done: 346 u128_to_be128((be128 *)walk->iv, &ctrblk); 347 return nbytes; 348 } 349 350 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 351 struct scatterlist *src, unsigned int nbytes) 352 { 353 struct blkcipher_walk walk; 354 int err; 355 356 blkcipher_walk_init(&walk, dst, src, nbytes); 357 err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE); 358 359 while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) { 360 nbytes = __ctr_crypt(desc, &walk); 361 err = blkcipher_walk_done(desc, &walk, nbytes); 362 } 363 364 if (walk.nbytes) { 365 ctr_crypt_final(desc, &walk); 366 err = blkcipher_walk_done(desc, &walk, 0); 367 } 368 369 return err; 370 } 371 372 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 373 { 374 const unsigned int bsize = TF_BLOCK_SIZE; 375 struct twofish_ctx *ctx = priv; 376 int i; 377 378 if (nbytes == 3 * bsize) { 379 twofish_enc_blk_3way(ctx, srcdst, srcdst); 380 return; 381 } 382 383 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 384 twofish_enc_blk(ctx, srcdst, srcdst); 385 } 386 387 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 388 { 389 const unsigned int bsize = TF_BLOCK_SIZE; 390 struct twofish_ctx *ctx = priv; 391 int i; 392 393 if (nbytes == 3 * bsize) { 394 twofish_dec_blk_3way(ctx, srcdst, srcdst); 395 return; 396 } 397 398 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 399 twofish_dec_blk(ctx, srcdst, srcdst); 400 } 401 402 struct twofish_lrw_ctx { 403 struct lrw_table_ctx lrw_table; 404 struct twofish_ctx twofish_ctx; 405 }; 406 407 static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, 408 unsigned int keylen) 409 { 410 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 411 int err; 412 413 err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE, 414 &tfm->crt_flags); 415 if (err) 416 return err; 417 418 return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE); 419 } 420 421 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 422 struct scatterlist *src, unsigned int nbytes) 423 { 424 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 425 be128 buf[3]; 426 struct lrw_crypt_req req = { 427 .tbuf = buf, 428 .tbuflen = sizeof(buf), 429 430 .table_ctx = &ctx->lrw_table, 431 .crypt_ctx = &ctx->twofish_ctx, 432 .crypt_fn = encrypt_callback, 433 }; 434 435 return lrw_crypt(desc, dst, src, nbytes, &req); 436 } 437 438 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 439 struct scatterlist *src, unsigned int nbytes) 440 { 441 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 442 be128 buf[3]; 443 struct lrw_crypt_req req = { 444 .tbuf = buf, 445 .tbuflen = sizeof(buf), 446 447 .table_ctx = &ctx->lrw_table, 448 .crypt_ctx = &ctx->twofish_ctx, 449 .crypt_fn = decrypt_callback, 450 }; 451 452 return lrw_crypt(desc, dst, src, nbytes, &req); 453 } 454 455 static void lrw_exit_tfm(struct crypto_tfm *tfm) 456 { 457 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 458 459 lrw_free_table(&ctx->lrw_table); 460 } 461 462 struct twofish_xts_ctx { 463 struct twofish_ctx tweak_ctx; 464 struct twofish_ctx crypt_ctx; 465 }; 466 467 static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, 468 unsigned int keylen) 469 { 470 struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm); 471 u32 *flags = &tfm->crt_flags; 472 int err; 473 474 /* key consists of keys of equal size concatenated, therefore 475 * the length must be even 476 */ 477 if (keylen % 2) { 478 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 479 return -EINVAL; 480 } 481 482 /* first half of xts-key is for crypt */ 483 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); 484 if (err) 485 return err; 486 487 /* second half of xts-key is for tweak */ 488 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, 489 flags); 490 } 491 492 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 493 struct scatterlist *src, unsigned int nbytes) 494 { 495 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 496 be128 buf[3]; 497 struct xts_crypt_req req = { 498 .tbuf = buf, 499 .tbuflen = sizeof(buf), 500 501 .tweak_ctx = &ctx->tweak_ctx, 502 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), 503 .crypt_ctx = &ctx->crypt_ctx, 504 .crypt_fn = encrypt_callback, 505 }; 506 507 return xts_crypt(desc, dst, src, nbytes, &req); 508 } 509 510 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 511 struct scatterlist *src, unsigned int nbytes) 512 { 513 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 514 be128 buf[3]; 515 struct xts_crypt_req req = { 516 .tbuf = buf, 517 .tbuflen = sizeof(buf), 518 519 .tweak_ctx = &ctx->tweak_ctx, 520 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), 521 .crypt_ctx = &ctx->crypt_ctx, 522 .crypt_fn = decrypt_callback, 523 }; 524 525 return xts_crypt(desc, dst, src, nbytes, &req); 526 } 527 528 static struct crypto_alg tf_algs[5] = { { 529 .cra_name = "ecb(twofish)", 530 .cra_driver_name = "ecb-twofish-3way", 531 .cra_priority = 300, 532 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 533 .cra_blocksize = TF_BLOCK_SIZE, 534 .cra_ctxsize = sizeof(struct twofish_ctx), 535 .cra_alignmask = 0, 536 .cra_type = &crypto_blkcipher_type, 537 .cra_module = THIS_MODULE, 538 .cra_list = LIST_HEAD_INIT(tf_algs[0].cra_list), 539 .cra_u = { 540 .blkcipher = { 541 .min_keysize = TF_MIN_KEY_SIZE, 542 .max_keysize = TF_MAX_KEY_SIZE, 543 .setkey = twofish_setkey, 544 .encrypt = ecb_encrypt, 545 .decrypt = ecb_decrypt, 546 }, 547 }, 548 }, { 549 .cra_name = "cbc(twofish)", 550 .cra_driver_name = "cbc-twofish-3way", 551 .cra_priority = 300, 552 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 553 .cra_blocksize = TF_BLOCK_SIZE, 554 .cra_ctxsize = sizeof(struct twofish_ctx), 555 .cra_alignmask = 0, 556 .cra_type = &crypto_blkcipher_type, 557 .cra_module = THIS_MODULE, 558 .cra_list = LIST_HEAD_INIT(tf_algs[1].cra_list), 559 .cra_u = { 560 .blkcipher = { 561 .min_keysize = TF_MIN_KEY_SIZE, 562 .max_keysize = TF_MAX_KEY_SIZE, 563 .ivsize = TF_BLOCK_SIZE, 564 .setkey = twofish_setkey, 565 .encrypt = cbc_encrypt, 566 .decrypt = cbc_decrypt, 567 }, 568 }, 569 }, { 570 .cra_name = "ctr(twofish)", 571 .cra_driver_name = "ctr-twofish-3way", 572 .cra_priority = 300, 573 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 574 .cra_blocksize = 1, 575 .cra_ctxsize = sizeof(struct twofish_ctx), 576 .cra_alignmask = 0, 577 .cra_type = &crypto_blkcipher_type, 578 .cra_module = THIS_MODULE, 579 .cra_list = LIST_HEAD_INIT(tf_algs[2].cra_list), 580 .cra_u = { 581 .blkcipher = { 582 .min_keysize = TF_MIN_KEY_SIZE, 583 .max_keysize = TF_MAX_KEY_SIZE, 584 .ivsize = TF_BLOCK_SIZE, 585 .setkey = twofish_setkey, 586 .encrypt = ctr_crypt, 587 .decrypt = ctr_crypt, 588 }, 589 }, 590 }, { 591 .cra_name = "lrw(twofish)", 592 .cra_driver_name = "lrw-twofish-3way", 593 .cra_priority = 300, 594 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 595 .cra_blocksize = TF_BLOCK_SIZE, 596 .cra_ctxsize = sizeof(struct twofish_lrw_ctx), 597 .cra_alignmask = 0, 598 .cra_type = &crypto_blkcipher_type, 599 .cra_module = THIS_MODULE, 600 .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list), 601 .cra_exit = lrw_exit_tfm, 602 .cra_u = { 603 .blkcipher = { 604 .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, 605 .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE, 606 .ivsize = TF_BLOCK_SIZE, 607 .setkey = lrw_twofish_setkey, 608 .encrypt = lrw_encrypt, 609 .decrypt = lrw_decrypt, 610 }, 611 }, 612 }, { 613 .cra_name = "xts(twofish)", 614 .cra_driver_name = "xts-twofish-3way", 615 .cra_priority = 300, 616 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 617 .cra_blocksize = TF_BLOCK_SIZE, 618 .cra_ctxsize = sizeof(struct twofish_xts_ctx), 619 .cra_alignmask = 0, 620 .cra_type = &crypto_blkcipher_type, 621 .cra_module = THIS_MODULE, 622 .cra_list = LIST_HEAD_INIT(tf_algs[4].cra_list), 623 .cra_u = { 624 .blkcipher = { 625 .min_keysize = TF_MIN_KEY_SIZE * 2, 626 .max_keysize = TF_MAX_KEY_SIZE * 2, 627 .ivsize = TF_BLOCK_SIZE, 628 .setkey = xts_twofish_setkey, 629 .encrypt = xts_encrypt, 630 .decrypt = xts_decrypt, 631 }, 632 }, 633 } }; 634 635 static bool is_blacklisted_cpu(void) 636 { 637 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 638 return false; 639 640 if (boot_cpu_data.x86 == 0x06 && 641 (boot_cpu_data.x86_model == 0x1c || 642 boot_cpu_data.x86_model == 0x26 || 643 boot_cpu_data.x86_model == 0x36)) { 644 /* 645 * On Atom, twofish-3way is slower than original assembler 646 * implementation. Twofish-3way trades off some performance in 647 * storing blocks in 64bit registers to allow three blocks to 648 * be processed parallel. Parallel operation then allows gaining 649 * more performance than was trade off, on out-of-order CPUs. 650 * However Atom does not benefit from this parallellism and 651 * should be blacklisted. 652 */ 653 return true; 654 } 655 656 if (boot_cpu_data.x86 == 0x0f) { 657 /* 658 * On Pentium 4, twofish-3way is slower than original assembler 659 * implementation because excessive uses of 64bit rotate and 660 * left-shifts (which are really slow on P4) needed to store and 661 * handle 128bit block in two 64bit registers. 662 */ 663 return true; 664 } 665 666 return false; 667 } 668 669 static int force; 670 module_param(force, int, 0); 671 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 672 673 static int __init init(void) 674 { 675 if (!force && is_blacklisted_cpu()) { 676 printk(KERN_INFO 677 "twofish-x86_64-3way: performance on this CPU " 678 "would be suboptimal: disabling " 679 "twofish-x86_64-3way.\n"); 680 return -ENODEV; 681 } 682 683 return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs)); 684 } 685 686 static void __exit fini(void) 687 { 688 crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs)); 689 } 690 691 module_init(init); 692 module_exit(fini); 693 694 MODULE_LICENSE("GPL"); 695 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized"); 696 MODULE_ALIAS("twofish"); 697 MODULE_ALIAS("twofish-asm"); 698