1 /* 2 * Glue Code for 3-way parallel assembler optimized version of Twofish 3 * 4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 24 * USA 25 * 26 */ 27 28 #include <linux/crypto.h> 29 #include <linux/init.h> 30 #include <linux/module.h> 31 #include <linux/types.h> 32 #include <crypto/algapi.h> 33 #include <crypto/twofish.h> 34 #include <crypto/b128ops.h> 35 #include <crypto/lrw.h> 36 37 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) 38 #define HAS_LRW 39 #endif 40 41 /* regular block cipher functions from twofish_x86_64 module */ 42 asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst, 43 const u8 *src); 44 asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst, 45 const u8 *src); 46 47 /* 3-way parallel cipher functions */ 48 asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 49 const u8 *src, bool xor); 50 asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst, 51 const u8 *src); 52 53 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 54 const u8 *src) 55 { 56 __twofish_enc_blk_3way(ctx, dst, src, false); 57 } 58 59 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst, 60 const u8 *src) 61 { 62 __twofish_enc_blk_3way(ctx, dst, src, true); 63 } 64 65 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 66 void (*fn)(struct twofish_ctx *, u8 *, const u8 *), 67 void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *)) 68 { 69 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 70 unsigned int bsize = TF_BLOCK_SIZE; 71 unsigned int nbytes; 72 int err; 73 74 err = blkcipher_walk_virt(desc, walk); 75 76 while ((nbytes = walk->nbytes)) { 77 u8 *wsrc = walk->src.virt.addr; 78 u8 *wdst = walk->dst.virt.addr; 79 80 /* Process three block batch */ 81 if (nbytes >= bsize * 3) { 82 do { 83 fn_3way(ctx, wdst, wsrc); 84 85 wsrc += bsize * 3; 86 wdst += bsize * 3; 87 nbytes -= bsize * 3; 88 } while (nbytes >= bsize * 3); 89 90 if (nbytes < bsize) 91 goto done; 92 } 93 94 /* Handle leftovers */ 95 do { 96 fn(ctx, wdst, wsrc); 97 98 wsrc += bsize; 99 wdst += bsize; 100 nbytes -= bsize; 101 } while (nbytes >= bsize); 102 103 done: 104 err = blkcipher_walk_done(desc, walk, nbytes); 105 } 106 107 return err; 108 } 109 110 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 111 struct scatterlist *src, unsigned int nbytes) 112 { 113 struct blkcipher_walk walk; 114 115 blkcipher_walk_init(&walk, dst, src, nbytes); 116 return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way); 117 } 118 119 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 120 struct scatterlist *src, unsigned int nbytes) 121 { 122 struct blkcipher_walk walk; 123 124 blkcipher_walk_init(&walk, dst, src, nbytes); 125 return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way); 126 } 127 128 static struct crypto_alg blk_ecb_alg = { 129 .cra_name = "ecb(twofish)", 130 .cra_driver_name = "ecb-twofish-3way", 131 .cra_priority = 300, 132 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 133 .cra_blocksize = TF_BLOCK_SIZE, 134 .cra_ctxsize = sizeof(struct twofish_ctx), 135 .cra_alignmask = 0, 136 .cra_type = &crypto_blkcipher_type, 137 .cra_module = THIS_MODULE, 138 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), 139 .cra_u = { 140 .blkcipher = { 141 .min_keysize = TF_MIN_KEY_SIZE, 142 .max_keysize = TF_MAX_KEY_SIZE, 143 .setkey = twofish_setkey, 144 .encrypt = ecb_encrypt, 145 .decrypt = ecb_decrypt, 146 }, 147 }, 148 }; 149 150 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 151 struct blkcipher_walk *walk) 152 { 153 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 154 unsigned int bsize = TF_BLOCK_SIZE; 155 unsigned int nbytes = walk->nbytes; 156 u128 *src = (u128 *)walk->src.virt.addr; 157 u128 *dst = (u128 *)walk->dst.virt.addr; 158 u128 *iv = (u128 *)walk->iv; 159 160 do { 161 u128_xor(dst, src, iv); 162 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 163 iv = dst; 164 165 src += 1; 166 dst += 1; 167 nbytes -= bsize; 168 } while (nbytes >= bsize); 169 170 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); 171 return nbytes; 172 } 173 174 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 175 struct scatterlist *src, unsigned int nbytes) 176 { 177 struct blkcipher_walk walk; 178 int err; 179 180 blkcipher_walk_init(&walk, dst, src, nbytes); 181 err = blkcipher_walk_virt(desc, &walk); 182 183 while ((nbytes = walk.nbytes)) { 184 nbytes = __cbc_encrypt(desc, &walk); 185 err = blkcipher_walk_done(desc, &walk, nbytes); 186 } 187 188 return err; 189 } 190 191 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 192 struct blkcipher_walk *walk) 193 { 194 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 195 unsigned int bsize = TF_BLOCK_SIZE; 196 unsigned int nbytes = walk->nbytes; 197 u128 *src = (u128 *)walk->src.virt.addr; 198 u128 *dst = (u128 *)walk->dst.virt.addr; 199 u128 ivs[3 - 1]; 200 u128 last_iv; 201 202 /* Start of the last block. */ 203 src += nbytes / bsize - 1; 204 dst += nbytes / bsize - 1; 205 206 last_iv = *src; 207 208 /* Process three block batch */ 209 if (nbytes >= bsize * 3) { 210 do { 211 nbytes -= bsize * (3 - 1); 212 src -= 3 - 1; 213 dst -= 3 - 1; 214 215 ivs[0] = src[0]; 216 ivs[1] = src[1]; 217 218 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); 219 220 u128_xor(dst + 1, dst + 1, ivs + 0); 221 u128_xor(dst + 2, dst + 2, ivs + 1); 222 223 nbytes -= bsize; 224 if (nbytes < bsize) 225 goto done; 226 227 u128_xor(dst, dst, src - 1); 228 src -= 1; 229 dst -= 1; 230 } while (nbytes >= bsize * 3); 231 232 if (nbytes < bsize) 233 goto done; 234 } 235 236 /* Handle leftovers */ 237 for (;;) { 238 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src); 239 240 nbytes -= bsize; 241 if (nbytes < bsize) 242 break; 243 244 u128_xor(dst, dst, src - 1); 245 src -= 1; 246 dst -= 1; 247 } 248 249 done: 250 u128_xor(dst, dst, (u128 *)walk->iv); 251 *(u128 *)walk->iv = last_iv; 252 253 return nbytes; 254 } 255 256 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 257 struct scatterlist *src, unsigned int nbytes) 258 { 259 struct blkcipher_walk walk; 260 int err; 261 262 blkcipher_walk_init(&walk, dst, src, nbytes); 263 err = blkcipher_walk_virt(desc, &walk); 264 265 while ((nbytes = walk.nbytes)) { 266 nbytes = __cbc_decrypt(desc, &walk); 267 err = blkcipher_walk_done(desc, &walk, nbytes); 268 } 269 270 return err; 271 } 272 273 static struct crypto_alg blk_cbc_alg = { 274 .cra_name = "cbc(twofish)", 275 .cra_driver_name = "cbc-twofish-3way", 276 .cra_priority = 300, 277 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 278 .cra_blocksize = TF_BLOCK_SIZE, 279 .cra_ctxsize = sizeof(struct twofish_ctx), 280 .cra_alignmask = 0, 281 .cra_type = &crypto_blkcipher_type, 282 .cra_module = THIS_MODULE, 283 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), 284 .cra_u = { 285 .blkcipher = { 286 .min_keysize = TF_MIN_KEY_SIZE, 287 .max_keysize = TF_MAX_KEY_SIZE, 288 .ivsize = TF_BLOCK_SIZE, 289 .setkey = twofish_setkey, 290 .encrypt = cbc_encrypt, 291 .decrypt = cbc_decrypt, 292 }, 293 }, 294 }; 295 296 static inline void u128_to_be128(be128 *dst, const u128 *src) 297 { 298 dst->a = cpu_to_be64(src->a); 299 dst->b = cpu_to_be64(src->b); 300 } 301 302 static inline void be128_to_u128(u128 *dst, const be128 *src) 303 { 304 dst->a = be64_to_cpu(src->a); 305 dst->b = be64_to_cpu(src->b); 306 } 307 308 static inline void u128_inc(u128 *i) 309 { 310 i->b++; 311 if (!i->b) 312 i->a++; 313 } 314 315 static void ctr_crypt_final(struct blkcipher_desc *desc, 316 struct blkcipher_walk *walk) 317 { 318 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 319 u8 *ctrblk = walk->iv; 320 u8 keystream[TF_BLOCK_SIZE]; 321 u8 *src = walk->src.virt.addr; 322 u8 *dst = walk->dst.virt.addr; 323 unsigned int nbytes = walk->nbytes; 324 325 twofish_enc_blk(ctx, keystream, ctrblk); 326 crypto_xor(keystream, src, nbytes); 327 memcpy(dst, keystream, nbytes); 328 329 crypto_inc(ctrblk, TF_BLOCK_SIZE); 330 } 331 332 static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 333 struct blkcipher_walk *walk) 334 { 335 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 336 unsigned int bsize = TF_BLOCK_SIZE; 337 unsigned int nbytes = walk->nbytes; 338 u128 *src = (u128 *)walk->src.virt.addr; 339 u128 *dst = (u128 *)walk->dst.virt.addr; 340 u128 ctrblk; 341 be128 ctrblocks[3]; 342 343 be128_to_u128(&ctrblk, (be128 *)walk->iv); 344 345 /* Process three block batch */ 346 if (nbytes >= bsize * 3) { 347 do { 348 if (dst != src) { 349 dst[0] = src[0]; 350 dst[1] = src[1]; 351 dst[2] = src[2]; 352 } 353 354 /* create ctrblks for parallel encrypt */ 355 u128_to_be128(&ctrblocks[0], &ctrblk); 356 u128_inc(&ctrblk); 357 u128_to_be128(&ctrblocks[1], &ctrblk); 358 u128_inc(&ctrblk); 359 u128_to_be128(&ctrblocks[2], &ctrblk); 360 u128_inc(&ctrblk); 361 362 twofish_enc_blk_xor_3way(ctx, (u8 *)dst, 363 (u8 *)ctrblocks); 364 365 src += 3; 366 dst += 3; 367 nbytes -= bsize * 3; 368 } while (nbytes >= bsize * 3); 369 370 if (nbytes < bsize) 371 goto done; 372 } 373 374 /* Handle leftovers */ 375 do { 376 if (dst != src) 377 *dst = *src; 378 379 u128_to_be128(&ctrblocks[0], &ctrblk); 380 u128_inc(&ctrblk); 381 382 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); 383 u128_xor(dst, dst, (u128 *)ctrblocks); 384 385 src += 1; 386 dst += 1; 387 nbytes -= bsize; 388 } while (nbytes >= bsize); 389 390 done: 391 u128_to_be128((be128 *)walk->iv, &ctrblk); 392 return nbytes; 393 } 394 395 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 396 struct scatterlist *src, unsigned int nbytes) 397 { 398 struct blkcipher_walk walk; 399 int err; 400 401 blkcipher_walk_init(&walk, dst, src, nbytes); 402 err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE); 403 404 while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) { 405 nbytes = __ctr_crypt(desc, &walk); 406 err = blkcipher_walk_done(desc, &walk, nbytes); 407 } 408 409 if (walk.nbytes) { 410 ctr_crypt_final(desc, &walk); 411 err = blkcipher_walk_done(desc, &walk, 0); 412 } 413 414 return err; 415 } 416 417 static struct crypto_alg blk_ctr_alg = { 418 .cra_name = "ctr(twofish)", 419 .cra_driver_name = "ctr-twofish-3way", 420 .cra_priority = 300, 421 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 422 .cra_blocksize = 1, 423 .cra_ctxsize = sizeof(struct twofish_ctx), 424 .cra_alignmask = 0, 425 .cra_type = &crypto_blkcipher_type, 426 .cra_module = THIS_MODULE, 427 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), 428 .cra_u = { 429 .blkcipher = { 430 .min_keysize = TF_MIN_KEY_SIZE, 431 .max_keysize = TF_MAX_KEY_SIZE, 432 .ivsize = TF_BLOCK_SIZE, 433 .setkey = twofish_setkey, 434 .encrypt = ctr_crypt, 435 .decrypt = ctr_crypt, 436 }, 437 }, 438 }; 439 440 #ifdef HAS_LRW 441 442 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 443 { 444 const unsigned int bsize = TF_BLOCK_SIZE; 445 struct twofish_ctx *ctx = priv; 446 int i; 447 448 if (nbytes == 3 * bsize) { 449 twofish_enc_blk_3way(ctx, srcdst, srcdst); 450 return; 451 } 452 453 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 454 twofish_enc_blk(ctx, srcdst, srcdst); 455 } 456 457 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 458 { 459 const unsigned int bsize = TF_BLOCK_SIZE; 460 struct twofish_ctx *ctx = priv; 461 int i; 462 463 if (nbytes == 3 * bsize) { 464 twofish_dec_blk_3way(ctx, srcdst, srcdst); 465 return; 466 } 467 468 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 469 twofish_dec_blk(ctx, srcdst, srcdst); 470 } 471 472 struct twofish_lrw_ctx { 473 struct lrw_table_ctx lrw_table; 474 struct twofish_ctx twofish_ctx; 475 }; 476 477 static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, 478 unsigned int keylen) 479 { 480 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 481 int err; 482 483 err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE, 484 &tfm->crt_flags); 485 if (err) 486 return err; 487 488 return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE); 489 } 490 491 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 492 struct scatterlist *src, unsigned int nbytes) 493 { 494 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 495 be128 buf[3]; 496 struct lrw_crypt_req req = { 497 .tbuf = buf, 498 .tbuflen = sizeof(buf), 499 500 .table_ctx = &ctx->lrw_table, 501 .crypt_ctx = &ctx->twofish_ctx, 502 .crypt_fn = encrypt_callback, 503 }; 504 505 return lrw_crypt(desc, dst, src, nbytes, &req); 506 } 507 508 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 509 struct scatterlist *src, unsigned int nbytes) 510 { 511 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 512 be128 buf[3]; 513 struct lrw_crypt_req req = { 514 .tbuf = buf, 515 .tbuflen = sizeof(buf), 516 517 .table_ctx = &ctx->lrw_table, 518 .crypt_ctx = &ctx->twofish_ctx, 519 .crypt_fn = decrypt_callback, 520 }; 521 522 return lrw_crypt(desc, dst, src, nbytes, &req); 523 } 524 525 static void lrw_exit_tfm(struct crypto_tfm *tfm) 526 { 527 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 528 529 lrw_free_table(&ctx->lrw_table); 530 } 531 532 static struct crypto_alg blk_lrw_alg = { 533 .cra_name = "lrw(twofish)", 534 .cra_driver_name = "lrw-twofish-3way", 535 .cra_priority = 300, 536 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 537 .cra_blocksize = TF_BLOCK_SIZE, 538 .cra_ctxsize = sizeof(struct twofish_lrw_ctx), 539 .cra_alignmask = 0, 540 .cra_type = &crypto_blkcipher_type, 541 .cra_module = THIS_MODULE, 542 .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list), 543 .cra_exit = lrw_exit_tfm, 544 .cra_u = { 545 .blkcipher = { 546 .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, 547 .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE, 548 .ivsize = TF_BLOCK_SIZE, 549 .setkey = lrw_twofish_setkey, 550 .encrypt = lrw_encrypt, 551 .decrypt = lrw_decrypt, 552 }, 553 }, 554 }; 555 556 #endif 557 558 int __init init(void) 559 { 560 int err; 561 562 err = crypto_register_alg(&blk_ecb_alg); 563 if (err) 564 goto ecb_err; 565 err = crypto_register_alg(&blk_cbc_alg); 566 if (err) 567 goto cbc_err; 568 err = crypto_register_alg(&blk_ctr_alg); 569 if (err) 570 goto ctr_err; 571 #ifdef HAS_LRW 572 err = crypto_register_alg(&blk_lrw_alg); 573 if (err) 574 goto blk_lrw_err; 575 #endif 576 577 return 0; 578 579 #ifdef HAS_LRW 580 blk_lrw_err: 581 crypto_unregister_alg(&blk_ctr_alg); 582 #endif 583 ctr_err: 584 crypto_unregister_alg(&blk_cbc_alg); 585 cbc_err: 586 crypto_unregister_alg(&blk_ecb_alg); 587 ecb_err: 588 return err; 589 } 590 591 void __exit fini(void) 592 { 593 #ifdef HAS_LRW 594 crypto_unregister_alg(&blk_lrw_alg); 595 #endif 596 crypto_unregister_alg(&blk_ctr_alg); 597 crypto_unregister_alg(&blk_cbc_alg); 598 crypto_unregister_alg(&blk_ecb_alg); 599 } 600 601 module_init(init); 602 module_exit(fini); 603 604 MODULE_LICENSE("GPL"); 605 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized"); 606 MODULE_ALIAS("twofish"); 607 MODULE_ALIAS("twofish-asm"); 608