1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for 3-way parallel assembler optimized version of Twofish 4 * 5 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 */ 7 8 #include <asm/cpu_device_id.h> 9 #include <crypto/algapi.h> 10 #include <crypto/twofish.h> 11 #include <linux/crypto.h> 12 #include <linux/init.h> 13 #include <linux/module.h> 14 #include <linux/types.h> 15 16 #include "twofish.h" 17 #include "ecb_cbc_helpers.h" 18 19 EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way); 20 EXPORT_SYMBOL_GPL(twofish_dec_blk_3way); 21 22 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, 23 const u8 *key, unsigned int keylen) 24 { 25 return twofish_setkey(&tfm->base, key, keylen); 26 } 27 28 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) 29 { 30 __twofish_enc_blk_3way(ctx, dst, src, false); 31 } 32 33 void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src) 34 { 35 u8 buf[2][TF_BLOCK_SIZE]; 36 const u8 *s = src; 37 38 if (dst == src) 39 s = memcpy(buf, src, sizeof(buf)); 40 twofish_dec_blk_3way(ctx, dst, src); 41 crypto_xor(dst + TF_BLOCK_SIZE, s, sizeof(buf)); 42 43 } 44 EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way); 45 46 static int ecb_encrypt(struct skcipher_request *req) 47 { 48 ECB_WALK_START(req, TF_BLOCK_SIZE, -1); 49 ECB_BLOCK(3, twofish_enc_blk_3way); 50 ECB_BLOCK(1, twofish_enc_blk); 51 ECB_WALK_END(); 52 } 53 54 static int ecb_decrypt(struct skcipher_request *req) 55 { 56 ECB_WALK_START(req, TF_BLOCK_SIZE, -1); 57 ECB_BLOCK(3, twofish_dec_blk_3way); 58 ECB_BLOCK(1, twofish_dec_blk); 59 ECB_WALK_END(); 60 } 61 62 static int cbc_encrypt(struct skcipher_request *req) 63 { 64 CBC_WALK_START(req, TF_BLOCK_SIZE, -1); 65 CBC_ENC_BLOCK(twofish_enc_blk); 66 CBC_WALK_END(); 67 } 68 69 static int cbc_decrypt(struct skcipher_request *req) 70 { 71 CBC_WALK_START(req, TF_BLOCK_SIZE, -1); 72 CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way); 73 CBC_DEC_BLOCK(1, twofish_dec_blk); 74 CBC_WALK_END(); 75 } 76 77 static struct skcipher_alg tf_skciphers[] = { 78 { 79 .base.cra_name = "ecb(twofish)", 80 .base.cra_driver_name = "ecb-twofish-3way", 81 .base.cra_priority = 300, 82 .base.cra_blocksize = TF_BLOCK_SIZE, 83 .base.cra_ctxsize = sizeof(struct twofish_ctx), 84 .base.cra_module = THIS_MODULE, 85 .min_keysize = TF_MIN_KEY_SIZE, 86 .max_keysize = TF_MAX_KEY_SIZE, 87 .setkey = twofish_setkey_skcipher, 88 .encrypt = ecb_encrypt, 89 .decrypt = ecb_decrypt, 90 }, { 91 .base.cra_name = "cbc(twofish)", 92 .base.cra_driver_name = "cbc-twofish-3way", 93 .base.cra_priority = 300, 94 .base.cra_blocksize = TF_BLOCK_SIZE, 95 .base.cra_ctxsize = sizeof(struct twofish_ctx), 96 .base.cra_module = THIS_MODULE, 97 .min_keysize = TF_MIN_KEY_SIZE, 98 .max_keysize = TF_MAX_KEY_SIZE, 99 .ivsize = TF_BLOCK_SIZE, 100 .setkey = twofish_setkey_skcipher, 101 .encrypt = cbc_encrypt, 102 .decrypt = cbc_decrypt, 103 }, 104 }; 105 106 static bool is_blacklisted_cpu(void) 107 { 108 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 109 return false; 110 111 switch (boot_cpu_data.x86_vfm) { 112 case INTEL_ATOM_BONNELL: 113 case INTEL_ATOM_BONNELL_MID: 114 case INTEL_ATOM_SALTWELL: 115 /* 116 * On Atom, twofish-3way is slower than original assembler 117 * implementation. Twofish-3way trades off some performance in 118 * storing blocks in 64bit registers to allow three blocks to 119 * be processed parallel. Parallel operation then allows gaining 120 * more performance than was trade off, on out-of-order CPUs. 121 * However Atom does not benefit from this parallelism and 122 * should be blacklisted. 123 */ 124 return true; 125 } 126 127 if (boot_cpu_data.x86 == 0x0f) { 128 /* 129 * On Pentium 4, twofish-3way is slower than original assembler 130 * implementation because excessive uses of 64bit rotate and 131 * left-shifts (which are really slow on P4) needed to store and 132 * handle 128bit block in two 64bit registers. 133 */ 134 return true; 135 } 136 137 return false; 138 } 139 140 static int force; 141 module_param(force, int, 0); 142 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 143 144 static int __init twofish_3way_init(void) 145 { 146 if (!force && is_blacklisted_cpu()) { 147 printk(KERN_INFO 148 "twofish-x86_64-3way: performance on this CPU " 149 "would be suboptimal: disabling " 150 "twofish-x86_64-3way.\n"); 151 return -ENODEV; 152 } 153 154 return crypto_register_skciphers(tf_skciphers, 155 ARRAY_SIZE(tf_skciphers)); 156 } 157 158 static void __exit twofish_3way_fini(void) 159 { 160 crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers)); 161 } 162 163 module_init(twofish_3way_init); 164 module_exit(twofish_3way_fini); 165 166 MODULE_LICENSE("GPL"); 167 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized"); 168 MODULE_ALIAS_CRYPTO("twofish"); 169 MODULE_ALIAS_CRYPTO("twofish-asm"); 170