1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for assembler optimized version of Blowfish 4 * 5 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 * 7 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 9 */ 10 11 #include <crypto/algapi.h> 12 #include <crypto/blowfish.h> 13 #include <crypto/internal/skcipher.h> 14 #include <linux/crypto.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/types.h> 18 19 /* regular block cipher functions */ 20 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src, 21 bool xor); 22 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src); 23 24 /* 4-way parallel cipher functions */ 25 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 26 const u8 *src, bool xor); 27 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst, 28 const u8 *src); 29 30 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src) 31 { 32 __blowfish_enc_blk(ctx, dst, src, false); 33 } 34 35 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 36 const u8 *src) 37 { 38 __blowfish_enc_blk_4way(ctx, dst, src, false); 39 } 40 41 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 42 { 43 blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src); 44 } 45 46 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 47 { 48 blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src); 49 } 50 51 static int blowfish_setkey_skcipher(struct crypto_skcipher *tfm, 52 const u8 *key, unsigned int keylen) 53 { 54 return blowfish_setkey(&tfm->base, key, keylen); 55 } 56 57 static int ecb_crypt(struct skcipher_request *req, 58 void (*fn)(struct bf_ctx *, u8 *, const u8 *), 59 void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *)) 60 { 61 unsigned int bsize = BF_BLOCK_SIZE; 62 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 63 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 64 struct skcipher_walk walk; 65 unsigned int nbytes; 66 int err; 67 68 err = skcipher_walk_virt(&walk, req, false); 69 70 while ((nbytes = walk.nbytes)) { 71 u8 *wsrc = walk.src.virt.addr; 72 u8 *wdst = walk.dst.virt.addr; 73 74 /* Process four block batch */ 75 if (nbytes >= bsize * 4) { 76 do { 77 fn_4way(ctx, wdst, wsrc); 78 79 wsrc += bsize * 4; 80 wdst += bsize * 4; 81 nbytes -= bsize * 4; 82 } while (nbytes >= bsize * 4); 83 84 if (nbytes < bsize) 85 goto done; 86 } 87 88 /* Handle leftovers */ 89 do { 90 fn(ctx, wdst, wsrc); 91 92 wsrc += bsize; 93 wdst += bsize; 94 nbytes -= bsize; 95 } while (nbytes >= bsize); 96 97 done: 98 err = skcipher_walk_done(&walk, nbytes); 99 } 100 101 return err; 102 } 103 104 static int ecb_encrypt(struct skcipher_request *req) 105 { 106 return ecb_crypt(req, blowfish_enc_blk, blowfish_enc_blk_4way); 107 } 108 109 static int ecb_decrypt(struct skcipher_request *req) 110 { 111 return ecb_crypt(req, blowfish_dec_blk, blowfish_dec_blk_4way); 112 } 113 114 static unsigned int __cbc_encrypt(struct bf_ctx *ctx, 115 struct skcipher_walk *walk) 116 { 117 unsigned int bsize = BF_BLOCK_SIZE; 118 unsigned int nbytes = walk->nbytes; 119 u64 *src = (u64 *)walk->src.virt.addr; 120 u64 *dst = (u64 *)walk->dst.virt.addr; 121 u64 *iv = (u64 *)walk->iv; 122 123 do { 124 *dst = *src ^ *iv; 125 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 126 iv = dst; 127 128 src += 1; 129 dst += 1; 130 nbytes -= bsize; 131 } while (nbytes >= bsize); 132 133 *(u64 *)walk->iv = *iv; 134 return nbytes; 135 } 136 137 static int cbc_encrypt(struct skcipher_request *req) 138 { 139 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 140 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 141 struct skcipher_walk walk; 142 unsigned int nbytes; 143 int err; 144 145 err = skcipher_walk_virt(&walk, req, false); 146 147 while ((nbytes = walk.nbytes)) { 148 nbytes = __cbc_encrypt(ctx, &walk); 149 err = skcipher_walk_done(&walk, nbytes); 150 } 151 152 return err; 153 } 154 155 static unsigned int __cbc_decrypt(struct bf_ctx *ctx, 156 struct skcipher_walk *walk) 157 { 158 unsigned int bsize = BF_BLOCK_SIZE; 159 unsigned int nbytes = walk->nbytes; 160 u64 *src = (u64 *)walk->src.virt.addr; 161 u64 *dst = (u64 *)walk->dst.virt.addr; 162 u64 ivs[4 - 1]; 163 u64 last_iv; 164 165 /* Start of the last block. */ 166 src += nbytes / bsize - 1; 167 dst += nbytes / bsize - 1; 168 169 last_iv = *src; 170 171 /* Process four block batch */ 172 if (nbytes >= bsize * 4) { 173 do { 174 nbytes -= bsize * 4 - bsize; 175 src -= 4 - 1; 176 dst -= 4 - 1; 177 178 ivs[0] = src[0]; 179 ivs[1] = src[1]; 180 ivs[2] = src[2]; 181 182 blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src); 183 184 dst[1] ^= ivs[0]; 185 dst[2] ^= ivs[1]; 186 dst[3] ^= ivs[2]; 187 188 nbytes -= bsize; 189 if (nbytes < bsize) 190 goto done; 191 192 *dst ^= *(src - 1); 193 src -= 1; 194 dst -= 1; 195 } while (nbytes >= bsize * 4); 196 } 197 198 /* Handle leftovers */ 199 for (;;) { 200 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); 201 202 nbytes -= bsize; 203 if (nbytes < bsize) 204 break; 205 206 *dst ^= *(src - 1); 207 src -= 1; 208 dst -= 1; 209 } 210 211 done: 212 *dst ^= *(u64 *)walk->iv; 213 *(u64 *)walk->iv = last_iv; 214 215 return nbytes; 216 } 217 218 static int cbc_decrypt(struct skcipher_request *req) 219 { 220 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 221 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 222 struct skcipher_walk walk; 223 unsigned int nbytes; 224 int err; 225 226 err = skcipher_walk_virt(&walk, req, false); 227 228 while ((nbytes = walk.nbytes)) { 229 nbytes = __cbc_decrypt(ctx, &walk); 230 err = skcipher_walk_done(&walk, nbytes); 231 } 232 233 return err; 234 } 235 236 static struct crypto_alg bf_cipher_alg = { 237 .cra_name = "blowfish", 238 .cra_driver_name = "blowfish-asm", 239 .cra_priority = 200, 240 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 241 .cra_blocksize = BF_BLOCK_SIZE, 242 .cra_ctxsize = sizeof(struct bf_ctx), 243 .cra_alignmask = 0, 244 .cra_module = THIS_MODULE, 245 .cra_u = { 246 .cipher = { 247 .cia_min_keysize = BF_MIN_KEY_SIZE, 248 .cia_max_keysize = BF_MAX_KEY_SIZE, 249 .cia_setkey = blowfish_setkey, 250 .cia_encrypt = blowfish_encrypt, 251 .cia_decrypt = blowfish_decrypt, 252 } 253 } 254 }; 255 256 static struct skcipher_alg bf_skcipher_algs[] = { 257 { 258 .base.cra_name = "ecb(blowfish)", 259 .base.cra_driver_name = "ecb-blowfish-asm", 260 .base.cra_priority = 300, 261 .base.cra_blocksize = BF_BLOCK_SIZE, 262 .base.cra_ctxsize = sizeof(struct bf_ctx), 263 .base.cra_module = THIS_MODULE, 264 .min_keysize = BF_MIN_KEY_SIZE, 265 .max_keysize = BF_MAX_KEY_SIZE, 266 .setkey = blowfish_setkey_skcipher, 267 .encrypt = ecb_encrypt, 268 .decrypt = ecb_decrypt, 269 }, { 270 .base.cra_name = "cbc(blowfish)", 271 .base.cra_driver_name = "cbc-blowfish-asm", 272 .base.cra_priority = 300, 273 .base.cra_blocksize = BF_BLOCK_SIZE, 274 .base.cra_ctxsize = sizeof(struct bf_ctx), 275 .base.cra_module = THIS_MODULE, 276 .min_keysize = BF_MIN_KEY_SIZE, 277 .max_keysize = BF_MAX_KEY_SIZE, 278 .ivsize = BF_BLOCK_SIZE, 279 .setkey = blowfish_setkey_skcipher, 280 .encrypt = cbc_encrypt, 281 .decrypt = cbc_decrypt, 282 }, 283 }; 284 285 static bool is_blacklisted_cpu(void) 286 { 287 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 288 return false; 289 290 if (boot_cpu_data.x86 == 0x0f) { 291 /* 292 * On Pentium 4, blowfish-x86_64 is slower than generic C 293 * implementation because use of 64bit rotates (which are really 294 * slow on P4). Therefore blacklist P4s. 295 */ 296 return true; 297 } 298 299 return false; 300 } 301 302 static int force; 303 module_param(force, int, 0); 304 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 305 306 static int __init blowfish_init(void) 307 { 308 int err; 309 310 if (!force && is_blacklisted_cpu()) { 311 printk(KERN_INFO 312 "blowfish-x86_64: performance on this CPU " 313 "would be suboptimal: disabling " 314 "blowfish-x86_64.\n"); 315 return -ENODEV; 316 } 317 318 err = crypto_register_alg(&bf_cipher_alg); 319 if (err) 320 return err; 321 322 err = crypto_register_skciphers(bf_skcipher_algs, 323 ARRAY_SIZE(bf_skcipher_algs)); 324 if (err) 325 crypto_unregister_alg(&bf_cipher_alg); 326 327 return err; 328 } 329 330 static void __exit blowfish_fini(void) 331 { 332 crypto_unregister_alg(&bf_cipher_alg); 333 crypto_unregister_skciphers(bf_skcipher_algs, 334 ARRAY_SIZE(bf_skcipher_algs)); 335 } 336 337 module_init(blowfish_init); 338 module_exit(blowfish_fini); 339 340 MODULE_LICENSE("GPL"); 341 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized"); 342 MODULE_ALIAS_CRYPTO("blowfish"); 343 MODULE_ALIAS_CRYPTO("blowfish-asm"); 344