1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for AVX assembler version of Twofish Cipher 4 * 5 * Copyright (C) 2012 Johannes Goetzfried 6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 7 * 8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/types.h> 13 #include <linux/crypto.h> 14 #include <linux/err.h> 15 #include <crypto/algapi.h> 16 #include <crypto/internal/simd.h> 17 #include <crypto/twofish.h> 18 #include <crypto/xts.h> 19 #include <asm/crypto/glue_helper.h> 20 #include <asm/crypto/twofish.h> 21 22 #define TWOFISH_PARALLEL_BLOCKS 8 23 24 /* 8-way parallel cipher functions */ 25 asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); 26 asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); 27 28 asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); 29 asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src, 30 le128 *iv); 31 32 asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, 33 le128 *iv); 34 asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, 35 le128 *iv); 36 37 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, 38 const u8 *key, unsigned int keylen) 39 { 40 return twofish_setkey(&tfm->base, key, keylen); 41 } 42 43 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) 44 { 45 __twofish_enc_blk_3way(ctx, dst, src, false); 46 } 47 48 static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) 49 { 50 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk); 51 } 52 53 static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) 54 { 55 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk); 56 } 57 58 struct twofish_xts_ctx { 59 struct twofish_ctx tweak_ctx; 60 struct twofish_ctx crypt_ctx; 61 }; 62 63 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, 64 unsigned int keylen) 65 { 66 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 67 int err; 68 69 err = xts_verify_key(tfm, key, keylen); 70 if (err) 71 return err; 72 73 /* first half of xts-key is for crypt */ 74 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2); 75 if (err) 76 return err; 77 78 /* second half of xts-key is for tweak */ 79 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); 80 } 81 82 static const struct common_glue_ctx twofish_enc = { 83 .num_funcs = 3, 84 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 85 86 .funcs = { { 87 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 88 .fn_u = { .ecb = twofish_ecb_enc_8way } 89 }, { 90 .num_blocks = 3, 91 .fn_u = { .ecb = twofish_enc_blk_3way } 92 }, { 93 .num_blocks = 1, 94 .fn_u = { .ecb = twofish_enc_blk } 95 } } 96 }; 97 98 static const struct common_glue_ctx twofish_ctr = { 99 .num_funcs = 3, 100 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 101 102 .funcs = { { 103 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 104 .fn_u = { .ctr = twofish_ctr_8way } 105 }, { 106 .num_blocks = 3, 107 .fn_u = { .ctr = twofish_enc_blk_ctr_3way } 108 }, { 109 .num_blocks = 1, 110 .fn_u = { .ctr = twofish_enc_blk_ctr } 111 } } 112 }; 113 114 static const struct common_glue_ctx twofish_enc_xts = { 115 .num_funcs = 2, 116 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 117 118 .funcs = { { 119 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 120 .fn_u = { .xts = twofish_xts_enc_8way } 121 }, { 122 .num_blocks = 1, 123 .fn_u = { .xts = twofish_xts_enc } 124 } } 125 }; 126 127 static const struct common_glue_ctx twofish_dec = { 128 .num_funcs = 3, 129 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 130 131 .funcs = { { 132 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 133 .fn_u = { .ecb = twofish_ecb_dec_8way } 134 }, { 135 .num_blocks = 3, 136 .fn_u = { .ecb = twofish_dec_blk_3way } 137 }, { 138 .num_blocks = 1, 139 .fn_u = { .ecb = twofish_dec_blk } 140 } } 141 }; 142 143 static const struct common_glue_ctx twofish_dec_cbc = { 144 .num_funcs = 3, 145 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 146 147 .funcs = { { 148 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 149 .fn_u = { .cbc = twofish_cbc_dec_8way } 150 }, { 151 .num_blocks = 3, 152 .fn_u = { .cbc = twofish_dec_blk_cbc_3way } 153 }, { 154 .num_blocks = 1, 155 .fn_u = { .cbc = twofish_dec_blk } 156 } } 157 }; 158 159 static const struct common_glue_ctx twofish_dec_xts = { 160 .num_funcs = 2, 161 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 162 163 .funcs = { { 164 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 165 .fn_u = { .xts = twofish_xts_dec_8way } 166 }, { 167 .num_blocks = 1, 168 .fn_u = { .xts = twofish_xts_dec } 169 } } 170 }; 171 172 static int ecb_encrypt(struct skcipher_request *req) 173 { 174 return glue_ecb_req_128bit(&twofish_enc, req); 175 } 176 177 static int ecb_decrypt(struct skcipher_request *req) 178 { 179 return glue_ecb_req_128bit(&twofish_dec, req); 180 } 181 182 static int cbc_encrypt(struct skcipher_request *req) 183 { 184 return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); 185 } 186 187 static int cbc_decrypt(struct skcipher_request *req) 188 { 189 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); 190 } 191 192 static int ctr_crypt(struct skcipher_request *req) 193 { 194 return glue_ctr_req_128bit(&twofish_ctr, req); 195 } 196 197 static int xts_encrypt(struct skcipher_request *req) 198 { 199 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 200 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 201 202 return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk, 203 &ctx->tweak_ctx, &ctx->crypt_ctx, false); 204 } 205 206 static int xts_decrypt(struct skcipher_request *req) 207 { 208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 210 211 return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk, 212 &ctx->tweak_ctx, &ctx->crypt_ctx, true); 213 } 214 215 static struct skcipher_alg twofish_algs[] = { 216 { 217 .base.cra_name = "__ecb(twofish)", 218 .base.cra_driver_name = "__ecb-twofish-avx", 219 .base.cra_priority = 400, 220 .base.cra_flags = CRYPTO_ALG_INTERNAL, 221 .base.cra_blocksize = TF_BLOCK_SIZE, 222 .base.cra_ctxsize = sizeof(struct twofish_ctx), 223 .base.cra_module = THIS_MODULE, 224 .min_keysize = TF_MIN_KEY_SIZE, 225 .max_keysize = TF_MAX_KEY_SIZE, 226 .setkey = twofish_setkey_skcipher, 227 .encrypt = ecb_encrypt, 228 .decrypt = ecb_decrypt, 229 }, { 230 .base.cra_name = "__cbc(twofish)", 231 .base.cra_driver_name = "__cbc-twofish-avx", 232 .base.cra_priority = 400, 233 .base.cra_flags = CRYPTO_ALG_INTERNAL, 234 .base.cra_blocksize = TF_BLOCK_SIZE, 235 .base.cra_ctxsize = sizeof(struct twofish_ctx), 236 .base.cra_module = THIS_MODULE, 237 .min_keysize = TF_MIN_KEY_SIZE, 238 .max_keysize = TF_MAX_KEY_SIZE, 239 .ivsize = TF_BLOCK_SIZE, 240 .setkey = twofish_setkey_skcipher, 241 .encrypt = cbc_encrypt, 242 .decrypt = cbc_decrypt, 243 }, { 244 .base.cra_name = "__ctr(twofish)", 245 .base.cra_driver_name = "__ctr-twofish-avx", 246 .base.cra_priority = 400, 247 .base.cra_flags = CRYPTO_ALG_INTERNAL, 248 .base.cra_blocksize = 1, 249 .base.cra_ctxsize = sizeof(struct twofish_ctx), 250 .base.cra_module = THIS_MODULE, 251 .min_keysize = TF_MIN_KEY_SIZE, 252 .max_keysize = TF_MAX_KEY_SIZE, 253 .ivsize = TF_BLOCK_SIZE, 254 .chunksize = TF_BLOCK_SIZE, 255 .setkey = twofish_setkey_skcipher, 256 .encrypt = ctr_crypt, 257 .decrypt = ctr_crypt, 258 }, { 259 .base.cra_name = "__xts(twofish)", 260 .base.cra_driver_name = "__xts-twofish-avx", 261 .base.cra_priority = 400, 262 .base.cra_flags = CRYPTO_ALG_INTERNAL, 263 .base.cra_blocksize = TF_BLOCK_SIZE, 264 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), 265 .base.cra_module = THIS_MODULE, 266 .min_keysize = 2 * TF_MIN_KEY_SIZE, 267 .max_keysize = 2 * TF_MAX_KEY_SIZE, 268 .ivsize = TF_BLOCK_SIZE, 269 .setkey = xts_twofish_setkey, 270 .encrypt = xts_encrypt, 271 .decrypt = xts_decrypt, 272 }, 273 }; 274 275 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; 276 277 static int __init twofish_init(void) 278 { 279 const char *feature_name; 280 281 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { 282 pr_info("CPU feature '%s' is not supported.\n", feature_name); 283 return -ENODEV; 284 } 285 286 return simd_register_skciphers_compat(twofish_algs, 287 ARRAY_SIZE(twofish_algs), 288 twofish_simd_algs); 289 } 290 291 static void __exit twofish_exit(void) 292 { 293 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), 294 twofish_simd_algs); 295 } 296 297 module_init(twofish_init); 298 module_exit(twofish_exit); 299 300 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 301 MODULE_LICENSE("GPL"); 302 MODULE_ALIAS_CRYPTO("twofish"); 303