1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent 4 * 5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/types.h> 10 #include <linux/crypto.h> 11 #include <linux/err.h> 12 #include <crypto/algapi.h> 13 #include <crypto/internal/simd.h> 14 #include <crypto/serpent.h> 15 #include <crypto/xts.h> 16 #include <asm/crypto/glue_helper.h> 17 #include <asm/crypto/serpent-avx.h> 18 19 #define SERPENT_AVX2_PARALLEL_BLOCKS 16 20 21 /* 16-way AVX2 parallel cipher functions */ 22 asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst, 23 const u8 *src); 24 asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst, 25 const u8 *src); 26 asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src); 27 28 asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src, 29 le128 *iv); 30 asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst, 31 const u8 *src, le128 *iv); 32 asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst, 33 const u8 *src, le128 *iv); 34 35 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, 36 const u8 *key, unsigned int keylen) 37 { 38 return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); 39 } 40 41 static const struct common_glue_ctx serpent_enc = { 42 .num_funcs = 3, 43 .fpu_blocks_limit = 8, 44 45 .funcs = { { 46 .num_blocks = 16, 47 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) } 48 }, { 49 .num_blocks = 8, 50 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) } 51 }, { 52 .num_blocks = 1, 53 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) } 54 } } 55 }; 56 57 static const struct common_glue_ctx serpent_ctr = { 58 .num_funcs = 3, 59 .fpu_blocks_limit = 8, 60 61 .funcs = { { 62 .num_blocks = 16, 63 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) } 64 }, { 65 .num_blocks = 8, 66 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) } 67 }, { 68 .num_blocks = 1, 69 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) } 70 } } 71 }; 72 73 static const struct common_glue_ctx serpent_enc_xts = { 74 .num_funcs = 3, 75 .fpu_blocks_limit = 8, 76 77 .funcs = { { 78 .num_blocks = 16, 79 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) } 80 }, { 81 .num_blocks = 8, 82 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) } 83 }, { 84 .num_blocks = 1, 85 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) } 86 } } 87 }; 88 89 static const struct common_glue_ctx serpent_dec = { 90 .num_funcs = 3, 91 .fpu_blocks_limit = 8, 92 93 .funcs = { { 94 .num_blocks = 16, 95 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) } 96 }, { 97 .num_blocks = 8, 98 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) } 99 }, { 100 .num_blocks = 1, 101 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) } 102 } } 103 }; 104 105 static const struct common_glue_ctx serpent_dec_cbc = { 106 .num_funcs = 3, 107 .fpu_blocks_limit = 8, 108 109 .funcs = { { 110 .num_blocks = 16, 111 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) } 112 }, { 113 .num_blocks = 8, 114 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) } 115 }, { 116 .num_blocks = 1, 117 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) } 118 } } 119 }; 120 121 static const struct common_glue_ctx serpent_dec_xts = { 122 .num_funcs = 3, 123 .fpu_blocks_limit = 8, 124 125 .funcs = { { 126 .num_blocks = 16, 127 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) } 128 }, { 129 .num_blocks = 8, 130 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) } 131 }, { 132 .num_blocks = 1, 133 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) } 134 } } 135 }; 136 137 static int ecb_encrypt(struct skcipher_request *req) 138 { 139 return glue_ecb_req_128bit(&serpent_enc, req); 140 } 141 142 static int ecb_decrypt(struct skcipher_request *req) 143 { 144 return glue_ecb_req_128bit(&serpent_dec, req); 145 } 146 147 static int cbc_encrypt(struct skcipher_request *req) 148 { 149 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt), 150 req); 151 } 152 153 static int cbc_decrypt(struct skcipher_request *req) 154 { 155 return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); 156 } 157 158 static int ctr_crypt(struct skcipher_request *req) 159 { 160 return glue_ctr_req_128bit(&serpent_ctr, req); 161 } 162 163 static int xts_encrypt(struct skcipher_request *req) 164 { 165 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 166 struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 167 168 return glue_xts_req_128bit(&serpent_enc_xts, req, 169 XTS_TWEAK_CAST(__serpent_encrypt), 170 &ctx->tweak_ctx, &ctx->crypt_ctx, false); 171 } 172 173 static int xts_decrypt(struct skcipher_request *req) 174 { 175 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 176 struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 177 178 return glue_xts_req_128bit(&serpent_dec_xts, req, 179 XTS_TWEAK_CAST(__serpent_encrypt), 180 &ctx->tweak_ctx, &ctx->crypt_ctx, true); 181 } 182 183 static struct skcipher_alg serpent_algs[] = { 184 { 185 .base.cra_name = "__ecb(serpent)", 186 .base.cra_driver_name = "__ecb-serpent-avx2", 187 .base.cra_priority = 600, 188 .base.cra_flags = CRYPTO_ALG_INTERNAL, 189 .base.cra_blocksize = SERPENT_BLOCK_SIZE, 190 .base.cra_ctxsize = sizeof(struct serpent_ctx), 191 .base.cra_module = THIS_MODULE, 192 .min_keysize = SERPENT_MIN_KEY_SIZE, 193 .max_keysize = SERPENT_MAX_KEY_SIZE, 194 .setkey = serpent_setkey_skcipher, 195 .encrypt = ecb_encrypt, 196 .decrypt = ecb_decrypt, 197 }, { 198 .base.cra_name = "__cbc(serpent)", 199 .base.cra_driver_name = "__cbc-serpent-avx2", 200 .base.cra_priority = 600, 201 .base.cra_flags = CRYPTO_ALG_INTERNAL, 202 .base.cra_blocksize = SERPENT_BLOCK_SIZE, 203 .base.cra_ctxsize = sizeof(struct serpent_ctx), 204 .base.cra_module = THIS_MODULE, 205 .min_keysize = SERPENT_MIN_KEY_SIZE, 206 .max_keysize = SERPENT_MAX_KEY_SIZE, 207 .ivsize = SERPENT_BLOCK_SIZE, 208 .setkey = serpent_setkey_skcipher, 209 .encrypt = cbc_encrypt, 210 .decrypt = cbc_decrypt, 211 }, { 212 .base.cra_name = "__ctr(serpent)", 213 .base.cra_driver_name = "__ctr-serpent-avx2", 214 .base.cra_priority = 600, 215 .base.cra_flags = CRYPTO_ALG_INTERNAL, 216 .base.cra_blocksize = 1, 217 .base.cra_ctxsize = sizeof(struct serpent_ctx), 218 .base.cra_module = THIS_MODULE, 219 .min_keysize = SERPENT_MIN_KEY_SIZE, 220 .max_keysize = SERPENT_MAX_KEY_SIZE, 221 .ivsize = SERPENT_BLOCK_SIZE, 222 .chunksize = SERPENT_BLOCK_SIZE, 223 .setkey = serpent_setkey_skcipher, 224 .encrypt = ctr_crypt, 225 .decrypt = ctr_crypt, 226 }, { 227 .base.cra_name = "__xts(serpent)", 228 .base.cra_driver_name = "__xts-serpent-avx2", 229 .base.cra_priority = 600, 230 .base.cra_flags = CRYPTO_ALG_INTERNAL, 231 .base.cra_blocksize = SERPENT_BLOCK_SIZE, 232 .base.cra_ctxsize = sizeof(struct serpent_xts_ctx), 233 .base.cra_module = THIS_MODULE, 234 .min_keysize = 2 * SERPENT_MIN_KEY_SIZE, 235 .max_keysize = 2 * SERPENT_MAX_KEY_SIZE, 236 .ivsize = SERPENT_BLOCK_SIZE, 237 .setkey = xts_serpent_setkey, 238 .encrypt = xts_encrypt, 239 .decrypt = xts_decrypt, 240 }, 241 }; 242 243 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; 244 245 static int __init init(void) 246 { 247 const char *feature_name; 248 249 if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 250 pr_info("AVX2 instructions are not detected.\n"); 251 return -ENODEV; 252 } 253 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 254 &feature_name)) { 255 pr_info("CPU feature '%s' is not supported.\n", feature_name); 256 return -ENODEV; 257 } 258 259 return simd_register_skciphers_compat(serpent_algs, 260 ARRAY_SIZE(serpent_algs), 261 serpent_simd_algs); 262 } 263 264 static void __exit fini(void) 265 { 266 simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs), 267 serpent_simd_algs); 268 } 269 270 module_init(init); 271 module_exit(fini); 272 273 MODULE_LICENSE("GPL"); 274 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized"); 275 MODULE_ALIAS_CRYPTO("serpent"); 276 MODULE_ALIAS_CRYPTO("serpent-asm"); 277