1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia 4 * 5 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 */ 7 8 #include <crypto/algapi.h> 9 #include <crypto/internal/simd.h> 10 #include <linux/crypto.h> 11 #include <linux/err.h> 12 #include <linux/module.h> 13 #include <linux/types.h> 14 15 #include "camellia.h" 16 #include "ecb_cbc_helpers.h" 17 18 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 19 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 20 21 /* 32-way AVX2/AES-NI parallel cipher functions */ 22 asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src); 23 asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src); 24 25 asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src); 26 27 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 28 unsigned int keylen) 29 { 30 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen); 31 } 32 33 static int ecb_encrypt(struct skcipher_request *req) 34 { 35 ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 36 ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_enc_32way); 37 ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); 38 ECB_BLOCK(2, camellia_enc_blk_2way); 39 ECB_BLOCK(1, camellia_enc_blk); 40 ECB_WALK_END(); 41 } 42 43 static int ecb_decrypt(struct skcipher_request *req) 44 { 45 ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 46 ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_dec_32way); 47 ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); 48 ECB_BLOCK(2, camellia_dec_blk_2way); 49 ECB_BLOCK(1, camellia_dec_blk); 50 ECB_WALK_END(); 51 } 52 53 static int cbc_encrypt(struct skcipher_request *req) 54 { 55 CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); 56 CBC_ENC_BLOCK(camellia_enc_blk); 57 CBC_WALK_END(); 58 } 59 60 static int cbc_decrypt(struct skcipher_request *req) 61 { 62 CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 63 CBC_DEC_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_cbc_dec_32way); 64 CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); 65 CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); 66 CBC_DEC_BLOCK(1, camellia_dec_blk); 67 CBC_WALK_END(); 68 } 69 70 static struct skcipher_alg camellia_algs[] = { 71 { 72 .base.cra_name = "__ecb(camellia)", 73 .base.cra_driver_name = "__ecb-camellia-aesni-avx2", 74 .base.cra_priority = 500, 75 .base.cra_flags = CRYPTO_ALG_INTERNAL, 76 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 77 .base.cra_ctxsize = sizeof(struct camellia_ctx), 78 .base.cra_module = THIS_MODULE, 79 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 80 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 81 .setkey = camellia_setkey, 82 .encrypt = ecb_encrypt, 83 .decrypt = ecb_decrypt, 84 }, { 85 .base.cra_name = "__cbc(camellia)", 86 .base.cra_driver_name = "__cbc-camellia-aesni-avx2", 87 .base.cra_priority = 500, 88 .base.cra_flags = CRYPTO_ALG_INTERNAL, 89 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 90 .base.cra_ctxsize = sizeof(struct camellia_ctx), 91 .base.cra_module = THIS_MODULE, 92 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 93 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 94 .ivsize = CAMELLIA_BLOCK_SIZE, 95 .setkey = camellia_setkey, 96 .encrypt = cbc_encrypt, 97 .decrypt = cbc_decrypt, 98 }, 99 }; 100 101 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; 102 103 static int __init camellia_aesni_init(void) 104 { 105 const char *feature_name; 106 107 if (!boot_cpu_has(X86_FEATURE_AVX) || 108 !boot_cpu_has(X86_FEATURE_AVX2) || 109 !boot_cpu_has(X86_FEATURE_AES) || 110 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 111 pr_info("AVX2 or AES-NI instructions are not detected.\n"); 112 return -ENODEV; 113 } 114 115 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 116 &feature_name)) { 117 pr_info("CPU feature '%s' is not supported.\n", feature_name); 118 return -ENODEV; 119 } 120 121 return simd_register_skciphers_compat(camellia_algs, 122 ARRAY_SIZE(camellia_algs), 123 camellia_simd_algs); 124 } 125 126 static void __exit camellia_aesni_fini(void) 127 { 128 simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), 129 camellia_simd_algs); 130 } 131 132 module_init(camellia_aesni_init); 133 module_exit(camellia_aesni_fini); 134 135 MODULE_LICENSE("GPL"); 136 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized"); 137 MODULE_ALIAS_CRYPTO("camellia"); 138 MODULE_ALIAS_CRYPTO("camellia-asm"); 139