1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia 4 * 5 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 */ 7 8 #include <crypto/algapi.h> 9 #include <linux/crypto.h> 10 #include <linux/err.h> 11 #include <linux/module.h> 12 #include <linux/types.h> 13 14 #include "camellia.h" 15 #include "ecb_cbc_helpers.h" 16 17 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 18 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 19 20 /* 32-way AVX2/AES-NI parallel cipher functions */ 21 asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src); 22 asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src); 23 24 asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src); 25 26 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 27 unsigned int keylen) 28 { 29 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen); 30 } 31 32 static int ecb_encrypt(struct skcipher_request *req) 33 { 34 ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 35 ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_enc_32way); 36 ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); 37 ECB_BLOCK(2, camellia_enc_blk_2way); 38 ECB_BLOCK(1, camellia_enc_blk); 39 ECB_WALK_END(); 40 } 41 42 static int ecb_decrypt(struct skcipher_request *req) 43 { 44 ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 45 ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_dec_32way); 46 ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); 47 ECB_BLOCK(2, camellia_dec_blk_2way); 48 ECB_BLOCK(1, camellia_dec_blk); 49 ECB_WALK_END(); 50 } 51 52 static int cbc_encrypt(struct skcipher_request *req) 53 { 54 CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); 55 CBC_ENC_BLOCK(camellia_enc_blk); 56 CBC_WALK_END(); 57 } 58 59 static int cbc_decrypt(struct skcipher_request *req) 60 { 61 CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 62 CBC_DEC_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_cbc_dec_32way); 63 CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); 64 CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); 65 CBC_DEC_BLOCK(1, camellia_dec_blk); 66 CBC_WALK_END(); 67 } 68 69 static struct skcipher_alg camellia_algs[] = { 70 { 71 .base.cra_name = "ecb(camellia)", 72 .base.cra_driver_name = "ecb-camellia-aesni-avx2", 73 .base.cra_priority = 500, 74 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 75 .base.cra_ctxsize = sizeof(struct camellia_ctx), 76 .base.cra_module = THIS_MODULE, 77 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 78 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 79 .setkey = camellia_setkey, 80 .encrypt = ecb_encrypt, 81 .decrypt = ecb_decrypt, 82 }, { 83 .base.cra_name = "cbc(camellia)", 84 .base.cra_driver_name = "cbc-camellia-aesni-avx2", 85 .base.cra_priority = 500, 86 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 87 .base.cra_ctxsize = sizeof(struct camellia_ctx), 88 .base.cra_module = THIS_MODULE, 89 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 90 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 91 .ivsize = CAMELLIA_BLOCK_SIZE, 92 .setkey = camellia_setkey, 93 .encrypt = cbc_encrypt, 94 .decrypt = cbc_decrypt, 95 }, 96 }; 97 98 static int __init camellia_aesni_init(void) 99 { 100 const char *feature_name; 101 102 if (!boot_cpu_has(X86_FEATURE_AVX) || 103 !boot_cpu_has(X86_FEATURE_AVX2) || 104 !boot_cpu_has(X86_FEATURE_AES) || 105 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 106 pr_info("AVX2 or AES-NI instructions are not detected.\n"); 107 return -ENODEV; 108 } 109 110 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 111 &feature_name)) { 112 pr_info("CPU feature '%s' is not supported.\n", feature_name); 113 return -ENODEV; 114 } 115 116 return crypto_register_skciphers(camellia_algs, 117 ARRAY_SIZE(camellia_algs)); 118 } 119 120 static void __exit camellia_aesni_fini(void) 121 { 122 crypto_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs)); 123 } 124 125 module_init(camellia_aesni_init); 126 module_exit(camellia_aesni_fini); 127 128 MODULE_LICENSE("GPL"); 129 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized"); 130 MODULE_ALIAS_CRYPTO("camellia"); 131 MODULE_ALIAS_CRYPTO("camellia-asm"); 132