1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia 4 * 5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 6 */ 7 8 #include <crypto/algapi.h> 9 #include <crypto/internal/simd.h> 10 #include <linux/crypto.h> 11 #include <linux/err.h> 12 #include <linux/module.h> 13 #include <linux/types.h> 14 15 #include "camellia.h" 16 #include "ecb_cbc_helpers.h" 17 18 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 19 20 /* 16-way parallel cipher functions (avx/aes-ni) */ 21 asmlinkage void camellia_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); 22 EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way); 23 24 asmlinkage void camellia_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); 25 EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way); 26 27 asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); 28 EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way); 29 30 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 31 unsigned int keylen) 32 { 33 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen); 34 } 35 36 static int ecb_encrypt(struct skcipher_request *req) 37 { 38 ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 39 ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); 40 ECB_BLOCK(2, camellia_enc_blk_2way); 41 ECB_BLOCK(1, camellia_enc_blk); 42 ECB_WALK_END(); 43 } 44 45 static int ecb_decrypt(struct skcipher_request *req) 46 { 47 ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 48 ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); 49 ECB_BLOCK(2, camellia_dec_blk_2way); 50 ECB_BLOCK(1, camellia_dec_blk); 51 ECB_WALK_END(); 52 } 53 54 static int cbc_encrypt(struct skcipher_request *req) 55 { 56 CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); 57 CBC_ENC_BLOCK(camellia_enc_blk); 58 CBC_WALK_END(); 59 } 60 61 static int cbc_decrypt(struct skcipher_request *req) 62 { 63 CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); 64 CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); 65 CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); 66 CBC_DEC_BLOCK(1, camellia_dec_blk); 67 CBC_WALK_END(); 68 } 69 70 static struct skcipher_alg camellia_algs[] = { 71 { 72 .base.cra_name = "__ecb(camellia)", 73 .base.cra_driver_name = "__ecb-camellia-aesni", 74 .base.cra_priority = 400, 75 .base.cra_flags = CRYPTO_ALG_INTERNAL, 76 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 77 .base.cra_ctxsize = sizeof(struct camellia_ctx), 78 .base.cra_module = THIS_MODULE, 79 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 80 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 81 .setkey = camellia_setkey, 82 .encrypt = ecb_encrypt, 83 .decrypt = ecb_decrypt, 84 }, { 85 .base.cra_name = "__cbc(camellia)", 86 .base.cra_driver_name = "__cbc-camellia-aesni", 87 .base.cra_priority = 400, 88 .base.cra_flags = CRYPTO_ALG_INTERNAL, 89 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 90 .base.cra_ctxsize = sizeof(struct camellia_ctx), 91 .base.cra_module = THIS_MODULE, 92 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 93 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 94 .ivsize = CAMELLIA_BLOCK_SIZE, 95 .setkey = camellia_setkey, 96 .encrypt = cbc_encrypt, 97 .decrypt = cbc_decrypt, 98 } 99 }; 100 101 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; 102 103 static int __init camellia_aesni_init(void) 104 { 105 const char *feature_name; 106 107 if (!boot_cpu_has(X86_FEATURE_AVX) || 108 !boot_cpu_has(X86_FEATURE_AES) || 109 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 110 pr_info("AVX or AES-NI instructions are not detected.\n"); 111 return -ENODEV; 112 } 113 114 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 115 &feature_name)) { 116 pr_info("CPU feature '%s' is not supported.\n", feature_name); 117 return -ENODEV; 118 } 119 120 return simd_register_skciphers_compat(camellia_algs, 121 ARRAY_SIZE(camellia_algs), 122 camellia_simd_algs); 123 } 124 125 static void __exit camellia_aesni_fini(void) 126 { 127 simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), 128 camellia_simd_algs); 129 } 130 131 module_init(camellia_aesni_init); 132 module_exit(camellia_aesni_fini); 133 134 MODULE_LICENSE("GPL"); 135 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized"); 136 MODULE_ALIAS_CRYPTO("camellia"); 137 MODULE_ALIAS_CRYPTO("camellia-asm"); 138