1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized. 4 * as specified in 5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html 6 * 7 * Copyright (c) 2021, Alibaba Group. 8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11 #include <asm/fpu/api.h> 12 #include <linux/module.h> 13 #include <linux/crypto.h> 14 #include <linux/kernel.h> 15 #include <crypto/internal/skcipher.h> 16 #include <crypto/sm4.h> 17 #include "sm4-avx.h" 18 19 #define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16) 20 21 asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst, 22 const u8 *src, u8 *iv); 23 asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst, 24 const u8 *src, u8 *iv); 25 26 static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 27 unsigned int key_len) 28 { 29 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 30 31 return sm4_expandkey(ctx, key, key_len); 32 } 33 34 static int cbc_decrypt(struct skcipher_request *req) 35 { 36 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE, 37 sm4_aesni_avx2_cbc_dec_blk16); 38 } 39 40 41 static int ctr_crypt(struct skcipher_request *req) 42 { 43 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE, 44 sm4_aesni_avx2_ctr_enc_blk16); 45 } 46 47 static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { 48 { 49 .base = { 50 .cra_name = "ecb(sm4)", 51 .cra_driver_name = "ecb-sm4-aesni-avx2", 52 .cra_priority = 500, 53 .cra_blocksize = SM4_BLOCK_SIZE, 54 .cra_ctxsize = sizeof(struct sm4_ctx), 55 .cra_module = THIS_MODULE, 56 }, 57 .min_keysize = SM4_KEY_SIZE, 58 .max_keysize = SM4_KEY_SIZE, 59 .walksize = 16 * SM4_BLOCK_SIZE, 60 .setkey = sm4_skcipher_setkey, 61 .encrypt = sm4_avx_ecb_encrypt, 62 .decrypt = sm4_avx_ecb_decrypt, 63 }, { 64 .base = { 65 .cra_name = "cbc(sm4)", 66 .cra_driver_name = "cbc-sm4-aesni-avx2", 67 .cra_priority = 500, 68 .cra_blocksize = SM4_BLOCK_SIZE, 69 .cra_ctxsize = sizeof(struct sm4_ctx), 70 .cra_module = THIS_MODULE, 71 }, 72 .min_keysize = SM4_KEY_SIZE, 73 .max_keysize = SM4_KEY_SIZE, 74 .ivsize = SM4_BLOCK_SIZE, 75 .walksize = 16 * SM4_BLOCK_SIZE, 76 .setkey = sm4_skcipher_setkey, 77 .encrypt = sm4_cbc_encrypt, 78 .decrypt = cbc_decrypt, 79 }, { 80 .base = { 81 .cra_name = "ctr(sm4)", 82 .cra_driver_name = "ctr-sm4-aesni-avx2", 83 .cra_priority = 500, 84 .cra_blocksize = 1, 85 .cra_ctxsize = sizeof(struct sm4_ctx), 86 .cra_module = THIS_MODULE, 87 }, 88 .min_keysize = SM4_KEY_SIZE, 89 .max_keysize = SM4_KEY_SIZE, 90 .ivsize = SM4_BLOCK_SIZE, 91 .chunksize = SM4_BLOCK_SIZE, 92 .walksize = 16 * SM4_BLOCK_SIZE, 93 .setkey = sm4_skcipher_setkey, 94 .encrypt = ctr_crypt, 95 .decrypt = ctr_crypt, 96 } 97 }; 98 99 static int __init sm4_init(void) 100 { 101 const char *feature_name; 102 103 if (!boot_cpu_has(X86_FEATURE_AVX) || 104 !boot_cpu_has(X86_FEATURE_AVX2) || 105 !boot_cpu_has(X86_FEATURE_AES) || 106 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 107 pr_info("AVX2 or AES-NI instructions are not detected.\n"); 108 return -ENODEV; 109 } 110 111 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 112 &feature_name)) { 113 pr_info("CPU feature '%s' is not supported.\n", feature_name); 114 return -ENODEV; 115 } 116 117 return crypto_register_skciphers(sm4_aesni_avx2_skciphers, 118 ARRAY_SIZE(sm4_aesni_avx2_skciphers)); 119 } 120 121 static void __exit sm4_exit(void) 122 { 123 crypto_unregister_skciphers(sm4_aesni_avx2_skciphers, 124 ARRAY_SIZE(sm4_aesni_avx2_skciphers)); 125 } 126 127 module_init(sm4_init); 128 module_exit(sm4_exit); 129 130 MODULE_LICENSE("GPL v2"); 131 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); 132 MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized"); 133 MODULE_ALIAS_CRYPTO("sm4"); 134 MODULE_ALIAS_CRYPTO("sm4-aesni-avx2"); 135