1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * SM4 Cipher Algorithm, using ARMv8 NEON 4 * as specified in 5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html 6 * 7 * Copyright (C) 2022, Alibaba Group. 8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/crypto.h> 13 #include <linux/kernel.h> 14 #include <linux/cpufeature.h> 15 #include <asm/neon.h> 16 #include <asm/simd.h> 17 #include <crypto/internal/simd.h> 18 #include <crypto/internal/skcipher.h> 19 #include <crypto/sm4.h> 20 21 asmlinkage void sm4_neon_crypt(const u32 *rkey, u8 *dst, const u8 *src, 22 unsigned int nblocks); 23 asmlinkage void sm4_neon_cbc_dec(const u32 *rkey_dec, u8 *dst, const u8 *src, 24 u8 *iv, unsigned int nblocks); 25 asmlinkage void sm4_neon_ctr_crypt(const u32 *rkey_enc, u8 *dst, const u8 *src, 26 u8 *iv, unsigned int nblocks); 27 28 static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key, 29 unsigned int key_len) 30 { 31 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 32 33 return sm4_expandkey(ctx, key, key_len); 34 } 35 36 static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey) 37 { 38 struct skcipher_walk walk; 39 unsigned int nbytes; 40 int err; 41 42 err = skcipher_walk_virt(&walk, req, false); 43 44 while ((nbytes = walk.nbytes) > 0) { 45 const u8 *src = walk.src.virt.addr; 46 u8 *dst = walk.dst.virt.addr; 47 unsigned int nblocks; 48 49 nblocks = nbytes / SM4_BLOCK_SIZE; 50 if (nblocks) { 51 kernel_neon_begin(); 52 53 sm4_neon_crypt(rkey, dst, src, nblocks); 54 55 kernel_neon_end(); 56 } 57 58 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); 59 } 60 61 return err; 62 } 63 64 static int sm4_ecb_encrypt(struct skcipher_request *req) 65 { 66 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 67 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 68 69 return sm4_ecb_do_crypt(req, ctx->rkey_enc); 70 } 71 72 static int sm4_ecb_decrypt(struct skcipher_request *req) 73 { 74 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 75 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 76 77 return sm4_ecb_do_crypt(req, ctx->rkey_dec); 78 } 79 80 static int sm4_cbc_encrypt(struct skcipher_request *req) 81 { 82 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 83 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 84 struct skcipher_walk walk; 85 unsigned int nbytes; 86 int err; 87 88 err = skcipher_walk_virt(&walk, req, false); 89 90 while ((nbytes = walk.nbytes) > 0) { 91 const u8 *iv = walk.iv; 92 const u8 *src = walk.src.virt.addr; 93 u8 *dst = walk.dst.virt.addr; 94 95 while (nbytes >= SM4_BLOCK_SIZE) { 96 crypto_xor_cpy(dst, src, iv, SM4_BLOCK_SIZE); 97 sm4_crypt_block(ctx->rkey_enc, dst, dst); 98 iv = dst; 99 src += SM4_BLOCK_SIZE; 100 dst += SM4_BLOCK_SIZE; 101 nbytes -= SM4_BLOCK_SIZE; 102 } 103 if (iv != walk.iv) 104 memcpy(walk.iv, iv, SM4_BLOCK_SIZE); 105 106 err = skcipher_walk_done(&walk, nbytes); 107 } 108 109 return err; 110 } 111 112 static int sm4_cbc_decrypt(struct skcipher_request *req) 113 { 114 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 115 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 116 struct skcipher_walk walk; 117 unsigned int nbytes; 118 int err; 119 120 err = skcipher_walk_virt(&walk, req, false); 121 122 while ((nbytes = walk.nbytes) > 0) { 123 const u8 *src = walk.src.virt.addr; 124 u8 *dst = walk.dst.virt.addr; 125 unsigned int nblocks; 126 127 nblocks = nbytes / SM4_BLOCK_SIZE; 128 if (nblocks) { 129 kernel_neon_begin(); 130 131 sm4_neon_cbc_dec(ctx->rkey_dec, dst, src, 132 walk.iv, nblocks); 133 134 kernel_neon_end(); 135 } 136 137 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); 138 } 139 140 return err; 141 } 142 143 static int sm4_ctr_crypt(struct skcipher_request *req) 144 { 145 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 146 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 147 struct skcipher_walk walk; 148 unsigned int nbytes; 149 int err; 150 151 err = skcipher_walk_virt(&walk, req, false); 152 153 while ((nbytes = walk.nbytes) > 0) { 154 const u8 *src = walk.src.virt.addr; 155 u8 *dst = walk.dst.virt.addr; 156 unsigned int nblocks; 157 158 nblocks = nbytes / SM4_BLOCK_SIZE; 159 if (nblocks) { 160 kernel_neon_begin(); 161 162 sm4_neon_ctr_crypt(ctx->rkey_enc, dst, src, 163 walk.iv, nblocks); 164 165 kernel_neon_end(); 166 167 dst += nblocks * SM4_BLOCK_SIZE; 168 src += nblocks * SM4_BLOCK_SIZE; 169 nbytes -= nblocks * SM4_BLOCK_SIZE; 170 } 171 172 /* tail */ 173 if (walk.nbytes == walk.total && nbytes > 0) { 174 u8 keystream[SM4_BLOCK_SIZE]; 175 176 sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); 177 crypto_inc(walk.iv, SM4_BLOCK_SIZE); 178 crypto_xor_cpy(dst, src, keystream, nbytes); 179 nbytes = 0; 180 } 181 182 err = skcipher_walk_done(&walk, nbytes); 183 } 184 185 return err; 186 } 187 188 static struct skcipher_alg sm4_algs[] = { 189 { 190 .base = { 191 .cra_name = "ecb(sm4)", 192 .cra_driver_name = "ecb-sm4-neon", 193 .cra_priority = 200, 194 .cra_blocksize = SM4_BLOCK_SIZE, 195 .cra_ctxsize = sizeof(struct sm4_ctx), 196 .cra_module = THIS_MODULE, 197 }, 198 .min_keysize = SM4_KEY_SIZE, 199 .max_keysize = SM4_KEY_SIZE, 200 .setkey = sm4_setkey, 201 .encrypt = sm4_ecb_encrypt, 202 .decrypt = sm4_ecb_decrypt, 203 }, { 204 .base = { 205 .cra_name = "cbc(sm4)", 206 .cra_driver_name = "cbc-sm4-neon", 207 .cra_priority = 200, 208 .cra_blocksize = SM4_BLOCK_SIZE, 209 .cra_ctxsize = sizeof(struct sm4_ctx), 210 .cra_module = THIS_MODULE, 211 }, 212 .min_keysize = SM4_KEY_SIZE, 213 .max_keysize = SM4_KEY_SIZE, 214 .ivsize = SM4_BLOCK_SIZE, 215 .setkey = sm4_setkey, 216 .encrypt = sm4_cbc_encrypt, 217 .decrypt = sm4_cbc_decrypt, 218 }, { 219 .base = { 220 .cra_name = "ctr(sm4)", 221 .cra_driver_name = "ctr-sm4-neon", 222 .cra_priority = 200, 223 .cra_blocksize = 1, 224 .cra_ctxsize = sizeof(struct sm4_ctx), 225 .cra_module = THIS_MODULE, 226 }, 227 .min_keysize = SM4_KEY_SIZE, 228 .max_keysize = SM4_KEY_SIZE, 229 .ivsize = SM4_BLOCK_SIZE, 230 .chunksize = SM4_BLOCK_SIZE, 231 .setkey = sm4_setkey, 232 .encrypt = sm4_ctr_crypt, 233 .decrypt = sm4_ctr_crypt, 234 } 235 }; 236 237 static int __init sm4_init(void) 238 { 239 return crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 240 } 241 242 static void __exit sm4_exit(void) 243 { 244 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 245 } 246 247 module_init(sm4_init); 248 module_exit(sm4_exit); 249 250 MODULE_DESCRIPTION("SM4 ECB/CBC/CTR using ARMv8 NEON"); 251 MODULE_ALIAS_CRYPTO("sm4-neon"); 252 MODULE_ALIAS_CRYPTO("sm4"); 253 MODULE_ALIAS_CRYPTO("ecb(sm4)"); 254 MODULE_ALIAS_CRYPTO("cbc(sm4)"); 255 MODULE_ALIAS_CRYPTO("ctr(sm4)"); 256 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); 257 MODULE_LICENSE("GPL v2"); 258