1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * SM4 Cipher Algorithm, using ARMv8 Crypto Extensions
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (C) 2022, Alibaba Group.
8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11 #include <linux/module.h>
12 #include <linux/crypto.h>
13 #include <linux/kernel.h>
14 #include <linux/cpufeature.h>
15 #include <asm/neon.h>
16 #include <asm/simd.h>
17 #include <crypto/b128ops.h>
18 #include <crypto/internal/simd.h>
19 #include <crypto/internal/skcipher.h>
20 #include <crypto/internal/hash.h>
21 #include <crypto/scatterwalk.h>
22 #include <crypto/xts.h>
23 #include <crypto/sm4.h>
24
25 #define BYTES2BLKS(nbytes) ((nbytes) >> 4)
26
27 asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec,
28 const u32 *fk, const u32 *ck);
29 asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src);
30 asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src,
31 unsigned int nblks);
32 asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src,
33 u8 *iv, unsigned int nblocks);
34 asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src,
35 u8 *iv, unsigned int nblocks);
36 asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src,
37 u8 *iv, unsigned int nbytes);
38 asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src,
39 u8 *iv, unsigned int nbytes);
40 asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src,
41 u8 *iv, unsigned int nblks);
42 asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src,
43 u8 *tweak, unsigned int nbytes,
44 const u32 *rkey2_enc);
45 asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src,
46 u8 *tweak, unsigned int nbytes,
47 const u32 *rkey2_enc);
48 asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest,
49 const u8 *src, unsigned int nblocks,
50 bool enc_before, bool enc_after);
51
52 EXPORT_SYMBOL(sm4_ce_expand_key);
53 EXPORT_SYMBOL(sm4_ce_crypt_block);
54 EXPORT_SYMBOL(sm4_ce_cbc_enc);
55
56 struct sm4_xts_ctx {
57 struct sm4_ctx key1;
58 struct sm4_ctx key2;
59 };
60
61 struct sm4_mac_tfm_ctx {
62 struct sm4_ctx key;
63 u8 __aligned(8) consts[];
64 };
65
66 struct sm4_mac_desc_ctx {
67 unsigned int len;
68 u8 digest[SM4_BLOCK_SIZE];
69 };
70
sm4_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)71 static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
72 unsigned int key_len)
73 {
74 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
75
76 if (key_len != SM4_KEY_SIZE)
77 return -EINVAL;
78
79 kernel_neon_begin();
80 sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec,
81 crypto_sm4_fk, crypto_sm4_ck);
82 kernel_neon_end();
83 return 0;
84 }
85
sm4_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)86 static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
87 unsigned int key_len)
88 {
89 struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
90 int ret;
91
92 if (key_len != SM4_KEY_SIZE * 2)
93 return -EINVAL;
94
95 ret = xts_verify_key(tfm, key, key_len);
96 if (ret)
97 return ret;
98
99 kernel_neon_begin();
100 sm4_ce_expand_key(key, ctx->key1.rkey_enc,
101 ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
102 sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc,
103 ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
104 kernel_neon_end();
105
106 return 0;
107 }
108
sm4_ecb_do_crypt(struct skcipher_request * req,const u32 * rkey)109 static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
110 {
111 struct skcipher_walk walk;
112 unsigned int nbytes;
113 int err;
114
115 err = skcipher_walk_virt(&walk, req, false);
116
117 while ((nbytes = walk.nbytes) > 0) {
118 const u8 *src = walk.src.virt.addr;
119 u8 *dst = walk.dst.virt.addr;
120 unsigned int nblks;
121
122 kernel_neon_begin();
123
124 nblks = BYTES2BLKS(nbytes);
125 if (nblks) {
126 sm4_ce_crypt(rkey, dst, src, nblks);
127 nbytes -= nblks * SM4_BLOCK_SIZE;
128 }
129
130 kernel_neon_end();
131
132 err = skcipher_walk_done(&walk, nbytes);
133 }
134
135 return err;
136 }
137
sm4_ecb_encrypt(struct skcipher_request * req)138 static int sm4_ecb_encrypt(struct skcipher_request *req)
139 {
140 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
141 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
142
143 return sm4_ecb_do_crypt(req, ctx->rkey_enc);
144 }
145
sm4_ecb_decrypt(struct skcipher_request * req)146 static int sm4_ecb_decrypt(struct skcipher_request *req)
147 {
148 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
149 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
150
151 return sm4_ecb_do_crypt(req, ctx->rkey_dec);
152 }
153
sm4_cbc_crypt(struct skcipher_request * req,struct sm4_ctx * ctx,bool encrypt)154 static int sm4_cbc_crypt(struct skcipher_request *req,
155 struct sm4_ctx *ctx, bool encrypt)
156 {
157 struct skcipher_walk walk;
158 unsigned int nbytes;
159 int err;
160
161 err = skcipher_walk_virt(&walk, req, false);
162 if (err)
163 return err;
164
165 while ((nbytes = walk.nbytes) > 0) {
166 const u8 *src = walk.src.virt.addr;
167 u8 *dst = walk.dst.virt.addr;
168 unsigned int nblocks;
169
170 nblocks = nbytes / SM4_BLOCK_SIZE;
171 if (nblocks) {
172 kernel_neon_begin();
173
174 if (encrypt)
175 sm4_ce_cbc_enc(ctx->rkey_enc, dst, src,
176 walk.iv, nblocks);
177 else
178 sm4_ce_cbc_dec(ctx->rkey_dec, dst, src,
179 walk.iv, nblocks);
180
181 kernel_neon_end();
182 }
183
184 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
185 }
186
187 return err;
188 }
189
sm4_cbc_encrypt(struct skcipher_request * req)190 static int sm4_cbc_encrypt(struct skcipher_request *req)
191 {
192 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
193 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
194
195 return sm4_cbc_crypt(req, ctx, true);
196 }
197
sm4_cbc_decrypt(struct skcipher_request * req)198 static int sm4_cbc_decrypt(struct skcipher_request *req)
199 {
200 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
201 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
202
203 return sm4_cbc_crypt(req, ctx, false);
204 }
205
sm4_cbc_cts_crypt(struct skcipher_request * req,bool encrypt)206 static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt)
207 {
208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
209 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
210 struct scatterlist *src = req->src;
211 struct scatterlist *dst = req->dst;
212 struct scatterlist sg_src[2], sg_dst[2];
213 struct skcipher_request subreq;
214 struct skcipher_walk walk;
215 int cbc_blocks;
216 int err;
217
218 if (req->cryptlen < SM4_BLOCK_SIZE)
219 return -EINVAL;
220
221 if (req->cryptlen == SM4_BLOCK_SIZE)
222 return sm4_cbc_crypt(req, ctx, encrypt);
223
224 skcipher_request_set_tfm(&subreq, tfm);
225 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
226 NULL, NULL);
227
228 /* handle the CBC cryption part */
229 cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
230 if (cbc_blocks) {
231 skcipher_request_set_crypt(&subreq, src, dst,
232 cbc_blocks * SM4_BLOCK_SIZE,
233 req->iv);
234
235 err = sm4_cbc_crypt(&subreq, ctx, encrypt);
236 if (err)
237 return err;
238
239 dst = src = scatterwalk_ffwd(sg_src, src, subreq.cryptlen);
240 if (req->dst != req->src)
241 dst = scatterwalk_ffwd(sg_dst, req->dst,
242 subreq.cryptlen);
243 }
244
245 /* handle ciphertext stealing */
246 skcipher_request_set_crypt(&subreq, src, dst,
247 req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE,
248 req->iv);
249
250 err = skcipher_walk_virt(&walk, &subreq, false);
251 if (err)
252 return err;
253
254 kernel_neon_begin();
255
256 if (encrypt)
257 sm4_ce_cbc_cts_enc(ctx->rkey_enc, walk.dst.virt.addr,
258 walk.src.virt.addr, walk.iv, walk.nbytes);
259 else
260 sm4_ce_cbc_cts_dec(ctx->rkey_dec, walk.dst.virt.addr,
261 walk.src.virt.addr, walk.iv, walk.nbytes);
262
263 kernel_neon_end();
264
265 return skcipher_walk_done(&walk, 0);
266 }
267
sm4_cbc_cts_encrypt(struct skcipher_request * req)268 static int sm4_cbc_cts_encrypt(struct skcipher_request *req)
269 {
270 return sm4_cbc_cts_crypt(req, true);
271 }
272
sm4_cbc_cts_decrypt(struct skcipher_request * req)273 static int sm4_cbc_cts_decrypt(struct skcipher_request *req)
274 {
275 return sm4_cbc_cts_crypt(req, false);
276 }
277
sm4_ctr_crypt(struct skcipher_request * req)278 static int sm4_ctr_crypt(struct skcipher_request *req)
279 {
280 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
281 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
282 struct skcipher_walk walk;
283 unsigned int nbytes;
284 int err;
285
286 err = skcipher_walk_virt(&walk, req, false);
287
288 while ((nbytes = walk.nbytes) > 0) {
289 const u8 *src = walk.src.virt.addr;
290 u8 *dst = walk.dst.virt.addr;
291 unsigned int nblks;
292
293 kernel_neon_begin();
294
295 nblks = BYTES2BLKS(nbytes);
296 if (nblks) {
297 sm4_ce_ctr_enc(ctx->rkey_enc, dst, src, walk.iv, nblks);
298 dst += nblks * SM4_BLOCK_SIZE;
299 src += nblks * SM4_BLOCK_SIZE;
300 nbytes -= nblks * SM4_BLOCK_SIZE;
301 }
302
303 /* tail */
304 if (walk.nbytes == walk.total && nbytes > 0) {
305 u8 keystream[SM4_BLOCK_SIZE];
306
307 sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
308 crypto_inc(walk.iv, SM4_BLOCK_SIZE);
309 crypto_xor_cpy(dst, src, keystream, nbytes);
310 nbytes = 0;
311 }
312
313 kernel_neon_end();
314
315 err = skcipher_walk_done(&walk, nbytes);
316 }
317
318 return err;
319 }
320
sm4_xts_crypt(struct skcipher_request * req,bool encrypt)321 static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt)
322 {
323 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
324 struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
325 int tail = req->cryptlen % SM4_BLOCK_SIZE;
326 const u32 *rkey2_enc = ctx->key2.rkey_enc;
327 struct scatterlist sg_src[2], sg_dst[2];
328 struct skcipher_request subreq;
329 struct scatterlist *src, *dst;
330 struct skcipher_walk walk;
331 unsigned int nbytes;
332 int err;
333
334 if (req->cryptlen < SM4_BLOCK_SIZE)
335 return -EINVAL;
336
337 err = skcipher_walk_virt(&walk, req, false);
338 if (err)
339 return err;
340
341 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
342 int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
343
344 skcipher_walk_abort(&walk);
345
346 skcipher_request_set_tfm(&subreq, tfm);
347 skcipher_request_set_callback(&subreq,
348 skcipher_request_flags(req),
349 NULL, NULL);
350 skcipher_request_set_crypt(&subreq, req->src, req->dst,
351 nblocks * SM4_BLOCK_SIZE, req->iv);
352
353 err = skcipher_walk_virt(&walk, &subreq, false);
354 if (err)
355 return err;
356 } else {
357 tail = 0;
358 }
359
360 while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) {
361 if (nbytes < walk.total)
362 nbytes &= ~(SM4_BLOCK_SIZE - 1);
363
364 kernel_neon_begin();
365
366 if (encrypt)
367 sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
368 walk.src.virt.addr, walk.iv, nbytes,
369 rkey2_enc);
370 else
371 sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
372 walk.src.virt.addr, walk.iv, nbytes,
373 rkey2_enc);
374
375 kernel_neon_end();
376
377 rkey2_enc = NULL;
378
379 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
380 if (err)
381 return err;
382 }
383
384 if (likely(tail == 0))
385 return 0;
386
387 /* handle ciphertext stealing */
388
389 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
390 if (req->dst != req->src)
391 dst = scatterwalk_ffwd(sg_dst, req->dst, subreq.cryptlen);
392
393 skcipher_request_set_crypt(&subreq, src, dst, SM4_BLOCK_SIZE + tail,
394 req->iv);
395
396 err = skcipher_walk_virt(&walk, &subreq, false);
397 if (err)
398 return err;
399
400 kernel_neon_begin();
401
402 if (encrypt)
403 sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
404 walk.src.virt.addr, walk.iv, walk.nbytes,
405 rkey2_enc);
406 else
407 sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
408 walk.src.virt.addr, walk.iv, walk.nbytes,
409 rkey2_enc);
410
411 kernel_neon_end();
412
413 return skcipher_walk_done(&walk, 0);
414 }
415
sm4_xts_encrypt(struct skcipher_request * req)416 static int sm4_xts_encrypt(struct skcipher_request *req)
417 {
418 return sm4_xts_crypt(req, true);
419 }
420
sm4_xts_decrypt(struct skcipher_request * req)421 static int sm4_xts_decrypt(struct skcipher_request *req)
422 {
423 return sm4_xts_crypt(req, false);
424 }
425
426 static struct skcipher_alg sm4_algs[] = {
427 {
428 .base = {
429 .cra_name = "ecb(sm4)",
430 .cra_driver_name = "ecb-sm4-ce",
431 .cra_priority = 400,
432 .cra_blocksize = SM4_BLOCK_SIZE,
433 .cra_ctxsize = sizeof(struct sm4_ctx),
434 .cra_module = THIS_MODULE,
435 },
436 .min_keysize = SM4_KEY_SIZE,
437 .max_keysize = SM4_KEY_SIZE,
438 .setkey = sm4_setkey,
439 .encrypt = sm4_ecb_encrypt,
440 .decrypt = sm4_ecb_decrypt,
441 }, {
442 .base = {
443 .cra_name = "cbc(sm4)",
444 .cra_driver_name = "cbc-sm4-ce",
445 .cra_priority = 400,
446 .cra_blocksize = SM4_BLOCK_SIZE,
447 .cra_ctxsize = sizeof(struct sm4_ctx),
448 .cra_module = THIS_MODULE,
449 },
450 .min_keysize = SM4_KEY_SIZE,
451 .max_keysize = SM4_KEY_SIZE,
452 .ivsize = SM4_BLOCK_SIZE,
453 .setkey = sm4_setkey,
454 .encrypt = sm4_cbc_encrypt,
455 .decrypt = sm4_cbc_decrypt,
456 }, {
457 .base = {
458 .cra_name = "ctr(sm4)",
459 .cra_driver_name = "ctr-sm4-ce",
460 .cra_priority = 400,
461 .cra_blocksize = 1,
462 .cra_ctxsize = sizeof(struct sm4_ctx),
463 .cra_module = THIS_MODULE,
464 },
465 .min_keysize = SM4_KEY_SIZE,
466 .max_keysize = SM4_KEY_SIZE,
467 .ivsize = SM4_BLOCK_SIZE,
468 .chunksize = SM4_BLOCK_SIZE,
469 .setkey = sm4_setkey,
470 .encrypt = sm4_ctr_crypt,
471 .decrypt = sm4_ctr_crypt,
472 }, {
473 .base = {
474 .cra_name = "cts(cbc(sm4))",
475 .cra_driver_name = "cts-cbc-sm4-ce",
476 .cra_priority = 400,
477 .cra_blocksize = SM4_BLOCK_SIZE,
478 .cra_ctxsize = sizeof(struct sm4_ctx),
479 .cra_module = THIS_MODULE,
480 },
481 .min_keysize = SM4_KEY_SIZE,
482 .max_keysize = SM4_KEY_SIZE,
483 .ivsize = SM4_BLOCK_SIZE,
484 .walksize = SM4_BLOCK_SIZE * 2,
485 .setkey = sm4_setkey,
486 .encrypt = sm4_cbc_cts_encrypt,
487 .decrypt = sm4_cbc_cts_decrypt,
488 }, {
489 .base = {
490 .cra_name = "xts(sm4)",
491 .cra_driver_name = "xts-sm4-ce",
492 .cra_priority = 400,
493 .cra_blocksize = SM4_BLOCK_SIZE,
494 .cra_ctxsize = sizeof(struct sm4_xts_ctx),
495 .cra_module = THIS_MODULE,
496 },
497 .min_keysize = SM4_KEY_SIZE * 2,
498 .max_keysize = SM4_KEY_SIZE * 2,
499 .ivsize = SM4_BLOCK_SIZE,
500 .walksize = SM4_BLOCK_SIZE * 2,
501 .setkey = sm4_xts_setkey,
502 .encrypt = sm4_xts_encrypt,
503 .decrypt = sm4_xts_decrypt,
504 }
505 };
506
sm4_cbcmac_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int key_len)507 static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key,
508 unsigned int key_len)
509 {
510 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
511
512 if (key_len != SM4_KEY_SIZE)
513 return -EINVAL;
514
515 kernel_neon_begin();
516 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
517 crypto_sm4_fk, crypto_sm4_ck);
518 kernel_neon_end();
519
520 return 0;
521 }
522
sm4_cmac_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int key_len)523 static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key,
524 unsigned int key_len)
525 {
526 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
527 be128 *consts = (be128 *)ctx->consts;
528 u64 a, b;
529
530 if (key_len != SM4_KEY_SIZE)
531 return -EINVAL;
532
533 memset(consts, 0, SM4_BLOCK_SIZE);
534
535 kernel_neon_begin();
536
537 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
538 crypto_sm4_fk, crypto_sm4_ck);
539
540 /* encrypt the zero block */
541 sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts);
542
543 kernel_neon_end();
544
545 /* gf(2^128) multiply zero-ciphertext with u and u^2 */
546 a = be64_to_cpu(consts[0].a);
547 b = be64_to_cpu(consts[0].b);
548 consts[0].a = cpu_to_be64((a << 1) | (b >> 63));
549 consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
550
551 a = be64_to_cpu(consts[0].a);
552 b = be64_to_cpu(consts[0].b);
553 consts[1].a = cpu_to_be64((a << 1) | (b >> 63));
554 consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
555
556 return 0;
557 }
558
sm4_xcbc_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int key_len)559 static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key,
560 unsigned int key_len)
561 {
562 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
563 u8 __aligned(8) key2[SM4_BLOCK_SIZE];
564 static u8 const ks[3][SM4_BLOCK_SIZE] = {
565 { [0 ... SM4_BLOCK_SIZE - 1] = 0x1},
566 { [0 ... SM4_BLOCK_SIZE - 1] = 0x2},
567 { [0 ... SM4_BLOCK_SIZE - 1] = 0x3},
568 };
569
570 if (key_len != SM4_KEY_SIZE)
571 return -EINVAL;
572
573 kernel_neon_begin();
574
575 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
576 crypto_sm4_fk, crypto_sm4_ck);
577
578 sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]);
579 sm4_ce_crypt(ctx->key.rkey_enc, ctx->consts, ks[1], 2);
580
581 sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec,
582 crypto_sm4_fk, crypto_sm4_ck);
583
584 kernel_neon_end();
585
586 return 0;
587 }
588
sm4_mac_init(struct shash_desc * desc)589 static int sm4_mac_init(struct shash_desc *desc)
590 {
591 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
592
593 memset(ctx->digest, 0, SM4_BLOCK_SIZE);
594 ctx->len = 0;
595
596 return 0;
597 }
598
sm4_mac_update(struct shash_desc * desc,const u8 * p,unsigned int len)599 static int sm4_mac_update(struct shash_desc *desc, const u8 *p,
600 unsigned int len)
601 {
602 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
603 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
604 unsigned int l, nblocks;
605
606 if (len == 0)
607 return 0;
608
609 if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) {
610 l = min(len, SM4_BLOCK_SIZE - ctx->len);
611
612 crypto_xor(ctx->digest + ctx->len, p, l);
613 ctx->len += l;
614 len -= l;
615 p += l;
616 }
617
618 if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) {
619 kernel_neon_begin();
620
621 if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) {
622 sm4_ce_crypt_block(tctx->key.rkey_enc,
623 ctx->digest, ctx->digest);
624 ctx->len = 0;
625 } else {
626 nblocks = len / SM4_BLOCK_SIZE;
627 len %= SM4_BLOCK_SIZE;
628
629 sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p,
630 nblocks, (ctx->len == SM4_BLOCK_SIZE),
631 (len != 0));
632
633 p += nblocks * SM4_BLOCK_SIZE;
634
635 if (len == 0)
636 ctx->len = SM4_BLOCK_SIZE;
637 }
638
639 kernel_neon_end();
640
641 if (len) {
642 crypto_xor(ctx->digest, p, len);
643 ctx->len = len;
644 }
645 }
646
647 return 0;
648 }
649
sm4_cmac_final(struct shash_desc * desc,u8 * out)650 static int sm4_cmac_final(struct shash_desc *desc, u8 *out)
651 {
652 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
653 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
654 const u8 *consts = tctx->consts;
655
656 if (ctx->len != SM4_BLOCK_SIZE) {
657 ctx->digest[ctx->len] ^= 0x80;
658 consts += SM4_BLOCK_SIZE;
659 }
660
661 kernel_neon_begin();
662 sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1,
663 false, true);
664 kernel_neon_end();
665
666 memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
667
668 return 0;
669 }
670
sm4_cbcmac_final(struct shash_desc * desc,u8 * out)671 static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out)
672 {
673 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
674 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
675
676 if (ctx->len) {
677 kernel_neon_begin();
678 sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest,
679 ctx->digest);
680 kernel_neon_end();
681 }
682
683 memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
684
685 return 0;
686 }
687
688 static struct shash_alg sm4_mac_algs[] = {
689 {
690 .base = {
691 .cra_name = "cmac(sm4)",
692 .cra_driver_name = "cmac-sm4-ce",
693 .cra_priority = 400,
694 .cra_blocksize = SM4_BLOCK_SIZE,
695 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
696 + SM4_BLOCK_SIZE * 2,
697 .cra_module = THIS_MODULE,
698 },
699 .digestsize = SM4_BLOCK_SIZE,
700 .init = sm4_mac_init,
701 .update = sm4_mac_update,
702 .final = sm4_cmac_final,
703 .setkey = sm4_cmac_setkey,
704 .descsize = sizeof(struct sm4_mac_desc_ctx),
705 }, {
706 .base = {
707 .cra_name = "xcbc(sm4)",
708 .cra_driver_name = "xcbc-sm4-ce",
709 .cra_priority = 400,
710 .cra_blocksize = SM4_BLOCK_SIZE,
711 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
712 + SM4_BLOCK_SIZE * 2,
713 .cra_module = THIS_MODULE,
714 },
715 .digestsize = SM4_BLOCK_SIZE,
716 .init = sm4_mac_init,
717 .update = sm4_mac_update,
718 .final = sm4_cmac_final,
719 .setkey = sm4_xcbc_setkey,
720 .descsize = sizeof(struct sm4_mac_desc_ctx),
721 }, {
722 .base = {
723 .cra_name = "cbcmac(sm4)",
724 .cra_driver_name = "cbcmac-sm4-ce",
725 .cra_priority = 400,
726 .cra_blocksize = 1,
727 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx),
728 .cra_module = THIS_MODULE,
729 },
730 .digestsize = SM4_BLOCK_SIZE,
731 .init = sm4_mac_init,
732 .update = sm4_mac_update,
733 .final = sm4_cbcmac_final,
734 .setkey = sm4_cbcmac_setkey,
735 .descsize = sizeof(struct sm4_mac_desc_ctx),
736 }
737 };
738
sm4_init(void)739 static int __init sm4_init(void)
740 {
741 int err;
742
743 err = crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
744 if (err)
745 return err;
746
747 err = crypto_register_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
748 if (err)
749 goto out_err;
750
751 return 0;
752
753 out_err:
754 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
755 return err;
756 }
757
sm4_exit(void)758 static void __exit sm4_exit(void)
759 {
760 crypto_unregister_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
761 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
762 }
763
764 module_cpu_feature_match(SM4, sm4_init);
765 module_exit(sm4_exit);
766
767 MODULE_DESCRIPTION("SM4 ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
768 MODULE_ALIAS_CRYPTO("sm4-ce");
769 MODULE_ALIAS_CRYPTO("sm4");
770 MODULE_ALIAS_CRYPTO("ecb(sm4)");
771 MODULE_ALIAS_CRYPTO("cbc(sm4)");
772 MODULE_ALIAS_CRYPTO("ctr(sm4)");
773 MODULE_ALIAS_CRYPTO("cts(cbc(sm4))");
774 MODULE_ALIAS_CRYPTO("xts(sm4)");
775 MODULE_ALIAS_CRYPTO("cmac(sm4)");
776 MODULE_ALIAS_CRYPTO("xcbc(sm4)");
777 MODULE_ALIAS_CRYPTO("cbcmac(sm4)");
778 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
779 MODULE_LICENSE("GPL v2");
780