xref: /linux/arch/arm64/crypto/sha3-ce-glue.c (revision 7ec462100ef9142344ddbf86f2c3008b97acddbe)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions
4  *
5  * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 
12 #include <asm/hwcap.h>
13 #include <asm/neon.h>
14 #include <asm/simd.h>
15 #include <linux/unaligned.h>
16 #include <crypto/internal/hash.h>
17 #include <crypto/internal/simd.h>
18 #include <crypto/sha3.h>
19 #include <linux/cpufeature.h>
20 #include <linux/crypto.h>
21 #include <linux/module.h>
22 
23 MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions");
24 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
25 MODULE_LICENSE("GPL v2");
26 MODULE_ALIAS_CRYPTO("sha3-224");
27 MODULE_ALIAS_CRYPTO("sha3-256");
28 MODULE_ALIAS_CRYPTO("sha3-384");
29 MODULE_ALIAS_CRYPTO("sha3-512");
30 
31 asmlinkage int sha3_ce_transform(u64 *st, const u8 *data, int blocks,
32 				 int md_len);
33 
sha3_update(struct shash_desc * desc,const u8 * data,unsigned int len)34 static int sha3_update(struct shash_desc *desc, const u8 *data,
35 		       unsigned int len)
36 {
37 	struct sha3_state *sctx = shash_desc_ctx(desc);
38 	unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
39 
40 	if (!crypto_simd_usable())
41 		return crypto_sha3_update(desc, data, len);
42 
43 	if ((sctx->partial + len) >= sctx->rsiz) {
44 		int blocks;
45 
46 		if (sctx->partial) {
47 			int p = sctx->rsiz - sctx->partial;
48 
49 			memcpy(sctx->buf + sctx->partial, data, p);
50 			kernel_neon_begin();
51 			sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size);
52 			kernel_neon_end();
53 
54 			data += p;
55 			len -= p;
56 			sctx->partial = 0;
57 		}
58 
59 		blocks = len / sctx->rsiz;
60 		len %= sctx->rsiz;
61 
62 		while (blocks) {
63 			int rem;
64 
65 			kernel_neon_begin();
66 			rem = sha3_ce_transform(sctx->st, data, blocks,
67 						digest_size);
68 			kernel_neon_end();
69 			data += (blocks - rem) * sctx->rsiz;
70 			blocks = rem;
71 		}
72 	}
73 
74 	if (len) {
75 		memcpy(sctx->buf + sctx->partial, data, len);
76 		sctx->partial += len;
77 	}
78 	return 0;
79 }
80 
sha3_final(struct shash_desc * desc,u8 * out)81 static int sha3_final(struct shash_desc *desc, u8 *out)
82 {
83 	struct sha3_state *sctx = shash_desc_ctx(desc);
84 	unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
85 	__le64 *digest = (__le64 *)out;
86 	int i;
87 
88 	if (!crypto_simd_usable())
89 		return crypto_sha3_final(desc, out);
90 
91 	sctx->buf[sctx->partial++] = 0x06;
92 	memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial);
93 	sctx->buf[sctx->rsiz - 1] |= 0x80;
94 
95 	kernel_neon_begin();
96 	sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size);
97 	kernel_neon_end();
98 
99 	for (i = 0; i < digest_size / 8; i++)
100 		put_unaligned_le64(sctx->st[i], digest++);
101 
102 	if (digest_size & 4)
103 		put_unaligned_le32(sctx->st[i], (__le32 *)digest);
104 
105 	memzero_explicit(sctx, sizeof(*sctx));
106 	return 0;
107 }
108 
109 static struct shash_alg algs[] = { {
110 	.digestsize		= SHA3_224_DIGEST_SIZE,
111 	.init			= crypto_sha3_init,
112 	.update			= sha3_update,
113 	.final			= sha3_final,
114 	.descsize		= sizeof(struct sha3_state),
115 	.base.cra_name		= "sha3-224",
116 	.base.cra_driver_name	= "sha3-224-ce",
117 	.base.cra_blocksize	= SHA3_224_BLOCK_SIZE,
118 	.base.cra_module	= THIS_MODULE,
119 	.base.cra_priority	= 200,
120 }, {
121 	.digestsize		= SHA3_256_DIGEST_SIZE,
122 	.init			= crypto_sha3_init,
123 	.update			= sha3_update,
124 	.final			= sha3_final,
125 	.descsize		= sizeof(struct sha3_state),
126 	.base.cra_name		= "sha3-256",
127 	.base.cra_driver_name	= "sha3-256-ce",
128 	.base.cra_blocksize	= SHA3_256_BLOCK_SIZE,
129 	.base.cra_module	= THIS_MODULE,
130 	.base.cra_priority	= 200,
131 }, {
132 	.digestsize		= SHA3_384_DIGEST_SIZE,
133 	.init			= crypto_sha3_init,
134 	.update			= sha3_update,
135 	.final			= sha3_final,
136 	.descsize		= sizeof(struct sha3_state),
137 	.base.cra_name		= "sha3-384",
138 	.base.cra_driver_name	= "sha3-384-ce",
139 	.base.cra_blocksize	= SHA3_384_BLOCK_SIZE,
140 	.base.cra_module	= THIS_MODULE,
141 	.base.cra_priority	= 200,
142 }, {
143 	.digestsize		= SHA3_512_DIGEST_SIZE,
144 	.init			= crypto_sha3_init,
145 	.update			= sha3_update,
146 	.final			= sha3_final,
147 	.descsize		= sizeof(struct sha3_state),
148 	.base.cra_name		= "sha3-512",
149 	.base.cra_driver_name	= "sha3-512-ce",
150 	.base.cra_blocksize	= SHA3_512_BLOCK_SIZE,
151 	.base.cra_module	= THIS_MODULE,
152 	.base.cra_priority	= 200,
153 } };
154 
sha3_neon_mod_init(void)155 static int __init sha3_neon_mod_init(void)
156 {
157 	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
158 }
159 
sha3_neon_mod_fini(void)160 static void __exit sha3_neon_mod_fini(void)
161 {
162 	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
163 }
164 
165 module_cpu_feature_match(SHA3, sha3_neon_mod_init);
166 module_exit(sha3_neon_mod_fini);
167