xref: /freebsd/sys/contrib/openzfs/module/icp/algs/sha2/sha512_impl.c (revision f126890ac5386406dadf7c4cfa9566cbb56537c5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
24  */
25 
26 #include <sys/simd.h>
27 #include <sys/zfs_context.h>
28 #include <sys/zfs_impl.h>
29 #include <sys/sha2.h>
30 
31 #include <sha2/sha2_impl.h>
32 #include <sys/asm_linkage.h>
33 
34 #define	TF(E, N) \
35 	extern void ASMABI E(uint64_t s[8], const void *, size_t); \
36 	static inline void N(uint64_t s[8], const void *d, size_t b) { \
37 	kfpu_begin(); E(s, d, b); kfpu_end(); \
38 }
39 
40 /* some implementation is always okay */
41 static inline boolean_t sha2_is_supported(void)
42 {
43 	return (B_TRUE);
44 }
45 
46 #if defined(__x86_64)
47 
48 /* Users of ASMABI requires all calls to be from wrappers */
49 extern void ASMABI
50 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
51 
52 static inline void
53 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
54 {
55 	zfs_sha512_transform_x64(s, d, b);
56 }
57 const sha512_ops_t sha512_x64_impl = {
58 	.is_supported = sha2_is_supported,
59 	.transform = tf_sha512_transform_x64,
60 	.name = "x64"
61 };
62 
63 #if defined(HAVE_AVX)
64 static boolean_t sha2_have_avx(void)
65 {
66 	return (kfpu_allowed() && zfs_avx_available());
67 }
68 
69 TF(zfs_sha512_transform_avx, tf_sha512_avx);
70 const sha512_ops_t sha512_avx_impl = {
71 	.is_supported = sha2_have_avx,
72 	.transform = tf_sha512_avx,
73 	.name = "avx"
74 };
75 #endif
76 
77 #if defined(HAVE_AVX2)
78 static boolean_t sha2_have_avx2(void)
79 {
80 	return (kfpu_allowed() && zfs_avx2_available());
81 }
82 
83 TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
84 const sha512_ops_t sha512_avx2_impl = {
85 	.is_supported = sha2_have_avx2,
86 	.transform = tf_sha512_avx2,
87 	.name = "avx2"
88 };
89 #endif
90 
91 #elif defined(__aarch64__) || defined(__arm__)
92 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
93 const sha512_ops_t sha512_armv7_impl = {
94 	.is_supported = sha2_is_supported,
95 	.transform = zfs_sha512_block_armv7,
96 	.name = "armv7"
97 };
98 
99 #if defined(__aarch64__)
100 static boolean_t sha512_have_armv8ce(void)
101 {
102 	return (kfpu_allowed() && zfs_sha512_available());
103 }
104 
105 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
106 const sha512_ops_t sha512_armv8_impl = {
107 	.is_supported = sha512_have_armv8ce,
108 	.transform = tf_sha512_armv8ce,
109 	.name = "armv8-ce"
110 };
111 #endif
112 
113 #if defined(__arm__) && __ARM_ARCH > 6
114 static boolean_t sha512_have_neon(void)
115 {
116 	return (kfpu_allowed() && zfs_neon_available());
117 }
118 
119 TF(zfs_sha512_block_neon, tf_sha512_neon);
120 const sha512_ops_t sha512_neon_impl = {
121 	.is_supported = sha512_have_neon,
122 	.transform = tf_sha512_neon,
123 	.name = "neon"
124 };
125 #endif
126 
127 #elif defined(__PPC64__)
128 TF(zfs_sha512_ppc, tf_sha512_ppc);
129 const sha512_ops_t sha512_ppc_impl = {
130 	.is_supported = sha2_is_supported,
131 	.transform = tf_sha512_ppc,
132 	.name = "ppc"
133 };
134 
135 static boolean_t sha512_have_isa207(void)
136 {
137 	return (kfpu_allowed() && zfs_isa207_available());
138 }
139 
140 TF(zfs_sha512_power8, tf_sha512_power8);
141 const sha512_ops_t sha512_power8_impl = {
142 	.is_supported = sha512_have_isa207,
143 	.transform = tf_sha512_power8,
144 	.name = "power8"
145 };
146 #endif /* __PPC64__ */
147 
148 /* the two generic ones */
149 extern const sha512_ops_t sha512_generic_impl;
150 
151 /* array with all sha512 implementations */
152 static const sha512_ops_t *const sha512_impls[] = {
153 	&sha512_generic_impl,
154 #if defined(__x86_64)
155 	&sha512_x64_impl,
156 #endif
157 #if defined(__x86_64) && defined(HAVE_AVX)
158 	&sha512_avx_impl,
159 #endif
160 #if defined(__x86_64) && defined(HAVE_AVX2)
161 	&sha512_avx2_impl,
162 #endif
163 #if defined(__aarch64__) || defined(__arm__)
164 	&sha512_armv7_impl,
165 #if defined(__aarch64__)
166 	&sha512_armv8_impl,
167 #endif
168 #if defined(__arm__) && __ARM_ARCH > 6
169 	&sha512_neon_impl,
170 #endif
171 #endif
172 #if defined(__PPC64__)
173 	&sha512_ppc_impl,
174 	&sha512_power8_impl,
175 #endif /* __PPC64__ */
176 };
177 
178 /* use the generic implementation functions */
179 #define	IMPL_NAME		"sha512"
180 #define	IMPL_OPS_T		sha512_ops_t
181 #define	IMPL_ARRAY		sha512_impls
182 #define	IMPL_GET_OPS		sha512_get_ops
183 #define	ZFS_IMPL_OPS		zfs_sha512_ops
184 #include <generic_impl.c>
185 
186 #ifdef _KERNEL
187 
188 #define	IMPL_FMT(impl, i)	(((impl) == (i)) ? "[%s] " : "%s ")
189 
190 #if defined(__linux__)
191 
192 static int
193 sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
194 {
195 	const uint32_t impl = IMPL_READ(generic_impl_chosen);
196 	char *fmt;
197 	int cnt = 0;
198 
199 	/* cycling */
200 	fmt = IMPL_FMT(impl, IMPL_CYCLE);
201 	cnt += sprintf(buffer + cnt, fmt, "cycle");
202 
203 	/* list fastest */
204 	fmt = IMPL_FMT(impl, IMPL_FASTEST);
205 	cnt += sprintf(buffer + cnt, fmt, "fastest");
206 
207 	/* list all supported implementations */
208 	generic_impl_init();
209 	for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
210 		fmt = IMPL_FMT(impl, i);
211 		cnt += sprintf(buffer + cnt, fmt,
212 		    generic_supp_impls[i]->name);
213 	}
214 
215 	return (cnt);
216 }
217 
218 static int
219 sha512_param_set(const char *val, zfs_kernel_param_t *unused)
220 {
221 	(void) unused;
222 	return (generic_impl_setname(val));
223 }
224 
225 #elif defined(__FreeBSD__)
226 
227 #include <sys/sbuf.h>
228 
229 static int
230 sha512_param(ZFS_MODULE_PARAM_ARGS)
231 {
232 	int err;
233 
234 	generic_impl_init();
235 	if (req->newptr == NULL) {
236 		const uint32_t impl = IMPL_READ(generic_impl_chosen);
237 		const int init_buflen = 64;
238 		const char *fmt;
239 		struct sbuf *s;
240 
241 		s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
242 
243 		/* cycling */
244 		fmt = IMPL_FMT(impl, IMPL_CYCLE);
245 		(void) sbuf_printf(s, fmt, "cycle");
246 
247 		/* list fastest */
248 		fmt = IMPL_FMT(impl, IMPL_FASTEST);
249 		(void) sbuf_printf(s, fmt, "fastest");
250 
251 		/* list all supported implementations */
252 		for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
253 			fmt = IMPL_FMT(impl, i);
254 			(void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
255 		}
256 
257 		err = sbuf_finish(s);
258 		sbuf_delete(s);
259 
260 		return (err);
261 	}
262 
263 	/* we got module parameter */
264 	char buf[16];
265 
266 	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
267 	if (err) {
268 		return (err);
269 	}
270 
271 	return (-generic_impl_setname(buf));
272 }
273 #endif
274 
275 #undef IMPL_FMT
276 
277 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
278     sha512_param_set, sha512_param_get, ZMOD_RW, \
279 	"Select SHA512 implementation.");
280 #endif
281 
282 #undef TF
283