1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 */
26
27 #include <sys/simd.h>
28 #include <sys/zfs_context.h>
29 #include <sys/zfs_impl.h>
30 #include <sys/sha2.h>
31
32 #include <sha2/sha2_impl.h>
33 #include <sys/asm_linkage.h>
34
35 #define TF(E, N) \
36 extern void ASMABI E(uint32_t s[8], const void *, size_t); \
37 static inline void N(uint32_t s[8], const void *d, size_t b) { \
38 kfpu_begin(); E(s, d, b); kfpu_end(); \
39 }
40
41 /* some implementation is always okay */
sha2_is_supported(void)42 static inline boolean_t sha2_is_supported(void)
43 {
44 return (B_TRUE);
45 }
46
47 #if defined(__x86_64)
48
49 /* Users of ASMABI requires all calls to be from wrappers */
50 extern void ASMABI
51 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
52
53 static inline void
tf_sha256_transform_x64(uint32_t s[8],const void * d,size_t b)54 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
55 {
56 zfs_sha256_transform_x64(s, d, b);
57 }
58
59 const sha256_ops_t sha256_x64_impl = {
60 .is_supported = sha2_is_supported,
61 .transform = tf_sha256_transform_x64,
62 .name = "x64"
63 };
64
65 #if defined(HAVE_SSSE3)
sha2_have_ssse3(void)66 static boolean_t sha2_have_ssse3(void)
67 {
68 return (kfpu_allowed() && zfs_ssse3_available());
69 }
70
71 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
72 const sha256_ops_t sha256_ssse3_impl = {
73 .is_supported = sha2_have_ssse3,
74 .transform = tf_sha256_ssse3,
75 .name = "ssse3"
76 };
77 #endif
78
79 #if defined(HAVE_AVX)
sha2_have_avx(void)80 static boolean_t sha2_have_avx(void)
81 {
82 return (kfpu_allowed() && zfs_avx_available());
83 }
84
85 TF(zfs_sha256_transform_avx, tf_sha256_avx);
86 const sha256_ops_t sha256_avx_impl = {
87 .is_supported = sha2_have_avx,
88 .transform = tf_sha256_avx,
89 .name = "avx"
90 };
91 #endif
92
93 #if defined(HAVE_AVX2)
sha2_have_avx2(void)94 static boolean_t sha2_have_avx2(void)
95 {
96 return (kfpu_allowed() && zfs_avx2_available());
97 }
98
99 TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
100 const sha256_ops_t sha256_avx2_impl = {
101 .is_supported = sha2_have_avx2,
102 .transform = tf_sha256_avx2,
103 .name = "avx2"
104 };
105 #endif
106
107 #if defined(HAVE_SSE4_1)
sha2_have_shani(void)108 static boolean_t sha2_have_shani(void)
109 {
110 return (kfpu_allowed() && zfs_sse4_1_available() && \
111 zfs_shani_available());
112 }
113
114 TF(zfs_sha256_transform_shani, tf_sha256_shani);
115 const sha256_ops_t sha256_shani_impl = {
116 .is_supported = sha2_have_shani,
117 .transform = tf_sha256_shani,
118 .name = "shani"
119 };
120 #endif
121
122 #elif defined(__aarch64__) || defined(__arm__)
123 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
124 const sha256_ops_t sha256_armv7_impl = {
125 .is_supported = sha2_is_supported,
126 .transform = zfs_sha256_block_armv7,
127 .name = "armv7"
128 };
129
130 #if __ARM_ARCH > 6
sha256_have_neon(void)131 static boolean_t sha256_have_neon(void)
132 {
133 return (kfpu_allowed() && zfs_neon_available());
134 }
135
sha256_have_armv8ce(void)136 static boolean_t sha256_have_armv8ce(void)
137 {
138 return (kfpu_allowed() && zfs_sha256_available());
139 }
140
141 TF(zfs_sha256_block_neon, tf_sha256_neon);
142 const sha256_ops_t sha256_neon_impl = {
143 .is_supported = sha256_have_neon,
144 .transform = tf_sha256_neon,
145 .name = "neon"
146 };
147
148 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
149 const sha256_ops_t sha256_armv8_impl = {
150 .is_supported = sha256_have_armv8ce,
151 .transform = tf_sha256_armv8ce,
152 .name = "armv8-ce"
153 };
154 #endif
155
156 #elif defined(__PPC64__)
sha256_have_isa207(void)157 static boolean_t sha256_have_isa207(void)
158 {
159 return (kfpu_allowed() && zfs_isa207_available());
160 }
161
162 TF(zfs_sha256_ppc, tf_sha256_ppc);
163 const sha256_ops_t sha256_ppc_impl = {
164 .is_supported = sha2_is_supported,
165 .transform = tf_sha256_ppc,
166 .name = "ppc"
167 };
168
169 TF(zfs_sha256_power8, tf_sha256_power8);
170 const sha256_ops_t sha256_power8_impl = {
171 .is_supported = sha256_have_isa207,
172 .transform = tf_sha256_power8,
173 .name = "power8"
174 };
175 #endif /* __PPC64__ */
176
177 /* the two generic ones */
178 extern const sha256_ops_t sha256_generic_impl;
179
180 /* array with all sha256 implementations */
181 static const sha256_ops_t *const sha256_impls[] = {
182 &sha256_generic_impl,
183 #if defined(__x86_64)
184 &sha256_x64_impl,
185 #endif
186 #if defined(__x86_64) && defined(HAVE_SSSE3)
187 &sha256_ssse3_impl,
188 #endif
189 #if defined(__x86_64) && defined(HAVE_AVX)
190 &sha256_avx_impl,
191 #endif
192 #if defined(__x86_64) && defined(HAVE_AVX2)
193 &sha256_avx2_impl,
194 #endif
195 #if defined(__x86_64) && defined(HAVE_SSE4_1)
196 &sha256_shani_impl,
197 #endif
198 #if defined(__aarch64__) || defined(__arm__)
199 &sha256_armv7_impl,
200 #if __ARM_ARCH > 6
201 &sha256_neon_impl,
202 &sha256_armv8_impl,
203 #endif
204 #endif
205 #if defined(__PPC64__)
206 &sha256_ppc_impl,
207 &sha256_power8_impl,
208 #endif /* __PPC64__ */
209 };
210
211 /* use the generic implementation functions */
212 #define IMPL_NAME "sha256"
213 #define IMPL_OPS_T sha256_ops_t
214 #define IMPL_ARRAY sha256_impls
215 #define IMPL_GET_OPS sha256_get_ops
216 #define ZFS_IMPL_OPS zfs_sha256_ops
217 #include <generic_impl.c>
218
219 #ifdef _KERNEL
220
221 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
222
223 #if defined(__linux__)
224
225 static int
sha256_param_get(char * buffer,zfs_kernel_param_t * unused)226 sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
227 {
228 const uint32_t impl = IMPL_READ(generic_impl_chosen);
229 char *fmt;
230 int cnt = 0;
231
232 /* cycling */
233 fmt = IMPL_FMT(impl, IMPL_CYCLE);
234 cnt += sprintf(buffer + cnt, fmt, "cycle");
235
236 /* list fastest */
237 fmt = IMPL_FMT(impl, IMPL_FASTEST);
238 cnt += sprintf(buffer + cnt, fmt, "fastest");
239
240 /* list all supported implementations */
241 generic_impl_init();
242 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
243 fmt = IMPL_FMT(impl, i);
244 cnt += sprintf(buffer + cnt, fmt,
245 generic_supp_impls[i]->name);
246 }
247
248 return (cnt);
249 }
250
251 static int
sha256_param_set(const char * val,zfs_kernel_param_t * unused)252 sha256_param_set(const char *val, zfs_kernel_param_t *unused)
253 {
254 (void) unused;
255 return (generic_impl_setname(val));
256 }
257
258 #elif defined(__FreeBSD__)
259
260 #include <sys/sbuf.h>
261
262 static int
sha256_param(ZFS_MODULE_PARAM_ARGS)263 sha256_param(ZFS_MODULE_PARAM_ARGS)
264 {
265 int err;
266
267 generic_impl_init();
268 if (req->newptr == NULL) {
269 const uint32_t impl = IMPL_READ(generic_impl_chosen);
270 const int init_buflen = 64;
271 const char *fmt;
272 struct sbuf *s;
273
274 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
275
276 /* cycling */
277 fmt = IMPL_FMT(impl, IMPL_CYCLE);
278 (void) sbuf_printf(s, fmt, "cycle");
279
280 /* list fastest */
281 fmt = IMPL_FMT(impl, IMPL_FASTEST);
282 (void) sbuf_printf(s, fmt, "fastest");
283
284 /* list all supported implementations */
285 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
286 fmt = IMPL_FMT(impl, i);
287 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
288 }
289
290 err = sbuf_finish(s);
291 sbuf_delete(s);
292
293 return (err);
294 }
295
296 char buf[16];
297
298 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
299 if (err) {
300 return (err);
301 }
302
303 return (-generic_impl_setname(buf));
304 }
305 #endif
306
307 #undef IMPL_FMT
308
309 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
310 sha256_param_set, sha256_param_get, ZMOD_RW, \
311 "Select SHA256 implementation.");
312 #endif
313
314 #undef TF
315