1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 */
26
27 #include <sys/simd.h>
28 #include <sys/zfs_context.h>
29 #include <sys/zfs_impl.h>
30 #include <sys/sha2.h>
31
32 #include <sha2/sha2_impl.h>
33 #include <sys/asm_linkage.h>
34
35 #define TF(E, N) \
36 extern void ASMABI E(uint32_t s[8], const void *, size_t); \
37 static inline void N(uint32_t s[8], const void *d, size_t b) { \
38 kfpu_begin(); E(s, d, b); kfpu_end(); \
39 }
40
41 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
42 defined(__PPC64__)
43 /* some implementation is always okay */
sha2_is_supported(void)44 static inline boolean_t sha2_is_supported(void)
45 {
46 return (B_TRUE);
47 }
48 #endif
49
50 #if defined(__x86_64)
51
52 /* Users of ASMABI requires all calls to be from wrappers */
53 extern void ASMABI
54 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
55
56 static inline void
tf_sha256_transform_x64(uint32_t s[8],const void * d,size_t b)57 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
58 {
59 zfs_sha256_transform_x64(s, d, b);
60 }
61
62 const sha256_ops_t sha256_x64_impl = {
63 .is_supported = sha2_is_supported,
64 .transform = tf_sha256_transform_x64,
65 .name = "x64"
66 };
67
68 #if defined(HAVE_SSSE3)
sha2_have_ssse3(void)69 static boolean_t sha2_have_ssse3(void)
70 {
71 return (kfpu_allowed() && zfs_ssse3_available());
72 }
73
74 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
75 const sha256_ops_t sha256_ssse3_impl = {
76 .is_supported = sha2_have_ssse3,
77 .transform = tf_sha256_ssse3,
78 .name = "ssse3"
79 };
80 #endif
81
82 #if defined(HAVE_AVX)
sha2_have_avx(void)83 static boolean_t sha2_have_avx(void)
84 {
85 return (kfpu_allowed() && zfs_avx_available());
86 }
87
88 TF(zfs_sha256_transform_avx, tf_sha256_avx);
89 const sha256_ops_t sha256_avx_impl = {
90 .is_supported = sha2_have_avx,
91 .transform = tf_sha256_avx,
92 .name = "avx"
93 };
94 #endif
95
96 #if defined(HAVE_AVX2)
sha2_have_avx2(void)97 static boolean_t sha2_have_avx2(void)
98 {
99 return (kfpu_allowed() && zfs_avx2_available());
100 }
101
102 TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
103 const sha256_ops_t sha256_avx2_impl = {
104 .is_supported = sha2_have_avx2,
105 .transform = tf_sha256_avx2,
106 .name = "avx2"
107 };
108 #endif
109
110 #if defined(HAVE_SSE4_1)
sha2_have_shani(void)111 static boolean_t sha2_have_shani(void)
112 {
113 return (kfpu_allowed() && zfs_sse4_1_available() && \
114 zfs_shani_available());
115 }
116
117 TF(zfs_sha256_transform_shani, tf_sha256_shani);
118 const sha256_ops_t sha256_shani_impl = {
119 .is_supported = sha2_have_shani,
120 .transform = tf_sha256_shani,
121 .name = "shani"
122 };
123 #endif
124
125 #elif defined(__aarch64__) || defined(__arm__)
126 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
127 const sha256_ops_t sha256_armv7_impl = {
128 .is_supported = sha2_is_supported,
129 .transform = zfs_sha256_block_armv7,
130 .name = "armv7"
131 };
132
133 #if __ARM_ARCH > 6
sha256_have_neon(void)134 static boolean_t sha256_have_neon(void)
135 {
136 return (kfpu_allowed() && zfs_neon_available());
137 }
138
sha256_have_armv8ce(void)139 static boolean_t sha256_have_armv8ce(void)
140 {
141 return (kfpu_allowed() && zfs_sha256_available());
142 }
143
144 TF(zfs_sha256_block_neon, tf_sha256_neon);
145 const sha256_ops_t sha256_neon_impl = {
146 .is_supported = sha256_have_neon,
147 .transform = tf_sha256_neon,
148 .name = "neon"
149 };
150
151 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
152 const sha256_ops_t sha256_armv8_impl = {
153 .is_supported = sha256_have_armv8ce,
154 .transform = tf_sha256_armv8ce,
155 .name = "armv8-ce"
156 };
157 #endif
158
159 #elif defined(__PPC64__)
sha256_have_isa207(void)160 static boolean_t sha256_have_isa207(void)
161 {
162 return (kfpu_allowed() && zfs_isa207_available());
163 }
164
165 TF(zfs_sha256_ppc, tf_sha256_ppc);
166 const sha256_ops_t sha256_ppc_impl = {
167 .is_supported = sha2_is_supported,
168 .transform = tf_sha256_ppc,
169 .name = "ppc"
170 };
171
172 TF(zfs_sha256_power8, tf_sha256_power8);
173 const sha256_ops_t sha256_power8_impl = {
174 .is_supported = sha256_have_isa207,
175 .transform = tf_sha256_power8,
176 .name = "power8"
177 };
178 #endif /* __PPC64__ */
179
180 /* the two generic ones */
181 extern const sha256_ops_t sha256_generic_impl;
182
183 /* array with all sha256 implementations */
184 static const sha256_ops_t *const sha256_impls[] = {
185 &sha256_generic_impl,
186 #if defined(__x86_64)
187 &sha256_x64_impl,
188 #endif
189 #if defined(__x86_64) && defined(HAVE_SSSE3)
190 &sha256_ssse3_impl,
191 #endif
192 #if defined(__x86_64) && defined(HAVE_AVX)
193 &sha256_avx_impl,
194 #endif
195 #if defined(__x86_64) && defined(HAVE_AVX2)
196 &sha256_avx2_impl,
197 #endif
198 #if defined(__x86_64) && defined(HAVE_SSE4_1)
199 &sha256_shani_impl,
200 #endif
201 #if defined(__aarch64__) || defined(__arm__)
202 &sha256_armv7_impl,
203 #if __ARM_ARCH > 6
204 &sha256_neon_impl,
205 &sha256_armv8_impl,
206 #endif
207 #endif
208 #if defined(__PPC64__)
209 &sha256_ppc_impl,
210 &sha256_power8_impl,
211 #endif /* __PPC64__ */
212 };
213
214 /* use the generic implementation functions */
215 #define IMPL_NAME "sha256"
216 #define IMPL_OPS_T sha256_ops_t
217 #define IMPL_ARRAY sha256_impls
218 #define IMPL_GET_OPS sha256_get_ops
219 #define ZFS_IMPL_OPS zfs_sha256_ops
220 #include <generic_impl.c>
221
222 #ifdef _KERNEL
223
224 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
225
226 #if defined(__linux__)
227
228 static int
sha256_param_get(char * buffer,zfs_kernel_param_t * unused)229 sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
230 {
231 const uint32_t impl = IMPL_READ(generic_impl_chosen);
232 char *fmt;
233 int cnt = 0;
234
235 /* cycling */
236 fmt = IMPL_FMT(impl, IMPL_CYCLE);
237 cnt += sprintf(buffer + cnt, fmt, "cycle");
238
239 /* list fastest */
240 fmt = IMPL_FMT(impl, IMPL_FASTEST);
241 cnt += sprintf(buffer + cnt, fmt, "fastest");
242
243 /* list all supported implementations */
244 generic_impl_init();
245 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
246 fmt = IMPL_FMT(impl, i);
247 cnt += sprintf(buffer + cnt, fmt,
248 generic_supp_impls[i]->name);
249 }
250
251 return (cnt);
252 }
253
254 static int
sha256_param_set(const char * val,zfs_kernel_param_t * unused)255 sha256_param_set(const char *val, zfs_kernel_param_t *unused)
256 {
257 (void) unused;
258 return (generic_impl_setname(val));
259 }
260
261 #elif defined(__FreeBSD__)
262
263 #include <sys/sbuf.h>
264
265 static int
sha256_param(ZFS_MODULE_PARAM_ARGS)266 sha256_param(ZFS_MODULE_PARAM_ARGS)
267 {
268 int err;
269
270 generic_impl_init();
271 if (req->newptr == NULL) {
272 const uint32_t impl = IMPL_READ(generic_impl_chosen);
273 const int init_buflen = 64;
274 const char *fmt;
275 struct sbuf *s;
276
277 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
278
279 /* cycling */
280 fmt = IMPL_FMT(impl, IMPL_CYCLE);
281 (void) sbuf_printf(s, fmt, "cycle");
282
283 /* list fastest */
284 fmt = IMPL_FMT(impl, IMPL_FASTEST);
285 (void) sbuf_printf(s, fmt, "fastest");
286
287 /* list all supported implementations */
288 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
289 fmt = IMPL_FMT(impl, i);
290 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
291 }
292
293 err = sbuf_finish(s);
294 sbuf_delete(s);
295
296 return (err);
297 }
298
299 char buf[16];
300
301 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
302 if (err) {
303 return (err);
304 }
305
306 return (-generic_impl_setname(buf));
307 }
308 #endif
309
310 #undef IMPL_FMT
311
312 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
313 sha256_param_set, sha256_param_get, ZMOD_RW, \
314 "Select SHA256 implementation.");
315 #endif
316
317 #undef TF
318