1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
15 */
16
17 #include <sys/simd.h>
18 #include <sys/zfs_context.h>
19 #include <sys/zfs_impl.h>
20 #include <sys/sha2.h>
21
22 #include <sha2/sha2_impl.h>
23 #include <sys/asm_linkage.h>
24
25 #define TF(E, N) \
26 extern void ASMABI E(uint32_t s[8], const void *, size_t); \
27 static inline void N(uint32_t s[8], const void *d, size_t b) { \
28 kfpu_begin(); E(s, d, b); kfpu_end(); \
29 }
30
31 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
32 defined(__PPC64__)
33 /* some implementation is always okay */
sha2_is_supported(void)34 static inline boolean_t sha2_is_supported(void)
35 {
36 return (B_TRUE);
37 }
38 #endif
39
40 #if defined(__x86_64)
41
42 /* Users of ASMABI requires all calls to be from wrappers */
43 extern void ASMABI
44 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
45
46 static inline void
tf_sha256_transform_x64(uint32_t s[8],const void * d,size_t b)47 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
48 {
49 zfs_sha256_transform_x64(s, d, b);
50 }
51
52 const sha256_ops_t sha256_x64_impl = {
53 .is_supported = sha2_is_supported,
54 .transform = tf_sha256_transform_x64,
55 .name = "x64"
56 };
57
58 #if HAVE_SIMD(SSSE3)
sha2_have_ssse3(void)59 static boolean_t sha2_have_ssse3(void)
60 {
61 return (kfpu_allowed() && zfs_ssse3_available());
62 }
63
64 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
65 const sha256_ops_t sha256_ssse3_impl = {
66 .is_supported = sha2_have_ssse3,
67 .transform = tf_sha256_ssse3,
68 .name = "ssse3"
69 };
70 #endif
71
72 #if HAVE_SIMD(AVX)
sha2_have_avx(void)73 static boolean_t sha2_have_avx(void)
74 {
75 return (kfpu_allowed() && zfs_avx_available());
76 }
77
78 TF(zfs_sha256_transform_avx, tf_sha256_avx);
79 const sha256_ops_t sha256_avx_impl = {
80 .is_supported = sha2_have_avx,
81 .transform = tf_sha256_avx,
82 .name = "avx"
83 };
84 #endif
85
86 #if HAVE_SIMD(AVX2)
sha2_have_avx2(void)87 static boolean_t sha2_have_avx2(void)
88 {
89 return (kfpu_allowed() && zfs_avx2_available());
90 }
91
92 TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
93 const sha256_ops_t sha256_avx2_impl = {
94 .is_supported = sha2_have_avx2,
95 .transform = tf_sha256_avx2,
96 .name = "avx2"
97 };
98 #endif
99
100 #if HAVE_SIMD(SSE4_1)
sha2_have_shani(void)101 static boolean_t sha2_have_shani(void)
102 {
103 return (kfpu_allowed() && zfs_sse4_1_available() && \
104 zfs_shani_available());
105 }
106
107 TF(zfs_sha256_transform_shani, tf_sha256_shani);
108 const sha256_ops_t sha256_shani_impl = {
109 .is_supported = sha2_have_shani,
110 .transform = tf_sha256_shani,
111 .name = "shani"
112 };
113 #endif
114
115 #elif defined(__aarch64__) || defined(__arm__)
116 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
117 const sha256_ops_t sha256_armv7_impl = {
118 .is_supported = sha2_is_supported,
119 .transform = zfs_sha256_block_armv7,
120 .name = "armv7"
121 };
122
123 #if __ARM_ARCH > 6
sha256_have_neon(void)124 static boolean_t sha256_have_neon(void)
125 {
126 return (kfpu_allowed() && zfs_neon_available());
127 }
128
sha256_have_armv8ce(void)129 static boolean_t sha256_have_armv8ce(void)
130 {
131 return (kfpu_allowed() && zfs_sha256_available());
132 }
133
134 TF(zfs_sha256_block_neon, tf_sha256_neon);
135 const sha256_ops_t sha256_neon_impl = {
136 .is_supported = sha256_have_neon,
137 .transform = tf_sha256_neon,
138 .name = "neon"
139 };
140
141 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
142 const sha256_ops_t sha256_armv8_impl = {
143 .is_supported = sha256_have_armv8ce,
144 .transform = tf_sha256_armv8ce,
145 .name = "armv8-ce"
146 };
147 #endif
148
149 #elif defined(__PPC64__)
sha256_have_isa207(void)150 static boolean_t sha256_have_isa207(void)
151 {
152 return (kfpu_allowed() && zfs_isa207_available());
153 }
154
155 TF(zfs_sha256_ppc, tf_sha256_ppc);
156 const sha256_ops_t sha256_ppc_impl = {
157 .is_supported = sha2_is_supported,
158 .transform = tf_sha256_ppc,
159 .name = "ppc"
160 };
161
162 TF(zfs_sha256_power8, tf_sha256_power8);
163 const sha256_ops_t sha256_power8_impl = {
164 .is_supported = sha256_have_isa207,
165 .transform = tf_sha256_power8,
166 .name = "power8"
167 };
168 #endif /* __PPC64__ */
169
170 /* the two generic ones */
171 extern const sha256_ops_t sha256_generic_impl;
172
173 /* array with all sha256 implementations */
174 static const sha256_ops_t *const sha256_impls[] = {
175 &sha256_generic_impl,
176 #if defined(__x86_64)
177 &sha256_x64_impl,
178 #endif
179 #if defined(__x86_64) && HAVE_SIMD(SSSE3)
180 &sha256_ssse3_impl,
181 #endif
182 #if defined(__x86_64) && HAVE_SIMD(AVX)
183 &sha256_avx_impl,
184 #endif
185 #if defined(__x86_64) && HAVE_SIMD(AVX2)
186 &sha256_avx2_impl,
187 #endif
188 #if defined(__x86_64) && HAVE_SIMD(SSE4_1)
189 &sha256_shani_impl,
190 #endif
191 #if defined(__aarch64__) || defined(__arm__)
192 &sha256_armv7_impl,
193 #if __ARM_ARCH > 6
194 &sha256_neon_impl,
195 &sha256_armv8_impl,
196 #endif
197 #endif
198 #if defined(__PPC64__)
199 &sha256_ppc_impl,
200 &sha256_power8_impl,
201 #endif /* __PPC64__ */
202 };
203
204 /* use the generic implementation functions */
205 #define IMPL_NAME "sha256"
206 #define IMPL_OPS_T sha256_ops_t
207 #define IMPL_ARRAY sha256_impls
208 #define IMPL_GET_OPS sha256_get_ops
209 #define ZFS_IMPL_OPS zfs_sha256_ops
210 #include <generic_impl.c>
211
212 #ifdef _KERNEL
213
214 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
215
216 #if defined(__linux__)
217
218 static int
sha256_param_get(char * buffer,zfs_kernel_param_t * unused)219 sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
220 {
221 const uint32_t impl = IMPL_READ(generic_impl_chosen);
222 char *fmt;
223 int cnt = 0;
224
225 /* cycling */
226 fmt = IMPL_FMT(impl, IMPL_CYCLE);
227 cnt += sprintf(buffer + cnt, fmt, "cycle");
228
229 /* list fastest */
230 fmt = IMPL_FMT(impl, IMPL_FASTEST);
231 cnt += sprintf(buffer + cnt, fmt, "fastest");
232
233 /* list all supported implementations */
234 generic_impl_init();
235 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
236 fmt = IMPL_FMT(impl, i);
237 cnt += sprintf(buffer + cnt, fmt,
238 generic_supp_impls[i]->name);
239 }
240
241 return (cnt);
242 }
243
244 static int
sha256_param_set(const char * val,zfs_kernel_param_t * unused)245 sha256_param_set(const char *val, zfs_kernel_param_t *unused)
246 {
247 (void) unused;
248 return (generic_impl_setname(val));
249 }
250
251 #elif defined(__FreeBSD__)
252
253 #include <sys/sbuf.h>
254
255 static int
sha256_param(ZFS_MODULE_PARAM_ARGS)256 sha256_param(ZFS_MODULE_PARAM_ARGS)
257 {
258 int err;
259
260 generic_impl_init();
261 if (req->newptr == NULL) {
262 const uint32_t impl = IMPL_READ(generic_impl_chosen);
263 const int init_buflen = 64;
264 const char *fmt;
265 struct sbuf *s;
266
267 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
268
269 /* cycling */
270 fmt = IMPL_FMT(impl, IMPL_CYCLE);
271 (void) sbuf_printf(s, fmt, "cycle");
272
273 /* list fastest */
274 fmt = IMPL_FMT(impl, IMPL_FASTEST);
275 (void) sbuf_printf(s, fmt, "fastest");
276
277 /* list all supported implementations */
278 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
279 fmt = IMPL_FMT(impl, i);
280 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
281 }
282
283 err = sbuf_finish(s);
284 sbuf_delete(s);
285
286 return (err);
287 }
288
289 char buf[16];
290
291 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
292 if (err) {
293 return (err);
294 }
295
296 return (-generic_impl_setname(buf));
297 }
298 #endif
299
300 #undef IMPL_FMT
301
302 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
303 sha256_param_set, sha256_param_get, ZMOD_RW, \
304 "Select SHA256 implementation.");
305 #endif
306
307 #undef TF
308