1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 */
26
27 #include <sys/simd.h>
28 #include <sys/zfs_context.h>
29 #include <sys/zfs_impl.h>
30 #include <sys/sha2.h>
31
32 #include <sha2/sha2_impl.h>
33 #include <sys/asm_linkage.h>
34
35 #define TF(E, N) \
36 extern void ASMABI E(uint64_t s[8], const void *, size_t); \
37 static inline void N(uint64_t s[8], const void *d, size_t b) { \
38 kfpu_begin(); E(s, d, b); kfpu_end(); \
39 }
40
41 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
42 defined(__aarch64__) || defined(__arm__) || defined(__PPC64__)
43 /* some implementation is always okay */
sha2_is_supported(void)44 static inline boolean_t sha2_is_supported(void)
45 {
46 return (B_TRUE);
47 }
48 #endif
49
50 #if defined(__x86_64)
51
52 /* Users of ASMABI requires all calls to be from wrappers */
53 extern void ASMABI
54 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
55
56 static inline void
tf_sha512_transform_x64(uint64_t s[8],const void * d,size_t b)57 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
58 {
59 zfs_sha512_transform_x64(s, d, b);
60 }
61 const sha512_ops_t sha512_x64_impl = {
62 .is_supported = sha2_is_supported,
63 .transform = tf_sha512_transform_x64,
64 .name = "x64"
65 };
66
67 #if defined(HAVE_AVX)
sha2_have_avx(void)68 static boolean_t sha2_have_avx(void)
69 {
70 return (kfpu_allowed() && zfs_avx_available());
71 }
72
73 TF(zfs_sha512_transform_avx, tf_sha512_avx);
74 const sha512_ops_t sha512_avx_impl = {
75 .is_supported = sha2_have_avx,
76 .transform = tf_sha512_avx,
77 .name = "avx"
78 };
79 #endif
80
81 #if defined(HAVE_AVX2)
sha2_have_avx2(void)82 static boolean_t sha2_have_avx2(void)
83 {
84 return (kfpu_allowed() && zfs_avx2_available());
85 }
86
87 TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
88 const sha512_ops_t sha512_avx2_impl = {
89 .is_supported = sha2_have_avx2,
90 .transform = tf_sha512_avx2,
91 .name = "avx2"
92 };
93 #endif
94
95 #elif defined(__aarch64__) || defined(__arm__)
96 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
97 const sha512_ops_t sha512_armv7_impl = {
98 .is_supported = sha2_is_supported,
99 .transform = zfs_sha512_block_armv7,
100 .name = "armv7"
101 };
102
103 #if defined(__aarch64__)
sha512_have_armv8ce(void)104 static boolean_t sha512_have_armv8ce(void)
105 {
106 return (kfpu_allowed() && zfs_sha512_available());
107 }
108
109 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
110 const sha512_ops_t sha512_armv8_impl = {
111 .is_supported = sha512_have_armv8ce,
112 .transform = tf_sha512_armv8ce,
113 .name = "armv8-ce"
114 };
115 #endif
116
117 #if defined(__arm__) && __ARM_ARCH > 6
sha512_have_neon(void)118 static boolean_t sha512_have_neon(void)
119 {
120 return (kfpu_allowed() && zfs_neon_available());
121 }
122
123 TF(zfs_sha512_block_neon, tf_sha512_neon);
124 const sha512_ops_t sha512_neon_impl = {
125 .is_supported = sha512_have_neon,
126 .transform = tf_sha512_neon,
127 .name = "neon"
128 };
129 #endif
130
131 #elif defined(__PPC64__)
132 TF(zfs_sha512_ppc, tf_sha512_ppc);
133 const sha512_ops_t sha512_ppc_impl = {
134 .is_supported = sha2_is_supported,
135 .transform = tf_sha512_ppc,
136 .name = "ppc"
137 };
138
sha512_have_isa207(void)139 static boolean_t sha512_have_isa207(void)
140 {
141 return (kfpu_allowed() && zfs_isa207_available());
142 }
143
144 TF(zfs_sha512_power8, tf_sha512_power8);
145 const sha512_ops_t sha512_power8_impl = {
146 .is_supported = sha512_have_isa207,
147 .transform = tf_sha512_power8,
148 .name = "power8"
149 };
150 #endif /* __PPC64__ */
151
152 /* the two generic ones */
153 extern const sha512_ops_t sha512_generic_impl;
154
155 /* array with all sha512 implementations */
156 static const sha512_ops_t *const sha512_impls[] = {
157 &sha512_generic_impl,
158 #if defined(__x86_64)
159 &sha512_x64_impl,
160 #endif
161 #if defined(__x86_64) && defined(HAVE_AVX)
162 &sha512_avx_impl,
163 #endif
164 #if defined(__x86_64) && defined(HAVE_AVX2)
165 &sha512_avx2_impl,
166 #endif
167 #if defined(__aarch64__) || defined(__arm__)
168 &sha512_armv7_impl,
169 #if defined(__aarch64__)
170 &sha512_armv8_impl,
171 #endif
172 #if defined(__arm__) && __ARM_ARCH > 6
173 &sha512_neon_impl,
174 #endif
175 #endif
176 #if defined(__PPC64__)
177 &sha512_ppc_impl,
178 &sha512_power8_impl,
179 #endif /* __PPC64__ */
180 };
181
182 /* use the generic implementation functions */
183 #define IMPL_NAME "sha512"
184 #define IMPL_OPS_T sha512_ops_t
185 #define IMPL_ARRAY sha512_impls
186 #define IMPL_GET_OPS sha512_get_ops
187 #define ZFS_IMPL_OPS zfs_sha512_ops
188 #include <generic_impl.c>
189
190 #ifdef _KERNEL
191
192 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
193
194 #if defined(__linux__)
195
196 static int
sha512_param_get(char * buffer,zfs_kernel_param_t * unused)197 sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
198 {
199 const uint32_t impl = IMPL_READ(generic_impl_chosen);
200 char *fmt;
201 int cnt = 0;
202
203 /* cycling */
204 fmt = IMPL_FMT(impl, IMPL_CYCLE);
205 cnt += sprintf(buffer + cnt, fmt, "cycle");
206
207 /* list fastest */
208 fmt = IMPL_FMT(impl, IMPL_FASTEST);
209 cnt += sprintf(buffer + cnt, fmt, "fastest");
210
211 /* list all supported implementations */
212 generic_impl_init();
213 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
214 fmt = IMPL_FMT(impl, i);
215 cnt += sprintf(buffer + cnt, fmt,
216 generic_supp_impls[i]->name);
217 }
218
219 return (cnt);
220 }
221
222 static int
sha512_param_set(const char * val,zfs_kernel_param_t * unused)223 sha512_param_set(const char *val, zfs_kernel_param_t *unused)
224 {
225 (void) unused;
226 return (generic_impl_setname(val));
227 }
228
229 #elif defined(__FreeBSD__)
230
231 #include <sys/sbuf.h>
232
233 static int
sha512_param(ZFS_MODULE_PARAM_ARGS)234 sha512_param(ZFS_MODULE_PARAM_ARGS)
235 {
236 int err;
237
238 generic_impl_init();
239 if (req->newptr == NULL) {
240 const uint32_t impl = IMPL_READ(generic_impl_chosen);
241 const int init_buflen = 64;
242 const char *fmt;
243 struct sbuf *s;
244
245 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
246
247 /* cycling */
248 fmt = IMPL_FMT(impl, IMPL_CYCLE);
249 (void) sbuf_printf(s, fmt, "cycle");
250
251 /* list fastest */
252 fmt = IMPL_FMT(impl, IMPL_FASTEST);
253 (void) sbuf_printf(s, fmt, "fastest");
254
255 /* list all supported implementations */
256 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
257 fmt = IMPL_FMT(impl, i);
258 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
259 }
260
261 err = sbuf_finish(s);
262 sbuf_delete(s);
263
264 return (err);
265 }
266
267 /* we got module parameter */
268 char buf[16];
269
270 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
271 if (err) {
272 return (err);
273 }
274
275 return (-generic_impl_setname(buf));
276 }
277 #endif
278
279 #undef IMPL_FMT
280
281 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
282 sha512_param_set, sha512_param_get, ZMOD_RW, \
283 "Select SHA512 implementation.");
284 #endif
285
286 #undef TF
287