1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 */
26
27 #include <sys/simd.h>
28 #include <sys/zfs_context.h>
29 #include <sys/zfs_impl.h>
30 #include <sys/sha2.h>
31
32 #include <sha2/sha2_impl.h>
33 #include <sys/asm_linkage.h>
34
35 #define TF(E, N) \
36 extern void ASMABI E(uint64_t s[8], const void *, size_t); \
37 static inline void N(uint64_t s[8], const void *d, size_t b) { \
38 kfpu_begin(); E(s, d, b); kfpu_end(); \
39 }
40
41 /* some implementation is always okay */
sha2_is_supported(void)42 static inline boolean_t sha2_is_supported(void)
43 {
44 return (B_TRUE);
45 }
46
47 #if defined(__x86_64)
48
49 /* Users of ASMABI requires all calls to be from wrappers */
50 extern void ASMABI
51 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
52
53 static inline void
tf_sha512_transform_x64(uint64_t s[8],const void * d,size_t b)54 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
55 {
56 zfs_sha512_transform_x64(s, d, b);
57 }
58 const sha512_ops_t sha512_x64_impl = {
59 .is_supported = sha2_is_supported,
60 .transform = tf_sha512_transform_x64,
61 .name = "x64"
62 };
63
64 #if defined(HAVE_AVX)
sha2_have_avx(void)65 static boolean_t sha2_have_avx(void)
66 {
67 return (kfpu_allowed() && zfs_avx_available());
68 }
69
70 TF(zfs_sha512_transform_avx, tf_sha512_avx);
71 const sha512_ops_t sha512_avx_impl = {
72 .is_supported = sha2_have_avx,
73 .transform = tf_sha512_avx,
74 .name = "avx"
75 };
76 #endif
77
78 #if defined(HAVE_AVX2)
sha2_have_avx2(void)79 static boolean_t sha2_have_avx2(void)
80 {
81 return (kfpu_allowed() && zfs_avx2_available());
82 }
83
84 TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
85 const sha512_ops_t sha512_avx2_impl = {
86 .is_supported = sha2_have_avx2,
87 .transform = tf_sha512_avx2,
88 .name = "avx2"
89 };
90 #endif
91
92 #elif defined(__aarch64__) || defined(__arm__)
93 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
94 const sha512_ops_t sha512_armv7_impl = {
95 .is_supported = sha2_is_supported,
96 .transform = zfs_sha512_block_armv7,
97 .name = "armv7"
98 };
99
100 #if defined(__aarch64__)
sha512_have_armv8ce(void)101 static boolean_t sha512_have_armv8ce(void)
102 {
103 return (kfpu_allowed() && zfs_sha512_available());
104 }
105
106 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
107 const sha512_ops_t sha512_armv8_impl = {
108 .is_supported = sha512_have_armv8ce,
109 .transform = tf_sha512_armv8ce,
110 .name = "armv8-ce"
111 };
112 #endif
113
114 #if defined(__arm__) && __ARM_ARCH > 6
sha512_have_neon(void)115 static boolean_t sha512_have_neon(void)
116 {
117 return (kfpu_allowed() && zfs_neon_available());
118 }
119
120 TF(zfs_sha512_block_neon, tf_sha512_neon);
121 const sha512_ops_t sha512_neon_impl = {
122 .is_supported = sha512_have_neon,
123 .transform = tf_sha512_neon,
124 .name = "neon"
125 };
126 #endif
127
128 #elif defined(__PPC64__)
129 TF(zfs_sha512_ppc, tf_sha512_ppc);
130 const sha512_ops_t sha512_ppc_impl = {
131 .is_supported = sha2_is_supported,
132 .transform = tf_sha512_ppc,
133 .name = "ppc"
134 };
135
sha512_have_isa207(void)136 static boolean_t sha512_have_isa207(void)
137 {
138 return (kfpu_allowed() && zfs_isa207_available());
139 }
140
141 TF(zfs_sha512_power8, tf_sha512_power8);
142 const sha512_ops_t sha512_power8_impl = {
143 .is_supported = sha512_have_isa207,
144 .transform = tf_sha512_power8,
145 .name = "power8"
146 };
147 #endif /* __PPC64__ */
148
149 /* the two generic ones */
150 extern const sha512_ops_t sha512_generic_impl;
151
152 /* array with all sha512 implementations */
153 static const sha512_ops_t *const sha512_impls[] = {
154 &sha512_generic_impl,
155 #if defined(__x86_64)
156 &sha512_x64_impl,
157 #endif
158 #if defined(__x86_64) && defined(HAVE_AVX)
159 &sha512_avx_impl,
160 #endif
161 #if defined(__x86_64) && defined(HAVE_AVX2)
162 &sha512_avx2_impl,
163 #endif
164 #if defined(__aarch64__) || defined(__arm__)
165 &sha512_armv7_impl,
166 #if defined(__aarch64__)
167 &sha512_armv8_impl,
168 #endif
169 #if defined(__arm__) && __ARM_ARCH > 6
170 &sha512_neon_impl,
171 #endif
172 #endif
173 #if defined(__PPC64__)
174 &sha512_ppc_impl,
175 &sha512_power8_impl,
176 #endif /* __PPC64__ */
177 };
178
179 /* use the generic implementation functions */
180 #define IMPL_NAME "sha512"
181 #define IMPL_OPS_T sha512_ops_t
182 #define IMPL_ARRAY sha512_impls
183 #define IMPL_GET_OPS sha512_get_ops
184 #define ZFS_IMPL_OPS zfs_sha512_ops
185 #include <generic_impl.c>
186
187 #ifdef _KERNEL
188
189 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
190
191 #if defined(__linux__)
192
193 static int
sha512_param_get(char * buffer,zfs_kernel_param_t * unused)194 sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
195 {
196 const uint32_t impl = IMPL_READ(generic_impl_chosen);
197 char *fmt;
198 int cnt = 0;
199
200 /* cycling */
201 fmt = IMPL_FMT(impl, IMPL_CYCLE);
202 cnt += sprintf(buffer + cnt, fmt, "cycle");
203
204 /* list fastest */
205 fmt = IMPL_FMT(impl, IMPL_FASTEST);
206 cnt += sprintf(buffer + cnt, fmt, "fastest");
207
208 /* list all supported implementations */
209 generic_impl_init();
210 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
211 fmt = IMPL_FMT(impl, i);
212 cnt += sprintf(buffer + cnt, fmt,
213 generic_supp_impls[i]->name);
214 }
215
216 return (cnt);
217 }
218
219 static int
sha512_param_set(const char * val,zfs_kernel_param_t * unused)220 sha512_param_set(const char *val, zfs_kernel_param_t *unused)
221 {
222 (void) unused;
223 return (generic_impl_setname(val));
224 }
225
226 #elif defined(__FreeBSD__)
227
228 #include <sys/sbuf.h>
229
230 static int
sha512_param(ZFS_MODULE_PARAM_ARGS)231 sha512_param(ZFS_MODULE_PARAM_ARGS)
232 {
233 int err;
234
235 generic_impl_init();
236 if (req->newptr == NULL) {
237 const uint32_t impl = IMPL_READ(generic_impl_chosen);
238 const int init_buflen = 64;
239 const char *fmt;
240 struct sbuf *s;
241
242 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
243
244 /* cycling */
245 fmt = IMPL_FMT(impl, IMPL_CYCLE);
246 (void) sbuf_printf(s, fmt, "cycle");
247
248 /* list fastest */
249 fmt = IMPL_FMT(impl, IMPL_FASTEST);
250 (void) sbuf_printf(s, fmt, "fastest");
251
252 /* list all supported implementations */
253 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
254 fmt = IMPL_FMT(impl, i);
255 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
256 }
257
258 err = sbuf_finish(s);
259 sbuf_delete(s);
260
261 return (err);
262 }
263
264 /* we got module parameter */
265 char buf[16];
266
267 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
268 if (err) {
269 return (err);
270 }
271
272 return (-generic_impl_setname(buf));
273 }
274 #endif
275
276 #undef IMPL_FMT
277
278 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
279 sha512_param_set, sha512_param_get, ZMOD_RW, \
280 "Select SHA512 implementation.");
281 #endif
282
283 #undef TF
284