1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
15 * Copyright (c) 2026, TrueNAS.
16 */
17
18 #include <sys/simd.h>
19 #include <sys/zfs_context.h>
20 #include <sys/zfs_impl.h>
21 #include <sys/sha2.h>
22
23 #include <sha2/sha2_impl.h>
24 #include <sys/asm_linkage.h>
25
26 #define TF(E, N) \
27 extern void ASMABI E(uint64_t s[8], const void *, size_t); \
28 static inline void N(uint64_t s[8], const void *d, size_t b) { \
29 kfpu_begin(); E(s, d, b); kfpu_end(); \
30 }
31
32 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
33 defined(__aarch64__) || defined(__arm__) || defined(__PPC64__)
34 /* some implementation is always okay */
sha2_is_supported(void)35 static inline boolean_t sha2_is_supported(void)
36 {
37 return (B_TRUE);
38 }
39 #endif
40
41 #if defined(__x86_64)
42
43 /* Users of ASMABI requires all calls to be from wrappers */
44 extern void ASMABI
45 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
46
47 static inline void
tf_sha512_transform_x64(uint64_t s[8],const void * d,size_t b)48 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
49 {
50 zfs_sha512_transform_x64(s, d, b);
51 }
52 const sha512_ops_t sha512_x64_impl = {
53 .is_supported = sha2_is_supported,
54 .transform = tf_sha512_transform_x64,
55 .name = "x64"
56 };
57
58 #if HAVE_SIMD(AVX)
sha2_have_avx(void)59 static boolean_t sha2_have_avx(void)
60 {
61 return (kfpu_allowed() && zfs_avx_available());
62 }
63
64 TF(zfs_sha512_transform_avx, tf_sha512_avx);
65 const sha512_ops_t sha512_avx_impl = {
66 .is_supported = sha2_have_avx,
67 .transform = tf_sha512_avx,
68 .name = "avx"
69 };
70 #endif
71
72 #if HAVE_SIMD(AVX2)
sha2_have_avx2(void)73 static boolean_t sha2_have_avx2(void)
74 {
75 return (kfpu_allowed() && zfs_avx2_available());
76 }
77
78 TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
79 const sha512_ops_t sha512_avx2_impl = {
80 .is_supported = sha2_have_avx2,
81 .transform = tf_sha512_avx2,
82 .name = "avx2"
83 };
84 #endif
85
86 #if HAVE_SIMD(SHA512EXT)
sha2_have_sha512ext(void)87 static boolean_t sha2_have_sha512ext(void)
88 {
89 return (kfpu_allowed() && zfs_sha512ext_available());
90 }
91
92 TF(zfs_sha512_transform_sha512ext, tf_sha512_sha512ext);
93 const sha512_ops_t sha512_sha512ext_impl = {
94 .is_supported = sha2_have_sha512ext,
95 .transform = tf_sha512_sha512ext,
96 .name = "sha512ext"
97 };
98 #endif
99
100 #elif defined(__aarch64__) || defined(__arm__)
101 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
102 const sha512_ops_t sha512_armv7_impl = {
103 .is_supported = sha2_is_supported,
104 .transform = zfs_sha512_block_armv7,
105 .name = "armv7"
106 };
107
108 #if defined(__aarch64__)
sha512_have_armv8ce(void)109 static boolean_t sha512_have_armv8ce(void)
110 {
111 return (kfpu_allowed() && zfs_sha512_available());
112 }
113
114 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
115 const sha512_ops_t sha512_armv8_impl = {
116 .is_supported = sha512_have_armv8ce,
117 .transform = tf_sha512_armv8ce,
118 .name = "armv8-ce"
119 };
120 #endif
121
122 #if defined(__arm__) && __ARM_ARCH > 6
sha512_have_neon(void)123 static boolean_t sha512_have_neon(void)
124 {
125 return (kfpu_allowed() && zfs_neon_available());
126 }
127
128 TF(zfs_sha512_block_neon, tf_sha512_neon);
129 const sha512_ops_t sha512_neon_impl = {
130 .is_supported = sha512_have_neon,
131 .transform = tf_sha512_neon,
132 .name = "neon"
133 };
134 #endif
135
136 #elif defined(__PPC64__)
137 TF(zfs_sha512_ppc, tf_sha512_ppc);
138 const sha512_ops_t sha512_ppc_impl = {
139 .is_supported = sha2_is_supported,
140 .transform = tf_sha512_ppc,
141 .name = "ppc"
142 };
143
sha512_have_isa207(void)144 static boolean_t sha512_have_isa207(void)
145 {
146 return (kfpu_allowed() && zfs_isa207_available());
147 }
148
149 TF(zfs_sha512_power8, tf_sha512_power8);
150 const sha512_ops_t sha512_power8_impl = {
151 .is_supported = sha512_have_isa207,
152 .transform = tf_sha512_power8,
153 .name = "power8"
154 };
155 #endif /* __PPC64__ */
156
157 /* the two generic ones */
158 extern const sha512_ops_t sha512_generic_impl;
159
160 /* array with all sha512 implementations */
161 static const sha512_ops_t *const sha512_impls[] = {
162 &sha512_generic_impl,
163 #if defined(__x86_64)
164 &sha512_x64_impl,
165 #endif
166 #if defined(__x86_64) && HAVE_SIMD(AVX)
167 &sha512_avx_impl,
168 #endif
169 #if defined(__x86_64) && HAVE_SIMD(AVX2)
170 &sha512_avx2_impl,
171 #endif
172 #if defined(__x86_64) && HAVE_SIMD(SHA512EXT)
173 &sha512_sha512ext_impl,
174 #endif
175 #if defined(__aarch64__) || defined(__arm__)
176 &sha512_armv7_impl,
177 #if defined(__aarch64__)
178 &sha512_armv8_impl,
179 #endif
180 #if defined(__arm__) && __ARM_ARCH > 6
181 &sha512_neon_impl,
182 #endif
183 #endif
184 #if defined(__PPC64__)
185 &sha512_ppc_impl,
186 &sha512_power8_impl,
187 #endif /* __PPC64__ */
188 };
189
190 /* use the generic implementation functions */
191 #define IMPL_NAME "sha512"
192 #define IMPL_OPS_T sha512_ops_t
193 #define IMPL_ARRAY sha512_impls
194 #define IMPL_GET_OPS sha512_get_ops
195 #define ZFS_IMPL_OPS zfs_sha512_ops
196 #include <generic_impl.c>
197
198 #ifdef _KERNEL
199
200 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
201
202 #if defined(__linux__)
203
204 static int
sha512_param_get(char * buffer,zfs_kernel_param_t * unused)205 sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
206 {
207 const uint32_t impl = IMPL_READ(generic_impl_chosen);
208 char *fmt;
209 int cnt = 0;
210
211 /* cycling */
212 fmt = IMPL_FMT(impl, IMPL_CYCLE);
213 cnt += sprintf(buffer + cnt, fmt, "cycle");
214
215 /* list fastest */
216 fmt = IMPL_FMT(impl, IMPL_FASTEST);
217 cnt += sprintf(buffer + cnt, fmt, "fastest");
218
219 /* list all supported implementations */
220 generic_impl_init();
221 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
222 fmt = IMPL_FMT(impl, i);
223 cnt += sprintf(buffer + cnt, fmt,
224 generic_supp_impls[i]->name);
225 }
226
227 return (cnt);
228 }
229
230 static int
sha512_param_set(const char * val,zfs_kernel_param_t * unused)231 sha512_param_set(const char *val, zfs_kernel_param_t *unused)
232 {
233 (void) unused;
234 return (generic_impl_setname(val));
235 }
236
237 #elif defined(__FreeBSD__)
238
239 #include <sys/sbuf.h>
240
241 static int
sha512_param(ZFS_MODULE_PARAM_ARGS)242 sha512_param(ZFS_MODULE_PARAM_ARGS)
243 {
244 int err;
245
246 generic_impl_init();
247 if (req->newptr == NULL) {
248 const uint32_t impl = IMPL_READ(generic_impl_chosen);
249 const int init_buflen = 64;
250 const char *fmt;
251 struct sbuf *s;
252
253 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
254
255 /* cycling */
256 fmt = IMPL_FMT(impl, IMPL_CYCLE);
257 (void) sbuf_printf(s, fmt, "cycle");
258
259 /* list fastest */
260 fmt = IMPL_FMT(impl, IMPL_FASTEST);
261 (void) sbuf_printf(s, fmt, "fastest");
262
263 /* list all supported implementations */
264 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
265 fmt = IMPL_FMT(impl, i);
266 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
267 }
268
269 err = sbuf_finish(s);
270 sbuf_delete(s);
271
272 return (err);
273 }
274
275 /* we got module parameter */
276 char buf[16];
277
278 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
279 if (err) {
280 return (err);
281 }
282
283 return (-generic_impl_setname(buf));
284 }
285 #endif
286
287 #undef IMPL_FMT
288
289 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
290 sha512_param_set, sha512_param_get, ZMOD_RW, \
291 "Select SHA512 implementation.");
292 #endif
293
294 #undef TF
295