1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
15 */
16
17 #include <sys/simd.h>
18 #include <sys/zfs_context.h>
19 #include <sys/zfs_impl.h>
20 #include <sys/blake3.h>
21
22 #include "blake3_impl.h"
23
24 #if !defined(OMIT_SIMD) && (defined(__aarch64__) || \
25 (defined(__x86_64) && HAVE_SIMD(SSE2)) || \
26 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)))
27 #define USE_SIMD
28 #endif
29
30 #ifdef USE_SIMD
31 extern void ASMABI zfs_blake3_compress_in_place_sse2(uint32_t cv[8],
32 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
33 uint64_t counter, uint8_t flags);
34
35 extern void ASMABI zfs_blake3_compress_xof_sse2(const uint32_t cv[8],
36 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
37 uint64_t counter, uint8_t flags, uint8_t out[64]);
38
39 extern void ASMABI zfs_blake3_hash_many_sse2(const uint8_t * const *inputs,
40 size_t num_inputs, size_t blocks, const uint32_t key[8],
41 uint64_t counter, boolean_t increment_counter, uint8_t flags,
42 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
43
blake3_compress_in_place_sse2(uint32_t cv[8],const uint8_t block[BLAKE3_BLOCK_LEN],uint8_t block_len,uint64_t counter,uint8_t flags)44 static void blake3_compress_in_place_sse2(uint32_t cv[8],
45 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
46 uint64_t counter, uint8_t flags) {
47 kfpu_begin();
48 zfs_blake3_compress_in_place_sse2(cv, block, block_len, counter,
49 flags);
50 kfpu_end();
51 }
52
blake3_compress_xof_sse2(const uint32_t cv[8],const uint8_t block[BLAKE3_BLOCK_LEN],uint8_t block_len,uint64_t counter,uint8_t flags,uint8_t out[64])53 static void blake3_compress_xof_sse2(const uint32_t cv[8],
54 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
55 uint64_t counter, uint8_t flags, uint8_t out[64]) {
56 kfpu_begin();
57 zfs_blake3_compress_xof_sse2(cv, block, block_len, counter, flags,
58 out);
59 kfpu_end();
60 }
61
blake3_hash_many_sse2(const uint8_t * const * inputs,size_t num_inputs,size_t blocks,const uint32_t key[8],uint64_t counter,boolean_t increment_counter,uint8_t flags,uint8_t flags_start,uint8_t flags_end,uint8_t * out)62 static void blake3_hash_many_sse2(const uint8_t * const *inputs,
63 size_t num_inputs, size_t blocks, const uint32_t key[8],
64 uint64_t counter, boolean_t increment_counter, uint8_t flags,
65 uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
66 kfpu_begin();
67 zfs_blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter,
68 increment_counter, flags, flags_start, flags_end, out);
69 kfpu_end();
70 }
71
blake3_is_sse2_supported(void)72 static boolean_t blake3_is_sse2_supported(void)
73 {
74 #if defined(__x86_64)
75 return (kfpu_allowed() && zfs_sse2_available());
76 #elif defined(__PPC64__)
77 return (kfpu_allowed() && zfs_vsx_available());
78 #else
79 return (kfpu_allowed());
80 #endif
81 }
82
83 const blake3_ops_t blake3_sse2_impl = {
84 .compress_in_place = blake3_compress_in_place_sse2,
85 .compress_xof = blake3_compress_xof_sse2,
86 .hash_many = blake3_hash_many_sse2,
87 .is_supported = blake3_is_sse2_supported,
88 .degree = 4,
89 .name = "sse2"
90 };
91 #endif
92
93 #ifdef USE_SIMD
94
95 extern void ASMABI zfs_blake3_compress_in_place_sse41(uint32_t cv[8],
96 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
97 uint64_t counter, uint8_t flags);
98
99 extern void ASMABI zfs_blake3_compress_xof_sse41(const uint32_t cv[8],
100 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
101 uint64_t counter, uint8_t flags, uint8_t out[64]);
102
103 extern void ASMABI zfs_blake3_hash_many_sse41(const uint8_t * const *inputs,
104 size_t num_inputs, size_t blocks, const uint32_t key[8],
105 uint64_t counter, boolean_t increment_counter, uint8_t flags,
106 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
107
blake3_compress_in_place_sse41(uint32_t cv[8],const uint8_t block[BLAKE3_BLOCK_LEN],uint8_t block_len,uint64_t counter,uint8_t flags)108 static void blake3_compress_in_place_sse41(uint32_t cv[8],
109 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
110 uint64_t counter, uint8_t flags) {
111 kfpu_begin();
112 zfs_blake3_compress_in_place_sse41(cv, block, block_len, counter,
113 flags);
114 kfpu_end();
115 }
116
blake3_compress_xof_sse41(const uint32_t cv[8],const uint8_t block[BLAKE3_BLOCK_LEN],uint8_t block_len,uint64_t counter,uint8_t flags,uint8_t out[64])117 static void blake3_compress_xof_sse41(const uint32_t cv[8],
118 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
119 uint64_t counter, uint8_t flags, uint8_t out[64]) {
120 kfpu_begin();
121 zfs_blake3_compress_xof_sse41(cv, block, block_len, counter, flags,
122 out);
123 kfpu_end();
124 }
125
blake3_hash_many_sse41(const uint8_t * const * inputs,size_t num_inputs,size_t blocks,const uint32_t key[8],uint64_t counter,boolean_t increment_counter,uint8_t flags,uint8_t flags_start,uint8_t flags_end,uint8_t * out)126 static void blake3_hash_many_sse41(const uint8_t * const *inputs,
127 size_t num_inputs, size_t blocks, const uint32_t key[8],
128 uint64_t counter, boolean_t increment_counter, uint8_t flags,
129 uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
130 kfpu_begin();
131 zfs_blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter,
132 increment_counter, flags, flags_start, flags_end, out);
133 kfpu_end();
134 }
135
blake3_is_sse41_supported(void)136 static boolean_t blake3_is_sse41_supported(void)
137 {
138 #if defined(__x86_64)
139 return (kfpu_allowed() && zfs_sse4_1_available());
140 #elif defined(__PPC64__)
141 return (kfpu_allowed() && zfs_vsx_available());
142 #else
143 return (kfpu_allowed());
144 #endif
145 }
146
147 const blake3_ops_t blake3_sse41_impl = {
148 .compress_in_place = blake3_compress_in_place_sse41,
149 .compress_xof = blake3_compress_xof_sse41,
150 .hash_many = blake3_hash_many_sse41,
151 .is_supported = blake3_is_sse41_supported,
152 .degree = 4,
153 .name = "sse41"
154 };
155 #endif
156
157 #if defined(__x86_64) && HAVE_SIMD(SSE4_1) && HAVE_SIMD(AVX2)
158 extern void ASMABI zfs_blake3_hash_many_avx2(const uint8_t * const *inputs,
159 size_t num_inputs, size_t blocks, const uint32_t key[8],
160 uint64_t counter, boolean_t increment_counter, uint8_t flags,
161 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
162
blake3_hash_many_avx2(const uint8_t * const * inputs,size_t num_inputs,size_t blocks,const uint32_t key[8],uint64_t counter,boolean_t increment_counter,uint8_t flags,uint8_t flags_start,uint8_t flags_end,uint8_t * out)163 static void blake3_hash_many_avx2(const uint8_t * const *inputs,
164 size_t num_inputs, size_t blocks, const uint32_t key[8],
165 uint64_t counter, boolean_t increment_counter, uint8_t flags,
166 uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
167 kfpu_begin();
168 zfs_blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter,
169 increment_counter, flags, flags_start, flags_end, out);
170 kfpu_end();
171 }
172
blake3_is_avx2_supported(void)173 static boolean_t blake3_is_avx2_supported(void)
174 {
175 return (kfpu_allowed() && zfs_sse4_1_available() &&
176 zfs_avx2_available());
177 }
178
179 const blake3_ops_t
180 blake3_avx2_impl = {
181 .compress_in_place = blake3_compress_in_place_sse41,
182 .compress_xof = blake3_compress_xof_sse41,
183 .hash_many = blake3_hash_many_avx2,
184 .is_supported = blake3_is_avx2_supported,
185 .degree = 8,
186 .name = "avx2"
187 };
188 #endif
189
190 #if defined(__x86_64) && HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL)
191 extern void ASMABI zfs_blake3_compress_in_place_avx512(uint32_t cv[8],
192 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
193 uint64_t counter, uint8_t flags);
194
195 extern void ASMABI zfs_blake3_compress_xof_avx512(const uint32_t cv[8],
196 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
197 uint64_t counter, uint8_t flags, uint8_t out[64]);
198
199 extern void ASMABI zfs_blake3_hash_many_avx512(const uint8_t * const *inputs,
200 size_t num_inputs, size_t blocks, const uint32_t key[8],
201 uint64_t counter, boolean_t increment_counter, uint8_t flags,
202 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
203
blake3_compress_in_place_avx512(uint32_t cv[8],const uint8_t block[BLAKE3_BLOCK_LEN],uint8_t block_len,uint64_t counter,uint8_t flags)204 static void blake3_compress_in_place_avx512(uint32_t cv[8],
205 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
206 uint64_t counter, uint8_t flags) {
207 kfpu_begin();
208 zfs_blake3_compress_in_place_avx512(cv, block, block_len, counter,
209 flags);
210 kfpu_end();
211 }
212
blake3_compress_xof_avx512(const uint32_t cv[8],const uint8_t block[BLAKE3_BLOCK_LEN],uint8_t block_len,uint64_t counter,uint8_t flags,uint8_t out[64])213 static void blake3_compress_xof_avx512(const uint32_t cv[8],
214 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
215 uint64_t counter, uint8_t flags, uint8_t out[64]) {
216 kfpu_begin();
217 zfs_blake3_compress_xof_avx512(cv, block, block_len, counter, flags,
218 out);
219 kfpu_end();
220 }
221
blake3_hash_many_avx512(const uint8_t * const * inputs,size_t num_inputs,size_t blocks,const uint32_t key[8],uint64_t counter,boolean_t increment_counter,uint8_t flags,uint8_t flags_start,uint8_t flags_end,uint8_t * out)222 static void blake3_hash_many_avx512(const uint8_t * const *inputs,
223 size_t num_inputs, size_t blocks, const uint32_t key[8],
224 uint64_t counter, boolean_t increment_counter, uint8_t flags,
225 uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
226 kfpu_begin();
227 zfs_blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter,
228 increment_counter, flags, flags_start, flags_end, out);
229 kfpu_end();
230 }
231
blake3_is_avx512_supported(void)232 static boolean_t blake3_is_avx512_supported(void)
233 {
234 return (kfpu_allowed() && zfs_avx512f_available() &&
235 zfs_avx512vl_available());
236 }
237
238 const blake3_ops_t blake3_avx512_impl = {
239 .compress_in_place = blake3_compress_in_place_avx512,
240 .compress_xof = blake3_compress_xof_avx512,
241 .hash_many = blake3_hash_many_avx512,
242 .is_supported = blake3_is_avx512_supported,
243 .degree = 16,
244 .name = "avx512"
245 };
246 #endif
247
248 extern const blake3_ops_t blake3_generic_impl;
249
250 static const blake3_ops_t *const blake3_impls[] = {
251 &blake3_generic_impl,
252 #ifdef USE_SIMD
253 #if defined(__aarch64__) || \
254 (defined(__x86_64) && HAVE_SIMD(SSE2)) || \
255 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
256 &blake3_sse2_impl,
257 #endif
258 #if defined(__aarch64__) || \
259 (defined(__x86_64) && HAVE_SIMD(SSE4_1)) || \
260 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
261 &blake3_sse41_impl,
262 #endif
263 #if defined(__x86_64) && HAVE_SIMD(SSE4_1) && HAVE_SIMD(AVX2)
264 &blake3_avx2_impl,
265 #endif
266 #if defined(__x86_64) && HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL)
267 &blake3_avx512_impl,
268 #endif
269 #endif
270 };
271
272 /* use the generic implementation functions */
273 #define IMPL_NAME "blake3"
274 #define IMPL_OPS_T blake3_ops_t
275 #define IMPL_ARRAY blake3_impls
276 #define IMPL_GET_OPS blake3_get_ops
277 #define ZFS_IMPL_OPS zfs_blake3_ops
278 #include <generic_impl.c>
279
280 #ifdef _KERNEL
281 void **blake3_per_cpu_ctx;
282
283 void
blake3_per_cpu_ctx_init(void)284 blake3_per_cpu_ctx_init(void)
285 {
286 /*
287 * Create "The Godfather" ptr to hold all blake3 ctx
288 */
289 blake3_per_cpu_ctx = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP);
290 for (int i = 0; i < max_ncpus; i++) {
291 blake3_per_cpu_ctx[i] = kmem_alloc(sizeof (BLAKE3_CTX),
292 KM_SLEEP);
293 }
294 }
295
296 void
blake3_per_cpu_ctx_fini(void)297 blake3_per_cpu_ctx_fini(void)
298 {
299 for (int i = 0; i < max_ncpus; i++) {
300 memset(blake3_per_cpu_ctx[i], 0, sizeof (BLAKE3_CTX));
301 kmem_free(blake3_per_cpu_ctx[i], sizeof (BLAKE3_CTX));
302 }
303 memset(blake3_per_cpu_ctx, 0, max_ncpus * sizeof (void *));
304 kmem_free(blake3_per_cpu_ctx, max_ncpus * sizeof (void *));
305 }
306
307 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
308
309 #if defined(__linux__)
310
311 static int
blake3_param_get(char * buffer,zfs_kernel_param_t * unused)312 blake3_param_get(char *buffer, zfs_kernel_param_t *unused)
313 {
314 const uint32_t impl = IMPL_READ(generic_impl_chosen);
315 char *fmt;
316 int cnt = 0;
317
318 /* cycling */
319 fmt = IMPL_FMT(impl, IMPL_CYCLE);
320 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt, "cycle");
321
322 /* list fastest */
323 fmt = IMPL_FMT(impl, IMPL_FASTEST);
324 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt, "fastest");
325
326 /* list all supported implementations */
327 generic_impl_init();
328 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
329 fmt = IMPL_FMT(impl, i);
330 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
331 blake3_impls[i]->name);
332 }
333
334 return (cnt);
335 }
336
337 static int
blake3_param_set(const char * val,zfs_kernel_param_t * unused)338 blake3_param_set(const char *val, zfs_kernel_param_t *unused)
339 {
340 (void) unused;
341 return (generic_impl_setname(val));
342 }
343
344 #elif defined(__FreeBSD__)
345
346 #include <sys/sbuf.h>
347
348 static int
blake3_param(ZFS_MODULE_PARAM_ARGS)349 blake3_param(ZFS_MODULE_PARAM_ARGS)
350 {
351 int err;
352
353 generic_impl_init();
354 if (req->newptr == NULL) {
355 const uint32_t impl = IMPL_READ(generic_impl_chosen);
356 const int init_buflen = 64;
357 const char *fmt;
358 struct sbuf *s;
359
360 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
361
362 /* cycling */
363 fmt = IMPL_FMT(impl, IMPL_CYCLE);
364 (void) sbuf_printf(s, fmt, "cycle");
365
366 /* list fastest */
367 fmt = IMPL_FMT(impl, IMPL_FASTEST);
368 (void) sbuf_printf(s, fmt, "fastest");
369
370 /* list all supported implementations */
371 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
372 fmt = IMPL_FMT(impl, i);
373 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
374 }
375
376 err = sbuf_finish(s);
377 sbuf_delete(s);
378
379 return (err);
380 }
381
382 char buf[16];
383
384 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
385 if (err) {
386 return (err);
387 }
388
389 return (-generic_impl_setname(buf));
390 }
391 #endif
392
393 #undef IMPL_FMT
394
395 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, blake3_impl,
396 blake3_param_set, blake3_param_get, ZMOD_RW, \
397 "Select BLAKE3 implementation.");
398 #endif
399