1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 */
26
27 #include <sys/zio_checksum.h>
28 #include <sys/zfs_context.h>
29 #include <sys/zfs_chksum.h>
30 #include <sys/zfs_impl.h>
31
32 #include <sys/blake3.h>
33 #include <sys/sha2.h>
34
35 typedef struct {
36 const char *name;
37 const char *impl;
38 uint64_t bs1k;
39 uint64_t bs4k;
40 uint64_t bs16k;
41 uint64_t bs64k;
42 uint64_t bs256k;
43 uint64_t bs1m;
44 uint64_t bs4m;
45 uint64_t bs16m;
46 zio_cksum_salt_t salt;
47 zio_checksum_t *(func);
48 zio_checksum_tmpl_init_t *(init);
49 zio_checksum_tmpl_free_t *(free);
50 } chksum_stat_t;
51
52 #define AT_STARTUP 0
53 #define AT_BENCHMARK 1
54 #define AT_DONE 2
55
56 static chksum_stat_t *chksum_stat_data = 0;
57 static kstat_t *chksum_kstat = NULL;
58 static int chksum_stat_limit = AT_STARTUP;
59 static int chksum_stat_cnt = 0;
60 static void chksum_benchmark(void);
61
62 /*
63 * Sample output on i3-1005G1 System:
64 *
65 * implementation 1k 4k 16k 64k 256k 1m 4m 16m
66 * edonr-generic 1278 1625 1769 1776 1783 1778 1771 1767
67 * skein-generic 548 594 613 623 621 623 621 486
68 * sha256-generic 255 270 281 278 279 281 283 283
69 * sha256-x64 288 310 316 317 318 317 317 316
70 * sha256-ssse3 304 342 351 355 356 357 356 356
71 * sha256-avx 311 348 359 362 362 363 363 362
72 * sha256-avx2 330 378 389 395 395 395 395 395
73 * sha256-shani 908 1127 1212 1230 1233 1234 1223 1230
74 * sha512-generic 359 409 431 427 429 430 428 423
75 * sha512-x64 420 473 490 496 497 497 496 495
76 * sha512-avx 406 522 546 560 560 560 556 560
77 * sha512-avx2 464 568 601 606 609 610 607 608
78 * blake3-generic 330 327 324 323 324 320 323 322
79 * blake3-sse2 424 1366 1449 1468 1458 1453 1395 1408
80 * blake3-sse41 453 1554 1658 1703 1689 1669 1622 1630
81 * blake3-avx2 452 2013 3225 3351 3356 3261 3076 3101
82 * blake3-avx512 498 2869 5269 5926 5872 5643 5014 5005
83 */
84 static int
chksum_kstat_headers(char * buf,size_t size)85 chksum_kstat_headers(char *buf, size_t size)
86 {
87 ssize_t off = 0;
88
89 off += kmem_scnprintf(buf + off, size, "%-23s", "implementation");
90 off += kmem_scnprintf(buf + off, size - off, "%8s", "1k");
91 off += kmem_scnprintf(buf + off, size - off, "%8s", "4k");
92 off += kmem_scnprintf(buf + off, size - off, "%8s", "16k");
93 off += kmem_scnprintf(buf + off, size - off, "%8s", "64k");
94 off += kmem_scnprintf(buf + off, size - off, "%8s", "256k");
95 off += kmem_scnprintf(buf + off, size - off, "%8s", "1m");
96 off += kmem_scnprintf(buf + off, size - off, "%8s", "4m");
97 (void) kmem_scnprintf(buf + off, size - off, "%8s\n", "16m");
98
99 return (0);
100 }
101
102 static int
chksum_kstat_data(char * buf,size_t size,void * data)103 chksum_kstat_data(char *buf, size_t size, void *data)
104 {
105 chksum_stat_t *cs;
106 ssize_t off = 0;
107 char b[24];
108
109 cs = (chksum_stat_t *)data;
110 kmem_scnprintf(b, 23, "%s-%s", cs->name, cs->impl);
111 off += kmem_scnprintf(buf + off, size - off, "%-23s", b);
112 off += kmem_scnprintf(buf + off, size - off, "%8llu",
113 (u_longlong_t)cs->bs1k);
114 off += kmem_scnprintf(buf + off, size - off, "%8llu",
115 (u_longlong_t)cs->bs4k);
116 off += kmem_scnprintf(buf + off, size - off, "%8llu",
117 (u_longlong_t)cs->bs16k);
118 off += kmem_scnprintf(buf + off, size - off, "%8llu",
119 (u_longlong_t)cs->bs64k);
120 off += kmem_scnprintf(buf + off, size - off, "%8llu",
121 (u_longlong_t)cs->bs256k);
122 off += kmem_scnprintf(buf + off, size - off, "%8llu",
123 (u_longlong_t)cs->bs1m);
124 off += kmem_scnprintf(buf + off, size - off, "%8llu",
125 (u_longlong_t)cs->bs4m);
126 (void) kmem_scnprintf(buf + off, size - off, "%8llu\n",
127 (u_longlong_t)cs->bs16m);
128
129 return (0);
130 }
131
132 static void *
chksum_kstat_addr(kstat_t * ksp,loff_t n)133 chksum_kstat_addr(kstat_t *ksp, loff_t n)
134 {
135 /* full benchmark */
136 chksum_benchmark();
137
138 if (n < chksum_stat_cnt)
139 ksp->ks_private = (void *)(chksum_stat_data + n);
140 else
141 ksp->ks_private = NULL;
142
143 return (ksp->ks_private);
144 }
145
146 static void
chksum_run(chksum_stat_t * cs,abd_t * abd,void * ctx,int round,uint64_t * result)147 chksum_run(chksum_stat_t *cs, abd_t *abd, void *ctx, int round,
148 uint64_t *result)
149 {
150 hrtime_t start;
151 uint64_t run_bw, run_time_ns, run_count = 0, size = 0;
152 uint32_t l, loops = 0;
153 zio_cksum_t zcp;
154
155 switch (round) {
156 case 1: /* 1k */
157 size = 1<<10; loops = 128; break;
158 case 2: /* 4k */
159 size = 1<<12; loops = 64; break;
160 case 3: /* 16k */
161 size = 1<<14; loops = 32; break;
162 case 4: /* 64k */
163 size = 1<<16; loops = 16; break;
164 case 5: /* 256k */
165 size = 1<<18; loops = 8; break;
166 case 6: /* 1m */
167 size = 1<<20; loops = 4; break;
168 case 7: /* 4m */
169 size = 1<<22; loops = 1; break;
170 case 8: /* 16m */
171 size = 1<<24; loops = 1; break;
172 }
173
174 kpreempt_disable();
175 start = gethrtime();
176 do {
177 for (l = 0; l < loops; l++, run_count++)
178 cs->func(abd, size, ctx, &zcp);
179
180 run_time_ns = gethrtime() - start;
181 } while (run_time_ns < MSEC2NSEC(1));
182 kpreempt_enable();
183
184 run_bw = size * run_count * NANOSEC;
185 run_bw /= run_time_ns; /* B/s */
186 *result = run_bw/1024/1024; /* MiB/s */
187 }
188
189 static void
chksum_benchit(chksum_stat_t * cs)190 chksum_benchit(chksum_stat_t *cs)
191 {
192 abd_t *abd;
193 void *ctx = 0;
194 void *salt = &cs->salt.zcs_bytes;
195
196 memset(salt, 0, sizeof (cs->salt.zcs_bytes));
197 if (cs->init)
198 ctx = cs->init(&cs->salt);
199
200 /* benchmarks in startup mode */
201 if (chksum_stat_limit == AT_STARTUP) {
202 abd = abd_alloc_linear(1<<18, B_FALSE);
203 chksum_run(cs, abd, ctx, 5, &cs->bs256k);
204 goto done;
205 }
206
207 /* allocate test memory via abd linear interface */
208 abd = abd_alloc_linear(1<<20, B_FALSE);
209
210 /* benchmarks when requested */
211 chksum_run(cs, abd, ctx, 1, &cs->bs1k);
212 chksum_run(cs, abd, ctx, 2, &cs->bs4k);
213 chksum_run(cs, abd, ctx, 3, &cs->bs16k);
214 chksum_run(cs, abd, ctx, 4, &cs->bs64k);
215 chksum_run(cs, abd, ctx, 5, &cs->bs256k);
216 chksum_run(cs, abd, ctx, 6, &cs->bs1m);
217 abd_free(abd);
218
219 /* allocate test memory via abd non linear interface */
220 abd = abd_alloc(1<<24, B_FALSE);
221 chksum_run(cs, abd, ctx, 7, &cs->bs4m);
222 chksum_run(cs, abd, ctx, 8, &cs->bs16m);
223
224 done:
225 abd_free(abd);
226
227 /* free up temp memory */
228 if (cs->free)
229 cs->free(ctx);
230 }
231
232 /*
233 * Initialize and benchmark all supported implementations.
234 */
235 static void
chksum_benchmark(void)236 chksum_benchmark(void)
237 {
238 #ifndef _KERNEL
239 /* we need the benchmark only for the kernel module */
240 return;
241 #endif
242 chksum_stat_t *cs;
243 uint64_t max;
244 uint32_t id, cbid = 0, id_save;
245 const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3");
246 const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256");
247 const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512");
248
249 /* benchmarks are done */
250 if (chksum_stat_limit == AT_DONE)
251 return;
252
253 /* count implementations */
254 if (chksum_stat_limit == AT_STARTUP) {
255 chksum_stat_cnt = 1; /* edonr */
256 chksum_stat_cnt += 1; /* skein */
257 chksum_stat_cnt += sha256->getcnt();
258 chksum_stat_cnt += sha512->getcnt();
259 chksum_stat_cnt += blake3->getcnt();
260 chksum_stat_data = kmem_zalloc(
261 sizeof (chksum_stat_t) * chksum_stat_cnt, KM_SLEEP);
262 }
263
264 /* edonr - needs to be the first one here (slow CPU check) */
265 cs = &chksum_stat_data[cbid++];
266
267 /* edonr */
268 cs->init = abd_checksum_edonr_tmpl_init;
269 cs->func = abd_checksum_edonr_native;
270 cs->free = abd_checksum_edonr_tmpl_free;
271 cs->name = "edonr";
272 cs->impl = "generic";
273 chksum_benchit(cs);
274
275 /* skein */
276 cs = &chksum_stat_data[cbid++];
277 cs->init = abd_checksum_skein_tmpl_init;
278 cs->func = abd_checksum_skein_native;
279 cs->free = abd_checksum_skein_tmpl_free;
280 cs->name = "skein";
281 cs->impl = "generic";
282 chksum_benchit(cs);
283
284 /* sha256 */
285 id_save = sha256->getid();
286 for (max = 0, id = 0; id < sha256->getcnt(); id++) {
287 sha256->setid(id);
288 cs = &chksum_stat_data[cbid++];
289 cs->init = 0;
290 cs->func = abd_checksum_sha256;
291 cs->free = 0;
292 cs->name = sha256->name;
293 cs->impl = sha256->getname();
294 chksum_benchit(cs);
295 if (cs->bs256k > max) {
296 max = cs->bs256k;
297 sha256->set_fastest(id);
298 }
299 }
300 sha256->setid(id_save);
301
302 /* sha512 */
303 id_save = sha512->getid();
304 for (max = 0, id = 0; id < sha512->getcnt(); id++) {
305 sha512->setid(id);
306 cs = &chksum_stat_data[cbid++];
307 cs->init = 0;
308 cs->func = abd_checksum_sha512_native;
309 cs->free = 0;
310 cs->name = sha512->name;
311 cs->impl = sha512->getname();
312 chksum_benchit(cs);
313 if (cs->bs256k > max) {
314 max = cs->bs256k;
315 sha512->set_fastest(id);
316 }
317 }
318 sha512->setid(id_save);
319
320 /* blake3 */
321 id_save = blake3->getid();
322 for (max = 0, id = 0; id < blake3->getcnt(); id++) {
323 blake3->setid(id);
324 cs = &chksum_stat_data[cbid++];
325 cs->init = abd_checksum_blake3_tmpl_init;
326 cs->func = abd_checksum_blake3_native;
327 cs->free = abd_checksum_blake3_tmpl_free;
328 cs->name = blake3->name;
329 cs->impl = blake3->getname();
330 chksum_benchit(cs);
331 if (cs->bs256k > max) {
332 max = cs->bs256k;
333 blake3->set_fastest(id);
334 }
335 }
336 blake3->setid(id_save);
337
338 switch (chksum_stat_limit) {
339 case AT_STARTUP:
340 /* next time we want a full benchmark */
341 chksum_stat_limit = AT_BENCHMARK;
342 break;
343 case AT_BENCHMARK:
344 /* no further benchmarks */
345 chksum_stat_limit = AT_DONE;
346 break;
347 }
348 }
349
350 void
chksum_init(void)351 chksum_init(void)
352 {
353 #ifdef _KERNEL
354 blake3_per_cpu_ctx_init();
355 #endif
356
357 /* 256KiB benchmark */
358 chksum_benchmark();
359
360 /* Install kstats for all implementations */
361 chksum_kstat = kstat_create("zfs", 0, "chksum_bench", "misc",
362 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
363
364 if (chksum_kstat != NULL) {
365 chksum_kstat->ks_data = NULL;
366 chksum_kstat->ks_ndata = UINT32_MAX;
367 kstat_set_raw_ops(chksum_kstat,
368 chksum_kstat_headers,
369 chksum_kstat_data,
370 chksum_kstat_addr);
371 kstat_install(chksum_kstat);
372 }
373 }
374
375 void
chksum_fini(void)376 chksum_fini(void)
377 {
378 if (chksum_kstat != NULL) {
379 kstat_delete(chksum_kstat);
380 chksum_kstat = NULL;
381 }
382
383 if (chksum_stat_cnt) {
384 kmem_free(chksum_stat_data,
385 sizeof (chksum_stat_t) * chksum_stat_cnt);
386 chksum_stat_cnt = 0;
387 chksum_stat_data = 0;
388 }
389
390 #ifdef _KERNEL
391 blake3_per_cpu_ctx_fini();
392 #endif
393 }
394