1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 */
26
27 #include <sys/zio_checksum.h>
28 #include <sys/zfs_context.h>
29 #include <sys/zfs_chksum.h>
30 #include <sys/zfs_impl.h>
31
32 #include <sys/blake3.h>
33 #include <sys/sha2.h>
34
35 /* limit benchmarking to max 256KiB, when EdonR is slower then this: */
36 #define LIMIT_PERF_MBS 300
37
38 typedef struct {
39 const char *name;
40 const char *impl;
41 uint64_t bs1k;
42 uint64_t bs4k;
43 uint64_t bs16k;
44 uint64_t bs64k;
45 uint64_t bs256k;
46 uint64_t bs1m;
47 uint64_t bs4m;
48 uint64_t bs16m;
49 zio_cksum_salt_t salt;
50 zio_checksum_t *(func);
51 zio_checksum_tmpl_init_t *(init);
52 zio_checksum_tmpl_free_t *(free);
53 } chksum_stat_t;
54
55 static chksum_stat_t *chksum_stat_data = 0;
56 static int chksum_stat_cnt = 0;
57 static kstat_t *chksum_kstat = NULL;
58
59 /*
60 * Sample output on i3-1005G1 System:
61 *
62 * implementation 1k 4k 16k 64k 256k 1m 4m 16m
63 * edonr-generic 1278 1625 1769 1776 1783 1778 1771 1767
64 * skein-generic 548 594 613 623 621 623 621 486
65 * sha256-generic 255 270 281 278 279 281 283 283
66 * sha256-x64 288 310 316 317 318 317 317 316
67 * sha256-ssse3 304 342 351 355 356 357 356 356
68 * sha256-avx 311 348 359 362 362 363 363 362
69 * sha256-avx2 330 378 389 395 395 395 395 395
70 * sha256-shani 908 1127 1212 1230 1233 1234 1223 1230
71 * sha512-generic 359 409 431 427 429 430 428 423
72 * sha512-x64 420 473 490 496 497 497 496 495
73 * sha512-avx 406 522 546 560 560 560 556 560
74 * sha512-avx2 464 568 601 606 609 610 607 608
75 * blake3-generic 330 327 324 323 324 320 323 322
76 * blake3-sse2 424 1366 1449 1468 1458 1453 1395 1408
77 * blake3-sse41 453 1554 1658 1703 1689 1669 1622 1630
78 * blake3-avx2 452 2013 3225 3351 3356 3261 3076 3101
79 * blake3-avx512 498 2869 5269 5926 5872 5643 5014 5005
80 */
81 static int
chksum_kstat_headers(char * buf,size_t size)82 chksum_kstat_headers(char *buf, size_t size)
83 {
84 ssize_t off = 0;
85
86 off += kmem_scnprintf(buf + off, size, "%-23s", "implementation");
87 off += kmem_scnprintf(buf + off, size - off, "%8s", "1k");
88 off += kmem_scnprintf(buf + off, size - off, "%8s", "4k");
89 off += kmem_scnprintf(buf + off, size - off, "%8s", "16k");
90 off += kmem_scnprintf(buf + off, size - off, "%8s", "64k");
91 off += kmem_scnprintf(buf + off, size - off, "%8s", "256k");
92 off += kmem_scnprintf(buf + off, size - off, "%8s", "1m");
93 off += kmem_scnprintf(buf + off, size - off, "%8s", "4m");
94 (void) kmem_scnprintf(buf + off, size - off, "%8s\n", "16m");
95
96 return (0);
97 }
98
99 static int
chksum_kstat_data(char * buf,size_t size,void * data)100 chksum_kstat_data(char *buf, size_t size, void *data)
101 {
102 chksum_stat_t *cs;
103 ssize_t off = 0;
104 char b[24];
105
106 cs = (chksum_stat_t *)data;
107 kmem_scnprintf(b, 23, "%s-%s", cs->name, cs->impl);
108 off += kmem_scnprintf(buf + off, size - off, "%-23s", b);
109 off += kmem_scnprintf(buf + off, size - off, "%8llu",
110 (u_longlong_t)cs->bs1k);
111 off += kmem_scnprintf(buf + off, size - off, "%8llu",
112 (u_longlong_t)cs->bs4k);
113 off += kmem_scnprintf(buf + off, size - off, "%8llu",
114 (u_longlong_t)cs->bs16k);
115 off += kmem_scnprintf(buf + off, size - off, "%8llu",
116 (u_longlong_t)cs->bs64k);
117 off += kmem_scnprintf(buf + off, size - off, "%8llu",
118 (u_longlong_t)cs->bs256k);
119 off += kmem_scnprintf(buf + off, size - off, "%8llu",
120 (u_longlong_t)cs->bs1m);
121 off += kmem_scnprintf(buf + off, size - off, "%8llu",
122 (u_longlong_t)cs->bs4m);
123 (void) kmem_scnprintf(buf + off, size - off, "%8llu\n",
124 (u_longlong_t)cs->bs16m);
125
126 return (0);
127 }
128
129 static void *
chksum_kstat_addr(kstat_t * ksp,loff_t n)130 chksum_kstat_addr(kstat_t *ksp, loff_t n)
131 {
132 if (n < chksum_stat_cnt)
133 ksp->ks_private = (void *)(chksum_stat_data + n);
134 else
135 ksp->ks_private = NULL;
136
137 return (ksp->ks_private);
138 }
139
140 static void
chksum_run(chksum_stat_t * cs,abd_t * abd,void * ctx,int round,uint64_t * result)141 chksum_run(chksum_stat_t *cs, abd_t *abd, void *ctx, int round,
142 uint64_t *result)
143 {
144 hrtime_t start;
145 uint64_t run_bw, run_time_ns, run_count = 0, size = 0;
146 uint32_t l, loops = 0;
147 zio_cksum_t zcp;
148
149 switch (round) {
150 case 1: /* 1k */
151 size = 1<<10; loops = 128; break;
152 case 2: /* 2k */
153 size = 1<<12; loops = 64; break;
154 case 3: /* 4k */
155 size = 1<<14; loops = 32; break;
156 case 4: /* 16k */
157 size = 1<<16; loops = 16; break;
158 case 5: /* 256k */
159 size = 1<<18; loops = 8; break;
160 case 6: /* 1m */
161 size = 1<<20; loops = 4; break;
162 case 7: /* 4m */
163 size = 1<<22; loops = 1; break;
164 case 8: /* 16m */
165 size = 1<<24; loops = 1; break;
166 }
167
168 kpreempt_disable();
169 start = gethrtime();
170 do {
171 for (l = 0; l < loops; l++, run_count++)
172 cs->func(abd, size, ctx, &zcp);
173
174 run_time_ns = gethrtime() - start;
175 } while (run_time_ns < MSEC2NSEC(1));
176 kpreempt_enable();
177
178 run_bw = size * run_count * NANOSEC;
179 run_bw /= run_time_ns; /* B/s */
180 *result = run_bw/1024/1024; /* MiB/s */
181 }
182
183 #define LIMIT_INIT 0
184 #define LIMIT_NEEDED 1
185 #define LIMIT_NOLIMIT 2
186
187 static void
chksum_benchit(chksum_stat_t * cs)188 chksum_benchit(chksum_stat_t *cs)
189 {
190 abd_t *abd;
191 void *ctx = 0;
192 void *salt = &cs->salt.zcs_bytes;
193 static int chksum_stat_limit = LIMIT_INIT;
194
195 memset(salt, 0, sizeof (cs->salt.zcs_bytes));
196 if (cs->init)
197 ctx = cs->init(&cs->salt);
198
199 /* allocate test memory via abd linear interface */
200 abd = abd_alloc_linear(1<<20, B_FALSE);
201 chksum_run(cs, abd, ctx, 1, &cs->bs1k);
202 chksum_run(cs, abd, ctx, 2, &cs->bs4k);
203 chksum_run(cs, abd, ctx, 3, &cs->bs16k);
204 chksum_run(cs, abd, ctx, 4, &cs->bs64k);
205 chksum_run(cs, abd, ctx, 5, &cs->bs256k);
206
207 /* check if we ran on a slow cpu */
208 if (chksum_stat_limit == LIMIT_INIT) {
209 if (cs->bs1k < LIMIT_PERF_MBS) {
210 chksum_stat_limit = LIMIT_NEEDED;
211 } else {
212 chksum_stat_limit = LIMIT_NOLIMIT;
213 }
214 }
215
216 /* skip benchmarks >= 1MiB when the CPU is to slow */
217 if (chksum_stat_limit == LIMIT_NEEDED)
218 goto abort;
219
220 chksum_run(cs, abd, ctx, 6, &cs->bs1m);
221 abd_free(abd);
222
223 /* allocate test memory via abd non linear interface */
224 abd = abd_alloc(1<<24, B_FALSE);
225 chksum_run(cs, abd, ctx, 7, &cs->bs4m);
226 chksum_run(cs, abd, ctx, 8, &cs->bs16m);
227
228 abort:
229 abd_free(abd);
230
231 /* free up temp memory */
232 if (cs->free)
233 cs->free(ctx);
234 }
235
236 /*
237 * Initialize and benchmark all supported implementations.
238 */
239 static void
chksum_benchmark(void)240 chksum_benchmark(void)
241 {
242 #ifndef _KERNEL
243 /* we need the benchmark only for the kernel module */
244 return;
245 #endif
246
247 chksum_stat_t *cs;
248 uint64_t max;
249 uint32_t id, cbid = 0, id_save;
250 const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3");
251 const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256");
252 const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512");
253
254 /* count implementations */
255 chksum_stat_cnt = 2;
256 chksum_stat_cnt += sha256->getcnt();
257 chksum_stat_cnt += sha512->getcnt();
258 chksum_stat_cnt += blake3->getcnt();
259 chksum_stat_data = kmem_zalloc(
260 sizeof (chksum_stat_t) * chksum_stat_cnt, KM_SLEEP);
261
262 /* edonr - needs to be the first one here (slow CPU check) */
263 cs = &chksum_stat_data[cbid++];
264
265 /* edonr */
266 cs->init = abd_checksum_edonr_tmpl_init;
267 cs->func = abd_checksum_edonr_native;
268 cs->free = abd_checksum_edonr_tmpl_free;
269 cs->name = "edonr";
270 cs->impl = "generic";
271 chksum_benchit(cs);
272
273 /* skein */
274 cs = &chksum_stat_data[cbid++];
275 cs->init = abd_checksum_skein_tmpl_init;
276 cs->func = abd_checksum_skein_native;
277 cs->free = abd_checksum_skein_tmpl_free;
278 cs->name = "skein";
279 cs->impl = "generic";
280 chksum_benchit(cs);
281
282 /* sha256 */
283 id_save = sha256->getid();
284 for (max = 0, id = 0; id < sha256->getcnt(); id++) {
285 sha256->setid(id);
286 cs = &chksum_stat_data[cbid++];
287 cs->init = 0;
288 cs->func = abd_checksum_sha256;
289 cs->free = 0;
290 cs->name = sha256->name;
291 cs->impl = sha256->getname();
292 chksum_benchit(cs);
293 if (cs->bs256k > max) {
294 max = cs->bs256k;
295 sha256->set_fastest(id);
296 }
297 }
298 sha256->setid(id_save);
299
300 /* sha512 */
301 id_save = sha512->getid();
302 for (max = 0, id = 0; id < sha512->getcnt(); id++) {
303 sha512->setid(id);
304 cs = &chksum_stat_data[cbid++];
305 cs->init = 0;
306 cs->func = abd_checksum_sha512_native;
307 cs->free = 0;
308 cs->name = sha512->name;
309 cs->impl = sha512->getname();
310 chksum_benchit(cs);
311 if (cs->bs256k > max) {
312 max = cs->bs256k;
313 sha512->set_fastest(id);
314 }
315 }
316 sha512->setid(id_save);
317
318 /* blake3 */
319 id_save = blake3->getid();
320 for (max = 0, id = 0; id < blake3->getcnt(); id++) {
321 blake3->setid(id);
322 cs = &chksum_stat_data[cbid++];
323 cs->init = abd_checksum_blake3_tmpl_init;
324 cs->func = abd_checksum_blake3_native;
325 cs->free = abd_checksum_blake3_tmpl_free;
326 cs->name = blake3->name;
327 cs->impl = blake3->getname();
328 chksum_benchit(cs);
329 if (cs->bs256k > max) {
330 max = cs->bs256k;
331 blake3->set_fastest(id);
332 }
333 }
334 blake3->setid(id_save);
335 }
336
337 void
chksum_init(void)338 chksum_init(void)
339 {
340 #ifdef _KERNEL
341 blake3_per_cpu_ctx_init();
342 #endif
343
344 /* Benchmark supported implementations */
345 chksum_benchmark();
346
347 /* Install kstats for all implementations */
348 chksum_kstat = kstat_create("zfs", 0, "chksum_bench", "misc",
349 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
350
351 if (chksum_kstat != NULL) {
352 chksum_kstat->ks_data = NULL;
353 chksum_kstat->ks_ndata = UINT32_MAX;
354 kstat_set_raw_ops(chksum_kstat,
355 chksum_kstat_headers,
356 chksum_kstat_data,
357 chksum_kstat_addr);
358 kstat_install(chksum_kstat);
359 }
360 }
361
362 void
chksum_fini(void)363 chksum_fini(void)
364 {
365 if (chksum_kstat != NULL) {
366 kstat_delete(chksum_kstat);
367 chksum_kstat = NULL;
368 }
369
370 if (chksum_stat_cnt) {
371 kmem_free(chksum_stat_data,
372 sizeof (chksum_stat_t) * chksum_stat_cnt);
373 chksum_stat_cnt = 0;
374 chksum_stat_data = 0;
375 }
376
377 #ifdef _KERNEL
378 blake3_per_cpu_ctx_fini();
379 #endif
380 }
381