1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de> 25 */ 26 27 #include <sys/zio_checksum.h> 28 #include <sys/zfs_context.h> 29 #include <sys/zfs_chksum.h> 30 #include <sys/zfs_impl.h> 31 32 #include <sys/blake3.h> 33 #include <sys/sha2.h> 34 35 /* limit benchmarking to max 256KiB, when EdonR is slower then this: */ 36 #define LIMIT_PERF_MBS 300 37 38 typedef struct { 39 const char *name; 40 const char *impl; 41 uint64_t bs1k; 42 uint64_t bs4k; 43 uint64_t bs16k; 44 uint64_t bs64k; 45 uint64_t bs256k; 46 uint64_t bs1m; 47 uint64_t bs4m; 48 uint64_t bs16m; 49 zio_cksum_salt_t salt; 50 zio_checksum_t *(func); 51 zio_checksum_tmpl_init_t *(init); 52 zio_checksum_tmpl_free_t *(free); 53 } chksum_stat_t; 54 55 static chksum_stat_t *chksum_stat_data = 0; 56 static int chksum_stat_cnt = 0; 57 static kstat_t *chksum_kstat = NULL; 58 59 /* 60 * Sample output on i3-1005G1 System: 61 * 62 * implementation 1k 4k 16k 64k 256k 1m 4m 16m 63 * edonr-generic 1278 1625 1769 1776 1783 1778 1771 1767 64 * skein-generic 548 594 613 623 621 623 621 486 65 * sha256-generic 255 270 281 278 279 281 283 283 66 * sha256-x64 288 310 316 317 318 317 317 316 67 * sha256-ssse3 304 342 351 355 356 357 356 356 68 * sha256-avx 311 348 359 362 362 363 363 362 69 * sha256-avx2 330 378 389 395 395 395 395 395 70 * sha256-shani 908 1127 1212 1230 1233 1234 1223 1230 71 * sha512-generic 359 409 431 427 429 430 428 423 72 * sha512-x64 420 473 490 496 497 497 496 495 73 * sha512-avx 406 522 546 560 560 560 556 560 74 * sha512-avx2 464 568 601 606 609 610 607 608 75 * blake3-generic 330 327 324 323 324 320 323 322 76 * blake3-sse2 424 1366 1449 1468 1458 1453 1395 1408 77 * blake3-sse41 453 1554 1658 1703 1689 1669 1622 1630 78 * blake3-avx2 452 2013 3225 3351 3356 3261 3076 3101 79 * blake3-avx512 498 2869 5269 5926 5872 5643 5014 5005 80 */ 81 static int 82 chksum_kstat_headers(char *buf, size_t size) 83 { 84 ssize_t off = 0; 85 86 off += kmem_scnprintf(buf + off, size, "%-23s", "implementation"); 87 off += kmem_scnprintf(buf + off, size - off, "%8s", "1k"); 88 off += kmem_scnprintf(buf + off, size - off, "%8s", "4k"); 89 off += kmem_scnprintf(buf + off, size - off, "%8s", "16k"); 90 off += kmem_scnprintf(buf + off, size - off, "%8s", "64k"); 91 off += kmem_scnprintf(buf + off, size - off, "%8s", "256k"); 92 off += kmem_scnprintf(buf + off, size - off, "%8s", "1m"); 93 off += kmem_scnprintf(buf + off, size - off, "%8s", "4m"); 94 (void) kmem_scnprintf(buf + off, size - off, "%8s\n", "16m"); 95 96 return (0); 97 } 98 99 static int 100 chksum_kstat_data(char *buf, size_t size, void *data) 101 { 102 chksum_stat_t *cs; 103 ssize_t off = 0; 104 char b[24]; 105 106 cs = (chksum_stat_t *)data; 107 kmem_scnprintf(b, 23, "%s-%s", cs->name, cs->impl); 108 off += kmem_scnprintf(buf + off, size - off, "%-23s", b); 109 off += kmem_scnprintf(buf + off, size - off, "%8llu", 110 (u_longlong_t)cs->bs1k); 111 off += kmem_scnprintf(buf + off, size - off, "%8llu", 112 (u_longlong_t)cs->bs4k); 113 off += kmem_scnprintf(buf + off, size - off, "%8llu", 114 (u_longlong_t)cs->bs16k); 115 off += kmem_scnprintf(buf + off, size - off, "%8llu", 116 (u_longlong_t)cs->bs64k); 117 off += kmem_scnprintf(buf + off, size - off, "%8llu", 118 (u_longlong_t)cs->bs256k); 119 off += kmem_scnprintf(buf + off, size - off, "%8llu", 120 (u_longlong_t)cs->bs1m); 121 off += kmem_scnprintf(buf + off, size - off, "%8llu", 122 (u_longlong_t)cs->bs4m); 123 (void) kmem_scnprintf(buf + off, size - off, "%8llu\n", 124 (u_longlong_t)cs->bs16m); 125 126 return (0); 127 } 128 129 static void * 130 chksum_kstat_addr(kstat_t *ksp, loff_t n) 131 { 132 if (n < chksum_stat_cnt) 133 ksp->ks_private = (void *)(chksum_stat_data + n); 134 else 135 ksp->ks_private = NULL; 136 137 return (ksp->ks_private); 138 } 139 140 static void 141 chksum_run(chksum_stat_t *cs, abd_t *abd, void *ctx, int round, 142 uint64_t *result) 143 { 144 hrtime_t start; 145 uint64_t run_bw, run_time_ns, run_count = 0, size = 0; 146 uint32_t l, loops = 0; 147 zio_cksum_t zcp; 148 149 switch (round) { 150 case 1: /* 1k */ 151 size = 1<<10; loops = 128; break; 152 case 2: /* 2k */ 153 size = 1<<12; loops = 64; break; 154 case 3: /* 4k */ 155 size = 1<<14; loops = 32; break; 156 case 4: /* 16k */ 157 size = 1<<16; loops = 16; break; 158 case 5: /* 256k */ 159 size = 1<<18; loops = 8; break; 160 case 6: /* 1m */ 161 size = 1<<20; loops = 4; break; 162 case 7: /* 4m */ 163 size = 1<<22; loops = 1; break; 164 case 8: /* 16m */ 165 size = 1<<24; loops = 1; break; 166 } 167 168 kpreempt_disable(); 169 start = gethrtime(); 170 do { 171 for (l = 0; l < loops; l++, run_count++) 172 cs->func(abd, size, ctx, &zcp); 173 174 run_time_ns = gethrtime() - start; 175 } while (run_time_ns < MSEC2NSEC(1)); 176 kpreempt_enable(); 177 178 run_bw = size * run_count * NANOSEC; 179 run_bw /= run_time_ns; /* B/s */ 180 *result = run_bw/1024/1024; /* MiB/s */ 181 } 182 183 #define LIMIT_INIT 0 184 #define LIMIT_NEEDED 1 185 #define LIMIT_NOLIMIT 2 186 187 static void 188 chksum_benchit(chksum_stat_t *cs) 189 { 190 abd_t *abd; 191 void *ctx = 0; 192 void *salt = &cs->salt.zcs_bytes; 193 static int chksum_stat_limit = LIMIT_INIT; 194 195 memset(salt, 0, sizeof (cs->salt.zcs_bytes)); 196 if (cs->init) 197 ctx = cs->init(&cs->salt); 198 199 /* allocate test memory via abd linear interface */ 200 abd = abd_alloc_linear(1<<20, B_FALSE); 201 chksum_run(cs, abd, ctx, 1, &cs->bs1k); 202 chksum_run(cs, abd, ctx, 2, &cs->bs4k); 203 chksum_run(cs, abd, ctx, 3, &cs->bs16k); 204 chksum_run(cs, abd, ctx, 4, &cs->bs64k); 205 chksum_run(cs, abd, ctx, 5, &cs->bs256k); 206 207 /* check if we ran on a slow cpu */ 208 if (chksum_stat_limit == LIMIT_INIT) { 209 if (cs->bs1k < LIMIT_PERF_MBS) { 210 chksum_stat_limit = LIMIT_NEEDED; 211 } else { 212 chksum_stat_limit = LIMIT_NOLIMIT; 213 } 214 } 215 216 /* skip benchmarks >= 1MiB when the CPU is to slow */ 217 if (chksum_stat_limit == LIMIT_NEEDED) 218 goto abort; 219 220 chksum_run(cs, abd, ctx, 6, &cs->bs1m); 221 abd_free(abd); 222 223 /* allocate test memory via abd non linear interface */ 224 abd = abd_alloc(1<<24, B_FALSE); 225 chksum_run(cs, abd, ctx, 7, &cs->bs4m); 226 chksum_run(cs, abd, ctx, 8, &cs->bs16m); 227 228 abort: 229 abd_free(abd); 230 231 /* free up temp memory */ 232 if (cs->free) 233 cs->free(ctx); 234 } 235 236 /* 237 * Initialize and benchmark all supported implementations. 238 */ 239 static void 240 chksum_benchmark(void) 241 { 242 #ifndef _KERNEL 243 /* we need the benchmark only for the kernel module */ 244 return; 245 #endif 246 247 chksum_stat_t *cs; 248 uint64_t max; 249 uint32_t id, cbid = 0, id_save; 250 const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3"); 251 const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256"); 252 const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512"); 253 254 /* count implementations */ 255 chksum_stat_cnt = 2; 256 chksum_stat_cnt += sha256->getcnt(); 257 chksum_stat_cnt += sha512->getcnt(); 258 chksum_stat_cnt += blake3->getcnt(); 259 chksum_stat_data = kmem_zalloc( 260 sizeof (chksum_stat_t) * chksum_stat_cnt, KM_SLEEP); 261 262 /* edonr - needs to be the first one here (slow CPU check) */ 263 cs = &chksum_stat_data[cbid++]; 264 265 /* edonr */ 266 cs->init = abd_checksum_edonr_tmpl_init; 267 cs->func = abd_checksum_edonr_native; 268 cs->free = abd_checksum_edonr_tmpl_free; 269 cs->name = "edonr"; 270 cs->impl = "generic"; 271 chksum_benchit(cs); 272 273 /* skein */ 274 cs = &chksum_stat_data[cbid++]; 275 cs->init = abd_checksum_skein_tmpl_init; 276 cs->func = abd_checksum_skein_native; 277 cs->free = abd_checksum_skein_tmpl_free; 278 cs->name = "skein"; 279 cs->impl = "generic"; 280 chksum_benchit(cs); 281 282 /* sha256 */ 283 id_save = sha256->getid(); 284 for (max = 0, id = 0; id < sha256->getcnt(); id++) { 285 sha256->setid(id); 286 cs = &chksum_stat_data[cbid++]; 287 cs->init = 0; 288 cs->func = abd_checksum_sha256; 289 cs->free = 0; 290 cs->name = sha256->name; 291 cs->impl = sha256->getname(); 292 chksum_benchit(cs); 293 if (cs->bs256k > max) { 294 max = cs->bs256k; 295 sha256->set_fastest(id); 296 } 297 } 298 sha256->setid(id_save); 299 300 /* sha512 */ 301 id_save = sha512->getid(); 302 for (max = 0, id = 0; id < sha512->getcnt(); id++) { 303 sha512->setid(id); 304 cs = &chksum_stat_data[cbid++]; 305 cs->init = 0; 306 cs->func = abd_checksum_sha512_native; 307 cs->free = 0; 308 cs->name = sha512->name; 309 cs->impl = sha512->getname(); 310 chksum_benchit(cs); 311 if (cs->bs256k > max) { 312 max = cs->bs256k; 313 sha512->set_fastest(id); 314 } 315 } 316 sha512->setid(id_save); 317 318 /* blake3 */ 319 id_save = blake3->getid(); 320 for (max = 0, id = 0; id < blake3->getcnt(); id++) { 321 blake3->setid(id); 322 cs = &chksum_stat_data[cbid++]; 323 cs->init = abd_checksum_blake3_tmpl_init; 324 cs->func = abd_checksum_blake3_native; 325 cs->free = abd_checksum_blake3_tmpl_free; 326 cs->name = blake3->name; 327 cs->impl = blake3->getname(); 328 chksum_benchit(cs); 329 if (cs->bs256k > max) { 330 max = cs->bs256k; 331 blake3->set_fastest(id); 332 } 333 } 334 blake3->setid(id_save); 335 } 336 337 void 338 chksum_init(void) 339 { 340 #ifdef _KERNEL 341 blake3_per_cpu_ctx_init(); 342 #endif 343 344 /* Benchmark supported implementations */ 345 chksum_benchmark(); 346 347 /* Install kstats for all implementations */ 348 chksum_kstat = kstat_create("zfs", 0, "chksum_bench", "misc", 349 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); 350 351 if (chksum_kstat != NULL) { 352 chksum_kstat->ks_data = NULL; 353 chksum_kstat->ks_ndata = UINT32_MAX; 354 kstat_set_raw_ops(chksum_kstat, 355 chksum_kstat_headers, 356 chksum_kstat_data, 357 chksum_kstat_addr); 358 kstat_install(chksum_kstat); 359 } 360 } 361 362 void 363 chksum_fini(void) 364 { 365 if (chksum_kstat != NULL) { 366 kstat_delete(chksum_kstat); 367 chksum_kstat = NULL; 368 } 369 370 if (chksum_stat_cnt) { 371 kmem_free(chksum_stat_data, 372 sizeof (chksum_stat_t) * chksum_stat_cnt); 373 chksum_stat_cnt = 0; 374 chksum_stat_data = 0; 375 } 376 377 #ifdef _KERNEL 378 blake3_per_cpu_ctx_fini(); 379 #endif 380 } 381