1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de> 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/zio_checksum.h> 28 29 #include "blake3_impl.h" 30 31 static const blake3_ops_t *const blake3_impls[] = { 32 &blake3_generic_impl, 33 #if defined(__aarch64__) || \ 34 (defined(__x86_64) && defined(HAVE_SSE2)) || \ 35 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) 36 &blake3_sse2_impl, 37 #endif 38 #if defined(__aarch64__) || \ 39 (defined(__x86_64) && defined(HAVE_SSE4_1)) || \ 40 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) 41 &blake3_sse41_impl, 42 #endif 43 #if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2) 44 &blake3_avx2_impl, 45 #endif 46 #if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL) 47 &blake3_avx512_impl, 48 #endif 49 }; 50 51 /* Select BLAKE3 implementation */ 52 #define IMPL_FASTEST (UINT32_MAX) 53 #define IMPL_CYCLE (UINT32_MAX - 1) 54 55 #define IMPL_READ(i) (*(volatile uint32_t *) &(i)) 56 57 /* Indicate that benchmark has been done */ 58 static boolean_t blake3_initialized = B_FALSE; 59 60 /* Implementation that contains the fastest methods */ 61 static blake3_ops_t blake3_fastest_impl = { 62 .name = "fastest" 63 }; 64 65 /* Hold all supported implementations */ 66 static const blake3_ops_t *blake3_supp_impls[ARRAY_SIZE(blake3_impls)]; 67 static uint32_t blake3_supp_impls_cnt = 0; 68 69 /* Currently selected implementation */ 70 static uint32_t blake3_impl_chosen = IMPL_FASTEST; 71 72 static struct blake3_impl_selector { 73 const char *name; 74 uint32_t sel; 75 } blake3_impl_selectors[] = { 76 { "cycle", IMPL_CYCLE }, 77 { "fastest", IMPL_FASTEST } 78 }; 79 80 /* check the supported implementations */ 81 static void blake3_impl_init(void) 82 { 83 int i, c; 84 85 /* init only once */ 86 if (likely(blake3_initialized)) 87 return; 88 89 /* move supported implementations into blake3_supp_impls */ 90 for (i = 0, c = 0; i < ARRAY_SIZE(blake3_impls); i++) { 91 const blake3_ops_t *impl = blake3_impls[i]; 92 93 if (impl->is_supported && impl->is_supported()) 94 blake3_supp_impls[c++] = impl; 95 } 96 blake3_supp_impls_cnt = c; 97 98 /* first init generic impl, may be changed via set_fastest() */ 99 memcpy(&blake3_fastest_impl, blake3_impls[0], 100 sizeof (blake3_fastest_impl)); 101 blake3_initialized = B_TRUE; 102 } 103 104 /* get number of supported implementations */ 105 uint32_t 106 blake3_impl_getcnt(void) 107 { 108 blake3_impl_init(); 109 return (blake3_supp_impls_cnt); 110 } 111 112 /* get id of selected implementation */ 113 uint32_t 114 blake3_impl_getid(void) 115 { 116 return (IMPL_READ(blake3_impl_chosen)); 117 } 118 119 /* get name of selected implementation */ 120 const char * 121 blake3_impl_getname(void) 122 { 123 uint32_t impl = IMPL_READ(blake3_impl_chosen); 124 125 blake3_impl_init(); 126 switch (impl) { 127 case IMPL_FASTEST: 128 return ("fastest"); 129 case IMPL_CYCLE: 130 return ("cycle"); 131 default: 132 return (blake3_supp_impls[impl]->name); 133 } 134 } 135 136 /* setup id as fastest implementation */ 137 void 138 blake3_impl_set_fastest(uint32_t id) 139 { 140 /* setup fastest impl */ 141 memcpy(&blake3_fastest_impl, blake3_supp_impls[id], 142 sizeof (blake3_fastest_impl)); 143 } 144 145 /* set implementation by id */ 146 void 147 blake3_impl_setid(uint32_t id) 148 { 149 blake3_impl_init(); 150 switch (id) { 151 case IMPL_FASTEST: 152 atomic_swap_32(&blake3_impl_chosen, IMPL_FASTEST); 153 break; 154 case IMPL_CYCLE: 155 atomic_swap_32(&blake3_impl_chosen, IMPL_CYCLE); 156 break; 157 default: 158 ASSERT3U(id, <, blake3_supp_impls_cnt); 159 atomic_swap_32(&blake3_impl_chosen, id); 160 break; 161 } 162 } 163 164 /* set implementation by name */ 165 int 166 blake3_impl_setname(const char *val) 167 { 168 uint32_t impl = IMPL_READ(blake3_impl_chosen); 169 size_t val_len; 170 int i, err = -EINVAL; 171 172 blake3_impl_init(); 173 val_len = strlen(val); 174 while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */ 175 val_len--; 176 177 /* check mandatory implementations */ 178 for (i = 0; i < ARRAY_SIZE(blake3_impl_selectors); i++) { 179 const char *name = blake3_impl_selectors[i].name; 180 181 if (val_len == strlen(name) && 182 strncmp(val, name, val_len) == 0) { 183 impl = blake3_impl_selectors[i].sel; 184 err = 0; 185 break; 186 } 187 } 188 189 if (err != 0 && blake3_initialized) { 190 /* check all supported implementations */ 191 for (i = 0; i < blake3_supp_impls_cnt; i++) { 192 const char *name = blake3_supp_impls[i]->name; 193 194 if (val_len == strlen(name) && 195 strncmp(val, name, val_len) == 0) { 196 impl = i; 197 err = 0; 198 break; 199 } 200 } 201 } 202 203 if (err == 0) { 204 atomic_swap_32(&blake3_impl_chosen, impl); 205 } 206 207 return (err); 208 } 209 210 const blake3_ops_t * 211 blake3_impl_get_ops(void) 212 { 213 const blake3_ops_t *ops = NULL; 214 uint32_t impl = IMPL_READ(blake3_impl_chosen); 215 216 blake3_impl_init(); 217 switch (impl) { 218 case IMPL_FASTEST: 219 ASSERT(blake3_initialized); 220 ops = &blake3_fastest_impl; 221 break; 222 case IMPL_CYCLE: 223 /* Cycle through supported implementations */ 224 ASSERT(blake3_initialized); 225 ASSERT3U(blake3_supp_impls_cnt, >, 0); 226 static uint32_t cycle_count = 0; 227 uint32_t idx = (++cycle_count) % blake3_supp_impls_cnt; 228 ops = blake3_supp_impls[idx]; 229 break; 230 default: 231 ASSERT3U(blake3_supp_impls_cnt, >, 0); 232 ASSERT3U(impl, <, blake3_supp_impls_cnt); 233 ops = blake3_supp_impls[impl]; 234 break; 235 } 236 237 ASSERT3P(ops, !=, NULL); 238 return (ops); 239 } 240 241 #if defined(_KERNEL) 242 243 void **blake3_per_cpu_ctx; 244 245 void 246 blake3_per_cpu_ctx_init(void) 247 { 248 /* 249 * Create "The Godfather" ptr to hold all blake3 ctx 250 */ 251 blake3_per_cpu_ctx = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP); 252 for (int i = 0; i < max_ncpus; i++) { 253 blake3_per_cpu_ctx[i] = kmem_alloc(sizeof (BLAKE3_CTX), 254 KM_SLEEP); 255 } 256 257 /* init once in kernel mode */ 258 blake3_impl_init(); 259 } 260 261 void 262 blake3_per_cpu_ctx_fini(void) 263 { 264 for (int i = 0; i < max_ncpus; i++) { 265 memset(blake3_per_cpu_ctx[i], 0, sizeof (BLAKE3_CTX)); 266 kmem_free(blake3_per_cpu_ctx[i], sizeof (BLAKE3_CTX)); 267 } 268 memset(blake3_per_cpu_ctx, 0, max_ncpus * sizeof (void *)); 269 kmem_free(blake3_per_cpu_ctx, max_ncpus * sizeof (void *)); 270 } 271 272 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 273 274 #if defined(__linux__) 275 276 static int 277 blake3_param_get(char *buffer, zfs_kernel_param_t *unused) 278 { 279 const uint32_t impl = IMPL_READ(blake3_impl_chosen); 280 char *fmt; 281 int cnt = 0; 282 283 /* cycling */ 284 fmt = IMPL_FMT(impl, IMPL_CYCLE); 285 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt, "cycle"); 286 287 /* list fastest */ 288 fmt = IMPL_FMT(impl, IMPL_FASTEST); 289 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt, "fastest"); 290 291 /* list all supported implementations */ 292 for (uint32_t i = 0; i < blake3_supp_impls_cnt; ++i) { 293 fmt = IMPL_FMT(impl, i); 294 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt, 295 blake3_supp_impls[i]->name); 296 } 297 298 return (cnt); 299 } 300 301 static int 302 blake3_param_set(const char *val, zfs_kernel_param_t *unused) 303 { 304 (void) unused; 305 return (blake3_impl_setname(val)); 306 } 307 308 #elif defined(__FreeBSD__) 309 310 #include <sys/sbuf.h> 311 312 static int 313 blake3_param(ZFS_MODULE_PARAM_ARGS) 314 { 315 int err; 316 317 if (req->newptr == NULL) { 318 const uint32_t impl = IMPL_READ(blake3_impl_chosen); 319 const int init_buflen = 64; 320 const char *fmt; 321 struct sbuf *s; 322 323 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 324 325 /* cycling */ 326 fmt = IMPL_FMT(impl, IMPL_CYCLE); 327 (void) sbuf_printf(s, fmt, "cycle"); 328 329 /* list fastest */ 330 fmt = IMPL_FMT(impl, IMPL_FASTEST); 331 (void) sbuf_printf(s, fmt, "fastest"); 332 333 /* list all supported implementations */ 334 for (uint32_t i = 0; i < blake3_supp_impls_cnt; ++i) { 335 fmt = IMPL_FMT(impl, i); 336 (void) sbuf_printf(s, fmt, blake3_supp_impls[i]->name); 337 } 338 339 err = sbuf_finish(s); 340 sbuf_delete(s); 341 342 return (err); 343 } 344 345 char buf[16]; 346 347 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 348 if (err) { 349 return (err); 350 } 351 352 return (-blake3_impl_setname(buf)); 353 } 354 #endif 355 356 #undef IMPL_FMT 357 358 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, blake3_impl, 359 blake3_param_set, blake3_param_get, ZMOD_RW, \ 360 "Select BLAKE3 implementation."); 361 #endif 362