1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de> 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/zio_checksum.h> 28 29 #include "blake3_impl.h" 30 31 static const blake3_impl_ops_t *const blake3_impls[] = { 32 &blake3_generic_impl, 33 #if defined(__aarch64__) || \ 34 (defined(__x86_64) && defined(HAVE_SSE2)) || \ 35 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) 36 &blake3_sse2_impl, 37 #endif 38 #if defined(__aarch64__) || \ 39 (defined(__x86_64) && defined(HAVE_SSE4_1)) || \ 40 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) 41 &blake3_sse41_impl, 42 #endif 43 #if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2) 44 &blake3_avx2_impl, 45 #endif 46 #if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL) 47 &blake3_avx512_impl, 48 #endif 49 }; 50 51 /* this pointer holds current ops for implementation */ 52 static const blake3_impl_ops_t *blake3_selected_impl = &blake3_generic_impl; 53 54 /* special implementation selections */ 55 #define IMPL_FASTEST (UINT32_MAX) 56 #define IMPL_CYCLE (UINT32_MAX-1) 57 #define IMPL_USER (UINT32_MAX-2) 58 #define IMPL_PARAM (UINT32_MAX-3) 59 60 #define IMPL_READ(i) (*(volatile uint32_t *) &(i)) 61 static uint32_t icp_blake3_impl = IMPL_FASTEST; 62 63 #define BLAKE3_IMPL_NAME_MAX 16 64 65 /* id of fastest implementation */ 66 static uint32_t blake3_fastest_id = 0; 67 68 /* currently used id */ 69 static uint32_t blake3_current_id = 0; 70 71 /* id of module parameter (-1 == unused) */ 72 static int blake3_param_id = -1; 73 74 /* return number of supported implementations */ 75 int 76 blake3_get_impl_count(void) 77 { 78 static int impls = 0; 79 int i; 80 81 if (impls) 82 return (impls); 83 84 for (i = 0; i < ARRAY_SIZE(blake3_impls); i++) { 85 if (!blake3_impls[i]->is_supported()) continue; 86 impls++; 87 } 88 89 return (impls); 90 } 91 92 /* return id of selected implementation */ 93 int 94 blake3_get_impl_id(void) 95 { 96 return (blake3_current_id); 97 } 98 99 /* return name of selected implementation */ 100 const char * 101 blake3_get_impl_name(void) 102 { 103 return (blake3_selected_impl->name); 104 } 105 106 /* setup id as fastest implementation */ 107 void 108 blake3_set_impl_fastest(uint32_t id) 109 { 110 blake3_fastest_id = id; 111 } 112 113 /* set implementation by id */ 114 void 115 blake3_set_impl_id(uint32_t id) 116 { 117 int i, cid; 118 119 /* select fastest */ 120 if (id == IMPL_FASTEST) 121 id = blake3_fastest_id; 122 123 /* select next or first */ 124 if (id == IMPL_CYCLE) 125 id = (++blake3_current_id) % blake3_get_impl_count(); 126 127 /* 0..N for the real impl */ 128 for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) { 129 if (!blake3_impls[i]->is_supported()) continue; 130 if (cid == id) { 131 blake3_current_id = cid; 132 blake3_selected_impl = blake3_impls[i]; 133 return; 134 } 135 cid++; 136 } 137 } 138 139 /* set implementation by name */ 140 int 141 blake3_set_impl_name(const char *name) 142 { 143 int i, cid; 144 145 if (strcmp(name, "fastest") == 0) { 146 atomic_swap_32(&icp_blake3_impl, IMPL_FASTEST); 147 blake3_set_impl_id(IMPL_FASTEST); 148 return (0); 149 } else if (strcmp(name, "cycle") == 0) { 150 atomic_swap_32(&icp_blake3_impl, IMPL_CYCLE); 151 blake3_set_impl_id(IMPL_CYCLE); 152 return (0); 153 } 154 155 for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) { 156 if (!blake3_impls[i]->is_supported()) continue; 157 if (strcmp(name, blake3_impls[i]->name) == 0) { 158 if (icp_blake3_impl == IMPL_PARAM) { 159 blake3_param_id = cid; 160 return (0); 161 } 162 blake3_selected_impl = blake3_impls[i]; 163 blake3_current_id = cid; 164 return (0); 165 } 166 cid++; 167 } 168 169 return (-EINVAL); 170 } 171 172 /* setup implementation */ 173 void 174 blake3_setup_impl(void) 175 { 176 switch (IMPL_READ(icp_blake3_impl)) { 177 case IMPL_PARAM: 178 blake3_set_impl_id(blake3_param_id); 179 atomic_swap_32(&icp_blake3_impl, IMPL_USER); 180 break; 181 case IMPL_FASTEST: 182 blake3_set_impl_id(IMPL_FASTEST); 183 break; 184 case IMPL_CYCLE: 185 blake3_set_impl_id(IMPL_CYCLE); 186 break; 187 default: 188 blake3_set_impl_id(blake3_current_id); 189 break; 190 } 191 } 192 193 /* return selected implementation */ 194 const blake3_impl_ops_t * 195 blake3_impl_get_ops(void) 196 { 197 /* each call to ops will cycle */ 198 if (icp_blake3_impl == IMPL_CYCLE) 199 blake3_set_impl_id(IMPL_CYCLE); 200 201 return (blake3_selected_impl); 202 } 203 204 #if defined(_KERNEL) 205 void **blake3_per_cpu_ctx; 206 207 void 208 blake3_per_cpu_ctx_init(void) 209 { 210 /* 211 * Create "The Godfather" ptr to hold all blake3 ctx 212 */ 213 blake3_per_cpu_ctx = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP); 214 for (int i = 0; i < max_ncpus; i++) { 215 blake3_per_cpu_ctx[i] = kmem_alloc(sizeof (BLAKE3_CTX), 216 KM_SLEEP); 217 } 218 } 219 220 void 221 blake3_per_cpu_ctx_fini(void) 222 { 223 for (int i = 0; i < max_ncpus; i++) { 224 memset(blake3_per_cpu_ctx[i], 0, sizeof (BLAKE3_CTX)); 225 kmem_free(blake3_per_cpu_ctx[i], sizeof (BLAKE3_CTX)); 226 } 227 memset(blake3_per_cpu_ctx, 0, max_ncpus * sizeof (void *)); 228 kmem_free(blake3_per_cpu_ctx, max_ncpus * sizeof (void *)); 229 } 230 #endif 231 232 #if defined(_KERNEL) && defined(__linux__) 233 static int 234 icp_blake3_impl_set(const char *name, zfs_kernel_param_t *kp) 235 { 236 char req_name[BLAKE3_IMPL_NAME_MAX]; 237 size_t i; 238 239 /* sanitize input */ 240 i = strnlen(name, BLAKE3_IMPL_NAME_MAX); 241 if (i == 0 || i >= BLAKE3_IMPL_NAME_MAX) 242 return (-EINVAL); 243 244 strlcpy(req_name, name, BLAKE3_IMPL_NAME_MAX); 245 while (i > 0 && isspace(req_name[i-1])) 246 i--; 247 req_name[i] = '\0'; 248 249 atomic_swap_32(&icp_blake3_impl, IMPL_PARAM); 250 return (blake3_set_impl_name(req_name)); 251 } 252 253 static int 254 icp_blake3_impl_get(char *buffer, zfs_kernel_param_t *kp) 255 { 256 int i, cid, cnt = 0; 257 char *fmt; 258 259 /* cycling */ 260 fmt = (icp_blake3_impl == IMPL_CYCLE) ? "[cycle] " : "cycle "; 261 cnt += sprintf(buffer + cnt, fmt); 262 263 /* fastest one */ 264 fmt = (icp_blake3_impl == IMPL_FASTEST) ? "[fastest] " : "fastest "; 265 cnt += sprintf(buffer + cnt, fmt); 266 267 /* user selected */ 268 for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) { 269 if (!blake3_impls[i]->is_supported()) continue; 270 fmt = (icp_blake3_impl == IMPL_USER && 271 cid == blake3_current_id) ? "[%s] " : "%s "; 272 cnt += sprintf(buffer + cnt, fmt, blake3_impls[i]->name); 273 cid++; 274 } 275 276 buffer[cnt] = 0; 277 278 return (cnt); 279 } 280 281 module_param_call(icp_blake3_impl, icp_blake3_impl_set, icp_blake3_impl_get, 282 NULL, 0644); 283 MODULE_PARM_DESC(icp_blake3_impl, "Select BLAKE3 implementation."); 284 #endif 285