1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 25 */ 26 27 #include <sys/simd.h> 28 #include <sys/zfs_context.h> 29 #include <sys/zfs_impl.h> 30 #include <sys/sha2.h> 31 32 #include <sha2/sha2_impl.h> 33 #include <sys/asm_linkage.h> 34 35 #define TF(E, N) \ 36 extern void ASMABI E(uint32_t s[8], const void *, size_t); \ 37 static inline void N(uint32_t s[8], const void *d, size_t b) { \ 38 kfpu_begin(); E(s, d, b); kfpu_end(); \ 39 } 40 41 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \ 42 defined(__PPC64__) 43 /* some implementation is always okay */ 44 static inline boolean_t sha2_is_supported(void) 45 { 46 return (B_TRUE); 47 } 48 #endif 49 50 #if defined(__x86_64) 51 52 /* Users of ASMABI requires all calls to be from wrappers */ 53 extern void ASMABI 54 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t); 55 56 static inline void 57 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b) 58 { 59 zfs_sha256_transform_x64(s, d, b); 60 } 61 62 const sha256_ops_t sha256_x64_impl = { 63 .is_supported = sha2_is_supported, 64 .transform = tf_sha256_transform_x64, 65 .name = "x64" 66 }; 67 68 #if defined(HAVE_SSSE3) 69 static boolean_t sha2_have_ssse3(void) 70 { 71 return (kfpu_allowed() && zfs_ssse3_available()); 72 } 73 74 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3); 75 const sha256_ops_t sha256_ssse3_impl = { 76 .is_supported = sha2_have_ssse3, 77 .transform = tf_sha256_ssse3, 78 .name = "ssse3" 79 }; 80 #endif 81 82 #if defined(HAVE_AVX) 83 static boolean_t sha2_have_avx(void) 84 { 85 return (kfpu_allowed() && zfs_avx_available()); 86 } 87 88 TF(zfs_sha256_transform_avx, tf_sha256_avx); 89 const sha256_ops_t sha256_avx_impl = { 90 .is_supported = sha2_have_avx, 91 .transform = tf_sha256_avx, 92 .name = "avx" 93 }; 94 #endif 95 96 #if defined(HAVE_AVX2) 97 static boolean_t sha2_have_avx2(void) 98 { 99 return (kfpu_allowed() && zfs_avx2_available()); 100 } 101 102 TF(zfs_sha256_transform_avx2, tf_sha256_avx2); 103 const sha256_ops_t sha256_avx2_impl = { 104 .is_supported = sha2_have_avx2, 105 .transform = tf_sha256_avx2, 106 .name = "avx2" 107 }; 108 #endif 109 110 #if defined(HAVE_SSE4_1) 111 static boolean_t sha2_have_shani(void) 112 { 113 return (kfpu_allowed() && zfs_sse4_1_available() && \ 114 zfs_shani_available()); 115 } 116 117 TF(zfs_sha256_transform_shani, tf_sha256_shani); 118 const sha256_ops_t sha256_shani_impl = { 119 .is_supported = sha2_have_shani, 120 .transform = tf_sha256_shani, 121 .name = "shani" 122 }; 123 #endif 124 125 #elif defined(__aarch64__) || defined(__arm__) 126 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t); 127 const sha256_ops_t sha256_armv7_impl = { 128 .is_supported = sha2_is_supported, 129 .transform = zfs_sha256_block_armv7, 130 .name = "armv7" 131 }; 132 133 #if __ARM_ARCH > 6 134 static boolean_t sha256_have_neon(void) 135 { 136 return (kfpu_allowed() && zfs_neon_available()); 137 } 138 139 static boolean_t sha256_have_armv8ce(void) 140 { 141 return (kfpu_allowed() && zfs_sha256_available()); 142 } 143 144 TF(zfs_sha256_block_neon, tf_sha256_neon); 145 const sha256_ops_t sha256_neon_impl = { 146 .is_supported = sha256_have_neon, 147 .transform = tf_sha256_neon, 148 .name = "neon" 149 }; 150 151 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce); 152 const sha256_ops_t sha256_armv8_impl = { 153 .is_supported = sha256_have_armv8ce, 154 .transform = tf_sha256_armv8ce, 155 .name = "armv8-ce" 156 }; 157 #endif 158 159 #elif defined(__PPC64__) 160 static boolean_t sha256_have_isa207(void) 161 { 162 return (kfpu_allowed() && zfs_isa207_available()); 163 } 164 165 TF(zfs_sha256_ppc, tf_sha256_ppc); 166 const sha256_ops_t sha256_ppc_impl = { 167 .is_supported = sha2_is_supported, 168 .transform = tf_sha256_ppc, 169 .name = "ppc" 170 }; 171 172 TF(zfs_sha256_power8, tf_sha256_power8); 173 const sha256_ops_t sha256_power8_impl = { 174 .is_supported = sha256_have_isa207, 175 .transform = tf_sha256_power8, 176 .name = "power8" 177 }; 178 #endif /* __PPC64__ */ 179 180 /* the two generic ones */ 181 extern const sha256_ops_t sha256_generic_impl; 182 183 /* array with all sha256 implementations */ 184 static const sha256_ops_t *const sha256_impls[] = { 185 &sha256_generic_impl, 186 #if defined(__x86_64) 187 &sha256_x64_impl, 188 #endif 189 #if defined(__x86_64) && defined(HAVE_SSSE3) 190 &sha256_ssse3_impl, 191 #endif 192 #if defined(__x86_64) && defined(HAVE_AVX) 193 &sha256_avx_impl, 194 #endif 195 #if defined(__x86_64) && defined(HAVE_AVX2) 196 &sha256_avx2_impl, 197 #endif 198 #if defined(__x86_64) && defined(HAVE_SSE4_1) 199 &sha256_shani_impl, 200 #endif 201 #if defined(__aarch64__) || defined(__arm__) 202 &sha256_armv7_impl, 203 #if __ARM_ARCH > 6 204 &sha256_neon_impl, 205 &sha256_armv8_impl, 206 #endif 207 #endif 208 #if defined(__PPC64__) 209 &sha256_ppc_impl, 210 &sha256_power8_impl, 211 #endif /* __PPC64__ */ 212 }; 213 214 /* use the generic implementation functions */ 215 #define IMPL_NAME "sha256" 216 #define IMPL_OPS_T sha256_ops_t 217 #define IMPL_ARRAY sha256_impls 218 #define IMPL_GET_OPS sha256_get_ops 219 #define ZFS_IMPL_OPS zfs_sha256_ops 220 #include <generic_impl.c> 221 222 #ifdef _KERNEL 223 224 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 225 226 #if defined(__linux__) 227 228 static int 229 sha256_param_get(char *buffer, zfs_kernel_param_t *unused) 230 { 231 const uint32_t impl = IMPL_READ(generic_impl_chosen); 232 char *fmt; 233 int cnt = 0; 234 235 /* cycling */ 236 fmt = IMPL_FMT(impl, IMPL_CYCLE); 237 cnt += sprintf(buffer + cnt, fmt, "cycle"); 238 239 /* list fastest */ 240 fmt = IMPL_FMT(impl, IMPL_FASTEST); 241 cnt += sprintf(buffer + cnt, fmt, "fastest"); 242 243 /* list all supported implementations */ 244 generic_impl_init(); 245 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 246 fmt = IMPL_FMT(impl, i); 247 cnt += sprintf(buffer + cnt, fmt, 248 generic_supp_impls[i]->name); 249 } 250 251 return (cnt); 252 } 253 254 static int 255 sha256_param_set(const char *val, zfs_kernel_param_t *unused) 256 { 257 (void) unused; 258 return (generic_impl_setname(val)); 259 } 260 261 #elif defined(__FreeBSD__) 262 263 #include <sys/sbuf.h> 264 265 static int 266 sha256_param(ZFS_MODULE_PARAM_ARGS) 267 { 268 int err; 269 270 generic_impl_init(); 271 if (req->newptr == NULL) { 272 const uint32_t impl = IMPL_READ(generic_impl_chosen); 273 const int init_buflen = 64; 274 const char *fmt; 275 struct sbuf *s; 276 277 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 278 279 /* cycling */ 280 fmt = IMPL_FMT(impl, IMPL_CYCLE); 281 (void) sbuf_printf(s, fmt, "cycle"); 282 283 /* list fastest */ 284 fmt = IMPL_FMT(impl, IMPL_FASTEST); 285 (void) sbuf_printf(s, fmt, "fastest"); 286 287 /* list all supported implementations */ 288 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 289 fmt = IMPL_FMT(impl, i); 290 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 291 } 292 293 err = sbuf_finish(s); 294 sbuf_delete(s); 295 296 return (err); 297 } 298 299 char buf[16]; 300 301 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 302 if (err) { 303 return (err); 304 } 305 306 return (-generic_impl_setname(buf)); 307 } 308 #endif 309 310 #undef IMPL_FMT 311 312 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl, 313 sha256_param_set, sha256_param_get, ZMOD_RW, \ 314 "Select SHA256 implementation."); 315 #endif 316 317 #undef TF 318