1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 25 */ 26 27 #include <sys/simd.h> 28 #include <sys/zfs_context.h> 29 #include <sys/zfs_impl.h> 30 #include <sys/sha2.h> 31 32 #include <sha2/sha2_impl.h> 33 #include <sys/asm_linkage.h> 34 35 #define TF(E, N) \ 36 extern void ASMABI E(uint32_t s[8], const void *, size_t); \ 37 static inline void N(uint32_t s[8], const void *d, size_t b) { \ 38 kfpu_begin(); E(s, d, b); kfpu_end(); \ 39 } 40 41 /* some implementation is always okay */ 42 static inline boolean_t sha2_is_supported(void) 43 { 44 return (B_TRUE); 45 } 46 47 #if defined(__x86_64) 48 49 /* Users of ASMABI requires all calls to be from wrappers */ 50 extern void ASMABI 51 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t); 52 53 static inline void 54 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b) 55 { 56 zfs_sha256_transform_x64(s, d, b); 57 } 58 59 const sha256_ops_t sha256_x64_impl = { 60 .is_supported = sha2_is_supported, 61 .transform = tf_sha256_transform_x64, 62 .name = "x64" 63 }; 64 65 #if defined(HAVE_SSSE3) 66 static boolean_t sha2_have_ssse3(void) 67 { 68 return (kfpu_allowed() && zfs_ssse3_available()); 69 } 70 71 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3); 72 const sha256_ops_t sha256_ssse3_impl = { 73 .is_supported = sha2_have_ssse3, 74 .transform = tf_sha256_ssse3, 75 .name = "ssse3" 76 }; 77 #endif 78 79 #if defined(HAVE_AVX) 80 static boolean_t sha2_have_avx(void) 81 { 82 return (kfpu_allowed() && zfs_avx_available()); 83 } 84 85 TF(zfs_sha256_transform_avx, tf_sha256_avx); 86 const sha256_ops_t sha256_avx_impl = { 87 .is_supported = sha2_have_avx, 88 .transform = tf_sha256_avx, 89 .name = "avx" 90 }; 91 #endif 92 93 #if defined(HAVE_AVX2) 94 static boolean_t sha2_have_avx2(void) 95 { 96 return (kfpu_allowed() && zfs_avx2_available()); 97 } 98 99 TF(zfs_sha256_transform_avx2, tf_sha256_avx2); 100 const sha256_ops_t sha256_avx2_impl = { 101 .is_supported = sha2_have_avx2, 102 .transform = tf_sha256_avx2, 103 .name = "avx2" 104 }; 105 #endif 106 107 #if defined(HAVE_SSE4_1) 108 static boolean_t sha2_have_shani(void) 109 { 110 return (kfpu_allowed() && zfs_sse4_1_available() && \ 111 zfs_shani_available()); 112 } 113 114 TF(zfs_sha256_transform_shani, tf_sha256_shani); 115 const sha256_ops_t sha256_shani_impl = { 116 .is_supported = sha2_have_shani, 117 .transform = tf_sha256_shani, 118 .name = "shani" 119 }; 120 #endif 121 122 #elif defined(__aarch64__) || defined(__arm__) 123 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t); 124 const sha256_ops_t sha256_armv7_impl = { 125 .is_supported = sha2_is_supported, 126 .transform = zfs_sha256_block_armv7, 127 .name = "armv7" 128 }; 129 130 #if __ARM_ARCH > 6 131 static boolean_t sha256_have_neon(void) 132 { 133 return (kfpu_allowed() && zfs_neon_available()); 134 } 135 136 static boolean_t sha256_have_armv8ce(void) 137 { 138 return (kfpu_allowed() && zfs_sha256_available()); 139 } 140 141 TF(zfs_sha256_block_neon, tf_sha256_neon); 142 const sha256_ops_t sha256_neon_impl = { 143 .is_supported = sha256_have_neon, 144 .transform = tf_sha256_neon, 145 .name = "neon" 146 }; 147 148 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce); 149 const sha256_ops_t sha256_armv8_impl = { 150 .is_supported = sha256_have_armv8ce, 151 .transform = tf_sha256_armv8ce, 152 .name = "armv8-ce" 153 }; 154 #endif 155 156 #elif defined(__PPC64__) 157 static boolean_t sha256_have_isa207(void) 158 { 159 return (kfpu_allowed() && zfs_isa207_available()); 160 } 161 162 TF(zfs_sha256_ppc, tf_sha256_ppc); 163 const sha256_ops_t sha256_ppc_impl = { 164 .is_supported = sha2_is_supported, 165 .transform = tf_sha256_ppc, 166 .name = "ppc" 167 }; 168 169 TF(zfs_sha256_power8, tf_sha256_power8); 170 const sha256_ops_t sha256_power8_impl = { 171 .is_supported = sha256_have_isa207, 172 .transform = tf_sha256_power8, 173 .name = "power8" 174 }; 175 #endif /* __PPC64__ */ 176 177 /* the two generic ones */ 178 extern const sha256_ops_t sha256_generic_impl; 179 180 /* array with all sha256 implementations */ 181 static const sha256_ops_t *const sha256_impls[] = { 182 &sha256_generic_impl, 183 #if defined(__x86_64) 184 &sha256_x64_impl, 185 #endif 186 #if defined(__x86_64) && defined(HAVE_SSSE3) 187 &sha256_ssse3_impl, 188 #endif 189 #if defined(__x86_64) && defined(HAVE_AVX) 190 &sha256_avx_impl, 191 #endif 192 #if defined(__x86_64) && defined(HAVE_AVX2) 193 &sha256_avx2_impl, 194 #endif 195 #if defined(__x86_64) && defined(HAVE_SSE4_1) 196 &sha256_shani_impl, 197 #endif 198 #if defined(__aarch64__) || defined(__arm__) 199 &sha256_armv7_impl, 200 #if __ARM_ARCH > 6 201 &sha256_neon_impl, 202 &sha256_armv8_impl, 203 #endif 204 #endif 205 #if defined(__PPC64__) 206 &sha256_ppc_impl, 207 &sha256_power8_impl, 208 #endif /* __PPC64__ */ 209 }; 210 211 /* use the generic implementation functions */ 212 #define IMPL_NAME "sha256" 213 #define IMPL_OPS_T sha256_ops_t 214 #define IMPL_ARRAY sha256_impls 215 #define IMPL_GET_OPS sha256_get_ops 216 #define ZFS_IMPL_OPS zfs_sha256_ops 217 #include <generic_impl.c> 218 219 #ifdef _KERNEL 220 221 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 222 223 #if defined(__linux__) 224 225 static int 226 sha256_param_get(char *buffer, zfs_kernel_param_t *unused) 227 { 228 const uint32_t impl = IMPL_READ(generic_impl_chosen); 229 char *fmt; 230 int cnt = 0; 231 232 /* cycling */ 233 fmt = IMPL_FMT(impl, IMPL_CYCLE); 234 cnt += sprintf(buffer + cnt, fmt, "cycle"); 235 236 /* list fastest */ 237 fmt = IMPL_FMT(impl, IMPL_FASTEST); 238 cnt += sprintf(buffer + cnt, fmt, "fastest"); 239 240 /* list all supported implementations */ 241 generic_impl_init(); 242 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 243 fmt = IMPL_FMT(impl, i); 244 cnt += sprintf(buffer + cnt, fmt, 245 generic_supp_impls[i]->name); 246 } 247 248 return (cnt); 249 } 250 251 static int 252 sha256_param_set(const char *val, zfs_kernel_param_t *unused) 253 { 254 (void) unused; 255 return (generic_impl_setname(val)); 256 } 257 258 #elif defined(__FreeBSD__) 259 260 #include <sys/sbuf.h> 261 262 static int 263 sha256_param(ZFS_MODULE_PARAM_ARGS) 264 { 265 int err; 266 267 generic_impl_init(); 268 if (req->newptr == NULL) { 269 const uint32_t impl = IMPL_READ(generic_impl_chosen); 270 const int init_buflen = 64; 271 const char *fmt; 272 struct sbuf *s; 273 274 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 275 276 /* cycling */ 277 fmt = IMPL_FMT(impl, IMPL_CYCLE); 278 (void) sbuf_printf(s, fmt, "cycle"); 279 280 /* list fastest */ 281 fmt = IMPL_FMT(impl, IMPL_FASTEST); 282 (void) sbuf_printf(s, fmt, "fastest"); 283 284 /* list all supported implementations */ 285 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 286 fmt = IMPL_FMT(impl, i); 287 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 288 } 289 290 err = sbuf_finish(s); 291 sbuf_delete(s); 292 293 return (err); 294 } 295 296 char buf[16]; 297 298 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 299 if (err) { 300 return (err); 301 } 302 303 return (-generic_impl_setname(buf)); 304 } 305 #endif 306 307 #undef IMPL_FMT 308 309 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl, 310 sha256_param_set, sha256_param_get, ZMOD_RW, \ 311 "Select SHA256 implementation."); 312 #endif 313 314 #undef TF 315