1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 24 */ 25 26 #include <sys/simd.h> 27 #include <sys/zfs_context.h> 28 #include <sys/zfs_impl.h> 29 #include <sys/sha2.h> 30 31 #include <sha2/sha2_impl.h> 32 #include <sys/asm_linkage.h> 33 34 #define TF(E, N) \ 35 extern void ASMABI E(uint32_t s[8], const void *, size_t); \ 36 static inline void N(uint32_t s[8], const void *d, size_t b) { \ 37 kfpu_begin(); E(s, d, b); kfpu_end(); \ 38 } 39 40 /* some implementation is always okay */ 41 static inline boolean_t sha2_is_supported(void) 42 { 43 return (B_TRUE); 44 } 45 46 #if defined(__x86_64) 47 48 /* Users of ASMABI requires all calls to be from wrappers */ 49 extern void ASMABI 50 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t); 51 52 static inline void 53 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b) 54 { 55 zfs_sha256_transform_x64(s, d, b); 56 } 57 58 const sha256_ops_t sha256_x64_impl = { 59 .is_supported = sha2_is_supported, 60 .transform = tf_sha256_transform_x64, 61 .name = "x64" 62 }; 63 64 #if defined(HAVE_SSSE3) 65 static boolean_t sha2_have_ssse3(void) 66 { 67 return (kfpu_allowed() && zfs_ssse3_available()); 68 } 69 70 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3); 71 const sha256_ops_t sha256_ssse3_impl = { 72 .is_supported = sha2_have_ssse3, 73 .transform = tf_sha256_ssse3, 74 .name = "ssse3" 75 }; 76 #endif 77 78 #if defined(HAVE_AVX) 79 static boolean_t sha2_have_avx(void) 80 { 81 return (kfpu_allowed() && zfs_avx_available()); 82 } 83 84 TF(zfs_sha256_transform_avx, tf_sha256_avx); 85 const sha256_ops_t sha256_avx_impl = { 86 .is_supported = sha2_have_avx, 87 .transform = tf_sha256_avx, 88 .name = "avx" 89 }; 90 #endif 91 92 #if defined(HAVE_AVX2) 93 static boolean_t sha2_have_avx2(void) 94 { 95 return (kfpu_allowed() && zfs_avx2_available()); 96 } 97 98 TF(zfs_sha256_transform_avx2, tf_sha256_avx2); 99 const sha256_ops_t sha256_avx2_impl = { 100 .is_supported = sha2_have_avx2, 101 .transform = tf_sha256_avx2, 102 .name = "avx2" 103 }; 104 #endif 105 106 #if defined(HAVE_SSE4_1) 107 static boolean_t sha2_have_shani(void) 108 { 109 return (kfpu_allowed() && zfs_sse4_1_available() && \ 110 zfs_shani_available()); 111 } 112 113 TF(zfs_sha256_transform_shani, tf_sha256_shani); 114 const sha256_ops_t sha256_shani_impl = { 115 .is_supported = sha2_have_shani, 116 .transform = tf_sha256_shani, 117 .name = "shani" 118 }; 119 #endif 120 121 #elif defined(__aarch64__) || defined(__arm__) 122 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t); 123 const sha256_ops_t sha256_armv7_impl = { 124 .is_supported = sha2_is_supported, 125 .transform = zfs_sha256_block_armv7, 126 .name = "armv7" 127 }; 128 129 #if __ARM_ARCH > 6 130 static boolean_t sha256_have_neon(void) 131 { 132 return (kfpu_allowed() && zfs_neon_available()); 133 } 134 135 static boolean_t sha256_have_armv8ce(void) 136 { 137 return (kfpu_allowed() && zfs_sha256_available()); 138 } 139 140 TF(zfs_sha256_block_neon, tf_sha256_neon); 141 const sha256_ops_t sha256_neon_impl = { 142 .is_supported = sha256_have_neon, 143 .transform = tf_sha256_neon, 144 .name = "neon" 145 }; 146 147 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce); 148 const sha256_ops_t sha256_armv8_impl = { 149 .is_supported = sha256_have_armv8ce, 150 .transform = tf_sha256_armv8ce, 151 .name = "armv8-ce" 152 }; 153 #endif 154 155 #elif defined(__PPC64__) 156 static boolean_t sha256_have_isa207(void) 157 { 158 return (kfpu_allowed() && zfs_isa207_available()); 159 } 160 161 TF(zfs_sha256_ppc, tf_sha256_ppc); 162 const sha256_ops_t sha256_ppc_impl = { 163 .is_supported = sha2_is_supported, 164 .transform = tf_sha256_ppc, 165 .name = "ppc" 166 }; 167 168 TF(zfs_sha256_power8, tf_sha256_power8); 169 const sha256_ops_t sha256_power8_impl = { 170 .is_supported = sha256_have_isa207, 171 .transform = tf_sha256_power8, 172 .name = "power8" 173 }; 174 #endif /* __PPC64__ */ 175 176 /* the two generic ones */ 177 extern const sha256_ops_t sha256_generic_impl; 178 179 /* array with all sha256 implementations */ 180 static const sha256_ops_t *const sha256_impls[] = { 181 &sha256_generic_impl, 182 #if defined(__x86_64) 183 &sha256_x64_impl, 184 #endif 185 #if defined(__x86_64) && defined(HAVE_SSSE3) 186 &sha256_ssse3_impl, 187 #endif 188 #if defined(__x86_64) && defined(HAVE_AVX) 189 &sha256_avx_impl, 190 #endif 191 #if defined(__x86_64) && defined(HAVE_AVX2) 192 &sha256_avx2_impl, 193 #endif 194 #if defined(__x86_64) && defined(HAVE_SSE4_1) 195 &sha256_shani_impl, 196 #endif 197 #if defined(__aarch64__) || defined(__arm__) 198 &sha256_armv7_impl, 199 #if __ARM_ARCH > 6 200 &sha256_neon_impl, 201 &sha256_armv8_impl, 202 #endif 203 #endif 204 #if defined(__PPC64__) 205 &sha256_ppc_impl, 206 &sha256_power8_impl, 207 #endif /* __PPC64__ */ 208 }; 209 210 /* use the generic implementation functions */ 211 #define IMPL_NAME "sha256" 212 #define IMPL_OPS_T sha256_ops_t 213 #define IMPL_ARRAY sha256_impls 214 #define IMPL_GET_OPS sha256_get_ops 215 #define ZFS_IMPL_OPS zfs_sha256_ops 216 #include <generic_impl.c> 217 218 #ifdef _KERNEL 219 220 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 221 222 #if defined(__linux__) 223 224 static int 225 sha256_param_get(char *buffer, zfs_kernel_param_t *unused) 226 { 227 const uint32_t impl = IMPL_READ(generic_impl_chosen); 228 char *fmt; 229 int cnt = 0; 230 231 /* cycling */ 232 fmt = IMPL_FMT(impl, IMPL_CYCLE); 233 cnt += sprintf(buffer + cnt, fmt, "cycle"); 234 235 /* list fastest */ 236 fmt = IMPL_FMT(impl, IMPL_FASTEST); 237 cnt += sprintf(buffer + cnt, fmt, "fastest"); 238 239 /* list all supported implementations */ 240 generic_impl_init(); 241 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 242 fmt = IMPL_FMT(impl, i); 243 cnt += sprintf(buffer + cnt, fmt, 244 generic_supp_impls[i]->name); 245 } 246 247 return (cnt); 248 } 249 250 static int 251 sha256_param_set(const char *val, zfs_kernel_param_t *unused) 252 { 253 (void) unused; 254 return (generic_impl_setname(val)); 255 } 256 257 #elif defined(__FreeBSD__) 258 259 #include <sys/sbuf.h> 260 261 static int 262 sha256_param(ZFS_MODULE_PARAM_ARGS) 263 { 264 int err; 265 266 generic_impl_init(); 267 if (req->newptr == NULL) { 268 const uint32_t impl = IMPL_READ(generic_impl_chosen); 269 const int init_buflen = 64; 270 const char *fmt; 271 struct sbuf *s; 272 273 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 274 275 /* cycling */ 276 fmt = IMPL_FMT(impl, IMPL_CYCLE); 277 (void) sbuf_printf(s, fmt, "cycle"); 278 279 /* list fastest */ 280 fmt = IMPL_FMT(impl, IMPL_FASTEST); 281 (void) sbuf_printf(s, fmt, "fastest"); 282 283 /* list all supported implementations */ 284 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 285 fmt = IMPL_FMT(impl, i); 286 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 287 } 288 289 err = sbuf_finish(s); 290 sbuf_delete(s); 291 292 return (err); 293 } 294 295 char buf[16]; 296 297 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 298 if (err) { 299 return (err); 300 } 301 302 return (-generic_impl_setname(buf)); 303 } 304 #endif 305 306 #undef IMPL_FMT 307 308 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl, 309 sha256_param_set, sha256_param_get, ZMOD_RW, \ 310 "Select SHA256 implementation."); 311 #endif 312 313 #undef TF 314