1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 24 */ 25 26 #include <sys/simd.h> 27 #include <sys/zfs_context.h> 28 #include <sys/zfs_impl.h> 29 #include <sys/sha2.h> 30 31 #include <sha2/sha2_impl.h> 32 #include <sys/asm_linkage.h> 33 34 #define TF(E, N) \ 35 extern void ASMABI E(uint32_t s[8], const void *, size_t); \ 36 static inline void N(uint32_t s[8], const void *d, size_t b) { \ 37 kfpu_begin(); E(s, d, b); kfpu_end(); \ 38 } 39 40 /* some implementation is always okay */ 41 static inline boolean_t sha2_is_supported(void) 42 { 43 return (B_TRUE); 44 } 45 46 #if defined(__x86_64) 47 48 /* Users of ASMABI requires all calls to be from wrappers */ 49 extern void ASMABI 50 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t); 51 52 static inline void 53 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b) 54 { 55 zfs_sha256_transform_x64(s, d, b); 56 } 57 58 const sha256_ops_t sha256_x64_impl = { 59 .is_supported = sha2_is_supported, 60 .transform = tf_sha256_transform_x64, 61 .name = "x64" 62 }; 63 64 #if defined(HAVE_SSSE3) 65 static boolean_t sha2_have_ssse3(void) 66 { 67 return (kfpu_allowed() && zfs_ssse3_available()); 68 } 69 70 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3); 71 const sha256_ops_t sha256_ssse3_impl = { 72 .is_supported = sha2_have_ssse3, 73 .transform = tf_sha256_ssse3, 74 .name = "ssse3" 75 }; 76 #endif 77 78 #if defined(HAVE_AVX) 79 static boolean_t sha2_have_avx(void) 80 { 81 return (kfpu_allowed() && zfs_avx_available()); 82 } 83 84 TF(zfs_sha256_transform_avx, tf_sha256_avx); 85 const sha256_ops_t sha256_avx_impl = { 86 .is_supported = sha2_have_avx, 87 .transform = tf_sha256_avx, 88 .name = "avx" 89 }; 90 #endif 91 92 #if defined(HAVE_AVX2) 93 static boolean_t sha2_have_avx2(void) 94 { 95 return (kfpu_allowed() && zfs_avx2_available()); 96 } 97 98 TF(zfs_sha256_transform_avx2, tf_sha256_avx2); 99 const sha256_ops_t sha256_avx2_impl = { 100 .is_supported = sha2_have_avx2, 101 .transform = tf_sha256_avx2, 102 .name = "avx2" 103 }; 104 #endif 105 106 #if defined(HAVE_SSE4_1) 107 static boolean_t sha2_have_shani(void) 108 { 109 return (kfpu_allowed() && zfs_sse4_1_available() && \ 110 zfs_shani_available()); 111 } 112 113 TF(zfs_sha256_transform_shani, tf_sha256_shani); 114 const sha256_ops_t sha256_shani_impl = { 115 .is_supported = sha2_have_shani, 116 .transform = tf_sha256_shani, 117 .name = "shani" 118 }; 119 #endif 120 121 #elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH > 6) 122 static boolean_t sha256_have_neon(void) 123 { 124 return (kfpu_allowed() && zfs_neon_available()); 125 } 126 127 static boolean_t sha256_have_armv8ce(void) 128 { 129 return (kfpu_allowed() && zfs_sha256_available()); 130 } 131 132 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t); 133 const sha256_ops_t sha256_armv7_impl = { 134 .is_supported = sha2_is_supported, 135 .transform = zfs_sha256_block_armv7, 136 .name = "armv7" 137 }; 138 139 TF(zfs_sha256_block_neon, tf_sha256_neon); 140 const sha256_ops_t sha256_neon_impl = { 141 .is_supported = sha256_have_neon, 142 .transform = tf_sha256_neon, 143 .name = "neon" 144 }; 145 146 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce); 147 const sha256_ops_t sha256_armv8_impl = { 148 .is_supported = sha256_have_armv8ce, 149 .transform = tf_sha256_armv8ce, 150 .name = "armv8-ce" 151 }; 152 153 #elif defined(__PPC64__) 154 static boolean_t sha256_have_isa207(void) 155 { 156 return (kfpu_allowed() && zfs_isa207_available()); 157 } 158 159 TF(zfs_sha256_ppc, tf_sha256_ppc); 160 const sha256_ops_t sha256_ppc_impl = { 161 .is_supported = sha2_is_supported, 162 .transform = tf_sha256_ppc, 163 .name = "ppc" 164 }; 165 166 TF(zfs_sha256_power8, tf_sha256_power8); 167 const sha256_ops_t sha256_power8_impl = { 168 .is_supported = sha256_have_isa207, 169 .transform = tf_sha256_power8, 170 .name = "power8" 171 }; 172 #endif /* __PPC64__ */ 173 174 /* the two generic ones */ 175 extern const sha256_ops_t sha256_generic_impl; 176 177 /* array with all sha256 implementations */ 178 static const sha256_ops_t *const sha256_impls[] = { 179 &sha256_generic_impl, 180 #if defined(__x86_64) 181 &sha256_x64_impl, 182 #endif 183 #if defined(__x86_64) && defined(HAVE_SSSE3) 184 &sha256_ssse3_impl, 185 #endif 186 #if defined(__x86_64) && defined(HAVE_AVX) 187 &sha256_avx_impl, 188 #endif 189 #if defined(__x86_64) && defined(HAVE_AVX2) 190 &sha256_avx2_impl, 191 #endif 192 #if defined(__x86_64) && defined(HAVE_SSE4_1) 193 &sha256_shani_impl, 194 #endif 195 #if defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH > 6) 196 &sha256_armv7_impl, 197 &sha256_neon_impl, 198 &sha256_armv8_impl, 199 #endif 200 #if defined(__PPC64__) 201 &sha256_ppc_impl, 202 &sha256_power8_impl, 203 #endif /* __PPC64__ */ 204 }; 205 206 /* use the generic implementation functions */ 207 #define IMPL_NAME "sha256" 208 #define IMPL_OPS_T sha256_ops_t 209 #define IMPL_ARRAY sha256_impls 210 #define IMPL_GET_OPS sha256_get_ops 211 #define ZFS_IMPL_OPS zfs_sha256_ops 212 #include <generic_impl.c> 213 214 #ifdef _KERNEL 215 216 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 217 218 #if defined(__linux__) 219 220 static int 221 sha256_param_get(char *buffer, zfs_kernel_param_t *unused) 222 { 223 const uint32_t impl = IMPL_READ(generic_impl_chosen); 224 char *fmt; 225 int cnt = 0; 226 227 /* cycling */ 228 fmt = IMPL_FMT(impl, IMPL_CYCLE); 229 cnt += sprintf(buffer + cnt, fmt, "cycle"); 230 231 /* list fastest */ 232 fmt = IMPL_FMT(impl, IMPL_FASTEST); 233 cnt += sprintf(buffer + cnt, fmt, "fastest"); 234 235 /* list all supported implementations */ 236 generic_impl_init(); 237 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 238 fmt = IMPL_FMT(impl, i); 239 cnt += sprintf(buffer + cnt, fmt, 240 generic_supp_impls[i]->name); 241 } 242 243 return (cnt); 244 } 245 246 static int 247 sha256_param_set(const char *val, zfs_kernel_param_t *unused) 248 { 249 (void) unused; 250 return (generic_impl_setname(val)); 251 } 252 253 #elif defined(__FreeBSD__) 254 255 #include <sys/sbuf.h> 256 257 static int 258 sha256_param(ZFS_MODULE_PARAM_ARGS) 259 { 260 int err; 261 262 generic_impl_init(); 263 if (req->newptr == NULL) { 264 const uint32_t impl = IMPL_READ(generic_impl_chosen); 265 const int init_buflen = 64; 266 const char *fmt; 267 struct sbuf *s; 268 269 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 270 271 /* cycling */ 272 fmt = IMPL_FMT(impl, IMPL_CYCLE); 273 (void) sbuf_printf(s, fmt, "cycle"); 274 275 /* list fastest */ 276 fmt = IMPL_FMT(impl, IMPL_FASTEST); 277 (void) sbuf_printf(s, fmt, "fastest"); 278 279 /* list all supported implementations */ 280 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 281 fmt = IMPL_FMT(impl, i); 282 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 283 } 284 285 err = sbuf_finish(s); 286 sbuf_delete(s); 287 288 return (err); 289 } 290 291 char buf[16]; 292 293 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 294 if (err) { 295 return (err); 296 } 297 298 return (-generic_impl_setname(buf)); 299 } 300 #endif 301 302 #undef IMPL_FMT 303 304 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl, 305 sha256_param_set, sha256_param_get, ZMOD_RW, \ 306 "Select SHA256 implementation."); 307 #endif 308 309 #undef TF 310