1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 25 */ 26 27 #include <sys/simd.h> 28 #include <sys/zfs_context.h> 29 #include <sys/zfs_impl.h> 30 #include <sys/sha2.h> 31 32 #include <sha2/sha2_impl.h> 33 #include <sys/asm_linkage.h> 34 35 #define TF(E, N) \ 36 extern void ASMABI E(uint64_t s[8], const void *, size_t); \ 37 static inline void N(uint64_t s[8], const void *d, size_t b) { \ 38 kfpu_begin(); E(s, d, b); kfpu_end(); \ 39 } 40 41 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \ 42 defined(__aarch64__) || defined(__arm__) || defined(__PPC64__) 43 /* some implementation is always okay */ 44 static inline boolean_t sha2_is_supported(void) 45 { 46 return (B_TRUE); 47 } 48 #endif 49 50 #if defined(__x86_64) 51 52 /* Users of ASMABI requires all calls to be from wrappers */ 53 extern void ASMABI 54 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t); 55 56 static inline void 57 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b) 58 { 59 zfs_sha512_transform_x64(s, d, b); 60 } 61 const sha512_ops_t sha512_x64_impl = { 62 .is_supported = sha2_is_supported, 63 .transform = tf_sha512_transform_x64, 64 .name = "x64" 65 }; 66 67 #if defined(HAVE_AVX) 68 static boolean_t sha2_have_avx(void) 69 { 70 return (kfpu_allowed() && zfs_avx_available()); 71 } 72 73 TF(zfs_sha512_transform_avx, tf_sha512_avx); 74 const sha512_ops_t sha512_avx_impl = { 75 .is_supported = sha2_have_avx, 76 .transform = tf_sha512_avx, 77 .name = "avx" 78 }; 79 #endif 80 81 #if defined(HAVE_AVX2) 82 static boolean_t sha2_have_avx2(void) 83 { 84 return (kfpu_allowed() && zfs_avx2_available()); 85 } 86 87 TF(zfs_sha512_transform_avx2, tf_sha512_avx2); 88 const sha512_ops_t sha512_avx2_impl = { 89 .is_supported = sha2_have_avx2, 90 .transform = tf_sha512_avx2, 91 .name = "avx2" 92 }; 93 #endif 94 95 #elif defined(__aarch64__) || defined(__arm__) 96 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t); 97 const sha512_ops_t sha512_armv7_impl = { 98 .is_supported = sha2_is_supported, 99 .transform = zfs_sha512_block_armv7, 100 .name = "armv7" 101 }; 102 103 #if defined(__aarch64__) 104 static boolean_t sha512_have_armv8ce(void) 105 { 106 return (kfpu_allowed() && zfs_sha512_available()); 107 } 108 109 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce); 110 const sha512_ops_t sha512_armv8_impl = { 111 .is_supported = sha512_have_armv8ce, 112 .transform = tf_sha512_armv8ce, 113 .name = "armv8-ce" 114 }; 115 #endif 116 117 #if defined(__arm__) && __ARM_ARCH > 6 118 static boolean_t sha512_have_neon(void) 119 { 120 return (kfpu_allowed() && zfs_neon_available()); 121 } 122 123 TF(zfs_sha512_block_neon, tf_sha512_neon); 124 const sha512_ops_t sha512_neon_impl = { 125 .is_supported = sha512_have_neon, 126 .transform = tf_sha512_neon, 127 .name = "neon" 128 }; 129 #endif 130 131 #elif defined(__PPC64__) 132 TF(zfs_sha512_ppc, tf_sha512_ppc); 133 const sha512_ops_t sha512_ppc_impl = { 134 .is_supported = sha2_is_supported, 135 .transform = tf_sha512_ppc, 136 .name = "ppc" 137 }; 138 139 static boolean_t sha512_have_isa207(void) 140 { 141 return (kfpu_allowed() && zfs_isa207_available()); 142 } 143 144 TF(zfs_sha512_power8, tf_sha512_power8); 145 const sha512_ops_t sha512_power8_impl = { 146 .is_supported = sha512_have_isa207, 147 .transform = tf_sha512_power8, 148 .name = "power8" 149 }; 150 #endif /* __PPC64__ */ 151 152 /* the two generic ones */ 153 extern const sha512_ops_t sha512_generic_impl; 154 155 /* array with all sha512 implementations */ 156 static const sha512_ops_t *const sha512_impls[] = { 157 &sha512_generic_impl, 158 #if defined(__x86_64) 159 &sha512_x64_impl, 160 #endif 161 #if defined(__x86_64) && defined(HAVE_AVX) 162 &sha512_avx_impl, 163 #endif 164 #if defined(__x86_64) && defined(HAVE_AVX2) 165 &sha512_avx2_impl, 166 #endif 167 #if defined(__aarch64__) || defined(__arm__) 168 &sha512_armv7_impl, 169 #if defined(__aarch64__) 170 &sha512_armv8_impl, 171 #endif 172 #if defined(__arm__) && __ARM_ARCH > 6 173 &sha512_neon_impl, 174 #endif 175 #endif 176 #if defined(__PPC64__) 177 &sha512_ppc_impl, 178 &sha512_power8_impl, 179 #endif /* __PPC64__ */ 180 }; 181 182 /* use the generic implementation functions */ 183 #define IMPL_NAME "sha512" 184 #define IMPL_OPS_T sha512_ops_t 185 #define IMPL_ARRAY sha512_impls 186 #define IMPL_GET_OPS sha512_get_ops 187 #define ZFS_IMPL_OPS zfs_sha512_ops 188 #include <generic_impl.c> 189 190 #ifdef _KERNEL 191 192 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 193 194 #if defined(__linux__) 195 196 static int 197 sha512_param_get(char *buffer, zfs_kernel_param_t *unused) 198 { 199 const uint32_t impl = IMPL_READ(generic_impl_chosen); 200 char *fmt; 201 int cnt = 0; 202 203 /* cycling */ 204 fmt = IMPL_FMT(impl, IMPL_CYCLE); 205 cnt += sprintf(buffer + cnt, fmt, "cycle"); 206 207 /* list fastest */ 208 fmt = IMPL_FMT(impl, IMPL_FASTEST); 209 cnt += sprintf(buffer + cnt, fmt, "fastest"); 210 211 /* list all supported implementations */ 212 generic_impl_init(); 213 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 214 fmt = IMPL_FMT(impl, i); 215 cnt += sprintf(buffer + cnt, fmt, 216 generic_supp_impls[i]->name); 217 } 218 219 return (cnt); 220 } 221 222 static int 223 sha512_param_set(const char *val, zfs_kernel_param_t *unused) 224 { 225 (void) unused; 226 return (generic_impl_setname(val)); 227 } 228 229 #elif defined(__FreeBSD__) 230 231 #include <sys/sbuf.h> 232 233 static int 234 sha512_param(ZFS_MODULE_PARAM_ARGS) 235 { 236 int err; 237 238 generic_impl_init(); 239 if (req->newptr == NULL) { 240 const uint32_t impl = IMPL_READ(generic_impl_chosen); 241 const int init_buflen = 64; 242 const char *fmt; 243 struct sbuf *s; 244 245 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 246 247 /* cycling */ 248 fmt = IMPL_FMT(impl, IMPL_CYCLE); 249 (void) sbuf_printf(s, fmt, "cycle"); 250 251 /* list fastest */ 252 fmt = IMPL_FMT(impl, IMPL_FASTEST); 253 (void) sbuf_printf(s, fmt, "fastest"); 254 255 /* list all supported implementations */ 256 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 257 fmt = IMPL_FMT(impl, i); 258 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 259 } 260 261 err = sbuf_finish(s); 262 sbuf_delete(s); 263 264 return (err); 265 } 266 267 /* we got module parameter */ 268 char buf[16]; 269 270 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 271 if (err) { 272 return (err); 273 } 274 275 return (-generic_impl_setname(buf)); 276 } 277 #endif 278 279 #undef IMPL_FMT 280 281 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl, 282 sha512_param_set, sha512_param_get, ZMOD_RW, \ 283 "Select SHA512 implementation."); 284 #endif 285 286 #undef TF 287