1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 25 */ 26 27 #include <sys/simd.h> 28 #include <sys/zfs_context.h> 29 #include <sys/zfs_impl.h> 30 #include <sys/sha2.h> 31 32 #include <sha2/sha2_impl.h> 33 #include <sys/asm_linkage.h> 34 35 #define TF(E, N) \ 36 extern void ASMABI E(uint64_t s[8], const void *, size_t); \ 37 static inline void N(uint64_t s[8], const void *d, size_t b) { \ 38 kfpu_begin(); E(s, d, b); kfpu_end(); \ 39 } 40 41 /* some implementation is always okay */ 42 static inline boolean_t sha2_is_supported(void) 43 { 44 return (B_TRUE); 45 } 46 47 #if defined(__x86_64) 48 49 /* Users of ASMABI requires all calls to be from wrappers */ 50 extern void ASMABI 51 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t); 52 53 static inline void 54 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b) 55 { 56 zfs_sha512_transform_x64(s, d, b); 57 } 58 const sha512_ops_t sha512_x64_impl = { 59 .is_supported = sha2_is_supported, 60 .transform = tf_sha512_transform_x64, 61 .name = "x64" 62 }; 63 64 #if defined(HAVE_AVX) 65 static boolean_t sha2_have_avx(void) 66 { 67 return (kfpu_allowed() && zfs_avx_available()); 68 } 69 70 TF(zfs_sha512_transform_avx, tf_sha512_avx); 71 const sha512_ops_t sha512_avx_impl = { 72 .is_supported = sha2_have_avx, 73 .transform = tf_sha512_avx, 74 .name = "avx" 75 }; 76 #endif 77 78 #if defined(HAVE_AVX2) 79 static boolean_t sha2_have_avx2(void) 80 { 81 return (kfpu_allowed() && zfs_avx2_available()); 82 } 83 84 TF(zfs_sha512_transform_avx2, tf_sha512_avx2); 85 const sha512_ops_t sha512_avx2_impl = { 86 .is_supported = sha2_have_avx2, 87 .transform = tf_sha512_avx2, 88 .name = "avx2" 89 }; 90 #endif 91 92 #elif defined(__aarch64__) || defined(__arm__) 93 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t); 94 const sha512_ops_t sha512_armv7_impl = { 95 .is_supported = sha2_is_supported, 96 .transform = zfs_sha512_block_armv7, 97 .name = "armv7" 98 }; 99 100 #if defined(__aarch64__) 101 static boolean_t sha512_have_armv8ce(void) 102 { 103 return (kfpu_allowed() && zfs_sha512_available()); 104 } 105 106 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce); 107 const sha512_ops_t sha512_armv8_impl = { 108 .is_supported = sha512_have_armv8ce, 109 .transform = tf_sha512_armv8ce, 110 .name = "armv8-ce" 111 }; 112 #endif 113 114 #if defined(__arm__) && __ARM_ARCH > 6 115 static boolean_t sha512_have_neon(void) 116 { 117 return (kfpu_allowed() && zfs_neon_available()); 118 } 119 120 TF(zfs_sha512_block_neon, tf_sha512_neon); 121 const sha512_ops_t sha512_neon_impl = { 122 .is_supported = sha512_have_neon, 123 .transform = tf_sha512_neon, 124 .name = "neon" 125 }; 126 #endif 127 128 #elif defined(__PPC64__) 129 TF(zfs_sha512_ppc, tf_sha512_ppc); 130 const sha512_ops_t sha512_ppc_impl = { 131 .is_supported = sha2_is_supported, 132 .transform = tf_sha512_ppc, 133 .name = "ppc" 134 }; 135 136 static boolean_t sha512_have_isa207(void) 137 { 138 return (kfpu_allowed() && zfs_isa207_available()); 139 } 140 141 TF(zfs_sha512_power8, tf_sha512_power8); 142 const sha512_ops_t sha512_power8_impl = { 143 .is_supported = sha512_have_isa207, 144 .transform = tf_sha512_power8, 145 .name = "power8" 146 }; 147 #endif /* __PPC64__ */ 148 149 /* the two generic ones */ 150 extern const sha512_ops_t sha512_generic_impl; 151 152 /* array with all sha512 implementations */ 153 static const sha512_ops_t *const sha512_impls[] = { 154 &sha512_generic_impl, 155 #if defined(__x86_64) 156 &sha512_x64_impl, 157 #endif 158 #if defined(__x86_64) && defined(HAVE_AVX) 159 &sha512_avx_impl, 160 #endif 161 #if defined(__x86_64) && defined(HAVE_AVX2) 162 &sha512_avx2_impl, 163 #endif 164 #if defined(__aarch64__) || defined(__arm__) 165 &sha512_armv7_impl, 166 #if defined(__aarch64__) 167 &sha512_armv8_impl, 168 #endif 169 #if defined(__arm__) && __ARM_ARCH > 6 170 &sha512_neon_impl, 171 #endif 172 #endif 173 #if defined(__PPC64__) 174 &sha512_ppc_impl, 175 &sha512_power8_impl, 176 #endif /* __PPC64__ */ 177 }; 178 179 /* use the generic implementation functions */ 180 #define IMPL_NAME "sha512" 181 #define IMPL_OPS_T sha512_ops_t 182 #define IMPL_ARRAY sha512_impls 183 #define IMPL_GET_OPS sha512_get_ops 184 #define ZFS_IMPL_OPS zfs_sha512_ops 185 #include <generic_impl.c> 186 187 #ifdef _KERNEL 188 189 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 190 191 #if defined(__linux__) 192 193 static int 194 sha512_param_get(char *buffer, zfs_kernel_param_t *unused) 195 { 196 const uint32_t impl = IMPL_READ(generic_impl_chosen); 197 char *fmt; 198 int cnt = 0; 199 200 /* cycling */ 201 fmt = IMPL_FMT(impl, IMPL_CYCLE); 202 cnt += sprintf(buffer + cnt, fmt, "cycle"); 203 204 /* list fastest */ 205 fmt = IMPL_FMT(impl, IMPL_FASTEST); 206 cnt += sprintf(buffer + cnt, fmt, "fastest"); 207 208 /* list all supported implementations */ 209 generic_impl_init(); 210 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 211 fmt = IMPL_FMT(impl, i); 212 cnt += sprintf(buffer + cnt, fmt, 213 generic_supp_impls[i]->name); 214 } 215 216 return (cnt); 217 } 218 219 static int 220 sha512_param_set(const char *val, zfs_kernel_param_t *unused) 221 { 222 (void) unused; 223 return (generic_impl_setname(val)); 224 } 225 226 #elif defined(__FreeBSD__) 227 228 #include <sys/sbuf.h> 229 230 static int 231 sha512_param(ZFS_MODULE_PARAM_ARGS) 232 { 233 int err; 234 235 generic_impl_init(); 236 if (req->newptr == NULL) { 237 const uint32_t impl = IMPL_READ(generic_impl_chosen); 238 const int init_buflen = 64; 239 const char *fmt; 240 struct sbuf *s; 241 242 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 243 244 /* cycling */ 245 fmt = IMPL_FMT(impl, IMPL_CYCLE); 246 (void) sbuf_printf(s, fmt, "cycle"); 247 248 /* list fastest */ 249 fmt = IMPL_FMT(impl, IMPL_FASTEST); 250 (void) sbuf_printf(s, fmt, "fastest"); 251 252 /* list all supported implementations */ 253 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 254 fmt = IMPL_FMT(impl, i); 255 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 256 } 257 258 err = sbuf_finish(s); 259 sbuf_delete(s); 260 261 return (err); 262 } 263 264 /* we got module parameter */ 265 char buf[16]; 266 267 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 268 if (err) { 269 return (err); 270 } 271 272 return (-generic_impl_setname(buf)); 273 } 274 #endif 275 276 #undef IMPL_FMT 277 278 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl, 279 sha512_param_set, sha512_param_get, ZMOD_RW, \ 280 "Select SHA512 implementation."); 281 #endif 282 283 #undef TF 284