1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 24 */ 25 26 #include <sys/simd.h> 27 #include <sys/zfs_context.h> 28 #include <sys/zfs_impl.h> 29 #include <sys/sha2.h> 30 31 #include <sha2/sha2_impl.h> 32 #include <sys/asm_linkage.h> 33 34 #define TF(E, N) \ 35 extern void ASMABI E(uint64_t s[8], const void *, size_t); \ 36 static inline void N(uint64_t s[8], const void *d, size_t b) { \ 37 kfpu_begin(); E(s, d, b); kfpu_end(); \ 38 } 39 40 /* some implementation is always okay */ 41 static inline boolean_t sha2_is_supported(void) 42 { 43 return (B_TRUE); 44 } 45 46 #if defined(__x86_64) 47 48 /* Users of ASMABI requires all calls to be from wrappers */ 49 extern void ASMABI 50 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t); 51 52 static inline void 53 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b) 54 { 55 zfs_sha512_transform_x64(s, d, b); 56 } 57 const sha512_ops_t sha512_x64_impl = { 58 .is_supported = sha2_is_supported, 59 .transform = tf_sha512_transform_x64, 60 .name = "x64" 61 }; 62 63 #if defined(HAVE_AVX) 64 static boolean_t sha2_have_avx(void) 65 { 66 return (kfpu_allowed() && zfs_avx_available()); 67 } 68 69 TF(zfs_sha512_transform_avx, tf_sha512_avx); 70 const sha512_ops_t sha512_avx_impl = { 71 .is_supported = sha2_have_avx, 72 .transform = tf_sha512_avx, 73 .name = "avx" 74 }; 75 #endif 76 77 #if defined(HAVE_AVX2) 78 static boolean_t sha2_have_avx2(void) 79 { 80 return (kfpu_allowed() && zfs_avx2_available()); 81 } 82 83 TF(zfs_sha512_transform_avx2, tf_sha512_avx2); 84 const sha512_ops_t sha512_avx2_impl = { 85 .is_supported = sha2_have_avx2, 86 .transform = tf_sha512_avx2, 87 .name = "avx2" 88 }; 89 #endif 90 91 #elif defined(__aarch64__) 92 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t); 93 const sha512_ops_t sha512_armv7_impl = { 94 .is_supported = sha2_is_supported, 95 .transform = zfs_sha512_block_armv7, 96 .name = "armv7" 97 }; 98 99 static boolean_t sha512_have_armv8ce(void) 100 { 101 return (kfpu_allowed() && zfs_sha512_available()); 102 } 103 104 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce); 105 const sha512_ops_t sha512_armv8_impl = { 106 .is_supported = sha512_have_armv8ce, 107 .transform = tf_sha512_armv8ce, 108 .name = "armv8-ce" 109 }; 110 111 #elif defined(__arm__) && __ARM_ARCH > 6 112 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t); 113 const sha512_ops_t sha512_armv7_impl = { 114 .is_supported = sha2_is_supported, 115 .transform = zfs_sha512_block_armv7, 116 .name = "armv7" 117 }; 118 119 static boolean_t sha512_have_neon(void) 120 { 121 return (kfpu_allowed() && zfs_neon_available()); 122 } 123 124 TF(zfs_sha512_block_neon, tf_sha512_neon); 125 const sha512_ops_t sha512_neon_impl = { 126 .is_supported = sha512_have_neon, 127 .transform = tf_sha512_neon, 128 .name = "neon" 129 }; 130 131 #elif defined(__PPC64__) 132 TF(zfs_sha512_ppc, tf_sha512_ppc); 133 const sha512_ops_t sha512_ppc_impl = { 134 .is_supported = sha2_is_supported, 135 .transform = tf_sha512_ppc, 136 .name = "ppc" 137 }; 138 139 static boolean_t sha512_have_isa207(void) 140 { 141 return (kfpu_allowed() && zfs_isa207_available()); 142 } 143 144 TF(zfs_sha512_power8, tf_sha512_power8); 145 const sha512_ops_t sha512_power8_impl = { 146 .is_supported = sha512_have_isa207, 147 .transform = tf_sha512_power8, 148 .name = "power8" 149 }; 150 #endif /* __PPC64__ */ 151 152 /* the two generic ones */ 153 extern const sha512_ops_t sha512_generic_impl; 154 155 /* array with all sha512 implementations */ 156 static const sha512_ops_t *const sha512_impls[] = { 157 &sha512_generic_impl, 158 #if defined(__x86_64) 159 &sha512_x64_impl, 160 #endif 161 #if defined(__x86_64) && defined(HAVE_AVX) 162 &sha512_avx_impl, 163 #endif 164 #if defined(__x86_64) && defined(HAVE_AVX2) 165 &sha512_avx2_impl, 166 #endif 167 #if defined(__aarch64__) 168 &sha512_armv7_impl, 169 &sha512_armv8_impl, 170 #endif 171 #if defined(__arm__) && __ARM_ARCH > 6 172 &sha512_armv7_impl, 173 &sha512_neon_impl, 174 #endif 175 #if defined(__PPC64__) 176 &sha512_ppc_impl, 177 &sha512_power8_impl, 178 #endif /* __PPC64__ */ 179 }; 180 181 /* use the generic implementation functions */ 182 #define IMPL_NAME "sha512" 183 #define IMPL_OPS_T sha512_ops_t 184 #define IMPL_ARRAY sha512_impls 185 #define IMPL_GET_OPS sha512_get_ops 186 #define ZFS_IMPL_OPS zfs_sha512_ops 187 #include <generic_impl.c> 188 189 #ifdef _KERNEL 190 191 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 192 193 #if defined(__linux__) 194 195 static int 196 sha512_param_get(char *buffer, zfs_kernel_param_t *unused) 197 { 198 const uint32_t impl = IMPL_READ(generic_impl_chosen); 199 char *fmt; 200 int cnt = 0; 201 202 /* cycling */ 203 fmt = IMPL_FMT(impl, IMPL_CYCLE); 204 cnt += sprintf(buffer + cnt, fmt, "cycle"); 205 206 /* list fastest */ 207 fmt = IMPL_FMT(impl, IMPL_FASTEST); 208 cnt += sprintf(buffer + cnt, fmt, "fastest"); 209 210 /* list all supported implementations */ 211 generic_impl_init(); 212 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 213 fmt = IMPL_FMT(impl, i); 214 cnt += sprintf(buffer + cnt, fmt, 215 generic_supp_impls[i]->name); 216 } 217 218 return (cnt); 219 } 220 221 static int 222 sha512_param_set(const char *val, zfs_kernel_param_t *unused) 223 { 224 (void) unused; 225 return (generic_impl_setname(val)); 226 } 227 228 #elif defined(__FreeBSD__) 229 230 #include <sys/sbuf.h> 231 232 static int 233 sha512_param(ZFS_MODULE_PARAM_ARGS) 234 { 235 int err; 236 237 generic_impl_init(); 238 if (req->newptr == NULL) { 239 const uint32_t impl = IMPL_READ(generic_impl_chosen); 240 const int init_buflen = 64; 241 const char *fmt; 242 struct sbuf *s; 243 244 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 245 246 /* cycling */ 247 fmt = IMPL_FMT(impl, IMPL_CYCLE); 248 (void) sbuf_printf(s, fmt, "cycle"); 249 250 /* list fastest */ 251 fmt = IMPL_FMT(impl, IMPL_FASTEST); 252 (void) sbuf_printf(s, fmt, "fastest"); 253 254 /* list all supported implementations */ 255 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 256 fmt = IMPL_FMT(impl, i); 257 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 258 } 259 260 err = sbuf_finish(s); 261 sbuf_delete(s); 262 263 return (err); 264 } 265 266 /* we got module parameter */ 267 char buf[16]; 268 269 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 270 if (err) { 271 return (err); 272 } 273 274 return (-generic_impl_setname(buf)); 275 } 276 #endif 277 278 #undef IMPL_FMT 279 280 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl, 281 sha512_param_set, sha512_param_get, ZMOD_RW, \ 282 "Select SHA512 implementation."); 283 #endif 284 285 #undef TF 286