1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020 Joyent, Inc. 14 * Copyright 2024 Oxide Computer Company 15 */ 16 17 #ifndef _SIMD_H 18 #define _SIMD_H 19 20 #if defined(__amd64__) || defined(__i386__) 21 22 #define kfpu_initialize(tsk) do {} while (0) 23 #define kfpu_init() (0) 24 #define kfpu_fini() do {} while (0) 25 26 #ifdef _KERNEL 27 #include <sys/x86_archext.h> 28 #include <sys/archsystm.h> 29 #include <sys/systm.h> 30 #include <sys/kfpu.h> 31 #include <sys/proc.h> 32 #include <sys/disp.h> 33 #include <sys/cpuvar.h> 34 35 static inline int 36 kfpu_allowed(void) 37 { 38 extern int zfs_fpu_enabled; 39 40 /* 41 * When panicking, play it safe and avoid kfpu use. This gives the best 42 * chance of being able to dump successfully, particularly if the panic 43 * occured around an FPU context switch. 44 */ 45 if (panicstr != NULL) 46 return (0); 47 48 return (zfs_fpu_enabled != 0 ? 1 : 0); 49 } 50 51 static inline void 52 kfpu_begin(void) 53 { 54 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) { 55 kernel_fpu_begin(NULL, KFPU_USE_LWP); 56 } else { 57 kpreempt_disable(); 58 kernel_fpu_begin(NULL, KFPU_NO_STATE); 59 } 60 } 61 62 static inline void 63 kfpu_end(void) 64 { 65 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) { 66 kernel_fpu_end(NULL, KFPU_USE_LWP); 67 } else { 68 kernel_fpu_end(NULL, KFPU_NO_STATE); 69 kpreempt_enable(); 70 } 71 } 72 73 /* 74 * Check if various vector instruction sets are available. 75 */ 76 77 static inline boolean_t 78 zfs_sse_available(void) 79 { 80 return (is_x86_feature(x86_featureset, X86FSET_SSE)); 81 } 82 83 static inline boolean_t 84 zfs_sse2_available(void) 85 { 86 return (is_x86_feature(x86_featureset, X86FSET_SSE2)); 87 } 88 89 static inline boolean_t 90 zfs_sse3_available(void) 91 { 92 return (is_x86_feature(x86_featureset, X86FSET_SSE3)); 93 } 94 95 static inline boolean_t 96 zfs_ssse3_available(void) 97 { 98 return (is_x86_feature(x86_featureset, X86FSET_SSSE3)); 99 } 100 101 static inline boolean_t 102 zfs_avx_available(void) 103 { 104 return (is_x86_feature(x86_featureset, X86FSET_AVX)); 105 } 106 107 static inline boolean_t 108 zfs_avx2_available(void) 109 { 110 return (is_x86_feature(x86_featureset, X86FSET_AVX2)); 111 } 112 113 static inline boolean_t 114 zfs_avx512f_available(void) 115 { 116 return (is_x86_feature(x86_featureset, X86FSET_AVX512F)); 117 } 118 119 static inline boolean_t 120 zfs_avx512bw_available(void) 121 { 122 return (is_x86_feature(x86_featureset, X86FSET_AVX512BW)); 123 } 124 125 #else /* ! _KERNEL */ 126 127 #include <sys/auxv.h> 128 #include <sys/auxv_386.h> 129 130 #define kfpu_allowed() 1 131 #define kfpu_begin() do {} while (0) 132 #define kfpu_end() do {} while (0) 133 134 /* 135 * User-level check if various vector instruction sets are available. 136 */ 137 138 static inline boolean_t 139 zfs_sse_available(void) 140 { 141 uint32_t u = 0; 142 143 (void) getisax(&u, 1); 144 return ((u & AV_386_SSE) != 0); 145 } 146 147 static inline boolean_t 148 zfs_sse2_available(void) 149 { 150 uint32_t u = 0; 151 152 (void) getisax(&u, 1); 153 return ((u & AV_386_SSE2) != 0); 154 } 155 156 static inline boolean_t 157 zfs_sse3_available(void) 158 { 159 uint32_t u = 0; 160 161 (void) getisax(&u, 1); 162 return ((u & AV_386_SSE3) != 0); 163 } 164 165 static inline boolean_t 166 zfs_ssse3_available(void) 167 { 168 uint32_t u = 0; 169 170 (void) getisax(&u, 1); 171 return ((u & AV_386_SSSE3) != 0); 172 } 173 174 static inline boolean_t 175 zfs_avx_available(void) 176 { 177 uint_t u = 0; 178 179 (void) getisax(&u, 1); 180 return ((u & AV_386_AVX) != 0); 181 } 182 183 static inline boolean_t 184 zfs_avx2_available(void) 185 { 186 uint32_t u[2] = { 0 }; 187 188 (void) getisax((uint32_t *)&u, 2); 189 return ((u[1] & AV_386_2_AVX2) != 0); 190 } 191 192 static inline boolean_t 193 zfs_avx512f_available(void) 194 { 195 uint32_t u[2] = { 0 }; 196 197 (void) getisax((uint32_t *)&u, 2); 198 return ((u[1] & AV_386_2_AVX512F) != 0); 199 } 200 201 static inline boolean_t 202 zfs_avx512bw_available(void) 203 { 204 uint32_t u[2] = { 0 }; 205 206 (void) getisax((uint32_t *)&u, 2); 207 return ((u[1] & AV_386_2_AVX512BW) != 0); 208 } 209 210 #endif /* _KERNEL */ 211 212 213 #else 214 215 /* Non-x86 CPUs currently always disallow kernel FPU support */ 216 #define kfpu_allowed() 0 217 #define kfpu_initialize(tsk) do {} while (0) 218 #define kfpu_begin() do {} while (0) 219 #define kfpu_end() do {} while (0) 220 #define kfpu_init() (0) 221 #define kfpu_fini() do {} while (0) 222 #endif 223 224 #endif /* _SIMD_H */ 225