1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020 Joyent, Inc. 14 */ 15 16 #ifndef _SIMD_H 17 #define _SIMD_H 18 19 #if defined(__amd64__) || defined(__i386__) 20 21 #define kfpu_initialize(tsk) do {} while (0) 22 #define kfpu_init() (0) 23 #define kfpu_fini() do {} while (0) 24 25 #ifdef _KERNEL 26 #include <sys/x86_archext.h> 27 #include <sys/archsystm.h> 28 #include <sys/kfpu.h> 29 #include <sys/proc.h> 30 #include <sys/disp.h> 31 #include <sys/cpuvar.h> 32 33 static inline int 34 kfpu_allowed(void) 35 { 36 extern int zfs_fpu_enabled; 37 38 return (zfs_fpu_enabled != 0 ? 1 : 0); 39 } 40 41 static inline void 42 kfpu_begin(void) 43 { 44 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) { 45 kernel_fpu_begin(NULL, KFPU_USE_LWP); 46 } else { 47 kpreempt_disable(); 48 kernel_fpu_begin(NULL, KFPU_NO_STATE); 49 } 50 } 51 52 static inline void 53 kfpu_end(void) 54 { 55 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) { 56 kernel_fpu_end(NULL, KFPU_USE_LWP); 57 } else { 58 kernel_fpu_end(NULL, KFPU_NO_STATE); 59 kpreempt_enable(); 60 } 61 } 62 63 /* 64 * Check if various vector instruction sets are available. 65 */ 66 67 static inline boolean_t 68 zfs_sse_available(void) 69 { 70 return (is_x86_feature(x86_featureset, X86FSET_SSE)); 71 } 72 73 static inline boolean_t 74 zfs_sse2_available(void) 75 { 76 return (is_x86_feature(x86_featureset, X86FSET_SSE2)); 77 } 78 79 static inline boolean_t 80 zfs_sse3_available(void) 81 { 82 return (is_x86_feature(x86_featureset, X86FSET_SSE3)); 83 } 84 85 static inline boolean_t 86 zfs_ssse3_available(void) 87 { 88 return (is_x86_feature(x86_featureset, X86FSET_SSSE3)); 89 } 90 91 static inline boolean_t 92 zfs_avx_available(void) 93 { 94 return (is_x86_feature(x86_featureset, X86FSET_AVX)); 95 } 96 97 static inline boolean_t 98 zfs_avx2_available(void) 99 { 100 return (is_x86_feature(x86_featureset, X86FSET_AVX2)); 101 } 102 103 static inline boolean_t 104 zfs_avx512f_available(void) 105 { 106 return (is_x86_feature(x86_featureset, X86FSET_AVX512F)); 107 } 108 109 static inline boolean_t 110 zfs_avx512bw_available(void) 111 { 112 return (is_x86_feature(x86_featureset, X86FSET_AVX512BW)); 113 } 114 115 #else /* ! _KERNEL */ 116 117 #include <sys/auxv.h> 118 #include <sys/auxv_386.h> 119 120 #define kfpu_allowed() 1 121 #define kfpu_begin() do {} while (0) 122 #define kfpu_end() do {} while (0) 123 124 /* 125 * User-level check if various vector instruction sets are available. 126 */ 127 128 static inline boolean_t 129 zfs_sse_available(void) 130 { 131 uint32_t u = 0; 132 133 (void) getisax(&u, 1); 134 return ((u & AV_386_SSE) != 0); 135 } 136 137 static inline boolean_t 138 zfs_sse2_available(void) 139 { 140 uint32_t u = 0; 141 142 (void) getisax(&u, 1); 143 return ((u & AV_386_SSE2) != 0); 144 } 145 146 static inline boolean_t 147 zfs_sse3_available(void) 148 { 149 uint32_t u = 0; 150 151 (void) getisax(&u, 1); 152 return ((u & AV_386_SSE3) != 0); 153 } 154 155 static inline boolean_t 156 zfs_ssse3_available(void) 157 { 158 uint32_t u = 0; 159 160 (void) getisax(&u, 1); 161 return ((u & AV_386_SSSE3) != 0); 162 } 163 164 static inline boolean_t 165 zfs_avx_available(void) 166 { 167 uint_t u = 0; 168 169 (void) getisax(&u, 1); 170 return ((u & AV_386_AVX) != 0); 171 } 172 173 static inline boolean_t 174 zfs_avx2_available(void) 175 { 176 uint32_t u[2] = { 0 }; 177 178 (void) getisax((uint32_t *)&u, 2); 179 return ((u[1] & AV_386_2_AVX2) != 0); 180 } 181 182 static inline boolean_t 183 zfs_avx512f_available(void) 184 { 185 uint32_t u[2] = { 0 }; 186 187 (void) getisax((uint32_t *)&u, 2); 188 return ((u[1] & AV_386_2_AVX512F) != 0); 189 } 190 191 static inline boolean_t 192 zfs_avx512bw_available(void) 193 { 194 uint32_t u[2] = { 0 }; 195 196 (void) getisax((uint32_t *)&u, 2); 197 return ((u[1] & AV_386_2_AVX512BW) != 0); 198 } 199 200 #endif /* _KERNEL */ 201 202 203 #else 204 205 /* Non-x86 CPUs currently always disallow kernel FPU support */ 206 #define kfpu_allowed() 0 207 #define kfpu_initialize(tsk) do {} while (0) 208 #define kfpu_begin() do {} while (0) 209 #define kfpu_end() do {} while (0) 210 #define kfpu_init() (0) 211 #define kfpu_fini() do {} while (0) 212 #endif 213 214 #endif /* _SIMD_H */ 215