1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>.
24 */
25
26 /*
27 * USER API:
28 *
29 * Kernel fpu methods:
30 * kfpu_allowed()
31 * kfpu_begin()
32 * kfpu_end()
33 * kfpu_init()
34 * kfpu_fini()
35 *
36 * SIMD support:
37 *
38 * Following functions should be called to determine whether CPU feature
39 * is supported. All functions are usable in kernel and user space.
40 * If a SIMD algorithm is using more than one instruction set
41 * all relevant feature test functions should be called.
42 *
43 * Supported features:
44 * zfs_sse_available()
45 * zfs_sse2_available()
46 * zfs_sse3_available()
47 * zfs_ssse3_available()
48 * zfs_sse4_1_available()
49 * zfs_sse4_2_available()
50 *
51 * zfs_avx_available()
52 * zfs_avx2_available()
53 *
54 * zfs_bmi1_available()
55 * zfs_bmi2_available()
56 *
57 * zfs_shani_available()
58 *
59 * zfs_avx512f_available()
60 * zfs_avx512cd_available()
61 * zfs_avx512er_available()
62 * zfs_avx512pf_available()
63 * zfs_avx512bw_available()
64 * zfs_avx512dq_available()
65 * zfs_avx512vl_available()
66 * zfs_avx512ifma_available()
67 * zfs_avx512vbmi_available()
68 *
69 * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers
70 * also add zfs_avx512vl_available() to feature check.
71 */
72
73 #ifndef _LINUX_SIMD_X86_H
74 #define _LINUX_SIMD_X86_H
75
76 /* only for __x86 */
77 #if defined(__x86)
78
79 #include <sys/types.h>
80 #include <asm/cpufeature.h>
81
82 /*
83 * Disable the WARN_ON_FPU() macro to prevent additional dependencies
84 * when providing the kfpu_* functions. Relevant warnings are included
85 * as appropriate and are unconditionally enabled.
86 */
87 #if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU)
88 #undef CONFIG_X86_DEBUG_FPU
89 #endif
90
91 /*
92 * The following cases are for kernels which export either the
93 * kernel_fpu_* or __kernel_fpu_* functions.
94 */
95 #if defined(KERNEL_EXPORTS_X86_FPU)
96
97 #if defined(HAVE_KERNEL_FPU_API_HEADER)
98 #include <asm/fpu/api.h>
99 #if defined(HAVE_KERNEL_FPU_INTERNAL_HEADER)
100 #include <asm/fpu/internal.h>
101 #endif
102 #else
103 #include <asm/i387.h>
104 #endif
105
106 #define kfpu_allowed() 1
107 #define kfpu_init() 0
108 #define kfpu_fini() ((void) 0)
109
110 #if defined(HAVE_UNDERSCORE_KERNEL_FPU)
111 #define kfpu_begin() \
112 { \
113 preempt_disable(); \
114 __kernel_fpu_begin(); \
115 }
116 #define kfpu_end() \
117 { \
118 __kernel_fpu_end(); \
119 preempt_enable(); \
120 }
121
122 #elif defined(HAVE_KERNEL_FPU)
123 #define kfpu_begin() kernel_fpu_begin()
124 #define kfpu_end() kernel_fpu_end()
125
126 #else
127 /*
128 * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then
129 * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined.
130 */
131 #error "Unreachable kernel configuration"
132 #endif
133
134 #else /* defined(KERNEL_EXPORTS_X86_FPU) */
135
136 /*
137 * When the kernel_fpu_* symbols are unavailable then provide our own
138 * versions which allow the FPU to be safely used.
139 */
140 #if defined(HAVE_KERNEL_FPU_INTERNAL)
141
142 #ifndef XFEATURE_MASK_XTILE
143 /*
144 * For kernels where this doesn't exist yet, we still don't want to break
145 * by save/restoring this broken nonsense.
146 * See issue #14989 or Intel errata SPR4 for why
147 */
148 #define XFEATURE_MASK_XTILE 0x60000
149 #endif
150
151 #include <linux/mm.h>
152 #include <linux/slab.h>
153
154 extern uint8_t **zfs_kfpu_fpregs;
155
156 /*
157 * Return the size in bytes required by the XSAVE instruction for an
158 * XSAVE area containing all the user state components supported by this CPU.
159 * See: Intel 64 and IA-32 Architectures Software Developer’s Manual.
160 * Dec. 2021. Vol. 2A p. 3-222.
161 */
162 static inline uint32_t
get_xsave_area_size(void)163 get_xsave_area_size(void)
164 {
165 if (!boot_cpu_has(X86_FEATURE_OSXSAVE)) {
166 return (0);
167 }
168 /*
169 * Call CPUID with leaf 13 and subleaf 0. The size is in ecx.
170 * We don't need to check for cpuid_max here, since if this CPU has
171 * OSXSAVE set, it has leaf 13 (0x0D) as well.
172 */
173 uint32_t eax, ebx, ecx, edx;
174
175 eax = 13U;
176 ecx = 0U;
177 __asm__ __volatile__("cpuid"
178 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
179 : "a" (eax), "c" (ecx));
180
181 return (ecx);
182 }
183
184 /*
185 * Return the allocation order of the maximum buffer size required to save the
186 * FPU state on this architecture. The value returned is the same as Linux'
187 * get_order() function would return (i.e. 2^order = nr. of pages required).
188 * Currently this will always return 0 since the save area is below 4k even for
189 * a full fledged AVX-512 implementation.
190 */
191 static inline int
get_fpuregs_save_area_order(void)192 get_fpuregs_save_area_order(void)
193 {
194 size_t area_size = (size_t)get_xsave_area_size();
195
196 /*
197 * If we are dealing with a CPU not supporting XSAVE,
198 * get_xsave_area_size() will return 0. Thus the maximum memory
199 * required is the FXSAVE area size which is 512 bytes. See: Intel 64
200 * and IA-32 Architectures Software Developer’s Manual. Dec. 2021.
201 * Vol. 2A p. 3-451.
202 */
203 if (area_size == 0) {
204 area_size = 512;
205 }
206 return (get_order(area_size));
207 }
208
209 /*
210 * Initialize per-cpu variables to store FPU state.
211 */
212 static inline void
kfpu_fini(void)213 kfpu_fini(void)
214 {
215 int cpu;
216 int order = get_fpuregs_save_area_order();
217
218 for_each_possible_cpu(cpu) {
219 if (zfs_kfpu_fpregs[cpu] != NULL) {
220 free_pages((unsigned long)zfs_kfpu_fpregs[cpu], order);
221 }
222 }
223
224 kfree(zfs_kfpu_fpregs);
225 }
226
227 static inline int
kfpu_init(void)228 kfpu_init(void)
229 {
230 zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (uint8_t *),
231 GFP_KERNEL);
232
233 if (zfs_kfpu_fpregs == NULL)
234 return (-ENOMEM);
235
236 /*
237 * The fxsave and xsave operations require 16-/64-byte alignment of
238 * the target memory. Since kmalloc() provides no alignment
239 * guarantee instead use alloc_pages_node().
240 */
241 int cpu;
242 int order = get_fpuregs_save_area_order();
243
244 for_each_possible_cpu(cpu) {
245 struct page *page = alloc_pages_node(cpu_to_node(cpu),
246 GFP_KERNEL | __GFP_ZERO, order);
247 if (page == NULL) {
248 kfpu_fini();
249 return (-ENOMEM);
250 }
251
252 zfs_kfpu_fpregs[cpu] = page_address(page);
253 }
254
255 return (0);
256 }
257
258 #define kfpu_allowed() 1
259
260 /*
261 * FPU save and restore instructions.
262 */
263 #define __asm __asm__ __volatile__
264 #define kfpu_fxsave(addr) __asm("fxsave %0" : "=m" (*(addr)))
265 #define kfpu_fxsaveq(addr) __asm("fxsaveq %0" : "=m" (*(addr)))
266 #define kfpu_fnsave(addr) __asm("fnsave %0; fwait" : "=m" (*(addr)))
267 #define kfpu_fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr)))
268 #define kfpu_fxrstorq(addr) __asm("fxrstorq %0" : : "m" (*(addr)))
269 #define kfpu_frstor(addr) __asm("frstor %0" : : "m" (*(addr)))
270 #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \
271 : : [addr] "m" (rval));
272
273 #define kfpu_do_xsave(instruction, addr, mask) \
274 { \
275 uint32_t low, hi; \
276 \
277 low = mask; \
278 hi = (uint64_t)(mask) >> 32; \
279 __asm(instruction " %[dst]\n\t" \
280 : \
281 : [dst] "m" (*(addr)), "a" (low), "d" (hi) \
282 : "memory"); \
283 }
284
285 static inline void
kfpu_save_fxsr(uint8_t * addr)286 kfpu_save_fxsr(uint8_t *addr)
287 {
288 if (IS_ENABLED(CONFIG_X86_32))
289 kfpu_fxsave(addr);
290 else
291 kfpu_fxsaveq(addr);
292 }
293
294 static inline void
kfpu_save_fsave(uint8_t * addr)295 kfpu_save_fsave(uint8_t *addr)
296 {
297 kfpu_fnsave(addr);
298 }
299
300 static inline void
kfpu_begin(void)301 kfpu_begin(void)
302 {
303 /*
304 * Preemption and interrupts must be disabled for the critical
305 * region where the FPU state is being modified.
306 */
307 preempt_disable();
308 local_irq_disable();
309
310 /*
311 * The current FPU registers need to be preserved by kfpu_begin()
312 * and restored by kfpu_end(). They are stored in a dedicated
313 * per-cpu variable, not in the task struct, this allows any user
314 * FPU state to be correctly preserved and restored.
315 */
316 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
317 #if defined(HAVE_XSAVES)
318 if (static_cpu_has(X86_FEATURE_XSAVES)) {
319 kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE);
320 return;
321 }
322 #endif
323 #if defined(HAVE_XSAVEOPT)
324 if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
325 kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE);
326 return;
327 }
328 #endif
329 #if defined(HAVE_XSAVE)
330 if (static_cpu_has(X86_FEATURE_XSAVE)) {
331 kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE);
332 return;
333 }
334 #endif
335 if (static_cpu_has(X86_FEATURE_FXSR)) {
336 kfpu_save_fxsr(state);
337 } else {
338 kfpu_save_fsave(state);
339 }
340 }
341
342 #define kfpu_do_xrstor(instruction, addr, mask) \
343 { \
344 uint32_t low, hi; \
345 \
346 low = mask; \
347 hi = (uint64_t)(mask) >> 32; \
348 __asm(instruction " %[src]" \
349 : \
350 : [src] "m" (*(addr)), "a" (low), "d" (hi) \
351 : "memory"); \
352 }
353
354 static inline void
kfpu_restore_fxsr(uint8_t * addr)355 kfpu_restore_fxsr(uint8_t *addr)
356 {
357 /*
358 * On AuthenticAMD K7 and K8 processors the fxrstor instruction only
359 * restores the _x87 FOP, FIP, and FDP registers when an exception
360 * is pending. Clean the _x87 state to force the restore.
361 */
362 if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK)))
363 kfpu_fxsr_clean(addr);
364
365 if (IS_ENABLED(CONFIG_X86_32)) {
366 kfpu_fxrstor(addr);
367 } else {
368 kfpu_fxrstorq(addr);
369 }
370 }
371
372 static inline void
kfpu_restore_fsave(uint8_t * addr)373 kfpu_restore_fsave(uint8_t *addr)
374 {
375 kfpu_frstor(addr);
376 }
377
378 static inline void
kfpu_end(void)379 kfpu_end(void)
380 {
381 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
382 #if defined(HAVE_XSAVES)
383 if (static_cpu_has(X86_FEATURE_XSAVES)) {
384 kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE);
385 goto out;
386 }
387 #endif
388 #if defined(HAVE_XSAVE)
389 if (static_cpu_has(X86_FEATURE_XSAVE)) {
390 kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE);
391 goto out;
392 }
393 #endif
394 if (static_cpu_has(X86_FEATURE_FXSR)) {
395 kfpu_restore_fxsr(state);
396 } else {
397 kfpu_restore_fsave(state);
398 }
399 out:
400 local_irq_enable();
401 preempt_enable();
402
403 }
404
405 #else
406
407 #error "Exactly one of KERNEL_EXPORTS_X86_FPU or HAVE_KERNEL_FPU_INTERNAL" \
408 " must be defined"
409
410 #endif /* defined(HAVE_KERNEL_FPU_INTERNAL */
411 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */
412
413 /*
414 * Linux kernel provides an interface for CPU feature testing.
415 */
416
417 /*
418 * Detect register set support
419 */
420
421 /*
422 * Check if OS supports AVX and AVX2 by checking XCR0
423 * Only call this function if CPUID indicates that AVX feature is
424 * supported by the CPU, otherwise it might be an illegal instruction.
425 */
426 static inline uint64_t
zfs_xgetbv(uint32_t index)427 zfs_xgetbv(uint32_t index)
428 {
429 uint32_t eax, edx;
430 /* xgetbv - instruction byte code */
431 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0"
432 : "=a" (eax), "=d" (edx)
433 : "c" (index));
434
435 return ((((uint64_t)edx)<<32) | (uint64_t)eax);
436 }
437
438
439 static inline boolean_t
__simd_state_enabled(const uint64_t state)440 __simd_state_enabled(const uint64_t state)
441 {
442 boolean_t has_osxsave;
443 uint64_t xcr0;
444
445 #if defined(X86_FEATURE_OSXSAVE)
446 has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE);
447 #else
448 has_osxsave = B_FALSE;
449 #endif
450 if (!has_osxsave)
451 return (B_FALSE);
452
453 xcr0 = zfs_xgetbv(0);
454 return ((xcr0 & state) == state);
455 }
456
457 #define _XSTATE_SSE_AVX (0x2 | 0x4)
458 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX)
459
460 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
461 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)
462
463 /*
464 * Check if SSE instruction set is available
465 */
466 static inline boolean_t
zfs_sse_available(void)467 zfs_sse_available(void)
468 {
469 return (!!boot_cpu_has(X86_FEATURE_XMM));
470 }
471
472 /*
473 * Check if SSE2 instruction set is available
474 */
475 static inline boolean_t
zfs_sse2_available(void)476 zfs_sse2_available(void)
477 {
478 return (!!boot_cpu_has(X86_FEATURE_XMM2));
479 }
480
481 /*
482 * Check if SSE3 instruction set is available
483 */
484 static inline boolean_t
zfs_sse3_available(void)485 zfs_sse3_available(void)
486 {
487 return (!!boot_cpu_has(X86_FEATURE_XMM3));
488 }
489
490 /*
491 * Check if SSSE3 instruction set is available
492 */
493 static inline boolean_t
zfs_ssse3_available(void)494 zfs_ssse3_available(void)
495 {
496 return (!!boot_cpu_has(X86_FEATURE_SSSE3));
497 }
498
499 /*
500 * Check if SSE4.1 instruction set is available
501 */
502 static inline boolean_t
zfs_sse4_1_available(void)503 zfs_sse4_1_available(void)
504 {
505 return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
506 }
507
508 /*
509 * Check if SSE4.2 instruction set is available
510 */
511 static inline boolean_t
zfs_sse4_2_available(void)512 zfs_sse4_2_available(void)
513 {
514 return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
515 }
516
517 /*
518 * Check if AVX instruction set is available
519 */
520 static inline boolean_t
zfs_avx_available(void)521 zfs_avx_available(void)
522 {
523 return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled());
524 }
525
526 /*
527 * Check if AVX2 instruction set is available
528 */
529 static inline boolean_t
zfs_avx2_available(void)530 zfs_avx2_available(void)
531 {
532 return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled());
533 }
534
535 /*
536 * Check if BMI1 instruction set is available
537 */
538 static inline boolean_t
zfs_bmi1_available(void)539 zfs_bmi1_available(void)
540 {
541 #if defined(X86_FEATURE_BMI1)
542 return (!!boot_cpu_has(X86_FEATURE_BMI1));
543 #else
544 return (B_FALSE);
545 #endif
546 }
547
548 /*
549 * Check if BMI2 instruction set is available
550 */
551 static inline boolean_t
zfs_bmi2_available(void)552 zfs_bmi2_available(void)
553 {
554 #if defined(X86_FEATURE_BMI2)
555 return (!!boot_cpu_has(X86_FEATURE_BMI2));
556 #else
557 return (B_FALSE);
558 #endif
559 }
560
561 /*
562 * Check if AES instruction set is available
563 */
564 static inline boolean_t
zfs_aes_available(void)565 zfs_aes_available(void)
566 {
567 #if defined(X86_FEATURE_AES)
568 return (!!boot_cpu_has(X86_FEATURE_AES));
569 #else
570 return (B_FALSE);
571 #endif
572 }
573
574 /*
575 * Check if PCLMULQDQ instruction set is available
576 */
577 static inline boolean_t
zfs_pclmulqdq_available(void)578 zfs_pclmulqdq_available(void)
579 {
580 #if defined(X86_FEATURE_PCLMULQDQ)
581 return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
582 #else
583 return (B_FALSE);
584 #endif
585 }
586
587 /*
588 * Check if MOVBE instruction is available
589 */
590 static inline boolean_t
zfs_movbe_available(void)591 zfs_movbe_available(void)
592 {
593 #if defined(X86_FEATURE_MOVBE)
594 return (!!boot_cpu_has(X86_FEATURE_MOVBE));
595 #else
596 return (B_FALSE);
597 #endif
598 }
599
600 /*
601 * Check if VAES instruction set is available
602 */
603 static inline boolean_t
zfs_vaes_available(void)604 zfs_vaes_available(void)
605 {
606 #if defined(X86_FEATURE_VAES)
607 return (!!boot_cpu_has(X86_FEATURE_VAES));
608 #else
609 return (B_FALSE);
610 #endif
611 }
612
613 /*
614 * Check if VPCLMULQDQ instruction set is available
615 */
616 static inline boolean_t
zfs_vpclmulqdq_available(void)617 zfs_vpclmulqdq_available(void)
618 {
619 #if defined(X86_FEATURE_VPCLMULQDQ)
620 return (!!boot_cpu_has(X86_FEATURE_VPCLMULQDQ));
621 #else
622 return (B_FALSE);
623 #endif
624 }
625
626 /*
627 * Check if SHA_NI instruction set is available
628 */
629 static inline boolean_t
zfs_shani_available(void)630 zfs_shani_available(void)
631 {
632 #if defined(X86_FEATURE_SHA_NI)
633 return (!!boot_cpu_has(X86_FEATURE_SHA_NI));
634 #else
635 return (B_FALSE);
636 #endif
637 }
638
639 /*
640 * AVX-512 family of instruction sets:
641 *
642 * AVX512F Foundation
643 * AVX512CD Conflict Detection Instructions
644 * AVX512ER Exponential and Reciprocal Instructions
645 * AVX512PF Prefetch Instructions
646 *
647 * AVX512BW Byte and Word Instructions
648 * AVX512DQ Double-word and Quadword Instructions
649 * AVX512VL Vector Length Extensions
650 *
651 * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4)
652 * AVX512VBMI Vector Byte Manipulation Instructions
653 */
654
655 /*
656 * Check if AVX512F instruction set is available
657 */
658 static inline boolean_t
zfs_avx512f_available(void)659 zfs_avx512f_available(void)
660 {
661 boolean_t has_avx512 = B_FALSE;
662
663 #if defined(X86_FEATURE_AVX512F)
664 has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F);
665 #endif
666 return (has_avx512 && __zmm_enabled());
667 }
668
669 /*
670 * Check if AVX512CD instruction set is available
671 */
672 static inline boolean_t
zfs_avx512cd_available(void)673 zfs_avx512cd_available(void)
674 {
675 boolean_t has_avx512 = B_FALSE;
676
677 #if defined(X86_FEATURE_AVX512CD)
678 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
679 boot_cpu_has(X86_FEATURE_AVX512CD);
680 #endif
681 return (has_avx512 && __zmm_enabled());
682 }
683
684 /*
685 * Check if AVX512ER instruction set is available
686 */
687 static inline boolean_t
zfs_avx512er_available(void)688 zfs_avx512er_available(void)
689 {
690 boolean_t has_avx512 = B_FALSE;
691
692 #if defined(X86_FEATURE_AVX512ER)
693 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
694 boot_cpu_has(X86_FEATURE_AVX512ER);
695 #endif
696 return (has_avx512 && __zmm_enabled());
697 }
698
699 /*
700 * Check if AVX512PF instruction set is available
701 */
702 static inline boolean_t
zfs_avx512pf_available(void)703 zfs_avx512pf_available(void)
704 {
705 boolean_t has_avx512 = B_FALSE;
706
707 #if defined(X86_FEATURE_AVX512PF)
708 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
709 boot_cpu_has(X86_FEATURE_AVX512PF);
710 #endif
711 return (has_avx512 && __zmm_enabled());
712 }
713
714 /*
715 * Check if AVX512BW instruction set is available
716 */
717 static inline boolean_t
zfs_avx512bw_available(void)718 zfs_avx512bw_available(void)
719 {
720 boolean_t has_avx512 = B_FALSE;
721
722 #if defined(X86_FEATURE_AVX512BW)
723 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
724 boot_cpu_has(X86_FEATURE_AVX512BW);
725 #endif
726
727 return (has_avx512 && __zmm_enabled());
728 }
729
730 /*
731 * Check if AVX512DQ instruction set is available
732 */
733 static inline boolean_t
zfs_avx512dq_available(void)734 zfs_avx512dq_available(void)
735 {
736 boolean_t has_avx512 = B_FALSE;
737
738 #if defined(X86_FEATURE_AVX512DQ)
739 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
740 boot_cpu_has(X86_FEATURE_AVX512DQ);
741 #endif
742 return (has_avx512 && __zmm_enabled());
743 }
744
745 /*
746 * Check if AVX512VL instruction set is available
747 */
748 static inline boolean_t
zfs_avx512vl_available(void)749 zfs_avx512vl_available(void)
750 {
751 boolean_t has_avx512 = B_FALSE;
752
753 #if defined(X86_FEATURE_AVX512VL)
754 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
755 boot_cpu_has(X86_FEATURE_AVX512VL);
756 #endif
757 return (has_avx512 && __zmm_enabled());
758 }
759
760 /*
761 * Check if AVX512IFMA instruction set is available
762 */
763 static inline boolean_t
zfs_avx512ifma_available(void)764 zfs_avx512ifma_available(void)
765 {
766 boolean_t has_avx512 = B_FALSE;
767
768 #if defined(X86_FEATURE_AVX512IFMA)
769 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
770 boot_cpu_has(X86_FEATURE_AVX512IFMA);
771 #endif
772 return (has_avx512 && __zmm_enabled());
773 }
774
775 /*
776 * Check if AVX512VBMI instruction set is available
777 */
778 static inline boolean_t
zfs_avx512vbmi_available(void)779 zfs_avx512vbmi_available(void)
780 {
781 boolean_t has_avx512 = B_FALSE;
782
783 #if defined(X86_FEATURE_AVX512VBMI)
784 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
785 boot_cpu_has(X86_FEATURE_AVX512VBMI);
786 #endif
787 return (has_avx512 && __zmm_enabled());
788 }
789
790 #endif /* defined(__x86) */
791
792 #endif /* _LINUX_SIMD_X86_H */
793