1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>.
14 * Copyright (c) 2026, TrueNAS.
15 */
16
17 /*
18 * USER API:
19 *
20 * Kernel fpu methods:
21 * kfpu_allowed()
22 * kfpu_begin()
23 * kfpu_end()
24 * kfpu_init()
25 * kfpu_fini()
26 *
27 * SIMD support:
28 *
29 * Following functions should be called to determine whether CPU feature
30 * is supported. All functions are usable in kernel and user space.
31 * If a SIMD algorithm is using more than one instruction set
32 * all relevant feature test functions should be called.
33 *
34 * Supported features:
35 * zfs_sse_available()
36 * zfs_sse2_available()
37 * zfs_sse3_available()
38 * zfs_ssse3_available()
39 * zfs_sse4_1_available()
40 * zfs_sse4_2_available()
41 *
42 * zfs_avx_available()
43 * zfs_avx2_available()
44 *
45 * zfs_bmi1_available()
46 * zfs_bmi2_available()
47 *
48 * zfs_shani_available()
49 *
50 * zfs_avx512f_available()
51 * zfs_avx512cd_available()
52 * zfs_avx512er_available()
53 * zfs_avx512pf_available()
54 * zfs_avx512bw_available()
55 * zfs_avx512dq_available()
56 * zfs_avx512vl_available()
57 * zfs_avx512ifma_available()
58 * zfs_avx512vbmi_available()
59 *
60 * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers
61 * also add zfs_avx512vl_available() to feature check.
62 */
63
64 #ifndef _LINUX_SIMD_X86_H
65 #define _LINUX_SIMD_X86_H
66
67 /* only for __x86 */
68 #if defined(__x86)
69
70 #include <sys/types.h>
71 #include <asm/cpufeature.h>
72
73 /*
74 * Disable the WARN_ON_FPU() macro to prevent additional dependencies
75 * when providing the kfpu_* functions. Relevant warnings are included
76 * as appropriate and are unconditionally enabled.
77 */
78 #if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU)
79 #undef CONFIG_X86_DEBUG_FPU
80 #endif
81
82 /*
83 * The following cases are for kernels which export either the
84 * kernel_fpu_* or __kernel_fpu_* functions.
85 */
86 #if defined(KERNEL_EXPORTS_X86_FPU)
87
88 #if defined(HAVE_KERNEL_FPU_API_HEADER)
89 #include <asm/fpu/api.h>
90 #if defined(HAVE_KERNEL_FPU_INTERNAL_HEADER)
91 #include <asm/fpu/internal.h>
92 #endif
93 #else
94 #include <asm/i387.h>
95 #endif
96
97 #define kfpu_allowed() 1
98 #define kfpu_init() 0
99 #define kfpu_fini() ((void) 0)
100
101 #if defined(HAVE_UNDERSCORE_KERNEL_FPU)
102 #define kfpu_begin() \
103 { \
104 preempt_disable(); \
105 __kernel_fpu_begin(); \
106 }
107 #define kfpu_end() \
108 { \
109 __kernel_fpu_end(); \
110 preempt_enable(); \
111 }
112
113 #elif defined(HAVE_KERNEL_FPU)
114 #define kfpu_begin() kernel_fpu_begin()
115 #define kfpu_end() kernel_fpu_end()
116
117 #else
118 /*
119 * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then
120 * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined.
121 */
122 #error "Unreachable kernel configuration"
123 #endif
124
125 #else /* defined(KERNEL_EXPORTS_X86_FPU) */
126
127 /*
128 * When the kernel_fpu_* symbols are unavailable then provide our own
129 * versions which allow the FPU to be safely used.
130 */
131 #if defined(HAVE_KERNEL_FPU_INTERNAL)
132
133 #ifndef XFEATURE_MASK_XTILE
134 /*
135 * For kernels where this doesn't exist yet, we still don't want to break
136 * by save/restoring this broken nonsense.
137 * See issue #14989 or Intel errata SPR4 for why
138 */
139 #define XFEATURE_MASK_XTILE 0x60000
140 #endif
141
142 #include <linux/mm.h>
143 #include <linux/slab.h>
144
145 extern uint8_t **zfs_kfpu_fpregs;
146
147 /*
148 * Return the size in bytes required by the XSAVE instruction for an
149 * XSAVE area containing all the user state components supported by this CPU.
150 * See: Intel 64 and IA-32 Architectures Software Developer’s Manual.
151 * Dec. 2021. Vol. 2A p. 3-222.
152 */
153 static inline uint32_t
get_xsave_area_size(void)154 get_xsave_area_size(void)
155 {
156 if (!boot_cpu_has(X86_FEATURE_OSXSAVE)) {
157 return (0);
158 }
159 /*
160 * Call CPUID with leaf 13 and subleaf 0. The size is in ecx.
161 * We don't need to check for cpuid_max here, since if this CPU has
162 * OSXSAVE set, it has leaf 13 (0x0D) as well.
163 */
164 uint32_t eax, ebx, ecx, edx;
165
166 eax = 13U;
167 ecx = 0U;
168 __asm__ __volatile__("cpuid"
169 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
170 : "a" (eax), "c" (ecx));
171
172 return (ecx);
173 }
174
175 /*
176 * Return the allocation order of the maximum buffer size required to save the
177 * FPU state on this architecture. The value returned is the same as Linux'
178 * get_order() function would return (i.e. 2^order = nr. of pages required).
179 * Currently this will always return 0 since the save area is below 4k even for
180 * a full fledged AVX-512 implementation.
181 */
182 static inline int
get_fpuregs_save_area_order(void)183 get_fpuregs_save_area_order(void)
184 {
185 size_t area_size = (size_t)get_xsave_area_size();
186
187 /*
188 * If we are dealing with a CPU not supporting XSAVE,
189 * get_xsave_area_size() will return 0. Thus the maximum memory
190 * required is the FXSAVE area size which is 512 bytes. See: Intel 64
191 * and IA-32 Architectures Software Developer’s Manual. Dec. 2021.
192 * Vol. 2A p. 3-451.
193 */
194 if (area_size == 0) {
195 area_size = 512;
196 }
197 return (get_order(area_size));
198 }
199
200 /*
201 * Initialize per-cpu variables to store FPU state.
202 */
203 static inline void
kfpu_fini(void)204 kfpu_fini(void)
205 {
206 int cpu;
207 int order = get_fpuregs_save_area_order();
208
209 for_each_possible_cpu(cpu) {
210 if (zfs_kfpu_fpregs[cpu] != NULL) {
211 free_pages((unsigned long)zfs_kfpu_fpregs[cpu], order);
212 }
213 }
214
215 kfree(zfs_kfpu_fpregs);
216 }
217
218 static inline int
kfpu_init(void)219 kfpu_init(void)
220 {
221 zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (uint8_t *),
222 GFP_KERNEL);
223
224 if (zfs_kfpu_fpregs == NULL)
225 return (-ENOMEM);
226
227 /*
228 * The fxsave and xsave operations require 16-/64-byte alignment of
229 * the target memory. Since kmalloc() provides no alignment
230 * guarantee instead use alloc_pages_node().
231 */
232 int cpu;
233 int order = get_fpuregs_save_area_order();
234
235 for_each_possible_cpu(cpu) {
236 struct page *page = alloc_pages_node(cpu_to_node(cpu),
237 GFP_KERNEL | __GFP_ZERO, order);
238 if (page == NULL) {
239 kfpu_fini();
240 return (-ENOMEM);
241 }
242
243 zfs_kfpu_fpregs[cpu] = page_address(page);
244 }
245
246 return (0);
247 }
248
249 #define kfpu_allowed() 1
250
251 /*
252 * FPU save and restore instructions.
253 */
254 #define __asm __asm__ __volatile__
255 #define kfpu_fxsave(addr) __asm("fxsave %0" : "=m" (*(addr)))
256 #define kfpu_fxsaveq(addr) __asm("fxsaveq %0" : "=m" (*(addr)))
257 #define kfpu_fnsave(addr) __asm("fnsave %0; fwait" : "=m" (*(addr)))
258 #define kfpu_fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr)))
259 #define kfpu_fxrstorq(addr) __asm("fxrstorq %0" : : "m" (*(addr)))
260 #define kfpu_frstor(addr) __asm("frstor %0" : : "m" (*(addr)))
261 #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \
262 : : [addr] "m" (rval));
263
264 #define kfpu_do_xsave(instruction, addr, mask) \
265 { \
266 uint32_t low, hi; \
267 \
268 low = mask; \
269 hi = (uint64_t)(mask) >> 32; \
270 __asm(instruction " %[dst]\n\t" \
271 : \
272 : [dst] "m" (*(addr)), "a" (low), "d" (hi) \
273 : "memory"); \
274 }
275
276 static inline void
kfpu_save_fxsr(uint8_t * addr)277 kfpu_save_fxsr(uint8_t *addr)
278 {
279 if (IS_ENABLED(CONFIG_X86_32))
280 kfpu_fxsave(addr);
281 else
282 kfpu_fxsaveq(addr);
283 }
284
285 static inline void
kfpu_save_fsave(uint8_t * addr)286 kfpu_save_fsave(uint8_t *addr)
287 {
288 kfpu_fnsave(addr);
289 }
290
291 static inline void
kfpu_begin(void)292 kfpu_begin(void)
293 {
294 /*
295 * Preemption and interrupts must be disabled for the critical
296 * region where the FPU state is being modified.
297 */
298 preempt_disable();
299 local_irq_disable();
300
301 /*
302 * The current FPU registers need to be preserved by kfpu_begin()
303 * and restored by kfpu_end(). They are stored in a dedicated
304 * per-cpu variable, not in the task struct, this allows any user
305 * FPU state to be correctly preserved and restored.
306 */
307 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
308 #if HAVE_SIMD(XSAVES)
309 if (static_cpu_has(X86_FEATURE_XSAVES)) {
310 kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE);
311 return;
312 }
313 #endif
314 #if HAVE_SIMD(XSAVEOPT)
315 if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
316 kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE);
317 return;
318 }
319 #endif
320 #if HAVE_SIMD(XSAVE)
321 if (static_cpu_has(X86_FEATURE_XSAVE)) {
322 kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE);
323 return;
324 }
325 #endif
326 if (static_cpu_has(X86_FEATURE_FXSR)) {
327 kfpu_save_fxsr(state);
328 } else {
329 kfpu_save_fsave(state);
330 }
331 }
332
333 #define kfpu_do_xrstor(instruction, addr, mask) \
334 { \
335 uint32_t low, hi; \
336 \
337 low = mask; \
338 hi = (uint64_t)(mask) >> 32; \
339 __asm(instruction " %[src]" \
340 : \
341 : [src] "m" (*(addr)), "a" (low), "d" (hi) \
342 : "memory"); \
343 }
344
345 static inline void
kfpu_restore_fxsr(uint8_t * addr)346 kfpu_restore_fxsr(uint8_t *addr)
347 {
348 /*
349 * On AuthenticAMD K7 and K8 processors the fxrstor instruction only
350 * restores the _x87 FOP, FIP, and FDP registers when an exception
351 * is pending. Clean the _x87 state to force the restore.
352 */
353 if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK)))
354 kfpu_fxsr_clean(addr);
355
356 if (IS_ENABLED(CONFIG_X86_32)) {
357 kfpu_fxrstor(addr);
358 } else {
359 kfpu_fxrstorq(addr);
360 }
361 }
362
363 static inline void
kfpu_restore_fsave(uint8_t * addr)364 kfpu_restore_fsave(uint8_t *addr)
365 {
366 kfpu_frstor(addr);
367 }
368
369 static inline void
kfpu_end(void)370 kfpu_end(void)
371 {
372 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
373 #if HAVE_SIMD(XSAVES)
374 if (static_cpu_has(X86_FEATURE_XSAVES)) {
375 kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE);
376 goto out;
377 }
378 #endif
379 #if HAVE_SIMD(XSAVE)
380 if (static_cpu_has(X86_FEATURE_XSAVE)) {
381 kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE);
382 goto out;
383 }
384 #endif
385 if (static_cpu_has(X86_FEATURE_FXSR)) {
386 kfpu_restore_fxsr(state);
387 } else {
388 kfpu_restore_fsave(state);
389 }
390 out:
391 local_irq_enable();
392 preempt_enable();
393
394 }
395
396 #else
397
398 #error "Exactly one of KERNEL_EXPORTS_X86_FPU or HAVE_KERNEL_FPU_INTERNAL" \
399 " must be defined"
400
401 #endif /* defined(HAVE_KERNEL_FPU_INTERNAL */
402 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */
403
404 /*
405 * Linux kernel provides an interface for CPU feature testing.
406 */
407
408 /*
409 * Detect register set support
410 */
411
412 /*
413 * Check if OS supports AVX and AVX2 by checking XCR0
414 * Only call this function if CPUID indicates that AVX feature is
415 * supported by the CPU, otherwise it might be an illegal instruction.
416 */
417 static inline uint64_t
zfs_xgetbv(uint32_t index)418 zfs_xgetbv(uint32_t index)
419 {
420 uint32_t eax, edx;
421 /* xgetbv - instruction byte code */
422 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0"
423 : "=a" (eax), "=d" (edx)
424 : "c" (index));
425
426 return ((((uint64_t)edx)<<32) | (uint64_t)eax);
427 }
428
429
430 static inline boolean_t
__simd_state_enabled(const uint64_t state)431 __simd_state_enabled(const uint64_t state)
432 {
433 uint64_t xcr0;
434
435 if (!boot_cpu_has(X86_FEATURE_OSXSAVE))
436 return (B_FALSE);
437
438 xcr0 = zfs_xgetbv(0);
439 return ((xcr0 & state) == state);
440 }
441
442 #define _XSTATE_SSE_AVX (0x2 | 0x4)
443 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX)
444
445 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
446 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)
447
448 /*
449 * Check if SSE instruction set is available
450 */
451 static inline boolean_t
zfs_sse_available(void)452 zfs_sse_available(void)
453 {
454 return (!!boot_cpu_has(X86_FEATURE_XMM));
455 }
456
457 /*
458 * Check if SSE2 instruction set is available
459 */
460 static inline boolean_t
zfs_sse2_available(void)461 zfs_sse2_available(void)
462 {
463 return (!!boot_cpu_has(X86_FEATURE_XMM2));
464 }
465
466 /*
467 * Check if SSE3 instruction set is available
468 */
469 static inline boolean_t
zfs_sse3_available(void)470 zfs_sse3_available(void)
471 {
472 return (!!boot_cpu_has(X86_FEATURE_XMM3));
473 }
474
475 /*
476 * Check if SSSE3 instruction set is available
477 */
478 static inline boolean_t
zfs_ssse3_available(void)479 zfs_ssse3_available(void)
480 {
481 return (!!boot_cpu_has(X86_FEATURE_SSSE3));
482 }
483
484 /*
485 * Check if SSE4.1 instruction set is available
486 */
487 static inline boolean_t
zfs_sse4_1_available(void)488 zfs_sse4_1_available(void)
489 {
490 return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
491 }
492
493 /*
494 * Check if SSE4.2 instruction set is available
495 */
496 static inline boolean_t
zfs_sse4_2_available(void)497 zfs_sse4_2_available(void)
498 {
499 return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
500 }
501
502 /*
503 * Check if AVX instruction set is available
504 */
505 static inline boolean_t
zfs_avx_available(void)506 zfs_avx_available(void)
507 {
508 return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled());
509 }
510
511 /*
512 * Check if AVX2 instruction set is available
513 */
514 static inline boolean_t
zfs_avx2_available(void)515 zfs_avx2_available(void)
516 {
517 return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled());
518 }
519
520 /*
521 * Check if BMI1 instruction set is available
522 */
523 static inline boolean_t
zfs_bmi1_available(void)524 zfs_bmi1_available(void)
525 {
526 return (!!boot_cpu_has(X86_FEATURE_BMI1));
527 }
528
529 /*
530 * Check if BMI2 instruction set is available
531 */
532 static inline boolean_t
zfs_bmi2_available(void)533 zfs_bmi2_available(void)
534 {
535 return (!!boot_cpu_has(X86_FEATURE_BMI2));
536 }
537
538 /*
539 * Check if AES instruction set is available
540 */
541 static inline boolean_t
zfs_aes_available(void)542 zfs_aes_available(void)
543 {
544 return (!!boot_cpu_has(X86_FEATURE_AES));
545 }
546
547 /*
548 * Check if PCLMULQDQ instruction set is available
549 */
550 static inline boolean_t
zfs_pclmulqdq_available(void)551 zfs_pclmulqdq_available(void)
552 {
553 return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
554 }
555
556 /*
557 * Check if MOVBE instruction is available
558 */
559 static inline boolean_t
zfs_movbe_available(void)560 zfs_movbe_available(void)
561 {
562 return (!!boot_cpu_has(X86_FEATURE_MOVBE));
563 }
564
565 /*
566 * Check if VAES instruction set is available
567 */
568 static inline boolean_t
zfs_vaes_available(void)569 zfs_vaes_available(void)
570 {
571 return (!!boot_cpu_has(X86_FEATURE_VAES));
572 }
573
574 /*
575 * Check if VPCLMULQDQ instruction set is available
576 */
577 static inline boolean_t
zfs_vpclmulqdq_available(void)578 zfs_vpclmulqdq_available(void)
579 {
580 return (!!boot_cpu_has(X86_FEATURE_VPCLMULQDQ));
581 }
582
583 /*
584 * Check if SHA512 instructions are available
585 * Kernel added X86_FEATURE_SHA512 in 6.13 (torvalds/linux@a0423af92cb3)
586 */
587 static inline boolean_t
zfs_sha512ext_available(void)588 zfs_sha512ext_available(void)
589 {
590 #if defined(X86_FEATURE_SHA512)
591 return (!!boot_cpu_has(X86_FEATURE_SHA512));
592 #else
593 return (B_FALSE);
594 #endif
595 }
596
597 /*
598 * Check if SHA_NI instruction set is available
599 */
600 static inline boolean_t
zfs_shani_available(void)601 zfs_shani_available(void)
602 {
603 return (!!boot_cpu_has(X86_FEATURE_SHA_NI));
604 }
605
606 /*
607 * AVX-512 family of instruction sets:
608 *
609 * AVX512F Foundation
610 * AVX512CD Conflict Detection Instructions
611 * AVX512ER Exponential and Reciprocal Instructions
612 * AVX512PF Prefetch Instructions
613 *
614 * AVX512BW Byte and Word Instructions
615 * AVX512DQ Double-word and Quadword Instructions
616 * AVX512VL Vector Length Extensions
617 *
618 * AVX512IFMA Integer Fused Multiply Add
619 * AVX512VBMI Vector Byte Manipulation Instructions
620 */
621
622 /*
623 * Check if AVX512F instruction set is available
624 */
625 static inline boolean_t
zfs_avx512f_available(void)626 zfs_avx512f_available(void)
627 {
628 return (boot_cpu_has(X86_FEATURE_AVX512F) && __zmm_enabled());
629 }
630
631 /*
632 * Check if AVX512CD instruction set is available
633 */
634 static inline boolean_t
zfs_avx512cd_available(void)635 zfs_avx512cd_available(void)
636 {
637 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512F));
638 }
639
640 /*
641 * Check if AVX512ER instruction set is available
642 */
643 static inline boolean_t
zfs_avx512er_available(void)644 zfs_avx512er_available(void)
645 {
646 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512ER));
647 }
648
649 /*
650 * Check if AVX512PF instruction set is available
651 */
652 static inline boolean_t
zfs_avx512pf_available(void)653 zfs_avx512pf_available(void)
654 {
655 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512PF));
656 }
657
658 /*
659 * Check if AVX512BW instruction set is available
660 */
661 static inline boolean_t
zfs_avx512bw_available(void)662 zfs_avx512bw_available(void)
663 {
664 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512BW));
665 }
666
667 /*
668 * Check if AVX512DQ instruction set is available
669 */
670 static inline boolean_t
zfs_avx512dq_available(void)671 zfs_avx512dq_available(void)
672 {
673 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512DQ));
674 }
675
676 /*
677 * Check if AVX512VL instruction set is available
678 */
679 static inline boolean_t
zfs_avx512vl_available(void)680 zfs_avx512vl_available(void)
681 {
682 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512VL));
683 }
684
685 /*
686 * Check if AVX512IFMA instruction set is available
687 */
688 static inline boolean_t
zfs_avx512ifma_available(void)689 zfs_avx512ifma_available(void)
690 {
691 return (zfs_avx512f_available() &&
692 boot_cpu_has(X86_FEATURE_AVX512IFMA));
693 }
694
695 /*
696 * Check if AVX512VBMI instruction set is available
697 */
698 static inline boolean_t
zfs_avx512vbmi_available(void)699 zfs_avx512vbmi_available(void)
700 {
701 return (zfs_avx512f_available() &&
702 boot_cpu_has(X86_FEATURE_AVX512VBMI));
703 }
704
705 #endif /* defined(__x86) */
706
707 #endif /* _LINUX_SIMD_X86_H */
708