1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>.
24 * Copyright (c) 2026, TrueNAS.
25 */
26
27 /*
28 * USER API:
29 *
30 * Kernel fpu methods:
31 * kfpu_allowed()
32 * kfpu_begin()
33 * kfpu_end()
34 * kfpu_init()
35 * kfpu_fini()
36 *
37 * SIMD support:
38 *
39 * Following functions should be called to determine whether CPU feature
40 * is supported. All functions are usable in kernel and user space.
41 * If a SIMD algorithm is using more than one instruction set
42 * all relevant feature test functions should be called.
43 *
44 * Supported features:
45 * zfs_sse_available()
46 * zfs_sse2_available()
47 * zfs_sse3_available()
48 * zfs_ssse3_available()
49 * zfs_sse4_1_available()
50 * zfs_sse4_2_available()
51 *
52 * zfs_avx_available()
53 * zfs_avx2_available()
54 *
55 * zfs_bmi1_available()
56 * zfs_bmi2_available()
57 *
58 * zfs_shani_available()
59 *
60 * zfs_avx512f_available()
61 * zfs_avx512cd_available()
62 * zfs_avx512er_available()
63 * zfs_avx512pf_available()
64 * zfs_avx512bw_available()
65 * zfs_avx512dq_available()
66 * zfs_avx512vl_available()
67 * zfs_avx512ifma_available()
68 * zfs_avx512vbmi_available()
69 *
70 * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers
71 * also add zfs_avx512vl_available() to feature check.
72 */
73
74 #ifndef _LINUX_SIMD_X86_H
75 #define _LINUX_SIMD_X86_H
76
77 /* only for __x86 */
78 #if defined(__x86)
79
80 #include <sys/types.h>
81 #include <asm/cpufeature.h>
82
83 /*
84 * Disable the WARN_ON_FPU() macro to prevent additional dependencies
85 * when providing the kfpu_* functions. Relevant warnings are included
86 * as appropriate and are unconditionally enabled.
87 */
88 #if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU)
89 #undef CONFIG_X86_DEBUG_FPU
90 #endif
91
92 /*
93 * The following cases are for kernels which export either the
94 * kernel_fpu_* or __kernel_fpu_* functions.
95 */
96 #if defined(KERNEL_EXPORTS_X86_FPU)
97
98 #if defined(HAVE_KERNEL_FPU_API_HEADER)
99 #include <asm/fpu/api.h>
100 #if defined(HAVE_KERNEL_FPU_INTERNAL_HEADER)
101 #include <asm/fpu/internal.h>
102 #endif
103 #else
104 #include <asm/i387.h>
105 #endif
106
107 #define kfpu_allowed() 1
108 #define kfpu_init() 0
109 #define kfpu_fini() ((void) 0)
110
111 #if defined(HAVE_UNDERSCORE_KERNEL_FPU)
112 #define kfpu_begin() \
113 { \
114 preempt_disable(); \
115 __kernel_fpu_begin(); \
116 }
117 #define kfpu_end() \
118 { \
119 __kernel_fpu_end(); \
120 preempt_enable(); \
121 }
122
123 #elif defined(HAVE_KERNEL_FPU)
124 #define kfpu_begin() kernel_fpu_begin()
125 #define kfpu_end() kernel_fpu_end()
126
127 #else
128 /*
129 * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then
130 * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined.
131 */
132 #error "Unreachable kernel configuration"
133 #endif
134
135 #else /* defined(KERNEL_EXPORTS_X86_FPU) */
136
137 /*
138 * When the kernel_fpu_* symbols are unavailable then provide our own
139 * versions which allow the FPU to be safely used.
140 */
141 #if defined(HAVE_KERNEL_FPU_INTERNAL)
142
143 #ifndef XFEATURE_MASK_XTILE
144 /*
145 * For kernels where this doesn't exist yet, we still don't want to break
146 * by save/restoring this broken nonsense.
147 * See issue #14989 or Intel errata SPR4 for why
148 */
149 #define XFEATURE_MASK_XTILE 0x60000
150 #endif
151
152 #include <linux/mm.h>
153 #include <linux/slab.h>
154
155 extern uint8_t **zfs_kfpu_fpregs;
156
157 /*
158 * Return the size in bytes required by the XSAVE instruction for an
159 * XSAVE area containing all the user state components supported by this CPU.
160 * See: Intel 64 and IA-32 Architectures Software Developer’s Manual.
161 * Dec. 2021. Vol. 2A p. 3-222.
162 */
163 static inline uint32_t
get_xsave_area_size(void)164 get_xsave_area_size(void)
165 {
166 if (!boot_cpu_has(X86_FEATURE_OSXSAVE)) {
167 return (0);
168 }
169 /*
170 * Call CPUID with leaf 13 and subleaf 0. The size is in ecx.
171 * We don't need to check for cpuid_max here, since if this CPU has
172 * OSXSAVE set, it has leaf 13 (0x0D) as well.
173 */
174 uint32_t eax, ebx, ecx, edx;
175
176 eax = 13U;
177 ecx = 0U;
178 __asm__ __volatile__("cpuid"
179 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
180 : "a" (eax), "c" (ecx));
181
182 return (ecx);
183 }
184
185 /*
186 * Return the allocation order of the maximum buffer size required to save the
187 * FPU state on this architecture. The value returned is the same as Linux'
188 * get_order() function would return (i.e. 2^order = nr. of pages required).
189 * Currently this will always return 0 since the save area is below 4k even for
190 * a full fledged AVX-512 implementation.
191 */
192 static inline int
get_fpuregs_save_area_order(void)193 get_fpuregs_save_area_order(void)
194 {
195 size_t area_size = (size_t)get_xsave_area_size();
196
197 /*
198 * If we are dealing with a CPU not supporting XSAVE,
199 * get_xsave_area_size() will return 0. Thus the maximum memory
200 * required is the FXSAVE area size which is 512 bytes. See: Intel 64
201 * and IA-32 Architectures Software Developer’s Manual. Dec. 2021.
202 * Vol. 2A p. 3-451.
203 */
204 if (area_size == 0) {
205 area_size = 512;
206 }
207 return (get_order(area_size));
208 }
209
210 /*
211 * Initialize per-cpu variables to store FPU state.
212 */
213 static inline void
kfpu_fini(void)214 kfpu_fini(void)
215 {
216 int cpu;
217 int order = get_fpuregs_save_area_order();
218
219 for_each_possible_cpu(cpu) {
220 if (zfs_kfpu_fpregs[cpu] != NULL) {
221 free_pages((unsigned long)zfs_kfpu_fpregs[cpu], order);
222 }
223 }
224
225 kfree(zfs_kfpu_fpregs);
226 }
227
228 static inline int
kfpu_init(void)229 kfpu_init(void)
230 {
231 zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (uint8_t *),
232 GFP_KERNEL);
233
234 if (zfs_kfpu_fpregs == NULL)
235 return (-ENOMEM);
236
237 /*
238 * The fxsave and xsave operations require 16-/64-byte alignment of
239 * the target memory. Since kmalloc() provides no alignment
240 * guarantee instead use alloc_pages_node().
241 */
242 int cpu;
243 int order = get_fpuregs_save_area_order();
244
245 for_each_possible_cpu(cpu) {
246 struct page *page = alloc_pages_node(cpu_to_node(cpu),
247 GFP_KERNEL | __GFP_ZERO, order);
248 if (page == NULL) {
249 kfpu_fini();
250 return (-ENOMEM);
251 }
252
253 zfs_kfpu_fpregs[cpu] = page_address(page);
254 }
255
256 return (0);
257 }
258
259 #define kfpu_allowed() 1
260
261 /*
262 * FPU save and restore instructions.
263 */
264 #define __asm __asm__ __volatile__
265 #define kfpu_fxsave(addr) __asm("fxsave %0" : "=m" (*(addr)))
266 #define kfpu_fxsaveq(addr) __asm("fxsaveq %0" : "=m" (*(addr)))
267 #define kfpu_fnsave(addr) __asm("fnsave %0; fwait" : "=m" (*(addr)))
268 #define kfpu_fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr)))
269 #define kfpu_fxrstorq(addr) __asm("fxrstorq %0" : : "m" (*(addr)))
270 #define kfpu_frstor(addr) __asm("frstor %0" : : "m" (*(addr)))
271 #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \
272 : : [addr] "m" (rval));
273
274 #define kfpu_do_xsave(instruction, addr, mask) \
275 { \
276 uint32_t low, hi; \
277 \
278 low = mask; \
279 hi = (uint64_t)(mask) >> 32; \
280 __asm(instruction " %[dst]\n\t" \
281 : \
282 : [dst] "m" (*(addr)), "a" (low), "d" (hi) \
283 : "memory"); \
284 }
285
286 static inline void
kfpu_save_fxsr(uint8_t * addr)287 kfpu_save_fxsr(uint8_t *addr)
288 {
289 if (IS_ENABLED(CONFIG_X86_32))
290 kfpu_fxsave(addr);
291 else
292 kfpu_fxsaveq(addr);
293 }
294
295 static inline void
kfpu_save_fsave(uint8_t * addr)296 kfpu_save_fsave(uint8_t *addr)
297 {
298 kfpu_fnsave(addr);
299 }
300
301 static inline void
kfpu_begin(void)302 kfpu_begin(void)
303 {
304 /*
305 * Preemption and interrupts must be disabled for the critical
306 * region where the FPU state is being modified.
307 */
308 preempt_disable();
309 local_irq_disable();
310
311 /*
312 * The current FPU registers need to be preserved by kfpu_begin()
313 * and restored by kfpu_end(). They are stored in a dedicated
314 * per-cpu variable, not in the task struct, this allows any user
315 * FPU state to be correctly preserved and restored.
316 */
317 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
318 #if HAVE_SIMD(XSAVES)
319 if (static_cpu_has(X86_FEATURE_XSAVES)) {
320 kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE);
321 return;
322 }
323 #endif
324 #if HAVE_SIMD(XSAVEOPT)
325 if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
326 kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE);
327 return;
328 }
329 #endif
330 #if HAVE_SIMD(XSAVE)
331 if (static_cpu_has(X86_FEATURE_XSAVE)) {
332 kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE);
333 return;
334 }
335 #endif
336 if (static_cpu_has(X86_FEATURE_FXSR)) {
337 kfpu_save_fxsr(state);
338 } else {
339 kfpu_save_fsave(state);
340 }
341 }
342
343 #define kfpu_do_xrstor(instruction, addr, mask) \
344 { \
345 uint32_t low, hi; \
346 \
347 low = mask; \
348 hi = (uint64_t)(mask) >> 32; \
349 __asm(instruction " %[src]" \
350 : \
351 : [src] "m" (*(addr)), "a" (low), "d" (hi) \
352 : "memory"); \
353 }
354
355 static inline void
kfpu_restore_fxsr(uint8_t * addr)356 kfpu_restore_fxsr(uint8_t *addr)
357 {
358 /*
359 * On AuthenticAMD K7 and K8 processors the fxrstor instruction only
360 * restores the _x87 FOP, FIP, and FDP registers when an exception
361 * is pending. Clean the _x87 state to force the restore.
362 */
363 if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK)))
364 kfpu_fxsr_clean(addr);
365
366 if (IS_ENABLED(CONFIG_X86_32)) {
367 kfpu_fxrstor(addr);
368 } else {
369 kfpu_fxrstorq(addr);
370 }
371 }
372
373 static inline void
kfpu_restore_fsave(uint8_t * addr)374 kfpu_restore_fsave(uint8_t *addr)
375 {
376 kfpu_frstor(addr);
377 }
378
379 static inline void
kfpu_end(void)380 kfpu_end(void)
381 {
382 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
383 #if HAVE_SIMD(XSAVES)
384 if (static_cpu_has(X86_FEATURE_XSAVES)) {
385 kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE);
386 goto out;
387 }
388 #endif
389 #if HAVE_SIMD(XSAVE)
390 if (static_cpu_has(X86_FEATURE_XSAVE)) {
391 kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE);
392 goto out;
393 }
394 #endif
395 if (static_cpu_has(X86_FEATURE_FXSR)) {
396 kfpu_restore_fxsr(state);
397 } else {
398 kfpu_restore_fsave(state);
399 }
400 out:
401 local_irq_enable();
402 preempt_enable();
403
404 }
405
406 #else
407
408 #error "Exactly one of KERNEL_EXPORTS_X86_FPU or HAVE_KERNEL_FPU_INTERNAL" \
409 " must be defined"
410
411 #endif /* defined(HAVE_KERNEL_FPU_INTERNAL */
412 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */
413
414 /*
415 * Linux kernel provides an interface for CPU feature testing.
416 */
417
418 /*
419 * Detect register set support
420 */
421
422 /*
423 * Check if OS supports AVX and AVX2 by checking XCR0
424 * Only call this function if CPUID indicates that AVX feature is
425 * supported by the CPU, otherwise it might be an illegal instruction.
426 */
427 static inline uint64_t
zfs_xgetbv(uint32_t index)428 zfs_xgetbv(uint32_t index)
429 {
430 uint32_t eax, edx;
431 /* xgetbv - instruction byte code */
432 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0"
433 : "=a" (eax), "=d" (edx)
434 : "c" (index));
435
436 return ((((uint64_t)edx)<<32) | (uint64_t)eax);
437 }
438
439
440 static inline boolean_t
__simd_state_enabled(const uint64_t state)441 __simd_state_enabled(const uint64_t state)
442 {
443 uint64_t xcr0;
444
445 if (!boot_cpu_has(X86_FEATURE_OSXSAVE))
446 return (B_FALSE);
447
448 xcr0 = zfs_xgetbv(0);
449 return ((xcr0 & state) == state);
450 }
451
452 #define _XSTATE_SSE_AVX (0x2 | 0x4)
453 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX)
454
455 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
456 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)
457
458 /*
459 * Check if SSE instruction set is available
460 */
461 static inline boolean_t
zfs_sse_available(void)462 zfs_sse_available(void)
463 {
464 return (!!boot_cpu_has(X86_FEATURE_XMM));
465 }
466
467 /*
468 * Check if SSE2 instruction set is available
469 */
470 static inline boolean_t
zfs_sse2_available(void)471 zfs_sse2_available(void)
472 {
473 return (!!boot_cpu_has(X86_FEATURE_XMM2));
474 }
475
476 /*
477 * Check if SSE3 instruction set is available
478 */
479 static inline boolean_t
zfs_sse3_available(void)480 zfs_sse3_available(void)
481 {
482 return (!!boot_cpu_has(X86_FEATURE_XMM3));
483 }
484
485 /*
486 * Check if SSSE3 instruction set is available
487 */
488 static inline boolean_t
zfs_ssse3_available(void)489 zfs_ssse3_available(void)
490 {
491 return (!!boot_cpu_has(X86_FEATURE_SSSE3));
492 }
493
494 /*
495 * Check if SSE4.1 instruction set is available
496 */
497 static inline boolean_t
zfs_sse4_1_available(void)498 zfs_sse4_1_available(void)
499 {
500 return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
501 }
502
503 /*
504 * Check if SSE4.2 instruction set is available
505 */
506 static inline boolean_t
zfs_sse4_2_available(void)507 zfs_sse4_2_available(void)
508 {
509 return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
510 }
511
512 /*
513 * Check if AVX instruction set is available
514 */
515 static inline boolean_t
zfs_avx_available(void)516 zfs_avx_available(void)
517 {
518 return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled());
519 }
520
521 /*
522 * Check if AVX2 instruction set is available
523 */
524 static inline boolean_t
zfs_avx2_available(void)525 zfs_avx2_available(void)
526 {
527 return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled());
528 }
529
530 /*
531 * Check if BMI1 instruction set is available
532 */
533 static inline boolean_t
zfs_bmi1_available(void)534 zfs_bmi1_available(void)
535 {
536 return (!!boot_cpu_has(X86_FEATURE_BMI1));
537 }
538
539 /*
540 * Check if BMI2 instruction set is available
541 */
542 static inline boolean_t
zfs_bmi2_available(void)543 zfs_bmi2_available(void)
544 {
545 return (!!boot_cpu_has(X86_FEATURE_BMI2));
546 }
547
548 /*
549 * Check if AES instruction set is available
550 */
551 static inline boolean_t
zfs_aes_available(void)552 zfs_aes_available(void)
553 {
554 return (!!boot_cpu_has(X86_FEATURE_AES));
555 }
556
557 /*
558 * Check if PCLMULQDQ instruction set is available
559 */
560 static inline boolean_t
zfs_pclmulqdq_available(void)561 zfs_pclmulqdq_available(void)
562 {
563 return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
564 }
565
566 /*
567 * Check if MOVBE instruction is available
568 */
569 static inline boolean_t
zfs_movbe_available(void)570 zfs_movbe_available(void)
571 {
572 return (!!boot_cpu_has(X86_FEATURE_MOVBE));
573 }
574
575 /*
576 * Check if VAES instruction set is available
577 */
578 static inline boolean_t
zfs_vaes_available(void)579 zfs_vaes_available(void)
580 {
581 return (!!boot_cpu_has(X86_FEATURE_VAES));
582 }
583
584 /*
585 * Check if VPCLMULQDQ instruction set is available
586 */
587 static inline boolean_t
zfs_vpclmulqdq_available(void)588 zfs_vpclmulqdq_available(void)
589 {
590 return (!!boot_cpu_has(X86_FEATURE_VPCLMULQDQ));
591 }
592
593 /*
594 * Check if SHA512 instructions are available
595 * Kernel added X86_FEATURE_SHA512 in 6.13 (torvalds/linux@a0423af92cb3)
596 */
597 static inline boolean_t
zfs_sha512ext_available(void)598 zfs_sha512ext_available(void)
599 {
600 #if defined(X86_FEATURE_SHA512)
601 return (!!boot_cpu_has(X86_FEATURE_SHA512));
602 #else
603 return (B_FALSE);
604 #endif
605 }
606
607 /*
608 * Check if SHA_NI instruction set is available
609 */
610 static inline boolean_t
zfs_shani_available(void)611 zfs_shani_available(void)
612 {
613 return (!!boot_cpu_has(X86_FEATURE_SHA_NI));
614 }
615
616 /*
617 * AVX-512 family of instruction sets:
618 *
619 * AVX512F Foundation
620 * AVX512CD Conflict Detection Instructions
621 * AVX512ER Exponential and Reciprocal Instructions
622 * AVX512PF Prefetch Instructions
623 *
624 * AVX512BW Byte and Word Instructions
625 * AVX512DQ Double-word and Quadword Instructions
626 * AVX512VL Vector Length Extensions
627 *
628 * AVX512IFMA Integer Fused Multiply Add
629 * AVX512VBMI Vector Byte Manipulation Instructions
630 */
631
632 /*
633 * Check if AVX512F instruction set is available
634 */
635 static inline boolean_t
zfs_avx512f_available(void)636 zfs_avx512f_available(void)
637 {
638 return (boot_cpu_has(X86_FEATURE_AVX512F) && __zmm_enabled());
639 }
640
641 /*
642 * Check if AVX512CD instruction set is available
643 */
644 static inline boolean_t
zfs_avx512cd_available(void)645 zfs_avx512cd_available(void)
646 {
647 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512F));
648 }
649
650 /*
651 * Check if AVX512ER instruction set is available
652 */
653 static inline boolean_t
zfs_avx512er_available(void)654 zfs_avx512er_available(void)
655 {
656 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512ER));
657 }
658
659 /*
660 * Check if AVX512PF instruction set is available
661 */
662 static inline boolean_t
zfs_avx512pf_available(void)663 zfs_avx512pf_available(void)
664 {
665 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512PF));
666 }
667
668 /*
669 * Check if AVX512BW instruction set is available
670 */
671 static inline boolean_t
zfs_avx512bw_available(void)672 zfs_avx512bw_available(void)
673 {
674 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512BW));
675 }
676
677 /*
678 * Check if AVX512DQ instruction set is available
679 */
680 static inline boolean_t
zfs_avx512dq_available(void)681 zfs_avx512dq_available(void)
682 {
683 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512DQ));
684 }
685
686 /*
687 * Check if AVX512VL instruction set is available
688 */
689 static inline boolean_t
zfs_avx512vl_available(void)690 zfs_avx512vl_available(void)
691 {
692 return (zfs_avx512f_available() && boot_cpu_has(X86_FEATURE_AVX512VL));
693 }
694
695 /*
696 * Check if AVX512IFMA instruction set is available
697 */
698 static inline boolean_t
zfs_avx512ifma_available(void)699 zfs_avx512ifma_available(void)
700 {
701 return (zfs_avx512f_available() &&
702 boot_cpu_has(X86_FEATURE_AVX512IFMA));
703 }
704
705 /*
706 * Check if AVX512VBMI instruction set is available
707 */
708 static inline boolean_t
zfs_avx512vbmi_available(void)709 zfs_avx512vbmi_available(void)
710 {
711 return (zfs_avx512f_available() &&
712 boot_cpu_has(X86_FEATURE_AVX512VBMI));
713 }
714
715 #endif /* defined(__x86) */
716
717 #endif /* _LINUX_SIMD_X86_H */
718