xref: /linux/arch/x86/include/asm/fpu/types.h (revision c9f016e72b5cc7d4d68fac51f8e72c8c7a69c06e)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * FPU data structures:
4  */
5 #ifndef _ASM_X86_FPU_TYPES_H
6 #define _ASM_X86_FPU_TYPES_H
7 
8 #include <asm/page_types.h>
9 
10 /*
11  * The legacy x87 FPU state format, as saved by FSAVE and
12  * restored by the FRSTOR instructions:
13  */
14 struct fregs_state {
15 	u32			cwd;	/* FPU Control Word		*/
16 	u32			swd;	/* FPU Status Word		*/
17 	u32			twd;	/* FPU Tag Word			*/
18 	u32			fip;	/* FPU IP Offset		*/
19 	u32			fcs;	/* FPU IP Selector		*/
20 	u32			foo;	/* FPU Operand Pointer Offset	*/
21 	u32			fos;	/* FPU Operand Pointer Selector	*/
22 
23 	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
24 	u32			st_space[20];
25 
26 	/* Software status information [not touched by FSAVE]:		*/
27 	u32			status;
28 };
29 
30 /*
31  * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
32  * restored by the FXRSTOR instructions. It's similar to the FSAVE
33  * format, but differs in some areas, plus has extensions at
34  * the end for the XMM registers.
35  */
36 struct fxregs_state {
37 	u16			cwd; /* Control Word			*/
38 	u16			swd; /* Status Word			*/
39 	u16			twd; /* Tag Word			*/
40 	u16			fop; /* Last Instruction Opcode		*/
41 	union {
42 		struct {
43 			u64	rip; /* Instruction Pointer		*/
44 			u64	rdp; /* Data Pointer			*/
45 		};
46 		struct {
47 			u32	fip; /* FPU IP Offset			*/
48 			u32	fcs; /* FPU IP Selector			*/
49 			u32	foo; /* FPU Operand Offset		*/
50 			u32	fos; /* FPU Operand Selector		*/
51 		};
52 	};
53 	u32			mxcsr;		/* MXCSR Register State */
54 	u32			mxcsr_mask;	/* MXCSR Mask		*/
55 
56 	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
57 	u32			st_space[32];
58 
59 	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
60 	u32			xmm_space[64];
61 
62 	u32			padding[12];
63 
64 	union {
65 		u32		padding1[12];
66 		u32		sw_reserved[12];
67 	};
68 
69 } __attribute__((aligned(16)));
70 
71 /* Default value for fxregs_state.mxcsr: */
72 #define MXCSR_DEFAULT		0x1f80
73 
74 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
75 #define MXCSR_AND_FLAGS_SIZE sizeof(u64)
76 
77 /*
78  * Software based FPU emulation state. This is arbitrary really,
79  * it matches the x87 format to make it easier to understand:
80  */
81 struct swregs_state {
82 	u32			cwd;
83 	u32			swd;
84 	u32			twd;
85 	u32			fip;
86 	u32			fcs;
87 	u32			foo;
88 	u32			fos;
89 	/* 8*10 bytes for each FP-reg = 80 bytes: */
90 	u32			st_space[20];
91 	u8			ftop;
92 	u8			changed;
93 	u8			lookahead;
94 	u8			no_update;
95 	u8			rm;
96 	u8			alimit;
97 	struct math_emu_info	*info;
98 	u32			entry_eip;
99 };
100 
101 /*
102  * List of XSAVE features Linux knows about:
103  */
104 enum xfeature {
105 	XFEATURE_FP,
106 	XFEATURE_SSE,
107 	/*
108 	 * Values above here are "legacy states".
109 	 * Those below are "extended states".
110 	 */
111 	XFEATURE_YMM,
112 	XFEATURE_BNDREGS,
113 	XFEATURE_BNDCSR,
114 	XFEATURE_OPMASK,
115 	XFEATURE_ZMM_Hi256,
116 	XFEATURE_Hi16_ZMM,
117 	XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
118 	XFEATURE_PKRU,
119 	XFEATURE_PASID,
120 	XFEATURE_CET_USER,
121 	XFEATURE_CET_KERNEL_UNUSED,
122 	XFEATURE_RSRVD_COMP_13,
123 	XFEATURE_RSRVD_COMP_14,
124 	XFEATURE_LBR,
125 	XFEATURE_RSRVD_COMP_16,
126 	XFEATURE_XTILE_CFG,
127 	XFEATURE_XTILE_DATA,
128 
129 	XFEATURE_MAX,
130 };
131 
132 #define XFEATURE_MASK_FP		(1 << XFEATURE_FP)
133 #define XFEATURE_MASK_SSE		(1 << XFEATURE_SSE)
134 #define XFEATURE_MASK_YMM		(1 << XFEATURE_YMM)
135 #define XFEATURE_MASK_BNDREGS		(1 << XFEATURE_BNDREGS)
136 #define XFEATURE_MASK_BNDCSR		(1 << XFEATURE_BNDCSR)
137 #define XFEATURE_MASK_OPMASK		(1 << XFEATURE_OPMASK)
138 #define XFEATURE_MASK_ZMM_Hi256		(1 << XFEATURE_ZMM_Hi256)
139 #define XFEATURE_MASK_Hi16_ZMM		(1 << XFEATURE_Hi16_ZMM)
140 #define XFEATURE_MASK_PT		(1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
141 #define XFEATURE_MASK_PKRU		(1 << XFEATURE_PKRU)
142 #define XFEATURE_MASK_PASID		(1 << XFEATURE_PASID)
143 #define XFEATURE_MASK_CET_USER		(1 << XFEATURE_CET_USER)
144 #define XFEATURE_MASK_CET_KERNEL	(1 << XFEATURE_CET_KERNEL_UNUSED)
145 #define XFEATURE_MASK_LBR		(1 << XFEATURE_LBR)
146 #define XFEATURE_MASK_XTILE_CFG		(1 << XFEATURE_XTILE_CFG)
147 #define XFEATURE_MASK_XTILE_DATA	(1 << XFEATURE_XTILE_DATA)
148 
149 #define XFEATURE_MASK_FPSSE		(XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
150 #define XFEATURE_MASK_AVX512		(XFEATURE_MASK_OPMASK \
151 					 | XFEATURE_MASK_ZMM_Hi256 \
152 					 | XFEATURE_MASK_Hi16_ZMM)
153 
154 #ifdef CONFIG_X86_64
155 # define XFEATURE_MASK_XTILE		(XFEATURE_MASK_XTILE_DATA \
156 					 | XFEATURE_MASK_XTILE_CFG)
157 #else
158 # define XFEATURE_MASK_XTILE		(0)
159 #endif
160 
161 #define FIRST_EXTENDED_XFEATURE	XFEATURE_YMM
162 
163 struct reg_128_bit {
164 	u8      regbytes[128/8];
165 };
166 struct reg_256_bit {
167 	u8	regbytes[256/8];
168 };
169 struct reg_512_bit {
170 	u8	regbytes[512/8];
171 };
172 struct reg_1024_byte {
173 	u8	regbytes[1024];
174 };
175 
176 /*
177  * State component 2:
178  *
179  * There are 16x 256-bit AVX registers named YMM0-YMM15.
180  * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
181  * and are stored in 'struct fxregs_state::xmm_space[]' in the
182  * "legacy" area.
183  *
184  * The high 128 bits are stored here.
185  */
186 struct ymmh_struct {
187 	struct reg_128_bit              hi_ymm[16];
188 } __packed;
189 
190 /* Intel MPX support: */
191 
192 struct mpx_bndreg {
193 	u64				lower_bound;
194 	u64				upper_bound;
195 } __packed;
196 /*
197  * State component 3 is used for the 4 128-bit bounds registers
198  */
199 struct mpx_bndreg_state {
200 	struct mpx_bndreg		bndreg[4];
201 } __packed;
202 
203 /*
204  * State component 4 is used for the 64-bit user-mode MPX
205  * configuration register BNDCFGU and the 64-bit MPX status
206  * register BNDSTATUS.  We call the pair "BNDCSR".
207  */
208 struct mpx_bndcsr {
209 	u64				bndcfgu;
210 	u64				bndstatus;
211 } __packed;
212 
213 /*
214  * The BNDCSR state is padded out to be 64-bytes in size.
215  */
216 struct mpx_bndcsr_state {
217 	union {
218 		struct mpx_bndcsr		bndcsr;
219 		u8				pad_to_64_bytes[64];
220 	};
221 } __packed;
222 
223 /* AVX-512 Components: */
224 
225 /*
226  * State component 5 is used for the 8 64-bit opmask registers
227  * k0-k7 (opmask state).
228  */
229 struct avx_512_opmask_state {
230 	u64				opmask_reg[8];
231 } __packed;
232 
233 /*
234  * State component 6 is used for the upper 256 bits of the
235  * registers ZMM0-ZMM15. These 16 256-bit values are denoted
236  * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
237  */
238 struct avx_512_zmm_uppers_state {
239 	struct reg_256_bit		zmm_upper[16];
240 } __packed;
241 
242 /*
243  * State component 7 is used for the 16 512-bit registers
244  * ZMM16-ZMM31 (Hi16_ZMM state).
245  */
246 struct avx_512_hi16_state {
247 	struct reg_512_bit		hi16_zmm[16];
248 } __packed;
249 
250 /*
251  * State component 9: 32-bit PKRU register.  The state is
252  * 8 bytes long but only 4 bytes is used currently.
253  */
254 struct pkru_state {
255 	u32				pkru;
256 	u32				pad;
257 } __packed;
258 
259 /*
260  * State component 11 is Control-flow Enforcement user states
261  */
262 struct cet_user_state {
263 	/* user control-flow settings */
264 	u64 user_cet;
265 	/* user shadow stack pointer */
266 	u64 user_ssp;
267 };
268 
269 /*
270  * State component 15: Architectural LBR configuration state.
271  * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
272  */
273 
274 struct lbr_entry {
275 	u64 from;
276 	u64 to;
277 	u64 info;
278 };
279 
280 struct arch_lbr_state {
281 	u64 lbr_ctl;
282 	u64 lbr_depth;
283 	u64 ler_from;
284 	u64 ler_to;
285 	u64 ler_info;
286 	struct lbr_entry		entries[];
287 };
288 
289 /*
290  * State component 17: 64-byte tile configuration register.
291  */
292 struct xtile_cfg {
293 	u64				tcfg[8];
294 } __packed;
295 
296 /*
297  * State component 18: 1KB tile data register.
298  * Each register represents 16 64-byte rows of the matrix
299  * data. But the number of registers depends on the actual
300  * implementation.
301  */
302 struct xtile_data {
303 	struct reg_1024_byte		tmm;
304 } __packed;
305 
306 /*
307  * State component 10 is supervisor state used for context-switching the
308  * PASID state.
309  */
310 struct ia32_pasid_state {
311 	u64 pasid;
312 } __packed;
313 
314 struct xstate_header {
315 	u64				xfeatures;
316 	u64				xcomp_bv;
317 	u64				reserved[6];
318 } __attribute__((packed));
319 
320 /*
321  * xstate_header.xcomp_bv[63] indicates that the extended_state_area
322  * is in compacted format.
323  */
324 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
325 
326 /*
327  * This is our most modern FPU state format, as saved by the XSAVE
328  * and restored by the XRSTOR instructions.
329  *
330  * It consists of a legacy fxregs portion, an xstate header and
331  * subsequent areas as defined by the xstate header.  Not all CPUs
332  * support all the extensions, so the size of the extended area
333  * can vary quite a bit between CPUs.
334  */
335 struct xregs_state {
336 	struct fxregs_state		i387;
337 	struct xstate_header		header;
338 	u8				extended_state_area[];
339 } __attribute__ ((packed, aligned (64)));
340 
341 /*
342  * This is a union of all the possible FPU state formats
343  * put together, so that we can pick the right one runtime.
344  *
345  * The size of the structure is determined by the largest
346  * member - which is the xsave area.  The padding is there
347  * to ensure that statically-allocated task_structs (just
348  * the init_task today) have enough space.
349  */
350 union fpregs_state {
351 	struct fregs_state		fsave;
352 	struct fxregs_state		fxsave;
353 	struct swregs_state		soft;
354 	struct xregs_state		xsave;
355 	u8 __padding[PAGE_SIZE];
356 };
357 
358 struct fpstate {
359 	/* @kernel_size: The size of the kernel register image */
360 	unsigned int		size;
361 
362 	/* @user_size: The size in non-compacted UABI format */
363 	unsigned int		user_size;
364 
365 	/* @xfeatures:		xfeatures for which the storage is sized */
366 	u64			xfeatures;
367 
368 	/* @user_xfeatures:	xfeatures valid in UABI buffers */
369 	u64			user_xfeatures;
370 
371 	/* @xfd:		xfeatures disabled to trap userspace use. */
372 	u64			xfd;
373 
374 	/* @is_valloc:		Indicator for dynamically allocated state */
375 	unsigned int		is_valloc	: 1;
376 
377 	/* @is_guest:		Indicator for guest state (KVM) */
378 	unsigned int		is_guest	: 1;
379 
380 	/*
381 	 * @is_confidential:	Indicator for KVM confidential mode.
382 	 *			The FPU registers are restored by the
383 	 *			vmentry firmware from encrypted guest
384 	 *			memory. On vmexit the FPU registers are
385 	 *			saved by firmware to encrypted guest memory
386 	 *			and the registers are scrubbed before
387 	 *			returning to the host. So there is no
388 	 *			content which is worth saving and restoring.
389 	 *			The fpstate has to be there so that
390 	 *			preemption and softirq FPU usage works
391 	 *			without special casing.
392 	 */
393 	unsigned int		is_confidential	: 1;
394 
395 	/* @in_use:		State is in use */
396 	unsigned int		in_use		: 1;
397 
398 	/* @regs: The register state union for all supported formats */
399 	union fpregs_state	regs;
400 
401 	/* @regs is dynamically sized! Don't add anything after @regs! */
402 } __aligned(64);
403 
404 #define FPU_GUEST_PERM_LOCKED		BIT_ULL(63)
405 
406 struct fpu_state_perm {
407 	/*
408 	 * @__state_perm:
409 	 *
410 	 * This bitmap indicates the permission for state components, which
411 	 * are available to a thread group. The permission prctl() sets the
412 	 * enabled state bits in thread_group_leader()->thread.fpu.
413 	 *
414 	 * All run time operations use the per thread information in the
415 	 * currently active fpu.fpstate which contains the xfeature masks
416 	 * and sizes for kernel and user space.
417 	 *
418 	 * This master permission field is only to be used when
419 	 * task.fpu.fpstate based checks fail to validate whether the task
420 	 * is allowed to expand its xfeatures set which requires to
421 	 * allocate a larger sized fpstate buffer.
422 	 *
423 	 * Do not access this field directly.  Use the provided helper
424 	 * function. Unlocked access is possible for quick checks.
425 	 */
426 	u64				__state_perm;
427 
428 	/*
429 	 * @__state_size:
430 	 *
431 	 * The size required for @__state_perm. Only valid to access
432 	 * with sighand locked.
433 	 */
434 	unsigned int			__state_size;
435 
436 	/*
437 	 * @__user_state_size:
438 	 *
439 	 * The size required for @__state_perm user part. Only valid to
440 	 * access with sighand locked.
441 	 */
442 	unsigned int			__user_state_size;
443 };
444 
445 /*
446  * Highest level per task FPU state data structure that
447  * contains the FPU register state plus various FPU
448  * state fields:
449  */
450 struct fpu {
451 	/*
452 	 * @last_cpu:
453 	 *
454 	 * Records the last CPU on which this context was loaded into
455 	 * FPU registers. (In the lazy-restore case we might be
456 	 * able to reuse FPU registers across multiple context switches
457 	 * this way, if no intermediate task used the FPU.)
458 	 *
459 	 * A value of -1 is used to indicate that the FPU state in context
460 	 * memory is newer than the FPU state in registers, and that the
461 	 * FPU state should be reloaded next time the task is run.
462 	 */
463 	unsigned int			last_cpu;
464 
465 	/*
466 	 * @avx512_timestamp:
467 	 *
468 	 * Records the timestamp of AVX512 use during last context switch.
469 	 */
470 	unsigned long			avx512_timestamp;
471 
472 	/*
473 	 * @fpstate:
474 	 *
475 	 * Pointer to the active struct fpstate. Initialized to
476 	 * point at @__fpstate below.
477 	 */
478 	struct fpstate			*fpstate;
479 
480 	/*
481 	 * @__task_fpstate:
482 	 *
483 	 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
484 	 * used only for KVM support to swap out the regular task fpstate.
485 	 */
486 	struct fpstate			*__task_fpstate;
487 
488 	/*
489 	 * @perm:
490 	 *
491 	 * Permission related information
492 	 */
493 	struct fpu_state_perm		perm;
494 
495 	/*
496 	 * @guest_perm:
497 	 *
498 	 * Permission related information for guest pseudo FPUs
499 	 */
500 	struct fpu_state_perm		guest_perm;
501 
502 	/*
503 	 * @__fpstate:
504 	 *
505 	 * Initial in-memory storage for FPU registers which are saved in
506 	 * context switch and when the kernel uses the FPU. The registers
507 	 * are restored from this storage on return to user space if they
508 	 * are not longer containing the tasks FPU register state.
509 	 */
510 	struct fpstate			__fpstate;
511 	/*
512 	 * WARNING: '__fpstate' is dynamically-sized.  Do not put
513 	 * anything after it here.
514 	 */
515 };
516 
517 /*
518  * Guest pseudo FPU container
519  */
520 struct fpu_guest {
521 	/*
522 	 * @xfeatures:			xfeature bitmap of features which are
523 	 *				currently enabled for the guest vCPU.
524 	 */
525 	u64				xfeatures;
526 
527 	/*
528 	 * @perm:			xfeature bitmap of features which are
529 	 *				permitted to be enabled for the guest
530 	 *				vCPU.
531 	 */
532 	u64				perm;
533 
534 	/*
535 	 * @xfd_err:			Save the guest value.
536 	 */
537 	u64				xfd_err;
538 
539 	/*
540 	 * @uabi_size:			Size required for save/restore
541 	 */
542 	unsigned int			uabi_size;
543 
544 	/*
545 	 * @fpstate:			Pointer to the allocated guest fpstate
546 	 */
547 	struct fpstate			*fpstate;
548 };
549 
550 /*
551  * FPU state configuration data. Initialized at boot time. Read only after init.
552  */
553 struct fpu_state_config {
554 	/*
555 	 * @max_size:
556 	 *
557 	 * The maximum size of the register state buffer. Includes all
558 	 * supported features except independent managed features.
559 	 */
560 	unsigned int		max_size;
561 
562 	/*
563 	 * @default_size:
564 	 *
565 	 * The default size of the register state buffer. Includes all
566 	 * supported features except independent managed features and
567 	 * features which have to be requested by user space before usage.
568 	 */
569 	unsigned int		default_size;
570 
571 	/*
572 	 * @max_features:
573 	 *
574 	 * The maximum supported features bitmap. Does not include
575 	 * independent managed features.
576 	 */
577 	u64 max_features;
578 
579 	/*
580 	 * @default_features:
581 	 *
582 	 * The default supported features bitmap. Does not include
583 	 * independent managed features and features which have to
584 	 * be requested by user space before usage.
585 	 */
586 	u64 default_features;
587 	/*
588 	 * @legacy_features:
589 	 *
590 	 * Features which can be reported back to user space
591 	 * even without XSAVE support, i.e. legacy features FP + SSE
592 	 */
593 	u64 legacy_features;
594 	/*
595 	 * @independent_features:
596 	 *
597 	 * Features that are supported by XSAVES, but not managed as part of
598 	 * the FPU core, such as LBR
599 	 */
600 	u64 independent_features;
601 };
602 
603 /* FPU state configuration information */
604 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
605 
606 #endif /* _ASM_X86_FPU_TYPES_H */
607