1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * FPU data structures: 4 */ 5 #ifndef _ASM_X86_FPU_TYPES_H 6 #define _ASM_X86_FPU_TYPES_H 7 8 #include <asm/page_types.h> 9 10 /* 11 * The legacy x87 FPU state format, as saved by FSAVE and 12 * restored by the FRSTOR instructions: 13 */ 14 struct fregs_state { 15 u32 cwd; /* FPU Control Word */ 16 u32 swd; /* FPU Status Word */ 17 u32 twd; /* FPU Tag Word */ 18 u32 fip; /* FPU IP Offset */ 19 u32 fcs; /* FPU IP Selector */ 20 u32 foo; /* FPU Operand Pointer Offset */ 21 u32 fos; /* FPU Operand Pointer Selector */ 22 23 /* 8*10 bytes for each FP-reg = 80 bytes: */ 24 u32 st_space[20]; 25 26 /* Software status information [not touched by FSAVE]: */ 27 u32 status; 28 }; 29 30 /* 31 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and 32 * restored by the FXRSTOR instructions. It's similar to the FSAVE 33 * format, but differs in some areas, plus has extensions at 34 * the end for the XMM registers. 35 */ 36 struct fxregs_state { 37 u16 cwd; /* Control Word */ 38 u16 swd; /* Status Word */ 39 u16 twd; /* Tag Word */ 40 u16 fop; /* Last Instruction Opcode */ 41 union { 42 struct { 43 u64 rip; /* Instruction Pointer */ 44 u64 rdp; /* Data Pointer */ 45 }; 46 struct { 47 u32 fip; /* FPU IP Offset */ 48 u32 fcs; /* FPU IP Selector */ 49 u32 foo; /* FPU Operand Offset */ 50 u32 fos; /* FPU Operand Selector */ 51 }; 52 }; 53 u32 mxcsr; /* MXCSR Register State */ 54 u32 mxcsr_mask; /* MXCSR Mask */ 55 56 /* 8*16 bytes for each FP-reg = 128 bytes: */ 57 u32 st_space[32]; 58 59 /* 16*16 bytes for each XMM-reg = 256 bytes: */ 60 u32 xmm_space[64]; 61 62 u32 padding[12]; 63 64 union { 65 u32 padding1[12]; 66 u32 sw_reserved[12]; 67 }; 68 69 } __attribute__((aligned(16))); 70 71 /* Default value for fxregs_state.mxcsr: */ 72 #define MXCSR_DEFAULT 0x1f80 73 74 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */ 75 #define MXCSR_AND_FLAGS_SIZE sizeof(u64) 76 77 /* 78 * Software based FPU emulation state. This is arbitrary really, 79 * it matches the x87 format to make it easier to understand: 80 */ 81 struct swregs_state { 82 u32 cwd; 83 u32 swd; 84 u32 twd; 85 u32 fip; 86 u32 fcs; 87 u32 foo; 88 u32 fos; 89 /* 8*10 bytes for each FP-reg = 80 bytes: */ 90 u32 st_space[20]; 91 u8 ftop; 92 u8 changed; 93 u8 lookahead; 94 u8 no_update; 95 u8 rm; 96 u8 alimit; 97 struct math_emu_info *info; 98 u32 entry_eip; 99 }; 100 101 /* 102 * List of XSAVE features Linux knows about: 103 */ 104 enum xfeature { 105 XFEATURE_FP, 106 XFEATURE_SSE, 107 /* 108 * Values above here are "legacy states". 109 * Those below are "extended states". 110 */ 111 XFEATURE_YMM, 112 XFEATURE_BNDREGS, 113 XFEATURE_BNDCSR, 114 XFEATURE_OPMASK, 115 XFEATURE_ZMM_Hi256, 116 XFEATURE_Hi16_ZMM, 117 XFEATURE_PT_UNIMPLEMENTED_SO_FAR, 118 XFEATURE_PKRU, 119 XFEATURE_PASID, 120 XFEATURE_CET_USER, 121 XFEATURE_CET_KERNEL, 122 XFEATURE_RSRVD_COMP_13, 123 XFEATURE_RSRVD_COMP_14, 124 XFEATURE_LBR, 125 XFEATURE_RSRVD_COMP_16, 126 XFEATURE_XTILE_CFG, 127 XFEATURE_XTILE_DATA, 128 XFEATURE_APX, 129 130 XFEATURE_MAX, 131 }; 132 133 #define XFEATURE_MASK_FP (1 << XFEATURE_FP) 134 #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) 135 #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) 136 #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) 137 #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) 138 #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) 139 #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) 140 #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) 141 #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) 142 #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) 143 #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) 144 #define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) 145 #define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL) 146 #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) 147 #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) 148 #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) 149 #define XFEATURE_MASK_APX (1 << XFEATURE_APX) 150 151 #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) 152 #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ 153 | XFEATURE_MASK_ZMM_Hi256 \ 154 | XFEATURE_MASK_Hi16_ZMM) 155 156 #ifdef CONFIG_X86_64 157 # define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ 158 | XFEATURE_MASK_XTILE_CFG) 159 #else 160 # define XFEATURE_MASK_XTILE (0) 161 #endif 162 163 #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM 164 165 struct reg_128_bit { 166 u8 regbytes[128/8]; 167 }; 168 struct reg_256_bit { 169 u8 regbytes[256/8]; 170 }; 171 struct reg_512_bit { 172 u8 regbytes[512/8]; 173 }; 174 struct reg_1024_byte { 175 u8 regbytes[1024]; 176 }; 177 178 /* 179 * State component 2: 180 * 181 * There are 16x 256-bit AVX registers named YMM0-YMM15. 182 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) 183 * and are stored in 'struct fxregs_state::xmm_space[]' in the 184 * "legacy" area. 185 * 186 * The high 128 bits are stored here. 187 */ 188 struct ymmh_struct { 189 struct reg_128_bit hi_ymm[16]; 190 } __packed; 191 192 /* Intel MPX support: */ 193 194 struct mpx_bndreg { 195 u64 lower_bound; 196 u64 upper_bound; 197 } __packed; 198 /* 199 * State component 3 is used for the 4 128-bit bounds registers 200 */ 201 struct mpx_bndreg_state { 202 struct mpx_bndreg bndreg[4]; 203 } __packed; 204 205 /* 206 * State component 4 is used for the 64-bit user-mode MPX 207 * configuration register BNDCFGU and the 64-bit MPX status 208 * register BNDSTATUS. We call the pair "BNDCSR". 209 */ 210 struct mpx_bndcsr { 211 u64 bndcfgu; 212 u64 bndstatus; 213 } __packed; 214 215 /* 216 * The BNDCSR state is padded out to be 64-bytes in size. 217 */ 218 struct mpx_bndcsr_state { 219 union { 220 struct mpx_bndcsr bndcsr; 221 u8 pad_to_64_bytes[64]; 222 }; 223 } __packed; 224 225 /* AVX-512 Components: */ 226 227 /* 228 * State component 5 is used for the 8 64-bit opmask registers 229 * k0-k7 (opmask state). 230 */ 231 struct avx_512_opmask_state { 232 u64 opmask_reg[8]; 233 } __packed; 234 235 /* 236 * State component 6 is used for the upper 256 bits of the 237 * registers ZMM0-ZMM15. These 16 256-bit values are denoted 238 * ZMM0_H-ZMM15_H (ZMM_Hi256 state). 239 */ 240 struct avx_512_zmm_uppers_state { 241 struct reg_256_bit zmm_upper[16]; 242 } __packed; 243 244 /* 245 * State component 7 is used for the 16 512-bit registers 246 * ZMM16-ZMM31 (Hi16_ZMM state). 247 */ 248 struct avx_512_hi16_state { 249 struct reg_512_bit hi16_zmm[16]; 250 } __packed; 251 252 /* 253 * State component 9: 32-bit PKRU register. The state is 254 * 8 bytes long but only 4 bytes is used currently. 255 */ 256 struct pkru_state { 257 u32 pkru; 258 u32 pad; 259 } __packed; 260 261 /* 262 * State component 11 is Control-flow Enforcement user states 263 */ 264 struct cet_user_state { 265 /* user control-flow settings */ 266 u64 user_cet; 267 /* user shadow stack pointer */ 268 u64 user_ssp; 269 }; 270 271 /* 272 * State component 12 is Control-flow Enforcement supervisor states. 273 * This state includes SSP pointers for privilege levels 0 through 2. 274 */ 275 struct cet_supervisor_state { 276 u64 pl0_ssp; 277 u64 pl1_ssp; 278 u64 pl2_ssp; 279 } __packed; 280 281 /* 282 * State component 15: Architectural LBR configuration state. 283 * The size of Arch LBR state depends on the number of LBRs (lbr_depth). 284 */ 285 286 struct lbr_entry { 287 u64 from; 288 u64 to; 289 u64 info; 290 }; 291 292 struct arch_lbr_state { 293 u64 lbr_ctl; 294 u64 lbr_depth; 295 u64 ler_from; 296 u64 ler_to; 297 u64 ler_info; 298 struct lbr_entry entries[]; 299 }; 300 301 /* 302 * State component 17: 64-byte tile configuration register. 303 */ 304 struct xtile_cfg { 305 u64 tcfg[8]; 306 } __packed; 307 308 /* 309 * State component 18: 1KB tile data register. 310 * Each register represents 16 64-byte rows of the matrix 311 * data. But the number of registers depends on the actual 312 * implementation. 313 */ 314 struct xtile_data { 315 struct reg_1024_byte tmm; 316 } __packed; 317 318 /* 319 * State component 19: 8B extended general purpose register. 320 */ 321 struct apx_state { 322 u64 egpr[16]; 323 } __packed; 324 325 /* 326 * State component 10 is supervisor state used for context-switching the 327 * PASID state. 328 */ 329 struct ia32_pasid_state { 330 u64 pasid; 331 } __packed; 332 333 struct xstate_header { 334 u64 xfeatures; 335 u64 xcomp_bv; 336 u64 reserved[6]; 337 } __attribute__((packed)); 338 339 /* 340 * xstate_header.xcomp_bv[63] indicates that the extended_state_area 341 * is in compacted format. 342 */ 343 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) 344 345 /* 346 * This is our most modern FPU state format, as saved by the XSAVE 347 * and restored by the XRSTOR instructions. 348 * 349 * It consists of a legacy fxregs portion, an xstate header and 350 * subsequent areas as defined by the xstate header. Not all CPUs 351 * support all the extensions, so the size of the extended area 352 * can vary quite a bit between CPUs. 353 */ 354 struct xregs_state { 355 struct fxregs_state i387; 356 struct xstate_header header; 357 u8 extended_state_area[]; 358 } __attribute__ ((packed, aligned (64))); 359 360 /* 361 * This is a union of all the possible FPU state formats 362 * put together, so that we can pick the right one runtime. 363 * 364 * The size of the structure is determined by the largest 365 * member - which is the xsave area. The padding is there 366 * to ensure that statically-allocated task_structs (just 367 * the init_task today) have enough space. 368 */ 369 union fpregs_state { 370 struct fregs_state fsave; 371 struct fxregs_state fxsave; 372 struct swregs_state soft; 373 struct xregs_state xsave; 374 u8 __padding[PAGE_SIZE]; 375 }; 376 377 struct fpstate { 378 /* @kernel_size: The size of the kernel register image */ 379 unsigned int size; 380 381 /* @user_size: The size in non-compacted UABI format */ 382 unsigned int user_size; 383 384 /* @xfeatures: xfeatures for which the storage is sized */ 385 u64 xfeatures; 386 387 /* @user_xfeatures: xfeatures valid in UABI buffers */ 388 u64 user_xfeatures; 389 390 /* @xfd: xfeatures disabled to trap userspace use. */ 391 u64 xfd; 392 393 /* @is_valloc: Indicator for dynamically allocated state */ 394 unsigned int is_valloc : 1; 395 396 /* @is_guest: Indicator for guest state (KVM) */ 397 unsigned int is_guest : 1; 398 399 /* 400 * @is_confidential: Indicator for KVM confidential mode. 401 * The FPU registers are restored by the 402 * vmentry firmware from encrypted guest 403 * memory. On vmexit the FPU registers are 404 * saved by firmware to encrypted guest memory 405 * and the registers are scrubbed before 406 * returning to the host. So there is no 407 * content which is worth saving and restoring. 408 * The fpstate has to be there so that 409 * preemption and softirq FPU usage works 410 * without special casing. 411 */ 412 unsigned int is_confidential : 1; 413 414 /* @in_use: State is in use */ 415 unsigned int in_use : 1; 416 417 /* @regs: The register state union for all supported formats */ 418 union fpregs_state regs; 419 420 /* @regs is dynamically sized! Don't add anything after @regs! */ 421 } __aligned(64); 422 423 #define FPU_GUEST_PERM_LOCKED BIT_ULL(63) 424 425 struct fpu_state_perm { 426 /* 427 * @__state_perm: 428 * 429 * This bitmap indicates the permission for state components 430 * available to a thread group, including both user and supervisor 431 * components and software-defined bits like FPU_GUEST_PERM_LOCKED. 432 * The permission prctl() sets the enabled state bits in 433 * thread_group_leader()->thread.fpu. 434 * 435 * All run time operations use the per thread information in the 436 * currently active fpu.fpstate which contains the xfeature masks 437 * and sizes for kernel and user space. 438 * 439 * This master permission field is only to be used when 440 * task.fpu.fpstate based checks fail to validate whether the task 441 * is allowed to expand its xfeatures set which requires to 442 * allocate a larger sized fpstate buffer. 443 * 444 * Do not access this field directly. Use the provided helper 445 * function. Unlocked access is possible for quick checks. 446 */ 447 u64 __state_perm; 448 449 /* 450 * @__state_size: 451 * 452 * The size required for @__state_perm. Only valid to access 453 * with sighand locked. 454 */ 455 unsigned int __state_size; 456 457 /* 458 * @__user_state_size: 459 * 460 * The size required for @__state_perm user part. Only valid to 461 * access with sighand locked. 462 */ 463 unsigned int __user_state_size; 464 }; 465 466 /* 467 * Highest level per task FPU state data structure that 468 * contains the FPU register state plus various FPU 469 * state fields: 470 */ 471 struct fpu { 472 /* 473 * @last_cpu: 474 * 475 * Records the last CPU on which this context was loaded into 476 * FPU registers. (In the lazy-restore case we might be 477 * able to reuse FPU registers across multiple context switches 478 * this way, if no intermediate task used the FPU.) 479 * 480 * A value of -1 is used to indicate that the FPU state in context 481 * memory is newer than the FPU state in registers, and that the 482 * FPU state should be reloaded next time the task is run. 483 */ 484 unsigned int last_cpu; 485 486 /* 487 * @avx512_timestamp: 488 * 489 * Records the timestamp of AVX512 use during last context switch. 490 */ 491 unsigned long avx512_timestamp; 492 493 /* 494 * @fpstate: 495 * 496 * Pointer to the active struct fpstate. Initialized to 497 * point at @__fpstate below. 498 */ 499 struct fpstate *fpstate; 500 501 /* 502 * @__task_fpstate: 503 * 504 * Pointer to an inactive struct fpstate. Initialized to NULL. Is 505 * used only for KVM support to swap out the regular task fpstate. 506 */ 507 struct fpstate *__task_fpstate; 508 509 /* 510 * @perm: 511 * 512 * Permission related information 513 */ 514 struct fpu_state_perm perm; 515 516 /* 517 * @guest_perm: 518 * 519 * Permission related information for guest pseudo FPUs 520 */ 521 struct fpu_state_perm guest_perm; 522 523 /* 524 * @__fpstate: 525 * 526 * Initial in-memory storage for FPU registers which are saved in 527 * context switch and when the kernel uses the FPU. The registers 528 * are restored from this storage on return to user space if they 529 * are not longer containing the tasks FPU register state. 530 */ 531 struct fpstate __fpstate; 532 /* 533 * WARNING: '__fpstate' is dynamically-sized. Do not put 534 * anything after it here. 535 */ 536 }; 537 538 /* 539 * Guest pseudo FPU container 540 */ 541 struct fpu_guest { 542 /* 543 * @xfeatures: xfeature bitmap of features which are 544 * currently enabled for the guest vCPU. 545 */ 546 u64 xfeatures; 547 548 /* 549 * @xfd_err: Save the guest value. 550 */ 551 u64 xfd_err; 552 553 /* 554 * @uabi_size: Size required for save/restore 555 */ 556 unsigned int uabi_size; 557 558 /* 559 * @fpstate: Pointer to the allocated guest fpstate 560 */ 561 struct fpstate *fpstate; 562 }; 563 564 /* 565 * FPU state configuration data for fpu_guest. 566 * Initialized at boot time. Read only after init. 567 */ 568 struct vcpu_fpu_config { 569 /* 570 * @size: 571 * 572 * The default size of the register state buffer in guest FPUs. 573 * Includes all supported features except independent managed 574 * features and features which have to be requested by user space 575 * before usage. 576 */ 577 unsigned int size; 578 579 /* 580 * @features: 581 * 582 * The default supported features bitmap in guest FPUs. Does not 583 * include independent managed features and features which have to 584 * be requested by user space before usage. 585 */ 586 u64 features; 587 }; 588 589 /* 590 * FPU state configuration data. Initialized at boot time. Read only after init. 591 */ 592 struct fpu_state_config { 593 /* 594 * @max_size: 595 * 596 * The maximum size of the register state buffer. Includes all 597 * supported features except independent managed features. 598 */ 599 unsigned int max_size; 600 601 /* 602 * @default_size: 603 * 604 * The default size of the register state buffer. Includes all 605 * supported features except independent managed features, 606 * guest-only features and features which have to be requested by 607 * user space before usage. 608 */ 609 unsigned int default_size; 610 611 /* 612 * @max_features: 613 * 614 * The maximum supported features bitmap. Does not include 615 * independent managed features. 616 */ 617 u64 max_features; 618 619 /* 620 * @default_features: 621 * 622 * The default supported features bitmap. Does not include 623 * independent managed features, guest-only features and features 624 * which have to be requested by user space before usage. 625 */ 626 u64 default_features; 627 /* 628 * @legacy_features: 629 * 630 * Features which can be reported back to user space 631 * even without XSAVE support, i.e. legacy features FP + SSE 632 */ 633 u64 legacy_features; 634 /* 635 * @independent_features: 636 * 637 * Features that are supported by XSAVES, but not managed as part of 638 * the FPU core, such as LBR 639 */ 640 u64 independent_features; 641 }; 642 643 /* FPU state configuration information */ 644 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; 645 extern struct vcpu_fpu_config guest_default_cfg; 646 647 #endif /* _ASM_X86_FPU_TYPES_H */ 648