1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * FPU data structures: 4 */ 5 #ifndef _ASM_X86_FPU_TYPES_H 6 #define _ASM_X86_FPU_TYPES_H 7 8 #include <asm/page_types.h> 9 10 /* 11 * The legacy x87 FPU state format, as saved by FSAVE and 12 * restored by the FRSTOR instructions: 13 */ 14 struct fregs_state { 15 u32 cwd; /* FPU Control Word */ 16 u32 swd; /* FPU Status Word */ 17 u32 twd; /* FPU Tag Word */ 18 u32 fip; /* FPU IP Offset */ 19 u32 fcs; /* FPU IP Selector */ 20 u32 foo; /* FPU Operand Pointer Offset */ 21 u32 fos; /* FPU Operand Pointer Selector */ 22 23 /* 8*10 bytes for each FP-reg = 80 bytes: */ 24 u32 st_space[20]; 25 26 /* Software status information [not touched by FSAVE]: */ 27 u32 status; 28 }; 29 30 /* 31 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and 32 * restored by the FXRSTOR instructions. It's similar to the FSAVE 33 * format, but differs in some areas, plus has extensions at 34 * the end for the XMM registers. 35 */ 36 struct fxregs_state { 37 u16 cwd; /* Control Word */ 38 u16 swd; /* Status Word */ 39 u16 twd; /* Tag Word */ 40 u16 fop; /* Last Instruction Opcode */ 41 union { 42 struct { 43 u64 rip; /* Instruction Pointer */ 44 u64 rdp; /* Data Pointer */ 45 }; 46 struct { 47 u32 fip; /* FPU IP Offset */ 48 u32 fcs; /* FPU IP Selector */ 49 u32 foo; /* FPU Operand Offset */ 50 u32 fos; /* FPU Operand Selector */ 51 }; 52 }; 53 u32 mxcsr; /* MXCSR Register State */ 54 u32 mxcsr_mask; /* MXCSR Mask */ 55 56 /* 8*16 bytes for each FP-reg = 128 bytes: */ 57 u32 st_space[32]; 58 59 /* 16*16 bytes for each XMM-reg = 256 bytes: */ 60 u32 xmm_space[64]; 61 62 u32 padding[12]; 63 64 union { 65 u32 padding1[12]; 66 u32 sw_reserved[12]; 67 }; 68 69 } __attribute__((aligned(16))); 70 71 /* Default value for fxregs_state.mxcsr: */ 72 #define MXCSR_DEFAULT 0x1f80 73 74 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */ 75 #define MXCSR_AND_FLAGS_SIZE sizeof(u64) 76 77 /* 78 * List of XSAVE features Linux knows about: 79 */ 80 enum xfeature { 81 XFEATURE_FP, 82 XFEATURE_SSE, 83 /* 84 * Values above here are "legacy states". 85 * Those below are "extended states". 86 */ 87 XFEATURE_YMM, 88 XFEATURE_BNDREGS, 89 XFEATURE_BNDCSR, 90 XFEATURE_OPMASK, 91 XFEATURE_ZMM_Hi256, 92 XFEATURE_Hi16_ZMM, 93 XFEATURE_PT_UNIMPLEMENTED_SO_FAR, 94 XFEATURE_PKRU, 95 XFEATURE_PASID, 96 XFEATURE_CET_USER, 97 XFEATURE_CET_KERNEL, 98 XFEATURE_RSRVD_COMP_13, 99 XFEATURE_RSRVD_COMP_14, 100 XFEATURE_LBR, 101 XFEATURE_RSRVD_COMP_16, 102 XFEATURE_XTILE_CFG, 103 XFEATURE_XTILE_DATA, 104 XFEATURE_APX, 105 106 XFEATURE_MAX, 107 }; 108 109 #define XFEATURE_MASK_FP (1 << XFEATURE_FP) 110 #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) 111 #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) 112 #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) 113 #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) 114 #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) 115 #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) 116 #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) 117 #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) 118 #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) 119 #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) 120 #define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) 121 #define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL) 122 #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) 123 #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) 124 #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) 125 #define XFEATURE_MASK_APX (1 << XFEATURE_APX) 126 127 #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) 128 #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ 129 | XFEATURE_MASK_ZMM_Hi256 \ 130 | XFEATURE_MASK_Hi16_ZMM) 131 132 #ifdef CONFIG_X86_64 133 # define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ 134 | XFEATURE_MASK_XTILE_CFG) 135 #else 136 # define XFEATURE_MASK_XTILE (0) 137 #endif 138 139 #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM 140 141 struct reg_128_bit { 142 u8 regbytes[128/8]; 143 }; 144 struct reg_256_bit { 145 u8 regbytes[256/8]; 146 }; 147 struct reg_512_bit { 148 u8 regbytes[512/8]; 149 }; 150 struct reg_1024_byte { 151 u8 regbytes[1024]; 152 }; 153 154 /* 155 * State component 2: 156 * 157 * There are 16x 256-bit AVX registers named YMM0-YMM15. 158 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) 159 * and are stored in 'struct fxregs_state::xmm_space[]' in the 160 * "legacy" area. 161 * 162 * The high 128 bits are stored here. 163 */ 164 struct ymmh_struct { 165 struct reg_128_bit hi_ymm[16]; 166 } __packed; 167 168 /* Intel MPX support: */ 169 170 struct mpx_bndreg { 171 u64 lower_bound; 172 u64 upper_bound; 173 } __packed; 174 /* 175 * State component 3 is used for the 4 128-bit bounds registers 176 */ 177 struct mpx_bndreg_state { 178 struct mpx_bndreg bndreg[4]; 179 } __packed; 180 181 /* 182 * State component 4 is used for the 64-bit user-mode MPX 183 * configuration register BNDCFGU and the 64-bit MPX status 184 * register BNDSTATUS. We call the pair "BNDCSR". 185 */ 186 struct mpx_bndcsr { 187 u64 bndcfgu; 188 u64 bndstatus; 189 } __packed; 190 191 /* 192 * The BNDCSR state is padded out to be 64-bytes in size. 193 */ 194 struct mpx_bndcsr_state { 195 union { 196 struct mpx_bndcsr bndcsr; 197 u8 pad_to_64_bytes[64]; 198 }; 199 } __packed; 200 201 /* AVX-512 Components: */ 202 203 /* 204 * State component 5 is used for the 8 64-bit opmask registers 205 * k0-k7 (opmask state). 206 */ 207 struct avx_512_opmask_state { 208 u64 opmask_reg[8]; 209 } __packed; 210 211 /* 212 * State component 6 is used for the upper 256 bits of the 213 * registers ZMM0-ZMM15. These 16 256-bit values are denoted 214 * ZMM0_H-ZMM15_H (ZMM_Hi256 state). 215 */ 216 struct avx_512_zmm_uppers_state { 217 struct reg_256_bit zmm_upper[16]; 218 } __packed; 219 220 /* 221 * State component 7 is used for the 16 512-bit registers 222 * ZMM16-ZMM31 (Hi16_ZMM state). 223 */ 224 struct avx_512_hi16_state { 225 struct reg_512_bit hi16_zmm[16]; 226 } __packed; 227 228 /* 229 * State component 9: 32-bit PKRU register. The state is 230 * 8 bytes long but only 4 bytes is used currently. 231 */ 232 struct pkru_state { 233 u32 pkru; 234 u32 pad; 235 } __packed; 236 237 /* 238 * State component 11 is Control-flow Enforcement user states 239 */ 240 struct cet_user_state { 241 /* user control-flow settings */ 242 u64 user_cet; 243 /* user shadow stack pointer */ 244 u64 user_ssp; 245 }; 246 247 /* 248 * State component 12 is Control-flow Enforcement supervisor states. 249 * This state includes SSP pointers for privilege levels 0 through 2. 250 */ 251 struct cet_supervisor_state { 252 u64 pl0_ssp; 253 u64 pl1_ssp; 254 u64 pl2_ssp; 255 } __packed; 256 257 /* 258 * State component 15: Architectural LBR configuration state. 259 * The size of Arch LBR state depends on the number of LBRs (lbr_depth). 260 */ 261 262 struct lbr_entry { 263 u64 from; 264 u64 to; 265 u64 info; 266 }; 267 268 struct arch_lbr_state { 269 u64 lbr_ctl; 270 u64 lbr_depth; 271 u64 ler_from; 272 u64 ler_to; 273 u64 ler_info; 274 struct lbr_entry entries[]; 275 }; 276 277 /* 278 * State component 17: 64-byte tile configuration register. 279 */ 280 struct xtile_cfg { 281 u64 tcfg[8]; 282 } __packed; 283 284 /* 285 * State component 18: 1KB tile data register. 286 * Each register represents 16 64-byte rows of the matrix 287 * data. But the number of registers depends on the actual 288 * implementation. 289 */ 290 struct xtile_data { 291 struct reg_1024_byte tmm; 292 } __packed; 293 294 /* 295 * State component 19: 8B extended general purpose register. 296 */ 297 struct apx_state { 298 u64 egpr[16]; 299 } __packed; 300 301 /* 302 * State component 10 is supervisor state used for context-switching the 303 * PASID state. 304 */ 305 struct ia32_pasid_state { 306 u64 pasid; 307 } __packed; 308 309 struct xstate_header { 310 u64 xfeatures; 311 u64 xcomp_bv; 312 u64 reserved[6]; 313 } __attribute__((packed)); 314 315 /* 316 * xstate_header.xcomp_bv[63] indicates that the extended_state_area 317 * is in compacted format. 318 */ 319 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) 320 321 /* 322 * This is our most modern FPU state format, as saved by the XSAVE 323 * and restored by the XRSTOR instructions. 324 * 325 * It consists of a legacy fxregs portion, an xstate header and 326 * subsequent areas as defined by the xstate header. Not all CPUs 327 * support all the extensions, so the size of the extended area 328 * can vary quite a bit between CPUs. 329 */ 330 struct xregs_state { 331 struct fxregs_state i387; 332 struct xstate_header header; 333 u8 extended_state_area[]; 334 } __attribute__ ((packed, aligned (64))); 335 336 /* 337 * This is a union of all the possible FPU state formats 338 * put together, so that we can pick the right one runtime. 339 * 340 * The size of the structure is determined by the largest 341 * member - which is the xsave area. The padding is there 342 * to ensure that statically-allocated task_structs (just 343 * the init_task today) have enough space. 344 */ 345 union fpregs_state { 346 struct fregs_state fsave; 347 struct fxregs_state fxsave; 348 struct xregs_state xsave; 349 u8 __padding[PAGE_SIZE]; 350 }; 351 352 struct fpstate { 353 /* @kernel_size: The size of the kernel register image */ 354 unsigned int size; 355 356 /* @user_size: The size in non-compacted UABI format */ 357 unsigned int user_size; 358 359 /* @xfeatures: xfeatures for which the storage is sized */ 360 u64 xfeatures; 361 362 /* @user_xfeatures: xfeatures valid in UABI buffers */ 363 u64 user_xfeatures; 364 365 /* @xfd: xfeatures disabled to trap userspace use. */ 366 u64 xfd; 367 368 /* @is_valloc: Indicator for dynamically allocated state */ 369 unsigned int is_valloc : 1; 370 371 /* @is_guest: Indicator for guest state (KVM) */ 372 unsigned int is_guest : 1; 373 374 /* 375 * @is_confidential: Indicator for KVM confidential mode. 376 * The FPU registers are restored by the 377 * vmentry firmware from encrypted guest 378 * memory. On vmexit the FPU registers are 379 * saved by firmware to encrypted guest memory 380 * and the registers are scrubbed before 381 * returning to the host. So there is no 382 * content which is worth saving and restoring. 383 * The fpstate has to be there so that 384 * preemption and softirq FPU usage works 385 * without special casing. 386 */ 387 unsigned int is_confidential : 1; 388 389 /* @in_use: State is in use */ 390 unsigned int in_use : 1; 391 392 /* @regs: The register state union for all supported formats */ 393 union fpregs_state regs; 394 395 /* @regs is dynamically sized! Don't add anything after @regs! */ 396 } __aligned(64); 397 398 #define FPU_GUEST_PERM_LOCKED BIT_ULL(63) 399 400 struct fpu_state_perm { 401 /* 402 * @__state_perm: 403 * 404 * This bitmap indicates the permission for state components 405 * available to a thread group, including both user and supervisor 406 * components and software-defined bits like FPU_GUEST_PERM_LOCKED. 407 * The permission prctl() sets the enabled state bits in 408 * thread_group_leader()->thread.fpu. 409 * 410 * All run time operations use the per thread information in the 411 * currently active fpu.fpstate which contains the xfeature masks 412 * and sizes for kernel and user space. 413 * 414 * This master permission field is only to be used when 415 * task.fpu.fpstate based checks fail to validate whether the task 416 * is allowed to expand its xfeatures set which requires to 417 * allocate a larger sized fpstate buffer. 418 * 419 * Do not access this field directly. Use the provided helper 420 * function. Unlocked access is possible for quick checks. 421 */ 422 u64 __state_perm; 423 424 /* 425 * @__state_size: 426 * 427 * The size required for @__state_perm. Only valid to access 428 * with sighand locked. 429 */ 430 unsigned int __state_size; 431 432 /* 433 * @__user_state_size: 434 * 435 * The size required for @__state_perm user part. Only valid to 436 * access with sighand locked. 437 */ 438 unsigned int __user_state_size; 439 }; 440 441 /* 442 * Highest level per task FPU state data structure that 443 * contains the FPU register state plus various FPU 444 * state fields: 445 */ 446 struct fpu { 447 /* 448 * @last_cpu: 449 * 450 * Records the last CPU on which this context was loaded into 451 * FPU registers. (In the lazy-restore case we might be 452 * able to reuse FPU registers across multiple context switches 453 * this way, if no intermediate task used the FPU.) 454 * 455 * A value of -1 is used to indicate that the FPU state in context 456 * memory is newer than the FPU state in registers, and that the 457 * FPU state should be reloaded next time the task is run. 458 */ 459 unsigned int last_cpu; 460 461 /* 462 * @avx512_timestamp: 463 * 464 * Records the timestamp of AVX512 use during last context switch. 465 */ 466 unsigned long avx512_timestamp; 467 468 /* 469 * @fpstate: 470 * 471 * Pointer to the active struct fpstate. Initialized to 472 * point at @__fpstate below. 473 */ 474 struct fpstate *fpstate; 475 476 /* 477 * @__task_fpstate: 478 * 479 * Pointer to an inactive struct fpstate. Initialized to NULL. Is 480 * used only for KVM support to swap out the regular task fpstate. 481 */ 482 struct fpstate *__task_fpstate; 483 484 /* 485 * @perm: 486 * 487 * Permission related information 488 */ 489 struct fpu_state_perm perm; 490 491 /* 492 * @guest_perm: 493 * 494 * Permission related information for guest pseudo FPUs 495 */ 496 struct fpu_state_perm guest_perm; 497 498 /* 499 * @__fpstate: 500 * 501 * Initial in-memory storage for FPU registers which are saved in 502 * context switch and when the kernel uses the FPU. The registers 503 * are restored from this storage on return to user space if they 504 * are not longer containing the tasks FPU register state. 505 */ 506 struct fpstate __fpstate; 507 /* 508 * WARNING: '__fpstate' is dynamically-sized. Do not put 509 * anything after it here. 510 */ 511 }; 512 513 /* 514 * Guest pseudo FPU container 515 */ 516 struct fpu_guest { 517 /* 518 * @xfeatures: xfeature bitmap of features which are 519 * currently enabled for the guest vCPU. 520 */ 521 u64 xfeatures; 522 523 /* 524 * @xfd_err: Save the guest value. 525 */ 526 u64 xfd_err; 527 528 /* 529 * @uabi_size: Size required for save/restore 530 */ 531 unsigned int uabi_size; 532 533 /* 534 * @fpstate: Pointer to the allocated guest fpstate 535 */ 536 struct fpstate *fpstate; 537 }; 538 539 /* 540 * FPU state configuration data for fpu_guest. 541 * Initialized at boot time. Read only after init. 542 */ 543 struct vcpu_fpu_config { 544 /* 545 * @size: 546 * 547 * The default size of the register state buffer in guest FPUs. 548 * Includes all supported features except independent managed 549 * features and features which have to be requested by user space 550 * before usage. 551 */ 552 unsigned int size; 553 554 /* 555 * @features: 556 * 557 * The default supported features bitmap in guest FPUs. Does not 558 * include independent managed features and features which have to 559 * be requested by user space before usage. 560 */ 561 u64 features; 562 }; 563 564 /* 565 * FPU state configuration data. Initialized at boot time. Read only after init. 566 */ 567 struct fpu_state_config { 568 /* 569 * @max_size: 570 * 571 * The maximum size of the register state buffer. Includes all 572 * supported features except independent managed features. 573 */ 574 unsigned int max_size; 575 576 /* 577 * @default_size: 578 * 579 * The default size of the register state buffer. Includes all 580 * supported features except independent managed features, 581 * guest-only features and features which have to be requested by 582 * user space before usage. 583 */ 584 unsigned int default_size; 585 586 /* 587 * @max_features: 588 * 589 * The maximum supported features bitmap. Does not include 590 * independent managed features. 591 */ 592 u64 max_features; 593 594 /* 595 * @default_features: 596 * 597 * The default supported features bitmap. Does not include 598 * independent managed features, guest-only features and features 599 * which have to be requested by user space before usage. 600 */ 601 u64 default_features; 602 /* 603 * @legacy_features: 604 * 605 * Features which can be reported back to user space 606 * even without XSAVE support, i.e. legacy features FP + SSE 607 */ 608 u64 legacy_features; 609 /* 610 * @independent_features: 611 * 612 * Features that are supported by XSAVES, but not managed as part of 613 * the FPU core, such as LBR 614 */ 615 u64 independent_features; 616 }; 617 618 /* FPU state configuration information */ 619 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; 620 extern struct vcpu_fpu_config guest_default_cfg; 621 622 #endif /* _ASM_X86_FPU_TYPES_H */ 623