1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * xsave/xrstor support. 4 * 5 * Author: Suresh Siddha <suresh.b.siddha@intel.com> 6 */ 7 #include <linux/bitops.h> 8 #include <linux/compat.h> 9 #include <linux/cpu.h> 10 #include <linux/mman.h> 11 #include <linux/kvm_types.h> 12 #include <linux/nospec.h> 13 #include <linux/pkeys.h> 14 #include <linux/seq_file.h> 15 #include <linux/proc_fs.h> 16 #include <linux/vmalloc.h> 17 #include <linux/coredump.h> 18 #include <linux/sort.h> 19 20 #include <asm/fpu/api.h> 21 #include <asm/fpu/regset.h> 22 #include <asm/fpu/signal.h> 23 #include <asm/fpu/xcr.h> 24 25 #include <asm/cpuid/api.h> 26 #include <asm/msr.h> 27 #include <asm/tlbflush.h> 28 #include <asm/prctl.h> 29 #include <asm/elf.h> 30 31 #include <uapi/asm/elf.h> 32 33 #include "context.h" 34 #include "internal.h" 35 #include "legacy.h" 36 #include "xstate.h" 37 38 #define for_each_extended_xfeature(bit, mask) \ 39 (bit) = FIRST_EXTENDED_XFEATURE; \ 40 for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) 41 42 /* 43 * Although we spell it out in here, the Processor Trace 44 * xfeature is completely unused. We use other mechanisms 45 * to save/restore PT state in Linux. 46 */ 47 static const char *xfeature_names[] = 48 { 49 "x87 floating point registers", 50 "SSE registers", 51 "AVX registers", 52 "MPX bounds registers", 53 "MPX CSR", 54 "AVX-512 opmask", 55 "AVX-512 Hi256", 56 "AVX-512 ZMM_Hi256", 57 "Processor Trace (unused)", 58 "Protection Keys User registers", 59 "PASID state", 60 "Control-flow User registers", 61 "Control-flow Kernel registers (KVM only)", 62 "unknown xstate feature", 63 "unknown xstate feature", 64 "unknown xstate feature", 65 "unknown xstate feature", 66 "AMX Tile config", 67 "AMX Tile data", 68 "APX registers", 69 "unknown xstate feature", 70 }; 71 72 static unsigned short xsave_cpuid_features[] __initdata = { 73 [XFEATURE_FP] = X86_FEATURE_FPU, 74 [XFEATURE_SSE] = X86_FEATURE_XMM, 75 [XFEATURE_YMM] = X86_FEATURE_AVX, 76 [XFEATURE_BNDREGS] = X86_FEATURE_MPX, 77 [XFEATURE_BNDCSR] = X86_FEATURE_MPX, 78 [XFEATURE_OPMASK] = X86_FEATURE_AVX512F, 79 [XFEATURE_ZMM_Hi256] = X86_FEATURE_AVX512F, 80 [XFEATURE_Hi16_ZMM] = X86_FEATURE_AVX512F, 81 [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT, 82 [XFEATURE_PKRU] = X86_FEATURE_OSPKE, 83 [XFEATURE_PASID] = X86_FEATURE_ENQCMD, 84 [XFEATURE_CET_USER] = X86_FEATURE_SHSTK, 85 [XFEATURE_CET_KERNEL] = X86_FEATURE_SHSTK, 86 [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE, 87 [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE, 88 [XFEATURE_APX] = X86_FEATURE_APX, 89 }; 90 91 static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = 92 { [ 0 ... XFEATURE_MAX - 1] = -1}; 93 static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = 94 { [ 0 ... XFEATURE_MAX - 1] = -1}; 95 static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init; 96 97 /* 98 * Ordering of xstate components in uncompacted format: The xfeature 99 * number does not necessarily indicate its position in the XSAVE buffer. 100 * This array defines the traversal order of xstate features. 101 */ 102 static unsigned int xfeature_uncompact_order[XFEATURE_MAX] __ro_after_init = 103 { [ 0 ... XFEATURE_MAX - 1] = -1}; 104 105 static inline unsigned int next_xfeature_order(unsigned int i, u64 mask) 106 { 107 for (; xfeature_uncompact_order[i] != -1; i++) { 108 if (mask & BIT_ULL(xfeature_uncompact_order[i])) 109 break; 110 } 111 112 return i; 113 } 114 115 /* Iterate xstate features in uncompacted order: */ 116 #define for_each_extended_xfeature_in_order(i, mask) \ 117 for (i = 0; \ 118 i = next_xfeature_order(i, mask), \ 119 xfeature_uncompact_order[i] != -1; \ 120 i++) 121 122 #define XSTATE_FLAG_SUPERVISOR BIT(0) 123 #define XSTATE_FLAG_ALIGNED64 BIT(1) 124 125 /* 126 * Return whether the system supports a given xfeature. 127 * 128 * Also return the name of the (most advanced) feature that the caller requested: 129 */ 130 int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) 131 { 132 u64 xfeatures_missing = xfeatures_needed & ~fpu_kernel_cfg.max_features; 133 134 if (unlikely(feature_name)) { 135 long xfeature_idx, max_idx; 136 u64 xfeatures_print; 137 /* 138 * So we use FLS here to be able to print the most advanced 139 * feature that was requested but is missing. So if a driver 140 * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the 141 * missing AVX feature - this is the most informative message 142 * to users: 143 */ 144 if (xfeatures_missing) 145 xfeatures_print = xfeatures_missing; 146 else 147 xfeatures_print = xfeatures_needed; 148 149 xfeature_idx = fls64(xfeatures_print)-1; 150 max_idx = ARRAY_SIZE(xfeature_names)-1; 151 xfeature_idx = min(xfeature_idx, max_idx); 152 153 *feature_name = xfeature_names[xfeature_idx]; 154 } 155 156 if (xfeatures_missing) 157 return 0; 158 159 return 1; 160 } 161 EXPORT_SYMBOL_GPL(cpu_has_xfeatures); 162 163 static bool xfeature_is_aligned64(int xfeature_nr) 164 { 165 return xstate_flags[xfeature_nr] & XSTATE_FLAG_ALIGNED64; 166 } 167 168 static bool xfeature_is_supervisor(int xfeature_nr) 169 { 170 return xstate_flags[xfeature_nr] & XSTATE_FLAG_SUPERVISOR; 171 } 172 173 static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature) 174 { 175 unsigned int offs, i; 176 177 /* 178 * Non-compacted format and legacy features use the cached fixed 179 * offsets. 180 */ 181 if (!cpu_feature_enabled(X86_FEATURE_XCOMPACTED) || 182 xfeature <= XFEATURE_SSE) 183 return xstate_offsets[xfeature]; 184 185 /* 186 * Compacted format offsets depend on the actual content of the 187 * compacted xsave area which is determined by the xcomp_bv header 188 * field. 189 */ 190 offs = FXSAVE_SIZE + XSAVE_HDR_SIZE; 191 for_each_extended_xfeature(i, xcomp_bv) { 192 if (xfeature_is_aligned64(i)) 193 offs = ALIGN(offs, 64); 194 if (i == xfeature) 195 break; 196 offs += xstate_sizes[i]; 197 } 198 return offs; 199 } 200 201 /* 202 * Enable the extended processor state save/restore feature. 203 * Called once per CPU onlining. 204 */ 205 void fpu__init_cpu_xstate(void) 206 { 207 if (!boot_cpu_has(X86_FEATURE_XSAVE) || !fpu_kernel_cfg.max_features) 208 return; 209 210 cr4_set_bits(X86_CR4_OSXSAVE); 211 212 /* 213 * Must happen after CR4 setup and before xsetbv() to allow KVM 214 * lazy passthrough. Write independent of the dynamic state static 215 * key as that does not work on the boot CPU. This also ensures 216 * that any stale state is wiped out from XFD. Reset the per CPU 217 * xfd cache too. 218 */ 219 if (cpu_feature_enabled(X86_FEATURE_XFD)) 220 xfd_set_state(init_fpstate.xfd); 221 222 /* 223 * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features 224 * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user 225 * states can be set here. 226 */ 227 xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features); 228 229 /* 230 * MSR_IA32_XSS sets supervisor states managed by XSAVES. 231 */ 232 if (boot_cpu_has(X86_FEATURE_XSAVES)) { 233 wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor() | 234 xfeatures_mask_independent()); 235 } 236 } 237 238 static bool xfeature_enabled(enum xfeature xfeature) 239 { 240 return fpu_kernel_cfg.max_features & BIT_ULL(xfeature); 241 } 242 243 static int compare_xstate_offsets(const void *xfeature1, const void *xfeature2) 244 { 245 return xstate_offsets[*(unsigned int *)xfeature1] - 246 xstate_offsets[*(unsigned int *)xfeature2]; 247 } 248 249 /* 250 * Record the offsets and sizes of various xstates contained 251 * in the XSAVE state memory layout. Also, create an ordered 252 * list of xfeatures for handling out-of-order offsets. 253 */ 254 static void __init setup_xstate_cache(void) 255 { 256 u32 eax, ebx, ecx, edx, xfeature, i = 0; 257 /* 258 * The FP xstates and SSE xstates are legacy states. They are always 259 * in the fixed offsets in the xsave area in either compacted form 260 * or standard form. 261 */ 262 xstate_offsets[XFEATURE_FP] = 0; 263 xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state, 264 xmm_space); 265 266 xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP]; 267 xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, 268 xmm_space); 269 270 for_each_extended_xfeature(xfeature, fpu_kernel_cfg.max_features) { 271 cpuid_count(CPUID_LEAF_XSTATE, xfeature, &eax, &ebx, &ecx, &edx); 272 273 xstate_sizes[xfeature] = eax; 274 xstate_flags[xfeature] = ecx; 275 276 /* 277 * If an xfeature is supervisor state, the offset in EBX is 278 * invalid, leave it to -1. 279 */ 280 if (xfeature_is_supervisor(xfeature)) 281 continue; 282 283 xstate_offsets[xfeature] = ebx; 284 285 /* Populate the list of xfeatures before sorting */ 286 xfeature_uncompact_order[i++] = xfeature; 287 } 288 289 /* 290 * Sort xfeatures by their offsets to support out-of-order 291 * offsets in the uncompacted format. 292 */ 293 sort(xfeature_uncompact_order, i, sizeof(unsigned int), compare_xstate_offsets, NULL); 294 } 295 296 /* 297 * Print out all the supported xstate features: 298 */ 299 static void __init print_xstate_features(void) 300 { 301 int i; 302 303 for (i = 0; i < XFEATURE_MAX; i++) { 304 u64 mask = BIT_ULL(i); 305 const char *name; 306 307 if (cpu_has_xfeatures(mask, &name)) 308 pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", mask, name); 309 } 310 } 311 312 /* 313 * This check is important because it is easy to get XSTATE_* 314 * confused with XSTATE_BIT_*. 315 */ 316 #define CHECK_XFEATURE(nr) do { \ 317 WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ 318 WARN_ON(nr >= XFEATURE_MAX); \ 319 } while (0) 320 321 /* 322 * Print out xstate component offsets and sizes 323 */ 324 static void __init print_xstate_offset_size(void) 325 { 326 int i; 327 328 for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { 329 pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", 330 i, xfeature_get_offset(fpu_kernel_cfg.max_features, i), 331 i, xstate_sizes[i]); 332 } 333 } 334 335 /* 336 * This function is called only during boot time when x86 caps are not set 337 * up and alternative can not be used yet. 338 */ 339 static __init void os_xrstor_booting(struct xregs_state *xstate) 340 { 341 u64 mask = fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSTATE; 342 u32 lmask = mask; 343 u32 hmask = mask >> 32; 344 int err; 345 346 if (cpu_feature_enabled(X86_FEATURE_XSAVES)) 347 XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); 348 else 349 XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); 350 351 /* 352 * We should never fault when copying from a kernel buffer, and the FPU 353 * state we set at boot time should be valid. 354 */ 355 WARN_ON_FPU(err); 356 } 357 358 /* 359 * All supported features have either init state all zeros or are 360 * handled in setup_init_fpu() individually. This is an explicit 361 * feature list and does not use XFEATURE_MASK*SUPPORTED to catch 362 * newly added supported features at build time and make people 363 * actually look at the init state for the new feature. 364 */ 365 #define XFEATURES_INIT_FPSTATE_HANDLED \ 366 (XFEATURE_MASK_FP | \ 367 XFEATURE_MASK_SSE | \ 368 XFEATURE_MASK_YMM | \ 369 XFEATURE_MASK_OPMASK | \ 370 XFEATURE_MASK_ZMM_Hi256 | \ 371 XFEATURE_MASK_Hi16_ZMM | \ 372 XFEATURE_MASK_PKRU | \ 373 XFEATURE_MASK_BNDREGS | \ 374 XFEATURE_MASK_BNDCSR | \ 375 XFEATURE_MASK_PASID | \ 376 XFEATURE_MASK_CET_USER | \ 377 XFEATURE_MASK_CET_KERNEL | \ 378 XFEATURE_MASK_XTILE | \ 379 XFEATURE_MASK_APX) 380 381 /* 382 * setup the xstate image representing the init state 383 */ 384 static void __init setup_init_fpu_buf(void) 385 { 386 BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED | 387 XFEATURE_MASK_SUPERVISOR_SUPPORTED) != 388 XFEATURES_INIT_FPSTATE_HANDLED); 389 390 if (!boot_cpu_has(X86_FEATURE_XSAVE)) 391 return; 392 393 print_xstate_features(); 394 395 xstate_init_xcomp_bv(&init_fpstate.regs.xsave, init_fpstate.xfeatures); 396 397 /* 398 * Init all the features state with header.xfeatures being 0x0 399 */ 400 os_xrstor_booting(&init_fpstate.regs.xsave); 401 402 /* 403 * All components are now in init state. Read the state back so 404 * that init_fpstate contains all non-zero init state. This only 405 * works with XSAVE, but not with XSAVEOPT and XSAVEC/S because 406 * those use the init optimization which skips writing data for 407 * components in init state. 408 * 409 * XSAVE could be used, but that would require to reshuffle the 410 * data when XSAVEC/S is available because XSAVEC/S uses xstate 411 * compaction. But doing so is a pointless exercise because most 412 * components have an all zeros init state except for the legacy 413 * ones (FP and SSE). Those can be saved with FXSAVE into the 414 * legacy area. Adding new features requires to ensure that init 415 * state is all zeroes or if not to add the necessary handling 416 * here. 417 */ 418 fxsave(&init_fpstate.regs.fxsave); 419 } 420 421 int xfeature_size(int xfeature_nr) 422 { 423 u32 eax, ebx, ecx, edx; 424 425 CHECK_XFEATURE(xfeature_nr); 426 cpuid_count(CPUID_LEAF_XSTATE, xfeature_nr, &eax, &ebx, &ecx, &edx); 427 return eax; 428 } 429 430 /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ 431 static int validate_user_xstate_header(const struct xstate_header *hdr, 432 struct fpstate *fpstate) 433 { 434 /* No unknown or supervisor features may be set */ 435 if (hdr->xfeatures & ~fpstate->user_xfeatures) 436 return -EINVAL; 437 438 /* Userspace must use the uncompacted format */ 439 if (hdr->xcomp_bv) 440 return -EINVAL; 441 442 /* 443 * If 'reserved' is shrunken to add a new field, make sure to validate 444 * that new field here! 445 */ 446 BUILD_BUG_ON(sizeof(hdr->reserved) != 48); 447 448 /* No reserved bits may be set */ 449 if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) 450 return -EINVAL; 451 452 return 0; 453 } 454 455 static void __init __xstate_dump_leaves(void) 456 { 457 int i; 458 u32 eax, ebx, ecx, edx; 459 static int should_dump = 1; 460 461 if (!should_dump) 462 return; 463 should_dump = 0; 464 /* 465 * Dump out a few leaves past the ones that we support 466 * just in case there are some goodies up there 467 */ 468 for (i = 0; i < XFEATURE_MAX + 10; i++) { 469 cpuid_count(CPUID_LEAF_XSTATE, i, &eax, &ebx, &ecx, &edx); 470 pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", 471 CPUID_LEAF_XSTATE, i, eax, ebx, ecx, edx); 472 } 473 } 474 475 #define XSTATE_WARN_ON(x, fmt, ...) do { \ 476 if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) { \ 477 __xstate_dump_leaves(); \ 478 } \ 479 } while (0) 480 481 #define XCHECK_SZ(sz, nr, __struct) ({ \ 482 if (WARN_ONCE(sz != sizeof(__struct), \ 483 "[%s]: struct is %zu bytes, cpu state %d bytes\n", \ 484 xfeature_names[nr], sizeof(__struct), sz)) { \ 485 __xstate_dump_leaves(); \ 486 } \ 487 true; \ 488 }) 489 490 491 /** 492 * check_xtile_data_against_struct - Check tile data state size. 493 * 494 * Calculate the state size by multiplying the single tile size which is 495 * recorded in a C struct, and the number of tiles that the CPU informs. 496 * Compare the provided size with the calculation. 497 * 498 * @size: The tile data state size 499 * 500 * Returns: 0 on success, -EINVAL on mismatch. 501 */ 502 static int __init check_xtile_data_against_struct(int size) 503 { 504 u32 max_palid, palid, state_size; 505 u32 eax, ebx, ecx, edx; 506 u16 max_tile; 507 508 /* 509 * Check the maximum palette id: 510 * eax: the highest numbered palette subleaf. 511 */ 512 cpuid_count(CPUID_LEAF_TILE, 0, &max_palid, &ebx, &ecx, &edx); 513 514 /* 515 * Cross-check each tile size and find the maximum number of 516 * supported tiles. 517 */ 518 for (palid = 1, max_tile = 0; palid <= max_palid; palid++) { 519 u16 tile_size, max; 520 521 /* 522 * Check the tile size info: 523 * eax[31:16]: bytes per title 524 * ebx[31:16]: the max names (or max number of tiles) 525 */ 526 cpuid_count(CPUID_LEAF_TILE, palid, &eax, &ebx, &edx, &edx); 527 tile_size = eax >> 16; 528 max = ebx >> 16; 529 530 if (tile_size != sizeof(struct xtile_data)) { 531 pr_err("%s: struct is %zu bytes, cpu xtile %d bytes\n", 532 __stringify(XFEATURE_XTILE_DATA), 533 sizeof(struct xtile_data), tile_size); 534 __xstate_dump_leaves(); 535 return -EINVAL; 536 } 537 538 if (max > max_tile) 539 max_tile = max; 540 } 541 542 state_size = sizeof(struct xtile_data) * max_tile; 543 if (size != state_size) { 544 pr_err("%s: calculated size is %u bytes, cpu state %d bytes\n", 545 __stringify(XFEATURE_XTILE_DATA), state_size, size); 546 __xstate_dump_leaves(); 547 return -EINVAL; 548 } 549 return 0; 550 } 551 552 /* 553 * We have a C struct for each 'xstate'. We need to ensure 554 * that our software representation matches what the CPU 555 * tells us about the state's size. 556 */ 557 static bool __init check_xstate_against_struct(int nr) 558 { 559 /* 560 * Ask the CPU for the size of the state. 561 */ 562 int sz = xfeature_size(nr); 563 564 /* 565 * Match each CPU state with the corresponding software 566 * structure. 567 */ 568 switch (nr) { 569 case XFEATURE_YMM: return XCHECK_SZ(sz, nr, struct ymmh_struct); 570 case XFEATURE_BNDREGS: return XCHECK_SZ(sz, nr, struct mpx_bndreg_state); 571 case XFEATURE_BNDCSR: return XCHECK_SZ(sz, nr, struct mpx_bndcsr_state); 572 case XFEATURE_OPMASK: return XCHECK_SZ(sz, nr, struct avx_512_opmask_state); 573 case XFEATURE_ZMM_Hi256: return XCHECK_SZ(sz, nr, struct avx_512_zmm_uppers_state); 574 case XFEATURE_Hi16_ZMM: return XCHECK_SZ(sz, nr, struct avx_512_hi16_state); 575 case XFEATURE_PKRU: return XCHECK_SZ(sz, nr, struct pkru_state); 576 case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state); 577 case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg); 578 case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state); 579 case XFEATURE_CET_KERNEL: return XCHECK_SZ(sz, nr, struct cet_supervisor_state); 580 case XFEATURE_APX: return XCHECK_SZ(sz, nr, struct apx_state); 581 case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true; 582 default: 583 XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr); 584 return false; 585 } 586 587 return true; 588 } 589 590 unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) 591 { 592 unsigned int topmost, offset, i; 593 594 if (!(xfeatures & ~XFEATURE_MASK_FPSSE)) 595 return sizeof(struct xregs_state); 596 597 topmost = fls64(xfeatures) - 1; 598 599 if (compacted) { 600 offset = xfeature_get_offset(xfeatures, topmost); 601 } else { 602 /* Walk through the xfeature order to pick the last */ 603 for_each_extended_xfeature_in_order(i, xfeatures) 604 topmost = xfeature_uncompact_order[i]; 605 offset = xstate_offsets[topmost]; 606 } 607 608 return offset + xstate_sizes[topmost]; 609 } 610 611 /* 612 * This essentially double-checks what the cpu told us about 613 * how large the XSAVE buffer needs to be. We are recalculating 614 * it to be safe. 615 * 616 * Independent XSAVE features allocate their own buffers and are not 617 * covered by these checks. Only the size of the buffer for task->fpu 618 * is checked here. 619 */ 620 static bool __init paranoid_xstate_size_valid(unsigned int kernel_size) 621 { 622 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); 623 bool xsaves = cpu_feature_enabled(X86_FEATURE_XSAVES); 624 unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE; 625 int i; 626 627 for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { 628 if (!check_xstate_against_struct(i)) 629 return false; 630 /* 631 * Supervisor state components can be managed only by 632 * XSAVES. 633 */ 634 if (!xsaves && xfeature_is_supervisor(i)) { 635 XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i); 636 return false; 637 } 638 } 639 size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted); 640 XSTATE_WARN_ON(size != kernel_size, 641 "size %u != kernel_size %u\n", size, kernel_size); 642 return size == kernel_size; 643 } 644 645 /* 646 * Get total size of enabled xstates in XCR0 | IA32_XSS. 647 * 648 * Note the SDM's wording here. "sub-function 0" only enumerates 649 * the size of the *user* states. If we use it to size a buffer 650 * that we use 'XSAVES' on, we could potentially overflow the 651 * buffer because 'XSAVES' saves system states too. 652 * 653 * This also takes compaction into account. So this works for 654 * XSAVEC as well. 655 */ 656 static unsigned int __init get_compacted_size(void) 657 { 658 unsigned int eax, ebx, ecx, edx; 659 /* 660 * - CPUID function 0DH, sub-function 1: 661 * EBX enumerates the size (in bytes) required by 662 * the XSAVES instruction for an XSAVE area 663 * containing all the state components 664 * corresponding to bits currently set in 665 * XCR0 | IA32_XSS. 666 * 667 * When XSAVES is not available but XSAVEC is (virt), then there 668 * are no supervisor states, but XSAVEC still uses compacted 669 * format. 670 */ 671 cpuid_count(CPUID_LEAF_XSTATE, 1, &eax, &ebx, &ecx, &edx); 672 return ebx; 673 } 674 675 /* 676 * Get the total size of the enabled xstates without the independent supervisor 677 * features. 678 */ 679 static unsigned int __init get_xsave_compacted_size(void) 680 { 681 u64 mask = xfeatures_mask_independent(); 682 unsigned int size; 683 684 if (!mask) 685 return get_compacted_size(); 686 687 /* Disable independent features. */ 688 wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor()); 689 690 /* 691 * Ask the hardware what size is required of the buffer. 692 * This is the size required for the task->fpu buffer. 693 */ 694 size = get_compacted_size(); 695 696 /* Re-enable independent features so XSAVES will work on them again. */ 697 wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); 698 699 return size; 700 } 701 702 static unsigned int __init get_xsave_size_user(void) 703 { 704 unsigned int eax, ebx, ecx, edx; 705 /* 706 * - CPUID function 0DH, sub-function 0: 707 * EBX enumerates the size (in bytes) required by 708 * the XSAVE instruction for an XSAVE area 709 * containing all the *user* state components 710 * corresponding to bits currently set in XCR0. 711 */ 712 cpuid_count(CPUID_LEAF_XSTATE, 0, &eax, &ebx, &ecx, &edx); 713 return ebx; 714 } 715 716 static int __init init_xstate_size(void) 717 { 718 /* Recompute the context size for enabled features: */ 719 unsigned int user_size, kernel_size, kernel_default_size; 720 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); 721 722 /* Uncompacted user space size */ 723 user_size = get_xsave_size_user(); 724 725 /* 726 * XSAVES kernel size includes supervisor states and uses compacted 727 * format. XSAVEC uses compacted format, but does not save 728 * supervisor states. 729 * 730 * XSAVE[OPT] do not support supervisor states so kernel and user 731 * size is identical. 732 */ 733 if (compacted) 734 kernel_size = get_xsave_compacted_size(); 735 else 736 kernel_size = user_size; 737 738 kernel_default_size = 739 xstate_calculate_size(fpu_kernel_cfg.default_features, compacted); 740 741 if (!paranoid_xstate_size_valid(kernel_size)) 742 return -EINVAL; 743 744 fpu_kernel_cfg.max_size = kernel_size; 745 fpu_user_cfg.max_size = user_size; 746 747 fpu_kernel_cfg.default_size = kernel_default_size; 748 fpu_user_cfg.default_size = 749 xstate_calculate_size(fpu_user_cfg.default_features, false); 750 751 guest_default_cfg.size = 752 xstate_calculate_size(guest_default_cfg.features, compacted); 753 754 return 0; 755 } 756 757 /* 758 * We enabled the XSAVE hardware, but something went wrong and 759 * we can not use it. Disable it. 760 */ 761 static void __init fpu__init_disable_system_xstate(unsigned int legacy_size) 762 { 763 pr_info("x86/fpu: XSAVE disabled\n"); 764 765 fpu_kernel_cfg.max_features = 0; 766 cr4_clear_bits(X86_CR4_OSXSAVE); 767 setup_clear_cpu_cap(X86_FEATURE_XSAVE); 768 769 /* Restore the legacy size.*/ 770 fpu_kernel_cfg.max_size = legacy_size; 771 fpu_kernel_cfg.default_size = legacy_size; 772 fpu_user_cfg.max_size = legacy_size; 773 fpu_user_cfg.default_size = legacy_size; 774 guest_default_cfg.size = legacy_size; 775 776 /* 777 * Prevent enabling the static branch which enables writes to the 778 * XFD MSR. 779 */ 780 init_fpstate.xfd = 0; 781 782 fpstate_reset(x86_task_fpu(current)); 783 } 784 785 static u64 __init host_default_mask(void) 786 { 787 /* 788 * Exclude dynamic features (require userspace opt-in) and features 789 * that are supported only for KVM guests. 790 */ 791 return ~((u64)XFEATURE_MASK_USER_DYNAMIC | XFEATURE_MASK_GUEST_SUPERVISOR); 792 } 793 794 static u64 __init guest_default_mask(void) 795 { 796 /* 797 * Exclude dynamic features, which require userspace opt-in even 798 * for KVM guests. 799 */ 800 return ~(u64)XFEATURE_MASK_USER_DYNAMIC; 801 } 802 803 /* 804 * Enable and initialize the xsave feature. 805 * Called once per system bootup. 806 */ 807 void __init fpu__init_system_xstate(unsigned int legacy_size) 808 { 809 unsigned int eax, ebx, ecx, edx; 810 u64 xfeatures, mask; 811 int err; 812 int i; 813 814 if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 815 pr_info("x86/fpu: x87 FPU will use %s\n", 816 boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE"); 817 /* Disable all dependent flags too */ 818 setup_clear_cpu_cap(X86_FEATURE_XSAVE); 819 return; 820 } 821 822 /* 823 * Find user xstates supported by the processor. 824 */ 825 cpuid_count(CPUID_LEAF_XSTATE, 0, &eax, &ebx, &ecx, &edx); 826 fpu_kernel_cfg.max_features = eax + ((u64)edx << 32); 827 828 /* 829 * Find supervisor xstates supported by the processor. 830 */ 831 cpuid_count(CPUID_LEAF_XSTATE, 1, &eax, &ebx, &ecx, &edx); 832 fpu_kernel_cfg.max_features |= ecx + ((u64)edx << 32); 833 834 mask = XFEATURE_MASK_FPSSE; 835 if ((fpu_kernel_cfg.max_features & mask) != mask) { 836 /* 837 * This indicates that something really unexpected happened 838 * with the enumeration. Disable XSAVE and try to continue 839 * booting without it. This is too early to BUG(). 840 */ 841 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", 842 fpu_kernel_cfg.max_features); 843 goto out_disable; 844 } 845 846 mask |= XFEATURE_MASK_YMM; 847 if (boot_cpu_has(X86_FEATURE_AVX)) { 848 if ((fpu_kernel_cfg.max_features & mask) != mask) { 849 pr_err(FW_BUG 850 "x86/fpu: Disabling AVX support due to missing xstate features\n"); 851 setup_clear_cpu_cap(X86_FEATURE_AVX); 852 } 853 } 854 855 mask |= XFEATURE_MASK_AVX512; 856 if (boot_cpu_has(X86_FEATURE_AVX512F)) { 857 if ((fpu_kernel_cfg.max_features & mask) != mask) { 858 pr_err(FW_BUG 859 "x86/fpu: Disabling AVX-512 support due to missing xstate features\n"); 860 setup_clear_cpu_cap(X86_FEATURE_AVX512F); 861 } 862 } 863 864 if (fpu_kernel_cfg.max_features & XFEATURE_MASK_APX && 865 fpu_kernel_cfg.max_features & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR)) { 866 /* 867 * This is a problematic CPU configuration where two 868 * conflicting state components are both enumerated. 869 */ 870 pr_err("x86/fpu: Both APX/MPX present in the CPU's xstate features: 0x%llx.\n", 871 fpu_kernel_cfg.max_features); 872 goto out_disable; 873 } 874 875 fpu_kernel_cfg.independent_features = fpu_kernel_cfg.max_features & 876 XFEATURE_MASK_INDEPENDENT; 877 878 /* 879 * Clear XSAVE features that are disabled in the normal CPUID. 880 */ 881 for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { 882 unsigned short cid = xsave_cpuid_features[i]; 883 884 /* Careful: X86_FEATURE_FPU is 0! */ 885 if ((i != XFEATURE_FP && !cid) || !boot_cpu_has(cid)) 886 fpu_kernel_cfg.max_features &= ~BIT_ULL(i); 887 } 888 889 if (!cpu_feature_enabled(X86_FEATURE_XFD)) 890 fpu_kernel_cfg.max_features &= ~XFEATURE_MASK_USER_DYNAMIC; 891 892 if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) 893 fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED; 894 else 895 fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED | 896 XFEATURE_MASK_SUPERVISOR_SUPPORTED; 897 898 fpu_user_cfg.max_features = fpu_kernel_cfg.max_features; 899 fpu_user_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED; 900 901 /* 902 * Now, given maximum feature set, determine default values by 903 * applying default masks. 904 */ 905 fpu_kernel_cfg.default_features = fpu_kernel_cfg.max_features & host_default_mask(); 906 fpu_user_cfg.default_features = fpu_user_cfg.max_features & host_default_mask(); 907 guest_default_cfg.features = fpu_kernel_cfg.max_features & guest_default_mask(); 908 909 /* Store it for paranoia check at the end */ 910 xfeatures = fpu_kernel_cfg.max_features; 911 912 /* 913 * Initialize the default XFD state in initfp_state and enable the 914 * dynamic sizing mechanism if dynamic states are available. The 915 * static key cannot be enabled here because this runs before 916 * jump_label_init(). This is delayed to an initcall. 917 */ 918 init_fpstate.xfd = fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC; 919 920 /* Set up compaction feature bit */ 921 if (cpu_feature_enabled(X86_FEATURE_XSAVEC) || 922 cpu_feature_enabled(X86_FEATURE_XSAVES)) 923 setup_force_cpu_cap(X86_FEATURE_XCOMPACTED); 924 925 /* Enable xstate instructions to be able to continue with initialization: */ 926 fpu__init_cpu_xstate(); 927 928 /* Cache size, offset and flags for initialization */ 929 setup_xstate_cache(); 930 931 err = init_xstate_size(); 932 if (err) 933 goto out_disable; 934 935 /* 936 * Update info used for ptrace frames; use standard-format size and no 937 * supervisor xstates: 938 */ 939 update_regset_xstate_info(fpu_user_cfg.max_size, 940 fpu_user_cfg.max_features); 941 942 /* 943 * init_fpstate excludes dynamic states as they are large but init 944 * state is zero. 945 */ 946 init_fpstate.size = fpu_kernel_cfg.default_size; 947 init_fpstate.xfeatures = fpu_kernel_cfg.default_features; 948 949 if (init_fpstate.size > sizeof(init_fpstate.regs)) { 950 pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d)\n", 951 sizeof(init_fpstate.regs), init_fpstate.size); 952 goto out_disable; 953 } 954 955 setup_init_fpu_buf(); 956 957 /* 958 * Paranoia check whether something in the setup modified the 959 * xfeatures mask. 960 */ 961 if (xfeatures != fpu_kernel_cfg.max_features) { 962 pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init\n", 963 xfeatures, fpu_kernel_cfg.max_features); 964 goto out_disable; 965 } 966 967 /* 968 * CPU capabilities initialization runs before FPU init. So 969 * X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely 970 * functional, set the feature bit so depending code works. 971 */ 972 setup_force_cpu_cap(X86_FEATURE_OSXSAVE); 973 974 print_xstate_offset_size(); 975 pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", 976 fpu_kernel_cfg.max_features, 977 fpu_kernel_cfg.max_size, 978 boot_cpu_has(X86_FEATURE_XCOMPACTED) ? "compacted" : "standard"); 979 return; 980 981 out_disable: 982 /* something went wrong, try to boot without any XSAVE support */ 983 fpu__init_disable_system_xstate(legacy_size); 984 } 985 986 /* 987 * Restore minimal FPU state after suspend: 988 */ 989 void fpu__resume_cpu(void) 990 { 991 /* 992 * Restore XCR0 on xsave capable CPUs: 993 */ 994 if (cpu_feature_enabled(X86_FEATURE_XSAVE)) 995 xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features); 996 997 /* 998 * Restore IA32_XSS. The same CPUID bit enumerates support 999 * of XSAVES and MSR_IA32_XSS. 1000 */ 1001 if (cpu_feature_enabled(X86_FEATURE_XSAVES)) { 1002 wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor() | 1003 xfeatures_mask_independent()); 1004 } 1005 1006 if (fpu_state_size_dynamic()) 1007 wrmsrq(MSR_IA32_XFD, x86_task_fpu(current)->fpstate->xfd); 1008 } 1009 1010 /* 1011 * Given an xstate feature nr, calculate where in the xsave 1012 * buffer the state is. Callers should ensure that the buffer 1013 * is valid. 1014 */ 1015 static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 1016 { 1017 u64 xcomp_bv = xsave->header.xcomp_bv; 1018 1019 if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) 1020 return NULL; 1021 1022 if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED)) { 1023 if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr)))) 1024 return NULL; 1025 } 1026 1027 return (void *)xsave + xfeature_get_offset(xcomp_bv, xfeature_nr); 1028 } 1029 1030 /* 1031 * Given the xsave area and a state inside, this function returns the 1032 * address of the state. 1033 * 1034 * This is the API that is called to get xstate address in either 1035 * standard format or compacted format of xsave area. 1036 * 1037 * Note that if there is no data for the field in the xsave buffer 1038 * this will return NULL. 1039 * 1040 * Inputs: 1041 * xstate: the thread's storage area for all FPU data 1042 * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, 1043 * XFEATURE_SSE, etc...) 1044 * Output: 1045 * address of the state in the xsave area, or NULL if the 1046 * field is not present in the xsave buffer. 1047 */ 1048 void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 1049 { 1050 /* 1051 * Do we even *have* xsave state? 1052 */ 1053 if (!boot_cpu_has(X86_FEATURE_XSAVE)) 1054 return NULL; 1055 1056 /* 1057 * We should not ever be requesting features that we 1058 * have not enabled. 1059 */ 1060 if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) 1061 return NULL; 1062 1063 /* 1064 * This assumes the last 'xsave*' instruction to 1065 * have requested that 'xfeature_nr' be saved. 1066 * If it did not, we might be seeing and old value 1067 * of the field in the buffer. 1068 * 1069 * This can happen because the last 'xsave' did not 1070 * request that this feature be saved (unlikely) 1071 * or because the "init optimization" caused it 1072 * to not be saved. 1073 */ 1074 if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr))) 1075 return NULL; 1076 1077 return __raw_xsave_addr(xsave, xfeature_nr); 1078 } 1079 EXPORT_SYMBOL_FOR_KVM(get_xsave_addr); 1080 1081 /* 1082 * Given an xstate feature nr, calculate where in the xsave buffer the state is. 1083 * The xsave buffer should be in standard format, not compacted (e.g. user mode 1084 * signal frames). 1085 */ 1086 void __user *get_xsave_addr_user(struct xregs_state __user *xsave, int xfeature_nr) 1087 { 1088 if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) 1089 return NULL; 1090 1091 return (void __user *)xsave + xstate_offsets[xfeature_nr]; 1092 } 1093 1094 #ifdef CONFIG_ARCH_HAS_PKEYS 1095 1096 /* 1097 * This will go out and modify PKRU register to set the access 1098 * rights for @pkey to @init_val. 1099 */ 1100 int arch_set_user_pkey_access(int pkey, unsigned long init_val) 1101 { 1102 u32 old_pkru, new_pkru_bits = 0; 1103 int pkey_shift; 1104 1105 /* 1106 * This check implies XSAVE support. OSPKE only gets 1107 * set if we enable XSAVE and we enable PKU in XCR0. 1108 */ 1109 if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) 1110 return -EINVAL; 1111 1112 /* 1113 * This code should only be called with valid 'pkey' 1114 * values originating from in-kernel users. Complain 1115 * if a bad value is observed. 1116 */ 1117 if (WARN_ON_ONCE(pkey >= arch_max_pkey())) 1118 return -EINVAL; 1119 1120 /* Set the bits we need in PKRU: */ 1121 if (init_val & PKEY_DISABLE_ACCESS) 1122 new_pkru_bits |= PKRU_AD_BIT; 1123 if (init_val & PKEY_DISABLE_WRITE) 1124 new_pkru_bits |= PKRU_WD_BIT; 1125 1126 /* Shift the bits in to the correct place in PKRU for pkey: */ 1127 pkey_shift = pkey * PKRU_BITS_PER_PKEY; 1128 new_pkru_bits <<= pkey_shift; 1129 1130 /* Get old PKRU and mask off any old bits in place: */ 1131 old_pkru = read_pkru(); 1132 old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); 1133 1134 /* Write old part along with new part: */ 1135 write_pkru(old_pkru | new_pkru_bits); 1136 1137 return 0; 1138 } 1139 #endif /* ! CONFIG_ARCH_HAS_PKEYS */ 1140 1141 static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, 1142 void *init_xstate, unsigned int size) 1143 { 1144 membuf_write(to, from_xstate ? xstate : init_xstate, size); 1145 } 1146 1147 /** 1148 * __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer 1149 * @to: membuf descriptor 1150 * @fpstate: The fpstate buffer from which to copy 1151 * @xfeatures: The mask of xfeatures to save (XSAVE mode only) 1152 * @pkru_val: The PKRU value to store in the PKRU component 1153 * @copy_mode: The requested copy mode 1154 * 1155 * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming 1156 * format, i.e. from the kernel internal hardware dependent storage format 1157 * to the requested @mode. UABI XSTATE is always uncompacted! 1158 * 1159 * It supports partial copy but @to.pos always starts from zero. 1160 */ 1161 void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, 1162 u64 xfeatures, u32 pkru_val, 1163 enum xstate_copy_mode copy_mode) 1164 { 1165 const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); 1166 struct xregs_state *xinit = &init_fpstate.regs.xsave; 1167 struct xregs_state *xsave = &fpstate->regs.xsave; 1168 unsigned int zerofrom, i, xfeature; 1169 struct xstate_header header; 1170 u64 mask; 1171 1172 memset(&header, 0, sizeof(header)); 1173 header.xfeatures = xsave->header.xfeatures; 1174 1175 /* Mask out the feature bits depending on copy mode */ 1176 switch (copy_mode) { 1177 case XSTATE_COPY_FP: 1178 header.xfeatures &= XFEATURE_MASK_FP; 1179 break; 1180 1181 case XSTATE_COPY_FX: 1182 header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE; 1183 break; 1184 1185 case XSTATE_COPY_XSAVE: 1186 header.xfeatures &= fpstate->user_xfeatures & xfeatures; 1187 break; 1188 } 1189 1190 /* Copy FP state up to MXCSR */ 1191 copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387, 1192 &xinit->i387, off_mxcsr); 1193 1194 /* Copy MXCSR when SSE or YMM are set in the feature mask */ 1195 copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM), 1196 &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr, 1197 MXCSR_AND_FLAGS_SIZE); 1198 1199 /* Copy the remaining FP state */ 1200 copy_feature(header.xfeatures & XFEATURE_MASK_FP, 1201 &to, &xsave->i387.st_space, &xinit->i387.st_space, 1202 sizeof(xsave->i387.st_space)); 1203 1204 /* Copy the SSE state - shared with YMM, but independently managed */ 1205 copy_feature(header.xfeatures & XFEATURE_MASK_SSE, 1206 &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space, 1207 sizeof(xsave->i387.xmm_space)); 1208 1209 if (copy_mode != XSTATE_COPY_XSAVE) 1210 goto out; 1211 1212 /* Zero the padding area */ 1213 membuf_zero(&to, sizeof(xsave->i387.padding)); 1214 1215 /* Copy xsave->i387.sw_reserved */ 1216 membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved)); 1217 1218 /* Copy the user space relevant state of @xsave->header */ 1219 membuf_write(&to, &header, sizeof(header)); 1220 1221 zerofrom = offsetof(struct xregs_state, extended_state_area); 1222 1223 /* 1224 * This 'mask' indicates which states to copy from fpstate. 1225 * Those extended states that are not present in fpstate are 1226 * either disabled or initialized: 1227 * 1228 * In non-compacted format, disabled features still occupy 1229 * state space but there is no state to copy from in the 1230 * compacted init_fpstate. The gap tracking will zero these 1231 * states. 1232 * 1233 * The extended features have an all zeroes init state. Thus, 1234 * remove them from 'mask' to zero those features in the user 1235 * buffer instead of retrieving them from init_fpstate. 1236 */ 1237 mask = header.xfeatures; 1238 1239 for_each_extended_xfeature_in_order(i, mask) { 1240 xfeature = xfeature_uncompact_order[i]; 1241 /* 1242 * If there was a feature or alignment gap, zero the space 1243 * in the destination buffer. 1244 */ 1245 if (zerofrom < xstate_offsets[xfeature]) 1246 membuf_zero(&to, xstate_offsets[xfeature] - zerofrom); 1247 1248 if (xfeature == XFEATURE_PKRU) { 1249 struct pkru_state pkru = {0}; 1250 /* 1251 * PKRU is not necessarily up to date in the 1252 * XSAVE buffer. Use the provided value. 1253 */ 1254 pkru.pkru = pkru_val; 1255 membuf_write(&to, &pkru, sizeof(pkru)); 1256 } else { 1257 membuf_write(&to, 1258 __raw_xsave_addr(xsave, xfeature), 1259 xstate_sizes[xfeature]); 1260 } 1261 /* 1262 * Keep track of the last copied state in the non-compacted 1263 * target buffer for gap zeroing. 1264 */ 1265 zerofrom = xstate_offsets[xfeature] + xstate_sizes[xfeature]; 1266 } 1267 1268 out: 1269 if (to.left) 1270 membuf_zero(&to, to.left); 1271 } 1272 1273 /** 1274 * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer 1275 * @to: membuf descriptor 1276 * @tsk: The task from which to copy the saved xstate 1277 * @copy_mode: The requested copy mode 1278 * 1279 * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming 1280 * format, i.e. from the kernel internal hardware dependent storage format 1281 * to the requested @mode. UABI XSTATE is always uncompacted! 1282 * 1283 * It supports partial copy but @to.pos always starts from zero. 1284 */ 1285 void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, 1286 enum xstate_copy_mode copy_mode) 1287 { 1288 __copy_xstate_to_uabi_buf(to, x86_task_fpu(tsk)->fpstate, 1289 x86_task_fpu(tsk)->fpstate->user_xfeatures, 1290 tsk->thread.pkru, copy_mode); 1291 } 1292 1293 static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size, 1294 const void *kbuf, const void __user *ubuf) 1295 { 1296 if (kbuf) { 1297 memcpy(dst, kbuf + offset, size); 1298 } else { 1299 if (copy_from_user(dst, ubuf + offset, size)) 1300 return -EFAULT; 1301 } 1302 return 0; 1303 } 1304 1305 1306 /** 1307 * copy_uabi_to_xstate - Copy a UABI format buffer to the kernel xstate 1308 * @fpstate: The fpstate buffer to copy to 1309 * @kbuf: The UABI format buffer, if it comes from the kernel 1310 * @ubuf: The UABI format buffer, if it comes from userspace 1311 * @pkru: The location to write the PKRU value to 1312 * 1313 * Converts from the UABI format into the kernel internal hardware 1314 * dependent format. 1315 * 1316 * This function ultimately has three different callers with distinct PKRU 1317 * behavior. 1318 * 1. When called from sigreturn the PKRU register will be restored from 1319 * @fpstate via an XRSTOR. Correctly copying the UABI format buffer to 1320 * @fpstate is sufficient to cover this case, but the caller will also 1321 * pass a pointer to the thread_struct's pkru field in @pkru and updating 1322 * it is harmless. 1323 * 2. When called from ptrace the PKRU register will be restored from the 1324 * thread_struct's pkru field. A pointer to that is passed in @pkru. 1325 * The kernel will restore it manually, so the XRSTOR behavior that resets 1326 * the PKRU register to the hardware init value (0) if the corresponding 1327 * xfeatures bit is not set is emulated here. 1328 * 3. When called from KVM the PKRU register will be restored from the vcpu's 1329 * pkru field. A pointer to that is passed in @pkru. KVM hasn't used 1330 * XRSTOR and hasn't had the PKRU resetting behavior described above. To 1331 * preserve that KVM behavior, it passes NULL for @pkru if the xfeatures 1332 * bit is not set. 1333 */ 1334 static int copy_uabi_to_xstate(struct fpstate *fpstate, const void *kbuf, 1335 const void __user *ubuf, u32 *pkru) 1336 { 1337 struct xregs_state *xsave = &fpstate->regs.xsave; 1338 unsigned int offset, size; 1339 struct xstate_header hdr; 1340 u64 mask; 1341 int i; 1342 1343 offset = offsetof(struct xregs_state, header); 1344 if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf)) 1345 return -EFAULT; 1346 1347 if (validate_user_xstate_header(&hdr, fpstate)) 1348 return -EINVAL; 1349 1350 /* Validate MXCSR when any of the related features is in use */ 1351 mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM; 1352 if (hdr.xfeatures & mask) { 1353 u32 mxcsr[2]; 1354 1355 offset = offsetof(struct fxregs_state, mxcsr); 1356 if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf)) 1357 return -EFAULT; 1358 1359 /* Reserved bits in MXCSR must be zero. */ 1360 if (mxcsr[0] & ~mxcsr_feature_mask) 1361 return -EINVAL; 1362 1363 /* SSE and YMM require MXCSR even when FP is not in use. */ 1364 if (!(hdr.xfeatures & XFEATURE_MASK_FP)) { 1365 xsave->i387.mxcsr = mxcsr[0]; 1366 xsave->i387.mxcsr_mask = mxcsr[1]; 1367 } 1368 } 1369 1370 for (i = 0; i < XFEATURE_MAX; i++) { 1371 mask = BIT_ULL(i); 1372 1373 if (hdr.xfeatures & mask) { 1374 void *dst = __raw_xsave_addr(xsave, i); 1375 1376 offset = xstate_offsets[i]; 1377 size = xstate_sizes[i]; 1378 1379 if (copy_from_buffer(dst, offset, size, kbuf, ubuf)) 1380 return -EFAULT; 1381 } 1382 } 1383 1384 if (hdr.xfeatures & XFEATURE_MASK_PKRU) { 1385 struct pkru_state *xpkru; 1386 1387 xpkru = __raw_xsave_addr(xsave, XFEATURE_PKRU); 1388 *pkru = xpkru->pkru; 1389 } else { 1390 /* 1391 * KVM may pass NULL here to indicate that it does not need 1392 * PKRU updated. 1393 */ 1394 if (pkru) 1395 *pkru = 0; 1396 } 1397 1398 /* 1399 * The state that came in from userspace was user-state only. 1400 * Mask all the user states out of 'xfeatures': 1401 */ 1402 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; 1403 1404 /* 1405 * Add back in the features that came in from userspace: 1406 */ 1407 xsave->header.xfeatures |= hdr.xfeatures; 1408 1409 return 0; 1410 } 1411 1412 /* 1413 * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] 1414 * format and copy to the target thread. Used by ptrace and KVM. 1415 */ 1416 int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru) 1417 { 1418 return copy_uabi_to_xstate(fpstate, kbuf, NULL, pkru); 1419 } 1420 1421 /* 1422 * Convert from a sigreturn standard-format user-space buffer to kernel 1423 * XSAVE[S] format and copy to the target thread. This is called from the 1424 * sigreturn() and rt_sigreturn() system calls. 1425 */ 1426 int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, 1427 const void __user *ubuf) 1428 { 1429 return copy_uabi_to_xstate(x86_task_fpu(tsk)->fpstate, NULL, ubuf, &tsk->thread.pkru); 1430 } 1431 1432 static bool validate_independent_components(u64 mask) 1433 { 1434 u64 xchk; 1435 1436 if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES))) 1437 return false; 1438 1439 xchk = ~xfeatures_mask_independent(); 1440 1441 if (WARN_ON_ONCE(!mask || mask & xchk)) 1442 return false; 1443 1444 return true; 1445 } 1446 1447 /** 1448 * xsaves - Save selected components to a kernel xstate buffer 1449 * @xstate: Pointer to the buffer 1450 * @mask: Feature mask to select the components to save 1451 * 1452 * The @xstate buffer must be 64 byte aligned and correctly initialized as 1453 * XSAVES does not write the full xstate header. Before first use the 1454 * buffer should be zeroed otherwise a consecutive XRSTORS from that buffer 1455 * can #GP. 1456 * 1457 * The feature mask must be a subset of the independent features. 1458 */ 1459 void xsaves(struct xregs_state *xstate, u64 mask) 1460 { 1461 int err; 1462 1463 if (!validate_independent_components(mask)) 1464 return; 1465 1466 XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); 1467 WARN_ON_ONCE(err); 1468 } 1469 1470 /** 1471 * xrstors - Restore selected components from a kernel xstate buffer 1472 * @xstate: Pointer to the buffer 1473 * @mask: Feature mask to select the components to restore 1474 * 1475 * The @xstate buffer must be 64 byte aligned and correctly initialized 1476 * otherwise XRSTORS from that buffer can #GP. 1477 * 1478 * Proper usage is to restore the state which was saved with 1479 * xsaves() into @xstate. 1480 * 1481 * The feature mask must be a subset of the independent features. 1482 */ 1483 void xrstors(struct xregs_state *xstate, u64 mask) 1484 { 1485 int err; 1486 1487 if (!validate_independent_components(mask)) 1488 return; 1489 1490 XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err); 1491 WARN_ON_ONCE(err); 1492 } 1493 1494 /** 1495 * xsaves_nmi - Save selected components to a kernel xstate buffer in NMI 1496 * @xstate: Pointer to the buffer 1497 * @mask: Feature mask to select the components to save 1498 * 1499 * This function is similar to xsaves(), but should only be called within 1500 * the NMI handler. This function returns the actual register contents at 1501 * the moment the NMI occurs. 1502 * 1503 * Currently, the perf subsystem is the sole user of this helper. It uses 1504 * the function to snapshot SIMD (XMM/YMM/ZMM) and APX eGPRs registers. 1505 */ 1506 void xsaves_nmi(struct xregs_state *xstate, u64 mask) 1507 { 1508 int err; 1509 1510 if (!in_nmi()) 1511 return; 1512 1513 XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); 1514 WARN_ON_ONCE(err); 1515 } 1516 1517 #if IS_ENABLED(CONFIG_KVM) 1518 void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature) 1519 { 1520 void *addr = get_xsave_addr(&fpstate->regs.xsave, xfeature); 1521 1522 if (addr) 1523 memset(addr, 0, xstate_sizes[xfeature]); 1524 } 1525 EXPORT_SYMBOL_FOR_KVM(fpstate_clear_xstate_component); 1526 #endif 1527 1528 #ifdef CONFIG_X86_64 1529 1530 #ifdef CONFIG_X86_DEBUG_FPU 1531 /* 1532 * Ensure that a subsequent XSAVE* or XRSTOR* instruction with RFBM=@mask 1533 * can safely operate on the @fpstate buffer. 1534 */ 1535 static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor) 1536 { 1537 u64 xfd = __this_cpu_read(xfd_state); 1538 1539 if (fpstate->xfd == xfd) 1540 return true; 1541 1542 /* 1543 * The XFD MSR does not match fpstate->xfd. That's invalid when 1544 * the passed in fpstate is current's fpstate. 1545 */ 1546 if (fpstate->xfd == x86_task_fpu(current)->fpstate->xfd) 1547 return false; 1548 1549 /* 1550 * XRSTOR(S) from init_fpstate are always correct as it will just 1551 * bring all components into init state and not read from the 1552 * buffer. XSAVE(S) raises #PF after init. 1553 */ 1554 if (fpstate == &init_fpstate) 1555 return rstor; 1556 1557 /* 1558 * XSAVE(S): clone(), fpu_swap_kvm_fpstate() 1559 * XRSTORS(S): fpu_swap_kvm_fpstate() 1560 */ 1561 1562 /* 1563 * No XSAVE/XRSTOR instructions (except XSAVE itself) touch 1564 * the buffer area for XFD-disabled state components. 1565 */ 1566 mask &= ~xfd; 1567 1568 /* 1569 * Remove features which are valid in fpstate. They 1570 * have space allocated in fpstate. 1571 */ 1572 mask &= ~fpstate->xfeatures; 1573 1574 /* 1575 * Any remaining state components in 'mask' might be written 1576 * by XSAVE/XRSTOR. Fail validation it found. 1577 */ 1578 return !mask; 1579 } 1580 1581 void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) 1582 { 1583 WARN_ON_ONCE(!xstate_op_valid(fpstate, mask, rstor)); 1584 } 1585 #endif /* CONFIG_X86_DEBUG_FPU */ 1586 1587 static int __init xfd_update_static_branch(void) 1588 { 1589 /* 1590 * If init_fpstate.xfd has bits set then dynamic features are 1591 * available and the dynamic sizing must be enabled. 1592 */ 1593 if (init_fpstate.xfd) 1594 static_branch_enable(&__fpu_state_size_dynamic); 1595 return 0; 1596 } 1597 arch_initcall(xfd_update_static_branch) 1598 1599 void fpstate_free(struct fpu *fpu) 1600 { 1601 if (fpu->fpstate && fpu->fpstate != &fpu->__fpstate) 1602 vfree(fpu->fpstate); 1603 } 1604 1605 /** 1606 * fpstate_realloc - Reallocate struct fpstate for the requested new features 1607 * 1608 * @xfeatures: A bitmap of xstate features which extend the enabled features 1609 * of that task 1610 * @ksize: The required size for the kernel buffer 1611 * @usize: The required size for user space buffers 1612 * @guest_fpu: Pointer to a guest FPU container. NULL for host allocations 1613 * 1614 * Note vs. vmalloc(): If the task with a vzalloc()-allocated buffer 1615 * terminates quickly, vfree()-induced IPIs may be a concern, but tasks 1616 * with large states are likely to live longer. 1617 * 1618 * Returns: 0 on success, -ENOMEM on allocation error. 1619 */ 1620 static int fpstate_realloc(u64 xfeatures, unsigned int ksize, 1621 unsigned int usize, struct fpu_guest *guest_fpu) 1622 { 1623 struct fpu *fpu = x86_task_fpu(current); 1624 struct fpstate *curfps, *newfps = NULL; 1625 unsigned int fpsize; 1626 bool in_use; 1627 1628 fpsize = ksize + ALIGN(offsetof(struct fpstate, regs), 64); 1629 1630 newfps = vzalloc(fpsize); 1631 if (!newfps) 1632 return -ENOMEM; 1633 newfps->size = ksize; 1634 newfps->user_size = usize; 1635 newfps->is_valloc = true; 1636 1637 /* 1638 * When a guest FPU is supplied, use @guest_fpu->fpstate 1639 * as reference independent whether it is in use or not. 1640 */ 1641 curfps = guest_fpu ? guest_fpu->fpstate : fpu->fpstate; 1642 1643 /* Determine whether @curfps is the active fpstate */ 1644 in_use = fpu->fpstate == curfps; 1645 1646 if (guest_fpu) { 1647 newfps->is_guest = true; 1648 newfps->is_confidential = curfps->is_confidential; 1649 newfps->in_use = curfps->in_use; 1650 guest_fpu->xfeatures |= xfeatures; 1651 guest_fpu->uabi_size = usize; 1652 } 1653 1654 fpregs_lock(); 1655 /* 1656 * If @curfps is in use, ensure that the current state is in the 1657 * registers before swapping fpstate as that might invalidate it 1658 * due to layout changes. 1659 */ 1660 if (in_use && test_thread_flag(TIF_NEED_FPU_LOAD)) 1661 fpregs_restore_userregs(); 1662 1663 newfps->xfeatures = curfps->xfeatures | xfeatures; 1664 newfps->user_xfeatures = curfps->user_xfeatures | xfeatures; 1665 newfps->xfd = curfps->xfd & ~xfeatures; 1666 1667 /* Do the final updates within the locked region */ 1668 xstate_init_xcomp_bv(&newfps->regs.xsave, newfps->xfeatures); 1669 1670 if (guest_fpu) { 1671 guest_fpu->fpstate = newfps; 1672 /* If curfps is active, update the FPU fpstate pointer */ 1673 if (in_use) 1674 fpu->fpstate = newfps; 1675 } else { 1676 fpu->fpstate = newfps; 1677 } 1678 1679 if (in_use) 1680 xfd_update_state(fpu->fpstate); 1681 fpregs_unlock(); 1682 1683 /* Only free valloc'ed state */ 1684 if (curfps && curfps->is_valloc) 1685 vfree(curfps); 1686 1687 return 0; 1688 } 1689 1690 static int validate_sigaltstack(unsigned int usize) 1691 { 1692 struct task_struct *thread, *leader = current->group_leader; 1693 unsigned long framesize = get_sigframe_size(); 1694 1695 lockdep_assert_held(¤t->sighand->siglock); 1696 1697 /* get_sigframe_size() is based on fpu_user_cfg.max_size */ 1698 framesize -= fpu_user_cfg.max_size; 1699 framesize += usize; 1700 for_each_thread(leader, thread) { 1701 if (thread->sas_ss_size && thread->sas_ss_size < framesize) 1702 return -ENOSPC; 1703 } 1704 return 0; 1705 } 1706 1707 static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) 1708 { 1709 /* 1710 * This deliberately does not exclude !XSAVES as we still might 1711 * decide to optionally context switch XCR0 or talk the silicon 1712 * vendors into extending XFD for the pre AMX states, especially 1713 * AVX512. 1714 */ 1715 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); 1716 struct fpu *fpu = x86_task_fpu(current->group_leader); 1717 struct fpu_state_perm *perm; 1718 unsigned int ksize, usize; 1719 u64 mask; 1720 int ret = 0; 1721 1722 /* Check whether fully enabled */ 1723 if ((permitted & requested) == requested) 1724 return 0; 1725 1726 /* 1727 * Calculate the resulting kernel state size. Note, @permitted also 1728 * contains supervisor xfeatures even though supervisor are always 1729 * permitted for kernel and guest FPUs, and never permitted for user 1730 * FPUs. 1731 */ 1732 mask = permitted | requested; 1733 ksize = xstate_calculate_size(mask, compacted); 1734 1735 /* 1736 * Calculate the resulting user state size. Take care not to clobber 1737 * the supervisor xfeatures in the new mask! 1738 */ 1739 usize = xstate_calculate_size(mask & XFEATURE_MASK_USER_SUPPORTED, false); 1740 1741 if (!guest) { 1742 ret = validate_sigaltstack(usize); 1743 if (ret) 1744 return ret; 1745 } 1746 1747 perm = guest ? &fpu->guest_perm : &fpu->perm; 1748 /* Pairs with the READ_ONCE() in xstate_get_group_perm() */ 1749 WRITE_ONCE(perm->__state_perm, mask); 1750 /* Protected by sighand lock */ 1751 perm->__state_size = ksize; 1752 perm->__user_state_size = usize; 1753 return ret; 1754 } 1755 1756 /* 1757 * Permissions array to map facilities with more than one component 1758 */ 1759 static const u64 xstate_prctl_req[XFEATURE_MAX] = { 1760 [XFEATURE_XTILE_DATA] = XFEATURE_MASK_XTILE_DATA, 1761 }; 1762 1763 static int xstate_request_perm(unsigned long idx, bool guest) 1764 { 1765 u64 permitted, requested; 1766 int ret; 1767 1768 if (idx >= XFEATURE_MAX) 1769 return -EINVAL; 1770 1771 /* 1772 * Look up the facility mask which can require more than 1773 * one xstate component. 1774 */ 1775 idx = array_index_nospec(idx, ARRAY_SIZE(xstate_prctl_req)); 1776 requested = xstate_prctl_req[idx]; 1777 if (!requested) 1778 return -EOPNOTSUPP; 1779 1780 if ((fpu_user_cfg.max_features & requested) != requested) 1781 return -EOPNOTSUPP; 1782 1783 /* Lockless quick check */ 1784 permitted = xstate_get_group_perm(guest); 1785 if ((permitted & requested) == requested) 1786 return 0; 1787 1788 /* Protect against concurrent modifications */ 1789 spin_lock_irq(¤t->sighand->siglock); 1790 permitted = xstate_get_group_perm(guest); 1791 1792 /* First vCPU allocation locks the permissions. */ 1793 if (guest && (permitted & FPU_GUEST_PERM_LOCKED)) 1794 ret = -EBUSY; 1795 else 1796 ret = __xstate_request_perm(permitted, requested, guest); 1797 spin_unlock_irq(¤t->sighand->siglock); 1798 return ret; 1799 } 1800 1801 int __xfd_enable_feature(u64 xfd_err, struct fpu_guest *guest_fpu) 1802 { 1803 u64 xfd_event = xfd_err & XFEATURE_MASK_USER_DYNAMIC; 1804 struct fpu_state_perm *perm; 1805 unsigned int ksize, usize; 1806 struct fpu *fpu; 1807 1808 if (!xfd_event) { 1809 if (!guest_fpu) 1810 pr_err_once("XFD: Invalid xfd error: %016llx\n", xfd_err); 1811 return 0; 1812 } 1813 1814 /* Protect against concurrent modifications */ 1815 spin_lock_irq(¤t->sighand->siglock); 1816 1817 /* If not permitted let it die */ 1818 if ((xstate_get_group_perm(!!guest_fpu) & xfd_event) != xfd_event) { 1819 spin_unlock_irq(¤t->sighand->siglock); 1820 return -EPERM; 1821 } 1822 1823 fpu = x86_task_fpu(current->group_leader); 1824 perm = guest_fpu ? &fpu->guest_perm : &fpu->perm; 1825 ksize = perm->__state_size; 1826 usize = perm->__user_state_size; 1827 1828 /* 1829 * The feature is permitted. State size is sufficient. Dropping 1830 * the lock is safe here even if more features are added from 1831 * another task, the retrieved buffer sizes are valid for the 1832 * currently requested feature(s). 1833 */ 1834 spin_unlock_irq(¤t->sighand->siglock); 1835 1836 /* 1837 * Try to allocate a new fpstate. If that fails there is no way 1838 * out. 1839 */ 1840 if (fpstate_realloc(xfd_event, ksize, usize, guest_fpu)) 1841 return -EFAULT; 1842 return 0; 1843 } 1844 1845 int xfd_enable_feature(u64 xfd_err) 1846 { 1847 return __xfd_enable_feature(xfd_err, NULL); 1848 } 1849 1850 #else /* CONFIG_X86_64 */ 1851 static inline int xstate_request_perm(unsigned long idx, bool guest) 1852 { 1853 return -EPERM; 1854 } 1855 #endif /* !CONFIG_X86_64 */ 1856 1857 u64 xstate_get_guest_group_perm(void) 1858 { 1859 return xstate_get_group_perm(true); 1860 } 1861 EXPORT_SYMBOL_FOR_KVM(xstate_get_guest_group_perm); 1862 1863 /** 1864 * fpu_xstate_prctl - xstate permission operations 1865 * @option: A subfunction of arch_prctl() 1866 * @arg2: option argument 1867 * Return: 0 if successful; otherwise, an error code 1868 * 1869 * Option arguments: 1870 * 1871 * ARCH_GET_XCOMP_SUPP: Pointer to user space u64 to store the info 1872 * ARCH_GET_XCOMP_PERM: Pointer to user space u64 to store the info 1873 * ARCH_REQ_XCOMP_PERM: Facility number requested 1874 * 1875 * For facilities which require more than one XSTATE component, the request 1876 * must be the highest state component number related to that facility, 1877 * e.g. for AMX which requires XFEATURE_XTILE_CFG(17) and 1878 * XFEATURE_XTILE_DATA(18) this would be XFEATURE_XTILE_DATA(18). 1879 */ 1880 long fpu_xstate_prctl(int option, unsigned long arg2) 1881 { 1882 u64 __user *uptr = (u64 __user *)arg2; 1883 u64 permitted, supported; 1884 unsigned long idx = arg2; 1885 bool guest = false; 1886 1887 switch (option) { 1888 case ARCH_GET_XCOMP_SUPP: 1889 supported = fpu_user_cfg.max_features | fpu_user_cfg.legacy_features; 1890 return put_user(supported, uptr); 1891 1892 case ARCH_GET_XCOMP_PERM: 1893 /* 1894 * Lockless snapshot as it can also change right after the 1895 * dropping the lock. 1896 */ 1897 permitted = xstate_get_host_group_perm(); 1898 permitted &= XFEATURE_MASK_USER_SUPPORTED; 1899 return put_user(permitted, uptr); 1900 1901 case ARCH_GET_XCOMP_GUEST_PERM: 1902 permitted = xstate_get_guest_group_perm(); 1903 permitted &= XFEATURE_MASK_USER_SUPPORTED; 1904 return put_user(permitted, uptr); 1905 1906 case ARCH_REQ_XCOMP_GUEST_PERM: 1907 guest = true; 1908 fallthrough; 1909 1910 case ARCH_REQ_XCOMP_PERM: 1911 if (!IS_ENABLED(CONFIG_X86_64)) 1912 return -EOPNOTSUPP; 1913 1914 return xstate_request_perm(idx, guest); 1915 1916 default: 1917 return -EINVAL; 1918 } 1919 } 1920 1921 #ifdef CONFIG_PROC_PID_ARCH_STATUS 1922 /* 1923 * Report the amount of time elapsed in millisecond since last AVX512 1924 * use in the task. Report -1 if no AVX-512 usage. 1925 */ 1926 static void avx512_status(struct seq_file *m, struct task_struct *task) 1927 { 1928 unsigned long timestamp; 1929 long delta = -1; 1930 1931 /* AVX-512 usage is not tracked for kernel threads. Don't report anything. */ 1932 if (task->flags & (PF_KTHREAD | PF_USER_WORKER)) 1933 return; 1934 1935 timestamp = READ_ONCE(x86_task_fpu(task)->avx512_timestamp); 1936 1937 if (timestamp) { 1938 delta = (long)(jiffies - timestamp); 1939 /* 1940 * Cap to LONG_MAX if time difference > LONG_MAX 1941 */ 1942 if (delta < 0) 1943 delta = LONG_MAX; 1944 delta = jiffies_to_msecs(delta); 1945 } 1946 1947 seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta); 1948 seq_putc(m, '\n'); 1949 } 1950 1951 /* 1952 * Report architecture specific information 1953 */ 1954 int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, 1955 struct pid *pid, struct task_struct *task) 1956 { 1957 /* 1958 * Report AVX512 state if the processor and build option supported. 1959 */ 1960 if (cpu_feature_enabled(X86_FEATURE_AVX512F)) 1961 avx512_status(m, task); 1962 1963 return 0; 1964 } 1965 #endif /* CONFIG_PROC_PID_ARCH_STATUS */ 1966 1967 #ifdef CONFIG_COREDUMP 1968 static const char owner_name[] = "LINUX"; 1969 1970 /* 1971 * Dump type, size, offset and flag values for every xfeature that is present. 1972 */ 1973 static int dump_xsave_layout_desc(struct coredump_params *cprm) 1974 { 1975 int num_records = 0; 1976 int i; 1977 1978 for_each_extended_xfeature(i, fpu_user_cfg.max_features) { 1979 struct x86_xfeat_component xc = { 1980 .type = i, 1981 .size = xstate_sizes[i], 1982 .offset = xstate_offsets[i], 1983 /* reserved for future use */ 1984 .flags = 0, 1985 }; 1986 1987 if (!dump_emit(cprm, &xc, sizeof(xc))) 1988 return -1; 1989 1990 num_records++; 1991 } 1992 return num_records; 1993 } 1994 1995 static u32 get_xsave_desc_size(void) 1996 { 1997 u32 cnt = 0; 1998 u32 i; 1999 2000 for_each_extended_xfeature(i, fpu_user_cfg.max_features) 2001 cnt++; 2002 2003 return cnt * (sizeof(struct x86_xfeat_component)); 2004 } 2005 2006 int elf_coredump_extra_notes_write(struct coredump_params *cprm) 2007 { 2008 int num_records = 0; 2009 struct elf_note en; 2010 2011 if (!fpu_user_cfg.max_features) 2012 return 0; 2013 2014 en.n_namesz = sizeof(owner_name); 2015 en.n_descsz = get_xsave_desc_size(); 2016 en.n_type = NT_X86_XSAVE_LAYOUT; 2017 2018 if (!dump_emit(cprm, &en, sizeof(en))) 2019 return 1; 2020 if (!dump_emit(cprm, owner_name, en.n_namesz)) 2021 return 1; 2022 if (!dump_align(cprm, 4)) 2023 return 1; 2024 2025 num_records = dump_xsave_layout_desc(cprm); 2026 if (num_records < 0) 2027 return 1; 2028 2029 /* Total size should be equal to the number of records */ 2030 if ((sizeof(struct x86_xfeat_component) * num_records) != en.n_descsz) 2031 return 1; 2032 2033 return 0; 2034 } 2035 2036 int elf_coredump_extra_notes_size(void) 2037 { 2038 int size; 2039 2040 if (!fpu_user_cfg.max_features) 2041 return 0; 2042 2043 /* .note header */ 2044 size = sizeof(struct elf_note); 2045 /* Name plus alignment to 4 bytes */ 2046 size += roundup(sizeof(owner_name), 4); 2047 size += get_xsave_desc_size(); 2048 2049 return size; 2050 } 2051 #endif /* CONFIG_COREDUMP */ 2052