1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Pentium III FXSR, SSE support 6 * General FPU state handling cleanups 7 * Gareth Hughes <gareth@valinux.com>, May 2000 8 */ 9 #include <asm/fpu/api.h> 10 #include <asm/fpu/regset.h> 11 #include <asm/fpu/sched.h> 12 #include <asm/fpu/signal.h> 13 #include <asm/fpu/types.h> 14 #include <asm/msr.h> 15 #include <asm/traps.h> 16 #include <asm/irq_regs.h> 17 18 #include <uapi/asm/kvm.h> 19 20 #include <linux/hardirq.h> 21 #include <linux/pkeys.h> 22 #include <linux/vmalloc.h> 23 24 #include "context.h" 25 #include "internal.h" 26 #include "legacy.h" 27 #include "xstate.h" 28 29 #define CREATE_TRACE_POINTS 30 #include <asm/trace/fpu.h> 31 32 #ifdef CONFIG_X86_64 33 DEFINE_STATIC_KEY_FALSE(__fpu_state_size_dynamic); 34 DEFINE_PER_CPU(u64, xfd_state); 35 #endif 36 37 /* The FPU state configuration data for kernel and user space */ 38 struct fpu_state_config fpu_kernel_cfg __ro_after_init; 39 struct fpu_state_config fpu_user_cfg __ro_after_init; 40 struct vcpu_fpu_config guest_default_cfg __ro_after_init; 41 42 /* 43 * Represents the initial FPU state. It's mostly (but not completely) zeroes, 44 * depending on the FPU hardware format: 45 */ 46 struct fpstate init_fpstate __ro_after_init; 47 48 /* 49 * Track FPU initialization and kernel-mode usage. 'true' means the FPU is 50 * initialized and is not currently being used by the kernel: 51 */ 52 DEFINE_PER_CPU(bool, kernel_fpu_allowed); 53 54 /* 55 * Track which context is using the FPU on the CPU: 56 */ 57 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); 58 59 #ifdef CONFIG_X86_DEBUG_FPU 60 struct fpu *x86_task_fpu(struct task_struct *task) 61 { 62 if (WARN_ON_ONCE(task->flags & PF_KTHREAD)) 63 return NULL; 64 65 return (void *)task + sizeof(*task); 66 } 67 #endif 68 69 /* 70 * Can we use the FPU in kernel mode with the 71 * whole "kernel_fpu_begin/end()" sequence? 72 */ 73 bool irq_fpu_usable(void) 74 { 75 if (WARN_ON_ONCE(in_nmi())) 76 return false; 77 78 /* 79 * Return false in the following cases: 80 * 81 * - FPU is not yet initialized. This can happen only when the call is 82 * coming from CPU onlining, for example for microcode checksumming. 83 * - The kernel is already using the FPU, either because of explicit 84 * nesting (which should never be done), or because of implicit 85 * nesting when a hardirq interrupted a kernel-mode FPU section. 86 * 87 * The single boolean check below handles both cases: 88 */ 89 if (!this_cpu_read(kernel_fpu_allowed)) 90 return false; 91 92 /* 93 * When not in NMI or hard interrupt context, FPU can be used in: 94 * 95 * - Task context except from within fpregs_lock()'ed critical 96 * regions. 97 * 98 * - Soft interrupt processing context which cannot happen 99 * while in a fpregs_lock()'ed critical region. 100 */ 101 if (!in_hardirq()) 102 return true; 103 104 /* 105 * In hard interrupt context it's safe when soft interrupts 106 * are enabled, which means the interrupt did not hit in 107 * a fpregs_lock()'ed critical region. 108 */ 109 return !softirq_count(); 110 } 111 EXPORT_SYMBOL(irq_fpu_usable); 112 113 /* 114 * Track AVX512 state use because it is known to slow the max clock 115 * speed of the core. 116 */ 117 static void update_avx_timestamp(struct fpu *fpu) 118 { 119 120 #define AVX512_TRACKING_MASK (XFEATURE_MASK_ZMM_Hi256 | XFEATURE_MASK_Hi16_ZMM) 121 122 if (fpu->fpstate->regs.xsave.header.xfeatures & AVX512_TRACKING_MASK) 123 fpu->avx512_timestamp = jiffies; 124 } 125 126 /* 127 * Save the FPU register state in fpu->fpstate->regs. The register state is 128 * preserved. 129 * 130 * Must be called with fpregs_lock() held. 131 * 132 * The legacy FNSAVE instruction clears all FPU state unconditionally, so 133 * register state has to be reloaded. That might be a pointless exercise 134 * when the FPU is going to be used by another task right after that. But 135 * this only affects 20+ years old 32bit systems and avoids conditionals all 136 * over the place. 137 * 138 * FXSAVE and all XSAVE variants preserve the FPU register state. 139 */ 140 void save_fpregs_to_fpstate(struct fpu *fpu) 141 { 142 if (likely(use_xsave())) { 143 os_xsave(fpu->fpstate); 144 update_avx_timestamp(fpu); 145 return; 146 } 147 148 if (likely(use_fxsr())) { 149 fxsave(&fpu->fpstate->regs.fxsave); 150 return; 151 } 152 153 /* 154 * Legacy FPU register saving, FNSAVE always clears FPU registers, 155 * so we have to reload them from the memory state. 156 */ 157 asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->fpstate->regs.fsave)); 158 frstor(&fpu->fpstate->regs.fsave); 159 } 160 161 void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask) 162 { 163 /* 164 * AMD K7/K8 and later CPUs up to Zen don't save/restore 165 * FDP/FIP/FOP unless an exception is pending. Clear the x87 state 166 * here by setting it to fixed values. "m" is a random variable 167 * that should be in L1. 168 */ 169 if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) { 170 asm volatile( 171 "fnclex\n\t" 172 "emms\n\t" 173 "fildl %[addr]" /* set F?P to defined value */ 174 : : [addr] "m" (*fpstate)); 175 } 176 177 if (use_xsave()) { 178 /* 179 * Dynamically enabled features are enabled in XCR0, but 180 * usage requires also that the corresponding bits in XFD 181 * are cleared. If the bits are set then using a related 182 * instruction will raise #NM. This allows to do the 183 * allocation of the larger FPU buffer lazy from #NM or if 184 * the task has no permission to kill it which would happen 185 * via #UD if the feature is disabled in XCR0. 186 * 187 * XFD state is following the same life time rules as 188 * XSTATE and to restore state correctly XFD has to be 189 * updated before XRSTORS otherwise the component would 190 * stay in or go into init state even if the bits are set 191 * in fpstate::regs::xsave::xfeatures. 192 */ 193 xfd_update_state(fpstate); 194 195 /* 196 * Restoring state always needs to modify all features 197 * which are in @mask even if the current task cannot use 198 * extended features. 199 * 200 * So fpstate->xfeatures cannot be used here, because then 201 * a feature for which the task has no permission but was 202 * used by the previous task would not go into init state. 203 */ 204 mask = fpu_kernel_cfg.max_features & mask; 205 206 os_xrstor(fpstate, mask); 207 } else { 208 if (use_fxsr()) 209 fxrstor(&fpstate->regs.fxsave); 210 else 211 frstor(&fpstate->regs.fsave); 212 } 213 } 214 215 void fpu_reset_from_exception_fixup(void) 216 { 217 restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE); 218 } 219 220 #if IS_ENABLED(CONFIG_KVM) 221 static void __fpstate_reset(struct fpstate *fpstate); 222 223 static void fpu_lock_guest_permissions(void) 224 { 225 struct fpu_state_perm *fpuperm; 226 u64 perm; 227 228 if (!IS_ENABLED(CONFIG_X86_64)) 229 return; 230 231 spin_lock_irq(¤t->sighand->siglock); 232 fpuperm = &x86_task_fpu(current->group_leader)->guest_perm; 233 perm = fpuperm->__state_perm; 234 235 /* First fpstate allocation locks down permissions. */ 236 WRITE_ONCE(fpuperm->__state_perm, perm | FPU_GUEST_PERM_LOCKED); 237 238 spin_unlock_irq(¤t->sighand->siglock); 239 } 240 241 bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) 242 { 243 struct fpstate *fpstate; 244 unsigned int size; 245 246 size = guest_default_cfg.size + ALIGN(offsetof(struct fpstate, regs), 64); 247 248 fpstate = vzalloc(size); 249 if (!fpstate) 250 return false; 251 252 /* Initialize indicators to reflect properties of the fpstate */ 253 fpstate->is_valloc = true; 254 fpstate->is_guest = true; 255 256 __fpstate_reset(fpstate); 257 fpstate_init_user(fpstate); 258 259 gfpu->fpstate = fpstate; 260 gfpu->xfeatures = guest_default_cfg.features; 261 262 /* 263 * KVM sets the FP+SSE bits in the XSAVE header when copying FPU state 264 * to userspace, even when XSAVE is unsupported, so that restoring FPU 265 * state on a different CPU that does support XSAVE can cleanly load 266 * the incoming state using its natural XSAVE. In other words, KVM's 267 * uABI size may be larger than this host's default size. Conversely, 268 * the default size should never be larger than KVM's base uABI size; 269 * all features that can expand the uABI size must be opt-in. 270 */ 271 gfpu->uabi_size = sizeof(struct kvm_xsave); 272 if (WARN_ON_ONCE(fpu_user_cfg.default_size > gfpu->uabi_size)) 273 gfpu->uabi_size = fpu_user_cfg.default_size; 274 275 fpu_lock_guest_permissions(); 276 277 return true; 278 } 279 EXPORT_SYMBOL_GPL(fpu_alloc_guest_fpstate); 280 281 void fpu_free_guest_fpstate(struct fpu_guest *gfpu) 282 { 283 struct fpstate *fpstate = gfpu->fpstate; 284 285 if (!fpstate) 286 return; 287 288 if (WARN_ON_ONCE(!fpstate->is_valloc || !fpstate->is_guest || fpstate->in_use)) 289 return; 290 291 gfpu->fpstate = NULL; 292 vfree(fpstate); 293 } 294 EXPORT_SYMBOL_GPL(fpu_free_guest_fpstate); 295 296 /* 297 * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable 298 * @guest_fpu: Pointer to the guest FPU container 299 * @xfeatures: Features requested by guest CPUID 300 * 301 * Enable all dynamic xfeatures according to guest perm and requested CPUID. 302 * 303 * Return: 0 on success, error code otherwise 304 */ 305 int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures) 306 { 307 lockdep_assert_preemption_enabled(); 308 309 /* Nothing to do if all requested features are already enabled. */ 310 xfeatures &= ~guest_fpu->xfeatures; 311 if (!xfeatures) 312 return 0; 313 314 return __xfd_enable_feature(xfeatures, guest_fpu); 315 } 316 EXPORT_SYMBOL_GPL(fpu_enable_guest_xfd_features); 317 318 #ifdef CONFIG_X86_64 319 void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) 320 { 321 fpregs_lock(); 322 guest_fpu->fpstate->xfd = xfd; 323 if (guest_fpu->fpstate->in_use) 324 xfd_update_state(guest_fpu->fpstate); 325 fpregs_unlock(); 326 } 327 EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); 328 329 /** 330 * fpu_sync_guest_vmexit_xfd_state - Synchronize XFD MSR and software state 331 * 332 * Must be invoked from KVM after a VMEXIT before enabling interrupts when 333 * XFD write emulation is disabled. This is required because the guest can 334 * freely modify XFD and the state at VMEXIT is not guaranteed to be the 335 * same as the state on VMENTER. So software state has to be updated before 336 * any operation which depends on it can take place. 337 * 338 * Note: It can be invoked unconditionally even when write emulation is 339 * enabled for the price of a then pointless MSR read. 340 */ 341 void fpu_sync_guest_vmexit_xfd_state(void) 342 { 343 struct fpstate *fpstate = x86_task_fpu(current)->fpstate; 344 345 lockdep_assert_irqs_disabled(); 346 if (fpu_state_size_dynamic()) { 347 rdmsrq(MSR_IA32_XFD, fpstate->xfd); 348 __this_cpu_write(xfd_state, fpstate->xfd); 349 } 350 } 351 EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state); 352 #endif /* CONFIG_X86_64 */ 353 354 int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) 355 { 356 struct fpstate *guest_fps = guest_fpu->fpstate; 357 struct fpu *fpu = x86_task_fpu(current); 358 struct fpstate *cur_fps = fpu->fpstate; 359 360 fpregs_lock(); 361 if (!cur_fps->is_confidential && !test_thread_flag(TIF_NEED_FPU_LOAD)) 362 save_fpregs_to_fpstate(fpu); 363 364 /* Swap fpstate */ 365 if (enter_guest) { 366 fpu->__task_fpstate = cur_fps; 367 fpu->fpstate = guest_fps; 368 guest_fps->in_use = true; 369 } else { 370 guest_fps->in_use = false; 371 fpu->fpstate = fpu->__task_fpstate; 372 fpu->__task_fpstate = NULL; 373 } 374 375 cur_fps = fpu->fpstate; 376 377 if (!cur_fps->is_confidential) { 378 /* Includes XFD update */ 379 restore_fpregs_from_fpstate(cur_fps, XFEATURE_MASK_FPSTATE); 380 } else { 381 /* 382 * XSTATE is restored by firmware from encrypted 383 * memory. Make sure XFD state is correct while 384 * running with guest fpstate 385 */ 386 xfd_update_state(cur_fps); 387 } 388 389 fpregs_mark_activate(); 390 fpregs_unlock(); 391 return 0; 392 } 393 EXPORT_SYMBOL_GPL(fpu_swap_kvm_fpstate); 394 395 void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf, 396 unsigned int size, u64 xfeatures, u32 pkru) 397 { 398 struct fpstate *kstate = gfpu->fpstate; 399 union fpregs_state *ustate = buf; 400 struct membuf mb = { .p = buf, .left = size }; 401 402 if (cpu_feature_enabled(X86_FEATURE_XSAVE)) { 403 __copy_xstate_to_uabi_buf(mb, kstate, xfeatures, pkru, 404 XSTATE_COPY_XSAVE); 405 } else { 406 memcpy(&ustate->fxsave, &kstate->regs.fxsave, 407 sizeof(ustate->fxsave)); 408 /* Make it restorable on a XSAVE enabled host */ 409 ustate->xsave.header.xfeatures = XFEATURE_MASK_FPSSE; 410 } 411 } 412 EXPORT_SYMBOL_GPL(fpu_copy_guest_fpstate_to_uabi); 413 414 int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, 415 u64 xcr0, u32 *vpkru) 416 { 417 struct fpstate *kstate = gfpu->fpstate; 418 const union fpregs_state *ustate = buf; 419 420 if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) { 421 if (ustate->xsave.header.xfeatures & ~XFEATURE_MASK_FPSSE) 422 return -EINVAL; 423 if (ustate->fxsave.mxcsr & ~mxcsr_feature_mask) 424 return -EINVAL; 425 memcpy(&kstate->regs.fxsave, &ustate->fxsave, sizeof(ustate->fxsave)); 426 return 0; 427 } 428 429 if (ustate->xsave.header.xfeatures & ~xcr0) 430 return -EINVAL; 431 432 /* 433 * Nullify @vpkru to preserve its current value if PKRU's bit isn't set 434 * in the header. KVM's odd ABI is to leave PKRU untouched in this 435 * case (all other components are eventually re-initialized). 436 */ 437 if (!(ustate->xsave.header.xfeatures & XFEATURE_MASK_PKRU)) 438 vpkru = NULL; 439 440 return copy_uabi_from_kernel_to_xstate(kstate, ustate, vpkru); 441 } 442 EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate); 443 #endif /* CONFIG_KVM */ 444 445 void kernel_fpu_begin_mask(unsigned int kfpu_mask) 446 { 447 if (!irqs_disabled()) 448 fpregs_lock(); 449 450 WARN_ON_FPU(!irq_fpu_usable()); 451 452 /* Toggle kernel_fpu_allowed to false: */ 453 WARN_ON_FPU(!this_cpu_read(kernel_fpu_allowed)); 454 this_cpu_write(kernel_fpu_allowed, false); 455 456 if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) && 457 !test_thread_flag(TIF_NEED_FPU_LOAD)) { 458 set_thread_flag(TIF_NEED_FPU_LOAD); 459 save_fpregs_to_fpstate(x86_task_fpu(current)); 460 } 461 __cpu_invalidate_fpregs_state(); 462 463 /* Put sane initial values into the control registers. */ 464 if (likely(kfpu_mask & KFPU_MXCSR) && boot_cpu_has(X86_FEATURE_XMM)) 465 ldmxcsr(MXCSR_DEFAULT); 466 467 if (unlikely(kfpu_mask & KFPU_387) && boot_cpu_has(X86_FEATURE_FPU)) 468 asm volatile ("fninit"); 469 } 470 EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask); 471 472 void kernel_fpu_end(void) 473 { 474 /* Toggle kernel_fpu_allowed back to true: */ 475 WARN_ON_FPU(this_cpu_read(kernel_fpu_allowed)); 476 this_cpu_write(kernel_fpu_allowed, true); 477 478 if (!irqs_disabled()) 479 fpregs_unlock(); 480 } 481 EXPORT_SYMBOL_GPL(kernel_fpu_end); 482 483 /* 484 * Sync the FPU register state to current's memory register state when the 485 * current task owns the FPU. The hardware register state is preserved. 486 */ 487 void fpu_sync_fpstate(struct fpu *fpu) 488 { 489 WARN_ON_FPU(fpu != x86_task_fpu(current)); 490 491 fpregs_lock(); 492 trace_x86_fpu_before_save(fpu); 493 494 if (!test_thread_flag(TIF_NEED_FPU_LOAD)) 495 save_fpregs_to_fpstate(fpu); 496 497 trace_x86_fpu_after_save(fpu); 498 fpregs_unlock(); 499 } 500 501 static inline unsigned int init_fpstate_copy_size(void) 502 { 503 if (!use_xsave()) 504 return fpu_kernel_cfg.default_size; 505 506 /* XSAVE(S) just needs the legacy and the xstate header part */ 507 return sizeof(init_fpstate.regs.xsave); 508 } 509 510 static inline void fpstate_init_fxstate(struct fpstate *fpstate) 511 { 512 fpstate->regs.fxsave.cwd = 0x37f; 513 fpstate->regs.fxsave.mxcsr = MXCSR_DEFAULT; 514 } 515 516 /* 517 * Legacy x87 fpstate state init: 518 */ 519 static inline void fpstate_init_fstate(struct fpstate *fpstate) 520 { 521 fpstate->regs.fsave.cwd = 0xffff037fu; 522 fpstate->regs.fsave.swd = 0xffff0000u; 523 fpstate->regs.fsave.twd = 0xffffffffu; 524 fpstate->regs.fsave.fos = 0xffff0000u; 525 } 526 527 /* 528 * Used in two places: 529 * 1) Early boot to setup init_fpstate for non XSAVE systems 530 * 2) fpu_alloc_guest_fpstate() which is invoked from KVM 531 */ 532 void fpstate_init_user(struct fpstate *fpstate) 533 { 534 if (!cpu_feature_enabled(X86_FEATURE_FPU)) { 535 fpstate_init_soft(&fpstate->regs.soft); 536 return; 537 } 538 539 xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures); 540 541 if (cpu_feature_enabled(X86_FEATURE_FXSR)) 542 fpstate_init_fxstate(fpstate); 543 else 544 fpstate_init_fstate(fpstate); 545 } 546 547 static void __fpstate_reset(struct fpstate *fpstate) 548 { 549 /* 550 * Supervisor features (and thus sizes) may diverge between guest 551 * FPUs and host FPUs, as some supervisor features are supported 552 * for guests despite not being utilized by the host. User 553 * features and sizes are always identical, which allows for 554 * common guest and userspace ABI. 555 * 556 * For the host, set XFD to the kernel's desired initialization 557 * value. For guests, set XFD to its architectural RESET value. 558 */ 559 if (fpstate->is_guest) { 560 fpstate->size = guest_default_cfg.size; 561 fpstate->xfeatures = guest_default_cfg.features; 562 fpstate->xfd = 0; 563 } else { 564 fpstate->size = fpu_kernel_cfg.default_size; 565 fpstate->xfeatures = fpu_kernel_cfg.default_features; 566 fpstate->xfd = init_fpstate.xfd; 567 } 568 569 fpstate->user_size = fpu_user_cfg.default_size; 570 fpstate->user_xfeatures = fpu_user_cfg.default_features; 571 } 572 573 void fpstate_reset(struct fpu *fpu) 574 { 575 /* Set the fpstate pointer to the default fpstate */ 576 fpu->fpstate = &fpu->__fpstate; 577 __fpstate_reset(fpu->fpstate); 578 579 /* Initialize the permission related info in fpu */ 580 fpu->perm.__state_perm = fpu_kernel_cfg.default_features; 581 fpu->perm.__state_size = fpu_kernel_cfg.default_size; 582 fpu->perm.__user_state_size = fpu_user_cfg.default_size; 583 584 fpu->guest_perm.__state_perm = guest_default_cfg.features; 585 fpu->guest_perm.__state_size = guest_default_cfg.size; 586 /* 587 * User features and sizes are always identical between host and 588 * guest FPUs, which allows for common guest and userspace ABI. 589 */ 590 fpu->guest_perm.__user_state_size = fpu_user_cfg.default_size; 591 } 592 593 static inline void fpu_inherit_perms(struct fpu *dst_fpu) 594 { 595 if (fpu_state_size_dynamic()) { 596 struct fpu *src_fpu = x86_task_fpu(current->group_leader); 597 598 spin_lock_irq(¤t->sighand->siglock); 599 /* Fork also inherits the permissions of the parent */ 600 dst_fpu->perm = src_fpu->perm; 601 dst_fpu->guest_perm = src_fpu->guest_perm; 602 spin_unlock_irq(¤t->sighand->siglock); 603 } 604 } 605 606 /* A passed ssp of zero will not cause any update */ 607 static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) 608 { 609 #ifdef CONFIG_X86_USER_SHADOW_STACK 610 struct cet_user_state *xstate; 611 612 /* If ssp update is not needed. */ 613 if (!ssp) 614 return 0; 615 616 xstate = get_xsave_addr(&x86_task_fpu(dst)->fpstate->regs.xsave, 617 XFEATURE_CET_USER); 618 619 /* 620 * If there is a non-zero ssp, then 'dst' must be configured with a shadow 621 * stack and the fpu state should be up to date since it was just copied 622 * from the parent in fpu_clone(). So there must be a valid non-init CET 623 * state location in the buffer. 624 */ 625 if (WARN_ON_ONCE(!xstate)) 626 return 1; 627 628 xstate->user_ssp = (u64)ssp; 629 #endif 630 return 0; 631 } 632 633 /* Clone current's FPU state on fork */ 634 int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, 635 unsigned long ssp) 636 { 637 /* 638 * We allocate the new FPU structure right after the end of the task struct. 639 * task allocation size already took this into account. 640 * 641 * This is safe because task_struct size is a multiple of cacheline size, 642 * thus x86_task_fpu() will always be cacheline aligned as well. 643 */ 644 struct fpu *dst_fpu = (void *)dst + sizeof(*dst); 645 646 BUILD_BUG_ON(sizeof(*dst) % SMP_CACHE_BYTES != 0); 647 648 /* The new task's FPU state cannot be valid in the hardware. */ 649 dst_fpu->last_cpu = -1; 650 651 fpstate_reset(dst_fpu); 652 653 if (!cpu_feature_enabled(X86_FEATURE_FPU)) 654 return 0; 655 656 /* 657 * Enforce reload for user space tasks and prevent kernel threads 658 * from trying to save the FPU registers on context switch. 659 */ 660 set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD); 661 662 /* 663 * No FPU state inheritance for kernel threads and IO 664 * worker threads. 665 */ 666 if (minimal) { 667 /* Clear out the minimal state */ 668 memcpy(&dst_fpu->fpstate->regs, &init_fpstate.regs, 669 init_fpstate_copy_size()); 670 return 0; 671 } 672 673 /* 674 * If a new feature is added, ensure all dynamic features are 675 * caller-saved from here! 676 */ 677 BUILD_BUG_ON(XFEATURE_MASK_USER_DYNAMIC != XFEATURE_MASK_XTILE_DATA); 678 679 /* 680 * Save the default portion of the current FPU state into the 681 * clone. Assume all dynamic features to be defined as caller- 682 * saved, which enables skipping both the expansion of fpstate 683 * and the copying of any dynamic state. 684 * 685 * Do not use memcpy() when TIF_NEED_FPU_LOAD is set because 686 * copying is not valid when current uses non-default states. 687 */ 688 fpregs_lock(); 689 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 690 fpregs_restore_userregs(); 691 save_fpregs_to_fpstate(dst_fpu); 692 fpregs_unlock(); 693 if (!(clone_flags & CLONE_THREAD)) 694 fpu_inherit_perms(dst_fpu); 695 696 /* 697 * Children never inherit PASID state. 698 * Force it to have its init value: 699 */ 700 if (use_xsave()) 701 dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID; 702 703 /* 704 * Update shadow stack pointer, in case it changed during clone. 705 */ 706 if (update_fpu_shstk(dst, ssp)) 707 return 1; 708 709 trace_x86_fpu_copy_dst(dst_fpu); 710 711 return 0; 712 } 713 714 /* 715 * While struct fpu is no longer part of struct thread_struct, it is still 716 * allocated after struct task_struct in the "task_struct" kmem cache. But 717 * since FPU is expected to be part of struct thread_struct, we have to 718 * adjust for it here. 719 */ 720 void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size) 721 { 722 /* The allocation follows struct task_struct. */ 723 *offset = sizeof(struct task_struct) - offsetof(struct task_struct, thread); 724 *offset += offsetof(struct fpu, __fpstate.regs); 725 *size = fpu_kernel_cfg.default_size; 726 } 727 728 /* 729 * Drops current FPU state: deactivates the fpregs and 730 * the fpstate. NOTE: it still leaves previous contents 731 * in the fpregs in the eager-FPU case. 732 * 733 * This function can be used in cases where we know that 734 * a state-restore is coming: either an explicit one, 735 * or a reschedule. 736 */ 737 void fpu__drop(struct task_struct *tsk) 738 { 739 struct fpu *fpu; 740 741 if (test_tsk_thread_flag(tsk, TIF_NEED_FPU_LOAD)) 742 return; 743 744 fpu = x86_task_fpu(tsk); 745 746 preempt_disable(); 747 748 if (fpu == x86_task_fpu(current)) { 749 /* Ignore delayed exceptions from user space */ 750 asm volatile("1: fwait\n" 751 "2:\n" 752 _ASM_EXTABLE(1b, 2b)); 753 fpregs_deactivate(fpu); 754 } 755 756 trace_x86_fpu_dropped(fpu); 757 758 preempt_enable(); 759 } 760 761 /* 762 * Clear FPU registers by setting them up from the init fpstate. 763 * Caller must do fpregs_[un]lock() around it. 764 */ 765 static inline void restore_fpregs_from_init_fpstate(u64 features_mask) 766 { 767 if (use_xsave()) 768 os_xrstor(&init_fpstate, features_mask); 769 else if (use_fxsr()) 770 fxrstor(&init_fpstate.regs.fxsave); 771 else 772 frstor(&init_fpstate.regs.fsave); 773 774 pkru_write_default(); 775 } 776 777 /* 778 * Reset current->fpu memory state to the init values. 779 */ 780 static void fpu_reset_fpstate_regs(void) 781 { 782 struct fpu *fpu = x86_task_fpu(current); 783 784 fpregs_lock(); 785 __fpu_invalidate_fpregs_state(fpu); 786 /* 787 * This does not change the actual hardware registers. It just 788 * resets the memory image and sets TIF_NEED_FPU_LOAD so a 789 * subsequent return to usermode will reload the registers from the 790 * task's memory image. 791 * 792 * Do not use fpstate_init() here. Just copy init_fpstate which has 793 * the correct content already except for PKRU. 794 * 795 * PKRU handling does not rely on the xstate when restoring for 796 * user space as PKRU is eagerly written in switch_to() and 797 * flush_thread(). 798 */ 799 memcpy(&fpu->fpstate->regs, &init_fpstate.regs, init_fpstate_copy_size()); 800 set_thread_flag(TIF_NEED_FPU_LOAD); 801 fpregs_unlock(); 802 } 803 804 /* 805 * Reset current's user FPU states to the init states. current's 806 * supervisor states, if any, are not modified by this function. The 807 * caller guarantees that the XSTATE header in memory is intact. 808 */ 809 void fpu__clear_user_states(struct fpu *fpu) 810 { 811 WARN_ON_FPU(fpu != x86_task_fpu(current)); 812 813 fpregs_lock(); 814 if (!cpu_feature_enabled(X86_FEATURE_FPU)) { 815 fpu_reset_fpstate_regs(); 816 fpregs_unlock(); 817 return; 818 } 819 820 /* 821 * Ensure that current's supervisor states are loaded into their 822 * corresponding registers. 823 */ 824 if (xfeatures_mask_supervisor() && 825 !fpregs_state_valid(fpu, smp_processor_id())) 826 os_xrstor_supervisor(fpu->fpstate); 827 828 /* Reset user states in registers. */ 829 restore_fpregs_from_init_fpstate(XFEATURE_MASK_USER_RESTORE); 830 831 /* 832 * Now all FPU registers have their desired values. Inform the FPU 833 * state machine that current's FPU registers are in the hardware 834 * registers. The memory image does not need to be updated because 835 * any operation relying on it has to save the registers first when 836 * current's FPU is marked active. 837 */ 838 fpregs_mark_activate(); 839 fpregs_unlock(); 840 } 841 842 void fpu_flush_thread(void) 843 { 844 fpstate_reset(x86_task_fpu(current)); 845 fpu_reset_fpstate_regs(); 846 } 847 /* 848 * Load FPU context before returning to userspace. 849 */ 850 void switch_fpu_return(void) 851 { 852 if (!static_cpu_has(X86_FEATURE_FPU)) 853 return; 854 855 fpregs_restore_userregs(); 856 } 857 EXPORT_SYMBOL_GPL(switch_fpu_return); 858 859 void fpregs_lock_and_load(void) 860 { 861 /* 862 * fpregs_lock() only disables preemption (mostly). So modifying state 863 * in an interrupt could screw up some in progress fpregs operation. 864 * Warn about it. 865 */ 866 WARN_ON_ONCE(!irq_fpu_usable()); 867 WARN_ON_ONCE(current->flags & PF_KTHREAD); 868 869 fpregs_lock(); 870 871 fpregs_assert_state_consistent(); 872 873 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 874 fpregs_restore_userregs(); 875 } 876 877 #ifdef CONFIG_X86_DEBUG_FPU 878 /* 879 * If current FPU state according to its tracking (loaded FPU context on this 880 * CPU) is not valid then we must have TIF_NEED_FPU_LOAD set so the context is 881 * loaded on return to userland. 882 */ 883 void fpregs_assert_state_consistent(void) 884 { 885 struct fpu *fpu = x86_task_fpu(current); 886 887 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 888 return; 889 890 WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id())); 891 } 892 EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent); 893 #endif 894 895 void fpregs_mark_activate(void) 896 { 897 struct fpu *fpu = x86_task_fpu(current); 898 899 fpregs_activate(fpu); 900 fpu->last_cpu = smp_processor_id(); 901 clear_thread_flag(TIF_NEED_FPU_LOAD); 902 } 903 904 /* 905 * x87 math exception handling: 906 */ 907 908 int fpu__exception_code(struct fpu *fpu, int trap_nr) 909 { 910 int err; 911 912 if (trap_nr == X86_TRAP_MF) { 913 unsigned short cwd, swd; 914 /* 915 * (~cwd & swd) will mask out exceptions that are not set to unmasked 916 * status. 0x3f is the exception bits in these regs, 0x200 is the 917 * C1 reg you need in case of a stack fault, 0x040 is the stack 918 * fault bit. We should only be taking one exception at a time, 919 * so if this combination doesn't produce any single exception, 920 * then we have a bad program that isn't synchronizing its FPU usage 921 * and it will suffer the consequences since we won't be able to 922 * fully reproduce the context of the exception. 923 */ 924 if (boot_cpu_has(X86_FEATURE_FXSR)) { 925 cwd = fpu->fpstate->regs.fxsave.cwd; 926 swd = fpu->fpstate->regs.fxsave.swd; 927 } else { 928 cwd = (unsigned short)fpu->fpstate->regs.fsave.cwd; 929 swd = (unsigned short)fpu->fpstate->regs.fsave.swd; 930 } 931 932 err = swd & ~cwd; 933 } else { 934 /* 935 * The SIMD FPU exceptions are handled a little differently, as there 936 * is only a single status/control register. Thus, to determine which 937 * unmasked exception was caught we must mask the exception mask bits 938 * at 0x1f80, and then use these to mask the exception bits at 0x3f. 939 */ 940 unsigned short mxcsr = MXCSR_DEFAULT; 941 942 if (boot_cpu_has(X86_FEATURE_XMM)) 943 mxcsr = fpu->fpstate->regs.fxsave.mxcsr; 944 945 err = ~(mxcsr >> 7) & mxcsr; 946 } 947 948 if (err & 0x001) { /* Invalid op */ 949 /* 950 * swd & 0x240 == 0x040: Stack Underflow 951 * swd & 0x240 == 0x240: Stack Overflow 952 * User must clear the SF bit (0x40) if set 953 */ 954 return FPE_FLTINV; 955 } else if (err & 0x004) { /* Divide by Zero */ 956 return FPE_FLTDIV; 957 } else if (err & 0x008) { /* Overflow */ 958 return FPE_FLTOVF; 959 } else if (err & 0x012) { /* Denormal, Underflow */ 960 return FPE_FLTUND; 961 } else if (err & 0x020) { /* Precision */ 962 return FPE_FLTRES; 963 } 964 965 /* 966 * If we're using IRQ 13, or supposedly even some trap 967 * X86_TRAP_MF implementations, it's possible 968 * we get a spurious trap, which is not an error. 969 */ 970 return 0; 971 } 972 973 /* 974 * Initialize register state that may prevent from entering low-power idle. 975 * This function will be invoked from the cpuidle driver only when needed. 976 */ 977 noinstr void fpu_idle_fpregs(void) 978 { 979 /* Note: AMX_TILE being enabled implies XGETBV1 support */ 980 if (cpu_feature_enabled(X86_FEATURE_AMX_TILE) && 981 (xfeatures_in_use() & XFEATURE_MASK_XTILE)) { 982 tile_release(); 983 __this_cpu_write(fpu_fpregs_owner_ctx, NULL); 984 } 985 } 986