1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/errno.h> 5 #include <linux/kernel.h> 6 #include <linux/mm.h> 7 #include <linux/smp.h> 8 #include <linux/cpu.h> 9 #include <linux/prctl.h> 10 #include <linux/slab.h> 11 #include <linux/sched.h> 12 #include <linux/sched/idle.h> 13 #include <linux/sched/debug.h> 14 #include <linux/sched/task.h> 15 #include <linux/sched/task_stack.h> 16 #include <linux/init.h> 17 #include <linux/export.h> 18 #include <linux/pm.h> 19 #include <linux/tick.h> 20 #include <linux/random.h> 21 #include <linux/user-return-notifier.h> 22 #include <linux/dmi.h> 23 #include <linux/utsname.h> 24 #include <linux/stackprotector.h> 25 #include <linux/cpuidle.h> 26 #include <linux/acpi.h> 27 #include <linux/elf-randomize.h> 28 #include <linux/static_call.h> 29 #include <trace/events/power.h> 30 #include <linux/hw_breakpoint.h> 31 #include <linux/entry-common.h> 32 #include <asm/cpu.h> 33 #include <asm/cpuid/api.h> 34 #include <asm/apic.h> 35 #include <linux/uaccess.h> 36 #include <asm/mwait.h> 37 #include <asm/fpu/api.h> 38 #include <asm/fpu/sched.h> 39 #include <asm/fpu/xstate.h> 40 #include <asm/debugreg.h> 41 #include <asm/nmi.h> 42 #include <asm/tlbflush.h> 43 #include <asm/mce.h> 44 #include <asm/vm86.h> 45 #include <asm/switch_to.h> 46 #include <asm/desc.h> 47 #include <asm/prctl.h> 48 #include <asm/spec-ctrl.h> 49 #include <asm/io_bitmap.h> 50 #include <asm/proto.h> 51 #include <asm/frame.h> 52 #include <asm/unwind.h> 53 #include <asm/tdx.h> 54 #include <asm/mmu_context.h> 55 #include <asm/msr.h> 56 #include <asm/shstk.h> 57 58 #include "process.h" 59 60 /* 61 * per-CPU TSS segments. Threads are completely 'soft' on Linux, 62 * no more per-task TSS's. The TSS size is kept cacheline-aligned 63 * so they are allowed to end up in the .data..cacheline_aligned 64 * section. Since TSS's are completely CPU-local, we want them 65 * on exact cacheline boundaries, to eliminate cacheline ping-pong. 66 */ 67 __visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = { 68 .x86_tss = { 69 /* 70 * .sp0 is only used when entering ring 0 from a lower 71 * privilege level. Since the init task never runs anything 72 * but ring 0 code, there is no need for a valid value here. 73 * Poison it. 74 */ 75 .sp0 = (1UL << (BITS_PER_LONG-1)) + 1, 76 77 #ifdef CONFIG_X86_32 78 .sp1 = TOP_OF_INIT_STACK, 79 80 .ss0 = __KERNEL_DS, 81 .ss1 = __KERNEL_CS, 82 #endif 83 .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, 84 }, 85 }; 86 EXPORT_PER_CPU_SYMBOL(cpu_tss_rw); 87 88 DEFINE_PER_CPU(bool, __tss_limit_invalid); 89 EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid); 90 91 /* 92 * The cache may be in an incoherent state and needs flushing during kexec. 93 * E.g., on SME/TDX platforms, dirty cacheline aliases with and without 94 * encryption bit(s) can coexist and the cache needs to be flushed before 95 * booting to the new kernel to avoid the silent memory corruption due to 96 * dirty cachelines with different encryption property being written back 97 * to the memory. 98 */ 99 DEFINE_PER_CPU(bool, cache_state_incoherent); 100 101 /* 102 * this gets called so that we can store lazy state into memory and copy the 103 * current task into the new thread. 104 */ 105 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) 106 { 107 /* fpu_clone() will initialize the "dst_fpu" memory */ 108 memcpy_and_pad(dst, arch_task_struct_size, src, sizeof(*dst), 0); 109 110 #ifdef CONFIG_VM86 111 dst->thread.vm86 = NULL; 112 #endif 113 114 return 0; 115 } 116 117 #ifdef CONFIG_X86_64 118 void arch_release_task_struct(struct task_struct *tsk) 119 { 120 if (fpu_state_size_dynamic() && !(tsk->flags & (PF_KTHREAD | PF_USER_WORKER))) 121 fpstate_free(x86_task_fpu(tsk)); 122 } 123 #endif 124 125 /* 126 * Free thread data structures etc.. 127 */ 128 void exit_thread(struct task_struct *tsk) 129 { 130 struct thread_struct *t = &tsk->thread; 131 132 if (test_thread_flag(TIF_IO_BITMAP)) 133 io_bitmap_exit(tsk); 134 135 free_vm86(t); 136 137 shstk_free(tsk); 138 fpu__drop(tsk); 139 } 140 141 static int set_new_tls(struct task_struct *p, unsigned long tls) 142 { 143 struct user_desc __user *utls = (struct user_desc __user *)tls; 144 145 if (in_ia32_syscall()) 146 return do_set_thread_area(p, -1, utls, 0); 147 else 148 return do_set_thread_area_64(p, ARCH_SET_FS, tls); 149 } 150 151 __visible void ret_from_fork(struct task_struct *prev, struct pt_regs *regs, 152 int (*fn)(void *), void *fn_arg) 153 { 154 schedule_tail(prev); 155 156 /* Is this a kernel thread? */ 157 if (unlikely(fn)) { 158 fn(fn_arg); 159 /* 160 * A kernel thread is allowed to return here after successfully 161 * calling kernel_execve(). Exit to userspace to complete the 162 * execve() syscall. 163 */ 164 regs->ax = 0; 165 } 166 167 syscall_exit_to_user_mode(regs); 168 } 169 170 int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) 171 { 172 u64 clone_flags = args->flags; 173 unsigned long sp = args->stack; 174 unsigned long tls = args->tls; 175 struct inactive_task_frame *frame; 176 struct fork_frame *fork_frame; 177 struct pt_regs *childregs; 178 unsigned long new_ssp; 179 int ret = 0; 180 181 childregs = task_pt_regs(p); 182 fork_frame = container_of(childregs, struct fork_frame, regs); 183 frame = &fork_frame->frame; 184 185 frame->bp = encode_frame_pointer(childregs); 186 frame->ret_addr = (unsigned long) ret_from_fork_asm; 187 p->thread.sp = (unsigned long) fork_frame; 188 p->thread.io_bitmap = NULL; 189 clear_tsk_thread_flag(p, TIF_IO_BITMAP); 190 p->thread.iopl_warn = 0; 191 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); 192 193 #ifdef CONFIG_X86_64 194 current_save_fsgs(); 195 p->thread.fsindex = current->thread.fsindex; 196 p->thread.fsbase = current->thread.fsbase; 197 p->thread.gsindex = current->thread.gsindex; 198 p->thread.gsbase = current->thread.gsbase; 199 200 savesegment(es, p->thread.es); 201 savesegment(ds, p->thread.ds); 202 203 if (p->mm && (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM) 204 set_bit(MM_CONTEXT_LOCK_LAM, &p->mm->context.flags); 205 #else 206 p->thread.sp0 = (unsigned long) (childregs + 1); 207 savesegment(gs, p->thread.gs); 208 /* 209 * Clear all status flags including IF and set fixed bit. 64bit 210 * does not have this initialization as the frame does not contain 211 * flags. The flags consistency (especially vs. AC) is there 212 * ensured via objtool, which lacks 32bit support. 213 */ 214 frame->flags = X86_EFLAGS_FIXED; 215 #endif 216 217 /* 218 * Allocate a new shadow stack for thread if needed. If shadow stack, 219 * is disabled, new_ssp will remain 0, and fpu_clone() will know not to 220 * update it. 221 */ 222 new_ssp = shstk_alloc_thread_stack(p, clone_flags, args->stack_size); 223 if (IS_ERR_VALUE(new_ssp)) 224 return PTR_ERR((void *)new_ssp); 225 226 fpu_clone(p, clone_flags, args->fn, new_ssp); 227 228 /* Kernel thread ? */ 229 if (unlikely(p->flags & PF_KTHREAD)) { 230 p->thread.pkru = pkru_get_init_value(); 231 memset(childregs, 0, sizeof(struct pt_regs)); 232 kthread_frame_init(frame, args->fn, args->fn_arg); 233 return 0; 234 } 235 236 /* 237 * Clone current's PKRU value from hardware. tsk->thread.pkru 238 * is only valid when scheduled out. 239 */ 240 p->thread.pkru = read_pkru(); 241 242 frame->bx = 0; 243 *childregs = *current_pt_regs(); 244 childregs->ax = 0; 245 if (sp) 246 childregs->sp = sp; 247 248 if (unlikely(args->fn)) { 249 /* 250 * A user space thread, but it doesn't return to 251 * ret_after_fork(). 252 * 253 * In order to indicate that to tools like gdb, 254 * we reset the stack and instruction pointers. 255 * 256 * It does the same kernel frame setup to return to a kernel 257 * function that a kernel thread does. 258 */ 259 childregs->sp = 0; 260 childregs->ip = 0; 261 kthread_frame_init(frame, args->fn, args->fn_arg); 262 return 0; 263 } 264 265 /* Set a new TLS for the child thread? */ 266 if (clone_flags & CLONE_SETTLS) 267 ret = set_new_tls(p, tls); 268 269 if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP))) 270 io_bitmap_share(p); 271 272 return ret; 273 } 274 275 static void pkru_flush_thread(void) 276 { 277 /* 278 * If PKRU is enabled the default PKRU value has to be loaded into 279 * the hardware right here (similar to context switch). 280 */ 281 pkru_write_default(); 282 } 283 284 void flush_thread(void) 285 { 286 struct task_struct *tsk = current; 287 288 flush_ptrace_hw_breakpoint(tsk); 289 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 290 291 fpu_flush_thread(); 292 pkru_flush_thread(); 293 } 294 295 void disable_TSC(void) 296 { 297 preempt_disable(); 298 if (!test_and_set_thread_flag(TIF_NOTSC)) 299 /* 300 * Must flip the CPU state synchronously with 301 * TIF_NOTSC in the current running context. 302 */ 303 cr4_set_bits(X86_CR4_TSD); 304 preempt_enable(); 305 } 306 307 static void enable_TSC(void) 308 { 309 preempt_disable(); 310 if (test_and_clear_thread_flag(TIF_NOTSC)) 311 /* 312 * Must flip the CPU state synchronously with 313 * TIF_NOTSC in the current running context. 314 */ 315 cr4_clear_bits(X86_CR4_TSD); 316 preempt_enable(); 317 } 318 319 int get_tsc_mode(unsigned long adr) 320 { 321 unsigned int val; 322 323 if (test_thread_flag(TIF_NOTSC)) 324 val = PR_TSC_SIGSEGV; 325 else 326 val = PR_TSC_ENABLE; 327 328 return put_user(val, (unsigned int __user *)adr); 329 } 330 331 int set_tsc_mode(unsigned int val) 332 { 333 if (val == PR_TSC_SIGSEGV) 334 disable_TSC(); 335 else if (val == PR_TSC_ENABLE) 336 enable_TSC(); 337 else 338 return -EINVAL; 339 340 return 0; 341 } 342 343 DEFINE_PER_CPU(u64, msr_misc_features_shadow); 344 345 static void set_cpuid_faulting(bool on) 346 { 347 348 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 349 u64 msrval; 350 351 msrval = this_cpu_read(msr_misc_features_shadow); 352 msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT; 353 msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT); 354 this_cpu_write(msr_misc_features_shadow, msrval); 355 wrmsrq(MSR_MISC_FEATURES_ENABLES, msrval); 356 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { 357 if (on) 358 msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_CPUID_USER_DIS_BIT); 359 else 360 msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_CPUID_USER_DIS_BIT); 361 } 362 } 363 364 static void disable_cpuid(void) 365 { 366 preempt_disable(); 367 if (!test_and_set_thread_flag(TIF_NOCPUID)) { 368 /* 369 * Must flip the CPU state synchronously with 370 * TIF_NOCPUID in the current running context. 371 */ 372 set_cpuid_faulting(true); 373 } 374 preempt_enable(); 375 } 376 377 static void enable_cpuid(void) 378 { 379 preempt_disable(); 380 if (test_and_clear_thread_flag(TIF_NOCPUID)) { 381 /* 382 * Must flip the CPU state synchronously with 383 * TIF_NOCPUID in the current running context. 384 */ 385 set_cpuid_faulting(false); 386 } 387 preempt_enable(); 388 } 389 390 static int get_cpuid_mode(void) 391 { 392 return !test_thread_flag(TIF_NOCPUID); 393 } 394 395 static int set_cpuid_mode(unsigned long cpuid_enabled) 396 { 397 if (!boot_cpu_has(X86_FEATURE_CPUID_FAULT)) 398 return -ENODEV; 399 400 if (cpuid_enabled) 401 enable_cpuid(); 402 else 403 disable_cpuid(); 404 405 return 0; 406 } 407 408 /* 409 * Called immediately after a successful exec. 410 */ 411 void arch_setup_new_exec(void) 412 { 413 /* If cpuid was previously disabled for this task, re-enable it. */ 414 if (test_thread_flag(TIF_NOCPUID)) 415 enable_cpuid(); 416 417 /* 418 * Don't inherit TIF_SSBD across exec boundary when 419 * PR_SPEC_DISABLE_NOEXEC is used. 420 */ 421 if (test_thread_flag(TIF_SSBD) && 422 task_spec_ssb_noexec(current)) { 423 clear_thread_flag(TIF_SSBD); 424 task_clear_spec_ssb_disable(current); 425 task_clear_spec_ssb_noexec(current); 426 speculation_ctrl_update(read_thread_flags()); 427 } 428 429 mm_reset_untag_mask(current->mm); 430 } 431 432 #ifdef CONFIG_X86_IOPL_IOPERM 433 static inline void switch_to_bitmap(unsigned long tifp) 434 { 435 /* 436 * Invalidate I/O bitmap if the previous task used it. This prevents 437 * any possible leakage of an active I/O bitmap. 438 * 439 * If the next task has an I/O bitmap it will handle it on exit to 440 * user mode. 441 */ 442 if (tifp & _TIF_IO_BITMAP) 443 tss_invalidate_io_bitmap(); 444 } 445 446 static void tss_copy_io_bitmap(struct tss_struct *tss, struct io_bitmap *iobm) 447 { 448 /* 449 * Copy at least the byte range of the incoming tasks bitmap which 450 * covers the permitted I/O ports. 451 * 452 * If the previous task which used an I/O bitmap had more bits 453 * permitted, then the copy needs to cover those as well so they 454 * get turned off. 455 */ 456 memcpy(tss->io_bitmap.bitmap, iobm->bitmap, 457 max(tss->io_bitmap.prev_max, iobm->max)); 458 459 /* 460 * Store the new max and the sequence number of this bitmap 461 * and a pointer to the bitmap itself. 462 */ 463 tss->io_bitmap.prev_max = iobm->max; 464 tss->io_bitmap.prev_sequence = iobm->sequence; 465 } 466 467 /** 468 * native_tss_update_io_bitmap - Update I/O bitmap before exiting to user mode 469 */ 470 void native_tss_update_io_bitmap(void) 471 { 472 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); 473 struct thread_struct *t = ¤t->thread; 474 u16 *base = &tss->x86_tss.io_bitmap_base; 475 476 if (!test_thread_flag(TIF_IO_BITMAP)) { 477 native_tss_invalidate_io_bitmap(); 478 return; 479 } 480 481 if (IS_ENABLED(CONFIG_X86_IOPL_IOPERM) && t->iopl_emul == 3) { 482 *base = IO_BITMAP_OFFSET_VALID_ALL; 483 } else { 484 struct io_bitmap *iobm = t->io_bitmap; 485 486 if (WARN_ON_ONCE(!iobm)) { 487 clear_thread_flag(TIF_IO_BITMAP); 488 native_tss_invalidate_io_bitmap(); 489 return; 490 } 491 492 /* 493 * Only copy bitmap data when the sequence number differs. The 494 * update time is accounted to the incoming task. 495 */ 496 if (tss->io_bitmap.prev_sequence != iobm->sequence) 497 tss_copy_io_bitmap(tss, iobm); 498 499 /* Enable the bitmap */ 500 *base = IO_BITMAP_OFFSET_VALID_MAP; 501 } 502 503 /* 504 * Make sure that the TSS limit is covering the IO bitmap. It might have 505 * been cut down by a VMEXIT to 0x67 which would cause a subsequent I/O 506 * access from user space to trigger a #GP because the bitmap is outside 507 * the TSS limit. 508 */ 509 refresh_tss_limit(); 510 } 511 #else /* CONFIG_X86_IOPL_IOPERM */ 512 static inline void switch_to_bitmap(unsigned long tifp) { } 513 #endif 514 515 #ifdef CONFIG_SMP 516 517 struct ssb_state { 518 struct ssb_state *shared_state; 519 raw_spinlock_t lock; 520 unsigned int disable_state; 521 unsigned long local_state; 522 }; 523 524 #define LSTATE_SSB 0 525 526 static DEFINE_PER_CPU(struct ssb_state, ssb_state); 527 528 void speculative_store_bypass_ht_init(void) 529 { 530 struct ssb_state *st = this_cpu_ptr(&ssb_state); 531 unsigned int this_cpu = smp_processor_id(); 532 unsigned int cpu; 533 534 st->local_state = 0; 535 536 /* 537 * Shared state setup happens once on the first bringup 538 * of the CPU. It's not destroyed on CPU hotunplug. 539 */ 540 if (st->shared_state) 541 return; 542 543 raw_spin_lock_init(&st->lock); 544 545 /* 546 * Go over HT siblings and check whether one of them has set up the 547 * shared state pointer already. 548 */ 549 for_each_cpu(cpu, topology_sibling_cpumask(this_cpu)) { 550 if (cpu == this_cpu) 551 continue; 552 553 if (!per_cpu(ssb_state, cpu).shared_state) 554 continue; 555 556 /* Link it to the state of the sibling: */ 557 st->shared_state = per_cpu(ssb_state, cpu).shared_state; 558 return; 559 } 560 561 /* 562 * First HT sibling to come up on the core. Link shared state of 563 * the first HT sibling to itself. The siblings on the same core 564 * which come up later will see the shared state pointer and link 565 * themselves to the state of this CPU. 566 */ 567 st->shared_state = st; 568 } 569 570 /* 571 * Logic is: First HT sibling enables SSBD for both siblings in the core 572 * and last sibling to disable it, disables it for the whole core. This how 573 * MSR_SPEC_CTRL works in "hardware": 574 * 575 * CORE_SPEC_CTRL = THREAD0_SPEC_CTRL | THREAD1_SPEC_CTRL 576 */ 577 static __always_inline void amd_set_core_ssb_state(unsigned long tifn) 578 { 579 struct ssb_state *st = this_cpu_ptr(&ssb_state); 580 u64 msr = x86_amd_ls_cfg_base; 581 582 if (!static_cpu_has(X86_FEATURE_ZEN)) { 583 msr |= ssbd_tif_to_amd_ls_cfg(tifn); 584 wrmsrq(MSR_AMD64_LS_CFG, msr); 585 return; 586 } 587 588 if (tifn & _TIF_SSBD) { 589 /* 590 * Since this can race with prctl(), block reentry on the 591 * same CPU. 592 */ 593 if (__test_and_set_bit(LSTATE_SSB, &st->local_state)) 594 return; 595 596 msr |= x86_amd_ls_cfg_ssbd_mask; 597 598 raw_spin_lock(&st->shared_state->lock); 599 /* First sibling enables SSBD: */ 600 if (!st->shared_state->disable_state) 601 wrmsrq(MSR_AMD64_LS_CFG, msr); 602 st->shared_state->disable_state++; 603 raw_spin_unlock(&st->shared_state->lock); 604 } else { 605 if (!__test_and_clear_bit(LSTATE_SSB, &st->local_state)) 606 return; 607 608 raw_spin_lock(&st->shared_state->lock); 609 st->shared_state->disable_state--; 610 if (!st->shared_state->disable_state) 611 wrmsrq(MSR_AMD64_LS_CFG, msr); 612 raw_spin_unlock(&st->shared_state->lock); 613 } 614 } 615 #else 616 static __always_inline void amd_set_core_ssb_state(unsigned long tifn) 617 { 618 u64 msr = x86_amd_ls_cfg_base | ssbd_tif_to_amd_ls_cfg(tifn); 619 620 wrmsrq(MSR_AMD64_LS_CFG, msr); 621 } 622 #endif 623 624 static __always_inline void amd_set_ssb_virt_state(unsigned long tifn) 625 { 626 /* 627 * SSBD has the same definition in SPEC_CTRL and VIRT_SPEC_CTRL, 628 * so ssbd_tif_to_spec_ctrl() just works. 629 */ 630 wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn)); 631 } 632 633 /* 634 * Update the MSRs managing speculation control, during context switch. 635 * 636 * tifp: Previous task's thread flags 637 * tifn: Next task's thread flags 638 */ 639 static __always_inline void __speculation_ctrl_update(unsigned long tifp, 640 unsigned long tifn) 641 { 642 unsigned long tif_diff = tifp ^ tifn; 643 u64 msr = x86_spec_ctrl_base; 644 bool updmsr = false; 645 646 lockdep_assert_irqs_disabled(); 647 648 /* Handle change of TIF_SSBD depending on the mitigation method. */ 649 if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) { 650 if (tif_diff & _TIF_SSBD) 651 amd_set_ssb_virt_state(tifn); 652 } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) { 653 if (tif_diff & _TIF_SSBD) 654 amd_set_core_ssb_state(tifn); 655 } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 656 static_cpu_has(X86_FEATURE_AMD_SSBD)) { 657 updmsr |= !!(tif_diff & _TIF_SSBD); 658 msr |= ssbd_tif_to_spec_ctrl(tifn); 659 } 660 661 /* Only evaluate TIF_SPEC_IB if conditional STIBP is enabled. */ 662 if (IS_ENABLED(CONFIG_SMP) && 663 static_branch_unlikely(&switch_to_cond_stibp)) { 664 updmsr |= !!(tif_diff & _TIF_SPEC_IB); 665 msr |= stibp_tif_to_spec_ctrl(tifn); 666 } 667 668 if (updmsr) 669 update_spec_ctrl_cond(msr); 670 } 671 672 static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk) 673 { 674 if (test_and_clear_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE)) { 675 if (task_spec_ssb_disable(tsk)) 676 set_tsk_thread_flag(tsk, TIF_SSBD); 677 else 678 clear_tsk_thread_flag(tsk, TIF_SSBD); 679 680 if (task_spec_ib_disable(tsk)) 681 set_tsk_thread_flag(tsk, TIF_SPEC_IB); 682 else 683 clear_tsk_thread_flag(tsk, TIF_SPEC_IB); 684 } 685 /* Return the updated threadinfo flags*/ 686 return read_task_thread_flags(tsk); 687 } 688 689 void speculation_ctrl_update(unsigned long tif) 690 { 691 unsigned long flags; 692 693 /* Forced update. Make sure all relevant TIF flags are different */ 694 local_irq_save(flags); 695 __speculation_ctrl_update(~tif, tif); 696 local_irq_restore(flags); 697 } 698 699 /* Called from seccomp/prctl update */ 700 void speculation_ctrl_update_current(void) 701 { 702 preempt_disable(); 703 speculation_ctrl_update(speculation_ctrl_update_tif(current)); 704 preempt_enable(); 705 } 706 707 static inline void cr4_toggle_bits_irqsoff(unsigned long mask) 708 { 709 unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4); 710 711 newval = cr4 ^ mask; 712 if (newval != cr4) { 713 this_cpu_write(cpu_tlbstate.cr4, newval); 714 __write_cr4(newval); 715 } 716 } 717 718 void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) 719 { 720 unsigned long tifp, tifn; 721 722 tifn = read_task_thread_flags(next_p); 723 tifp = read_task_thread_flags(prev_p); 724 725 switch_to_bitmap(tifp); 726 727 propagate_user_return_notify(prev_p, next_p); 728 729 if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) && 730 arch_has_block_step()) { 731 unsigned long debugctl, msk; 732 733 rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 734 debugctl &= ~DEBUGCTLMSR_BTF; 735 msk = tifn & _TIF_BLOCKSTEP; 736 debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT; 737 wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 738 } 739 740 if ((tifp ^ tifn) & _TIF_NOTSC) 741 cr4_toggle_bits_irqsoff(X86_CR4_TSD); 742 743 if ((tifp ^ tifn) & _TIF_NOCPUID) 744 set_cpuid_faulting(!!(tifn & _TIF_NOCPUID)); 745 746 if (likely(!((tifp | tifn) & _TIF_SPEC_FORCE_UPDATE))) { 747 __speculation_ctrl_update(tifp, tifn); 748 } else { 749 speculation_ctrl_update_tif(prev_p); 750 tifn = speculation_ctrl_update_tif(next_p); 751 752 /* Enforce MSR update to ensure consistent state */ 753 __speculation_ctrl_update(~tifn, tifn); 754 } 755 } 756 757 /* 758 * Idle related variables and functions 759 */ 760 unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; 761 EXPORT_SYMBOL(boot_option_idle_override); 762 763 /* 764 * We use this if we don't have any better idle routine.. 765 */ 766 void __cpuidle default_idle(void) 767 { 768 raw_safe_halt(); 769 raw_local_irq_disable(); 770 } 771 #if defined(CONFIG_APM_MODULE) || defined(CONFIG_HALTPOLL_CPUIDLE_MODULE) 772 EXPORT_SYMBOL(default_idle); 773 #endif 774 775 DEFINE_STATIC_CALL_NULL(x86_idle, default_idle); 776 777 static bool x86_idle_set(void) 778 { 779 return !!static_call_query(x86_idle); 780 } 781 782 #ifndef CONFIG_SMP 783 static inline void __noreturn play_dead(void) 784 { 785 BUG(); 786 } 787 #endif 788 789 void arch_cpu_idle_enter(void) 790 { 791 tsc_verify_tsc_adjust(false); 792 local_touch_nmi(); 793 } 794 795 void __noreturn arch_cpu_idle_dead(void) 796 { 797 play_dead(); 798 } 799 800 /* 801 * Called from the generic idle code. 802 */ 803 void __cpuidle arch_cpu_idle(void) 804 { 805 static_call(x86_idle)(); 806 } 807 EXPORT_SYMBOL_GPL(arch_cpu_idle); 808 809 #ifdef CONFIG_XEN 810 bool xen_set_default_idle(void) 811 { 812 bool ret = x86_idle_set(); 813 814 static_call_update(x86_idle, default_idle); 815 816 return ret; 817 } 818 #endif 819 820 struct cpumask cpus_stop_mask; 821 822 void __noreturn stop_this_cpu(void *dummy) 823 { 824 struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info); 825 unsigned int cpu = smp_processor_id(); 826 827 local_irq_disable(); 828 829 /* 830 * Remove this CPU from the online mask and disable it 831 * unconditionally. This might be redundant in case that the reboot 832 * vector was handled late and stop_other_cpus() sent an NMI. 833 * 834 * According to SDM and APM NMIs can be accepted even after soft 835 * disabling the local APIC. 836 */ 837 set_cpu_online(cpu, false); 838 disable_local_APIC(); 839 mcheck_cpu_clear(c); 840 841 if (this_cpu_read(cache_state_incoherent)) 842 wbinvd(); 843 844 /* 845 * This brings a cache line back and dirties it, but 846 * native_stop_other_cpus() will overwrite cpus_stop_mask after it 847 * observed that all CPUs reported stop. This write will invalidate 848 * the related cache line on this CPU. 849 */ 850 cpumask_clear_cpu(cpu, &cpus_stop_mask); 851 852 #ifdef CONFIG_SMP 853 if (smp_ops.stop_this_cpu) { 854 smp_ops.stop_this_cpu(); 855 BUG(); 856 } 857 #endif 858 859 for (;;) { 860 /* 861 * Use native_halt() so that memory contents don't change 862 * (stack usage and variables) after possibly issuing the 863 * wbinvd() above. 864 */ 865 native_halt(); 866 } 867 } 868 869 /* 870 * Prefer MWAIT over HALT if MWAIT is supported, MWAIT_CPUID leaf 871 * exists and whenever MONITOR/MWAIT extensions are present there is at 872 * least one C1 substate. 873 * 874 * Do not prefer MWAIT if MONITOR instruction has a bug or idle=nomwait 875 * is passed to kernel commandline parameter. 876 */ 877 static __init bool prefer_mwait_c1_over_halt(void) 878 { 879 const struct cpuinfo_x86 *c = &boot_cpu_data; 880 u32 eax, ebx, ecx, edx; 881 882 /* If override is enforced on the command line, fall back to HALT. */ 883 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 884 return false; 885 886 /* MWAIT is not supported on this platform. Fallback to HALT */ 887 if (!cpu_has(c, X86_FEATURE_MWAIT)) 888 return false; 889 890 /* Monitor has a bug or APIC stops in C1E. Fallback to HALT */ 891 if (boot_cpu_has_bug(X86_BUG_MONITOR) || boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) 892 return false; 893 894 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &edx); 895 896 /* 897 * If MWAIT extensions are not available, it is safe to use MWAIT 898 * with EAX=0, ECX=0. 899 */ 900 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) 901 return true; 902 903 /* 904 * If MWAIT extensions are available, there should be at least one 905 * MWAIT C1 substate present. 906 */ 907 return !!(edx & MWAIT_C1_SUBSTATE_MASK); 908 } 909 910 /* 911 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT 912 * with interrupts enabled and no flags, which is backwards compatible with the 913 * original MWAIT implementation. 914 */ 915 static __cpuidle void mwait_idle(void) 916 { 917 if (need_resched()) 918 return; 919 920 x86_idle_clear_cpu_buffers(); 921 922 if (!current_set_polling_and_test()) { 923 const void *addr = ¤t_thread_info()->flags; 924 925 alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); 926 __monitor(addr, 0, 0); 927 if (need_resched()) 928 goto out; 929 930 __sti_mwait(0, 0); 931 raw_local_irq_disable(); 932 } 933 934 out: 935 __current_clr_polling(); 936 } 937 938 void __init select_idle_routine(void) 939 { 940 if (boot_option_idle_override == IDLE_POLL) { 941 if (IS_ENABLED(CONFIG_SMP) && __max_threads_per_core > 1) 942 pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n"); 943 return; 944 } 945 946 /* Required to guard against xen_set_default_idle() */ 947 if (x86_idle_set()) 948 return; 949 950 if (prefer_mwait_c1_over_halt()) { 951 pr_info("using mwait in idle threads\n"); 952 static_call_update(x86_idle, mwait_idle); 953 } else if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) { 954 pr_info("using TDX aware idle routine\n"); 955 static_call_update(x86_idle, tdx_halt); 956 } else { 957 static_call_update(x86_idle, default_idle); 958 } 959 } 960 961 void amd_e400_c1e_apic_setup(void) 962 { 963 if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) { 964 pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id()); 965 local_irq_disable(); 966 tick_broadcast_force(); 967 local_irq_enable(); 968 } 969 } 970 971 void __init arch_post_acpi_subsys_init(void) 972 { 973 u64 val; 974 975 if (!boot_cpu_has_bug(X86_BUG_AMD_E400)) 976 return; 977 978 /* 979 * AMD E400 detection needs to happen after ACPI has been enabled. If 980 * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in 981 * MSR_K8_INT_PENDING_MSG. 982 */ 983 rdmsrq(MSR_K8_INT_PENDING_MSG, val); 984 if (!(val & K8_INTP_C1E_ACTIVE_MASK)) 985 return; 986 987 boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E); 988 989 if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 990 mark_tsc_unstable("TSC halt in AMD C1E"); 991 992 if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST_IDLE)) 993 static_branch_enable(&arch_needs_tick_broadcast); 994 pr_info("System has AMD C1E erratum E400. Workaround enabled.\n"); 995 } 996 997 static int __init idle_setup(char *str) 998 { 999 if (!str) 1000 return -EINVAL; 1001 1002 if (!strcmp(str, "poll")) { 1003 pr_info("using polling idle threads\n"); 1004 boot_option_idle_override = IDLE_POLL; 1005 cpu_idle_poll_ctrl(true); 1006 } else if (!strcmp(str, "halt")) { 1007 /* 'idle=halt' HALT for idle. C-states are disabled. */ 1008 boot_option_idle_override = IDLE_HALT; 1009 } else if (!strcmp(str, "nomwait")) { 1010 /* 'idle=nomwait' disables MWAIT for idle */ 1011 boot_option_idle_override = IDLE_NOMWAIT; 1012 } else { 1013 return -EINVAL; 1014 } 1015 1016 return 0; 1017 } 1018 early_param("idle", idle_setup); 1019 1020 unsigned long arch_align_stack(unsigned long sp) 1021 { 1022 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) 1023 sp -= get_random_u32_below(8192); 1024 return sp & ~0xf; 1025 } 1026 1027 unsigned long arch_randomize_brk(struct mm_struct *mm) 1028 { 1029 if (mmap_is_ia32()) 1030 return randomize_page(mm->brk, SZ_32M); 1031 1032 return randomize_page(mm->brk, SZ_1G); 1033 } 1034 1035 /* 1036 * Called from fs/proc with a reference on @p to find the function 1037 * which called into schedule(). This needs to be done carefully 1038 * because the task might wake up and we might look at a stack 1039 * changing under us. 1040 */ 1041 unsigned long __get_wchan(struct task_struct *p) 1042 { 1043 struct unwind_state state; 1044 unsigned long addr = 0; 1045 1046 if (!try_get_task_stack(p)) 1047 return 0; 1048 1049 for (unwind_start(&state, p, NULL, NULL); !unwind_done(&state); 1050 unwind_next_frame(&state)) { 1051 addr = unwind_get_return_address(&state); 1052 if (!addr) 1053 break; 1054 if (in_sched_functions(addr)) 1055 continue; 1056 break; 1057 } 1058 1059 put_task_stack(p); 1060 1061 return addr; 1062 } 1063 1064 SYSCALL_DEFINE2(arch_prctl, int, option, unsigned long, arg2) 1065 { 1066 switch (option) { 1067 case ARCH_GET_CPUID: 1068 return get_cpuid_mode(); 1069 case ARCH_SET_CPUID: 1070 return set_cpuid_mode(arg2); 1071 case ARCH_GET_XCOMP_SUPP: 1072 case ARCH_GET_XCOMP_PERM: 1073 case ARCH_REQ_XCOMP_PERM: 1074 case ARCH_GET_XCOMP_GUEST_PERM: 1075 case ARCH_REQ_XCOMP_GUEST_PERM: 1076 return fpu_xstate_prctl(option, arg2); 1077 } 1078 1079 if (!in_ia32_syscall()) 1080 return do_arch_prctl_64(current, option, arg2); 1081 1082 return -EINVAL; 1083 } 1084 1085 SYSCALL_DEFINE0(ni_syscall) 1086 { 1087 return -ENOSYS; 1088 } 1089