1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/utsname.h> 13 #include <linux/cpu.h> 14 #include <linux/module.h> 15 #include <linux/nospec.h> 16 #include <linux/prctl.h> 17 #include <linux/sched/smt.h> 18 #include <linux/pgtable.h> 19 #include <linux/bpf.h> 20 21 #include <asm/spec-ctrl.h> 22 #include <asm/cmdline.h> 23 #include <asm/bugs.h> 24 #include <asm/processor.h> 25 #include <asm/processor-flags.h> 26 #include <asm/fpu/api.h> 27 #include <asm/msr.h> 28 #include <asm/vmx.h> 29 #include <asm/paravirt.h> 30 #include <asm/alternative.h> 31 #include <asm/set_memory.h> 32 #include <asm/intel-family.h> 33 #include <asm/e820/api.h> 34 #include <asm/hypervisor.h> 35 #include <asm/tlbflush.h> 36 37 #include "cpu.h" 38 39 static void __init spectre_v1_select_mitigation(void); 40 static void __init spectre_v2_select_mitigation(void); 41 static void __init ssb_select_mitigation(void); 42 static void __init l1tf_select_mitigation(void); 43 static void __init mds_select_mitigation(void); 44 static void __init mds_print_mitigation(void); 45 static void __init taa_select_mitigation(void); 46 static void __init srbds_select_mitigation(void); 47 static void __init l1d_flush_select_mitigation(void); 48 49 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */ 50 u64 x86_spec_ctrl_base; 51 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 52 static DEFINE_MUTEX(spec_ctrl_mutex); 53 54 /* 55 * The vendor and possibly platform specific bits which can be modified in 56 * x86_spec_ctrl_base. 57 */ 58 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS; 59 60 /* 61 * AMD specific MSR info for Speculative Store Bypass control. 62 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 63 */ 64 u64 __ro_after_init x86_amd_ls_cfg_base; 65 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 66 67 /* Control conditional STIBP in switch_to() */ 68 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 69 /* Control conditional IBPB in switch_mm() */ 70 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 71 /* Control unconditional IBPB in switch_mm() */ 72 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 73 74 /* Control MDS CPU buffer clear before returning to user space */ 75 DEFINE_STATIC_KEY_FALSE(mds_user_clear); 76 EXPORT_SYMBOL_GPL(mds_user_clear); 77 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 78 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 79 EXPORT_SYMBOL_GPL(mds_idle_clear); 80 81 /* 82 * Controls whether l1d flush based mitigations are enabled, 83 * based on hw features and admin setting via boot parameter 84 * defaults to false 85 */ 86 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 87 88 void __init check_bugs(void) 89 { 90 identify_boot_cpu(); 91 92 /* 93 * identify_boot_cpu() initialized SMT support information, let the 94 * core code know. 95 */ 96 cpu_smt_check_topology(); 97 98 if (!IS_ENABLED(CONFIG_SMP)) { 99 pr_info("CPU: "); 100 print_cpu_info(&boot_cpu_data); 101 } 102 103 /* 104 * Read the SPEC_CTRL MSR to account for reserved bits which may 105 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 106 * init code as it is not enumerated and depends on the family. 107 */ 108 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 109 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 110 111 /* Allow STIBP in MSR_SPEC_CTRL if supported */ 112 if (boot_cpu_has(X86_FEATURE_STIBP)) 113 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP; 114 115 /* Select the proper CPU mitigations before patching alternatives: */ 116 spectre_v1_select_mitigation(); 117 spectre_v2_select_mitigation(); 118 ssb_select_mitigation(); 119 l1tf_select_mitigation(); 120 mds_select_mitigation(); 121 taa_select_mitigation(); 122 srbds_select_mitigation(); 123 l1d_flush_select_mitigation(); 124 125 /* 126 * As MDS and TAA mitigations are inter-related, print MDS 127 * mitigation until after TAA mitigation selection is done. 128 */ 129 mds_print_mitigation(); 130 131 arch_smt_update(); 132 133 #ifdef CONFIG_X86_32 134 /* 135 * Check whether we are able to run this kernel safely on SMP. 136 * 137 * - i386 is no longer supported. 138 * - In order to run on anything without a TSC, we need to be 139 * compiled for a i486. 140 */ 141 if (boot_cpu_data.x86 < 4) 142 panic("Kernel requires i486+ for 'invlpg' and other features"); 143 144 init_utsname()->machine[1] = 145 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 146 alternative_instructions(); 147 148 fpu__init_check_bugs(); 149 #else /* CONFIG_X86_64 */ 150 alternative_instructions(); 151 152 /* 153 * Make sure the first 2MB area is not mapped by huge pages 154 * There are typically fixed size MTRRs in there and overlapping 155 * MTRRs into large pages causes slow downs. 156 * 157 * Right now we don't do that with gbpages because there seems 158 * very little benefit for that case. 159 */ 160 if (!direct_gbpages) 161 set_memory_4k((unsigned long)__va(0), 1); 162 #endif 163 } 164 165 void 166 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) 167 { 168 u64 msrval, guestval, hostval = x86_spec_ctrl_base; 169 struct thread_info *ti = current_thread_info(); 170 171 /* Is MSR_SPEC_CTRL implemented ? */ 172 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) { 173 /* 174 * Restrict guest_spec_ctrl to supported values. Clear the 175 * modifiable bits in the host base value and or the 176 * modifiable bits from the guest value. 177 */ 178 guestval = hostval & ~x86_spec_ctrl_mask; 179 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask; 180 181 /* SSBD controlled in MSR_SPEC_CTRL */ 182 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 183 static_cpu_has(X86_FEATURE_AMD_SSBD)) 184 hostval |= ssbd_tif_to_spec_ctrl(ti->flags); 185 186 /* Conditional STIBP enabled? */ 187 if (static_branch_unlikely(&switch_to_cond_stibp)) 188 hostval |= stibp_tif_to_spec_ctrl(ti->flags); 189 190 if (hostval != guestval) { 191 msrval = setguest ? guestval : hostval; 192 wrmsrl(MSR_IA32_SPEC_CTRL, msrval); 193 } 194 } 195 196 /* 197 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 198 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 199 */ 200 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 201 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 202 return; 203 204 /* 205 * If the host has SSBD mitigation enabled, force it in the host's 206 * virtual MSR value. If its not permanently enabled, evaluate 207 * current's TIF_SSBD thread flag. 208 */ 209 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 210 hostval = SPEC_CTRL_SSBD; 211 else 212 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 213 214 /* Sanitize the guest value */ 215 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 216 217 if (hostval != guestval) { 218 unsigned long tif; 219 220 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 221 ssbd_spec_ctrl_to_tif(hostval); 222 223 speculation_ctrl_update(tif); 224 } 225 } 226 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 227 228 static void x86_amd_ssb_disable(void) 229 { 230 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 231 232 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 233 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 234 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 235 wrmsrl(MSR_AMD64_LS_CFG, msrval); 236 } 237 238 #undef pr_fmt 239 #define pr_fmt(fmt) "MDS: " fmt 240 241 /* Default mitigation for MDS-affected CPUs */ 242 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 243 static bool mds_nosmt __ro_after_init = false; 244 245 static const char * const mds_strings[] = { 246 [MDS_MITIGATION_OFF] = "Vulnerable", 247 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 248 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 249 }; 250 251 static void __init mds_select_mitigation(void) 252 { 253 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 254 mds_mitigation = MDS_MITIGATION_OFF; 255 return; 256 } 257 258 if (mds_mitigation == MDS_MITIGATION_FULL) { 259 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 260 mds_mitigation = MDS_MITIGATION_VMWERV; 261 262 static_branch_enable(&mds_user_clear); 263 264 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 265 (mds_nosmt || cpu_mitigations_auto_nosmt())) 266 cpu_smt_disable(false); 267 } 268 } 269 270 static void __init mds_print_mitigation(void) 271 { 272 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) 273 return; 274 275 pr_info("%s\n", mds_strings[mds_mitigation]); 276 } 277 278 static int __init mds_cmdline(char *str) 279 { 280 if (!boot_cpu_has_bug(X86_BUG_MDS)) 281 return 0; 282 283 if (!str) 284 return -EINVAL; 285 286 if (!strcmp(str, "off")) 287 mds_mitigation = MDS_MITIGATION_OFF; 288 else if (!strcmp(str, "full")) 289 mds_mitigation = MDS_MITIGATION_FULL; 290 else if (!strcmp(str, "full,nosmt")) { 291 mds_mitigation = MDS_MITIGATION_FULL; 292 mds_nosmt = true; 293 } 294 295 return 0; 296 } 297 early_param("mds", mds_cmdline); 298 299 #undef pr_fmt 300 #define pr_fmt(fmt) "TAA: " fmt 301 302 enum taa_mitigations { 303 TAA_MITIGATION_OFF, 304 TAA_MITIGATION_UCODE_NEEDED, 305 TAA_MITIGATION_VERW, 306 TAA_MITIGATION_TSX_DISABLED, 307 }; 308 309 /* Default mitigation for TAA-affected CPUs */ 310 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 311 static bool taa_nosmt __ro_after_init; 312 313 static const char * const taa_strings[] = { 314 [TAA_MITIGATION_OFF] = "Vulnerable", 315 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 316 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 317 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 318 }; 319 320 static void __init taa_select_mitigation(void) 321 { 322 u64 ia32_cap; 323 324 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 325 taa_mitigation = TAA_MITIGATION_OFF; 326 return; 327 } 328 329 /* TSX previously disabled by tsx=off */ 330 if (!boot_cpu_has(X86_FEATURE_RTM)) { 331 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 332 goto out; 333 } 334 335 if (cpu_mitigations_off()) { 336 taa_mitigation = TAA_MITIGATION_OFF; 337 return; 338 } 339 340 /* 341 * TAA mitigation via VERW is turned off if both 342 * tsx_async_abort=off and mds=off are specified. 343 */ 344 if (taa_mitigation == TAA_MITIGATION_OFF && 345 mds_mitigation == MDS_MITIGATION_OFF) 346 goto out; 347 348 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 349 taa_mitigation = TAA_MITIGATION_VERW; 350 else 351 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 352 353 /* 354 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 355 * A microcode update fixes this behavior to clear CPU buffers. It also 356 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 357 * ARCH_CAP_TSX_CTRL_MSR bit. 358 * 359 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 360 * update is required. 361 */ 362 ia32_cap = x86_read_arch_cap_msr(); 363 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 364 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 365 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 366 367 /* 368 * TSX is enabled, select alternate mitigation for TAA which is 369 * the same as MDS. Enable MDS static branch to clear CPU buffers. 370 * 371 * For guests that can't determine whether the correct microcode is 372 * present on host, enable the mitigation for UCODE_NEEDED as well. 373 */ 374 static_branch_enable(&mds_user_clear); 375 376 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 377 cpu_smt_disable(false); 378 379 /* 380 * Update MDS mitigation, if necessary, as the mds_user_clear is 381 * now enabled for TAA mitigation. 382 */ 383 if (mds_mitigation == MDS_MITIGATION_OFF && 384 boot_cpu_has_bug(X86_BUG_MDS)) { 385 mds_mitigation = MDS_MITIGATION_FULL; 386 mds_select_mitigation(); 387 } 388 out: 389 pr_info("%s\n", taa_strings[taa_mitigation]); 390 } 391 392 static int __init tsx_async_abort_parse_cmdline(char *str) 393 { 394 if (!boot_cpu_has_bug(X86_BUG_TAA)) 395 return 0; 396 397 if (!str) 398 return -EINVAL; 399 400 if (!strcmp(str, "off")) { 401 taa_mitigation = TAA_MITIGATION_OFF; 402 } else if (!strcmp(str, "full")) { 403 taa_mitigation = TAA_MITIGATION_VERW; 404 } else if (!strcmp(str, "full,nosmt")) { 405 taa_mitigation = TAA_MITIGATION_VERW; 406 taa_nosmt = true; 407 } 408 409 return 0; 410 } 411 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 412 413 #undef pr_fmt 414 #define pr_fmt(fmt) "SRBDS: " fmt 415 416 enum srbds_mitigations { 417 SRBDS_MITIGATION_OFF, 418 SRBDS_MITIGATION_UCODE_NEEDED, 419 SRBDS_MITIGATION_FULL, 420 SRBDS_MITIGATION_TSX_OFF, 421 SRBDS_MITIGATION_HYPERVISOR, 422 }; 423 424 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL; 425 426 static const char * const srbds_strings[] = { 427 [SRBDS_MITIGATION_OFF] = "Vulnerable", 428 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 429 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 430 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 431 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 432 }; 433 434 static bool srbds_off; 435 436 void update_srbds_msr(void) 437 { 438 u64 mcu_ctrl; 439 440 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 441 return; 442 443 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 444 return; 445 446 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 447 return; 448 449 /* 450 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 451 * being disabled and it hasn't received the SRBDS MSR microcode. 452 */ 453 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 454 return; 455 456 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 457 458 switch (srbds_mitigation) { 459 case SRBDS_MITIGATION_OFF: 460 case SRBDS_MITIGATION_TSX_OFF: 461 mcu_ctrl |= RNGDS_MITG_DIS; 462 break; 463 case SRBDS_MITIGATION_FULL: 464 mcu_ctrl &= ~RNGDS_MITG_DIS; 465 break; 466 default: 467 break; 468 } 469 470 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 471 } 472 473 static void __init srbds_select_mitigation(void) 474 { 475 u64 ia32_cap; 476 477 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 478 return; 479 480 /* 481 * Check to see if this is one of the MDS_NO systems supporting 482 * TSX that are only exposed to SRBDS when TSX is enabled. 483 */ 484 ia32_cap = x86_read_arch_cap_msr(); 485 if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM)) 486 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 487 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 488 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 489 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 490 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 491 else if (cpu_mitigations_off() || srbds_off) 492 srbds_mitigation = SRBDS_MITIGATION_OFF; 493 494 update_srbds_msr(); 495 pr_info("%s\n", srbds_strings[srbds_mitigation]); 496 } 497 498 static int __init srbds_parse_cmdline(char *str) 499 { 500 if (!str) 501 return -EINVAL; 502 503 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 504 return 0; 505 506 srbds_off = !strcmp(str, "off"); 507 return 0; 508 } 509 early_param("srbds", srbds_parse_cmdline); 510 511 #undef pr_fmt 512 #define pr_fmt(fmt) "L1D Flush : " fmt 513 514 enum l1d_flush_mitigations { 515 L1D_FLUSH_OFF = 0, 516 L1D_FLUSH_ON, 517 }; 518 519 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 520 521 static void __init l1d_flush_select_mitigation(void) 522 { 523 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 524 return; 525 526 static_branch_enable(&switch_mm_cond_l1d_flush); 527 pr_info("Conditional flush on switch_mm() enabled\n"); 528 } 529 530 static int __init l1d_flush_parse_cmdline(char *str) 531 { 532 if (!strcmp(str, "on")) 533 l1d_flush_mitigation = L1D_FLUSH_ON; 534 535 return 0; 536 } 537 early_param("l1d_flush", l1d_flush_parse_cmdline); 538 539 #undef pr_fmt 540 #define pr_fmt(fmt) "Spectre V1 : " fmt 541 542 enum spectre_v1_mitigation { 543 SPECTRE_V1_MITIGATION_NONE, 544 SPECTRE_V1_MITIGATION_AUTO, 545 }; 546 547 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 548 SPECTRE_V1_MITIGATION_AUTO; 549 550 static const char * const spectre_v1_strings[] = { 551 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 552 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 553 }; 554 555 /* 556 * Does SMAP provide full mitigation against speculative kernel access to 557 * userspace? 558 */ 559 static bool smap_works_speculatively(void) 560 { 561 if (!boot_cpu_has(X86_FEATURE_SMAP)) 562 return false; 563 564 /* 565 * On CPUs which are vulnerable to Meltdown, SMAP does not 566 * prevent speculative access to user data in the L1 cache. 567 * Consider SMAP to be non-functional as a mitigation on these 568 * CPUs. 569 */ 570 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 571 return false; 572 573 return true; 574 } 575 576 static void __init spectre_v1_select_mitigation(void) 577 { 578 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 579 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 580 return; 581 } 582 583 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 584 /* 585 * With Spectre v1, a user can speculatively control either 586 * path of a conditional swapgs with a user-controlled GS 587 * value. The mitigation is to add lfences to both code paths. 588 * 589 * If FSGSBASE is enabled, the user can put a kernel address in 590 * GS, in which case SMAP provides no protection. 591 * 592 * If FSGSBASE is disabled, the user can only put a user space 593 * address in GS. That makes an attack harder, but still 594 * possible if there's no SMAP protection. 595 */ 596 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 597 !smap_works_speculatively()) { 598 /* 599 * Mitigation can be provided from SWAPGS itself or 600 * PTI as the CR3 write in the Meltdown mitigation 601 * is serializing. 602 * 603 * If neither is there, mitigate with an LFENCE to 604 * stop speculation through swapgs. 605 */ 606 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 607 !boot_cpu_has(X86_FEATURE_PTI)) 608 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 609 610 /* 611 * Enable lfences in the kernel entry (non-swapgs) 612 * paths, to prevent user entry from speculatively 613 * skipping swapgs. 614 */ 615 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 616 } 617 } 618 619 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 620 } 621 622 static int __init nospectre_v1_cmdline(char *str) 623 { 624 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 625 return 0; 626 } 627 early_param("nospectre_v1", nospectre_v1_cmdline); 628 629 #undef pr_fmt 630 #define pr_fmt(fmt) "Spectre V2 : " fmt 631 632 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = 633 SPECTRE_V2_NONE; 634 635 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 636 SPECTRE_V2_USER_NONE; 637 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 638 SPECTRE_V2_USER_NONE; 639 640 #ifdef CONFIG_RETPOLINE 641 static bool spectre_v2_bad_module; 642 643 bool retpoline_module_ok(bool has_retpoline) 644 { 645 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 646 return true; 647 648 pr_err("System may be vulnerable to spectre v2\n"); 649 spectre_v2_bad_module = true; 650 return false; 651 } 652 653 static inline const char *spectre_v2_module_string(void) 654 { 655 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 656 } 657 #else 658 static inline const char *spectre_v2_module_string(void) { return ""; } 659 #endif 660 661 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 662 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 663 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 664 665 #ifdef CONFIG_BPF_SYSCALL 666 void unpriv_ebpf_notify(int new_state) 667 { 668 if (new_state) 669 return; 670 671 /* Unprivileged eBPF is enabled */ 672 673 switch (spectre_v2_enabled) { 674 case SPECTRE_V2_EIBRS: 675 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 676 break; 677 case SPECTRE_V2_EIBRS_LFENCE: 678 if (sched_smt_active()) 679 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 680 break; 681 default: 682 break; 683 } 684 } 685 #endif 686 687 static inline bool match_option(const char *arg, int arglen, const char *opt) 688 { 689 int len = strlen(opt); 690 691 return len == arglen && !strncmp(arg, opt, len); 692 } 693 694 /* The kernel command line selection for spectre v2 */ 695 enum spectre_v2_mitigation_cmd { 696 SPECTRE_V2_CMD_NONE, 697 SPECTRE_V2_CMD_AUTO, 698 SPECTRE_V2_CMD_FORCE, 699 SPECTRE_V2_CMD_RETPOLINE, 700 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 701 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 702 SPECTRE_V2_CMD_EIBRS, 703 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 704 SPECTRE_V2_CMD_EIBRS_LFENCE, 705 }; 706 707 enum spectre_v2_user_cmd { 708 SPECTRE_V2_USER_CMD_NONE, 709 SPECTRE_V2_USER_CMD_AUTO, 710 SPECTRE_V2_USER_CMD_FORCE, 711 SPECTRE_V2_USER_CMD_PRCTL, 712 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 713 SPECTRE_V2_USER_CMD_SECCOMP, 714 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 715 }; 716 717 static const char * const spectre_v2_user_strings[] = { 718 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 719 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 720 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 721 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 722 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 723 }; 724 725 static const struct { 726 const char *option; 727 enum spectre_v2_user_cmd cmd; 728 bool secure; 729 } v2_user_options[] __initconst = { 730 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 731 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 732 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 733 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 734 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 735 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 736 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 737 }; 738 739 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 740 { 741 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 742 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 743 } 744 745 static enum spectre_v2_user_cmd __init 746 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) 747 { 748 char arg[20]; 749 int ret, i; 750 751 switch (v2_cmd) { 752 case SPECTRE_V2_CMD_NONE: 753 return SPECTRE_V2_USER_CMD_NONE; 754 case SPECTRE_V2_CMD_FORCE: 755 return SPECTRE_V2_USER_CMD_FORCE; 756 default: 757 break; 758 } 759 760 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 761 arg, sizeof(arg)); 762 if (ret < 0) 763 return SPECTRE_V2_USER_CMD_AUTO; 764 765 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 766 if (match_option(arg, ret, v2_user_options[i].option)) { 767 spec_v2_user_print_cond(v2_user_options[i].option, 768 v2_user_options[i].secure); 769 return v2_user_options[i].cmd; 770 } 771 } 772 773 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 774 return SPECTRE_V2_USER_CMD_AUTO; 775 } 776 777 static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode) 778 { 779 return (mode == SPECTRE_V2_EIBRS || 780 mode == SPECTRE_V2_EIBRS_RETPOLINE || 781 mode == SPECTRE_V2_EIBRS_LFENCE); 782 } 783 784 static void __init 785 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) 786 { 787 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 788 bool smt_possible = IS_ENABLED(CONFIG_SMP); 789 enum spectre_v2_user_cmd cmd; 790 791 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 792 return; 793 794 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 795 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 796 smt_possible = false; 797 798 cmd = spectre_v2_parse_user_cmdline(v2_cmd); 799 switch (cmd) { 800 case SPECTRE_V2_USER_CMD_NONE: 801 goto set_mode; 802 case SPECTRE_V2_USER_CMD_FORCE: 803 mode = SPECTRE_V2_USER_STRICT; 804 break; 805 case SPECTRE_V2_USER_CMD_AUTO: 806 case SPECTRE_V2_USER_CMD_PRCTL: 807 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 808 mode = SPECTRE_V2_USER_PRCTL; 809 break; 810 case SPECTRE_V2_USER_CMD_SECCOMP: 811 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 812 if (IS_ENABLED(CONFIG_SECCOMP)) 813 mode = SPECTRE_V2_USER_SECCOMP; 814 else 815 mode = SPECTRE_V2_USER_PRCTL; 816 break; 817 } 818 819 /* Initialize Indirect Branch Prediction Barrier */ 820 if (boot_cpu_has(X86_FEATURE_IBPB)) { 821 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 822 823 spectre_v2_user_ibpb = mode; 824 switch (cmd) { 825 case SPECTRE_V2_USER_CMD_FORCE: 826 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 827 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 828 static_branch_enable(&switch_mm_always_ibpb); 829 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 830 break; 831 case SPECTRE_V2_USER_CMD_PRCTL: 832 case SPECTRE_V2_USER_CMD_AUTO: 833 case SPECTRE_V2_USER_CMD_SECCOMP: 834 static_branch_enable(&switch_mm_cond_ibpb); 835 break; 836 default: 837 break; 838 } 839 840 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 841 static_key_enabled(&switch_mm_always_ibpb) ? 842 "always-on" : "conditional"); 843 } 844 845 /* 846 * If no STIBP, enhanced IBRS is enabled or SMT impossible, STIBP is not 847 * required. 848 */ 849 if (!boot_cpu_has(X86_FEATURE_STIBP) || 850 !smt_possible || 851 spectre_v2_in_eibrs_mode(spectre_v2_enabled)) 852 return; 853 854 /* 855 * At this point, an STIBP mode other than "off" has been set. 856 * If STIBP support is not being forced, check if STIBP always-on 857 * is preferred. 858 */ 859 if (mode != SPECTRE_V2_USER_STRICT && 860 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 861 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 862 863 spectre_v2_user_stibp = mode; 864 865 set_mode: 866 pr_info("%s\n", spectre_v2_user_strings[mode]); 867 } 868 869 static const char * const spectre_v2_strings[] = { 870 [SPECTRE_V2_NONE] = "Vulnerable", 871 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 872 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 873 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", 874 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", 875 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", 876 }; 877 878 static const struct { 879 const char *option; 880 enum spectre_v2_mitigation_cmd cmd; 881 bool secure; 882 } mitigation_options[] __initconst = { 883 { "off", SPECTRE_V2_CMD_NONE, false }, 884 { "on", SPECTRE_V2_CMD_FORCE, true }, 885 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 886 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 887 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 888 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 889 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 890 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 891 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 892 { "auto", SPECTRE_V2_CMD_AUTO, false }, 893 }; 894 895 static void __init spec_v2_print_cond(const char *reason, bool secure) 896 { 897 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 898 pr_info("%s selected on command line.\n", reason); 899 } 900 901 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 902 { 903 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 904 char arg[20]; 905 int ret, i; 906 907 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 908 cpu_mitigations_off()) 909 return SPECTRE_V2_CMD_NONE; 910 911 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 912 if (ret < 0) 913 return SPECTRE_V2_CMD_AUTO; 914 915 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 916 if (!match_option(arg, ret, mitigation_options[i].option)) 917 continue; 918 cmd = mitigation_options[i].cmd; 919 break; 920 } 921 922 if (i >= ARRAY_SIZE(mitigation_options)) { 923 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 924 return SPECTRE_V2_CMD_AUTO; 925 } 926 927 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 928 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 929 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 930 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 931 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 932 !IS_ENABLED(CONFIG_RETPOLINE)) { 933 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 934 mitigation_options[i].option); 935 return SPECTRE_V2_CMD_AUTO; 936 } 937 938 if ((cmd == SPECTRE_V2_CMD_EIBRS || 939 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 940 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 941 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 942 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", 943 mitigation_options[i].option); 944 return SPECTRE_V2_CMD_AUTO; 945 } 946 947 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 948 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 949 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 950 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 951 mitigation_options[i].option); 952 return SPECTRE_V2_CMD_AUTO; 953 } 954 955 spec_v2_print_cond(mitigation_options[i].option, 956 mitigation_options[i].secure); 957 return cmd; 958 } 959 960 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 961 { 962 if (!IS_ENABLED(CONFIG_RETPOLINE)) { 963 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 964 return SPECTRE_V2_NONE; 965 } 966 967 return SPECTRE_V2_RETPOLINE; 968 } 969 970 static void __init spectre_v2_select_mitigation(void) 971 { 972 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 973 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 974 975 /* 976 * If the CPU is not affected and the command line mode is NONE or AUTO 977 * then nothing to do. 978 */ 979 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 980 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 981 return; 982 983 switch (cmd) { 984 case SPECTRE_V2_CMD_NONE: 985 return; 986 987 case SPECTRE_V2_CMD_FORCE: 988 case SPECTRE_V2_CMD_AUTO: 989 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 990 mode = SPECTRE_V2_EIBRS; 991 break; 992 } 993 994 mode = spectre_v2_select_retpoline(); 995 break; 996 997 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 998 pr_err(SPECTRE_V2_LFENCE_MSG); 999 mode = SPECTRE_V2_LFENCE; 1000 break; 1001 1002 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 1003 mode = SPECTRE_V2_RETPOLINE; 1004 break; 1005 1006 case SPECTRE_V2_CMD_RETPOLINE: 1007 mode = spectre_v2_select_retpoline(); 1008 break; 1009 1010 case SPECTRE_V2_CMD_EIBRS: 1011 mode = SPECTRE_V2_EIBRS; 1012 break; 1013 1014 case SPECTRE_V2_CMD_EIBRS_LFENCE: 1015 mode = SPECTRE_V2_EIBRS_LFENCE; 1016 break; 1017 1018 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 1019 mode = SPECTRE_V2_EIBRS_RETPOLINE; 1020 break; 1021 } 1022 1023 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1024 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1025 1026 if (spectre_v2_in_eibrs_mode(mode)) { 1027 /* Force it so VMEXIT will restore correctly */ 1028 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 1029 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1030 } 1031 1032 switch (mode) { 1033 case SPECTRE_V2_NONE: 1034 case SPECTRE_V2_EIBRS: 1035 break; 1036 1037 case SPECTRE_V2_LFENCE: 1038 case SPECTRE_V2_EIBRS_LFENCE: 1039 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 1040 fallthrough; 1041 1042 case SPECTRE_V2_RETPOLINE: 1043 case SPECTRE_V2_EIBRS_RETPOLINE: 1044 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 1045 break; 1046 } 1047 1048 spectre_v2_enabled = mode; 1049 pr_info("%s\n", spectre_v2_strings[mode]); 1050 1051 /* 1052 * If spectre v2 protection has been enabled, unconditionally fill 1053 * RSB during a context switch; this protects against two independent 1054 * issues: 1055 * 1056 * - RSB underflow (and switch to BTB) on Skylake+ 1057 * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs 1058 */ 1059 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1060 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 1061 1062 /* 1063 * Retpoline means the kernel is safe because it has no indirect 1064 * branches. Enhanced IBRS protects firmware too, so, enable restricted 1065 * speculation around firmware calls only when Enhanced IBRS isn't 1066 * supported. 1067 * 1068 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 1069 * the user might select retpoline on the kernel command line and if 1070 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 1071 * enable IBRS around firmware calls. 1072 */ 1073 if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) { 1074 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 1075 pr_info("Enabling Restricted Speculation for firmware calls\n"); 1076 } 1077 1078 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 1079 spectre_v2_user_select_mitigation(cmd); 1080 } 1081 1082 static void update_stibp_msr(void * __unused) 1083 { 1084 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1085 } 1086 1087 /* Update x86_spec_ctrl_base in case SMT state changed. */ 1088 static void update_stibp_strict(void) 1089 { 1090 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 1091 1092 if (sched_smt_active()) 1093 mask |= SPEC_CTRL_STIBP; 1094 1095 if (mask == x86_spec_ctrl_base) 1096 return; 1097 1098 pr_info("Update user space SMT mitigation: STIBP %s\n", 1099 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 1100 x86_spec_ctrl_base = mask; 1101 on_each_cpu(update_stibp_msr, NULL, 1); 1102 } 1103 1104 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 1105 static void update_indir_branch_cond(void) 1106 { 1107 if (sched_smt_active()) 1108 static_branch_enable(&switch_to_cond_stibp); 1109 else 1110 static_branch_disable(&switch_to_cond_stibp); 1111 } 1112 1113 #undef pr_fmt 1114 #define pr_fmt(fmt) fmt 1115 1116 /* Update the static key controlling the MDS CPU buffer clear in idle */ 1117 static void update_mds_branch_idle(void) 1118 { 1119 /* 1120 * Enable the idle clearing if SMT is active on CPUs which are 1121 * affected only by MSBDS and not any other MDS variant. 1122 * 1123 * The other variants cannot be mitigated when SMT is enabled, so 1124 * clearing the buffers on idle just to prevent the Store Buffer 1125 * repartitioning leak would be a window dressing exercise. 1126 */ 1127 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 1128 return; 1129 1130 if (sched_smt_active()) 1131 static_branch_enable(&mds_idle_clear); 1132 else 1133 static_branch_disable(&mds_idle_clear); 1134 } 1135 1136 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 1137 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 1138 1139 void cpu_bugs_smt_update(void) 1140 { 1141 mutex_lock(&spec_ctrl_mutex); 1142 1143 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1144 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1145 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1146 1147 switch (spectre_v2_user_stibp) { 1148 case SPECTRE_V2_USER_NONE: 1149 break; 1150 case SPECTRE_V2_USER_STRICT: 1151 case SPECTRE_V2_USER_STRICT_PREFERRED: 1152 update_stibp_strict(); 1153 break; 1154 case SPECTRE_V2_USER_PRCTL: 1155 case SPECTRE_V2_USER_SECCOMP: 1156 update_indir_branch_cond(); 1157 break; 1158 } 1159 1160 switch (mds_mitigation) { 1161 case MDS_MITIGATION_FULL: 1162 case MDS_MITIGATION_VMWERV: 1163 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 1164 pr_warn_once(MDS_MSG_SMT); 1165 update_mds_branch_idle(); 1166 break; 1167 case MDS_MITIGATION_OFF: 1168 break; 1169 } 1170 1171 switch (taa_mitigation) { 1172 case TAA_MITIGATION_VERW: 1173 case TAA_MITIGATION_UCODE_NEEDED: 1174 if (sched_smt_active()) 1175 pr_warn_once(TAA_MSG_SMT); 1176 break; 1177 case TAA_MITIGATION_TSX_DISABLED: 1178 case TAA_MITIGATION_OFF: 1179 break; 1180 } 1181 1182 mutex_unlock(&spec_ctrl_mutex); 1183 } 1184 1185 #undef pr_fmt 1186 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 1187 1188 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 1189 1190 /* The kernel command line selection */ 1191 enum ssb_mitigation_cmd { 1192 SPEC_STORE_BYPASS_CMD_NONE, 1193 SPEC_STORE_BYPASS_CMD_AUTO, 1194 SPEC_STORE_BYPASS_CMD_ON, 1195 SPEC_STORE_BYPASS_CMD_PRCTL, 1196 SPEC_STORE_BYPASS_CMD_SECCOMP, 1197 }; 1198 1199 static const char * const ssb_strings[] = { 1200 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 1201 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 1202 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 1203 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 1204 }; 1205 1206 static const struct { 1207 const char *option; 1208 enum ssb_mitigation_cmd cmd; 1209 } ssb_mitigation_options[] __initconst = { 1210 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 1211 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 1212 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 1213 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 1214 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 1215 }; 1216 1217 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 1218 { 1219 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 1220 char arg[20]; 1221 int ret, i; 1222 1223 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 1224 cpu_mitigations_off()) { 1225 return SPEC_STORE_BYPASS_CMD_NONE; 1226 } else { 1227 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 1228 arg, sizeof(arg)); 1229 if (ret < 0) 1230 return SPEC_STORE_BYPASS_CMD_AUTO; 1231 1232 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1233 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1234 continue; 1235 1236 cmd = ssb_mitigation_options[i].cmd; 1237 break; 1238 } 1239 1240 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1241 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1242 return SPEC_STORE_BYPASS_CMD_AUTO; 1243 } 1244 } 1245 1246 return cmd; 1247 } 1248 1249 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1250 { 1251 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1252 enum ssb_mitigation_cmd cmd; 1253 1254 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1255 return mode; 1256 1257 cmd = ssb_parse_cmdline(); 1258 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1259 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1260 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1261 return mode; 1262 1263 switch (cmd) { 1264 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1265 /* 1266 * Choose prctl+seccomp as the default mode if seccomp is 1267 * enabled. 1268 */ 1269 if (IS_ENABLED(CONFIG_SECCOMP)) 1270 mode = SPEC_STORE_BYPASS_SECCOMP; 1271 else 1272 mode = SPEC_STORE_BYPASS_PRCTL; 1273 break; 1274 case SPEC_STORE_BYPASS_CMD_ON: 1275 mode = SPEC_STORE_BYPASS_DISABLE; 1276 break; 1277 case SPEC_STORE_BYPASS_CMD_AUTO: 1278 case SPEC_STORE_BYPASS_CMD_PRCTL: 1279 mode = SPEC_STORE_BYPASS_PRCTL; 1280 break; 1281 case SPEC_STORE_BYPASS_CMD_NONE: 1282 break; 1283 } 1284 1285 /* 1286 * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper 1287 * bit in the mask to allow guests to use the mitigation even in the 1288 * case where the host does not enable it. 1289 */ 1290 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 1291 static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1292 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD; 1293 } 1294 1295 /* 1296 * We have three CPU feature flags that are in play here: 1297 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1298 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1299 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1300 */ 1301 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1302 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1303 /* 1304 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1305 * use a completely different MSR and bit dependent on family. 1306 */ 1307 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1308 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1309 x86_amd_ssb_disable(); 1310 } else { 1311 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1312 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1313 } 1314 } 1315 1316 return mode; 1317 } 1318 1319 static void ssb_select_mitigation(void) 1320 { 1321 ssb_mode = __ssb_select_mitigation(); 1322 1323 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1324 pr_info("%s\n", ssb_strings[ssb_mode]); 1325 } 1326 1327 #undef pr_fmt 1328 #define pr_fmt(fmt) "Speculation prctl: " fmt 1329 1330 static void task_update_spec_tif(struct task_struct *tsk) 1331 { 1332 /* Force the update of the real TIF bits */ 1333 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1334 1335 /* 1336 * Immediately update the speculation control MSRs for the current 1337 * task, but for a non-current task delay setting the CPU 1338 * mitigation until it is scheduled next. 1339 * 1340 * This can only happen for SECCOMP mitigation. For PRCTL it's 1341 * always the current task. 1342 */ 1343 if (tsk == current) 1344 speculation_ctrl_update_current(); 1345 } 1346 1347 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 1348 { 1349 1350 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 1351 return -EPERM; 1352 1353 switch (ctrl) { 1354 case PR_SPEC_ENABLE: 1355 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 1356 return 0; 1357 case PR_SPEC_DISABLE: 1358 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 1359 return 0; 1360 default: 1361 return -ERANGE; 1362 } 1363 } 1364 1365 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 1366 { 1367 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 1368 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 1369 return -ENXIO; 1370 1371 switch (ctrl) { 1372 case PR_SPEC_ENABLE: 1373 /* If speculation is force disabled, enable is not allowed */ 1374 if (task_spec_ssb_force_disable(task)) 1375 return -EPERM; 1376 task_clear_spec_ssb_disable(task); 1377 task_clear_spec_ssb_noexec(task); 1378 task_update_spec_tif(task); 1379 break; 1380 case PR_SPEC_DISABLE: 1381 task_set_spec_ssb_disable(task); 1382 task_clear_spec_ssb_noexec(task); 1383 task_update_spec_tif(task); 1384 break; 1385 case PR_SPEC_FORCE_DISABLE: 1386 task_set_spec_ssb_disable(task); 1387 task_set_spec_ssb_force_disable(task); 1388 task_clear_spec_ssb_noexec(task); 1389 task_update_spec_tif(task); 1390 break; 1391 case PR_SPEC_DISABLE_NOEXEC: 1392 if (task_spec_ssb_force_disable(task)) 1393 return -EPERM; 1394 task_set_spec_ssb_disable(task); 1395 task_set_spec_ssb_noexec(task); 1396 task_update_spec_tif(task); 1397 break; 1398 default: 1399 return -ERANGE; 1400 } 1401 return 0; 1402 } 1403 1404 static bool is_spec_ib_user_controlled(void) 1405 { 1406 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 1407 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 1408 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1409 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 1410 } 1411 1412 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 1413 { 1414 switch (ctrl) { 1415 case PR_SPEC_ENABLE: 1416 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1417 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1418 return 0; 1419 1420 /* 1421 * With strict mode for both IBPB and STIBP, the instruction 1422 * code paths avoid checking this task flag and instead, 1423 * unconditionally run the instruction. However, STIBP and IBPB 1424 * are independent and either can be set to conditionally 1425 * enabled regardless of the mode of the other. 1426 * 1427 * If either is set to conditional, allow the task flag to be 1428 * updated, unless it was force-disabled by a previous prctl 1429 * call. Currently, this is possible on an AMD CPU which has the 1430 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 1431 * kernel is booted with 'spectre_v2_user=seccomp', then 1432 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 1433 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 1434 */ 1435 if (!is_spec_ib_user_controlled() || 1436 task_spec_ib_force_disable(task)) 1437 return -EPERM; 1438 1439 task_clear_spec_ib_disable(task); 1440 task_update_spec_tif(task); 1441 break; 1442 case PR_SPEC_DISABLE: 1443 case PR_SPEC_FORCE_DISABLE: 1444 /* 1445 * Indirect branch speculation is always allowed when 1446 * mitigation is force disabled. 1447 */ 1448 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1449 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1450 return -EPERM; 1451 1452 if (!is_spec_ib_user_controlled()) 1453 return 0; 1454 1455 task_set_spec_ib_disable(task); 1456 if (ctrl == PR_SPEC_FORCE_DISABLE) 1457 task_set_spec_ib_force_disable(task); 1458 task_update_spec_tif(task); 1459 break; 1460 default: 1461 return -ERANGE; 1462 } 1463 return 0; 1464 } 1465 1466 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 1467 unsigned long ctrl) 1468 { 1469 switch (which) { 1470 case PR_SPEC_STORE_BYPASS: 1471 return ssb_prctl_set(task, ctrl); 1472 case PR_SPEC_INDIRECT_BRANCH: 1473 return ib_prctl_set(task, ctrl); 1474 case PR_SPEC_L1D_FLUSH: 1475 return l1d_flush_prctl_set(task, ctrl); 1476 default: 1477 return -ENODEV; 1478 } 1479 } 1480 1481 #ifdef CONFIG_SECCOMP 1482 void arch_seccomp_spec_mitigate(struct task_struct *task) 1483 { 1484 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 1485 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 1486 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 1487 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 1488 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 1489 } 1490 #endif 1491 1492 static int l1d_flush_prctl_get(struct task_struct *task) 1493 { 1494 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 1495 return PR_SPEC_FORCE_DISABLE; 1496 1497 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 1498 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1499 else 1500 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1501 } 1502 1503 static int ssb_prctl_get(struct task_struct *task) 1504 { 1505 switch (ssb_mode) { 1506 case SPEC_STORE_BYPASS_DISABLE: 1507 return PR_SPEC_DISABLE; 1508 case SPEC_STORE_BYPASS_SECCOMP: 1509 case SPEC_STORE_BYPASS_PRCTL: 1510 if (task_spec_ssb_force_disable(task)) 1511 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 1512 if (task_spec_ssb_noexec(task)) 1513 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 1514 if (task_spec_ssb_disable(task)) 1515 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1516 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1517 default: 1518 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1519 return PR_SPEC_ENABLE; 1520 return PR_SPEC_NOT_AFFECTED; 1521 } 1522 } 1523 1524 static int ib_prctl_get(struct task_struct *task) 1525 { 1526 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 1527 return PR_SPEC_NOT_AFFECTED; 1528 1529 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1530 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1531 return PR_SPEC_ENABLE; 1532 else if (is_spec_ib_user_controlled()) { 1533 if (task_spec_ib_force_disable(task)) 1534 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 1535 if (task_spec_ib_disable(task)) 1536 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1537 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1538 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 1539 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 1540 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 1541 return PR_SPEC_DISABLE; 1542 else 1543 return PR_SPEC_NOT_AFFECTED; 1544 } 1545 1546 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 1547 { 1548 switch (which) { 1549 case PR_SPEC_STORE_BYPASS: 1550 return ssb_prctl_get(task); 1551 case PR_SPEC_INDIRECT_BRANCH: 1552 return ib_prctl_get(task); 1553 case PR_SPEC_L1D_FLUSH: 1554 return l1d_flush_prctl_get(task); 1555 default: 1556 return -ENODEV; 1557 } 1558 } 1559 1560 void x86_spec_ctrl_setup_ap(void) 1561 { 1562 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 1563 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1564 1565 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 1566 x86_amd_ssb_disable(); 1567 } 1568 1569 bool itlb_multihit_kvm_mitigation; 1570 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 1571 1572 #undef pr_fmt 1573 #define pr_fmt(fmt) "L1TF: " fmt 1574 1575 /* Default mitigation for L1TF-affected CPUs */ 1576 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 1577 #if IS_ENABLED(CONFIG_KVM_INTEL) 1578 EXPORT_SYMBOL_GPL(l1tf_mitigation); 1579 #endif 1580 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 1581 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 1582 1583 /* 1584 * These CPUs all support 44bits physical address space internally in the 1585 * cache but CPUID can report a smaller number of physical address bits. 1586 * 1587 * The L1TF mitigation uses the top most address bit for the inversion of 1588 * non present PTEs. When the installed memory reaches into the top most 1589 * address bit due to memory holes, which has been observed on machines 1590 * which report 36bits physical address bits and have 32G RAM installed, 1591 * then the mitigation range check in l1tf_select_mitigation() triggers. 1592 * This is a false positive because the mitigation is still possible due to 1593 * the fact that the cache uses 44bit internally. Use the cache bits 1594 * instead of the reported physical bits and adjust them on the affected 1595 * machines to 44bit if the reported bits are less than 44. 1596 */ 1597 static void override_cache_bits(struct cpuinfo_x86 *c) 1598 { 1599 if (c->x86 != 6) 1600 return; 1601 1602 switch (c->x86_model) { 1603 case INTEL_FAM6_NEHALEM: 1604 case INTEL_FAM6_WESTMERE: 1605 case INTEL_FAM6_SANDYBRIDGE: 1606 case INTEL_FAM6_IVYBRIDGE: 1607 case INTEL_FAM6_HASWELL: 1608 case INTEL_FAM6_HASWELL_L: 1609 case INTEL_FAM6_HASWELL_G: 1610 case INTEL_FAM6_BROADWELL: 1611 case INTEL_FAM6_BROADWELL_G: 1612 case INTEL_FAM6_SKYLAKE_L: 1613 case INTEL_FAM6_SKYLAKE: 1614 case INTEL_FAM6_KABYLAKE_L: 1615 case INTEL_FAM6_KABYLAKE: 1616 if (c->x86_cache_bits < 44) 1617 c->x86_cache_bits = 44; 1618 break; 1619 } 1620 } 1621 1622 static void __init l1tf_select_mitigation(void) 1623 { 1624 u64 half_pa; 1625 1626 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 1627 return; 1628 1629 if (cpu_mitigations_off()) 1630 l1tf_mitigation = L1TF_MITIGATION_OFF; 1631 else if (cpu_mitigations_auto_nosmt()) 1632 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 1633 1634 override_cache_bits(&boot_cpu_data); 1635 1636 switch (l1tf_mitigation) { 1637 case L1TF_MITIGATION_OFF: 1638 case L1TF_MITIGATION_FLUSH_NOWARN: 1639 case L1TF_MITIGATION_FLUSH: 1640 break; 1641 case L1TF_MITIGATION_FLUSH_NOSMT: 1642 case L1TF_MITIGATION_FULL: 1643 cpu_smt_disable(false); 1644 break; 1645 case L1TF_MITIGATION_FULL_FORCE: 1646 cpu_smt_disable(true); 1647 break; 1648 } 1649 1650 #if CONFIG_PGTABLE_LEVELS == 2 1651 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 1652 return; 1653 #endif 1654 1655 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 1656 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 1657 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 1658 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 1659 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 1660 half_pa); 1661 pr_info("However, doing so will make a part of your RAM unusable.\n"); 1662 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 1663 return; 1664 } 1665 1666 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 1667 } 1668 1669 static int __init l1tf_cmdline(char *str) 1670 { 1671 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 1672 return 0; 1673 1674 if (!str) 1675 return -EINVAL; 1676 1677 if (!strcmp(str, "off")) 1678 l1tf_mitigation = L1TF_MITIGATION_OFF; 1679 else if (!strcmp(str, "flush,nowarn")) 1680 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 1681 else if (!strcmp(str, "flush")) 1682 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 1683 else if (!strcmp(str, "flush,nosmt")) 1684 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 1685 else if (!strcmp(str, "full")) 1686 l1tf_mitigation = L1TF_MITIGATION_FULL; 1687 else if (!strcmp(str, "full,force")) 1688 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 1689 1690 return 0; 1691 } 1692 early_param("l1tf", l1tf_cmdline); 1693 1694 #undef pr_fmt 1695 #define pr_fmt(fmt) fmt 1696 1697 #ifdef CONFIG_SYSFS 1698 1699 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 1700 1701 #if IS_ENABLED(CONFIG_KVM_INTEL) 1702 static const char * const l1tf_vmx_states[] = { 1703 [VMENTER_L1D_FLUSH_AUTO] = "auto", 1704 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 1705 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 1706 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 1707 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 1708 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 1709 }; 1710 1711 static ssize_t l1tf_show_state(char *buf) 1712 { 1713 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 1714 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); 1715 1716 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 1717 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 1718 sched_smt_active())) { 1719 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 1720 l1tf_vmx_states[l1tf_vmx_mitigation]); 1721 } 1722 1723 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 1724 l1tf_vmx_states[l1tf_vmx_mitigation], 1725 sched_smt_active() ? "vulnerable" : "disabled"); 1726 } 1727 1728 static ssize_t itlb_multihit_show_state(char *buf) 1729 { 1730 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 1731 !boot_cpu_has(X86_FEATURE_VMX)) 1732 return sprintf(buf, "KVM: Mitigation: VMX unsupported\n"); 1733 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 1734 return sprintf(buf, "KVM: Mitigation: VMX disabled\n"); 1735 else if (itlb_multihit_kvm_mitigation) 1736 return sprintf(buf, "KVM: Mitigation: Split huge pages\n"); 1737 else 1738 return sprintf(buf, "KVM: Vulnerable\n"); 1739 } 1740 #else 1741 static ssize_t l1tf_show_state(char *buf) 1742 { 1743 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); 1744 } 1745 1746 static ssize_t itlb_multihit_show_state(char *buf) 1747 { 1748 return sprintf(buf, "Processor vulnerable\n"); 1749 } 1750 #endif 1751 1752 static ssize_t mds_show_state(char *buf) 1753 { 1754 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1755 return sprintf(buf, "%s; SMT Host state unknown\n", 1756 mds_strings[mds_mitigation]); 1757 } 1758 1759 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 1760 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 1761 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 1762 sched_smt_active() ? "mitigated" : "disabled")); 1763 } 1764 1765 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 1766 sched_smt_active() ? "vulnerable" : "disabled"); 1767 } 1768 1769 static ssize_t tsx_async_abort_show_state(char *buf) 1770 { 1771 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 1772 (taa_mitigation == TAA_MITIGATION_OFF)) 1773 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]); 1774 1775 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1776 return sprintf(buf, "%s; SMT Host state unknown\n", 1777 taa_strings[taa_mitigation]); 1778 } 1779 1780 return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 1781 sched_smt_active() ? "vulnerable" : "disabled"); 1782 } 1783 1784 static char *stibp_state(void) 1785 { 1786 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) 1787 return ""; 1788 1789 switch (spectre_v2_user_stibp) { 1790 case SPECTRE_V2_USER_NONE: 1791 return ", STIBP: disabled"; 1792 case SPECTRE_V2_USER_STRICT: 1793 return ", STIBP: forced"; 1794 case SPECTRE_V2_USER_STRICT_PREFERRED: 1795 return ", STIBP: always-on"; 1796 case SPECTRE_V2_USER_PRCTL: 1797 case SPECTRE_V2_USER_SECCOMP: 1798 if (static_key_enabled(&switch_to_cond_stibp)) 1799 return ", STIBP: conditional"; 1800 } 1801 return ""; 1802 } 1803 1804 static char *ibpb_state(void) 1805 { 1806 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1807 if (static_key_enabled(&switch_mm_always_ibpb)) 1808 return ", IBPB: always-on"; 1809 if (static_key_enabled(&switch_mm_cond_ibpb)) 1810 return ", IBPB: conditional"; 1811 return ", IBPB: disabled"; 1812 } 1813 return ""; 1814 } 1815 1816 static ssize_t spectre_v2_show_state(char *buf) 1817 { 1818 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 1819 return sprintf(buf, "Vulnerable: LFENCE\n"); 1820 1821 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1822 return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 1823 1824 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1825 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1826 return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 1827 1828 return sprintf(buf, "%s%s%s%s%s%s\n", 1829 spectre_v2_strings[spectre_v2_enabled], 1830 ibpb_state(), 1831 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 1832 stibp_state(), 1833 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 1834 spectre_v2_module_string()); 1835 } 1836 1837 static ssize_t srbds_show_state(char *buf) 1838 { 1839 return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]); 1840 } 1841 1842 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 1843 char *buf, unsigned int bug) 1844 { 1845 if (!boot_cpu_has_bug(bug)) 1846 return sprintf(buf, "Not affected\n"); 1847 1848 switch (bug) { 1849 case X86_BUG_CPU_MELTDOWN: 1850 if (boot_cpu_has(X86_FEATURE_PTI)) 1851 return sprintf(buf, "Mitigation: PTI\n"); 1852 1853 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 1854 return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 1855 1856 break; 1857 1858 case X86_BUG_SPECTRE_V1: 1859 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1860 1861 case X86_BUG_SPECTRE_V2: 1862 return spectre_v2_show_state(buf); 1863 1864 case X86_BUG_SPEC_STORE_BYPASS: 1865 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); 1866 1867 case X86_BUG_L1TF: 1868 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 1869 return l1tf_show_state(buf); 1870 break; 1871 1872 case X86_BUG_MDS: 1873 return mds_show_state(buf); 1874 1875 case X86_BUG_TAA: 1876 return tsx_async_abort_show_state(buf); 1877 1878 case X86_BUG_ITLB_MULTIHIT: 1879 return itlb_multihit_show_state(buf); 1880 1881 case X86_BUG_SRBDS: 1882 return srbds_show_state(buf); 1883 1884 default: 1885 break; 1886 } 1887 1888 return sprintf(buf, "Vulnerable\n"); 1889 } 1890 1891 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 1892 { 1893 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 1894 } 1895 1896 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 1897 { 1898 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 1899 } 1900 1901 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 1902 { 1903 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 1904 } 1905 1906 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 1907 { 1908 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 1909 } 1910 1911 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 1912 { 1913 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 1914 } 1915 1916 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 1917 { 1918 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 1919 } 1920 1921 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 1922 { 1923 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 1924 } 1925 1926 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 1927 { 1928 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 1929 } 1930 1931 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 1932 { 1933 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 1934 } 1935 #endif 1936