1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 20 #include <asm/spec-ctrl.h> 21 #include <asm/cmdline.h> 22 #include <asm/bugs.h> 23 #include <asm/processor.h> 24 #include <asm/processor-flags.h> 25 #include <asm/fpu/api.h> 26 #include <asm/msr.h> 27 #include <asm/vmx.h> 28 #include <asm/paravirt.h> 29 #include <asm/intel-family.h> 30 #include <asm/e820/api.h> 31 #include <asm/hypervisor.h> 32 #include <asm/tlbflush.h> 33 #include <asm/cpu.h> 34 35 #include "cpu.h" 36 37 static void __init spectre_v1_select_mitigation(void); 38 static void __init spectre_v2_select_mitigation(void); 39 static void __init retbleed_select_mitigation(void); 40 static void __init spectre_v2_user_select_mitigation(void); 41 static void __init ssb_select_mitigation(void); 42 static void __init l1tf_select_mitigation(void); 43 static void __init mds_select_mitigation(void); 44 static void __init md_clear_update_mitigation(void); 45 static void __init md_clear_select_mitigation(void); 46 static void __init taa_select_mitigation(void); 47 static void __init mmio_select_mitigation(void); 48 static void __init srbds_select_mitigation(void); 49 static void __init l1d_flush_select_mitigation(void); 50 static void __init srso_select_mitigation(void); 51 static void __init gds_select_mitigation(void); 52 53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 54 u64 x86_spec_ctrl_base; 55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 56 57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 59 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); 60 61 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 62 EXPORT_SYMBOL_GPL(x86_pred_cmd); 63 64 static DEFINE_MUTEX(spec_ctrl_mutex); 65 66 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 67 68 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 69 static void update_spec_ctrl(u64 val) 70 { 71 this_cpu_write(x86_spec_ctrl_current, val); 72 wrmsrl(MSR_IA32_SPEC_CTRL, val); 73 } 74 75 /* 76 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 77 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 78 */ 79 void update_spec_ctrl_cond(u64 val) 80 { 81 if (this_cpu_read(x86_spec_ctrl_current) == val) 82 return; 83 84 this_cpu_write(x86_spec_ctrl_current, val); 85 86 /* 87 * When KERNEL_IBRS this MSR is written on return-to-user, unless 88 * forced the update can be delayed until that time. 89 */ 90 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 91 wrmsrl(MSR_IA32_SPEC_CTRL, val); 92 } 93 94 noinstr u64 spec_ctrl_current(void) 95 { 96 return this_cpu_read(x86_spec_ctrl_current); 97 } 98 EXPORT_SYMBOL_GPL(spec_ctrl_current); 99 100 /* 101 * AMD specific MSR info for Speculative Store Bypass control. 102 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 103 */ 104 u64 __ro_after_init x86_amd_ls_cfg_base; 105 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 106 107 /* Control conditional STIBP in switch_to() */ 108 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 109 /* Control conditional IBPB in switch_mm() */ 110 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 111 /* Control unconditional IBPB in switch_mm() */ 112 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 113 114 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 115 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 116 EXPORT_SYMBOL_GPL(mds_idle_clear); 117 118 /* 119 * Controls whether l1d flush based mitigations are enabled, 120 * based on hw features and admin setting via boot parameter 121 * defaults to false 122 */ 123 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 124 125 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ 126 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); 127 EXPORT_SYMBOL_GPL(mmio_stale_data_clear); 128 129 void __init cpu_select_mitigations(void) 130 { 131 /* 132 * Read the SPEC_CTRL MSR to account for reserved bits which may 133 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 134 * init code as it is not enumerated and depends on the family. 135 */ 136 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 137 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 138 139 /* 140 * Previously running kernel (kexec), may have some controls 141 * turned ON. Clear them and let the mitigations setup below 142 * rediscover them based on configuration. 143 */ 144 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 145 } 146 147 /* Select the proper CPU mitigations before patching alternatives: */ 148 spectre_v1_select_mitigation(); 149 spectre_v2_select_mitigation(); 150 /* 151 * retbleed_select_mitigation() relies on the state set by 152 * spectre_v2_select_mitigation(); specifically it wants to know about 153 * spectre_v2=ibrs. 154 */ 155 retbleed_select_mitigation(); 156 /* 157 * spectre_v2_user_select_mitigation() relies on the state set by 158 * retbleed_select_mitigation(); specifically the STIBP selection is 159 * forced for UNRET or IBPB. 160 */ 161 spectre_v2_user_select_mitigation(); 162 ssb_select_mitigation(); 163 l1tf_select_mitigation(); 164 md_clear_select_mitigation(); 165 srbds_select_mitigation(); 166 l1d_flush_select_mitigation(); 167 168 /* 169 * srso_select_mitigation() depends and must run after 170 * retbleed_select_mitigation(). 171 */ 172 srso_select_mitigation(); 173 gds_select_mitigation(); 174 } 175 176 /* 177 * NOTE: This function is *only* called for SVM, since Intel uses 178 * MSR_IA32_SPEC_CTRL for SSBD. 179 */ 180 void 181 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 182 { 183 u64 guestval, hostval; 184 struct thread_info *ti = current_thread_info(); 185 186 /* 187 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 188 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 189 */ 190 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 191 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 192 return; 193 194 /* 195 * If the host has SSBD mitigation enabled, force it in the host's 196 * virtual MSR value. If its not permanently enabled, evaluate 197 * current's TIF_SSBD thread flag. 198 */ 199 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 200 hostval = SPEC_CTRL_SSBD; 201 else 202 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 203 204 /* Sanitize the guest value */ 205 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 206 207 if (hostval != guestval) { 208 unsigned long tif; 209 210 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 211 ssbd_spec_ctrl_to_tif(hostval); 212 213 speculation_ctrl_update(tif); 214 } 215 } 216 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 217 218 static void x86_amd_ssb_disable(void) 219 { 220 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 221 222 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 223 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 224 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 225 wrmsrl(MSR_AMD64_LS_CFG, msrval); 226 } 227 228 #undef pr_fmt 229 #define pr_fmt(fmt) "MDS: " fmt 230 231 /* Default mitigation for MDS-affected CPUs */ 232 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 233 static bool mds_nosmt __ro_after_init = false; 234 235 static const char * const mds_strings[] = { 236 [MDS_MITIGATION_OFF] = "Vulnerable", 237 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 238 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 239 }; 240 241 static void __init mds_select_mitigation(void) 242 { 243 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 244 mds_mitigation = MDS_MITIGATION_OFF; 245 return; 246 } 247 248 if (mds_mitigation == MDS_MITIGATION_FULL) { 249 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 250 mds_mitigation = MDS_MITIGATION_VMWERV; 251 252 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 253 254 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 255 (mds_nosmt || cpu_mitigations_auto_nosmt())) 256 cpu_smt_disable(false); 257 } 258 } 259 260 static int __init mds_cmdline(char *str) 261 { 262 if (!boot_cpu_has_bug(X86_BUG_MDS)) 263 return 0; 264 265 if (!str) 266 return -EINVAL; 267 268 if (!strcmp(str, "off")) 269 mds_mitigation = MDS_MITIGATION_OFF; 270 else if (!strcmp(str, "full")) 271 mds_mitigation = MDS_MITIGATION_FULL; 272 else if (!strcmp(str, "full,nosmt")) { 273 mds_mitigation = MDS_MITIGATION_FULL; 274 mds_nosmt = true; 275 } 276 277 return 0; 278 } 279 early_param("mds", mds_cmdline); 280 281 #undef pr_fmt 282 #define pr_fmt(fmt) "TAA: " fmt 283 284 enum taa_mitigations { 285 TAA_MITIGATION_OFF, 286 TAA_MITIGATION_UCODE_NEEDED, 287 TAA_MITIGATION_VERW, 288 TAA_MITIGATION_TSX_DISABLED, 289 }; 290 291 /* Default mitigation for TAA-affected CPUs */ 292 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 293 static bool taa_nosmt __ro_after_init; 294 295 static const char * const taa_strings[] = { 296 [TAA_MITIGATION_OFF] = "Vulnerable", 297 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 298 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 299 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 300 }; 301 302 static void __init taa_select_mitigation(void) 303 { 304 u64 ia32_cap; 305 306 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 307 taa_mitigation = TAA_MITIGATION_OFF; 308 return; 309 } 310 311 /* TSX previously disabled by tsx=off */ 312 if (!boot_cpu_has(X86_FEATURE_RTM)) { 313 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 314 return; 315 } 316 317 if (cpu_mitigations_off()) { 318 taa_mitigation = TAA_MITIGATION_OFF; 319 return; 320 } 321 322 /* 323 * TAA mitigation via VERW is turned off if both 324 * tsx_async_abort=off and mds=off are specified. 325 */ 326 if (taa_mitigation == TAA_MITIGATION_OFF && 327 mds_mitigation == MDS_MITIGATION_OFF) 328 return; 329 330 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 331 taa_mitigation = TAA_MITIGATION_VERW; 332 else 333 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 334 335 /* 336 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 337 * A microcode update fixes this behavior to clear CPU buffers. It also 338 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 339 * ARCH_CAP_TSX_CTRL_MSR bit. 340 * 341 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 342 * update is required. 343 */ 344 ia32_cap = x86_read_arch_cap_msr(); 345 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 346 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 347 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 348 349 /* 350 * TSX is enabled, select alternate mitigation for TAA which is 351 * the same as MDS. Enable MDS static branch to clear CPU buffers. 352 * 353 * For guests that can't determine whether the correct microcode is 354 * present on host, enable the mitigation for UCODE_NEEDED as well. 355 */ 356 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 357 358 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 359 cpu_smt_disable(false); 360 } 361 362 static int __init tsx_async_abort_parse_cmdline(char *str) 363 { 364 if (!boot_cpu_has_bug(X86_BUG_TAA)) 365 return 0; 366 367 if (!str) 368 return -EINVAL; 369 370 if (!strcmp(str, "off")) { 371 taa_mitigation = TAA_MITIGATION_OFF; 372 } else if (!strcmp(str, "full")) { 373 taa_mitigation = TAA_MITIGATION_VERW; 374 } else if (!strcmp(str, "full,nosmt")) { 375 taa_mitigation = TAA_MITIGATION_VERW; 376 taa_nosmt = true; 377 } 378 379 return 0; 380 } 381 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 382 383 #undef pr_fmt 384 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 385 386 enum mmio_mitigations { 387 MMIO_MITIGATION_OFF, 388 MMIO_MITIGATION_UCODE_NEEDED, 389 MMIO_MITIGATION_VERW, 390 }; 391 392 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 393 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW; 394 static bool mmio_nosmt __ro_after_init = false; 395 396 static const char * const mmio_strings[] = { 397 [MMIO_MITIGATION_OFF] = "Vulnerable", 398 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 399 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 400 }; 401 402 static void __init mmio_select_mitigation(void) 403 { 404 u64 ia32_cap; 405 406 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || 407 boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || 408 cpu_mitigations_off()) { 409 mmio_mitigation = MMIO_MITIGATION_OFF; 410 return; 411 } 412 413 if (mmio_mitigation == MMIO_MITIGATION_OFF) 414 return; 415 416 ia32_cap = x86_read_arch_cap_msr(); 417 418 /* 419 * Enable CPU buffer clear mitigation for host and VMM, if also affected 420 * by MDS or TAA. Otherwise, enable mitigation for VMM only. 421 */ 422 if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && 423 boot_cpu_has(X86_FEATURE_RTM))) 424 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 425 else 426 static_branch_enable(&mmio_stale_data_clear); 427 428 /* 429 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 430 * be propagated to uncore buffers, clearing the Fill buffers on idle 431 * is required irrespective of SMT state. 432 */ 433 if (!(ia32_cap & ARCH_CAP_FBSDP_NO)) 434 static_branch_enable(&mds_idle_clear); 435 436 /* 437 * Check if the system has the right microcode. 438 * 439 * CPU Fill buffer clear mitigation is enumerated by either an explicit 440 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 441 * affected systems. 442 */ 443 if ((ia32_cap & ARCH_CAP_FB_CLEAR) || 444 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 445 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 446 !(ia32_cap & ARCH_CAP_MDS_NO))) 447 mmio_mitigation = MMIO_MITIGATION_VERW; 448 else 449 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 450 451 if (mmio_nosmt || cpu_mitigations_auto_nosmt()) 452 cpu_smt_disable(false); 453 } 454 455 static int __init mmio_stale_data_parse_cmdline(char *str) 456 { 457 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 458 return 0; 459 460 if (!str) 461 return -EINVAL; 462 463 if (!strcmp(str, "off")) { 464 mmio_mitigation = MMIO_MITIGATION_OFF; 465 } else if (!strcmp(str, "full")) { 466 mmio_mitigation = MMIO_MITIGATION_VERW; 467 } else if (!strcmp(str, "full,nosmt")) { 468 mmio_mitigation = MMIO_MITIGATION_VERW; 469 mmio_nosmt = true; 470 } 471 472 return 0; 473 } 474 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 475 476 #undef pr_fmt 477 #define pr_fmt(fmt) "" fmt 478 479 static void __init md_clear_update_mitigation(void) 480 { 481 if (cpu_mitigations_off()) 482 return; 483 484 if (!boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) 485 goto out; 486 487 /* 488 * X86_FEATURE_CLEAR_CPU_BUF is now enabled. Update MDS, TAA and MMIO 489 * Stale Data mitigation, if necessary. 490 */ 491 if (mds_mitigation == MDS_MITIGATION_OFF && 492 boot_cpu_has_bug(X86_BUG_MDS)) { 493 mds_mitigation = MDS_MITIGATION_FULL; 494 mds_select_mitigation(); 495 } 496 if (taa_mitigation == TAA_MITIGATION_OFF && 497 boot_cpu_has_bug(X86_BUG_TAA)) { 498 taa_mitigation = TAA_MITIGATION_VERW; 499 taa_select_mitigation(); 500 } 501 if (mmio_mitigation == MMIO_MITIGATION_OFF && 502 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 503 mmio_mitigation = MMIO_MITIGATION_VERW; 504 mmio_select_mitigation(); 505 } 506 out: 507 if (boot_cpu_has_bug(X86_BUG_MDS)) 508 pr_info("MDS: %s\n", mds_strings[mds_mitigation]); 509 if (boot_cpu_has_bug(X86_BUG_TAA)) 510 pr_info("TAA: %s\n", taa_strings[taa_mitigation]); 511 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 512 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); 513 else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 514 pr_info("MMIO Stale Data: Unknown: No mitigations\n"); 515 } 516 517 static void __init md_clear_select_mitigation(void) 518 { 519 mds_select_mitigation(); 520 taa_select_mitigation(); 521 mmio_select_mitigation(); 522 523 /* 524 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update 525 * and print their mitigation after MDS, TAA and MMIO Stale Data 526 * mitigation selection is done. 527 */ 528 md_clear_update_mitigation(); 529 } 530 531 #undef pr_fmt 532 #define pr_fmt(fmt) "SRBDS: " fmt 533 534 enum srbds_mitigations { 535 SRBDS_MITIGATION_OFF, 536 SRBDS_MITIGATION_UCODE_NEEDED, 537 SRBDS_MITIGATION_FULL, 538 SRBDS_MITIGATION_TSX_OFF, 539 SRBDS_MITIGATION_HYPERVISOR, 540 }; 541 542 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL; 543 544 static const char * const srbds_strings[] = { 545 [SRBDS_MITIGATION_OFF] = "Vulnerable", 546 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 547 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 548 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 549 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 550 }; 551 552 static bool srbds_off; 553 554 void update_srbds_msr(void) 555 { 556 u64 mcu_ctrl; 557 558 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 559 return; 560 561 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 562 return; 563 564 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 565 return; 566 567 /* 568 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 569 * being disabled and it hasn't received the SRBDS MSR microcode. 570 */ 571 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 572 return; 573 574 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 575 576 switch (srbds_mitigation) { 577 case SRBDS_MITIGATION_OFF: 578 case SRBDS_MITIGATION_TSX_OFF: 579 mcu_ctrl |= RNGDS_MITG_DIS; 580 break; 581 case SRBDS_MITIGATION_FULL: 582 mcu_ctrl &= ~RNGDS_MITG_DIS; 583 break; 584 default: 585 break; 586 } 587 588 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 589 } 590 591 static void __init srbds_select_mitigation(void) 592 { 593 u64 ia32_cap; 594 595 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 596 return; 597 598 /* 599 * Check to see if this is one of the MDS_NO systems supporting TSX that 600 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 601 * by Processor MMIO Stale Data vulnerability. 602 */ 603 ia32_cap = x86_read_arch_cap_msr(); 604 if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 605 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 606 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 607 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 608 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 609 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 610 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 611 else if (cpu_mitigations_off() || srbds_off) 612 srbds_mitigation = SRBDS_MITIGATION_OFF; 613 614 update_srbds_msr(); 615 pr_info("%s\n", srbds_strings[srbds_mitigation]); 616 } 617 618 static int __init srbds_parse_cmdline(char *str) 619 { 620 if (!str) 621 return -EINVAL; 622 623 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 624 return 0; 625 626 srbds_off = !strcmp(str, "off"); 627 return 0; 628 } 629 early_param("srbds", srbds_parse_cmdline); 630 631 #undef pr_fmt 632 #define pr_fmt(fmt) "L1D Flush : " fmt 633 634 enum l1d_flush_mitigations { 635 L1D_FLUSH_OFF = 0, 636 L1D_FLUSH_ON, 637 }; 638 639 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 640 641 static void __init l1d_flush_select_mitigation(void) 642 { 643 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 644 return; 645 646 static_branch_enable(&switch_mm_cond_l1d_flush); 647 pr_info("Conditional flush on switch_mm() enabled\n"); 648 } 649 650 static int __init l1d_flush_parse_cmdline(char *str) 651 { 652 if (!strcmp(str, "on")) 653 l1d_flush_mitigation = L1D_FLUSH_ON; 654 655 return 0; 656 } 657 early_param("l1d_flush", l1d_flush_parse_cmdline); 658 659 #undef pr_fmt 660 #define pr_fmt(fmt) "GDS: " fmt 661 662 enum gds_mitigations { 663 GDS_MITIGATION_OFF, 664 GDS_MITIGATION_UCODE_NEEDED, 665 GDS_MITIGATION_FORCE, 666 GDS_MITIGATION_FULL, 667 GDS_MITIGATION_FULL_LOCKED, 668 GDS_MITIGATION_HYPERVISOR, 669 }; 670 671 #if IS_ENABLED(CONFIG_GDS_FORCE_MITIGATION) 672 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE; 673 #else 674 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL; 675 #endif 676 677 static const char * const gds_strings[] = { 678 [GDS_MITIGATION_OFF] = "Vulnerable", 679 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 680 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 681 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 682 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 683 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 684 }; 685 686 bool gds_ucode_mitigated(void) 687 { 688 return (gds_mitigation == GDS_MITIGATION_FULL || 689 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 690 } 691 EXPORT_SYMBOL_GPL(gds_ucode_mitigated); 692 693 void update_gds_msr(void) 694 { 695 u64 mcu_ctrl_after; 696 u64 mcu_ctrl; 697 698 switch (gds_mitigation) { 699 case GDS_MITIGATION_OFF: 700 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 701 mcu_ctrl |= GDS_MITG_DIS; 702 break; 703 case GDS_MITIGATION_FULL_LOCKED: 704 /* 705 * The LOCKED state comes from the boot CPU. APs might not have 706 * the same state. Make sure the mitigation is enabled on all 707 * CPUs. 708 */ 709 case GDS_MITIGATION_FULL: 710 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 711 mcu_ctrl &= ~GDS_MITG_DIS; 712 break; 713 case GDS_MITIGATION_FORCE: 714 case GDS_MITIGATION_UCODE_NEEDED: 715 case GDS_MITIGATION_HYPERVISOR: 716 return; 717 } 718 719 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 720 721 /* 722 * Check to make sure that the WRMSR value was not ignored. Writes to 723 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 724 * processor was not. 725 */ 726 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 727 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 728 } 729 730 static void __init gds_select_mitigation(void) 731 { 732 u64 mcu_ctrl; 733 734 if (!boot_cpu_has_bug(X86_BUG_GDS)) 735 return; 736 737 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 738 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 739 goto out; 740 } 741 742 if (cpu_mitigations_off()) 743 gds_mitigation = GDS_MITIGATION_OFF; 744 /* Will verify below that mitigation _can_ be disabled */ 745 746 /* No microcode */ 747 if (!(x86_read_arch_cap_msr() & ARCH_CAP_GDS_CTRL)) { 748 if (gds_mitigation == GDS_MITIGATION_FORCE) { 749 /* 750 * This only needs to be done on the boot CPU so do it 751 * here rather than in update_gds_msr() 752 */ 753 setup_clear_cpu_cap(X86_FEATURE_AVX); 754 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 755 } else { 756 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 757 } 758 goto out; 759 } 760 761 /* Microcode has mitigation, use it */ 762 if (gds_mitigation == GDS_MITIGATION_FORCE) 763 gds_mitigation = GDS_MITIGATION_FULL; 764 765 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 766 if (mcu_ctrl & GDS_MITG_LOCKED) { 767 if (gds_mitigation == GDS_MITIGATION_OFF) 768 pr_warn("Mitigation locked. Disable failed.\n"); 769 770 /* 771 * The mitigation is selected from the boot CPU. All other CPUs 772 * _should_ have the same state. If the boot CPU isn't locked 773 * but others are then update_gds_msr() will WARN() of the state 774 * mismatch. If the boot CPU is locked update_gds_msr() will 775 * ensure the other CPUs have the mitigation enabled. 776 */ 777 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 778 } 779 780 update_gds_msr(); 781 out: 782 pr_info("%s\n", gds_strings[gds_mitigation]); 783 } 784 785 static int __init gds_parse_cmdline(char *str) 786 { 787 if (!str) 788 return -EINVAL; 789 790 if (!boot_cpu_has_bug(X86_BUG_GDS)) 791 return 0; 792 793 if (!strcmp(str, "off")) 794 gds_mitigation = GDS_MITIGATION_OFF; 795 else if (!strcmp(str, "force")) 796 gds_mitigation = GDS_MITIGATION_FORCE; 797 798 return 0; 799 } 800 early_param("gather_data_sampling", gds_parse_cmdline); 801 802 #undef pr_fmt 803 #define pr_fmt(fmt) "Spectre V1 : " fmt 804 805 enum spectre_v1_mitigation { 806 SPECTRE_V1_MITIGATION_NONE, 807 SPECTRE_V1_MITIGATION_AUTO, 808 }; 809 810 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 811 SPECTRE_V1_MITIGATION_AUTO; 812 813 static const char * const spectre_v1_strings[] = { 814 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 815 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 816 }; 817 818 /* 819 * Does SMAP provide full mitigation against speculative kernel access to 820 * userspace? 821 */ 822 static bool smap_works_speculatively(void) 823 { 824 if (!boot_cpu_has(X86_FEATURE_SMAP)) 825 return false; 826 827 /* 828 * On CPUs which are vulnerable to Meltdown, SMAP does not 829 * prevent speculative access to user data in the L1 cache. 830 * Consider SMAP to be non-functional as a mitigation on these 831 * CPUs. 832 */ 833 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 834 return false; 835 836 return true; 837 } 838 839 static void __init spectre_v1_select_mitigation(void) 840 { 841 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 842 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 843 return; 844 } 845 846 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 847 /* 848 * With Spectre v1, a user can speculatively control either 849 * path of a conditional swapgs with a user-controlled GS 850 * value. The mitigation is to add lfences to both code paths. 851 * 852 * If FSGSBASE is enabled, the user can put a kernel address in 853 * GS, in which case SMAP provides no protection. 854 * 855 * If FSGSBASE is disabled, the user can only put a user space 856 * address in GS. That makes an attack harder, but still 857 * possible if there's no SMAP protection. 858 */ 859 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 860 !smap_works_speculatively()) { 861 /* 862 * Mitigation can be provided from SWAPGS itself or 863 * PTI as the CR3 write in the Meltdown mitigation 864 * is serializing. 865 * 866 * If neither is there, mitigate with an LFENCE to 867 * stop speculation through swapgs. 868 */ 869 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 870 !boot_cpu_has(X86_FEATURE_PTI)) 871 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 872 873 /* 874 * Enable lfences in the kernel entry (non-swapgs) 875 * paths, to prevent user entry from speculatively 876 * skipping swapgs. 877 */ 878 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 879 } 880 } 881 882 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 883 } 884 885 static int __init nospectre_v1_cmdline(char *str) 886 { 887 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 888 return 0; 889 } 890 early_param("nospectre_v1", nospectre_v1_cmdline); 891 892 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 893 894 #undef pr_fmt 895 #define pr_fmt(fmt) "RETBleed: " fmt 896 897 enum retbleed_mitigation { 898 RETBLEED_MITIGATION_NONE, 899 RETBLEED_MITIGATION_UNRET, 900 RETBLEED_MITIGATION_IBPB, 901 RETBLEED_MITIGATION_IBRS, 902 RETBLEED_MITIGATION_EIBRS, 903 RETBLEED_MITIGATION_STUFF, 904 }; 905 906 enum retbleed_mitigation_cmd { 907 RETBLEED_CMD_OFF, 908 RETBLEED_CMD_AUTO, 909 RETBLEED_CMD_UNRET, 910 RETBLEED_CMD_IBPB, 911 RETBLEED_CMD_STUFF, 912 }; 913 914 static const char * const retbleed_strings[] = { 915 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 916 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 917 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 918 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 919 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 920 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 921 }; 922 923 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 924 RETBLEED_MITIGATION_NONE; 925 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = 926 RETBLEED_CMD_AUTO; 927 928 static int __ro_after_init retbleed_nosmt = false; 929 930 static int __init retbleed_parse_cmdline(char *str) 931 { 932 if (!str) 933 return -EINVAL; 934 935 while (str) { 936 char *next = strchr(str, ','); 937 if (next) { 938 *next = 0; 939 next++; 940 } 941 942 if (!strcmp(str, "off")) { 943 retbleed_cmd = RETBLEED_CMD_OFF; 944 } else if (!strcmp(str, "auto")) { 945 retbleed_cmd = RETBLEED_CMD_AUTO; 946 } else if (!strcmp(str, "unret")) { 947 retbleed_cmd = RETBLEED_CMD_UNRET; 948 } else if (!strcmp(str, "ibpb")) { 949 retbleed_cmd = RETBLEED_CMD_IBPB; 950 } else if (!strcmp(str, "stuff")) { 951 retbleed_cmd = RETBLEED_CMD_STUFF; 952 } else if (!strcmp(str, "nosmt")) { 953 retbleed_nosmt = true; 954 } else if (!strcmp(str, "force")) { 955 setup_force_cpu_bug(X86_BUG_RETBLEED); 956 } else { 957 pr_err("Ignoring unknown retbleed option (%s).", str); 958 } 959 960 str = next; 961 } 962 963 return 0; 964 } 965 early_param("retbleed", retbleed_parse_cmdline); 966 967 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 968 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 969 970 static void __init retbleed_select_mitigation(void) 971 { 972 bool mitigate_smt = false; 973 974 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) 975 return; 976 977 switch (retbleed_cmd) { 978 case RETBLEED_CMD_OFF: 979 return; 980 981 case RETBLEED_CMD_UNRET: 982 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) { 983 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 984 } else { 985 pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n"); 986 goto do_cmd_auto; 987 } 988 break; 989 990 case RETBLEED_CMD_IBPB: 991 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 992 pr_err("WARNING: CPU does not support IBPB.\n"); 993 goto do_cmd_auto; 994 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { 995 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 996 } else { 997 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n"); 998 goto do_cmd_auto; 999 } 1000 break; 1001 1002 case RETBLEED_CMD_STUFF: 1003 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) && 1004 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { 1005 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1006 1007 } else { 1008 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING)) 1009 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); 1010 else 1011 pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n"); 1012 1013 goto do_cmd_auto; 1014 } 1015 break; 1016 1017 do_cmd_auto: 1018 case RETBLEED_CMD_AUTO: 1019 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1020 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1021 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) 1022 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1023 else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB)) 1024 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1025 } 1026 1027 /* 1028 * The Intel mitigation (IBRS or eIBRS) was already selected in 1029 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will 1030 * be set accordingly below. 1031 */ 1032 1033 break; 1034 } 1035 1036 switch (retbleed_mitigation) { 1037 case RETBLEED_MITIGATION_UNRET: 1038 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1039 setup_force_cpu_cap(X86_FEATURE_UNRET); 1040 1041 x86_return_thunk = retbleed_return_thunk; 1042 1043 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1044 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1045 pr_err(RETBLEED_UNTRAIN_MSG); 1046 1047 mitigate_smt = true; 1048 break; 1049 1050 case RETBLEED_MITIGATION_IBPB: 1051 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1052 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1053 mitigate_smt = true; 1054 break; 1055 1056 case RETBLEED_MITIGATION_STUFF: 1057 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1058 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1059 1060 x86_return_thunk = call_depth_return_thunk; 1061 break; 1062 1063 default: 1064 break; 1065 } 1066 1067 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1068 (retbleed_nosmt || cpu_mitigations_auto_nosmt())) 1069 cpu_smt_disable(false); 1070 1071 /* 1072 * Let IBRS trump all on Intel without affecting the effects of the 1073 * retbleed= cmdline option except for call depth based stuffing 1074 */ 1075 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1076 switch (spectre_v2_enabled) { 1077 case SPECTRE_V2_IBRS: 1078 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1079 break; 1080 case SPECTRE_V2_EIBRS: 1081 case SPECTRE_V2_EIBRS_RETPOLINE: 1082 case SPECTRE_V2_EIBRS_LFENCE: 1083 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1084 break; 1085 default: 1086 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 1087 pr_err(RETBLEED_INTEL_MSG); 1088 } 1089 } 1090 1091 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1092 } 1093 1094 #undef pr_fmt 1095 #define pr_fmt(fmt) "Spectre V2 : " fmt 1096 1097 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1098 SPECTRE_V2_USER_NONE; 1099 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1100 SPECTRE_V2_USER_NONE; 1101 1102 #ifdef CONFIG_RETPOLINE 1103 static bool spectre_v2_bad_module; 1104 1105 bool retpoline_module_ok(bool has_retpoline) 1106 { 1107 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1108 return true; 1109 1110 pr_err("System may be vulnerable to spectre v2\n"); 1111 spectre_v2_bad_module = true; 1112 return false; 1113 } 1114 1115 static inline const char *spectre_v2_module_string(void) 1116 { 1117 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1118 } 1119 #else 1120 static inline const char *spectre_v2_module_string(void) { return ""; } 1121 #endif 1122 1123 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1124 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1125 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1126 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1127 1128 #ifdef CONFIG_BPF_SYSCALL 1129 void unpriv_ebpf_notify(int new_state) 1130 { 1131 if (new_state) 1132 return; 1133 1134 /* Unprivileged eBPF is enabled */ 1135 1136 switch (spectre_v2_enabled) { 1137 case SPECTRE_V2_EIBRS: 1138 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1139 break; 1140 case SPECTRE_V2_EIBRS_LFENCE: 1141 if (sched_smt_active()) 1142 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1143 break; 1144 default: 1145 break; 1146 } 1147 } 1148 #endif 1149 1150 static inline bool match_option(const char *arg, int arglen, const char *opt) 1151 { 1152 int len = strlen(opt); 1153 1154 return len == arglen && !strncmp(arg, opt, len); 1155 } 1156 1157 /* The kernel command line selection for spectre v2 */ 1158 enum spectre_v2_mitigation_cmd { 1159 SPECTRE_V2_CMD_NONE, 1160 SPECTRE_V2_CMD_AUTO, 1161 SPECTRE_V2_CMD_FORCE, 1162 SPECTRE_V2_CMD_RETPOLINE, 1163 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1164 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1165 SPECTRE_V2_CMD_EIBRS, 1166 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1167 SPECTRE_V2_CMD_EIBRS_LFENCE, 1168 SPECTRE_V2_CMD_IBRS, 1169 }; 1170 1171 enum spectre_v2_user_cmd { 1172 SPECTRE_V2_USER_CMD_NONE, 1173 SPECTRE_V2_USER_CMD_AUTO, 1174 SPECTRE_V2_USER_CMD_FORCE, 1175 SPECTRE_V2_USER_CMD_PRCTL, 1176 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1177 SPECTRE_V2_USER_CMD_SECCOMP, 1178 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1179 }; 1180 1181 static const char * const spectre_v2_user_strings[] = { 1182 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1183 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1184 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1185 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1186 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1187 }; 1188 1189 static const struct { 1190 const char *option; 1191 enum spectre_v2_user_cmd cmd; 1192 bool secure; 1193 } v2_user_options[] __initconst = { 1194 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 1195 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 1196 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 1197 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 1198 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 1199 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 1200 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 1201 }; 1202 1203 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 1204 { 1205 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1206 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 1207 } 1208 1209 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; 1210 1211 static enum spectre_v2_user_cmd __init 1212 spectre_v2_parse_user_cmdline(void) 1213 { 1214 char arg[20]; 1215 int ret, i; 1216 1217 switch (spectre_v2_cmd) { 1218 case SPECTRE_V2_CMD_NONE: 1219 return SPECTRE_V2_USER_CMD_NONE; 1220 case SPECTRE_V2_CMD_FORCE: 1221 return SPECTRE_V2_USER_CMD_FORCE; 1222 default: 1223 break; 1224 } 1225 1226 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 1227 arg, sizeof(arg)); 1228 if (ret < 0) 1229 return SPECTRE_V2_USER_CMD_AUTO; 1230 1231 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 1232 if (match_option(arg, ret, v2_user_options[i].option)) { 1233 spec_v2_user_print_cond(v2_user_options[i].option, 1234 v2_user_options[i].secure); 1235 return v2_user_options[i].cmd; 1236 } 1237 } 1238 1239 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 1240 return SPECTRE_V2_USER_CMD_AUTO; 1241 } 1242 1243 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1244 { 1245 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1246 } 1247 1248 static void __init 1249 spectre_v2_user_select_mitigation(void) 1250 { 1251 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 1252 bool smt_possible = IS_ENABLED(CONFIG_SMP); 1253 enum spectre_v2_user_cmd cmd; 1254 1255 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1256 return; 1257 1258 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 1259 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 1260 smt_possible = false; 1261 1262 cmd = spectre_v2_parse_user_cmdline(); 1263 switch (cmd) { 1264 case SPECTRE_V2_USER_CMD_NONE: 1265 goto set_mode; 1266 case SPECTRE_V2_USER_CMD_FORCE: 1267 mode = SPECTRE_V2_USER_STRICT; 1268 break; 1269 case SPECTRE_V2_USER_CMD_AUTO: 1270 case SPECTRE_V2_USER_CMD_PRCTL: 1271 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1272 mode = SPECTRE_V2_USER_PRCTL; 1273 break; 1274 case SPECTRE_V2_USER_CMD_SECCOMP: 1275 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1276 if (IS_ENABLED(CONFIG_SECCOMP)) 1277 mode = SPECTRE_V2_USER_SECCOMP; 1278 else 1279 mode = SPECTRE_V2_USER_PRCTL; 1280 break; 1281 } 1282 1283 /* Initialize Indirect Branch Prediction Barrier */ 1284 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1285 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 1286 1287 spectre_v2_user_ibpb = mode; 1288 switch (cmd) { 1289 case SPECTRE_V2_USER_CMD_NONE: 1290 break; 1291 case SPECTRE_V2_USER_CMD_FORCE: 1292 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1293 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1294 static_branch_enable(&switch_mm_always_ibpb); 1295 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1296 break; 1297 case SPECTRE_V2_USER_CMD_PRCTL: 1298 case SPECTRE_V2_USER_CMD_AUTO: 1299 case SPECTRE_V2_USER_CMD_SECCOMP: 1300 static_branch_enable(&switch_mm_cond_ibpb); 1301 break; 1302 } 1303 1304 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1305 static_key_enabled(&switch_mm_always_ibpb) ? 1306 "always-on" : "conditional"); 1307 } 1308 1309 /* 1310 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1311 * is not required. 1312 * 1313 * Intel's Enhanced IBRS also protects against cross-thread branch target 1314 * injection in user-mode as the IBRS bit remains always set which 1315 * implicitly enables cross-thread protections. However, in legacy IBRS 1316 * mode, the IBRS bit is set only on kernel entry and cleared on return 1317 * to userspace. AMD Automatic IBRS also does not protect userspace. 1318 * These modes therefore disable the implicit cross-thread protection, 1319 * so allow for STIBP to be selected in those cases. 1320 */ 1321 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1322 !smt_possible || 1323 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1324 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) 1325 return; 1326 1327 /* 1328 * At this point, an STIBP mode other than "off" has been set. 1329 * If STIBP support is not being forced, check if STIBP always-on 1330 * is preferred. 1331 */ 1332 if (mode != SPECTRE_V2_USER_STRICT && 1333 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1334 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1335 1336 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1337 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 1338 if (mode != SPECTRE_V2_USER_STRICT && 1339 mode != SPECTRE_V2_USER_STRICT_PREFERRED) 1340 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1341 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1342 } 1343 1344 spectre_v2_user_stibp = mode; 1345 1346 set_mode: 1347 pr_info("%s\n", spectre_v2_user_strings[mode]); 1348 } 1349 1350 static const char * const spectre_v2_strings[] = { 1351 [SPECTRE_V2_NONE] = "Vulnerable", 1352 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1353 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 1354 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1355 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1356 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1357 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1358 }; 1359 1360 static const struct { 1361 const char *option; 1362 enum spectre_v2_mitigation_cmd cmd; 1363 bool secure; 1364 } mitigation_options[] __initconst = { 1365 { "off", SPECTRE_V2_CMD_NONE, false }, 1366 { "on", SPECTRE_V2_CMD_FORCE, true }, 1367 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 1368 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1369 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1370 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 1371 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 1372 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 1373 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 1374 { "auto", SPECTRE_V2_CMD_AUTO, false }, 1375 { "ibrs", SPECTRE_V2_CMD_IBRS, false }, 1376 }; 1377 1378 static void __init spec_v2_print_cond(const char *reason, bool secure) 1379 { 1380 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1381 pr_info("%s selected on command line.\n", reason); 1382 } 1383 1384 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 1385 { 1386 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 1387 char arg[20]; 1388 int ret, i; 1389 1390 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 1391 cpu_mitigations_off()) 1392 return SPECTRE_V2_CMD_NONE; 1393 1394 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 1395 if (ret < 0) 1396 return SPECTRE_V2_CMD_AUTO; 1397 1398 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 1399 if (!match_option(arg, ret, mitigation_options[i].option)) 1400 continue; 1401 cmd = mitigation_options[i].cmd; 1402 break; 1403 } 1404 1405 if (i >= ARRAY_SIZE(mitigation_options)) { 1406 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1407 return SPECTRE_V2_CMD_AUTO; 1408 } 1409 1410 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 1411 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1412 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 1413 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1414 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1415 !IS_ENABLED(CONFIG_RETPOLINE)) { 1416 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1417 mitigation_options[i].option); 1418 return SPECTRE_V2_CMD_AUTO; 1419 } 1420 1421 if ((cmd == SPECTRE_V2_CMD_EIBRS || 1422 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1423 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1424 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1425 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n", 1426 mitigation_options[i].option); 1427 return SPECTRE_V2_CMD_AUTO; 1428 } 1429 1430 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1431 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 1432 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 1433 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 1434 mitigation_options[i].option); 1435 return SPECTRE_V2_CMD_AUTO; 1436 } 1437 1438 if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) { 1439 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1440 mitigation_options[i].option); 1441 return SPECTRE_V2_CMD_AUTO; 1442 } 1443 1444 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1445 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n", 1446 mitigation_options[i].option); 1447 return SPECTRE_V2_CMD_AUTO; 1448 } 1449 1450 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 1451 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n", 1452 mitigation_options[i].option); 1453 return SPECTRE_V2_CMD_AUTO; 1454 } 1455 1456 if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 1457 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n", 1458 mitigation_options[i].option); 1459 return SPECTRE_V2_CMD_AUTO; 1460 } 1461 1462 spec_v2_print_cond(mitigation_options[i].option, 1463 mitigation_options[i].secure); 1464 return cmd; 1465 } 1466 1467 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1468 { 1469 if (!IS_ENABLED(CONFIG_RETPOLINE)) { 1470 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1471 return SPECTRE_V2_NONE; 1472 } 1473 1474 return SPECTRE_V2_RETPOLINE; 1475 } 1476 1477 /* Disable in-kernel use of non-RSB RET predictors */ 1478 static void __init spec_ctrl_disable_kernel_rrsba(void) 1479 { 1480 u64 ia32_cap; 1481 1482 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1483 return; 1484 1485 ia32_cap = x86_read_arch_cap_msr(); 1486 1487 if (ia32_cap & ARCH_CAP_RRSBA) { 1488 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1489 update_spec_ctrl(x86_spec_ctrl_base); 1490 } 1491 } 1492 1493 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode) 1494 { 1495 /* 1496 * Similar to context switches, there are two types of RSB attacks 1497 * after VM exit: 1498 * 1499 * 1) RSB underflow 1500 * 1501 * 2) Poisoned RSB entry 1502 * 1503 * When retpoline is enabled, both are mitigated by filling/clearing 1504 * the RSB. 1505 * 1506 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch 1507 * prediction isolation protections, RSB still needs to be cleared 1508 * because of #2. Note that SMEP provides no protection here, unlike 1509 * user-space-poisoned RSB entries. 1510 * 1511 * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB 1512 * bug is present then a LITE version of RSB protection is required, 1513 * just a single call needs to retire before a RET is executed. 1514 */ 1515 switch (mode) { 1516 case SPECTRE_V2_NONE: 1517 return; 1518 1519 case SPECTRE_V2_EIBRS_LFENCE: 1520 case SPECTRE_V2_EIBRS: 1521 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 1522 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 1523 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 1524 } 1525 return; 1526 1527 case SPECTRE_V2_EIBRS_RETPOLINE: 1528 case SPECTRE_V2_RETPOLINE: 1529 case SPECTRE_V2_LFENCE: 1530 case SPECTRE_V2_IBRS: 1531 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1532 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n"); 1533 return; 1534 } 1535 1536 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit"); 1537 dump_stack(); 1538 } 1539 1540 static void __init spectre_v2_select_mitigation(void) 1541 { 1542 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 1543 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 1544 1545 /* 1546 * If the CPU is not affected and the command line mode is NONE or AUTO 1547 * then nothing to do. 1548 */ 1549 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 1550 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 1551 return; 1552 1553 switch (cmd) { 1554 case SPECTRE_V2_CMD_NONE: 1555 return; 1556 1557 case SPECTRE_V2_CMD_FORCE: 1558 case SPECTRE_V2_CMD_AUTO: 1559 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1560 mode = SPECTRE_V2_EIBRS; 1561 break; 1562 } 1563 1564 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) && 1565 boot_cpu_has_bug(X86_BUG_RETBLEED) && 1566 retbleed_cmd != RETBLEED_CMD_OFF && 1567 retbleed_cmd != RETBLEED_CMD_STUFF && 1568 boot_cpu_has(X86_FEATURE_IBRS) && 1569 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1570 mode = SPECTRE_V2_IBRS; 1571 break; 1572 } 1573 1574 mode = spectre_v2_select_retpoline(); 1575 break; 1576 1577 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 1578 pr_err(SPECTRE_V2_LFENCE_MSG); 1579 mode = SPECTRE_V2_LFENCE; 1580 break; 1581 1582 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 1583 mode = SPECTRE_V2_RETPOLINE; 1584 break; 1585 1586 case SPECTRE_V2_CMD_RETPOLINE: 1587 mode = spectre_v2_select_retpoline(); 1588 break; 1589 1590 case SPECTRE_V2_CMD_IBRS: 1591 mode = SPECTRE_V2_IBRS; 1592 break; 1593 1594 case SPECTRE_V2_CMD_EIBRS: 1595 mode = SPECTRE_V2_EIBRS; 1596 break; 1597 1598 case SPECTRE_V2_CMD_EIBRS_LFENCE: 1599 mode = SPECTRE_V2_EIBRS_LFENCE; 1600 break; 1601 1602 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 1603 mode = SPECTRE_V2_EIBRS_RETPOLINE; 1604 break; 1605 } 1606 1607 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1608 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1609 1610 if (spectre_v2_in_ibrs_mode(mode)) { 1611 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 1612 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 1613 } else { 1614 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 1615 update_spec_ctrl(x86_spec_ctrl_base); 1616 } 1617 } 1618 1619 switch (mode) { 1620 case SPECTRE_V2_NONE: 1621 case SPECTRE_V2_EIBRS: 1622 break; 1623 1624 case SPECTRE_V2_IBRS: 1625 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 1626 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1627 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 1628 break; 1629 1630 case SPECTRE_V2_LFENCE: 1631 case SPECTRE_V2_EIBRS_LFENCE: 1632 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 1633 fallthrough; 1634 1635 case SPECTRE_V2_RETPOLINE: 1636 case SPECTRE_V2_EIBRS_RETPOLINE: 1637 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 1638 break; 1639 } 1640 1641 /* 1642 * Disable alternate RSB predictions in kernel when indirect CALLs and 1643 * JMPs gets protection against BHI and Intramode-BTI, but RET 1644 * prediction from a non-RSB predictor is still a risk. 1645 */ 1646 if (mode == SPECTRE_V2_EIBRS_LFENCE || 1647 mode == SPECTRE_V2_EIBRS_RETPOLINE || 1648 mode == SPECTRE_V2_RETPOLINE) 1649 spec_ctrl_disable_kernel_rrsba(); 1650 1651 spectre_v2_enabled = mode; 1652 pr_info("%s\n", spectre_v2_strings[mode]); 1653 1654 /* 1655 * If Spectre v2 protection has been enabled, fill the RSB during a 1656 * context switch. In general there are two types of RSB attacks 1657 * across context switches, for which the CALLs/RETs may be unbalanced. 1658 * 1659 * 1) RSB underflow 1660 * 1661 * Some Intel parts have "bottomless RSB". When the RSB is empty, 1662 * speculated return targets may come from the branch predictor, 1663 * which could have a user-poisoned BTB or BHB entry. 1664 * 1665 * AMD has it even worse: *all* returns are speculated from the BTB, 1666 * regardless of the state of the RSB. 1667 * 1668 * When IBRS or eIBRS is enabled, the "user -> kernel" attack 1669 * scenario is mitigated by the IBRS branch prediction isolation 1670 * properties, so the RSB buffer filling wouldn't be necessary to 1671 * protect against this type of attack. 1672 * 1673 * The "user -> user" attack scenario is mitigated by RSB filling. 1674 * 1675 * 2) Poisoned RSB entry 1676 * 1677 * If the 'next' in-kernel return stack is shorter than 'prev', 1678 * 'next' could be tricked into speculating with a user-poisoned RSB 1679 * entry. 1680 * 1681 * The "user -> kernel" attack scenario is mitigated by SMEP and 1682 * eIBRS. 1683 * 1684 * The "user -> user" scenario, also known as SpectreBHB, requires 1685 * RSB clearing. 1686 * 1687 * So to mitigate all cases, unconditionally fill RSB on context 1688 * switches. 1689 * 1690 * FIXME: Is this pointless for retbleed-affected AMD? 1691 */ 1692 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1693 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 1694 1695 spectre_v2_determine_rsb_fill_type_at_vmexit(mode); 1696 1697 /* 1698 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 1699 * and Enhanced IBRS protect firmware too, so enable IBRS around 1700 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 1701 * otherwise enabled. 1702 * 1703 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 1704 * the user might select retpoline on the kernel command line and if 1705 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 1706 * enable IBRS around firmware calls. 1707 */ 1708 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 1709 boot_cpu_has(X86_FEATURE_IBPB) && 1710 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1711 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 1712 1713 if (retbleed_cmd != RETBLEED_CMD_IBPB) { 1714 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 1715 pr_info("Enabling Speculation Barrier for firmware calls\n"); 1716 } 1717 1718 } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { 1719 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 1720 pr_info("Enabling Restricted Speculation for firmware calls\n"); 1721 } 1722 1723 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 1724 spectre_v2_cmd = cmd; 1725 } 1726 1727 static void update_stibp_msr(void * __unused) 1728 { 1729 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 1730 update_spec_ctrl(val); 1731 } 1732 1733 /* Update x86_spec_ctrl_base in case SMT state changed. */ 1734 static void update_stibp_strict(void) 1735 { 1736 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 1737 1738 if (sched_smt_active()) 1739 mask |= SPEC_CTRL_STIBP; 1740 1741 if (mask == x86_spec_ctrl_base) 1742 return; 1743 1744 pr_info("Update user space SMT mitigation: STIBP %s\n", 1745 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 1746 x86_spec_ctrl_base = mask; 1747 on_each_cpu(update_stibp_msr, NULL, 1); 1748 } 1749 1750 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 1751 static void update_indir_branch_cond(void) 1752 { 1753 if (sched_smt_active()) 1754 static_branch_enable(&switch_to_cond_stibp); 1755 else 1756 static_branch_disable(&switch_to_cond_stibp); 1757 } 1758 1759 #undef pr_fmt 1760 #define pr_fmt(fmt) fmt 1761 1762 /* Update the static key controlling the MDS CPU buffer clear in idle */ 1763 static void update_mds_branch_idle(void) 1764 { 1765 u64 ia32_cap = x86_read_arch_cap_msr(); 1766 1767 /* 1768 * Enable the idle clearing if SMT is active on CPUs which are 1769 * affected only by MSBDS and not any other MDS variant. 1770 * 1771 * The other variants cannot be mitigated when SMT is enabled, so 1772 * clearing the buffers on idle just to prevent the Store Buffer 1773 * repartitioning leak would be a window dressing exercise. 1774 */ 1775 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 1776 return; 1777 1778 if (sched_smt_active()) { 1779 static_branch_enable(&mds_idle_clear); 1780 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 1781 (ia32_cap & ARCH_CAP_FBSDP_NO)) { 1782 static_branch_disable(&mds_idle_clear); 1783 } 1784 } 1785 1786 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 1787 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 1788 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 1789 1790 void cpu_bugs_smt_update(void) 1791 { 1792 mutex_lock(&spec_ctrl_mutex); 1793 1794 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1795 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1796 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1797 1798 switch (spectre_v2_user_stibp) { 1799 case SPECTRE_V2_USER_NONE: 1800 break; 1801 case SPECTRE_V2_USER_STRICT: 1802 case SPECTRE_V2_USER_STRICT_PREFERRED: 1803 update_stibp_strict(); 1804 break; 1805 case SPECTRE_V2_USER_PRCTL: 1806 case SPECTRE_V2_USER_SECCOMP: 1807 update_indir_branch_cond(); 1808 break; 1809 } 1810 1811 switch (mds_mitigation) { 1812 case MDS_MITIGATION_FULL: 1813 case MDS_MITIGATION_VMWERV: 1814 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 1815 pr_warn_once(MDS_MSG_SMT); 1816 update_mds_branch_idle(); 1817 break; 1818 case MDS_MITIGATION_OFF: 1819 break; 1820 } 1821 1822 switch (taa_mitigation) { 1823 case TAA_MITIGATION_VERW: 1824 case TAA_MITIGATION_UCODE_NEEDED: 1825 if (sched_smt_active()) 1826 pr_warn_once(TAA_MSG_SMT); 1827 break; 1828 case TAA_MITIGATION_TSX_DISABLED: 1829 case TAA_MITIGATION_OFF: 1830 break; 1831 } 1832 1833 switch (mmio_mitigation) { 1834 case MMIO_MITIGATION_VERW: 1835 case MMIO_MITIGATION_UCODE_NEEDED: 1836 if (sched_smt_active()) 1837 pr_warn_once(MMIO_MSG_SMT); 1838 break; 1839 case MMIO_MITIGATION_OFF: 1840 break; 1841 } 1842 1843 mutex_unlock(&spec_ctrl_mutex); 1844 } 1845 1846 #undef pr_fmt 1847 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 1848 1849 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 1850 1851 /* The kernel command line selection */ 1852 enum ssb_mitigation_cmd { 1853 SPEC_STORE_BYPASS_CMD_NONE, 1854 SPEC_STORE_BYPASS_CMD_AUTO, 1855 SPEC_STORE_BYPASS_CMD_ON, 1856 SPEC_STORE_BYPASS_CMD_PRCTL, 1857 SPEC_STORE_BYPASS_CMD_SECCOMP, 1858 }; 1859 1860 static const char * const ssb_strings[] = { 1861 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 1862 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 1863 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 1864 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 1865 }; 1866 1867 static const struct { 1868 const char *option; 1869 enum ssb_mitigation_cmd cmd; 1870 } ssb_mitigation_options[] __initconst = { 1871 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 1872 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 1873 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 1874 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 1875 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 1876 }; 1877 1878 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 1879 { 1880 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 1881 char arg[20]; 1882 int ret, i; 1883 1884 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 1885 cpu_mitigations_off()) { 1886 return SPEC_STORE_BYPASS_CMD_NONE; 1887 } else { 1888 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 1889 arg, sizeof(arg)); 1890 if (ret < 0) 1891 return SPEC_STORE_BYPASS_CMD_AUTO; 1892 1893 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1894 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1895 continue; 1896 1897 cmd = ssb_mitigation_options[i].cmd; 1898 break; 1899 } 1900 1901 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1902 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1903 return SPEC_STORE_BYPASS_CMD_AUTO; 1904 } 1905 } 1906 1907 return cmd; 1908 } 1909 1910 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1911 { 1912 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1913 enum ssb_mitigation_cmd cmd; 1914 1915 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1916 return mode; 1917 1918 cmd = ssb_parse_cmdline(); 1919 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1920 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1921 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1922 return mode; 1923 1924 switch (cmd) { 1925 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1926 /* 1927 * Choose prctl+seccomp as the default mode if seccomp is 1928 * enabled. 1929 */ 1930 if (IS_ENABLED(CONFIG_SECCOMP)) 1931 mode = SPEC_STORE_BYPASS_SECCOMP; 1932 else 1933 mode = SPEC_STORE_BYPASS_PRCTL; 1934 break; 1935 case SPEC_STORE_BYPASS_CMD_ON: 1936 mode = SPEC_STORE_BYPASS_DISABLE; 1937 break; 1938 case SPEC_STORE_BYPASS_CMD_AUTO: 1939 case SPEC_STORE_BYPASS_CMD_PRCTL: 1940 mode = SPEC_STORE_BYPASS_PRCTL; 1941 break; 1942 case SPEC_STORE_BYPASS_CMD_NONE: 1943 break; 1944 } 1945 1946 /* 1947 * We have three CPU feature flags that are in play here: 1948 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1949 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1950 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1951 */ 1952 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1953 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1954 /* 1955 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1956 * use a completely different MSR and bit dependent on family. 1957 */ 1958 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1959 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1960 x86_amd_ssb_disable(); 1961 } else { 1962 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1963 update_spec_ctrl(x86_spec_ctrl_base); 1964 } 1965 } 1966 1967 return mode; 1968 } 1969 1970 static void ssb_select_mitigation(void) 1971 { 1972 ssb_mode = __ssb_select_mitigation(); 1973 1974 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1975 pr_info("%s\n", ssb_strings[ssb_mode]); 1976 } 1977 1978 #undef pr_fmt 1979 #define pr_fmt(fmt) "Speculation prctl: " fmt 1980 1981 static void task_update_spec_tif(struct task_struct *tsk) 1982 { 1983 /* Force the update of the real TIF bits */ 1984 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1985 1986 /* 1987 * Immediately update the speculation control MSRs for the current 1988 * task, but for a non-current task delay setting the CPU 1989 * mitigation until it is scheduled next. 1990 * 1991 * This can only happen for SECCOMP mitigation. For PRCTL it's 1992 * always the current task. 1993 */ 1994 if (tsk == current) 1995 speculation_ctrl_update_current(); 1996 } 1997 1998 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 1999 { 2000 2001 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2002 return -EPERM; 2003 2004 switch (ctrl) { 2005 case PR_SPEC_ENABLE: 2006 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2007 return 0; 2008 case PR_SPEC_DISABLE: 2009 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2010 return 0; 2011 default: 2012 return -ERANGE; 2013 } 2014 } 2015 2016 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2017 { 2018 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2019 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2020 return -ENXIO; 2021 2022 switch (ctrl) { 2023 case PR_SPEC_ENABLE: 2024 /* If speculation is force disabled, enable is not allowed */ 2025 if (task_spec_ssb_force_disable(task)) 2026 return -EPERM; 2027 task_clear_spec_ssb_disable(task); 2028 task_clear_spec_ssb_noexec(task); 2029 task_update_spec_tif(task); 2030 break; 2031 case PR_SPEC_DISABLE: 2032 task_set_spec_ssb_disable(task); 2033 task_clear_spec_ssb_noexec(task); 2034 task_update_spec_tif(task); 2035 break; 2036 case PR_SPEC_FORCE_DISABLE: 2037 task_set_spec_ssb_disable(task); 2038 task_set_spec_ssb_force_disable(task); 2039 task_clear_spec_ssb_noexec(task); 2040 task_update_spec_tif(task); 2041 break; 2042 case PR_SPEC_DISABLE_NOEXEC: 2043 if (task_spec_ssb_force_disable(task)) 2044 return -EPERM; 2045 task_set_spec_ssb_disable(task); 2046 task_set_spec_ssb_noexec(task); 2047 task_update_spec_tif(task); 2048 break; 2049 default: 2050 return -ERANGE; 2051 } 2052 return 0; 2053 } 2054 2055 static bool is_spec_ib_user_controlled(void) 2056 { 2057 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2058 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2059 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2060 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2061 } 2062 2063 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2064 { 2065 switch (ctrl) { 2066 case PR_SPEC_ENABLE: 2067 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2068 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2069 return 0; 2070 2071 /* 2072 * With strict mode for both IBPB and STIBP, the instruction 2073 * code paths avoid checking this task flag and instead, 2074 * unconditionally run the instruction. However, STIBP and IBPB 2075 * are independent and either can be set to conditionally 2076 * enabled regardless of the mode of the other. 2077 * 2078 * If either is set to conditional, allow the task flag to be 2079 * updated, unless it was force-disabled by a previous prctl 2080 * call. Currently, this is possible on an AMD CPU which has the 2081 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2082 * kernel is booted with 'spectre_v2_user=seccomp', then 2083 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2084 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2085 */ 2086 if (!is_spec_ib_user_controlled() || 2087 task_spec_ib_force_disable(task)) 2088 return -EPERM; 2089 2090 task_clear_spec_ib_disable(task); 2091 task_update_spec_tif(task); 2092 break; 2093 case PR_SPEC_DISABLE: 2094 case PR_SPEC_FORCE_DISABLE: 2095 /* 2096 * Indirect branch speculation is always allowed when 2097 * mitigation is force disabled. 2098 */ 2099 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2100 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2101 return -EPERM; 2102 2103 if (!is_spec_ib_user_controlled()) 2104 return 0; 2105 2106 task_set_spec_ib_disable(task); 2107 if (ctrl == PR_SPEC_FORCE_DISABLE) 2108 task_set_spec_ib_force_disable(task); 2109 task_update_spec_tif(task); 2110 if (task == current) 2111 indirect_branch_prediction_barrier(); 2112 break; 2113 default: 2114 return -ERANGE; 2115 } 2116 return 0; 2117 } 2118 2119 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2120 unsigned long ctrl) 2121 { 2122 switch (which) { 2123 case PR_SPEC_STORE_BYPASS: 2124 return ssb_prctl_set(task, ctrl); 2125 case PR_SPEC_INDIRECT_BRANCH: 2126 return ib_prctl_set(task, ctrl); 2127 case PR_SPEC_L1D_FLUSH: 2128 return l1d_flush_prctl_set(task, ctrl); 2129 default: 2130 return -ENODEV; 2131 } 2132 } 2133 2134 #ifdef CONFIG_SECCOMP 2135 void arch_seccomp_spec_mitigate(struct task_struct *task) 2136 { 2137 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2138 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2139 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2140 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2141 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2142 } 2143 #endif 2144 2145 static int l1d_flush_prctl_get(struct task_struct *task) 2146 { 2147 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2148 return PR_SPEC_FORCE_DISABLE; 2149 2150 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2151 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2152 else 2153 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2154 } 2155 2156 static int ssb_prctl_get(struct task_struct *task) 2157 { 2158 switch (ssb_mode) { 2159 case SPEC_STORE_BYPASS_NONE: 2160 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2161 return PR_SPEC_ENABLE; 2162 return PR_SPEC_NOT_AFFECTED; 2163 case SPEC_STORE_BYPASS_DISABLE: 2164 return PR_SPEC_DISABLE; 2165 case SPEC_STORE_BYPASS_SECCOMP: 2166 case SPEC_STORE_BYPASS_PRCTL: 2167 if (task_spec_ssb_force_disable(task)) 2168 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2169 if (task_spec_ssb_noexec(task)) 2170 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2171 if (task_spec_ssb_disable(task)) 2172 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2173 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2174 } 2175 BUG(); 2176 } 2177 2178 static int ib_prctl_get(struct task_struct *task) 2179 { 2180 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2181 return PR_SPEC_NOT_AFFECTED; 2182 2183 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2184 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2185 return PR_SPEC_ENABLE; 2186 else if (is_spec_ib_user_controlled()) { 2187 if (task_spec_ib_force_disable(task)) 2188 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2189 if (task_spec_ib_disable(task)) 2190 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2191 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2192 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2193 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2194 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2195 return PR_SPEC_DISABLE; 2196 else 2197 return PR_SPEC_NOT_AFFECTED; 2198 } 2199 2200 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2201 { 2202 switch (which) { 2203 case PR_SPEC_STORE_BYPASS: 2204 return ssb_prctl_get(task); 2205 case PR_SPEC_INDIRECT_BRANCH: 2206 return ib_prctl_get(task); 2207 case PR_SPEC_L1D_FLUSH: 2208 return l1d_flush_prctl_get(task); 2209 default: 2210 return -ENODEV; 2211 } 2212 } 2213 2214 void x86_spec_ctrl_setup_ap(void) 2215 { 2216 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2217 update_spec_ctrl(x86_spec_ctrl_base); 2218 2219 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2220 x86_amd_ssb_disable(); 2221 } 2222 2223 bool itlb_multihit_kvm_mitigation; 2224 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 2225 2226 #undef pr_fmt 2227 #define pr_fmt(fmt) "L1TF: " fmt 2228 2229 /* Default mitigation for L1TF-affected CPUs */ 2230 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 2231 #if IS_ENABLED(CONFIG_KVM_INTEL) 2232 EXPORT_SYMBOL_GPL(l1tf_mitigation); 2233 #endif 2234 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2235 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 2236 2237 /* 2238 * These CPUs all support 44bits physical address space internally in the 2239 * cache but CPUID can report a smaller number of physical address bits. 2240 * 2241 * The L1TF mitigation uses the top most address bit for the inversion of 2242 * non present PTEs. When the installed memory reaches into the top most 2243 * address bit due to memory holes, which has been observed on machines 2244 * which report 36bits physical address bits and have 32G RAM installed, 2245 * then the mitigation range check in l1tf_select_mitigation() triggers. 2246 * This is a false positive because the mitigation is still possible due to 2247 * the fact that the cache uses 44bit internally. Use the cache bits 2248 * instead of the reported physical bits and adjust them on the affected 2249 * machines to 44bit if the reported bits are less than 44. 2250 */ 2251 static void override_cache_bits(struct cpuinfo_x86 *c) 2252 { 2253 if (c->x86 != 6) 2254 return; 2255 2256 switch (c->x86_model) { 2257 case INTEL_FAM6_NEHALEM: 2258 case INTEL_FAM6_WESTMERE: 2259 case INTEL_FAM6_SANDYBRIDGE: 2260 case INTEL_FAM6_IVYBRIDGE: 2261 case INTEL_FAM6_HASWELL: 2262 case INTEL_FAM6_HASWELL_L: 2263 case INTEL_FAM6_HASWELL_G: 2264 case INTEL_FAM6_BROADWELL: 2265 case INTEL_FAM6_BROADWELL_G: 2266 case INTEL_FAM6_SKYLAKE_L: 2267 case INTEL_FAM6_SKYLAKE: 2268 case INTEL_FAM6_KABYLAKE_L: 2269 case INTEL_FAM6_KABYLAKE: 2270 if (c->x86_cache_bits < 44) 2271 c->x86_cache_bits = 44; 2272 break; 2273 } 2274 } 2275 2276 static void __init l1tf_select_mitigation(void) 2277 { 2278 u64 half_pa; 2279 2280 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2281 return; 2282 2283 if (cpu_mitigations_off()) 2284 l1tf_mitigation = L1TF_MITIGATION_OFF; 2285 else if (cpu_mitigations_auto_nosmt()) 2286 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2287 2288 override_cache_bits(&boot_cpu_data); 2289 2290 switch (l1tf_mitigation) { 2291 case L1TF_MITIGATION_OFF: 2292 case L1TF_MITIGATION_FLUSH_NOWARN: 2293 case L1TF_MITIGATION_FLUSH: 2294 break; 2295 case L1TF_MITIGATION_FLUSH_NOSMT: 2296 case L1TF_MITIGATION_FULL: 2297 cpu_smt_disable(false); 2298 break; 2299 case L1TF_MITIGATION_FULL_FORCE: 2300 cpu_smt_disable(true); 2301 break; 2302 } 2303 2304 #if CONFIG_PGTABLE_LEVELS == 2 2305 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2306 return; 2307 #endif 2308 2309 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2310 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2311 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2312 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2313 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2314 half_pa); 2315 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2316 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2317 return; 2318 } 2319 2320 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2321 } 2322 2323 static int __init l1tf_cmdline(char *str) 2324 { 2325 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2326 return 0; 2327 2328 if (!str) 2329 return -EINVAL; 2330 2331 if (!strcmp(str, "off")) 2332 l1tf_mitigation = L1TF_MITIGATION_OFF; 2333 else if (!strcmp(str, "flush,nowarn")) 2334 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2335 else if (!strcmp(str, "flush")) 2336 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2337 else if (!strcmp(str, "flush,nosmt")) 2338 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2339 else if (!strcmp(str, "full")) 2340 l1tf_mitigation = L1TF_MITIGATION_FULL; 2341 else if (!strcmp(str, "full,force")) 2342 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2343 2344 return 0; 2345 } 2346 early_param("l1tf", l1tf_cmdline); 2347 2348 #undef pr_fmt 2349 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 2350 2351 enum srso_mitigation { 2352 SRSO_MITIGATION_NONE, 2353 SRSO_MITIGATION_UCODE_NEEDED, 2354 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 2355 SRSO_MITIGATION_MICROCODE, 2356 SRSO_MITIGATION_SAFE_RET, 2357 SRSO_MITIGATION_IBPB, 2358 SRSO_MITIGATION_IBPB_ON_VMEXIT, 2359 }; 2360 2361 enum srso_mitigation_cmd { 2362 SRSO_CMD_OFF, 2363 SRSO_CMD_MICROCODE, 2364 SRSO_CMD_SAFE_RET, 2365 SRSO_CMD_IBPB, 2366 SRSO_CMD_IBPB_ON_VMEXIT, 2367 }; 2368 2369 static const char * const srso_strings[] = { 2370 [SRSO_MITIGATION_NONE] = "Vulnerable", 2371 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 2372 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 2373 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 2374 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 2375 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 2376 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only" 2377 }; 2378 2379 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE; 2380 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET; 2381 2382 static int __init srso_parse_cmdline(char *str) 2383 { 2384 if (!str) 2385 return -EINVAL; 2386 2387 if (!strcmp(str, "off")) 2388 srso_cmd = SRSO_CMD_OFF; 2389 else if (!strcmp(str, "microcode")) 2390 srso_cmd = SRSO_CMD_MICROCODE; 2391 else if (!strcmp(str, "safe-ret")) 2392 srso_cmd = SRSO_CMD_SAFE_RET; 2393 else if (!strcmp(str, "ibpb")) 2394 srso_cmd = SRSO_CMD_IBPB; 2395 else if (!strcmp(str, "ibpb-vmexit")) 2396 srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT; 2397 else 2398 pr_err("Ignoring unknown SRSO option (%s).", str); 2399 2400 return 0; 2401 } 2402 early_param("spec_rstack_overflow", srso_parse_cmdline); 2403 2404 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 2405 2406 static void __init srso_select_mitigation(void) 2407 { 2408 bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); 2409 2410 if (cpu_mitigations_off()) 2411 return; 2412 2413 if (!boot_cpu_has_bug(X86_BUG_SRSO)) { 2414 if (boot_cpu_has(X86_FEATURE_SBPB)) 2415 x86_pred_cmd = PRED_CMD_SBPB; 2416 return; 2417 } 2418 2419 if (has_microcode) { 2420 /* 2421 * Zen1/2 with SMT off aren't vulnerable after the right 2422 * IBPB microcode has been applied. 2423 * 2424 * Zen1/2 don't have SBPB, no need to try to enable it here. 2425 */ 2426 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 2427 setup_force_cpu_cap(X86_FEATURE_SRSO_NO); 2428 return; 2429 } 2430 2431 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2432 srso_mitigation = SRSO_MITIGATION_IBPB; 2433 goto out; 2434 } 2435 } else { 2436 pr_warn("IBPB-extending microcode not applied!\n"); 2437 pr_warn(SRSO_NOTICE); 2438 2439 /* may be overwritten by SRSO_CMD_SAFE_RET below */ 2440 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 2441 } 2442 2443 switch (srso_cmd) { 2444 case SRSO_CMD_OFF: 2445 if (boot_cpu_has(X86_FEATURE_SBPB)) 2446 x86_pred_cmd = PRED_CMD_SBPB; 2447 return; 2448 2449 case SRSO_CMD_MICROCODE: 2450 if (has_microcode) { 2451 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2452 pr_warn(SRSO_NOTICE); 2453 } 2454 break; 2455 2456 case SRSO_CMD_SAFE_RET: 2457 if (IS_ENABLED(CONFIG_CPU_SRSO)) { 2458 /* 2459 * Enable the return thunk for generated code 2460 * like ftrace, static_call, etc. 2461 */ 2462 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 2463 setup_force_cpu_cap(X86_FEATURE_UNRET); 2464 2465 if (boot_cpu_data.x86 == 0x19) { 2466 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 2467 x86_return_thunk = srso_alias_return_thunk; 2468 } else { 2469 setup_force_cpu_cap(X86_FEATURE_SRSO); 2470 x86_return_thunk = srso_return_thunk; 2471 } 2472 if (has_microcode) 2473 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2474 else 2475 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 2476 } else { 2477 pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); 2478 } 2479 break; 2480 2481 case SRSO_CMD_IBPB: 2482 if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { 2483 if (has_microcode) { 2484 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 2485 srso_mitigation = SRSO_MITIGATION_IBPB; 2486 } 2487 } else { 2488 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n"); 2489 } 2490 break; 2491 2492 case SRSO_CMD_IBPB_ON_VMEXIT: 2493 if (IS_ENABLED(CONFIG_CPU_SRSO)) { 2494 if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) { 2495 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 2496 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2497 } 2498 } else { 2499 pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); 2500 } 2501 break; 2502 } 2503 2504 out: 2505 pr_info("%s\n", srso_strings[srso_mitigation]); 2506 } 2507 2508 #undef pr_fmt 2509 #define pr_fmt(fmt) fmt 2510 2511 #ifdef CONFIG_SYSFS 2512 2513 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 2514 2515 #if IS_ENABLED(CONFIG_KVM_INTEL) 2516 static const char * const l1tf_vmx_states[] = { 2517 [VMENTER_L1D_FLUSH_AUTO] = "auto", 2518 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 2519 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 2520 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 2521 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 2522 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 2523 }; 2524 2525 static ssize_t l1tf_show_state(char *buf) 2526 { 2527 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 2528 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2529 2530 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 2531 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 2532 sched_smt_active())) { 2533 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 2534 l1tf_vmx_states[l1tf_vmx_mitigation]); 2535 } 2536 2537 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 2538 l1tf_vmx_states[l1tf_vmx_mitigation], 2539 sched_smt_active() ? "vulnerable" : "disabled"); 2540 } 2541 2542 static ssize_t itlb_multihit_show_state(char *buf) 2543 { 2544 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2545 !boot_cpu_has(X86_FEATURE_VMX)) 2546 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 2547 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 2548 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 2549 else if (itlb_multihit_kvm_mitigation) 2550 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 2551 else 2552 return sysfs_emit(buf, "KVM: Vulnerable\n"); 2553 } 2554 #else 2555 static ssize_t l1tf_show_state(char *buf) 2556 { 2557 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2558 } 2559 2560 static ssize_t itlb_multihit_show_state(char *buf) 2561 { 2562 return sysfs_emit(buf, "Processor vulnerable\n"); 2563 } 2564 #endif 2565 2566 static ssize_t mds_show_state(char *buf) 2567 { 2568 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2569 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2570 mds_strings[mds_mitigation]); 2571 } 2572 2573 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 2574 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2575 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 2576 sched_smt_active() ? "mitigated" : "disabled")); 2577 } 2578 2579 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2580 sched_smt_active() ? "vulnerable" : "disabled"); 2581 } 2582 2583 static ssize_t tsx_async_abort_show_state(char *buf) 2584 { 2585 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 2586 (taa_mitigation == TAA_MITIGATION_OFF)) 2587 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 2588 2589 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2590 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2591 taa_strings[taa_mitigation]); 2592 } 2593 2594 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 2595 sched_smt_active() ? "vulnerable" : "disabled"); 2596 } 2597 2598 static ssize_t mmio_stale_data_show_state(char *buf) 2599 { 2600 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2601 return sysfs_emit(buf, "Unknown: No mitigations\n"); 2602 2603 if (mmio_mitigation == MMIO_MITIGATION_OFF) 2604 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 2605 2606 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2607 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2608 mmio_strings[mmio_mitigation]); 2609 } 2610 2611 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 2612 sched_smt_active() ? "vulnerable" : "disabled"); 2613 } 2614 2615 static char *stibp_state(void) 2616 { 2617 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 2618 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 2619 return ""; 2620 2621 switch (spectre_v2_user_stibp) { 2622 case SPECTRE_V2_USER_NONE: 2623 return ", STIBP: disabled"; 2624 case SPECTRE_V2_USER_STRICT: 2625 return ", STIBP: forced"; 2626 case SPECTRE_V2_USER_STRICT_PREFERRED: 2627 return ", STIBP: always-on"; 2628 case SPECTRE_V2_USER_PRCTL: 2629 case SPECTRE_V2_USER_SECCOMP: 2630 if (static_key_enabled(&switch_to_cond_stibp)) 2631 return ", STIBP: conditional"; 2632 } 2633 return ""; 2634 } 2635 2636 static char *ibpb_state(void) 2637 { 2638 if (boot_cpu_has(X86_FEATURE_IBPB)) { 2639 if (static_key_enabled(&switch_mm_always_ibpb)) 2640 return ", IBPB: always-on"; 2641 if (static_key_enabled(&switch_mm_cond_ibpb)) 2642 return ", IBPB: conditional"; 2643 return ", IBPB: disabled"; 2644 } 2645 return ""; 2646 } 2647 2648 static char *pbrsb_eibrs_state(void) 2649 { 2650 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2651 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 2652 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 2653 return ", PBRSB-eIBRS: SW sequence"; 2654 else 2655 return ", PBRSB-eIBRS: Vulnerable"; 2656 } else { 2657 return ", PBRSB-eIBRS: Not affected"; 2658 } 2659 } 2660 2661 static ssize_t spectre_v2_show_state(char *buf) 2662 { 2663 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 2664 return sysfs_emit(buf, "Vulnerable: LFENCE\n"); 2665 2666 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2667 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 2668 2669 if (sched_smt_active() && unprivileged_ebpf_enabled() && 2670 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 2671 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 2672 2673 return sysfs_emit(buf, "%s%s%s%s%s%s%s\n", 2674 spectre_v2_strings[spectre_v2_enabled], 2675 ibpb_state(), 2676 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 2677 stibp_state(), 2678 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 2679 pbrsb_eibrs_state(), 2680 spectre_v2_module_string()); 2681 } 2682 2683 static ssize_t srbds_show_state(char *buf) 2684 { 2685 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 2686 } 2687 2688 static ssize_t retbleed_show_state(char *buf) 2689 { 2690 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 2691 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2692 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 2693 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 2694 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 2695 2696 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 2697 !sched_smt_active() ? "disabled" : 2698 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2699 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 2700 "enabled with STIBP protection" : "vulnerable"); 2701 } 2702 2703 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 2704 } 2705 2706 static ssize_t srso_show_state(char *buf) 2707 { 2708 if (boot_cpu_has(X86_FEATURE_SRSO_NO)) 2709 return sysfs_emit(buf, "Mitigation: SMT disabled\n"); 2710 2711 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 2712 } 2713 2714 static ssize_t gds_show_state(char *buf) 2715 { 2716 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 2717 } 2718 2719 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 2720 char *buf, unsigned int bug) 2721 { 2722 if (!boot_cpu_has_bug(bug)) 2723 return sysfs_emit(buf, "Not affected\n"); 2724 2725 switch (bug) { 2726 case X86_BUG_CPU_MELTDOWN: 2727 if (boot_cpu_has(X86_FEATURE_PTI)) 2728 return sysfs_emit(buf, "Mitigation: PTI\n"); 2729 2730 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 2731 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 2732 2733 break; 2734 2735 case X86_BUG_SPECTRE_V1: 2736 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 2737 2738 case X86_BUG_SPECTRE_V2: 2739 return spectre_v2_show_state(buf); 2740 2741 case X86_BUG_SPEC_STORE_BYPASS: 2742 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 2743 2744 case X86_BUG_L1TF: 2745 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 2746 return l1tf_show_state(buf); 2747 break; 2748 2749 case X86_BUG_MDS: 2750 return mds_show_state(buf); 2751 2752 case X86_BUG_TAA: 2753 return tsx_async_abort_show_state(buf); 2754 2755 case X86_BUG_ITLB_MULTIHIT: 2756 return itlb_multihit_show_state(buf); 2757 2758 case X86_BUG_SRBDS: 2759 return srbds_show_state(buf); 2760 2761 case X86_BUG_MMIO_STALE_DATA: 2762 case X86_BUG_MMIO_UNKNOWN: 2763 return mmio_stale_data_show_state(buf); 2764 2765 case X86_BUG_RETBLEED: 2766 return retbleed_show_state(buf); 2767 2768 case X86_BUG_SRSO: 2769 return srso_show_state(buf); 2770 2771 case X86_BUG_GDS: 2772 return gds_show_state(buf); 2773 2774 default: 2775 break; 2776 } 2777 2778 return sysfs_emit(buf, "Vulnerable\n"); 2779 } 2780 2781 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 2782 { 2783 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 2784 } 2785 2786 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 2787 { 2788 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 2789 } 2790 2791 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 2792 { 2793 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 2794 } 2795 2796 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 2797 { 2798 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 2799 } 2800 2801 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 2802 { 2803 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 2804 } 2805 2806 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 2807 { 2808 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 2809 } 2810 2811 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 2812 { 2813 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 2814 } 2815 2816 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 2817 { 2818 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 2819 } 2820 2821 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 2822 { 2823 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 2824 } 2825 2826 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 2827 { 2828 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2829 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); 2830 else 2831 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 2832 } 2833 2834 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 2835 { 2836 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 2837 } 2838 2839 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 2840 { 2841 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 2842 } 2843 2844 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 2845 { 2846 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 2847 } 2848 #endif 2849