1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 20 #include <asm/spec-ctrl.h> 21 #include <asm/cmdline.h> 22 #include <asm/bugs.h> 23 #include <asm/processor.h> 24 #include <asm/processor-flags.h> 25 #include <asm/fpu/api.h> 26 #include <asm/msr.h> 27 #include <asm/vmx.h> 28 #include <asm/paravirt.h> 29 #include <asm/intel-family.h> 30 #include <asm/e820/api.h> 31 #include <asm/hypervisor.h> 32 #include <asm/tlbflush.h> 33 #include <asm/cpu.h> 34 35 #include "cpu.h" 36 37 static void __init spectre_v1_select_mitigation(void); 38 static void __init spectre_v2_select_mitigation(void); 39 static void __init retbleed_select_mitigation(void); 40 static void __init spectre_v2_user_select_mitigation(void); 41 static void __init ssb_select_mitigation(void); 42 static void __init l1tf_select_mitigation(void); 43 static void __init mds_select_mitigation(void); 44 static void __init md_clear_update_mitigation(void); 45 static void __init md_clear_select_mitigation(void); 46 static void __init taa_select_mitigation(void); 47 static void __init mmio_select_mitigation(void); 48 static void __init srbds_select_mitigation(void); 49 static void __init l1d_flush_select_mitigation(void); 50 static void __init srso_select_mitigation(void); 51 static void __init gds_select_mitigation(void); 52 53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 54 u64 x86_spec_ctrl_base; 55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 56 57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 59 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); 60 61 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 62 EXPORT_SYMBOL_GPL(x86_pred_cmd); 63 64 static DEFINE_MUTEX(spec_ctrl_mutex); 65 66 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 67 68 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 69 static void update_spec_ctrl(u64 val) 70 { 71 this_cpu_write(x86_spec_ctrl_current, val); 72 wrmsrl(MSR_IA32_SPEC_CTRL, val); 73 } 74 75 /* 76 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 77 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 78 */ 79 void update_spec_ctrl_cond(u64 val) 80 { 81 if (this_cpu_read(x86_spec_ctrl_current) == val) 82 return; 83 84 this_cpu_write(x86_spec_ctrl_current, val); 85 86 /* 87 * When KERNEL_IBRS this MSR is written on return-to-user, unless 88 * forced the update can be delayed until that time. 89 */ 90 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 91 wrmsrl(MSR_IA32_SPEC_CTRL, val); 92 } 93 94 noinstr u64 spec_ctrl_current(void) 95 { 96 return this_cpu_read(x86_spec_ctrl_current); 97 } 98 EXPORT_SYMBOL_GPL(spec_ctrl_current); 99 100 /* 101 * AMD specific MSR info for Speculative Store Bypass control. 102 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 103 */ 104 u64 __ro_after_init x86_amd_ls_cfg_base; 105 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 106 107 /* Control conditional STIBP in switch_to() */ 108 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 109 /* Control conditional IBPB in switch_mm() */ 110 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 111 /* Control unconditional IBPB in switch_mm() */ 112 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 113 114 /* Control MDS CPU buffer clear before returning to user space */ 115 DEFINE_STATIC_KEY_FALSE(mds_user_clear); 116 EXPORT_SYMBOL_GPL(mds_user_clear); 117 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 118 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 119 EXPORT_SYMBOL_GPL(mds_idle_clear); 120 121 /* 122 * Controls whether l1d flush based mitigations are enabled, 123 * based on hw features and admin setting via boot parameter 124 * defaults to false 125 */ 126 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 127 128 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ 129 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); 130 EXPORT_SYMBOL_GPL(mmio_stale_data_clear); 131 132 void __init cpu_select_mitigations(void) 133 { 134 /* 135 * Read the SPEC_CTRL MSR to account for reserved bits which may 136 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 137 * init code as it is not enumerated and depends on the family. 138 */ 139 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 140 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 141 142 /* 143 * Previously running kernel (kexec), may have some controls 144 * turned ON. Clear them and let the mitigations setup below 145 * rediscover them based on configuration. 146 */ 147 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 148 } 149 150 /* Select the proper CPU mitigations before patching alternatives: */ 151 spectre_v1_select_mitigation(); 152 spectre_v2_select_mitigation(); 153 /* 154 * retbleed_select_mitigation() relies on the state set by 155 * spectre_v2_select_mitigation(); specifically it wants to know about 156 * spectre_v2=ibrs. 157 */ 158 retbleed_select_mitigation(); 159 /* 160 * spectre_v2_user_select_mitigation() relies on the state set by 161 * retbleed_select_mitigation(); specifically the STIBP selection is 162 * forced for UNRET or IBPB. 163 */ 164 spectre_v2_user_select_mitigation(); 165 ssb_select_mitigation(); 166 l1tf_select_mitigation(); 167 md_clear_select_mitigation(); 168 srbds_select_mitigation(); 169 l1d_flush_select_mitigation(); 170 171 /* 172 * srso_select_mitigation() depends and must run after 173 * retbleed_select_mitigation(). 174 */ 175 srso_select_mitigation(); 176 gds_select_mitigation(); 177 } 178 179 /* 180 * NOTE: This function is *only* called for SVM, since Intel uses 181 * MSR_IA32_SPEC_CTRL for SSBD. 182 */ 183 void 184 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 185 { 186 u64 guestval, hostval; 187 struct thread_info *ti = current_thread_info(); 188 189 /* 190 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 191 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 192 */ 193 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 194 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 195 return; 196 197 /* 198 * If the host has SSBD mitigation enabled, force it in the host's 199 * virtual MSR value. If its not permanently enabled, evaluate 200 * current's TIF_SSBD thread flag. 201 */ 202 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 203 hostval = SPEC_CTRL_SSBD; 204 else 205 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 206 207 /* Sanitize the guest value */ 208 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 209 210 if (hostval != guestval) { 211 unsigned long tif; 212 213 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 214 ssbd_spec_ctrl_to_tif(hostval); 215 216 speculation_ctrl_update(tif); 217 } 218 } 219 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 220 221 static void x86_amd_ssb_disable(void) 222 { 223 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 224 225 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 226 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 227 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 228 wrmsrl(MSR_AMD64_LS_CFG, msrval); 229 } 230 231 #undef pr_fmt 232 #define pr_fmt(fmt) "MDS: " fmt 233 234 /* Default mitigation for MDS-affected CPUs */ 235 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 236 static bool mds_nosmt __ro_after_init = false; 237 238 static const char * const mds_strings[] = { 239 [MDS_MITIGATION_OFF] = "Vulnerable", 240 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 241 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 242 }; 243 244 static void __init mds_select_mitigation(void) 245 { 246 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 247 mds_mitigation = MDS_MITIGATION_OFF; 248 return; 249 } 250 251 if (mds_mitigation == MDS_MITIGATION_FULL) { 252 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 253 mds_mitigation = MDS_MITIGATION_VMWERV; 254 255 static_branch_enable(&mds_user_clear); 256 257 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 258 (mds_nosmt || cpu_mitigations_auto_nosmt())) 259 cpu_smt_disable(false); 260 } 261 } 262 263 static int __init mds_cmdline(char *str) 264 { 265 if (!boot_cpu_has_bug(X86_BUG_MDS)) 266 return 0; 267 268 if (!str) 269 return -EINVAL; 270 271 if (!strcmp(str, "off")) 272 mds_mitigation = MDS_MITIGATION_OFF; 273 else if (!strcmp(str, "full")) 274 mds_mitigation = MDS_MITIGATION_FULL; 275 else if (!strcmp(str, "full,nosmt")) { 276 mds_mitigation = MDS_MITIGATION_FULL; 277 mds_nosmt = true; 278 } 279 280 return 0; 281 } 282 early_param("mds", mds_cmdline); 283 284 #undef pr_fmt 285 #define pr_fmt(fmt) "TAA: " fmt 286 287 enum taa_mitigations { 288 TAA_MITIGATION_OFF, 289 TAA_MITIGATION_UCODE_NEEDED, 290 TAA_MITIGATION_VERW, 291 TAA_MITIGATION_TSX_DISABLED, 292 }; 293 294 /* Default mitigation for TAA-affected CPUs */ 295 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 296 static bool taa_nosmt __ro_after_init; 297 298 static const char * const taa_strings[] = { 299 [TAA_MITIGATION_OFF] = "Vulnerable", 300 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 301 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 302 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 303 }; 304 305 static void __init taa_select_mitigation(void) 306 { 307 u64 ia32_cap; 308 309 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 310 taa_mitigation = TAA_MITIGATION_OFF; 311 return; 312 } 313 314 /* TSX previously disabled by tsx=off */ 315 if (!boot_cpu_has(X86_FEATURE_RTM)) { 316 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 317 return; 318 } 319 320 if (cpu_mitigations_off()) { 321 taa_mitigation = TAA_MITIGATION_OFF; 322 return; 323 } 324 325 /* 326 * TAA mitigation via VERW is turned off if both 327 * tsx_async_abort=off and mds=off are specified. 328 */ 329 if (taa_mitigation == TAA_MITIGATION_OFF && 330 mds_mitigation == MDS_MITIGATION_OFF) 331 return; 332 333 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 334 taa_mitigation = TAA_MITIGATION_VERW; 335 else 336 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 337 338 /* 339 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 340 * A microcode update fixes this behavior to clear CPU buffers. It also 341 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 342 * ARCH_CAP_TSX_CTRL_MSR bit. 343 * 344 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 345 * update is required. 346 */ 347 ia32_cap = x86_read_arch_cap_msr(); 348 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 349 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 350 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 351 352 /* 353 * TSX is enabled, select alternate mitigation for TAA which is 354 * the same as MDS. Enable MDS static branch to clear CPU buffers. 355 * 356 * For guests that can't determine whether the correct microcode is 357 * present on host, enable the mitigation for UCODE_NEEDED as well. 358 */ 359 static_branch_enable(&mds_user_clear); 360 361 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 362 cpu_smt_disable(false); 363 } 364 365 static int __init tsx_async_abort_parse_cmdline(char *str) 366 { 367 if (!boot_cpu_has_bug(X86_BUG_TAA)) 368 return 0; 369 370 if (!str) 371 return -EINVAL; 372 373 if (!strcmp(str, "off")) { 374 taa_mitigation = TAA_MITIGATION_OFF; 375 } else if (!strcmp(str, "full")) { 376 taa_mitigation = TAA_MITIGATION_VERW; 377 } else if (!strcmp(str, "full,nosmt")) { 378 taa_mitigation = TAA_MITIGATION_VERW; 379 taa_nosmt = true; 380 } 381 382 return 0; 383 } 384 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 385 386 #undef pr_fmt 387 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 388 389 enum mmio_mitigations { 390 MMIO_MITIGATION_OFF, 391 MMIO_MITIGATION_UCODE_NEEDED, 392 MMIO_MITIGATION_VERW, 393 }; 394 395 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 396 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW; 397 static bool mmio_nosmt __ro_after_init = false; 398 399 static const char * const mmio_strings[] = { 400 [MMIO_MITIGATION_OFF] = "Vulnerable", 401 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 402 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 403 }; 404 405 static void __init mmio_select_mitigation(void) 406 { 407 u64 ia32_cap; 408 409 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || 410 boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || 411 cpu_mitigations_off()) { 412 mmio_mitigation = MMIO_MITIGATION_OFF; 413 return; 414 } 415 416 if (mmio_mitigation == MMIO_MITIGATION_OFF) 417 return; 418 419 ia32_cap = x86_read_arch_cap_msr(); 420 421 /* 422 * Enable CPU buffer clear mitigation for host and VMM, if also affected 423 * by MDS or TAA. Otherwise, enable mitigation for VMM only. 424 */ 425 if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && 426 boot_cpu_has(X86_FEATURE_RTM))) 427 static_branch_enable(&mds_user_clear); 428 else 429 static_branch_enable(&mmio_stale_data_clear); 430 431 /* 432 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 433 * be propagated to uncore buffers, clearing the Fill buffers on idle 434 * is required irrespective of SMT state. 435 */ 436 if (!(ia32_cap & ARCH_CAP_FBSDP_NO)) 437 static_branch_enable(&mds_idle_clear); 438 439 /* 440 * Check if the system has the right microcode. 441 * 442 * CPU Fill buffer clear mitigation is enumerated by either an explicit 443 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 444 * affected systems. 445 */ 446 if ((ia32_cap & ARCH_CAP_FB_CLEAR) || 447 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 448 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 449 !(ia32_cap & ARCH_CAP_MDS_NO))) 450 mmio_mitigation = MMIO_MITIGATION_VERW; 451 else 452 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 453 454 if (mmio_nosmt || cpu_mitigations_auto_nosmt()) 455 cpu_smt_disable(false); 456 } 457 458 static int __init mmio_stale_data_parse_cmdline(char *str) 459 { 460 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 461 return 0; 462 463 if (!str) 464 return -EINVAL; 465 466 if (!strcmp(str, "off")) { 467 mmio_mitigation = MMIO_MITIGATION_OFF; 468 } else if (!strcmp(str, "full")) { 469 mmio_mitigation = MMIO_MITIGATION_VERW; 470 } else if (!strcmp(str, "full,nosmt")) { 471 mmio_mitigation = MMIO_MITIGATION_VERW; 472 mmio_nosmt = true; 473 } 474 475 return 0; 476 } 477 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 478 479 #undef pr_fmt 480 #define pr_fmt(fmt) "" fmt 481 482 static void __init md_clear_update_mitigation(void) 483 { 484 if (cpu_mitigations_off()) 485 return; 486 487 if (!static_key_enabled(&mds_user_clear)) 488 goto out; 489 490 /* 491 * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data 492 * mitigation, if necessary. 493 */ 494 if (mds_mitigation == MDS_MITIGATION_OFF && 495 boot_cpu_has_bug(X86_BUG_MDS)) { 496 mds_mitigation = MDS_MITIGATION_FULL; 497 mds_select_mitigation(); 498 } 499 if (taa_mitigation == TAA_MITIGATION_OFF && 500 boot_cpu_has_bug(X86_BUG_TAA)) { 501 taa_mitigation = TAA_MITIGATION_VERW; 502 taa_select_mitigation(); 503 } 504 if (mmio_mitigation == MMIO_MITIGATION_OFF && 505 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 506 mmio_mitigation = MMIO_MITIGATION_VERW; 507 mmio_select_mitigation(); 508 } 509 out: 510 if (boot_cpu_has_bug(X86_BUG_MDS)) 511 pr_info("MDS: %s\n", mds_strings[mds_mitigation]); 512 if (boot_cpu_has_bug(X86_BUG_TAA)) 513 pr_info("TAA: %s\n", taa_strings[taa_mitigation]); 514 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 515 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); 516 else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 517 pr_info("MMIO Stale Data: Unknown: No mitigations\n"); 518 } 519 520 static void __init md_clear_select_mitigation(void) 521 { 522 mds_select_mitigation(); 523 taa_select_mitigation(); 524 mmio_select_mitigation(); 525 526 /* 527 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update 528 * and print their mitigation after MDS, TAA and MMIO Stale Data 529 * mitigation selection is done. 530 */ 531 md_clear_update_mitigation(); 532 } 533 534 #undef pr_fmt 535 #define pr_fmt(fmt) "SRBDS: " fmt 536 537 enum srbds_mitigations { 538 SRBDS_MITIGATION_OFF, 539 SRBDS_MITIGATION_UCODE_NEEDED, 540 SRBDS_MITIGATION_FULL, 541 SRBDS_MITIGATION_TSX_OFF, 542 SRBDS_MITIGATION_HYPERVISOR, 543 }; 544 545 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL; 546 547 static const char * const srbds_strings[] = { 548 [SRBDS_MITIGATION_OFF] = "Vulnerable", 549 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 550 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 551 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 552 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 553 }; 554 555 static bool srbds_off; 556 557 void update_srbds_msr(void) 558 { 559 u64 mcu_ctrl; 560 561 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 562 return; 563 564 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 565 return; 566 567 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 568 return; 569 570 /* 571 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 572 * being disabled and it hasn't received the SRBDS MSR microcode. 573 */ 574 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 575 return; 576 577 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 578 579 switch (srbds_mitigation) { 580 case SRBDS_MITIGATION_OFF: 581 case SRBDS_MITIGATION_TSX_OFF: 582 mcu_ctrl |= RNGDS_MITG_DIS; 583 break; 584 case SRBDS_MITIGATION_FULL: 585 mcu_ctrl &= ~RNGDS_MITG_DIS; 586 break; 587 default: 588 break; 589 } 590 591 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 592 } 593 594 static void __init srbds_select_mitigation(void) 595 { 596 u64 ia32_cap; 597 598 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 599 return; 600 601 /* 602 * Check to see if this is one of the MDS_NO systems supporting TSX that 603 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 604 * by Processor MMIO Stale Data vulnerability. 605 */ 606 ia32_cap = x86_read_arch_cap_msr(); 607 if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 608 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 609 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 610 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 611 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 612 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 613 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 614 else if (cpu_mitigations_off() || srbds_off) 615 srbds_mitigation = SRBDS_MITIGATION_OFF; 616 617 update_srbds_msr(); 618 pr_info("%s\n", srbds_strings[srbds_mitigation]); 619 } 620 621 static int __init srbds_parse_cmdline(char *str) 622 { 623 if (!str) 624 return -EINVAL; 625 626 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 627 return 0; 628 629 srbds_off = !strcmp(str, "off"); 630 return 0; 631 } 632 early_param("srbds", srbds_parse_cmdline); 633 634 #undef pr_fmt 635 #define pr_fmt(fmt) "L1D Flush : " fmt 636 637 enum l1d_flush_mitigations { 638 L1D_FLUSH_OFF = 0, 639 L1D_FLUSH_ON, 640 }; 641 642 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 643 644 static void __init l1d_flush_select_mitigation(void) 645 { 646 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 647 return; 648 649 static_branch_enable(&switch_mm_cond_l1d_flush); 650 pr_info("Conditional flush on switch_mm() enabled\n"); 651 } 652 653 static int __init l1d_flush_parse_cmdline(char *str) 654 { 655 if (!strcmp(str, "on")) 656 l1d_flush_mitigation = L1D_FLUSH_ON; 657 658 return 0; 659 } 660 early_param("l1d_flush", l1d_flush_parse_cmdline); 661 662 #undef pr_fmt 663 #define pr_fmt(fmt) "GDS: " fmt 664 665 enum gds_mitigations { 666 GDS_MITIGATION_OFF, 667 GDS_MITIGATION_UCODE_NEEDED, 668 GDS_MITIGATION_FORCE, 669 GDS_MITIGATION_FULL, 670 GDS_MITIGATION_FULL_LOCKED, 671 GDS_MITIGATION_HYPERVISOR, 672 }; 673 674 #if IS_ENABLED(CONFIG_GDS_FORCE_MITIGATION) 675 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE; 676 #else 677 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL; 678 #endif 679 680 static const char * const gds_strings[] = { 681 [GDS_MITIGATION_OFF] = "Vulnerable", 682 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 683 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 684 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 685 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 686 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 687 }; 688 689 bool gds_ucode_mitigated(void) 690 { 691 return (gds_mitigation == GDS_MITIGATION_FULL || 692 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 693 } 694 EXPORT_SYMBOL_GPL(gds_ucode_mitigated); 695 696 void update_gds_msr(void) 697 { 698 u64 mcu_ctrl_after; 699 u64 mcu_ctrl; 700 701 switch (gds_mitigation) { 702 case GDS_MITIGATION_OFF: 703 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 704 mcu_ctrl |= GDS_MITG_DIS; 705 break; 706 case GDS_MITIGATION_FULL_LOCKED: 707 /* 708 * The LOCKED state comes from the boot CPU. APs might not have 709 * the same state. Make sure the mitigation is enabled on all 710 * CPUs. 711 */ 712 case GDS_MITIGATION_FULL: 713 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 714 mcu_ctrl &= ~GDS_MITG_DIS; 715 break; 716 case GDS_MITIGATION_FORCE: 717 case GDS_MITIGATION_UCODE_NEEDED: 718 case GDS_MITIGATION_HYPERVISOR: 719 return; 720 } 721 722 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 723 724 /* 725 * Check to make sure that the WRMSR value was not ignored. Writes to 726 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 727 * processor was not. 728 */ 729 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 730 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 731 } 732 733 static void __init gds_select_mitigation(void) 734 { 735 u64 mcu_ctrl; 736 737 if (!boot_cpu_has_bug(X86_BUG_GDS)) 738 return; 739 740 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 741 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 742 goto out; 743 } 744 745 if (cpu_mitigations_off()) 746 gds_mitigation = GDS_MITIGATION_OFF; 747 /* Will verify below that mitigation _can_ be disabled */ 748 749 /* No microcode */ 750 if (!(x86_read_arch_cap_msr() & ARCH_CAP_GDS_CTRL)) { 751 if (gds_mitigation == GDS_MITIGATION_FORCE) { 752 /* 753 * This only needs to be done on the boot CPU so do it 754 * here rather than in update_gds_msr() 755 */ 756 setup_clear_cpu_cap(X86_FEATURE_AVX); 757 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 758 } else { 759 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 760 } 761 goto out; 762 } 763 764 /* Microcode has mitigation, use it */ 765 if (gds_mitigation == GDS_MITIGATION_FORCE) 766 gds_mitigation = GDS_MITIGATION_FULL; 767 768 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 769 if (mcu_ctrl & GDS_MITG_LOCKED) { 770 if (gds_mitigation == GDS_MITIGATION_OFF) 771 pr_warn("Mitigation locked. Disable failed.\n"); 772 773 /* 774 * The mitigation is selected from the boot CPU. All other CPUs 775 * _should_ have the same state. If the boot CPU isn't locked 776 * but others are then update_gds_msr() will WARN() of the state 777 * mismatch. If the boot CPU is locked update_gds_msr() will 778 * ensure the other CPUs have the mitigation enabled. 779 */ 780 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 781 } 782 783 update_gds_msr(); 784 out: 785 pr_info("%s\n", gds_strings[gds_mitigation]); 786 } 787 788 static int __init gds_parse_cmdline(char *str) 789 { 790 if (!str) 791 return -EINVAL; 792 793 if (!boot_cpu_has_bug(X86_BUG_GDS)) 794 return 0; 795 796 if (!strcmp(str, "off")) 797 gds_mitigation = GDS_MITIGATION_OFF; 798 else if (!strcmp(str, "force")) 799 gds_mitigation = GDS_MITIGATION_FORCE; 800 801 return 0; 802 } 803 early_param("gather_data_sampling", gds_parse_cmdline); 804 805 #undef pr_fmt 806 #define pr_fmt(fmt) "Spectre V1 : " fmt 807 808 enum spectre_v1_mitigation { 809 SPECTRE_V1_MITIGATION_NONE, 810 SPECTRE_V1_MITIGATION_AUTO, 811 }; 812 813 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 814 SPECTRE_V1_MITIGATION_AUTO; 815 816 static const char * const spectre_v1_strings[] = { 817 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 818 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 819 }; 820 821 /* 822 * Does SMAP provide full mitigation against speculative kernel access to 823 * userspace? 824 */ 825 static bool smap_works_speculatively(void) 826 { 827 if (!boot_cpu_has(X86_FEATURE_SMAP)) 828 return false; 829 830 /* 831 * On CPUs which are vulnerable to Meltdown, SMAP does not 832 * prevent speculative access to user data in the L1 cache. 833 * Consider SMAP to be non-functional as a mitigation on these 834 * CPUs. 835 */ 836 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 837 return false; 838 839 return true; 840 } 841 842 static void __init spectre_v1_select_mitigation(void) 843 { 844 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 845 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 846 return; 847 } 848 849 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 850 /* 851 * With Spectre v1, a user can speculatively control either 852 * path of a conditional swapgs with a user-controlled GS 853 * value. The mitigation is to add lfences to both code paths. 854 * 855 * If FSGSBASE is enabled, the user can put a kernel address in 856 * GS, in which case SMAP provides no protection. 857 * 858 * If FSGSBASE is disabled, the user can only put a user space 859 * address in GS. That makes an attack harder, but still 860 * possible if there's no SMAP protection. 861 */ 862 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 863 !smap_works_speculatively()) { 864 /* 865 * Mitigation can be provided from SWAPGS itself or 866 * PTI as the CR3 write in the Meltdown mitigation 867 * is serializing. 868 * 869 * If neither is there, mitigate with an LFENCE to 870 * stop speculation through swapgs. 871 */ 872 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 873 !boot_cpu_has(X86_FEATURE_PTI)) 874 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 875 876 /* 877 * Enable lfences in the kernel entry (non-swapgs) 878 * paths, to prevent user entry from speculatively 879 * skipping swapgs. 880 */ 881 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 882 } 883 } 884 885 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 886 } 887 888 static int __init nospectre_v1_cmdline(char *str) 889 { 890 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 891 return 0; 892 } 893 early_param("nospectre_v1", nospectre_v1_cmdline); 894 895 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 896 897 #undef pr_fmt 898 #define pr_fmt(fmt) "RETBleed: " fmt 899 900 enum retbleed_mitigation { 901 RETBLEED_MITIGATION_NONE, 902 RETBLEED_MITIGATION_UNRET, 903 RETBLEED_MITIGATION_IBPB, 904 RETBLEED_MITIGATION_IBRS, 905 RETBLEED_MITIGATION_EIBRS, 906 RETBLEED_MITIGATION_STUFF, 907 }; 908 909 enum retbleed_mitigation_cmd { 910 RETBLEED_CMD_OFF, 911 RETBLEED_CMD_AUTO, 912 RETBLEED_CMD_UNRET, 913 RETBLEED_CMD_IBPB, 914 RETBLEED_CMD_STUFF, 915 }; 916 917 static const char * const retbleed_strings[] = { 918 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 919 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 920 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 921 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 922 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 923 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 924 }; 925 926 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 927 RETBLEED_MITIGATION_NONE; 928 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = 929 RETBLEED_CMD_AUTO; 930 931 static int __ro_after_init retbleed_nosmt = false; 932 933 static int __init retbleed_parse_cmdline(char *str) 934 { 935 if (!str) 936 return -EINVAL; 937 938 while (str) { 939 char *next = strchr(str, ','); 940 if (next) { 941 *next = 0; 942 next++; 943 } 944 945 if (!strcmp(str, "off")) { 946 retbleed_cmd = RETBLEED_CMD_OFF; 947 } else if (!strcmp(str, "auto")) { 948 retbleed_cmd = RETBLEED_CMD_AUTO; 949 } else if (!strcmp(str, "unret")) { 950 retbleed_cmd = RETBLEED_CMD_UNRET; 951 } else if (!strcmp(str, "ibpb")) { 952 retbleed_cmd = RETBLEED_CMD_IBPB; 953 } else if (!strcmp(str, "stuff")) { 954 retbleed_cmd = RETBLEED_CMD_STUFF; 955 } else if (!strcmp(str, "nosmt")) { 956 retbleed_nosmt = true; 957 } else if (!strcmp(str, "force")) { 958 setup_force_cpu_bug(X86_BUG_RETBLEED); 959 } else { 960 pr_err("Ignoring unknown retbleed option (%s).", str); 961 } 962 963 str = next; 964 } 965 966 return 0; 967 } 968 early_param("retbleed", retbleed_parse_cmdline); 969 970 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 971 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 972 973 static void __init retbleed_select_mitigation(void) 974 { 975 bool mitigate_smt = false; 976 977 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) 978 return; 979 980 switch (retbleed_cmd) { 981 case RETBLEED_CMD_OFF: 982 return; 983 984 case RETBLEED_CMD_UNRET: 985 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) { 986 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 987 } else { 988 pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n"); 989 goto do_cmd_auto; 990 } 991 break; 992 993 case RETBLEED_CMD_IBPB: 994 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 995 pr_err("WARNING: CPU does not support IBPB.\n"); 996 goto do_cmd_auto; 997 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { 998 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 999 } else { 1000 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n"); 1001 goto do_cmd_auto; 1002 } 1003 break; 1004 1005 case RETBLEED_CMD_STUFF: 1006 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) && 1007 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { 1008 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1009 1010 } else { 1011 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING)) 1012 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); 1013 else 1014 pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n"); 1015 1016 goto do_cmd_auto; 1017 } 1018 break; 1019 1020 do_cmd_auto: 1021 case RETBLEED_CMD_AUTO: 1022 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1023 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1024 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) 1025 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1026 else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB)) 1027 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1028 } 1029 1030 /* 1031 * The Intel mitigation (IBRS or eIBRS) was already selected in 1032 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will 1033 * be set accordingly below. 1034 */ 1035 1036 break; 1037 } 1038 1039 switch (retbleed_mitigation) { 1040 case RETBLEED_MITIGATION_UNRET: 1041 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1042 setup_force_cpu_cap(X86_FEATURE_UNRET); 1043 1044 x86_return_thunk = retbleed_return_thunk; 1045 1046 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1047 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1048 pr_err(RETBLEED_UNTRAIN_MSG); 1049 1050 mitigate_smt = true; 1051 break; 1052 1053 case RETBLEED_MITIGATION_IBPB: 1054 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1055 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1056 mitigate_smt = true; 1057 break; 1058 1059 case RETBLEED_MITIGATION_STUFF: 1060 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1061 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1062 1063 x86_return_thunk = call_depth_return_thunk; 1064 break; 1065 1066 default: 1067 break; 1068 } 1069 1070 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1071 (retbleed_nosmt || cpu_mitigations_auto_nosmt())) 1072 cpu_smt_disable(false); 1073 1074 /* 1075 * Let IBRS trump all on Intel without affecting the effects of the 1076 * retbleed= cmdline option except for call depth based stuffing 1077 */ 1078 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1079 switch (spectre_v2_enabled) { 1080 case SPECTRE_V2_IBRS: 1081 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1082 break; 1083 case SPECTRE_V2_EIBRS: 1084 case SPECTRE_V2_EIBRS_RETPOLINE: 1085 case SPECTRE_V2_EIBRS_LFENCE: 1086 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1087 break; 1088 default: 1089 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 1090 pr_err(RETBLEED_INTEL_MSG); 1091 } 1092 } 1093 1094 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1095 } 1096 1097 #undef pr_fmt 1098 #define pr_fmt(fmt) "Spectre V2 : " fmt 1099 1100 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1101 SPECTRE_V2_USER_NONE; 1102 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1103 SPECTRE_V2_USER_NONE; 1104 1105 #ifdef CONFIG_RETPOLINE 1106 static bool spectre_v2_bad_module; 1107 1108 bool retpoline_module_ok(bool has_retpoline) 1109 { 1110 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1111 return true; 1112 1113 pr_err("System may be vulnerable to spectre v2\n"); 1114 spectre_v2_bad_module = true; 1115 return false; 1116 } 1117 1118 static inline const char *spectre_v2_module_string(void) 1119 { 1120 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1121 } 1122 #else 1123 static inline const char *spectre_v2_module_string(void) { return ""; } 1124 #endif 1125 1126 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1127 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1128 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1129 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1130 1131 #ifdef CONFIG_BPF_SYSCALL 1132 void unpriv_ebpf_notify(int new_state) 1133 { 1134 if (new_state) 1135 return; 1136 1137 /* Unprivileged eBPF is enabled */ 1138 1139 switch (spectre_v2_enabled) { 1140 case SPECTRE_V2_EIBRS: 1141 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1142 break; 1143 case SPECTRE_V2_EIBRS_LFENCE: 1144 if (sched_smt_active()) 1145 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1146 break; 1147 default: 1148 break; 1149 } 1150 } 1151 #endif 1152 1153 static inline bool match_option(const char *arg, int arglen, const char *opt) 1154 { 1155 int len = strlen(opt); 1156 1157 return len == arglen && !strncmp(arg, opt, len); 1158 } 1159 1160 /* The kernel command line selection for spectre v2 */ 1161 enum spectre_v2_mitigation_cmd { 1162 SPECTRE_V2_CMD_NONE, 1163 SPECTRE_V2_CMD_AUTO, 1164 SPECTRE_V2_CMD_FORCE, 1165 SPECTRE_V2_CMD_RETPOLINE, 1166 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1167 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1168 SPECTRE_V2_CMD_EIBRS, 1169 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1170 SPECTRE_V2_CMD_EIBRS_LFENCE, 1171 SPECTRE_V2_CMD_IBRS, 1172 }; 1173 1174 enum spectre_v2_user_cmd { 1175 SPECTRE_V2_USER_CMD_NONE, 1176 SPECTRE_V2_USER_CMD_AUTO, 1177 SPECTRE_V2_USER_CMD_FORCE, 1178 SPECTRE_V2_USER_CMD_PRCTL, 1179 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1180 SPECTRE_V2_USER_CMD_SECCOMP, 1181 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1182 }; 1183 1184 static const char * const spectre_v2_user_strings[] = { 1185 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1186 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1187 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1188 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1189 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1190 }; 1191 1192 static const struct { 1193 const char *option; 1194 enum spectre_v2_user_cmd cmd; 1195 bool secure; 1196 } v2_user_options[] __initconst = { 1197 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 1198 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 1199 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 1200 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 1201 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 1202 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 1203 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 1204 }; 1205 1206 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 1207 { 1208 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1209 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 1210 } 1211 1212 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; 1213 1214 static enum spectre_v2_user_cmd __init 1215 spectre_v2_parse_user_cmdline(void) 1216 { 1217 char arg[20]; 1218 int ret, i; 1219 1220 switch (spectre_v2_cmd) { 1221 case SPECTRE_V2_CMD_NONE: 1222 return SPECTRE_V2_USER_CMD_NONE; 1223 case SPECTRE_V2_CMD_FORCE: 1224 return SPECTRE_V2_USER_CMD_FORCE; 1225 default: 1226 break; 1227 } 1228 1229 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 1230 arg, sizeof(arg)); 1231 if (ret < 0) 1232 return SPECTRE_V2_USER_CMD_AUTO; 1233 1234 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 1235 if (match_option(arg, ret, v2_user_options[i].option)) { 1236 spec_v2_user_print_cond(v2_user_options[i].option, 1237 v2_user_options[i].secure); 1238 return v2_user_options[i].cmd; 1239 } 1240 } 1241 1242 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 1243 return SPECTRE_V2_USER_CMD_AUTO; 1244 } 1245 1246 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1247 { 1248 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1249 } 1250 1251 static void __init 1252 spectre_v2_user_select_mitigation(void) 1253 { 1254 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 1255 bool smt_possible = IS_ENABLED(CONFIG_SMP); 1256 enum spectre_v2_user_cmd cmd; 1257 1258 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1259 return; 1260 1261 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 1262 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 1263 smt_possible = false; 1264 1265 cmd = spectre_v2_parse_user_cmdline(); 1266 switch (cmd) { 1267 case SPECTRE_V2_USER_CMD_NONE: 1268 goto set_mode; 1269 case SPECTRE_V2_USER_CMD_FORCE: 1270 mode = SPECTRE_V2_USER_STRICT; 1271 break; 1272 case SPECTRE_V2_USER_CMD_AUTO: 1273 case SPECTRE_V2_USER_CMD_PRCTL: 1274 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1275 mode = SPECTRE_V2_USER_PRCTL; 1276 break; 1277 case SPECTRE_V2_USER_CMD_SECCOMP: 1278 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1279 if (IS_ENABLED(CONFIG_SECCOMP)) 1280 mode = SPECTRE_V2_USER_SECCOMP; 1281 else 1282 mode = SPECTRE_V2_USER_PRCTL; 1283 break; 1284 } 1285 1286 /* Initialize Indirect Branch Prediction Barrier */ 1287 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1288 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 1289 1290 spectre_v2_user_ibpb = mode; 1291 switch (cmd) { 1292 case SPECTRE_V2_USER_CMD_NONE: 1293 break; 1294 case SPECTRE_V2_USER_CMD_FORCE: 1295 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1296 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1297 static_branch_enable(&switch_mm_always_ibpb); 1298 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1299 break; 1300 case SPECTRE_V2_USER_CMD_PRCTL: 1301 case SPECTRE_V2_USER_CMD_AUTO: 1302 case SPECTRE_V2_USER_CMD_SECCOMP: 1303 static_branch_enable(&switch_mm_cond_ibpb); 1304 break; 1305 } 1306 1307 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1308 static_key_enabled(&switch_mm_always_ibpb) ? 1309 "always-on" : "conditional"); 1310 } 1311 1312 /* 1313 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1314 * is not required. 1315 * 1316 * Intel's Enhanced IBRS also protects against cross-thread branch target 1317 * injection in user-mode as the IBRS bit remains always set which 1318 * implicitly enables cross-thread protections. However, in legacy IBRS 1319 * mode, the IBRS bit is set only on kernel entry and cleared on return 1320 * to userspace. AMD Automatic IBRS also does not protect userspace. 1321 * These modes therefore disable the implicit cross-thread protection, 1322 * so allow for STIBP to be selected in those cases. 1323 */ 1324 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1325 !smt_possible || 1326 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1327 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) 1328 return; 1329 1330 /* 1331 * At this point, an STIBP mode other than "off" has been set. 1332 * If STIBP support is not being forced, check if STIBP always-on 1333 * is preferred. 1334 */ 1335 if (mode != SPECTRE_V2_USER_STRICT && 1336 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1337 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1338 1339 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1340 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 1341 if (mode != SPECTRE_V2_USER_STRICT && 1342 mode != SPECTRE_V2_USER_STRICT_PREFERRED) 1343 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1344 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1345 } 1346 1347 spectre_v2_user_stibp = mode; 1348 1349 set_mode: 1350 pr_info("%s\n", spectre_v2_user_strings[mode]); 1351 } 1352 1353 static const char * const spectre_v2_strings[] = { 1354 [SPECTRE_V2_NONE] = "Vulnerable", 1355 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1356 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 1357 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1358 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1359 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1360 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1361 }; 1362 1363 static const struct { 1364 const char *option; 1365 enum spectre_v2_mitigation_cmd cmd; 1366 bool secure; 1367 } mitigation_options[] __initconst = { 1368 { "off", SPECTRE_V2_CMD_NONE, false }, 1369 { "on", SPECTRE_V2_CMD_FORCE, true }, 1370 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 1371 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1372 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1373 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 1374 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 1375 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 1376 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 1377 { "auto", SPECTRE_V2_CMD_AUTO, false }, 1378 { "ibrs", SPECTRE_V2_CMD_IBRS, false }, 1379 }; 1380 1381 static void __init spec_v2_print_cond(const char *reason, bool secure) 1382 { 1383 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1384 pr_info("%s selected on command line.\n", reason); 1385 } 1386 1387 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 1388 { 1389 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 1390 char arg[20]; 1391 int ret, i; 1392 1393 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 1394 cpu_mitigations_off()) 1395 return SPECTRE_V2_CMD_NONE; 1396 1397 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 1398 if (ret < 0) 1399 return SPECTRE_V2_CMD_AUTO; 1400 1401 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 1402 if (!match_option(arg, ret, mitigation_options[i].option)) 1403 continue; 1404 cmd = mitigation_options[i].cmd; 1405 break; 1406 } 1407 1408 if (i >= ARRAY_SIZE(mitigation_options)) { 1409 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1410 return SPECTRE_V2_CMD_AUTO; 1411 } 1412 1413 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 1414 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1415 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 1416 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1417 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1418 !IS_ENABLED(CONFIG_RETPOLINE)) { 1419 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1420 mitigation_options[i].option); 1421 return SPECTRE_V2_CMD_AUTO; 1422 } 1423 1424 if ((cmd == SPECTRE_V2_CMD_EIBRS || 1425 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1426 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1427 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1428 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n", 1429 mitigation_options[i].option); 1430 return SPECTRE_V2_CMD_AUTO; 1431 } 1432 1433 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1434 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 1435 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 1436 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 1437 mitigation_options[i].option); 1438 return SPECTRE_V2_CMD_AUTO; 1439 } 1440 1441 if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) { 1442 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1443 mitigation_options[i].option); 1444 return SPECTRE_V2_CMD_AUTO; 1445 } 1446 1447 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1448 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n", 1449 mitigation_options[i].option); 1450 return SPECTRE_V2_CMD_AUTO; 1451 } 1452 1453 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 1454 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n", 1455 mitigation_options[i].option); 1456 return SPECTRE_V2_CMD_AUTO; 1457 } 1458 1459 if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 1460 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n", 1461 mitigation_options[i].option); 1462 return SPECTRE_V2_CMD_AUTO; 1463 } 1464 1465 spec_v2_print_cond(mitigation_options[i].option, 1466 mitigation_options[i].secure); 1467 return cmd; 1468 } 1469 1470 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1471 { 1472 if (!IS_ENABLED(CONFIG_RETPOLINE)) { 1473 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1474 return SPECTRE_V2_NONE; 1475 } 1476 1477 return SPECTRE_V2_RETPOLINE; 1478 } 1479 1480 /* Disable in-kernel use of non-RSB RET predictors */ 1481 static void __init spec_ctrl_disable_kernel_rrsba(void) 1482 { 1483 u64 ia32_cap; 1484 1485 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1486 return; 1487 1488 ia32_cap = x86_read_arch_cap_msr(); 1489 1490 if (ia32_cap & ARCH_CAP_RRSBA) { 1491 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1492 update_spec_ctrl(x86_spec_ctrl_base); 1493 } 1494 } 1495 1496 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode) 1497 { 1498 /* 1499 * Similar to context switches, there are two types of RSB attacks 1500 * after VM exit: 1501 * 1502 * 1) RSB underflow 1503 * 1504 * 2) Poisoned RSB entry 1505 * 1506 * When retpoline is enabled, both are mitigated by filling/clearing 1507 * the RSB. 1508 * 1509 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch 1510 * prediction isolation protections, RSB still needs to be cleared 1511 * because of #2. Note that SMEP provides no protection here, unlike 1512 * user-space-poisoned RSB entries. 1513 * 1514 * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB 1515 * bug is present then a LITE version of RSB protection is required, 1516 * just a single call needs to retire before a RET is executed. 1517 */ 1518 switch (mode) { 1519 case SPECTRE_V2_NONE: 1520 return; 1521 1522 case SPECTRE_V2_EIBRS_LFENCE: 1523 case SPECTRE_V2_EIBRS: 1524 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 1525 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 1526 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 1527 } 1528 return; 1529 1530 case SPECTRE_V2_EIBRS_RETPOLINE: 1531 case SPECTRE_V2_RETPOLINE: 1532 case SPECTRE_V2_LFENCE: 1533 case SPECTRE_V2_IBRS: 1534 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1535 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n"); 1536 return; 1537 } 1538 1539 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit"); 1540 dump_stack(); 1541 } 1542 1543 static void __init spectre_v2_select_mitigation(void) 1544 { 1545 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 1546 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 1547 1548 /* 1549 * If the CPU is not affected and the command line mode is NONE or AUTO 1550 * then nothing to do. 1551 */ 1552 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 1553 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 1554 return; 1555 1556 switch (cmd) { 1557 case SPECTRE_V2_CMD_NONE: 1558 return; 1559 1560 case SPECTRE_V2_CMD_FORCE: 1561 case SPECTRE_V2_CMD_AUTO: 1562 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1563 mode = SPECTRE_V2_EIBRS; 1564 break; 1565 } 1566 1567 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) && 1568 boot_cpu_has_bug(X86_BUG_RETBLEED) && 1569 retbleed_cmd != RETBLEED_CMD_OFF && 1570 retbleed_cmd != RETBLEED_CMD_STUFF && 1571 boot_cpu_has(X86_FEATURE_IBRS) && 1572 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1573 mode = SPECTRE_V2_IBRS; 1574 break; 1575 } 1576 1577 mode = spectre_v2_select_retpoline(); 1578 break; 1579 1580 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 1581 pr_err(SPECTRE_V2_LFENCE_MSG); 1582 mode = SPECTRE_V2_LFENCE; 1583 break; 1584 1585 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 1586 mode = SPECTRE_V2_RETPOLINE; 1587 break; 1588 1589 case SPECTRE_V2_CMD_RETPOLINE: 1590 mode = spectre_v2_select_retpoline(); 1591 break; 1592 1593 case SPECTRE_V2_CMD_IBRS: 1594 mode = SPECTRE_V2_IBRS; 1595 break; 1596 1597 case SPECTRE_V2_CMD_EIBRS: 1598 mode = SPECTRE_V2_EIBRS; 1599 break; 1600 1601 case SPECTRE_V2_CMD_EIBRS_LFENCE: 1602 mode = SPECTRE_V2_EIBRS_LFENCE; 1603 break; 1604 1605 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 1606 mode = SPECTRE_V2_EIBRS_RETPOLINE; 1607 break; 1608 } 1609 1610 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1611 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1612 1613 if (spectre_v2_in_ibrs_mode(mode)) { 1614 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 1615 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 1616 } else { 1617 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 1618 update_spec_ctrl(x86_spec_ctrl_base); 1619 } 1620 } 1621 1622 switch (mode) { 1623 case SPECTRE_V2_NONE: 1624 case SPECTRE_V2_EIBRS: 1625 break; 1626 1627 case SPECTRE_V2_IBRS: 1628 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 1629 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1630 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 1631 break; 1632 1633 case SPECTRE_V2_LFENCE: 1634 case SPECTRE_V2_EIBRS_LFENCE: 1635 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 1636 fallthrough; 1637 1638 case SPECTRE_V2_RETPOLINE: 1639 case SPECTRE_V2_EIBRS_RETPOLINE: 1640 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 1641 break; 1642 } 1643 1644 /* 1645 * Disable alternate RSB predictions in kernel when indirect CALLs and 1646 * JMPs gets protection against BHI and Intramode-BTI, but RET 1647 * prediction from a non-RSB predictor is still a risk. 1648 */ 1649 if (mode == SPECTRE_V2_EIBRS_LFENCE || 1650 mode == SPECTRE_V2_EIBRS_RETPOLINE || 1651 mode == SPECTRE_V2_RETPOLINE) 1652 spec_ctrl_disable_kernel_rrsba(); 1653 1654 spectre_v2_enabled = mode; 1655 pr_info("%s\n", spectre_v2_strings[mode]); 1656 1657 /* 1658 * If Spectre v2 protection has been enabled, fill the RSB during a 1659 * context switch. In general there are two types of RSB attacks 1660 * across context switches, for which the CALLs/RETs may be unbalanced. 1661 * 1662 * 1) RSB underflow 1663 * 1664 * Some Intel parts have "bottomless RSB". When the RSB is empty, 1665 * speculated return targets may come from the branch predictor, 1666 * which could have a user-poisoned BTB or BHB entry. 1667 * 1668 * AMD has it even worse: *all* returns are speculated from the BTB, 1669 * regardless of the state of the RSB. 1670 * 1671 * When IBRS or eIBRS is enabled, the "user -> kernel" attack 1672 * scenario is mitigated by the IBRS branch prediction isolation 1673 * properties, so the RSB buffer filling wouldn't be necessary to 1674 * protect against this type of attack. 1675 * 1676 * The "user -> user" attack scenario is mitigated by RSB filling. 1677 * 1678 * 2) Poisoned RSB entry 1679 * 1680 * If the 'next' in-kernel return stack is shorter than 'prev', 1681 * 'next' could be tricked into speculating with a user-poisoned RSB 1682 * entry. 1683 * 1684 * The "user -> kernel" attack scenario is mitigated by SMEP and 1685 * eIBRS. 1686 * 1687 * The "user -> user" scenario, also known as SpectreBHB, requires 1688 * RSB clearing. 1689 * 1690 * So to mitigate all cases, unconditionally fill RSB on context 1691 * switches. 1692 * 1693 * FIXME: Is this pointless for retbleed-affected AMD? 1694 */ 1695 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1696 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 1697 1698 spectre_v2_determine_rsb_fill_type_at_vmexit(mode); 1699 1700 /* 1701 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 1702 * and Enhanced IBRS protect firmware too, so enable IBRS around 1703 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 1704 * otherwise enabled. 1705 * 1706 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 1707 * the user might select retpoline on the kernel command line and if 1708 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 1709 * enable IBRS around firmware calls. 1710 */ 1711 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 1712 boot_cpu_has(X86_FEATURE_IBPB) && 1713 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1714 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 1715 1716 if (retbleed_cmd != RETBLEED_CMD_IBPB) { 1717 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 1718 pr_info("Enabling Speculation Barrier for firmware calls\n"); 1719 } 1720 1721 } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { 1722 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 1723 pr_info("Enabling Restricted Speculation for firmware calls\n"); 1724 } 1725 1726 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 1727 spectre_v2_cmd = cmd; 1728 } 1729 1730 static void update_stibp_msr(void * __unused) 1731 { 1732 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 1733 update_spec_ctrl(val); 1734 } 1735 1736 /* Update x86_spec_ctrl_base in case SMT state changed. */ 1737 static void update_stibp_strict(void) 1738 { 1739 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 1740 1741 if (sched_smt_active()) 1742 mask |= SPEC_CTRL_STIBP; 1743 1744 if (mask == x86_spec_ctrl_base) 1745 return; 1746 1747 pr_info("Update user space SMT mitigation: STIBP %s\n", 1748 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 1749 x86_spec_ctrl_base = mask; 1750 on_each_cpu(update_stibp_msr, NULL, 1); 1751 } 1752 1753 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 1754 static void update_indir_branch_cond(void) 1755 { 1756 if (sched_smt_active()) 1757 static_branch_enable(&switch_to_cond_stibp); 1758 else 1759 static_branch_disable(&switch_to_cond_stibp); 1760 } 1761 1762 #undef pr_fmt 1763 #define pr_fmt(fmt) fmt 1764 1765 /* Update the static key controlling the MDS CPU buffer clear in idle */ 1766 static void update_mds_branch_idle(void) 1767 { 1768 u64 ia32_cap = x86_read_arch_cap_msr(); 1769 1770 /* 1771 * Enable the idle clearing if SMT is active on CPUs which are 1772 * affected only by MSBDS and not any other MDS variant. 1773 * 1774 * The other variants cannot be mitigated when SMT is enabled, so 1775 * clearing the buffers on idle just to prevent the Store Buffer 1776 * repartitioning leak would be a window dressing exercise. 1777 */ 1778 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 1779 return; 1780 1781 if (sched_smt_active()) { 1782 static_branch_enable(&mds_idle_clear); 1783 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 1784 (ia32_cap & ARCH_CAP_FBSDP_NO)) { 1785 static_branch_disable(&mds_idle_clear); 1786 } 1787 } 1788 1789 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 1790 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 1791 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 1792 1793 void cpu_bugs_smt_update(void) 1794 { 1795 mutex_lock(&spec_ctrl_mutex); 1796 1797 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1798 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1799 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1800 1801 switch (spectre_v2_user_stibp) { 1802 case SPECTRE_V2_USER_NONE: 1803 break; 1804 case SPECTRE_V2_USER_STRICT: 1805 case SPECTRE_V2_USER_STRICT_PREFERRED: 1806 update_stibp_strict(); 1807 break; 1808 case SPECTRE_V2_USER_PRCTL: 1809 case SPECTRE_V2_USER_SECCOMP: 1810 update_indir_branch_cond(); 1811 break; 1812 } 1813 1814 switch (mds_mitigation) { 1815 case MDS_MITIGATION_FULL: 1816 case MDS_MITIGATION_VMWERV: 1817 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 1818 pr_warn_once(MDS_MSG_SMT); 1819 update_mds_branch_idle(); 1820 break; 1821 case MDS_MITIGATION_OFF: 1822 break; 1823 } 1824 1825 switch (taa_mitigation) { 1826 case TAA_MITIGATION_VERW: 1827 case TAA_MITIGATION_UCODE_NEEDED: 1828 if (sched_smt_active()) 1829 pr_warn_once(TAA_MSG_SMT); 1830 break; 1831 case TAA_MITIGATION_TSX_DISABLED: 1832 case TAA_MITIGATION_OFF: 1833 break; 1834 } 1835 1836 switch (mmio_mitigation) { 1837 case MMIO_MITIGATION_VERW: 1838 case MMIO_MITIGATION_UCODE_NEEDED: 1839 if (sched_smt_active()) 1840 pr_warn_once(MMIO_MSG_SMT); 1841 break; 1842 case MMIO_MITIGATION_OFF: 1843 break; 1844 } 1845 1846 mutex_unlock(&spec_ctrl_mutex); 1847 } 1848 1849 #undef pr_fmt 1850 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 1851 1852 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 1853 1854 /* The kernel command line selection */ 1855 enum ssb_mitigation_cmd { 1856 SPEC_STORE_BYPASS_CMD_NONE, 1857 SPEC_STORE_BYPASS_CMD_AUTO, 1858 SPEC_STORE_BYPASS_CMD_ON, 1859 SPEC_STORE_BYPASS_CMD_PRCTL, 1860 SPEC_STORE_BYPASS_CMD_SECCOMP, 1861 }; 1862 1863 static const char * const ssb_strings[] = { 1864 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 1865 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 1866 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 1867 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 1868 }; 1869 1870 static const struct { 1871 const char *option; 1872 enum ssb_mitigation_cmd cmd; 1873 } ssb_mitigation_options[] __initconst = { 1874 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 1875 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 1876 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 1877 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 1878 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 1879 }; 1880 1881 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 1882 { 1883 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 1884 char arg[20]; 1885 int ret, i; 1886 1887 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 1888 cpu_mitigations_off()) { 1889 return SPEC_STORE_BYPASS_CMD_NONE; 1890 } else { 1891 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 1892 arg, sizeof(arg)); 1893 if (ret < 0) 1894 return SPEC_STORE_BYPASS_CMD_AUTO; 1895 1896 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1897 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1898 continue; 1899 1900 cmd = ssb_mitigation_options[i].cmd; 1901 break; 1902 } 1903 1904 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1905 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1906 return SPEC_STORE_BYPASS_CMD_AUTO; 1907 } 1908 } 1909 1910 return cmd; 1911 } 1912 1913 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1914 { 1915 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1916 enum ssb_mitigation_cmd cmd; 1917 1918 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1919 return mode; 1920 1921 cmd = ssb_parse_cmdline(); 1922 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1923 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1924 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1925 return mode; 1926 1927 switch (cmd) { 1928 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1929 /* 1930 * Choose prctl+seccomp as the default mode if seccomp is 1931 * enabled. 1932 */ 1933 if (IS_ENABLED(CONFIG_SECCOMP)) 1934 mode = SPEC_STORE_BYPASS_SECCOMP; 1935 else 1936 mode = SPEC_STORE_BYPASS_PRCTL; 1937 break; 1938 case SPEC_STORE_BYPASS_CMD_ON: 1939 mode = SPEC_STORE_BYPASS_DISABLE; 1940 break; 1941 case SPEC_STORE_BYPASS_CMD_AUTO: 1942 case SPEC_STORE_BYPASS_CMD_PRCTL: 1943 mode = SPEC_STORE_BYPASS_PRCTL; 1944 break; 1945 case SPEC_STORE_BYPASS_CMD_NONE: 1946 break; 1947 } 1948 1949 /* 1950 * We have three CPU feature flags that are in play here: 1951 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1952 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1953 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1954 */ 1955 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1956 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1957 /* 1958 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1959 * use a completely different MSR and bit dependent on family. 1960 */ 1961 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1962 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1963 x86_amd_ssb_disable(); 1964 } else { 1965 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1966 update_spec_ctrl(x86_spec_ctrl_base); 1967 } 1968 } 1969 1970 return mode; 1971 } 1972 1973 static void ssb_select_mitigation(void) 1974 { 1975 ssb_mode = __ssb_select_mitigation(); 1976 1977 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1978 pr_info("%s\n", ssb_strings[ssb_mode]); 1979 } 1980 1981 #undef pr_fmt 1982 #define pr_fmt(fmt) "Speculation prctl: " fmt 1983 1984 static void task_update_spec_tif(struct task_struct *tsk) 1985 { 1986 /* Force the update of the real TIF bits */ 1987 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1988 1989 /* 1990 * Immediately update the speculation control MSRs for the current 1991 * task, but for a non-current task delay setting the CPU 1992 * mitigation until it is scheduled next. 1993 * 1994 * This can only happen for SECCOMP mitigation. For PRCTL it's 1995 * always the current task. 1996 */ 1997 if (tsk == current) 1998 speculation_ctrl_update_current(); 1999 } 2000 2001 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 2002 { 2003 2004 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2005 return -EPERM; 2006 2007 switch (ctrl) { 2008 case PR_SPEC_ENABLE: 2009 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2010 return 0; 2011 case PR_SPEC_DISABLE: 2012 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2013 return 0; 2014 default: 2015 return -ERANGE; 2016 } 2017 } 2018 2019 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2020 { 2021 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2022 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2023 return -ENXIO; 2024 2025 switch (ctrl) { 2026 case PR_SPEC_ENABLE: 2027 /* If speculation is force disabled, enable is not allowed */ 2028 if (task_spec_ssb_force_disable(task)) 2029 return -EPERM; 2030 task_clear_spec_ssb_disable(task); 2031 task_clear_spec_ssb_noexec(task); 2032 task_update_spec_tif(task); 2033 break; 2034 case PR_SPEC_DISABLE: 2035 task_set_spec_ssb_disable(task); 2036 task_clear_spec_ssb_noexec(task); 2037 task_update_spec_tif(task); 2038 break; 2039 case PR_SPEC_FORCE_DISABLE: 2040 task_set_spec_ssb_disable(task); 2041 task_set_spec_ssb_force_disable(task); 2042 task_clear_spec_ssb_noexec(task); 2043 task_update_spec_tif(task); 2044 break; 2045 case PR_SPEC_DISABLE_NOEXEC: 2046 if (task_spec_ssb_force_disable(task)) 2047 return -EPERM; 2048 task_set_spec_ssb_disable(task); 2049 task_set_spec_ssb_noexec(task); 2050 task_update_spec_tif(task); 2051 break; 2052 default: 2053 return -ERANGE; 2054 } 2055 return 0; 2056 } 2057 2058 static bool is_spec_ib_user_controlled(void) 2059 { 2060 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2061 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2062 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2063 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2064 } 2065 2066 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2067 { 2068 switch (ctrl) { 2069 case PR_SPEC_ENABLE: 2070 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2071 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2072 return 0; 2073 2074 /* 2075 * With strict mode for both IBPB and STIBP, the instruction 2076 * code paths avoid checking this task flag and instead, 2077 * unconditionally run the instruction. However, STIBP and IBPB 2078 * are independent and either can be set to conditionally 2079 * enabled regardless of the mode of the other. 2080 * 2081 * If either is set to conditional, allow the task flag to be 2082 * updated, unless it was force-disabled by a previous prctl 2083 * call. Currently, this is possible on an AMD CPU which has the 2084 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2085 * kernel is booted with 'spectre_v2_user=seccomp', then 2086 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2087 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2088 */ 2089 if (!is_spec_ib_user_controlled() || 2090 task_spec_ib_force_disable(task)) 2091 return -EPERM; 2092 2093 task_clear_spec_ib_disable(task); 2094 task_update_spec_tif(task); 2095 break; 2096 case PR_SPEC_DISABLE: 2097 case PR_SPEC_FORCE_DISABLE: 2098 /* 2099 * Indirect branch speculation is always allowed when 2100 * mitigation is force disabled. 2101 */ 2102 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2103 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2104 return -EPERM; 2105 2106 if (!is_spec_ib_user_controlled()) 2107 return 0; 2108 2109 task_set_spec_ib_disable(task); 2110 if (ctrl == PR_SPEC_FORCE_DISABLE) 2111 task_set_spec_ib_force_disable(task); 2112 task_update_spec_tif(task); 2113 if (task == current) 2114 indirect_branch_prediction_barrier(); 2115 break; 2116 default: 2117 return -ERANGE; 2118 } 2119 return 0; 2120 } 2121 2122 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2123 unsigned long ctrl) 2124 { 2125 switch (which) { 2126 case PR_SPEC_STORE_BYPASS: 2127 return ssb_prctl_set(task, ctrl); 2128 case PR_SPEC_INDIRECT_BRANCH: 2129 return ib_prctl_set(task, ctrl); 2130 case PR_SPEC_L1D_FLUSH: 2131 return l1d_flush_prctl_set(task, ctrl); 2132 default: 2133 return -ENODEV; 2134 } 2135 } 2136 2137 #ifdef CONFIG_SECCOMP 2138 void arch_seccomp_spec_mitigate(struct task_struct *task) 2139 { 2140 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2141 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2142 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2143 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2144 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2145 } 2146 #endif 2147 2148 static int l1d_flush_prctl_get(struct task_struct *task) 2149 { 2150 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2151 return PR_SPEC_FORCE_DISABLE; 2152 2153 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2154 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2155 else 2156 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2157 } 2158 2159 static int ssb_prctl_get(struct task_struct *task) 2160 { 2161 switch (ssb_mode) { 2162 case SPEC_STORE_BYPASS_NONE: 2163 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2164 return PR_SPEC_ENABLE; 2165 return PR_SPEC_NOT_AFFECTED; 2166 case SPEC_STORE_BYPASS_DISABLE: 2167 return PR_SPEC_DISABLE; 2168 case SPEC_STORE_BYPASS_SECCOMP: 2169 case SPEC_STORE_BYPASS_PRCTL: 2170 if (task_spec_ssb_force_disable(task)) 2171 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2172 if (task_spec_ssb_noexec(task)) 2173 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2174 if (task_spec_ssb_disable(task)) 2175 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2176 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2177 } 2178 BUG(); 2179 } 2180 2181 static int ib_prctl_get(struct task_struct *task) 2182 { 2183 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2184 return PR_SPEC_NOT_AFFECTED; 2185 2186 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2187 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2188 return PR_SPEC_ENABLE; 2189 else if (is_spec_ib_user_controlled()) { 2190 if (task_spec_ib_force_disable(task)) 2191 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2192 if (task_spec_ib_disable(task)) 2193 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2194 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2195 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2196 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2197 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2198 return PR_SPEC_DISABLE; 2199 else 2200 return PR_SPEC_NOT_AFFECTED; 2201 } 2202 2203 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2204 { 2205 switch (which) { 2206 case PR_SPEC_STORE_BYPASS: 2207 return ssb_prctl_get(task); 2208 case PR_SPEC_INDIRECT_BRANCH: 2209 return ib_prctl_get(task); 2210 case PR_SPEC_L1D_FLUSH: 2211 return l1d_flush_prctl_get(task); 2212 default: 2213 return -ENODEV; 2214 } 2215 } 2216 2217 void x86_spec_ctrl_setup_ap(void) 2218 { 2219 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2220 update_spec_ctrl(x86_spec_ctrl_base); 2221 2222 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2223 x86_amd_ssb_disable(); 2224 } 2225 2226 bool itlb_multihit_kvm_mitigation; 2227 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 2228 2229 #undef pr_fmt 2230 #define pr_fmt(fmt) "L1TF: " fmt 2231 2232 /* Default mitigation for L1TF-affected CPUs */ 2233 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 2234 #if IS_ENABLED(CONFIG_KVM_INTEL) 2235 EXPORT_SYMBOL_GPL(l1tf_mitigation); 2236 #endif 2237 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2238 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 2239 2240 /* 2241 * These CPUs all support 44bits physical address space internally in the 2242 * cache but CPUID can report a smaller number of physical address bits. 2243 * 2244 * The L1TF mitigation uses the top most address bit for the inversion of 2245 * non present PTEs. When the installed memory reaches into the top most 2246 * address bit due to memory holes, which has been observed on machines 2247 * which report 36bits physical address bits and have 32G RAM installed, 2248 * then the mitigation range check in l1tf_select_mitigation() triggers. 2249 * This is a false positive because the mitigation is still possible due to 2250 * the fact that the cache uses 44bit internally. Use the cache bits 2251 * instead of the reported physical bits and adjust them on the affected 2252 * machines to 44bit if the reported bits are less than 44. 2253 */ 2254 static void override_cache_bits(struct cpuinfo_x86 *c) 2255 { 2256 if (c->x86 != 6) 2257 return; 2258 2259 switch (c->x86_model) { 2260 case INTEL_FAM6_NEHALEM: 2261 case INTEL_FAM6_WESTMERE: 2262 case INTEL_FAM6_SANDYBRIDGE: 2263 case INTEL_FAM6_IVYBRIDGE: 2264 case INTEL_FAM6_HASWELL: 2265 case INTEL_FAM6_HASWELL_L: 2266 case INTEL_FAM6_HASWELL_G: 2267 case INTEL_FAM6_BROADWELL: 2268 case INTEL_FAM6_BROADWELL_G: 2269 case INTEL_FAM6_SKYLAKE_L: 2270 case INTEL_FAM6_SKYLAKE: 2271 case INTEL_FAM6_KABYLAKE_L: 2272 case INTEL_FAM6_KABYLAKE: 2273 if (c->x86_cache_bits < 44) 2274 c->x86_cache_bits = 44; 2275 break; 2276 } 2277 } 2278 2279 static void __init l1tf_select_mitigation(void) 2280 { 2281 u64 half_pa; 2282 2283 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2284 return; 2285 2286 if (cpu_mitigations_off()) 2287 l1tf_mitigation = L1TF_MITIGATION_OFF; 2288 else if (cpu_mitigations_auto_nosmt()) 2289 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2290 2291 override_cache_bits(&boot_cpu_data); 2292 2293 switch (l1tf_mitigation) { 2294 case L1TF_MITIGATION_OFF: 2295 case L1TF_MITIGATION_FLUSH_NOWARN: 2296 case L1TF_MITIGATION_FLUSH: 2297 break; 2298 case L1TF_MITIGATION_FLUSH_NOSMT: 2299 case L1TF_MITIGATION_FULL: 2300 cpu_smt_disable(false); 2301 break; 2302 case L1TF_MITIGATION_FULL_FORCE: 2303 cpu_smt_disable(true); 2304 break; 2305 } 2306 2307 #if CONFIG_PGTABLE_LEVELS == 2 2308 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2309 return; 2310 #endif 2311 2312 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2313 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2314 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2315 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2316 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2317 half_pa); 2318 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2319 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2320 return; 2321 } 2322 2323 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2324 } 2325 2326 static int __init l1tf_cmdline(char *str) 2327 { 2328 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2329 return 0; 2330 2331 if (!str) 2332 return -EINVAL; 2333 2334 if (!strcmp(str, "off")) 2335 l1tf_mitigation = L1TF_MITIGATION_OFF; 2336 else if (!strcmp(str, "flush,nowarn")) 2337 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2338 else if (!strcmp(str, "flush")) 2339 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2340 else if (!strcmp(str, "flush,nosmt")) 2341 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2342 else if (!strcmp(str, "full")) 2343 l1tf_mitigation = L1TF_MITIGATION_FULL; 2344 else if (!strcmp(str, "full,force")) 2345 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2346 2347 return 0; 2348 } 2349 early_param("l1tf", l1tf_cmdline); 2350 2351 #undef pr_fmt 2352 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 2353 2354 enum srso_mitigation { 2355 SRSO_MITIGATION_NONE, 2356 SRSO_MITIGATION_UCODE_NEEDED, 2357 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 2358 SRSO_MITIGATION_MICROCODE, 2359 SRSO_MITIGATION_SAFE_RET, 2360 SRSO_MITIGATION_IBPB, 2361 SRSO_MITIGATION_IBPB_ON_VMEXIT, 2362 }; 2363 2364 enum srso_mitigation_cmd { 2365 SRSO_CMD_OFF, 2366 SRSO_CMD_MICROCODE, 2367 SRSO_CMD_SAFE_RET, 2368 SRSO_CMD_IBPB, 2369 SRSO_CMD_IBPB_ON_VMEXIT, 2370 }; 2371 2372 static const char * const srso_strings[] = { 2373 [SRSO_MITIGATION_NONE] = "Vulnerable", 2374 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 2375 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 2376 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 2377 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 2378 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 2379 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only" 2380 }; 2381 2382 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE; 2383 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET; 2384 2385 static int __init srso_parse_cmdline(char *str) 2386 { 2387 if (!str) 2388 return -EINVAL; 2389 2390 if (!strcmp(str, "off")) 2391 srso_cmd = SRSO_CMD_OFF; 2392 else if (!strcmp(str, "microcode")) 2393 srso_cmd = SRSO_CMD_MICROCODE; 2394 else if (!strcmp(str, "safe-ret")) 2395 srso_cmd = SRSO_CMD_SAFE_RET; 2396 else if (!strcmp(str, "ibpb")) 2397 srso_cmd = SRSO_CMD_IBPB; 2398 else if (!strcmp(str, "ibpb-vmexit")) 2399 srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT; 2400 else 2401 pr_err("Ignoring unknown SRSO option (%s).", str); 2402 2403 return 0; 2404 } 2405 early_param("spec_rstack_overflow", srso_parse_cmdline); 2406 2407 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 2408 2409 static void __init srso_select_mitigation(void) 2410 { 2411 bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); 2412 2413 if (cpu_mitigations_off()) 2414 return; 2415 2416 if (!boot_cpu_has_bug(X86_BUG_SRSO)) { 2417 if (boot_cpu_has(X86_FEATURE_SBPB)) 2418 x86_pred_cmd = PRED_CMD_SBPB; 2419 return; 2420 } 2421 2422 if (has_microcode) { 2423 /* 2424 * Zen1/2 with SMT off aren't vulnerable after the right 2425 * IBPB microcode has been applied. 2426 * 2427 * Zen1/2 don't have SBPB, no need to try to enable it here. 2428 */ 2429 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 2430 setup_force_cpu_cap(X86_FEATURE_SRSO_NO); 2431 return; 2432 } 2433 2434 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2435 srso_mitigation = SRSO_MITIGATION_IBPB; 2436 goto out; 2437 } 2438 } else { 2439 pr_warn("IBPB-extending microcode not applied!\n"); 2440 pr_warn(SRSO_NOTICE); 2441 2442 /* may be overwritten by SRSO_CMD_SAFE_RET below */ 2443 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 2444 } 2445 2446 switch (srso_cmd) { 2447 case SRSO_CMD_OFF: 2448 if (boot_cpu_has(X86_FEATURE_SBPB)) 2449 x86_pred_cmd = PRED_CMD_SBPB; 2450 return; 2451 2452 case SRSO_CMD_MICROCODE: 2453 if (has_microcode) { 2454 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2455 pr_warn(SRSO_NOTICE); 2456 } 2457 break; 2458 2459 case SRSO_CMD_SAFE_RET: 2460 if (IS_ENABLED(CONFIG_CPU_SRSO)) { 2461 /* 2462 * Enable the return thunk for generated code 2463 * like ftrace, static_call, etc. 2464 */ 2465 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 2466 setup_force_cpu_cap(X86_FEATURE_UNRET); 2467 2468 if (boot_cpu_data.x86 == 0x19) { 2469 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 2470 x86_return_thunk = srso_alias_return_thunk; 2471 } else { 2472 setup_force_cpu_cap(X86_FEATURE_SRSO); 2473 x86_return_thunk = srso_return_thunk; 2474 } 2475 if (has_microcode) 2476 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2477 else 2478 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 2479 } else { 2480 pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); 2481 } 2482 break; 2483 2484 case SRSO_CMD_IBPB: 2485 if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { 2486 if (has_microcode) { 2487 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 2488 srso_mitigation = SRSO_MITIGATION_IBPB; 2489 } 2490 } else { 2491 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n"); 2492 } 2493 break; 2494 2495 case SRSO_CMD_IBPB_ON_VMEXIT: 2496 if (IS_ENABLED(CONFIG_CPU_SRSO)) { 2497 if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) { 2498 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 2499 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2500 } 2501 } else { 2502 pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); 2503 } 2504 break; 2505 } 2506 2507 out: 2508 pr_info("%s\n", srso_strings[srso_mitigation]); 2509 } 2510 2511 #undef pr_fmt 2512 #define pr_fmt(fmt) fmt 2513 2514 #ifdef CONFIG_SYSFS 2515 2516 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 2517 2518 #if IS_ENABLED(CONFIG_KVM_INTEL) 2519 static const char * const l1tf_vmx_states[] = { 2520 [VMENTER_L1D_FLUSH_AUTO] = "auto", 2521 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 2522 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 2523 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 2524 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 2525 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 2526 }; 2527 2528 static ssize_t l1tf_show_state(char *buf) 2529 { 2530 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 2531 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2532 2533 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 2534 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 2535 sched_smt_active())) { 2536 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 2537 l1tf_vmx_states[l1tf_vmx_mitigation]); 2538 } 2539 2540 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 2541 l1tf_vmx_states[l1tf_vmx_mitigation], 2542 sched_smt_active() ? "vulnerable" : "disabled"); 2543 } 2544 2545 static ssize_t itlb_multihit_show_state(char *buf) 2546 { 2547 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2548 !boot_cpu_has(X86_FEATURE_VMX)) 2549 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 2550 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 2551 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 2552 else if (itlb_multihit_kvm_mitigation) 2553 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 2554 else 2555 return sysfs_emit(buf, "KVM: Vulnerable\n"); 2556 } 2557 #else 2558 static ssize_t l1tf_show_state(char *buf) 2559 { 2560 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2561 } 2562 2563 static ssize_t itlb_multihit_show_state(char *buf) 2564 { 2565 return sysfs_emit(buf, "Processor vulnerable\n"); 2566 } 2567 #endif 2568 2569 static ssize_t mds_show_state(char *buf) 2570 { 2571 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2572 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2573 mds_strings[mds_mitigation]); 2574 } 2575 2576 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 2577 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2578 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 2579 sched_smt_active() ? "mitigated" : "disabled")); 2580 } 2581 2582 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2583 sched_smt_active() ? "vulnerable" : "disabled"); 2584 } 2585 2586 static ssize_t tsx_async_abort_show_state(char *buf) 2587 { 2588 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 2589 (taa_mitigation == TAA_MITIGATION_OFF)) 2590 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 2591 2592 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2593 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2594 taa_strings[taa_mitigation]); 2595 } 2596 2597 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 2598 sched_smt_active() ? "vulnerable" : "disabled"); 2599 } 2600 2601 static ssize_t mmio_stale_data_show_state(char *buf) 2602 { 2603 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2604 return sysfs_emit(buf, "Unknown: No mitigations\n"); 2605 2606 if (mmio_mitigation == MMIO_MITIGATION_OFF) 2607 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 2608 2609 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2610 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2611 mmio_strings[mmio_mitigation]); 2612 } 2613 2614 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 2615 sched_smt_active() ? "vulnerable" : "disabled"); 2616 } 2617 2618 static char *stibp_state(void) 2619 { 2620 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 2621 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 2622 return ""; 2623 2624 switch (spectre_v2_user_stibp) { 2625 case SPECTRE_V2_USER_NONE: 2626 return ", STIBP: disabled"; 2627 case SPECTRE_V2_USER_STRICT: 2628 return ", STIBP: forced"; 2629 case SPECTRE_V2_USER_STRICT_PREFERRED: 2630 return ", STIBP: always-on"; 2631 case SPECTRE_V2_USER_PRCTL: 2632 case SPECTRE_V2_USER_SECCOMP: 2633 if (static_key_enabled(&switch_to_cond_stibp)) 2634 return ", STIBP: conditional"; 2635 } 2636 return ""; 2637 } 2638 2639 static char *ibpb_state(void) 2640 { 2641 if (boot_cpu_has(X86_FEATURE_IBPB)) { 2642 if (static_key_enabled(&switch_mm_always_ibpb)) 2643 return ", IBPB: always-on"; 2644 if (static_key_enabled(&switch_mm_cond_ibpb)) 2645 return ", IBPB: conditional"; 2646 return ", IBPB: disabled"; 2647 } 2648 return ""; 2649 } 2650 2651 static char *pbrsb_eibrs_state(void) 2652 { 2653 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2654 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 2655 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 2656 return ", PBRSB-eIBRS: SW sequence"; 2657 else 2658 return ", PBRSB-eIBRS: Vulnerable"; 2659 } else { 2660 return ", PBRSB-eIBRS: Not affected"; 2661 } 2662 } 2663 2664 static ssize_t spectre_v2_show_state(char *buf) 2665 { 2666 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 2667 return sysfs_emit(buf, "Vulnerable: LFENCE\n"); 2668 2669 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2670 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 2671 2672 if (sched_smt_active() && unprivileged_ebpf_enabled() && 2673 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 2674 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 2675 2676 return sysfs_emit(buf, "%s%s%s%s%s%s%s\n", 2677 spectre_v2_strings[spectre_v2_enabled], 2678 ibpb_state(), 2679 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 2680 stibp_state(), 2681 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 2682 pbrsb_eibrs_state(), 2683 spectre_v2_module_string()); 2684 } 2685 2686 static ssize_t srbds_show_state(char *buf) 2687 { 2688 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 2689 } 2690 2691 static ssize_t retbleed_show_state(char *buf) 2692 { 2693 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 2694 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2695 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 2696 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 2697 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 2698 2699 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 2700 !sched_smt_active() ? "disabled" : 2701 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2702 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 2703 "enabled with STIBP protection" : "vulnerable"); 2704 } 2705 2706 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 2707 } 2708 2709 static ssize_t srso_show_state(char *buf) 2710 { 2711 if (boot_cpu_has(X86_FEATURE_SRSO_NO)) 2712 return sysfs_emit(buf, "Mitigation: SMT disabled\n"); 2713 2714 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 2715 } 2716 2717 static ssize_t gds_show_state(char *buf) 2718 { 2719 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 2720 } 2721 2722 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 2723 char *buf, unsigned int bug) 2724 { 2725 if (!boot_cpu_has_bug(bug)) 2726 return sysfs_emit(buf, "Not affected\n"); 2727 2728 switch (bug) { 2729 case X86_BUG_CPU_MELTDOWN: 2730 if (boot_cpu_has(X86_FEATURE_PTI)) 2731 return sysfs_emit(buf, "Mitigation: PTI\n"); 2732 2733 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 2734 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 2735 2736 break; 2737 2738 case X86_BUG_SPECTRE_V1: 2739 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 2740 2741 case X86_BUG_SPECTRE_V2: 2742 return spectre_v2_show_state(buf); 2743 2744 case X86_BUG_SPEC_STORE_BYPASS: 2745 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 2746 2747 case X86_BUG_L1TF: 2748 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 2749 return l1tf_show_state(buf); 2750 break; 2751 2752 case X86_BUG_MDS: 2753 return mds_show_state(buf); 2754 2755 case X86_BUG_TAA: 2756 return tsx_async_abort_show_state(buf); 2757 2758 case X86_BUG_ITLB_MULTIHIT: 2759 return itlb_multihit_show_state(buf); 2760 2761 case X86_BUG_SRBDS: 2762 return srbds_show_state(buf); 2763 2764 case X86_BUG_MMIO_STALE_DATA: 2765 case X86_BUG_MMIO_UNKNOWN: 2766 return mmio_stale_data_show_state(buf); 2767 2768 case X86_BUG_RETBLEED: 2769 return retbleed_show_state(buf); 2770 2771 case X86_BUG_SRSO: 2772 return srso_show_state(buf); 2773 2774 case X86_BUG_GDS: 2775 return gds_show_state(buf); 2776 2777 default: 2778 break; 2779 } 2780 2781 return sysfs_emit(buf, "Vulnerable\n"); 2782 } 2783 2784 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 2785 { 2786 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 2787 } 2788 2789 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 2790 { 2791 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 2792 } 2793 2794 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 2795 { 2796 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 2797 } 2798 2799 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 2800 { 2801 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 2802 } 2803 2804 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 2805 { 2806 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 2807 } 2808 2809 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 2810 { 2811 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 2812 } 2813 2814 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 2815 { 2816 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 2817 } 2818 2819 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 2820 { 2821 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 2822 } 2823 2824 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 2825 { 2826 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 2827 } 2828 2829 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 2830 { 2831 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2832 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); 2833 else 2834 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 2835 } 2836 2837 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 2838 { 2839 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 2840 } 2841 2842 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 2843 { 2844 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 2845 } 2846 2847 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 2848 { 2849 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 2850 } 2851 #endif 2852