1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 20 #include <asm/spec-ctrl.h> 21 #include <asm/cmdline.h> 22 #include <asm/bugs.h> 23 #include <asm/processor.h> 24 #include <asm/processor-flags.h> 25 #include <asm/fpu/api.h> 26 #include <asm/msr.h> 27 #include <asm/vmx.h> 28 #include <asm/paravirt.h> 29 #include <asm/cpu_device_id.h> 30 #include <asm/e820/api.h> 31 #include <asm/hypervisor.h> 32 #include <asm/tlbflush.h> 33 #include <asm/cpu.h> 34 35 #include "cpu.h" 36 37 static void __init spectre_v1_select_mitigation(void); 38 static void __init spectre_v2_select_mitigation(void); 39 static void __init retbleed_select_mitigation(void); 40 static void __init spectre_v2_user_select_mitigation(void); 41 static void __init ssb_select_mitigation(void); 42 static void __init l1tf_select_mitigation(void); 43 static void __init mds_select_mitigation(void); 44 static void __init md_clear_update_mitigation(void); 45 static void __init md_clear_select_mitigation(void); 46 static void __init taa_select_mitigation(void); 47 static void __init mmio_select_mitigation(void); 48 static void __init srbds_select_mitigation(void); 49 static void __init l1d_flush_select_mitigation(void); 50 static void __init srso_select_mitigation(void); 51 static void __init gds_select_mitigation(void); 52 53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 54 u64 x86_spec_ctrl_base; 55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 56 57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 59 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current); 60 61 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 62 63 static u64 __ro_after_init x86_arch_cap_msr; 64 65 static DEFINE_MUTEX(spec_ctrl_mutex); 66 67 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 68 69 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 70 static void update_spec_ctrl(u64 val) 71 { 72 this_cpu_write(x86_spec_ctrl_current, val); 73 wrmsrl(MSR_IA32_SPEC_CTRL, val); 74 } 75 76 /* 77 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 78 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 79 */ 80 void update_spec_ctrl_cond(u64 val) 81 { 82 if (this_cpu_read(x86_spec_ctrl_current) == val) 83 return; 84 85 this_cpu_write(x86_spec_ctrl_current, val); 86 87 /* 88 * When KERNEL_IBRS this MSR is written on return-to-user, unless 89 * forced the update can be delayed until that time. 90 */ 91 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 92 wrmsrl(MSR_IA32_SPEC_CTRL, val); 93 } 94 95 noinstr u64 spec_ctrl_current(void) 96 { 97 return this_cpu_read(x86_spec_ctrl_current); 98 } 99 EXPORT_SYMBOL_GPL(spec_ctrl_current); 100 101 /* 102 * AMD specific MSR info for Speculative Store Bypass control. 103 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 104 */ 105 u64 __ro_after_init x86_amd_ls_cfg_base; 106 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 107 108 /* Control conditional STIBP in switch_to() */ 109 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 110 /* Control conditional IBPB in switch_mm() */ 111 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 112 /* Control unconditional IBPB in switch_mm() */ 113 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 114 115 /* Control IBPB on vCPU load */ 116 DEFINE_STATIC_KEY_FALSE(switch_vcpu_ibpb); 117 EXPORT_SYMBOL_GPL(switch_vcpu_ibpb); 118 119 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 120 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 121 EXPORT_SYMBOL_GPL(mds_idle_clear); 122 123 /* 124 * Controls whether l1d flush based mitigations are enabled, 125 * based on hw features and admin setting via boot parameter 126 * defaults to false 127 */ 128 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 129 130 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ 131 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); 132 EXPORT_SYMBOL_GPL(mmio_stale_data_clear); 133 134 void __init cpu_select_mitigations(void) 135 { 136 /* 137 * Read the SPEC_CTRL MSR to account for reserved bits which may 138 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 139 * init code as it is not enumerated and depends on the family. 140 */ 141 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 142 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 143 144 /* 145 * Previously running kernel (kexec), may have some controls 146 * turned ON. Clear them and let the mitigations setup below 147 * rediscover them based on configuration. 148 */ 149 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 150 } 151 152 x86_arch_cap_msr = x86_read_arch_cap_msr(); 153 154 /* Select the proper CPU mitigations before patching alternatives: */ 155 spectre_v1_select_mitigation(); 156 spectre_v2_select_mitigation(); 157 /* 158 * retbleed_select_mitigation() relies on the state set by 159 * spectre_v2_select_mitigation(); specifically it wants to know about 160 * spectre_v2=ibrs. 161 */ 162 retbleed_select_mitigation(); 163 /* 164 * spectre_v2_user_select_mitigation() relies on the state set by 165 * retbleed_select_mitigation(); specifically the STIBP selection is 166 * forced for UNRET or IBPB. 167 */ 168 spectre_v2_user_select_mitigation(); 169 ssb_select_mitigation(); 170 l1tf_select_mitigation(); 171 md_clear_select_mitigation(); 172 srbds_select_mitigation(); 173 l1d_flush_select_mitigation(); 174 175 /* 176 * srso_select_mitigation() depends and must run after 177 * retbleed_select_mitigation(). 178 */ 179 srso_select_mitigation(); 180 gds_select_mitigation(); 181 } 182 183 /* 184 * NOTE: This function is *only* called for SVM, since Intel uses 185 * MSR_IA32_SPEC_CTRL for SSBD. 186 */ 187 void 188 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 189 { 190 u64 guestval, hostval; 191 struct thread_info *ti = current_thread_info(); 192 193 /* 194 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 195 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 196 */ 197 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 198 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 199 return; 200 201 /* 202 * If the host has SSBD mitigation enabled, force it in the host's 203 * virtual MSR value. If its not permanently enabled, evaluate 204 * current's TIF_SSBD thread flag. 205 */ 206 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 207 hostval = SPEC_CTRL_SSBD; 208 else 209 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 210 211 /* Sanitize the guest value */ 212 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 213 214 if (hostval != guestval) { 215 unsigned long tif; 216 217 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 218 ssbd_spec_ctrl_to_tif(hostval); 219 220 speculation_ctrl_update(tif); 221 } 222 } 223 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 224 225 static void x86_amd_ssb_disable(void) 226 { 227 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 228 229 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 230 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 231 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 232 wrmsrl(MSR_AMD64_LS_CFG, msrval); 233 } 234 235 #undef pr_fmt 236 #define pr_fmt(fmt) "MDS: " fmt 237 238 /* Default mitigation for MDS-affected CPUs */ 239 static enum mds_mitigations mds_mitigation __ro_after_init = 240 IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_AUTO : MDS_MITIGATION_OFF; 241 static bool mds_nosmt __ro_after_init = false; 242 243 static const char * const mds_strings[] = { 244 [MDS_MITIGATION_OFF] = "Vulnerable", 245 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 246 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 247 }; 248 249 enum taa_mitigations { 250 TAA_MITIGATION_OFF, 251 TAA_MITIGATION_AUTO, 252 TAA_MITIGATION_UCODE_NEEDED, 253 TAA_MITIGATION_VERW, 254 TAA_MITIGATION_TSX_DISABLED, 255 }; 256 257 /* Default mitigation for TAA-affected CPUs */ 258 static enum taa_mitigations taa_mitigation __ro_after_init = 259 IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_AUTO : TAA_MITIGATION_OFF; 260 261 enum mmio_mitigations { 262 MMIO_MITIGATION_OFF, 263 MMIO_MITIGATION_AUTO, 264 MMIO_MITIGATION_UCODE_NEEDED, 265 MMIO_MITIGATION_VERW, 266 }; 267 268 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 269 static enum mmio_mitigations mmio_mitigation __ro_after_init = 270 IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ? MMIO_MITIGATION_AUTO : MMIO_MITIGATION_OFF; 271 272 enum rfds_mitigations { 273 RFDS_MITIGATION_OFF, 274 RFDS_MITIGATION_AUTO, 275 RFDS_MITIGATION_VERW, 276 RFDS_MITIGATION_UCODE_NEEDED, 277 }; 278 279 /* Default mitigation for Register File Data Sampling */ 280 static enum rfds_mitigations rfds_mitigation __ro_after_init = 281 IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF; 282 283 static void __init mds_select_mitigation(void) 284 { 285 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 286 mds_mitigation = MDS_MITIGATION_OFF; 287 return; 288 } 289 290 if (mds_mitigation == MDS_MITIGATION_AUTO) 291 mds_mitigation = MDS_MITIGATION_FULL; 292 293 if (mds_mitigation == MDS_MITIGATION_FULL) { 294 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 295 mds_mitigation = MDS_MITIGATION_VMWERV; 296 297 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 298 299 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 300 (mds_nosmt || cpu_mitigations_auto_nosmt())) 301 cpu_smt_disable(false); 302 } 303 } 304 305 static int __init mds_cmdline(char *str) 306 { 307 if (!boot_cpu_has_bug(X86_BUG_MDS)) 308 return 0; 309 310 if (!str) 311 return -EINVAL; 312 313 if (!strcmp(str, "off")) 314 mds_mitigation = MDS_MITIGATION_OFF; 315 else if (!strcmp(str, "full")) 316 mds_mitigation = MDS_MITIGATION_FULL; 317 else if (!strcmp(str, "full,nosmt")) { 318 mds_mitigation = MDS_MITIGATION_FULL; 319 mds_nosmt = true; 320 } 321 322 return 0; 323 } 324 early_param("mds", mds_cmdline); 325 326 #undef pr_fmt 327 #define pr_fmt(fmt) "TAA: " fmt 328 329 static bool taa_nosmt __ro_after_init; 330 331 static const char * const taa_strings[] = { 332 [TAA_MITIGATION_OFF] = "Vulnerable", 333 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 334 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 335 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 336 }; 337 338 static void __init taa_select_mitigation(void) 339 { 340 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 341 taa_mitigation = TAA_MITIGATION_OFF; 342 return; 343 } 344 345 /* TSX previously disabled by tsx=off */ 346 if (!boot_cpu_has(X86_FEATURE_RTM)) { 347 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 348 return; 349 } 350 351 if (cpu_mitigations_off()) { 352 taa_mitigation = TAA_MITIGATION_OFF; 353 return; 354 } 355 356 /* 357 * TAA mitigation via VERW is turned off if both 358 * tsx_async_abort=off and mds=off are specified. 359 */ 360 if (taa_mitigation == TAA_MITIGATION_OFF && 361 mds_mitigation == MDS_MITIGATION_OFF) 362 return; 363 364 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 365 taa_mitigation = TAA_MITIGATION_VERW; 366 else 367 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 368 369 /* 370 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 371 * A microcode update fixes this behavior to clear CPU buffers. It also 372 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 373 * ARCH_CAP_TSX_CTRL_MSR bit. 374 * 375 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 376 * update is required. 377 */ 378 if ( (x86_arch_cap_msr & ARCH_CAP_MDS_NO) && 379 !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) 380 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 381 382 /* 383 * TSX is enabled, select alternate mitigation for TAA which is 384 * the same as MDS. Enable MDS static branch to clear CPU buffers. 385 * 386 * For guests that can't determine whether the correct microcode is 387 * present on host, enable the mitigation for UCODE_NEEDED as well. 388 */ 389 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 390 391 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 392 cpu_smt_disable(false); 393 } 394 395 static int __init tsx_async_abort_parse_cmdline(char *str) 396 { 397 if (!boot_cpu_has_bug(X86_BUG_TAA)) 398 return 0; 399 400 if (!str) 401 return -EINVAL; 402 403 if (!strcmp(str, "off")) { 404 taa_mitigation = TAA_MITIGATION_OFF; 405 } else if (!strcmp(str, "full")) { 406 taa_mitigation = TAA_MITIGATION_VERW; 407 } else if (!strcmp(str, "full,nosmt")) { 408 taa_mitigation = TAA_MITIGATION_VERW; 409 taa_nosmt = true; 410 } 411 412 return 0; 413 } 414 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 415 416 #undef pr_fmt 417 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 418 419 static bool mmio_nosmt __ro_after_init = false; 420 421 static const char * const mmio_strings[] = { 422 [MMIO_MITIGATION_OFF] = "Vulnerable", 423 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 424 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 425 }; 426 427 static void __init mmio_select_mitigation(void) 428 { 429 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || 430 boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || 431 cpu_mitigations_off()) { 432 mmio_mitigation = MMIO_MITIGATION_OFF; 433 return; 434 } 435 436 if (mmio_mitigation == MMIO_MITIGATION_OFF) 437 return; 438 439 /* 440 * Enable CPU buffer clear mitigation for host and VMM, if also affected 441 * by MDS or TAA. Otherwise, enable mitigation for VMM only. 442 */ 443 if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && 444 boot_cpu_has(X86_FEATURE_RTM))) 445 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 446 447 /* 448 * X86_FEATURE_CLEAR_CPU_BUF could be enabled by other VERW based 449 * mitigations, disable KVM-only mitigation in that case. 450 */ 451 if (boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) 452 static_branch_disable(&mmio_stale_data_clear); 453 else 454 static_branch_enable(&mmio_stale_data_clear); 455 456 /* 457 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 458 * be propagated to uncore buffers, clearing the Fill buffers on idle 459 * is required irrespective of SMT state. 460 */ 461 if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) 462 static_branch_enable(&mds_idle_clear); 463 464 /* 465 * Check if the system has the right microcode. 466 * 467 * CPU Fill buffer clear mitigation is enumerated by either an explicit 468 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 469 * affected systems. 470 */ 471 if ((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || 472 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 473 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 474 !(x86_arch_cap_msr & ARCH_CAP_MDS_NO))) 475 mmio_mitigation = MMIO_MITIGATION_VERW; 476 else 477 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 478 479 if (mmio_nosmt || cpu_mitigations_auto_nosmt()) 480 cpu_smt_disable(false); 481 } 482 483 static int __init mmio_stale_data_parse_cmdline(char *str) 484 { 485 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 486 return 0; 487 488 if (!str) 489 return -EINVAL; 490 491 if (!strcmp(str, "off")) { 492 mmio_mitigation = MMIO_MITIGATION_OFF; 493 } else if (!strcmp(str, "full")) { 494 mmio_mitigation = MMIO_MITIGATION_VERW; 495 } else if (!strcmp(str, "full,nosmt")) { 496 mmio_mitigation = MMIO_MITIGATION_VERW; 497 mmio_nosmt = true; 498 } 499 500 return 0; 501 } 502 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 503 504 #undef pr_fmt 505 #define pr_fmt(fmt) "Register File Data Sampling: " fmt 506 507 static const char * const rfds_strings[] = { 508 [RFDS_MITIGATION_OFF] = "Vulnerable", 509 [RFDS_MITIGATION_VERW] = "Mitigation: Clear Register File", 510 [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 511 }; 512 513 static void __init rfds_select_mitigation(void) 514 { 515 if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) { 516 rfds_mitigation = RFDS_MITIGATION_OFF; 517 return; 518 } 519 if (rfds_mitigation == RFDS_MITIGATION_OFF) 520 return; 521 522 if (rfds_mitigation == RFDS_MITIGATION_AUTO) 523 rfds_mitigation = RFDS_MITIGATION_VERW; 524 525 if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR) 526 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 527 else 528 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; 529 } 530 531 static __init int rfds_parse_cmdline(char *str) 532 { 533 if (!str) 534 return -EINVAL; 535 536 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 537 return 0; 538 539 if (!strcmp(str, "off")) 540 rfds_mitigation = RFDS_MITIGATION_OFF; 541 else if (!strcmp(str, "on")) 542 rfds_mitigation = RFDS_MITIGATION_VERW; 543 544 return 0; 545 } 546 early_param("reg_file_data_sampling", rfds_parse_cmdline); 547 548 #undef pr_fmt 549 #define pr_fmt(fmt) "" fmt 550 551 static void __init md_clear_update_mitigation(void) 552 { 553 if (cpu_mitigations_off()) 554 return; 555 556 if (!boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) 557 goto out; 558 559 /* 560 * X86_FEATURE_CLEAR_CPU_BUF is now enabled. Update MDS, TAA and MMIO 561 * Stale Data mitigation, if necessary. 562 */ 563 if (mds_mitigation == MDS_MITIGATION_OFF && 564 boot_cpu_has_bug(X86_BUG_MDS)) { 565 mds_mitigation = MDS_MITIGATION_FULL; 566 mds_select_mitigation(); 567 } 568 if (taa_mitigation == TAA_MITIGATION_OFF && 569 boot_cpu_has_bug(X86_BUG_TAA)) { 570 taa_mitigation = TAA_MITIGATION_VERW; 571 taa_select_mitigation(); 572 } 573 /* 574 * MMIO_MITIGATION_OFF is not checked here so that mmio_stale_data_clear 575 * gets updated correctly as per X86_FEATURE_CLEAR_CPU_BUF state. 576 */ 577 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 578 mmio_mitigation = MMIO_MITIGATION_VERW; 579 mmio_select_mitigation(); 580 } 581 if (rfds_mitigation == RFDS_MITIGATION_OFF && 582 boot_cpu_has_bug(X86_BUG_RFDS)) { 583 rfds_mitigation = RFDS_MITIGATION_VERW; 584 rfds_select_mitigation(); 585 } 586 out: 587 if (boot_cpu_has_bug(X86_BUG_MDS)) 588 pr_info("MDS: %s\n", mds_strings[mds_mitigation]); 589 if (boot_cpu_has_bug(X86_BUG_TAA)) 590 pr_info("TAA: %s\n", taa_strings[taa_mitigation]); 591 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 592 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); 593 else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 594 pr_info("MMIO Stale Data: Unknown: No mitigations\n"); 595 if (boot_cpu_has_bug(X86_BUG_RFDS)) 596 pr_info("Register File Data Sampling: %s\n", rfds_strings[rfds_mitigation]); 597 } 598 599 static void __init md_clear_select_mitigation(void) 600 { 601 mds_select_mitigation(); 602 taa_select_mitigation(); 603 mmio_select_mitigation(); 604 rfds_select_mitigation(); 605 606 /* 607 * As these mitigations are inter-related and rely on VERW instruction 608 * to clear the microarchitural buffers, update and print their status 609 * after mitigation selection is done for each of these vulnerabilities. 610 */ 611 md_clear_update_mitigation(); 612 } 613 614 #undef pr_fmt 615 #define pr_fmt(fmt) "SRBDS: " fmt 616 617 enum srbds_mitigations { 618 SRBDS_MITIGATION_OFF, 619 SRBDS_MITIGATION_UCODE_NEEDED, 620 SRBDS_MITIGATION_FULL, 621 SRBDS_MITIGATION_TSX_OFF, 622 SRBDS_MITIGATION_HYPERVISOR, 623 }; 624 625 static enum srbds_mitigations srbds_mitigation __ro_after_init = 626 IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_FULL : SRBDS_MITIGATION_OFF; 627 628 static const char * const srbds_strings[] = { 629 [SRBDS_MITIGATION_OFF] = "Vulnerable", 630 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 631 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 632 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 633 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 634 }; 635 636 static bool srbds_off; 637 638 void update_srbds_msr(void) 639 { 640 u64 mcu_ctrl; 641 642 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 643 return; 644 645 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 646 return; 647 648 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 649 return; 650 651 /* 652 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 653 * being disabled and it hasn't received the SRBDS MSR microcode. 654 */ 655 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 656 return; 657 658 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 659 660 switch (srbds_mitigation) { 661 case SRBDS_MITIGATION_OFF: 662 case SRBDS_MITIGATION_TSX_OFF: 663 mcu_ctrl |= RNGDS_MITG_DIS; 664 break; 665 case SRBDS_MITIGATION_FULL: 666 mcu_ctrl &= ~RNGDS_MITG_DIS; 667 break; 668 default: 669 break; 670 } 671 672 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 673 } 674 675 static void __init srbds_select_mitigation(void) 676 { 677 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 678 return; 679 680 /* 681 * Check to see if this is one of the MDS_NO systems supporting TSX that 682 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 683 * by Processor MMIO Stale Data vulnerability. 684 */ 685 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 686 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 687 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 688 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 689 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 690 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 691 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 692 else if (cpu_mitigations_off() || srbds_off) 693 srbds_mitigation = SRBDS_MITIGATION_OFF; 694 695 update_srbds_msr(); 696 pr_info("%s\n", srbds_strings[srbds_mitigation]); 697 } 698 699 static int __init srbds_parse_cmdline(char *str) 700 { 701 if (!str) 702 return -EINVAL; 703 704 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 705 return 0; 706 707 srbds_off = !strcmp(str, "off"); 708 return 0; 709 } 710 early_param("srbds", srbds_parse_cmdline); 711 712 #undef pr_fmt 713 #define pr_fmt(fmt) "L1D Flush : " fmt 714 715 enum l1d_flush_mitigations { 716 L1D_FLUSH_OFF = 0, 717 L1D_FLUSH_ON, 718 }; 719 720 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 721 722 static void __init l1d_flush_select_mitigation(void) 723 { 724 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 725 return; 726 727 static_branch_enable(&switch_mm_cond_l1d_flush); 728 pr_info("Conditional flush on switch_mm() enabled\n"); 729 } 730 731 static int __init l1d_flush_parse_cmdline(char *str) 732 { 733 if (!strcmp(str, "on")) 734 l1d_flush_mitigation = L1D_FLUSH_ON; 735 736 return 0; 737 } 738 early_param("l1d_flush", l1d_flush_parse_cmdline); 739 740 #undef pr_fmt 741 #define pr_fmt(fmt) "GDS: " fmt 742 743 enum gds_mitigations { 744 GDS_MITIGATION_OFF, 745 GDS_MITIGATION_UCODE_NEEDED, 746 GDS_MITIGATION_FORCE, 747 GDS_MITIGATION_FULL, 748 GDS_MITIGATION_FULL_LOCKED, 749 GDS_MITIGATION_HYPERVISOR, 750 }; 751 752 static enum gds_mitigations gds_mitigation __ro_after_init = 753 IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_FULL : GDS_MITIGATION_OFF; 754 755 static const char * const gds_strings[] = { 756 [GDS_MITIGATION_OFF] = "Vulnerable", 757 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 758 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 759 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 760 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 761 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 762 }; 763 764 bool gds_ucode_mitigated(void) 765 { 766 return (gds_mitigation == GDS_MITIGATION_FULL || 767 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 768 } 769 EXPORT_SYMBOL_GPL(gds_ucode_mitigated); 770 771 void update_gds_msr(void) 772 { 773 u64 mcu_ctrl_after; 774 u64 mcu_ctrl; 775 776 switch (gds_mitigation) { 777 case GDS_MITIGATION_OFF: 778 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 779 mcu_ctrl |= GDS_MITG_DIS; 780 break; 781 case GDS_MITIGATION_FULL_LOCKED: 782 /* 783 * The LOCKED state comes from the boot CPU. APs might not have 784 * the same state. Make sure the mitigation is enabled on all 785 * CPUs. 786 */ 787 case GDS_MITIGATION_FULL: 788 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 789 mcu_ctrl &= ~GDS_MITG_DIS; 790 break; 791 case GDS_MITIGATION_FORCE: 792 case GDS_MITIGATION_UCODE_NEEDED: 793 case GDS_MITIGATION_HYPERVISOR: 794 return; 795 } 796 797 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 798 799 /* 800 * Check to make sure that the WRMSR value was not ignored. Writes to 801 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 802 * processor was not. 803 */ 804 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 805 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 806 } 807 808 static void __init gds_select_mitigation(void) 809 { 810 u64 mcu_ctrl; 811 812 if (!boot_cpu_has_bug(X86_BUG_GDS)) 813 return; 814 815 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 816 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 817 goto out; 818 } 819 820 if (cpu_mitigations_off()) 821 gds_mitigation = GDS_MITIGATION_OFF; 822 /* Will verify below that mitigation _can_ be disabled */ 823 824 /* No microcode */ 825 if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) { 826 if (gds_mitigation == GDS_MITIGATION_FORCE) { 827 /* 828 * This only needs to be done on the boot CPU so do it 829 * here rather than in update_gds_msr() 830 */ 831 setup_clear_cpu_cap(X86_FEATURE_AVX); 832 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 833 } else { 834 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 835 } 836 goto out; 837 } 838 839 /* Microcode has mitigation, use it */ 840 if (gds_mitigation == GDS_MITIGATION_FORCE) 841 gds_mitigation = GDS_MITIGATION_FULL; 842 843 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 844 if (mcu_ctrl & GDS_MITG_LOCKED) { 845 if (gds_mitigation == GDS_MITIGATION_OFF) 846 pr_warn("Mitigation locked. Disable failed.\n"); 847 848 /* 849 * The mitigation is selected from the boot CPU. All other CPUs 850 * _should_ have the same state. If the boot CPU isn't locked 851 * but others are then update_gds_msr() will WARN() of the state 852 * mismatch. If the boot CPU is locked update_gds_msr() will 853 * ensure the other CPUs have the mitigation enabled. 854 */ 855 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 856 } 857 858 update_gds_msr(); 859 out: 860 pr_info("%s\n", gds_strings[gds_mitigation]); 861 } 862 863 static int __init gds_parse_cmdline(char *str) 864 { 865 if (!str) 866 return -EINVAL; 867 868 if (!boot_cpu_has_bug(X86_BUG_GDS)) 869 return 0; 870 871 if (!strcmp(str, "off")) 872 gds_mitigation = GDS_MITIGATION_OFF; 873 else if (!strcmp(str, "force")) 874 gds_mitigation = GDS_MITIGATION_FORCE; 875 876 return 0; 877 } 878 early_param("gather_data_sampling", gds_parse_cmdline); 879 880 #undef pr_fmt 881 #define pr_fmt(fmt) "Spectre V1 : " fmt 882 883 enum spectre_v1_mitigation { 884 SPECTRE_V1_MITIGATION_NONE, 885 SPECTRE_V1_MITIGATION_AUTO, 886 }; 887 888 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 889 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ? 890 SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE; 891 892 static const char * const spectre_v1_strings[] = { 893 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 894 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 895 }; 896 897 /* 898 * Does SMAP provide full mitigation against speculative kernel access to 899 * userspace? 900 */ 901 static bool smap_works_speculatively(void) 902 { 903 if (!boot_cpu_has(X86_FEATURE_SMAP)) 904 return false; 905 906 /* 907 * On CPUs which are vulnerable to Meltdown, SMAP does not 908 * prevent speculative access to user data in the L1 cache. 909 * Consider SMAP to be non-functional as a mitigation on these 910 * CPUs. 911 */ 912 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 913 return false; 914 915 return true; 916 } 917 918 static void __init spectre_v1_select_mitigation(void) 919 { 920 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 921 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 922 return; 923 } 924 925 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 926 /* 927 * With Spectre v1, a user can speculatively control either 928 * path of a conditional swapgs with a user-controlled GS 929 * value. The mitigation is to add lfences to both code paths. 930 * 931 * If FSGSBASE is enabled, the user can put a kernel address in 932 * GS, in which case SMAP provides no protection. 933 * 934 * If FSGSBASE is disabled, the user can only put a user space 935 * address in GS. That makes an attack harder, but still 936 * possible if there's no SMAP protection. 937 */ 938 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 939 !smap_works_speculatively()) { 940 /* 941 * Mitigation can be provided from SWAPGS itself or 942 * PTI as the CR3 write in the Meltdown mitigation 943 * is serializing. 944 * 945 * If neither is there, mitigate with an LFENCE to 946 * stop speculation through swapgs. 947 */ 948 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 949 !boot_cpu_has(X86_FEATURE_PTI)) 950 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 951 952 /* 953 * Enable lfences in the kernel entry (non-swapgs) 954 * paths, to prevent user entry from speculatively 955 * skipping swapgs. 956 */ 957 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 958 } 959 } 960 961 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 962 } 963 964 static int __init nospectre_v1_cmdline(char *str) 965 { 966 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 967 return 0; 968 } 969 early_param("nospectre_v1", nospectre_v1_cmdline); 970 971 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 972 973 #undef pr_fmt 974 #define pr_fmt(fmt) "RETBleed: " fmt 975 976 enum retbleed_mitigation { 977 RETBLEED_MITIGATION_NONE, 978 RETBLEED_MITIGATION_UNRET, 979 RETBLEED_MITIGATION_IBPB, 980 RETBLEED_MITIGATION_IBRS, 981 RETBLEED_MITIGATION_EIBRS, 982 RETBLEED_MITIGATION_STUFF, 983 }; 984 985 enum retbleed_mitigation_cmd { 986 RETBLEED_CMD_OFF, 987 RETBLEED_CMD_AUTO, 988 RETBLEED_CMD_UNRET, 989 RETBLEED_CMD_IBPB, 990 RETBLEED_CMD_STUFF, 991 }; 992 993 static const char * const retbleed_strings[] = { 994 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 995 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 996 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 997 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 998 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 999 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 1000 }; 1001 1002 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 1003 RETBLEED_MITIGATION_NONE; 1004 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = 1005 IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_CMD_AUTO : RETBLEED_CMD_OFF; 1006 1007 static int __ro_after_init retbleed_nosmt = false; 1008 1009 static int __init retbleed_parse_cmdline(char *str) 1010 { 1011 if (!str) 1012 return -EINVAL; 1013 1014 while (str) { 1015 char *next = strchr(str, ','); 1016 if (next) { 1017 *next = 0; 1018 next++; 1019 } 1020 1021 if (!strcmp(str, "off")) { 1022 retbleed_cmd = RETBLEED_CMD_OFF; 1023 } else if (!strcmp(str, "auto")) { 1024 retbleed_cmd = RETBLEED_CMD_AUTO; 1025 } else if (!strcmp(str, "unret")) { 1026 retbleed_cmd = RETBLEED_CMD_UNRET; 1027 } else if (!strcmp(str, "ibpb")) { 1028 retbleed_cmd = RETBLEED_CMD_IBPB; 1029 } else if (!strcmp(str, "stuff")) { 1030 retbleed_cmd = RETBLEED_CMD_STUFF; 1031 } else if (!strcmp(str, "nosmt")) { 1032 retbleed_nosmt = true; 1033 } else if (!strcmp(str, "force")) { 1034 setup_force_cpu_bug(X86_BUG_RETBLEED); 1035 } else { 1036 pr_err("Ignoring unknown retbleed option (%s).", str); 1037 } 1038 1039 str = next; 1040 } 1041 1042 return 0; 1043 } 1044 early_param("retbleed", retbleed_parse_cmdline); 1045 1046 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 1047 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 1048 1049 static void __init retbleed_select_mitigation(void) 1050 { 1051 bool mitigate_smt = false; 1052 1053 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) 1054 return; 1055 1056 switch (retbleed_cmd) { 1057 case RETBLEED_CMD_OFF: 1058 return; 1059 1060 case RETBLEED_CMD_UNRET: 1061 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { 1062 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1063 } else { 1064 pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); 1065 goto do_cmd_auto; 1066 } 1067 break; 1068 1069 case RETBLEED_CMD_IBPB: 1070 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 1071 pr_err("WARNING: CPU does not support IBPB.\n"); 1072 goto do_cmd_auto; 1073 } else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 1074 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1075 } else { 1076 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 1077 goto do_cmd_auto; 1078 } 1079 break; 1080 1081 case RETBLEED_CMD_STUFF: 1082 if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) && 1083 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { 1084 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1085 1086 } else { 1087 if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) 1088 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); 1089 else 1090 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); 1091 1092 goto do_cmd_auto; 1093 } 1094 break; 1095 1096 do_cmd_auto: 1097 case RETBLEED_CMD_AUTO: 1098 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1099 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1100 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) 1101 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1102 else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && 1103 boot_cpu_has(X86_FEATURE_IBPB)) 1104 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1105 } 1106 1107 /* 1108 * The Intel mitigation (IBRS or eIBRS) was already selected in 1109 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will 1110 * be set accordingly below. 1111 */ 1112 1113 break; 1114 } 1115 1116 switch (retbleed_mitigation) { 1117 case RETBLEED_MITIGATION_UNRET: 1118 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1119 setup_force_cpu_cap(X86_FEATURE_UNRET); 1120 1121 x86_return_thunk = retbleed_return_thunk; 1122 1123 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1124 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1125 pr_err(RETBLEED_UNTRAIN_MSG); 1126 1127 mitigate_smt = true; 1128 break; 1129 1130 case RETBLEED_MITIGATION_IBPB: 1131 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1132 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1133 mitigate_smt = true; 1134 1135 /* 1136 * IBPB on entry already obviates the need for 1137 * software-based untraining so clear those in case some 1138 * other mitigation like SRSO has selected them. 1139 */ 1140 setup_clear_cpu_cap(X86_FEATURE_UNRET); 1141 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 1142 1143 /* 1144 * There is no need for RSB filling: write_ibpb() ensures 1145 * all predictions, including the RSB, are invalidated, 1146 * regardless of IBPB implementation. 1147 */ 1148 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1149 1150 break; 1151 1152 case RETBLEED_MITIGATION_STUFF: 1153 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1154 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1155 1156 x86_return_thunk = call_depth_return_thunk; 1157 break; 1158 1159 default: 1160 break; 1161 } 1162 1163 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1164 (retbleed_nosmt || cpu_mitigations_auto_nosmt())) 1165 cpu_smt_disable(false); 1166 1167 /* 1168 * Let IBRS trump all on Intel without affecting the effects of the 1169 * retbleed= cmdline option except for call depth based stuffing 1170 */ 1171 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1172 switch (spectre_v2_enabled) { 1173 case SPECTRE_V2_IBRS: 1174 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1175 break; 1176 case SPECTRE_V2_EIBRS: 1177 case SPECTRE_V2_EIBRS_RETPOLINE: 1178 case SPECTRE_V2_EIBRS_LFENCE: 1179 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1180 break; 1181 default: 1182 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 1183 pr_err(RETBLEED_INTEL_MSG); 1184 } 1185 } 1186 1187 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1188 } 1189 1190 #undef pr_fmt 1191 #define pr_fmt(fmt) "Spectre V2 : " fmt 1192 1193 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1194 SPECTRE_V2_USER_NONE; 1195 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1196 SPECTRE_V2_USER_NONE; 1197 1198 #ifdef CONFIG_MITIGATION_RETPOLINE 1199 static bool spectre_v2_bad_module; 1200 1201 bool retpoline_module_ok(bool has_retpoline) 1202 { 1203 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1204 return true; 1205 1206 pr_err("System may be vulnerable to spectre v2\n"); 1207 spectre_v2_bad_module = true; 1208 return false; 1209 } 1210 1211 static inline const char *spectre_v2_module_string(void) 1212 { 1213 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1214 } 1215 #else 1216 static inline const char *spectre_v2_module_string(void) { return ""; } 1217 #endif 1218 1219 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1220 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1221 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1222 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1223 1224 #ifdef CONFIG_BPF_SYSCALL 1225 void unpriv_ebpf_notify(int new_state) 1226 { 1227 if (new_state) 1228 return; 1229 1230 /* Unprivileged eBPF is enabled */ 1231 1232 switch (spectre_v2_enabled) { 1233 case SPECTRE_V2_EIBRS: 1234 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1235 break; 1236 case SPECTRE_V2_EIBRS_LFENCE: 1237 if (sched_smt_active()) 1238 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1239 break; 1240 default: 1241 break; 1242 } 1243 } 1244 #endif 1245 1246 static inline bool match_option(const char *arg, int arglen, const char *opt) 1247 { 1248 int len = strlen(opt); 1249 1250 return len == arglen && !strncmp(arg, opt, len); 1251 } 1252 1253 /* The kernel command line selection for spectre v2 */ 1254 enum spectre_v2_mitigation_cmd { 1255 SPECTRE_V2_CMD_NONE, 1256 SPECTRE_V2_CMD_AUTO, 1257 SPECTRE_V2_CMD_FORCE, 1258 SPECTRE_V2_CMD_RETPOLINE, 1259 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1260 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1261 SPECTRE_V2_CMD_EIBRS, 1262 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1263 SPECTRE_V2_CMD_EIBRS_LFENCE, 1264 SPECTRE_V2_CMD_IBRS, 1265 }; 1266 1267 enum spectre_v2_user_cmd { 1268 SPECTRE_V2_USER_CMD_NONE, 1269 SPECTRE_V2_USER_CMD_AUTO, 1270 SPECTRE_V2_USER_CMD_FORCE, 1271 SPECTRE_V2_USER_CMD_PRCTL, 1272 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1273 SPECTRE_V2_USER_CMD_SECCOMP, 1274 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1275 }; 1276 1277 static const char * const spectre_v2_user_strings[] = { 1278 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1279 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1280 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1281 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1282 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1283 }; 1284 1285 static const struct { 1286 const char *option; 1287 enum spectre_v2_user_cmd cmd; 1288 bool secure; 1289 } v2_user_options[] __initconst = { 1290 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 1291 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 1292 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 1293 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 1294 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 1295 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 1296 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 1297 }; 1298 1299 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 1300 { 1301 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1302 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 1303 } 1304 1305 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; 1306 1307 static enum spectre_v2_user_cmd __init 1308 spectre_v2_parse_user_cmdline(void) 1309 { 1310 enum spectre_v2_user_cmd mode; 1311 char arg[20]; 1312 int ret, i; 1313 1314 mode = IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? 1315 SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE; 1316 1317 switch (spectre_v2_cmd) { 1318 case SPECTRE_V2_CMD_NONE: 1319 return SPECTRE_V2_USER_CMD_NONE; 1320 case SPECTRE_V2_CMD_FORCE: 1321 return SPECTRE_V2_USER_CMD_FORCE; 1322 default: 1323 break; 1324 } 1325 1326 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 1327 arg, sizeof(arg)); 1328 if (ret < 0) 1329 return mode; 1330 1331 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 1332 if (match_option(arg, ret, v2_user_options[i].option)) { 1333 spec_v2_user_print_cond(v2_user_options[i].option, 1334 v2_user_options[i].secure); 1335 return v2_user_options[i].cmd; 1336 } 1337 } 1338 1339 pr_err("Unknown user space protection option (%s). Switching to default\n", arg); 1340 return mode; 1341 } 1342 1343 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1344 { 1345 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1346 } 1347 1348 static void __init 1349 spectre_v2_user_select_mitigation(void) 1350 { 1351 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 1352 enum spectre_v2_user_cmd cmd; 1353 1354 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1355 return; 1356 1357 cmd = spectre_v2_parse_user_cmdline(); 1358 switch (cmd) { 1359 case SPECTRE_V2_USER_CMD_NONE: 1360 goto set_mode; 1361 case SPECTRE_V2_USER_CMD_FORCE: 1362 mode = SPECTRE_V2_USER_STRICT; 1363 break; 1364 case SPECTRE_V2_USER_CMD_AUTO: 1365 case SPECTRE_V2_USER_CMD_PRCTL: 1366 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1367 mode = SPECTRE_V2_USER_PRCTL; 1368 break; 1369 case SPECTRE_V2_USER_CMD_SECCOMP: 1370 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1371 if (IS_ENABLED(CONFIG_SECCOMP)) 1372 mode = SPECTRE_V2_USER_SECCOMP; 1373 else 1374 mode = SPECTRE_V2_USER_PRCTL; 1375 break; 1376 } 1377 1378 /* Initialize Indirect Branch Prediction Barrier */ 1379 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1380 static_branch_enable(&switch_vcpu_ibpb); 1381 1382 spectre_v2_user_ibpb = mode; 1383 switch (cmd) { 1384 case SPECTRE_V2_USER_CMD_NONE: 1385 break; 1386 case SPECTRE_V2_USER_CMD_FORCE: 1387 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1388 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1389 static_branch_enable(&switch_mm_always_ibpb); 1390 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1391 break; 1392 case SPECTRE_V2_USER_CMD_PRCTL: 1393 case SPECTRE_V2_USER_CMD_AUTO: 1394 case SPECTRE_V2_USER_CMD_SECCOMP: 1395 static_branch_enable(&switch_mm_cond_ibpb); 1396 break; 1397 } 1398 1399 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1400 static_key_enabled(&switch_mm_always_ibpb) ? 1401 "always-on" : "conditional"); 1402 } 1403 1404 /* 1405 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1406 * is not required. 1407 * 1408 * Intel's Enhanced IBRS also protects against cross-thread branch target 1409 * injection in user-mode as the IBRS bit remains always set which 1410 * implicitly enables cross-thread protections. However, in legacy IBRS 1411 * mode, the IBRS bit is set only on kernel entry and cleared on return 1412 * to userspace. AMD Automatic IBRS also does not protect userspace. 1413 * These modes therefore disable the implicit cross-thread protection, 1414 * so allow for STIBP to be selected in those cases. 1415 */ 1416 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1417 !cpu_smt_possible() || 1418 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1419 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) 1420 return; 1421 1422 /* 1423 * At this point, an STIBP mode other than "off" has been set. 1424 * If STIBP support is not being forced, check if STIBP always-on 1425 * is preferred. 1426 */ 1427 if (mode != SPECTRE_V2_USER_STRICT && 1428 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1429 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1430 1431 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1432 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 1433 if (mode != SPECTRE_V2_USER_STRICT && 1434 mode != SPECTRE_V2_USER_STRICT_PREFERRED) 1435 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1436 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1437 } 1438 1439 spectre_v2_user_stibp = mode; 1440 1441 set_mode: 1442 pr_info("%s\n", spectre_v2_user_strings[mode]); 1443 } 1444 1445 static const char * const spectre_v2_strings[] = { 1446 [SPECTRE_V2_NONE] = "Vulnerable", 1447 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1448 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 1449 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1450 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1451 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1452 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1453 }; 1454 1455 static const struct { 1456 const char *option; 1457 enum spectre_v2_mitigation_cmd cmd; 1458 bool secure; 1459 } mitigation_options[] __initconst = { 1460 { "off", SPECTRE_V2_CMD_NONE, false }, 1461 { "on", SPECTRE_V2_CMD_FORCE, true }, 1462 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 1463 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1464 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1465 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 1466 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 1467 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 1468 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 1469 { "auto", SPECTRE_V2_CMD_AUTO, false }, 1470 { "ibrs", SPECTRE_V2_CMD_IBRS, false }, 1471 }; 1472 1473 static void __init spec_v2_print_cond(const char *reason, bool secure) 1474 { 1475 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1476 pr_info("%s selected on command line.\n", reason); 1477 } 1478 1479 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 1480 { 1481 enum spectre_v2_mitigation_cmd cmd; 1482 char arg[20]; 1483 int ret, i; 1484 1485 cmd = IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE; 1486 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 1487 cpu_mitigations_off()) 1488 return SPECTRE_V2_CMD_NONE; 1489 1490 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 1491 if (ret < 0) 1492 return cmd; 1493 1494 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 1495 if (!match_option(arg, ret, mitigation_options[i].option)) 1496 continue; 1497 cmd = mitigation_options[i].cmd; 1498 break; 1499 } 1500 1501 if (i >= ARRAY_SIZE(mitigation_options)) { 1502 pr_err("unknown option (%s). Switching to default mode\n", arg); 1503 return cmd; 1504 } 1505 1506 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 1507 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1508 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 1509 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1510 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1511 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 1512 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1513 mitigation_options[i].option); 1514 return SPECTRE_V2_CMD_AUTO; 1515 } 1516 1517 if ((cmd == SPECTRE_V2_CMD_EIBRS || 1518 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1519 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1520 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1521 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n", 1522 mitigation_options[i].option); 1523 return SPECTRE_V2_CMD_AUTO; 1524 } 1525 1526 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1527 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 1528 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 1529 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 1530 mitigation_options[i].option); 1531 return SPECTRE_V2_CMD_AUTO; 1532 } 1533 1534 if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) { 1535 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1536 mitigation_options[i].option); 1537 return SPECTRE_V2_CMD_AUTO; 1538 } 1539 1540 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1541 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n", 1542 mitigation_options[i].option); 1543 return SPECTRE_V2_CMD_AUTO; 1544 } 1545 1546 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 1547 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n", 1548 mitigation_options[i].option); 1549 return SPECTRE_V2_CMD_AUTO; 1550 } 1551 1552 if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 1553 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n", 1554 mitigation_options[i].option); 1555 return SPECTRE_V2_CMD_AUTO; 1556 } 1557 1558 spec_v2_print_cond(mitigation_options[i].option, 1559 mitigation_options[i].secure); 1560 return cmd; 1561 } 1562 1563 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1564 { 1565 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 1566 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1567 return SPECTRE_V2_NONE; 1568 } 1569 1570 return SPECTRE_V2_RETPOLINE; 1571 } 1572 1573 static bool __ro_after_init rrsba_disabled; 1574 1575 /* Disable in-kernel use of non-RSB RET predictors */ 1576 static void __init spec_ctrl_disable_kernel_rrsba(void) 1577 { 1578 if (rrsba_disabled) 1579 return; 1580 1581 if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) { 1582 rrsba_disabled = true; 1583 return; 1584 } 1585 1586 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1587 return; 1588 1589 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1590 update_spec_ctrl(x86_spec_ctrl_base); 1591 rrsba_disabled = true; 1592 } 1593 1594 static void __init spectre_v2_select_rsb_mitigation(enum spectre_v2_mitigation mode) 1595 { 1596 /* 1597 * WARNING! There are many subtleties to consider when changing *any* 1598 * code related to RSB-related mitigations. Before doing so, carefully 1599 * read the following document, and update if necessary: 1600 * 1601 * Documentation/admin-guide/hw-vuln/rsb.rst 1602 * 1603 * In an overly simplified nutshell: 1604 * 1605 * - User->user RSB attacks are conditionally mitigated during 1606 * context switches by cond_mitigation -> write_ibpb(). 1607 * 1608 * - User->kernel and guest->host attacks are mitigated by eIBRS or 1609 * RSB filling. 1610 * 1611 * Though, depending on config, note that other alternative 1612 * mitigations may end up getting used instead, e.g., IBPB on 1613 * entry/vmexit, call depth tracking, or return thunks. 1614 */ 1615 1616 switch (mode) { 1617 case SPECTRE_V2_NONE: 1618 break; 1619 1620 case SPECTRE_V2_EIBRS: 1621 case SPECTRE_V2_EIBRS_LFENCE: 1622 case SPECTRE_V2_EIBRS_RETPOLINE: 1623 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 1624 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 1625 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 1626 } 1627 break; 1628 1629 case SPECTRE_V2_RETPOLINE: 1630 case SPECTRE_V2_LFENCE: 1631 case SPECTRE_V2_IBRS: 1632 pr_info("Spectre v2 / SpectreRSB: Filling RSB on context switch and VMEXIT\n"); 1633 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1634 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1635 break; 1636 1637 default: 1638 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation\n"); 1639 dump_stack(); 1640 break; 1641 } 1642 } 1643 1644 /* 1645 * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by 1646 * branch history in userspace. Not needed if BHI_NO is set. 1647 */ 1648 static bool __init spec_ctrl_bhi_dis(void) 1649 { 1650 if (!boot_cpu_has(X86_FEATURE_BHI_CTRL)) 1651 return false; 1652 1653 x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S; 1654 update_spec_ctrl(x86_spec_ctrl_base); 1655 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW); 1656 1657 return true; 1658 } 1659 1660 enum bhi_mitigations { 1661 BHI_MITIGATION_OFF, 1662 BHI_MITIGATION_ON, 1663 BHI_MITIGATION_VMEXIT_ONLY, 1664 }; 1665 1666 static enum bhi_mitigations bhi_mitigation __ro_after_init = 1667 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_ON : BHI_MITIGATION_OFF; 1668 1669 static int __init spectre_bhi_parse_cmdline(char *str) 1670 { 1671 if (!str) 1672 return -EINVAL; 1673 1674 if (!strcmp(str, "off")) 1675 bhi_mitigation = BHI_MITIGATION_OFF; 1676 else if (!strcmp(str, "on")) 1677 bhi_mitigation = BHI_MITIGATION_ON; 1678 else if (!strcmp(str, "vmexit")) 1679 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 1680 else 1681 pr_err("Ignoring unknown spectre_bhi option (%s)", str); 1682 1683 return 0; 1684 } 1685 early_param("spectre_bhi", spectre_bhi_parse_cmdline); 1686 1687 static void __init bhi_select_mitigation(void) 1688 { 1689 if (bhi_mitigation == BHI_MITIGATION_OFF) 1690 return; 1691 1692 /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ 1693 if (boot_cpu_has(X86_FEATURE_RETPOLINE) && 1694 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { 1695 spec_ctrl_disable_kernel_rrsba(); 1696 if (rrsba_disabled) 1697 return; 1698 } 1699 1700 /* Mitigate in hardware if supported */ 1701 if (spec_ctrl_bhi_dis()) 1702 return; 1703 1704 if (!IS_ENABLED(CONFIG_X86_64)) 1705 return; 1706 1707 if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) { 1708 pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n"); 1709 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT); 1710 return; 1711 } 1712 1713 pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n"); 1714 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP); 1715 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT); 1716 } 1717 1718 static void __init spectre_v2_select_mitigation(void) 1719 { 1720 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 1721 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 1722 1723 /* 1724 * If the CPU is not affected and the command line mode is NONE or AUTO 1725 * then nothing to do. 1726 */ 1727 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 1728 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 1729 return; 1730 1731 switch (cmd) { 1732 case SPECTRE_V2_CMD_NONE: 1733 return; 1734 1735 case SPECTRE_V2_CMD_FORCE: 1736 case SPECTRE_V2_CMD_AUTO: 1737 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1738 mode = SPECTRE_V2_EIBRS; 1739 break; 1740 } 1741 1742 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && 1743 boot_cpu_has_bug(X86_BUG_RETBLEED) && 1744 retbleed_cmd != RETBLEED_CMD_OFF && 1745 retbleed_cmd != RETBLEED_CMD_STUFF && 1746 boot_cpu_has(X86_FEATURE_IBRS) && 1747 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1748 mode = SPECTRE_V2_IBRS; 1749 break; 1750 } 1751 1752 mode = spectre_v2_select_retpoline(); 1753 break; 1754 1755 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 1756 pr_err(SPECTRE_V2_LFENCE_MSG); 1757 mode = SPECTRE_V2_LFENCE; 1758 break; 1759 1760 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 1761 mode = SPECTRE_V2_RETPOLINE; 1762 break; 1763 1764 case SPECTRE_V2_CMD_RETPOLINE: 1765 mode = spectre_v2_select_retpoline(); 1766 break; 1767 1768 case SPECTRE_V2_CMD_IBRS: 1769 mode = SPECTRE_V2_IBRS; 1770 break; 1771 1772 case SPECTRE_V2_CMD_EIBRS: 1773 mode = SPECTRE_V2_EIBRS; 1774 break; 1775 1776 case SPECTRE_V2_CMD_EIBRS_LFENCE: 1777 mode = SPECTRE_V2_EIBRS_LFENCE; 1778 break; 1779 1780 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 1781 mode = SPECTRE_V2_EIBRS_RETPOLINE; 1782 break; 1783 } 1784 1785 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1786 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1787 1788 if (spectre_v2_in_ibrs_mode(mode)) { 1789 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 1790 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 1791 } else { 1792 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 1793 update_spec_ctrl(x86_spec_ctrl_base); 1794 } 1795 } 1796 1797 switch (mode) { 1798 case SPECTRE_V2_NONE: 1799 case SPECTRE_V2_EIBRS: 1800 break; 1801 1802 case SPECTRE_V2_IBRS: 1803 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 1804 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1805 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 1806 break; 1807 1808 case SPECTRE_V2_LFENCE: 1809 case SPECTRE_V2_EIBRS_LFENCE: 1810 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 1811 fallthrough; 1812 1813 case SPECTRE_V2_RETPOLINE: 1814 case SPECTRE_V2_EIBRS_RETPOLINE: 1815 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 1816 break; 1817 } 1818 1819 /* 1820 * Disable alternate RSB predictions in kernel when indirect CALLs and 1821 * JMPs gets protection against BHI and Intramode-BTI, but RET 1822 * prediction from a non-RSB predictor is still a risk. 1823 */ 1824 if (mode == SPECTRE_V2_EIBRS_LFENCE || 1825 mode == SPECTRE_V2_EIBRS_RETPOLINE || 1826 mode == SPECTRE_V2_RETPOLINE) 1827 spec_ctrl_disable_kernel_rrsba(); 1828 1829 if (boot_cpu_has(X86_BUG_BHI)) 1830 bhi_select_mitigation(); 1831 1832 spectre_v2_enabled = mode; 1833 pr_info("%s\n", spectre_v2_strings[mode]); 1834 1835 spectre_v2_select_rsb_mitigation(mode); 1836 1837 /* 1838 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 1839 * and Enhanced IBRS protect firmware too, so enable IBRS around 1840 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 1841 * otherwise enabled. 1842 * 1843 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 1844 * the user might select retpoline on the kernel command line and if 1845 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 1846 * enable IBRS around firmware calls. 1847 */ 1848 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 1849 boot_cpu_has(X86_FEATURE_IBPB) && 1850 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1851 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 1852 1853 if (retbleed_cmd != RETBLEED_CMD_IBPB) { 1854 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 1855 pr_info("Enabling Speculation Barrier for firmware calls\n"); 1856 } 1857 1858 } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { 1859 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 1860 pr_info("Enabling Restricted Speculation for firmware calls\n"); 1861 } 1862 1863 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 1864 spectre_v2_cmd = cmd; 1865 } 1866 1867 static void update_stibp_msr(void * __unused) 1868 { 1869 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 1870 update_spec_ctrl(val); 1871 } 1872 1873 /* Update x86_spec_ctrl_base in case SMT state changed. */ 1874 static void update_stibp_strict(void) 1875 { 1876 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 1877 1878 if (sched_smt_active()) 1879 mask |= SPEC_CTRL_STIBP; 1880 1881 if (mask == x86_spec_ctrl_base) 1882 return; 1883 1884 pr_info("Update user space SMT mitigation: STIBP %s\n", 1885 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 1886 x86_spec_ctrl_base = mask; 1887 on_each_cpu(update_stibp_msr, NULL, 1); 1888 } 1889 1890 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 1891 static void update_indir_branch_cond(void) 1892 { 1893 if (sched_smt_active()) 1894 static_branch_enable(&switch_to_cond_stibp); 1895 else 1896 static_branch_disable(&switch_to_cond_stibp); 1897 } 1898 1899 #undef pr_fmt 1900 #define pr_fmt(fmt) fmt 1901 1902 /* Update the static key controlling the MDS CPU buffer clear in idle */ 1903 static void update_mds_branch_idle(void) 1904 { 1905 /* 1906 * Enable the idle clearing if SMT is active on CPUs which are 1907 * affected only by MSBDS and not any other MDS variant. 1908 * 1909 * The other variants cannot be mitigated when SMT is enabled, so 1910 * clearing the buffers on idle just to prevent the Store Buffer 1911 * repartitioning leak would be a window dressing exercise. 1912 */ 1913 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 1914 return; 1915 1916 if (sched_smt_active()) { 1917 static_branch_enable(&mds_idle_clear); 1918 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 1919 (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) { 1920 static_branch_disable(&mds_idle_clear); 1921 } 1922 } 1923 1924 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 1925 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 1926 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 1927 1928 void cpu_bugs_smt_update(void) 1929 { 1930 mutex_lock(&spec_ctrl_mutex); 1931 1932 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1933 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1934 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1935 1936 switch (spectre_v2_user_stibp) { 1937 case SPECTRE_V2_USER_NONE: 1938 break; 1939 case SPECTRE_V2_USER_STRICT: 1940 case SPECTRE_V2_USER_STRICT_PREFERRED: 1941 update_stibp_strict(); 1942 break; 1943 case SPECTRE_V2_USER_PRCTL: 1944 case SPECTRE_V2_USER_SECCOMP: 1945 update_indir_branch_cond(); 1946 break; 1947 } 1948 1949 switch (mds_mitigation) { 1950 case MDS_MITIGATION_FULL: 1951 case MDS_MITIGATION_AUTO: 1952 case MDS_MITIGATION_VMWERV: 1953 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 1954 pr_warn_once(MDS_MSG_SMT); 1955 update_mds_branch_idle(); 1956 break; 1957 case MDS_MITIGATION_OFF: 1958 break; 1959 } 1960 1961 switch (taa_mitigation) { 1962 case TAA_MITIGATION_VERW: 1963 case TAA_MITIGATION_AUTO: 1964 case TAA_MITIGATION_UCODE_NEEDED: 1965 if (sched_smt_active()) 1966 pr_warn_once(TAA_MSG_SMT); 1967 break; 1968 case TAA_MITIGATION_TSX_DISABLED: 1969 case TAA_MITIGATION_OFF: 1970 break; 1971 } 1972 1973 switch (mmio_mitigation) { 1974 case MMIO_MITIGATION_VERW: 1975 case MMIO_MITIGATION_AUTO: 1976 case MMIO_MITIGATION_UCODE_NEEDED: 1977 if (sched_smt_active()) 1978 pr_warn_once(MMIO_MSG_SMT); 1979 break; 1980 case MMIO_MITIGATION_OFF: 1981 break; 1982 } 1983 1984 mutex_unlock(&spec_ctrl_mutex); 1985 } 1986 1987 #undef pr_fmt 1988 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 1989 1990 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 1991 1992 /* The kernel command line selection */ 1993 enum ssb_mitigation_cmd { 1994 SPEC_STORE_BYPASS_CMD_NONE, 1995 SPEC_STORE_BYPASS_CMD_AUTO, 1996 SPEC_STORE_BYPASS_CMD_ON, 1997 SPEC_STORE_BYPASS_CMD_PRCTL, 1998 SPEC_STORE_BYPASS_CMD_SECCOMP, 1999 }; 2000 2001 static const char * const ssb_strings[] = { 2002 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 2003 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 2004 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 2005 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 2006 }; 2007 2008 static const struct { 2009 const char *option; 2010 enum ssb_mitigation_cmd cmd; 2011 } ssb_mitigation_options[] __initconst = { 2012 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 2013 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 2014 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 2015 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 2016 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 2017 }; 2018 2019 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 2020 { 2021 enum ssb_mitigation_cmd cmd; 2022 char arg[20]; 2023 int ret, i; 2024 2025 cmd = IS_ENABLED(CONFIG_MITIGATION_SSB) ? 2026 SPEC_STORE_BYPASS_CMD_AUTO : SPEC_STORE_BYPASS_CMD_NONE; 2027 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 2028 cpu_mitigations_off()) { 2029 return SPEC_STORE_BYPASS_CMD_NONE; 2030 } else { 2031 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 2032 arg, sizeof(arg)); 2033 if (ret < 0) 2034 return cmd; 2035 2036 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 2037 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 2038 continue; 2039 2040 cmd = ssb_mitigation_options[i].cmd; 2041 break; 2042 } 2043 2044 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 2045 pr_err("unknown option (%s). Switching to default mode\n", arg); 2046 return cmd; 2047 } 2048 } 2049 2050 return cmd; 2051 } 2052 2053 static enum ssb_mitigation __init __ssb_select_mitigation(void) 2054 { 2055 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 2056 enum ssb_mitigation_cmd cmd; 2057 2058 if (!boot_cpu_has(X86_FEATURE_SSBD)) 2059 return mode; 2060 2061 cmd = ssb_parse_cmdline(); 2062 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 2063 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 2064 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 2065 return mode; 2066 2067 switch (cmd) { 2068 case SPEC_STORE_BYPASS_CMD_SECCOMP: 2069 /* 2070 * Choose prctl+seccomp as the default mode if seccomp is 2071 * enabled. 2072 */ 2073 if (IS_ENABLED(CONFIG_SECCOMP)) 2074 mode = SPEC_STORE_BYPASS_SECCOMP; 2075 else 2076 mode = SPEC_STORE_BYPASS_PRCTL; 2077 break; 2078 case SPEC_STORE_BYPASS_CMD_ON: 2079 mode = SPEC_STORE_BYPASS_DISABLE; 2080 break; 2081 case SPEC_STORE_BYPASS_CMD_AUTO: 2082 case SPEC_STORE_BYPASS_CMD_PRCTL: 2083 mode = SPEC_STORE_BYPASS_PRCTL; 2084 break; 2085 case SPEC_STORE_BYPASS_CMD_NONE: 2086 break; 2087 } 2088 2089 /* 2090 * We have three CPU feature flags that are in play here: 2091 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 2092 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 2093 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 2094 */ 2095 if (mode == SPEC_STORE_BYPASS_DISABLE) { 2096 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 2097 /* 2098 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 2099 * use a completely different MSR and bit dependent on family. 2100 */ 2101 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 2102 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 2103 x86_amd_ssb_disable(); 2104 } else { 2105 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 2106 update_spec_ctrl(x86_spec_ctrl_base); 2107 } 2108 } 2109 2110 return mode; 2111 } 2112 2113 static void ssb_select_mitigation(void) 2114 { 2115 ssb_mode = __ssb_select_mitigation(); 2116 2117 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2118 pr_info("%s\n", ssb_strings[ssb_mode]); 2119 } 2120 2121 #undef pr_fmt 2122 #define pr_fmt(fmt) "Speculation prctl: " fmt 2123 2124 static void task_update_spec_tif(struct task_struct *tsk) 2125 { 2126 /* Force the update of the real TIF bits */ 2127 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 2128 2129 /* 2130 * Immediately update the speculation control MSRs for the current 2131 * task, but for a non-current task delay setting the CPU 2132 * mitigation until it is scheduled next. 2133 * 2134 * This can only happen for SECCOMP mitigation. For PRCTL it's 2135 * always the current task. 2136 */ 2137 if (tsk == current) 2138 speculation_ctrl_update_current(); 2139 } 2140 2141 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 2142 { 2143 2144 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2145 return -EPERM; 2146 2147 switch (ctrl) { 2148 case PR_SPEC_ENABLE: 2149 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2150 return 0; 2151 case PR_SPEC_DISABLE: 2152 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2153 return 0; 2154 default: 2155 return -ERANGE; 2156 } 2157 } 2158 2159 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2160 { 2161 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2162 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2163 return -ENXIO; 2164 2165 switch (ctrl) { 2166 case PR_SPEC_ENABLE: 2167 /* If speculation is force disabled, enable is not allowed */ 2168 if (task_spec_ssb_force_disable(task)) 2169 return -EPERM; 2170 task_clear_spec_ssb_disable(task); 2171 task_clear_spec_ssb_noexec(task); 2172 task_update_spec_tif(task); 2173 break; 2174 case PR_SPEC_DISABLE: 2175 task_set_spec_ssb_disable(task); 2176 task_clear_spec_ssb_noexec(task); 2177 task_update_spec_tif(task); 2178 break; 2179 case PR_SPEC_FORCE_DISABLE: 2180 task_set_spec_ssb_disable(task); 2181 task_set_spec_ssb_force_disable(task); 2182 task_clear_spec_ssb_noexec(task); 2183 task_update_spec_tif(task); 2184 break; 2185 case PR_SPEC_DISABLE_NOEXEC: 2186 if (task_spec_ssb_force_disable(task)) 2187 return -EPERM; 2188 task_set_spec_ssb_disable(task); 2189 task_set_spec_ssb_noexec(task); 2190 task_update_spec_tif(task); 2191 break; 2192 default: 2193 return -ERANGE; 2194 } 2195 return 0; 2196 } 2197 2198 static bool is_spec_ib_user_controlled(void) 2199 { 2200 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2201 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2202 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2203 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2204 } 2205 2206 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2207 { 2208 switch (ctrl) { 2209 case PR_SPEC_ENABLE: 2210 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2211 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2212 return 0; 2213 2214 /* 2215 * With strict mode for both IBPB and STIBP, the instruction 2216 * code paths avoid checking this task flag and instead, 2217 * unconditionally run the instruction. However, STIBP and IBPB 2218 * are independent and either can be set to conditionally 2219 * enabled regardless of the mode of the other. 2220 * 2221 * If either is set to conditional, allow the task flag to be 2222 * updated, unless it was force-disabled by a previous prctl 2223 * call. Currently, this is possible on an AMD CPU which has the 2224 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2225 * kernel is booted with 'spectre_v2_user=seccomp', then 2226 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2227 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2228 */ 2229 if (!is_spec_ib_user_controlled() || 2230 task_spec_ib_force_disable(task)) 2231 return -EPERM; 2232 2233 task_clear_spec_ib_disable(task); 2234 task_update_spec_tif(task); 2235 break; 2236 case PR_SPEC_DISABLE: 2237 case PR_SPEC_FORCE_DISABLE: 2238 /* 2239 * Indirect branch speculation is always allowed when 2240 * mitigation is force disabled. 2241 */ 2242 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2243 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2244 return -EPERM; 2245 2246 if (!is_spec_ib_user_controlled()) 2247 return 0; 2248 2249 task_set_spec_ib_disable(task); 2250 if (ctrl == PR_SPEC_FORCE_DISABLE) 2251 task_set_spec_ib_force_disable(task); 2252 task_update_spec_tif(task); 2253 if (task == current) 2254 indirect_branch_prediction_barrier(); 2255 break; 2256 default: 2257 return -ERANGE; 2258 } 2259 return 0; 2260 } 2261 2262 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2263 unsigned long ctrl) 2264 { 2265 switch (which) { 2266 case PR_SPEC_STORE_BYPASS: 2267 return ssb_prctl_set(task, ctrl); 2268 case PR_SPEC_INDIRECT_BRANCH: 2269 return ib_prctl_set(task, ctrl); 2270 case PR_SPEC_L1D_FLUSH: 2271 return l1d_flush_prctl_set(task, ctrl); 2272 default: 2273 return -ENODEV; 2274 } 2275 } 2276 2277 #ifdef CONFIG_SECCOMP 2278 void arch_seccomp_spec_mitigate(struct task_struct *task) 2279 { 2280 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2281 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2282 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2283 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2284 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2285 } 2286 #endif 2287 2288 static int l1d_flush_prctl_get(struct task_struct *task) 2289 { 2290 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2291 return PR_SPEC_FORCE_DISABLE; 2292 2293 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2294 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2295 else 2296 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2297 } 2298 2299 static int ssb_prctl_get(struct task_struct *task) 2300 { 2301 switch (ssb_mode) { 2302 case SPEC_STORE_BYPASS_NONE: 2303 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2304 return PR_SPEC_ENABLE; 2305 return PR_SPEC_NOT_AFFECTED; 2306 case SPEC_STORE_BYPASS_DISABLE: 2307 return PR_SPEC_DISABLE; 2308 case SPEC_STORE_BYPASS_SECCOMP: 2309 case SPEC_STORE_BYPASS_PRCTL: 2310 if (task_spec_ssb_force_disable(task)) 2311 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2312 if (task_spec_ssb_noexec(task)) 2313 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2314 if (task_spec_ssb_disable(task)) 2315 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2316 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2317 } 2318 BUG(); 2319 } 2320 2321 static int ib_prctl_get(struct task_struct *task) 2322 { 2323 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2324 return PR_SPEC_NOT_AFFECTED; 2325 2326 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2327 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2328 return PR_SPEC_ENABLE; 2329 else if (is_spec_ib_user_controlled()) { 2330 if (task_spec_ib_force_disable(task)) 2331 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2332 if (task_spec_ib_disable(task)) 2333 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2334 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2335 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2336 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2337 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2338 return PR_SPEC_DISABLE; 2339 else 2340 return PR_SPEC_NOT_AFFECTED; 2341 } 2342 2343 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2344 { 2345 switch (which) { 2346 case PR_SPEC_STORE_BYPASS: 2347 return ssb_prctl_get(task); 2348 case PR_SPEC_INDIRECT_BRANCH: 2349 return ib_prctl_get(task); 2350 case PR_SPEC_L1D_FLUSH: 2351 return l1d_flush_prctl_get(task); 2352 default: 2353 return -ENODEV; 2354 } 2355 } 2356 2357 void x86_spec_ctrl_setup_ap(void) 2358 { 2359 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2360 update_spec_ctrl(x86_spec_ctrl_base); 2361 2362 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2363 x86_amd_ssb_disable(); 2364 } 2365 2366 bool itlb_multihit_kvm_mitigation; 2367 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 2368 2369 #undef pr_fmt 2370 #define pr_fmt(fmt) "L1TF: " fmt 2371 2372 /* Default mitigation for L1TF-affected CPUs */ 2373 enum l1tf_mitigations l1tf_mitigation __ro_after_init = 2374 IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_FLUSH : L1TF_MITIGATION_OFF; 2375 #if IS_ENABLED(CONFIG_KVM_INTEL) 2376 EXPORT_SYMBOL_GPL(l1tf_mitigation); 2377 #endif 2378 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2379 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 2380 2381 /* 2382 * These CPUs all support 44bits physical address space internally in the 2383 * cache but CPUID can report a smaller number of physical address bits. 2384 * 2385 * The L1TF mitigation uses the top most address bit for the inversion of 2386 * non present PTEs. When the installed memory reaches into the top most 2387 * address bit due to memory holes, which has been observed on machines 2388 * which report 36bits physical address bits and have 32G RAM installed, 2389 * then the mitigation range check in l1tf_select_mitigation() triggers. 2390 * This is a false positive because the mitigation is still possible due to 2391 * the fact that the cache uses 44bit internally. Use the cache bits 2392 * instead of the reported physical bits and adjust them on the affected 2393 * machines to 44bit if the reported bits are less than 44. 2394 */ 2395 static void override_cache_bits(struct cpuinfo_x86 *c) 2396 { 2397 if (c->x86 != 6) 2398 return; 2399 2400 switch (c->x86_vfm) { 2401 case INTEL_NEHALEM: 2402 case INTEL_WESTMERE: 2403 case INTEL_SANDYBRIDGE: 2404 case INTEL_IVYBRIDGE: 2405 case INTEL_HASWELL: 2406 case INTEL_HASWELL_L: 2407 case INTEL_HASWELL_G: 2408 case INTEL_BROADWELL: 2409 case INTEL_BROADWELL_G: 2410 case INTEL_SKYLAKE_L: 2411 case INTEL_SKYLAKE: 2412 case INTEL_KABYLAKE_L: 2413 case INTEL_KABYLAKE: 2414 if (c->x86_cache_bits < 44) 2415 c->x86_cache_bits = 44; 2416 break; 2417 } 2418 } 2419 2420 static void __init l1tf_select_mitigation(void) 2421 { 2422 u64 half_pa; 2423 2424 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2425 return; 2426 2427 if (cpu_mitigations_off()) 2428 l1tf_mitigation = L1TF_MITIGATION_OFF; 2429 else if (cpu_mitigations_auto_nosmt()) 2430 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2431 2432 override_cache_bits(&boot_cpu_data); 2433 2434 switch (l1tf_mitigation) { 2435 case L1TF_MITIGATION_OFF: 2436 case L1TF_MITIGATION_FLUSH_NOWARN: 2437 case L1TF_MITIGATION_FLUSH: 2438 break; 2439 case L1TF_MITIGATION_FLUSH_NOSMT: 2440 case L1TF_MITIGATION_FULL: 2441 cpu_smt_disable(false); 2442 break; 2443 case L1TF_MITIGATION_FULL_FORCE: 2444 cpu_smt_disable(true); 2445 break; 2446 } 2447 2448 #if CONFIG_PGTABLE_LEVELS == 2 2449 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2450 return; 2451 #endif 2452 2453 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2454 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2455 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2456 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2457 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2458 half_pa); 2459 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2460 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2461 return; 2462 } 2463 2464 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2465 } 2466 2467 static int __init l1tf_cmdline(char *str) 2468 { 2469 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2470 return 0; 2471 2472 if (!str) 2473 return -EINVAL; 2474 2475 if (!strcmp(str, "off")) 2476 l1tf_mitigation = L1TF_MITIGATION_OFF; 2477 else if (!strcmp(str, "flush,nowarn")) 2478 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2479 else if (!strcmp(str, "flush")) 2480 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2481 else if (!strcmp(str, "flush,nosmt")) 2482 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2483 else if (!strcmp(str, "full")) 2484 l1tf_mitigation = L1TF_MITIGATION_FULL; 2485 else if (!strcmp(str, "full,force")) 2486 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2487 2488 return 0; 2489 } 2490 early_param("l1tf", l1tf_cmdline); 2491 2492 #undef pr_fmt 2493 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 2494 2495 enum srso_mitigation { 2496 SRSO_MITIGATION_NONE, 2497 SRSO_MITIGATION_UCODE_NEEDED, 2498 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 2499 SRSO_MITIGATION_MICROCODE, 2500 SRSO_MITIGATION_SAFE_RET, 2501 SRSO_MITIGATION_IBPB, 2502 SRSO_MITIGATION_IBPB_ON_VMEXIT, 2503 SRSO_MITIGATION_BP_SPEC_REDUCE, 2504 }; 2505 2506 enum srso_mitigation_cmd { 2507 SRSO_CMD_OFF, 2508 SRSO_CMD_MICROCODE, 2509 SRSO_CMD_SAFE_RET, 2510 SRSO_CMD_IBPB, 2511 SRSO_CMD_IBPB_ON_VMEXIT, 2512 }; 2513 2514 static const char * const srso_strings[] = { 2515 [SRSO_MITIGATION_NONE] = "Vulnerable", 2516 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 2517 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 2518 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 2519 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 2520 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 2521 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only", 2522 [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation" 2523 }; 2524 2525 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE; 2526 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET; 2527 2528 static int __init srso_parse_cmdline(char *str) 2529 { 2530 if (!str) 2531 return -EINVAL; 2532 2533 if (!strcmp(str, "off")) 2534 srso_cmd = SRSO_CMD_OFF; 2535 else if (!strcmp(str, "microcode")) 2536 srso_cmd = SRSO_CMD_MICROCODE; 2537 else if (!strcmp(str, "safe-ret")) 2538 srso_cmd = SRSO_CMD_SAFE_RET; 2539 else if (!strcmp(str, "ibpb")) 2540 srso_cmd = SRSO_CMD_IBPB; 2541 else if (!strcmp(str, "ibpb-vmexit")) 2542 srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT; 2543 else 2544 pr_err("Ignoring unknown SRSO option (%s).", str); 2545 2546 return 0; 2547 } 2548 early_param("spec_rstack_overflow", srso_parse_cmdline); 2549 2550 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 2551 2552 static void __init srso_select_mitigation(void) 2553 { 2554 bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); 2555 2556 if (!boot_cpu_has_bug(X86_BUG_SRSO) || 2557 cpu_mitigations_off() || 2558 srso_cmd == SRSO_CMD_OFF) { 2559 if (boot_cpu_has(X86_FEATURE_SBPB)) 2560 x86_pred_cmd = PRED_CMD_SBPB; 2561 goto out; 2562 } 2563 2564 if (has_microcode) { 2565 /* 2566 * Zen1/2 with SMT off aren't vulnerable after the right 2567 * IBPB microcode has been applied. 2568 * 2569 * Zen1/2 don't have SBPB, no need to try to enable it here. 2570 */ 2571 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 2572 setup_force_cpu_cap(X86_FEATURE_SRSO_NO); 2573 goto out; 2574 } 2575 2576 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2577 srso_mitigation = SRSO_MITIGATION_IBPB; 2578 goto out; 2579 } 2580 } else { 2581 pr_warn("IBPB-extending microcode not applied!\n"); 2582 pr_warn(SRSO_NOTICE); 2583 2584 /* may be overwritten by SRSO_CMD_SAFE_RET below */ 2585 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 2586 } 2587 2588 switch (srso_cmd) { 2589 case SRSO_CMD_MICROCODE: 2590 if (has_microcode) { 2591 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2592 pr_warn(SRSO_NOTICE); 2593 } 2594 break; 2595 2596 case SRSO_CMD_SAFE_RET: 2597 if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) 2598 goto ibpb_on_vmexit; 2599 2600 if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) { 2601 /* 2602 * Enable the return thunk for generated code 2603 * like ftrace, static_call, etc. 2604 */ 2605 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 2606 setup_force_cpu_cap(X86_FEATURE_UNRET); 2607 2608 if (boot_cpu_data.x86 == 0x19) { 2609 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 2610 x86_return_thunk = srso_alias_return_thunk; 2611 } else { 2612 setup_force_cpu_cap(X86_FEATURE_SRSO); 2613 x86_return_thunk = srso_return_thunk; 2614 } 2615 if (has_microcode) 2616 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2617 else 2618 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 2619 } else { 2620 pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n"); 2621 } 2622 break; 2623 2624 case SRSO_CMD_IBPB: 2625 if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 2626 if (has_microcode) { 2627 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 2628 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 2629 srso_mitigation = SRSO_MITIGATION_IBPB; 2630 2631 /* 2632 * IBPB on entry already obviates the need for 2633 * software-based untraining so clear those in case some 2634 * other mitigation like Retbleed has selected them. 2635 */ 2636 setup_clear_cpu_cap(X86_FEATURE_UNRET); 2637 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 2638 2639 /* 2640 * There is no need for RSB filling: write_ibpb() ensures 2641 * all predictions, including the RSB, are invalidated, 2642 * regardless of IBPB implementation. 2643 */ 2644 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 2645 } 2646 } else { 2647 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 2648 } 2649 break; 2650 2651 ibpb_on_vmexit: 2652 case SRSO_CMD_IBPB_ON_VMEXIT: 2653 if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) { 2654 pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n"); 2655 srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE; 2656 break; 2657 } 2658 2659 if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 2660 if (has_microcode) { 2661 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 2662 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2663 2664 /* 2665 * There is no need for RSB filling: write_ibpb() ensures 2666 * all predictions, including the RSB, are invalidated, 2667 * regardless of IBPB implementation. 2668 */ 2669 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 2670 } 2671 } else { 2672 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 2673 } 2674 break; 2675 default: 2676 break; 2677 } 2678 2679 out: 2680 /* 2681 * Clear the feature flag if this mitigation is not selected as that 2682 * feature flag controls the BpSpecReduce MSR bit toggling in KVM. 2683 */ 2684 if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE) 2685 setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE); 2686 2687 if (srso_mitigation != SRSO_MITIGATION_NONE) 2688 pr_info("%s\n", srso_strings[srso_mitigation]); 2689 } 2690 2691 #undef pr_fmt 2692 #define pr_fmt(fmt) fmt 2693 2694 #ifdef CONFIG_SYSFS 2695 2696 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 2697 2698 #if IS_ENABLED(CONFIG_KVM_INTEL) 2699 static const char * const l1tf_vmx_states[] = { 2700 [VMENTER_L1D_FLUSH_AUTO] = "auto", 2701 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 2702 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 2703 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 2704 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 2705 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 2706 }; 2707 2708 static ssize_t l1tf_show_state(char *buf) 2709 { 2710 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 2711 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2712 2713 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 2714 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 2715 sched_smt_active())) { 2716 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 2717 l1tf_vmx_states[l1tf_vmx_mitigation]); 2718 } 2719 2720 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 2721 l1tf_vmx_states[l1tf_vmx_mitigation], 2722 sched_smt_active() ? "vulnerable" : "disabled"); 2723 } 2724 2725 static ssize_t itlb_multihit_show_state(char *buf) 2726 { 2727 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2728 !boot_cpu_has(X86_FEATURE_VMX)) 2729 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 2730 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 2731 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 2732 else if (itlb_multihit_kvm_mitigation) 2733 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 2734 else 2735 return sysfs_emit(buf, "KVM: Vulnerable\n"); 2736 } 2737 #else 2738 static ssize_t l1tf_show_state(char *buf) 2739 { 2740 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2741 } 2742 2743 static ssize_t itlb_multihit_show_state(char *buf) 2744 { 2745 return sysfs_emit(buf, "Processor vulnerable\n"); 2746 } 2747 #endif 2748 2749 static ssize_t mds_show_state(char *buf) 2750 { 2751 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2752 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2753 mds_strings[mds_mitigation]); 2754 } 2755 2756 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 2757 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2758 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 2759 sched_smt_active() ? "mitigated" : "disabled")); 2760 } 2761 2762 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2763 sched_smt_active() ? "vulnerable" : "disabled"); 2764 } 2765 2766 static ssize_t tsx_async_abort_show_state(char *buf) 2767 { 2768 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 2769 (taa_mitigation == TAA_MITIGATION_OFF)) 2770 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 2771 2772 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2773 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2774 taa_strings[taa_mitigation]); 2775 } 2776 2777 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 2778 sched_smt_active() ? "vulnerable" : "disabled"); 2779 } 2780 2781 static ssize_t mmio_stale_data_show_state(char *buf) 2782 { 2783 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2784 return sysfs_emit(buf, "Unknown: No mitigations\n"); 2785 2786 if (mmio_mitigation == MMIO_MITIGATION_OFF) 2787 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 2788 2789 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2790 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2791 mmio_strings[mmio_mitigation]); 2792 } 2793 2794 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 2795 sched_smt_active() ? "vulnerable" : "disabled"); 2796 } 2797 2798 static ssize_t rfds_show_state(char *buf) 2799 { 2800 return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); 2801 } 2802 2803 static char *stibp_state(void) 2804 { 2805 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 2806 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 2807 return ""; 2808 2809 switch (spectre_v2_user_stibp) { 2810 case SPECTRE_V2_USER_NONE: 2811 return "; STIBP: disabled"; 2812 case SPECTRE_V2_USER_STRICT: 2813 return "; STIBP: forced"; 2814 case SPECTRE_V2_USER_STRICT_PREFERRED: 2815 return "; STIBP: always-on"; 2816 case SPECTRE_V2_USER_PRCTL: 2817 case SPECTRE_V2_USER_SECCOMP: 2818 if (static_key_enabled(&switch_to_cond_stibp)) 2819 return "; STIBP: conditional"; 2820 } 2821 return ""; 2822 } 2823 2824 static char *ibpb_state(void) 2825 { 2826 if (boot_cpu_has(X86_FEATURE_IBPB)) { 2827 if (static_key_enabled(&switch_mm_always_ibpb)) 2828 return "; IBPB: always-on"; 2829 if (static_key_enabled(&switch_mm_cond_ibpb)) 2830 return "; IBPB: conditional"; 2831 return "; IBPB: disabled"; 2832 } 2833 return ""; 2834 } 2835 2836 static char *pbrsb_eibrs_state(void) 2837 { 2838 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2839 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 2840 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 2841 return "; PBRSB-eIBRS: SW sequence"; 2842 else 2843 return "; PBRSB-eIBRS: Vulnerable"; 2844 } else { 2845 return "; PBRSB-eIBRS: Not affected"; 2846 } 2847 } 2848 2849 static const char *spectre_bhi_state(void) 2850 { 2851 if (!boot_cpu_has_bug(X86_BUG_BHI)) 2852 return "; BHI: Not affected"; 2853 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW)) 2854 return "; BHI: BHI_DIS_S"; 2855 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) 2856 return "; BHI: SW loop, KVM: SW loop"; 2857 else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && 2858 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) && 2859 rrsba_disabled) 2860 return "; BHI: Retpoline"; 2861 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT)) 2862 return "; BHI: Vulnerable, KVM: SW loop"; 2863 2864 return "; BHI: Vulnerable"; 2865 } 2866 2867 static ssize_t spectre_v2_show_state(char *buf) 2868 { 2869 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 2870 return sysfs_emit(buf, "Vulnerable: LFENCE\n"); 2871 2872 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2873 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 2874 2875 if (sched_smt_active() && unprivileged_ebpf_enabled() && 2876 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 2877 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 2878 2879 return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n", 2880 spectre_v2_strings[spectre_v2_enabled], 2881 ibpb_state(), 2882 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "", 2883 stibp_state(), 2884 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "", 2885 pbrsb_eibrs_state(), 2886 spectre_bhi_state(), 2887 /* this should always be at the end */ 2888 spectre_v2_module_string()); 2889 } 2890 2891 static ssize_t srbds_show_state(char *buf) 2892 { 2893 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 2894 } 2895 2896 static ssize_t retbleed_show_state(char *buf) 2897 { 2898 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 2899 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2900 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 2901 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 2902 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 2903 2904 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 2905 !sched_smt_active() ? "disabled" : 2906 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2907 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 2908 "enabled with STIBP protection" : "vulnerable"); 2909 } 2910 2911 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 2912 } 2913 2914 static ssize_t srso_show_state(char *buf) 2915 { 2916 if (boot_cpu_has(X86_FEATURE_SRSO_NO)) 2917 return sysfs_emit(buf, "Mitigation: SMT disabled\n"); 2918 2919 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 2920 } 2921 2922 static ssize_t gds_show_state(char *buf) 2923 { 2924 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 2925 } 2926 2927 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 2928 char *buf, unsigned int bug) 2929 { 2930 if (!boot_cpu_has_bug(bug)) 2931 return sysfs_emit(buf, "Not affected\n"); 2932 2933 switch (bug) { 2934 case X86_BUG_CPU_MELTDOWN: 2935 if (boot_cpu_has(X86_FEATURE_PTI)) 2936 return sysfs_emit(buf, "Mitigation: PTI\n"); 2937 2938 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 2939 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 2940 2941 break; 2942 2943 case X86_BUG_SPECTRE_V1: 2944 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 2945 2946 case X86_BUG_SPECTRE_V2: 2947 return spectre_v2_show_state(buf); 2948 2949 case X86_BUG_SPEC_STORE_BYPASS: 2950 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 2951 2952 case X86_BUG_L1TF: 2953 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 2954 return l1tf_show_state(buf); 2955 break; 2956 2957 case X86_BUG_MDS: 2958 return mds_show_state(buf); 2959 2960 case X86_BUG_TAA: 2961 return tsx_async_abort_show_state(buf); 2962 2963 case X86_BUG_ITLB_MULTIHIT: 2964 return itlb_multihit_show_state(buf); 2965 2966 case X86_BUG_SRBDS: 2967 return srbds_show_state(buf); 2968 2969 case X86_BUG_MMIO_STALE_DATA: 2970 case X86_BUG_MMIO_UNKNOWN: 2971 return mmio_stale_data_show_state(buf); 2972 2973 case X86_BUG_RETBLEED: 2974 return retbleed_show_state(buf); 2975 2976 case X86_BUG_SRSO: 2977 return srso_show_state(buf); 2978 2979 case X86_BUG_GDS: 2980 return gds_show_state(buf); 2981 2982 case X86_BUG_RFDS: 2983 return rfds_show_state(buf); 2984 2985 default: 2986 break; 2987 } 2988 2989 return sysfs_emit(buf, "Vulnerable\n"); 2990 } 2991 2992 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 2993 { 2994 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 2995 } 2996 2997 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 2998 { 2999 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 3000 } 3001 3002 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 3003 { 3004 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 3005 } 3006 3007 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 3008 { 3009 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 3010 } 3011 3012 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 3013 { 3014 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 3015 } 3016 3017 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 3018 { 3019 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 3020 } 3021 3022 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 3023 { 3024 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 3025 } 3026 3027 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 3028 { 3029 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 3030 } 3031 3032 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 3033 { 3034 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 3035 } 3036 3037 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 3038 { 3039 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 3040 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); 3041 else 3042 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 3043 } 3044 3045 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 3046 { 3047 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 3048 } 3049 3050 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 3051 { 3052 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 3053 } 3054 3055 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 3056 { 3057 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 3058 } 3059 3060 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf) 3061 { 3062 return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); 3063 } 3064 #endif 3065 3066 void __warn_thunk(void) 3067 { 3068 WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); 3069 } 3070