1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 #include <linux/filter.h> 20 #include <linux/kvm_types.h> 21 22 #include <asm/spec-ctrl.h> 23 #include <asm/cmdline.h> 24 #include <asm/bugs.h> 25 #include <asm/processor.h> 26 #include <asm/processor-flags.h> 27 #include <asm/fpu/api.h> 28 #include <asm/msr.h> 29 #include <asm/vmx.h> 30 #include <asm/cpu_device_id.h> 31 #include <asm/e820/api.h> 32 #include <asm/hypervisor.h> 33 #include <asm/tlbflush.h> 34 #include <asm/cpu.h> 35 36 #include "cpu.h" 37 38 /* 39 * Speculation Vulnerability Handling 40 * 41 * Each vulnerability is handled with the following functions: 42 * <vuln>_select_mitigation() -- Selects a mitigation to use. This should 43 * take into account all relevant command line 44 * options. 45 * <vuln>_update_mitigation() -- This is called after all vulnerabilities have 46 * selected a mitigation, in case the selection 47 * may want to change based on other choices 48 * made. This function is optional. 49 * <vuln>_apply_mitigation() -- Enable the selected mitigation. 50 * 51 * The compile-time mitigation in all cases should be AUTO. An explicit 52 * command-line option can override AUTO. If no such option is 53 * provided, <vuln>_select_mitigation() will override AUTO to the best 54 * mitigation option. 55 */ 56 57 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 58 u64 x86_spec_ctrl_base; 59 60 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 61 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 62 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current); 63 64 /* 65 * Set when the CPU has run a potentially malicious guest. An IBPB will 66 * be needed to before running userspace. That IBPB will flush the branch 67 * predictor content. 68 */ 69 DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user); 70 EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user); 71 72 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 73 74 static u64 __ro_after_init x86_arch_cap_msr; 75 76 static DEFINE_MUTEX(spec_ctrl_mutex); 77 78 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 79 80 static void __init set_return_thunk(void *thunk) 81 { 82 x86_return_thunk = thunk; 83 84 pr_info("active return thunk: %ps\n", thunk); 85 } 86 87 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 88 static void update_spec_ctrl(u64 val) 89 { 90 this_cpu_write(x86_spec_ctrl_current, val); 91 wrmsrq(MSR_IA32_SPEC_CTRL, val); 92 } 93 94 /* 95 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 96 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 97 */ 98 void update_spec_ctrl_cond(u64 val) 99 { 100 if (this_cpu_read(x86_spec_ctrl_current) == val) 101 return; 102 103 this_cpu_write(x86_spec_ctrl_current, val); 104 105 /* 106 * When KERNEL_IBRS this MSR is written on return-to-user, unless 107 * forced the update can be delayed until that time. 108 */ 109 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 110 wrmsrq(MSR_IA32_SPEC_CTRL, val); 111 } 112 113 noinstr u64 spec_ctrl_current(void) 114 { 115 return this_cpu_read(x86_spec_ctrl_current); 116 } 117 EXPORT_SYMBOL_GPL(spec_ctrl_current); 118 119 /* 120 * AMD specific MSR info for Speculative Store Bypass control. 121 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 122 */ 123 u64 __ro_after_init x86_amd_ls_cfg_base; 124 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 125 126 /* Control conditional STIBP in switch_to() */ 127 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 128 /* Control conditional IBPB in switch_mm() */ 129 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 130 /* Control unconditional IBPB in switch_mm() */ 131 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 132 133 /* Control IBPB on vCPU load */ 134 DEFINE_STATIC_KEY_FALSE(switch_vcpu_ibpb); 135 EXPORT_SYMBOL_FOR_KVM(switch_vcpu_ibpb); 136 137 /* Control CPU buffer clear before idling (halt, mwait) */ 138 DEFINE_STATIC_KEY_FALSE(cpu_buf_idle_clear); 139 EXPORT_SYMBOL_GPL(cpu_buf_idle_clear); 140 141 /* 142 * Controls whether l1d flush based mitigations are enabled, 143 * based on hw features and admin setting via boot parameter 144 * defaults to false 145 */ 146 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 147 148 #undef pr_fmt 149 #define pr_fmt(fmt) "mitigations: " fmt 150 151 static void __init cpu_print_attack_vectors(void) 152 { 153 pr_info("Enabled attack vectors: "); 154 155 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL)) 156 pr_cont("user_kernel, "); 157 158 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) 159 pr_cont("user_user, "); 160 161 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST)) 162 pr_cont("guest_host, "); 163 164 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) 165 pr_cont("guest_guest, "); 166 167 pr_cont("SMT mitigations: "); 168 169 switch (smt_mitigations) { 170 case SMT_MITIGATIONS_OFF: 171 pr_cont("off\n"); 172 break; 173 case SMT_MITIGATIONS_AUTO: 174 pr_cont("auto\n"); 175 break; 176 case SMT_MITIGATIONS_ON: 177 pr_cont("on\n"); 178 } 179 } 180 181 /* 182 * NOTE: This function is *only* called for SVM, since Intel uses 183 * MSR_IA32_SPEC_CTRL for SSBD. 184 */ 185 void 186 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 187 { 188 u64 guestval, hostval; 189 struct thread_info *ti = current_thread_info(); 190 191 /* 192 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 193 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 194 */ 195 if (!cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD) && 196 !cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) 197 return; 198 199 /* 200 * If the host has SSBD mitigation enabled, force it in the host's 201 * virtual MSR value. If its not permanently enabled, evaluate 202 * current's TIF_SSBD thread flag. 203 */ 204 if (cpu_feature_enabled(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 205 hostval = SPEC_CTRL_SSBD; 206 else 207 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 208 209 /* Sanitize the guest value */ 210 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 211 212 if (hostval != guestval) { 213 unsigned long tif; 214 215 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 216 ssbd_spec_ctrl_to_tif(hostval); 217 218 speculation_ctrl_update(tif); 219 } 220 } 221 EXPORT_SYMBOL_FOR_KVM(x86_virt_spec_ctrl); 222 223 static void x86_amd_ssb_disable(void) 224 { 225 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 226 227 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 228 wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 229 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 230 wrmsrq(MSR_AMD64_LS_CFG, msrval); 231 } 232 233 #undef pr_fmt 234 #define pr_fmt(fmt) "MDS: " fmt 235 236 /* 237 * Returns true if vulnerability should be mitigated based on the 238 * selected attack vector controls. 239 * 240 * See Documentation/admin-guide/hw-vuln/attack_vector_controls.rst 241 */ 242 static bool __init should_mitigate_vuln(unsigned int bug) 243 { 244 switch (bug) { 245 /* 246 * The only runtime-selected spectre_v1 mitigations in the kernel are 247 * related to SWAPGS protection on kernel entry. Therefore, protection 248 * is only required for the user->kernel attack vector. 249 */ 250 case X86_BUG_SPECTRE_V1: 251 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL); 252 253 case X86_BUG_SPECTRE_V2: 254 case X86_BUG_RETBLEED: 255 case X86_BUG_L1TF: 256 case X86_BUG_ITS: 257 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 258 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST); 259 260 case X86_BUG_SPECTRE_V2_USER: 261 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 262 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST); 263 264 /* 265 * All the vulnerabilities below allow potentially leaking data 266 * across address spaces. Therefore, mitigation is required for 267 * any of these 4 attack vectors. 268 */ 269 case X86_BUG_MDS: 270 case X86_BUG_TAA: 271 case X86_BUG_MMIO_STALE_DATA: 272 case X86_BUG_RFDS: 273 case X86_BUG_SRBDS: 274 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 275 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 276 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 277 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST); 278 279 case X86_BUG_GDS: 280 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 281 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 282 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 283 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST) || 284 (smt_mitigations != SMT_MITIGATIONS_OFF); 285 286 case X86_BUG_SPEC_STORE_BYPASS: 287 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER); 288 289 case X86_BUG_VMSCAPE: 290 return cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST); 291 292 default: 293 WARN(1, "Unknown bug %x\n", bug); 294 return false; 295 } 296 } 297 298 /* Default mitigation for MDS-affected CPUs */ 299 static enum mds_mitigations mds_mitigation __ro_after_init = 300 IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_AUTO : MDS_MITIGATION_OFF; 301 static bool mds_nosmt __ro_after_init = false; 302 303 static const char * const mds_strings[] = { 304 [MDS_MITIGATION_OFF] = "Vulnerable", 305 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 306 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 307 }; 308 309 enum taa_mitigations { 310 TAA_MITIGATION_OFF, 311 TAA_MITIGATION_AUTO, 312 TAA_MITIGATION_UCODE_NEEDED, 313 TAA_MITIGATION_VERW, 314 TAA_MITIGATION_TSX_DISABLED, 315 }; 316 317 /* Default mitigation for TAA-affected CPUs */ 318 static enum taa_mitigations taa_mitigation __ro_after_init = 319 IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_AUTO : TAA_MITIGATION_OFF; 320 321 enum mmio_mitigations { 322 MMIO_MITIGATION_OFF, 323 MMIO_MITIGATION_AUTO, 324 MMIO_MITIGATION_UCODE_NEEDED, 325 MMIO_MITIGATION_VERW, 326 }; 327 328 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 329 static enum mmio_mitigations mmio_mitigation __ro_after_init = 330 IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ? MMIO_MITIGATION_AUTO : MMIO_MITIGATION_OFF; 331 332 enum rfds_mitigations { 333 RFDS_MITIGATION_OFF, 334 RFDS_MITIGATION_AUTO, 335 RFDS_MITIGATION_VERW, 336 RFDS_MITIGATION_UCODE_NEEDED, 337 }; 338 339 /* Default mitigation for Register File Data Sampling */ 340 static enum rfds_mitigations rfds_mitigation __ro_after_init = 341 IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF; 342 343 /* 344 * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing on exit to 345 * userspace *and* on entry to KVM guests. 346 */ 347 static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init; 348 349 static void __init mds_select_mitigation(void) 350 { 351 if (!boot_cpu_has_bug(X86_BUG_MDS)) { 352 mds_mitigation = MDS_MITIGATION_OFF; 353 return; 354 } 355 356 if (mds_mitigation == MDS_MITIGATION_AUTO) { 357 if (should_mitigate_vuln(X86_BUG_MDS)) 358 mds_mitigation = MDS_MITIGATION_FULL; 359 else 360 mds_mitigation = MDS_MITIGATION_OFF; 361 } 362 363 if (mds_mitigation == MDS_MITIGATION_OFF) 364 return; 365 366 verw_clear_cpu_buf_mitigation_selected = true; 367 } 368 369 static void __init mds_update_mitigation(void) 370 { 371 if (!boot_cpu_has_bug(X86_BUG_MDS)) 372 return; 373 374 /* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */ 375 if (verw_clear_cpu_buf_mitigation_selected) 376 mds_mitigation = MDS_MITIGATION_FULL; 377 378 if (mds_mitigation == MDS_MITIGATION_FULL) { 379 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 380 mds_mitigation = MDS_MITIGATION_VMWERV; 381 } 382 383 pr_info("%s\n", mds_strings[mds_mitigation]); 384 } 385 386 static void __init mds_apply_mitigation(void) 387 { 388 if (mds_mitigation == MDS_MITIGATION_FULL || 389 mds_mitigation == MDS_MITIGATION_VMWERV) { 390 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 391 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 392 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 393 (mds_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)) 394 cpu_smt_disable(false); 395 } 396 } 397 398 static int __init mds_cmdline(char *str) 399 { 400 if (!boot_cpu_has_bug(X86_BUG_MDS)) 401 return 0; 402 403 if (!str) 404 return -EINVAL; 405 406 if (!strcmp(str, "off")) 407 mds_mitigation = MDS_MITIGATION_OFF; 408 else if (!strcmp(str, "full")) 409 mds_mitigation = MDS_MITIGATION_FULL; 410 else if (!strcmp(str, "full,nosmt")) { 411 mds_mitigation = MDS_MITIGATION_FULL; 412 mds_nosmt = true; 413 } 414 415 return 0; 416 } 417 early_param("mds", mds_cmdline); 418 419 #undef pr_fmt 420 #define pr_fmt(fmt) "TAA: " fmt 421 422 static bool taa_nosmt __ro_after_init; 423 424 static const char * const taa_strings[] = { 425 [TAA_MITIGATION_OFF] = "Vulnerable", 426 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 427 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 428 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 429 }; 430 431 static bool __init taa_vulnerable(void) 432 { 433 return boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM); 434 } 435 436 static void __init taa_select_mitigation(void) 437 { 438 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 439 taa_mitigation = TAA_MITIGATION_OFF; 440 return; 441 } 442 443 /* TSX previously disabled by tsx=off */ 444 if (!boot_cpu_has(X86_FEATURE_RTM)) { 445 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 446 return; 447 } 448 449 /* Microcode will be checked in taa_update_mitigation(). */ 450 if (taa_mitigation == TAA_MITIGATION_AUTO) { 451 if (should_mitigate_vuln(X86_BUG_TAA)) 452 taa_mitigation = TAA_MITIGATION_VERW; 453 else 454 taa_mitigation = TAA_MITIGATION_OFF; 455 } 456 457 if (taa_mitigation != TAA_MITIGATION_OFF) 458 verw_clear_cpu_buf_mitigation_selected = true; 459 } 460 461 static void __init taa_update_mitigation(void) 462 { 463 if (!taa_vulnerable()) 464 return; 465 466 if (verw_clear_cpu_buf_mitigation_selected) 467 taa_mitigation = TAA_MITIGATION_VERW; 468 469 if (taa_mitigation == TAA_MITIGATION_VERW) { 470 /* Check if the requisite ucode is available. */ 471 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 472 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 473 474 /* 475 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 476 * A microcode update fixes this behavior to clear CPU buffers. It also 477 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 478 * ARCH_CAP_TSX_CTRL_MSR bit. 479 * 480 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 481 * update is required. 482 */ 483 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && 484 !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) 485 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 486 } 487 488 pr_info("%s\n", taa_strings[taa_mitigation]); 489 } 490 491 static void __init taa_apply_mitigation(void) 492 { 493 if (taa_mitigation == TAA_MITIGATION_VERW || 494 taa_mitigation == TAA_MITIGATION_UCODE_NEEDED) { 495 /* 496 * TSX is enabled, select alternate mitigation for TAA which is 497 * the same as MDS. Enable MDS static branch to clear CPU buffers. 498 * 499 * For guests that can't determine whether the correct microcode is 500 * present on host, enable the mitigation for UCODE_NEEDED as well. 501 */ 502 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 503 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 504 505 if (taa_nosmt || smt_mitigations == SMT_MITIGATIONS_ON) 506 cpu_smt_disable(false); 507 } 508 } 509 510 static int __init tsx_async_abort_parse_cmdline(char *str) 511 { 512 if (!boot_cpu_has_bug(X86_BUG_TAA)) 513 return 0; 514 515 if (!str) 516 return -EINVAL; 517 518 if (!strcmp(str, "off")) { 519 taa_mitigation = TAA_MITIGATION_OFF; 520 } else if (!strcmp(str, "full")) { 521 taa_mitigation = TAA_MITIGATION_VERW; 522 } else if (!strcmp(str, "full,nosmt")) { 523 taa_mitigation = TAA_MITIGATION_VERW; 524 taa_nosmt = true; 525 } 526 527 return 0; 528 } 529 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 530 531 #undef pr_fmt 532 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 533 534 static bool mmio_nosmt __ro_after_init = false; 535 536 static const char * const mmio_strings[] = { 537 [MMIO_MITIGATION_OFF] = "Vulnerable", 538 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 539 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 540 }; 541 542 static void __init mmio_select_mitigation(void) 543 { 544 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 545 mmio_mitigation = MMIO_MITIGATION_OFF; 546 return; 547 } 548 549 /* Microcode will be checked in mmio_update_mitigation(). */ 550 if (mmio_mitigation == MMIO_MITIGATION_AUTO) { 551 if (should_mitigate_vuln(X86_BUG_MMIO_STALE_DATA)) 552 mmio_mitigation = MMIO_MITIGATION_VERW; 553 else 554 mmio_mitigation = MMIO_MITIGATION_OFF; 555 } 556 557 if (mmio_mitigation == MMIO_MITIGATION_OFF) 558 return; 559 560 /* 561 * Enable CPU buffer clear mitigation for host and VMM, if also affected 562 * by MDS or TAA. 563 */ 564 if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable()) 565 verw_clear_cpu_buf_mitigation_selected = true; 566 } 567 568 static void __init mmio_update_mitigation(void) 569 { 570 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 571 return; 572 573 if (verw_clear_cpu_buf_mitigation_selected) 574 mmio_mitigation = MMIO_MITIGATION_VERW; 575 576 if (mmio_mitigation == MMIO_MITIGATION_VERW) { 577 /* 578 * Check if the system has the right microcode. 579 * 580 * CPU Fill buffer clear mitigation is enumerated by either an explicit 581 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 582 * affected systems. 583 */ 584 if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || 585 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 586 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 587 !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))) 588 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 589 } 590 591 pr_info("%s\n", mmio_strings[mmio_mitigation]); 592 } 593 594 static void __init mmio_apply_mitigation(void) 595 { 596 if (mmio_mitigation == MMIO_MITIGATION_OFF) 597 return; 598 599 /* 600 * Only enable the VMM mitigation if the CPU buffer clear mitigation is 601 * not being used. 602 */ 603 if (verw_clear_cpu_buf_mitigation_selected) { 604 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 605 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 606 } else { 607 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO); 608 } 609 610 /* 611 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 612 * be propagated to uncore buffers, clearing the Fill buffers on idle 613 * is required irrespective of SMT state. 614 */ 615 if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) 616 static_branch_enable(&cpu_buf_idle_clear); 617 618 if (mmio_nosmt || smt_mitigations == SMT_MITIGATIONS_ON) 619 cpu_smt_disable(false); 620 } 621 622 static int __init mmio_stale_data_parse_cmdline(char *str) 623 { 624 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 625 return 0; 626 627 if (!str) 628 return -EINVAL; 629 630 if (!strcmp(str, "off")) { 631 mmio_mitigation = MMIO_MITIGATION_OFF; 632 } else if (!strcmp(str, "full")) { 633 mmio_mitigation = MMIO_MITIGATION_VERW; 634 } else if (!strcmp(str, "full,nosmt")) { 635 mmio_mitigation = MMIO_MITIGATION_VERW; 636 mmio_nosmt = true; 637 } 638 639 return 0; 640 } 641 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 642 643 #undef pr_fmt 644 #define pr_fmt(fmt) "Register File Data Sampling: " fmt 645 646 static const char * const rfds_strings[] = { 647 [RFDS_MITIGATION_OFF] = "Vulnerable", 648 [RFDS_MITIGATION_VERW] = "Mitigation: Clear Register File", 649 [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 650 }; 651 652 static inline bool __init verw_clears_cpu_reg_file(void) 653 { 654 return (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR); 655 } 656 657 static void __init rfds_select_mitigation(void) 658 { 659 if (!boot_cpu_has_bug(X86_BUG_RFDS)) { 660 rfds_mitigation = RFDS_MITIGATION_OFF; 661 return; 662 } 663 664 if (rfds_mitigation == RFDS_MITIGATION_AUTO) { 665 if (should_mitigate_vuln(X86_BUG_RFDS)) 666 rfds_mitigation = RFDS_MITIGATION_VERW; 667 else 668 rfds_mitigation = RFDS_MITIGATION_OFF; 669 } 670 671 if (rfds_mitigation == RFDS_MITIGATION_OFF) 672 return; 673 674 if (verw_clears_cpu_reg_file()) 675 verw_clear_cpu_buf_mitigation_selected = true; 676 } 677 678 static void __init rfds_update_mitigation(void) 679 { 680 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 681 return; 682 683 if (verw_clear_cpu_buf_mitigation_selected) 684 rfds_mitigation = RFDS_MITIGATION_VERW; 685 686 if (rfds_mitigation == RFDS_MITIGATION_VERW) { 687 if (!verw_clears_cpu_reg_file()) 688 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; 689 } 690 691 pr_info("%s\n", rfds_strings[rfds_mitigation]); 692 } 693 694 static void __init rfds_apply_mitigation(void) 695 { 696 if (rfds_mitigation == RFDS_MITIGATION_VERW) { 697 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 698 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 699 } 700 } 701 702 static __init int rfds_parse_cmdline(char *str) 703 { 704 if (!str) 705 return -EINVAL; 706 707 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 708 return 0; 709 710 if (!strcmp(str, "off")) 711 rfds_mitigation = RFDS_MITIGATION_OFF; 712 else if (!strcmp(str, "on")) 713 rfds_mitigation = RFDS_MITIGATION_VERW; 714 715 return 0; 716 } 717 early_param("reg_file_data_sampling", rfds_parse_cmdline); 718 719 #undef pr_fmt 720 #define pr_fmt(fmt) "SRBDS: " fmt 721 722 enum srbds_mitigations { 723 SRBDS_MITIGATION_OFF, 724 SRBDS_MITIGATION_AUTO, 725 SRBDS_MITIGATION_UCODE_NEEDED, 726 SRBDS_MITIGATION_FULL, 727 SRBDS_MITIGATION_TSX_OFF, 728 SRBDS_MITIGATION_HYPERVISOR, 729 }; 730 731 static enum srbds_mitigations srbds_mitigation __ro_after_init = 732 IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_AUTO : SRBDS_MITIGATION_OFF; 733 734 static const char * const srbds_strings[] = { 735 [SRBDS_MITIGATION_OFF] = "Vulnerable", 736 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 737 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 738 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 739 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 740 }; 741 742 static bool srbds_off; 743 744 void update_srbds_msr(void) 745 { 746 u64 mcu_ctrl; 747 748 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 749 return; 750 751 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 752 return; 753 754 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 755 return; 756 757 /* 758 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 759 * being disabled and it hasn't received the SRBDS MSR microcode. 760 */ 761 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 762 return; 763 764 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 765 766 switch (srbds_mitigation) { 767 case SRBDS_MITIGATION_OFF: 768 case SRBDS_MITIGATION_TSX_OFF: 769 mcu_ctrl |= RNGDS_MITG_DIS; 770 break; 771 case SRBDS_MITIGATION_FULL: 772 mcu_ctrl &= ~RNGDS_MITG_DIS; 773 break; 774 default: 775 break; 776 } 777 778 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 779 } 780 781 static void __init srbds_select_mitigation(void) 782 { 783 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) { 784 srbds_mitigation = SRBDS_MITIGATION_OFF; 785 return; 786 } 787 788 if (srbds_mitigation == SRBDS_MITIGATION_AUTO) { 789 if (should_mitigate_vuln(X86_BUG_SRBDS)) 790 srbds_mitigation = SRBDS_MITIGATION_FULL; 791 else { 792 srbds_mitigation = SRBDS_MITIGATION_OFF; 793 return; 794 } 795 } 796 797 /* 798 * Check to see if this is one of the MDS_NO systems supporting TSX that 799 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 800 * by Processor MMIO Stale Data vulnerability. 801 */ 802 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 803 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 804 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 805 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 806 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 807 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 808 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 809 else if (srbds_off) 810 srbds_mitigation = SRBDS_MITIGATION_OFF; 811 812 pr_info("%s\n", srbds_strings[srbds_mitigation]); 813 } 814 815 static void __init srbds_apply_mitigation(void) 816 { 817 update_srbds_msr(); 818 } 819 820 static int __init srbds_parse_cmdline(char *str) 821 { 822 if (!str) 823 return -EINVAL; 824 825 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 826 return 0; 827 828 srbds_off = !strcmp(str, "off"); 829 return 0; 830 } 831 early_param("srbds", srbds_parse_cmdline); 832 833 #undef pr_fmt 834 #define pr_fmt(fmt) "L1D Flush : " fmt 835 836 enum l1d_flush_mitigations { 837 L1D_FLUSH_OFF = 0, 838 L1D_FLUSH_ON, 839 }; 840 841 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 842 843 static void __init l1d_flush_select_mitigation(void) 844 { 845 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 846 return; 847 848 static_branch_enable(&switch_mm_cond_l1d_flush); 849 pr_info("Conditional flush on switch_mm() enabled\n"); 850 } 851 852 static int __init l1d_flush_parse_cmdline(char *str) 853 { 854 if (!strcmp(str, "on")) 855 l1d_flush_mitigation = L1D_FLUSH_ON; 856 857 return 0; 858 } 859 early_param("l1d_flush", l1d_flush_parse_cmdline); 860 861 #undef pr_fmt 862 #define pr_fmt(fmt) "GDS: " fmt 863 864 enum gds_mitigations { 865 GDS_MITIGATION_OFF, 866 GDS_MITIGATION_AUTO, 867 GDS_MITIGATION_UCODE_NEEDED, 868 GDS_MITIGATION_FORCE, 869 GDS_MITIGATION_FULL, 870 GDS_MITIGATION_FULL_LOCKED, 871 GDS_MITIGATION_HYPERVISOR, 872 }; 873 874 static enum gds_mitigations gds_mitigation __ro_after_init = 875 IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_AUTO : GDS_MITIGATION_OFF; 876 877 static const char * const gds_strings[] = { 878 [GDS_MITIGATION_OFF] = "Vulnerable", 879 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 880 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 881 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 882 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 883 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 884 }; 885 886 bool gds_ucode_mitigated(void) 887 { 888 return (gds_mitigation == GDS_MITIGATION_FULL || 889 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 890 } 891 EXPORT_SYMBOL_FOR_KVM(gds_ucode_mitigated); 892 893 void update_gds_msr(void) 894 { 895 u64 mcu_ctrl_after; 896 u64 mcu_ctrl; 897 898 switch (gds_mitigation) { 899 case GDS_MITIGATION_OFF: 900 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 901 mcu_ctrl |= GDS_MITG_DIS; 902 break; 903 case GDS_MITIGATION_FULL_LOCKED: 904 /* 905 * The LOCKED state comes from the boot CPU. APs might not have 906 * the same state. Make sure the mitigation is enabled on all 907 * CPUs. 908 */ 909 case GDS_MITIGATION_FULL: 910 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 911 mcu_ctrl &= ~GDS_MITG_DIS; 912 break; 913 case GDS_MITIGATION_FORCE: 914 case GDS_MITIGATION_UCODE_NEEDED: 915 case GDS_MITIGATION_HYPERVISOR: 916 case GDS_MITIGATION_AUTO: 917 return; 918 } 919 920 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 921 922 /* 923 * Check to make sure that the WRMSR value was not ignored. Writes to 924 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 925 * processor was not. 926 */ 927 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 928 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 929 } 930 931 static void __init gds_select_mitigation(void) 932 { 933 u64 mcu_ctrl; 934 935 if (!boot_cpu_has_bug(X86_BUG_GDS)) 936 return; 937 938 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 939 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 940 return; 941 } 942 943 /* Will verify below that mitigation _can_ be disabled */ 944 if (gds_mitigation == GDS_MITIGATION_AUTO) { 945 if (should_mitigate_vuln(X86_BUG_GDS)) 946 gds_mitigation = GDS_MITIGATION_FULL; 947 else 948 gds_mitigation = GDS_MITIGATION_OFF; 949 } 950 951 /* No microcode */ 952 if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) { 953 if (gds_mitigation != GDS_MITIGATION_FORCE) 954 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 955 return; 956 } 957 958 /* Microcode has mitigation, use it */ 959 if (gds_mitigation == GDS_MITIGATION_FORCE) 960 gds_mitigation = GDS_MITIGATION_FULL; 961 962 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 963 if (mcu_ctrl & GDS_MITG_LOCKED) { 964 if (gds_mitigation == GDS_MITIGATION_OFF) 965 pr_warn("Mitigation locked. Disable failed.\n"); 966 967 /* 968 * The mitigation is selected from the boot CPU. All other CPUs 969 * _should_ have the same state. If the boot CPU isn't locked 970 * but others are then update_gds_msr() will WARN() of the state 971 * mismatch. If the boot CPU is locked update_gds_msr() will 972 * ensure the other CPUs have the mitigation enabled. 973 */ 974 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 975 } 976 } 977 978 static void __init gds_apply_mitigation(void) 979 { 980 if (!boot_cpu_has_bug(X86_BUG_GDS)) 981 return; 982 983 /* Microcode is present */ 984 if (x86_arch_cap_msr & ARCH_CAP_GDS_CTRL) 985 update_gds_msr(); 986 else if (gds_mitigation == GDS_MITIGATION_FORCE) { 987 /* 988 * This only needs to be done on the boot CPU so do it 989 * here rather than in update_gds_msr() 990 */ 991 setup_clear_cpu_cap(X86_FEATURE_AVX); 992 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 993 } 994 995 pr_info("%s\n", gds_strings[gds_mitigation]); 996 } 997 998 static int __init gds_parse_cmdline(char *str) 999 { 1000 if (!str) 1001 return -EINVAL; 1002 1003 if (!boot_cpu_has_bug(X86_BUG_GDS)) 1004 return 0; 1005 1006 if (!strcmp(str, "off")) 1007 gds_mitigation = GDS_MITIGATION_OFF; 1008 else if (!strcmp(str, "force")) 1009 gds_mitigation = GDS_MITIGATION_FORCE; 1010 1011 return 0; 1012 } 1013 early_param("gather_data_sampling", gds_parse_cmdline); 1014 1015 #undef pr_fmt 1016 #define pr_fmt(fmt) "Spectre V1 : " fmt 1017 1018 enum spectre_v1_mitigation { 1019 SPECTRE_V1_MITIGATION_NONE, 1020 SPECTRE_V1_MITIGATION_AUTO, 1021 }; 1022 1023 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 1024 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ? 1025 SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE; 1026 1027 static const char * const spectre_v1_strings[] = { 1028 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 1029 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 1030 }; 1031 1032 /* 1033 * Does SMAP provide full mitigation against speculative kernel access to 1034 * userspace? 1035 */ 1036 static bool smap_works_speculatively(void) 1037 { 1038 if (!boot_cpu_has(X86_FEATURE_SMAP)) 1039 return false; 1040 1041 /* 1042 * On CPUs which are vulnerable to Meltdown, SMAP does not 1043 * prevent speculative access to user data in the L1 cache. 1044 * Consider SMAP to be non-functional as a mitigation on these 1045 * CPUs. 1046 */ 1047 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 1048 return false; 1049 1050 return true; 1051 } 1052 1053 static void __init spectre_v1_select_mitigation(void) 1054 { 1055 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 1056 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1057 1058 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V1)) 1059 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1060 } 1061 1062 static void __init spectre_v1_apply_mitigation(void) 1063 { 1064 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 1065 return; 1066 1067 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 1068 /* 1069 * With Spectre v1, a user can speculatively control either 1070 * path of a conditional swapgs with a user-controlled GS 1071 * value. The mitigation is to add lfences to both code paths. 1072 * 1073 * If FSGSBASE is enabled, the user can put a kernel address in 1074 * GS, in which case SMAP provides no protection. 1075 * 1076 * If FSGSBASE is disabled, the user can only put a user space 1077 * address in GS. That makes an attack harder, but still 1078 * possible if there's no SMAP protection. 1079 */ 1080 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 1081 !smap_works_speculatively()) { 1082 /* 1083 * Mitigation can be provided from SWAPGS itself or 1084 * PTI as the CR3 write in the Meltdown mitigation 1085 * is serializing. 1086 * 1087 * If neither is there, mitigate with an LFENCE to 1088 * stop speculation through swapgs. 1089 */ 1090 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 1091 !boot_cpu_has(X86_FEATURE_PTI)) 1092 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 1093 1094 /* 1095 * Enable lfences in the kernel entry (non-swapgs) 1096 * paths, to prevent user entry from speculatively 1097 * skipping swapgs. 1098 */ 1099 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 1100 } 1101 } 1102 1103 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1104 } 1105 1106 static int __init nospectre_v1_cmdline(char *str) 1107 { 1108 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1109 return 0; 1110 } 1111 early_param("nospectre_v1", nospectre_v1_cmdline); 1112 1113 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 1114 1115 /* Depends on spectre_v2 mitigation selected already */ 1116 static inline bool cdt_possible(enum spectre_v2_mitigation mode) 1117 { 1118 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) || 1119 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) 1120 return false; 1121 1122 if (mode == SPECTRE_V2_RETPOLINE || 1123 mode == SPECTRE_V2_EIBRS_RETPOLINE) 1124 return true; 1125 1126 return false; 1127 } 1128 1129 #undef pr_fmt 1130 #define pr_fmt(fmt) "RETBleed: " fmt 1131 1132 enum its_mitigation { 1133 ITS_MITIGATION_OFF, 1134 ITS_MITIGATION_AUTO, 1135 ITS_MITIGATION_VMEXIT_ONLY, 1136 ITS_MITIGATION_ALIGNED_THUNKS, 1137 ITS_MITIGATION_RETPOLINE_STUFF, 1138 }; 1139 1140 static enum its_mitigation its_mitigation __ro_after_init = 1141 IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_MITIGATION_AUTO : ITS_MITIGATION_OFF; 1142 1143 enum retbleed_mitigation { 1144 RETBLEED_MITIGATION_NONE, 1145 RETBLEED_MITIGATION_AUTO, 1146 RETBLEED_MITIGATION_UNRET, 1147 RETBLEED_MITIGATION_IBPB, 1148 RETBLEED_MITIGATION_IBRS, 1149 RETBLEED_MITIGATION_EIBRS, 1150 RETBLEED_MITIGATION_STUFF, 1151 }; 1152 1153 static const char * const retbleed_strings[] = { 1154 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 1155 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 1156 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 1157 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 1158 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 1159 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 1160 }; 1161 1162 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 1163 IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE; 1164 1165 static int __ro_after_init retbleed_nosmt = false; 1166 1167 enum srso_mitigation { 1168 SRSO_MITIGATION_NONE, 1169 SRSO_MITIGATION_AUTO, 1170 SRSO_MITIGATION_UCODE_NEEDED, 1171 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 1172 SRSO_MITIGATION_MICROCODE, 1173 SRSO_MITIGATION_NOSMT, 1174 SRSO_MITIGATION_SAFE_RET, 1175 SRSO_MITIGATION_IBPB, 1176 SRSO_MITIGATION_IBPB_ON_VMEXIT, 1177 SRSO_MITIGATION_BP_SPEC_REDUCE, 1178 }; 1179 1180 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO; 1181 1182 static int __init retbleed_parse_cmdline(char *str) 1183 { 1184 if (!str) 1185 return -EINVAL; 1186 1187 while (str) { 1188 char *next = strchr(str, ','); 1189 if (next) { 1190 *next = 0; 1191 next++; 1192 } 1193 1194 if (!strcmp(str, "off")) { 1195 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1196 } else if (!strcmp(str, "auto")) { 1197 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1198 } else if (!strcmp(str, "unret")) { 1199 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1200 } else if (!strcmp(str, "ibpb")) { 1201 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1202 } else if (!strcmp(str, "stuff")) { 1203 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1204 } else if (!strcmp(str, "nosmt")) { 1205 retbleed_nosmt = true; 1206 } else if (!strcmp(str, "force")) { 1207 setup_force_cpu_bug(X86_BUG_RETBLEED); 1208 } else { 1209 pr_err("Ignoring unknown retbleed option (%s).", str); 1210 } 1211 1212 str = next; 1213 } 1214 1215 return 0; 1216 } 1217 early_param("retbleed", retbleed_parse_cmdline); 1218 1219 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 1220 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 1221 1222 static void __init retbleed_select_mitigation(void) 1223 { 1224 if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) { 1225 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1226 return; 1227 } 1228 1229 switch (retbleed_mitigation) { 1230 case RETBLEED_MITIGATION_UNRET: 1231 if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { 1232 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1233 pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); 1234 } 1235 break; 1236 case RETBLEED_MITIGATION_IBPB: 1237 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 1238 pr_err("WARNING: CPU does not support IBPB.\n"); 1239 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1240 } else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 1241 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 1242 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1243 } 1244 break; 1245 case RETBLEED_MITIGATION_STUFF: 1246 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1247 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); 1248 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1249 } else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1250 pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); 1251 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1252 } 1253 break; 1254 default: 1255 break; 1256 } 1257 1258 if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO) 1259 return; 1260 1261 if (!should_mitigate_vuln(X86_BUG_RETBLEED)) { 1262 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1263 return; 1264 } 1265 1266 /* Intel mitigation selected in retbleed_update_mitigation() */ 1267 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1268 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1269 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) 1270 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1271 else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && 1272 boot_cpu_has(X86_FEATURE_IBPB)) 1273 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1274 else 1275 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1276 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1277 /* Final mitigation depends on spectre-v2 selection */ 1278 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1279 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1280 else if (boot_cpu_has(X86_FEATURE_IBRS)) 1281 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1282 else 1283 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1284 } 1285 } 1286 1287 static void __init retbleed_update_mitigation(void) 1288 { 1289 if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) 1290 return; 1291 1292 /* ITS can also enable stuffing */ 1293 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF) 1294 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1295 1296 /* If SRSO is using IBPB, that works for retbleed too */ 1297 if (srso_mitigation == SRSO_MITIGATION_IBPB) 1298 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1299 1300 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF && 1301 !cdt_possible(spectre_v2_enabled)) { 1302 pr_err("WARNING: retbleed=stuff depends on retpoline\n"); 1303 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1304 } 1305 1306 /* 1307 * Let IBRS trump all on Intel without affecting the effects of the 1308 * retbleed= cmdline option except for call depth based stuffing 1309 */ 1310 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1311 switch (spectre_v2_enabled) { 1312 case SPECTRE_V2_IBRS: 1313 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1314 break; 1315 case SPECTRE_V2_EIBRS: 1316 case SPECTRE_V2_EIBRS_RETPOLINE: 1317 case SPECTRE_V2_EIBRS_LFENCE: 1318 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1319 break; 1320 default: 1321 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) { 1322 if (retbleed_mitigation != RETBLEED_MITIGATION_NONE) 1323 pr_err(RETBLEED_INTEL_MSG); 1324 1325 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1326 } 1327 } 1328 } 1329 1330 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1331 } 1332 1333 static void __init retbleed_apply_mitigation(void) 1334 { 1335 bool mitigate_smt = false; 1336 1337 switch (retbleed_mitigation) { 1338 case RETBLEED_MITIGATION_NONE: 1339 return; 1340 1341 case RETBLEED_MITIGATION_UNRET: 1342 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1343 setup_force_cpu_cap(X86_FEATURE_UNRET); 1344 1345 set_return_thunk(retbleed_return_thunk); 1346 1347 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1348 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1349 pr_err(RETBLEED_UNTRAIN_MSG); 1350 1351 mitigate_smt = true; 1352 break; 1353 1354 case RETBLEED_MITIGATION_IBPB: 1355 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1356 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1357 mitigate_smt = true; 1358 1359 /* 1360 * IBPB on entry already obviates the need for 1361 * software-based untraining so clear those in case some 1362 * other mitigation like SRSO has selected them. 1363 */ 1364 setup_clear_cpu_cap(X86_FEATURE_UNRET); 1365 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 1366 1367 /* 1368 * There is no need for RSB filling: write_ibpb() ensures 1369 * all predictions, including the RSB, are invalidated, 1370 * regardless of IBPB implementation. 1371 */ 1372 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1373 1374 break; 1375 1376 case RETBLEED_MITIGATION_STUFF: 1377 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1378 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1379 1380 set_return_thunk(call_depth_return_thunk); 1381 break; 1382 1383 default: 1384 break; 1385 } 1386 1387 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1388 (retbleed_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)) 1389 cpu_smt_disable(false); 1390 } 1391 1392 #undef pr_fmt 1393 #define pr_fmt(fmt) "ITS: " fmt 1394 1395 static const char * const its_strings[] = { 1396 [ITS_MITIGATION_OFF] = "Vulnerable", 1397 [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", 1398 [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", 1399 [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", 1400 }; 1401 1402 static int __init its_parse_cmdline(char *str) 1403 { 1404 if (!str) 1405 return -EINVAL; 1406 1407 if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) { 1408 pr_err("Mitigation disabled at compile time, ignoring option (%s)", str); 1409 return 0; 1410 } 1411 1412 if (!strcmp(str, "off")) { 1413 its_mitigation = ITS_MITIGATION_OFF; 1414 } else if (!strcmp(str, "on")) { 1415 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1416 } else if (!strcmp(str, "force")) { 1417 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1418 setup_force_cpu_bug(X86_BUG_ITS); 1419 } else if (!strcmp(str, "vmexit")) { 1420 its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; 1421 } else if (!strcmp(str, "stuff")) { 1422 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1423 } else { 1424 pr_err("Ignoring unknown indirect_target_selection option (%s).", str); 1425 } 1426 1427 return 0; 1428 } 1429 early_param("indirect_target_selection", its_parse_cmdline); 1430 1431 static void __init its_select_mitigation(void) 1432 { 1433 if (!boot_cpu_has_bug(X86_BUG_ITS)) { 1434 its_mitigation = ITS_MITIGATION_OFF; 1435 return; 1436 } 1437 1438 if (its_mitigation == ITS_MITIGATION_AUTO) { 1439 if (should_mitigate_vuln(X86_BUG_ITS)) 1440 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1441 else 1442 its_mitigation = ITS_MITIGATION_OFF; 1443 } 1444 1445 if (its_mitigation == ITS_MITIGATION_OFF) 1446 return; 1447 1448 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || 1449 !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) { 1450 pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); 1451 its_mitigation = ITS_MITIGATION_OFF; 1452 return; 1453 } 1454 1455 if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { 1456 pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); 1457 its_mitigation = ITS_MITIGATION_OFF; 1458 return; 1459 } 1460 1461 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1462 !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1463 pr_err("RSB stuff mitigation not supported, using default\n"); 1464 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1465 } 1466 1467 if (its_mitigation == ITS_MITIGATION_VMEXIT_ONLY && 1468 !boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) 1469 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1470 } 1471 1472 static void __init its_update_mitigation(void) 1473 { 1474 if (!boot_cpu_has_bug(X86_BUG_ITS)) 1475 return; 1476 1477 switch (spectre_v2_enabled) { 1478 case SPECTRE_V2_NONE: 1479 if (its_mitigation != ITS_MITIGATION_OFF) 1480 pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); 1481 its_mitigation = ITS_MITIGATION_OFF; 1482 break; 1483 case SPECTRE_V2_RETPOLINE: 1484 case SPECTRE_V2_EIBRS_RETPOLINE: 1485 /* Retpoline+CDT mitigates ITS */ 1486 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) 1487 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1488 break; 1489 case SPECTRE_V2_LFENCE: 1490 case SPECTRE_V2_EIBRS_LFENCE: 1491 pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); 1492 its_mitigation = ITS_MITIGATION_OFF; 1493 break; 1494 default: 1495 break; 1496 } 1497 1498 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1499 !cdt_possible(spectre_v2_enabled)) 1500 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1501 1502 pr_info("%s\n", its_strings[its_mitigation]); 1503 } 1504 1505 static void __init its_apply_mitigation(void) 1506 { 1507 switch (its_mitigation) { 1508 case ITS_MITIGATION_OFF: 1509 case ITS_MITIGATION_AUTO: 1510 case ITS_MITIGATION_VMEXIT_ONLY: 1511 break; 1512 case ITS_MITIGATION_ALIGNED_THUNKS: 1513 if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) 1514 setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); 1515 1516 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1517 set_return_thunk(its_return_thunk); 1518 break; 1519 case ITS_MITIGATION_RETPOLINE_STUFF: 1520 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1521 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1522 set_return_thunk(call_depth_return_thunk); 1523 break; 1524 } 1525 } 1526 1527 #undef pr_fmt 1528 #define pr_fmt(fmt) "Transient Scheduler Attacks: " fmt 1529 1530 enum tsa_mitigations { 1531 TSA_MITIGATION_NONE, 1532 TSA_MITIGATION_AUTO, 1533 TSA_MITIGATION_UCODE_NEEDED, 1534 TSA_MITIGATION_USER_KERNEL, 1535 TSA_MITIGATION_VM, 1536 TSA_MITIGATION_FULL, 1537 }; 1538 1539 static const char * const tsa_strings[] = { 1540 [TSA_MITIGATION_NONE] = "Vulnerable", 1541 [TSA_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 1542 [TSA_MITIGATION_USER_KERNEL] = "Mitigation: Clear CPU buffers: user/kernel boundary", 1543 [TSA_MITIGATION_VM] = "Mitigation: Clear CPU buffers: VM", 1544 [TSA_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 1545 }; 1546 1547 static enum tsa_mitigations tsa_mitigation __ro_after_init = 1548 IS_ENABLED(CONFIG_MITIGATION_TSA) ? TSA_MITIGATION_AUTO : TSA_MITIGATION_NONE; 1549 1550 static int __init tsa_parse_cmdline(char *str) 1551 { 1552 if (!str) 1553 return -EINVAL; 1554 1555 if (!strcmp(str, "off")) 1556 tsa_mitigation = TSA_MITIGATION_NONE; 1557 else if (!strcmp(str, "on")) 1558 tsa_mitigation = TSA_MITIGATION_FULL; 1559 else if (!strcmp(str, "user")) 1560 tsa_mitigation = TSA_MITIGATION_USER_KERNEL; 1561 else if (!strcmp(str, "vm")) 1562 tsa_mitigation = TSA_MITIGATION_VM; 1563 else 1564 pr_err("Ignoring unknown tsa=%s option.\n", str); 1565 1566 return 0; 1567 } 1568 early_param("tsa", tsa_parse_cmdline); 1569 1570 static void __init tsa_select_mitigation(void) 1571 { 1572 if (!boot_cpu_has_bug(X86_BUG_TSA)) { 1573 tsa_mitigation = TSA_MITIGATION_NONE; 1574 return; 1575 } 1576 1577 if (tsa_mitigation == TSA_MITIGATION_AUTO) { 1578 bool vm = false, uk = false; 1579 1580 tsa_mitigation = TSA_MITIGATION_NONE; 1581 1582 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 1583 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) { 1584 tsa_mitigation = TSA_MITIGATION_USER_KERNEL; 1585 uk = true; 1586 } 1587 1588 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 1589 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) { 1590 tsa_mitigation = TSA_MITIGATION_VM; 1591 vm = true; 1592 } 1593 1594 if (uk && vm) 1595 tsa_mitigation = TSA_MITIGATION_FULL; 1596 } 1597 1598 if (tsa_mitigation == TSA_MITIGATION_NONE) 1599 return; 1600 1601 if (!boot_cpu_has(X86_FEATURE_VERW_CLEAR)) 1602 tsa_mitigation = TSA_MITIGATION_UCODE_NEEDED; 1603 1604 /* 1605 * No need to set verw_clear_cpu_buf_mitigation_selected - it 1606 * doesn't fit all cases here and it is not needed because this 1607 * is the only VERW-based mitigation on AMD. 1608 */ 1609 pr_info("%s\n", tsa_strings[tsa_mitigation]); 1610 } 1611 1612 static void __init tsa_apply_mitigation(void) 1613 { 1614 switch (tsa_mitigation) { 1615 case TSA_MITIGATION_USER_KERNEL: 1616 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 1617 break; 1618 case TSA_MITIGATION_VM: 1619 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 1620 break; 1621 case TSA_MITIGATION_FULL: 1622 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 1623 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 1624 break; 1625 default: 1626 break; 1627 } 1628 } 1629 1630 #undef pr_fmt 1631 #define pr_fmt(fmt) "Spectre V2 : " fmt 1632 1633 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1634 SPECTRE_V2_USER_NONE; 1635 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1636 SPECTRE_V2_USER_NONE; 1637 1638 #ifdef CONFIG_MITIGATION_RETPOLINE 1639 static bool spectre_v2_bad_module; 1640 1641 bool retpoline_module_ok(bool has_retpoline) 1642 { 1643 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1644 return true; 1645 1646 pr_err("System may be vulnerable to spectre v2\n"); 1647 spectre_v2_bad_module = true; 1648 return false; 1649 } 1650 1651 static inline const char *spectre_v2_module_string(void) 1652 { 1653 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1654 } 1655 1656 /* 1657 * The "retpoline sequence" is the "call;mov;ret" sequence that 1658 * replaces normal indirect branch instructions. Differentiate 1659 * *the* retpoline sequence from the LFENCE-prefixed indirect 1660 * branches that simply use the retpoline infrastructure. 1661 */ 1662 static inline bool retpoline_seq_enabled(void) 1663 { 1664 return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE); 1665 } 1666 1667 #else 1668 static inline const char *spectre_v2_module_string(void) { return ""; } 1669 static inline bool retpoline_seq_enabled(void) { return false; } 1670 #endif 1671 1672 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1673 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1674 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1675 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1676 1677 #ifdef CONFIG_BPF_SYSCALL 1678 void unpriv_ebpf_notify(int new_state) 1679 { 1680 if (new_state) 1681 return; 1682 1683 /* Unprivileged eBPF is enabled */ 1684 1685 switch (spectre_v2_enabled) { 1686 case SPECTRE_V2_EIBRS: 1687 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1688 break; 1689 case SPECTRE_V2_EIBRS_LFENCE: 1690 if (sched_smt_active()) 1691 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1692 break; 1693 default: 1694 break; 1695 } 1696 } 1697 #endif 1698 1699 /* The kernel command line selection for spectre v2 */ 1700 enum spectre_v2_mitigation_cmd { 1701 SPECTRE_V2_CMD_NONE, 1702 SPECTRE_V2_CMD_AUTO, 1703 SPECTRE_V2_CMD_FORCE, 1704 SPECTRE_V2_CMD_RETPOLINE, 1705 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1706 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1707 SPECTRE_V2_CMD_EIBRS, 1708 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1709 SPECTRE_V2_CMD_EIBRS_LFENCE, 1710 SPECTRE_V2_CMD_IBRS, 1711 }; 1712 1713 static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init = 1714 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE; 1715 1716 enum spectre_v2_user_mitigation_cmd { 1717 SPECTRE_V2_USER_CMD_NONE, 1718 SPECTRE_V2_USER_CMD_AUTO, 1719 SPECTRE_V2_USER_CMD_FORCE, 1720 SPECTRE_V2_USER_CMD_PRCTL, 1721 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1722 SPECTRE_V2_USER_CMD_SECCOMP, 1723 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1724 }; 1725 1726 static enum spectre_v2_user_mitigation_cmd spectre_v2_user_cmd __ro_after_init = 1727 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE; 1728 1729 static const char * const spectre_v2_user_strings[] = { 1730 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1731 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1732 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1733 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1734 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1735 }; 1736 1737 static int __init spectre_v2_user_parse_cmdline(char *str) 1738 { 1739 if (!str) 1740 return -EINVAL; 1741 1742 if (!strcmp(str, "auto")) 1743 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_AUTO; 1744 else if (!strcmp(str, "off")) 1745 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_NONE; 1746 else if (!strcmp(str, "on")) 1747 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_FORCE; 1748 else if (!strcmp(str, "prctl")) 1749 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL; 1750 else if (!strcmp(str, "prctl,ibpb")) 1751 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL_IBPB; 1752 else if (!strcmp(str, "seccomp")) 1753 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP; 1754 else if (!strcmp(str, "seccomp,ibpb")) 1755 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP_IBPB; 1756 else 1757 pr_err("Ignoring unknown spectre_v2_user option (%s).", str); 1758 1759 return 0; 1760 } 1761 early_param("spectre_v2_user", spectre_v2_user_parse_cmdline); 1762 1763 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1764 { 1765 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1766 } 1767 1768 static void __init spectre_v2_user_select_mitigation(void) 1769 { 1770 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1771 return; 1772 1773 switch (spectre_v2_user_cmd) { 1774 case SPECTRE_V2_USER_CMD_NONE: 1775 return; 1776 case SPECTRE_V2_USER_CMD_FORCE: 1777 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1778 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1779 break; 1780 case SPECTRE_V2_USER_CMD_AUTO: 1781 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2_USER)) 1782 break; 1783 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1784 if (smt_mitigations == SMT_MITIGATIONS_OFF) 1785 break; 1786 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1787 break; 1788 case SPECTRE_V2_USER_CMD_PRCTL: 1789 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1790 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1791 break; 1792 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1793 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1794 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1795 break; 1796 case SPECTRE_V2_USER_CMD_SECCOMP: 1797 if (IS_ENABLED(CONFIG_SECCOMP)) 1798 spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP; 1799 else 1800 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1801 spectre_v2_user_stibp = spectre_v2_user_ibpb; 1802 break; 1803 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1804 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1805 if (IS_ENABLED(CONFIG_SECCOMP)) 1806 spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP; 1807 else 1808 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1809 break; 1810 } 1811 1812 /* 1813 * At this point, an STIBP mode other than "off" has been set. 1814 * If STIBP support is not being forced, check if STIBP always-on 1815 * is preferred. 1816 */ 1817 if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1818 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) && 1819 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1820 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1821 1822 if (!boot_cpu_has(X86_FEATURE_IBPB)) 1823 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1824 1825 if (!boot_cpu_has(X86_FEATURE_STIBP)) 1826 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1827 } 1828 1829 static void __init spectre_v2_user_update_mitigation(void) 1830 { 1831 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1832 return; 1833 1834 /* The spectre_v2 cmd line can override spectre_v2_user options */ 1835 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) { 1836 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1837 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1838 } else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) { 1839 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1840 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1841 } 1842 1843 /* 1844 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1845 * is not required. 1846 * 1847 * Intel's Enhanced IBRS also protects against cross-thread branch target 1848 * injection in user-mode as the IBRS bit remains always set which 1849 * implicitly enables cross-thread protections. However, in legacy IBRS 1850 * mode, the IBRS bit is set only on kernel entry and cleared on return 1851 * to userspace. AMD Automatic IBRS also does not protect userspace. 1852 * These modes therefore disable the implicit cross-thread protection, 1853 * so allow for STIBP to be selected in those cases. 1854 */ 1855 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1856 !cpu_smt_possible() || 1857 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1858 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) { 1859 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1860 return; 1861 } 1862 1863 if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE && 1864 (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1865 retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) { 1866 if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT && 1867 spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED) 1868 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1869 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1870 } 1871 pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]); 1872 } 1873 1874 static void __init spectre_v2_user_apply_mitigation(void) 1875 { 1876 /* Initialize Indirect Branch Prediction Barrier */ 1877 if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) { 1878 static_branch_enable(&switch_vcpu_ibpb); 1879 1880 switch (spectre_v2_user_ibpb) { 1881 case SPECTRE_V2_USER_STRICT: 1882 static_branch_enable(&switch_mm_always_ibpb); 1883 break; 1884 case SPECTRE_V2_USER_PRCTL: 1885 case SPECTRE_V2_USER_SECCOMP: 1886 static_branch_enable(&switch_mm_cond_ibpb); 1887 break; 1888 default: 1889 break; 1890 } 1891 1892 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1893 static_key_enabled(&switch_mm_always_ibpb) ? 1894 "always-on" : "conditional"); 1895 } 1896 } 1897 1898 static const char * const spectre_v2_strings[] = { 1899 [SPECTRE_V2_NONE] = "Vulnerable", 1900 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1901 [SPECTRE_V2_LFENCE] = "Vulnerable: LFENCE", 1902 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1903 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1904 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1905 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1906 }; 1907 1908 static bool nospectre_v2 __ro_after_init; 1909 1910 static int __init nospectre_v2_parse_cmdline(char *str) 1911 { 1912 nospectre_v2 = true; 1913 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 1914 return 0; 1915 } 1916 early_param("nospectre_v2", nospectre_v2_parse_cmdline); 1917 1918 static int __init spectre_v2_parse_cmdline(char *str) 1919 { 1920 if (!str) 1921 return -EINVAL; 1922 1923 if (nospectre_v2) 1924 return 0; 1925 1926 if (!strcmp(str, "off")) { 1927 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 1928 } else if (!strcmp(str, "on")) { 1929 spectre_v2_cmd = SPECTRE_V2_CMD_FORCE; 1930 setup_force_cpu_bug(X86_BUG_SPECTRE_V2); 1931 setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER); 1932 } else if (!strcmp(str, "retpoline")) { 1933 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE; 1934 } else if (!strcmp(str, "retpoline,amd") || 1935 !strcmp(str, "retpoline,lfence")) { 1936 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_LFENCE; 1937 } else if (!strcmp(str, "retpoline,generic")) { 1938 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_GENERIC; 1939 } else if (!strcmp(str, "eibrs")) { 1940 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS; 1941 } else if (!strcmp(str, "eibrs,lfence")) { 1942 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_LFENCE; 1943 } else if (!strcmp(str, "eibrs,retpoline")) { 1944 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_RETPOLINE; 1945 } else if (!strcmp(str, "auto")) { 1946 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 1947 } else if (!strcmp(str, "ibrs")) { 1948 spectre_v2_cmd = SPECTRE_V2_CMD_IBRS; 1949 } else { 1950 pr_err("Ignoring unknown spectre_v2 option (%s).", str); 1951 } 1952 1953 return 0; 1954 } 1955 early_param("spectre_v2", spectre_v2_parse_cmdline); 1956 1957 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1958 { 1959 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 1960 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1961 return SPECTRE_V2_NONE; 1962 } 1963 1964 return SPECTRE_V2_RETPOLINE; 1965 } 1966 1967 static bool __ro_after_init rrsba_disabled; 1968 1969 /* Disable in-kernel use of non-RSB RET predictors */ 1970 static void __init spec_ctrl_disable_kernel_rrsba(void) 1971 { 1972 if (rrsba_disabled) 1973 return; 1974 1975 if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) { 1976 rrsba_disabled = true; 1977 return; 1978 } 1979 1980 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1981 return; 1982 1983 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1984 update_spec_ctrl(x86_spec_ctrl_base); 1985 rrsba_disabled = true; 1986 } 1987 1988 static void __init spectre_v2_select_rsb_mitigation(enum spectre_v2_mitigation mode) 1989 { 1990 /* 1991 * WARNING! There are many subtleties to consider when changing *any* 1992 * code related to RSB-related mitigations. Before doing so, carefully 1993 * read the following document, and update if necessary: 1994 * 1995 * Documentation/admin-guide/hw-vuln/rsb.rst 1996 * 1997 * In an overly simplified nutshell: 1998 * 1999 * - User->user RSB attacks are conditionally mitigated during 2000 * context switches by cond_mitigation -> write_ibpb(). 2001 * 2002 * - User->kernel and guest->host attacks are mitigated by eIBRS or 2003 * RSB filling. 2004 * 2005 * Though, depending on config, note that other alternative 2006 * mitigations may end up getting used instead, e.g., IBPB on 2007 * entry/vmexit, call depth tracking, or return thunks. 2008 */ 2009 2010 switch (mode) { 2011 case SPECTRE_V2_NONE: 2012 break; 2013 2014 case SPECTRE_V2_EIBRS: 2015 case SPECTRE_V2_EIBRS_LFENCE: 2016 case SPECTRE_V2_EIBRS_RETPOLINE: 2017 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2018 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 2019 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 2020 } 2021 break; 2022 2023 case SPECTRE_V2_RETPOLINE: 2024 case SPECTRE_V2_LFENCE: 2025 case SPECTRE_V2_IBRS: 2026 pr_info("Spectre v2 / SpectreRSB: Filling RSB on context switch and VMEXIT\n"); 2027 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 2028 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 2029 break; 2030 2031 default: 2032 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation\n"); 2033 dump_stack(); 2034 break; 2035 } 2036 } 2037 2038 /* 2039 * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by 2040 * branch history in userspace. Not needed if BHI_NO is set. 2041 */ 2042 static bool __init spec_ctrl_bhi_dis(void) 2043 { 2044 if (!boot_cpu_has(X86_FEATURE_BHI_CTRL)) 2045 return false; 2046 2047 x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S; 2048 update_spec_ctrl(x86_spec_ctrl_base); 2049 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW); 2050 2051 return true; 2052 } 2053 2054 enum bhi_mitigations { 2055 BHI_MITIGATION_OFF, 2056 BHI_MITIGATION_AUTO, 2057 BHI_MITIGATION_ON, 2058 BHI_MITIGATION_VMEXIT_ONLY, 2059 }; 2060 2061 static enum bhi_mitigations bhi_mitigation __ro_after_init = 2062 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF; 2063 2064 static int __init spectre_bhi_parse_cmdline(char *str) 2065 { 2066 if (!str) 2067 return -EINVAL; 2068 2069 if (!strcmp(str, "off")) 2070 bhi_mitigation = BHI_MITIGATION_OFF; 2071 else if (!strcmp(str, "on")) 2072 bhi_mitigation = BHI_MITIGATION_ON; 2073 else if (!strcmp(str, "vmexit")) 2074 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 2075 else 2076 pr_err("Ignoring unknown spectre_bhi option (%s)", str); 2077 2078 return 0; 2079 } 2080 early_param("spectre_bhi", spectre_bhi_parse_cmdline); 2081 2082 static void __init bhi_select_mitigation(void) 2083 { 2084 if (!boot_cpu_has(X86_BUG_BHI)) 2085 bhi_mitigation = BHI_MITIGATION_OFF; 2086 2087 if (bhi_mitigation != BHI_MITIGATION_AUTO) 2088 return; 2089 2090 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST)) { 2091 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL)) 2092 bhi_mitigation = BHI_MITIGATION_ON; 2093 else 2094 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 2095 } else { 2096 bhi_mitigation = BHI_MITIGATION_OFF; 2097 } 2098 } 2099 2100 static void __init bhi_update_mitigation(void) 2101 { 2102 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) 2103 bhi_mitigation = BHI_MITIGATION_OFF; 2104 } 2105 2106 static void __init bhi_apply_mitigation(void) 2107 { 2108 if (bhi_mitigation == BHI_MITIGATION_OFF) 2109 return; 2110 2111 /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ 2112 if (retpoline_seq_enabled()) { 2113 spec_ctrl_disable_kernel_rrsba(); 2114 if (rrsba_disabled) 2115 return; 2116 } 2117 2118 if (!IS_ENABLED(CONFIG_X86_64)) 2119 return; 2120 2121 /* Mitigate in hardware if supported */ 2122 if (spec_ctrl_bhi_dis()) 2123 return; 2124 2125 if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) { 2126 pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n"); 2127 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2128 return; 2129 } 2130 2131 pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n"); 2132 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP); 2133 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2134 } 2135 2136 static void __init spectre_v2_select_mitigation(void) 2137 { 2138 if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE || 2139 spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 2140 spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 2141 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 2142 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 2143 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 2144 pr_err("RETPOLINE selected but not compiled in. Switching to AUTO select\n"); 2145 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2146 } 2147 2148 if ((spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS || 2149 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 2150 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 2151 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2152 pr_err("EIBRS selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n"); 2153 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2154 } 2155 2156 if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 2157 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 2158 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 2159 pr_err("LFENCE selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n"); 2160 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2161 } 2162 2163 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) { 2164 pr_err("IBRS selected but not compiled in. Switching to AUTO select\n"); 2165 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2166 } 2167 2168 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 2169 pr_err("IBRS selected but not Intel CPU. Switching to AUTO select\n"); 2170 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2171 } 2172 2173 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 2174 pr_err("IBRS selected but CPU doesn't have IBRS. Switching to AUTO select\n"); 2175 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2176 } 2177 2178 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 2179 pr_err("IBRS selected but running as XenPV guest. Switching to AUTO select\n"); 2180 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2181 } 2182 2183 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) { 2184 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 2185 return; 2186 } 2187 2188 switch (spectre_v2_cmd) { 2189 case SPECTRE_V2_CMD_NONE: 2190 return; 2191 2192 case SPECTRE_V2_CMD_AUTO: 2193 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2)) 2194 break; 2195 fallthrough; 2196 case SPECTRE_V2_CMD_FORCE: 2197 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2198 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2199 break; 2200 } 2201 2202 spectre_v2_enabled = spectre_v2_select_retpoline(); 2203 break; 2204 2205 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 2206 pr_err(SPECTRE_V2_LFENCE_MSG); 2207 spectre_v2_enabled = SPECTRE_V2_LFENCE; 2208 break; 2209 2210 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 2211 spectre_v2_enabled = SPECTRE_V2_RETPOLINE; 2212 break; 2213 2214 case SPECTRE_V2_CMD_RETPOLINE: 2215 spectre_v2_enabled = spectre_v2_select_retpoline(); 2216 break; 2217 2218 case SPECTRE_V2_CMD_IBRS: 2219 spectre_v2_enabled = SPECTRE_V2_IBRS; 2220 break; 2221 2222 case SPECTRE_V2_CMD_EIBRS: 2223 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2224 break; 2225 2226 case SPECTRE_V2_CMD_EIBRS_LFENCE: 2227 spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE; 2228 break; 2229 2230 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 2231 spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE; 2232 break; 2233 } 2234 } 2235 2236 static void __init spectre_v2_update_mitigation(void) 2237 { 2238 if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO && 2239 !spectre_v2_in_eibrs_mode(spectre_v2_enabled)) { 2240 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && 2241 boot_cpu_has_bug(X86_BUG_RETBLEED) && 2242 retbleed_mitigation != RETBLEED_MITIGATION_NONE && 2243 retbleed_mitigation != RETBLEED_MITIGATION_STUFF && 2244 boot_cpu_has(X86_FEATURE_IBRS) && 2245 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 2246 spectre_v2_enabled = SPECTRE_V2_IBRS; 2247 } 2248 } 2249 2250 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2251 pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); 2252 } 2253 2254 #ifdef CONFIG_BPF_JIT 2255 static void __bpf_arch_ibpb(void *unused) 2256 { 2257 write_ibpb(); 2258 } 2259 2260 void bpf_arch_ibpb(void) 2261 { 2262 on_each_cpu(__bpf_arch_ibpb, NULL, 1); 2263 } 2264 2265 static bool __init cpu_wants_ibpb_bpf(void) 2266 { 2267 /* A genuine retpoline already neutralizes ring0 indirect predictions */ 2268 if (retpoline_seq_enabled()) 2269 return false; 2270 2271 return boot_cpu_has(X86_FEATURE_IBPB); 2272 } 2273 #endif 2274 2275 static void __init spectre_v2_apply_mitigation(void) 2276 { 2277 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2278 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 2279 2280 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2281 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 2282 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 2283 } else { 2284 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 2285 update_spec_ctrl(x86_spec_ctrl_base); 2286 } 2287 } 2288 2289 switch (spectre_v2_enabled) { 2290 case SPECTRE_V2_NONE: 2291 return; 2292 2293 case SPECTRE_V2_EIBRS: 2294 break; 2295 2296 case SPECTRE_V2_IBRS: 2297 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 2298 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 2299 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 2300 break; 2301 2302 case SPECTRE_V2_LFENCE: 2303 case SPECTRE_V2_EIBRS_LFENCE: 2304 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 2305 fallthrough; 2306 2307 case SPECTRE_V2_RETPOLINE: 2308 case SPECTRE_V2_EIBRS_RETPOLINE: 2309 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 2310 break; 2311 } 2312 2313 /* 2314 * Disable alternate RSB predictions in kernel when indirect CALLs and 2315 * JMPs gets protection against BHI and Intramode-BTI, but RET 2316 * prediction from a non-RSB predictor is still a risk. 2317 */ 2318 if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE || 2319 spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE || 2320 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) 2321 spec_ctrl_disable_kernel_rrsba(); 2322 2323 spectre_v2_select_rsb_mitigation(spectre_v2_enabled); 2324 2325 /* 2326 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 2327 * and Enhanced IBRS protect firmware too, so enable IBRS around 2328 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 2329 * otherwise enabled. 2330 * 2331 * Use "spectre_v2_enabled" to check Enhanced IBRS instead of 2332 * boot_cpu_has(), because the user might select retpoline on the kernel 2333 * command line and if the CPU supports Enhanced IBRS, kernel might 2334 * un-intentionally not enable IBRS around firmware calls. 2335 */ 2336 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 2337 boot_cpu_has(X86_FEATURE_IBPB) && 2338 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2339 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 2340 2341 if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) { 2342 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 2343 pr_info("Enabling Speculation Barrier for firmware calls\n"); 2344 } 2345 2346 } else if (boot_cpu_has(X86_FEATURE_IBRS) && 2347 !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2348 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 2349 pr_info("Enabling Restricted Speculation for firmware calls\n"); 2350 } 2351 2352 #ifdef CONFIG_BPF_JIT 2353 if (cpu_wants_ibpb_bpf()) { 2354 static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb); 2355 static_branch_enable(&bpf_pred_flush_enabled); 2356 pr_info("Enabling IBPB for BPF\n"); 2357 } 2358 #endif 2359 } 2360 2361 static void update_stibp_msr(void * __unused) 2362 { 2363 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 2364 update_spec_ctrl(val); 2365 } 2366 2367 /* Update x86_spec_ctrl_base in case SMT state changed. */ 2368 static void update_stibp_strict(void) 2369 { 2370 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 2371 2372 if (sched_smt_active()) 2373 mask |= SPEC_CTRL_STIBP; 2374 2375 if (mask == x86_spec_ctrl_base) 2376 return; 2377 2378 pr_info("Update user space SMT mitigation: STIBP %s\n", 2379 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 2380 x86_spec_ctrl_base = mask; 2381 on_each_cpu(update_stibp_msr, NULL, 1); 2382 } 2383 2384 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 2385 static void update_indir_branch_cond(void) 2386 { 2387 if (sched_smt_active()) 2388 static_branch_enable(&switch_to_cond_stibp); 2389 else 2390 static_branch_disable(&switch_to_cond_stibp); 2391 } 2392 2393 #undef pr_fmt 2394 #define pr_fmt(fmt) fmt 2395 2396 /* Update the static key controlling the MDS CPU buffer clear in idle */ 2397 static void update_mds_branch_idle(void) 2398 { 2399 /* 2400 * Enable the idle clearing if SMT is active on CPUs which are 2401 * affected only by MSBDS and not any other MDS variant. 2402 * 2403 * The other variants cannot be mitigated when SMT is enabled, so 2404 * clearing the buffers on idle just to prevent the Store Buffer 2405 * repartitioning leak would be a window dressing exercise. 2406 */ 2407 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 2408 return; 2409 2410 if (sched_smt_active()) { 2411 static_branch_enable(&cpu_buf_idle_clear); 2412 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 2413 (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) { 2414 static_branch_disable(&cpu_buf_idle_clear); 2415 } 2416 } 2417 2418 #undef pr_fmt 2419 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 2420 2421 static enum ssb_mitigation ssb_mode __ro_after_init = 2422 IS_ENABLED(CONFIG_MITIGATION_SSB) ? SPEC_STORE_BYPASS_AUTO : SPEC_STORE_BYPASS_NONE; 2423 2424 static const char * const ssb_strings[] = { 2425 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 2426 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 2427 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 2428 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 2429 }; 2430 2431 static bool nossb __ro_after_init; 2432 2433 static int __init nossb_parse_cmdline(char *str) 2434 { 2435 nossb = true; 2436 ssb_mode = SPEC_STORE_BYPASS_NONE; 2437 return 0; 2438 } 2439 early_param("nospec_store_bypass_disable", nossb_parse_cmdline); 2440 2441 static int __init ssb_parse_cmdline(char *str) 2442 { 2443 if (!str) 2444 return -EINVAL; 2445 2446 if (nossb) 2447 return 0; 2448 2449 if (!strcmp(str, "auto")) 2450 ssb_mode = SPEC_STORE_BYPASS_AUTO; 2451 else if (!strcmp(str, "on")) 2452 ssb_mode = SPEC_STORE_BYPASS_DISABLE; 2453 else if (!strcmp(str, "off")) 2454 ssb_mode = SPEC_STORE_BYPASS_NONE; 2455 else if (!strcmp(str, "prctl")) 2456 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2457 else if (!strcmp(str, "seccomp")) 2458 ssb_mode = IS_ENABLED(CONFIG_SECCOMP) ? 2459 SPEC_STORE_BYPASS_SECCOMP : SPEC_STORE_BYPASS_PRCTL; 2460 else 2461 pr_err("Ignoring unknown spec_store_bypass_disable option (%s).\n", 2462 str); 2463 2464 return 0; 2465 } 2466 early_param("spec_store_bypass_disable", ssb_parse_cmdline); 2467 2468 static void __init ssb_select_mitigation(void) 2469 { 2470 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) { 2471 ssb_mode = SPEC_STORE_BYPASS_NONE; 2472 return; 2473 } 2474 2475 if (ssb_mode == SPEC_STORE_BYPASS_AUTO) { 2476 if (should_mitigate_vuln(X86_BUG_SPEC_STORE_BYPASS)) 2477 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2478 else 2479 ssb_mode = SPEC_STORE_BYPASS_NONE; 2480 } 2481 2482 if (!boot_cpu_has(X86_FEATURE_SSBD)) 2483 ssb_mode = SPEC_STORE_BYPASS_NONE; 2484 2485 pr_info("%s\n", ssb_strings[ssb_mode]); 2486 } 2487 2488 static void __init ssb_apply_mitigation(void) 2489 { 2490 /* 2491 * We have three CPU feature flags that are in play here: 2492 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 2493 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 2494 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 2495 */ 2496 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) { 2497 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 2498 /* 2499 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 2500 * use a completely different MSR and bit dependent on family. 2501 */ 2502 if (!cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) && 2503 !cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) { 2504 x86_amd_ssb_disable(); 2505 } else { 2506 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 2507 update_spec_ctrl(x86_spec_ctrl_base); 2508 } 2509 } 2510 } 2511 2512 #undef pr_fmt 2513 #define pr_fmt(fmt) "Speculation prctl: " fmt 2514 2515 static void task_update_spec_tif(struct task_struct *tsk) 2516 { 2517 /* Force the update of the real TIF bits */ 2518 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 2519 2520 /* 2521 * Immediately update the speculation control MSRs for the current 2522 * task, but for a non-current task delay setting the CPU 2523 * mitigation until it is scheduled next. 2524 * 2525 * This can only happen for SECCOMP mitigation. For PRCTL it's 2526 * always the current task. 2527 */ 2528 if (tsk == current) 2529 speculation_ctrl_update_current(); 2530 } 2531 2532 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 2533 { 2534 2535 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2536 return -EPERM; 2537 2538 switch (ctrl) { 2539 case PR_SPEC_ENABLE: 2540 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2541 return 0; 2542 case PR_SPEC_DISABLE: 2543 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2544 return 0; 2545 default: 2546 return -ERANGE; 2547 } 2548 } 2549 2550 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2551 { 2552 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2553 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2554 return -ENXIO; 2555 2556 switch (ctrl) { 2557 case PR_SPEC_ENABLE: 2558 /* If speculation is force disabled, enable is not allowed */ 2559 if (task_spec_ssb_force_disable(task)) 2560 return -EPERM; 2561 task_clear_spec_ssb_disable(task); 2562 task_clear_spec_ssb_noexec(task); 2563 task_update_spec_tif(task); 2564 break; 2565 case PR_SPEC_DISABLE: 2566 task_set_spec_ssb_disable(task); 2567 task_clear_spec_ssb_noexec(task); 2568 task_update_spec_tif(task); 2569 break; 2570 case PR_SPEC_FORCE_DISABLE: 2571 task_set_spec_ssb_disable(task); 2572 task_set_spec_ssb_force_disable(task); 2573 task_clear_spec_ssb_noexec(task); 2574 task_update_spec_tif(task); 2575 break; 2576 case PR_SPEC_DISABLE_NOEXEC: 2577 if (task_spec_ssb_force_disable(task)) 2578 return -EPERM; 2579 task_set_spec_ssb_disable(task); 2580 task_set_spec_ssb_noexec(task); 2581 task_update_spec_tif(task); 2582 break; 2583 default: 2584 return -ERANGE; 2585 } 2586 return 0; 2587 } 2588 2589 static bool is_spec_ib_user_controlled(void) 2590 { 2591 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2592 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2593 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2594 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2595 } 2596 2597 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2598 { 2599 switch (ctrl) { 2600 case PR_SPEC_ENABLE: 2601 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2602 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2603 return 0; 2604 2605 /* 2606 * With strict mode for both IBPB and STIBP, the instruction 2607 * code paths avoid checking this task flag and instead, 2608 * unconditionally run the instruction. However, STIBP and IBPB 2609 * are independent and either can be set to conditionally 2610 * enabled regardless of the mode of the other. 2611 * 2612 * If either is set to conditional, allow the task flag to be 2613 * updated, unless it was force-disabled by a previous prctl 2614 * call. Currently, this is possible on an AMD CPU which has the 2615 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2616 * kernel is booted with 'spectre_v2_user=seccomp', then 2617 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2618 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2619 */ 2620 if (!is_spec_ib_user_controlled() || 2621 task_spec_ib_force_disable(task)) 2622 return -EPERM; 2623 2624 task_clear_spec_ib_disable(task); 2625 task_update_spec_tif(task); 2626 break; 2627 case PR_SPEC_DISABLE: 2628 case PR_SPEC_FORCE_DISABLE: 2629 /* 2630 * Indirect branch speculation is always allowed when 2631 * mitigation is force disabled. 2632 */ 2633 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2634 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2635 return -EPERM; 2636 2637 if (!is_spec_ib_user_controlled()) 2638 return 0; 2639 2640 task_set_spec_ib_disable(task); 2641 if (ctrl == PR_SPEC_FORCE_DISABLE) 2642 task_set_spec_ib_force_disable(task); 2643 task_update_spec_tif(task); 2644 if (task == current) 2645 indirect_branch_prediction_barrier(); 2646 break; 2647 default: 2648 return -ERANGE; 2649 } 2650 return 0; 2651 } 2652 2653 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2654 unsigned long ctrl) 2655 { 2656 switch (which) { 2657 case PR_SPEC_STORE_BYPASS: 2658 return ssb_prctl_set(task, ctrl); 2659 case PR_SPEC_INDIRECT_BRANCH: 2660 return ib_prctl_set(task, ctrl); 2661 case PR_SPEC_L1D_FLUSH: 2662 return l1d_flush_prctl_set(task, ctrl); 2663 default: 2664 return -ENODEV; 2665 } 2666 } 2667 2668 #ifdef CONFIG_SECCOMP 2669 void arch_seccomp_spec_mitigate(struct task_struct *task) 2670 { 2671 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2672 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2673 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2674 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2675 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2676 } 2677 #endif 2678 2679 static int l1d_flush_prctl_get(struct task_struct *task) 2680 { 2681 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2682 return PR_SPEC_FORCE_DISABLE; 2683 2684 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2685 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2686 else 2687 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2688 } 2689 2690 static int ssb_prctl_get(struct task_struct *task) 2691 { 2692 switch (ssb_mode) { 2693 case SPEC_STORE_BYPASS_NONE: 2694 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2695 return PR_SPEC_ENABLE; 2696 return PR_SPEC_NOT_AFFECTED; 2697 case SPEC_STORE_BYPASS_DISABLE: 2698 return PR_SPEC_DISABLE; 2699 case SPEC_STORE_BYPASS_SECCOMP: 2700 case SPEC_STORE_BYPASS_PRCTL: 2701 case SPEC_STORE_BYPASS_AUTO: 2702 if (task_spec_ssb_force_disable(task)) 2703 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2704 if (task_spec_ssb_noexec(task)) 2705 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2706 if (task_spec_ssb_disable(task)) 2707 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2708 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2709 } 2710 BUG(); 2711 } 2712 2713 static int ib_prctl_get(struct task_struct *task) 2714 { 2715 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2716 return PR_SPEC_NOT_AFFECTED; 2717 2718 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2719 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2720 return PR_SPEC_ENABLE; 2721 else if (is_spec_ib_user_controlled()) { 2722 if (task_spec_ib_force_disable(task)) 2723 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2724 if (task_spec_ib_disable(task)) 2725 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2726 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2727 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2728 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2729 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2730 return PR_SPEC_DISABLE; 2731 else 2732 return PR_SPEC_NOT_AFFECTED; 2733 } 2734 2735 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2736 { 2737 switch (which) { 2738 case PR_SPEC_STORE_BYPASS: 2739 return ssb_prctl_get(task); 2740 case PR_SPEC_INDIRECT_BRANCH: 2741 return ib_prctl_get(task); 2742 case PR_SPEC_L1D_FLUSH: 2743 return l1d_flush_prctl_get(task); 2744 default: 2745 return -ENODEV; 2746 } 2747 } 2748 2749 void x86_spec_ctrl_setup_ap(void) 2750 { 2751 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2752 update_spec_ctrl(x86_spec_ctrl_base); 2753 2754 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2755 x86_amd_ssb_disable(); 2756 } 2757 2758 bool itlb_multihit_kvm_mitigation; 2759 EXPORT_SYMBOL_FOR_KVM(itlb_multihit_kvm_mitigation); 2760 2761 #undef pr_fmt 2762 #define pr_fmt(fmt) "L1TF: " fmt 2763 2764 /* Default mitigation for L1TF-affected CPUs */ 2765 enum l1tf_mitigations l1tf_mitigation __ro_after_init = 2766 IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_AUTO : L1TF_MITIGATION_OFF; 2767 EXPORT_SYMBOL_FOR_KVM(l1tf_mitigation); 2768 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2769 EXPORT_SYMBOL_FOR_KVM(l1tf_vmx_mitigation); 2770 2771 /* 2772 * These CPUs all support 44bits physical address space internally in the 2773 * cache but CPUID can report a smaller number of physical address bits. 2774 * 2775 * The L1TF mitigation uses the top most address bit for the inversion of 2776 * non present PTEs. When the installed memory reaches into the top most 2777 * address bit due to memory holes, which has been observed on machines 2778 * which report 36bits physical address bits and have 32G RAM installed, 2779 * then the mitigation range check in l1tf_select_mitigation() triggers. 2780 * This is a false positive because the mitigation is still possible due to 2781 * the fact that the cache uses 44bit internally. Use the cache bits 2782 * instead of the reported physical bits and adjust them on the affected 2783 * machines to 44bit if the reported bits are less than 44. 2784 */ 2785 static void override_cache_bits(struct cpuinfo_x86 *c) 2786 { 2787 if (c->x86 != 6) 2788 return; 2789 2790 switch (c->x86_vfm) { 2791 case INTEL_NEHALEM: 2792 case INTEL_WESTMERE: 2793 case INTEL_SANDYBRIDGE: 2794 case INTEL_IVYBRIDGE: 2795 case INTEL_HASWELL: 2796 case INTEL_HASWELL_L: 2797 case INTEL_HASWELL_G: 2798 case INTEL_BROADWELL: 2799 case INTEL_BROADWELL_G: 2800 case INTEL_SKYLAKE_L: 2801 case INTEL_SKYLAKE: 2802 case INTEL_KABYLAKE_L: 2803 case INTEL_KABYLAKE: 2804 if (c->x86_cache_bits < 44) 2805 c->x86_cache_bits = 44; 2806 break; 2807 } 2808 } 2809 2810 static void __init l1tf_select_mitigation(void) 2811 { 2812 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 2813 l1tf_mitigation = L1TF_MITIGATION_OFF; 2814 return; 2815 } 2816 2817 if (l1tf_mitigation != L1TF_MITIGATION_AUTO) 2818 return; 2819 2820 if (!should_mitigate_vuln(X86_BUG_L1TF)) { 2821 l1tf_mitigation = L1TF_MITIGATION_OFF; 2822 return; 2823 } 2824 2825 if (smt_mitigations == SMT_MITIGATIONS_ON) 2826 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2827 else 2828 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2829 } 2830 2831 static void __init l1tf_apply_mitigation(void) 2832 { 2833 u64 half_pa; 2834 2835 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2836 return; 2837 2838 override_cache_bits(&boot_cpu_data); 2839 2840 switch (l1tf_mitigation) { 2841 case L1TF_MITIGATION_OFF: 2842 case L1TF_MITIGATION_FLUSH_NOWARN: 2843 case L1TF_MITIGATION_FLUSH: 2844 case L1TF_MITIGATION_AUTO: 2845 break; 2846 case L1TF_MITIGATION_FLUSH_NOSMT: 2847 case L1TF_MITIGATION_FULL: 2848 cpu_smt_disable(false); 2849 break; 2850 case L1TF_MITIGATION_FULL_FORCE: 2851 cpu_smt_disable(true); 2852 break; 2853 } 2854 2855 #if CONFIG_PGTABLE_LEVELS == 2 2856 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2857 return; 2858 #endif 2859 2860 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2861 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2862 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2863 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2864 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2865 half_pa); 2866 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2867 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2868 return; 2869 } 2870 2871 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2872 } 2873 2874 static int __init l1tf_cmdline(char *str) 2875 { 2876 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2877 return 0; 2878 2879 if (!str) 2880 return -EINVAL; 2881 2882 if (!strcmp(str, "off")) 2883 l1tf_mitigation = L1TF_MITIGATION_OFF; 2884 else if (!strcmp(str, "flush,nowarn")) 2885 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2886 else if (!strcmp(str, "flush")) 2887 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2888 else if (!strcmp(str, "flush,nosmt")) 2889 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2890 else if (!strcmp(str, "full")) 2891 l1tf_mitigation = L1TF_MITIGATION_FULL; 2892 else if (!strcmp(str, "full,force")) 2893 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2894 2895 return 0; 2896 } 2897 early_param("l1tf", l1tf_cmdline); 2898 2899 #undef pr_fmt 2900 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 2901 2902 static const char * const srso_strings[] = { 2903 [SRSO_MITIGATION_NONE] = "Vulnerable", 2904 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 2905 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 2906 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 2907 [SRSO_MITIGATION_NOSMT] = "Mitigation: SMT disabled", 2908 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 2909 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 2910 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only", 2911 [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation" 2912 }; 2913 2914 static int __init srso_parse_cmdline(char *str) 2915 { 2916 if (!str) 2917 return -EINVAL; 2918 2919 if (!strcmp(str, "off")) 2920 srso_mitigation = SRSO_MITIGATION_NONE; 2921 else if (!strcmp(str, "microcode")) 2922 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2923 else if (!strcmp(str, "safe-ret")) 2924 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2925 else if (!strcmp(str, "ibpb")) 2926 srso_mitigation = SRSO_MITIGATION_IBPB; 2927 else if (!strcmp(str, "ibpb-vmexit")) 2928 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2929 else 2930 pr_err("Ignoring unknown SRSO option (%s).", str); 2931 2932 return 0; 2933 } 2934 early_param("spec_rstack_overflow", srso_parse_cmdline); 2935 2936 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 2937 2938 static void __init srso_select_mitigation(void) 2939 { 2940 if (!boot_cpu_has_bug(X86_BUG_SRSO)) { 2941 srso_mitigation = SRSO_MITIGATION_NONE; 2942 return; 2943 } 2944 2945 if (srso_mitigation == SRSO_MITIGATION_AUTO) { 2946 /* 2947 * Use safe-RET if user->kernel or guest->host protection is 2948 * required. Otherwise the 'microcode' mitigation is sufficient 2949 * to protect the user->user and guest->guest vectors. 2950 */ 2951 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 2952 (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) && 2953 !boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))) { 2954 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2955 } else if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 2956 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) { 2957 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2958 } else { 2959 srso_mitigation = SRSO_MITIGATION_NONE; 2960 return; 2961 } 2962 } 2963 2964 /* Zen1/2 with SMT off aren't vulnerable to SRSO. */ 2965 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 2966 srso_mitigation = SRSO_MITIGATION_NOSMT; 2967 return; 2968 } 2969 2970 if (!boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) { 2971 pr_warn("IBPB-extending microcode not applied!\n"); 2972 pr_warn(SRSO_NOTICE); 2973 2974 /* 2975 * Safe-RET provides partial mitigation without microcode, but 2976 * other mitigations require microcode to provide any 2977 * mitigations. 2978 */ 2979 if (srso_mitigation == SRSO_MITIGATION_SAFE_RET) 2980 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 2981 else 2982 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 2983 } 2984 2985 switch (srso_mitigation) { 2986 case SRSO_MITIGATION_SAFE_RET: 2987 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 2988 if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) { 2989 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2990 goto ibpb_on_vmexit; 2991 } 2992 2993 if (!IS_ENABLED(CONFIG_MITIGATION_SRSO)) { 2994 pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n"); 2995 srso_mitigation = SRSO_MITIGATION_NONE; 2996 } 2997 break; 2998 ibpb_on_vmexit: 2999 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3000 if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) { 3001 pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n"); 3002 srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE; 3003 break; 3004 } 3005 fallthrough; 3006 case SRSO_MITIGATION_IBPB: 3007 if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 3008 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 3009 srso_mitigation = SRSO_MITIGATION_NONE; 3010 } 3011 break; 3012 default: 3013 break; 3014 } 3015 } 3016 3017 static void __init srso_update_mitigation(void) 3018 { 3019 if (!boot_cpu_has_bug(X86_BUG_SRSO)) 3020 return; 3021 3022 /* If retbleed is using IBPB, that works for SRSO as well */ 3023 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB && 3024 boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) 3025 srso_mitigation = SRSO_MITIGATION_IBPB; 3026 3027 pr_info("%s\n", srso_strings[srso_mitigation]); 3028 } 3029 3030 static void __init srso_apply_mitigation(void) 3031 { 3032 /* 3033 * Clear the feature flag if this mitigation is not selected as that 3034 * feature flag controls the BpSpecReduce MSR bit toggling in KVM. 3035 */ 3036 if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE) 3037 setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE); 3038 3039 if (srso_mitigation == SRSO_MITIGATION_NONE) { 3040 if (boot_cpu_has(X86_FEATURE_SBPB)) 3041 x86_pred_cmd = PRED_CMD_SBPB; 3042 return; 3043 } 3044 3045 switch (srso_mitigation) { 3046 case SRSO_MITIGATION_SAFE_RET: 3047 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 3048 /* 3049 * Enable the return thunk for generated code 3050 * like ftrace, static_call, etc. 3051 */ 3052 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 3053 setup_force_cpu_cap(X86_FEATURE_UNRET); 3054 3055 if (boot_cpu_data.x86 == 0x19) { 3056 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 3057 set_return_thunk(srso_alias_return_thunk); 3058 } else { 3059 setup_force_cpu_cap(X86_FEATURE_SRSO); 3060 set_return_thunk(srso_return_thunk); 3061 } 3062 break; 3063 case SRSO_MITIGATION_IBPB: 3064 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 3065 /* 3066 * IBPB on entry already obviates the need for 3067 * software-based untraining so clear those in case some 3068 * other mitigation like Retbleed has selected them. 3069 */ 3070 setup_clear_cpu_cap(X86_FEATURE_UNRET); 3071 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 3072 fallthrough; 3073 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3074 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 3075 /* 3076 * There is no need for RSB filling: entry_ibpb() ensures 3077 * all predictions, including the RSB, are invalidated, 3078 * regardless of IBPB implementation. 3079 */ 3080 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 3081 break; 3082 default: 3083 break; 3084 } 3085 } 3086 3087 #undef pr_fmt 3088 #define pr_fmt(fmt) "VMSCAPE: " fmt 3089 3090 enum vmscape_mitigations { 3091 VMSCAPE_MITIGATION_NONE, 3092 VMSCAPE_MITIGATION_AUTO, 3093 VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER, 3094 VMSCAPE_MITIGATION_IBPB_ON_VMEXIT, 3095 }; 3096 3097 static const char * const vmscape_strings[] = { 3098 [VMSCAPE_MITIGATION_NONE] = "Vulnerable", 3099 /* [VMSCAPE_MITIGATION_AUTO] */ 3100 [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER] = "Mitigation: IBPB before exit to userspace", 3101 [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT", 3102 }; 3103 3104 static enum vmscape_mitigations vmscape_mitigation __ro_after_init = 3105 IS_ENABLED(CONFIG_MITIGATION_VMSCAPE) ? VMSCAPE_MITIGATION_AUTO : VMSCAPE_MITIGATION_NONE; 3106 3107 static int __init vmscape_parse_cmdline(char *str) 3108 { 3109 if (!str) 3110 return -EINVAL; 3111 3112 if (!strcmp(str, "off")) { 3113 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3114 } else if (!strcmp(str, "ibpb")) { 3115 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; 3116 } else if (!strcmp(str, "force")) { 3117 setup_force_cpu_bug(X86_BUG_VMSCAPE); 3118 vmscape_mitigation = VMSCAPE_MITIGATION_AUTO; 3119 } else { 3120 pr_err("Ignoring unknown vmscape=%s option.\n", str); 3121 } 3122 3123 return 0; 3124 } 3125 early_param("vmscape", vmscape_parse_cmdline); 3126 3127 static void __init vmscape_select_mitigation(void) 3128 { 3129 if (!boot_cpu_has_bug(X86_BUG_VMSCAPE) || 3130 !boot_cpu_has(X86_FEATURE_IBPB)) { 3131 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3132 return; 3133 } 3134 3135 if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO) { 3136 if (should_mitigate_vuln(X86_BUG_VMSCAPE)) 3137 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; 3138 else 3139 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3140 } 3141 } 3142 3143 static void __init vmscape_update_mitigation(void) 3144 { 3145 if (!boot_cpu_has_bug(X86_BUG_VMSCAPE)) 3146 return; 3147 3148 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB || 3149 srso_mitigation == SRSO_MITIGATION_IBPB_ON_VMEXIT) 3150 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_ON_VMEXIT; 3151 3152 pr_info("%s\n", vmscape_strings[vmscape_mitigation]); 3153 } 3154 3155 static void __init vmscape_apply_mitigation(void) 3156 { 3157 if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER) 3158 setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER); 3159 } 3160 3161 #undef pr_fmt 3162 #define pr_fmt(fmt) fmt 3163 3164 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 3165 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 3166 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 3167 #define VMSCAPE_MSG_SMT "VMSCAPE: SMT on, STIBP is required for full protection. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/vmscape.html for more details.\n" 3168 3169 void cpu_bugs_smt_update(void) 3170 { 3171 mutex_lock(&spec_ctrl_mutex); 3172 3173 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3174 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3175 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 3176 3177 switch (spectre_v2_user_stibp) { 3178 case SPECTRE_V2_USER_NONE: 3179 break; 3180 case SPECTRE_V2_USER_STRICT: 3181 case SPECTRE_V2_USER_STRICT_PREFERRED: 3182 update_stibp_strict(); 3183 break; 3184 case SPECTRE_V2_USER_PRCTL: 3185 case SPECTRE_V2_USER_SECCOMP: 3186 update_indir_branch_cond(); 3187 break; 3188 } 3189 3190 switch (mds_mitigation) { 3191 case MDS_MITIGATION_FULL: 3192 case MDS_MITIGATION_AUTO: 3193 case MDS_MITIGATION_VMWERV: 3194 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 3195 pr_warn_once(MDS_MSG_SMT); 3196 update_mds_branch_idle(); 3197 break; 3198 case MDS_MITIGATION_OFF: 3199 break; 3200 } 3201 3202 switch (taa_mitigation) { 3203 case TAA_MITIGATION_VERW: 3204 case TAA_MITIGATION_AUTO: 3205 case TAA_MITIGATION_UCODE_NEEDED: 3206 if (sched_smt_active()) 3207 pr_warn_once(TAA_MSG_SMT); 3208 break; 3209 case TAA_MITIGATION_TSX_DISABLED: 3210 case TAA_MITIGATION_OFF: 3211 break; 3212 } 3213 3214 switch (mmio_mitigation) { 3215 case MMIO_MITIGATION_VERW: 3216 case MMIO_MITIGATION_AUTO: 3217 case MMIO_MITIGATION_UCODE_NEEDED: 3218 if (sched_smt_active()) 3219 pr_warn_once(MMIO_MSG_SMT); 3220 break; 3221 case MMIO_MITIGATION_OFF: 3222 break; 3223 } 3224 3225 switch (tsa_mitigation) { 3226 case TSA_MITIGATION_USER_KERNEL: 3227 case TSA_MITIGATION_VM: 3228 case TSA_MITIGATION_AUTO: 3229 case TSA_MITIGATION_FULL: 3230 /* 3231 * TSA-SQ can potentially lead to info leakage between 3232 * SMT threads. 3233 */ 3234 if (sched_smt_active()) 3235 static_branch_enable(&cpu_buf_idle_clear); 3236 else 3237 static_branch_disable(&cpu_buf_idle_clear); 3238 break; 3239 case TSA_MITIGATION_NONE: 3240 case TSA_MITIGATION_UCODE_NEEDED: 3241 break; 3242 } 3243 3244 switch (vmscape_mitigation) { 3245 case VMSCAPE_MITIGATION_NONE: 3246 case VMSCAPE_MITIGATION_AUTO: 3247 break; 3248 case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT: 3249 case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER: 3250 /* 3251 * Hypervisors can be attacked across-threads, warn for SMT when 3252 * STIBP is not already enabled system-wide. 3253 * 3254 * Intel eIBRS (!AUTOIBRS) implies STIBP on. 3255 */ 3256 if (!sched_smt_active() || 3257 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3258 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED || 3259 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3260 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) 3261 break; 3262 pr_warn_once(VMSCAPE_MSG_SMT); 3263 break; 3264 } 3265 3266 mutex_unlock(&spec_ctrl_mutex); 3267 } 3268 3269 void __init cpu_select_mitigations(void) 3270 { 3271 /* 3272 * Read the SPEC_CTRL MSR to account for reserved bits which may 3273 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 3274 * init code as it is not enumerated and depends on the family. 3275 */ 3276 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 3277 rdmsrq(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 3278 3279 /* 3280 * Previously running kernel (kexec), may have some controls 3281 * turned ON. Clear them and let the mitigations setup below 3282 * rediscover them based on configuration. 3283 */ 3284 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 3285 } 3286 3287 x86_arch_cap_msr = x86_read_arch_cap_msr(); 3288 3289 cpu_print_attack_vectors(); 3290 3291 /* Select the proper CPU mitigations before patching alternatives: */ 3292 spectre_v1_select_mitigation(); 3293 spectre_v2_select_mitigation(); 3294 retbleed_select_mitigation(); 3295 spectre_v2_user_select_mitigation(); 3296 ssb_select_mitigation(); 3297 l1tf_select_mitigation(); 3298 mds_select_mitigation(); 3299 taa_select_mitigation(); 3300 mmio_select_mitigation(); 3301 rfds_select_mitigation(); 3302 srbds_select_mitigation(); 3303 l1d_flush_select_mitigation(); 3304 srso_select_mitigation(); 3305 gds_select_mitigation(); 3306 its_select_mitigation(); 3307 bhi_select_mitigation(); 3308 tsa_select_mitigation(); 3309 vmscape_select_mitigation(); 3310 3311 /* 3312 * After mitigations are selected, some may need to update their 3313 * choices. 3314 */ 3315 spectre_v2_update_mitigation(); 3316 /* 3317 * retbleed_update_mitigation() relies on the state set by 3318 * spectre_v2_update_mitigation(); specifically it wants to know about 3319 * spectre_v2=ibrs. 3320 */ 3321 retbleed_update_mitigation(); 3322 /* 3323 * its_update_mitigation() depends on spectre_v2_update_mitigation() 3324 * and retbleed_update_mitigation(). 3325 */ 3326 its_update_mitigation(); 3327 3328 /* 3329 * spectre_v2_user_update_mitigation() depends on 3330 * retbleed_update_mitigation(), specifically the STIBP 3331 * selection is forced for UNRET or IBPB. 3332 */ 3333 spectre_v2_user_update_mitigation(); 3334 mds_update_mitigation(); 3335 taa_update_mitigation(); 3336 mmio_update_mitigation(); 3337 rfds_update_mitigation(); 3338 bhi_update_mitigation(); 3339 /* srso_update_mitigation() depends on retbleed_update_mitigation(). */ 3340 srso_update_mitigation(); 3341 vmscape_update_mitigation(); 3342 3343 spectre_v1_apply_mitigation(); 3344 spectre_v2_apply_mitigation(); 3345 retbleed_apply_mitigation(); 3346 spectre_v2_user_apply_mitigation(); 3347 ssb_apply_mitigation(); 3348 l1tf_apply_mitigation(); 3349 mds_apply_mitigation(); 3350 taa_apply_mitigation(); 3351 mmio_apply_mitigation(); 3352 rfds_apply_mitigation(); 3353 srbds_apply_mitigation(); 3354 srso_apply_mitigation(); 3355 gds_apply_mitigation(); 3356 its_apply_mitigation(); 3357 bhi_apply_mitigation(); 3358 tsa_apply_mitigation(); 3359 vmscape_apply_mitigation(); 3360 } 3361 3362 #ifdef CONFIG_SYSFS 3363 3364 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 3365 3366 #if IS_ENABLED(CONFIG_KVM_INTEL) 3367 static const char * const l1tf_vmx_states[] = { 3368 [VMENTER_L1D_FLUSH_AUTO] = "auto", 3369 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 3370 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 3371 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 3372 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 3373 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 3374 }; 3375 3376 static ssize_t l1tf_show_state(char *buf) 3377 { 3378 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 3379 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3380 3381 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 3382 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 3383 sched_smt_active())) { 3384 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 3385 l1tf_vmx_states[l1tf_vmx_mitigation]); 3386 } 3387 3388 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 3389 l1tf_vmx_states[l1tf_vmx_mitigation], 3390 sched_smt_active() ? "vulnerable" : "disabled"); 3391 } 3392 3393 static ssize_t itlb_multihit_show_state(char *buf) 3394 { 3395 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 3396 !boot_cpu_has(X86_FEATURE_VMX)) 3397 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 3398 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 3399 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 3400 else if (itlb_multihit_kvm_mitigation) 3401 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 3402 else 3403 return sysfs_emit(buf, "KVM: Vulnerable\n"); 3404 } 3405 #else 3406 static ssize_t l1tf_show_state(char *buf) 3407 { 3408 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3409 } 3410 3411 static ssize_t itlb_multihit_show_state(char *buf) 3412 { 3413 return sysfs_emit(buf, "Processor vulnerable\n"); 3414 } 3415 #endif 3416 3417 static ssize_t mds_show_state(char *buf) 3418 { 3419 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3420 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3421 mds_strings[mds_mitigation]); 3422 } 3423 3424 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 3425 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3426 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 3427 sched_smt_active() ? "mitigated" : "disabled")); 3428 } 3429 3430 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3431 sched_smt_active() ? "vulnerable" : "disabled"); 3432 } 3433 3434 static ssize_t tsx_async_abort_show_state(char *buf) 3435 { 3436 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 3437 (taa_mitigation == TAA_MITIGATION_OFF)) 3438 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 3439 3440 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3441 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3442 taa_strings[taa_mitigation]); 3443 } 3444 3445 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 3446 sched_smt_active() ? "vulnerable" : "disabled"); 3447 } 3448 3449 static ssize_t mmio_stale_data_show_state(char *buf) 3450 { 3451 if (mmio_mitigation == MMIO_MITIGATION_OFF) 3452 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 3453 3454 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3455 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3456 mmio_strings[mmio_mitigation]); 3457 } 3458 3459 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 3460 sched_smt_active() ? "vulnerable" : "disabled"); 3461 } 3462 3463 static ssize_t rfds_show_state(char *buf) 3464 { 3465 return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); 3466 } 3467 3468 static ssize_t old_microcode_show_state(char *buf) 3469 { 3470 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 3471 return sysfs_emit(buf, "Unknown: running under hypervisor"); 3472 3473 return sysfs_emit(buf, "Vulnerable\n"); 3474 } 3475 3476 static ssize_t its_show_state(char *buf) 3477 { 3478 return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]); 3479 } 3480 3481 static char *stibp_state(void) 3482 { 3483 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3484 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 3485 return ""; 3486 3487 switch (spectre_v2_user_stibp) { 3488 case SPECTRE_V2_USER_NONE: 3489 return "; STIBP: disabled"; 3490 case SPECTRE_V2_USER_STRICT: 3491 return "; STIBP: forced"; 3492 case SPECTRE_V2_USER_STRICT_PREFERRED: 3493 return "; STIBP: always-on"; 3494 case SPECTRE_V2_USER_PRCTL: 3495 case SPECTRE_V2_USER_SECCOMP: 3496 if (static_key_enabled(&switch_to_cond_stibp)) 3497 return "; STIBP: conditional"; 3498 } 3499 return ""; 3500 } 3501 3502 static char *ibpb_state(void) 3503 { 3504 if (boot_cpu_has(X86_FEATURE_IBPB)) { 3505 if (static_key_enabled(&switch_mm_always_ibpb)) 3506 return "; IBPB: always-on"; 3507 if (static_key_enabled(&switch_mm_cond_ibpb)) 3508 return "; IBPB: conditional"; 3509 return "; IBPB: disabled"; 3510 } 3511 return ""; 3512 } 3513 3514 static char *pbrsb_eibrs_state(void) 3515 { 3516 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 3517 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 3518 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 3519 return "; PBRSB-eIBRS: SW sequence"; 3520 else 3521 return "; PBRSB-eIBRS: Vulnerable"; 3522 } else { 3523 return "; PBRSB-eIBRS: Not affected"; 3524 } 3525 } 3526 3527 static const char *spectre_bhi_state(void) 3528 { 3529 if (!boot_cpu_has_bug(X86_BUG_BHI)) 3530 return "; BHI: Not affected"; 3531 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW)) 3532 return "; BHI: BHI_DIS_S"; 3533 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) 3534 return "; BHI: SW loop, KVM: SW loop"; 3535 else if (retpoline_seq_enabled() && rrsba_disabled) 3536 return "; BHI: Retpoline"; 3537 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) 3538 return "; BHI: Vulnerable, KVM: SW loop"; 3539 3540 return "; BHI: Vulnerable"; 3541 } 3542 3543 static ssize_t spectre_v2_show_state(char *buf) 3544 { 3545 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 3546 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 3547 3548 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3549 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3550 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 3551 3552 return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n", 3553 spectre_v2_strings[spectre_v2_enabled], 3554 ibpb_state(), 3555 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "", 3556 stibp_state(), 3557 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "", 3558 pbrsb_eibrs_state(), 3559 spectre_bhi_state(), 3560 /* this should always be at the end */ 3561 spectre_v2_module_string()); 3562 } 3563 3564 static ssize_t srbds_show_state(char *buf) 3565 { 3566 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 3567 } 3568 3569 static ssize_t retbleed_show_state(char *buf) 3570 { 3571 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 3572 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 3573 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 3574 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 3575 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 3576 3577 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 3578 !sched_smt_active() ? "disabled" : 3579 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3580 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 3581 "enabled with STIBP protection" : "vulnerable"); 3582 } 3583 3584 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 3585 } 3586 3587 static ssize_t srso_show_state(char *buf) 3588 { 3589 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 3590 } 3591 3592 static ssize_t gds_show_state(char *buf) 3593 { 3594 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 3595 } 3596 3597 static ssize_t tsa_show_state(char *buf) 3598 { 3599 return sysfs_emit(buf, "%s\n", tsa_strings[tsa_mitigation]); 3600 } 3601 3602 static ssize_t vmscape_show_state(char *buf) 3603 { 3604 return sysfs_emit(buf, "%s\n", vmscape_strings[vmscape_mitigation]); 3605 } 3606 3607 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 3608 char *buf, unsigned int bug) 3609 { 3610 if (!boot_cpu_has_bug(bug)) 3611 return sysfs_emit(buf, "Not affected\n"); 3612 3613 switch (bug) { 3614 case X86_BUG_CPU_MELTDOWN: 3615 if (boot_cpu_has(X86_FEATURE_PTI)) 3616 return sysfs_emit(buf, "Mitigation: PTI\n"); 3617 3618 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 3619 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 3620 3621 break; 3622 3623 case X86_BUG_SPECTRE_V1: 3624 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 3625 3626 case X86_BUG_SPECTRE_V2: 3627 return spectre_v2_show_state(buf); 3628 3629 case X86_BUG_SPEC_STORE_BYPASS: 3630 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 3631 3632 case X86_BUG_L1TF: 3633 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 3634 return l1tf_show_state(buf); 3635 break; 3636 3637 case X86_BUG_MDS: 3638 return mds_show_state(buf); 3639 3640 case X86_BUG_TAA: 3641 return tsx_async_abort_show_state(buf); 3642 3643 case X86_BUG_ITLB_MULTIHIT: 3644 return itlb_multihit_show_state(buf); 3645 3646 case X86_BUG_SRBDS: 3647 return srbds_show_state(buf); 3648 3649 case X86_BUG_MMIO_STALE_DATA: 3650 return mmio_stale_data_show_state(buf); 3651 3652 case X86_BUG_RETBLEED: 3653 return retbleed_show_state(buf); 3654 3655 case X86_BUG_SRSO: 3656 return srso_show_state(buf); 3657 3658 case X86_BUG_GDS: 3659 return gds_show_state(buf); 3660 3661 case X86_BUG_RFDS: 3662 return rfds_show_state(buf); 3663 3664 case X86_BUG_OLD_MICROCODE: 3665 return old_microcode_show_state(buf); 3666 3667 case X86_BUG_ITS: 3668 return its_show_state(buf); 3669 3670 case X86_BUG_TSA: 3671 return tsa_show_state(buf); 3672 3673 case X86_BUG_VMSCAPE: 3674 return vmscape_show_state(buf); 3675 3676 default: 3677 break; 3678 } 3679 3680 return sysfs_emit(buf, "Vulnerable\n"); 3681 } 3682 3683 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 3684 { 3685 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 3686 } 3687 3688 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 3689 { 3690 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 3691 } 3692 3693 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 3694 { 3695 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 3696 } 3697 3698 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 3699 { 3700 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 3701 } 3702 3703 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 3704 { 3705 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 3706 } 3707 3708 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 3709 { 3710 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 3711 } 3712 3713 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 3714 { 3715 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 3716 } 3717 3718 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 3719 { 3720 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 3721 } 3722 3723 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 3724 { 3725 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 3726 } 3727 3728 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 3729 { 3730 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 3731 } 3732 3733 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 3734 { 3735 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 3736 } 3737 3738 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 3739 { 3740 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 3741 } 3742 3743 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 3744 { 3745 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 3746 } 3747 3748 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf) 3749 { 3750 return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); 3751 } 3752 3753 ssize_t cpu_show_old_microcode(struct device *dev, struct device_attribute *attr, char *buf) 3754 { 3755 return cpu_show_common(dev, attr, buf, X86_BUG_OLD_MICROCODE); 3756 } 3757 3758 ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf) 3759 { 3760 return cpu_show_common(dev, attr, buf, X86_BUG_ITS); 3761 } 3762 3763 ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf) 3764 { 3765 return cpu_show_common(dev, attr, buf, X86_BUG_TSA); 3766 } 3767 3768 ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf) 3769 { 3770 return cpu_show_common(dev, attr, buf, X86_BUG_VMSCAPE); 3771 } 3772 #endif 3773 3774 void __warn_thunk(void) 3775 { 3776 WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); 3777 } 3778 3779 #ifdef CONFIG_MITIGATION_SRSO 3780 /* 3781 * Called during exception/interrupt entry if interrupted during the 3782 * safe-RET sequence. The safe-RET sequence consists of 3 instructions: 3783 * 3784 * CALL 3785 * LEA 8(%RSP), %RSP 3786 * RET 3787 * 3788 * An interrupt after the CALL or after the LEA could potentially lead 3789 * to branch predictor poisoning and results in the sequence not being 3790 * able to be safely resumed. 3791 * 3792 * Therefore, modify the regs state as if the remaining part of the 3793 * safe-RET sequence executed so the interrupt returns back to the 3794 * desired return target, instead of the to the safe-RET sequence. 3795 */ 3796 void noinstr handle_interrupted_saferet(struct pt_regs *regs) 3797 { 3798 unsigned long rip = regs->ip; 3799 3800 if (rip == (unsigned long) srso_safe_ret || 3801 rip == (unsigned long) srso_alias_safe_ret) { 3802 /* Modify stack pointer as if LEA executed: */ 3803 regs->sp += 8; 3804 } 3805 3806 /* 3807 * Adjust registers as if RET executed: 3808 * 3809 * 1. Read the return address off the stack and into rIP: 3810 */ 3811 regs->ip = *(unsigned long *)(regs->sp); 3812 3813 /* 2. Pop rIP off the stack: */ 3814 regs->sp += 8; 3815 } 3816 #endif /* CONFIG_MITIGATION_SRSO */ 3817