1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 #include <linux/filter.h> 20 #include <linux/kvm_types.h> 21 22 #include <asm/spec-ctrl.h> 23 #include <asm/cmdline.h> 24 #include <asm/bugs.h> 25 #include <asm/processor.h> 26 #include <asm/processor-flags.h> 27 #include <asm/fpu/api.h> 28 #include <asm/msr.h> 29 #include <asm/vmx.h> 30 #include <asm/cpu_device_id.h> 31 #include <asm/e820/api.h> 32 #include <asm/hypervisor.h> 33 #include <asm/tlbflush.h> 34 #include <asm/cpu.h> 35 36 #include "cpu.h" 37 38 /* 39 * Speculation Vulnerability Handling 40 * 41 * Each vulnerability is handled with the following functions: 42 * <vuln>_select_mitigation() -- Selects a mitigation to use. This should 43 * take into account all relevant command line 44 * options. 45 * <vuln>_update_mitigation() -- This is called after all vulnerabilities have 46 * selected a mitigation, in case the selection 47 * may want to change based on other choices 48 * made. This function is optional. 49 * <vuln>_apply_mitigation() -- Enable the selected mitigation. 50 * 51 * The compile-time mitigation in all cases should be AUTO. An explicit 52 * command-line option can override AUTO. If no such option is 53 * provided, <vuln>_select_mitigation() will override AUTO to the best 54 * mitigation option. 55 */ 56 57 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 58 u64 x86_spec_ctrl_base; 59 60 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 61 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 62 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current); 63 64 /* 65 * Set when the CPU has run a potentially malicious guest. An IBPB will 66 * be needed to before running userspace. That IBPB will flush the branch 67 * predictor content. 68 */ 69 DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user); 70 EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user); 71 72 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 73 74 static u64 __ro_after_init x86_arch_cap_msr; 75 76 static DEFINE_MUTEX(spec_ctrl_mutex); 77 78 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 79 80 static void __init set_return_thunk(void *thunk) 81 { 82 x86_return_thunk = thunk; 83 84 pr_info("active return thunk: %ps\n", thunk); 85 } 86 87 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 88 static void update_spec_ctrl(u64 val) 89 { 90 this_cpu_write(x86_spec_ctrl_current, val); 91 wrmsrq(MSR_IA32_SPEC_CTRL, val); 92 } 93 94 /* 95 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 96 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 97 */ 98 void update_spec_ctrl_cond(u64 val) 99 { 100 if (this_cpu_read(x86_spec_ctrl_current) == val) 101 return; 102 103 this_cpu_write(x86_spec_ctrl_current, val); 104 105 /* 106 * When KERNEL_IBRS this MSR is written on return-to-user, unless 107 * forced the update can be delayed until that time. 108 */ 109 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 110 wrmsrq(MSR_IA32_SPEC_CTRL, val); 111 } 112 113 noinstr u64 spec_ctrl_current(void) 114 { 115 return this_cpu_read(x86_spec_ctrl_current); 116 } 117 EXPORT_SYMBOL_GPL(spec_ctrl_current); 118 119 /* 120 * AMD specific MSR info for Speculative Store Bypass control. 121 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 122 */ 123 u64 __ro_after_init x86_amd_ls_cfg_base; 124 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 125 126 /* Control conditional STIBP in switch_to() */ 127 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 128 /* Control conditional IBPB in switch_mm() */ 129 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 130 /* Control unconditional IBPB in switch_mm() */ 131 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 132 133 /* Control IBPB on vCPU load */ 134 DEFINE_STATIC_KEY_FALSE(switch_vcpu_ibpb); 135 EXPORT_SYMBOL_FOR_KVM(switch_vcpu_ibpb); 136 137 /* Control CPU buffer clear before idling (halt, mwait) */ 138 DEFINE_STATIC_KEY_FALSE(cpu_buf_idle_clear); 139 EXPORT_SYMBOL_GPL(cpu_buf_idle_clear); 140 141 /* 142 * Controls whether l1d flush based mitigations are enabled, 143 * based on hw features and admin setting via boot parameter 144 * defaults to false 145 */ 146 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 147 148 #undef pr_fmt 149 #define pr_fmt(fmt) "mitigations: " fmt 150 151 static void __init cpu_print_attack_vectors(void) 152 { 153 pr_info("Enabled attack vectors: "); 154 155 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL)) 156 pr_cont("user_kernel, "); 157 158 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) 159 pr_cont("user_user, "); 160 161 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST)) 162 pr_cont("guest_host, "); 163 164 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) 165 pr_cont("guest_guest, "); 166 167 pr_cont("SMT mitigations: "); 168 169 switch (smt_mitigations) { 170 case SMT_MITIGATIONS_OFF: 171 pr_cont("off\n"); 172 break; 173 case SMT_MITIGATIONS_AUTO: 174 pr_cont("auto\n"); 175 break; 176 case SMT_MITIGATIONS_ON: 177 pr_cont("on\n"); 178 } 179 } 180 181 /* 182 * NOTE: This function is *only* called for SVM, since Intel uses 183 * MSR_IA32_SPEC_CTRL for SSBD. 184 */ 185 void 186 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 187 { 188 u64 guestval, hostval; 189 struct thread_info *ti = current_thread_info(); 190 191 /* 192 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 193 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 194 */ 195 if (!cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD) && 196 !cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) 197 return; 198 199 /* 200 * If the host has SSBD mitigation enabled, force it in the host's 201 * virtual MSR value. If its not permanently enabled, evaluate 202 * current's TIF_SSBD thread flag. 203 */ 204 if (cpu_feature_enabled(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 205 hostval = SPEC_CTRL_SSBD; 206 else 207 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 208 209 /* Sanitize the guest value */ 210 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 211 212 if (hostval != guestval) { 213 unsigned long tif; 214 215 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 216 ssbd_spec_ctrl_to_tif(hostval); 217 218 speculation_ctrl_update(tif); 219 } 220 } 221 EXPORT_SYMBOL_FOR_KVM(x86_virt_spec_ctrl); 222 223 static void x86_amd_ssb_disable(void) 224 { 225 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 226 227 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 228 wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 229 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 230 wrmsrq(MSR_AMD64_LS_CFG, msrval); 231 } 232 233 #undef pr_fmt 234 #define pr_fmt(fmt) "MDS: " fmt 235 236 /* 237 * Returns true if vulnerability should be mitigated based on the 238 * selected attack vector controls. 239 * 240 * See Documentation/admin-guide/hw-vuln/attack_vector_controls.rst 241 */ 242 static bool __init should_mitigate_vuln(unsigned int bug) 243 { 244 switch (bug) { 245 /* 246 * The only runtime-selected spectre_v1 mitigations in the kernel are 247 * related to SWAPGS protection on kernel entry. Therefore, protection 248 * is only required for the user->kernel attack vector. 249 */ 250 case X86_BUG_SPECTRE_V1: 251 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL); 252 253 case X86_BUG_SPECTRE_V2: 254 case X86_BUG_RETBLEED: 255 case X86_BUG_L1TF: 256 case X86_BUG_ITS: 257 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 258 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST); 259 260 case X86_BUG_SPECTRE_V2_USER: 261 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 262 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST); 263 264 /* 265 * All the vulnerabilities below allow potentially leaking data 266 * across address spaces. Therefore, mitigation is required for 267 * any of these 4 attack vectors. 268 */ 269 case X86_BUG_MDS: 270 case X86_BUG_TAA: 271 case X86_BUG_MMIO_STALE_DATA: 272 case X86_BUG_RFDS: 273 case X86_BUG_SRBDS: 274 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 275 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 276 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 277 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST); 278 279 case X86_BUG_GDS: 280 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 281 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 282 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 283 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST) || 284 (smt_mitigations != SMT_MITIGATIONS_OFF); 285 286 case X86_BUG_SPEC_STORE_BYPASS: 287 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER); 288 289 case X86_BUG_VMSCAPE: 290 return cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST); 291 292 default: 293 WARN(1, "Unknown bug %x\n", bug); 294 return false; 295 } 296 } 297 298 /* Default mitigation for MDS-affected CPUs */ 299 static enum mds_mitigations mds_mitigation __ro_after_init = 300 IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_AUTO : MDS_MITIGATION_OFF; 301 static bool mds_nosmt __ro_after_init = false; 302 303 static const char * const mds_strings[] = { 304 [MDS_MITIGATION_OFF] = "Vulnerable", 305 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 306 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 307 }; 308 309 enum taa_mitigations { 310 TAA_MITIGATION_OFF, 311 TAA_MITIGATION_AUTO, 312 TAA_MITIGATION_UCODE_NEEDED, 313 TAA_MITIGATION_VERW, 314 TAA_MITIGATION_TSX_DISABLED, 315 }; 316 317 /* Default mitigation for TAA-affected CPUs */ 318 static enum taa_mitigations taa_mitigation __ro_after_init = 319 IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_AUTO : TAA_MITIGATION_OFF; 320 321 enum mmio_mitigations { 322 MMIO_MITIGATION_OFF, 323 MMIO_MITIGATION_AUTO, 324 MMIO_MITIGATION_UCODE_NEEDED, 325 MMIO_MITIGATION_VERW, 326 }; 327 328 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 329 static enum mmio_mitigations mmio_mitigation __ro_after_init = 330 IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ? MMIO_MITIGATION_AUTO : MMIO_MITIGATION_OFF; 331 332 enum rfds_mitigations { 333 RFDS_MITIGATION_OFF, 334 RFDS_MITIGATION_AUTO, 335 RFDS_MITIGATION_VERW, 336 RFDS_MITIGATION_UCODE_NEEDED, 337 }; 338 339 /* Default mitigation for Register File Data Sampling */ 340 static enum rfds_mitigations rfds_mitigation __ro_after_init = 341 IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF; 342 343 /* 344 * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing on exit to 345 * userspace *and* on entry to KVM guests. 346 */ 347 static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init; 348 349 static void __init mds_select_mitigation(void) 350 { 351 if (!boot_cpu_has_bug(X86_BUG_MDS)) { 352 mds_mitigation = MDS_MITIGATION_OFF; 353 return; 354 } 355 356 if (mds_mitigation == MDS_MITIGATION_AUTO) { 357 if (should_mitigate_vuln(X86_BUG_MDS)) 358 mds_mitigation = MDS_MITIGATION_FULL; 359 else 360 mds_mitigation = MDS_MITIGATION_OFF; 361 } 362 363 if (mds_mitigation == MDS_MITIGATION_OFF) 364 return; 365 366 verw_clear_cpu_buf_mitigation_selected = true; 367 } 368 369 static void __init mds_update_mitigation(void) 370 { 371 if (!boot_cpu_has_bug(X86_BUG_MDS)) 372 return; 373 374 /* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */ 375 if (verw_clear_cpu_buf_mitigation_selected) 376 mds_mitigation = MDS_MITIGATION_FULL; 377 378 if (mds_mitigation == MDS_MITIGATION_FULL) { 379 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 380 mds_mitigation = MDS_MITIGATION_VMWERV; 381 } 382 383 pr_info("%s\n", mds_strings[mds_mitigation]); 384 } 385 386 static void __init mds_apply_mitigation(void) 387 { 388 if (mds_mitigation == MDS_MITIGATION_FULL || 389 mds_mitigation == MDS_MITIGATION_VMWERV) { 390 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 391 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 392 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 393 (mds_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)) 394 cpu_smt_disable(false); 395 } 396 } 397 398 static int __init mds_cmdline(char *str) 399 { 400 if (!boot_cpu_has_bug(X86_BUG_MDS)) 401 return 0; 402 403 if (!str) 404 return -EINVAL; 405 406 if (!strcmp(str, "off")) 407 mds_mitigation = MDS_MITIGATION_OFF; 408 else if (!strcmp(str, "full")) 409 mds_mitigation = MDS_MITIGATION_FULL; 410 else if (!strcmp(str, "full,nosmt")) { 411 mds_mitigation = MDS_MITIGATION_FULL; 412 mds_nosmt = true; 413 } 414 415 return 0; 416 } 417 early_param("mds", mds_cmdline); 418 419 #undef pr_fmt 420 #define pr_fmt(fmt) "TAA: " fmt 421 422 static bool taa_nosmt __ro_after_init; 423 424 static const char * const taa_strings[] = { 425 [TAA_MITIGATION_OFF] = "Vulnerable", 426 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 427 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 428 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 429 }; 430 431 static bool __init taa_vulnerable(void) 432 { 433 return boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM); 434 } 435 436 static void __init taa_select_mitigation(void) 437 { 438 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 439 taa_mitigation = TAA_MITIGATION_OFF; 440 return; 441 } 442 443 /* TSX previously disabled by tsx=off */ 444 if (!boot_cpu_has(X86_FEATURE_RTM)) { 445 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 446 return; 447 } 448 449 /* Microcode will be checked in taa_update_mitigation(). */ 450 if (taa_mitigation == TAA_MITIGATION_AUTO) { 451 if (should_mitigate_vuln(X86_BUG_TAA)) 452 taa_mitigation = TAA_MITIGATION_VERW; 453 else 454 taa_mitigation = TAA_MITIGATION_OFF; 455 } 456 457 if (taa_mitigation != TAA_MITIGATION_OFF) 458 verw_clear_cpu_buf_mitigation_selected = true; 459 } 460 461 static void __init taa_update_mitigation(void) 462 { 463 if (!taa_vulnerable()) 464 return; 465 466 if (verw_clear_cpu_buf_mitigation_selected) 467 taa_mitigation = TAA_MITIGATION_VERW; 468 469 if (taa_mitigation == TAA_MITIGATION_VERW) { 470 /* Check if the requisite ucode is available. */ 471 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 472 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 473 474 /* 475 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 476 * A microcode update fixes this behavior to clear CPU buffers. It also 477 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 478 * ARCH_CAP_TSX_CTRL_MSR bit. 479 * 480 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 481 * update is required. 482 */ 483 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && 484 !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) 485 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 486 } 487 488 pr_info("%s\n", taa_strings[taa_mitigation]); 489 } 490 491 static void __init taa_apply_mitigation(void) 492 { 493 if (taa_mitigation == TAA_MITIGATION_VERW || 494 taa_mitigation == TAA_MITIGATION_UCODE_NEEDED) { 495 /* 496 * TSX is enabled, select alternate mitigation for TAA which is 497 * the same as MDS. Enable MDS static branch to clear CPU buffers. 498 * 499 * For guests that can't determine whether the correct microcode is 500 * present on host, enable the mitigation for UCODE_NEEDED as well. 501 */ 502 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 503 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 504 505 if (taa_nosmt || smt_mitigations == SMT_MITIGATIONS_ON) 506 cpu_smt_disable(false); 507 } 508 } 509 510 static int __init tsx_async_abort_parse_cmdline(char *str) 511 { 512 if (!boot_cpu_has_bug(X86_BUG_TAA)) 513 return 0; 514 515 if (!str) 516 return -EINVAL; 517 518 if (!strcmp(str, "off")) { 519 taa_mitigation = TAA_MITIGATION_OFF; 520 } else if (!strcmp(str, "full")) { 521 taa_mitigation = TAA_MITIGATION_VERW; 522 } else if (!strcmp(str, "full,nosmt")) { 523 taa_mitigation = TAA_MITIGATION_VERW; 524 taa_nosmt = true; 525 } 526 527 return 0; 528 } 529 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 530 531 #undef pr_fmt 532 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 533 534 static bool mmio_nosmt __ro_after_init = false; 535 536 static const char * const mmio_strings[] = { 537 [MMIO_MITIGATION_OFF] = "Vulnerable", 538 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 539 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 540 }; 541 542 static void __init mmio_select_mitigation(void) 543 { 544 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 545 mmio_mitigation = MMIO_MITIGATION_OFF; 546 return; 547 } 548 549 /* Microcode will be checked in mmio_update_mitigation(). */ 550 if (mmio_mitigation == MMIO_MITIGATION_AUTO) { 551 if (should_mitigate_vuln(X86_BUG_MMIO_STALE_DATA)) 552 mmio_mitigation = MMIO_MITIGATION_VERW; 553 else 554 mmio_mitigation = MMIO_MITIGATION_OFF; 555 } 556 557 if (mmio_mitigation == MMIO_MITIGATION_OFF) 558 return; 559 560 /* 561 * Enable CPU buffer clear mitigation for host and VMM, if also affected 562 * by MDS or TAA. 563 */ 564 if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable()) 565 verw_clear_cpu_buf_mitigation_selected = true; 566 } 567 568 static void __init mmio_update_mitigation(void) 569 { 570 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 571 return; 572 573 if (verw_clear_cpu_buf_mitigation_selected) 574 mmio_mitigation = MMIO_MITIGATION_VERW; 575 576 if (mmio_mitigation == MMIO_MITIGATION_VERW) { 577 /* 578 * Check if the system has the right microcode. 579 * 580 * CPU Fill buffer clear mitigation is enumerated by either an explicit 581 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 582 * affected systems. 583 */ 584 if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || 585 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 586 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 587 !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))) 588 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 589 } 590 591 pr_info("%s\n", mmio_strings[mmio_mitigation]); 592 } 593 594 static void __init mmio_apply_mitigation(void) 595 { 596 if (mmio_mitigation == MMIO_MITIGATION_OFF) 597 return; 598 599 /* 600 * Only enable the VMM mitigation if the CPU buffer clear mitigation is 601 * not being used. 602 */ 603 if (verw_clear_cpu_buf_mitigation_selected) { 604 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 605 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 606 } else { 607 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO); 608 } 609 610 /* 611 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 612 * be propagated to uncore buffers, clearing the Fill buffers on idle 613 * is required irrespective of SMT state. 614 */ 615 if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) 616 static_branch_enable(&cpu_buf_idle_clear); 617 618 if (mmio_nosmt || smt_mitigations == SMT_MITIGATIONS_ON) 619 cpu_smt_disable(false); 620 } 621 622 static int __init mmio_stale_data_parse_cmdline(char *str) 623 { 624 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 625 return 0; 626 627 if (!str) 628 return -EINVAL; 629 630 if (!strcmp(str, "off")) { 631 mmio_mitigation = MMIO_MITIGATION_OFF; 632 } else if (!strcmp(str, "full")) { 633 mmio_mitigation = MMIO_MITIGATION_VERW; 634 } else if (!strcmp(str, "full,nosmt")) { 635 mmio_mitigation = MMIO_MITIGATION_VERW; 636 mmio_nosmt = true; 637 } 638 639 return 0; 640 } 641 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 642 643 #undef pr_fmt 644 #define pr_fmt(fmt) "Register File Data Sampling: " fmt 645 646 static const char * const rfds_strings[] = { 647 [RFDS_MITIGATION_OFF] = "Vulnerable", 648 [RFDS_MITIGATION_VERW] = "Mitigation: Clear Register File", 649 [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 650 }; 651 652 static inline bool __init verw_clears_cpu_reg_file(void) 653 { 654 return (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR); 655 } 656 657 static void __init rfds_select_mitigation(void) 658 { 659 if (!boot_cpu_has_bug(X86_BUG_RFDS)) { 660 rfds_mitigation = RFDS_MITIGATION_OFF; 661 return; 662 } 663 664 if (rfds_mitigation == RFDS_MITIGATION_AUTO) { 665 if (should_mitigate_vuln(X86_BUG_RFDS)) 666 rfds_mitigation = RFDS_MITIGATION_VERW; 667 else 668 rfds_mitigation = RFDS_MITIGATION_OFF; 669 } 670 671 if (rfds_mitigation == RFDS_MITIGATION_OFF) 672 return; 673 674 if (verw_clears_cpu_reg_file()) 675 verw_clear_cpu_buf_mitigation_selected = true; 676 } 677 678 static void __init rfds_update_mitigation(void) 679 { 680 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 681 return; 682 683 if (verw_clear_cpu_buf_mitigation_selected) 684 rfds_mitigation = RFDS_MITIGATION_VERW; 685 686 if (rfds_mitigation == RFDS_MITIGATION_VERW) { 687 if (!verw_clears_cpu_reg_file()) 688 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; 689 } 690 691 pr_info("%s\n", rfds_strings[rfds_mitigation]); 692 } 693 694 static void __init rfds_apply_mitigation(void) 695 { 696 if (rfds_mitigation == RFDS_MITIGATION_VERW) { 697 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 698 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 699 } 700 } 701 702 static __init int rfds_parse_cmdline(char *str) 703 { 704 if (!str) 705 return -EINVAL; 706 707 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 708 return 0; 709 710 if (!strcmp(str, "off")) 711 rfds_mitigation = RFDS_MITIGATION_OFF; 712 else if (!strcmp(str, "on")) 713 rfds_mitigation = RFDS_MITIGATION_VERW; 714 715 return 0; 716 } 717 early_param("reg_file_data_sampling", rfds_parse_cmdline); 718 719 #undef pr_fmt 720 #define pr_fmt(fmt) "SRBDS: " fmt 721 722 enum srbds_mitigations { 723 SRBDS_MITIGATION_OFF, 724 SRBDS_MITIGATION_AUTO, 725 SRBDS_MITIGATION_UCODE_NEEDED, 726 SRBDS_MITIGATION_FULL, 727 SRBDS_MITIGATION_TSX_OFF, 728 SRBDS_MITIGATION_HYPERVISOR, 729 }; 730 731 static enum srbds_mitigations srbds_mitigation __ro_after_init = 732 IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_AUTO : SRBDS_MITIGATION_OFF; 733 734 static const char * const srbds_strings[] = { 735 [SRBDS_MITIGATION_OFF] = "Vulnerable", 736 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 737 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 738 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 739 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 740 }; 741 742 static bool srbds_off; 743 744 void update_srbds_msr(void) 745 { 746 u64 mcu_ctrl; 747 748 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 749 return; 750 751 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 752 return; 753 754 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 755 return; 756 757 /* 758 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 759 * being disabled and it hasn't received the SRBDS MSR microcode. 760 */ 761 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 762 return; 763 764 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 765 766 switch (srbds_mitigation) { 767 case SRBDS_MITIGATION_OFF: 768 case SRBDS_MITIGATION_TSX_OFF: 769 mcu_ctrl |= RNGDS_MITG_DIS; 770 break; 771 case SRBDS_MITIGATION_FULL: 772 mcu_ctrl &= ~RNGDS_MITG_DIS; 773 break; 774 default: 775 break; 776 } 777 778 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 779 } 780 781 static void __init srbds_select_mitigation(void) 782 { 783 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) { 784 srbds_mitigation = SRBDS_MITIGATION_OFF; 785 return; 786 } 787 788 if (srbds_mitigation == SRBDS_MITIGATION_AUTO) { 789 if (should_mitigate_vuln(X86_BUG_SRBDS)) 790 srbds_mitigation = SRBDS_MITIGATION_FULL; 791 else { 792 srbds_mitigation = SRBDS_MITIGATION_OFF; 793 return; 794 } 795 } 796 797 /* 798 * Check to see if this is one of the MDS_NO systems supporting TSX that 799 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 800 * by Processor MMIO Stale Data vulnerability. 801 */ 802 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 803 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 804 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 805 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 806 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 807 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 808 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 809 else if (srbds_off) 810 srbds_mitigation = SRBDS_MITIGATION_OFF; 811 812 pr_info("%s\n", srbds_strings[srbds_mitigation]); 813 } 814 815 static void __init srbds_apply_mitigation(void) 816 { 817 update_srbds_msr(); 818 } 819 820 static int __init srbds_parse_cmdline(char *str) 821 { 822 if (!str) 823 return -EINVAL; 824 825 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 826 return 0; 827 828 srbds_off = !strcmp(str, "off"); 829 return 0; 830 } 831 early_param("srbds", srbds_parse_cmdline); 832 833 #undef pr_fmt 834 #define pr_fmt(fmt) "L1D Flush : " fmt 835 836 enum l1d_flush_mitigations { 837 L1D_FLUSH_OFF = 0, 838 L1D_FLUSH_ON, 839 }; 840 841 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 842 843 static void __init l1d_flush_select_mitigation(void) 844 { 845 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 846 return; 847 848 static_branch_enable(&switch_mm_cond_l1d_flush); 849 pr_info("Conditional flush on switch_mm() enabled\n"); 850 } 851 852 static int __init l1d_flush_parse_cmdline(char *str) 853 { 854 if (!strcmp(str, "on")) 855 l1d_flush_mitigation = L1D_FLUSH_ON; 856 857 return 0; 858 } 859 early_param("l1d_flush", l1d_flush_parse_cmdline); 860 861 #undef pr_fmt 862 #define pr_fmt(fmt) "GDS: " fmt 863 864 enum gds_mitigations { 865 GDS_MITIGATION_OFF, 866 GDS_MITIGATION_AUTO, 867 GDS_MITIGATION_UCODE_NEEDED, 868 GDS_MITIGATION_FORCE, 869 GDS_MITIGATION_FULL, 870 GDS_MITIGATION_FULL_LOCKED, 871 GDS_MITIGATION_HYPERVISOR, 872 }; 873 874 static enum gds_mitigations gds_mitigation __ro_after_init = 875 IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_AUTO : GDS_MITIGATION_OFF; 876 877 static const char * const gds_strings[] = { 878 [GDS_MITIGATION_OFF] = "Vulnerable", 879 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 880 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 881 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 882 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 883 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 884 }; 885 886 bool gds_ucode_mitigated(void) 887 { 888 return (gds_mitigation == GDS_MITIGATION_FULL || 889 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 890 } 891 EXPORT_SYMBOL_FOR_KVM(gds_ucode_mitigated); 892 893 void update_gds_msr(void) 894 { 895 u64 mcu_ctrl_after; 896 u64 mcu_ctrl; 897 898 switch (gds_mitigation) { 899 case GDS_MITIGATION_OFF: 900 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 901 mcu_ctrl |= GDS_MITG_DIS; 902 break; 903 case GDS_MITIGATION_FULL_LOCKED: 904 /* 905 * The LOCKED state comes from the boot CPU. APs might not have 906 * the same state. Make sure the mitigation is enabled on all 907 * CPUs. 908 */ 909 case GDS_MITIGATION_FULL: 910 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 911 mcu_ctrl &= ~GDS_MITG_DIS; 912 break; 913 case GDS_MITIGATION_FORCE: 914 case GDS_MITIGATION_UCODE_NEEDED: 915 case GDS_MITIGATION_HYPERVISOR: 916 case GDS_MITIGATION_AUTO: 917 return; 918 } 919 920 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 921 922 /* 923 * Check to make sure that the WRMSR value was not ignored. Writes to 924 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 925 * processor was not. 926 */ 927 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 928 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 929 } 930 931 static void __init gds_select_mitigation(void) 932 { 933 u64 mcu_ctrl; 934 935 if (!boot_cpu_has_bug(X86_BUG_GDS)) 936 return; 937 938 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 939 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 940 return; 941 } 942 943 /* Will verify below that mitigation _can_ be disabled */ 944 if (gds_mitigation == GDS_MITIGATION_AUTO) { 945 if (should_mitigate_vuln(X86_BUG_GDS)) 946 gds_mitigation = GDS_MITIGATION_FULL; 947 else 948 gds_mitigation = GDS_MITIGATION_OFF; 949 } 950 951 /* No microcode */ 952 if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) { 953 if (gds_mitigation != GDS_MITIGATION_FORCE) 954 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 955 return; 956 } 957 958 /* Microcode has mitigation, use it */ 959 if (gds_mitigation == GDS_MITIGATION_FORCE) 960 gds_mitigation = GDS_MITIGATION_FULL; 961 962 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 963 if (mcu_ctrl & GDS_MITG_LOCKED) { 964 if (gds_mitigation == GDS_MITIGATION_OFF) 965 pr_warn("Mitigation locked. Disable failed.\n"); 966 967 /* 968 * The mitigation is selected from the boot CPU. All other CPUs 969 * _should_ have the same state. If the boot CPU isn't locked 970 * but others are then update_gds_msr() will WARN() of the state 971 * mismatch. If the boot CPU is locked update_gds_msr() will 972 * ensure the other CPUs have the mitigation enabled. 973 */ 974 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 975 } 976 } 977 978 static void __init gds_apply_mitigation(void) 979 { 980 if (!boot_cpu_has_bug(X86_BUG_GDS)) 981 return; 982 983 /* Microcode is present */ 984 if (x86_arch_cap_msr & ARCH_CAP_GDS_CTRL) 985 update_gds_msr(); 986 else if (gds_mitigation == GDS_MITIGATION_FORCE) { 987 /* 988 * This only needs to be done on the boot CPU so do it 989 * here rather than in update_gds_msr() 990 */ 991 setup_clear_cpu_cap(X86_FEATURE_AVX); 992 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 993 } 994 995 pr_info("%s\n", gds_strings[gds_mitigation]); 996 } 997 998 static int __init gds_parse_cmdline(char *str) 999 { 1000 if (!str) 1001 return -EINVAL; 1002 1003 if (!boot_cpu_has_bug(X86_BUG_GDS)) 1004 return 0; 1005 1006 if (!strcmp(str, "off")) 1007 gds_mitigation = GDS_MITIGATION_OFF; 1008 else if (!strcmp(str, "force")) 1009 gds_mitigation = GDS_MITIGATION_FORCE; 1010 1011 return 0; 1012 } 1013 early_param("gather_data_sampling", gds_parse_cmdline); 1014 1015 #undef pr_fmt 1016 #define pr_fmt(fmt) "Spectre V1 : " fmt 1017 1018 enum spectre_v1_mitigation { 1019 SPECTRE_V1_MITIGATION_NONE, 1020 SPECTRE_V1_MITIGATION_AUTO, 1021 }; 1022 1023 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 1024 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ? 1025 SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE; 1026 1027 static const char * const spectre_v1_strings[] = { 1028 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 1029 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 1030 }; 1031 1032 /* 1033 * Does SMAP provide full mitigation against speculative kernel access to 1034 * userspace? 1035 */ 1036 static bool smap_works_speculatively(void) 1037 { 1038 if (!boot_cpu_has(X86_FEATURE_SMAP)) 1039 return false; 1040 1041 /* 1042 * On CPUs which are vulnerable to Meltdown, SMAP does not 1043 * prevent speculative access to user data in the L1 cache. 1044 * Consider SMAP to be non-functional as a mitigation on these 1045 * CPUs. 1046 */ 1047 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 1048 return false; 1049 1050 return true; 1051 } 1052 1053 static void __init spectre_v1_select_mitigation(void) 1054 { 1055 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 1056 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1057 1058 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V1)) 1059 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1060 } 1061 1062 static void __init spectre_v1_apply_mitigation(void) 1063 { 1064 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 1065 return; 1066 1067 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 1068 /* 1069 * With Spectre v1, a user can speculatively control either 1070 * path of a conditional swapgs with a user-controlled GS 1071 * value. The mitigation is to add lfences to both code paths. 1072 * 1073 * If FSGSBASE is enabled, the user can put a kernel address in 1074 * GS, in which case SMAP provides no protection. 1075 * 1076 * If FSGSBASE is disabled, the user can only put a user space 1077 * address in GS. That makes an attack harder, but still 1078 * possible if there's no SMAP protection. 1079 */ 1080 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 1081 !smap_works_speculatively()) { 1082 /* 1083 * Mitigation can be provided from SWAPGS itself or 1084 * PTI as the CR3 write in the Meltdown mitigation 1085 * is serializing. 1086 * 1087 * If neither is there, mitigate with an LFENCE to 1088 * stop speculation through swapgs. 1089 */ 1090 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 1091 !boot_cpu_has(X86_FEATURE_PTI)) 1092 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 1093 1094 /* 1095 * Enable lfences in the kernel entry (non-swapgs) 1096 * paths, to prevent user entry from speculatively 1097 * skipping swapgs. 1098 */ 1099 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 1100 } 1101 } 1102 1103 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1104 } 1105 1106 static int __init nospectre_v1_cmdline(char *str) 1107 { 1108 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1109 return 0; 1110 } 1111 early_param("nospectre_v1", nospectre_v1_cmdline); 1112 1113 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 1114 1115 /* Depends on spectre_v2 mitigation selected already */ 1116 static inline bool cdt_possible(enum spectre_v2_mitigation mode) 1117 { 1118 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) || 1119 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) 1120 return false; 1121 1122 if (mode == SPECTRE_V2_RETPOLINE || 1123 mode == SPECTRE_V2_EIBRS_RETPOLINE) 1124 return true; 1125 1126 return false; 1127 } 1128 1129 #undef pr_fmt 1130 #define pr_fmt(fmt) "RETBleed: " fmt 1131 1132 enum its_mitigation { 1133 ITS_MITIGATION_OFF, 1134 ITS_MITIGATION_AUTO, 1135 ITS_MITIGATION_VMEXIT_ONLY, 1136 ITS_MITIGATION_ALIGNED_THUNKS, 1137 ITS_MITIGATION_RETPOLINE_STUFF, 1138 }; 1139 1140 static enum its_mitigation its_mitigation __ro_after_init = 1141 IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_MITIGATION_AUTO : ITS_MITIGATION_OFF; 1142 1143 enum retbleed_mitigation { 1144 RETBLEED_MITIGATION_NONE, 1145 RETBLEED_MITIGATION_AUTO, 1146 RETBLEED_MITIGATION_UNRET, 1147 RETBLEED_MITIGATION_IBPB, 1148 RETBLEED_MITIGATION_IBRS, 1149 RETBLEED_MITIGATION_EIBRS, 1150 RETBLEED_MITIGATION_STUFF, 1151 }; 1152 1153 static const char * const retbleed_strings[] = { 1154 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 1155 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 1156 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 1157 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 1158 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 1159 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 1160 }; 1161 1162 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 1163 IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE; 1164 1165 static int __ro_after_init retbleed_nosmt = false; 1166 1167 enum srso_mitigation { 1168 SRSO_MITIGATION_NONE, 1169 SRSO_MITIGATION_AUTO, 1170 SRSO_MITIGATION_UCODE_NEEDED, 1171 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 1172 SRSO_MITIGATION_MICROCODE, 1173 SRSO_MITIGATION_NOSMT, 1174 SRSO_MITIGATION_SAFE_RET, 1175 SRSO_MITIGATION_IBPB, 1176 SRSO_MITIGATION_IBPB_ON_VMEXIT, 1177 SRSO_MITIGATION_BP_SPEC_REDUCE, 1178 SRSO_MITIGATION_USER_IBPB, 1179 }; 1180 1181 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO; 1182 1183 static int __init retbleed_parse_cmdline(char *str) 1184 { 1185 if (!str) 1186 return -EINVAL; 1187 1188 while (str) { 1189 char *next = strchr(str, ','); 1190 if (next) { 1191 *next = 0; 1192 next++; 1193 } 1194 1195 if (!strcmp(str, "off")) { 1196 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1197 } else if (!strcmp(str, "auto")) { 1198 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1199 } else if (!strcmp(str, "unret")) { 1200 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1201 } else if (!strcmp(str, "ibpb")) { 1202 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1203 } else if (!strcmp(str, "stuff")) { 1204 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1205 } else if (!strcmp(str, "nosmt")) { 1206 retbleed_nosmt = true; 1207 } else if (!strcmp(str, "force")) { 1208 setup_force_cpu_bug(X86_BUG_RETBLEED); 1209 } else { 1210 pr_err("Ignoring unknown retbleed option (%s).", str); 1211 } 1212 1213 str = next; 1214 } 1215 1216 return 0; 1217 } 1218 early_param("retbleed", retbleed_parse_cmdline); 1219 1220 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 1221 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 1222 1223 static void __init retbleed_select_mitigation(void) 1224 { 1225 if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) { 1226 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1227 return; 1228 } 1229 1230 switch (retbleed_mitigation) { 1231 case RETBLEED_MITIGATION_UNRET: 1232 if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { 1233 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1234 pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); 1235 } 1236 break; 1237 case RETBLEED_MITIGATION_IBPB: 1238 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 1239 pr_err("WARNING: CPU does not support IBPB.\n"); 1240 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1241 } else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 1242 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 1243 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1244 } 1245 break; 1246 case RETBLEED_MITIGATION_STUFF: 1247 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1248 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); 1249 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1250 } else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1251 pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); 1252 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1253 } 1254 break; 1255 default: 1256 break; 1257 } 1258 1259 if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO) 1260 return; 1261 1262 if (!should_mitigate_vuln(X86_BUG_RETBLEED)) { 1263 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1264 return; 1265 } 1266 1267 /* Intel mitigation selected in retbleed_update_mitigation() */ 1268 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1269 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1270 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) 1271 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1272 else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && 1273 boot_cpu_has(X86_FEATURE_IBPB)) 1274 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1275 else 1276 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1277 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1278 /* Final mitigation depends on spectre-v2 selection */ 1279 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1280 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1281 else if (boot_cpu_has(X86_FEATURE_IBRS)) 1282 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1283 else 1284 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1285 } 1286 } 1287 1288 static void __init retbleed_update_mitigation(void) 1289 { 1290 if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) 1291 return; 1292 1293 /* ITS can also enable stuffing */ 1294 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF) 1295 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1296 1297 /* If SRSO is using IBPB, that works for retbleed too */ 1298 if (srso_mitigation == SRSO_MITIGATION_IBPB) 1299 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1300 1301 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF && 1302 !cdt_possible(spectre_v2_enabled)) { 1303 pr_err("WARNING: retbleed=stuff depends on retpoline\n"); 1304 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1305 } 1306 1307 /* 1308 * Let IBRS trump all on Intel without affecting the effects of the 1309 * retbleed= cmdline option except for call depth based stuffing 1310 */ 1311 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1312 switch (spectre_v2_enabled) { 1313 case SPECTRE_V2_IBRS: 1314 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1315 break; 1316 case SPECTRE_V2_EIBRS: 1317 case SPECTRE_V2_EIBRS_RETPOLINE: 1318 case SPECTRE_V2_EIBRS_LFENCE: 1319 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1320 break; 1321 default: 1322 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) { 1323 if (retbleed_mitigation != RETBLEED_MITIGATION_NONE) 1324 pr_err(RETBLEED_INTEL_MSG); 1325 1326 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1327 } 1328 } 1329 } 1330 1331 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1332 } 1333 1334 static void __init retbleed_apply_mitigation(void) 1335 { 1336 bool mitigate_smt = false; 1337 1338 switch (retbleed_mitigation) { 1339 case RETBLEED_MITIGATION_NONE: 1340 return; 1341 1342 case RETBLEED_MITIGATION_UNRET: 1343 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1344 setup_force_cpu_cap(X86_FEATURE_UNRET); 1345 1346 set_return_thunk(retbleed_return_thunk); 1347 1348 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1349 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1350 pr_err(RETBLEED_UNTRAIN_MSG); 1351 1352 mitigate_smt = true; 1353 break; 1354 1355 case RETBLEED_MITIGATION_IBPB: 1356 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1357 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1358 mitigate_smt = true; 1359 1360 /* 1361 * IBPB on entry already obviates the need for 1362 * software-based untraining so clear those in case some 1363 * other mitigation like SRSO has selected them. 1364 */ 1365 setup_clear_cpu_cap(X86_FEATURE_UNRET); 1366 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 1367 1368 /* 1369 * There is no need for RSB filling: write_ibpb() ensures 1370 * all predictions, including the RSB, are invalidated, 1371 * regardless of IBPB implementation. 1372 */ 1373 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1374 1375 break; 1376 1377 case RETBLEED_MITIGATION_STUFF: 1378 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1379 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1380 1381 set_return_thunk(call_depth_return_thunk); 1382 break; 1383 1384 default: 1385 break; 1386 } 1387 1388 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1389 (retbleed_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)) 1390 cpu_smt_disable(false); 1391 } 1392 1393 #undef pr_fmt 1394 #define pr_fmt(fmt) "ITS: " fmt 1395 1396 static const char * const its_strings[] = { 1397 [ITS_MITIGATION_OFF] = "Vulnerable", 1398 [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", 1399 [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", 1400 [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", 1401 }; 1402 1403 static int __init its_parse_cmdline(char *str) 1404 { 1405 if (!str) 1406 return -EINVAL; 1407 1408 if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) { 1409 pr_err("Mitigation disabled at compile time, ignoring option (%s)", str); 1410 return 0; 1411 } 1412 1413 if (!strcmp(str, "off")) { 1414 its_mitigation = ITS_MITIGATION_OFF; 1415 } else if (!strcmp(str, "on")) { 1416 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1417 } else if (!strcmp(str, "force")) { 1418 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1419 setup_force_cpu_bug(X86_BUG_ITS); 1420 } else if (!strcmp(str, "vmexit")) { 1421 its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; 1422 } else if (!strcmp(str, "stuff")) { 1423 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1424 } else { 1425 pr_err("Ignoring unknown indirect_target_selection option (%s).", str); 1426 } 1427 1428 return 0; 1429 } 1430 early_param("indirect_target_selection", its_parse_cmdline); 1431 1432 static void __init its_select_mitigation(void) 1433 { 1434 if (!boot_cpu_has_bug(X86_BUG_ITS)) { 1435 its_mitigation = ITS_MITIGATION_OFF; 1436 return; 1437 } 1438 1439 if (its_mitigation == ITS_MITIGATION_AUTO) { 1440 if (should_mitigate_vuln(X86_BUG_ITS)) 1441 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1442 else 1443 its_mitigation = ITS_MITIGATION_OFF; 1444 } 1445 1446 if (its_mitigation == ITS_MITIGATION_OFF) 1447 return; 1448 1449 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || 1450 !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) { 1451 pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); 1452 its_mitigation = ITS_MITIGATION_OFF; 1453 return; 1454 } 1455 1456 if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { 1457 pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); 1458 its_mitigation = ITS_MITIGATION_OFF; 1459 return; 1460 } 1461 1462 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1463 !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1464 pr_err("RSB stuff mitigation not supported, using default\n"); 1465 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1466 } 1467 1468 if (its_mitigation == ITS_MITIGATION_VMEXIT_ONLY && 1469 !boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) 1470 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1471 } 1472 1473 static void __init its_update_mitigation(void) 1474 { 1475 if (!boot_cpu_has_bug(X86_BUG_ITS)) 1476 return; 1477 1478 switch (spectre_v2_enabled) { 1479 case SPECTRE_V2_NONE: 1480 if (its_mitigation != ITS_MITIGATION_OFF) 1481 pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); 1482 its_mitigation = ITS_MITIGATION_OFF; 1483 break; 1484 case SPECTRE_V2_RETPOLINE: 1485 case SPECTRE_V2_EIBRS_RETPOLINE: 1486 /* Retpoline+CDT mitigates ITS */ 1487 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) 1488 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1489 break; 1490 case SPECTRE_V2_LFENCE: 1491 case SPECTRE_V2_EIBRS_LFENCE: 1492 pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); 1493 its_mitigation = ITS_MITIGATION_OFF; 1494 break; 1495 default: 1496 break; 1497 } 1498 1499 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1500 !cdt_possible(spectre_v2_enabled)) 1501 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1502 1503 pr_info("%s\n", its_strings[its_mitigation]); 1504 } 1505 1506 static void __init its_apply_mitigation(void) 1507 { 1508 switch (its_mitigation) { 1509 case ITS_MITIGATION_OFF: 1510 case ITS_MITIGATION_AUTO: 1511 case ITS_MITIGATION_VMEXIT_ONLY: 1512 break; 1513 case ITS_MITIGATION_ALIGNED_THUNKS: 1514 if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) 1515 setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); 1516 1517 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1518 set_return_thunk(its_return_thunk); 1519 break; 1520 case ITS_MITIGATION_RETPOLINE_STUFF: 1521 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1522 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1523 set_return_thunk(call_depth_return_thunk); 1524 break; 1525 } 1526 } 1527 1528 #undef pr_fmt 1529 #define pr_fmt(fmt) "Transient Scheduler Attacks: " fmt 1530 1531 enum tsa_mitigations { 1532 TSA_MITIGATION_NONE, 1533 TSA_MITIGATION_AUTO, 1534 TSA_MITIGATION_UCODE_NEEDED, 1535 TSA_MITIGATION_USER_KERNEL, 1536 TSA_MITIGATION_VM, 1537 TSA_MITIGATION_FULL, 1538 }; 1539 1540 static const char * const tsa_strings[] = { 1541 [TSA_MITIGATION_NONE] = "Vulnerable", 1542 [TSA_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 1543 [TSA_MITIGATION_USER_KERNEL] = "Mitigation: Clear CPU buffers: user/kernel boundary", 1544 [TSA_MITIGATION_VM] = "Mitigation: Clear CPU buffers: VM", 1545 [TSA_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 1546 }; 1547 1548 static enum tsa_mitigations tsa_mitigation __ro_after_init = 1549 IS_ENABLED(CONFIG_MITIGATION_TSA) ? TSA_MITIGATION_AUTO : TSA_MITIGATION_NONE; 1550 1551 static int __init tsa_parse_cmdline(char *str) 1552 { 1553 if (!str) 1554 return -EINVAL; 1555 1556 if (!strcmp(str, "off")) 1557 tsa_mitigation = TSA_MITIGATION_NONE; 1558 else if (!strcmp(str, "on")) 1559 tsa_mitigation = TSA_MITIGATION_FULL; 1560 else if (!strcmp(str, "user")) 1561 tsa_mitigation = TSA_MITIGATION_USER_KERNEL; 1562 else if (!strcmp(str, "vm")) 1563 tsa_mitigation = TSA_MITIGATION_VM; 1564 else 1565 pr_err("Ignoring unknown tsa=%s option.\n", str); 1566 1567 return 0; 1568 } 1569 early_param("tsa", tsa_parse_cmdline); 1570 1571 static void __init tsa_select_mitigation(void) 1572 { 1573 if (!boot_cpu_has_bug(X86_BUG_TSA)) { 1574 tsa_mitigation = TSA_MITIGATION_NONE; 1575 return; 1576 } 1577 1578 if (tsa_mitigation == TSA_MITIGATION_AUTO) { 1579 bool vm = false, uk = false; 1580 1581 tsa_mitigation = TSA_MITIGATION_NONE; 1582 1583 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 1584 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) { 1585 tsa_mitigation = TSA_MITIGATION_USER_KERNEL; 1586 uk = true; 1587 } 1588 1589 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 1590 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) { 1591 tsa_mitigation = TSA_MITIGATION_VM; 1592 vm = true; 1593 } 1594 1595 if (uk && vm) 1596 tsa_mitigation = TSA_MITIGATION_FULL; 1597 } 1598 1599 if (tsa_mitigation == TSA_MITIGATION_NONE) 1600 return; 1601 1602 if (!boot_cpu_has(X86_FEATURE_VERW_CLEAR)) 1603 tsa_mitigation = TSA_MITIGATION_UCODE_NEEDED; 1604 1605 /* 1606 * No need to set verw_clear_cpu_buf_mitigation_selected - it 1607 * doesn't fit all cases here and it is not needed because this 1608 * is the only VERW-based mitigation on AMD. 1609 */ 1610 pr_info("%s\n", tsa_strings[tsa_mitigation]); 1611 } 1612 1613 static void __init tsa_apply_mitigation(void) 1614 { 1615 switch (tsa_mitigation) { 1616 case TSA_MITIGATION_USER_KERNEL: 1617 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 1618 break; 1619 case TSA_MITIGATION_VM: 1620 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 1621 break; 1622 case TSA_MITIGATION_FULL: 1623 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 1624 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 1625 break; 1626 default: 1627 break; 1628 } 1629 } 1630 1631 #undef pr_fmt 1632 #define pr_fmt(fmt) "Spectre V2 : " fmt 1633 1634 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1635 SPECTRE_V2_USER_NONE; 1636 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1637 SPECTRE_V2_USER_NONE; 1638 1639 #ifdef CONFIG_MITIGATION_RETPOLINE 1640 static bool spectre_v2_bad_module; 1641 1642 bool retpoline_module_ok(bool has_retpoline) 1643 { 1644 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1645 return true; 1646 1647 pr_err("System may be vulnerable to spectre v2\n"); 1648 spectre_v2_bad_module = true; 1649 return false; 1650 } 1651 1652 static inline const char *spectre_v2_module_string(void) 1653 { 1654 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1655 } 1656 1657 /* 1658 * The "retpoline sequence" is the "call;mov;ret" sequence that 1659 * replaces normal indirect branch instructions. Differentiate 1660 * *the* retpoline sequence from the LFENCE-prefixed indirect 1661 * branches that simply use the retpoline infrastructure. 1662 */ 1663 static inline bool retpoline_seq_enabled(void) 1664 { 1665 return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE); 1666 } 1667 1668 #else 1669 static inline const char *spectre_v2_module_string(void) { return ""; } 1670 static inline bool retpoline_seq_enabled(void) { return false; } 1671 #endif 1672 1673 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1674 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1675 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1676 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1677 1678 #ifdef CONFIG_BPF_SYSCALL 1679 void unpriv_ebpf_notify(int new_state) 1680 { 1681 if (new_state) 1682 return; 1683 1684 /* Unprivileged eBPF is enabled */ 1685 1686 switch (spectre_v2_enabled) { 1687 case SPECTRE_V2_EIBRS: 1688 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1689 break; 1690 case SPECTRE_V2_EIBRS_LFENCE: 1691 if (sched_smt_active()) 1692 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1693 break; 1694 default: 1695 break; 1696 } 1697 } 1698 #endif 1699 1700 /* The kernel command line selection for spectre v2 */ 1701 enum spectre_v2_mitigation_cmd { 1702 SPECTRE_V2_CMD_NONE, 1703 SPECTRE_V2_CMD_AUTO, 1704 SPECTRE_V2_CMD_FORCE, 1705 SPECTRE_V2_CMD_RETPOLINE, 1706 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1707 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1708 SPECTRE_V2_CMD_EIBRS, 1709 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1710 SPECTRE_V2_CMD_EIBRS_LFENCE, 1711 SPECTRE_V2_CMD_IBRS, 1712 }; 1713 1714 static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init = 1715 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE; 1716 1717 enum spectre_v2_user_mitigation_cmd { 1718 SPECTRE_V2_USER_CMD_NONE, 1719 SPECTRE_V2_USER_CMD_AUTO, 1720 SPECTRE_V2_USER_CMD_FORCE, 1721 SPECTRE_V2_USER_CMD_PRCTL, 1722 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1723 SPECTRE_V2_USER_CMD_SECCOMP, 1724 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1725 }; 1726 1727 static enum spectre_v2_user_mitigation_cmd spectre_v2_user_cmd __ro_after_init = 1728 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE; 1729 1730 static const char * const spectre_v2_user_strings[] = { 1731 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1732 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1733 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1734 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1735 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1736 }; 1737 1738 static int __init spectre_v2_user_parse_cmdline(char *str) 1739 { 1740 if (!str) 1741 return -EINVAL; 1742 1743 if (!strcmp(str, "auto")) 1744 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_AUTO; 1745 else if (!strcmp(str, "off")) 1746 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_NONE; 1747 else if (!strcmp(str, "on")) 1748 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_FORCE; 1749 else if (!strcmp(str, "prctl")) 1750 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL; 1751 else if (!strcmp(str, "prctl,ibpb")) 1752 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL_IBPB; 1753 else if (!strcmp(str, "seccomp")) 1754 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP; 1755 else if (!strcmp(str, "seccomp,ibpb")) 1756 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP_IBPB; 1757 else 1758 pr_err("Ignoring unknown spectre_v2_user option (%s).", str); 1759 1760 return 0; 1761 } 1762 early_param("spectre_v2_user", spectre_v2_user_parse_cmdline); 1763 1764 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1765 { 1766 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1767 } 1768 1769 static void __init spectre_v2_user_select_mitigation(void) 1770 { 1771 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1772 return; 1773 1774 switch (spectre_v2_user_cmd) { 1775 case SPECTRE_V2_USER_CMD_NONE: 1776 return; 1777 case SPECTRE_V2_USER_CMD_FORCE: 1778 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1779 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1780 break; 1781 case SPECTRE_V2_USER_CMD_AUTO: 1782 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2_USER)) 1783 break; 1784 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1785 if (smt_mitigations == SMT_MITIGATIONS_OFF) 1786 break; 1787 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1788 break; 1789 case SPECTRE_V2_USER_CMD_PRCTL: 1790 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1791 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1792 break; 1793 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1794 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1795 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1796 break; 1797 case SPECTRE_V2_USER_CMD_SECCOMP: 1798 if (IS_ENABLED(CONFIG_SECCOMP)) 1799 spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP; 1800 else 1801 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1802 spectre_v2_user_stibp = spectre_v2_user_ibpb; 1803 break; 1804 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1805 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1806 if (IS_ENABLED(CONFIG_SECCOMP)) 1807 spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP; 1808 else 1809 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1810 break; 1811 } 1812 1813 /* 1814 * At this point, an STIBP mode other than "off" has been set. 1815 * If STIBP support is not being forced, check if STIBP always-on 1816 * is preferred. 1817 */ 1818 if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1819 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) && 1820 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1821 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1822 1823 if (!boot_cpu_has(X86_FEATURE_IBPB)) 1824 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1825 1826 if (!boot_cpu_has(X86_FEATURE_STIBP)) 1827 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1828 } 1829 1830 static void __init spectre_v2_user_update_mitigation(void) 1831 { 1832 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1833 return; 1834 1835 /* The spectre_v2 cmd line can override spectre_v2_user options */ 1836 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) { 1837 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1838 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1839 } else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) { 1840 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1841 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1842 } 1843 1844 /* 1845 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1846 * is not required. 1847 * 1848 * Intel's Enhanced IBRS also protects against cross-thread branch target 1849 * injection in user-mode as the IBRS bit remains always set which 1850 * implicitly enables cross-thread protections. However, in legacy IBRS 1851 * mode, the IBRS bit is set only on kernel entry and cleared on return 1852 * to userspace. AMD Automatic IBRS also does not protect userspace. 1853 * These modes therefore disable the implicit cross-thread protection, 1854 * so allow for STIBP to be selected in those cases. 1855 */ 1856 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1857 !cpu_smt_possible() || 1858 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1859 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) { 1860 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1861 return; 1862 } 1863 1864 if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE && 1865 (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1866 retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) { 1867 if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT && 1868 spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED) 1869 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1870 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1871 } 1872 pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]); 1873 } 1874 1875 static void __init spectre_v2_user_apply_mitigation(void) 1876 { 1877 /* Initialize Indirect Branch Prediction Barrier */ 1878 if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) { 1879 static_branch_enable(&switch_vcpu_ibpb); 1880 1881 switch (spectre_v2_user_ibpb) { 1882 case SPECTRE_V2_USER_STRICT: 1883 static_branch_enable(&switch_mm_always_ibpb); 1884 break; 1885 case SPECTRE_V2_USER_PRCTL: 1886 case SPECTRE_V2_USER_SECCOMP: 1887 static_branch_enable(&switch_mm_cond_ibpb); 1888 break; 1889 default: 1890 break; 1891 } 1892 1893 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1894 static_key_enabled(&switch_mm_always_ibpb) ? 1895 "always-on" : "conditional"); 1896 } 1897 } 1898 1899 static const char * const spectre_v2_strings[] = { 1900 [SPECTRE_V2_NONE] = "Vulnerable", 1901 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1902 [SPECTRE_V2_LFENCE] = "Vulnerable: LFENCE", 1903 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1904 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1905 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1906 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1907 }; 1908 1909 static bool nospectre_v2 __ro_after_init; 1910 1911 static int __init nospectre_v2_parse_cmdline(char *str) 1912 { 1913 nospectre_v2 = true; 1914 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 1915 return 0; 1916 } 1917 early_param("nospectre_v2", nospectre_v2_parse_cmdline); 1918 1919 static int __init spectre_v2_parse_cmdline(char *str) 1920 { 1921 if (!str) 1922 return -EINVAL; 1923 1924 if (nospectre_v2) 1925 return 0; 1926 1927 if (!strcmp(str, "off")) { 1928 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 1929 } else if (!strcmp(str, "on")) { 1930 spectre_v2_cmd = SPECTRE_V2_CMD_FORCE; 1931 setup_force_cpu_bug(X86_BUG_SPECTRE_V2); 1932 setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER); 1933 } else if (!strcmp(str, "retpoline")) { 1934 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE; 1935 } else if (!strcmp(str, "retpoline,amd") || 1936 !strcmp(str, "retpoline,lfence")) { 1937 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_LFENCE; 1938 } else if (!strcmp(str, "retpoline,generic")) { 1939 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_GENERIC; 1940 } else if (!strcmp(str, "eibrs")) { 1941 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS; 1942 } else if (!strcmp(str, "eibrs,lfence")) { 1943 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_LFENCE; 1944 } else if (!strcmp(str, "eibrs,retpoline")) { 1945 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_RETPOLINE; 1946 } else if (!strcmp(str, "auto")) { 1947 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 1948 } else if (!strcmp(str, "ibrs")) { 1949 spectre_v2_cmd = SPECTRE_V2_CMD_IBRS; 1950 } else { 1951 pr_err("Ignoring unknown spectre_v2 option (%s).", str); 1952 } 1953 1954 return 0; 1955 } 1956 early_param("spectre_v2", spectre_v2_parse_cmdline); 1957 1958 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1959 { 1960 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 1961 pr_err("Kernel not compiled with retpoline"); 1962 return SPECTRE_V2_NONE; 1963 } 1964 1965 return SPECTRE_V2_RETPOLINE; 1966 } 1967 1968 static bool __ro_after_init rrsba_disabled; 1969 1970 /* Disable in-kernel use of non-RSB RET predictors */ 1971 static void __init spec_ctrl_disable_kernel_rrsba(void) 1972 { 1973 if (rrsba_disabled) 1974 return; 1975 1976 if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) { 1977 rrsba_disabled = true; 1978 return; 1979 } 1980 1981 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1982 return; 1983 1984 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1985 update_spec_ctrl(x86_spec_ctrl_base); 1986 rrsba_disabled = true; 1987 } 1988 1989 static void __init spectre_v2_select_rsb_mitigation(enum spectre_v2_mitigation mode) 1990 { 1991 /* 1992 * WARNING! There are many subtleties to consider when changing *any* 1993 * code related to RSB-related mitigations. Before doing so, carefully 1994 * read the following document, and update if necessary: 1995 * 1996 * Documentation/admin-guide/hw-vuln/rsb.rst 1997 * 1998 * In an overly simplified nutshell: 1999 * 2000 * - User->user RSB attacks are conditionally mitigated during 2001 * context switches by cond_mitigation -> write_ibpb(). 2002 * 2003 * - User->kernel and guest->host attacks are mitigated by eIBRS or 2004 * RSB filling. 2005 * 2006 * Though, depending on config, note that other alternative 2007 * mitigations may end up getting used instead, e.g., IBPB on 2008 * entry/vmexit, call depth tracking, or return thunks. 2009 */ 2010 2011 switch (mode) { 2012 case SPECTRE_V2_NONE: 2013 break; 2014 2015 case SPECTRE_V2_EIBRS: 2016 case SPECTRE_V2_EIBRS_LFENCE: 2017 case SPECTRE_V2_EIBRS_RETPOLINE: 2018 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2019 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 2020 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 2021 } 2022 break; 2023 2024 case SPECTRE_V2_RETPOLINE: 2025 case SPECTRE_V2_LFENCE: 2026 case SPECTRE_V2_IBRS: 2027 pr_info("Spectre v2 / SpectreRSB: Filling RSB on context switch and VMEXIT\n"); 2028 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 2029 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 2030 break; 2031 2032 default: 2033 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation\n"); 2034 dump_stack(); 2035 break; 2036 } 2037 } 2038 2039 /* 2040 * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by 2041 * branch history in userspace. Not needed if BHI_NO is set. 2042 */ 2043 static bool __init spec_ctrl_bhi_dis(void) 2044 { 2045 if (!boot_cpu_has(X86_FEATURE_BHI_CTRL)) 2046 return false; 2047 2048 x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S; 2049 update_spec_ctrl(x86_spec_ctrl_base); 2050 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW); 2051 2052 return true; 2053 } 2054 2055 enum bhi_mitigations { 2056 BHI_MITIGATION_OFF, 2057 BHI_MITIGATION_AUTO, 2058 BHI_MITIGATION_ON, 2059 BHI_MITIGATION_VMEXIT_ONLY, 2060 }; 2061 2062 static enum bhi_mitigations bhi_mitigation __ro_after_init = 2063 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF; 2064 2065 static int __init spectre_bhi_parse_cmdline(char *str) 2066 { 2067 if (!str) 2068 return -EINVAL; 2069 2070 if (!strcmp(str, "off")) 2071 bhi_mitigation = BHI_MITIGATION_OFF; 2072 else if (!strcmp(str, "on")) 2073 bhi_mitigation = BHI_MITIGATION_ON; 2074 else if (!strcmp(str, "vmexit")) 2075 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 2076 else 2077 pr_err("Ignoring unknown spectre_bhi option (%s)", str); 2078 2079 return 0; 2080 } 2081 early_param("spectre_bhi", spectre_bhi_parse_cmdline); 2082 2083 static void __init bhi_select_mitigation(void) 2084 { 2085 if (!boot_cpu_has(X86_BUG_BHI)) 2086 bhi_mitigation = BHI_MITIGATION_OFF; 2087 2088 if (bhi_mitigation != BHI_MITIGATION_AUTO) 2089 return; 2090 2091 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL)) 2092 bhi_mitigation = BHI_MITIGATION_ON; 2093 else if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST)) 2094 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 2095 else 2096 bhi_mitigation = BHI_MITIGATION_OFF; 2097 } 2098 2099 static void __init bhi_update_mitigation(void) 2100 { 2101 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) 2102 bhi_mitigation = BHI_MITIGATION_OFF; 2103 } 2104 2105 static void __init bhi_apply_mitigation(void) 2106 { 2107 if (bhi_mitigation == BHI_MITIGATION_OFF) 2108 return; 2109 2110 /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ 2111 if (retpoline_seq_enabled()) { 2112 spec_ctrl_disable_kernel_rrsba(); 2113 if (rrsba_disabled) 2114 return; 2115 } 2116 2117 if (!IS_ENABLED(CONFIG_X86_64)) 2118 return; 2119 2120 /* Mitigate in hardware if supported */ 2121 if (spec_ctrl_bhi_dis()) 2122 return; 2123 2124 if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) { 2125 pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n"); 2126 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2127 return; 2128 } 2129 2130 pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n"); 2131 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP); 2132 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2133 } 2134 2135 static void __init spectre_v2_select_mitigation(void) 2136 { 2137 if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE || 2138 spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 2139 spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 2140 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 2141 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 2142 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 2143 pr_err("RETPOLINE selected but not compiled in. Switching to AUTO select\n"); 2144 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2145 } 2146 2147 if ((spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS || 2148 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 2149 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 2150 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2151 pr_err("EIBRS selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n"); 2152 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2153 } 2154 2155 if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 2156 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 2157 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 2158 pr_err("LFENCE selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n"); 2159 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2160 } 2161 2162 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) { 2163 pr_err("IBRS selected but not compiled in. Switching to AUTO select\n"); 2164 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2165 } 2166 2167 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 2168 pr_err("IBRS selected but not Intel CPU. Switching to AUTO select\n"); 2169 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2170 } 2171 2172 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 2173 pr_err("IBRS selected but CPU doesn't have IBRS. Switching to AUTO select\n"); 2174 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2175 } 2176 2177 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 2178 pr_err("IBRS selected but running as XenPV guest. Switching to AUTO select\n"); 2179 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2180 } 2181 2182 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) { 2183 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 2184 return; 2185 } 2186 2187 switch (spectre_v2_cmd) { 2188 case SPECTRE_V2_CMD_NONE: 2189 return; 2190 2191 case SPECTRE_V2_CMD_AUTO: 2192 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2)) 2193 break; 2194 fallthrough; 2195 case SPECTRE_V2_CMD_FORCE: 2196 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2197 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2198 break; 2199 } 2200 2201 spectre_v2_enabled = spectre_v2_select_retpoline(); 2202 break; 2203 2204 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 2205 pr_err(SPECTRE_V2_LFENCE_MSG); 2206 spectre_v2_enabled = SPECTRE_V2_LFENCE; 2207 break; 2208 2209 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 2210 spectre_v2_enabled = SPECTRE_V2_RETPOLINE; 2211 break; 2212 2213 case SPECTRE_V2_CMD_RETPOLINE: 2214 spectre_v2_enabled = spectre_v2_select_retpoline(); 2215 break; 2216 2217 case SPECTRE_V2_CMD_IBRS: 2218 spectre_v2_enabled = SPECTRE_V2_IBRS; 2219 break; 2220 2221 case SPECTRE_V2_CMD_EIBRS: 2222 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2223 break; 2224 2225 case SPECTRE_V2_CMD_EIBRS_LFENCE: 2226 spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE; 2227 break; 2228 2229 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 2230 spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE; 2231 break; 2232 } 2233 } 2234 2235 static void __init spectre_v2_update_mitigation(void) 2236 { 2237 if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO && 2238 !spectre_v2_in_eibrs_mode(spectre_v2_enabled)) { 2239 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && 2240 boot_cpu_has_bug(X86_BUG_RETBLEED) && 2241 retbleed_mitigation != RETBLEED_MITIGATION_NONE && 2242 retbleed_mitigation != RETBLEED_MITIGATION_STUFF && 2243 boot_cpu_has(X86_FEATURE_IBRS) && 2244 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 2245 spectre_v2_enabled = SPECTRE_V2_IBRS; 2246 } 2247 } 2248 2249 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2250 pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); 2251 } 2252 2253 #ifdef CONFIG_BPF_JIT 2254 static void __bpf_arch_ibpb(void *unused) 2255 { 2256 write_ibpb(); 2257 } 2258 2259 void bpf_arch_ibpb(void) 2260 { 2261 on_each_cpu(__bpf_arch_ibpb, NULL, 1); 2262 } 2263 2264 static bool __init cpu_wants_ibpb_bpf(void) 2265 { 2266 /* A genuine retpoline already neutralizes ring0 indirect predictions */ 2267 if (retpoline_seq_enabled()) 2268 return false; 2269 2270 return boot_cpu_has(X86_FEATURE_IBPB); 2271 } 2272 #endif 2273 2274 static void __init spectre_v2_apply_mitigation(void) 2275 { 2276 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2277 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 2278 2279 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2280 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 2281 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 2282 } else { 2283 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 2284 update_spec_ctrl(x86_spec_ctrl_base); 2285 } 2286 } 2287 2288 switch (spectre_v2_enabled) { 2289 case SPECTRE_V2_NONE: 2290 return; 2291 2292 case SPECTRE_V2_EIBRS: 2293 break; 2294 2295 case SPECTRE_V2_IBRS: 2296 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 2297 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 2298 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 2299 break; 2300 2301 case SPECTRE_V2_LFENCE: 2302 case SPECTRE_V2_EIBRS_LFENCE: 2303 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 2304 fallthrough; 2305 2306 case SPECTRE_V2_RETPOLINE: 2307 case SPECTRE_V2_EIBRS_RETPOLINE: 2308 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 2309 break; 2310 } 2311 2312 /* 2313 * Disable alternate RSB predictions in kernel when indirect CALLs and 2314 * JMPs gets protection against BHI and Intramode-BTI, but RET 2315 * prediction from a non-RSB predictor is still a risk. 2316 */ 2317 if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE || 2318 spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE || 2319 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) 2320 spec_ctrl_disable_kernel_rrsba(); 2321 2322 spectre_v2_select_rsb_mitigation(spectre_v2_enabled); 2323 2324 /* 2325 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 2326 * and Enhanced IBRS protect firmware too, so enable IBRS around 2327 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 2328 * otherwise enabled. 2329 * 2330 * Use "spectre_v2_enabled" to check Enhanced IBRS instead of 2331 * boot_cpu_has(), because the user might select retpoline on the kernel 2332 * command line and if the CPU supports Enhanced IBRS, kernel might 2333 * un-intentionally not enable IBRS around firmware calls. 2334 */ 2335 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 2336 boot_cpu_has(X86_FEATURE_IBPB) && 2337 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2338 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 2339 2340 if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) { 2341 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 2342 pr_info("Enabling Speculation Barrier for firmware calls\n"); 2343 } 2344 2345 } else if (boot_cpu_has(X86_FEATURE_IBRS) && 2346 !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2347 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 2348 pr_info("Enabling Restricted Speculation for firmware calls\n"); 2349 } 2350 2351 #ifdef CONFIG_BPF_JIT 2352 if (cpu_wants_ibpb_bpf()) { 2353 static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb); 2354 static_branch_enable(&bpf_pred_flush_enabled); 2355 pr_info("Enabling IBPB for BPF\n"); 2356 } 2357 #endif 2358 } 2359 2360 static void update_stibp_msr(void * __unused) 2361 { 2362 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 2363 update_spec_ctrl(val); 2364 } 2365 2366 /* Update x86_spec_ctrl_base in case SMT state changed. */ 2367 static void update_stibp_strict(void) 2368 { 2369 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 2370 2371 if (sched_smt_active()) 2372 mask |= SPEC_CTRL_STIBP; 2373 2374 if (mask == x86_spec_ctrl_base) 2375 return; 2376 2377 pr_info("Update user space SMT mitigation: STIBP %s\n", 2378 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 2379 x86_spec_ctrl_base = mask; 2380 on_each_cpu(update_stibp_msr, NULL, 1); 2381 } 2382 2383 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 2384 static void update_indir_branch_cond(void) 2385 { 2386 if (sched_smt_active()) 2387 static_branch_enable(&switch_to_cond_stibp); 2388 else 2389 static_branch_disable(&switch_to_cond_stibp); 2390 } 2391 2392 #undef pr_fmt 2393 #define pr_fmt(fmt) fmt 2394 2395 /* Update the static key controlling the MDS CPU buffer clear in idle */ 2396 static void update_mds_branch_idle(void) 2397 { 2398 /* 2399 * Enable the idle clearing if SMT is active on CPUs which are 2400 * affected only by MSBDS and not any other MDS variant. 2401 * 2402 * The other variants cannot be mitigated when SMT is enabled, so 2403 * clearing the buffers on idle just to prevent the Store Buffer 2404 * repartitioning leak would be a window dressing exercise. 2405 */ 2406 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 2407 return; 2408 2409 if (sched_smt_active()) { 2410 static_branch_enable(&cpu_buf_idle_clear); 2411 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 2412 (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) { 2413 static_branch_disable(&cpu_buf_idle_clear); 2414 } 2415 } 2416 2417 #undef pr_fmt 2418 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 2419 2420 static enum ssb_mitigation ssb_mode __ro_after_init = 2421 IS_ENABLED(CONFIG_MITIGATION_SSB) ? SPEC_STORE_BYPASS_AUTO : SPEC_STORE_BYPASS_NONE; 2422 2423 static const char * const ssb_strings[] = { 2424 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 2425 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 2426 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 2427 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 2428 }; 2429 2430 static bool nossb __ro_after_init; 2431 2432 static int __init nossb_parse_cmdline(char *str) 2433 { 2434 nossb = true; 2435 ssb_mode = SPEC_STORE_BYPASS_NONE; 2436 return 0; 2437 } 2438 early_param("nospec_store_bypass_disable", nossb_parse_cmdline); 2439 2440 static int __init ssb_parse_cmdline(char *str) 2441 { 2442 if (!str) 2443 return -EINVAL; 2444 2445 if (nossb) 2446 return 0; 2447 2448 if (!strcmp(str, "auto")) 2449 ssb_mode = SPEC_STORE_BYPASS_AUTO; 2450 else if (!strcmp(str, "on")) 2451 ssb_mode = SPEC_STORE_BYPASS_DISABLE; 2452 else if (!strcmp(str, "off")) 2453 ssb_mode = SPEC_STORE_BYPASS_NONE; 2454 else if (!strcmp(str, "prctl")) 2455 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2456 else if (!strcmp(str, "seccomp")) 2457 ssb_mode = IS_ENABLED(CONFIG_SECCOMP) ? 2458 SPEC_STORE_BYPASS_SECCOMP : SPEC_STORE_BYPASS_PRCTL; 2459 else 2460 pr_err("Ignoring unknown spec_store_bypass_disable option (%s).\n", 2461 str); 2462 2463 return 0; 2464 } 2465 early_param("spec_store_bypass_disable", ssb_parse_cmdline); 2466 2467 static void __init ssb_select_mitigation(void) 2468 { 2469 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) { 2470 ssb_mode = SPEC_STORE_BYPASS_NONE; 2471 return; 2472 } 2473 2474 if (ssb_mode == SPEC_STORE_BYPASS_AUTO) { 2475 if (should_mitigate_vuln(X86_BUG_SPEC_STORE_BYPASS)) 2476 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2477 else 2478 ssb_mode = SPEC_STORE_BYPASS_NONE; 2479 } 2480 2481 if (!boot_cpu_has(X86_FEATURE_SSBD)) 2482 ssb_mode = SPEC_STORE_BYPASS_NONE; 2483 2484 pr_info("%s\n", ssb_strings[ssb_mode]); 2485 } 2486 2487 static void __init ssb_apply_mitigation(void) 2488 { 2489 /* 2490 * We have three CPU feature flags that are in play here: 2491 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 2492 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 2493 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 2494 */ 2495 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) { 2496 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 2497 /* 2498 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 2499 * use a completely different MSR and bit dependent on family. 2500 */ 2501 if (!cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) && 2502 !cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) { 2503 x86_amd_ssb_disable(); 2504 } else { 2505 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 2506 update_spec_ctrl(x86_spec_ctrl_base); 2507 } 2508 } 2509 } 2510 2511 #undef pr_fmt 2512 #define pr_fmt(fmt) "Speculation prctl: " fmt 2513 2514 static void task_update_spec_tif(struct task_struct *tsk) 2515 { 2516 /* Force the update of the real TIF bits */ 2517 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 2518 2519 /* 2520 * Immediately update the speculation control MSRs for the current 2521 * task, but for a non-current task delay setting the CPU 2522 * mitigation until it is scheduled next. 2523 * 2524 * This can only happen for SECCOMP mitigation. For PRCTL it's 2525 * always the current task. 2526 */ 2527 if (tsk == current) 2528 speculation_ctrl_update_current(); 2529 } 2530 2531 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 2532 { 2533 2534 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2535 return -EPERM; 2536 2537 switch (ctrl) { 2538 case PR_SPEC_ENABLE: 2539 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2540 return 0; 2541 case PR_SPEC_DISABLE: 2542 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2543 return 0; 2544 default: 2545 return -ERANGE; 2546 } 2547 } 2548 2549 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2550 { 2551 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2552 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2553 return -ENXIO; 2554 2555 switch (ctrl) { 2556 case PR_SPEC_ENABLE: 2557 /* If speculation is force disabled, enable is not allowed */ 2558 if (task_spec_ssb_force_disable(task)) 2559 return -EPERM; 2560 task_clear_spec_ssb_disable(task); 2561 task_clear_spec_ssb_noexec(task); 2562 task_update_spec_tif(task); 2563 break; 2564 case PR_SPEC_DISABLE: 2565 task_set_spec_ssb_disable(task); 2566 task_clear_spec_ssb_noexec(task); 2567 task_update_spec_tif(task); 2568 break; 2569 case PR_SPEC_FORCE_DISABLE: 2570 task_set_spec_ssb_disable(task); 2571 task_set_spec_ssb_force_disable(task); 2572 task_clear_spec_ssb_noexec(task); 2573 task_update_spec_tif(task); 2574 break; 2575 case PR_SPEC_DISABLE_NOEXEC: 2576 if (task_spec_ssb_force_disable(task)) 2577 return -EPERM; 2578 task_set_spec_ssb_disable(task); 2579 task_set_spec_ssb_noexec(task); 2580 task_update_spec_tif(task); 2581 break; 2582 default: 2583 return -ERANGE; 2584 } 2585 return 0; 2586 } 2587 2588 static bool is_spec_ib_user_controlled(void) 2589 { 2590 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2591 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2592 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2593 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2594 } 2595 2596 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2597 { 2598 switch (ctrl) { 2599 case PR_SPEC_ENABLE: 2600 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2601 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2602 return 0; 2603 2604 /* 2605 * With strict mode for both IBPB and STIBP, the instruction 2606 * code paths avoid checking this task flag and instead, 2607 * unconditionally run the instruction. However, STIBP and IBPB 2608 * are independent and either can be set to conditionally 2609 * enabled regardless of the mode of the other. 2610 * 2611 * If either is set to conditional, allow the task flag to be 2612 * updated, unless it was force-disabled by a previous prctl 2613 * call. Currently, this is possible on an AMD CPU which has the 2614 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2615 * kernel is booted with 'spectre_v2_user=seccomp', then 2616 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2617 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2618 */ 2619 if (!is_spec_ib_user_controlled() || 2620 task_spec_ib_force_disable(task)) 2621 return -EPERM; 2622 2623 task_clear_spec_ib_disable(task); 2624 task_update_spec_tif(task); 2625 break; 2626 case PR_SPEC_DISABLE: 2627 case PR_SPEC_FORCE_DISABLE: 2628 /* 2629 * Indirect branch speculation is always allowed when 2630 * mitigation is force disabled. 2631 */ 2632 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2633 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2634 return -EPERM; 2635 2636 if (!is_spec_ib_user_controlled()) 2637 return 0; 2638 2639 task_set_spec_ib_disable(task); 2640 if (ctrl == PR_SPEC_FORCE_DISABLE) 2641 task_set_spec_ib_force_disable(task); 2642 task_update_spec_tif(task); 2643 if (task == current) 2644 indirect_branch_prediction_barrier(); 2645 break; 2646 default: 2647 return -ERANGE; 2648 } 2649 return 0; 2650 } 2651 2652 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2653 unsigned long ctrl) 2654 { 2655 switch (which) { 2656 case PR_SPEC_STORE_BYPASS: 2657 return ssb_prctl_set(task, ctrl); 2658 case PR_SPEC_INDIRECT_BRANCH: 2659 return ib_prctl_set(task, ctrl); 2660 case PR_SPEC_L1D_FLUSH: 2661 return l1d_flush_prctl_set(task, ctrl); 2662 default: 2663 return -ENODEV; 2664 } 2665 } 2666 2667 #ifdef CONFIG_SECCOMP 2668 void arch_seccomp_spec_mitigate(struct task_struct *task) 2669 { 2670 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2671 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2672 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2673 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2674 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2675 } 2676 #endif 2677 2678 static int l1d_flush_prctl_get(struct task_struct *task) 2679 { 2680 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2681 return PR_SPEC_FORCE_DISABLE; 2682 2683 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2684 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2685 else 2686 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2687 } 2688 2689 static int ssb_prctl_get(struct task_struct *task) 2690 { 2691 switch (ssb_mode) { 2692 case SPEC_STORE_BYPASS_NONE: 2693 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2694 return PR_SPEC_ENABLE; 2695 return PR_SPEC_NOT_AFFECTED; 2696 case SPEC_STORE_BYPASS_DISABLE: 2697 return PR_SPEC_DISABLE; 2698 case SPEC_STORE_BYPASS_SECCOMP: 2699 case SPEC_STORE_BYPASS_PRCTL: 2700 case SPEC_STORE_BYPASS_AUTO: 2701 if (task_spec_ssb_force_disable(task)) 2702 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2703 if (task_spec_ssb_noexec(task)) 2704 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2705 if (task_spec_ssb_disable(task)) 2706 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2707 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2708 } 2709 BUG(); 2710 } 2711 2712 static int ib_prctl_get(struct task_struct *task) 2713 { 2714 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2715 return PR_SPEC_NOT_AFFECTED; 2716 2717 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2718 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2719 return PR_SPEC_ENABLE; 2720 else if (is_spec_ib_user_controlled()) { 2721 if (task_spec_ib_force_disable(task)) 2722 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2723 if (task_spec_ib_disable(task)) 2724 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2725 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2726 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2727 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2728 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2729 return PR_SPEC_DISABLE; 2730 else 2731 return PR_SPEC_NOT_AFFECTED; 2732 } 2733 2734 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2735 { 2736 switch (which) { 2737 case PR_SPEC_STORE_BYPASS: 2738 return ssb_prctl_get(task); 2739 case PR_SPEC_INDIRECT_BRANCH: 2740 return ib_prctl_get(task); 2741 case PR_SPEC_L1D_FLUSH: 2742 return l1d_flush_prctl_get(task); 2743 default: 2744 return -ENODEV; 2745 } 2746 } 2747 2748 void x86_spec_ctrl_setup_ap(void) 2749 { 2750 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2751 update_spec_ctrl(x86_spec_ctrl_base); 2752 2753 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2754 x86_amd_ssb_disable(); 2755 } 2756 2757 bool itlb_multihit_kvm_mitigation; 2758 EXPORT_SYMBOL_FOR_KVM(itlb_multihit_kvm_mitigation); 2759 2760 #undef pr_fmt 2761 #define pr_fmt(fmt) "L1TF: " fmt 2762 2763 /* Default mitigation for L1TF-affected CPUs */ 2764 enum l1tf_mitigations l1tf_mitigation __ro_after_init = 2765 IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_AUTO : L1TF_MITIGATION_OFF; 2766 EXPORT_SYMBOL_FOR_KVM(l1tf_mitigation); 2767 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2768 EXPORT_SYMBOL_FOR_KVM(l1tf_vmx_mitigation); 2769 2770 /* 2771 * These CPUs all support 44bits physical address space internally in the 2772 * cache but CPUID can report a smaller number of physical address bits. 2773 * 2774 * The L1TF mitigation uses the top most address bit for the inversion of 2775 * non present PTEs. When the installed memory reaches into the top most 2776 * address bit due to memory holes, which has been observed on machines 2777 * which report 36bits physical address bits and have 32G RAM installed, 2778 * then the mitigation range check in l1tf_select_mitigation() triggers. 2779 * This is a false positive because the mitigation is still possible due to 2780 * the fact that the cache uses 44bit internally. Use the cache bits 2781 * instead of the reported physical bits and adjust them on the affected 2782 * machines to 44bit if the reported bits are less than 44. 2783 */ 2784 static void override_cache_bits(struct cpuinfo_x86 *c) 2785 { 2786 if (c->x86 != 6) 2787 return; 2788 2789 switch (c->x86_vfm) { 2790 case INTEL_NEHALEM: 2791 case INTEL_WESTMERE: 2792 case INTEL_SANDYBRIDGE: 2793 case INTEL_IVYBRIDGE: 2794 case INTEL_HASWELL: 2795 case INTEL_HASWELL_L: 2796 case INTEL_HASWELL_G: 2797 case INTEL_BROADWELL: 2798 case INTEL_BROADWELL_G: 2799 case INTEL_SKYLAKE_L: 2800 case INTEL_SKYLAKE: 2801 case INTEL_KABYLAKE_L: 2802 case INTEL_KABYLAKE: 2803 if (c->x86_cache_bits < 44) 2804 c->x86_cache_bits = 44; 2805 break; 2806 } 2807 } 2808 2809 static void __init l1tf_select_mitigation(void) 2810 { 2811 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 2812 l1tf_mitigation = L1TF_MITIGATION_OFF; 2813 return; 2814 } 2815 2816 if (l1tf_mitigation != L1TF_MITIGATION_AUTO) 2817 return; 2818 2819 if (!should_mitigate_vuln(X86_BUG_L1TF)) { 2820 l1tf_mitigation = L1TF_MITIGATION_OFF; 2821 return; 2822 } 2823 2824 if (smt_mitigations == SMT_MITIGATIONS_ON) 2825 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2826 else 2827 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2828 } 2829 2830 static void __init l1tf_apply_mitigation(void) 2831 { 2832 u64 half_pa; 2833 2834 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2835 return; 2836 2837 override_cache_bits(&boot_cpu_data); 2838 2839 switch (l1tf_mitigation) { 2840 case L1TF_MITIGATION_OFF: 2841 case L1TF_MITIGATION_FLUSH_NOWARN: 2842 case L1TF_MITIGATION_FLUSH: 2843 case L1TF_MITIGATION_AUTO: 2844 break; 2845 case L1TF_MITIGATION_FLUSH_NOSMT: 2846 case L1TF_MITIGATION_FULL: 2847 cpu_smt_disable(false); 2848 break; 2849 case L1TF_MITIGATION_FULL_FORCE: 2850 cpu_smt_disable(true); 2851 break; 2852 } 2853 2854 #if CONFIG_PGTABLE_LEVELS == 2 2855 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2856 return; 2857 #endif 2858 2859 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2860 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2861 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2862 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2863 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2864 half_pa); 2865 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2866 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2867 return; 2868 } 2869 2870 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2871 } 2872 2873 static int __init l1tf_cmdline(char *str) 2874 { 2875 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2876 return 0; 2877 2878 if (!str) 2879 return -EINVAL; 2880 2881 if (!strcmp(str, "off")) 2882 l1tf_mitigation = L1TF_MITIGATION_OFF; 2883 else if (!strcmp(str, "flush,nowarn")) 2884 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2885 else if (!strcmp(str, "flush")) 2886 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2887 else if (!strcmp(str, "flush,nosmt")) 2888 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2889 else if (!strcmp(str, "full")) 2890 l1tf_mitigation = L1TF_MITIGATION_FULL; 2891 else if (!strcmp(str, "full,force")) 2892 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2893 2894 return 0; 2895 } 2896 early_param("l1tf", l1tf_cmdline); 2897 2898 #undef pr_fmt 2899 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 2900 2901 static const char * const srso_strings[] = { 2902 [SRSO_MITIGATION_NONE] = "Vulnerable", 2903 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 2904 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 2905 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 2906 [SRSO_MITIGATION_NOSMT] = "Mitigation: SMT disabled", 2907 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 2908 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 2909 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only", 2910 [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation", 2911 [SRSO_MITIGATION_USER_IBPB] = "Mitigation: IBPB on context switch", 2912 }; 2913 2914 static int __init srso_parse_cmdline(char *str) 2915 { 2916 if (!str) 2917 return -EINVAL; 2918 2919 if (!strcmp(str, "off")) 2920 srso_mitigation = SRSO_MITIGATION_NONE; 2921 else if (!strcmp(str, "microcode")) 2922 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2923 else if (!strcmp(str, "safe-ret")) 2924 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2925 else if (!strcmp(str, "ibpb")) 2926 srso_mitigation = SRSO_MITIGATION_IBPB; 2927 else if (!strcmp(str, "ibpb-vmexit")) 2928 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2929 else 2930 pr_err("Ignoring unknown SRSO option (%s).", str); 2931 2932 return 0; 2933 } 2934 early_param("spec_rstack_overflow", srso_parse_cmdline); 2935 2936 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 2937 2938 static void __init srso_select_mitigation(void) 2939 { 2940 if (!boot_cpu_has_bug(X86_BUG_SRSO)) { 2941 srso_mitigation = SRSO_MITIGATION_NONE; 2942 return; 2943 } 2944 2945 if (srso_mitigation == SRSO_MITIGATION_AUTO) { 2946 /* 2947 * Use safe-RET if user->kernel or guest->host protection is 2948 * required. Otherwise the 'microcode' mitigation is sufficient 2949 * to protect the user->user and guest->guest vectors. 2950 */ 2951 if ((cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) && 2952 !boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION)) 2953 || 2954 (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) && 2955 !boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))) { 2956 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2957 } else if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 2958 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) { 2959 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2960 } else { 2961 srso_mitigation = SRSO_MITIGATION_NONE; 2962 return; 2963 } 2964 } 2965 2966 /* Zen1/2 with SMT off aren't vulnerable to SRSO. */ 2967 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 2968 srso_mitigation = SRSO_MITIGATION_NOSMT; 2969 return; 2970 } 2971 2972 if (!boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) { 2973 pr_warn("IBPB-extending microcode not applied!\n"); 2974 pr_warn(SRSO_NOTICE); 2975 2976 /* 2977 * Safe-RET provides partial mitigation without microcode, but 2978 * other mitigations require microcode to provide any 2979 * mitigations. 2980 */ 2981 if (srso_mitigation == SRSO_MITIGATION_SAFE_RET) 2982 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 2983 else 2984 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 2985 } 2986 2987 switch (srso_mitigation) { 2988 case SRSO_MITIGATION_SAFE_RET: 2989 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 2990 if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) { 2991 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2992 goto ibpb_on_vmexit; 2993 } 2994 2995 if (!IS_ENABLED(CONFIG_MITIGATION_SRSO)) { 2996 pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n"); 2997 srso_mitigation = SRSO_MITIGATION_NONE; 2998 } 2999 break; 3000 ibpb_on_vmexit: 3001 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3002 if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) { 3003 pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n"); 3004 srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE; 3005 break; 3006 } 3007 fallthrough; 3008 case SRSO_MITIGATION_IBPB: 3009 if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 3010 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 3011 srso_mitigation = SRSO_MITIGATION_NONE; 3012 } 3013 break; 3014 default: 3015 break; 3016 } 3017 } 3018 3019 static void __init srso_update_mitigation(void) 3020 { 3021 if (!boot_cpu_has_bug(X86_BUG_SRSO)) 3022 return; 3023 3024 /* If retbleed is using IBPB, that works for SRSO as well */ 3025 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB && 3026 boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) 3027 srso_mitigation = SRSO_MITIGATION_IBPB; 3028 3029 /* 3030 * See if IBPB on context switch is the only thing needed to address 3031 * GUEST/GUEST and USER/USER vectors. 3032 */ 3033 if (srso_mitigation == SRSO_MITIGATION_MICROCODE && 3034 boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO) && 3035 boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION) && 3036 spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) 3037 srso_mitigation = SRSO_MITIGATION_USER_IBPB; 3038 3039 pr_info("%s\n", srso_strings[srso_mitigation]); 3040 } 3041 3042 static void __init srso_apply_mitigation(void) 3043 { 3044 /* 3045 * Clear the feature flag if this mitigation is not selected as that 3046 * feature flag controls the BpSpecReduce MSR bit toggling in KVM. 3047 */ 3048 if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE) 3049 setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE); 3050 3051 if (srso_mitigation == SRSO_MITIGATION_NONE) { 3052 if (boot_cpu_has(X86_FEATURE_SBPB)) 3053 x86_pred_cmd = PRED_CMD_SBPB; 3054 return; 3055 } 3056 3057 switch (srso_mitigation) { 3058 case SRSO_MITIGATION_SAFE_RET: 3059 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 3060 /* 3061 * Enable the return thunk for generated code 3062 * like ftrace, static_call, etc. 3063 */ 3064 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 3065 setup_force_cpu_cap(X86_FEATURE_UNRET); 3066 3067 if (boot_cpu_data.x86 == 0x19) { 3068 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 3069 set_return_thunk(srso_alias_return_thunk); 3070 } else { 3071 setup_force_cpu_cap(X86_FEATURE_SRSO); 3072 set_return_thunk(srso_return_thunk); 3073 } 3074 break; 3075 case SRSO_MITIGATION_IBPB: 3076 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 3077 /* 3078 * IBPB on entry already obviates the need for 3079 * software-based untraining so clear those in case some 3080 * other mitigation like Retbleed has selected them. 3081 */ 3082 setup_clear_cpu_cap(X86_FEATURE_UNRET); 3083 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 3084 fallthrough; 3085 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3086 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 3087 /* 3088 * There is no need for RSB filling: entry_ibpb() ensures 3089 * all predictions, including the RSB, are invalidated, 3090 * regardless of IBPB implementation. 3091 */ 3092 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 3093 break; 3094 default: 3095 break; 3096 } 3097 } 3098 3099 #undef pr_fmt 3100 #define pr_fmt(fmt) "VMSCAPE: " fmt 3101 3102 enum vmscape_mitigations { 3103 VMSCAPE_MITIGATION_NONE, 3104 VMSCAPE_MITIGATION_AUTO, 3105 VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER, 3106 VMSCAPE_MITIGATION_IBPB_ON_VMEXIT, 3107 }; 3108 3109 static const char * const vmscape_strings[] = { 3110 [VMSCAPE_MITIGATION_NONE] = "Vulnerable", 3111 /* [VMSCAPE_MITIGATION_AUTO] */ 3112 [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER] = "Mitigation: IBPB before exit to userspace", 3113 [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT", 3114 }; 3115 3116 static enum vmscape_mitigations vmscape_mitigation __ro_after_init = 3117 IS_ENABLED(CONFIG_MITIGATION_VMSCAPE) ? VMSCAPE_MITIGATION_AUTO : VMSCAPE_MITIGATION_NONE; 3118 3119 static int __init vmscape_parse_cmdline(char *str) 3120 { 3121 if (!str) 3122 return -EINVAL; 3123 3124 if (!strcmp(str, "off")) { 3125 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3126 } else if (!strcmp(str, "ibpb")) { 3127 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; 3128 } else if (!strcmp(str, "force")) { 3129 setup_force_cpu_bug(X86_BUG_VMSCAPE); 3130 vmscape_mitigation = VMSCAPE_MITIGATION_AUTO; 3131 } else { 3132 pr_err("Ignoring unknown vmscape=%s option.\n", str); 3133 } 3134 3135 return 0; 3136 } 3137 early_param("vmscape", vmscape_parse_cmdline); 3138 3139 static void __init vmscape_select_mitigation(void) 3140 { 3141 if (!boot_cpu_has_bug(X86_BUG_VMSCAPE) || 3142 !boot_cpu_has(X86_FEATURE_IBPB)) { 3143 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3144 return; 3145 } 3146 3147 if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO) { 3148 if (should_mitigate_vuln(X86_BUG_VMSCAPE)) 3149 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; 3150 else 3151 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3152 } 3153 } 3154 3155 static void __init vmscape_update_mitigation(void) 3156 { 3157 if (!boot_cpu_has_bug(X86_BUG_VMSCAPE)) 3158 return; 3159 3160 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB || 3161 srso_mitigation == SRSO_MITIGATION_IBPB_ON_VMEXIT) 3162 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_ON_VMEXIT; 3163 3164 pr_info("%s\n", vmscape_strings[vmscape_mitigation]); 3165 } 3166 3167 static void __init vmscape_apply_mitigation(void) 3168 { 3169 if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER) 3170 setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER); 3171 } 3172 3173 #undef pr_fmt 3174 #define pr_fmt(fmt) fmt 3175 3176 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 3177 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 3178 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 3179 #define VMSCAPE_MSG_SMT "VMSCAPE: SMT on, STIBP is required for full protection. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/vmscape.html for more details.\n" 3180 3181 void cpu_bugs_smt_update(void) 3182 { 3183 mutex_lock(&spec_ctrl_mutex); 3184 3185 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3186 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3187 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 3188 3189 switch (spectre_v2_user_stibp) { 3190 case SPECTRE_V2_USER_NONE: 3191 break; 3192 case SPECTRE_V2_USER_STRICT: 3193 case SPECTRE_V2_USER_STRICT_PREFERRED: 3194 update_stibp_strict(); 3195 break; 3196 case SPECTRE_V2_USER_PRCTL: 3197 case SPECTRE_V2_USER_SECCOMP: 3198 update_indir_branch_cond(); 3199 break; 3200 } 3201 3202 switch (mds_mitigation) { 3203 case MDS_MITIGATION_FULL: 3204 case MDS_MITIGATION_AUTO: 3205 case MDS_MITIGATION_VMWERV: 3206 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 3207 pr_warn_once(MDS_MSG_SMT); 3208 update_mds_branch_idle(); 3209 break; 3210 case MDS_MITIGATION_OFF: 3211 break; 3212 } 3213 3214 switch (taa_mitigation) { 3215 case TAA_MITIGATION_VERW: 3216 case TAA_MITIGATION_AUTO: 3217 case TAA_MITIGATION_UCODE_NEEDED: 3218 if (sched_smt_active()) 3219 pr_warn_once(TAA_MSG_SMT); 3220 break; 3221 case TAA_MITIGATION_TSX_DISABLED: 3222 case TAA_MITIGATION_OFF: 3223 break; 3224 } 3225 3226 switch (mmio_mitigation) { 3227 case MMIO_MITIGATION_VERW: 3228 case MMIO_MITIGATION_AUTO: 3229 case MMIO_MITIGATION_UCODE_NEEDED: 3230 if (sched_smt_active()) 3231 pr_warn_once(MMIO_MSG_SMT); 3232 break; 3233 case MMIO_MITIGATION_OFF: 3234 break; 3235 } 3236 3237 switch (tsa_mitigation) { 3238 case TSA_MITIGATION_USER_KERNEL: 3239 case TSA_MITIGATION_VM: 3240 case TSA_MITIGATION_AUTO: 3241 case TSA_MITIGATION_FULL: 3242 /* 3243 * TSA-SQ can potentially lead to info leakage between 3244 * SMT threads. 3245 */ 3246 if (sched_smt_active()) 3247 static_branch_enable(&cpu_buf_idle_clear); 3248 else 3249 static_branch_disable(&cpu_buf_idle_clear); 3250 break; 3251 case TSA_MITIGATION_NONE: 3252 case TSA_MITIGATION_UCODE_NEEDED: 3253 break; 3254 } 3255 3256 switch (vmscape_mitigation) { 3257 case VMSCAPE_MITIGATION_NONE: 3258 case VMSCAPE_MITIGATION_AUTO: 3259 break; 3260 case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT: 3261 case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER: 3262 /* 3263 * Hypervisors can be attacked across-threads, warn for SMT when 3264 * STIBP is not already enabled system-wide. 3265 * 3266 * Intel eIBRS (!AUTOIBRS) implies STIBP on. 3267 */ 3268 if (!sched_smt_active() || 3269 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3270 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED || 3271 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3272 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) 3273 break; 3274 pr_warn_once(VMSCAPE_MSG_SMT); 3275 break; 3276 } 3277 3278 mutex_unlock(&spec_ctrl_mutex); 3279 } 3280 3281 void __init cpu_select_mitigations(void) 3282 { 3283 /* 3284 * Read the SPEC_CTRL MSR to account for reserved bits which may 3285 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 3286 * init code as it is not enumerated and depends on the family. 3287 */ 3288 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 3289 rdmsrq(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 3290 3291 /* 3292 * Previously running kernel (kexec), may have some controls 3293 * turned ON. Clear them and let the mitigations setup below 3294 * rediscover them based on configuration. 3295 */ 3296 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 3297 } 3298 3299 x86_arch_cap_msr = x86_read_arch_cap_msr(); 3300 3301 cpu_print_attack_vectors(); 3302 3303 /* Select the proper CPU mitigations before patching alternatives: */ 3304 spectre_v1_select_mitigation(); 3305 spectre_v2_select_mitigation(); 3306 retbleed_select_mitigation(); 3307 spectre_v2_user_select_mitigation(); 3308 ssb_select_mitigation(); 3309 l1tf_select_mitigation(); 3310 mds_select_mitigation(); 3311 taa_select_mitigation(); 3312 mmio_select_mitigation(); 3313 rfds_select_mitigation(); 3314 srbds_select_mitigation(); 3315 l1d_flush_select_mitigation(); 3316 srso_select_mitigation(); 3317 gds_select_mitigation(); 3318 its_select_mitigation(); 3319 bhi_select_mitigation(); 3320 tsa_select_mitigation(); 3321 vmscape_select_mitigation(); 3322 3323 /* 3324 * After mitigations are selected, some may need to update their 3325 * choices. 3326 */ 3327 spectre_v2_update_mitigation(); 3328 /* 3329 * retbleed_update_mitigation() relies on the state set by 3330 * spectre_v2_update_mitigation(); specifically it wants to know about 3331 * spectre_v2=ibrs. 3332 */ 3333 retbleed_update_mitigation(); 3334 /* 3335 * its_update_mitigation() depends on spectre_v2_update_mitigation() 3336 * and retbleed_update_mitigation(). 3337 */ 3338 its_update_mitigation(); 3339 3340 /* 3341 * spectre_v2_user_update_mitigation() depends on 3342 * retbleed_update_mitigation(), specifically the STIBP 3343 * selection is forced for UNRET or IBPB. 3344 */ 3345 spectre_v2_user_update_mitigation(); 3346 mds_update_mitigation(); 3347 taa_update_mitigation(); 3348 mmio_update_mitigation(); 3349 rfds_update_mitigation(); 3350 bhi_update_mitigation(); 3351 /* srso_update_mitigation() depends on retbleed_update_mitigation(). */ 3352 srso_update_mitigation(); 3353 vmscape_update_mitigation(); 3354 3355 spectre_v1_apply_mitigation(); 3356 spectre_v2_apply_mitigation(); 3357 retbleed_apply_mitigation(); 3358 spectre_v2_user_apply_mitigation(); 3359 ssb_apply_mitigation(); 3360 l1tf_apply_mitigation(); 3361 mds_apply_mitigation(); 3362 taa_apply_mitigation(); 3363 mmio_apply_mitigation(); 3364 rfds_apply_mitigation(); 3365 srbds_apply_mitigation(); 3366 srso_apply_mitigation(); 3367 gds_apply_mitigation(); 3368 its_apply_mitigation(); 3369 bhi_apply_mitigation(); 3370 tsa_apply_mitigation(); 3371 vmscape_apply_mitigation(); 3372 } 3373 3374 #ifdef CONFIG_SYSFS 3375 3376 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 3377 3378 #if IS_ENABLED(CONFIG_KVM_INTEL) 3379 static const char * const l1tf_vmx_states[] = { 3380 [VMENTER_L1D_FLUSH_AUTO] = "auto", 3381 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 3382 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 3383 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 3384 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 3385 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 3386 }; 3387 3388 static ssize_t l1tf_show_state(char *buf) 3389 { 3390 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 3391 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3392 3393 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 3394 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 3395 sched_smt_active())) { 3396 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 3397 l1tf_vmx_states[l1tf_vmx_mitigation]); 3398 } 3399 3400 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 3401 l1tf_vmx_states[l1tf_vmx_mitigation], 3402 sched_smt_active() ? "vulnerable" : "disabled"); 3403 } 3404 3405 static ssize_t itlb_multihit_show_state(char *buf) 3406 { 3407 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 3408 !boot_cpu_has(X86_FEATURE_VMX)) 3409 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 3410 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 3411 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 3412 else if (itlb_multihit_kvm_mitigation) 3413 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 3414 else 3415 return sysfs_emit(buf, "KVM: Vulnerable\n"); 3416 } 3417 #else 3418 static ssize_t l1tf_show_state(char *buf) 3419 { 3420 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3421 } 3422 3423 static ssize_t itlb_multihit_show_state(char *buf) 3424 { 3425 return sysfs_emit(buf, "Processor vulnerable\n"); 3426 } 3427 #endif 3428 3429 static ssize_t mds_show_state(char *buf) 3430 { 3431 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3432 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3433 mds_strings[mds_mitigation]); 3434 } 3435 3436 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 3437 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3438 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 3439 sched_smt_active() ? "mitigated" : "disabled")); 3440 } 3441 3442 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3443 sched_smt_active() ? "vulnerable" : "disabled"); 3444 } 3445 3446 static ssize_t tsx_async_abort_show_state(char *buf) 3447 { 3448 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 3449 (taa_mitigation == TAA_MITIGATION_OFF)) 3450 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 3451 3452 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3453 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3454 taa_strings[taa_mitigation]); 3455 } 3456 3457 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 3458 sched_smt_active() ? "vulnerable" : "disabled"); 3459 } 3460 3461 static ssize_t mmio_stale_data_show_state(char *buf) 3462 { 3463 if (mmio_mitigation == MMIO_MITIGATION_OFF) 3464 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 3465 3466 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3467 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3468 mmio_strings[mmio_mitigation]); 3469 } 3470 3471 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 3472 sched_smt_active() ? "vulnerable" : "disabled"); 3473 } 3474 3475 static ssize_t rfds_show_state(char *buf) 3476 { 3477 return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); 3478 } 3479 3480 static ssize_t old_microcode_show_state(char *buf) 3481 { 3482 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 3483 return sysfs_emit(buf, "Unknown: running under hypervisor"); 3484 3485 return sysfs_emit(buf, "Vulnerable\n"); 3486 } 3487 3488 static ssize_t its_show_state(char *buf) 3489 { 3490 return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]); 3491 } 3492 3493 static char *stibp_state(void) 3494 { 3495 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3496 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 3497 return ""; 3498 3499 switch (spectre_v2_user_stibp) { 3500 case SPECTRE_V2_USER_NONE: 3501 return "; STIBP: disabled"; 3502 case SPECTRE_V2_USER_STRICT: 3503 return "; STIBP: forced"; 3504 case SPECTRE_V2_USER_STRICT_PREFERRED: 3505 return "; STIBP: always-on"; 3506 case SPECTRE_V2_USER_PRCTL: 3507 case SPECTRE_V2_USER_SECCOMP: 3508 if (static_key_enabled(&switch_to_cond_stibp)) 3509 return "; STIBP: conditional"; 3510 } 3511 return ""; 3512 } 3513 3514 static char *ibpb_state(void) 3515 { 3516 if (boot_cpu_has(X86_FEATURE_IBPB)) { 3517 if (static_key_enabled(&switch_mm_always_ibpb)) 3518 return "; IBPB: always-on"; 3519 if (static_key_enabled(&switch_mm_cond_ibpb)) 3520 return "; IBPB: conditional"; 3521 return "; IBPB: disabled"; 3522 } 3523 return ""; 3524 } 3525 3526 static char *pbrsb_eibrs_state(void) 3527 { 3528 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 3529 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 3530 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 3531 return "; PBRSB-eIBRS: SW sequence"; 3532 else 3533 return "; PBRSB-eIBRS: Vulnerable"; 3534 } else { 3535 return "; PBRSB-eIBRS: Not affected"; 3536 } 3537 } 3538 3539 static const char *spectre_bhi_state(void) 3540 { 3541 if (!boot_cpu_has_bug(X86_BUG_BHI)) 3542 return "; BHI: Not affected"; 3543 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW)) 3544 return "; BHI: BHI_DIS_S"; 3545 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) 3546 return "; BHI: SW loop, KVM: SW loop"; 3547 else if (retpoline_seq_enabled() && rrsba_disabled) 3548 return "; BHI: Retpoline"; 3549 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) 3550 return "; BHI: Vulnerable, KVM: SW loop"; 3551 3552 return "; BHI: Vulnerable"; 3553 } 3554 3555 static ssize_t spectre_v2_show_state(char *buf) 3556 { 3557 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 3558 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 3559 3560 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3561 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3562 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 3563 3564 return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n", 3565 spectre_v2_strings[spectre_v2_enabled], 3566 ibpb_state(), 3567 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "", 3568 stibp_state(), 3569 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "", 3570 pbrsb_eibrs_state(), 3571 spectre_bhi_state(), 3572 /* this should always be at the end */ 3573 spectre_v2_module_string()); 3574 } 3575 3576 static ssize_t srbds_show_state(char *buf) 3577 { 3578 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 3579 } 3580 3581 static ssize_t retbleed_show_state(char *buf) 3582 { 3583 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 3584 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 3585 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 3586 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 3587 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 3588 3589 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 3590 !sched_smt_active() ? "disabled" : 3591 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3592 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 3593 "enabled with STIBP protection" : "vulnerable"); 3594 } 3595 3596 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 3597 } 3598 3599 static ssize_t srso_show_state(char *buf) 3600 { 3601 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 3602 } 3603 3604 static ssize_t gds_show_state(char *buf) 3605 { 3606 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 3607 } 3608 3609 static ssize_t tsa_show_state(char *buf) 3610 { 3611 return sysfs_emit(buf, "%s\n", tsa_strings[tsa_mitigation]); 3612 } 3613 3614 static ssize_t vmscape_show_state(char *buf) 3615 { 3616 return sysfs_emit(buf, "%s\n", vmscape_strings[vmscape_mitigation]); 3617 } 3618 3619 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 3620 char *buf, unsigned int bug) 3621 { 3622 if (!boot_cpu_has_bug(bug)) 3623 return sysfs_emit(buf, "Not affected\n"); 3624 3625 switch (bug) { 3626 case X86_BUG_CPU_MELTDOWN: 3627 if (boot_cpu_has(X86_FEATURE_PTI)) 3628 return sysfs_emit(buf, "Mitigation: PTI\n"); 3629 3630 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 3631 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 3632 3633 break; 3634 3635 case X86_BUG_SPECTRE_V1: 3636 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 3637 3638 case X86_BUG_SPECTRE_V2: 3639 return spectre_v2_show_state(buf); 3640 3641 case X86_BUG_SPEC_STORE_BYPASS: 3642 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 3643 3644 case X86_BUG_L1TF: 3645 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 3646 return l1tf_show_state(buf); 3647 break; 3648 3649 case X86_BUG_MDS: 3650 return mds_show_state(buf); 3651 3652 case X86_BUG_TAA: 3653 return tsx_async_abort_show_state(buf); 3654 3655 case X86_BUG_ITLB_MULTIHIT: 3656 return itlb_multihit_show_state(buf); 3657 3658 case X86_BUG_SRBDS: 3659 return srbds_show_state(buf); 3660 3661 case X86_BUG_MMIO_STALE_DATA: 3662 return mmio_stale_data_show_state(buf); 3663 3664 case X86_BUG_RETBLEED: 3665 return retbleed_show_state(buf); 3666 3667 case X86_BUG_SRSO: 3668 return srso_show_state(buf); 3669 3670 case X86_BUG_GDS: 3671 return gds_show_state(buf); 3672 3673 case X86_BUG_RFDS: 3674 return rfds_show_state(buf); 3675 3676 case X86_BUG_OLD_MICROCODE: 3677 return old_microcode_show_state(buf); 3678 3679 case X86_BUG_ITS: 3680 return its_show_state(buf); 3681 3682 case X86_BUG_TSA: 3683 return tsa_show_state(buf); 3684 3685 case X86_BUG_VMSCAPE: 3686 return vmscape_show_state(buf); 3687 3688 default: 3689 break; 3690 } 3691 3692 return sysfs_emit(buf, "Vulnerable\n"); 3693 } 3694 3695 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 3696 { 3697 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 3698 } 3699 3700 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 3701 { 3702 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 3703 } 3704 3705 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 3706 { 3707 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 3708 } 3709 3710 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 3711 { 3712 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 3713 } 3714 3715 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 3716 { 3717 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 3718 } 3719 3720 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 3721 { 3722 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 3723 } 3724 3725 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 3726 { 3727 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 3728 } 3729 3730 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 3731 { 3732 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 3733 } 3734 3735 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 3736 { 3737 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 3738 } 3739 3740 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 3741 { 3742 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 3743 } 3744 3745 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 3746 { 3747 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 3748 } 3749 3750 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 3751 { 3752 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 3753 } 3754 3755 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 3756 { 3757 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 3758 } 3759 3760 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf) 3761 { 3762 return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); 3763 } 3764 3765 ssize_t cpu_show_old_microcode(struct device *dev, struct device_attribute *attr, char *buf) 3766 { 3767 return cpu_show_common(dev, attr, buf, X86_BUG_OLD_MICROCODE); 3768 } 3769 3770 ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf) 3771 { 3772 return cpu_show_common(dev, attr, buf, X86_BUG_ITS); 3773 } 3774 3775 ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf) 3776 { 3777 return cpu_show_common(dev, attr, buf, X86_BUG_TSA); 3778 } 3779 3780 ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf) 3781 { 3782 return cpu_show_common(dev, attr, buf, X86_BUG_VMSCAPE); 3783 } 3784 #endif 3785 3786 void __warn_thunk(void) 3787 { 3788 WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); 3789 } 3790 3791 #ifdef CONFIG_MITIGATION_SRSO 3792 /* 3793 * Called during exception/interrupt entry if interrupted during the 3794 * safe-RET sequence. The safe-RET sequence consists of 3 instructions: 3795 * 3796 * CALL 3797 * LEA 8(%RSP), %RSP 3798 * RET 3799 * 3800 * An interrupt after the CALL or after the LEA could potentially lead 3801 * to branch predictor poisoning and results in the sequence not being 3802 * able to be safely resumed. 3803 * 3804 * Therefore, modify the regs state as if the remaining part of the 3805 * safe-RET sequence executed so the interrupt returns back to the 3806 * desired return target, instead of the to the safe-RET sequence. 3807 */ 3808 void noinstr handle_interrupted_saferet(struct pt_regs *regs) 3809 { 3810 unsigned long rip = regs->ip; 3811 3812 if (rip == (unsigned long) srso_safe_ret || 3813 rip == (unsigned long) srso_alias_safe_ret) { 3814 /* Modify stack pointer as if LEA executed: */ 3815 regs->sp += 8; 3816 } 3817 3818 /* 3819 * Adjust registers as if RET executed: 3820 * 3821 * 1. Read the return address off the stack and into rIP: 3822 */ 3823 regs->ip = *(unsigned long *)(regs->sp); 3824 3825 /* 2. Pop rIP off the stack: */ 3826 regs->sp += 8; 3827 } 3828 #endif /* CONFIG_MITIGATION_SRSO */ 3829