1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 #include <linux/kvm_types.h> 20 21 #include <asm/spec-ctrl.h> 22 #include <asm/cmdline.h> 23 #include <asm/bugs.h> 24 #include <asm/processor.h> 25 #include <asm/processor-flags.h> 26 #include <asm/fpu/api.h> 27 #include <asm/msr.h> 28 #include <asm/vmx.h> 29 #include <asm/paravirt.h> 30 #include <asm/cpu_device_id.h> 31 #include <asm/e820/api.h> 32 #include <asm/hypervisor.h> 33 #include <asm/tlbflush.h> 34 #include <asm/cpu.h> 35 36 #include "cpu.h" 37 38 /* 39 * Speculation Vulnerability Handling 40 * 41 * Each vulnerability is handled with the following functions: 42 * <vuln>_select_mitigation() -- Selects a mitigation to use. This should 43 * take into account all relevant command line 44 * options. 45 * <vuln>_update_mitigation() -- This is called after all vulnerabilities have 46 * selected a mitigation, in case the selection 47 * may want to change based on other choices 48 * made. This function is optional. 49 * <vuln>_apply_mitigation() -- Enable the selected mitigation. 50 * 51 * The compile-time mitigation in all cases should be AUTO. An explicit 52 * command-line option can override AUTO. If no such option is 53 * provided, <vuln>_select_mitigation() will override AUTO to the best 54 * mitigation option. 55 */ 56 57 static void __init spectre_v1_select_mitigation(void); 58 static void __init spectre_v1_apply_mitigation(void); 59 static void __init spectre_v2_select_mitigation(void); 60 static void __init spectre_v2_update_mitigation(void); 61 static void __init spectre_v2_apply_mitigation(void); 62 static void __init retbleed_select_mitigation(void); 63 static void __init retbleed_update_mitigation(void); 64 static void __init retbleed_apply_mitigation(void); 65 static void __init spectre_v2_user_select_mitigation(void); 66 static void __init spectre_v2_user_update_mitigation(void); 67 static void __init spectre_v2_user_apply_mitigation(void); 68 static void __init ssb_select_mitigation(void); 69 static void __init ssb_apply_mitigation(void); 70 static void __init l1tf_select_mitigation(void); 71 static void __init l1tf_apply_mitigation(void); 72 static void __init mds_select_mitigation(void); 73 static void __init mds_update_mitigation(void); 74 static void __init mds_apply_mitigation(void); 75 static void __init taa_select_mitigation(void); 76 static void __init taa_update_mitigation(void); 77 static void __init taa_apply_mitigation(void); 78 static void __init mmio_select_mitigation(void); 79 static void __init mmio_update_mitigation(void); 80 static void __init mmio_apply_mitigation(void); 81 static void __init rfds_select_mitigation(void); 82 static void __init rfds_update_mitigation(void); 83 static void __init rfds_apply_mitigation(void); 84 static void __init srbds_select_mitigation(void); 85 static void __init srbds_apply_mitigation(void); 86 static void __init l1d_flush_select_mitigation(void); 87 static void __init srso_select_mitigation(void); 88 static void __init srso_update_mitigation(void); 89 static void __init srso_apply_mitigation(void); 90 static void __init gds_select_mitigation(void); 91 static void __init gds_apply_mitigation(void); 92 static void __init bhi_select_mitigation(void); 93 static void __init bhi_update_mitigation(void); 94 static void __init bhi_apply_mitigation(void); 95 static void __init its_select_mitigation(void); 96 static void __init its_update_mitigation(void); 97 static void __init its_apply_mitigation(void); 98 static void __init tsa_select_mitigation(void); 99 static void __init tsa_apply_mitigation(void); 100 static void __init vmscape_select_mitigation(void); 101 static void __init vmscape_update_mitigation(void); 102 static void __init vmscape_apply_mitigation(void); 103 104 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 105 u64 x86_spec_ctrl_base; 106 107 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 108 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 109 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current); 110 111 /* 112 * Set when the CPU has run a potentially malicious guest. An IBPB will 113 * be needed to before running userspace. That IBPB will flush the branch 114 * predictor content. 115 */ 116 DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user); 117 EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user); 118 119 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 120 121 static u64 __ro_after_init x86_arch_cap_msr; 122 123 static DEFINE_MUTEX(spec_ctrl_mutex); 124 125 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 126 127 static void __init set_return_thunk(void *thunk) 128 { 129 x86_return_thunk = thunk; 130 131 pr_info("active return thunk: %ps\n", thunk); 132 } 133 134 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 135 static void update_spec_ctrl(u64 val) 136 { 137 this_cpu_write(x86_spec_ctrl_current, val); 138 wrmsrq(MSR_IA32_SPEC_CTRL, val); 139 } 140 141 /* 142 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 143 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 144 */ 145 void update_spec_ctrl_cond(u64 val) 146 { 147 if (this_cpu_read(x86_spec_ctrl_current) == val) 148 return; 149 150 this_cpu_write(x86_spec_ctrl_current, val); 151 152 /* 153 * When KERNEL_IBRS this MSR is written on return-to-user, unless 154 * forced the update can be delayed until that time. 155 */ 156 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 157 wrmsrq(MSR_IA32_SPEC_CTRL, val); 158 } 159 160 noinstr u64 spec_ctrl_current(void) 161 { 162 return this_cpu_read(x86_spec_ctrl_current); 163 } 164 EXPORT_SYMBOL_GPL(spec_ctrl_current); 165 166 /* 167 * AMD specific MSR info for Speculative Store Bypass control. 168 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 169 */ 170 u64 __ro_after_init x86_amd_ls_cfg_base; 171 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 172 173 /* Control conditional STIBP in switch_to() */ 174 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 175 /* Control conditional IBPB in switch_mm() */ 176 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 177 /* Control unconditional IBPB in switch_mm() */ 178 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 179 180 /* Control IBPB on vCPU load */ 181 DEFINE_STATIC_KEY_FALSE(switch_vcpu_ibpb); 182 EXPORT_SYMBOL_FOR_KVM(switch_vcpu_ibpb); 183 184 /* Control CPU buffer clear before idling (halt, mwait) */ 185 DEFINE_STATIC_KEY_FALSE(cpu_buf_idle_clear); 186 EXPORT_SYMBOL_GPL(cpu_buf_idle_clear); 187 188 /* 189 * Controls whether l1d flush based mitigations are enabled, 190 * based on hw features and admin setting via boot parameter 191 * defaults to false 192 */ 193 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 194 195 /* 196 * Controls CPU Fill buffer clear before VMenter. This is a subset of 197 * X86_FEATURE_CLEAR_CPU_BUF, and should only be enabled when KVM-only 198 * mitigation is required. 199 */ 200 DEFINE_STATIC_KEY_FALSE(cpu_buf_vm_clear); 201 EXPORT_SYMBOL_FOR_KVM(cpu_buf_vm_clear); 202 203 #undef pr_fmt 204 #define pr_fmt(fmt) "mitigations: " fmt 205 206 static void __init cpu_print_attack_vectors(void) 207 { 208 pr_info("Enabled attack vectors: "); 209 210 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL)) 211 pr_cont("user_kernel, "); 212 213 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) 214 pr_cont("user_user, "); 215 216 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST)) 217 pr_cont("guest_host, "); 218 219 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) 220 pr_cont("guest_guest, "); 221 222 pr_cont("SMT mitigations: "); 223 224 switch (smt_mitigations) { 225 case SMT_MITIGATIONS_OFF: 226 pr_cont("off\n"); 227 break; 228 case SMT_MITIGATIONS_AUTO: 229 pr_cont("auto\n"); 230 break; 231 case SMT_MITIGATIONS_ON: 232 pr_cont("on\n"); 233 } 234 } 235 236 void __init cpu_select_mitigations(void) 237 { 238 /* 239 * Read the SPEC_CTRL MSR to account for reserved bits which may 240 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 241 * init code as it is not enumerated and depends on the family. 242 */ 243 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 244 rdmsrq(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 245 246 /* 247 * Previously running kernel (kexec), may have some controls 248 * turned ON. Clear them and let the mitigations setup below 249 * rediscover them based on configuration. 250 */ 251 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 252 } 253 254 x86_arch_cap_msr = x86_read_arch_cap_msr(); 255 256 cpu_print_attack_vectors(); 257 258 /* Select the proper CPU mitigations before patching alternatives: */ 259 spectre_v1_select_mitigation(); 260 spectre_v2_select_mitigation(); 261 retbleed_select_mitigation(); 262 spectre_v2_user_select_mitigation(); 263 ssb_select_mitigation(); 264 l1tf_select_mitigation(); 265 mds_select_mitigation(); 266 taa_select_mitigation(); 267 mmio_select_mitigation(); 268 rfds_select_mitigation(); 269 srbds_select_mitigation(); 270 l1d_flush_select_mitigation(); 271 srso_select_mitigation(); 272 gds_select_mitigation(); 273 its_select_mitigation(); 274 bhi_select_mitigation(); 275 tsa_select_mitigation(); 276 vmscape_select_mitigation(); 277 278 /* 279 * After mitigations are selected, some may need to update their 280 * choices. 281 */ 282 spectre_v2_update_mitigation(); 283 /* 284 * retbleed_update_mitigation() relies on the state set by 285 * spectre_v2_update_mitigation(); specifically it wants to know about 286 * spectre_v2=ibrs. 287 */ 288 retbleed_update_mitigation(); 289 /* 290 * its_update_mitigation() depends on spectre_v2_update_mitigation() 291 * and retbleed_update_mitigation(). 292 */ 293 its_update_mitigation(); 294 295 /* 296 * spectre_v2_user_update_mitigation() depends on 297 * retbleed_update_mitigation(), specifically the STIBP 298 * selection is forced for UNRET or IBPB. 299 */ 300 spectre_v2_user_update_mitigation(); 301 mds_update_mitigation(); 302 taa_update_mitigation(); 303 mmio_update_mitigation(); 304 rfds_update_mitigation(); 305 bhi_update_mitigation(); 306 /* srso_update_mitigation() depends on retbleed_update_mitigation(). */ 307 srso_update_mitigation(); 308 vmscape_update_mitigation(); 309 310 spectre_v1_apply_mitigation(); 311 spectre_v2_apply_mitigation(); 312 retbleed_apply_mitigation(); 313 spectre_v2_user_apply_mitigation(); 314 ssb_apply_mitigation(); 315 l1tf_apply_mitigation(); 316 mds_apply_mitigation(); 317 taa_apply_mitigation(); 318 mmio_apply_mitigation(); 319 rfds_apply_mitigation(); 320 srbds_apply_mitigation(); 321 srso_apply_mitigation(); 322 gds_apply_mitigation(); 323 its_apply_mitigation(); 324 bhi_apply_mitigation(); 325 tsa_apply_mitigation(); 326 vmscape_apply_mitigation(); 327 } 328 329 /* 330 * NOTE: This function is *only* called for SVM, since Intel uses 331 * MSR_IA32_SPEC_CTRL for SSBD. 332 */ 333 void 334 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 335 { 336 u64 guestval, hostval; 337 struct thread_info *ti = current_thread_info(); 338 339 /* 340 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 341 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 342 */ 343 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 344 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 345 return; 346 347 /* 348 * If the host has SSBD mitigation enabled, force it in the host's 349 * virtual MSR value. If its not permanently enabled, evaluate 350 * current's TIF_SSBD thread flag. 351 */ 352 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 353 hostval = SPEC_CTRL_SSBD; 354 else 355 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 356 357 /* Sanitize the guest value */ 358 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 359 360 if (hostval != guestval) { 361 unsigned long tif; 362 363 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 364 ssbd_spec_ctrl_to_tif(hostval); 365 366 speculation_ctrl_update(tif); 367 } 368 } 369 EXPORT_SYMBOL_FOR_KVM(x86_virt_spec_ctrl); 370 371 static void x86_amd_ssb_disable(void) 372 { 373 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 374 375 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 376 wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 377 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 378 wrmsrq(MSR_AMD64_LS_CFG, msrval); 379 } 380 381 #undef pr_fmt 382 #define pr_fmt(fmt) "MDS: " fmt 383 384 /* 385 * Returns true if vulnerability should be mitigated based on the 386 * selected attack vector controls. 387 * 388 * See Documentation/admin-guide/hw-vuln/attack_vector_controls.rst 389 */ 390 static bool __init should_mitigate_vuln(unsigned int bug) 391 { 392 switch (bug) { 393 /* 394 * The only runtime-selected spectre_v1 mitigations in the kernel are 395 * related to SWAPGS protection on kernel entry. Therefore, protection 396 * is only required for the user->kernel attack vector. 397 */ 398 case X86_BUG_SPECTRE_V1: 399 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL); 400 401 case X86_BUG_SPECTRE_V2: 402 case X86_BUG_RETBLEED: 403 case X86_BUG_L1TF: 404 case X86_BUG_ITS: 405 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 406 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST); 407 408 case X86_BUG_SPECTRE_V2_USER: 409 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 410 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST); 411 412 /* 413 * All the vulnerabilities below allow potentially leaking data 414 * across address spaces. Therefore, mitigation is required for 415 * any of these 4 attack vectors. 416 */ 417 case X86_BUG_MDS: 418 case X86_BUG_TAA: 419 case X86_BUG_MMIO_STALE_DATA: 420 case X86_BUG_RFDS: 421 case X86_BUG_SRBDS: 422 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 423 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 424 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 425 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST); 426 427 case X86_BUG_GDS: 428 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 429 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 430 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 431 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST) || 432 (smt_mitigations != SMT_MITIGATIONS_OFF); 433 434 case X86_BUG_SPEC_STORE_BYPASS: 435 return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER); 436 437 case X86_BUG_VMSCAPE: 438 return cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST); 439 440 default: 441 WARN(1, "Unknown bug %x\n", bug); 442 return false; 443 } 444 } 445 446 /* Default mitigation for MDS-affected CPUs */ 447 static enum mds_mitigations mds_mitigation __ro_after_init = 448 IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_AUTO : MDS_MITIGATION_OFF; 449 static bool mds_nosmt __ro_after_init = false; 450 451 static const char * const mds_strings[] = { 452 [MDS_MITIGATION_OFF] = "Vulnerable", 453 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 454 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 455 }; 456 457 enum taa_mitigations { 458 TAA_MITIGATION_OFF, 459 TAA_MITIGATION_AUTO, 460 TAA_MITIGATION_UCODE_NEEDED, 461 TAA_MITIGATION_VERW, 462 TAA_MITIGATION_TSX_DISABLED, 463 }; 464 465 /* Default mitigation for TAA-affected CPUs */ 466 static enum taa_mitigations taa_mitigation __ro_after_init = 467 IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_AUTO : TAA_MITIGATION_OFF; 468 469 enum mmio_mitigations { 470 MMIO_MITIGATION_OFF, 471 MMIO_MITIGATION_AUTO, 472 MMIO_MITIGATION_UCODE_NEEDED, 473 MMIO_MITIGATION_VERW, 474 }; 475 476 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 477 static enum mmio_mitigations mmio_mitigation __ro_after_init = 478 IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ? MMIO_MITIGATION_AUTO : MMIO_MITIGATION_OFF; 479 480 enum rfds_mitigations { 481 RFDS_MITIGATION_OFF, 482 RFDS_MITIGATION_AUTO, 483 RFDS_MITIGATION_VERW, 484 RFDS_MITIGATION_UCODE_NEEDED, 485 }; 486 487 /* Default mitigation for Register File Data Sampling */ 488 static enum rfds_mitigations rfds_mitigation __ro_after_init = 489 IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF; 490 491 /* 492 * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing 493 * through X86_FEATURE_CLEAR_CPU_BUF on kernel and guest entry. 494 */ 495 static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init; 496 497 static void __init mds_select_mitigation(void) 498 { 499 if (!boot_cpu_has_bug(X86_BUG_MDS)) { 500 mds_mitigation = MDS_MITIGATION_OFF; 501 return; 502 } 503 504 if (mds_mitigation == MDS_MITIGATION_AUTO) { 505 if (should_mitigate_vuln(X86_BUG_MDS)) 506 mds_mitigation = MDS_MITIGATION_FULL; 507 else 508 mds_mitigation = MDS_MITIGATION_OFF; 509 } 510 511 if (mds_mitigation == MDS_MITIGATION_OFF) 512 return; 513 514 verw_clear_cpu_buf_mitigation_selected = true; 515 } 516 517 static void __init mds_update_mitigation(void) 518 { 519 if (!boot_cpu_has_bug(X86_BUG_MDS)) 520 return; 521 522 /* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */ 523 if (verw_clear_cpu_buf_mitigation_selected) 524 mds_mitigation = MDS_MITIGATION_FULL; 525 526 if (mds_mitigation == MDS_MITIGATION_FULL) { 527 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 528 mds_mitigation = MDS_MITIGATION_VMWERV; 529 } 530 531 pr_info("%s\n", mds_strings[mds_mitigation]); 532 } 533 534 static void __init mds_apply_mitigation(void) 535 { 536 if (mds_mitigation == MDS_MITIGATION_FULL || 537 mds_mitigation == MDS_MITIGATION_VMWERV) { 538 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 539 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 540 (mds_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)) 541 cpu_smt_disable(false); 542 } 543 } 544 545 static int __init mds_cmdline(char *str) 546 { 547 if (!boot_cpu_has_bug(X86_BUG_MDS)) 548 return 0; 549 550 if (!str) 551 return -EINVAL; 552 553 if (!strcmp(str, "off")) 554 mds_mitigation = MDS_MITIGATION_OFF; 555 else if (!strcmp(str, "full")) 556 mds_mitigation = MDS_MITIGATION_FULL; 557 else if (!strcmp(str, "full,nosmt")) { 558 mds_mitigation = MDS_MITIGATION_FULL; 559 mds_nosmt = true; 560 } 561 562 return 0; 563 } 564 early_param("mds", mds_cmdline); 565 566 #undef pr_fmt 567 #define pr_fmt(fmt) "TAA: " fmt 568 569 static bool taa_nosmt __ro_after_init; 570 571 static const char * const taa_strings[] = { 572 [TAA_MITIGATION_OFF] = "Vulnerable", 573 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 574 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 575 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 576 }; 577 578 static bool __init taa_vulnerable(void) 579 { 580 return boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM); 581 } 582 583 static void __init taa_select_mitigation(void) 584 { 585 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 586 taa_mitigation = TAA_MITIGATION_OFF; 587 return; 588 } 589 590 /* TSX previously disabled by tsx=off */ 591 if (!boot_cpu_has(X86_FEATURE_RTM)) { 592 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 593 return; 594 } 595 596 /* Microcode will be checked in taa_update_mitigation(). */ 597 if (taa_mitigation == TAA_MITIGATION_AUTO) { 598 if (should_mitigate_vuln(X86_BUG_TAA)) 599 taa_mitigation = TAA_MITIGATION_VERW; 600 else 601 taa_mitigation = TAA_MITIGATION_OFF; 602 } 603 604 if (taa_mitigation != TAA_MITIGATION_OFF) 605 verw_clear_cpu_buf_mitigation_selected = true; 606 } 607 608 static void __init taa_update_mitigation(void) 609 { 610 if (!taa_vulnerable()) 611 return; 612 613 if (verw_clear_cpu_buf_mitigation_selected) 614 taa_mitigation = TAA_MITIGATION_VERW; 615 616 if (taa_mitigation == TAA_MITIGATION_VERW) { 617 /* Check if the requisite ucode is available. */ 618 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 619 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 620 621 /* 622 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 623 * A microcode update fixes this behavior to clear CPU buffers. It also 624 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 625 * ARCH_CAP_TSX_CTRL_MSR bit. 626 * 627 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 628 * update is required. 629 */ 630 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && 631 !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) 632 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 633 } 634 635 pr_info("%s\n", taa_strings[taa_mitigation]); 636 } 637 638 static void __init taa_apply_mitigation(void) 639 { 640 if (taa_mitigation == TAA_MITIGATION_VERW || 641 taa_mitigation == TAA_MITIGATION_UCODE_NEEDED) { 642 /* 643 * TSX is enabled, select alternate mitigation for TAA which is 644 * the same as MDS. Enable MDS static branch to clear CPU buffers. 645 * 646 * For guests that can't determine whether the correct microcode is 647 * present on host, enable the mitigation for UCODE_NEEDED as well. 648 */ 649 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 650 651 if (taa_nosmt || smt_mitigations == SMT_MITIGATIONS_ON) 652 cpu_smt_disable(false); 653 } 654 } 655 656 static int __init tsx_async_abort_parse_cmdline(char *str) 657 { 658 if (!boot_cpu_has_bug(X86_BUG_TAA)) 659 return 0; 660 661 if (!str) 662 return -EINVAL; 663 664 if (!strcmp(str, "off")) { 665 taa_mitigation = TAA_MITIGATION_OFF; 666 } else if (!strcmp(str, "full")) { 667 taa_mitigation = TAA_MITIGATION_VERW; 668 } else if (!strcmp(str, "full,nosmt")) { 669 taa_mitigation = TAA_MITIGATION_VERW; 670 taa_nosmt = true; 671 } 672 673 return 0; 674 } 675 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 676 677 #undef pr_fmt 678 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 679 680 static bool mmio_nosmt __ro_after_init = false; 681 682 static const char * const mmio_strings[] = { 683 [MMIO_MITIGATION_OFF] = "Vulnerable", 684 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 685 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 686 }; 687 688 static void __init mmio_select_mitigation(void) 689 { 690 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 691 mmio_mitigation = MMIO_MITIGATION_OFF; 692 return; 693 } 694 695 /* Microcode will be checked in mmio_update_mitigation(). */ 696 if (mmio_mitigation == MMIO_MITIGATION_AUTO) { 697 if (should_mitigate_vuln(X86_BUG_MMIO_STALE_DATA)) 698 mmio_mitigation = MMIO_MITIGATION_VERW; 699 else 700 mmio_mitigation = MMIO_MITIGATION_OFF; 701 } 702 703 if (mmio_mitigation == MMIO_MITIGATION_OFF) 704 return; 705 706 /* 707 * Enable CPU buffer clear mitigation for host and VMM, if also affected 708 * by MDS or TAA. 709 */ 710 if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable()) 711 verw_clear_cpu_buf_mitigation_selected = true; 712 } 713 714 static void __init mmio_update_mitigation(void) 715 { 716 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 717 return; 718 719 if (verw_clear_cpu_buf_mitigation_selected) 720 mmio_mitigation = MMIO_MITIGATION_VERW; 721 722 if (mmio_mitigation == MMIO_MITIGATION_VERW) { 723 /* 724 * Check if the system has the right microcode. 725 * 726 * CPU Fill buffer clear mitigation is enumerated by either an explicit 727 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 728 * affected systems. 729 */ 730 if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || 731 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 732 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 733 !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))) 734 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 735 } 736 737 pr_info("%s\n", mmio_strings[mmio_mitigation]); 738 } 739 740 static void __init mmio_apply_mitigation(void) 741 { 742 if (mmio_mitigation == MMIO_MITIGATION_OFF) 743 return; 744 745 /* 746 * Only enable the VMM mitigation if the CPU buffer clear mitigation is 747 * not being used. 748 */ 749 if (verw_clear_cpu_buf_mitigation_selected) { 750 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 751 static_branch_disable(&cpu_buf_vm_clear); 752 } else { 753 static_branch_enable(&cpu_buf_vm_clear); 754 } 755 756 /* 757 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 758 * be propagated to uncore buffers, clearing the Fill buffers on idle 759 * is required irrespective of SMT state. 760 */ 761 if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) 762 static_branch_enable(&cpu_buf_idle_clear); 763 764 if (mmio_nosmt || smt_mitigations == SMT_MITIGATIONS_ON) 765 cpu_smt_disable(false); 766 } 767 768 static int __init mmio_stale_data_parse_cmdline(char *str) 769 { 770 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 771 return 0; 772 773 if (!str) 774 return -EINVAL; 775 776 if (!strcmp(str, "off")) { 777 mmio_mitigation = MMIO_MITIGATION_OFF; 778 } else if (!strcmp(str, "full")) { 779 mmio_mitigation = MMIO_MITIGATION_VERW; 780 } else if (!strcmp(str, "full,nosmt")) { 781 mmio_mitigation = MMIO_MITIGATION_VERW; 782 mmio_nosmt = true; 783 } 784 785 return 0; 786 } 787 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 788 789 #undef pr_fmt 790 #define pr_fmt(fmt) "Register File Data Sampling: " fmt 791 792 static const char * const rfds_strings[] = { 793 [RFDS_MITIGATION_OFF] = "Vulnerable", 794 [RFDS_MITIGATION_VERW] = "Mitigation: Clear Register File", 795 [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 796 }; 797 798 static inline bool __init verw_clears_cpu_reg_file(void) 799 { 800 return (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR); 801 } 802 803 static void __init rfds_select_mitigation(void) 804 { 805 if (!boot_cpu_has_bug(X86_BUG_RFDS)) { 806 rfds_mitigation = RFDS_MITIGATION_OFF; 807 return; 808 } 809 810 if (rfds_mitigation == RFDS_MITIGATION_AUTO) { 811 if (should_mitigate_vuln(X86_BUG_RFDS)) 812 rfds_mitigation = RFDS_MITIGATION_VERW; 813 else 814 rfds_mitigation = RFDS_MITIGATION_OFF; 815 } 816 817 if (rfds_mitigation == RFDS_MITIGATION_OFF) 818 return; 819 820 if (verw_clears_cpu_reg_file()) 821 verw_clear_cpu_buf_mitigation_selected = true; 822 } 823 824 static void __init rfds_update_mitigation(void) 825 { 826 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 827 return; 828 829 if (verw_clear_cpu_buf_mitigation_selected) 830 rfds_mitigation = RFDS_MITIGATION_VERW; 831 832 if (rfds_mitigation == RFDS_MITIGATION_VERW) { 833 if (!verw_clears_cpu_reg_file()) 834 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; 835 } 836 837 pr_info("%s\n", rfds_strings[rfds_mitigation]); 838 } 839 840 static void __init rfds_apply_mitigation(void) 841 { 842 if (rfds_mitigation == RFDS_MITIGATION_VERW) 843 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 844 } 845 846 static __init int rfds_parse_cmdline(char *str) 847 { 848 if (!str) 849 return -EINVAL; 850 851 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 852 return 0; 853 854 if (!strcmp(str, "off")) 855 rfds_mitigation = RFDS_MITIGATION_OFF; 856 else if (!strcmp(str, "on")) 857 rfds_mitigation = RFDS_MITIGATION_VERW; 858 859 return 0; 860 } 861 early_param("reg_file_data_sampling", rfds_parse_cmdline); 862 863 #undef pr_fmt 864 #define pr_fmt(fmt) "SRBDS: " fmt 865 866 enum srbds_mitigations { 867 SRBDS_MITIGATION_OFF, 868 SRBDS_MITIGATION_AUTO, 869 SRBDS_MITIGATION_UCODE_NEEDED, 870 SRBDS_MITIGATION_FULL, 871 SRBDS_MITIGATION_TSX_OFF, 872 SRBDS_MITIGATION_HYPERVISOR, 873 }; 874 875 static enum srbds_mitigations srbds_mitigation __ro_after_init = 876 IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_AUTO : SRBDS_MITIGATION_OFF; 877 878 static const char * const srbds_strings[] = { 879 [SRBDS_MITIGATION_OFF] = "Vulnerable", 880 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 881 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 882 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 883 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 884 }; 885 886 static bool srbds_off; 887 888 void update_srbds_msr(void) 889 { 890 u64 mcu_ctrl; 891 892 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 893 return; 894 895 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 896 return; 897 898 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 899 return; 900 901 /* 902 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 903 * being disabled and it hasn't received the SRBDS MSR microcode. 904 */ 905 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 906 return; 907 908 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 909 910 switch (srbds_mitigation) { 911 case SRBDS_MITIGATION_OFF: 912 case SRBDS_MITIGATION_TSX_OFF: 913 mcu_ctrl |= RNGDS_MITG_DIS; 914 break; 915 case SRBDS_MITIGATION_FULL: 916 mcu_ctrl &= ~RNGDS_MITG_DIS; 917 break; 918 default: 919 break; 920 } 921 922 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 923 } 924 925 static void __init srbds_select_mitigation(void) 926 { 927 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) { 928 srbds_mitigation = SRBDS_MITIGATION_OFF; 929 return; 930 } 931 932 if (srbds_mitigation == SRBDS_MITIGATION_AUTO) { 933 if (should_mitigate_vuln(X86_BUG_SRBDS)) 934 srbds_mitigation = SRBDS_MITIGATION_FULL; 935 else { 936 srbds_mitigation = SRBDS_MITIGATION_OFF; 937 return; 938 } 939 } 940 941 /* 942 * Check to see if this is one of the MDS_NO systems supporting TSX that 943 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 944 * by Processor MMIO Stale Data vulnerability. 945 */ 946 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 947 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 948 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 949 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 950 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 951 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 952 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 953 else if (srbds_off) 954 srbds_mitigation = SRBDS_MITIGATION_OFF; 955 956 pr_info("%s\n", srbds_strings[srbds_mitigation]); 957 } 958 959 static void __init srbds_apply_mitigation(void) 960 { 961 update_srbds_msr(); 962 } 963 964 static int __init srbds_parse_cmdline(char *str) 965 { 966 if (!str) 967 return -EINVAL; 968 969 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 970 return 0; 971 972 srbds_off = !strcmp(str, "off"); 973 return 0; 974 } 975 early_param("srbds", srbds_parse_cmdline); 976 977 #undef pr_fmt 978 #define pr_fmt(fmt) "L1D Flush : " fmt 979 980 enum l1d_flush_mitigations { 981 L1D_FLUSH_OFF = 0, 982 L1D_FLUSH_ON, 983 }; 984 985 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 986 987 static void __init l1d_flush_select_mitigation(void) 988 { 989 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 990 return; 991 992 static_branch_enable(&switch_mm_cond_l1d_flush); 993 pr_info("Conditional flush on switch_mm() enabled\n"); 994 } 995 996 static int __init l1d_flush_parse_cmdline(char *str) 997 { 998 if (!strcmp(str, "on")) 999 l1d_flush_mitigation = L1D_FLUSH_ON; 1000 1001 return 0; 1002 } 1003 early_param("l1d_flush", l1d_flush_parse_cmdline); 1004 1005 #undef pr_fmt 1006 #define pr_fmt(fmt) "GDS: " fmt 1007 1008 enum gds_mitigations { 1009 GDS_MITIGATION_OFF, 1010 GDS_MITIGATION_AUTO, 1011 GDS_MITIGATION_UCODE_NEEDED, 1012 GDS_MITIGATION_FORCE, 1013 GDS_MITIGATION_FULL, 1014 GDS_MITIGATION_FULL_LOCKED, 1015 GDS_MITIGATION_HYPERVISOR, 1016 }; 1017 1018 static enum gds_mitigations gds_mitigation __ro_after_init = 1019 IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_AUTO : GDS_MITIGATION_OFF; 1020 1021 static const char * const gds_strings[] = { 1022 [GDS_MITIGATION_OFF] = "Vulnerable", 1023 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 1024 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 1025 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 1026 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 1027 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 1028 }; 1029 1030 bool gds_ucode_mitigated(void) 1031 { 1032 return (gds_mitigation == GDS_MITIGATION_FULL || 1033 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 1034 } 1035 EXPORT_SYMBOL_FOR_KVM(gds_ucode_mitigated); 1036 1037 void update_gds_msr(void) 1038 { 1039 u64 mcu_ctrl_after; 1040 u64 mcu_ctrl; 1041 1042 switch (gds_mitigation) { 1043 case GDS_MITIGATION_OFF: 1044 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 1045 mcu_ctrl |= GDS_MITG_DIS; 1046 break; 1047 case GDS_MITIGATION_FULL_LOCKED: 1048 /* 1049 * The LOCKED state comes from the boot CPU. APs might not have 1050 * the same state. Make sure the mitigation is enabled on all 1051 * CPUs. 1052 */ 1053 case GDS_MITIGATION_FULL: 1054 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 1055 mcu_ctrl &= ~GDS_MITG_DIS; 1056 break; 1057 case GDS_MITIGATION_FORCE: 1058 case GDS_MITIGATION_UCODE_NEEDED: 1059 case GDS_MITIGATION_HYPERVISOR: 1060 case GDS_MITIGATION_AUTO: 1061 return; 1062 } 1063 1064 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 1065 1066 /* 1067 * Check to make sure that the WRMSR value was not ignored. Writes to 1068 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 1069 * processor was not. 1070 */ 1071 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 1072 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 1073 } 1074 1075 static void __init gds_select_mitigation(void) 1076 { 1077 u64 mcu_ctrl; 1078 1079 if (!boot_cpu_has_bug(X86_BUG_GDS)) 1080 return; 1081 1082 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1083 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 1084 return; 1085 } 1086 1087 /* Will verify below that mitigation _can_ be disabled */ 1088 if (gds_mitigation == GDS_MITIGATION_AUTO) { 1089 if (should_mitigate_vuln(X86_BUG_GDS)) 1090 gds_mitigation = GDS_MITIGATION_FULL; 1091 else 1092 gds_mitigation = GDS_MITIGATION_OFF; 1093 } 1094 1095 /* No microcode */ 1096 if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) { 1097 if (gds_mitigation != GDS_MITIGATION_FORCE) 1098 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 1099 return; 1100 } 1101 1102 /* Microcode has mitigation, use it */ 1103 if (gds_mitigation == GDS_MITIGATION_FORCE) 1104 gds_mitigation = GDS_MITIGATION_FULL; 1105 1106 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 1107 if (mcu_ctrl & GDS_MITG_LOCKED) { 1108 if (gds_mitigation == GDS_MITIGATION_OFF) 1109 pr_warn("Mitigation locked. Disable failed.\n"); 1110 1111 /* 1112 * The mitigation is selected from the boot CPU. All other CPUs 1113 * _should_ have the same state. If the boot CPU isn't locked 1114 * but others are then update_gds_msr() will WARN() of the state 1115 * mismatch. If the boot CPU is locked update_gds_msr() will 1116 * ensure the other CPUs have the mitigation enabled. 1117 */ 1118 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 1119 } 1120 } 1121 1122 static void __init gds_apply_mitigation(void) 1123 { 1124 if (!boot_cpu_has_bug(X86_BUG_GDS)) 1125 return; 1126 1127 /* Microcode is present */ 1128 if (x86_arch_cap_msr & ARCH_CAP_GDS_CTRL) 1129 update_gds_msr(); 1130 else if (gds_mitigation == GDS_MITIGATION_FORCE) { 1131 /* 1132 * This only needs to be done on the boot CPU so do it 1133 * here rather than in update_gds_msr() 1134 */ 1135 setup_clear_cpu_cap(X86_FEATURE_AVX); 1136 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 1137 } 1138 1139 pr_info("%s\n", gds_strings[gds_mitigation]); 1140 } 1141 1142 static int __init gds_parse_cmdline(char *str) 1143 { 1144 if (!str) 1145 return -EINVAL; 1146 1147 if (!boot_cpu_has_bug(X86_BUG_GDS)) 1148 return 0; 1149 1150 if (!strcmp(str, "off")) 1151 gds_mitigation = GDS_MITIGATION_OFF; 1152 else if (!strcmp(str, "force")) 1153 gds_mitigation = GDS_MITIGATION_FORCE; 1154 1155 return 0; 1156 } 1157 early_param("gather_data_sampling", gds_parse_cmdline); 1158 1159 #undef pr_fmt 1160 #define pr_fmt(fmt) "Spectre V1 : " fmt 1161 1162 enum spectre_v1_mitigation { 1163 SPECTRE_V1_MITIGATION_NONE, 1164 SPECTRE_V1_MITIGATION_AUTO, 1165 }; 1166 1167 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 1168 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ? 1169 SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE; 1170 1171 static const char * const spectre_v1_strings[] = { 1172 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 1173 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 1174 }; 1175 1176 /* 1177 * Does SMAP provide full mitigation against speculative kernel access to 1178 * userspace? 1179 */ 1180 static bool smap_works_speculatively(void) 1181 { 1182 if (!boot_cpu_has(X86_FEATURE_SMAP)) 1183 return false; 1184 1185 /* 1186 * On CPUs which are vulnerable to Meltdown, SMAP does not 1187 * prevent speculative access to user data in the L1 cache. 1188 * Consider SMAP to be non-functional as a mitigation on these 1189 * CPUs. 1190 */ 1191 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 1192 return false; 1193 1194 return true; 1195 } 1196 1197 static void __init spectre_v1_select_mitigation(void) 1198 { 1199 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 1200 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1201 1202 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V1)) 1203 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1204 } 1205 1206 static void __init spectre_v1_apply_mitigation(void) 1207 { 1208 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 1209 return; 1210 1211 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 1212 /* 1213 * With Spectre v1, a user can speculatively control either 1214 * path of a conditional swapgs with a user-controlled GS 1215 * value. The mitigation is to add lfences to both code paths. 1216 * 1217 * If FSGSBASE is enabled, the user can put a kernel address in 1218 * GS, in which case SMAP provides no protection. 1219 * 1220 * If FSGSBASE is disabled, the user can only put a user space 1221 * address in GS. That makes an attack harder, but still 1222 * possible if there's no SMAP protection. 1223 */ 1224 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 1225 !smap_works_speculatively()) { 1226 /* 1227 * Mitigation can be provided from SWAPGS itself or 1228 * PTI as the CR3 write in the Meltdown mitigation 1229 * is serializing. 1230 * 1231 * If neither is there, mitigate with an LFENCE to 1232 * stop speculation through swapgs. 1233 */ 1234 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 1235 !boot_cpu_has(X86_FEATURE_PTI)) 1236 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 1237 1238 /* 1239 * Enable lfences in the kernel entry (non-swapgs) 1240 * paths, to prevent user entry from speculatively 1241 * skipping swapgs. 1242 */ 1243 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 1244 } 1245 } 1246 1247 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1248 } 1249 1250 static int __init nospectre_v1_cmdline(char *str) 1251 { 1252 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1253 return 0; 1254 } 1255 early_param("nospectre_v1", nospectre_v1_cmdline); 1256 1257 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 1258 1259 /* Depends on spectre_v2 mitigation selected already */ 1260 static inline bool cdt_possible(enum spectre_v2_mitigation mode) 1261 { 1262 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) || 1263 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) 1264 return false; 1265 1266 if (mode == SPECTRE_V2_RETPOLINE || 1267 mode == SPECTRE_V2_EIBRS_RETPOLINE) 1268 return true; 1269 1270 return false; 1271 } 1272 1273 #undef pr_fmt 1274 #define pr_fmt(fmt) "RETBleed: " fmt 1275 1276 enum its_mitigation { 1277 ITS_MITIGATION_OFF, 1278 ITS_MITIGATION_AUTO, 1279 ITS_MITIGATION_VMEXIT_ONLY, 1280 ITS_MITIGATION_ALIGNED_THUNKS, 1281 ITS_MITIGATION_RETPOLINE_STUFF, 1282 }; 1283 1284 static enum its_mitigation its_mitigation __ro_after_init = 1285 IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_MITIGATION_AUTO : ITS_MITIGATION_OFF; 1286 1287 enum retbleed_mitigation { 1288 RETBLEED_MITIGATION_NONE, 1289 RETBLEED_MITIGATION_AUTO, 1290 RETBLEED_MITIGATION_UNRET, 1291 RETBLEED_MITIGATION_IBPB, 1292 RETBLEED_MITIGATION_IBRS, 1293 RETBLEED_MITIGATION_EIBRS, 1294 RETBLEED_MITIGATION_STUFF, 1295 }; 1296 1297 static const char * const retbleed_strings[] = { 1298 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 1299 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 1300 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 1301 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 1302 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 1303 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 1304 }; 1305 1306 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 1307 IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE; 1308 1309 static int __ro_after_init retbleed_nosmt = false; 1310 1311 enum srso_mitigation { 1312 SRSO_MITIGATION_NONE, 1313 SRSO_MITIGATION_AUTO, 1314 SRSO_MITIGATION_UCODE_NEEDED, 1315 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 1316 SRSO_MITIGATION_MICROCODE, 1317 SRSO_MITIGATION_NOSMT, 1318 SRSO_MITIGATION_SAFE_RET, 1319 SRSO_MITIGATION_IBPB, 1320 SRSO_MITIGATION_IBPB_ON_VMEXIT, 1321 SRSO_MITIGATION_BP_SPEC_REDUCE, 1322 }; 1323 1324 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO; 1325 1326 static int __init retbleed_parse_cmdline(char *str) 1327 { 1328 if (!str) 1329 return -EINVAL; 1330 1331 while (str) { 1332 char *next = strchr(str, ','); 1333 if (next) { 1334 *next = 0; 1335 next++; 1336 } 1337 1338 if (!strcmp(str, "off")) { 1339 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1340 } else if (!strcmp(str, "auto")) { 1341 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1342 } else if (!strcmp(str, "unret")) { 1343 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1344 } else if (!strcmp(str, "ibpb")) { 1345 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1346 } else if (!strcmp(str, "stuff")) { 1347 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1348 } else if (!strcmp(str, "nosmt")) { 1349 retbleed_nosmt = true; 1350 } else if (!strcmp(str, "force")) { 1351 setup_force_cpu_bug(X86_BUG_RETBLEED); 1352 } else { 1353 pr_err("Ignoring unknown retbleed option (%s).", str); 1354 } 1355 1356 str = next; 1357 } 1358 1359 return 0; 1360 } 1361 early_param("retbleed", retbleed_parse_cmdline); 1362 1363 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 1364 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 1365 1366 static void __init retbleed_select_mitigation(void) 1367 { 1368 if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) { 1369 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1370 return; 1371 } 1372 1373 switch (retbleed_mitigation) { 1374 case RETBLEED_MITIGATION_UNRET: 1375 if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { 1376 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1377 pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); 1378 } 1379 break; 1380 case RETBLEED_MITIGATION_IBPB: 1381 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 1382 pr_err("WARNING: CPU does not support IBPB.\n"); 1383 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1384 } else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 1385 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 1386 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1387 } 1388 break; 1389 case RETBLEED_MITIGATION_STUFF: 1390 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1391 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); 1392 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1393 } else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1394 pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); 1395 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1396 } 1397 break; 1398 default: 1399 break; 1400 } 1401 1402 if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO) 1403 return; 1404 1405 if (!should_mitigate_vuln(X86_BUG_RETBLEED)) { 1406 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1407 return; 1408 } 1409 1410 /* Intel mitigation selected in retbleed_update_mitigation() */ 1411 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1412 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1413 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) 1414 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1415 else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && 1416 boot_cpu_has(X86_FEATURE_IBPB)) 1417 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1418 else 1419 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1420 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1421 /* Final mitigation depends on spectre-v2 selection */ 1422 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1423 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1424 else if (boot_cpu_has(X86_FEATURE_IBRS)) 1425 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1426 else 1427 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1428 } 1429 } 1430 1431 static void __init retbleed_update_mitigation(void) 1432 { 1433 if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) 1434 return; 1435 1436 /* ITS can also enable stuffing */ 1437 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF) 1438 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1439 1440 /* If SRSO is using IBPB, that works for retbleed too */ 1441 if (srso_mitigation == SRSO_MITIGATION_IBPB) 1442 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1443 1444 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF && 1445 !cdt_possible(spectre_v2_enabled)) { 1446 pr_err("WARNING: retbleed=stuff depends on retpoline\n"); 1447 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1448 } 1449 1450 /* 1451 * Let IBRS trump all on Intel without affecting the effects of the 1452 * retbleed= cmdline option except for call depth based stuffing 1453 */ 1454 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1455 switch (spectre_v2_enabled) { 1456 case SPECTRE_V2_IBRS: 1457 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1458 break; 1459 case SPECTRE_V2_EIBRS: 1460 case SPECTRE_V2_EIBRS_RETPOLINE: 1461 case SPECTRE_V2_EIBRS_LFENCE: 1462 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1463 break; 1464 default: 1465 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) { 1466 if (retbleed_mitigation != RETBLEED_MITIGATION_NONE) 1467 pr_err(RETBLEED_INTEL_MSG); 1468 1469 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1470 } 1471 } 1472 } 1473 1474 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1475 } 1476 1477 static void __init retbleed_apply_mitigation(void) 1478 { 1479 bool mitigate_smt = false; 1480 1481 switch (retbleed_mitigation) { 1482 case RETBLEED_MITIGATION_NONE: 1483 return; 1484 1485 case RETBLEED_MITIGATION_UNRET: 1486 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1487 setup_force_cpu_cap(X86_FEATURE_UNRET); 1488 1489 set_return_thunk(retbleed_return_thunk); 1490 1491 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1492 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1493 pr_err(RETBLEED_UNTRAIN_MSG); 1494 1495 mitigate_smt = true; 1496 break; 1497 1498 case RETBLEED_MITIGATION_IBPB: 1499 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1500 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1501 mitigate_smt = true; 1502 1503 /* 1504 * IBPB on entry already obviates the need for 1505 * software-based untraining so clear those in case some 1506 * other mitigation like SRSO has selected them. 1507 */ 1508 setup_clear_cpu_cap(X86_FEATURE_UNRET); 1509 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 1510 1511 /* 1512 * There is no need for RSB filling: write_ibpb() ensures 1513 * all predictions, including the RSB, are invalidated, 1514 * regardless of IBPB implementation. 1515 */ 1516 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1517 1518 break; 1519 1520 case RETBLEED_MITIGATION_STUFF: 1521 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1522 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1523 1524 set_return_thunk(call_depth_return_thunk); 1525 break; 1526 1527 default: 1528 break; 1529 } 1530 1531 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1532 (retbleed_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)) 1533 cpu_smt_disable(false); 1534 } 1535 1536 #undef pr_fmt 1537 #define pr_fmt(fmt) "ITS: " fmt 1538 1539 static const char * const its_strings[] = { 1540 [ITS_MITIGATION_OFF] = "Vulnerable", 1541 [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", 1542 [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", 1543 [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", 1544 }; 1545 1546 static int __init its_parse_cmdline(char *str) 1547 { 1548 if (!str) 1549 return -EINVAL; 1550 1551 if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) { 1552 pr_err("Mitigation disabled at compile time, ignoring option (%s)", str); 1553 return 0; 1554 } 1555 1556 if (!strcmp(str, "off")) { 1557 its_mitigation = ITS_MITIGATION_OFF; 1558 } else if (!strcmp(str, "on")) { 1559 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1560 } else if (!strcmp(str, "force")) { 1561 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1562 setup_force_cpu_bug(X86_BUG_ITS); 1563 } else if (!strcmp(str, "vmexit")) { 1564 its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; 1565 } else if (!strcmp(str, "stuff")) { 1566 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1567 } else { 1568 pr_err("Ignoring unknown indirect_target_selection option (%s).", str); 1569 } 1570 1571 return 0; 1572 } 1573 early_param("indirect_target_selection", its_parse_cmdline); 1574 1575 static void __init its_select_mitigation(void) 1576 { 1577 if (!boot_cpu_has_bug(X86_BUG_ITS)) { 1578 its_mitigation = ITS_MITIGATION_OFF; 1579 return; 1580 } 1581 1582 if (its_mitigation == ITS_MITIGATION_AUTO) { 1583 if (should_mitigate_vuln(X86_BUG_ITS)) 1584 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1585 else 1586 its_mitigation = ITS_MITIGATION_OFF; 1587 } 1588 1589 if (its_mitigation == ITS_MITIGATION_OFF) 1590 return; 1591 1592 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || 1593 !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) { 1594 pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); 1595 its_mitigation = ITS_MITIGATION_OFF; 1596 return; 1597 } 1598 1599 if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { 1600 pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); 1601 its_mitigation = ITS_MITIGATION_OFF; 1602 return; 1603 } 1604 1605 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1606 !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1607 pr_err("RSB stuff mitigation not supported, using default\n"); 1608 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1609 } 1610 1611 if (its_mitigation == ITS_MITIGATION_VMEXIT_ONLY && 1612 !boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) 1613 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1614 } 1615 1616 static void __init its_update_mitigation(void) 1617 { 1618 if (!boot_cpu_has_bug(X86_BUG_ITS)) 1619 return; 1620 1621 switch (spectre_v2_enabled) { 1622 case SPECTRE_V2_NONE: 1623 if (its_mitigation != ITS_MITIGATION_OFF) 1624 pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); 1625 its_mitigation = ITS_MITIGATION_OFF; 1626 break; 1627 case SPECTRE_V2_RETPOLINE: 1628 case SPECTRE_V2_EIBRS_RETPOLINE: 1629 /* Retpoline+CDT mitigates ITS */ 1630 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) 1631 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1632 break; 1633 case SPECTRE_V2_LFENCE: 1634 case SPECTRE_V2_EIBRS_LFENCE: 1635 pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); 1636 its_mitigation = ITS_MITIGATION_OFF; 1637 break; 1638 default: 1639 break; 1640 } 1641 1642 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1643 !cdt_possible(spectre_v2_enabled)) 1644 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1645 1646 pr_info("%s\n", its_strings[its_mitigation]); 1647 } 1648 1649 static void __init its_apply_mitigation(void) 1650 { 1651 switch (its_mitigation) { 1652 case ITS_MITIGATION_OFF: 1653 case ITS_MITIGATION_AUTO: 1654 case ITS_MITIGATION_VMEXIT_ONLY: 1655 break; 1656 case ITS_MITIGATION_ALIGNED_THUNKS: 1657 if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) 1658 setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); 1659 1660 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1661 set_return_thunk(its_return_thunk); 1662 break; 1663 case ITS_MITIGATION_RETPOLINE_STUFF: 1664 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1665 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1666 set_return_thunk(call_depth_return_thunk); 1667 break; 1668 } 1669 } 1670 1671 #undef pr_fmt 1672 #define pr_fmt(fmt) "Transient Scheduler Attacks: " fmt 1673 1674 enum tsa_mitigations { 1675 TSA_MITIGATION_NONE, 1676 TSA_MITIGATION_AUTO, 1677 TSA_MITIGATION_UCODE_NEEDED, 1678 TSA_MITIGATION_USER_KERNEL, 1679 TSA_MITIGATION_VM, 1680 TSA_MITIGATION_FULL, 1681 }; 1682 1683 static const char * const tsa_strings[] = { 1684 [TSA_MITIGATION_NONE] = "Vulnerable", 1685 [TSA_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 1686 [TSA_MITIGATION_USER_KERNEL] = "Mitigation: Clear CPU buffers: user/kernel boundary", 1687 [TSA_MITIGATION_VM] = "Mitigation: Clear CPU buffers: VM", 1688 [TSA_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 1689 }; 1690 1691 static enum tsa_mitigations tsa_mitigation __ro_after_init = 1692 IS_ENABLED(CONFIG_MITIGATION_TSA) ? TSA_MITIGATION_AUTO : TSA_MITIGATION_NONE; 1693 1694 static int __init tsa_parse_cmdline(char *str) 1695 { 1696 if (!str) 1697 return -EINVAL; 1698 1699 if (!strcmp(str, "off")) 1700 tsa_mitigation = TSA_MITIGATION_NONE; 1701 else if (!strcmp(str, "on")) 1702 tsa_mitigation = TSA_MITIGATION_FULL; 1703 else if (!strcmp(str, "user")) 1704 tsa_mitigation = TSA_MITIGATION_USER_KERNEL; 1705 else if (!strcmp(str, "vm")) 1706 tsa_mitigation = TSA_MITIGATION_VM; 1707 else 1708 pr_err("Ignoring unknown tsa=%s option.\n", str); 1709 1710 return 0; 1711 } 1712 early_param("tsa", tsa_parse_cmdline); 1713 1714 static void __init tsa_select_mitigation(void) 1715 { 1716 if (!boot_cpu_has_bug(X86_BUG_TSA)) { 1717 tsa_mitigation = TSA_MITIGATION_NONE; 1718 return; 1719 } 1720 1721 if (tsa_mitigation == TSA_MITIGATION_AUTO) { 1722 bool vm = false, uk = false; 1723 1724 tsa_mitigation = TSA_MITIGATION_NONE; 1725 1726 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) || 1727 cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) { 1728 tsa_mitigation = TSA_MITIGATION_USER_KERNEL; 1729 uk = true; 1730 } 1731 1732 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 1733 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) { 1734 tsa_mitigation = TSA_MITIGATION_VM; 1735 vm = true; 1736 } 1737 1738 if (uk && vm) 1739 tsa_mitigation = TSA_MITIGATION_FULL; 1740 } 1741 1742 if (tsa_mitigation == TSA_MITIGATION_NONE) 1743 return; 1744 1745 if (!boot_cpu_has(X86_FEATURE_VERW_CLEAR)) 1746 tsa_mitigation = TSA_MITIGATION_UCODE_NEEDED; 1747 1748 /* 1749 * No need to set verw_clear_cpu_buf_mitigation_selected - it 1750 * doesn't fit all cases here and it is not needed because this 1751 * is the only VERW-based mitigation on AMD. 1752 */ 1753 pr_info("%s\n", tsa_strings[tsa_mitigation]); 1754 } 1755 1756 static void __init tsa_apply_mitigation(void) 1757 { 1758 switch (tsa_mitigation) { 1759 case TSA_MITIGATION_USER_KERNEL: 1760 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 1761 break; 1762 case TSA_MITIGATION_VM: 1763 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 1764 break; 1765 case TSA_MITIGATION_FULL: 1766 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 1767 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM); 1768 break; 1769 default: 1770 break; 1771 } 1772 } 1773 1774 #undef pr_fmt 1775 #define pr_fmt(fmt) "Spectre V2 : " fmt 1776 1777 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1778 SPECTRE_V2_USER_NONE; 1779 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1780 SPECTRE_V2_USER_NONE; 1781 1782 #ifdef CONFIG_MITIGATION_RETPOLINE 1783 static bool spectre_v2_bad_module; 1784 1785 bool retpoline_module_ok(bool has_retpoline) 1786 { 1787 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1788 return true; 1789 1790 pr_err("System may be vulnerable to spectre v2\n"); 1791 spectre_v2_bad_module = true; 1792 return false; 1793 } 1794 1795 static inline const char *spectre_v2_module_string(void) 1796 { 1797 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1798 } 1799 #else 1800 static inline const char *spectre_v2_module_string(void) { return ""; } 1801 #endif 1802 1803 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1804 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1805 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1806 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1807 1808 #ifdef CONFIG_BPF_SYSCALL 1809 void unpriv_ebpf_notify(int new_state) 1810 { 1811 if (new_state) 1812 return; 1813 1814 /* Unprivileged eBPF is enabled */ 1815 1816 switch (spectre_v2_enabled) { 1817 case SPECTRE_V2_EIBRS: 1818 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1819 break; 1820 case SPECTRE_V2_EIBRS_LFENCE: 1821 if (sched_smt_active()) 1822 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1823 break; 1824 default: 1825 break; 1826 } 1827 } 1828 #endif 1829 1830 /* The kernel command line selection for spectre v2 */ 1831 enum spectre_v2_mitigation_cmd { 1832 SPECTRE_V2_CMD_NONE, 1833 SPECTRE_V2_CMD_AUTO, 1834 SPECTRE_V2_CMD_FORCE, 1835 SPECTRE_V2_CMD_RETPOLINE, 1836 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1837 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1838 SPECTRE_V2_CMD_EIBRS, 1839 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1840 SPECTRE_V2_CMD_EIBRS_LFENCE, 1841 SPECTRE_V2_CMD_IBRS, 1842 }; 1843 1844 static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init = 1845 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE; 1846 1847 enum spectre_v2_user_mitigation_cmd { 1848 SPECTRE_V2_USER_CMD_NONE, 1849 SPECTRE_V2_USER_CMD_AUTO, 1850 SPECTRE_V2_USER_CMD_FORCE, 1851 SPECTRE_V2_USER_CMD_PRCTL, 1852 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1853 SPECTRE_V2_USER_CMD_SECCOMP, 1854 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1855 }; 1856 1857 static enum spectre_v2_user_mitigation_cmd spectre_v2_user_cmd __ro_after_init = 1858 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE; 1859 1860 static const char * const spectre_v2_user_strings[] = { 1861 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1862 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1863 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1864 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1865 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1866 }; 1867 1868 static int __init spectre_v2_user_parse_cmdline(char *str) 1869 { 1870 if (!str) 1871 return -EINVAL; 1872 1873 if (!strcmp(str, "auto")) 1874 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_AUTO; 1875 else if (!strcmp(str, "off")) 1876 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_NONE; 1877 else if (!strcmp(str, "on")) 1878 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_FORCE; 1879 else if (!strcmp(str, "prctl")) 1880 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL; 1881 else if (!strcmp(str, "prctl,ibpb")) 1882 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL_IBPB; 1883 else if (!strcmp(str, "seccomp")) 1884 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP; 1885 else if (!strcmp(str, "seccomp,ibpb")) 1886 spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP_IBPB; 1887 else 1888 pr_err("Ignoring unknown spectre_v2_user option (%s).", str); 1889 1890 return 0; 1891 } 1892 early_param("spectre_v2_user", spectre_v2_user_parse_cmdline); 1893 1894 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1895 { 1896 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1897 } 1898 1899 static void __init spectre_v2_user_select_mitigation(void) 1900 { 1901 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1902 return; 1903 1904 switch (spectre_v2_user_cmd) { 1905 case SPECTRE_V2_USER_CMD_NONE: 1906 return; 1907 case SPECTRE_V2_USER_CMD_FORCE: 1908 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1909 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1910 break; 1911 case SPECTRE_V2_USER_CMD_AUTO: 1912 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2_USER)) 1913 break; 1914 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1915 if (smt_mitigations == SMT_MITIGATIONS_OFF) 1916 break; 1917 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1918 break; 1919 case SPECTRE_V2_USER_CMD_PRCTL: 1920 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1921 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1922 break; 1923 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1924 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1925 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1926 break; 1927 case SPECTRE_V2_USER_CMD_SECCOMP: 1928 if (IS_ENABLED(CONFIG_SECCOMP)) 1929 spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP; 1930 else 1931 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1932 spectre_v2_user_stibp = spectre_v2_user_ibpb; 1933 break; 1934 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1935 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1936 if (IS_ENABLED(CONFIG_SECCOMP)) 1937 spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP; 1938 else 1939 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1940 break; 1941 } 1942 1943 /* 1944 * At this point, an STIBP mode other than "off" has been set. 1945 * If STIBP support is not being forced, check if STIBP always-on 1946 * is preferred. 1947 */ 1948 if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1949 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) && 1950 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1951 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1952 1953 if (!boot_cpu_has(X86_FEATURE_IBPB)) 1954 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1955 1956 if (!boot_cpu_has(X86_FEATURE_STIBP)) 1957 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1958 } 1959 1960 static void __init spectre_v2_user_update_mitigation(void) 1961 { 1962 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1963 return; 1964 1965 /* The spectre_v2 cmd line can override spectre_v2_user options */ 1966 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) { 1967 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1968 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1969 } else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) { 1970 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1971 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1972 } 1973 1974 /* 1975 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1976 * is not required. 1977 * 1978 * Intel's Enhanced IBRS also protects against cross-thread branch target 1979 * injection in user-mode as the IBRS bit remains always set which 1980 * implicitly enables cross-thread protections. However, in legacy IBRS 1981 * mode, the IBRS bit is set only on kernel entry and cleared on return 1982 * to userspace. AMD Automatic IBRS also does not protect userspace. 1983 * These modes therefore disable the implicit cross-thread protection, 1984 * so allow for STIBP to be selected in those cases. 1985 */ 1986 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1987 !cpu_smt_possible() || 1988 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1989 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) { 1990 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1991 return; 1992 } 1993 1994 if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE && 1995 (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1996 retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) { 1997 if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT && 1998 spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED) 1999 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 2000 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 2001 } 2002 pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]); 2003 } 2004 2005 static void __init spectre_v2_user_apply_mitigation(void) 2006 { 2007 /* Initialize Indirect Branch Prediction Barrier */ 2008 if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) { 2009 static_branch_enable(&switch_vcpu_ibpb); 2010 2011 switch (spectre_v2_user_ibpb) { 2012 case SPECTRE_V2_USER_STRICT: 2013 static_branch_enable(&switch_mm_always_ibpb); 2014 break; 2015 case SPECTRE_V2_USER_PRCTL: 2016 case SPECTRE_V2_USER_SECCOMP: 2017 static_branch_enable(&switch_mm_cond_ibpb); 2018 break; 2019 default: 2020 break; 2021 } 2022 2023 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 2024 static_key_enabled(&switch_mm_always_ibpb) ? 2025 "always-on" : "conditional"); 2026 } 2027 } 2028 2029 static const char * const spectre_v2_strings[] = { 2030 [SPECTRE_V2_NONE] = "Vulnerable", 2031 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 2032 [SPECTRE_V2_LFENCE] = "Vulnerable: LFENCE", 2033 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 2034 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 2035 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 2036 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 2037 }; 2038 2039 static bool nospectre_v2 __ro_after_init; 2040 2041 static int __init nospectre_v2_parse_cmdline(char *str) 2042 { 2043 nospectre_v2 = true; 2044 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 2045 return 0; 2046 } 2047 early_param("nospectre_v2", nospectre_v2_parse_cmdline); 2048 2049 static int __init spectre_v2_parse_cmdline(char *str) 2050 { 2051 if (!str) 2052 return -EINVAL; 2053 2054 if (nospectre_v2) 2055 return 0; 2056 2057 if (!strcmp(str, "off")) { 2058 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 2059 } else if (!strcmp(str, "on")) { 2060 spectre_v2_cmd = SPECTRE_V2_CMD_FORCE; 2061 setup_force_cpu_bug(X86_BUG_SPECTRE_V2); 2062 setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER); 2063 } else if (!strcmp(str, "retpoline")) { 2064 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE; 2065 } else if (!strcmp(str, "retpoline,amd") || 2066 !strcmp(str, "retpoline,lfence")) { 2067 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_LFENCE; 2068 } else if (!strcmp(str, "retpoline,generic")) { 2069 spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_GENERIC; 2070 } else if (!strcmp(str, "eibrs")) { 2071 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS; 2072 } else if (!strcmp(str, "eibrs,lfence")) { 2073 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_LFENCE; 2074 } else if (!strcmp(str, "eibrs,retpoline")) { 2075 spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_RETPOLINE; 2076 } else if (!strcmp(str, "auto")) { 2077 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2078 } else if (!strcmp(str, "ibrs")) { 2079 spectre_v2_cmd = SPECTRE_V2_CMD_IBRS; 2080 } else { 2081 pr_err("Ignoring unknown spectre_v2 option (%s).", str); 2082 } 2083 2084 return 0; 2085 } 2086 early_param("spectre_v2", spectre_v2_parse_cmdline); 2087 2088 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 2089 { 2090 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 2091 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 2092 return SPECTRE_V2_NONE; 2093 } 2094 2095 return SPECTRE_V2_RETPOLINE; 2096 } 2097 2098 static bool __ro_after_init rrsba_disabled; 2099 2100 /* Disable in-kernel use of non-RSB RET predictors */ 2101 static void __init spec_ctrl_disable_kernel_rrsba(void) 2102 { 2103 if (rrsba_disabled) 2104 return; 2105 2106 if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) { 2107 rrsba_disabled = true; 2108 return; 2109 } 2110 2111 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 2112 return; 2113 2114 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 2115 update_spec_ctrl(x86_spec_ctrl_base); 2116 rrsba_disabled = true; 2117 } 2118 2119 static void __init spectre_v2_select_rsb_mitigation(enum spectre_v2_mitigation mode) 2120 { 2121 /* 2122 * WARNING! There are many subtleties to consider when changing *any* 2123 * code related to RSB-related mitigations. Before doing so, carefully 2124 * read the following document, and update if necessary: 2125 * 2126 * Documentation/admin-guide/hw-vuln/rsb.rst 2127 * 2128 * In an overly simplified nutshell: 2129 * 2130 * - User->user RSB attacks are conditionally mitigated during 2131 * context switches by cond_mitigation -> write_ibpb(). 2132 * 2133 * - User->kernel and guest->host attacks are mitigated by eIBRS or 2134 * RSB filling. 2135 * 2136 * Though, depending on config, note that other alternative 2137 * mitigations may end up getting used instead, e.g., IBPB on 2138 * entry/vmexit, call depth tracking, or return thunks. 2139 */ 2140 2141 switch (mode) { 2142 case SPECTRE_V2_NONE: 2143 break; 2144 2145 case SPECTRE_V2_EIBRS: 2146 case SPECTRE_V2_EIBRS_LFENCE: 2147 case SPECTRE_V2_EIBRS_RETPOLINE: 2148 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2149 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 2150 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 2151 } 2152 break; 2153 2154 case SPECTRE_V2_RETPOLINE: 2155 case SPECTRE_V2_LFENCE: 2156 case SPECTRE_V2_IBRS: 2157 pr_info("Spectre v2 / SpectreRSB: Filling RSB on context switch and VMEXIT\n"); 2158 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 2159 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 2160 break; 2161 2162 default: 2163 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation\n"); 2164 dump_stack(); 2165 break; 2166 } 2167 } 2168 2169 /* 2170 * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by 2171 * branch history in userspace. Not needed if BHI_NO is set. 2172 */ 2173 static bool __init spec_ctrl_bhi_dis(void) 2174 { 2175 if (!boot_cpu_has(X86_FEATURE_BHI_CTRL)) 2176 return false; 2177 2178 x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S; 2179 update_spec_ctrl(x86_spec_ctrl_base); 2180 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW); 2181 2182 return true; 2183 } 2184 2185 enum bhi_mitigations { 2186 BHI_MITIGATION_OFF, 2187 BHI_MITIGATION_AUTO, 2188 BHI_MITIGATION_ON, 2189 BHI_MITIGATION_VMEXIT_ONLY, 2190 }; 2191 2192 static enum bhi_mitigations bhi_mitigation __ro_after_init = 2193 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF; 2194 2195 static int __init spectre_bhi_parse_cmdline(char *str) 2196 { 2197 if (!str) 2198 return -EINVAL; 2199 2200 if (!strcmp(str, "off")) 2201 bhi_mitigation = BHI_MITIGATION_OFF; 2202 else if (!strcmp(str, "on")) 2203 bhi_mitigation = BHI_MITIGATION_ON; 2204 else if (!strcmp(str, "vmexit")) 2205 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 2206 else 2207 pr_err("Ignoring unknown spectre_bhi option (%s)", str); 2208 2209 return 0; 2210 } 2211 early_param("spectre_bhi", spectre_bhi_parse_cmdline); 2212 2213 static void __init bhi_select_mitigation(void) 2214 { 2215 if (!boot_cpu_has(X86_BUG_BHI)) 2216 bhi_mitigation = BHI_MITIGATION_OFF; 2217 2218 if (bhi_mitigation != BHI_MITIGATION_AUTO) 2219 return; 2220 2221 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST)) { 2222 if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL)) 2223 bhi_mitigation = BHI_MITIGATION_ON; 2224 else 2225 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 2226 } else { 2227 bhi_mitigation = BHI_MITIGATION_OFF; 2228 } 2229 } 2230 2231 static void __init bhi_update_mitigation(void) 2232 { 2233 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) 2234 bhi_mitigation = BHI_MITIGATION_OFF; 2235 } 2236 2237 static void __init bhi_apply_mitigation(void) 2238 { 2239 if (bhi_mitigation == BHI_MITIGATION_OFF) 2240 return; 2241 2242 /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ 2243 if (boot_cpu_has(X86_FEATURE_RETPOLINE) && 2244 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { 2245 spec_ctrl_disable_kernel_rrsba(); 2246 if (rrsba_disabled) 2247 return; 2248 } 2249 2250 if (!IS_ENABLED(CONFIG_X86_64)) 2251 return; 2252 2253 /* Mitigate in hardware if supported */ 2254 if (spec_ctrl_bhi_dis()) 2255 return; 2256 2257 if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) { 2258 pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n"); 2259 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2260 return; 2261 } 2262 2263 pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n"); 2264 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP); 2265 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2266 } 2267 2268 static void __init spectre_v2_select_mitigation(void) 2269 { 2270 if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE || 2271 spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 2272 spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 2273 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 2274 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 2275 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 2276 pr_err("RETPOLINE selected but not compiled in. Switching to AUTO select\n"); 2277 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2278 } 2279 2280 if ((spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS || 2281 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 2282 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 2283 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2284 pr_err("EIBRS selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n"); 2285 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2286 } 2287 2288 if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 2289 spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 2290 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 2291 pr_err("LFENCE selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n"); 2292 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2293 } 2294 2295 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) { 2296 pr_err("IBRS selected but not compiled in. Switching to AUTO select\n"); 2297 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2298 } 2299 2300 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 2301 pr_err("IBRS selected but not Intel CPU. Switching to AUTO select\n"); 2302 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2303 } 2304 2305 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 2306 pr_err("IBRS selected but CPU doesn't have IBRS. Switching to AUTO select\n"); 2307 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2308 } 2309 2310 if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 2311 pr_err("IBRS selected but running as XenPV guest. Switching to AUTO select\n"); 2312 spectre_v2_cmd = SPECTRE_V2_CMD_AUTO; 2313 } 2314 2315 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) { 2316 spectre_v2_cmd = SPECTRE_V2_CMD_NONE; 2317 return; 2318 } 2319 2320 switch (spectre_v2_cmd) { 2321 case SPECTRE_V2_CMD_NONE: 2322 return; 2323 2324 case SPECTRE_V2_CMD_AUTO: 2325 if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2)) 2326 break; 2327 fallthrough; 2328 case SPECTRE_V2_CMD_FORCE: 2329 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2330 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2331 break; 2332 } 2333 2334 spectre_v2_enabled = spectre_v2_select_retpoline(); 2335 break; 2336 2337 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 2338 pr_err(SPECTRE_V2_LFENCE_MSG); 2339 spectre_v2_enabled = SPECTRE_V2_LFENCE; 2340 break; 2341 2342 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 2343 spectre_v2_enabled = SPECTRE_V2_RETPOLINE; 2344 break; 2345 2346 case SPECTRE_V2_CMD_RETPOLINE: 2347 spectre_v2_enabled = spectre_v2_select_retpoline(); 2348 break; 2349 2350 case SPECTRE_V2_CMD_IBRS: 2351 spectre_v2_enabled = SPECTRE_V2_IBRS; 2352 break; 2353 2354 case SPECTRE_V2_CMD_EIBRS: 2355 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2356 break; 2357 2358 case SPECTRE_V2_CMD_EIBRS_LFENCE: 2359 spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE; 2360 break; 2361 2362 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 2363 spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE; 2364 break; 2365 } 2366 } 2367 2368 static void __init spectre_v2_update_mitigation(void) 2369 { 2370 if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO && 2371 !spectre_v2_in_eibrs_mode(spectre_v2_enabled)) { 2372 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && 2373 boot_cpu_has_bug(X86_BUG_RETBLEED) && 2374 retbleed_mitigation != RETBLEED_MITIGATION_NONE && 2375 retbleed_mitigation != RETBLEED_MITIGATION_STUFF && 2376 boot_cpu_has(X86_FEATURE_IBRS) && 2377 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 2378 spectre_v2_enabled = SPECTRE_V2_IBRS; 2379 } 2380 } 2381 2382 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2383 pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); 2384 } 2385 2386 static void __init spectre_v2_apply_mitigation(void) 2387 { 2388 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2389 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 2390 2391 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2392 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 2393 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 2394 } else { 2395 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 2396 update_spec_ctrl(x86_spec_ctrl_base); 2397 } 2398 } 2399 2400 switch (spectre_v2_enabled) { 2401 case SPECTRE_V2_NONE: 2402 return; 2403 2404 case SPECTRE_V2_EIBRS: 2405 break; 2406 2407 case SPECTRE_V2_IBRS: 2408 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 2409 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 2410 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 2411 break; 2412 2413 case SPECTRE_V2_LFENCE: 2414 case SPECTRE_V2_EIBRS_LFENCE: 2415 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 2416 fallthrough; 2417 2418 case SPECTRE_V2_RETPOLINE: 2419 case SPECTRE_V2_EIBRS_RETPOLINE: 2420 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 2421 break; 2422 } 2423 2424 /* 2425 * Disable alternate RSB predictions in kernel when indirect CALLs and 2426 * JMPs gets protection against BHI and Intramode-BTI, but RET 2427 * prediction from a non-RSB predictor is still a risk. 2428 */ 2429 if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE || 2430 spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE || 2431 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) 2432 spec_ctrl_disable_kernel_rrsba(); 2433 2434 spectre_v2_select_rsb_mitigation(spectre_v2_enabled); 2435 2436 /* 2437 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 2438 * and Enhanced IBRS protect firmware too, so enable IBRS around 2439 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 2440 * otherwise enabled. 2441 * 2442 * Use "spectre_v2_enabled" to check Enhanced IBRS instead of 2443 * boot_cpu_has(), because the user might select retpoline on the kernel 2444 * command line and if the CPU supports Enhanced IBRS, kernel might 2445 * un-intentionally not enable IBRS around firmware calls. 2446 */ 2447 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 2448 boot_cpu_has(X86_FEATURE_IBPB) && 2449 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2450 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 2451 2452 if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) { 2453 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 2454 pr_info("Enabling Speculation Barrier for firmware calls\n"); 2455 } 2456 2457 } else if (boot_cpu_has(X86_FEATURE_IBRS) && 2458 !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2459 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 2460 pr_info("Enabling Restricted Speculation for firmware calls\n"); 2461 } 2462 } 2463 2464 static void update_stibp_msr(void * __unused) 2465 { 2466 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 2467 update_spec_ctrl(val); 2468 } 2469 2470 /* Update x86_spec_ctrl_base in case SMT state changed. */ 2471 static void update_stibp_strict(void) 2472 { 2473 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 2474 2475 if (sched_smt_active()) 2476 mask |= SPEC_CTRL_STIBP; 2477 2478 if (mask == x86_spec_ctrl_base) 2479 return; 2480 2481 pr_info("Update user space SMT mitigation: STIBP %s\n", 2482 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 2483 x86_spec_ctrl_base = mask; 2484 on_each_cpu(update_stibp_msr, NULL, 1); 2485 } 2486 2487 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 2488 static void update_indir_branch_cond(void) 2489 { 2490 if (sched_smt_active()) 2491 static_branch_enable(&switch_to_cond_stibp); 2492 else 2493 static_branch_disable(&switch_to_cond_stibp); 2494 } 2495 2496 #undef pr_fmt 2497 #define pr_fmt(fmt) fmt 2498 2499 /* Update the static key controlling the MDS CPU buffer clear in idle */ 2500 static void update_mds_branch_idle(void) 2501 { 2502 /* 2503 * Enable the idle clearing if SMT is active on CPUs which are 2504 * affected only by MSBDS and not any other MDS variant. 2505 * 2506 * The other variants cannot be mitigated when SMT is enabled, so 2507 * clearing the buffers on idle just to prevent the Store Buffer 2508 * repartitioning leak would be a window dressing exercise. 2509 */ 2510 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 2511 return; 2512 2513 if (sched_smt_active()) { 2514 static_branch_enable(&cpu_buf_idle_clear); 2515 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 2516 (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) { 2517 static_branch_disable(&cpu_buf_idle_clear); 2518 } 2519 } 2520 2521 #undef pr_fmt 2522 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 2523 2524 static enum ssb_mitigation ssb_mode __ro_after_init = 2525 IS_ENABLED(CONFIG_MITIGATION_SSB) ? SPEC_STORE_BYPASS_AUTO : SPEC_STORE_BYPASS_NONE; 2526 2527 static const char * const ssb_strings[] = { 2528 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 2529 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 2530 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 2531 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 2532 }; 2533 2534 static bool nossb __ro_after_init; 2535 2536 static int __init nossb_parse_cmdline(char *str) 2537 { 2538 nossb = true; 2539 ssb_mode = SPEC_STORE_BYPASS_NONE; 2540 return 0; 2541 } 2542 early_param("nospec_store_bypass_disable", nossb_parse_cmdline); 2543 2544 static int __init ssb_parse_cmdline(char *str) 2545 { 2546 if (!str) 2547 return -EINVAL; 2548 2549 if (nossb) 2550 return 0; 2551 2552 if (!strcmp(str, "auto")) 2553 ssb_mode = SPEC_STORE_BYPASS_AUTO; 2554 else if (!strcmp(str, "on")) 2555 ssb_mode = SPEC_STORE_BYPASS_DISABLE; 2556 else if (!strcmp(str, "off")) 2557 ssb_mode = SPEC_STORE_BYPASS_NONE; 2558 else if (!strcmp(str, "prctl")) 2559 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2560 else if (!strcmp(str, "seccomp")) 2561 ssb_mode = IS_ENABLED(CONFIG_SECCOMP) ? 2562 SPEC_STORE_BYPASS_SECCOMP : SPEC_STORE_BYPASS_PRCTL; 2563 else 2564 pr_err("Ignoring unknown spec_store_bypass_disable option (%s).\n", 2565 str); 2566 2567 return 0; 2568 } 2569 early_param("spec_store_bypass_disable", ssb_parse_cmdline); 2570 2571 static void __init ssb_select_mitigation(void) 2572 { 2573 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) { 2574 ssb_mode = SPEC_STORE_BYPASS_NONE; 2575 return; 2576 } 2577 2578 if (ssb_mode == SPEC_STORE_BYPASS_AUTO) { 2579 if (should_mitigate_vuln(X86_BUG_SPEC_STORE_BYPASS)) 2580 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2581 else 2582 ssb_mode = SPEC_STORE_BYPASS_NONE; 2583 } 2584 2585 if (!boot_cpu_has(X86_FEATURE_SSBD)) 2586 ssb_mode = SPEC_STORE_BYPASS_NONE; 2587 2588 pr_info("%s\n", ssb_strings[ssb_mode]); 2589 } 2590 2591 static void __init ssb_apply_mitigation(void) 2592 { 2593 /* 2594 * We have three CPU feature flags that are in play here: 2595 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 2596 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 2597 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 2598 */ 2599 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) { 2600 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 2601 /* 2602 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 2603 * use a completely different MSR and bit dependent on family. 2604 */ 2605 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 2606 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 2607 x86_amd_ssb_disable(); 2608 } else { 2609 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 2610 update_spec_ctrl(x86_spec_ctrl_base); 2611 } 2612 } 2613 } 2614 2615 #undef pr_fmt 2616 #define pr_fmt(fmt) "Speculation prctl: " fmt 2617 2618 static void task_update_spec_tif(struct task_struct *tsk) 2619 { 2620 /* Force the update of the real TIF bits */ 2621 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 2622 2623 /* 2624 * Immediately update the speculation control MSRs for the current 2625 * task, but for a non-current task delay setting the CPU 2626 * mitigation until it is scheduled next. 2627 * 2628 * This can only happen for SECCOMP mitigation. For PRCTL it's 2629 * always the current task. 2630 */ 2631 if (tsk == current) 2632 speculation_ctrl_update_current(); 2633 } 2634 2635 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 2636 { 2637 2638 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2639 return -EPERM; 2640 2641 switch (ctrl) { 2642 case PR_SPEC_ENABLE: 2643 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2644 return 0; 2645 case PR_SPEC_DISABLE: 2646 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2647 return 0; 2648 default: 2649 return -ERANGE; 2650 } 2651 } 2652 2653 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2654 { 2655 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2656 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2657 return -ENXIO; 2658 2659 switch (ctrl) { 2660 case PR_SPEC_ENABLE: 2661 /* If speculation is force disabled, enable is not allowed */ 2662 if (task_spec_ssb_force_disable(task)) 2663 return -EPERM; 2664 task_clear_spec_ssb_disable(task); 2665 task_clear_spec_ssb_noexec(task); 2666 task_update_spec_tif(task); 2667 break; 2668 case PR_SPEC_DISABLE: 2669 task_set_spec_ssb_disable(task); 2670 task_clear_spec_ssb_noexec(task); 2671 task_update_spec_tif(task); 2672 break; 2673 case PR_SPEC_FORCE_DISABLE: 2674 task_set_spec_ssb_disable(task); 2675 task_set_spec_ssb_force_disable(task); 2676 task_clear_spec_ssb_noexec(task); 2677 task_update_spec_tif(task); 2678 break; 2679 case PR_SPEC_DISABLE_NOEXEC: 2680 if (task_spec_ssb_force_disable(task)) 2681 return -EPERM; 2682 task_set_spec_ssb_disable(task); 2683 task_set_spec_ssb_noexec(task); 2684 task_update_spec_tif(task); 2685 break; 2686 default: 2687 return -ERANGE; 2688 } 2689 return 0; 2690 } 2691 2692 static bool is_spec_ib_user_controlled(void) 2693 { 2694 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2695 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2696 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2697 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2698 } 2699 2700 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2701 { 2702 switch (ctrl) { 2703 case PR_SPEC_ENABLE: 2704 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2705 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2706 return 0; 2707 2708 /* 2709 * With strict mode for both IBPB and STIBP, the instruction 2710 * code paths avoid checking this task flag and instead, 2711 * unconditionally run the instruction. However, STIBP and IBPB 2712 * are independent and either can be set to conditionally 2713 * enabled regardless of the mode of the other. 2714 * 2715 * If either is set to conditional, allow the task flag to be 2716 * updated, unless it was force-disabled by a previous prctl 2717 * call. Currently, this is possible on an AMD CPU which has the 2718 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2719 * kernel is booted with 'spectre_v2_user=seccomp', then 2720 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2721 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2722 */ 2723 if (!is_spec_ib_user_controlled() || 2724 task_spec_ib_force_disable(task)) 2725 return -EPERM; 2726 2727 task_clear_spec_ib_disable(task); 2728 task_update_spec_tif(task); 2729 break; 2730 case PR_SPEC_DISABLE: 2731 case PR_SPEC_FORCE_DISABLE: 2732 /* 2733 * Indirect branch speculation is always allowed when 2734 * mitigation is force disabled. 2735 */ 2736 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2737 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2738 return -EPERM; 2739 2740 if (!is_spec_ib_user_controlled()) 2741 return 0; 2742 2743 task_set_spec_ib_disable(task); 2744 if (ctrl == PR_SPEC_FORCE_DISABLE) 2745 task_set_spec_ib_force_disable(task); 2746 task_update_spec_tif(task); 2747 if (task == current) 2748 indirect_branch_prediction_barrier(); 2749 break; 2750 default: 2751 return -ERANGE; 2752 } 2753 return 0; 2754 } 2755 2756 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2757 unsigned long ctrl) 2758 { 2759 switch (which) { 2760 case PR_SPEC_STORE_BYPASS: 2761 return ssb_prctl_set(task, ctrl); 2762 case PR_SPEC_INDIRECT_BRANCH: 2763 return ib_prctl_set(task, ctrl); 2764 case PR_SPEC_L1D_FLUSH: 2765 return l1d_flush_prctl_set(task, ctrl); 2766 default: 2767 return -ENODEV; 2768 } 2769 } 2770 2771 #ifdef CONFIG_SECCOMP 2772 void arch_seccomp_spec_mitigate(struct task_struct *task) 2773 { 2774 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2775 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2776 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2777 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2778 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2779 } 2780 #endif 2781 2782 static int l1d_flush_prctl_get(struct task_struct *task) 2783 { 2784 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2785 return PR_SPEC_FORCE_DISABLE; 2786 2787 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2788 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2789 else 2790 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2791 } 2792 2793 static int ssb_prctl_get(struct task_struct *task) 2794 { 2795 switch (ssb_mode) { 2796 case SPEC_STORE_BYPASS_NONE: 2797 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2798 return PR_SPEC_ENABLE; 2799 return PR_SPEC_NOT_AFFECTED; 2800 case SPEC_STORE_BYPASS_DISABLE: 2801 return PR_SPEC_DISABLE; 2802 case SPEC_STORE_BYPASS_SECCOMP: 2803 case SPEC_STORE_BYPASS_PRCTL: 2804 case SPEC_STORE_BYPASS_AUTO: 2805 if (task_spec_ssb_force_disable(task)) 2806 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2807 if (task_spec_ssb_noexec(task)) 2808 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2809 if (task_spec_ssb_disable(task)) 2810 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2811 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2812 } 2813 BUG(); 2814 } 2815 2816 static int ib_prctl_get(struct task_struct *task) 2817 { 2818 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2819 return PR_SPEC_NOT_AFFECTED; 2820 2821 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2822 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2823 return PR_SPEC_ENABLE; 2824 else if (is_spec_ib_user_controlled()) { 2825 if (task_spec_ib_force_disable(task)) 2826 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2827 if (task_spec_ib_disable(task)) 2828 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2829 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2830 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2831 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2832 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2833 return PR_SPEC_DISABLE; 2834 else 2835 return PR_SPEC_NOT_AFFECTED; 2836 } 2837 2838 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2839 { 2840 switch (which) { 2841 case PR_SPEC_STORE_BYPASS: 2842 return ssb_prctl_get(task); 2843 case PR_SPEC_INDIRECT_BRANCH: 2844 return ib_prctl_get(task); 2845 case PR_SPEC_L1D_FLUSH: 2846 return l1d_flush_prctl_get(task); 2847 default: 2848 return -ENODEV; 2849 } 2850 } 2851 2852 void x86_spec_ctrl_setup_ap(void) 2853 { 2854 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2855 update_spec_ctrl(x86_spec_ctrl_base); 2856 2857 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2858 x86_amd_ssb_disable(); 2859 } 2860 2861 bool itlb_multihit_kvm_mitigation; 2862 EXPORT_SYMBOL_FOR_KVM(itlb_multihit_kvm_mitigation); 2863 2864 #undef pr_fmt 2865 #define pr_fmt(fmt) "L1TF: " fmt 2866 2867 /* Default mitigation for L1TF-affected CPUs */ 2868 enum l1tf_mitigations l1tf_mitigation __ro_after_init = 2869 IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_AUTO : L1TF_MITIGATION_OFF; 2870 EXPORT_SYMBOL_FOR_KVM(l1tf_mitigation); 2871 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2872 EXPORT_SYMBOL_FOR_KVM(l1tf_vmx_mitigation); 2873 2874 /* 2875 * These CPUs all support 44bits physical address space internally in the 2876 * cache but CPUID can report a smaller number of physical address bits. 2877 * 2878 * The L1TF mitigation uses the top most address bit for the inversion of 2879 * non present PTEs. When the installed memory reaches into the top most 2880 * address bit due to memory holes, which has been observed on machines 2881 * which report 36bits physical address bits and have 32G RAM installed, 2882 * then the mitigation range check in l1tf_select_mitigation() triggers. 2883 * This is a false positive because the mitigation is still possible due to 2884 * the fact that the cache uses 44bit internally. Use the cache bits 2885 * instead of the reported physical bits and adjust them on the affected 2886 * machines to 44bit if the reported bits are less than 44. 2887 */ 2888 static void override_cache_bits(struct cpuinfo_x86 *c) 2889 { 2890 if (c->x86 != 6) 2891 return; 2892 2893 switch (c->x86_vfm) { 2894 case INTEL_NEHALEM: 2895 case INTEL_WESTMERE: 2896 case INTEL_SANDYBRIDGE: 2897 case INTEL_IVYBRIDGE: 2898 case INTEL_HASWELL: 2899 case INTEL_HASWELL_L: 2900 case INTEL_HASWELL_G: 2901 case INTEL_BROADWELL: 2902 case INTEL_BROADWELL_G: 2903 case INTEL_SKYLAKE_L: 2904 case INTEL_SKYLAKE: 2905 case INTEL_KABYLAKE_L: 2906 case INTEL_KABYLAKE: 2907 if (c->x86_cache_bits < 44) 2908 c->x86_cache_bits = 44; 2909 break; 2910 } 2911 } 2912 2913 static void __init l1tf_select_mitigation(void) 2914 { 2915 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 2916 l1tf_mitigation = L1TF_MITIGATION_OFF; 2917 return; 2918 } 2919 2920 if (l1tf_mitigation != L1TF_MITIGATION_AUTO) 2921 return; 2922 2923 if (!should_mitigate_vuln(X86_BUG_L1TF)) { 2924 l1tf_mitigation = L1TF_MITIGATION_OFF; 2925 return; 2926 } 2927 2928 if (smt_mitigations == SMT_MITIGATIONS_ON) 2929 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2930 else 2931 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2932 } 2933 2934 static void __init l1tf_apply_mitigation(void) 2935 { 2936 u64 half_pa; 2937 2938 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2939 return; 2940 2941 override_cache_bits(&boot_cpu_data); 2942 2943 switch (l1tf_mitigation) { 2944 case L1TF_MITIGATION_OFF: 2945 case L1TF_MITIGATION_FLUSH_NOWARN: 2946 case L1TF_MITIGATION_FLUSH: 2947 case L1TF_MITIGATION_AUTO: 2948 break; 2949 case L1TF_MITIGATION_FLUSH_NOSMT: 2950 case L1TF_MITIGATION_FULL: 2951 cpu_smt_disable(false); 2952 break; 2953 case L1TF_MITIGATION_FULL_FORCE: 2954 cpu_smt_disable(true); 2955 break; 2956 } 2957 2958 #if CONFIG_PGTABLE_LEVELS == 2 2959 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2960 return; 2961 #endif 2962 2963 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2964 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2965 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2966 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2967 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2968 half_pa); 2969 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2970 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2971 return; 2972 } 2973 2974 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2975 } 2976 2977 static int __init l1tf_cmdline(char *str) 2978 { 2979 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2980 return 0; 2981 2982 if (!str) 2983 return -EINVAL; 2984 2985 if (!strcmp(str, "off")) 2986 l1tf_mitigation = L1TF_MITIGATION_OFF; 2987 else if (!strcmp(str, "flush,nowarn")) 2988 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2989 else if (!strcmp(str, "flush")) 2990 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2991 else if (!strcmp(str, "flush,nosmt")) 2992 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2993 else if (!strcmp(str, "full")) 2994 l1tf_mitigation = L1TF_MITIGATION_FULL; 2995 else if (!strcmp(str, "full,force")) 2996 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2997 2998 return 0; 2999 } 3000 early_param("l1tf", l1tf_cmdline); 3001 3002 #undef pr_fmt 3003 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 3004 3005 static const char * const srso_strings[] = { 3006 [SRSO_MITIGATION_NONE] = "Vulnerable", 3007 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 3008 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 3009 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 3010 [SRSO_MITIGATION_NOSMT] = "Mitigation: SMT disabled", 3011 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 3012 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 3013 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only", 3014 [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation" 3015 }; 3016 3017 static int __init srso_parse_cmdline(char *str) 3018 { 3019 if (!str) 3020 return -EINVAL; 3021 3022 if (!strcmp(str, "off")) 3023 srso_mitigation = SRSO_MITIGATION_NONE; 3024 else if (!strcmp(str, "microcode")) 3025 srso_mitigation = SRSO_MITIGATION_MICROCODE; 3026 else if (!strcmp(str, "safe-ret")) 3027 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 3028 else if (!strcmp(str, "ibpb")) 3029 srso_mitigation = SRSO_MITIGATION_IBPB; 3030 else if (!strcmp(str, "ibpb-vmexit")) 3031 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 3032 else 3033 pr_err("Ignoring unknown SRSO option (%s).", str); 3034 3035 return 0; 3036 } 3037 early_param("spec_rstack_overflow", srso_parse_cmdline); 3038 3039 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 3040 3041 static void __init srso_select_mitigation(void) 3042 { 3043 if (!boot_cpu_has_bug(X86_BUG_SRSO)) { 3044 srso_mitigation = SRSO_MITIGATION_NONE; 3045 return; 3046 } 3047 3048 if (srso_mitigation == SRSO_MITIGATION_AUTO) { 3049 /* 3050 * Use safe-RET if user->kernel or guest->host protection is 3051 * required. Otherwise the 'microcode' mitigation is sufficient 3052 * to protect the user->user and guest->guest vectors. 3053 */ 3054 if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || 3055 (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) && 3056 !boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))) { 3057 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 3058 } else if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) || 3059 cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) { 3060 srso_mitigation = SRSO_MITIGATION_MICROCODE; 3061 } else { 3062 srso_mitigation = SRSO_MITIGATION_NONE; 3063 return; 3064 } 3065 } 3066 3067 /* Zen1/2 with SMT off aren't vulnerable to SRSO. */ 3068 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 3069 srso_mitigation = SRSO_MITIGATION_NOSMT; 3070 return; 3071 } 3072 3073 if (!boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) { 3074 pr_warn("IBPB-extending microcode not applied!\n"); 3075 pr_warn(SRSO_NOTICE); 3076 3077 /* 3078 * Safe-RET provides partial mitigation without microcode, but 3079 * other mitigations require microcode to provide any 3080 * mitigations. 3081 */ 3082 if (srso_mitigation == SRSO_MITIGATION_SAFE_RET) 3083 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 3084 else 3085 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 3086 } 3087 3088 switch (srso_mitigation) { 3089 case SRSO_MITIGATION_SAFE_RET: 3090 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 3091 if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) { 3092 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 3093 goto ibpb_on_vmexit; 3094 } 3095 3096 if (!IS_ENABLED(CONFIG_MITIGATION_SRSO)) { 3097 pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n"); 3098 srso_mitigation = SRSO_MITIGATION_NONE; 3099 } 3100 break; 3101 ibpb_on_vmexit: 3102 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3103 if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) { 3104 pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n"); 3105 srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE; 3106 break; 3107 } 3108 fallthrough; 3109 case SRSO_MITIGATION_IBPB: 3110 if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 3111 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 3112 srso_mitigation = SRSO_MITIGATION_NONE; 3113 } 3114 break; 3115 default: 3116 break; 3117 } 3118 } 3119 3120 static void __init srso_update_mitigation(void) 3121 { 3122 if (!boot_cpu_has_bug(X86_BUG_SRSO)) 3123 return; 3124 3125 /* If retbleed is using IBPB, that works for SRSO as well */ 3126 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB && 3127 boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) 3128 srso_mitigation = SRSO_MITIGATION_IBPB; 3129 3130 pr_info("%s\n", srso_strings[srso_mitigation]); 3131 } 3132 3133 static void __init srso_apply_mitigation(void) 3134 { 3135 /* 3136 * Clear the feature flag if this mitigation is not selected as that 3137 * feature flag controls the BpSpecReduce MSR bit toggling in KVM. 3138 */ 3139 if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE) 3140 setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE); 3141 3142 if (srso_mitigation == SRSO_MITIGATION_NONE) { 3143 if (boot_cpu_has(X86_FEATURE_SBPB)) 3144 x86_pred_cmd = PRED_CMD_SBPB; 3145 return; 3146 } 3147 3148 switch (srso_mitigation) { 3149 case SRSO_MITIGATION_SAFE_RET: 3150 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 3151 /* 3152 * Enable the return thunk for generated code 3153 * like ftrace, static_call, etc. 3154 */ 3155 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 3156 setup_force_cpu_cap(X86_FEATURE_UNRET); 3157 3158 if (boot_cpu_data.x86 == 0x19) { 3159 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 3160 set_return_thunk(srso_alias_return_thunk); 3161 } else { 3162 setup_force_cpu_cap(X86_FEATURE_SRSO); 3163 set_return_thunk(srso_return_thunk); 3164 } 3165 break; 3166 case SRSO_MITIGATION_IBPB: 3167 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 3168 /* 3169 * IBPB on entry already obviates the need for 3170 * software-based untraining so clear those in case some 3171 * other mitigation like Retbleed has selected them. 3172 */ 3173 setup_clear_cpu_cap(X86_FEATURE_UNRET); 3174 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 3175 fallthrough; 3176 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3177 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 3178 /* 3179 * There is no need for RSB filling: entry_ibpb() ensures 3180 * all predictions, including the RSB, are invalidated, 3181 * regardless of IBPB implementation. 3182 */ 3183 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 3184 break; 3185 default: 3186 break; 3187 } 3188 } 3189 3190 #undef pr_fmt 3191 #define pr_fmt(fmt) "VMSCAPE: " fmt 3192 3193 enum vmscape_mitigations { 3194 VMSCAPE_MITIGATION_NONE, 3195 VMSCAPE_MITIGATION_AUTO, 3196 VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER, 3197 VMSCAPE_MITIGATION_IBPB_ON_VMEXIT, 3198 }; 3199 3200 static const char * const vmscape_strings[] = { 3201 [VMSCAPE_MITIGATION_NONE] = "Vulnerable", 3202 /* [VMSCAPE_MITIGATION_AUTO] */ 3203 [VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER] = "Mitigation: IBPB before exit to userspace", 3204 [VMSCAPE_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT", 3205 }; 3206 3207 static enum vmscape_mitigations vmscape_mitigation __ro_after_init = 3208 IS_ENABLED(CONFIG_MITIGATION_VMSCAPE) ? VMSCAPE_MITIGATION_AUTO : VMSCAPE_MITIGATION_NONE; 3209 3210 static int __init vmscape_parse_cmdline(char *str) 3211 { 3212 if (!str) 3213 return -EINVAL; 3214 3215 if (!strcmp(str, "off")) { 3216 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3217 } else if (!strcmp(str, "ibpb")) { 3218 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; 3219 } else if (!strcmp(str, "force")) { 3220 setup_force_cpu_bug(X86_BUG_VMSCAPE); 3221 vmscape_mitigation = VMSCAPE_MITIGATION_AUTO; 3222 } else { 3223 pr_err("Ignoring unknown vmscape=%s option.\n", str); 3224 } 3225 3226 return 0; 3227 } 3228 early_param("vmscape", vmscape_parse_cmdline); 3229 3230 static void __init vmscape_select_mitigation(void) 3231 { 3232 if (!boot_cpu_has_bug(X86_BUG_VMSCAPE) || 3233 !boot_cpu_has(X86_FEATURE_IBPB)) { 3234 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3235 return; 3236 } 3237 3238 if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO) { 3239 if (should_mitigate_vuln(X86_BUG_VMSCAPE)) 3240 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER; 3241 else 3242 vmscape_mitigation = VMSCAPE_MITIGATION_NONE; 3243 } 3244 } 3245 3246 static void __init vmscape_update_mitigation(void) 3247 { 3248 if (!boot_cpu_has_bug(X86_BUG_VMSCAPE)) 3249 return; 3250 3251 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB || 3252 srso_mitigation == SRSO_MITIGATION_IBPB_ON_VMEXIT) 3253 vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_ON_VMEXIT; 3254 3255 pr_info("%s\n", vmscape_strings[vmscape_mitigation]); 3256 } 3257 3258 static void __init vmscape_apply_mitigation(void) 3259 { 3260 if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER) 3261 setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER); 3262 } 3263 3264 #undef pr_fmt 3265 #define pr_fmt(fmt) fmt 3266 3267 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 3268 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 3269 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 3270 #define VMSCAPE_MSG_SMT "VMSCAPE: SMT on, STIBP is required for full protection. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/vmscape.html for more details.\n" 3271 3272 void cpu_bugs_smt_update(void) 3273 { 3274 mutex_lock(&spec_ctrl_mutex); 3275 3276 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3277 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3278 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 3279 3280 switch (spectre_v2_user_stibp) { 3281 case SPECTRE_V2_USER_NONE: 3282 break; 3283 case SPECTRE_V2_USER_STRICT: 3284 case SPECTRE_V2_USER_STRICT_PREFERRED: 3285 update_stibp_strict(); 3286 break; 3287 case SPECTRE_V2_USER_PRCTL: 3288 case SPECTRE_V2_USER_SECCOMP: 3289 update_indir_branch_cond(); 3290 break; 3291 } 3292 3293 switch (mds_mitigation) { 3294 case MDS_MITIGATION_FULL: 3295 case MDS_MITIGATION_AUTO: 3296 case MDS_MITIGATION_VMWERV: 3297 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 3298 pr_warn_once(MDS_MSG_SMT); 3299 update_mds_branch_idle(); 3300 break; 3301 case MDS_MITIGATION_OFF: 3302 break; 3303 } 3304 3305 switch (taa_mitigation) { 3306 case TAA_MITIGATION_VERW: 3307 case TAA_MITIGATION_AUTO: 3308 case TAA_MITIGATION_UCODE_NEEDED: 3309 if (sched_smt_active()) 3310 pr_warn_once(TAA_MSG_SMT); 3311 break; 3312 case TAA_MITIGATION_TSX_DISABLED: 3313 case TAA_MITIGATION_OFF: 3314 break; 3315 } 3316 3317 switch (mmio_mitigation) { 3318 case MMIO_MITIGATION_VERW: 3319 case MMIO_MITIGATION_AUTO: 3320 case MMIO_MITIGATION_UCODE_NEEDED: 3321 if (sched_smt_active()) 3322 pr_warn_once(MMIO_MSG_SMT); 3323 break; 3324 case MMIO_MITIGATION_OFF: 3325 break; 3326 } 3327 3328 switch (tsa_mitigation) { 3329 case TSA_MITIGATION_USER_KERNEL: 3330 case TSA_MITIGATION_VM: 3331 case TSA_MITIGATION_AUTO: 3332 case TSA_MITIGATION_FULL: 3333 /* 3334 * TSA-SQ can potentially lead to info leakage between 3335 * SMT threads. 3336 */ 3337 if (sched_smt_active()) 3338 static_branch_enable(&cpu_buf_idle_clear); 3339 else 3340 static_branch_disable(&cpu_buf_idle_clear); 3341 break; 3342 case TSA_MITIGATION_NONE: 3343 case TSA_MITIGATION_UCODE_NEEDED: 3344 break; 3345 } 3346 3347 switch (vmscape_mitigation) { 3348 case VMSCAPE_MITIGATION_NONE: 3349 case VMSCAPE_MITIGATION_AUTO: 3350 break; 3351 case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT: 3352 case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER: 3353 /* 3354 * Hypervisors can be attacked across-threads, warn for SMT when 3355 * STIBP is not already enabled system-wide. 3356 * 3357 * Intel eIBRS (!AUTOIBRS) implies STIBP on. 3358 */ 3359 if (!sched_smt_active() || 3360 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3361 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED || 3362 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3363 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) 3364 break; 3365 pr_warn_once(VMSCAPE_MSG_SMT); 3366 break; 3367 } 3368 3369 mutex_unlock(&spec_ctrl_mutex); 3370 } 3371 3372 #ifdef CONFIG_SYSFS 3373 3374 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 3375 3376 #if IS_ENABLED(CONFIG_KVM_INTEL) 3377 static const char * const l1tf_vmx_states[] = { 3378 [VMENTER_L1D_FLUSH_AUTO] = "auto", 3379 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 3380 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 3381 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 3382 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 3383 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 3384 }; 3385 3386 static ssize_t l1tf_show_state(char *buf) 3387 { 3388 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 3389 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3390 3391 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 3392 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 3393 sched_smt_active())) { 3394 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 3395 l1tf_vmx_states[l1tf_vmx_mitigation]); 3396 } 3397 3398 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 3399 l1tf_vmx_states[l1tf_vmx_mitigation], 3400 sched_smt_active() ? "vulnerable" : "disabled"); 3401 } 3402 3403 static ssize_t itlb_multihit_show_state(char *buf) 3404 { 3405 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 3406 !boot_cpu_has(X86_FEATURE_VMX)) 3407 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 3408 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 3409 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 3410 else if (itlb_multihit_kvm_mitigation) 3411 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 3412 else 3413 return sysfs_emit(buf, "KVM: Vulnerable\n"); 3414 } 3415 #else 3416 static ssize_t l1tf_show_state(char *buf) 3417 { 3418 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3419 } 3420 3421 static ssize_t itlb_multihit_show_state(char *buf) 3422 { 3423 return sysfs_emit(buf, "Processor vulnerable\n"); 3424 } 3425 #endif 3426 3427 static ssize_t mds_show_state(char *buf) 3428 { 3429 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3430 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3431 mds_strings[mds_mitigation]); 3432 } 3433 3434 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 3435 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3436 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 3437 sched_smt_active() ? "mitigated" : "disabled")); 3438 } 3439 3440 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3441 sched_smt_active() ? "vulnerable" : "disabled"); 3442 } 3443 3444 static ssize_t tsx_async_abort_show_state(char *buf) 3445 { 3446 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 3447 (taa_mitigation == TAA_MITIGATION_OFF)) 3448 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 3449 3450 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3451 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3452 taa_strings[taa_mitigation]); 3453 } 3454 3455 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 3456 sched_smt_active() ? "vulnerable" : "disabled"); 3457 } 3458 3459 static ssize_t mmio_stale_data_show_state(char *buf) 3460 { 3461 if (mmio_mitigation == MMIO_MITIGATION_OFF) 3462 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 3463 3464 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3465 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3466 mmio_strings[mmio_mitigation]); 3467 } 3468 3469 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 3470 sched_smt_active() ? "vulnerable" : "disabled"); 3471 } 3472 3473 static ssize_t rfds_show_state(char *buf) 3474 { 3475 return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); 3476 } 3477 3478 static ssize_t old_microcode_show_state(char *buf) 3479 { 3480 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 3481 return sysfs_emit(buf, "Unknown: running under hypervisor"); 3482 3483 return sysfs_emit(buf, "Vulnerable\n"); 3484 } 3485 3486 static ssize_t its_show_state(char *buf) 3487 { 3488 return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]); 3489 } 3490 3491 static char *stibp_state(void) 3492 { 3493 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3494 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 3495 return ""; 3496 3497 switch (spectre_v2_user_stibp) { 3498 case SPECTRE_V2_USER_NONE: 3499 return "; STIBP: disabled"; 3500 case SPECTRE_V2_USER_STRICT: 3501 return "; STIBP: forced"; 3502 case SPECTRE_V2_USER_STRICT_PREFERRED: 3503 return "; STIBP: always-on"; 3504 case SPECTRE_V2_USER_PRCTL: 3505 case SPECTRE_V2_USER_SECCOMP: 3506 if (static_key_enabled(&switch_to_cond_stibp)) 3507 return "; STIBP: conditional"; 3508 } 3509 return ""; 3510 } 3511 3512 static char *ibpb_state(void) 3513 { 3514 if (boot_cpu_has(X86_FEATURE_IBPB)) { 3515 if (static_key_enabled(&switch_mm_always_ibpb)) 3516 return "; IBPB: always-on"; 3517 if (static_key_enabled(&switch_mm_cond_ibpb)) 3518 return "; IBPB: conditional"; 3519 return "; IBPB: disabled"; 3520 } 3521 return ""; 3522 } 3523 3524 static char *pbrsb_eibrs_state(void) 3525 { 3526 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 3527 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 3528 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 3529 return "; PBRSB-eIBRS: SW sequence"; 3530 else 3531 return "; PBRSB-eIBRS: Vulnerable"; 3532 } else { 3533 return "; PBRSB-eIBRS: Not affected"; 3534 } 3535 } 3536 3537 static const char *spectre_bhi_state(void) 3538 { 3539 if (!boot_cpu_has_bug(X86_BUG_BHI)) 3540 return "; BHI: Not affected"; 3541 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW)) 3542 return "; BHI: BHI_DIS_S"; 3543 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) 3544 return "; BHI: SW loop, KVM: SW loop"; 3545 else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && 3546 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) && 3547 rrsba_disabled) 3548 return "; BHI: Retpoline"; 3549 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) 3550 return "; BHI: Vulnerable, KVM: SW loop"; 3551 3552 return "; BHI: Vulnerable"; 3553 } 3554 3555 static ssize_t spectre_v2_show_state(char *buf) 3556 { 3557 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 3558 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 3559 3560 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3561 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3562 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 3563 3564 return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n", 3565 spectre_v2_strings[spectre_v2_enabled], 3566 ibpb_state(), 3567 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "", 3568 stibp_state(), 3569 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "", 3570 pbrsb_eibrs_state(), 3571 spectre_bhi_state(), 3572 /* this should always be at the end */ 3573 spectre_v2_module_string()); 3574 } 3575 3576 static ssize_t srbds_show_state(char *buf) 3577 { 3578 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 3579 } 3580 3581 static ssize_t retbleed_show_state(char *buf) 3582 { 3583 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 3584 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 3585 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 3586 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 3587 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 3588 3589 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 3590 !sched_smt_active() ? "disabled" : 3591 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3592 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 3593 "enabled with STIBP protection" : "vulnerable"); 3594 } 3595 3596 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 3597 } 3598 3599 static ssize_t srso_show_state(char *buf) 3600 { 3601 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 3602 } 3603 3604 static ssize_t gds_show_state(char *buf) 3605 { 3606 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 3607 } 3608 3609 static ssize_t tsa_show_state(char *buf) 3610 { 3611 return sysfs_emit(buf, "%s\n", tsa_strings[tsa_mitigation]); 3612 } 3613 3614 static ssize_t vmscape_show_state(char *buf) 3615 { 3616 return sysfs_emit(buf, "%s\n", vmscape_strings[vmscape_mitigation]); 3617 } 3618 3619 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 3620 char *buf, unsigned int bug) 3621 { 3622 if (!boot_cpu_has_bug(bug)) 3623 return sysfs_emit(buf, "Not affected\n"); 3624 3625 switch (bug) { 3626 case X86_BUG_CPU_MELTDOWN: 3627 if (boot_cpu_has(X86_FEATURE_PTI)) 3628 return sysfs_emit(buf, "Mitigation: PTI\n"); 3629 3630 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 3631 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 3632 3633 break; 3634 3635 case X86_BUG_SPECTRE_V1: 3636 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 3637 3638 case X86_BUG_SPECTRE_V2: 3639 return spectre_v2_show_state(buf); 3640 3641 case X86_BUG_SPEC_STORE_BYPASS: 3642 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 3643 3644 case X86_BUG_L1TF: 3645 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 3646 return l1tf_show_state(buf); 3647 break; 3648 3649 case X86_BUG_MDS: 3650 return mds_show_state(buf); 3651 3652 case X86_BUG_TAA: 3653 return tsx_async_abort_show_state(buf); 3654 3655 case X86_BUG_ITLB_MULTIHIT: 3656 return itlb_multihit_show_state(buf); 3657 3658 case X86_BUG_SRBDS: 3659 return srbds_show_state(buf); 3660 3661 case X86_BUG_MMIO_STALE_DATA: 3662 return mmio_stale_data_show_state(buf); 3663 3664 case X86_BUG_RETBLEED: 3665 return retbleed_show_state(buf); 3666 3667 case X86_BUG_SRSO: 3668 return srso_show_state(buf); 3669 3670 case X86_BUG_GDS: 3671 return gds_show_state(buf); 3672 3673 case X86_BUG_RFDS: 3674 return rfds_show_state(buf); 3675 3676 case X86_BUG_OLD_MICROCODE: 3677 return old_microcode_show_state(buf); 3678 3679 case X86_BUG_ITS: 3680 return its_show_state(buf); 3681 3682 case X86_BUG_TSA: 3683 return tsa_show_state(buf); 3684 3685 case X86_BUG_VMSCAPE: 3686 return vmscape_show_state(buf); 3687 3688 default: 3689 break; 3690 } 3691 3692 return sysfs_emit(buf, "Vulnerable\n"); 3693 } 3694 3695 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 3696 { 3697 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 3698 } 3699 3700 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 3701 { 3702 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 3703 } 3704 3705 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 3706 { 3707 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 3708 } 3709 3710 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 3711 { 3712 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 3713 } 3714 3715 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 3716 { 3717 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 3718 } 3719 3720 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 3721 { 3722 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 3723 } 3724 3725 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 3726 { 3727 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 3728 } 3729 3730 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 3731 { 3732 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 3733 } 3734 3735 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 3736 { 3737 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 3738 } 3739 3740 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 3741 { 3742 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 3743 } 3744 3745 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 3746 { 3747 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 3748 } 3749 3750 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 3751 { 3752 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 3753 } 3754 3755 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 3756 { 3757 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 3758 } 3759 3760 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf) 3761 { 3762 return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); 3763 } 3764 3765 ssize_t cpu_show_old_microcode(struct device *dev, struct device_attribute *attr, char *buf) 3766 { 3767 return cpu_show_common(dev, attr, buf, X86_BUG_OLD_MICROCODE); 3768 } 3769 3770 ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf) 3771 { 3772 return cpu_show_common(dev, attr, buf, X86_BUG_ITS); 3773 } 3774 3775 ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf) 3776 { 3777 return cpu_show_common(dev, attr, buf, X86_BUG_TSA); 3778 } 3779 3780 ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf) 3781 { 3782 return cpu_show_common(dev, attr, buf, X86_BUG_VMSCAPE); 3783 } 3784 #endif 3785 3786 void __warn_thunk(void) 3787 { 3788 WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); 3789 } 3790