1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <linux/module.h> 14 #include <linux/nospec.h> 15 #include <linux/prctl.h> 16 #include <linux/sched/smt.h> 17 #include <linux/pgtable.h> 18 #include <linux/bpf.h> 19 20 #include <asm/spec-ctrl.h> 21 #include <asm/cmdline.h> 22 #include <asm/bugs.h> 23 #include <asm/processor.h> 24 #include <asm/processor-flags.h> 25 #include <asm/fpu/api.h> 26 #include <asm/msr.h> 27 #include <asm/vmx.h> 28 #include <asm/paravirt.h> 29 #include <asm/cpu_device_id.h> 30 #include <asm/e820/api.h> 31 #include <asm/hypervisor.h> 32 #include <asm/tlbflush.h> 33 #include <asm/cpu.h> 34 35 #include "cpu.h" 36 37 /* 38 * Speculation Vulnerability Handling 39 * 40 * Each vulnerability is handled with the following functions: 41 * <vuln>_select_mitigation() -- Selects a mitigation to use. This should 42 * take into account all relevant command line 43 * options. 44 * <vuln>_update_mitigation() -- This is called after all vulnerabilities have 45 * selected a mitigation, in case the selection 46 * may want to change based on other choices 47 * made. This function is optional. 48 * <vuln>_apply_mitigation() -- Enable the selected mitigation. 49 * 50 * The compile-time mitigation in all cases should be AUTO. An explicit 51 * command-line option can override AUTO. If no such option is 52 * provided, <vuln>_select_mitigation() will override AUTO to the best 53 * mitigation option. 54 */ 55 56 static void __init spectre_v1_select_mitigation(void); 57 static void __init spectre_v1_apply_mitigation(void); 58 static void __init spectre_v2_select_mitigation(void); 59 static void __init spectre_v2_update_mitigation(void); 60 static void __init spectre_v2_apply_mitigation(void); 61 static void __init retbleed_select_mitigation(void); 62 static void __init retbleed_update_mitigation(void); 63 static void __init retbleed_apply_mitigation(void); 64 static void __init spectre_v2_user_select_mitigation(void); 65 static void __init spectre_v2_user_update_mitigation(void); 66 static void __init spectre_v2_user_apply_mitigation(void); 67 static void __init ssb_select_mitigation(void); 68 static void __init ssb_apply_mitigation(void); 69 static void __init l1tf_select_mitigation(void); 70 static void __init l1tf_apply_mitigation(void); 71 static void __init mds_select_mitigation(void); 72 static void __init mds_update_mitigation(void); 73 static void __init mds_apply_mitigation(void); 74 static void __init taa_select_mitigation(void); 75 static void __init taa_update_mitigation(void); 76 static void __init taa_apply_mitigation(void); 77 static void __init mmio_select_mitigation(void); 78 static void __init mmio_update_mitigation(void); 79 static void __init mmio_apply_mitigation(void); 80 static void __init rfds_select_mitigation(void); 81 static void __init rfds_update_mitigation(void); 82 static void __init rfds_apply_mitigation(void); 83 static void __init srbds_select_mitigation(void); 84 static void __init srbds_apply_mitigation(void); 85 static void __init l1d_flush_select_mitigation(void); 86 static void __init srso_select_mitigation(void); 87 static void __init srso_update_mitigation(void); 88 static void __init srso_apply_mitigation(void); 89 static void __init gds_select_mitigation(void); 90 static void __init gds_apply_mitigation(void); 91 static void __init bhi_select_mitigation(void); 92 static void __init bhi_update_mitigation(void); 93 static void __init bhi_apply_mitigation(void); 94 static void __init its_select_mitigation(void); 95 static void __init its_update_mitigation(void); 96 static void __init its_apply_mitigation(void); 97 98 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 99 u64 x86_spec_ctrl_base; 100 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 101 102 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 103 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 104 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current); 105 106 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; 107 108 static u64 __ro_after_init x86_arch_cap_msr; 109 110 static DEFINE_MUTEX(spec_ctrl_mutex); 111 112 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; 113 114 static void __init set_return_thunk(void *thunk) 115 { 116 if (x86_return_thunk != __x86_return_thunk) 117 pr_warn("x86/bugs: return thunk changed\n"); 118 119 x86_return_thunk = thunk; 120 } 121 122 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 123 static void update_spec_ctrl(u64 val) 124 { 125 this_cpu_write(x86_spec_ctrl_current, val); 126 wrmsrq(MSR_IA32_SPEC_CTRL, val); 127 } 128 129 /* 130 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 131 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 132 */ 133 void update_spec_ctrl_cond(u64 val) 134 { 135 if (this_cpu_read(x86_spec_ctrl_current) == val) 136 return; 137 138 this_cpu_write(x86_spec_ctrl_current, val); 139 140 /* 141 * When KERNEL_IBRS this MSR is written on return-to-user, unless 142 * forced the update can be delayed until that time. 143 */ 144 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 145 wrmsrq(MSR_IA32_SPEC_CTRL, val); 146 } 147 148 noinstr u64 spec_ctrl_current(void) 149 { 150 return this_cpu_read(x86_spec_ctrl_current); 151 } 152 EXPORT_SYMBOL_GPL(spec_ctrl_current); 153 154 /* 155 * AMD specific MSR info for Speculative Store Bypass control. 156 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 157 */ 158 u64 __ro_after_init x86_amd_ls_cfg_base; 159 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 160 161 /* Control conditional STIBP in switch_to() */ 162 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 163 /* Control conditional IBPB in switch_mm() */ 164 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 165 /* Control unconditional IBPB in switch_mm() */ 166 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 167 168 /* Control IBPB on vCPU load */ 169 DEFINE_STATIC_KEY_FALSE(switch_vcpu_ibpb); 170 EXPORT_SYMBOL_GPL(switch_vcpu_ibpb); 171 172 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 173 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 174 EXPORT_SYMBOL_GPL(mds_idle_clear); 175 176 /* 177 * Controls whether l1d flush based mitigations are enabled, 178 * based on hw features and admin setting via boot parameter 179 * defaults to false 180 */ 181 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 182 183 /* 184 * Controls CPU Fill buffer clear before VMenter. This is a subset of 185 * X86_FEATURE_CLEAR_CPU_BUF, and should only be enabled when KVM-only 186 * mitigation is required. 187 */ 188 DEFINE_STATIC_KEY_FALSE(cpu_buf_vm_clear); 189 EXPORT_SYMBOL_GPL(cpu_buf_vm_clear); 190 191 void __init cpu_select_mitigations(void) 192 { 193 /* 194 * Read the SPEC_CTRL MSR to account for reserved bits which may 195 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 196 * init code as it is not enumerated and depends on the family. 197 */ 198 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 199 rdmsrq(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 200 201 /* 202 * Previously running kernel (kexec), may have some controls 203 * turned ON. Clear them and let the mitigations setup below 204 * rediscover them based on configuration. 205 */ 206 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 207 } 208 209 x86_arch_cap_msr = x86_read_arch_cap_msr(); 210 211 /* Select the proper CPU mitigations before patching alternatives: */ 212 spectre_v1_select_mitigation(); 213 spectre_v2_select_mitigation(); 214 retbleed_select_mitigation(); 215 spectre_v2_user_select_mitigation(); 216 ssb_select_mitigation(); 217 l1tf_select_mitigation(); 218 mds_select_mitigation(); 219 taa_select_mitigation(); 220 mmio_select_mitigation(); 221 rfds_select_mitigation(); 222 srbds_select_mitigation(); 223 l1d_flush_select_mitigation(); 224 srso_select_mitigation(); 225 gds_select_mitigation(); 226 its_select_mitigation(); 227 bhi_select_mitigation(); 228 229 /* 230 * After mitigations are selected, some may need to update their 231 * choices. 232 */ 233 spectre_v2_update_mitigation(); 234 /* 235 * retbleed_update_mitigation() relies on the state set by 236 * spectre_v2_update_mitigation(); specifically it wants to know about 237 * spectre_v2=ibrs. 238 */ 239 retbleed_update_mitigation(); 240 /* 241 * its_update_mitigation() depends on spectre_v2_update_mitigation() 242 * and retbleed_update_mitigation(). 243 */ 244 its_update_mitigation(); 245 246 /* 247 * spectre_v2_user_update_mitigation() depends on 248 * retbleed_update_mitigation(), specifically the STIBP 249 * selection is forced for UNRET or IBPB. 250 */ 251 spectre_v2_user_update_mitigation(); 252 mds_update_mitigation(); 253 taa_update_mitigation(); 254 mmio_update_mitigation(); 255 rfds_update_mitigation(); 256 bhi_update_mitigation(); 257 /* srso_update_mitigation() depends on retbleed_update_mitigation(). */ 258 srso_update_mitigation(); 259 260 spectre_v1_apply_mitigation(); 261 spectre_v2_apply_mitigation(); 262 retbleed_apply_mitigation(); 263 spectre_v2_user_apply_mitigation(); 264 ssb_apply_mitigation(); 265 l1tf_apply_mitigation(); 266 mds_apply_mitigation(); 267 taa_apply_mitigation(); 268 mmio_apply_mitigation(); 269 rfds_apply_mitigation(); 270 srbds_apply_mitigation(); 271 srso_apply_mitigation(); 272 gds_apply_mitigation(); 273 its_apply_mitigation(); 274 bhi_apply_mitigation(); 275 } 276 277 /* 278 * NOTE: This function is *only* called for SVM, since Intel uses 279 * MSR_IA32_SPEC_CTRL for SSBD. 280 */ 281 void 282 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 283 { 284 u64 guestval, hostval; 285 struct thread_info *ti = current_thread_info(); 286 287 /* 288 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 289 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 290 */ 291 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 292 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 293 return; 294 295 /* 296 * If the host has SSBD mitigation enabled, force it in the host's 297 * virtual MSR value. If its not permanently enabled, evaluate 298 * current's TIF_SSBD thread flag. 299 */ 300 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 301 hostval = SPEC_CTRL_SSBD; 302 else 303 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 304 305 /* Sanitize the guest value */ 306 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 307 308 if (hostval != guestval) { 309 unsigned long tif; 310 311 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 312 ssbd_spec_ctrl_to_tif(hostval); 313 314 speculation_ctrl_update(tif); 315 } 316 } 317 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 318 319 static void x86_amd_ssb_disable(void) 320 { 321 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 322 323 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 324 wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 325 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 326 wrmsrq(MSR_AMD64_LS_CFG, msrval); 327 } 328 329 #undef pr_fmt 330 #define pr_fmt(fmt) "MDS: " fmt 331 332 /* Default mitigation for MDS-affected CPUs */ 333 static enum mds_mitigations mds_mitigation __ro_after_init = 334 IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_AUTO : MDS_MITIGATION_OFF; 335 static bool mds_nosmt __ro_after_init = false; 336 337 static const char * const mds_strings[] = { 338 [MDS_MITIGATION_OFF] = "Vulnerable", 339 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 340 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 341 }; 342 343 enum taa_mitigations { 344 TAA_MITIGATION_OFF, 345 TAA_MITIGATION_AUTO, 346 TAA_MITIGATION_UCODE_NEEDED, 347 TAA_MITIGATION_VERW, 348 TAA_MITIGATION_TSX_DISABLED, 349 }; 350 351 /* Default mitigation for TAA-affected CPUs */ 352 static enum taa_mitigations taa_mitigation __ro_after_init = 353 IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_AUTO : TAA_MITIGATION_OFF; 354 355 enum mmio_mitigations { 356 MMIO_MITIGATION_OFF, 357 MMIO_MITIGATION_AUTO, 358 MMIO_MITIGATION_UCODE_NEEDED, 359 MMIO_MITIGATION_VERW, 360 }; 361 362 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 363 static enum mmio_mitigations mmio_mitigation __ro_after_init = 364 IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ? MMIO_MITIGATION_AUTO : MMIO_MITIGATION_OFF; 365 366 enum rfds_mitigations { 367 RFDS_MITIGATION_OFF, 368 RFDS_MITIGATION_AUTO, 369 RFDS_MITIGATION_VERW, 370 RFDS_MITIGATION_UCODE_NEEDED, 371 }; 372 373 /* Default mitigation for Register File Data Sampling */ 374 static enum rfds_mitigations rfds_mitigation __ro_after_init = 375 IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF; 376 377 /* 378 * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing 379 * through X86_FEATURE_CLEAR_CPU_BUF on kernel and guest entry. 380 */ 381 static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init; 382 383 static void __init mds_select_mitigation(void) 384 { 385 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 386 mds_mitigation = MDS_MITIGATION_OFF; 387 return; 388 } 389 390 if (mds_mitigation == MDS_MITIGATION_AUTO) 391 mds_mitigation = MDS_MITIGATION_FULL; 392 393 if (mds_mitigation == MDS_MITIGATION_OFF) 394 return; 395 396 verw_clear_cpu_buf_mitigation_selected = true; 397 } 398 399 static void __init mds_update_mitigation(void) 400 { 401 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) 402 return; 403 404 /* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */ 405 if (verw_clear_cpu_buf_mitigation_selected) 406 mds_mitigation = MDS_MITIGATION_FULL; 407 408 if (mds_mitigation == MDS_MITIGATION_FULL) { 409 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 410 mds_mitigation = MDS_MITIGATION_VMWERV; 411 } 412 413 pr_info("%s\n", mds_strings[mds_mitigation]); 414 } 415 416 static void __init mds_apply_mitigation(void) 417 { 418 if (mds_mitigation == MDS_MITIGATION_FULL || 419 mds_mitigation == MDS_MITIGATION_VMWERV) { 420 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 421 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 422 (mds_nosmt || cpu_mitigations_auto_nosmt())) 423 cpu_smt_disable(false); 424 } 425 } 426 427 static int __init mds_cmdline(char *str) 428 { 429 if (!boot_cpu_has_bug(X86_BUG_MDS)) 430 return 0; 431 432 if (!str) 433 return -EINVAL; 434 435 if (!strcmp(str, "off")) 436 mds_mitigation = MDS_MITIGATION_OFF; 437 else if (!strcmp(str, "full")) 438 mds_mitigation = MDS_MITIGATION_FULL; 439 else if (!strcmp(str, "full,nosmt")) { 440 mds_mitigation = MDS_MITIGATION_FULL; 441 mds_nosmt = true; 442 } 443 444 return 0; 445 } 446 early_param("mds", mds_cmdline); 447 448 #undef pr_fmt 449 #define pr_fmt(fmt) "TAA: " fmt 450 451 static bool taa_nosmt __ro_after_init; 452 453 static const char * const taa_strings[] = { 454 [TAA_MITIGATION_OFF] = "Vulnerable", 455 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 456 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 457 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 458 }; 459 460 static bool __init taa_vulnerable(void) 461 { 462 return boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM); 463 } 464 465 static void __init taa_select_mitigation(void) 466 { 467 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 468 taa_mitigation = TAA_MITIGATION_OFF; 469 return; 470 } 471 472 /* TSX previously disabled by tsx=off */ 473 if (!boot_cpu_has(X86_FEATURE_RTM)) { 474 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 475 return; 476 } 477 478 if (cpu_mitigations_off()) 479 taa_mitigation = TAA_MITIGATION_OFF; 480 481 /* Microcode will be checked in taa_update_mitigation(). */ 482 if (taa_mitigation == TAA_MITIGATION_AUTO) 483 taa_mitigation = TAA_MITIGATION_VERW; 484 485 if (taa_mitigation != TAA_MITIGATION_OFF) 486 verw_clear_cpu_buf_mitigation_selected = true; 487 } 488 489 static void __init taa_update_mitigation(void) 490 { 491 if (!taa_vulnerable() || cpu_mitigations_off()) 492 return; 493 494 if (verw_clear_cpu_buf_mitigation_selected) 495 taa_mitigation = TAA_MITIGATION_VERW; 496 497 if (taa_mitigation == TAA_MITIGATION_VERW) { 498 /* Check if the requisite ucode is available. */ 499 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 500 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 501 502 /* 503 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 504 * A microcode update fixes this behavior to clear CPU buffers. It also 505 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 506 * ARCH_CAP_TSX_CTRL_MSR bit. 507 * 508 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 509 * update is required. 510 */ 511 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && 512 !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) 513 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 514 } 515 516 pr_info("%s\n", taa_strings[taa_mitigation]); 517 } 518 519 static void __init taa_apply_mitigation(void) 520 { 521 if (taa_mitigation == TAA_MITIGATION_VERW || 522 taa_mitigation == TAA_MITIGATION_UCODE_NEEDED) { 523 /* 524 * TSX is enabled, select alternate mitigation for TAA which is 525 * the same as MDS. Enable MDS static branch to clear CPU buffers. 526 * 527 * For guests that can't determine whether the correct microcode is 528 * present on host, enable the mitigation for UCODE_NEEDED as well. 529 */ 530 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 531 532 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 533 cpu_smt_disable(false); 534 } 535 } 536 537 static int __init tsx_async_abort_parse_cmdline(char *str) 538 { 539 if (!boot_cpu_has_bug(X86_BUG_TAA)) 540 return 0; 541 542 if (!str) 543 return -EINVAL; 544 545 if (!strcmp(str, "off")) { 546 taa_mitigation = TAA_MITIGATION_OFF; 547 } else if (!strcmp(str, "full")) { 548 taa_mitigation = TAA_MITIGATION_VERW; 549 } else if (!strcmp(str, "full,nosmt")) { 550 taa_mitigation = TAA_MITIGATION_VERW; 551 taa_nosmt = true; 552 } 553 554 return 0; 555 } 556 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 557 558 #undef pr_fmt 559 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 560 561 static bool mmio_nosmt __ro_after_init = false; 562 563 static const char * const mmio_strings[] = { 564 [MMIO_MITIGATION_OFF] = "Vulnerable", 565 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 566 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 567 }; 568 569 static void __init mmio_select_mitigation(void) 570 { 571 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || 572 cpu_mitigations_off()) { 573 mmio_mitigation = MMIO_MITIGATION_OFF; 574 return; 575 } 576 577 /* Microcode will be checked in mmio_update_mitigation(). */ 578 if (mmio_mitigation == MMIO_MITIGATION_AUTO) 579 mmio_mitigation = MMIO_MITIGATION_VERW; 580 581 if (mmio_mitigation == MMIO_MITIGATION_OFF) 582 return; 583 584 /* 585 * Enable CPU buffer clear mitigation for host and VMM, if also affected 586 * by MDS or TAA. 587 */ 588 if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable()) 589 verw_clear_cpu_buf_mitigation_selected = true; 590 } 591 592 static void __init mmio_update_mitigation(void) 593 { 594 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || cpu_mitigations_off()) 595 return; 596 597 if (verw_clear_cpu_buf_mitigation_selected) 598 mmio_mitigation = MMIO_MITIGATION_VERW; 599 600 if (mmio_mitigation == MMIO_MITIGATION_VERW) { 601 /* 602 * Check if the system has the right microcode. 603 * 604 * CPU Fill buffer clear mitigation is enumerated by either an explicit 605 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 606 * affected systems. 607 */ 608 if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || 609 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 610 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 611 !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))) 612 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 613 } 614 615 pr_info("%s\n", mmio_strings[mmio_mitigation]); 616 } 617 618 static void __init mmio_apply_mitigation(void) 619 { 620 if (mmio_mitigation == MMIO_MITIGATION_OFF) 621 return; 622 623 /* 624 * Only enable the VMM mitigation if the CPU buffer clear mitigation is 625 * not being used. 626 */ 627 if (verw_clear_cpu_buf_mitigation_selected) { 628 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 629 static_branch_disable(&cpu_buf_vm_clear); 630 } else { 631 static_branch_enable(&cpu_buf_vm_clear); 632 } 633 634 /* 635 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 636 * be propagated to uncore buffers, clearing the Fill buffers on idle 637 * is required irrespective of SMT state. 638 */ 639 if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) 640 static_branch_enable(&mds_idle_clear); 641 642 if (mmio_nosmt || cpu_mitigations_auto_nosmt()) 643 cpu_smt_disable(false); 644 } 645 646 static int __init mmio_stale_data_parse_cmdline(char *str) 647 { 648 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 649 return 0; 650 651 if (!str) 652 return -EINVAL; 653 654 if (!strcmp(str, "off")) { 655 mmio_mitigation = MMIO_MITIGATION_OFF; 656 } else if (!strcmp(str, "full")) { 657 mmio_mitigation = MMIO_MITIGATION_VERW; 658 } else if (!strcmp(str, "full,nosmt")) { 659 mmio_mitigation = MMIO_MITIGATION_VERW; 660 mmio_nosmt = true; 661 } 662 663 return 0; 664 } 665 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 666 667 #undef pr_fmt 668 #define pr_fmt(fmt) "Register File Data Sampling: " fmt 669 670 static const char * const rfds_strings[] = { 671 [RFDS_MITIGATION_OFF] = "Vulnerable", 672 [RFDS_MITIGATION_VERW] = "Mitigation: Clear Register File", 673 [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 674 }; 675 676 static inline bool __init verw_clears_cpu_reg_file(void) 677 { 678 return (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR); 679 } 680 681 static void __init rfds_select_mitigation(void) 682 { 683 if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) { 684 rfds_mitigation = RFDS_MITIGATION_OFF; 685 return; 686 } 687 688 if (rfds_mitigation == RFDS_MITIGATION_AUTO) 689 rfds_mitigation = RFDS_MITIGATION_VERW; 690 691 if (rfds_mitigation == RFDS_MITIGATION_OFF) 692 return; 693 694 if (verw_clears_cpu_reg_file()) 695 verw_clear_cpu_buf_mitigation_selected = true; 696 } 697 698 static void __init rfds_update_mitigation(void) 699 { 700 if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) 701 return; 702 703 if (verw_clear_cpu_buf_mitigation_selected) 704 rfds_mitigation = RFDS_MITIGATION_VERW; 705 706 if (rfds_mitigation == RFDS_MITIGATION_VERW) { 707 if (!verw_clears_cpu_reg_file()) 708 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; 709 } 710 711 pr_info("%s\n", rfds_strings[rfds_mitigation]); 712 } 713 714 static void __init rfds_apply_mitigation(void) 715 { 716 if (rfds_mitigation == RFDS_MITIGATION_VERW) 717 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); 718 } 719 720 static __init int rfds_parse_cmdline(char *str) 721 { 722 if (!str) 723 return -EINVAL; 724 725 if (!boot_cpu_has_bug(X86_BUG_RFDS)) 726 return 0; 727 728 if (!strcmp(str, "off")) 729 rfds_mitigation = RFDS_MITIGATION_OFF; 730 else if (!strcmp(str, "on")) 731 rfds_mitigation = RFDS_MITIGATION_VERW; 732 733 return 0; 734 } 735 early_param("reg_file_data_sampling", rfds_parse_cmdline); 736 737 #undef pr_fmt 738 #define pr_fmt(fmt) "SRBDS: " fmt 739 740 enum srbds_mitigations { 741 SRBDS_MITIGATION_OFF, 742 SRBDS_MITIGATION_AUTO, 743 SRBDS_MITIGATION_UCODE_NEEDED, 744 SRBDS_MITIGATION_FULL, 745 SRBDS_MITIGATION_TSX_OFF, 746 SRBDS_MITIGATION_HYPERVISOR, 747 }; 748 749 static enum srbds_mitigations srbds_mitigation __ro_after_init = 750 IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_AUTO : SRBDS_MITIGATION_OFF; 751 752 static const char * const srbds_strings[] = { 753 [SRBDS_MITIGATION_OFF] = "Vulnerable", 754 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 755 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 756 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 757 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 758 }; 759 760 static bool srbds_off; 761 762 void update_srbds_msr(void) 763 { 764 u64 mcu_ctrl; 765 766 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 767 return; 768 769 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 770 return; 771 772 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 773 return; 774 775 /* 776 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 777 * being disabled and it hasn't received the SRBDS MSR microcode. 778 */ 779 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 780 return; 781 782 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 783 784 switch (srbds_mitigation) { 785 case SRBDS_MITIGATION_OFF: 786 case SRBDS_MITIGATION_TSX_OFF: 787 mcu_ctrl |= RNGDS_MITG_DIS; 788 break; 789 case SRBDS_MITIGATION_FULL: 790 mcu_ctrl &= ~RNGDS_MITG_DIS; 791 break; 792 default: 793 break; 794 } 795 796 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 797 } 798 799 static void __init srbds_select_mitigation(void) 800 { 801 if (!boot_cpu_has_bug(X86_BUG_SRBDS) || cpu_mitigations_off()) { 802 srbds_mitigation = SRBDS_MITIGATION_OFF; 803 return; 804 } 805 806 if (srbds_mitigation == SRBDS_MITIGATION_AUTO) 807 srbds_mitigation = SRBDS_MITIGATION_FULL; 808 809 /* 810 * Check to see if this is one of the MDS_NO systems supporting TSX that 811 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 812 * by Processor MMIO Stale Data vulnerability. 813 */ 814 if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 815 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 816 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 817 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 818 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 819 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 820 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 821 else if (srbds_off) 822 srbds_mitigation = SRBDS_MITIGATION_OFF; 823 824 pr_info("%s\n", srbds_strings[srbds_mitigation]); 825 } 826 827 static void __init srbds_apply_mitigation(void) 828 { 829 update_srbds_msr(); 830 } 831 832 static int __init srbds_parse_cmdline(char *str) 833 { 834 if (!str) 835 return -EINVAL; 836 837 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 838 return 0; 839 840 srbds_off = !strcmp(str, "off"); 841 return 0; 842 } 843 early_param("srbds", srbds_parse_cmdline); 844 845 #undef pr_fmt 846 #define pr_fmt(fmt) "L1D Flush : " fmt 847 848 enum l1d_flush_mitigations { 849 L1D_FLUSH_OFF = 0, 850 L1D_FLUSH_ON, 851 }; 852 853 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 854 855 static void __init l1d_flush_select_mitigation(void) 856 { 857 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 858 return; 859 860 static_branch_enable(&switch_mm_cond_l1d_flush); 861 pr_info("Conditional flush on switch_mm() enabled\n"); 862 } 863 864 static int __init l1d_flush_parse_cmdline(char *str) 865 { 866 if (!strcmp(str, "on")) 867 l1d_flush_mitigation = L1D_FLUSH_ON; 868 869 return 0; 870 } 871 early_param("l1d_flush", l1d_flush_parse_cmdline); 872 873 #undef pr_fmt 874 #define pr_fmt(fmt) "GDS: " fmt 875 876 enum gds_mitigations { 877 GDS_MITIGATION_OFF, 878 GDS_MITIGATION_AUTO, 879 GDS_MITIGATION_UCODE_NEEDED, 880 GDS_MITIGATION_FORCE, 881 GDS_MITIGATION_FULL, 882 GDS_MITIGATION_FULL_LOCKED, 883 GDS_MITIGATION_HYPERVISOR, 884 }; 885 886 static enum gds_mitigations gds_mitigation __ro_after_init = 887 IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_AUTO : GDS_MITIGATION_OFF; 888 889 static const char * const gds_strings[] = { 890 [GDS_MITIGATION_OFF] = "Vulnerable", 891 [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 892 [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode", 893 [GDS_MITIGATION_FULL] = "Mitigation: Microcode", 894 [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)", 895 [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 896 }; 897 898 bool gds_ucode_mitigated(void) 899 { 900 return (gds_mitigation == GDS_MITIGATION_FULL || 901 gds_mitigation == GDS_MITIGATION_FULL_LOCKED); 902 } 903 EXPORT_SYMBOL_GPL(gds_ucode_mitigated); 904 905 void update_gds_msr(void) 906 { 907 u64 mcu_ctrl_after; 908 u64 mcu_ctrl; 909 910 switch (gds_mitigation) { 911 case GDS_MITIGATION_OFF: 912 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 913 mcu_ctrl |= GDS_MITG_DIS; 914 break; 915 case GDS_MITIGATION_FULL_LOCKED: 916 /* 917 * The LOCKED state comes from the boot CPU. APs might not have 918 * the same state. Make sure the mitigation is enabled on all 919 * CPUs. 920 */ 921 case GDS_MITIGATION_FULL: 922 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 923 mcu_ctrl &= ~GDS_MITG_DIS; 924 break; 925 case GDS_MITIGATION_FORCE: 926 case GDS_MITIGATION_UCODE_NEEDED: 927 case GDS_MITIGATION_HYPERVISOR: 928 case GDS_MITIGATION_AUTO: 929 return; 930 } 931 932 wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 933 934 /* 935 * Check to make sure that the WRMSR value was not ignored. Writes to 936 * GDS_MITG_DIS will be ignored if this processor is locked but the boot 937 * processor was not. 938 */ 939 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); 940 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); 941 } 942 943 static void __init gds_select_mitigation(void) 944 { 945 u64 mcu_ctrl; 946 947 if (!boot_cpu_has_bug(X86_BUG_GDS)) 948 return; 949 950 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 951 gds_mitigation = GDS_MITIGATION_HYPERVISOR; 952 return; 953 } 954 955 if (cpu_mitigations_off()) 956 gds_mitigation = GDS_MITIGATION_OFF; 957 /* Will verify below that mitigation _can_ be disabled */ 958 959 if (gds_mitigation == GDS_MITIGATION_AUTO) 960 gds_mitigation = GDS_MITIGATION_FULL; 961 962 /* No microcode */ 963 if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) { 964 if (gds_mitigation != GDS_MITIGATION_FORCE) 965 gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; 966 return; 967 } 968 969 /* Microcode has mitigation, use it */ 970 if (gds_mitigation == GDS_MITIGATION_FORCE) 971 gds_mitigation = GDS_MITIGATION_FULL; 972 973 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 974 if (mcu_ctrl & GDS_MITG_LOCKED) { 975 if (gds_mitigation == GDS_MITIGATION_OFF) 976 pr_warn("Mitigation locked. Disable failed.\n"); 977 978 /* 979 * The mitigation is selected from the boot CPU. All other CPUs 980 * _should_ have the same state. If the boot CPU isn't locked 981 * but others are then update_gds_msr() will WARN() of the state 982 * mismatch. If the boot CPU is locked update_gds_msr() will 983 * ensure the other CPUs have the mitigation enabled. 984 */ 985 gds_mitigation = GDS_MITIGATION_FULL_LOCKED; 986 } 987 } 988 989 static void __init gds_apply_mitigation(void) 990 { 991 if (!boot_cpu_has_bug(X86_BUG_GDS)) 992 return; 993 994 /* Microcode is present */ 995 if (x86_arch_cap_msr & ARCH_CAP_GDS_CTRL) 996 update_gds_msr(); 997 else if (gds_mitigation == GDS_MITIGATION_FORCE) { 998 /* 999 * This only needs to be done on the boot CPU so do it 1000 * here rather than in update_gds_msr() 1001 */ 1002 setup_clear_cpu_cap(X86_FEATURE_AVX); 1003 pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); 1004 } 1005 1006 pr_info("%s\n", gds_strings[gds_mitigation]); 1007 } 1008 1009 static int __init gds_parse_cmdline(char *str) 1010 { 1011 if (!str) 1012 return -EINVAL; 1013 1014 if (!boot_cpu_has_bug(X86_BUG_GDS)) 1015 return 0; 1016 1017 if (!strcmp(str, "off")) 1018 gds_mitigation = GDS_MITIGATION_OFF; 1019 else if (!strcmp(str, "force")) 1020 gds_mitigation = GDS_MITIGATION_FORCE; 1021 1022 return 0; 1023 } 1024 early_param("gather_data_sampling", gds_parse_cmdline); 1025 1026 #undef pr_fmt 1027 #define pr_fmt(fmt) "Spectre V1 : " fmt 1028 1029 enum spectre_v1_mitigation { 1030 SPECTRE_V1_MITIGATION_NONE, 1031 SPECTRE_V1_MITIGATION_AUTO, 1032 }; 1033 1034 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 1035 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ? 1036 SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE; 1037 1038 static const char * const spectre_v1_strings[] = { 1039 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 1040 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 1041 }; 1042 1043 /* 1044 * Does SMAP provide full mitigation against speculative kernel access to 1045 * userspace? 1046 */ 1047 static bool smap_works_speculatively(void) 1048 { 1049 if (!boot_cpu_has(X86_FEATURE_SMAP)) 1050 return false; 1051 1052 /* 1053 * On CPUs which are vulnerable to Meltdown, SMAP does not 1054 * prevent speculative access to user data in the L1 cache. 1055 * Consider SMAP to be non-functional as a mitigation on these 1056 * CPUs. 1057 */ 1058 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 1059 return false; 1060 1061 return true; 1062 } 1063 1064 static void __init spectre_v1_select_mitigation(void) 1065 { 1066 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) 1067 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1068 } 1069 1070 static void __init spectre_v1_apply_mitigation(void) 1071 { 1072 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) 1073 return; 1074 1075 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 1076 /* 1077 * With Spectre v1, a user can speculatively control either 1078 * path of a conditional swapgs with a user-controlled GS 1079 * value. The mitigation is to add lfences to both code paths. 1080 * 1081 * If FSGSBASE is enabled, the user can put a kernel address in 1082 * GS, in which case SMAP provides no protection. 1083 * 1084 * If FSGSBASE is disabled, the user can only put a user space 1085 * address in GS. That makes an attack harder, but still 1086 * possible if there's no SMAP protection. 1087 */ 1088 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 1089 !smap_works_speculatively()) { 1090 /* 1091 * Mitigation can be provided from SWAPGS itself or 1092 * PTI as the CR3 write in the Meltdown mitigation 1093 * is serializing. 1094 * 1095 * If neither is there, mitigate with an LFENCE to 1096 * stop speculation through swapgs. 1097 */ 1098 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 1099 !boot_cpu_has(X86_FEATURE_PTI)) 1100 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 1101 1102 /* 1103 * Enable lfences in the kernel entry (non-swapgs) 1104 * paths, to prevent user entry from speculatively 1105 * skipping swapgs. 1106 */ 1107 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 1108 } 1109 } 1110 1111 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1112 } 1113 1114 static int __init nospectre_v1_cmdline(char *str) 1115 { 1116 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 1117 return 0; 1118 } 1119 early_param("nospectre_v1", nospectre_v1_cmdline); 1120 1121 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 1122 1123 #undef pr_fmt 1124 #define pr_fmt(fmt) "RETBleed: " fmt 1125 1126 enum its_mitigation { 1127 ITS_MITIGATION_OFF, 1128 ITS_MITIGATION_AUTO, 1129 ITS_MITIGATION_VMEXIT_ONLY, 1130 ITS_MITIGATION_ALIGNED_THUNKS, 1131 ITS_MITIGATION_RETPOLINE_STUFF, 1132 }; 1133 1134 static enum its_mitigation its_mitigation __ro_after_init = 1135 IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_MITIGATION_AUTO : ITS_MITIGATION_OFF; 1136 1137 enum retbleed_mitigation { 1138 RETBLEED_MITIGATION_NONE, 1139 RETBLEED_MITIGATION_AUTO, 1140 RETBLEED_MITIGATION_UNRET, 1141 RETBLEED_MITIGATION_IBPB, 1142 RETBLEED_MITIGATION_IBRS, 1143 RETBLEED_MITIGATION_EIBRS, 1144 RETBLEED_MITIGATION_STUFF, 1145 }; 1146 1147 static const char * const retbleed_strings[] = { 1148 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 1149 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 1150 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 1151 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 1152 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 1153 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 1154 }; 1155 1156 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 1157 IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE; 1158 1159 static int __ro_after_init retbleed_nosmt = false; 1160 1161 static int __init retbleed_parse_cmdline(char *str) 1162 { 1163 if (!str) 1164 return -EINVAL; 1165 1166 while (str) { 1167 char *next = strchr(str, ','); 1168 if (next) { 1169 *next = 0; 1170 next++; 1171 } 1172 1173 if (!strcmp(str, "off")) { 1174 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1175 } else if (!strcmp(str, "auto")) { 1176 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1177 } else if (!strcmp(str, "unret")) { 1178 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1179 } else if (!strcmp(str, "ibpb")) { 1180 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1181 } else if (!strcmp(str, "stuff")) { 1182 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1183 } else if (!strcmp(str, "nosmt")) { 1184 retbleed_nosmt = true; 1185 } else if (!strcmp(str, "force")) { 1186 setup_force_cpu_bug(X86_BUG_RETBLEED); 1187 } else { 1188 pr_err("Ignoring unknown retbleed option (%s).", str); 1189 } 1190 1191 str = next; 1192 } 1193 1194 return 0; 1195 } 1196 early_param("retbleed", retbleed_parse_cmdline); 1197 1198 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 1199 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 1200 1201 static void __init retbleed_select_mitigation(void) 1202 { 1203 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) { 1204 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1205 return; 1206 } 1207 1208 switch (retbleed_mitigation) { 1209 case RETBLEED_MITIGATION_UNRET: 1210 if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { 1211 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1212 pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); 1213 } 1214 break; 1215 case RETBLEED_MITIGATION_IBPB: 1216 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 1217 pr_err("WARNING: CPU does not support IBPB.\n"); 1218 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1219 } else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 1220 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 1221 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1222 } 1223 break; 1224 case RETBLEED_MITIGATION_STUFF: 1225 if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1226 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); 1227 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1228 } else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1229 pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); 1230 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1231 } 1232 break; 1233 default: 1234 break; 1235 } 1236 1237 if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO) 1238 return; 1239 1240 /* Intel mitigation selected in retbleed_update_mitigation() */ 1241 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1242 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 1243 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) 1244 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 1245 else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && 1246 boot_cpu_has(X86_FEATURE_IBPB)) 1247 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 1248 else 1249 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1250 } 1251 } 1252 1253 static void __init retbleed_update_mitigation(void) 1254 { 1255 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) 1256 return; 1257 1258 if (retbleed_mitigation == RETBLEED_MITIGATION_NONE) 1259 goto out; 1260 1261 /* 1262 * retbleed=stuff is only allowed on Intel. If stuffing can't be used 1263 * then a different mitigation will be selected below. 1264 * 1265 * its=stuff will also attempt to enable stuffing. 1266 */ 1267 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF || 1268 its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF) { 1269 if (spectre_v2_enabled != SPECTRE_V2_RETPOLINE) { 1270 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); 1271 retbleed_mitigation = RETBLEED_MITIGATION_AUTO; 1272 } else { 1273 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 1274 pr_info("Retbleed mitigation updated to stuffing\n"); 1275 1276 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 1277 } 1278 } 1279 /* 1280 * Let IBRS trump all on Intel without affecting the effects of the 1281 * retbleed= cmdline option except for call depth based stuffing 1282 */ 1283 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1284 switch (spectre_v2_enabled) { 1285 case SPECTRE_V2_IBRS: 1286 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 1287 break; 1288 case SPECTRE_V2_EIBRS: 1289 case SPECTRE_V2_EIBRS_RETPOLINE: 1290 case SPECTRE_V2_EIBRS_LFENCE: 1291 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 1292 break; 1293 default: 1294 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 1295 pr_err(RETBLEED_INTEL_MSG); 1296 } 1297 /* If nothing has set the mitigation yet, default to NONE. */ 1298 if (retbleed_mitigation == RETBLEED_MITIGATION_AUTO) 1299 retbleed_mitigation = RETBLEED_MITIGATION_NONE; 1300 } 1301 out: 1302 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 1303 } 1304 1305 1306 static void __init retbleed_apply_mitigation(void) 1307 { 1308 bool mitigate_smt = false; 1309 1310 switch (retbleed_mitigation) { 1311 case RETBLEED_MITIGATION_NONE: 1312 return; 1313 1314 case RETBLEED_MITIGATION_UNRET: 1315 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1316 setup_force_cpu_cap(X86_FEATURE_UNRET); 1317 1318 set_return_thunk(retbleed_return_thunk); 1319 1320 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1321 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1322 pr_err(RETBLEED_UNTRAIN_MSG); 1323 1324 mitigate_smt = true; 1325 break; 1326 1327 case RETBLEED_MITIGATION_IBPB: 1328 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 1329 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 1330 mitigate_smt = true; 1331 1332 /* 1333 * IBPB on entry already obviates the need for 1334 * software-based untraining so clear those in case some 1335 * other mitigation like SRSO has selected them. 1336 */ 1337 setup_clear_cpu_cap(X86_FEATURE_UNRET); 1338 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 1339 1340 /* 1341 * There is no need for RSB filling: write_ibpb() ensures 1342 * all predictions, including the RSB, are invalidated, 1343 * regardless of IBPB implementation. 1344 */ 1345 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1346 1347 break; 1348 1349 case RETBLEED_MITIGATION_STUFF: 1350 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1351 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 1352 1353 set_return_thunk(call_depth_return_thunk); 1354 break; 1355 1356 default: 1357 break; 1358 } 1359 1360 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 1361 (retbleed_nosmt || cpu_mitigations_auto_nosmt())) 1362 cpu_smt_disable(false); 1363 } 1364 1365 #undef pr_fmt 1366 #define pr_fmt(fmt) "ITS: " fmt 1367 1368 static const char * const its_strings[] = { 1369 [ITS_MITIGATION_OFF] = "Vulnerable", 1370 [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", 1371 [ITS_MITIGATION_ALIGNED_THUNKS] = "Mitigation: Aligned branch/return thunks", 1372 [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", 1373 }; 1374 1375 static int __init its_parse_cmdline(char *str) 1376 { 1377 if (!str) 1378 return -EINVAL; 1379 1380 if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) { 1381 pr_err("Mitigation disabled at compile time, ignoring option (%s)", str); 1382 return 0; 1383 } 1384 1385 if (!strcmp(str, "off")) { 1386 its_mitigation = ITS_MITIGATION_OFF; 1387 } else if (!strcmp(str, "on")) { 1388 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1389 } else if (!strcmp(str, "force")) { 1390 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1391 setup_force_cpu_bug(X86_BUG_ITS); 1392 } else if (!strcmp(str, "vmexit")) { 1393 its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; 1394 } else if (!strcmp(str, "stuff")) { 1395 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1396 } else { 1397 pr_err("Ignoring unknown indirect_target_selection option (%s).", str); 1398 } 1399 1400 return 0; 1401 } 1402 early_param("indirect_target_selection", its_parse_cmdline); 1403 1404 static void __init its_select_mitigation(void) 1405 { 1406 if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) { 1407 its_mitigation = ITS_MITIGATION_OFF; 1408 return; 1409 } 1410 1411 if (its_mitigation == ITS_MITIGATION_AUTO) 1412 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1413 1414 if (its_mitigation == ITS_MITIGATION_OFF) 1415 return; 1416 1417 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || 1418 !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) { 1419 pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); 1420 its_mitigation = ITS_MITIGATION_OFF; 1421 return; 1422 } 1423 1424 if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { 1425 pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); 1426 its_mitigation = ITS_MITIGATION_OFF; 1427 return; 1428 } 1429 1430 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1431 !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { 1432 pr_err("RSB stuff mitigation not supported, using default\n"); 1433 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1434 } 1435 1436 if (its_mitigation == ITS_MITIGATION_VMEXIT_ONLY && 1437 !boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) 1438 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1439 } 1440 1441 static void __init its_update_mitigation(void) 1442 { 1443 if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) 1444 return; 1445 1446 switch (spectre_v2_enabled) { 1447 case SPECTRE_V2_NONE: 1448 pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); 1449 its_mitigation = ITS_MITIGATION_OFF; 1450 break; 1451 case SPECTRE_V2_RETPOLINE: 1452 /* Retpoline+CDT mitigates ITS */ 1453 if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) 1454 its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; 1455 break; 1456 case SPECTRE_V2_LFENCE: 1457 case SPECTRE_V2_EIBRS_LFENCE: 1458 pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); 1459 its_mitigation = ITS_MITIGATION_OFF; 1460 break; 1461 default: 1462 break; 1463 } 1464 1465 /* 1466 * retbleed_update_mitigation() will try to do stuffing if its=stuff. 1467 * If it can't, such as if spectre_v2!=retpoline, then fall back to 1468 * aligned thunks. 1469 */ 1470 if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && 1471 retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 1472 its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; 1473 1474 pr_info("%s\n", its_strings[its_mitigation]); 1475 } 1476 1477 static void __init its_apply_mitigation(void) 1478 { 1479 /* its=stuff forces retbleed stuffing and is enabled there. */ 1480 if (its_mitigation != ITS_MITIGATION_ALIGNED_THUNKS) 1481 return; 1482 1483 if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) 1484 setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); 1485 1486 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 1487 set_return_thunk(its_return_thunk); 1488 } 1489 1490 #undef pr_fmt 1491 #define pr_fmt(fmt) "Spectre V2 : " fmt 1492 1493 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 1494 SPECTRE_V2_USER_NONE; 1495 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 1496 SPECTRE_V2_USER_NONE; 1497 1498 #ifdef CONFIG_MITIGATION_RETPOLINE 1499 static bool spectre_v2_bad_module; 1500 1501 bool retpoline_module_ok(bool has_retpoline) 1502 { 1503 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1504 return true; 1505 1506 pr_err("System may be vulnerable to spectre v2\n"); 1507 spectre_v2_bad_module = true; 1508 return false; 1509 } 1510 1511 static inline const char *spectre_v2_module_string(void) 1512 { 1513 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1514 } 1515 #else 1516 static inline const char *spectre_v2_module_string(void) { return ""; } 1517 #endif 1518 1519 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1520 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1521 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1522 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1523 1524 #ifdef CONFIG_BPF_SYSCALL 1525 void unpriv_ebpf_notify(int new_state) 1526 { 1527 if (new_state) 1528 return; 1529 1530 /* Unprivileged eBPF is enabled */ 1531 1532 switch (spectre_v2_enabled) { 1533 case SPECTRE_V2_EIBRS: 1534 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1535 break; 1536 case SPECTRE_V2_EIBRS_LFENCE: 1537 if (sched_smt_active()) 1538 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1539 break; 1540 default: 1541 break; 1542 } 1543 } 1544 #endif 1545 1546 static inline bool match_option(const char *arg, int arglen, const char *opt) 1547 { 1548 int len = strlen(opt); 1549 1550 return len == arglen && !strncmp(arg, opt, len); 1551 } 1552 1553 /* The kernel command line selection for spectre v2 */ 1554 enum spectre_v2_mitigation_cmd { 1555 SPECTRE_V2_CMD_NONE, 1556 SPECTRE_V2_CMD_AUTO, 1557 SPECTRE_V2_CMD_FORCE, 1558 SPECTRE_V2_CMD_RETPOLINE, 1559 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1560 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1561 SPECTRE_V2_CMD_EIBRS, 1562 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1563 SPECTRE_V2_CMD_EIBRS_LFENCE, 1564 SPECTRE_V2_CMD_IBRS, 1565 }; 1566 1567 static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init = SPECTRE_V2_CMD_AUTO; 1568 1569 enum spectre_v2_user_cmd { 1570 SPECTRE_V2_USER_CMD_NONE, 1571 SPECTRE_V2_USER_CMD_AUTO, 1572 SPECTRE_V2_USER_CMD_FORCE, 1573 SPECTRE_V2_USER_CMD_PRCTL, 1574 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1575 SPECTRE_V2_USER_CMD_SECCOMP, 1576 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1577 }; 1578 1579 static const char * const spectre_v2_user_strings[] = { 1580 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1581 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1582 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1583 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1584 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1585 }; 1586 1587 static const struct { 1588 const char *option; 1589 enum spectre_v2_user_cmd cmd; 1590 bool secure; 1591 } v2_user_options[] __initconst = { 1592 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 1593 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 1594 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 1595 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 1596 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 1597 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 1598 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 1599 }; 1600 1601 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 1602 { 1603 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1604 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 1605 } 1606 1607 static enum spectre_v2_user_cmd __init spectre_v2_parse_user_cmdline(void) 1608 { 1609 char arg[20]; 1610 int ret, i; 1611 1612 if (cpu_mitigations_off() || !IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2)) 1613 return SPECTRE_V2_USER_CMD_NONE; 1614 1615 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 1616 arg, sizeof(arg)); 1617 if (ret < 0) 1618 return SPECTRE_V2_USER_CMD_AUTO; 1619 1620 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 1621 if (match_option(arg, ret, v2_user_options[i].option)) { 1622 spec_v2_user_print_cond(v2_user_options[i].option, 1623 v2_user_options[i].secure); 1624 return v2_user_options[i].cmd; 1625 } 1626 } 1627 1628 pr_err("Unknown user space protection option (%s). Switching to default\n", arg); 1629 return SPECTRE_V2_USER_CMD_AUTO; 1630 } 1631 1632 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1633 { 1634 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1635 } 1636 1637 static void __init spectre_v2_user_select_mitigation(void) 1638 { 1639 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1640 return; 1641 1642 switch (spectre_v2_parse_user_cmdline()) { 1643 case SPECTRE_V2_USER_CMD_NONE: 1644 return; 1645 case SPECTRE_V2_USER_CMD_FORCE: 1646 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1647 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1648 break; 1649 case SPECTRE_V2_USER_CMD_AUTO: 1650 case SPECTRE_V2_USER_CMD_PRCTL: 1651 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1652 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1653 break; 1654 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1655 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1656 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1657 break; 1658 case SPECTRE_V2_USER_CMD_SECCOMP: 1659 if (IS_ENABLED(CONFIG_SECCOMP)) 1660 spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP; 1661 else 1662 spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; 1663 spectre_v2_user_stibp = spectre_v2_user_ibpb; 1664 break; 1665 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1666 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1667 if (IS_ENABLED(CONFIG_SECCOMP)) 1668 spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP; 1669 else 1670 spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; 1671 break; 1672 } 1673 1674 /* 1675 * At this point, an STIBP mode other than "off" has been set. 1676 * If STIBP support is not being forced, check if STIBP always-on 1677 * is preferred. 1678 */ 1679 if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1680 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) && 1681 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1682 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1683 1684 if (!boot_cpu_has(X86_FEATURE_IBPB)) 1685 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1686 1687 if (!boot_cpu_has(X86_FEATURE_STIBP)) 1688 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1689 } 1690 1691 static void __init spectre_v2_user_update_mitigation(void) 1692 { 1693 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1694 return; 1695 1696 /* The spectre_v2 cmd line can override spectre_v2_user options */ 1697 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) { 1698 spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; 1699 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1700 } else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) { 1701 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1702 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; 1703 } 1704 1705 /* 1706 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP 1707 * is not required. 1708 * 1709 * Intel's Enhanced IBRS also protects against cross-thread branch target 1710 * injection in user-mode as the IBRS bit remains always set which 1711 * implicitly enables cross-thread protections. However, in legacy IBRS 1712 * mode, the IBRS bit is set only on kernel entry and cleared on return 1713 * to userspace. AMD Automatic IBRS also does not protect userspace. 1714 * These modes therefore disable the implicit cross-thread protection, 1715 * so allow for STIBP to be selected in those cases. 1716 */ 1717 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1718 !cpu_smt_possible() || 1719 (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 1720 !boot_cpu_has(X86_FEATURE_AUTOIBRS))) { 1721 spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; 1722 return; 1723 } 1724 1725 if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE && 1726 (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1727 retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) { 1728 if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT && 1729 spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED) 1730 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1731 spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; 1732 } 1733 pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]); 1734 } 1735 1736 static void __init spectre_v2_user_apply_mitigation(void) 1737 { 1738 /* Initialize Indirect Branch Prediction Barrier */ 1739 if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) { 1740 static_branch_enable(&switch_vcpu_ibpb); 1741 1742 switch (spectre_v2_user_ibpb) { 1743 case SPECTRE_V2_USER_STRICT: 1744 static_branch_enable(&switch_mm_always_ibpb); 1745 break; 1746 case SPECTRE_V2_USER_PRCTL: 1747 case SPECTRE_V2_USER_SECCOMP: 1748 static_branch_enable(&switch_mm_cond_ibpb); 1749 break; 1750 default: 1751 break; 1752 } 1753 1754 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1755 static_key_enabled(&switch_mm_always_ibpb) ? 1756 "always-on" : "conditional"); 1757 } 1758 } 1759 1760 static const char * const spectre_v2_strings[] = { 1761 [SPECTRE_V2_NONE] = "Vulnerable", 1762 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1763 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 1764 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1765 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1766 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1767 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1768 }; 1769 1770 static const struct { 1771 const char *option; 1772 enum spectre_v2_mitigation_cmd cmd; 1773 bool secure; 1774 } mitigation_options[] __initconst = { 1775 { "off", SPECTRE_V2_CMD_NONE, false }, 1776 { "on", SPECTRE_V2_CMD_FORCE, true }, 1777 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 1778 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1779 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1780 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 1781 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 1782 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 1783 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 1784 { "auto", SPECTRE_V2_CMD_AUTO, false }, 1785 { "ibrs", SPECTRE_V2_CMD_IBRS, false }, 1786 }; 1787 1788 static void __init spec_v2_print_cond(const char *reason, bool secure) 1789 { 1790 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1791 pr_info("%s selected on command line.\n", reason); 1792 } 1793 1794 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 1795 { 1796 enum spectre_v2_mitigation_cmd cmd; 1797 char arg[20]; 1798 int ret, i; 1799 1800 cmd = IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE; 1801 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 1802 cpu_mitigations_off()) 1803 return SPECTRE_V2_CMD_NONE; 1804 1805 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 1806 if (ret < 0) 1807 return cmd; 1808 1809 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 1810 if (!match_option(arg, ret, mitigation_options[i].option)) 1811 continue; 1812 cmd = mitigation_options[i].cmd; 1813 break; 1814 } 1815 1816 if (i >= ARRAY_SIZE(mitigation_options)) { 1817 pr_err("unknown option (%s). Switching to default mode\n", arg); 1818 return cmd; 1819 } 1820 1821 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 1822 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1823 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 1824 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1825 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1826 !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 1827 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1828 mitigation_options[i].option); 1829 return SPECTRE_V2_CMD_AUTO; 1830 } 1831 1832 if ((cmd == SPECTRE_V2_CMD_EIBRS || 1833 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1834 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1835 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1836 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n", 1837 mitigation_options[i].option); 1838 return SPECTRE_V2_CMD_AUTO; 1839 } 1840 1841 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1842 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 1843 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 1844 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 1845 mitigation_options[i].option); 1846 return SPECTRE_V2_CMD_AUTO; 1847 } 1848 1849 if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) { 1850 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1851 mitigation_options[i].option); 1852 return SPECTRE_V2_CMD_AUTO; 1853 } 1854 1855 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1856 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n", 1857 mitigation_options[i].option); 1858 return SPECTRE_V2_CMD_AUTO; 1859 } 1860 1861 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 1862 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n", 1863 mitigation_options[i].option); 1864 return SPECTRE_V2_CMD_AUTO; 1865 } 1866 1867 if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 1868 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n", 1869 mitigation_options[i].option); 1870 return SPECTRE_V2_CMD_AUTO; 1871 } 1872 1873 spec_v2_print_cond(mitigation_options[i].option, 1874 mitigation_options[i].secure); 1875 return cmd; 1876 } 1877 1878 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1879 { 1880 if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) { 1881 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1882 return SPECTRE_V2_NONE; 1883 } 1884 1885 return SPECTRE_V2_RETPOLINE; 1886 } 1887 1888 static bool __ro_after_init rrsba_disabled; 1889 1890 /* Disable in-kernel use of non-RSB RET predictors */ 1891 static void __init spec_ctrl_disable_kernel_rrsba(void) 1892 { 1893 if (rrsba_disabled) 1894 return; 1895 1896 if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) { 1897 rrsba_disabled = true; 1898 return; 1899 } 1900 1901 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1902 return; 1903 1904 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1905 update_spec_ctrl(x86_spec_ctrl_base); 1906 rrsba_disabled = true; 1907 } 1908 1909 static void __init spectre_v2_select_rsb_mitigation(enum spectre_v2_mitigation mode) 1910 { 1911 /* 1912 * WARNING! There are many subtleties to consider when changing *any* 1913 * code related to RSB-related mitigations. Before doing so, carefully 1914 * read the following document, and update if necessary: 1915 * 1916 * Documentation/admin-guide/hw-vuln/rsb.rst 1917 * 1918 * In an overly simplified nutshell: 1919 * 1920 * - User->user RSB attacks are conditionally mitigated during 1921 * context switches by cond_mitigation -> write_ibpb(). 1922 * 1923 * - User->kernel and guest->host attacks are mitigated by eIBRS or 1924 * RSB filling. 1925 * 1926 * Though, depending on config, note that other alternative 1927 * mitigations may end up getting used instead, e.g., IBPB on 1928 * entry/vmexit, call depth tracking, or return thunks. 1929 */ 1930 1931 switch (mode) { 1932 case SPECTRE_V2_NONE: 1933 break; 1934 1935 case SPECTRE_V2_EIBRS: 1936 case SPECTRE_V2_EIBRS_LFENCE: 1937 case SPECTRE_V2_EIBRS_RETPOLINE: 1938 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 1939 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 1940 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 1941 } 1942 break; 1943 1944 case SPECTRE_V2_RETPOLINE: 1945 case SPECTRE_V2_LFENCE: 1946 case SPECTRE_V2_IBRS: 1947 pr_info("Spectre v2 / SpectreRSB: Filling RSB on context switch and VMEXIT\n"); 1948 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1949 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1950 break; 1951 1952 default: 1953 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation\n"); 1954 dump_stack(); 1955 break; 1956 } 1957 } 1958 1959 /* 1960 * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by 1961 * branch history in userspace. Not needed if BHI_NO is set. 1962 */ 1963 static bool __init spec_ctrl_bhi_dis(void) 1964 { 1965 if (!boot_cpu_has(X86_FEATURE_BHI_CTRL)) 1966 return false; 1967 1968 x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S; 1969 update_spec_ctrl(x86_spec_ctrl_base); 1970 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW); 1971 1972 return true; 1973 } 1974 1975 enum bhi_mitigations { 1976 BHI_MITIGATION_OFF, 1977 BHI_MITIGATION_AUTO, 1978 BHI_MITIGATION_ON, 1979 BHI_MITIGATION_VMEXIT_ONLY, 1980 }; 1981 1982 static enum bhi_mitigations bhi_mitigation __ro_after_init = 1983 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF; 1984 1985 static int __init spectre_bhi_parse_cmdline(char *str) 1986 { 1987 if (!str) 1988 return -EINVAL; 1989 1990 if (!strcmp(str, "off")) 1991 bhi_mitigation = BHI_MITIGATION_OFF; 1992 else if (!strcmp(str, "on")) 1993 bhi_mitigation = BHI_MITIGATION_ON; 1994 else if (!strcmp(str, "vmexit")) 1995 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY; 1996 else 1997 pr_err("Ignoring unknown spectre_bhi option (%s)", str); 1998 1999 return 0; 2000 } 2001 early_param("spectre_bhi", spectre_bhi_parse_cmdline); 2002 2003 static void __init bhi_select_mitigation(void) 2004 { 2005 if (!boot_cpu_has(X86_BUG_BHI) || cpu_mitigations_off()) 2006 bhi_mitigation = BHI_MITIGATION_OFF; 2007 2008 if (bhi_mitigation == BHI_MITIGATION_AUTO) 2009 bhi_mitigation = BHI_MITIGATION_ON; 2010 } 2011 2012 static void __init bhi_update_mitigation(void) 2013 { 2014 if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) 2015 bhi_mitigation = BHI_MITIGATION_OFF; 2016 2017 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 2018 spectre_v2_cmd == SPECTRE_V2_CMD_AUTO) 2019 bhi_mitigation = BHI_MITIGATION_OFF; 2020 } 2021 2022 static void __init bhi_apply_mitigation(void) 2023 { 2024 if (bhi_mitigation == BHI_MITIGATION_OFF) 2025 return; 2026 2027 /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */ 2028 if (boot_cpu_has(X86_FEATURE_RETPOLINE) && 2029 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { 2030 spec_ctrl_disable_kernel_rrsba(); 2031 if (rrsba_disabled) 2032 return; 2033 } 2034 2035 if (!IS_ENABLED(CONFIG_X86_64)) 2036 return; 2037 2038 /* Mitigate in hardware if supported */ 2039 if (spec_ctrl_bhi_dis()) 2040 return; 2041 2042 if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) { 2043 pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n"); 2044 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2045 return; 2046 } 2047 2048 pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n"); 2049 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP); 2050 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); 2051 } 2052 2053 static void __init spectre_v2_select_mitigation(void) 2054 { 2055 spectre_v2_cmd = spectre_v2_parse_cmdline(); 2056 2057 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 2058 (spectre_v2_cmd == SPECTRE_V2_CMD_NONE || spectre_v2_cmd == SPECTRE_V2_CMD_AUTO)) 2059 return; 2060 2061 switch (spectre_v2_cmd) { 2062 case SPECTRE_V2_CMD_NONE: 2063 return; 2064 2065 case SPECTRE_V2_CMD_FORCE: 2066 case SPECTRE_V2_CMD_AUTO: 2067 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 2068 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2069 break; 2070 } 2071 2072 spectre_v2_enabled = spectre_v2_select_retpoline(); 2073 break; 2074 2075 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 2076 pr_err(SPECTRE_V2_LFENCE_MSG); 2077 spectre_v2_enabled = SPECTRE_V2_LFENCE; 2078 break; 2079 2080 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 2081 spectre_v2_enabled = SPECTRE_V2_RETPOLINE; 2082 break; 2083 2084 case SPECTRE_V2_CMD_RETPOLINE: 2085 spectre_v2_enabled = spectre_v2_select_retpoline(); 2086 break; 2087 2088 case SPECTRE_V2_CMD_IBRS: 2089 spectre_v2_enabled = SPECTRE_V2_IBRS; 2090 break; 2091 2092 case SPECTRE_V2_CMD_EIBRS: 2093 spectre_v2_enabled = SPECTRE_V2_EIBRS; 2094 break; 2095 2096 case SPECTRE_V2_CMD_EIBRS_LFENCE: 2097 spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE; 2098 break; 2099 2100 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 2101 spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE; 2102 break; 2103 } 2104 } 2105 2106 static void __init spectre_v2_update_mitigation(void) 2107 { 2108 if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO && 2109 !spectre_v2_in_eibrs_mode(spectre_v2_enabled)) { 2110 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && 2111 boot_cpu_has_bug(X86_BUG_RETBLEED) && 2112 retbleed_mitigation != RETBLEED_MITIGATION_NONE && 2113 retbleed_mitigation != RETBLEED_MITIGATION_STUFF && 2114 boot_cpu_has(X86_FEATURE_IBRS) && 2115 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 2116 spectre_v2_enabled = SPECTRE_V2_IBRS; 2117 } 2118 } 2119 2120 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && !cpu_mitigations_off()) 2121 pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); 2122 } 2123 2124 static void __init spectre_v2_apply_mitigation(void) 2125 { 2126 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2127 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 2128 2129 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2130 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 2131 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 2132 } else { 2133 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 2134 update_spec_ctrl(x86_spec_ctrl_base); 2135 } 2136 } 2137 2138 switch (spectre_v2_enabled) { 2139 case SPECTRE_V2_NONE: 2140 return; 2141 2142 case SPECTRE_V2_EIBRS: 2143 break; 2144 2145 case SPECTRE_V2_IBRS: 2146 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 2147 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 2148 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 2149 break; 2150 2151 case SPECTRE_V2_LFENCE: 2152 case SPECTRE_V2_EIBRS_LFENCE: 2153 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 2154 fallthrough; 2155 2156 case SPECTRE_V2_RETPOLINE: 2157 case SPECTRE_V2_EIBRS_RETPOLINE: 2158 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 2159 break; 2160 } 2161 2162 /* 2163 * Disable alternate RSB predictions in kernel when indirect CALLs and 2164 * JMPs gets protection against BHI and Intramode-BTI, but RET 2165 * prediction from a non-RSB predictor is still a risk. 2166 */ 2167 if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE || 2168 spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE || 2169 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) 2170 spec_ctrl_disable_kernel_rrsba(); 2171 2172 spectre_v2_select_rsb_mitigation(spectre_v2_enabled); 2173 2174 /* 2175 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 2176 * and Enhanced IBRS protect firmware too, so enable IBRS around 2177 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 2178 * otherwise enabled. 2179 * 2180 * Use "spectre_v2_enabled" to check Enhanced IBRS instead of 2181 * boot_cpu_has(), because the user might select retpoline on the kernel 2182 * command line and if the CPU supports Enhanced IBRS, kernel might 2183 * un-intentionally not enable IBRS around firmware calls. 2184 */ 2185 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 2186 boot_cpu_has(X86_FEATURE_IBPB) && 2187 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2188 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 2189 2190 if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) { 2191 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 2192 pr_info("Enabling Speculation Barrier for firmware calls\n"); 2193 } 2194 2195 } else if (boot_cpu_has(X86_FEATURE_IBRS) && 2196 !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { 2197 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 2198 pr_info("Enabling Restricted Speculation for firmware calls\n"); 2199 } 2200 } 2201 2202 static void update_stibp_msr(void * __unused) 2203 { 2204 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 2205 update_spec_ctrl(val); 2206 } 2207 2208 /* Update x86_spec_ctrl_base in case SMT state changed. */ 2209 static void update_stibp_strict(void) 2210 { 2211 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 2212 2213 if (sched_smt_active()) 2214 mask |= SPEC_CTRL_STIBP; 2215 2216 if (mask == x86_spec_ctrl_base) 2217 return; 2218 2219 pr_info("Update user space SMT mitigation: STIBP %s\n", 2220 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 2221 x86_spec_ctrl_base = mask; 2222 on_each_cpu(update_stibp_msr, NULL, 1); 2223 } 2224 2225 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 2226 static void update_indir_branch_cond(void) 2227 { 2228 if (sched_smt_active()) 2229 static_branch_enable(&switch_to_cond_stibp); 2230 else 2231 static_branch_disable(&switch_to_cond_stibp); 2232 } 2233 2234 #undef pr_fmt 2235 #define pr_fmt(fmt) fmt 2236 2237 /* Update the static key controlling the MDS CPU buffer clear in idle */ 2238 static void update_mds_branch_idle(void) 2239 { 2240 /* 2241 * Enable the idle clearing if SMT is active on CPUs which are 2242 * affected only by MSBDS and not any other MDS variant. 2243 * 2244 * The other variants cannot be mitigated when SMT is enabled, so 2245 * clearing the buffers on idle just to prevent the Store Buffer 2246 * repartitioning leak would be a window dressing exercise. 2247 */ 2248 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 2249 return; 2250 2251 if (sched_smt_active()) { 2252 static_branch_enable(&mds_idle_clear); 2253 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 2254 (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) { 2255 static_branch_disable(&mds_idle_clear); 2256 } 2257 } 2258 2259 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 2260 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 2261 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 2262 2263 void cpu_bugs_smt_update(void) 2264 { 2265 mutex_lock(&spec_ctrl_mutex); 2266 2267 if (sched_smt_active() && unprivileged_ebpf_enabled() && 2268 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 2269 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 2270 2271 switch (spectre_v2_user_stibp) { 2272 case SPECTRE_V2_USER_NONE: 2273 break; 2274 case SPECTRE_V2_USER_STRICT: 2275 case SPECTRE_V2_USER_STRICT_PREFERRED: 2276 update_stibp_strict(); 2277 break; 2278 case SPECTRE_V2_USER_PRCTL: 2279 case SPECTRE_V2_USER_SECCOMP: 2280 update_indir_branch_cond(); 2281 break; 2282 } 2283 2284 switch (mds_mitigation) { 2285 case MDS_MITIGATION_FULL: 2286 case MDS_MITIGATION_AUTO: 2287 case MDS_MITIGATION_VMWERV: 2288 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 2289 pr_warn_once(MDS_MSG_SMT); 2290 update_mds_branch_idle(); 2291 break; 2292 case MDS_MITIGATION_OFF: 2293 break; 2294 } 2295 2296 switch (taa_mitigation) { 2297 case TAA_MITIGATION_VERW: 2298 case TAA_MITIGATION_AUTO: 2299 case TAA_MITIGATION_UCODE_NEEDED: 2300 if (sched_smt_active()) 2301 pr_warn_once(TAA_MSG_SMT); 2302 break; 2303 case TAA_MITIGATION_TSX_DISABLED: 2304 case TAA_MITIGATION_OFF: 2305 break; 2306 } 2307 2308 switch (mmio_mitigation) { 2309 case MMIO_MITIGATION_VERW: 2310 case MMIO_MITIGATION_AUTO: 2311 case MMIO_MITIGATION_UCODE_NEEDED: 2312 if (sched_smt_active()) 2313 pr_warn_once(MMIO_MSG_SMT); 2314 break; 2315 case MMIO_MITIGATION_OFF: 2316 break; 2317 } 2318 2319 mutex_unlock(&spec_ctrl_mutex); 2320 } 2321 2322 #undef pr_fmt 2323 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 2324 2325 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 2326 2327 /* The kernel command line selection */ 2328 enum ssb_mitigation_cmd { 2329 SPEC_STORE_BYPASS_CMD_NONE, 2330 SPEC_STORE_BYPASS_CMD_AUTO, 2331 SPEC_STORE_BYPASS_CMD_ON, 2332 SPEC_STORE_BYPASS_CMD_PRCTL, 2333 SPEC_STORE_BYPASS_CMD_SECCOMP, 2334 }; 2335 2336 static const char * const ssb_strings[] = { 2337 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 2338 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 2339 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 2340 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 2341 }; 2342 2343 static const struct { 2344 const char *option; 2345 enum ssb_mitigation_cmd cmd; 2346 } ssb_mitigation_options[] __initconst = { 2347 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 2348 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 2349 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 2350 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 2351 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 2352 }; 2353 2354 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 2355 { 2356 enum ssb_mitigation_cmd cmd; 2357 char arg[20]; 2358 int ret, i; 2359 2360 cmd = IS_ENABLED(CONFIG_MITIGATION_SSB) ? 2361 SPEC_STORE_BYPASS_CMD_AUTO : SPEC_STORE_BYPASS_CMD_NONE; 2362 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 2363 cpu_mitigations_off()) { 2364 return SPEC_STORE_BYPASS_CMD_NONE; 2365 } else { 2366 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 2367 arg, sizeof(arg)); 2368 if (ret < 0) 2369 return cmd; 2370 2371 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 2372 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 2373 continue; 2374 2375 cmd = ssb_mitigation_options[i].cmd; 2376 break; 2377 } 2378 2379 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 2380 pr_err("unknown option (%s). Switching to default mode\n", arg); 2381 return cmd; 2382 } 2383 } 2384 2385 return cmd; 2386 } 2387 2388 static void __init ssb_select_mitigation(void) 2389 { 2390 enum ssb_mitigation_cmd cmd; 2391 2392 if (!boot_cpu_has(X86_FEATURE_SSBD)) 2393 goto out; 2394 2395 cmd = ssb_parse_cmdline(); 2396 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 2397 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 2398 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 2399 return; 2400 2401 switch (cmd) { 2402 case SPEC_STORE_BYPASS_CMD_SECCOMP: 2403 /* 2404 * Choose prctl+seccomp as the default mode if seccomp is 2405 * enabled. 2406 */ 2407 if (IS_ENABLED(CONFIG_SECCOMP)) 2408 ssb_mode = SPEC_STORE_BYPASS_SECCOMP; 2409 else 2410 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2411 break; 2412 case SPEC_STORE_BYPASS_CMD_ON: 2413 ssb_mode = SPEC_STORE_BYPASS_DISABLE; 2414 break; 2415 case SPEC_STORE_BYPASS_CMD_AUTO: 2416 case SPEC_STORE_BYPASS_CMD_PRCTL: 2417 ssb_mode = SPEC_STORE_BYPASS_PRCTL; 2418 break; 2419 case SPEC_STORE_BYPASS_CMD_NONE: 2420 break; 2421 } 2422 2423 out: 2424 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2425 pr_info("%s\n", ssb_strings[ssb_mode]); 2426 } 2427 2428 static void __init ssb_apply_mitigation(void) 2429 { 2430 /* 2431 * We have three CPU feature flags that are in play here: 2432 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 2433 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 2434 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 2435 */ 2436 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) { 2437 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 2438 /* 2439 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 2440 * use a completely different MSR and bit dependent on family. 2441 */ 2442 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 2443 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 2444 x86_amd_ssb_disable(); 2445 } else { 2446 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 2447 update_spec_ctrl(x86_spec_ctrl_base); 2448 } 2449 } 2450 } 2451 2452 #undef pr_fmt 2453 #define pr_fmt(fmt) "Speculation prctl: " fmt 2454 2455 static void task_update_spec_tif(struct task_struct *tsk) 2456 { 2457 /* Force the update of the real TIF bits */ 2458 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 2459 2460 /* 2461 * Immediately update the speculation control MSRs for the current 2462 * task, but for a non-current task delay setting the CPU 2463 * mitigation until it is scheduled next. 2464 * 2465 * This can only happen for SECCOMP mitigation. For PRCTL it's 2466 * always the current task. 2467 */ 2468 if (tsk == current) 2469 speculation_ctrl_update_current(); 2470 } 2471 2472 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 2473 { 2474 2475 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2476 return -EPERM; 2477 2478 switch (ctrl) { 2479 case PR_SPEC_ENABLE: 2480 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2481 return 0; 2482 case PR_SPEC_DISABLE: 2483 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 2484 return 0; 2485 default: 2486 return -ERANGE; 2487 } 2488 } 2489 2490 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 2491 { 2492 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 2493 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 2494 return -ENXIO; 2495 2496 switch (ctrl) { 2497 case PR_SPEC_ENABLE: 2498 /* If speculation is force disabled, enable is not allowed */ 2499 if (task_spec_ssb_force_disable(task)) 2500 return -EPERM; 2501 task_clear_spec_ssb_disable(task); 2502 task_clear_spec_ssb_noexec(task); 2503 task_update_spec_tif(task); 2504 break; 2505 case PR_SPEC_DISABLE: 2506 task_set_spec_ssb_disable(task); 2507 task_clear_spec_ssb_noexec(task); 2508 task_update_spec_tif(task); 2509 break; 2510 case PR_SPEC_FORCE_DISABLE: 2511 task_set_spec_ssb_disable(task); 2512 task_set_spec_ssb_force_disable(task); 2513 task_clear_spec_ssb_noexec(task); 2514 task_update_spec_tif(task); 2515 break; 2516 case PR_SPEC_DISABLE_NOEXEC: 2517 if (task_spec_ssb_force_disable(task)) 2518 return -EPERM; 2519 task_set_spec_ssb_disable(task); 2520 task_set_spec_ssb_noexec(task); 2521 task_update_spec_tif(task); 2522 break; 2523 default: 2524 return -ERANGE; 2525 } 2526 return 0; 2527 } 2528 2529 static bool is_spec_ib_user_controlled(void) 2530 { 2531 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 2532 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2533 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 2534 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 2535 } 2536 2537 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 2538 { 2539 switch (ctrl) { 2540 case PR_SPEC_ENABLE: 2541 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2542 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2543 return 0; 2544 2545 /* 2546 * With strict mode for both IBPB and STIBP, the instruction 2547 * code paths avoid checking this task flag and instead, 2548 * unconditionally run the instruction. However, STIBP and IBPB 2549 * are independent and either can be set to conditionally 2550 * enabled regardless of the mode of the other. 2551 * 2552 * If either is set to conditional, allow the task flag to be 2553 * updated, unless it was force-disabled by a previous prctl 2554 * call. Currently, this is possible on an AMD CPU which has the 2555 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 2556 * kernel is booted with 'spectre_v2_user=seccomp', then 2557 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 2558 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 2559 */ 2560 if (!is_spec_ib_user_controlled() || 2561 task_spec_ib_force_disable(task)) 2562 return -EPERM; 2563 2564 task_clear_spec_ib_disable(task); 2565 task_update_spec_tif(task); 2566 break; 2567 case PR_SPEC_DISABLE: 2568 case PR_SPEC_FORCE_DISABLE: 2569 /* 2570 * Indirect branch speculation is always allowed when 2571 * mitigation is force disabled. 2572 */ 2573 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2574 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2575 return -EPERM; 2576 2577 if (!is_spec_ib_user_controlled()) 2578 return 0; 2579 2580 task_set_spec_ib_disable(task); 2581 if (ctrl == PR_SPEC_FORCE_DISABLE) 2582 task_set_spec_ib_force_disable(task); 2583 task_update_spec_tif(task); 2584 if (task == current) 2585 indirect_branch_prediction_barrier(); 2586 break; 2587 default: 2588 return -ERANGE; 2589 } 2590 return 0; 2591 } 2592 2593 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2594 unsigned long ctrl) 2595 { 2596 switch (which) { 2597 case PR_SPEC_STORE_BYPASS: 2598 return ssb_prctl_set(task, ctrl); 2599 case PR_SPEC_INDIRECT_BRANCH: 2600 return ib_prctl_set(task, ctrl); 2601 case PR_SPEC_L1D_FLUSH: 2602 return l1d_flush_prctl_set(task, ctrl); 2603 default: 2604 return -ENODEV; 2605 } 2606 } 2607 2608 #ifdef CONFIG_SECCOMP 2609 void arch_seccomp_spec_mitigate(struct task_struct *task) 2610 { 2611 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2612 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2613 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2614 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2615 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2616 } 2617 #endif 2618 2619 static int l1d_flush_prctl_get(struct task_struct *task) 2620 { 2621 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2622 return PR_SPEC_FORCE_DISABLE; 2623 2624 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2625 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2626 else 2627 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2628 } 2629 2630 static int ssb_prctl_get(struct task_struct *task) 2631 { 2632 switch (ssb_mode) { 2633 case SPEC_STORE_BYPASS_NONE: 2634 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2635 return PR_SPEC_ENABLE; 2636 return PR_SPEC_NOT_AFFECTED; 2637 case SPEC_STORE_BYPASS_DISABLE: 2638 return PR_SPEC_DISABLE; 2639 case SPEC_STORE_BYPASS_SECCOMP: 2640 case SPEC_STORE_BYPASS_PRCTL: 2641 if (task_spec_ssb_force_disable(task)) 2642 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2643 if (task_spec_ssb_noexec(task)) 2644 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2645 if (task_spec_ssb_disable(task)) 2646 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2647 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2648 } 2649 BUG(); 2650 } 2651 2652 static int ib_prctl_get(struct task_struct *task) 2653 { 2654 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2655 return PR_SPEC_NOT_AFFECTED; 2656 2657 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2658 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2659 return PR_SPEC_ENABLE; 2660 else if (is_spec_ib_user_controlled()) { 2661 if (task_spec_ib_force_disable(task)) 2662 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2663 if (task_spec_ib_disable(task)) 2664 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2665 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2666 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2667 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2668 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2669 return PR_SPEC_DISABLE; 2670 else 2671 return PR_SPEC_NOT_AFFECTED; 2672 } 2673 2674 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2675 { 2676 switch (which) { 2677 case PR_SPEC_STORE_BYPASS: 2678 return ssb_prctl_get(task); 2679 case PR_SPEC_INDIRECT_BRANCH: 2680 return ib_prctl_get(task); 2681 case PR_SPEC_L1D_FLUSH: 2682 return l1d_flush_prctl_get(task); 2683 default: 2684 return -ENODEV; 2685 } 2686 } 2687 2688 void x86_spec_ctrl_setup_ap(void) 2689 { 2690 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2691 update_spec_ctrl(x86_spec_ctrl_base); 2692 2693 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2694 x86_amd_ssb_disable(); 2695 } 2696 2697 bool itlb_multihit_kvm_mitigation; 2698 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 2699 2700 #undef pr_fmt 2701 #define pr_fmt(fmt) "L1TF: " fmt 2702 2703 /* Default mitigation for L1TF-affected CPUs */ 2704 enum l1tf_mitigations l1tf_mitigation __ro_after_init = 2705 IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_AUTO : L1TF_MITIGATION_OFF; 2706 #if IS_ENABLED(CONFIG_KVM_INTEL) 2707 EXPORT_SYMBOL_GPL(l1tf_mitigation); 2708 #endif 2709 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2710 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 2711 2712 /* 2713 * These CPUs all support 44bits physical address space internally in the 2714 * cache but CPUID can report a smaller number of physical address bits. 2715 * 2716 * The L1TF mitigation uses the top most address bit for the inversion of 2717 * non present PTEs. When the installed memory reaches into the top most 2718 * address bit due to memory holes, which has been observed on machines 2719 * which report 36bits physical address bits and have 32G RAM installed, 2720 * then the mitigation range check in l1tf_select_mitigation() triggers. 2721 * This is a false positive because the mitigation is still possible due to 2722 * the fact that the cache uses 44bit internally. Use the cache bits 2723 * instead of the reported physical bits and adjust them on the affected 2724 * machines to 44bit if the reported bits are less than 44. 2725 */ 2726 static void override_cache_bits(struct cpuinfo_x86 *c) 2727 { 2728 if (c->x86 != 6) 2729 return; 2730 2731 switch (c->x86_vfm) { 2732 case INTEL_NEHALEM: 2733 case INTEL_WESTMERE: 2734 case INTEL_SANDYBRIDGE: 2735 case INTEL_IVYBRIDGE: 2736 case INTEL_HASWELL: 2737 case INTEL_HASWELL_L: 2738 case INTEL_HASWELL_G: 2739 case INTEL_BROADWELL: 2740 case INTEL_BROADWELL_G: 2741 case INTEL_SKYLAKE_L: 2742 case INTEL_SKYLAKE: 2743 case INTEL_KABYLAKE_L: 2744 case INTEL_KABYLAKE: 2745 if (c->x86_cache_bits < 44) 2746 c->x86_cache_bits = 44; 2747 break; 2748 } 2749 } 2750 2751 static void __init l1tf_select_mitigation(void) 2752 { 2753 if (!boot_cpu_has_bug(X86_BUG_L1TF) || cpu_mitigations_off()) { 2754 l1tf_mitigation = L1TF_MITIGATION_OFF; 2755 return; 2756 } 2757 2758 if (l1tf_mitigation == L1TF_MITIGATION_AUTO) { 2759 if (cpu_mitigations_auto_nosmt()) 2760 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2761 else 2762 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2763 } 2764 } 2765 2766 static void __init l1tf_apply_mitigation(void) 2767 { 2768 u64 half_pa; 2769 2770 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2771 return; 2772 2773 override_cache_bits(&boot_cpu_data); 2774 2775 switch (l1tf_mitigation) { 2776 case L1TF_MITIGATION_OFF: 2777 case L1TF_MITIGATION_FLUSH_NOWARN: 2778 case L1TF_MITIGATION_FLUSH: 2779 case L1TF_MITIGATION_AUTO: 2780 break; 2781 case L1TF_MITIGATION_FLUSH_NOSMT: 2782 case L1TF_MITIGATION_FULL: 2783 cpu_smt_disable(false); 2784 break; 2785 case L1TF_MITIGATION_FULL_FORCE: 2786 cpu_smt_disable(true); 2787 break; 2788 } 2789 2790 #if CONFIG_PGTABLE_LEVELS == 2 2791 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2792 return; 2793 #endif 2794 2795 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2796 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2797 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2798 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2799 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2800 half_pa); 2801 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2802 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2803 return; 2804 } 2805 2806 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2807 } 2808 2809 static int __init l1tf_cmdline(char *str) 2810 { 2811 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2812 return 0; 2813 2814 if (!str) 2815 return -EINVAL; 2816 2817 if (!strcmp(str, "off")) 2818 l1tf_mitigation = L1TF_MITIGATION_OFF; 2819 else if (!strcmp(str, "flush,nowarn")) 2820 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2821 else if (!strcmp(str, "flush")) 2822 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2823 else if (!strcmp(str, "flush,nosmt")) 2824 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2825 else if (!strcmp(str, "full")) 2826 l1tf_mitigation = L1TF_MITIGATION_FULL; 2827 else if (!strcmp(str, "full,force")) 2828 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2829 2830 return 0; 2831 } 2832 early_param("l1tf", l1tf_cmdline); 2833 2834 #undef pr_fmt 2835 #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt 2836 2837 enum srso_mitigation { 2838 SRSO_MITIGATION_NONE, 2839 SRSO_MITIGATION_AUTO, 2840 SRSO_MITIGATION_UCODE_NEEDED, 2841 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, 2842 SRSO_MITIGATION_MICROCODE, 2843 SRSO_MITIGATION_SAFE_RET, 2844 SRSO_MITIGATION_IBPB, 2845 SRSO_MITIGATION_IBPB_ON_VMEXIT, 2846 SRSO_MITIGATION_BP_SPEC_REDUCE, 2847 }; 2848 2849 static const char * const srso_strings[] = { 2850 [SRSO_MITIGATION_NONE] = "Vulnerable", 2851 [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 2852 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode", 2853 [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET", 2854 [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", 2855 [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", 2856 [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only", 2857 [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation" 2858 }; 2859 2860 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO; 2861 2862 static int __init srso_parse_cmdline(char *str) 2863 { 2864 if (!str) 2865 return -EINVAL; 2866 2867 if (!strcmp(str, "off")) 2868 srso_mitigation = SRSO_MITIGATION_NONE; 2869 else if (!strcmp(str, "microcode")) 2870 srso_mitigation = SRSO_MITIGATION_MICROCODE; 2871 else if (!strcmp(str, "safe-ret")) 2872 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2873 else if (!strcmp(str, "ibpb")) 2874 srso_mitigation = SRSO_MITIGATION_IBPB; 2875 else if (!strcmp(str, "ibpb-vmexit")) 2876 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2877 else 2878 pr_err("Ignoring unknown SRSO option (%s).", str); 2879 2880 return 0; 2881 } 2882 early_param("spec_rstack_overflow", srso_parse_cmdline); 2883 2884 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." 2885 2886 static void __init srso_select_mitigation(void) 2887 { 2888 bool has_microcode; 2889 2890 if (!boot_cpu_has_bug(X86_BUG_SRSO) || cpu_mitigations_off()) 2891 srso_mitigation = SRSO_MITIGATION_NONE; 2892 2893 if (srso_mitigation == SRSO_MITIGATION_NONE) 2894 return; 2895 2896 if (srso_mitigation == SRSO_MITIGATION_AUTO) 2897 srso_mitigation = SRSO_MITIGATION_SAFE_RET; 2898 2899 has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); 2900 if (has_microcode) { 2901 /* 2902 * Zen1/2 with SMT off aren't vulnerable after the right 2903 * IBPB microcode has been applied. 2904 */ 2905 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { 2906 setup_force_cpu_cap(X86_FEATURE_SRSO_NO); 2907 srso_mitigation = SRSO_MITIGATION_NONE; 2908 return; 2909 } 2910 } else { 2911 pr_warn("IBPB-extending microcode not applied!\n"); 2912 pr_warn(SRSO_NOTICE); 2913 } 2914 2915 switch (srso_mitigation) { 2916 case SRSO_MITIGATION_SAFE_RET: 2917 if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) { 2918 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; 2919 goto ibpb_on_vmexit; 2920 } 2921 2922 if (!IS_ENABLED(CONFIG_MITIGATION_SRSO)) { 2923 pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n"); 2924 srso_mitigation = SRSO_MITIGATION_NONE; 2925 } 2926 2927 if (!has_microcode) 2928 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; 2929 break; 2930 ibpb_on_vmexit: 2931 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 2932 if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) { 2933 pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n"); 2934 srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE; 2935 break; 2936 } 2937 fallthrough; 2938 case SRSO_MITIGATION_IBPB: 2939 if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { 2940 pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); 2941 srso_mitigation = SRSO_MITIGATION_NONE; 2942 } 2943 2944 if (!has_microcode) 2945 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; 2946 break; 2947 default: 2948 break; 2949 } 2950 } 2951 2952 static void __init srso_update_mitigation(void) 2953 { 2954 /* If retbleed is using IBPB, that works for SRSO as well */ 2955 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB && 2956 boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) 2957 srso_mitigation = SRSO_MITIGATION_IBPB; 2958 2959 if (boot_cpu_has_bug(X86_BUG_SRSO) && 2960 !cpu_mitigations_off() && 2961 !boot_cpu_has(X86_FEATURE_SRSO_NO)) 2962 pr_info("%s\n", srso_strings[srso_mitigation]); 2963 } 2964 2965 static void __init srso_apply_mitigation(void) 2966 { 2967 /* 2968 * Clear the feature flag if this mitigation is not selected as that 2969 * feature flag controls the BpSpecReduce MSR bit toggling in KVM. 2970 */ 2971 if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE) 2972 setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE); 2973 2974 if (srso_mitigation == SRSO_MITIGATION_NONE) { 2975 if (boot_cpu_has(X86_FEATURE_SBPB)) 2976 x86_pred_cmd = PRED_CMD_SBPB; 2977 return; 2978 } 2979 2980 switch (srso_mitigation) { 2981 case SRSO_MITIGATION_SAFE_RET: 2982 case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: 2983 /* 2984 * Enable the return thunk for generated code 2985 * like ftrace, static_call, etc. 2986 */ 2987 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 2988 setup_force_cpu_cap(X86_FEATURE_UNRET); 2989 2990 if (boot_cpu_data.x86 == 0x19) { 2991 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); 2992 set_return_thunk(srso_alias_return_thunk); 2993 } else { 2994 setup_force_cpu_cap(X86_FEATURE_SRSO); 2995 set_return_thunk(srso_return_thunk); 2996 } 2997 break; 2998 case SRSO_MITIGATION_IBPB: 2999 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 3000 /* 3001 * IBPB on entry already obviates the need for 3002 * software-based untraining so clear those in case some 3003 * other mitigation like Retbleed has selected them. 3004 */ 3005 setup_clear_cpu_cap(X86_FEATURE_UNRET); 3006 setup_clear_cpu_cap(X86_FEATURE_RETHUNK); 3007 fallthrough; 3008 case SRSO_MITIGATION_IBPB_ON_VMEXIT: 3009 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); 3010 /* 3011 * There is no need for RSB filling: entry_ibpb() ensures 3012 * all predictions, including the RSB, are invalidated, 3013 * regardless of IBPB implementation. 3014 */ 3015 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); 3016 break; 3017 default: 3018 break; 3019 } 3020 } 3021 3022 #undef pr_fmt 3023 #define pr_fmt(fmt) fmt 3024 3025 #ifdef CONFIG_SYSFS 3026 3027 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 3028 3029 #if IS_ENABLED(CONFIG_KVM_INTEL) 3030 static const char * const l1tf_vmx_states[] = { 3031 [VMENTER_L1D_FLUSH_AUTO] = "auto", 3032 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 3033 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 3034 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 3035 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 3036 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 3037 }; 3038 3039 static ssize_t l1tf_show_state(char *buf) 3040 { 3041 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 3042 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3043 3044 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 3045 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 3046 sched_smt_active())) { 3047 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 3048 l1tf_vmx_states[l1tf_vmx_mitigation]); 3049 } 3050 3051 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 3052 l1tf_vmx_states[l1tf_vmx_mitigation], 3053 sched_smt_active() ? "vulnerable" : "disabled"); 3054 } 3055 3056 static ssize_t itlb_multihit_show_state(char *buf) 3057 { 3058 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 3059 !boot_cpu_has(X86_FEATURE_VMX)) 3060 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 3061 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 3062 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 3063 else if (itlb_multihit_kvm_mitigation) 3064 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 3065 else 3066 return sysfs_emit(buf, "KVM: Vulnerable\n"); 3067 } 3068 #else 3069 static ssize_t l1tf_show_state(char *buf) 3070 { 3071 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 3072 } 3073 3074 static ssize_t itlb_multihit_show_state(char *buf) 3075 { 3076 return sysfs_emit(buf, "Processor vulnerable\n"); 3077 } 3078 #endif 3079 3080 static ssize_t mds_show_state(char *buf) 3081 { 3082 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3083 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3084 mds_strings[mds_mitigation]); 3085 } 3086 3087 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 3088 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3089 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 3090 sched_smt_active() ? "mitigated" : "disabled")); 3091 } 3092 3093 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 3094 sched_smt_active() ? "vulnerable" : "disabled"); 3095 } 3096 3097 static ssize_t tsx_async_abort_show_state(char *buf) 3098 { 3099 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 3100 (taa_mitigation == TAA_MITIGATION_OFF)) 3101 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 3102 3103 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3104 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3105 taa_strings[taa_mitigation]); 3106 } 3107 3108 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 3109 sched_smt_active() ? "vulnerable" : "disabled"); 3110 } 3111 3112 static ssize_t mmio_stale_data_show_state(char *buf) 3113 { 3114 if (mmio_mitigation == MMIO_MITIGATION_OFF) 3115 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 3116 3117 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 3118 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 3119 mmio_strings[mmio_mitigation]); 3120 } 3121 3122 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 3123 sched_smt_active() ? "vulnerable" : "disabled"); 3124 } 3125 3126 static ssize_t rfds_show_state(char *buf) 3127 { 3128 return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); 3129 } 3130 3131 static ssize_t old_microcode_show_state(char *buf) 3132 { 3133 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 3134 return sysfs_emit(buf, "Unknown: running under hypervisor"); 3135 3136 return sysfs_emit(buf, "Vulnerable\n"); 3137 } 3138 3139 static ssize_t its_show_state(char *buf) 3140 { 3141 return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]); 3142 } 3143 3144 static char *stibp_state(void) 3145 { 3146 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && 3147 !boot_cpu_has(X86_FEATURE_AUTOIBRS)) 3148 return ""; 3149 3150 switch (spectre_v2_user_stibp) { 3151 case SPECTRE_V2_USER_NONE: 3152 return "; STIBP: disabled"; 3153 case SPECTRE_V2_USER_STRICT: 3154 return "; STIBP: forced"; 3155 case SPECTRE_V2_USER_STRICT_PREFERRED: 3156 return "; STIBP: always-on"; 3157 case SPECTRE_V2_USER_PRCTL: 3158 case SPECTRE_V2_USER_SECCOMP: 3159 if (static_key_enabled(&switch_to_cond_stibp)) 3160 return "; STIBP: conditional"; 3161 } 3162 return ""; 3163 } 3164 3165 static char *ibpb_state(void) 3166 { 3167 if (boot_cpu_has(X86_FEATURE_IBPB)) { 3168 if (static_key_enabled(&switch_mm_always_ibpb)) 3169 return "; IBPB: always-on"; 3170 if (static_key_enabled(&switch_mm_cond_ibpb)) 3171 return "; IBPB: conditional"; 3172 return "; IBPB: disabled"; 3173 } 3174 return ""; 3175 } 3176 3177 static char *pbrsb_eibrs_state(void) 3178 { 3179 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 3180 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 3181 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 3182 return "; PBRSB-eIBRS: SW sequence"; 3183 else 3184 return "; PBRSB-eIBRS: Vulnerable"; 3185 } else { 3186 return "; PBRSB-eIBRS: Not affected"; 3187 } 3188 } 3189 3190 static const char *spectre_bhi_state(void) 3191 { 3192 if (!boot_cpu_has_bug(X86_BUG_BHI)) 3193 return "; BHI: Not affected"; 3194 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW)) 3195 return "; BHI: BHI_DIS_S"; 3196 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP)) 3197 return "; BHI: SW loop, KVM: SW loop"; 3198 else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && 3199 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) && 3200 rrsba_disabled) 3201 return "; BHI: Retpoline"; 3202 else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) 3203 return "; BHI: Vulnerable, KVM: SW loop"; 3204 3205 return "; BHI: Vulnerable"; 3206 } 3207 3208 static ssize_t spectre_v2_show_state(char *buf) 3209 { 3210 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 3211 return sysfs_emit(buf, "Vulnerable: LFENCE\n"); 3212 3213 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 3214 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 3215 3216 if (sched_smt_active() && unprivileged_ebpf_enabled() && 3217 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 3218 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 3219 3220 return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n", 3221 spectre_v2_strings[spectre_v2_enabled], 3222 ibpb_state(), 3223 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "", 3224 stibp_state(), 3225 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "", 3226 pbrsb_eibrs_state(), 3227 spectre_bhi_state(), 3228 /* this should always be at the end */ 3229 spectre_v2_module_string()); 3230 } 3231 3232 static ssize_t srbds_show_state(char *buf) 3233 { 3234 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 3235 } 3236 3237 static ssize_t retbleed_show_state(char *buf) 3238 { 3239 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 3240 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 3241 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 3242 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 3243 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 3244 3245 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 3246 !sched_smt_active() ? "disabled" : 3247 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 3248 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 3249 "enabled with STIBP protection" : "vulnerable"); 3250 } 3251 3252 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 3253 } 3254 3255 static ssize_t srso_show_state(char *buf) 3256 { 3257 if (boot_cpu_has(X86_FEATURE_SRSO_NO)) 3258 return sysfs_emit(buf, "Mitigation: SMT disabled\n"); 3259 3260 return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]); 3261 } 3262 3263 static ssize_t gds_show_state(char *buf) 3264 { 3265 return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]); 3266 } 3267 3268 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 3269 char *buf, unsigned int bug) 3270 { 3271 if (!boot_cpu_has_bug(bug)) 3272 return sysfs_emit(buf, "Not affected\n"); 3273 3274 switch (bug) { 3275 case X86_BUG_CPU_MELTDOWN: 3276 if (boot_cpu_has(X86_FEATURE_PTI)) 3277 return sysfs_emit(buf, "Mitigation: PTI\n"); 3278 3279 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 3280 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 3281 3282 break; 3283 3284 case X86_BUG_SPECTRE_V1: 3285 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 3286 3287 case X86_BUG_SPECTRE_V2: 3288 return spectre_v2_show_state(buf); 3289 3290 case X86_BUG_SPEC_STORE_BYPASS: 3291 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 3292 3293 case X86_BUG_L1TF: 3294 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 3295 return l1tf_show_state(buf); 3296 break; 3297 3298 case X86_BUG_MDS: 3299 return mds_show_state(buf); 3300 3301 case X86_BUG_TAA: 3302 return tsx_async_abort_show_state(buf); 3303 3304 case X86_BUG_ITLB_MULTIHIT: 3305 return itlb_multihit_show_state(buf); 3306 3307 case X86_BUG_SRBDS: 3308 return srbds_show_state(buf); 3309 3310 case X86_BUG_MMIO_STALE_DATA: 3311 return mmio_stale_data_show_state(buf); 3312 3313 case X86_BUG_RETBLEED: 3314 return retbleed_show_state(buf); 3315 3316 case X86_BUG_SRSO: 3317 return srso_show_state(buf); 3318 3319 case X86_BUG_GDS: 3320 return gds_show_state(buf); 3321 3322 case X86_BUG_RFDS: 3323 return rfds_show_state(buf); 3324 3325 case X86_BUG_OLD_MICROCODE: 3326 return old_microcode_show_state(buf); 3327 3328 case X86_BUG_ITS: 3329 return its_show_state(buf); 3330 3331 default: 3332 break; 3333 } 3334 3335 return sysfs_emit(buf, "Vulnerable\n"); 3336 } 3337 3338 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 3339 { 3340 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 3341 } 3342 3343 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 3344 { 3345 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 3346 } 3347 3348 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 3349 { 3350 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 3351 } 3352 3353 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 3354 { 3355 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 3356 } 3357 3358 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 3359 { 3360 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 3361 } 3362 3363 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 3364 { 3365 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 3366 } 3367 3368 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 3369 { 3370 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 3371 } 3372 3373 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 3374 { 3375 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 3376 } 3377 3378 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 3379 { 3380 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 3381 } 3382 3383 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 3384 { 3385 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 3386 } 3387 3388 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 3389 { 3390 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 3391 } 3392 3393 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) 3394 { 3395 return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); 3396 } 3397 3398 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) 3399 { 3400 return cpu_show_common(dev, attr, buf, X86_BUG_GDS); 3401 } 3402 3403 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf) 3404 { 3405 return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); 3406 } 3407 3408 ssize_t cpu_show_old_microcode(struct device *dev, struct device_attribute *attr, char *buf) 3409 { 3410 return cpu_show_common(dev, attr, buf, X86_BUG_OLD_MICROCODE); 3411 } 3412 3413 ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf) 3414 { 3415 return cpu_show_common(dev, attr, buf, X86_BUG_ITS); 3416 } 3417 #endif 3418 3419 void __warn_thunk(void) 3420 { 3421 WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); 3422 } 3423