xref: /linux/arch/x86/kernel/cpu/bugs.c (revision 3457b3bcd344005af894876cab4e634cabdcbb62)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright (C) 1994  Linus Torvalds
4  *
5  *  Cyrix stuff, June 1998 by:
6  *	- Rafael R. Reilova (moved everything from head.S),
7  *        <rreilova@ececs.uc.edu>
8  *	- Channing Corn (tests & fixes),
9  *	- Andrew D. Balsa (code cleanup).
10  */
11 #include <linux/init.h>
12 #include <linux/cpu.h>
13 #include <linux/module.h>
14 #include <linux/nospec.h>
15 #include <linux/prctl.h>
16 #include <linux/sched/smt.h>
17 #include <linux/pgtable.h>
18 #include <linux/bpf.h>
19 #include <linux/filter.h>
20 #include <linux/kvm_types.h>
21 
22 #include <asm/spec-ctrl.h>
23 #include <asm/cmdline.h>
24 #include <asm/bugs.h>
25 #include <asm/processor.h>
26 #include <asm/processor-flags.h>
27 #include <asm/fpu/api.h>
28 #include <asm/msr.h>
29 #include <asm/vmx.h>
30 #include <asm/cpu_device_id.h>
31 #include <asm/e820/api.h>
32 #include <asm/hypervisor.h>
33 #include <asm/tlbflush.h>
34 #include <asm/cpu.h>
35 
36 #include "cpu.h"
37 
38 /*
39  * Speculation Vulnerability Handling
40  *
41  * Each vulnerability is handled with the following functions:
42  *   <vuln>_select_mitigation() -- Selects a mitigation to use.  This should
43  *				   take into account all relevant command line
44  *				   options.
45  *   <vuln>_update_mitigation() -- This is called after all vulnerabilities have
46  *				   selected a mitigation, in case the selection
47  *				   may want to change based on other choices
48  *				   made.  This function is optional.
49  *   <vuln>_apply_mitigation() -- Enable the selected mitigation.
50  *
51  * The compile-time mitigation in all cases should be AUTO.  An explicit
52  * command-line option can override AUTO.  If no such option is
53  * provided, <vuln>_select_mitigation() will override AUTO to the best
54  * mitigation option.
55  */
56 
57 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
58 u64 x86_spec_ctrl_base;
59 
60 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
61 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
62 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current);
63 
64 /*
65  * Set when the CPU has run a potentially malicious guest. An IBPB will
66  * be needed to before running userspace. That IBPB will flush the branch
67  * predictor content.
68  */
69 DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user);
70 EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user);
71 
72 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
73 
74 static u64 __ro_after_init x86_arch_cap_msr;
75 
76 static DEFINE_MUTEX(spec_ctrl_mutex);
77 
78 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;
79 
80 static void __init set_return_thunk(void *thunk)
81 {
82 	x86_return_thunk = thunk;
83 
84 	pr_info("active return thunk: %ps\n", thunk);
85 }
86 
87 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
88 static void update_spec_ctrl(u64 val)
89 {
90 	this_cpu_write(x86_spec_ctrl_current, val);
91 	wrmsrq(MSR_IA32_SPEC_CTRL, val);
92 }
93 
94 /*
95  * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
96  * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
97  */
98 void update_spec_ctrl_cond(u64 val)
99 {
100 	if (this_cpu_read(x86_spec_ctrl_current) == val)
101 		return;
102 
103 	this_cpu_write(x86_spec_ctrl_current, val);
104 
105 	/*
106 	 * When KERNEL_IBRS this MSR is written on return-to-user, unless
107 	 * forced the update can be delayed until that time.
108 	 */
109 	if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
110 		wrmsrq(MSR_IA32_SPEC_CTRL, val);
111 }
112 
113 noinstr u64 spec_ctrl_current(void)
114 {
115 	return this_cpu_read(x86_spec_ctrl_current);
116 }
117 EXPORT_SYMBOL_GPL(spec_ctrl_current);
118 
119 /*
120  * AMD specific MSR info for Speculative Store Bypass control.
121  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
122  */
123 u64 __ro_after_init x86_amd_ls_cfg_base;
124 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
125 
126 /* Control conditional STIBP in switch_to() */
127 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
128 /* Control conditional IBPB in switch_mm() */
129 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
130 /* Control unconditional IBPB in switch_mm() */
131 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
132 
133 /* Control IBPB on vCPU load */
134 DEFINE_STATIC_KEY_FALSE(switch_vcpu_ibpb);
135 EXPORT_SYMBOL_FOR_KVM(switch_vcpu_ibpb);
136 
137 /* Control CPU buffer clear before idling (halt, mwait) */
138 DEFINE_STATIC_KEY_FALSE(cpu_buf_idle_clear);
139 EXPORT_SYMBOL_GPL(cpu_buf_idle_clear);
140 
141 /*
142  * Controls whether l1d flush based mitigations are enabled,
143  * based on hw features and admin setting via boot parameter
144  * defaults to false
145  */
146 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
147 
148 #undef pr_fmt
149 #define pr_fmt(fmt)	"mitigations: " fmt
150 
151 static void __init cpu_print_attack_vectors(void)
152 {
153 	pr_info("Enabled attack vectors: ");
154 
155 	if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL))
156 		pr_cont("user_kernel, ");
157 
158 	if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER))
159 		pr_cont("user_user, ");
160 
161 	if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST))
162 		pr_cont("guest_host, ");
163 
164 	if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST))
165 		pr_cont("guest_guest, ");
166 
167 	pr_cont("SMT mitigations: ");
168 
169 	switch (smt_mitigations) {
170 	case SMT_MITIGATIONS_OFF:
171 		pr_cont("off\n");
172 		break;
173 	case SMT_MITIGATIONS_AUTO:
174 		pr_cont("auto\n");
175 		break;
176 	case SMT_MITIGATIONS_ON:
177 		pr_cont("on\n");
178 	}
179 }
180 
181 /*
182  * NOTE: This function is *only* called for SVM, since Intel uses
183  * MSR_IA32_SPEC_CTRL for SSBD.
184  */
185 void
186 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest)
187 {
188 	u64 guestval, hostval;
189 	struct thread_info *ti = current_thread_info();
190 
191 	/*
192 	 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
193 	 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
194 	 */
195 	if (!cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD) &&
196 	    !cpu_feature_enabled(X86_FEATURE_VIRT_SSBD))
197 		return;
198 
199 	/*
200 	 * If the host has SSBD mitigation enabled, force it in the host's
201 	 * virtual MSR value. If its not permanently enabled, evaluate
202 	 * current's TIF_SSBD thread flag.
203 	 */
204 	if (cpu_feature_enabled(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
205 		hostval = SPEC_CTRL_SSBD;
206 	else
207 		hostval = ssbd_tif_to_spec_ctrl(ti->flags);
208 
209 	/* Sanitize the guest value */
210 	guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
211 
212 	if (hostval != guestval) {
213 		unsigned long tif;
214 
215 		tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
216 				 ssbd_spec_ctrl_to_tif(hostval);
217 
218 		speculation_ctrl_update(tif);
219 	}
220 }
221 EXPORT_SYMBOL_FOR_KVM(x86_virt_spec_ctrl);
222 
223 static void x86_amd_ssb_disable(void)
224 {
225 	u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
226 
227 	if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
228 		wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
229 	else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
230 		wrmsrq(MSR_AMD64_LS_CFG, msrval);
231 }
232 
233 #undef pr_fmt
234 #define pr_fmt(fmt)	"MDS: " fmt
235 
236 /*
237  * Returns true if vulnerability should be mitigated based on the
238  * selected attack vector controls.
239  *
240  * See Documentation/admin-guide/hw-vuln/attack_vector_controls.rst
241  */
242 static bool __init should_mitigate_vuln(unsigned int bug)
243 {
244 	switch (bug) {
245 	/*
246 	 * The only runtime-selected spectre_v1 mitigations in the kernel are
247 	 * related to SWAPGS protection on kernel entry.  Therefore, protection
248 	 * is only required for the user->kernel attack vector.
249 	 */
250 	case X86_BUG_SPECTRE_V1:
251 		return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL);
252 
253 	case X86_BUG_SPECTRE_V2:
254 	case X86_BUG_RETBLEED:
255 	case X86_BUG_L1TF:
256 	case X86_BUG_ITS:
257 		return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) ||
258 		       cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST);
259 
260 	case X86_BUG_SPECTRE_V2_USER:
261 		return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) ||
262 		       cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST);
263 
264 	/*
265 	 * All the vulnerabilities below allow potentially leaking data
266 	 * across address spaces.  Therefore, mitigation is required for
267 	 * any of these 4 attack vectors.
268 	 */
269 	case X86_BUG_MDS:
270 	case X86_BUG_TAA:
271 	case X86_BUG_MMIO_STALE_DATA:
272 	case X86_BUG_RFDS:
273 	case X86_BUG_SRBDS:
274 		return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) ||
275 		       cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) ||
276 		       cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) ||
277 		       cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST);
278 
279 	case X86_BUG_GDS:
280 		return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) ||
281 		       cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) ||
282 		       cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) ||
283 		       cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST) ||
284 		       (smt_mitigations != SMT_MITIGATIONS_OFF);
285 
286 	case X86_BUG_SPEC_STORE_BYPASS:
287 		return cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER);
288 
289 	case X86_BUG_VMSCAPE:
290 		return cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST);
291 
292 	default:
293 		WARN(1, "Unknown bug %x\n", bug);
294 		return false;
295 	}
296 }
297 
298 /* Default mitigation for MDS-affected CPUs */
299 static enum mds_mitigations mds_mitigation __ro_after_init =
300 	IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_AUTO : MDS_MITIGATION_OFF;
301 static bool mds_nosmt __ro_after_init = false;
302 
303 static const char * const mds_strings[] = {
304 	[MDS_MITIGATION_OFF]	= "Vulnerable",
305 	[MDS_MITIGATION_FULL]	= "Mitigation: Clear CPU buffers",
306 	[MDS_MITIGATION_VMWERV]	= "Vulnerable: Clear CPU buffers attempted, no microcode",
307 };
308 
309 enum taa_mitigations {
310 	TAA_MITIGATION_OFF,
311 	TAA_MITIGATION_AUTO,
312 	TAA_MITIGATION_UCODE_NEEDED,
313 	TAA_MITIGATION_VERW,
314 	TAA_MITIGATION_TSX_DISABLED,
315 };
316 
317 /* Default mitigation for TAA-affected CPUs */
318 static enum taa_mitigations taa_mitigation __ro_after_init =
319 	IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_AUTO : TAA_MITIGATION_OFF;
320 
321 enum mmio_mitigations {
322 	MMIO_MITIGATION_OFF,
323 	MMIO_MITIGATION_AUTO,
324 	MMIO_MITIGATION_UCODE_NEEDED,
325 	MMIO_MITIGATION_VERW,
326 };
327 
328 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
329 static enum mmio_mitigations mmio_mitigation __ro_after_init =
330 	IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ?	MMIO_MITIGATION_AUTO : MMIO_MITIGATION_OFF;
331 
332 enum rfds_mitigations {
333 	RFDS_MITIGATION_OFF,
334 	RFDS_MITIGATION_AUTO,
335 	RFDS_MITIGATION_VERW,
336 	RFDS_MITIGATION_UCODE_NEEDED,
337 };
338 
339 /* Default mitigation for Register File Data Sampling */
340 static enum rfds_mitigations rfds_mitigation __ro_after_init =
341 	IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF;
342 
343 /*
344  * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing on exit to
345  * userspace *and* on entry to KVM guests.
346  */
347 static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init;
348 
349 static void __init mds_select_mitigation(void)
350 {
351 	if (!boot_cpu_has_bug(X86_BUG_MDS)) {
352 		mds_mitigation = MDS_MITIGATION_OFF;
353 		return;
354 	}
355 
356 	if (mds_mitigation == MDS_MITIGATION_AUTO) {
357 		if (should_mitigate_vuln(X86_BUG_MDS))
358 			mds_mitigation = MDS_MITIGATION_FULL;
359 		else
360 			mds_mitigation = MDS_MITIGATION_OFF;
361 	}
362 
363 	if (mds_mitigation == MDS_MITIGATION_OFF)
364 		return;
365 
366 	verw_clear_cpu_buf_mitigation_selected = true;
367 }
368 
369 static void __init mds_update_mitigation(void)
370 {
371 	if (!boot_cpu_has_bug(X86_BUG_MDS))
372 		return;
373 
374 	/* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */
375 	if (verw_clear_cpu_buf_mitigation_selected)
376 		mds_mitigation = MDS_MITIGATION_FULL;
377 
378 	if (mds_mitigation == MDS_MITIGATION_FULL) {
379 		if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
380 			mds_mitigation = MDS_MITIGATION_VMWERV;
381 	}
382 
383 	pr_info("%s\n", mds_strings[mds_mitigation]);
384 }
385 
386 static void __init mds_apply_mitigation(void)
387 {
388 	if (mds_mitigation == MDS_MITIGATION_FULL ||
389 	    mds_mitigation == MDS_MITIGATION_VMWERV) {
390 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
391 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
392 		if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
393 		    (mds_nosmt || smt_mitigations == SMT_MITIGATIONS_ON))
394 			cpu_smt_disable(false);
395 	}
396 }
397 
398 static int __init mds_cmdline(char *str)
399 {
400 	if (!boot_cpu_has_bug(X86_BUG_MDS))
401 		return 0;
402 
403 	if (!str)
404 		return -EINVAL;
405 
406 	if (!strcmp(str, "off"))
407 		mds_mitigation = MDS_MITIGATION_OFF;
408 	else if (!strcmp(str, "full"))
409 		mds_mitigation = MDS_MITIGATION_FULL;
410 	else if (!strcmp(str, "full,nosmt")) {
411 		mds_mitigation = MDS_MITIGATION_FULL;
412 		mds_nosmt = true;
413 	}
414 
415 	return 0;
416 }
417 early_param("mds", mds_cmdline);
418 
419 #undef pr_fmt
420 #define pr_fmt(fmt)	"TAA: " fmt
421 
422 static bool taa_nosmt __ro_after_init;
423 
424 static const char * const taa_strings[] = {
425 	[TAA_MITIGATION_OFF]		= "Vulnerable",
426 	[TAA_MITIGATION_UCODE_NEEDED]	= "Vulnerable: Clear CPU buffers attempted, no microcode",
427 	[TAA_MITIGATION_VERW]		= "Mitigation: Clear CPU buffers",
428 	[TAA_MITIGATION_TSX_DISABLED]	= "Mitigation: TSX disabled",
429 };
430 
431 static bool __init taa_vulnerable(void)
432 {
433 	return boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM);
434 }
435 
436 static void __init taa_select_mitigation(void)
437 {
438 	if (!boot_cpu_has_bug(X86_BUG_TAA)) {
439 		taa_mitigation = TAA_MITIGATION_OFF;
440 		return;
441 	}
442 
443 	/* TSX previously disabled by tsx=off */
444 	if (!boot_cpu_has(X86_FEATURE_RTM)) {
445 		taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
446 		return;
447 	}
448 
449 	/* Microcode will be checked in taa_update_mitigation(). */
450 	if (taa_mitigation == TAA_MITIGATION_AUTO) {
451 		if (should_mitigate_vuln(X86_BUG_TAA))
452 			taa_mitigation = TAA_MITIGATION_VERW;
453 		else
454 			taa_mitigation = TAA_MITIGATION_OFF;
455 	}
456 
457 	if (taa_mitigation != TAA_MITIGATION_OFF)
458 		verw_clear_cpu_buf_mitigation_selected = true;
459 }
460 
461 static void __init taa_update_mitigation(void)
462 {
463 	if (!taa_vulnerable())
464 		return;
465 
466 	if (verw_clear_cpu_buf_mitigation_selected)
467 		taa_mitigation = TAA_MITIGATION_VERW;
468 
469 	if (taa_mitigation == TAA_MITIGATION_VERW) {
470 		/* Check if the requisite ucode is available. */
471 		if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
472 			taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
473 
474 		/*
475 		 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
476 		 * A microcode update fixes this behavior to clear CPU buffers. It also
477 		 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
478 		 * ARCH_CAP_TSX_CTRL_MSR bit.
479 		 *
480 		 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
481 		 * update is required.
482 		 */
483 		if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) &&
484 		   !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR))
485 			taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
486 	}
487 
488 	pr_info("%s\n", taa_strings[taa_mitigation]);
489 }
490 
491 static void __init taa_apply_mitigation(void)
492 {
493 	if (taa_mitigation == TAA_MITIGATION_VERW ||
494 	    taa_mitigation == TAA_MITIGATION_UCODE_NEEDED) {
495 		/*
496 		 * TSX is enabled, select alternate mitigation for TAA which is
497 		 * the same as MDS. Enable MDS static branch to clear CPU buffers.
498 		 *
499 		 * For guests that can't determine whether the correct microcode is
500 		 * present on host, enable the mitigation for UCODE_NEEDED as well.
501 		 */
502 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
503 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
504 
505 		if (taa_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)
506 			cpu_smt_disable(false);
507 	}
508 }
509 
510 static int __init tsx_async_abort_parse_cmdline(char *str)
511 {
512 	if (!boot_cpu_has_bug(X86_BUG_TAA))
513 		return 0;
514 
515 	if (!str)
516 		return -EINVAL;
517 
518 	if (!strcmp(str, "off")) {
519 		taa_mitigation = TAA_MITIGATION_OFF;
520 	} else if (!strcmp(str, "full")) {
521 		taa_mitigation = TAA_MITIGATION_VERW;
522 	} else if (!strcmp(str, "full,nosmt")) {
523 		taa_mitigation = TAA_MITIGATION_VERW;
524 		taa_nosmt = true;
525 	}
526 
527 	return 0;
528 }
529 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
530 
531 #undef pr_fmt
532 #define pr_fmt(fmt)	"MMIO Stale Data: " fmt
533 
534 static bool mmio_nosmt __ro_after_init = false;
535 
536 static const char * const mmio_strings[] = {
537 	[MMIO_MITIGATION_OFF]		= "Vulnerable",
538 	[MMIO_MITIGATION_UCODE_NEEDED]	= "Vulnerable: Clear CPU buffers attempted, no microcode",
539 	[MMIO_MITIGATION_VERW]		= "Mitigation: Clear CPU buffers",
540 };
541 
542 static void __init mmio_select_mitigation(void)
543 {
544 	if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
545 		mmio_mitigation = MMIO_MITIGATION_OFF;
546 		return;
547 	}
548 
549 	/* Microcode will be checked in mmio_update_mitigation(). */
550 	if (mmio_mitigation == MMIO_MITIGATION_AUTO) {
551 		if (should_mitigate_vuln(X86_BUG_MMIO_STALE_DATA))
552 			mmio_mitigation = MMIO_MITIGATION_VERW;
553 		else
554 			mmio_mitigation = MMIO_MITIGATION_OFF;
555 	}
556 
557 	if (mmio_mitigation == MMIO_MITIGATION_OFF)
558 		return;
559 
560 	/*
561 	 * Enable CPU buffer clear mitigation for host and VMM, if also affected
562 	 * by MDS or TAA.
563 	 */
564 	if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable())
565 		verw_clear_cpu_buf_mitigation_selected = true;
566 }
567 
568 static void __init mmio_update_mitigation(void)
569 {
570 	if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
571 		return;
572 
573 	if (verw_clear_cpu_buf_mitigation_selected)
574 		mmio_mitigation = MMIO_MITIGATION_VERW;
575 
576 	if (mmio_mitigation == MMIO_MITIGATION_VERW) {
577 		/*
578 		 * Check if the system has the right microcode.
579 		 *
580 		 * CPU Fill buffer clear mitigation is enumerated by either an explicit
581 		 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
582 		 * affected systems.
583 		 */
584 		if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) ||
585 		      (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
586 		       boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
587 		     !(x86_arch_cap_msr & ARCH_CAP_MDS_NO))))
588 			mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
589 	}
590 
591 	pr_info("%s\n", mmio_strings[mmio_mitigation]);
592 }
593 
594 static void __init mmio_apply_mitigation(void)
595 {
596 	if (mmio_mitigation == MMIO_MITIGATION_OFF)
597 		return;
598 
599 	/*
600 	 * Only enable the VMM mitigation if the CPU buffer clear mitigation is
601 	 * not being used.
602 	 */
603 	if (verw_clear_cpu_buf_mitigation_selected) {
604 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
605 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
606 	} else {
607 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO);
608 	}
609 
610 	/*
611 	 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
612 	 * be propagated to uncore buffers, clearing the Fill buffers on idle
613 	 * is required irrespective of SMT state.
614 	 */
615 	if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO))
616 		static_branch_enable(&cpu_buf_idle_clear);
617 
618 	if (mmio_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)
619 		cpu_smt_disable(false);
620 }
621 
622 static int __init mmio_stale_data_parse_cmdline(char *str)
623 {
624 	if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
625 		return 0;
626 
627 	if (!str)
628 		return -EINVAL;
629 
630 	if (!strcmp(str, "off")) {
631 		mmio_mitigation = MMIO_MITIGATION_OFF;
632 	} else if (!strcmp(str, "full")) {
633 		mmio_mitigation = MMIO_MITIGATION_VERW;
634 	} else if (!strcmp(str, "full,nosmt")) {
635 		mmio_mitigation = MMIO_MITIGATION_VERW;
636 		mmio_nosmt = true;
637 	}
638 
639 	return 0;
640 }
641 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
642 
643 #undef pr_fmt
644 #define pr_fmt(fmt)	"Register File Data Sampling: " fmt
645 
646 static const char * const rfds_strings[] = {
647 	[RFDS_MITIGATION_OFF]			= "Vulnerable",
648 	[RFDS_MITIGATION_VERW]			= "Mitigation: Clear Register File",
649 	[RFDS_MITIGATION_UCODE_NEEDED]		= "Vulnerable: No microcode",
650 };
651 
652 static inline bool __init verw_clears_cpu_reg_file(void)
653 {
654 	return (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR);
655 }
656 
657 static void __init rfds_select_mitigation(void)
658 {
659 	if (!boot_cpu_has_bug(X86_BUG_RFDS)) {
660 		rfds_mitigation = RFDS_MITIGATION_OFF;
661 		return;
662 	}
663 
664 	if (rfds_mitigation == RFDS_MITIGATION_AUTO) {
665 		if (should_mitigate_vuln(X86_BUG_RFDS))
666 			rfds_mitigation = RFDS_MITIGATION_VERW;
667 		else
668 			rfds_mitigation = RFDS_MITIGATION_OFF;
669 	}
670 
671 	if (rfds_mitigation == RFDS_MITIGATION_OFF)
672 		return;
673 
674 	if (verw_clears_cpu_reg_file())
675 		verw_clear_cpu_buf_mitigation_selected = true;
676 }
677 
678 static void __init rfds_update_mitigation(void)
679 {
680 	if (!boot_cpu_has_bug(X86_BUG_RFDS))
681 		return;
682 
683 	if (verw_clear_cpu_buf_mitigation_selected)
684 		rfds_mitigation = RFDS_MITIGATION_VERW;
685 
686 	if (rfds_mitigation == RFDS_MITIGATION_VERW) {
687 		if (!verw_clears_cpu_reg_file())
688 			rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED;
689 	}
690 
691 	pr_info("%s\n", rfds_strings[rfds_mitigation]);
692 }
693 
694 static void __init rfds_apply_mitigation(void)
695 {
696 	if (rfds_mitigation == RFDS_MITIGATION_VERW) {
697 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
698 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
699 	}
700 }
701 
702 static __init int rfds_parse_cmdline(char *str)
703 {
704 	if (!str)
705 		return -EINVAL;
706 
707 	if (!boot_cpu_has_bug(X86_BUG_RFDS))
708 		return 0;
709 
710 	if (!strcmp(str, "off"))
711 		rfds_mitigation = RFDS_MITIGATION_OFF;
712 	else if (!strcmp(str, "on"))
713 		rfds_mitigation = RFDS_MITIGATION_VERW;
714 
715 	return 0;
716 }
717 early_param("reg_file_data_sampling", rfds_parse_cmdline);
718 
719 #undef pr_fmt
720 #define pr_fmt(fmt)	"SRBDS: " fmt
721 
722 enum srbds_mitigations {
723 	SRBDS_MITIGATION_OFF,
724 	SRBDS_MITIGATION_AUTO,
725 	SRBDS_MITIGATION_UCODE_NEEDED,
726 	SRBDS_MITIGATION_FULL,
727 	SRBDS_MITIGATION_TSX_OFF,
728 	SRBDS_MITIGATION_HYPERVISOR,
729 };
730 
731 static enum srbds_mitigations srbds_mitigation __ro_after_init =
732 	IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_AUTO : SRBDS_MITIGATION_OFF;
733 
734 static const char * const srbds_strings[] = {
735 	[SRBDS_MITIGATION_OFF]		= "Vulnerable",
736 	[SRBDS_MITIGATION_UCODE_NEEDED]	= "Vulnerable: No microcode",
737 	[SRBDS_MITIGATION_FULL]		= "Mitigation: Microcode",
738 	[SRBDS_MITIGATION_TSX_OFF]	= "Mitigation: TSX disabled",
739 	[SRBDS_MITIGATION_HYPERVISOR]	= "Unknown: Dependent on hypervisor status",
740 };
741 
742 static bool srbds_off;
743 
744 void update_srbds_msr(void)
745 {
746 	u64 mcu_ctrl;
747 
748 	if (!boot_cpu_has_bug(X86_BUG_SRBDS))
749 		return;
750 
751 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
752 		return;
753 
754 	if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
755 		return;
756 
757 	/*
758 	 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX
759 	 * being disabled and it hasn't received the SRBDS MSR microcode.
760 	 */
761 	if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
762 		return;
763 
764 	rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
765 
766 	switch (srbds_mitigation) {
767 	case SRBDS_MITIGATION_OFF:
768 	case SRBDS_MITIGATION_TSX_OFF:
769 		mcu_ctrl |= RNGDS_MITG_DIS;
770 		break;
771 	case SRBDS_MITIGATION_FULL:
772 		mcu_ctrl &= ~RNGDS_MITG_DIS;
773 		break;
774 	default:
775 		break;
776 	}
777 
778 	wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
779 }
780 
781 static void __init srbds_select_mitigation(void)
782 {
783 	if (!boot_cpu_has_bug(X86_BUG_SRBDS)) {
784 		srbds_mitigation = SRBDS_MITIGATION_OFF;
785 		return;
786 	}
787 
788 	if (srbds_mitigation == SRBDS_MITIGATION_AUTO) {
789 		if (should_mitigate_vuln(X86_BUG_SRBDS))
790 			srbds_mitigation = SRBDS_MITIGATION_FULL;
791 		else {
792 			srbds_mitigation = SRBDS_MITIGATION_OFF;
793 			return;
794 		}
795 	}
796 
797 	/*
798 	 * Check to see if this is one of the MDS_NO systems supporting TSX that
799 	 * are only exposed to SRBDS when TSX is enabled or when CPU is affected
800 	 * by Processor MMIO Stale Data vulnerability.
801 	 */
802 	if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
803 	    !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
804 		srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
805 	else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
806 		srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
807 	else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
808 		srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
809 	else if (srbds_off)
810 		srbds_mitigation = SRBDS_MITIGATION_OFF;
811 
812 	pr_info("%s\n", srbds_strings[srbds_mitigation]);
813 }
814 
815 static void __init srbds_apply_mitigation(void)
816 {
817 	update_srbds_msr();
818 }
819 
820 static int __init srbds_parse_cmdline(char *str)
821 {
822 	if (!str)
823 		return -EINVAL;
824 
825 	if (!boot_cpu_has_bug(X86_BUG_SRBDS))
826 		return 0;
827 
828 	srbds_off = !strcmp(str, "off");
829 	return 0;
830 }
831 early_param("srbds", srbds_parse_cmdline);
832 
833 #undef pr_fmt
834 #define pr_fmt(fmt)     "L1D Flush : " fmt
835 
836 enum l1d_flush_mitigations {
837 	L1D_FLUSH_OFF = 0,
838 	L1D_FLUSH_ON,
839 };
840 
841 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
842 
843 static void __init l1d_flush_select_mitigation(void)
844 {
845 	if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
846 		return;
847 
848 	static_branch_enable(&switch_mm_cond_l1d_flush);
849 	pr_info("Conditional flush on switch_mm() enabled\n");
850 }
851 
852 static int __init l1d_flush_parse_cmdline(char *str)
853 {
854 	if (!strcmp(str, "on"))
855 		l1d_flush_mitigation = L1D_FLUSH_ON;
856 
857 	return 0;
858 }
859 early_param("l1d_flush", l1d_flush_parse_cmdline);
860 
861 #undef pr_fmt
862 #define pr_fmt(fmt)	"GDS: " fmt
863 
864 enum gds_mitigations {
865 	GDS_MITIGATION_OFF,
866 	GDS_MITIGATION_AUTO,
867 	GDS_MITIGATION_UCODE_NEEDED,
868 	GDS_MITIGATION_FORCE,
869 	GDS_MITIGATION_FULL,
870 	GDS_MITIGATION_FULL_LOCKED,
871 	GDS_MITIGATION_HYPERVISOR,
872 };
873 
874 static enum gds_mitigations gds_mitigation __ro_after_init =
875 	IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_AUTO : GDS_MITIGATION_OFF;
876 
877 static const char * const gds_strings[] = {
878 	[GDS_MITIGATION_OFF]		= "Vulnerable",
879 	[GDS_MITIGATION_UCODE_NEEDED]	= "Vulnerable: No microcode",
880 	[GDS_MITIGATION_FORCE]		= "Mitigation: AVX disabled, no microcode",
881 	[GDS_MITIGATION_FULL]		= "Mitigation: Microcode",
882 	[GDS_MITIGATION_FULL_LOCKED]	= "Mitigation: Microcode (locked)",
883 	[GDS_MITIGATION_HYPERVISOR]	= "Unknown: Dependent on hypervisor status",
884 };
885 
886 bool gds_ucode_mitigated(void)
887 {
888 	return (gds_mitigation == GDS_MITIGATION_FULL ||
889 		gds_mitigation == GDS_MITIGATION_FULL_LOCKED);
890 }
891 EXPORT_SYMBOL_FOR_KVM(gds_ucode_mitigated);
892 
893 void update_gds_msr(void)
894 {
895 	u64 mcu_ctrl_after;
896 	u64 mcu_ctrl;
897 
898 	switch (gds_mitigation) {
899 	case GDS_MITIGATION_OFF:
900 		rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
901 		mcu_ctrl |= GDS_MITG_DIS;
902 		break;
903 	case GDS_MITIGATION_FULL_LOCKED:
904 		/*
905 		 * The LOCKED state comes from the boot CPU. APs might not have
906 		 * the same state. Make sure the mitigation is enabled on all
907 		 * CPUs.
908 		 */
909 	case GDS_MITIGATION_FULL:
910 		rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
911 		mcu_ctrl &= ~GDS_MITG_DIS;
912 		break;
913 	case GDS_MITIGATION_FORCE:
914 	case GDS_MITIGATION_UCODE_NEEDED:
915 	case GDS_MITIGATION_HYPERVISOR:
916 	case GDS_MITIGATION_AUTO:
917 		return;
918 	}
919 
920 	wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
921 
922 	/*
923 	 * Check to make sure that the WRMSR value was not ignored. Writes to
924 	 * GDS_MITG_DIS will be ignored if this processor is locked but the boot
925 	 * processor was not.
926 	 */
927 	rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after);
928 	WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after);
929 }
930 
931 static void __init gds_select_mitigation(void)
932 {
933 	u64 mcu_ctrl;
934 
935 	if (!boot_cpu_has_bug(X86_BUG_GDS))
936 		return;
937 
938 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
939 		gds_mitigation = GDS_MITIGATION_HYPERVISOR;
940 		return;
941 	}
942 
943 	/* Will verify below that mitigation _can_ be disabled */
944 	if (gds_mitigation == GDS_MITIGATION_AUTO) {
945 		if (should_mitigate_vuln(X86_BUG_GDS))
946 			gds_mitigation = GDS_MITIGATION_FULL;
947 		else
948 			gds_mitigation = GDS_MITIGATION_OFF;
949 	}
950 
951 	/* No microcode */
952 	if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) {
953 		if (gds_mitigation != GDS_MITIGATION_FORCE)
954 			gds_mitigation = GDS_MITIGATION_UCODE_NEEDED;
955 		return;
956 	}
957 
958 	/* Microcode has mitigation, use it */
959 	if (gds_mitigation == GDS_MITIGATION_FORCE)
960 		gds_mitigation = GDS_MITIGATION_FULL;
961 
962 	rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
963 	if (mcu_ctrl & GDS_MITG_LOCKED) {
964 		if (gds_mitigation == GDS_MITIGATION_OFF)
965 			pr_warn("Mitigation locked. Disable failed.\n");
966 
967 		/*
968 		 * The mitigation is selected from the boot CPU. All other CPUs
969 		 * _should_ have the same state. If the boot CPU isn't locked
970 		 * but others are then update_gds_msr() will WARN() of the state
971 		 * mismatch. If the boot CPU is locked update_gds_msr() will
972 		 * ensure the other CPUs have the mitigation enabled.
973 		 */
974 		gds_mitigation = GDS_MITIGATION_FULL_LOCKED;
975 	}
976 }
977 
978 static void __init gds_apply_mitigation(void)
979 {
980 	if (!boot_cpu_has_bug(X86_BUG_GDS))
981 		return;
982 
983 	/* Microcode is present */
984 	if (x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)
985 		update_gds_msr();
986 	else if (gds_mitigation == GDS_MITIGATION_FORCE) {
987 		/*
988 		 * This only needs to be done on the boot CPU so do it
989 		 * here rather than in update_gds_msr()
990 		 */
991 		setup_clear_cpu_cap(X86_FEATURE_AVX);
992 		pr_warn("Microcode update needed! Disabling AVX as mitigation.\n");
993 	}
994 
995 	pr_info("%s\n", gds_strings[gds_mitigation]);
996 }
997 
998 static int __init gds_parse_cmdline(char *str)
999 {
1000 	if (!str)
1001 		return -EINVAL;
1002 
1003 	if (!boot_cpu_has_bug(X86_BUG_GDS))
1004 		return 0;
1005 
1006 	if (!strcmp(str, "off"))
1007 		gds_mitigation = GDS_MITIGATION_OFF;
1008 	else if (!strcmp(str, "force"))
1009 		gds_mitigation = GDS_MITIGATION_FORCE;
1010 
1011 	return 0;
1012 }
1013 early_param("gather_data_sampling", gds_parse_cmdline);
1014 
1015 #undef pr_fmt
1016 #define pr_fmt(fmt)     "Spectre V1 : " fmt
1017 
1018 enum spectre_v1_mitigation {
1019 	SPECTRE_V1_MITIGATION_NONE,
1020 	SPECTRE_V1_MITIGATION_AUTO,
1021 };
1022 
1023 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
1024 	IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ?
1025 		SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE;
1026 
1027 static const char * const spectre_v1_strings[] = {
1028 	[SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
1029 	[SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
1030 };
1031 
1032 /*
1033  * Does SMAP provide full mitigation against speculative kernel access to
1034  * userspace?
1035  */
1036 static bool smap_works_speculatively(void)
1037 {
1038 	if (!boot_cpu_has(X86_FEATURE_SMAP))
1039 		return false;
1040 
1041 	/*
1042 	 * On CPUs which are vulnerable to Meltdown, SMAP does not
1043 	 * prevent speculative access to user data in the L1 cache.
1044 	 * Consider SMAP to be non-functional as a mitigation on these
1045 	 * CPUs.
1046 	 */
1047 	if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
1048 		return false;
1049 
1050 	return true;
1051 }
1052 
1053 static void __init spectre_v1_select_mitigation(void)
1054 {
1055 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1))
1056 		spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
1057 
1058 	if (!should_mitigate_vuln(X86_BUG_SPECTRE_V1))
1059 		spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
1060 }
1061 
1062 static void __init spectre_v1_apply_mitigation(void)
1063 {
1064 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1))
1065 		return;
1066 
1067 	if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
1068 		/*
1069 		 * With Spectre v1, a user can speculatively control either
1070 		 * path of a conditional swapgs with a user-controlled GS
1071 		 * value.  The mitigation is to add lfences to both code paths.
1072 		 *
1073 		 * If FSGSBASE is enabled, the user can put a kernel address in
1074 		 * GS, in which case SMAP provides no protection.
1075 		 *
1076 		 * If FSGSBASE is disabled, the user can only put a user space
1077 		 * address in GS.  That makes an attack harder, but still
1078 		 * possible if there's no SMAP protection.
1079 		 */
1080 		if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
1081 		    !smap_works_speculatively()) {
1082 			/*
1083 			 * Mitigation can be provided from SWAPGS itself or
1084 			 * PTI as the CR3 write in the Meltdown mitigation
1085 			 * is serializing.
1086 			 *
1087 			 * If neither is there, mitigate with an LFENCE to
1088 			 * stop speculation through swapgs.
1089 			 */
1090 			if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
1091 			    !boot_cpu_has(X86_FEATURE_PTI))
1092 				setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
1093 
1094 			/*
1095 			 * Enable lfences in the kernel entry (non-swapgs)
1096 			 * paths, to prevent user entry from speculatively
1097 			 * skipping swapgs.
1098 			 */
1099 			setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
1100 		}
1101 	}
1102 
1103 	pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1104 }
1105 
1106 static int __init nospectre_v1_cmdline(char *str)
1107 {
1108 	spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
1109 	return 0;
1110 }
1111 early_param("nospectre_v1", nospectre_v1_cmdline);
1112 
1113 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE;
1114 
1115 /* Depends on spectre_v2 mitigation selected already */
1116 static inline bool cdt_possible(enum spectre_v2_mitigation mode)
1117 {
1118 	if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) ||
1119 	    !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE))
1120 		return false;
1121 
1122 	if (mode == SPECTRE_V2_RETPOLINE ||
1123 	    mode == SPECTRE_V2_EIBRS_RETPOLINE)
1124 		return true;
1125 
1126 	return false;
1127 }
1128 
1129 #undef pr_fmt
1130 #define pr_fmt(fmt)     "RETBleed: " fmt
1131 
1132 enum its_mitigation {
1133 	ITS_MITIGATION_OFF,
1134 	ITS_MITIGATION_AUTO,
1135 	ITS_MITIGATION_VMEXIT_ONLY,
1136 	ITS_MITIGATION_ALIGNED_THUNKS,
1137 	ITS_MITIGATION_RETPOLINE_STUFF,
1138 };
1139 
1140 static enum its_mitigation its_mitigation __ro_after_init =
1141 	IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_MITIGATION_AUTO : ITS_MITIGATION_OFF;
1142 
1143 enum retbleed_mitigation {
1144 	RETBLEED_MITIGATION_NONE,
1145 	RETBLEED_MITIGATION_AUTO,
1146 	RETBLEED_MITIGATION_UNRET,
1147 	RETBLEED_MITIGATION_IBPB,
1148 	RETBLEED_MITIGATION_IBRS,
1149 	RETBLEED_MITIGATION_EIBRS,
1150 	RETBLEED_MITIGATION_STUFF,
1151 };
1152 
1153 static const char * const retbleed_strings[] = {
1154 	[RETBLEED_MITIGATION_NONE]	= "Vulnerable",
1155 	[RETBLEED_MITIGATION_UNRET]	= "Mitigation: untrained return thunk",
1156 	[RETBLEED_MITIGATION_IBPB]	= "Mitigation: IBPB",
1157 	[RETBLEED_MITIGATION_IBRS]	= "Mitigation: IBRS",
1158 	[RETBLEED_MITIGATION_EIBRS]	= "Mitigation: Enhanced IBRS",
1159 	[RETBLEED_MITIGATION_STUFF]	= "Mitigation: Stuffing",
1160 };
1161 
1162 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
1163 	IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE;
1164 
1165 static int __ro_after_init retbleed_nosmt = false;
1166 
1167 enum srso_mitigation {
1168 	SRSO_MITIGATION_NONE,
1169 	SRSO_MITIGATION_AUTO,
1170 	SRSO_MITIGATION_UCODE_NEEDED,
1171 	SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED,
1172 	SRSO_MITIGATION_MICROCODE,
1173 	SRSO_MITIGATION_NOSMT,
1174 	SRSO_MITIGATION_SAFE_RET,
1175 	SRSO_MITIGATION_IBPB,
1176 	SRSO_MITIGATION_IBPB_ON_VMEXIT,
1177 	SRSO_MITIGATION_BP_SPEC_REDUCE,
1178 	SRSO_MITIGATION_USER_IBPB,
1179 };
1180 
1181 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO;
1182 
1183 static int __init retbleed_parse_cmdline(char *str)
1184 {
1185 	if (!str)
1186 		return -EINVAL;
1187 
1188 	while (str) {
1189 		char *next = strchr(str, ',');
1190 		if (next) {
1191 			*next = 0;
1192 			next++;
1193 		}
1194 
1195 		if (!strcmp(str, "off")) {
1196 			retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1197 		} else if (!strcmp(str, "auto")) {
1198 			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
1199 		} else if (!strcmp(str, "unret")) {
1200 			retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1201 		} else if (!strcmp(str, "ibpb")) {
1202 			retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1203 		} else if (!strcmp(str, "stuff")) {
1204 			retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
1205 		} else if (!strcmp(str, "nosmt")) {
1206 			retbleed_nosmt = true;
1207 		} else if (!strcmp(str, "force")) {
1208 			setup_force_cpu_bug(X86_BUG_RETBLEED);
1209 		} else {
1210 			pr_err("Ignoring unknown retbleed option (%s).", str);
1211 		}
1212 
1213 		str = next;
1214 	}
1215 
1216 	return 0;
1217 }
1218 early_param("retbleed", retbleed_parse_cmdline);
1219 
1220 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
1221 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
1222 
1223 static void __init retbleed_select_mitigation(void)
1224 {
1225 	if (!boot_cpu_has_bug(X86_BUG_RETBLEED)) {
1226 		retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1227 		return;
1228 	}
1229 
1230 	switch (retbleed_mitigation) {
1231 	case RETBLEED_MITIGATION_UNRET:
1232 		if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {
1233 			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
1234 			pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n");
1235 		}
1236 		break;
1237 	case RETBLEED_MITIGATION_IBPB:
1238 		if (!boot_cpu_has(X86_FEATURE_IBPB)) {
1239 			pr_err("WARNING: CPU does not support IBPB.\n");
1240 			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
1241 		} else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
1242 			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
1243 			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
1244 		}
1245 		break;
1246 	case RETBLEED_MITIGATION_STUFF:
1247 		if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) {
1248 			pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n");
1249 			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
1250 		} else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
1251 			pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n");
1252 			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
1253 		}
1254 		break;
1255 	default:
1256 		break;
1257 	}
1258 
1259 	if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO)
1260 		return;
1261 
1262 	if (!should_mitigate_vuln(X86_BUG_RETBLEED)) {
1263 		retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1264 		return;
1265 	}
1266 
1267 	/* Intel mitigation selected in retbleed_update_mitigation() */
1268 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1269 	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
1270 		if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY))
1271 			retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1272 		else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) &&
1273 			 boot_cpu_has(X86_FEATURE_IBPB))
1274 			retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1275 		else
1276 			retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1277 	} else if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1278 		/* Final mitigation depends on spectre-v2 selection */
1279 		if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
1280 			retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
1281 		else if (boot_cpu_has(X86_FEATURE_IBRS))
1282 			retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
1283 		else
1284 			retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1285 	}
1286 }
1287 
1288 static void __init retbleed_update_mitigation(void)
1289 {
1290 	if (!boot_cpu_has_bug(X86_BUG_RETBLEED))
1291 		return;
1292 
1293 	 /* ITS can also enable stuffing */
1294 	if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF)
1295 		retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
1296 
1297 	/* If SRSO is using IBPB, that works for retbleed too */
1298 	if (srso_mitigation == SRSO_MITIGATION_IBPB)
1299 		retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1300 
1301 	if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF &&
1302 	    !cdt_possible(spectre_v2_enabled)) {
1303 		pr_err("WARNING: retbleed=stuff depends on retpoline\n");
1304 		retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1305 	}
1306 
1307 	/*
1308 	 * Let IBRS trump all on Intel without affecting the effects of the
1309 	 * retbleed= cmdline option except for call depth based stuffing
1310 	 */
1311 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1312 		switch (spectre_v2_enabled) {
1313 		case SPECTRE_V2_IBRS:
1314 			retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
1315 			break;
1316 		case SPECTRE_V2_EIBRS:
1317 		case SPECTRE_V2_EIBRS_RETPOLINE:
1318 		case SPECTRE_V2_EIBRS_LFENCE:
1319 			retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
1320 			break;
1321 		default:
1322 			if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) {
1323 				if (retbleed_mitigation != RETBLEED_MITIGATION_NONE)
1324 					pr_err(RETBLEED_INTEL_MSG);
1325 
1326 				retbleed_mitigation = RETBLEED_MITIGATION_NONE;
1327 			}
1328 		}
1329 	}
1330 
1331 	pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
1332 }
1333 
1334 static void __init retbleed_apply_mitigation(void)
1335 {
1336 	bool mitigate_smt = false;
1337 
1338 	switch (retbleed_mitigation) {
1339 	case RETBLEED_MITIGATION_NONE:
1340 		return;
1341 
1342 	case RETBLEED_MITIGATION_UNRET:
1343 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1344 		setup_force_cpu_cap(X86_FEATURE_UNRET);
1345 
1346 		set_return_thunk(retbleed_return_thunk);
1347 
1348 		if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1349 		    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1350 			pr_err(RETBLEED_UNTRAIN_MSG);
1351 
1352 		mitigate_smt = true;
1353 		break;
1354 
1355 	case RETBLEED_MITIGATION_IBPB:
1356 		setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
1357 		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
1358 		mitigate_smt = true;
1359 
1360 		/*
1361 		 * IBPB on entry already obviates the need for
1362 		 * software-based untraining so clear those in case some
1363 		 * other mitigation like SRSO has selected them.
1364 		 */
1365 		setup_clear_cpu_cap(X86_FEATURE_UNRET);
1366 		setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
1367 
1368 		/*
1369 		 * There is no need for RSB filling: write_ibpb() ensures
1370 		 * all predictions, including the RSB, are invalidated,
1371 		 * regardless of IBPB implementation.
1372 		 */
1373 		setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
1374 
1375 		break;
1376 
1377 	case RETBLEED_MITIGATION_STUFF:
1378 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1379 		setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
1380 
1381 		set_return_thunk(call_depth_return_thunk);
1382 		break;
1383 
1384 	default:
1385 		break;
1386 	}
1387 
1388 	if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
1389 	    (retbleed_nosmt || smt_mitigations == SMT_MITIGATIONS_ON))
1390 		cpu_smt_disable(false);
1391 }
1392 
1393 #undef pr_fmt
1394 #define pr_fmt(fmt)     "ITS: " fmt
1395 
1396 static const char * const its_strings[] = {
1397 	[ITS_MITIGATION_OFF]			= "Vulnerable",
1398 	[ITS_MITIGATION_VMEXIT_ONLY]		= "Mitigation: Vulnerable, KVM: Not affected",
1399 	[ITS_MITIGATION_ALIGNED_THUNKS]		= "Mitigation: Aligned branch/return thunks",
1400 	[ITS_MITIGATION_RETPOLINE_STUFF]	= "Mitigation: Retpolines, Stuffing RSB",
1401 };
1402 
1403 static int __init its_parse_cmdline(char *str)
1404 {
1405 	if (!str)
1406 		return -EINVAL;
1407 
1408 	if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) {
1409 		pr_err("Mitigation disabled at compile time, ignoring option (%s)", str);
1410 		return 0;
1411 	}
1412 
1413 	if (!strcmp(str, "off")) {
1414 		its_mitigation = ITS_MITIGATION_OFF;
1415 	} else if (!strcmp(str, "on")) {
1416 		its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
1417 	} else if (!strcmp(str, "force")) {
1418 		its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
1419 		setup_force_cpu_bug(X86_BUG_ITS);
1420 	} else if (!strcmp(str, "vmexit")) {
1421 		its_mitigation = ITS_MITIGATION_VMEXIT_ONLY;
1422 	} else if (!strcmp(str, "stuff")) {
1423 		its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF;
1424 	} else {
1425 		pr_err("Ignoring unknown indirect_target_selection option (%s).", str);
1426 	}
1427 
1428 	return 0;
1429 }
1430 early_param("indirect_target_selection", its_parse_cmdline);
1431 
1432 static void __init its_select_mitigation(void)
1433 {
1434 	if (!boot_cpu_has_bug(X86_BUG_ITS)) {
1435 		its_mitigation = ITS_MITIGATION_OFF;
1436 		return;
1437 	}
1438 
1439 	if (its_mitigation == ITS_MITIGATION_AUTO) {
1440 		if (should_mitigate_vuln(X86_BUG_ITS))
1441 			its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
1442 		else
1443 			its_mitigation = ITS_MITIGATION_OFF;
1444 	}
1445 
1446 	if (its_mitigation == ITS_MITIGATION_OFF)
1447 		return;
1448 
1449 	if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) ||
1450 	    !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) {
1451 		pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n");
1452 		its_mitigation = ITS_MITIGATION_OFF;
1453 		return;
1454 	}
1455 
1456 	if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) {
1457 		pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n");
1458 		its_mitigation = ITS_MITIGATION_OFF;
1459 		return;
1460 	}
1461 
1462 	if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF &&
1463 	    !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) {
1464 		pr_err("RSB stuff mitigation not supported, using default\n");
1465 		its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
1466 	}
1467 
1468 	if (its_mitigation == ITS_MITIGATION_VMEXIT_ONLY &&
1469 	    !boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY))
1470 		its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
1471 }
1472 
1473 static void __init its_update_mitigation(void)
1474 {
1475 	if (!boot_cpu_has_bug(X86_BUG_ITS))
1476 		return;
1477 
1478 	switch (spectre_v2_enabled) {
1479 	case SPECTRE_V2_NONE:
1480 		if (its_mitigation != ITS_MITIGATION_OFF)
1481 			pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n");
1482 		its_mitigation = ITS_MITIGATION_OFF;
1483 		break;
1484 	case SPECTRE_V2_RETPOLINE:
1485 	case SPECTRE_V2_EIBRS_RETPOLINE:
1486 		/* Retpoline+CDT mitigates ITS */
1487 		if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF)
1488 			its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF;
1489 		break;
1490 	case SPECTRE_V2_LFENCE:
1491 	case SPECTRE_V2_EIBRS_LFENCE:
1492 		pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n");
1493 		its_mitigation = ITS_MITIGATION_OFF;
1494 		break;
1495 	default:
1496 		break;
1497 	}
1498 
1499 	if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF &&
1500 	    !cdt_possible(spectre_v2_enabled))
1501 		its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
1502 
1503 	pr_info("%s\n", its_strings[its_mitigation]);
1504 }
1505 
1506 static void __init its_apply_mitigation(void)
1507 {
1508 	switch (its_mitigation) {
1509 	case ITS_MITIGATION_OFF:
1510 	case ITS_MITIGATION_AUTO:
1511 	case ITS_MITIGATION_VMEXIT_ONLY:
1512 		break;
1513 	case ITS_MITIGATION_ALIGNED_THUNKS:
1514 		if (!boot_cpu_has(X86_FEATURE_RETPOLINE))
1515 			setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS);
1516 
1517 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1518 		set_return_thunk(its_return_thunk);
1519 		break;
1520 	case ITS_MITIGATION_RETPOLINE_STUFF:
1521 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1522 		setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
1523 		set_return_thunk(call_depth_return_thunk);
1524 		break;
1525 	}
1526 }
1527 
1528 #undef pr_fmt
1529 #define pr_fmt(fmt)	"Transient Scheduler Attacks: " fmt
1530 
1531 enum tsa_mitigations {
1532 	TSA_MITIGATION_NONE,
1533 	TSA_MITIGATION_AUTO,
1534 	TSA_MITIGATION_UCODE_NEEDED,
1535 	TSA_MITIGATION_USER_KERNEL,
1536 	TSA_MITIGATION_VM,
1537 	TSA_MITIGATION_FULL,
1538 };
1539 
1540 static const char * const tsa_strings[] = {
1541 	[TSA_MITIGATION_NONE]		= "Vulnerable",
1542 	[TSA_MITIGATION_UCODE_NEEDED]	= "Vulnerable: No microcode",
1543 	[TSA_MITIGATION_USER_KERNEL]	= "Mitigation: Clear CPU buffers: user/kernel boundary",
1544 	[TSA_MITIGATION_VM]		= "Mitigation: Clear CPU buffers: VM",
1545 	[TSA_MITIGATION_FULL]		= "Mitigation: Clear CPU buffers",
1546 };
1547 
1548 static enum tsa_mitigations tsa_mitigation __ro_after_init =
1549 	IS_ENABLED(CONFIG_MITIGATION_TSA) ? TSA_MITIGATION_AUTO : TSA_MITIGATION_NONE;
1550 
1551 static int __init tsa_parse_cmdline(char *str)
1552 {
1553 	if (!str)
1554 		return -EINVAL;
1555 
1556 	if (!strcmp(str, "off"))
1557 		tsa_mitigation = TSA_MITIGATION_NONE;
1558 	else if (!strcmp(str, "on"))
1559 		tsa_mitigation = TSA_MITIGATION_FULL;
1560 	else if (!strcmp(str, "user"))
1561 		tsa_mitigation = TSA_MITIGATION_USER_KERNEL;
1562 	else if (!strcmp(str, "vm"))
1563 		tsa_mitigation = TSA_MITIGATION_VM;
1564 	else
1565 		pr_err("Ignoring unknown tsa=%s option.\n", str);
1566 
1567 	return 0;
1568 }
1569 early_param("tsa", tsa_parse_cmdline);
1570 
1571 static void __init tsa_select_mitigation(void)
1572 {
1573 	if (!boot_cpu_has_bug(X86_BUG_TSA)) {
1574 		tsa_mitigation = TSA_MITIGATION_NONE;
1575 		return;
1576 	}
1577 
1578 	if (tsa_mitigation == TSA_MITIGATION_AUTO) {
1579 		bool vm = false, uk = false;
1580 
1581 		tsa_mitigation = TSA_MITIGATION_NONE;
1582 
1583 		if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) ||
1584 		    cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER)) {
1585 			tsa_mitigation = TSA_MITIGATION_USER_KERNEL;
1586 			uk = true;
1587 		}
1588 
1589 		if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) ||
1590 		    cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) {
1591 			tsa_mitigation = TSA_MITIGATION_VM;
1592 			vm = true;
1593 		}
1594 
1595 		if (uk && vm)
1596 			tsa_mitigation = TSA_MITIGATION_FULL;
1597 	}
1598 
1599 	if (tsa_mitigation == TSA_MITIGATION_NONE)
1600 		return;
1601 
1602 	if (!boot_cpu_has(X86_FEATURE_VERW_CLEAR))
1603 		tsa_mitigation = TSA_MITIGATION_UCODE_NEEDED;
1604 
1605 	/*
1606 	 * No need to set verw_clear_cpu_buf_mitigation_selected - it
1607 	 * doesn't fit all cases here and it is not needed because this
1608 	 * is the only VERW-based mitigation on AMD.
1609 	 */
1610 	pr_info("%s\n", tsa_strings[tsa_mitigation]);
1611 }
1612 
1613 static void __init tsa_apply_mitigation(void)
1614 {
1615 	switch (tsa_mitigation) {
1616 	case TSA_MITIGATION_USER_KERNEL:
1617 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
1618 		break;
1619 	case TSA_MITIGATION_VM:
1620 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
1621 		break;
1622 	case TSA_MITIGATION_FULL:
1623 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
1624 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
1625 		break;
1626 	default:
1627 		break;
1628 	}
1629 }
1630 
1631 #undef pr_fmt
1632 #define pr_fmt(fmt)     "Spectre V2 : " fmt
1633 
1634 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
1635 	SPECTRE_V2_USER_NONE;
1636 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
1637 	SPECTRE_V2_USER_NONE;
1638 
1639 #ifdef CONFIG_MITIGATION_RETPOLINE
1640 static bool spectre_v2_bad_module;
1641 
1642 bool retpoline_module_ok(bool has_retpoline)
1643 {
1644 	if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
1645 		return true;
1646 
1647 	pr_err("System may be vulnerable to spectre v2\n");
1648 	spectre_v2_bad_module = true;
1649 	return false;
1650 }
1651 
1652 static inline const char *spectre_v2_module_string(void)
1653 {
1654 	return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
1655 }
1656 
1657 /*
1658  * The "retpoline sequence" is the "call;mov;ret" sequence that
1659  * replaces normal indirect branch instructions. Differentiate
1660  * *the* retpoline sequence from the LFENCE-prefixed indirect
1661  * branches that simply use the retpoline infrastructure.
1662  */
1663 static inline bool retpoline_seq_enabled(void)
1664 {
1665 	return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE);
1666 }
1667 
1668 #else
1669 static inline const char *spectre_v2_module_string(void) { return ""; }
1670 static inline bool retpoline_seq_enabled(void) { return false; }
1671 #endif
1672 
1673 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
1674 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
1675 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
1676 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n"
1677 
1678 #ifdef CONFIG_BPF_SYSCALL
1679 void unpriv_ebpf_notify(int new_state)
1680 {
1681 	if (new_state)
1682 		return;
1683 
1684 	/* Unprivileged eBPF is enabled */
1685 
1686 	switch (spectre_v2_enabled) {
1687 	case SPECTRE_V2_EIBRS:
1688 		pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1689 		break;
1690 	case SPECTRE_V2_EIBRS_LFENCE:
1691 		if (sched_smt_active())
1692 			pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1693 		break;
1694 	default:
1695 		break;
1696 	}
1697 }
1698 #endif
1699 
1700 /* The kernel command line selection for spectre v2 */
1701 enum spectre_v2_mitigation_cmd {
1702 	SPECTRE_V2_CMD_NONE,
1703 	SPECTRE_V2_CMD_AUTO,
1704 	SPECTRE_V2_CMD_FORCE,
1705 	SPECTRE_V2_CMD_RETPOLINE,
1706 	SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1707 	SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1708 	SPECTRE_V2_CMD_EIBRS,
1709 	SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1710 	SPECTRE_V2_CMD_EIBRS_LFENCE,
1711 	SPECTRE_V2_CMD_IBRS,
1712 };
1713 
1714 static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init =
1715 	IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE;
1716 
1717 enum spectre_v2_user_mitigation_cmd {
1718 	SPECTRE_V2_USER_CMD_NONE,
1719 	SPECTRE_V2_USER_CMD_AUTO,
1720 	SPECTRE_V2_USER_CMD_FORCE,
1721 	SPECTRE_V2_USER_CMD_PRCTL,
1722 	SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1723 	SPECTRE_V2_USER_CMD_SECCOMP,
1724 	SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1725 };
1726 
1727 static enum spectre_v2_user_mitigation_cmd spectre_v2_user_cmd __ro_after_init =
1728 	IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE;
1729 
1730 static const char * const spectre_v2_user_strings[] = {
1731 	[SPECTRE_V2_USER_NONE]			= "User space: Vulnerable",
1732 	[SPECTRE_V2_USER_STRICT]		= "User space: Mitigation: STIBP protection",
1733 	[SPECTRE_V2_USER_STRICT_PREFERRED]	= "User space: Mitigation: STIBP always-on protection",
1734 	[SPECTRE_V2_USER_PRCTL]			= "User space: Mitigation: STIBP via prctl",
1735 	[SPECTRE_V2_USER_SECCOMP]		= "User space: Mitigation: STIBP via seccomp and prctl",
1736 };
1737 
1738 static int __init spectre_v2_user_parse_cmdline(char *str)
1739 {
1740 	if (!str)
1741 		return -EINVAL;
1742 
1743 	if (!strcmp(str, "auto"))
1744 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_AUTO;
1745 	else if (!strcmp(str, "off"))
1746 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_NONE;
1747 	else if (!strcmp(str, "on"))
1748 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_FORCE;
1749 	else if (!strcmp(str, "prctl"))
1750 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL;
1751 	else if (!strcmp(str, "prctl,ibpb"))
1752 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_PRCTL_IBPB;
1753 	else if (!strcmp(str, "seccomp"))
1754 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP;
1755 	else if (!strcmp(str, "seccomp,ibpb"))
1756 		spectre_v2_user_cmd = SPECTRE_V2_USER_CMD_SECCOMP_IBPB;
1757 	else
1758 		pr_err("Ignoring unknown spectre_v2_user option (%s).", str);
1759 
1760 	return 0;
1761 }
1762 early_param("spectre_v2_user", spectre_v2_user_parse_cmdline);
1763 
1764 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
1765 {
1766 	return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS;
1767 }
1768 
1769 static void __init spectre_v2_user_select_mitigation(void)
1770 {
1771 	if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1772 		return;
1773 
1774 	switch (spectre_v2_user_cmd) {
1775 	case SPECTRE_V2_USER_CMD_NONE:
1776 		return;
1777 	case SPECTRE_V2_USER_CMD_FORCE:
1778 		spectre_v2_user_ibpb  = SPECTRE_V2_USER_STRICT;
1779 		spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT;
1780 		break;
1781 	case SPECTRE_V2_USER_CMD_AUTO:
1782 		if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2_USER))
1783 			break;
1784 		spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL;
1785 		if (smt_mitigations == SMT_MITIGATIONS_OFF)
1786 			break;
1787 		spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
1788 		break;
1789 	case SPECTRE_V2_USER_CMD_PRCTL:
1790 		spectre_v2_user_ibpb  = SPECTRE_V2_USER_PRCTL;
1791 		spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
1792 		break;
1793 	case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1794 		spectre_v2_user_ibpb  = SPECTRE_V2_USER_STRICT;
1795 		spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
1796 		break;
1797 	case SPECTRE_V2_USER_CMD_SECCOMP:
1798 		if (IS_ENABLED(CONFIG_SECCOMP))
1799 			spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP;
1800 		else
1801 			spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL;
1802 		spectre_v2_user_stibp = spectre_v2_user_ibpb;
1803 		break;
1804 	case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1805 		spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1806 		if (IS_ENABLED(CONFIG_SECCOMP))
1807 			spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP;
1808 		else
1809 			spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
1810 		break;
1811 	}
1812 
1813 	/*
1814 	 * At this point, an STIBP mode other than "off" has been set.
1815 	 * If STIBP support is not being forced, check if STIBP always-on
1816 	 * is preferred.
1817 	 */
1818 	if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
1819 	     spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) &&
1820 	    boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
1821 		spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED;
1822 
1823 	if (!boot_cpu_has(X86_FEATURE_IBPB))
1824 		spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE;
1825 
1826 	if (!boot_cpu_has(X86_FEATURE_STIBP))
1827 		spectre_v2_user_stibp = SPECTRE_V2_USER_NONE;
1828 }
1829 
1830 static void __init spectre_v2_user_update_mitigation(void)
1831 {
1832 	if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1833 		return;
1834 
1835 	/* The spectre_v2 cmd line can override spectre_v2_user options */
1836 	if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) {
1837 		spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE;
1838 		spectre_v2_user_stibp = SPECTRE_V2_USER_NONE;
1839 	} else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) {
1840 		spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1841 		spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT;
1842 	}
1843 
1844 	/*
1845 	 * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP
1846 	 * is not required.
1847 	 *
1848 	 * Intel's Enhanced IBRS also protects against cross-thread branch target
1849 	 * injection in user-mode as the IBRS bit remains always set which
1850 	 * implicitly enables cross-thread protections.  However, in legacy IBRS
1851 	 * mode, the IBRS bit is set only on kernel entry and cleared on return
1852 	 * to userspace.  AMD Automatic IBRS also does not protect userspace.
1853 	 * These modes therefore disable the implicit cross-thread protection,
1854 	 * so allow for STIBP to be selected in those cases.
1855 	 */
1856 	if (!boot_cpu_has(X86_FEATURE_STIBP) ||
1857 	    !cpu_smt_possible() ||
1858 	    (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
1859 	     !boot_cpu_has(X86_FEATURE_AUTOIBRS))) {
1860 		spectre_v2_user_stibp = SPECTRE_V2_USER_NONE;
1861 		return;
1862 	}
1863 
1864 	if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE &&
1865 	    (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
1866 	     retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) {
1867 		if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT &&
1868 		    spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED)
1869 			pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
1870 		spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED;
1871 	}
1872 	pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]);
1873 }
1874 
1875 static void __init spectre_v2_user_apply_mitigation(void)
1876 {
1877 	/* Initialize Indirect Branch Prediction Barrier */
1878 	if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) {
1879 		static_branch_enable(&switch_vcpu_ibpb);
1880 
1881 		switch (spectre_v2_user_ibpb) {
1882 		case SPECTRE_V2_USER_STRICT:
1883 			static_branch_enable(&switch_mm_always_ibpb);
1884 			break;
1885 		case SPECTRE_V2_USER_PRCTL:
1886 		case SPECTRE_V2_USER_SECCOMP:
1887 			static_branch_enable(&switch_mm_cond_ibpb);
1888 			break;
1889 		default:
1890 			break;
1891 		}
1892 
1893 		pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1894 			static_key_enabled(&switch_mm_always_ibpb) ?
1895 			"always-on" : "conditional");
1896 	}
1897 }
1898 
1899 static const char * const spectre_v2_strings[] = {
1900 	[SPECTRE_V2_NONE]			= "Vulnerable",
1901 	[SPECTRE_V2_RETPOLINE]			= "Mitigation: Retpolines",
1902 	[SPECTRE_V2_LFENCE]			= "Vulnerable: LFENCE",
1903 	[SPECTRE_V2_EIBRS]			= "Mitigation: Enhanced / Automatic IBRS",
1904 	[SPECTRE_V2_EIBRS_LFENCE]		= "Mitigation: Enhanced / Automatic IBRS + LFENCE",
1905 	[SPECTRE_V2_EIBRS_RETPOLINE]		= "Mitigation: Enhanced / Automatic IBRS + Retpolines",
1906 	[SPECTRE_V2_IBRS]			= "Mitigation: IBRS",
1907 };
1908 
1909 static bool nospectre_v2 __ro_after_init;
1910 
1911 static int __init nospectre_v2_parse_cmdline(char *str)
1912 {
1913 	nospectre_v2 = true;
1914 	spectre_v2_cmd = SPECTRE_V2_CMD_NONE;
1915 	return 0;
1916 }
1917 early_param("nospectre_v2", nospectre_v2_parse_cmdline);
1918 
1919 static int __init spectre_v2_parse_cmdline(char *str)
1920 {
1921 	if (!str)
1922 		return -EINVAL;
1923 
1924 	if (nospectre_v2)
1925 		return 0;
1926 
1927 	if (!strcmp(str, "off")) {
1928 		spectre_v2_cmd = SPECTRE_V2_CMD_NONE;
1929 	} else if (!strcmp(str, "on")) {
1930 		spectre_v2_cmd = SPECTRE_V2_CMD_FORCE;
1931 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1932 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER);
1933 	} else if (!strcmp(str, "retpoline")) {
1934 		spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE;
1935 	} else if (!strcmp(str, "retpoline,amd") ||
1936 		 !strcmp(str, "retpoline,lfence")) {
1937 		spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_LFENCE;
1938 	} else if (!strcmp(str, "retpoline,generic")) {
1939 		spectre_v2_cmd = SPECTRE_V2_CMD_RETPOLINE_GENERIC;
1940 	} else if (!strcmp(str, "eibrs")) {
1941 		spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS;
1942 	} else if (!strcmp(str, "eibrs,lfence")) {
1943 		spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_LFENCE;
1944 	} else if (!strcmp(str, "eibrs,retpoline")) {
1945 		spectre_v2_cmd = SPECTRE_V2_CMD_EIBRS_RETPOLINE;
1946 	} else if (!strcmp(str, "auto")) {
1947 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
1948 	} else if (!strcmp(str, "ibrs")) {
1949 		spectre_v2_cmd = SPECTRE_V2_CMD_IBRS;
1950 	} else {
1951 		pr_err("Ignoring unknown spectre_v2 option (%s).", str);
1952 	}
1953 
1954 	return 0;
1955 }
1956 early_param("spectre_v2", spectre_v2_parse_cmdline);
1957 
1958 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
1959 {
1960 	if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) {
1961 		pr_err("Kernel not compiled with retpoline");
1962 		return SPECTRE_V2_NONE;
1963 	}
1964 
1965 	return SPECTRE_V2_RETPOLINE;
1966 }
1967 
1968 static bool __ro_after_init rrsba_disabled;
1969 
1970 /* Disable in-kernel use of non-RSB RET predictors */
1971 static void __init spec_ctrl_disable_kernel_rrsba(void)
1972 {
1973 	if (rrsba_disabled)
1974 		return;
1975 
1976 	if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) {
1977 		rrsba_disabled = true;
1978 		return;
1979 	}
1980 
1981 	if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL))
1982 		return;
1983 
1984 	x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S;
1985 	update_spec_ctrl(x86_spec_ctrl_base);
1986 	rrsba_disabled = true;
1987 }
1988 
1989 static void __init spectre_v2_select_rsb_mitigation(enum spectre_v2_mitigation mode)
1990 {
1991 	/*
1992 	 * WARNING! There are many subtleties to consider when changing *any*
1993 	 * code related to RSB-related mitigations.  Before doing so, carefully
1994 	 * read the following document, and update if necessary:
1995 	 *
1996 	 *   Documentation/admin-guide/hw-vuln/rsb.rst
1997 	 *
1998 	 * In an overly simplified nutshell:
1999 	 *
2000 	 *   - User->user RSB attacks are conditionally mitigated during
2001 	 *     context switches by cond_mitigation -> write_ibpb().
2002 	 *
2003 	 *   - User->kernel and guest->host attacks are mitigated by eIBRS or
2004 	 *     RSB filling.
2005 	 *
2006 	 *     Though, depending on config, note that other alternative
2007 	 *     mitigations may end up getting used instead, e.g., IBPB on
2008 	 *     entry/vmexit, call depth tracking, or return thunks.
2009 	 */
2010 
2011 	switch (mode) {
2012 	case SPECTRE_V2_NONE:
2013 		break;
2014 
2015 	case SPECTRE_V2_EIBRS:
2016 	case SPECTRE_V2_EIBRS_LFENCE:
2017 	case SPECTRE_V2_EIBRS_RETPOLINE:
2018 		if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
2019 			pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");
2020 			setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);
2021 		}
2022 		break;
2023 
2024 	case SPECTRE_V2_RETPOLINE:
2025 	case SPECTRE_V2_LFENCE:
2026 	case SPECTRE_V2_IBRS:
2027 		pr_info("Spectre v2 / SpectreRSB: Filling RSB on context switch and VMEXIT\n");
2028 		setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
2029 		setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);
2030 		break;
2031 
2032 	default:
2033 		pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation\n");
2034 		dump_stack();
2035 		break;
2036 	}
2037 }
2038 
2039 /*
2040  * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by
2041  * branch history in userspace. Not needed if BHI_NO is set.
2042  */
2043 static bool __init spec_ctrl_bhi_dis(void)
2044 {
2045 	if (!boot_cpu_has(X86_FEATURE_BHI_CTRL))
2046 		return false;
2047 
2048 	x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S;
2049 	update_spec_ctrl(x86_spec_ctrl_base);
2050 	setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW);
2051 
2052 	return true;
2053 }
2054 
2055 enum bhi_mitigations {
2056 	BHI_MITIGATION_OFF,
2057 	BHI_MITIGATION_AUTO,
2058 	BHI_MITIGATION_ON,
2059 	BHI_MITIGATION_VMEXIT_ONLY,
2060 };
2061 
2062 static enum bhi_mitigations bhi_mitigation __ro_after_init =
2063 	IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF;
2064 
2065 static int __init spectre_bhi_parse_cmdline(char *str)
2066 {
2067 	if (!str)
2068 		return -EINVAL;
2069 
2070 	if (!strcmp(str, "off"))
2071 		bhi_mitigation = BHI_MITIGATION_OFF;
2072 	else if (!strcmp(str, "on"))
2073 		bhi_mitigation = BHI_MITIGATION_ON;
2074 	else if (!strcmp(str, "vmexit"))
2075 		bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY;
2076 	else
2077 		pr_err("Ignoring unknown spectre_bhi option (%s)", str);
2078 
2079 	return 0;
2080 }
2081 early_param("spectre_bhi", spectre_bhi_parse_cmdline);
2082 
2083 static void __init bhi_select_mitigation(void)
2084 {
2085 	if (!boot_cpu_has(X86_BUG_BHI))
2086 		bhi_mitigation = BHI_MITIGATION_OFF;
2087 
2088 	if (bhi_mitigation != BHI_MITIGATION_AUTO)
2089 		return;
2090 
2091 	if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL))
2092 		bhi_mitigation = BHI_MITIGATION_ON;
2093 	else if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST))
2094 		bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY;
2095 	else
2096 		bhi_mitigation = BHI_MITIGATION_OFF;
2097 }
2098 
2099 static void __init bhi_update_mitigation(void)
2100 {
2101 	if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE)
2102 		bhi_mitigation = BHI_MITIGATION_OFF;
2103 }
2104 
2105 static void __init bhi_apply_mitigation(void)
2106 {
2107 	if (bhi_mitigation == BHI_MITIGATION_OFF)
2108 		return;
2109 
2110 	/* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
2111 	if (retpoline_seq_enabled()) {
2112 		spec_ctrl_disable_kernel_rrsba();
2113 		if (rrsba_disabled)
2114 			return;
2115 	}
2116 
2117 	if (!IS_ENABLED(CONFIG_X86_64))
2118 		return;
2119 
2120 	/* Mitigate in hardware if supported */
2121 	if (spec_ctrl_bhi_dis())
2122 		return;
2123 
2124 	if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) {
2125 		pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n");
2126 		setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT);
2127 		return;
2128 	}
2129 
2130 	pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n");
2131 	setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP);
2132 	setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT);
2133 }
2134 
2135 static void __init spectre_v2_select_mitigation(void)
2136 {
2137 	if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE ||
2138 	     spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
2139 	     spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
2140 	     spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
2141 	     spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
2142 	    !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) {
2143 		pr_err("RETPOLINE selected but not compiled in. Switching to AUTO select\n");
2144 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2145 	}
2146 
2147 	if ((spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS ||
2148 	     spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
2149 	     spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
2150 	    !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
2151 		pr_err("EIBRS selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n");
2152 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2153 	}
2154 
2155 	if ((spectre_v2_cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
2156 	     spectre_v2_cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
2157 	    !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
2158 		pr_err("LFENCE selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n");
2159 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2160 	}
2161 
2162 	if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) {
2163 		pr_err("IBRS selected but not compiled in. Switching to AUTO select\n");
2164 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2165 	}
2166 
2167 	if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
2168 		pr_err("IBRS selected but not Intel CPU. Switching to AUTO select\n");
2169 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2170 	}
2171 
2172 	if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
2173 		pr_err("IBRS selected but CPU doesn't have IBRS. Switching to AUTO select\n");
2174 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2175 	}
2176 
2177 	if (spectre_v2_cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) {
2178 		pr_err("IBRS selected but running as XenPV guest. Switching to AUTO select\n");
2179 		spectre_v2_cmd = SPECTRE_V2_CMD_AUTO;
2180 	}
2181 
2182 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) {
2183 		spectre_v2_cmd = SPECTRE_V2_CMD_NONE;
2184 		return;
2185 	}
2186 
2187 	switch (spectre_v2_cmd) {
2188 	case SPECTRE_V2_CMD_NONE:
2189 		return;
2190 
2191 	case SPECTRE_V2_CMD_AUTO:
2192 		if (!should_mitigate_vuln(X86_BUG_SPECTRE_V2))
2193 			break;
2194 		fallthrough;
2195 	case SPECTRE_V2_CMD_FORCE:
2196 		if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
2197 			spectre_v2_enabled = SPECTRE_V2_EIBRS;
2198 			break;
2199 		}
2200 
2201 		spectre_v2_enabled = spectre_v2_select_retpoline();
2202 		break;
2203 
2204 	case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
2205 		pr_err(SPECTRE_V2_LFENCE_MSG);
2206 		spectre_v2_enabled = SPECTRE_V2_LFENCE;
2207 		break;
2208 
2209 	case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
2210 		spectre_v2_enabled = SPECTRE_V2_RETPOLINE;
2211 		break;
2212 
2213 	case SPECTRE_V2_CMD_RETPOLINE:
2214 		spectre_v2_enabled = spectre_v2_select_retpoline();
2215 		break;
2216 
2217 	case SPECTRE_V2_CMD_IBRS:
2218 		spectre_v2_enabled = SPECTRE_V2_IBRS;
2219 		break;
2220 
2221 	case SPECTRE_V2_CMD_EIBRS:
2222 		spectre_v2_enabled = SPECTRE_V2_EIBRS;
2223 		break;
2224 
2225 	case SPECTRE_V2_CMD_EIBRS_LFENCE:
2226 		spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE;
2227 		break;
2228 
2229 	case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
2230 		spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE;
2231 		break;
2232 	}
2233 }
2234 
2235 static void __init spectre_v2_update_mitigation(void)
2236 {
2237 	if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO &&
2238 	    !spectre_v2_in_eibrs_mode(spectre_v2_enabled)) {
2239 		if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&
2240 		    boot_cpu_has_bug(X86_BUG_RETBLEED) &&
2241 		    retbleed_mitigation != RETBLEED_MITIGATION_NONE &&
2242 		    retbleed_mitigation != RETBLEED_MITIGATION_STUFF &&
2243 		    boot_cpu_has(X86_FEATURE_IBRS) &&
2244 		    boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
2245 			spectre_v2_enabled = SPECTRE_V2_IBRS;
2246 		}
2247 	}
2248 
2249 	if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
2250 		pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]);
2251 }
2252 
2253 #ifdef CONFIG_BPF_JIT
2254 static void __bpf_arch_ibpb(void *unused)
2255 {
2256 	write_ibpb();
2257 }
2258 
2259 void bpf_arch_ibpb(void)
2260 {
2261 	on_each_cpu(__bpf_arch_ibpb, NULL, 1);
2262 }
2263 
2264 static bool __init cpu_wants_ibpb_bpf(void)
2265 {
2266 	/* A genuine retpoline already neutralizes ring0 indirect predictions */
2267 	if (retpoline_seq_enabled())
2268 		return false;
2269 
2270 	return boot_cpu_has(X86_FEATURE_IBPB);
2271 }
2272 #endif
2273 
2274 static void __init spectre_v2_apply_mitigation(void)
2275 {
2276 	if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
2277 		pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
2278 
2279 	if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) {
2280 		if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
2281 			msr_set_bit(MSR_EFER, _EFER_AUTOIBRS);
2282 		} else {
2283 			x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
2284 			update_spec_ctrl(x86_spec_ctrl_base);
2285 		}
2286 	}
2287 
2288 	switch (spectre_v2_enabled) {
2289 	case SPECTRE_V2_NONE:
2290 		return;
2291 
2292 	case SPECTRE_V2_EIBRS:
2293 		break;
2294 
2295 	case SPECTRE_V2_IBRS:
2296 		setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
2297 		if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
2298 			pr_warn(SPECTRE_V2_IBRS_PERF_MSG);
2299 		break;
2300 
2301 	case SPECTRE_V2_LFENCE:
2302 	case SPECTRE_V2_EIBRS_LFENCE:
2303 		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
2304 		fallthrough;
2305 
2306 	case SPECTRE_V2_RETPOLINE:
2307 	case SPECTRE_V2_EIBRS_RETPOLINE:
2308 		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
2309 		break;
2310 	}
2311 
2312 	/*
2313 	 * Disable alternate RSB predictions in kernel when indirect CALLs and
2314 	 * JMPs gets protection against BHI and Intramode-BTI, but RET
2315 	 * prediction from a non-RSB predictor is still a risk.
2316 	 */
2317 	if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE ||
2318 	    spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE ||
2319 	    spectre_v2_enabled == SPECTRE_V2_RETPOLINE)
2320 		spec_ctrl_disable_kernel_rrsba();
2321 
2322 	spectre_v2_select_rsb_mitigation(spectre_v2_enabled);
2323 
2324 	/*
2325 	 * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
2326 	 * and Enhanced IBRS protect firmware too, so enable IBRS around
2327 	 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't
2328 	 * otherwise enabled.
2329 	 *
2330 	 * Use "spectre_v2_enabled" to check Enhanced IBRS instead of
2331 	 * boot_cpu_has(), because the user might select retpoline on the kernel
2332 	 * command line and if the CPU supports Enhanced IBRS, kernel might
2333 	 * un-intentionally not enable IBRS around firmware calls.
2334 	 */
2335 	if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
2336 	    boot_cpu_has(X86_FEATURE_IBPB) &&
2337 	    (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
2338 	     boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
2339 
2340 		if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) {
2341 			setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
2342 			pr_info("Enabling Speculation Barrier for firmware calls\n");
2343 		}
2344 
2345 	} else if (boot_cpu_has(X86_FEATURE_IBRS) &&
2346 		   !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) {
2347 		setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
2348 		pr_info("Enabling Restricted Speculation for firmware calls\n");
2349 	}
2350 
2351 #ifdef CONFIG_BPF_JIT
2352 	if (cpu_wants_ibpb_bpf()) {
2353 		static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb);
2354 		static_branch_enable(&bpf_pred_flush_enabled);
2355 		pr_info("Enabling IBPB for BPF\n");
2356 	}
2357 #endif
2358 }
2359 
2360 static void update_stibp_msr(void * __unused)
2361 {
2362 	u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP);
2363 	update_spec_ctrl(val);
2364 }
2365 
2366 /* Update x86_spec_ctrl_base in case SMT state changed. */
2367 static void update_stibp_strict(void)
2368 {
2369 	u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
2370 
2371 	if (sched_smt_active())
2372 		mask |= SPEC_CTRL_STIBP;
2373 
2374 	if (mask == x86_spec_ctrl_base)
2375 		return;
2376 
2377 	pr_info("Update user space SMT mitigation: STIBP %s\n",
2378 		mask & SPEC_CTRL_STIBP ? "always-on" : "off");
2379 	x86_spec_ctrl_base = mask;
2380 	on_each_cpu(update_stibp_msr, NULL, 1);
2381 }
2382 
2383 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
2384 static void update_indir_branch_cond(void)
2385 {
2386 	if (sched_smt_active())
2387 		static_branch_enable(&switch_to_cond_stibp);
2388 	else
2389 		static_branch_disable(&switch_to_cond_stibp);
2390 }
2391 
2392 #undef pr_fmt
2393 #define pr_fmt(fmt) fmt
2394 
2395 /* Update the static key controlling the MDS CPU buffer clear in idle */
2396 static void update_mds_branch_idle(void)
2397 {
2398 	/*
2399 	 * Enable the idle clearing if SMT is active on CPUs which are
2400 	 * affected only by MSBDS and not any other MDS variant.
2401 	 *
2402 	 * The other variants cannot be mitigated when SMT is enabled, so
2403 	 * clearing the buffers on idle just to prevent the Store Buffer
2404 	 * repartitioning leak would be a window dressing exercise.
2405 	 */
2406 	if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
2407 		return;
2408 
2409 	if (sched_smt_active()) {
2410 		static_branch_enable(&cpu_buf_idle_clear);
2411 	} else if (mmio_mitigation == MMIO_MITIGATION_OFF ||
2412 		   (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) {
2413 		static_branch_disable(&cpu_buf_idle_clear);
2414 	}
2415 }
2416 
2417 #undef pr_fmt
2418 #define pr_fmt(fmt)	"Speculative Store Bypass: " fmt
2419 
2420 static enum ssb_mitigation ssb_mode __ro_after_init =
2421 	IS_ENABLED(CONFIG_MITIGATION_SSB) ? SPEC_STORE_BYPASS_AUTO : SPEC_STORE_BYPASS_NONE;
2422 
2423 static const char * const ssb_strings[] = {
2424 	[SPEC_STORE_BYPASS_NONE]	= "Vulnerable",
2425 	[SPEC_STORE_BYPASS_DISABLE]	= "Mitigation: Speculative Store Bypass disabled",
2426 	[SPEC_STORE_BYPASS_PRCTL]	= "Mitigation: Speculative Store Bypass disabled via prctl",
2427 	[SPEC_STORE_BYPASS_SECCOMP]	= "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
2428 };
2429 
2430 static bool nossb __ro_after_init;
2431 
2432 static int __init nossb_parse_cmdline(char *str)
2433 {
2434 	nossb = true;
2435 	ssb_mode = SPEC_STORE_BYPASS_NONE;
2436 	return 0;
2437 }
2438 early_param("nospec_store_bypass_disable", nossb_parse_cmdline);
2439 
2440 static int __init ssb_parse_cmdline(char *str)
2441 {
2442 	if (!str)
2443 		return -EINVAL;
2444 
2445 	if (nossb)
2446 		return 0;
2447 
2448 	if (!strcmp(str, "auto"))
2449 		ssb_mode = SPEC_STORE_BYPASS_AUTO;
2450 	else if (!strcmp(str, "on"))
2451 		ssb_mode = SPEC_STORE_BYPASS_DISABLE;
2452 	else if (!strcmp(str, "off"))
2453 		ssb_mode = SPEC_STORE_BYPASS_NONE;
2454 	else if (!strcmp(str, "prctl"))
2455 		ssb_mode = SPEC_STORE_BYPASS_PRCTL;
2456 	else if (!strcmp(str, "seccomp"))
2457 		ssb_mode = IS_ENABLED(CONFIG_SECCOMP) ?
2458 			SPEC_STORE_BYPASS_SECCOMP : SPEC_STORE_BYPASS_PRCTL;
2459 	else
2460 		pr_err("Ignoring unknown spec_store_bypass_disable option (%s).\n",
2461 			str);
2462 
2463 	return 0;
2464 }
2465 early_param("spec_store_bypass_disable", ssb_parse_cmdline);
2466 
2467 static void __init ssb_select_mitigation(void)
2468 {
2469 	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) {
2470 		ssb_mode = SPEC_STORE_BYPASS_NONE;
2471 		return;
2472 	}
2473 
2474 	if (ssb_mode == SPEC_STORE_BYPASS_AUTO) {
2475 		if (should_mitigate_vuln(X86_BUG_SPEC_STORE_BYPASS))
2476 			ssb_mode = SPEC_STORE_BYPASS_PRCTL;
2477 		else
2478 			ssb_mode = SPEC_STORE_BYPASS_NONE;
2479 	}
2480 
2481 	if (!boot_cpu_has(X86_FEATURE_SSBD))
2482 		ssb_mode = SPEC_STORE_BYPASS_NONE;
2483 
2484 	pr_info("%s\n", ssb_strings[ssb_mode]);
2485 }
2486 
2487 static void __init ssb_apply_mitigation(void)
2488 {
2489 	/*
2490 	 * We have three CPU feature flags that are in play here:
2491 	 *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
2492 	 *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
2493 	 *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
2494 	 */
2495 	if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) {
2496 		setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
2497 		/*
2498 		 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
2499 		 * use a completely different MSR and bit dependent on family.
2500 		 */
2501 		if (!cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) &&
2502 		    !cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) {
2503 			x86_amd_ssb_disable();
2504 		} else {
2505 			x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
2506 			update_spec_ctrl(x86_spec_ctrl_base);
2507 		}
2508 	}
2509 }
2510 
2511 #undef pr_fmt
2512 #define pr_fmt(fmt)     "Speculation prctl: " fmt
2513 
2514 static void task_update_spec_tif(struct task_struct *tsk)
2515 {
2516 	/* Force the update of the real TIF bits */
2517 	set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
2518 
2519 	/*
2520 	 * Immediately update the speculation control MSRs for the current
2521 	 * task, but for a non-current task delay setting the CPU
2522 	 * mitigation until it is scheduled next.
2523 	 *
2524 	 * This can only happen for SECCOMP mitigation. For PRCTL it's
2525 	 * always the current task.
2526 	 */
2527 	if (tsk == current)
2528 		speculation_ctrl_update_current();
2529 }
2530 
2531 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
2532 {
2533 
2534 	if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2535 		return -EPERM;
2536 
2537 	switch (ctrl) {
2538 	case PR_SPEC_ENABLE:
2539 		set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
2540 		return 0;
2541 	case PR_SPEC_DISABLE:
2542 		clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
2543 		return 0;
2544 	default:
2545 		return -ERANGE;
2546 	}
2547 }
2548 
2549 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
2550 {
2551 	if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
2552 	    ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
2553 		return -ENXIO;
2554 
2555 	switch (ctrl) {
2556 	case PR_SPEC_ENABLE:
2557 		/* If speculation is force disabled, enable is not allowed */
2558 		if (task_spec_ssb_force_disable(task))
2559 			return -EPERM;
2560 		task_clear_spec_ssb_disable(task);
2561 		task_clear_spec_ssb_noexec(task);
2562 		task_update_spec_tif(task);
2563 		break;
2564 	case PR_SPEC_DISABLE:
2565 		task_set_spec_ssb_disable(task);
2566 		task_clear_spec_ssb_noexec(task);
2567 		task_update_spec_tif(task);
2568 		break;
2569 	case PR_SPEC_FORCE_DISABLE:
2570 		task_set_spec_ssb_disable(task);
2571 		task_set_spec_ssb_force_disable(task);
2572 		task_clear_spec_ssb_noexec(task);
2573 		task_update_spec_tif(task);
2574 		break;
2575 	case PR_SPEC_DISABLE_NOEXEC:
2576 		if (task_spec_ssb_force_disable(task))
2577 			return -EPERM;
2578 		task_set_spec_ssb_disable(task);
2579 		task_set_spec_ssb_noexec(task);
2580 		task_update_spec_tif(task);
2581 		break;
2582 	default:
2583 		return -ERANGE;
2584 	}
2585 	return 0;
2586 }
2587 
2588 static bool is_spec_ib_user_controlled(void)
2589 {
2590 	return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
2591 		spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2592 		spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
2593 		spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
2594 }
2595 
2596 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
2597 {
2598 	switch (ctrl) {
2599 	case PR_SPEC_ENABLE:
2600 		if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2601 		    spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2602 			return 0;
2603 
2604 		/*
2605 		 * With strict mode for both IBPB and STIBP, the instruction
2606 		 * code paths avoid checking this task flag and instead,
2607 		 * unconditionally run the instruction. However, STIBP and IBPB
2608 		 * are independent and either can be set to conditionally
2609 		 * enabled regardless of the mode of the other.
2610 		 *
2611 		 * If either is set to conditional, allow the task flag to be
2612 		 * updated, unless it was force-disabled by a previous prctl
2613 		 * call. Currently, this is possible on an AMD CPU which has the
2614 		 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
2615 		 * kernel is booted with 'spectre_v2_user=seccomp', then
2616 		 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
2617 		 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
2618 		 */
2619 		if (!is_spec_ib_user_controlled() ||
2620 		    task_spec_ib_force_disable(task))
2621 			return -EPERM;
2622 
2623 		task_clear_spec_ib_disable(task);
2624 		task_update_spec_tif(task);
2625 		break;
2626 	case PR_SPEC_DISABLE:
2627 	case PR_SPEC_FORCE_DISABLE:
2628 		/*
2629 		 * Indirect branch speculation is always allowed when
2630 		 * mitigation is force disabled.
2631 		 */
2632 		if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2633 		    spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2634 			return -EPERM;
2635 
2636 		if (!is_spec_ib_user_controlled())
2637 			return 0;
2638 
2639 		task_set_spec_ib_disable(task);
2640 		if (ctrl == PR_SPEC_FORCE_DISABLE)
2641 			task_set_spec_ib_force_disable(task);
2642 		task_update_spec_tif(task);
2643 		if (task == current)
2644 			indirect_branch_prediction_barrier();
2645 		break;
2646 	default:
2647 		return -ERANGE;
2648 	}
2649 	return 0;
2650 }
2651 
2652 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
2653 			     unsigned long ctrl)
2654 {
2655 	switch (which) {
2656 	case PR_SPEC_STORE_BYPASS:
2657 		return ssb_prctl_set(task, ctrl);
2658 	case PR_SPEC_INDIRECT_BRANCH:
2659 		return ib_prctl_set(task, ctrl);
2660 	case PR_SPEC_L1D_FLUSH:
2661 		return l1d_flush_prctl_set(task, ctrl);
2662 	default:
2663 		return -ENODEV;
2664 	}
2665 }
2666 
2667 #ifdef CONFIG_SECCOMP
2668 void arch_seccomp_spec_mitigate(struct task_struct *task)
2669 {
2670 	if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
2671 		ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2672 	if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2673 	    spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
2674 		ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2675 }
2676 #endif
2677 
2678 static int l1d_flush_prctl_get(struct task_struct *task)
2679 {
2680 	if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2681 		return PR_SPEC_FORCE_DISABLE;
2682 
2683 	if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
2684 		return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2685 	else
2686 		return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2687 }
2688 
2689 static int ssb_prctl_get(struct task_struct *task)
2690 {
2691 	switch (ssb_mode) {
2692 	case SPEC_STORE_BYPASS_NONE:
2693 		if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2694 			return PR_SPEC_ENABLE;
2695 		return PR_SPEC_NOT_AFFECTED;
2696 	case SPEC_STORE_BYPASS_DISABLE:
2697 		return PR_SPEC_DISABLE;
2698 	case SPEC_STORE_BYPASS_SECCOMP:
2699 	case SPEC_STORE_BYPASS_PRCTL:
2700 	case SPEC_STORE_BYPASS_AUTO:
2701 		if (task_spec_ssb_force_disable(task))
2702 			return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2703 		if (task_spec_ssb_noexec(task))
2704 			return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
2705 		if (task_spec_ssb_disable(task))
2706 			return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2707 		return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2708 	}
2709 	BUG();
2710 }
2711 
2712 static int ib_prctl_get(struct task_struct *task)
2713 {
2714 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
2715 		return PR_SPEC_NOT_AFFECTED;
2716 
2717 	if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2718 	    spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2719 		return PR_SPEC_ENABLE;
2720 	else if (is_spec_ib_user_controlled()) {
2721 		if (task_spec_ib_force_disable(task))
2722 			return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2723 		if (task_spec_ib_disable(task))
2724 			return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2725 		return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2726 	} else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
2727 	    spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2728 	    spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
2729 		return PR_SPEC_DISABLE;
2730 	else
2731 		return PR_SPEC_NOT_AFFECTED;
2732 }
2733 
2734 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
2735 {
2736 	switch (which) {
2737 	case PR_SPEC_STORE_BYPASS:
2738 		return ssb_prctl_get(task);
2739 	case PR_SPEC_INDIRECT_BRANCH:
2740 		return ib_prctl_get(task);
2741 	case PR_SPEC_L1D_FLUSH:
2742 		return l1d_flush_prctl_get(task);
2743 	default:
2744 		return -ENODEV;
2745 	}
2746 }
2747 
2748 void x86_spec_ctrl_setup_ap(void)
2749 {
2750 	if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
2751 		update_spec_ctrl(x86_spec_ctrl_base);
2752 
2753 	if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
2754 		x86_amd_ssb_disable();
2755 }
2756 
2757 bool itlb_multihit_kvm_mitigation;
2758 EXPORT_SYMBOL_FOR_KVM(itlb_multihit_kvm_mitigation);
2759 
2760 #undef pr_fmt
2761 #define pr_fmt(fmt)	"L1TF: " fmt
2762 
2763 /* Default mitigation for L1TF-affected CPUs */
2764 enum l1tf_mitigations l1tf_mitigation __ro_after_init =
2765 	IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_AUTO : L1TF_MITIGATION_OFF;
2766 EXPORT_SYMBOL_FOR_KVM(l1tf_mitigation);
2767 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
2768 EXPORT_SYMBOL_FOR_KVM(l1tf_vmx_mitigation);
2769 
2770 /*
2771  * These CPUs all support 44bits physical address space internally in the
2772  * cache but CPUID can report a smaller number of physical address bits.
2773  *
2774  * The L1TF mitigation uses the top most address bit for the inversion of
2775  * non present PTEs. When the installed memory reaches into the top most
2776  * address bit due to memory holes, which has been observed on machines
2777  * which report 36bits physical address bits and have 32G RAM installed,
2778  * then the mitigation range check in l1tf_select_mitigation() triggers.
2779  * This is a false positive because the mitigation is still possible due to
2780  * the fact that the cache uses 44bit internally. Use the cache bits
2781  * instead of the reported physical bits and adjust them on the affected
2782  * machines to 44bit if the reported bits are less than 44.
2783  */
2784 static void override_cache_bits(struct cpuinfo_x86 *c)
2785 {
2786 	if (c->x86 != 6)
2787 		return;
2788 
2789 	switch (c->x86_vfm) {
2790 	case INTEL_NEHALEM:
2791 	case INTEL_WESTMERE:
2792 	case INTEL_SANDYBRIDGE:
2793 	case INTEL_IVYBRIDGE:
2794 	case INTEL_HASWELL:
2795 	case INTEL_HASWELL_L:
2796 	case INTEL_HASWELL_G:
2797 	case INTEL_BROADWELL:
2798 	case INTEL_BROADWELL_G:
2799 	case INTEL_SKYLAKE_L:
2800 	case INTEL_SKYLAKE:
2801 	case INTEL_KABYLAKE_L:
2802 	case INTEL_KABYLAKE:
2803 		if (c->x86_cache_bits < 44)
2804 			c->x86_cache_bits = 44;
2805 		break;
2806 	}
2807 }
2808 
2809 static void __init l1tf_select_mitigation(void)
2810 {
2811 	if (!boot_cpu_has_bug(X86_BUG_L1TF)) {
2812 		l1tf_mitigation = L1TF_MITIGATION_OFF;
2813 		return;
2814 	}
2815 
2816 	if (l1tf_mitigation != L1TF_MITIGATION_AUTO)
2817 		return;
2818 
2819 	if (!should_mitigate_vuln(X86_BUG_L1TF)) {
2820 		l1tf_mitigation = L1TF_MITIGATION_OFF;
2821 		return;
2822 	}
2823 
2824 	if (smt_mitigations == SMT_MITIGATIONS_ON)
2825 		l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2826 	else
2827 		l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2828 }
2829 
2830 static void __init l1tf_apply_mitigation(void)
2831 {
2832 	u64 half_pa;
2833 
2834 	if (!boot_cpu_has_bug(X86_BUG_L1TF))
2835 		return;
2836 
2837 	override_cache_bits(&boot_cpu_data);
2838 
2839 	switch (l1tf_mitigation) {
2840 	case L1TF_MITIGATION_OFF:
2841 	case L1TF_MITIGATION_FLUSH_NOWARN:
2842 	case L1TF_MITIGATION_FLUSH:
2843 	case L1TF_MITIGATION_AUTO:
2844 		break;
2845 	case L1TF_MITIGATION_FLUSH_NOSMT:
2846 	case L1TF_MITIGATION_FULL:
2847 		cpu_smt_disable(false);
2848 		break;
2849 	case L1TF_MITIGATION_FULL_FORCE:
2850 		cpu_smt_disable(true);
2851 		break;
2852 	}
2853 
2854 #if CONFIG_PGTABLE_LEVELS == 2
2855 	pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2856 	return;
2857 #endif
2858 
2859 	half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
2860 	if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
2861 			e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
2862 		pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2863 		pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2864 				half_pa);
2865 		pr_info("However, doing so will make a part of your RAM unusable.\n");
2866 		pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2867 		return;
2868 	}
2869 
2870 	setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
2871 }
2872 
2873 static int __init l1tf_cmdline(char *str)
2874 {
2875 	if (!boot_cpu_has_bug(X86_BUG_L1TF))
2876 		return 0;
2877 
2878 	if (!str)
2879 		return -EINVAL;
2880 
2881 	if (!strcmp(str, "off"))
2882 		l1tf_mitigation = L1TF_MITIGATION_OFF;
2883 	else if (!strcmp(str, "flush,nowarn"))
2884 		l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
2885 	else if (!strcmp(str, "flush"))
2886 		l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2887 	else if (!strcmp(str, "flush,nosmt"))
2888 		l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2889 	else if (!strcmp(str, "full"))
2890 		l1tf_mitigation = L1TF_MITIGATION_FULL;
2891 	else if (!strcmp(str, "full,force"))
2892 		l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
2893 
2894 	return 0;
2895 }
2896 early_param("l1tf", l1tf_cmdline);
2897 
2898 #undef pr_fmt
2899 #define pr_fmt(fmt)	"Speculative Return Stack Overflow: " fmt
2900 
2901 static const char * const srso_strings[] = {
2902 	[SRSO_MITIGATION_NONE]			= "Vulnerable",
2903 	[SRSO_MITIGATION_UCODE_NEEDED]		= "Vulnerable: No microcode",
2904 	[SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED]	= "Vulnerable: Safe RET, no microcode",
2905 	[SRSO_MITIGATION_MICROCODE]		= "Vulnerable: Microcode, no safe RET",
2906 	[SRSO_MITIGATION_NOSMT]			= "Mitigation: SMT disabled",
2907 	[SRSO_MITIGATION_SAFE_RET]		= "Mitigation: Safe RET",
2908 	[SRSO_MITIGATION_IBPB]			= "Mitigation: IBPB",
2909 	[SRSO_MITIGATION_IBPB_ON_VMEXIT]	= "Mitigation: IBPB on VMEXIT only",
2910 	[SRSO_MITIGATION_BP_SPEC_REDUCE]	= "Mitigation: Reduced Speculation",
2911 	[SRSO_MITIGATION_USER_IBPB]		= "Mitigation: IBPB on context switch",
2912 };
2913 
2914 static int __init srso_parse_cmdline(char *str)
2915 {
2916 	if (!str)
2917 		return -EINVAL;
2918 
2919 	if (!strcmp(str, "off"))
2920 		srso_mitigation = SRSO_MITIGATION_NONE;
2921 	else if (!strcmp(str, "microcode"))
2922 		srso_mitigation = SRSO_MITIGATION_MICROCODE;
2923 	else if (!strcmp(str, "safe-ret"))
2924 		srso_mitigation = SRSO_MITIGATION_SAFE_RET;
2925 	else if (!strcmp(str, "ibpb"))
2926 		srso_mitigation = SRSO_MITIGATION_IBPB;
2927 	else if (!strcmp(str, "ibpb-vmexit"))
2928 		srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
2929 	else
2930 		pr_err("Ignoring unknown SRSO option (%s).", str);
2931 
2932 	return 0;
2933 }
2934 early_param("spec_rstack_overflow", srso_parse_cmdline);
2935 
2936 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options."
2937 
2938 static void __init srso_select_mitigation(void)
2939 {
2940 	if (!boot_cpu_has_bug(X86_BUG_SRSO)) {
2941 		srso_mitigation = SRSO_MITIGATION_NONE;
2942 		return;
2943 	}
2944 
2945 	if (srso_mitigation == SRSO_MITIGATION_AUTO) {
2946 		/*
2947 		 * Use safe-RET if user->kernel or guest->host protection is
2948 		 * required.  Otherwise the 'microcode' mitigation is sufficient
2949 		 * to protect the user->user and guest->guest vectors.
2950 		 */
2951 		if ((cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) &&
2952 		    !boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION))
2953 					||
2954 		    (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) &&
2955 		     !boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))) {
2956 			srso_mitigation = SRSO_MITIGATION_SAFE_RET;
2957 		} else if (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_USER) ||
2958 			   cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_GUEST)) {
2959 			srso_mitigation = SRSO_MITIGATION_MICROCODE;
2960 		} else {
2961 			srso_mitigation = SRSO_MITIGATION_NONE;
2962 			return;
2963 		}
2964 	}
2965 
2966 	/* Zen1/2 with SMT off aren't vulnerable to SRSO. */
2967 	if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
2968 		srso_mitigation = SRSO_MITIGATION_NOSMT;
2969 		return;
2970 	}
2971 
2972 	if (!boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) {
2973 		pr_warn("IBPB-extending microcode not applied!\n");
2974 		pr_warn(SRSO_NOTICE);
2975 
2976 		/*
2977 		 * Safe-RET provides partial mitigation without microcode, but
2978 		 * other mitigations require microcode to provide any
2979 		 * mitigations.
2980 		 */
2981 		if (srso_mitigation == SRSO_MITIGATION_SAFE_RET)
2982 			srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED;
2983 		else
2984 			srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED;
2985 	}
2986 
2987 	switch (srso_mitigation) {
2988 	case SRSO_MITIGATION_SAFE_RET:
2989 	case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED:
2990 		if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) {
2991 			srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
2992 			goto ibpb_on_vmexit;
2993 		}
2994 
2995 		if (!IS_ENABLED(CONFIG_MITIGATION_SRSO)) {
2996 			pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n");
2997 			srso_mitigation = SRSO_MITIGATION_NONE;
2998 		}
2999 		break;
3000 ibpb_on_vmexit:
3001 	case SRSO_MITIGATION_IBPB_ON_VMEXIT:
3002 		if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) {
3003 			pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n");
3004 			srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE;
3005 			break;
3006 		}
3007 		fallthrough;
3008 	case SRSO_MITIGATION_IBPB:
3009 		if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
3010 			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
3011 			srso_mitigation = SRSO_MITIGATION_NONE;
3012 		}
3013 		break;
3014 	default:
3015 		break;
3016 	}
3017 }
3018 
3019 static void __init srso_update_mitigation(void)
3020 {
3021 	if (!boot_cpu_has_bug(X86_BUG_SRSO))
3022 		return;
3023 
3024 	/* If retbleed is using IBPB, that works for SRSO as well */
3025 	if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB &&
3026 	    boot_cpu_has(X86_FEATURE_IBPB_BRTYPE))
3027 		srso_mitigation = SRSO_MITIGATION_IBPB;
3028 
3029 	/*
3030 	 * See if IBPB on context switch is the only thing needed to address
3031 	 * GUEST/GUEST and USER/USER vectors.
3032 	 */
3033 	if (srso_mitigation == SRSO_MITIGATION_MICROCODE  &&
3034 	    boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO) &&
3035 	    boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION) &&
3036 	    spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE)
3037 		srso_mitigation = SRSO_MITIGATION_USER_IBPB;
3038 
3039 	pr_info("%s\n", srso_strings[srso_mitigation]);
3040 }
3041 
3042 static void __init srso_apply_mitigation(void)
3043 {
3044 	/*
3045 	 * Clear the feature flag if this mitigation is not selected as that
3046 	 * feature flag controls the BpSpecReduce MSR bit toggling in KVM.
3047 	 */
3048 	if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE)
3049 		setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE);
3050 
3051 	if (srso_mitigation == SRSO_MITIGATION_NONE) {
3052 		if (boot_cpu_has(X86_FEATURE_SBPB))
3053 			x86_pred_cmd = PRED_CMD_SBPB;
3054 		return;
3055 	}
3056 
3057 	switch (srso_mitigation) {
3058 	case SRSO_MITIGATION_SAFE_RET:
3059 	case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED:
3060 		/*
3061 		 * Enable the return thunk for generated code
3062 		 * like ftrace, static_call, etc.
3063 		 */
3064 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
3065 		setup_force_cpu_cap(X86_FEATURE_UNRET);
3066 
3067 		if (boot_cpu_data.x86 == 0x19) {
3068 			setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
3069 			set_return_thunk(srso_alias_return_thunk);
3070 		} else {
3071 			setup_force_cpu_cap(X86_FEATURE_SRSO);
3072 			set_return_thunk(srso_return_thunk);
3073 		}
3074 		break;
3075 	case SRSO_MITIGATION_IBPB:
3076 		setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
3077 		/*
3078 		 * IBPB on entry already obviates the need for
3079 		 * software-based untraining so clear those in case some
3080 		 * other mitigation like Retbleed has selected them.
3081 		 */
3082 		setup_clear_cpu_cap(X86_FEATURE_UNRET);
3083 		setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
3084 		fallthrough;
3085 	case SRSO_MITIGATION_IBPB_ON_VMEXIT:
3086 		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
3087 		/*
3088 		 * There is no need for RSB filling: entry_ibpb() ensures
3089 		 * all predictions, including the RSB, are invalidated,
3090 		 * regardless of IBPB implementation.
3091 		 */
3092 		setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
3093 		break;
3094 	default:
3095 		break;
3096 	}
3097 }
3098 
3099 #undef pr_fmt
3100 #define pr_fmt(fmt)	"VMSCAPE: " fmt
3101 
3102 enum vmscape_mitigations {
3103 	VMSCAPE_MITIGATION_NONE,
3104 	VMSCAPE_MITIGATION_AUTO,
3105 	VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER,
3106 	VMSCAPE_MITIGATION_IBPB_ON_VMEXIT,
3107 };
3108 
3109 static const char * const vmscape_strings[] = {
3110 	[VMSCAPE_MITIGATION_NONE]		= "Vulnerable",
3111 	/* [VMSCAPE_MITIGATION_AUTO] */
3112 	[VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER]	= "Mitigation: IBPB before exit to userspace",
3113 	[VMSCAPE_MITIGATION_IBPB_ON_VMEXIT]	= "Mitigation: IBPB on VMEXIT",
3114 };
3115 
3116 static enum vmscape_mitigations vmscape_mitigation __ro_after_init =
3117 	IS_ENABLED(CONFIG_MITIGATION_VMSCAPE) ? VMSCAPE_MITIGATION_AUTO : VMSCAPE_MITIGATION_NONE;
3118 
3119 static int __init vmscape_parse_cmdline(char *str)
3120 {
3121 	if (!str)
3122 		return -EINVAL;
3123 
3124 	if (!strcmp(str, "off")) {
3125 		vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
3126 	} else if (!strcmp(str, "ibpb")) {
3127 		vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
3128 	} else if (!strcmp(str, "force")) {
3129 		setup_force_cpu_bug(X86_BUG_VMSCAPE);
3130 		vmscape_mitigation = VMSCAPE_MITIGATION_AUTO;
3131 	} else {
3132 		pr_err("Ignoring unknown vmscape=%s option.\n", str);
3133 	}
3134 
3135 	return 0;
3136 }
3137 early_param("vmscape", vmscape_parse_cmdline);
3138 
3139 static void __init vmscape_select_mitigation(void)
3140 {
3141 	if (!boot_cpu_has_bug(X86_BUG_VMSCAPE) ||
3142 	    !boot_cpu_has(X86_FEATURE_IBPB)) {
3143 		vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
3144 		return;
3145 	}
3146 
3147 	if (vmscape_mitigation == VMSCAPE_MITIGATION_AUTO) {
3148 		if (should_mitigate_vuln(X86_BUG_VMSCAPE))
3149 			vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER;
3150 		else
3151 			vmscape_mitigation = VMSCAPE_MITIGATION_NONE;
3152 	}
3153 }
3154 
3155 static void __init vmscape_update_mitigation(void)
3156 {
3157 	if (!boot_cpu_has_bug(X86_BUG_VMSCAPE))
3158 		return;
3159 
3160 	if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB ||
3161 	    srso_mitigation == SRSO_MITIGATION_IBPB_ON_VMEXIT)
3162 		vmscape_mitigation = VMSCAPE_MITIGATION_IBPB_ON_VMEXIT;
3163 
3164 	pr_info("%s\n", vmscape_strings[vmscape_mitigation]);
3165 }
3166 
3167 static void __init vmscape_apply_mitigation(void)
3168 {
3169 	if (vmscape_mitigation == VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER)
3170 		setup_force_cpu_cap(X86_FEATURE_IBPB_EXIT_TO_USER);
3171 }
3172 
3173 #undef pr_fmt
3174 #define pr_fmt(fmt) fmt
3175 
3176 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
3177 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
3178 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
3179 #define VMSCAPE_MSG_SMT "VMSCAPE: SMT on, STIBP is required for full protection. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/vmscape.html for more details.\n"
3180 
3181 void cpu_bugs_smt_update(void)
3182 {
3183 	mutex_lock(&spec_ctrl_mutex);
3184 
3185 	if (sched_smt_active() && unprivileged_ebpf_enabled() &&
3186 	    spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
3187 		pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
3188 
3189 	switch (spectre_v2_user_stibp) {
3190 	case SPECTRE_V2_USER_NONE:
3191 		break;
3192 	case SPECTRE_V2_USER_STRICT:
3193 	case SPECTRE_V2_USER_STRICT_PREFERRED:
3194 		update_stibp_strict();
3195 		break;
3196 	case SPECTRE_V2_USER_PRCTL:
3197 	case SPECTRE_V2_USER_SECCOMP:
3198 		update_indir_branch_cond();
3199 		break;
3200 	}
3201 
3202 	switch (mds_mitigation) {
3203 	case MDS_MITIGATION_FULL:
3204 	case MDS_MITIGATION_AUTO:
3205 	case MDS_MITIGATION_VMWERV:
3206 		if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
3207 			pr_warn_once(MDS_MSG_SMT);
3208 		update_mds_branch_idle();
3209 		break;
3210 	case MDS_MITIGATION_OFF:
3211 		break;
3212 	}
3213 
3214 	switch (taa_mitigation) {
3215 	case TAA_MITIGATION_VERW:
3216 	case TAA_MITIGATION_AUTO:
3217 	case TAA_MITIGATION_UCODE_NEEDED:
3218 		if (sched_smt_active())
3219 			pr_warn_once(TAA_MSG_SMT);
3220 		break;
3221 	case TAA_MITIGATION_TSX_DISABLED:
3222 	case TAA_MITIGATION_OFF:
3223 		break;
3224 	}
3225 
3226 	switch (mmio_mitigation) {
3227 	case MMIO_MITIGATION_VERW:
3228 	case MMIO_MITIGATION_AUTO:
3229 	case MMIO_MITIGATION_UCODE_NEEDED:
3230 		if (sched_smt_active())
3231 			pr_warn_once(MMIO_MSG_SMT);
3232 		break;
3233 	case MMIO_MITIGATION_OFF:
3234 		break;
3235 	}
3236 
3237 	switch (tsa_mitigation) {
3238 	case TSA_MITIGATION_USER_KERNEL:
3239 	case TSA_MITIGATION_VM:
3240 	case TSA_MITIGATION_AUTO:
3241 	case TSA_MITIGATION_FULL:
3242 		/*
3243 		 * TSA-SQ can potentially lead to info leakage between
3244 		 * SMT threads.
3245 		 */
3246 		if (sched_smt_active())
3247 			static_branch_enable(&cpu_buf_idle_clear);
3248 		else
3249 			static_branch_disable(&cpu_buf_idle_clear);
3250 		break;
3251 	case TSA_MITIGATION_NONE:
3252 	case TSA_MITIGATION_UCODE_NEEDED:
3253 		break;
3254 	}
3255 
3256 	switch (vmscape_mitigation) {
3257 	case VMSCAPE_MITIGATION_NONE:
3258 	case VMSCAPE_MITIGATION_AUTO:
3259 		break;
3260 	case VMSCAPE_MITIGATION_IBPB_ON_VMEXIT:
3261 	case VMSCAPE_MITIGATION_IBPB_EXIT_TO_USER:
3262 		/*
3263 		 * Hypervisors can be attacked across-threads, warn for SMT when
3264 		 * STIBP is not already enabled system-wide.
3265 		 *
3266 		 * Intel eIBRS (!AUTOIBRS) implies STIBP on.
3267 		 */
3268 		if (!sched_smt_active() ||
3269 		    spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
3270 		    spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ||
3271 		    (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
3272 		     !boot_cpu_has(X86_FEATURE_AUTOIBRS)))
3273 			break;
3274 		pr_warn_once(VMSCAPE_MSG_SMT);
3275 		break;
3276 	}
3277 
3278 	mutex_unlock(&spec_ctrl_mutex);
3279 }
3280 
3281 void __init cpu_select_mitigations(void)
3282 {
3283 	/*
3284 	 * Read the SPEC_CTRL MSR to account for reserved bits which may
3285 	 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
3286 	 * init code as it is not enumerated and depends on the family.
3287 	 */
3288 	if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) {
3289 		rdmsrq(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
3290 
3291 		/*
3292 		 * Previously running kernel (kexec), may have some controls
3293 		 * turned ON. Clear them and let the mitigations setup below
3294 		 * rediscover them based on configuration.
3295 		 */
3296 		x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
3297 	}
3298 
3299 	x86_arch_cap_msr = x86_read_arch_cap_msr();
3300 
3301 	cpu_print_attack_vectors();
3302 
3303 	/* Select the proper CPU mitigations before patching alternatives: */
3304 	spectre_v1_select_mitigation();
3305 	spectre_v2_select_mitigation();
3306 	retbleed_select_mitigation();
3307 	spectre_v2_user_select_mitigation();
3308 	ssb_select_mitigation();
3309 	l1tf_select_mitigation();
3310 	mds_select_mitigation();
3311 	taa_select_mitigation();
3312 	mmio_select_mitigation();
3313 	rfds_select_mitigation();
3314 	srbds_select_mitigation();
3315 	l1d_flush_select_mitigation();
3316 	srso_select_mitigation();
3317 	gds_select_mitigation();
3318 	its_select_mitigation();
3319 	bhi_select_mitigation();
3320 	tsa_select_mitigation();
3321 	vmscape_select_mitigation();
3322 
3323 	/*
3324 	 * After mitigations are selected, some may need to update their
3325 	 * choices.
3326 	 */
3327 	spectre_v2_update_mitigation();
3328 	/*
3329 	 * retbleed_update_mitigation() relies on the state set by
3330 	 * spectre_v2_update_mitigation(); specifically it wants to know about
3331 	 * spectre_v2=ibrs.
3332 	 */
3333 	retbleed_update_mitigation();
3334 	/*
3335 	 * its_update_mitigation() depends on spectre_v2_update_mitigation()
3336 	 * and retbleed_update_mitigation().
3337 	 */
3338 	its_update_mitigation();
3339 
3340 	/*
3341 	 * spectre_v2_user_update_mitigation() depends on
3342 	 * retbleed_update_mitigation(), specifically the STIBP
3343 	 * selection is forced for UNRET or IBPB.
3344 	 */
3345 	spectre_v2_user_update_mitigation();
3346 	mds_update_mitigation();
3347 	taa_update_mitigation();
3348 	mmio_update_mitigation();
3349 	rfds_update_mitigation();
3350 	bhi_update_mitigation();
3351 	/* srso_update_mitigation() depends on retbleed_update_mitigation(). */
3352 	srso_update_mitigation();
3353 	vmscape_update_mitigation();
3354 
3355 	spectre_v1_apply_mitigation();
3356 	spectre_v2_apply_mitigation();
3357 	retbleed_apply_mitigation();
3358 	spectre_v2_user_apply_mitigation();
3359 	ssb_apply_mitigation();
3360 	l1tf_apply_mitigation();
3361 	mds_apply_mitigation();
3362 	taa_apply_mitigation();
3363 	mmio_apply_mitigation();
3364 	rfds_apply_mitigation();
3365 	srbds_apply_mitigation();
3366 	srso_apply_mitigation();
3367 	gds_apply_mitigation();
3368 	its_apply_mitigation();
3369 	bhi_apply_mitigation();
3370 	tsa_apply_mitigation();
3371 	vmscape_apply_mitigation();
3372 }
3373 
3374 #ifdef CONFIG_SYSFS
3375 
3376 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
3377 
3378 #if IS_ENABLED(CONFIG_KVM_INTEL)
3379 static const char * const l1tf_vmx_states[] = {
3380 	[VMENTER_L1D_FLUSH_AUTO]		= "auto",
3381 	[VMENTER_L1D_FLUSH_NEVER]		= "vulnerable",
3382 	[VMENTER_L1D_FLUSH_COND]		= "conditional cache flushes",
3383 	[VMENTER_L1D_FLUSH_ALWAYS]		= "cache flushes",
3384 	[VMENTER_L1D_FLUSH_EPT_DISABLED]	= "EPT disabled",
3385 	[VMENTER_L1D_FLUSH_NOT_REQUIRED]	= "flush not necessary"
3386 };
3387 
3388 static ssize_t l1tf_show_state(char *buf)
3389 {
3390 	if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
3391 		return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG);
3392 
3393 	if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
3394 	    (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
3395 	     sched_smt_active())) {
3396 		return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
3397 				  l1tf_vmx_states[l1tf_vmx_mitigation]);
3398 	}
3399 
3400 	return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
3401 			  l1tf_vmx_states[l1tf_vmx_mitigation],
3402 			  sched_smt_active() ? "vulnerable" : "disabled");
3403 }
3404 
3405 static ssize_t itlb_multihit_show_state(char *buf)
3406 {
3407 	if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
3408 	    !boot_cpu_has(X86_FEATURE_VMX))
3409 		return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n");
3410 	else if (!(cr4_read_shadow() & X86_CR4_VMXE))
3411 		return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n");
3412 	else if (itlb_multihit_kvm_mitigation)
3413 		return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n");
3414 	else
3415 		return sysfs_emit(buf, "KVM: Vulnerable\n");
3416 }
3417 #else
3418 static ssize_t l1tf_show_state(char *buf)
3419 {
3420 	return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG);
3421 }
3422 
3423 static ssize_t itlb_multihit_show_state(char *buf)
3424 {
3425 	return sysfs_emit(buf, "Processor vulnerable\n");
3426 }
3427 #endif
3428 
3429 static ssize_t mds_show_state(char *buf)
3430 {
3431 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
3432 		return sysfs_emit(buf, "%s; SMT Host state unknown\n",
3433 				  mds_strings[mds_mitigation]);
3434 	}
3435 
3436 	if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
3437 		return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
3438 				  (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
3439 				   sched_smt_active() ? "mitigated" : "disabled"));
3440 	}
3441 
3442 	return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
3443 			  sched_smt_active() ? "vulnerable" : "disabled");
3444 }
3445 
3446 static ssize_t tsx_async_abort_show_state(char *buf)
3447 {
3448 	if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
3449 	    (taa_mitigation == TAA_MITIGATION_OFF))
3450 		return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]);
3451 
3452 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
3453 		return sysfs_emit(buf, "%s; SMT Host state unknown\n",
3454 				  taa_strings[taa_mitigation]);
3455 	}
3456 
3457 	return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
3458 			  sched_smt_active() ? "vulnerable" : "disabled");
3459 }
3460 
3461 static ssize_t mmio_stale_data_show_state(char *buf)
3462 {
3463 	if (mmio_mitigation == MMIO_MITIGATION_OFF)
3464 		return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
3465 
3466 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
3467 		return sysfs_emit(buf, "%s; SMT Host state unknown\n",
3468 				  mmio_strings[mmio_mitigation]);
3469 	}
3470 
3471 	return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
3472 			  sched_smt_active() ? "vulnerable" : "disabled");
3473 }
3474 
3475 static ssize_t rfds_show_state(char *buf)
3476 {
3477 	return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]);
3478 }
3479 
3480 static ssize_t old_microcode_show_state(char *buf)
3481 {
3482 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
3483 		return sysfs_emit(buf, "Unknown: running under hypervisor");
3484 
3485 	return sysfs_emit(buf, "Vulnerable\n");
3486 }
3487 
3488 static ssize_t its_show_state(char *buf)
3489 {
3490 	return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]);
3491 }
3492 
3493 static char *stibp_state(void)
3494 {
3495 	if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
3496 	    !boot_cpu_has(X86_FEATURE_AUTOIBRS))
3497 		return "";
3498 
3499 	switch (spectre_v2_user_stibp) {
3500 	case SPECTRE_V2_USER_NONE:
3501 		return "; STIBP: disabled";
3502 	case SPECTRE_V2_USER_STRICT:
3503 		return "; STIBP: forced";
3504 	case SPECTRE_V2_USER_STRICT_PREFERRED:
3505 		return "; STIBP: always-on";
3506 	case SPECTRE_V2_USER_PRCTL:
3507 	case SPECTRE_V2_USER_SECCOMP:
3508 		if (static_key_enabled(&switch_to_cond_stibp))
3509 			return "; STIBP: conditional";
3510 	}
3511 	return "";
3512 }
3513 
3514 static char *ibpb_state(void)
3515 {
3516 	if (boot_cpu_has(X86_FEATURE_IBPB)) {
3517 		if (static_key_enabled(&switch_mm_always_ibpb))
3518 			return "; IBPB: always-on";
3519 		if (static_key_enabled(&switch_mm_cond_ibpb))
3520 			return "; IBPB: conditional";
3521 		return "; IBPB: disabled";
3522 	}
3523 	return "";
3524 }
3525 
3526 static char *pbrsb_eibrs_state(void)
3527 {
3528 	if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
3529 		if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) ||
3530 		    boot_cpu_has(X86_FEATURE_RSB_VMEXIT))
3531 			return "; PBRSB-eIBRS: SW sequence";
3532 		else
3533 			return "; PBRSB-eIBRS: Vulnerable";
3534 	} else {
3535 		return "; PBRSB-eIBRS: Not affected";
3536 	}
3537 }
3538 
3539 static const char *spectre_bhi_state(void)
3540 {
3541 	if (!boot_cpu_has_bug(X86_BUG_BHI))
3542 		return "; BHI: Not affected";
3543 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
3544 		return "; BHI: BHI_DIS_S";
3545 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
3546 		return "; BHI: SW loop, KVM: SW loop";
3547 	else if (retpoline_seq_enabled() && rrsba_disabled)
3548 		return "; BHI: Retpoline";
3549 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT))
3550 		return "; BHI: Vulnerable, KVM: SW loop";
3551 
3552 	return "; BHI: Vulnerable";
3553 }
3554 
3555 static ssize_t spectre_v2_show_state(char *buf)
3556 {
3557 	if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
3558 		return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
3559 
3560 	if (sched_smt_active() && unprivileged_ebpf_enabled() &&
3561 	    spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
3562 		return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
3563 
3564 	return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n",
3565 			  spectre_v2_strings[spectre_v2_enabled],
3566 			  ibpb_state(),
3567 			  boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "",
3568 			  stibp_state(),
3569 			  boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "",
3570 			  pbrsb_eibrs_state(),
3571 			  spectre_bhi_state(),
3572 			  /* this should always be at the end */
3573 			  spectre_v2_module_string());
3574 }
3575 
3576 static ssize_t srbds_show_state(char *buf)
3577 {
3578 	return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]);
3579 }
3580 
3581 static ssize_t retbleed_show_state(char *buf)
3582 {
3583 	if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
3584 	    retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
3585 		if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
3586 		    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
3587 			return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");
3588 
3589 		return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation],
3590 				  !sched_smt_active() ? "disabled" :
3591 				  spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
3592 				  spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
3593 				  "enabled with STIBP protection" : "vulnerable");
3594 	}
3595 
3596 	return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
3597 }
3598 
3599 static ssize_t srso_show_state(char *buf)
3600 {
3601 	return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]);
3602 }
3603 
3604 static ssize_t gds_show_state(char *buf)
3605 {
3606 	return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
3607 }
3608 
3609 static ssize_t tsa_show_state(char *buf)
3610 {
3611 	return sysfs_emit(buf, "%s\n", tsa_strings[tsa_mitigation]);
3612 }
3613 
3614 static ssize_t vmscape_show_state(char *buf)
3615 {
3616 	return sysfs_emit(buf, "%s\n", vmscape_strings[vmscape_mitigation]);
3617 }
3618 
3619 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
3620 			       char *buf, unsigned int bug)
3621 {
3622 	if (!boot_cpu_has_bug(bug))
3623 		return sysfs_emit(buf, "Not affected\n");
3624 
3625 	switch (bug) {
3626 	case X86_BUG_CPU_MELTDOWN:
3627 		if (boot_cpu_has(X86_FEATURE_PTI))
3628 			return sysfs_emit(buf, "Mitigation: PTI\n");
3629 
3630 		if (hypervisor_is_type(X86_HYPER_XEN_PV))
3631 			return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
3632 
3633 		break;
3634 
3635 	case X86_BUG_SPECTRE_V1:
3636 		return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
3637 
3638 	case X86_BUG_SPECTRE_V2:
3639 		return spectre_v2_show_state(buf);
3640 
3641 	case X86_BUG_SPEC_STORE_BYPASS:
3642 		return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]);
3643 
3644 	case X86_BUG_L1TF:
3645 		if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
3646 			return l1tf_show_state(buf);
3647 		break;
3648 
3649 	case X86_BUG_MDS:
3650 		return mds_show_state(buf);
3651 
3652 	case X86_BUG_TAA:
3653 		return tsx_async_abort_show_state(buf);
3654 
3655 	case X86_BUG_ITLB_MULTIHIT:
3656 		return itlb_multihit_show_state(buf);
3657 
3658 	case X86_BUG_SRBDS:
3659 		return srbds_show_state(buf);
3660 
3661 	case X86_BUG_MMIO_STALE_DATA:
3662 		return mmio_stale_data_show_state(buf);
3663 
3664 	case X86_BUG_RETBLEED:
3665 		return retbleed_show_state(buf);
3666 
3667 	case X86_BUG_SRSO:
3668 		return srso_show_state(buf);
3669 
3670 	case X86_BUG_GDS:
3671 		return gds_show_state(buf);
3672 
3673 	case X86_BUG_RFDS:
3674 		return rfds_show_state(buf);
3675 
3676 	case X86_BUG_OLD_MICROCODE:
3677 		return old_microcode_show_state(buf);
3678 
3679 	case X86_BUG_ITS:
3680 		return its_show_state(buf);
3681 
3682 	case X86_BUG_TSA:
3683 		return tsa_show_state(buf);
3684 
3685 	case X86_BUG_VMSCAPE:
3686 		return vmscape_show_state(buf);
3687 
3688 	default:
3689 		break;
3690 	}
3691 
3692 	return sysfs_emit(buf, "Vulnerable\n");
3693 }
3694 
3695 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
3696 {
3697 	return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
3698 }
3699 
3700 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
3701 {
3702 	return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
3703 }
3704 
3705 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
3706 {
3707 	return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
3708 }
3709 
3710 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
3711 {
3712 	return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
3713 }
3714 
3715 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
3716 {
3717 	return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
3718 }
3719 
3720 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
3721 {
3722 	return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
3723 }
3724 
3725 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
3726 {
3727 	return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
3728 }
3729 
3730 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
3731 {
3732 	return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
3733 }
3734 
3735 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
3736 {
3737 	return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
3738 }
3739 
3740 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf)
3741 {
3742 	return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
3743 }
3744 
3745 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf)
3746 {
3747 	return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);
3748 }
3749 
3750 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf)
3751 {
3752 	return cpu_show_common(dev, attr, buf, X86_BUG_SRSO);
3753 }
3754 
3755 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf)
3756 {
3757 	return cpu_show_common(dev, attr, buf, X86_BUG_GDS);
3758 }
3759 
3760 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf)
3761 {
3762 	return cpu_show_common(dev, attr, buf, X86_BUG_RFDS);
3763 }
3764 
3765 ssize_t cpu_show_old_microcode(struct device *dev, struct device_attribute *attr, char *buf)
3766 {
3767 	return cpu_show_common(dev, attr, buf, X86_BUG_OLD_MICROCODE);
3768 }
3769 
3770 ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf)
3771 {
3772 	return cpu_show_common(dev, attr, buf, X86_BUG_ITS);
3773 }
3774 
3775 ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf)
3776 {
3777 	return cpu_show_common(dev, attr, buf, X86_BUG_TSA);
3778 }
3779 
3780 ssize_t cpu_show_vmscape(struct device *dev, struct device_attribute *attr, char *buf)
3781 {
3782 	return cpu_show_common(dev, attr, buf, X86_BUG_VMSCAPE);
3783 }
3784 #endif
3785 
3786 void __warn_thunk(void)
3787 {
3788 	WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n");
3789 }
3790 
3791 #ifdef CONFIG_MITIGATION_SRSO
3792 /*
3793  * Called during exception/interrupt entry if interrupted during the
3794  * safe-RET sequence.  The safe-RET sequence consists of 3 instructions:
3795  *
3796  *	CALL
3797  *	LEA 8(%RSP), %RSP
3798  *	RET
3799  *
3800  * An interrupt after the CALL or after the LEA could potentially lead
3801  * to branch predictor poisoning and results in the sequence not being
3802  * able to be safely resumed.
3803  *
3804  * Therefore, modify the regs state as if the remaining part of the
3805  * safe-RET sequence executed so the interrupt returns back to the
3806  * desired return target, instead of the to the safe-RET sequence.
3807  */
3808 void noinstr handle_interrupted_saferet(struct pt_regs *regs)
3809 {
3810 	unsigned long rip = regs->ip;
3811 
3812 	if (rip == (unsigned long) srso_safe_ret ||
3813 	    rip == (unsigned long) srso_alias_safe_ret) {
3814 	    /* Modify stack pointer as if LEA executed: */
3815 	    regs->sp += 8;
3816 	}
3817 
3818 	/*
3819 	 * Adjust registers as if RET executed:
3820 	 *
3821 	 * 1. Read the return address off the stack and into rIP:
3822 	 */
3823 	regs->ip = *(unsigned long *)(regs->sp);
3824 
3825 	/* 2. Pop rIP off the stack: */
3826 	regs->sp += 8;
3827 }
3828 #endif /* CONFIG_MITIGATION_SRSO */
3829