xref: /linux/arch/x86/kernel/cpu/common.c (revision 223ba8ee0a3986718c874b66ed24e7f87f6b8124)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* cpu_feature_enabled() cannot be used this early */
3 #define USE_EARLY_PGTABLE_L5
4 
5 #include <linux/memblock.h>
6 #include <linux/linkage.h>
7 #include <linux/bitops.h>
8 #include <linux/kernel.h>
9 #include <linux/export.h>
10 #include <linux/percpu.h>
11 #include <linux/string.h>
12 #include <linux/ctype.h>
13 #include <linux/delay.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/clock.h>
16 #include <linux/sched/task.h>
17 #include <linux/sched/smt.h>
18 #include <linux/init.h>
19 #include <linux/kprobes.h>
20 #include <linux/kgdb.h>
21 #include <linux/mem_encrypt.h>
22 #include <linux/smp.h>
23 #include <linux/cpu.h>
24 #include <linux/io.h>
25 #include <linux/syscore_ops.h>
26 #include <linux/pgtable.h>
27 #include <linux/stackprotector.h>
28 #include <linux/utsname.h>
29 #include <linux/efi.h>
30 
31 #include <asm/alternative.h>
32 #include <asm/cmdline.h>
33 #include <asm/cpuid/api.h>
34 #include <asm/perf_event.h>
35 #include <asm/mmu_context.h>
36 #include <asm/doublefault.h>
37 #include <asm/archrandom.h>
38 #include <asm/hypervisor.h>
39 #include <asm/processor.h>
40 #include <asm/tlbflush.h>
41 #include <asm/debugreg.h>
42 #include <asm/sections.h>
43 #include <asm/vsyscall.h>
44 #include <linux/topology.h>
45 #include <linux/cpumask.h>
46 #include <linux/atomic.h>
47 #include <asm/proto.h>
48 #include <asm/setup.h>
49 #include <asm/apic.h>
50 #include <asm/desc.h>
51 #include <asm/fpu/api.h>
52 #include <asm/mtrr.h>
53 #include <asm/hwcap2.h>
54 #include <linux/numa.h>
55 #include <asm/numa.h>
56 #include <asm/asm.h>
57 #include <asm/bugs.h>
58 #include <asm/cpu.h>
59 #include <asm/mce.h>
60 #include <asm/msr.h>
61 #include <asm/cacheinfo.h>
62 #include <asm/memtype.h>
63 #include <asm/microcode.h>
64 #include <asm/intel-family.h>
65 #include <asm/cpu_device_id.h>
66 #include <asm/fred.h>
67 #include <asm/uv/uv.h>
68 #include <asm/ia32.h>
69 #include <asm/set_memory.h>
70 #include <asm/traps.h>
71 #include <asm/sev.h>
72 #include <asm/tdx.h>
73 #include <asm/posted_intr.h>
74 #include <asm/runtime-const.h>
75 
76 #include "cpu.h"
77 
78 DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
79 EXPORT_PER_CPU_SYMBOL(cpu_info);
80 
81 u32 elf_hwcap2 __read_mostly;
82 
83 /* Number of siblings per CPU package */
84 unsigned int __max_threads_per_core __ro_after_init = 1;
85 EXPORT_SYMBOL(__max_threads_per_core);
86 
87 unsigned int __max_dies_per_package __ro_after_init = 1;
88 EXPORT_SYMBOL(__max_dies_per_package);
89 
90 unsigned int __max_logical_packages __ro_after_init = 1;
91 EXPORT_SYMBOL(__max_logical_packages);
92 
93 unsigned int __num_cores_per_package __ro_after_init = 1;
94 EXPORT_SYMBOL(__num_cores_per_package);
95 
96 unsigned int __num_threads_per_package __ro_after_init = 1;
97 EXPORT_SYMBOL(__num_threads_per_package);
98 
99 static struct ppin_info {
100 	int	feature;
101 	int	msr_ppin_ctl;
102 	int	msr_ppin;
103 } ppin_info[] = {
104 	[X86_VENDOR_INTEL] = {
105 		.feature = X86_FEATURE_INTEL_PPIN,
106 		.msr_ppin_ctl = MSR_PPIN_CTL,
107 		.msr_ppin = MSR_PPIN
108 	},
109 	[X86_VENDOR_AMD] = {
110 		.feature = X86_FEATURE_AMD_PPIN,
111 		.msr_ppin_ctl = MSR_AMD_PPIN_CTL,
112 		.msr_ppin = MSR_AMD_PPIN
113 	},
114 };
115 
116 static const struct x86_cpu_id ppin_cpuids[] = {
117 	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
118 	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
119 
120 	/* Legacy models without CPUID enumeration */
121 	X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
122 	X86_MATCH_VFM(INTEL_HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
123 	X86_MATCH_VFM(INTEL_BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
124 	X86_MATCH_VFM(INTEL_BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
125 	X86_MATCH_VFM(INTEL_SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
126 	X86_MATCH_VFM(INTEL_ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
127 	X86_MATCH_VFM(INTEL_ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
128 	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
129 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
130 	X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
131 	X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
132 
133 	{}
134 };
135 
ppin_init(struct cpuinfo_x86 * c)136 static void ppin_init(struct cpuinfo_x86 *c)
137 {
138 	const struct x86_cpu_id *id;
139 	unsigned long long val;
140 	struct ppin_info *info;
141 
142 	id = x86_match_cpu(ppin_cpuids);
143 	if (!id)
144 		return;
145 
146 	/*
147 	 * Testing the presence of the MSR is not enough. Need to check
148 	 * that the PPIN_CTL allows reading of the PPIN.
149 	 */
150 	info = (struct ppin_info *)id->driver_data;
151 
152 	if (rdmsrq_safe(info->msr_ppin_ctl, &val))
153 		goto clear_ppin;
154 
155 	if ((val & 3UL) == 1UL) {
156 		/* PPIN locked in disabled mode */
157 		goto clear_ppin;
158 	}
159 
160 	/* If PPIN is disabled, try to enable */
161 	if (!(val & 2UL)) {
162 		wrmsrq_safe(info->msr_ppin_ctl,  val | 2UL);
163 		rdmsrq_safe(info->msr_ppin_ctl, &val);
164 	}
165 
166 	/* Is the enable bit set? */
167 	if (val & 2UL) {
168 		c->ppin = native_rdmsrq(info->msr_ppin);
169 		set_cpu_cap(c, info->feature);
170 		return;
171 	}
172 
173 clear_ppin:
174 	setup_clear_cpu_cap(info->feature);
175 }
176 
default_init(struct cpuinfo_x86 * c)177 static void default_init(struct cpuinfo_x86 *c)
178 {
179 #ifdef CONFIG_X86_64
180 	cpu_detect_cache_sizes(c);
181 #else
182 	/* Not much we can do here... */
183 	/* Check if at least it has cpuid */
184 	if (c->cpuid_level == -1) {
185 		/* No cpuid. It must be an ancient CPU */
186 		if (c->x86 == 4)
187 			strcpy(c->x86_model_id, "486");
188 		else if (c->x86 == 3)
189 			strcpy(c->x86_model_id, "386");
190 	}
191 #endif
192 }
193 
194 static const struct cpu_dev default_cpu = {
195 	.c_init		= default_init,
196 	.c_vendor	= "Unknown",
197 	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
198 };
199 
200 static const struct cpu_dev *this_cpu = &default_cpu;
201 
202 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
203 #ifdef CONFIG_X86_64
204 	/*
205 	 * We need valid kernel segments for data and code in long mode too
206 	 * IRET will check the segment types  kkeil 2000/10/28
207 	 * Also sysret mandates a special GDT layout
208 	 *
209 	 * TLS descriptors are currently at a different place compared to i386.
210 	 * Hopefully nobody expects them at a fixed place (Wine?)
211 	 */
212 	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
213 	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
214 	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
215 	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
216 	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
217 	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
218 #else
219 	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
220 	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
221 	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
222 	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
223 	/*
224 	 * Segments used for calling PnP BIOS have byte granularity.
225 	 * They code segments and data segments have fixed 64k limits,
226 	 * the transfer segment sizes are set at run time.
227 	 */
228 	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
229 	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
230 	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
231 	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
232 	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
233 	/*
234 	 * The APM segments have byte granularity and their bases
235 	 * are set at run time.  All have 64k limits.
236 	 */
237 	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
238 	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
239 	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
240 
241 	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
242 	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
243 #endif
244 } };
245 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
246 SYM_PIC_ALIAS(gdt_page);
247 
248 #ifdef CONFIG_X86_64
x86_nopcid_setup(char * s)249 static int __init x86_nopcid_setup(char *s)
250 {
251 	/* nopcid doesn't accept parameters */
252 	if (s)
253 		return -EINVAL;
254 
255 	/* do not emit a message if the feature is not present */
256 	if (!boot_cpu_has(X86_FEATURE_PCID))
257 		return 0;
258 
259 	setup_clear_cpu_cap(X86_FEATURE_PCID);
260 	pr_info("nopcid: PCID feature disabled\n");
261 	return 0;
262 }
263 early_param("nopcid", x86_nopcid_setup);
264 #endif
265 
x86_noinvpcid_setup(char * s)266 static int __init x86_noinvpcid_setup(char *s)
267 {
268 	/* noinvpcid doesn't accept parameters */
269 	if (s)
270 		return -EINVAL;
271 
272 	/* do not emit a message if the feature is not present */
273 	if (!boot_cpu_has(X86_FEATURE_INVPCID))
274 		return 0;
275 
276 	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
277 	pr_info("noinvpcid: INVPCID feature disabled\n");
278 	return 0;
279 }
280 early_param("noinvpcid", x86_noinvpcid_setup);
281 
282 /* Standard macro to see if a specific flag is changeable */
flag_is_changeable_p(unsigned long flag)283 static inline bool flag_is_changeable_p(unsigned long flag)
284 {
285 	unsigned long f1, f2;
286 
287 	if (!IS_ENABLED(CONFIG_X86_32))
288 		return true;
289 
290 	/*
291 	 * Cyrix and IDT cpus allow disabling of CPUID
292 	 * so the code below may return different results
293 	 * when it is executed before and after enabling
294 	 * the CPUID. Add "volatile" to not allow gcc to
295 	 * optimize the subsequent calls to this function.
296 	 */
297 	asm volatile ("pushfl		\n\t"
298 		      "pushfl		\n\t"
299 		      "popl %0		\n\t"
300 		      "movl %0, %1	\n\t"
301 		      "xorl %2, %0	\n\t"
302 		      "pushl %0		\n\t"
303 		      "popfl		\n\t"
304 		      "pushfl		\n\t"
305 		      "popl %0		\n\t"
306 		      "popfl		\n\t"
307 
308 		      : "=&r" (f1), "=&r" (f2)
309 		      : "ir" (flag));
310 
311 	return (f1 ^ f2) & flag;
312 }
313 
314 #ifdef CONFIG_X86_32
315 static int cachesize_override = -1;
316 static int disable_x86_serial_nr = 1;
317 
cachesize_setup(char * str)318 static int __init cachesize_setup(char *str)
319 {
320 	get_option(&str, &cachesize_override);
321 	return 1;
322 }
323 __setup("cachesize=", cachesize_setup);
324 
325 /* Probe for the CPUID instruction */
cpuid_feature(void)326 bool cpuid_feature(void)
327 {
328 	return flag_is_changeable_p(X86_EFLAGS_ID);
329 }
330 
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)331 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
332 {
333 	unsigned long lo, hi;
334 
335 	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
336 		return;
337 
338 	/* Disable processor serial number: */
339 
340 	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
341 	lo |= 0x200000;
342 	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
343 
344 	pr_notice("CPU serial number disabled.\n");
345 	clear_cpu_cap(c, X86_FEATURE_PN);
346 
347 	/* Disabling the serial number may affect the cpuid level */
348 	c->cpuid_level = cpuid_eax(0);
349 }
350 
x86_serial_nr_setup(char * s)351 static int __init x86_serial_nr_setup(char *s)
352 {
353 	disable_x86_serial_nr = 0;
354 	return 1;
355 }
356 __setup("serialnumber", x86_serial_nr_setup);
357 #else
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)358 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
359 {
360 }
361 #endif
362 
setup_smep(struct cpuinfo_x86 * c)363 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
364 {
365 	if (cpu_has(c, X86_FEATURE_SMEP))
366 		cr4_set_bits(X86_CR4_SMEP);
367 }
368 
setup_smap(struct cpuinfo_x86 * c)369 static __always_inline void setup_smap(struct cpuinfo_x86 *c)
370 {
371 	unsigned long eflags = native_save_fl();
372 
373 	/* This should have been cleared long ago */
374 	BUG_ON(eflags & X86_EFLAGS_AC);
375 
376 	if (cpu_has(c, X86_FEATURE_SMAP))
377 		cr4_set_bits(X86_CR4_SMAP);
378 }
379 
setup_umip(struct cpuinfo_x86 * c)380 static __always_inline void setup_umip(struct cpuinfo_x86 *c)
381 {
382 	/* Check the boot processor, plus build option for UMIP. */
383 	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
384 		goto out;
385 
386 	/* Check the current processor's cpuid bits. */
387 	if (!cpu_has(c, X86_FEATURE_UMIP))
388 		goto out;
389 
390 	cr4_set_bits(X86_CR4_UMIP);
391 
392 	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
393 
394 	return;
395 
396 out:
397 	/*
398 	 * Make sure UMIP is disabled in case it was enabled in a
399 	 * previous boot (e.g., via kexec).
400 	 */
401 	cr4_clear_bits(X86_CR4_UMIP);
402 }
403 
404 /* These bits should not change their value after CPU init is finished. */
405 static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
406 					     X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED;
407 static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
408 static unsigned long cr4_pinned_bits __ro_after_init;
409 
native_write_cr0(unsigned long val)410 void native_write_cr0(unsigned long val)
411 {
412 	unsigned long bits_missing = 0;
413 
414 set_register:
415 	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
416 
417 	if (static_branch_likely(&cr_pinning)) {
418 		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
419 			bits_missing = X86_CR0_WP;
420 			val |= bits_missing;
421 			goto set_register;
422 		}
423 		/* Warn after we've set the missing bits. */
424 		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
425 	}
426 }
427 EXPORT_SYMBOL(native_write_cr0);
428 
native_write_cr4(unsigned long val)429 void __no_profile native_write_cr4(unsigned long val)
430 {
431 	unsigned long bits_changed = 0;
432 
433 set_register:
434 	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
435 
436 	if (static_branch_likely(&cr_pinning)) {
437 		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
438 			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
439 			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
440 			goto set_register;
441 		}
442 		/* Warn after we've corrected the changed bits. */
443 		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
444 			  bits_changed);
445 	}
446 }
447 #if IS_MODULE(CONFIG_LKDTM)
448 EXPORT_SYMBOL_GPL(native_write_cr4);
449 #endif
450 
cr4_update_irqsoff(unsigned long set,unsigned long clear)451 void cr4_update_irqsoff(unsigned long set, unsigned long clear)
452 {
453 	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
454 
455 	lockdep_assert_irqs_disabled();
456 
457 	newval = (cr4 & ~clear) | set;
458 	if (newval != cr4) {
459 		this_cpu_write(cpu_tlbstate.cr4, newval);
460 		__write_cr4(newval);
461 	}
462 }
463 EXPORT_SYMBOL(cr4_update_irqsoff);
464 
465 /* Read the CR4 shadow. */
cr4_read_shadow(void)466 unsigned long cr4_read_shadow(void)
467 {
468 	return this_cpu_read(cpu_tlbstate.cr4);
469 }
470 EXPORT_SYMBOL_GPL(cr4_read_shadow);
471 
cr4_init(void)472 void cr4_init(void)
473 {
474 	unsigned long cr4 = __read_cr4();
475 
476 	if (boot_cpu_has(X86_FEATURE_PCID))
477 		cr4 |= X86_CR4_PCIDE;
478 	if (static_branch_likely(&cr_pinning))
479 		cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
480 
481 	__write_cr4(cr4);
482 
483 	/* Initialize cr4 shadow for this CPU. */
484 	this_cpu_write(cpu_tlbstate.cr4, cr4);
485 }
486 
487 /*
488  * Once CPU feature detection is finished (and boot params have been
489  * parsed), record any of the sensitive CR bits that are set, and
490  * enable CR pinning.
491  */
setup_cr_pinning(void)492 static void __init setup_cr_pinning(void)
493 {
494 	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
495 	static_key_enable(&cr_pinning.key);
496 }
497 
x86_nofsgsbase_setup(char * arg)498 static __init int x86_nofsgsbase_setup(char *arg)
499 {
500 	/* Require an exact match without trailing characters. */
501 	if (strlen(arg))
502 		return 0;
503 
504 	/* Do not emit a message if the feature is not present. */
505 	if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
506 		return 1;
507 
508 	setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
509 	pr_info("FSGSBASE disabled via kernel command line\n");
510 	return 1;
511 }
512 __setup("nofsgsbase", x86_nofsgsbase_setup);
513 
514 /*
515  * Protection Keys are not available in 32-bit mode.
516  */
517 static bool pku_disabled;
518 
setup_pku(struct cpuinfo_x86 * c)519 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
520 {
521 	if (c == &boot_cpu_data) {
522 		if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
523 			return;
524 		/*
525 		 * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
526 		 * bit to be set.  Enforce it.
527 		 */
528 		setup_force_cpu_cap(X86_FEATURE_OSPKE);
529 
530 	} else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
531 		return;
532 	}
533 
534 	cr4_set_bits(X86_CR4_PKE);
535 	/* Load the default PKRU value */
536 	pkru_write_default();
537 }
538 
539 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
setup_disable_pku(char * arg)540 static __init int setup_disable_pku(char *arg)
541 {
542 	/*
543 	 * Do not clear the X86_FEATURE_PKU bit.  All of the
544 	 * runtime checks are against OSPKE so clearing the
545 	 * bit does nothing.
546 	 *
547 	 * This way, we will see "pku" in cpuinfo, but not
548 	 * "ospke", which is exactly what we want.  It shows
549 	 * that the CPU has PKU, but the OS has not enabled it.
550 	 * This happens to be exactly how a system would look
551 	 * if we disabled the config option.
552 	 */
553 	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
554 	pku_disabled = true;
555 	return 1;
556 }
557 __setup("nopku", setup_disable_pku);
558 #endif
559 
560 #ifdef CONFIG_X86_KERNEL_IBT
561 
ibt_save(bool disable)562 __noendbr u64 ibt_save(bool disable)
563 {
564 	u64 msr = 0;
565 
566 	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
567 		rdmsrq(MSR_IA32_S_CET, msr);
568 		if (disable)
569 			wrmsrq(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
570 	}
571 
572 	return msr;
573 }
574 
ibt_restore(u64 save)575 __noendbr void ibt_restore(u64 save)
576 {
577 	u64 msr;
578 
579 	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
580 		rdmsrq(MSR_IA32_S_CET, msr);
581 		msr &= ~CET_ENDBR_EN;
582 		msr |= (save & CET_ENDBR_EN);
583 		wrmsrq(MSR_IA32_S_CET, msr);
584 	}
585 }
586 
587 #endif
588 
setup_cet(struct cpuinfo_x86 * c)589 static __always_inline void setup_cet(struct cpuinfo_x86 *c)
590 {
591 	bool user_shstk, kernel_ibt;
592 
593 	if (!IS_ENABLED(CONFIG_X86_CET))
594 		return;
595 
596 	kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
597 	user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
598 		     IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
599 
600 	if (!kernel_ibt && !user_shstk)
601 		return;
602 
603 	if (user_shstk)
604 		set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
605 
606 	if (kernel_ibt)
607 		wrmsrq(MSR_IA32_S_CET, CET_ENDBR_EN);
608 	else
609 		wrmsrq(MSR_IA32_S_CET, 0);
610 
611 	cr4_set_bits(X86_CR4_CET);
612 
613 	if (kernel_ibt && ibt_selftest()) {
614 		pr_err("IBT selftest: Failed!\n");
615 		wrmsrq(MSR_IA32_S_CET, 0);
616 		setup_clear_cpu_cap(X86_FEATURE_IBT);
617 	}
618 }
619 
cet_disable(void)620 __noendbr void cet_disable(void)
621 {
622 	if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
623 	      cpu_feature_enabled(X86_FEATURE_SHSTK)))
624 		return;
625 
626 	wrmsrq(MSR_IA32_S_CET, 0);
627 	wrmsrq(MSR_IA32_U_CET, 0);
628 }
629 
630 /*
631  * Some CPU features depend on higher CPUID levels, which may not always
632  * be available due to CPUID level capping or broken virtualization
633  * software.  Add those features to this table to auto-disable them.
634  */
635 struct cpuid_dependent_feature {
636 	u32 feature;
637 	u32 level;
638 };
639 
640 static const struct cpuid_dependent_feature
641 cpuid_dependent_features[] = {
642 	{ X86_FEATURE_MWAIT,		CPUID_LEAF_MWAIT },
643 	{ X86_FEATURE_DCA,		CPUID_LEAF_DCA },
644 	{ X86_FEATURE_XSAVE,		CPUID_LEAF_XSTATE },
645 	{ 0, 0 }
646 };
647 
filter_cpuid_features(struct cpuinfo_x86 * c,bool warn)648 static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
649 {
650 	const struct cpuid_dependent_feature *df;
651 
652 	for (df = cpuid_dependent_features; df->feature; df++) {
653 
654 		if (!cpu_has(c, df->feature))
655 			continue;
656 		/*
657 		 * Note: cpuid_level is set to -1 if unavailable, but
658 		 * extended_extended_level is set to 0 if unavailable
659 		 * and the legitimate extended levels are all negative
660 		 * when signed; hence the weird messing around with
661 		 * signs here...
662 		 */
663 		if (!((s32)df->level < 0 ?
664 		     (u32)df->level > (u32)c->extended_cpuid_level :
665 		     (s32)df->level > (s32)c->cpuid_level))
666 			continue;
667 
668 		clear_cpu_cap(c, df->feature);
669 		if (!warn)
670 			continue;
671 
672 		pr_warn("CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
673 			x86_cap_flags[df->feature], df->level);
674 	}
675 }
676 
677 /*
678  * Naming convention should be: <Name> [(<Codename>)]
679  * This table only is used unless init_<vendor>() below doesn't set it;
680  * in particular, if CPUID levels 0x80000002..4 are supported, this
681  * isn't used
682  */
683 
684 /* Look up CPU names by table lookup. */
table_lookup_model(struct cpuinfo_x86 * c)685 static const char *table_lookup_model(struct cpuinfo_x86 *c)
686 {
687 #ifdef CONFIG_X86_32
688 	const struct legacy_cpu_model_info *info;
689 
690 	if (c->x86_model >= 16)
691 		return NULL;	/* Range check */
692 
693 	if (!this_cpu)
694 		return NULL;
695 
696 	info = this_cpu->legacy_models;
697 
698 	while (info->family) {
699 		if (info->family == c->x86)
700 			return info->model_names[c->x86_model];
701 		info++;
702 	}
703 #endif
704 	return NULL;		/* Not found */
705 }
706 
707 /* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
708 __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
709 __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
710 
711 #ifdef CONFIG_X86_32
712 /* The 32-bit entry code needs to find cpu_entry_area. */
713 DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
714 #endif
715 
716 /* Load the original GDT from the per-cpu structure */
load_direct_gdt(int cpu)717 void load_direct_gdt(int cpu)
718 {
719 	struct desc_ptr gdt_descr;
720 
721 	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
722 	gdt_descr.size = GDT_SIZE - 1;
723 	load_gdt(&gdt_descr);
724 }
725 EXPORT_SYMBOL_GPL(load_direct_gdt);
726 
727 /* Load a fixmap remapping of the per-cpu GDT */
load_fixmap_gdt(int cpu)728 void load_fixmap_gdt(int cpu)
729 {
730 	struct desc_ptr gdt_descr;
731 
732 	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
733 	gdt_descr.size = GDT_SIZE - 1;
734 	load_gdt(&gdt_descr);
735 }
736 EXPORT_SYMBOL_GPL(load_fixmap_gdt);
737 
738 /**
739  * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
740  * @cpu:	The CPU number for which this is invoked
741  *
742  * Invoked during early boot to switch from early GDT and early per CPU to
743  * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
744  * switch is implicit by loading the direct GDT. On 64bit this requires
745  * to update GSBASE.
746  */
switch_gdt_and_percpu_base(int cpu)747 void __init switch_gdt_and_percpu_base(int cpu)
748 {
749 	load_direct_gdt(cpu);
750 
751 #ifdef CONFIG_X86_64
752 	/*
753 	 * No need to load %gs. It is already correct.
754 	 *
755 	 * Writing %gs on 64bit would zero GSBASE which would make any per
756 	 * CPU operation up to the point of the wrmsrq() fault.
757 	 *
758 	 * Set GSBASE to the new offset. Until the wrmsrq() happens the
759 	 * early mapping is still valid. That means the GSBASE update will
760 	 * lose any prior per CPU data which was not copied over in
761 	 * setup_per_cpu_areas().
762 	 *
763 	 * This works even with stackprotector enabled because the
764 	 * per CPU stack canary is 0 in both per CPU areas.
765 	 */
766 	wrmsrq(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
767 #else
768 	/*
769 	 * %fs is already set to __KERNEL_PERCPU, but after switching GDT
770 	 * it is required to load FS again so that the 'hidden' part is
771 	 * updated from the new GDT. Up to this point the early per CPU
772 	 * translation is active. Any content of the early per CPU data
773 	 * which was not copied over in setup_per_cpu_areas() is lost.
774 	 */
775 	loadsegment(fs, __KERNEL_PERCPU);
776 #endif
777 }
778 
779 static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
780 
get_model_name(struct cpuinfo_x86 * c)781 static void get_model_name(struct cpuinfo_x86 *c)
782 {
783 	unsigned int *v;
784 	char *p, *q, *s;
785 
786 	if (c->extended_cpuid_level < 0x80000004)
787 		return;
788 
789 	v = (unsigned int *)c->x86_model_id;
790 	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
791 	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
792 	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
793 	c->x86_model_id[48] = 0;
794 
795 	/* Trim whitespace */
796 	p = q = s = &c->x86_model_id[0];
797 
798 	while (*p == ' ')
799 		p++;
800 
801 	while (*p) {
802 		/* Note the last non-whitespace index */
803 		if (!isspace(*p))
804 			s = q;
805 
806 		*q++ = *p++;
807 	}
808 
809 	*(s + 1) = '\0';
810 }
811 
cpu_detect_cache_sizes(struct cpuinfo_x86 * c)812 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
813 {
814 	unsigned int n, dummy, ebx, ecx, edx, l2size;
815 
816 	n = c->extended_cpuid_level;
817 
818 	if (n >= 0x80000005) {
819 		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
820 		c->x86_cache_size = (ecx>>24) + (edx>>24);
821 #ifdef CONFIG_X86_64
822 		/* On K8 L1 TLB is inclusive, so don't count it */
823 		c->x86_tlbsize = 0;
824 #endif
825 	}
826 
827 	if (n < 0x80000006)	/* Some chips just has a large L1. */
828 		return;
829 
830 	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
831 	l2size = ecx >> 16;
832 
833 #ifdef CONFIG_X86_64
834 	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
835 #else
836 	/* do processor-specific cache resizing */
837 	if (this_cpu->legacy_cache_size)
838 		l2size = this_cpu->legacy_cache_size(c, l2size);
839 
840 	/* Allow user to override all this if necessary. */
841 	if (cachesize_override != -1)
842 		l2size = cachesize_override;
843 
844 	if (l2size == 0)
845 		return;		/* Again, no L2 cache is possible */
846 #endif
847 
848 	c->x86_cache_size = l2size;
849 }
850 
851 u16 __read_mostly tlb_lli_4k;
852 u16 __read_mostly tlb_lli_2m;
853 u16 __read_mostly tlb_lli_4m;
854 u16 __read_mostly tlb_lld_4k;
855 u16 __read_mostly tlb_lld_2m;
856 u16 __read_mostly tlb_lld_4m;
857 u16 __read_mostly tlb_lld_1g;
858 
cpu_detect_tlb(struct cpuinfo_x86 * c)859 static void cpu_detect_tlb(struct cpuinfo_x86 *c)
860 {
861 	if (this_cpu->c_detect_tlb)
862 		this_cpu->c_detect_tlb(c);
863 
864 	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
865 		tlb_lli_4k, tlb_lli_2m, tlb_lli_4m);
866 
867 	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
868 		tlb_lld_4k, tlb_lld_2m, tlb_lld_4m, tlb_lld_1g);
869 }
870 
get_cpu_vendor(struct cpuinfo_x86 * c)871 void get_cpu_vendor(struct cpuinfo_x86 *c)
872 {
873 	char *v = c->x86_vendor_id;
874 	int i;
875 
876 	for (i = 0; i < X86_VENDOR_NUM; i++) {
877 		if (!cpu_devs[i])
878 			break;
879 
880 		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
881 		    (cpu_devs[i]->c_ident[1] &&
882 		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
883 
884 			this_cpu = cpu_devs[i];
885 			c->x86_vendor = this_cpu->c_x86_vendor;
886 			return;
887 		}
888 	}
889 
890 	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
891 		    "CPU: Your system may be unstable.\n", v);
892 
893 	c->x86_vendor = X86_VENDOR_UNKNOWN;
894 	this_cpu = &default_cpu;
895 }
896 
cpu_detect(struct cpuinfo_x86 * c)897 void cpu_detect(struct cpuinfo_x86 *c)
898 {
899 	/* Get vendor name */
900 	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
901 	      (unsigned int *)&c->x86_vendor_id[0],
902 	      (unsigned int *)&c->x86_vendor_id[8],
903 	      (unsigned int *)&c->x86_vendor_id[4]);
904 
905 	c->x86 = 4;
906 	/* Intel-defined flags: level 0x00000001 */
907 	if (c->cpuid_level >= 0x00000001) {
908 		u32 junk, tfms, cap0, misc;
909 
910 		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
911 		c->x86		= x86_family(tfms);
912 		c->x86_model	= x86_model(tfms);
913 		c->x86_stepping	= x86_stepping(tfms);
914 
915 		if (cap0 & (1<<19)) {
916 			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
917 			c->x86_cache_alignment = c->x86_clflush_size;
918 		}
919 	}
920 }
921 
apply_forced_caps(struct cpuinfo_x86 * c)922 static void apply_forced_caps(struct cpuinfo_x86 *c)
923 {
924 	int i;
925 
926 	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
927 		c->x86_capability[i] &= ~cpu_caps_cleared[i];
928 		c->x86_capability[i] |= cpu_caps_set[i];
929 	}
930 }
931 
init_speculation_control(struct cpuinfo_x86 * c)932 static void init_speculation_control(struct cpuinfo_x86 *c)
933 {
934 	/*
935 	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
936 	 * and they also have a different bit for STIBP support. Also,
937 	 * a hypervisor might have set the individual AMD bits even on
938 	 * Intel CPUs, for finer-grained selection of what's available.
939 	 */
940 	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
941 		set_cpu_cap(c, X86_FEATURE_IBRS);
942 		set_cpu_cap(c, X86_FEATURE_IBPB);
943 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
944 	}
945 
946 	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
947 		set_cpu_cap(c, X86_FEATURE_STIBP);
948 
949 	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
950 	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
951 		set_cpu_cap(c, X86_FEATURE_SSBD);
952 
953 	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
954 		set_cpu_cap(c, X86_FEATURE_IBRS);
955 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
956 	}
957 
958 	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
959 		set_cpu_cap(c, X86_FEATURE_IBPB);
960 
961 	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
962 		set_cpu_cap(c, X86_FEATURE_STIBP);
963 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
964 	}
965 
966 	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
967 		set_cpu_cap(c, X86_FEATURE_SSBD);
968 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
969 		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
970 	}
971 }
972 
get_cpu_cap(struct cpuinfo_x86 * c)973 void get_cpu_cap(struct cpuinfo_x86 *c)
974 {
975 	u32 eax, ebx, ecx, edx;
976 
977 	/* Intel-defined flags: level 0x00000001 */
978 	if (c->cpuid_level >= 0x00000001) {
979 		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
980 
981 		c->x86_capability[CPUID_1_ECX] = ecx;
982 		c->x86_capability[CPUID_1_EDX] = edx;
983 	}
984 
985 	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
986 	if (c->cpuid_level >= 0x00000006)
987 		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
988 
989 	/* Additional Intel-defined flags: level 0x00000007 */
990 	if (c->cpuid_level >= 0x00000007) {
991 		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
992 		c->x86_capability[CPUID_7_0_EBX] = ebx;
993 		c->x86_capability[CPUID_7_ECX] = ecx;
994 		c->x86_capability[CPUID_7_EDX] = edx;
995 
996 		/* Check valid sub-leaf index before accessing it */
997 		if (eax >= 1) {
998 			cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
999 			c->x86_capability[CPUID_7_1_EAX] = eax;
1000 		}
1001 	}
1002 
1003 	/* Extended state features: level 0x0000000d */
1004 	if (c->cpuid_level >= 0x0000000d) {
1005 		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1006 
1007 		c->x86_capability[CPUID_D_1_EAX] = eax;
1008 	}
1009 
1010 	/*
1011 	 * Check if extended CPUID leaves are implemented: Max extended
1012 	 * CPUID leaf must be in the 0x80000001-0x8000ffff range.
1013 	 */
1014 	eax = cpuid_eax(0x80000000);
1015 	c->extended_cpuid_level = ((eax & 0xffff0000) == 0x80000000) ? eax : 0;
1016 
1017 	if (c->extended_cpuid_level >= 0x80000001) {
1018 		cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1019 
1020 		c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1021 		c->x86_capability[CPUID_8000_0001_EDX] = edx;
1022 	}
1023 
1024 	if (c->extended_cpuid_level >= 0x80000007) {
1025 		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1026 
1027 		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1028 		c->x86_power = edx;
1029 	}
1030 
1031 	if (c->extended_cpuid_level >= 0x80000008) {
1032 		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1033 		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1034 	}
1035 
1036 	if (c->extended_cpuid_level >= 0x8000000a)
1037 		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1038 
1039 	if (c->extended_cpuid_level >= 0x8000001f)
1040 		c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1041 
1042 	if (c->extended_cpuid_level >= 0x80000021)
1043 		c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
1044 
1045 	init_scattered_cpuid_features(c);
1046 	init_speculation_control(c);
1047 
1048 	/*
1049 	 * Clear/Set all flags overridden by options, after probe.
1050 	 * This needs to happen each time we re-probe, which may happen
1051 	 * several times during CPU initialization.
1052 	 */
1053 	apply_forced_caps(c);
1054 }
1055 
get_cpu_address_sizes(struct cpuinfo_x86 * c)1056 void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1057 {
1058 	u32 eax, ebx, ecx, edx;
1059 
1060 	if (!cpu_has(c, X86_FEATURE_CPUID) ||
1061 	    (c->extended_cpuid_level < 0x80000008)) {
1062 		if (IS_ENABLED(CONFIG_X86_64)) {
1063 			c->x86_clflush_size = 64;
1064 			c->x86_phys_bits = 36;
1065 			c->x86_virt_bits = 48;
1066 		} else {
1067 			c->x86_clflush_size = 32;
1068 			c->x86_virt_bits = 32;
1069 			c->x86_phys_bits = 32;
1070 
1071 			if (cpu_has(c, X86_FEATURE_PAE) ||
1072 			    cpu_has(c, X86_FEATURE_PSE36))
1073 				c->x86_phys_bits = 36;
1074 		}
1075 	} else {
1076 		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1077 
1078 		c->x86_virt_bits = (eax >> 8) & 0xff;
1079 		c->x86_phys_bits = eax & 0xff;
1080 
1081 		/* Provide a sane default if not enumerated: */
1082 		if (!c->x86_clflush_size)
1083 			c->x86_clflush_size = 32;
1084 	}
1085 
1086 	c->x86_cache_bits = c->x86_phys_bits;
1087 	c->x86_cache_alignment = c->x86_clflush_size;
1088 }
1089 
identify_cpu_without_cpuid(struct cpuinfo_x86 * c)1090 static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1091 {
1092 	int i;
1093 
1094 	/*
1095 	 * First of all, decide if this is a 486 or higher
1096 	 * It's a 486 if we can modify the AC flag
1097 	 */
1098 	if (flag_is_changeable_p(X86_EFLAGS_AC))
1099 		c->x86 = 4;
1100 	else
1101 		c->x86 = 3;
1102 
1103 	for (i = 0; i < X86_VENDOR_NUM; i++)
1104 		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1105 			c->x86_vendor_id[0] = 0;
1106 			cpu_devs[i]->c_identify(c);
1107 			if (c->x86_vendor_id[0]) {
1108 				get_cpu_vendor(c);
1109 				break;
1110 			}
1111 		}
1112 }
1113 
1114 #define NO_SPECULATION		BIT(0)
1115 #define NO_MELTDOWN		BIT(1)
1116 #define NO_SSB			BIT(2)
1117 #define NO_L1TF			BIT(3)
1118 #define NO_MDS			BIT(4)
1119 #define MSBDS_ONLY		BIT(5)
1120 #define NO_SWAPGS		BIT(6)
1121 #define NO_ITLB_MULTIHIT	BIT(7)
1122 #define NO_SPECTRE_V2		BIT(8)
1123 #define NO_MMIO			BIT(9)
1124 #define NO_EIBRS_PBRSB		BIT(10)
1125 #define NO_BHI			BIT(11)
1126 
1127 #define VULNWL(vendor, family, model, whitelist)	\
1128 	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1129 
1130 #define VULNWL_INTEL(vfm, whitelist)		\
1131 	X86_MATCH_VFM(vfm, whitelist)
1132 
1133 #define VULNWL_AMD(family, whitelist)		\
1134 	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1135 
1136 #define VULNWL_HYGON(family, whitelist)		\
1137 	VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1138 
1139 static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1140 	VULNWL(ANY,	4, X86_MODEL_ANY,	NO_SPECULATION),
1141 	VULNWL(CENTAUR,	5, X86_MODEL_ANY,	NO_SPECULATION),
1142 	VULNWL(INTEL,	5, X86_MODEL_ANY,	NO_SPECULATION),
1143 	VULNWL(NSC,	5, X86_MODEL_ANY,	NO_SPECULATION),
1144 	VULNWL(VORTEX,	5, X86_MODEL_ANY,	NO_SPECULATION),
1145 	VULNWL(VORTEX,	6, X86_MODEL_ANY,	NO_SPECULATION),
1146 
1147 	/* Intel Family 6 */
1148 	VULNWL_INTEL(INTEL_TIGERLAKE,		NO_MMIO),
1149 	VULNWL_INTEL(INTEL_TIGERLAKE_L,		NO_MMIO),
1150 	VULNWL_INTEL(INTEL_ALDERLAKE,		NO_MMIO),
1151 	VULNWL_INTEL(INTEL_ALDERLAKE_L,		NO_MMIO),
1152 
1153 	VULNWL_INTEL(INTEL_ATOM_SALTWELL,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1154 	VULNWL_INTEL(INTEL_ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT),
1155 	VULNWL_INTEL(INTEL_ATOM_SALTWELL_MID,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1156 	VULNWL_INTEL(INTEL_ATOM_BONNELL,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1157 	VULNWL_INTEL(INTEL_ATOM_BONNELL_MID,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1158 
1159 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1160 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_D,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1161 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1162 	VULNWL_INTEL(INTEL_ATOM_AIRMONT,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1163 	VULNWL_INTEL(INTEL_XEON_PHI_KNL,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1164 	VULNWL_INTEL(INTEL_XEON_PHI_KNM,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1165 
1166 	VULNWL_INTEL(INTEL_CORE_YONAH,		NO_SSB),
1167 
1168 	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID2,NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | MSBDS_ONLY),
1169 	VULNWL_INTEL(INTEL_ATOM_AIRMONT_NP,	NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1170 
1171 	VULNWL_INTEL(INTEL_ATOM_GOLDMONT,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1172 	VULNWL_INTEL(INTEL_ATOM_GOLDMONT_D,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1173 	VULNWL_INTEL(INTEL_ATOM_GOLDMONT_PLUS,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1174 
1175 	/*
1176 	 * Technically, swapgs isn't serializing on AMD (despite it previously
1177 	 * being documented as such in the APM).  But according to AMD, %gs is
1178 	 * updated non-speculatively, and the issuing of %gs-relative memory
1179 	 * operands will be blocked until the %gs update completes, which is
1180 	 * good enough for our purposes.
1181 	 */
1182 
1183 	VULNWL_INTEL(INTEL_ATOM_TREMONT,	NO_EIBRS_PBRSB),
1184 	VULNWL_INTEL(INTEL_ATOM_TREMONT_L,	NO_EIBRS_PBRSB),
1185 	VULNWL_INTEL(INTEL_ATOM_TREMONT_D,	NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1186 
1187 	/* AMD Family 0xf - 0x12 */
1188 	VULNWL_AMD(0x0f,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1189 	VULNWL_AMD(0x10,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1190 	VULNWL_AMD(0x11,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1191 	VULNWL_AMD(0x12,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1192 
1193 	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1194 	VULNWL_AMD(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1195 	VULNWL_HYGON(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1196 
1197 	/* Zhaoxin Family 7 */
1198 	VULNWL(CENTAUR,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1199 	VULNWL(ZHAOXIN,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1200 	{}
1201 };
1202 
1203 #define VULNBL(vendor, family, model, blacklist)	\
1204 	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1205 
1206 #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues)		   \
1207 	X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues)
1208 
1209 #define VULNBL_INTEL_TYPE(vfm, cpu_type, issues)	\
1210 	X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues)
1211 
1212 #define VULNBL_AMD(family, blacklist)		\
1213 	VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1214 
1215 #define VULNBL_HYGON(family, blacklist)		\
1216 	VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1217 
1218 #define SRBDS		BIT(0)
1219 /* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1220 #define MMIO		BIT(1)
1221 /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1222 #define MMIO_SBDS	BIT(2)
1223 /* CPU is affected by RETbleed, speculating where you would not expect it */
1224 #define RETBLEED	BIT(3)
1225 /* CPU is affected by SMT (cross-thread) return predictions */
1226 #define SMT_RSB		BIT(4)
1227 /* CPU is affected by SRSO */
1228 #define SRSO		BIT(5)
1229 /* CPU is affected by GDS */
1230 #define GDS		BIT(6)
1231 /* CPU is affected by Register File Data Sampling */
1232 #define RFDS		BIT(7)
1233 /* CPU is affected by Indirect Target Selection */
1234 #define ITS		BIT(8)
1235 /* CPU is affected by Indirect Target Selection, but guest-host isolation is not affected */
1236 #define ITS_NATIVE_ONLY	BIT(9)
1237 /* CPU is affected by Transient Scheduler Attacks */
1238 #define TSA		BIT(10)
1239 /* CPU is affected by VMSCAPE */
1240 #define VMSCAPE		BIT(11)
1241 
1242 static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1243 	VULNBL_INTEL_STEPS(INTEL_SANDYBRIDGE_X,	     X86_STEP_MAX,	VMSCAPE),
1244 	VULNBL_INTEL_STEPS(INTEL_SANDYBRIDGE,	     X86_STEP_MAX,	VMSCAPE),
1245 	VULNBL_INTEL_STEPS(INTEL_IVYBRIDGE_X,	     X86_STEP_MAX,	VMSCAPE),
1246 	VULNBL_INTEL_STEPS(INTEL_IVYBRIDGE,	     X86_STEP_MAX,	SRBDS | VMSCAPE),
1247 	VULNBL_INTEL_STEPS(INTEL_HASWELL,	     X86_STEP_MAX,	SRBDS | VMSCAPE),
1248 	VULNBL_INTEL_STEPS(INTEL_HASWELL_L,	     X86_STEP_MAX,	SRBDS | VMSCAPE),
1249 	VULNBL_INTEL_STEPS(INTEL_HASWELL_G,	     X86_STEP_MAX,	SRBDS | VMSCAPE),
1250 	VULNBL_INTEL_STEPS(INTEL_HASWELL_X,	     X86_STEP_MAX,	MMIO | VMSCAPE),
1251 	VULNBL_INTEL_STEPS(INTEL_BROADWELL_D,	     X86_STEP_MAX,	MMIO | VMSCAPE),
1252 	VULNBL_INTEL_STEPS(INTEL_BROADWELL_X,	     X86_STEP_MAX,	MMIO | VMSCAPE),
1253 	VULNBL_INTEL_STEPS(INTEL_BROADWELL_G,	     X86_STEP_MAX,	SRBDS | VMSCAPE),
1254 	VULNBL_INTEL_STEPS(INTEL_BROADWELL,	     X86_STEP_MAX,	SRBDS | VMSCAPE),
1255 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X,		      0x5,	MMIO | RETBLEED | GDS | VMSCAPE),
1256 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | ITS | VMSCAPE),
1257 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_L,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
1258 	VULNBL_INTEL_STEPS(INTEL_SKYLAKE,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
1259 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L,		      0xb,	MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
1260 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS | ITS | VMSCAPE),
1261 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE,		      0xc,	MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
1262 	VULNBL_INTEL_STEPS(INTEL_KABYLAKE,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | SRBDS | ITS | VMSCAPE),
1263 	VULNBL_INTEL_STEPS(INTEL_CANNONLAKE_L,	     X86_STEP_MAX,	RETBLEED | VMSCAPE),
1264 	VULNBL_INTEL_STEPS(INTEL_ICELAKE_L,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
1265 	VULNBL_INTEL_STEPS(INTEL_ICELAKE_D,	     X86_STEP_MAX,	MMIO | GDS | ITS | ITS_NATIVE_ONLY),
1266 	VULNBL_INTEL_STEPS(INTEL_ICELAKE_X,	     X86_STEP_MAX,	MMIO | GDS | ITS | ITS_NATIVE_ONLY),
1267 	VULNBL_INTEL_STEPS(INTEL_COMETLAKE,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | VMSCAPE),
1268 	VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L,		      0x0,	MMIO | RETBLEED | ITS | VMSCAPE),
1269 	VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | VMSCAPE),
1270 	VULNBL_INTEL_STEPS(INTEL_TIGERLAKE_L,	     X86_STEP_MAX,	GDS | ITS | ITS_NATIVE_ONLY),
1271 	VULNBL_INTEL_STEPS(INTEL_TIGERLAKE,	     X86_STEP_MAX,	GDS | ITS | ITS_NATIVE_ONLY),
1272 	VULNBL_INTEL_STEPS(INTEL_LAKEFIELD,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RETBLEED),
1273 	VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE,	     X86_STEP_MAX,	MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
1274 	VULNBL_INTEL_TYPE(INTEL_ALDERLAKE,		     ATOM,	RFDS | VMSCAPE),
1275 	VULNBL_INTEL_STEPS(INTEL_ALDERLAKE,	     X86_STEP_MAX,	VMSCAPE),
1276 	VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L,	     X86_STEP_MAX,	RFDS | VMSCAPE),
1277 	VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE,		     ATOM,	RFDS | VMSCAPE),
1278 	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE,	     X86_STEP_MAX,	VMSCAPE),
1279 	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P,	     X86_STEP_MAX,	RFDS | VMSCAPE),
1280 	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S,	     X86_STEP_MAX,	RFDS | VMSCAPE),
1281 	VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L,	     X86_STEP_MAX,	VMSCAPE),
1282 	VULNBL_INTEL_STEPS(INTEL_ARROWLAKE_H,	     X86_STEP_MAX,	VMSCAPE),
1283 	VULNBL_INTEL_STEPS(INTEL_ARROWLAKE,	     X86_STEP_MAX,	VMSCAPE),
1284 	VULNBL_INTEL_STEPS(INTEL_ARROWLAKE_U,	     X86_STEP_MAX,	VMSCAPE),
1285 	VULNBL_INTEL_STEPS(INTEL_LUNARLAKE_M,	     X86_STEP_MAX,	VMSCAPE),
1286 	VULNBL_INTEL_STEPS(INTEL_SAPPHIRERAPIDS_X,   X86_STEP_MAX,	VMSCAPE),
1287 	VULNBL_INTEL_STEPS(INTEL_GRANITERAPIDS_X,    X86_STEP_MAX,	VMSCAPE),
1288 	VULNBL_INTEL_STEPS(INTEL_EMERALDRAPIDS_X,    X86_STEP_MAX,	VMSCAPE),
1289 	VULNBL_INTEL_STEPS(INTEL_ATOM_GRACEMONT,     X86_STEP_MAX,	RFDS | VMSCAPE),
1290 	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT,	     X86_STEP_MAX,	MMIO | MMIO_SBDS | RFDS),
1291 	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_D,     X86_STEP_MAX,	MMIO | RFDS),
1292 	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_L,     X86_STEP_MAX,	MMIO | MMIO_SBDS | RFDS),
1293 	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT,      X86_STEP_MAX,	RFDS),
1294 	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_D,    X86_STEP_MAX,	RFDS),
1295 	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_PLUS, X86_STEP_MAX,	RFDS),
1296 	VULNBL_INTEL_STEPS(INTEL_ATOM_CRESTMONT_X,   X86_STEP_MAX,	VMSCAPE),
1297 
1298 	VULNBL_AMD(0x15, RETBLEED),
1299 	VULNBL_AMD(0x16, RETBLEED),
1300 	VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO | VMSCAPE),
1301 	VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO | VMSCAPE),
1302 	VULNBL_AMD(0x19, SRSO | TSA | VMSCAPE),
1303 	VULNBL_AMD(0x1a, SRSO | VMSCAPE),
1304 	{}
1305 };
1306 
cpu_matches(const struct x86_cpu_id * table,unsigned long which)1307 static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1308 {
1309 	const struct x86_cpu_id *m = x86_match_cpu(table);
1310 
1311 	return m && !!(m->driver_data & which);
1312 }
1313 
x86_read_arch_cap_msr(void)1314 u64 x86_read_arch_cap_msr(void)
1315 {
1316 	u64 x86_arch_cap_msr = 0;
1317 
1318 	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1319 		rdmsrq(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr);
1320 
1321 	return x86_arch_cap_msr;
1322 }
1323 
arch_cap_mmio_immune(u64 x86_arch_cap_msr)1324 static bool arch_cap_mmio_immune(u64 x86_arch_cap_msr)
1325 {
1326 	return (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO &&
1327 		x86_arch_cap_msr & ARCH_CAP_PSDP_NO &&
1328 		x86_arch_cap_msr & ARCH_CAP_SBDR_SSDP_NO);
1329 }
1330 
vulnerable_to_rfds(u64 x86_arch_cap_msr)1331 static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr)
1332 {
1333 	/* The "immunity" bit trumps everything else: */
1334 	if (x86_arch_cap_msr & ARCH_CAP_RFDS_NO)
1335 		return false;
1336 
1337 	/*
1338 	 * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
1339 	 * indicate that mitigation is needed because guest is running on a
1340 	 * vulnerable hardware or may migrate to such hardware:
1341 	 */
1342 	if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
1343 		return true;
1344 
1345 	/* Only consult the blacklist when there is no enumeration: */
1346 	return cpu_matches(cpu_vuln_blacklist, RFDS);
1347 }
1348 
vulnerable_to_its(u64 x86_arch_cap_msr)1349 static bool __init vulnerable_to_its(u64 x86_arch_cap_msr)
1350 {
1351 	/* The "immunity" bit trumps everything else: */
1352 	if (x86_arch_cap_msr & ARCH_CAP_ITS_NO)
1353 		return false;
1354 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
1355 		return false;
1356 
1357 	/* None of the affected CPUs have BHI_CTRL */
1358 	if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
1359 		return false;
1360 
1361 	/*
1362 	 * If a VMM did not expose ITS_NO, assume that a guest could
1363 	 * be running on a vulnerable hardware or may migrate to such
1364 	 * hardware.
1365 	 */
1366 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1367 		return true;
1368 
1369 	if (cpu_matches(cpu_vuln_blacklist, ITS))
1370 		return true;
1371 
1372 	return false;
1373 }
1374 
1375 static struct x86_cpu_id cpu_latest_microcode[] = {
1376 #include "microcode/intel-ucode-defs.h"
1377 	{}
1378 };
1379 
cpu_has_old_microcode(void)1380 static bool __init cpu_has_old_microcode(void)
1381 {
1382 	const struct x86_cpu_id *m = x86_match_cpu(cpu_latest_microcode);
1383 
1384 	/* Give unknown CPUs a pass: */
1385 	if (!m) {
1386 		/* Intel CPUs should be in the list. Warn if not: */
1387 		if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1388 			pr_info("x86/CPU: Model not found in latest microcode list\n");
1389 		return false;
1390 	}
1391 
1392 	/*
1393 	 * Hosts usually lie to guests with a super high microcode
1394 	 * version. Just ignore what hosts tell guests:
1395 	 */
1396 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1397 		return false;
1398 
1399 	/* Consider all debug microcode to be old: */
1400 	if (boot_cpu_data.microcode & BIT(31))
1401 		return true;
1402 
1403 	/* Give new microcode a pass: */
1404 	if (boot_cpu_data.microcode >= m->driver_data)
1405 		return false;
1406 
1407 	/* Uh oh, too old: */
1408 	return true;
1409 }
1410 
cpu_set_bug_bits(struct cpuinfo_x86 * c)1411 static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1412 {
1413 	u64 x86_arch_cap_msr = x86_read_arch_cap_msr();
1414 
1415 	if (cpu_has_old_microcode()) {
1416 		pr_warn("x86/CPU: Running old microcode\n");
1417 		setup_force_cpu_bug(X86_BUG_OLD_MICROCODE);
1418 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1419 	}
1420 
1421 	/* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1422 	if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1423 	    !(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO))
1424 		setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1425 
1426 	if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1427 		return;
1428 
1429 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1430 
1431 	if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2)) {
1432 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1433 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER);
1434 	}
1435 
1436 	if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1437 	    !(x86_arch_cap_msr & ARCH_CAP_SSB_NO) &&
1438 	   !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1439 		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1440 
1441 	/*
1442 	 * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1443 	 * flag and protect from vendor-specific bugs via the whitelist.
1444 	 *
1445 	 * Don't use AutoIBRS when SNP is enabled because it degrades host
1446 	 * userspace indirect branch performance.
1447 	 */
1448 	if ((x86_arch_cap_msr & ARCH_CAP_IBRS_ALL) ||
1449 	    (cpu_has(c, X86_FEATURE_AUTOIBRS) &&
1450 	     !cpu_feature_enabled(X86_FEATURE_SEV_SNP))) {
1451 		setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1452 		if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1453 		    !(x86_arch_cap_msr & ARCH_CAP_PBRSB_NO))
1454 			setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1455 	}
1456 
1457 	if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1458 	    !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)) {
1459 		setup_force_cpu_bug(X86_BUG_MDS);
1460 		if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1461 			setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1462 	}
1463 
1464 	if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1465 		setup_force_cpu_bug(X86_BUG_SWAPGS);
1466 
1467 	/*
1468 	 * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1469 	 *	- TSX is supported or
1470 	 *	- TSX_CTRL is present
1471 	 *
1472 	 * TSX_CTRL check is needed for cases when TSX could be disabled before
1473 	 * the kernel boot e.g. kexec.
1474 	 * TSX_CTRL check alone is not sufficient for cases when the microcode
1475 	 * update is not present or running as guest that don't get TSX_CTRL.
1476 	 */
1477 	if (!(x86_arch_cap_msr & ARCH_CAP_TAA_NO) &&
1478 	    (cpu_has(c, X86_FEATURE_RTM) ||
1479 	     (x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)))
1480 		setup_force_cpu_bug(X86_BUG_TAA);
1481 
1482 	/*
1483 	 * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1484 	 * in the vulnerability blacklist.
1485 	 *
1486 	 * Some of the implications and mitigation of Shared Buffers Data
1487 	 * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1488 	 * SRBDS.
1489 	 */
1490 	if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1491 	     cpu_has(c, X86_FEATURE_RDSEED)) &&
1492 	    cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1493 		    setup_force_cpu_bug(X86_BUG_SRBDS);
1494 
1495 	/*
1496 	 * Processor MMIO Stale Data bug enumeration
1497 	 *
1498 	 * Affected CPU list is generally enough to enumerate the vulnerability,
1499 	 * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1500 	 * not want the guest to enumerate the bug.
1501 	 */
1502 	if (!arch_cap_mmio_immune(x86_arch_cap_msr)) {
1503 		if (cpu_matches(cpu_vuln_blacklist, MMIO))
1504 			setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1505 	}
1506 
1507 	if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1508 		if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (x86_arch_cap_msr & ARCH_CAP_RSBA))
1509 			setup_force_cpu_bug(X86_BUG_RETBLEED);
1510 	}
1511 
1512 	if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
1513 		setup_force_cpu_bug(X86_BUG_SMT_RSB);
1514 
1515 	if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1516 		if (cpu_matches(cpu_vuln_blacklist, SRSO))
1517 			setup_force_cpu_bug(X86_BUG_SRSO);
1518 	}
1519 
1520 	/*
1521 	 * Check if CPU is vulnerable to GDS. If running in a virtual machine on
1522 	 * an affected processor, the VMM may have disabled the use of GATHER by
1523 	 * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1524 	 * which means that AVX will be disabled.
1525 	 */
1526 	if (cpu_matches(cpu_vuln_blacklist, GDS) && !(x86_arch_cap_msr & ARCH_CAP_GDS_NO) &&
1527 	    boot_cpu_has(X86_FEATURE_AVX))
1528 		setup_force_cpu_bug(X86_BUG_GDS);
1529 
1530 	if (vulnerable_to_rfds(x86_arch_cap_msr))
1531 		setup_force_cpu_bug(X86_BUG_RFDS);
1532 
1533 	/*
1534 	 * Intel parts with eIBRS are vulnerable to BHI attacks. Parts with
1535 	 * BHI_NO still need to use the BHI mitigation to prevent Intra-mode
1536 	 * attacks.  When virtualized, eIBRS could be hidden, assume vulnerable.
1537 	 */
1538 	if (!cpu_matches(cpu_vuln_whitelist, NO_BHI) &&
1539 	    (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) ||
1540 	     boot_cpu_has(X86_FEATURE_HYPERVISOR)))
1541 		setup_force_cpu_bug(X86_BUG_BHI);
1542 
1543 	if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET))
1544 		setup_force_cpu_bug(X86_BUG_IBPB_NO_RET);
1545 
1546 	if (vulnerable_to_its(x86_arch_cap_msr)) {
1547 		setup_force_cpu_bug(X86_BUG_ITS);
1548 		if (cpu_matches(cpu_vuln_blacklist, ITS_NATIVE_ONLY))
1549 			setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY);
1550 	}
1551 
1552 	if (c->x86_vendor == X86_VENDOR_AMD) {
1553 		if (!cpu_has(c, X86_FEATURE_TSA_SQ_NO) ||
1554 		    !cpu_has(c, X86_FEATURE_TSA_L1_NO)) {
1555 			if (cpu_matches(cpu_vuln_blacklist, TSA) ||
1556 			    /* Enable bug on Zen guests to allow for live migration. */
1557 			    (cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_ZEN)))
1558 				setup_force_cpu_bug(X86_BUG_TSA);
1559 		}
1560 	}
1561 
1562 	/*
1563 	 * Set the bug only on bare-metal. A nested hypervisor should already be
1564 	 * deploying IBPB to isolate itself from nested guests.
1565 	 */
1566 	if (cpu_matches(cpu_vuln_blacklist, VMSCAPE) &&
1567 	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
1568 		setup_force_cpu_bug(X86_BUG_VMSCAPE);
1569 
1570 	if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1571 		return;
1572 
1573 	/* Rogue Data Cache Load? No! */
1574 	if (x86_arch_cap_msr & ARCH_CAP_RDCL_NO)
1575 		return;
1576 
1577 	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1578 
1579 	if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1580 		return;
1581 
1582 	setup_force_cpu_bug(X86_BUG_L1TF);
1583 }
1584 
1585 /*
1586  * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1587  * unfortunately, that's not true in practice because of early VIA
1588  * chips and (more importantly) broken virtualizers that are not easy
1589  * to detect. In the latter case it doesn't even *fail* reliably, so
1590  * probing for it doesn't even work. Disable it completely on 32-bit
1591  * unless we can find a reliable way to detect all the broken cases.
1592  * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1593  */
detect_nopl(void)1594 static void detect_nopl(void)
1595 {
1596 #ifdef CONFIG_X86_32
1597 	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1598 #else
1599 	setup_force_cpu_cap(X86_FEATURE_NOPL);
1600 #endif
1601 }
1602 
parse_set_clear_cpuid(char * arg,bool set)1603 static inline bool parse_set_clear_cpuid(char *arg, bool set)
1604 {
1605 	char *opt;
1606 	int taint = 0;
1607 
1608 	while (arg) {
1609 		bool found __maybe_unused = false;
1610 		unsigned int bit;
1611 
1612 		opt = strsep(&arg, ",");
1613 
1614 		/*
1615 		 * Handle naked numbers first for feature flags which don't
1616 		 * have names. It doesn't make sense for a bug not to have a
1617 		 * name so don't handle bug flags here.
1618 		 */
1619 		if (!kstrtouint(opt, 10, &bit)) {
1620 			if (bit < NCAPINTS * 32) {
1621 
1622 				if (set) {
1623 					pr_warn("setcpuid: force-enabling CPU feature flag:");
1624 					setup_force_cpu_cap(bit);
1625 				} else {
1626 					pr_warn("clearcpuid: force-disabling CPU feature flag:");
1627 					setup_clear_cpu_cap(bit);
1628 				}
1629 				/* empty-string, i.e., ""-defined feature flags */
1630 				if (!x86_cap_flags[bit])
1631 					pr_cont(" %d:%d\n", bit >> 5, bit & 31);
1632 				else
1633 					pr_cont(" %s\n", x86_cap_flags[bit]);
1634 
1635 				taint++;
1636 			}
1637 			/*
1638 			 * The assumption is that there are no feature names with only
1639 			 * numbers in the name thus go to the next argument.
1640 			 */
1641 			continue;
1642 		}
1643 
1644 		for (bit = 0; bit < 32 * (NCAPINTS + NBUGINTS); bit++) {
1645 			const char *flag;
1646 			const char *kind;
1647 
1648 			if (bit < 32 * NCAPINTS) {
1649 				flag = x86_cap_flags[bit];
1650 				kind = "feature";
1651 			} else {
1652 				kind = "bug";
1653 				flag = x86_bug_flags[bit - (32 * NCAPINTS)];
1654 			}
1655 
1656 			if (!flag)
1657 				continue;
1658 
1659 			if (strcmp(flag, opt))
1660 				continue;
1661 
1662 			if (set) {
1663 				pr_warn("setcpuid: force-enabling CPU %s flag: %s\n",
1664 					kind, flag);
1665 				setup_force_cpu_cap(bit);
1666 			} else {
1667 				pr_warn("clearcpuid: force-disabling CPU %s flag: %s\n",
1668 					kind, flag);
1669 				setup_clear_cpu_cap(bit);
1670 			}
1671 			taint++;
1672 			found = true;
1673 			break;
1674 		}
1675 
1676 		if (!found)
1677 			pr_warn("%s: unknown CPU flag: %s", set ? "setcpuid" : "clearcpuid", opt);
1678 	}
1679 
1680 	return taint;
1681 }
1682 
1683 
1684 /*
1685  * We parse cpu parameters early because fpu__init_system() is executed
1686  * before parse_early_param().
1687  */
cpu_parse_early_param(void)1688 static void __init cpu_parse_early_param(void)
1689 {
1690 	bool cpuid_taint = false;
1691 	char arg[128];
1692 	int arglen;
1693 
1694 #ifdef CONFIG_X86_32
1695 	if (cmdline_find_option_bool(boot_command_line, "no387"))
1696 #ifdef CONFIG_MATH_EMULATION
1697 		setup_clear_cpu_cap(X86_FEATURE_FPU);
1698 #else
1699 		pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1700 #endif
1701 
1702 	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1703 		setup_clear_cpu_cap(X86_FEATURE_FXSR);
1704 #endif
1705 
1706 	if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1707 		setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1708 
1709 	if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1710 		setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1711 
1712 	if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1713 		setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1714 
1715 	if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1716 		setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1717 
1718 	/* Minimize the gap between FRED is available and available but disabled. */
1719 	arglen = cmdline_find_option(boot_command_line, "fred", arg, sizeof(arg));
1720 	if (arglen != 2 || strncmp(arg, "on", 2))
1721 		setup_clear_cpu_cap(X86_FEATURE_FRED);
1722 
1723 	arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1724 	if (arglen > 0)
1725 		cpuid_taint |= parse_set_clear_cpuid(arg, false);
1726 
1727 	arglen = cmdline_find_option(boot_command_line, "setcpuid", arg, sizeof(arg));
1728 	if (arglen > 0)
1729 		cpuid_taint |= parse_set_clear_cpuid(arg, true);
1730 
1731 	if (cpuid_taint) {
1732 		pr_warn("!!! setcpuid=/clearcpuid= in use, this is for TESTING ONLY, may break things horribly. Tainting kernel.\n");
1733 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1734 	}
1735 }
1736 
1737 /*
1738  * Do minimum CPU detection early.
1739  * Fields really needed: vendor, cpuid_level, family, model, mask,
1740  * cache alignment.
1741  * The others are not touched to avoid unwanted side effects.
1742  *
1743  * WARNING: this function is only called on the boot CPU.  Don't add code
1744  * here that is supposed to run on all CPUs.
1745  */
early_identify_cpu(struct cpuinfo_x86 * c)1746 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1747 {
1748 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1749 	c->extended_cpuid_level = 0;
1750 
1751 	if (!cpuid_feature())
1752 		identify_cpu_without_cpuid(c);
1753 
1754 	/* cyrix could have cpuid enabled via c_identify()*/
1755 	if (cpuid_feature()) {
1756 		cpu_detect(c);
1757 		get_cpu_vendor(c);
1758 		intel_unlock_cpuid_leafs(c);
1759 		get_cpu_cap(c);
1760 		setup_force_cpu_cap(X86_FEATURE_CPUID);
1761 		get_cpu_address_sizes(c);
1762 		cpu_parse_early_param();
1763 
1764 		cpu_init_topology(c);
1765 
1766 		if (this_cpu->c_early_init)
1767 			this_cpu->c_early_init(c);
1768 
1769 		c->cpu_index = 0;
1770 		filter_cpuid_features(c, false);
1771 		check_cpufeature_deps(c);
1772 
1773 		if (this_cpu->c_bsp_init)
1774 			this_cpu->c_bsp_init(c);
1775 	} else {
1776 		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1777 		get_cpu_address_sizes(c);
1778 		cpu_init_topology(c);
1779 	}
1780 
1781 	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1782 
1783 	cpu_set_bug_bits(c);
1784 
1785 	sld_setup(c);
1786 
1787 #ifdef CONFIG_X86_32
1788 	/*
1789 	 * Regardless of whether PCID is enumerated, the SDM says
1790 	 * that it can't be enabled in 32-bit mode.
1791 	 */
1792 	setup_clear_cpu_cap(X86_FEATURE_PCID);
1793 #endif
1794 
1795 	/*
1796 	 * Later in the boot process pgtable_l5_enabled() relies on
1797 	 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1798 	 * enabled by this point we need to clear the feature bit to avoid
1799 	 * false-positives at the later stage.
1800 	 *
1801 	 * pgtable_l5_enabled() can be false here for several reasons:
1802 	 *  - 5-level paging is disabled compile-time;
1803 	 *  - it's 32-bit kernel;
1804 	 *  - machine doesn't support 5-level paging;
1805 	 *  - user specified 'no5lvl' in kernel command line.
1806 	 */
1807 	if (!pgtable_l5_enabled())
1808 		setup_clear_cpu_cap(X86_FEATURE_LA57);
1809 
1810 	detect_nopl();
1811 }
1812 
init_cpu_devs(void)1813 void __init init_cpu_devs(void)
1814 {
1815 	const struct cpu_dev *const *cdev;
1816 	int count = 0;
1817 
1818 	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1819 		const struct cpu_dev *cpudev = *cdev;
1820 
1821 		if (count >= X86_VENDOR_NUM)
1822 			break;
1823 		cpu_devs[count] = cpudev;
1824 		count++;
1825 	}
1826 }
1827 
early_cpu_init(void)1828 void __init early_cpu_init(void)
1829 {
1830 #ifdef CONFIG_PROCESSOR_SELECT
1831 	unsigned int i, j;
1832 
1833 	pr_info("KERNEL supported cpus:\n");
1834 #endif
1835 
1836 	init_cpu_devs();
1837 
1838 #ifdef CONFIG_PROCESSOR_SELECT
1839 	for (i = 0; i < X86_VENDOR_NUM && cpu_devs[i]; i++) {
1840 		for (j = 0; j < 2; j++) {
1841 			if (!cpu_devs[i]->c_ident[j])
1842 				continue;
1843 			pr_info("  %s %s\n", cpu_devs[i]->c_vendor,
1844 				cpu_devs[i]->c_ident[j]);
1845 		}
1846 	}
1847 #endif
1848 
1849 	early_identify_cpu(&boot_cpu_data);
1850 }
1851 
detect_null_seg_behavior(void)1852 static bool detect_null_seg_behavior(void)
1853 {
1854 	/*
1855 	 * Empirically, writing zero to a segment selector on AMD does
1856 	 * not clear the base, whereas writing zero to a segment
1857 	 * selector on Intel does clear the base.  Intel's behavior
1858 	 * allows slightly faster context switches in the common case
1859 	 * where GS is unused by the prev and next threads.
1860 	 *
1861 	 * Since neither vendor documents this anywhere that I can see,
1862 	 * detect it directly instead of hard-coding the choice by
1863 	 * vendor.
1864 	 *
1865 	 * I've designated AMD's behavior as the "bug" because it's
1866 	 * counterintuitive and less friendly.
1867 	 */
1868 
1869 	unsigned long old_base, tmp;
1870 	rdmsrq(MSR_FS_BASE, old_base);
1871 	wrmsrq(MSR_FS_BASE, 1);
1872 	loadsegment(fs, 0);
1873 	rdmsrq(MSR_FS_BASE, tmp);
1874 	wrmsrq(MSR_FS_BASE, old_base);
1875 	return tmp == 0;
1876 }
1877 
check_null_seg_clears_base(struct cpuinfo_x86 * c)1878 void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1879 {
1880 	/* BUG_NULL_SEG is only relevant with 64bit userspace */
1881 	if (!IS_ENABLED(CONFIG_X86_64))
1882 		return;
1883 
1884 	if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
1885 		return;
1886 
1887 	/*
1888 	 * CPUID bit above wasn't set. If this kernel is still running
1889 	 * as a HV guest, then the HV has decided not to advertize
1890 	 * that CPUID bit for whatever reason.	For example, one
1891 	 * member of the migration pool might be vulnerable.  Which
1892 	 * means, the bug is present: set the BUG flag and return.
1893 	 */
1894 	if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1895 		set_cpu_bug(c, X86_BUG_NULL_SEG);
1896 		return;
1897 	}
1898 
1899 	/*
1900 	 * Zen2 CPUs also have this behaviour, but no CPUID bit.
1901 	 * 0x18 is the respective family for Hygon.
1902 	 */
1903 	if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1904 	    detect_null_seg_behavior())
1905 		return;
1906 
1907 	/* All the remaining ones are affected */
1908 	set_cpu_bug(c, X86_BUG_NULL_SEG);
1909 }
1910 
generic_identify(struct cpuinfo_x86 * c)1911 static void generic_identify(struct cpuinfo_x86 *c)
1912 {
1913 	c->extended_cpuid_level = 0;
1914 
1915 	if (!cpuid_feature())
1916 		identify_cpu_without_cpuid(c);
1917 
1918 	/* cyrix could have cpuid enabled via c_identify()*/
1919 	if (!cpuid_feature())
1920 		return;
1921 
1922 	cpu_detect(c);
1923 
1924 	get_cpu_vendor(c);
1925 	intel_unlock_cpuid_leafs(c);
1926 	get_cpu_cap(c);
1927 
1928 	get_cpu_address_sizes(c);
1929 
1930 	get_model_name(c); /* Default name */
1931 
1932 	/*
1933 	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1934 	 * systems that run Linux at CPL > 0 may or may not have the
1935 	 * issue, but, even if they have the issue, there's absolutely
1936 	 * nothing we can do about it because we can't use the real IRET
1937 	 * instruction.
1938 	 *
1939 	 * NB: For the time being, only 32-bit kernels support
1940 	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1941 	 * whether to apply espfix using paravirt hooks.  If any
1942 	 * non-paravirt system ever shows up that does *not* have the
1943 	 * ESPFIX issue, we can change this.
1944 	 */
1945 #ifdef CONFIG_X86_32
1946 	set_cpu_bug(c, X86_BUG_ESPFIX);
1947 #endif
1948 }
1949 
1950 /*
1951  * This does the hard work of actually picking apart the CPU stuff...
1952  */
identify_cpu(struct cpuinfo_x86 * c)1953 static void identify_cpu(struct cpuinfo_x86 *c)
1954 {
1955 	int i;
1956 
1957 	c->loops_per_jiffy = loops_per_jiffy;
1958 	c->x86_cache_size = 0;
1959 	c->x86_vendor = X86_VENDOR_UNKNOWN;
1960 	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
1961 	c->x86_vendor_id[0] = '\0'; /* Unset */
1962 	c->x86_model_id[0] = '\0';  /* Unset */
1963 #ifdef CONFIG_X86_64
1964 	c->x86_clflush_size = 64;
1965 	c->x86_phys_bits = 36;
1966 	c->x86_virt_bits = 48;
1967 #else
1968 	c->cpuid_level = -1;	/* CPUID not detected */
1969 	c->x86_clflush_size = 32;
1970 	c->x86_phys_bits = 32;
1971 	c->x86_virt_bits = 32;
1972 #endif
1973 	c->x86_cache_alignment = c->x86_clflush_size;
1974 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1975 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
1976 	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1977 #endif
1978 
1979 	generic_identify(c);
1980 
1981 	cpu_parse_topology(c);
1982 
1983 	if (this_cpu->c_identify)
1984 		this_cpu->c_identify(c);
1985 
1986 	/* Clear/Set all flags overridden by options, after probe */
1987 	apply_forced_caps(c);
1988 
1989 	/*
1990 	 * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
1991 	 * Hygon will clear it in ->c_init() below.
1992 	 */
1993 	set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1994 
1995 	/*
1996 	 * Vendor-specific initialization.  In this section we
1997 	 * canonicalize the feature flags, meaning if there are
1998 	 * features a certain CPU supports which CPUID doesn't
1999 	 * tell us, CPUID claiming incorrect flags, or other bugs,
2000 	 * we handle them here.
2001 	 *
2002 	 * At the end of this section, c->x86_capability better
2003 	 * indicate the features this CPU genuinely supports!
2004 	 */
2005 	if (this_cpu->c_init)
2006 		this_cpu->c_init(c);
2007 
2008 	bus_lock_init();
2009 
2010 	/* Disable the PN if appropriate */
2011 	squash_the_stupid_serial_number(c);
2012 
2013 	/* Set up SMEP/SMAP/UMIP */
2014 	setup_smep(c);
2015 	setup_smap(c);
2016 	setup_umip(c);
2017 
2018 	/* Enable FSGSBASE instructions if available. */
2019 	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
2020 		cr4_set_bits(X86_CR4_FSGSBASE);
2021 		elf_hwcap2 |= HWCAP2_FSGSBASE;
2022 	}
2023 
2024 	/*
2025 	 * The vendor-specific functions might have changed features.
2026 	 * Now we do "generic changes."
2027 	 */
2028 
2029 	/* Filter out anything that depends on CPUID levels we don't have */
2030 	filter_cpuid_features(c, true);
2031 
2032 	/* Check for unmet dependencies based on the CPUID dependency table */
2033 	check_cpufeature_deps(c);
2034 
2035 	/* If the model name is still unset, do table lookup. */
2036 	if (!c->x86_model_id[0]) {
2037 		const char *p;
2038 		p = table_lookup_model(c);
2039 		if (p)
2040 			strcpy(c->x86_model_id, p);
2041 		else
2042 			/* Last resort... */
2043 			sprintf(c->x86_model_id, "%02x/%02x",
2044 				c->x86, c->x86_model);
2045 	}
2046 
2047 	x86_init_rdrand(c);
2048 	setup_pku(c);
2049 	setup_cet(c);
2050 
2051 	/*
2052 	 * Clear/Set all flags overridden by options, need do it
2053 	 * before following smp all cpus cap AND.
2054 	 */
2055 	apply_forced_caps(c);
2056 
2057 	/*
2058 	 * On SMP, boot_cpu_data holds the common feature set between
2059 	 * all CPUs; so make sure that we indicate which features are
2060 	 * common between the CPUs.  The first time this routine gets
2061 	 * executed, c == &boot_cpu_data.
2062 	 */
2063 	if (c != &boot_cpu_data) {
2064 		/* AND the already accumulated flags with these */
2065 		for (i = 0; i < NCAPINTS; i++)
2066 			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
2067 
2068 		/* OR, i.e. replicate the bug flags */
2069 		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
2070 			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
2071 	}
2072 
2073 	ppin_init(c);
2074 
2075 	/* Init Machine Check Exception if available. */
2076 	mcheck_cpu_init(c);
2077 
2078 	numa_add_cpu(smp_processor_id());
2079 }
2080 
2081 /*
2082  * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
2083  * on 32-bit kernels:
2084  */
2085 #ifdef CONFIG_X86_32
enable_sep_cpu(void)2086 void enable_sep_cpu(void)
2087 {
2088 	struct tss_struct *tss;
2089 	int cpu;
2090 
2091 	if (!boot_cpu_has(X86_FEATURE_SEP))
2092 		return;
2093 
2094 	cpu = get_cpu();
2095 	tss = &per_cpu(cpu_tss_rw, cpu);
2096 
2097 	/*
2098 	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
2099 	 * see the big comment in struct x86_hw_tss's definition.
2100 	 */
2101 
2102 	tss->x86_tss.ss1 = __KERNEL_CS;
2103 	wrmsrq(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1);
2104 	wrmsrq(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1));
2105 	wrmsrq(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32);
2106 
2107 	put_cpu();
2108 }
2109 #endif
2110 
identify_boot_cpu(void)2111 static __init void identify_boot_cpu(void)
2112 {
2113 	identify_cpu(&boot_cpu_data);
2114 	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
2115 		pr_info("CET detected: Indirect Branch Tracking enabled\n");
2116 #ifdef CONFIG_X86_32
2117 	enable_sep_cpu();
2118 #endif
2119 	cpu_detect_tlb(&boot_cpu_data);
2120 	setup_cr_pinning();
2121 
2122 	tsx_init();
2123 	tdx_init();
2124 	lkgs_init();
2125 }
2126 
identify_secondary_cpu(unsigned int cpu)2127 void identify_secondary_cpu(unsigned int cpu)
2128 {
2129 	struct cpuinfo_x86 *c = &cpu_data(cpu);
2130 
2131 	/* Copy boot_cpu_data only on the first bringup */
2132 	if (!c->initialized)
2133 		*c = boot_cpu_data;
2134 	c->cpu_index = cpu;
2135 
2136 	identify_cpu(c);
2137 #ifdef CONFIG_X86_32
2138 	enable_sep_cpu();
2139 #endif
2140 	x86_spec_ctrl_setup_ap();
2141 	update_srbds_msr();
2142 	if (boot_cpu_has_bug(X86_BUG_GDS))
2143 		update_gds_msr();
2144 
2145 	tsx_ap_init();
2146 	c->initialized = true;
2147 }
2148 
print_cpu_info(struct cpuinfo_x86 * c)2149 void print_cpu_info(struct cpuinfo_x86 *c)
2150 {
2151 	const char *vendor = NULL;
2152 
2153 	if (c->x86_vendor < X86_VENDOR_NUM) {
2154 		vendor = this_cpu->c_vendor;
2155 	} else {
2156 		if (c->cpuid_level >= 0)
2157 			vendor = c->x86_vendor_id;
2158 	}
2159 
2160 	if (vendor && !strstr(c->x86_model_id, vendor))
2161 		pr_cont("%s ", vendor);
2162 
2163 	if (c->x86_model_id[0])
2164 		pr_cont("%s", c->x86_model_id);
2165 	else
2166 		pr_cont("%d86", c->x86);
2167 
2168 	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
2169 
2170 	if (c->x86_stepping || c->cpuid_level >= 0)
2171 		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
2172 	else
2173 		pr_cont(")\n");
2174 }
2175 
2176 /*
2177  * clearcpuid= and setcpuid= were already parsed in cpu_parse_early_param().
2178  * These dummy functions prevent them from becoming an environment variable for
2179  * init.
2180  */
2181 
setup_clearcpuid(char * arg)2182 static __init int setup_clearcpuid(char *arg)
2183 {
2184 	return 1;
2185 }
2186 __setup("clearcpuid=", setup_clearcpuid);
2187 
setup_setcpuid(char * arg)2188 static __init int setup_setcpuid(char *arg)
2189 {
2190 	return 1;
2191 }
2192 __setup("setcpuid=", setup_setcpuid);
2193 
2194 DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
2195 EXPORT_PER_CPU_SYMBOL(current_task);
2196 EXPORT_PER_CPU_SYMBOL(const_current_task);
2197 
2198 DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
2199 EXPORT_PER_CPU_SYMBOL(__preempt_count);
2200 
2201 DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2202 
2203 #ifdef CONFIG_X86_64
2204 /*
2205  * Note: Do not make this dependant on CONFIG_MITIGATION_CALL_DEPTH_TRACKING
2206  * so that this space is reserved in the hot cache section even when the
2207  * mitigation is disabled.
2208  */
2209 DEFINE_PER_CPU_CACHE_HOT(u64, __x86_call_depth);
2210 EXPORT_PER_CPU_SYMBOL(__x86_call_depth);
2211 
wrmsrq_cstar(unsigned long val)2212 static void wrmsrq_cstar(unsigned long val)
2213 {
2214 	/*
2215 	 * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2216 	 * is so far ignored by the CPU, but raises a #VE trap in a TDX
2217 	 * guest. Avoid the pointless write on all Intel CPUs.
2218 	 */
2219 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2220 		wrmsrq(MSR_CSTAR, val);
2221 }
2222 
idt_syscall_init(void)2223 static inline void idt_syscall_init(void)
2224 {
2225 	wrmsrq(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2226 
2227 	if (ia32_enabled()) {
2228 		wrmsrq_cstar((unsigned long)entry_SYSCALL_compat);
2229 		/*
2230 		 * This only works on Intel CPUs.
2231 		 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2232 		 * This does not cause SYSENTER to jump to the wrong location, because
2233 		 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2234 		 */
2235 		wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2236 		wrmsrq_safe(MSR_IA32_SYSENTER_ESP,
2237 			    (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2238 		wrmsrq_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2239 	} else {
2240 		wrmsrq_cstar((unsigned long)entry_SYSCALL32_ignore);
2241 		wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2242 		wrmsrq_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2243 		wrmsrq_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2244 	}
2245 
2246 	/*
2247 	 * Flags to clear on syscall; clear as much as possible
2248 	 * to minimize user space-kernel interference.
2249 	 */
2250 	wrmsrq(MSR_SYSCALL_MASK,
2251 	       X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2252 	       X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2253 	       X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2254 	       X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2255 	       X86_EFLAGS_AC|X86_EFLAGS_ID);
2256 }
2257 
2258 /* May not be marked __init: used by software suspend */
syscall_init(void)2259 void syscall_init(void)
2260 {
2261 	/* The default user and kernel segments */
2262 	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2263 
2264 	/*
2265 	 * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
2266 	 * SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED
2267 	 * entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit
2268 	 * instruction to return to ring 3 (both sysexit and sysret cause
2269 	 * #UD when FRED is enabled).
2270 	 */
2271 	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2272 		idt_syscall_init();
2273 }
2274 #endif /* CONFIG_X86_64 */
2275 
2276 #ifdef CONFIG_STACKPROTECTOR
2277 DEFINE_PER_CPU_CACHE_HOT(unsigned long, __stack_chk_guard);
2278 #ifndef CONFIG_SMP
2279 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2280 #endif
2281 #endif
2282 
initialize_debug_regs(void)2283 static void initialize_debug_regs(void)
2284 {
2285 	/* Control register first -- to make sure everything is disabled. */
2286 	set_debugreg(DR7_FIXED_1, 7);
2287 	set_debugreg(DR6_RESERVED, 6);
2288 	/* dr5 and dr4 don't exist */
2289 	set_debugreg(0, 3);
2290 	set_debugreg(0, 2);
2291 	set_debugreg(0, 1);
2292 	set_debugreg(0, 0);
2293 }
2294 
2295 #ifdef CONFIG_KGDB
2296 /*
2297  * Restore debug regs if using kgdbwait and you have a kernel debugger
2298  * connection established.
2299  */
dbg_restore_debug_regs(void)2300 static void dbg_restore_debug_regs(void)
2301 {
2302 	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2303 		arch_kgdb_ops.correct_hw_break();
2304 }
2305 #else /* ! CONFIG_KGDB */
2306 #define dbg_restore_debug_regs()
2307 #endif /* ! CONFIG_KGDB */
2308 
setup_getcpu(int cpu)2309 static inline void setup_getcpu(int cpu)
2310 {
2311 	unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2312 	struct desc_struct d = { };
2313 
2314 	if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2315 		wrmsrq(MSR_TSC_AUX, cpudata);
2316 
2317 	/* Store CPU and node number in limit. */
2318 	d.limit0 = cpudata;
2319 	d.limit1 = cpudata >> 16;
2320 
2321 	d.type = 5;		/* RO data, expand down, accessed */
2322 	d.dpl = 3;		/* Visible to user code */
2323 	d.s = 1;		/* Not a system segment */
2324 	d.p = 1;		/* Present */
2325 	d.d = 1;		/* 32-bit */
2326 
2327 	write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2328 }
2329 
2330 #ifdef CONFIG_X86_64
tss_setup_ist(struct tss_struct * tss)2331 static inline void tss_setup_ist(struct tss_struct *tss)
2332 {
2333 	/* Set up the per-CPU TSS IST stacks */
2334 	tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2335 	tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2336 	tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2337 	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2338 	/* Only mapped when SEV-ES is active */
2339 	tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2340 }
2341 #else /* CONFIG_X86_64 */
tss_setup_ist(struct tss_struct * tss)2342 static inline void tss_setup_ist(struct tss_struct *tss) { }
2343 #endif /* !CONFIG_X86_64 */
2344 
tss_setup_io_bitmap(struct tss_struct * tss)2345 static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2346 {
2347 	tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2348 
2349 #ifdef CONFIG_X86_IOPL_IOPERM
2350 	tss->io_bitmap.prev_max = 0;
2351 	tss->io_bitmap.prev_sequence = 0;
2352 	memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2353 	/*
2354 	 * Invalidate the extra array entry past the end of the all
2355 	 * permission bitmap as required by the hardware.
2356 	 */
2357 	tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2358 #endif
2359 }
2360 
2361 /*
2362  * Setup everything needed to handle exceptions from the IDT, including the IST
2363  * exceptions which use paranoid_entry().
2364  */
cpu_init_exception_handling(bool boot_cpu)2365 void cpu_init_exception_handling(bool boot_cpu)
2366 {
2367 	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2368 	int cpu = raw_smp_processor_id();
2369 
2370 	/* paranoid_entry() gets the CPU number from the GDT */
2371 	setup_getcpu(cpu);
2372 
2373 	/* For IDT mode, IST vectors need to be set in TSS. */
2374 	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2375 		tss_setup_ist(tss);
2376 	tss_setup_io_bitmap(tss);
2377 	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2378 
2379 	load_TR_desc();
2380 
2381 	/* GHCB needs to be setup to handle #VC. */
2382 	setup_ghcb();
2383 
2384 	if (cpu_feature_enabled(X86_FEATURE_FRED)) {
2385 		/* The boot CPU has enabled FRED during early boot */
2386 		if (!boot_cpu)
2387 			cpu_init_fred_exceptions();
2388 
2389 		cpu_init_fred_rsps();
2390 	} else {
2391 		load_current_idt();
2392 	}
2393 }
2394 
cpu_init_replace_early_idt(void)2395 void __init cpu_init_replace_early_idt(void)
2396 {
2397 	if (cpu_feature_enabled(X86_FEATURE_FRED))
2398 		cpu_init_fred_exceptions();
2399 	else
2400 		idt_setup_early_pf();
2401 }
2402 
2403 /*
2404  * cpu_init() initializes state that is per-CPU. Some data is already
2405  * initialized (naturally) in the bootstrap process, such as the GDT.  We
2406  * reload it nevertheless, this function acts as a 'CPU state barrier',
2407  * nothing should get across.
2408  */
cpu_init(void)2409 void cpu_init(void)
2410 {
2411 	struct task_struct *cur = current;
2412 	int cpu = raw_smp_processor_id();
2413 
2414 #ifdef CONFIG_NUMA
2415 	if (this_cpu_read(numa_node) == 0 &&
2416 	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
2417 		set_numa_node(early_cpu_to_node(cpu));
2418 #endif
2419 	pr_debug("Initializing CPU#%d\n", cpu);
2420 
2421 	if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2422 	    boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2423 		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2424 
2425 	if (IS_ENABLED(CONFIG_X86_64)) {
2426 		loadsegment(fs, 0);
2427 		memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2428 		syscall_init();
2429 
2430 		wrmsrq(MSR_FS_BASE, 0);
2431 		wrmsrq(MSR_KERNEL_GS_BASE, 0);
2432 		barrier();
2433 
2434 		x2apic_setup();
2435 
2436 		intel_posted_msi_init();
2437 	}
2438 
2439 	mmgrab(&init_mm);
2440 	cur->active_mm = &init_mm;
2441 	BUG_ON(cur->mm);
2442 	initialize_tlbstate_and_flush();
2443 	enter_lazy_tlb(&init_mm, cur);
2444 
2445 	/*
2446 	 * sp0 points to the entry trampoline stack regardless of what task
2447 	 * is running.
2448 	 */
2449 	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2450 
2451 	load_mm_ldt(&init_mm);
2452 
2453 	initialize_debug_regs();
2454 	dbg_restore_debug_regs();
2455 
2456 	doublefault_init_cpu_tss();
2457 
2458 	if (is_uv_system())
2459 		uv_cpu_init();
2460 
2461 	load_fixmap_gdt(cpu);
2462 }
2463 
2464 #ifdef CONFIG_MICROCODE_LATE_LOADING
2465 /**
2466  * store_cpu_caps() - Store a snapshot of CPU capabilities
2467  * @curr_info: Pointer where to store it
2468  *
2469  * Returns: None
2470  */
store_cpu_caps(struct cpuinfo_x86 * curr_info)2471 void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2472 {
2473 	/* Reload CPUID max function as it might've changed. */
2474 	curr_info->cpuid_level = cpuid_eax(0);
2475 
2476 	/* Copy all capability leafs and pick up the synthetic ones. */
2477 	memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2478 	       sizeof(curr_info->x86_capability));
2479 
2480 	/* Get the hardware CPUID leafs */
2481 	get_cpu_cap(curr_info);
2482 }
2483 
2484 /**
2485  * microcode_check() - Check if any CPU capabilities changed after an update.
2486  * @prev_info:	CPU capabilities stored before an update.
2487  *
2488  * The microcode loader calls this upon late microcode load to recheck features,
2489  * only when microcode has been updated. Caller holds and CPU hotplug lock.
2490  *
2491  * Return: None
2492  */
microcode_check(struct cpuinfo_x86 * prev_info)2493 void microcode_check(struct cpuinfo_x86 *prev_info)
2494 {
2495 	struct cpuinfo_x86 curr_info;
2496 
2497 	perf_check_microcode();
2498 
2499 	amd_check_microcode();
2500 
2501 	store_cpu_caps(&curr_info);
2502 
2503 	if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
2504 		    sizeof(prev_info->x86_capability)))
2505 		return;
2506 
2507 	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2508 	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2509 }
2510 #endif
2511 
2512 /*
2513  * Invoked from core CPU hotplug code after hotplug operations
2514  */
arch_smt_update(void)2515 void arch_smt_update(void)
2516 {
2517 	/* Handle the speculative execution misfeatures */
2518 	cpu_bugs_smt_update();
2519 	/* Check whether IPI broadcasting can be enabled */
2520 	apic_smt_update();
2521 }
2522 
arch_cpu_finalize_init(void)2523 void __init arch_cpu_finalize_init(void)
2524 {
2525 	struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info);
2526 
2527 	identify_boot_cpu();
2528 
2529 	select_idle_routine();
2530 
2531 	/*
2532 	 * identify_boot_cpu() initialized SMT support information, let the
2533 	 * core code know.
2534 	 */
2535 	cpu_smt_set_num_threads(__max_threads_per_core, __max_threads_per_core);
2536 
2537 	if (!IS_ENABLED(CONFIG_SMP)) {
2538 		pr_info("CPU: ");
2539 		print_cpu_info(&boot_cpu_data);
2540 	}
2541 
2542 	cpu_select_mitigations();
2543 
2544 	arch_smt_update();
2545 
2546 	if (IS_ENABLED(CONFIG_X86_32)) {
2547 		/*
2548 		 * Check whether this is a real i386 which is not longer
2549 		 * supported and fixup the utsname.
2550 		 */
2551 		if (boot_cpu_data.x86 < 4)
2552 			panic("Kernel requires i486+ for 'invlpg' and other features");
2553 
2554 		init_utsname()->machine[1] =
2555 			'0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2556 	}
2557 
2558 	/*
2559 	 * Must be before alternatives because it might set or clear
2560 	 * feature bits.
2561 	 */
2562 	fpu__init_system();
2563 	fpu__init_cpu();
2564 
2565 	/*
2566 	 * This needs to follow the FPU initializtion, since EFI depends on it.
2567 	 */
2568 	if (efi_enabled(EFI_RUNTIME_SERVICES))
2569 		efi_enter_virtual_mode();
2570 
2571 	/*
2572 	 * Ensure that access to the per CPU representation has the initial
2573 	 * boot CPU configuration.
2574 	 */
2575 	*c = boot_cpu_data;
2576 	c->initialized = true;
2577 
2578 	alternative_instructions();
2579 
2580 	if (IS_ENABLED(CONFIG_X86_64)) {
2581 		unsigned long USER_PTR_MAX = TASK_SIZE_MAX;
2582 
2583 		/*
2584 		 * Enable this when LAM is gated on LASS support
2585 		if (cpu_feature_enabled(X86_FEATURE_LAM))
2586 			USER_PTR_MAX = (1ul << 63) - PAGE_SIZE;
2587 		 */
2588 		runtime_const_init(ptr, USER_PTR_MAX);
2589 
2590 		/*
2591 		 * Make sure the first 2MB area is not mapped by huge pages
2592 		 * There are typically fixed size MTRRs in there and overlapping
2593 		 * MTRRs into large pages causes slow downs.
2594 		 *
2595 		 * Right now we don't do that with gbpages because there seems
2596 		 * very little benefit for that case.
2597 		 */
2598 		if (!direct_gbpages)
2599 			set_memory_4k((unsigned long)__va(0), 1);
2600 	} else {
2601 		fpu__init_check_bugs();
2602 	}
2603 
2604 	/*
2605 	 * This needs to be called before any devices perform DMA
2606 	 * operations that might use the SWIOTLB bounce buffers. It will
2607 	 * mark the bounce buffers as decrypted so that their usage will
2608 	 * not cause "plain-text" data to be decrypted when accessed. It
2609 	 * must be called after late_time_init() so that Hyper-V x86/x64
2610 	 * hypercalls work when the SWIOTLB bounce buffers are decrypted.
2611 	 */
2612 	mem_encrypt_init();
2613 }
2614