xref: /linux/arch/x86/kernel/smpboot.c (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2  /*
3  *	x86 SMP booting functions
4  *
5  *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
6  *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
7  *	Copyright 2001 Andi Kleen, SuSE Labs.
8  *
9  *	Much of the core SMP work is based on previous work by Thomas Radke, to
10  *	whom a great many thanks are extended.
11  *
12  *	Thanks to Intel for making available several different Pentium,
13  *	Pentium Pro and Pentium-II/Xeon MP machines.
14  *	Original development of Linux SMP code supported by Caldera.
15  *
16  *	Fixes
17  *		Felix Koop	:	NR_CPUS used properly
18  *		Jose Renau	:	Handle single CPU case.
19  *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
20  *		Greg Wright	:	Fix for kernel stacks panic.
21  *		Erich Boleyn	:	MP v1.4 and additional changes.
22  *	Matthias Sattler	:	Changes for 2.1 kernel map.
23  *	Michel Lespinasse	:	Changes for 2.1 kernel map.
24  *	Michael Chastain	:	Change trampoline.S to gnu as.
25  *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
26  *		Ingo Molnar	:	Added APIC timers, based on code
27  *					from Jose Renau
28  *		Ingo Molnar	:	various cleanups and rewrites
29  *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
30  *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
31  *	Andi Kleen		:	Changed for SMP boot into long mode.
32  *		Martin J. Bligh	: 	Added support for multi-quad systems
33  *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
34  *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
35  *      Andi Kleen              :       Converted to new state machine.
36  *	Ashok Raj		: 	CPU hotplug support
37  *	Glauber Costa		:	i386 and x86_64 integration
38  */
39 
40 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
41 
42 #include <linux/init.h>
43 #include <linux/smp.h>
44 #include <linux/export.h>
45 #include <linux/sched.h>
46 #include <linux/sched/topology.h>
47 #include <linux/sched/hotplug.h>
48 #include <linux/sched/task_stack.h>
49 #include <linux/percpu.h>
50 #include <linux/memblock.h>
51 #include <linux/err.h>
52 #include <linux/nmi.h>
53 #include <linux/tboot.h>
54 #include <linux/gfp.h>
55 #include <linux/cpuidle.h>
56 #include <linux/kexec.h>
57 #include <linux/numa.h>
58 #include <linux/pgtable.h>
59 #include <linux/overflow.h>
60 #include <linux/stackprotector.h>
61 #include <linux/cpuhotplug.h>
62 #include <linux/mc146818rtc.h>
63 #include <linux/acpi.h>
64 
65 #include <asm/acpi.h>
66 #include <asm/cacheinfo.h>
67 #include <asm/desc.h>
68 #include <asm/nmi.h>
69 #include <asm/irq.h>
70 #include <asm/realmode.h>
71 #include <asm/cpu.h>
72 #include <asm/numa.h>
73 #include <asm/tlbflush.h>
74 #include <asm/mtrr.h>
75 #include <asm/mwait.h>
76 #include <asm/apic.h>
77 #include <asm/io_apic.h>
78 #include <asm/fpu/api.h>
79 #include <asm/setup.h>
80 #include <asm/uv/uv.h>
81 #include <asm/microcode.h>
82 #include <asm/i8259.h>
83 #include <asm/misc.h>
84 #include <asm/qspinlock.h>
85 #include <asm/intel-family.h>
86 #include <asm/cpu_device_id.h>
87 #include <asm/spec-ctrl.h>
88 #include <asm/hw_irq.h>
89 #include <asm/stackprotector.h>
90 #include <asm/sev.h>
91 #include <asm/spec-ctrl.h>
92 
93 /* representing HT siblings of each logical CPU */
94 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
95 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
96 
97 /* representing HT and core siblings of each logical CPU */
98 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
99 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
100 
101 /* representing HT, core, and die siblings of each logical CPU */
102 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map);
103 EXPORT_PER_CPU_SYMBOL(cpu_die_map);
104 
105 /* CPUs which are the primary SMT threads */
106 struct cpumask __cpu_primary_thread_mask __read_mostly;
107 
108 /* Representing CPUs for which sibling maps can be computed */
109 static cpumask_var_t cpu_sibling_setup_mask;
110 
111 struct mwait_cpu_dead {
112 	unsigned int	control;
113 	unsigned int	status;
114 };
115 
116 #define CPUDEAD_MWAIT_WAIT	0xDEADBEEF
117 #define CPUDEAD_MWAIT_KEXEC_HLT	0x4A17DEAD
118 
119 /*
120  * Cache line aligned data for mwait_play_dead(). Separate on purpose so
121  * that it's unlikely to be touched by other CPUs.
122  */
123 static DEFINE_PER_CPU_ALIGNED(struct mwait_cpu_dead, mwait_cpu_dead);
124 
125 /* Maximum number of SMT threads on any online core */
126 int __read_mostly __max_smt_threads = 1;
127 
128 /* Flag to indicate if a complete sched domain rebuild is required */
129 bool x86_topology_update;
130 
arch_update_cpu_topology(void)131 int arch_update_cpu_topology(void)
132 {
133 	int retval = x86_topology_update;
134 
135 	x86_topology_update = false;
136 	return retval;
137 }
138 
139 static unsigned int smpboot_warm_reset_vector_count;
140 
smpboot_setup_warm_reset_vector(unsigned long start_eip)141 static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
142 {
143 	unsigned long flags;
144 
145 	spin_lock_irqsave(&rtc_lock, flags);
146 	if (!smpboot_warm_reset_vector_count++) {
147 		CMOS_WRITE(0xa, 0xf);
148 		*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) = start_eip >> 4;
149 		*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = start_eip & 0xf;
150 	}
151 	spin_unlock_irqrestore(&rtc_lock, flags);
152 }
153 
smpboot_restore_warm_reset_vector(void)154 static inline void smpboot_restore_warm_reset_vector(void)
155 {
156 	unsigned long flags;
157 
158 	/*
159 	 * Paranoid:  Set warm reset code and vector here back
160 	 * to default values.
161 	 */
162 	spin_lock_irqsave(&rtc_lock, flags);
163 	if (!--smpboot_warm_reset_vector_count) {
164 		CMOS_WRITE(0, 0xf);
165 		*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
166 	}
167 	spin_unlock_irqrestore(&rtc_lock, flags);
168 
169 }
170 
171 /* Run the next set of setup steps for the upcoming CPU */
ap_starting(void)172 static void ap_starting(void)
173 {
174 	int cpuid = smp_processor_id();
175 
176 	/* Mop up eventual mwait_play_dead() wreckage */
177 	this_cpu_write(mwait_cpu_dead.status, 0);
178 	this_cpu_write(mwait_cpu_dead.control, 0);
179 
180 	/*
181 	 * If woken up by an INIT in an 82489DX configuration the alive
182 	 * synchronization guarantees that the CPU does not reach this
183 	 * point before an INIT_deassert IPI reaches the local APIC, so it
184 	 * is now safe to touch the local APIC.
185 	 *
186 	 * Set up this CPU, first the APIC, which is probably redundant on
187 	 * most boards.
188 	 */
189 	apic_ap_setup();
190 
191 	/* Save the processor parameters. */
192 	smp_store_cpu_info(cpuid);
193 
194 	/*
195 	 * The topology information must be up to date before
196 	 * notify_cpu_starting().
197 	 */
198 	set_cpu_sibling_map(cpuid);
199 
200 	ap_init_aperfmperf();
201 
202 	pr_debug("Stack at about %p\n", &cpuid);
203 
204 	wmb();
205 
206 	/*
207 	 * This runs the AP through all the cpuhp states to its target
208 	 * state CPUHP_ONLINE.
209 	 */
210 	notify_cpu_starting(cpuid);
211 }
212 
ap_calibrate_delay(void)213 static void ap_calibrate_delay(void)
214 {
215 	/*
216 	 * Calibrate the delay loop and update loops_per_jiffy in cpu_data.
217 	 * smp_store_cpu_info() stored a value that is close but not as
218 	 * accurate as the value just calculated.
219 	 *
220 	 * As this is invoked after the TSC synchronization check,
221 	 * calibrate_delay_is_known() will skip the calibration routine
222 	 * when TSC is synchronized across sockets.
223 	 */
224 	calibrate_delay();
225 	cpu_data(smp_processor_id()).loops_per_jiffy = loops_per_jiffy;
226 }
227 
228 /*
229  * Activate a secondary processor.
230  */
start_secondary(void * unused)231 static void notrace start_secondary(void *unused)
232 {
233 	/*
234 	 * Don't put *anything* except direct CPU state initialization
235 	 * before cpu_init(), SMP booting is too fragile that we want to
236 	 * limit the things done here to the most necessary things.
237 	 */
238 	cr4_init();
239 
240 	/*
241 	 * 32-bit specific. 64-bit reaches this code with the correct page
242 	 * table established. Yet another historical divergence.
243 	 */
244 	if (IS_ENABLED(CONFIG_X86_32)) {
245 		/* switch away from the initial page table */
246 		load_cr3(swapper_pg_dir);
247 		__flush_tlb_all();
248 	}
249 
250 	cpu_init_exception_handling(false);
251 
252 	/*
253 	 * Load the microcode before reaching the AP alive synchronization
254 	 * point below so it is not part of the full per CPU serialized
255 	 * bringup part when "parallel" bringup is enabled.
256 	 *
257 	 * That's even safe when hyperthreading is enabled in the CPU as
258 	 * the core code starts the primary threads first and leaves the
259 	 * secondary threads waiting for SIPI. Loading microcode on
260 	 * physical cores concurrently is a safe operation.
261 	 *
262 	 * This covers both the Intel specific issue that concurrent
263 	 * microcode loading on SMT siblings must be prohibited and the
264 	 * vendor independent issue`that microcode loading which changes
265 	 * CPUID, MSRs etc. must be strictly serialized to maintain
266 	 * software state correctness.
267 	 */
268 	load_ucode_ap();
269 
270 	/*
271 	 * Synchronization point with the hotplug core. Sets this CPUs
272 	 * synchronization state to ALIVE and spin-waits for the control CPU to
273 	 * release this CPU for further bringup.
274 	 */
275 	cpuhp_ap_sync_alive();
276 
277 	cpu_init();
278 	fpu__init_cpu();
279 	rcutree_report_cpu_starting(raw_smp_processor_id());
280 	x86_cpuinit.early_percpu_clock_init();
281 
282 	ap_starting();
283 
284 	/* Check TSC synchronization with the control CPU. */
285 	check_tsc_sync_target();
286 
287 	/*
288 	 * Calibrate the delay loop after the TSC synchronization check.
289 	 * This allows to skip the calibration when TSC is synchronized
290 	 * across sockets.
291 	 */
292 	ap_calibrate_delay();
293 
294 	speculative_store_bypass_ht_init();
295 
296 	/*
297 	 * Lock vector_lock, set CPU online and bring the vector
298 	 * allocator online. Online must be set with vector_lock held
299 	 * to prevent a concurrent irq setup/teardown from seeing a
300 	 * half valid vector space.
301 	 */
302 	lock_vector_lock();
303 	set_cpu_online(smp_processor_id(), true);
304 	lapic_online();
305 	unlock_vector_lock();
306 	x86_platform.nmi_init();
307 
308 	/* enable local interrupts */
309 	local_irq_enable();
310 
311 	x86_cpuinit.setup_percpu_clockev();
312 
313 	wmb();
314 	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
315 }
316 
317 /*
318  * The bootstrap kernel entry code has set these up. Save them for
319  * a given CPU
320  */
smp_store_cpu_info(int id)321 void smp_store_cpu_info(int id)
322 {
323 	struct cpuinfo_x86 *c = &cpu_data(id);
324 
325 	/* Copy boot_cpu_data only on the first bringup */
326 	if (!c->initialized)
327 		*c = boot_cpu_data;
328 	c->cpu_index = id;
329 	/*
330 	 * During boot time, CPU0 has this setup already. Save the info when
331 	 * bringing up an AP.
332 	 */
333 	identify_secondary_cpu(c);
334 	c->initialized = true;
335 }
336 
337 static bool
topology_same_node(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o)338 topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
339 {
340 	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
341 
342 	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
343 }
344 
345 static bool
topology_sane(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o,const char * name)346 topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
347 {
348 	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
349 
350 	return !WARN_ONCE(!topology_same_node(c, o),
351 		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
352 		"[node: %d != %d]. Ignoring dependency.\n",
353 		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
354 }
355 
356 #define link_mask(mfunc, c1, c2)					\
357 do {									\
358 	cpumask_set_cpu((c1), mfunc(c2));				\
359 	cpumask_set_cpu((c2), mfunc(c1));				\
360 } while (0)
361 
match_smt(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o)362 static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
363 {
364 	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
365 		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
366 
367 		if (c->topo.pkg_id == o->topo.pkg_id &&
368 		    c->topo.die_id == o->topo.die_id &&
369 		    c->topo.amd_node_id == o->topo.amd_node_id &&
370 		    per_cpu_llc_id(cpu1) == per_cpu_llc_id(cpu2)) {
371 			if (c->topo.core_id == o->topo.core_id)
372 				return topology_sane(c, o, "smt");
373 
374 			if ((c->topo.cu_id != 0xff) &&
375 			    (o->topo.cu_id != 0xff) &&
376 			    (c->topo.cu_id == o->topo.cu_id))
377 				return topology_sane(c, o, "smt");
378 		}
379 
380 	} else if (c->topo.pkg_id == o->topo.pkg_id &&
381 		   c->topo.die_id == o->topo.die_id &&
382 		   c->topo.core_id == o->topo.core_id) {
383 		return topology_sane(c, o, "smt");
384 	}
385 
386 	return false;
387 }
388 
match_die(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o)389 static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
390 {
391 	if (c->topo.pkg_id != o->topo.pkg_id || c->topo.die_id != o->topo.die_id)
392 		return false;
393 
394 	if (cpu_feature_enabled(X86_FEATURE_TOPOEXT) && topology_amd_nodes_per_pkg() > 1)
395 		return c->topo.amd_node_id == o->topo.amd_node_id;
396 
397 	return true;
398 }
399 
match_l2c(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o)400 static bool match_l2c(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
401 {
402 	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
403 
404 	/* If the arch didn't set up l2c_id, fall back to SMT */
405 	if (per_cpu_l2c_id(cpu1) == BAD_APICID)
406 		return match_smt(c, o);
407 
408 	/* Do not match if L2 cache id does not match: */
409 	if (per_cpu_l2c_id(cpu1) != per_cpu_l2c_id(cpu2))
410 		return false;
411 
412 	return topology_sane(c, o, "l2c");
413 }
414 
415 /*
416  * Unlike the other levels, we do not enforce keeping a
417  * multicore group inside a NUMA node.  If this happens, we will
418  * discard the MC level of the topology later.
419  */
match_pkg(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o)420 static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
421 {
422 	if (c->topo.pkg_id == o->topo.pkg_id)
423 		return true;
424 	return false;
425 }
426 
427 /*
428  * Define intel_cod_cpu[] for Intel COD (Cluster-on-Die) CPUs.
429  *
430  * Any Intel CPU that has multiple nodes per package and does not
431  * match intel_cod_cpu[] has the SNC (Sub-NUMA Cluster) topology.
432  *
433  * When in SNC mode, these CPUs enumerate an LLC that is shared
434  * by multiple NUMA nodes. The LLC is shared for off-package data
435  * access but private to the NUMA node (half of the package) for
436  * on-package access. CPUID (the source of the information about
437  * the LLC) can only enumerate the cache as shared or unshared,
438  * but not this particular configuration.
439  */
440 
441 static const struct x86_cpu_id intel_cod_cpu[] = {
442 	X86_MATCH_VFM(INTEL_HASWELL_X,	 0),	/* COD */
443 	X86_MATCH_VFM(INTEL_BROADWELL_X, 0),	/* COD */
444 	X86_MATCH_VFM(INTEL_ANY,	 1),	/* SNC */
445 	{}
446 };
447 
match_llc(struct cpuinfo_x86 * c,struct cpuinfo_x86 * o)448 static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
449 {
450 	const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
451 	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
452 	bool intel_snc = id && id->driver_data;
453 
454 	/* Do not match if we do not have a valid APICID for cpu: */
455 	if (per_cpu_llc_id(cpu1) == BAD_APICID)
456 		return false;
457 
458 	/* Do not match if LLC id does not match: */
459 	if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2))
460 		return false;
461 
462 	/*
463 	 * Allow the SNC topology without warning. Return of false
464 	 * means 'c' does not share the LLC of 'o'. This will be
465 	 * reflected to userspace.
466 	 */
467 	if (match_pkg(c, o) && !topology_same_node(c, o) && intel_snc)
468 		return false;
469 
470 	return topology_sane(c, o, "llc");
471 }
472 
473 
x86_sched_itmt_flags(void)474 static inline int x86_sched_itmt_flags(void)
475 {
476 	return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0;
477 }
478 
479 #ifdef CONFIG_SCHED_MC
x86_core_flags(void)480 static int x86_core_flags(void)
481 {
482 	return cpu_core_flags() | x86_sched_itmt_flags();
483 }
484 #endif
485 #ifdef CONFIG_SCHED_SMT
x86_smt_flags(void)486 static int x86_smt_flags(void)
487 {
488 	return cpu_smt_flags();
489 }
490 #endif
491 #ifdef CONFIG_SCHED_CLUSTER
x86_cluster_flags(void)492 static int x86_cluster_flags(void)
493 {
494 	return cpu_cluster_flags() | x86_sched_itmt_flags();
495 }
496 #endif
497 
x86_die_flags(void)498 static int x86_die_flags(void)
499 {
500 	if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU) ||
501 	    cpu_feature_enabled(X86_FEATURE_AMD_HETEROGENEOUS_CORES))
502 		return x86_sched_itmt_flags();
503 
504 	return 0;
505 }
506 
507 /*
508  * Set if a package/die has multiple NUMA nodes inside.
509  * AMD Magny-Cours, Intel Cluster-on-Die, and Intel
510  * Sub-NUMA Clustering have this.
511  */
512 static bool x86_has_numa_in_package;
513 
514 static struct sched_domain_topology_level x86_topology[6];
515 
build_sched_topology(void)516 static void __init build_sched_topology(void)
517 {
518 	int i = 0;
519 
520 #ifdef CONFIG_SCHED_SMT
521 	x86_topology[i++] = (struct sched_domain_topology_level){
522 		cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT)
523 	};
524 #endif
525 #ifdef CONFIG_SCHED_CLUSTER
526 	x86_topology[i++] = (struct sched_domain_topology_level){
527 		cpu_clustergroup_mask, x86_cluster_flags, SD_INIT_NAME(CLS)
528 	};
529 #endif
530 #ifdef CONFIG_SCHED_MC
531 	x86_topology[i++] = (struct sched_domain_topology_level){
532 		cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC)
533 	};
534 #endif
535 	/*
536 	 * When there is NUMA topology inside the package skip the PKG domain
537 	 * since the NUMA domains will auto-magically create the right spanning
538 	 * domains based on the SLIT.
539 	 */
540 	if (!x86_has_numa_in_package) {
541 		x86_topology[i++] = (struct sched_domain_topology_level){
542 			cpu_cpu_mask, x86_die_flags, SD_INIT_NAME(PKG)
543 		};
544 	}
545 
546 	/*
547 	 * There must be one trailing NULL entry left.
548 	 */
549 	BUG_ON(i >= ARRAY_SIZE(x86_topology)-1);
550 
551 	set_sched_topology(x86_topology);
552 }
553 
set_cpu_sibling_map(int cpu)554 void set_cpu_sibling_map(int cpu)
555 {
556 	bool has_smt = __max_threads_per_core > 1;
557 	bool has_mp = has_smt || topology_num_cores_per_package() > 1;
558 	struct cpuinfo_x86 *c = &cpu_data(cpu);
559 	struct cpuinfo_x86 *o;
560 	int i, threads;
561 
562 	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
563 
564 	if (!has_mp) {
565 		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
566 		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
567 		cpumask_set_cpu(cpu, cpu_l2c_shared_mask(cpu));
568 		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
569 		cpumask_set_cpu(cpu, topology_die_cpumask(cpu));
570 		c->booted_cores = 1;
571 		return;
572 	}
573 
574 	for_each_cpu(i, cpu_sibling_setup_mask) {
575 		o = &cpu_data(i);
576 
577 		if (match_pkg(c, o) && !topology_same_node(c, o))
578 			x86_has_numa_in_package = true;
579 
580 		if ((i == cpu) || (has_smt && match_smt(c, o)))
581 			link_mask(topology_sibling_cpumask, cpu, i);
582 
583 		if ((i == cpu) || (has_mp && match_llc(c, o)))
584 			link_mask(cpu_llc_shared_mask, cpu, i);
585 
586 		if ((i == cpu) || (has_mp && match_l2c(c, o)))
587 			link_mask(cpu_l2c_shared_mask, cpu, i);
588 
589 		if ((i == cpu) || (has_mp && match_die(c, o)))
590 			link_mask(topology_die_cpumask, cpu, i);
591 	}
592 
593 	threads = cpumask_weight(topology_sibling_cpumask(cpu));
594 	if (threads > __max_smt_threads)
595 		__max_smt_threads = threads;
596 
597 	for_each_cpu(i, topology_sibling_cpumask(cpu))
598 		cpu_data(i).smt_active = threads > 1;
599 
600 	/*
601 	 * This needs a separate iteration over the cpus because we rely on all
602 	 * topology_sibling_cpumask links to be set-up.
603 	 */
604 	for_each_cpu(i, cpu_sibling_setup_mask) {
605 		o = &cpu_data(i);
606 
607 		if ((i == cpu) || (has_mp && match_pkg(c, o))) {
608 			link_mask(topology_core_cpumask, cpu, i);
609 
610 			/*
611 			 *  Does this new cpu bringup a new core?
612 			 */
613 			if (threads == 1) {
614 				/*
615 				 * for each core in package, increment
616 				 * the booted_cores for this new cpu
617 				 */
618 				if (cpumask_first(
619 				    topology_sibling_cpumask(i)) == i)
620 					c->booted_cores++;
621 				/*
622 				 * increment the core count for all
623 				 * the other cpus in this package
624 				 */
625 				if (i != cpu)
626 					cpu_data(i).booted_cores++;
627 			} else if (i != cpu && !c->booted_cores)
628 				c->booted_cores = cpu_data(i).booted_cores;
629 		}
630 	}
631 }
632 
633 /* maps the cpu to the sched domain representing multi-core */
cpu_coregroup_mask(int cpu)634 const struct cpumask *cpu_coregroup_mask(int cpu)
635 {
636 	return cpu_llc_shared_mask(cpu);
637 }
638 
cpu_clustergroup_mask(int cpu)639 const struct cpumask *cpu_clustergroup_mask(int cpu)
640 {
641 	return cpu_l2c_shared_mask(cpu);
642 }
643 EXPORT_SYMBOL_GPL(cpu_clustergroup_mask);
644 
impress_friends(void)645 static void impress_friends(void)
646 {
647 	int cpu;
648 	unsigned long bogosum = 0;
649 	/*
650 	 * Allow the user to impress friends.
651 	 */
652 	pr_debug("Before bogomips\n");
653 	for_each_online_cpu(cpu)
654 		bogosum += cpu_data(cpu).loops_per_jiffy;
655 
656 	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
657 		num_online_cpus(),
658 		bogosum/(500000/HZ),
659 		(bogosum/(5000/HZ))%100);
660 
661 	pr_debug("Before bogocount - setting activated=1\n");
662 }
663 
664 /*
665  * The Multiprocessor Specification 1.4 (1997) example code suggests
666  * that there should be a 10ms delay between the BSP asserting INIT
667  * and de-asserting INIT, when starting a remote processor.
668  * But that slows boot and resume on modern processors, which include
669  * many cores and don't require that delay.
670  *
671  * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
672  * Modern processor families are quirked to remove the delay entirely.
673  */
674 #define UDELAY_10MS_DEFAULT 10000
675 
676 static unsigned int init_udelay = UINT_MAX;
677 
cpu_init_udelay(char * str)678 static int __init cpu_init_udelay(char *str)
679 {
680 	get_option(&str, &init_udelay);
681 
682 	return 0;
683 }
684 early_param("cpu_init_udelay", cpu_init_udelay);
685 
smp_quirk_init_udelay(void)686 static void __init smp_quirk_init_udelay(void)
687 {
688 	/* if cmdline changed it from default, leave it alone */
689 	if (init_udelay != UINT_MAX)
690 		return;
691 
692 	/* if modern processor, use no delay */
693 	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
694 	    ((boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) && (boot_cpu_data.x86 >= 0x18)) ||
695 	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
696 		init_udelay = 0;
697 		return;
698 	}
699 	/* else, use legacy delay */
700 	init_udelay = UDELAY_10MS_DEFAULT;
701 }
702 
703 /*
704  * Wake up AP by INIT, INIT, STARTUP sequence.
705  */
send_init_sequence(u32 phys_apicid)706 static void send_init_sequence(u32 phys_apicid)
707 {
708 	int maxlvt = lapic_get_maxlvt();
709 
710 	/* Be paranoid about clearing APIC errors. */
711 	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
712 		/* Due to the Pentium erratum 3AP.  */
713 		if (maxlvt > 3)
714 			apic_write(APIC_ESR, 0);
715 		apic_read(APIC_ESR);
716 	}
717 
718 	/* Assert INIT on the target CPU */
719 	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT, phys_apicid);
720 	safe_apic_wait_icr_idle();
721 
722 	udelay(init_udelay);
723 
724 	/* Deassert INIT on the target CPU */
725 	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
726 	safe_apic_wait_icr_idle();
727 }
728 
729 /*
730  * Wake up AP by INIT, INIT, STARTUP sequence.
731  */
wakeup_secondary_cpu_via_init(u32 phys_apicid,unsigned long start_eip)732 static int wakeup_secondary_cpu_via_init(u32 phys_apicid, unsigned long start_eip)
733 {
734 	unsigned long send_status = 0, accept_status = 0;
735 	int num_starts, j, maxlvt;
736 
737 	preempt_disable();
738 	maxlvt = lapic_get_maxlvt();
739 	send_init_sequence(phys_apicid);
740 
741 	mb();
742 
743 	/*
744 	 * Should we send STARTUP IPIs ?
745 	 *
746 	 * Determine this based on the APIC version.
747 	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
748 	 */
749 	if (APIC_INTEGRATED(boot_cpu_apic_version))
750 		num_starts = 2;
751 	else
752 		num_starts = 0;
753 
754 	/*
755 	 * Run STARTUP IPI loop.
756 	 */
757 	pr_debug("#startup loops: %d\n", num_starts);
758 
759 	for (j = 1; j <= num_starts; j++) {
760 		pr_debug("Sending STARTUP #%d\n", j);
761 		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
762 			apic_write(APIC_ESR, 0);
763 		apic_read(APIC_ESR);
764 		pr_debug("After apic_write\n");
765 
766 		/*
767 		 * STARTUP IPI
768 		 */
769 
770 		/* Target chip */
771 		/* Boot on the stack */
772 		/* Kick the second */
773 		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
774 			       phys_apicid);
775 
776 		/*
777 		 * Give the other CPU some time to accept the IPI.
778 		 */
779 		if (init_udelay == 0)
780 			udelay(10);
781 		else
782 			udelay(300);
783 
784 		pr_debug("Startup point 1\n");
785 
786 		pr_debug("Waiting for send to finish...\n");
787 		send_status = safe_apic_wait_icr_idle();
788 
789 		/*
790 		 * Give the other CPU some time to accept the IPI.
791 		 */
792 		if (init_udelay == 0)
793 			udelay(10);
794 		else
795 			udelay(200);
796 
797 		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
798 			apic_write(APIC_ESR, 0);
799 		accept_status = (apic_read(APIC_ESR) & 0xEF);
800 		if (send_status || accept_status)
801 			break;
802 	}
803 	pr_debug("After Startup\n");
804 
805 	if (send_status)
806 		pr_err("APIC never delivered???\n");
807 	if (accept_status)
808 		pr_err("APIC delivery error (%lx)\n", accept_status);
809 
810 	preempt_enable();
811 	return (send_status | accept_status);
812 }
813 
814 /* reduce the number of lines printed when booting a large cpu count system */
announce_cpu(int cpu,int apicid)815 static void announce_cpu(int cpu, int apicid)
816 {
817 	static int width, node_width, first = 1;
818 	static int current_node = NUMA_NO_NODE;
819 	int node = early_cpu_to_node(cpu);
820 
821 	if (!width)
822 		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
823 
824 	if (!node_width)
825 		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */
826 
827 	if (system_state < SYSTEM_RUNNING) {
828 		if (first)
829 			pr_info("x86: Booting SMP configuration:\n");
830 
831 		if (node != current_node) {
832 			if (current_node > (-1))
833 				pr_cont("\n");
834 			current_node = node;
835 
836 			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
837 			       node_width - num_digits(node), " ", node);
838 		}
839 
840 		/* Add padding for the BSP */
841 		if (first)
842 			pr_cont("%*s", width + 1, " ");
843 		first = 0;
844 
845 		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);
846 	} else
847 		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
848 			node, cpu, apicid);
849 }
850 
common_cpu_up(unsigned int cpu,struct task_struct * idle)851 int common_cpu_up(unsigned int cpu, struct task_struct *idle)
852 {
853 	int ret;
854 
855 	/* Just in case we booted with a single CPU. */
856 	alternatives_enable_smp();
857 
858 	per_cpu(pcpu_hot.current_task, cpu) = idle;
859 	cpu_init_stack_canary(cpu, idle);
860 
861 	/* Initialize the interrupt stack(s) */
862 	ret = irq_init_percpu_irqstack(cpu);
863 	if (ret)
864 		return ret;
865 
866 #ifdef CONFIG_X86_32
867 	/* Stack for startup_32 can be just as for start_secondary onwards */
868 	per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle);
869 #endif
870 	return 0;
871 }
872 
873 /*
874  * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
875  * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
876  * Returns zero if startup was successfully sent, else error code from
877  * ->wakeup_secondary_cpu.
878  */
do_boot_cpu(u32 apicid,int cpu,struct task_struct * idle)879 static int do_boot_cpu(u32 apicid, int cpu, struct task_struct *idle)
880 {
881 	unsigned long start_ip = real_mode_header->trampoline_start;
882 	int ret;
883 
884 #ifdef CONFIG_X86_64
885 	/* If 64-bit wakeup method exists, use the 64-bit mode trampoline IP */
886 	if (apic->wakeup_secondary_cpu_64)
887 		start_ip = real_mode_header->trampoline_start64;
888 #endif
889 	idle->thread.sp = (unsigned long)task_pt_regs(idle);
890 	initial_code = (unsigned long)start_secondary;
891 
892 	if (IS_ENABLED(CONFIG_X86_32)) {
893 		early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu);
894 		initial_stack  = idle->thread.sp;
895 	} else if (!(smpboot_control & STARTUP_PARALLEL_MASK)) {
896 		smpboot_control = cpu;
897 	}
898 
899 	/* Enable the espfix hack for this CPU */
900 	init_espfix_ap(cpu);
901 
902 	/* So we see what's up */
903 	announce_cpu(cpu, apicid);
904 
905 	/*
906 	 * This grunge runs the startup process for
907 	 * the targeted processor.
908 	 */
909 	if (x86_platform.legacy.warm_reset) {
910 
911 		pr_debug("Setting warm reset code and vector.\n");
912 
913 		smpboot_setup_warm_reset_vector(start_ip);
914 		/*
915 		 * Be paranoid about clearing APIC errors.
916 		*/
917 		if (APIC_INTEGRATED(boot_cpu_apic_version)) {
918 			apic_write(APIC_ESR, 0);
919 			apic_read(APIC_ESR);
920 		}
921 	}
922 
923 	smp_mb();
924 
925 	/*
926 	 * Wake up a CPU in difference cases:
927 	 * - Use a method from the APIC driver if one defined, with wakeup
928 	 *   straight to 64-bit mode preferred over wakeup to RM.
929 	 * Otherwise,
930 	 * - Use an INIT boot APIC message
931 	 */
932 	if (apic->wakeup_secondary_cpu_64)
933 		ret = apic->wakeup_secondary_cpu_64(apicid, start_ip);
934 	else if (apic->wakeup_secondary_cpu)
935 		ret = apic->wakeup_secondary_cpu(apicid, start_ip);
936 	else
937 		ret = wakeup_secondary_cpu_via_init(apicid, start_ip);
938 
939 	/* If the wakeup mechanism failed, cleanup the warm reset vector */
940 	if (ret)
941 		arch_cpuhp_cleanup_kick_cpu(cpu);
942 	return ret;
943 }
944 
native_kick_ap(unsigned int cpu,struct task_struct * tidle)945 int native_kick_ap(unsigned int cpu, struct task_struct *tidle)
946 {
947 	u32 apicid = apic->cpu_present_to_apicid(cpu);
948 	int err;
949 
950 	lockdep_assert_irqs_enabled();
951 
952 	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
953 
954 	if (apicid == BAD_APICID || !apic_id_valid(apicid)) {
955 		pr_err("CPU %u has invalid APIC ID %x. Aborting bringup\n", cpu, apicid);
956 		return -EINVAL;
957 	}
958 
959 	if (!test_bit(apicid, phys_cpu_present_map)) {
960 		pr_err("CPU %u APIC ID %x is not present. Aborting bringup\n", cpu, apicid);
961 		return -EINVAL;
962 	}
963 
964 	/*
965 	 * Save current MTRR state in case it was changed since early boot
966 	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
967 	 */
968 	mtrr_save_state();
969 
970 	/* the FPU context is blank, nobody can own it */
971 	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
972 
973 	err = common_cpu_up(cpu, tidle);
974 	if (err)
975 		return err;
976 
977 	err = do_boot_cpu(apicid, cpu, tidle);
978 	if (err)
979 		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
980 
981 	return err;
982 }
983 
arch_cpuhp_kick_ap_alive(unsigned int cpu,struct task_struct * tidle)984 int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
985 {
986 	return smp_ops.kick_ap_alive(cpu, tidle);
987 }
988 
arch_cpuhp_cleanup_kick_cpu(unsigned int cpu)989 void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu)
990 {
991 	/* Cleanup possible dangling ends... */
992 	if (smp_ops.kick_ap_alive == native_kick_ap && x86_platform.legacy.warm_reset)
993 		smpboot_restore_warm_reset_vector();
994 }
995 
arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)996 void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)
997 {
998 	if (smp_ops.cleanup_dead_cpu)
999 		smp_ops.cleanup_dead_cpu(cpu);
1000 
1001 	if (system_state == SYSTEM_RUNNING)
1002 		pr_info("CPU %u is now offline\n", cpu);
1003 }
1004 
arch_cpuhp_sync_state_poll(void)1005 void arch_cpuhp_sync_state_poll(void)
1006 {
1007 	if (smp_ops.poll_sync_state)
1008 		smp_ops.poll_sync_state();
1009 }
1010 
1011 /**
1012  * arch_disable_smp_support() - Disables SMP support for x86 at boottime
1013  */
arch_disable_smp_support(void)1014 void __init arch_disable_smp_support(void)
1015 {
1016 	disable_ioapic_support();
1017 }
1018 
1019 /*
1020  * Fall back to non SMP mode after errors.
1021  *
1022  * RED-PEN audit/test this more. I bet there is more state messed up here.
1023  */
disable_smp(void)1024 static __init void disable_smp(void)
1025 {
1026 	pr_info("SMP disabled\n");
1027 
1028 	disable_ioapic_support();
1029 	topology_reset_possible_cpus_up();
1030 
1031 	cpumask_set_cpu(0, topology_sibling_cpumask(0));
1032 	cpumask_set_cpu(0, topology_core_cpumask(0));
1033 	cpumask_set_cpu(0, topology_die_cpumask(0));
1034 }
1035 
smp_prepare_cpus_common(void)1036 void __init smp_prepare_cpus_common(void)
1037 {
1038 	unsigned int cpu, node;
1039 
1040 	/* Mark all except the boot CPU as hotpluggable */
1041 	for_each_possible_cpu(cpu) {
1042 		if (cpu)
1043 			per_cpu(cpu_info.cpu_index, cpu) = nr_cpu_ids;
1044 	}
1045 
1046 	for_each_possible_cpu(cpu) {
1047 		node = cpu_to_node(cpu);
1048 
1049 		zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map,    cpu), GFP_KERNEL, node);
1050 		zalloc_cpumask_var_node(&per_cpu(cpu_core_map,       cpu), GFP_KERNEL, node);
1051 		zalloc_cpumask_var_node(&per_cpu(cpu_die_map,        cpu), GFP_KERNEL, node);
1052 		zalloc_cpumask_var_node(&per_cpu(cpu_llc_shared_map, cpu), GFP_KERNEL, node);
1053 		zalloc_cpumask_var_node(&per_cpu(cpu_l2c_shared_map, cpu), GFP_KERNEL, node);
1054 	}
1055 
1056 	set_cpu_sibling_map(0);
1057 }
1058 
smp_prepare_boot_cpu(void)1059 void __init smp_prepare_boot_cpu(void)
1060 {
1061 	smp_ops.smp_prepare_boot_cpu();
1062 }
1063 
1064 #ifdef CONFIG_X86_64
1065 /* Establish whether parallel bringup can be supported. */
arch_cpuhp_init_parallel_bringup(void)1066 bool __init arch_cpuhp_init_parallel_bringup(void)
1067 {
1068 	if (!x86_cpuinit.parallel_bringup) {
1069 		pr_info("Parallel CPU startup disabled by the platform\n");
1070 		return false;
1071 	}
1072 
1073 	smpboot_control = STARTUP_READ_APICID;
1074 	pr_debug("Parallel CPU startup enabled: 0x%08x\n", smpboot_control);
1075 	return true;
1076 }
1077 #endif
1078 
1079 /*
1080  * Prepare for SMP bootup.
1081  * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
1082  *            for common interface support.
1083  */
native_smp_prepare_cpus(unsigned int max_cpus)1084 void __init native_smp_prepare_cpus(unsigned int max_cpus)
1085 {
1086 	smp_prepare_cpus_common();
1087 
1088 	switch (apic_intr_mode) {
1089 	case APIC_PIC:
1090 	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1091 		disable_smp();
1092 		return;
1093 	case APIC_SYMMETRIC_IO_NO_ROUTING:
1094 		disable_smp();
1095 		/* Setup local timer */
1096 		x86_init.timers.setup_percpu_clockev();
1097 		return;
1098 	case APIC_VIRTUAL_WIRE:
1099 	case APIC_SYMMETRIC_IO:
1100 		break;
1101 	}
1102 
1103 	/* Setup local timer */
1104 	x86_init.timers.setup_percpu_clockev();
1105 
1106 	pr_info("CPU0: ");
1107 	print_cpu_info(&cpu_data(0));
1108 
1109 	uv_system_init();
1110 
1111 	smp_quirk_init_udelay();
1112 
1113 	speculative_store_bypass_ht_init();
1114 
1115 	snp_set_wakeup_secondary_cpu();
1116 }
1117 
arch_thaw_secondary_cpus_begin(void)1118 void arch_thaw_secondary_cpus_begin(void)
1119 {
1120 	set_cache_aps_delayed_init(true);
1121 }
1122 
arch_thaw_secondary_cpus_end(void)1123 void arch_thaw_secondary_cpus_end(void)
1124 {
1125 	cache_aps_init();
1126 }
1127 
1128 /*
1129  * Early setup to make printk work.
1130  */
native_smp_prepare_boot_cpu(void)1131 void __init native_smp_prepare_boot_cpu(void)
1132 {
1133 	int me = smp_processor_id();
1134 
1135 	/* SMP handles this from setup_per_cpu_areas() */
1136 	if (!IS_ENABLED(CONFIG_SMP))
1137 		switch_gdt_and_percpu_base(me);
1138 
1139 	native_pv_lock_init();
1140 }
1141 
native_smp_cpus_done(unsigned int max_cpus)1142 void __init native_smp_cpus_done(unsigned int max_cpus)
1143 {
1144 	pr_debug("Boot done\n");
1145 
1146 	build_sched_topology();
1147 	nmi_selftest();
1148 	impress_friends();
1149 	cache_aps_init();
1150 }
1151 
1152 /* correctly size the local cpu masks */
setup_cpu_local_masks(void)1153 void __init setup_cpu_local_masks(void)
1154 {
1155 	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
1156 }
1157 
1158 #ifdef CONFIG_HOTPLUG_CPU
1159 
1160 /* Recompute SMT state for all CPUs on offline */
recompute_smt_state(void)1161 static void recompute_smt_state(void)
1162 {
1163 	int max_threads, cpu;
1164 
1165 	max_threads = 0;
1166 	for_each_online_cpu (cpu) {
1167 		int threads = cpumask_weight(topology_sibling_cpumask(cpu));
1168 
1169 		if (threads > max_threads)
1170 			max_threads = threads;
1171 	}
1172 	__max_smt_threads = max_threads;
1173 }
1174 
remove_siblinginfo(int cpu)1175 static void remove_siblinginfo(int cpu)
1176 {
1177 	int sibling;
1178 	struct cpuinfo_x86 *c = &cpu_data(cpu);
1179 
1180 	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
1181 		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1182 		/*/
1183 		 * last thread sibling in this cpu core going down
1184 		 */
1185 		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1186 			cpu_data(sibling).booted_cores--;
1187 	}
1188 
1189 	for_each_cpu(sibling, topology_die_cpumask(cpu))
1190 		cpumask_clear_cpu(cpu, topology_die_cpumask(sibling));
1191 
1192 	for_each_cpu(sibling, topology_sibling_cpumask(cpu)) {
1193 		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1194 		if (cpumask_weight(topology_sibling_cpumask(sibling)) == 1)
1195 			cpu_data(sibling).smt_active = false;
1196 	}
1197 
1198 	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
1199 		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
1200 	for_each_cpu(sibling, cpu_l2c_shared_mask(cpu))
1201 		cpumask_clear_cpu(cpu, cpu_l2c_shared_mask(sibling));
1202 	cpumask_clear(cpu_llc_shared_mask(cpu));
1203 	cpumask_clear(cpu_l2c_shared_mask(cpu));
1204 	cpumask_clear(topology_sibling_cpumask(cpu));
1205 	cpumask_clear(topology_core_cpumask(cpu));
1206 	cpumask_clear(topology_die_cpumask(cpu));
1207 	c->topo.core_id = 0;
1208 	c->booted_cores = 0;
1209 	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1210 	recompute_smt_state();
1211 }
1212 
remove_cpu_from_maps(int cpu)1213 static void remove_cpu_from_maps(int cpu)
1214 {
1215 	set_cpu_online(cpu, false);
1216 	numa_remove_cpu(cpu);
1217 }
1218 
cpu_disable_common(void)1219 void cpu_disable_common(void)
1220 {
1221 	int cpu = smp_processor_id();
1222 
1223 	remove_siblinginfo(cpu);
1224 
1225 	/* It's now safe to remove this processor from the online map */
1226 	lock_vector_lock();
1227 	remove_cpu_from_maps(cpu);
1228 	unlock_vector_lock();
1229 	fixup_irqs();
1230 	lapic_offline();
1231 }
1232 
native_cpu_disable(void)1233 int native_cpu_disable(void)
1234 {
1235 	int ret;
1236 
1237 	ret = lapic_can_unplug_cpu();
1238 	if (ret)
1239 		return ret;
1240 
1241 	cpu_disable_common();
1242 
1243         /*
1244          * Disable the local APIC. Otherwise IPI broadcasts will reach
1245          * it. It still responds normally to INIT, NMI, SMI, and SIPI
1246          * messages.
1247          *
1248          * Disabling the APIC must happen after cpu_disable_common()
1249          * which invokes fixup_irqs().
1250          *
1251          * Disabling the APIC preserves already set bits in IRR, but
1252          * an interrupt arriving after disabling the local APIC does not
1253          * set the corresponding IRR bit.
1254          *
1255          * fixup_irqs() scans IRR for set bits so it can raise a not
1256          * yet handled interrupt on the new destination CPU via an IPI
1257          * but obviously it can't do so for IRR bits which are not set.
1258          * IOW, interrupts arriving after disabling the local APIC will
1259          * be lost.
1260          */
1261 	apic_soft_disable();
1262 
1263 	return 0;
1264 }
1265 
play_dead_common(void)1266 void play_dead_common(void)
1267 {
1268 	idle_task_exit();
1269 
1270 	cpuhp_ap_report_dead();
1271 
1272 	local_irq_disable();
1273 }
1274 
1275 /*
1276  * We need to flush the caches before going to sleep, lest we have
1277  * dirty data in our caches when we come back up.
1278  */
mwait_play_dead(void)1279 static inline void mwait_play_dead(void)
1280 {
1281 	struct mwait_cpu_dead *md = this_cpu_ptr(&mwait_cpu_dead);
1282 	unsigned int eax, ebx, ecx, edx;
1283 	unsigned int highest_cstate = 0;
1284 	unsigned int highest_subcstate = 0;
1285 	int i;
1286 
1287 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1288 	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
1289 		return;
1290 	if (!this_cpu_has(X86_FEATURE_MWAIT))
1291 		return;
1292 	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1293 		return;
1294 	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1295 		return;
1296 
1297 	eax = CPUID_MWAIT_LEAF;
1298 	ecx = 0;
1299 	native_cpuid(&eax, &ebx, &ecx, &edx);
1300 
1301 	/*
1302 	 * eax will be 0 if EDX enumeration is not valid.
1303 	 * Initialized below to cstate, sub_cstate value when EDX is valid.
1304 	 */
1305 	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
1306 		eax = 0;
1307 	} else {
1308 		edx >>= MWAIT_SUBSTATE_SIZE;
1309 		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
1310 			if (edx & MWAIT_SUBSTATE_MASK) {
1311 				highest_cstate = i;
1312 				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
1313 			}
1314 		}
1315 		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
1316 			(highest_subcstate - 1);
1317 	}
1318 
1319 	/* Set up state for the kexec() hack below */
1320 	md->status = CPUDEAD_MWAIT_WAIT;
1321 	md->control = CPUDEAD_MWAIT_WAIT;
1322 
1323 	wbinvd();
1324 
1325 	while (1) {
1326 		/*
1327 		 * The CLFLUSH is a workaround for erratum AAI65 for
1328 		 * the Xeon 7400 series.  It's not clear it is actually
1329 		 * needed, but it should be harmless in either case.
1330 		 * The WBINVD is insufficient due to the spurious-wakeup
1331 		 * case where we return around the loop.
1332 		 */
1333 		mb();
1334 		clflush(md);
1335 		mb();
1336 		__monitor(md, 0, 0);
1337 		mb();
1338 		__mwait(eax, 0);
1339 
1340 		if (READ_ONCE(md->control) == CPUDEAD_MWAIT_KEXEC_HLT) {
1341 			/*
1342 			 * Kexec is about to happen. Don't go back into mwait() as
1343 			 * the kexec kernel might overwrite text and data including
1344 			 * page tables and stack. So mwait() would resume when the
1345 			 * monitor cache line is written to and then the CPU goes
1346 			 * south due to overwritten text, page tables and stack.
1347 			 *
1348 			 * Note: This does _NOT_ protect against a stray MCE, NMI,
1349 			 * SMI. They will resume execution at the instruction
1350 			 * following the HLT instruction and run into the problem
1351 			 * which this is trying to prevent.
1352 			 */
1353 			WRITE_ONCE(md->status, CPUDEAD_MWAIT_KEXEC_HLT);
1354 			while(1)
1355 				native_halt();
1356 		}
1357 	}
1358 }
1359 
1360 /*
1361  * Kick all "offline" CPUs out of mwait on kexec(). See comment in
1362  * mwait_play_dead().
1363  */
smp_kick_mwait_play_dead(void)1364 void smp_kick_mwait_play_dead(void)
1365 {
1366 	u32 newstate = CPUDEAD_MWAIT_KEXEC_HLT;
1367 	struct mwait_cpu_dead *md;
1368 	unsigned int cpu, i;
1369 
1370 	for_each_cpu_andnot(cpu, cpu_present_mask, cpu_online_mask) {
1371 		md = per_cpu_ptr(&mwait_cpu_dead, cpu);
1372 
1373 		/* Does it sit in mwait_play_dead() ? */
1374 		if (READ_ONCE(md->status) != CPUDEAD_MWAIT_WAIT)
1375 			continue;
1376 
1377 		/* Wait up to 5ms */
1378 		for (i = 0; READ_ONCE(md->status) != newstate && i < 1000; i++) {
1379 			/* Bring it out of mwait */
1380 			WRITE_ONCE(md->control, newstate);
1381 			udelay(5);
1382 		}
1383 
1384 		if (READ_ONCE(md->status) != newstate)
1385 			pr_err_once("CPU%u is stuck in mwait_play_dead()\n", cpu);
1386 	}
1387 }
1388 
hlt_play_dead(void)1389 void __noreturn hlt_play_dead(void)
1390 {
1391 	if (__this_cpu_read(cpu_info.x86) >= 4)
1392 		wbinvd();
1393 
1394 	while (1)
1395 		native_halt();
1396 }
1397 
1398 /*
1399  * native_play_dead() is essentially a __noreturn function, but it can't
1400  * be marked as such as the compiler may complain about it.
1401  */
native_play_dead(void)1402 void native_play_dead(void)
1403 {
1404 	if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
1405 		__update_spec_ctrl(0);
1406 
1407 	play_dead_common();
1408 	tboot_shutdown(TB_SHUTDOWN_WFS);
1409 
1410 	mwait_play_dead();
1411 	if (cpuidle_play_dead())
1412 		hlt_play_dead();
1413 }
1414 
1415 #else /* ... !CONFIG_HOTPLUG_CPU */
native_cpu_disable(void)1416 int native_cpu_disable(void)
1417 {
1418 	return -ENOSYS;
1419 }
1420 
native_play_dead(void)1421 void native_play_dead(void)
1422 {
1423 	BUG();
1424 }
1425 
1426 #endif
1427