xref: /linux/arch/arm64/kernel/smp.c (revision 59fff63cc2b75dcfe08f9eeb4b2187d73e53843d)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * SMP initialisation and IPI support
4  * Based on arch/arm/kernel/smp.c
5  *
6  * Copyright (C) 2012 ARM Ltd.
7  */
8 
9 #include <linux/acpi.h>
10 #include <linux/arm_sdei.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/hotplug.h>
16 #include <linux/sched/task_stack.h>
17 #include <linux/interrupt.h>
18 #include <linux/cache.h>
19 #include <linux/profile.h>
20 #include <linux/errno.h>
21 #include <linux/mm.h>
22 #include <linux/err.h>
23 #include <linux/cpu.h>
24 #include <linux/smp.h>
25 #include <linux/seq_file.h>
26 #include <linux/irq.h>
27 #include <linux/irqchip/arm-gic-v3.h>
28 #include <linux/percpu.h>
29 #include <linux/clockchips.h>
30 #include <linux/completion.h>
31 #include <linux/of.h>
32 #include <linux/irq_work.h>
33 #include <linux/kernel_stat.h>
34 #include <linux/kexec.h>
35 #include <linux/kvm_host.h>
36 
37 #include <asm/alternative.h>
38 #include <asm/atomic.h>
39 #include <asm/cacheflush.h>
40 #include <asm/cpu.h>
41 #include <asm/cputype.h>
42 #include <asm/cpu_ops.h>
43 #include <asm/daifflags.h>
44 #include <asm/kvm_mmu.h>
45 #include <asm/mmu_context.h>
46 #include <asm/numa.h>
47 #include <asm/processor.h>
48 #include <asm/smp_plat.h>
49 #include <asm/sections.h>
50 #include <asm/tlbflush.h>
51 #include <asm/ptrace.h>
52 #include <asm/virt.h>
53 
54 #include <trace/events/ipi.h>
55 
56 DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
57 EXPORT_PER_CPU_SYMBOL(cpu_number);
58 
59 /*
60  * as from 2.5, kernels no longer have an init_tasks structure
61  * so we need some other way of telling a new secondary core
62  * where to place its SVC stack
63  */
64 struct secondary_data secondary_data;
65 /* Number of CPUs which aren't online, but looping in kernel text. */
66 static int cpus_stuck_in_kernel;
67 
68 enum ipi_msg_type {
69 	IPI_RESCHEDULE,
70 	IPI_CALL_FUNC,
71 	IPI_CPU_STOP,
72 	IPI_CPU_CRASH_STOP,
73 	IPI_TIMER,
74 	IPI_IRQ_WORK,
75 	IPI_WAKEUP,
76 	NR_IPI
77 };
78 
79 static int ipi_irq_base __read_mostly;
80 static int nr_ipi __read_mostly = NR_IPI;
81 static struct irq_desc *ipi_desc[NR_IPI] __read_mostly;
82 
83 static void ipi_setup(int cpu);
84 
85 #ifdef CONFIG_HOTPLUG_CPU
86 static void ipi_teardown(int cpu);
87 static int op_cpu_kill(unsigned int cpu);
88 #else
89 static inline int op_cpu_kill(unsigned int cpu)
90 {
91 	return -ENOSYS;
92 }
93 #endif
94 
95 
96 /*
97  * Boot a secondary CPU, and assign it the specified idle task.
98  * This also gives us the initial stack to use for this CPU.
99  */
100 static int boot_secondary(unsigned int cpu, struct task_struct *idle)
101 {
102 	const struct cpu_operations *ops = get_cpu_ops(cpu);
103 
104 	if (ops->cpu_boot)
105 		return ops->cpu_boot(cpu);
106 
107 	return -EOPNOTSUPP;
108 }
109 
110 static DECLARE_COMPLETION(cpu_running);
111 
112 int __cpu_up(unsigned int cpu, struct task_struct *idle)
113 {
114 	int ret;
115 	long status;
116 
117 	/*
118 	 * We need to tell the secondary core where to find its stack and the
119 	 * page tables.
120 	 */
121 	secondary_data.task = idle;
122 	update_cpu_boot_status(CPU_MMU_OFF);
123 
124 	/* Now bring the CPU into our world */
125 	ret = boot_secondary(cpu, idle);
126 	if (ret) {
127 		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
128 		return ret;
129 	}
130 
131 	/*
132 	 * CPU was successfully started, wait for it to come online or
133 	 * time out.
134 	 */
135 	wait_for_completion_timeout(&cpu_running,
136 				    msecs_to_jiffies(5000));
137 	if (cpu_online(cpu))
138 		return 0;
139 
140 	pr_crit("CPU%u: failed to come online\n", cpu);
141 	secondary_data.task = NULL;
142 	status = READ_ONCE(secondary_data.status);
143 	if (status == CPU_MMU_OFF)
144 		status = READ_ONCE(__early_cpu_boot_status);
145 
146 	switch (status & CPU_BOOT_STATUS_MASK) {
147 	default:
148 		pr_err("CPU%u: failed in unknown state : 0x%lx\n",
149 		       cpu, status);
150 		cpus_stuck_in_kernel++;
151 		break;
152 	case CPU_KILL_ME:
153 		if (!op_cpu_kill(cpu)) {
154 			pr_crit("CPU%u: died during early boot\n", cpu);
155 			break;
156 		}
157 		pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
158 		fallthrough;
159 	case CPU_STUCK_IN_KERNEL:
160 		pr_crit("CPU%u: is stuck in kernel\n", cpu);
161 		if (status & CPU_STUCK_REASON_52_BIT_VA)
162 			pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);
163 		if (status & CPU_STUCK_REASON_NO_GRAN) {
164 			pr_crit("CPU%u: does not support %luK granule\n",
165 				cpu, PAGE_SIZE / SZ_1K);
166 		}
167 		cpus_stuck_in_kernel++;
168 		break;
169 	case CPU_PANIC_KERNEL:
170 		panic("CPU%u detected unsupported configuration\n", cpu);
171 	}
172 
173 	return -EIO;
174 }
175 
176 static void init_gic_priority_masking(void)
177 {
178 	u32 cpuflags;
179 
180 	if (WARN_ON(!gic_enable_sre()))
181 		return;
182 
183 	cpuflags = read_sysreg(daif);
184 
185 	WARN_ON(!(cpuflags & PSR_I_BIT));
186 	WARN_ON(!(cpuflags & PSR_F_BIT));
187 
188 	gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
189 }
190 
191 /*
192  * This is the secondary CPU boot entry.  We're using this CPUs
193  * idle thread stack, but a set of temporary page tables.
194  */
195 asmlinkage notrace void secondary_start_kernel(void)
196 {
197 	u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
198 	struct mm_struct *mm = &init_mm;
199 	const struct cpu_operations *ops;
200 	unsigned int cpu = smp_processor_id();
201 
202 	/*
203 	 * All kernel threads share the same mm context; grab a
204 	 * reference and switch to it.
205 	 */
206 	mmgrab(mm);
207 	current->active_mm = mm;
208 
209 	/*
210 	 * TTBR0 is only used for the identity mapping at this stage. Make it
211 	 * point to zero page to avoid speculatively fetching new entries.
212 	 */
213 	cpu_uninstall_idmap();
214 
215 	if (system_uses_irq_prio_masking())
216 		init_gic_priority_masking();
217 
218 	rcutree_report_cpu_starting(cpu);
219 	trace_hardirqs_off();
220 
221 	/*
222 	 * If the system has established the capabilities, make sure
223 	 * this CPU ticks all of those. If it doesn't, the CPU will
224 	 * fail to come online.
225 	 */
226 	check_local_cpu_capabilities();
227 
228 	ops = get_cpu_ops(cpu);
229 	if (ops->cpu_postboot)
230 		ops->cpu_postboot();
231 
232 	/*
233 	 * Log the CPU info before it is marked online and might get read.
234 	 */
235 	cpuinfo_store_cpu();
236 	store_cpu_topology(cpu);
237 
238 	/*
239 	 * Enable GIC and timers.
240 	 */
241 	notify_cpu_starting(cpu);
242 
243 	ipi_setup(cpu);
244 
245 	numa_add_cpu(cpu);
246 
247 	/*
248 	 * OK, now it's safe to let the boot CPU continue.  Wait for
249 	 * the CPU migration code to notice that the CPU is online
250 	 * before we continue.
251 	 */
252 	pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",
253 					 cpu, (unsigned long)mpidr,
254 					 read_cpuid_id());
255 	update_cpu_boot_status(CPU_BOOT_SUCCESS);
256 	set_cpu_online(cpu, true);
257 	complete(&cpu_running);
258 
259 	local_daif_restore(DAIF_PROCCTX);
260 
261 	/*
262 	 * OK, it's off to the idle thread for us
263 	 */
264 	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
265 }
266 
267 #ifdef CONFIG_HOTPLUG_CPU
268 static int op_cpu_disable(unsigned int cpu)
269 {
270 	const struct cpu_operations *ops = get_cpu_ops(cpu);
271 
272 	/*
273 	 * If we don't have a cpu_die method, abort before we reach the point
274 	 * of no return. CPU0 may not have an cpu_ops, so test for it.
275 	 */
276 	if (!ops || !ops->cpu_die)
277 		return -EOPNOTSUPP;
278 
279 	/*
280 	 * We may need to abort a hot unplug for some other mechanism-specific
281 	 * reason.
282 	 */
283 	if (ops->cpu_disable)
284 		return ops->cpu_disable(cpu);
285 
286 	return 0;
287 }
288 
289 /*
290  * __cpu_disable runs on the processor to be shutdown.
291  */
292 int __cpu_disable(void)
293 {
294 	unsigned int cpu = smp_processor_id();
295 	int ret;
296 
297 	ret = op_cpu_disable(cpu);
298 	if (ret)
299 		return ret;
300 
301 	remove_cpu_topology(cpu);
302 	numa_remove_cpu(cpu);
303 
304 	/*
305 	 * Take this CPU offline.  Once we clear this, we can't return,
306 	 * and we must not schedule until we're ready to give up the cpu.
307 	 */
308 	set_cpu_online(cpu, false);
309 	ipi_teardown(cpu);
310 
311 	/*
312 	 * OK - migrate IRQs away from this CPU
313 	 */
314 	irq_migrate_all_off_this_cpu();
315 
316 	return 0;
317 }
318 
319 static int op_cpu_kill(unsigned int cpu)
320 {
321 	const struct cpu_operations *ops = get_cpu_ops(cpu);
322 
323 	/*
324 	 * If we have no means of synchronising with the dying CPU, then assume
325 	 * that it is really dead. We can only wait for an arbitrary length of
326 	 * time and hope that it's dead, so let's skip the wait and just hope.
327 	 */
328 	if (!ops->cpu_kill)
329 		return 0;
330 
331 	return ops->cpu_kill(cpu);
332 }
333 
334 /*
335  * Called on the thread which is asking for a CPU to be shutdown after the
336  * shutdown completed.
337  */
338 void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)
339 {
340 	int err;
341 
342 	pr_debug("CPU%u: shutdown\n", cpu);
343 
344 	/*
345 	 * Now that the dying CPU is beyond the point of no return w.r.t.
346 	 * in-kernel synchronisation, try to get the firwmare to help us to
347 	 * verify that it has really left the kernel before we consider
348 	 * clobbering anything it might still be using.
349 	 */
350 	err = op_cpu_kill(cpu);
351 	if (err)
352 		pr_warn("CPU%d may not have shut down cleanly: %d\n", cpu, err);
353 }
354 
355 /*
356  * Called from the idle thread for the CPU which has been shutdown.
357  *
358  */
359 void __noreturn cpu_die(void)
360 {
361 	unsigned int cpu = smp_processor_id();
362 	const struct cpu_operations *ops = get_cpu_ops(cpu);
363 
364 	idle_task_exit();
365 
366 	local_daif_mask();
367 
368 	/* Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose of */
369 	cpuhp_ap_report_dead();
370 
371 	/*
372 	 * Actually shutdown the CPU. This must never fail. The specific hotplug
373 	 * mechanism must perform all required cache maintenance to ensure that
374 	 * no dirty lines are lost in the process of shutting down the CPU.
375 	 */
376 	ops->cpu_die(cpu);
377 
378 	BUG();
379 }
380 #endif
381 
382 static void __cpu_try_die(int cpu)
383 {
384 #ifdef CONFIG_HOTPLUG_CPU
385 	const struct cpu_operations *ops = get_cpu_ops(cpu);
386 
387 	if (ops && ops->cpu_die)
388 		ops->cpu_die(cpu);
389 #endif
390 }
391 
392 /*
393  * Kill the calling secondary CPU, early in bringup before it is turned
394  * online.
395  */
396 void __noreturn cpu_die_early(void)
397 {
398 	int cpu = smp_processor_id();
399 
400 	pr_crit("CPU%d: will not boot\n", cpu);
401 
402 	/* Mark this CPU absent */
403 	set_cpu_present(cpu, 0);
404 	rcutree_report_cpu_dead();
405 
406 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
407 		update_cpu_boot_status(CPU_KILL_ME);
408 		__cpu_try_die(cpu);
409 	}
410 
411 	update_cpu_boot_status(CPU_STUCK_IN_KERNEL);
412 
413 	cpu_park_loop();
414 }
415 
416 static void __init hyp_mode_check(void)
417 {
418 	if (is_hyp_mode_available())
419 		pr_info("CPU: All CPU(s) started at EL2\n");
420 	else if (is_hyp_mode_mismatched())
421 		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
422 			   "CPU: CPUs started in inconsistent modes");
423 	else
424 		pr_info("CPU: All CPU(s) started at EL1\n");
425 	if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {
426 		kvm_compute_layout();
427 		kvm_apply_hyp_relocations();
428 	}
429 }
430 
431 void __init smp_cpus_done(unsigned int max_cpus)
432 {
433 	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
434 	setup_cpu_features();
435 	hyp_mode_check();
436 	apply_alternatives_all();
437 	mark_linear_text_alias_ro();
438 }
439 
440 void __init smp_prepare_boot_cpu(void)
441 {
442 	/*
443 	 * The runtime per-cpu areas have been allocated by
444 	 * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be
445 	 * freed shortly, so we must move over to the runtime per-cpu area.
446 	 */
447 	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
448 	cpuinfo_store_boot_cpu();
449 
450 	/*
451 	 * We now know enough about the boot CPU to apply the
452 	 * alternatives that cannot wait until interrupt handling
453 	 * and/or scheduling is enabled.
454 	 */
455 	apply_boot_alternatives();
456 
457 	/* Conditionally switch to GIC PMR for interrupt masking */
458 	if (system_uses_irq_prio_masking())
459 		init_gic_priority_masking();
460 
461 	kasan_init_hw_tags();
462 }
463 
464 /*
465  * Duplicate MPIDRs are a recipe for disaster. Scan all initialized
466  * entries and check for duplicates. If any is found just ignore the
467  * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
468  * matching valid MPIDR values.
469  */
470 static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)
471 {
472 	unsigned int i;
473 
474 	for (i = 1; (i < cpu) && (i < NR_CPUS); i++)
475 		if (cpu_logical_map(i) == hwid)
476 			return true;
477 	return false;
478 }
479 
480 /*
481  * Initialize cpu operations for a logical cpu and
482  * set it in the possible mask on success
483  */
484 static int __init smp_cpu_setup(int cpu)
485 {
486 	const struct cpu_operations *ops;
487 
488 	if (init_cpu_ops(cpu))
489 		return -ENODEV;
490 
491 	ops = get_cpu_ops(cpu);
492 	if (ops->cpu_init(cpu))
493 		return -ENODEV;
494 
495 	set_cpu_possible(cpu, true);
496 
497 	return 0;
498 }
499 
500 static bool bootcpu_valid __initdata;
501 static unsigned int cpu_count = 1;
502 
503 #ifdef CONFIG_ACPI
504 static struct acpi_madt_generic_interrupt cpu_madt_gicc[NR_CPUS];
505 
506 struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)
507 {
508 	return &cpu_madt_gicc[cpu];
509 }
510 EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc);
511 
512 /*
513  * acpi_map_gic_cpu_interface - parse processor MADT entry
514  *
515  * Carry out sanity checks on MADT processor entry and initialize
516  * cpu_logical_map on success
517  */
518 static void __init
519 acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
520 {
521 	u64 hwid = processor->arm_mpidr;
522 
523 	if (!(processor->flags & ACPI_MADT_ENABLED)) {
524 		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
525 		return;
526 	}
527 
528 	if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) {
529 		pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);
530 		return;
531 	}
532 
533 	if (is_mpidr_duplicate(cpu_count, hwid)) {
534 		pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
535 		return;
536 	}
537 
538 	/* Check if GICC structure of boot CPU is available in the MADT */
539 	if (cpu_logical_map(0) == hwid) {
540 		if (bootcpu_valid) {
541 			pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",
542 			       hwid);
543 			return;
544 		}
545 		bootcpu_valid = true;
546 		cpu_madt_gicc[0] = *processor;
547 		return;
548 	}
549 
550 	if (cpu_count >= NR_CPUS)
551 		return;
552 
553 	/* map the logical cpu id to cpu MPIDR */
554 	set_cpu_logical_map(cpu_count, hwid);
555 
556 	cpu_madt_gicc[cpu_count] = *processor;
557 
558 	/*
559 	 * Set-up the ACPI parking protocol cpu entries
560 	 * while initializing the cpu_logical_map to
561 	 * avoid parsing MADT entries multiple times for
562 	 * nothing (ie a valid cpu_logical_map entry should
563 	 * contain a valid parking protocol data set to
564 	 * initialize the cpu if the parking protocol is
565 	 * the only available enable method).
566 	 */
567 	acpi_set_mailbox_entry(cpu_count, processor);
568 
569 	cpu_count++;
570 }
571 
572 static int __init
573 acpi_parse_gic_cpu_interface(union acpi_subtable_headers *header,
574 			     const unsigned long end)
575 {
576 	struct acpi_madt_generic_interrupt *processor;
577 
578 	processor = (struct acpi_madt_generic_interrupt *)header;
579 	if (BAD_MADT_GICC_ENTRY(processor, end))
580 		return -EINVAL;
581 
582 	acpi_table_print_madt_entry(&header->common);
583 
584 	acpi_map_gic_cpu_interface(processor);
585 
586 	return 0;
587 }
588 
589 static void __init acpi_parse_and_init_cpus(void)
590 {
591 	int i;
592 
593 	/*
594 	 * do a walk of MADT to determine how many CPUs
595 	 * we have including disabled CPUs, and get information
596 	 * we need for SMP init.
597 	 */
598 	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
599 				      acpi_parse_gic_cpu_interface, 0);
600 
601 	/*
602 	 * In ACPI, SMP and CPU NUMA information is provided in separate
603 	 * static tables, namely the MADT and the SRAT.
604 	 *
605 	 * Thus, it is simpler to first create the cpu logical map through
606 	 * an MADT walk and then map the logical cpus to their node ids
607 	 * as separate steps.
608 	 */
609 	acpi_map_cpus_to_nodes();
610 
611 	for (i = 0; i < nr_cpu_ids; i++)
612 		early_map_cpu_to_node(i, acpi_numa_get_nid(i));
613 }
614 #else
615 #define acpi_parse_and_init_cpus(...)	do { } while (0)
616 #endif
617 
618 /*
619  * Enumerate the possible CPU set from the device tree and build the
620  * cpu logical map array containing MPIDR values related to logical
621  * cpus. Assumes that cpu_logical_map(0) has already been initialized.
622  */
623 static void __init of_parse_and_init_cpus(void)
624 {
625 	struct device_node *dn;
626 
627 	for_each_of_cpu_node(dn) {
628 		u64 hwid = of_get_cpu_hwid(dn, 0);
629 
630 		if (hwid & ~MPIDR_HWID_BITMASK)
631 			goto next;
632 
633 		if (is_mpidr_duplicate(cpu_count, hwid)) {
634 			pr_err("%pOF: duplicate cpu reg properties in the DT\n",
635 				dn);
636 			goto next;
637 		}
638 
639 		/*
640 		 * The numbering scheme requires that the boot CPU
641 		 * must be assigned logical id 0. Record it so that
642 		 * the logical map built from DT is validated and can
643 		 * be used.
644 		 */
645 		if (hwid == cpu_logical_map(0)) {
646 			if (bootcpu_valid) {
647 				pr_err("%pOF: duplicate boot cpu reg property in DT\n",
648 					dn);
649 				goto next;
650 			}
651 
652 			bootcpu_valid = true;
653 			early_map_cpu_to_node(0, of_node_to_nid(dn));
654 
655 			/*
656 			 * cpu_logical_map has already been
657 			 * initialized and the boot cpu doesn't need
658 			 * the enable-method so continue without
659 			 * incrementing cpu.
660 			 */
661 			continue;
662 		}
663 
664 		if (cpu_count >= NR_CPUS)
665 			goto next;
666 
667 		pr_debug("cpu logical map 0x%llx\n", hwid);
668 		set_cpu_logical_map(cpu_count, hwid);
669 
670 		early_map_cpu_to_node(cpu_count, of_node_to_nid(dn));
671 next:
672 		cpu_count++;
673 	}
674 }
675 
676 /*
677  * Enumerate the possible CPU set from the device tree or ACPI and build the
678  * cpu logical map array containing MPIDR values related to logical
679  * cpus. Assumes that cpu_logical_map(0) has already been initialized.
680  */
681 void __init smp_init_cpus(void)
682 {
683 	int i;
684 
685 	if (acpi_disabled)
686 		of_parse_and_init_cpus();
687 	else
688 		acpi_parse_and_init_cpus();
689 
690 	if (cpu_count > nr_cpu_ids)
691 		pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
692 			cpu_count, nr_cpu_ids);
693 
694 	if (!bootcpu_valid) {
695 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
696 		return;
697 	}
698 
699 	/*
700 	 * We need to set the cpu_logical_map entries before enabling
701 	 * the cpus so that cpu processor description entries (DT cpu nodes
702 	 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
703 	 * with entries in cpu_logical_map while initializing the cpus.
704 	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
705 	 */
706 	for (i = 1; i < nr_cpu_ids; i++) {
707 		if (cpu_logical_map(i) != INVALID_HWID) {
708 			if (smp_cpu_setup(i))
709 				set_cpu_logical_map(i, INVALID_HWID);
710 		}
711 	}
712 }
713 
714 void __init smp_prepare_cpus(unsigned int max_cpus)
715 {
716 	const struct cpu_operations *ops;
717 	int err;
718 	unsigned int cpu;
719 	unsigned int this_cpu;
720 
721 	init_cpu_topology();
722 
723 	this_cpu = smp_processor_id();
724 	store_cpu_topology(this_cpu);
725 	numa_store_cpu_info(this_cpu);
726 	numa_add_cpu(this_cpu);
727 
728 	/*
729 	 * If UP is mandated by "nosmp" (which implies "maxcpus=0"), don't set
730 	 * secondary CPUs present.
731 	 */
732 	if (max_cpus == 0)
733 		return;
734 
735 	/*
736 	 * Initialise the present map (which describes the set of CPUs
737 	 * actually populated at the present time) and release the
738 	 * secondaries from the bootloader.
739 	 */
740 	for_each_possible_cpu(cpu) {
741 
742 		per_cpu(cpu_number, cpu) = cpu;
743 
744 		if (cpu == smp_processor_id())
745 			continue;
746 
747 		ops = get_cpu_ops(cpu);
748 		if (!ops)
749 			continue;
750 
751 		err = ops->cpu_prepare(cpu);
752 		if (err)
753 			continue;
754 
755 		set_cpu_present(cpu, true);
756 		numa_store_cpu_info(cpu);
757 	}
758 }
759 
760 static const char *ipi_types[NR_IPI] __tracepoint_string = {
761 	[IPI_RESCHEDULE]	= "Rescheduling interrupts",
762 	[IPI_CALL_FUNC]		= "Function call interrupts",
763 	[IPI_CPU_STOP]		= "CPU stop interrupts",
764 	[IPI_CPU_CRASH_STOP]	= "CPU stop (for crash dump) interrupts",
765 	[IPI_TIMER]		= "Timer broadcast interrupts",
766 	[IPI_IRQ_WORK]		= "IRQ work interrupts",
767 	[IPI_WAKEUP]		= "CPU wake-up interrupts",
768 };
769 
770 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
771 
772 unsigned long irq_err_count;
773 
774 int arch_show_interrupts(struct seq_file *p, int prec)
775 {
776 	unsigned int cpu, i;
777 
778 	for (i = 0; i < NR_IPI; i++) {
779 		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
780 			   prec >= 4 ? " " : "");
781 		for_each_online_cpu(cpu)
782 			seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
783 		seq_printf(p, "      %s\n", ipi_types[i]);
784 	}
785 
786 	seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
787 	return 0;
788 }
789 
790 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
791 {
792 	smp_cross_call(mask, IPI_CALL_FUNC);
793 }
794 
795 void arch_send_call_function_single_ipi(int cpu)
796 {
797 	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
798 }
799 
800 #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
801 void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
802 {
803 	smp_cross_call(mask, IPI_WAKEUP);
804 }
805 #endif
806 
807 #ifdef CONFIG_IRQ_WORK
808 void arch_irq_work_raise(void)
809 {
810 	smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
811 }
812 #endif
813 
814 static void __noreturn local_cpu_stop(void)
815 {
816 	set_cpu_online(smp_processor_id(), false);
817 
818 	local_daif_mask();
819 	sdei_mask_local_cpu();
820 	cpu_park_loop();
821 }
822 
823 /*
824  * We need to implement panic_smp_self_stop() for parallel panic() calls, so
825  * that cpu_online_mask gets correctly updated and smp_send_stop() can skip
826  * CPUs that have already stopped themselves.
827  */
828 void __noreturn panic_smp_self_stop(void)
829 {
830 	local_cpu_stop();
831 }
832 
833 #ifdef CONFIG_KEXEC_CORE
834 static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0);
835 #endif
836 
837 static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
838 {
839 #ifdef CONFIG_KEXEC_CORE
840 	crash_save_cpu(regs, cpu);
841 
842 	atomic_dec(&waiting_for_crash_ipi);
843 
844 	local_irq_disable();
845 	sdei_mask_local_cpu();
846 
847 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
848 		__cpu_try_die(cpu);
849 
850 	/* just in case */
851 	cpu_park_loop();
852 #else
853 	BUG();
854 #endif
855 }
856 
857 /*
858  * Main handler for inter-processor interrupts
859  */
860 static void do_handle_IPI(int ipinr)
861 {
862 	unsigned int cpu = smp_processor_id();
863 
864 	if ((unsigned)ipinr < NR_IPI)
865 		trace_ipi_entry(ipi_types[ipinr]);
866 
867 	switch (ipinr) {
868 	case IPI_RESCHEDULE:
869 		scheduler_ipi();
870 		break;
871 
872 	case IPI_CALL_FUNC:
873 		generic_smp_call_function_interrupt();
874 		break;
875 
876 	case IPI_CPU_STOP:
877 		local_cpu_stop();
878 		break;
879 
880 	case IPI_CPU_CRASH_STOP:
881 		if (IS_ENABLED(CONFIG_KEXEC_CORE)) {
882 			ipi_cpu_crash_stop(cpu, get_irq_regs());
883 
884 			unreachable();
885 		}
886 		break;
887 
888 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
889 	case IPI_TIMER:
890 		tick_receive_broadcast();
891 		break;
892 #endif
893 
894 #ifdef CONFIG_IRQ_WORK
895 	case IPI_IRQ_WORK:
896 		irq_work_run();
897 		break;
898 #endif
899 
900 #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
901 	case IPI_WAKEUP:
902 		WARN_ONCE(!acpi_parking_protocol_valid(cpu),
903 			  "CPU%u: Wake-up IPI outside the ACPI parking protocol\n",
904 			  cpu);
905 		break;
906 #endif
907 
908 	default:
909 		pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
910 		break;
911 	}
912 
913 	if ((unsigned)ipinr < NR_IPI)
914 		trace_ipi_exit(ipi_types[ipinr]);
915 }
916 
917 static irqreturn_t ipi_handler(int irq, void *data)
918 {
919 	do_handle_IPI(irq - ipi_irq_base);
920 	return IRQ_HANDLED;
921 }
922 
923 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
924 {
925 	trace_ipi_raise(target, ipi_types[ipinr]);
926 	__ipi_send_mask(ipi_desc[ipinr], target);
927 }
928 
929 static void ipi_setup(int cpu)
930 {
931 	int i;
932 
933 	if (WARN_ON_ONCE(!ipi_irq_base))
934 		return;
935 
936 	for (i = 0; i < nr_ipi; i++)
937 		enable_percpu_irq(ipi_irq_base + i, 0);
938 }
939 
940 #ifdef CONFIG_HOTPLUG_CPU
941 static void ipi_teardown(int cpu)
942 {
943 	int i;
944 
945 	if (WARN_ON_ONCE(!ipi_irq_base))
946 		return;
947 
948 	for (i = 0; i < nr_ipi; i++)
949 		disable_percpu_irq(ipi_irq_base + i);
950 }
951 #endif
952 
953 void __init set_smp_ipi_range(int ipi_base, int n)
954 {
955 	int i;
956 
957 	WARN_ON(n < NR_IPI);
958 	nr_ipi = min(n, NR_IPI);
959 
960 	for (i = 0; i < nr_ipi; i++) {
961 		int err;
962 
963 		err = request_percpu_irq(ipi_base + i, ipi_handler,
964 					 "IPI", &cpu_number);
965 		WARN_ON(err);
966 
967 		ipi_desc[i] = irq_to_desc(ipi_base + i);
968 		irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
969 	}
970 
971 	ipi_irq_base = ipi_base;
972 
973 	/* Setup the boot CPU immediately */
974 	ipi_setup(smp_processor_id());
975 }
976 
977 void arch_smp_send_reschedule(int cpu)
978 {
979 	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
980 }
981 
982 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
983 void tick_broadcast(const struct cpumask *mask)
984 {
985 	smp_cross_call(mask, IPI_TIMER);
986 }
987 #endif
988 
989 /*
990  * The number of CPUs online, not counting this CPU (which may not be
991  * fully online and so not counted in num_online_cpus()).
992  */
993 static inline unsigned int num_other_online_cpus(void)
994 {
995 	unsigned int this_cpu_online = cpu_online(smp_processor_id());
996 
997 	return num_online_cpus() - this_cpu_online;
998 }
999 
1000 void smp_send_stop(void)
1001 {
1002 	unsigned long timeout;
1003 
1004 	if (num_other_online_cpus()) {
1005 		cpumask_t mask;
1006 
1007 		cpumask_copy(&mask, cpu_online_mask);
1008 		cpumask_clear_cpu(smp_processor_id(), &mask);
1009 
1010 		if (system_state <= SYSTEM_RUNNING)
1011 			pr_crit("SMP: stopping secondary CPUs\n");
1012 		smp_cross_call(&mask, IPI_CPU_STOP);
1013 	}
1014 
1015 	/* Wait up to one second for other CPUs to stop */
1016 	timeout = USEC_PER_SEC;
1017 	while (num_other_online_cpus() && timeout--)
1018 		udelay(1);
1019 
1020 	if (num_other_online_cpus())
1021 		pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
1022 			cpumask_pr_args(cpu_online_mask));
1023 
1024 	sdei_mask_local_cpu();
1025 }
1026 
1027 #ifdef CONFIG_KEXEC_CORE
1028 void crash_smp_send_stop(void)
1029 {
1030 	static int cpus_stopped;
1031 	cpumask_t mask;
1032 	unsigned long timeout;
1033 
1034 	/*
1035 	 * This function can be called twice in panic path, but obviously
1036 	 * we execute this only once.
1037 	 */
1038 	if (cpus_stopped)
1039 		return;
1040 
1041 	cpus_stopped = 1;
1042 
1043 	/*
1044 	 * If this cpu is the only one alive at this point in time, online or
1045 	 * not, there are no stop messages to be sent around, so just back out.
1046 	 */
1047 	if (num_other_online_cpus() == 0)
1048 		goto skip_ipi;
1049 
1050 	cpumask_copy(&mask, cpu_online_mask);
1051 	cpumask_clear_cpu(smp_processor_id(), &mask);
1052 
1053 	atomic_set(&waiting_for_crash_ipi, num_other_online_cpus());
1054 
1055 	pr_crit("SMP: stopping secondary CPUs\n");
1056 	smp_cross_call(&mask, IPI_CPU_CRASH_STOP);
1057 
1058 	/* Wait up to one second for other CPUs to stop */
1059 	timeout = USEC_PER_SEC;
1060 	while ((atomic_read(&waiting_for_crash_ipi) > 0) && timeout--)
1061 		udelay(1);
1062 
1063 	if (atomic_read(&waiting_for_crash_ipi) > 0)
1064 		pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
1065 			cpumask_pr_args(&mask));
1066 
1067 skip_ipi:
1068 	sdei_mask_local_cpu();
1069 	sdei_handler_abort();
1070 }
1071 
1072 bool smp_crash_stop_failed(void)
1073 {
1074 	return (atomic_read(&waiting_for_crash_ipi) > 0);
1075 }
1076 #endif
1077 
1078 static bool have_cpu_die(void)
1079 {
1080 #ifdef CONFIG_HOTPLUG_CPU
1081 	int any_cpu = raw_smp_processor_id();
1082 	const struct cpu_operations *ops = get_cpu_ops(any_cpu);
1083 
1084 	if (ops && ops->cpu_die)
1085 		return true;
1086 #endif
1087 	return false;
1088 }
1089 
1090 bool cpus_are_stuck_in_kernel(void)
1091 {
1092 	bool smp_spin_tables = (num_possible_cpus() > 1 && !have_cpu_die());
1093 
1094 	return !!cpus_stuck_in_kernel || smp_spin_tables ||
1095 		is_protected_kvm_enabled();
1096 }
1097