1 /*-
2 * Copyright (c) 1996, by Steve Passe
3 * Copyright (c) 2003, by Peter Wemm
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. The name of the developer may NOT be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include "opt_acpi.h"
29 #ifdef __i386__
30 #include "opt_apic.h"
31 #endif
32 #include "opt_cpu.h"
33 #include "opt_ddb.h"
34 #include "opt_gdb.h"
35 #include "opt_kstack_pages.h"
36 #include "opt_pmap.h"
37 #include "opt_sched.h"
38 #include "opt_smp.h"
39 #include "opt_stack.h"
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/asan.h>
44 #include <sys/bus.h>
45 #include <sys/cons.h> /* cngetc() */
46 #include <sys/cpuset.h>
47 #include <sys/csan.h>
48 #include <sys/interrupt.h>
49 #include <sys/kdb.h>
50 #include <sys/kernel.h>
51 #include <sys/ktr.h>
52 #include <sys/lock.h>
53 #include <sys/malloc.h>
54 #include <sys/memrange.h>
55 #include <sys/msan.h>
56 #include <sys/mutex.h>
57 #include <sys/pcpu.h>
58 #include <sys/proc.h>
59 #include <sys/sched.h>
60 #include <sys/smp.h>
61 #include <sys/sysctl.h>
62
63 #include <vm/vm.h>
64 #include <vm/vm_param.h>
65 #include <vm/pmap.h>
66 #include <vm/vm_kern.h>
67 #include <vm/vm_extern.h>
68 #include <vm/vm_map.h>
69
70 #include <x86/apicreg.h>
71 #include <machine/clock.h>
72 #include <machine/cpu.h>
73 #include <machine/cputypes.h>
74 #include <x86/mca.h>
75 #include <machine/md_var.h>
76 #include <machine/pcb.h>
77 #include <machine/psl.h>
78 #include <machine/smp.h>
79 #include <machine/specialreg.h>
80 #include <machine/stack.h>
81 #include <x86/ucode.h>
82
83 #ifdef DEV_ACPI
84 #include <contrib/dev/acpica/include/acpi.h>
85 #include <dev/acpica/acpivar.h>
86 #endif
87
88 static MALLOC_DEFINE(M_CPUS, "cpus", "CPU items");
89
90 int mp_naps; /* # of Applications processors */
91 int boot_cpu_id = -1; /* designated BSP */
92
93 /* AP uses this during bootstrap. Do not staticize. */
94 char *bootSTK;
95 int bootAP;
96
97 /* Free these after use */
98 void *bootstacks[MAXCPU];
99 void *dpcpu;
100
101 struct susppcb **susppcbs;
102
103 #ifdef COUNT_IPIS
104 /* Interrupt counts. */
105 static u_long *ipi_preempt_counts[MAXCPU];
106 static u_long *ipi_ast_counts[MAXCPU];
107 u_long *ipi_invltlb_counts[MAXCPU];
108 u_long *ipi_invlrng_counts[MAXCPU];
109 u_long *ipi_invlpg_counts[MAXCPU];
110 u_long *ipi_invlcache_counts[MAXCPU];
111 u_long *ipi_rendezvous_counts[MAXCPU];
112 static u_long *ipi_hardclock_counts[MAXCPU];
113 #endif
114
115 /* Default cpu_ops implementation. */
116 struct cpu_ops cpu_ops;
117
118 /*
119 * Local data and functions.
120 */
121
122 static volatile cpuset_t ipi_stop_nmi_pending;
123
124 volatile cpuset_t resuming_cpus;
125 volatile cpuset_t toresume_cpus;
126
127 /* used to hold the AP's until we are ready to release them */
128 static int ap_boot_lock;
129
130 /* Set to 1 once we're ready to let the APs out of the pen. */
131 volatile int aps_ready = 0;
132
133 /*
134 * Store data from cpu_add() until later in the boot when we actually setup
135 * the APs.
136 */
137 struct cpu_info *cpu_info;
138 int *apic_cpuids;
139 int cpu_apic_ids[MAXCPU];
140 _Static_assert(MAXCPU <= MAX_APIC_ID,
141 "MAXCPU cannot be larger that MAX_APIC_ID");
142 _Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID,
143 "xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID");
144
145 static void release_aps(void *dummy);
146 static void cpustop_handler_post(u_int cpu);
147
148 static int hyperthreading_allowed = 1;
149 SYSCTL_INT(_machdep, OID_AUTO, hyperthreading_allowed, CTLFLAG_RDTUN,
150 &hyperthreading_allowed, 0, "Use Intel HTT logical CPUs");
151
152 static int hyperthreading_intr_allowed = 0;
153 SYSCTL_INT(_machdep, OID_AUTO, hyperthreading_intr_allowed, CTLFLAG_RDTUN,
154 &hyperthreading_intr_allowed, 0,
155 "Allow interrupts on HTT logical CPUs");
156
157 static int intr_apic_id_limit = -1;
158 SYSCTL_INT(_machdep, OID_AUTO, intr_apic_id_limit, CTLFLAG_RDTUN,
159 &intr_apic_id_limit, 0,
160 "Maximum permitted APIC ID for interrupt delivery (-1 is unlimited)");
161
162 static struct topo_node topo_root;
163
164 static int pkg_id_shift;
165 static int node_id_shift;
166 static int core_id_shift;
167 static int disabled_cpus;
168
169 struct cache_info {
170 int id_shift;
171 int present;
172 } static caches[MAX_CACHE_LEVELS];
173
174 static bool stop_mwait = false;
175 SYSCTL_BOOL(_machdep, OID_AUTO, stop_mwait, CTLFLAG_RWTUN, &stop_mwait, 0,
176 "Use MONITOR/MWAIT when stopping CPU, if available");
177
178 void
mem_range_AP_init(void)179 mem_range_AP_init(void)
180 {
181
182 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP)
183 mem_range_softc.mr_op->initAP(&mem_range_softc);
184 }
185
186 /*
187 * Compute ceil(log2(x)). Returns -1 if x is zero.
188 */
189 static __inline int
mask_width(u_int x)190 mask_width(u_int x)
191 {
192
193 return (x == 0 ? -1 : order_base_2(x));
194 }
195
196 /*
197 * Add a cache level to the cache topology description.
198 */
199 static int
add_deterministic_cache(int type,int level,int share_count)200 add_deterministic_cache(int type, int level, int share_count)
201 {
202
203 if (type == 0)
204 return (0);
205 if (type > 3) {
206 printf("unexpected cache type %d\n", type);
207 return (1);
208 }
209 if (type == 2) /* ignore instruction cache */
210 return (1);
211 if (level == 0 || level > MAX_CACHE_LEVELS) {
212 printf("unexpected cache level %d\n", level);
213 return (1);
214 }
215
216 if (caches[level - 1].present) {
217 printf("WARNING: multiple entries for L%u data cache\n", level);
218 printf("%u => %u\n", caches[level - 1].id_shift,
219 mask_width(share_count));
220 }
221 caches[level - 1].id_shift = mask_width(share_count);
222 caches[level - 1].present = 1;
223
224 if (caches[level - 1].id_shift > pkg_id_shift) {
225 printf("WARNING: L%u data cache covers more "
226 "APIC IDs than a package (%u > %u)\n", level,
227 caches[level - 1].id_shift, pkg_id_shift);
228 caches[level - 1].id_shift = pkg_id_shift;
229 }
230 if (caches[level - 1].id_shift < core_id_shift) {
231 printf("WARNING: L%u data cache covers fewer "
232 "APIC IDs than a core (%u < %u)\n", level,
233 caches[level - 1].id_shift, core_id_shift);
234 caches[level - 1].id_shift = core_id_shift;
235 }
236
237 return (1);
238 }
239
240 /*
241 * Determine topology of processing units and caches for AMD CPUs.
242 * See:
243 * - AMD CPUID Specification (Publication # 25481)
244 * - BKDG for AMD NPT Family 0Fh Processors (Publication # 32559)
245 * - BKDG For AMD Family 10h Processors (Publication # 31116)
246 * - BKDG For AMD Family 15h Models 00h-0Fh Processors (Publication # 42301)
247 * - BKDG For AMD Family 16h Models 00h-0Fh Processors (Publication # 48751)
248 * - PPR For AMD Family 17h Models 00h-0Fh Processors (Publication # 54945)
249 */
250 static void
topo_probe_amd(void)251 topo_probe_amd(void)
252 {
253 u_int p[4];
254 uint64_t v;
255 int level;
256 int nodes_per_socket;
257 int share_count;
258 int type;
259 int i;
260
261 /* No multi-core capability. */
262 if ((amd_feature2 & AMDID2_CMP) == 0)
263 return;
264
265 /*
266 * XXX Lack of an AMD IOMMU driver prevents use of APIC IDs above
267 * xAPIC_MAX_APIC_ID. This is a workaround so we boot and function on
268 * AMD systems with high thread counts, albeit with reduced interrupt
269 * performance.
270 *
271 * We should really set the limit to xAPIC_MAX_APIC_ID by default, and
272 * have the IOMMU driver increase it. That way if a driver is present
273 * but disabled, or is otherwise not able to route the interrupts, the
274 * system can fall back to a functional state. That will require a more
275 * substantial change though, including having the IOMMU initialize
276 * earlier.
277 */
278 if (intr_apic_id_limit == -1)
279 intr_apic_id_limit = xAPIC_MAX_APIC_ID;
280
281 /* For families 10h and newer. */
282 pkg_id_shift = (cpu_procinfo2 & AMDID_COREID_SIZE) >>
283 AMDID_COREID_SIZE_SHIFT;
284
285 /* For 0Fh family. */
286 if (pkg_id_shift == 0)
287 pkg_id_shift =
288 mask_width((cpu_procinfo2 & AMDID_CMP_CORES) + 1);
289
290 /*
291 * Families prior to 16h define the following value as
292 * cores per compute unit and we don't really care about the AMD
293 * compute units at the moment. Perhaps we should treat them as
294 * cores and cores within the compute units as hardware threads,
295 * but that's up for debate.
296 * Later families define the value as threads per compute unit,
297 * so we are following AMD's nomenclature here.
298 */
299 if ((amd_feature2 & AMDID2_TOPOLOGY) != 0 &&
300 CPUID_TO_FAMILY(cpu_id) >= 0x16) {
301 cpuid_count(0x8000001e, 0, p);
302 share_count = ((p[1] >> 8) & 0xff) + 1;
303 core_id_shift = mask_width(share_count);
304
305 /*
306 * For Zen (17h), gather Nodes per Processor. Each node is a
307 * Zeppelin die; TR and EPYC CPUs will have multiple dies per
308 * package. Communication latency between dies is higher than
309 * within them.
310 */
311 nodes_per_socket = ((p[2] >> 8) & 0x7) + 1;
312 node_id_shift = pkg_id_shift - mask_width(nodes_per_socket);
313 }
314
315 if ((amd_feature2 & AMDID2_TOPOLOGY) != 0) {
316 for (i = 0; ; i++) {
317 cpuid_count(0x8000001d, i, p);
318 type = p[0] & 0x1f;
319 level = (p[0] >> 5) & 0x7;
320 share_count = 1 + ((p[0] >> 14) & 0xfff);
321
322 if (!add_deterministic_cache(type, level, share_count))
323 break;
324 }
325 } else {
326 if (cpu_exthigh >= 0x80000005) {
327 cpuid_count(0x80000005, 0, p);
328 if (((p[2] >> 24) & 0xff) != 0) {
329 caches[0].id_shift = 0;
330 caches[0].present = 1;
331 }
332 }
333 if (cpu_exthigh >= 0x80000006) {
334 cpuid_count(0x80000006, 0, p);
335 if (((p[2] >> 16) & 0xffff) != 0) {
336 caches[1].id_shift = 0;
337 caches[1].present = 1;
338 }
339 if (((p[3] >> 18) & 0x3fff) != 0) {
340 nodes_per_socket = 1;
341 if ((amd_feature2 & AMDID2_NODE_ID) != 0) {
342 /*
343 * Handle multi-node processors that
344 * have multiple chips, each with its
345 * own L3 cache, on the same die.
346 */
347 v = rdmsr(0xc001100c);
348 nodes_per_socket = 1 + ((v >> 3) & 0x7);
349 }
350 caches[2].id_shift =
351 pkg_id_shift - mask_width(nodes_per_socket);
352 caches[2].present = 1;
353 }
354 }
355 }
356 }
357
358 /*
359 * Determine topology of processing units for Intel CPUs
360 * using CPUID Leaf 1 and Leaf 4, if supported.
361 * See:
362 * - Intel 64 Architecture Processor Topology Enumeration
363 * - Intel 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
364 * Volume 3A: System Programming Guide, PROGRAMMING CONSIDERATIONS
365 * FOR HARDWARE MULTI-THREADING CAPABLE PROCESSORS
366 */
367 static void
topo_probe_intel_0x4(void)368 topo_probe_intel_0x4(void)
369 {
370 u_int p[4];
371 int max_cores;
372 int max_logical;
373
374 /* Both zero and one here mean one logical processor per package. */
375 max_logical = (cpu_feature & CPUID_HTT) != 0 ?
376 (cpu_procinfo & CPUID_HTT_CORES) >> 16 : 1;
377 if (max_logical <= 1)
378 return;
379
380 if (cpu_high >= 0x4) {
381 cpuid_count(0x04, 0, p);
382 max_cores = ((p[0] >> 26) & 0x3f) + 1;
383 } else
384 max_cores = 1;
385
386 core_id_shift = mask_width(max_logical/max_cores);
387 KASSERT(core_id_shift >= 0,
388 ("intel topo: max_cores > max_logical\n"));
389 pkg_id_shift = core_id_shift + mask_width(max_cores);
390 }
391
392 /*
393 * Determine topology of processing units for Intel CPUs
394 * using CPUID Leaf 1Fh or 0Bh, if supported.
395 * See:
396 * - Intel 64 Architecture Processor Topology Enumeration
397 * - Intel 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
398 * Volume 3A: System Programming Guide, PROGRAMMING CONSIDERATIONS
399 * FOR HARDWARE MULTI-THREADING CAPABLE PROCESSORS
400 */
401 static void
topo_probe_intel_0xb(void)402 topo_probe_intel_0xb(void)
403 {
404 u_int leaf;
405 u_int p[4] = { 0 };
406 int bits;
407 int type;
408 int i;
409
410 /* Prefer leaf 1Fh (V2 Extended Topology Enumeration). */
411 if (cpu_high >= 0x1f) {
412 leaf = 0x1f;
413 cpuid_count(leaf, 0, p);
414 }
415 /* Fall back to leaf 0Bh (Extended Topology Enumeration). */
416 if (p[1] == 0) {
417 leaf = 0x0b;
418 cpuid_count(leaf, 0, p);
419 }
420 /* Fall back to leaf 04h (Deterministic Cache Parameters). */
421 if (p[1] == 0) {
422 topo_probe_intel_0x4();
423 return;
424 }
425
426 /* We only support three levels for now. */
427 for (i = 0; ; i++) {
428 cpuid_count(leaf, i, p);
429
430 bits = p[0] & 0x1f;
431 type = (p[2] >> 8) & 0xff;
432
433 if (type == 0)
434 break;
435
436 if (type == CPUID_TYPE_SMT)
437 core_id_shift = bits;
438 else if (type == CPUID_TYPE_CORE)
439 pkg_id_shift = bits;
440 else if (bootverbose)
441 printf("Topology level type %d shift: %d\n", type, bits);
442 }
443
444 if (pkg_id_shift < core_id_shift) {
445 printf("WARNING: core covers more APIC IDs than a package\n");
446 core_id_shift = pkg_id_shift;
447 }
448 }
449
450 /*
451 * Determine topology of caches for Intel CPUs.
452 * See:
453 * - Intel 64 Architecture Processor Topology Enumeration
454 * - Intel 64 and IA-32 Architectures Software Developer’s Manual
455 * Volume 2A: Instruction Set Reference, A-M,
456 * CPUID instruction
457 */
458 static void
topo_probe_intel_caches(void)459 topo_probe_intel_caches(void)
460 {
461 u_int p[4];
462 int level;
463 int share_count;
464 int type;
465 int i;
466
467 if (cpu_high < 0x4) {
468 /*
469 * Available cache level and sizes can be determined
470 * via CPUID leaf 2, but that requires a huge table of hardcoded
471 * values, so for now just assume L1 and L2 caches potentially
472 * shared only by HTT processing units, if HTT is present.
473 */
474 caches[0].id_shift = pkg_id_shift;
475 caches[0].present = 1;
476 caches[1].id_shift = pkg_id_shift;
477 caches[1].present = 1;
478 return;
479 }
480
481 for (i = 0; ; i++) {
482 cpuid_count(0x4, i, p);
483 type = p[0] & 0x1f;
484 level = (p[0] >> 5) & 0x7;
485 share_count = 1 + ((p[0] >> 14) & 0xfff);
486
487 if (!add_deterministic_cache(type, level, share_count))
488 break;
489 }
490 }
491
492 /*
493 * Determine topology of processing units and caches for Intel CPUs.
494 * See:
495 * - Intel 64 Architecture Processor Topology Enumeration
496 */
497 static void
topo_probe_intel(void)498 topo_probe_intel(void)
499 {
500
501 /*
502 * Note that 0x1 <= cpu_high < 4 case should be
503 * compatible with topo_probe_intel_0x4() logic when
504 * CPUID.1:EBX[23:16] > 0 (cpu_cores will be 1)
505 * or it should trigger the fallback otherwise.
506 */
507 if (cpu_high >= 0xb)
508 topo_probe_intel_0xb();
509 else if (cpu_high >= 0x1)
510 topo_probe_intel_0x4();
511
512 topo_probe_intel_caches();
513 }
514
515 /*
516 * Topology information is queried only on BSP, on which this
517 * code runs and for which it can query CPUID information.
518 * Then topology is extrapolated on all packages using an
519 * assumption that APIC ID to hardware component ID mapping is
520 * homogenious.
521 * That doesn't necesserily imply that the topology is uniform.
522 */
523 void
topo_probe(void)524 topo_probe(void)
525 {
526 static int cpu_topo_probed = 0;
527 struct x86_topo_layer {
528 int type;
529 int subtype;
530 int id_shift;
531 } topo_layers[MAX_CACHE_LEVELS + 5];
532 struct topo_node *parent;
533 struct topo_node *node;
534 int layer;
535 int nlayers;
536 int node_id;
537 int i;
538 #if defined(DEV_ACPI) && MAXMEMDOM > 1
539 int d, domain;
540 #endif
541
542 if (cpu_topo_probed)
543 return;
544
545 CPU_ZERO(&logical_cpus_mask);
546
547 if (mp_ncpus <= 1)
548 ; /* nothing */
549 else if (cpu_vendor_id == CPU_VENDOR_AMD ||
550 cpu_vendor_id == CPU_VENDOR_HYGON)
551 topo_probe_amd();
552 else if (cpu_vendor_id == CPU_VENDOR_INTEL)
553 topo_probe_intel();
554
555 KASSERT(pkg_id_shift >= core_id_shift,
556 ("bug in APIC topology discovery"));
557
558 nlayers = 0;
559 bzero(topo_layers, sizeof(topo_layers));
560
561 topo_layers[nlayers].type = TOPO_TYPE_PKG;
562 topo_layers[nlayers].id_shift = pkg_id_shift;
563 if (bootverbose)
564 printf("Package ID shift: %u\n", topo_layers[nlayers].id_shift);
565 nlayers++;
566
567 if (pkg_id_shift > node_id_shift && node_id_shift != 0) {
568 topo_layers[nlayers].type = TOPO_TYPE_GROUP;
569 topo_layers[nlayers].id_shift = node_id_shift;
570 if (bootverbose)
571 printf("Node ID shift: %u\n",
572 topo_layers[nlayers].id_shift);
573 nlayers++;
574 }
575
576 /*
577 * Consider all caches to be within a package/chip
578 * and "in front" of all sub-components like
579 * cores and hardware threads.
580 */
581 for (i = MAX_CACHE_LEVELS - 1; i >= 0; --i) {
582 if (caches[i].present) {
583 if (node_id_shift != 0)
584 KASSERT(caches[i].id_shift <= node_id_shift,
585 ("bug in APIC topology discovery"));
586 KASSERT(caches[i].id_shift <= pkg_id_shift,
587 ("bug in APIC topology discovery"));
588 KASSERT(caches[i].id_shift >= core_id_shift,
589 ("bug in APIC topology discovery"));
590
591 topo_layers[nlayers].type = TOPO_TYPE_CACHE;
592 topo_layers[nlayers].subtype = i + 1;
593 topo_layers[nlayers].id_shift = caches[i].id_shift;
594 if (bootverbose)
595 printf("L%u cache ID shift: %u\n",
596 topo_layers[nlayers].subtype,
597 topo_layers[nlayers].id_shift);
598 nlayers++;
599 }
600 }
601
602 if (pkg_id_shift > core_id_shift) {
603 topo_layers[nlayers].type = TOPO_TYPE_CORE;
604 topo_layers[nlayers].id_shift = core_id_shift;
605 if (bootverbose)
606 printf("Core ID shift: %u\n",
607 topo_layers[nlayers].id_shift);
608 nlayers++;
609 }
610
611 topo_layers[nlayers].type = TOPO_TYPE_PU;
612 topo_layers[nlayers].id_shift = 0;
613 nlayers++;
614
615 #if defined(DEV_ACPI) && MAXMEMDOM > 1
616 if (vm_ndomains > 1) {
617 for (layer = 0; layer < nlayers; ++layer) {
618 for (i = 0; i <= max_apic_id; ++i) {
619 if ((i & ((1 << topo_layers[layer].id_shift) - 1)) == 0)
620 domain = -1;
621 if (!cpu_info[i].cpu_present)
622 continue;
623 d = acpi_pxm_get_cpu_locality(i);
624 if (domain >= 0 && domain != d)
625 break;
626 domain = d;
627 }
628 if (i > max_apic_id)
629 break;
630 }
631 KASSERT(layer < nlayers, ("NUMA domain smaller than PU"));
632 memmove(&topo_layers[layer+1], &topo_layers[layer],
633 sizeof(*topo_layers) * (nlayers - layer));
634 topo_layers[layer].type = TOPO_TYPE_NODE;
635 topo_layers[layer].subtype = CG_SHARE_NONE;
636 nlayers++;
637 }
638 #endif
639
640 topo_init_root(&topo_root);
641 for (i = 0; i <= max_apic_id; ++i) {
642 if (!cpu_info[i].cpu_present)
643 continue;
644
645 parent = &topo_root;
646 for (layer = 0; layer < nlayers; ++layer) {
647 #if defined(DEV_ACPI) && MAXMEMDOM > 1
648 if (topo_layers[layer].type == TOPO_TYPE_NODE) {
649 node_id = acpi_pxm_get_cpu_locality(i);
650 } else
651 #endif
652 node_id = i >> topo_layers[layer].id_shift;
653 parent = topo_add_node_by_hwid(parent, node_id,
654 topo_layers[layer].type,
655 topo_layers[layer].subtype);
656 }
657 }
658
659 parent = &topo_root;
660 for (layer = 0; layer < nlayers; ++layer) {
661 #if defined(DEV_ACPI) && MAXMEMDOM > 1
662 if (topo_layers[layer].type == TOPO_TYPE_NODE)
663 node_id = acpi_pxm_get_cpu_locality(boot_cpu_id);
664 else
665 #endif
666 node_id = boot_cpu_id >> topo_layers[layer].id_shift;
667 node = topo_find_node_by_hwid(parent, node_id,
668 topo_layers[layer].type,
669 topo_layers[layer].subtype);
670 topo_promote_child(node);
671 parent = node;
672 }
673
674 cpu_topo_probed = 1;
675 }
676
677 /*
678 * Assign logical CPU IDs to local APICs.
679 */
680 void
assign_cpu_ids(void)681 assign_cpu_ids(void)
682 {
683 struct topo_node *node;
684 u_int smt_mask;
685 int nhyper;
686
687 smt_mask = (1u << core_id_shift) - 1;
688
689 /*
690 * Assign CPU IDs to local APIC IDs and disable any CPUs
691 * beyond MAXCPU. CPU 0 is always assigned to the BSP.
692 */
693 mp_ncpus = 0;
694 nhyper = 0;
695 TOPO_FOREACH(node, &topo_root) {
696 if (node->type != TOPO_TYPE_PU)
697 continue;
698
699 if ((node->hwid & smt_mask) != (boot_cpu_id & smt_mask))
700 cpu_info[node->hwid].cpu_hyperthread = 1;
701
702 if (resource_disabled("lapic", node->hwid)) {
703 if (node->hwid != boot_cpu_id)
704 cpu_info[node->hwid].cpu_disabled = 1;
705 else
706 printf("Cannot disable BSP, APIC ID = %d\n",
707 node->hwid);
708 }
709
710 if (!hyperthreading_allowed &&
711 cpu_info[node->hwid].cpu_hyperthread)
712 cpu_info[node->hwid].cpu_disabled = 1;
713
714 if (mp_ncpus >= MAXCPU)
715 cpu_info[node->hwid].cpu_disabled = 1;
716
717 if (cpu_info[node->hwid].cpu_disabled) {
718 disabled_cpus++;
719 continue;
720 }
721
722 if (cpu_info[node->hwid].cpu_hyperthread)
723 nhyper++;
724
725 cpu_apic_ids[mp_ncpus] = node->hwid;
726 apic_cpuids[node->hwid] = mp_ncpus;
727 topo_set_pu_id(node, mp_ncpus);
728 mp_ncpus++;
729 }
730
731 KASSERT(mp_maxid >= mp_ncpus - 1,
732 ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid,
733 mp_ncpus));
734
735 mp_ncores = mp_ncpus - nhyper;
736 smp_threads_per_core = mp_ncpus / mp_ncores;
737 }
738
739 /*
740 * Print various information about the SMP system hardware and setup.
741 */
742 void
cpu_mp_announce(void)743 cpu_mp_announce(void)
744 {
745 struct topo_node *node;
746 const char *hyperthread;
747 struct topo_analysis topology;
748
749 printf("FreeBSD/SMP: ");
750 if (topo_analyze(&topo_root, 1, &topology)) {
751 printf("%d package(s)", topology.entities[TOPO_LEVEL_PKG]);
752 if (topology.entities[TOPO_LEVEL_GROUP] > 1)
753 printf(" x %d groups",
754 topology.entities[TOPO_LEVEL_GROUP]);
755 if (topology.entities[TOPO_LEVEL_CACHEGROUP] > 1)
756 printf(" x %d cache groups",
757 topology.entities[TOPO_LEVEL_CACHEGROUP]);
758 if (topology.entities[TOPO_LEVEL_CORE] > 0)
759 printf(" x %d core(s)",
760 topology.entities[TOPO_LEVEL_CORE]);
761 if (topology.entities[TOPO_LEVEL_THREAD] > 1)
762 printf(" x %d hardware threads",
763 topology.entities[TOPO_LEVEL_THREAD]);
764 } else {
765 printf("Non-uniform topology");
766 }
767 printf("\n");
768
769 if (disabled_cpus) {
770 printf("FreeBSD/SMP Online: ");
771 if (topo_analyze(&topo_root, 0, &topology)) {
772 printf("%d package(s)",
773 topology.entities[TOPO_LEVEL_PKG]);
774 if (topology.entities[TOPO_LEVEL_GROUP] > 1)
775 printf(" x %d groups",
776 topology.entities[TOPO_LEVEL_GROUP]);
777 if (topology.entities[TOPO_LEVEL_CACHEGROUP] > 1)
778 printf(" x %d cache groups",
779 topology.entities[TOPO_LEVEL_CACHEGROUP]);
780 if (topology.entities[TOPO_LEVEL_CORE] > 0)
781 printf(" x %d core(s)",
782 topology.entities[TOPO_LEVEL_CORE]);
783 if (topology.entities[TOPO_LEVEL_THREAD] > 1)
784 printf(" x %d hardware threads",
785 topology.entities[TOPO_LEVEL_THREAD]);
786 } else {
787 printf("Non-uniform topology");
788 }
789 printf("\n");
790 }
791
792 if (!bootverbose)
793 return;
794
795 TOPO_FOREACH(node, &topo_root) {
796 switch (node->type) {
797 case TOPO_TYPE_PKG:
798 printf("Package HW ID = %u\n", node->hwid);
799 break;
800 case TOPO_TYPE_CORE:
801 printf("\tCore HW ID = %u\n", node->hwid);
802 break;
803 case TOPO_TYPE_PU:
804 if (cpu_info[node->hwid].cpu_hyperthread)
805 hyperthread = "/HT";
806 else
807 hyperthread = "";
808
809 if (node->subtype == 0)
810 printf("\t\tCPU (AP%s): APIC ID: %u"
811 "(disabled)\n", hyperthread, node->hwid);
812 else if (node->id == 0)
813 printf("\t\tCPU0 (BSP): APIC ID: %u\n",
814 node->hwid);
815 else
816 printf("\t\tCPU%u (AP%s): APIC ID: %u\n",
817 node->id, hyperthread, node->hwid);
818 break;
819 default:
820 /* ignored */
821 break;
822 }
823 }
824 }
825
826 /*
827 * Add a scheduling group, a group of logical processors sharing
828 * a particular cache (and, thus having an affinity), to the scheduling
829 * topology.
830 * This function recursively works on lower level caches.
831 */
832 static void
x86topo_add_sched_group(struct topo_node * root,struct cpu_group * cg_root)833 x86topo_add_sched_group(struct topo_node *root, struct cpu_group *cg_root)
834 {
835 struct topo_node *node;
836 int nchildren;
837 int ncores;
838 int i;
839
840 KASSERT(root->type == TOPO_TYPE_SYSTEM || root->type == TOPO_TYPE_CACHE ||
841 root->type == TOPO_TYPE_NODE || root->type == TOPO_TYPE_GROUP,
842 ("x86topo_add_sched_group: bad type: %u", root->type));
843 CPU_COPY(&root->cpuset, &cg_root->cg_mask);
844 cg_root->cg_count = root->cpu_count;
845 if (root->type == TOPO_TYPE_CACHE)
846 cg_root->cg_level = root->subtype;
847 else
848 cg_root->cg_level = CG_SHARE_NONE;
849 if (root->type == TOPO_TYPE_NODE)
850 cg_root->cg_flags = CG_FLAG_NODE;
851 else
852 cg_root->cg_flags = 0;
853
854 /*
855 * Check how many core nodes we have under the given root node.
856 * If we have multiple logical processors, but not multiple
857 * cores, then those processors must be hardware threads.
858 */
859 ncores = 0;
860 node = root;
861 while (node != NULL) {
862 if (node->type != TOPO_TYPE_CORE) {
863 node = topo_next_node(root, node);
864 continue;
865 }
866
867 ncores++;
868 node = topo_next_nonchild_node(root, node);
869 }
870
871 if (cg_root->cg_level != CG_SHARE_NONE &&
872 root->cpu_count > 1 && ncores < 2)
873 cg_root->cg_flags |= CG_FLAG_SMT;
874
875 /*
876 * Find out how many cache nodes we have under the given root node.
877 * We ignore cache nodes that cover all the same processors as the
878 * root node. Also, we do not descend below found cache nodes.
879 * That is, we count top-level "non-redundant" caches under the root
880 * node.
881 */
882 nchildren = 0;
883 node = root;
884 while (node != NULL) {
885 /*
886 * When some APICs are disabled by tunables, nodes can end up
887 * with an empty cpuset. Nodes with an empty cpuset will be
888 * translated into cpu groups with empty cpusets. smp_topo_fill
889 * will then set cg_first and cg_last to -1. This isn't
890 * correctly handled in all functions. E.g. when
891 * cpu_search_lowest and cpu_search_highest loop through all
892 * cpus, they call CPU_ISSET on cpu -1 which ends up in a
893 * general protection fault.
894 *
895 * We could fix the scheduler to handle empty cpu groups
896 * correctly. Nevertheless, empty cpu groups are causing
897 * overhead for no value. So, it makes more sense to just don't
898 * create them.
899 */
900 if (CPU_EMPTY(&node->cpuset)) {
901 node = topo_next_node(root, node);
902 continue;
903 }
904 if (CPU_CMP(&node->cpuset, &root->cpuset) == 0) {
905 if (node->type == TOPO_TYPE_CACHE &&
906 cg_root->cg_level < node->subtype)
907 cg_root->cg_level = node->subtype;
908 if (node->type == TOPO_TYPE_NODE)
909 cg_root->cg_flags |= CG_FLAG_NODE;
910 node = topo_next_node(root, node);
911 continue;
912 }
913 if (node->type != TOPO_TYPE_GROUP &&
914 node->type != TOPO_TYPE_NODE &&
915 node->type != TOPO_TYPE_CACHE) {
916 node = topo_next_node(root, node);
917 continue;
918 }
919 nchildren++;
920 node = topo_next_nonchild_node(root, node);
921 }
922
923 /*
924 * We are not interested in nodes including only one CPU each.
925 */
926 if (nchildren == root->cpu_count)
927 return;
928
929 /*
930 * We are not interested in nodes without children.
931 */
932 cg_root->cg_children = nchildren;
933 if (nchildren == 0)
934 return;
935
936 cg_root->cg_child = smp_topo_alloc(nchildren);
937
938 /*
939 * Now find again the same cache nodes as above and recursively
940 * build scheduling topologies for them.
941 */
942 node = root;
943 i = 0;
944 while (node != NULL) {
945 if ((node->type != TOPO_TYPE_GROUP &&
946 node->type != TOPO_TYPE_NODE &&
947 node->type != TOPO_TYPE_CACHE) ||
948 CPU_CMP(&node->cpuset, &root->cpuset) == 0 ||
949 CPU_EMPTY(&node->cpuset)) {
950 node = topo_next_node(root, node);
951 continue;
952 }
953 cg_root->cg_child[i].cg_parent = cg_root;
954 x86topo_add_sched_group(node, &cg_root->cg_child[i]);
955 i++;
956 node = topo_next_nonchild_node(root, node);
957 }
958 }
959
960 /*
961 * Build the MI scheduling topology from the discovered hardware topology.
962 */
963 struct cpu_group *
cpu_topo(void)964 cpu_topo(void)
965 {
966 struct cpu_group *cg_root;
967
968 if (mp_ncpus <= 1)
969 return (smp_topo_none());
970
971 cg_root = smp_topo_alloc(1);
972 x86topo_add_sched_group(&topo_root, cg_root);
973 return (cg_root);
974 }
975
976 static void
cpu_alloc(void * dummy __unused)977 cpu_alloc(void *dummy __unused)
978 {
979 /*
980 * Dynamically allocate the arrays that depend on the
981 * maximum APIC ID.
982 */
983 cpu_info = malloc(sizeof(*cpu_info) * (max_apic_id + 1), M_CPUS,
984 M_WAITOK | M_ZERO);
985 apic_cpuids = malloc(sizeof(*apic_cpuids) * (max_apic_id + 1), M_CPUS,
986 M_WAITOK | M_ZERO);
987 }
988 SYSINIT(cpu_alloc, SI_SUB_CPU, SI_ORDER_FIRST, cpu_alloc, NULL);
989
990 /*
991 * Add a logical CPU to the topology.
992 */
993 void
cpu_add(u_int apic_id,char boot_cpu)994 cpu_add(u_int apic_id, char boot_cpu)
995 {
996
997 if (apic_id > max_apic_id)
998 panic("SMP: APIC ID %d too high", apic_id);
999
1000 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %u added twice",
1001 apic_id));
1002 cpu_info[apic_id].cpu_present = 1;
1003 if (boot_cpu) {
1004 KASSERT(boot_cpu_id == -1,
1005 ("CPU %u claims to be BSP, but CPU %u already is", apic_id,
1006 boot_cpu_id));
1007 boot_cpu_id = apic_id;
1008 cpu_info[apic_id].cpu_bsp = 1;
1009 }
1010 if (bootverbose)
1011 printf("SMP: Added CPU %u (%s)\n", apic_id, boot_cpu ? "BSP" :
1012 "AP");
1013 }
1014
1015 void
cpu_mp_setmaxid(void)1016 cpu_mp_setmaxid(void)
1017 {
1018
1019 /*
1020 * mp_ncpus and mp_maxid should be already set by calls to cpu_add().
1021 * If there were no calls to cpu_add() assume this is a UP system.
1022 */
1023 if (mp_ncpus == 0)
1024 mp_ncpus = 1;
1025 }
1026
1027 int
cpu_mp_probe(void)1028 cpu_mp_probe(void)
1029 {
1030
1031 /*
1032 * Always record BSP in CPU map so that the mbuf init code works
1033 * correctly.
1034 */
1035 CPU_SETOF(0, &all_cpus);
1036 return (mp_ncpus > 1);
1037 }
1038
1039 /*
1040 * AP CPU's call this to initialize themselves.
1041 */
1042 void
init_secondary_tail(void)1043 init_secondary_tail(void)
1044 {
1045 u_int cpuid;
1046
1047 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace));
1048
1049 /*
1050 * On real hardware, switch to x2apic mode if possible. Do it
1051 * after aps_ready was signalled, to avoid manipulating the
1052 * mode while BSP might still want to send some IPI to us
1053 * (second startup IPI is ignored on modern hardware etc).
1054 */
1055 lapic_xapic_mode();
1056
1057 /* Initialize the PAT MSR. */
1058 pmap_init_pat();
1059
1060 /* set up CPU registers and state */
1061 cpu_setregs();
1062
1063 /* set up SSE/NX */
1064 initializecpu();
1065
1066 /* set up FPU state on the AP */
1067 #ifdef __amd64__
1068 fpuinit();
1069 #else
1070 npxinit(false);
1071 #endif
1072
1073 if (cpu_ops.cpu_init)
1074 cpu_ops.cpu_init();
1075
1076 /* A quick check from sanity claus */
1077 cpuid = PCPU_GET(cpuid);
1078 if (PCPU_GET(apic_id) != lapic_id()) {
1079 printf("SMP: cpuid = %d\n", cpuid);
1080 printf("SMP: actual apic_id = %d\n", lapic_id());
1081 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
1082 panic("cpuid mismatch! boom!!");
1083 }
1084
1085 /* Initialize curthread. */
1086 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
1087 PCPU_SET(curthread, PCPU_GET(idlethread));
1088 schedinit_ap();
1089
1090 mca_init();
1091
1092 /* Init local apic for irq's */
1093 lapic_setup(1);
1094
1095 /* Set memory range attributes for this CPU to match the BSP */
1096 mem_range_AP_init();
1097
1098 /*
1099 * Use naive spinning lock instead of the real spinlock, since
1100 * printfs() below might take a very long time and trigger
1101 * spinlock timeout panics. This is the only use of the
1102 * ap_boot_lock anyway.
1103 */
1104 while (atomic_cmpset_acq_int(&ap_boot_lock, 0, 1) == 0)
1105 ia32_pause();
1106
1107 smp_cpus++;
1108
1109 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
1110 if (bootverbose)
1111 printf("SMP: AP CPU #%d Launched!\n", cpuid);
1112 else
1113 printf("%s%d%s", smp_cpus == 2 ? "Launching APs: " : "",
1114 cpuid, smp_cpus == mp_ncpus ? "\n" : " ");
1115
1116 /* Determine if we are a logical CPU. */
1117 if (cpu_info[PCPU_GET(apic_id)].cpu_hyperthread)
1118 CPU_SET(cpuid, &logical_cpus_mask);
1119
1120 if (bootverbose)
1121 lapic_dump("AP");
1122
1123 if (smp_cpus == mp_ncpus) {
1124 /* enable IPI's, tlb shootdown, freezes etc */
1125 atomic_store_rel_int(&smp_started, 1);
1126 }
1127
1128 atomic_store_rel_int(&ap_boot_lock, 0);
1129
1130 #ifdef __amd64__
1131 if (pmap_pcid_enabled)
1132 load_cr4(rcr4() | CR4_PCIDE);
1133 load_ds(_udatasel);
1134 load_es(_udatasel);
1135 load_fs(_ufssel);
1136 #endif
1137
1138 /* Wait until all the AP's are up. */
1139 while (atomic_load_acq_int(&smp_started) == 0)
1140 ia32_pause();
1141
1142 kcsan_cpu_init(cpuid);
1143
1144 sched_ap_entry();
1145
1146 panic("scheduler returned us to %s", __func__);
1147 /* NOTREACHED */
1148 }
1149
1150 static void
smp_after_idle_runnable(void * arg __unused)1151 smp_after_idle_runnable(void *arg __unused)
1152 {
1153 int cpu;
1154
1155 if (mp_ncpus == 1)
1156 return;
1157
1158 KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__));
1159
1160 /*
1161 * Wait for all APs to handle an interrupt. After that, we know that
1162 * the APs have entered the scheduler at least once, so the boot stacks
1163 * are safe to free.
1164 */
1165 smp_rendezvous(smp_no_rendezvous_barrier, NULL,
1166 smp_no_rendezvous_barrier, NULL);
1167
1168 for (cpu = 1; cpu < mp_ncpus; cpu++) {
1169 kmem_free(bootstacks[cpu], kstack_pages * PAGE_SIZE);
1170 }
1171 }
1172 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
1173 smp_after_idle_runnable, NULL);
1174
1175 /*
1176 * We tell the I/O APIC code about all the CPUs we want to receive
1177 * interrupts. If we don't want certain CPUs to receive IRQs we
1178 * can simply not tell the I/O APIC code about them in this function.
1179 * We also do not tell it about the BSP since it tells itself about
1180 * the BSP internally to work with UP kernels and on UP machines.
1181 */
1182 void
set_interrupt_apic_ids(void)1183 set_interrupt_apic_ids(void)
1184 {
1185 u_int i, apic_id;
1186
1187 for (i = 0; i < MAXCPU; i++) {
1188 apic_id = cpu_apic_ids[i];
1189 if (apic_id == -1)
1190 continue;
1191 if (cpu_info[apic_id].cpu_bsp)
1192 continue;
1193 if (cpu_info[apic_id].cpu_disabled)
1194 continue;
1195 if (intr_apic_id_limit >= 0 && apic_id > intr_apic_id_limit)
1196 continue;
1197
1198 /* Don't let hyperthreads service interrupts. */
1199 if (cpu_info[apic_id].cpu_hyperthread &&
1200 !hyperthreading_intr_allowed)
1201 continue;
1202
1203 intr_add_cpu(i);
1204 }
1205 }
1206
1207 #ifdef COUNT_XINVLTLB_HITS
1208 u_int xhits_gbl[MAXCPU];
1209 u_int xhits_pg[MAXCPU];
1210 u_int xhits_rng[MAXCPU];
1211 static SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1212 "");
1213 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
1214 sizeof(xhits_gbl), "IU", "");
1215 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
1216 sizeof(xhits_pg), "IU", "");
1217 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng,
1218 sizeof(xhits_rng), "IU", "");
1219
1220 u_int ipi_global;
1221 u_int ipi_page;
1222 u_int ipi_range;
1223 u_int ipi_range_size;
1224 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, "");
1225 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
1226 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
1227 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
1228 0, "");
1229 #endif /* COUNT_XINVLTLB_HITS */
1230
1231 /*
1232 * Init and startup IPI.
1233 */
1234 void
ipi_startup(int apic_id,int vector)1235 ipi_startup(int apic_id, int vector)
1236 {
1237
1238 /*
1239 * This attempts to follow the algorithm described in the
1240 * Intel Multiprocessor Specification v1.4 in section B.4.
1241 * For each IPI, we allow the local APIC ~20us to deliver the
1242 * IPI. If that times out, we panic.
1243 */
1244
1245 /*
1246 * first we do an INIT IPI: this INIT IPI might be run, resetting
1247 * and running the target CPU. OR this INIT IPI might be latched (P5
1248 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1249 * ignored.
1250 */
1251 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL |
1252 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
1253 lapic_ipi_wait(100);
1254
1255 /* Explicitly deassert the INIT IPI. */
1256 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL |
1257 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT,
1258 apic_id);
1259
1260 DELAY(10000); /* wait ~10mS */
1261
1262 /*
1263 * next we do a STARTUP IPI: the previous INIT IPI might still be
1264 * latched, (P5 bug) this 1st STARTUP would then terminate
1265 * immediately, and the previously started INIT IPI would continue. OR
1266 * the previous INIT IPI has already run. and this STARTUP IPI will
1267 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1268 * will run.
1269 */
1270 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
1271 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
1272 vector, apic_id);
1273 if (!lapic_ipi_wait(100))
1274 panic("Failed to deliver first STARTUP IPI to APIC %d",
1275 apic_id);
1276 DELAY(200); /* wait ~200uS */
1277
1278 /*
1279 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1280 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1281 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1282 * recognized after hardware RESET or INIT IPI.
1283 */
1284 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
1285 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
1286 vector, apic_id);
1287 if (!lapic_ipi_wait(100))
1288 panic("Failed to deliver second STARTUP IPI to APIC %d",
1289 apic_id);
1290
1291 DELAY(200); /* wait ~200uS */
1292 }
1293
1294 static bool
ipi_bitmap_set(int cpu,u_int ipi)1295 ipi_bitmap_set(int cpu, u_int ipi)
1296 {
1297 u_int bitmap, old, new;
1298 u_int *cpu_bitmap;
1299
1300 bitmap = 1 << ipi;
1301 cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
1302 old = *cpu_bitmap;
1303 for (;;) {
1304 if ((old & bitmap) != 0)
1305 break;
1306 new = old | bitmap;
1307 if (atomic_fcmpset_int(cpu_bitmap, &old, new))
1308 break;
1309 }
1310 return (old != 0);
1311 }
1312
1313 /*
1314 * Send an IPI to specified CPU handling the bitmap logic.
1315 */
1316 static void
ipi_send_cpu(int cpu,u_int ipi)1317 ipi_send_cpu(int cpu, u_int ipi)
1318 {
1319
1320 KASSERT((u_int)cpu < MAXCPU && cpu_apic_ids[cpu] != -1,
1321 ("IPI to non-existent CPU %d", cpu));
1322
1323 if (IPI_IS_BITMAPED(ipi)) {
1324 if (ipi_bitmap_set(cpu, ipi))
1325 return;
1326 ipi = IPI_BITMAP_VECTOR;
1327 }
1328 lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
1329 }
1330
1331 void
ipi_bitmap_handler(struct trapframe * frame)1332 ipi_bitmap_handler(struct trapframe *frame)
1333 {
1334 struct trapframe *oldframe;
1335 struct thread *td;
1336 int cpu = PCPU_GET(cpuid);
1337 u_int ipi_bitmap;
1338
1339 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
1340 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
1341
1342 td = curthread;
1343 ipi_bitmap = atomic_readandclear_int(&cpuid_to_pcpu[cpu]->
1344 pc_ipi_bitmap);
1345
1346 /*
1347 * sched_preempt() must be called to clear the pending preempt
1348 * IPI to enable delivery of further preempts. However, the
1349 * critical section will cause extra scheduler lock thrashing
1350 * when used unconditionally. Only critical_enter() if
1351 * hardclock must also run, which requires the section entry.
1352 */
1353 if (ipi_bitmap & (1 << IPI_HARDCLOCK))
1354 critical_enter();
1355
1356 td->td_intr_nesting_level++;
1357 oldframe = td->td_intr_frame;
1358 td->td_intr_frame = frame;
1359 #if defined(STACK) || defined(DDB)
1360 if (ipi_bitmap & (1 << IPI_TRACE))
1361 stack_capture_intr();
1362 #endif
1363 if (ipi_bitmap & (1 << IPI_PREEMPT)) {
1364 #ifdef COUNT_IPIS
1365 (*ipi_preempt_counts[cpu])++;
1366 #endif
1367 sched_preempt(td);
1368 }
1369 if (ipi_bitmap & (1 << IPI_AST)) {
1370 #ifdef COUNT_IPIS
1371 (*ipi_ast_counts[cpu])++;
1372 #endif
1373 /* Nothing to do for AST */
1374 }
1375 if (ipi_bitmap & (1 << IPI_HARDCLOCK)) {
1376 #ifdef COUNT_IPIS
1377 (*ipi_hardclock_counts[cpu])++;
1378 #endif
1379 hardclockintr();
1380 }
1381 td->td_intr_frame = oldframe;
1382 td->td_intr_nesting_level--;
1383 if (ipi_bitmap & (1 << IPI_HARDCLOCK))
1384 critical_exit();
1385 }
1386
1387 /*
1388 * send an IPI to a set of cpus.
1389 */
1390 void
ipi_selected(cpuset_t cpus,u_int ipi)1391 ipi_selected(cpuset_t cpus, u_int ipi)
1392 {
1393 int cpu;
1394
1395 /*
1396 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1397 * of help in order to understand what is the source.
1398 * Set the mask of receiving CPUs for this purpose.
1399 */
1400 if (ipi == IPI_STOP_HARD)
1401 CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &cpus);
1402
1403 CPU_FOREACH_ISSET(cpu, &cpus) {
1404 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
1405 ipi_send_cpu(cpu, ipi);
1406 }
1407 }
1408
1409 /*
1410 * send an IPI to a specific CPU.
1411 */
1412 void
ipi_cpu(int cpu,u_int ipi)1413 ipi_cpu(int cpu, u_int ipi)
1414 {
1415
1416 /*
1417 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1418 * of help in order to understand what is the source.
1419 * Set the mask of receiving CPUs for this purpose.
1420 */
1421 if (ipi == IPI_STOP_HARD)
1422 CPU_SET_ATOMIC(cpu, &ipi_stop_nmi_pending);
1423
1424 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
1425 ipi_send_cpu(cpu, ipi);
1426 }
1427
1428 /*
1429 * send an IPI to all CPUs EXCEPT myself
1430 */
1431 void
ipi_all_but_self(u_int ipi)1432 ipi_all_but_self(u_int ipi)
1433 {
1434 cpuset_t other_cpus;
1435 int cpu, c;
1436
1437 if (mp_ncpus == 1)
1438 return;
1439
1440 /*
1441 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1442 * of help in order to understand what is the source.
1443 * Set the mask of receiving CPUs for this purpose.
1444 */
1445 if (ipi == IPI_STOP_HARD) {
1446 other_cpus = all_cpus;
1447 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
1448 CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &other_cpus);
1449 }
1450
1451 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1452 if (IPI_IS_BITMAPED(ipi)) {
1453 cpu = PCPU_GET(cpuid);
1454 CPU_FOREACH(c) {
1455 if (c != cpu)
1456 ipi_bitmap_set(c, ipi);
1457 }
1458 ipi = IPI_BITMAP_VECTOR;
1459 }
1460 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
1461 }
1462
1463 void
ipi_self_from_nmi(u_int vector)1464 ipi_self_from_nmi(u_int vector)
1465 {
1466
1467 lapic_ipi_vectored(vector, APIC_IPI_DEST_SELF);
1468
1469 /* Wait for IPI to finish. */
1470 if (!lapic_ipi_wait(50000)) {
1471 if (KERNEL_PANICKED())
1472 return;
1473 else
1474 panic("APIC: IPI is stuck");
1475 }
1476 }
1477
1478 int
ipi_nmi_handler(void)1479 ipi_nmi_handler(void)
1480 {
1481 u_int cpuid;
1482
1483 /*
1484 * As long as there is not a simple way to know about a NMI's
1485 * source, if the bitmask for the current CPU is present in
1486 * the global pending bitword an IPI_STOP_HARD has been issued
1487 * and should be handled.
1488 */
1489 cpuid = PCPU_GET(cpuid);
1490 if (!CPU_ISSET(cpuid, &ipi_stop_nmi_pending))
1491 return (1);
1492
1493 CPU_CLR_ATOMIC(cpuid, &ipi_stop_nmi_pending);
1494 cpustop_handler();
1495 return (0);
1496 }
1497
1498 int nmi_kdb_lock;
1499
1500 void
nmi_call_kdb_smp(u_int type,struct trapframe * frame)1501 nmi_call_kdb_smp(u_int type, struct trapframe *frame)
1502 {
1503 int cpu;
1504 bool call_post;
1505
1506 cpu = PCPU_GET(cpuid);
1507 if (atomic_cmpset_acq_int(&nmi_kdb_lock, 0, 1)) {
1508 nmi_call_kdb(cpu, type, frame);
1509 call_post = false;
1510 } else {
1511 savectx(&stoppcbs[cpu]);
1512 CPU_SET_ATOMIC(cpu, &stopped_cpus);
1513 while (!atomic_cmpset_acq_int(&nmi_kdb_lock, 0, 1))
1514 ia32_pause();
1515 call_post = true;
1516 }
1517 atomic_store_rel_int(&nmi_kdb_lock, 0);
1518 if (call_post)
1519 cpustop_handler_post(cpu);
1520 }
1521
1522 /*
1523 * Handle an IPI_STOP by saving our current context and spinning (or mwaiting,
1524 * if available) until we are resumed.
1525 */
1526 void
cpustop_handler(void)1527 cpustop_handler(void)
1528 {
1529 struct monitorbuf *mb;
1530 u_int cpu;
1531 bool use_mwait;
1532
1533 cpu = PCPU_GET(cpuid);
1534
1535 savectx(&stoppcbs[cpu]);
1536
1537 use_mwait = (stop_mwait && (cpu_feature2 & CPUID2_MON) != 0 &&
1538 !mwait_cpustop_broken);
1539 if (use_mwait) {
1540 mb = PCPU_PTR(monitorbuf);
1541 atomic_store_int(&mb->stop_state,
1542 MONITOR_STOPSTATE_STOPPED);
1543 }
1544
1545 /* Indicate that we are stopped */
1546 CPU_SET_ATOMIC(cpu, &stopped_cpus);
1547
1548 /* Wait for restart */
1549 while (!CPU_ISSET(cpu, &started_cpus)) {
1550 if (use_mwait) {
1551 cpu_monitor(mb, 0, 0);
1552 if (atomic_load_int(&mb->stop_state) ==
1553 MONITOR_STOPSTATE_STOPPED)
1554 cpu_mwait(0, MWAIT_C1);
1555 continue;
1556 }
1557
1558 ia32_pause();
1559
1560 /*
1561 * Halt non-BSP CPUs on panic -- we're never going to need them
1562 * again, and might as well save power / release resources
1563 * (e.g., overprovisioned VM infrastructure).
1564 */
1565 while (__predict_false(!IS_BSP() && KERNEL_PANICKED()))
1566 halt();
1567 }
1568
1569 cpustop_handler_post(cpu);
1570 }
1571
1572 static void
cpustop_handler_post(u_int cpu)1573 cpustop_handler_post(u_int cpu)
1574 {
1575
1576 CPU_CLR_ATOMIC(cpu, &started_cpus);
1577 CPU_CLR_ATOMIC(cpu, &stopped_cpus);
1578
1579 /*
1580 * We don't broadcast TLB invalidations to other CPUs when they are
1581 * stopped. Hence, we clear the TLB before resuming.
1582 */
1583 invltlb_glob();
1584
1585 #if defined(__amd64__) && (defined(DDB) || defined(GDB))
1586 amd64_db_resume_dbreg();
1587 #endif
1588
1589 if (cpu == 0 && cpustop_restartfunc != NULL) {
1590 cpustop_restartfunc();
1591 cpustop_restartfunc = NULL;
1592 }
1593 }
1594
1595 /*
1596 * Handle an IPI_SUSPEND by saving our current context and spinning until we
1597 * are resumed.
1598 */
1599 void
cpususpend_handler(void)1600 cpususpend_handler(void)
1601 {
1602 u_int cpu;
1603
1604 mtx_assert(&smp_ipi_mtx, MA_NOTOWNED);
1605
1606 #ifdef __amd64__
1607 if (vmm_suspend_p)
1608 vmm_suspend_p();
1609 #endif
1610
1611 cpu = PCPU_GET(cpuid);
1612
1613 #ifdef XENHVM
1614 /*
1615 * Some Xen guest types (PVH) expose a very minimal set of ACPI tables,
1616 * and for example have no support for SCI. That leads to the suspend
1617 * stacks not being allocated, and hence when attempting to perform a
1618 * Xen triggered suspension FreeBSD will hit a #PF. Avoid saving the
1619 * CPU and FPU contexts if the stacks are not allocated, as the
1620 * hypervisor will already take care of this. Note that we could even
1621 * do this for Xen triggered suspensions on guests that have full ACPI
1622 * support, but doing so would introduce extra complexity.
1623 */
1624 if (susppcbs == NULL) {
1625 KASSERT(vm_guest == VM_GUEST_XEN, ("Missing suspend stack"));
1626 CPU_SET_ATOMIC(cpu, &suspended_cpus);
1627 CPU_SET_ATOMIC(cpu, &resuming_cpus);
1628 } else
1629 #endif
1630 if (savectx(&susppcbs[cpu]->sp_pcb)) {
1631 #ifdef __amd64__
1632 fpususpend(susppcbs[cpu]->sp_fpususpend);
1633 #else
1634 npxsuspend(susppcbs[cpu]->sp_fpususpend);
1635 #endif
1636 /*
1637 * suspended_cpus is cleared shortly after each AP is restarted
1638 * by a Startup IPI, so that the BSP can proceed to restarting
1639 * the next AP.
1640 *
1641 * resuming_cpus gets cleared when the AP completes
1642 * initialization after having been released by the BSP.
1643 * resuming_cpus is probably not the best name for the
1644 * variable, because it is actually a set of processors that
1645 * haven't resumed yet and haven't necessarily started resuming.
1646 *
1647 * Note that suspended_cpus is meaningful only for ACPI suspend
1648 * as it's not really used for Xen suspend since the APs are
1649 * automatically restored to the running state and the correct
1650 * context. For the same reason resumectx is never called in
1651 * that case.
1652 */
1653 CPU_SET_ATOMIC(cpu, &suspended_cpus);
1654 CPU_SET_ATOMIC(cpu, &resuming_cpus);
1655
1656 /*
1657 * Invalidate the cache after setting the global status bits.
1658 * The last AP to set its bit may end up being an Owner of the
1659 * corresponding cache line in MOESI protocol. The AP may be
1660 * stopped before the cache line is written to the main memory.
1661 */
1662 wbinvd();
1663 } else {
1664 #ifdef __amd64__
1665 fpuresume(susppcbs[cpu]->sp_fpususpend);
1666 #else
1667 npxresume(susppcbs[cpu]->sp_fpususpend);
1668 #endif
1669 pmap_init_pat();
1670 initializecpu();
1671 PCPU_SET(switchtime, 0);
1672 PCPU_SET(switchticks, ticks);
1673
1674 /* Indicate that we have restarted and restored the context. */
1675 CPU_CLR_ATOMIC(cpu, &suspended_cpus);
1676 }
1677
1678 /* Wait for resume directive */
1679 while (!CPU_ISSET(cpu, &toresume_cpus))
1680 ia32_pause();
1681
1682 /* Re-apply microcode updates. */
1683 ucode_reload();
1684
1685 #ifdef __i386__
1686 /* Finish removing the identity mapping of low memory for this AP. */
1687 invltlb_glob();
1688 #endif
1689
1690 if (cpu_ops.cpu_resume)
1691 cpu_ops.cpu_resume();
1692 #ifdef __amd64__
1693 if (vmm_resume_p)
1694 vmm_resume_p();
1695 #endif
1696
1697 /* Resume MCA and local APIC */
1698 lapic_xapic_mode();
1699 mca_resume();
1700 lapic_setup(0);
1701
1702 /* Indicate that we are resumed */
1703 CPU_CLR_ATOMIC(cpu, &resuming_cpus);
1704 CPU_CLR_ATOMIC(cpu, &suspended_cpus);
1705 CPU_CLR_ATOMIC(cpu, &toresume_cpus);
1706 }
1707
1708 void
cpuoff_handler(void)1709 cpuoff_handler(void)
1710 {
1711 u_int cpu;
1712
1713 cpu = PCPU_GET(cpuid);
1714
1715 /* Time to go catatonic. A reset will be required to leave. */
1716 disable_intr();
1717 lapic_disable();
1718 CPU_SET_ATOMIC(cpu, &suspended_cpus);
1719
1720 /*
1721 * There technically should be no need for the `while` here, since it
1722 * cannot be interrupted (interrupts are disabled). Be safe anyway.
1723 * Any interrupt at this point will likely be fatal, as the page tables
1724 * are likely going away shortly.
1725 */
1726 while (1)
1727 halt();
1728 }
1729
1730 /*
1731 * Handle an IPI_SWI by waking delayed SWI thread.
1732 */
1733 void
ipi_swi_handler(struct trapframe * frame)1734 ipi_swi_handler(struct trapframe *frame)
1735 {
1736
1737 intr_event_handle(clk_intr_event, frame);
1738 }
1739
1740 /*
1741 * This is called once the rest of the system is up and running and we're
1742 * ready to let the AP's out of the pen.
1743 */
1744 static void
release_aps(void * dummy __unused)1745 release_aps(void *dummy __unused)
1746 {
1747
1748 if (mp_ncpus == 1)
1749 return;
1750 atomic_store_rel_int(&aps_ready, 1);
1751 while (smp_started == 0)
1752 ia32_pause();
1753 }
1754 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
1755
1756 #ifdef COUNT_IPIS
1757 /*
1758 * Setup interrupt counters for IPI handlers.
1759 */
1760 static void
mp_ipi_intrcnt(void * dummy)1761 mp_ipi_intrcnt(void *dummy)
1762 {
1763 char buf[64];
1764 int i;
1765
1766 CPU_FOREACH(i) {
1767 snprintf(buf, sizeof(buf), "cpu%d:invltlb", i);
1768 intrcnt_add(buf, &ipi_invltlb_counts[i]);
1769 snprintf(buf, sizeof(buf), "cpu%d:invlrng", i);
1770 intrcnt_add(buf, &ipi_invlrng_counts[i]);
1771 snprintf(buf, sizeof(buf), "cpu%d:invlpg", i);
1772 intrcnt_add(buf, &ipi_invlpg_counts[i]);
1773 snprintf(buf, sizeof(buf), "cpu%d:invlcache", i);
1774 intrcnt_add(buf, &ipi_invlcache_counts[i]);
1775 snprintf(buf, sizeof(buf), "cpu%d:preempt", i);
1776 intrcnt_add(buf, &ipi_preempt_counts[i]);
1777 snprintf(buf, sizeof(buf), "cpu%d:ast", i);
1778 intrcnt_add(buf, &ipi_ast_counts[i]);
1779 snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i);
1780 intrcnt_add(buf, &ipi_rendezvous_counts[i]);
1781 snprintf(buf, sizeof(buf), "cpu%d:hardclock", i);
1782 intrcnt_add(buf, &ipi_hardclock_counts[i]);
1783 }
1784 }
1785 SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL);
1786 #endif
1787