xref: /freebsd/sys/kern/subr_smp.c (revision 43e29d03f416d7dda52112a29600a7c82ee1a91e)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2001, John Baldwin <jhb@FreeBSD.org>.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 /*
29  * This module holds the global variables and machine independent functions
30  * used for the kernel SMP support.
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/ktr.h>
40 #include <sys/proc.h>
41 #include <sys/bus.h>
42 #include <sys/lock.h>
43 #include <sys/malloc.h>
44 #include <sys/mutex.h>
45 #include <sys/pcpu.h>
46 #include <sys/sched.h>
47 #include <sys/smp.h>
48 #include <sys/sysctl.h>
49 
50 #include <machine/cpu.h>
51 #include <machine/pcb.h>
52 #include <machine/smp.h>
53 
54 #include "opt_sched.h"
55 
56 #ifdef SMP
57 MALLOC_DEFINE(M_TOPO, "toponodes", "SMP topology data");
58 
59 volatile cpuset_t stopped_cpus;
60 volatile cpuset_t started_cpus;
61 volatile cpuset_t suspended_cpus;
62 cpuset_t hlt_cpus_mask;
63 cpuset_t logical_cpus_mask;
64 
65 void (*cpustop_restartfunc)(void);
66 #endif
67 
68 static int sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS);
69 
70 /* This is used in modules that need to work in both SMP and UP. */
71 cpuset_t all_cpus;
72 
73 int mp_ncpus;
74 /* export this for libkvm consumers. */
75 int mp_maxcpus = MAXCPU;
76 
77 volatile int smp_started;
78 u_int mp_maxid;
79 
80 /* Array of CPU contexts saved during a panic. */
81 struct pcb *stoppcbs;
82 
83 static SYSCTL_NODE(_kern, OID_AUTO, smp,
84     CTLFLAG_RD | CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL,
85     "Kernel SMP");
86 
87 SYSCTL_INT(_kern_smp, OID_AUTO, maxid, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxid, 0,
88     "Max CPU ID.");
89 
90 SYSCTL_INT(_kern_smp, OID_AUTO, maxcpus, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxcpus,
91     0, "Max number of CPUs that the system was compiled for.");
92 
93 SYSCTL_PROC(_kern_smp, OID_AUTO, active, CTLFLAG_RD|CTLTYPE_INT|CTLFLAG_MPSAFE,
94     NULL, 0, sysctl_kern_smp_active, "I",
95     "Indicates system is running in SMP mode");
96 
97 int smp_disabled = 0;	/* has smp been disabled? */
98 SYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RDTUN|CTLFLAG_CAPRD,
99     &smp_disabled, 0, "SMP has been disabled from the loader");
100 
101 int smp_cpus = 1;	/* how many cpu's running */
102 SYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD|CTLFLAG_CAPRD, &smp_cpus, 0,
103     "Number of CPUs online");
104 
105 int smp_threads_per_core = 1;	/* how many SMT threads are running per core */
106 SYSCTL_INT(_kern_smp, OID_AUTO, threads_per_core, CTLFLAG_RD|CTLFLAG_CAPRD,
107     &smp_threads_per_core, 0, "Number of SMT threads online per core");
108 
109 int mp_ncores = -1;	/* how many physical cores running */
110 SYSCTL_INT(_kern_smp, OID_AUTO, cores, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_ncores, 0,
111     "Number of physical cores online");
112 
113 int smp_topology = 0;	/* Which topology we're using. */
114 SYSCTL_INT(_kern_smp, OID_AUTO, topology, CTLFLAG_RDTUN, &smp_topology, 0,
115     "Topology override setting; 0 is default provided by hardware.");
116 
117 #ifdef SMP
118 /* Enable forwarding of a signal to a process running on a different CPU */
119 static int forward_signal_enabled = 1;
120 SYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
121 	   &forward_signal_enabled, 0,
122 	   "Forwarding of a signal to a process on a different CPU");
123 
124 /* Variables needed for SMP rendezvous. */
125 static volatile int smp_rv_ncpus;
126 static void (*volatile smp_rv_setup_func)(void *arg);
127 static void (*volatile smp_rv_action_func)(void *arg);
128 static void (*volatile smp_rv_teardown_func)(void *arg);
129 static void *volatile smp_rv_func_arg;
130 static volatile int smp_rv_waiters[4];
131 
132 /*
133  * Shared mutex to restrict busywaits between smp_rendezvous() and
134  * smp(_targeted)_tlb_shootdown().  A deadlock occurs if both of these
135  * functions trigger at once and cause multiple CPUs to busywait with
136  * interrupts disabled.
137  */
138 struct mtx smp_ipi_mtx;
139 
140 /*
141  * Let the MD SMP code initialize mp_maxid very early if it can.
142  */
143 static void
144 mp_setmaxid(void *dummy)
145 {
146 
147 	cpu_mp_setmaxid();
148 
149 	KASSERT(mp_ncpus >= 1, ("%s: CPU count < 1", __func__));
150 	KASSERT(mp_ncpus > 1 || mp_maxid == 0,
151 	    ("%s: one CPU but mp_maxid is not zero", __func__));
152 	KASSERT(mp_maxid >= mp_ncpus - 1,
153 	    ("%s: counters out of sync: max %d, count %d", __func__,
154 		mp_maxid, mp_ncpus));
155 
156 	cpusetsizemin = howmany(mp_maxid + 1, NBBY);
157 }
158 SYSINIT(cpu_mp_setmaxid, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setmaxid, NULL);
159 
160 /*
161  * Call the MD SMP initialization code.
162  */
163 static void
164 mp_start(void *dummy)
165 {
166 
167 	mtx_init(&smp_ipi_mtx, "smp rendezvous", NULL, MTX_SPIN);
168 
169 	/* Probe for MP hardware. */
170 	if (smp_disabled != 0 || cpu_mp_probe() == 0) {
171 		mp_ncores = 1;
172 		mp_ncpus = 1;
173 		CPU_SETOF(PCPU_GET(cpuid), &all_cpus);
174 		return;
175 	}
176 
177 	cpu_mp_start();
178 	printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n",
179 	    mp_ncpus);
180 
181 	/* Provide a default for most architectures that don't have SMT/HTT. */
182 	if (mp_ncores < 0)
183 		mp_ncores = mp_ncpus;
184 
185 	stoppcbs = mallocarray(mp_maxid + 1, sizeof(struct pcb), M_DEVBUF,
186 	    M_WAITOK | M_ZERO);
187 
188 	cpu_mp_announce();
189 }
190 SYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_THIRD, mp_start, NULL);
191 
192 void
193 forward_signal(struct thread *td)
194 {
195 	int id;
196 
197 	/*
198 	 * signotify() has already set TDA_AST and TDA_SIG on td_ast for
199 	 * this thread, so all we need to do is poke it if it is currently
200 	 * executing so that it executes ast().
201 	 */
202 	THREAD_LOCK_ASSERT(td, MA_OWNED);
203 	KASSERT(TD_IS_RUNNING(td),
204 	    ("forward_signal: thread is not TDS_RUNNING"));
205 
206 	CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc);
207 
208 	if (!smp_started || cold || KERNEL_PANICKED())
209 		return;
210 	if (!forward_signal_enabled)
211 		return;
212 
213 	/* No need to IPI ourself. */
214 	if (td == curthread)
215 		return;
216 
217 	id = td->td_oncpu;
218 	if (id == NOCPU)
219 		return;
220 	ipi_cpu(id, IPI_AST);
221 }
222 
223 /*
224  * When called the executing CPU will send an IPI to all other CPUs
225  *  requesting that they halt execution.
226  *
227  * Usually (but not necessarily) called with 'other_cpus' as its arg.
228  *
229  *  - Signals all CPUs in map to stop.
230  *  - Waits for each to stop.
231  *
232  * Returns:
233  *  -1: error
234  *   0: NA
235  *   1: ok
236  *
237  */
238 #if defined(__amd64__) || defined(__i386__)
239 #define	X86	1
240 #else
241 #define	X86	0
242 #endif
243 static int
244 generic_stop_cpus(cpuset_t map, u_int type)
245 {
246 #ifdef KTR
247 	char cpusetbuf[CPUSETBUFSIZ];
248 #endif
249 	static volatile u_int stopping_cpu = NOCPU;
250 	int i;
251 	volatile cpuset_t *cpus;
252 
253 	KASSERT(
254 	    type == IPI_STOP || type == IPI_STOP_HARD
255 #if X86
256 	    || type == IPI_SUSPEND
257 #endif
258 	    , ("%s: invalid stop type", __func__));
259 
260 	if (!smp_started)
261 		return (0);
262 
263 	CTR2(KTR_SMP, "stop_cpus(%s) with %u type",
264 	    cpusetobj_strprint(cpusetbuf, &map), type);
265 
266 #if X86
267 	/*
268 	 * When suspending, ensure there are are no IPIs in progress.
269 	 * IPIs that have been issued, but not yet delivered (e.g.
270 	 * not pending on a vCPU when running under virtualization)
271 	 * will be lost, violating FreeBSD's assumption of reliable
272 	 * IPI delivery.
273 	 */
274 	if (type == IPI_SUSPEND)
275 		mtx_lock_spin(&smp_ipi_mtx);
276 #endif
277 
278 #if X86
279 	if (!nmi_is_broadcast || nmi_kdb_lock == 0) {
280 #endif
281 	if (stopping_cpu != PCPU_GET(cpuid))
282 		while (atomic_cmpset_int(&stopping_cpu, NOCPU,
283 		    PCPU_GET(cpuid)) == 0)
284 			while (stopping_cpu != NOCPU)
285 				cpu_spinwait(); /* spin */
286 
287 	/* send the stop IPI to all CPUs in map */
288 	ipi_selected(map, type);
289 #if X86
290 	}
291 #endif
292 
293 #if X86
294 	if (type == IPI_SUSPEND)
295 		cpus = &suspended_cpus;
296 	else
297 #endif
298 		cpus = &stopped_cpus;
299 
300 	i = 0;
301 	while (!CPU_SUBSET(cpus, &map)) {
302 		/* spin */
303 		cpu_spinwait();
304 		i++;
305 		if (i == 100000000) {
306 			printf("timeout stopping cpus\n");
307 			break;
308 		}
309 	}
310 
311 #if X86
312 	if (type == IPI_SUSPEND)
313 		mtx_unlock_spin(&smp_ipi_mtx);
314 #endif
315 
316 	stopping_cpu = NOCPU;
317 	return (1);
318 }
319 
320 int
321 stop_cpus(cpuset_t map)
322 {
323 
324 	return (generic_stop_cpus(map, IPI_STOP));
325 }
326 
327 int
328 stop_cpus_hard(cpuset_t map)
329 {
330 
331 	return (generic_stop_cpus(map, IPI_STOP_HARD));
332 }
333 
334 #if X86
335 int
336 suspend_cpus(cpuset_t map)
337 {
338 
339 	return (generic_stop_cpus(map, IPI_SUSPEND));
340 }
341 #endif
342 
343 /*
344  * Called by a CPU to restart stopped CPUs.
345  *
346  * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
347  *
348  *  - Signals all CPUs in map to restart.
349  *  - Waits for each to restart.
350  *
351  * Returns:
352  *  -1: error
353  *   0: NA
354  *   1: ok
355  */
356 static int
357 generic_restart_cpus(cpuset_t map, u_int type)
358 {
359 #ifdef KTR
360 	char cpusetbuf[CPUSETBUFSIZ];
361 #endif
362 	volatile cpuset_t *cpus;
363 
364 #if X86
365 	KASSERT(type == IPI_STOP || type == IPI_STOP_HARD
366 	    || type == IPI_SUSPEND, ("%s: invalid stop type", __func__));
367 
368 	if (!smp_started)
369 		return (0);
370 
371 	CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map));
372 
373 	if (type == IPI_SUSPEND)
374 		cpus = &resuming_cpus;
375 	else
376 		cpus = &stopped_cpus;
377 
378 	/* signal other cpus to restart */
379 	if (type == IPI_SUSPEND)
380 		CPU_COPY_STORE_REL(&map, &toresume_cpus);
381 	else
382 		CPU_COPY_STORE_REL(&map, &started_cpus);
383 
384 	/*
385 	 * Wake up any CPUs stopped with MWAIT.  From MI code we can't tell if
386 	 * MONITOR/MWAIT is enabled, but the potentially redundant writes are
387 	 * relatively inexpensive.
388 	 */
389 	if (type == IPI_STOP) {
390 		struct monitorbuf *mb;
391 		u_int id;
392 
393 		CPU_FOREACH(id) {
394 			if (!CPU_ISSET(id, &map))
395 				continue;
396 
397 			mb = &pcpu_find(id)->pc_monitorbuf;
398 			atomic_store_int(&mb->stop_state,
399 			    MONITOR_STOPSTATE_RUNNING);
400 		}
401 	}
402 
403 	if (!nmi_is_broadcast || nmi_kdb_lock == 0) {
404 		/* wait for each to clear its bit */
405 		while (CPU_OVERLAP(cpus, &map))
406 			cpu_spinwait();
407 	}
408 #else /* !X86 */
409 	KASSERT(type == IPI_STOP || type == IPI_STOP_HARD,
410 	    ("%s: invalid stop type", __func__));
411 
412 	if (!smp_started)
413 		return (0);
414 
415 	CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map));
416 
417 	cpus = &stopped_cpus;
418 
419 	/* signal other cpus to restart */
420 	CPU_COPY_STORE_REL(&map, &started_cpus);
421 
422 	/* wait for each to clear its bit */
423 	while (CPU_OVERLAP(cpus, &map))
424 		cpu_spinwait();
425 #endif
426 	return (1);
427 }
428 
429 int
430 restart_cpus(cpuset_t map)
431 {
432 
433 	return (generic_restart_cpus(map, IPI_STOP));
434 }
435 
436 #if X86
437 int
438 resume_cpus(cpuset_t map)
439 {
440 
441 	return (generic_restart_cpus(map, IPI_SUSPEND));
442 }
443 #endif
444 #undef X86
445 
446 /*
447  * All-CPU rendezvous.  CPUs are signalled, all execute the setup function
448  * (if specified), rendezvous, execute the action function (if specified),
449  * rendezvous again, execute the teardown function (if specified), and then
450  * resume.
451  *
452  * Note that the supplied external functions _must_ be reentrant and aware
453  * that they are running in parallel and in an unknown lock context.
454  */
455 void
456 smp_rendezvous_action(void)
457 {
458 	struct thread *td;
459 	void *local_func_arg;
460 	void (*local_setup_func)(void*);
461 	void (*local_action_func)(void*);
462 	void (*local_teardown_func)(void*);
463 #ifdef INVARIANTS
464 	int owepreempt;
465 #endif
466 
467 	/* Ensure we have up-to-date values. */
468 	atomic_add_acq_int(&smp_rv_waiters[0], 1);
469 	while (smp_rv_waiters[0] < smp_rv_ncpus)
470 		cpu_spinwait();
471 
472 	/* Fetch rendezvous parameters after acquire barrier. */
473 	local_func_arg = smp_rv_func_arg;
474 	local_setup_func = smp_rv_setup_func;
475 	local_action_func = smp_rv_action_func;
476 	local_teardown_func = smp_rv_teardown_func;
477 
478 	/*
479 	 * Use a nested critical section to prevent any preemptions
480 	 * from occurring during a rendezvous action routine.
481 	 * Specifically, if a rendezvous handler is invoked via an IPI
482 	 * and the interrupted thread was in the critical_exit()
483 	 * function after setting td_critnest to 0 but before
484 	 * performing a deferred preemption, this routine can be
485 	 * invoked with td_critnest set to 0 and td_owepreempt true.
486 	 * In that case, a critical_exit() during the rendezvous
487 	 * action would trigger a preemption which is not permitted in
488 	 * a rendezvous action.  To fix this, wrap all of the
489 	 * rendezvous action handlers in a critical section.  We
490 	 * cannot use a regular critical section however as having
491 	 * critical_exit() preempt from this routine would also be
492 	 * problematic (the preemption must not occur before the IPI
493 	 * has been acknowledged via an EOI).  Instead, we
494 	 * intentionally ignore td_owepreempt when leaving the
495 	 * critical section.  This should be harmless because we do
496 	 * not permit rendezvous action routines to schedule threads,
497 	 * and thus td_owepreempt should never transition from 0 to 1
498 	 * during this routine.
499 	 */
500 	td = curthread;
501 	td->td_critnest++;
502 #ifdef INVARIANTS
503 	owepreempt = td->td_owepreempt;
504 #endif
505 
506 	/*
507 	 * If requested, run a setup function before the main action
508 	 * function.  Ensure all CPUs have completed the setup
509 	 * function before moving on to the action function.
510 	 */
511 	if (local_setup_func != smp_no_rendezvous_barrier) {
512 		if (local_setup_func != NULL)
513 			local_setup_func(local_func_arg);
514 		atomic_add_int(&smp_rv_waiters[1], 1);
515 		while (smp_rv_waiters[1] < smp_rv_ncpus)
516                 	cpu_spinwait();
517 	}
518 
519 	if (local_action_func != NULL)
520 		local_action_func(local_func_arg);
521 
522 	if (local_teardown_func != smp_no_rendezvous_barrier) {
523 		/*
524 		 * Signal that the main action has been completed.  If a
525 		 * full exit rendezvous is requested, then all CPUs will
526 		 * wait here until all CPUs have finished the main action.
527 		 */
528 		atomic_add_int(&smp_rv_waiters[2], 1);
529 		while (smp_rv_waiters[2] < smp_rv_ncpus)
530 			cpu_spinwait();
531 
532 		if (local_teardown_func != NULL)
533 			local_teardown_func(local_func_arg);
534 	}
535 
536 	/*
537 	 * Signal that the rendezvous is fully completed by this CPU.
538 	 * This means that no member of smp_rv_* pseudo-structure will be
539 	 * accessed by this target CPU after this point; in particular,
540 	 * memory pointed by smp_rv_func_arg.
541 	 *
542 	 * The release semantic ensures that all accesses performed by
543 	 * the current CPU are visible when smp_rendezvous_cpus()
544 	 * returns, by synchronizing with the
545 	 * atomic_load_acq_int(&smp_rv_waiters[3]).
546 	 */
547 	atomic_add_rel_int(&smp_rv_waiters[3], 1);
548 
549 	td->td_critnest--;
550 	KASSERT(owepreempt == td->td_owepreempt,
551 	    ("rendezvous action changed td_owepreempt"));
552 }
553 
554 void
555 smp_rendezvous_cpus(cpuset_t map,
556 	void (* setup_func)(void *),
557 	void (* action_func)(void *),
558 	void (* teardown_func)(void *),
559 	void *arg)
560 {
561 	int curcpumap, i, ncpus = 0;
562 
563 	/* See comments in the !SMP case. */
564 	if (!smp_started) {
565 		spinlock_enter();
566 		if (setup_func != NULL)
567 			setup_func(arg);
568 		if (action_func != NULL)
569 			action_func(arg);
570 		if (teardown_func != NULL)
571 			teardown_func(arg);
572 		spinlock_exit();
573 		return;
574 	}
575 
576 	/*
577 	 * Make sure we come here with interrupts enabled.  Otherwise we
578 	 * livelock if smp_ipi_mtx is owned by a thread which sent us an IPI.
579 	 */
580 	MPASS(curthread->td_md.md_spinlock_count == 0);
581 
582 	CPU_FOREACH(i) {
583 		if (CPU_ISSET(i, &map))
584 			ncpus++;
585 	}
586 	if (ncpus == 0)
587 		panic("ncpus is 0 with non-zero map");
588 
589 	mtx_lock_spin(&smp_ipi_mtx);
590 
591 	/* Pass rendezvous parameters via global variables. */
592 	smp_rv_ncpus = ncpus;
593 	smp_rv_setup_func = setup_func;
594 	smp_rv_action_func = action_func;
595 	smp_rv_teardown_func = teardown_func;
596 	smp_rv_func_arg = arg;
597 	smp_rv_waiters[1] = 0;
598 	smp_rv_waiters[2] = 0;
599 	smp_rv_waiters[3] = 0;
600 	atomic_store_rel_int(&smp_rv_waiters[0], 0);
601 
602 	/*
603 	 * Signal other processors, which will enter the IPI with
604 	 * interrupts off.
605 	 */
606 	curcpumap = CPU_ISSET(curcpu, &map);
607 	CPU_CLR(curcpu, &map);
608 	ipi_selected(map, IPI_RENDEZVOUS);
609 
610 	/* Check if the current CPU is in the map */
611 	if (curcpumap != 0)
612 		smp_rendezvous_action();
613 
614 	/*
615 	 * Ensure that the master CPU waits for all the other
616 	 * CPUs to finish the rendezvous, so that smp_rv_*
617 	 * pseudo-structure and the arg are guaranteed to not
618 	 * be in use.
619 	 *
620 	 * Load acquire synchronizes with the release add in
621 	 * smp_rendezvous_action(), which ensures that our caller sees
622 	 * all memory actions done by the called functions on other
623 	 * CPUs.
624 	 */
625 	while (atomic_load_acq_int(&smp_rv_waiters[3]) < ncpus)
626 		cpu_spinwait();
627 
628 	mtx_unlock_spin(&smp_ipi_mtx);
629 }
630 
631 void
632 smp_rendezvous(void (* setup_func)(void *),
633 	       void (* action_func)(void *),
634 	       void (* teardown_func)(void *),
635 	       void *arg)
636 {
637 	smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func, arg);
638 }
639 
640 static void
641 smp_topo_fill(struct cpu_group *cg)
642 {
643 	int c;
644 
645 	for (c = 0; c < cg->cg_children; c++)
646 		smp_topo_fill(&cg->cg_child[c]);
647 	cg->cg_first = CPU_FFS(&cg->cg_mask) - 1;
648 	cg->cg_last = CPU_FLS(&cg->cg_mask) - 1;
649 }
650 
651 struct cpu_group *
652 smp_topo(void)
653 {
654 	char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ];
655 	static struct cpu_group *top = NULL;
656 
657 	/*
658 	 * The first call to smp_topo() is guaranteed to occur
659 	 * during the kernel boot while we are still single-threaded.
660 	 */
661 	if (top != NULL)
662 		return (top);
663 
664 	/*
665 	 * Check for a fake topology request for debugging purposes.
666 	 */
667 	switch (smp_topology) {
668 	case 1:
669 		/* Dual core with no sharing.  */
670 		top = smp_topo_1level(CG_SHARE_NONE, 2, 0);
671 		break;
672 	case 2:
673 		/* No topology, all cpus are equal. */
674 		top = smp_topo_none();
675 		break;
676 	case 3:
677 		/* Dual core with shared L2.  */
678 		top = smp_topo_1level(CG_SHARE_L2, 2, 0);
679 		break;
680 	case 4:
681 		/* quad core, shared l3 among each package, private l2.  */
682 		top = smp_topo_1level(CG_SHARE_L3, 4, 0);
683 		break;
684 	case 5:
685 		/* quad core,  2 dualcore parts on each package share l2.  */
686 		top = smp_topo_2level(CG_SHARE_NONE, 2, CG_SHARE_L2, 2, 0);
687 		break;
688 	case 6:
689 		/* Single-core 2xHTT */
690 		top = smp_topo_1level(CG_SHARE_L1, 2, CG_FLAG_HTT);
691 		break;
692 	case 7:
693 		/* quad core with a shared l3, 8 threads sharing L2.  */
694 		top = smp_topo_2level(CG_SHARE_L3, 4, CG_SHARE_L2, 8,
695 		    CG_FLAG_SMT);
696 		break;
697 	default:
698 		/* Default, ask the system what it wants. */
699 		top = cpu_topo();
700 		break;
701 	}
702 	/*
703 	 * Verify the returned topology.
704 	 */
705 	if (top->cg_count != mp_ncpus)
706 		panic("Built bad topology at %p.  CPU count %d != %d",
707 		    top, top->cg_count, mp_ncpus);
708 	if (CPU_CMP(&top->cg_mask, &all_cpus))
709 		panic("Built bad topology at %p.  CPU mask (%s) != (%s)",
710 		    top, cpusetobj_strprint(cpusetbuf, &top->cg_mask),
711 		    cpusetobj_strprint(cpusetbuf2, &all_cpus));
712 
713 	/*
714 	 * Collapse nonsense levels that may be created out of convenience by
715 	 * the MD layers.  They cause extra work in the search functions.
716 	 */
717 	while (top->cg_children == 1) {
718 		top = &top->cg_child[0];
719 		top->cg_parent = NULL;
720 	}
721 	smp_topo_fill(top);
722 	return (top);
723 }
724 
725 struct cpu_group *
726 smp_topo_alloc(u_int count)
727 {
728 	static struct cpu_group *group = NULL;
729 	static u_int index;
730 	u_int curr;
731 
732 	if (group == NULL) {
733 		group = mallocarray((mp_maxid + 1) * MAX_CACHE_LEVELS + 1,
734 		    sizeof(*group), M_DEVBUF, M_WAITOK | M_ZERO);
735 	}
736 	curr = index;
737 	index += count;
738 	return (&group[curr]);
739 }
740 
741 struct cpu_group *
742 smp_topo_none(void)
743 {
744 	struct cpu_group *top;
745 
746 	top = smp_topo_alloc(1);
747 	top->cg_parent = NULL;
748 	top->cg_child = NULL;
749 	top->cg_mask = all_cpus;
750 	top->cg_count = mp_ncpus;
751 	top->cg_children = 0;
752 	top->cg_level = CG_SHARE_NONE;
753 	top->cg_flags = 0;
754 
755 	return (top);
756 }
757 
758 static int
759 smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share,
760     int count, int flags, int start)
761 {
762 	char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ];
763 	cpuset_t mask;
764 	int i;
765 
766 	CPU_ZERO(&mask);
767 	for (i = 0; i < count; i++, start++)
768 		CPU_SET(start, &mask);
769 	child->cg_parent = parent;
770 	child->cg_child = NULL;
771 	child->cg_children = 0;
772 	child->cg_level = share;
773 	child->cg_count = count;
774 	child->cg_flags = flags;
775 	child->cg_mask = mask;
776 	parent->cg_children++;
777 	for (; parent != NULL; parent = parent->cg_parent) {
778 		if (CPU_OVERLAP(&parent->cg_mask, &child->cg_mask))
779 			panic("Duplicate children in %p.  mask (%s) child (%s)",
780 			    parent,
781 			    cpusetobj_strprint(cpusetbuf, &parent->cg_mask),
782 			    cpusetobj_strprint(cpusetbuf2, &child->cg_mask));
783 		CPU_OR(&parent->cg_mask, &parent->cg_mask, &child->cg_mask);
784 		parent->cg_count += child->cg_count;
785 	}
786 
787 	return (start);
788 }
789 
790 struct cpu_group *
791 smp_topo_1level(int share, int count, int flags)
792 {
793 	struct cpu_group *child;
794 	struct cpu_group *top;
795 	int packages;
796 	int cpu;
797 	int i;
798 
799 	cpu = 0;
800 	packages = mp_ncpus / count;
801 	top = smp_topo_alloc(1 + packages);
802 	top->cg_child = child = top + 1;
803 	top->cg_level = CG_SHARE_NONE;
804 	for (i = 0; i < packages; i++, child++)
805 		cpu = smp_topo_addleaf(top, child, share, count, flags, cpu);
806 	return (top);
807 }
808 
809 struct cpu_group *
810 smp_topo_2level(int l2share, int l2count, int l1share, int l1count,
811     int l1flags)
812 {
813 	struct cpu_group *top;
814 	struct cpu_group *l1g;
815 	struct cpu_group *l2g;
816 	int cpu;
817 	int i;
818 	int j;
819 
820 	cpu = 0;
821 	top = smp_topo_alloc(1 + mp_ncpus / (l2count * l1count) +
822 	    mp_ncpus / l1count);
823 	l2g = top + 1;
824 	top->cg_child = l2g;
825 	top->cg_level = CG_SHARE_NONE;
826 	top->cg_children = mp_ncpus / (l2count * l1count);
827 	l1g = l2g + top->cg_children;
828 	for (i = 0; i < top->cg_children; i++, l2g++) {
829 		l2g->cg_parent = top;
830 		l2g->cg_child = l1g;
831 		l2g->cg_level = l2share;
832 		for (j = 0; j < l2count; j++, l1g++)
833 			cpu = smp_topo_addleaf(l2g, l1g, l1share, l1count,
834 			    l1flags, cpu);
835 	}
836 	return (top);
837 }
838 
839 struct cpu_group *
840 smp_topo_find(struct cpu_group *top, int cpu)
841 {
842 	struct cpu_group *cg;
843 	cpuset_t mask;
844 	int children;
845 	int i;
846 
847 	CPU_SETOF(cpu, &mask);
848 	cg = top;
849 	for (;;) {
850 		if (!CPU_OVERLAP(&cg->cg_mask, &mask))
851 			return (NULL);
852 		if (cg->cg_children == 0)
853 			return (cg);
854 		children = cg->cg_children;
855 		for (i = 0, cg = cg->cg_child; i < children; cg++, i++)
856 			if (CPU_OVERLAP(&cg->cg_mask, &mask))
857 				break;
858 	}
859 	return (NULL);
860 }
861 #else /* !SMP */
862 
863 void
864 smp_rendezvous_cpus(cpuset_t map,
865 	void (*setup_func)(void *),
866 	void (*action_func)(void *),
867 	void (*teardown_func)(void *),
868 	void *arg)
869 {
870 	/*
871 	 * In the !SMP case we just need to ensure the same initial conditions
872 	 * as the SMP case.
873 	 */
874 	spinlock_enter();
875 	if (setup_func != NULL)
876 		setup_func(arg);
877 	if (action_func != NULL)
878 		action_func(arg);
879 	if (teardown_func != NULL)
880 		teardown_func(arg);
881 	spinlock_exit();
882 }
883 
884 void
885 smp_rendezvous(void (*setup_func)(void *),
886 	       void (*action_func)(void *),
887 	       void (*teardown_func)(void *),
888 	       void *arg)
889 {
890 
891 	smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func,
892 	    arg);
893 }
894 
895 /*
896  * Provide dummy SMP support for UP kernels.  Modules that need to use SMP
897  * APIs will still work using this dummy support.
898  */
899 static void
900 mp_setvariables_for_up(void *dummy)
901 {
902 	mp_ncpus = 1;
903 	mp_ncores = 1;
904 	mp_maxid = PCPU_GET(cpuid);
905 	CPU_SETOF(mp_maxid, &all_cpus);
906 	KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero"));
907 }
908 SYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST,
909     mp_setvariables_for_up, NULL);
910 #endif /* SMP */
911 
912 void
913 smp_no_rendezvous_barrier(void *dummy)
914 {
915 #ifdef SMP
916 	KASSERT((!smp_started),("smp_no_rendezvous called and smp is started"));
917 #endif
918 }
919 
920 void
921 smp_rendezvous_cpus_retry(cpuset_t map,
922 	void (* setup_func)(void *),
923 	void (* action_func)(void *),
924 	void (* teardown_func)(void *),
925 	void (* wait_func)(void *, int),
926 	struct smp_rendezvous_cpus_retry_arg *arg)
927 {
928 	int cpu;
929 
930 	CPU_COPY(&map, &arg->cpus);
931 
932 	/*
933 	 * Only one CPU to execute on.
934 	 */
935 	if (!smp_started) {
936 		spinlock_enter();
937 		if (setup_func != NULL)
938 			setup_func(arg);
939 		if (action_func != NULL)
940 			action_func(arg);
941 		if (teardown_func != NULL)
942 			teardown_func(arg);
943 		spinlock_exit();
944 		return;
945 	}
946 
947 	/*
948 	 * Execute an action on all specified CPUs while retrying until they
949 	 * all acknowledge completion.
950 	 */
951 	for (;;) {
952 		smp_rendezvous_cpus(
953 		    arg->cpus,
954 		    setup_func,
955 		    action_func,
956 		    teardown_func,
957 		    arg);
958 
959 		if (CPU_EMPTY(&arg->cpus))
960 			break;
961 
962 		CPU_FOREACH(cpu) {
963 			if (!CPU_ISSET(cpu, &arg->cpus))
964 				continue;
965 			wait_func(arg, cpu);
966 		}
967 	}
968 }
969 
970 void
971 smp_rendezvous_cpus_done(struct smp_rendezvous_cpus_retry_arg *arg)
972 {
973 
974 	CPU_CLR_ATOMIC(curcpu, &arg->cpus);
975 }
976 
977 /*
978  * If (prio & PDROP) == 0:
979  * Wait for specified idle threads to switch once.  This ensures that even
980  * preempted threads have cycled through the switch function once,
981  * exiting their codepaths.  This allows us to change global pointers
982  * with no other synchronization.
983  * If (prio & PDROP) != 0:
984  * Force the specified CPUs to switch context at least once.
985  */
986 int
987 quiesce_cpus(cpuset_t map, const char *wmesg, int prio)
988 {
989 	struct pcpu *pcpu;
990 	u_int *gen;
991 	int error;
992 	int cpu;
993 
994 	error = 0;
995 	if ((prio & PDROP) == 0) {
996 		gen = mallocarray(sizeof(u_int), mp_maxid + 1, M_TEMP,
997 		    M_WAITOK);
998 		for (cpu = 0; cpu <= mp_maxid; cpu++) {
999 			if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
1000 				continue;
1001 			pcpu = pcpu_find(cpu);
1002 			gen[cpu] = pcpu->pc_idlethread->td_generation;
1003 		}
1004 	}
1005 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
1006 		if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
1007 			continue;
1008 		pcpu = pcpu_find(cpu);
1009 		thread_lock(curthread);
1010 		sched_bind(curthread, cpu);
1011 		thread_unlock(curthread);
1012 		if ((prio & PDROP) != 0)
1013 			continue;
1014 		while (gen[cpu] == pcpu->pc_idlethread->td_generation) {
1015 			error = tsleep(quiesce_cpus, prio & ~PDROP, wmesg, 1);
1016 			if (error != EWOULDBLOCK)
1017 				goto out;
1018 			error = 0;
1019 		}
1020 	}
1021 out:
1022 	thread_lock(curthread);
1023 	sched_unbind(curthread);
1024 	thread_unlock(curthread);
1025 	if ((prio & PDROP) == 0)
1026 		free(gen, M_TEMP);
1027 
1028 	return (error);
1029 }
1030 
1031 int
1032 quiesce_all_cpus(const char *wmesg, int prio)
1033 {
1034 
1035 	return quiesce_cpus(all_cpus, wmesg, prio);
1036 }
1037 
1038 /*
1039  * Observe all CPUs not executing in critical section.
1040  * We are not in one so the check for us is safe. If the found
1041  * thread changes to something else we know the section was
1042  * exited as well.
1043  */
1044 void
1045 quiesce_all_critical(void)
1046 {
1047 	struct thread *td, *newtd;
1048 	struct pcpu *pcpu;
1049 	int cpu;
1050 
1051 	MPASS(curthread->td_critnest == 0);
1052 
1053 	CPU_FOREACH(cpu) {
1054 		pcpu = cpuid_to_pcpu[cpu];
1055 		td = pcpu->pc_curthread;
1056 		for (;;) {
1057 			if (td->td_critnest == 0)
1058 				break;
1059 			cpu_spinwait();
1060 			newtd = (struct thread *)
1061 			    atomic_load_acq_ptr((void *)pcpu->pc_curthread);
1062 			if (td != newtd)
1063 				break;
1064 		}
1065 	}
1066 }
1067 
1068 static void
1069 cpus_fence_seq_cst_issue(void *arg __unused)
1070 {
1071 
1072 	atomic_thread_fence_seq_cst();
1073 }
1074 
1075 /*
1076  * Send an IPI forcing a sequentially consistent fence.
1077  *
1078  * Allows replacement of an explicitly fence with a compiler barrier.
1079  * Trades speed up during normal execution for a significant slowdown when
1080  * the barrier is needed.
1081  */
1082 void
1083 cpus_fence_seq_cst(void)
1084 {
1085 
1086 #ifdef SMP
1087 	smp_rendezvous(
1088 	    smp_no_rendezvous_barrier,
1089 	    cpus_fence_seq_cst_issue,
1090 	    smp_no_rendezvous_barrier,
1091 	    NULL
1092 	);
1093 #else
1094 	cpus_fence_seq_cst_issue(NULL);
1095 #endif
1096 }
1097 
1098 /* Extra care is taken with this sysctl because the data type is volatile */
1099 static int
1100 sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS)
1101 {
1102 	int error, active;
1103 
1104 	active = smp_started;
1105 	error = SYSCTL_OUT(req, &active, sizeof(active));
1106 	return (error);
1107 }
1108 
1109 #ifdef SMP
1110 void
1111 topo_init_node(struct topo_node *node)
1112 {
1113 
1114 	bzero(node, sizeof(*node));
1115 	TAILQ_INIT(&node->children);
1116 }
1117 
1118 void
1119 topo_init_root(struct topo_node *root)
1120 {
1121 
1122 	topo_init_node(root);
1123 	root->type = TOPO_TYPE_SYSTEM;
1124 }
1125 
1126 /*
1127  * Add a child node with the given ID under the given parent.
1128  * Do nothing if there is already a child with that ID.
1129  */
1130 struct topo_node *
1131 topo_add_node_by_hwid(struct topo_node *parent, int hwid,
1132     topo_node_type type, uintptr_t subtype)
1133 {
1134 	struct topo_node *node;
1135 
1136 	TAILQ_FOREACH_REVERSE(node, &parent->children,
1137 	    topo_children, siblings) {
1138 		if (node->hwid == hwid
1139 		    && node->type == type && node->subtype == subtype) {
1140 			return (node);
1141 		}
1142 	}
1143 
1144 	node = malloc(sizeof(*node), M_TOPO, M_WAITOK);
1145 	topo_init_node(node);
1146 	node->parent = parent;
1147 	node->hwid = hwid;
1148 	node->type = type;
1149 	node->subtype = subtype;
1150 	TAILQ_INSERT_TAIL(&parent->children, node, siblings);
1151 	parent->nchildren++;
1152 
1153 	return (node);
1154 }
1155 
1156 /*
1157  * Find a child node with the given ID under the given parent.
1158  */
1159 struct topo_node *
1160 topo_find_node_by_hwid(struct topo_node *parent, int hwid,
1161     topo_node_type type, uintptr_t subtype)
1162 {
1163 
1164 	struct topo_node *node;
1165 
1166 	TAILQ_FOREACH(node, &parent->children, siblings) {
1167 		if (node->hwid == hwid
1168 		    && node->type == type && node->subtype == subtype) {
1169 			return (node);
1170 		}
1171 	}
1172 
1173 	return (NULL);
1174 }
1175 
1176 /*
1177  * Given a node change the order of its parent's child nodes such
1178  * that the node becomes the firt child while preserving the cyclic
1179  * order of the children.  In other words, the given node is promoted
1180  * by rotation.
1181  */
1182 void
1183 topo_promote_child(struct topo_node *child)
1184 {
1185 	struct topo_node *next;
1186 	struct topo_node *node;
1187 	struct topo_node *parent;
1188 
1189 	parent = child->parent;
1190 	next = TAILQ_NEXT(child, siblings);
1191 	TAILQ_REMOVE(&parent->children, child, siblings);
1192 	TAILQ_INSERT_HEAD(&parent->children, child, siblings);
1193 
1194 	while (next != NULL) {
1195 		node = next;
1196 		next = TAILQ_NEXT(node, siblings);
1197 		TAILQ_REMOVE(&parent->children, node, siblings);
1198 		TAILQ_INSERT_AFTER(&parent->children, child, node, siblings);
1199 		child = node;
1200 	}
1201 }
1202 
1203 /*
1204  * Iterate to the next node in the depth-first search (traversal) of
1205  * the topology tree.
1206  */
1207 struct topo_node *
1208 topo_next_node(struct topo_node *top, struct topo_node *node)
1209 {
1210 	struct topo_node *next;
1211 
1212 	if ((next = TAILQ_FIRST(&node->children)) != NULL)
1213 		return (next);
1214 
1215 	if ((next = TAILQ_NEXT(node, siblings)) != NULL)
1216 		return (next);
1217 
1218 	while (node != top && (node = node->parent) != top)
1219 		if ((next = TAILQ_NEXT(node, siblings)) != NULL)
1220 			return (next);
1221 
1222 	return (NULL);
1223 }
1224 
1225 /*
1226  * Iterate to the next node in the depth-first search of the topology tree,
1227  * but without descending below the current node.
1228  */
1229 struct topo_node *
1230 topo_next_nonchild_node(struct topo_node *top, struct topo_node *node)
1231 {
1232 	struct topo_node *next;
1233 
1234 	if ((next = TAILQ_NEXT(node, siblings)) != NULL)
1235 		return (next);
1236 
1237 	while (node != top && (node = node->parent) != top)
1238 		if ((next = TAILQ_NEXT(node, siblings)) != NULL)
1239 			return (next);
1240 
1241 	return (NULL);
1242 }
1243 
1244 /*
1245  * Assign the given ID to the given topology node that represents a logical
1246  * processor.
1247  */
1248 void
1249 topo_set_pu_id(struct topo_node *node, cpuid_t id)
1250 {
1251 
1252 	KASSERT(node->type == TOPO_TYPE_PU,
1253 	    ("topo_set_pu_id: wrong node type: %u", node->type));
1254 	KASSERT(CPU_EMPTY(&node->cpuset) && node->cpu_count == 0,
1255 	    ("topo_set_pu_id: cpuset already not empty"));
1256 	node->id = id;
1257 	CPU_SET(id, &node->cpuset);
1258 	node->cpu_count = 1;
1259 	node->subtype = 1;
1260 
1261 	while ((node = node->parent) != NULL) {
1262 		KASSERT(!CPU_ISSET(id, &node->cpuset),
1263 		    ("logical ID %u is already set in node %p", id, node));
1264 		CPU_SET(id, &node->cpuset);
1265 		node->cpu_count++;
1266 	}
1267 }
1268 
1269 static struct topology_spec {
1270 	topo_node_type	type;
1271 	bool		match_subtype;
1272 	uintptr_t	subtype;
1273 } topology_level_table[TOPO_LEVEL_COUNT] = {
1274 	[TOPO_LEVEL_PKG] = { .type = TOPO_TYPE_PKG, },
1275 	[TOPO_LEVEL_GROUP] = { .type = TOPO_TYPE_GROUP, },
1276 	[TOPO_LEVEL_CACHEGROUP] = {
1277 		.type = TOPO_TYPE_CACHE,
1278 		.match_subtype = true,
1279 		.subtype = CG_SHARE_L3,
1280 	},
1281 	[TOPO_LEVEL_CORE] = { .type = TOPO_TYPE_CORE, },
1282 	[TOPO_LEVEL_THREAD] = { .type = TOPO_TYPE_PU, },
1283 };
1284 
1285 static bool
1286 topo_analyze_table(struct topo_node *root, int all, enum topo_level level,
1287     struct topo_analysis *results)
1288 {
1289 	struct topology_spec *spec;
1290 	struct topo_node *node;
1291 	int count;
1292 
1293 	if (level >= TOPO_LEVEL_COUNT)
1294 		return (true);
1295 
1296 	spec = &topology_level_table[level];
1297 	count = 0;
1298 	node = topo_next_node(root, root);
1299 
1300 	while (node != NULL) {
1301 		if (node->type != spec->type ||
1302 		    (spec->match_subtype && node->subtype != spec->subtype)) {
1303 			node = topo_next_node(root, node);
1304 			continue;
1305 		}
1306 		if (!all && CPU_EMPTY(&node->cpuset)) {
1307 			node = topo_next_nonchild_node(root, node);
1308 			continue;
1309 		}
1310 
1311 		count++;
1312 
1313 		if (!topo_analyze_table(node, all, level + 1, results))
1314 			return (false);
1315 
1316 		node = topo_next_nonchild_node(root, node);
1317 	}
1318 
1319 	/* No explicit subgroups is essentially one subgroup. */
1320 	if (count == 0) {
1321 		count = 1;
1322 
1323 		if (!topo_analyze_table(root, all, level + 1, results))
1324 			return (false);
1325 	}
1326 
1327 	if (results->entities[level] == -1)
1328 		results->entities[level] = count;
1329 	else if (results->entities[level] != count)
1330 		return (false);
1331 
1332 	return (true);
1333 }
1334 
1335 /*
1336  * Check if the topology is uniform, that is, each package has the same number
1337  * of cores in it and each core has the same number of threads (logical
1338  * processors) in it.  If so, calculate the number of packages, the number of
1339  * groups per package, the number of cachegroups per group, and the number of
1340  * logical processors per cachegroup.  'all' parameter tells whether to include
1341  * administratively disabled logical processors into the analysis.
1342  */
1343 int
1344 topo_analyze(struct topo_node *topo_root, int all,
1345     struct topo_analysis *results)
1346 {
1347 
1348 	results->entities[TOPO_LEVEL_PKG] = -1;
1349 	results->entities[TOPO_LEVEL_CORE] = -1;
1350 	results->entities[TOPO_LEVEL_THREAD] = -1;
1351 	results->entities[TOPO_LEVEL_GROUP] = -1;
1352 	results->entities[TOPO_LEVEL_CACHEGROUP] = -1;
1353 
1354 	if (!topo_analyze_table(topo_root, all, TOPO_LEVEL_PKG, results))
1355 		return (0);
1356 
1357 	KASSERT(results->entities[TOPO_LEVEL_PKG] > 0,
1358 		("bug in topology or analysis"));
1359 
1360 	return (1);
1361 }
1362 
1363 #endif /* SMP */
1364