xref: /titanic_41/usr/src/uts/sun4/os/mp_startup.c (revision a8c92b49f4e8de856d9e4775e15e4fa90a687664)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/sysmacros.h>
28 #include <sys/prom_plat.h>
29 #include <sys/prom_debug.h>
30 #include <vm/hat_sfmmu.h>
31 #include <vm/seg_kp.h>
32 #include <vm/seg_kmem.h>
33 #include <sys/machsystm.h>
34 #include <sys/callb.h>
35 #include <sys/cpu_module.h>
36 #include <sys/pg.h>
37 #include <sys/cmt.h>
38 #include <sys/dtrace.h>
39 #include <sys/reboot.h>
40 #include <sys/kdi.h>
41 #include <sys/traptrace.h>
42 #ifdef TRAPTRACE
43 #include <sys/bootconf.h>
44 #endif /* TRAPTRACE */
45 #include <sys/cpu_sgnblk_defs.h>
46 
47 extern int cpu_intrq_setup(struct cpu *);
48 extern void cpu_intrq_cleanup(struct cpu *);
49 extern void cpu_intrq_register(struct cpu *);
50 
51 struct cpu	*cpus;	/* pointer to other cpus; dynamically allocate */
52 struct cpu	*cpu[NCPU];	/* pointers to all CPUs */
53 uint64_t	cpu_pa[NCPU];	/* pointers to all CPUs in PA */
54 cpu_core_t	cpu_core[NCPU];	/* cpu_core structures */
55 
56 #ifdef TRAPTRACE
57 caddr_t	ttrace_buf;	/* kmem64 traptrace for all cpus except 0 */
58 #endif /* TRAPTRACE */
59 
60 /* bit mask of cpus ready for x-calls, protected by cpu_lock */
61 cpuset_t cpu_ready_set;
62 
63 /* bit mask used to communicate with cpus during bringup */
64 static cpuset_t proxy_ready_set;
65 
66 static void	slave_startup(void);
67 
68 /*
69  * Defined in $KARCH/os/mach_mp_startup.c
70  */
71 #pragma weak init_cpu_info
72 
73 /*
74  * Amount of time (in milliseconds) we should wait before giving up on CPU
75  * initialization and assuming that the CPU we're trying to wake up is dead
76  * or out of control.
77  */
78 #define	CPU_WAKEUP_GRACE_MSEC 1000
79 
80 #ifdef	TRAPTRACE
81 /*
82  * This function sets traptrace buffers for all cpus
83  * other than boot cpu.
84  */
85 size_t
86 calc_traptrace_sz(void)
87 {
88 	return (TRAP_TSIZE * (max_ncpus - 1));
89 }
90 #endif	/* TRAPTRACE */
91 
92 
93 /*
94  * common slave cpu initialization code
95  */
96 void
97 common_startup_init(cpu_t *cp, int cpuid)
98 {
99 	kthread_id_t tp;
100 	sfmmu_t *sfmmup;
101 	caddr_t	sp;
102 
103 	/*
104 	 * Allocate and initialize the startup thread for this CPU.
105 	 */
106 	tp = thread_create(NULL, 0, slave_startup, NULL, 0, &p0,
107 	    TS_STOPPED, maxclsyspri);
108 
109 	/*
110 	 * Set state to TS_ONPROC since this thread will start running
111 	 * as soon as the CPU comes online.
112 	 *
113 	 * All the other fields of the thread structure are setup by
114 	 * thread_create().
115 	 */
116 	THREAD_ONPROC(tp, cp);
117 	tp->t_preempt = 1;
118 	tp->t_bound_cpu = cp;
119 	tp->t_affinitycnt = 1;
120 	tp->t_cpu = cp;
121 	tp->t_disp_queue = cp->cpu_disp;
122 
123 	sfmmup = astosfmmu(&kas);
124 	CPUSET_ADD(sfmmup->sfmmu_cpusran, cpuid);
125 
126 	/*
127 	 * Setup thread to start in slave_startup.
128 	 */
129 	sp = tp->t_stk;
130 	tp->t_pc = (uintptr_t)slave_startup - 8;
131 	tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS;
132 
133 	cp->cpu_id = cpuid;
134 	cp->cpu_self = cp;
135 	cp->cpu_thread = tp;
136 	cp->cpu_lwp = NULL;
137 	cp->cpu_dispthread = tp;
138 	cp->cpu_dispatch_pri = DISP_PRIO(tp);
139 	cp->cpu_startup_thread = tp;
140 
141 	/*
142 	 * The dispatcher may discover the CPU before it is in cpu_ready_set
143 	 * and attempt to poke it. Before the CPU is in cpu_ready_set, any
144 	 * cross calls to it will be dropped. We initialize
145 	 * poke_cpu_outstanding to true so that poke_cpu will ignore any poke
146 	 * requests for this CPU. Pokes that come in before the CPU is in
147 	 * cpu_ready_set can be ignored because the CPU is about to come
148 	 * online.
149 	 */
150 	cp->cpu_m.poke_cpu_outstanding = B_TRUE;
151 }
152 
153 /*
154  * parametric flag setting functions.  these routines set the cpu
155  * state just prior to releasing the slave cpu.
156  */
157 void
158 cold_flag_set(int cpuid)
159 {
160 	cpu_t *cp;
161 
162 	ASSERT(MUTEX_HELD(&cpu_lock));
163 
164 	cp = cpu[cpuid];
165 	cp->cpu_flags |= CPU_RUNNING | CPU_ENABLE | CPU_EXISTS;
166 	cpu_add_active(cp);
167 	/*
168 	 * Add CPU_READY after the cpu_add_active() call
169 	 * to avoid pausing cp.
170 	 */
171 	cp->cpu_flags |= CPU_READY;		/* ready */
172 	cpu_set_state(cp);
173 }
174 
175 static void
176 warm_flag_set(int cpuid)
177 {
178 	cpu_t *cp;
179 
180 	ASSERT(MUTEX_HELD(&cpu_lock));
181 
182 	/*
183 	 * warm start activates cpus into the OFFLINE state
184 	 */
185 	cp = cpu[cpuid];
186 	cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS
187 	    | CPU_OFFLINE | CPU_QUIESCED;
188 	cpu_set_state(cp);
189 }
190 
191 /*
192  * Internal cpu startup sequencer
193  * The sequence is as follows:
194  *
195  * MASTER	SLAVE
196  * -------	----------
197  * assume the kernel data is initialized
198  * clear the proxy bit
199  * start the slave cpu
200  * wait for the slave cpu to set the proxy
201  *
202  *		the slave runs slave_startup and then sets the proxy
203  *		the slave waits for the master to add slave to the ready set
204  *
205  * the master finishes the initialization and
206  * adds the slave to the ready set
207  *
208  *		the slave exits the startup thread and is running
209  */
210 void
211 start_cpu(int cpuid, void(*flag_func)(int))
212 {
213 	extern void cpu_startup(int);
214 	int timout;
215 
216 	ASSERT(MUTEX_HELD(&cpu_lock));
217 
218 	/*
219 	 * Before we begin the dance, tell DTrace that we're about to start
220 	 * a CPU.
221 	 */
222 	if (dtrace_cpustart_init != NULL)
223 		(*dtrace_cpustart_init)();
224 
225 	/* start the slave cpu */
226 	CPUSET_DEL(proxy_ready_set, cpuid);
227 	if (prom_test("SUNW,start-cpu-by-cpuid") == 0) {
228 		(void) prom_startcpu_bycpuid(cpuid, (caddr_t)&cpu_startup,
229 		    cpuid);
230 	} else {
231 		/* "by-cpuid" interface didn't exist.  Do it the old way */
232 		pnode_t nodeid = cpunodes[cpuid].nodeid;
233 
234 		ASSERT(nodeid != (pnode_t)0);
235 		(void) prom_startcpu(nodeid, (caddr_t)&cpu_startup, cpuid);
236 	}
237 
238 	/* wait for the slave cpu to check in. */
239 	for (timout = CPU_WAKEUP_GRACE_MSEC; timout; timout--) {
240 		if (CPU_IN_SET(proxy_ready_set, cpuid))
241 			break;
242 		DELAY(1000);
243 	}
244 	if (timout == 0) {
245 		panic("cpu%d failed to start (2)", cpuid);
246 	}
247 
248 	/*
249 	 * The slave has started; we can tell DTrace that it's safe again.
250 	 */
251 	if (dtrace_cpustart_fini != NULL)
252 		(*dtrace_cpustart_fini)();
253 
254 	/* run the master side of stick synchronization for the slave cpu */
255 	sticksync_master();
256 
257 	/*
258 	 * deal with the cpu flags in a phase-specific manner
259 	 * for various reasons, this needs to run after the slave
260 	 * is checked in but before the slave is released.
261 	 */
262 	(*flag_func)(cpuid);
263 
264 	/* release the slave */
265 	CPUSET_ADD(cpu_ready_set, cpuid);
266 }
267 
268 #ifdef TRAPTRACE
269 int trap_tr0_inuse = 1;	/* it is always used on the boot cpu */
270 int trap_trace_inuse[NCPU];
271 #endif /* TRAPTRACE */
272 
273 #define	cpu_next_free	cpu_prev
274 
275 /*
276  * Routine to set up a CPU to prepare for starting it up.
277  */
278 int
279 setup_cpu_common(int cpuid)
280 {
281 	struct cpu *cp = NULL;
282 	kthread_id_t tp;
283 #ifdef TRAPTRACE
284 	int tt_index;
285 	TRAP_TRACE_CTL	*ctlp;
286 	caddr_t	newbuf;
287 #endif /* TRAPTRACE */
288 
289 	extern void idle();
290 	int	rval;
291 
292 	ASSERT(MUTEX_HELD(&cpu_lock));
293 	ASSERT(cpu[cpuid] == NULL);
294 
295 	ASSERT(ncpus <= max_ncpus);
296 
297 #ifdef TRAPTRACE
298 	/*
299 	 * allocate a traptrace buffer for this CPU.
300 	 */
301 	ctlp = &trap_trace_ctl[cpuid];
302 	if (!trap_tr0_inuse) {
303 		trap_tr0_inuse = 1;
304 		newbuf = trap_tr0;
305 		tt_index = -1;
306 	} else {
307 		for (tt_index = 0; tt_index < (max_ncpus-1); tt_index++)
308 			if (!trap_trace_inuse[tt_index])
309 				break;
310 		ASSERT(tt_index < max_ncpus - 1);
311 		trap_trace_inuse[tt_index] = 1;
312 		newbuf = (caddr_t)(ttrace_buf + (tt_index * TRAP_TSIZE));
313 	}
314 	ctlp->d.vaddr_base = newbuf;
315 	ctlp->d.offset = ctlp->d.last_offset = 0;
316 	ctlp->d.limit = trap_trace_bufsize;
317 	ctlp->d.paddr_base = va_to_pa(newbuf);
318 	ASSERT(ctlp->d.paddr_base != (uint64_t)-1);
319 #endif /* TRAPTRACE */
320 	/*
321 	 * initialize hv traptrace buffer for this CPU
322 	 */
323 	mach_htraptrace_setup(cpuid);
324 
325 	/*
326 	 * Obtain pointer to the appropriate cpu structure.
327 	 */
328 	if (cpu0.cpu_flags == 0) {
329 		cp = &cpu0;
330 	} else {
331 		/*
332 		 *  When dynamically allocating cpu structs,
333 		 *  cpus is used as a pointer to a list of freed
334 		 *  cpu structs.
335 		 */
336 		if (cpus) {
337 			/* grab the first cpu struct on the free list */
338 			cp = cpus;
339 			if (cp->cpu_next_free)
340 				cpus = cp->cpu_next_free;
341 			else
342 				cpus = NULL;
343 		}
344 	}
345 
346 	if (cp == NULL)
347 		cp = vmem_xalloc(static_alloc_arena, CPU_ALLOC_SIZE,
348 		    CPU_ALLOC_SIZE, 0, 0, NULL, NULL, VM_SLEEP);
349 
350 	bzero(cp, sizeof (*cp));
351 
352 	cp->cpu_id = cpuid;
353 	cp->cpu_self = cp;
354 
355 	/*
356 	 * Initialize ptl1_panic stack
357 	 */
358 	ptl1_init_cpu(cp);
359 
360 	/*
361 	 * Initialize the dispatcher for this CPU.
362 	 */
363 	disp_cpu_init(cp);
364 
365 	/*
366 	 * Bootstrap the CPU's PG data
367 	 */
368 	pg_cpu_bootstrap(cp);
369 
370 	cpu_vm_data_init(cp);
371 
372 	/*
373 	 * Now, initialize per-CPU idle thread for this CPU.
374 	 */
375 	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_ONPROC, -1);
376 
377 	cp->cpu_idle_thread = tp;
378 
379 	tp->t_preempt = 1;
380 	tp->t_bound_cpu = cp;
381 	tp->t_affinitycnt = 1;
382 	tp->t_cpu = cp;
383 	tp->t_disp_queue = cp->cpu_disp;
384 
385 	/*
386 	 * Registering a thread in the callback table is usually
387 	 * done in the initialization code of the thread. In this
388 	 * case, we do it right after thread creation to avoid
389 	 * blocking idle thread while registering itself. It also
390 	 * avoids the possibility of reregistration in case a CPU
391 	 * restarts its idle thread.
392 	 */
393 	CALLB_CPR_INIT_SAFE(tp, "idle");
394 
395 	init_cpu_info(cp);
396 
397 	/*
398 	 * Initialize the interrupt threads for this CPU
399 	 */
400 	cpu_intr_alloc(cp, NINTR_THREADS);
401 
402 	/*
403 	 * Add CPU to list of available CPUs.
404 	 * It'll be on the active list after it is started.
405 	 */
406 	cpu_add_unit(cp);
407 
408 	/*
409 	 * Allocate and init cpu module private data structures,
410 	 * including scrubber.
411 	 */
412 	cpu_init_private(cp);
413 	populate_idstr(cp);
414 
415 	/*
416 	 * Initialize the CPUs physical ID cache, and processor groups
417 	 */
418 	pghw_physid_create(cp);
419 	(void) pg_cpu_init(cp, B_FALSE);
420 
421 	if ((rval = cpu_intrq_setup(cp)) != 0) {
422 		return (rval);
423 	}
424 
425 	/*
426 	 * Initialize MMU context domain information.
427 	 */
428 	sfmmu_cpu_init(cp);
429 
430 	return (0);
431 }
432 
433 /*
434  * Routine to clean up a CPU after shutting it down.
435  */
436 int
437 cleanup_cpu_common(int cpuid)
438 {
439 	struct cpu *cp;
440 #ifdef TRAPTRACE
441 	int i;
442 	TRAP_TRACE_CTL	*ctlp;
443 	caddr_t	newbuf;
444 #endif /* TRAPTRACE */
445 
446 	ASSERT(MUTEX_HELD(&cpu_lock));
447 	ASSERT(cpu[cpuid] != NULL);
448 
449 	cp = cpu[cpuid];
450 
451 	/* Free cpu module private data structures, including scrubber. */
452 	cpu_uninit_private(cp);
453 
454 	/* Free cpu ID string and brand string. */
455 	if (cp->cpu_idstr)
456 		kmem_free(cp->cpu_idstr, strlen(cp->cpu_idstr) + 1);
457 	if (cp->cpu_brandstr)
458 		kmem_free(cp->cpu_brandstr, strlen(cp->cpu_brandstr) + 1);
459 
460 	cpu_vm_data_destroy(cp);
461 
462 	/*
463 	 * Remove CPU from list of available CPUs.
464 	 */
465 	cpu_del_unit(cpuid);
466 
467 	/*
468 	 * Clean any machine specific interrupt states.
469 	 */
470 	cpu_intrq_cleanup(cp);
471 
472 	/*
473 	 * At this point, the only threads bound to this CPU should be
474 	 * special per-cpu threads: it's idle thread, it's pause thread,
475 	 * and it's interrupt threads.  Clean these up.
476 	 */
477 	cpu_destroy_bound_threads(cp);
478 
479 	/*
480 	 * Free the interrupt stack.
481 	 */
482 	segkp_release(segkp, cp->cpu_intr_stack);
483 
484 	/*
485 	 * Free hv traptrace buffer for this CPU.
486 	 */
487 	mach_htraptrace_cleanup(cpuid);
488 #ifdef TRAPTRACE
489 	/*
490 	 * Free the traptrace buffer for this CPU.
491 	 */
492 	ctlp = &trap_trace_ctl[cpuid];
493 	newbuf = ctlp->d.vaddr_base;
494 	i = (newbuf - ttrace_buf) / (TRAP_TSIZE);
495 	if (((newbuf - ttrace_buf) % (TRAP_TSIZE) == 0) &&
496 	    ((i >= 0) && (i < (max_ncpus-1)))) {
497 		/*
498 		 * This CPU got it's trap trace buffer from the
499 		 * boot-alloc'd bunch of them.
500 		 */
501 		trap_trace_inuse[i] = 0;
502 		bzero(newbuf, (TRAP_TSIZE));
503 	} else if (newbuf == trap_tr0) {
504 		trap_tr0_inuse = 0;
505 		bzero(trap_tr0, (TRAP_TSIZE));
506 	} else {
507 		cmn_err(CE_WARN, "failed to free trap trace buffer from cpu%d",
508 		    cpuid);
509 	}
510 	bzero(ctlp, sizeof (*ctlp));
511 #endif /* TRAPTRACE */
512 
513 	/*
514 	 * There is a race condition with mutex_vector_enter() which
515 	 * caches a cpu pointer. The race is detected by checking cpu_next.
516 	 */
517 	disp_cpu_fini(cp);
518 	cpu_pa[cpuid] = 0;
519 	if (CPU_MMU_CTXP(cp))
520 		sfmmu_cpu_cleanup(cp);
521 	bzero(cp, sizeof (*cp));
522 
523 	/*
524 	 * Place the freed cpu structure on the list of freed cpus.
525 	 */
526 	if (cp != &cpu0) {
527 		if (cpus) {
528 			cp->cpu_next_free = cpus;
529 			cpus = cp;
530 		}
531 		else
532 			cpus = cp;
533 	}
534 
535 	return (0);
536 }
537 
538 /*
539  * This routine is used to start a previously powered off processor.
540  * Note that restarted cpus are initialized into the offline state.
541  */
542 void
543 restart_other_cpu(int cpuid)
544 {
545 	struct cpu *cp;
546 	kthread_id_t tp;
547 	caddr_t	sp;
548 	extern void idle();
549 
550 	ASSERT(MUTEX_HELD(&cpu_lock));
551 	ASSERT(cpuid < NCPU && cpu[cpuid] != NULL);
552 
553 	/*
554 	 * Obtain pointer to the appropriate cpu structure.
555 	 */
556 	cp = cpu[cpuid];
557 
558 	common_startup_init(cp, cpuid);
559 
560 	/*
561 	 * idle thread t_lock is held when the idle thread is suspended.
562 	 * Manually unlock the t_lock of idle loop so that we can resume
563 	 * the suspended idle thread.
564 	 * Also adjust the PC of idle thread for re-retry.
565 	 */
566 	cp->cpu_intr_actv = 0;	/* clear the value from previous life */
567 	cp->cpu_m.mutex_ready = 0; /* we are not ready yet */
568 	lock_clear(&cp->cpu_idle_thread->t_lock);
569 	tp = cp->cpu_idle_thread;
570 
571 	sp = tp->t_stk;
572 	tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS;
573 	tp->t_pc = (uintptr_t)idle - 8;
574 
575 	/*
576 	 * restart the cpu now
577 	 */
578 	promsafe_pause_cpus();
579 	start_cpu(cpuid, warm_flag_set);
580 	start_cpus();
581 
582 	/* call cmn_err outside pause_cpus/start_cpus to avoid deadlock */
583 	cmn_err(CE_CONT, "!cpu%d initialization complete - restarted\n",
584 	    cpuid);
585 }
586 
587 /*
588  * Startup function executed on 'other' CPUs.  This is the first
589  * C function after cpu_start sets up the cpu registers.
590  */
591 static void
592 slave_startup(void)
593 {
594 	struct cpu	*cp = CPU;
595 	ushort_t	original_flags = cp->cpu_flags;
596 
597 	mach_htraptrace_configure(cp->cpu_id);
598 	cpu_intrq_register(CPU);
599 	cp->cpu_m.mutex_ready = 1;
600 
601 	/* acknowledge that we are done with initialization */
602 	CPUSET_ADD(proxy_ready_set, cp->cpu_id);
603 
604 	/* synchronize STICK */
605 	sticksync_slave();
606 
607 	if (boothowto & RB_DEBUG)
608 		kdi_dvec_cpu_init(cp);
609 
610 	/*
611 	 * the slave will wait here forever -- assuming that the master
612 	 * will get back to us.  if it doesn't we've got bigger problems
613 	 * than a master not replying to this slave.
614 	 * the small delay improves the slave's responsiveness to the
615 	 * master's ack and decreases the time window between master and
616 	 * slave operations.
617 	 */
618 	while (!CPU_IN_SET(cpu_ready_set, cp->cpu_id))
619 		DELAY(1);
620 
621 	/*
622 	 * The CPU is now in cpu_ready_set, safely able to take pokes.
623 	 */
624 	cp->cpu_m.poke_cpu_outstanding = B_FALSE;
625 
626 	/* enable interrupts */
627 	(void) spl0();
628 
629 	/*
630 	 * Signature block update to indicate that this CPU is in OS now.
631 	 * This needs to be done after the PIL is lowered since on
632 	 * some platforms the update code may block.
633 	 */
634 	CPU_SIGNATURE(OS_SIG, SIGST_RUN, SIGSUBST_NULL, cp->cpu_id);
635 
636 	/*
637 	 * park the slave thread in a safe/quiet state and wait for the master
638 	 * to finish configuring this CPU before proceeding to thread_exit().
639 	 */
640 	while (((volatile ushort_t)cp->cpu_flags) & CPU_QUIESCED)
641 		DELAY(1);
642 
643 	/*
644 	 * Initialize CPC CPU state.
645 	 */
646 	kcpc_hw_startup_cpu(original_flags);
647 
648 	/*
649 	 * Notify the PG subsystem that the CPU  has started
650 	 */
651 	pg_cmt_cpu_startup(CPU);
652 
653 	/*
654 	 * Now we are done with the startup thread, so free it up.
655 	 */
656 	thread_exit();
657 	cmn_err(CE_PANIC, "slave_startup: cannot return");
658 	/*NOTREACHED*/
659 }
660 
661 extern struct cpu	*cpu[NCPU];	/* pointers to all CPUs */
662 
663 /*
664  * cpu_bringup_set is a tunable (via /etc/system, debugger, etc.) that
665  * can be used during debugging to control which processors are brought
666  * online at boot time.  The variable represents a bitmap of the id's
667  * of the processors that will be brought online.  The initialization
668  * of this variable depends on the type of cpuset_t, which varies
669  * depending on the number of processors supported (see cpuvar.h).
670  */
671 cpuset_t cpu_bringup_set;
672 
673 
674 /*
675  * Generic start-all cpus entry.  Typically used during cold initialization.
676  * Note that cold start cpus are initialized into the online state.
677  */
678 /*ARGSUSED*/
679 void
680 start_other_cpus(int flag)
681 {
682 	int cpuid;
683 	extern void idlestop_init(void);
684 	int bootcpu;
685 
686 	/*
687 	 * Check if cpu_bringup_set has been explicitly set before
688 	 * initializing it.
689 	 */
690 	if (CPUSET_ISNULL(cpu_bringup_set)) {
691 		CPUSET_ALL(cpu_bringup_set);
692 	}
693 
694 	if (&cpu_feature_init)
695 		cpu_feature_init();
696 
697 	/*
698 	 * Initialize CPC.
699 	 */
700 	kcpc_hw_init();
701 
702 	mutex_enter(&cpu_lock);
703 
704 	/*
705 	 * Initialize our own cpu_info.
706 	 */
707 	init_cpu_info(CPU);
708 
709 	/*
710 	 * Initialize CPU 0 cpu module private data area, including scrubber.
711 	 */
712 	cpu_init_private(CPU);
713 	populate_idstr(CPU);
714 
715 	/*
716 	 * perform such initialization as is needed
717 	 * to be able to take CPUs on- and off-line.
718 	 */
719 	cpu_pause_init();
720 	xc_init();		/* initialize processor crosscalls */
721 	idlestop_init();
722 
723 	if (!use_mp) {
724 		mutex_exit(&cpu_lock);
725 		cmn_err(CE_CONT, "?***** Not in MP mode\n");
726 		return;
727 	}
728 	/*
729 	 * should we be initializing this cpu?
730 	 */
731 	bootcpu = getprocessorid();
732 
733 	/*
734 	 * launch all the slave cpus now
735 	 */
736 	for (cpuid = 0; cpuid < NCPU; cpuid++) {
737 		pnode_t nodeid = cpunodes[cpuid].nodeid;
738 
739 		if (nodeid == (pnode_t)0)
740 			continue;
741 
742 		if (cpuid == bootcpu) {
743 			if (!CPU_IN_SET(cpu_bringup_set, cpuid)) {
744 				cmn_err(CE_WARN, "boot cpu not a member "
745 				    "of cpu_bringup_set, adding it");
746 				CPUSET_ADD(cpu_bringup_set, cpuid);
747 			}
748 			continue;
749 		}
750 		if (!CPU_IN_SET(cpu_bringup_set, cpuid))
751 			continue;
752 
753 		ASSERT(cpu[cpuid] == NULL);
754 
755 		if (setup_cpu_common(cpuid)) {
756 			cmn_err(CE_PANIC, "cpu%d: setup failed", cpuid);
757 		}
758 
759 		common_startup_init(cpu[cpuid], cpuid);
760 
761 		start_cpu(cpuid, cold_flag_set);
762 		/*
763 		 * Because slave_startup() gets fired off after init()
764 		 * starts, we can't use the '?' trick to do 'boot -v'
765 		 * printing - so we always direct the 'cpu .. online'
766 		 * messages to the log.
767 		 */
768 		cmn_err(CE_CONT, "!cpu%d initialization complete - online\n",
769 		    cpuid);
770 
771 		cpu_state_change_notify(cpuid, CPU_SETUP);
772 
773 		if (dtrace_cpu_init != NULL)
774 			(*dtrace_cpu_init)(cpuid);
775 	}
776 
777 	/*
778 	 * since all the cpus are online now, redistribute interrupts to them.
779 	 */
780 	intr_redist_all_cpus();
781 
782 	mutex_exit(&cpu_lock);
783 
784 	/*
785 	 * Start the Ecache scrubber.  Must be done after all calls to
786 	 * cpu_init_private for every cpu (including CPU 0).
787 	 */
788 	cpu_init_cache_scrub();
789 
790 	if (&cpu_mp_init)
791 		cpu_mp_init();
792 }
793