1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/sysmacros.h>
28 #include <sys/prom_plat.h>
29 #include <sys/prom_debug.h>
30 #include <vm/hat_sfmmu.h>
31 #include <vm/seg_kp.h>
32 #include <vm/seg_kmem.h>
33 #include <sys/machsystm.h>
34 #include <sys/callb.h>
35 #include <sys/cpu_module.h>
36 #include <sys/pg.h>
37 #include <sys/cmt.h>
38 #include <sys/dtrace.h>
39 #include <sys/reboot.h>
40 #include <sys/kdi.h>
41 #include <sys/traptrace.h>
42 #ifdef TRAPTRACE
43 #include <sys/bootconf.h>
44 #endif /* TRAPTRACE */
45 #include <sys/cpu_sgnblk_defs.h>
46
47 extern int cpu_intrq_setup(struct cpu *);
48 extern void cpu_intrq_cleanup(struct cpu *);
49 extern void cpu_intrq_register(struct cpu *);
50
51 struct cpu *cpus; /* pointer to other cpus; dynamically allocate */
52 struct cpu *cpu[NCPU]; /* pointers to all CPUs */
53 uint64_t cpu_pa[NCPU]; /* pointers to all CPUs in PA */
54 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */
55
56 #ifdef TRAPTRACE
57 caddr_t ttrace_buf; /* kmem64 traptrace for all cpus except 0 */
58 #endif /* TRAPTRACE */
59
60 /* bit mask of cpus ready for x-calls, protected by cpu_lock */
61 cpuset_t cpu_ready_set;
62
63 /* bit mask used to communicate with cpus during bringup */
64 static cpuset_t proxy_ready_set;
65
66 static void slave_startup(void);
67
68 /*
69 * Defined in $KARCH/os/mach_mp_startup.c
70 */
71 #pragma weak init_cpu_info
72
73 /*
74 * Amount of time (in milliseconds) we should wait before giving up on CPU
75 * initialization and assuming that the CPU we're trying to wake up is dead
76 * or out of control.
77 */
78 #define CPU_WAKEUP_GRACE_MSEC 1000
79
80 #ifdef TRAPTRACE
81 /*
82 * This function sets traptrace buffers for all cpus
83 * other than boot cpu.
84 */
85 size_t
calc_traptrace_sz(void)86 calc_traptrace_sz(void)
87 {
88 return (TRAP_TSIZE * (max_ncpus - 1));
89 }
90 #endif /* TRAPTRACE */
91
92
93 /*
94 * common slave cpu initialization code
95 */
96 void
common_startup_init(cpu_t * cp,int cpuid)97 common_startup_init(cpu_t *cp, int cpuid)
98 {
99 kthread_id_t tp;
100 sfmmu_t *sfmmup;
101 caddr_t sp;
102
103 /*
104 * Allocate and initialize the startup thread for this CPU.
105 */
106 tp = thread_create(NULL, 0, slave_startup, NULL, 0, &p0,
107 TS_STOPPED, maxclsyspri);
108
109 /*
110 * Set state to TS_ONPROC since this thread will start running
111 * as soon as the CPU comes online.
112 *
113 * All the other fields of the thread structure are setup by
114 * thread_create().
115 */
116 THREAD_ONPROC(tp, cp);
117 tp->t_preempt = 1;
118 tp->t_bound_cpu = cp;
119 tp->t_affinitycnt = 1;
120 tp->t_cpu = cp;
121 tp->t_disp_queue = cp->cpu_disp;
122
123 sfmmup = astosfmmu(&kas);
124 CPUSET_ADD(sfmmup->sfmmu_cpusran, cpuid);
125
126 /*
127 * Setup thread to start in slave_startup.
128 */
129 sp = tp->t_stk;
130 tp->t_pc = (uintptr_t)slave_startup - 8;
131 tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS;
132
133 cp->cpu_id = cpuid;
134 cp->cpu_self = cp;
135 cp->cpu_thread = tp;
136 cp->cpu_lwp = NULL;
137 cp->cpu_dispthread = tp;
138 cp->cpu_dispatch_pri = DISP_PRIO(tp);
139 cp->cpu_startup_thread = tp;
140
141 /*
142 * The dispatcher may discover the CPU before it is in cpu_ready_set
143 * and attempt to poke it. Before the CPU is in cpu_ready_set, any
144 * cross calls to it will be dropped. We initialize
145 * poke_cpu_outstanding to true so that poke_cpu will ignore any poke
146 * requests for this CPU. Pokes that come in before the CPU is in
147 * cpu_ready_set can be ignored because the CPU is about to come
148 * online.
149 */
150 cp->cpu_m.poke_cpu_outstanding = B_TRUE;
151 }
152
153 /*
154 * parametric flag setting functions. these routines set the cpu
155 * state just prior to releasing the slave cpu.
156 */
157 void
cold_flag_set(int cpuid)158 cold_flag_set(int cpuid)
159 {
160 cpu_t *cp;
161
162 ASSERT(MUTEX_HELD(&cpu_lock));
163
164 cp = cpu[cpuid];
165 cp->cpu_flags |= CPU_RUNNING | CPU_ENABLE | CPU_EXISTS;
166 cpu_add_active(cp);
167 /*
168 * Add CPU_READY after the cpu_add_active() call
169 * to avoid pausing cp.
170 */
171 cp->cpu_flags |= CPU_READY; /* ready */
172 cpu_set_state(cp);
173 }
174
175 static void
warm_flag_set(int cpuid)176 warm_flag_set(int cpuid)
177 {
178 cpu_t *cp;
179
180 ASSERT(MUTEX_HELD(&cpu_lock));
181
182 /*
183 * warm start activates cpus into the OFFLINE state
184 */
185 cp = cpu[cpuid];
186 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS
187 | CPU_OFFLINE | CPU_QUIESCED;
188 cpu_set_state(cp);
189 }
190
191 /*
192 * Internal cpu startup sequencer
193 * The sequence is as follows:
194 *
195 * MASTER SLAVE
196 * ------- ----------
197 * assume the kernel data is initialized
198 * clear the proxy bit
199 * start the slave cpu
200 * wait for the slave cpu to set the proxy
201 *
202 * the slave runs slave_startup and then sets the proxy
203 * the slave waits for the master to add slave to the ready set
204 *
205 * the master finishes the initialization and
206 * adds the slave to the ready set
207 *
208 * the slave exits the startup thread and is running
209 */
210 void
start_cpu(int cpuid,void (* flag_func)(int))211 start_cpu(int cpuid, void(*flag_func)(int))
212 {
213 extern void cpu_startup(int);
214 int timout;
215
216 ASSERT(MUTEX_HELD(&cpu_lock));
217
218 /*
219 * Before we begin the dance, tell DTrace that we're about to start
220 * a CPU.
221 */
222 if (dtrace_cpustart_init != NULL)
223 (*dtrace_cpustart_init)();
224
225 /* start the slave cpu */
226 CPUSET_DEL(proxy_ready_set, cpuid);
227 if (prom_test("SUNW,start-cpu-by-cpuid") == 0) {
228 (void) prom_startcpu_bycpuid(cpuid, (caddr_t)&cpu_startup,
229 cpuid);
230 } else {
231 /* "by-cpuid" interface didn't exist. Do it the old way */
232 pnode_t nodeid = cpunodes[cpuid].nodeid;
233
234 ASSERT(nodeid != (pnode_t)0);
235 (void) prom_startcpu(nodeid, (caddr_t)&cpu_startup, cpuid);
236 }
237
238 /* wait for the slave cpu to check in. */
239 for (timout = CPU_WAKEUP_GRACE_MSEC; timout; timout--) {
240 if (CPU_IN_SET(proxy_ready_set, cpuid))
241 break;
242 DELAY(1000);
243 }
244 if (timout == 0) {
245 panic("cpu%d failed to start (2)", cpuid);
246 }
247
248 /*
249 * The slave has started; we can tell DTrace that it's safe again.
250 */
251 if (dtrace_cpustart_fini != NULL)
252 (*dtrace_cpustart_fini)();
253
254 /* run the master side of stick synchronization for the slave cpu */
255 sticksync_master();
256
257 /*
258 * deal with the cpu flags in a phase-specific manner
259 * for various reasons, this needs to run after the slave
260 * is checked in but before the slave is released.
261 */
262 (*flag_func)(cpuid);
263
264 /* release the slave */
265 CPUSET_ADD(cpu_ready_set, cpuid);
266 }
267
268 #ifdef TRAPTRACE
269 int trap_tr0_inuse = 1; /* it is always used on the boot cpu */
270 int trap_trace_inuse[NCPU];
271 #endif /* TRAPTRACE */
272
273 #define cpu_next_free cpu_prev
274
275 /*
276 * Routine to set up a CPU to prepare for starting it up.
277 */
278 int
setup_cpu_common(int cpuid)279 setup_cpu_common(int cpuid)
280 {
281 struct cpu *cp = NULL;
282 kthread_id_t tp;
283 #ifdef TRAPTRACE
284 int tt_index;
285 TRAP_TRACE_CTL *ctlp;
286 caddr_t newbuf;
287 #endif /* TRAPTRACE */
288
289 extern void idle();
290 int rval;
291
292 ASSERT(MUTEX_HELD(&cpu_lock));
293 ASSERT(cpu[cpuid] == NULL);
294
295 ASSERT(ncpus <= max_ncpus);
296
297 #ifdef TRAPTRACE
298 /*
299 * allocate a traptrace buffer for this CPU.
300 */
301 ctlp = &trap_trace_ctl[cpuid];
302 if (!trap_tr0_inuse) {
303 trap_tr0_inuse = 1;
304 newbuf = trap_tr0;
305 tt_index = -1;
306 } else {
307 for (tt_index = 0; tt_index < (max_ncpus-1); tt_index++)
308 if (!trap_trace_inuse[tt_index])
309 break;
310 ASSERT(tt_index < max_ncpus - 1);
311 trap_trace_inuse[tt_index] = 1;
312 newbuf = (caddr_t)(ttrace_buf + (tt_index * TRAP_TSIZE));
313 }
314 ctlp->d.vaddr_base = newbuf;
315 ctlp->d.offset = ctlp->d.last_offset = 0;
316 ctlp->d.limit = trap_trace_bufsize;
317 ctlp->d.paddr_base = va_to_pa(newbuf);
318 ASSERT(ctlp->d.paddr_base != (uint64_t)-1);
319 #endif /* TRAPTRACE */
320 /*
321 * initialize hv traptrace buffer for this CPU
322 */
323 mach_htraptrace_setup(cpuid);
324
325 /*
326 * Obtain pointer to the appropriate cpu structure.
327 */
328 if (cpu0.cpu_flags == 0) {
329 cp = &cpu0;
330 } else {
331 /*
332 * When dynamically allocating cpu structs,
333 * cpus is used as a pointer to a list of freed
334 * cpu structs.
335 */
336 if (cpus) {
337 /* grab the first cpu struct on the free list */
338 cp = cpus;
339 if (cp->cpu_next_free)
340 cpus = cp->cpu_next_free;
341 else
342 cpus = NULL;
343 }
344 }
345
346 if (cp == NULL)
347 cp = vmem_xalloc(static_alloc_arena, CPU_ALLOC_SIZE,
348 CPU_ALLOC_SIZE, 0, 0, NULL, NULL, VM_SLEEP);
349
350 bzero(cp, sizeof (*cp));
351
352 cp->cpu_id = cpuid;
353 cp->cpu_self = cp;
354
355 /*
356 * Initialize ptl1_panic stack
357 */
358 ptl1_init_cpu(cp);
359
360 /*
361 * Initialize the dispatcher for this CPU.
362 */
363 disp_cpu_init(cp);
364
365 /*
366 * Bootstrap the CPU's PG data
367 */
368 pg_cpu_bootstrap(cp);
369
370 cpu_vm_data_init(cp);
371
372 /*
373 * Now, initialize per-CPU idle thread for this CPU.
374 */
375 tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_ONPROC, -1);
376
377 cp->cpu_idle_thread = tp;
378
379 tp->t_preempt = 1;
380 tp->t_bound_cpu = cp;
381 tp->t_affinitycnt = 1;
382 tp->t_cpu = cp;
383 tp->t_disp_queue = cp->cpu_disp;
384
385 /*
386 * Registering a thread in the callback table is usually
387 * done in the initialization code of the thread. In this
388 * case, we do it right after thread creation to avoid
389 * blocking idle thread while registering itself. It also
390 * avoids the possibility of reregistration in case a CPU
391 * restarts its idle thread.
392 */
393 CALLB_CPR_INIT_SAFE(tp, "idle");
394
395 init_cpu_info(cp);
396
397 /*
398 * Initialize the interrupt threads for this CPU
399 */
400 cpu_intr_alloc(cp, NINTR_THREADS);
401
402 /*
403 * Add CPU to list of available CPUs.
404 * It'll be on the active list after it is started.
405 */
406 cpu_add_unit(cp);
407
408 /*
409 * Allocate and init cpu module private data structures,
410 * including scrubber.
411 */
412 cpu_init_private(cp);
413 populate_idstr(cp);
414
415 /*
416 * Initialize the CPUs physical ID cache, and processor groups
417 */
418 pghw_physid_create(cp);
419 (void) pg_cpu_init(cp, B_FALSE);
420
421 if ((rval = cpu_intrq_setup(cp)) != 0) {
422 return (rval);
423 }
424
425 /*
426 * Initialize MMU context domain information.
427 */
428 sfmmu_cpu_init(cp);
429
430 return (0);
431 }
432
433 /*
434 * Routine to clean up a CPU after shutting it down.
435 */
436 int
cleanup_cpu_common(int cpuid)437 cleanup_cpu_common(int cpuid)
438 {
439 struct cpu *cp;
440 #ifdef TRAPTRACE
441 int i;
442 TRAP_TRACE_CTL *ctlp;
443 caddr_t newbuf;
444 #endif /* TRAPTRACE */
445
446 ASSERT(MUTEX_HELD(&cpu_lock));
447 ASSERT(cpu[cpuid] != NULL);
448
449 cp = cpu[cpuid];
450
451 /* Free cpu module private data structures, including scrubber. */
452 cpu_uninit_private(cp);
453
454 /* Free cpu ID string and brand string. */
455 if (cp->cpu_idstr)
456 kmem_free(cp->cpu_idstr, strlen(cp->cpu_idstr) + 1);
457 if (cp->cpu_brandstr)
458 kmem_free(cp->cpu_brandstr, strlen(cp->cpu_brandstr) + 1);
459
460 cpu_vm_data_destroy(cp);
461
462 /*
463 * Remove CPU from list of available CPUs.
464 */
465 cpu_del_unit(cpuid);
466
467 /*
468 * Clean any machine specific interrupt states.
469 */
470 cpu_intrq_cleanup(cp);
471
472 /*
473 * At this point, the only threads bound to this CPU should be
474 * special per-cpu threads: it's idle thread, it's pause thread,
475 * and it's interrupt threads. Clean these up.
476 */
477 cpu_destroy_bound_threads(cp);
478
479 /*
480 * Free the interrupt stack.
481 */
482 segkp_release(segkp, cp->cpu_intr_stack);
483
484 /*
485 * Free hv traptrace buffer for this CPU.
486 */
487 mach_htraptrace_cleanup(cpuid);
488 #ifdef TRAPTRACE
489 /*
490 * Free the traptrace buffer for this CPU.
491 */
492 ctlp = &trap_trace_ctl[cpuid];
493 newbuf = ctlp->d.vaddr_base;
494 i = (newbuf - ttrace_buf) / (TRAP_TSIZE);
495 if (((newbuf - ttrace_buf) % (TRAP_TSIZE) == 0) &&
496 ((i >= 0) && (i < (max_ncpus-1)))) {
497 /*
498 * This CPU got it's trap trace buffer from the
499 * boot-alloc'd bunch of them.
500 */
501 trap_trace_inuse[i] = 0;
502 bzero(newbuf, (TRAP_TSIZE));
503 } else if (newbuf == trap_tr0) {
504 trap_tr0_inuse = 0;
505 bzero(trap_tr0, (TRAP_TSIZE));
506 } else {
507 cmn_err(CE_WARN, "failed to free trap trace buffer from cpu%d",
508 cpuid);
509 }
510 bzero(ctlp, sizeof (*ctlp));
511 #endif /* TRAPTRACE */
512
513 /*
514 * There is a race condition with mutex_vector_enter() which
515 * caches a cpu pointer. The race is detected by checking cpu_next.
516 */
517 disp_cpu_fini(cp);
518 cpu_pa[cpuid] = 0;
519 if (CPU_MMU_CTXP(cp))
520 sfmmu_cpu_cleanup(cp);
521 bzero(cp, sizeof (*cp));
522
523 /*
524 * Place the freed cpu structure on the list of freed cpus.
525 */
526 if (cp != &cpu0) {
527 if (cpus) {
528 cp->cpu_next_free = cpus;
529 cpus = cp;
530 }
531 else
532 cpus = cp;
533 }
534
535 return (0);
536 }
537
538 /*
539 * This routine is used to start a previously powered off processor.
540 * Note that restarted cpus are initialized into the offline state.
541 */
542 void
restart_other_cpu(int cpuid)543 restart_other_cpu(int cpuid)
544 {
545 struct cpu *cp;
546 kthread_id_t tp;
547 caddr_t sp;
548 extern void idle();
549
550 ASSERT(MUTEX_HELD(&cpu_lock));
551 ASSERT(cpuid < NCPU && cpu[cpuid] != NULL);
552
553 /*
554 * Obtain pointer to the appropriate cpu structure.
555 */
556 cp = cpu[cpuid];
557
558 common_startup_init(cp, cpuid);
559
560 /*
561 * idle thread t_lock is held when the idle thread is suspended.
562 * Manually unlock the t_lock of idle loop so that we can resume
563 * the suspended idle thread.
564 * Also adjust the PC of idle thread for re-retry.
565 */
566 cp->cpu_intr_actv = 0; /* clear the value from previous life */
567 cp->cpu_m.mutex_ready = 0; /* we are not ready yet */
568 lock_clear(&cp->cpu_idle_thread->t_lock);
569 tp = cp->cpu_idle_thread;
570
571 sp = tp->t_stk;
572 tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS;
573 tp->t_pc = (uintptr_t)idle - 8;
574
575 /*
576 * restart the cpu now
577 */
578 promsafe_pause_cpus();
579 start_cpu(cpuid, warm_flag_set);
580 start_cpus();
581
582 /* call cmn_err outside pause_cpus/start_cpus to avoid deadlock */
583 cmn_err(CE_CONT, "!cpu%d initialization complete - restarted\n",
584 cpuid);
585 }
586
587 /*
588 * Startup function executed on 'other' CPUs. This is the first
589 * C function after cpu_start sets up the cpu registers.
590 */
591 static void
slave_startup(void)592 slave_startup(void)
593 {
594 struct cpu *cp = CPU;
595 ushort_t original_flags = cp->cpu_flags;
596
597 mach_htraptrace_configure(cp->cpu_id);
598 cpu_intrq_register(CPU);
599 cp->cpu_m.mutex_ready = 1;
600
601 /* acknowledge that we are done with initialization */
602 CPUSET_ADD(proxy_ready_set, cp->cpu_id);
603
604 /* synchronize STICK */
605 sticksync_slave();
606
607 if (boothowto & RB_DEBUG)
608 kdi_dvec_cpu_init(cp);
609
610 /*
611 * the slave will wait here forever -- assuming that the master
612 * will get back to us. if it doesn't we've got bigger problems
613 * than a master not replying to this slave.
614 * the small delay improves the slave's responsiveness to the
615 * master's ack and decreases the time window between master and
616 * slave operations.
617 */
618 while (!CPU_IN_SET(cpu_ready_set, cp->cpu_id))
619 DELAY(1);
620
621 /*
622 * The CPU is now in cpu_ready_set, safely able to take pokes.
623 */
624 cp->cpu_m.poke_cpu_outstanding = B_FALSE;
625
626 /* enable interrupts */
627 (void) spl0();
628
629 /*
630 * Signature block update to indicate that this CPU is in OS now.
631 * This needs to be done after the PIL is lowered since on
632 * some platforms the update code may block.
633 */
634 CPU_SIGNATURE(OS_SIG, SIGST_RUN, SIGSUBST_NULL, cp->cpu_id);
635
636 /*
637 * park the slave thread in a safe/quiet state and wait for the master
638 * to finish configuring this CPU before proceeding to thread_exit().
639 */
640 while (((volatile ushort_t)cp->cpu_flags) & CPU_QUIESCED)
641 DELAY(1);
642
643 /*
644 * Initialize CPC CPU state.
645 */
646 kcpc_hw_startup_cpu(original_flags);
647
648 /*
649 * Notify the PG subsystem that the CPU has started
650 */
651 pg_cmt_cpu_startup(CPU);
652
653 /*
654 * Now we are done with the startup thread, so free it up.
655 */
656 thread_exit();
657 cmn_err(CE_PANIC, "slave_startup: cannot return");
658 /*NOTREACHED*/
659 }
660
661 extern struct cpu *cpu[NCPU]; /* pointers to all CPUs */
662
663 /*
664 * cpu_bringup_set is a tunable (via /etc/system, debugger, etc.) that
665 * can be used during debugging to control which processors are brought
666 * online at boot time. The variable represents a bitmap of the id's
667 * of the processors that will be brought online. The initialization
668 * of this variable depends on the type of cpuset_t, which varies
669 * depending on the number of processors supported (see cpuvar.h).
670 */
671 cpuset_t cpu_bringup_set;
672
673
674 /*
675 * Generic start-all cpus entry. Typically used during cold initialization.
676 * Note that cold start cpus are initialized into the online state.
677 */
678 /*ARGSUSED*/
679 void
start_other_cpus(int flag)680 start_other_cpus(int flag)
681 {
682 int cpuid;
683 extern void idlestop_init(void);
684 int bootcpu;
685
686 /*
687 * Check if cpu_bringup_set has been explicitly set before
688 * initializing it.
689 */
690 if (CPUSET_ISNULL(cpu_bringup_set)) {
691 CPUSET_ALL(cpu_bringup_set);
692 }
693
694 if (&cpu_feature_init)
695 cpu_feature_init();
696
697 /*
698 * Initialize CPC.
699 */
700 kcpc_hw_init();
701
702 mutex_enter(&cpu_lock);
703
704 /*
705 * Initialize our own cpu_info.
706 */
707 init_cpu_info(CPU);
708
709 /*
710 * Initialize CPU 0 cpu module private data area, including scrubber.
711 */
712 cpu_init_private(CPU);
713 populate_idstr(CPU);
714
715 /*
716 * perform such initialization as is needed
717 * to be able to take CPUs on- and off-line.
718 */
719 cpu_pause_init();
720 xc_init(); /* initialize processor crosscalls */
721 idlestop_init();
722
723 if (!use_mp) {
724 mutex_exit(&cpu_lock);
725 cmn_err(CE_CONT, "?***** Not in MP mode\n");
726 return;
727 }
728 /*
729 * should we be initializing this cpu?
730 */
731 bootcpu = getprocessorid();
732
733 /*
734 * launch all the slave cpus now
735 */
736 for (cpuid = 0; cpuid < NCPU; cpuid++) {
737 pnode_t nodeid = cpunodes[cpuid].nodeid;
738
739 if (nodeid == (pnode_t)0)
740 continue;
741
742 if (cpuid == bootcpu) {
743 if (!CPU_IN_SET(cpu_bringup_set, cpuid)) {
744 cmn_err(CE_WARN, "boot cpu not a member "
745 "of cpu_bringup_set, adding it");
746 CPUSET_ADD(cpu_bringup_set, cpuid);
747 }
748 continue;
749 }
750 if (!CPU_IN_SET(cpu_bringup_set, cpuid))
751 continue;
752
753 ASSERT(cpu[cpuid] == NULL);
754
755 if (setup_cpu_common(cpuid)) {
756 cmn_err(CE_PANIC, "cpu%d: setup failed", cpuid);
757 }
758
759 common_startup_init(cpu[cpuid], cpuid);
760
761 start_cpu(cpuid, cold_flag_set);
762 /*
763 * Because slave_startup() gets fired off after init()
764 * starts, we can't use the '?' trick to do 'boot -v'
765 * printing - so we always direct the 'cpu .. online'
766 * messages to the log.
767 */
768 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n",
769 cpuid);
770
771 cpu_state_change_notify(cpuid, CPU_SETUP);
772
773 if (dtrace_cpu_init != NULL)
774 (*dtrace_cpu_init)(cpuid);
775 }
776
777 /*
778 * since all the cpus are online now, redistribute interrupts to them.
779 */
780 intr_redist_all_cpus();
781
782 mutex_exit(&cpu_lock);
783
784 /*
785 * Start the Ecache scrubber. Must be done after all calls to
786 * cpu_init_private for every cpu (including CPU 0).
787 */
788 cpu_init_cache_scrub();
789
790 if (&cpu_mp_init)
791 cpu_mp_init();
792 }
793