1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/sysmacros.h> 28 #include <sys/prom_plat.h> 29 #include <sys/prom_debug.h> 30 #include <vm/hat_sfmmu.h> 31 #include <vm/seg_kp.h> 32 #include <vm/seg_kmem.h> 33 #include <sys/machsystm.h> 34 #include <sys/callb.h> 35 #include <sys/cpu_module.h> 36 #include <sys/pg.h> 37 #include <sys/cmt.h> 38 #include <sys/dtrace.h> 39 #include <sys/reboot.h> 40 #include <sys/kdi.h> 41 #include <sys/traptrace.h> 42 #ifdef TRAPTRACE 43 #include <sys/bootconf.h> 44 #endif /* TRAPTRACE */ 45 #include <sys/cpu_sgnblk_defs.h> 46 47 extern int cpu_intrq_setup(struct cpu *); 48 extern void cpu_intrq_cleanup(struct cpu *); 49 extern void cpu_intrq_register(struct cpu *); 50 51 struct cpu *cpus; /* pointer to other cpus; dynamically allocate */ 52 struct cpu *cpu[NCPU]; /* pointers to all CPUs */ 53 uint64_t cpu_pa[NCPU]; /* pointers to all CPUs in PA */ 54 cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 55 56 #ifdef TRAPTRACE 57 caddr_t ttrace_buf; /* kmem64 traptrace for all cpus except 0 */ 58 #endif /* TRAPTRACE */ 59 60 /* bit mask of cpus ready for x-calls, protected by cpu_lock */ 61 cpuset_t cpu_ready_set; 62 63 /* bit mask used to communicate with cpus during bringup */ 64 static cpuset_t proxy_ready_set; 65 66 static void slave_startup(void); 67 68 /* 69 * Defined in $KARCH/os/mach_mp_startup.c 70 */ 71 #pragma weak init_cpu_info 72 73 /* 74 * Amount of time (in milliseconds) we should wait before giving up on CPU 75 * initialization and assuming that the CPU we're trying to wake up is dead 76 * or out of control. 77 */ 78 #define CPU_WAKEUP_GRACE_MSEC 1000 79 80 #ifdef TRAPTRACE 81 /* 82 * This function sets traptrace buffers for all cpus 83 * other than boot cpu. 84 */ 85 size_t 86 calc_traptrace_sz(void) 87 { 88 return (TRAP_TSIZE * (max_ncpus - 1)); 89 } 90 #endif /* TRAPTRACE */ 91 92 93 /* 94 * common slave cpu initialization code 95 */ 96 void 97 common_startup_init(cpu_t *cp, int cpuid) 98 { 99 kthread_id_t tp; 100 sfmmu_t *sfmmup; 101 caddr_t sp; 102 103 /* 104 * Allocate and initialize the startup thread for this CPU. 105 */ 106 tp = thread_create(NULL, 0, slave_startup, NULL, 0, &p0, 107 TS_STOPPED, maxclsyspri); 108 109 /* 110 * Set state to TS_ONPROC since this thread will start running 111 * as soon as the CPU comes online. 112 * 113 * All the other fields of the thread structure are setup by 114 * thread_create(). 115 */ 116 THREAD_ONPROC(tp, cp); 117 tp->t_preempt = 1; 118 tp->t_bound_cpu = cp; 119 tp->t_affinitycnt = 1; 120 tp->t_cpu = cp; 121 tp->t_disp_queue = cp->cpu_disp; 122 123 sfmmup = astosfmmu(&kas); 124 CPUSET_ADD(sfmmup->sfmmu_cpusran, cpuid); 125 126 /* 127 * Setup thread to start in slave_startup. 128 */ 129 sp = tp->t_stk; 130 tp->t_pc = (uintptr_t)slave_startup - 8; 131 tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS; 132 133 cp->cpu_id = cpuid; 134 cp->cpu_self = cp; 135 cp->cpu_thread = tp; 136 cp->cpu_lwp = NULL; 137 cp->cpu_dispthread = tp; 138 cp->cpu_dispatch_pri = DISP_PRIO(tp); 139 cp->cpu_startup_thread = tp; 140 141 /* 142 * The dispatcher may discover the CPU before it is in cpu_ready_set 143 * and attempt to poke it. Before the CPU is in cpu_ready_set, any 144 * cross calls to it will be dropped. We initialize 145 * poke_cpu_outstanding to true so that poke_cpu will ignore any poke 146 * requests for this CPU. Pokes that come in before the CPU is in 147 * cpu_ready_set can be ignored because the CPU is about to come 148 * online. 149 */ 150 cp->cpu_m.poke_cpu_outstanding = B_TRUE; 151 } 152 153 /* 154 * parametric flag setting functions. these routines set the cpu 155 * state just prior to releasing the slave cpu. 156 */ 157 void 158 cold_flag_set(int cpuid) 159 { 160 cpu_t *cp; 161 162 ASSERT(MUTEX_HELD(&cpu_lock)); 163 164 cp = cpu[cpuid]; 165 cp->cpu_flags |= CPU_RUNNING | CPU_ENABLE | CPU_EXISTS; 166 cpu_add_active(cp); 167 /* 168 * Add CPU_READY after the cpu_add_active() call 169 * to avoid pausing cp. 170 */ 171 cp->cpu_flags |= CPU_READY; /* ready */ 172 cpu_set_state(cp); 173 } 174 175 static void 176 warm_flag_set(int cpuid) 177 { 178 cpu_t *cp; 179 180 ASSERT(MUTEX_HELD(&cpu_lock)); 181 182 /* 183 * warm start activates cpus into the OFFLINE state 184 */ 185 cp = cpu[cpuid]; 186 cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS 187 | CPU_OFFLINE | CPU_QUIESCED; 188 cpu_set_state(cp); 189 } 190 191 /* 192 * Internal cpu startup sequencer 193 * The sequence is as follows: 194 * 195 * MASTER SLAVE 196 * ------- ---------- 197 * assume the kernel data is initialized 198 * clear the proxy bit 199 * start the slave cpu 200 * wait for the slave cpu to set the proxy 201 * 202 * the slave runs slave_startup and then sets the proxy 203 * the slave waits for the master to add slave to the ready set 204 * 205 * the master finishes the initialization and 206 * adds the slave to the ready set 207 * 208 * the slave exits the startup thread and is running 209 */ 210 void 211 start_cpu(int cpuid, void(*flag_func)(int)) 212 { 213 extern void cpu_startup(int); 214 int timout; 215 216 ASSERT(MUTEX_HELD(&cpu_lock)); 217 218 /* 219 * Before we begin the dance, tell DTrace that we're about to start 220 * a CPU. 221 */ 222 if (dtrace_cpustart_init != NULL) 223 (*dtrace_cpustart_init)(); 224 225 /* start the slave cpu */ 226 CPUSET_DEL(proxy_ready_set, cpuid); 227 if (prom_test("SUNW,start-cpu-by-cpuid") == 0) { 228 (void) prom_startcpu_bycpuid(cpuid, (caddr_t)&cpu_startup, 229 cpuid); 230 } else { 231 /* "by-cpuid" interface didn't exist. Do it the old way */ 232 pnode_t nodeid = cpunodes[cpuid].nodeid; 233 234 ASSERT(nodeid != (pnode_t)0); 235 (void) prom_startcpu(nodeid, (caddr_t)&cpu_startup, cpuid); 236 } 237 238 /* wait for the slave cpu to check in. */ 239 for (timout = CPU_WAKEUP_GRACE_MSEC; timout; timout--) { 240 if (CPU_IN_SET(proxy_ready_set, cpuid)) 241 break; 242 DELAY(1000); 243 } 244 if (timout == 0) { 245 panic("cpu%d failed to start (2)", cpuid); 246 } 247 248 /* 249 * The slave has started; we can tell DTrace that it's safe again. 250 */ 251 if (dtrace_cpustart_fini != NULL) 252 (*dtrace_cpustart_fini)(); 253 254 /* run the master side of stick synchronization for the slave cpu */ 255 sticksync_master(); 256 257 /* 258 * deal with the cpu flags in a phase-specific manner 259 * for various reasons, this needs to run after the slave 260 * is checked in but before the slave is released. 261 */ 262 (*flag_func)(cpuid); 263 264 /* release the slave */ 265 CPUSET_ADD(cpu_ready_set, cpuid); 266 } 267 268 #ifdef TRAPTRACE 269 int trap_tr0_inuse = 1; /* it is always used on the boot cpu */ 270 int trap_trace_inuse[NCPU]; 271 #endif /* TRAPTRACE */ 272 273 #define cpu_next_free cpu_prev 274 275 /* 276 * Routine to set up a CPU to prepare for starting it up. 277 */ 278 int 279 setup_cpu_common(int cpuid) 280 { 281 struct cpu *cp = NULL; 282 kthread_id_t tp; 283 #ifdef TRAPTRACE 284 int tt_index; 285 TRAP_TRACE_CTL *ctlp; 286 caddr_t newbuf; 287 #endif /* TRAPTRACE */ 288 289 extern void idle(); 290 int rval; 291 292 ASSERT(MUTEX_HELD(&cpu_lock)); 293 ASSERT(cpu[cpuid] == NULL); 294 295 ASSERT(ncpus <= max_ncpus); 296 297 #ifdef TRAPTRACE 298 /* 299 * allocate a traptrace buffer for this CPU. 300 */ 301 ctlp = &trap_trace_ctl[cpuid]; 302 if (!trap_tr0_inuse) { 303 trap_tr0_inuse = 1; 304 newbuf = trap_tr0; 305 tt_index = -1; 306 } else { 307 for (tt_index = 0; tt_index < (max_ncpus-1); tt_index++) 308 if (!trap_trace_inuse[tt_index]) 309 break; 310 ASSERT(tt_index < max_ncpus - 1); 311 trap_trace_inuse[tt_index] = 1; 312 newbuf = (caddr_t)(ttrace_buf + (tt_index * TRAP_TSIZE)); 313 } 314 ctlp->d.vaddr_base = newbuf; 315 ctlp->d.offset = ctlp->d.last_offset = 0; 316 ctlp->d.limit = trap_trace_bufsize; 317 ctlp->d.paddr_base = va_to_pa(newbuf); 318 ASSERT(ctlp->d.paddr_base != (uint64_t)-1); 319 #endif /* TRAPTRACE */ 320 /* 321 * initialize hv traptrace buffer for this CPU 322 */ 323 mach_htraptrace_setup(cpuid); 324 325 /* 326 * Obtain pointer to the appropriate cpu structure. 327 */ 328 if (cpu0.cpu_flags == 0) { 329 cp = &cpu0; 330 } else { 331 /* 332 * When dynamically allocating cpu structs, 333 * cpus is used as a pointer to a list of freed 334 * cpu structs. 335 */ 336 if (cpus) { 337 /* grab the first cpu struct on the free list */ 338 cp = cpus; 339 if (cp->cpu_next_free) 340 cpus = cp->cpu_next_free; 341 else 342 cpus = NULL; 343 } 344 } 345 346 if (cp == NULL) 347 cp = vmem_xalloc(static_alloc_arena, CPU_ALLOC_SIZE, 348 CPU_ALLOC_SIZE, 0, 0, NULL, NULL, VM_SLEEP); 349 350 bzero(cp, sizeof (*cp)); 351 352 cp->cpu_id = cpuid; 353 cp->cpu_self = cp; 354 355 /* 356 * Initialize ptl1_panic stack 357 */ 358 ptl1_init_cpu(cp); 359 360 /* 361 * Initialize the dispatcher for this CPU. 362 */ 363 disp_cpu_init(cp); 364 365 /* 366 * Bootstrap the CPU's PG data 367 */ 368 pg_cpu_bootstrap(cp); 369 370 cpu_vm_data_init(cp); 371 372 /* 373 * Now, initialize per-CPU idle thread for this CPU. 374 */ 375 tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_ONPROC, -1); 376 377 cp->cpu_idle_thread = tp; 378 379 tp->t_preempt = 1; 380 tp->t_bound_cpu = cp; 381 tp->t_affinitycnt = 1; 382 tp->t_cpu = cp; 383 tp->t_disp_queue = cp->cpu_disp; 384 385 /* 386 * Registering a thread in the callback table is usually 387 * done in the initialization code of the thread. In this 388 * case, we do it right after thread creation to avoid 389 * blocking idle thread while registering itself. It also 390 * avoids the possibility of reregistration in case a CPU 391 * restarts its idle thread. 392 */ 393 CALLB_CPR_INIT_SAFE(tp, "idle"); 394 395 init_cpu_info(cp); 396 397 /* 398 * Initialize the interrupt threads for this CPU 399 */ 400 cpu_intr_alloc(cp, NINTR_THREADS); 401 402 /* 403 * Add CPU to list of available CPUs. 404 * It'll be on the active list after it is started. 405 */ 406 cpu_add_unit(cp); 407 408 /* 409 * Allocate and init cpu module private data structures, 410 * including scrubber. 411 */ 412 cpu_init_private(cp); 413 populate_idstr(cp); 414 415 /* 416 * Initialize the CPUs physical ID cache, and processor groups 417 */ 418 pghw_physid_create(cp); 419 (void) pg_cpu_init(cp, B_FALSE); 420 421 if ((rval = cpu_intrq_setup(cp)) != 0) { 422 return (rval); 423 } 424 425 /* 426 * Initialize MMU context domain information. 427 */ 428 sfmmu_cpu_init(cp); 429 430 return (0); 431 } 432 433 /* 434 * Routine to clean up a CPU after shutting it down. 435 */ 436 int 437 cleanup_cpu_common(int cpuid) 438 { 439 struct cpu *cp; 440 #ifdef TRAPTRACE 441 int i; 442 TRAP_TRACE_CTL *ctlp; 443 caddr_t newbuf; 444 #endif /* TRAPTRACE */ 445 446 ASSERT(MUTEX_HELD(&cpu_lock)); 447 ASSERT(cpu[cpuid] != NULL); 448 449 cp = cpu[cpuid]; 450 451 /* Free cpu module private data structures, including scrubber. */ 452 cpu_uninit_private(cp); 453 454 /* Free cpu ID string and brand string. */ 455 if (cp->cpu_idstr) 456 kmem_free(cp->cpu_idstr, strlen(cp->cpu_idstr) + 1); 457 if (cp->cpu_brandstr) 458 kmem_free(cp->cpu_brandstr, strlen(cp->cpu_brandstr) + 1); 459 460 cpu_vm_data_destroy(cp); 461 462 /* 463 * Remove CPU from list of available CPUs. 464 */ 465 cpu_del_unit(cpuid); 466 467 /* 468 * Clean any machine specific interrupt states. 469 */ 470 cpu_intrq_cleanup(cp); 471 472 /* 473 * At this point, the only threads bound to this CPU should be 474 * special per-cpu threads: it's idle thread, it's pause thread, 475 * and it's interrupt threads. Clean these up. 476 */ 477 cpu_destroy_bound_threads(cp); 478 479 /* 480 * Free the interrupt stack. 481 */ 482 segkp_release(segkp, cp->cpu_intr_stack); 483 484 /* 485 * Free hv traptrace buffer for this CPU. 486 */ 487 mach_htraptrace_cleanup(cpuid); 488 #ifdef TRAPTRACE 489 /* 490 * Free the traptrace buffer for this CPU. 491 */ 492 ctlp = &trap_trace_ctl[cpuid]; 493 newbuf = ctlp->d.vaddr_base; 494 i = (newbuf - ttrace_buf) / (TRAP_TSIZE); 495 if (((newbuf - ttrace_buf) % (TRAP_TSIZE) == 0) && 496 ((i >= 0) && (i < (max_ncpus-1)))) { 497 /* 498 * This CPU got it's trap trace buffer from the 499 * boot-alloc'd bunch of them. 500 */ 501 trap_trace_inuse[i] = 0; 502 bzero(newbuf, (TRAP_TSIZE)); 503 } else if (newbuf == trap_tr0) { 504 trap_tr0_inuse = 0; 505 bzero(trap_tr0, (TRAP_TSIZE)); 506 } else { 507 cmn_err(CE_WARN, "failed to free trap trace buffer from cpu%d", 508 cpuid); 509 } 510 bzero(ctlp, sizeof (*ctlp)); 511 #endif /* TRAPTRACE */ 512 513 /* 514 * There is a race condition with mutex_vector_enter() which 515 * caches a cpu pointer. The race is detected by checking cpu_next. 516 */ 517 disp_cpu_fini(cp); 518 cpu_pa[cpuid] = 0; 519 if (CPU_MMU_CTXP(cp)) 520 sfmmu_cpu_cleanup(cp); 521 bzero(cp, sizeof (*cp)); 522 523 /* 524 * Place the freed cpu structure on the list of freed cpus. 525 */ 526 if (cp != &cpu0) { 527 if (cpus) { 528 cp->cpu_next_free = cpus; 529 cpus = cp; 530 } 531 else 532 cpus = cp; 533 } 534 535 return (0); 536 } 537 538 /* 539 * This routine is used to start a previously powered off processor. 540 * Note that restarted cpus are initialized into the offline state. 541 */ 542 void 543 restart_other_cpu(int cpuid) 544 { 545 struct cpu *cp; 546 kthread_id_t tp; 547 caddr_t sp; 548 extern void idle(); 549 550 ASSERT(MUTEX_HELD(&cpu_lock)); 551 ASSERT(cpuid < NCPU && cpu[cpuid] != NULL); 552 553 /* 554 * Obtain pointer to the appropriate cpu structure. 555 */ 556 cp = cpu[cpuid]; 557 558 common_startup_init(cp, cpuid); 559 560 /* 561 * idle thread t_lock is held when the idle thread is suspended. 562 * Manually unlock the t_lock of idle loop so that we can resume 563 * the suspended idle thread. 564 * Also adjust the PC of idle thread for re-retry. 565 */ 566 cp->cpu_intr_actv = 0; /* clear the value from previous life */ 567 cp->cpu_m.mutex_ready = 0; /* we are not ready yet */ 568 lock_clear(&cp->cpu_idle_thread->t_lock); 569 tp = cp->cpu_idle_thread; 570 571 sp = tp->t_stk; 572 tp->t_sp = (uintptr_t)((struct rwindow *)sp - 1) - STACK_BIAS; 573 tp->t_pc = (uintptr_t)idle - 8; 574 575 /* 576 * restart the cpu now 577 */ 578 promsafe_pause_cpus(); 579 start_cpu(cpuid, warm_flag_set); 580 start_cpus(); 581 582 /* call cmn_err outside pause_cpus/start_cpus to avoid deadlock */ 583 cmn_err(CE_CONT, "!cpu%d initialization complete - restarted\n", 584 cpuid); 585 } 586 587 /* 588 * Startup function executed on 'other' CPUs. This is the first 589 * C function after cpu_start sets up the cpu registers. 590 */ 591 static void 592 slave_startup(void) 593 { 594 struct cpu *cp = CPU; 595 ushort_t original_flags = cp->cpu_flags; 596 597 mach_htraptrace_configure(cp->cpu_id); 598 cpu_intrq_register(CPU); 599 cp->cpu_m.mutex_ready = 1; 600 601 /* acknowledge that we are done with initialization */ 602 CPUSET_ADD(proxy_ready_set, cp->cpu_id); 603 604 /* synchronize STICK */ 605 sticksync_slave(); 606 607 if (boothowto & RB_DEBUG) 608 kdi_dvec_cpu_init(cp); 609 610 /* 611 * the slave will wait here forever -- assuming that the master 612 * will get back to us. if it doesn't we've got bigger problems 613 * than a master not replying to this slave. 614 * the small delay improves the slave's responsiveness to the 615 * master's ack and decreases the time window between master and 616 * slave operations. 617 */ 618 while (!CPU_IN_SET(cpu_ready_set, cp->cpu_id)) 619 DELAY(1); 620 621 /* 622 * The CPU is now in cpu_ready_set, safely able to take pokes. 623 */ 624 cp->cpu_m.poke_cpu_outstanding = B_FALSE; 625 626 /* enable interrupts */ 627 (void) spl0(); 628 629 /* 630 * Signature block update to indicate that this CPU is in OS now. 631 * This needs to be done after the PIL is lowered since on 632 * some platforms the update code may block. 633 */ 634 CPU_SIGNATURE(OS_SIG, SIGST_RUN, SIGSUBST_NULL, cp->cpu_id); 635 636 /* 637 * park the slave thread in a safe/quiet state and wait for the master 638 * to finish configuring this CPU before proceeding to thread_exit(). 639 */ 640 while (((volatile ushort_t)cp->cpu_flags) & CPU_QUIESCED) 641 DELAY(1); 642 643 /* 644 * Initialize CPC CPU state. 645 */ 646 kcpc_hw_startup_cpu(original_flags); 647 648 /* 649 * Notify the PG subsystem that the CPU has started 650 */ 651 pg_cmt_cpu_startup(CPU); 652 653 /* 654 * Now we are done with the startup thread, so free it up. 655 */ 656 thread_exit(); 657 cmn_err(CE_PANIC, "slave_startup: cannot return"); 658 /*NOTREACHED*/ 659 } 660 661 extern struct cpu *cpu[NCPU]; /* pointers to all CPUs */ 662 663 /* 664 * cpu_bringup_set is a tunable (via /etc/system, debugger, etc.) that 665 * can be used during debugging to control which processors are brought 666 * online at boot time. The variable represents a bitmap of the id's 667 * of the processors that will be brought online. The initialization 668 * of this variable depends on the type of cpuset_t, which varies 669 * depending on the number of processors supported (see cpuvar.h). 670 */ 671 cpuset_t cpu_bringup_set; 672 673 674 /* 675 * Generic start-all cpus entry. Typically used during cold initialization. 676 * Note that cold start cpus are initialized into the online state. 677 */ 678 /*ARGSUSED*/ 679 void 680 start_other_cpus(int flag) 681 { 682 int cpuid; 683 extern void idlestop_init(void); 684 int bootcpu; 685 686 /* 687 * Check if cpu_bringup_set has been explicitly set before 688 * initializing it. 689 */ 690 if (CPUSET_ISNULL(cpu_bringup_set)) { 691 CPUSET_ALL(cpu_bringup_set); 692 } 693 694 if (&cpu_feature_init) 695 cpu_feature_init(); 696 697 /* 698 * Initialize CPC. 699 */ 700 kcpc_hw_init(); 701 702 mutex_enter(&cpu_lock); 703 704 /* 705 * Initialize our own cpu_info. 706 */ 707 init_cpu_info(CPU); 708 709 /* 710 * Initialize CPU 0 cpu module private data area, including scrubber. 711 */ 712 cpu_init_private(CPU); 713 populate_idstr(CPU); 714 715 /* 716 * perform such initialization as is needed 717 * to be able to take CPUs on- and off-line. 718 */ 719 cpu_pause_init(); 720 xc_init(); /* initialize processor crosscalls */ 721 idlestop_init(); 722 723 if (!use_mp) { 724 mutex_exit(&cpu_lock); 725 cmn_err(CE_CONT, "?***** Not in MP mode\n"); 726 return; 727 } 728 /* 729 * should we be initializing this cpu? 730 */ 731 bootcpu = getprocessorid(); 732 733 /* 734 * launch all the slave cpus now 735 */ 736 for (cpuid = 0; cpuid < NCPU; cpuid++) { 737 pnode_t nodeid = cpunodes[cpuid].nodeid; 738 739 if (nodeid == (pnode_t)0) 740 continue; 741 742 if (cpuid == bootcpu) { 743 if (!CPU_IN_SET(cpu_bringup_set, cpuid)) { 744 cmn_err(CE_WARN, "boot cpu not a member " 745 "of cpu_bringup_set, adding it"); 746 CPUSET_ADD(cpu_bringup_set, cpuid); 747 } 748 continue; 749 } 750 if (!CPU_IN_SET(cpu_bringup_set, cpuid)) 751 continue; 752 753 ASSERT(cpu[cpuid] == NULL); 754 755 if (setup_cpu_common(cpuid)) { 756 cmn_err(CE_PANIC, "cpu%d: setup failed", cpuid); 757 } 758 759 common_startup_init(cpu[cpuid], cpuid); 760 761 start_cpu(cpuid, cold_flag_set); 762 /* 763 * Because slave_startup() gets fired off after init() 764 * starts, we can't use the '?' trick to do 'boot -v' 765 * printing - so we always direct the 'cpu .. online' 766 * messages to the log. 767 */ 768 cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 769 cpuid); 770 771 cpu_state_change_notify(cpuid, CPU_SETUP); 772 773 if (dtrace_cpu_init != NULL) 774 (*dtrace_cpu_init)(cpuid); 775 } 776 777 /* 778 * since all the cpus are online now, redistribute interrupts to them. 779 */ 780 intr_redist_all_cpus(); 781 782 mutex_exit(&cpu_lock); 783 784 /* 785 * Start the Ecache scrubber. Must be done after all calls to 786 * cpu_init_private for every cpu (including CPU 0). 787 */ 788 cpu_init_cache_scrub(); 789 790 if (&cpu_mp_init) 791 cpu_mp_init(); 792 } 793