xref: /titanic_44/usr/src/uts/i86pc/os/mp_startup.c (revision c3d7cb8d25139b078de5a359a2967be1b7166ccc)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/thread.h>
29 #include <sys/cpuvar.h>
30 #include <sys/t_lock.h>
31 #include <sys/param.h>
32 #include <sys/proc.h>
33 #include <sys/disp.h>
34 #include <sys/class.h>
35 #include <sys/cmn_err.h>
36 #include <sys/debug.h>
37 #include <sys/asm_linkage.h>
38 #include <sys/x_call.h>
39 #include <sys/systm.h>
40 #include <sys/var.h>
41 #include <sys/vtrace.h>
42 #include <vm/hat.h>
43 #include <vm/as.h>
44 #include <vm/seg_kmem.h>
45 #include <vm/seg_kp.h>
46 #include <sys/segments.h>
47 #include <sys/kmem.h>
48 #include <sys/stack.h>
49 #include <sys/smp_impldefs.h>
50 #include <sys/x86_archext.h>
51 #include <sys/machsystm.h>
52 #include <sys/traptrace.h>
53 #include <sys/clock.h>
54 #include <sys/cpc_impl.h>
55 #include <sys/pg.h>
56 #include <sys/cmt.h>
57 #include <sys/dtrace.h>
58 #include <sys/archsystm.h>
59 #include <sys/fp.h>
60 #include <sys/reboot.h>
61 #include <sys/kdi_machimpl.h>
62 #include <vm/hat_i86.h>
63 #include <sys/memnode.h>
64 #include <sys/pci_cfgspace.h>
65 #include <sys/mach_mmu.h>
66 #include <sys/sysmacros.h>
67 #if defined(__xpv)
68 #include <sys/hypervisor.h>
69 #endif
70 #include <sys/cpu_module.h>
71 
72 struct cpu	cpus[1];			/* CPU data */
73 struct cpu	*cpu[NCPU] = {&cpus[0]};	/* pointers to all CPUs */
74 cpu_core_t	cpu_core[NCPU];			/* cpu_core structures */
75 
76 /*
77  * Useful for disabling MP bring-up on a MP capable system.
78  */
79 int use_mp = 1;
80 
81 /*
82  * to be set by a PSM to indicate what cpus
83  * are sitting around on the system.
84  */
85 cpuset_t mp_cpus;
86 
87 /*
88  * This variable is used by the hat layer to decide whether or not
89  * critical sections are needed to prevent race conditions.  For sun4m,
90  * this variable is set once enough MP initialization has been done in
91  * order to allow cross calls.
92  */
93 int flushes_require_xcalls;
94 
95 cpuset_t cpu_ready_set;		/* initialized in startup() */
96 
97 static 	void	mp_startup(void);
98 
99 static void cpu_sep_enable(void);
100 static void cpu_sep_disable(void);
101 static void cpu_asysc_enable(void);
102 static void cpu_asysc_disable(void);
103 
104 /*
105  * Init CPU info - get CPU type info for processor_info system call.
106  */
107 void
108 init_cpu_info(struct cpu *cp)
109 {
110 	processor_info_t *pi = &cp->cpu_type_info;
111 	char buf[CPU_IDSTRLEN];
112 
113 	/*
114 	 * Get clock-frequency property for the CPU.
115 	 */
116 	pi->pi_clock = cpu_freq;
117 
118 	/*
119 	 * Current frequency in Hz.
120 	 */
121 	cp->cpu_curr_clock = cpu_freq_hz;
122 
123 	(void) strcpy(pi->pi_processor_type, "i386");
124 	if (fpu_exists)
125 		(void) strcpy(pi->pi_fputypes, "i387 compatible");
126 
127 	(void) cpuid_getidstr(cp, buf, sizeof (buf));
128 
129 	cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP);
130 	(void) strcpy(cp->cpu_idstr, buf);
131 
132 	(void) cpuid_getbrandstr(cp, buf, sizeof (buf));
133 	cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP);
134 	(void) strcpy(cp->cpu_brandstr, buf);
135 }
136 
137 /*
138  * Configure syscall support on this CPU.
139  */
140 /*ARGSUSED*/
141 void
142 init_cpu_syscall(struct cpu *cp)
143 {
144 	kpreempt_disable();
145 
146 #if defined(__amd64)
147 	if ((x86_feature & (X86_MSR | X86_ASYSC)) == (X86_MSR | X86_ASYSC)) {
148 
149 #if !defined(__lint)
150 		/*
151 		 * The syscall instruction imposes a certain ordering on
152 		 * segment selectors, so we double-check that ordering
153 		 * here.
154 		 */
155 		ASSERT(KDS_SEL == KCS_SEL + 8);
156 		ASSERT(UDS_SEL == U32CS_SEL + 8);
157 		ASSERT(UCS_SEL == U32CS_SEL + 16);
158 #endif
159 		/*
160 		 * Turn syscall/sysret extensions on.
161 		 */
162 		cpu_asysc_enable();
163 
164 		/*
165 		 * Program the magic registers ..
166 		 */
167 		wrmsr(MSR_AMD_STAR,
168 		    ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
169 		wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall);
170 		wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32);
171 
172 		/*
173 		 * This list of flags is masked off the incoming
174 		 * %rfl when we enter the kernel.
175 		 */
176 		wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T));
177 	}
178 #endif
179 
180 	/*
181 	 * On 32-bit kernels, we use sysenter/sysexit because it's too
182 	 * hard to use syscall/sysret, and it is more portable anyway.
183 	 *
184 	 * On 64-bit kernels on Nocona machines, the 32-bit syscall
185 	 * variant isn't available to 32-bit applications, but sysenter is.
186 	 */
187 	if ((x86_feature & (X86_MSR | X86_SEP)) == (X86_MSR | X86_SEP)) {
188 
189 #if !defined(__lint)
190 		/*
191 		 * The sysenter instruction imposes a certain ordering on
192 		 * segment selectors, so we double-check that ordering
193 		 * here. See "sysenter" in Intel document 245471-012, "IA-32
194 		 * Intel Architecture Software Developer's Manual Volume 2:
195 		 * Instruction Set Reference"
196 		 */
197 		ASSERT(KDS_SEL == KCS_SEL + 8);
198 
199 		ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3));
200 		ASSERT32(UDS_SEL == UCS_SEL + 8);
201 
202 		ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3));
203 		ASSERT64(UDS_SEL == U32CS_SEL + 8);
204 #endif
205 
206 		cpu_sep_enable();
207 
208 		/*
209 		 * resume() sets this value to the base of the threads stack
210 		 * via a context handler.
211 		 */
212 		wrmsr(MSR_INTC_SEP_ESP, 0);
213 		wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter);
214 	}
215 
216 	kpreempt_enable();
217 }
218 
219 /*
220  * Multiprocessor initialization.
221  *
222  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
223  * startup and idle threads for the specified CPU.
224  */
225 struct cpu *
226 mp_startup_init(int cpun)
227 {
228 	struct cpu *cp;
229 	kthread_id_t tp;
230 	caddr_t	sp;
231 	proc_t *procp;
232 #if !defined(__xpv)
233 	extern int idle_cpu_prefer_mwait;
234 	extern void cpu_idle_mwait();
235 #endif
236 	extern void idle();
237 	extern void cpu_idle();
238 
239 #ifdef TRAPTRACE
240 	trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
241 #endif
242 
243 	ASSERT(cpun < NCPU && cpu[cpun] == NULL);
244 
245 	cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);
246 #if !defined(__xpv)
247 	if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) {
248 		cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);
249 		cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
250 	} else
251 #endif
252 		cp->cpu_m.mcpu_idle_cpu = cpu_idle;
253 
254 	procp = curthread->t_procp;
255 
256 	mutex_enter(&cpu_lock);
257 	/*
258 	 * Initialize the dispatcher first.
259 	 */
260 	disp_cpu_init(cp);
261 	mutex_exit(&cpu_lock);
262 
263 	cpu_vm_data_init(cp);
264 
265 	/*
266 	 * Allocate and initialize the startup thread for this CPU.
267 	 * Interrupt and process switch stacks get allocated later
268 	 * when the CPU starts running.
269 	 */
270 	tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
271 	    TS_STOPPED, maxclsyspri);
272 
273 	/*
274 	 * Set state to TS_ONPROC since this thread will start running
275 	 * as soon as the CPU comes online.
276 	 *
277 	 * All the other fields of the thread structure are setup by
278 	 * thread_create().
279 	 */
280 	THREAD_ONPROC(tp, cp);
281 	tp->t_preempt = 1;
282 	tp->t_bound_cpu = cp;
283 	tp->t_affinitycnt = 1;
284 	tp->t_cpu = cp;
285 	tp->t_disp_queue = cp->cpu_disp;
286 
287 	/*
288 	 * Setup thread to start in mp_startup.
289 	 */
290 	sp = tp->t_stk;
291 	tp->t_pc = (uintptr_t)mp_startup;
292 	tp->t_sp = (uintptr_t)(sp - MINFRAME);
293 #if defined(__amd64)
294 	tp->t_sp -= STACK_ENTRY_ALIGN;		/* fake a call */
295 #endif
296 
297 	cp->cpu_id = cpun;
298 	cp->cpu_self = cp;
299 	cp->cpu_thread = tp;
300 	cp->cpu_lwp = NULL;
301 	cp->cpu_dispthread = tp;
302 	cp->cpu_dispatch_pri = DISP_PRIO(tp);
303 
304 	/*
305 	 * cpu_base_spl must be set explicitly here to prevent any blocking
306 	 * operations in mp_startup from causing the spl of the cpu to drop
307 	 * to 0 (allowing device interrupts before we're ready) in resume().
308 	 * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.
309 	 * As an extra bit of security on DEBUG kernels, this is enforced with
310 	 * an assertion in mp_startup() -- before cpu_base_spl is set to its
311 	 * proper value.
312 	 */
313 	cp->cpu_base_spl = ipltospl(LOCK_LEVEL);
314 
315 	/*
316 	 * Now, initialize per-CPU idle thread for this CPU.
317 	 */
318 	tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
319 
320 	cp->cpu_idle_thread = tp;
321 
322 	tp->t_preempt = 1;
323 	tp->t_bound_cpu = cp;
324 	tp->t_affinitycnt = 1;
325 	tp->t_cpu = cp;
326 	tp->t_disp_queue = cp->cpu_disp;
327 
328 	/*
329 	 * Bootstrap the CPU's PG data
330 	 */
331 	pg_cpu_bootstrap(cp);
332 
333 	/*
334 	 * Perform CPC initialization on the new CPU.
335 	 */
336 	kcpc_hw_init(cp);
337 
338 	/*
339 	 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
340 	 * for each CPU.
341 	 */
342 	setup_vaddr_for_ppcopy(cp);
343 
344 	/*
345 	 * Allocate page for new GDT and initialize from current GDT.
346 	 */
347 #if !defined(__lint)
348 	ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);
349 #endif
350 	cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);
351 	bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));
352 
353 #if defined(__i386)
354 	/*
355 	 * setup kernel %gs.
356 	 */
357 	set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,
358 	    SEL_KPL, 0, 1);
359 #endif
360 
361 	/*
362 	 * If we have more than one node, each cpu gets a copy of IDT
363 	 * local to its node. If this is a Pentium box, we use cpu 0's
364 	 * IDT. cpu 0's IDT has been made read-only to workaround the
365 	 * cmpxchgl register bug
366 	 */
367 	if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) {
368 #if !defined(__lint)
369 		ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);
370 #endif
371 		cp->cpu_idt = kmem_zalloc(PAGESIZE, KM_SLEEP);
372 		bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);
373 	} else {
374 		cp->cpu_idt = CPU->cpu_idt;
375 	}
376 
377 	/*
378 	 * Get interrupt priority data from cpu 0.
379 	 */
380 	cp->cpu_pri_data = CPU->cpu_pri_data;
381 
382 	/*
383 	 * alloc space for cpuid info
384 	 */
385 	cpuid_alloc_space(cp);
386 
387 	/*
388 	 * alloc space for ucode_info
389 	 */
390 	ucode_alloc_space(cp);
391 
392 	hat_cpu_online(cp);
393 
394 #ifdef TRAPTRACE
395 	/*
396 	 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers
397 	 */
398 	ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
399 	ttc->ttc_next = ttc->ttc_first;
400 	ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
401 #endif
402 	/*
403 	 * Record that we have another CPU.
404 	 */
405 	mutex_enter(&cpu_lock);
406 	/*
407 	 * Initialize the interrupt threads for this CPU
408 	 */
409 	cpu_intr_alloc(cp, NINTR_THREADS);
410 	/*
411 	 * Add CPU to list of available CPUs.  It'll be on the active list
412 	 * after mp_startup().
413 	 */
414 	cpu_add_unit(cp);
415 	mutex_exit(&cpu_lock);
416 
417 	return (cp);
418 }
419 
420 /*
421  * Undo what was done in mp_startup_init
422  */
423 static void
424 mp_startup_fini(struct cpu *cp, int error)
425 {
426 	mutex_enter(&cpu_lock);
427 
428 	/*
429 	 * Remove the CPU from the list of available CPUs.
430 	 */
431 	cpu_del_unit(cp->cpu_id);
432 
433 	if (error == ETIMEDOUT) {
434 		/*
435 		 * The cpu was started, but never *seemed* to run any
436 		 * code in the kernel; it's probably off spinning in its
437 		 * own private world, though with potential references to
438 		 * our kmem-allocated IDTs and GDTs (for example).
439 		 *
440 		 * Worse still, it may actually wake up some time later,
441 		 * so rather than guess what it might or might not do, we
442 		 * leave the fundamental data structures intact.
443 		 */
444 		cp->cpu_flags = 0;
445 		mutex_exit(&cpu_lock);
446 		return;
447 	}
448 
449 	/*
450 	 * At this point, the only threads bound to this CPU should
451 	 * special per-cpu threads: it's idle thread, it's pause threads,
452 	 * and it's interrupt threads.  Clean these up.
453 	 */
454 	cpu_destroy_bound_threads(cp);
455 	cp->cpu_idle_thread = NULL;
456 
457 	/*
458 	 * Free the interrupt stack.
459 	 */
460 	segkp_release(segkp,
461 	    cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));
462 
463 	mutex_exit(&cpu_lock);
464 
465 #ifdef TRAPTRACE
466 	/*
467 	 * Discard the trap trace buffer
468 	 */
469 	{
470 		trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];
471 
472 		kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);
473 		ttc->ttc_first = NULL;
474 	}
475 #endif
476 
477 	hat_cpu_offline(cp);
478 
479 	cpuid_free_space(cp);
480 
481 	ucode_free_space(cp);
482 
483 	if (cp->cpu_idt != CPU->cpu_idt)
484 		kmem_free(cp->cpu_idt, PAGESIZE);
485 	cp->cpu_idt = NULL;
486 
487 	kmem_free(cp->cpu_gdt, PAGESIZE);
488 	cp->cpu_gdt = NULL;
489 
490 	teardown_vaddr_for_ppcopy(cp);
491 
492 	kcpc_hw_fini(cp);
493 
494 	cp->cpu_dispthread = NULL;
495 	cp->cpu_thread = NULL;	/* discarded by cpu_destroy_bound_threads() */
496 
497 	cpu_vm_data_destroy(cp);
498 
499 	mutex_enter(&cpu_lock);
500 	disp_cpu_fini(cp);
501 	mutex_exit(&cpu_lock);
502 
503 #if !defined(__xpv)
504 	if (cp->cpu_m.mcpu_mwait != NULL)
505 		cpuid_mwait_free(cp);
506 #endif
507 	kmem_free(cp, sizeof (*cp));
508 }
509 
510 /*
511  * Apply workarounds for known errata, and warn about those that are absent.
512  *
513  * System vendors occasionally create configurations which contain different
514  * revisions of the CPUs that are almost but not exactly the same.  At the
515  * time of writing, this meant that their clock rates were the same, their
516  * feature sets were the same, but the required workaround were -not-
517  * necessarily the same.  So, this routine is invoked on -every- CPU soon
518  * after starting to make sure that the resulting system contains the most
519  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
520  * system.
521  *
522  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
523  * mp_startup() for all slave CPUs. Slaves process workaround_errata prior
524  * to acknowledging their readiness to the master, so this routine will
525  * never be executed by multiple CPUs in parallel, thus making updates to
526  * global data safe.
527  *
528  * These workarounds are based on Rev 3.57 of the Revision Guide for
529  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
530  */
531 
532 #if defined(OPTERON_ERRATUM_88)
533 int opteron_erratum_88;		/* if non-zero -> at least one cpu has it */
534 #endif
535 
536 #if defined(OPTERON_ERRATUM_91)
537 int opteron_erratum_91;		/* if non-zero -> at least one cpu has it */
538 #endif
539 
540 #if defined(OPTERON_ERRATUM_93)
541 int opteron_erratum_93;		/* if non-zero -> at least one cpu has it */
542 #endif
543 
544 #if defined(OPTERON_ERRATUM_95)
545 int opteron_erratum_95;		/* if non-zero -> at least one cpu has it */
546 #endif
547 
548 #if defined(OPTERON_ERRATUM_100)
549 int opteron_erratum_100;	/* if non-zero -> at least one cpu has it */
550 #endif
551 
552 #if defined(OPTERON_ERRATUM_108)
553 int opteron_erratum_108;	/* if non-zero -> at least one cpu has it */
554 #endif
555 
556 #if defined(OPTERON_ERRATUM_109)
557 int opteron_erratum_109;	/* if non-zero -> at least one cpu has it */
558 #endif
559 
560 #if defined(OPTERON_ERRATUM_121)
561 int opteron_erratum_121;	/* if non-zero -> at least one cpu has it */
562 #endif
563 
564 #if defined(OPTERON_ERRATUM_122)
565 int opteron_erratum_122;	/* if non-zero -> at least one cpu has it */
566 #endif
567 
568 #if defined(OPTERON_ERRATUM_123)
569 int opteron_erratum_123;	/* if non-zero -> at least one cpu has it */
570 #endif
571 
572 #if defined(OPTERON_ERRATUM_131)
573 int opteron_erratum_131;	/* if non-zero -> at least one cpu has it */
574 #endif
575 
576 #if defined(OPTERON_WORKAROUND_6336786)
577 int opteron_workaround_6336786;	/* non-zero -> WA relevant and applied */
578 int opteron_workaround_6336786_UP = 0;	/* Not needed for UP */
579 #endif
580 
581 #if defined(OPTERON_WORKAROUND_6323525)
582 int opteron_workaround_6323525;	/* if non-zero -> at least one cpu has it */
583 #endif
584 
585 #if defined(OPTERON_ERRATUM_298)
586 int opteron_erratum_298;
587 #endif
588 
589 static void
590 workaround_warning(cpu_t *cp, uint_t erratum)
591 {
592 	cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u",
593 	    cp->cpu_id, erratum);
594 }
595 
596 static void
597 workaround_applied(uint_t erratum)
598 {
599 	if (erratum > 1000000)
600 		cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n",
601 		    erratum);
602 	else
603 		cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n",
604 		    erratum);
605 }
606 
607 static void
608 msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error)
609 {
610 	cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d",
611 	    cp->cpu_id, rw, msr, error);
612 }
613 
614 /*
615  * Determine the number of nodes in a Hammer / Greyhound / Griffin family
616  * system.
617  */
618 static uint_t
619 opteron_get_nnodes(void)
620 {
621 	static uint_t nnodes = 0;
622 
623 	if (nnodes == 0) {
624 #ifdef	DEBUG
625 		uint_t family;
626 
627 		/*
628 		 * This routine uses a PCI config space based mechanism
629 		 * for retrieving the number of nodes in the system.
630 		 * Device 24, function 0, offset 0x60 as used here is not
631 		 * AMD processor architectural, and may not work on processor
632 		 * families other than those listed below.
633 		 *
634 		 * Callers of this routine must ensure that we're running on
635 		 * a processor which supports this mechanism.
636 		 * The assertion below is meant to catch calls on unsupported
637 		 * processors.
638 		 */
639 		family = cpuid_getfamily(CPU);
640 		ASSERT(family == 0xf || family == 0x10 || family == 0x11);
641 #endif	/* DEBUG */
642 
643 		/*
644 		 * Obtain the number of nodes in the system from
645 		 * bits [6:4] of the Node ID register on node 0.
646 		 *
647 		 * The actual node count is NodeID[6:4] + 1
648 		 *
649 		 * The Node ID register is accessed via function 0,
650 		 * offset 0x60. Node 0 is device 24.
651 		 */
652 		nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1;
653 	}
654 	return (nnodes);
655 }
656 
657 #if defined(__xpv)
658 
659 /*
660  * On dom0, we can determine the number of physical cpus on the machine.
661  * This number is important when figuring out what workarounds are
662  * appropriate, so compute it now.
663  */
664 uint_t
665 xen_get_nphyscpus(void)
666 {
667 	static uint_t nphyscpus = 0;
668 
669 	ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
670 
671 	if (nphyscpus == 0) {
672 		xen_sysctl_t op;
673 		xen_sysctl_physinfo_t *pi = &op.u.physinfo;
674 
675 		op.cmd = XEN_SYSCTL_physinfo;
676 		op.interface_version = XEN_SYSCTL_INTERFACE_VERSION;
677 		if (HYPERVISOR_sysctl(&op) == 0)
678 			nphyscpus = pi->threads_per_core *
679 			    pi->cores_per_socket * pi->sockets_per_node *
680 			    pi->nr_nodes;
681 	}
682 	return (nphyscpus);
683 }
684 #endif
685 
686 uint_t
687 do_erratum_298(struct cpu *cpu)
688 {
689 	static int	osvwrc = -3;
690 	extern int	osvw_opteron_erratum(cpu_t *, uint_t);
691 
692 	/*
693 	 * L2 Eviction May Occur During Processor Operation To Set
694 	 * Accessed or Dirty Bit.
695 	 */
696 	if (osvwrc == -3) {
697 		osvwrc = osvw_opteron_erratum(cpu, 298);
698 	} else {
699 		/* osvw return codes should be consistent for all cpus */
700 		ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298));
701 	}
702 
703 	switch (osvwrc) {
704 	case 0:		/* erratum is not present: do nothing */
705 		break;
706 	case 1:		/* erratum is present: BIOS workaround applied */
707 		/*
708 		 * check if workaround is actually in place and issue warning
709 		 * if not.
710 		 */
711 		if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
712 		    ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) {
713 #if defined(OPTERON_ERRATUM_298)
714 			opteron_erratum_298++;
715 #else
716 			workaround_warning(cpu, 298);
717 			return (1);
718 #endif
719 		}
720 		break;
721 	case -1:	/* cannot determine via osvw: check cpuid */
722 		if ((cpuid_opteron_erratum(cpu, 298) > 0) &&
723 		    (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
724 		    ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) {
725 #if defined(OPTERON_ERRATUM_298)
726 			opteron_erratum_298++;
727 #else
728 			workaround_warning(cpu, 298);
729 			return (1);
730 #endif
731 		}
732 		break;
733 	}
734 	return (0);
735 }
736 
737 uint_t
738 workaround_errata(struct cpu *cpu)
739 {
740 	uint_t missing = 0;
741 
742 	ASSERT(cpu == CPU);
743 
744 	/*LINTED*/
745 	if (cpuid_opteron_erratum(cpu, 88) > 0) {
746 		/*
747 		 * SWAPGS May Fail To Read Correct GS Base
748 		 */
749 #if defined(OPTERON_ERRATUM_88)
750 		/*
751 		 * The workaround is an mfence in the relevant assembler code
752 		 */
753 		opteron_erratum_88++;
754 #else
755 		workaround_warning(cpu, 88);
756 		missing++;
757 #endif
758 	}
759 
760 	if (cpuid_opteron_erratum(cpu, 91) > 0) {
761 		/*
762 		 * Software Prefetches May Report A Page Fault
763 		 */
764 #if defined(OPTERON_ERRATUM_91)
765 		/*
766 		 * fix is in trap.c
767 		 */
768 		opteron_erratum_91++;
769 #else
770 		workaround_warning(cpu, 91);
771 		missing++;
772 #endif
773 	}
774 
775 	if (cpuid_opteron_erratum(cpu, 93) > 0) {
776 		/*
777 		 * RSM Auto-Halt Restart Returns to Incorrect RIP
778 		 */
779 #if defined(OPTERON_ERRATUM_93)
780 		/*
781 		 * fix is in trap.c
782 		 */
783 		opteron_erratum_93++;
784 #else
785 		workaround_warning(cpu, 93);
786 		missing++;
787 #endif
788 	}
789 
790 	/*LINTED*/
791 	if (cpuid_opteron_erratum(cpu, 95) > 0) {
792 		/*
793 		 * RET Instruction May Return to Incorrect EIP
794 		 */
795 #if defined(OPTERON_ERRATUM_95)
796 #if defined(_LP64)
797 		/*
798 		 * Workaround this by ensuring that 32-bit user code and
799 		 * 64-bit kernel code never occupy the same address
800 		 * range mod 4G.
801 		 */
802 		if (_userlimit32 > 0xc0000000ul)
803 			*(uintptr_t *)&_userlimit32 = 0xc0000000ul;
804 
805 		/*LINTED*/
806 		ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
807 		opteron_erratum_95++;
808 #endif	/* _LP64 */
809 #else
810 		workaround_warning(cpu, 95);
811 		missing++;
812 #endif
813 	}
814 
815 	if (cpuid_opteron_erratum(cpu, 100) > 0) {
816 		/*
817 		 * Compatibility Mode Branches Transfer to Illegal Address
818 		 */
819 #if defined(OPTERON_ERRATUM_100)
820 		/*
821 		 * fix is in trap.c
822 		 */
823 		opteron_erratum_100++;
824 #else
825 		workaround_warning(cpu, 100);
826 		missing++;
827 #endif
828 	}
829 
830 	/*LINTED*/
831 	if (cpuid_opteron_erratum(cpu, 108) > 0) {
832 		/*
833 		 * CPUID Instruction May Return Incorrect Model Number In
834 		 * Some Processors
835 		 */
836 #if defined(OPTERON_ERRATUM_108)
837 		/*
838 		 * (Our cpuid-handling code corrects the model number on
839 		 * those processors)
840 		 */
841 #else
842 		workaround_warning(cpu, 108);
843 		missing++;
844 #endif
845 	}
846 
847 	/*LINTED*/
848 	if (cpuid_opteron_erratum(cpu, 109) > 0) do {
849 		/*
850 		 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour
851 		 */
852 #if defined(OPTERON_ERRATUM_109)
853 		/*
854 		 * The "workaround" is to print a warning to upgrade the BIOS
855 		 */
856 		uint64_t value;
857 		const uint_t msr = MSR_AMD_PATCHLEVEL;
858 		int err;
859 
860 		if ((err = checked_rdmsr(msr, &value)) != 0) {
861 			msr_warning(cpu, "rd", msr, err);
862 			workaround_warning(cpu, 109);
863 			missing++;
864 		}
865 		if (value == 0)
866 			opteron_erratum_109++;
867 #else
868 		workaround_warning(cpu, 109);
869 		missing++;
870 #endif
871 	/*CONSTANTCONDITION*/
872 	} while (0);
873 
874 	/*LINTED*/
875 	if (cpuid_opteron_erratum(cpu, 121) > 0) {
876 		/*
877 		 * Sequential Execution Across Non_Canonical Boundary Caused
878 		 * Processor Hang
879 		 */
880 #if defined(OPTERON_ERRATUM_121)
881 #if defined(_LP64)
882 		/*
883 		 * Erratum 121 is only present in long (64 bit) mode.
884 		 * Workaround is to include the page immediately before the
885 		 * va hole to eliminate the possibility of system hangs due to
886 		 * sequential execution across the va hole boundary.
887 		 */
888 		if (opteron_erratum_121)
889 			opteron_erratum_121++;
890 		else {
891 			if (hole_start) {
892 				hole_start -= PAGESIZE;
893 			} else {
894 				/*
895 				 * hole_start not yet initialized by
896 				 * mmu_init. Initialize hole_start
897 				 * with value to be subtracted.
898 				 */
899 				hole_start = PAGESIZE;
900 			}
901 			opteron_erratum_121++;
902 		}
903 #endif	/* _LP64 */
904 #else
905 		workaround_warning(cpu, 121);
906 		missing++;
907 #endif
908 	}
909 
910 	/*LINTED*/
911 	if (cpuid_opteron_erratum(cpu, 122) > 0) do {
912 		/*
913 		 * TLB Flush Filter May Cause Coherency Problem in
914 		 * Multiprocessor Systems
915 		 */
916 #if defined(OPTERON_ERRATUM_122)
917 		uint64_t value;
918 		const uint_t msr = MSR_AMD_HWCR;
919 		int error;
920 
921 		/*
922 		 * Erratum 122 is only present in MP configurations (multi-core
923 		 * or multi-processor).
924 		 */
925 #if defined(__xpv)
926 		if (!DOMAIN_IS_INITDOMAIN(xen_info))
927 			break;
928 		if (!opteron_erratum_122 && xen_get_nphyscpus() == 1)
929 			break;
930 #else
931 		if (!opteron_erratum_122 && opteron_get_nnodes() == 1 &&
932 		    cpuid_get_ncpu_per_chip(cpu) == 1)
933 			break;
934 #endif
935 		/* disable TLB Flush Filter */
936 
937 		if ((error = checked_rdmsr(msr, &value)) != 0) {
938 			msr_warning(cpu, "rd", msr, error);
939 			workaround_warning(cpu, 122);
940 			missing++;
941 		} else {
942 			value |= (uint64_t)AMD_HWCR_FFDIS;
943 			if ((error = checked_wrmsr(msr, value)) != 0) {
944 				msr_warning(cpu, "wr", msr, error);
945 				workaround_warning(cpu, 122);
946 				missing++;
947 			}
948 		}
949 		opteron_erratum_122++;
950 #else
951 		workaround_warning(cpu, 122);
952 		missing++;
953 #endif
954 	/*CONSTANTCONDITION*/
955 	} while (0);
956 
957 	/*LINTED*/
958 	if (cpuid_opteron_erratum(cpu, 123) > 0) do {
959 		/*
960 		 * Bypassed Reads May Cause Data Corruption of System Hang in
961 		 * Dual Core Processors
962 		 */
963 #if defined(OPTERON_ERRATUM_123)
964 		uint64_t value;
965 		const uint_t msr = MSR_AMD_PATCHLEVEL;
966 		int err;
967 
968 		/*
969 		 * Erratum 123 applies only to multi-core cpus.
970 		 */
971 		if (cpuid_get_ncpu_per_chip(cpu) < 2)
972 			break;
973 #if defined(__xpv)
974 		if (!DOMAIN_IS_INITDOMAIN(xen_info))
975 			break;
976 #endif
977 		/*
978 		 * The "workaround" is to print a warning to upgrade the BIOS
979 		 */
980 		if ((err = checked_rdmsr(msr, &value)) != 0) {
981 			msr_warning(cpu, "rd", msr, err);
982 			workaround_warning(cpu, 123);
983 			missing++;
984 		}
985 		if (value == 0)
986 			opteron_erratum_123++;
987 #else
988 		workaround_warning(cpu, 123);
989 		missing++;
990 
991 #endif
992 	/*CONSTANTCONDITION*/
993 	} while (0);
994 
995 	/*LINTED*/
996 	if (cpuid_opteron_erratum(cpu, 131) > 0) do {
997 		/*
998 		 * Multiprocessor Systems with Four or More Cores May Deadlock
999 		 * Waiting for a Probe Response
1000 		 */
1001 #if defined(OPTERON_ERRATUM_131)
1002 		uint64_t nbcfg;
1003 		const uint_t msr = MSR_AMD_NB_CFG;
1004 		const uint64_t wabits =
1005 		    AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR;
1006 		int error;
1007 
1008 		/*
1009 		 * Erratum 131 applies to any system with four or more cores.
1010 		 */
1011 		if (opteron_erratum_131)
1012 			break;
1013 #if defined(__xpv)
1014 		if (!DOMAIN_IS_INITDOMAIN(xen_info))
1015 			break;
1016 		if (xen_get_nphyscpus() < 4)
1017 			break;
1018 #else
1019 		if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4)
1020 			break;
1021 #endif
1022 		/*
1023 		 * Print a warning if neither of the workarounds for
1024 		 * erratum 131 is present.
1025 		 */
1026 		if ((error = checked_rdmsr(msr, &nbcfg)) != 0) {
1027 			msr_warning(cpu, "rd", msr, error);
1028 			workaround_warning(cpu, 131);
1029 			missing++;
1030 		} else if ((nbcfg & wabits) == 0) {
1031 			opteron_erratum_131++;
1032 		} else {
1033 			/* cannot have both workarounds set */
1034 			ASSERT((nbcfg & wabits) != wabits);
1035 		}
1036 #else
1037 		workaround_warning(cpu, 131);
1038 		missing++;
1039 #endif
1040 	/*CONSTANTCONDITION*/
1041 	} while (0);
1042 
1043 	/*
1044 	 * This isn't really an erratum, but for convenience the
1045 	 * detection/workaround code lives here and in cpuid_opteron_erratum.
1046 	 */
1047 	if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
1048 #if defined(OPTERON_WORKAROUND_6336786)
1049 		/*
1050 		 * Disable C1-Clock ramping on multi-core/multi-processor
1051 		 * K8 platforms to guard against TSC drift.
1052 		 */
1053 		if (opteron_workaround_6336786) {
1054 			opteron_workaround_6336786++;
1055 #if defined(__xpv)
1056 		} else if ((DOMAIN_IS_INITDOMAIN(xen_info) &&
1057 		    xen_get_nphyscpus() > 1) ||
1058 		    opteron_workaround_6336786_UP) {
1059 			/*
1060 			 * XXPV	Hmm.  We can't walk the Northbridges on
1061 			 *	the hypervisor; so just complain and drive
1062 			 *	on.  This probably needs to be fixed in
1063 			 *	the hypervisor itself.
1064 			 */
1065 			opteron_workaround_6336786++;
1066 			workaround_warning(cpu, 6336786);
1067 #else	/* __xpv */
1068 		} else if ((opteron_get_nnodes() *
1069 		    cpuid_get_ncpu_per_chip(cpu) > 1) ||
1070 		    opteron_workaround_6336786_UP) {
1071 
1072 			uint_t	node, nnodes;
1073 			uint8_t data;
1074 
1075 			nnodes = opteron_get_nnodes();
1076 			for (node = 0; node < nnodes; node++) {
1077 				/*
1078 				 * Clear PMM7[1:0] (function 3, offset 0x87)
1079 				 * Northbridge device is the node id + 24.
1080 				 */
1081 				data = pci_getb_func(0, node + 24, 3, 0x87);
1082 				data &= 0xFC;
1083 				pci_putb_func(0, node + 24, 3, 0x87, data);
1084 			}
1085 			opteron_workaround_6336786++;
1086 #endif	/* __xpv */
1087 		}
1088 #else
1089 		workaround_warning(cpu, 6336786);
1090 		missing++;
1091 #endif
1092 	}
1093 
1094 	/*LINTED*/
1095 	/*
1096 	 * Mutex primitives don't work as expected.
1097 	 */
1098 	if (cpuid_opteron_erratum(cpu, 6323525) > 0) {
1099 #if defined(OPTERON_WORKAROUND_6323525)
1100 		/*
1101 		 * This problem only occurs with 2 or more cores. If bit in
1102 		 * MSR_AMD_BU_CFG set, then not applicable. The workaround
1103 		 * is to patch the semaphone routines with the lfence
1104 		 * instruction to provide necessary load memory barrier with
1105 		 * possible subsequent read-modify-write ops.
1106 		 *
1107 		 * It is too early in boot to call the patch routine so
1108 		 * set erratum variable to be done in startup_end().
1109 		 */
1110 		if (opteron_workaround_6323525) {
1111 			opteron_workaround_6323525++;
1112 #if defined(__xpv)
1113 		} else if (x86_feature & X86_SSE2) {
1114 			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1115 				/*
1116 				 * XXPV	Use dom0_msr here when extended
1117 				 *	operations are supported?
1118 				 */
1119 				if (xen_get_nphyscpus() > 1)
1120 					opteron_workaround_6323525++;
1121 			} else {
1122 				/*
1123 				 * We have no way to tell how many physical
1124 				 * cpus there are, or even if this processor
1125 				 * has the problem, so enable the workaround
1126 				 * unconditionally (at some performance cost).
1127 				 */
1128 				opteron_workaround_6323525++;
1129 			}
1130 #else	/* __xpv */
1131 		} else if ((x86_feature & X86_SSE2) && ((opteron_get_nnodes() *
1132 		    cpuid_get_ncpu_per_chip(cpu)) > 1)) {
1133 			if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0)
1134 				opteron_workaround_6323525++;
1135 #endif	/* __xpv */
1136 		}
1137 #else
1138 		workaround_warning(cpu, 6323525);
1139 		missing++;
1140 #endif
1141 	}
1142 
1143 	missing += do_erratum_298(cpu);
1144 
1145 #ifdef __xpv
1146 	return (0);
1147 #else
1148 	return (missing);
1149 #endif
1150 }
1151 
1152 void
1153 workaround_errata_end()
1154 {
1155 #if defined(OPTERON_ERRATUM_88)
1156 	if (opteron_erratum_88)
1157 		workaround_applied(88);
1158 #endif
1159 #if defined(OPTERON_ERRATUM_91)
1160 	if (opteron_erratum_91)
1161 		workaround_applied(91);
1162 #endif
1163 #if defined(OPTERON_ERRATUM_93)
1164 	if (opteron_erratum_93)
1165 		workaround_applied(93);
1166 #endif
1167 #if defined(OPTERON_ERRATUM_95)
1168 	if (opteron_erratum_95)
1169 		workaround_applied(95);
1170 #endif
1171 #if defined(OPTERON_ERRATUM_100)
1172 	if (opteron_erratum_100)
1173 		workaround_applied(100);
1174 #endif
1175 #if defined(OPTERON_ERRATUM_108)
1176 	if (opteron_erratum_108)
1177 		workaround_applied(108);
1178 #endif
1179 #if defined(OPTERON_ERRATUM_109)
1180 	if (opteron_erratum_109) {
1181 		cmn_err(CE_WARN,
1182 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1183 		    " processor\nerratum 109 was not detected; updating your"
1184 		    " system's BIOS to a version\ncontaining this"
1185 		    " microcode patch is HIGHLY recommended or erroneous"
1186 		    " system\noperation may occur.\n");
1187 	}
1188 #endif
1189 #if defined(OPTERON_ERRATUM_121)
1190 	if (opteron_erratum_121)
1191 		workaround_applied(121);
1192 #endif
1193 #if defined(OPTERON_ERRATUM_122)
1194 	if (opteron_erratum_122)
1195 		workaround_applied(122);
1196 #endif
1197 #if defined(OPTERON_ERRATUM_123)
1198 	if (opteron_erratum_123) {
1199 		cmn_err(CE_WARN,
1200 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1201 		    " processor\nerratum 123 was not detected; updating your"
1202 		    " system's BIOS to a version\ncontaining this"
1203 		    " microcode patch is HIGHLY recommended or erroneous"
1204 		    " system\noperation may occur.\n");
1205 	}
1206 #endif
1207 #if defined(OPTERON_ERRATUM_131)
1208 	if (opteron_erratum_131) {
1209 		cmn_err(CE_WARN,
1210 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1211 		    " processor\nerratum 131 was not detected; updating your"
1212 		    " system's BIOS to a version\ncontaining this"
1213 		    " microcode patch is HIGHLY recommended or erroneous"
1214 		    " system\noperation may occur.\n");
1215 	}
1216 #endif
1217 #if defined(OPTERON_WORKAROUND_6336786)
1218 	if (opteron_workaround_6336786)
1219 		workaround_applied(6336786);
1220 #endif
1221 #if defined(OPTERON_WORKAROUND_6323525)
1222 	if (opteron_workaround_6323525)
1223 		workaround_applied(6323525);
1224 #endif
1225 #if defined(OPTERON_ERRATUM_298)
1226 	if (opteron_erratum_298) {
1227 		cmn_err(CE_WARN,
1228 		    "BIOS microcode patch for AMD 64/Opteron(tm)"
1229 		    " processor\nerratum 298 was not detected; updating your"
1230 		    " system's BIOS to a version\ncontaining this"
1231 		    " microcode patch is HIGHLY recommended or erroneous"
1232 		    " system\noperation may occur.\n");
1233 	}
1234 #endif
1235 }
1236 
1237 static cpuset_t procset;
1238 
1239 /*
1240  * Start a single cpu, assuming that the kernel context is available
1241  * to successfully start another cpu.
1242  *
1243  * (For example, real mode code is mapped into the right place
1244  * in memory and is ready to be run.)
1245  */
1246 int
1247 start_cpu(processorid_t who)
1248 {
1249 	void *ctx;
1250 	cpu_t *cp;
1251 	int delays;
1252 	int error = 0;
1253 
1254 	ASSERT(who != 0);
1255 
1256 	/*
1257 	 * Check if there's at least a Mbyte of kmem available
1258 	 * before attempting to start the cpu.
1259 	 */
1260 	if (kmem_avail() < 1024 * 1024) {
1261 		/*
1262 		 * Kick off a reap in case that helps us with
1263 		 * later attempts ..
1264 		 */
1265 		kmem_reap();
1266 		return (ENOMEM);
1267 	}
1268 
1269 	cp = mp_startup_init(who);
1270 	if ((ctx = mach_cpucontext_alloc(cp)) == NULL ||
1271 	    (error = mach_cpu_start(cp, ctx)) != 0) {
1272 
1273 		/*
1274 		 * Something went wrong before we even started it
1275 		 */
1276 		if (ctx)
1277 			cmn_err(CE_WARN,
1278 			    "cpu%d: failed to start error %d",
1279 			    cp->cpu_id, error);
1280 		else
1281 			cmn_err(CE_WARN,
1282 			    "cpu%d: failed to allocate context", cp->cpu_id);
1283 
1284 		if (ctx)
1285 			mach_cpucontext_free(cp, ctx, error);
1286 		else
1287 			error = EAGAIN;		/* hmm. */
1288 		mp_startup_fini(cp, error);
1289 		return (error);
1290 	}
1291 
1292 	for (delays = 0; !CPU_IN_SET(procset, who); delays++) {
1293 		if (delays == 500) {
1294 			/*
1295 			 * After five seconds, things are probably looking
1296 			 * a bit bleak - explain the hang.
1297 			 */
1298 			cmn_err(CE_NOTE, "cpu%d: started, "
1299 			    "but not running in the kernel yet", who);
1300 		} else if (delays > 2000) {
1301 			/*
1302 			 * We waited at least 20 seconds, bail ..
1303 			 */
1304 			error = ETIMEDOUT;
1305 			cmn_err(CE_WARN, "cpu%d: timed out", who);
1306 			mach_cpucontext_free(cp, ctx, error);
1307 			mp_startup_fini(cp, error);
1308 			return (error);
1309 		}
1310 
1311 		/*
1312 		 * wait at least 10ms, then check again..
1313 		 */
1314 		delay(USEC_TO_TICK_ROUNDUP(10000));
1315 	}
1316 
1317 	mach_cpucontext_free(cp, ctx, 0);
1318 
1319 #ifndef __xpv
1320 	if (tsc_gethrtime_enable)
1321 		tsc_sync_master(who);
1322 #endif
1323 
1324 	if (dtrace_cpu_init != NULL) {
1325 		/*
1326 		 * DTrace CPU initialization expects cpu_lock to be held.
1327 		 */
1328 		mutex_enter(&cpu_lock);
1329 		(*dtrace_cpu_init)(who);
1330 		mutex_exit(&cpu_lock);
1331 	}
1332 
1333 	while (!CPU_IN_SET(cpu_ready_set, who))
1334 		delay(1);
1335 
1336 	return (0);
1337 }
1338 
1339 
1340 /*ARGSUSED*/
1341 void
1342 start_other_cpus(int cprboot)
1343 {
1344 	uint_t who;
1345 	uint_t skipped = 0;
1346 	uint_t bootcpuid = 0;
1347 
1348 	/*
1349 	 * Initialize our own cpu_info.
1350 	 */
1351 	init_cpu_info(CPU);
1352 
1353 	cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr);
1354 	cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr);
1355 
1356 	/*
1357 	 * Initialize our syscall handlers
1358 	 */
1359 	init_cpu_syscall(CPU);
1360 
1361 	/*
1362 	 * Take the boot cpu out of the mp_cpus set because we know
1363 	 * it's already running.  Add it to the cpu_ready_set for
1364 	 * precisely the same reason.
1365 	 */
1366 	CPUSET_DEL(mp_cpus, bootcpuid);
1367 	CPUSET_ADD(cpu_ready_set, bootcpuid);
1368 
1369 	/*
1370 	 * if only 1 cpu or not using MP, skip the rest of this
1371 	 */
1372 	if (CPUSET_ISNULL(mp_cpus) || use_mp == 0) {
1373 		if (use_mp == 0)
1374 			cmn_err(CE_CONT, "?***** Not in MP mode\n");
1375 		goto done;
1376 	}
1377 
1378 	/*
1379 	 * perform such initialization as is needed
1380 	 * to be able to take CPUs on- and off-line.
1381 	 */
1382 	cpu_pause_init();
1383 
1384 	xc_init();		/* initialize processor crosscalls */
1385 
1386 	if (mach_cpucontext_init() != 0)
1387 		goto done;
1388 
1389 	flushes_require_xcalls = 1;
1390 
1391 	/*
1392 	 * We lock our affinity to the master CPU to ensure that all slave CPUs
1393 	 * do their TSC syncs with the same CPU.
1394 	 */
1395 	affinity_set(CPU_CURRENT);
1396 
1397 	for (who = 0; who < NCPU; who++) {
1398 
1399 		if (!CPU_IN_SET(mp_cpus, who))
1400 			continue;
1401 		ASSERT(who != bootcpuid);
1402 		if (ncpus >= max_ncpus) {
1403 			skipped = who;
1404 			continue;
1405 		}
1406 		if (start_cpu(who) != 0)
1407 			CPUSET_DEL(mp_cpus, who);
1408 
1409 		mutex_enter(&cpu_lock);
1410 		cpu_state_change_notify(who, CPU_SETUP);
1411 		mutex_exit(&cpu_lock);
1412 	}
1413 
1414 	/* Free the space allocated to hold the microcode file */
1415 	ucode_cleanup();
1416 
1417 	affinity_clear();
1418 
1419 	if (skipped) {
1420 		cmn_err(CE_NOTE,
1421 		    "System detected %d cpus, but "
1422 		    "only %d cpu(s) were enabled during boot.",
1423 		    skipped + 1, ncpus);
1424 		cmn_err(CE_NOTE,
1425 		    "Use \"boot-ncpus\" parameter to enable more CPU(s). "
1426 		    "See eeprom(1M).");
1427 	}
1428 
1429 done:
1430 	workaround_errata_end();
1431 	mach_cpucontext_fini();
1432 
1433 	cmi_post_mpstartup();
1434 }
1435 
1436 /*
1437  * Dummy functions - no i86pc platforms support dynamic cpu allocation.
1438  */
1439 /*ARGSUSED*/
1440 int
1441 mp_cpu_configure(int cpuid)
1442 {
1443 	return (ENOTSUP);		/* not supported */
1444 }
1445 
1446 /*ARGSUSED*/
1447 int
1448 mp_cpu_unconfigure(int cpuid)
1449 {
1450 	return (ENOTSUP);		/* not supported */
1451 }
1452 
1453 /*
1454  * Startup function for 'other' CPUs (besides boot cpu).
1455  * Called from real_mode_start.
1456  *
1457  * WARNING: until CPU_READY is set, mp_startup and routines called by
1458  * mp_startup should not call routines (e.g. kmem_free) that could call
1459  * hat_unload which requires CPU_READY to be set.
1460  */
1461 void
1462 mp_startup(void)
1463 {
1464 	struct cpu *cp = CPU;
1465 	uint_t new_x86_feature;
1466 #ifndef __xpv
1467 	extern void cpupm_init(cpu_t *);
1468 #endif
1469 
1470 	/*
1471 	 * We need to get TSC on this proc synced (i.e., any delta
1472 	 * from cpu0 accounted for) as soon as we can, because many
1473 	 * many things use gethrtime/pc_gethrestime, including
1474 	 * interrupts, cmn_err, etc.
1475 	 */
1476 
1477 	/* Let cpu0 continue into tsc_sync_master() */
1478 	CPUSET_ATOMIC_ADD(procset, cp->cpu_id);
1479 
1480 #ifndef __xpv
1481 	if (tsc_gethrtime_enable)
1482 		tsc_sync_slave();
1483 #endif
1484 
1485 	/*
1486 	 * Once this was done from assembly, but it's safer here; if
1487 	 * it blocks, we need to be able to swtch() to and from, and
1488 	 * since we get here by calling t_pc, we need to do that call
1489 	 * before swtch() overwrites it.
1490 	 */
1491 	(void) (*ap_mlsetup)();
1492 
1493 	new_x86_feature = cpuid_pass1(cp);
1494 
1495 #ifndef __xpv
1496 	/*
1497 	 * Program this cpu's PAT
1498 	 */
1499 	if (x86_feature & X86_PAT)
1500 		pat_sync();
1501 #endif
1502 
1503 	/*
1504 	 * Set up TSC_AUX to contain the cpuid for this processor
1505 	 * for the rdtscp instruction.
1506 	 */
1507 	if (x86_feature & X86_TSCP)
1508 		(void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id);
1509 
1510 	/*
1511 	 * Initialize this CPU's syscall handlers
1512 	 */
1513 	init_cpu_syscall(cp);
1514 
1515 	/*
1516 	 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
1517 	 * highest level at which a routine is permitted to block on
1518 	 * an adaptive mutex (allows for cpu poke interrupt in case
1519 	 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
1520 	 * device interrupts that may end up in the hat layer issuing cross
1521 	 * calls before CPU_READY is set.
1522 	 */
1523 	splx(ipltospl(LOCK_LEVEL));
1524 	sti();
1525 
1526 	/*
1527 	 * Do a sanity check to make sure this new CPU is a sane thing
1528 	 * to add to the collection of processors running this system.
1529 	 *
1530 	 * XXX	Clearly this needs to get more sophisticated, if x86
1531 	 * systems start to get built out of heterogenous CPUs; as is
1532 	 * likely to happen once the number of processors in a configuration
1533 	 * gets large enough.
1534 	 */
1535 	if ((x86_feature & new_x86_feature) != x86_feature) {
1536 		cmn_err(CE_CONT, "?cpu%d: %b\n",
1537 		    cp->cpu_id, new_x86_feature, FMT_X86_FEATURE);
1538 		cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
1539 	}
1540 
1541 	/*
1542 	 * We do not support cpus with mixed monitor/mwait support if the
1543 	 * boot cpu supports monitor/mwait.
1544 	 */
1545 	if ((x86_feature & ~new_x86_feature) & X86_MWAIT)
1546 		panic("unsupported mixed cpu monitor/mwait support detected");
1547 
1548 	/*
1549 	 * We could be more sophisticated here, and just mark the CPU
1550 	 * as "faulted" but at this point we'll opt for the easier
1551 	 * answer of dieing horribly.  Provided the boot cpu is ok,
1552 	 * the system can be recovered by booting with use_mp set to zero.
1553 	 */
1554 	if (workaround_errata(cp) != 0)
1555 		panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
1556 
1557 	cpuid_pass2(cp);
1558 	cpuid_pass3(cp);
1559 	(void) cpuid_pass4(cp);
1560 
1561 	init_cpu_info(cp);
1562 
1563 	mutex_enter(&cpu_lock);
1564 
1565 	cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS;
1566 
1567 	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1568 	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1569 
1570 	if (dtrace_cpu_init != NULL) {
1571 		(*dtrace_cpu_init)(cp->cpu_id);
1572 	}
1573 
1574 	/*
1575 	 * Fill out cpu_ucode_info.  Update microcode if necessary.
1576 	 */
1577 	ucode_check(cp);
1578 
1579 	mutex_exit(&cpu_lock);
1580 
1581 	/*
1582 	 * Enable preemption here so that contention for any locks acquired
1583 	 * later in mp_startup may be preempted if the thread owning those
1584 	 * locks is continously executing on other CPUs (for example, this
1585 	 * CPU must be preemptible to allow other CPUs to pause it during their
1586 	 * startup phases).  It's safe to enable preemption here because the
1587 	 * CPU state is pretty-much fully constructed.
1588 	 */
1589 	curthread->t_preempt = 0;
1590 
1591 	/* The base spl should still be at LOCK LEVEL here */
1592 	ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL));
1593 	set_base_spl();		/* Restore the spl to its proper value */
1594 
1595 #ifndef __xpv
1596 	cpupm_init(cp);
1597 #endif
1598 	add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi);
1599 
1600 	/*
1601 	 * Processor group initialization for this CPU is dependent on the
1602 	 * cpuid probing, which must be done in the context of the current
1603 	 * CPU, as well as the CPU's device node initialization (for ACPI).
1604 	 */
1605 	mutex_enter(&cpu_lock);
1606 	pghw_physid_create(cp);
1607 	pg_cpu_init(cp);
1608 	pg_cmt_cpu_startup(cp);
1609 	mutex_exit(&cpu_lock);
1610 
1611 	/* Enable interrupts */
1612 	(void) spl0();
1613 
1614 	mutex_enter(&cpu_lock);
1615 	cpu_enable_intr(cp);
1616 	cpu_add_active(cp);
1617 	mutex_exit(&cpu_lock);
1618 
1619 #ifndef __xpv
1620 	{
1621 		/*
1622 		 * Set up the CPU module for this CPU.  This can't be done
1623 		 * before this CPU is made CPU_READY, because we may (in
1624 		 * heterogeneous systems) need to go load another CPU module.
1625 		 * The act of attempting to load a module may trigger a
1626 		 * cross-call, which will ASSERT unless this cpu is CPU_READY.
1627 		 */
1628 		cmi_hdl_t hdl;
1629 
1630 		if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1631 		    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1632 			if (x86_feature & X86_MCA)
1633 				cmi_mca_init(hdl);
1634 		}
1635 	}
1636 #endif /* __xpv */
1637 
1638 	if (boothowto & RB_DEBUG)
1639 		kdi_cpu_init();
1640 
1641 	/*
1642 	 * Setting the bit in cpu_ready_set must be the last operation in
1643 	 * processor initialization; the boot CPU will continue to boot once
1644 	 * it sees this bit set for all active CPUs.
1645 	 */
1646 	CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
1647 
1648 	/*
1649 	 * Because mp_startup() gets fired off after init() starts, we
1650 	 * can't use the '?' trick to do 'boot -v' printing - so we
1651 	 * always direct the 'cpu .. online' messages to the log.
1652 	 */
1653 	cmn_err(CE_CONT, "!cpu%d initialization complete - online\n",
1654 	    cp->cpu_id);
1655 
1656 	/*
1657 	 * Now we are done with the startup thread, so free it up.
1658 	 */
1659 	thread_exit();
1660 	panic("mp_startup: cannot return");
1661 	/*NOTREACHED*/
1662 }
1663 
1664 
1665 /*
1666  * Start CPU on user request.
1667  */
1668 /* ARGSUSED */
1669 int
1670 mp_cpu_start(struct cpu *cp)
1671 {
1672 	ASSERT(MUTEX_HELD(&cpu_lock));
1673 	return (0);
1674 }
1675 
1676 /*
1677  * Stop CPU on user request.
1678  */
1679 /* ARGSUSED */
1680 int
1681 mp_cpu_stop(struct cpu *cp)
1682 {
1683 	extern int cbe_psm_timer_mode;
1684 	ASSERT(MUTEX_HELD(&cpu_lock));
1685 
1686 #ifdef __xpv
1687 	/*
1688 	 * We can't offline vcpu0.
1689 	 */
1690 	if (cp->cpu_id == 0)
1691 		return (EBUSY);
1692 #endif
1693 
1694 	/*
1695 	 * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1696 	 * can't stop it.  (This is true only for machines with no TSC.)
1697 	 */
1698 
1699 	if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1700 		return (EBUSY);
1701 
1702 	return (0);
1703 }
1704 
1705 /*
1706  * Take the specified CPU out of participation in interrupts.
1707  */
1708 int
1709 cpu_disable_intr(struct cpu *cp)
1710 {
1711 	if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
1712 		return (EBUSY);
1713 
1714 	cp->cpu_flags &= ~CPU_ENABLE;
1715 	return (0);
1716 }
1717 
1718 /*
1719  * Allow the specified CPU to participate in interrupts.
1720  */
1721 void
1722 cpu_enable_intr(struct cpu *cp)
1723 {
1724 	ASSERT(MUTEX_HELD(&cpu_lock));
1725 	cp->cpu_flags |= CPU_ENABLE;
1726 	psm_enable_intr(cp->cpu_id);
1727 }
1728 
1729 
1730 /*ARGSUSED*/
1731 void
1732 mp_cpu_faulted_enter(struct cpu *cp)
1733 {
1734 #ifndef __xpv
1735 	cmi_hdl_t hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
1736 	    cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
1737 
1738 	if (hdl != NULL) {
1739 		cmi_faulted_enter(hdl);
1740 		cmi_hdl_rele(hdl);
1741 	}
1742 #endif
1743 }
1744 
1745 /*ARGSUSED*/
1746 void
1747 mp_cpu_faulted_exit(struct cpu *cp)
1748 {
1749 #ifndef __xpv
1750 	cmi_hdl_t hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
1751 	    cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
1752 
1753 	if (hdl != NULL) {
1754 		cmi_faulted_exit(hdl);
1755 		cmi_hdl_rele(hdl);
1756 	}
1757 #endif
1758 }
1759 
1760 /*
1761  * The following two routines are used as context operators on threads belonging
1762  * to processes with a private LDT (see sysi86).  Due to the rarity of such
1763  * processes, these routines are currently written for best code readability and
1764  * organization rather than speed.  We could avoid checking x86_feature at every
1765  * context switch by installing different context ops, depending on the
1766  * x86_feature flags, at LDT creation time -- one for each combination of fast
1767  * syscall feature flags.
1768  */
1769 
1770 /*ARGSUSED*/
1771 void
1772 cpu_fast_syscall_disable(void *arg)
1773 {
1774 	if ((x86_feature & (X86_MSR | X86_SEP)) == (X86_MSR | X86_SEP))
1775 		cpu_sep_disable();
1776 	if ((x86_feature & (X86_MSR | X86_ASYSC)) == (X86_MSR | X86_ASYSC))
1777 		cpu_asysc_disable();
1778 }
1779 
1780 /*ARGSUSED*/
1781 void
1782 cpu_fast_syscall_enable(void *arg)
1783 {
1784 	if ((x86_feature & (X86_MSR | X86_SEP)) == (X86_MSR | X86_SEP))
1785 		cpu_sep_enable();
1786 	if ((x86_feature & (X86_MSR | X86_ASYSC)) == (X86_MSR | X86_ASYSC))
1787 		cpu_asysc_enable();
1788 }
1789 
1790 static void
1791 cpu_sep_enable(void)
1792 {
1793 	ASSERT(x86_feature & X86_SEP);
1794 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
1795 
1796 	wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
1797 }
1798 
1799 static void
1800 cpu_sep_disable(void)
1801 {
1802 	ASSERT(x86_feature & X86_SEP);
1803 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
1804 
1805 	/*
1806 	 * Setting the SYSENTER_CS_MSR register to 0 causes software executing
1807 	 * the sysenter or sysexit instruction to trigger a #gp fault.
1808 	 */
1809 	wrmsr(MSR_INTC_SEP_CS, 0);
1810 }
1811 
1812 static void
1813 cpu_asysc_enable(void)
1814 {
1815 	ASSERT(x86_feature & X86_ASYSC);
1816 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
1817 
1818 	wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
1819 	    (uint64_t)(uintptr_t)AMD_EFER_SCE);
1820 }
1821 
1822 static void
1823 cpu_asysc_disable(void)
1824 {
1825 	ASSERT(x86_feature & X86_ASYSC);
1826 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
1827 
1828 	/*
1829 	 * Turn off the SCE (syscall enable) bit in the EFER register. Software
1830 	 * executing syscall or sysret with this bit off will incur a #ud trap.
1831 	 */
1832 	wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
1833 	    ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
1834 }
1835