xref: /illumos-gate/usr/src/uts/i86pc/os/mp_machdep.c (revision 24b9abbad58fdd63dad716fd35a99a7944c4e3eb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #define	PSMI_1_6
29 #include <sys/smp_impldefs.h>
30 #include <sys/psm.h>
31 #include <sys/psm_modctl.h>
32 #include <sys/pit.h>
33 #include <sys/cmn_err.h>
34 #include <sys/strlog.h>
35 #include <sys/clock.h>
36 #include <sys/debug.h>
37 #include <sys/rtc.h>
38 #include <sys/x86_archext.h>
39 #include <sys/cpupart.h>
40 #include <sys/cpuvar.h>
41 #include <sys/cmt.h>
42 #include <sys/cpu.h>
43 #include <sys/disp.h>
44 #include <sys/archsystm.h>
45 #include <sys/machsystm.h>
46 #include <sys/sysmacros.h>
47 #include <sys/memlist.h>
48 #include <sys/param.h>
49 #include <sys/promif.h>
50 #if defined(__xpv)
51 #include <sys/hypervisor.h>
52 #endif
53 #include <sys/mach_intr.h>
54 #include <vm/hat_i86.h>
55 #include <sys/kdi_machimpl.h>
56 #include <sys/sdt.h>
57 
58 #define	OFFSETOF(s, m)		(size_t)(&(((s *)0)->m))
59 
60 /*
61  *	Local function prototypes
62  */
63 static int mp_disable_intr(processorid_t cpun);
64 static void mp_enable_intr(processorid_t cpun);
65 static void mach_init();
66 static void mach_picinit();
67 static int machhztomhz(uint64_t cpu_freq_hz);
68 static uint64_t mach_getcpufreq(void);
69 static void mach_fixcpufreq(void);
70 static int mach_clkinit(int, int *);
71 static void mach_smpinit(void);
72 static int mach_softlvl_to_vect(int ipl);
73 static void mach_get_platform(int owner);
74 static void mach_construct_info();
75 static int mach_translate_irq(dev_info_t *dip, int irqno);
76 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
77     psm_intr_op_t, int *);
78 static void mach_notify_error(int level, char *errmsg);
79 static hrtime_t dummy_hrtime(void);
80 static void dummy_scalehrtime(hrtime_t *);
81 static void cpu_idle(void);
82 static void cpu_wakeup(cpu_t *, int);
83 #ifndef __xpv
84 static void cpu_idle_mwait(void);
85 static void cpu_wakeup_mwait(cpu_t *, int);
86 #endif
87 /*
88  *	External reference functions
89  */
90 extern void return_instr();
91 extern uint64_t freq_tsc(uint32_t *);
92 #if defined(__i386)
93 extern uint64_t freq_notsc(uint32_t *);
94 #endif
95 extern void pc_gethrestime(timestruc_t *);
96 extern int cpuid_get_coreid(cpu_t *);
97 extern int cpuid_get_chipid(cpu_t *);
98 
99 /*
100  *	PSM functions initialization
101  */
102 void (*psm_shutdownf)(int, int)	= (void (*)(int, int))return_instr;
103 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr;
104 void (*psm_notifyf)(int)	= (void (*)(int))return_instr;
105 void (*psm_set_idle_cpuf)(int)	= (void (*)(int))return_instr;
106 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr;
107 void (*psminitf)()		= mach_init;
108 void (*picinitf)() 		= return_instr;
109 int (*clkinitf)(int, int *) 	= (int (*)(int, int *))return_instr;
110 int (*ap_mlsetup)() 		= (int (*)(void))return_instr;
111 void (*send_dirintf)() 		= return_instr;
112 void (*setspl)(int)		= (void (*)(int))return_instr;
113 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
114 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
115 void (*kdisetsoftint)(int, struct av_softinfo *)=
116 	(void (*)(int, struct av_softinfo *))return_instr;
117 void (*setsoftint)(int, struct av_softinfo *)=
118 	(void (*)(int, struct av_softinfo *))return_instr;
119 int (*slvltovect)(int)		= (int (*)(int))return_instr;
120 int (*setlvl)(int, int *)	= (int (*)(int, int *))return_instr;
121 void (*setlvlx)(int, int)	= (void (*)(int, int))return_instr;
122 int (*psm_disable_intr)(int)	= mp_disable_intr;
123 void (*psm_enable_intr)(int)	= mp_enable_intr;
124 hrtime_t (*gethrtimef)(void)	= dummy_hrtime;
125 hrtime_t (*gethrtimeunscaledf)(void)	= dummy_hrtime;
126 void (*scalehrtimef)(hrtime_t *)	= dummy_scalehrtime;
127 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq;
128 void (*gethrestimef)(timestruc_t *) = pc_gethrestime;
129 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL;
130 int (*psm_get_clockirq)(int) = NULL;
131 int (*psm_get_ipivect)(int, int) = NULL;
132 
133 int (*psm_clkinit)(int) = NULL;
134 void (*psm_timer_reprogram)(hrtime_t) = NULL;
135 void (*psm_timer_enable)(void) = NULL;
136 void (*psm_timer_disable)(void) = NULL;
137 void (*psm_post_cyclic_setup)(void *arg) = NULL;
138 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t,
139     int *) = mach_intr_ops;
140 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *))
141     return_instr;
142 
143 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr;
144 void (*hrtime_tick)(void)	= return_instr;
145 
146 /*
147  * True if the generic TSC code is our source of hrtime, rather than whatever
148  * the PSM can provide.
149  */
150 #ifdef __xpv
151 int tsc_gethrtime_enable = 0;
152 #else
153 int tsc_gethrtime_enable = 1;
154 #endif
155 int tsc_gethrtime_initted = 0;
156 
157 /*
158  * True if the hrtime implementation is "hires"; namely, better than microdata.
159  */
160 int gethrtime_hires = 0;
161 
162 /*
163  * Local Static Data
164  */
165 static struct psm_ops mach_ops;
166 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL};
167 static ushort_t mach_ver[4] = {0, 0, 0, 0};
168 
169 /*
170  * If non-zero, idle cpus will become "halted" when there's
171  * no work to do.
172  */
173 int	idle_cpu_use_hlt = 1;
174 
175 #ifndef __xpv
176 /*
177  * If non-zero, idle cpus will use mwait if available to halt instead of hlt.
178  */
179 int	idle_cpu_prefer_mwait = 1;
180 #endif
181 
182 /*ARGSUSED*/
183 int
184 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw)
185 {
186 	switch (hw) {
187 	case PGHW_IPIPE:
188 		if (x86_feature & (X86_HTT)) {
189 			/*
190 			 * Hyper-threading is SMT
191 			 */
192 			return (1);
193 		} else {
194 			return (0);
195 		}
196 	case PGHW_CHIP:
197 		if (x86_feature & (X86_CMP|X86_HTT))
198 			return (1);
199 		else
200 			return (0);
201 	case PGHW_CACHE:
202 		if (cpuid_get_ncpu_sharing_last_cache(cp) > 1)
203 			return (1);
204 		else
205 			return (0);
206 	default:
207 		return (0);
208 	}
209 }
210 
211 /*
212  * Compare two CPUs and see if they have a pghw_type_t sharing relationship
213  * If pghw_type_t is an unsupported hardware type, then return -1
214  */
215 int
216 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw)
217 {
218 	id_t pgp_a, pgp_b;
219 
220 	pgp_a = pg_plat_hw_instance_id(cpu_a, hw);
221 	pgp_b = pg_plat_hw_instance_id(cpu_b, hw);
222 
223 	if (pgp_a == -1 || pgp_b == -1)
224 		return (-1);
225 
226 	return (pgp_a == pgp_b);
227 }
228 
229 /*
230  * Return a physical instance identifier for known hardware sharing
231  * relationships
232  */
233 id_t
234 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw)
235 {
236 	switch (hw) {
237 	case PGHW_IPIPE:
238 		return (cpuid_get_coreid(cpu));
239 	case PGHW_CACHE:
240 		return (cpuid_get_last_lvl_cacheid(cpu));
241 	case PGHW_CHIP:
242 		return (cpuid_get_chipid(cpu));
243 	default:
244 		return (-1);
245 	}
246 }
247 
248 int
249 pg_plat_hw_level(pghw_type_t hw)
250 {
251 	int i;
252 	static pghw_type_t hw_hier[] = {
253 		PGHW_IPIPE,
254 		PGHW_CACHE,
255 		PGHW_CHIP,
256 		PGHW_NUM_COMPONENTS
257 	};
258 
259 	for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) {
260 		if (hw_hier[i] == hw)
261 			return (i);
262 	}
263 	return (-1);
264 }
265 
266 /*
267  * Return 1 if CMT load balancing policies should be
268  * implemented across instances of the specified hardware
269  * sharing relationship.
270  */
271 int
272 pg_plat_cmt_load_bal_hw(pghw_type_t hw)
273 {
274 	if (hw == PGHW_IPIPE ||
275 	    hw == PGHW_FPU ||
276 	    hw == PGHW_CHIP ||
277 	    hw == PGHW_CACHE)
278 		return (1);
279 	else
280 		return (0);
281 }
282 
283 
284 /*
285  * Return 1 if thread affinity polices should be implemented
286  * for instances of the specifed hardware sharing relationship.
287  */
288 int
289 pg_plat_cmt_affinity_hw(pghw_type_t hw)
290 {
291 	if (hw == PGHW_CACHE)
292 		return (1);
293 	else
294 		return (0);
295 }
296 
297 id_t
298 pg_plat_get_core_id(cpu_t *cpu)
299 {
300 	return ((id_t)cpuid_get_coreid(cpu));
301 }
302 
303 void
304 cmp_set_nosteal_interval(void)
305 {
306 	/* Set the nosteal interval (used by disp_getbest()) to 100us */
307 	nosteal_nsec = 100000UL;
308 }
309 
310 /*
311  * Routine to ensure initial callers to hrtime gets 0 as return
312  */
313 static hrtime_t
314 dummy_hrtime(void)
315 {
316 	return (0);
317 }
318 
319 /* ARGSUSED */
320 static void
321 dummy_scalehrtime(hrtime_t *ticks)
322 {}
323 
324 /*
325  * Idle the present CPU until awoken via an interrupt
326  */
327 static void
328 cpu_idle(void)
329 {
330 	cpu_t		*cpup = CPU;
331 	processorid_t	cpun = cpup->cpu_id;
332 	cpupart_t	*cp = cpup->cpu_part;
333 	int		hset_update = 1;
334 
335 	/*
336 	 * If this CPU is online, and there's multiple CPUs
337 	 * in the system, then we should notate our halting
338 	 * by adding ourselves to the partition's halted CPU
339 	 * bitmap. This allows other CPUs to find/awaken us when
340 	 * work becomes available.
341 	 */
342 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
343 		hset_update = 0;
344 
345 	/*
346 	 * Add ourselves to the partition's halted CPUs bitmask
347 	 * and set our HALTED flag, if necessary.
348 	 *
349 	 * When a thread becomes runnable, it is placed on the queue
350 	 * and then the halted cpuset is checked to determine who
351 	 * (if anyone) should be awoken. We therefore need to first
352 	 * add ourselves to the halted cpuset, and and then check if there
353 	 * is any work available.
354 	 *
355 	 * Note that memory barriers after updating the HALTED flag
356 	 * are not necessary since an atomic operation (updating the bitmap)
357 	 * immediately follows. On x86 the atomic operation acts as a
358 	 * memory barrier for the update of cpu_disp_flags.
359 	 */
360 	if (hset_update) {
361 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
362 		CPUSET_ATOMIC_ADD(cp->cp_mach->mc_haltset, cpun);
363 	}
364 
365 	/*
366 	 * Check to make sure there's really nothing to do.
367 	 * Work destined for this CPU may become available after
368 	 * this check. We'll be notified through the clearing of our
369 	 * bit in the halted CPU bitmask, and a poke.
370 	 */
371 	if (disp_anywork()) {
372 		if (hset_update) {
373 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
374 			CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun);
375 		}
376 		return;
377 	}
378 
379 	/*
380 	 * We're on our way to being halted.
381 	 *
382 	 * Disable interrupts now, so that we'll awaken immediately
383 	 * after halting if someone tries to poke us between now and
384 	 * the time we actually halt.
385 	 *
386 	 * We check for the presence of our bit after disabling interrupts.
387 	 * If it's cleared, we'll return. If the bit is cleared after
388 	 * we check then the poke will pop us out of the halted state.
389 	 *
390 	 * This means that the ordering of the poke and the clearing
391 	 * of the bit by cpu_wakeup is important.
392 	 * cpu_wakeup() must clear, then poke.
393 	 * cpu_idle() must disable interrupts, then check for the bit.
394 	 */
395 	cli();
396 
397 	if (hset_update && !CPU_IN_SET(cp->cp_mach->mc_haltset, cpun)) {
398 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
399 		sti();
400 		return;
401 	}
402 
403 	/*
404 	 * The check for anything locally runnable is here for performance
405 	 * and isn't needed for correctness. disp_nrunnable ought to be
406 	 * in our cache still, so it's inexpensive to check, and if there
407 	 * is anything runnable we won't have to wait for the poke.
408 	 */
409 	if (cpup->cpu_disp->disp_nrunnable != 0) {
410 		if (hset_update) {
411 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
412 			CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun);
413 		}
414 		sti();
415 		return;
416 	}
417 
418 	DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C1);
419 
420 	mach_cpu_idle();
421 
422 	DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C0);
423 
424 	/*
425 	 * We're no longer halted
426 	 */
427 	if (hset_update) {
428 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
429 		CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun);
430 	}
431 }
432 
433 
434 /*
435  * If "cpu" is halted, then wake it up clearing its halted bit in advance.
436  * Otherwise, see if other CPUs in the cpu partition are halted and need to
437  * be woken up so that they can steal the thread we placed on this CPU.
438  * This function is only used on MP systems.
439  */
440 static void
441 cpu_wakeup(cpu_t *cpu, int bound)
442 {
443 	uint_t		cpu_found;
444 	int		result;
445 	cpupart_t	*cp;
446 
447 	cp = cpu->cpu_part;
448 	if (CPU_IN_SET(cp->cp_mach->mc_haltset, cpu->cpu_id)) {
449 		/*
450 		 * Clear the halted bit for that CPU since it will be
451 		 * poked in a moment.
452 		 */
453 		CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpu->cpu_id);
454 		/*
455 		 * We may find the current CPU present in the halted cpuset
456 		 * if we're in the context of an interrupt that occurred
457 		 * before we had a chance to clear our bit in cpu_idle().
458 		 * Poking ourself is obviously unnecessary, since if
459 		 * we're here, we're not halted.
460 		 */
461 		if (cpu != CPU)
462 			poke_cpu(cpu->cpu_id);
463 		return;
464 	} else {
465 		/*
466 		 * This cpu isn't halted, but it's idle or undergoing a
467 		 * context switch. No need to awaken anyone else.
468 		 */
469 		if (cpu->cpu_thread == cpu->cpu_idle_thread ||
470 		    cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL)
471 			return;
472 	}
473 
474 	/*
475 	 * No need to wake up other CPUs if the thread we just enqueued
476 	 * is bound.
477 	 */
478 	if (bound)
479 		return;
480 
481 
482 	/*
483 	 * See if there's any other halted CPUs. If there are, then
484 	 * select one, and awaken it.
485 	 * It's possible that after we find a CPU, somebody else
486 	 * will awaken it before we get the chance.
487 	 * In that case, look again.
488 	 */
489 	do {
490 		CPUSET_FIND(cp->cp_mach->mc_haltset, cpu_found);
491 		if (cpu_found == CPUSET_NOTINSET)
492 			return;
493 
494 		ASSERT(cpu_found >= 0 && cpu_found < NCPU);
495 		CPUSET_ATOMIC_XDEL(cp->cp_mach->mc_haltset, cpu_found, result);
496 	} while (result < 0);
497 
498 	if (cpu_found != CPU->cpu_id)
499 		poke_cpu(cpu_found);
500 }
501 
502 #ifndef __xpv
503 /*
504  * Idle the present CPU until awoken via touching its monitored line
505  */
506 static void
507 cpu_idle_mwait(void)
508 {
509 	volatile uint32_t	*mcpu_mwait = CPU->cpu_m.mcpu_mwait;
510 	cpu_t			*cpup = CPU;
511 	processorid_t		cpun = cpup->cpu_id;
512 	cpupart_t		*cp = cpup->cpu_part;
513 	int			hset_update = 1;
514 
515 	/*
516 	 * Set our mcpu_mwait here, so we can tell if anyone trys to
517 	 * wake us between now and when we call mwait.  No other cpu will
518 	 * attempt to set our mcpu_mwait until we add ourself to the haltset.
519 	 */
520 	*mcpu_mwait = MWAIT_HALTED;
521 
522 	/*
523 	 * If this CPU is online, and there's multiple CPUs
524 	 * in the system, then we should notate our halting
525 	 * by adding ourselves to the partition's halted CPU
526 	 * bitmap. This allows other CPUs to find/awaken us when
527 	 * work becomes available.
528 	 */
529 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
530 		hset_update = 0;
531 
532 	/*
533 	 * Add ourselves to the partition's halted CPUs bitmask
534 	 * and set our HALTED flag, if necessary.
535 	 *
536 	 * When a thread becomes runnable, it is placed on the queue
537 	 * and then the halted cpuset is checked to determine who
538 	 * (if anyone) should be awoken. We therefore need to first
539 	 * add ourselves to the halted cpuset, and and then check if there
540 	 * is any work available.
541 	 *
542 	 * Note that memory barriers after updating the HALTED flag
543 	 * are not necessary since an atomic operation (updating the bitmap)
544 	 * immediately follows. On x86 the atomic operation acts as a
545 	 * memory barrier for the update of cpu_disp_flags.
546 	 */
547 	if (hset_update) {
548 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
549 		CPUSET_ATOMIC_ADD(cp->cp_mach->mc_haltset, cpun);
550 	}
551 
552 	/*
553 	 * Check to make sure there's really nothing to do.
554 	 * Work destined for this CPU may become available after
555 	 * this check. We'll be notified through the clearing of our
556 	 * bit in the halted CPU bitmask, and a write to our mcpu_mwait.
557 	 *
558 	 * disp_anywork() checks disp_nrunnable, so we do not have to later.
559 	 */
560 	if (disp_anywork()) {
561 		if (hset_update) {
562 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
563 			CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun);
564 		}
565 		return;
566 	}
567 
568 	/*
569 	 * We're on our way to being halted.
570 	 * To avoid a lost wakeup, arm the monitor before checking if another
571 	 * cpu wrote to mcpu_mwait to wake us up.
572 	 */
573 	i86_monitor(mcpu_mwait, 0, 0);
574 	if (*mcpu_mwait == MWAIT_HALTED) {
575 		DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C1);
576 
577 		tlb_going_idle();
578 		i86_mwait(0, 0);
579 		tlb_service();
580 
581 		DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C0);
582 	}
583 
584 	/*
585 	 * We're no longer halted
586 	 */
587 	if (hset_update) {
588 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
589 		CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun);
590 	}
591 }
592 
593 /*
594  * If "cpu" is halted in mwait, then wake it up clearing its halted bit in
595  * advance.  Otherwise, see if other CPUs in the cpu partition are halted and
596  * need to be woken up so that they can steal the thread we placed on this CPU.
597  * This function is only used on MP systems.
598  */
599 static void
600 cpu_wakeup_mwait(cpu_t *cp, int bound)
601 {
602 	cpupart_t	*cpu_part;
603 	uint_t		cpu_found;
604 	int		result;
605 
606 	cpu_part = cp->cpu_part;
607 
608 	/*
609 	 * Clear the halted bit for that CPU since it will be woken up
610 	 * in a moment.
611 	 */
612 	if (CPU_IN_SET(cpu_part->cp_mach->mc_haltset, cp->cpu_id)) {
613 		/*
614 		 * Clear the halted bit for that CPU since it will be
615 		 * poked in a moment.
616 		 */
617 		CPUSET_ATOMIC_DEL(cpu_part->cp_mach->mc_haltset, cp->cpu_id);
618 		/*
619 		 * We may find the current CPU present in the halted cpuset
620 		 * if we're in the context of an interrupt that occurred
621 		 * before we had a chance to clear our bit in cpu_idle().
622 		 * Waking ourself is obviously unnecessary, since if
623 		 * we're here, we're not halted.
624 		 *
625 		 * monitor/mwait wakeup via writing to our cache line is
626 		 * harmless and less expensive than always checking if we
627 		 * are waking ourself which is an uncommon case.
628 		 */
629 		MWAIT_WAKEUP(cp);	/* write to monitored line */
630 		return;
631 	} else {
632 		/*
633 		 * This cpu isn't halted, but it's idle or undergoing a
634 		 * context switch. No need to awaken anyone else.
635 		 */
636 		if (cp->cpu_thread == cp->cpu_idle_thread ||
637 		    cp->cpu_disp_flags & CPU_DISP_DONTSTEAL)
638 			return;
639 	}
640 
641 	/*
642 	 * No need to wake up other CPUs if the thread we just enqueued
643 	 * is bound.
644 	 */
645 	if (bound)
646 		return;
647 
648 
649 	/*
650 	 * See if there's any other halted CPUs. If there are, then
651 	 * select one, and awaken it.
652 	 * It's possible that after we find a CPU, somebody else
653 	 * will awaken it before we get the chance.
654 	 * In that case, look again.
655 	 */
656 	do {
657 		CPUSET_FIND(cpu_part->cp_mach->mc_haltset, cpu_found);
658 		if (cpu_found == CPUSET_NOTINSET)
659 			return;
660 
661 		ASSERT(cpu_found >= 0 && cpu_found < NCPU);
662 		CPUSET_ATOMIC_XDEL(cpu_part->cp_mach->mc_haltset, cpu_found,
663 		    result);
664 	} while (result < 0);
665 
666 	/*
667 	 * Do not check if cpu_found is ourself as monitor/mwait wakeup is
668 	 * cheap.
669 	 */
670 	MWAIT_WAKEUP(cpu[cpu_found]);	/* write to monitored line */
671 }
672 #endif
673 
674 void (*cpu_pause_handler)(volatile char *) = NULL;
675 
676 static int
677 mp_disable_intr(int cpun)
678 {
679 	/*
680 	 * switch to the offline cpu
681 	 */
682 	affinity_set(cpun);
683 	/*
684 	 * raise ipl to just below cross call
685 	 */
686 	splx(XC_MED_PIL-1);
687 	/*
688 	 *	set base spl to prevent the next swtch to idle from
689 	 *	lowering back to ipl 0
690 	 */
691 	CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1));
692 	set_base_spl();
693 	affinity_clear();
694 	return (DDI_SUCCESS);
695 }
696 
697 static void
698 mp_enable_intr(int cpun)
699 {
700 	/*
701 	 * switch to the online cpu
702 	 */
703 	affinity_set(cpun);
704 	/*
705 	 * clear the interrupt active mask
706 	 */
707 	CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1));
708 	set_base_spl();
709 	(void) spl0();
710 	affinity_clear();
711 }
712 
713 static void
714 mach_get_platform(int owner)
715 {
716 	void		**srv_opsp;
717 	void		**clt_opsp;
718 	int		i;
719 	int		total_ops;
720 
721 	/* fix up psm ops */
722 	srv_opsp = (void **)mach_set[0];
723 	clt_opsp = (void **)mach_set[owner];
724 	if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01)
725 		total_ops = sizeof (struct psm_ops_ver01) /
726 		    sizeof (void (*)(void));
727 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1)
728 		/* no psm_notify_func */
729 		total_ops = OFFSETOF(struct psm_ops, psm_notify_func) /
730 		    sizeof (void (*)(void));
731 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2)
732 		/* no psm_timer funcs */
733 		total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) /
734 		    sizeof (void (*)(void));
735 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3)
736 		/* no psm_preshutdown function */
737 		total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) /
738 		    sizeof (void (*)(void));
739 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4)
740 		/* no psm_preshutdown function */
741 		total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) /
742 		    sizeof (void (*)(void));
743 	else
744 		total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void));
745 
746 	/*
747 	 * Save the version of the PSM module, in case we need to
748 	 * bahave differently based on version.
749 	 */
750 	mach_ver[0] = mach_ver[owner];
751 
752 	for (i = 0; i < total_ops; i++)
753 		if (clt_opsp[i] != NULL)
754 			srv_opsp[i] = clt_opsp[i];
755 }
756 
757 static void
758 mach_construct_info()
759 {
760 	struct psm_sw *swp;
761 	int	mach_cnt[PSM_OWN_OVERRIDE+1] = {0};
762 	int	conflict_owner = 0;
763 
764 	if (psmsw->psw_forw == psmsw)
765 		panic("No valid PSM modules found");
766 	mutex_enter(&psmsw_lock);
767 	for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
768 		if (!(swp->psw_flag & PSM_MOD_IDENTIFY))
769 			continue;
770 		mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops;
771 		mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version;
772 		mach_cnt[swp->psw_infop->p_owner]++;
773 	}
774 	mutex_exit(&psmsw_lock);
775 
776 	mach_get_platform(PSM_OWN_SYS_DEFAULT);
777 
778 	/* check to see are there any conflicts */
779 	if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1)
780 		conflict_owner = PSM_OWN_EXCLUSIVE;
781 	if (mach_cnt[PSM_OWN_OVERRIDE] > 1)
782 		conflict_owner = PSM_OWN_OVERRIDE;
783 	if (conflict_owner) {
784 		/* remove all psm modules except uppc */
785 		cmn_err(CE_WARN,
786 		    "Conflicts detected on the following PSM modules:");
787 		mutex_enter(&psmsw_lock);
788 		for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
789 			if (swp->psw_infop->p_owner == conflict_owner)
790 				cmn_err(CE_WARN, "%s ",
791 				    swp->psw_infop->p_mach_idstring);
792 		}
793 		mutex_exit(&psmsw_lock);
794 		cmn_err(CE_WARN,
795 		    "Setting the system back to SINGLE processor mode!");
796 		cmn_err(CE_WARN,
797 		    "Please edit /etc/mach to remove the invalid PSM module.");
798 		return;
799 	}
800 
801 	if (mach_set[PSM_OWN_EXCLUSIVE])
802 		mach_get_platform(PSM_OWN_EXCLUSIVE);
803 
804 	if (mach_set[PSM_OWN_OVERRIDE])
805 		mach_get_platform(PSM_OWN_OVERRIDE);
806 }
807 
808 static void
809 mach_init()
810 {
811 	struct psm_ops  *pops;
812 
813 	mach_construct_info();
814 
815 	pops = mach_set[0];
816 
817 	/* register the interrupt and clock initialization rotuines */
818 	picinitf = mach_picinit;
819 	clkinitf = mach_clkinit;
820 	psm_get_clockirq = pops->psm_get_clockirq;
821 
822 	/* register the interrupt setup code */
823 	slvltovect = mach_softlvl_to_vect;
824 	addspl	= pops->psm_addspl;
825 	delspl	= pops->psm_delspl;
826 
827 	if (pops->psm_translate_irq)
828 		psm_translate_irq = pops->psm_translate_irq;
829 	if (pops->psm_intr_ops)
830 		psm_intr_ops = pops->psm_intr_ops;
831 
832 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4)
833 	/*
834 	 * Time-of-day functionality now handled in TOD modules.
835 	 * (Warn about PSM modules that think that we're going to use
836 	 * their ops vectors.)
837 	 */
838 	if (pops->psm_tod_get)
839 		cmn_err(CE_WARN, "obsolete psm_tod_get op %p",
840 		    (void *)pops->psm_tod_get);
841 
842 	if (pops->psm_tod_set)
843 		cmn_err(CE_WARN, "obsolete psm_tod_set op %p",
844 		    (void *)pops->psm_tod_set);
845 #endif
846 
847 	if (pops->psm_notify_error) {
848 		psm_notify_error = mach_notify_error;
849 		notify_error = pops->psm_notify_error;
850 	}
851 
852 	(*pops->psm_softinit)();
853 
854 	/*
855 	 * Initialize the dispatcher's function hooks
856 	 * to enable CPU halting when idle.
857 	 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle)
858 	 * or idle_cpu_prefer_mwait is not set.
859 	 * Allocate monitor/mwait buffer for cpu0.
860 	 */
861 	if (idle_cpu_use_hlt) {
862 		idle_cpu = cpu_idle;
863 #ifndef __xpv
864 		if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) {
865 			CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);
866 			/*
867 			 * Protect ourself from insane mwait size.
868 			 */
869 			if (CPU->cpu_m.mcpu_mwait == NULL) {
870 #ifdef DEBUG
871 				cmn_err(CE_NOTE, "Using hlt idle.  Cannot "
872 				    "handle cpu 0 mwait size.");
873 #endif
874 				idle_cpu_prefer_mwait = 0;
875 				idle_cpu = cpu_idle;
876 			} else {
877 				idle_cpu = cpu_idle_mwait;
878 			}
879 		} else {
880 			idle_cpu = cpu_idle;
881 		}
882 #endif
883 	}
884 
885 	mach_smpinit();
886 }
887 
888 static void
889 mach_smpinit(void)
890 {
891 	struct psm_ops  *pops;
892 	processorid_t cpu_id;
893 	int cnt;
894 	cpuset_t cpumask;
895 
896 	pops = mach_set[0];
897 	CPUSET_ZERO(cpumask);
898 
899 	cpu_id = -1;
900 	cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
901 	for (cnt = 0; cpu_id != -1; cnt++) {
902 		CPUSET_ADD(cpumask, cpu_id);
903 		cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
904 	}
905 
906 	mp_cpus = cpumask;
907 
908 	/* MP related routines */
909 	ap_mlsetup = pops->psm_post_cpu_start;
910 	send_dirintf = pops->psm_send_ipi;
911 
912 	/* optional MP related routines */
913 	if (pops->psm_shutdown)
914 		psm_shutdownf = pops->psm_shutdown;
915 	if (pops->psm_preshutdown)
916 		psm_preshutdownf = pops->psm_preshutdown;
917 	if (pops->psm_notify_func)
918 		psm_notifyf = pops->psm_notify_func;
919 	if (pops->psm_set_idlecpu)
920 		psm_set_idle_cpuf = pops->psm_set_idlecpu;
921 	if (pops->psm_unset_idlecpu)
922 		psm_unset_idle_cpuf = pops->psm_unset_idlecpu;
923 
924 	psm_clkinit = pops->psm_clkinit;
925 
926 	if (pops->psm_timer_reprogram)
927 		psm_timer_reprogram = pops->psm_timer_reprogram;
928 
929 	if (pops->psm_timer_enable)
930 		psm_timer_enable = pops->psm_timer_enable;
931 
932 	if (pops->psm_timer_disable)
933 		psm_timer_disable = pops->psm_timer_disable;
934 
935 	if (pops->psm_post_cyclic_setup)
936 		psm_post_cyclic_setup = pops->psm_post_cyclic_setup;
937 
938 	if (pops->psm_state)
939 		psm_state = pops->psm_state;
940 
941 	/* check for multiple cpu's */
942 	if (cnt < 2)
943 		return;
944 
945 	/* check for MP platforms */
946 	if (pops->psm_cpu_start == NULL)
947 		return;
948 
949 	/*
950 	 * Set the dispatcher hook to enable cpu "wake up"
951 	 * when a thread becomes runnable.
952 	 */
953 	if (idle_cpu_use_hlt) {
954 		disp_enq_thread = cpu_wakeup;
955 #ifndef __xpv
956 		if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait)
957 			disp_enq_thread = cpu_wakeup_mwait;
958 #endif
959 	}
960 
961 	if (pops->psm_disable_intr)
962 		psm_disable_intr = pops->psm_disable_intr;
963 	if (pops->psm_enable_intr)
964 		psm_enable_intr  = pops->psm_enable_intr;
965 
966 	psm_get_ipivect = pops->psm_get_ipivect;
967 
968 	(void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr",
969 	    (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI),
970 	    (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL);
971 	(void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr",
972 	    (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO),
973 	    (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL);
974 
975 	(void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE);
976 }
977 
978 static void
979 mach_picinit()
980 {
981 	struct psm_ops  *pops;
982 
983 	pops = mach_set[0];
984 
985 	/* register the interrupt handlers */
986 	setlvl = pops->psm_intr_enter;
987 	setlvlx = pops->psm_intr_exit;
988 
989 	/* initialize the interrupt hardware */
990 	(*pops->psm_picinit)();
991 
992 	/* set interrupt mask for current ipl */
993 	setspl = pops->psm_setspl;
994 	cli();
995 	setspl(CPU->cpu_pri);
996 }
997 
998 uint_t	cpu_freq;	/* MHz */
999 uint64_t cpu_freq_hz;	/* measured (in hertz) */
1000 
1001 #define	MEGA_HZ		1000000
1002 
1003 #ifdef __xpv
1004 
1005 int xpv_cpufreq_workaround = 1;
1006 int xpv_cpufreq_verbose = 0;
1007 
1008 #else	/* __xpv */
1009 
1010 static uint64_t
1011 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks)
1012 {
1013 	uint64_t cpu_hz;
1014 
1015 	if ((pit_counter == 0) || (*processor_clks == 0) ||
1016 	    (*processor_clks > (((uint64_t)-1) / PIT_HZ)))
1017 		return (0);
1018 
1019 	cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter;
1020 
1021 	return (cpu_hz);
1022 }
1023 
1024 #endif	/* __xpv */
1025 
1026 static uint64_t
1027 mach_getcpufreq(void)
1028 {
1029 #if defined(__xpv)
1030 	vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time;
1031 	uint64_t cpu_hz;
1032 
1033 	/*
1034 	 * During dom0 bringup, it was noted that on at least one older
1035 	 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul
1036 	 * value that is quite wrong (the 3.06GHz clock was reported
1037 	 * as 4.77GHz)
1038 	 *
1039 	 * The curious thing is, that if you stop the kernel at entry,
1040 	 * breakpoint here and inspect the value with kmdb, the value
1041 	 * is correct - but if you don't stop and simply enable the
1042 	 * printf statement (below), you can see the bad value printed
1043 	 * here.  Almost as if something kmdb did caused the hypervisor to
1044 	 * figure it out correctly.  And, note that the hypervisor
1045 	 * eventually -does- figure it out correctly ... if you look at
1046 	 * the field later in the life of dom0, it is correct.
1047 	 *
1048 	 * For now, on dom0, we employ a slightly cheesy workaround of
1049 	 * using the DOM0_PHYSINFO hypercall.
1050 	 */
1051 	if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) {
1052 		xen_sysctl_t op0, *op = &op0;
1053 
1054 		op->cmd = XEN_SYSCTL_physinfo;
1055 		op->interface_version = XEN_SYSCTL_INTERFACE_VERSION;
1056 		if (HYPERVISOR_sysctl(op) != 0)
1057 			panic("physinfo op refused");
1058 
1059 		cpu_hz = 1000 * (uint64_t)op->u.physinfo.cpu_khz;
1060 	} else {
1061 		cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul;
1062 
1063 		if (vti->tsc_shift < 0)
1064 			cpu_hz <<= -vti->tsc_shift;
1065 		else
1066 			cpu_hz >>= vti->tsc_shift;
1067 	}
1068 
1069 	if (xpv_cpufreq_verbose)
1070 		printf("mach_getcpufreq: system_mul 0x%x, shift %d, "
1071 		    "cpu_hz %" PRId64 "Hz\n",
1072 		    vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz);
1073 
1074 	return (cpu_hz);
1075 #else	/* __xpv */
1076 	uint32_t pit_counter;
1077 	uint64_t processor_clks;
1078 
1079 	if (x86_feature & X86_TSC) {
1080 		/*
1081 		 * We have a TSC. freq_tsc() knows how to measure the number
1082 		 * of clock cycles sampled against the PIT.
1083 		 */
1084 		ulong_t flags = clear_int_flag();
1085 		processor_clks = freq_tsc(&pit_counter);
1086 		restore_int_flag(flags);
1087 		return (mach_calchz(pit_counter, &processor_clks));
1088 	} else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) {
1089 #if defined(__amd64)
1090 		panic("mach_getcpufreq: no TSC!");
1091 #elif defined(__i386)
1092 		/*
1093 		 * We are a Cyrix based on a 6x86 core or an Intel Pentium
1094 		 * for which freq_notsc() knows how to measure the number of
1095 		 * elapsed clock cycles sampled against the PIT
1096 		 */
1097 		ulong_t flags = clear_int_flag();
1098 		processor_clks = freq_notsc(&pit_counter);
1099 		restore_int_flag(flags);
1100 		return (mach_calchz(pit_counter, &processor_clks));
1101 #endif	/* __i386 */
1102 	}
1103 
1104 	/* We do not know how to calculate cpu frequency for this cpu. */
1105 	return (0);
1106 #endif	/* __xpv */
1107 }
1108 
1109 /*
1110  * If the clock speed of a cpu is found to be reported incorrectly, do not add
1111  * to this array, instead improve the accuracy of the algorithm that determines
1112  * the clock speed of the processor or extend the implementation to support the
1113  * vendor as appropriate. This is here only to support adjusting the speed on
1114  * older slower processors that mach_fixcpufreq() would not be able to account
1115  * for otherwise.
1116  */
1117 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 };
1118 
1119 /*
1120  * On fast processors the clock frequency that is measured may be off by
1121  * a few MHz from the value printed on the part. This is a combination of
1122  * the factors that for such fast parts being off by this much is within
1123  * the tolerances for manufacture and because of the difficulties in the
1124  * measurement that can lead to small error. This function uses some
1125  * heuristics in order to tweak the value that was measured to match what
1126  * is most likely printed on the part.
1127  *
1128  * Some examples:
1129  * 	AMD Athlon 1000 mhz measured as 998 mhz
1130  * 	Intel Pentium III Xeon 733 mhz measured as 731 mhz
1131  * 	Intel Pentium IV 1500 mhz measured as 1495mhz
1132  *
1133  * If in the future this function is no longer sufficient to correct
1134  * for the error in the measurement, then the algorithm used to perform
1135  * the measurement will have to be improved in order to increase accuracy
1136  * rather than adding horrible and questionable kludges here.
1137  *
1138  * This is called after the cyclics subsystem because of the potential
1139  * that the heuristics within may give a worse estimate of the clock
1140  * frequency than the value that was measured.
1141  */
1142 static void
1143 mach_fixcpufreq(void)
1144 {
1145 	uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i;
1146 
1147 	freq = (uint32_t)cpu_freq;
1148 
1149 	/*
1150 	 * Find the nearest integer multiple of 200/3 (about 66) MHz to the
1151 	 * measured speed taking into account that the 667 MHz parts were
1152 	 * the first to round-up.
1153 	 */
1154 	mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200);
1155 	near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3);
1156 	delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66);
1157 
1158 	/* Find the nearest integer multiple of 50 MHz to the measured speed */
1159 	mul = (freq + 25) / 50;
1160 	near50 = mul * 50;
1161 	delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50);
1162 
1163 	/* Find the closer of the two */
1164 	if (delta66 < delta50) {
1165 		fixed = near66;
1166 		delta = delta66;
1167 	} else {
1168 		fixed = near50;
1169 		delta = delta50;
1170 	}
1171 
1172 	if (fixed > INT_MAX)
1173 		return;
1174 
1175 	/*
1176 	 * Some older parts have a core clock frequency that is not an
1177 	 * integral multiple of 50 or 66 MHz. Check if one of the old
1178 	 * clock frequencies is closer to the measured value than any
1179 	 * of the integral multiples of 50 an 66, and if so set fixed
1180 	 * and delta appropriately to represent the closest value.
1181 	 */
1182 	i = sizeof (x86_cpu_freq) / sizeof (int);
1183 	while (i > 0) {
1184 		i--;
1185 
1186 		if (x86_cpu_freq[i] <= freq) {
1187 			mul = freq - x86_cpu_freq[i];
1188 
1189 			if (mul < delta) {
1190 				fixed = x86_cpu_freq[i];
1191 				delta = mul;
1192 			}
1193 
1194 			break;
1195 		}
1196 
1197 		mul = x86_cpu_freq[i] - freq;
1198 
1199 		if (mul < delta) {
1200 			fixed = x86_cpu_freq[i];
1201 			delta = mul;
1202 		}
1203 	}
1204 
1205 	/*
1206 	 * Set a reasonable maximum for how much to correct the measured
1207 	 * result by. This check is here to prevent the adjustment made
1208 	 * by this function from being more harm than good. It is entirely
1209 	 * possible that in the future parts will be made that are not
1210 	 * integral multiples of 66 or 50 in clock frequency or that
1211 	 * someone may overclock a part to some odd frequency. If the
1212 	 * measured value is farther from the corrected value than
1213 	 * allowed, then assume the corrected value is in error and use
1214 	 * the measured value.
1215 	 */
1216 	if (6 < delta)
1217 		return;
1218 
1219 	cpu_freq = (int)fixed;
1220 }
1221 
1222 
1223 static int
1224 machhztomhz(uint64_t cpu_freq_hz)
1225 {
1226 	uint64_t cpu_mhz;
1227 
1228 	/* Round to nearest MHZ */
1229 	cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ;
1230 
1231 	if (cpu_mhz > INT_MAX)
1232 		return (0);
1233 
1234 	return ((int)cpu_mhz);
1235 
1236 }
1237 
1238 
1239 static int
1240 mach_clkinit(int preferred_mode, int *set_mode)
1241 {
1242 	struct psm_ops  *pops;
1243 	int resolution;
1244 
1245 	pops = mach_set[0];
1246 
1247 	cpu_freq_hz = mach_getcpufreq();
1248 
1249 	cpu_freq = machhztomhz(cpu_freq_hz);
1250 
1251 	if (!(x86_feature & X86_TSC) || (cpu_freq == 0))
1252 		tsc_gethrtime_enable = 0;
1253 
1254 #ifndef __xpv
1255 	if (tsc_gethrtime_enable) {
1256 		tsc_hrtimeinit(cpu_freq_hz);
1257 	} else
1258 #endif
1259 	{
1260 		if (pops->psm_hrtimeinit)
1261 			(*pops->psm_hrtimeinit)();
1262 		gethrtimef = pops->psm_gethrtime;
1263 		gethrtimeunscaledf = gethrtimef;
1264 		/* scalehrtimef will remain dummy */
1265 	}
1266 
1267 	mach_fixcpufreq();
1268 
1269 	if (mach_ver[0] >= PSM_INFO_VER01_3) {
1270 		if (preferred_mode == TIMER_ONESHOT) {
1271 
1272 			resolution = (*pops->psm_clkinit)(0);
1273 			if (resolution != 0)  {
1274 				*set_mode = TIMER_ONESHOT;
1275 				return (resolution);
1276 			}
1277 		}
1278 
1279 		/*
1280 		 * either periodic mode was requested or could not set to
1281 		 * one-shot mode
1282 		 */
1283 		resolution = (*pops->psm_clkinit)(hz);
1284 		/*
1285 		 * psm should be able to do periodic, so we do not check
1286 		 * for return value of psm_clkinit here.
1287 		 */
1288 		*set_mode = TIMER_PERIODIC;
1289 		return (resolution);
1290 	} else {
1291 		/*
1292 		 * PSMI interface prior to PSMI_3 does not define a return
1293 		 * value for psm_clkinit, so the return value is ignored.
1294 		 */
1295 		(void) (*pops->psm_clkinit)(hz);
1296 		*set_mode = TIMER_PERIODIC;
1297 		return (nsec_per_tick);
1298 	}
1299 }
1300 
1301 
1302 /*ARGSUSED*/
1303 static int
1304 mach_softlvl_to_vect(int ipl)
1305 {
1306 	setsoftint = av_set_softint_pending;
1307 	kdisetsoftint = kdi_av_set_softint_pending;
1308 
1309 	return (PSM_SV_SOFTWARE);
1310 }
1311 
1312 #ifdef DEBUG
1313 /*
1314  * This is here to allow us to simulate cpus that refuse to start.
1315  */
1316 cpuset_t cpufailset;
1317 #endif
1318 
1319 int
1320 mach_cpu_start(struct cpu *cp, void *ctx)
1321 {
1322 	struct psm_ops *pops = mach_set[0];
1323 	processorid_t id = cp->cpu_id;
1324 
1325 #ifdef DEBUG
1326 	if (CPU_IN_SET(cpufailset, id))
1327 		return (0);
1328 #endif
1329 	return ((*pops->psm_cpu_start)(id, ctx));
1330 }
1331 
1332 int
1333 mach_cpuid_start(processorid_t id, void *ctx)
1334 {
1335 	struct psm_ops *pops = mach_set[0];
1336 
1337 #ifdef DEBUG
1338 	if (CPU_IN_SET(cpufailset, id))
1339 		return (0);
1340 #endif
1341 	return ((*pops->psm_cpu_start)(id, ctx));
1342 }
1343 
1344 /*ARGSUSED*/
1345 static int
1346 mach_translate_irq(dev_info_t *dip, int irqno)
1347 {
1348 	return (irqno);	/* default to NO translation */
1349 }
1350 
1351 static void
1352 mach_notify_error(int level, char *errmsg)
1353 {
1354 	/*
1355 	 * SL_FATAL is pass in once panicstr is set, deliver it
1356 	 * as CE_PANIC.  Also, translate SL_ codes back to CE_
1357 	 * codes for the psmi handler
1358 	 */
1359 	if (level & SL_FATAL)
1360 		(*notify_error)(CE_PANIC, errmsg);
1361 	else if (level & SL_WARN)
1362 		(*notify_error)(CE_WARN, errmsg);
1363 	else if (level & SL_NOTE)
1364 		(*notify_error)(CE_NOTE, errmsg);
1365 	else if (level & SL_CONSOLE)
1366 		(*notify_error)(CE_CONT, errmsg);
1367 }
1368 
1369 /*
1370  * It provides the default basic intr_ops interface for the new DDI
1371  * interrupt framework if the PSM doesn't have one.
1372  *
1373  * Input:
1374  * dip     - pointer to the dev_info structure of the requested device
1375  * hdlp    - pointer to the internal interrupt handle structure for the
1376  *	     requested interrupt
1377  * intr_op - opcode for this call
1378  * result  - pointer to the integer that will hold the result to be
1379  *	     passed back if return value is PSM_SUCCESS
1380  *
1381  * Output:
1382  * return value is either PSM_SUCCESS or PSM_FAILURE
1383  */
1384 static int
1385 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1386     psm_intr_op_t intr_op, int *result)
1387 {
1388 	struct intrspec *ispec;
1389 
1390 	switch (intr_op) {
1391 	case PSM_INTR_OP_CHECK_MSI:
1392 		*result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1393 		    DDI_INTR_TYPE_MSIX);
1394 		break;
1395 	case PSM_INTR_OP_ALLOC_VECTORS:
1396 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1397 			*result = 1;
1398 		else
1399 			*result = 0;
1400 		break;
1401 	case PSM_INTR_OP_FREE_VECTORS:
1402 		break;
1403 	case PSM_INTR_OP_NAVAIL_VECTORS:
1404 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1405 			*result = 1;
1406 		else
1407 			*result = 0;
1408 		break;
1409 	case PSM_INTR_OP_XLATE_VECTOR:
1410 		ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1411 		*result = psm_translate_irq(dip, ispec->intrspec_vec);
1412 		break;
1413 	case PSM_INTR_OP_GET_CAP:
1414 		*result = 0;
1415 		break;
1416 	case PSM_INTR_OP_GET_PENDING:
1417 	case PSM_INTR_OP_CLEAR_MASK:
1418 	case PSM_INTR_OP_SET_MASK:
1419 	case PSM_INTR_OP_GET_SHARED:
1420 	case PSM_INTR_OP_SET_PRI:
1421 	case PSM_INTR_OP_SET_CAP:
1422 	case PSM_INTR_OP_SET_CPU:
1423 	case PSM_INTR_OP_GET_INTR:
1424 	default:
1425 		return (PSM_FAILURE);
1426 	}
1427 	return (PSM_SUCCESS);
1428 }
1429 /*
1430  * Return 1 if CMT load balancing policies should be
1431  * implemented across instances of the specified hardware
1432  * sharing relationship.
1433  */
1434 int
1435 pg_cmt_load_bal_hw(pghw_type_t hw)
1436 {
1437 	if (hw == PGHW_IPIPE ||
1438 	    hw == PGHW_FPU ||
1439 	    hw == PGHW_CHIP)
1440 		return (1);
1441 	else
1442 		return (0);
1443 }
1444 /*
1445  * Return 1 if thread affinity polices should be implemented
1446  * for instances of the specifed hardware sharing relationship.
1447  */
1448 int
1449 pg_cmt_affinity_hw(pghw_type_t hw)
1450 {
1451 	if (hw == PGHW_CACHE)
1452 		return (1);
1453 	else
1454 		return (0);
1455 }
1456