xref: /illumos-gate/usr/src/uts/i86pc/os/mp_machdep.c (revision c9eab9d4e096bb9b983e9b007577edfa73c32eff)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #define	PSMI_1_6
27 #include <sys/smp_impldefs.h>
28 #include <sys/psm.h>
29 #include <sys/psm_modctl.h>
30 #include <sys/pit.h>
31 #include <sys/cmn_err.h>
32 #include <sys/strlog.h>
33 #include <sys/clock.h>
34 #include <sys/debug.h>
35 #include <sys/rtc.h>
36 #include <sys/x86_archext.h>
37 #include <sys/cpupart.h>
38 #include <sys/cpuvar.h>
39 #include <sys/cmt.h>
40 #include <sys/cpu.h>
41 #include <sys/disp.h>
42 #include <sys/archsystm.h>
43 #include <sys/machsystm.h>
44 #include <sys/sysmacros.h>
45 #include <sys/memlist.h>
46 #include <sys/param.h>
47 #include <sys/promif.h>
48 #if defined(__xpv)
49 #include <sys/hypervisor.h>
50 #endif
51 #include <sys/mach_intr.h>
52 #include <vm/hat_i86.h>
53 #include <sys/kdi_machimpl.h>
54 #include <sys/sdt.h>
55 
56 #define	OFFSETOF(s, m)		(size_t)(&(((s *)0)->m))
57 
58 /*
59  *	Local function prototypes
60  */
61 static int mp_disable_intr(processorid_t cpun);
62 static void mp_enable_intr(processorid_t cpun);
63 static void mach_init();
64 static void mach_picinit();
65 static int machhztomhz(uint64_t cpu_freq_hz);
66 static uint64_t mach_getcpufreq(void);
67 static void mach_fixcpufreq(void);
68 static int mach_clkinit(int, int *);
69 static void mach_smpinit(void);
70 static int mach_softlvl_to_vect(int ipl);
71 static void mach_get_platform(int owner);
72 static void mach_construct_info();
73 static int mach_translate_irq(dev_info_t *dip, int irqno);
74 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
75     psm_intr_op_t, int *);
76 static void mach_notify_error(int level, char *errmsg);
77 static hrtime_t dummy_hrtime(void);
78 static void dummy_scalehrtime(hrtime_t *);
79 static void cpu_idle(void);
80 static void cpu_wakeup(cpu_t *, int);
81 #ifndef __xpv
82 static void cpu_idle_mwait(void);
83 static void cpu_wakeup_mwait(cpu_t *, int);
84 #endif
85 /*
86  *	External reference functions
87  */
88 extern void return_instr();
89 extern uint64_t freq_tsc(uint32_t *);
90 #if defined(__i386)
91 extern uint64_t freq_notsc(uint32_t *);
92 #endif
93 extern void pc_gethrestime(timestruc_t *);
94 extern int cpuid_get_coreid(cpu_t *);
95 extern int cpuid_get_chipid(cpu_t *);
96 
97 /*
98  *	PSM functions initialization
99  */
100 void (*psm_shutdownf)(int, int)	= (void (*)(int, int))return_instr;
101 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr;
102 void (*psm_notifyf)(int)	= (void (*)(int))return_instr;
103 void (*psm_set_idle_cpuf)(int)	= (void (*)(int))return_instr;
104 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr;
105 void (*psminitf)()		= mach_init;
106 void (*picinitf)() 		= return_instr;
107 int (*clkinitf)(int, int *) 	= (int (*)(int, int *))return_instr;
108 int (*ap_mlsetup)() 		= (int (*)(void))return_instr;
109 void (*send_dirintf)() 		= return_instr;
110 void (*setspl)(int)		= (void (*)(int))return_instr;
111 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
112 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
113 void (*kdisetsoftint)(int, struct av_softinfo *)=
114 	(void (*)(int, struct av_softinfo *))return_instr;
115 void (*setsoftint)(int, struct av_softinfo *)=
116 	(void (*)(int, struct av_softinfo *))return_instr;
117 int (*slvltovect)(int)		= (int (*)(int))return_instr;
118 int (*setlvl)(int, int *)	= (int (*)(int, int *))return_instr;
119 void (*setlvlx)(int, int)	= (void (*)(int, int))return_instr;
120 int (*psm_disable_intr)(int)	= mp_disable_intr;
121 void (*psm_enable_intr)(int)	= mp_enable_intr;
122 hrtime_t (*gethrtimef)(void)	= dummy_hrtime;
123 hrtime_t (*gethrtimeunscaledf)(void)	= dummy_hrtime;
124 void (*scalehrtimef)(hrtime_t *)	= dummy_scalehrtime;
125 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq;
126 void (*gethrestimef)(timestruc_t *) = pc_gethrestime;
127 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL;
128 int (*psm_get_clockirq)(int) = NULL;
129 int (*psm_get_ipivect)(int, int) = NULL;
130 
131 int (*psm_clkinit)(int) = NULL;
132 void (*psm_timer_reprogram)(hrtime_t) = NULL;
133 void (*psm_timer_enable)(void) = NULL;
134 void (*psm_timer_disable)(void) = NULL;
135 void (*psm_post_cyclic_setup)(void *arg) = NULL;
136 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t,
137     int *) = mach_intr_ops;
138 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *))
139     return_instr;
140 
141 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr;
142 void (*hrtime_tick)(void)	= return_instr;
143 
144 /*
145  * True if the generic TSC code is our source of hrtime, rather than whatever
146  * the PSM can provide.
147  */
148 #ifdef __xpv
149 int tsc_gethrtime_enable = 0;
150 #else
151 int tsc_gethrtime_enable = 1;
152 #endif
153 int tsc_gethrtime_initted = 0;
154 
155 /*
156  * True if the hrtime implementation is "hires"; namely, better than microdata.
157  */
158 int gethrtime_hires = 0;
159 
160 /*
161  * Local Static Data
162  */
163 static struct psm_ops mach_ops;
164 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL};
165 static ushort_t mach_ver[4] = {0, 0, 0, 0};
166 
167 /*
168  * virtualization support for psm
169  */
170 void *psm_vt_ops = NULL;
171 /*
172  * If non-zero, idle cpus will become "halted" when there's
173  * no work to do.
174  */
175 int	idle_cpu_use_hlt = 1;
176 
177 #ifndef __xpv
178 /*
179  * If non-zero, idle cpus will use mwait if available to halt instead of hlt.
180  */
181 int	idle_cpu_prefer_mwait = 1;
182 /*
183  * Set to 0 to avoid MONITOR+CLFLUSH assertion.
184  */
185 int	idle_cpu_assert_cflush_monitor = 1;
186 
187 #endif
188 
189 /*ARGSUSED*/
190 int
191 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw)
192 {
193 	switch (hw) {
194 	case PGHW_IPIPE:
195 		if (x86_feature & (X86_HTT)) {
196 			/*
197 			 * Hyper-threading is SMT
198 			 */
199 			return (1);
200 		} else {
201 			return (0);
202 		}
203 	case PGHW_CHIP:
204 		if (x86_feature & (X86_CMP|X86_HTT))
205 			return (1);
206 		else
207 			return (0);
208 	case PGHW_CACHE:
209 		if (cpuid_get_ncpu_sharing_last_cache(cp) > 1)
210 			return (1);
211 		else
212 			return (0);
213 	default:
214 		return (0);
215 	}
216 }
217 
218 /*
219  * Compare two CPUs and see if they have a pghw_type_t sharing relationship
220  * If pghw_type_t is an unsupported hardware type, then return -1
221  */
222 int
223 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw)
224 {
225 	id_t pgp_a, pgp_b;
226 
227 	pgp_a = pg_plat_hw_instance_id(cpu_a, hw);
228 	pgp_b = pg_plat_hw_instance_id(cpu_b, hw);
229 
230 	if (pgp_a == -1 || pgp_b == -1)
231 		return (-1);
232 
233 	return (pgp_a == pgp_b);
234 }
235 
236 /*
237  * Return a physical instance identifier for known hardware sharing
238  * relationships
239  */
240 id_t
241 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw)
242 {
243 	switch (hw) {
244 	case PGHW_IPIPE:
245 		return (cpuid_get_coreid(cpu));
246 	case PGHW_CACHE:
247 		return (cpuid_get_last_lvl_cacheid(cpu));
248 	case PGHW_CHIP:
249 		return (cpuid_get_chipid(cpu));
250 	default:
251 		return (-1);
252 	}
253 }
254 
255 int
256 pg_plat_hw_level(pghw_type_t hw)
257 {
258 	int i;
259 	static pghw_type_t hw_hier[] = {
260 		PGHW_IPIPE,
261 		PGHW_CACHE,
262 		PGHW_CHIP,
263 		PGHW_NUM_COMPONENTS
264 	};
265 
266 	for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) {
267 		if (hw_hier[i] == hw)
268 			return (i);
269 	}
270 	return (-1);
271 }
272 
273 /*
274  * Return 1 if CMT load balancing policies should be
275  * implemented across instances of the specified hardware
276  * sharing relationship.
277  */
278 int
279 pg_plat_cmt_load_bal_hw(pghw_type_t hw)
280 {
281 	if (hw == PGHW_IPIPE ||
282 	    hw == PGHW_FPU ||
283 	    hw == PGHW_CHIP ||
284 	    hw == PGHW_CACHE)
285 		return (1);
286 	else
287 		return (0);
288 }
289 
290 
291 /*
292  * Return 1 if thread affinity polices should be implemented
293  * for instances of the specifed hardware sharing relationship.
294  */
295 int
296 pg_plat_cmt_affinity_hw(pghw_type_t hw)
297 {
298 	if (hw == PGHW_CACHE)
299 		return (1);
300 	else
301 		return (0);
302 }
303 
304 id_t
305 pg_plat_get_core_id(cpu_t *cpu)
306 {
307 	return ((id_t)cpuid_get_coreid(cpu));
308 }
309 
310 void
311 cmp_set_nosteal_interval(void)
312 {
313 	/* Set the nosteal interval (used by disp_getbest()) to 100us */
314 	nosteal_nsec = 100000UL;
315 }
316 
317 /*
318  * Routine to ensure initial callers to hrtime gets 0 as return
319  */
320 static hrtime_t
321 dummy_hrtime(void)
322 {
323 	return (0);
324 }
325 
326 /* ARGSUSED */
327 static void
328 dummy_scalehrtime(hrtime_t *ticks)
329 {}
330 
331 /*
332  * Idle the present CPU until awoken via an interrupt
333  */
334 static void
335 cpu_idle(void)
336 {
337 	cpu_t		*cpup = CPU;
338 	processorid_t	cpu_sid = cpup->cpu_seqid;
339 	cpupart_t	*cp = cpup->cpu_part;
340 	int		hset_update = 1;
341 
342 	/*
343 	 * If this CPU is online, and there's multiple CPUs
344 	 * in the system, then we should notate our halting
345 	 * by adding ourselves to the partition's halted CPU
346 	 * bitmap. This allows other CPUs to find/awaken us when
347 	 * work becomes available.
348 	 */
349 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
350 		hset_update = 0;
351 
352 	/*
353 	 * Add ourselves to the partition's halted CPUs bitmap
354 	 * and set our HALTED flag, if necessary.
355 	 *
356 	 * When a thread becomes runnable, it is placed on the queue
357 	 * and then the halted CPU bitmap is checked to determine who
358 	 * (if anyone) should be awoken. We therefore need to first
359 	 * add ourselves to the bitmap, and and then check if there
360 	 * is any work available. The order is important to prevent a race
361 	 * that can lead to work languishing on a run queue somewhere while
362 	 * this CPU remains halted.
363 	 *
364 	 * Either the producing CPU will see we're halted and will awaken us,
365 	 * or this CPU will see the work available in disp_anywork().
366 	 *
367 	 * Note that memory barriers after updating the HALTED flag
368 	 * are not necessary since an atomic operation (updating the bitset)
369 	 * immediately follows. On x86 the atomic operation acts as a
370 	 * memory barrier for the update of cpu_disp_flags.
371 	 */
372 	if (hset_update) {
373 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
374 		bitset_atomic_add(&cp->cp_haltset, cpu_sid);
375 	}
376 
377 	/*
378 	 * Check to make sure there's really nothing to do.
379 	 * Work destined for this CPU may become available after
380 	 * this check. We'll be notified through the clearing of our
381 	 * bit in the halted CPU bitmap, and a poke.
382 	 */
383 	if (disp_anywork()) {
384 		if (hset_update) {
385 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
386 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
387 		}
388 		return;
389 	}
390 
391 	/*
392 	 * We're on our way to being halted.
393 	 *
394 	 * Disable interrupts now, so that we'll awaken immediately
395 	 * after halting if someone tries to poke us between now and
396 	 * the time we actually halt.
397 	 *
398 	 * We check for the presence of our bit after disabling interrupts.
399 	 * If it's cleared, we'll return. If the bit is cleared after
400 	 * we check then the poke will pop us out of the halted state.
401 	 *
402 	 * This means that the ordering of the poke and the clearing
403 	 * of the bit by cpu_wakeup is important.
404 	 * cpu_wakeup() must clear, then poke.
405 	 * cpu_idle() must disable interrupts, then check for the bit.
406 	 */
407 	cli();
408 
409 	if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) {
410 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
411 		sti();
412 		return;
413 	}
414 
415 	/*
416 	 * The check for anything locally runnable is here for performance
417 	 * and isn't needed for correctness. disp_nrunnable ought to be
418 	 * in our cache still, so it's inexpensive to check, and if there
419 	 * is anything runnable we won't have to wait for the poke.
420 	 */
421 	if (cpup->cpu_disp->disp_nrunnable != 0) {
422 		if (hset_update) {
423 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
424 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
425 		}
426 		sti();
427 		return;
428 	}
429 
430 	DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C1);
431 
432 	mach_cpu_idle();
433 
434 	DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C0);
435 
436 	/*
437 	 * We're no longer halted
438 	 */
439 	if (hset_update) {
440 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
441 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
442 	}
443 }
444 
445 
446 /*
447  * If "cpu" is halted, then wake it up clearing its halted bit in advance.
448  * Otherwise, see if other CPUs in the cpu partition are halted and need to
449  * be woken up so that they can steal the thread we placed on this CPU.
450  * This function is only used on MP systems.
451  */
452 static void
453 cpu_wakeup(cpu_t *cpu, int bound)
454 {
455 	uint_t		cpu_found;
456 	processorid_t	cpu_sid;
457 	cpupart_t	*cp;
458 
459 	cp = cpu->cpu_part;
460 	cpu_sid = cpu->cpu_seqid;
461 	if (bitset_in_set(&cp->cp_haltset, cpu_sid)) {
462 		/*
463 		 * Clear the halted bit for that CPU since it will be
464 		 * poked in a moment.
465 		 */
466 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
467 		/*
468 		 * We may find the current CPU present in the halted cpuset
469 		 * if we're in the context of an interrupt that occurred
470 		 * before we had a chance to clear our bit in cpu_idle().
471 		 * Poking ourself is obviously unnecessary, since if
472 		 * we're here, we're not halted.
473 		 */
474 		if (cpu != CPU)
475 			poke_cpu(cpu->cpu_id);
476 		return;
477 	} else {
478 		/*
479 		 * This cpu isn't halted, but it's idle or undergoing a
480 		 * context switch. No need to awaken anyone else.
481 		 */
482 		if (cpu->cpu_thread == cpu->cpu_idle_thread ||
483 		    cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL)
484 			return;
485 	}
486 
487 	/*
488 	 * No need to wake up other CPUs if this is for a bound thread.
489 	 */
490 	if (bound)
491 		return;
492 
493 	/*
494 	 * The CPU specified for wakeup isn't currently halted, so check
495 	 * to see if there are any other halted CPUs in the partition,
496 	 * and if there are then awaken one.
497 	 */
498 	do {
499 		cpu_found = bitset_find(&cp->cp_haltset);
500 		if (cpu_found == (uint_t)-1)
501 			return;
502 	} while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0);
503 
504 	if (cpu_found != CPU->cpu_seqid) {
505 		poke_cpu(cpu_seq[cpu_found]->cpu_id);
506 	}
507 }
508 
509 #ifndef __xpv
510 /*
511  * Idle the present CPU until awoken via touching its monitored line
512  */
513 static void
514 cpu_idle_mwait(void)
515 {
516 	volatile uint32_t	*mcpu_mwait = CPU->cpu_m.mcpu_mwait;
517 	cpu_t			*cpup = CPU;
518 	processorid_t		cpu_sid = cpup->cpu_seqid;
519 	cpupart_t		*cp = cpup->cpu_part;
520 	int			hset_update = 1;
521 
522 	/*
523 	 * Set our mcpu_mwait here, so we can tell if anyone trys to
524 	 * wake us between now and when we call mwait.  No other cpu will
525 	 * attempt to set our mcpu_mwait until we add ourself to the halted
526 	 * CPU bitmap.
527 	 */
528 	*mcpu_mwait = MWAIT_HALTED;
529 
530 	/*
531 	 * If this CPU is online, and there's multiple CPUs
532 	 * in the system, then we should notate our halting
533 	 * by adding ourselves to the partition's halted CPU
534 	 * bitmap. This allows other CPUs to find/awaken us when
535 	 * work becomes available.
536 	 */
537 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
538 		hset_update = 0;
539 
540 	/*
541 	 * Add ourselves to the partition's halted CPUs bitmap
542 	 * and set our HALTED flag, if necessary.
543 	 *
544 	 * When a thread becomes runnable, it is placed on the queue
545 	 * and then the halted CPU bitmap is checked to determine who
546 	 * (if anyone) should be awoken. We therefore need to first
547 	 * add ourselves to the bitmap, and and then check if there
548 	 * is any work available.
549 	 *
550 	 * Note that memory barriers after updating the HALTED flag
551 	 * are not necessary since an atomic operation (updating the bitmap)
552 	 * immediately follows. On x86 the atomic operation acts as a
553 	 * memory barrier for the update of cpu_disp_flags.
554 	 */
555 	if (hset_update) {
556 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
557 		bitset_atomic_add(&cp->cp_haltset, cpu_sid);
558 	}
559 
560 	/*
561 	 * Check to make sure there's really nothing to do.
562 	 * Work destined for this CPU may become available after
563 	 * this check. We'll be notified through the clearing of our
564 	 * bit in the halted CPU bitmap, and a write to our mcpu_mwait.
565 	 *
566 	 * disp_anywork() checks disp_nrunnable, so we do not have to later.
567 	 */
568 	if (disp_anywork()) {
569 		if (hset_update) {
570 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
571 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
572 		}
573 		return;
574 	}
575 
576 	/*
577 	 * We're on our way to being halted.
578 	 * To avoid a lost wakeup, arm the monitor before checking if another
579 	 * cpu wrote to mcpu_mwait to wake us up.
580 	 */
581 	i86_monitor(mcpu_mwait, 0, 0);
582 	if (*mcpu_mwait == MWAIT_HALTED) {
583 		DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C1);
584 
585 		tlb_going_idle();
586 		i86_mwait(0, 0);
587 		tlb_service();
588 
589 		DTRACE_PROBE1(idle__state__transition, uint_t, IDLE_STATE_C0);
590 	}
591 
592 	/*
593 	 * We're no longer halted
594 	 */
595 	if (hset_update) {
596 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
597 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
598 	}
599 }
600 
601 /*
602  * If "cpu" is halted in mwait, then wake it up clearing its halted bit in
603  * advance.  Otherwise, see if other CPUs in the cpu partition are halted and
604  * need to be woken up so that they can steal the thread we placed on this CPU.
605  * This function is only used on MP systems.
606  */
607 static void
608 cpu_wakeup_mwait(cpu_t *cp, int bound)
609 {
610 	cpupart_t	*cpu_part;
611 	uint_t		cpu_found;
612 	processorid_t	cpu_sid;
613 
614 	cpu_part = cp->cpu_part;
615 	cpu_sid = cp->cpu_seqid;
616 
617 	/*
618 	 * Clear the halted bit for that CPU since it will be woken up
619 	 * in a moment.
620 	 */
621 	if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) {
622 		/*
623 		 * Clear the halted bit for that CPU since it will be
624 		 * poked in a moment.
625 		 */
626 		bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid);
627 		/*
628 		 * We may find the current CPU present in the halted cpuset
629 		 * if we're in the context of an interrupt that occurred
630 		 * before we had a chance to clear our bit in cpu_idle().
631 		 * Waking ourself is obviously unnecessary, since if
632 		 * we're here, we're not halted.
633 		 *
634 		 * monitor/mwait wakeup via writing to our cache line is
635 		 * harmless and less expensive than always checking if we
636 		 * are waking ourself which is an uncommon case.
637 		 */
638 		MWAIT_WAKEUP(cp);	/* write to monitored line */
639 		return;
640 	} else {
641 		/*
642 		 * This cpu isn't halted, but it's idle or undergoing a
643 		 * context switch. No need to awaken anyone else.
644 		 */
645 		if (cp->cpu_thread == cp->cpu_idle_thread ||
646 		    cp->cpu_disp_flags & CPU_DISP_DONTSTEAL)
647 			return;
648 	}
649 
650 	/*
651 	 * No need to wake up other CPUs if the thread we just enqueued
652 	 * is bound.
653 	 */
654 	if (bound || ncpus == 1)
655 		return;
656 
657 	/*
658 	 * See if there's any other halted CPUs. If there are, then
659 	 * select one, and awaken it.
660 	 * It's possible that after we find a CPU, somebody else
661 	 * will awaken it before we get the chance.
662 	 * In that case, look again.
663 	 */
664 	do {
665 		cpu_found = bitset_find(&cpu_part->cp_haltset);
666 		if (cpu_found == (uint_t)-1)
667 			return;
668 	} while (bitset_atomic_test_and_del(&cpu_part->cp_haltset,
669 	    cpu_found) < 0);
670 
671 	/*
672 	 * Do not check if cpu_found is ourself as monitor/mwait
673 	 * wakeup is cheap.
674 	 */
675 	MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */
676 }
677 
678 #endif
679 
680 void (*cpu_pause_handler)(volatile char *) = NULL;
681 
682 static int
683 mp_disable_intr(int cpun)
684 {
685 	/*
686 	 * switch to the offline cpu
687 	 */
688 	affinity_set(cpun);
689 	/*
690 	 * raise ipl to just below cross call
691 	 */
692 	splx(XC_MED_PIL-1);
693 	/*
694 	 *	set base spl to prevent the next swtch to idle from
695 	 *	lowering back to ipl 0
696 	 */
697 	CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1));
698 	set_base_spl();
699 	affinity_clear();
700 	return (DDI_SUCCESS);
701 }
702 
703 static void
704 mp_enable_intr(int cpun)
705 {
706 	/*
707 	 * switch to the online cpu
708 	 */
709 	affinity_set(cpun);
710 	/*
711 	 * clear the interrupt active mask
712 	 */
713 	CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1));
714 	set_base_spl();
715 	(void) spl0();
716 	affinity_clear();
717 }
718 
719 static void
720 mach_get_platform(int owner)
721 {
722 	void		**srv_opsp;
723 	void		**clt_opsp;
724 	int		i;
725 	int		total_ops;
726 
727 	/* fix up psm ops */
728 	srv_opsp = (void **)mach_set[0];
729 	clt_opsp = (void **)mach_set[owner];
730 	if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01)
731 		total_ops = sizeof (struct psm_ops_ver01) /
732 		    sizeof (void (*)(void));
733 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1)
734 		/* no psm_notify_func */
735 		total_ops = OFFSETOF(struct psm_ops, psm_notify_func) /
736 		    sizeof (void (*)(void));
737 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2)
738 		/* no psm_timer funcs */
739 		total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) /
740 		    sizeof (void (*)(void));
741 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3)
742 		/* no psm_preshutdown function */
743 		total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) /
744 		    sizeof (void (*)(void));
745 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4)
746 		/* no psm_preshutdown function */
747 		total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) /
748 		    sizeof (void (*)(void));
749 	else
750 		total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void));
751 
752 	/*
753 	 * Save the version of the PSM module, in case we need to
754 	 * bahave differently based on version.
755 	 */
756 	mach_ver[0] = mach_ver[owner];
757 
758 	for (i = 0; i < total_ops; i++)
759 		if (clt_opsp[i] != NULL)
760 			srv_opsp[i] = clt_opsp[i];
761 }
762 
763 static void
764 mach_construct_info()
765 {
766 	struct psm_sw *swp;
767 	int	mach_cnt[PSM_OWN_OVERRIDE+1] = {0};
768 	int	conflict_owner = 0;
769 
770 	if (psmsw->psw_forw == psmsw)
771 		panic("No valid PSM modules found");
772 	mutex_enter(&psmsw_lock);
773 	for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
774 		if (!(swp->psw_flag & PSM_MOD_IDENTIFY))
775 			continue;
776 		mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops;
777 		mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version;
778 		mach_cnt[swp->psw_infop->p_owner]++;
779 	}
780 	mutex_exit(&psmsw_lock);
781 
782 	mach_get_platform(PSM_OWN_SYS_DEFAULT);
783 
784 	/* check to see are there any conflicts */
785 	if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1)
786 		conflict_owner = PSM_OWN_EXCLUSIVE;
787 	if (mach_cnt[PSM_OWN_OVERRIDE] > 1)
788 		conflict_owner = PSM_OWN_OVERRIDE;
789 	if (conflict_owner) {
790 		/* remove all psm modules except uppc */
791 		cmn_err(CE_WARN,
792 		    "Conflicts detected on the following PSM modules:");
793 		mutex_enter(&psmsw_lock);
794 		for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
795 			if (swp->psw_infop->p_owner == conflict_owner)
796 				cmn_err(CE_WARN, "%s ",
797 				    swp->psw_infop->p_mach_idstring);
798 		}
799 		mutex_exit(&psmsw_lock);
800 		cmn_err(CE_WARN,
801 		    "Setting the system back to SINGLE processor mode!");
802 		cmn_err(CE_WARN,
803 		    "Please edit /etc/mach to remove the invalid PSM module.");
804 		return;
805 	}
806 
807 	if (mach_set[PSM_OWN_EXCLUSIVE])
808 		mach_get_platform(PSM_OWN_EXCLUSIVE);
809 
810 	if (mach_set[PSM_OWN_OVERRIDE])
811 		mach_get_platform(PSM_OWN_OVERRIDE);
812 }
813 
814 static void
815 mach_init()
816 {
817 	struct psm_ops  *pops;
818 
819 	mach_construct_info();
820 
821 	pops = mach_set[0];
822 
823 	/* register the interrupt and clock initialization rotuines */
824 	picinitf = mach_picinit;
825 	clkinitf = mach_clkinit;
826 	psm_get_clockirq = pops->psm_get_clockirq;
827 
828 	/* register the interrupt setup code */
829 	slvltovect = mach_softlvl_to_vect;
830 	addspl	= pops->psm_addspl;
831 	delspl	= pops->psm_delspl;
832 
833 	if (pops->psm_translate_irq)
834 		psm_translate_irq = pops->psm_translate_irq;
835 	if (pops->psm_intr_ops)
836 		psm_intr_ops = pops->psm_intr_ops;
837 
838 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4)
839 	/*
840 	 * Time-of-day functionality now handled in TOD modules.
841 	 * (Warn about PSM modules that think that we're going to use
842 	 * their ops vectors.)
843 	 */
844 	if (pops->psm_tod_get)
845 		cmn_err(CE_WARN, "obsolete psm_tod_get op %p",
846 		    (void *)pops->psm_tod_get);
847 
848 	if (pops->psm_tod_set)
849 		cmn_err(CE_WARN, "obsolete psm_tod_set op %p",
850 		    (void *)pops->psm_tod_set);
851 #endif
852 
853 	if (pops->psm_notify_error) {
854 		psm_notify_error = mach_notify_error;
855 		notify_error = pops->psm_notify_error;
856 	}
857 
858 	(*pops->psm_softinit)();
859 
860 	/*
861 	 * Initialize the dispatcher's function hooks
862 	 * to enable CPU halting when idle.
863 	 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle)
864 	 * or idle_cpu_prefer_mwait is not set.
865 	 * Allocate monitor/mwait buffer for cpu0.
866 	 */
867 	if (idle_cpu_use_hlt) {
868 		idle_cpu = cpu_idle;
869 #ifndef __xpv
870 		if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) {
871 			CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);
872 			/*
873 			 * Protect ourself from insane mwait size.
874 			 */
875 			if (CPU->cpu_m.mcpu_mwait == NULL) {
876 #ifdef DEBUG
877 				cmn_err(CE_NOTE, "Using hlt idle.  Cannot "
878 				    "handle cpu 0 mwait size.");
879 #endif
880 				idle_cpu_prefer_mwait = 0;
881 				idle_cpu = cpu_idle;
882 			} else {
883 				idle_cpu = cpu_idle_mwait;
884 			}
885 		} else {
886 			idle_cpu = cpu_idle;
887 		}
888 #endif
889 	}
890 
891 	mach_smpinit();
892 }
893 
894 static void
895 mach_smpinit(void)
896 {
897 	struct psm_ops  *pops;
898 	processorid_t cpu_id;
899 	int cnt;
900 	cpuset_t cpumask;
901 
902 	pops = mach_set[0];
903 	CPUSET_ZERO(cpumask);
904 
905 	cpu_id = -1;
906 	cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
907 	for (cnt = 0; cpu_id != -1; cnt++) {
908 		CPUSET_ADD(cpumask, cpu_id);
909 		cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
910 	}
911 
912 	mp_cpus = cpumask;
913 
914 	/* MP related routines */
915 	ap_mlsetup = pops->psm_post_cpu_start;
916 	send_dirintf = pops->psm_send_ipi;
917 
918 	/* optional MP related routines */
919 	if (pops->psm_shutdown)
920 		psm_shutdownf = pops->psm_shutdown;
921 	if (pops->psm_preshutdown)
922 		psm_preshutdownf = pops->psm_preshutdown;
923 	if (pops->psm_notify_func)
924 		psm_notifyf = pops->psm_notify_func;
925 	if (pops->psm_set_idlecpu)
926 		psm_set_idle_cpuf = pops->psm_set_idlecpu;
927 	if (pops->psm_unset_idlecpu)
928 		psm_unset_idle_cpuf = pops->psm_unset_idlecpu;
929 
930 	psm_clkinit = pops->psm_clkinit;
931 
932 	if (pops->psm_timer_reprogram)
933 		psm_timer_reprogram = pops->psm_timer_reprogram;
934 
935 	if (pops->psm_timer_enable)
936 		psm_timer_enable = pops->psm_timer_enable;
937 
938 	if (pops->psm_timer_disable)
939 		psm_timer_disable = pops->psm_timer_disable;
940 
941 	if (pops->psm_post_cyclic_setup)
942 		psm_post_cyclic_setup = pops->psm_post_cyclic_setup;
943 
944 	if (pops->psm_state)
945 		psm_state = pops->psm_state;
946 
947 	/*
948 	 * Set these vectors here so they can be used by Suspend/Resume
949 	 * on UP machines.
950 	 */
951 	if (pops->psm_disable_intr)
952 		psm_disable_intr = pops->psm_disable_intr;
953 	if (pops->psm_enable_intr)
954 		psm_enable_intr  = pops->psm_enable_intr;
955 
956 	/* check for multiple CPUs */
957 	if (cnt < 2)
958 		return;
959 
960 	/* check for MP platforms */
961 	if (pops->psm_cpu_start == NULL)
962 		return;
963 
964 	/*
965 	 * Set the dispatcher hook to enable cpu "wake up"
966 	 * when a thread becomes runnable.
967 	 */
968 	if (idle_cpu_use_hlt) {
969 		disp_enq_thread = cpu_wakeup;
970 #ifndef __xpv
971 		if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait)
972 			disp_enq_thread = cpu_wakeup_mwait;
973 #endif
974 	}
975 
976 	psm_get_ipivect = pops->psm_get_ipivect;
977 
978 	(void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr",
979 	    (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI),
980 	    (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL);
981 	(void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr",
982 	    (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO),
983 	    (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL);
984 
985 	(void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE);
986 }
987 
988 static void
989 mach_picinit()
990 {
991 	struct psm_ops  *pops;
992 
993 	pops = mach_set[0];
994 
995 	/* register the interrupt handlers */
996 	setlvl = pops->psm_intr_enter;
997 	setlvlx = pops->psm_intr_exit;
998 
999 	/* initialize the interrupt hardware */
1000 	(*pops->psm_picinit)();
1001 
1002 	/* set interrupt mask for current ipl */
1003 	setspl = pops->psm_setspl;
1004 	cli();
1005 	setspl(CPU->cpu_pri);
1006 }
1007 
1008 uint_t	cpu_freq;	/* MHz */
1009 uint64_t cpu_freq_hz;	/* measured (in hertz) */
1010 
1011 #define	MEGA_HZ		1000000
1012 
1013 #ifdef __xpv
1014 
1015 int xpv_cpufreq_workaround = 1;
1016 int xpv_cpufreq_verbose = 0;
1017 
1018 #else	/* __xpv */
1019 
1020 static uint64_t
1021 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks)
1022 {
1023 	uint64_t cpu_hz;
1024 
1025 	if ((pit_counter == 0) || (*processor_clks == 0) ||
1026 	    (*processor_clks > (((uint64_t)-1) / PIT_HZ)))
1027 		return (0);
1028 
1029 	cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter;
1030 
1031 	return (cpu_hz);
1032 }
1033 
1034 #endif	/* __xpv */
1035 
1036 static uint64_t
1037 mach_getcpufreq(void)
1038 {
1039 #if defined(__xpv)
1040 	vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time;
1041 	uint64_t cpu_hz;
1042 
1043 	/*
1044 	 * During dom0 bringup, it was noted that on at least one older
1045 	 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul
1046 	 * value that is quite wrong (the 3.06GHz clock was reported
1047 	 * as 4.77GHz)
1048 	 *
1049 	 * The curious thing is, that if you stop the kernel at entry,
1050 	 * breakpoint here and inspect the value with kmdb, the value
1051 	 * is correct - but if you don't stop and simply enable the
1052 	 * printf statement (below), you can see the bad value printed
1053 	 * here.  Almost as if something kmdb did caused the hypervisor to
1054 	 * figure it out correctly.  And, note that the hypervisor
1055 	 * eventually -does- figure it out correctly ... if you look at
1056 	 * the field later in the life of dom0, it is correct.
1057 	 *
1058 	 * For now, on dom0, we employ a slightly cheesy workaround of
1059 	 * using the DOM0_PHYSINFO hypercall.
1060 	 */
1061 	if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) {
1062 		xen_sysctl_t op0, *op = &op0;
1063 
1064 		op->cmd = XEN_SYSCTL_physinfo;
1065 		op->interface_version = XEN_SYSCTL_INTERFACE_VERSION;
1066 		if (HYPERVISOR_sysctl(op) != 0)
1067 			panic("physinfo op refused");
1068 
1069 		cpu_hz = 1000 * (uint64_t)op->u.physinfo.cpu_khz;
1070 	} else {
1071 		cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul;
1072 
1073 		if (vti->tsc_shift < 0)
1074 			cpu_hz <<= -vti->tsc_shift;
1075 		else
1076 			cpu_hz >>= vti->tsc_shift;
1077 	}
1078 
1079 	if (xpv_cpufreq_verbose)
1080 		printf("mach_getcpufreq: system_mul 0x%x, shift %d, "
1081 		    "cpu_hz %" PRId64 "Hz\n",
1082 		    vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz);
1083 
1084 	return (cpu_hz);
1085 #else	/* __xpv */
1086 	uint32_t pit_counter;
1087 	uint64_t processor_clks;
1088 
1089 	if (x86_feature & X86_TSC) {
1090 		/*
1091 		 * We have a TSC. freq_tsc() knows how to measure the number
1092 		 * of clock cycles sampled against the PIT.
1093 		 */
1094 		ulong_t flags = clear_int_flag();
1095 		processor_clks = freq_tsc(&pit_counter);
1096 		restore_int_flag(flags);
1097 		return (mach_calchz(pit_counter, &processor_clks));
1098 	} else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) {
1099 #if defined(__amd64)
1100 		panic("mach_getcpufreq: no TSC!");
1101 #elif defined(__i386)
1102 		/*
1103 		 * We are a Cyrix based on a 6x86 core or an Intel Pentium
1104 		 * for which freq_notsc() knows how to measure the number of
1105 		 * elapsed clock cycles sampled against the PIT
1106 		 */
1107 		ulong_t flags = clear_int_flag();
1108 		processor_clks = freq_notsc(&pit_counter);
1109 		restore_int_flag(flags);
1110 		return (mach_calchz(pit_counter, &processor_clks));
1111 #endif	/* __i386 */
1112 	}
1113 
1114 	/* We do not know how to calculate cpu frequency for this cpu. */
1115 	return (0);
1116 #endif	/* __xpv */
1117 }
1118 
1119 /*
1120  * If the clock speed of a cpu is found to be reported incorrectly, do not add
1121  * to this array, instead improve the accuracy of the algorithm that determines
1122  * the clock speed of the processor or extend the implementation to support the
1123  * vendor as appropriate. This is here only to support adjusting the speed on
1124  * older slower processors that mach_fixcpufreq() would not be able to account
1125  * for otherwise.
1126  */
1127 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 };
1128 
1129 /*
1130  * On fast processors the clock frequency that is measured may be off by
1131  * a few MHz from the value printed on the part. This is a combination of
1132  * the factors that for such fast parts being off by this much is within
1133  * the tolerances for manufacture and because of the difficulties in the
1134  * measurement that can lead to small error. This function uses some
1135  * heuristics in order to tweak the value that was measured to match what
1136  * is most likely printed on the part.
1137  *
1138  * Some examples:
1139  * 	AMD Athlon 1000 mhz measured as 998 mhz
1140  * 	Intel Pentium III Xeon 733 mhz measured as 731 mhz
1141  * 	Intel Pentium IV 1500 mhz measured as 1495mhz
1142  *
1143  * If in the future this function is no longer sufficient to correct
1144  * for the error in the measurement, then the algorithm used to perform
1145  * the measurement will have to be improved in order to increase accuracy
1146  * rather than adding horrible and questionable kludges here.
1147  *
1148  * This is called after the cyclics subsystem because of the potential
1149  * that the heuristics within may give a worse estimate of the clock
1150  * frequency than the value that was measured.
1151  */
1152 static void
1153 mach_fixcpufreq(void)
1154 {
1155 	uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i;
1156 
1157 	freq = (uint32_t)cpu_freq;
1158 
1159 	/*
1160 	 * Find the nearest integer multiple of 200/3 (about 66) MHz to the
1161 	 * measured speed taking into account that the 667 MHz parts were
1162 	 * the first to round-up.
1163 	 */
1164 	mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200);
1165 	near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3);
1166 	delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66);
1167 
1168 	/* Find the nearest integer multiple of 50 MHz to the measured speed */
1169 	mul = (freq + 25) / 50;
1170 	near50 = mul * 50;
1171 	delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50);
1172 
1173 	/* Find the closer of the two */
1174 	if (delta66 < delta50) {
1175 		fixed = near66;
1176 		delta = delta66;
1177 	} else {
1178 		fixed = near50;
1179 		delta = delta50;
1180 	}
1181 
1182 	if (fixed > INT_MAX)
1183 		return;
1184 
1185 	/*
1186 	 * Some older parts have a core clock frequency that is not an
1187 	 * integral multiple of 50 or 66 MHz. Check if one of the old
1188 	 * clock frequencies is closer to the measured value than any
1189 	 * of the integral multiples of 50 an 66, and if so set fixed
1190 	 * and delta appropriately to represent the closest value.
1191 	 */
1192 	i = sizeof (x86_cpu_freq) / sizeof (int);
1193 	while (i > 0) {
1194 		i--;
1195 
1196 		if (x86_cpu_freq[i] <= freq) {
1197 			mul = freq - x86_cpu_freq[i];
1198 
1199 			if (mul < delta) {
1200 				fixed = x86_cpu_freq[i];
1201 				delta = mul;
1202 			}
1203 
1204 			break;
1205 		}
1206 
1207 		mul = x86_cpu_freq[i] - freq;
1208 
1209 		if (mul < delta) {
1210 			fixed = x86_cpu_freq[i];
1211 			delta = mul;
1212 		}
1213 	}
1214 
1215 	/*
1216 	 * Set a reasonable maximum for how much to correct the measured
1217 	 * result by. This check is here to prevent the adjustment made
1218 	 * by this function from being more harm than good. It is entirely
1219 	 * possible that in the future parts will be made that are not
1220 	 * integral multiples of 66 or 50 in clock frequency or that
1221 	 * someone may overclock a part to some odd frequency. If the
1222 	 * measured value is farther from the corrected value than
1223 	 * allowed, then assume the corrected value is in error and use
1224 	 * the measured value.
1225 	 */
1226 	if (6 < delta)
1227 		return;
1228 
1229 	cpu_freq = (int)fixed;
1230 }
1231 
1232 
1233 static int
1234 machhztomhz(uint64_t cpu_freq_hz)
1235 {
1236 	uint64_t cpu_mhz;
1237 
1238 	/* Round to nearest MHZ */
1239 	cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ;
1240 
1241 	if (cpu_mhz > INT_MAX)
1242 		return (0);
1243 
1244 	return ((int)cpu_mhz);
1245 
1246 }
1247 
1248 
1249 static int
1250 mach_clkinit(int preferred_mode, int *set_mode)
1251 {
1252 	struct psm_ops  *pops;
1253 	int resolution;
1254 
1255 	pops = mach_set[0];
1256 
1257 	cpu_freq_hz = mach_getcpufreq();
1258 
1259 	cpu_freq = machhztomhz(cpu_freq_hz);
1260 
1261 	if (!(x86_feature & X86_TSC) || (cpu_freq == 0))
1262 		tsc_gethrtime_enable = 0;
1263 
1264 #ifndef __xpv
1265 	if (tsc_gethrtime_enable) {
1266 		tsc_hrtimeinit(cpu_freq_hz);
1267 	} else
1268 #endif
1269 	{
1270 		if (pops->psm_hrtimeinit)
1271 			(*pops->psm_hrtimeinit)();
1272 		gethrtimef = pops->psm_gethrtime;
1273 		gethrtimeunscaledf = gethrtimef;
1274 		/* scalehrtimef will remain dummy */
1275 	}
1276 
1277 	mach_fixcpufreq();
1278 
1279 	if (mach_ver[0] >= PSM_INFO_VER01_3) {
1280 		if (preferred_mode == TIMER_ONESHOT) {
1281 
1282 			resolution = (*pops->psm_clkinit)(0);
1283 			if (resolution != 0)  {
1284 				*set_mode = TIMER_ONESHOT;
1285 				return (resolution);
1286 			}
1287 		}
1288 
1289 		/*
1290 		 * either periodic mode was requested or could not set to
1291 		 * one-shot mode
1292 		 */
1293 		resolution = (*pops->psm_clkinit)(hz);
1294 		/*
1295 		 * psm should be able to do periodic, so we do not check
1296 		 * for return value of psm_clkinit here.
1297 		 */
1298 		*set_mode = TIMER_PERIODIC;
1299 		return (resolution);
1300 	} else {
1301 		/*
1302 		 * PSMI interface prior to PSMI_3 does not define a return
1303 		 * value for psm_clkinit, so the return value is ignored.
1304 		 */
1305 		(void) (*pops->psm_clkinit)(hz);
1306 		*set_mode = TIMER_PERIODIC;
1307 		return (nsec_per_tick);
1308 	}
1309 }
1310 
1311 
1312 /*ARGSUSED*/
1313 static int
1314 mach_softlvl_to_vect(int ipl)
1315 {
1316 	setsoftint = av_set_softint_pending;
1317 	kdisetsoftint = kdi_av_set_softint_pending;
1318 
1319 	return (PSM_SV_SOFTWARE);
1320 }
1321 
1322 #ifdef DEBUG
1323 /*
1324  * This is here to allow us to simulate cpus that refuse to start.
1325  */
1326 cpuset_t cpufailset;
1327 #endif
1328 
1329 int
1330 mach_cpu_start(struct cpu *cp, void *ctx)
1331 {
1332 	struct psm_ops *pops = mach_set[0];
1333 	processorid_t id = cp->cpu_id;
1334 
1335 #ifdef DEBUG
1336 	if (CPU_IN_SET(cpufailset, id))
1337 		return (0);
1338 #endif
1339 	return ((*pops->psm_cpu_start)(id, ctx));
1340 }
1341 
1342 int
1343 mach_cpuid_start(processorid_t id, void *ctx)
1344 {
1345 	struct psm_ops *pops = mach_set[0];
1346 
1347 #ifdef DEBUG
1348 	if (CPU_IN_SET(cpufailset, id))
1349 		return (0);
1350 #endif
1351 	return ((*pops->psm_cpu_start)(id, ctx));
1352 }
1353 
1354 /*ARGSUSED*/
1355 static int
1356 mach_translate_irq(dev_info_t *dip, int irqno)
1357 {
1358 	return (irqno);	/* default to NO translation */
1359 }
1360 
1361 static void
1362 mach_notify_error(int level, char *errmsg)
1363 {
1364 	/*
1365 	 * SL_FATAL is pass in once panicstr is set, deliver it
1366 	 * as CE_PANIC.  Also, translate SL_ codes back to CE_
1367 	 * codes for the psmi handler
1368 	 */
1369 	if (level & SL_FATAL)
1370 		(*notify_error)(CE_PANIC, errmsg);
1371 	else if (level & SL_WARN)
1372 		(*notify_error)(CE_WARN, errmsg);
1373 	else if (level & SL_NOTE)
1374 		(*notify_error)(CE_NOTE, errmsg);
1375 	else if (level & SL_CONSOLE)
1376 		(*notify_error)(CE_CONT, errmsg);
1377 }
1378 
1379 /*
1380  * It provides the default basic intr_ops interface for the new DDI
1381  * interrupt framework if the PSM doesn't have one.
1382  *
1383  * Input:
1384  * dip     - pointer to the dev_info structure of the requested device
1385  * hdlp    - pointer to the internal interrupt handle structure for the
1386  *	     requested interrupt
1387  * intr_op - opcode for this call
1388  * result  - pointer to the integer that will hold the result to be
1389  *	     passed back if return value is PSM_SUCCESS
1390  *
1391  * Output:
1392  * return value is either PSM_SUCCESS or PSM_FAILURE
1393  */
1394 static int
1395 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1396     psm_intr_op_t intr_op, int *result)
1397 {
1398 	struct intrspec *ispec;
1399 
1400 	switch (intr_op) {
1401 	case PSM_INTR_OP_CHECK_MSI:
1402 		*result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1403 		    DDI_INTR_TYPE_MSIX);
1404 		break;
1405 	case PSM_INTR_OP_ALLOC_VECTORS:
1406 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1407 			*result = 1;
1408 		else
1409 			*result = 0;
1410 		break;
1411 	case PSM_INTR_OP_FREE_VECTORS:
1412 		break;
1413 	case PSM_INTR_OP_NAVAIL_VECTORS:
1414 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1415 			*result = 1;
1416 		else
1417 			*result = 0;
1418 		break;
1419 	case PSM_INTR_OP_XLATE_VECTOR:
1420 		ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1421 		*result = psm_translate_irq(dip, ispec->intrspec_vec);
1422 		break;
1423 	case PSM_INTR_OP_GET_CAP:
1424 		*result = 0;
1425 		break;
1426 	case PSM_INTR_OP_GET_PENDING:
1427 	case PSM_INTR_OP_CLEAR_MASK:
1428 	case PSM_INTR_OP_SET_MASK:
1429 	case PSM_INTR_OP_GET_SHARED:
1430 	case PSM_INTR_OP_SET_PRI:
1431 	case PSM_INTR_OP_SET_CAP:
1432 	case PSM_INTR_OP_SET_CPU:
1433 	case PSM_INTR_OP_GET_INTR:
1434 	default:
1435 		return (PSM_FAILURE);
1436 	}
1437 	return (PSM_SUCCESS);
1438 }
1439 /*
1440  * Return 1 if CMT load balancing policies should be
1441  * implemented across instances of the specified hardware
1442  * sharing relationship.
1443  */
1444 int
1445 pg_cmt_load_bal_hw(pghw_type_t hw)
1446 {
1447 	if (hw == PGHW_IPIPE ||
1448 	    hw == PGHW_FPU ||
1449 	    hw == PGHW_CHIP)
1450 		return (1);
1451 	else
1452 		return (0);
1453 }
1454 /*
1455  * Return 1 if thread affinity polices should be implemented
1456  * for instances of the specifed hardware sharing relationship.
1457  */
1458 int
1459 pg_cmt_affinity_hw(pghw_type_t hw)
1460 {
1461 	if (hw == PGHW_CACHE)
1462 		return (1);
1463 	else
1464 		return (0);
1465 }
1466