xref: /titanic_52/usr/src/uts/i86pc/os/mp_machdep.c (revision 20d217c8569fadc52e6956aa7fcc78efd8d1f1b5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #define	PSMI_1_6
27 #include <sys/smp_impldefs.h>
28 #include <sys/psm.h>
29 #include <sys/psm_modctl.h>
30 #include <sys/pit.h>
31 #include <sys/cmn_err.h>
32 #include <sys/strlog.h>
33 #include <sys/clock.h>
34 #include <sys/debug.h>
35 #include <sys/rtc.h>
36 #include <sys/x86_archext.h>
37 #include <sys/cpupart.h>
38 #include <sys/cpuvar.h>
39 #include <sys/cpu_event.h>
40 #include <sys/cmt.h>
41 #include <sys/cpu.h>
42 #include <sys/disp.h>
43 #include <sys/archsystm.h>
44 #include <sys/machsystm.h>
45 #include <sys/sysmacros.h>
46 #include <sys/memlist.h>
47 #include <sys/param.h>
48 #include <sys/promif.h>
49 #include <sys/cpu_pm.h>
50 #if defined(__xpv)
51 #include <sys/hypervisor.h>
52 #endif
53 #include <sys/mach_intr.h>
54 #include <vm/hat_i86.h>
55 #include <sys/kdi_machimpl.h>
56 #include <sys/sdt.h>
57 #include <sys/hpet.h>
58 
59 #define	OFFSETOF(s, m)		(size_t)(&(((s *)0)->m))
60 
61 /*
62  *	Local function prototypes
63  */
64 static int mp_disable_intr(processorid_t cpun);
65 static void mp_enable_intr(processorid_t cpun);
66 static void mach_init();
67 static void mach_picinit();
68 static int machhztomhz(uint64_t cpu_freq_hz);
69 static uint64_t mach_getcpufreq(void);
70 static void mach_fixcpufreq(void);
71 static int mach_clkinit(int, int *);
72 static void mach_smpinit(void);
73 static int mach_softlvl_to_vect(int ipl);
74 static void mach_get_platform(int owner);
75 static void mach_construct_info();
76 static int mach_translate_irq(dev_info_t *dip, int irqno);
77 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
78     psm_intr_op_t, int *);
79 static void mach_notify_error(int level, char *errmsg);
80 static hrtime_t dummy_hrtime(void);
81 static void dummy_scalehrtime(hrtime_t *);
82 void cpu_idle(void);
83 static void cpu_wakeup(cpu_t *, int);
84 #ifndef __xpv
85 void cpu_idle_mwait(void);
86 static void cpu_wakeup_mwait(cpu_t *, int);
87 #endif
88 /*
89  *	External reference functions
90  */
91 extern void return_instr();
92 extern uint64_t freq_tsc(uint32_t *);
93 #if defined(__i386)
94 extern uint64_t freq_notsc(uint32_t *);
95 #endif
96 extern void pc_gethrestime(timestruc_t *);
97 extern int cpuid_get_coreid(cpu_t *);
98 extern int cpuid_get_chipid(cpu_t *);
99 
100 /*
101  *	PSM functions initialization
102  */
103 void (*psm_shutdownf)(int, int)	= (void (*)(int, int))return_instr;
104 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr;
105 void (*psm_notifyf)(int)	= (void (*)(int))return_instr;
106 void (*psm_set_idle_cpuf)(int)	= (void (*)(int))return_instr;
107 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr;
108 void (*psminitf)()		= mach_init;
109 void (*picinitf)() 		= return_instr;
110 int (*clkinitf)(int, int *) 	= (int (*)(int, int *))return_instr;
111 int (*ap_mlsetup)() 		= (int (*)(void))return_instr;
112 void (*send_dirintf)() 		= return_instr;
113 void (*setspl)(int)		= (void (*)(int))return_instr;
114 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
115 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
116 void (*kdisetsoftint)(int, struct av_softinfo *)=
117 	(void (*)(int, struct av_softinfo *))return_instr;
118 void (*setsoftint)(int, struct av_softinfo *)=
119 	(void (*)(int, struct av_softinfo *))return_instr;
120 int (*slvltovect)(int)		= (int (*)(int))return_instr;
121 int (*setlvl)(int, int *)	= (int (*)(int, int *))return_instr;
122 void (*setlvlx)(int, int)	= (void (*)(int, int))return_instr;
123 int (*psm_disable_intr)(int)	= mp_disable_intr;
124 void (*psm_enable_intr)(int)	= mp_enable_intr;
125 hrtime_t (*gethrtimef)(void)	= dummy_hrtime;
126 hrtime_t (*gethrtimeunscaledf)(void)	= dummy_hrtime;
127 void (*scalehrtimef)(hrtime_t *)	= dummy_scalehrtime;
128 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq;
129 void (*gethrestimef)(timestruc_t *) = pc_gethrestime;
130 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL;
131 int (*psm_get_clockirq)(int) = NULL;
132 int (*psm_get_ipivect)(int, int) = NULL;
133 
134 int (*psm_clkinit)(int) = NULL;
135 void (*psm_timer_reprogram)(hrtime_t) = NULL;
136 void (*psm_timer_enable)(void) = NULL;
137 void (*psm_timer_disable)(void) = NULL;
138 void (*psm_post_cyclic_setup)(void *arg) = NULL;
139 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t,
140     int *) = mach_intr_ops;
141 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *))
142     return_instr;
143 
144 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr;
145 void (*hrtime_tick)(void)	= return_instr;
146 
147 /*
148  * True if the generic TSC code is our source of hrtime, rather than whatever
149  * the PSM can provide.
150  */
151 #ifdef __xpv
152 int tsc_gethrtime_enable = 0;
153 #else
154 int tsc_gethrtime_enable = 1;
155 #endif
156 int tsc_gethrtime_initted = 0;
157 
158 /*
159  * True if the hrtime implementation is "hires"; namely, better than microdata.
160  */
161 int gethrtime_hires = 0;
162 
163 /*
164  * Local Static Data
165  */
166 static struct psm_ops mach_ops;
167 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL};
168 static ushort_t mach_ver[4] = {0, 0, 0, 0};
169 
170 /*
171  * virtualization support for psm
172  */
173 void *psm_vt_ops = NULL;
174 /*
175  * If non-zero, idle cpus will become "halted" when there's
176  * no work to do.
177  */
178 int	idle_cpu_use_hlt = 1;
179 
180 #ifndef __xpv
181 /*
182  * If non-zero, idle cpus will use mwait if available to halt instead of hlt.
183  */
184 int	idle_cpu_prefer_mwait = 1;
185 /*
186  * Set to 0 to avoid MONITOR+CLFLUSH assertion.
187  */
188 int	idle_cpu_assert_cflush_monitor = 1;
189 
190 /*
191  * If non-zero, idle cpus will not use power saving Deep C-States idle loop.
192  */
193 int	idle_cpu_no_deep_c = 0;
194 /*
195  * Non-power saving idle loop and wakeup pointers.
196  * Allows user to toggle Deep Idle power saving feature on/off.
197  */
198 void	(*non_deep_idle_cpu)() = cpu_idle;
199 void	(*non_deep_idle_disp_enq_thread)(cpu_t *, int);
200 
201 /*
202  * Object for the kernel to access the HPET.
203  */
204 hpet_t hpet;
205 
206 #endif	/* ifndef __xpv */
207 
208 /*ARGSUSED*/
209 int
210 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw)
211 {
212 	switch (hw) {
213 	case PGHW_IPIPE:
214 		if (x86_feature & (X86_HTT)) {
215 			/*
216 			 * Hyper-threading is SMT
217 			 */
218 			return (1);
219 		} else {
220 			return (0);
221 		}
222 	case PGHW_CHIP:
223 		if (x86_feature & (X86_CMP|X86_HTT))
224 			return (1);
225 		else
226 			return (0);
227 	case PGHW_CACHE:
228 		if (cpuid_get_ncpu_sharing_last_cache(cp) > 1)
229 			return (1);
230 		else
231 			return (0);
232 	case PGHW_POW_ACTIVE:
233 		if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1)
234 			return (1);
235 		else
236 			return (0);
237 	case PGHW_POW_IDLE:
238 		if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1)
239 			return (1);
240 		else
241 			return (0);
242 	default:
243 		return (0);
244 	}
245 }
246 
247 /*
248  * Compare two CPUs and see if they have a pghw_type_t sharing relationship
249  * If pghw_type_t is an unsupported hardware type, then return -1
250  */
251 int
252 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw)
253 {
254 	id_t pgp_a, pgp_b;
255 
256 	pgp_a = pg_plat_hw_instance_id(cpu_a, hw);
257 	pgp_b = pg_plat_hw_instance_id(cpu_b, hw);
258 
259 	if (pgp_a == -1 || pgp_b == -1)
260 		return (-1);
261 
262 	return (pgp_a == pgp_b);
263 }
264 
265 /*
266  * Return a physical instance identifier for known hardware sharing
267  * relationships
268  */
269 id_t
270 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw)
271 {
272 	switch (hw) {
273 	case PGHW_IPIPE:
274 		return (cpuid_get_coreid(cpu));
275 	case PGHW_CACHE:
276 		return (cpuid_get_last_lvl_cacheid(cpu));
277 	case PGHW_CHIP:
278 		return (cpuid_get_chipid(cpu));
279 	case PGHW_POW_ACTIVE:
280 		return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE));
281 	case PGHW_POW_IDLE:
282 		return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE));
283 	default:
284 		return (-1);
285 	}
286 }
287 
288 /*
289  * Express preference for optimizing for sharing relationship
290  * hw1 vs hw2
291  */
292 pghw_type_t
293 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2)
294 {
295 	int i, rank1, rank2;
296 
297 	static pghw_type_t hw_hier[] = {
298 		PGHW_IPIPE,
299 		PGHW_CACHE,
300 		PGHW_CHIP,
301 		PGHW_POW_IDLE,
302 		PGHW_POW_ACTIVE,
303 		PGHW_NUM_COMPONENTS
304 	};
305 
306 	for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) {
307 		if (hw_hier[i] == hw1)
308 			rank1 = i;
309 		if (hw_hier[i] == hw2)
310 			rank2 = i;
311 	}
312 
313 	if (rank1 > rank2)
314 		return (hw1);
315 	else
316 		return (hw2);
317 }
318 
319 /*
320  * Override the default CMT dispatcher policy for the specified
321  * hardware sharing relationship
322  */
323 pg_cmt_policy_t
324 pg_plat_cmt_policy(pghw_type_t hw)
325 {
326 	/*
327 	 * For shared caches, also load balance across them to
328 	 * maximize aggregate cache capacity
329 	 */
330 	switch (hw) {
331 	case PGHW_CACHE:
332 		return (CMT_BALANCE|CMT_AFFINITY);
333 	default:
334 		return (CMT_NO_POLICY);
335 	}
336 }
337 
338 id_t
339 pg_plat_get_core_id(cpu_t *cpu)
340 {
341 	return ((id_t)cpuid_get_coreid(cpu));
342 }
343 
344 void
345 cmp_set_nosteal_interval(void)
346 {
347 	/* Set the nosteal interval (used by disp_getbest()) to 100us */
348 	nosteal_nsec = 100000UL;
349 }
350 
351 /*
352  * Routine to ensure initial callers to hrtime gets 0 as return
353  */
354 static hrtime_t
355 dummy_hrtime(void)
356 {
357 	return (0);
358 }
359 
360 /* ARGSUSED */
361 static void
362 dummy_scalehrtime(hrtime_t *ticks)
363 {}
364 
365 /*
366  * Supports Deep C-State power saving idle loop.
367  */
368 void
369 cpu_idle_adaptive(void)
370 {
371 	(*CPU->cpu_m.mcpu_idle_cpu)();
372 }
373 
374 /*
375  * Function called by CPU idle notification framework to check whether CPU
376  * has been awakened. It will be called with interrupt disabled.
377  * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle
378  * notification framework.
379  */
380 /*ARGSUSED*/
381 static void
382 cpu_idle_check_wakeup(void *arg)
383 {
384 	/*
385 	 * Toggle interrupt flag to detect pending interrupts.
386 	 * If interrupt happened, do_interrupt() will notify CPU idle
387 	 * notification framework so no need to call cpu_idle_exit() here.
388 	 */
389 	sti();
390 	SMT_PAUSE();
391 	cli();
392 }
393 
394 /*
395  * Idle the present CPU until wakened via an interrupt
396  */
397 void
398 cpu_idle(void)
399 {
400 	cpu_t		*cpup = CPU;
401 	processorid_t	cpu_sid = cpup->cpu_seqid;
402 	cpupart_t	*cp = cpup->cpu_part;
403 	int		hset_update = 1;
404 
405 	/*
406 	 * If this CPU is online, and there's multiple CPUs
407 	 * in the system, then we should notate our halting
408 	 * by adding ourselves to the partition's halted CPU
409 	 * bitmap. This allows other CPUs to find/awaken us when
410 	 * work becomes available.
411 	 */
412 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
413 		hset_update = 0;
414 
415 	/*
416 	 * Add ourselves to the partition's halted CPUs bitmap
417 	 * and set our HALTED flag, if necessary.
418 	 *
419 	 * When a thread becomes runnable, it is placed on the queue
420 	 * and then the halted CPU bitmap is checked to determine who
421 	 * (if anyone) should be awakened. We therefore need to first
422 	 * add ourselves to the bitmap, and and then check if there
423 	 * is any work available. The order is important to prevent a race
424 	 * that can lead to work languishing on a run queue somewhere while
425 	 * this CPU remains halted.
426 	 *
427 	 * Either the producing CPU will see we're halted and will awaken us,
428 	 * or this CPU will see the work available in disp_anywork().
429 	 *
430 	 * Note that memory barriers after updating the HALTED flag
431 	 * are not necessary since an atomic operation (updating the bitset)
432 	 * immediately follows. On x86 the atomic operation acts as a
433 	 * memory barrier for the update of cpu_disp_flags.
434 	 */
435 	if (hset_update) {
436 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
437 		bitset_atomic_add(&cp->cp_haltset, cpu_sid);
438 	}
439 
440 	/*
441 	 * Check to make sure there's really nothing to do.
442 	 * Work destined for this CPU may become available after
443 	 * this check. We'll be notified through the clearing of our
444 	 * bit in the halted CPU bitmap, and a poke.
445 	 */
446 	if (disp_anywork()) {
447 		if (hset_update) {
448 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
449 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
450 		}
451 		return;
452 	}
453 
454 	/*
455 	 * We're on our way to being halted.
456 	 *
457 	 * Disable interrupts now, so that we'll awaken immediately
458 	 * after halting if someone tries to poke us between now and
459 	 * the time we actually halt.
460 	 *
461 	 * We check for the presence of our bit after disabling interrupts.
462 	 * If it's cleared, we'll return. If the bit is cleared after
463 	 * we check then the poke will pop us out of the halted state.
464 	 *
465 	 * This means that the ordering of the poke and the clearing
466 	 * of the bit by cpu_wakeup is important.
467 	 * cpu_wakeup() must clear, then poke.
468 	 * cpu_idle() must disable interrupts, then check for the bit.
469 	 */
470 	cli();
471 
472 	if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) {
473 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
474 		sti();
475 		return;
476 	}
477 
478 	/*
479 	 * The check for anything locally runnable is here for performance
480 	 * and isn't needed for correctness. disp_nrunnable ought to be
481 	 * in our cache still, so it's inexpensive to check, and if there
482 	 * is anything runnable we won't have to wait for the poke.
483 	 */
484 	if (cpup->cpu_disp->disp_nrunnable != 0) {
485 		if (hset_update) {
486 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
487 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
488 		}
489 		sti();
490 		return;
491 	}
492 
493 	if (cpu_idle_enter(IDLE_STATE_C1, 0,
494 	    cpu_idle_check_wakeup, NULL) == 0) {
495 		mach_cpu_idle();
496 		cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
497 	}
498 
499 	/*
500 	 * We're no longer halted
501 	 */
502 	if (hset_update) {
503 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
504 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
505 	}
506 }
507 
508 
509 /*
510  * If "cpu" is halted, then wake it up clearing its halted bit in advance.
511  * Otherwise, see if other CPUs in the cpu partition are halted and need to
512  * be woken up so that they can steal the thread we placed on this CPU.
513  * This function is only used on MP systems.
514  */
515 static void
516 cpu_wakeup(cpu_t *cpu, int bound)
517 {
518 	uint_t		cpu_found;
519 	processorid_t	cpu_sid;
520 	cpupart_t	*cp;
521 
522 	cp = cpu->cpu_part;
523 	cpu_sid = cpu->cpu_seqid;
524 	if (bitset_in_set(&cp->cp_haltset, cpu_sid)) {
525 		/*
526 		 * Clear the halted bit for that CPU since it will be
527 		 * poked in a moment.
528 		 */
529 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
530 		/*
531 		 * We may find the current CPU present in the halted cpuset
532 		 * if we're in the context of an interrupt that occurred
533 		 * before we had a chance to clear our bit in cpu_idle().
534 		 * Poking ourself is obviously unnecessary, since if
535 		 * we're here, we're not halted.
536 		 */
537 		if (cpu != CPU)
538 			poke_cpu(cpu->cpu_id);
539 		return;
540 	} else {
541 		/*
542 		 * This cpu isn't halted, but it's idle or undergoing a
543 		 * context switch. No need to awaken anyone else.
544 		 */
545 		if (cpu->cpu_thread == cpu->cpu_idle_thread ||
546 		    cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL)
547 			return;
548 	}
549 
550 	/*
551 	 * No need to wake up other CPUs if this is for a bound thread.
552 	 */
553 	if (bound)
554 		return;
555 
556 	/*
557 	 * The CPU specified for wakeup isn't currently halted, so check
558 	 * to see if there are any other halted CPUs in the partition,
559 	 * and if there are then awaken one.
560 	 */
561 	do {
562 		cpu_found = bitset_find(&cp->cp_haltset);
563 		if (cpu_found == (uint_t)-1)
564 			return;
565 	} while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0);
566 
567 	if (cpu_found != CPU->cpu_seqid) {
568 		poke_cpu(cpu_seq[cpu_found]->cpu_id);
569 	}
570 }
571 
572 #ifndef __xpv
573 /*
574  * Function called by CPU idle notification framework to check whether CPU
575  * has been awakened. It will be called with interrupt disabled.
576  * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle
577  * notification framework.
578  */
579 static void
580 cpu_idle_mwait_check_wakeup(void *arg)
581 {
582 	volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg;
583 
584 	ASSERT(arg != NULL);
585 	if (*mcpu_mwait != MWAIT_HALTED) {
586 		/*
587 		 * CPU has been awakened, notify CPU idle notification system.
588 		 */
589 		cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
590 	} else {
591 		/*
592 		 * Toggle interrupt flag to detect pending interrupts.
593 		 * If interrupt happened, do_interrupt() will notify CPU idle
594 		 * notification framework so no need to call cpu_idle_exit()
595 		 * here.
596 		 */
597 		sti();
598 		SMT_PAUSE();
599 		cli();
600 	}
601 }
602 
603 /*
604  * Idle the present CPU until awakened via touching its monitored line
605  */
606 void
607 cpu_idle_mwait(void)
608 {
609 	volatile uint32_t	*mcpu_mwait = CPU->cpu_m.mcpu_mwait;
610 	cpu_t			*cpup = CPU;
611 	processorid_t		cpu_sid = cpup->cpu_seqid;
612 	cpupart_t		*cp = cpup->cpu_part;
613 	int			hset_update = 1;
614 
615 	/*
616 	 * Set our mcpu_mwait here, so we can tell if anyone tries to
617 	 * wake us between now and when we call mwait.  No other cpu will
618 	 * attempt to set our mcpu_mwait until we add ourself to the halted
619 	 * CPU bitmap.
620 	 */
621 	*mcpu_mwait = MWAIT_HALTED;
622 
623 	/*
624 	 * If this CPU is online, and there's multiple CPUs
625 	 * in the system, then we should note our halting
626 	 * by adding ourselves to the partition's halted CPU
627 	 * bitmap. This allows other CPUs to find/awaken us when
628 	 * work becomes available.
629 	 */
630 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
631 		hset_update = 0;
632 
633 	/*
634 	 * Add ourselves to the partition's halted CPUs bitmap
635 	 * and set our HALTED flag, if necessary.
636 	 *
637 	 * When a thread becomes runnable, it is placed on the queue
638 	 * and then the halted CPU bitmap is checked to determine who
639 	 * (if anyone) should be awakened. We therefore need to first
640 	 * add ourselves to the bitmap, and and then check if there
641 	 * is any work available.
642 	 *
643 	 * Note that memory barriers after updating the HALTED flag
644 	 * are not necessary since an atomic operation (updating the bitmap)
645 	 * immediately follows. On x86 the atomic operation acts as a
646 	 * memory barrier for the update of cpu_disp_flags.
647 	 */
648 	if (hset_update) {
649 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
650 		bitset_atomic_add(&cp->cp_haltset, cpu_sid);
651 	}
652 
653 	/*
654 	 * Check to make sure there's really nothing to do.
655 	 * Work destined for this CPU may become available after
656 	 * this check. We'll be notified through the clearing of our
657 	 * bit in the halted CPU bitmap, and a write to our mcpu_mwait.
658 	 *
659 	 * disp_anywork() checks disp_nrunnable, so we do not have to later.
660 	 */
661 	if (disp_anywork()) {
662 		if (hset_update) {
663 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
664 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
665 		}
666 		return;
667 	}
668 
669 	/*
670 	 * We're on our way to being halted.
671 	 * To avoid a lost wakeup, arm the monitor before checking if another
672 	 * cpu wrote to mcpu_mwait to wake us up.
673 	 */
674 	i86_monitor(mcpu_mwait, 0, 0);
675 	if (*mcpu_mwait == MWAIT_HALTED) {
676 		if (cpu_idle_enter(IDLE_STATE_C1, 0,
677 		    cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) {
678 			if (*mcpu_mwait == MWAIT_HALTED) {
679 				i86_mwait(0, 0);
680 			}
681 			cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
682 		}
683 	}
684 
685 	/*
686 	 * We're no longer halted
687 	 */
688 	if (hset_update) {
689 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
690 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
691 	}
692 }
693 
694 /*
695  * If "cpu" is halted in mwait, then wake it up clearing its halted bit in
696  * advance.  Otherwise, see if other CPUs in the cpu partition are halted and
697  * need to be woken up so that they can steal the thread we placed on this CPU.
698  * This function is only used on MP systems.
699  */
700 static void
701 cpu_wakeup_mwait(cpu_t *cp, int bound)
702 {
703 	cpupart_t	*cpu_part;
704 	uint_t		cpu_found;
705 	processorid_t	cpu_sid;
706 
707 	cpu_part = cp->cpu_part;
708 	cpu_sid = cp->cpu_seqid;
709 
710 	/*
711 	 * Clear the halted bit for that CPU since it will be woken up
712 	 * in a moment.
713 	 */
714 	if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) {
715 		/*
716 		 * Clear the halted bit for that CPU since it will be
717 		 * poked in a moment.
718 		 */
719 		bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid);
720 		/*
721 		 * We may find the current CPU present in the halted cpuset
722 		 * if we're in the context of an interrupt that occurred
723 		 * before we had a chance to clear our bit in cpu_idle().
724 		 * Waking ourself is obviously unnecessary, since if
725 		 * we're here, we're not halted.
726 		 *
727 		 * monitor/mwait wakeup via writing to our cache line is
728 		 * harmless and less expensive than always checking if we
729 		 * are waking ourself which is an uncommon case.
730 		 */
731 		MWAIT_WAKEUP(cp);	/* write to monitored line */
732 		return;
733 	} else {
734 		/*
735 		 * This cpu isn't halted, but it's idle or undergoing a
736 		 * context switch. No need to awaken anyone else.
737 		 */
738 		if (cp->cpu_thread == cp->cpu_idle_thread ||
739 		    cp->cpu_disp_flags & CPU_DISP_DONTSTEAL)
740 			return;
741 	}
742 
743 	/*
744 	 * No need to wake up other CPUs if the thread we just enqueued
745 	 * is bound.
746 	 */
747 	if (bound || ncpus == 1)
748 		return;
749 
750 	/*
751 	 * See if there's any other halted CPUs. If there are, then
752 	 * select one, and awaken it.
753 	 * It's possible that after we find a CPU, somebody else
754 	 * will awaken it before we get the chance.
755 	 * In that case, look again.
756 	 */
757 	do {
758 		cpu_found = bitset_find(&cpu_part->cp_haltset);
759 		if (cpu_found == (uint_t)-1)
760 			return;
761 	} while (bitset_atomic_test_and_del(&cpu_part->cp_haltset,
762 	    cpu_found) < 0);
763 
764 	/*
765 	 * Do not check if cpu_found is ourself as monitor/mwait
766 	 * wakeup is cheap.
767 	 */
768 	MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */
769 }
770 
771 #endif
772 
773 void (*cpu_pause_handler)(volatile char *) = NULL;
774 
775 static int
776 mp_disable_intr(int cpun)
777 {
778 	/*
779 	 * switch to the offline cpu
780 	 */
781 	affinity_set(cpun);
782 	/*
783 	 * raise ipl to just below cross call
784 	 */
785 	splx(XC_SYS_PIL - 1);
786 	/*
787 	 *	set base spl to prevent the next swtch to idle from
788 	 *	lowering back to ipl 0
789 	 */
790 	CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1));
791 	set_base_spl();
792 	affinity_clear();
793 	return (DDI_SUCCESS);
794 }
795 
796 static void
797 mp_enable_intr(int cpun)
798 {
799 	/*
800 	 * switch to the online cpu
801 	 */
802 	affinity_set(cpun);
803 	/*
804 	 * clear the interrupt active mask
805 	 */
806 	CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1));
807 	set_base_spl();
808 	(void) spl0();
809 	affinity_clear();
810 }
811 
812 static void
813 mach_get_platform(int owner)
814 {
815 	void		**srv_opsp;
816 	void		**clt_opsp;
817 	int		i;
818 	int		total_ops;
819 
820 	/* fix up psm ops */
821 	srv_opsp = (void **)mach_set[0];
822 	clt_opsp = (void **)mach_set[owner];
823 	if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01)
824 		total_ops = sizeof (struct psm_ops_ver01) /
825 		    sizeof (void (*)(void));
826 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1)
827 		/* no psm_notify_func */
828 		total_ops = OFFSETOF(struct psm_ops, psm_notify_func) /
829 		    sizeof (void (*)(void));
830 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2)
831 		/* no psm_timer funcs */
832 		total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) /
833 		    sizeof (void (*)(void));
834 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3)
835 		/* no psm_preshutdown function */
836 		total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) /
837 		    sizeof (void (*)(void));
838 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4)
839 		/* no psm_preshutdown function */
840 		total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) /
841 		    sizeof (void (*)(void));
842 	else
843 		total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void));
844 
845 	/*
846 	 * Save the version of the PSM module, in case we need to
847 	 * behave differently based on version.
848 	 */
849 	mach_ver[0] = mach_ver[owner];
850 
851 	for (i = 0; i < total_ops; i++)
852 		if (clt_opsp[i] != NULL)
853 			srv_opsp[i] = clt_opsp[i];
854 }
855 
856 static void
857 mach_construct_info()
858 {
859 	struct psm_sw *swp;
860 	int	mach_cnt[PSM_OWN_OVERRIDE+1] = {0};
861 	int	conflict_owner = 0;
862 
863 	if (psmsw->psw_forw == psmsw)
864 		panic("No valid PSM modules found");
865 	mutex_enter(&psmsw_lock);
866 	for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
867 		if (!(swp->psw_flag & PSM_MOD_IDENTIFY))
868 			continue;
869 		mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops;
870 		mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version;
871 		mach_cnt[swp->psw_infop->p_owner]++;
872 	}
873 	mutex_exit(&psmsw_lock);
874 
875 	mach_get_platform(PSM_OWN_SYS_DEFAULT);
876 
877 	/* check to see are there any conflicts */
878 	if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1)
879 		conflict_owner = PSM_OWN_EXCLUSIVE;
880 	if (mach_cnt[PSM_OWN_OVERRIDE] > 1)
881 		conflict_owner = PSM_OWN_OVERRIDE;
882 	if (conflict_owner) {
883 		/* remove all psm modules except uppc */
884 		cmn_err(CE_WARN,
885 		    "Conflicts detected on the following PSM modules:");
886 		mutex_enter(&psmsw_lock);
887 		for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
888 			if (swp->psw_infop->p_owner == conflict_owner)
889 				cmn_err(CE_WARN, "%s ",
890 				    swp->psw_infop->p_mach_idstring);
891 		}
892 		mutex_exit(&psmsw_lock);
893 		cmn_err(CE_WARN,
894 		    "Setting the system back to SINGLE processor mode!");
895 		cmn_err(CE_WARN,
896 		    "Please edit /etc/mach to remove the invalid PSM module.");
897 		return;
898 	}
899 
900 	if (mach_set[PSM_OWN_EXCLUSIVE])
901 		mach_get_platform(PSM_OWN_EXCLUSIVE);
902 
903 	if (mach_set[PSM_OWN_OVERRIDE])
904 		mach_get_platform(PSM_OWN_OVERRIDE);
905 }
906 
907 static void
908 mach_init()
909 {
910 	struct psm_ops  *pops;
911 
912 	mach_construct_info();
913 
914 	pops = mach_set[0];
915 
916 	/* register the interrupt and clock initialization rotuines */
917 	picinitf = mach_picinit;
918 	clkinitf = mach_clkinit;
919 	psm_get_clockirq = pops->psm_get_clockirq;
920 
921 	/* register the interrupt setup code */
922 	slvltovect = mach_softlvl_to_vect;
923 	addspl	= pops->psm_addspl;
924 	delspl	= pops->psm_delspl;
925 
926 	if (pops->psm_translate_irq)
927 		psm_translate_irq = pops->psm_translate_irq;
928 	if (pops->psm_intr_ops)
929 		psm_intr_ops = pops->psm_intr_ops;
930 
931 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4)
932 	/*
933 	 * Time-of-day functionality now handled in TOD modules.
934 	 * (Warn about PSM modules that think that we're going to use
935 	 * their ops vectors.)
936 	 */
937 	if (pops->psm_tod_get)
938 		cmn_err(CE_WARN, "obsolete psm_tod_get op %p",
939 		    (void *)pops->psm_tod_get);
940 
941 	if (pops->psm_tod_set)
942 		cmn_err(CE_WARN, "obsolete psm_tod_set op %p",
943 		    (void *)pops->psm_tod_set);
944 #endif
945 
946 	if (pops->psm_notify_error) {
947 		psm_notify_error = mach_notify_error;
948 		notify_error = pops->psm_notify_error;
949 	}
950 
951 	(*pops->psm_softinit)();
952 
953 	/*
954 	 * Initialize the dispatcher's function hooks to enable CPU halting
955 	 * when idle.  Set both the deep-idle and non-deep-idle hooks.
956 	 *
957 	 * Assume we can use power saving deep-idle loop cpu_idle_adaptive.
958 	 * Platform deep-idle driver will reset our idle loop to
959 	 * non_deep_idle_cpu if power saving deep-idle feature is not available.
960 	 *
961 	 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle)
962 	 * or idle_cpu_prefer_mwait is not set.
963 	 * Allocate monitor/mwait buffer for cpu0.
964 	 */
965 #ifndef __xpv
966 	non_deep_idle_disp_enq_thread = disp_enq_thread;
967 #endif
968 	if (idle_cpu_use_hlt) {
969 		idle_cpu = cpu_idle_adaptive;
970 		CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
971 #ifndef __xpv
972 		if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) {
973 			CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);
974 			/*
975 			 * Protect ourself from insane mwait size.
976 			 */
977 			if (CPU->cpu_m.mcpu_mwait == NULL) {
978 #ifdef DEBUG
979 				cmn_err(CE_NOTE, "Using hlt idle.  Cannot "
980 				    "handle cpu 0 mwait size.");
981 #endif
982 				idle_cpu_prefer_mwait = 0;
983 				CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
984 			} else {
985 				CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
986 			}
987 		} else {
988 			CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
989 		}
990 		non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu;
991 
992 		/*
993 		 * Disable power saving deep idle loop?
994 		 */
995 		if (idle_cpu_no_deep_c) {
996 			idle_cpu = non_deep_idle_cpu;
997 		}
998 #endif
999 	}
1000 
1001 	mach_smpinit();
1002 }
1003 
1004 static void
1005 mach_smpinit(void)
1006 {
1007 	struct psm_ops  *pops;
1008 	processorid_t cpu_id;
1009 	int cnt;
1010 	cpuset_t cpumask;
1011 
1012 	pops = mach_set[0];
1013 	CPUSET_ZERO(cpumask);
1014 
1015 	cpu_id = -1;
1016 	cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
1017 	for (cnt = 0; cpu_id != -1; cnt++) {
1018 		CPUSET_ADD(cpumask, cpu_id);
1019 		cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
1020 	}
1021 
1022 	mp_cpus = cpumask;
1023 
1024 	/* MP related routines */
1025 	ap_mlsetup = pops->psm_post_cpu_start;
1026 	send_dirintf = pops->psm_send_ipi;
1027 
1028 	/* optional MP related routines */
1029 	if (pops->psm_shutdown)
1030 		psm_shutdownf = pops->psm_shutdown;
1031 	if (pops->psm_preshutdown)
1032 		psm_preshutdownf = pops->psm_preshutdown;
1033 	if (pops->psm_notify_func)
1034 		psm_notifyf = pops->psm_notify_func;
1035 	if (pops->psm_set_idlecpu)
1036 		psm_set_idle_cpuf = pops->psm_set_idlecpu;
1037 	if (pops->psm_unset_idlecpu)
1038 		psm_unset_idle_cpuf = pops->psm_unset_idlecpu;
1039 
1040 	psm_clkinit = pops->psm_clkinit;
1041 
1042 	if (pops->psm_timer_reprogram)
1043 		psm_timer_reprogram = pops->psm_timer_reprogram;
1044 
1045 	if (pops->psm_timer_enable)
1046 		psm_timer_enable = pops->psm_timer_enable;
1047 
1048 	if (pops->psm_timer_disable)
1049 		psm_timer_disable = pops->psm_timer_disable;
1050 
1051 	if (pops->psm_post_cyclic_setup)
1052 		psm_post_cyclic_setup = pops->psm_post_cyclic_setup;
1053 
1054 	if (pops->psm_state)
1055 		psm_state = pops->psm_state;
1056 
1057 	/*
1058 	 * Set these vectors here so they can be used by Suspend/Resume
1059 	 * on UP machines.
1060 	 */
1061 	if (pops->psm_disable_intr)
1062 		psm_disable_intr = pops->psm_disable_intr;
1063 	if (pops->psm_enable_intr)
1064 		psm_enable_intr  = pops->psm_enable_intr;
1065 
1066 	/* check for multiple CPUs */
1067 	if (cnt < 2)
1068 		return;
1069 
1070 	/* check for MP platforms */
1071 	if (pops->psm_cpu_start == NULL)
1072 		return;
1073 
1074 	/*
1075 	 * Set the dispatcher hook to enable cpu "wake up"
1076 	 * when a thread becomes runnable.
1077 	 */
1078 	if (idle_cpu_use_hlt) {
1079 		disp_enq_thread = cpu_wakeup;
1080 #ifndef __xpv
1081 		if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait)
1082 			disp_enq_thread = cpu_wakeup_mwait;
1083 		non_deep_idle_disp_enq_thread = disp_enq_thread;
1084 #endif
1085 	}
1086 
1087 	psm_get_ipivect = pops->psm_get_ipivect;
1088 
1089 	(void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr",
1090 	    (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI),
1091 	    NULL, NULL, NULL, NULL);
1092 
1093 	(void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE);
1094 }
1095 
1096 static void
1097 mach_picinit()
1098 {
1099 	struct psm_ops  *pops;
1100 
1101 	pops = mach_set[0];
1102 
1103 	/* register the interrupt handlers */
1104 	setlvl = pops->psm_intr_enter;
1105 	setlvlx = pops->psm_intr_exit;
1106 
1107 	/* initialize the interrupt hardware */
1108 	(*pops->psm_picinit)();
1109 
1110 	/* set interrupt mask for current ipl */
1111 	setspl = pops->psm_setspl;
1112 	cli();
1113 	setspl(CPU->cpu_pri);
1114 }
1115 
1116 uint_t	cpu_freq;	/* MHz */
1117 uint64_t cpu_freq_hz;	/* measured (in hertz) */
1118 
1119 #define	MEGA_HZ		1000000
1120 
1121 #ifdef __xpv
1122 
1123 int xpv_cpufreq_workaround = 1;
1124 int xpv_cpufreq_verbose = 0;
1125 
1126 #else	/* __xpv */
1127 
1128 static uint64_t
1129 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks)
1130 {
1131 	uint64_t cpu_hz;
1132 
1133 	if ((pit_counter == 0) || (*processor_clks == 0) ||
1134 	    (*processor_clks > (((uint64_t)-1) / PIT_HZ)))
1135 		return (0);
1136 
1137 	cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter;
1138 
1139 	return (cpu_hz);
1140 }
1141 
1142 #endif	/* __xpv */
1143 
1144 static uint64_t
1145 mach_getcpufreq(void)
1146 {
1147 #if defined(__xpv)
1148 	vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time;
1149 	uint64_t cpu_hz;
1150 
1151 	/*
1152 	 * During dom0 bringup, it was noted that on at least one older
1153 	 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul
1154 	 * value that is quite wrong (the 3.06GHz clock was reported
1155 	 * as 4.77GHz)
1156 	 *
1157 	 * The curious thing is, that if you stop the kernel at entry,
1158 	 * breakpoint here and inspect the value with kmdb, the value
1159 	 * is correct - but if you don't stop and simply enable the
1160 	 * printf statement (below), you can see the bad value printed
1161 	 * here.  Almost as if something kmdb did caused the hypervisor to
1162 	 * figure it out correctly.  And, note that the hypervisor
1163 	 * eventually -does- figure it out correctly ... if you look at
1164 	 * the field later in the life of dom0, it is correct.
1165 	 *
1166 	 * For now, on dom0, we employ a slightly cheesy workaround of
1167 	 * using the DOM0_PHYSINFO hypercall.
1168 	 */
1169 	if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) {
1170 		xen_sysctl_t op0, *op = &op0;
1171 
1172 		op->cmd = XEN_SYSCTL_physinfo;
1173 		op->interface_version = XEN_SYSCTL_INTERFACE_VERSION;
1174 		if (HYPERVISOR_sysctl(op) != 0)
1175 			panic("physinfo op refused");
1176 
1177 		cpu_hz = 1000 * (uint64_t)op->u.physinfo.cpu_khz;
1178 	} else {
1179 		cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul;
1180 
1181 		if (vti->tsc_shift < 0)
1182 			cpu_hz <<= -vti->tsc_shift;
1183 		else
1184 			cpu_hz >>= vti->tsc_shift;
1185 	}
1186 
1187 	if (xpv_cpufreq_verbose)
1188 		printf("mach_getcpufreq: system_mul 0x%x, shift %d, "
1189 		    "cpu_hz %" PRId64 "Hz\n",
1190 		    vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz);
1191 
1192 	return (cpu_hz);
1193 #else	/* __xpv */
1194 	uint32_t pit_counter;
1195 	uint64_t processor_clks;
1196 
1197 	if (x86_feature & X86_TSC) {
1198 		/*
1199 		 * We have a TSC. freq_tsc() knows how to measure the number
1200 		 * of clock cycles sampled against the PIT.
1201 		 */
1202 		ulong_t flags = clear_int_flag();
1203 		processor_clks = freq_tsc(&pit_counter);
1204 		restore_int_flag(flags);
1205 		return (mach_calchz(pit_counter, &processor_clks));
1206 	} else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) {
1207 #if defined(__amd64)
1208 		panic("mach_getcpufreq: no TSC!");
1209 #elif defined(__i386)
1210 		/*
1211 		 * We are a Cyrix based on a 6x86 core or an Intel Pentium
1212 		 * for which freq_notsc() knows how to measure the number of
1213 		 * elapsed clock cycles sampled against the PIT
1214 		 */
1215 		ulong_t flags = clear_int_flag();
1216 		processor_clks = freq_notsc(&pit_counter);
1217 		restore_int_flag(flags);
1218 		return (mach_calchz(pit_counter, &processor_clks));
1219 #endif	/* __i386 */
1220 	}
1221 
1222 	/* We do not know how to calculate cpu frequency for this cpu. */
1223 	return (0);
1224 #endif	/* __xpv */
1225 }
1226 
1227 /*
1228  * If the clock speed of a cpu is found to be reported incorrectly, do not add
1229  * to this array, instead improve the accuracy of the algorithm that determines
1230  * the clock speed of the processor or extend the implementation to support the
1231  * vendor as appropriate. This is here only to support adjusting the speed on
1232  * older slower processors that mach_fixcpufreq() would not be able to account
1233  * for otherwise.
1234  */
1235 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 };
1236 
1237 /*
1238  * On fast processors the clock frequency that is measured may be off by
1239  * a few MHz from the value printed on the part. This is a combination of
1240  * the factors that for such fast parts being off by this much is within
1241  * the tolerances for manufacture and because of the difficulties in the
1242  * measurement that can lead to small error. This function uses some
1243  * heuristics in order to tweak the value that was measured to match what
1244  * is most likely printed on the part.
1245  *
1246  * Some examples:
1247  * 	AMD Athlon 1000 mhz measured as 998 mhz
1248  * 	Intel Pentium III Xeon 733 mhz measured as 731 mhz
1249  * 	Intel Pentium IV 1500 mhz measured as 1495mhz
1250  *
1251  * If in the future this function is no longer sufficient to correct
1252  * for the error in the measurement, then the algorithm used to perform
1253  * the measurement will have to be improved in order to increase accuracy
1254  * rather than adding horrible and questionable kludges here.
1255  *
1256  * This is called after the cyclics subsystem because of the potential
1257  * that the heuristics within may give a worse estimate of the clock
1258  * frequency than the value that was measured.
1259  */
1260 static void
1261 mach_fixcpufreq(void)
1262 {
1263 	uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i;
1264 
1265 	freq = (uint32_t)cpu_freq;
1266 
1267 	/*
1268 	 * Find the nearest integer multiple of 200/3 (about 66) MHz to the
1269 	 * measured speed taking into account that the 667 MHz parts were
1270 	 * the first to round-up.
1271 	 */
1272 	mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200);
1273 	near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3);
1274 	delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66);
1275 
1276 	/* Find the nearest integer multiple of 50 MHz to the measured speed */
1277 	mul = (freq + 25) / 50;
1278 	near50 = mul * 50;
1279 	delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50);
1280 
1281 	/* Find the closer of the two */
1282 	if (delta66 < delta50) {
1283 		fixed = near66;
1284 		delta = delta66;
1285 	} else {
1286 		fixed = near50;
1287 		delta = delta50;
1288 	}
1289 
1290 	if (fixed > INT_MAX)
1291 		return;
1292 
1293 	/*
1294 	 * Some older parts have a core clock frequency that is not an
1295 	 * integral multiple of 50 or 66 MHz. Check if one of the old
1296 	 * clock frequencies is closer to the measured value than any
1297 	 * of the integral multiples of 50 an 66, and if so set fixed
1298 	 * and delta appropriately to represent the closest value.
1299 	 */
1300 	i = sizeof (x86_cpu_freq) / sizeof (int);
1301 	while (i > 0) {
1302 		i--;
1303 
1304 		if (x86_cpu_freq[i] <= freq) {
1305 			mul = freq - x86_cpu_freq[i];
1306 
1307 			if (mul < delta) {
1308 				fixed = x86_cpu_freq[i];
1309 				delta = mul;
1310 			}
1311 
1312 			break;
1313 		}
1314 
1315 		mul = x86_cpu_freq[i] - freq;
1316 
1317 		if (mul < delta) {
1318 			fixed = x86_cpu_freq[i];
1319 			delta = mul;
1320 		}
1321 	}
1322 
1323 	/*
1324 	 * Set a reasonable maximum for how much to correct the measured
1325 	 * result by. This check is here to prevent the adjustment made
1326 	 * by this function from being more harm than good. It is entirely
1327 	 * possible that in the future parts will be made that are not
1328 	 * integral multiples of 66 or 50 in clock frequency or that
1329 	 * someone may overclock a part to some odd frequency. If the
1330 	 * measured value is farther from the corrected value than
1331 	 * allowed, then assume the corrected value is in error and use
1332 	 * the measured value.
1333 	 */
1334 	if (6 < delta)
1335 		return;
1336 
1337 	cpu_freq = (int)fixed;
1338 }
1339 
1340 
1341 static int
1342 machhztomhz(uint64_t cpu_freq_hz)
1343 {
1344 	uint64_t cpu_mhz;
1345 
1346 	/* Round to nearest MHZ */
1347 	cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ;
1348 
1349 	if (cpu_mhz > INT_MAX)
1350 		return (0);
1351 
1352 	return ((int)cpu_mhz);
1353 
1354 }
1355 
1356 
1357 static int
1358 mach_clkinit(int preferred_mode, int *set_mode)
1359 {
1360 	struct psm_ops  *pops;
1361 	int resolution;
1362 
1363 	pops = mach_set[0];
1364 
1365 	cpu_freq_hz = mach_getcpufreq();
1366 
1367 	cpu_freq = machhztomhz(cpu_freq_hz);
1368 
1369 	if (!(x86_feature & X86_TSC) || (cpu_freq == 0))
1370 		tsc_gethrtime_enable = 0;
1371 
1372 #ifndef __xpv
1373 	if (tsc_gethrtime_enable) {
1374 		tsc_hrtimeinit(cpu_freq_hz);
1375 	} else
1376 #endif
1377 	{
1378 		if (pops->psm_hrtimeinit)
1379 			(*pops->psm_hrtimeinit)();
1380 		gethrtimef = pops->psm_gethrtime;
1381 		gethrtimeunscaledf = gethrtimef;
1382 		/* scalehrtimef will remain dummy */
1383 	}
1384 
1385 	mach_fixcpufreq();
1386 
1387 	if (mach_ver[0] >= PSM_INFO_VER01_3) {
1388 		if (preferred_mode == TIMER_ONESHOT) {
1389 
1390 			resolution = (*pops->psm_clkinit)(0);
1391 			if (resolution != 0)  {
1392 				*set_mode = TIMER_ONESHOT;
1393 				return (resolution);
1394 			}
1395 		}
1396 
1397 		/*
1398 		 * either periodic mode was requested or could not set to
1399 		 * one-shot mode
1400 		 */
1401 		resolution = (*pops->psm_clkinit)(hz);
1402 		/*
1403 		 * psm should be able to do periodic, so we do not check
1404 		 * for return value of psm_clkinit here.
1405 		 */
1406 		*set_mode = TIMER_PERIODIC;
1407 		return (resolution);
1408 	} else {
1409 		/*
1410 		 * PSMI interface prior to PSMI_3 does not define a return
1411 		 * value for psm_clkinit, so the return value is ignored.
1412 		 */
1413 		(void) (*pops->psm_clkinit)(hz);
1414 		*set_mode = TIMER_PERIODIC;
1415 		return (nsec_per_tick);
1416 	}
1417 }
1418 
1419 
1420 /*ARGSUSED*/
1421 static int
1422 mach_softlvl_to_vect(int ipl)
1423 {
1424 	setsoftint = av_set_softint_pending;
1425 	kdisetsoftint = kdi_av_set_softint_pending;
1426 
1427 	return (PSM_SV_SOFTWARE);
1428 }
1429 
1430 #ifdef DEBUG
1431 /*
1432  * This is here to allow us to simulate cpus that refuse to start.
1433  */
1434 cpuset_t cpufailset;
1435 #endif
1436 
1437 int
1438 mach_cpu_start(struct cpu *cp, void *ctx)
1439 {
1440 	struct psm_ops *pops = mach_set[0];
1441 	processorid_t id = cp->cpu_id;
1442 
1443 #ifdef DEBUG
1444 	if (CPU_IN_SET(cpufailset, id))
1445 		return (0);
1446 #endif
1447 	return ((*pops->psm_cpu_start)(id, ctx));
1448 }
1449 
1450 int
1451 mach_cpuid_start(processorid_t id, void *ctx)
1452 {
1453 	struct psm_ops *pops = mach_set[0];
1454 
1455 #ifdef DEBUG
1456 	if (CPU_IN_SET(cpufailset, id))
1457 		return (0);
1458 #endif
1459 	return ((*pops->psm_cpu_start)(id, ctx));
1460 }
1461 
1462 /*ARGSUSED*/
1463 static int
1464 mach_translate_irq(dev_info_t *dip, int irqno)
1465 {
1466 	return (irqno);	/* default to NO translation */
1467 }
1468 
1469 static void
1470 mach_notify_error(int level, char *errmsg)
1471 {
1472 	/*
1473 	 * SL_FATAL is pass in once panicstr is set, deliver it
1474 	 * as CE_PANIC.  Also, translate SL_ codes back to CE_
1475 	 * codes for the psmi handler
1476 	 */
1477 	if (level & SL_FATAL)
1478 		(*notify_error)(CE_PANIC, errmsg);
1479 	else if (level & SL_WARN)
1480 		(*notify_error)(CE_WARN, errmsg);
1481 	else if (level & SL_NOTE)
1482 		(*notify_error)(CE_NOTE, errmsg);
1483 	else if (level & SL_CONSOLE)
1484 		(*notify_error)(CE_CONT, errmsg);
1485 }
1486 
1487 /*
1488  * It provides the default basic intr_ops interface for the new DDI
1489  * interrupt framework if the PSM doesn't have one.
1490  *
1491  * Input:
1492  * dip     - pointer to the dev_info structure of the requested device
1493  * hdlp    - pointer to the internal interrupt handle structure for the
1494  *	     requested interrupt
1495  * intr_op - opcode for this call
1496  * result  - pointer to the integer that will hold the result to be
1497  *	     passed back if return value is PSM_SUCCESS
1498  *
1499  * Output:
1500  * return value is either PSM_SUCCESS or PSM_FAILURE
1501  */
1502 static int
1503 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1504     psm_intr_op_t intr_op, int *result)
1505 {
1506 	struct intrspec *ispec;
1507 
1508 	switch (intr_op) {
1509 	case PSM_INTR_OP_CHECK_MSI:
1510 		*result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1511 		    DDI_INTR_TYPE_MSIX);
1512 		break;
1513 	case PSM_INTR_OP_ALLOC_VECTORS:
1514 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1515 			*result = 1;
1516 		else
1517 			*result = 0;
1518 		break;
1519 	case PSM_INTR_OP_FREE_VECTORS:
1520 		break;
1521 	case PSM_INTR_OP_NAVAIL_VECTORS:
1522 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1523 			*result = 1;
1524 		else
1525 			*result = 0;
1526 		break;
1527 	case PSM_INTR_OP_XLATE_VECTOR:
1528 		ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1529 		*result = psm_translate_irq(dip, ispec->intrspec_vec);
1530 		break;
1531 	case PSM_INTR_OP_GET_CAP:
1532 		*result = 0;
1533 		break;
1534 	case PSM_INTR_OP_GET_PENDING:
1535 	case PSM_INTR_OP_CLEAR_MASK:
1536 	case PSM_INTR_OP_SET_MASK:
1537 	case PSM_INTR_OP_GET_SHARED:
1538 	case PSM_INTR_OP_SET_PRI:
1539 	case PSM_INTR_OP_SET_CAP:
1540 	case PSM_INTR_OP_SET_CPU:
1541 	case PSM_INTR_OP_GET_INTR:
1542 	default:
1543 		return (PSM_FAILURE);
1544 	}
1545 	return (PSM_SUCCESS);
1546 }
1547 /*
1548  * Return 1 if CMT load balancing policies should be
1549  * implemented across instances of the specified hardware
1550  * sharing relationship.
1551  */
1552 int
1553 pg_cmt_load_bal_hw(pghw_type_t hw)
1554 {
1555 	if (hw == PGHW_IPIPE ||
1556 	    hw == PGHW_FPU ||
1557 	    hw == PGHW_CHIP)
1558 		return (1);
1559 	else
1560 		return (0);
1561 }
1562 /*
1563  * Return 1 if thread affinity polices should be implemented
1564  * for instances of the specifed hardware sharing relationship.
1565  */
1566 int
1567 pg_cmt_affinity_hw(pghw_type_t hw)
1568 {
1569 	if (hw == PGHW_CACHE)
1570 		return (1);
1571 	else
1572 		return (0);
1573 }
1574