xref: /illumos-gate/usr/src/uts/i86pc/os/mp_machdep.c (revision b531f6d16eb39863e7bbc34773fb7ef7a282a0a2)
1 
2 /*
3  * CDDL HEADER START
4  *
5  * The contents of this file are subject to the terms of the
6  * Common Development and Distribution License (the "License").
7  * You may not use this file except in compliance with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 /*
26  * Copyright (c) 2009-2010, Intel Corporation.
27  * All rights reserved.
28  * Copyright 2018 Joyent, Inc.
29  */
30 
31 #define	PSMI_1_7
32 #include <sys/smp_impldefs.h>
33 #include <sys/psm.h>
34 #include <sys/psm_modctl.h>
35 #include <sys/pit.h>
36 #include <sys/cmn_err.h>
37 #include <sys/strlog.h>
38 #include <sys/clock.h>
39 #include <sys/debug.h>
40 #include <sys/rtc.h>
41 #include <sys/x86_archext.h>
42 #include <sys/cpupart.h>
43 #include <sys/cpuvar.h>
44 #include <sys/cpu_event.h>
45 #include <sys/cmt.h>
46 #include <sys/cpu.h>
47 #include <sys/disp.h>
48 #include <sys/archsystm.h>
49 #include <sys/machsystm.h>
50 #include <sys/sysmacros.h>
51 #include <sys/memlist.h>
52 #include <sys/param.h>
53 #include <sys/promif.h>
54 #include <sys/cpu_pm.h>
55 #if defined(__xpv)
56 #include <sys/hypervisor.h>
57 #endif
58 #include <sys/mach_intr.h>
59 #include <vm/hat_i86.h>
60 #include <sys/kdi_machimpl.h>
61 #include <sys/sdt.h>
62 #include <sys/hpet.h>
63 #include <sys/sunddi.h>
64 #include <sys/sunndi.h>
65 #include <sys/cpc_pcbe.h>
66 
67 #define	OFFSETOF(s, m)		(size_t)(&(((s *)0)->m))
68 
69 /*
70  *	Local function prototypes
71  */
72 static int mp_disable_intr(processorid_t cpun);
73 static void mp_enable_intr(processorid_t cpun);
74 static void mach_init();
75 static void mach_picinit();
76 static int machhztomhz(uint64_t cpu_freq_hz);
77 static uint64_t mach_getcpufreq(void);
78 static void mach_fixcpufreq(void);
79 static int mach_clkinit(int, int *);
80 static void mach_smpinit(void);
81 static int mach_softlvl_to_vect(int ipl);
82 static void mach_get_platform(int owner);
83 static void mach_construct_info();
84 static int mach_translate_irq(dev_info_t *dip, int irqno);
85 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
86     psm_intr_op_t, int *);
87 static void mach_notify_error(int level, char *errmsg);
88 static hrtime_t dummy_hrtime(void);
89 static void dummy_scalehrtime(hrtime_t *);
90 static uint64_t dummy_unscalehrtime(hrtime_t);
91 void cpu_idle(void);
92 static void cpu_wakeup(cpu_t *, int);
93 #ifndef __xpv
94 void cpu_idle_mwait(void);
95 static void cpu_wakeup_mwait(cpu_t *, int);
96 #endif
97 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp);
98 
99 /*
100  *	External reference functions
101  */
102 extern void return_instr();
103 extern uint64_t freq_tsc(uint32_t *);
104 #if defined(__i386)
105 extern uint64_t freq_notsc(uint32_t *);
106 #endif
107 extern void pc_gethrestime(timestruc_t *);
108 extern int cpuid_get_coreid(cpu_t *);
109 extern int cpuid_get_chipid(cpu_t *);
110 
111 /*
112  *	PSM functions initialization
113  */
114 void (*psm_shutdownf)(int, int)	= (void (*)(int, int))return_instr;
115 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr;
116 void (*psm_notifyf)(int)	= (void (*)(int))return_instr;
117 void (*psm_set_idle_cpuf)(int)	= (void (*)(int))return_instr;
118 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr;
119 void (*psminitf)()		= mach_init;
120 void (*picinitf)()		= return_instr;
121 int (*clkinitf)(int, int *)	= (int (*)(int, int *))return_instr;
122 int (*ap_mlsetup)()		= (int (*)(void))return_instr;
123 void (*send_dirintf)()		= return_instr;
124 void (*setspl)(int)		= (void (*)(int))return_instr;
125 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
126 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
127 int (*get_pending_spl)(void)	= (int (*)(void))return_instr;
128 int (*addintr)(void *, int, avfunc, char *, int, caddr_t, caddr_t,
129     uint64_t *, dev_info_t *) = NULL;
130 void (*remintr)(void *, int, avfunc, int) = NULL;
131 void (*kdisetsoftint)(int, struct av_softinfo *)=
132 	(void (*)(int, struct av_softinfo *))return_instr;
133 void (*setsoftint)(int, struct av_softinfo *)=
134 	(void (*)(int, struct av_softinfo *))return_instr;
135 int (*slvltovect)(int)		= (int (*)(int))return_instr;
136 int (*setlvl)(int, int *)	= (int (*)(int, int *))return_instr;
137 void (*setlvlx)(int, int)	= (void (*)(int, int))return_instr;
138 int (*psm_disable_intr)(int)	= mp_disable_intr;
139 void (*psm_enable_intr)(int)	= mp_enable_intr;
140 hrtime_t (*gethrtimef)(void)	= dummy_hrtime;
141 hrtime_t (*gethrtimeunscaledf)(void)	= dummy_hrtime;
142 void (*scalehrtimef)(hrtime_t *)	= dummy_scalehrtime;
143 uint64_t (*unscalehrtimef)(hrtime_t)	= dummy_unscalehrtime;
144 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq;
145 void (*gethrestimef)(timestruc_t *) = pc_gethrestime;
146 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL;
147 int (*psm_get_clockirq)(int) = NULL;
148 int (*psm_get_ipivect)(int, int) = NULL;
149 uchar_t (*psm_get_ioapicid)(uchar_t) = NULL;
150 uint32_t (*psm_get_localapicid)(uint32_t) = NULL;
151 uchar_t (*psm_xlate_vector_by_irq)(uchar_t) = NULL;
152 int (*psm_get_pir_ipivect)(void) = NULL;
153 void (*psm_send_pir_ipi)(processorid_t) = NULL;
154 void (*psm_cmci_setup)(processorid_t, boolean_t) = NULL;
155 
156 int (*psm_clkinit)(int) = NULL;
157 void (*psm_timer_reprogram)(hrtime_t) = NULL;
158 void (*psm_timer_enable)(void) = NULL;
159 void (*psm_timer_disable)(void) = NULL;
160 void (*psm_post_cyclic_setup)(void *arg) = NULL;
161 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t,
162     int *) = mach_intr_ops;
163 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *))
164     return_instr;
165 
166 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr;
167 void (*hrtime_tick)(void)	= return_instr;
168 
169 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo;
170 int (*psm_cpu_get_devinfo)(cpu_t *, dev_info_t **) = NULL;
171 
172 /* global IRM pool for APIX (PSM) module */
173 ddi_irm_pool_t *apix_irm_pool_p = NULL;
174 
175 /*
176  * True if the generic TSC code is our source of hrtime, rather than whatever
177  * the PSM can provide.
178  */
179 #ifdef __xpv
180 int tsc_gethrtime_enable = 0;
181 #else
182 int tsc_gethrtime_enable = 1;
183 #endif
184 int tsc_gethrtime_initted = 0;
185 
186 /*
187  * True if the hrtime implementation is "hires"; namely, better than microdata.
188  */
189 int gethrtime_hires = 0;
190 
191 /*
192  * Local Static Data
193  */
194 static struct psm_ops mach_ops;
195 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL};
196 static ushort_t mach_ver[4] = {0, 0, 0, 0};
197 
198 /*
199  * virtualization support for psm
200  */
201 void *psm_vt_ops = NULL;
202 /*
203  * If non-zero, idle cpus will become "halted" when there's
204  * no work to do.
205  */
206 int	idle_cpu_use_hlt = 1;
207 
208 #ifndef __xpv
209 /*
210  * If non-zero, idle cpus will use mwait if available to halt instead of hlt.
211  */
212 int	idle_cpu_prefer_mwait = 1;
213 /*
214  * Set to 0 to avoid MONITOR+CLFLUSH assertion.
215  */
216 int	idle_cpu_assert_cflush_monitor = 1;
217 
218 /*
219  * If non-zero, idle cpus will not use power saving Deep C-States idle loop.
220  */
221 int	idle_cpu_no_deep_c = 0;
222 /*
223  * Non-power saving idle loop and wakeup pointers.
224  * Allows user to toggle Deep Idle power saving feature on/off.
225  */
226 void	(*non_deep_idle_cpu)() = cpu_idle;
227 void	(*non_deep_idle_disp_enq_thread)(cpu_t *, int);
228 
229 /*
230  * Object for the kernel to access the HPET.
231  */
232 hpet_t hpet;
233 
234 #endif	/* ifndef __xpv */
235 
236 uint_t cp_haltset_fanout = 0;
237 
238 /*ARGSUSED*/
239 int
240 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw)
241 {
242 	switch (hw) {
243 	case PGHW_IPIPE:
244 		if (is_x86_feature(x86_featureset, X86FSET_HTT)) {
245 			/*
246 			 * Hyper-threading is SMT
247 			 */
248 			return (1);
249 		} else {
250 			return (0);
251 		}
252 	case PGHW_FPU:
253 		if (cpuid_get_cores_per_compunit(cp) > 1)
254 			return (1);
255 		else
256 			return (0);
257 	case PGHW_PROCNODE:
258 		if (cpuid_get_procnodes_per_pkg(cp) > 1)
259 			return (1);
260 		else
261 			return (0);
262 	case PGHW_CHIP:
263 		if (is_x86_feature(x86_featureset, X86FSET_CMP) ||
264 		    is_x86_feature(x86_featureset, X86FSET_HTT))
265 			return (1);
266 		else
267 			return (0);
268 	case PGHW_CACHE:
269 		if (cpuid_get_ncpu_sharing_last_cache(cp) > 1)
270 			return (1);
271 		else
272 			return (0);
273 	case PGHW_POW_ACTIVE:
274 		if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1)
275 			return (1);
276 		else
277 			return (0);
278 	case PGHW_POW_IDLE:
279 		if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1)
280 			return (1);
281 		else
282 			return (0);
283 	default:
284 		return (0);
285 	}
286 }
287 
288 /*
289  * Compare two CPUs and see if they have a pghw_type_t sharing relationship
290  * If pghw_type_t is an unsupported hardware type, then return -1
291  */
292 int
293 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw)
294 {
295 	id_t pgp_a, pgp_b;
296 
297 	pgp_a = pg_plat_hw_instance_id(cpu_a, hw);
298 	pgp_b = pg_plat_hw_instance_id(cpu_b, hw);
299 
300 	if (pgp_a == -1 || pgp_b == -1)
301 		return (-1);
302 
303 	return (pgp_a == pgp_b);
304 }
305 
306 /*
307  * Return a physical instance identifier for known hardware sharing
308  * relationships
309  */
310 id_t
311 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw)
312 {
313 	switch (hw) {
314 	case PGHW_IPIPE:
315 		return (cpuid_get_coreid(cpu));
316 	case PGHW_CACHE:
317 		return (cpuid_get_last_lvl_cacheid(cpu));
318 	case PGHW_FPU:
319 		return (cpuid_get_compunitid(cpu));
320 	case PGHW_PROCNODE:
321 		return (cpuid_get_procnodeid(cpu));
322 	case PGHW_CHIP:
323 		return (cpuid_get_chipid(cpu));
324 	case PGHW_POW_ACTIVE:
325 		return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE));
326 	case PGHW_POW_IDLE:
327 		return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE));
328 	default:
329 		return (-1);
330 	}
331 }
332 
333 /*
334  * Express preference for optimizing for sharing relationship
335  * hw1 vs hw2
336  */
337 pghw_type_t
338 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2)
339 {
340 	int i, rank1, rank2;
341 
342 	static pghw_type_t hw_hier[] = {
343 		PGHW_IPIPE,
344 		PGHW_CACHE,
345 		PGHW_FPU,
346 		PGHW_PROCNODE,
347 		PGHW_CHIP,
348 		PGHW_POW_IDLE,
349 		PGHW_POW_ACTIVE,
350 		PGHW_NUM_COMPONENTS
351 	};
352 
353 	rank1 = 0;
354 	rank2 = 0;
355 
356 	for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) {
357 		if (hw_hier[i] == hw1)
358 			rank1 = i;
359 		if (hw_hier[i] == hw2)
360 			rank2 = i;
361 	}
362 
363 	if (rank1 > rank2)
364 		return (hw1);
365 	else
366 		return (hw2);
367 }
368 
369 /*
370  * Override the default CMT dispatcher policy for the specified
371  * hardware sharing relationship
372  */
373 pg_cmt_policy_t
374 pg_plat_cmt_policy(pghw_type_t hw)
375 {
376 	/*
377 	 * For shared caches, also load balance across them to
378 	 * maximize aggregate cache capacity
379 	 *
380 	 * On AMD family 0x15 CPUs, cores come in pairs called
381 	 * compute units, sharing the FPU and the I$ and L2
382 	 * caches. Use balancing and cache affinity.
383 	 */
384 	switch (hw) {
385 	case PGHW_FPU:
386 	case PGHW_CACHE:
387 		return (CMT_BALANCE|CMT_AFFINITY);
388 	default:
389 		return (CMT_NO_POLICY);
390 	}
391 }
392 
393 id_t
394 pg_plat_get_core_id(cpu_t *cpu)
395 {
396 	return ((id_t)cpuid_get_coreid(cpu));
397 }
398 
399 void
400 cmp_set_nosteal_interval(void)
401 {
402 	/* Set the nosteal interval (used by disp_getbest()) to 100us */
403 	nosteal_nsec = 100000UL;
404 }
405 
406 /*
407  * Routine to ensure initial callers to hrtime gets 0 as return
408  */
409 static hrtime_t
410 dummy_hrtime(void)
411 {
412 	return (0);
413 }
414 
415 /* ARGSUSED */
416 static void
417 dummy_scalehrtime(hrtime_t *ticks)
418 {}
419 
420 static uint64_t
421 dummy_unscalehrtime(hrtime_t nsecs)
422 {
423 	return ((uint64_t)nsecs);
424 }
425 
426 /*
427  * Supports Deep C-State power saving idle loop.
428  */
429 void
430 cpu_idle_adaptive(void)
431 {
432 	(*CPU->cpu_m.mcpu_idle_cpu)();
433 }
434 
435 /*
436  * Function called by CPU idle notification framework to check whether CPU
437  * has been awakened. It will be called with interrupt disabled.
438  * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle
439  * notification framework.
440  */
441 /*ARGSUSED*/
442 static void
443 cpu_idle_check_wakeup(void *arg)
444 {
445 	/*
446 	 * Toggle interrupt flag to detect pending interrupts.
447 	 * If interrupt happened, do_interrupt() will notify CPU idle
448 	 * notification framework so no need to call cpu_idle_exit() here.
449 	 */
450 	sti();
451 	SMT_PAUSE();
452 	cli();
453 }
454 
455 /*
456  * Idle the present CPU until wakened via an interrupt
457  */
458 void
459 cpu_idle(void)
460 {
461 	cpu_t		*cpup = CPU;
462 	processorid_t	cpu_sid = cpup->cpu_seqid;
463 	cpupart_t	*cp = cpup->cpu_part;
464 	int		hset_update = 1;
465 
466 	/*
467 	 * If this CPU is online, and there's multiple CPUs
468 	 * in the system, then we should notate our halting
469 	 * by adding ourselves to the partition's halted CPU
470 	 * bitmap. This allows other CPUs to find/awaken us when
471 	 * work becomes available.
472 	 */
473 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
474 		hset_update = 0;
475 
476 	/*
477 	 * Add ourselves to the partition's halted CPUs bitmap
478 	 * and set our HALTED flag, if necessary.
479 	 *
480 	 * When a thread becomes runnable, it is placed on the queue
481 	 * and then the halted CPU bitmap is checked to determine who
482 	 * (if anyone) should be awakened. We therefore need to first
483 	 * add ourselves to the bitmap, and and then check if there
484 	 * is any work available. The order is important to prevent a race
485 	 * that can lead to work languishing on a run queue somewhere while
486 	 * this CPU remains halted.
487 	 *
488 	 * Either the producing CPU will see we're halted and will awaken us,
489 	 * or this CPU will see the work available in disp_anywork().
490 	 *
491 	 * Note that memory barriers after updating the HALTED flag
492 	 * are not necessary since an atomic operation (updating the bitset)
493 	 * immediately follows. On x86 the atomic operation acts as a
494 	 * memory barrier for the update of cpu_disp_flags.
495 	 */
496 	if (hset_update) {
497 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
498 		bitset_atomic_add(&cp->cp_haltset, cpu_sid);
499 	}
500 
501 	/*
502 	 * Check to make sure there's really nothing to do.
503 	 * Work destined for this CPU may become available after
504 	 * this check. We'll be notified through the clearing of our
505 	 * bit in the halted CPU bitmap, and a poke.
506 	 */
507 	if (disp_anywork()) {
508 		if (hset_update) {
509 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
510 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
511 		}
512 		return;
513 	}
514 
515 	/*
516 	 * We're on our way to being halted.
517 	 *
518 	 * Disable interrupts now, so that we'll awaken immediately
519 	 * after halting if someone tries to poke us between now and
520 	 * the time we actually halt.
521 	 *
522 	 * We check for the presence of our bit after disabling interrupts.
523 	 * If it's cleared, we'll return. If the bit is cleared after
524 	 * we check then the poke will pop us out of the halted state.
525 	 *
526 	 * This means that the ordering of the poke and the clearing
527 	 * of the bit by cpu_wakeup is important.
528 	 * cpu_wakeup() must clear, then poke.
529 	 * cpu_idle() must disable interrupts, then check for the bit.
530 	 */
531 	cli();
532 
533 	if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) {
534 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
535 		sti();
536 		return;
537 	}
538 
539 	/*
540 	 * The check for anything locally runnable is here for performance
541 	 * and isn't needed for correctness. disp_nrunnable ought to be
542 	 * in our cache still, so it's inexpensive to check, and if there
543 	 * is anything runnable we won't have to wait for the poke.
544 	 */
545 	if (cpup->cpu_disp->disp_nrunnable != 0) {
546 		if (hset_update) {
547 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
548 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
549 		}
550 		sti();
551 		return;
552 	}
553 
554 	if (cpu_idle_enter(IDLE_STATE_C1, 0,
555 	    cpu_idle_check_wakeup, NULL) == 0) {
556 		mach_cpu_idle();
557 		cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
558 	}
559 
560 	/*
561 	 * We're no longer halted
562 	 */
563 	if (hset_update) {
564 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
565 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
566 	}
567 }
568 
569 
570 /*
571  * If "cpu" is halted, then wake it up clearing its halted bit in advance.
572  * Otherwise, see if other CPUs in the cpu partition are halted and need to
573  * be woken up so that they can steal the thread we placed on this CPU.
574  * This function is only used on MP systems.
575  */
576 static void
577 cpu_wakeup(cpu_t *cpu, int bound)
578 {
579 	uint_t		cpu_found;
580 	processorid_t	cpu_sid;
581 	cpupart_t	*cp;
582 
583 	cp = cpu->cpu_part;
584 	cpu_sid = cpu->cpu_seqid;
585 	if (bitset_in_set(&cp->cp_haltset, cpu_sid)) {
586 		/*
587 		 * Clear the halted bit for that CPU since it will be
588 		 * poked in a moment.
589 		 */
590 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
591 		/*
592 		 * We may find the current CPU present in the halted cpuset
593 		 * if we're in the context of an interrupt that occurred
594 		 * before we had a chance to clear our bit in cpu_idle().
595 		 * Poking ourself is obviously unnecessary, since if
596 		 * we're here, we're not halted.
597 		 */
598 		if (cpu != CPU)
599 			poke_cpu(cpu->cpu_id);
600 		return;
601 	} else {
602 		/*
603 		 * This cpu isn't halted, but it's idle or undergoing a
604 		 * context switch. No need to awaken anyone else.
605 		 */
606 		if (cpu->cpu_thread == cpu->cpu_idle_thread ||
607 		    cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL)
608 			return;
609 	}
610 
611 	/*
612 	 * No need to wake up other CPUs if this is for a bound thread.
613 	 */
614 	if (bound)
615 		return;
616 
617 	/*
618 	 * The CPU specified for wakeup isn't currently halted, so check
619 	 * to see if there are any other halted CPUs in the partition,
620 	 * and if there are then awaken one.
621 	 */
622 	do {
623 		cpu_found = bitset_find(&cp->cp_haltset);
624 		if (cpu_found == (uint_t)-1)
625 			return;
626 	} while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0);
627 
628 	if (cpu_found != CPU->cpu_seqid) {
629 		poke_cpu(cpu_seq[cpu_found]->cpu_id);
630 	}
631 }
632 
633 #ifndef __xpv
634 /*
635  * Function called by CPU idle notification framework to check whether CPU
636  * has been awakened. It will be called with interrupt disabled.
637  * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle
638  * notification framework.
639  */
640 static void
641 cpu_idle_mwait_check_wakeup(void *arg)
642 {
643 	volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg;
644 
645 	ASSERT(arg != NULL);
646 	if (*mcpu_mwait != MWAIT_HALTED) {
647 		/*
648 		 * CPU has been awakened, notify CPU idle notification system.
649 		 */
650 		cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
651 	} else {
652 		/*
653 		 * Toggle interrupt flag to detect pending interrupts.
654 		 * If interrupt happened, do_interrupt() will notify CPU idle
655 		 * notification framework so no need to call cpu_idle_exit()
656 		 * here.
657 		 */
658 		sti();
659 		SMT_PAUSE();
660 		cli();
661 	}
662 }
663 
664 /*
665  * Idle the present CPU until awakened via touching its monitored line
666  */
667 void
668 cpu_idle_mwait(void)
669 {
670 	volatile uint32_t	*mcpu_mwait = CPU->cpu_m.mcpu_mwait;
671 	cpu_t			*cpup = CPU;
672 	processorid_t		cpu_sid = cpup->cpu_seqid;
673 	cpupart_t		*cp = cpup->cpu_part;
674 	int			hset_update = 1;
675 
676 	/*
677 	 * Set our mcpu_mwait here, so we can tell if anyone tries to
678 	 * wake us between now and when we call mwait.  No other cpu will
679 	 * attempt to set our mcpu_mwait until we add ourself to the halted
680 	 * CPU bitmap.
681 	 */
682 	*mcpu_mwait = MWAIT_HALTED;
683 
684 	/*
685 	 * If this CPU is online, and there's multiple CPUs
686 	 * in the system, then we should note our halting
687 	 * by adding ourselves to the partition's halted CPU
688 	 * bitmap. This allows other CPUs to find/awaken us when
689 	 * work becomes available.
690 	 */
691 	if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
692 		hset_update = 0;
693 
694 	/*
695 	 * Add ourselves to the partition's halted CPUs bitmap
696 	 * and set our HALTED flag, if necessary.
697 	 *
698 	 * When a thread becomes runnable, it is placed on the queue
699 	 * and then the halted CPU bitmap is checked to determine who
700 	 * (if anyone) should be awakened. We therefore need to first
701 	 * add ourselves to the bitmap, and and then check if there
702 	 * is any work available.
703 	 *
704 	 * Note that memory barriers after updating the HALTED flag
705 	 * are not necessary since an atomic operation (updating the bitmap)
706 	 * immediately follows. On x86 the atomic operation acts as a
707 	 * memory barrier for the update of cpu_disp_flags.
708 	 */
709 	if (hset_update) {
710 		cpup->cpu_disp_flags |= CPU_DISP_HALTED;
711 		bitset_atomic_add(&cp->cp_haltset, cpu_sid);
712 	}
713 
714 	/*
715 	 * Check to make sure there's really nothing to do.
716 	 * Work destined for this CPU may become available after
717 	 * this check. We'll be notified through the clearing of our
718 	 * bit in the halted CPU bitmap, and a write to our mcpu_mwait.
719 	 *
720 	 * disp_anywork() checks disp_nrunnable, so we do not have to later.
721 	 */
722 	if (disp_anywork()) {
723 		if (hset_update) {
724 			cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
725 			bitset_atomic_del(&cp->cp_haltset, cpu_sid);
726 		}
727 		return;
728 	}
729 
730 	/*
731 	 * We're on our way to being halted.
732 	 * To avoid a lost wakeup, arm the monitor before checking if another
733 	 * cpu wrote to mcpu_mwait to wake us up.
734 	 */
735 	i86_monitor(mcpu_mwait, 0, 0);
736 	if (*mcpu_mwait == MWAIT_HALTED) {
737 		if (cpu_idle_enter(IDLE_STATE_C1, 0,
738 		    cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) {
739 			if (*mcpu_mwait == MWAIT_HALTED) {
740 				i86_mwait(0, 0);
741 			}
742 			cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
743 		}
744 	}
745 
746 	/*
747 	 * We're no longer halted
748 	 */
749 	if (hset_update) {
750 		cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
751 		bitset_atomic_del(&cp->cp_haltset, cpu_sid);
752 	}
753 }
754 
755 /*
756  * If "cpu" is halted in mwait, then wake it up clearing its halted bit in
757  * advance.  Otherwise, see if other CPUs in the cpu partition are halted and
758  * need to be woken up so that they can steal the thread we placed on this CPU.
759  * This function is only used on MP systems.
760  */
761 static void
762 cpu_wakeup_mwait(cpu_t *cp, int bound)
763 {
764 	cpupart_t	*cpu_part;
765 	uint_t		cpu_found;
766 	processorid_t	cpu_sid;
767 
768 	cpu_part = cp->cpu_part;
769 	cpu_sid = cp->cpu_seqid;
770 
771 	/*
772 	 * Clear the halted bit for that CPU since it will be woken up
773 	 * in a moment.
774 	 */
775 	if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) {
776 		/*
777 		 * Clear the halted bit for that CPU since it will be
778 		 * poked in a moment.
779 		 */
780 		bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid);
781 		/*
782 		 * We may find the current CPU present in the halted cpuset
783 		 * if we're in the context of an interrupt that occurred
784 		 * before we had a chance to clear our bit in cpu_idle().
785 		 * Waking ourself is obviously unnecessary, since if
786 		 * we're here, we're not halted.
787 		 *
788 		 * monitor/mwait wakeup via writing to our cache line is
789 		 * harmless and less expensive than always checking if we
790 		 * are waking ourself which is an uncommon case.
791 		 */
792 		MWAIT_WAKEUP(cp);	/* write to monitored line */
793 		return;
794 	} else {
795 		/*
796 		 * This cpu isn't halted, but it's idle or undergoing a
797 		 * context switch. No need to awaken anyone else.
798 		 */
799 		if (cp->cpu_thread == cp->cpu_idle_thread ||
800 		    cp->cpu_disp_flags & CPU_DISP_DONTSTEAL)
801 			return;
802 	}
803 
804 	/*
805 	 * No need to wake up other CPUs if the thread we just enqueued
806 	 * is bound.
807 	 */
808 	if (bound || ncpus == 1)
809 		return;
810 
811 	/*
812 	 * See if there's any other halted CPUs. If there are, then
813 	 * select one, and awaken it.
814 	 * It's possible that after we find a CPU, somebody else
815 	 * will awaken it before we get the chance.
816 	 * In that case, look again.
817 	 */
818 	do {
819 		cpu_found = bitset_find(&cpu_part->cp_haltset);
820 		if (cpu_found == (uint_t)-1)
821 			return;
822 	} while (bitset_atomic_test_and_del(&cpu_part->cp_haltset,
823 	    cpu_found) < 0);
824 
825 	/*
826 	 * Do not check if cpu_found is ourself as monitor/mwait
827 	 * wakeup is cheap.
828 	 */
829 	MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */
830 }
831 
832 #endif
833 
834 void (*cpu_pause_handler)(volatile char *) = NULL;
835 
836 static int
837 mp_disable_intr(int cpun)
838 {
839 	/*
840 	 * switch to the offline cpu
841 	 */
842 	affinity_set(cpun);
843 	/*
844 	 * raise ipl to just below cross call
845 	 */
846 	splx(XC_SYS_PIL - 1);
847 	/*
848 	 *	set base spl to prevent the next swtch to idle from
849 	 *	lowering back to ipl 0
850 	 */
851 	CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1));
852 	set_base_spl();
853 	affinity_clear();
854 	return (DDI_SUCCESS);
855 }
856 
857 static void
858 mp_enable_intr(int cpun)
859 {
860 	/*
861 	 * switch to the online cpu
862 	 */
863 	affinity_set(cpun);
864 	/*
865 	 * clear the interrupt active mask
866 	 */
867 	CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1));
868 	set_base_spl();
869 	(void) spl0();
870 	affinity_clear();
871 }
872 
873 static void
874 mach_get_platform(int owner)
875 {
876 	void		**srv_opsp;
877 	void		**clt_opsp;
878 	int		i;
879 	int		total_ops;
880 
881 	/* fix up psm ops */
882 	srv_opsp = (void **)mach_set[0];
883 	clt_opsp = (void **)mach_set[owner];
884 	if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01)
885 		total_ops = sizeof (struct psm_ops_ver01) /
886 		    sizeof (void (*)(void));
887 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1)
888 		/* no psm_notify_func */
889 		total_ops = OFFSETOF(struct psm_ops, psm_notify_func) /
890 		    sizeof (void (*)(void));
891 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2)
892 		/* no psm_timer funcs */
893 		total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) /
894 		    sizeof (void (*)(void));
895 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3)
896 		/* no psm_preshutdown function */
897 		total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) /
898 		    sizeof (void (*)(void));
899 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4)
900 		/* no psm_intr_ops function */
901 		total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) /
902 		    sizeof (void (*)(void));
903 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_5)
904 		/* no psm_state function */
905 		total_ops = OFFSETOF(struct psm_ops, psm_state) /
906 		    sizeof (void (*)(void));
907 	else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_6)
908 		/* no psm_cpu_ops function */
909 		total_ops = OFFSETOF(struct psm_ops, psm_cpu_ops) /
910 		    sizeof (void (*)(void));
911 	else
912 		total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void));
913 
914 	/*
915 	 * Save the version of the PSM module, in case we need to
916 	 * behave differently based on version.
917 	 */
918 	mach_ver[0] = mach_ver[owner];
919 
920 	for (i = 0; i < total_ops; i++)
921 		if (clt_opsp[i] != NULL)
922 			srv_opsp[i] = clt_opsp[i];
923 }
924 
925 static void
926 mach_construct_info()
927 {
928 	struct psm_sw *swp;
929 	int	mach_cnt[PSM_OWN_OVERRIDE+1] = {0};
930 	int	conflict_owner = 0;
931 
932 	if (psmsw->psw_forw == psmsw)
933 		panic("No valid PSM modules found");
934 	mutex_enter(&psmsw_lock);
935 	for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
936 		if (!(swp->psw_flag & PSM_MOD_IDENTIFY))
937 			continue;
938 		mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops;
939 		mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version;
940 		mach_cnt[swp->psw_infop->p_owner]++;
941 	}
942 	mutex_exit(&psmsw_lock);
943 
944 	mach_get_platform(PSM_OWN_SYS_DEFAULT);
945 
946 	/* check to see are there any conflicts */
947 	if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1)
948 		conflict_owner = PSM_OWN_EXCLUSIVE;
949 	if (mach_cnt[PSM_OWN_OVERRIDE] > 1)
950 		conflict_owner = PSM_OWN_OVERRIDE;
951 	if (conflict_owner) {
952 		/* remove all psm modules except uppc */
953 		cmn_err(CE_WARN,
954 		    "Conflicts detected on the following PSM modules:");
955 		mutex_enter(&psmsw_lock);
956 		for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
957 			if (swp->psw_infop->p_owner == conflict_owner)
958 				cmn_err(CE_WARN, "%s ",
959 				    swp->psw_infop->p_mach_idstring);
960 		}
961 		mutex_exit(&psmsw_lock);
962 		cmn_err(CE_WARN,
963 		    "Setting the system back to SINGLE processor mode!");
964 		cmn_err(CE_WARN,
965 		    "Please edit /etc/mach to remove the invalid PSM module.");
966 		return;
967 	}
968 
969 	if (mach_set[PSM_OWN_EXCLUSIVE])
970 		mach_get_platform(PSM_OWN_EXCLUSIVE);
971 
972 	if (mach_set[PSM_OWN_OVERRIDE])
973 		mach_get_platform(PSM_OWN_OVERRIDE);
974 }
975 
976 static void
977 mach_init()
978 {
979 	struct psm_ops  *pops;
980 
981 	mach_construct_info();
982 
983 	pops = mach_set[0];
984 
985 	/* register the interrupt and clock initialization rotuines */
986 	picinitf = mach_picinit;
987 	clkinitf = mach_clkinit;
988 	psm_get_clockirq = pops->psm_get_clockirq;
989 
990 	/* register the interrupt setup code */
991 	slvltovect = mach_softlvl_to_vect;
992 	addspl	= pops->psm_addspl;
993 	delspl	= pops->psm_delspl;
994 
995 	if (pops->psm_translate_irq)
996 		psm_translate_irq = pops->psm_translate_irq;
997 	if (pops->psm_intr_ops)
998 		psm_intr_ops = pops->psm_intr_ops;
999 
1000 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4)
1001 	/*
1002 	 * Time-of-day functionality now handled in TOD modules.
1003 	 * (Warn about PSM modules that think that we're going to use
1004 	 * their ops vectors.)
1005 	 */
1006 	if (pops->psm_tod_get)
1007 		cmn_err(CE_WARN, "obsolete psm_tod_get op %p",
1008 		    (void *)pops->psm_tod_get);
1009 
1010 	if (pops->psm_tod_set)
1011 		cmn_err(CE_WARN, "obsolete psm_tod_set op %p",
1012 		    (void *)pops->psm_tod_set);
1013 #endif
1014 
1015 	if (pops->psm_notify_error) {
1016 		psm_notify_error = mach_notify_error;
1017 		notify_error = pops->psm_notify_error;
1018 	}
1019 
1020 	(*pops->psm_softinit)();
1021 
1022 	/*
1023 	 * Initialize the dispatcher's function hooks to enable CPU halting
1024 	 * when idle.  Set both the deep-idle and non-deep-idle hooks.
1025 	 *
1026 	 * Assume we can use power saving deep-idle loop cpu_idle_adaptive.
1027 	 * Platform deep-idle driver will reset our idle loop to
1028 	 * non_deep_idle_cpu if power saving deep-idle feature is not available.
1029 	 *
1030 	 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle)
1031 	 * or idle_cpu_prefer_mwait is not set.
1032 	 * Allocate monitor/mwait buffer for cpu0.
1033 	 */
1034 #ifndef __xpv
1035 	non_deep_idle_disp_enq_thread = disp_enq_thread;
1036 #endif
1037 	if (idle_cpu_use_hlt) {
1038 		idle_cpu = cpu_idle_adaptive;
1039 		CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
1040 #ifndef __xpv
1041 		if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
1042 		    idle_cpu_prefer_mwait) {
1043 			CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);
1044 			/*
1045 			 * Protect ourself from insane mwait size.
1046 			 */
1047 			if (CPU->cpu_m.mcpu_mwait == NULL) {
1048 #ifdef DEBUG
1049 				cmn_err(CE_NOTE, "Using hlt idle.  Cannot "
1050 				    "handle cpu 0 mwait size.");
1051 #endif
1052 				idle_cpu_prefer_mwait = 0;
1053 				CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
1054 			} else {
1055 				CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
1056 			}
1057 		} else {
1058 			CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
1059 		}
1060 		non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu;
1061 
1062 		/*
1063 		 * Disable power saving deep idle loop?
1064 		 */
1065 		if (idle_cpu_no_deep_c) {
1066 			idle_cpu = non_deep_idle_cpu;
1067 		}
1068 #endif
1069 	}
1070 
1071 	mach_smpinit();
1072 }
1073 
1074 static void
1075 mach_smpinit(void)
1076 {
1077 	struct psm_ops  *pops;
1078 	processorid_t cpu_id;
1079 	int cnt;
1080 	cpuset_t cpumask;
1081 
1082 	pops = mach_set[0];
1083 	CPUSET_ZERO(cpumask);
1084 
1085 	cpu_id = -1;
1086 	cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
1087 	/*
1088 	 * Only add boot_ncpus CPUs to mp_cpus. Other CPUs will be handled
1089 	 * by CPU DR driver at runtime.
1090 	 */
1091 	for (cnt = 0; cpu_id != -1 && cnt < boot_ncpus; cnt++) {
1092 		CPUSET_ADD(cpumask, cpu_id);
1093 		cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
1094 	}
1095 
1096 	mp_cpus = cpumask;
1097 
1098 	/* MP related routines */
1099 	ap_mlsetup = pops->psm_post_cpu_start;
1100 	send_dirintf = pops->psm_send_ipi;
1101 
1102 	/* optional MP related routines */
1103 	if (pops->psm_shutdown)
1104 		psm_shutdownf = pops->psm_shutdown;
1105 	if (pops->psm_preshutdown)
1106 		psm_preshutdownf = pops->psm_preshutdown;
1107 	if (pops->psm_notify_func)
1108 		psm_notifyf = pops->psm_notify_func;
1109 	if (pops->psm_set_idlecpu)
1110 		psm_set_idle_cpuf = pops->psm_set_idlecpu;
1111 	if (pops->psm_unset_idlecpu)
1112 		psm_unset_idle_cpuf = pops->psm_unset_idlecpu;
1113 
1114 	psm_clkinit = pops->psm_clkinit;
1115 
1116 	if (pops->psm_timer_reprogram)
1117 		psm_timer_reprogram = pops->psm_timer_reprogram;
1118 
1119 	if (pops->psm_timer_enable)
1120 		psm_timer_enable = pops->psm_timer_enable;
1121 
1122 	if (pops->psm_timer_disable)
1123 		psm_timer_disable = pops->psm_timer_disable;
1124 
1125 	if (pops->psm_post_cyclic_setup)
1126 		psm_post_cyclic_setup = pops->psm_post_cyclic_setup;
1127 
1128 	if (pops->psm_state)
1129 		psm_state = pops->psm_state;
1130 
1131 	/*
1132 	 * Set these vectors here so they can be used by Suspend/Resume
1133 	 * on UP machines.
1134 	 */
1135 	if (pops->psm_disable_intr)
1136 		psm_disable_intr = pops->psm_disable_intr;
1137 	if (pops->psm_enable_intr)
1138 		psm_enable_intr  = pops->psm_enable_intr;
1139 
1140 	/*
1141 	 * Set this vector so it can be used by vmbus (for Hyper-V)
1142 	 * Need this even for single-CPU systems.  This works for
1143 	 * "pcplusmp" and "apix" platforms, but not "uppc" (because
1144 	 * "Uni-processor PC" does not provide a _get_ipivect).
1145 	 */
1146 	psm_get_ipivect = pops->psm_get_ipivect;
1147 
1148 	/* check for multiple CPUs */
1149 	if (cnt < 2 && plat_dr_support_cpu() == B_FALSE)
1150 		return;
1151 
1152 	/* check for MP platforms */
1153 	if (pops->psm_cpu_start == NULL)
1154 		return;
1155 
1156 	/*
1157 	 * Set the dispatcher hook to enable cpu "wake up"
1158 	 * when a thread becomes runnable.
1159 	 */
1160 	if (idle_cpu_use_hlt) {
1161 		disp_enq_thread = cpu_wakeup;
1162 #ifndef __xpv
1163 		if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
1164 		    idle_cpu_prefer_mwait)
1165 			disp_enq_thread = cpu_wakeup_mwait;
1166 		non_deep_idle_disp_enq_thread = disp_enq_thread;
1167 #endif
1168 	}
1169 
1170 	psm_get_pir_ipivect = pops->psm_get_pir_ipivect;
1171 	psm_send_pir_ipi = pops->psm_send_pir_ipi;
1172 	psm_cmci_setup = pops->psm_cmci_setup;
1173 
1174 
1175 	(void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr",
1176 	    (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI),
1177 	    NULL, NULL, NULL, NULL);
1178 
1179 	(void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE);
1180 }
1181 
1182 static void
1183 mach_picinit()
1184 {
1185 	struct psm_ops  *pops;
1186 
1187 	pops = mach_set[0];
1188 
1189 	/* register the interrupt handlers */
1190 	setlvl = pops->psm_intr_enter;
1191 	setlvlx = pops->psm_intr_exit;
1192 
1193 	/* initialize the interrupt hardware */
1194 	(*pops->psm_picinit)();
1195 
1196 	/* set interrupt mask for current ipl */
1197 	setspl = pops->psm_setspl;
1198 	cli();
1199 	setspl(CPU->cpu_pri);
1200 }
1201 
1202 uint_t	cpu_freq;	/* MHz */
1203 uint64_t cpu_freq_hz;	/* measured (in hertz) */
1204 
1205 #define	MEGA_HZ		1000000
1206 
1207 #ifdef __xpv
1208 
1209 int xpv_cpufreq_workaround = 1;
1210 int xpv_cpufreq_verbose = 0;
1211 
1212 #else	/* __xpv */
1213 
1214 static uint64_t
1215 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks)
1216 {
1217 	uint64_t cpu_hz;
1218 
1219 	if ((pit_counter == 0) || (*processor_clks == 0) ||
1220 	    (*processor_clks > (((uint64_t)-1) / PIT_HZ)))
1221 		return (0);
1222 
1223 	cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter;
1224 
1225 	return (cpu_hz);
1226 }
1227 
1228 #endif	/* __xpv */
1229 
1230 static uint64_t
1231 mach_getcpufreq(void)
1232 {
1233 #if defined(__xpv)
1234 	vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time;
1235 	uint64_t cpu_hz;
1236 
1237 	/*
1238 	 * During dom0 bringup, it was noted that on at least one older
1239 	 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul
1240 	 * value that is quite wrong (the 3.06GHz clock was reported
1241 	 * as 4.77GHz)
1242 	 *
1243 	 * The curious thing is, that if you stop the kernel at entry,
1244 	 * breakpoint here and inspect the value with kmdb, the value
1245 	 * is correct - but if you don't stop and simply enable the
1246 	 * printf statement (below), you can see the bad value printed
1247 	 * here.  Almost as if something kmdb did caused the hypervisor to
1248 	 * figure it out correctly.  And, note that the hypervisor
1249 	 * eventually -does- figure it out correctly ... if you look at
1250 	 * the field later in the life of dom0, it is correct.
1251 	 *
1252 	 * For now, on dom0, we employ a slightly cheesy workaround of
1253 	 * using the DOM0_PHYSINFO hypercall.
1254 	 */
1255 	if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) {
1256 		cpu_hz = 1000 * xpv_cpu_khz();
1257 	} else {
1258 		cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul;
1259 
1260 		if (vti->tsc_shift < 0)
1261 			cpu_hz <<= -vti->tsc_shift;
1262 		else
1263 			cpu_hz >>= vti->tsc_shift;
1264 	}
1265 
1266 	if (xpv_cpufreq_verbose)
1267 		printf("mach_getcpufreq: system_mul 0x%x, shift %d, "
1268 		    "cpu_hz %" PRId64 "Hz\n",
1269 		    vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz);
1270 
1271 	return (cpu_hz);
1272 #else	/* __xpv */
1273 	uint32_t pit_counter;
1274 	uint64_t processor_clks;
1275 
1276 	if (is_x86_feature(x86_featureset, X86FSET_TSC)) {
1277 		/*
1278 		 * We have a TSC. freq_tsc() knows how to measure the number
1279 		 * of clock cycles sampled against the PIT.
1280 		 */
1281 		ulong_t flags = clear_int_flag();
1282 		processor_clks = freq_tsc(&pit_counter);
1283 		restore_int_flag(flags);
1284 		return (mach_calchz(pit_counter, &processor_clks));
1285 	} else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) {
1286 #if defined(__amd64)
1287 		panic("mach_getcpufreq: no TSC!");
1288 #elif defined(__i386)
1289 		/*
1290 		 * We are a Cyrix based on a 6x86 core or an Intel Pentium
1291 		 * for which freq_notsc() knows how to measure the number of
1292 		 * elapsed clock cycles sampled against the PIT
1293 		 */
1294 		ulong_t flags = clear_int_flag();
1295 		processor_clks = freq_notsc(&pit_counter);
1296 		restore_int_flag(flags);
1297 		return (mach_calchz(pit_counter, &processor_clks));
1298 #endif	/* __i386 */
1299 	}
1300 
1301 	/* We do not know how to calculate cpu frequency for this cpu. */
1302 	return (0);
1303 #endif	/* __xpv */
1304 }
1305 
1306 /*
1307  * If the clock speed of a cpu is found to be reported incorrectly, do not add
1308  * to this array, instead improve the accuracy of the algorithm that determines
1309  * the clock speed of the processor or extend the implementation to support the
1310  * vendor as appropriate. This is here only to support adjusting the speed on
1311  * older slower processors that mach_fixcpufreq() would not be able to account
1312  * for otherwise.
1313  */
1314 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 };
1315 
1316 /*
1317  * On fast processors the clock frequency that is measured may be off by
1318  * a few MHz from the value printed on the part. This is a combination of
1319  * the factors that for such fast parts being off by this much is within
1320  * the tolerances for manufacture and because of the difficulties in the
1321  * measurement that can lead to small error. This function uses some
1322  * heuristics in order to tweak the value that was measured to match what
1323  * is most likely printed on the part.
1324  *
1325  * Some examples:
1326  *	AMD Athlon 1000 mhz measured as 998 mhz
1327  *	Intel Pentium III Xeon 733 mhz measured as 731 mhz
1328  *	Intel Pentium IV 1500 mhz measured as 1495mhz
1329  *
1330  * If in the future this function is no longer sufficient to correct
1331  * for the error in the measurement, then the algorithm used to perform
1332  * the measurement will have to be improved in order to increase accuracy
1333  * rather than adding horrible and questionable kludges here.
1334  *
1335  * This is called after the cyclics subsystem because of the potential
1336  * that the heuristics within may give a worse estimate of the clock
1337  * frequency than the value that was measured.
1338  */
1339 static void
1340 mach_fixcpufreq(void)
1341 {
1342 	uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i;
1343 
1344 	freq = (uint32_t)cpu_freq;
1345 
1346 	/*
1347 	 * Find the nearest integer multiple of 200/3 (about 66) MHz to the
1348 	 * measured speed taking into account that the 667 MHz parts were
1349 	 * the first to round-up.
1350 	 */
1351 	mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200);
1352 	near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3);
1353 	delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66);
1354 
1355 	/* Find the nearest integer multiple of 50 MHz to the measured speed */
1356 	mul = (freq + 25) / 50;
1357 	near50 = mul * 50;
1358 	delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50);
1359 
1360 	/* Find the closer of the two */
1361 	if (delta66 < delta50) {
1362 		fixed = near66;
1363 		delta = delta66;
1364 	} else {
1365 		fixed = near50;
1366 		delta = delta50;
1367 	}
1368 
1369 	if (fixed > INT_MAX)
1370 		return;
1371 
1372 	/*
1373 	 * Some older parts have a core clock frequency that is not an
1374 	 * integral multiple of 50 or 66 MHz. Check if one of the old
1375 	 * clock frequencies is closer to the measured value than any
1376 	 * of the integral multiples of 50 an 66, and if so set fixed
1377 	 * and delta appropriately to represent the closest value.
1378 	 */
1379 	i = sizeof (x86_cpu_freq) / sizeof (int);
1380 	while (i > 0) {
1381 		i--;
1382 
1383 		if (x86_cpu_freq[i] <= freq) {
1384 			mul = freq - x86_cpu_freq[i];
1385 
1386 			if (mul < delta) {
1387 				fixed = x86_cpu_freq[i];
1388 				delta = mul;
1389 			}
1390 
1391 			break;
1392 		}
1393 
1394 		mul = x86_cpu_freq[i] - freq;
1395 
1396 		if (mul < delta) {
1397 			fixed = x86_cpu_freq[i];
1398 			delta = mul;
1399 		}
1400 	}
1401 
1402 	/*
1403 	 * Set a reasonable maximum for how much to correct the measured
1404 	 * result by. This check is here to prevent the adjustment made
1405 	 * by this function from being more harm than good. It is entirely
1406 	 * possible that in the future parts will be made that are not
1407 	 * integral multiples of 66 or 50 in clock frequency or that
1408 	 * someone may overclock a part to some odd frequency. If the
1409 	 * measured value is farther from the corrected value than
1410 	 * allowed, then assume the corrected value is in error and use
1411 	 * the measured value.
1412 	 */
1413 	if (6 < delta)
1414 		return;
1415 
1416 	cpu_freq = (int)fixed;
1417 }
1418 
1419 
1420 static int
1421 machhztomhz(uint64_t cpu_freq_hz)
1422 {
1423 	uint64_t cpu_mhz;
1424 
1425 	/* Round to nearest MHZ */
1426 	cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ;
1427 
1428 	if (cpu_mhz > INT_MAX)
1429 		return (0);
1430 
1431 	return ((int)cpu_mhz);
1432 
1433 }
1434 
1435 
1436 static int
1437 mach_clkinit(int preferred_mode, int *set_mode)
1438 {
1439 	struct psm_ops  *pops;
1440 	int resolution;
1441 
1442 	pops = mach_set[0];
1443 
1444 	cpu_freq_hz = mach_getcpufreq();
1445 
1446 	cpu_freq = machhztomhz(cpu_freq_hz);
1447 
1448 	if (!is_x86_feature(x86_featureset, X86FSET_TSC) || (cpu_freq == 0))
1449 		tsc_gethrtime_enable = 0;
1450 
1451 #ifndef __xpv
1452 	if (tsc_gethrtime_enable) {
1453 		tsc_hrtimeinit(cpu_freq_hz);
1454 	} else
1455 #endif
1456 	{
1457 		if (pops->psm_hrtimeinit)
1458 			(*pops->psm_hrtimeinit)();
1459 		gethrtimef = pops->psm_gethrtime;
1460 		gethrtimeunscaledf = gethrtimef;
1461 		/* scalehrtimef will remain dummy */
1462 	}
1463 
1464 	mach_fixcpufreq();
1465 
1466 	if (mach_ver[0] >= PSM_INFO_VER01_3) {
1467 		if (preferred_mode == TIMER_ONESHOT) {
1468 
1469 			resolution = (*pops->psm_clkinit)(0);
1470 			if (resolution != 0)  {
1471 				*set_mode = TIMER_ONESHOT;
1472 				return (resolution);
1473 			}
1474 		}
1475 
1476 		/*
1477 		 * either periodic mode was requested or could not set to
1478 		 * one-shot mode
1479 		 */
1480 		resolution = (*pops->psm_clkinit)(hz);
1481 		/*
1482 		 * psm should be able to do periodic, so we do not check
1483 		 * for return value of psm_clkinit here.
1484 		 */
1485 		*set_mode = TIMER_PERIODIC;
1486 		return (resolution);
1487 	} else {
1488 		/*
1489 		 * PSMI interface prior to PSMI_3 does not define a return
1490 		 * value for psm_clkinit, so the return value is ignored.
1491 		 */
1492 		(void) (*pops->psm_clkinit)(hz);
1493 		*set_mode = TIMER_PERIODIC;
1494 		return (nsec_per_tick);
1495 	}
1496 }
1497 
1498 
1499 /*ARGSUSED*/
1500 static int
1501 mach_softlvl_to_vect(int ipl)
1502 {
1503 	setsoftint = av_set_softint_pending;
1504 	kdisetsoftint = kdi_av_set_softint_pending;
1505 
1506 	return (PSM_SV_SOFTWARE);
1507 }
1508 
1509 #ifdef DEBUG
1510 /*
1511  * This is here to allow us to simulate cpus that refuse to start.
1512  */
1513 cpuset_t cpufailset;
1514 #endif
1515 
1516 int
1517 mach_cpu_start(struct cpu *cp, void *ctx)
1518 {
1519 	struct psm_ops *pops = mach_set[0];
1520 	processorid_t id = cp->cpu_id;
1521 
1522 #ifdef DEBUG
1523 	if (CPU_IN_SET(cpufailset, id))
1524 		return (0);
1525 #endif
1526 	return ((*pops->psm_cpu_start)(id, ctx));
1527 }
1528 
1529 int
1530 mach_cpuid_start(processorid_t id, void *ctx)
1531 {
1532 	struct psm_ops *pops = mach_set[0];
1533 
1534 #ifdef DEBUG
1535 	if (CPU_IN_SET(cpufailset, id))
1536 		return (0);
1537 #endif
1538 	return ((*pops->psm_cpu_start)(id, ctx));
1539 }
1540 
1541 int
1542 mach_cpu_stop(cpu_t *cp, void *ctx)
1543 {
1544 	struct psm_ops *pops = mach_set[0];
1545 	psm_cpu_request_t request;
1546 
1547 	if (pops->psm_cpu_ops == NULL) {
1548 		return (ENOTSUP);
1549 	}
1550 
1551 	ASSERT(cp->cpu_id != -1);
1552 	request.pcr_cmd = PSM_CPU_STOP;
1553 	request.req.cpu_stop.cpuid = cp->cpu_id;
1554 	request.req.cpu_stop.ctx = ctx;
1555 
1556 	return ((*pops->psm_cpu_ops)(&request));
1557 }
1558 
1559 int
1560 mach_cpu_add(mach_cpu_add_arg_t *argp, processorid_t *cpuidp)
1561 {
1562 	int rc;
1563 	struct psm_ops *pops = mach_set[0];
1564 	psm_cpu_request_t request;
1565 
1566 	if (pops->psm_cpu_ops == NULL) {
1567 		return (ENOTSUP);
1568 	}
1569 
1570 	request.pcr_cmd = PSM_CPU_ADD;
1571 	request.req.cpu_add.argp = argp;
1572 	request.req.cpu_add.cpuid = -1;
1573 	rc = (*pops->psm_cpu_ops)(&request);
1574 	if (rc == 0) {
1575 		ASSERT(request.req.cpu_add.cpuid != -1);
1576 		*cpuidp = request.req.cpu_add.cpuid;
1577 	}
1578 
1579 	return (rc);
1580 }
1581 
1582 int
1583 mach_cpu_remove(processorid_t cpuid)
1584 {
1585 	struct psm_ops *pops = mach_set[0];
1586 	psm_cpu_request_t request;
1587 
1588 	if (pops->psm_cpu_ops == NULL) {
1589 		return (ENOTSUP);
1590 	}
1591 
1592 	request.pcr_cmd = PSM_CPU_REMOVE;
1593 	request.req.cpu_remove.cpuid = cpuid;
1594 
1595 	return ((*pops->psm_cpu_ops)(&request));
1596 }
1597 
1598 /*
1599  * Default handler to create device node for CPU.
1600  * One reference count will be held on created device node.
1601  */
1602 static int
1603 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp)
1604 {
1605 	int rv, circ;
1606 	dev_info_t *dip;
1607 	static kmutex_t cpu_node_lock;
1608 	static dev_info_t *cpu_nex_devi = NULL;
1609 
1610 	ASSERT(cp != NULL);
1611 	ASSERT(dipp != NULL);
1612 	*dipp = NULL;
1613 
1614 	if (cpu_nex_devi == NULL) {
1615 		mutex_enter(&cpu_node_lock);
1616 		/* First check whether cpus exists. */
1617 		cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0);
1618 		/* Create cpus if it doesn't exist. */
1619 		if (cpu_nex_devi == NULL) {
1620 			ndi_devi_enter(ddi_root_node(), &circ);
1621 			rv = ndi_devi_alloc(ddi_root_node(), "cpus",
1622 			    (pnode_t)DEVI_SID_NODEID, &dip);
1623 			if (rv != NDI_SUCCESS) {
1624 				mutex_exit(&cpu_node_lock);
1625 				cmn_err(CE_CONT,
1626 				    "?failed to create cpu nexus device.\n");
1627 				return (PSM_FAILURE);
1628 			}
1629 			ASSERT(dip != NULL);
1630 			(void) ndi_devi_online(dip, 0);
1631 			ndi_devi_exit(ddi_root_node(), circ);
1632 			cpu_nex_devi = dip;
1633 		}
1634 		mutex_exit(&cpu_node_lock);
1635 	}
1636 
1637 	/*
1638 	 * create a child node for cpu identified as 'cpu_id'
1639 	 */
1640 	ndi_devi_enter(cpu_nex_devi, &circ);
1641 	dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1);
1642 	if (dip == NULL) {
1643 		cmn_err(CE_CONT,
1644 		    "?failed to create device node for cpu%d.\n", cp->cpu_id);
1645 		rv = PSM_FAILURE;
1646 	} else {
1647 		*dipp = dip;
1648 		(void) ndi_hold_devi(dip);
1649 		rv = PSM_SUCCESS;
1650 	}
1651 	ndi_devi_exit(cpu_nex_devi, circ);
1652 
1653 	return (rv);
1654 }
1655 
1656 /*
1657  * Create cpu device node in device tree and online it.
1658  * Return created dip with reference count held if requested.
1659  */
1660 int
1661 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp)
1662 {
1663 	int rv;
1664 	dev_info_t *dip = NULL;
1665 
1666 	ASSERT(psm_cpu_create_devinfo != NULL);
1667 	rv = psm_cpu_create_devinfo(cp, &dip);
1668 	if (rv == PSM_SUCCESS) {
1669 		cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi);
1670 		/* Recursively attach driver for parent nexus device. */
1671 		if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) ==
1672 		    DDI_SUCCESS) {
1673 			/* Configure cpu itself and descendants. */
1674 			(void) ndi_devi_online(dip,
1675 			    NDI_ONLINE_ATTACH | NDI_CONFIG);
1676 		}
1677 		if (dipp != NULL) {
1678 			*dipp = dip;
1679 		} else {
1680 			(void) ndi_rele_devi(dip);
1681 		}
1682 	}
1683 
1684 	return (rv);
1685 }
1686 
1687 /*
1688  * The dipp contains one of following values on return:
1689  * - NULL if no device node found
1690  * - pointer to device node if found
1691  */
1692 int
1693 mach_cpu_get_device_node(struct cpu *cp, dev_info_t **dipp)
1694 {
1695 	*dipp = NULL;
1696 	if (psm_cpu_get_devinfo != NULL) {
1697 		if (psm_cpu_get_devinfo(cp, dipp) == PSM_SUCCESS) {
1698 			return (PSM_SUCCESS);
1699 		}
1700 	}
1701 
1702 	return (PSM_FAILURE);
1703 }
1704 
1705 /*ARGSUSED*/
1706 static int
1707 mach_translate_irq(dev_info_t *dip, int irqno)
1708 {
1709 	return (irqno);	/* default to NO translation */
1710 }
1711 
1712 static void
1713 mach_notify_error(int level, char *errmsg)
1714 {
1715 	/*
1716 	 * SL_FATAL is pass in once panicstr is set, deliver it
1717 	 * as CE_PANIC.  Also, translate SL_ codes back to CE_
1718 	 * codes for the psmi handler
1719 	 */
1720 	if (level & SL_FATAL)
1721 		(*notify_error)(CE_PANIC, errmsg);
1722 	else if (level & SL_WARN)
1723 		(*notify_error)(CE_WARN, errmsg);
1724 	else if (level & SL_NOTE)
1725 		(*notify_error)(CE_NOTE, errmsg);
1726 	else if (level & SL_CONSOLE)
1727 		(*notify_error)(CE_CONT, errmsg);
1728 }
1729 
1730 /*
1731  * It provides the default basic intr_ops interface for the new DDI
1732  * interrupt framework if the PSM doesn't have one.
1733  *
1734  * Input:
1735  * dip     - pointer to the dev_info structure of the requested device
1736  * hdlp    - pointer to the internal interrupt handle structure for the
1737  *	     requested interrupt
1738  * intr_op - opcode for this call
1739  * result  - pointer to the integer that will hold the result to be
1740  *	     passed back if return value is PSM_SUCCESS
1741  *
1742  * Output:
1743  * return value is either PSM_SUCCESS or PSM_FAILURE
1744  */
1745 static int
1746 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1747     psm_intr_op_t intr_op, int *result)
1748 {
1749 	struct intrspec *ispec;
1750 
1751 	switch (intr_op) {
1752 	case PSM_INTR_OP_CHECK_MSI:
1753 		*result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1754 		    DDI_INTR_TYPE_MSIX);
1755 		break;
1756 	case PSM_INTR_OP_ALLOC_VECTORS:
1757 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1758 			*result = 1;
1759 		else
1760 			*result = 0;
1761 		break;
1762 	case PSM_INTR_OP_FREE_VECTORS:
1763 		break;
1764 	case PSM_INTR_OP_NAVAIL_VECTORS:
1765 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1766 			*result = 1;
1767 		else
1768 			*result = 0;
1769 		break;
1770 	case PSM_INTR_OP_XLATE_VECTOR:
1771 		ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1772 		*result = psm_translate_irq(dip, ispec->intrspec_vec);
1773 		break;
1774 	case PSM_INTR_OP_GET_CAP:
1775 		*result = 0;
1776 		break;
1777 	case PSM_INTR_OP_GET_PENDING:
1778 	case PSM_INTR_OP_CLEAR_MASK:
1779 	case PSM_INTR_OP_SET_MASK:
1780 	case PSM_INTR_OP_GET_SHARED:
1781 	case PSM_INTR_OP_SET_PRI:
1782 	case PSM_INTR_OP_SET_CAP:
1783 	case PSM_INTR_OP_SET_CPU:
1784 	case PSM_INTR_OP_GET_INTR:
1785 	default:
1786 		return (PSM_FAILURE);
1787 	}
1788 	return (PSM_SUCCESS);
1789 }
1790 /*
1791  * Return 1 if CMT load balancing policies should be
1792  * implemented across instances of the specified hardware
1793  * sharing relationship.
1794  */
1795 int
1796 pg_cmt_load_bal_hw(pghw_type_t hw)
1797 {
1798 	if (hw == PGHW_IPIPE ||
1799 	    hw == PGHW_FPU ||
1800 	    hw == PGHW_PROCNODE ||
1801 	    hw == PGHW_CHIP)
1802 		return (1);
1803 	else
1804 		return (0);
1805 }
1806 /*
1807  * Return 1 if thread affinity polices should be implemented
1808  * for instances of the specifed hardware sharing relationship.
1809  */
1810 int
1811 pg_cmt_affinity_hw(pghw_type_t hw)
1812 {
1813 	if (hw == PGHW_CACHE)
1814 		return (1);
1815 	else
1816 		return (0);
1817 }
1818 
1819 /*
1820  * Return number of counter events requested to measure hardware capacity and
1821  * utilization and setup CPC requests for specified CPU as needed
1822  *
1823  * May return 0 when platform or processor specific code knows that no CPC
1824  * events should be programmed on this CPU or -1 when platform or processor
1825  * specific code doesn't know which counter events are best to use and common
1826  * code should decide for itself
1827  */
1828 int
1829 /* LINTED E_FUNC_ARG_UNUSED */
1830 cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs)
1831 {
1832 	const char	*impl_name;
1833 
1834 	/*
1835 	 * Return error if pcbe_ops not set
1836 	 */
1837 	if (pcbe_ops == NULL)
1838 		return (-1);
1839 
1840 	/*
1841 	 * Return that no CPC events should be programmed on hyperthreaded
1842 	 * Pentium 4 and return error for all other x86 processors to tell
1843 	 * common code to decide what counter events to program on those CPUs
1844 	 * for measuring hardware capacity and utilization
1845 	 */
1846 	impl_name = pcbe_ops->pcbe_impl_name();
1847 	if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0)
1848 		return (0);
1849 	else
1850 		return (-1);
1851 }
1852