xref: /titanic_51/usr/src/uts/i86pc/io/pcplusmp/apic.c (revision 15d9d0b528387242011cdcc6190c9e598cfe3a07)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
31  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
32  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
33  * PSMI 1.5 extensions are supported in Solaris Nevada.
34  * PSMI 1.6 extensions are supported in Solaris Nevada.
35  */
36 #define	PSMI_1_6
37 
38 #include <sys/processor.h>
39 #include <sys/time.h>
40 #include <sys/psm.h>
41 #include <sys/smp_impldefs.h>
42 #include <sys/cram.h>
43 #include <sys/acpi/acpi.h>
44 #include <sys/acpica.h>
45 #include <sys/psm_common.h>
46 #include <sys/apic.h>
47 #include <sys/pit.h>
48 #include <sys/ddi.h>
49 #include <sys/sunddi.h>
50 #include <sys/ddi_impldefs.h>
51 #include <sys/pci.h>
52 #include <sys/promif.h>
53 #include <sys/x86_archext.h>
54 #include <sys/cpc_impl.h>
55 #include <sys/uadmin.h>
56 #include <sys/panic.h>
57 #include <sys/debug.h>
58 #include <sys/archsystm.h>
59 #include <sys/trap.h>
60 #include <sys/machsystm.h>
61 #include <sys/sysmacros.h>
62 #include <sys/cpuvar.h>
63 #include <sys/rm_platter.h>
64 #include <sys/privregs.h>
65 #include <sys/note.h>
66 #include <sys/pci_intr_lib.h>
67 #include <sys/spl.h>
68 #include <sys/clock.h>
69 #include <sys/dditypes.h>
70 #include <sys/sunddi.h>
71 
72 /*
73  *	Local Function Prototypes
74  */
75 static void apic_init_intr();
76 static void apic_ret();
77 static int get_apic_cmd1();
78 static int get_apic_pri();
79 static void apic_nmi_intr(caddr_t arg, struct regs *rp);
80 
81 /*
82  *	standard MP entries
83  */
84 static int	apic_probe();
85 static int	apic_clkinit();
86 static int	apic_getclkirq(int ipl);
87 static uint_t	apic_calibrate(volatile uint32_t *addr,
88     uint16_t *pit_ticks_adj);
89 static hrtime_t apic_gettime();
90 static hrtime_t apic_gethrtime();
91 static void	apic_init();
92 static void	apic_picinit(void);
93 static int	apic_cpu_start(processorid_t, caddr_t);
94 static int	apic_post_cpu_start(void);
95 static void	apic_send_ipi(int cpun, int ipl);
96 static void	apic_set_idlecpu(processorid_t cpun);
97 static void	apic_unset_idlecpu(processorid_t cpun);
98 static int	apic_intr_enter(int ipl, int *vect);
99 static void	apic_setspl(int ipl);
100 static int	apic_addspl(int ipl, int vector, int min_ipl, int max_ipl);
101 static int	apic_delspl(int ipl, int vector, int min_ipl, int max_ipl);
102 static void	apic_shutdown(int cmd, int fcn);
103 static void	apic_preshutdown(int cmd, int fcn);
104 static int	apic_disable_intr(processorid_t cpun);
105 static void	apic_enable_intr(processorid_t cpun);
106 static processorid_t	apic_get_next_processorid(processorid_t cpun);
107 static int		apic_get_ipivect(int ipl, int type);
108 static void	apic_timer_reprogram(hrtime_t time);
109 static void	apic_timer_enable(void);
110 static void	apic_timer_disable(void);
111 static void	apic_post_cyclic_setup(void *arg);
112 
113 static int	apic_oneshot = 0;
114 int	apic_oneshot_enable = 1; /* to allow disabling one-shot capability */
115 
116 /* Now the ones for Dynamic Interrupt distribution */
117 int	apic_enable_dynamic_migration = 0;
118 
119 
120 /*
121  * These variables are frequently accessed in apic_intr_enter(),
122  * apic_intr_exit and apic_setspl, so group them together
123  */
124 volatile uint32_t *apicadr =  NULL;	/* virtual addr of local APIC	*/
125 int apic_setspl_delay = 1;		/* apic_setspl - delay enable	*/
126 int apic_clkvect;
127 
128 /* vector at which error interrupts come in */
129 int apic_errvect;
130 int apic_enable_error_intr = 1;
131 int apic_error_display_delay = 100;
132 
133 /* vector at which performance counter overflow interrupts come in */
134 int apic_cpcovf_vect;
135 int apic_enable_cpcovf_intr = 1;
136 
137 /*
138  * The following vector assignments influence the value of ipltopri and
139  * vectortoipl. Note that vectors 0 - 0x1f are not used. We can program
140  * idle to 0 and IPL 0 to 0xf to differentiate idle in case
141  * we care to do so in future. Note some IPLs which are rarely used
142  * will share the vector ranges and heavily used IPLs (5 and 6) have
143  * a wide range.
144  *
145  * This array is used to initialize apic_ipls[] (in apic_init()).
146  *
147  *	IPL		Vector range.		as passed to intr_enter
148  *	0		none.
149  *	1,2,3		0x20-0x2f		0x0-0xf
150  *	4		0x30-0x3f		0x10-0x1f
151  *	5		0x40-0x5f		0x20-0x3f
152  *	6		0x60-0x7f		0x40-0x5f
153  *	7,8,9		0x80-0x8f		0x60-0x6f
154  *	10		0x90-0x9f		0x70-0x7f
155  *	11		0xa0-0xaf		0x80-0x8f
156  *	...		...
157  *	15		0xe0-0xef		0xc0-0xcf
158  *	15		0xf0-0xff		0xd0-0xdf
159  */
160 uchar_t apic_vectortoipl[APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL] = {
161 	3, 4, 5, 5, 6, 6, 9, 10, 11, 12, 13, 14, 15, 15
162 };
163 	/*
164 	 * The ipl of an ISR at vector X is apic_vectortoipl[X>>4]
165 	 * NOTE that this is vector as passed into intr_enter which is
166 	 * programmed vector - 0x20 (APIC_BASE_VECT)
167 	 */
168 
169 uchar_t	apic_ipltopri[MAXIPL + 1];	/* unix ipl to apic pri	*/
170 	/* The taskpri to be programmed into apic to mask given ipl */
171 
172 #if defined(__amd64)
173 uchar_t	apic_cr8pri[MAXIPL + 1];	/* unix ipl to cr8 pri	*/
174 #endif
175 
176 /*
177  * Correlation of the hardware vector to the IPL in use, initialized
178  * from apic_vectortoipl[] in apic_init().  The final IPLs may not correlate
179  * to the IPLs in apic_vectortoipl on some systems that share interrupt lines
180  * connected to errata-stricken IOAPICs
181  */
182 uchar_t apic_ipls[APIC_AVAIL_VECTOR];
183 
184 /*
185  * Patchable global variables.
186  */
187 int	apic_forceload = 0;
188 
189 int	apic_coarse_hrtime = 1;		/* 0 - use accurate slow gethrtime() */
190 					/* 1 - use gettime() for performance */
191 int	apic_flat_model = 0;		/* 0 - clustered. 1 - flat */
192 int	apic_enable_hwsoftint = 0;	/* 0 - disable, 1 - enable	*/
193 int	apic_enable_bind_log = 1;	/* 1 - display interrupt binding log */
194 int	apic_panic_on_nmi = 0;
195 int	apic_panic_on_apic_error = 0;
196 
197 int	apic_verbose = 0;
198 
199 /* minimum number of timer ticks to program to */
200 int apic_min_timer_ticks = 1;
201 /*
202  *	Local static data
203  */
204 static struct	psm_ops apic_ops = {
205 	apic_probe,
206 
207 	apic_init,
208 	apic_picinit,
209 	apic_intr_enter,
210 	apic_intr_exit,
211 	apic_setspl,
212 	apic_addspl,
213 	apic_delspl,
214 	apic_disable_intr,
215 	apic_enable_intr,
216 	(int (*)(int))NULL,		/* psm_softlvl_to_irq */
217 	(void (*)(int))NULL,		/* psm_set_softintr */
218 
219 	apic_set_idlecpu,
220 	apic_unset_idlecpu,
221 
222 	apic_clkinit,
223 	apic_getclkirq,
224 	(void (*)(void))NULL,		/* psm_hrtimeinit */
225 	apic_gethrtime,
226 
227 	apic_get_next_processorid,
228 	apic_cpu_start,
229 	apic_post_cpu_start,
230 	apic_shutdown,
231 	apic_get_ipivect,
232 	apic_send_ipi,
233 
234 	(int (*)(dev_info_t *, int))NULL,	/* psm_translate_irq */
235 	(void (*)(int, char *))NULL,	/* psm_notify_error */
236 	(void (*)(int))NULL,		/* psm_notify_func */
237 	apic_timer_reprogram,
238 	apic_timer_enable,
239 	apic_timer_disable,
240 	apic_post_cyclic_setup,
241 	apic_preshutdown,
242 	apic_intr_ops,			/* Advanced DDI Interrupt framework */
243 	apic_state,			/* save, restore apic state for S3 */
244 };
245 
246 
247 static struct	psm_info apic_psm_info = {
248 	PSM_INFO_VER01_6,			/* version */
249 	PSM_OWN_EXCLUSIVE,			/* ownership */
250 	(struct psm_ops *)&apic_ops,		/* operation */
251 	APIC_PCPLUSMP_NAME,			/* machine name */
252 	"pcplusmp v1.4 compatible %I%",
253 };
254 
255 static void *apic_hdlp;
256 
257 #ifdef DEBUG
258 int	apic_debug = 0;
259 int	apic_restrict_vector = 0;
260 
261 int	apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE];
262 int	apic_debug_msgbufindex = 0;
263 
264 #endif /* DEBUG */
265 
266 apic_cpus_info_t	*apic_cpus;
267 
268 cpuset_t	apic_cpumask;
269 uint_t	apic_picinit_called;
270 
271 /* Flag to indicate that we need to shut down all processors */
272 static uint_t	apic_shutdown_processors;
273 
274 uint_t apic_nsec_per_intr = 0;
275 
276 /*
277  * apic_let_idle_redistribute can have the following values:
278  * 0 - If clock decremented it from 1 to 0, clock has to call redistribute.
279  * apic_redistribute_lock prevents multiple idle cpus from redistributing
280  */
281 int	apic_num_idle_redistributions = 0;
282 static	int apic_let_idle_redistribute = 0;
283 static	uint_t apic_nticks = 0;
284 static	uint_t apic_skipped_redistribute = 0;
285 
286 /* to gather intr data and redistribute */
287 static void apic_redistribute_compute(void);
288 
289 static	uint_t last_count_read = 0;
290 static	lock_t	apic_gethrtime_lock;
291 volatile int	apic_hrtime_stamp = 0;
292 volatile hrtime_t apic_nsec_since_boot = 0;
293 static uint_t apic_hertz_count;
294 
295 uint64_t apic_ticks_per_SFnsecs;	/* # of ticks in SF nsecs */
296 
297 static hrtime_t apic_nsec_max;
298 
299 static	hrtime_t	apic_last_hrtime = 0;
300 int		apic_hrtime_error = 0;
301 int		apic_remote_hrterr = 0;
302 int		apic_num_nmis = 0;
303 int		apic_apic_error = 0;
304 int		apic_num_apic_errors = 0;
305 int		apic_num_cksum_errors = 0;
306 
307 int	apic_error = 0;
308 static	int	apic_cmos_ssb_set = 0;
309 
310 /* use to make sure only one cpu handles the nmi */
311 static	lock_t	apic_nmi_lock;
312 /* use to make sure only one cpu handles the error interrupt */
313 static	lock_t	apic_error_lock;
314 
315 static	struct {
316 	uchar_t	cntl;
317 	uchar_t	data;
318 } aspen_bmc[] = {
319 	{ CC_SMS_WR_START,	0x18 },		/* NetFn/LUN */
320 	{ CC_SMS_WR_NEXT,	0x24 },		/* Cmd SET_WATCHDOG_TIMER */
321 	{ CC_SMS_WR_NEXT,	0x84 },		/* DataByte 1: SMS/OS no log */
322 	{ CC_SMS_WR_NEXT,	0x2 },		/* DataByte 2: Power Down */
323 	{ CC_SMS_WR_NEXT,	0x0 },		/* DataByte 3: no pre-timeout */
324 	{ CC_SMS_WR_NEXT,	0x0 },		/* DataByte 4: timer expir. */
325 	{ CC_SMS_WR_NEXT,	0xa },		/* DataByte 5: init countdown */
326 	{ CC_SMS_WR_END,	0x0 },		/* DataByte 6: init countdown */
327 
328 	{ CC_SMS_WR_START,	0x18 },		/* NetFn/LUN */
329 	{ CC_SMS_WR_END,	0x22 }		/* Cmd RESET_WATCHDOG_TIMER */
330 };
331 
332 static	struct {
333 	int	port;
334 	uchar_t	data;
335 } sitka_bmc[] = {
336 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_START },
337 	{ SMS_DATA_REGISTER,	0x18 },		/* NetFn/LUN */
338 	{ SMS_DATA_REGISTER,	0x24 },		/* Cmd SET_WATCHDOG_TIMER */
339 	{ SMS_DATA_REGISTER,	0x84 },		/* DataByte 1: SMS/OS no log */
340 	{ SMS_DATA_REGISTER,	0x2 },		/* DataByte 2: Power Down */
341 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 3: no pre-timeout */
342 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 4: timer expir. */
343 	{ SMS_DATA_REGISTER,	0xa },		/* DataByte 5: init countdown */
344 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_END },
345 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 6: init countdown */
346 
347 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_START },
348 	{ SMS_DATA_REGISTER,	0x18 },		/* NetFn/LUN */
349 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_END },
350 	{ SMS_DATA_REGISTER,	0x22 }		/* Cmd RESET_WATCHDOG_TIMER */
351 };
352 
353 /* Patchable global variables. */
354 int		apic_kmdb_on_nmi = 0;		/* 0 - no, 1 - yes enter kmdb */
355 uint32_t	apic_divide_reg_init = 0;	/* 0 - divide by 2 */
356 
357 /*
358  *	This is the loadable module wrapper
359  */
360 
361 int
362 _init(void)
363 {
364 	if (apic_coarse_hrtime)
365 		apic_ops.psm_gethrtime = &apic_gettime;
366 	return (psm_mod_init(&apic_hdlp, &apic_psm_info));
367 }
368 
369 int
370 _fini(void)
371 {
372 	return (psm_mod_fini(&apic_hdlp, &apic_psm_info));
373 }
374 
375 int
376 _info(struct modinfo *modinfop)
377 {
378 	return (psm_mod_info(&apic_hdlp, &apic_psm_info, modinfop));
379 }
380 
381 
382 static int
383 apic_probe()
384 {
385 	return (apic_probe_common(apic_psm_info.p_mach_idstring));
386 }
387 
388 void
389 apic_init()
390 {
391 	int i;
392 	int	j = 1;
393 
394 	apic_ipltopri[0] = APIC_VECTOR_PER_IPL; /* leave 0 for idle */
395 	for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
396 		if ((i < ((APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL) - 1)) &&
397 		    (apic_vectortoipl[i + 1] == apic_vectortoipl[i]))
398 			/* get to highest vector at the same ipl */
399 			continue;
400 		for (; j <= apic_vectortoipl[i]; j++) {
401 			apic_ipltopri[j] = (i << APIC_IPL_SHIFT) +
402 			    APIC_BASE_VECT;
403 		}
404 	}
405 	for (; j < MAXIPL + 1; j++)
406 		/* fill up any empty ipltopri slots */
407 		apic_ipltopri[j] = (i << APIC_IPL_SHIFT) + APIC_BASE_VECT;
408 	apic_init_common();
409 #if defined(__amd64)
410 	/*
411 	 * Make cpu-specific interrupt info point to cr8pri vector
412 	 */
413 	for (i = 0; i <= MAXIPL; i++)
414 		apic_cr8pri[i] = apic_ipltopri[i] >> APIC_IPL_SHIFT;
415 	CPU->cpu_pri_data = apic_cr8pri;
416 #endif	/* __amd64 */
417 }
418 
419 /*
420  * handler for APIC Error interrupt. Just print a warning and continue
421  */
422 static int
423 apic_error_intr()
424 {
425 	uint_t	error0, error1, error;
426 	uint_t	i;
427 
428 	/*
429 	 * We need to write before read as per 7.4.17 of system prog manual.
430 	 * We do both and or the results to be safe
431 	 */
432 	error0 = apicadr[APIC_ERROR_STATUS];
433 	apicadr[APIC_ERROR_STATUS] = 0;
434 	error1 = apicadr[APIC_ERROR_STATUS];
435 	error = error0 | error1;
436 
437 	/*
438 	 * Clear the APIC error status (do this on all cpus that enter here)
439 	 * (two writes are required due to the semantics of accessing the
440 	 * error status register.)
441 	 */
442 	apicadr[APIC_ERROR_STATUS] = 0;
443 	apicadr[APIC_ERROR_STATUS] = 0;
444 
445 	/*
446 	 * Prevent more than 1 CPU from handling error interrupt causing
447 	 * double printing (interleave of characters from multiple
448 	 * CPU's when using prom_printf)
449 	 */
450 	if (lock_try(&apic_error_lock) == 0)
451 		return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
452 	if (error) {
453 #if	DEBUG
454 		if (apic_debug)
455 			debug_enter("pcplusmp: APIC Error interrupt received");
456 #endif /* DEBUG */
457 		if (apic_panic_on_apic_error)
458 			cmn_err(CE_PANIC,
459 			    "APIC Error interrupt on CPU %d. Status = %x\n",
460 			    psm_get_cpu_id(), error);
461 		else {
462 			if ((error & ~APIC_CS_ERRORS) == 0) {
463 				/* cksum error only */
464 				apic_error |= APIC_ERR_APIC_ERROR;
465 				apic_apic_error |= error;
466 				apic_num_apic_errors++;
467 				apic_num_cksum_errors++;
468 			} else {
469 				/*
470 				 * prom_printf is the best shot we have of
471 				 * something which is problem free from
472 				 * high level/NMI type of interrupts
473 				 */
474 				prom_printf("APIC Error interrupt on CPU %d. "
475 				    "Status 0 = %x, Status 1 = %x\n",
476 				    psm_get_cpu_id(), error0, error1);
477 				apic_error |= APIC_ERR_APIC_ERROR;
478 				apic_apic_error |= error;
479 				apic_num_apic_errors++;
480 				for (i = 0; i < apic_error_display_delay; i++) {
481 					tenmicrosec();
482 				}
483 				/*
484 				 * provide more delay next time limited to
485 				 * roughly 1 clock tick time
486 				 */
487 				if (apic_error_display_delay < 500)
488 					apic_error_display_delay *= 2;
489 			}
490 		}
491 		lock_clear(&apic_error_lock);
492 		return (DDI_INTR_CLAIMED);
493 	} else {
494 		lock_clear(&apic_error_lock);
495 		return (DDI_INTR_UNCLAIMED);
496 	}
497 	/* NOTREACHED */
498 }
499 
500 /*
501  * Turn off the mask bit in the performance counter Local Vector Table entry.
502  */
503 static void
504 apic_cpcovf_mask_clear(void)
505 {
506 	apicadr[APIC_PCINT_VECT] &= ~APIC_LVT_MASK;
507 }
508 
509 static void
510 apic_init_intr()
511 {
512 	processorid_t	cpun = psm_get_cpu_id();
513 
514 #if defined(__amd64)
515 	setcr8((ulong_t)(APIC_MASK_ALL >> APIC_IPL_SHIFT));
516 #else
517 	apicadr[APIC_TASK_REG] = APIC_MASK_ALL;
518 #endif
519 
520 	if (apic_flat_model)
521 		apicadr[APIC_FORMAT_REG] = APIC_FLAT_MODEL;
522 	else
523 		apicadr[APIC_FORMAT_REG] = APIC_CLUSTER_MODEL;
524 	apicadr[APIC_DEST_REG] = AV_HIGH_ORDER >> cpun;
525 
526 	/* need to enable APIC before unmasking NMI */
527 	apicadr[APIC_SPUR_INT_REG] = AV_UNIT_ENABLE | APIC_SPUR_INTR;
528 
529 	apicadr[APIC_LOCAL_TIMER] = AV_MASK;
530 	apicadr[APIC_INT_VECT0]	= AV_MASK;	/* local intr reg 0 */
531 	apicadr[APIC_INT_VECT1] = AV_NMI;	/* enable NMI */
532 
533 	if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS)
534 		return;
535 
536 	/* Enable performance counter overflow interrupt */
537 
538 	if ((x86_feature & X86_MSR) != X86_MSR)
539 		apic_enable_cpcovf_intr = 0;
540 	if (apic_enable_cpcovf_intr) {
541 		if (apic_cpcovf_vect == 0) {
542 			int ipl = APIC_PCINT_IPL;
543 			int irq = apic_get_ipivect(ipl, -1);
544 
545 			ASSERT(irq != -1);
546 			apic_cpcovf_vect = apic_irq_table[irq]->airq_vector;
547 			ASSERT(apic_cpcovf_vect);
548 			(void) add_avintr(NULL, ipl,
549 			    (avfunc)kcpc_hw_overflow_intr,
550 			    "apic pcint", irq, NULL, NULL, NULL, NULL);
551 			kcpc_hw_overflow_intr_installed = 1;
552 			kcpc_hw_enable_cpc_intr = apic_cpcovf_mask_clear;
553 		}
554 		apicadr[APIC_PCINT_VECT] = apic_cpcovf_vect;
555 	}
556 
557 	/* Enable error interrupt */
558 
559 	if (apic_enable_error_intr) {
560 		if (apic_errvect == 0) {
561 			int ipl = 0xf;	/* get highest priority intr */
562 			int irq = apic_get_ipivect(ipl, -1);
563 
564 			ASSERT(irq != -1);
565 			apic_errvect = apic_irq_table[irq]->airq_vector;
566 			ASSERT(apic_errvect);
567 			/*
568 			 * Not PSMI compliant, but we are going to merge
569 			 * with ON anyway
570 			 */
571 			(void) add_avintr((void *)NULL, ipl,
572 			    (avfunc)apic_error_intr, "apic error intr",
573 			    irq, NULL, NULL, NULL, NULL);
574 		}
575 		apicadr[APIC_ERR_VECT] = apic_errvect;
576 		apicadr[APIC_ERROR_STATUS] = 0;
577 		apicadr[APIC_ERROR_STATUS] = 0;
578 	}
579 }
580 
581 static void
582 apic_disable_local_apic()
583 {
584 	apicadr[APIC_TASK_REG] = APIC_MASK_ALL;
585 	apicadr[APIC_LOCAL_TIMER] = AV_MASK;
586 	apicadr[APIC_INT_VECT0] = AV_MASK;	/* local intr reg 0 */
587 	apicadr[APIC_INT_VECT1] = AV_MASK;	/* disable NMI */
588 	apicadr[APIC_ERR_VECT] = AV_MASK;	/* and error interrupt */
589 	apicadr[APIC_PCINT_VECT] = AV_MASK;	/* and perf counter intr */
590 	apicadr[APIC_SPUR_INT_REG] = APIC_SPUR_INTR;
591 }
592 
593 static void
594 apic_picinit(void)
595 {
596 	int i, j;
597 	uint_t isr;
598 
599 	/*
600 	 * On UniSys Model 6520, the BIOS leaves vector 0x20 isr
601 	 * bit on without clearing it with EOI.  Since softint
602 	 * uses vector 0x20 to interrupt itself, so softint will
603 	 * not work on this machine.  In order to fix this problem
604 	 * a check is made to verify all the isr bits are clear.
605 	 * If not, EOIs are issued to clear the bits.
606 	 */
607 	for (i = 7; i >= 1; i--) {
608 		if ((isr = apicadr[APIC_ISR_REG + (i * 4)]) != 0)
609 			for (j = 0; ((j < 32) && (isr != 0)); j++)
610 				if (isr & (1 << j)) {
611 					apicadr[APIC_EOI_REG] = 0;
612 					isr &= ~(1 << j);
613 					apic_error |= APIC_ERR_BOOT_EOI;
614 				}
615 	}
616 
617 	/* set a flag so we know we have run apic_picinit() */
618 	apic_picinit_called = 1;
619 	LOCK_INIT_CLEAR(&apic_gethrtime_lock);
620 	LOCK_INIT_CLEAR(&apic_ioapic_lock);
621 	LOCK_INIT_CLEAR(&apic_error_lock);
622 
623 	picsetup();	 /* initialise the 8259 */
624 
625 	/* add nmi handler - least priority nmi handler */
626 	LOCK_INIT_CLEAR(&apic_nmi_lock);
627 
628 	if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr,
629 	    "pcplusmp NMI handler", (caddr_t)NULL))
630 		cmn_err(CE_WARN, "pcplusmp: Unable to add nmi handler");
631 
632 	apic_init_intr();
633 
634 	/* enable apic mode if imcr present */
635 	if (apic_imcrp) {
636 		outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
637 		outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC);
638 	}
639 
640 	ioapic_init_intr(IOAPIC_MASK);
641 }
642 
643 
644 /*ARGSUSED1*/
645 static int
646 apic_cpu_start(processorid_t cpun, caddr_t arg)
647 {
648 	int		loop_count;
649 	uint32_t	vector;
650 	uint_t		cpu_id;
651 	ulong_t		iflag;
652 
653 	cpu_id = apic_cpus[cpun].aci_local_id;
654 
655 	apic_cmos_ssb_set = 1;
656 
657 	/*
658 	 * Interrupts on BSP cpu will be disabled during these startup
659 	 * steps in order to avoid unwanted side effects from
660 	 * executing interrupt handlers on a problematic BIOS.
661 	 */
662 
663 	iflag = intr_clear();
664 	outb(CMOS_ADDR, SSB);
665 	outb(CMOS_DATA, BIOS_SHUTDOWN);
666 
667 	while (get_apic_cmd1() & AV_PENDING)
668 		apic_ret();
669 
670 	/* for integrated - make sure there is one INIT IPI in buffer */
671 	/* for external - it will wake up the cpu */
672 	apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
673 	apicadr[APIC_INT_CMD1] = AV_ASSERT | AV_RESET;
674 
675 	/* If only 1 CPU is installed, PENDING bit will not go low */
676 	for (loop_count = 0x1000; loop_count; loop_count--)
677 		if (get_apic_cmd1() & AV_PENDING)
678 			apic_ret();
679 		else
680 			break;
681 
682 	apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
683 	apicadr[APIC_INT_CMD1] = AV_DEASSERT | AV_RESET;
684 
685 	drv_usecwait(20000);		/* 20 milli sec */
686 
687 	if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) {
688 		/* integrated apic */
689 
690 		vector = (rm_platter_pa >> MMU_PAGESHIFT) &
691 		    (APIC_VECTOR_MASK | APIC_IPL_MASK);
692 
693 		/* to offset the INIT IPI queue up in the buffer */
694 		apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
695 		apicadr[APIC_INT_CMD1] = vector | AV_STARTUP;
696 
697 		drv_usecwait(200);		/* 20 micro sec */
698 
699 		apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
700 		apicadr[APIC_INT_CMD1] = vector | AV_STARTUP;
701 
702 		drv_usecwait(200);		/* 20 micro sec */
703 	}
704 	intr_restore(iflag);
705 	return (0);
706 }
707 
708 
709 #ifdef	DEBUG
710 int	apic_break_on_cpu = 9;
711 int	apic_stretch_interrupts = 0;
712 int	apic_stretch_ISR = 1 << 3;	/* IPL of 3 matches nothing now */
713 
714 void
715 apic_break()
716 {
717 }
718 #endif /* DEBUG */
719 
720 /*
721  * platform_intr_enter
722  *
723  *	Called at the beginning of the interrupt service routine to
724  *	mask all level equal to and below the interrupt priority
725  *	of the interrupting vector.  An EOI should be given to
726  *	the interrupt controller to enable other HW interrupts.
727  *
728  *	Return -1 for spurious interrupts
729  *
730  */
731 /*ARGSUSED*/
732 static int
733 apic_intr_enter(int ipl, int *vectorp)
734 {
735 	uchar_t vector;
736 	int nipl;
737 	int irq;
738 	ulong_t iflag;
739 	apic_cpus_info_t *cpu_infop;
740 
741 	/*
742 	 * The real vector delivered is (*vectorp + 0x20), but our caller
743 	 * subtracts 0x20 from the vector before passing it to us.
744 	 * (That's why APIC_BASE_VECT is 0x20.)
745 	 */
746 	vector = (uchar_t)*vectorp;
747 
748 	/* if interrupted by the clock, increment apic_nsec_since_boot */
749 	if (vector == apic_clkvect) {
750 		if (!apic_oneshot) {
751 			/* NOTE: this is not MT aware */
752 			apic_hrtime_stamp++;
753 			apic_nsec_since_boot += apic_nsec_per_intr;
754 			apic_hrtime_stamp++;
755 			last_count_read = apic_hertz_count;
756 			apic_redistribute_compute();
757 		}
758 
759 		/* We will avoid all the book keeping overhead for clock */
760 		nipl = apic_ipls[vector];
761 
762 #if defined(__amd64)
763 		setcr8((ulong_t)apic_cr8pri[nipl]);
764 #else
765 		apicadr[APIC_TASK_REG] = apic_ipltopri[nipl];
766 #endif
767 		*vectorp = apic_vector_to_irq[vector + APIC_BASE_VECT];
768 		apicadr[APIC_EOI_REG] = 0;
769 		return (nipl);
770 	}
771 
772 	cpu_infop = &apic_cpus[psm_get_cpu_id()];
773 
774 	if (vector == (APIC_SPUR_INTR - APIC_BASE_VECT)) {
775 		cpu_infop->aci_spur_cnt++;
776 		return (APIC_INT_SPURIOUS);
777 	}
778 
779 	/* Check if the vector we got is really what we need */
780 	if (apic_revector_pending) {
781 		/*
782 		 * Disable interrupts for the duration of
783 		 * the vector translation to prevent a self-race for
784 		 * the apic_revector_lock.  This cannot be done
785 		 * in apic_xlate_vector because it is recursive and
786 		 * we want the vector translation to be atomic with
787 		 * respect to other (higher-priority) interrupts.
788 		 */
789 		iflag = intr_clear();
790 		vector = apic_xlate_vector(vector + APIC_BASE_VECT) -
791 		    APIC_BASE_VECT;
792 		intr_restore(iflag);
793 	}
794 
795 	nipl = apic_ipls[vector];
796 	*vectorp = irq = apic_vector_to_irq[vector + APIC_BASE_VECT];
797 
798 #if defined(__amd64)
799 	setcr8((ulong_t)apic_cr8pri[nipl]);
800 #else
801 	apicadr[APIC_TASK_REG] = apic_ipltopri[nipl];
802 #endif
803 
804 	cpu_infop->aci_current[nipl] = (uchar_t)irq;
805 	cpu_infop->aci_curipl = (uchar_t)nipl;
806 	cpu_infop->aci_ISR_in_progress |= 1 << nipl;
807 
808 	/*
809 	 * apic_level_intr could have been assimilated into the irq struct.
810 	 * but, having it as a character array is more efficient in terms of
811 	 * cache usage. So, we leave it as is.
812 	 */
813 	if (!apic_level_intr[irq])
814 		apicadr[APIC_EOI_REG] = 0;
815 
816 #ifdef	DEBUG
817 	APIC_DEBUG_BUF_PUT(vector);
818 	APIC_DEBUG_BUF_PUT(irq);
819 	APIC_DEBUG_BUF_PUT(nipl);
820 	APIC_DEBUG_BUF_PUT(psm_get_cpu_id());
821 	if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl)))
822 		drv_usecwait(apic_stretch_interrupts);
823 
824 	if (apic_break_on_cpu == psm_get_cpu_id())
825 		apic_break();
826 #endif /* DEBUG */
827 	return (nipl);
828 }
829 
830 void
831 apic_intr_exit(int prev_ipl, int irq)
832 {
833 	apic_cpus_info_t *cpu_infop;
834 
835 #if defined(__amd64)
836 	setcr8((ulong_t)apic_cr8pri[prev_ipl]);
837 #else
838 	apicadr[APIC_TASK_REG] = apic_ipltopri[prev_ipl];
839 #endif
840 
841 	cpu_infop = &apic_cpus[psm_get_cpu_id()];
842 	if (apic_level_intr[irq])
843 		apicadr[APIC_EOI_REG] = 0;
844 
845 	cpu_infop->aci_curipl = (uchar_t)prev_ipl;
846 	/* ISR above current pri could not be in progress */
847 	cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1;
848 }
849 
850 intr_exit_fn_t
851 psm_intr_exit_fn(void)
852 {
853 	return (apic_intr_exit);
854 }
855 
856 /*
857  * Mask all interrupts below or equal to the given IPL
858  */
859 static void
860 apic_setspl(int ipl)
861 {
862 
863 #if defined(__amd64)
864 	setcr8((ulong_t)apic_cr8pri[ipl]);
865 #else
866 	apicadr[APIC_TASK_REG] = apic_ipltopri[ipl];
867 #endif
868 
869 	/* interrupts at ipl above this cannot be in progress */
870 	apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1;
871 	/*
872 	 * this is a patch fix for the ALR QSMP P5 machine, so that interrupts
873 	 * have enough time to come in before the priority is raised again
874 	 * during the idle() loop.
875 	 */
876 	if (apic_setspl_delay)
877 		(void) get_apic_pri();
878 }
879 
880 /*
881  * generates an interprocessor interrupt to another CPU
882  */
883 static void
884 apic_send_ipi(int cpun, int ipl)
885 {
886 	int vector;
887 	ulong_t flag;
888 
889 	vector = apic_resv_vector[ipl];
890 
891 	flag = intr_clear();
892 
893 	while (get_apic_cmd1() & AV_PENDING)
894 		apic_ret();
895 
896 	apicadr[APIC_INT_CMD2] =
897 	    apic_cpus[cpun].aci_local_id << APIC_ICR_ID_BIT_OFFSET;
898 	apicadr[APIC_INT_CMD1] = vector;
899 
900 	intr_restore(flag);
901 }
902 
903 
904 /*ARGSUSED*/
905 static void
906 apic_set_idlecpu(processorid_t cpun)
907 {
908 }
909 
910 /*ARGSUSED*/
911 static void
912 apic_unset_idlecpu(processorid_t cpun)
913 {
914 }
915 
916 
917 static void
918 apic_ret()
919 {
920 }
921 
922 static int
923 get_apic_cmd1()
924 {
925 	return (apicadr[APIC_INT_CMD1]);
926 }
927 
928 static int
929 get_apic_pri()
930 {
931 #if defined(__amd64)
932 	return ((int)getcr8());
933 #else
934 	return (apicadr[APIC_TASK_REG]);
935 #endif
936 }
937 
938 /*
939  * If apic_coarse_time == 1, then apic_gettime() is used instead of
940  * apic_gethrtime().  This is used for performance instead of accuracy.
941  */
942 
943 static hrtime_t
944 apic_gettime()
945 {
946 	int old_hrtime_stamp;
947 	hrtime_t temp;
948 
949 	/*
950 	 * In one-shot mode, we do not keep time, so if anyone
951 	 * calls psm_gettime() directly, we vector over to
952 	 * gethrtime().
953 	 * one-shot mode MUST NOT be enabled if this psm is the source of
954 	 * hrtime.
955 	 */
956 
957 	if (apic_oneshot)
958 		return (gethrtime());
959 
960 
961 gettime_again:
962 	while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
963 		apic_ret();
964 
965 	temp = apic_nsec_since_boot;
966 
967 	if (apic_hrtime_stamp != old_hrtime_stamp) {	/* got an interrupt */
968 		goto gettime_again;
969 	}
970 	return (temp);
971 }
972 
973 /*
974  * Here we return the number of nanoseconds since booting.  Note every
975  * clock interrupt increments apic_nsec_since_boot by the appropriate
976  * amount.
977  */
978 static hrtime_t
979 apic_gethrtime()
980 {
981 	int curr_timeval, countval, elapsed_ticks;
982 	int old_hrtime_stamp, status;
983 	hrtime_t temp;
984 	uchar_t	cpun;
985 	ulong_t oflags;
986 
987 	/*
988 	 * In one-shot mode, we do not keep time, so if anyone
989 	 * calls psm_gethrtime() directly, we vector over to
990 	 * gethrtime().
991 	 * one-shot mode MUST NOT be enabled if this psm is the source of
992 	 * hrtime.
993 	 */
994 
995 	if (apic_oneshot)
996 		return (gethrtime());
997 
998 	oflags = intr_clear();	/* prevent migration */
999 
1000 	cpun = (uchar_t)((uint_t)apicadr[APIC_LID_REG] >> APIC_ID_BIT_OFFSET);
1001 
1002 	lock_set(&apic_gethrtime_lock);
1003 
1004 gethrtime_again:
1005 	while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
1006 		apic_ret();
1007 
1008 	/*
1009 	 * Check to see which CPU we are on.  Note the time is kept on
1010 	 * the local APIC of CPU 0.  If on CPU 0, simply read the current
1011 	 * counter.  If on another CPU, issue a remote read command to CPU 0.
1012 	 */
1013 	if (cpun == apic_cpus[0].aci_local_id) {
1014 		countval = apicadr[APIC_CURR_COUNT];
1015 	} else {
1016 		while (get_apic_cmd1() & AV_PENDING)
1017 			apic_ret();
1018 
1019 		apicadr[APIC_INT_CMD2] =
1020 		    apic_cpus[0].aci_local_id << APIC_ICR_ID_BIT_OFFSET;
1021 		apicadr[APIC_INT_CMD1] = APIC_CURR_ADD|AV_REMOTE;
1022 
1023 		while ((status = get_apic_cmd1()) & AV_READ_PENDING)
1024 			apic_ret();
1025 
1026 		if (status & AV_REMOTE_STATUS)	/* 1 = valid */
1027 			countval = apicadr[APIC_REMOTE_READ];
1028 		else {	/* 0 = invalid */
1029 			apic_remote_hrterr++;
1030 			/*
1031 			 * return last hrtime right now, will need more
1032 			 * testing if change to retry
1033 			 */
1034 			temp = apic_last_hrtime;
1035 
1036 			lock_clear(&apic_gethrtime_lock);
1037 
1038 			intr_restore(oflags);
1039 
1040 			return (temp);
1041 		}
1042 	}
1043 	if (countval > last_count_read)
1044 		countval = 0;
1045 	else
1046 		last_count_read = countval;
1047 
1048 	elapsed_ticks = apic_hertz_count - countval;
1049 
1050 	curr_timeval = APIC_TICKS_TO_NSECS(elapsed_ticks);
1051 	temp = apic_nsec_since_boot + curr_timeval;
1052 
1053 	if (apic_hrtime_stamp != old_hrtime_stamp) {	/* got an interrupt */
1054 		/* we might have clobbered last_count_read. Restore it */
1055 		last_count_read = apic_hertz_count;
1056 		goto gethrtime_again;
1057 	}
1058 
1059 	if (temp < apic_last_hrtime) {
1060 		/* return last hrtime if error occurs */
1061 		apic_hrtime_error++;
1062 		temp = apic_last_hrtime;
1063 	}
1064 	else
1065 		apic_last_hrtime = temp;
1066 
1067 	lock_clear(&apic_gethrtime_lock);
1068 	intr_restore(oflags);
1069 
1070 	return (temp);
1071 }
1072 
1073 /* apic NMI handler */
1074 /*ARGSUSED*/
1075 static void
1076 apic_nmi_intr(caddr_t arg, struct regs *rp)
1077 {
1078 	if (apic_shutdown_processors) {
1079 		apic_disable_local_apic();
1080 		return;
1081 	}
1082 
1083 	apic_error |= APIC_ERR_NMI;
1084 
1085 	if (!lock_try(&apic_nmi_lock))
1086 		return;
1087 	apic_num_nmis++;
1088 
1089 	if (apic_kmdb_on_nmi && psm_debugger()) {
1090 		debug_enter("NMI received: entering kmdb\n");
1091 	} else if (apic_panic_on_nmi) {
1092 		/* Keep panic from entering kmdb. */
1093 		nopanicdebug = 1;
1094 		panic("NMI received\n");
1095 	} else {
1096 		/*
1097 		 * prom_printf is the best shot we have of something which is
1098 		 * problem free from high level/NMI type of interrupts
1099 		 */
1100 		prom_printf("NMI received\n");
1101 	}
1102 
1103 	lock_clear(&apic_nmi_lock);
1104 }
1105 
1106 /*ARGSUSED*/
1107 static int
1108 apic_addspl(int irqno, int ipl, int min_ipl, int max_ipl)
1109 {
1110 	return (apic_addspl_common(irqno, ipl, min_ipl, max_ipl));
1111 }
1112 
1113 static int
1114 apic_delspl(int irqno, int ipl, int min_ipl, int max_ipl)
1115 {
1116 	return (apic_delspl_common(irqno, ipl, min_ipl,  max_ipl));
1117 }
1118 
1119 static int
1120 apic_post_cpu_start()
1121 {
1122 	int i, cpun;
1123 	ulong_t iflag;
1124 	apic_irq_t *irq_ptr;
1125 
1126 	splx(ipltospl(LOCK_LEVEL));
1127 	apic_init_intr();
1128 
1129 	/*
1130 	 * since some systems don't enable the internal cache on the non-boot
1131 	 * cpus, so we have to enable them here
1132 	 */
1133 	setcr0(getcr0() & ~(CR0_CD | CR0_NW));
1134 
1135 	while (get_apic_cmd1() & AV_PENDING)
1136 		apic_ret();
1137 
1138 	cpun = psm_get_cpu_id();
1139 	apic_cpus[cpun].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
1140 
1141 	for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
1142 		irq_ptr = apic_irq_table[i];
1143 		if ((irq_ptr == NULL) ||
1144 		    ((irq_ptr->airq_cpu & ~IRQ_USER_BOUND) != cpun))
1145 			continue;
1146 
1147 		while (irq_ptr) {
1148 			if (irq_ptr->airq_temp_cpu != IRQ_UNINIT) {
1149 				iflag = intr_clear();
1150 				lock_set(&apic_ioapic_lock);
1151 
1152 				(void) apic_rebind(irq_ptr, cpun, NULL);
1153 
1154 				lock_clear(&apic_ioapic_lock);
1155 				intr_restore(iflag);
1156 			}
1157 			irq_ptr = irq_ptr->airq_next;
1158 		}
1159 	}
1160 
1161 	apicadr[APIC_DIVIDE_REG] = apic_divide_reg_init;
1162 	return (PSM_SUCCESS);
1163 }
1164 
1165 processorid_t
1166 apic_get_next_processorid(processorid_t cpu_id)
1167 {
1168 
1169 	int i;
1170 
1171 	if (cpu_id == -1)
1172 		return ((processorid_t)0);
1173 
1174 	for (i = cpu_id + 1; i < NCPU; i++) {
1175 		if (CPU_IN_SET(apic_cpumask, i))
1176 			return (i);
1177 	}
1178 
1179 	return ((processorid_t)-1);
1180 }
1181 
1182 
1183 /*
1184  * type == -1 indicates it is an internal request. Do not change
1185  * resv_vector for these requests
1186  */
1187 static int
1188 apic_get_ipivect(int ipl, int type)
1189 {
1190 	uchar_t vector;
1191 	int irq;
1192 
1193 	if (irq = apic_allocate_irq(APIC_VECTOR(ipl))) {
1194 		if (vector = apic_allocate_vector(ipl, irq, 1)) {
1195 			apic_irq_table[irq]->airq_mps_intr_index =
1196 			    RESERVE_INDEX;
1197 			apic_irq_table[irq]->airq_vector = vector;
1198 			if (type != -1) {
1199 				apic_resv_vector[ipl] = vector;
1200 			}
1201 			return (irq);
1202 		}
1203 	}
1204 	apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
1205 	return (-1);	/* shouldn't happen */
1206 }
1207 
1208 static int
1209 apic_getclkirq(int ipl)
1210 {
1211 	int	irq;
1212 
1213 	if ((irq = apic_get_ipivect(ipl, -1)) == -1)
1214 		return (-1);
1215 	/*
1216 	 * Note the vector in apic_clkvect for per clock handling.
1217 	 */
1218 	apic_clkvect = apic_irq_table[irq]->airq_vector - APIC_BASE_VECT;
1219 	APIC_VERBOSE_IOAPIC((CE_NOTE, "get_clkirq: vector = %x\n",
1220 	    apic_clkvect));
1221 	return (irq);
1222 }
1223 
1224 
1225 /*
1226  * Return the number of APIC clock ticks elapsed for 8245 to decrement
1227  * (APIC_TIME_COUNT + pit_ticks_adj) ticks.
1228  */
1229 static uint_t
1230 apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj)
1231 {
1232 	uint8_t		pit_tick_lo;
1233 	uint16_t	pit_tick, target_pit_tick;
1234 	uint32_t	start_apic_tick, end_apic_tick;
1235 	ulong_t		iflag;
1236 
1237 	addr += APIC_CURR_COUNT;
1238 
1239 	iflag = intr_clear();
1240 
1241 	do {
1242 		pit_tick_lo = inb(PITCTR0_PORT);
1243 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1244 	} while (pit_tick < APIC_TIME_MIN ||
1245 	    pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX);
1246 
1247 	/*
1248 	 * Wait for the 8254 to decrement by 5 ticks to ensure
1249 	 * we didn't start in the middle of a tick.
1250 	 * Compare with 0x10 for the wrap around case.
1251 	 */
1252 	target_pit_tick = pit_tick - 5;
1253 	do {
1254 		pit_tick_lo = inb(PITCTR0_PORT);
1255 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1256 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1257 
1258 	start_apic_tick = *addr;
1259 
1260 	/*
1261 	 * Wait for the 8254 to decrement by
1262 	 * (APIC_TIME_COUNT + pit_ticks_adj) ticks
1263 	 */
1264 	target_pit_tick = pit_tick - APIC_TIME_COUNT;
1265 	do {
1266 		pit_tick_lo = inb(PITCTR0_PORT);
1267 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1268 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1269 
1270 	end_apic_tick = *addr;
1271 
1272 	*pit_ticks_adj = target_pit_tick - pit_tick;
1273 
1274 	intr_restore(iflag);
1275 
1276 	return (start_apic_tick - end_apic_tick);
1277 }
1278 
1279 /*
1280  * Initialise the APIC timer on the local APIC of CPU 0 to the desired
1281  * frequency.  Note at this stage in the boot sequence, the boot processor
1282  * is the only active processor.
1283  * hertz value of 0 indicates a one-shot mode request.  In this case
1284  * the function returns the resolution (in nanoseconds) for the hardware
1285  * timer interrupt.  If one-shot mode capability is not available,
1286  * the return value will be 0. apic_enable_oneshot is a global switch
1287  * for disabling the functionality.
1288  * A non-zero positive value for hertz indicates a periodic mode request.
1289  * In this case the hardware will be programmed to generate clock interrupts
1290  * at hertz frequency and returns the resolution of interrupts in
1291  * nanosecond.
1292  */
1293 
1294 static int
1295 apic_clkinit(int hertz)
1296 {
1297 	uint_t		apic_ticks = 0;
1298 	uint_t		pit_ticks;
1299 	int		ret;
1300 	uint16_t	pit_ticks_adj;
1301 	static int	firsttime = 1;
1302 
1303 	if (firsttime) {
1304 		/* first time calibrate on CPU0 only */
1305 
1306 		apicadr[APIC_DIVIDE_REG] = apic_divide_reg_init;
1307 		apicadr[APIC_INIT_COUNT] = APIC_MAXVAL;
1308 		apic_ticks = apic_calibrate(apicadr, &pit_ticks_adj);
1309 
1310 		/* total number of PIT ticks corresponding to apic_ticks */
1311 		pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
1312 
1313 		/*
1314 		 * Determine the number of nanoseconds per APIC clock tick
1315 		 * and then determine how many APIC ticks to interrupt at the
1316 		 * desired frequency
1317 		 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
1318 		 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
1319 		 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
1320 		 * pic_ticks_per_SFns =
1321 		 *   (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
1322 		 */
1323 		apic_ticks_per_SFnsecs =
1324 		    ((SF * apic_ticks * PIT_HZ) /
1325 		    ((uint64_t)pit_ticks * NANOSEC));
1326 
1327 		/* the interval timer initial count is 32 bit max */
1328 		apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL);
1329 		firsttime = 0;
1330 	}
1331 
1332 	if (hertz != 0) {
1333 		/* periodic */
1334 		apic_nsec_per_intr = NANOSEC / hertz;
1335 		apic_hertz_count = APIC_NSECS_TO_TICKS(apic_nsec_per_intr);
1336 	}
1337 
1338 	apic_int_busy_mark = (apic_int_busy_mark *
1339 	    apic_sample_factor_redistribution) / 100;
1340 	apic_int_free_mark = (apic_int_free_mark *
1341 	    apic_sample_factor_redistribution) / 100;
1342 	apic_diff_for_redistribution = (apic_diff_for_redistribution *
1343 	    apic_sample_factor_redistribution) / 100;
1344 
1345 	if (hertz == 0) {
1346 		/* requested one_shot */
1347 		if (!tsc_gethrtime_enable || !apic_oneshot_enable)
1348 			return (0);
1349 		apic_oneshot = 1;
1350 		ret = (int)APIC_TICKS_TO_NSECS(1);
1351 	} else {
1352 		/* program the local APIC to interrupt at the given frequency */
1353 		apicadr[APIC_INIT_COUNT] = apic_hertz_count;
1354 		apicadr[APIC_LOCAL_TIMER] =
1355 		    (apic_clkvect + APIC_BASE_VECT) | AV_TIME;
1356 		apic_oneshot = 0;
1357 		ret = NANOSEC / hertz;
1358 	}
1359 
1360 	return (ret);
1361 
1362 }
1363 
1364 /*
1365  * apic_preshutdown:
1366  * Called early in shutdown whilst we can still access filesystems to do
1367  * things like loading modules which will be required to complete shutdown
1368  * after filesystems are all unmounted.
1369  */
1370 static void
1371 apic_preshutdown(int cmd, int fcn)
1372 {
1373 	APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n",
1374 	    cmd, fcn, apic_poweroff_method, apic_enable_acpi));
1375 
1376 	if ((cmd != A_SHUTDOWN) || (fcn != AD_POWEROFF)) {
1377 		return;
1378 	}
1379 }
1380 
1381 static void
1382 apic_shutdown(int cmd, int fcn)
1383 {
1384 	int restarts, attempts;
1385 	int i;
1386 	uchar_t	byte;
1387 	ulong_t iflag;
1388 
1389 	/* Send NMI to all CPUs except self to do per processor shutdown */
1390 	iflag = intr_clear();
1391 	while (get_apic_cmd1() & AV_PENDING)
1392 		apic_ret();
1393 	apic_shutdown_processors = 1;
1394 	apicadr[APIC_INT_CMD1] = AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF;
1395 
1396 	/* restore cmos shutdown byte before reboot */
1397 	if (apic_cmos_ssb_set) {
1398 		outb(CMOS_ADDR, SSB);
1399 		outb(CMOS_DATA, 0);
1400 	}
1401 
1402 	ioapic_disable_redirection();
1403 
1404 	/*	disable apic mode if imcr present	*/
1405 	if (apic_imcrp) {
1406 		outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
1407 		outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC);
1408 	}
1409 
1410 	apic_disable_local_apic();
1411 
1412 	intr_restore(iflag);
1413 
1414 	/* remainder of function is for shutdown cases only */
1415 	if (cmd != A_SHUTDOWN)
1416 		return;
1417 
1418 	/*
1419 	 * Switch system back into Legacy-Mode if using ACPI and
1420 	 * not powering-off.  Some BIOSes need to remain in ACPI-mode
1421 	 * for power-off to succeed (Dell Dimension 4600)
1422 	 */
1423 	if (apic_enable_acpi && (fcn != AD_POWEROFF))
1424 		(void) AcpiDisable();
1425 
1426 	/* remainder of function is for shutdown+poweroff case only */
1427 	if (fcn != AD_POWEROFF)
1428 		return;
1429 
1430 	switch (apic_poweroff_method) {
1431 		case APIC_POWEROFF_VIA_RTC:
1432 
1433 			/* select the extended NVRAM bank in the RTC */
1434 			outb(CMOS_ADDR, RTC_REGA);
1435 			byte = inb(CMOS_DATA);
1436 			outb(CMOS_DATA, (byte | EXT_BANK));
1437 
1438 			outb(CMOS_ADDR, PFR_REG);
1439 
1440 			/* for Predator must toggle the PAB bit */
1441 			byte = inb(CMOS_DATA);
1442 
1443 			/*
1444 			 * clear power active bar, wakeup alarm and
1445 			 * kickstart
1446 			 */
1447 			byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG);
1448 			outb(CMOS_DATA, byte);
1449 
1450 			/* delay before next write */
1451 			drv_usecwait(1000);
1452 
1453 			/* for S40 the following would suffice */
1454 			byte = inb(CMOS_DATA);
1455 
1456 			/* power active bar control bit */
1457 			byte |= PAB_CBIT;
1458 			outb(CMOS_DATA, byte);
1459 
1460 			break;
1461 
1462 		case APIC_POWEROFF_VIA_ASPEN_BMC:
1463 			restarts = 0;
1464 restart_aspen_bmc:
1465 			if (++restarts == 3)
1466 				break;
1467 			attempts = 0;
1468 			do {
1469 				byte = inb(MISMIC_FLAG_REGISTER);
1470 				byte &= MISMIC_BUSY_MASK;
1471 				if (byte != 0) {
1472 					drv_usecwait(1000);
1473 					if (attempts >= 3)
1474 						goto restart_aspen_bmc;
1475 					++attempts;
1476 				}
1477 			} while (byte != 0);
1478 			outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS);
1479 			byte = inb(MISMIC_FLAG_REGISTER);
1480 			byte |= 0x1;
1481 			outb(MISMIC_FLAG_REGISTER, byte);
1482 			i = 0;
1483 			for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0]));
1484 			    i++) {
1485 				attempts = 0;
1486 				do {
1487 					byte = inb(MISMIC_FLAG_REGISTER);
1488 					byte &= MISMIC_BUSY_MASK;
1489 					if (byte != 0) {
1490 						drv_usecwait(1000);
1491 						if (attempts >= 3)
1492 							goto restart_aspen_bmc;
1493 						++attempts;
1494 					}
1495 				} while (byte != 0);
1496 				outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl);
1497 				outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data);
1498 				byte = inb(MISMIC_FLAG_REGISTER);
1499 				byte |= 0x1;
1500 				outb(MISMIC_FLAG_REGISTER, byte);
1501 			}
1502 			break;
1503 
1504 		case APIC_POWEROFF_VIA_SITKA_BMC:
1505 			restarts = 0;
1506 restart_sitka_bmc:
1507 			if (++restarts == 3)
1508 				break;
1509 			attempts = 0;
1510 			do {
1511 				byte = inb(SMS_STATUS_REGISTER);
1512 				byte &= SMS_STATE_MASK;
1513 				if ((byte == SMS_READ_STATE) ||
1514 				    (byte == SMS_WRITE_STATE)) {
1515 					drv_usecwait(1000);
1516 					if (attempts >= 3)
1517 						goto restart_sitka_bmc;
1518 					++attempts;
1519 				}
1520 			} while ((byte == SMS_READ_STATE) ||
1521 			    (byte == SMS_WRITE_STATE));
1522 			outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS);
1523 			i = 0;
1524 			for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0]));
1525 			    i++) {
1526 				attempts = 0;
1527 				do {
1528 					byte = inb(SMS_STATUS_REGISTER);
1529 					byte &= SMS_IBF_MASK;
1530 					if (byte != 0) {
1531 						drv_usecwait(1000);
1532 						if (attempts >= 3)
1533 							goto restart_sitka_bmc;
1534 						++attempts;
1535 					}
1536 				} while (byte != 0);
1537 				outb(sitka_bmc[i].port, sitka_bmc[i].data);
1538 			}
1539 			break;
1540 
1541 		case APIC_POWEROFF_NONE:
1542 
1543 			/* If no APIC direct method, we will try using ACPI */
1544 			if (apic_enable_acpi) {
1545 				if (acpi_poweroff() == 1)
1546 					return;
1547 			} else
1548 				return;
1549 
1550 			break;
1551 	}
1552 	/*
1553 	 * Wait a limited time here for power to go off.
1554 	 * If the power does not go off, then there was a
1555 	 * problem and we should continue to the halt which
1556 	 * prints a message for the user to press a key to
1557 	 * reboot.
1558 	 */
1559 	drv_usecwait(7000000); /* wait seven seconds */
1560 
1561 }
1562 
1563 /*
1564  * Try and disable all interrupts. We just assign interrupts to other
1565  * processors based on policy. If any were bound by user request, we
1566  * let them continue and return failure. We do not bother to check
1567  * for cache affinity while rebinding.
1568  */
1569 
1570 static int
1571 apic_disable_intr(processorid_t cpun)
1572 {
1573 	int bind_cpu = 0, i, hardbound = 0;
1574 	apic_irq_t *irq_ptr;
1575 	ulong_t iflag;
1576 
1577 	iflag = intr_clear();
1578 	lock_set(&apic_ioapic_lock);
1579 
1580 	for (i = 0; i <= APIC_MAX_VECTOR; i++) {
1581 		if (apic_reprogram_info[i].done == B_FALSE) {
1582 			if (apic_reprogram_info[i].bindcpu == cpun) {
1583 				/*
1584 				 * CPU is busy -- it's the target of
1585 				 * a pending reprogramming attempt
1586 				 */
1587 				lock_clear(&apic_ioapic_lock);
1588 				intr_restore(iflag);
1589 				return (PSM_FAILURE);
1590 			}
1591 		}
1592 	}
1593 
1594 	apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE;
1595 
1596 	apic_cpus[cpun].aci_curipl = 0;
1597 
1598 	i = apic_min_device_irq;
1599 	for (; i <= apic_max_device_irq; i++) {
1600 		/*
1601 		 * If there are bound interrupts on this cpu, then
1602 		 * rebind them to other processors.
1603 		 */
1604 		if ((irq_ptr = apic_irq_table[i]) != NULL) {
1605 			ASSERT((irq_ptr->airq_temp_cpu == IRQ_UNBOUND) ||
1606 			    (irq_ptr->airq_temp_cpu == IRQ_UNINIT) ||
1607 			    ((irq_ptr->airq_temp_cpu & ~IRQ_USER_BOUND) <
1608 			    apic_nproc));
1609 
1610 			if (irq_ptr->airq_temp_cpu == (cpun | IRQ_USER_BOUND)) {
1611 				hardbound = 1;
1612 				continue;
1613 			}
1614 
1615 			if (irq_ptr->airq_temp_cpu == cpun) {
1616 				do {
1617 					bind_cpu = apic_next_bind_cpu++;
1618 					if (bind_cpu >= apic_nproc) {
1619 						apic_next_bind_cpu = 1;
1620 						bind_cpu = 0;
1621 
1622 					}
1623 				} while (apic_rebind_all(irq_ptr, bind_cpu));
1624 			}
1625 		}
1626 	}
1627 
1628 	lock_clear(&apic_ioapic_lock);
1629 	intr_restore(iflag);
1630 
1631 	if (hardbound) {
1632 		cmn_err(CE_WARN, "Could not disable interrupts on %d"
1633 		    "due to user bound interrupts", cpun);
1634 		return (PSM_FAILURE);
1635 	}
1636 	else
1637 		return (PSM_SUCCESS);
1638 }
1639 
1640 static void
1641 apic_enable_intr(processorid_t cpun)
1642 {
1643 	int	i;
1644 	apic_irq_t *irq_ptr;
1645 	ulong_t iflag;
1646 
1647 	iflag = intr_clear();
1648 	lock_set(&apic_ioapic_lock);
1649 
1650 	apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE;
1651 
1652 	i = apic_min_device_irq;
1653 	for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
1654 		if ((irq_ptr = apic_irq_table[i]) != NULL) {
1655 			if ((irq_ptr->airq_cpu & ~IRQ_USER_BOUND) == cpun) {
1656 				(void) apic_rebind_all(irq_ptr,
1657 				    irq_ptr->airq_cpu);
1658 			}
1659 		}
1660 	}
1661 
1662 	lock_clear(&apic_ioapic_lock);
1663 	intr_restore(iflag);
1664 }
1665 
1666 
1667 /*
1668  * This function will reprogram the timer.
1669  *
1670  * When in oneshot mode the argument is the absolute time in future to
1671  * generate the interrupt at.
1672  *
1673  * When in periodic mode, the argument is the interval at which the
1674  * interrupts should be generated. There is no need to support the periodic
1675  * mode timer change at this time.
1676  */
1677 static void
1678 apic_timer_reprogram(hrtime_t time)
1679 {
1680 	hrtime_t now;
1681 	uint_t ticks;
1682 	int64_t delta;
1683 
1684 	/*
1685 	 * We should be called from high PIL context (CBE_HIGH_PIL),
1686 	 * so kpreempt is disabled.
1687 	 */
1688 
1689 	if (!apic_oneshot) {
1690 		/* time is the interval for periodic mode */
1691 		ticks = APIC_NSECS_TO_TICKS(time);
1692 	} else {
1693 		/* one shot mode */
1694 
1695 		now = gethrtime();
1696 		delta = time - now;
1697 
1698 		if (delta <= 0) {
1699 			/*
1700 			 * requested to generate an interrupt in the past
1701 			 * generate an interrupt as soon as possible
1702 			 */
1703 			ticks = apic_min_timer_ticks;
1704 		} else if (delta > apic_nsec_max) {
1705 			/*
1706 			 * requested to generate an interrupt at a time
1707 			 * further than what we are capable of. Set to max
1708 			 * the hardware can handle
1709 			 */
1710 
1711 			ticks = APIC_MAXVAL;
1712 #ifdef DEBUG
1713 			cmn_err(CE_CONT, "apic_timer_reprogram, request at"
1714 			    "  %lld  too far in future, current time"
1715 			    "  %lld \n", time, now);
1716 #endif
1717 		} else
1718 			ticks = APIC_NSECS_TO_TICKS(delta);
1719 	}
1720 
1721 	if (ticks < apic_min_timer_ticks)
1722 		ticks = apic_min_timer_ticks;
1723 
1724 	apicadr[APIC_INIT_COUNT] = ticks;
1725 
1726 }
1727 
1728 /*
1729  * This function will enable timer interrupts.
1730  */
1731 static void
1732 apic_timer_enable(void)
1733 {
1734 	/*
1735 	 * We should be Called from high PIL context (CBE_HIGH_PIL),
1736 	 * so kpreempt is disabled.
1737 	 */
1738 
1739 	if (!apic_oneshot)
1740 		apicadr[APIC_LOCAL_TIMER] =
1741 		    (apic_clkvect + APIC_BASE_VECT) | AV_TIME;
1742 	else {
1743 		/* one shot */
1744 		apicadr[APIC_LOCAL_TIMER] = (apic_clkvect + APIC_BASE_VECT);
1745 	}
1746 }
1747 
1748 /*
1749  * This function will disable timer interrupts.
1750  */
1751 static void
1752 apic_timer_disable(void)
1753 {
1754 	/*
1755 	 * We should be Called from high PIL context (CBE_HIGH_PIL),
1756 	 * so kpreempt is disabled.
1757 	 */
1758 
1759 	apicadr[APIC_LOCAL_TIMER] = (apic_clkvect + APIC_BASE_VECT) | AV_MASK;
1760 }
1761 
1762 
1763 ddi_periodic_t apic_periodic_id;
1764 
1765 /*
1766  * If this module needs a periodic handler for the interrupt distribution, it
1767  * can be added here. The argument to the periodic handler is not currently
1768  * used, but is reserved for future.
1769  */
1770 static void
1771 apic_post_cyclic_setup(void *arg)
1772 {
1773 _NOTE(ARGUNUSED(arg))
1774 	/* cpu_lock is held */
1775 	/* set up a periodic handler for intr redistribution */
1776 
1777 	/*
1778 	 * In peridoc mode intr redistribution processing is done in
1779 	 * apic_intr_enter during clk intr processing
1780 	 */
1781 	if (!apic_oneshot)
1782 		return;
1783 	/*
1784 	 * Register a periodical handler for the redistribution processing.
1785 	 * On X86, CY_LOW_LEVEL is mapped to the level 2 interrupt, so
1786 	 * DDI_IPL_2 should be passed to ddi_periodic_add() here.
1787 	 */
1788 	apic_periodic_id = ddi_periodic_add(
1789 	    (void (*)(void *))apic_redistribute_compute, NULL,
1790 	    apic_redistribute_sample_interval, DDI_IPL_2);
1791 }
1792 
1793 static void
1794 apic_redistribute_compute(void)
1795 {
1796 	int	i, j, max_busy;
1797 
1798 	if (apic_enable_dynamic_migration) {
1799 		if (++apic_nticks == apic_sample_factor_redistribution) {
1800 			/*
1801 			 * Time to call apic_intr_redistribute().
1802 			 * reset apic_nticks. This will cause max_busy
1803 			 * to be calculated below and if it is more than
1804 			 * apic_int_busy, we will do the whole thing
1805 			 */
1806 			apic_nticks = 0;
1807 		}
1808 		max_busy = 0;
1809 		for (i = 0; i < apic_nproc; i++) {
1810 
1811 			/*
1812 			 * Check if curipl is non zero & if ISR is in
1813 			 * progress
1814 			 */
1815 			if (((j = apic_cpus[i].aci_curipl) != 0) &&
1816 			    (apic_cpus[i].aci_ISR_in_progress & (1 << j))) {
1817 
1818 				int	irq;
1819 				apic_cpus[i].aci_busy++;
1820 				irq = apic_cpus[i].aci_current[j];
1821 				apic_irq_table[irq]->airq_busy++;
1822 			}
1823 
1824 			if (!apic_nticks &&
1825 			    (apic_cpus[i].aci_busy > max_busy))
1826 				max_busy = apic_cpus[i].aci_busy;
1827 		}
1828 		if (!apic_nticks) {
1829 			if (max_busy > apic_int_busy_mark) {
1830 			/*
1831 			 * We could make the following check be
1832 			 * skipped > 1 in which case, we get a
1833 			 * redistribution at half the busy mark (due to
1834 			 * double interval). Need to be able to collect
1835 			 * more empirical data to decide if that is a
1836 			 * good strategy. Punt for now.
1837 			 */
1838 				if (apic_skipped_redistribute) {
1839 					apic_cleanup_busy();
1840 					apic_skipped_redistribute = 0;
1841 				} else {
1842 					apic_intr_redistribute();
1843 				}
1844 			} else
1845 				apic_skipped_redistribute++;
1846 		}
1847 	}
1848 }
1849 
1850 
1851 /*
1852  * The following functions are in the platform specific file so that they
1853  * can be different functions depending on whether we are running on
1854  * bare metal or a hypervisor.
1855  */
1856 
1857 /*
1858  * map an apic for memory-mapped access
1859  */
1860 uint32_t *
1861 mapin_apic(uint32_t addr, size_t len, int flags)
1862 {
1863 	/*LINTED: pointer cast may result in improper alignment */
1864 	return ((uint32_t *)psm_map_phys(addr, len, flags));
1865 }
1866 
1867 uint32_t *
1868 mapin_ioapic(uint32_t addr, size_t len, int flags)
1869 {
1870 	return (mapin_apic(addr, len, flags));
1871 }
1872 
1873 /*
1874  * unmap an apic
1875  */
1876 void
1877 mapout_apic(caddr_t addr, size_t len)
1878 {
1879 	psm_unmap_phys(addr, len);
1880 }
1881 
1882 void
1883 mapout_ioapic(caddr_t addr, size_t len)
1884 {
1885 	mapout_apic(addr, len);
1886 }
1887 
1888 /*
1889  * Check to make sure there are enough irq slots
1890  */
1891 int
1892 apic_check_free_irqs(int count)
1893 {
1894 	int i, avail;
1895 
1896 	avail = 0;
1897 	for (i = APIC_FIRST_FREE_IRQ; i < APIC_RESV_IRQ; i++) {
1898 		if ((apic_irq_table[i] == NULL) ||
1899 		    apic_irq_table[i]->airq_mps_intr_index == FREE_INDEX) {
1900 			if (++avail >= count)
1901 				return (PSM_SUCCESS);
1902 		}
1903 	}
1904 	return (PSM_FAILURE);
1905 }
1906 
1907 /*
1908  * This function allocates "count" MSI vector(s) for the given "dip/pri/type"
1909  */
1910 int
1911 apic_alloc_msi_vectors(dev_info_t *dip, int inum, int count, int pri,
1912     int behavior)
1913 {
1914 	int	rcount, i;
1915 	uchar_t	start, irqno, cpu;
1916 	major_t	major;
1917 	apic_irq_t	*irqptr;
1918 
1919 	DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: dip=0x%p "
1920 	    "inum=0x%x  pri=0x%x count=0x%x behavior=%d\n",
1921 	    (void *)dip, inum, pri, count, behavior));
1922 
1923 	if (count > 1) {
1924 		if (behavior == DDI_INTR_ALLOC_STRICT &&
1925 		    (apic_multi_msi_enable == 0 || count > apic_multi_msi_max))
1926 			return (0);
1927 
1928 		if (apic_multi_msi_enable == 0)
1929 			count = 1;
1930 		else if (count > apic_multi_msi_max)
1931 			count = apic_multi_msi_max;
1932 	}
1933 
1934 	if ((rcount = apic_navail_vector(dip, pri)) > count)
1935 		rcount = count;
1936 	else if (rcount == 0 || (rcount < count &&
1937 	    behavior == DDI_INTR_ALLOC_STRICT))
1938 		return (0);
1939 
1940 	/* if not ISP2, then round it down */
1941 	if (!ISP2(rcount))
1942 		rcount = 1 << (highbit(rcount) - 1);
1943 
1944 	mutex_enter(&airq_mutex);
1945 
1946 	for (start = 0; rcount > 0; rcount >>= 1) {
1947 		if ((start = apic_find_multi_vectors(pri, rcount)) != 0 ||
1948 		    behavior == DDI_INTR_ALLOC_STRICT)
1949 			break;
1950 	}
1951 
1952 	if (start == 0) {
1953 		/* no vector available */
1954 		mutex_exit(&airq_mutex);
1955 		return (0);
1956 	}
1957 
1958 	if (apic_check_free_irqs(rcount) == PSM_FAILURE) {
1959 		/* not enough free irq slots available */
1960 		mutex_exit(&airq_mutex);
1961 		return (0);
1962 	}
1963 
1964 	major = (dip != NULL) ? ddi_name_to_major(ddi_get_name(dip)) : 0;
1965 	for (i = 0; i < rcount; i++) {
1966 		if ((irqno = apic_allocate_irq(apic_first_avail_irq)) ==
1967 		    (uchar_t)-1) {
1968 			/*
1969 			 * shouldn't happen because of the
1970 			 * apic_check_free_irqs() check earlier
1971 			 */
1972 			mutex_exit(&airq_mutex);
1973 			DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: "
1974 			    "apic_allocate_irq failed\n"));
1975 			return (i);
1976 		}
1977 		apic_max_device_irq = max(irqno, apic_max_device_irq);
1978 		apic_min_device_irq = min(irqno, apic_min_device_irq);
1979 		irqptr = apic_irq_table[irqno];
1980 #ifdef	DEBUG
1981 		if (apic_vector_to_irq[start + i] != APIC_RESV_IRQ)
1982 			DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: "
1983 			    "apic_vector_to_irq is not APIC_RESV_IRQ\n"));
1984 #endif
1985 		apic_vector_to_irq[start + i] = (uchar_t)irqno;
1986 
1987 		irqptr->airq_vector = (uchar_t)(start + i);
1988 		irqptr->airq_ioapicindex = (uchar_t)inum;	/* start */
1989 		irqptr->airq_intin_no = (uchar_t)rcount;
1990 		irqptr->airq_ipl = pri;
1991 		irqptr->airq_vector = start + i;
1992 		irqptr->airq_origirq = (uchar_t)(inum + i);
1993 		irqptr->airq_share_id = 0;
1994 		irqptr->airq_mps_intr_index = MSI_INDEX;
1995 		irqptr->airq_dip = dip;
1996 		irqptr->airq_major = major;
1997 		if (i == 0) /* they all bound to the same cpu */
1998 			cpu = irqptr->airq_cpu = apic_bind_intr(dip, irqno,
1999 			    0xff, 0xff);
2000 		else
2001 			irqptr->airq_cpu = cpu;
2002 		DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: irq=0x%x "
2003 		    "dip=0x%p vector=0x%x origirq=0x%x pri=0x%x\n", irqno,
2004 		    (void *)irqptr->airq_dip, irqptr->airq_vector,
2005 		    irqptr->airq_origirq, pri));
2006 	}
2007 	mutex_exit(&airq_mutex);
2008 	return (rcount);
2009 }
2010 
2011 /*
2012  * This function allocates "count" MSI-X vector(s) for the given "dip/pri/type"
2013  */
2014 int
2015 apic_alloc_msix_vectors(dev_info_t *dip, int inum, int count, int pri,
2016     int behavior)
2017 {
2018 	int	rcount, i;
2019 	major_t	major;
2020 
2021 	if (count > 1) {
2022 		if (behavior == DDI_INTR_ALLOC_STRICT) {
2023 			if (count > apic_msix_max)
2024 				return (0);
2025 		} else if (count > apic_msix_max)
2026 			count = apic_msix_max;
2027 	}
2028 
2029 	mutex_enter(&airq_mutex);
2030 
2031 	if ((rcount = apic_navail_vector(dip, pri)) > count)
2032 		rcount = count;
2033 	else if (rcount == 0 || (rcount < count &&
2034 	    behavior == DDI_INTR_ALLOC_STRICT)) {
2035 		rcount = 0;
2036 		goto out;
2037 	}
2038 
2039 	if (apic_check_free_irqs(rcount) == PSM_FAILURE) {
2040 		/* not enough free irq slots available */
2041 		rcount = 0;
2042 		goto out;
2043 	}
2044 
2045 	major = (dip != NULL) ? ddi_name_to_major(ddi_get_name(dip)) : 0;
2046 	for (i = 0; i < rcount; i++) {
2047 		uchar_t	vector, irqno;
2048 		apic_irq_t	*irqptr;
2049 
2050 		if ((irqno = apic_allocate_irq(apic_first_avail_irq)) ==
2051 		    (uchar_t)-1) {
2052 			/*
2053 			 * shouldn't happen because of the
2054 			 * apic_check_free_irqs() check earlier
2055 			 */
2056 			DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: "
2057 			    "apic_allocate_irq failed\n"));
2058 			rcount = i;
2059 			goto out;
2060 		}
2061 		if ((vector = apic_allocate_vector(pri, irqno, 1)) == 0) {
2062 			/*
2063 			 * shouldn't happen because of the
2064 			 * apic_navail_vector() call earlier
2065 			 */
2066 			DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: "
2067 			    "apic_allocate_vector failed\n"));
2068 			rcount = i;
2069 			goto out;
2070 		}
2071 		apic_max_device_irq = max(irqno, apic_max_device_irq);
2072 		apic_min_device_irq = min(irqno, apic_min_device_irq);
2073 		irqptr = apic_irq_table[irqno];
2074 		irqptr->airq_vector = (uchar_t)vector;
2075 		irqptr->airq_ipl = pri;
2076 		irqptr->airq_origirq = (uchar_t)(inum + i);
2077 		irqptr->airq_share_id = 0;
2078 		irqptr->airq_mps_intr_index = MSIX_INDEX;
2079 		irqptr->airq_dip = dip;
2080 		irqptr->airq_major = major;
2081 		irqptr->airq_cpu = apic_bind_intr(dip, irqno, 0xff, 0xff);
2082 	}
2083 out:
2084 	mutex_exit(&airq_mutex);
2085 	return (rcount);
2086 }
2087 
2088 /*
2089  * Allocate a free vector for irq at ipl. Takes care of merging of multiple
2090  * IPLs into a single APIC level as well as stretching some IPLs onto multiple
2091  * levels. APIC_HI_PRI_VECTS interrupts are reserved for high priority
2092  * requests and allocated only when pri is set.
2093  */
2094 uchar_t
2095 apic_allocate_vector(int ipl, int irq, int pri)
2096 {
2097 	int	lowest, highest, i;
2098 
2099 	highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
2100 	lowest = apic_ipltopri[ipl - 1] + APIC_VECTOR_PER_IPL;
2101 
2102 	if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */
2103 		lowest -= APIC_VECTOR_PER_IPL;
2104 
2105 #ifdef	DEBUG
2106 	if (apic_restrict_vector)	/* for testing shared interrupt logic */
2107 		highest = lowest + apic_restrict_vector + APIC_HI_PRI_VECTS;
2108 #endif /* DEBUG */
2109 	if (pri == 0)
2110 		highest -= APIC_HI_PRI_VECTS;
2111 
2112 	for (i = lowest; i < highest; i++) {
2113 		if (APIC_CHECK_RESERVE_VECTORS(i))
2114 			continue;
2115 		if (apic_vector_to_irq[i] == APIC_RESV_IRQ) {
2116 			apic_vector_to_irq[i] = (uchar_t)irq;
2117 			return (i);
2118 		}
2119 	}
2120 
2121 	return (0);
2122 }
2123 
2124 /* Mark vector as not being used by any irq */
2125 void
2126 apic_free_vector(uchar_t vector)
2127 {
2128 	apic_vector_to_irq[vector] = APIC_RESV_IRQ;
2129 }
2130 
2131 uint32_t
2132 ioapic_read(int ioapic_ix, uint32_t reg)
2133 {
2134 	volatile uint32_t *ioapic;
2135 
2136 	ioapic = apicioadr[ioapic_ix];
2137 	ioapic[APIC_IO_REG] = reg;
2138 	return (ioapic[APIC_IO_DATA]);
2139 }
2140 
2141 void
2142 ioapic_write(int ioapic_ix, uint32_t reg, uint32_t value)
2143 {
2144 	volatile uint32_t *ioapic;
2145 
2146 	ioapic = apicioadr[ioapic_ix];
2147 	ioapic[APIC_IO_REG] = reg;
2148 	ioapic[APIC_IO_DATA] = value;
2149 }
2150 
2151 static processorid_t
2152 apic_find_cpu(int flag)
2153 {
2154 	processorid_t acid = 0;
2155 	int i;
2156 
2157 	/* Find the first CPU with the passed-in flag set */
2158 	for (i = 0; i < apic_nproc; i++) {
2159 		if (apic_cpus[i].aci_status & flag) {
2160 			acid = i;
2161 			break;
2162 		}
2163 	}
2164 
2165 	ASSERT((apic_cpus[acid].aci_status & flag) != 0);
2166 	return (acid);
2167 }
2168 
2169 /*
2170  * Call rebind to do the actual programming.
2171  * Must be called with interrupts disabled and apic_ioapic_lock held
2172  * 'p' is polymorphic -- if this function is called to process a deferred
2173  * reprogramming, p is of type 'struct ioapic_reprogram_data *', from which
2174  * the irq pointer is retrieved.  If not doing deferred reprogramming,
2175  * p is of the type 'apic_irq_t *'.
2176  *
2177  * apic_ioapic_lock must be held across this call, as it protects apic_rebind
2178  * and it protects apic_find_cpu() from a race in which a CPU can be taken
2179  * offline after a cpu is selected, but before apic_rebind is called to
2180  * bind interrupts to it.
2181  */
2182 int
2183 apic_setup_io_intr(void *p, int irq, boolean_t deferred)
2184 {
2185 	apic_irq_t *irqptr;
2186 	struct ioapic_reprogram_data *drep = NULL;
2187 	int rv;
2188 
2189 	if (deferred) {
2190 		drep = (struct ioapic_reprogram_data *)p;
2191 		ASSERT(drep != NULL);
2192 		irqptr = drep->irqp;
2193 	} else
2194 		irqptr = (apic_irq_t *)p;
2195 
2196 	ASSERT(irqptr != NULL);
2197 
2198 	rv = apic_rebind(irqptr, apic_irq_table[irq]->airq_cpu, drep);
2199 	if (rv) {
2200 		/*
2201 		 * CPU is not up or interrupts are disabled. Fall back to
2202 		 * the first available CPU
2203 		 */
2204 		rv = apic_rebind(irqptr, apic_find_cpu(APIC_CPU_INTR_ENABLE),
2205 		    drep);
2206 	}
2207 
2208 	return (rv);
2209 }
2210 
2211 
2212 uchar_t
2213 apic_modify_vector(uchar_t vector, int irq)
2214 {
2215 	apic_vector_to_irq[vector] = (uchar_t)irq;
2216 	return (vector);
2217 }
2218 
2219 char *
2220 apic_get_apic_type()
2221 {
2222 	return (apic_psm_info.p_mach_idstring);
2223 }
2224