xref: /illumos-gate/usr/src/uts/i86pc/io/mp_platform_common.c (revision d8a7fe16f62711cdc5c4267da8b34ff24a6b668c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
28  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
29  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
30  * PSMI 1.5 extensions are supported in Solaris Nevada.
31  * PSMI 1.6 extensions are supported in Solaris Nevada.
32  */
33 #define	PSMI_1_6
34 
35 #include <sys/processor.h>
36 #include <sys/time.h>
37 #include <sys/psm.h>
38 #include <sys/smp_impldefs.h>
39 #include <sys/cram.h>
40 #include <sys/acpi/acpi.h>
41 #include <sys/acpica.h>
42 #include <sys/psm_common.h>
43 #include <sys/apic.h>
44 #include <sys/pit.h>
45 #include <sys/ddi.h>
46 #include <sys/sunddi.h>
47 #include <sys/ddi_impldefs.h>
48 #include <sys/pci.h>
49 #include <sys/promif.h>
50 #include <sys/x86_archext.h>
51 #include <sys/cpc_impl.h>
52 #include <sys/uadmin.h>
53 #include <sys/panic.h>
54 #include <sys/debug.h>
55 #include <sys/archsystm.h>
56 #include <sys/trap.h>
57 #include <sys/machsystm.h>
58 #include <sys/cpuvar.h>
59 #include <sys/rm_platter.h>
60 #include <sys/privregs.h>
61 #include <sys/cyclic.h>
62 #include <sys/note.h>
63 #include <sys/pci_intr_lib.h>
64 #include <sys/sunndi.h>
65 #if !defined(__xpv)
66 #include <sys/hpet.h>
67 #include <sys/clock.h>
68 #endif
69 
70 /*
71  *	Local Function Prototypes
72  */
73 static int apic_handle_defconf();
74 static int apic_parse_mpct(caddr_t mpct, int bypass);
75 static struct apic_mpfps_hdr *apic_find_fps_sig(caddr_t fptr, int size);
76 static int apic_checksum(caddr_t bptr, int len);
77 static int apic_find_bus_type(char *bus);
78 static int apic_find_bus(int busid);
79 static int apic_find_bus_id(int bustype);
80 static struct apic_io_intr *apic_find_io_intr(int irqno);
81 static int apic_find_free_irq(int start, int end);
82 static void apic_mark_vector(uchar_t oldvector, uchar_t newvector);
83 static void apic_xlate_vector_free_timeout_handler(void *arg);
84 static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
85     int new_bind_cpu, int apicindex, int intin_no, int which_irq,
86     struct ioapic_reprogram_data *drep);
87 static void apic_record_rdt_entry(apic_irq_t *irqptr, int irq);
88 static struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid);
89 static int apic_find_intin(uchar_t ioapic, uchar_t intin);
90 static int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno,
91     int child_ipin, struct apic_io_intr **intrp);
92 static int apic_setup_irq_table(dev_info_t *dip, int irqno,
93     struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp,
94     int type);
95 static void apic_set_pwroff_method_from_mpcnfhdr(struct apic_mp_cnf_hdr *hdrp);
96 static void apic_try_deferred_reprogram(int ipl, int vect);
97 static void delete_defer_repro_ent(int which_irq);
98 static void apic_ioapic_wait_pending_clear(int ioapicindex,
99     int intin_no);
100 static boolean_t apic_is_ioapic_AMD_813x(uint32_t physaddr);
101 static int apic_acpi_enter_apicmode(void);
102 
103 int apic_debug_mps_id = 0;	/* 1 - print MPS ID strings */
104 
105 /* ACPI SCI interrupt configuration; -1 if SCI not used */
106 int apic_sci_vect = -1;
107 iflag_t apic_sci_flags;
108 
109 #if !defined(__xpv)
110 /* ACPI HPET interrupt configuration; -1 if HPET not used */
111 int apic_hpet_vect = -1;
112 iflag_t apic_hpet_flags;
113 #endif
114 
115 /*
116  * psm name pointer
117  */
118 static char *psm_name;
119 
120 /* ACPI support routines */
121 static int acpi_probe(char *);
122 static int apic_acpi_irq_configure(acpi_psm_lnk_t *acpipsmlnkp, dev_info_t *dip,
123     int *pci_irqp, iflag_t *intr_flagp);
124 
125 static int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
126     int ipin, int *pci_irqp, iflag_t *intr_flagp);
127 static uchar_t acpi_find_ioapic(int irq);
128 static int acpi_intr_compatible(iflag_t iflag1, iflag_t iflag2);
129 
130 /*
131  * number of bits per byte, from <sys/param.h>
132  */
133 #define	UCHAR_MAX	((1 << NBBY) - 1)
134 
135 /* Max wait time (in repetitions) for flags to clear in an RDT entry. */
136 int apic_max_reps_clear_pending = 1000;
137 
138 /* The irq # is implicit in the array index: */
139 struct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1];
140 /*
141  * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
142  * is indexed by IRQ number, NOT by vector number.
143  */
144 
145 int	apic_intr_policy = INTR_ROUND_ROBIN;
146 
147 int	apic_next_bind_cpu = 1; /* For round robin assignment */
148 				/* start with cpu 1 */
149 
150 /*
151  * If enabled, the distribution works as follows:
152  * On every interrupt entry, the current ipl for the CPU is set in cpu_info
153  * and the irq corresponding to the ipl is also set in the aci_current array.
154  * interrupt exit and setspl (due to soft interrupts) will cause the current
155  * ipl to be be changed. This is cache friendly as these frequently used
156  * paths write into a per cpu structure.
157  *
158  * Sampling is done by checking the structures for all CPUs and incrementing
159  * the busy field of the irq (if any) executing on each CPU and the busy field
160  * of the corresponding CPU.
161  * In periodic mode this is done on every clock interrupt.
162  * In one-shot mode, this is done thru a cyclic with an interval of
163  * apic_redistribute_sample_interval (default 10 milli sec).
164  *
165  * Every apic_sample_factor_redistribution times we sample, we do computations
166  * to decide which interrupt needs to be migrated (see comments
167  * before apic_intr_redistribute().
168  */
169 
170 /*
171  * Following 3 variables start as % and can be patched or set using an
172  * API to be defined in future. They will be scaled to
173  * sample_factor_redistribution which is in turn set to hertz+1 (in periodic
174  * mode), or 101 in one-shot mode to stagger it away from one sec processing
175  */
176 
177 int	apic_int_busy_mark = 60;
178 int	apic_int_free_mark = 20;
179 int	apic_diff_for_redistribution = 10;
180 
181 /* sampling interval for interrupt redistribution for dynamic migration */
182 int	apic_redistribute_sample_interval = NANOSEC / 100; /* 10 millisec */
183 
184 /*
185  * number of times we sample before deciding to redistribute interrupts
186  * for dynamic migration
187  */
188 int	apic_sample_factor_redistribution = 101;
189 
190 /* timeout for xlate_vector, mark_vector */
191 int	apic_revector_timeout = 16 * 10000; /* 160 millisec */
192 
193 int	apic_redist_cpu_skip = 0;
194 int	apic_num_imbalance = 0;
195 int	apic_num_rebind = 0;
196 
197 int	apic_nproc = 0;
198 size_t	apic_cpus_size = 0;
199 int	apic_defconf = 0;
200 int	apic_irq_translate = 0;
201 int	apic_spec_rev = 0;
202 int	apic_imcrp = 0;
203 
204 int	apic_use_acpi = 1;	/* 1 = use ACPI, 0 = don't use ACPI */
205 int	apic_use_acpi_madt_only = 0;	/* 1=ONLY use MADT from ACPI */
206 
207 /*
208  * For interrupt link devices, if apic_unconditional_srs is set, an irq resource
209  * will be assigned (via _SRS). If it is not set, use the current
210  * irq setting (via _CRS), but only if that irq is in the set of possible
211  * irqs (returned by _PRS) for the device.
212  */
213 int	apic_unconditional_srs = 1;
214 
215 /*
216  * For interrupt link devices, if apic_prefer_crs is set when we are
217  * assigning an IRQ resource to a device, prefer the current IRQ setting
218  * over other possible irq settings under same conditions.
219  */
220 
221 int	apic_prefer_crs = 1;
222 
223 uchar_t	apic_io_id[MAX_IO_APIC];
224 volatile uint32_t *apicioadr[MAX_IO_APIC];
225 static	uchar_t	apic_io_ver[MAX_IO_APIC];
226 static	uchar_t	apic_io_vectbase[MAX_IO_APIC];
227 static	uchar_t	apic_io_vectend[MAX_IO_APIC];
228 uchar_t apic_reserved_irqlist[MAX_ISA_IRQ + 1];
229 uint32_t apic_physaddr[MAX_IO_APIC];
230 
231 static	boolean_t ioapic_mask_workaround[MAX_IO_APIC];
232 
233 /*
234  * First available slot to be used as IRQ index into the apic_irq_table
235  * for those interrupts (like MSI/X) that don't have a physical IRQ.
236  */
237 int apic_first_avail_irq  = APIC_FIRST_FREE_IRQ;
238 
239 /*
240  * apic_ioapic_lock protects the ioapics (reg select), the status, temp_bound
241  * and bound elements of cpus_info and the temp_cpu element of irq_struct
242  */
243 lock_t	apic_ioapic_lock;
244 
245 /*
246  * apic_defer_reprogram_lock ensures that only one processor is handling
247  * deferred interrupt programming at *_intr_exit time.
248  */
249 static	lock_t	apic_defer_reprogram_lock;
250 
251 /*
252  * The current number of deferred reprogrammings outstanding
253  */
254 uint_t	apic_reprogram_outstanding = 0;
255 
256 #ifdef DEBUG
257 /*
258  * Counters that keep track of deferred reprogramming stats
259  */
260 uint_t	apic_intr_deferrals = 0;
261 uint_t	apic_intr_deliver_timeouts = 0;
262 uint_t	apic_last_ditch_reprogram_failures = 0;
263 uint_t	apic_deferred_setup_failures = 0;
264 uint_t	apic_defer_repro_total_retries = 0;
265 uint_t	apic_defer_repro_successes = 0;
266 uint_t	apic_deferred_spurious_enters = 0;
267 #endif
268 
269 static	int	apic_io_max = 0;	/* no. of i/o apics enabled */
270 
271 static	struct apic_io_intr *apic_io_intrp = 0;
272 static	struct apic_bus	*apic_busp;
273 
274 uchar_t	apic_vector_to_irq[APIC_MAX_VECTOR+1];
275 uchar_t	apic_resv_vector[MAXIPL+1];
276 
277 char	apic_level_intr[APIC_MAX_VECTOR+1];
278 
279 static	uint32_t	eisa_level_intr_mask = 0;
280 	/* At least MSB will be set if EISA bus */
281 
282 static	int	apic_pci_bus_total = 0;
283 static	uchar_t	apic_single_pci_busid = 0;
284 
285 /*
286  * airq_mutex protects additions to the apic_irq_table - the first
287  * pointer and any airq_nexts off of that one. It also protects
288  * apic_max_device_irq & apic_min_device_irq. It also guarantees
289  * that share_id is unique as new ids are generated only when new
290  * irq_t structs are linked in. Once linked in the structs are never
291  * deleted. temp_cpu & mps_intr_index field indicate if it is programmed
292  * or allocated. Note that there is a slight gap between allocating in
293  * apic_introp_xlate and programming in addspl.
294  */
295 kmutex_t	airq_mutex;
296 apic_irq_t	*apic_irq_table[APIC_MAX_VECTOR+1];
297 int		apic_max_device_irq = 0;
298 int		apic_min_device_irq = APIC_MAX_VECTOR;
299 
300 /*
301  * Following declarations are for revectoring; used when ISRs at different
302  * IPLs share an irq.
303  */
304 static	lock_t	apic_revector_lock;
305 int	apic_revector_pending = 0;
306 static	uchar_t	*apic_oldvec_to_newvec;
307 static	uchar_t	*apic_newvec_to_oldvec;
308 
309 typedef struct prs_irq_list_ent {
310 	int			list_prio;
311 	int32_t			irq;
312 	iflag_t			intrflags;
313 	acpi_prs_private_t	prsprv;
314 	struct prs_irq_list_ent	*next;
315 } prs_irq_list_t;
316 
317 
318 /*
319  * ACPI variables
320  */
321 /* 1 = acpi is enabled & working, 0 = acpi is not enabled or not there */
322 int apic_enable_acpi = 0;
323 
324 /* ACPI Multiple APIC Description Table ptr */
325 static	ACPI_TABLE_MADT *acpi_mapic_dtp = NULL;
326 
327 /* ACPI Interrupt Source Override Structure ptr */
328 static	ACPI_MADT_INTERRUPT_OVERRIDE *acpi_isop = NULL;
329 static	int acpi_iso_cnt = 0;
330 
331 /* ACPI Non-maskable Interrupt Sources ptr */
332 static	ACPI_MADT_NMI_SOURCE *acpi_nmi_sp = NULL;
333 static	int acpi_nmi_scnt = 0;
334 static	ACPI_MADT_LOCAL_APIC_NMI *acpi_nmi_cp = NULL;
335 static	int acpi_nmi_ccnt = 0;
336 
337 /*
338  * The following added to identify a software poweroff method if available.
339  */
340 
341 static struct {
342 	int	poweroff_method;
343 	char	oem_id[APIC_MPS_OEM_ID_LEN + 1];	/* MAX + 1 for NULL */
344 	char	prod_id[APIC_MPS_PROD_ID_LEN + 1];	/* MAX + 1 for NULL */
345 } apic_mps_ids[] = {
346 	{ APIC_POWEROFF_VIA_RTC,	"INTEL",	"ALDER" },   /* 4300 */
347 	{ APIC_POWEROFF_VIA_RTC,	"NCR",		"AMC" },    /* 4300 */
348 	{ APIC_POWEROFF_VIA_ASPEN_BMC,	"INTEL",	"A450NX" },  /* 4400? */
349 	{ APIC_POWEROFF_VIA_ASPEN_BMC,	"INTEL",	"AD450NX" }, /* 4400 */
350 	{ APIC_POWEROFF_VIA_ASPEN_BMC,	"INTEL",	"AC450NX" }, /* 4400R */
351 	{ APIC_POWEROFF_VIA_SITKA_BMC,	"INTEL",	"S450NX" },  /* S50  */
352 	{ APIC_POWEROFF_VIA_SITKA_BMC,	"INTEL",	"SC450NX" }  /* S50? */
353 };
354 
355 int	apic_poweroff_method = APIC_POWEROFF_NONE;
356 
357 /*
358  * Auto-configuration routines
359  */
360 
361 /*
362  * Look at MPSpec 1.4 (Intel Order # 242016-005) for details of what we do here
363  * May work with 1.1 - but not guaranteed.
364  * According to the MP Spec, the MP floating pointer structure
365  * will be searched in the order described below:
366  * 1. In the first kilobyte of Extended BIOS Data Area (EBDA)
367  * 2. Within the last kilobyte of system base memory
368  * 3. In the BIOS ROM address space between 0F0000h and 0FFFFh
369  * Once we find the right signature with proper checksum, we call
370  * either handle_defconf or parse_mpct to get all info necessary for
371  * subsequent operations.
372  */
373 int
374 apic_probe_common(char *modname)
375 {
376 	uint32_t mpct_addr, ebda_start = 0, base_mem_end;
377 	caddr_t	biosdatap;
378 	caddr_t	mpct;
379 	caddr_t	fptr;
380 	int	i, mpct_size, mapsize, retval = PSM_FAILURE;
381 	ushort_t	ebda_seg, base_mem_size;
382 	struct	apic_mpfps_hdr	*fpsp;
383 	struct	apic_mp_cnf_hdr	*hdrp;
384 	int bypass_cpu_and_ioapics_in_mptables;
385 	int acpi_user_options;
386 
387 	if (apic_forceload < 0)
388 		return (retval);
389 
390 	/*
391 	 * Remember who we are
392 	 */
393 	psm_name = modname;
394 
395 	/* Allow override for MADT-only mode */
396 	acpi_user_options = ddi_prop_get_int(DDI_DEV_T_ANY, ddi_root_node(), 0,
397 	    "acpi-user-options", 0);
398 	apic_use_acpi_madt_only = ((acpi_user_options & ACPI_OUSER_MADT) != 0);
399 
400 	/* Allow apic_use_acpi to override MADT-only mode */
401 	if (!apic_use_acpi)
402 		apic_use_acpi_madt_only = 0;
403 
404 	retval = acpi_probe(modname);
405 
406 	/*
407 	 * mapin the bios data area 40:0
408 	 * 40:13h - two-byte location reports the base memory size
409 	 * 40:0Eh - two-byte location for the exact starting address of
410 	 *	    the EBDA segment for EISA
411 	 */
412 	biosdatap = psm_map_phys(0x400, 0x20, PROT_READ);
413 	if (!biosdatap)
414 		return (retval);
415 	fpsp = (struct apic_mpfps_hdr *)NULL;
416 	mapsize = MPFPS_RAM_WIN_LEN;
417 	/*LINTED: pointer cast may result in improper alignment */
418 	ebda_seg = *((ushort_t *)(biosdatap+0xe));
419 	/* check the 1k of EBDA */
420 	if (ebda_seg) {
421 		ebda_start = ((uint32_t)ebda_seg) << 4;
422 		fptr = psm_map_phys(ebda_start, MPFPS_RAM_WIN_LEN, PROT_READ);
423 		if (fptr) {
424 			if (!(fpsp =
425 			    apic_find_fps_sig(fptr, MPFPS_RAM_WIN_LEN)))
426 				psm_unmap_phys(fptr, MPFPS_RAM_WIN_LEN);
427 		}
428 	}
429 	/* If not in EBDA, check the last k of system base memory */
430 	if (!fpsp) {
431 		/*LINTED: pointer cast may result in improper alignment */
432 		base_mem_size = *((ushort_t *)(biosdatap + 0x13));
433 
434 		if (base_mem_size > 512)
435 			base_mem_end = 639 * 1024;
436 		else
437 			base_mem_end = 511 * 1024;
438 		/* if ebda == last k of base mem, skip to check BIOS ROM */
439 		if (base_mem_end != ebda_start) {
440 
441 			fptr = psm_map_phys(base_mem_end, MPFPS_RAM_WIN_LEN,
442 			    PROT_READ);
443 
444 			if (fptr) {
445 				if (!(fpsp = apic_find_fps_sig(fptr,
446 				    MPFPS_RAM_WIN_LEN)))
447 					psm_unmap_phys(fptr, MPFPS_RAM_WIN_LEN);
448 			}
449 		}
450 	}
451 	psm_unmap_phys(biosdatap, 0x20);
452 
453 	/* If still cannot find it, check the BIOS ROM space */
454 	if (!fpsp) {
455 		mapsize = MPFPS_ROM_WIN_LEN;
456 		fptr = psm_map_phys(MPFPS_ROM_WIN_START,
457 		    MPFPS_ROM_WIN_LEN, PROT_READ);
458 		if (fptr) {
459 			if (!(fpsp =
460 			    apic_find_fps_sig(fptr, MPFPS_ROM_WIN_LEN))) {
461 				psm_unmap_phys(fptr, MPFPS_ROM_WIN_LEN);
462 				return (retval);
463 			}
464 		}
465 	}
466 
467 	if (apic_checksum((caddr_t)fpsp, fpsp->mpfps_length * 16) != 0) {
468 		psm_unmap_phys(fptr, MPFPS_ROM_WIN_LEN);
469 		return (retval);
470 	}
471 
472 	apic_spec_rev = fpsp->mpfps_spec_rev;
473 	if ((apic_spec_rev != 04) && (apic_spec_rev != 01)) {
474 		psm_unmap_phys(fptr, MPFPS_ROM_WIN_LEN);
475 		return (retval);
476 	}
477 
478 	/* check IMCR is present or not */
479 	apic_imcrp = fpsp->mpfps_featinfo2 & MPFPS_FEATINFO2_IMCRP;
480 
481 	/* check default configuration (dual CPUs) */
482 	if ((apic_defconf = fpsp->mpfps_featinfo1) != 0) {
483 		psm_unmap_phys(fptr, mapsize);
484 		return (apic_handle_defconf());
485 	}
486 
487 	/* MP Configuration Table */
488 	mpct_addr = (uint32_t)(fpsp->mpfps_mpct_paddr);
489 
490 	psm_unmap_phys(fptr, mapsize); /* unmap floating ptr struct */
491 
492 	/*
493 	 * Map in enough memory for the MP Configuration Table Header.
494 	 * Use this table to read the total length of the BIOS data and
495 	 * map in all the info
496 	 */
497 	/*LINTED: pointer cast may result in improper alignment */
498 	hdrp = (struct apic_mp_cnf_hdr *)psm_map_phys(mpct_addr,
499 	    sizeof (struct apic_mp_cnf_hdr), PROT_READ);
500 	if (!hdrp)
501 		return (retval);
502 
503 	/* check mp configuration table signature PCMP */
504 	if (hdrp->mpcnf_sig != 0x504d4350) {
505 		psm_unmap_phys((caddr_t)hdrp, sizeof (struct apic_mp_cnf_hdr));
506 		return (retval);
507 	}
508 	mpct_size = (int)hdrp->mpcnf_tbl_length;
509 
510 	apic_set_pwroff_method_from_mpcnfhdr(hdrp);
511 
512 	psm_unmap_phys((caddr_t)hdrp, sizeof (struct apic_mp_cnf_hdr));
513 
514 	if ((retval == PSM_SUCCESS) && !apic_use_acpi_madt_only) {
515 		/* This is an ACPI machine No need for further checks */
516 		return (retval);
517 	}
518 
519 	/*
520 	 * Map in the entries for this machine, ie. Processor
521 	 * Entry Tables, Bus Entry Tables, etc.
522 	 * They are in fixed order following one another
523 	 */
524 	mpct = psm_map_phys(mpct_addr, mpct_size, PROT_READ);
525 	if (!mpct)
526 		return (retval);
527 
528 	if (apic_checksum(mpct, mpct_size) != 0)
529 		goto apic_fail1;
530 
531 
532 	/*LINTED: pointer cast may result in improper alignment */
533 	hdrp = (struct apic_mp_cnf_hdr *)mpct;
534 	apicadr = (uint32_t *)mapin_apic((uint32_t)hdrp->mpcnf_local_apic,
535 	    APIC_LOCAL_MEMLEN, PROT_READ | PROT_WRITE);
536 	if (!apicadr)
537 		goto apic_fail1;
538 
539 	/* Parse all information in the tables */
540 	bypass_cpu_and_ioapics_in_mptables = (retval == PSM_SUCCESS);
541 	if (apic_parse_mpct(mpct, bypass_cpu_and_ioapics_in_mptables) ==
542 	    PSM_SUCCESS)
543 		return (PSM_SUCCESS);
544 
545 	for (i = 0; i < apic_io_max; i++)
546 		mapout_ioapic((caddr_t)apicioadr[i], APIC_IO_MEMLEN);
547 	if (apic_cpus)
548 		kmem_free(apic_cpus, apic_cpus_size);
549 	if (apicadr)
550 		mapout_apic((caddr_t)apicadr, APIC_LOCAL_MEMLEN);
551 apic_fail1:
552 	psm_unmap_phys(mpct, mpct_size);
553 	return (retval);
554 }
555 
556 static void
557 apic_set_pwroff_method_from_mpcnfhdr(struct apic_mp_cnf_hdr *hdrp)
558 {
559 	int	i;
560 
561 	for (i = 0; i < (sizeof (apic_mps_ids) / sizeof (apic_mps_ids[0]));
562 	    i++) {
563 		if ((strncmp(hdrp->mpcnf_oem_str, apic_mps_ids[i].oem_id,
564 		    strlen(apic_mps_ids[i].oem_id)) == 0) &&
565 		    (strncmp(hdrp->mpcnf_prod_str, apic_mps_ids[i].prod_id,
566 		    strlen(apic_mps_ids[i].prod_id)) == 0)) {
567 
568 			apic_poweroff_method = apic_mps_ids[i].poweroff_method;
569 			break;
570 		}
571 	}
572 
573 	if (apic_debug_mps_id != 0) {
574 		cmn_err(CE_CONT, "%s: MPS OEM ID = '%c%c%c%c%c%c%c%c'"
575 		    "Product ID = '%c%c%c%c%c%c%c%c%c%c%c%c'\n",
576 		    psm_name,
577 		    hdrp->mpcnf_oem_str[0],
578 		    hdrp->mpcnf_oem_str[1],
579 		    hdrp->mpcnf_oem_str[2],
580 		    hdrp->mpcnf_oem_str[3],
581 		    hdrp->mpcnf_oem_str[4],
582 		    hdrp->mpcnf_oem_str[5],
583 		    hdrp->mpcnf_oem_str[6],
584 		    hdrp->mpcnf_oem_str[7],
585 		    hdrp->mpcnf_prod_str[0],
586 		    hdrp->mpcnf_prod_str[1],
587 		    hdrp->mpcnf_prod_str[2],
588 		    hdrp->mpcnf_prod_str[3],
589 		    hdrp->mpcnf_prod_str[4],
590 		    hdrp->mpcnf_prod_str[5],
591 		    hdrp->mpcnf_prod_str[6],
592 		    hdrp->mpcnf_prod_str[7],
593 		    hdrp->mpcnf_prod_str[8],
594 		    hdrp->mpcnf_prod_str[9],
595 		    hdrp->mpcnf_prod_str[10],
596 		    hdrp->mpcnf_prod_str[11]);
597 	}
598 }
599 
600 static int
601 acpi_probe(char *modname)
602 {
603 	int			i, intmax, index;
604 	uint32_t		id, ver;
605 	int			acpi_verboseflags = 0;
606 	int			madt_seen, madt_size;
607 	ACPI_SUBTABLE_HEADER		*ap;
608 	ACPI_MADT_LOCAL_APIC	*mpa;
609 	ACPI_MADT_LOCAL_X2APIC	*mpx2a;
610 	ACPI_MADT_IO_APIC		*mia;
611 	ACPI_MADT_IO_SAPIC		*misa;
612 	ACPI_MADT_INTERRUPT_OVERRIDE	*mio;
613 	ACPI_MADT_NMI_SOURCE		*mns;
614 	ACPI_MADT_INTERRUPT_SOURCE	*mis;
615 	ACPI_MADT_LOCAL_APIC_NMI	*mlan;
616 	ACPI_MADT_LOCAL_X2APIC_NMI	*mx2alan;
617 	ACPI_MADT_LOCAL_APIC_OVERRIDE	*mao;
618 	int			sci;
619 	iflag_t			sci_flags;
620 	volatile uint32_t	*ioapic;
621 	int			ioapic_ix;
622 	uint32_t		local_ids[NCPU];
623 	uint32_t		proc_ids[NCPU];
624 	uchar_t			hid;
625 	int			warned = 0;
626 
627 	if (!apic_use_acpi)
628 		return (PSM_FAILURE);
629 
630 	if (AcpiGetTable(ACPI_SIG_MADT, 1,
631 	    (ACPI_TABLE_HEADER **) &acpi_mapic_dtp) != AE_OK)
632 		return (PSM_FAILURE);
633 
634 	apicadr = mapin_apic((uint32_t)acpi_mapic_dtp->Address,
635 	    APIC_LOCAL_MEMLEN, PROT_READ | PROT_WRITE);
636 	if (!apicadr)
637 		return (PSM_FAILURE);
638 
639 	id = apic_reg_ops->apic_read(APIC_LID_REG);
640 	local_ids[0] = (uchar_t)(id >> 24);
641 	apic_nproc = index = 1;
642 	CPUSET_ONLY(apic_cpumask, 0);
643 	apic_io_max = 0;
644 
645 	ap = (ACPI_SUBTABLE_HEADER *) (acpi_mapic_dtp + 1);
646 	madt_size = acpi_mapic_dtp->Header.Length;
647 	madt_seen = sizeof (*acpi_mapic_dtp);
648 
649 	while (madt_seen < madt_size) {
650 		switch (ap->Type) {
651 		case ACPI_MADT_TYPE_LOCAL_APIC:
652 			mpa = (ACPI_MADT_LOCAL_APIC *) ap;
653 			if (mpa->LapicFlags & ACPI_MADT_ENABLED) {
654 				if (mpa->Id == local_ids[0]) {
655 					proc_ids[0] = mpa->ProcessorId;
656 					(void) acpica_map_cpu(0,
657 					    mpa->ProcessorId);
658 				} else if (apic_nproc < NCPU && use_mp &&
659 				    apic_nproc < boot_ncpus) {
660 					local_ids[index] = mpa->Id;
661 					proc_ids[index] = mpa->ProcessorId;
662 					CPUSET_ADD(apic_cpumask, index);
663 					(void) acpica_map_cpu(index,
664 					    mpa->ProcessorId);
665 					index++;
666 					apic_nproc++;
667 				} else if (apic_nproc == NCPU && !warned) {
668 					cmn_err(CE_WARN, "%s: CPU limit "
669 					    "exceeded"
670 #if !defined(__amd64)
671 					    " for 32-bit mode"
672 #endif
673 					    "; Solaris will use %d CPUs.",
674 					    psm_name,  NCPU);
675 					warned = 1;
676 				}
677 			}
678 			break;
679 
680 		case ACPI_MADT_TYPE_IO_APIC:
681 			mia = (ACPI_MADT_IO_APIC *) ap;
682 			if (apic_io_max < MAX_IO_APIC) {
683 				ioapic_ix = apic_io_max;
684 				apic_io_id[apic_io_max] = mia->Id;
685 				apic_io_vectbase[apic_io_max] =
686 				    mia->GlobalIrqBase;
687 				apic_physaddr[apic_io_max] =
688 				    (uint32_t)mia->Address;
689 				ioapic = apicioadr[apic_io_max] =
690 				    mapin_ioapic((uint32_t)mia->Address,
691 				    APIC_IO_MEMLEN, PROT_READ | PROT_WRITE);
692 				if (!ioapic)
693 					goto cleanup;
694 				ioapic_mask_workaround[apic_io_max] =
695 				    apic_is_ioapic_AMD_813x(mia->Address);
696 				apic_io_max++;
697 			}
698 			break;
699 
700 		case ACPI_MADT_TYPE_INTERRUPT_OVERRIDE:
701 			mio = (ACPI_MADT_INTERRUPT_OVERRIDE *) ap;
702 			if (acpi_isop == NULL)
703 				acpi_isop = mio;
704 			acpi_iso_cnt++;
705 			break;
706 
707 		case ACPI_MADT_TYPE_NMI_SOURCE:
708 			/* UNIMPLEMENTED */
709 			mns = (ACPI_MADT_NMI_SOURCE *) ap;
710 			if (acpi_nmi_sp == NULL)
711 				acpi_nmi_sp = mns;
712 			acpi_nmi_scnt++;
713 
714 			cmn_err(CE_NOTE, "!apic: nmi source: %d 0x%x\n",
715 			    mns->GlobalIrq, mns->IntiFlags);
716 			break;
717 
718 		case ACPI_MADT_TYPE_LOCAL_APIC_NMI:
719 			/* UNIMPLEMENTED */
720 			mlan = (ACPI_MADT_LOCAL_APIC_NMI *) ap;
721 			if (acpi_nmi_cp == NULL)
722 				acpi_nmi_cp = mlan;
723 			acpi_nmi_ccnt++;
724 
725 			cmn_err(CE_NOTE, "!apic: local nmi: %d 0x%x %d\n",
726 			    mlan->ProcessorId, mlan->IntiFlags,
727 			    mlan->Lint);
728 			break;
729 
730 		case ACPI_MADT_TYPE_LOCAL_APIC_OVERRIDE:
731 			/* UNIMPLEMENTED */
732 			mao = (ACPI_MADT_LOCAL_APIC_OVERRIDE *) ap;
733 			cmn_err(CE_NOTE, "!apic: address override: %lx\n",
734 			    (long)mao->Address);
735 			break;
736 
737 		case ACPI_MADT_TYPE_IO_SAPIC:
738 			/* UNIMPLEMENTED */
739 			misa = (ACPI_MADT_IO_SAPIC *) ap;
740 
741 			cmn_err(CE_NOTE, "!apic: io sapic: %d %d %lx\n",
742 			    misa->Id, misa->GlobalIrqBase,
743 			    (long)misa->Address);
744 			break;
745 
746 		case ACPI_MADT_TYPE_INTERRUPT_SOURCE:
747 			/* UNIMPLEMENTED */
748 			mis = (ACPI_MADT_INTERRUPT_SOURCE *) ap;
749 
750 			cmn_err(CE_NOTE,
751 			    "!apic: irq source: %d %d %d 0x%x %d %d\n",
752 			    mis->Id, mis->Eid, mis->GlobalIrq,
753 			    mis->IntiFlags, mis->Type,
754 			    mis->IoSapicVector);
755 			break;
756 
757 		case ACPI_MADT_TYPE_LOCAL_X2APIC:
758 			mpx2a = (ACPI_MADT_LOCAL_X2APIC *) ap;
759 
760 			/*
761 			 * All logical processors with APIC ID values
762 			 * of 255 and greater will have their APIC
763 			 * reported through Processor X2APIC structure.
764 			 * All logical processors with APIC ID less than
765 			 * 255 will have their APIC reported through
766 			 * Processor Local APIC.
767 			 */
768 			if ((mpx2a->LapicFlags & ACPI_MADT_ENABLED) &&
769 			    (mpx2a->LocalApicId >> 8)) {
770 				if (apic_nproc < NCPU && use_mp &&
771 				    apic_nproc < boot_ncpus) {
772 					local_ids[index] = mpx2a->LocalApicId;
773 					CPUSET_ADD(apic_cpumask, index);
774 					(void) acpica_map_cpu(index,
775 					    mpx2a->Uid);
776 					index++;
777 					apic_nproc++;
778 				} else if (apic_nproc == NCPU && !warned) {
779 					cmn_err(CE_WARN, "%s: CPU limit "
780 					    "exceeded"
781 #if !defined(__amd64)
782 					    " for 32-bit mode"
783 #endif
784 					    "; Solaris will use %d CPUs.",
785 					    psm_name,  NCPU);
786 					warned = 1;
787 				}
788 			}
789 
790 			break;
791 
792 		case ACPI_MADT_TYPE_LOCAL_X2APIC_NMI:
793 			/* UNIMPLEMENTED */
794 			mx2alan = (ACPI_MADT_LOCAL_X2APIC_NMI *) ap;
795 			if (mx2alan->Uid >> 8)
796 				acpi_nmi_ccnt++;
797 
798 #ifdef	DEBUG
799 			cmn_err(CE_NOTE,
800 			    "!apic: local x2apic nmi: %d 0x%x %d\n",
801 			    mx2alan->Uid, mx2alan->IntiFlags, mx2alan->Lint);
802 #endif
803 
804 			break;
805 
806 		case ACPI_MADT_TYPE_RESERVED:
807 		default:
808 			break;
809 		}
810 
811 		/* advance to next entry */
812 		madt_seen += ap->Length;
813 		ap = (ACPI_SUBTABLE_HEADER *)(((char *)ap) + ap->Length);
814 	}
815 
816 	apic_cpus_size = apic_nproc * sizeof (*apic_cpus);
817 	if ((apic_cpus = kmem_zalloc(apic_cpus_size, KM_NOSLEEP)) == NULL)
818 		goto cleanup;
819 
820 	/*
821 	 * ACPI doesn't provide the local apic ver, get it directly from the
822 	 * local apic
823 	 */
824 	ver = apic_reg_ops->apic_read(APIC_VERS_REG);
825 	for (i = 0; i < apic_nproc; i++) {
826 		apic_cpus[i].aci_local_id = local_ids[i];
827 		apic_cpus[i].aci_local_ver = (uchar_t)(ver & 0xFF);
828 	}
829 
830 	for (i = 0; i < apic_io_max; i++) {
831 		ioapic_ix = i;
832 
833 		/*
834 		 * need to check Sitka on the following acpi problem
835 		 * On the Sitka, the ioapic's apic_id field isn't reporting
836 		 * the actual io apic id. We have reported this problem
837 		 * to Intel. Until they fix the problem, we will get the
838 		 * actual id directly from the ioapic.
839 		 */
840 		id = ioapic_read(ioapic_ix, APIC_ID_CMD);
841 		hid = (uchar_t)(id >> 24);
842 
843 		if (hid != apic_io_id[i]) {
844 			if (apic_io_id[i] == 0)
845 				apic_io_id[i] = hid;
846 			else { /* set ioapic id to whatever reported by ACPI */
847 				id = ((uint32_t)apic_io_id[i]) << 24;
848 				ioapic_write(ioapic_ix, APIC_ID_CMD, id);
849 			}
850 		}
851 		ver = ioapic_read(ioapic_ix, APIC_VERS_CMD);
852 		apic_io_ver[i] = (uchar_t)(ver & 0xff);
853 		intmax = (ver >> 16) & 0xff;
854 		apic_io_vectend[i] = apic_io_vectbase[i] + intmax;
855 		if (apic_first_avail_irq <= apic_io_vectend[i])
856 			apic_first_avail_irq = apic_io_vectend[i] + 1;
857 	}
858 
859 
860 	/*
861 	 * Process SCI configuration here
862 	 * An error may be returned here if
863 	 * acpi-user-options specifies legacy mode
864 	 * (no SCI, no ACPI mode)
865 	 */
866 	if (acpica_get_sci(&sci, &sci_flags) != AE_OK)
867 		sci = -1;
868 
869 	/*
870 	 * Now call acpi_init() to generate namespaces
871 	 * If this fails, we don't attempt to use ACPI
872 	 * even if we were able to get a MADT above
873 	 */
874 	if (acpica_init() != AE_OK)
875 		goto cleanup;
876 
877 	/*
878 	 * Call acpica_build_processor_map() now that we have
879 	 * ACPI namesspace access
880 	 */
881 	(void) acpica_build_processor_map();
882 
883 	/*
884 	 * Squirrel away the SCI and flags for later on
885 	 * in apic_picinit() when we're ready
886 	 */
887 	apic_sci_vect = sci;
888 	apic_sci_flags = sci_flags;
889 
890 	if (apic_verbose & APIC_VERBOSE_IRQ_FLAG)
891 		acpi_verboseflags |= PSM_VERBOSE_IRQ_FLAG;
892 
893 	if (apic_verbose & APIC_VERBOSE_POWEROFF_FLAG)
894 		acpi_verboseflags |= PSM_VERBOSE_POWEROFF_FLAG;
895 
896 	if (apic_verbose & APIC_VERBOSE_POWEROFF_PAUSE_FLAG)
897 		acpi_verboseflags |= PSM_VERBOSE_POWEROFF_PAUSE_FLAG;
898 
899 	if (acpi_psm_init(modname, acpi_verboseflags) == ACPI_PSM_FAILURE)
900 		goto cleanup;
901 
902 	/* Enable ACPI APIC interrupt routing */
903 	if (apic_acpi_enter_apicmode() != PSM_FAILURE) {
904 		build_reserved_irqlist((uchar_t *)apic_reserved_irqlist);
905 		apic_enable_acpi = 1;
906 		if (apic_sci_vect > 0) {
907 			acpica_set_core_feature(ACPI_FEATURE_SCI_EVENT);
908 		}
909 		if (apic_use_acpi_madt_only) {
910 			cmn_err(CE_CONT,
911 			    "?Using ACPI for CPU/IOAPIC information ONLY\n");
912 		}
913 
914 #if !defined(__xpv)
915 		/*
916 		 * probe ACPI for hpet information here which is used later
917 		 * in apic_picinit().
918 		 */
919 		if (hpet_acpi_init(&apic_hpet_vect, &apic_hpet_flags) < 0) {
920 			cmn_err(CE_NOTE, "!ACPI HPET table query failed\n");
921 		}
922 #endif
923 
924 		return (PSM_SUCCESS);
925 	}
926 	/* if setting APIC mode failed above, we fall through to cleanup */
927 
928 cleanup:
929 	if (apicadr != NULL) {
930 		mapout_apic((caddr_t)apicadr, APIC_LOCAL_MEMLEN);
931 		apicadr = NULL;
932 	}
933 	apic_nproc = 0;
934 	for (i = 0; i < apic_io_max; i++) {
935 		mapout_ioapic((caddr_t)apicioadr[i], APIC_IO_MEMLEN);
936 		apicioadr[i] = NULL;
937 	}
938 	apic_io_max = 0;
939 	acpi_isop = NULL;
940 	acpi_iso_cnt = 0;
941 	acpi_nmi_sp = NULL;
942 	acpi_nmi_scnt = 0;
943 	acpi_nmi_cp = NULL;
944 	acpi_nmi_ccnt = 0;
945 	return (PSM_FAILURE);
946 }
947 
948 /*
949  * Handle default configuration. Fill in reqd global variables & tables
950  * Fill all details as MP table does not give any more info
951  */
952 static int
953 apic_handle_defconf()
954 {
955 	uint_t	lid;
956 
957 	/*LINTED: pointer cast may result in improper alignment */
958 	apicioadr[0] = mapin_ioapic(APIC_IO_ADDR,
959 	    APIC_IO_MEMLEN, PROT_READ | PROT_WRITE);
960 	/*LINTED: pointer cast may result in improper alignment */
961 	apicadr = (uint32_t *)psm_map_phys(APIC_LOCAL_ADDR,
962 	    APIC_LOCAL_MEMLEN, PROT_READ);
963 	apic_cpus_size = 2 * sizeof (*apic_cpus);
964 	apic_cpus = (apic_cpus_info_t *)
965 	    kmem_zalloc(apic_cpus_size, KM_NOSLEEP);
966 	if ((!apicadr) || (!apicioadr[0]) || (!apic_cpus))
967 		goto apic_handle_defconf_fail;
968 	CPUSET_ONLY(apic_cpumask, 0);
969 	CPUSET_ADD(apic_cpumask, 1);
970 	apic_nproc = 2;
971 	lid = apic_reg_ops->apic_read(APIC_LID_REG);
972 	apic_cpus[0].aci_local_id = (uchar_t)(lid >> APIC_ID_BIT_OFFSET);
973 	/*
974 	 * According to the PC+MP spec 1.1, the local ids
975 	 * for the default configuration has to be 0 or 1
976 	 */
977 	if (apic_cpus[0].aci_local_id == 1)
978 		apic_cpus[1].aci_local_id = 0;
979 	else if (apic_cpus[0].aci_local_id == 0)
980 		apic_cpus[1].aci_local_id = 1;
981 	else
982 		goto apic_handle_defconf_fail;
983 
984 	apic_io_id[0] = 2;
985 	apic_io_max = 1;
986 	if (apic_defconf >= 5) {
987 		apic_cpus[0].aci_local_ver = APIC_INTEGRATED_VERS;
988 		apic_cpus[1].aci_local_ver = APIC_INTEGRATED_VERS;
989 		apic_io_ver[0] = APIC_INTEGRATED_VERS;
990 	} else {
991 		apic_cpus[0].aci_local_ver = 0;		/* 82489 DX */
992 		apic_cpus[1].aci_local_ver = 0;
993 		apic_io_ver[0] = 0;
994 	}
995 	if (apic_defconf == 2 || apic_defconf == 3 || apic_defconf == 6)
996 		eisa_level_intr_mask = (inb(EISA_LEVEL_CNTL + 1) << 8) |
997 		    inb(EISA_LEVEL_CNTL) | ((uint_t)INT32_MAX + 1);
998 	return (PSM_SUCCESS);
999 
1000 apic_handle_defconf_fail:
1001 	if (apic_cpus)
1002 		kmem_free(apic_cpus, apic_cpus_size);
1003 	if (apicadr)
1004 		mapout_apic((caddr_t)apicadr, APIC_LOCAL_MEMLEN);
1005 	if (apicioadr[0])
1006 		mapout_ioapic((caddr_t)apicioadr[0], APIC_IO_MEMLEN);
1007 	return (PSM_FAILURE);
1008 }
1009 
1010 /* Parse the entries in MP configuration table and collect info that we need */
1011 static int
1012 apic_parse_mpct(caddr_t mpct, int bypass_cpus_and_ioapics)
1013 {
1014 	struct	apic_procent	*procp;
1015 	struct	apic_bus	*busp;
1016 	struct	apic_io_entry	*ioapicp;
1017 	struct	apic_io_intr	*intrp;
1018 	int			ioapic_ix;
1019 	uint_t	lid;
1020 	uint32_t	id;
1021 	uchar_t hid;
1022 	int	warned = 0;
1023 
1024 	/*LINTED: pointer cast may result in improper alignment */
1025 	procp = (struct apic_procent *)(mpct + sizeof (struct apic_mp_cnf_hdr));
1026 
1027 	/* No need to count cpu entries if we won't use them */
1028 	if (!bypass_cpus_and_ioapics) {
1029 
1030 		/* Find max # of CPUS and allocate structure accordingly */
1031 		apic_nproc = 0;
1032 		CPUSET_ZERO(apic_cpumask);
1033 		while (procp->proc_entry == APIC_CPU_ENTRY) {
1034 			if (procp->proc_cpuflags & CPUFLAGS_EN) {
1035 				if (apic_nproc < NCPU && use_mp &&
1036 				    apic_nproc < boot_ncpus) {
1037 					CPUSET_ADD(apic_cpumask, apic_nproc);
1038 					apic_nproc++;
1039 				} else if (apic_nproc == NCPU && !warned) {
1040 					cmn_err(CE_WARN, "%s: CPU limit "
1041 					    "exceeded"
1042 #if !defined(__amd64)
1043 					    " for 32-bit mode"
1044 #endif
1045 					    "; Solaris will use %d CPUs.",
1046 					    psm_name,  NCPU);
1047 					warned = 1;
1048 				}
1049 
1050 			}
1051 			procp++;
1052 		}
1053 		apic_cpus_size = apic_nproc * sizeof (*apic_cpus);
1054 		if (!apic_nproc || !(apic_cpus = (apic_cpus_info_t *)
1055 		    kmem_zalloc(apic_cpus_size, KM_NOSLEEP)))
1056 			return (PSM_FAILURE);
1057 	}
1058 
1059 	/*LINTED: pointer cast may result in improper alignment */
1060 	procp = (struct apic_procent *)(mpct + sizeof (struct apic_mp_cnf_hdr));
1061 
1062 	/*
1063 	 * start with index 1 as 0 needs to be filled in with Boot CPU, but
1064 	 * if we're bypassing this information, it has already been filled
1065 	 * in by acpi_probe(), so don't overwrite it.
1066 	 */
1067 	if (!bypass_cpus_and_ioapics)
1068 		apic_nproc = 1;
1069 
1070 	while (procp->proc_entry == APIC_CPU_ENTRY) {
1071 		/* check whether the cpu exists or not */
1072 		if (!bypass_cpus_and_ioapics &&
1073 		    procp->proc_cpuflags & CPUFLAGS_EN) {
1074 			if (procp->proc_cpuflags & CPUFLAGS_BP) { /* Boot CPU */
1075 				lid = apic_reg_ops->apic_read(APIC_LID_REG);
1076 				apic_cpus[0].aci_local_id = procp->proc_apicid;
1077 				if (apic_cpus[0].aci_local_id !=
1078 				    (uchar_t)(lid >> APIC_ID_BIT_OFFSET)) {
1079 					return (PSM_FAILURE);
1080 				}
1081 				apic_cpus[0].aci_local_ver =
1082 				    procp->proc_version;
1083 			} else if (apic_nproc < NCPU && use_mp &&
1084 			    apic_nproc < boot_ncpus) {
1085 				apic_cpus[apic_nproc].aci_local_id =
1086 				    procp->proc_apicid;
1087 
1088 				apic_cpus[apic_nproc].aci_local_ver =
1089 				    procp->proc_version;
1090 				apic_nproc++;
1091 
1092 			}
1093 		}
1094 		procp++;
1095 	}
1096 
1097 	/*
1098 	 * Save start of bus entries for later use.
1099 	 * Get EISA level cntrl if EISA bus is present.
1100 	 * Also get the CPI bus id for single CPI bus case
1101 	 */
1102 	apic_busp = busp = (struct apic_bus *)procp;
1103 	while (busp->bus_entry == APIC_BUS_ENTRY) {
1104 		lid = apic_find_bus_type((char *)&busp->bus_str1);
1105 		if (lid	== BUS_EISA) {
1106 			eisa_level_intr_mask = (inb(EISA_LEVEL_CNTL + 1) << 8) |
1107 			    inb(EISA_LEVEL_CNTL) | ((uint_t)INT32_MAX + 1);
1108 		} else if (lid == BUS_PCI) {
1109 			/*
1110 			 * apic_single_pci_busid will be used only if
1111 			 * apic_pic_bus_total is equal to 1
1112 			 */
1113 			apic_pci_bus_total++;
1114 			apic_single_pci_busid = busp->bus_id;
1115 		}
1116 		busp++;
1117 	}
1118 
1119 	ioapicp = (struct apic_io_entry *)busp;
1120 
1121 	if (!bypass_cpus_and_ioapics)
1122 		apic_io_max = 0;
1123 	do {
1124 		if (!bypass_cpus_and_ioapics && apic_io_max < MAX_IO_APIC) {
1125 			if (ioapicp->io_flags & IOAPIC_FLAGS_EN) {
1126 				apic_io_id[apic_io_max] = ioapicp->io_apicid;
1127 				apic_io_ver[apic_io_max] = ioapicp->io_version;
1128 		/*LINTED: pointer cast may result in improper alignment */
1129 				apicioadr[apic_io_max] =
1130 				    mapin_ioapic(
1131 				    (uint32_t)ioapicp->io_apic_addr,
1132 				    APIC_IO_MEMLEN, PROT_READ | PROT_WRITE);
1133 
1134 				if (!apicioadr[apic_io_max])
1135 					return (PSM_FAILURE);
1136 
1137 				ioapic_mask_workaround[apic_io_max] =
1138 				    apic_is_ioapic_AMD_813x(
1139 				    ioapicp->io_apic_addr);
1140 
1141 				ioapic_ix = apic_io_max;
1142 				id = ioapic_read(ioapic_ix, APIC_ID_CMD);
1143 				hid = (uchar_t)(id >> 24);
1144 
1145 				if (hid != apic_io_id[apic_io_max]) {
1146 					if (apic_io_id[apic_io_max] == 0)
1147 						apic_io_id[apic_io_max] = hid;
1148 					else {
1149 						/*
1150 						 * set ioapic id to whatever
1151 						 * reported by MPS
1152 						 *
1153 						 * may not need to set index
1154 						 * again ???
1155 						 * take it out and try
1156 						 */
1157 
1158 						id = ((uint32_t)
1159 						    apic_io_id[apic_io_max]) <<
1160 						    24;
1161 
1162 						ioapic_write(ioapic_ix,
1163 						    APIC_ID_CMD, id);
1164 					}
1165 				}
1166 				apic_io_max++;
1167 			}
1168 		}
1169 		ioapicp++;
1170 	} while (ioapicp->io_entry == APIC_IO_ENTRY);
1171 
1172 	apic_io_intrp = (struct apic_io_intr *)ioapicp;
1173 
1174 	intrp = apic_io_intrp;
1175 	while (intrp->intr_entry == APIC_IO_INTR_ENTRY) {
1176 		if ((intrp->intr_irq > APIC_MAX_ISA_IRQ) ||
1177 		    (apic_find_bus(intrp->intr_busid) == BUS_PCI)) {
1178 			apic_irq_translate = 1;
1179 			break;
1180 		}
1181 		intrp++;
1182 	}
1183 
1184 	return (PSM_SUCCESS);
1185 }
1186 
1187 boolean_t
1188 apic_cpu_in_range(int cpu)
1189 {
1190 	return ((cpu & ~IRQ_USER_BOUND) < apic_nproc);
1191 }
1192 
1193 uint16_t
1194 apic_get_apic_version()
1195 {
1196 	int i;
1197 	uchar_t min_io_apic_ver = 0;
1198 	static uint16_t version;		/* Cache as value is constant */
1199 	static boolean_t found = B_FALSE;	/* Accomodate zero version */
1200 
1201 	if (found == B_FALSE) {
1202 		found = B_TRUE;
1203 
1204 		/*
1205 		 * Don't assume all IO APICs in the system are the same.
1206 		 *
1207 		 * Set to the minimum version.
1208 		 */
1209 		for (i = 0; i < apic_io_max; i++) {
1210 			if ((apic_io_ver[i] != 0) &&
1211 			    ((min_io_apic_ver == 0) ||
1212 			    (min_io_apic_ver >= apic_io_ver[i])))
1213 				min_io_apic_ver = apic_io_ver[i];
1214 		}
1215 
1216 		/* Assume all local APICs are of the same version. */
1217 		version = (min_io_apic_ver << 8) | apic_cpus[0].aci_local_ver;
1218 	}
1219 	return (version);
1220 }
1221 
1222 static struct apic_mpfps_hdr *
1223 apic_find_fps_sig(caddr_t cptr, int len)
1224 {
1225 	int	i;
1226 
1227 	/* Look for the pattern "_MP_" */
1228 	for (i = 0; i < len; i += 16) {
1229 		if ((*(cptr+i) == '_') &&
1230 		    (*(cptr+i+1) == 'M') &&
1231 		    (*(cptr+i+2) == 'P') &&
1232 		    (*(cptr+i+3) == '_'))
1233 		    /*LINTED: pointer cast may result in improper alignment */
1234 			return ((struct apic_mpfps_hdr *)(cptr + i));
1235 	}
1236 	return (NULL);
1237 }
1238 
1239 static int
1240 apic_checksum(caddr_t bptr, int len)
1241 {
1242 	int	i;
1243 	uchar_t	cksum;
1244 
1245 	cksum = 0;
1246 	for (i = 0; i < len; i++)
1247 		cksum += *bptr++;
1248 	return ((int)cksum);
1249 }
1250 
1251 
1252 /*
1253  * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
1254  * are also set to NULL. vector->irq is set to a value which cannot map
1255  * to a real irq to show that it is free.
1256  */
1257 void
1258 apic_init_common()
1259 {
1260 	int	i, j, indx;
1261 	int	*iptr;
1262 
1263 	/*
1264 	 * Initialize apic_ipls from apic_vectortoipl.  This array is
1265 	 * used in apic_intr_enter to determine the IPL to use for the
1266 	 * corresponding vector.  On some systems, due to hardware errata
1267 	 * and interrupt sharing, the IPL may not correspond to the IPL listed
1268 	 * in apic_vectortoipl (see apic_addspl and apic_delspl).
1269 	 */
1270 	for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
1271 		indx = i * APIC_VECTOR_PER_IPL;
1272 
1273 		for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++)
1274 			apic_ipls[indx] = apic_vectortoipl[i];
1275 	}
1276 
1277 	/* cpu 0 is always up (for now) */
1278 	apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
1279 
1280 	iptr = (int *)&apic_irq_table[0];
1281 	for (i = 0; i <= APIC_MAX_VECTOR; i++) {
1282 		apic_level_intr[i] = 0;
1283 		*iptr++ = NULL;
1284 		apic_vector_to_irq[i] = APIC_RESV_IRQ;
1285 
1286 		/* These *must* be initted to B_TRUE! */
1287 		apic_reprogram_info[i].done = B_TRUE;
1288 		apic_reprogram_info[i].irqp = NULL;
1289 		apic_reprogram_info[i].tries = 0;
1290 		apic_reprogram_info[i].bindcpu = 0;
1291 	}
1292 
1293 	/*
1294 	 * Allocate a dummy irq table entry for the reserved entry.
1295 	 * This takes care of the race between removing an irq and
1296 	 * clock detecting a CPU in that irq during interrupt load
1297 	 * sampling.
1298 	 */
1299 	apic_irq_table[APIC_RESV_IRQ] =
1300 	    kmem_zalloc(sizeof (apic_irq_t), KM_NOSLEEP);
1301 
1302 	mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
1303 }
1304 
1305 void
1306 ioapic_init_intr(int mask_apic)
1307 {
1308 	int ioapic_ix;
1309 	struct intrspec ispec;
1310 	apic_irq_t *irqptr;
1311 	int i, j;
1312 	ulong_t iflag;
1313 
1314 	LOCK_INIT_CLEAR(&apic_revector_lock);
1315 	LOCK_INIT_CLEAR(&apic_defer_reprogram_lock);
1316 
1317 	/* mask interrupt vectors */
1318 	for (j = 0; j < apic_io_max && mask_apic; j++) {
1319 		int intin_max;
1320 
1321 		ioapic_ix = j;
1322 		/* Bits 23-16 define the maximum redirection entries */
1323 		intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
1324 		    & 0xff;
1325 		for (i = 0; i <= intin_max; i++)
1326 			ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK);
1327 	}
1328 
1329 	/*
1330 	 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
1331 	 */
1332 	if (apic_sci_vect > 0) {
1333 		/*
1334 		 * acpica has already done add_avintr(); we just
1335 		 * to finish the job by mimicing translate_irq()
1336 		 *
1337 		 * Fake up an intrspec and setup the tables
1338 		 */
1339 		ispec.intrspec_vec = apic_sci_vect;
1340 		ispec.intrspec_pri = SCI_IPL;
1341 
1342 		if (apic_setup_irq_table(NULL, apic_sci_vect, NULL,
1343 		    &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) {
1344 			cmn_err(CE_WARN, "!apic: SCI setup failed");
1345 			return;
1346 		}
1347 		irqptr = apic_irq_table[apic_sci_vect];
1348 
1349 		iflag = intr_clear();
1350 		lock_set(&apic_ioapic_lock);
1351 
1352 		/* Program I/O APIC */
1353 		(void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE);
1354 
1355 		lock_clear(&apic_ioapic_lock);
1356 		intr_restore(iflag);
1357 
1358 		irqptr->airq_share++;
1359 	}
1360 
1361 #if !defined(__xpv)
1362 	/*
1363 	 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
1364 	 */
1365 	if (apic_hpet_vect > 0) {
1366 		/*
1367 		 * hpet has already done add_avintr(); we just need
1368 		 * to finish the job by mimicing translate_irq()
1369 		 *
1370 		 * Fake up an intrspec and setup the tables
1371 		 */
1372 		ispec.intrspec_vec = apic_hpet_vect;
1373 		ispec.intrspec_pri = CBE_HIGH_PIL;
1374 
1375 		if (apic_setup_irq_table(NULL, apic_hpet_vect, NULL,
1376 		    &ispec, &apic_hpet_flags, DDI_INTR_TYPE_FIXED) < 0) {
1377 			cmn_err(CE_WARN, "!apic: HPET setup failed");
1378 			return;
1379 		}
1380 		irqptr = apic_irq_table[apic_hpet_vect];
1381 
1382 		iflag = intr_clear();
1383 		lock_set(&apic_ioapic_lock);
1384 
1385 		/* Program I/O APIC */
1386 		(void) apic_setup_io_intr(irqptr, apic_hpet_vect, B_FALSE);
1387 
1388 		lock_clear(&apic_ioapic_lock);
1389 		intr_restore(iflag);
1390 
1391 		irqptr->airq_share++;
1392 	}
1393 #endif	/* !defined(__xpv) */
1394 }
1395 
1396 /*
1397  * Add mask bits to disable interrupt vector from happening
1398  * at or above IPL. In addition, it should remove mask bits
1399  * to enable interrupt vectors below the given IPL.
1400  *
1401  * Both add and delspl are complicated by the fact that different interrupts
1402  * may share IRQs. This can happen in two ways.
1403  * 1. The same H/W line is shared by more than 1 device
1404  * 1a. with interrupts at different IPLs
1405  * 1b. with interrupts at same IPL
1406  * 2. We ran out of vectors at a given IPL and started sharing vectors.
1407  * 1b and 2 should be handled gracefully, except for the fact some ISRs
1408  * will get called often when no interrupt is pending for the device.
1409  * For 1a, we just hope that the machine blows up with the person who
1410  * set it up that way!. In the meantime, we handle it at the higher IPL.
1411  */
1412 /*ARGSUSED*/
1413 int
1414 apic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
1415 {
1416 	uchar_t vector;
1417 	ulong_t iflag;
1418 	apic_irq_t *irqptr, *irqheadptr;
1419 	int irqindex;
1420 
1421 	ASSERT(max_ipl <= UCHAR_MAX);
1422 	irqindex = IRQINDEX(irqno);
1423 
1424 	if ((irqindex == -1) || (!apic_irq_table[irqindex]))
1425 		return (PSM_FAILURE);
1426 
1427 	mutex_enter(&airq_mutex);
1428 	irqptr = irqheadptr = apic_irq_table[irqindex];
1429 
1430 	DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
1431 	    "vector=0x%x\n", (void *)irqptr->airq_dip,
1432 	    irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
1433 
1434 	while (irqptr) {
1435 		if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
1436 			break;
1437 		irqptr = irqptr->airq_next;
1438 	}
1439 	irqptr->airq_share++;
1440 
1441 	mutex_exit(&airq_mutex);
1442 
1443 	/* return if it is not hardware interrupt */
1444 	if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
1445 		return (PSM_SUCCESS);
1446 
1447 	/* Or if there are more interupts at a higher IPL */
1448 	if (ipl != max_ipl)
1449 		return (PSM_SUCCESS);
1450 
1451 	/*
1452 	 * if apic_picinit() has not been called yet, just return.
1453 	 * At the end of apic_picinit(), we will call setup_io_intr().
1454 	 */
1455 
1456 	if (!apic_picinit_called)
1457 		return (PSM_SUCCESS);
1458 
1459 	/*
1460 	 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
1461 	 * return failure. Not very elegant, but then we hope the
1462 	 * machine will blow up with ...
1463 	 */
1464 	if (irqptr->airq_ipl != max_ipl &&
1465 	    !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
1466 
1467 		vector = apic_allocate_vector(max_ipl, irqindex, 1);
1468 		if (vector == 0) {
1469 			irqptr->airq_share--;
1470 			return (PSM_FAILURE);
1471 		}
1472 		irqptr = irqheadptr;
1473 		apic_mark_vector(irqptr->airq_vector, vector);
1474 		while (irqptr) {
1475 			irqptr->airq_vector = vector;
1476 			irqptr->airq_ipl = (uchar_t)max_ipl;
1477 			/*
1478 			 * reprogram irq being added and every one else
1479 			 * who is not in the UNINIT state
1480 			 */
1481 			if ((VIRTIRQ(irqindex, irqptr->airq_share_id) ==
1482 			    irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) {
1483 				apic_record_rdt_entry(irqptr, irqindex);
1484 
1485 				iflag = intr_clear();
1486 				lock_set(&apic_ioapic_lock);
1487 
1488 				(void) apic_setup_io_intr(irqptr, irqindex,
1489 				    B_FALSE);
1490 
1491 				lock_clear(&apic_ioapic_lock);
1492 				intr_restore(iflag);
1493 			}
1494 			irqptr = irqptr->airq_next;
1495 		}
1496 		return (PSM_SUCCESS);
1497 
1498 	} else if (irqptr->airq_ipl != max_ipl &&
1499 	    ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
1500 		/*
1501 		 * We cannot upgrade the vector, but we can change
1502 		 * the IPL that this vector induces.
1503 		 *
1504 		 * Note that we subtract APIC_BASE_VECT from the vector
1505 		 * here because this array is used in apic_intr_enter
1506 		 * (no need to add APIC_BASE_VECT in that hot code
1507 		 * path since we can do it in the rarely-executed path
1508 		 * here).
1509 		 */
1510 		apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] =
1511 		    (uchar_t)max_ipl;
1512 
1513 		irqptr = irqheadptr;
1514 		while (irqptr) {
1515 			irqptr->airq_ipl = (uchar_t)max_ipl;
1516 			irqptr = irqptr->airq_next;
1517 		}
1518 
1519 		return (PSM_SUCCESS);
1520 	}
1521 
1522 	ASSERT(irqptr);
1523 
1524 	iflag = intr_clear();
1525 	lock_set(&apic_ioapic_lock);
1526 
1527 	(void) apic_setup_io_intr(irqptr, irqindex, B_FALSE);
1528 
1529 	lock_clear(&apic_ioapic_lock);
1530 	intr_restore(iflag);
1531 
1532 	return (PSM_SUCCESS);
1533 }
1534 
1535 /*
1536  * Recompute mask bits for the given interrupt vector.
1537  * If there is no interrupt servicing routine for this
1538  * vector, this function should disable interrupt vector
1539  * from happening at all IPLs. If there are still
1540  * handlers using the given vector, this function should
1541  * disable the given vector from happening below the lowest
1542  * IPL of the remaining hadlers.
1543  */
1544 /*ARGSUSED*/
1545 int
1546 apic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
1547 {
1548 	uchar_t vector;
1549 	uint32_t bind_cpu;
1550 	int intin, irqindex;
1551 	int ioapic_ix;
1552 	apic_irq_t	*irqptr, *irqheadptr, *irqp;
1553 	ulong_t iflag;
1554 
1555 	mutex_enter(&airq_mutex);
1556 	irqindex = IRQINDEX(irqno);
1557 	irqptr = irqheadptr = apic_irq_table[irqindex];
1558 
1559 	DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
1560 	    "vector=0x%x\n", (void *)irqptr->airq_dip,
1561 	    irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
1562 
1563 	while (irqptr) {
1564 		if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
1565 			break;
1566 		irqptr = irqptr->airq_next;
1567 	}
1568 	ASSERT(irqptr);
1569 
1570 	irqptr->airq_share--;
1571 
1572 	mutex_exit(&airq_mutex);
1573 
1574 	if (ipl < max_ipl)
1575 		return (PSM_SUCCESS);
1576 
1577 	/* return if it is not hardware interrupt */
1578 	if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
1579 		return (PSM_SUCCESS);
1580 
1581 	if (!apic_picinit_called) {
1582 		/*
1583 		 * Clear irq_struct. If two devices shared an intpt
1584 		 * line & 1 unloaded before picinit, we are hosed. But, then
1585 		 * we hope the machine will ...
1586 		 */
1587 		irqptr->airq_mps_intr_index = FREE_INDEX;
1588 		irqptr->airq_temp_cpu = IRQ_UNINIT;
1589 		apic_free_vector(irqptr->airq_vector);
1590 		return (PSM_SUCCESS);
1591 	}
1592 	/*
1593 	 * Downgrade vector to new max_ipl if needed.If we cannot allocate,
1594 	 * use old IPL. Not very elegant, but then we hope ...
1595 	 */
1596 	if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) &&
1597 	    !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
1598 		apic_irq_t	*irqp;
1599 		if (vector = apic_allocate_vector(max_ipl, irqno, 1)) {
1600 			apic_mark_vector(irqheadptr->airq_vector, vector);
1601 			irqp = irqheadptr;
1602 			while (irqp) {
1603 				irqp->airq_vector = vector;
1604 				irqp->airq_ipl = (uchar_t)max_ipl;
1605 				if (irqp->airq_temp_cpu != IRQ_UNINIT) {
1606 					apic_record_rdt_entry(irqp, irqindex);
1607 
1608 					iflag = intr_clear();
1609 					lock_set(&apic_ioapic_lock);
1610 
1611 					(void) apic_setup_io_intr(irqp,
1612 					    irqindex, B_FALSE);
1613 
1614 					lock_clear(&apic_ioapic_lock);
1615 					intr_restore(iflag);
1616 				}
1617 				irqp = irqp->airq_next;
1618 			}
1619 		}
1620 
1621 	} else if (irqptr->airq_ipl != max_ipl &&
1622 	    max_ipl != PSM_INVALID_IPL &&
1623 	    ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
1624 
1625 	/*
1626 	 * We cannot downgrade the IPL of the vector below the vector's
1627 	 * hardware priority. If we did, it would be possible for a
1628 	 * higher-priority hardware vector to interrupt a CPU running at an IPL
1629 	 * lower than the hardware priority of the interrupting vector (but
1630 	 * higher than the soft IPL of this IRQ). When this happens, we would
1631 	 * then try to drop the IPL BELOW what it was (effectively dropping
1632 	 * below base_spl) which would be potentially catastrophic.
1633 	 *
1634 	 * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
1635 	 * (hardware IPL of 4).  Further assume that the old IPL of this IRQ
1636 	 * was 4, but the new IPL is 1.  If we forced vector 0x40 to result in
1637 	 * an IPL of 1, it would be possible for the processor to be executing
1638 	 * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
1639 	 * the currently-executing ISR.  When apic_intr_enter consults
1640 	 * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
1641 	 * so even though the processor was running at IPL 4, an IPL 1
1642 	 * interrupt will have interrupted it, which must not happen)).
1643 	 *
1644 	 * Effectively, this means that the hardware priority corresponding to
1645 	 * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
1646 	 * hardware priority.
1647 	 *
1648 	 * (In the above example, then, after removal of the IPL 4 device's
1649 	 * interrupt handler, the new IPL will continue to be 4 because the
1650 	 * hardware priority that IPL 1 implies is lower than the hardware
1651 	 * priority of the vector used.)
1652 	 */
1653 		/* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
1654 		const int apic_ipls_index = irqptr->airq_vector -
1655 		    APIC_BASE_VECT;
1656 		const int vect_inherent_hwpri = irqptr->airq_vector >>
1657 		    APIC_IPL_SHIFT;
1658 
1659 		/*
1660 		 * If there are still devices using this IRQ, determine the
1661 		 * new ipl to use.
1662 		 */
1663 		if (irqptr->airq_share) {
1664 			int vect_desired_hwpri, hwpri;
1665 
1666 			ASSERT(max_ipl < MAXIPL);
1667 			vect_desired_hwpri = apic_ipltopri[max_ipl] >>
1668 			    APIC_IPL_SHIFT;
1669 
1670 			/*
1671 			 * If the desired IPL's hardware priority is lower
1672 			 * than that of the vector, use the hardware priority
1673 			 * of the vector to determine the new IPL.
1674 			 */
1675 			hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ?
1676 			    vect_inherent_hwpri : vect_desired_hwpri;
1677 
1678 			/*
1679 			 * Now, to get the right index for apic_vectortoipl,
1680 			 * we need to subtract APIC_BASE_VECT from the
1681 			 * hardware-vector-equivalent (in hwpri).  Since hwpri
1682 			 * is already shifted, we shift APIC_BASE_VECT before
1683 			 * doing the subtraction.
1684 			 */
1685 			hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT);
1686 
1687 			ASSERT(hwpri >= 0);
1688 			ASSERT(hwpri < MAXIPL);
1689 			max_ipl = apic_vectortoipl[hwpri];
1690 			apic_ipls[apic_ipls_index] = max_ipl;
1691 
1692 			irqp = irqheadptr;
1693 			while (irqp) {
1694 				irqp->airq_ipl = (uchar_t)max_ipl;
1695 				irqp = irqp->airq_next;
1696 			}
1697 		} else {
1698 			/*
1699 			 * No more devices on this IRQ, so reset this vector's
1700 			 * element in apic_ipls to the original IPL for this
1701 			 * vector
1702 			 */
1703 			apic_ipls[apic_ipls_index] =
1704 			    apic_vectortoipl[vect_inherent_hwpri];
1705 		}
1706 	}
1707 
1708 	if (irqptr->airq_share)
1709 		return (PSM_SUCCESS);
1710 
1711 	iflag = intr_clear();
1712 	lock_set(&apic_ioapic_lock);
1713 
1714 	if (irqptr->airq_mps_intr_index == MSI_INDEX) {
1715 		/*
1716 		 * Disable the MSI vector
1717 		 * Make sure we only disable on the last
1718 		 * of the multi-MSI support
1719 		 */
1720 		if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
1721 			apic_pci_msi_disable_mode(irqptr->airq_dip,
1722 			    DDI_INTR_TYPE_MSI);
1723 		}
1724 	} else if (irqptr->airq_mps_intr_index == MSIX_INDEX) {
1725 		/*
1726 		 * Disable the MSI-X vector
1727 		 * needs to clear its mask and addr/data for each MSI-X
1728 		 */
1729 		apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX,
1730 		    irqptr->airq_origirq);
1731 		/*
1732 		 * Make sure we only disable on the last MSI-X
1733 		 */
1734 		if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
1735 			apic_pci_msi_disable_mode(irqptr->airq_dip,
1736 			    DDI_INTR_TYPE_MSIX);
1737 		}
1738 	} else {
1739 		/*
1740 		 * The assumption here is that this is safe, even for
1741 		 * systems with IOAPICs that suffer from the hardware
1742 		 * erratum because all devices have been quiesced before
1743 		 * they unregister their interrupt handlers.  If that
1744 		 * assumption turns out to be false, this mask operation
1745 		 * can induce the same erratum result we're trying to
1746 		 * avoid.
1747 		 */
1748 		ioapic_ix = irqptr->airq_ioapicindex;
1749 		intin = irqptr->airq_intin_no;
1750 		ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK);
1751 	}
1752 
1753 #if !defined(__xpv)
1754 	apic_vt_ops->apic_intrr_free_entry(irqptr);
1755 #endif
1756 
1757 	if (max_ipl == PSM_INVALID_IPL) {
1758 		ASSERT(irqheadptr == irqptr);
1759 		bind_cpu = irqptr->airq_temp_cpu;
1760 		if (((uint32_t)bind_cpu != IRQ_UNBOUND) &&
1761 		    ((uint32_t)bind_cpu != IRQ_UNINIT)) {
1762 			ASSERT((bind_cpu & ~IRQ_USER_BOUND) < apic_nproc);
1763 			if (bind_cpu & IRQ_USER_BOUND) {
1764 				/* If hardbound, temp_cpu == cpu */
1765 				bind_cpu &= ~IRQ_USER_BOUND;
1766 				apic_cpus[bind_cpu].aci_bound--;
1767 			} else
1768 				apic_cpus[bind_cpu].aci_temp_bound--;
1769 		}
1770 		irqptr->airq_temp_cpu = IRQ_UNINIT;
1771 		irqptr->airq_mps_intr_index = FREE_INDEX;
1772 		lock_clear(&apic_ioapic_lock);
1773 		intr_restore(iflag);
1774 		apic_free_vector(irqptr->airq_vector);
1775 		return (PSM_SUCCESS);
1776 	}
1777 	lock_clear(&apic_ioapic_lock);
1778 	intr_restore(iflag);
1779 
1780 	mutex_enter(&airq_mutex);
1781 	if ((irqptr == apic_irq_table[irqindex])) {
1782 		apic_irq_t	*oldirqptr;
1783 		/* Move valid irq entry to the head */
1784 		irqheadptr = oldirqptr = irqptr;
1785 		irqptr = irqptr->airq_next;
1786 		ASSERT(irqptr);
1787 		while (irqptr) {
1788 			if (irqptr->airq_mps_intr_index != FREE_INDEX)
1789 				break;
1790 			oldirqptr = irqptr;
1791 			irqptr = irqptr->airq_next;
1792 		}
1793 		/* remove all invalid ones from the beginning */
1794 		apic_irq_table[irqindex] = irqptr;
1795 		/*
1796 		 * and link them back after the head. The invalid ones
1797 		 * begin with irqheadptr and end at oldirqptr
1798 		 */
1799 		oldirqptr->airq_next = irqptr->airq_next;
1800 		irqptr->airq_next = irqheadptr;
1801 	}
1802 	mutex_exit(&airq_mutex);
1803 
1804 	irqptr->airq_temp_cpu = IRQ_UNINIT;
1805 	irqptr->airq_mps_intr_index = FREE_INDEX;
1806 
1807 	return (PSM_SUCCESS);
1808 }
1809 
1810 /*
1811  * apic_introp_xlate() replaces apic_translate_irq() and is
1812  * called only from apic_intr_ops().  With the new ADII framework,
1813  * the priority can no longer be retrieved through i_ddi_get_intrspec().
1814  * It has to be passed in from the caller.
1815  */
1816 int
1817 apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type)
1818 {
1819 	char dev_type[16];
1820 	int dev_len, pci_irq, newirq, bustype, devid, busid, i;
1821 	int irqno = ispec->intrspec_vec;
1822 	ddi_acc_handle_t cfg_handle;
1823 	uchar_t ipin;
1824 	struct apic_io_intr *intrp;
1825 	iflag_t intr_flag;
1826 	ACPI_SUBTABLE_HEADER	*hp;
1827 	ACPI_MADT_INTERRUPT_OVERRIDE *isop;
1828 	apic_irq_t *airqp;
1829 	int parent_is_pci_or_pciex = 0;
1830 	int child_is_pciex = 0;
1831 
1832 	DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s "
1833 	    "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type,
1834 	    irqno));
1835 
1836 	dev_len = sizeof (dev_type);
1837 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
1838 	    DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
1839 	    &dev_len) == DDI_PROP_SUCCESS) {
1840 		if ((strcmp(dev_type, "pci") == 0) ||
1841 		    (strcmp(dev_type, "pciex") == 0))
1842 			parent_is_pci_or_pciex = 1;
1843 	}
1844 
1845 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
1846 	    DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
1847 	    &dev_len) == DDI_PROP_SUCCESS) {
1848 		if (strstr(dev_type, "pciex"))
1849 			child_is_pciex = 1;
1850 	}
1851 
1852 
1853 	if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
1854 		if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) {
1855 			airqp->airq_iflag.bustype =
1856 			    child_is_pciex ? BUS_PCIE : BUS_PCI;
1857 			return (apic_vector_to_irq[airqp->airq_vector]);
1858 		}
1859 		return (apic_setup_irq_table(dip, irqno, NULL, ispec,
1860 		    NULL, type));
1861 	}
1862 
1863 	bustype = 0;
1864 
1865 	/* check if we have already translated this irq */
1866 	mutex_enter(&airq_mutex);
1867 	newirq = apic_min_device_irq;
1868 	for (; newirq <= apic_max_device_irq; newirq++) {
1869 		airqp = apic_irq_table[newirq];
1870 		while (airqp) {
1871 			if ((airqp->airq_dip == dip) &&
1872 			    (airqp->airq_origirq == irqno) &&
1873 			    (airqp->airq_mps_intr_index != FREE_INDEX)) {
1874 
1875 				mutex_exit(&airq_mutex);
1876 				return (VIRTIRQ(newirq, airqp->airq_share_id));
1877 			}
1878 			airqp = airqp->airq_next;
1879 		}
1880 	}
1881 	mutex_exit(&airq_mutex);
1882 
1883 	if (apic_defconf)
1884 		goto defconf;
1885 
1886 	if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi))
1887 		goto nonpci;
1888 
1889 	if (parent_is_pci_or_pciex) {
1890 		/* pci device */
1891 		if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
1892 			goto nonpci;
1893 		if (busid == 0 && apic_pci_bus_total == 1)
1894 			busid = (int)apic_single_pci_busid;
1895 
1896 		if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
1897 			goto nonpci;
1898 		ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
1899 		pci_config_teardown(&cfg_handle);
1900 		if (apic_enable_acpi && !apic_use_acpi_madt_only) {
1901 			if (apic_acpi_translate_pci_irq(dip, busid, devid,
1902 			    ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
1903 				goto nonpci;
1904 
1905 			intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
1906 			if ((newirq = apic_setup_irq_table(dip, pci_irq, NULL,
1907 			    ispec, &intr_flag, type)) == -1)
1908 				goto nonpci;
1909 			return (newirq);
1910 		} else {
1911 			pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
1912 			if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid))
1913 			    == NULL) {
1914 				if ((pci_irq = apic_handle_pci_pci_bridge(dip,
1915 				    devid, ipin, &intrp)) == -1)
1916 					goto nonpci;
1917 			}
1918 			if ((newirq = apic_setup_irq_table(dip, pci_irq, intrp,
1919 			    ispec, NULL, type)) == -1)
1920 				goto nonpci;
1921 			return (newirq);
1922 		}
1923 	} else if (strcmp(dev_type, "isa") == 0)
1924 		bustype = BUS_ISA;
1925 	else if (strcmp(dev_type, "eisa") == 0)
1926 		bustype = BUS_EISA;
1927 
1928 nonpci:
1929 	if (apic_enable_acpi && !apic_use_acpi_madt_only) {
1930 		/* search iso entries first */
1931 		if (acpi_iso_cnt != 0) {
1932 			hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
1933 			i = 0;
1934 			while (i < acpi_iso_cnt) {
1935 				if (hp->Type ==
1936 				    ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
1937 					isop =
1938 					    (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
1939 					if (isop->Bus == 0 &&
1940 					    isop->SourceIrq == irqno) {
1941 						newirq = isop->GlobalIrq;
1942 						intr_flag.intr_po =
1943 						    isop->IntiFlags &
1944 						    ACPI_MADT_POLARITY_MASK;
1945 						intr_flag.intr_el =
1946 						    (isop->IntiFlags &
1947 						    ACPI_MADT_TRIGGER_MASK)
1948 						    >> 2;
1949 						intr_flag.bustype = BUS_ISA;
1950 
1951 						return (apic_setup_irq_table(
1952 						    dip, newirq, NULL, ispec,
1953 						    &intr_flag, type));
1954 
1955 					}
1956 					i++;
1957 				}
1958 				hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
1959 				    hp->Length);
1960 			}
1961 		}
1962 		intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
1963 		intr_flag.intr_el = INTR_EL_EDGE;
1964 		intr_flag.bustype = BUS_ISA;
1965 		return (apic_setup_irq_table(dip, irqno, NULL, ispec,
1966 		    &intr_flag, type));
1967 	} else {
1968 		if (bustype == 0)
1969 			bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
1970 		for (i = 0; i < 2; i++) {
1971 			if (((busid = apic_find_bus_id(bustype)) != -1) &&
1972 			    ((intrp = apic_find_io_intr_w_busid(irqno, busid))
1973 			    != NULL)) {
1974 				if ((newirq = apic_setup_irq_table(dip, irqno,
1975 				    intrp, ispec, NULL, type)) != -1) {
1976 					return (newirq);
1977 				}
1978 				goto defconf;
1979 			}
1980 			bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
1981 		}
1982 	}
1983 
1984 /* MPS default configuration */
1985 defconf:
1986 	newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type);
1987 	if (newirq == -1)
1988 		return (newirq);
1989 	ASSERT(IRQINDEX(newirq) == irqno);
1990 	ASSERT(apic_irq_table[irqno]);
1991 	return (newirq);
1992 }
1993 
1994 
1995 
1996 
1997 
1998 
1999 /*
2000  * On machines with PCI-PCI bridges, a device behind a PCI-PCI bridge
2001  * needs special handling.  We may need to chase up the device tree,
2002  * using the PCI-PCI Bridge specification's "rotating IPIN assumptions",
2003  * to find the IPIN at the root bus that relates to the IPIN on the
2004  * subsidiary bus (for ACPI or MP).  We may, however, have an entry
2005  * in the MP table or the ACPI namespace for this device itself.
2006  * We handle both cases in the search below.
2007  */
2008 /* this is the non-acpi version */
2009 static int
2010 apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno, int child_ipin,
2011 			struct apic_io_intr **intrp)
2012 {
2013 	dev_info_t *dipp, *dip;
2014 	int pci_irq;
2015 	ddi_acc_handle_t cfg_handle;
2016 	int bridge_devno, bridge_bus;
2017 	int ipin;
2018 
2019 	dip = idip;
2020 
2021 	/*CONSTCOND*/
2022 	while (1) {
2023 		if (((dipp = ddi_get_parent(dip)) == (dev_info_t *)NULL) ||
2024 		    (pci_config_setup(dipp, &cfg_handle) != DDI_SUCCESS))
2025 			return (-1);
2026 		if ((pci_config_get8(cfg_handle, PCI_CONF_BASCLASS) ==
2027 		    PCI_CLASS_BRIDGE) && (pci_config_get8(cfg_handle,
2028 		    PCI_CONF_SUBCLASS) == PCI_BRIDGE_PCI)) {
2029 			pci_config_teardown(&cfg_handle);
2030 			if (acpica_get_bdf(dipp, &bridge_bus, &bridge_devno,
2031 			    NULL) != 0)
2032 				return (-1);
2033 			/*
2034 			 * This is the rotating scheme documented in the
2035 			 * PCI-to-PCI spec.  If the PCI-to-PCI bridge is
2036 			 * behind another PCI-to-PCI bridge, then it needs
2037 			 * to keep ascending until an interrupt entry is
2038 			 * found or the root is reached.
2039 			 */
2040 			ipin = (child_devno + child_ipin) % PCI_INTD;
2041 				if (bridge_bus == 0 && apic_pci_bus_total == 1)
2042 					bridge_bus = (int)apic_single_pci_busid;
2043 				pci_irq = ((bridge_devno & 0x1f) << 2) |
2044 				    (ipin & 0x3);
2045 				if ((*intrp = apic_find_io_intr_w_busid(pci_irq,
2046 				    bridge_bus)) != NULL) {
2047 					return (pci_irq);
2048 				}
2049 			dip = dipp;
2050 			child_devno = bridge_devno;
2051 			child_ipin = ipin;
2052 		} else {
2053 			pci_config_teardown(&cfg_handle);
2054 			return (-1);
2055 		}
2056 	}
2057 	/*LINTED: function will not fall off the bottom */
2058 }
2059 
2060 
2061 
2062 
2063 static uchar_t
2064 acpi_find_ioapic(int irq)
2065 {
2066 	int i;
2067 
2068 	for (i = 0; i < apic_io_max; i++) {
2069 		if (irq >= apic_io_vectbase[i] && irq <= apic_io_vectend[i])
2070 			return (i);
2071 	}
2072 	return (0xFF);	/* shouldn't happen */
2073 }
2074 
2075 /*
2076  * See if two irqs are compatible for sharing a vector.
2077  * Currently we only support sharing of PCI devices.
2078  */
2079 static int
2080 acpi_intr_compatible(iflag_t iflag1, iflag_t iflag2)
2081 {
2082 	uint_t	level1, po1;
2083 	uint_t	level2, po2;
2084 
2085 	/* Assume active high by default */
2086 	po1 = 0;
2087 	po2 = 0;
2088 
2089 	if (iflag1.bustype != iflag2.bustype || iflag1.bustype != BUS_PCI)
2090 		return (0);
2091 
2092 	if (iflag1.intr_el == INTR_EL_CONFORM)
2093 		level1 = AV_LEVEL;
2094 	else
2095 		level1 = (iflag1.intr_el == INTR_EL_LEVEL) ? AV_LEVEL : 0;
2096 
2097 	if (level1 && ((iflag1.intr_po == INTR_PO_ACTIVE_LOW) ||
2098 	    (iflag1.intr_po == INTR_PO_CONFORM)))
2099 		po1 = AV_ACTIVE_LOW;
2100 
2101 	if (iflag2.intr_el == INTR_EL_CONFORM)
2102 		level2 = AV_LEVEL;
2103 	else
2104 		level2 = (iflag2.intr_el == INTR_EL_LEVEL) ? AV_LEVEL : 0;
2105 
2106 	if (level2 && ((iflag2.intr_po == INTR_PO_ACTIVE_LOW) ||
2107 	    (iflag2.intr_po == INTR_PO_CONFORM)))
2108 		po2 = AV_ACTIVE_LOW;
2109 
2110 	if ((level1 == level2) && (po1 == po2))
2111 		return (1);
2112 
2113 	return (0);
2114 }
2115 
2116 /*
2117  * Attempt to share vector with someone else
2118  */
2119 static int
2120 apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl,
2121 	uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
2122 {
2123 #ifdef DEBUG
2124 	apic_irq_t *tmpirqp = NULL;
2125 #endif /* DEBUG */
2126 	apic_irq_t *irqptr, dummyirq;
2127 	int	newirq, chosen_irq = -1, share = 127;
2128 	int	lowest, highest, i;
2129 	uchar_t	share_id;
2130 
2131 	DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x "
2132 	    "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl));
2133 
2134 	highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
2135 	lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL;
2136 
2137 	if (highest < lowest) /* Both ipl and ipl-1 map to same pri */
2138 		lowest -= APIC_VECTOR_PER_IPL;
2139 	dummyirq.airq_mps_intr_index = intr_index;
2140 	dummyirq.airq_ioapicindex = ioapicindex;
2141 	dummyirq.airq_intin_no = ipin;
2142 	if (intr_flagp)
2143 		dummyirq.airq_iflag = *intr_flagp;
2144 	apic_record_rdt_entry(&dummyirq, irqno);
2145 	for (i = lowest; i <= highest; i++) {
2146 		newirq = apic_vector_to_irq[i];
2147 		if (newirq == APIC_RESV_IRQ)
2148 			continue;
2149 		irqptr = apic_irq_table[newirq];
2150 
2151 		if ((dummyirq.airq_rdt_entry & 0xFF00) !=
2152 		    (irqptr->airq_rdt_entry & 0xFF00))
2153 			/* not compatible */
2154 			continue;
2155 
2156 		if (irqptr->airq_share < share) {
2157 			share = irqptr->airq_share;
2158 			chosen_irq = newirq;
2159 		}
2160 	}
2161 	if (chosen_irq != -1) {
2162 		/*
2163 		 * Assign a share id which is free or which is larger
2164 		 * than the largest one.
2165 		 */
2166 		share_id = 1;
2167 		mutex_enter(&airq_mutex);
2168 		irqptr = apic_irq_table[chosen_irq];
2169 		while (irqptr) {
2170 			if (irqptr->airq_mps_intr_index == FREE_INDEX) {
2171 				share_id = irqptr->airq_share_id;
2172 				break;
2173 			}
2174 			if (share_id <= irqptr->airq_share_id)
2175 				share_id = irqptr->airq_share_id + 1;
2176 #ifdef DEBUG
2177 			tmpirqp = irqptr;
2178 #endif /* DEBUG */
2179 			irqptr = irqptr->airq_next;
2180 		}
2181 		if (!irqptr) {
2182 			irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2183 			irqptr->airq_temp_cpu = IRQ_UNINIT;
2184 			irqptr->airq_next =
2185 			    apic_irq_table[chosen_irq]->airq_next;
2186 			apic_irq_table[chosen_irq]->airq_next = irqptr;
2187 #ifdef	DEBUG
2188 			tmpirqp = apic_irq_table[chosen_irq];
2189 #endif /* DEBUG */
2190 		}
2191 		irqptr->airq_mps_intr_index = intr_index;
2192 		irqptr->airq_ioapicindex = ioapicindex;
2193 		irqptr->airq_intin_no = ipin;
2194 		if (intr_flagp)
2195 			irqptr->airq_iflag = *intr_flagp;
2196 		irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector;
2197 		irqptr->airq_share_id = share_id;
2198 		apic_record_rdt_entry(irqptr, irqno);
2199 		*irqptrp = irqptr;
2200 #ifdef	DEBUG
2201 		/* shuffle the pointers to test apic_delspl path */
2202 		if (tmpirqp) {
2203 			tmpirqp->airq_next = irqptr->airq_next;
2204 			irqptr->airq_next = apic_irq_table[chosen_irq];
2205 			apic_irq_table[chosen_irq] = irqptr;
2206 		}
2207 #endif /* DEBUG */
2208 		mutex_exit(&airq_mutex);
2209 		return (VIRTIRQ(chosen_irq, share_id));
2210 	}
2211 	return (-1);
2212 }
2213 
2214 /*
2215  *
2216  */
2217 static int
2218 apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp,
2219     struct intrspec *ispec, iflag_t *intr_flagp, int type)
2220 {
2221 	int origirq = ispec->intrspec_vec;
2222 	uchar_t ipl = ispec->intrspec_pri;
2223 	int	newirq, intr_index;
2224 	uchar_t	ipin, ioapic, ioapicindex, vector;
2225 	apic_irq_t *irqptr;
2226 	major_t	major;
2227 	dev_info_t	*sdip;
2228 
2229 	DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d "
2230 	    "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq));
2231 
2232 	ASSERT(ispec != NULL);
2233 
2234 	major =  (dip != NULL) ? ddi_driver_major(dip) : 0;
2235 
2236 	if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
2237 		/* MSI/X doesn't need to setup ioapic stuffs */
2238 		ioapicindex = 0xff;
2239 		ioapic = 0xff;
2240 		ipin = (uchar_t)0xff;
2241 		intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX :
2242 		    MSIX_INDEX;
2243 		mutex_enter(&airq_mutex);
2244 		if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) {
2245 			mutex_exit(&airq_mutex);
2246 			/* need an irq for MSI/X to index into autovect[] */
2247 			cmn_err(CE_WARN, "No interrupt irq: %s instance %d",
2248 			    ddi_get_name(dip), ddi_get_instance(dip));
2249 			return (-1);
2250 		}
2251 		mutex_exit(&airq_mutex);
2252 
2253 	} else if (intrp != NULL) {
2254 		intr_index = (int)(intrp - apic_io_intrp);
2255 		ioapic = intrp->intr_destid;
2256 		ipin = intrp->intr_destintin;
2257 		/* Find ioapicindex. If destid was ALL, we will exit with 0. */
2258 		for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
2259 			if (apic_io_id[ioapicindex] == ioapic)
2260 				break;
2261 		ASSERT((ioapic == apic_io_id[ioapicindex]) ||
2262 		    (ioapic == INTR_ALL_APIC));
2263 
2264 		/* check whether this intin# has been used by another irqno */
2265 		if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) {
2266 			return (newirq);
2267 		}
2268 
2269 	} else if (intr_flagp != NULL) {
2270 		/* ACPI case */
2271 		intr_index = ACPI_INDEX;
2272 		ioapicindex = acpi_find_ioapic(irqno);
2273 		ASSERT(ioapicindex != 0xFF);
2274 		ioapic = apic_io_id[ioapicindex];
2275 		ipin = irqno - apic_io_vectbase[ioapicindex];
2276 		if (apic_irq_table[irqno] &&
2277 		    apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
2278 			ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
2279 			    apic_irq_table[irqno]->airq_ioapicindex ==
2280 			    ioapicindex);
2281 			return (irqno);
2282 		}
2283 
2284 	} else {
2285 		/* default configuration */
2286 		ioapicindex = 0;
2287 		ioapic = apic_io_id[ioapicindex];
2288 		ipin = (uchar_t)irqno;
2289 		intr_index = DEFAULT_INDEX;
2290 	}
2291 
2292 	if (ispec == NULL) {
2293 		APIC_VERBOSE_IOAPIC((CE_WARN, "No intrspec for irqno = %x\n",
2294 		    irqno));
2295 	} else if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
2296 		if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index,
2297 		    ipl, ioapicindex, ipin, &irqptr)) != -1) {
2298 			irqptr->airq_ipl = ipl;
2299 			irqptr->airq_origirq = (uchar_t)origirq;
2300 			irqptr->airq_dip = dip;
2301 			irqptr->airq_major = major;
2302 			sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip;
2303 			/* This is OK to do really */
2304 			if (sdip == NULL) {
2305 				cmn_err(CE_WARN, "Sharing vectors: %s"
2306 				    " instance %d and SCI",
2307 				    ddi_get_name(dip), ddi_get_instance(dip));
2308 			} else {
2309 				cmn_err(CE_WARN, "Sharing vectors: %s"
2310 				    " instance %d and %s instance %d",
2311 				    ddi_get_name(sdip), ddi_get_instance(sdip),
2312 				    ddi_get_name(dip), ddi_get_instance(dip));
2313 			}
2314 			return (newirq);
2315 		}
2316 		/* try high priority allocation now  that share has failed */
2317 		if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) {
2318 			cmn_err(CE_WARN, "No interrupt vector: %s instance %d",
2319 			    ddi_get_name(dip), ddi_get_instance(dip));
2320 			return (-1);
2321 		}
2322 	}
2323 
2324 	mutex_enter(&airq_mutex);
2325 	if (apic_irq_table[irqno] == NULL) {
2326 		irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2327 		irqptr->airq_temp_cpu = IRQ_UNINIT;
2328 		apic_irq_table[irqno] = irqptr;
2329 	} else {
2330 		irqptr = apic_irq_table[irqno];
2331 		if (irqptr->airq_mps_intr_index != FREE_INDEX) {
2332 			/*
2333 			 * The slot is used by another irqno, so allocate
2334 			 * a free irqno for this interrupt
2335 			 */
2336 			newirq = apic_allocate_irq(apic_first_avail_irq);
2337 			if (newirq == -1) {
2338 				mutex_exit(&airq_mutex);
2339 				return (-1);
2340 			}
2341 			irqno = newirq;
2342 			irqptr = apic_irq_table[irqno];
2343 			if (irqptr == NULL) {
2344 				irqptr = kmem_zalloc(sizeof (apic_irq_t),
2345 				    KM_SLEEP);
2346 				irqptr->airq_temp_cpu = IRQ_UNINIT;
2347 				apic_irq_table[irqno] = irqptr;
2348 			}
2349 			vector = apic_modify_vector(vector, newirq);
2350 		}
2351 	}
2352 	apic_max_device_irq = max(irqno, apic_max_device_irq);
2353 	apic_min_device_irq = min(irqno, apic_min_device_irq);
2354 	mutex_exit(&airq_mutex);
2355 	irqptr->airq_ioapicindex = ioapicindex;
2356 	irqptr->airq_intin_no = ipin;
2357 	irqptr->airq_ipl = ipl;
2358 	irqptr->airq_vector = vector;
2359 	irqptr->airq_origirq = (uchar_t)origirq;
2360 	irqptr->airq_share_id = 0;
2361 	irqptr->airq_mps_intr_index = (short)intr_index;
2362 	irqptr->airq_dip = dip;
2363 	irqptr->airq_major = major;
2364 	irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin);
2365 	if (intr_flagp)
2366 		irqptr->airq_iflag = *intr_flagp;
2367 
2368 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
2369 		/* setup I/O APIC entry for non-MSI/X interrupts */
2370 		apic_record_rdt_entry(irqptr, irqno);
2371 	}
2372 	return (irqno);
2373 }
2374 
2375 /*
2376  * return the cpu to which this intr should be bound.
2377  * Check properties or any other mechanism to see if user wants it
2378  * bound to a specific CPU. If so, return the cpu id with high bit set.
2379  * If not, use the policy to choose a cpu and return the id.
2380  */
2381 uint32_t
2382 apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin)
2383 {
2384 	int	instance, instno, prop_len, bind_cpu, count;
2385 	uint_t	i, rc;
2386 	uint32_t cpu;
2387 	major_t	major;
2388 	char	*name, *drv_name, *prop_val, *cptr;
2389 	char	prop_name[32];
2390 
2391 
2392 	if (apic_intr_policy == INTR_LOWEST_PRIORITY)
2393 		return (IRQ_UNBOUND);
2394 
2395 	if (apic_nproc == 1)
2396 		return (0);
2397 
2398 	drv_name = NULL;
2399 	rc = DDI_PROP_NOT_FOUND;
2400 	major = (major_t)-1;
2401 	if (dip != NULL) {
2402 		name = ddi_get_name(dip);
2403 		major = ddi_name_to_major(name);
2404 		drv_name = ddi_major_to_name(major);
2405 		instance = ddi_get_instance(dip);
2406 		if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
2407 			i = apic_min_device_irq;
2408 			for (; i <= apic_max_device_irq; i++) {
2409 
2410 				if ((i == irq) || (apic_irq_table[i] == NULL) ||
2411 				    (apic_irq_table[i]->airq_mps_intr_index
2412 				    == FREE_INDEX))
2413 					continue;
2414 
2415 				if ((apic_irq_table[i]->airq_major == major) &&
2416 				    (!(apic_irq_table[i]->airq_cpu &
2417 				    IRQ_USER_BOUND))) {
2418 
2419 					cpu = apic_irq_table[i]->airq_cpu;
2420 
2421 					cmn_err(CE_CONT,
2422 					    "!%s: %s (%s) instance #%d "
2423 					    "irq 0x%x vector 0x%x ioapic 0x%x "
2424 					    "intin 0x%x is bound to cpu %d\n",
2425 					    psm_name,
2426 					    name, drv_name, instance, irq,
2427 					    apic_irq_table[irq]->airq_vector,
2428 					    ioapicid, intin, cpu);
2429 					return (cpu);
2430 				}
2431 			}
2432 		}
2433 		/*
2434 		 * search for "drvname"_intpt_bind_cpus property first, the
2435 		 * syntax of the property should be "a[,b,c,...]" where
2436 		 * instance 0 binds to cpu a, instance 1 binds to cpu b,
2437 		 * instance 3 binds to cpu c...
2438 		 * ddi_getlongprop() will search /option first, then /
2439 		 * if "drvname"_intpt_bind_cpus doesn't exist, then find
2440 		 * intpt_bind_cpus property.  The syntax is the same, and
2441 		 * it applies to all the devices if its "drvname" specific
2442 		 * property doesn't exist
2443 		 */
2444 		(void) strcpy(prop_name, drv_name);
2445 		(void) strcat(prop_name, "_intpt_bind_cpus");
2446 		rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
2447 		    (caddr_t)&prop_val, &prop_len);
2448 		if (rc != DDI_PROP_SUCCESS) {
2449 			rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
2450 			    "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
2451 		}
2452 	}
2453 	if (rc == DDI_PROP_SUCCESS) {
2454 		for (i = count = 0; i < (prop_len - 1); i++)
2455 			if (prop_val[i] == ',')
2456 				count++;
2457 		if (prop_val[i-1] != ',')
2458 			count++;
2459 		/*
2460 		 * if somehow the binding instances defined in the
2461 		 * property are not enough for this instno., then
2462 		 * reuse the pattern for the next instance until
2463 		 * it reaches the requested instno
2464 		 */
2465 		instno = instance % count;
2466 		i = 0;
2467 		cptr = prop_val;
2468 		while (i < instno)
2469 			if (*cptr++ == ',')
2470 				i++;
2471 		bind_cpu = stoi(&cptr);
2472 		kmem_free(prop_val, prop_len);
2473 		/* if specific cpu is bogus, then default to cpu 0 */
2474 		if (bind_cpu >= apic_nproc) {
2475 			cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present",
2476 			    psm_name, prop_name, prop_val, bind_cpu);
2477 			bind_cpu = 0;
2478 		} else {
2479 			/* indicate that we are bound at user request */
2480 			bind_cpu |= IRQ_USER_BOUND;
2481 		}
2482 		/*
2483 		 * no need to check apic_cpus[].aci_status, if specific cpu is
2484 		 * not up, then post_cpu_start will handle it.
2485 		 */
2486 	} else {
2487 		bind_cpu = apic_next_bind_cpu++;
2488 		if (bind_cpu >= apic_nproc) {
2489 			apic_next_bind_cpu = 1;
2490 			bind_cpu = 0;
2491 		}
2492 	}
2493 	if (drv_name != NULL)
2494 		cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
2495 		    "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
2496 		    psm_name, name, drv_name, instance, irq,
2497 		    apic_irq_table[irq]->airq_vector, ioapicid, intin,
2498 		    bind_cpu & ~IRQ_USER_BOUND);
2499 	else
2500 		cmn_err(CE_CONT, "!%s: irq 0x%x "
2501 		    "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
2502 		    psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
2503 		    intin, bind_cpu & ~IRQ_USER_BOUND);
2504 
2505 	return ((uint32_t)bind_cpu);
2506 }
2507 
2508 static struct apic_io_intr *
2509 apic_find_io_intr_w_busid(int irqno, int busid)
2510 {
2511 	struct	apic_io_intr	*intrp;
2512 
2513 	/*
2514 	 * It can have more than 1 entry with same source bus IRQ,
2515 	 * but unique with the source bus id
2516 	 */
2517 	intrp = apic_io_intrp;
2518 	if (intrp != NULL) {
2519 		while (intrp->intr_entry == APIC_IO_INTR_ENTRY) {
2520 			if (intrp->intr_irq == irqno &&
2521 			    intrp->intr_busid == busid &&
2522 			    intrp->intr_type == IO_INTR_INT)
2523 				return (intrp);
2524 			intrp++;
2525 		}
2526 	}
2527 	APIC_VERBOSE_IOAPIC((CE_NOTE, "Did not find io intr for irqno:"
2528 	    "busid %x:%x\n", irqno, busid));
2529 	return ((struct apic_io_intr *)NULL);
2530 }
2531 
2532 
2533 struct mps_bus_info {
2534 	char	*bus_name;
2535 	int	bus_id;
2536 } bus_info_array[] = {
2537 	"ISA ", BUS_ISA,
2538 	"PCI ", BUS_PCI,
2539 	"EISA ", BUS_EISA,
2540 	"XPRESS", BUS_XPRESS,
2541 	"PCMCIA", BUS_PCMCIA,
2542 	"VL ", BUS_VL,
2543 	"CBUS ", BUS_CBUS,
2544 	"CBUSII", BUS_CBUSII,
2545 	"FUTURE", BUS_FUTURE,
2546 	"INTERN", BUS_INTERN,
2547 	"MBI ", BUS_MBI,
2548 	"MBII ", BUS_MBII,
2549 	"MPI ", BUS_MPI,
2550 	"MPSA ", BUS_MPSA,
2551 	"NUBUS ", BUS_NUBUS,
2552 	"TC ", BUS_TC,
2553 	"VME ", BUS_VME,
2554 	"PCI-E ", BUS_PCIE
2555 };
2556 
2557 static int
2558 apic_find_bus_type(char *bus)
2559 {
2560 	int	i = 0;
2561 
2562 	for (; i < sizeof (bus_info_array)/sizeof (struct mps_bus_info); i++)
2563 		if (strncmp(bus, bus_info_array[i].bus_name,
2564 		    strlen(bus_info_array[i].bus_name)) == 0)
2565 			return (bus_info_array[i].bus_id);
2566 	APIC_VERBOSE_IOAPIC((CE_WARN, "Did not find bus type for bus %s", bus));
2567 	return (0);
2568 }
2569 
2570 static int
2571 apic_find_bus(int busid)
2572 {
2573 	struct	apic_bus	*busp;
2574 
2575 	busp = apic_busp;
2576 	while (busp->bus_entry == APIC_BUS_ENTRY) {
2577 		if (busp->bus_id == busid)
2578 			return (apic_find_bus_type((char *)&busp->bus_str1));
2579 		busp++;
2580 	}
2581 	APIC_VERBOSE_IOAPIC((CE_WARN, "Did not find bus for bus id %x", busid));
2582 	return (0);
2583 }
2584 
2585 static int
2586 apic_find_bus_id(int bustype)
2587 {
2588 	struct	apic_bus	*busp;
2589 
2590 	busp = apic_busp;
2591 	while (busp->bus_entry == APIC_BUS_ENTRY) {
2592 		if (apic_find_bus_type((char *)&busp->bus_str1) == bustype)
2593 			return (busp->bus_id);
2594 		busp++;
2595 	}
2596 	APIC_VERBOSE_IOAPIC((CE_WARN, "Did not find bus id for bustype %x",
2597 	    bustype));
2598 	return (-1);
2599 }
2600 
2601 /*
2602  * Check if a particular irq need to be reserved for any io_intr
2603  */
2604 static struct apic_io_intr *
2605 apic_find_io_intr(int irqno)
2606 {
2607 	struct	apic_io_intr	*intrp;
2608 
2609 	intrp = apic_io_intrp;
2610 	if (intrp != NULL) {
2611 		while (intrp->intr_entry == APIC_IO_INTR_ENTRY) {
2612 			if (intrp->intr_irq == irqno &&
2613 			    intrp->intr_type == IO_INTR_INT)
2614 				return (intrp);
2615 			intrp++;
2616 		}
2617 	}
2618 	return ((struct apic_io_intr *)NULL);
2619 }
2620 
2621 /*
2622  * Check if the given ioapicindex intin combination has already been assigned
2623  * an irq. If so return irqno. Else -1
2624  */
2625 static int
2626 apic_find_intin(uchar_t ioapic, uchar_t intin)
2627 {
2628 	apic_irq_t *irqptr;
2629 	int	i;
2630 
2631 	/* find ioapic and intin in the apic_irq_table[] and return the index */
2632 	for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
2633 		irqptr = apic_irq_table[i];
2634 		while (irqptr) {
2635 			if ((irqptr->airq_mps_intr_index >= 0) &&
2636 			    (irqptr->airq_intin_no == intin) &&
2637 			    (irqptr->airq_ioapicindex == ioapic)) {
2638 				APIC_VERBOSE_IOAPIC((CE_NOTE, "!Found irq "
2639 				    "entry for ioapic:intin %x:%x "
2640 				    "shared interrupts ?", ioapic, intin));
2641 				return (i);
2642 			}
2643 			irqptr = irqptr->airq_next;
2644 		}
2645 	}
2646 	return (-1);
2647 }
2648 
2649 int
2650 apic_allocate_irq(int irq)
2651 {
2652 	int	freeirq, i;
2653 
2654 	if ((freeirq = apic_find_free_irq(irq, (APIC_RESV_IRQ - 1))) == -1)
2655 		if ((freeirq = apic_find_free_irq(APIC_FIRST_FREE_IRQ,
2656 		    (irq - 1))) == -1) {
2657 			/*
2658 			 * if BIOS really defines every single irq in the mps
2659 			 * table, then don't worry about conflicting with
2660 			 * them, just use any free slot in apic_irq_table
2661 			 */
2662 			for (i = APIC_FIRST_FREE_IRQ; i < APIC_RESV_IRQ; i++) {
2663 				if ((apic_irq_table[i] == NULL) ||
2664 				    apic_irq_table[i]->airq_mps_intr_index ==
2665 				    FREE_INDEX) {
2666 				freeirq = i;
2667 				break;
2668 			}
2669 		}
2670 		if (freeirq == -1) {
2671 			/* This shouldn't happen, but just in case */
2672 			cmn_err(CE_WARN, "%s: NO available IRQ", psm_name);
2673 			return (-1);
2674 		}
2675 	}
2676 	if (apic_irq_table[freeirq] == NULL) {
2677 		apic_irq_table[freeirq] =
2678 		    kmem_zalloc(sizeof (apic_irq_t), KM_NOSLEEP);
2679 		if (apic_irq_table[freeirq] == NULL) {
2680 			cmn_err(CE_WARN, "%s: NO memory to allocate IRQ",
2681 			    psm_name);
2682 			return (-1);
2683 		}
2684 		apic_irq_table[freeirq]->airq_mps_intr_index = FREE_INDEX;
2685 	}
2686 	return (freeirq);
2687 }
2688 
2689 static int
2690 apic_find_free_irq(int start, int end)
2691 {
2692 	int	i;
2693 
2694 	for (i = start; i <= end; i++)
2695 		/* Check if any I/O entry needs this IRQ */
2696 		if (apic_find_io_intr(i) == NULL) {
2697 			/* Then see if it is free */
2698 			if ((apic_irq_table[i] == NULL) ||
2699 			    (apic_irq_table[i]->airq_mps_intr_index ==
2700 			    FREE_INDEX)) {
2701 				return (i);
2702 			}
2703 		}
2704 	return (-1);
2705 }
2706 
2707 
2708 /*
2709  * Mark vector as being in the process of being deleted. Interrupts
2710  * may still come in on some CPU. The moment an interrupt comes with
2711  * the new vector, we know we can free the old one. Called only from
2712  * addspl and delspl with interrupts disabled. Because an interrupt
2713  * can be shared, but no interrupt from either device may come in,
2714  * we also use a timeout mechanism, which we arbitrarily set to
2715  * apic_revector_timeout microseconds.
2716  */
2717 static void
2718 apic_mark_vector(uchar_t oldvector, uchar_t newvector)
2719 {
2720 	ulong_t iflag;
2721 
2722 	iflag = intr_clear();
2723 	lock_set(&apic_revector_lock);
2724 	if (!apic_oldvec_to_newvec) {
2725 		apic_oldvec_to_newvec =
2726 		    kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2,
2727 		    KM_NOSLEEP);
2728 
2729 		if (!apic_oldvec_to_newvec) {
2730 			/*
2731 			 * This failure is not catastrophic.
2732 			 * But, the oldvec will never be freed.
2733 			 */
2734 			apic_error |= APIC_ERR_MARK_VECTOR_FAIL;
2735 			lock_clear(&apic_revector_lock);
2736 			intr_restore(iflag);
2737 			return;
2738 		}
2739 		apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR];
2740 	}
2741 
2742 	/* See if we already did this for drivers which do double addintrs */
2743 	if (apic_oldvec_to_newvec[oldvector] != newvector) {
2744 		apic_oldvec_to_newvec[oldvector] = newvector;
2745 		apic_newvec_to_oldvec[newvector] = oldvector;
2746 		apic_revector_pending++;
2747 	}
2748 	lock_clear(&apic_revector_lock);
2749 	intr_restore(iflag);
2750 	(void) timeout(apic_xlate_vector_free_timeout_handler,
2751 	    (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout));
2752 }
2753 
2754 /*
2755  * xlate_vector is called from intr_enter if revector_pending is set.
2756  * It will xlate it if needed and mark the old vector as free.
2757  */
2758 uchar_t
2759 apic_xlate_vector(uchar_t vector)
2760 {
2761 	uchar_t	newvector, oldvector = 0;
2762 
2763 	lock_set(&apic_revector_lock);
2764 	/* Do we really need to do this ? */
2765 	if (!apic_revector_pending) {
2766 		lock_clear(&apic_revector_lock);
2767 		return (vector);
2768 	}
2769 	if ((newvector = apic_oldvec_to_newvec[vector]) != 0)
2770 		oldvector = vector;
2771 	else {
2772 		/*
2773 		 * The incoming vector is new . See if a stale entry is
2774 		 * remaining
2775 		 */
2776 		if ((oldvector = apic_newvec_to_oldvec[vector]) != 0)
2777 			newvector = vector;
2778 	}
2779 
2780 	if (oldvector) {
2781 		apic_revector_pending--;
2782 		apic_oldvec_to_newvec[oldvector] = 0;
2783 		apic_newvec_to_oldvec[newvector] = 0;
2784 		apic_free_vector(oldvector);
2785 		lock_clear(&apic_revector_lock);
2786 		/* There could have been more than one reprogramming! */
2787 		return (apic_xlate_vector(newvector));
2788 	}
2789 	lock_clear(&apic_revector_lock);
2790 	return (vector);
2791 }
2792 
2793 void
2794 apic_xlate_vector_free_timeout_handler(void *arg)
2795 {
2796 	ulong_t iflag;
2797 	uchar_t oldvector, newvector;
2798 
2799 	oldvector = (uchar_t)(uintptr_t)arg;
2800 	iflag = intr_clear();
2801 	lock_set(&apic_revector_lock);
2802 	if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) {
2803 		apic_free_vector(oldvector);
2804 		apic_oldvec_to_newvec[oldvector] = 0;
2805 		apic_newvec_to_oldvec[newvector] = 0;
2806 		apic_revector_pending--;
2807 	}
2808 
2809 	lock_clear(&apic_revector_lock);
2810 	intr_restore(iflag);
2811 }
2812 
2813 
2814 /*
2815  * compute the polarity, trigger mode and vector for programming into
2816  * the I/O apic and record in airq_rdt_entry.
2817  */
2818 static void
2819 apic_record_rdt_entry(apic_irq_t *irqptr, int irq)
2820 {
2821 	int	ioapicindex, bus_type, vector;
2822 	short	intr_index;
2823 	uint_t	level, po, io_po;
2824 	struct apic_io_intr *iointrp;
2825 
2826 	intr_index = irqptr->airq_mps_intr_index;
2827 	DDI_INTR_IMPLDBG((CE_CONT, "apic_record_rdt_entry: intr_index=%d "
2828 	    "irq = 0x%x dip = 0x%p vector = 0x%x\n", intr_index, irq,
2829 	    (void *)irqptr->airq_dip, irqptr->airq_vector));
2830 
2831 	if (intr_index == RESERVE_INDEX) {
2832 		apic_error |= APIC_ERR_INVALID_INDEX;
2833 		return;
2834 	} else if (APIC_IS_MSI_OR_MSIX_INDEX(intr_index)) {
2835 		return;
2836 	}
2837 
2838 	vector = irqptr->airq_vector;
2839 	ioapicindex = irqptr->airq_ioapicindex;
2840 	/* Assume edge triggered by default */
2841 	level = 0;
2842 	/* Assume active high by default */
2843 	po = 0;
2844 
2845 	if (intr_index == DEFAULT_INDEX || intr_index == FREE_INDEX) {
2846 		ASSERT(irq < 16);
2847 		if (eisa_level_intr_mask & (1 << irq))
2848 			level = AV_LEVEL;
2849 		if (intr_index == FREE_INDEX && apic_defconf == 0)
2850 			apic_error |= APIC_ERR_INVALID_INDEX;
2851 	} else if (intr_index == ACPI_INDEX) {
2852 		bus_type = irqptr->airq_iflag.bustype;
2853 		if (irqptr->airq_iflag.intr_el == INTR_EL_CONFORM) {
2854 			if (bus_type == BUS_PCI)
2855 				level = AV_LEVEL;
2856 		} else
2857 			level = (irqptr->airq_iflag.intr_el == INTR_EL_LEVEL) ?
2858 			    AV_LEVEL : 0;
2859 		if (level &&
2860 		    ((irqptr->airq_iflag.intr_po == INTR_PO_ACTIVE_LOW) ||
2861 		    (irqptr->airq_iflag.intr_po == INTR_PO_CONFORM &&
2862 		    bus_type == BUS_PCI)))
2863 			po = AV_ACTIVE_LOW;
2864 	} else {
2865 		iointrp = apic_io_intrp + intr_index;
2866 		bus_type = apic_find_bus(iointrp->intr_busid);
2867 		if (iointrp->intr_el == INTR_EL_CONFORM) {
2868 			if ((irq < 16) && (eisa_level_intr_mask & (1 << irq)))
2869 				level = AV_LEVEL;
2870 			else if (bus_type == BUS_PCI)
2871 				level = AV_LEVEL;
2872 		} else
2873 			level = (iointrp->intr_el == INTR_EL_LEVEL) ?
2874 			    AV_LEVEL : 0;
2875 		if (level && ((iointrp->intr_po == INTR_PO_ACTIVE_LOW) ||
2876 		    (iointrp->intr_po == INTR_PO_CONFORM &&
2877 		    bus_type == BUS_PCI)))
2878 			po = AV_ACTIVE_LOW;
2879 	}
2880 	if (level)
2881 		apic_level_intr[irq] = 1;
2882 	/*
2883 	 * The 82489DX External APIC cannot do active low polarity interrupts.
2884 	 */
2885 	if (po && (apic_io_ver[ioapicindex] != IOAPIC_VER_82489DX))
2886 		io_po = po;
2887 	else
2888 		io_po = 0;
2889 
2890 	if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG)
2891 		printf("setio: ioapic=%x intin=%x level=%x po=%x vector=%x\n",
2892 		    ioapicindex, irqptr->airq_intin_no, level, io_po, vector);
2893 
2894 	irqptr->airq_rdt_entry = level|io_po|vector;
2895 }
2896 
2897 /*
2898  * Bind interrupt corresponding to irq_ptr to bind_cpu.
2899  * Must be called with interrupts disabled and apic_ioapic_lock held
2900  */
2901 int
2902 apic_rebind(apic_irq_t *irq_ptr, int bind_cpu,
2903     struct ioapic_reprogram_data *drep)
2904 {
2905 	int			ioapicindex, intin_no;
2906 	uint32_t		airq_temp_cpu;
2907 	apic_cpus_info_t	*cpu_infop;
2908 	uint32_t		rdt_entry;
2909 	int			which_irq;
2910 	ioapic_rdt_t		irdt;
2911 
2912 	which_irq = apic_vector_to_irq[irq_ptr->airq_vector];
2913 
2914 	intin_no = irq_ptr->airq_intin_no;
2915 	ioapicindex = irq_ptr->airq_ioapicindex;
2916 	airq_temp_cpu = irq_ptr->airq_temp_cpu;
2917 	if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) {
2918 		if (airq_temp_cpu & IRQ_USER_BOUND)
2919 			/* Mask off high bit so it can be used as array index */
2920 			airq_temp_cpu &= ~IRQ_USER_BOUND;
2921 
2922 		ASSERT(airq_temp_cpu < apic_nproc);
2923 	}
2924 
2925 	/*
2926 	 * Can't bind to a CPU that's not accepting interrupts:
2927 	 */
2928 	cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND];
2929 	if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE))
2930 		return (1);
2931 
2932 	/*
2933 	 * If we are about to change the interrupt vector for this interrupt,
2934 	 * and this interrupt is level-triggered, attached to an IOAPIC,
2935 	 * has been delivered to a CPU and that CPU has not handled it
2936 	 * yet, we cannot reprogram the IOAPIC now.
2937 	 */
2938 	if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
2939 
2940 		rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex,
2941 		    intin_no);
2942 
2943 		if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) &&
2944 		    apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu,
2945 		    bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) {
2946 
2947 			return (0);
2948 		}
2949 
2950 		/*
2951 		 * NOTE: We do not unmask the RDT here, as an interrupt MAY
2952 		 * still come in before we have a chance to reprogram it below.
2953 		 * The reprogramming below will simultaneously change and
2954 		 * unmask the RDT entry.
2955 		 */
2956 
2957 		if ((uint32_t)bind_cpu == IRQ_UNBOUND) {
2958 			irdt.ir_lo =  AV_LDEST | AV_LOPRI |
2959 			    irq_ptr->airq_rdt_entry;
2960 #if !defined(__xpv)
2961 			irdt.ir_hi = AV_TOALL >> APIC_ID_BIT_OFFSET;
2962 
2963 			apic_vt_ops->apic_intrr_alloc_entry(irq_ptr);
2964 			apic_vt_ops->apic_intrr_map_entry(
2965 			    irq_ptr, (void *)&irdt);
2966 			apic_vt_ops->apic_intrr_record_rdt(irq_ptr, &irdt);
2967 
2968 			/* Write the RDT entry -- no specific CPU binding */
2969 			WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
2970 			    irdt.ir_hi | AV_TOALL);
2971 #else
2972 			WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
2973 			    AV_TOALL);
2974 #endif
2975 			if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu !=
2976 			    IRQ_UNBOUND)
2977 				apic_cpus[airq_temp_cpu].aci_temp_bound--;
2978 
2979 			/*
2980 			 * Write the vector, trigger, and polarity portion of
2981 			 * the RDT
2982 			 */
2983 			WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
2984 			    irdt.ir_lo);
2985 
2986 			irq_ptr->airq_temp_cpu = IRQ_UNBOUND;
2987 			return (0);
2988 		}
2989 	}
2990 
2991 	if (bind_cpu & IRQ_USER_BOUND) {
2992 		cpu_infop->aci_bound++;
2993 	} else {
2994 		cpu_infop->aci_temp_bound++;
2995 	}
2996 	ASSERT((bind_cpu & ~IRQ_USER_BOUND) < apic_nproc);
2997 
2998 	if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) {
2999 		apic_cpus[airq_temp_cpu].aci_temp_bound--;
3000 	}
3001 	if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
3002 
3003 		irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry;
3004 		irdt.ir_hi = cpu_infop->aci_local_id;
3005 
3006 #if !defined(__xpv)
3007 		apic_vt_ops->apic_intrr_alloc_entry(irq_ptr);
3008 		apic_vt_ops->apic_intrr_map_entry(irq_ptr, (void *)&irdt);
3009 		apic_vt_ops->apic_intrr_record_rdt(irq_ptr, &irdt);
3010 
3011 		/* Write the RDT entry -- bind to a specific CPU: */
3012 		WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
3013 		    irdt.ir_hi);
3014 #else
3015 		/* Write the RDT entry -- bind to a specific CPU: */
3016 		WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
3017 		    irdt.ir_hi << APIC_ID_BIT_OFFSET);
3018 #endif
3019 		/* Write the vector, trigger, and polarity portion of the RDT */
3020 		WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
3021 		    irdt.ir_lo);
3022 
3023 	} else {
3024 		int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ?
3025 		    DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX;
3026 		if (type == DDI_INTR_TYPE_MSI) {
3027 			if (irq_ptr->airq_ioapicindex ==
3028 			    irq_ptr->airq_origirq) {
3029 				/* first one */
3030 				DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
3031 				    "apic_pci_msi_enable_vector\n"));
3032 				apic_pci_msi_enable_vector(irq_ptr,
3033 				    type, which_irq, irq_ptr->airq_vector,
3034 				    irq_ptr->airq_intin_no,
3035 				    cpu_infop->aci_local_id);
3036 			}
3037 			if ((irq_ptr->airq_ioapicindex +
3038 			    irq_ptr->airq_intin_no - 1) ==
3039 			    irq_ptr->airq_origirq) { /* last one */
3040 				DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
3041 				    "apic_pci_msi_enable_mode\n"));
3042 				apic_pci_msi_enable_mode(irq_ptr->airq_dip,
3043 				    type, which_irq);
3044 			}
3045 		} else { /* MSI-X */
3046 			apic_pci_msi_enable_vector(irq_ptr, type,
3047 			    irq_ptr->airq_origirq, irq_ptr->airq_vector, 1,
3048 			    cpu_infop->aci_local_id);
3049 			apic_pci_msi_enable_mode(irq_ptr->airq_dip, type,
3050 			    irq_ptr->airq_origirq);
3051 		}
3052 	}
3053 	irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu;
3054 	apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND));
3055 	return (0);
3056 }
3057 
3058 static void
3059 apic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no)
3060 {
3061 	if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no)
3062 	    & AV_REMOTE_IRR) != 0) {
3063 		/*
3064 		 * Trying to clear the bit through normal
3065 		 * channels has failed.  So as a last-ditch
3066 		 * effort, try to set the trigger mode to
3067 		 * edge, then to level.  This has been
3068 		 * observed to work on many systems.
3069 		 */
3070 		WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3071 		    intin_no,
3072 		    READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3073 		    intin_no) & ~AV_LEVEL);
3074 
3075 		WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3076 		    intin_no,
3077 		    READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3078 		    intin_no) | AV_LEVEL);
3079 
3080 		/*
3081 		 * If the bit's STILL set, this interrupt may
3082 		 * be hosed.
3083 		 */
3084 		if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3085 		    intin_no) & AV_REMOTE_IRR) != 0) {
3086 
3087 			prom_printf("%s: Remote IRR still "
3088 			    "not clear for IOAPIC %d intin %d.\n"
3089 			    "\tInterrupts to this pin may cease "
3090 			    "functioning.\n", psm_name, ioapic_ix,
3091 			    intin_no);
3092 #ifdef DEBUG
3093 			apic_last_ditch_reprogram_failures++;
3094 #endif
3095 		}
3096 	}
3097 }
3098 
3099 /*
3100  * This function is protected by apic_ioapic_lock coupled with the
3101  * fact that interrupts are disabled.
3102  */
3103 static void
3104 delete_defer_repro_ent(int which_irq)
3105 {
3106 	ASSERT(which_irq >= 0);
3107 	ASSERT(which_irq <= 255);
3108 
3109 	if (apic_reprogram_info[which_irq].done)
3110 		return;
3111 
3112 	apic_reprogram_info[which_irq].done = B_TRUE;
3113 
3114 #ifdef DEBUG
3115 	apic_defer_repro_total_retries +=
3116 	    apic_reprogram_info[which_irq].tries;
3117 
3118 	apic_defer_repro_successes++;
3119 #endif
3120 
3121 	if (--apic_reprogram_outstanding == 0) {
3122 
3123 		setlvlx = psm_intr_exit_fn();
3124 	}
3125 }
3126 
3127 
3128 /*
3129  * Interrupts must be disabled during this function to prevent
3130  * self-deadlock.  Interrupts are disabled because this function
3131  * is called from apic_check_stuck_interrupt(), which is called
3132  * from apic_rebind(), which requires its caller to disable interrupts.
3133  */
3134 static void
3135 add_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu)
3136 {
3137 	ASSERT(which_irq >= 0);
3138 	ASSERT(which_irq <= 255);
3139 
3140 	/*
3141 	 * On the off-chance that there's already a deferred
3142 	 * reprogramming on this irq, check, and if so, just update the
3143 	 * CPU and irq pointer to which the interrupt is targeted, then return.
3144 	 */
3145 	if (!apic_reprogram_info[which_irq].done) {
3146 		apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
3147 		apic_reprogram_info[which_irq].irqp = irq_ptr;
3148 		return;
3149 	}
3150 
3151 	apic_reprogram_info[which_irq].irqp = irq_ptr;
3152 	apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
3153 	apic_reprogram_info[which_irq].tries = 0;
3154 	/*
3155 	 * This must be the last thing set, since we're not
3156 	 * grabbing any locks, apic_try_deferred_reprogram() will
3157 	 * make its decision about using this entry iff done
3158 	 * is false.
3159 	 */
3160 	apic_reprogram_info[which_irq].done = B_FALSE;
3161 
3162 	/*
3163 	 * If there were previously no deferred reprogrammings, change
3164 	 * setlvlx to call apic_try_deferred_reprogram()
3165 	 */
3166 	if (++apic_reprogram_outstanding == 1) {
3167 
3168 		setlvlx = apic_try_deferred_reprogram;
3169 	}
3170 }
3171 
3172 static void
3173 apic_try_deferred_reprogram(int prev_ipl, int irq)
3174 {
3175 	int reproirq;
3176 	ulong_t iflag;
3177 	struct ioapic_reprogram_data *drep;
3178 
3179 	(*psm_intr_exit_fn())(prev_ipl, irq);
3180 
3181 	if (!lock_try(&apic_defer_reprogram_lock)) {
3182 		return;
3183 	}
3184 
3185 	/*
3186 	 * Acquire the apic_ioapic_lock so that any other operations that
3187 	 * may affect the apic_reprogram_info state are serialized.
3188 	 * It's still possible for the last deferred reprogramming to clear
3189 	 * between the time we entered this function and the time we get to
3190 	 * the for loop below.  In that case, *setlvlx will have been set
3191 	 * back to *_intr_exit and drep will be NULL. (There's no way to
3192 	 * stop that from happening -- we would need to grab a lock before
3193 	 * calling *setlvlx, which is neither realistic nor prudent).
3194 	 */
3195 	iflag = intr_clear();
3196 	lock_set(&apic_ioapic_lock);
3197 
3198 	/*
3199 	 * For each deferred RDT entry, try to reprogram it now.  Note that
3200 	 * there is no lock acquisition to read apic_reprogram_info because
3201 	 * '.done' is set only after the other fields in the structure are set.
3202 	 */
3203 
3204 	drep = NULL;
3205 	for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) {
3206 		if (apic_reprogram_info[reproirq].done == B_FALSE) {
3207 			drep = &apic_reprogram_info[reproirq];
3208 			break;
3209 		}
3210 	}
3211 
3212 	/*
3213 	 * Either we found a deferred action to perform, or
3214 	 * we entered this function spuriously, after *setlvlx
3215 	 * was restored to point to *_intr_exit.  Any other
3216 	 * permutation is invalid.
3217 	 */
3218 	ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn());
3219 
3220 	/*
3221 	 * Though we can't really do anything about errors
3222 	 * at this point, keep track of them for reporting.
3223 	 * Note that it is very possible for apic_setup_io_intr
3224 	 * to re-register this very timeout if the Remote IRR bit
3225 	 * has not yet cleared.
3226 	 */
3227 
3228 #ifdef DEBUG
3229 	if (drep != NULL) {
3230 		if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) {
3231 			apic_deferred_setup_failures++;
3232 		}
3233 	} else {
3234 		apic_deferred_spurious_enters++;
3235 	}
3236 #else
3237 	if (drep != NULL)
3238 		(void) apic_setup_io_intr(drep, reproirq, B_TRUE);
3239 #endif
3240 
3241 	lock_clear(&apic_ioapic_lock);
3242 	intr_restore(iflag);
3243 
3244 	lock_clear(&apic_defer_reprogram_lock);
3245 }
3246 
3247 static void
3248 apic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no)
3249 {
3250 	int waited;
3251 
3252 	/*
3253 	 * Wait for the delivery pending bit to clear.
3254 	 */
3255 	if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
3256 	    (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) {
3257 
3258 		/*
3259 		 * If we're still waiting on the delivery of this interrupt,
3260 		 * continue to wait here until it is delivered (this should be
3261 		 * a very small amount of time, but include a timeout just in
3262 		 * case).
3263 		 */
3264 		for (waited = 0; waited < apic_max_reps_clear_pending;
3265 		    waited++) {
3266 			if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3267 			    intin_no) & AV_PENDING) == 0) {
3268 				break;
3269 			}
3270 		}
3271 	}
3272 }
3273 
3274 
3275 /*
3276  * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
3277  * bit set.  Calls functions that modify the function that setlvlx points to,
3278  * so that the reprogramming can be retried very shortly.
3279  *
3280  * This function will mask the RDT entry if the interrupt is level-triggered.
3281  * (The caller is responsible for unmasking the RDT entry.)
3282  *
3283  * Returns non-zero if the caller should defer IOAPIC reprogramming.
3284  */
3285 static int
3286 apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
3287     int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq,
3288     struct ioapic_reprogram_data *drep)
3289 {
3290 	int32_t			rdt_entry;
3291 	int			waited;
3292 	int			reps = 0;
3293 
3294 	/*
3295 	 * Wait for the delivery pending bit to clear.
3296 	 */
3297 	do {
3298 		++reps;
3299 
3300 		apic_ioapic_wait_pending_clear(ioapic_ix, intin_no);
3301 
3302 		/*
3303 		 * Mask the RDT entry, but only if it's a level-triggered
3304 		 * interrupt
3305 		 */
3306 		rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3307 		    intin_no);
3308 		if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) {
3309 
3310 			/* Mask it */
3311 			WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
3312 			    AV_MASK | rdt_entry);
3313 		}
3314 
3315 		if ((rdt_entry & AV_LEVEL) == AV_LEVEL) {
3316 			/*
3317 			 * If there was a race and an interrupt was injected
3318 			 * just before we masked, check for that case here.
3319 			 * Then, unmask the RDT entry and try again.  If we're
3320 			 * on our last try, don't unmask (because we want the
3321 			 * RDT entry to remain masked for the rest of the
3322 			 * function).
3323 			 */
3324 			rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3325 			    intin_no);
3326 			if ((rdt_entry & AV_PENDING) &&
3327 			    (reps < apic_max_reps_clear_pending)) {
3328 				/* Unmask it */
3329 				WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3330 				    intin_no, rdt_entry & ~AV_MASK);
3331 			}
3332 		}
3333 
3334 	} while ((rdt_entry & AV_PENDING) &&
3335 	    (reps < apic_max_reps_clear_pending));
3336 
3337 #ifdef DEBUG
3338 		if (rdt_entry & AV_PENDING)
3339 			apic_intr_deliver_timeouts++;
3340 #endif
3341 
3342 	/*
3343 	 * If the remote IRR bit is set, then the interrupt has been sent
3344 	 * to a CPU for processing.  We have no choice but to wait for
3345 	 * that CPU to process the interrupt, at which point the remote IRR
3346 	 * bit will be cleared.
3347 	 */
3348 	if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
3349 	    (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) {
3350 
3351 		/*
3352 		 * If the CPU that this RDT is bound to is NOT the current
3353 		 * CPU, wait until that CPU handles the interrupt and ACKs
3354 		 * it.  If this interrupt is not bound to any CPU (that is,
3355 		 * if it's bound to the logical destination of "anyone"), it
3356 		 * may have been delivered to the current CPU so handle that
3357 		 * case by deferring the reprogramming (below).
3358 		 */
3359 		if ((old_bind_cpu != IRQ_UNBOUND) &&
3360 		    (old_bind_cpu != IRQ_UNINIT) &&
3361 		    (old_bind_cpu != psm_get_cpu_id())) {
3362 			for (waited = 0; waited < apic_max_reps_clear_pending;
3363 			    waited++) {
3364 				if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
3365 				    intin_no) & AV_REMOTE_IRR) == 0) {
3366 
3367 					delete_defer_repro_ent(which_irq);
3368 
3369 					/* Remote IRR has cleared! */
3370 					return (0);
3371 				}
3372 			}
3373 		}
3374 
3375 		/*
3376 		 * If we waited and the Remote IRR bit is still not cleared,
3377 		 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
3378 		 * times for this interrupt, try the last-ditch workaround:
3379 		 */
3380 		if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) {
3381 
3382 			apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no);
3383 
3384 			/* Mark this one as reprogrammed: */
3385 			delete_defer_repro_ent(which_irq);
3386 
3387 			return (0);
3388 		} else {
3389 #ifdef DEBUG
3390 			apic_intr_deferrals++;
3391 #endif
3392 
3393 			/*
3394 			 * If waiting for the Remote IRR bit (above) didn't
3395 			 * allow it to clear, defer the reprogramming.
3396 			 * Add a new deferred-programming entry if the
3397 			 * caller passed a NULL one (and update the existing one
3398 			 * in case anything changed).
3399 			 */
3400 			add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu);
3401 			if (drep)
3402 				drep->tries++;
3403 
3404 			/* Inform caller to defer IOAPIC programming: */
3405 			return (1);
3406 		}
3407 
3408 	}
3409 
3410 	/* Remote IRR is clear */
3411 	delete_defer_repro_ent(which_irq);
3412 
3413 	return (0);
3414 }
3415 
3416 /*
3417  * Called to migrate all interrupts at an irq to another cpu.
3418  * Must be called with interrupts disabled and apic_ioapic_lock held
3419  */
3420 int
3421 apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu)
3422 {
3423 	apic_irq_t	*irqptr = irq_ptr;
3424 	int		retval = 0;
3425 
3426 	while (irqptr) {
3427 		if (irqptr->airq_temp_cpu != IRQ_UNINIT)
3428 			retval |= apic_rebind(irqptr, bind_cpu, NULL);
3429 		irqptr = irqptr->airq_next;
3430 	}
3431 
3432 	return (retval);
3433 }
3434 
3435 /*
3436  * apic_intr_redistribute does all the messy computations for identifying
3437  * which interrupt to move to which CPU. Currently we do just one interrupt
3438  * at a time. This reduces the time we spent doing all this within clock
3439  * interrupt. When it is done in idle, we could do more than 1.
3440  * First we find the most busy and the most free CPU (time in ISR only)
3441  * skipping those CPUs that has been identified as being ineligible (cpu_skip)
3442  * Then we look for IRQs which are closest to the difference between the
3443  * most busy CPU and the average ISR load. We try to find one whose load
3444  * is less than difference.If none exists, then we chose one larger than the
3445  * difference, provided it does not make the most idle CPU worse than the
3446  * most busy one. In the end, we clear all the busy fields for CPUs. For
3447  * IRQs, they are cleared as they are scanned.
3448  */
3449 void
3450 apic_intr_redistribute()
3451 {
3452 	int busiest_cpu, most_free_cpu;
3453 	int cpu_free, cpu_busy, max_busy, min_busy;
3454 	int min_free, diff;
3455 	int average_busy, cpus_online;
3456 	int i, busy;
3457 	ulong_t iflag;
3458 	apic_cpus_info_t *cpu_infop;
3459 	apic_irq_t *min_busy_irq = NULL;
3460 	apic_irq_t *max_busy_irq = NULL;
3461 
3462 	busiest_cpu = most_free_cpu = -1;
3463 	cpu_free = cpu_busy = max_busy = average_busy = 0;
3464 	min_free = apic_sample_factor_redistribution;
3465 	cpus_online = 0;
3466 	/*
3467 	 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
3468 	 * without ioapic_lock. That is OK as we are just doing statistical
3469 	 * sampling anyway and any inaccuracy now will get corrected next time
3470 	 * The call to rebind which actually changes things will make sure
3471 	 * we are consistent.
3472 	 */
3473 	for (i = 0; i < apic_nproc; i++) {
3474 		if (!(apic_redist_cpu_skip & (1 << i)) &&
3475 		    (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) {
3476 
3477 			cpu_infop = &apic_cpus[i];
3478 			/*
3479 			 * If no unbound interrupts or only 1 total on this
3480 			 * CPU, skip
3481 			 */
3482 			if (!cpu_infop->aci_temp_bound ||
3483 			    (cpu_infop->aci_bound + cpu_infop->aci_temp_bound)
3484 			    == 1) {
3485 				apic_redist_cpu_skip |= 1 << i;
3486 				continue;
3487 			}
3488 
3489 			busy = cpu_infop->aci_busy;
3490 			average_busy += busy;
3491 			cpus_online++;
3492 			if (max_busy < busy) {
3493 				max_busy = busy;
3494 				busiest_cpu = i;
3495 			}
3496 			if (min_free > busy) {
3497 				min_free = busy;
3498 				most_free_cpu = i;
3499 			}
3500 			if (busy > apic_int_busy_mark) {
3501 				cpu_busy |= 1 << i;
3502 			} else {
3503 				if (busy < apic_int_free_mark)
3504 					cpu_free |= 1 << i;
3505 			}
3506 		}
3507 	}
3508 	if ((cpu_busy && cpu_free) ||
3509 	    (max_busy >= (min_free + apic_diff_for_redistribution))) {
3510 
3511 		apic_num_imbalance++;
3512 #ifdef	DEBUG
3513 		if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
3514 			prom_printf(
3515 			    "redistribute busy=%x free=%x max=%x min=%x",
3516 			    cpu_busy, cpu_free, max_busy, min_free);
3517 		}
3518 #endif /* DEBUG */
3519 
3520 
3521 		average_busy /= cpus_online;
3522 
3523 		diff = max_busy - average_busy;
3524 		min_busy = max_busy; /* start with the max possible value */
3525 		max_busy = 0;
3526 		min_busy_irq = max_busy_irq = NULL;
3527 		i = apic_min_device_irq;
3528 		for (; i <= apic_max_device_irq; i++) {
3529 			apic_irq_t *irq_ptr;
3530 			/* Change to linked list per CPU ? */
3531 			if ((irq_ptr = apic_irq_table[i]) == NULL)
3532 				continue;
3533 			/* Check for irq_busy & decide which one to move */
3534 			/* Also zero them for next round */
3535 			if ((irq_ptr->airq_temp_cpu == busiest_cpu) &&
3536 			    irq_ptr->airq_busy) {
3537 				if (irq_ptr->airq_busy < diff) {
3538 					/*
3539 					 * Check for least busy CPU,
3540 					 * best fit or what ?
3541 					 */
3542 					if (max_busy < irq_ptr->airq_busy) {
3543 						/*
3544 						 * Most busy within the
3545 						 * required differential
3546 						 */
3547 						max_busy = irq_ptr->airq_busy;
3548 						max_busy_irq = irq_ptr;
3549 					}
3550 				} else {
3551 					if (min_busy > irq_ptr->airq_busy) {
3552 						/*
3553 						 * least busy, but more than
3554 						 * the reqd diff
3555 						 */
3556 						if (min_busy <
3557 						    (diff + average_busy -
3558 						    min_free)) {
3559 							/*
3560 							 * Making sure new cpu
3561 							 * will not end up
3562 							 * worse
3563 							 */
3564 							min_busy =
3565 							    irq_ptr->airq_busy;
3566 
3567 							min_busy_irq = irq_ptr;
3568 						}
3569 					}
3570 				}
3571 			}
3572 			irq_ptr->airq_busy = 0;
3573 		}
3574 
3575 		if (max_busy_irq != NULL) {
3576 #ifdef	DEBUG
3577 			if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
3578 				prom_printf("rebinding %x to %x",
3579 				    max_busy_irq->airq_vector, most_free_cpu);
3580 			}
3581 #endif /* DEBUG */
3582 			iflag = intr_clear();
3583 			if (lock_try(&apic_ioapic_lock)) {
3584 				if (apic_rebind_all(max_busy_irq,
3585 				    most_free_cpu) == 0) {
3586 					/* Make change permenant */
3587 					max_busy_irq->airq_cpu =
3588 					    (uint32_t)most_free_cpu;
3589 				}
3590 				lock_clear(&apic_ioapic_lock);
3591 			}
3592 			intr_restore(iflag);
3593 
3594 		} else if (min_busy_irq != NULL) {
3595 #ifdef	DEBUG
3596 			if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
3597 				prom_printf("rebinding %x to %x",
3598 				    min_busy_irq->airq_vector, most_free_cpu);
3599 			}
3600 #endif /* DEBUG */
3601 
3602 			iflag = intr_clear();
3603 			if (lock_try(&apic_ioapic_lock)) {
3604 				if (apic_rebind_all(min_busy_irq,
3605 				    most_free_cpu) == 0) {
3606 					/* Make change permenant */
3607 					min_busy_irq->airq_cpu =
3608 					    (uint32_t)most_free_cpu;
3609 				}
3610 				lock_clear(&apic_ioapic_lock);
3611 			}
3612 			intr_restore(iflag);
3613 
3614 		} else {
3615 			if (cpu_busy != (1 << busiest_cpu)) {
3616 				apic_redist_cpu_skip |= 1 << busiest_cpu;
3617 				/*
3618 				 * We leave cpu_skip set so that next time we
3619 				 * can choose another cpu
3620 				 */
3621 			}
3622 		}
3623 		apic_num_rebind++;
3624 	} else {
3625 		/*
3626 		 * found nothing. Could be that we skipped over valid CPUs
3627 		 * or we have balanced everything. If we had a variable
3628 		 * ticks_for_redistribution, it could be increased here.
3629 		 * apic_int_busy, int_free etc would also need to be
3630 		 * changed.
3631 		 */
3632 		if (apic_redist_cpu_skip)
3633 			apic_redist_cpu_skip = 0;
3634 	}
3635 	for (i = 0; i < apic_nproc; i++) {
3636 		apic_cpus[i].aci_busy = 0;
3637 	}
3638 }
3639 
3640 void
3641 apic_cleanup_busy()
3642 {
3643 	int i;
3644 	apic_irq_t *irq_ptr;
3645 
3646 	for (i = 0; i < apic_nproc; i++) {
3647 		apic_cpus[i].aci_busy = 0;
3648 	}
3649 
3650 	for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
3651 		if ((irq_ptr = apic_irq_table[i]) != NULL)
3652 			irq_ptr->airq_busy = 0;
3653 	}
3654 }
3655 
3656 
3657 static int
3658 apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
3659     int ipin, int *pci_irqp, iflag_t *intr_flagp)
3660 {
3661 
3662 	int status;
3663 	acpi_psm_lnk_t acpipsmlnk;
3664 
3665 	if ((status = acpi_get_irq_cache_ent(busid, devid, ipin, pci_irqp,
3666 	    intr_flagp)) == ACPI_PSM_SUCCESS) {
3667 		APIC_VERBOSE_IRQ((CE_CONT, "!%s: Found irqno %d "
3668 		    "from cache for device %s, instance #%d\n", psm_name,
3669 		    *pci_irqp, ddi_get_name(dip), ddi_get_instance(dip)));
3670 		return (status);
3671 	}
3672 
3673 	bzero(&acpipsmlnk, sizeof (acpi_psm_lnk_t));
3674 
3675 	if ((status = acpi_translate_pci_irq(dip, ipin, pci_irqp, intr_flagp,
3676 	    &acpipsmlnk)) == ACPI_PSM_FAILURE) {
3677 		APIC_VERBOSE_IRQ((CE_WARN, "%s: "
3678 		    " acpi_translate_pci_irq failed for device %s, instance"
3679 		    " #%d", psm_name, ddi_get_name(dip),
3680 		    ddi_get_instance(dip)));
3681 		return (status);
3682 	}
3683 
3684 	if (status == ACPI_PSM_PARTIAL && acpipsmlnk.lnkobj != NULL) {
3685 		status = apic_acpi_irq_configure(&acpipsmlnk, dip, pci_irqp,
3686 		    intr_flagp);
3687 		if (status != ACPI_PSM_SUCCESS) {
3688 			status = acpi_get_current_irq_resource(&acpipsmlnk,
3689 			    pci_irqp, intr_flagp);
3690 		}
3691 	}
3692 
3693 	if (status == ACPI_PSM_SUCCESS) {
3694 		acpi_new_irq_cache_ent(busid, devid, ipin, *pci_irqp,
3695 		    intr_flagp, &acpipsmlnk);
3696 
3697 		APIC_VERBOSE_IRQ((CE_CONT, "%s: [ACPI] "
3698 		    "new irq %d for device %s, instance #%d\n", psm_name,
3699 		    *pci_irqp, ddi_get_name(dip), ddi_get_instance(dip)));
3700 	}
3701 
3702 	return (status);
3703 }
3704 
3705 /*
3706  * Adds an entry to the irq list passed in, and returns the new list.
3707  * Entries are added in priority order (lower numerical priorities are
3708  * placed closer to the head of the list)
3709  */
3710 static prs_irq_list_t *
3711 acpi_insert_prs_irq_ent(prs_irq_list_t *listp, int priority, int irq,
3712     iflag_t *iflagp, acpi_prs_private_t *prsprvp)
3713 {
3714 	struct prs_irq_list_ent *newent, *prevp = NULL, *origlistp;
3715 
3716 	newent = kmem_zalloc(sizeof (struct prs_irq_list_ent), KM_SLEEP);
3717 
3718 	newent->list_prio = priority;
3719 	newent->irq = irq;
3720 	newent->intrflags = *iflagp;
3721 	newent->prsprv = *prsprvp;
3722 	/* ->next is NULL from kmem_zalloc */
3723 
3724 	/*
3725 	 * New list -- return the new entry as the list.
3726 	 */
3727 	if (listp == NULL)
3728 		return (newent);
3729 
3730 	/*
3731 	 * Save original list pointer for return (since we're not modifying
3732 	 * the head)
3733 	 */
3734 	origlistp = listp;
3735 
3736 	/*
3737 	 * Insertion sort, with entries with identical keys stored AFTER
3738 	 * existing entries (the less-than-or-equal test of priority does
3739 	 * this for us).
3740 	 */
3741 	while (listp != NULL && listp->list_prio <= priority) {
3742 		prevp = listp;
3743 		listp = listp->next;
3744 	}
3745 
3746 	newent->next = listp;
3747 
3748 	if (prevp == NULL) { /* Add at head of list (newent is the new head) */
3749 		return (newent);
3750 	} else {
3751 		prevp->next = newent;
3752 		return (origlistp);
3753 	}
3754 }
3755 
3756 /*
3757  * Frees the list passed in, deallocating all memory and leaving *listpp
3758  * set to NULL.
3759  */
3760 static void
3761 acpi_destroy_prs_irq_list(prs_irq_list_t **listpp)
3762 {
3763 	struct prs_irq_list_ent *nextp;
3764 
3765 	ASSERT(listpp != NULL);
3766 
3767 	while (*listpp != NULL) {
3768 		nextp = (*listpp)->next;
3769 		kmem_free(*listpp, sizeof (struct prs_irq_list_ent));
3770 		*listpp = nextp;
3771 	}
3772 }
3773 
3774 /*
3775  * apic_choose_irqs_from_prs returns a list of irqs selected from the list of
3776  * irqs returned by the link device's _PRS method.  The irqs are chosen
3777  * to minimize contention in situations where the interrupt link device
3778  * can be programmed to steer interrupts to different interrupt controller
3779  * inputs (some of which may already be in use).  The list is sorted in order
3780  * of irqs to use, with the highest priority given to interrupt controller
3781  * inputs that are not shared.   When an interrupt controller input
3782  * must be shared, apic_choose_irqs_from_prs adds the possible irqs to the
3783  * returned list in the order that minimizes sharing (thereby ensuring lowest
3784  * possible latency from interrupt trigger time to ISR execution time).
3785  */
3786 static prs_irq_list_t *
3787 apic_choose_irqs_from_prs(acpi_irqlist_t *irqlistent, dev_info_t *dip,
3788     int crs_irq)
3789 {
3790 	int32_t irq;
3791 	int i;
3792 	prs_irq_list_t *prsirqlistp = NULL;
3793 	iflag_t iflags;
3794 
3795 	while (irqlistent != NULL) {
3796 		irqlistent->intr_flags.bustype = BUS_PCI;
3797 
3798 		for (i = 0; i < irqlistent->num_irqs; i++) {
3799 
3800 			irq = irqlistent->irqs[i];
3801 
3802 			if (irq <= 0) {
3803 				/* invalid irq number */
3804 				continue;
3805 			}
3806 
3807 			if ((irq < 16) && (apic_reserved_irqlist[irq]))
3808 				continue;
3809 
3810 			if ((apic_irq_table[irq] == NULL) ||
3811 			    (apic_irq_table[irq]->airq_dip == dip)) {
3812 
3813 				prsirqlistp = acpi_insert_prs_irq_ent(
3814 				    prsirqlistp, 0 /* Highest priority */, irq,
3815 				    &irqlistent->intr_flags,
3816 				    &irqlistent->acpi_prs_prv);
3817 
3818 				/*
3819 				 * If we do not prefer the current irq from _CRS
3820 				 * or if we do and this irq is the same as the
3821 				 * current irq from _CRS, this is the one
3822 				 * to pick.
3823 				 */
3824 				if (!(apic_prefer_crs) || (irq == crs_irq)) {
3825 					return (prsirqlistp);
3826 				}
3827 				continue;
3828 			}
3829 
3830 			/*
3831 			 * Edge-triggered interrupts cannot be shared
3832 			 */
3833 			if (irqlistent->intr_flags.intr_el == INTR_EL_EDGE)
3834 				continue;
3835 
3836 			/*
3837 			 * To work around BIOSes that contain incorrect
3838 			 * interrupt polarity information in interrupt
3839 			 * descriptors returned by _PRS, we assume that
3840 			 * the polarity of the other device sharing this
3841 			 * interrupt controller input is compatible.
3842 			 * If it's not, the caller will catch it when
3843 			 * the caller invokes the link device's _CRS method
3844 			 * (after invoking its _SRS method).
3845 			 */
3846 			iflags = irqlistent->intr_flags;
3847 			iflags.intr_po =
3848 			    apic_irq_table[irq]->airq_iflag.intr_po;
3849 
3850 			if (!acpi_intr_compatible(iflags,
3851 			    apic_irq_table[irq]->airq_iflag)) {
3852 				APIC_VERBOSE_IRQ((CE_CONT, "!%s: irq %d "
3853 				    "not compatible [%x:%x:%x !~ %x:%x:%x]",
3854 				    psm_name, irq,
3855 				    iflags.intr_po,
3856 				    iflags.intr_el,
3857 				    iflags.bustype,
3858 				    apic_irq_table[irq]->airq_iflag.intr_po,
3859 				    apic_irq_table[irq]->airq_iflag.intr_el,
3860 				    apic_irq_table[irq]->airq_iflag.bustype));
3861 				continue;
3862 			}
3863 
3864 			/*
3865 			 * If we prefer the irq from _CRS, no need
3866 			 * to search any further (and make sure
3867 			 * to add this irq with the highest priority
3868 			 * so it's tried first).
3869 			 */
3870 			if (crs_irq == irq && apic_prefer_crs) {
3871 
3872 				return (acpi_insert_prs_irq_ent(
3873 				    prsirqlistp,
3874 				    0 /* Highest priority */,
3875 				    irq, &iflags,
3876 				    &irqlistent->acpi_prs_prv));
3877 			}
3878 
3879 			/*
3880 			 * Priority is equal to the share count (lower
3881 			 * share count is higher priority). Note that
3882 			 * the intr flags passed in here are the ones we
3883 			 * changed above -- if incorrect, it will be
3884 			 * caught by the caller's _CRS flags comparison.
3885 			 */
3886 			prsirqlistp = acpi_insert_prs_irq_ent(
3887 			    prsirqlistp,
3888 			    apic_irq_table[irq]->airq_share, irq,
3889 			    &iflags, &irqlistent->acpi_prs_prv);
3890 		}
3891 
3892 		/* Go to the next irqlist entry */
3893 		irqlistent = irqlistent->next;
3894 	}
3895 
3896 	return (prsirqlistp);
3897 }
3898 
3899 /*
3900  * Configures the irq for the interrupt link device identified by
3901  * acpipsmlnkp.
3902  *
3903  * Gets the current and the list of possible irq settings for the
3904  * device. If apic_unconditional_srs is not set, and the current
3905  * resource setting is in the list of possible irq settings,
3906  * current irq resource setting is passed to the caller.
3907  *
3908  * Otherwise, picks an irq number from the list of possible irq
3909  * settings, and sets the irq of the device to this value.
3910  * If prefer_crs is set, among a set of irq numbers in the list that have
3911  * the least number of devices sharing the interrupt, we pick current irq
3912  * resource setting if it is a member of this set.
3913  *
3914  * Passes the irq number in the value pointed to by pci_irqp, and
3915  * polarity and sensitivity in the structure pointed to by dipintrflagp
3916  * to the caller.
3917  *
3918  * Note that if setting the irq resource failed, but successfuly obtained
3919  * the current irq resource settings, passes the current irq resources
3920  * and considers it a success.
3921  *
3922  * Returns:
3923  * ACPI_PSM_SUCCESS on success.
3924  *
3925  * ACPI_PSM_FAILURE if an error occured during the configuration or
3926  * if a suitable irq was not found for this device, or if setting the
3927  * irq resource and obtaining the current resource fails.
3928  *
3929  */
3930 static int
3931 apic_acpi_irq_configure(acpi_psm_lnk_t *acpipsmlnkp, dev_info_t *dip,
3932     int *pci_irqp, iflag_t *dipintr_flagp)
3933 {
3934 	int32_t irq;
3935 	int cur_irq = -1;
3936 	acpi_irqlist_t *irqlistp;
3937 	prs_irq_list_t *prs_irq_listp, *prs_irq_entp;
3938 	boolean_t found_irq = B_FALSE;
3939 
3940 	dipintr_flagp->bustype = BUS_PCI;
3941 
3942 	if ((acpi_get_possible_irq_resources(acpipsmlnkp, &irqlistp))
3943 	    == ACPI_PSM_FAILURE) {
3944 		APIC_VERBOSE_IRQ((CE_WARN, "!%s: Unable to determine "
3945 		    "or assign IRQ for device %s, instance #%d: The system was "
3946 		    "unable to get the list of potential IRQs from ACPI.",
3947 		    psm_name, ddi_get_name(dip), ddi_get_instance(dip)));
3948 
3949 		return (ACPI_PSM_FAILURE);
3950 	}
3951 
3952 	if ((acpi_get_current_irq_resource(acpipsmlnkp, &cur_irq,
3953 	    dipintr_flagp) == ACPI_PSM_SUCCESS) && (!apic_unconditional_srs) &&
3954 	    (cur_irq > 0)) {
3955 		/*
3956 		 * If an IRQ is set in CRS and that IRQ exists in the set
3957 		 * returned from _PRS, return that IRQ, otherwise print
3958 		 * a warning
3959 		 */
3960 
3961 		if (acpi_irqlist_find_irq(irqlistp, cur_irq, NULL)
3962 		    == ACPI_PSM_SUCCESS) {
3963 
3964 			ASSERT(pci_irqp != NULL);
3965 			*pci_irqp = cur_irq;
3966 			acpi_free_irqlist(irqlistp);
3967 			return (ACPI_PSM_SUCCESS);
3968 		}
3969 
3970 		APIC_VERBOSE_IRQ((CE_WARN, "!%s: Could not find the "
3971 		    "current irq %d for device %s, instance #%d in ACPI's "
3972 		    "list of possible irqs for this device. Picking one from "
3973 		    " the latter list.", psm_name, cur_irq, ddi_get_name(dip),
3974 		    ddi_get_instance(dip)));
3975 	}
3976 
3977 	if ((prs_irq_listp = apic_choose_irqs_from_prs(irqlistp, dip,
3978 	    cur_irq)) == NULL) {
3979 
3980 		APIC_VERBOSE_IRQ((CE_WARN, "!%s: Could not find a "
3981 		    "suitable irq from the list of possible irqs for device "
3982 		    "%s, instance #%d in ACPI's list of possible irqs",
3983 		    psm_name, ddi_get_name(dip), ddi_get_instance(dip)));
3984 
3985 		acpi_free_irqlist(irqlistp);
3986 		return (ACPI_PSM_FAILURE);
3987 	}
3988 
3989 	acpi_free_irqlist(irqlistp);
3990 
3991 	for (prs_irq_entp = prs_irq_listp;
3992 	    prs_irq_entp != NULL && found_irq == B_FALSE;
3993 	    prs_irq_entp = prs_irq_entp->next) {
3994 
3995 		acpipsmlnkp->acpi_prs_prv = prs_irq_entp->prsprv;
3996 		irq = prs_irq_entp->irq;
3997 
3998 		APIC_VERBOSE_IRQ((CE_CONT, "!%s: Setting irq %d for "
3999 		    "device %s instance #%d\n", psm_name, irq,
4000 		    ddi_get_name(dip), ddi_get_instance(dip)));
4001 
4002 		if ((acpi_set_irq_resource(acpipsmlnkp, irq))
4003 		    == ACPI_PSM_SUCCESS) {
4004 			/*
4005 			 * setting irq was successful, check to make sure CRS
4006 			 * reflects that. If CRS does not agree with what we
4007 			 * set, return the irq that was set.
4008 			 */
4009 
4010 			if (acpi_get_current_irq_resource(acpipsmlnkp, &cur_irq,
4011 			    dipintr_flagp) == ACPI_PSM_SUCCESS) {
4012 
4013 				if (cur_irq != irq)
4014 					APIC_VERBOSE_IRQ((CE_WARN,
4015 					    "!%s: IRQ resource set "
4016 					    "(irqno %d) for device %s "
4017 					    "instance #%d, differs from "
4018 					    "current setting irqno %d",
4019 					    psm_name, irq, ddi_get_name(dip),
4020 					    ddi_get_instance(dip), cur_irq));
4021 			} else {
4022 				/*
4023 				 * On at least one system, there was a bug in
4024 				 * a DSDT method called by _STA, causing _STA to
4025 				 * indicate that the link device was disabled
4026 				 * (when, in fact, it was enabled).  Since _SRS
4027 				 * succeeded, assume that _CRS is lying and use
4028 				 * the iflags from this _PRS interrupt choice.
4029 				 * If we're wrong about the flags, the polarity
4030 				 * will be incorrect and we may get an interrupt
4031 				 * storm, but there's not much else we can do
4032 				 * at this point.
4033 				 */
4034 				*dipintr_flagp = prs_irq_entp->intrflags;
4035 			}
4036 
4037 			/*
4038 			 * Return the irq that was set, and not what _CRS
4039 			 * reports, since _CRS has been seen to return
4040 			 * different IRQs than what was passed to _SRS on some
4041 			 * systems (and just not return successfully on others).
4042 			 */
4043 			cur_irq = irq;
4044 			found_irq = B_TRUE;
4045 		} else {
4046 			APIC_VERBOSE_IRQ((CE_WARN, "!%s: set resource "
4047 			    "irq %d failed for device %s instance #%d",
4048 			    psm_name, irq, ddi_get_name(dip),
4049 			    ddi_get_instance(dip)));
4050 
4051 			if (cur_irq == -1) {
4052 				acpi_destroy_prs_irq_list(&prs_irq_listp);
4053 				return (ACPI_PSM_FAILURE);
4054 			}
4055 		}
4056 	}
4057 
4058 	acpi_destroy_prs_irq_list(&prs_irq_listp);
4059 
4060 	if (!found_irq)
4061 		return (ACPI_PSM_FAILURE);
4062 
4063 	ASSERT(pci_irqp != NULL);
4064 	*pci_irqp = cur_irq;
4065 	return (ACPI_PSM_SUCCESS);
4066 }
4067 
4068 void
4069 ioapic_disable_redirection()
4070 {
4071 	int ioapic_ix;
4072 	int intin_max;
4073 	int intin_ix;
4074 
4075 	/* Disable the I/O APIC redirection entries */
4076 	for (ioapic_ix = 0; ioapic_ix < apic_io_max; ioapic_ix++) {
4077 
4078 		/* Bits 23-16 define the maximum redirection entries */
4079 		intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
4080 		    & 0xff;
4081 
4082 		for (intin_ix = 0; intin_ix <= intin_max; intin_ix++) {
4083 			/*
4084 			 * The assumption here is that this is safe, even for
4085 			 * systems with IOAPICs that suffer from the hardware
4086 			 * erratum because all devices have been quiesced before
4087 			 * this function is called from apic_shutdown()
4088 			 * (or equivalent). If that assumption turns out to be
4089 			 * false, this mask operation can induce the same
4090 			 * erratum result we're trying to avoid.
4091 			 */
4092 			ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin_ix,
4093 			    AV_MASK);
4094 		}
4095 	}
4096 }
4097 
4098 /*
4099  * Looks for an IOAPIC with the specified physical address in the /ioapics
4100  * node in the device tree (created by the PCI enumerator).
4101  */
4102 static boolean_t
4103 apic_is_ioapic_AMD_813x(uint32_t physaddr)
4104 {
4105 	/*
4106 	 * Look in /ioapics, for the ioapic with
4107 	 * the physical address given
4108 	 */
4109 	dev_info_t *ioapicsnode = ddi_find_devinfo(IOAPICS_NODE_NAME, -1, 0);
4110 	dev_info_t *ioapic_child;
4111 	boolean_t rv = B_FALSE;
4112 	int vid, did;
4113 	uint64_t ioapic_paddr;
4114 	boolean_t done = B_FALSE;
4115 
4116 	if (ioapicsnode == NULL)
4117 		return (B_FALSE);
4118 
4119 	/* Load first child: */
4120 	ioapic_child = ddi_get_child(ioapicsnode);
4121 	while (!done && ioapic_child != 0) { /* Iterate over children */
4122 
4123 		if ((ioapic_paddr = (uint64_t)ddi_prop_get_int64(DDI_DEV_T_ANY,
4124 		    ioapic_child, DDI_PROP_DONTPASS, "reg", 0))
4125 		    != 0 && physaddr == ioapic_paddr) {
4126 
4127 			vid = ddi_prop_get_int(DDI_DEV_T_ANY, ioapic_child,
4128 			    DDI_PROP_DONTPASS, IOAPICS_PROP_VENID, 0);
4129 
4130 			if (vid == VENID_AMD) {
4131 
4132 				did = ddi_prop_get_int(DDI_DEV_T_ANY,
4133 				    ioapic_child, DDI_PROP_DONTPASS,
4134 				    IOAPICS_PROP_DEVID, 0);
4135 
4136 				if (did == DEVID_8131_IOAPIC ||
4137 				    did == DEVID_8132_IOAPIC) {
4138 
4139 					rv = B_TRUE;
4140 					done = B_TRUE;
4141 				}
4142 			}
4143 		}
4144 
4145 		if (!done)
4146 			ioapic_child = ddi_get_next_sibling(ioapic_child);
4147 	}
4148 
4149 	/* The ioapics node was held by ddi_find_devinfo, so release it */
4150 	ndi_rele_devi(ioapicsnode);
4151 	return (rv);
4152 }
4153 
4154 struct apic_state {
4155 	int32_t as_task_reg;
4156 	int32_t as_dest_reg;
4157 	int32_t as_format_reg;
4158 	int32_t as_local_timer;
4159 	int32_t as_pcint_vect;
4160 	int32_t as_int_vect0;
4161 	int32_t as_int_vect1;
4162 	int32_t as_err_vect;
4163 	int32_t as_init_count;
4164 	int32_t as_divide_reg;
4165 	int32_t as_spur_int_reg;
4166 	uint32_t as_ioapic_ids[MAX_IO_APIC];
4167 };
4168 
4169 
4170 static int
4171 apic_acpi_enter_apicmode(void)
4172 {
4173 	ACPI_OBJECT_LIST	arglist;
4174 	ACPI_OBJECT		arg;
4175 	ACPI_STATUS		status;
4176 
4177 	/* Setup parameter object */
4178 	arglist.Count = 1;
4179 	arglist.Pointer = &arg;
4180 	arg.Type = ACPI_TYPE_INTEGER;
4181 	arg.Integer.Value = ACPI_APIC_MODE;
4182 
4183 	status = AcpiEvaluateObject(NULL, "\\_PIC", &arglist, NULL);
4184 	if (ACPI_FAILURE(status))
4185 		return (PSM_FAILURE);
4186 	else
4187 		return (PSM_SUCCESS);
4188 }
4189 
4190 
4191 static void
4192 apic_save_state(struct apic_state *sp)
4193 {
4194 	int	i;
4195 	ulong_t	iflag;
4196 
4197 	PMD(PMD_SX, ("apic_save_state %p\n", (void *)sp))
4198 	/*
4199 	 * First the local APIC.
4200 	 */
4201 	sp->as_task_reg = apic_reg_ops->apic_get_pri();
4202 	sp->as_dest_reg =  apic_reg_ops->apic_read(APIC_DEST_REG);
4203 	if (apic_mode == LOCAL_APIC)
4204 		sp->as_format_reg = apic_reg_ops->apic_read(APIC_FORMAT_REG);
4205 	sp->as_local_timer = apic_reg_ops->apic_read(APIC_LOCAL_TIMER);
4206 	sp->as_pcint_vect = apic_reg_ops->apic_read(APIC_PCINT_VECT);
4207 	sp->as_int_vect0 = apic_reg_ops->apic_read(APIC_INT_VECT0);
4208 	sp->as_int_vect1 = apic_reg_ops->apic_read(APIC_INT_VECT1);
4209 	sp->as_err_vect = apic_reg_ops->apic_read(APIC_ERR_VECT);
4210 	sp->as_init_count = apic_reg_ops->apic_read(APIC_INIT_COUNT);
4211 	sp->as_divide_reg = apic_reg_ops->apic_read(APIC_DIVIDE_REG);
4212 	sp->as_spur_int_reg = apic_reg_ops->apic_read(APIC_SPUR_INT_REG);
4213 
4214 	/*
4215 	 * If on the boot processor then save the IOAPICs' IDs
4216 	 */
4217 	if (psm_get_cpu_id() == 0) {
4218 
4219 		iflag = intr_clear();
4220 		lock_set(&apic_ioapic_lock);
4221 
4222 		for (i = 0; i < apic_io_max; i++)
4223 			sp->as_ioapic_ids[i] = ioapic_read(i, APIC_ID_CMD);
4224 
4225 		lock_clear(&apic_ioapic_lock);
4226 		intr_restore(iflag);
4227 	}
4228 }
4229 
4230 static void
4231 apic_restore_state(struct apic_state *sp)
4232 {
4233 	int	i;
4234 	ulong_t	iflag;
4235 
4236 	/*
4237 	 * First the local APIC.
4238 	 */
4239 	apic_reg_ops->apic_write_task_reg(sp->as_task_reg);
4240 	if (apic_mode == LOCAL_APIC) {
4241 		apic_reg_ops->apic_write(APIC_DEST_REG, sp->as_dest_reg);
4242 		apic_reg_ops->apic_write(APIC_FORMAT_REG, sp->as_format_reg);
4243 	}
4244 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER, sp->as_local_timer);
4245 	apic_reg_ops->apic_write(APIC_PCINT_VECT, sp->as_pcint_vect);
4246 	apic_reg_ops->apic_write(APIC_INT_VECT0, sp->as_int_vect0);
4247 	apic_reg_ops->apic_write(APIC_INT_VECT1, sp->as_int_vect1);
4248 	apic_reg_ops->apic_write(APIC_ERR_VECT, sp->as_err_vect);
4249 	apic_reg_ops->apic_write(APIC_INIT_COUNT, sp->as_init_count);
4250 	apic_reg_ops->apic_write(APIC_DIVIDE_REG, sp->as_divide_reg);
4251 	apic_reg_ops->apic_write(APIC_SPUR_INT_REG, sp->as_spur_int_reg);
4252 
4253 	/*
4254 	 * the following only needs to be done once, so we do it on the
4255 	 * boot processor, since we know that we only have one of those
4256 	 */
4257 	if (psm_get_cpu_id() == 0) {
4258 
4259 		iflag = intr_clear();
4260 		lock_set(&apic_ioapic_lock);
4261 
4262 		/* Restore IOAPICs' APIC IDs */
4263 		for (i = 0; i < apic_io_max; i++) {
4264 			ioapic_write(i, APIC_ID_CMD, sp->as_ioapic_ids[i]);
4265 		}
4266 
4267 		lock_clear(&apic_ioapic_lock);
4268 		intr_restore(iflag);
4269 
4270 		/*
4271 		 * Reenter APIC mode before restoring LNK devices
4272 		 */
4273 		(void) apic_acpi_enter_apicmode();
4274 
4275 		/*
4276 		 * restore acpi link device mappings
4277 		 */
4278 		acpi_restore_link_devices();
4279 	}
4280 }
4281 
4282 /*
4283  * Returns 0 on success
4284  */
4285 int
4286 apic_state(psm_state_request_t *rp)
4287 {
4288 	PMD(PMD_SX, ("apic_state "))
4289 	switch (rp->psr_cmd) {
4290 	case PSM_STATE_ALLOC:
4291 		rp->req.psm_state_req.psr_state =
4292 		    kmem_zalloc(sizeof (struct apic_state), KM_NOSLEEP);
4293 		if (rp->req.psm_state_req.psr_state == NULL)
4294 			return (ENOMEM);
4295 		rp->req.psm_state_req.psr_state_size =
4296 		    sizeof (struct apic_state);
4297 		PMD(PMD_SX, (":STATE_ALLOC: state %p, size %lx\n",
4298 		    rp->req.psm_state_req.psr_state,
4299 		    rp->req.psm_state_req.psr_state_size))
4300 		return (0);
4301 
4302 	case PSM_STATE_FREE:
4303 		kmem_free(rp->req.psm_state_req.psr_state,
4304 		    rp->req.psm_state_req.psr_state_size);
4305 		PMD(PMD_SX, (" STATE_FREE: state %p, size %lx\n",
4306 		    rp->req.psm_state_req.psr_state,
4307 		    rp->req.psm_state_req.psr_state_size))
4308 		return (0);
4309 
4310 	case PSM_STATE_SAVE:
4311 		PMD(PMD_SX, (" STATE_SAVE: state %p, size %lx\n",
4312 		    rp->req.psm_state_req.psr_state,
4313 		    rp->req.psm_state_req.psr_state_size))
4314 		apic_save_state(rp->req.psm_state_req.psr_state);
4315 		return (0);
4316 
4317 	case PSM_STATE_RESTORE:
4318 		apic_restore_state(rp->req.psm_state_req.psr_state);
4319 		PMD(PMD_SX, (" STATE_RESTORE: state %p, size %lx\n",
4320 		    rp->req.psm_state_req.psr_state,
4321 		    rp->req.psm_state_req.psr_state_size))
4322 		return (0);
4323 
4324 	default:
4325 		return (EINVAL);
4326 	}
4327 }
4328