xref: /titanic_44/usr/src/uts/sun4u/io/pci/pci_intr.c (revision e824d57f8160a27ac5e650005c7a4f037109c2be)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * PCI nexus interrupt handling:
30  *	PCI device interrupt handler wrapper
31  *	pil lookup routine
32  *	PCI device interrupt related initchild code
33  */
34 
35 #include <sys/types.h>
36 #include <sys/kmem.h>
37 #include <sys/async.h>
38 #include <sys/spl.h>
39 #include <sys/sunddi.h>
40 #include <sys/machsystm.h>	/* e_ddi_nodeid_to_dip() */
41 #include <sys/ddi_impldefs.h>
42 #include <sys/pci/pci_obj.h>
43 #include <sys/sdt.h>
44 #include <sys/clock.h>
45 
46 #ifdef _STARFIRE
47 #include <sys/starfire.h>
48 #endif /* _STARFIRE */
49 
50 /*
51  * interrupt jabber:
52  *
53  * When an interrupt line is jabbering, every time the state machine for the
54  * associated ino is idled, a new mondo will be sent and the ino will go into
55  * the pending state again. The mondo will cause a new call to
56  * pci_intr_wrapper() which normally idles the ino's state machine which would
57  * precipitate another trip round the loop.
58  * The loop can be broken by preventing the ino's state machine from being
59  * idled when an interrupt line is jabbering. See the comment at the
60  * beginning of pci_intr_wrapper() explaining how the 'interrupt jabber
61  * protection' code does this.
62  */
63 
64 /*LINTLIBRARY*/
65 
66 #ifdef NOT_DEFINED
67 /*
68  * This array is used to determine the sparc PIL at the which the
69  * handler for a given INO will execute.  This table is for onboard
70  * devices only.  A different scheme will be used for plug-in cards.
71  */
72 
73 uint_t ino_to_pil[] = {
74 
75 	/* pil */		/* ino */
76 
77 	0, 0, 0, 0,  		/* 0x00 - 0x03: bus A slot 0 int#A, B, C, D */
78 	0, 0, 0, 0,		/* 0x04 - 0x07: bus A slot 1 int#A, B, C, D */
79 	0, 0, 0, 0,  		/* 0x08 - 0x0B: unused */
80 	0, 0, 0, 0,		/* 0x0C - 0x0F: unused */
81 
82 	0, 0, 0, 0,  		/* 0x10 - 0x13: bus B slot 0 int#A, B, C, D */
83 	0, 0, 0, 0,		/* 0x14 - 0x17: bus B slot 1 int#A, B, C, D */
84 	0, 0, 0, 0,  		/* 0x18 - 0x1B: bus B slot 2 int#A, B, C, D */
85 	4, 0, 0, 0,		/* 0x1C - 0x1F: bus B slot 3 int#A, B, C, D */
86 
87 	4,			/* 0x20: SCSI */
88 	6,			/* 0x21: ethernet */
89 	3,			/* 0x22: parallel port */
90 	9,			/* 0x23: audio record */
91 	9,			/* 0x24: audio playback */
92 	14,			/* 0x25: power fail */
93 	4,			/* 0x26: 2nd SCSI */
94 	8,			/* 0x27: floppy */
95 	14,			/* 0x28: thermal warning */
96 	12,			/* 0x29: keyboard */
97 	12,			/* 0x2A: mouse */
98 	12,			/* 0x2B: serial */
99 	0,			/* 0x2C: timer/counter 0 */
100 	0,			/* 0x2D: timer/counter 1 */
101 	14,			/* 0x2E: uncorrectable ECC errors */
102 	14,			/* 0x2F: correctable ECC errors */
103 	14,			/* 0x30: PCI bus A error */
104 	14,			/* 0x31: PCI bus B error */
105 	14,			/* 0x32: power management wakeup */
106 	14,			/* 0x33 */
107 	14,			/* 0x34 */
108 	14,			/* 0x35 */
109 	14,			/* 0x36 */
110 	14,			/* 0x37 */
111 	14,			/* 0x38 */
112 	14,			/* 0x39 */
113 	14,			/* 0x3a */
114 	14,			/* 0x3b */
115 	14,			/* 0x3c */
116 	14,			/* 0x3d */
117 	14,			/* 0x3e */
118 	14,			/* 0x3f */
119 	14			/* 0x40 */
120 };
121 #endif /* NOT_DEFINED */
122 
123 
124 #define	PCI_SIMBA_VENID		0x108e	/* vendor id for simba */
125 #define	PCI_SIMBA_DEVID		0x5000	/* device id for simba */
126 
127 /*
128  * map_pcidev_cfg_reg - create mapping to pci device configuration registers
129  *			if we have a simba AND a pci to pci bridge along the
130  *			device path.
131  *			Called with corresponding mutexes held!!
132  *
133  * XXX	  XXX	XXX	The purpose of this routine is to overcome a hardware
134  *			defect in Sabre CPU and Simba bridge configuration
135  *			which does not drain DMA write data stalled in
136  *			PCI to PCI bridges (such as the DEC bridge) beyond
137  *			Simba. This routine will setup the data structures
138  *			to allow the pci_intr_wrapper to perform a manual
139  *			drain data operation before passing the control to
140  *			interrupt handlers of device drivers.
141  * return value:
142  * DDI_SUCCESS
143  * DDI_FAILURE		if unable to create mapping
144  */
145 static int
146 map_pcidev_cfg_reg(dev_info_t *dip, dev_info_t *rdip, ddi_acc_handle_t *hdl_p)
147 {
148 	dev_info_t *cdip;
149 	dev_info_t *pci_dip = NULL;
150 	pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
151 	int simba_found = 0, pci_bridge_found = 0;
152 
153 	for (cdip = rdip; cdip && cdip != dip; cdip = ddi_get_parent(cdip)) {
154 		ddi_acc_handle_t config_handle;
155 		uint32_t vendor_id = ddi_getprop(DDI_DEV_T_ANY, cdip,
156 			DDI_PROP_DONTPASS, "vendor-id", 0xffff);
157 
158 		DEBUG4(DBG_A_INTX, pci_p->pci_dip,
159 			"map dev cfg reg for %s%d: @%s%d\n",
160 			ddi_driver_name(rdip), ddi_get_instance(rdip),
161 			ddi_driver_name(cdip), ddi_get_instance(cdip));
162 
163 		if (ddi_prop_exists(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
164 				"no-dma-interrupt-sync"))
165 			continue;
166 
167 		/* continue to search up-stream if not a PCI device */
168 		if (vendor_id == 0xffff)
169 			continue;
170 
171 		/* record the deepest pci device */
172 		if (!pci_dip)
173 			pci_dip = cdip;
174 
175 		/* look for simba */
176 		if (vendor_id == PCI_SIMBA_VENID) {
177 			uint32_t device_id = ddi_getprop(DDI_DEV_T_ANY,
178 			    cdip, DDI_PROP_DONTPASS, "device-id", -1);
179 			if (device_id == PCI_SIMBA_DEVID) {
180 				simba_found = 1;
181 				DEBUG0(DBG_A_INTX, pci_p->pci_dip,
182 					"\tFound simba\n");
183 				continue; /* do not check bridge if simba */
184 			}
185 		}
186 
187 		/* look for pci to pci bridge */
188 		if (pci_config_setup(cdip, &config_handle) != DDI_SUCCESS) {
189 			cmn_err(CE_WARN,
190 			    "%s%d: can't get brdg cfg space for %s%d\n",
191 				ddi_driver_name(dip), ddi_get_instance(dip),
192 				ddi_driver_name(cdip), ddi_get_instance(cdip));
193 			return (DDI_FAILURE);
194 		}
195 		if (pci_config_get8(config_handle, PCI_CONF_BASCLASS)
196 		    == PCI_CLASS_BRIDGE) {
197 			DEBUG0(DBG_A_INTX, pci_p->pci_dip,
198 				"\tFound PCI to xBus bridge\n");
199 			pci_bridge_found = 1;
200 		}
201 		pci_config_teardown(&config_handle);
202 	}
203 
204 	if (!pci_bridge_found)
205 		return (DDI_SUCCESS);
206 	if (!simba_found && (CHIP_TYPE(pci_p) < PCI_CHIP_SCHIZO))
207 		return (DDI_SUCCESS);
208 	if (pci_config_setup(pci_dip, hdl_p) != DDI_SUCCESS) {
209 		cmn_err(CE_WARN, "%s%d: can not get config space for %s%d\n",
210 			ddi_driver_name(dip), ddi_get_instance(dip),
211 			ddi_driver_name(cdip), ddi_get_instance(cdip));
212 		return (DDI_FAILURE);
213 	}
214 	return (DDI_SUCCESS);
215 }
216 
217 /*
218  * If the unclaimed interrupt count has reached the limit set by
219  * pci_unclaimed_intr_max within the time limit, then all interrupts
220  * on this ino is blocked by not idling the interrupt state machine.
221  */
222 static int
223 pci_spurintr(ib_ino_info_t *ino_p) {
224 	int i;
225 	ih_t *ih_p = ino_p->ino_ih_start;
226 	pci_t *pci_p = ino_p->ino_ib_p->ib_pci_p;
227 	char *err_fmt_str;
228 	boolean_t blocked = B_FALSE;
229 
230 	if (ino_p->ino_unclaimed > pci_unclaimed_intr_max)
231 		return (DDI_INTR_CLAIMED);
232 
233 	if (!ino_p->ino_unclaimed)
234 		ino_p->ino_spurintr_begin = ddi_get_lbolt();
235 
236 	ino_p->ino_unclaimed++;
237 
238 	if (ino_p->ino_unclaimed <= pci_unclaimed_intr_max)
239 		goto clear;
240 
241 	if (drv_hztousec(ddi_get_lbolt() - ino_p->ino_spurintr_begin)
242 	    > pci_spurintr_duration) {
243 		ino_p->ino_unclaimed = 0;
244 		goto clear;
245 	}
246 	err_fmt_str = "%s%d: ino 0x%x blocked";
247 	blocked = B_TRUE;
248 	goto warn;
249 clear:
250 	if (!pci_spurintr_msgs) { /* tomatillo errata #71 spurious mondo */
251 		/* clear the pending state */
252 		IB_INO_INTR_CLEAR(ino_p->ino_clr_reg);
253 		return (DDI_INTR_CLAIMED);
254 	}
255 
256 	err_fmt_str = "!%s%d: spurious interrupt from ino 0x%x";
257 warn:
258 	cmn_err(CE_WARN, err_fmt_str, NAMEINST(pci_p->pci_dip), ino_p->ino_ino);
259 	for (i = 0; i < ino_p->ino_ih_size; i++, ih_p = ih_p->ih_next)
260 		cmn_err(CE_CONT, "!%s-%d#%x ", NAMEINST(ih_p->ih_dip),
261 		    ih_p->ih_inum);
262 	cmn_err(CE_CONT, "!\n");
263 	if (blocked == B_FALSE)  /* clear the pending state */
264 		IB_INO_INTR_CLEAR(ino_p->ino_clr_reg);
265 
266 	return (DDI_INTR_CLAIMED);
267 }
268 
269 /*
270  * pci_intr_wrapper
271  *
272  * This routine is used as wrapper around interrupt handlers installed by child
273  * device drivers.  This routine invokes the driver interrupt handlers and
274  * examines the return codes.
275  * There is a count of unclaimed interrupts kept on a per-ino basis. If at
276  * least one handler claims the interrupt then the counter is halved and the
277  * interrupt state machine is idled. If no handler claims the interrupt then
278  * the counter is incremented by one and the state machine is idled.
279  * If the count ever reaches the limit value set by pci_unclaimed_intr_max
280  * then the interrupt state machine is not idled thus preventing any further
281  * interrupts on that ino. The state machine will only be idled again if a
282  * handler is subsequently added or removed.
283  *
284  * return value: DDI_INTR_CLAIMED if any handlers claimed the interrupt,
285  * DDI_INTR_UNCLAIMED otherwise.
286  */
287 
288 extern uint64_t intr_get_time(void);
289 
290 uint_t
291 pci_intr_wrapper(caddr_t arg)
292 {
293 	ib_ino_info_t *ino_p = (ib_ino_info_t *)arg;
294 	uint_t result = 0, r;
295 	pci_t *pci_p = ino_p->ino_ib_p->ib_pci_p;
296 	pbm_t *pbm_p = pci_p->pci_pbm_p;
297 	ih_t *ih_p = ino_p->ino_ih_start;
298 	int i;
299 
300 	for (i = 0; i < ino_p->ino_ih_size; i++, ih_p = ih_p->ih_next) {
301 		dev_info_t *dip = ih_p->ih_dip;
302 		uint_t (*handler)() = ih_p->ih_handler;
303 		caddr_t arg1 = ih_p->ih_handler_arg1;
304 		caddr_t arg2 = ih_p->ih_handler_arg2;
305 		ddi_acc_handle_t cfg_hdl = ih_p->ih_config_handle;
306 
307 		if (pci_intr_dma_sync && cfg_hdl && pbm_p->pbm_sync_reg_pa) {
308 			(void) pci_config_get16(cfg_hdl, PCI_CONF_VENID);
309 			pci_pbm_dma_sync(pbm_p, ino_p->ino_ino);
310 		}
311 
312 		if (ih_p->ih_intr_state == PCI_INTR_STATE_DISABLE) {
313 			DEBUG3(DBG_INTR, pci_p->pci_dip,
314 			    "pci_intr_wrapper: %s%d interrupt %d is disabled\n",
315 			    ddi_driver_name(dip), ddi_get_instance(dip),
316 			    ino_p->ino_ino);
317 
318 			continue;
319 		}
320 
321 		DTRACE_PROBE4(interrupt__start, dev_info_t, dip,
322 		    void *, handler, caddr_t, arg1, caddr_t, arg2);
323 
324 		r = (*handler)(arg1, arg2);
325 
326 		/*
327 		 * Account for time used by this interrupt. Protect against
328 		 * conflicting writes to ih_ticks from ib_intr_dist_all() by
329 		 * using atomic ops.
330 		 */
331 
332 		if (ino_p->ino_pil <= LOCK_LEVEL)
333 			atomic_add_64(&ih_p->ih_ticks, intr_get_time());
334 
335 		DTRACE_PROBE4(interrupt__complete, dev_info_t, dip,
336 		    void *, handler, caddr_t, arg1, int, r);
337 
338 		result += r;
339 
340 		if (pci_check_all_handlers)
341 			continue;
342 		if (result)
343 			break;
344 	}
345 
346 	if (!result)
347 		return (pci_spurintr(ino_p));
348 
349 	ino_p->ino_unclaimed = 0;
350 	IB_INO_INTR_CLEAR(ino_p->ino_clr_reg);  /* clear the pending state */
351 
352 	return (DDI_INTR_CLAIMED);
353 }
354 
355 dev_info_t *
356 get_my_childs_dip(dev_info_t *dip, dev_info_t *rdip)
357 {
358 	dev_info_t *cdip = rdip;
359 
360 	for (; ddi_get_parent(cdip) != dip; cdip = ddi_get_parent(cdip))
361 		;
362 
363 	return (cdip);
364 }
365 
366 /* default class to pil value mapping */
367 pci_class_val_t pci_default_pil [] = {
368 	{0x000000, 0xff0000, 0x1},	/* Class code for pre-2.0 devices */
369 	{0x010000, 0xff0000, 0x4},	/* Mass Storage Controller */
370 	{0x020000, 0xff0000, 0x6},	/* Network Controller */
371 	{0x030000, 0xff0000, 0x9},	/* Display Controller */
372 	{0x040000, 0xff0000, 0x9},	/* Multimedia Controller */
373 	{0x050000, 0xff0000, 0xb},	/* Memory Controller */
374 	{0x060000, 0xff0000, 0xb},	/* Bridge Controller */
375 	{0x0c0000, 0xffff00, 0x9},	/* Serial Bus, FireWire (IEEE 1394) */
376 	{0x0c0100, 0xffff00, 0x4},	/* Serial Bus, ACCESS.bus */
377 	{0x0c0200, 0xffff00, 0x4},	/* Serial Bus, SSA */
378 	{0x0c0300, 0xffff00, 0x9},	/* Serial Bus Universal Serial Bus */
379 	{0x0c0400, 0xffff00, 0x6},	/* Serial Bus, Fibre Channel */
380 	{0x0c0600, 0xffff00, 0x6}	/* Serial Bus, Infiniband */
381 };
382 
383 /*
384  * Default class to intr_weight value mapping (% of CPU).  A driver.conf
385  * entry on or above the pci node like
386  *
387  *	pci-class-intr-weights= 0x020000, 0xff0000, 30;
388  *
389  * can be used to augment or override entries in the default table below.
390  *
391  * NB: The values below give NICs preference on redistribution, and provide
392  * NICs some isolation from other interrupt sources. We need better interfaces
393  * that allow the NIC driver to identify a specific NIC instance as high
394  * bandwidth, and thus deserving of separation from other low bandwidth
395  * NICs additional isolation from other interrupt sources.
396  *
397  * NB: We treat Infiniband like a NIC.
398  */
399 pci_class_val_t pci_default_intr_weight [] = {
400 	{0x020000, 0xff0000, 35},	/* Network Controller */
401 	{0x010000, 0xff0000, 10},	/* Mass Storage Controller */
402 	{0x0c0400, 0xffff00, 10},	/* Serial Bus, Fibre Channel */
403 	{0x0c0600, 0xffff00, 50}	/* Serial Bus, Infiniband */
404 };
405 
406 static uint32_t
407 pci_match_class_val(uint32_t key, pci_class_val_t *rec_p, int nrec,
408     uint32_t default_val)
409 {
410 	int i;
411 
412 	for (i = 0; i < nrec; rec_p++, i++) {
413 		if ((rec_p->class_code & rec_p->class_mask) ==
414 		    (key & rec_p->class_mask))
415 			return (rec_p->class_val);
416 	}
417 
418 	return (default_val);
419 }
420 
421 /*
422  * Return the configuration value, based on class code and sub class code,
423  * from the specified property based or default pci_class_val_t table.
424  */
425 uint32_t
426 pci_class_to_val(dev_info_t *rdip, char *property_name, pci_class_val_t *rec_p,
427     int nrec, uint32_t default_val)
428 {
429 	int property_len;
430 	uint32_t class_code;
431 	pci_class_val_t *conf;
432 	uint32_t val = default_val;
433 
434 	/*
435 	 * Use the "class-code" property to get the base and sub class
436 	 * codes for the requesting device.
437 	 */
438 	class_code = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, rdip,
439 	    DDI_PROP_DONTPASS, "class-code", -1);
440 
441 	if (class_code == -1)
442 		return (val);
443 
444 	/* look up the val from the default table */
445 	val = pci_match_class_val(class_code, rec_p, nrec, val);
446 
447 
448 	/* see if there is a more specific property specified value */
449 	if (ddi_getlongprop(DDI_DEV_T_ANY, rdip, DDI_PROP_NOTPROM,
450 	    property_name, (caddr_t)&conf, &property_len))
451 			return (val);
452 
453 	if ((property_len % sizeof (pci_class_val_t)) == 0)
454 		val = pci_match_class_val(class_code, conf,
455 		    property_len / sizeof (pci_class_val_t), val);
456 	kmem_free(conf, property_len);
457 	return (val);
458 }
459 
460 /* pci_class_to_pil: return the pil for a given PCI device. */
461 uint32_t
462 pci_class_to_pil(dev_info_t *rdip)
463 {
464 	uint32_t pil;
465 
466 	/* default pil is 0 (uninitialized) */
467 	pil = pci_class_to_val(rdip,
468 	    "pci-class-priorities", pci_default_pil,
469 	    sizeof (pci_default_pil) / sizeof (pci_class_val_t), 0);
470 
471 	/* range check the result */
472 	if (pil >= 0xf)
473 		pil = 0;
474 
475 	return (pil);
476 }
477 
478 /* pci_class_to_intr_weight: return the intr_weight for a given PCI device. */
479 int32_t
480 pci_class_to_intr_weight(dev_info_t *rdip)
481 {
482 	int32_t intr_weight;
483 
484 	/* default weight is 0% */
485 	intr_weight = pci_class_to_val(rdip,
486 	    "pci-class-intr-weights", pci_default_intr_weight,
487 	    sizeof (pci_default_intr_weight) / sizeof (pci_class_val_t), 0);
488 
489 	/* range check the result */
490 	if (intr_weight < 0)
491 		intr_weight = 0;
492 	if (intr_weight > 1000)
493 		intr_weight = 1000;
494 
495 	return (intr_weight);
496 }
497 
498 static struct {
499 	kstat_named_t pciintr_ks_name;
500 	kstat_named_t pciintr_ks_type;
501 	kstat_named_t pciintr_ks_cpu;
502 	kstat_named_t pciintr_ks_pil;
503 	kstat_named_t pciintr_ks_time;
504 	kstat_named_t pciintr_ks_ino;
505 	kstat_named_t pciintr_ks_cookie;
506 	kstat_named_t pciintr_ks_devpath;
507 	kstat_named_t pciintr_ks_buspath;
508 } pciintr_ks_template = {
509 	{ "name",	KSTAT_DATA_CHAR },
510 	{ "type",	KSTAT_DATA_CHAR },
511 	{ "cpu",	KSTAT_DATA_UINT64 },
512 	{ "pil",	KSTAT_DATA_UINT64 },
513 	{ "time",	KSTAT_DATA_UINT64 },
514 	{ "ino",	KSTAT_DATA_UINT64 },
515 	{ "cookie",	KSTAT_DATA_UINT64 },
516 	{ "devpath",	KSTAT_DATA_STRING },
517 	{ "buspath",	KSTAT_DATA_STRING },
518 };
519 static uint32_t pciintr_ks_instance;
520 static char ih_devpath[MAXPATHLEN];
521 static char ih_buspath[MAXPATHLEN];
522 
523 kmutex_t pciintr_ks_template_lock;
524 
525 int
526 pci_ks_update(kstat_t *ksp, int rw)
527 {
528 	ih_t *ih_p = ksp->ks_private;
529 	int maxlen = sizeof (pciintr_ks_template.pciintr_ks_name.value.c);
530 	ib_t *ib_p = ih_p->ih_ino_p->ino_ib_p;
531 	pci_t *pci_p = ib_p->ib_pci_p;
532 	ib_ino_t ino;
533 
534 	ino = ih_p->ih_ino_p->ino_ino;
535 
536 	(void) snprintf(pciintr_ks_template.pciintr_ks_name.value.c, maxlen,
537 	    "%s%d", ddi_driver_name(ih_p->ih_dip),
538 	    ddi_get_instance(ih_p->ih_dip));
539 
540 	(void) ddi_pathname(ih_p->ih_dip, ih_devpath);
541 	(void) ddi_pathname(pci_p->pci_dip, ih_buspath);
542 	kstat_named_setstr(&pciintr_ks_template.pciintr_ks_devpath, ih_devpath);
543 	kstat_named_setstr(&pciintr_ks_template.pciintr_ks_buspath, ih_buspath);
544 
545 	if (ih_p->ih_intr_state == PCI_INTR_STATE_ENABLE) {
546 		(void) strcpy(pciintr_ks_template.pciintr_ks_type.value.c,
547 		    "fixed");
548 		pciintr_ks_template.pciintr_ks_cpu.value.ui64 =
549 		    ih_p->ih_ino_p->ino_cpuid;
550 		pciintr_ks_template.pciintr_ks_pil.value.ui64 =
551 		    ih_p->ih_ino_p->ino_pil;
552 		pciintr_ks_template.pciintr_ks_time.value.ui64 = ih_p->ih_nsec +
553 		    (uint64_t)tick2ns((hrtime_t)ih_p->ih_ticks,
554 			ih_p->ih_ino_p->ino_cpuid);
555 		pciintr_ks_template.pciintr_ks_ino.value.ui64 = ino;
556 		pciintr_ks_template.pciintr_ks_cookie.value.ui64 =
557 			IB_INO_TO_MONDO(ib_p, ino);
558 	} else {
559 		(void) strcpy(pciintr_ks_template.pciintr_ks_type.value.c,
560 		    "disabled");
561 		pciintr_ks_template.pciintr_ks_cpu.value.ui64 = 0;
562 		pciintr_ks_template.pciintr_ks_pil.value.ui64 = 0;
563 		pciintr_ks_template.pciintr_ks_time.value.ui64 = 0;
564 		pciintr_ks_template.pciintr_ks_ino.value.ui64 = 0;
565 		pciintr_ks_template.pciintr_ks_cookie.value.ui64 = 0;
566 	}
567 
568 	return (0);
569 }
570 
571 int
572 pci_add_intr(dev_info_t *dip, dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
573 {
574 	pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
575 	ib_t *ib_p = pci_p->pci_ib_p;
576 	cb_t *cb_p = pci_p->pci_cb_p;
577 	ih_t *ih_p;
578 	ib_ino_t ino;
579 	ib_ino_info_t *ino_p;		/* pulse interrupts have no ino */
580 	ib_mondo_t mondo;
581 	uint32_t cpu_id;
582 	int ret;
583 	int32_t weight;
584 
585 	ino = IB_MONDO_TO_INO(hdlp->ih_vector);
586 
587 	DEBUG3(DBG_A_INTX, dip, "pci_add_intr: rdip=%s%d ino=%x\n",
588 	    ddi_driver_name(rdip), ddi_get_instance(rdip), ino);
589 
590 	if (ino > ib_p->ib_max_ino) {
591 		DEBUG1(DBG_A_INTX, dip, "ino %x is invalid\n", ino);
592 		return (DDI_INTR_NOTFOUND);
593 	}
594 
595 	if (hdlp->ih_vector & PCI_PULSE_INO) {
596 		volatile uint64_t *map_reg_addr;
597 		map_reg_addr = ib_intr_map_reg_addr(ib_p, ino);
598 
599 		mondo = pci_xlate_intr(dip, rdip, ib_p, ino);
600 		if (mondo == 0)
601 			goto fail1;
602 
603 		hdlp->ih_vector = CB_MONDO_TO_XMONDO(cb_p, mondo);
604 
605 		if (i_ddi_add_ivintr(hdlp) != DDI_SUCCESS)
606 			goto fail1;
607 
608 		/*
609 		 * Select cpu and program.
610 		 *
611 		 * Since there is no good way to always derive cpuid in
612 		 * pci_remove_intr for PCI_PULSE_INO (esp. for STARFIRE), we
613 		 * don't add (or remove) device weight for pulsed interrupt
614 		 * sources.
615 		 */
616 		mutex_enter(&ib_p->ib_intr_lock);
617 		cpu_id = intr_dist_cpuid();
618 		*map_reg_addr = ib_get_map_reg(mondo, cpu_id);
619 		mutex_exit(&ib_p->ib_intr_lock);
620 		*map_reg_addr;	/* flush previous write */
621 		goto done;
622 	}
623 
624 	if ((mondo = pci_xlate_intr(dip, rdip, pci_p->pci_ib_p, ino)) == 0)
625 		goto fail1;
626 
627 	ino = IB_MONDO_TO_INO(mondo);
628 
629 	mutex_enter(&ib_p->ib_ino_lst_mutex);
630 	ih_p = ib_alloc_ih(rdip, hdlp->ih_inum,
631 	    hdlp->ih_cb_func, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2);
632 	if (map_pcidev_cfg_reg(dip, rdip, &ih_p->ih_config_handle))
633 		goto fail2;
634 
635 	if (ino_p = ib_locate_ino(ib_p, ino)) {		/* sharing ino */
636 		uint32_t intr_index = hdlp->ih_inum;
637 		if (ib_ino_locate_intr(ino_p, rdip, intr_index)) {
638 			DEBUG1(DBG_A_INTX, dip, "dup intr #%d\n", intr_index);
639 			goto fail3;
640 		}
641 
642 		/* add weight to the cpu that we are already targeting */
643 		cpu_id = ino_p->ino_cpuid;
644 		weight = pci_class_to_intr_weight(rdip);
645 		intr_dist_cpuid_add_device_weight(cpu_id, rdip, weight);
646 
647 		ib_ino_add_intr(pci_p, ino_p, ih_p);
648 		goto ino_done;
649 	}
650 
651 	ino_p = ib_new_ino(ib_p, ino, ih_p);
652 
653 	if (hdlp->ih_pri == 0)
654 		hdlp->ih_pri = pci_class_to_pil(rdip);
655 
656 	hdlp->ih_vector = CB_MONDO_TO_XMONDO(cb_p, mondo);
657 
658 	/* Store this global mondo */
659 	ino_p->ino_mondo = hdlp->ih_vector;
660 
661 	DEBUG2(DBG_A_INTX, dip, "pci_add_intr:  pil=0x%x mondo=0x%x\n",
662 	    hdlp->ih_pri, hdlp->ih_vector);
663 
664 	DDI_INTR_ASSIGN_HDLR_N_ARGS(hdlp,
665 	    (ddi_intr_handler_t *)pci_intr_wrapper, (caddr_t)ino_p, NULL);
666 
667 	ret = i_ddi_add_ivintr(hdlp);
668 
669 	/*
670 	 * Restore original interrupt handler
671 	 * and arguments in interrupt handle.
672 	 */
673 	DDI_INTR_ASSIGN_HDLR_N_ARGS(hdlp, ih_p->ih_handler,
674 	    ih_p->ih_handler_arg1, ih_p->ih_handler_arg2);
675 
676 	if (ret != DDI_SUCCESS)
677 		goto fail4;
678 
679 	/* Save the pil for this ino */
680 	ino_p->ino_pil = hdlp->ih_pri;
681 
682 	/* clear and enable interrupt */
683 	IB_INO_INTR_CLEAR(ino_p->ino_clr_reg);
684 
685 	/* select cpu and compute weight, saving both for sharing and removal */
686 	cpu_id = pci_intr_dist_cpuid(ib_p, ino_p);
687 	ino_p->ino_cpuid = cpu_id;
688 	ino_p->ino_established = 1;
689 	weight = pci_class_to_intr_weight(rdip);
690 	intr_dist_cpuid_add_device_weight(cpu_id, rdip, weight);
691 
692 #ifdef _STARFIRE
693 	cpu_id = pc_translate_tgtid(cb_p->cb_ittrans_cookie, cpu_id,
694 		IB_GET_MAPREG_INO(ino));
695 #endif /* _STARFIRE */
696 	*ino_p->ino_map_reg = ib_get_map_reg(mondo, cpu_id);
697 	*ino_p->ino_map_reg;
698 ino_done:
699 	ih_p->ih_ino_p = ino_p;
700 	ih_p->ih_ksp = kstat_create("pci_intrs",
701 	    atomic_inc_32_nv(&pciintr_ks_instance), "config", "interrupts",
702 	    KSTAT_TYPE_NAMED,
703 	    sizeof (pciintr_ks_template) / sizeof (kstat_named_t),
704 	    KSTAT_FLAG_VIRTUAL);
705 	if (ih_p->ih_ksp != NULL) {
706 		ih_p->ih_ksp->ks_data_size += MAXPATHLEN * 2;
707 		ih_p->ih_ksp->ks_lock = &pciintr_ks_template_lock;
708 		ih_p->ih_ksp->ks_data = &pciintr_ks_template;
709 		ih_p->ih_ksp->ks_private = ih_p;
710 		ih_p->ih_ksp->ks_update = pci_ks_update;
711 		kstat_install(ih_p->ih_ksp);
712 	}
713 	ib_ino_map_reg_share(ib_p, ino, ino_p);
714 	mutex_exit(&ib_p->ib_ino_lst_mutex);
715 done:
716 	DEBUG2(DBG_A_INTX, dip, "done! Interrupt 0x%x pil=%x\n",
717 		hdlp->ih_vector, hdlp->ih_pri);
718 	return (DDI_SUCCESS);
719 fail4:
720 	ib_delete_ino(ib_p, ino_p);
721 fail3:
722 	if (ih_p->ih_config_handle)
723 		pci_config_teardown(&ih_p->ih_config_handle);
724 fail2:
725 	mutex_exit(&ib_p->ib_ino_lst_mutex);
726 	kmem_free(ih_p, sizeof (ih_t));
727 fail1:
728 	DEBUG2(DBG_A_INTX, dip, "Failed! Interrupt 0x%x pil=%x\n",
729 		hdlp->ih_vector, hdlp->ih_pri);
730 	return (DDI_FAILURE);
731 }
732 
733 int
734 pci_remove_intr(dev_info_t *dip, dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
735 {
736 	pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
737 	ib_t *ib_p = pci_p->pci_ib_p;
738 	cb_t *cb_p = pci_p->pci_cb_p;
739 	ib_ino_t ino;
740 	ib_mondo_t mondo;
741 	ib_ino_info_t *ino_p;	/* non-pulse only */
742 	ih_t *ih_p;		/* non-pulse only */
743 
744 	ino = IB_MONDO_TO_INO(hdlp->ih_vector);
745 
746 	DEBUG3(DBG_R_INTX, dip, "pci_rem_intr: rdip=%s%d ino=%x\n",
747 	    ddi_driver_name(rdip), ddi_get_instance(rdip), ino);
748 
749 	if (hdlp->ih_vector & PCI_PULSE_INO) { /* pulse interrupt */
750 		volatile uint64_t *map_reg_addr;
751 
752 		/*
753 		 * No weight was added by pci_add_intr for PCI_PULSE_INO
754 		 * because it is difficult to determine cpuid here.
755 		 */
756 		map_reg_addr = ib_intr_map_reg_addr(ib_p, ino);
757 		IB_INO_INTR_RESET(map_reg_addr);	/* disable intr */
758 		*map_reg_addr;
759 
760 		mondo = pci_xlate_intr(dip, rdip, ib_p, ino);
761 		if (mondo == 0) {
762 			DEBUG1(DBG_R_INTX, dip,
763 				"can't get mondo for ino %x\n", ino);
764 			return (DDI_FAILURE);
765 		}
766 
767 		if (hdlp->ih_pri == 0)
768 			hdlp->ih_pri = pci_class_to_pil(rdip);
769 
770 		hdlp->ih_vector = CB_MONDO_TO_XMONDO(cb_p, mondo);
771 
772 		DEBUG2(DBG_R_INTX, dip, "pci_rem_intr: pil=0x%x mondo=0x%x\n",
773 		    hdlp->ih_pri, hdlp->ih_vector);
774 
775 		i_ddi_rem_ivintr(hdlp);
776 
777 		DEBUG2(DBG_R_INTX, dip, "pulse success mondo=%x reg=%p\n",
778 			mondo, map_reg_addr);
779 		return (DDI_SUCCESS);
780 	}
781 
782 	/* Translate the interrupt property */
783 	mondo = pci_xlate_intr(dip, rdip, pci_p->pci_ib_p, ino);
784 	if (mondo == 0) {
785 		DEBUG1(DBG_R_INTX, dip, "can't get mondo for ino %x\n", ino);
786 		return (DDI_FAILURE);
787 	}
788 	ino = IB_MONDO_TO_INO(mondo);
789 
790 	mutex_enter(&ib_p->ib_ino_lst_mutex);
791 	ino_p = ib_locate_ino(ib_p, ino);
792 	if (!ino_p) {
793 		int r = cb_remove_xintr(pci_p, dip, rdip, ino, mondo);
794 		if (r != DDI_SUCCESS)
795 			cmn_err(CE_WARN, "%s%d-xintr: ino %x is invalid",
796 			    ddi_driver_name(dip), ddi_get_instance(dip), ino);
797 		mutex_exit(&ib_p->ib_ino_lst_mutex);
798 		return (r);
799 	}
800 
801 	ih_p = ib_ino_locate_intr(ino_p, rdip, hdlp->ih_inum);
802 	ib_ino_rem_intr(pci_p, ino_p, ih_p);
803 	intr_dist_cpuid_rem_device_weight(ino_p->ino_cpuid, rdip);
804 	if (ino_p->ino_ih_size == 0) {
805 		IB_INO_INTR_PEND(ib_clear_intr_reg_addr(ib_p, ino));
806 		hdlp->ih_vector = CB_MONDO_TO_XMONDO(cb_p, mondo);
807 		if (hdlp->ih_pri == 0)
808 			hdlp->ih_pri = pci_class_to_pil(rdip);
809 
810 		i_ddi_rem_ivintr(hdlp);
811 		ib_delete_ino(ib_p, ino_p);
812 	}
813 
814 	/* re-enable interrupt only if mapping register still shared */
815 	if (ib_ino_map_reg_unshare(ib_p, ino, ino_p)) {
816 		IB_INO_INTR_ON(ino_p->ino_map_reg);
817 		*ino_p->ino_map_reg;
818 	}
819 	mutex_exit(&ib_p->ib_ino_lst_mutex);
820 
821 	if (ino_p->ino_ih_size == 0)
822 		kmem_free(ino_p, sizeof (ib_ino_info_t));
823 
824 	DEBUG1(DBG_R_INTX, dip, "success! mondo=%x\n", mondo);
825 	return (DDI_SUCCESS);
826 }
827 
828 /*
829  * free the pci_inos array allocated during pci_intr_setup. the actual
830  * interrupts are torn down by their respective block destroy routines:
831  * cb_destroy, pbm_destroy, and ib_destroy.
832  */
833 void
834 pci_intr_teardown(pci_t *pci_p)
835 {
836 	kmem_free(pci_p->pci_inos, pci_p->pci_inos_len);
837 	pci_p->pci_inos = NULL;
838 	pci_p->pci_inos_len = 0;
839 }
840