xref: /titanic_51/usr/src/uts/common/io/avintr.c (revision 7a364d25fde47aa82704b12b5251bf7fac37f02e)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
57c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
67c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
77c478bd9Sstevel@tonic-gate  * with the License.
87c478bd9Sstevel@tonic-gate  *
97c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
107c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
117c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
127c478bd9Sstevel@tonic-gate  * and limitations under the License.
137c478bd9Sstevel@tonic-gate  *
147c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
157c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
167c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
177c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
187c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
197c478bd9Sstevel@tonic-gate  *
207c478bd9Sstevel@tonic-gate  * CDDL HEADER END
217c478bd9Sstevel@tonic-gate  */
227c478bd9Sstevel@tonic-gate /*
237c478bd9Sstevel@tonic-gate  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate  * Autovectored Interrupt Configuration and Deconfiguration
317c478bd9Sstevel@tonic-gate  */
327c478bd9Sstevel@tonic-gate 
337c478bd9Sstevel@tonic-gate #include <sys/param.h>
347c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
357c478bd9Sstevel@tonic-gate #include <sys/trap.h>
367c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
377c478bd9Sstevel@tonic-gate #include <sys/avintr.h>
387c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
397c478bd9Sstevel@tonic-gate #include <sys/machlock.h>
407c478bd9Sstevel@tonic-gate #include <sys/systm.h>
417c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
427c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
437c478bd9Sstevel@tonic-gate #include <sys/x_call.h>
447c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
457c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
467c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h>
477c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
487c478bd9Sstevel@tonic-gate #include <sys/stack.h>
497c478bd9Sstevel@tonic-gate #include <sys/ddi_impldefs.h>
507c478bd9Sstevel@tonic-gate 
517c478bd9Sstevel@tonic-gate static int insert_av(void *intr_id, struct av_head *vectp, avfunc f,
52*7a364d25Sschwartz 	caddr_t arg1, caddr_t arg2, uint64_t *ticksp, int pri_level,
53*7a364d25Sschwartz 	dev_info_t *dip);
547c478bd9Sstevel@tonic-gate static void remove_av(void *intr_id, struct av_head *vectp, avfunc f,
557c478bd9Sstevel@tonic-gate 	int pri_level, int vect);
567c478bd9Sstevel@tonic-gate 
577c478bd9Sstevel@tonic-gate /*
587c478bd9Sstevel@tonic-gate  * Arrange for a driver to be called when a particular
597c478bd9Sstevel@tonic-gate  * auto-vectored interrupt occurs.
607c478bd9Sstevel@tonic-gate  * NOTE: if a device can generate interrupts on more than
617c478bd9Sstevel@tonic-gate  * one level, or if a driver services devices that interrupt
627c478bd9Sstevel@tonic-gate  * on more than one level, then the driver should install
637c478bd9Sstevel@tonic-gate  * itself on each of those levels.
647c478bd9Sstevel@tonic-gate  */
657c478bd9Sstevel@tonic-gate static char badsoft[] =
667c478bd9Sstevel@tonic-gate 	"add_avintr: bad soft interrupt level %d for driver '%s'\n";
677c478bd9Sstevel@tonic-gate static char multilevel[] =
687c478bd9Sstevel@tonic-gate 	"!IRQ%d is being shared by drivers with different interrupt levels.\n"
697c478bd9Sstevel@tonic-gate 	"This may result in reduced system performance.";
707c478bd9Sstevel@tonic-gate static char multilevel2[] =
717c478bd9Sstevel@tonic-gate 	"Cannot register interrupt for '%s' device at IPL %d because it\n"
727c478bd9Sstevel@tonic-gate 	"conflicts with another device using the same vector %d with an IPL\n"
737c478bd9Sstevel@tonic-gate 	"of %d. Reconfigure the conflicting devices to use different vectors.";
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate #define	MAX_VECT	256
767c478bd9Sstevel@tonic-gate struct autovec *nmivect = NULL;
777c478bd9Sstevel@tonic-gate struct av_head autovect[MAX_VECT];
787c478bd9Sstevel@tonic-gate struct av_head softvect[LOCK_LEVEL + 1];
797c478bd9Sstevel@tonic-gate kmutex_t av_lock;
807c478bd9Sstevel@tonic-gate ddi_softint_hdl_impl_t softlevel1_hdl =
817c478bd9Sstevel@tonic-gate 	{0, NULL, NULL, 0, 0, NULL, NULL, NULL};
827c478bd9Sstevel@tonic-gate 
837c478bd9Sstevel@tonic-gate void
847c478bd9Sstevel@tonic-gate set_pending(int pri)
857c478bd9Sstevel@tonic-gate {
867c478bd9Sstevel@tonic-gate 	atomic_or_32((uint32_t *)&CPU->cpu_softinfo.st_pending, 1 << pri);
877c478bd9Sstevel@tonic-gate }
887c478bd9Sstevel@tonic-gate 
897c478bd9Sstevel@tonic-gate /*
907c478bd9Sstevel@tonic-gate  * register nmi interrupt routine. The first arg is used only to order
917c478bd9Sstevel@tonic-gate  * various nmi interrupt service routines in the chain. Higher lvls will
927c478bd9Sstevel@tonic-gate  * be called first
937c478bd9Sstevel@tonic-gate  */
947c478bd9Sstevel@tonic-gate int
957c478bd9Sstevel@tonic-gate add_nmintr(int lvl, avfunc nmintr, char *name, caddr_t arg)
967c478bd9Sstevel@tonic-gate {
977c478bd9Sstevel@tonic-gate 	struct autovec  *mem;
987c478bd9Sstevel@tonic-gate 	struct autovec *p, *prev = NULL;
997c478bd9Sstevel@tonic-gate 
1007c478bd9Sstevel@tonic-gate 	if (nmintr == NULL) {
1017c478bd9Sstevel@tonic-gate 		printf("Attempt to add null vect for %s on nmi\n", name);
1027c478bd9Sstevel@tonic-gate 		return (0);
1037c478bd9Sstevel@tonic-gate 
1047c478bd9Sstevel@tonic-gate 	}
1057c478bd9Sstevel@tonic-gate 
1067c478bd9Sstevel@tonic-gate 	mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
1077c478bd9Sstevel@tonic-gate 	mem->av_vector = nmintr;
1087c478bd9Sstevel@tonic-gate 	mem->av_intarg1 = arg;
1097c478bd9Sstevel@tonic-gate 	mem->av_intarg2 = NULL;
1107c478bd9Sstevel@tonic-gate 	mem->av_intr_id = NULL;
1117c478bd9Sstevel@tonic-gate 	mem->av_prilevel = lvl;
1120b38a8bdSahl 	mem->av_dip = NULL;
1137c478bd9Sstevel@tonic-gate 	mem->av_link = NULL;
1147c478bd9Sstevel@tonic-gate 
1157c478bd9Sstevel@tonic-gate 	mutex_enter(&av_lock);
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate 	if (!nmivect) {
1187c478bd9Sstevel@tonic-gate 		nmivect = mem;
1197c478bd9Sstevel@tonic-gate 		mutex_exit(&av_lock);
1207c478bd9Sstevel@tonic-gate 		return (1);
1217c478bd9Sstevel@tonic-gate 	}
1227c478bd9Sstevel@tonic-gate 	/* find where it goes in list */
1237c478bd9Sstevel@tonic-gate 	for (p = nmivect; p != NULL; p = p->av_link) {
1247c478bd9Sstevel@tonic-gate 		if (p->av_vector == nmintr && p->av_intarg1 == arg) {
1257c478bd9Sstevel@tonic-gate 			/*
1267c478bd9Sstevel@tonic-gate 			 * already in list
1277c478bd9Sstevel@tonic-gate 			 * So? Somebody added the same interrupt twice.
1287c478bd9Sstevel@tonic-gate 			 */
1297c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN, "Driver already registered '%s'",
1307c478bd9Sstevel@tonic-gate 			    name);
1317c478bd9Sstevel@tonic-gate 			kmem_free(mem, sizeof (struct autovec));
1327c478bd9Sstevel@tonic-gate 			mutex_exit(&av_lock);
1337c478bd9Sstevel@tonic-gate 			return (0);
1347c478bd9Sstevel@tonic-gate 		}
1357c478bd9Sstevel@tonic-gate 		if (p->av_prilevel < lvl) {
1367c478bd9Sstevel@tonic-gate 			if (p == nmivect) {   /* it's at head of list */
1377c478bd9Sstevel@tonic-gate 				mem->av_link = p;
1387c478bd9Sstevel@tonic-gate 				nmivect = mem;
1397c478bd9Sstevel@tonic-gate 			} else {
1407c478bd9Sstevel@tonic-gate 				mem->av_link = p;
1417c478bd9Sstevel@tonic-gate 				prev->av_link = mem;
1427c478bd9Sstevel@tonic-gate 			}
1437c478bd9Sstevel@tonic-gate 			mutex_exit(&av_lock);
1447c478bd9Sstevel@tonic-gate 			return (1);
1457c478bd9Sstevel@tonic-gate 		}
1467c478bd9Sstevel@tonic-gate 		prev = p;
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate 	}
1497c478bd9Sstevel@tonic-gate 	/* didn't find it, add it to the end */
1507c478bd9Sstevel@tonic-gate 	prev->av_link = mem;
1517c478bd9Sstevel@tonic-gate 	mutex_exit(&av_lock);
1527c478bd9Sstevel@tonic-gate 	return (1);
1537c478bd9Sstevel@tonic-gate 
1547c478bd9Sstevel@tonic-gate }
1557c478bd9Sstevel@tonic-gate 
1567c478bd9Sstevel@tonic-gate /*
1577c478bd9Sstevel@tonic-gate  * register a hardware interrupt handler.
1587c478bd9Sstevel@tonic-gate  */
1597c478bd9Sstevel@tonic-gate int
1607c478bd9Sstevel@tonic-gate add_avintr(void *intr_id, int lvl, avfunc xxintr, char *name, int vect,
161*7a364d25Sschwartz     caddr_t arg1, caddr_t arg2, uint64_t *ticksp, dev_info_t *dip)
1627c478bd9Sstevel@tonic-gate {
1637c478bd9Sstevel@tonic-gate 	struct av_head *vecp = (struct av_head *)0;
1647c478bd9Sstevel@tonic-gate 	avfunc f;
1657c478bd9Sstevel@tonic-gate 	int s, vectindex;			/* save old spl value */
1667c478bd9Sstevel@tonic-gate 	ushort_t hi_pri;
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate 	if ((f = xxintr) == NULL) {
1697c478bd9Sstevel@tonic-gate 		printf("Attempt to add null vect for %s on vector %d\n",
1707c478bd9Sstevel@tonic-gate 			name, vect);
1717c478bd9Sstevel@tonic-gate 		return (0);
1727c478bd9Sstevel@tonic-gate 
1737c478bd9Sstevel@tonic-gate 	}
1747c478bd9Sstevel@tonic-gate 	vectindex = vect % MAX_VECT;
1757c478bd9Sstevel@tonic-gate 
1767c478bd9Sstevel@tonic-gate 	vecp = &autovect[vectindex];
1777c478bd9Sstevel@tonic-gate 
1787c478bd9Sstevel@tonic-gate 	/*
1797c478bd9Sstevel@tonic-gate 	 * "hi_pri == 0" implies all entries on list are "unused",
1807c478bd9Sstevel@tonic-gate 	 * which means that it's OK to just insert this one.
1817c478bd9Sstevel@tonic-gate 	 */
1827c478bd9Sstevel@tonic-gate 	hi_pri = vecp->avh_hi_pri;
1837c478bd9Sstevel@tonic-gate 	if (vecp->avh_link && (hi_pri != 0)) {
1847c478bd9Sstevel@tonic-gate 		if (((hi_pri > LOCK_LEVEL) && (lvl < LOCK_LEVEL)) ||
1857c478bd9Sstevel@tonic-gate 		    ((hi_pri < LOCK_LEVEL) && (lvl > LOCK_LEVEL))) {
1867c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN, multilevel2, name, lvl, vect,
1877c478bd9Sstevel@tonic-gate 				hi_pri);
1887c478bd9Sstevel@tonic-gate 			return (0);
1897c478bd9Sstevel@tonic-gate 		}
1907c478bd9Sstevel@tonic-gate 		if ((vecp->avh_lo_pri != lvl) || (hi_pri != lvl))
1917c478bd9Sstevel@tonic-gate 			cmn_err(CE_NOTE, multilevel, vect);
1927c478bd9Sstevel@tonic-gate 	}
1937c478bd9Sstevel@tonic-gate 
194*7a364d25Sschwartz 	if (!insert_av(intr_id, vecp, f, arg1, arg2, ticksp, lvl, dip))
1957c478bd9Sstevel@tonic-gate 		return (0);
1967c478bd9Sstevel@tonic-gate 	s = splhi();
1977c478bd9Sstevel@tonic-gate 	/*
1987c478bd9Sstevel@tonic-gate 	 * do what ever machine specific things are necessary
1997c478bd9Sstevel@tonic-gate 	 * to set priority level (e.g. set picmasks)
2007c478bd9Sstevel@tonic-gate 	 */
2017c478bd9Sstevel@tonic-gate 	mutex_enter(&av_lock);
2027c478bd9Sstevel@tonic-gate 	(*addspl)(vect, lvl, vecp->avh_lo_pri, vecp->avh_hi_pri);
2037c478bd9Sstevel@tonic-gate 	mutex_exit(&av_lock);
2047c478bd9Sstevel@tonic-gate 	splx(s);
2057c478bd9Sstevel@tonic-gate 	return (1);
2067c478bd9Sstevel@tonic-gate 
2077c478bd9Sstevel@tonic-gate }
2087c478bd9Sstevel@tonic-gate 
2097c478bd9Sstevel@tonic-gate void
2107c478bd9Sstevel@tonic-gate update_avsoftintr_args(void *intr_id, int lvl, caddr_t arg2)
2117c478bd9Sstevel@tonic-gate {
2127c478bd9Sstevel@tonic-gate 	struct autovec *p;
2137c478bd9Sstevel@tonic-gate 	struct autovec *target = NULL;
2147c478bd9Sstevel@tonic-gate 	struct av_head *vectp = (struct av_head *)&softvect[lvl];
2157c478bd9Sstevel@tonic-gate 
2167c478bd9Sstevel@tonic-gate 	for (p = vectp->avh_link; p && p->av_vector; p = p->av_link) {
2177c478bd9Sstevel@tonic-gate 		if (p->av_intr_id == intr_id) {
2187c478bd9Sstevel@tonic-gate 			target = p;
2197c478bd9Sstevel@tonic-gate 			break;
2207c478bd9Sstevel@tonic-gate 		}
2217c478bd9Sstevel@tonic-gate 	}
2227c478bd9Sstevel@tonic-gate 
2237c478bd9Sstevel@tonic-gate 	if (target == NULL)
2247c478bd9Sstevel@tonic-gate 		return;
2257c478bd9Sstevel@tonic-gate 	target->av_intarg2 = arg2;
2267c478bd9Sstevel@tonic-gate }
2277c478bd9Sstevel@tonic-gate 
2287c478bd9Sstevel@tonic-gate /*
2297c478bd9Sstevel@tonic-gate  * Register a software interrupt handler
2307c478bd9Sstevel@tonic-gate  */
2317c478bd9Sstevel@tonic-gate int
2327c478bd9Sstevel@tonic-gate add_avsoftintr(void *intr_id, int lvl, avfunc xxintr, char *name,
2337c478bd9Sstevel@tonic-gate     caddr_t arg1, caddr_t arg2)
2347c478bd9Sstevel@tonic-gate {
2357c478bd9Sstevel@tonic-gate 	int slvl;
2367c478bd9Sstevel@tonic-gate 
2377c478bd9Sstevel@tonic-gate 	if ((slvl = slvltovect(lvl)) != -1)
2387c478bd9Sstevel@tonic-gate 		return (add_avintr(intr_id, lvl, xxintr,
239*7a364d25Sschwartz 		    name, slvl, arg1, arg2, NULL, NULL));
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate 	if (intr_id == NULL) {
2427c478bd9Sstevel@tonic-gate 		printf("Attempt to add null intr_id for %s on level %d\n",
2437c478bd9Sstevel@tonic-gate 		    name, lvl);
2447c478bd9Sstevel@tonic-gate 		return (0);
2457c478bd9Sstevel@tonic-gate 	}
2467c478bd9Sstevel@tonic-gate 
2477c478bd9Sstevel@tonic-gate 	if (xxintr == NULL) {
2487c478bd9Sstevel@tonic-gate 		printf("Attempt to add null handler for %s on level %d\n",
2497c478bd9Sstevel@tonic-gate 		    name, lvl);
2507c478bd9Sstevel@tonic-gate 		return (0);
2517c478bd9Sstevel@tonic-gate 	}
2527c478bd9Sstevel@tonic-gate 
2537c478bd9Sstevel@tonic-gate 	if (lvl <= 0 || lvl > LOCK_LEVEL) {
2547c478bd9Sstevel@tonic-gate 		printf(badsoft, lvl, name);
2557c478bd9Sstevel@tonic-gate 		return (0);
2567c478bd9Sstevel@tonic-gate 	}
257*7a364d25Sschwartz 	if (!insert_av(intr_id, &softvect[lvl], xxintr, arg1, arg2, NULL,
2587c478bd9Sstevel@tonic-gate 	    lvl, NULL)) {
2597c478bd9Sstevel@tonic-gate 		return (0);
2607c478bd9Sstevel@tonic-gate 	}
2617c478bd9Sstevel@tonic-gate 	return (1);
2627c478bd9Sstevel@tonic-gate }
2637c478bd9Sstevel@tonic-gate 
2647c478bd9Sstevel@tonic-gate /* insert an interrupt vector into chain */
2657c478bd9Sstevel@tonic-gate static int
2667c478bd9Sstevel@tonic-gate insert_av(void *intr_id, struct av_head *vectp, avfunc f, caddr_t arg1,
267*7a364d25Sschwartz     caddr_t arg2, uint64_t *ticksp, int pri_level, dev_info_t *dip)
2687c478bd9Sstevel@tonic-gate {
2697c478bd9Sstevel@tonic-gate 	/*
2707c478bd9Sstevel@tonic-gate 	 * Protect rewrites of the list
2717c478bd9Sstevel@tonic-gate 	 */
2727c478bd9Sstevel@tonic-gate 	struct autovec *p, *mem;
2737c478bd9Sstevel@tonic-gate 
2747c478bd9Sstevel@tonic-gate 	mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
2757c478bd9Sstevel@tonic-gate 	mem->av_vector = f;
2767c478bd9Sstevel@tonic-gate 	mem->av_intarg1 = arg1;
2777c478bd9Sstevel@tonic-gate 	mem->av_intarg2 = arg2;
278*7a364d25Sschwartz 	mem->av_ticksp = ticksp;
2797c478bd9Sstevel@tonic-gate 	mem->av_intr_id = intr_id;
2807c478bd9Sstevel@tonic-gate 	mem->av_prilevel = pri_level;
2817c478bd9Sstevel@tonic-gate 	mem->av_dip = dip;
2827c478bd9Sstevel@tonic-gate 	mem->av_link = NULL;
2837c478bd9Sstevel@tonic-gate 
2847c478bd9Sstevel@tonic-gate 	mutex_enter(&av_lock);
2857c478bd9Sstevel@tonic-gate 
2867c478bd9Sstevel@tonic-gate 	if (vectp->avh_link == NULL) {	/* Nothing on list - put it at head */
2877c478bd9Sstevel@tonic-gate 		vectp->avh_link = mem;
2887c478bd9Sstevel@tonic-gate 		vectp->avh_hi_pri = vectp->avh_lo_pri = (ushort_t)pri_level;
2897c478bd9Sstevel@tonic-gate 
2907c478bd9Sstevel@tonic-gate 		mutex_exit(&av_lock);
2917c478bd9Sstevel@tonic-gate 		return (1);
2927c478bd9Sstevel@tonic-gate 	}
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate 	/* find where it goes in list */
2957c478bd9Sstevel@tonic-gate 	for (p = vectp->avh_link; p != NULL; p = p->av_link) {
2967c478bd9Sstevel@tonic-gate 		if (p->av_vector == NULL) {	/* freed struct available */
2977c478bd9Sstevel@tonic-gate 			kmem_free(mem, sizeof (struct autovec));
2987c478bd9Sstevel@tonic-gate 			p->av_intarg1 = arg1;
2997c478bd9Sstevel@tonic-gate 			p->av_intarg2 = arg2;
300*7a364d25Sschwartz 			p->av_ticksp = ticksp;
3017c478bd9Sstevel@tonic-gate 			p->av_intr_id = intr_id;
3027c478bd9Sstevel@tonic-gate 			p->av_prilevel = pri_level;
3030b38a8bdSahl 			p->av_dip = dip;
3047c478bd9Sstevel@tonic-gate 			if (pri_level > (int)vectp->avh_hi_pri) {
3057c478bd9Sstevel@tonic-gate 				vectp->avh_hi_pri = (ushort_t)pri_level;
3067c478bd9Sstevel@tonic-gate 			}
3077c478bd9Sstevel@tonic-gate 			if (pri_level < (int)vectp->avh_lo_pri) {
3087c478bd9Sstevel@tonic-gate 				vectp->avh_lo_pri = (ushort_t)pri_level;
3097c478bd9Sstevel@tonic-gate 			}
3107c478bd9Sstevel@tonic-gate 			p->av_vector = f;
3117c478bd9Sstevel@tonic-gate 			mutex_exit(&av_lock);
3127c478bd9Sstevel@tonic-gate 			return (1);
3137c478bd9Sstevel@tonic-gate 		}
3147c478bd9Sstevel@tonic-gate 	}
3157c478bd9Sstevel@tonic-gate 	/* insert new intpt at beginning of chain */
3167c478bd9Sstevel@tonic-gate 	mem->av_link = vectp->avh_link;
3177c478bd9Sstevel@tonic-gate 	vectp->avh_link = mem;
3187c478bd9Sstevel@tonic-gate 	if (pri_level > (int)vectp->avh_hi_pri) {
3197c478bd9Sstevel@tonic-gate 		vectp->avh_hi_pri = (ushort_t)pri_level;
3207c478bd9Sstevel@tonic-gate 	}
3217c478bd9Sstevel@tonic-gate 	if (pri_level < (int)vectp->avh_lo_pri) {
3227c478bd9Sstevel@tonic-gate 		vectp->avh_lo_pri = (ushort_t)pri_level;
3237c478bd9Sstevel@tonic-gate 	}
3247c478bd9Sstevel@tonic-gate 	mutex_exit(&av_lock);
3257c478bd9Sstevel@tonic-gate 
3267c478bd9Sstevel@tonic-gate 	return (1);
3277c478bd9Sstevel@tonic-gate }
3287c478bd9Sstevel@tonic-gate 
3297c478bd9Sstevel@tonic-gate /*
3307c478bd9Sstevel@tonic-gate  * Remove a driver from the autovector list.
3317c478bd9Sstevel@tonic-gate  */
3327c478bd9Sstevel@tonic-gate int
3337c478bd9Sstevel@tonic-gate rem_avsoftintr(void *intr_id, int lvl, avfunc xxintr)
3347c478bd9Sstevel@tonic-gate {
3357c478bd9Sstevel@tonic-gate 	struct av_head *vecp = (struct av_head *)0;
3367c478bd9Sstevel@tonic-gate 	int slvl;
3377c478bd9Sstevel@tonic-gate 
3387c478bd9Sstevel@tonic-gate 	if (xxintr == NULL)
3397c478bd9Sstevel@tonic-gate 		return (0);
3407c478bd9Sstevel@tonic-gate 
3417c478bd9Sstevel@tonic-gate 	if ((slvl = slvltovect(lvl)) != -1) {
3427c478bd9Sstevel@tonic-gate 		rem_avintr(intr_id, lvl, xxintr, slvl);
3437c478bd9Sstevel@tonic-gate 		return (1);
3447c478bd9Sstevel@tonic-gate 	}
3457c478bd9Sstevel@tonic-gate 
3467c478bd9Sstevel@tonic-gate 	if (lvl <= 0 && lvl >= LOCK_LEVEL) {
3477c478bd9Sstevel@tonic-gate 		return (0);
3487c478bd9Sstevel@tonic-gate 	}
3497c478bd9Sstevel@tonic-gate 	vecp = &softvect[lvl];
3507c478bd9Sstevel@tonic-gate 	remove_av(intr_id, vecp, xxintr, lvl, 0);
3517c478bd9Sstevel@tonic-gate 
3527c478bd9Sstevel@tonic-gate 	return (1);
3537c478bd9Sstevel@tonic-gate }
3547c478bd9Sstevel@tonic-gate 
3557c478bd9Sstevel@tonic-gate void
3567c478bd9Sstevel@tonic-gate rem_avintr(void *intr_id, int lvl, avfunc xxintr, int vect)
3577c478bd9Sstevel@tonic-gate {
3587c478bd9Sstevel@tonic-gate 	struct av_head *vecp = (struct av_head *)0;
3597c478bd9Sstevel@tonic-gate 	avfunc f;
3607c478bd9Sstevel@tonic-gate 	int s, vectindex;			/* save old spl value */
3617c478bd9Sstevel@tonic-gate 
3627c478bd9Sstevel@tonic-gate 	if ((f = xxintr) == NULL)
3637c478bd9Sstevel@tonic-gate 		return;
3647c478bd9Sstevel@tonic-gate 
3657c478bd9Sstevel@tonic-gate 	vectindex = vect % MAX_VECT;
3667c478bd9Sstevel@tonic-gate 	vecp = &autovect[vectindex];
3677c478bd9Sstevel@tonic-gate 	remove_av(intr_id, vecp, f, lvl, vect);
3687c478bd9Sstevel@tonic-gate 	s = splhi();
3697c478bd9Sstevel@tonic-gate 	mutex_enter(&av_lock);
3707c478bd9Sstevel@tonic-gate 	(*delspl)(vect, lvl, vecp->avh_lo_pri, vecp->avh_hi_pri);
3717c478bd9Sstevel@tonic-gate 	mutex_exit(&av_lock);
3727c478bd9Sstevel@tonic-gate 	splx(s);
3737c478bd9Sstevel@tonic-gate }
3747c478bd9Sstevel@tonic-gate 
3757c478bd9Sstevel@tonic-gate 
3767c478bd9Sstevel@tonic-gate /*
3777c478bd9Sstevel@tonic-gate  * After having made a change to an autovector list, wait until we have
3787c478bd9Sstevel@tonic-gate  * seen each cpu not executing an interrupt at that level--so we know our
3797c478bd9Sstevel@tonic-gate  * change has taken effect completely (no old state in registers, etc).
3807c478bd9Sstevel@tonic-gate  */
3817c478bd9Sstevel@tonic-gate void
3827c478bd9Sstevel@tonic-gate wait_till_seen(int ipl)
3837c478bd9Sstevel@tonic-gate {
3847c478bd9Sstevel@tonic-gate 	int cpu_in_chain, cix;
3857c478bd9Sstevel@tonic-gate 	struct cpu *cpup;
3867c478bd9Sstevel@tonic-gate 	cpuset_t cpus_to_check;
3877c478bd9Sstevel@tonic-gate 
3887c478bd9Sstevel@tonic-gate 	CPUSET_ALL(cpus_to_check);
3897c478bd9Sstevel@tonic-gate 	do {
3907c478bd9Sstevel@tonic-gate 		cpu_in_chain = 0;
3917c478bd9Sstevel@tonic-gate 		for (cix = 0; cix < NCPU; cix++) {
3927c478bd9Sstevel@tonic-gate 			cpup = cpu[cix];
3937c478bd9Sstevel@tonic-gate 			if (cpup != NULL && CPU_IN_SET(cpus_to_check, cix)) {
3947c478bd9Sstevel@tonic-gate 				if (intr_active(cpup, ipl)) {
3957c478bd9Sstevel@tonic-gate 					cpu_in_chain = 1;
3967c478bd9Sstevel@tonic-gate 				} else {
3977c478bd9Sstevel@tonic-gate 					CPUSET_DEL(cpus_to_check, cix);
3987c478bd9Sstevel@tonic-gate 				}
3997c478bd9Sstevel@tonic-gate 			}
4007c478bd9Sstevel@tonic-gate 		}
4017c478bd9Sstevel@tonic-gate 	} while (cpu_in_chain);
4027c478bd9Sstevel@tonic-gate }
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate /* remove an interrupt vector from the chain */
4057c478bd9Sstevel@tonic-gate static void
4067c478bd9Sstevel@tonic-gate remove_av(void *intr_id, struct av_head *vectp, avfunc f, int pri_level,
4077c478bd9Sstevel@tonic-gate 	int vect)
4087c478bd9Sstevel@tonic-gate {
4097c478bd9Sstevel@tonic-gate 	struct autovec *endp, *p, *target;
4107c478bd9Sstevel@tonic-gate 	int	lo_pri, hi_pri;
4117c478bd9Sstevel@tonic-gate 	int	ipl;
4127c478bd9Sstevel@tonic-gate 	/*
4137c478bd9Sstevel@tonic-gate 	 * Protect rewrites of the list
4147c478bd9Sstevel@tonic-gate 	 */
4157c478bd9Sstevel@tonic-gate 	target = NULL;
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate 	mutex_enter(&av_lock);
4187c478bd9Sstevel@tonic-gate 	ipl = pri_level;
4197c478bd9Sstevel@tonic-gate 	lo_pri = MAXIPL;
4207c478bd9Sstevel@tonic-gate 	hi_pri = 0;
4217c478bd9Sstevel@tonic-gate 	for (endp = p = vectp->avh_link; p && p->av_vector; p = p->av_link) {
4227c478bd9Sstevel@tonic-gate 		endp = p;
4237c478bd9Sstevel@tonic-gate 		if ((p->av_vector == f) && (p->av_intr_id == intr_id)) {
4247c478bd9Sstevel@tonic-gate 			/* found the handler */
4257c478bd9Sstevel@tonic-gate 			target = p;
4267c478bd9Sstevel@tonic-gate 			continue;
4277c478bd9Sstevel@tonic-gate 		}
4287c478bd9Sstevel@tonic-gate 		if (p->av_prilevel > hi_pri)
4297c478bd9Sstevel@tonic-gate 			hi_pri = p->av_prilevel;
4307c478bd9Sstevel@tonic-gate 		if (p->av_prilevel < lo_pri)
4317c478bd9Sstevel@tonic-gate 			lo_pri = p->av_prilevel;
4327c478bd9Sstevel@tonic-gate 	}
4337c478bd9Sstevel@tonic-gate 	if (ipl < hi_pri)
4347c478bd9Sstevel@tonic-gate 		ipl = hi_pri;
4357c478bd9Sstevel@tonic-gate 	if (target == NULL) {	/* not found */
4367c478bd9Sstevel@tonic-gate 		printf("Couldn't remove function %p at %d, %d\n",
4377c478bd9Sstevel@tonic-gate 			(void *)f, vect, pri_level);
4387c478bd9Sstevel@tonic-gate 		mutex_exit(&av_lock);
4397c478bd9Sstevel@tonic-gate 		return;
4407c478bd9Sstevel@tonic-gate 	}
4417c478bd9Sstevel@tonic-gate 
4427c478bd9Sstevel@tonic-gate 	target->av_vector = NULL;
443*7a364d25Sschwartz 	target->av_ticksp = NULL;
4447c478bd9Sstevel@tonic-gate 	wait_till_seen(ipl);
4457c478bd9Sstevel@tonic-gate 	if (endp != target) {	/* vector to be removed is not last in chain */
4460b38a8bdSahl 		target->av_vector = endp->av_vector;
4477c478bd9Sstevel@tonic-gate 		target->av_intarg1 = endp->av_intarg1;
4487c478bd9Sstevel@tonic-gate 		target->av_intarg2 = endp->av_intarg2;
449*7a364d25Sschwartz 		target->av_ticksp = endp->av_ticksp;
4507c478bd9Sstevel@tonic-gate 		target->av_intr_id = endp->av_intr_id;
4510b38a8bdSahl 		target->av_prilevel = endp->av_prilevel;
4520b38a8bdSahl 		target->av_dip = endp->av_dip;
4537c478bd9Sstevel@tonic-gate 		/*
4547c478bd9Sstevel@tonic-gate 		 * We have a hole here where the routine corresponding to
4557c478bd9Sstevel@tonic-gate 		 * endp may not get called. Do a wait_till_seen to take care
4567c478bd9Sstevel@tonic-gate 		 * of this.
4577c478bd9Sstevel@tonic-gate 		 */
4587c478bd9Sstevel@tonic-gate 		wait_till_seen(ipl);
4597c478bd9Sstevel@tonic-gate 		endp->av_vector = NULL;
460*7a364d25Sschwartz 		endp->av_ticksp = NULL;
4617c478bd9Sstevel@tonic-gate 	}
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate 	if (lo_pri > hi_pri) {	/* the chain is now empty */
4647c478bd9Sstevel@tonic-gate 		/* Leave the unused entries here for probable future use */
4657c478bd9Sstevel@tonic-gate 		vectp->avh_lo_pri = MAXIPL;
4667c478bd9Sstevel@tonic-gate 		vectp->avh_hi_pri = 0;
4677c478bd9Sstevel@tonic-gate 	} else {
4687c478bd9Sstevel@tonic-gate 		if ((int)vectp->avh_lo_pri < lo_pri)
4697c478bd9Sstevel@tonic-gate 			vectp->avh_lo_pri = (ushort_t)lo_pri;
4707c478bd9Sstevel@tonic-gate 		if ((int)vectp->avh_hi_pri > hi_pri)
4717c478bd9Sstevel@tonic-gate 			vectp->avh_hi_pri = (ushort_t)hi_pri;
4727c478bd9Sstevel@tonic-gate 	}
4737c478bd9Sstevel@tonic-gate 	mutex_exit(&av_lock);
4747c478bd9Sstevel@tonic-gate 	wait_till_seen(ipl);
4757c478bd9Sstevel@tonic-gate }
4767c478bd9Sstevel@tonic-gate 
4777c478bd9Sstevel@tonic-gate /*
4787c478bd9Sstevel@tonic-gate  * Trigger a soft interrupt.
4797c478bd9Sstevel@tonic-gate  */
4807c478bd9Sstevel@tonic-gate void
4817c478bd9Sstevel@tonic-gate siron(void)
4827c478bd9Sstevel@tonic-gate {
4837c478bd9Sstevel@tonic-gate 	softlevel1_hdl.ih_pending = 1;
4847c478bd9Sstevel@tonic-gate 	(*setsoftint)(1);
4857c478bd9Sstevel@tonic-gate }
4867c478bd9Sstevel@tonic-gate 
4877c478bd9Sstevel@tonic-gate /*
4887c478bd9Sstevel@tonic-gate  * Walk the autovector table for this vector, invoking each
4897c478bd9Sstevel@tonic-gate  * interrupt handler as we go.
4907c478bd9Sstevel@tonic-gate  */
491*7a364d25Sschwartz 
492*7a364d25Sschwartz extern uint64_t intr_get_time(void);
493*7a364d25Sschwartz 
4947c478bd9Sstevel@tonic-gate void
4957c478bd9Sstevel@tonic-gate av_dispatch_autovect(uint_t vec)
4967c478bd9Sstevel@tonic-gate {
4977c478bd9Sstevel@tonic-gate 	struct autovec *av;
4987c478bd9Sstevel@tonic-gate 
4997c478bd9Sstevel@tonic-gate 	ASSERT_STACK_ALIGNED();
5007c478bd9Sstevel@tonic-gate 
5017c478bd9Sstevel@tonic-gate 	while ((av = autovect[vec].avh_link) != NULL) {
5027c478bd9Sstevel@tonic-gate 		uint_t numcalled = 0;
5037c478bd9Sstevel@tonic-gate 		uint_t claimed = 0;
5047c478bd9Sstevel@tonic-gate 
5057c478bd9Sstevel@tonic-gate 		for (; av; av = av->av_link) {
5067c478bd9Sstevel@tonic-gate 			uint_t r;
5077c478bd9Sstevel@tonic-gate 			uint_t (*intr)() = av->av_vector;
5087c478bd9Sstevel@tonic-gate 			caddr_t arg1 = av->av_intarg1;
5097c478bd9Sstevel@tonic-gate 			caddr_t arg2 = av->av_intarg2;
5107c478bd9Sstevel@tonic-gate 			dev_info_t *dip = av->av_dip;
5117c478bd9Sstevel@tonic-gate 
5127c478bd9Sstevel@tonic-gate 			numcalled++;
5137c478bd9Sstevel@tonic-gate 			if (intr == NULL)
5147c478bd9Sstevel@tonic-gate 				break;
5157c478bd9Sstevel@tonic-gate 
5167c478bd9Sstevel@tonic-gate 			DTRACE_PROBE4(interrupt__start, dev_info_t *, dip,
5177c478bd9Sstevel@tonic-gate 			    void *, intr, caddr_t, arg1, caddr_t, arg2);
5187c478bd9Sstevel@tonic-gate 			r = (*intr)(arg1, arg2);
5197c478bd9Sstevel@tonic-gate 			DTRACE_PROBE4(interrupt__complete, dev_info_t *, dip,
5207c478bd9Sstevel@tonic-gate 			    void *, intr, caddr_t, arg1, uint_t, r);
5217c478bd9Sstevel@tonic-gate 			claimed |= r;
522*7a364d25Sschwartz 			if (av->av_ticksp && av->av_prilevel <= LOCK_LEVEL)
523*7a364d25Sschwartz 				atomic_add_64(av->av_ticksp, intr_get_time());
5247c478bd9Sstevel@tonic-gate 		}
5257c478bd9Sstevel@tonic-gate 
5267c478bd9Sstevel@tonic-gate 		/*
5277c478bd9Sstevel@tonic-gate 		 * If there's only one interrupt handler in the chain,
5287c478bd9Sstevel@tonic-gate 		 * or if no-one claimed the interrupt at all give up now.
5297c478bd9Sstevel@tonic-gate 		 */
5307c478bd9Sstevel@tonic-gate 		if (numcalled == 1 || claimed == 0)
5317c478bd9Sstevel@tonic-gate 			break;
5327c478bd9Sstevel@tonic-gate 	}
5337c478bd9Sstevel@tonic-gate }
5347c478bd9Sstevel@tonic-gate 
5357c478bd9Sstevel@tonic-gate /*
5367c478bd9Sstevel@tonic-gate  * Call every soft interrupt handler we can find at this level once.
5377c478bd9Sstevel@tonic-gate  */
5387c478bd9Sstevel@tonic-gate void
5397c478bd9Sstevel@tonic-gate av_dispatch_softvect(uint_t pil)
5407c478bd9Sstevel@tonic-gate {
5417c478bd9Sstevel@tonic-gate 	struct autovec *av;
5427c478bd9Sstevel@tonic-gate 	ddi_softint_hdl_impl_t	*hdlp;
5437c478bd9Sstevel@tonic-gate 	uint_t (*intr)();
5447c478bd9Sstevel@tonic-gate 	caddr_t arg1;
5457c478bd9Sstevel@tonic-gate 	caddr_t arg2;
5467c478bd9Sstevel@tonic-gate 
5477c478bd9Sstevel@tonic-gate 	ASSERT_STACK_ALIGNED();
5487c478bd9Sstevel@tonic-gate 	ASSERT(pil >= 0 && pil <= PIL_MAX);
5497c478bd9Sstevel@tonic-gate 
5507c478bd9Sstevel@tonic-gate 	for (av = softvect[pil].avh_link; av; av = av->av_link) {
5517c478bd9Sstevel@tonic-gate 		if ((intr = av->av_vector) == NULL)
5527c478bd9Sstevel@tonic-gate 			break;
5537c478bd9Sstevel@tonic-gate 		arg1 = av->av_intarg1;
5547c478bd9Sstevel@tonic-gate 		arg2 = av->av_intarg2;
5557c478bd9Sstevel@tonic-gate 
5567c478bd9Sstevel@tonic-gate 		hdlp = (ddi_softint_hdl_impl_t *)av->av_intr_id;
5577c478bd9Sstevel@tonic-gate 		ASSERT(hdlp);
5587c478bd9Sstevel@tonic-gate 
5597c478bd9Sstevel@tonic-gate 		if (hdlp->ih_pending) {
5607c478bd9Sstevel@tonic-gate 			hdlp->ih_pending = 0;
5617c478bd9Sstevel@tonic-gate 			(void) (*intr)(arg1, arg2);
5627c478bd9Sstevel@tonic-gate 		}
5637c478bd9Sstevel@tonic-gate 	}
5647c478bd9Sstevel@tonic-gate }
5657c478bd9Sstevel@tonic-gate 
5667c478bd9Sstevel@tonic-gate struct regs;
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate /*
5697c478bd9Sstevel@tonic-gate  * Call every NMI handler we know of once.
5707c478bd9Sstevel@tonic-gate  */
5717c478bd9Sstevel@tonic-gate void
5727c478bd9Sstevel@tonic-gate av_dispatch_nmivect(struct regs *rp)
5737c478bd9Sstevel@tonic-gate {
5747c478bd9Sstevel@tonic-gate 	struct autovec *av;
5757c478bd9Sstevel@tonic-gate 
5767c478bd9Sstevel@tonic-gate 	ASSERT_STACK_ALIGNED();
5777c478bd9Sstevel@tonic-gate 
5787c478bd9Sstevel@tonic-gate 	for (av = nmivect; av; av = av->av_link)
5797c478bd9Sstevel@tonic-gate 		(void) (av->av_vector)(av->av_intarg1, rp);
5807c478bd9Sstevel@tonic-gate }
581