xref: /freebsd/sys/kern/kern_intr.c (revision 44ad5475222d2043df40fcbe3f5703915ee16468)
19454b2d8SWarner Losh /*-
2425f9fdaSStefan Eßer  * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
3425f9fdaSStefan Eßer  * All rights reserved.
4425f9fdaSStefan Eßer  *
5425f9fdaSStefan Eßer  * Redistribution and use in source and binary forms, with or without
6425f9fdaSStefan Eßer  * modification, are permitted provided that the following conditions
7425f9fdaSStefan Eßer  * are met:
8425f9fdaSStefan Eßer  * 1. Redistributions of source code must retain the above copyright
9425f9fdaSStefan Eßer  *    notice unmodified, this list of conditions, and the following
10425f9fdaSStefan Eßer  *    disclaimer.
11425f9fdaSStefan Eßer  * 2. Redistributions in binary form must reproduce the above copyright
12425f9fdaSStefan Eßer  *    notice, this list of conditions and the following disclaimer in the
13425f9fdaSStefan Eßer  *    documentation and/or other materials provided with the distribution.
14425f9fdaSStefan Eßer  *
15425f9fdaSStefan Eßer  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16425f9fdaSStefan Eßer  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17425f9fdaSStefan Eßer  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18425f9fdaSStefan Eßer  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19425f9fdaSStefan Eßer  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20425f9fdaSStefan Eßer  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21425f9fdaSStefan Eßer  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22425f9fdaSStefan Eßer  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23425f9fdaSStefan Eßer  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24425f9fdaSStefan Eßer  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25425f9fdaSStefan Eßer  */
26425f9fdaSStefan Eßer 
27677b542eSDavid E. O'Brien #include <sys/cdefs.h>
28677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
293900ddb2SDoug Rabson 
308b201c42SJohn Baldwin #include "opt_ddb.h"
318b201c42SJohn Baldwin 
321c5bb3eaSPeter Wemm #include <sys/param.h>
339a94c9c5SJohn Baldwin #include <sys/bus.h>
34c11110eaSAlfred Perlstein #include <sys/conf.h>
359b33b154SJeff Roberson #include <sys/cpuset.h>
369a94c9c5SJohn Baldwin #include <sys/rtprio.h>
37425f9fdaSStefan Eßer #include <sys/systm.h>
3868352337SDoug Rabson #include <sys/interrupt.h>
391931cf94SJohn Baldwin #include <sys/kernel.h>
401931cf94SJohn Baldwin #include <sys/kthread.h>
411931cf94SJohn Baldwin #include <sys/ktr.h>
4205b2c96fSBruce Evans #include <sys/limits.h>
43f34fa851SJohn Baldwin #include <sys/lock.h>
441931cf94SJohn Baldwin #include <sys/malloc.h>
4535e0e5b3SJohn Baldwin #include <sys/mutex.h>
46cebc7fb1SJohn Baldwin #include <sys/priv.h>
471931cf94SJohn Baldwin #include <sys/proc.h>
483e5da754SJohn Baldwin #include <sys/random.h>
49b4151f71SJohn Baldwin #include <sys/resourcevar.h>
5063710c4dSJohn Baldwin #include <sys/sched.h>
51eaf86d16SJohn Baldwin #include <sys/smp.h>
52d279178dSThomas Moestl #include <sys/sysctl.h>
536205924aSKip Macy #include <sys/syslog.h>
541931cf94SJohn Baldwin #include <sys/unistd.h>
551931cf94SJohn Baldwin #include <sys/vmmeter.h>
561931cf94SJohn Baldwin #include <machine/atomic.h>
571931cf94SJohn Baldwin #include <machine/cpu.h>
588088699fSJohn Baldwin #include <machine/md_var.h>
59b4151f71SJohn Baldwin #include <machine/stdarg.h>
608b201c42SJohn Baldwin #ifdef DDB
618b201c42SJohn Baldwin #include <ddb/ddb.h>
628b201c42SJohn Baldwin #include <ddb/db_sym.h>
638b201c42SJohn Baldwin #endif
64425f9fdaSStefan Eßer 
65e0f66ef8SJohn Baldwin /*
66e0f66ef8SJohn Baldwin  * Describe an interrupt thread.  There is one of these per interrupt event.
67e0f66ef8SJohn Baldwin  */
68e0f66ef8SJohn Baldwin struct intr_thread {
69e0f66ef8SJohn Baldwin 	struct intr_event *it_event;
70e0f66ef8SJohn Baldwin 	struct thread *it_thread;	/* Kernel thread. */
71e0f66ef8SJohn Baldwin 	int	it_flags;		/* (j) IT_* flags. */
72e0f66ef8SJohn Baldwin 	int	it_need;		/* Needs service. */
733e5da754SJohn Baldwin };
743e5da754SJohn Baldwin 
75e0f66ef8SJohn Baldwin /* Interrupt thread flags kept in it_flags */
76e0f66ef8SJohn Baldwin #define	IT_DEAD		0x000001	/* Thread is waiting to exit. */
77e4cd31ddSJeff Roberson #define	IT_WAIT		0x000002	/* Thread is waiting for completion. */
78e0f66ef8SJohn Baldwin 
79e0f66ef8SJohn Baldwin struct	intr_entropy {
80e0f66ef8SJohn Baldwin 	struct	thread *td;
81e0f66ef8SJohn Baldwin 	uintptr_t event;
82e0f66ef8SJohn Baldwin };
83e0f66ef8SJohn Baldwin 
84e0f66ef8SJohn Baldwin struct	intr_event *clk_intr_event;
85e0f66ef8SJohn Baldwin struct	intr_event *tty_intr_event;
867b1fe905SBruce Evans void	*vm_ih;
877ab24ea3SJulian Elischer struct proc *intrproc;
881931cf94SJohn Baldwin 
89b4151f71SJohn Baldwin static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads");
90b4151f71SJohn Baldwin 
910ae62c18SNate Lawson static int intr_storm_threshold = 1000;
927870c3c6SJohn Baldwin TUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold);
937870c3c6SJohn Baldwin SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW,
947870c3c6SJohn Baldwin     &intr_storm_threshold, 0,
957b1fe905SBruce Evans     "Number of consecutive interrupts before storm protection is enabled");
96e0f66ef8SJohn Baldwin static TAILQ_HEAD(, intr_event) event_list =
97e0f66ef8SJohn Baldwin     TAILQ_HEAD_INITIALIZER(event_list);
989b33b154SJeff Roberson static struct mtx event_lock;
999b33b154SJeff Roberson MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF);
1007b1fe905SBruce Evans 
101e0f66ef8SJohn Baldwin static void	intr_event_update(struct intr_event *ie);
102bafe5a31SPaolo Pisati #ifdef INTR_FILTER
1031ee1b687SJohn Baldwin static int	intr_event_schedule_thread(struct intr_event *ie,
1041ee1b687SJohn Baldwin 		    struct intr_thread *ithd);
1051ee1b687SJohn Baldwin static int	intr_filter_loop(struct intr_event *ie,
1061ee1b687SJohn Baldwin 		    struct trapframe *frame, struct intr_thread **ithd);
107bafe5a31SPaolo Pisati static struct intr_thread *ithread_create(const char *name,
108bafe5a31SPaolo Pisati 			      struct intr_handler *ih);
109bafe5a31SPaolo Pisati #else
1101ee1b687SJohn Baldwin static int	intr_event_schedule_thread(struct intr_event *ie);
111e0f66ef8SJohn Baldwin static struct intr_thread *ithread_create(const char *name);
112bafe5a31SPaolo Pisati #endif
113e0f66ef8SJohn Baldwin static void	ithread_destroy(struct intr_thread *ithread);
114bafe5a31SPaolo Pisati static void	ithread_execute_handlers(struct proc *p,
115bafe5a31SPaolo Pisati 		    struct intr_event *ie);
116bafe5a31SPaolo Pisati #ifdef INTR_FILTER
117bafe5a31SPaolo Pisati static void	priv_ithread_execute_handler(struct proc *p,
118bafe5a31SPaolo Pisati 		    struct intr_handler *ih);
119bafe5a31SPaolo Pisati #endif
1207b1fe905SBruce Evans static void	ithread_loop(void *);
121e0f66ef8SJohn Baldwin static void	ithread_update(struct intr_thread *ithd);
1227b1fe905SBruce Evans static void	start_softintr(void *);
1237870c3c6SJohn Baldwin 
124bc17acb2SJohn Baldwin /* Map an interrupt type to an ithread priority. */
125b4151f71SJohn Baldwin u_char
126e0f66ef8SJohn Baldwin intr_priority(enum intr_type flags)
1279a94c9c5SJohn Baldwin {
128b4151f71SJohn Baldwin 	u_char pri;
1299a94c9c5SJohn Baldwin 
130b4151f71SJohn Baldwin 	flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET |
1315a280d9cSPeter Wemm 	    INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV);
1329a94c9c5SJohn Baldwin 	switch (flags) {
133b4151f71SJohn Baldwin 	case INTR_TYPE_TTY:
134d3305205SJohn Baldwin 		pri = PI_TTY;
1359a94c9c5SJohn Baldwin 		break;
1369a94c9c5SJohn Baldwin 	case INTR_TYPE_BIO:
1379a94c9c5SJohn Baldwin 		pri = PI_DISK;
1389a94c9c5SJohn Baldwin 		break;
1399a94c9c5SJohn Baldwin 	case INTR_TYPE_NET:
1409a94c9c5SJohn Baldwin 		pri = PI_NET;
1419a94c9c5SJohn Baldwin 		break;
1429a94c9c5SJohn Baldwin 	case INTR_TYPE_CAM:
143d3305205SJohn Baldwin 		pri = PI_DISK;
1449a94c9c5SJohn Baldwin 		break;
145d3305205SJohn Baldwin 	case INTR_TYPE_AV:
1465a280d9cSPeter Wemm 		pri = PI_AV;
1475a280d9cSPeter Wemm 		break;
148b4151f71SJohn Baldwin 	case INTR_TYPE_CLK:
149b4151f71SJohn Baldwin 		pri = PI_REALTIME;
150b4151f71SJohn Baldwin 		break;
1519a94c9c5SJohn Baldwin 	case INTR_TYPE_MISC:
1529a94c9c5SJohn Baldwin 		pri = PI_DULL;          /* don't care */
1539a94c9c5SJohn Baldwin 		break;
1549a94c9c5SJohn Baldwin 	default:
155b4151f71SJohn Baldwin 		/* We didn't specify an interrupt level. */
156e0f66ef8SJohn Baldwin 		panic("intr_priority: no interrupt type in flags");
1579a94c9c5SJohn Baldwin 	}
1589a94c9c5SJohn Baldwin 
1599a94c9c5SJohn Baldwin 	return pri;
1609a94c9c5SJohn Baldwin }
1619a94c9c5SJohn Baldwin 
162b4151f71SJohn Baldwin /*
163e0f66ef8SJohn Baldwin  * Update an ithread based on the associated intr_event.
164b4151f71SJohn Baldwin  */
165b4151f71SJohn Baldwin static void
166e0f66ef8SJohn Baldwin ithread_update(struct intr_thread *ithd)
167b4151f71SJohn Baldwin {
168e0f66ef8SJohn Baldwin 	struct intr_event *ie;
169b40ce416SJulian Elischer 	struct thread *td;
170e0f66ef8SJohn Baldwin 	u_char pri;
1718088699fSJohn Baldwin 
172e0f66ef8SJohn Baldwin 	ie = ithd->it_event;
173e0f66ef8SJohn Baldwin 	td = ithd->it_thread;
174b4151f71SJohn Baldwin 
175e0f66ef8SJohn Baldwin 	/* Determine the overall priority of this event. */
176e0f66ef8SJohn Baldwin 	if (TAILQ_EMPTY(&ie->ie_handlers))
177e0f66ef8SJohn Baldwin 		pri = PRI_MAX_ITHD;
178e0f66ef8SJohn Baldwin 	else
179e0f66ef8SJohn Baldwin 		pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri;
180e80fb434SRobert Drehmel 
181e0f66ef8SJohn Baldwin 	/* Update name and priority. */
1827ab24ea3SJulian Elischer 	strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name));
183*44ad5475SJohn Baldwin #ifdef KTR
184*44ad5475SJohn Baldwin 	sched_clear_tdname(td);
185*44ad5475SJohn Baldwin #endif
186982d11f8SJeff Roberson 	thread_lock(td);
187e0f66ef8SJohn Baldwin 	sched_prio(td, pri);
188982d11f8SJeff Roberson 	thread_unlock(td);
189b4151f71SJohn Baldwin }
190e0f66ef8SJohn Baldwin 
191e0f66ef8SJohn Baldwin /*
192e0f66ef8SJohn Baldwin  * Regenerate the full name of an interrupt event and update its priority.
193e0f66ef8SJohn Baldwin  */
194e0f66ef8SJohn Baldwin static void
195e0f66ef8SJohn Baldwin intr_event_update(struct intr_event *ie)
196e0f66ef8SJohn Baldwin {
197e0f66ef8SJohn Baldwin 	struct intr_handler *ih;
198e0f66ef8SJohn Baldwin 	char *last;
199e0f66ef8SJohn Baldwin 	int missed, space;
200e0f66ef8SJohn Baldwin 
201e0f66ef8SJohn Baldwin 	/* Start off with no entropy and just the name of the event. */
202e0f66ef8SJohn Baldwin 	mtx_assert(&ie->ie_lock, MA_OWNED);
203e0f66ef8SJohn Baldwin 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
204e0f66ef8SJohn Baldwin 	ie->ie_flags &= ~IE_ENTROPY;
2050811d60aSJohn Baldwin 	missed = 0;
206e0f66ef8SJohn Baldwin 	space = 1;
207e0f66ef8SJohn Baldwin 
208e0f66ef8SJohn Baldwin 	/* Run through all the handlers updating values. */
209e0f66ef8SJohn Baldwin 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
210e0f66ef8SJohn Baldwin 		if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
211e0f66ef8SJohn Baldwin 		    sizeof(ie->ie_fullname)) {
212e0f66ef8SJohn Baldwin 			strcat(ie->ie_fullname, " ");
213e0f66ef8SJohn Baldwin 			strcat(ie->ie_fullname, ih->ih_name);
214e0f66ef8SJohn Baldwin 			space = 0;
2150811d60aSJohn Baldwin 		} else
2160811d60aSJohn Baldwin 			missed++;
2170811d60aSJohn Baldwin 		if (ih->ih_flags & IH_ENTROPY)
218e0f66ef8SJohn Baldwin 			ie->ie_flags |= IE_ENTROPY;
2190811d60aSJohn Baldwin 	}
220e0f66ef8SJohn Baldwin 
221e0f66ef8SJohn Baldwin 	/*
222e0f66ef8SJohn Baldwin 	 * If the handler names were too long, add +'s to indicate missing
223e0f66ef8SJohn Baldwin 	 * names. If we run out of room and still have +'s to add, change
224e0f66ef8SJohn Baldwin 	 * the last character from a + to a *.
225e0f66ef8SJohn Baldwin 	 */
226e0f66ef8SJohn Baldwin 	last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
2270811d60aSJohn Baldwin 	while (missed-- > 0) {
228e0f66ef8SJohn Baldwin 		if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
229e0f66ef8SJohn Baldwin 			if (*last == '+') {
230e0f66ef8SJohn Baldwin 				*last = '*';
231e0f66ef8SJohn Baldwin 				break;
232b4151f71SJohn Baldwin 			} else
233e0f66ef8SJohn Baldwin 				*last = '+';
234e0f66ef8SJohn Baldwin 		} else if (space) {
235e0f66ef8SJohn Baldwin 			strcat(ie->ie_fullname, " +");
236e0f66ef8SJohn Baldwin 			space = 0;
237e0f66ef8SJohn Baldwin 		} else
238e0f66ef8SJohn Baldwin 			strcat(ie->ie_fullname, "+");
239b4151f71SJohn Baldwin 	}
240e0f66ef8SJohn Baldwin 
241e0f66ef8SJohn Baldwin 	/*
242e0f66ef8SJohn Baldwin 	 * If this event has an ithread, update it's priority and
243e0f66ef8SJohn Baldwin 	 * name.
244e0f66ef8SJohn Baldwin 	 */
245e0f66ef8SJohn Baldwin 	if (ie->ie_thread != NULL)
246e0f66ef8SJohn Baldwin 		ithread_update(ie->ie_thread);
247e0f66ef8SJohn Baldwin 	CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
248b4151f71SJohn Baldwin }
249b4151f71SJohn Baldwin 
250b4151f71SJohn Baldwin int
2519b33b154SJeff Roberson intr_event_create(struct intr_event **event, void *source, int flags, int irq,
2521ee1b687SJohn Baldwin     void (*pre_ithread)(void *), void (*post_ithread)(void *),
2531ee1b687SJohn Baldwin     void (*post_filter)(void *), int (*assign_cpu)(void *, u_char),
2541ee1b687SJohn Baldwin     const char *fmt, ...)
255bafe5a31SPaolo Pisati {
256bafe5a31SPaolo Pisati 	struct intr_event *ie;
257bafe5a31SPaolo Pisati 	va_list ap;
258bafe5a31SPaolo Pisati 
259bafe5a31SPaolo Pisati 	/* The only valid flag during creation is IE_SOFT. */
260bafe5a31SPaolo Pisati 	if ((flags & ~IE_SOFT) != 0)
261bafe5a31SPaolo Pisati 		return (EINVAL);
262bafe5a31SPaolo Pisati 	ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
263bafe5a31SPaolo Pisati 	ie->ie_source = source;
2641ee1b687SJohn Baldwin 	ie->ie_pre_ithread = pre_ithread;
2651ee1b687SJohn Baldwin 	ie->ie_post_ithread = post_ithread;
2661ee1b687SJohn Baldwin 	ie->ie_post_filter = post_filter;
2676d2d1c04SJohn Baldwin 	ie->ie_assign_cpu = assign_cpu;
268bafe5a31SPaolo Pisati 	ie->ie_flags = flags;
2699b33b154SJeff Roberson 	ie->ie_irq = irq;
270eaf86d16SJohn Baldwin 	ie->ie_cpu = NOCPU;
271bafe5a31SPaolo Pisati 	TAILQ_INIT(&ie->ie_handlers);
272bafe5a31SPaolo Pisati 	mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
273bafe5a31SPaolo Pisati 
274bafe5a31SPaolo Pisati 	va_start(ap, fmt);
275bafe5a31SPaolo Pisati 	vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
276bafe5a31SPaolo Pisati 	va_end(ap);
277bafe5a31SPaolo Pisati 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
2789b33b154SJeff Roberson 	mtx_lock(&event_lock);
279bafe5a31SPaolo Pisati 	TAILQ_INSERT_TAIL(&event_list, ie, ie_list);
2809b33b154SJeff Roberson 	mtx_unlock(&event_lock);
281bafe5a31SPaolo Pisati 	if (event != NULL)
282bafe5a31SPaolo Pisati 		*event = ie;
283bafe5a31SPaolo Pisati 	CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
284bafe5a31SPaolo Pisati 	return (0);
285bafe5a31SPaolo Pisati }
286b4151f71SJohn Baldwin 
287eaf86d16SJohn Baldwin /*
288eaf86d16SJohn Baldwin  * Bind an interrupt event to the specified CPU.  Note that not all
289eaf86d16SJohn Baldwin  * platforms support binding an interrupt to a CPU.  For those
290eaf86d16SJohn Baldwin  * platforms this request will fail.  For supported platforms, any
291eaf86d16SJohn Baldwin  * associated ithreads as well as the primary interrupt context will
292eaf86d16SJohn Baldwin  * be bound to the specificed CPU.  Using a cpu id of NOCPU unbinds
293eaf86d16SJohn Baldwin  * the interrupt event.
294eaf86d16SJohn Baldwin  */
295eaf86d16SJohn Baldwin int
296eaf86d16SJohn Baldwin intr_event_bind(struct intr_event *ie, u_char cpu)
297eaf86d16SJohn Baldwin {
2989b33b154SJeff Roberson 	cpuset_t mask;
2999b33b154SJeff Roberson 	lwpid_t id;
300eaf86d16SJohn Baldwin 	int error;
301eaf86d16SJohn Baldwin 
302eaf86d16SJohn Baldwin 	/* Need a CPU to bind to. */
303eaf86d16SJohn Baldwin 	if (cpu != NOCPU && CPU_ABSENT(cpu))
304eaf86d16SJohn Baldwin 		return (EINVAL);
305eaf86d16SJohn Baldwin 
306eaf86d16SJohn Baldwin 	if (ie->ie_assign_cpu == NULL)
307eaf86d16SJohn Baldwin 		return (EOPNOTSUPP);
308cebc7fb1SJohn Baldwin 
309cebc7fb1SJohn Baldwin 	error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR);
310cebc7fb1SJohn Baldwin 	if (error)
311cebc7fb1SJohn Baldwin 		return (error);
312cebc7fb1SJohn Baldwin 
3139b33b154SJeff Roberson 	/*
314cebc7fb1SJohn Baldwin 	 * If we have any ithreads try to set their mask first to verify
315cebc7fb1SJohn Baldwin 	 * permissions, etc.
3169b33b154SJeff Roberson 	 */
317eaf86d16SJohn Baldwin 	mtx_lock(&ie->ie_lock);
3189b33b154SJeff Roberson 	if (ie->ie_thread != NULL) {
3199b33b154SJeff Roberson 		CPU_ZERO(&mask);
3209b33b154SJeff Roberson 		if (cpu == NOCPU)
3219b33b154SJeff Roberson 			CPU_COPY(cpuset_root, &mask);
3229b33b154SJeff Roberson 		else
3239b33b154SJeff Roberson 			CPU_SET(cpu, &mask);
3249b33b154SJeff Roberson 		id = ie->ie_thread->it_thread->td_tid;
325eaf86d16SJohn Baldwin 		mtx_unlock(&ie->ie_lock);
3269b33b154SJeff Roberson 		error = cpuset_setthread(id, &mask);
3279b33b154SJeff Roberson 		if (error)
3289b33b154SJeff Roberson 			return (error);
3299b33b154SJeff Roberson 	} else
330eaf86d16SJohn Baldwin 		mtx_unlock(&ie->ie_lock);
331eaf86d16SJohn Baldwin 	error = ie->ie_assign_cpu(ie->ie_source, cpu);
332cebc7fb1SJohn Baldwin 	if (error) {
333cebc7fb1SJohn Baldwin 		mtx_lock(&ie->ie_lock);
334cebc7fb1SJohn Baldwin 		if (ie->ie_thread != NULL) {
335cebc7fb1SJohn Baldwin 			CPU_ZERO(&mask);
336cebc7fb1SJohn Baldwin 			if (ie->ie_cpu == NOCPU)
337cebc7fb1SJohn Baldwin 				CPU_COPY(cpuset_root, &mask);
338cebc7fb1SJohn Baldwin 			else
339cebc7fb1SJohn Baldwin 				CPU_SET(cpu, &mask);
340cebc7fb1SJohn Baldwin 			id = ie->ie_thread->it_thread->td_tid;
341cebc7fb1SJohn Baldwin 			mtx_unlock(&ie->ie_lock);
342cebc7fb1SJohn Baldwin 			(void)cpuset_setthread(id, &mask);
343cebc7fb1SJohn Baldwin 		} else
344cebc7fb1SJohn Baldwin 			mtx_unlock(&ie->ie_lock);
345eaf86d16SJohn Baldwin 		return (error);
346cebc7fb1SJohn Baldwin 	}
347cebc7fb1SJohn Baldwin 
348eaf86d16SJohn Baldwin 	mtx_lock(&ie->ie_lock);
349eaf86d16SJohn Baldwin 	ie->ie_cpu = cpu;
3509b33b154SJeff Roberson 	mtx_unlock(&ie->ie_lock);
3519b33b154SJeff Roberson 
3529b33b154SJeff Roberson 	return (error);
3539b33b154SJeff Roberson }
3549b33b154SJeff Roberson 
3559b33b154SJeff Roberson static struct intr_event *
3569b33b154SJeff Roberson intr_lookup(int irq)
3579b33b154SJeff Roberson {
3589b33b154SJeff Roberson 	struct intr_event *ie;
3599b33b154SJeff Roberson 
3609b33b154SJeff Roberson 	mtx_lock(&event_lock);
3619b33b154SJeff Roberson 	TAILQ_FOREACH(ie, &event_list, ie_list)
3629b33b154SJeff Roberson 		if (ie->ie_irq == irq &&
3639b33b154SJeff Roberson 		    (ie->ie_flags & IE_SOFT) == 0 &&
3649b33b154SJeff Roberson 		    TAILQ_FIRST(&ie->ie_handlers) != NULL)
3659b33b154SJeff Roberson 			break;
3669b33b154SJeff Roberson 	mtx_unlock(&event_lock);
3679b33b154SJeff Roberson 	return (ie);
3689b33b154SJeff Roberson }
3699b33b154SJeff Roberson 
3709b33b154SJeff Roberson int
3719b33b154SJeff Roberson intr_setaffinity(int irq, void *m)
3729b33b154SJeff Roberson {
3739b33b154SJeff Roberson 	struct intr_event *ie;
3749b33b154SJeff Roberson 	cpuset_t *mask;
3759b33b154SJeff Roberson 	u_char cpu;
3769b33b154SJeff Roberson 	int n;
3779b33b154SJeff Roberson 
3789b33b154SJeff Roberson 	mask = m;
3799b33b154SJeff Roberson 	cpu = NOCPU;
3809b33b154SJeff Roberson 	/*
3819b33b154SJeff Roberson 	 * If we're setting all cpus we can unbind.  Otherwise make sure
3829b33b154SJeff Roberson 	 * only one cpu is in the set.
3839b33b154SJeff Roberson 	 */
3849b33b154SJeff Roberson 	if (CPU_CMP(cpuset_root, mask)) {
3859b33b154SJeff Roberson 		for (n = 0; n < CPU_SETSIZE; n++) {
3869b33b154SJeff Roberson 			if (!CPU_ISSET(n, mask))
3879b33b154SJeff Roberson 				continue;
3889b33b154SJeff Roberson 			if (cpu != NOCPU)
3899b33b154SJeff Roberson 				return (EINVAL);
3909b33b154SJeff Roberson 			cpu = (u_char)n;
3919b33b154SJeff Roberson 		}
3929b33b154SJeff Roberson 	}
3939b33b154SJeff Roberson 	ie = intr_lookup(irq);
3949b33b154SJeff Roberson 	if (ie == NULL)
3959b33b154SJeff Roberson 		return (ESRCH);
3969bd55acfSJohn Baldwin 	return (intr_event_bind(ie, cpu));
3979b33b154SJeff Roberson }
3989b33b154SJeff Roberson 
3999b33b154SJeff Roberson int
4009b33b154SJeff Roberson intr_getaffinity(int irq, void *m)
4019b33b154SJeff Roberson {
4029b33b154SJeff Roberson 	struct intr_event *ie;
4039b33b154SJeff Roberson 	cpuset_t *mask;
4049b33b154SJeff Roberson 
4059b33b154SJeff Roberson 	mask = m;
4069b33b154SJeff Roberson 	ie = intr_lookup(irq);
4079b33b154SJeff Roberson 	if (ie == NULL)
4089b33b154SJeff Roberson 		return (ESRCH);
4099b33b154SJeff Roberson 	CPU_ZERO(mask);
4109b33b154SJeff Roberson 	mtx_lock(&ie->ie_lock);
4119b33b154SJeff Roberson 	if (ie->ie_cpu == NOCPU)
4129b33b154SJeff Roberson 		CPU_COPY(cpuset_root, mask);
4139b33b154SJeff Roberson 	else
4149b33b154SJeff Roberson 		CPU_SET(ie->ie_cpu, mask);
415eaf86d16SJohn Baldwin 	mtx_unlock(&ie->ie_lock);
416eaf86d16SJohn Baldwin 	return (0);
417eaf86d16SJohn Baldwin }
418eaf86d16SJohn Baldwin 
419b4151f71SJohn Baldwin int
420e0f66ef8SJohn Baldwin intr_event_destroy(struct intr_event *ie)
421b4151f71SJohn Baldwin {
422b4151f71SJohn Baldwin 
4239b33b154SJeff Roberson 	mtx_lock(&event_lock);
424e0f66ef8SJohn Baldwin 	mtx_lock(&ie->ie_lock);
425e0f66ef8SJohn Baldwin 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
426e0f66ef8SJohn Baldwin 		mtx_unlock(&ie->ie_lock);
4279b33b154SJeff Roberson 		mtx_unlock(&event_lock);
428e0f66ef8SJohn Baldwin 		return (EBUSY);
4294d29cb2dSJohn Baldwin 	}
430e0f66ef8SJohn Baldwin 	TAILQ_REMOVE(&event_list, ie, ie_list);
4319477358dSJohn Baldwin #ifndef notyet
4329477358dSJohn Baldwin 	if (ie->ie_thread != NULL) {
4339477358dSJohn Baldwin 		ithread_destroy(ie->ie_thread);
4349477358dSJohn Baldwin 		ie->ie_thread = NULL;
4359477358dSJohn Baldwin 	}
4369477358dSJohn Baldwin #endif
437e0f66ef8SJohn Baldwin 	mtx_unlock(&ie->ie_lock);
4389b33b154SJeff Roberson 	mtx_unlock(&event_lock);
439e0f66ef8SJohn Baldwin 	mtx_destroy(&ie->ie_lock);
440e0f66ef8SJohn Baldwin 	free(ie, M_ITHREAD);
441e0f66ef8SJohn Baldwin 	return (0);
442e0f66ef8SJohn Baldwin }
443e0f66ef8SJohn Baldwin 
444bafe5a31SPaolo Pisati #ifndef INTR_FILTER
445e0f66ef8SJohn Baldwin static struct intr_thread *
446e0f66ef8SJohn Baldwin ithread_create(const char *name)
447e0f66ef8SJohn Baldwin {
448e0f66ef8SJohn Baldwin 	struct intr_thread *ithd;
449e0f66ef8SJohn Baldwin 	struct thread *td;
450e0f66ef8SJohn Baldwin 	int error;
451e0f66ef8SJohn Baldwin 
452e0f66ef8SJohn Baldwin 	ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
453e0f66ef8SJohn Baldwin 
4547ab24ea3SJulian Elischer 	error = kproc_kthread_add(ithread_loop, ithd, &intrproc,
4557ab24ea3SJulian Elischer 		    &td, RFSTOPPED | RFHIGHPID,
4569ef95d01SJulian Elischer 	    	    0, "intr", "%s", name);
457e0f66ef8SJohn Baldwin 	if (error)
4583745c395SJulian Elischer 		panic("kproc_create() failed with %d", error);
459982d11f8SJeff Roberson 	thread_lock(td);
460ad1e7d28SJulian Elischer 	sched_class(td, PRI_ITHD);
461e0f66ef8SJohn Baldwin 	TD_SET_IWAIT(td);
462982d11f8SJeff Roberson 	thread_unlock(td);
463e0f66ef8SJohn Baldwin 	td->td_pflags |= TDP_ITHREAD;
464e0f66ef8SJohn Baldwin 	ithd->it_thread = td;
465e0f66ef8SJohn Baldwin 	CTR2(KTR_INTR, "%s: created %s", __func__, name);
466e0f66ef8SJohn Baldwin 	return (ithd);
467e0f66ef8SJohn Baldwin }
468bafe5a31SPaolo Pisati #else
469bafe5a31SPaolo Pisati static struct intr_thread *
470bafe5a31SPaolo Pisati ithread_create(const char *name, struct intr_handler *ih)
471bafe5a31SPaolo Pisati {
472bafe5a31SPaolo Pisati 	struct intr_thread *ithd;
473bafe5a31SPaolo Pisati 	struct thread *td;
474bafe5a31SPaolo Pisati 	int error;
475bafe5a31SPaolo Pisati 
476bafe5a31SPaolo Pisati 	ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
477bafe5a31SPaolo Pisati 
478539976ffSJulian Elischer 	error = kproc_kthread_add(ithread_loop, ih, &intrproc,
4797ab24ea3SJulian Elischer 		    &td, RFSTOPPED | RFHIGHPID,
4809ef95d01SJulian Elischer 	    	    0, "intr", "%s", name);
481bafe5a31SPaolo Pisati 	if (error)
4823745c395SJulian Elischer 		panic("kproc_create() failed with %d", error);
483982d11f8SJeff Roberson 	thread_lock(td);
484bafe5a31SPaolo Pisati 	sched_class(td, PRI_ITHD);
485bafe5a31SPaolo Pisati 	TD_SET_IWAIT(td);
486982d11f8SJeff Roberson 	thread_unlock(td);
487bafe5a31SPaolo Pisati 	td->td_pflags |= TDP_ITHREAD;
488bafe5a31SPaolo Pisati 	ithd->it_thread = td;
489bafe5a31SPaolo Pisati 	CTR2(KTR_INTR, "%s: created %s", __func__, name);
490bafe5a31SPaolo Pisati 	return (ithd);
491bafe5a31SPaolo Pisati }
492bafe5a31SPaolo Pisati #endif
493e0f66ef8SJohn Baldwin 
494e0f66ef8SJohn Baldwin static void
495e0f66ef8SJohn Baldwin ithread_destroy(struct intr_thread *ithread)
496e0f66ef8SJohn Baldwin {
497e0f66ef8SJohn Baldwin 	struct thread *td;
498e0f66ef8SJohn Baldwin 
499bb141be1SScott Long 	CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name);
500e0f66ef8SJohn Baldwin 	td = ithread->it_thread;
501982d11f8SJeff Roberson 	thread_lock(td);
502e0f66ef8SJohn Baldwin 	ithread->it_flags |= IT_DEAD;
50371fad9fdSJulian Elischer 	if (TD_AWAITING_INTR(td)) {
50471fad9fdSJulian Elischer 		TD_CLR_IWAIT(td);
505f0393f06SJeff Roberson 		sched_add(td, SRQ_INTR);
506b4151f71SJohn Baldwin 	}
507982d11f8SJeff Roberson 	thread_unlock(td);
508b4151f71SJohn Baldwin }
509b4151f71SJohn Baldwin 
510bafe5a31SPaolo Pisati #ifndef INTR_FILTER
511b4151f71SJohn Baldwin int
512e0f66ef8SJohn Baldwin intr_event_add_handler(struct intr_event *ie, const char *name,
513ef544f63SPaolo Pisati     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
514ef544f63SPaolo Pisati     enum intr_type flags, void **cookiep)
515b4151f71SJohn Baldwin {
516e0f66ef8SJohn Baldwin 	struct intr_handler *ih, *temp_ih;
517e0f66ef8SJohn Baldwin 	struct intr_thread *it;
518b4151f71SJohn Baldwin 
519ef544f63SPaolo Pisati 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
520b4151f71SJohn Baldwin 		return (EINVAL);
521b4151f71SJohn Baldwin 
522e0f66ef8SJohn Baldwin 	/* Allocate and populate an interrupt handler structure. */
523e0f66ef8SJohn Baldwin 	ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
524ef544f63SPaolo Pisati 	ih->ih_filter = filter;
525b4151f71SJohn Baldwin 	ih->ih_handler = handler;
526b4151f71SJohn Baldwin 	ih->ih_argument = arg;
52737b8ef16SJohn Baldwin 	strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
528e0f66ef8SJohn Baldwin 	ih->ih_event = ie;
529b4151f71SJohn Baldwin 	ih->ih_pri = pri;
530ef544f63SPaolo Pisati 	if (flags & INTR_EXCL)
531b4151f71SJohn Baldwin 		ih->ih_flags = IH_EXCLUSIVE;
532b4151f71SJohn Baldwin 	if (flags & INTR_MPSAFE)
533b4151f71SJohn Baldwin 		ih->ih_flags |= IH_MPSAFE;
534b4151f71SJohn Baldwin 	if (flags & INTR_ENTROPY)
535b4151f71SJohn Baldwin 		ih->ih_flags |= IH_ENTROPY;
536b4151f71SJohn Baldwin 
537e0f66ef8SJohn Baldwin 	/* We can only have one exclusive handler in a event. */
538e0f66ef8SJohn Baldwin 	mtx_lock(&ie->ie_lock);
539e0f66ef8SJohn Baldwin 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
540e0f66ef8SJohn Baldwin 		if ((flags & INTR_EXCL) ||
541e0f66ef8SJohn Baldwin 		    (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
542e0f66ef8SJohn Baldwin 			mtx_unlock(&ie->ie_lock);
543b4151f71SJohn Baldwin 			free(ih, M_ITHREAD);
544b4151f71SJohn Baldwin 			return (EINVAL);
545b4151f71SJohn Baldwin 		}
546e0f66ef8SJohn Baldwin 	}
547e0f66ef8SJohn Baldwin 
548e0f66ef8SJohn Baldwin 	/* Add the new handler to the event in priority order. */
549e0f66ef8SJohn Baldwin 	TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
550e0f66ef8SJohn Baldwin 		if (temp_ih->ih_pri > ih->ih_pri)
551e0f66ef8SJohn Baldwin 			break;
552e0f66ef8SJohn Baldwin 	}
553e0f66ef8SJohn Baldwin 	if (temp_ih == NULL)
554e0f66ef8SJohn Baldwin 		TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
555e0f66ef8SJohn Baldwin 	else
556e0f66ef8SJohn Baldwin 		TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
557e0f66ef8SJohn Baldwin 	intr_event_update(ie);
558e0f66ef8SJohn Baldwin 
559e0f66ef8SJohn Baldwin 	/* Create a thread if we need one. */
560ef544f63SPaolo Pisati 	while (ie->ie_thread == NULL && handler != NULL) {
561e0f66ef8SJohn Baldwin 		if (ie->ie_flags & IE_ADDING_THREAD)
5620f180a7cSJohn Baldwin 			msleep(ie, &ie->ie_lock, 0, "ithread", 0);
563e0f66ef8SJohn Baldwin 		else {
564e0f66ef8SJohn Baldwin 			ie->ie_flags |= IE_ADDING_THREAD;
565e0f66ef8SJohn Baldwin 			mtx_unlock(&ie->ie_lock);
566e0f66ef8SJohn Baldwin 			it = ithread_create("intr: newborn");
567e0f66ef8SJohn Baldwin 			mtx_lock(&ie->ie_lock);
568e0f66ef8SJohn Baldwin 			ie->ie_flags &= ~IE_ADDING_THREAD;
569e0f66ef8SJohn Baldwin 			ie->ie_thread = it;
570e0f66ef8SJohn Baldwin 			it->it_event = ie;
571e0f66ef8SJohn Baldwin 			ithread_update(it);
572e0f66ef8SJohn Baldwin 			wakeup(ie);
573e0f66ef8SJohn Baldwin 		}
574e0f66ef8SJohn Baldwin 	}
575e0f66ef8SJohn Baldwin 	CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
576e0f66ef8SJohn Baldwin 	    ie->ie_name);
577e0f66ef8SJohn Baldwin 	mtx_unlock(&ie->ie_lock);
578e0f66ef8SJohn Baldwin 
579e0f66ef8SJohn Baldwin 	if (cookiep != NULL)
580e0f66ef8SJohn Baldwin 		*cookiep = ih;
581e0f66ef8SJohn Baldwin 	return (0);
582e0f66ef8SJohn Baldwin }
583bafe5a31SPaolo Pisati #else
584bafe5a31SPaolo Pisati int
585bafe5a31SPaolo Pisati intr_event_add_handler(struct intr_event *ie, const char *name,
586bafe5a31SPaolo Pisati     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
587bafe5a31SPaolo Pisati     enum intr_type flags, void **cookiep)
588bafe5a31SPaolo Pisati {
589bafe5a31SPaolo Pisati 	struct intr_handler *ih, *temp_ih;
590bafe5a31SPaolo Pisati 	struct intr_thread *it;
591bafe5a31SPaolo Pisati 
592bafe5a31SPaolo Pisati 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
593bafe5a31SPaolo Pisati 		return (EINVAL);
594bafe5a31SPaolo Pisati 
595bafe5a31SPaolo Pisati 	/* Allocate and populate an interrupt handler structure. */
596bafe5a31SPaolo Pisati 	ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
597bafe5a31SPaolo Pisati 	ih->ih_filter = filter;
598bafe5a31SPaolo Pisati 	ih->ih_handler = handler;
599bafe5a31SPaolo Pisati 	ih->ih_argument = arg;
60037b8ef16SJohn Baldwin 	strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
601bafe5a31SPaolo Pisati 	ih->ih_event = ie;
602bafe5a31SPaolo Pisati 	ih->ih_pri = pri;
603bafe5a31SPaolo Pisati 	if (flags & INTR_EXCL)
604bafe5a31SPaolo Pisati 		ih->ih_flags = IH_EXCLUSIVE;
605bafe5a31SPaolo Pisati 	if (flags & INTR_MPSAFE)
606bafe5a31SPaolo Pisati 		ih->ih_flags |= IH_MPSAFE;
607bafe5a31SPaolo Pisati 	if (flags & INTR_ENTROPY)
608bafe5a31SPaolo Pisati 		ih->ih_flags |= IH_ENTROPY;
609bafe5a31SPaolo Pisati 
610bafe5a31SPaolo Pisati 	/* We can only have one exclusive handler in a event. */
611bafe5a31SPaolo Pisati 	mtx_lock(&ie->ie_lock);
612bafe5a31SPaolo Pisati 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
613bafe5a31SPaolo Pisati 		if ((flags & INTR_EXCL) ||
614bafe5a31SPaolo Pisati 		    (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
615bafe5a31SPaolo Pisati 			mtx_unlock(&ie->ie_lock);
616bafe5a31SPaolo Pisati 			free(ih, M_ITHREAD);
617bafe5a31SPaolo Pisati 			return (EINVAL);
618bafe5a31SPaolo Pisati 		}
619bafe5a31SPaolo Pisati 	}
620bafe5a31SPaolo Pisati 
621bafe5a31SPaolo Pisati 	/* Add the new handler to the event in priority order. */
622bafe5a31SPaolo Pisati 	TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
623bafe5a31SPaolo Pisati 		if (temp_ih->ih_pri > ih->ih_pri)
624bafe5a31SPaolo Pisati 			break;
625bafe5a31SPaolo Pisati 	}
626bafe5a31SPaolo Pisati 	if (temp_ih == NULL)
627bafe5a31SPaolo Pisati 		TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
628bafe5a31SPaolo Pisati 	else
629bafe5a31SPaolo Pisati 		TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
630bafe5a31SPaolo Pisati 	intr_event_update(ie);
631bafe5a31SPaolo Pisati 
632bafe5a31SPaolo Pisati 	/* For filtered handlers, create a private ithread to run on. */
633bafe5a31SPaolo Pisati 	if (filter != NULL && handler != NULL) {
634bafe5a31SPaolo Pisati 		mtx_unlock(&ie->ie_lock);
635bafe5a31SPaolo Pisati 		it = ithread_create("intr: newborn", ih);
636bafe5a31SPaolo Pisati 		mtx_lock(&ie->ie_lock);
637bafe5a31SPaolo Pisati 		it->it_event = ie;
638bafe5a31SPaolo Pisati 		ih->ih_thread = it;
639037f43d3SSergey Kandaurov 		ithread_update(it); /* XXX - do we really need this?!?!? */
640bafe5a31SPaolo Pisati 	} else { /* Create the global per-event thread if we need one. */
641bafe5a31SPaolo Pisati 		while (ie->ie_thread == NULL && handler != NULL) {
642bafe5a31SPaolo Pisati 			if (ie->ie_flags & IE_ADDING_THREAD)
643bafe5a31SPaolo Pisati 				msleep(ie, &ie->ie_lock, 0, "ithread", 0);
644bafe5a31SPaolo Pisati 			else {
645bafe5a31SPaolo Pisati 				ie->ie_flags |= IE_ADDING_THREAD;
646bafe5a31SPaolo Pisati 				mtx_unlock(&ie->ie_lock);
647bafe5a31SPaolo Pisati 				it = ithread_create("intr: newborn", ih);
648bafe5a31SPaolo Pisati 				mtx_lock(&ie->ie_lock);
649bafe5a31SPaolo Pisati 				ie->ie_flags &= ~IE_ADDING_THREAD;
650bafe5a31SPaolo Pisati 				ie->ie_thread = it;
651bafe5a31SPaolo Pisati 				it->it_event = ie;
652bafe5a31SPaolo Pisati 				ithread_update(it);
653bafe5a31SPaolo Pisati 				wakeup(ie);
654bafe5a31SPaolo Pisati 			}
655bafe5a31SPaolo Pisati 		}
656bafe5a31SPaolo Pisati 	}
657bafe5a31SPaolo Pisati 	CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
658bafe5a31SPaolo Pisati 	    ie->ie_name);
659bafe5a31SPaolo Pisati 	mtx_unlock(&ie->ie_lock);
660bafe5a31SPaolo Pisati 
661bafe5a31SPaolo Pisati 	if (cookiep != NULL)
662bafe5a31SPaolo Pisati 		*cookiep = ih;
663bafe5a31SPaolo Pisati 	return (0);
664bafe5a31SPaolo Pisati }
665bafe5a31SPaolo Pisati #endif
666b4151f71SJohn Baldwin 
667c3045318SJohn Baldwin /*
66837b8ef16SJohn Baldwin  * Append a description preceded by a ':' to the name of the specified
66937b8ef16SJohn Baldwin  * interrupt handler.
67037b8ef16SJohn Baldwin  */
67137b8ef16SJohn Baldwin int
67237b8ef16SJohn Baldwin intr_event_describe_handler(struct intr_event *ie, void *cookie,
67337b8ef16SJohn Baldwin     const char *descr)
67437b8ef16SJohn Baldwin {
67537b8ef16SJohn Baldwin 	struct intr_handler *ih;
67637b8ef16SJohn Baldwin 	size_t space;
67737b8ef16SJohn Baldwin 	char *start;
67837b8ef16SJohn Baldwin 
67937b8ef16SJohn Baldwin 	mtx_lock(&ie->ie_lock);
68037b8ef16SJohn Baldwin #ifdef INVARIANTS
68137b8ef16SJohn Baldwin 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
68237b8ef16SJohn Baldwin 		if (ih == cookie)
68337b8ef16SJohn Baldwin 			break;
68437b8ef16SJohn Baldwin 	}
68537b8ef16SJohn Baldwin 	if (ih == NULL) {
68637b8ef16SJohn Baldwin 		mtx_unlock(&ie->ie_lock);
687d0c9a291SJohn Baldwin 		panic("handler %p not found in interrupt event %p", cookie, ie);
68837b8ef16SJohn Baldwin 	}
68937b8ef16SJohn Baldwin #endif
69037b8ef16SJohn Baldwin 	ih = cookie;
69137b8ef16SJohn Baldwin 
69237b8ef16SJohn Baldwin 	/*
69337b8ef16SJohn Baldwin 	 * Look for an existing description by checking for an
69437b8ef16SJohn Baldwin 	 * existing ":".  This assumes device names do not include
69537b8ef16SJohn Baldwin 	 * colons.  If one is found, prepare to insert the new
69637b8ef16SJohn Baldwin 	 * description at that point.  If one is not found, find the
69737b8ef16SJohn Baldwin 	 * end of the name to use as the insertion point.
69837b8ef16SJohn Baldwin 	 */
699dc15eac0SEd Schouten 	start = strchr(ih->ih_name, ':');
70037b8ef16SJohn Baldwin 	if (start == NULL)
701dc15eac0SEd Schouten 		start = strchr(ih->ih_name, 0);
70237b8ef16SJohn Baldwin 
70337b8ef16SJohn Baldwin 	/*
70437b8ef16SJohn Baldwin 	 * See if there is enough remaining room in the string for the
70537b8ef16SJohn Baldwin 	 * description + ":".  The "- 1" leaves room for the trailing
70637b8ef16SJohn Baldwin 	 * '\0'.  The "+ 1" accounts for the colon.
70737b8ef16SJohn Baldwin 	 */
70837b8ef16SJohn Baldwin 	space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1;
70937b8ef16SJohn Baldwin 	if (strlen(descr) + 1 > space) {
71037b8ef16SJohn Baldwin 		mtx_unlock(&ie->ie_lock);
71137b8ef16SJohn Baldwin 		return (ENOSPC);
71237b8ef16SJohn Baldwin 	}
71337b8ef16SJohn Baldwin 
71437b8ef16SJohn Baldwin 	/* Append a colon followed by the description. */
71537b8ef16SJohn Baldwin 	*start = ':';
71637b8ef16SJohn Baldwin 	strcpy(start + 1, descr);
71737b8ef16SJohn Baldwin 	intr_event_update(ie);
71837b8ef16SJohn Baldwin 	mtx_unlock(&ie->ie_lock);
71937b8ef16SJohn Baldwin 	return (0);
72037b8ef16SJohn Baldwin }
72137b8ef16SJohn Baldwin 
72237b8ef16SJohn Baldwin /*
723c3045318SJohn Baldwin  * Return the ie_source field from the intr_event an intr_handler is
724c3045318SJohn Baldwin  * associated with.
725c3045318SJohn Baldwin  */
726c3045318SJohn Baldwin void *
727c3045318SJohn Baldwin intr_handler_source(void *cookie)
728c3045318SJohn Baldwin {
729c3045318SJohn Baldwin 	struct intr_handler *ih;
730c3045318SJohn Baldwin 	struct intr_event *ie;
731c3045318SJohn Baldwin 
732c3045318SJohn Baldwin 	ih = (struct intr_handler *)cookie;
733c3045318SJohn Baldwin 	if (ih == NULL)
734c3045318SJohn Baldwin 		return (NULL);
735c3045318SJohn Baldwin 	ie = ih->ih_event;
736c3045318SJohn Baldwin 	KASSERT(ie != NULL,
737c3045318SJohn Baldwin 	    ("interrupt handler \"%s\" has a NULL interrupt event",
738c3045318SJohn Baldwin 	    ih->ih_name));
739c3045318SJohn Baldwin 	return (ie->ie_source);
740c3045318SJohn Baldwin }
741c3045318SJohn Baldwin 
742e4cd31ddSJeff Roberson /*
743e4cd31ddSJeff Roberson  * Sleep until an ithread finishes executing an interrupt handler.
744e4cd31ddSJeff Roberson  *
745e4cd31ddSJeff Roberson  * XXX Doesn't currently handle interrupt filters or fast interrupt
746e4cd31ddSJeff Roberson  * handlers.  This is intended for compatibility with linux drivers
747e4cd31ddSJeff Roberson  * only.  Do not use in BSD code.
748e4cd31ddSJeff Roberson  */
749e4cd31ddSJeff Roberson void
750e4cd31ddSJeff Roberson _intr_drain(int irq)
751e4cd31ddSJeff Roberson {
752e4cd31ddSJeff Roberson 	struct intr_event *ie;
753e4cd31ddSJeff Roberson 	struct intr_thread *ithd;
754e4cd31ddSJeff Roberson 	struct thread *td;
755e4cd31ddSJeff Roberson 
756e4cd31ddSJeff Roberson 	ie = intr_lookup(irq);
757e4cd31ddSJeff Roberson 	if (ie == NULL)
758e4cd31ddSJeff Roberson 		return;
759e4cd31ddSJeff Roberson 	if (ie->ie_thread == NULL)
760e4cd31ddSJeff Roberson 		return;
761e4cd31ddSJeff Roberson 	ithd = ie->ie_thread;
762e4cd31ddSJeff Roberson 	td = ithd->it_thread;
7635bd186a6SJeff Roberson 	/*
7645bd186a6SJeff Roberson 	 * We set the flag and wait for it to be cleared to avoid
7655bd186a6SJeff Roberson 	 * long delays with potentially busy interrupt handlers
7665bd186a6SJeff Roberson 	 * were we to only sample TD_AWAITING_INTR() every tick.
7675bd186a6SJeff Roberson 	 */
768e4cd31ddSJeff Roberson 	thread_lock(td);
769e4cd31ddSJeff Roberson 	if (!TD_AWAITING_INTR(td)) {
770e4cd31ddSJeff Roberson 		ithd->it_flags |= IT_WAIT;
7715bd186a6SJeff Roberson 		while (ithd->it_flags & IT_WAIT) {
7725bd186a6SJeff Roberson 			thread_unlock(td);
7735bd186a6SJeff Roberson 			pause("idrain", 1);
7745bd186a6SJeff Roberson 			thread_lock(td);
775e4cd31ddSJeff Roberson 		}
7765bd186a6SJeff Roberson 	}
7775bd186a6SJeff Roberson 	thread_unlock(td);
778e4cd31ddSJeff Roberson 	return;
779e4cd31ddSJeff Roberson }
780e4cd31ddSJeff Roberson 
781e4cd31ddSJeff Roberson 
782bafe5a31SPaolo Pisati #ifndef INTR_FILTER
783b4151f71SJohn Baldwin int
784e0f66ef8SJohn Baldwin intr_event_remove_handler(void *cookie)
785b4151f71SJohn Baldwin {
786e0f66ef8SJohn Baldwin 	struct intr_handler *handler = (struct intr_handler *)cookie;
787e0f66ef8SJohn Baldwin 	struct intr_event *ie;
788b4151f71SJohn Baldwin #ifdef INVARIANTS
789e0f66ef8SJohn Baldwin 	struct intr_handler *ih;
790e0f66ef8SJohn Baldwin #endif
791e0f66ef8SJohn Baldwin #ifdef notyet
792e0f66ef8SJohn Baldwin 	int dead;
793b4151f71SJohn Baldwin #endif
794b4151f71SJohn Baldwin 
7953e5da754SJohn Baldwin 	if (handler == NULL)
796b4151f71SJohn Baldwin 		return (EINVAL);
797e0f66ef8SJohn Baldwin 	ie = handler->ih_event;
798e0f66ef8SJohn Baldwin 	KASSERT(ie != NULL,
799e0f66ef8SJohn Baldwin 	    ("interrupt handler \"%s\" has a NULL interrupt event",
8003e5da754SJohn Baldwin 	    handler->ih_name));
801e0f66ef8SJohn Baldwin 	mtx_lock(&ie->ie_lock);
80291f91617SDavid E. O'Brien 	CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
803e0f66ef8SJohn Baldwin 	    ie->ie_name);
804b4151f71SJohn Baldwin #ifdef INVARIANTS
805e0f66ef8SJohn Baldwin 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
8063e5da754SJohn Baldwin 		if (ih == handler)
8073e5da754SJohn Baldwin 			goto ok;
808e0f66ef8SJohn Baldwin 	mtx_unlock(&ie->ie_lock);
809e0f66ef8SJohn Baldwin 	panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
810e0f66ef8SJohn Baldwin 	    ih->ih_name, ie->ie_name);
8113e5da754SJohn Baldwin ok:
812b4151f71SJohn Baldwin #endif
813de271f01SJohn Baldwin 	/*
814e0f66ef8SJohn Baldwin 	 * If there is no ithread, then just remove the handler and return.
815e0f66ef8SJohn Baldwin 	 * XXX: Note that an INTR_FAST handler might be running on another
816e0f66ef8SJohn Baldwin 	 * CPU!
817e0f66ef8SJohn Baldwin 	 */
818e0f66ef8SJohn Baldwin 	if (ie->ie_thread == NULL) {
819e0f66ef8SJohn Baldwin 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
820e0f66ef8SJohn Baldwin 		mtx_unlock(&ie->ie_lock);
821e0f66ef8SJohn Baldwin 		free(handler, M_ITHREAD);
822e0f66ef8SJohn Baldwin 		return (0);
823e0f66ef8SJohn Baldwin 	}
824e0f66ef8SJohn Baldwin 
825e0f66ef8SJohn Baldwin 	/*
826de271f01SJohn Baldwin 	 * If the interrupt thread is already running, then just mark this
827de271f01SJohn Baldwin 	 * handler as being dead and let the ithread do the actual removal.
828288e351bSDon Lewis 	 *
829288e351bSDon Lewis 	 * During a cold boot while cold is set, msleep() does not sleep,
830288e351bSDon Lewis 	 * so we have to remove the handler here rather than letting the
831288e351bSDon Lewis 	 * thread do it.
832de271f01SJohn Baldwin 	 */
833982d11f8SJeff Roberson 	thread_lock(ie->ie_thread->it_thread);
834e0f66ef8SJohn Baldwin 	if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) {
835de271f01SJohn Baldwin 		handler->ih_flags |= IH_DEAD;
836de271f01SJohn Baldwin 
837de271f01SJohn Baldwin 		/*
838de271f01SJohn Baldwin 		 * Ensure that the thread will process the handler list
839de271f01SJohn Baldwin 		 * again and remove this handler if it has already passed
840de271f01SJohn Baldwin 		 * it on the list.
841de271f01SJohn Baldwin 		 */
842e0f66ef8SJohn Baldwin 		ie->ie_thread->it_need = 1;
8434d29cb2dSJohn Baldwin 	} else
844e0f66ef8SJohn Baldwin 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
845982d11f8SJeff Roberson 	thread_unlock(ie->ie_thread->it_thread);
846e0f66ef8SJohn Baldwin 	while (handler->ih_flags & IH_DEAD)
8470f180a7cSJohn Baldwin 		msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
848e0f66ef8SJohn Baldwin 	intr_event_update(ie);
849e0f66ef8SJohn Baldwin #ifdef notyet
850e0f66ef8SJohn Baldwin 	/*
851e0f66ef8SJohn Baldwin 	 * XXX: This could be bad in the case of ppbus(8).  Also, I think
852e0f66ef8SJohn Baldwin 	 * this could lead to races of stale data when servicing an
853e0f66ef8SJohn Baldwin 	 * interrupt.
854e0f66ef8SJohn Baldwin 	 */
855e0f66ef8SJohn Baldwin 	dead = 1;
856e0f66ef8SJohn Baldwin 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
857e0f66ef8SJohn Baldwin 		if (!(ih->ih_flags & IH_FAST)) {
858e0f66ef8SJohn Baldwin 			dead = 0;
859e0f66ef8SJohn Baldwin 			break;
860e0f66ef8SJohn Baldwin 		}
861e0f66ef8SJohn Baldwin 	}
862e0f66ef8SJohn Baldwin 	if (dead) {
863e0f66ef8SJohn Baldwin 		ithread_destroy(ie->ie_thread);
864e0f66ef8SJohn Baldwin 		ie->ie_thread = NULL;
865e0f66ef8SJohn Baldwin 	}
866e0f66ef8SJohn Baldwin #endif
867e0f66ef8SJohn Baldwin 	mtx_unlock(&ie->ie_lock);
868b4151f71SJohn Baldwin 	free(handler, M_ITHREAD);
869b4151f71SJohn Baldwin 	return (0);
870b4151f71SJohn Baldwin }
871b4151f71SJohn Baldwin 
8721ee1b687SJohn Baldwin static int
873e0f66ef8SJohn Baldwin intr_event_schedule_thread(struct intr_event *ie)
8743e5da754SJohn Baldwin {
875e0f66ef8SJohn Baldwin 	struct intr_entropy entropy;
876e0f66ef8SJohn Baldwin 	struct intr_thread *it;
877b40ce416SJulian Elischer 	struct thread *td;
87804774f23SJulian Elischer 	struct thread *ctd;
8793e5da754SJohn Baldwin 	struct proc *p;
8803e5da754SJohn Baldwin 
8813e5da754SJohn Baldwin 	/*
8823e5da754SJohn Baldwin 	 * If no ithread or no handlers, then we have a stray interrupt.
8833e5da754SJohn Baldwin 	 */
884e0f66ef8SJohn Baldwin 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) ||
885e0f66ef8SJohn Baldwin 	    ie->ie_thread == NULL)
8863e5da754SJohn Baldwin 		return (EINVAL);
8873e5da754SJohn Baldwin 
88804774f23SJulian Elischer 	ctd = curthread;
889e0f66ef8SJohn Baldwin 	it = ie->ie_thread;
890e0f66ef8SJohn Baldwin 	td = it->it_thread;
8916f40c417SRobert Watson 	p = td->td_proc;
892e0f66ef8SJohn Baldwin 
8933e5da754SJohn Baldwin 	/*
8943e5da754SJohn Baldwin 	 * If any of the handlers for this ithread claim to be good
8953e5da754SJohn Baldwin 	 * sources of entropy, then gather some.
8963e5da754SJohn Baldwin 	 */
897e0f66ef8SJohn Baldwin 	if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
8986f40c417SRobert Watson 		CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
8997ab24ea3SJulian Elischer 		    p->p_pid, td->td_name);
900e0f66ef8SJohn Baldwin 		entropy.event = (uintptr_t)ie;
901e0f66ef8SJohn Baldwin 		entropy.td = ctd;
9023e5da754SJohn Baldwin 		random_harvest(&entropy, sizeof(entropy), 2, 0,
9033e5da754SJohn Baldwin 		    RANDOM_INTERRUPT);
9043e5da754SJohn Baldwin 	}
9053e5da754SJohn Baldwin 
906e0f66ef8SJohn Baldwin 	KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
9073e5da754SJohn Baldwin 
9083e5da754SJohn Baldwin 	/*
9093e5da754SJohn Baldwin 	 * Set it_need to tell the thread to keep running if it is already
910982d11f8SJeff Roberson 	 * running.  Then, lock the thread and see if we actually need to
911982d11f8SJeff Roberson 	 * put it on the runqueue.
9123e5da754SJohn Baldwin 	 */
913e0f66ef8SJohn Baldwin 	it->it_need = 1;
914982d11f8SJeff Roberson 	thread_lock(td);
91571fad9fdSJulian Elischer 	if (TD_AWAITING_INTR(td)) {
916e0f66ef8SJohn Baldwin 		CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
9177ab24ea3SJulian Elischer 		    td->td_name);
91871fad9fdSJulian Elischer 		TD_CLR_IWAIT(td);
919f0393f06SJeff Roberson 		sched_add(td, SRQ_INTR);
9203e5da754SJohn Baldwin 	} else {
921e0f66ef8SJohn Baldwin 		CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
9227ab24ea3SJulian Elischer 		    __func__, p->p_pid, td->td_name, it->it_need, td->td_state);
9233e5da754SJohn Baldwin 	}
924982d11f8SJeff Roberson 	thread_unlock(td);
9253e5da754SJohn Baldwin 
9263e5da754SJohn Baldwin 	return (0);
9273e5da754SJohn Baldwin }
928bafe5a31SPaolo Pisati #else
929bafe5a31SPaolo Pisati int
930bafe5a31SPaolo Pisati intr_event_remove_handler(void *cookie)
931bafe5a31SPaolo Pisati {
932bafe5a31SPaolo Pisati 	struct intr_handler *handler = (struct intr_handler *)cookie;
933bafe5a31SPaolo Pisati 	struct intr_event *ie;
934bafe5a31SPaolo Pisati 	struct intr_thread *it;
935bafe5a31SPaolo Pisati #ifdef INVARIANTS
936bafe5a31SPaolo Pisati 	struct intr_handler *ih;
937bafe5a31SPaolo Pisati #endif
938bafe5a31SPaolo Pisati #ifdef notyet
939bafe5a31SPaolo Pisati 	int dead;
940bafe5a31SPaolo Pisati #endif
941bafe5a31SPaolo Pisati 
942bafe5a31SPaolo Pisati 	if (handler == NULL)
943bafe5a31SPaolo Pisati 		return (EINVAL);
944bafe5a31SPaolo Pisati 	ie = handler->ih_event;
945bafe5a31SPaolo Pisati 	KASSERT(ie != NULL,
946bafe5a31SPaolo Pisati 	    ("interrupt handler \"%s\" has a NULL interrupt event",
947bafe5a31SPaolo Pisati 	    handler->ih_name));
948bafe5a31SPaolo Pisati 	mtx_lock(&ie->ie_lock);
949bafe5a31SPaolo Pisati 	CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
950bafe5a31SPaolo Pisati 	    ie->ie_name);
951bafe5a31SPaolo Pisati #ifdef INVARIANTS
952bafe5a31SPaolo Pisati 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
953bafe5a31SPaolo Pisati 		if (ih == handler)
954bafe5a31SPaolo Pisati 			goto ok;
955bafe5a31SPaolo Pisati 	mtx_unlock(&ie->ie_lock);
956bafe5a31SPaolo Pisati 	panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
957bafe5a31SPaolo Pisati 	    ih->ih_name, ie->ie_name);
958bafe5a31SPaolo Pisati ok:
959bafe5a31SPaolo Pisati #endif
960bafe5a31SPaolo Pisati 	/*
961bafe5a31SPaolo Pisati 	 * If there are no ithreads (per event and per handler), then
962bafe5a31SPaolo Pisati 	 * just remove the handler and return.
963bafe5a31SPaolo Pisati 	 * XXX: Note that an INTR_FAST handler might be running on another CPU!
964bafe5a31SPaolo Pisati 	 */
965bafe5a31SPaolo Pisati 	if (ie->ie_thread == NULL && handler->ih_thread == NULL) {
966bafe5a31SPaolo Pisati 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
967bafe5a31SPaolo Pisati 		mtx_unlock(&ie->ie_lock);
968bafe5a31SPaolo Pisati 		free(handler, M_ITHREAD);
969bafe5a31SPaolo Pisati 		return (0);
970bafe5a31SPaolo Pisati 	}
971bafe5a31SPaolo Pisati 
972bafe5a31SPaolo Pisati 	/* Private or global ithread? */
973bafe5a31SPaolo Pisati 	it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread;
974bafe5a31SPaolo Pisati 	/*
975bafe5a31SPaolo Pisati 	 * If the interrupt thread is already running, then just mark this
976bafe5a31SPaolo Pisati 	 * handler as being dead and let the ithread do the actual removal.
977bafe5a31SPaolo Pisati 	 *
978bafe5a31SPaolo Pisati 	 * During a cold boot while cold is set, msleep() does not sleep,
979bafe5a31SPaolo Pisati 	 * so we have to remove the handler here rather than letting the
980bafe5a31SPaolo Pisati 	 * thread do it.
981bafe5a31SPaolo Pisati 	 */
982982d11f8SJeff Roberson 	thread_lock(it->it_thread);
983bafe5a31SPaolo Pisati 	if (!TD_AWAITING_INTR(it->it_thread) && !cold) {
984bafe5a31SPaolo Pisati 		handler->ih_flags |= IH_DEAD;
985bafe5a31SPaolo Pisati 
986bafe5a31SPaolo Pisati 		/*
987bafe5a31SPaolo Pisati 		 * Ensure that the thread will process the handler list
988bafe5a31SPaolo Pisati 		 * again and remove this handler if it has already passed
989bafe5a31SPaolo Pisati 		 * it on the list.
990bafe5a31SPaolo Pisati 		 */
991bafe5a31SPaolo Pisati 		it->it_need = 1;
992bafe5a31SPaolo Pisati 	} else
993bafe5a31SPaolo Pisati 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
994982d11f8SJeff Roberson 	thread_unlock(it->it_thread);
995bafe5a31SPaolo Pisati 	while (handler->ih_flags & IH_DEAD)
996bafe5a31SPaolo Pisati 		msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
997bafe5a31SPaolo Pisati 	/*
998bafe5a31SPaolo Pisati 	 * At this point, the handler has been disconnected from the event,
999bafe5a31SPaolo Pisati 	 * so we can kill the private ithread if any.
1000bafe5a31SPaolo Pisati 	 */
1001bafe5a31SPaolo Pisati 	if (handler->ih_thread) {
1002bafe5a31SPaolo Pisati 		ithread_destroy(handler->ih_thread);
1003bafe5a31SPaolo Pisati 		handler->ih_thread = NULL;
1004bafe5a31SPaolo Pisati 	}
1005bafe5a31SPaolo Pisati 	intr_event_update(ie);
1006bafe5a31SPaolo Pisati #ifdef notyet
1007bafe5a31SPaolo Pisati 	/*
1008bafe5a31SPaolo Pisati 	 * XXX: This could be bad in the case of ppbus(8).  Also, I think
1009bafe5a31SPaolo Pisati 	 * this could lead to races of stale data when servicing an
1010bafe5a31SPaolo Pisati 	 * interrupt.
1011bafe5a31SPaolo Pisati 	 */
1012bafe5a31SPaolo Pisati 	dead = 1;
1013bafe5a31SPaolo Pisati 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1014bafe5a31SPaolo Pisati 		if (handler != NULL) {
1015bafe5a31SPaolo Pisati 			dead = 0;
1016bafe5a31SPaolo Pisati 			break;
1017bafe5a31SPaolo Pisati 		}
1018bafe5a31SPaolo Pisati 	}
1019bafe5a31SPaolo Pisati 	if (dead) {
1020bafe5a31SPaolo Pisati 		ithread_destroy(ie->ie_thread);
1021bafe5a31SPaolo Pisati 		ie->ie_thread = NULL;
1022bafe5a31SPaolo Pisati 	}
1023bafe5a31SPaolo Pisati #endif
1024bafe5a31SPaolo Pisati 	mtx_unlock(&ie->ie_lock);
1025bafe5a31SPaolo Pisati 	free(handler, M_ITHREAD);
1026bafe5a31SPaolo Pisati 	return (0);
1027bafe5a31SPaolo Pisati }
1028bafe5a31SPaolo Pisati 
10291ee1b687SJohn Baldwin static int
1030bafe5a31SPaolo Pisati intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it)
1031bafe5a31SPaolo Pisati {
1032bafe5a31SPaolo Pisati 	struct intr_entropy entropy;
1033bafe5a31SPaolo Pisati 	struct thread *td;
1034bafe5a31SPaolo Pisati 	struct thread *ctd;
1035bafe5a31SPaolo Pisati 	struct proc *p;
1036bafe5a31SPaolo Pisati 
1037bafe5a31SPaolo Pisati 	/*
1038bafe5a31SPaolo Pisati 	 * If no ithread or no handlers, then we have a stray interrupt.
1039bafe5a31SPaolo Pisati 	 */
1040bafe5a31SPaolo Pisati 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL)
1041bafe5a31SPaolo Pisati 		return (EINVAL);
1042bafe5a31SPaolo Pisati 
1043bafe5a31SPaolo Pisati 	ctd = curthread;
1044bafe5a31SPaolo Pisati 	td = it->it_thread;
1045bafe5a31SPaolo Pisati 	p = td->td_proc;
1046bafe5a31SPaolo Pisati 
1047bafe5a31SPaolo Pisati 	/*
1048bafe5a31SPaolo Pisati 	 * If any of the handlers for this ithread claim to be good
1049bafe5a31SPaolo Pisati 	 * sources of entropy, then gather some.
1050bafe5a31SPaolo Pisati 	 */
1051bafe5a31SPaolo Pisati 	if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
1052bafe5a31SPaolo Pisati 		CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__,
10537ab24ea3SJulian Elischer 		    p->p_pid, td->td_name);
1054bafe5a31SPaolo Pisati 		entropy.event = (uintptr_t)ie;
1055bafe5a31SPaolo Pisati 		entropy.td = ctd;
1056bafe5a31SPaolo Pisati 		random_harvest(&entropy, sizeof(entropy), 2, 0,
1057bafe5a31SPaolo Pisati 		    RANDOM_INTERRUPT);
1058bafe5a31SPaolo Pisati 	}
1059bafe5a31SPaolo Pisati 
1060bafe5a31SPaolo Pisati 	KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
1061bafe5a31SPaolo Pisati 
1062bafe5a31SPaolo Pisati 	/*
1063bafe5a31SPaolo Pisati 	 * Set it_need to tell the thread to keep running if it is already
1064982d11f8SJeff Roberson 	 * running.  Then, lock the thread and see if we actually need to
1065982d11f8SJeff Roberson 	 * put it on the runqueue.
1066bafe5a31SPaolo Pisati 	 */
1067bafe5a31SPaolo Pisati 	it->it_need = 1;
1068982d11f8SJeff Roberson 	thread_lock(td);
1069bafe5a31SPaolo Pisati 	if (TD_AWAITING_INTR(td)) {
1070bafe5a31SPaolo Pisati 		CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
10713c1ffc32SJulian Elischer 		    td->td_name);
1072bafe5a31SPaolo Pisati 		TD_CLR_IWAIT(td);
1073bafe5a31SPaolo Pisati 		sched_add(td, SRQ_INTR);
1074bafe5a31SPaolo Pisati 	} else {
1075bafe5a31SPaolo Pisati 		CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
10767ab24ea3SJulian Elischer 		    __func__, p->p_pid, td->td_name, it->it_need, td->td_state);
1077bafe5a31SPaolo Pisati 	}
1078982d11f8SJeff Roberson 	thread_unlock(td);
1079bafe5a31SPaolo Pisati 
1080bafe5a31SPaolo Pisati 	return (0);
1081bafe5a31SPaolo Pisati }
1082bafe5a31SPaolo Pisati #endif
10833e5da754SJohn Baldwin 
1084fe486a37SJohn Baldwin /*
1085e84bcd84SRobert Watson  * Allow interrupt event binding for software interrupt handlers -- a no-op,
1086e84bcd84SRobert Watson  * since interrupts are generated in software rather than being directed by
1087e84bcd84SRobert Watson  * a PIC.
1088e84bcd84SRobert Watson  */
1089e84bcd84SRobert Watson static int
1090e84bcd84SRobert Watson swi_assign_cpu(void *arg, u_char cpu)
1091e84bcd84SRobert Watson {
1092e84bcd84SRobert Watson 
1093e84bcd84SRobert Watson 	return (0);
1094e84bcd84SRobert Watson }
1095e84bcd84SRobert Watson 
1096e84bcd84SRobert Watson /*
1097fe486a37SJohn Baldwin  * Add a software interrupt handler to a specified event.  If a given event
1098fe486a37SJohn Baldwin  * is not specified, then a new event is created.
1099fe486a37SJohn Baldwin  */
11003e5da754SJohn Baldwin int
1101e0f66ef8SJohn Baldwin swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
1102b4151f71SJohn Baldwin 	    void *arg, int pri, enum intr_type flags, void **cookiep)
11038088699fSJohn Baldwin {
11041b9d701fSAttilio Rao 	struct thread *td;
1105e0f66ef8SJohn Baldwin 	struct intr_event *ie;
1106b4151f71SJohn Baldwin 	int error;
11078088699fSJohn Baldwin 
1108bafe5a31SPaolo Pisati 	if (flags & INTR_ENTROPY)
11093e5da754SJohn Baldwin 		return (EINVAL);
11103e5da754SJohn Baldwin 
1111e0f66ef8SJohn Baldwin 	ie = (eventp != NULL) ? *eventp : NULL;
11128088699fSJohn Baldwin 
1113e0f66ef8SJohn Baldwin 	if (ie != NULL) {
1114e0f66ef8SJohn Baldwin 		if (!(ie->ie_flags & IE_SOFT))
11153e5da754SJohn Baldwin 			return (EINVAL);
11163e5da754SJohn Baldwin 	} else {
11179b33b154SJeff Roberson 		error = intr_event_create(&ie, NULL, IE_SOFT, 0,
1118e84bcd84SRobert Watson 		    NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri);
11198088699fSJohn Baldwin 		if (error)
1120b4151f71SJohn Baldwin 			return (error);
1121e0f66ef8SJohn Baldwin 		if (eventp != NULL)
1122e0f66ef8SJohn Baldwin 			*eventp = ie;
11238088699fSJohn Baldwin 	}
11248d809d50SJeff Roberson 	error = intr_event_add_handler(ie, name, NULL, handler, arg,
1125d3305205SJohn Baldwin 	    PI_SWI(pri), flags, cookiep);
11268d809d50SJeff Roberson 	if (error)
11278d809d50SJeff Roberson 		return (error);
11288d809d50SJeff Roberson 	if (pri == SWI_CLOCK) {
11291b9d701fSAttilio Rao 		td = ie->ie_thread->it_thread;
11301b9d701fSAttilio Rao 		thread_lock(td);
11311b9d701fSAttilio Rao 		td->td_flags |= TDF_NOLOAD;
11321b9d701fSAttilio Rao 		thread_unlock(td);
11338d809d50SJeff Roberson 	}
11348d809d50SJeff Roberson 	return (0);
11358088699fSJohn Baldwin }
11368088699fSJohn Baldwin 
11371931cf94SJohn Baldwin /*
1138e0f66ef8SJohn Baldwin  * Schedule a software interrupt thread.
11391931cf94SJohn Baldwin  */
11401931cf94SJohn Baldwin void
1141b4151f71SJohn Baldwin swi_sched(void *cookie, int flags)
11421931cf94SJohn Baldwin {
1143e0f66ef8SJohn Baldwin 	struct intr_handler *ih = (struct intr_handler *)cookie;
1144e0f66ef8SJohn Baldwin 	struct intr_event *ie = ih->ih_event;
11453e5da754SJohn Baldwin 	int error;
11468088699fSJohn Baldwin 
1147e0f66ef8SJohn Baldwin 	CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
1148e0f66ef8SJohn Baldwin 	    ih->ih_need);
11491931cf94SJohn Baldwin 
11501931cf94SJohn Baldwin 	/*
11513e5da754SJohn Baldwin 	 * Set ih_need for this handler so that if the ithread is already
11523e5da754SJohn Baldwin 	 * running it will execute this handler on the next pass.  Otherwise,
11533e5da754SJohn Baldwin 	 * it will execute it the next time it runs.
11541931cf94SJohn Baldwin 	 */
1155b4151f71SJohn Baldwin 	atomic_store_rel_int(&ih->ih_need, 1);
11561ca2c018SBruce Evans 
1157b4151f71SJohn Baldwin 	if (!(flags & SWI_DELAY)) {
115867596082SAttilio Rao 		PCPU_INC(cnt.v_soft);
1159bafe5a31SPaolo Pisati #ifdef INTR_FILTER
1160bafe5a31SPaolo Pisati 		error = intr_event_schedule_thread(ie, ie->ie_thread);
1161bafe5a31SPaolo Pisati #else
1162e0f66ef8SJohn Baldwin 		error = intr_event_schedule_thread(ie);
1163bafe5a31SPaolo Pisati #endif
11643e5da754SJohn Baldwin 		KASSERT(error == 0, ("stray software interrupt"));
11658088699fSJohn Baldwin 	}
11668088699fSJohn Baldwin }
11678088699fSJohn Baldwin 
1168fe486a37SJohn Baldwin /*
1169fe486a37SJohn Baldwin  * Remove a software interrupt handler.  Currently this code does not
1170fe486a37SJohn Baldwin  * remove the associated interrupt event if it becomes empty.  Calling code
1171fe486a37SJohn Baldwin  * may do so manually via intr_event_destroy(), but that's not really
1172fe486a37SJohn Baldwin  * an optimal interface.
1173fe486a37SJohn Baldwin  */
1174fe486a37SJohn Baldwin int
1175fe486a37SJohn Baldwin swi_remove(void *cookie)
1176fe486a37SJohn Baldwin {
1177fe486a37SJohn Baldwin 
1178fe486a37SJohn Baldwin 	return (intr_event_remove_handler(cookie));
1179fe486a37SJohn Baldwin }
1180fe486a37SJohn Baldwin 
1181bafe5a31SPaolo Pisati #ifdef INTR_FILTER
1182bafe5a31SPaolo Pisati static void
1183bafe5a31SPaolo Pisati priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih)
1184bafe5a31SPaolo Pisati {
1185bafe5a31SPaolo Pisati 	struct intr_event *ie;
1186bafe5a31SPaolo Pisati 
1187bafe5a31SPaolo Pisati 	ie = ih->ih_event;
1188bafe5a31SPaolo Pisati 	/*
1189bafe5a31SPaolo Pisati 	 * If this handler is marked for death, remove it from
1190bafe5a31SPaolo Pisati 	 * the list of handlers and wake up the sleeper.
1191bafe5a31SPaolo Pisati 	 */
1192bafe5a31SPaolo Pisati 	if (ih->ih_flags & IH_DEAD) {
1193bafe5a31SPaolo Pisati 		mtx_lock(&ie->ie_lock);
1194bafe5a31SPaolo Pisati 		TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1195bafe5a31SPaolo Pisati 		ih->ih_flags &= ~IH_DEAD;
1196bafe5a31SPaolo Pisati 		wakeup(ih);
1197bafe5a31SPaolo Pisati 		mtx_unlock(&ie->ie_lock);
1198bafe5a31SPaolo Pisati 		return;
1199bafe5a31SPaolo Pisati 	}
1200bafe5a31SPaolo Pisati 
1201bafe5a31SPaolo Pisati 	/* Execute this handler. */
1202bafe5a31SPaolo Pisati 	CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
1203bafe5a31SPaolo Pisati 	     __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument,
1204bafe5a31SPaolo Pisati 	     ih->ih_name, ih->ih_flags);
1205bafe5a31SPaolo Pisati 
1206bafe5a31SPaolo Pisati 	if (!(ih->ih_flags & IH_MPSAFE))
1207bafe5a31SPaolo Pisati 		mtx_lock(&Giant);
1208bafe5a31SPaolo Pisati 	ih->ih_handler(ih->ih_argument);
1209bafe5a31SPaolo Pisati 	if (!(ih->ih_flags & IH_MPSAFE))
1210bafe5a31SPaolo Pisati 		mtx_unlock(&Giant);
1211bafe5a31SPaolo Pisati }
1212bafe5a31SPaolo Pisati #endif
1213bafe5a31SPaolo Pisati 
121437e9511fSJohn Baldwin /*
121537e9511fSJohn Baldwin  * This is a public function for use by drivers that mux interrupt
121637e9511fSJohn Baldwin  * handlers for child devices from their interrupt handler.
121737e9511fSJohn Baldwin  */
121837e9511fSJohn Baldwin void
121937e9511fSJohn Baldwin intr_event_execute_handlers(struct proc *p, struct intr_event *ie)
1220e0f66ef8SJohn Baldwin {
1221e0f66ef8SJohn Baldwin 	struct intr_handler *ih, *ihn;
1222e0f66ef8SJohn Baldwin 
1223e0f66ef8SJohn Baldwin 	TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
1224e0f66ef8SJohn Baldwin 		/*
1225e0f66ef8SJohn Baldwin 		 * If this handler is marked for death, remove it from
1226e0f66ef8SJohn Baldwin 		 * the list of handlers and wake up the sleeper.
1227e0f66ef8SJohn Baldwin 		 */
1228e0f66ef8SJohn Baldwin 		if (ih->ih_flags & IH_DEAD) {
1229e0f66ef8SJohn Baldwin 			mtx_lock(&ie->ie_lock);
1230e0f66ef8SJohn Baldwin 			TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1231e0f66ef8SJohn Baldwin 			ih->ih_flags &= ~IH_DEAD;
1232e0f66ef8SJohn Baldwin 			wakeup(ih);
1233e0f66ef8SJohn Baldwin 			mtx_unlock(&ie->ie_lock);
1234e0f66ef8SJohn Baldwin 			continue;
1235e0f66ef8SJohn Baldwin 		}
1236e0f66ef8SJohn Baldwin 
1237f2d619c8SPaolo Pisati 		/* Skip filter only handlers */
1238f2d619c8SPaolo Pisati 		if (ih->ih_handler == NULL)
1239f2d619c8SPaolo Pisati 			continue;
1240f2d619c8SPaolo Pisati 
1241e0f66ef8SJohn Baldwin 		/*
1242e0f66ef8SJohn Baldwin 		 * For software interrupt threads, we only execute
1243e0f66ef8SJohn Baldwin 		 * handlers that have their need flag set.  Hardware
1244e0f66ef8SJohn Baldwin 		 * interrupt threads always invoke all of their handlers.
1245e0f66ef8SJohn Baldwin 		 */
1246e0f66ef8SJohn Baldwin 		if (ie->ie_flags & IE_SOFT) {
1247e0f66ef8SJohn Baldwin 			if (!ih->ih_need)
1248e0f66ef8SJohn Baldwin 				continue;
1249e0f66ef8SJohn Baldwin 			else
1250e0f66ef8SJohn Baldwin 				atomic_store_rel_int(&ih->ih_need, 0);
1251e0f66ef8SJohn Baldwin 		}
1252e0f66ef8SJohn Baldwin 
1253e0f66ef8SJohn Baldwin 		/* Execute this handler. */
1254e0f66ef8SJohn Baldwin 		CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
1255bafe5a31SPaolo Pisati 		    __func__, p->p_pid, (void *)ih->ih_handler,
1256bafe5a31SPaolo Pisati 		    ih->ih_argument, ih->ih_name, ih->ih_flags);
1257e0f66ef8SJohn Baldwin 
1258e0f66ef8SJohn Baldwin 		if (!(ih->ih_flags & IH_MPSAFE))
1259e0f66ef8SJohn Baldwin 			mtx_lock(&Giant);
1260e0f66ef8SJohn Baldwin 		ih->ih_handler(ih->ih_argument);
1261e0f66ef8SJohn Baldwin 		if (!(ih->ih_flags & IH_MPSAFE))
1262e0f66ef8SJohn Baldwin 			mtx_unlock(&Giant);
1263e0f66ef8SJohn Baldwin 	}
126437e9511fSJohn Baldwin }
126537e9511fSJohn Baldwin 
126637e9511fSJohn Baldwin static void
126737e9511fSJohn Baldwin ithread_execute_handlers(struct proc *p, struct intr_event *ie)
126837e9511fSJohn Baldwin {
126937e9511fSJohn Baldwin 
127037e9511fSJohn Baldwin 	/* Interrupt handlers should not sleep. */
127137e9511fSJohn Baldwin 	if (!(ie->ie_flags & IE_SOFT))
127237e9511fSJohn Baldwin 		THREAD_NO_SLEEPING();
127337e9511fSJohn Baldwin 	intr_event_execute_handlers(p, ie);
1274e0f66ef8SJohn Baldwin 	if (!(ie->ie_flags & IE_SOFT))
1275e0f66ef8SJohn Baldwin 		THREAD_SLEEPING_OK();
1276e0f66ef8SJohn Baldwin 
1277e0f66ef8SJohn Baldwin 	/*
1278e0f66ef8SJohn Baldwin 	 * Interrupt storm handling:
1279e0f66ef8SJohn Baldwin 	 *
1280e0f66ef8SJohn Baldwin 	 * If this interrupt source is currently storming, then throttle
1281e0f66ef8SJohn Baldwin 	 * it to only fire the handler once  per clock tick.
1282e0f66ef8SJohn Baldwin 	 *
1283e0f66ef8SJohn Baldwin 	 * If this interrupt source is not currently storming, but the
1284e0f66ef8SJohn Baldwin 	 * number of back to back interrupts exceeds the storm threshold,
1285e0f66ef8SJohn Baldwin 	 * then enter storming mode.
1286e0f66ef8SJohn Baldwin 	 */
1287e41bcf3cSJohn Baldwin 	if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold &&
1288e41bcf3cSJohn Baldwin 	    !(ie->ie_flags & IE_SOFT)) {
12890ae62c18SNate Lawson 		/* Report the message only once every second. */
12900ae62c18SNate Lawson 		if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
1291e0f66ef8SJohn Baldwin 			printf(
12920ae62c18SNate Lawson 	"interrupt storm detected on \"%s\"; throttling interrupt source\n",
1293e0f66ef8SJohn Baldwin 			    ie->ie_name);
1294e0f66ef8SJohn Baldwin 		}
1295e41bcf3cSJohn Baldwin 		pause("istorm", 1);
1296e0f66ef8SJohn Baldwin 	} else
1297e0f66ef8SJohn Baldwin 		ie->ie_count++;
1298e0f66ef8SJohn Baldwin 
1299e0f66ef8SJohn Baldwin 	/*
1300e0f66ef8SJohn Baldwin 	 * Now that all the handlers have had a chance to run, reenable
1301e0f66ef8SJohn Baldwin 	 * the interrupt source.
1302e0f66ef8SJohn Baldwin 	 */
13031ee1b687SJohn Baldwin 	if (ie->ie_post_ithread != NULL)
13041ee1b687SJohn Baldwin 		ie->ie_post_ithread(ie->ie_source);
1305e0f66ef8SJohn Baldwin }
1306e0f66ef8SJohn Baldwin 
1307bafe5a31SPaolo Pisati #ifndef INTR_FILTER
13088088699fSJohn Baldwin /*
1309b4151f71SJohn Baldwin  * This is the main code for interrupt threads.
13108088699fSJohn Baldwin  */
131137c84183SPoul-Henning Kamp static void
1312b4151f71SJohn Baldwin ithread_loop(void *arg)
13138088699fSJohn Baldwin {
1314e0f66ef8SJohn Baldwin 	struct intr_thread *ithd;
1315e0f66ef8SJohn Baldwin 	struct intr_event *ie;
1316b40ce416SJulian Elischer 	struct thread *td;
1317b4151f71SJohn Baldwin 	struct proc *p;
1318e4cd31ddSJeff Roberson 	int wake;
13198088699fSJohn Baldwin 
1320b40ce416SJulian Elischer 	td = curthread;
1321b40ce416SJulian Elischer 	p = td->td_proc;
1322e0f66ef8SJohn Baldwin 	ithd = (struct intr_thread *)arg;
1323e0f66ef8SJohn Baldwin 	KASSERT(ithd->it_thread == td,
132491f91617SDavid E. O'Brien 	    ("%s: ithread and proc linkage out of sync", __func__));
1325e0f66ef8SJohn Baldwin 	ie = ithd->it_event;
1326e0f66ef8SJohn Baldwin 	ie->ie_count = 0;
1327e4cd31ddSJeff Roberson 	wake = 0;
13288088699fSJohn Baldwin 
13298088699fSJohn Baldwin 	/*
13308088699fSJohn Baldwin 	 * As long as we have interrupts outstanding, go through the
13318088699fSJohn Baldwin 	 * list of handlers, giving each one a go at it.
13328088699fSJohn Baldwin 	 */
13338088699fSJohn Baldwin 	for (;;) {
1334b4151f71SJohn Baldwin 		/*
1335b4151f71SJohn Baldwin 		 * If we are an orphaned thread, then just die.
1336b4151f71SJohn Baldwin 		 */
1337b4151f71SJohn Baldwin 		if (ithd->it_flags & IT_DEAD) {
1338e0f66ef8SJohn Baldwin 			CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
13397ab24ea3SJulian Elischer 			    p->p_pid, td->td_name);
1340b4151f71SJohn Baldwin 			free(ithd, M_ITHREAD);
1341ca9a0ddfSJulian Elischer 			kthread_exit();
1342b4151f71SJohn Baldwin 		}
1343b4151f71SJohn Baldwin 
1344e0f66ef8SJohn Baldwin 		/*
1345e0f66ef8SJohn Baldwin 		 * Service interrupts.  If another interrupt arrives while
1346e0f66ef8SJohn Baldwin 		 * we are running, it will set it_need to note that we
1347e0f66ef8SJohn Baldwin 		 * should make another pass.
1348e0f66ef8SJohn Baldwin 		 */
1349b4151f71SJohn Baldwin 		while (ithd->it_need) {
13508088699fSJohn Baldwin 			/*
1351e0f66ef8SJohn Baldwin 			 * This might need a full read and write barrier
1352e0f66ef8SJohn Baldwin 			 * to make sure that this write posts before any
1353e0f66ef8SJohn Baldwin 			 * of the memory or device accesses in the
1354e0f66ef8SJohn Baldwin 			 * handlers.
13558088699fSJohn Baldwin 			 */
1356b4151f71SJohn Baldwin 			atomic_store_rel_int(&ithd->it_need, 0);
1357e0f66ef8SJohn Baldwin 			ithread_execute_handlers(p, ie);
13588088699fSJohn Baldwin 		}
13597870c3c6SJohn Baldwin 		WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
13607870c3c6SJohn Baldwin 		mtx_assert(&Giant, MA_NOTOWNED);
13618088699fSJohn Baldwin 
13628088699fSJohn Baldwin 		/*
13638088699fSJohn Baldwin 		 * Processed all our interrupts.  Now get the sched
13648088699fSJohn Baldwin 		 * lock.  This may take a while and it_need may get
13658088699fSJohn Baldwin 		 * set again, so we have to check it again.
13668088699fSJohn Baldwin 		 */
1367982d11f8SJeff Roberson 		thread_lock(td);
1368e4cd31ddSJeff Roberson 		if (!ithd->it_need && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) {
13697870c3c6SJohn Baldwin 			TD_SET_IWAIT(td);
1370e0f66ef8SJohn Baldwin 			ie->ie_count = 0;
13718df78c41SJeff Roberson 			mi_switch(SW_VOL | SWT_IWAIT, NULL);
13728088699fSJohn Baldwin 		}
1373e4cd31ddSJeff Roberson 		if (ithd->it_flags & IT_WAIT) {
1374e4cd31ddSJeff Roberson 			wake = 1;
1375e4cd31ddSJeff Roberson 			ithd->it_flags &= ~IT_WAIT;
1376e4cd31ddSJeff Roberson 		}
1377982d11f8SJeff Roberson 		thread_unlock(td);
1378e4cd31ddSJeff Roberson 		if (wake) {
1379e4cd31ddSJeff Roberson 			wakeup(ithd);
1380e4cd31ddSJeff Roberson 			wake = 0;
1381e4cd31ddSJeff Roberson 		}
13828088699fSJohn Baldwin 	}
13831931cf94SJohn Baldwin }
13841ee1b687SJohn Baldwin 
13851ee1b687SJohn Baldwin /*
13861ee1b687SJohn Baldwin  * Main interrupt handling body.
13871ee1b687SJohn Baldwin  *
13881ee1b687SJohn Baldwin  * Input:
13891ee1b687SJohn Baldwin  * o ie:                        the event connected to this interrupt.
13901ee1b687SJohn Baldwin  * o frame:                     some archs (i.e. i386) pass a frame to some.
13911ee1b687SJohn Baldwin  *                              handlers as their main argument.
13921ee1b687SJohn Baldwin  * Return value:
13931ee1b687SJohn Baldwin  * o 0:                         everything ok.
13941ee1b687SJohn Baldwin  * o EINVAL:                    stray interrupt.
13951ee1b687SJohn Baldwin  */
13961ee1b687SJohn Baldwin int
13971ee1b687SJohn Baldwin intr_event_handle(struct intr_event *ie, struct trapframe *frame)
13981ee1b687SJohn Baldwin {
13991ee1b687SJohn Baldwin 	struct intr_handler *ih;
14001f255bd3SAlexander Motin 	struct trapframe *oldframe;
14011ee1b687SJohn Baldwin 	struct thread *td;
14021ee1b687SJohn Baldwin 	int error, ret, thread;
14031ee1b687SJohn Baldwin 
14041ee1b687SJohn Baldwin 	td = curthread;
14051ee1b687SJohn Baldwin 
14061ee1b687SJohn Baldwin 	/* An interrupt with no event or handlers is a stray interrupt. */
14071ee1b687SJohn Baldwin 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
14081ee1b687SJohn Baldwin 		return (EINVAL);
14091ee1b687SJohn Baldwin 
14101ee1b687SJohn Baldwin 	/*
14111ee1b687SJohn Baldwin 	 * Execute fast interrupt handlers directly.
14121ee1b687SJohn Baldwin 	 * To support clock handlers, if a handler registers
14131ee1b687SJohn Baldwin 	 * with a NULL argument, then we pass it a pointer to
14141ee1b687SJohn Baldwin 	 * a trapframe as its argument.
14151ee1b687SJohn Baldwin 	 */
14161ee1b687SJohn Baldwin 	td->td_intr_nesting_level++;
14171ee1b687SJohn Baldwin 	thread = 0;
14181ee1b687SJohn Baldwin 	ret = 0;
14191ee1b687SJohn Baldwin 	critical_enter();
14201f255bd3SAlexander Motin 	oldframe = td->td_intr_frame;
14211f255bd3SAlexander Motin 	td->td_intr_frame = frame;
14221ee1b687SJohn Baldwin 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
14231ee1b687SJohn Baldwin 		if (ih->ih_filter == NULL) {
14241ee1b687SJohn Baldwin 			thread = 1;
14251ee1b687SJohn Baldwin 			continue;
14261ee1b687SJohn Baldwin 		}
14271ee1b687SJohn Baldwin 		CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
14281ee1b687SJohn Baldwin 		    ih->ih_filter, ih->ih_argument == NULL ? frame :
14291ee1b687SJohn Baldwin 		    ih->ih_argument, ih->ih_name);
14301ee1b687SJohn Baldwin 		if (ih->ih_argument == NULL)
14311ee1b687SJohn Baldwin 			ret = ih->ih_filter(frame);
14321ee1b687SJohn Baldwin 		else
14331ee1b687SJohn Baldwin 			ret = ih->ih_filter(ih->ih_argument);
143489fc20ccSAndriy Gapon 		KASSERT(ret == FILTER_STRAY ||
143589fc20ccSAndriy Gapon 		    ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
143689fc20ccSAndriy Gapon 		    (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
143789fc20ccSAndriy Gapon 		    ("%s: incorrect return value %#x from %s", __func__, ret,
143889fc20ccSAndriy Gapon 		    ih->ih_name));
143989fc20ccSAndriy Gapon 
14401ee1b687SJohn Baldwin 		/*
14411ee1b687SJohn Baldwin 		 * Wrapper handler special handling:
14421ee1b687SJohn Baldwin 		 *
14431ee1b687SJohn Baldwin 		 * in some particular cases (like pccard and pccbb),
14441ee1b687SJohn Baldwin 		 * the _real_ device handler is wrapped in a couple of
14451ee1b687SJohn Baldwin 		 * functions - a filter wrapper and an ithread wrapper.
14461ee1b687SJohn Baldwin 		 * In this case (and just in this case), the filter wrapper
14471ee1b687SJohn Baldwin 		 * could ask the system to schedule the ithread and mask
14481ee1b687SJohn Baldwin 		 * the interrupt source if the wrapped handler is composed
14491ee1b687SJohn Baldwin 		 * of just an ithread handler.
14501ee1b687SJohn Baldwin 		 *
14511ee1b687SJohn Baldwin 		 * TODO: write a generic wrapper to avoid people rolling
14521ee1b687SJohn Baldwin 		 * their own
14531ee1b687SJohn Baldwin 		 */
14541ee1b687SJohn Baldwin 		if (!thread) {
14551ee1b687SJohn Baldwin 			if (ret == FILTER_SCHEDULE_THREAD)
14561ee1b687SJohn Baldwin 				thread = 1;
14571ee1b687SJohn Baldwin 		}
14581ee1b687SJohn Baldwin 	}
14591f255bd3SAlexander Motin 	td->td_intr_frame = oldframe;
14601ee1b687SJohn Baldwin 
14611ee1b687SJohn Baldwin 	if (thread) {
14621ee1b687SJohn Baldwin 		if (ie->ie_pre_ithread != NULL)
14631ee1b687SJohn Baldwin 			ie->ie_pre_ithread(ie->ie_source);
14641ee1b687SJohn Baldwin 	} else {
14651ee1b687SJohn Baldwin 		if (ie->ie_post_filter != NULL)
14661ee1b687SJohn Baldwin 			ie->ie_post_filter(ie->ie_source);
14671ee1b687SJohn Baldwin 	}
14681ee1b687SJohn Baldwin 
14691ee1b687SJohn Baldwin 	/* Schedule the ithread if needed. */
14701ee1b687SJohn Baldwin 	if (thread) {
14711ee1b687SJohn Baldwin 		error = intr_event_schedule_thread(ie);
14726205924aSKip Macy #ifndef XEN
14731ee1b687SJohn Baldwin 		KASSERT(error == 0, ("bad stray interrupt"));
14746205924aSKip Macy #else
14756205924aSKip Macy 		if (error != 0)
14766205924aSKip Macy 			log(LOG_WARNING, "bad stray interrupt");
14776205924aSKip Macy #endif
14781ee1b687SJohn Baldwin 	}
14791ee1b687SJohn Baldwin 	critical_exit();
14801ee1b687SJohn Baldwin 	td->td_intr_nesting_level--;
14811ee1b687SJohn Baldwin 	return (0);
14821ee1b687SJohn Baldwin }
1483bafe5a31SPaolo Pisati #else
1484bafe5a31SPaolo Pisati /*
1485bafe5a31SPaolo Pisati  * This is the main code for interrupt threads.
1486bafe5a31SPaolo Pisati  */
1487bafe5a31SPaolo Pisati static void
1488bafe5a31SPaolo Pisati ithread_loop(void *arg)
1489bafe5a31SPaolo Pisati {
1490bafe5a31SPaolo Pisati 	struct intr_thread *ithd;
1491bafe5a31SPaolo Pisati 	struct intr_handler *ih;
1492bafe5a31SPaolo Pisati 	struct intr_event *ie;
1493bafe5a31SPaolo Pisati 	struct thread *td;
1494bafe5a31SPaolo Pisati 	struct proc *p;
1495bafe5a31SPaolo Pisati 	int priv;
1496e4cd31ddSJeff Roberson 	int wake;
1497bafe5a31SPaolo Pisati 
1498bafe5a31SPaolo Pisati 	td = curthread;
1499bafe5a31SPaolo Pisati 	p = td->td_proc;
1500bafe5a31SPaolo Pisati 	ih = (struct intr_handler *)arg;
1501bafe5a31SPaolo Pisati 	priv = (ih->ih_thread != NULL) ? 1 : 0;
1502bafe5a31SPaolo Pisati 	ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread;
1503bafe5a31SPaolo Pisati 	KASSERT(ithd->it_thread == td,
1504bafe5a31SPaolo Pisati 	    ("%s: ithread and proc linkage out of sync", __func__));
1505bafe5a31SPaolo Pisati 	ie = ithd->it_event;
1506bafe5a31SPaolo Pisati 	ie->ie_count = 0;
1507e4cd31ddSJeff Roberson 	wake = 0;
1508bafe5a31SPaolo Pisati 
1509bafe5a31SPaolo Pisati 	/*
1510bafe5a31SPaolo Pisati 	 * As long as we have interrupts outstanding, go through the
1511bafe5a31SPaolo Pisati 	 * list of handlers, giving each one a go at it.
1512bafe5a31SPaolo Pisati 	 */
1513bafe5a31SPaolo Pisati 	for (;;) {
1514bafe5a31SPaolo Pisati 		/*
1515bafe5a31SPaolo Pisati 		 * If we are an orphaned thread, then just die.
1516bafe5a31SPaolo Pisati 		 */
1517bafe5a31SPaolo Pisati 		if (ithd->it_flags & IT_DEAD) {
1518bafe5a31SPaolo Pisati 			CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
15197ab24ea3SJulian Elischer 			    p->p_pid, td->td_name);
1520bafe5a31SPaolo Pisati 			free(ithd, M_ITHREAD);
1521ca9a0ddfSJulian Elischer 			kthread_exit();
1522bafe5a31SPaolo Pisati 		}
1523bafe5a31SPaolo Pisati 
1524bafe5a31SPaolo Pisati 		/*
1525bafe5a31SPaolo Pisati 		 * Service interrupts.  If another interrupt arrives while
1526bafe5a31SPaolo Pisati 		 * we are running, it will set it_need to note that we
1527bafe5a31SPaolo Pisati 		 * should make another pass.
1528bafe5a31SPaolo Pisati 		 */
1529bafe5a31SPaolo Pisati 		while (ithd->it_need) {
1530bafe5a31SPaolo Pisati 			/*
1531bafe5a31SPaolo Pisati 			 * This might need a full read and write barrier
1532bafe5a31SPaolo Pisati 			 * to make sure that this write posts before any
1533bafe5a31SPaolo Pisati 			 * of the memory or device accesses in the
1534bafe5a31SPaolo Pisati 			 * handlers.
1535bafe5a31SPaolo Pisati 			 */
1536bafe5a31SPaolo Pisati 			atomic_store_rel_int(&ithd->it_need, 0);
1537bafe5a31SPaolo Pisati 			if (priv)
1538bafe5a31SPaolo Pisati 				priv_ithread_execute_handler(p, ih);
1539bafe5a31SPaolo Pisati 			else
1540bafe5a31SPaolo Pisati 				ithread_execute_handlers(p, ie);
1541bafe5a31SPaolo Pisati 		}
1542bafe5a31SPaolo Pisati 		WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
1543bafe5a31SPaolo Pisati 		mtx_assert(&Giant, MA_NOTOWNED);
1544bafe5a31SPaolo Pisati 
1545bafe5a31SPaolo Pisati 		/*
1546bafe5a31SPaolo Pisati 		 * Processed all our interrupts.  Now get the sched
1547bafe5a31SPaolo Pisati 		 * lock.  This may take a while and it_need may get
1548bafe5a31SPaolo Pisati 		 * set again, so we have to check it again.
1549bafe5a31SPaolo Pisati 		 */
1550982d11f8SJeff Roberson 		thread_lock(td);
1551e4cd31ddSJeff Roberson 		if (!ithd->it_need && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) {
1552bafe5a31SPaolo Pisati 			TD_SET_IWAIT(td);
1553bafe5a31SPaolo Pisati 			ie->ie_count = 0;
15548df78c41SJeff Roberson 			mi_switch(SW_VOL | SWT_IWAIT, NULL);
1555bafe5a31SPaolo Pisati 		}
1556e4cd31ddSJeff Roberson 		if (ithd->it_flags & IT_WAIT) {
1557e4cd31ddSJeff Roberson 			wake = 1;
1558e4cd31ddSJeff Roberson 			ithd->it_flags &= ~IT_WAIT;
1559e4cd31ddSJeff Roberson 		}
1560982d11f8SJeff Roberson 		thread_unlock(td);
1561e4cd31ddSJeff Roberson 		if (wake) {
1562e4cd31ddSJeff Roberson 			wakeup(ithd);
1563e4cd31ddSJeff Roberson 			wake = 0;
1564e4cd31ddSJeff Roberson 		}
1565bafe5a31SPaolo Pisati 	}
1566bafe5a31SPaolo Pisati }
1567bafe5a31SPaolo Pisati 
1568bafe5a31SPaolo Pisati /*
1569bafe5a31SPaolo Pisati  * Main loop for interrupt filter.
1570bafe5a31SPaolo Pisati  *
1571bafe5a31SPaolo Pisati  * Some architectures (i386, amd64 and arm) require the optional frame
1572bafe5a31SPaolo Pisati  * parameter, and use it as the main argument for fast handler execution
1573bafe5a31SPaolo Pisati  * when ih_argument == NULL.
1574bafe5a31SPaolo Pisati  *
1575bafe5a31SPaolo Pisati  * Return value:
1576bafe5a31SPaolo Pisati  * o FILTER_STRAY:              No filter recognized the event, and no
1577bafe5a31SPaolo Pisati  *                              filter-less handler is registered on this
1578bafe5a31SPaolo Pisati  *                              line.
1579bafe5a31SPaolo Pisati  * o FILTER_HANDLED:            A filter claimed the event and served it.
1580bafe5a31SPaolo Pisati  * o FILTER_SCHEDULE_THREAD:    No filter claimed the event, but there's at
1581bafe5a31SPaolo Pisati  *                              least one filter-less handler on this line.
1582bafe5a31SPaolo Pisati  * o FILTER_HANDLED |
1583bafe5a31SPaolo Pisati  *   FILTER_SCHEDULE_THREAD:    A filter claimed the event, and asked for
1584bafe5a31SPaolo Pisati  *                              scheduling the per-handler ithread.
1585bafe5a31SPaolo Pisati  *
1586bafe5a31SPaolo Pisati  * In case an ithread has to be scheduled, in *ithd there will be a
1587bafe5a31SPaolo Pisati  * pointer to a struct intr_thread containing the thread to be
1588bafe5a31SPaolo Pisati  * scheduled.
1589bafe5a31SPaolo Pisati  */
1590bafe5a31SPaolo Pisati 
15911ee1b687SJohn Baldwin static int
1592bafe5a31SPaolo Pisati intr_filter_loop(struct intr_event *ie, struct trapframe *frame,
1593bafe5a31SPaolo Pisati 		 struct intr_thread **ithd)
1594bafe5a31SPaolo Pisati {
1595bafe5a31SPaolo Pisati 	struct intr_handler *ih;
1596bafe5a31SPaolo Pisati 	void *arg;
1597bafe5a31SPaolo Pisati 	int ret, thread_only;
1598bafe5a31SPaolo Pisati 
1599bafe5a31SPaolo Pisati 	ret = 0;
1600bafe5a31SPaolo Pisati 	thread_only = 0;
1601bafe5a31SPaolo Pisati 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1602bafe5a31SPaolo Pisati 		/*
1603bafe5a31SPaolo Pisati 		 * Execute fast interrupt handlers directly.
1604bafe5a31SPaolo Pisati 		 * To support clock handlers, if a handler registers
1605bafe5a31SPaolo Pisati 		 * with a NULL argument, then we pass it a pointer to
1606bafe5a31SPaolo Pisati 		 * a trapframe as its argument.
1607bafe5a31SPaolo Pisati 		 */
1608bafe5a31SPaolo Pisati 		arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument);
1609bafe5a31SPaolo Pisati 
1610bafe5a31SPaolo Pisati 		CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__,
1611bafe5a31SPaolo Pisati 		     ih->ih_filter, ih->ih_handler, arg, ih->ih_name);
1612bafe5a31SPaolo Pisati 
1613bafe5a31SPaolo Pisati 		if (ih->ih_filter != NULL)
1614bafe5a31SPaolo Pisati 			ret = ih->ih_filter(arg);
1615bafe5a31SPaolo Pisati 		else {
1616bafe5a31SPaolo Pisati 			thread_only = 1;
1617bafe5a31SPaolo Pisati 			continue;
1618bafe5a31SPaolo Pisati 		}
161989fc20ccSAndriy Gapon 		KASSERT(ret == FILTER_STRAY ||
162089fc20ccSAndriy Gapon 		    ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
162189fc20ccSAndriy Gapon 		    (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
162289fc20ccSAndriy Gapon 		    ("%s: incorrect return value %#x from %s", __func__, ret,
162389fc20ccSAndriy Gapon 		    ih->ih_name));
1624bafe5a31SPaolo Pisati 		if (ret & FILTER_STRAY)
1625bafe5a31SPaolo Pisati 			continue;
1626bafe5a31SPaolo Pisati 		else {
1627bafe5a31SPaolo Pisati 			*ithd = ih->ih_thread;
1628bafe5a31SPaolo Pisati 			return (ret);
1629bafe5a31SPaolo Pisati 		}
1630bafe5a31SPaolo Pisati 	}
1631bafe5a31SPaolo Pisati 
1632bafe5a31SPaolo Pisati 	/*
1633bafe5a31SPaolo Pisati 	 * No filters handled the interrupt and we have at least
1634bafe5a31SPaolo Pisati 	 * one handler without a filter.  In this case, we schedule
1635bafe5a31SPaolo Pisati 	 * all of the filter-less handlers to run in the ithread.
1636bafe5a31SPaolo Pisati 	 */
1637bafe5a31SPaolo Pisati 	if (thread_only) {
1638bafe5a31SPaolo Pisati 		*ithd = ie->ie_thread;
1639bafe5a31SPaolo Pisati 		return (FILTER_SCHEDULE_THREAD);
1640bafe5a31SPaolo Pisati 	}
1641bafe5a31SPaolo Pisati 	return (FILTER_STRAY);
1642bafe5a31SPaolo Pisati }
1643bafe5a31SPaolo Pisati 
1644bafe5a31SPaolo Pisati /*
1645bafe5a31SPaolo Pisati  * Main interrupt handling body.
1646bafe5a31SPaolo Pisati  *
1647bafe5a31SPaolo Pisati  * Input:
1648bafe5a31SPaolo Pisati  * o ie:                        the event connected to this interrupt.
1649bafe5a31SPaolo Pisati  * o frame:                     some archs (i.e. i386) pass a frame to some.
1650bafe5a31SPaolo Pisati  *                              handlers as their main argument.
1651bafe5a31SPaolo Pisati  * Return value:
1652bafe5a31SPaolo Pisati  * o 0:                         everything ok.
1653bafe5a31SPaolo Pisati  * o EINVAL:                    stray interrupt.
1654bafe5a31SPaolo Pisati  */
1655bafe5a31SPaolo Pisati int
1656bafe5a31SPaolo Pisati intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1657bafe5a31SPaolo Pisati {
1658bafe5a31SPaolo Pisati 	struct intr_thread *ithd;
16591f255bd3SAlexander Motin 	struct trapframe *oldframe;
1660bafe5a31SPaolo Pisati 	struct thread *td;
1661bafe5a31SPaolo Pisati 	int thread;
1662bafe5a31SPaolo Pisati 
1663bafe5a31SPaolo Pisati 	ithd = NULL;
1664bafe5a31SPaolo Pisati 	td = curthread;
1665bafe5a31SPaolo Pisati 
1666bafe5a31SPaolo Pisati 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1667bafe5a31SPaolo Pisati 		return (EINVAL);
1668bafe5a31SPaolo Pisati 
1669bafe5a31SPaolo Pisati 	td->td_intr_nesting_level++;
1670bafe5a31SPaolo Pisati 	thread = 0;
1671bafe5a31SPaolo Pisati 	critical_enter();
16721f255bd3SAlexander Motin 	oldframe = td->td_intr_frame;
16731f255bd3SAlexander Motin 	td->td_intr_frame = frame;
1674bafe5a31SPaolo Pisati 	thread = intr_filter_loop(ie, frame, &ithd);
1675bafe5a31SPaolo Pisati 	if (thread & FILTER_HANDLED) {
16761ee1b687SJohn Baldwin 		if (ie->ie_post_filter != NULL)
16771ee1b687SJohn Baldwin 			ie->ie_post_filter(ie->ie_source);
1678bafe5a31SPaolo Pisati 	} else {
16791ee1b687SJohn Baldwin 		if (ie->ie_pre_ithread != NULL)
16801ee1b687SJohn Baldwin 			ie->ie_pre_ithread(ie->ie_source);
1681bafe5a31SPaolo Pisati 	}
16821f255bd3SAlexander Motin 	td->td_intr_frame = oldframe;
1683bafe5a31SPaolo Pisati 	critical_exit();
1684bafe5a31SPaolo Pisati 
1685bafe5a31SPaolo Pisati 	/* Interrupt storm logic */
1686bafe5a31SPaolo Pisati 	if (thread & FILTER_STRAY) {
1687bafe5a31SPaolo Pisati 		ie->ie_count++;
1688bafe5a31SPaolo Pisati 		if (ie->ie_count < intr_storm_threshold)
1689bafe5a31SPaolo Pisati 			printf("Interrupt stray detection not present\n");
1690bafe5a31SPaolo Pisati 	}
1691bafe5a31SPaolo Pisati 
1692bafe5a31SPaolo Pisati 	/* Schedule an ithread if needed. */
1693bafe5a31SPaolo Pisati 	if (thread & FILTER_SCHEDULE_THREAD) {
1694bafe5a31SPaolo Pisati 		if (intr_event_schedule_thread(ie, ithd) != 0)
1695bafe5a31SPaolo Pisati 			panic("%s: impossible stray interrupt", __func__);
1696bafe5a31SPaolo Pisati 	}
1697bafe5a31SPaolo Pisati 	td->td_intr_nesting_level--;
1698bafe5a31SPaolo Pisati 	return (0);
1699bafe5a31SPaolo Pisati }
1700bafe5a31SPaolo Pisati #endif
17011931cf94SJohn Baldwin 
17028b201c42SJohn Baldwin #ifdef DDB
17038b201c42SJohn Baldwin /*
17048b201c42SJohn Baldwin  * Dump details about an interrupt handler
17058b201c42SJohn Baldwin  */
17068b201c42SJohn Baldwin static void
1707e0f66ef8SJohn Baldwin db_dump_intrhand(struct intr_handler *ih)
17088b201c42SJohn Baldwin {
17098b201c42SJohn Baldwin 	int comma;
17108b201c42SJohn Baldwin 
17118b201c42SJohn Baldwin 	db_printf("\t%-10s ", ih->ih_name);
17128b201c42SJohn Baldwin 	switch (ih->ih_pri) {
17138b201c42SJohn Baldwin 	case PI_REALTIME:
17148b201c42SJohn Baldwin 		db_printf("CLK ");
17158b201c42SJohn Baldwin 		break;
17168b201c42SJohn Baldwin 	case PI_AV:
17178b201c42SJohn Baldwin 		db_printf("AV  ");
17188b201c42SJohn Baldwin 		break;
1719d3305205SJohn Baldwin 	case PI_TTY:
17208b201c42SJohn Baldwin 		db_printf("TTY ");
17218b201c42SJohn Baldwin 		break;
17228b201c42SJohn Baldwin 	case PI_NET:
17238b201c42SJohn Baldwin 		db_printf("NET ");
17248b201c42SJohn Baldwin 		break;
17258b201c42SJohn Baldwin 	case PI_DISK:
17268b201c42SJohn Baldwin 		db_printf("DISK");
17278b201c42SJohn Baldwin 		break;
17288b201c42SJohn Baldwin 	case PI_DULL:
17298b201c42SJohn Baldwin 		db_printf("DULL");
17308b201c42SJohn Baldwin 		break;
17318b201c42SJohn Baldwin 	default:
17328b201c42SJohn Baldwin 		if (ih->ih_pri >= PI_SOFT)
17338b201c42SJohn Baldwin 			db_printf("SWI ");
17348b201c42SJohn Baldwin 		else
17358b201c42SJohn Baldwin 			db_printf("%4u", ih->ih_pri);
17368b201c42SJohn Baldwin 		break;
17378b201c42SJohn Baldwin 	}
17388b201c42SJohn Baldwin 	db_printf(" ");
17398b201c42SJohn Baldwin 	db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC);
17408b201c42SJohn Baldwin 	db_printf("(%p)", ih->ih_argument);
17418b201c42SJohn Baldwin 	if (ih->ih_need ||
1742ef544f63SPaolo Pisati 	    (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD |
17438b201c42SJohn Baldwin 	    IH_MPSAFE)) != 0) {
17448b201c42SJohn Baldwin 		db_printf(" {");
17458b201c42SJohn Baldwin 		comma = 0;
17468b201c42SJohn Baldwin 		if (ih->ih_flags & IH_EXCLUSIVE) {
17478b201c42SJohn Baldwin 			if (comma)
17488b201c42SJohn Baldwin 				db_printf(", ");
17498b201c42SJohn Baldwin 			db_printf("EXCL");
17508b201c42SJohn Baldwin 			comma = 1;
17518b201c42SJohn Baldwin 		}
17528b201c42SJohn Baldwin 		if (ih->ih_flags & IH_ENTROPY) {
17538b201c42SJohn Baldwin 			if (comma)
17548b201c42SJohn Baldwin 				db_printf(", ");
17558b201c42SJohn Baldwin 			db_printf("ENTROPY");
17568b201c42SJohn Baldwin 			comma = 1;
17578b201c42SJohn Baldwin 		}
17588b201c42SJohn Baldwin 		if (ih->ih_flags & IH_DEAD) {
17598b201c42SJohn Baldwin 			if (comma)
17608b201c42SJohn Baldwin 				db_printf(", ");
17618b201c42SJohn Baldwin 			db_printf("DEAD");
17628b201c42SJohn Baldwin 			comma = 1;
17638b201c42SJohn Baldwin 		}
17648b201c42SJohn Baldwin 		if (ih->ih_flags & IH_MPSAFE) {
17658b201c42SJohn Baldwin 			if (comma)
17668b201c42SJohn Baldwin 				db_printf(", ");
17678b201c42SJohn Baldwin 			db_printf("MPSAFE");
17688b201c42SJohn Baldwin 			comma = 1;
17698b201c42SJohn Baldwin 		}
17708b201c42SJohn Baldwin 		if (ih->ih_need) {
17718b201c42SJohn Baldwin 			if (comma)
17728b201c42SJohn Baldwin 				db_printf(", ");
17738b201c42SJohn Baldwin 			db_printf("NEED");
17748b201c42SJohn Baldwin 		}
17758b201c42SJohn Baldwin 		db_printf("}");
17768b201c42SJohn Baldwin 	}
17778b201c42SJohn Baldwin 	db_printf("\n");
17788b201c42SJohn Baldwin }
17798b201c42SJohn Baldwin 
17808b201c42SJohn Baldwin /*
1781e0f66ef8SJohn Baldwin  * Dump details about a event.
17828b201c42SJohn Baldwin  */
17838b201c42SJohn Baldwin void
1784e0f66ef8SJohn Baldwin db_dump_intr_event(struct intr_event *ie, int handlers)
17858b201c42SJohn Baldwin {
1786e0f66ef8SJohn Baldwin 	struct intr_handler *ih;
1787e0f66ef8SJohn Baldwin 	struct intr_thread *it;
17888b201c42SJohn Baldwin 	int comma;
17898b201c42SJohn Baldwin 
1790e0f66ef8SJohn Baldwin 	db_printf("%s ", ie->ie_fullname);
1791e0f66ef8SJohn Baldwin 	it = ie->ie_thread;
1792e0f66ef8SJohn Baldwin 	if (it != NULL)
1793e0f66ef8SJohn Baldwin 		db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
1794e0f66ef8SJohn Baldwin 	else
1795e0f66ef8SJohn Baldwin 		db_printf("(no thread)");
1796e0f66ef8SJohn Baldwin 	if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
1797e0f66ef8SJohn Baldwin 	    (it != NULL && it->it_need)) {
17988b201c42SJohn Baldwin 		db_printf(" {");
17998b201c42SJohn Baldwin 		comma = 0;
1800e0f66ef8SJohn Baldwin 		if (ie->ie_flags & IE_SOFT) {
18018b201c42SJohn Baldwin 			db_printf("SOFT");
18028b201c42SJohn Baldwin 			comma = 1;
18038b201c42SJohn Baldwin 		}
1804e0f66ef8SJohn Baldwin 		if (ie->ie_flags & IE_ENTROPY) {
18058b201c42SJohn Baldwin 			if (comma)
18068b201c42SJohn Baldwin 				db_printf(", ");
18078b201c42SJohn Baldwin 			db_printf("ENTROPY");
18088b201c42SJohn Baldwin 			comma = 1;
18098b201c42SJohn Baldwin 		}
1810e0f66ef8SJohn Baldwin 		if (ie->ie_flags & IE_ADDING_THREAD) {
18118b201c42SJohn Baldwin 			if (comma)
18128b201c42SJohn Baldwin 				db_printf(", ");
1813e0f66ef8SJohn Baldwin 			db_printf("ADDING_THREAD");
18148b201c42SJohn Baldwin 			comma = 1;
18158b201c42SJohn Baldwin 		}
1816e0f66ef8SJohn Baldwin 		if (it != NULL && it->it_need) {
18178b201c42SJohn Baldwin 			if (comma)
18188b201c42SJohn Baldwin 				db_printf(", ");
18198b201c42SJohn Baldwin 			db_printf("NEED");
18208b201c42SJohn Baldwin 		}
18218b201c42SJohn Baldwin 		db_printf("}");
18228b201c42SJohn Baldwin 	}
18238b201c42SJohn Baldwin 	db_printf("\n");
18248b201c42SJohn Baldwin 
18258b201c42SJohn Baldwin 	if (handlers)
1826e0f66ef8SJohn Baldwin 		TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
18278b201c42SJohn Baldwin 		    db_dump_intrhand(ih);
18288b201c42SJohn Baldwin }
1829e0f66ef8SJohn Baldwin 
1830e0f66ef8SJohn Baldwin /*
1831e0f66ef8SJohn Baldwin  * Dump data about interrupt handlers
1832e0f66ef8SJohn Baldwin  */
1833e0f66ef8SJohn Baldwin DB_SHOW_COMMAND(intr, db_show_intr)
1834e0f66ef8SJohn Baldwin {
1835e0f66ef8SJohn Baldwin 	struct intr_event *ie;
183619e9205aSJohn Baldwin 	int all, verbose;
1837e0f66ef8SJohn Baldwin 
1838dc15eac0SEd Schouten 	verbose = strchr(modif, 'v') != NULL;
1839dc15eac0SEd Schouten 	all = strchr(modif, 'a') != NULL;
1840e0f66ef8SJohn Baldwin 	TAILQ_FOREACH(ie, &event_list, ie_list) {
1841e0f66ef8SJohn Baldwin 		if (!all && TAILQ_EMPTY(&ie->ie_handlers))
1842e0f66ef8SJohn Baldwin 			continue;
1843e0f66ef8SJohn Baldwin 		db_dump_intr_event(ie, verbose);
184419e9205aSJohn Baldwin 		if (db_pager_quit)
184519e9205aSJohn Baldwin 			break;
1846e0f66ef8SJohn Baldwin 	}
1847e0f66ef8SJohn Baldwin }
18488b201c42SJohn Baldwin #endif /* DDB */
18498b201c42SJohn Baldwin 
1850b4151f71SJohn Baldwin /*
18518088699fSJohn Baldwin  * Start standard software interrupt threads
18521931cf94SJohn Baldwin  */
18531931cf94SJohn Baldwin static void
1854b4151f71SJohn Baldwin start_softintr(void *dummy)
18551931cf94SJohn Baldwin {
1856b4151f71SJohn Baldwin 
18578d809d50SJeff Roberson 	if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
18588d809d50SJeff Roberson 		panic("died while creating vm swi ithread");
18591931cf94SJohn Baldwin }
1860237fdd78SRobert Watson SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr,
1861237fdd78SRobert Watson     NULL);
18621931cf94SJohn Baldwin 
1863d279178dSThomas Moestl /*
1864d279178dSThomas Moestl  * Sysctls used by systat and others: hw.intrnames and hw.intrcnt.
1865d279178dSThomas Moestl  * The data for this machine dependent, and the declarations are in machine
1866d279178dSThomas Moestl  * dependent code.  The layout of intrnames and intrcnt however is machine
1867d279178dSThomas Moestl  * independent.
1868d279178dSThomas Moestl  *
1869d279178dSThomas Moestl  * We do not know the length of intrcnt and intrnames at compile time, so
1870d279178dSThomas Moestl  * calculate things at run time.
1871d279178dSThomas Moestl  */
1872d279178dSThomas Moestl static int
1873d279178dSThomas Moestl sysctl_intrnames(SYSCTL_HANDLER_ARGS)
1874d279178dSThomas Moestl {
1875521ea19dSAttilio Rao 	return (sysctl_handle_opaque(oidp, intrnames, sintrnames, req));
1876d279178dSThomas Moestl }
1877d279178dSThomas Moestl 
1878d279178dSThomas Moestl SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD,
1879d279178dSThomas Moestl     NULL, 0, sysctl_intrnames, "", "Interrupt Names");
1880d279178dSThomas Moestl 
1881d279178dSThomas Moestl static int
1882d279178dSThomas Moestl sysctl_intrcnt(SYSCTL_HANDLER_ARGS)
1883d279178dSThomas Moestl {
1884521ea19dSAttilio Rao 	return (sysctl_handle_opaque(oidp, intrcnt, sintrcnt, req));
1885d279178dSThomas Moestl }
1886d279178dSThomas Moestl 
1887d279178dSThomas Moestl SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD,
1888d279178dSThomas Moestl     NULL, 0, sysctl_intrcnt, "", "Interrupt Counts");
18898b201c42SJohn Baldwin 
18908b201c42SJohn Baldwin #ifdef DDB
18918b201c42SJohn Baldwin /*
18928b201c42SJohn Baldwin  * DDB command to dump the interrupt statistics.
18938b201c42SJohn Baldwin  */
18948b201c42SJohn Baldwin DB_SHOW_COMMAND(intrcnt, db_show_intrcnt)
18958b201c42SJohn Baldwin {
18968b201c42SJohn Baldwin 	u_long *i;
18978b201c42SJohn Baldwin 	char *cp;
1898521ea19dSAttilio Rao 	u_int j;
18998b201c42SJohn Baldwin 
19008b201c42SJohn Baldwin 	cp = intrnames;
1901521ea19dSAttilio Rao 	j = 0;
1902521ea19dSAttilio Rao 	for (i = intrcnt; j < (sintrcnt / sizeof(u_long)) && !db_pager_quit;
1903521ea19dSAttilio Rao 	    i++, j++) {
19048b201c42SJohn Baldwin 		if (*cp == '\0')
19058b201c42SJohn Baldwin 			break;
19068b201c42SJohn Baldwin 		if (*i != 0)
19078b201c42SJohn Baldwin 			db_printf("%s\t%lu\n", cp, *i);
19088b201c42SJohn Baldwin 		cp += strlen(cp) + 1;
19098b201c42SJohn Baldwin 	}
19108b201c42SJohn Baldwin }
19118b201c42SJohn Baldwin #endif
1912