xref: /freebsd/sys/kern/kern_intr.c (revision 38d120bc13ac1de5b739b67b87016b9122149374)
1 /*-
2  * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include "opt_ddb.h"
31 #include "opt_kstack_usage_prof.h"
32 
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/conf.h>
36 #include <sys/cpuset.h>
37 #include <sys/rtprio.h>
38 #include <sys/systm.h>
39 #include <sys/interrupt.h>
40 #include <sys/kernel.h>
41 #include <sys/kthread.h>
42 #include <sys/ktr.h>
43 #include <sys/limits.h>
44 #include <sys/lock.h>
45 #include <sys/malloc.h>
46 #include <sys/mutex.h>
47 #include <sys/priv.h>
48 #include <sys/proc.h>
49 #include <sys/random.h>
50 #include <sys/resourcevar.h>
51 #include <sys/sched.h>
52 #include <sys/smp.h>
53 #include <sys/sysctl.h>
54 #include <sys/syslog.h>
55 #include <sys/unistd.h>
56 #include <sys/vmmeter.h>
57 #include <machine/atomic.h>
58 #include <machine/cpu.h>
59 #include <machine/md_var.h>
60 #include <machine/stdarg.h>
61 #ifdef DDB
62 #include <ddb/ddb.h>
63 #include <ddb/db_sym.h>
64 #endif
65 
66 /*
67  * Describe an interrupt thread.  There is one of these per interrupt event.
68  */
69 struct intr_thread {
70 	struct intr_event *it_event;
71 	struct thread *it_thread;	/* Kernel thread. */
72 	int	it_flags;		/* (j) IT_* flags. */
73 	int	it_need;		/* Needs service. */
74 };
75 
76 /* Interrupt thread flags kept in it_flags */
77 #define	IT_DEAD		0x000001	/* Thread is waiting to exit. */
78 #define	IT_WAIT		0x000002	/* Thread is waiting for completion. */
79 
80 struct	intr_entropy {
81 	struct	thread *td;
82 	uintptr_t event;
83 };
84 
85 struct	intr_event *clk_intr_event;
86 struct	intr_event *tty_intr_event;
87 void	*vm_ih;
88 struct proc *intrproc;
89 
90 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads");
91 
92 static int intr_storm_threshold = 1000;
93 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN,
94     &intr_storm_threshold, 0,
95     "Number of consecutive interrupts before storm protection is enabled");
96 static TAILQ_HEAD(, intr_event) event_list =
97     TAILQ_HEAD_INITIALIZER(event_list);
98 static struct mtx event_lock;
99 MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF);
100 
101 static void	intr_event_update(struct intr_event *ie);
102 #ifdef INTR_FILTER
103 static int	intr_event_schedule_thread(struct intr_event *ie,
104 		    struct intr_thread *ithd);
105 static int	intr_filter_loop(struct intr_event *ie,
106 		    struct trapframe *frame, struct intr_thread **ithd);
107 static struct intr_thread *ithread_create(const char *name,
108 			      struct intr_handler *ih);
109 #else
110 static int	intr_event_schedule_thread(struct intr_event *ie);
111 static struct intr_thread *ithread_create(const char *name);
112 #endif
113 static void	ithread_destroy(struct intr_thread *ithread);
114 static void	ithread_execute_handlers(struct proc *p,
115 		    struct intr_event *ie);
116 #ifdef INTR_FILTER
117 static void	priv_ithread_execute_handler(struct proc *p,
118 		    struct intr_handler *ih);
119 #endif
120 static void	ithread_loop(void *);
121 static void	ithread_update(struct intr_thread *ithd);
122 static void	start_softintr(void *);
123 
124 /* Map an interrupt type to an ithread priority. */
125 u_char
126 intr_priority(enum intr_type flags)
127 {
128 	u_char pri;
129 
130 	flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET |
131 	    INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV);
132 	switch (flags) {
133 	case INTR_TYPE_TTY:
134 		pri = PI_TTY;
135 		break;
136 	case INTR_TYPE_BIO:
137 		pri = PI_DISK;
138 		break;
139 	case INTR_TYPE_NET:
140 		pri = PI_NET;
141 		break;
142 	case INTR_TYPE_CAM:
143 		pri = PI_DISK;
144 		break;
145 	case INTR_TYPE_AV:
146 		pri = PI_AV;
147 		break;
148 	case INTR_TYPE_CLK:
149 		pri = PI_REALTIME;
150 		break;
151 	case INTR_TYPE_MISC:
152 		pri = PI_DULL;          /* don't care */
153 		break;
154 	default:
155 		/* We didn't specify an interrupt level. */
156 		panic("intr_priority: no interrupt type in flags");
157 	}
158 
159 	return pri;
160 }
161 
162 /*
163  * Update an ithread based on the associated intr_event.
164  */
165 static void
166 ithread_update(struct intr_thread *ithd)
167 {
168 	struct intr_event *ie;
169 	struct thread *td;
170 	u_char pri;
171 
172 	ie = ithd->it_event;
173 	td = ithd->it_thread;
174 
175 	/* Determine the overall priority of this event. */
176 	if (TAILQ_EMPTY(&ie->ie_handlers))
177 		pri = PRI_MAX_ITHD;
178 	else
179 		pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri;
180 
181 	/* Update name and priority. */
182 	strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name));
183 #ifdef KTR
184 	sched_clear_tdname(td);
185 #endif
186 	thread_lock(td);
187 	sched_prio(td, pri);
188 	thread_unlock(td);
189 }
190 
191 /*
192  * Regenerate the full name of an interrupt event and update its priority.
193  */
194 static void
195 intr_event_update(struct intr_event *ie)
196 {
197 	struct intr_handler *ih;
198 	char *last;
199 	int missed, space;
200 
201 	/* Start off with no entropy and just the name of the event. */
202 	mtx_assert(&ie->ie_lock, MA_OWNED);
203 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
204 	ie->ie_flags &= ~IE_ENTROPY;
205 	missed = 0;
206 	space = 1;
207 
208 	/* Run through all the handlers updating values. */
209 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
210 		if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
211 		    sizeof(ie->ie_fullname)) {
212 			strcat(ie->ie_fullname, " ");
213 			strcat(ie->ie_fullname, ih->ih_name);
214 			space = 0;
215 		} else
216 			missed++;
217 		if (ih->ih_flags & IH_ENTROPY)
218 			ie->ie_flags |= IE_ENTROPY;
219 	}
220 
221 	/*
222 	 * If the handler names were too long, add +'s to indicate missing
223 	 * names. If we run out of room and still have +'s to add, change
224 	 * the last character from a + to a *.
225 	 */
226 	last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
227 	while (missed-- > 0) {
228 		if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
229 			if (*last == '+') {
230 				*last = '*';
231 				break;
232 			} else
233 				*last = '+';
234 		} else if (space) {
235 			strcat(ie->ie_fullname, " +");
236 			space = 0;
237 		} else
238 			strcat(ie->ie_fullname, "+");
239 	}
240 
241 	/*
242 	 * If this event has an ithread, update it's priority and
243 	 * name.
244 	 */
245 	if (ie->ie_thread != NULL)
246 		ithread_update(ie->ie_thread);
247 	CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
248 }
249 
250 int
251 intr_event_create(struct intr_event **event, void *source, int flags, int irq,
252     void (*pre_ithread)(void *), void (*post_ithread)(void *),
253     void (*post_filter)(void *), int (*assign_cpu)(void *, int),
254     const char *fmt, ...)
255 {
256 	struct intr_event *ie;
257 	va_list ap;
258 
259 	/* The only valid flag during creation is IE_SOFT. */
260 	if ((flags & ~IE_SOFT) != 0)
261 		return (EINVAL);
262 	ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
263 	ie->ie_source = source;
264 	ie->ie_pre_ithread = pre_ithread;
265 	ie->ie_post_ithread = post_ithread;
266 	ie->ie_post_filter = post_filter;
267 	ie->ie_assign_cpu = assign_cpu;
268 	ie->ie_flags = flags;
269 	ie->ie_irq = irq;
270 	ie->ie_cpu = NOCPU;
271 	TAILQ_INIT(&ie->ie_handlers);
272 	mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
273 
274 	va_start(ap, fmt);
275 	vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
276 	va_end(ap);
277 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
278 	mtx_lock(&event_lock);
279 	TAILQ_INSERT_TAIL(&event_list, ie, ie_list);
280 	mtx_unlock(&event_lock);
281 	if (event != NULL)
282 		*event = ie;
283 	CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
284 	return (0);
285 }
286 
287 /*
288  * Bind an interrupt event to the specified CPU.  Note that not all
289  * platforms support binding an interrupt to a CPU.  For those
290  * platforms this request will fail.  For supported platforms, any
291  * associated ithreads as well as the primary interrupt context will
292  * be bound to the specificed CPU.  Using a cpu id of NOCPU unbinds
293  * the interrupt event.
294  */
295 int
296 intr_event_bind(struct intr_event *ie, int cpu)
297 {
298 	lwpid_t id;
299 	int error;
300 
301 	/* Need a CPU to bind to. */
302 	if (cpu != NOCPU && CPU_ABSENT(cpu))
303 		return (EINVAL);
304 
305 	if (ie->ie_assign_cpu == NULL)
306 		return (EOPNOTSUPP);
307 
308 	error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR);
309 	if (error)
310 		return (error);
311 
312 	/*
313 	 * If we have any ithreads try to set their mask first to verify
314 	 * permissions, etc.
315 	 */
316 	mtx_lock(&ie->ie_lock);
317 	if (ie->ie_thread != NULL) {
318 		id = ie->ie_thread->it_thread->td_tid;
319 		mtx_unlock(&ie->ie_lock);
320 		error = cpuset_setithread(id, cpu);
321 		if (error)
322 			return (error);
323 	} else
324 		mtx_unlock(&ie->ie_lock);
325 	error = ie->ie_assign_cpu(ie->ie_source, cpu);
326 	if (error) {
327 		mtx_lock(&ie->ie_lock);
328 		if (ie->ie_thread != NULL) {
329 			cpu = ie->ie_cpu;
330 			id = ie->ie_thread->it_thread->td_tid;
331 			mtx_unlock(&ie->ie_lock);
332 			(void)cpuset_setithread(id, cpu);
333 		} else
334 			mtx_unlock(&ie->ie_lock);
335 		return (error);
336 	}
337 
338 	mtx_lock(&ie->ie_lock);
339 	ie->ie_cpu = cpu;
340 	mtx_unlock(&ie->ie_lock);
341 
342 	return (error);
343 }
344 
345 static struct intr_event *
346 intr_lookup(int irq)
347 {
348 	struct intr_event *ie;
349 
350 	mtx_lock(&event_lock);
351 	TAILQ_FOREACH(ie, &event_list, ie_list)
352 		if (ie->ie_irq == irq &&
353 		    (ie->ie_flags & IE_SOFT) == 0 &&
354 		    TAILQ_FIRST(&ie->ie_handlers) != NULL)
355 			break;
356 	mtx_unlock(&event_lock);
357 	return (ie);
358 }
359 
360 int
361 intr_setaffinity(int irq, void *m)
362 {
363 	struct intr_event *ie;
364 	cpuset_t *mask;
365 	int cpu, n;
366 
367 	mask = m;
368 	cpu = NOCPU;
369 	/*
370 	 * If we're setting all cpus we can unbind.  Otherwise make sure
371 	 * only one cpu is in the set.
372 	 */
373 	if (CPU_CMP(cpuset_root, mask)) {
374 		for (n = 0; n < CPU_SETSIZE; n++) {
375 			if (!CPU_ISSET(n, mask))
376 				continue;
377 			if (cpu != NOCPU)
378 				return (EINVAL);
379 			cpu = n;
380 		}
381 	}
382 	ie = intr_lookup(irq);
383 	if (ie == NULL)
384 		return (ESRCH);
385 	return (intr_event_bind(ie, cpu));
386 }
387 
388 int
389 intr_getaffinity(int irq, void *m)
390 {
391 	struct intr_event *ie;
392 	cpuset_t *mask;
393 
394 	mask = m;
395 	ie = intr_lookup(irq);
396 	if (ie == NULL)
397 		return (ESRCH);
398 	CPU_ZERO(mask);
399 	mtx_lock(&ie->ie_lock);
400 	if (ie->ie_cpu == NOCPU)
401 		CPU_COPY(cpuset_root, mask);
402 	else
403 		CPU_SET(ie->ie_cpu, mask);
404 	mtx_unlock(&ie->ie_lock);
405 	return (0);
406 }
407 
408 int
409 intr_event_destroy(struct intr_event *ie)
410 {
411 
412 	mtx_lock(&event_lock);
413 	mtx_lock(&ie->ie_lock);
414 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
415 		mtx_unlock(&ie->ie_lock);
416 		mtx_unlock(&event_lock);
417 		return (EBUSY);
418 	}
419 	TAILQ_REMOVE(&event_list, ie, ie_list);
420 #ifndef notyet
421 	if (ie->ie_thread != NULL) {
422 		ithread_destroy(ie->ie_thread);
423 		ie->ie_thread = NULL;
424 	}
425 #endif
426 	mtx_unlock(&ie->ie_lock);
427 	mtx_unlock(&event_lock);
428 	mtx_destroy(&ie->ie_lock);
429 	free(ie, M_ITHREAD);
430 	return (0);
431 }
432 
433 #ifndef INTR_FILTER
434 static struct intr_thread *
435 ithread_create(const char *name)
436 {
437 	struct intr_thread *ithd;
438 	struct thread *td;
439 	int error;
440 
441 	ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
442 
443 	error = kproc_kthread_add(ithread_loop, ithd, &intrproc,
444 		    &td, RFSTOPPED | RFHIGHPID,
445 	    	    0, "intr", "%s", name);
446 	if (error)
447 		panic("kproc_create() failed with %d", error);
448 	thread_lock(td);
449 	sched_class(td, PRI_ITHD);
450 	TD_SET_IWAIT(td);
451 	thread_unlock(td);
452 	td->td_pflags |= TDP_ITHREAD;
453 	ithd->it_thread = td;
454 	CTR2(KTR_INTR, "%s: created %s", __func__, name);
455 	return (ithd);
456 }
457 #else
458 static struct intr_thread *
459 ithread_create(const char *name, struct intr_handler *ih)
460 {
461 	struct intr_thread *ithd;
462 	struct thread *td;
463 	int error;
464 
465 	ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO);
466 
467 	error = kproc_kthread_add(ithread_loop, ih, &intrproc,
468 		    &td, RFSTOPPED | RFHIGHPID,
469 	    	    0, "intr", "%s", name);
470 	if (error)
471 		panic("kproc_create() failed with %d", error);
472 	thread_lock(td);
473 	sched_class(td, PRI_ITHD);
474 	TD_SET_IWAIT(td);
475 	thread_unlock(td);
476 	td->td_pflags |= TDP_ITHREAD;
477 	ithd->it_thread = td;
478 	CTR2(KTR_INTR, "%s: created %s", __func__, name);
479 	return (ithd);
480 }
481 #endif
482 
483 static void
484 ithread_destroy(struct intr_thread *ithread)
485 {
486 	struct thread *td;
487 
488 	CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name);
489 	td = ithread->it_thread;
490 	thread_lock(td);
491 	ithread->it_flags |= IT_DEAD;
492 	if (TD_AWAITING_INTR(td)) {
493 		TD_CLR_IWAIT(td);
494 		sched_add(td, SRQ_INTR);
495 	}
496 	thread_unlock(td);
497 }
498 
499 #ifndef INTR_FILTER
500 int
501 intr_event_add_handler(struct intr_event *ie, const char *name,
502     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
503     enum intr_type flags, void **cookiep)
504 {
505 	struct intr_handler *ih, *temp_ih;
506 	struct intr_thread *it;
507 
508 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
509 		return (EINVAL);
510 
511 	/* Allocate and populate an interrupt handler structure. */
512 	ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
513 	ih->ih_filter = filter;
514 	ih->ih_handler = handler;
515 	ih->ih_argument = arg;
516 	strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
517 	ih->ih_event = ie;
518 	ih->ih_pri = pri;
519 	if (flags & INTR_EXCL)
520 		ih->ih_flags = IH_EXCLUSIVE;
521 	if (flags & INTR_MPSAFE)
522 		ih->ih_flags |= IH_MPSAFE;
523 	if (flags & INTR_ENTROPY)
524 		ih->ih_flags |= IH_ENTROPY;
525 
526 	/* We can only have one exclusive handler in a event. */
527 	mtx_lock(&ie->ie_lock);
528 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
529 		if ((flags & INTR_EXCL) ||
530 		    (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
531 			mtx_unlock(&ie->ie_lock);
532 			free(ih, M_ITHREAD);
533 			return (EINVAL);
534 		}
535 	}
536 
537 	/* Create a thread if we need one. */
538 	while (ie->ie_thread == NULL && handler != NULL) {
539 		if (ie->ie_flags & IE_ADDING_THREAD)
540 			msleep(ie, &ie->ie_lock, 0, "ithread", 0);
541 		else {
542 			ie->ie_flags |= IE_ADDING_THREAD;
543 			mtx_unlock(&ie->ie_lock);
544 			it = ithread_create("intr: newborn");
545 			mtx_lock(&ie->ie_lock);
546 			ie->ie_flags &= ~IE_ADDING_THREAD;
547 			ie->ie_thread = it;
548 			it->it_event = ie;
549 			ithread_update(it);
550 			wakeup(ie);
551 		}
552 	}
553 
554 	/* Add the new handler to the event in priority order. */
555 	TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
556 		if (temp_ih->ih_pri > ih->ih_pri)
557 			break;
558 	}
559 	if (temp_ih == NULL)
560 		TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
561 	else
562 		TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
563 	intr_event_update(ie);
564 
565 	CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
566 	    ie->ie_name);
567 	mtx_unlock(&ie->ie_lock);
568 
569 	if (cookiep != NULL)
570 		*cookiep = ih;
571 	return (0);
572 }
573 #else
574 int
575 intr_event_add_handler(struct intr_event *ie, const char *name,
576     driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri,
577     enum intr_type flags, void **cookiep)
578 {
579 	struct intr_handler *ih, *temp_ih;
580 	struct intr_thread *it;
581 
582 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
583 		return (EINVAL);
584 
585 	/* Allocate and populate an interrupt handler structure. */
586 	ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO);
587 	ih->ih_filter = filter;
588 	ih->ih_handler = handler;
589 	ih->ih_argument = arg;
590 	strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
591 	ih->ih_event = ie;
592 	ih->ih_pri = pri;
593 	if (flags & INTR_EXCL)
594 		ih->ih_flags = IH_EXCLUSIVE;
595 	if (flags & INTR_MPSAFE)
596 		ih->ih_flags |= IH_MPSAFE;
597 	if (flags & INTR_ENTROPY)
598 		ih->ih_flags |= IH_ENTROPY;
599 
600 	/* We can only have one exclusive handler in a event. */
601 	mtx_lock(&ie->ie_lock);
602 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
603 		if ((flags & INTR_EXCL) ||
604 		    (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
605 			mtx_unlock(&ie->ie_lock);
606 			free(ih, M_ITHREAD);
607 			return (EINVAL);
608 		}
609 	}
610 
611 	/* For filtered handlers, create a private ithread to run on. */
612 	if (filter != NULL && handler != NULL) {
613 		mtx_unlock(&ie->ie_lock);
614 		it = ithread_create("intr: newborn", ih);
615 		mtx_lock(&ie->ie_lock);
616 		it->it_event = ie;
617 		ih->ih_thread = it;
618 		ithread_update(it); /* XXX - do we really need this?!?!? */
619 	} else { /* Create the global per-event thread if we need one. */
620 		while (ie->ie_thread == NULL && handler != NULL) {
621 			if (ie->ie_flags & IE_ADDING_THREAD)
622 				msleep(ie, &ie->ie_lock, 0, "ithread", 0);
623 			else {
624 				ie->ie_flags |= IE_ADDING_THREAD;
625 				mtx_unlock(&ie->ie_lock);
626 				it = ithread_create("intr: newborn", ih);
627 				mtx_lock(&ie->ie_lock);
628 				ie->ie_flags &= ~IE_ADDING_THREAD;
629 				ie->ie_thread = it;
630 				it->it_event = ie;
631 				ithread_update(it);
632 				wakeup(ie);
633 			}
634 		}
635 	}
636 
637 	/* Add the new handler to the event in priority order. */
638 	TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
639 		if (temp_ih->ih_pri > ih->ih_pri)
640 			break;
641 	}
642 	if (temp_ih == NULL)
643 		TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
644 	else
645 		TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next);
646 	intr_event_update(ie);
647 
648 	CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
649 	    ie->ie_name);
650 	mtx_unlock(&ie->ie_lock);
651 
652 	if (cookiep != NULL)
653 		*cookiep = ih;
654 	return (0);
655 }
656 #endif
657 
658 /*
659  * Append a description preceded by a ':' to the name of the specified
660  * interrupt handler.
661  */
662 int
663 intr_event_describe_handler(struct intr_event *ie, void *cookie,
664     const char *descr)
665 {
666 	struct intr_handler *ih;
667 	size_t space;
668 	char *start;
669 
670 	mtx_lock(&ie->ie_lock);
671 #ifdef INVARIANTS
672 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
673 		if (ih == cookie)
674 			break;
675 	}
676 	if (ih == NULL) {
677 		mtx_unlock(&ie->ie_lock);
678 		panic("handler %p not found in interrupt event %p", cookie, ie);
679 	}
680 #endif
681 	ih = cookie;
682 
683 	/*
684 	 * Look for an existing description by checking for an
685 	 * existing ":".  This assumes device names do not include
686 	 * colons.  If one is found, prepare to insert the new
687 	 * description at that point.  If one is not found, find the
688 	 * end of the name to use as the insertion point.
689 	 */
690 	start = strchr(ih->ih_name, ':');
691 	if (start == NULL)
692 		start = strchr(ih->ih_name, 0);
693 
694 	/*
695 	 * See if there is enough remaining room in the string for the
696 	 * description + ":".  The "- 1" leaves room for the trailing
697 	 * '\0'.  The "+ 1" accounts for the colon.
698 	 */
699 	space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1;
700 	if (strlen(descr) + 1 > space) {
701 		mtx_unlock(&ie->ie_lock);
702 		return (ENOSPC);
703 	}
704 
705 	/* Append a colon followed by the description. */
706 	*start = ':';
707 	strcpy(start + 1, descr);
708 	intr_event_update(ie);
709 	mtx_unlock(&ie->ie_lock);
710 	return (0);
711 }
712 
713 /*
714  * Return the ie_source field from the intr_event an intr_handler is
715  * associated with.
716  */
717 void *
718 intr_handler_source(void *cookie)
719 {
720 	struct intr_handler *ih;
721 	struct intr_event *ie;
722 
723 	ih = (struct intr_handler *)cookie;
724 	if (ih == NULL)
725 		return (NULL);
726 	ie = ih->ih_event;
727 	KASSERT(ie != NULL,
728 	    ("interrupt handler \"%s\" has a NULL interrupt event",
729 	    ih->ih_name));
730 	return (ie->ie_source);
731 }
732 
733 /*
734  * Sleep until an ithread finishes executing an interrupt handler.
735  *
736  * XXX Doesn't currently handle interrupt filters or fast interrupt
737  * handlers.  This is intended for compatibility with linux drivers
738  * only.  Do not use in BSD code.
739  */
740 void
741 _intr_drain(int irq)
742 {
743 	struct intr_event *ie;
744 	struct intr_thread *ithd;
745 	struct thread *td;
746 
747 	ie = intr_lookup(irq);
748 	if (ie == NULL)
749 		return;
750 	if (ie->ie_thread == NULL)
751 		return;
752 	ithd = ie->ie_thread;
753 	td = ithd->it_thread;
754 	/*
755 	 * We set the flag and wait for it to be cleared to avoid
756 	 * long delays with potentially busy interrupt handlers
757 	 * were we to only sample TD_AWAITING_INTR() every tick.
758 	 */
759 	thread_lock(td);
760 	if (!TD_AWAITING_INTR(td)) {
761 		ithd->it_flags |= IT_WAIT;
762 		while (ithd->it_flags & IT_WAIT) {
763 			thread_unlock(td);
764 			pause("idrain", 1);
765 			thread_lock(td);
766 		}
767 	}
768 	thread_unlock(td);
769 	return;
770 }
771 
772 
773 #ifndef INTR_FILTER
774 int
775 intr_event_remove_handler(void *cookie)
776 {
777 	struct intr_handler *handler = (struct intr_handler *)cookie;
778 	struct intr_event *ie;
779 #ifdef INVARIANTS
780 	struct intr_handler *ih;
781 #endif
782 #ifdef notyet
783 	int dead;
784 #endif
785 
786 	if (handler == NULL)
787 		return (EINVAL);
788 	ie = handler->ih_event;
789 	KASSERT(ie != NULL,
790 	    ("interrupt handler \"%s\" has a NULL interrupt event",
791 	    handler->ih_name));
792 	mtx_lock(&ie->ie_lock);
793 	CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
794 	    ie->ie_name);
795 #ifdef INVARIANTS
796 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
797 		if (ih == handler)
798 			goto ok;
799 	mtx_unlock(&ie->ie_lock);
800 	panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
801 	    ih->ih_name, ie->ie_name);
802 ok:
803 #endif
804 	/*
805 	 * If there is no ithread, then just remove the handler and return.
806 	 * XXX: Note that an INTR_FAST handler might be running on another
807 	 * CPU!
808 	 */
809 	if (ie->ie_thread == NULL) {
810 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
811 		mtx_unlock(&ie->ie_lock);
812 		free(handler, M_ITHREAD);
813 		return (0);
814 	}
815 
816 	/*
817 	 * If the interrupt thread is already running, then just mark this
818 	 * handler as being dead and let the ithread do the actual removal.
819 	 *
820 	 * During a cold boot while cold is set, msleep() does not sleep,
821 	 * so we have to remove the handler here rather than letting the
822 	 * thread do it.
823 	 */
824 	thread_lock(ie->ie_thread->it_thread);
825 	if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) {
826 		handler->ih_flags |= IH_DEAD;
827 
828 		/*
829 		 * Ensure that the thread will process the handler list
830 		 * again and remove this handler if it has already passed
831 		 * it on the list.
832 		 */
833 		atomic_store_rel_int(&ie->ie_thread->it_need, 1);
834 	} else
835 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
836 	thread_unlock(ie->ie_thread->it_thread);
837 	while (handler->ih_flags & IH_DEAD)
838 		msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
839 	intr_event_update(ie);
840 #ifdef notyet
841 	/*
842 	 * XXX: This could be bad in the case of ppbus(8).  Also, I think
843 	 * this could lead to races of stale data when servicing an
844 	 * interrupt.
845 	 */
846 	dead = 1;
847 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
848 		if (!(ih->ih_flags & IH_FAST)) {
849 			dead = 0;
850 			break;
851 		}
852 	}
853 	if (dead) {
854 		ithread_destroy(ie->ie_thread);
855 		ie->ie_thread = NULL;
856 	}
857 #endif
858 	mtx_unlock(&ie->ie_lock);
859 	free(handler, M_ITHREAD);
860 	return (0);
861 }
862 
863 static int
864 intr_event_schedule_thread(struct intr_event *ie)
865 {
866 	struct intr_entropy entropy;
867 	struct intr_thread *it;
868 	struct thread *td;
869 	struct thread *ctd;
870 	struct proc *p;
871 
872 	/*
873 	 * If no ithread or no handlers, then we have a stray interrupt.
874 	 */
875 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) ||
876 	    ie->ie_thread == NULL)
877 		return (EINVAL);
878 
879 	ctd = curthread;
880 	it = ie->ie_thread;
881 	td = it->it_thread;
882 	p = td->td_proc;
883 
884 	/*
885 	 * If any of the handlers for this ithread claim to be good
886 	 * sources of entropy, then gather some.
887 	 */
888 	if (ie->ie_flags & IE_ENTROPY) {
889 		entropy.event = (uintptr_t)ie;
890 		entropy.td = ctd;
891 		random_harvest(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT);
892 	}
893 
894 	KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
895 
896 	/*
897 	 * Set it_need to tell the thread to keep running if it is already
898 	 * running.  Then, lock the thread and see if we actually need to
899 	 * put it on the runqueue.
900 	 */
901 	atomic_store_rel_int(&it->it_need, 1);
902 	thread_lock(td);
903 	if (TD_AWAITING_INTR(td)) {
904 		CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
905 		    td->td_name);
906 		TD_CLR_IWAIT(td);
907 		sched_add(td, SRQ_INTR);
908 	} else {
909 		CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
910 		    __func__, p->p_pid, td->td_name, it->it_need, td->td_state);
911 	}
912 	thread_unlock(td);
913 
914 	return (0);
915 }
916 #else
917 int
918 intr_event_remove_handler(void *cookie)
919 {
920 	struct intr_handler *handler = (struct intr_handler *)cookie;
921 	struct intr_event *ie;
922 	struct intr_thread *it;
923 #ifdef INVARIANTS
924 	struct intr_handler *ih;
925 #endif
926 #ifdef notyet
927 	int dead;
928 #endif
929 
930 	if (handler == NULL)
931 		return (EINVAL);
932 	ie = handler->ih_event;
933 	KASSERT(ie != NULL,
934 	    ("interrupt handler \"%s\" has a NULL interrupt event",
935 	    handler->ih_name));
936 	mtx_lock(&ie->ie_lock);
937 	CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
938 	    ie->ie_name);
939 #ifdef INVARIANTS
940 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
941 		if (ih == handler)
942 			goto ok;
943 	mtx_unlock(&ie->ie_lock);
944 	panic("interrupt handler \"%s\" not found in interrupt event \"%s\"",
945 	    ih->ih_name, ie->ie_name);
946 ok:
947 #endif
948 	/*
949 	 * If there are no ithreads (per event and per handler), then
950 	 * just remove the handler and return.
951 	 * XXX: Note that an INTR_FAST handler might be running on another CPU!
952 	 */
953 	if (ie->ie_thread == NULL && handler->ih_thread == NULL) {
954 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
955 		mtx_unlock(&ie->ie_lock);
956 		free(handler, M_ITHREAD);
957 		return (0);
958 	}
959 
960 	/* Private or global ithread? */
961 	it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread;
962 	/*
963 	 * If the interrupt thread is already running, then just mark this
964 	 * handler as being dead and let the ithread do the actual removal.
965 	 *
966 	 * During a cold boot while cold is set, msleep() does not sleep,
967 	 * so we have to remove the handler here rather than letting the
968 	 * thread do it.
969 	 */
970 	thread_lock(it->it_thread);
971 	if (!TD_AWAITING_INTR(it->it_thread) && !cold) {
972 		handler->ih_flags |= IH_DEAD;
973 
974 		/*
975 		 * Ensure that the thread will process the handler list
976 		 * again and remove this handler if it has already passed
977 		 * it on the list.
978 		 */
979 		atomic_store_rel_int(&it->it_need, 1);
980 	} else
981 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
982 	thread_unlock(it->it_thread);
983 	while (handler->ih_flags & IH_DEAD)
984 		msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
985 	/*
986 	 * At this point, the handler has been disconnected from the event,
987 	 * so we can kill the private ithread if any.
988 	 */
989 	if (handler->ih_thread) {
990 		ithread_destroy(handler->ih_thread);
991 		handler->ih_thread = NULL;
992 	}
993 	intr_event_update(ie);
994 #ifdef notyet
995 	/*
996 	 * XXX: This could be bad in the case of ppbus(8).  Also, I think
997 	 * this could lead to races of stale data when servicing an
998 	 * interrupt.
999 	 */
1000 	dead = 1;
1001 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1002 		if (handler != NULL) {
1003 			dead = 0;
1004 			break;
1005 		}
1006 	}
1007 	if (dead) {
1008 		ithread_destroy(ie->ie_thread);
1009 		ie->ie_thread = NULL;
1010 	}
1011 #endif
1012 	mtx_unlock(&ie->ie_lock);
1013 	free(handler, M_ITHREAD);
1014 	return (0);
1015 }
1016 
1017 static int
1018 intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it)
1019 {
1020 	struct intr_entropy entropy;
1021 	struct thread *td;
1022 	struct thread *ctd;
1023 	struct proc *p;
1024 
1025 	/*
1026 	 * If no ithread or no handlers, then we have a stray interrupt.
1027 	 */
1028 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL)
1029 		return (EINVAL);
1030 
1031 	ctd = curthread;
1032 	td = it->it_thread;
1033 	p = td->td_proc;
1034 
1035 	/*
1036 	 * If any of the handlers for this ithread claim to be good
1037 	 * sources of entropy, then gather some.
1038 	 */
1039 	if (ie->ie_flags & IE_ENTROPY) {
1040 		entropy.event = (uintptr_t)ie;
1041 		entropy.td = ctd;
1042 		random_harvest(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT);
1043 	}
1044 
1045 	KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
1046 
1047 	/*
1048 	 * Set it_need to tell the thread to keep running if it is already
1049 	 * running.  Then, lock the thread and see if we actually need to
1050 	 * put it on the runqueue.
1051 	 */
1052 	atomic_store_rel_int(&it->it_need, 1);
1053 	thread_lock(td);
1054 	if (TD_AWAITING_INTR(td)) {
1055 		CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid,
1056 		    td->td_name);
1057 		TD_CLR_IWAIT(td);
1058 		sched_add(td, SRQ_INTR);
1059 	} else {
1060 		CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d",
1061 		    __func__, p->p_pid, td->td_name, it->it_need, td->td_state);
1062 	}
1063 	thread_unlock(td);
1064 
1065 	return (0);
1066 }
1067 #endif
1068 
1069 /*
1070  * Allow interrupt event binding for software interrupt handlers -- a no-op,
1071  * since interrupts are generated in software rather than being directed by
1072  * a PIC.
1073  */
1074 static int
1075 swi_assign_cpu(void *arg, int cpu)
1076 {
1077 
1078 	return (0);
1079 }
1080 
1081 /*
1082  * Add a software interrupt handler to a specified event.  If a given event
1083  * is not specified, then a new event is created.
1084  */
1085 int
1086 swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
1087 	    void *arg, int pri, enum intr_type flags, void **cookiep)
1088 {
1089 	struct intr_event *ie;
1090 	int error;
1091 
1092 	if (flags & INTR_ENTROPY)
1093 		return (EINVAL);
1094 
1095 	ie = (eventp != NULL) ? *eventp : NULL;
1096 
1097 	if (ie != NULL) {
1098 		if (!(ie->ie_flags & IE_SOFT))
1099 			return (EINVAL);
1100 	} else {
1101 		error = intr_event_create(&ie, NULL, IE_SOFT, 0,
1102 		    NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri);
1103 		if (error)
1104 			return (error);
1105 		if (eventp != NULL)
1106 			*eventp = ie;
1107 	}
1108 	error = intr_event_add_handler(ie, name, NULL, handler, arg,
1109 	    PI_SWI(pri), flags, cookiep);
1110 	return (error);
1111 }
1112 
1113 /*
1114  * Schedule a software interrupt thread.
1115  */
1116 void
1117 swi_sched(void *cookie, int flags)
1118 {
1119 	struct intr_handler *ih = (struct intr_handler *)cookie;
1120 	struct intr_event *ie = ih->ih_event;
1121 	struct intr_entropy entropy;
1122 	int error;
1123 
1124 	CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
1125 	    ih->ih_need);
1126 
1127 	entropy.event = (uintptr_t)ih;
1128 	entropy.td = curthread;
1129 	random_harvest(&entropy, sizeof(entropy), 1, RANDOM_SWI);
1130 
1131 	/*
1132 	 * Set ih_need for this handler so that if the ithread is already
1133 	 * running it will execute this handler on the next pass.  Otherwise,
1134 	 * it will execute it the next time it runs.
1135 	 */
1136 	atomic_store_rel_int(&ih->ih_need, 1);
1137 
1138 	if (!(flags & SWI_DELAY)) {
1139 		PCPU_INC(cnt.v_soft);
1140 #ifdef INTR_FILTER
1141 		error = intr_event_schedule_thread(ie, ie->ie_thread);
1142 #else
1143 		error = intr_event_schedule_thread(ie);
1144 #endif
1145 		KASSERT(error == 0, ("stray software interrupt"));
1146 	}
1147 }
1148 
1149 /*
1150  * Remove a software interrupt handler.  Currently this code does not
1151  * remove the associated interrupt event if it becomes empty.  Calling code
1152  * may do so manually via intr_event_destroy(), but that's not really
1153  * an optimal interface.
1154  */
1155 int
1156 swi_remove(void *cookie)
1157 {
1158 
1159 	return (intr_event_remove_handler(cookie));
1160 }
1161 
1162 #ifdef INTR_FILTER
1163 static void
1164 priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih)
1165 {
1166 	struct intr_event *ie;
1167 
1168 	ie = ih->ih_event;
1169 	/*
1170 	 * If this handler is marked for death, remove it from
1171 	 * the list of handlers and wake up the sleeper.
1172 	 */
1173 	if (ih->ih_flags & IH_DEAD) {
1174 		mtx_lock(&ie->ie_lock);
1175 		TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1176 		ih->ih_flags &= ~IH_DEAD;
1177 		wakeup(ih);
1178 		mtx_unlock(&ie->ie_lock);
1179 		return;
1180 	}
1181 
1182 	/* Execute this handler. */
1183 	CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
1184 	     __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument,
1185 	     ih->ih_name, ih->ih_flags);
1186 
1187 	if (!(ih->ih_flags & IH_MPSAFE))
1188 		mtx_lock(&Giant);
1189 	ih->ih_handler(ih->ih_argument);
1190 	if (!(ih->ih_flags & IH_MPSAFE))
1191 		mtx_unlock(&Giant);
1192 }
1193 #endif
1194 
1195 /*
1196  * This is a public function for use by drivers that mux interrupt
1197  * handlers for child devices from their interrupt handler.
1198  */
1199 void
1200 intr_event_execute_handlers(struct proc *p, struct intr_event *ie)
1201 {
1202 	struct intr_handler *ih, *ihn;
1203 
1204 	TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
1205 		/*
1206 		 * If this handler is marked for death, remove it from
1207 		 * the list of handlers and wake up the sleeper.
1208 		 */
1209 		if (ih->ih_flags & IH_DEAD) {
1210 			mtx_lock(&ie->ie_lock);
1211 			TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1212 			ih->ih_flags &= ~IH_DEAD;
1213 			wakeup(ih);
1214 			mtx_unlock(&ie->ie_lock);
1215 			continue;
1216 		}
1217 
1218 		/* Skip filter only handlers */
1219 		if (ih->ih_handler == NULL)
1220 			continue;
1221 
1222 		/*
1223 		 * For software interrupt threads, we only execute
1224 		 * handlers that have their need flag set.  Hardware
1225 		 * interrupt threads always invoke all of their handlers.
1226 		 */
1227 		if (ie->ie_flags & IE_SOFT) {
1228 			if (atomic_load_acq_int(&ih->ih_need) == 0)
1229 				continue;
1230 			else
1231 				atomic_store_rel_int(&ih->ih_need, 0);
1232 		}
1233 
1234 		/* Execute this handler. */
1235 		CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x",
1236 		    __func__, p->p_pid, (void *)ih->ih_handler,
1237 		    ih->ih_argument, ih->ih_name, ih->ih_flags);
1238 
1239 		if (!(ih->ih_flags & IH_MPSAFE))
1240 			mtx_lock(&Giant);
1241 		ih->ih_handler(ih->ih_argument);
1242 		if (!(ih->ih_flags & IH_MPSAFE))
1243 			mtx_unlock(&Giant);
1244 	}
1245 }
1246 
1247 static void
1248 ithread_execute_handlers(struct proc *p, struct intr_event *ie)
1249 {
1250 
1251 	/* Interrupt handlers should not sleep. */
1252 	if (!(ie->ie_flags & IE_SOFT))
1253 		THREAD_NO_SLEEPING();
1254 	intr_event_execute_handlers(p, ie);
1255 	if (!(ie->ie_flags & IE_SOFT))
1256 		THREAD_SLEEPING_OK();
1257 
1258 	/*
1259 	 * Interrupt storm handling:
1260 	 *
1261 	 * If this interrupt source is currently storming, then throttle
1262 	 * it to only fire the handler once  per clock tick.
1263 	 *
1264 	 * If this interrupt source is not currently storming, but the
1265 	 * number of back to back interrupts exceeds the storm threshold,
1266 	 * then enter storming mode.
1267 	 */
1268 	if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold &&
1269 	    !(ie->ie_flags & IE_SOFT)) {
1270 		/* Report the message only once every second. */
1271 		if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
1272 			printf(
1273 	"interrupt storm detected on \"%s\"; throttling interrupt source\n",
1274 			    ie->ie_name);
1275 		}
1276 		pause("istorm", 1);
1277 	} else
1278 		ie->ie_count++;
1279 
1280 	/*
1281 	 * Now that all the handlers have had a chance to run, reenable
1282 	 * the interrupt source.
1283 	 */
1284 	if (ie->ie_post_ithread != NULL)
1285 		ie->ie_post_ithread(ie->ie_source);
1286 }
1287 
1288 #ifndef INTR_FILTER
1289 /*
1290  * This is the main code for interrupt threads.
1291  */
1292 static void
1293 ithread_loop(void *arg)
1294 {
1295 	struct intr_thread *ithd;
1296 	struct intr_event *ie;
1297 	struct thread *td;
1298 	struct proc *p;
1299 	int wake;
1300 
1301 	td = curthread;
1302 	p = td->td_proc;
1303 	ithd = (struct intr_thread *)arg;
1304 	KASSERT(ithd->it_thread == td,
1305 	    ("%s: ithread and proc linkage out of sync", __func__));
1306 	ie = ithd->it_event;
1307 	ie->ie_count = 0;
1308 	wake = 0;
1309 
1310 	/*
1311 	 * As long as we have interrupts outstanding, go through the
1312 	 * list of handlers, giving each one a go at it.
1313 	 */
1314 	for (;;) {
1315 		/*
1316 		 * If we are an orphaned thread, then just die.
1317 		 */
1318 		if (ithd->it_flags & IT_DEAD) {
1319 			CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
1320 			    p->p_pid, td->td_name);
1321 			free(ithd, M_ITHREAD);
1322 			kthread_exit();
1323 		}
1324 
1325 		/*
1326 		 * Service interrupts.  If another interrupt arrives while
1327 		 * we are running, it will set it_need to note that we
1328 		 * should make another pass.
1329 		 */
1330 		while (atomic_load_acq_int(&ithd->it_need) != 0) {
1331 			/*
1332 			 * This might need a full read and write barrier
1333 			 * to make sure that this write posts before any
1334 			 * of the memory or device accesses in the
1335 			 * handlers.
1336 			 */
1337 			atomic_store_rel_int(&ithd->it_need, 0);
1338 			ithread_execute_handlers(p, ie);
1339 		}
1340 		WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
1341 		mtx_assert(&Giant, MA_NOTOWNED);
1342 
1343 		/*
1344 		 * Processed all our interrupts.  Now get the sched
1345 		 * lock.  This may take a while and it_need may get
1346 		 * set again, so we have to check it again.
1347 		 */
1348 		thread_lock(td);
1349 		if ((atomic_load_acq_int(&ithd->it_need) == 0) &&
1350 		    !(ithd->it_flags & (IT_DEAD | IT_WAIT))) {
1351 			TD_SET_IWAIT(td);
1352 			ie->ie_count = 0;
1353 			mi_switch(SW_VOL | SWT_IWAIT, NULL);
1354 		}
1355 		if (ithd->it_flags & IT_WAIT) {
1356 			wake = 1;
1357 			ithd->it_flags &= ~IT_WAIT;
1358 		}
1359 		thread_unlock(td);
1360 		if (wake) {
1361 			wakeup(ithd);
1362 			wake = 0;
1363 		}
1364 	}
1365 }
1366 
1367 /*
1368  * Main interrupt handling body.
1369  *
1370  * Input:
1371  * o ie:                        the event connected to this interrupt.
1372  * o frame:                     some archs (i.e. i386) pass a frame to some.
1373  *                              handlers as their main argument.
1374  * Return value:
1375  * o 0:                         everything ok.
1376  * o EINVAL:                    stray interrupt.
1377  */
1378 int
1379 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1380 {
1381 	struct intr_handler *ih;
1382 	struct trapframe *oldframe;
1383 	struct thread *td;
1384 	int error, ret, thread;
1385 
1386 	td = curthread;
1387 
1388 #ifdef KSTACK_USAGE_PROF
1389 	intr_prof_stack_use(td, frame);
1390 #endif
1391 
1392 	/* An interrupt with no event or handlers is a stray interrupt. */
1393 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1394 		return (EINVAL);
1395 
1396 	/*
1397 	 * Execute fast interrupt handlers directly.
1398 	 * To support clock handlers, if a handler registers
1399 	 * with a NULL argument, then we pass it a pointer to
1400 	 * a trapframe as its argument.
1401 	 */
1402 	td->td_intr_nesting_level++;
1403 	thread = 0;
1404 	ret = 0;
1405 	critical_enter();
1406 	oldframe = td->td_intr_frame;
1407 	td->td_intr_frame = frame;
1408 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1409 		if (ih->ih_filter == NULL) {
1410 			thread = 1;
1411 			continue;
1412 		}
1413 		CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
1414 		    ih->ih_filter, ih->ih_argument == NULL ? frame :
1415 		    ih->ih_argument, ih->ih_name);
1416 		if (ih->ih_argument == NULL)
1417 			ret = ih->ih_filter(frame);
1418 		else
1419 			ret = ih->ih_filter(ih->ih_argument);
1420 		KASSERT(ret == FILTER_STRAY ||
1421 		    ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
1422 		    (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
1423 		    ("%s: incorrect return value %#x from %s", __func__, ret,
1424 		    ih->ih_name));
1425 
1426 		/*
1427 		 * Wrapper handler special handling:
1428 		 *
1429 		 * in some particular cases (like pccard and pccbb),
1430 		 * the _real_ device handler is wrapped in a couple of
1431 		 * functions - a filter wrapper and an ithread wrapper.
1432 		 * In this case (and just in this case), the filter wrapper
1433 		 * could ask the system to schedule the ithread and mask
1434 		 * the interrupt source if the wrapped handler is composed
1435 		 * of just an ithread handler.
1436 		 *
1437 		 * TODO: write a generic wrapper to avoid people rolling
1438 		 * their own
1439 		 */
1440 		if (!thread) {
1441 			if (ret == FILTER_SCHEDULE_THREAD)
1442 				thread = 1;
1443 		}
1444 	}
1445 	td->td_intr_frame = oldframe;
1446 
1447 	if (thread) {
1448 		if (ie->ie_pre_ithread != NULL)
1449 			ie->ie_pre_ithread(ie->ie_source);
1450 	} else {
1451 		if (ie->ie_post_filter != NULL)
1452 			ie->ie_post_filter(ie->ie_source);
1453 	}
1454 
1455 	/* Schedule the ithread if needed. */
1456 	if (thread) {
1457 		error = intr_event_schedule_thread(ie);
1458 #ifndef XEN
1459 		KASSERT(error == 0, ("bad stray interrupt"));
1460 #else
1461 		if (error != 0)
1462 			log(LOG_WARNING, "bad stray interrupt");
1463 #endif
1464 	}
1465 	critical_exit();
1466 	td->td_intr_nesting_level--;
1467 	return (0);
1468 }
1469 #else
1470 /*
1471  * This is the main code for interrupt threads.
1472  */
1473 static void
1474 ithread_loop(void *arg)
1475 {
1476 	struct intr_thread *ithd;
1477 	struct intr_handler *ih;
1478 	struct intr_event *ie;
1479 	struct thread *td;
1480 	struct proc *p;
1481 	int priv;
1482 	int wake;
1483 
1484 	td = curthread;
1485 	p = td->td_proc;
1486 	ih = (struct intr_handler *)arg;
1487 	priv = (ih->ih_thread != NULL) ? 1 : 0;
1488 	ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread;
1489 	KASSERT(ithd->it_thread == td,
1490 	    ("%s: ithread and proc linkage out of sync", __func__));
1491 	ie = ithd->it_event;
1492 	ie->ie_count = 0;
1493 	wake = 0;
1494 
1495 	/*
1496 	 * As long as we have interrupts outstanding, go through the
1497 	 * list of handlers, giving each one a go at it.
1498 	 */
1499 	for (;;) {
1500 		/*
1501 		 * If we are an orphaned thread, then just die.
1502 		 */
1503 		if (ithd->it_flags & IT_DEAD) {
1504 			CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__,
1505 			    p->p_pid, td->td_name);
1506 			free(ithd, M_ITHREAD);
1507 			kthread_exit();
1508 		}
1509 
1510 		/*
1511 		 * Service interrupts.  If another interrupt arrives while
1512 		 * we are running, it will set it_need to note that we
1513 		 * should make another pass.
1514 		 */
1515 		while (atomic_load_acq_int(&ithd->it_need) != 0) {
1516 			/*
1517 			 * This might need a full read and write barrier
1518 			 * to make sure that this write posts before any
1519 			 * of the memory or device accesses in the
1520 			 * handlers.
1521 			 */
1522 			atomic_store_rel_int(&ithd->it_need, 0);
1523 			if (priv)
1524 				priv_ithread_execute_handler(p, ih);
1525 			else
1526 				ithread_execute_handlers(p, ie);
1527 		}
1528 		WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
1529 		mtx_assert(&Giant, MA_NOTOWNED);
1530 
1531 		/*
1532 		 * Processed all our interrupts.  Now get the sched
1533 		 * lock.  This may take a while and it_need may get
1534 		 * set again, so we have to check it again.
1535 		 */
1536 		thread_lock(td);
1537 		if ((atomic_load_acq_int(&ithd->it_need) == 0) &&
1538 		    !(ithd->it_flags & (IT_DEAD | IT_WAIT))) {
1539 			TD_SET_IWAIT(td);
1540 			ie->ie_count = 0;
1541 			mi_switch(SW_VOL | SWT_IWAIT, NULL);
1542 		}
1543 		if (ithd->it_flags & IT_WAIT) {
1544 			wake = 1;
1545 			ithd->it_flags &= ~IT_WAIT;
1546 		}
1547 		thread_unlock(td);
1548 		if (wake) {
1549 			wakeup(ithd);
1550 			wake = 0;
1551 		}
1552 	}
1553 }
1554 
1555 /*
1556  * Main loop for interrupt filter.
1557  *
1558  * Some architectures (i386, amd64 and arm) require the optional frame
1559  * parameter, and use it as the main argument for fast handler execution
1560  * when ih_argument == NULL.
1561  *
1562  * Return value:
1563  * o FILTER_STRAY:              No filter recognized the event, and no
1564  *                              filter-less handler is registered on this
1565  *                              line.
1566  * o FILTER_HANDLED:            A filter claimed the event and served it.
1567  * o FILTER_SCHEDULE_THREAD:    No filter claimed the event, but there's at
1568  *                              least one filter-less handler on this line.
1569  * o FILTER_HANDLED |
1570  *   FILTER_SCHEDULE_THREAD:    A filter claimed the event, and asked for
1571  *                              scheduling the per-handler ithread.
1572  *
1573  * In case an ithread has to be scheduled, in *ithd there will be a
1574  * pointer to a struct intr_thread containing the thread to be
1575  * scheduled.
1576  */
1577 
1578 static int
1579 intr_filter_loop(struct intr_event *ie, struct trapframe *frame,
1580 		 struct intr_thread **ithd)
1581 {
1582 	struct intr_handler *ih;
1583 	void *arg;
1584 	int ret, thread_only;
1585 
1586 	ret = 0;
1587 	thread_only = 0;
1588 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1589 		/*
1590 		 * Execute fast interrupt handlers directly.
1591 		 * To support clock handlers, if a handler registers
1592 		 * with a NULL argument, then we pass it a pointer to
1593 		 * a trapframe as its argument.
1594 		 */
1595 		arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument);
1596 
1597 		CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__,
1598 		     ih->ih_filter, ih->ih_handler, arg, ih->ih_name);
1599 
1600 		if (ih->ih_filter != NULL)
1601 			ret = ih->ih_filter(arg);
1602 		else {
1603 			thread_only = 1;
1604 			continue;
1605 		}
1606 		KASSERT(ret == FILTER_STRAY ||
1607 		    ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 &&
1608 		    (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0),
1609 		    ("%s: incorrect return value %#x from %s", __func__, ret,
1610 		    ih->ih_name));
1611 		if (ret & FILTER_STRAY)
1612 			continue;
1613 		else {
1614 			*ithd = ih->ih_thread;
1615 			return (ret);
1616 		}
1617 	}
1618 
1619 	/*
1620 	 * No filters handled the interrupt and we have at least
1621 	 * one handler without a filter.  In this case, we schedule
1622 	 * all of the filter-less handlers to run in the ithread.
1623 	 */
1624 	if (thread_only) {
1625 		*ithd = ie->ie_thread;
1626 		return (FILTER_SCHEDULE_THREAD);
1627 	}
1628 	return (FILTER_STRAY);
1629 }
1630 
1631 /*
1632  * Main interrupt handling body.
1633  *
1634  * Input:
1635  * o ie:                        the event connected to this interrupt.
1636  * o frame:                     some archs (i.e. i386) pass a frame to some.
1637  *                              handlers as their main argument.
1638  * Return value:
1639  * o 0:                         everything ok.
1640  * o EINVAL:                    stray interrupt.
1641  */
1642 int
1643 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1644 {
1645 	struct intr_thread *ithd;
1646 	struct trapframe *oldframe;
1647 	struct thread *td;
1648 	int thread;
1649 
1650 	ithd = NULL;
1651 	td = curthread;
1652 
1653 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1654 		return (EINVAL);
1655 
1656 	td->td_intr_nesting_level++;
1657 	thread = 0;
1658 	critical_enter();
1659 	oldframe = td->td_intr_frame;
1660 	td->td_intr_frame = frame;
1661 	thread = intr_filter_loop(ie, frame, &ithd);
1662 	if (thread & FILTER_HANDLED) {
1663 		if (ie->ie_post_filter != NULL)
1664 			ie->ie_post_filter(ie->ie_source);
1665 	} else {
1666 		if (ie->ie_pre_ithread != NULL)
1667 			ie->ie_pre_ithread(ie->ie_source);
1668 	}
1669 	td->td_intr_frame = oldframe;
1670 	critical_exit();
1671 
1672 	/* Interrupt storm logic */
1673 	if (thread & FILTER_STRAY) {
1674 		ie->ie_count++;
1675 		if (ie->ie_count < intr_storm_threshold)
1676 			printf("Interrupt stray detection not present\n");
1677 	}
1678 
1679 	/* Schedule an ithread if needed. */
1680 	if (thread & FILTER_SCHEDULE_THREAD) {
1681 		if (intr_event_schedule_thread(ie, ithd) != 0)
1682 			panic("%s: impossible stray interrupt", __func__);
1683 	}
1684 	td->td_intr_nesting_level--;
1685 	return (0);
1686 }
1687 #endif
1688 
1689 #ifdef DDB
1690 /*
1691  * Dump details about an interrupt handler
1692  */
1693 static void
1694 db_dump_intrhand(struct intr_handler *ih)
1695 {
1696 	int comma;
1697 
1698 	db_printf("\t%-10s ", ih->ih_name);
1699 	switch (ih->ih_pri) {
1700 	case PI_REALTIME:
1701 		db_printf("CLK ");
1702 		break;
1703 	case PI_AV:
1704 		db_printf("AV  ");
1705 		break;
1706 	case PI_TTY:
1707 		db_printf("TTY ");
1708 		break;
1709 	case PI_NET:
1710 		db_printf("NET ");
1711 		break;
1712 	case PI_DISK:
1713 		db_printf("DISK");
1714 		break;
1715 	case PI_DULL:
1716 		db_printf("DULL");
1717 		break;
1718 	default:
1719 		if (ih->ih_pri >= PI_SOFT)
1720 			db_printf("SWI ");
1721 		else
1722 			db_printf("%4u", ih->ih_pri);
1723 		break;
1724 	}
1725 	db_printf(" ");
1726 	if (ih->ih_filter != NULL) {
1727 		db_printf("[F]");
1728 		db_printsym((uintptr_t)ih->ih_filter, DB_STGY_PROC);
1729 	}
1730 	if (ih->ih_handler != NULL) {
1731 		if (ih->ih_filter != NULL)
1732 			db_printf(",");
1733 		db_printf("[H]");
1734 		db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC);
1735 	}
1736 	db_printf("(%p)", ih->ih_argument);
1737 	if (ih->ih_need ||
1738 	    (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD |
1739 	    IH_MPSAFE)) != 0) {
1740 		db_printf(" {");
1741 		comma = 0;
1742 		if (ih->ih_flags & IH_EXCLUSIVE) {
1743 			if (comma)
1744 				db_printf(", ");
1745 			db_printf("EXCL");
1746 			comma = 1;
1747 		}
1748 		if (ih->ih_flags & IH_ENTROPY) {
1749 			if (comma)
1750 				db_printf(", ");
1751 			db_printf("ENTROPY");
1752 			comma = 1;
1753 		}
1754 		if (ih->ih_flags & IH_DEAD) {
1755 			if (comma)
1756 				db_printf(", ");
1757 			db_printf("DEAD");
1758 			comma = 1;
1759 		}
1760 		if (ih->ih_flags & IH_MPSAFE) {
1761 			if (comma)
1762 				db_printf(", ");
1763 			db_printf("MPSAFE");
1764 			comma = 1;
1765 		}
1766 		if (ih->ih_need) {
1767 			if (comma)
1768 				db_printf(", ");
1769 			db_printf("NEED");
1770 		}
1771 		db_printf("}");
1772 	}
1773 	db_printf("\n");
1774 }
1775 
1776 /*
1777  * Dump details about a event.
1778  */
1779 void
1780 db_dump_intr_event(struct intr_event *ie, int handlers)
1781 {
1782 	struct intr_handler *ih;
1783 	struct intr_thread *it;
1784 	int comma;
1785 
1786 	db_printf("%s ", ie->ie_fullname);
1787 	it = ie->ie_thread;
1788 	if (it != NULL)
1789 		db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
1790 	else
1791 		db_printf("(no thread)");
1792 	if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
1793 	    (it != NULL && it->it_need)) {
1794 		db_printf(" {");
1795 		comma = 0;
1796 		if (ie->ie_flags & IE_SOFT) {
1797 			db_printf("SOFT");
1798 			comma = 1;
1799 		}
1800 		if (ie->ie_flags & IE_ENTROPY) {
1801 			if (comma)
1802 				db_printf(", ");
1803 			db_printf("ENTROPY");
1804 			comma = 1;
1805 		}
1806 		if (ie->ie_flags & IE_ADDING_THREAD) {
1807 			if (comma)
1808 				db_printf(", ");
1809 			db_printf("ADDING_THREAD");
1810 			comma = 1;
1811 		}
1812 		if (it != NULL && it->it_need) {
1813 			if (comma)
1814 				db_printf(", ");
1815 			db_printf("NEED");
1816 		}
1817 		db_printf("}");
1818 	}
1819 	db_printf("\n");
1820 
1821 	if (handlers)
1822 		TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
1823 		    db_dump_intrhand(ih);
1824 }
1825 
1826 /*
1827  * Dump data about interrupt handlers
1828  */
1829 DB_SHOW_COMMAND(intr, db_show_intr)
1830 {
1831 	struct intr_event *ie;
1832 	int all, verbose;
1833 
1834 	verbose = strchr(modif, 'v') != NULL;
1835 	all = strchr(modif, 'a') != NULL;
1836 	TAILQ_FOREACH(ie, &event_list, ie_list) {
1837 		if (!all && TAILQ_EMPTY(&ie->ie_handlers))
1838 			continue;
1839 		db_dump_intr_event(ie, verbose);
1840 		if (db_pager_quit)
1841 			break;
1842 	}
1843 }
1844 #endif /* DDB */
1845 
1846 /*
1847  * Start standard software interrupt threads
1848  */
1849 static void
1850 start_softintr(void *dummy)
1851 {
1852 
1853 	if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
1854 		panic("died while creating vm swi ithread");
1855 }
1856 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr,
1857     NULL);
1858 
1859 /*
1860  * Sysctls used by systat and others: hw.intrnames and hw.intrcnt.
1861  * The data for this machine dependent, and the declarations are in machine
1862  * dependent code.  The layout of intrnames and intrcnt however is machine
1863  * independent.
1864  *
1865  * We do not know the length of intrcnt and intrnames at compile time, so
1866  * calculate things at run time.
1867  */
1868 static int
1869 sysctl_intrnames(SYSCTL_HANDLER_ARGS)
1870 {
1871 	return (sysctl_handle_opaque(oidp, intrnames, sintrnames, req));
1872 }
1873 
1874 SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD,
1875     NULL, 0, sysctl_intrnames, "", "Interrupt Names");
1876 
1877 static int
1878 sysctl_intrcnt(SYSCTL_HANDLER_ARGS)
1879 {
1880 #ifdef SCTL_MASK32
1881 	uint32_t *intrcnt32;
1882 	unsigned i;
1883 	int error;
1884 
1885 	if (req->flags & SCTL_MASK32) {
1886 		if (!req->oldptr)
1887 			return (sysctl_handle_opaque(oidp, NULL, sintrcnt / 2, req));
1888 		intrcnt32 = malloc(sintrcnt / 2, M_TEMP, M_NOWAIT);
1889 		if (intrcnt32 == NULL)
1890 			return (ENOMEM);
1891 		for (i = 0; i < sintrcnt / sizeof (u_long); i++)
1892 			intrcnt32[i] = intrcnt[i];
1893 		error = sysctl_handle_opaque(oidp, intrcnt32, sintrcnt / 2, req);
1894 		free(intrcnt32, M_TEMP);
1895 		return (error);
1896 	}
1897 #endif
1898 	return (sysctl_handle_opaque(oidp, intrcnt, sintrcnt, req));
1899 }
1900 
1901 SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD,
1902     NULL, 0, sysctl_intrcnt, "", "Interrupt Counts");
1903 
1904 #ifdef DDB
1905 /*
1906  * DDB command to dump the interrupt statistics.
1907  */
1908 DB_SHOW_COMMAND(intrcnt, db_show_intrcnt)
1909 {
1910 	u_long *i;
1911 	char *cp;
1912 	u_int j;
1913 
1914 	cp = intrnames;
1915 	j = 0;
1916 	for (i = intrcnt; j < (sintrcnt / sizeof(u_long)) && !db_pager_quit;
1917 	    i++, j++) {
1918 		if (*cp == '\0')
1919 			break;
1920 		if (*i != 0)
1921 			db_printf("%s\t%lu\n", cp, *i);
1922 		cp += strlen(cp) + 1;
1923 	}
1924 }
1925 #endif
1926