113 static void	intr_event_update(struct intr_event *ie);
114 static int	intr_event_schedule_thread(struct intr_event *ie, struct trapframe *frame);
118 		    struct intr_event *ie);
185 	struct intr_event *ie;  in ithread_update()  local
189 	ie = ithd->it_event;  in ithread_update()
191 	mtx_assert(&ie->ie_lock, MA_OWNED);  in ithread_update()
194 	if (CK_SLIST_EMPTY(&ie->ie_handlers))  in ithread_update()
197 		pri = CK_SLIST_FIRST(&ie->ie_handlers)->ih_pri;  in ithread_update()
200 	strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name));  in ithread_update()
213 intr_event_update(struct intr_event *ie)  in intr_event_update()  argument
220 	mtx_assert(&ie->ie_lock, MA_OWNED);  in intr_event_update()
221 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));  in intr_event_update()
227 	CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {  in intr_event_update()
228 		if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <  in intr_event_update()
229 		    sizeof(ie->ie_fullname)) {  in intr_event_update()
230 			strcat(ie->ie_fullname, " ");  in intr_event_update()
231 			strcat(ie->ie_fullname, ih->ih_name);  in intr_event_update()
237 	ie->ie_hflags = flags;  in intr_event_update()
247 		ih = CK_SLIST_FIRST(&ie->ie_handlers);  in intr_event_update()
248 		missed = strlen(ie->ie_fullname) + strlen(ih->ih_name) + 2 -  in intr_event_update()
249 		    sizeof(ie->ie_fullname);  in intr_event_update()
250 		strcat(ie->ie_fullname, (missed == 0) ? " " : "-");  in intr_event_update()
251 		strcat(ie->ie_fullname, &ih->ih_name[missed]);  in intr_event_update()
254 	last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];  in intr_event_update()
256 		if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {  in intr_event_update()
263 			strcat(ie->ie_fullname, " +");  in intr_event_update()
266 			strcat(ie->ie_fullname, "+");  in intr_event_update()
273 	if (ie->ie_thread != NULL)  in intr_event_update()
274 		ithread_update(ie->ie_thread);  in intr_event_update()
275 	CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);  in intr_event_update()
284 	struct intr_event *ie;  in intr_event_create()  local
290 	ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);  in intr_event_create()
291 	ie->ie_source = source;  in intr_event_create()
292 	ie->ie_pre_ithread = pre_ithread;  in intr_event_create()
293 	ie->ie_post_ithread = post_ithread;  in intr_event_create()
294 	ie->ie_post_filter = post_filter;  in intr_event_create()
295 	ie->ie_assign_cpu = assign_cpu;  in intr_event_create()
296 	ie->ie_flags = flags;  in intr_event_create()
297 	ie->ie_irq = irq;  in intr_event_create()
298 	ie->ie_cpu = NOCPU;  in intr_event_create()
299 	CK_SLIST_INIT(&ie->ie_handlers);  in intr_event_create()
300 	mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);  in intr_event_create()
303 	vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);  in intr_event_create()
305 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));  in intr_event_create()
307 	TAILQ_INSERT_TAIL(&event_list, ie, ie_list);  in intr_event_create()
310 		*event = ie;  in intr_event_create()
311 	CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);  in intr_event_create()
322 _intr_event_bind(struct intr_event *ie, int cpu, bool bindirq, bool bindithread)  in _intr_event_bind()  argument
331 	if (ie->ie_assign_cpu == NULL)  in _intr_event_bind()
343 		mtx_lock(&ie->ie_lock);  in _intr_event_bind()
344 		if (ie->ie_thread != NULL) {  in _intr_event_bind()
345 			id = ie->ie_thread->it_thread->td_tid;  in _intr_event_bind()
346 			mtx_unlock(&ie->ie_lock);  in _intr_event_bind()
351 			mtx_unlock(&ie->ie_lock);  in _intr_event_bind()
354 		error = ie->ie_assign_cpu(ie->ie_source, cpu);  in _intr_event_bind()
357 			mtx_lock(&ie->ie_lock);  in _intr_event_bind()
358 			if (ie->ie_thread != NULL) {  in _intr_event_bind()
359 				cpu = ie->ie_cpu;  in _intr_event_bind()
360 				id = ie->ie_thread->it_thread->td_tid;  in _intr_event_bind()
361 				mtx_unlock(&ie->ie_lock);  in _intr_event_bind()
364 				mtx_unlock(&ie->ie_lock);  in _intr_event_bind()
370 		mtx_lock(&ie->ie_lock);  in _intr_event_bind()
371 		ie->ie_cpu = cpu;  in _intr_event_bind()
372 		mtx_unlock(&ie->ie_lock);  in _intr_event_bind()
384 intr_event_bind(struct intr_event *ie, int cpu)  in intr_event_bind()  argument
387 	return (_intr_event_bind(ie, cpu, true, true));  in intr_event_bind()
395 intr_event_bind_irqonly(struct intr_event *ie, int cpu)  in intr_event_bind_irqonly()  argument
398 	return (_intr_event_bind(ie, cpu, true, false));  in intr_event_bind_irqonly()
405 intr_event_bind_ithread(struct intr_event *ie, int cpu)  in intr_event_bind_ithread()  argument
408 	return (_intr_event_bind(ie, cpu, false, true));  in intr_event_bind_ithread()
415 intr_event_bind_ithread_cpuset(struct intr_event *ie, cpuset_t *cs)  in intr_event_bind_ithread_cpuset()  argument
419 	mtx_lock(&ie->ie_lock);  in intr_event_bind_ithread_cpuset()
420 	if (ie->ie_thread != NULL) {  in intr_event_bind_ithread_cpuset()
421 		id = ie->ie_thread->it_thread->td_tid;  in intr_event_bind_ithread_cpuset()
422 		mtx_unlock(&ie->ie_lock);  in intr_event_bind_ithread_cpuset()
425 		mtx_unlock(&ie->ie_lock);  in intr_event_bind_ithread_cpuset()
433 	struct intr_event *ie;  in intr_lookup()  local
436 	TAILQ_FOREACH(ie, &event_list, ie_list)  in intr_lookup()
437 		if (ie->ie_irq == irq &&  in intr_lookup()
438 		    (ie->ie_flags & IE_SOFT) == 0 &&  in intr_lookup()
439 		    CK_SLIST_FIRST(&ie->ie_handlers) != NULL)  in intr_lookup()
442 	return (ie);  in intr_lookup()
448 	struct intr_event *ie;  in intr_setaffinity()  local
467 	ie = intr_lookup(irq);  in intr_setaffinity()
468 	if (ie == NULL)  in intr_setaffinity()
472 		return (intr_event_bind(ie, cpu));  in intr_setaffinity()
474 		return (intr_event_bind_irqonly(ie, cpu));  in intr_setaffinity()
476 		return (intr_event_bind_ithread(ie, cpu));  in intr_setaffinity()
485 	struct intr_event *ie;  in intr_getaffinity()  local
493 	ie = intr_lookup(irq);  in intr_getaffinity()
494 	if (ie == NULL)  in intr_getaffinity()
502 		mtx_lock(&ie->ie_lock);  in intr_getaffinity()
503 		if (ie->ie_cpu == NOCPU)  in intr_getaffinity()
506 			CPU_SET(ie->ie_cpu, mask);  in intr_getaffinity()
507 		mtx_unlock(&ie->ie_lock);  in intr_getaffinity()
510 		mtx_lock(&ie->ie_lock);  in intr_getaffinity()
511 		if (ie->ie_thread == NULL) {  in intr_getaffinity()
512 			mtx_unlock(&ie->ie_lock);  in intr_getaffinity()
515 			id = ie->ie_thread->it_thread->td_tid;  in intr_getaffinity()
516 			mtx_unlock(&ie->ie_lock);  in intr_getaffinity()
530 intr_event_destroy(struct intr_event *ie)  in intr_event_destroy()  argument
533 	if (ie == NULL)  in intr_event_destroy()
537 	mtx_lock(&ie->ie_lock);  in intr_event_destroy()
538 	if (!CK_SLIST_EMPTY(&ie->ie_handlers)) {  in intr_event_destroy()
539 		mtx_unlock(&ie->ie_lock);  in intr_event_destroy()
543 	TAILQ_REMOVE(&event_list, ie, ie_list);  in intr_event_destroy()
545 	if (ie->ie_thread != NULL)  in intr_event_destroy()
546 		ithread_destroy(ie->ie_thread);  in intr_event_destroy()
547 	mtx_unlock(&ie->ie_lock);  in intr_event_destroy()
548 	mtx_destroy(&ie->ie_lock);  in intr_event_destroy()
549 	free(ie, M_ITHREAD);  in intr_event_destroy()
580 	struct intr_event *ie;  in ithread_destroy()  local
584 	ie = ithread->it_event;  in ithread_destroy()
586 	mtx_assert(&ie->ie_lock, MA_OWNED);  in ithread_destroy()
588 	CTR2(KTR_INTR, "%s: killing %s", __func__, ie->ie_name);  in ithread_destroy()
597 	while (ie->ie_thread != NULL)  in ithread_destroy()
598 		msleep(ithread, &ie->ie_lock, 0, "ithd_dth", 0);  in ithread_destroy()
602 intr_event_add_handler(struct intr_event *ie, const char *name,  in intr_event_add_handler()  argument
610 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))  in intr_event_add_handler()
625 	ih->ih_event = ie;  in intr_event_add_handler()
637 	mtx_lock(&ie->ie_lock);  in intr_event_add_handler()
638 	if (!CK_SLIST_EMPTY(&ie->ie_handlers)) {  in intr_event_add_handler()
640 		    (CK_SLIST_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {  in intr_event_add_handler()
641 			mtx_unlock(&ie->ie_lock);  in intr_event_add_handler()
647 		ie->ie_flags |= IE_SLEEPABLE;  in intr_event_add_handler()
650 	while (ie->ie_thread == NULL && handler != NULL) {  in intr_event_add_handler()
651 		if (ie->ie_flags & IE_ADDING_THREAD)  in intr_event_add_handler()
652 			msleep(ie, &ie->ie_lock, 0, "ithread", 0);  in intr_event_add_handler()
654 			ie->ie_flags |= IE_ADDING_THREAD;  in intr_event_add_handler()
655 			mtx_unlock(&ie->ie_lock);  in intr_event_add_handler()
657 			mtx_lock(&ie->ie_lock);  in intr_event_add_handler()
658 			ie->ie_flags &= ~IE_ADDING_THREAD;  in intr_event_add_handler()
659 			ie->ie_thread = it;  in intr_event_add_handler()
660 			it->it_event = ie;  in intr_event_add_handler()
662 			wakeup(ie);  in intr_event_add_handler()
667 	CK_SLIST_FOREACH_PREVPTR(temp_ih, prevptr, &ie->ie_handlers, ih_next) {  in intr_event_add_handler()
673 	intr_event_update(ie);  in intr_event_add_handler()
676 	    ie->ie_name);  in intr_event_add_handler()
677 	mtx_unlock(&ie->ie_lock);  in intr_event_add_handler()
689 intr_event_describe_handler(struct intr_event *ie, void *cookie,  in intr_event_describe_handler()  argument
696 	mtx_lock(&ie->ie_lock);  in intr_event_describe_handler()
698 	CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {  in intr_event_describe_handler()
703 		mtx_unlock(&ie->ie_lock);  in intr_event_describe_handler()
704 		panic("handler %p not found in interrupt event %p", cookie, ie);  in intr_event_describe_handler()
727 		mtx_unlock(&ie->ie_lock);  in intr_event_describe_handler()
734 	intr_event_update(ie);  in intr_event_describe_handler()
735 	mtx_unlock(&ie->ie_lock);  in intr_event_describe_handler()
747 	struct intr_event *ie;  in intr_handler_source()  local
752 	ie = ih->ih_event;  in intr_handler_source()
753 	KASSERT(ie != NULL,  in intr_handler_source()
756 	return (ie->ie_source);  in intr_handler_source()
764 intr_event_barrier(struct intr_event *ie)  in intr_event_barrier()  argument
768 	mtx_assert(&ie->ie_lock, MA_OWNED);  in intr_event_barrier()
769 	phase = ie->ie_phase;  in intr_event_barrier()
775 	KASSERT(ie->ie_active[!phase] == 0, ("idle phase has activity"));  in intr_event_barrier()
776 	atomic_store_rel_int(&ie->ie_phase, !phase);  in intr_event_barrier()
793 	while (ie->ie_active[phase] > 0)  in intr_event_barrier()
801 	struct intr_event *ie;  in intr_handler_barrier()  local
803 	ie = handler->ih_event;  in intr_handler_barrier()
804 	mtx_assert(&ie->ie_lock, MA_OWNED);  in intr_handler_barrier()
808 	if (ie->ie_thread == NULL) {  in intr_handler_barrier()
809 		intr_event_barrier(ie);  in intr_handler_barrier()
814 		intr_event_schedule_thread(ie, NULL);  in intr_handler_barrier()
817 		msleep(handler, &ie->ie_lock, 0, "ih_barr", 0);  in intr_handler_barrier()
830 	struct intr_event *ie;  in _intr_drain()  local
834 	ie = intr_lookup(irq);  in _intr_drain()
835 	if (ie == NULL)  in _intr_drain()
837 	if (ie->ie_thread == NULL)  in _intr_drain()
839 	ithd = ie->ie_thread;  in _intr_drain()
863 	struct intr_event *ie;  in intr_event_remove_handler()  local
869 	ie = handler->ih_event;  in intr_event_remove_handler()
870 	KASSERT(ie != NULL,  in intr_event_remove_handler()
874 	mtx_lock(&ie->ie_lock);  in intr_event_remove_handler()
876 	    ie->ie_name);  in intr_event_remove_handler()
877 	CK_SLIST_FOREACH_PREVPTR(ih, prevptr, &ie->ie_handlers, ih_next) {  in intr_event_remove_handler()
883 		    "interrupt event \"%s\"", handler->ih_name, ie->ie_name);  in intr_event_remove_handler()
886 	if (ie->ie_thread == NULL) {  in intr_event_remove_handler()
895 		intr_event_barrier(ie);  in intr_event_remove_handler()
905 		intr_event_schedule_thread(ie, NULL);  in intr_event_remove_handler()
907 			msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);  in intr_event_remove_handler()
909 	intr_event_update(ie);  in intr_event_remove_handler()
910 	mtx_unlock(&ie->ie_lock);  in intr_event_remove_handler()
919 	struct intr_event *ie;  in intr_event_suspend_handler()  local
923 	ie = handler->ih_event;  in intr_event_suspend_handler()
924 	KASSERT(ie != NULL,  in intr_event_suspend_handler()
927 	mtx_lock(&ie->ie_lock);  in intr_event_suspend_handler()
930 	mtx_unlock(&ie->ie_lock);  in intr_event_suspend_handler()
938 	struct intr_event *ie;  in intr_event_resume_handler()  local
942 	ie = handler->ih_event;  in intr_event_resume_handler()
943 	KASSERT(ie != NULL,  in intr_event_resume_handler()
951 	mtx_lock(&ie->ie_lock);  in intr_event_resume_handler()
954 	mtx_unlock(&ie->ie_lock);  in intr_event_resume_handler()
959 intr_event_schedule_thread(struct intr_event *ie, struct trapframe *frame)  in intr_event_schedule_thread()  argument
969 	if (ie == NULL || CK_SLIST_EMPTY(&ie->ie_handlers) ||  in intr_event_schedule_thread()
970 	    ie->ie_thread == NULL)  in intr_event_schedule_thread()
974 	it = ie->ie_thread;  in intr_event_schedule_thread()
981 	if (ie->ie_hflags & IH_ENTROPY) {  in intr_event_schedule_thread()
982 		entropy.event = (uintptr_t)ie;  in intr_event_schedule_thread()
987 	KASSERT(td->td_proc != NULL, ("ithread %s has no process", ie->ie_name));  in intr_event_schedule_thread()
1045 	struct intr_event *ie;  in swi_add()  local
1051 	ie = (eventp != NULL) ? *eventp : NULL;  in swi_add()
1053 	if (ie != NULL) {  in swi_add()
1054 		if (!(ie->ie_flags & IE_SOFT))  in swi_add()
1057 		error = intr_event_create(&ie, NULL, IE_SOFT, 0,  in swi_add()
1062 			*eventp = ie;  in swi_add()
1065 		error = intr_event_add_handler(ie, name, NULL, handler, arg,  in swi_add()
1078 	struct intr_event *ie = ih->ih_event;  in swi_sched()  local
1082 	CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,  in swi_sched()
1103 		KASSERT(ie == clk_intr_event,  in swi_sched()
1109 		error = intr_event_schedule_thread(ie, NULL);  in swi_sched()
1128 intr_event_execute_handlers(struct proc *p, struct intr_event *ie)  in intr_event_execute_handlers()  argument
1133 	CK_SLIST_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {  in intr_event_execute_handlers()
1139 			mtx_lock(&ie->ie_lock);  in intr_event_execute_handlers()
1141 				CK_SLIST_REMOVE_HEAD(&ie->ie_handlers, ih_next);  in intr_event_execute_handlers()
1146 			mtx_unlock(&ie->ie_lock);  in intr_event_execute_handlers()
1157 			mtx_lock(&ie->ie_lock);  in intr_event_execute_handlers()
1160 			mtx_unlock(&ie->ie_lock);  in intr_event_execute_handlers()
1180 		if ((ie->ie_flags & IE_SOFT) != 0 &&  in intr_event_execute_handlers()
1198 ithread_execute_handlers(struct proc *p, struct intr_event *ie)  in ithread_execute_handlers()  argument
1202 	if (!(ie->ie_flags & (IE_SOFT | IE_SLEEPABLE)))  in ithread_execute_handlers()
1204 	intr_event_execute_handlers(p, ie);  in ithread_execute_handlers()
1205 	if (!(ie->ie_flags & (IE_SOFT | IE_SLEEPABLE)))  in ithread_execute_handlers()
1219 	    ie->ie_count >= intr_storm_threshold &&  in ithread_execute_handlers()
1220 	    (ie->ie_flags & IE_SOFT) == 0)) {  in ithread_execute_handlers()
1222 		if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {  in ithread_execute_handlers()
1225 			    ie->ie_name);  in ithread_execute_handlers()
1229 		ie->ie_count++;  in ithread_execute_handlers()
1235 	if (ie->ie_post_ithread != NULL)  in ithread_execute_handlers()
1236 		ie->ie_post_ithread(ie->ie_source);  in ithread_execute_handlers()
1247 	struct intr_event *ie;  in ithread_loop()  local
1258 	ie = ithd->it_event;  in ithread_loop()
1259 	ie->ie_count = 0;  in ithread_loop()
1272 			mtx_lock(&ie->ie_lock);  in ithread_loop()
1273 			ie->ie_thread = NULL;  in ithread_loop()
1275 			mtx_unlock(&ie->ie_lock);  in ithread_loop()
1291 		    (atomic_load_int(&ie->ie_hflags) & IH_NET) != 0;  in ithread_loop()
1297 			ithread_execute_handlers(p, ie);  in ithread_loop()
1319 			ie->ie_count = 0;  in ithread_loop()
1334  * o ie:                        the event connected to this interrupt.
1345 intr_event_handle(struct intr_event *ie, struct trapframe *frame)  in intr_event_handle()  argument
1361 	if (ie == NULL || CK_SLIST_EMPTY(&ie->ie_handlers))  in intr_event_handle()
1375 	phase = ie->ie_phase;  in intr_event_handle()
1376 	atomic_add_int(&ie->ie_active[phase], 1);  in intr_event_handle()
1384 	CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {  in intr_event_handle()
1387 		if ((ie->ie_flags & IE_SOFT) != 0 && ih->ih_need == 0)  in intr_event_handle()
1433 	atomic_add_rel_int(&ie->ie_active[phase], -1);  in intr_event_handle()
1438 		if (ie->ie_pre_ithread != NULL)  in intr_event_handle()
1439 			ie->ie_pre_ithread(ie->ie_source);  in intr_event_handle()
1441 		if (ie->ie_post_filter != NULL)  in intr_event_handle()
1442 			ie->ie_post_filter(ie->ie_source);  in intr_event_handle()
1449 		error =  intr_event_schedule_thread(ie, frame);  in intr_event_handle()
1541 db_dump_intr_event(struct intr_event *ie, int handlers)  in db_dump_intr_event()  argument
1547 	db_printf("%s ", ie->ie_fullname);  in db_dump_intr_event()
1548 	it = ie->ie_thread;  in db_dump_intr_event()
1553 	if ((ie->ie_flags & (IE_SOFT | IE_ADDING_THREAD)) != 0 ||  in db_dump_intr_event()
1557 		if (ie->ie_flags & IE_SOFT) {  in db_dump_intr_event()
1561 		if (ie->ie_flags & IE_ADDING_THREAD) {  in db_dump_intr_event()
1577 		CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next)  in db_dump_intr_event()
1586 	struct intr_event *ie;  in DB_SHOW_COMMAND_FLAGS()  local
1591 	TAILQ_FOREACH(ie, &event_list, ie_list) {  in DB_SHOW_COMMAND_FLAGS()
1592 		if (!all && CK_SLIST_EMPTY(&ie->ie_handlers))  in DB_SHOW_COMMAND_FLAGS()
1594 		db_dump_intr_event(ie, verbose);  in DB_SHOW_COMMAND_FLAGS()