xref: /freebsd/sys/kern/subr_intr.c (revision dc4250904956c3e438186dbb78254e0c9bf1b734)
12b3ad188SAdrian Chadd /*-
2bff6be3eSSvatopluk Kraus  * Copyright (c) 2015-2016 Svatopluk Kraus
3bff6be3eSSvatopluk Kraus  * Copyright (c) 2015-2016 Michal Meloun
42b3ad188SAdrian Chadd  * All rights reserved.
52b3ad188SAdrian Chadd  *
62b3ad188SAdrian Chadd  * Redistribution and use in source and binary forms, with or without
72b3ad188SAdrian Chadd  * modification, are permitted provided that the following conditions
82b3ad188SAdrian Chadd  * are met:
92b3ad188SAdrian Chadd  * 1. Redistributions of source code must retain the above copyright
102b3ad188SAdrian Chadd  *    notice, this list of conditions and the following disclaimer.
112b3ad188SAdrian Chadd  * 2. Redistributions in binary form must reproduce the above copyright
122b3ad188SAdrian Chadd  *    notice, this list of conditions and the following disclaimer in the
132b3ad188SAdrian Chadd  *    documentation and/or other materials provided with the distribution.
142b3ad188SAdrian Chadd  *
152b3ad188SAdrian Chadd  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
162b3ad188SAdrian Chadd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
172b3ad188SAdrian Chadd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
182b3ad188SAdrian Chadd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
192b3ad188SAdrian Chadd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
202b3ad188SAdrian Chadd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
212b3ad188SAdrian Chadd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
222b3ad188SAdrian Chadd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
232b3ad188SAdrian Chadd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
242b3ad188SAdrian Chadd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
252b3ad188SAdrian Chadd  * SUCH DAMAGE.
262b3ad188SAdrian Chadd  */
272b3ad188SAdrian Chadd 
282b3ad188SAdrian Chadd #include <sys/cdefs.h>
292b3ad188SAdrian Chadd __FBSDID("$FreeBSD$");
302b3ad188SAdrian Chadd 
312b3ad188SAdrian Chadd /*
322b3ad188SAdrian Chadd  *	New-style Interrupt Framework
332b3ad188SAdrian Chadd  *
34895c8b1cSMichal Meloun  *  TODO: - add support for disconnected PICs.
35895c8b1cSMichal Meloun  *        - to support IPI (PPI) enabling on other CPUs if already started.
36895c8b1cSMichal Meloun  *        - to complete things for removable PICs.
372b3ad188SAdrian Chadd  */
382b3ad188SAdrian Chadd 
392b3ad188SAdrian Chadd #include "opt_ddb.h"
40df7a2251SAndrew Turner #include "opt_hwpmc_hooks.h"
412b3ad188SAdrian Chadd 
422b3ad188SAdrian Chadd #include <sys/param.h>
432b3ad188SAdrian Chadd #include <sys/systm.h>
442b3ad188SAdrian Chadd #include <sys/kernel.h>
45e2e050c8SConrad Meyer #include <sys/lock.h>
46e2e050c8SConrad Meyer #include <sys/mutex.h>
472b3ad188SAdrian Chadd #include <sys/syslog.h>
482b3ad188SAdrian Chadd #include <sys/malloc.h>
492b3ad188SAdrian Chadd #include <sys/proc.h>
502b3ad188SAdrian Chadd #include <sys/queue.h>
512b3ad188SAdrian Chadd #include <sys/bus.h>
522b3ad188SAdrian Chadd #include <sys/interrupt.h>
532b3ad188SAdrian Chadd #include <sys/conf.h>
542b3ad188SAdrian Chadd #include <sys/cpuset.h>
556b42a1f4SAndrew Turner #include <sys/rman.h>
562b3ad188SAdrian Chadd #include <sys/sched.h>
572b3ad188SAdrian Chadd #include <sys/smp.h>
589ed01c32SGleb Smirnoff #include <sys/vmmeter.h>
59df7a2251SAndrew Turner #ifdef HWPMC_HOOKS
60df7a2251SAndrew Turner #include <sys/pmckern.h>
61df7a2251SAndrew Turner #endif
62df7a2251SAndrew Turner 
632b3ad188SAdrian Chadd #include <machine/atomic.h>
642b3ad188SAdrian Chadd #include <machine/intr.h>
652b3ad188SAdrian Chadd #include <machine/cpu.h>
662b3ad188SAdrian Chadd #include <machine/smp.h>
672b3ad188SAdrian Chadd #include <machine/stdarg.h>
682b3ad188SAdrian Chadd 
692b3ad188SAdrian Chadd #ifdef DDB
702b3ad188SAdrian Chadd #include <ddb/ddb.h>
712b3ad188SAdrian Chadd #endif
722b3ad188SAdrian Chadd 
732b3ad188SAdrian Chadd #include "pic_if.h"
743fc155dcSAndrew Turner #include "msi_if.h"
752b3ad188SAdrian Chadd 
762b3ad188SAdrian Chadd #define	INTRNAME_LEN	(2*MAXCOMLEN + 1)
772b3ad188SAdrian Chadd 
782b3ad188SAdrian Chadd #ifdef DEBUG
792b3ad188SAdrian Chadd #define debugf(fmt, args...) do { printf("%s(): ", __func__);	\
802b3ad188SAdrian Chadd     printf(fmt,##args); } while (0)
812b3ad188SAdrian Chadd #else
822b3ad188SAdrian Chadd #define debugf(fmt, args...)
832b3ad188SAdrian Chadd #endif
842b3ad188SAdrian Chadd 
852b3ad188SAdrian Chadd MALLOC_DECLARE(M_INTRNG);
862b3ad188SAdrian Chadd MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling");
872b3ad188SAdrian Chadd 
882b3ad188SAdrian Chadd /* Main interrupt handler called from assembler -> 'hidden' for C code. */
892b3ad188SAdrian Chadd void intr_irq_handler(struct trapframe *tf);
902b3ad188SAdrian Chadd 
912b3ad188SAdrian Chadd /* Root interrupt controller stuff. */
925b70c08cSSvatopluk Kraus device_t intr_irq_root_dev;
932b3ad188SAdrian Chadd static intr_irq_filter_t *irq_root_filter;
942b3ad188SAdrian Chadd static void *irq_root_arg;
952b3ad188SAdrian Chadd static u_int irq_root_ipicount;
962b3ad188SAdrian Chadd 
97d1605cdaSAndrew Turner struct intr_pic_child {
98d1605cdaSAndrew Turner 	SLIST_ENTRY(intr_pic_child)	 pc_next;
99d1605cdaSAndrew Turner 	struct intr_pic			*pc_pic;
100d1605cdaSAndrew Turner 	intr_child_irq_filter_t		*pc_filter;
101d1605cdaSAndrew Turner 	void				*pc_filter_arg;
102d1605cdaSAndrew Turner 	uintptr_t			 pc_start;
103d1605cdaSAndrew Turner 	uintptr_t			 pc_length;
104d1605cdaSAndrew Turner };
105d1605cdaSAndrew Turner 
1062b3ad188SAdrian Chadd /* Interrupt controller definition. */
1072b3ad188SAdrian Chadd struct intr_pic {
1082b3ad188SAdrian Chadd 	SLIST_ENTRY(intr_pic)	pic_next;
1092b3ad188SAdrian Chadd 	intptr_t		pic_xref;	/* hardware identification */
1102b3ad188SAdrian Chadd 	device_t		pic_dev;
111c0d52370SAndrew Turner /* Only one of FLAG_PIC or FLAG_MSI may be set */
1123fc155dcSAndrew Turner #define	FLAG_PIC	(1 << 0)
1133fc155dcSAndrew Turner #define	FLAG_MSI	(1 << 1)
114c0d52370SAndrew Turner #define	FLAG_TYPE_MASK	(FLAG_PIC | FLAG_MSI)
1153fc155dcSAndrew Turner 	u_int			pic_flags;
116d1605cdaSAndrew Turner 	struct mtx		pic_child_lock;
117d1605cdaSAndrew Turner 	SLIST_HEAD(, intr_pic_child) pic_children;
1182b3ad188SAdrian Chadd };
1192b3ad188SAdrian Chadd 
1202b3ad188SAdrian Chadd static struct mtx pic_list_lock;
1212b3ad188SAdrian Chadd static SLIST_HEAD(, intr_pic) pic_list;
1222b3ad188SAdrian Chadd 
123c0d52370SAndrew Turner static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, int flags);
1242b3ad188SAdrian Chadd 
1252b3ad188SAdrian Chadd /* Interrupt source definition. */
1262b3ad188SAdrian Chadd static struct mtx isrc_table_lock;
1272b3ad188SAdrian Chadd static struct intr_irqsrc *irq_sources[NIRQ];
1282b3ad188SAdrian Chadd u_int irq_next_free;
1292b3ad188SAdrian Chadd 
1302b3ad188SAdrian Chadd #ifdef SMP
131*dc425090SMitchell Horne #ifdef EARLY_AP_STARTUP
132*dc425090SMitchell Horne static bool irq_assign_cpu = true;
133*dc425090SMitchell Horne #else
134*dc425090SMitchell Horne static bool irq_assign_cpu = false;
135*dc425090SMitchell Horne #endif
1362b3ad188SAdrian Chadd #endif
1372b3ad188SAdrian Chadd 
1382b3ad188SAdrian Chadd /*
1392b3ad188SAdrian Chadd  * - 2 counters for each I/O interrupt.
1402b3ad188SAdrian Chadd  * - MAXCPU counters for each IPI counters for SMP.
1412b3ad188SAdrian Chadd  */
1422b3ad188SAdrian Chadd #ifdef SMP
1432b3ad188SAdrian Chadd #define INTRCNT_COUNT   (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU)
1442b3ad188SAdrian Chadd #else
1452b3ad188SAdrian Chadd #define INTRCNT_COUNT   (NIRQ * 2)
1462b3ad188SAdrian Chadd #endif
1472b3ad188SAdrian Chadd 
1482b3ad188SAdrian Chadd /* Data for MI statistics reporting. */
1492b3ad188SAdrian Chadd u_long intrcnt[INTRCNT_COUNT];
1502b3ad188SAdrian Chadd char intrnames[INTRCNT_COUNT * INTRNAME_LEN];
1512b3ad188SAdrian Chadd size_t sintrcnt = sizeof(intrcnt);
1522b3ad188SAdrian Chadd size_t sintrnames = sizeof(intrnames);
1532b3ad188SAdrian Chadd static u_int intrcnt_index;
1542b3ad188SAdrian Chadd 
155895c8b1cSMichal Meloun static struct intr_irqsrc *intr_map_get_isrc(u_int res_id);
156895c8b1cSMichal Meloun static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc);
157609b0fe9SOleksandr Tymoshenko static struct intr_map_data * intr_map_get_map_data(u_int res_id);
158895c8b1cSMichal Meloun static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref,
159895c8b1cSMichal Meloun     struct intr_map_data **data);
160895c8b1cSMichal Meloun 
1612b3ad188SAdrian Chadd /*
1622b3ad188SAdrian Chadd  *  Interrupt framework initialization routine.
1632b3ad188SAdrian Chadd  */
1642b3ad188SAdrian Chadd static void
1652b3ad188SAdrian Chadd intr_irq_init(void *dummy __unused)
1662b3ad188SAdrian Chadd {
1672b3ad188SAdrian Chadd 
1682b3ad188SAdrian Chadd 	SLIST_INIT(&pic_list);
1692b3ad188SAdrian Chadd 	mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF);
1703fc155dcSAndrew Turner 
1712b3ad188SAdrian Chadd 	mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF);
1722b3ad188SAdrian Chadd }
1732b3ad188SAdrian Chadd SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL);
1742b3ad188SAdrian Chadd 
1752b3ad188SAdrian Chadd static void
1762b3ad188SAdrian Chadd intrcnt_setname(const char *name, int index)
1772b3ad188SAdrian Chadd {
1782b3ad188SAdrian Chadd 
1792b3ad188SAdrian Chadd 	snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s",
1802b3ad188SAdrian Chadd 	    INTRNAME_LEN - 1, name);
1812b3ad188SAdrian Chadd }
1822b3ad188SAdrian Chadd 
1832b3ad188SAdrian Chadd /*
1842b3ad188SAdrian Chadd  *  Update name for interrupt source with interrupt event.
1852b3ad188SAdrian Chadd  */
1862b3ad188SAdrian Chadd static void
1872b3ad188SAdrian Chadd intrcnt_updatename(struct intr_irqsrc *isrc)
1882b3ad188SAdrian Chadd {
1892b3ad188SAdrian Chadd 
1902b3ad188SAdrian Chadd 	/* QQQ: What about stray counter name? */
1912b3ad188SAdrian Chadd 	mtx_assert(&isrc_table_lock, MA_OWNED);
1922b3ad188SAdrian Chadd 	intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index);
1932b3ad188SAdrian Chadd }
1942b3ad188SAdrian Chadd 
1952b3ad188SAdrian Chadd /*
1962b3ad188SAdrian Chadd  *  Virtualization for interrupt source interrupt counter increment.
1972b3ad188SAdrian Chadd  */
1982b3ad188SAdrian Chadd static inline void
1992b3ad188SAdrian Chadd isrc_increment_count(struct intr_irqsrc *isrc)
2002b3ad188SAdrian Chadd {
2012b3ad188SAdrian Chadd 
202bff6be3eSSvatopluk Kraus 	if (isrc->isrc_flags & INTR_ISRCF_PPI)
203bff6be3eSSvatopluk Kraus 		atomic_add_long(&isrc->isrc_count[0], 1);
204bff6be3eSSvatopluk Kraus 	else
2052b3ad188SAdrian Chadd 		isrc->isrc_count[0]++;
2062b3ad188SAdrian Chadd }
2072b3ad188SAdrian Chadd 
2082b3ad188SAdrian Chadd /*
2092b3ad188SAdrian Chadd  *  Virtualization for interrupt source interrupt stray counter increment.
2102b3ad188SAdrian Chadd  */
2112b3ad188SAdrian Chadd static inline void
2122b3ad188SAdrian Chadd isrc_increment_straycount(struct intr_irqsrc *isrc)
2132b3ad188SAdrian Chadd {
2142b3ad188SAdrian Chadd 
2152b3ad188SAdrian Chadd 	isrc->isrc_count[1]++;
2162b3ad188SAdrian Chadd }
2172b3ad188SAdrian Chadd 
2182b3ad188SAdrian Chadd /*
2192b3ad188SAdrian Chadd  *  Virtualization for interrupt source interrupt name update.
2202b3ad188SAdrian Chadd  */
2212b3ad188SAdrian Chadd static void
2222b3ad188SAdrian Chadd isrc_update_name(struct intr_irqsrc *isrc, const char *name)
2232b3ad188SAdrian Chadd {
2242b3ad188SAdrian Chadd 	char str[INTRNAME_LEN];
2252b3ad188SAdrian Chadd 
2262b3ad188SAdrian Chadd 	mtx_assert(&isrc_table_lock, MA_OWNED);
2272b3ad188SAdrian Chadd 
2282b3ad188SAdrian Chadd 	if (name != NULL) {
2292b3ad188SAdrian Chadd 		snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name);
2302b3ad188SAdrian Chadd 		intrcnt_setname(str, isrc->isrc_index);
2312b3ad188SAdrian Chadd 		snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name,
2322b3ad188SAdrian Chadd 		    name);
2332b3ad188SAdrian Chadd 		intrcnt_setname(str, isrc->isrc_index + 1);
2342b3ad188SAdrian Chadd 	} else {
2352b3ad188SAdrian Chadd 		snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name);
2362b3ad188SAdrian Chadd 		intrcnt_setname(str, isrc->isrc_index);
2372b3ad188SAdrian Chadd 		snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name);
2382b3ad188SAdrian Chadd 		intrcnt_setname(str, isrc->isrc_index + 1);
2392b3ad188SAdrian Chadd 	}
2402b3ad188SAdrian Chadd }
2412b3ad188SAdrian Chadd 
2422b3ad188SAdrian Chadd /*
2432b3ad188SAdrian Chadd  *  Virtualization for interrupt source interrupt counters setup.
2442b3ad188SAdrian Chadd  */
2452b3ad188SAdrian Chadd static void
2462b3ad188SAdrian Chadd isrc_setup_counters(struct intr_irqsrc *isrc)
2472b3ad188SAdrian Chadd {
2482b3ad188SAdrian Chadd 	u_int index;
2492b3ad188SAdrian Chadd 
2502b3ad188SAdrian Chadd 	/*
2512b3ad188SAdrian Chadd 	 *  XXX - it does not work well with removable controllers and
2522b3ad188SAdrian Chadd 	 *        interrupt sources !!!
2532b3ad188SAdrian Chadd 	 */
2542b3ad188SAdrian Chadd 	index = atomic_fetchadd_int(&intrcnt_index, 2);
2552b3ad188SAdrian Chadd 	isrc->isrc_index = index;
2562b3ad188SAdrian Chadd 	isrc->isrc_count = &intrcnt[index];
2572b3ad188SAdrian Chadd 	isrc_update_name(isrc, NULL);
2582b3ad188SAdrian Chadd }
2592b3ad188SAdrian Chadd 
260bff6be3eSSvatopluk Kraus /*
261bff6be3eSSvatopluk Kraus  *  Virtualization for interrupt source interrupt counters release.
262bff6be3eSSvatopluk Kraus  */
263bff6be3eSSvatopluk Kraus static void
264bff6be3eSSvatopluk Kraus isrc_release_counters(struct intr_irqsrc *isrc)
265bff6be3eSSvatopluk Kraus {
266bff6be3eSSvatopluk Kraus 
267bff6be3eSSvatopluk Kraus 	panic("%s: not implemented", __func__);
268bff6be3eSSvatopluk Kraus }
269bff6be3eSSvatopluk Kraus 
2702b3ad188SAdrian Chadd #ifdef SMP
2712b3ad188SAdrian Chadd /*
2722b3ad188SAdrian Chadd  *  Virtualization for interrupt source IPI counters setup.
2732b3ad188SAdrian Chadd  */
2745b70c08cSSvatopluk Kraus u_long *
2755b70c08cSSvatopluk Kraus intr_ipi_setup_counters(const char *name)
2762b3ad188SAdrian Chadd {
2772b3ad188SAdrian Chadd 	u_int index, i;
2782b3ad188SAdrian Chadd 	char str[INTRNAME_LEN];
2792b3ad188SAdrian Chadd 
2802b3ad188SAdrian Chadd 	index = atomic_fetchadd_int(&intrcnt_index, MAXCPU);
2812b3ad188SAdrian Chadd 	for (i = 0; i < MAXCPU; i++) {
2822b3ad188SAdrian Chadd 		snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name);
2832b3ad188SAdrian Chadd 		intrcnt_setname(str, index + i);
2842b3ad188SAdrian Chadd 	}
2855b70c08cSSvatopluk Kraus 	return (&intrcnt[index]);
2862b3ad188SAdrian Chadd }
2872b3ad188SAdrian Chadd #endif
2882b3ad188SAdrian Chadd 
2892b3ad188SAdrian Chadd /*
2902b3ad188SAdrian Chadd  *  Main interrupt dispatch handler. It's called straight
2912b3ad188SAdrian Chadd  *  from the assembler, where CPU interrupt is served.
2922b3ad188SAdrian Chadd  */
2932b3ad188SAdrian Chadd void
2942b3ad188SAdrian Chadd intr_irq_handler(struct trapframe *tf)
2952b3ad188SAdrian Chadd {
2962b3ad188SAdrian Chadd 	struct trapframe * oldframe;
2972b3ad188SAdrian Chadd 	struct thread * td;
2982b3ad188SAdrian Chadd 
2992b3ad188SAdrian Chadd 	KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__));
3002b3ad188SAdrian Chadd 
30183c9dea1SGleb Smirnoff 	VM_CNT_INC(v_intr);
3022b3ad188SAdrian Chadd 	critical_enter();
3032b3ad188SAdrian Chadd 	td = curthread;
3042b3ad188SAdrian Chadd 	oldframe = td->td_intr_frame;
3052b3ad188SAdrian Chadd 	td->td_intr_frame = tf;
3062b3ad188SAdrian Chadd 	irq_root_filter(irq_root_arg);
3072b3ad188SAdrian Chadd 	td->td_intr_frame = oldframe;
3082b3ad188SAdrian Chadd 	critical_exit();
309df7a2251SAndrew Turner #ifdef HWPMC_HOOKS
310974692e3SAndrew Turner 	if (pmc_hook && TRAPF_USERMODE(tf) &&
311974692e3SAndrew Turner 	    (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN))
312df7a2251SAndrew Turner 		pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf);
313df7a2251SAndrew Turner #endif
3142b3ad188SAdrian Chadd }
3152b3ad188SAdrian Chadd 
316d1605cdaSAndrew Turner int
317d1605cdaSAndrew Turner intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq)
318d1605cdaSAndrew Turner {
319d1605cdaSAndrew Turner 	struct intr_pic_child *child;
320d1605cdaSAndrew Turner 	bool found;
321d1605cdaSAndrew Turner 
322d1605cdaSAndrew Turner 	found = false;
323d1605cdaSAndrew Turner 	mtx_lock_spin(&parent->pic_child_lock);
324d1605cdaSAndrew Turner 	SLIST_FOREACH(child, &parent->pic_children, pc_next) {
325d1605cdaSAndrew Turner 		if (child->pc_start <= irq &&
326d1605cdaSAndrew Turner 		    irq < (child->pc_start + child->pc_length)) {
327d1605cdaSAndrew Turner 			found = true;
328d1605cdaSAndrew Turner 			break;
329d1605cdaSAndrew Turner 		}
330d1605cdaSAndrew Turner 	}
331d1605cdaSAndrew Turner 	mtx_unlock_spin(&parent->pic_child_lock);
332d1605cdaSAndrew Turner 
333d1605cdaSAndrew Turner 	if (found)
334d1605cdaSAndrew Turner 		return (child->pc_filter(child->pc_filter_arg, irq));
335d1605cdaSAndrew Turner 
336d1605cdaSAndrew Turner 	return (FILTER_STRAY);
337d1605cdaSAndrew Turner }
338d1605cdaSAndrew Turner 
3392b3ad188SAdrian Chadd /*
3402b3ad188SAdrian Chadd  *  interrupt controller dispatch function for interrupts. It should
3412b3ad188SAdrian Chadd  *  be called straight from the interrupt controller, when associated interrupt
3422b3ad188SAdrian Chadd  *  source is learned.
3432b3ad188SAdrian Chadd  */
344bff6be3eSSvatopluk Kraus int
345bff6be3eSSvatopluk Kraus intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf)
3462b3ad188SAdrian Chadd {
3472b3ad188SAdrian Chadd 
3482b3ad188SAdrian Chadd 	KASSERT(isrc != NULL, ("%s: no source", __func__));
3492b3ad188SAdrian Chadd 
3502b3ad188SAdrian Chadd 	isrc_increment_count(isrc);
3512b3ad188SAdrian Chadd 
3522b3ad188SAdrian Chadd #ifdef INTR_SOLO
3532b3ad188SAdrian Chadd 	if (isrc->isrc_filter != NULL) {
3542b3ad188SAdrian Chadd 		int error;
3552b3ad188SAdrian Chadd 		error = isrc->isrc_filter(isrc->isrc_arg, tf);
3562b3ad188SAdrian Chadd 		PIC_POST_FILTER(isrc->isrc_dev, isrc);
3572b3ad188SAdrian Chadd 		if (error == FILTER_HANDLED)
358bff6be3eSSvatopluk Kraus 			return (0);
3592b3ad188SAdrian Chadd 	} else
3602b3ad188SAdrian Chadd #endif
3612b3ad188SAdrian Chadd 	if (isrc->isrc_event != NULL) {
3622b3ad188SAdrian Chadd 		if (intr_event_handle(isrc->isrc_event, tf) == 0)
363bff6be3eSSvatopluk Kraus 			return (0);
3642b3ad188SAdrian Chadd 	}
3652b3ad188SAdrian Chadd 
3662b3ad188SAdrian Chadd 	isrc_increment_straycount(isrc);
367bff6be3eSSvatopluk Kraus 	return (EINVAL);
3682b3ad188SAdrian Chadd }
3692b3ad188SAdrian Chadd 
3702b3ad188SAdrian Chadd /*
3712b3ad188SAdrian Chadd  *  Alloc unique interrupt number (resource handle) for interrupt source.
3722b3ad188SAdrian Chadd  *
3732b3ad188SAdrian Chadd  *  There could be various strategies how to allocate free interrupt number
3742b3ad188SAdrian Chadd  *  (resource handle) for new interrupt source.
3752b3ad188SAdrian Chadd  *
3762b3ad188SAdrian Chadd  *  1. Handles are always allocated forward, so handles are not recycled
3772b3ad188SAdrian Chadd  *     immediately. However, if only one free handle left which is reused
3782b3ad188SAdrian Chadd  *     constantly...
3792b3ad188SAdrian Chadd  */
380bff6be3eSSvatopluk Kraus static inline int
381bff6be3eSSvatopluk Kraus isrc_alloc_irq(struct intr_irqsrc *isrc)
3822b3ad188SAdrian Chadd {
3832b3ad188SAdrian Chadd 	u_int maxirqs, irq;
3842b3ad188SAdrian Chadd 
3852b3ad188SAdrian Chadd 	mtx_assert(&isrc_table_lock, MA_OWNED);
3862b3ad188SAdrian Chadd 
3872b3ad188SAdrian Chadd 	maxirqs = nitems(irq_sources);
3882b3ad188SAdrian Chadd 	if (irq_next_free >= maxirqs)
3892b3ad188SAdrian Chadd 		return (ENOSPC);
3902b3ad188SAdrian Chadd 
3912b3ad188SAdrian Chadd 	for (irq = irq_next_free; irq < maxirqs; irq++) {
3922b3ad188SAdrian Chadd 		if (irq_sources[irq] == NULL)
3932b3ad188SAdrian Chadd 			goto found;
3942b3ad188SAdrian Chadd 	}
3952b3ad188SAdrian Chadd 	for (irq = 0; irq < irq_next_free; irq++) {
3962b3ad188SAdrian Chadd 		if (irq_sources[irq] == NULL)
3972b3ad188SAdrian Chadd 			goto found;
3982b3ad188SAdrian Chadd 	}
3992b3ad188SAdrian Chadd 
4002b3ad188SAdrian Chadd 	irq_next_free = maxirqs;
4012b3ad188SAdrian Chadd 	return (ENOSPC);
4022b3ad188SAdrian Chadd 
4032b3ad188SAdrian Chadd found:
4042b3ad188SAdrian Chadd 	isrc->isrc_irq = irq;
4052b3ad188SAdrian Chadd 	irq_sources[irq] = isrc;
4062b3ad188SAdrian Chadd 
4072b3ad188SAdrian Chadd 	irq_next_free = irq + 1;
4082b3ad188SAdrian Chadd 	if (irq_next_free >= maxirqs)
4092b3ad188SAdrian Chadd 		irq_next_free = 0;
4102b3ad188SAdrian Chadd 	return (0);
4112b3ad188SAdrian Chadd }
412bff6be3eSSvatopluk Kraus 
4132b3ad188SAdrian Chadd /*
4142b3ad188SAdrian Chadd  *  Free unique interrupt number (resource handle) from interrupt source.
4152b3ad188SAdrian Chadd  */
416bff6be3eSSvatopluk Kraus static inline int
4172b3ad188SAdrian Chadd isrc_free_irq(struct intr_irqsrc *isrc)
4182b3ad188SAdrian Chadd {
4192b3ad188SAdrian Chadd 
420bff6be3eSSvatopluk Kraus 	mtx_assert(&isrc_table_lock, MA_OWNED);
4212b3ad188SAdrian Chadd 
422bff6be3eSSvatopluk Kraus 	if (isrc->isrc_irq >= nitems(irq_sources))
4232b3ad188SAdrian Chadd 		return (EINVAL);
424bff6be3eSSvatopluk Kraus 	if (irq_sources[isrc->isrc_irq] != isrc)
4252b3ad188SAdrian Chadd 		return (EINVAL);
4262b3ad188SAdrian Chadd 
4272b3ad188SAdrian Chadd 	irq_sources[isrc->isrc_irq] = NULL;
4288442087fSMichal Meloun 	isrc->isrc_irq = INTR_IRQ_INVALID;	/* just to be safe */
4292b3ad188SAdrian Chadd 	return (0);
4302b3ad188SAdrian Chadd }
431bff6be3eSSvatopluk Kraus 
4322b3ad188SAdrian Chadd /*
433bff6be3eSSvatopluk Kraus  *  Initialize interrupt source and register it into global interrupt table.
4342b3ad188SAdrian Chadd  */
435bff6be3eSSvatopluk Kraus int
436bff6be3eSSvatopluk Kraus intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags,
437bff6be3eSSvatopluk Kraus     const char *fmt, ...)
4382b3ad188SAdrian Chadd {
439bff6be3eSSvatopluk Kraus 	int error;
440bff6be3eSSvatopluk Kraus 	va_list ap;
4412b3ad188SAdrian Chadd 
442bff6be3eSSvatopluk Kraus 	bzero(isrc, sizeof(struct intr_irqsrc));
443bff6be3eSSvatopluk Kraus 	isrc->isrc_dev = dev;
4448442087fSMichal Meloun 	isrc->isrc_irq = INTR_IRQ_INVALID;	/* just to be safe */
445bff6be3eSSvatopluk Kraus 	isrc->isrc_flags = flags;
4462b3ad188SAdrian Chadd 
447bff6be3eSSvatopluk Kraus 	va_start(ap, fmt);
448bff6be3eSSvatopluk Kraus 	vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap);
449bff6be3eSSvatopluk Kraus 	va_end(ap);
450bff6be3eSSvatopluk Kraus 
451bff6be3eSSvatopluk Kraus 	mtx_lock(&isrc_table_lock);
452bff6be3eSSvatopluk Kraus 	error = isrc_alloc_irq(isrc);
453bff6be3eSSvatopluk Kraus 	if (error != 0) {
454bff6be3eSSvatopluk Kraus 		mtx_unlock(&isrc_table_lock);
455bff6be3eSSvatopluk Kraus 		return (error);
4562b3ad188SAdrian Chadd 	}
457bff6be3eSSvatopluk Kraus 	/*
458bff6be3eSSvatopluk Kraus 	 * Setup interrupt counters, but not for IPI sources. Those are setup
459bff6be3eSSvatopluk Kraus 	 * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust
460bff6be3eSSvatopluk Kraus 	 * our counter pool.
461bff6be3eSSvatopluk Kraus 	 */
462bff6be3eSSvatopluk Kraus 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)
463bff6be3eSSvatopluk Kraus 		isrc_setup_counters(isrc);
464bff6be3eSSvatopluk Kraus 	mtx_unlock(&isrc_table_lock);
465bff6be3eSSvatopluk Kraus 	return (0);
4662b3ad188SAdrian Chadd }
4672b3ad188SAdrian Chadd 
4682b3ad188SAdrian Chadd /*
469bff6be3eSSvatopluk Kraus  *  Deregister interrupt source from global interrupt table.
470bff6be3eSSvatopluk Kraus  */
471bff6be3eSSvatopluk Kraus int
472bff6be3eSSvatopluk Kraus intr_isrc_deregister(struct intr_irqsrc *isrc)
473bff6be3eSSvatopluk Kraus {
474bff6be3eSSvatopluk Kraus 	int error;
475bff6be3eSSvatopluk Kraus 
476bff6be3eSSvatopluk Kraus 	mtx_lock(&isrc_table_lock);
477bff6be3eSSvatopluk Kraus 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)
478bff6be3eSSvatopluk Kraus 		isrc_release_counters(isrc);
479bff6be3eSSvatopluk Kraus 	error = isrc_free_irq(isrc);
480bff6be3eSSvatopluk Kraus 	mtx_unlock(&isrc_table_lock);
481bff6be3eSSvatopluk Kraus 	return (error);
482bff6be3eSSvatopluk Kraus }
483bff6be3eSSvatopluk Kraus 
4845b613c19SSvatopluk Kraus #ifdef SMP
4855b613c19SSvatopluk Kraus /*
4865b613c19SSvatopluk Kraus  *  A support function for a PIC to decide if provided ISRC should be inited
4875b613c19SSvatopluk Kraus  *  on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of
4885b613c19SSvatopluk Kraus  *  struct intr_irqsrc is the following:
4895b613c19SSvatopluk Kraus  *
4905b613c19SSvatopluk Kraus  *     If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus
4915b613c19SSvatopluk Kraus  *     set in isrc_cpu. If not, the ISRC should be inited on every cpu and
4925b613c19SSvatopluk Kraus  *     isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct.
4935b613c19SSvatopluk Kraus  */
4945b613c19SSvatopluk Kraus bool
4955b613c19SSvatopluk Kraus intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu)
4965b613c19SSvatopluk Kraus {
4975b613c19SSvatopluk Kraus 
4985b613c19SSvatopluk Kraus 	if (isrc->isrc_handlers == 0)
4995b613c19SSvatopluk Kraus 		return (false);
5005b613c19SSvatopluk Kraus 	if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0)
5015b613c19SSvatopluk Kraus 		return (false);
5025b613c19SSvatopluk Kraus 	if (isrc->isrc_flags & INTR_ISRCF_BOUND)
5035b613c19SSvatopluk Kraus 		return (CPU_ISSET(cpu, &isrc->isrc_cpu));
5045b613c19SSvatopluk Kraus 
5055b613c19SSvatopluk Kraus 	CPU_SET(cpu, &isrc->isrc_cpu);
5065b613c19SSvatopluk Kraus 	return (true);
5075b613c19SSvatopluk Kraus }
5085b613c19SSvatopluk Kraus #endif
5095b613c19SSvatopluk Kraus 
5102b3ad188SAdrian Chadd #ifdef INTR_SOLO
5112b3ad188SAdrian Chadd /*
5122b3ad188SAdrian Chadd  *  Setup filter into interrupt source.
5132b3ad188SAdrian Chadd  */
5142b3ad188SAdrian Chadd static int
5152b3ad188SAdrian Chadd iscr_setup_filter(struct intr_irqsrc *isrc, const char *name,
5162b3ad188SAdrian Chadd     intr_irq_filter_t *filter, void *arg, void **cookiep)
5172b3ad188SAdrian Chadd {
5182b3ad188SAdrian Chadd 
5192b3ad188SAdrian Chadd 	if (filter == NULL)
5202b3ad188SAdrian Chadd 		return (EINVAL);
5212b3ad188SAdrian Chadd 
5222b3ad188SAdrian Chadd 	mtx_lock(&isrc_table_lock);
5232b3ad188SAdrian Chadd 	/*
5242b3ad188SAdrian Chadd 	 * Make sure that we do not mix the two ways
5252b3ad188SAdrian Chadd 	 * how we handle interrupt sources.
5262b3ad188SAdrian Chadd 	 */
5272b3ad188SAdrian Chadd 	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {
5282b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
5292b3ad188SAdrian Chadd 		return (EBUSY);
5302b3ad188SAdrian Chadd 	}
5312b3ad188SAdrian Chadd 	isrc->isrc_filter = filter;
5322b3ad188SAdrian Chadd 	isrc->isrc_arg = arg;
5332b3ad188SAdrian Chadd 	isrc_update_name(isrc, name);
5342b3ad188SAdrian Chadd 	mtx_unlock(&isrc_table_lock);
5352b3ad188SAdrian Chadd 
5362b3ad188SAdrian Chadd 	*cookiep = isrc;
5372b3ad188SAdrian Chadd 	return (0);
5382b3ad188SAdrian Chadd }
5392b3ad188SAdrian Chadd #endif
5402b3ad188SAdrian Chadd 
5412b3ad188SAdrian Chadd /*
5422b3ad188SAdrian Chadd  *  Interrupt source pre_ithread method for MI interrupt framework.
5432b3ad188SAdrian Chadd  */
5442b3ad188SAdrian Chadd static void
5452b3ad188SAdrian Chadd intr_isrc_pre_ithread(void *arg)
5462b3ad188SAdrian Chadd {
5472b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc = arg;
5482b3ad188SAdrian Chadd 
5492b3ad188SAdrian Chadd 	PIC_PRE_ITHREAD(isrc->isrc_dev, isrc);
5502b3ad188SAdrian Chadd }
5512b3ad188SAdrian Chadd 
5522b3ad188SAdrian Chadd /*
5532b3ad188SAdrian Chadd  *  Interrupt source post_ithread method for MI interrupt framework.
5542b3ad188SAdrian Chadd  */
5552b3ad188SAdrian Chadd static void
5562b3ad188SAdrian Chadd intr_isrc_post_ithread(void *arg)
5572b3ad188SAdrian Chadd {
5582b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc = arg;
5592b3ad188SAdrian Chadd 
5602b3ad188SAdrian Chadd 	PIC_POST_ITHREAD(isrc->isrc_dev, isrc);
5612b3ad188SAdrian Chadd }
5622b3ad188SAdrian Chadd 
5632b3ad188SAdrian Chadd /*
5642b3ad188SAdrian Chadd  *  Interrupt source post_filter method for MI interrupt framework.
5652b3ad188SAdrian Chadd  */
5662b3ad188SAdrian Chadd static void
5672b3ad188SAdrian Chadd intr_isrc_post_filter(void *arg)
5682b3ad188SAdrian Chadd {
5692b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc = arg;
5702b3ad188SAdrian Chadd 
5712b3ad188SAdrian Chadd 	PIC_POST_FILTER(isrc->isrc_dev, isrc);
5722b3ad188SAdrian Chadd }
5732b3ad188SAdrian Chadd 
5742b3ad188SAdrian Chadd /*
5752b3ad188SAdrian Chadd  *  Interrupt source assign_cpu method for MI interrupt framework.
5762b3ad188SAdrian Chadd  */
5772b3ad188SAdrian Chadd static int
5782b3ad188SAdrian Chadd intr_isrc_assign_cpu(void *arg, int cpu)
5792b3ad188SAdrian Chadd {
5802b3ad188SAdrian Chadd #ifdef SMP
5812b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc = arg;
5822b3ad188SAdrian Chadd 	int error;
5832b3ad188SAdrian Chadd 
5845b70c08cSSvatopluk Kraus 	if (isrc->isrc_dev != intr_irq_root_dev)
5852b3ad188SAdrian Chadd 		return (EINVAL);
5862b3ad188SAdrian Chadd 
5872b3ad188SAdrian Chadd 	mtx_lock(&isrc_table_lock);
5882b3ad188SAdrian Chadd 	if (cpu == NOCPU) {
5892b3ad188SAdrian Chadd 		CPU_ZERO(&isrc->isrc_cpu);
5902b3ad188SAdrian Chadd 		isrc->isrc_flags &= ~INTR_ISRCF_BOUND;
5912b3ad188SAdrian Chadd 	} else {
5922b3ad188SAdrian Chadd 		CPU_SETOF(cpu, &isrc->isrc_cpu);
5932b3ad188SAdrian Chadd 		isrc->isrc_flags |= INTR_ISRCF_BOUND;
5942b3ad188SAdrian Chadd 	}
5952b3ad188SAdrian Chadd 
5962b3ad188SAdrian Chadd 	/*
5972b3ad188SAdrian Chadd 	 * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or
5982b3ad188SAdrian Chadd 	 * re-balance it to another CPU or enable it on more CPUs. However,
5992b3ad188SAdrian Chadd 	 * PIC is expected to change isrc_cpu appropriately to keep us well
600e3043798SPedro F. Giffuni 	 * informed if the call is successful.
6012b3ad188SAdrian Chadd 	 */
6022b3ad188SAdrian Chadd 	if (irq_assign_cpu) {
603bff6be3eSSvatopluk Kraus 		error = PIC_BIND_INTR(isrc->isrc_dev, isrc);
6042b3ad188SAdrian Chadd 		if (error) {
6052b3ad188SAdrian Chadd 			CPU_ZERO(&isrc->isrc_cpu);
6062b3ad188SAdrian Chadd 			mtx_unlock(&isrc_table_lock);
6072b3ad188SAdrian Chadd 			return (error);
6082b3ad188SAdrian Chadd 		}
6092b3ad188SAdrian Chadd 	}
6102b3ad188SAdrian Chadd 	mtx_unlock(&isrc_table_lock);
6112b3ad188SAdrian Chadd 	return (0);
6122b3ad188SAdrian Chadd #else
6132b3ad188SAdrian Chadd 	return (EOPNOTSUPP);
6142b3ad188SAdrian Chadd #endif
6152b3ad188SAdrian Chadd }
6162b3ad188SAdrian Chadd 
6172b3ad188SAdrian Chadd /*
6182b3ad188SAdrian Chadd  *  Create interrupt event for interrupt source.
6192b3ad188SAdrian Chadd  */
6202b3ad188SAdrian Chadd static int
6212b3ad188SAdrian Chadd isrc_event_create(struct intr_irqsrc *isrc)
6222b3ad188SAdrian Chadd {
6232b3ad188SAdrian Chadd 	struct intr_event *ie;
6242b3ad188SAdrian Chadd 	int error;
6252b3ad188SAdrian Chadd 
6262b3ad188SAdrian Chadd 	error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq,
6272b3ad188SAdrian Chadd 	    intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter,
6282b3ad188SAdrian Chadd 	    intr_isrc_assign_cpu, "%s:", isrc->isrc_name);
6292b3ad188SAdrian Chadd 	if (error)
6302b3ad188SAdrian Chadd 		return (error);
6312b3ad188SAdrian Chadd 
6322b3ad188SAdrian Chadd 	mtx_lock(&isrc_table_lock);
6332b3ad188SAdrian Chadd 	/*
6342b3ad188SAdrian Chadd 	 * Make sure that we do not mix the two ways
6352b3ad188SAdrian Chadd 	 * how we handle interrupt sources. Let contested event wins.
6362b3ad188SAdrian Chadd 	 */
637169e6abdSSvatopluk Kraus #ifdef INTR_SOLO
6382b3ad188SAdrian Chadd 	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {
639169e6abdSSvatopluk Kraus #else
640169e6abdSSvatopluk Kraus 	if (isrc->isrc_event != NULL) {
641169e6abdSSvatopluk Kraus #endif
6422b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
6432b3ad188SAdrian Chadd 		intr_event_destroy(ie);
6442b3ad188SAdrian Chadd 		return (isrc->isrc_event != NULL ? EBUSY : 0);
6452b3ad188SAdrian Chadd 	}
6462b3ad188SAdrian Chadd 	isrc->isrc_event = ie;
6472b3ad188SAdrian Chadd 	mtx_unlock(&isrc_table_lock);
6482b3ad188SAdrian Chadd 
6492b3ad188SAdrian Chadd 	return (0);
6502b3ad188SAdrian Chadd }
6512b3ad188SAdrian Chadd #ifdef notyet
6522b3ad188SAdrian Chadd /*
6532b3ad188SAdrian Chadd  *  Destroy interrupt event for interrupt source.
6542b3ad188SAdrian Chadd  */
6552b3ad188SAdrian Chadd static void
6562b3ad188SAdrian Chadd isrc_event_destroy(struct intr_irqsrc *isrc)
6572b3ad188SAdrian Chadd {
6582b3ad188SAdrian Chadd 	struct intr_event *ie;
6592b3ad188SAdrian Chadd 
6602b3ad188SAdrian Chadd 	mtx_lock(&isrc_table_lock);
6612b3ad188SAdrian Chadd 	ie = isrc->isrc_event;
6622b3ad188SAdrian Chadd 	isrc->isrc_event = NULL;
6632b3ad188SAdrian Chadd 	mtx_unlock(&isrc_table_lock);
6642b3ad188SAdrian Chadd 
6652b3ad188SAdrian Chadd 	if (ie != NULL)
6662b3ad188SAdrian Chadd 		intr_event_destroy(ie);
6672b3ad188SAdrian Chadd }
6682b3ad188SAdrian Chadd #endif
6692b3ad188SAdrian Chadd /*
6702b3ad188SAdrian Chadd  *  Add handler to interrupt source.
6712b3ad188SAdrian Chadd  */
6722b3ad188SAdrian Chadd static int
6732b3ad188SAdrian Chadd isrc_add_handler(struct intr_irqsrc *isrc, const char *name,
6742b3ad188SAdrian Chadd     driver_filter_t filter, driver_intr_t handler, void *arg,
6752b3ad188SAdrian Chadd     enum intr_type flags, void **cookiep)
6762b3ad188SAdrian Chadd {
6772b3ad188SAdrian Chadd 	int error;
6782b3ad188SAdrian Chadd 
6792b3ad188SAdrian Chadd 	if (isrc->isrc_event == NULL) {
6802b3ad188SAdrian Chadd 		error = isrc_event_create(isrc);
6812b3ad188SAdrian Chadd 		if (error)
6822b3ad188SAdrian Chadd 			return (error);
6832b3ad188SAdrian Chadd 	}
6842b3ad188SAdrian Chadd 
6852b3ad188SAdrian Chadd 	error = intr_event_add_handler(isrc->isrc_event, name, filter, handler,
6862b3ad188SAdrian Chadd 	    arg, intr_priority(flags), flags, cookiep);
6872b3ad188SAdrian Chadd 	if (error == 0) {
6882b3ad188SAdrian Chadd 		mtx_lock(&isrc_table_lock);
6892b3ad188SAdrian Chadd 		intrcnt_updatename(isrc);
6902b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
6912b3ad188SAdrian Chadd 	}
6922b3ad188SAdrian Chadd 
6932b3ad188SAdrian Chadd 	return (error);
6942b3ad188SAdrian Chadd }
6952b3ad188SAdrian Chadd 
6962b3ad188SAdrian Chadd /*
6972b3ad188SAdrian Chadd  *  Lookup interrupt controller locked.
6982b3ad188SAdrian Chadd  */
699bff6be3eSSvatopluk Kraus static inline struct intr_pic *
700c0d52370SAndrew Turner pic_lookup_locked(device_t dev, intptr_t xref, int flags)
7012b3ad188SAdrian Chadd {
7022b3ad188SAdrian Chadd 	struct intr_pic *pic;
7032b3ad188SAdrian Chadd 
7042b3ad188SAdrian Chadd 	mtx_assert(&pic_list_lock, MA_OWNED);
7052b3ad188SAdrian Chadd 
7064be58cbaSSvatopluk Kraus 	if (dev == NULL && xref == 0)
7074be58cbaSSvatopluk Kraus 		return (NULL);
7084be58cbaSSvatopluk Kraus 
7094be58cbaSSvatopluk Kraus 	/* Note that pic->pic_dev is never NULL on registered PIC. */
7102b3ad188SAdrian Chadd 	SLIST_FOREACH(pic, &pic_list, pic_next) {
711c0d52370SAndrew Turner 		if ((pic->pic_flags & FLAG_TYPE_MASK) !=
712c0d52370SAndrew Turner 		    (flags & FLAG_TYPE_MASK))
713c0d52370SAndrew Turner 			continue;
714c0d52370SAndrew Turner 
7154be58cbaSSvatopluk Kraus 		if (dev == NULL) {
7164be58cbaSSvatopluk Kraus 			if (xref == pic->pic_xref)
7174be58cbaSSvatopluk Kraus 				return (pic);
7184be58cbaSSvatopluk Kraus 		} else if (xref == 0 || pic->pic_xref == 0) {
7194be58cbaSSvatopluk Kraus 			if (dev == pic->pic_dev)
7204be58cbaSSvatopluk Kraus 				return (pic);
7214be58cbaSSvatopluk Kraus 		} else if (xref == pic->pic_xref && dev == pic->pic_dev)
7222b3ad188SAdrian Chadd 				return (pic);
7232b3ad188SAdrian Chadd 	}
7242b3ad188SAdrian Chadd 	return (NULL);
7252b3ad188SAdrian Chadd }
7262b3ad188SAdrian Chadd 
7272b3ad188SAdrian Chadd /*
7282b3ad188SAdrian Chadd  *  Lookup interrupt controller.
7292b3ad188SAdrian Chadd  */
7302b3ad188SAdrian Chadd static struct intr_pic *
731c0d52370SAndrew Turner pic_lookup(device_t dev, intptr_t xref, int flags)
7322b3ad188SAdrian Chadd {
7332b3ad188SAdrian Chadd 	struct intr_pic *pic;
7342b3ad188SAdrian Chadd 
7352b3ad188SAdrian Chadd 	mtx_lock(&pic_list_lock);
736c0d52370SAndrew Turner 	pic = pic_lookup_locked(dev, xref, flags);
7372b3ad188SAdrian Chadd 	mtx_unlock(&pic_list_lock);
7382b3ad188SAdrian Chadd 	return (pic);
7392b3ad188SAdrian Chadd }
7402b3ad188SAdrian Chadd 
7412b3ad188SAdrian Chadd /*
7422b3ad188SAdrian Chadd  *  Create interrupt controller.
7432b3ad188SAdrian Chadd  */
7442b3ad188SAdrian Chadd static struct intr_pic *
745c0d52370SAndrew Turner pic_create(device_t dev, intptr_t xref, int flags)
7462b3ad188SAdrian Chadd {
7472b3ad188SAdrian Chadd 	struct intr_pic *pic;
7482b3ad188SAdrian Chadd 
7492b3ad188SAdrian Chadd 	mtx_lock(&pic_list_lock);
750c0d52370SAndrew Turner 	pic = pic_lookup_locked(dev, xref, flags);
7512b3ad188SAdrian Chadd 	if (pic != NULL) {
7522b3ad188SAdrian Chadd 		mtx_unlock(&pic_list_lock);
7532b3ad188SAdrian Chadd 		return (pic);
7542b3ad188SAdrian Chadd 	}
7552b3ad188SAdrian Chadd 	pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO);
756b48c6083SAndrew Turner 	if (pic == NULL) {
757b48c6083SAndrew Turner 		mtx_unlock(&pic_list_lock);
758b48c6083SAndrew Turner 		return (NULL);
759b48c6083SAndrew Turner 	}
7602b3ad188SAdrian Chadd 	pic->pic_xref = xref;
7612b3ad188SAdrian Chadd 	pic->pic_dev = dev;
762c0d52370SAndrew Turner 	pic->pic_flags = flags;
763d1605cdaSAndrew Turner 	mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN);
7642b3ad188SAdrian Chadd 	SLIST_INSERT_HEAD(&pic_list, pic, pic_next);
7652b3ad188SAdrian Chadd 	mtx_unlock(&pic_list_lock);
7662b3ad188SAdrian Chadd 
7672b3ad188SAdrian Chadd 	return (pic);
7682b3ad188SAdrian Chadd }
7692b3ad188SAdrian Chadd #ifdef notyet
7702b3ad188SAdrian Chadd /*
7712b3ad188SAdrian Chadd  *  Destroy interrupt controller.
7722b3ad188SAdrian Chadd  */
7732b3ad188SAdrian Chadd static void
774c0d52370SAndrew Turner pic_destroy(device_t dev, intptr_t xref, int flags)
7752b3ad188SAdrian Chadd {
7762b3ad188SAdrian Chadd 	struct intr_pic *pic;
7772b3ad188SAdrian Chadd 
7782b3ad188SAdrian Chadd 	mtx_lock(&pic_list_lock);
779c0d52370SAndrew Turner 	pic = pic_lookup_locked(dev, xref, flags);
7802b3ad188SAdrian Chadd 	if (pic == NULL) {
7812b3ad188SAdrian Chadd 		mtx_unlock(&pic_list_lock);
7822b3ad188SAdrian Chadd 		return;
7832b3ad188SAdrian Chadd 	}
7842b3ad188SAdrian Chadd 	SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next);
7852b3ad188SAdrian Chadd 	mtx_unlock(&pic_list_lock);
7862b3ad188SAdrian Chadd 
7872b3ad188SAdrian Chadd 	free(pic, M_INTRNG);
7882b3ad188SAdrian Chadd }
7892b3ad188SAdrian Chadd #endif
7902b3ad188SAdrian Chadd /*
7912b3ad188SAdrian Chadd  *  Register interrupt controller.
7922b3ad188SAdrian Chadd  */
7939346e913SAndrew Turner struct intr_pic *
7942b3ad188SAdrian Chadd intr_pic_register(device_t dev, intptr_t xref)
7952b3ad188SAdrian Chadd {
7962b3ad188SAdrian Chadd 	struct intr_pic *pic;
7972b3ad188SAdrian Chadd 
7984be58cbaSSvatopluk Kraus 	if (dev == NULL)
7999346e913SAndrew Turner 		return (NULL);
800c0d52370SAndrew Turner 	pic = pic_create(dev, xref, FLAG_PIC);
8012b3ad188SAdrian Chadd 	if (pic == NULL)
8029346e913SAndrew Turner 		return (NULL);
8032b3ad188SAdrian Chadd 
804cff33fa8SEd Maste 	debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic,
805cff33fa8SEd Maste 	    device_get_nameunit(dev), dev, (uintmax_t)xref);
8069346e913SAndrew Turner 	return (pic);
8072b3ad188SAdrian Chadd }
8082b3ad188SAdrian Chadd 
8092b3ad188SAdrian Chadd /*
8102b3ad188SAdrian Chadd  *  Unregister interrupt controller.
8112b3ad188SAdrian Chadd  */
8122b3ad188SAdrian Chadd int
813bff6be3eSSvatopluk Kraus intr_pic_deregister(device_t dev, intptr_t xref)
8142b3ad188SAdrian Chadd {
8152b3ad188SAdrian Chadd 
8162b3ad188SAdrian Chadd 	panic("%s: not implemented", __func__);
8172b3ad188SAdrian Chadd }
8182b3ad188SAdrian Chadd 
8192b3ad188SAdrian Chadd /*
8202b3ad188SAdrian Chadd  *  Mark interrupt controller (itself) as a root one.
8212b3ad188SAdrian Chadd  *
8222b3ad188SAdrian Chadd  *  Note that only an interrupt controller can really know its position
8232b3ad188SAdrian Chadd  *  in interrupt controller's tree. So root PIC must claim itself as a root.
8242b3ad188SAdrian Chadd  *
8252b3ad188SAdrian Chadd  *  In FDT case, according to ePAPR approved version 1.1 from 08 April 2011,
8262b3ad188SAdrian Chadd  *  page 30:
8272b3ad188SAdrian Chadd  *    "The root of the interrupt tree is determined when traversal
8282b3ad188SAdrian Chadd  *     of the interrupt tree reaches an interrupt controller node without
8292b3ad188SAdrian Chadd  *     an interrupts property and thus no explicit interrupt parent."
8302b3ad188SAdrian Chadd  */
8312b3ad188SAdrian Chadd int
8322b3ad188SAdrian Chadd intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter,
8332b3ad188SAdrian Chadd     void *arg, u_int ipicount)
8342b3ad188SAdrian Chadd {
8353fc155dcSAndrew Turner 	struct intr_pic *pic;
8362b3ad188SAdrian Chadd 
837c0d52370SAndrew Turner 	pic = pic_lookup(dev, xref, FLAG_PIC);
8383fc155dcSAndrew Turner 	if (pic == NULL) {
8392b3ad188SAdrian Chadd 		device_printf(dev, "not registered\n");
8402b3ad188SAdrian Chadd 		return (EINVAL);
8412b3ad188SAdrian Chadd 	}
8423fc155dcSAndrew Turner 
843c0d52370SAndrew Turner 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC,
8443fc155dcSAndrew Turner 	    ("%s: Found a non-PIC controller: %s", __func__,
8453fc155dcSAndrew Turner 	     device_get_name(pic->pic_dev)));
8463fc155dcSAndrew Turner 
8472b3ad188SAdrian Chadd 	if (filter == NULL) {
8482b3ad188SAdrian Chadd 		device_printf(dev, "filter missing\n");
8492b3ad188SAdrian Chadd 		return (EINVAL);
8502b3ad188SAdrian Chadd 	}
8512b3ad188SAdrian Chadd 
8522b3ad188SAdrian Chadd 	/*
8532b3ad188SAdrian Chadd 	 * Only one interrupt controllers could be on the root for now.
8542b3ad188SAdrian Chadd 	 * Note that we further suppose that there is not threaded interrupt
8552b3ad188SAdrian Chadd 	 * routine (handler) on the root. See intr_irq_handler().
8562b3ad188SAdrian Chadd 	 */
8575b70c08cSSvatopluk Kraus 	if (intr_irq_root_dev != NULL) {
8582b3ad188SAdrian Chadd 		device_printf(dev, "another root already set\n");
8592b3ad188SAdrian Chadd 		return (EBUSY);
8602b3ad188SAdrian Chadd 	}
8612b3ad188SAdrian Chadd 
8625b70c08cSSvatopluk Kraus 	intr_irq_root_dev = dev;
8632b3ad188SAdrian Chadd 	irq_root_filter = filter;
8642b3ad188SAdrian Chadd 	irq_root_arg = arg;
8652b3ad188SAdrian Chadd 	irq_root_ipicount = ipicount;
8662b3ad188SAdrian Chadd 
8672b3ad188SAdrian Chadd 	debugf("irq root set to %s\n", device_get_nameunit(dev));
8682b3ad188SAdrian Chadd 	return (0);
8692b3ad188SAdrian Chadd }
8702b3ad188SAdrian Chadd 
871d1605cdaSAndrew Turner /*
872d1605cdaSAndrew Turner  * Add a handler to manage a sub range of a parents interrupts.
873d1605cdaSAndrew Turner  */
874d1605cdaSAndrew Turner struct intr_pic *
875d1605cdaSAndrew Turner intr_pic_add_handler(device_t parent, struct intr_pic *pic,
876d1605cdaSAndrew Turner     intr_child_irq_filter_t *filter, void *arg, uintptr_t start,
877d1605cdaSAndrew Turner     uintptr_t length)
878d1605cdaSAndrew Turner {
879d1605cdaSAndrew Turner 	struct intr_pic *parent_pic;
880d1605cdaSAndrew Turner 	struct intr_pic_child *newchild;
881d1605cdaSAndrew Turner #ifdef INVARIANTS
882d1605cdaSAndrew Turner 	struct intr_pic_child *child;
883d1605cdaSAndrew Turner #endif
884d1605cdaSAndrew Turner 
885c0d52370SAndrew Turner 	/* Find the parent PIC */
886c0d52370SAndrew Turner 	parent_pic = pic_lookup(parent, 0, FLAG_PIC);
887d1605cdaSAndrew Turner 	if (parent_pic == NULL)
888d1605cdaSAndrew Turner 		return (NULL);
889d1605cdaSAndrew Turner 
890d1605cdaSAndrew Turner 	newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO);
891d1605cdaSAndrew Turner 	newchild->pc_pic = pic;
892d1605cdaSAndrew Turner 	newchild->pc_filter = filter;
893d1605cdaSAndrew Turner 	newchild->pc_filter_arg = arg;
894d1605cdaSAndrew Turner 	newchild->pc_start = start;
895d1605cdaSAndrew Turner 	newchild->pc_length = length;
896d1605cdaSAndrew Turner 
897d1605cdaSAndrew Turner 	mtx_lock_spin(&parent_pic->pic_child_lock);
898d1605cdaSAndrew Turner #ifdef INVARIANTS
899d1605cdaSAndrew Turner 	SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) {
900d1605cdaSAndrew Turner 		KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice",
901d1605cdaSAndrew Turner 		    __func__));
902d1605cdaSAndrew Turner 	}
903d1605cdaSAndrew Turner #endif
904d1605cdaSAndrew Turner 	SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next);
905d1605cdaSAndrew Turner 	mtx_unlock_spin(&parent_pic->pic_child_lock);
906d1605cdaSAndrew Turner 
907d1605cdaSAndrew Turner 	return (pic);
908d1605cdaSAndrew Turner }
909d1605cdaSAndrew Turner 
910895c8b1cSMichal Meloun static int
911895c8b1cSMichal Meloun intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data,
912895c8b1cSMichal Meloun     struct intr_irqsrc **isrc)
9132b3ad188SAdrian Chadd {
914bff6be3eSSvatopluk Kraus 	struct intr_pic *pic;
915895c8b1cSMichal Meloun 	struct intr_map_data_msi *msi;
916bff6be3eSSvatopluk Kraus 
917bff6be3eSSvatopluk Kraus 	if (data == NULL)
918bff6be3eSSvatopluk Kraus 		return (EINVAL);
919bff6be3eSSvatopluk Kraus 
920c0d52370SAndrew Turner 	pic = pic_lookup(dev, xref,
921c0d52370SAndrew Turner 	    (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC);
92215adccc6SSvatopluk Kraus 	if (pic == NULL)
923bff6be3eSSvatopluk Kraus 		return (ESRCH);
924bff6be3eSSvatopluk Kraus 
925895c8b1cSMichal Meloun 	switch (data->type) {
926895c8b1cSMichal Meloun 	case INTR_MAP_DATA_MSI:
927c0d52370SAndrew Turner 		KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
928895c8b1cSMichal Meloun 		    ("%s: Found a non-MSI controller: %s", __func__,
929895c8b1cSMichal Meloun 		     device_get_name(pic->pic_dev)));
930895c8b1cSMichal Meloun 		msi = (struct intr_map_data_msi *)data;
931895c8b1cSMichal Meloun 		*isrc = msi->isrc;
932895c8b1cSMichal Meloun 		return (0);
933895c8b1cSMichal Meloun 
934895c8b1cSMichal Meloun 	default:
935c0d52370SAndrew Turner 		KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC,
9363fc155dcSAndrew Turner 		    ("%s: Found a non-PIC controller: %s", __func__,
9373fc155dcSAndrew Turner 		     device_get_name(pic->pic_dev)));
938895c8b1cSMichal Meloun 		return (PIC_MAP_INTR(pic->pic_dev, data, isrc));
9393fc155dcSAndrew Turner 
940895c8b1cSMichal Meloun 	}
941895c8b1cSMichal Meloun }
942895c8b1cSMichal Meloun 
943895c8b1cSMichal Meloun int
944895c8b1cSMichal Meloun intr_activate_irq(device_t dev, struct resource *res)
945895c8b1cSMichal Meloun {
946895c8b1cSMichal Meloun 	device_t map_dev;
947895c8b1cSMichal Meloun 	intptr_t map_xref;
948895c8b1cSMichal Meloun 	struct intr_map_data *data;
949895c8b1cSMichal Meloun 	struct intr_irqsrc *isrc;
950895c8b1cSMichal Meloun 	u_int res_id;
951895c8b1cSMichal Meloun 	int error;
952895c8b1cSMichal Meloun 
953895c8b1cSMichal Meloun 	KASSERT(rman_get_start(res) == rman_get_end(res),
954895c8b1cSMichal Meloun 	    ("%s: more interrupts in resource", __func__));
955895c8b1cSMichal Meloun 
956895c8b1cSMichal Meloun 	res_id = (u_int)rman_get_start(res);
957895c8b1cSMichal Meloun 	if (intr_map_get_isrc(res_id) != NULL)
958895c8b1cSMichal Meloun 		panic("Attempt to double activation of resource id: %u\n",
959895c8b1cSMichal Meloun 		    res_id);
960895c8b1cSMichal Meloun 	intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data);
961895c8b1cSMichal Meloun 	error = intr_resolve_irq(map_dev, map_xref, data, &isrc);
962895c8b1cSMichal Meloun 	if (error != 0) {
963895c8b1cSMichal Meloun 		free(data, M_INTRNG);
964895c8b1cSMichal Meloun 		/* XXX TODO DISCONECTED PICs */
965895c8b1cSMichal Meloun 		/* if (error == EINVAL) return(0); */
966bff6be3eSSvatopluk Kraus 		return (error);
967bff6be3eSSvatopluk Kraus 	}
968895c8b1cSMichal Meloun 	intr_map_set_isrc(res_id, isrc);
969895c8b1cSMichal Meloun 	rman_set_virtual(res, data);
970895c8b1cSMichal Meloun 	return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data));
971bff6be3eSSvatopluk Kraus }
972bff6be3eSSvatopluk Kraus 
973bff6be3eSSvatopluk Kraus int
974895c8b1cSMichal Meloun intr_deactivate_irq(device_t dev, struct resource *res)
975bff6be3eSSvatopluk Kraus {
976bff6be3eSSvatopluk Kraus 	struct intr_map_data *data;
977bff6be3eSSvatopluk Kraus 	struct intr_irqsrc *isrc;
978895c8b1cSMichal Meloun 	u_int res_id;
979895c8b1cSMichal Meloun 	int error;
980bff6be3eSSvatopluk Kraus 
981bff6be3eSSvatopluk Kraus 	KASSERT(rman_get_start(res) == rman_get_end(res),
982bff6be3eSSvatopluk Kraus 	    ("%s: more interrupts in resource", __func__));
983bff6be3eSSvatopluk Kraus 
984895c8b1cSMichal Meloun 	res_id = (u_int)rman_get_start(res);
985895c8b1cSMichal Meloun 	isrc = intr_map_get_isrc(res_id);
986bff6be3eSSvatopluk Kraus 	if (isrc == NULL)
987895c8b1cSMichal Meloun 		panic("Attempt to deactivate non-active resource id: %u\n",
988895c8b1cSMichal Meloun 		    res_id);
989bff6be3eSSvatopluk Kraus 
990c4263292SSvatopluk Kraus 	data = rman_get_virtual(res);
991895c8b1cSMichal Meloun 	error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data);
992895c8b1cSMichal Meloun 	intr_map_set_isrc(res_id, NULL);
993895c8b1cSMichal Meloun 	rman_set_virtual(res, NULL);
994895c8b1cSMichal Meloun 	free(data, M_INTRNG);
995895c8b1cSMichal Meloun 	return (error);
996bff6be3eSSvatopluk Kraus }
997bff6be3eSSvatopluk Kraus 
998bff6be3eSSvatopluk Kraus int
999bff6be3eSSvatopluk Kraus intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt,
1000bff6be3eSSvatopluk Kraus     driver_intr_t hand, void *arg, int flags, void **cookiep)
1001bff6be3eSSvatopluk Kraus {
1002bff6be3eSSvatopluk Kraus 	int error;
1003bff6be3eSSvatopluk Kraus 	struct intr_map_data *data;
1004bff6be3eSSvatopluk Kraus 	struct intr_irqsrc *isrc;
1005bff6be3eSSvatopluk Kraus 	const char *name;
1006895c8b1cSMichal Meloun 	u_int res_id;
1007bff6be3eSSvatopluk Kraus 
1008bff6be3eSSvatopluk Kraus 	KASSERT(rman_get_start(res) == rman_get_end(res),
1009bff6be3eSSvatopluk Kraus 	    ("%s: more interrupts in resource", __func__));
1010bff6be3eSSvatopluk Kraus 
1011895c8b1cSMichal Meloun 	res_id = (u_int)rman_get_start(res);
1012895c8b1cSMichal Meloun 	isrc = intr_map_get_isrc(res_id);
1013895c8b1cSMichal Meloun 	if (isrc == NULL) {
1014895c8b1cSMichal Meloun 		/* XXX TODO DISCONECTED PICs */
1015bff6be3eSSvatopluk Kraus 		return (EINVAL);
1016895c8b1cSMichal Meloun 	}
10172b3ad188SAdrian Chadd 
1018c4263292SSvatopluk Kraus 	data = rman_get_virtual(res);
10192b3ad188SAdrian Chadd 	name = device_get_nameunit(dev);
10202b3ad188SAdrian Chadd 
10212b3ad188SAdrian Chadd #ifdef INTR_SOLO
10222b3ad188SAdrian Chadd 	/*
1023e3043798SPedro F. Giffuni 	 * Standard handling is done through MI interrupt framework. However,
10242b3ad188SAdrian Chadd 	 * some interrupts could request solely own special handling. This
10252b3ad188SAdrian Chadd 	 * non standard handling can be used for interrupt controllers without
10262b3ad188SAdrian Chadd 	 * handler (filter only), so in case that interrupt controllers are
10272b3ad188SAdrian Chadd 	 * chained, MI interrupt framework is called only in leaf controller.
10282b3ad188SAdrian Chadd 	 *
10292b3ad188SAdrian Chadd 	 * Note that root interrupt controller routine is served as well,
10302b3ad188SAdrian Chadd 	 * however in intr_irq_handler(), i.e. main system dispatch routine.
10312b3ad188SAdrian Chadd 	 */
10322b3ad188SAdrian Chadd 	if (flags & INTR_SOLO && hand != NULL) {
10332b3ad188SAdrian Chadd 		debugf("irq %u cannot solo on %s\n", irq, name);
10342b3ad188SAdrian Chadd 		return (EINVAL);
10352b3ad188SAdrian Chadd 	}
10362b3ad188SAdrian Chadd 
10372b3ad188SAdrian Chadd 	if (flags & INTR_SOLO) {
10382b3ad188SAdrian Chadd 		error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt,
10392b3ad188SAdrian Chadd 		    arg, cookiep);
1040ce44a736SIan Lepore 		debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error,
10412b3ad188SAdrian Chadd 		    name);
10422b3ad188SAdrian Chadd 	} else
10432b3ad188SAdrian Chadd #endif
10442b3ad188SAdrian Chadd 		{
10452b3ad188SAdrian Chadd 		error = isrc_add_handler(isrc, name, filt, hand, arg, flags,
10462b3ad188SAdrian Chadd 		    cookiep);
1047ce44a736SIan Lepore 		debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name);
10482b3ad188SAdrian Chadd 	}
10492b3ad188SAdrian Chadd 	if (error != 0)
10502b3ad188SAdrian Chadd 		return (error);
10512b3ad188SAdrian Chadd 
10522b3ad188SAdrian Chadd 	mtx_lock(&isrc_table_lock);
1053bff6be3eSSvatopluk Kraus 	error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data);
1054bff6be3eSSvatopluk Kraus 	if (error == 0) {
10552b3ad188SAdrian Chadd 		isrc->isrc_handlers++;
1056bff6be3eSSvatopluk Kraus 		if (isrc->isrc_handlers == 1)
10572b3ad188SAdrian Chadd 			PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
10582b3ad188SAdrian Chadd 	}
10592b3ad188SAdrian Chadd 	mtx_unlock(&isrc_table_lock);
1060bff6be3eSSvatopluk Kraus 	if (error != 0)
1061bff6be3eSSvatopluk Kraus 		intr_event_remove_handler(*cookiep);
1062bff6be3eSSvatopluk Kraus 	return (error);
10632b3ad188SAdrian Chadd }
10642b3ad188SAdrian Chadd 
10652b3ad188SAdrian Chadd int
1066bff6be3eSSvatopluk Kraus intr_teardown_irq(device_t dev, struct resource *res, void *cookie)
10672b3ad188SAdrian Chadd {
10682b3ad188SAdrian Chadd 	int error;
1069bff6be3eSSvatopluk Kraus 	struct intr_map_data *data;
1070bff6be3eSSvatopluk Kraus 	struct intr_irqsrc *isrc;
1071895c8b1cSMichal Meloun 	u_int res_id;
10722b3ad188SAdrian Chadd 
1073bff6be3eSSvatopluk Kraus 	KASSERT(rman_get_start(res) == rman_get_end(res),
1074bff6be3eSSvatopluk Kraus 	    ("%s: more interrupts in resource", __func__));
1075bff6be3eSSvatopluk Kraus 
1076895c8b1cSMichal Meloun 	res_id = (u_int)rman_get_start(res);
1077895c8b1cSMichal Meloun 	isrc = intr_map_get_isrc(res_id);
10782b3ad188SAdrian Chadd 	if (isrc == NULL || isrc->isrc_handlers == 0)
10792b3ad188SAdrian Chadd 		return (EINVAL);
1080bff6be3eSSvatopluk Kraus 
1081c4263292SSvatopluk Kraus 	data = rman_get_virtual(res);
1082c4263292SSvatopluk Kraus 
1083169e6abdSSvatopluk Kraus #ifdef INTR_SOLO
10842b3ad188SAdrian Chadd 	if (isrc->isrc_filter != NULL) {
10852b3ad188SAdrian Chadd 		if (isrc != cookie)
10862b3ad188SAdrian Chadd 			return (EINVAL);
10872b3ad188SAdrian Chadd 
10882b3ad188SAdrian Chadd 		mtx_lock(&isrc_table_lock);
10892b3ad188SAdrian Chadd 		isrc->isrc_filter = NULL;
10902b3ad188SAdrian Chadd 		isrc->isrc_arg = NULL;
10912b3ad188SAdrian Chadd 		isrc->isrc_handlers = 0;
10922b3ad188SAdrian Chadd 		PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
1093bff6be3eSSvatopluk Kraus 		PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data);
10942b3ad188SAdrian Chadd 		isrc_update_name(isrc, NULL);
10952b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
10962b3ad188SAdrian Chadd 		return (0);
10972b3ad188SAdrian Chadd 	}
1098169e6abdSSvatopluk Kraus #endif
10992b3ad188SAdrian Chadd 	if (isrc != intr_handler_source(cookie))
11002b3ad188SAdrian Chadd 		return (EINVAL);
11012b3ad188SAdrian Chadd 
11022b3ad188SAdrian Chadd 	error = intr_event_remove_handler(cookie);
11032b3ad188SAdrian Chadd 	if (error == 0) {
11042b3ad188SAdrian Chadd 		mtx_lock(&isrc_table_lock);
11052b3ad188SAdrian Chadd 		isrc->isrc_handlers--;
1106bff6be3eSSvatopluk Kraus 		if (isrc->isrc_handlers == 0)
11072b3ad188SAdrian Chadd 			PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
1108bff6be3eSSvatopluk Kraus 		PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data);
11092b3ad188SAdrian Chadd 		intrcnt_updatename(isrc);
11102b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
11112b3ad188SAdrian Chadd 	}
11122b3ad188SAdrian Chadd 	return (error);
11132b3ad188SAdrian Chadd }
11142b3ad188SAdrian Chadd 
11152b3ad188SAdrian Chadd int
1116bff6be3eSSvatopluk Kraus intr_describe_irq(device_t dev, struct resource *res, void *cookie,
1117bff6be3eSSvatopluk Kraus     const char *descr)
11182b3ad188SAdrian Chadd {
11192b3ad188SAdrian Chadd 	int error;
1120bff6be3eSSvatopluk Kraus 	struct intr_irqsrc *isrc;
1121895c8b1cSMichal Meloun 	u_int res_id;
11222b3ad188SAdrian Chadd 
1123bff6be3eSSvatopluk Kraus 	KASSERT(rman_get_start(res) == rman_get_end(res),
1124bff6be3eSSvatopluk Kraus 	    ("%s: more interrupts in resource", __func__));
1125bff6be3eSSvatopluk Kraus 
1126895c8b1cSMichal Meloun 	res_id = (u_int)rman_get_start(res);
1127895c8b1cSMichal Meloun 	isrc = intr_map_get_isrc(res_id);
11282b3ad188SAdrian Chadd 	if (isrc == NULL || isrc->isrc_handlers == 0)
11292b3ad188SAdrian Chadd 		return (EINVAL);
1130169e6abdSSvatopluk Kraus #ifdef INTR_SOLO
11312b3ad188SAdrian Chadd 	if (isrc->isrc_filter != NULL) {
11322b3ad188SAdrian Chadd 		if (isrc != cookie)
11332b3ad188SAdrian Chadd 			return (EINVAL);
11342b3ad188SAdrian Chadd 
11352b3ad188SAdrian Chadd 		mtx_lock(&isrc_table_lock);
11362b3ad188SAdrian Chadd 		isrc_update_name(isrc, descr);
11372b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
11382b3ad188SAdrian Chadd 		return (0);
11392b3ad188SAdrian Chadd 	}
1140169e6abdSSvatopluk Kraus #endif
11412b3ad188SAdrian Chadd 	error = intr_event_describe_handler(isrc->isrc_event, cookie, descr);
11422b3ad188SAdrian Chadd 	if (error == 0) {
11432b3ad188SAdrian Chadd 		mtx_lock(&isrc_table_lock);
11442b3ad188SAdrian Chadd 		intrcnt_updatename(isrc);
11452b3ad188SAdrian Chadd 		mtx_unlock(&isrc_table_lock);
11462b3ad188SAdrian Chadd 	}
11472b3ad188SAdrian Chadd 	return (error);
11482b3ad188SAdrian Chadd }
11492b3ad188SAdrian Chadd 
11502b3ad188SAdrian Chadd #ifdef SMP
11512b3ad188SAdrian Chadd int
1152bff6be3eSSvatopluk Kraus intr_bind_irq(device_t dev, struct resource *res, int cpu)
11532b3ad188SAdrian Chadd {
11542b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc;
1155895c8b1cSMichal Meloun 	u_int res_id;
11562b3ad188SAdrian Chadd 
1157bff6be3eSSvatopluk Kraus 	KASSERT(rman_get_start(res) == rman_get_end(res),
1158bff6be3eSSvatopluk Kraus 	    ("%s: more interrupts in resource", __func__));
1159bff6be3eSSvatopluk Kraus 
1160895c8b1cSMichal Meloun 	res_id = (u_int)rman_get_start(res);
1161895c8b1cSMichal Meloun 	isrc = intr_map_get_isrc(res_id);
11622b3ad188SAdrian Chadd 	if (isrc == NULL || isrc->isrc_handlers == 0)
11632b3ad188SAdrian Chadd 		return (EINVAL);
1164169e6abdSSvatopluk Kraus #ifdef INTR_SOLO
11652b3ad188SAdrian Chadd 	if (isrc->isrc_filter != NULL)
11662b3ad188SAdrian Chadd 		return (intr_isrc_assign_cpu(isrc, cpu));
1167169e6abdSSvatopluk Kraus #endif
11682b3ad188SAdrian Chadd 	return (intr_event_bind(isrc->isrc_event, cpu));
11692b3ad188SAdrian Chadd }
11702b3ad188SAdrian Chadd 
11712b3ad188SAdrian Chadd /*
11722b3ad188SAdrian Chadd  * Return the CPU that the next interrupt source should use.
11732b3ad188SAdrian Chadd  * For now just returns the next CPU according to round-robin.
11742b3ad188SAdrian Chadd  */
11752b3ad188SAdrian Chadd u_int
11762b3ad188SAdrian Chadd intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask)
11772b3ad188SAdrian Chadd {
1178a92a2f00SAndrew Turner 	u_int cpu;
11792b3ad188SAdrian Chadd 
1180a92a2f00SAndrew Turner 	KASSERT(!CPU_EMPTY(cpumask), ("%s: Empty CPU mask", __func__));
1181a92a2f00SAndrew Turner 	if (!irq_assign_cpu || mp_ncpus == 1) {
1182a92a2f00SAndrew Turner 		cpu = PCPU_GET(cpuid);
1183a92a2f00SAndrew Turner 
1184a92a2f00SAndrew Turner 		if (CPU_ISSET(cpu, cpumask))
1185a92a2f00SAndrew Turner 			return (curcpu);
1186a92a2f00SAndrew Turner 
1187a92a2f00SAndrew Turner 		return (CPU_FFS(cpumask) - 1);
1188a92a2f00SAndrew Turner 	}
11892b3ad188SAdrian Chadd 
11902b3ad188SAdrian Chadd 	do {
11912b3ad188SAdrian Chadd 		last_cpu++;
11922b3ad188SAdrian Chadd 		if (last_cpu > mp_maxid)
11932b3ad188SAdrian Chadd 			last_cpu = 0;
11942b3ad188SAdrian Chadd 	} while (!CPU_ISSET(last_cpu, cpumask));
11952b3ad188SAdrian Chadd 	return (last_cpu);
11962b3ad188SAdrian Chadd }
11972b3ad188SAdrian Chadd 
1198*dc425090SMitchell Horne #ifndef EARLY_AP_STARTUP
11992b3ad188SAdrian Chadd /*
12002b3ad188SAdrian Chadd  *  Distribute all the interrupt sources among the available
12012b3ad188SAdrian Chadd  *  CPUs once the AP's have been launched.
12022b3ad188SAdrian Chadd  */
12032b3ad188SAdrian Chadd static void
12042b3ad188SAdrian Chadd intr_irq_shuffle(void *arg __unused)
12052b3ad188SAdrian Chadd {
12062b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc;
12072b3ad188SAdrian Chadd 	u_int i;
12082b3ad188SAdrian Chadd 
12092b3ad188SAdrian Chadd 	if (mp_ncpus == 1)
12102b3ad188SAdrian Chadd 		return;
12112b3ad188SAdrian Chadd 
12122b3ad188SAdrian Chadd 	mtx_lock(&isrc_table_lock);
1213*dc425090SMitchell Horne 	irq_assign_cpu = true;
12142b3ad188SAdrian Chadd 	for (i = 0; i < NIRQ; i++) {
12152b3ad188SAdrian Chadd 		isrc = irq_sources[i];
12162b3ad188SAdrian Chadd 		if (isrc == NULL || isrc->isrc_handlers == 0 ||
1217cf55df9fSSvatopluk Kraus 		    isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI))
12182b3ad188SAdrian Chadd 			continue;
12192b3ad188SAdrian Chadd 
12202b3ad188SAdrian Chadd 		if (isrc->isrc_event != NULL &&
12212b3ad188SAdrian Chadd 		    isrc->isrc_flags & INTR_ISRCF_BOUND &&
12222b3ad188SAdrian Chadd 		    isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1)
12232b3ad188SAdrian Chadd 			panic("%s: CPU inconsistency", __func__);
12242b3ad188SAdrian Chadd 
12252b3ad188SAdrian Chadd 		if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0)
12262b3ad188SAdrian Chadd 			CPU_ZERO(&isrc->isrc_cpu); /* start again */
12272b3ad188SAdrian Chadd 
12282b3ad188SAdrian Chadd 		/*
12292b3ad188SAdrian Chadd 		 * We are in wicked position here if the following call fails
12302b3ad188SAdrian Chadd 		 * for bound ISRC. The best thing we can do is to clear
12312b3ad188SAdrian Chadd 		 * isrc_cpu so inconsistency with ie_cpu will be detectable.
12322b3ad188SAdrian Chadd 		 */
1233bff6be3eSSvatopluk Kraus 		if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0)
12342b3ad188SAdrian Chadd 			CPU_ZERO(&isrc->isrc_cpu);
12352b3ad188SAdrian Chadd 	}
12362b3ad188SAdrian Chadd 	mtx_unlock(&isrc_table_lock);
12372b3ad188SAdrian Chadd }
12382b3ad188SAdrian Chadd SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL);
1239*dc425090SMitchell Horne #endif /* !EARLY_AP_STARTUP */
12402b3ad188SAdrian Chadd 
12412b3ad188SAdrian Chadd #else
12422b3ad188SAdrian Chadd u_int
12432b3ad188SAdrian Chadd intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask)
12442b3ad188SAdrian Chadd {
12452b3ad188SAdrian Chadd 
12462b3ad188SAdrian Chadd 	return (PCPU_GET(cpuid));
12472b3ad188SAdrian Chadd }
1248*dc425090SMitchell Horne #endif /* SMP */
12492b3ad188SAdrian Chadd 
12503fc155dcSAndrew Turner /*
1251895c8b1cSMichal Meloun  * Allocate memory for new intr_map_data structure.
1252895c8b1cSMichal Meloun  * Initialize common fields.
1253895c8b1cSMichal Meloun  */
1254895c8b1cSMichal Meloun struct intr_map_data *
1255895c8b1cSMichal Meloun intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags)
1256895c8b1cSMichal Meloun {
1257895c8b1cSMichal Meloun 	struct intr_map_data *data;
1258895c8b1cSMichal Meloun 
1259895c8b1cSMichal Meloun 	data = malloc(len, M_INTRNG, flags);
1260895c8b1cSMichal Meloun 	data->type = type;
1261895c8b1cSMichal Meloun 	data->len = len;
1262895c8b1cSMichal Meloun 	return (data);
1263895c8b1cSMichal Meloun }
1264895c8b1cSMichal Meloun 
1265895c8b1cSMichal Meloun void intr_free_intr_map_data(struct intr_map_data *data)
1266895c8b1cSMichal Meloun {
1267895c8b1cSMichal Meloun 
1268895c8b1cSMichal Meloun 	free(data, M_INTRNG);
1269895c8b1cSMichal Meloun }
1270895c8b1cSMichal Meloun 
1271895c8b1cSMichal Meloun /*
12723fc155dcSAndrew Turner  *  Register a MSI/MSI-X interrupt controller
12733fc155dcSAndrew Turner  */
12743fc155dcSAndrew Turner int
12753fc155dcSAndrew Turner intr_msi_register(device_t dev, intptr_t xref)
12763fc155dcSAndrew Turner {
12773fc155dcSAndrew Turner 	struct intr_pic *pic;
12783fc155dcSAndrew Turner 
12793fc155dcSAndrew Turner 	if (dev == NULL)
12803fc155dcSAndrew Turner 		return (EINVAL);
1281c0d52370SAndrew Turner 	pic = pic_create(dev, xref, FLAG_MSI);
12823fc155dcSAndrew Turner 	if (pic == NULL)
12833fc155dcSAndrew Turner 		return (ENOMEM);
12843fc155dcSAndrew Turner 
12853fc155dcSAndrew Turner 	debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic,
12863fc155dcSAndrew Turner 	    device_get_nameunit(dev), dev, (uintmax_t)xref);
12873fc155dcSAndrew Turner 	return (0);
12883fc155dcSAndrew Turner }
12893fc155dcSAndrew Turner 
12903fc155dcSAndrew Turner int
12913fc155dcSAndrew Turner intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count,
12923fc155dcSAndrew Turner     int maxcount, int *irqs)
12933fc155dcSAndrew Turner {
12943fc155dcSAndrew Turner 	struct intr_irqsrc **isrc;
12953fc155dcSAndrew Turner 	struct intr_pic *pic;
12963fc155dcSAndrew Turner 	device_t pdev;
1297895c8b1cSMichal Meloun 	struct intr_map_data_msi *msi;
12983fc155dcSAndrew Turner 	int err, i;
12993fc155dcSAndrew Turner 
1300c0d52370SAndrew Turner 	pic = pic_lookup(NULL, xref, FLAG_MSI);
13013fc155dcSAndrew Turner 	if (pic == NULL)
13023fc155dcSAndrew Turner 		return (ESRCH);
13033fc155dcSAndrew Turner 
1304c0d52370SAndrew Turner 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
13053fc155dcSAndrew Turner 	    ("%s: Found a non-MSI controller: %s", __func__,
13063fc155dcSAndrew Turner 	     device_get_name(pic->pic_dev)));
13073fc155dcSAndrew Turner 
13083fc155dcSAndrew Turner 	isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK);
13093fc155dcSAndrew Turner 	err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc);
1310895c8b1cSMichal Meloun 	if (err != 0) {
1311895c8b1cSMichal Meloun 		free(isrc, M_INTRNG);
1312895c8b1cSMichal Meloun 		return (err);
13133fc155dcSAndrew Turner 	}
13143fc155dcSAndrew Turner 
1315895c8b1cSMichal Meloun 	for (i = 0; i < count; i++) {
1316895c8b1cSMichal Meloun 		msi = (struct intr_map_data_msi *)intr_alloc_map_data(
1317895c8b1cSMichal Meloun 		    INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO);
1318895c8b1cSMichal Meloun 		msi-> isrc = isrc[i];
1319895c8b1cSMichal Meloun 		irqs[i] = intr_map_irq(pic->pic_dev, xref,
1320895c8b1cSMichal Meloun 		    (struct intr_map_data *)msi);
1321895c8b1cSMichal Meloun 
1322895c8b1cSMichal Meloun 	}
13233fc155dcSAndrew Turner 	free(isrc, M_INTRNG);
13243fc155dcSAndrew Turner 
13253fc155dcSAndrew Turner 	return (err);
13263fc155dcSAndrew Turner }
13273fc155dcSAndrew Turner 
13283fc155dcSAndrew Turner int
13293fc155dcSAndrew Turner intr_release_msi(device_t pci, device_t child, intptr_t xref, int count,
13303fc155dcSAndrew Turner     int *irqs)
13313fc155dcSAndrew Turner {
13323fc155dcSAndrew Turner 	struct intr_irqsrc **isrc;
13333fc155dcSAndrew Turner 	struct intr_pic *pic;
1334609b0fe9SOleksandr Tymoshenko 	struct intr_map_data_msi *msi;
13353fc155dcSAndrew Turner 	int i, err;
13363fc155dcSAndrew Turner 
1337c0d52370SAndrew Turner 	pic = pic_lookup(NULL, xref, FLAG_MSI);
13383fc155dcSAndrew Turner 	if (pic == NULL)
13393fc155dcSAndrew Turner 		return (ESRCH);
13403fc155dcSAndrew Turner 
1341c0d52370SAndrew Turner 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
13423fc155dcSAndrew Turner 	    ("%s: Found a non-MSI controller: %s", __func__,
13433fc155dcSAndrew Turner 	     device_get_name(pic->pic_dev)));
13443fc155dcSAndrew Turner 
13453fc155dcSAndrew Turner 	isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK);
13463fc155dcSAndrew Turner 
1347609b0fe9SOleksandr Tymoshenko 	for (i = 0; i < count; i++) {
1348609b0fe9SOleksandr Tymoshenko 		msi = (struct intr_map_data_msi *)
1349609b0fe9SOleksandr Tymoshenko 		    intr_map_get_map_data(irqs[i]);
1350609b0fe9SOleksandr Tymoshenko 		KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI,
1351609b0fe9SOleksandr Tymoshenko 		    ("%s: irq %d map data is not MSI", __func__,
1352609b0fe9SOleksandr Tymoshenko 		    irqs[i]));
1353609b0fe9SOleksandr Tymoshenko 		isrc[i] = msi->isrc;
1354609b0fe9SOleksandr Tymoshenko 	}
13553fc155dcSAndrew Turner 
13563fc155dcSAndrew Turner 	err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc);
1357895c8b1cSMichal Meloun 
1358895c8b1cSMichal Meloun 	for (i = 0; i < count; i++) {
1359895c8b1cSMichal Meloun 		if (isrc[i] != NULL)
1360895c8b1cSMichal Meloun 			intr_unmap_irq(irqs[i]);
1361895c8b1cSMichal Meloun 	}
1362895c8b1cSMichal Meloun 
13633fc155dcSAndrew Turner 	free(isrc, M_INTRNG);
13643fc155dcSAndrew Turner 	return (err);
13653fc155dcSAndrew Turner }
13663fc155dcSAndrew Turner 
13673fc155dcSAndrew Turner int
13683fc155dcSAndrew Turner intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq)
13693fc155dcSAndrew Turner {
13703fc155dcSAndrew Turner 	struct intr_irqsrc *isrc;
13713fc155dcSAndrew Turner 	struct intr_pic *pic;
13723fc155dcSAndrew Turner 	device_t pdev;
1373895c8b1cSMichal Meloun 	struct intr_map_data_msi *msi;
13743fc155dcSAndrew Turner 	int err;
13753fc155dcSAndrew Turner 
1376c0d52370SAndrew Turner 	pic = pic_lookup(NULL, xref, FLAG_MSI);
13773fc155dcSAndrew Turner 	if (pic == NULL)
13783fc155dcSAndrew Turner 		return (ESRCH);
13793fc155dcSAndrew Turner 
1380c0d52370SAndrew Turner 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
13813fc155dcSAndrew Turner 	    ("%s: Found a non-MSI controller: %s", __func__,
13823fc155dcSAndrew Turner 	     device_get_name(pic->pic_dev)));
13833fc155dcSAndrew Turner 
13843fc155dcSAndrew Turner 	err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc);
13853fc155dcSAndrew Turner 	if (err != 0)
13863fc155dcSAndrew Turner 		return (err);
13873fc155dcSAndrew Turner 
1388895c8b1cSMichal Meloun 	msi = (struct intr_map_data_msi *)intr_alloc_map_data(
1389895c8b1cSMichal Meloun 		    INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO);
1390895c8b1cSMichal Meloun 	msi->isrc = isrc;
1391895c8b1cSMichal Meloun 	*irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi);
13923fc155dcSAndrew Turner 	return (0);
13933fc155dcSAndrew Turner }
13943fc155dcSAndrew Turner 
13953fc155dcSAndrew Turner int
13963fc155dcSAndrew Turner intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq)
13973fc155dcSAndrew Turner {
13983fc155dcSAndrew Turner 	struct intr_irqsrc *isrc;
13993fc155dcSAndrew Turner 	struct intr_pic *pic;
1400609b0fe9SOleksandr Tymoshenko 	struct intr_map_data_msi *msi;
14013fc155dcSAndrew Turner 	int err;
14023fc155dcSAndrew Turner 
1403c0d52370SAndrew Turner 	pic = pic_lookup(NULL, xref, FLAG_MSI);
14043fc155dcSAndrew Turner 	if (pic == NULL)
14053fc155dcSAndrew Turner 		return (ESRCH);
14063fc155dcSAndrew Turner 
1407c0d52370SAndrew Turner 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
14083fc155dcSAndrew Turner 	    ("%s: Found a non-MSI controller: %s", __func__,
14093fc155dcSAndrew Turner 	     device_get_name(pic->pic_dev)));
14103fc155dcSAndrew Turner 
1411609b0fe9SOleksandr Tymoshenko 	msi = (struct intr_map_data_msi *)
1412609b0fe9SOleksandr Tymoshenko 	    intr_map_get_map_data(irq);
1413609b0fe9SOleksandr Tymoshenko 	KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI,
1414609b0fe9SOleksandr Tymoshenko 	    ("%s: irq %d map data is not MSI", __func__,
1415609b0fe9SOleksandr Tymoshenko 	    irq));
1416609b0fe9SOleksandr Tymoshenko 	isrc = msi->isrc;
1417895c8b1cSMichal Meloun 	if (isrc == NULL) {
1418895c8b1cSMichal Meloun 		intr_unmap_irq(irq);
14193fc155dcSAndrew Turner 		return (EINVAL);
1420895c8b1cSMichal Meloun 	}
14213fc155dcSAndrew Turner 
14223fc155dcSAndrew Turner 	err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc);
1423895c8b1cSMichal Meloun 	intr_unmap_irq(irq);
1424895c8b1cSMichal Meloun 
14253fc155dcSAndrew Turner 	return (err);
14263fc155dcSAndrew Turner }
14273fc155dcSAndrew Turner 
14283fc155dcSAndrew Turner int
14293fc155dcSAndrew Turner intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq,
14303fc155dcSAndrew Turner     uint64_t *addr, uint32_t *data)
14313fc155dcSAndrew Turner {
14323fc155dcSAndrew Turner 	struct intr_irqsrc *isrc;
14333fc155dcSAndrew Turner 	struct intr_pic *pic;
14343fc155dcSAndrew Turner 	int err;
14353fc155dcSAndrew Turner 
1436c0d52370SAndrew Turner 	pic = pic_lookup(NULL, xref, FLAG_MSI);
14373fc155dcSAndrew Turner 	if (pic == NULL)
14383fc155dcSAndrew Turner 		return (ESRCH);
14393fc155dcSAndrew Turner 
1440c0d52370SAndrew Turner 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
14413fc155dcSAndrew Turner 	    ("%s: Found a non-MSI controller: %s", __func__,
14423fc155dcSAndrew Turner 	     device_get_name(pic->pic_dev)));
14433fc155dcSAndrew Turner 
1444895c8b1cSMichal Meloun 	isrc = intr_map_get_isrc(irq);
14453fc155dcSAndrew Turner 	if (isrc == NULL)
14463fc155dcSAndrew Turner 		return (EINVAL);
14473fc155dcSAndrew Turner 
14483fc155dcSAndrew Turner 	err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data);
14493fc155dcSAndrew Turner 	return (err);
14503fc155dcSAndrew Turner }
14513fc155dcSAndrew Turner 
14522b3ad188SAdrian Chadd void dosoftints(void);
14532b3ad188SAdrian Chadd void
14542b3ad188SAdrian Chadd dosoftints(void)
14552b3ad188SAdrian Chadd {
14562b3ad188SAdrian Chadd }
14572b3ad188SAdrian Chadd 
14582b3ad188SAdrian Chadd #ifdef SMP
14592b3ad188SAdrian Chadd /*
14602b3ad188SAdrian Chadd  *  Init interrupt controller on another CPU.
14612b3ad188SAdrian Chadd  */
14622b3ad188SAdrian Chadd void
14632b3ad188SAdrian Chadd intr_pic_init_secondary(void)
14642b3ad188SAdrian Chadd {
14652b3ad188SAdrian Chadd 
14662b3ad188SAdrian Chadd 	/*
14672b3ad188SAdrian Chadd 	 * QQQ: Only root PIC is aware of other CPUs ???
14682b3ad188SAdrian Chadd 	 */
14695b70c08cSSvatopluk Kraus 	KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
14702b3ad188SAdrian Chadd 
14712b3ad188SAdrian Chadd 	//mtx_lock(&isrc_table_lock);
14725b70c08cSSvatopluk Kraus 	PIC_INIT_SECONDARY(intr_irq_root_dev);
14732b3ad188SAdrian Chadd 	//mtx_unlock(&isrc_table_lock);
14742b3ad188SAdrian Chadd }
14752b3ad188SAdrian Chadd #endif
14762b3ad188SAdrian Chadd 
14772b3ad188SAdrian Chadd #ifdef DDB
14782b3ad188SAdrian Chadd DB_SHOW_COMMAND(irqs, db_show_irqs)
14792b3ad188SAdrian Chadd {
14802b3ad188SAdrian Chadd 	u_int i, irqsum;
1481bff6be3eSSvatopluk Kraus 	u_long num;
14822b3ad188SAdrian Chadd 	struct intr_irqsrc *isrc;
14832b3ad188SAdrian Chadd 
14842b3ad188SAdrian Chadd 	for (irqsum = 0, i = 0; i < NIRQ; i++) {
14852b3ad188SAdrian Chadd 		isrc = irq_sources[i];
14862b3ad188SAdrian Chadd 		if (isrc == NULL)
14872b3ad188SAdrian Chadd 			continue;
14882b3ad188SAdrian Chadd 
1489bff6be3eSSvatopluk Kraus 		num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0;
14902b3ad188SAdrian Chadd 		db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i,
14912b3ad188SAdrian Chadd 		    isrc->isrc_name, isrc->isrc_cpu.__bits[0],
1492bff6be3eSSvatopluk Kraus 		    isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num);
1493bff6be3eSSvatopluk Kraus 		irqsum += num;
14942b3ad188SAdrian Chadd 	}
14952b3ad188SAdrian Chadd 	db_printf("irq total %u\n", irqsum);
14962b3ad188SAdrian Chadd }
14972b3ad188SAdrian Chadd #endif
1498895c8b1cSMichal Meloun 
1499895c8b1cSMichal Meloun /*
1500895c8b1cSMichal Meloun  * Interrupt mapping table functions.
1501895c8b1cSMichal Meloun  *
1502895c8b1cSMichal Meloun  * Please, keep this part separately, it can be transformed to
1503895c8b1cSMichal Meloun  * extension of standard resources.
1504895c8b1cSMichal Meloun  */
1505895c8b1cSMichal Meloun struct intr_map_entry
1506895c8b1cSMichal Meloun {
1507895c8b1cSMichal Meloun 	device_t 		dev;
1508895c8b1cSMichal Meloun 	intptr_t 		xref;
1509895c8b1cSMichal Meloun 	struct intr_map_data 	*map_data;
1510895c8b1cSMichal Meloun 	struct intr_irqsrc 	*isrc;
1511895c8b1cSMichal Meloun 	/* XXX TODO DISCONECTED PICs */
1512895c8b1cSMichal Meloun 	/*int			flags */
1513895c8b1cSMichal Meloun };
1514895c8b1cSMichal Meloun 
1515895c8b1cSMichal Meloun /* XXX Convert irq_map[] to dynamicaly expandable one. */
1516895c8b1cSMichal Meloun static struct intr_map_entry *irq_map[2 * NIRQ];
1517895c8b1cSMichal Meloun static int irq_map_count = nitems(irq_map);
1518895c8b1cSMichal Meloun static int irq_map_first_free_idx;
1519895c8b1cSMichal Meloun static struct mtx irq_map_lock;
1520895c8b1cSMichal Meloun 
1521895c8b1cSMichal Meloun static struct intr_irqsrc *
1522895c8b1cSMichal Meloun intr_map_get_isrc(u_int res_id)
1523895c8b1cSMichal Meloun {
1524895c8b1cSMichal Meloun 	struct intr_irqsrc *isrc;
1525895c8b1cSMichal Meloun 
1526ecc8ccb4SAndrew Turner 	isrc = NULL;
1527895c8b1cSMichal Meloun 	mtx_lock(&irq_map_lock);
1528ecc8ccb4SAndrew Turner 	if (res_id < irq_map_count && irq_map[res_id] != NULL)
1529895c8b1cSMichal Meloun 		isrc = irq_map[res_id]->isrc;
1530895c8b1cSMichal Meloun 	mtx_unlock(&irq_map_lock);
1531ecc8ccb4SAndrew Turner 
1532895c8b1cSMichal Meloun 	return (isrc);
1533895c8b1cSMichal Meloun }
1534895c8b1cSMichal Meloun 
1535895c8b1cSMichal Meloun static void
1536895c8b1cSMichal Meloun intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc)
1537895c8b1cSMichal Meloun {
1538895c8b1cSMichal Meloun 
1539895c8b1cSMichal Meloun 	mtx_lock(&irq_map_lock);
1540ecc8ccb4SAndrew Turner 	if (res_id < irq_map_count && irq_map[res_id] != NULL)
1541895c8b1cSMichal Meloun 		irq_map[res_id]->isrc = isrc;
1542895c8b1cSMichal Meloun 	mtx_unlock(&irq_map_lock);
1543895c8b1cSMichal Meloun }
1544895c8b1cSMichal Meloun 
1545895c8b1cSMichal Meloun /*
1546895c8b1cSMichal Meloun  * Get a copy of intr_map_entry data
1547895c8b1cSMichal Meloun  */
1548609b0fe9SOleksandr Tymoshenko static struct intr_map_data *
1549609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(u_int res_id)
1550609b0fe9SOleksandr Tymoshenko {
1551609b0fe9SOleksandr Tymoshenko 	struct intr_map_data *data;
1552609b0fe9SOleksandr Tymoshenko 
1553609b0fe9SOleksandr Tymoshenko 	data = NULL;
1554609b0fe9SOleksandr Tymoshenko 	mtx_lock(&irq_map_lock);
1555609b0fe9SOleksandr Tymoshenko 	if (res_id >= irq_map_count || irq_map[res_id] == NULL)
1556609b0fe9SOleksandr Tymoshenko 		panic("Attempt to copy invalid resource id: %u\n", res_id);
1557609b0fe9SOleksandr Tymoshenko 	data = irq_map[res_id]->map_data;
1558609b0fe9SOleksandr Tymoshenko 	mtx_unlock(&irq_map_lock);
1559609b0fe9SOleksandr Tymoshenko 
1560609b0fe9SOleksandr Tymoshenko 	return (data);
1561609b0fe9SOleksandr Tymoshenko }
1562609b0fe9SOleksandr Tymoshenko 
1563609b0fe9SOleksandr Tymoshenko /*
1564609b0fe9SOleksandr Tymoshenko  * Get a copy of intr_map_entry data
1565609b0fe9SOleksandr Tymoshenko  */
1566895c8b1cSMichal Meloun static void
1567895c8b1cSMichal Meloun intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref,
1568895c8b1cSMichal Meloun     struct intr_map_data **data)
1569895c8b1cSMichal Meloun {
1570895c8b1cSMichal Meloun 	size_t len;
1571895c8b1cSMichal Meloun 
1572895c8b1cSMichal Meloun 	len = 0;
1573895c8b1cSMichal Meloun 	mtx_lock(&irq_map_lock);
1574895c8b1cSMichal Meloun 	if (res_id >= irq_map_count || irq_map[res_id] == NULL)
1575895c8b1cSMichal Meloun 		panic("Attempt to copy invalid resource id: %u\n", res_id);
1576895c8b1cSMichal Meloun 	if (irq_map[res_id]->map_data != NULL)
1577895c8b1cSMichal Meloun 		len = irq_map[res_id]->map_data->len;
1578895c8b1cSMichal Meloun 	mtx_unlock(&irq_map_lock);
1579895c8b1cSMichal Meloun 
1580895c8b1cSMichal Meloun 	if (len == 0)
1581895c8b1cSMichal Meloun 		*data = NULL;
1582895c8b1cSMichal Meloun 	else
1583895c8b1cSMichal Meloun 		*data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO);
1584895c8b1cSMichal Meloun 	mtx_lock(&irq_map_lock);
1585895c8b1cSMichal Meloun 	if (irq_map[res_id] == NULL)
1586895c8b1cSMichal Meloun 		panic("Attempt to copy invalid resource id: %u\n", res_id);
1587895c8b1cSMichal Meloun 	if (len != 0) {
1588895c8b1cSMichal Meloun 		if (len != irq_map[res_id]->map_data->len)
1589895c8b1cSMichal Meloun 			panic("Resource id: %u has changed.\n", res_id);
1590895c8b1cSMichal Meloun 		memcpy(*data, irq_map[res_id]->map_data, len);
1591895c8b1cSMichal Meloun 	}
1592895c8b1cSMichal Meloun 	*map_dev = irq_map[res_id]->dev;
1593895c8b1cSMichal Meloun 	*map_xref = irq_map[res_id]->xref;
1594895c8b1cSMichal Meloun 	mtx_unlock(&irq_map_lock);
1595895c8b1cSMichal Meloun }
1596895c8b1cSMichal Meloun 
1597895c8b1cSMichal Meloun /*
1598895c8b1cSMichal Meloun  * Allocate and fill new entry in irq_map table.
1599895c8b1cSMichal Meloun  */
1600895c8b1cSMichal Meloun u_int
1601895c8b1cSMichal Meloun intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data)
1602895c8b1cSMichal Meloun {
1603895c8b1cSMichal Meloun 	u_int i;
1604895c8b1cSMichal Meloun 	struct intr_map_entry *entry;
1605895c8b1cSMichal Meloun 
1606895c8b1cSMichal Meloun 	/* Prepare new entry first. */
1607895c8b1cSMichal Meloun 	entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO);
1608895c8b1cSMichal Meloun 
1609895c8b1cSMichal Meloun 	entry->dev = dev;
1610895c8b1cSMichal Meloun 	entry->xref = xref;
1611895c8b1cSMichal Meloun 	entry->map_data = data;
1612895c8b1cSMichal Meloun 	entry->isrc = NULL;
1613895c8b1cSMichal Meloun 
1614895c8b1cSMichal Meloun 	mtx_lock(&irq_map_lock);
1615895c8b1cSMichal Meloun 	for (i = irq_map_first_free_idx; i < irq_map_count; i++) {
1616895c8b1cSMichal Meloun 		if (irq_map[i] == NULL) {
1617895c8b1cSMichal Meloun 			irq_map[i] = entry;
1618895c8b1cSMichal Meloun 			irq_map_first_free_idx = i + 1;
1619895c8b1cSMichal Meloun 			mtx_unlock(&irq_map_lock);
1620895c8b1cSMichal Meloun 			return (i);
1621895c8b1cSMichal Meloun 		}
1622895c8b1cSMichal Meloun 	}
1623895c8b1cSMichal Meloun 	mtx_unlock(&irq_map_lock);
1624895c8b1cSMichal Meloun 
1625895c8b1cSMichal Meloun 	/* XXX Expand irq_map table */
1626895c8b1cSMichal Meloun 	panic("IRQ mapping table is full.");
1627895c8b1cSMichal Meloun }
1628895c8b1cSMichal Meloun 
1629895c8b1cSMichal Meloun /*
1630895c8b1cSMichal Meloun  * Remove and free mapping entry.
1631895c8b1cSMichal Meloun  */
1632895c8b1cSMichal Meloun void
1633895c8b1cSMichal Meloun intr_unmap_irq(u_int res_id)
1634895c8b1cSMichal Meloun {
1635895c8b1cSMichal Meloun 	struct intr_map_entry *entry;
1636895c8b1cSMichal Meloun 
1637895c8b1cSMichal Meloun 	mtx_lock(&irq_map_lock);
1638895c8b1cSMichal Meloun 	if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL))
1639895c8b1cSMichal Meloun 		panic("Attempt to unmap invalid resource id: %u\n", res_id);
1640895c8b1cSMichal Meloun 	entry = irq_map[res_id];
1641895c8b1cSMichal Meloun 	irq_map[res_id] = NULL;
1642895c8b1cSMichal Meloun 	irq_map_first_free_idx = res_id;
1643895c8b1cSMichal Meloun 	mtx_unlock(&irq_map_lock);
1644895c8b1cSMichal Meloun 	intr_free_intr_map_data(entry->map_data);
1645895c8b1cSMichal Meloun 	free(entry, M_INTRNG);
1646895c8b1cSMichal Meloun }
1647895c8b1cSMichal Meloun 
1648895c8b1cSMichal Meloun /*
1649895c8b1cSMichal Meloun  * Clone mapping entry.
1650895c8b1cSMichal Meloun  */
1651895c8b1cSMichal Meloun u_int
1652895c8b1cSMichal Meloun intr_map_clone_irq(u_int old_res_id)
1653895c8b1cSMichal Meloun {
1654895c8b1cSMichal Meloun 	device_t map_dev;
1655895c8b1cSMichal Meloun 	intptr_t map_xref;
1656895c8b1cSMichal Meloun 	struct intr_map_data *data;
1657895c8b1cSMichal Meloun 
1658895c8b1cSMichal Meloun 	intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data);
1659895c8b1cSMichal Meloun 	return (intr_map_irq(map_dev, map_xref, data));
1660895c8b1cSMichal Meloun }
1661895c8b1cSMichal Meloun 
1662895c8b1cSMichal Meloun static void
1663895c8b1cSMichal Meloun intr_map_init(void *dummy __unused)
1664895c8b1cSMichal Meloun {
1665895c8b1cSMichal Meloun 
1666895c8b1cSMichal Meloun 	mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF);
1667895c8b1cSMichal Meloun }
1668895c8b1cSMichal Meloun SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL);
1669