12b3ad188SAdrian Chadd /*- 2bff6be3eSSvatopluk Kraus * Copyright (c) 2015-2016 Svatopluk Kraus 3bff6be3eSSvatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun 42b3ad188SAdrian Chadd * All rights reserved. 52b3ad188SAdrian Chadd * 62b3ad188SAdrian Chadd * Redistribution and use in source and binary forms, with or without 72b3ad188SAdrian Chadd * modification, are permitted provided that the following conditions 82b3ad188SAdrian Chadd * are met: 92b3ad188SAdrian Chadd * 1. Redistributions of source code must retain the above copyright 102b3ad188SAdrian Chadd * notice, this list of conditions and the following disclaimer. 112b3ad188SAdrian Chadd * 2. Redistributions in binary form must reproduce the above copyright 122b3ad188SAdrian Chadd * notice, this list of conditions and the following disclaimer in the 132b3ad188SAdrian Chadd * documentation and/or other materials provided with the distribution. 142b3ad188SAdrian Chadd * 152b3ad188SAdrian Chadd * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 162b3ad188SAdrian Chadd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 172b3ad188SAdrian Chadd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 182b3ad188SAdrian Chadd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 192b3ad188SAdrian Chadd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 202b3ad188SAdrian Chadd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 212b3ad188SAdrian Chadd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 222b3ad188SAdrian Chadd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 232b3ad188SAdrian Chadd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 242b3ad188SAdrian Chadd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 252b3ad188SAdrian Chadd * SUCH DAMAGE. 262b3ad188SAdrian Chadd */ 272b3ad188SAdrian Chadd 282b3ad188SAdrian Chadd #include <sys/cdefs.h> 292b3ad188SAdrian Chadd __FBSDID("$FreeBSD$"); 302b3ad188SAdrian Chadd 312b3ad188SAdrian Chadd /* 322b3ad188SAdrian Chadd * New-style Interrupt Framework 332b3ad188SAdrian Chadd * 34895c8b1cSMichal Meloun * TODO: - add support for disconnected PICs. 35895c8b1cSMichal Meloun * - to support IPI (PPI) enabling on other CPUs if already started. 36895c8b1cSMichal Meloun * - to complete things for removable PICs. 372b3ad188SAdrian Chadd */ 382b3ad188SAdrian Chadd 392b3ad188SAdrian Chadd #include "opt_ddb.h" 40df7a2251SAndrew Turner #include "opt_hwpmc_hooks.h" 412b3ad188SAdrian Chadd 422b3ad188SAdrian Chadd #include <sys/param.h> 432b3ad188SAdrian Chadd #include <sys/systm.h> 442b3ad188SAdrian Chadd #include <sys/kernel.h> 45e2e050c8SConrad Meyer #include <sys/lock.h> 46e2e050c8SConrad Meyer #include <sys/mutex.h> 472b3ad188SAdrian Chadd #include <sys/syslog.h> 482b3ad188SAdrian Chadd #include <sys/malloc.h> 492b3ad188SAdrian Chadd #include <sys/proc.h> 502b3ad188SAdrian Chadd #include <sys/queue.h> 512b3ad188SAdrian Chadd #include <sys/bus.h> 522b3ad188SAdrian Chadd #include <sys/interrupt.h> 532b3ad188SAdrian Chadd #include <sys/conf.h> 542b3ad188SAdrian Chadd #include <sys/cpuset.h> 556b42a1f4SAndrew Turner #include <sys/rman.h> 562b3ad188SAdrian Chadd #include <sys/sched.h> 572b3ad188SAdrian Chadd #include <sys/smp.h> 589ed01c32SGleb Smirnoff #include <sys/vmmeter.h> 59df7a2251SAndrew Turner #ifdef HWPMC_HOOKS 60df7a2251SAndrew Turner #include <sys/pmckern.h> 61df7a2251SAndrew Turner #endif 62df7a2251SAndrew Turner 632b3ad188SAdrian Chadd #include <machine/atomic.h> 642b3ad188SAdrian Chadd #include <machine/intr.h> 652b3ad188SAdrian Chadd #include <machine/cpu.h> 662b3ad188SAdrian Chadd #include <machine/smp.h> 672b3ad188SAdrian Chadd #include <machine/stdarg.h> 682b3ad188SAdrian Chadd 692b3ad188SAdrian Chadd #ifdef DDB 702b3ad188SAdrian Chadd #include <ddb/ddb.h> 712b3ad188SAdrian Chadd #endif 722b3ad188SAdrian Chadd 732b3ad188SAdrian Chadd #include "pic_if.h" 743fc155dcSAndrew Turner #include "msi_if.h" 752b3ad188SAdrian Chadd 762b3ad188SAdrian Chadd #define INTRNAME_LEN (2*MAXCOMLEN + 1) 772b3ad188SAdrian Chadd 782b3ad188SAdrian Chadd #ifdef DEBUG 792b3ad188SAdrian Chadd #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 802b3ad188SAdrian Chadd printf(fmt,##args); } while (0) 812b3ad188SAdrian Chadd #else 822b3ad188SAdrian Chadd #define debugf(fmt, args...) 832b3ad188SAdrian Chadd #endif 842b3ad188SAdrian Chadd 852b3ad188SAdrian Chadd MALLOC_DECLARE(M_INTRNG); 862b3ad188SAdrian Chadd MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 872b3ad188SAdrian Chadd 882b3ad188SAdrian Chadd /* Main interrupt handler called from assembler -> 'hidden' for C code. */ 892b3ad188SAdrian Chadd void intr_irq_handler(struct trapframe *tf); 902b3ad188SAdrian Chadd 912b3ad188SAdrian Chadd /* Root interrupt controller stuff. */ 925b70c08cSSvatopluk Kraus device_t intr_irq_root_dev; 932b3ad188SAdrian Chadd static intr_irq_filter_t *irq_root_filter; 942b3ad188SAdrian Chadd static void *irq_root_arg; 952b3ad188SAdrian Chadd static u_int irq_root_ipicount; 962b3ad188SAdrian Chadd 97d1605cdaSAndrew Turner struct intr_pic_child { 98d1605cdaSAndrew Turner SLIST_ENTRY(intr_pic_child) pc_next; 99d1605cdaSAndrew Turner struct intr_pic *pc_pic; 100d1605cdaSAndrew Turner intr_child_irq_filter_t *pc_filter; 101d1605cdaSAndrew Turner void *pc_filter_arg; 102d1605cdaSAndrew Turner uintptr_t pc_start; 103d1605cdaSAndrew Turner uintptr_t pc_length; 104d1605cdaSAndrew Turner }; 105d1605cdaSAndrew Turner 1062b3ad188SAdrian Chadd /* Interrupt controller definition. */ 1072b3ad188SAdrian Chadd struct intr_pic { 1082b3ad188SAdrian Chadd SLIST_ENTRY(intr_pic) pic_next; 1092b3ad188SAdrian Chadd intptr_t pic_xref; /* hardware identification */ 1102b3ad188SAdrian Chadd device_t pic_dev; 111c0d52370SAndrew Turner /* Only one of FLAG_PIC or FLAG_MSI may be set */ 1123fc155dcSAndrew Turner #define FLAG_PIC (1 << 0) 1133fc155dcSAndrew Turner #define FLAG_MSI (1 << 1) 114c0d52370SAndrew Turner #define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) 1153fc155dcSAndrew Turner u_int pic_flags; 116d1605cdaSAndrew Turner struct mtx pic_child_lock; 117d1605cdaSAndrew Turner SLIST_HEAD(, intr_pic_child) pic_children; 1182b3ad188SAdrian Chadd }; 1192b3ad188SAdrian Chadd 1202b3ad188SAdrian Chadd static struct mtx pic_list_lock; 1212b3ad188SAdrian Chadd static SLIST_HEAD(, intr_pic) pic_list; 1222b3ad188SAdrian Chadd 123c0d52370SAndrew Turner static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, int flags); 1242b3ad188SAdrian Chadd 1252b3ad188SAdrian Chadd /* Interrupt source definition. */ 1262b3ad188SAdrian Chadd static struct mtx isrc_table_lock; 1272b3ad188SAdrian Chadd static struct intr_irqsrc *irq_sources[NIRQ]; 1282b3ad188SAdrian Chadd u_int irq_next_free; 1292b3ad188SAdrian Chadd 1302b3ad188SAdrian Chadd #ifdef SMP 131*dc425090SMitchell Horne #ifdef EARLY_AP_STARTUP 132*dc425090SMitchell Horne static bool irq_assign_cpu = true; 133*dc425090SMitchell Horne #else 134*dc425090SMitchell Horne static bool irq_assign_cpu = false; 135*dc425090SMitchell Horne #endif 1362b3ad188SAdrian Chadd #endif 1372b3ad188SAdrian Chadd 1382b3ad188SAdrian Chadd /* 1392b3ad188SAdrian Chadd * - 2 counters for each I/O interrupt. 1402b3ad188SAdrian Chadd * - MAXCPU counters for each IPI counters for SMP. 1412b3ad188SAdrian Chadd */ 1422b3ad188SAdrian Chadd #ifdef SMP 1432b3ad188SAdrian Chadd #define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) 1442b3ad188SAdrian Chadd #else 1452b3ad188SAdrian Chadd #define INTRCNT_COUNT (NIRQ * 2) 1462b3ad188SAdrian Chadd #endif 1472b3ad188SAdrian Chadd 1482b3ad188SAdrian Chadd /* Data for MI statistics reporting. */ 1492b3ad188SAdrian Chadd u_long intrcnt[INTRCNT_COUNT]; 1502b3ad188SAdrian Chadd char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; 1512b3ad188SAdrian Chadd size_t sintrcnt = sizeof(intrcnt); 1522b3ad188SAdrian Chadd size_t sintrnames = sizeof(intrnames); 1532b3ad188SAdrian Chadd static u_int intrcnt_index; 1542b3ad188SAdrian Chadd 155895c8b1cSMichal Meloun static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); 156895c8b1cSMichal Meloun static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); 157609b0fe9SOleksandr Tymoshenko static struct intr_map_data * intr_map_get_map_data(u_int res_id); 158895c8b1cSMichal Meloun static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, 159895c8b1cSMichal Meloun struct intr_map_data **data); 160895c8b1cSMichal Meloun 1612b3ad188SAdrian Chadd /* 1622b3ad188SAdrian Chadd * Interrupt framework initialization routine. 1632b3ad188SAdrian Chadd */ 1642b3ad188SAdrian Chadd static void 1652b3ad188SAdrian Chadd intr_irq_init(void *dummy __unused) 1662b3ad188SAdrian Chadd { 1672b3ad188SAdrian Chadd 1682b3ad188SAdrian Chadd SLIST_INIT(&pic_list); 1692b3ad188SAdrian Chadd mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 1703fc155dcSAndrew Turner 1712b3ad188SAdrian Chadd mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 1722b3ad188SAdrian Chadd } 1732b3ad188SAdrian Chadd SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 1742b3ad188SAdrian Chadd 1752b3ad188SAdrian Chadd static void 1762b3ad188SAdrian Chadd intrcnt_setname(const char *name, int index) 1772b3ad188SAdrian Chadd { 1782b3ad188SAdrian Chadd 1792b3ad188SAdrian Chadd snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 1802b3ad188SAdrian Chadd INTRNAME_LEN - 1, name); 1812b3ad188SAdrian Chadd } 1822b3ad188SAdrian Chadd 1832b3ad188SAdrian Chadd /* 1842b3ad188SAdrian Chadd * Update name for interrupt source with interrupt event. 1852b3ad188SAdrian Chadd */ 1862b3ad188SAdrian Chadd static void 1872b3ad188SAdrian Chadd intrcnt_updatename(struct intr_irqsrc *isrc) 1882b3ad188SAdrian Chadd { 1892b3ad188SAdrian Chadd 1902b3ad188SAdrian Chadd /* QQQ: What about stray counter name? */ 1912b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 1922b3ad188SAdrian Chadd intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 1932b3ad188SAdrian Chadd } 1942b3ad188SAdrian Chadd 1952b3ad188SAdrian Chadd /* 1962b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt counter increment. 1972b3ad188SAdrian Chadd */ 1982b3ad188SAdrian Chadd static inline void 1992b3ad188SAdrian Chadd isrc_increment_count(struct intr_irqsrc *isrc) 2002b3ad188SAdrian Chadd { 2012b3ad188SAdrian Chadd 202bff6be3eSSvatopluk Kraus if (isrc->isrc_flags & INTR_ISRCF_PPI) 203bff6be3eSSvatopluk Kraus atomic_add_long(&isrc->isrc_count[0], 1); 204bff6be3eSSvatopluk Kraus else 2052b3ad188SAdrian Chadd isrc->isrc_count[0]++; 2062b3ad188SAdrian Chadd } 2072b3ad188SAdrian Chadd 2082b3ad188SAdrian Chadd /* 2092b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt stray counter increment. 2102b3ad188SAdrian Chadd */ 2112b3ad188SAdrian Chadd static inline void 2122b3ad188SAdrian Chadd isrc_increment_straycount(struct intr_irqsrc *isrc) 2132b3ad188SAdrian Chadd { 2142b3ad188SAdrian Chadd 2152b3ad188SAdrian Chadd isrc->isrc_count[1]++; 2162b3ad188SAdrian Chadd } 2172b3ad188SAdrian Chadd 2182b3ad188SAdrian Chadd /* 2192b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt name update. 2202b3ad188SAdrian Chadd */ 2212b3ad188SAdrian Chadd static void 2222b3ad188SAdrian Chadd isrc_update_name(struct intr_irqsrc *isrc, const char *name) 2232b3ad188SAdrian Chadd { 2242b3ad188SAdrian Chadd char str[INTRNAME_LEN]; 2252b3ad188SAdrian Chadd 2262b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 2272b3ad188SAdrian Chadd 2282b3ad188SAdrian Chadd if (name != NULL) { 2292b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 2302b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index); 2312b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 2322b3ad188SAdrian Chadd name); 2332b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index + 1); 2342b3ad188SAdrian Chadd } else { 2352b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 2362b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index); 2372b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 2382b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index + 1); 2392b3ad188SAdrian Chadd } 2402b3ad188SAdrian Chadd } 2412b3ad188SAdrian Chadd 2422b3ad188SAdrian Chadd /* 2432b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt counters setup. 2442b3ad188SAdrian Chadd */ 2452b3ad188SAdrian Chadd static void 2462b3ad188SAdrian Chadd isrc_setup_counters(struct intr_irqsrc *isrc) 2472b3ad188SAdrian Chadd { 2482b3ad188SAdrian Chadd u_int index; 2492b3ad188SAdrian Chadd 2502b3ad188SAdrian Chadd /* 2512b3ad188SAdrian Chadd * XXX - it does not work well with removable controllers and 2522b3ad188SAdrian Chadd * interrupt sources !!! 2532b3ad188SAdrian Chadd */ 2542b3ad188SAdrian Chadd index = atomic_fetchadd_int(&intrcnt_index, 2); 2552b3ad188SAdrian Chadd isrc->isrc_index = index; 2562b3ad188SAdrian Chadd isrc->isrc_count = &intrcnt[index]; 2572b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 2582b3ad188SAdrian Chadd } 2592b3ad188SAdrian Chadd 260bff6be3eSSvatopluk Kraus /* 261bff6be3eSSvatopluk Kraus * Virtualization for interrupt source interrupt counters release. 262bff6be3eSSvatopluk Kraus */ 263bff6be3eSSvatopluk Kraus static void 264bff6be3eSSvatopluk Kraus isrc_release_counters(struct intr_irqsrc *isrc) 265bff6be3eSSvatopluk Kraus { 266bff6be3eSSvatopluk Kraus 267bff6be3eSSvatopluk Kraus panic("%s: not implemented", __func__); 268bff6be3eSSvatopluk Kraus } 269bff6be3eSSvatopluk Kraus 2702b3ad188SAdrian Chadd #ifdef SMP 2712b3ad188SAdrian Chadd /* 2722b3ad188SAdrian Chadd * Virtualization for interrupt source IPI counters setup. 2732b3ad188SAdrian Chadd */ 2745b70c08cSSvatopluk Kraus u_long * 2755b70c08cSSvatopluk Kraus intr_ipi_setup_counters(const char *name) 2762b3ad188SAdrian Chadd { 2772b3ad188SAdrian Chadd u_int index, i; 2782b3ad188SAdrian Chadd char str[INTRNAME_LEN]; 2792b3ad188SAdrian Chadd 2802b3ad188SAdrian Chadd index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); 2812b3ad188SAdrian Chadd for (i = 0; i < MAXCPU; i++) { 2822b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 2832b3ad188SAdrian Chadd intrcnt_setname(str, index + i); 2842b3ad188SAdrian Chadd } 2855b70c08cSSvatopluk Kraus return (&intrcnt[index]); 2862b3ad188SAdrian Chadd } 2872b3ad188SAdrian Chadd #endif 2882b3ad188SAdrian Chadd 2892b3ad188SAdrian Chadd /* 2902b3ad188SAdrian Chadd * Main interrupt dispatch handler. It's called straight 2912b3ad188SAdrian Chadd * from the assembler, where CPU interrupt is served. 2922b3ad188SAdrian Chadd */ 2932b3ad188SAdrian Chadd void 2942b3ad188SAdrian Chadd intr_irq_handler(struct trapframe *tf) 2952b3ad188SAdrian Chadd { 2962b3ad188SAdrian Chadd struct trapframe * oldframe; 2972b3ad188SAdrian Chadd struct thread * td; 2982b3ad188SAdrian Chadd 2992b3ad188SAdrian Chadd KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); 3002b3ad188SAdrian Chadd 30183c9dea1SGleb Smirnoff VM_CNT_INC(v_intr); 3022b3ad188SAdrian Chadd critical_enter(); 3032b3ad188SAdrian Chadd td = curthread; 3042b3ad188SAdrian Chadd oldframe = td->td_intr_frame; 3052b3ad188SAdrian Chadd td->td_intr_frame = tf; 3062b3ad188SAdrian Chadd irq_root_filter(irq_root_arg); 3072b3ad188SAdrian Chadd td->td_intr_frame = oldframe; 3082b3ad188SAdrian Chadd critical_exit(); 309df7a2251SAndrew Turner #ifdef HWPMC_HOOKS 310974692e3SAndrew Turner if (pmc_hook && TRAPF_USERMODE(tf) && 311974692e3SAndrew Turner (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) 312df7a2251SAndrew Turner pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); 313df7a2251SAndrew Turner #endif 3142b3ad188SAdrian Chadd } 3152b3ad188SAdrian Chadd 316d1605cdaSAndrew Turner int 317d1605cdaSAndrew Turner intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) 318d1605cdaSAndrew Turner { 319d1605cdaSAndrew Turner struct intr_pic_child *child; 320d1605cdaSAndrew Turner bool found; 321d1605cdaSAndrew Turner 322d1605cdaSAndrew Turner found = false; 323d1605cdaSAndrew Turner mtx_lock_spin(&parent->pic_child_lock); 324d1605cdaSAndrew Turner SLIST_FOREACH(child, &parent->pic_children, pc_next) { 325d1605cdaSAndrew Turner if (child->pc_start <= irq && 326d1605cdaSAndrew Turner irq < (child->pc_start + child->pc_length)) { 327d1605cdaSAndrew Turner found = true; 328d1605cdaSAndrew Turner break; 329d1605cdaSAndrew Turner } 330d1605cdaSAndrew Turner } 331d1605cdaSAndrew Turner mtx_unlock_spin(&parent->pic_child_lock); 332d1605cdaSAndrew Turner 333d1605cdaSAndrew Turner if (found) 334d1605cdaSAndrew Turner return (child->pc_filter(child->pc_filter_arg, irq)); 335d1605cdaSAndrew Turner 336d1605cdaSAndrew Turner return (FILTER_STRAY); 337d1605cdaSAndrew Turner } 338d1605cdaSAndrew Turner 3392b3ad188SAdrian Chadd /* 3402b3ad188SAdrian Chadd * interrupt controller dispatch function for interrupts. It should 3412b3ad188SAdrian Chadd * be called straight from the interrupt controller, when associated interrupt 3422b3ad188SAdrian Chadd * source is learned. 3432b3ad188SAdrian Chadd */ 344bff6be3eSSvatopluk Kraus int 345bff6be3eSSvatopluk Kraus intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 3462b3ad188SAdrian Chadd { 3472b3ad188SAdrian Chadd 3482b3ad188SAdrian Chadd KASSERT(isrc != NULL, ("%s: no source", __func__)); 3492b3ad188SAdrian Chadd 3502b3ad188SAdrian Chadd isrc_increment_count(isrc); 3512b3ad188SAdrian Chadd 3522b3ad188SAdrian Chadd #ifdef INTR_SOLO 3532b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 3542b3ad188SAdrian Chadd int error; 3552b3ad188SAdrian Chadd error = isrc->isrc_filter(isrc->isrc_arg, tf); 3562b3ad188SAdrian Chadd PIC_POST_FILTER(isrc->isrc_dev, isrc); 3572b3ad188SAdrian Chadd if (error == FILTER_HANDLED) 358bff6be3eSSvatopluk Kraus return (0); 3592b3ad188SAdrian Chadd } else 3602b3ad188SAdrian Chadd #endif 3612b3ad188SAdrian Chadd if (isrc->isrc_event != NULL) { 3622b3ad188SAdrian Chadd if (intr_event_handle(isrc->isrc_event, tf) == 0) 363bff6be3eSSvatopluk Kraus return (0); 3642b3ad188SAdrian Chadd } 3652b3ad188SAdrian Chadd 3662b3ad188SAdrian Chadd isrc_increment_straycount(isrc); 367bff6be3eSSvatopluk Kraus return (EINVAL); 3682b3ad188SAdrian Chadd } 3692b3ad188SAdrian Chadd 3702b3ad188SAdrian Chadd /* 3712b3ad188SAdrian Chadd * Alloc unique interrupt number (resource handle) for interrupt source. 3722b3ad188SAdrian Chadd * 3732b3ad188SAdrian Chadd * There could be various strategies how to allocate free interrupt number 3742b3ad188SAdrian Chadd * (resource handle) for new interrupt source. 3752b3ad188SAdrian Chadd * 3762b3ad188SAdrian Chadd * 1. Handles are always allocated forward, so handles are not recycled 3772b3ad188SAdrian Chadd * immediately. However, if only one free handle left which is reused 3782b3ad188SAdrian Chadd * constantly... 3792b3ad188SAdrian Chadd */ 380bff6be3eSSvatopluk Kraus static inline int 381bff6be3eSSvatopluk Kraus isrc_alloc_irq(struct intr_irqsrc *isrc) 3822b3ad188SAdrian Chadd { 3832b3ad188SAdrian Chadd u_int maxirqs, irq; 3842b3ad188SAdrian Chadd 3852b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 3862b3ad188SAdrian Chadd 3872b3ad188SAdrian Chadd maxirqs = nitems(irq_sources); 3882b3ad188SAdrian Chadd if (irq_next_free >= maxirqs) 3892b3ad188SAdrian Chadd return (ENOSPC); 3902b3ad188SAdrian Chadd 3912b3ad188SAdrian Chadd for (irq = irq_next_free; irq < maxirqs; irq++) { 3922b3ad188SAdrian Chadd if (irq_sources[irq] == NULL) 3932b3ad188SAdrian Chadd goto found; 3942b3ad188SAdrian Chadd } 3952b3ad188SAdrian Chadd for (irq = 0; irq < irq_next_free; irq++) { 3962b3ad188SAdrian Chadd if (irq_sources[irq] == NULL) 3972b3ad188SAdrian Chadd goto found; 3982b3ad188SAdrian Chadd } 3992b3ad188SAdrian Chadd 4002b3ad188SAdrian Chadd irq_next_free = maxirqs; 4012b3ad188SAdrian Chadd return (ENOSPC); 4022b3ad188SAdrian Chadd 4032b3ad188SAdrian Chadd found: 4042b3ad188SAdrian Chadd isrc->isrc_irq = irq; 4052b3ad188SAdrian Chadd irq_sources[irq] = isrc; 4062b3ad188SAdrian Chadd 4072b3ad188SAdrian Chadd irq_next_free = irq + 1; 4082b3ad188SAdrian Chadd if (irq_next_free >= maxirqs) 4092b3ad188SAdrian Chadd irq_next_free = 0; 4102b3ad188SAdrian Chadd return (0); 4112b3ad188SAdrian Chadd } 412bff6be3eSSvatopluk Kraus 4132b3ad188SAdrian Chadd /* 4142b3ad188SAdrian Chadd * Free unique interrupt number (resource handle) from interrupt source. 4152b3ad188SAdrian Chadd */ 416bff6be3eSSvatopluk Kraus static inline int 4172b3ad188SAdrian Chadd isrc_free_irq(struct intr_irqsrc *isrc) 4182b3ad188SAdrian Chadd { 4192b3ad188SAdrian Chadd 420bff6be3eSSvatopluk Kraus mtx_assert(&isrc_table_lock, MA_OWNED); 4212b3ad188SAdrian Chadd 422bff6be3eSSvatopluk Kraus if (isrc->isrc_irq >= nitems(irq_sources)) 4232b3ad188SAdrian Chadd return (EINVAL); 424bff6be3eSSvatopluk Kraus if (irq_sources[isrc->isrc_irq] != isrc) 4252b3ad188SAdrian Chadd return (EINVAL); 4262b3ad188SAdrian Chadd 4272b3ad188SAdrian Chadd irq_sources[isrc->isrc_irq] = NULL; 4288442087fSMichal Meloun isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 4292b3ad188SAdrian Chadd return (0); 4302b3ad188SAdrian Chadd } 431bff6be3eSSvatopluk Kraus 4322b3ad188SAdrian Chadd /* 433bff6be3eSSvatopluk Kraus * Initialize interrupt source and register it into global interrupt table. 4342b3ad188SAdrian Chadd */ 435bff6be3eSSvatopluk Kraus int 436bff6be3eSSvatopluk Kraus intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, 437bff6be3eSSvatopluk Kraus const char *fmt, ...) 4382b3ad188SAdrian Chadd { 439bff6be3eSSvatopluk Kraus int error; 440bff6be3eSSvatopluk Kraus va_list ap; 4412b3ad188SAdrian Chadd 442bff6be3eSSvatopluk Kraus bzero(isrc, sizeof(struct intr_irqsrc)); 443bff6be3eSSvatopluk Kraus isrc->isrc_dev = dev; 4448442087fSMichal Meloun isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 445bff6be3eSSvatopluk Kraus isrc->isrc_flags = flags; 4462b3ad188SAdrian Chadd 447bff6be3eSSvatopluk Kraus va_start(ap, fmt); 448bff6be3eSSvatopluk Kraus vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 449bff6be3eSSvatopluk Kraus va_end(ap); 450bff6be3eSSvatopluk Kraus 451bff6be3eSSvatopluk Kraus mtx_lock(&isrc_table_lock); 452bff6be3eSSvatopluk Kraus error = isrc_alloc_irq(isrc); 453bff6be3eSSvatopluk Kraus if (error != 0) { 454bff6be3eSSvatopluk Kraus mtx_unlock(&isrc_table_lock); 455bff6be3eSSvatopluk Kraus return (error); 4562b3ad188SAdrian Chadd } 457bff6be3eSSvatopluk Kraus /* 458bff6be3eSSvatopluk Kraus * Setup interrupt counters, but not for IPI sources. Those are setup 459bff6be3eSSvatopluk Kraus * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust 460bff6be3eSSvatopluk Kraus * our counter pool. 461bff6be3eSSvatopluk Kraus */ 462bff6be3eSSvatopluk Kraus if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 463bff6be3eSSvatopluk Kraus isrc_setup_counters(isrc); 464bff6be3eSSvatopluk Kraus mtx_unlock(&isrc_table_lock); 465bff6be3eSSvatopluk Kraus return (0); 4662b3ad188SAdrian Chadd } 4672b3ad188SAdrian Chadd 4682b3ad188SAdrian Chadd /* 469bff6be3eSSvatopluk Kraus * Deregister interrupt source from global interrupt table. 470bff6be3eSSvatopluk Kraus */ 471bff6be3eSSvatopluk Kraus int 472bff6be3eSSvatopluk Kraus intr_isrc_deregister(struct intr_irqsrc *isrc) 473bff6be3eSSvatopluk Kraus { 474bff6be3eSSvatopluk Kraus int error; 475bff6be3eSSvatopluk Kraus 476bff6be3eSSvatopluk Kraus mtx_lock(&isrc_table_lock); 477bff6be3eSSvatopluk Kraus if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 478bff6be3eSSvatopluk Kraus isrc_release_counters(isrc); 479bff6be3eSSvatopluk Kraus error = isrc_free_irq(isrc); 480bff6be3eSSvatopluk Kraus mtx_unlock(&isrc_table_lock); 481bff6be3eSSvatopluk Kraus return (error); 482bff6be3eSSvatopluk Kraus } 483bff6be3eSSvatopluk Kraus 4845b613c19SSvatopluk Kraus #ifdef SMP 4855b613c19SSvatopluk Kraus /* 4865b613c19SSvatopluk Kraus * A support function for a PIC to decide if provided ISRC should be inited 4875b613c19SSvatopluk Kraus * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of 4885b613c19SSvatopluk Kraus * struct intr_irqsrc is the following: 4895b613c19SSvatopluk Kraus * 4905b613c19SSvatopluk Kraus * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus 4915b613c19SSvatopluk Kraus * set in isrc_cpu. If not, the ISRC should be inited on every cpu and 4925b613c19SSvatopluk Kraus * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. 4935b613c19SSvatopluk Kraus */ 4945b613c19SSvatopluk Kraus bool 4955b613c19SSvatopluk Kraus intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) 4965b613c19SSvatopluk Kraus { 4975b613c19SSvatopluk Kraus 4985b613c19SSvatopluk Kraus if (isrc->isrc_handlers == 0) 4995b613c19SSvatopluk Kraus return (false); 5005b613c19SSvatopluk Kraus if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) 5015b613c19SSvatopluk Kraus return (false); 5025b613c19SSvatopluk Kraus if (isrc->isrc_flags & INTR_ISRCF_BOUND) 5035b613c19SSvatopluk Kraus return (CPU_ISSET(cpu, &isrc->isrc_cpu)); 5045b613c19SSvatopluk Kraus 5055b613c19SSvatopluk Kraus CPU_SET(cpu, &isrc->isrc_cpu); 5065b613c19SSvatopluk Kraus return (true); 5075b613c19SSvatopluk Kraus } 5085b613c19SSvatopluk Kraus #endif 5095b613c19SSvatopluk Kraus 5102b3ad188SAdrian Chadd #ifdef INTR_SOLO 5112b3ad188SAdrian Chadd /* 5122b3ad188SAdrian Chadd * Setup filter into interrupt source. 5132b3ad188SAdrian Chadd */ 5142b3ad188SAdrian Chadd static int 5152b3ad188SAdrian Chadd iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 5162b3ad188SAdrian Chadd intr_irq_filter_t *filter, void *arg, void **cookiep) 5172b3ad188SAdrian Chadd { 5182b3ad188SAdrian Chadd 5192b3ad188SAdrian Chadd if (filter == NULL) 5202b3ad188SAdrian Chadd return (EINVAL); 5212b3ad188SAdrian Chadd 5222b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 5232b3ad188SAdrian Chadd /* 5242b3ad188SAdrian Chadd * Make sure that we do not mix the two ways 5252b3ad188SAdrian Chadd * how we handle interrupt sources. 5262b3ad188SAdrian Chadd */ 5272b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 5282b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 5292b3ad188SAdrian Chadd return (EBUSY); 5302b3ad188SAdrian Chadd } 5312b3ad188SAdrian Chadd isrc->isrc_filter = filter; 5322b3ad188SAdrian Chadd isrc->isrc_arg = arg; 5332b3ad188SAdrian Chadd isrc_update_name(isrc, name); 5342b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 5352b3ad188SAdrian Chadd 5362b3ad188SAdrian Chadd *cookiep = isrc; 5372b3ad188SAdrian Chadd return (0); 5382b3ad188SAdrian Chadd } 5392b3ad188SAdrian Chadd #endif 5402b3ad188SAdrian Chadd 5412b3ad188SAdrian Chadd /* 5422b3ad188SAdrian Chadd * Interrupt source pre_ithread method for MI interrupt framework. 5432b3ad188SAdrian Chadd */ 5442b3ad188SAdrian Chadd static void 5452b3ad188SAdrian Chadd intr_isrc_pre_ithread(void *arg) 5462b3ad188SAdrian Chadd { 5472b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 5482b3ad188SAdrian Chadd 5492b3ad188SAdrian Chadd PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 5502b3ad188SAdrian Chadd } 5512b3ad188SAdrian Chadd 5522b3ad188SAdrian Chadd /* 5532b3ad188SAdrian Chadd * Interrupt source post_ithread method for MI interrupt framework. 5542b3ad188SAdrian Chadd */ 5552b3ad188SAdrian Chadd static void 5562b3ad188SAdrian Chadd intr_isrc_post_ithread(void *arg) 5572b3ad188SAdrian Chadd { 5582b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 5592b3ad188SAdrian Chadd 5602b3ad188SAdrian Chadd PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 5612b3ad188SAdrian Chadd } 5622b3ad188SAdrian Chadd 5632b3ad188SAdrian Chadd /* 5642b3ad188SAdrian Chadd * Interrupt source post_filter method for MI interrupt framework. 5652b3ad188SAdrian Chadd */ 5662b3ad188SAdrian Chadd static void 5672b3ad188SAdrian Chadd intr_isrc_post_filter(void *arg) 5682b3ad188SAdrian Chadd { 5692b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 5702b3ad188SAdrian Chadd 5712b3ad188SAdrian Chadd PIC_POST_FILTER(isrc->isrc_dev, isrc); 5722b3ad188SAdrian Chadd } 5732b3ad188SAdrian Chadd 5742b3ad188SAdrian Chadd /* 5752b3ad188SAdrian Chadd * Interrupt source assign_cpu method for MI interrupt framework. 5762b3ad188SAdrian Chadd */ 5772b3ad188SAdrian Chadd static int 5782b3ad188SAdrian Chadd intr_isrc_assign_cpu(void *arg, int cpu) 5792b3ad188SAdrian Chadd { 5802b3ad188SAdrian Chadd #ifdef SMP 5812b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 5822b3ad188SAdrian Chadd int error; 5832b3ad188SAdrian Chadd 5845b70c08cSSvatopluk Kraus if (isrc->isrc_dev != intr_irq_root_dev) 5852b3ad188SAdrian Chadd return (EINVAL); 5862b3ad188SAdrian Chadd 5872b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 5882b3ad188SAdrian Chadd if (cpu == NOCPU) { 5892b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 5902b3ad188SAdrian Chadd isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 5912b3ad188SAdrian Chadd } else { 5922b3ad188SAdrian Chadd CPU_SETOF(cpu, &isrc->isrc_cpu); 5932b3ad188SAdrian Chadd isrc->isrc_flags |= INTR_ISRCF_BOUND; 5942b3ad188SAdrian Chadd } 5952b3ad188SAdrian Chadd 5962b3ad188SAdrian Chadd /* 5972b3ad188SAdrian Chadd * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 5982b3ad188SAdrian Chadd * re-balance it to another CPU or enable it on more CPUs. However, 5992b3ad188SAdrian Chadd * PIC is expected to change isrc_cpu appropriately to keep us well 600e3043798SPedro F. Giffuni * informed if the call is successful. 6012b3ad188SAdrian Chadd */ 6022b3ad188SAdrian Chadd if (irq_assign_cpu) { 603bff6be3eSSvatopluk Kraus error = PIC_BIND_INTR(isrc->isrc_dev, isrc); 6042b3ad188SAdrian Chadd if (error) { 6052b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 6062b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6072b3ad188SAdrian Chadd return (error); 6082b3ad188SAdrian Chadd } 6092b3ad188SAdrian Chadd } 6102b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6112b3ad188SAdrian Chadd return (0); 6122b3ad188SAdrian Chadd #else 6132b3ad188SAdrian Chadd return (EOPNOTSUPP); 6142b3ad188SAdrian Chadd #endif 6152b3ad188SAdrian Chadd } 6162b3ad188SAdrian Chadd 6172b3ad188SAdrian Chadd /* 6182b3ad188SAdrian Chadd * Create interrupt event for interrupt source. 6192b3ad188SAdrian Chadd */ 6202b3ad188SAdrian Chadd static int 6212b3ad188SAdrian Chadd isrc_event_create(struct intr_irqsrc *isrc) 6222b3ad188SAdrian Chadd { 6232b3ad188SAdrian Chadd struct intr_event *ie; 6242b3ad188SAdrian Chadd int error; 6252b3ad188SAdrian Chadd 6262b3ad188SAdrian Chadd error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 6272b3ad188SAdrian Chadd intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 6282b3ad188SAdrian Chadd intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 6292b3ad188SAdrian Chadd if (error) 6302b3ad188SAdrian Chadd return (error); 6312b3ad188SAdrian Chadd 6322b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 6332b3ad188SAdrian Chadd /* 6342b3ad188SAdrian Chadd * Make sure that we do not mix the two ways 6352b3ad188SAdrian Chadd * how we handle interrupt sources. Let contested event wins. 6362b3ad188SAdrian Chadd */ 637169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 6382b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 639169e6abdSSvatopluk Kraus #else 640169e6abdSSvatopluk Kraus if (isrc->isrc_event != NULL) { 641169e6abdSSvatopluk Kraus #endif 6422b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6432b3ad188SAdrian Chadd intr_event_destroy(ie); 6442b3ad188SAdrian Chadd return (isrc->isrc_event != NULL ? EBUSY : 0); 6452b3ad188SAdrian Chadd } 6462b3ad188SAdrian Chadd isrc->isrc_event = ie; 6472b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6482b3ad188SAdrian Chadd 6492b3ad188SAdrian Chadd return (0); 6502b3ad188SAdrian Chadd } 6512b3ad188SAdrian Chadd #ifdef notyet 6522b3ad188SAdrian Chadd /* 6532b3ad188SAdrian Chadd * Destroy interrupt event for interrupt source. 6542b3ad188SAdrian Chadd */ 6552b3ad188SAdrian Chadd static void 6562b3ad188SAdrian Chadd isrc_event_destroy(struct intr_irqsrc *isrc) 6572b3ad188SAdrian Chadd { 6582b3ad188SAdrian Chadd struct intr_event *ie; 6592b3ad188SAdrian Chadd 6602b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 6612b3ad188SAdrian Chadd ie = isrc->isrc_event; 6622b3ad188SAdrian Chadd isrc->isrc_event = NULL; 6632b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6642b3ad188SAdrian Chadd 6652b3ad188SAdrian Chadd if (ie != NULL) 6662b3ad188SAdrian Chadd intr_event_destroy(ie); 6672b3ad188SAdrian Chadd } 6682b3ad188SAdrian Chadd #endif 6692b3ad188SAdrian Chadd /* 6702b3ad188SAdrian Chadd * Add handler to interrupt source. 6712b3ad188SAdrian Chadd */ 6722b3ad188SAdrian Chadd static int 6732b3ad188SAdrian Chadd isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 6742b3ad188SAdrian Chadd driver_filter_t filter, driver_intr_t handler, void *arg, 6752b3ad188SAdrian Chadd enum intr_type flags, void **cookiep) 6762b3ad188SAdrian Chadd { 6772b3ad188SAdrian Chadd int error; 6782b3ad188SAdrian Chadd 6792b3ad188SAdrian Chadd if (isrc->isrc_event == NULL) { 6802b3ad188SAdrian Chadd error = isrc_event_create(isrc); 6812b3ad188SAdrian Chadd if (error) 6822b3ad188SAdrian Chadd return (error); 6832b3ad188SAdrian Chadd } 6842b3ad188SAdrian Chadd 6852b3ad188SAdrian Chadd error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 6862b3ad188SAdrian Chadd arg, intr_priority(flags), flags, cookiep); 6872b3ad188SAdrian Chadd if (error == 0) { 6882b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 6892b3ad188SAdrian Chadd intrcnt_updatename(isrc); 6902b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 6912b3ad188SAdrian Chadd } 6922b3ad188SAdrian Chadd 6932b3ad188SAdrian Chadd return (error); 6942b3ad188SAdrian Chadd } 6952b3ad188SAdrian Chadd 6962b3ad188SAdrian Chadd /* 6972b3ad188SAdrian Chadd * Lookup interrupt controller locked. 6982b3ad188SAdrian Chadd */ 699bff6be3eSSvatopluk Kraus static inline struct intr_pic * 700c0d52370SAndrew Turner pic_lookup_locked(device_t dev, intptr_t xref, int flags) 7012b3ad188SAdrian Chadd { 7022b3ad188SAdrian Chadd struct intr_pic *pic; 7032b3ad188SAdrian Chadd 7042b3ad188SAdrian Chadd mtx_assert(&pic_list_lock, MA_OWNED); 7052b3ad188SAdrian Chadd 7064be58cbaSSvatopluk Kraus if (dev == NULL && xref == 0) 7074be58cbaSSvatopluk Kraus return (NULL); 7084be58cbaSSvatopluk Kraus 7094be58cbaSSvatopluk Kraus /* Note that pic->pic_dev is never NULL on registered PIC. */ 7102b3ad188SAdrian Chadd SLIST_FOREACH(pic, &pic_list, pic_next) { 711c0d52370SAndrew Turner if ((pic->pic_flags & FLAG_TYPE_MASK) != 712c0d52370SAndrew Turner (flags & FLAG_TYPE_MASK)) 713c0d52370SAndrew Turner continue; 714c0d52370SAndrew Turner 7154be58cbaSSvatopluk Kraus if (dev == NULL) { 7164be58cbaSSvatopluk Kraus if (xref == pic->pic_xref) 7174be58cbaSSvatopluk Kraus return (pic); 7184be58cbaSSvatopluk Kraus } else if (xref == 0 || pic->pic_xref == 0) { 7194be58cbaSSvatopluk Kraus if (dev == pic->pic_dev) 7204be58cbaSSvatopluk Kraus return (pic); 7214be58cbaSSvatopluk Kraus } else if (xref == pic->pic_xref && dev == pic->pic_dev) 7222b3ad188SAdrian Chadd return (pic); 7232b3ad188SAdrian Chadd } 7242b3ad188SAdrian Chadd return (NULL); 7252b3ad188SAdrian Chadd } 7262b3ad188SAdrian Chadd 7272b3ad188SAdrian Chadd /* 7282b3ad188SAdrian Chadd * Lookup interrupt controller. 7292b3ad188SAdrian Chadd */ 7302b3ad188SAdrian Chadd static struct intr_pic * 731c0d52370SAndrew Turner pic_lookup(device_t dev, intptr_t xref, int flags) 7322b3ad188SAdrian Chadd { 7332b3ad188SAdrian Chadd struct intr_pic *pic; 7342b3ad188SAdrian Chadd 7352b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 736c0d52370SAndrew Turner pic = pic_lookup_locked(dev, xref, flags); 7372b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 7382b3ad188SAdrian Chadd return (pic); 7392b3ad188SAdrian Chadd } 7402b3ad188SAdrian Chadd 7412b3ad188SAdrian Chadd /* 7422b3ad188SAdrian Chadd * Create interrupt controller. 7432b3ad188SAdrian Chadd */ 7442b3ad188SAdrian Chadd static struct intr_pic * 745c0d52370SAndrew Turner pic_create(device_t dev, intptr_t xref, int flags) 7462b3ad188SAdrian Chadd { 7472b3ad188SAdrian Chadd struct intr_pic *pic; 7482b3ad188SAdrian Chadd 7492b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 750c0d52370SAndrew Turner pic = pic_lookup_locked(dev, xref, flags); 7512b3ad188SAdrian Chadd if (pic != NULL) { 7522b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 7532b3ad188SAdrian Chadd return (pic); 7542b3ad188SAdrian Chadd } 7552b3ad188SAdrian Chadd pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 756b48c6083SAndrew Turner if (pic == NULL) { 757b48c6083SAndrew Turner mtx_unlock(&pic_list_lock); 758b48c6083SAndrew Turner return (NULL); 759b48c6083SAndrew Turner } 7602b3ad188SAdrian Chadd pic->pic_xref = xref; 7612b3ad188SAdrian Chadd pic->pic_dev = dev; 762c0d52370SAndrew Turner pic->pic_flags = flags; 763d1605cdaSAndrew Turner mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); 7642b3ad188SAdrian Chadd SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 7652b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 7662b3ad188SAdrian Chadd 7672b3ad188SAdrian Chadd return (pic); 7682b3ad188SAdrian Chadd } 7692b3ad188SAdrian Chadd #ifdef notyet 7702b3ad188SAdrian Chadd /* 7712b3ad188SAdrian Chadd * Destroy interrupt controller. 7722b3ad188SAdrian Chadd */ 7732b3ad188SAdrian Chadd static void 774c0d52370SAndrew Turner pic_destroy(device_t dev, intptr_t xref, int flags) 7752b3ad188SAdrian Chadd { 7762b3ad188SAdrian Chadd struct intr_pic *pic; 7772b3ad188SAdrian Chadd 7782b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 779c0d52370SAndrew Turner pic = pic_lookup_locked(dev, xref, flags); 7802b3ad188SAdrian Chadd if (pic == NULL) { 7812b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 7822b3ad188SAdrian Chadd return; 7832b3ad188SAdrian Chadd } 7842b3ad188SAdrian Chadd SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 7852b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 7862b3ad188SAdrian Chadd 7872b3ad188SAdrian Chadd free(pic, M_INTRNG); 7882b3ad188SAdrian Chadd } 7892b3ad188SAdrian Chadd #endif 7902b3ad188SAdrian Chadd /* 7912b3ad188SAdrian Chadd * Register interrupt controller. 7922b3ad188SAdrian Chadd */ 7939346e913SAndrew Turner struct intr_pic * 7942b3ad188SAdrian Chadd intr_pic_register(device_t dev, intptr_t xref) 7952b3ad188SAdrian Chadd { 7962b3ad188SAdrian Chadd struct intr_pic *pic; 7972b3ad188SAdrian Chadd 7984be58cbaSSvatopluk Kraus if (dev == NULL) 7999346e913SAndrew Turner return (NULL); 800c0d52370SAndrew Turner pic = pic_create(dev, xref, FLAG_PIC); 8012b3ad188SAdrian Chadd if (pic == NULL) 8029346e913SAndrew Turner return (NULL); 8032b3ad188SAdrian Chadd 804cff33fa8SEd Maste debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 805cff33fa8SEd Maste device_get_nameunit(dev), dev, (uintmax_t)xref); 8069346e913SAndrew Turner return (pic); 8072b3ad188SAdrian Chadd } 8082b3ad188SAdrian Chadd 8092b3ad188SAdrian Chadd /* 8102b3ad188SAdrian Chadd * Unregister interrupt controller. 8112b3ad188SAdrian Chadd */ 8122b3ad188SAdrian Chadd int 813bff6be3eSSvatopluk Kraus intr_pic_deregister(device_t dev, intptr_t xref) 8142b3ad188SAdrian Chadd { 8152b3ad188SAdrian Chadd 8162b3ad188SAdrian Chadd panic("%s: not implemented", __func__); 8172b3ad188SAdrian Chadd } 8182b3ad188SAdrian Chadd 8192b3ad188SAdrian Chadd /* 8202b3ad188SAdrian Chadd * Mark interrupt controller (itself) as a root one. 8212b3ad188SAdrian Chadd * 8222b3ad188SAdrian Chadd * Note that only an interrupt controller can really know its position 8232b3ad188SAdrian Chadd * in interrupt controller's tree. So root PIC must claim itself as a root. 8242b3ad188SAdrian Chadd * 8252b3ad188SAdrian Chadd * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 8262b3ad188SAdrian Chadd * page 30: 8272b3ad188SAdrian Chadd * "The root of the interrupt tree is determined when traversal 8282b3ad188SAdrian Chadd * of the interrupt tree reaches an interrupt controller node without 8292b3ad188SAdrian Chadd * an interrupts property and thus no explicit interrupt parent." 8302b3ad188SAdrian Chadd */ 8312b3ad188SAdrian Chadd int 8322b3ad188SAdrian Chadd intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 8332b3ad188SAdrian Chadd void *arg, u_int ipicount) 8342b3ad188SAdrian Chadd { 8353fc155dcSAndrew Turner struct intr_pic *pic; 8362b3ad188SAdrian Chadd 837c0d52370SAndrew Turner pic = pic_lookup(dev, xref, FLAG_PIC); 8383fc155dcSAndrew Turner if (pic == NULL) { 8392b3ad188SAdrian Chadd device_printf(dev, "not registered\n"); 8402b3ad188SAdrian Chadd return (EINVAL); 8412b3ad188SAdrian Chadd } 8423fc155dcSAndrew Turner 843c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 8443fc155dcSAndrew Turner ("%s: Found a non-PIC controller: %s", __func__, 8453fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 8463fc155dcSAndrew Turner 8472b3ad188SAdrian Chadd if (filter == NULL) { 8482b3ad188SAdrian Chadd device_printf(dev, "filter missing\n"); 8492b3ad188SAdrian Chadd return (EINVAL); 8502b3ad188SAdrian Chadd } 8512b3ad188SAdrian Chadd 8522b3ad188SAdrian Chadd /* 8532b3ad188SAdrian Chadd * Only one interrupt controllers could be on the root for now. 8542b3ad188SAdrian Chadd * Note that we further suppose that there is not threaded interrupt 8552b3ad188SAdrian Chadd * routine (handler) on the root. See intr_irq_handler(). 8562b3ad188SAdrian Chadd */ 8575b70c08cSSvatopluk Kraus if (intr_irq_root_dev != NULL) { 8582b3ad188SAdrian Chadd device_printf(dev, "another root already set\n"); 8592b3ad188SAdrian Chadd return (EBUSY); 8602b3ad188SAdrian Chadd } 8612b3ad188SAdrian Chadd 8625b70c08cSSvatopluk Kraus intr_irq_root_dev = dev; 8632b3ad188SAdrian Chadd irq_root_filter = filter; 8642b3ad188SAdrian Chadd irq_root_arg = arg; 8652b3ad188SAdrian Chadd irq_root_ipicount = ipicount; 8662b3ad188SAdrian Chadd 8672b3ad188SAdrian Chadd debugf("irq root set to %s\n", device_get_nameunit(dev)); 8682b3ad188SAdrian Chadd return (0); 8692b3ad188SAdrian Chadd } 8702b3ad188SAdrian Chadd 871d1605cdaSAndrew Turner /* 872d1605cdaSAndrew Turner * Add a handler to manage a sub range of a parents interrupts. 873d1605cdaSAndrew Turner */ 874d1605cdaSAndrew Turner struct intr_pic * 875d1605cdaSAndrew Turner intr_pic_add_handler(device_t parent, struct intr_pic *pic, 876d1605cdaSAndrew Turner intr_child_irq_filter_t *filter, void *arg, uintptr_t start, 877d1605cdaSAndrew Turner uintptr_t length) 878d1605cdaSAndrew Turner { 879d1605cdaSAndrew Turner struct intr_pic *parent_pic; 880d1605cdaSAndrew Turner struct intr_pic_child *newchild; 881d1605cdaSAndrew Turner #ifdef INVARIANTS 882d1605cdaSAndrew Turner struct intr_pic_child *child; 883d1605cdaSAndrew Turner #endif 884d1605cdaSAndrew Turner 885c0d52370SAndrew Turner /* Find the parent PIC */ 886c0d52370SAndrew Turner parent_pic = pic_lookup(parent, 0, FLAG_PIC); 887d1605cdaSAndrew Turner if (parent_pic == NULL) 888d1605cdaSAndrew Turner return (NULL); 889d1605cdaSAndrew Turner 890d1605cdaSAndrew Turner newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); 891d1605cdaSAndrew Turner newchild->pc_pic = pic; 892d1605cdaSAndrew Turner newchild->pc_filter = filter; 893d1605cdaSAndrew Turner newchild->pc_filter_arg = arg; 894d1605cdaSAndrew Turner newchild->pc_start = start; 895d1605cdaSAndrew Turner newchild->pc_length = length; 896d1605cdaSAndrew Turner 897d1605cdaSAndrew Turner mtx_lock_spin(&parent_pic->pic_child_lock); 898d1605cdaSAndrew Turner #ifdef INVARIANTS 899d1605cdaSAndrew Turner SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { 900d1605cdaSAndrew Turner KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", 901d1605cdaSAndrew Turner __func__)); 902d1605cdaSAndrew Turner } 903d1605cdaSAndrew Turner #endif 904d1605cdaSAndrew Turner SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); 905d1605cdaSAndrew Turner mtx_unlock_spin(&parent_pic->pic_child_lock); 906d1605cdaSAndrew Turner 907d1605cdaSAndrew Turner return (pic); 908d1605cdaSAndrew Turner } 909d1605cdaSAndrew Turner 910895c8b1cSMichal Meloun static int 911895c8b1cSMichal Meloun intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, 912895c8b1cSMichal Meloun struct intr_irqsrc **isrc) 9132b3ad188SAdrian Chadd { 914bff6be3eSSvatopluk Kraus struct intr_pic *pic; 915895c8b1cSMichal Meloun struct intr_map_data_msi *msi; 916bff6be3eSSvatopluk Kraus 917bff6be3eSSvatopluk Kraus if (data == NULL) 918bff6be3eSSvatopluk Kraus return (EINVAL); 919bff6be3eSSvatopluk Kraus 920c0d52370SAndrew Turner pic = pic_lookup(dev, xref, 921c0d52370SAndrew Turner (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); 92215adccc6SSvatopluk Kraus if (pic == NULL) 923bff6be3eSSvatopluk Kraus return (ESRCH); 924bff6be3eSSvatopluk Kraus 925895c8b1cSMichal Meloun switch (data->type) { 926895c8b1cSMichal Meloun case INTR_MAP_DATA_MSI: 927c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 928895c8b1cSMichal Meloun ("%s: Found a non-MSI controller: %s", __func__, 929895c8b1cSMichal Meloun device_get_name(pic->pic_dev))); 930895c8b1cSMichal Meloun msi = (struct intr_map_data_msi *)data; 931895c8b1cSMichal Meloun *isrc = msi->isrc; 932895c8b1cSMichal Meloun return (0); 933895c8b1cSMichal Meloun 934895c8b1cSMichal Meloun default: 935c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 9363fc155dcSAndrew Turner ("%s: Found a non-PIC controller: %s", __func__, 9373fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 938895c8b1cSMichal Meloun return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); 9393fc155dcSAndrew Turner 940895c8b1cSMichal Meloun } 941895c8b1cSMichal Meloun } 942895c8b1cSMichal Meloun 943895c8b1cSMichal Meloun int 944895c8b1cSMichal Meloun intr_activate_irq(device_t dev, struct resource *res) 945895c8b1cSMichal Meloun { 946895c8b1cSMichal Meloun device_t map_dev; 947895c8b1cSMichal Meloun intptr_t map_xref; 948895c8b1cSMichal Meloun struct intr_map_data *data; 949895c8b1cSMichal Meloun struct intr_irqsrc *isrc; 950895c8b1cSMichal Meloun u_int res_id; 951895c8b1cSMichal Meloun int error; 952895c8b1cSMichal Meloun 953895c8b1cSMichal Meloun KASSERT(rman_get_start(res) == rman_get_end(res), 954895c8b1cSMichal Meloun ("%s: more interrupts in resource", __func__)); 955895c8b1cSMichal Meloun 956895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 957895c8b1cSMichal Meloun if (intr_map_get_isrc(res_id) != NULL) 958895c8b1cSMichal Meloun panic("Attempt to double activation of resource id: %u\n", 959895c8b1cSMichal Meloun res_id); 960895c8b1cSMichal Meloun intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); 961895c8b1cSMichal Meloun error = intr_resolve_irq(map_dev, map_xref, data, &isrc); 962895c8b1cSMichal Meloun if (error != 0) { 963895c8b1cSMichal Meloun free(data, M_INTRNG); 964895c8b1cSMichal Meloun /* XXX TODO DISCONECTED PICs */ 965895c8b1cSMichal Meloun /* if (error == EINVAL) return(0); */ 966bff6be3eSSvatopluk Kraus return (error); 967bff6be3eSSvatopluk Kraus } 968895c8b1cSMichal Meloun intr_map_set_isrc(res_id, isrc); 969895c8b1cSMichal Meloun rman_set_virtual(res, data); 970895c8b1cSMichal Meloun return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); 971bff6be3eSSvatopluk Kraus } 972bff6be3eSSvatopluk Kraus 973bff6be3eSSvatopluk Kraus int 974895c8b1cSMichal Meloun intr_deactivate_irq(device_t dev, struct resource *res) 975bff6be3eSSvatopluk Kraus { 976bff6be3eSSvatopluk Kraus struct intr_map_data *data; 977bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 978895c8b1cSMichal Meloun u_int res_id; 979895c8b1cSMichal Meloun int error; 980bff6be3eSSvatopluk Kraus 981bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 982bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 983bff6be3eSSvatopluk Kraus 984895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 985895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 986bff6be3eSSvatopluk Kraus if (isrc == NULL) 987895c8b1cSMichal Meloun panic("Attempt to deactivate non-active resource id: %u\n", 988895c8b1cSMichal Meloun res_id); 989bff6be3eSSvatopluk Kraus 990c4263292SSvatopluk Kraus data = rman_get_virtual(res); 991895c8b1cSMichal Meloun error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); 992895c8b1cSMichal Meloun intr_map_set_isrc(res_id, NULL); 993895c8b1cSMichal Meloun rman_set_virtual(res, NULL); 994895c8b1cSMichal Meloun free(data, M_INTRNG); 995895c8b1cSMichal Meloun return (error); 996bff6be3eSSvatopluk Kraus } 997bff6be3eSSvatopluk Kraus 998bff6be3eSSvatopluk Kraus int 999bff6be3eSSvatopluk Kraus intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, 1000bff6be3eSSvatopluk Kraus driver_intr_t hand, void *arg, int flags, void **cookiep) 1001bff6be3eSSvatopluk Kraus { 1002bff6be3eSSvatopluk Kraus int error; 1003bff6be3eSSvatopluk Kraus struct intr_map_data *data; 1004bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1005bff6be3eSSvatopluk Kraus const char *name; 1006895c8b1cSMichal Meloun u_int res_id; 1007bff6be3eSSvatopluk Kraus 1008bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1009bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1010bff6be3eSSvatopluk Kraus 1011895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1012895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 1013895c8b1cSMichal Meloun if (isrc == NULL) { 1014895c8b1cSMichal Meloun /* XXX TODO DISCONECTED PICs */ 1015bff6be3eSSvatopluk Kraus return (EINVAL); 1016895c8b1cSMichal Meloun } 10172b3ad188SAdrian Chadd 1018c4263292SSvatopluk Kraus data = rman_get_virtual(res); 10192b3ad188SAdrian Chadd name = device_get_nameunit(dev); 10202b3ad188SAdrian Chadd 10212b3ad188SAdrian Chadd #ifdef INTR_SOLO 10222b3ad188SAdrian Chadd /* 1023e3043798SPedro F. Giffuni * Standard handling is done through MI interrupt framework. However, 10242b3ad188SAdrian Chadd * some interrupts could request solely own special handling. This 10252b3ad188SAdrian Chadd * non standard handling can be used for interrupt controllers without 10262b3ad188SAdrian Chadd * handler (filter only), so in case that interrupt controllers are 10272b3ad188SAdrian Chadd * chained, MI interrupt framework is called only in leaf controller. 10282b3ad188SAdrian Chadd * 10292b3ad188SAdrian Chadd * Note that root interrupt controller routine is served as well, 10302b3ad188SAdrian Chadd * however in intr_irq_handler(), i.e. main system dispatch routine. 10312b3ad188SAdrian Chadd */ 10322b3ad188SAdrian Chadd if (flags & INTR_SOLO && hand != NULL) { 10332b3ad188SAdrian Chadd debugf("irq %u cannot solo on %s\n", irq, name); 10342b3ad188SAdrian Chadd return (EINVAL); 10352b3ad188SAdrian Chadd } 10362b3ad188SAdrian Chadd 10372b3ad188SAdrian Chadd if (flags & INTR_SOLO) { 10382b3ad188SAdrian Chadd error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 10392b3ad188SAdrian Chadd arg, cookiep); 1040ce44a736SIan Lepore debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error, 10412b3ad188SAdrian Chadd name); 10422b3ad188SAdrian Chadd } else 10432b3ad188SAdrian Chadd #endif 10442b3ad188SAdrian Chadd { 10452b3ad188SAdrian Chadd error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 10462b3ad188SAdrian Chadd cookiep); 1047ce44a736SIan Lepore debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name); 10482b3ad188SAdrian Chadd } 10492b3ad188SAdrian Chadd if (error != 0) 10502b3ad188SAdrian Chadd return (error); 10512b3ad188SAdrian Chadd 10522b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1053bff6be3eSSvatopluk Kraus error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); 1054bff6be3eSSvatopluk Kraus if (error == 0) { 10552b3ad188SAdrian Chadd isrc->isrc_handlers++; 1056bff6be3eSSvatopluk Kraus if (isrc->isrc_handlers == 1) 10572b3ad188SAdrian Chadd PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 10582b3ad188SAdrian Chadd } 10592b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1060bff6be3eSSvatopluk Kraus if (error != 0) 1061bff6be3eSSvatopluk Kraus intr_event_remove_handler(*cookiep); 1062bff6be3eSSvatopluk Kraus return (error); 10632b3ad188SAdrian Chadd } 10642b3ad188SAdrian Chadd 10652b3ad188SAdrian Chadd int 1066bff6be3eSSvatopluk Kraus intr_teardown_irq(device_t dev, struct resource *res, void *cookie) 10672b3ad188SAdrian Chadd { 10682b3ad188SAdrian Chadd int error; 1069bff6be3eSSvatopluk Kraus struct intr_map_data *data; 1070bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1071895c8b1cSMichal Meloun u_int res_id; 10722b3ad188SAdrian Chadd 1073bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1074bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1075bff6be3eSSvatopluk Kraus 1076895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1077895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 10782b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 10792b3ad188SAdrian Chadd return (EINVAL); 1080bff6be3eSSvatopluk Kraus 1081c4263292SSvatopluk Kraus data = rman_get_virtual(res); 1082c4263292SSvatopluk Kraus 1083169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 10842b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 10852b3ad188SAdrian Chadd if (isrc != cookie) 10862b3ad188SAdrian Chadd return (EINVAL); 10872b3ad188SAdrian Chadd 10882b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 10892b3ad188SAdrian Chadd isrc->isrc_filter = NULL; 10902b3ad188SAdrian Chadd isrc->isrc_arg = NULL; 10912b3ad188SAdrian Chadd isrc->isrc_handlers = 0; 10922b3ad188SAdrian Chadd PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1093bff6be3eSSvatopluk Kraus PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 10942b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 10952b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 10962b3ad188SAdrian Chadd return (0); 10972b3ad188SAdrian Chadd } 1098169e6abdSSvatopluk Kraus #endif 10992b3ad188SAdrian Chadd if (isrc != intr_handler_source(cookie)) 11002b3ad188SAdrian Chadd return (EINVAL); 11012b3ad188SAdrian Chadd 11022b3ad188SAdrian Chadd error = intr_event_remove_handler(cookie); 11032b3ad188SAdrian Chadd if (error == 0) { 11042b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 11052b3ad188SAdrian Chadd isrc->isrc_handlers--; 1106bff6be3eSSvatopluk Kraus if (isrc->isrc_handlers == 0) 11072b3ad188SAdrian Chadd PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1108bff6be3eSSvatopluk Kraus PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 11092b3ad188SAdrian Chadd intrcnt_updatename(isrc); 11102b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 11112b3ad188SAdrian Chadd } 11122b3ad188SAdrian Chadd return (error); 11132b3ad188SAdrian Chadd } 11142b3ad188SAdrian Chadd 11152b3ad188SAdrian Chadd int 1116bff6be3eSSvatopluk Kraus intr_describe_irq(device_t dev, struct resource *res, void *cookie, 1117bff6be3eSSvatopluk Kraus const char *descr) 11182b3ad188SAdrian Chadd { 11192b3ad188SAdrian Chadd int error; 1120bff6be3eSSvatopluk Kraus struct intr_irqsrc *isrc; 1121895c8b1cSMichal Meloun u_int res_id; 11222b3ad188SAdrian Chadd 1123bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1124bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1125bff6be3eSSvatopluk Kraus 1126895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1127895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 11282b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 11292b3ad188SAdrian Chadd return (EINVAL); 1130169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 11312b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 11322b3ad188SAdrian Chadd if (isrc != cookie) 11332b3ad188SAdrian Chadd return (EINVAL); 11342b3ad188SAdrian Chadd 11352b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 11362b3ad188SAdrian Chadd isrc_update_name(isrc, descr); 11372b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 11382b3ad188SAdrian Chadd return (0); 11392b3ad188SAdrian Chadd } 1140169e6abdSSvatopluk Kraus #endif 11412b3ad188SAdrian Chadd error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 11422b3ad188SAdrian Chadd if (error == 0) { 11432b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 11442b3ad188SAdrian Chadd intrcnt_updatename(isrc); 11452b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 11462b3ad188SAdrian Chadd } 11472b3ad188SAdrian Chadd return (error); 11482b3ad188SAdrian Chadd } 11492b3ad188SAdrian Chadd 11502b3ad188SAdrian Chadd #ifdef SMP 11512b3ad188SAdrian Chadd int 1152bff6be3eSSvatopluk Kraus intr_bind_irq(device_t dev, struct resource *res, int cpu) 11532b3ad188SAdrian Chadd { 11542b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1155895c8b1cSMichal Meloun u_int res_id; 11562b3ad188SAdrian Chadd 1157bff6be3eSSvatopluk Kraus KASSERT(rman_get_start(res) == rman_get_end(res), 1158bff6be3eSSvatopluk Kraus ("%s: more interrupts in resource", __func__)); 1159bff6be3eSSvatopluk Kraus 1160895c8b1cSMichal Meloun res_id = (u_int)rman_get_start(res); 1161895c8b1cSMichal Meloun isrc = intr_map_get_isrc(res_id); 11622b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 11632b3ad188SAdrian Chadd return (EINVAL); 1164169e6abdSSvatopluk Kraus #ifdef INTR_SOLO 11652b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) 11662b3ad188SAdrian Chadd return (intr_isrc_assign_cpu(isrc, cpu)); 1167169e6abdSSvatopluk Kraus #endif 11682b3ad188SAdrian Chadd return (intr_event_bind(isrc->isrc_event, cpu)); 11692b3ad188SAdrian Chadd } 11702b3ad188SAdrian Chadd 11712b3ad188SAdrian Chadd /* 11722b3ad188SAdrian Chadd * Return the CPU that the next interrupt source should use. 11732b3ad188SAdrian Chadd * For now just returns the next CPU according to round-robin. 11742b3ad188SAdrian Chadd */ 11752b3ad188SAdrian Chadd u_int 11762b3ad188SAdrian Chadd intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 11772b3ad188SAdrian Chadd { 1178a92a2f00SAndrew Turner u_int cpu; 11792b3ad188SAdrian Chadd 1180a92a2f00SAndrew Turner KASSERT(!CPU_EMPTY(cpumask), ("%s: Empty CPU mask", __func__)); 1181a92a2f00SAndrew Turner if (!irq_assign_cpu || mp_ncpus == 1) { 1182a92a2f00SAndrew Turner cpu = PCPU_GET(cpuid); 1183a92a2f00SAndrew Turner 1184a92a2f00SAndrew Turner if (CPU_ISSET(cpu, cpumask)) 1185a92a2f00SAndrew Turner return (curcpu); 1186a92a2f00SAndrew Turner 1187a92a2f00SAndrew Turner return (CPU_FFS(cpumask) - 1); 1188a92a2f00SAndrew Turner } 11892b3ad188SAdrian Chadd 11902b3ad188SAdrian Chadd do { 11912b3ad188SAdrian Chadd last_cpu++; 11922b3ad188SAdrian Chadd if (last_cpu > mp_maxid) 11932b3ad188SAdrian Chadd last_cpu = 0; 11942b3ad188SAdrian Chadd } while (!CPU_ISSET(last_cpu, cpumask)); 11952b3ad188SAdrian Chadd return (last_cpu); 11962b3ad188SAdrian Chadd } 11972b3ad188SAdrian Chadd 1198*dc425090SMitchell Horne #ifndef EARLY_AP_STARTUP 11992b3ad188SAdrian Chadd /* 12002b3ad188SAdrian Chadd * Distribute all the interrupt sources among the available 12012b3ad188SAdrian Chadd * CPUs once the AP's have been launched. 12022b3ad188SAdrian Chadd */ 12032b3ad188SAdrian Chadd static void 12042b3ad188SAdrian Chadd intr_irq_shuffle(void *arg __unused) 12052b3ad188SAdrian Chadd { 12062b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 12072b3ad188SAdrian Chadd u_int i; 12082b3ad188SAdrian Chadd 12092b3ad188SAdrian Chadd if (mp_ncpus == 1) 12102b3ad188SAdrian Chadd return; 12112b3ad188SAdrian Chadd 12122b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1213*dc425090SMitchell Horne irq_assign_cpu = true; 12142b3ad188SAdrian Chadd for (i = 0; i < NIRQ; i++) { 12152b3ad188SAdrian Chadd isrc = irq_sources[i]; 12162b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0 || 1217cf55df9fSSvatopluk Kraus isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) 12182b3ad188SAdrian Chadd continue; 12192b3ad188SAdrian Chadd 12202b3ad188SAdrian Chadd if (isrc->isrc_event != NULL && 12212b3ad188SAdrian Chadd isrc->isrc_flags & INTR_ISRCF_BOUND && 12222b3ad188SAdrian Chadd isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 12232b3ad188SAdrian Chadd panic("%s: CPU inconsistency", __func__); 12242b3ad188SAdrian Chadd 12252b3ad188SAdrian Chadd if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 12262b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); /* start again */ 12272b3ad188SAdrian Chadd 12282b3ad188SAdrian Chadd /* 12292b3ad188SAdrian Chadd * We are in wicked position here if the following call fails 12302b3ad188SAdrian Chadd * for bound ISRC. The best thing we can do is to clear 12312b3ad188SAdrian Chadd * isrc_cpu so inconsistency with ie_cpu will be detectable. 12322b3ad188SAdrian Chadd */ 1233bff6be3eSSvatopluk Kraus if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) 12342b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 12352b3ad188SAdrian Chadd } 12362b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 12372b3ad188SAdrian Chadd } 12382b3ad188SAdrian Chadd SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1239*dc425090SMitchell Horne #endif /* !EARLY_AP_STARTUP */ 12402b3ad188SAdrian Chadd 12412b3ad188SAdrian Chadd #else 12422b3ad188SAdrian Chadd u_int 12432b3ad188SAdrian Chadd intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 12442b3ad188SAdrian Chadd { 12452b3ad188SAdrian Chadd 12462b3ad188SAdrian Chadd return (PCPU_GET(cpuid)); 12472b3ad188SAdrian Chadd } 1248*dc425090SMitchell Horne #endif /* SMP */ 12492b3ad188SAdrian Chadd 12503fc155dcSAndrew Turner /* 1251895c8b1cSMichal Meloun * Allocate memory for new intr_map_data structure. 1252895c8b1cSMichal Meloun * Initialize common fields. 1253895c8b1cSMichal Meloun */ 1254895c8b1cSMichal Meloun struct intr_map_data * 1255895c8b1cSMichal Meloun intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) 1256895c8b1cSMichal Meloun { 1257895c8b1cSMichal Meloun struct intr_map_data *data; 1258895c8b1cSMichal Meloun 1259895c8b1cSMichal Meloun data = malloc(len, M_INTRNG, flags); 1260895c8b1cSMichal Meloun data->type = type; 1261895c8b1cSMichal Meloun data->len = len; 1262895c8b1cSMichal Meloun return (data); 1263895c8b1cSMichal Meloun } 1264895c8b1cSMichal Meloun 1265895c8b1cSMichal Meloun void intr_free_intr_map_data(struct intr_map_data *data) 1266895c8b1cSMichal Meloun { 1267895c8b1cSMichal Meloun 1268895c8b1cSMichal Meloun free(data, M_INTRNG); 1269895c8b1cSMichal Meloun } 1270895c8b1cSMichal Meloun 1271895c8b1cSMichal Meloun /* 12723fc155dcSAndrew Turner * Register a MSI/MSI-X interrupt controller 12733fc155dcSAndrew Turner */ 12743fc155dcSAndrew Turner int 12753fc155dcSAndrew Turner intr_msi_register(device_t dev, intptr_t xref) 12763fc155dcSAndrew Turner { 12773fc155dcSAndrew Turner struct intr_pic *pic; 12783fc155dcSAndrew Turner 12793fc155dcSAndrew Turner if (dev == NULL) 12803fc155dcSAndrew Turner return (EINVAL); 1281c0d52370SAndrew Turner pic = pic_create(dev, xref, FLAG_MSI); 12823fc155dcSAndrew Turner if (pic == NULL) 12833fc155dcSAndrew Turner return (ENOMEM); 12843fc155dcSAndrew Turner 12853fc155dcSAndrew Turner debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 12863fc155dcSAndrew Turner device_get_nameunit(dev), dev, (uintmax_t)xref); 12873fc155dcSAndrew Turner return (0); 12883fc155dcSAndrew Turner } 12893fc155dcSAndrew Turner 12903fc155dcSAndrew Turner int 12913fc155dcSAndrew Turner intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, 12923fc155dcSAndrew Turner int maxcount, int *irqs) 12933fc155dcSAndrew Turner { 12943fc155dcSAndrew Turner struct intr_irqsrc **isrc; 12953fc155dcSAndrew Turner struct intr_pic *pic; 12963fc155dcSAndrew Turner device_t pdev; 1297895c8b1cSMichal Meloun struct intr_map_data_msi *msi; 12983fc155dcSAndrew Turner int err, i; 12993fc155dcSAndrew Turner 1300c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 13013fc155dcSAndrew Turner if (pic == NULL) 13023fc155dcSAndrew Turner return (ESRCH); 13033fc155dcSAndrew Turner 1304c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 13053fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 13063fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 13073fc155dcSAndrew Turner 13083fc155dcSAndrew Turner isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 13093fc155dcSAndrew Turner err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); 1310895c8b1cSMichal Meloun if (err != 0) { 1311895c8b1cSMichal Meloun free(isrc, M_INTRNG); 1312895c8b1cSMichal Meloun return (err); 13133fc155dcSAndrew Turner } 13143fc155dcSAndrew Turner 1315895c8b1cSMichal Meloun for (i = 0; i < count; i++) { 1316895c8b1cSMichal Meloun msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1317895c8b1cSMichal Meloun INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1318895c8b1cSMichal Meloun msi-> isrc = isrc[i]; 1319895c8b1cSMichal Meloun irqs[i] = intr_map_irq(pic->pic_dev, xref, 1320895c8b1cSMichal Meloun (struct intr_map_data *)msi); 1321895c8b1cSMichal Meloun 1322895c8b1cSMichal Meloun } 13233fc155dcSAndrew Turner free(isrc, M_INTRNG); 13243fc155dcSAndrew Turner 13253fc155dcSAndrew Turner return (err); 13263fc155dcSAndrew Turner } 13273fc155dcSAndrew Turner 13283fc155dcSAndrew Turner int 13293fc155dcSAndrew Turner intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, 13303fc155dcSAndrew Turner int *irqs) 13313fc155dcSAndrew Turner { 13323fc155dcSAndrew Turner struct intr_irqsrc **isrc; 13333fc155dcSAndrew Turner struct intr_pic *pic; 1334609b0fe9SOleksandr Tymoshenko struct intr_map_data_msi *msi; 13353fc155dcSAndrew Turner int i, err; 13363fc155dcSAndrew Turner 1337c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 13383fc155dcSAndrew Turner if (pic == NULL) 13393fc155dcSAndrew Turner return (ESRCH); 13403fc155dcSAndrew Turner 1341c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 13423fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 13433fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 13443fc155dcSAndrew Turner 13453fc155dcSAndrew Turner isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 13463fc155dcSAndrew Turner 1347609b0fe9SOleksandr Tymoshenko for (i = 0; i < count; i++) { 1348609b0fe9SOleksandr Tymoshenko msi = (struct intr_map_data_msi *) 1349609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(irqs[i]); 1350609b0fe9SOleksandr Tymoshenko KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1351609b0fe9SOleksandr Tymoshenko ("%s: irq %d map data is not MSI", __func__, 1352609b0fe9SOleksandr Tymoshenko irqs[i])); 1353609b0fe9SOleksandr Tymoshenko isrc[i] = msi->isrc; 1354609b0fe9SOleksandr Tymoshenko } 13553fc155dcSAndrew Turner 13563fc155dcSAndrew Turner err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); 1357895c8b1cSMichal Meloun 1358895c8b1cSMichal Meloun for (i = 0; i < count; i++) { 1359895c8b1cSMichal Meloun if (isrc[i] != NULL) 1360895c8b1cSMichal Meloun intr_unmap_irq(irqs[i]); 1361895c8b1cSMichal Meloun } 1362895c8b1cSMichal Meloun 13633fc155dcSAndrew Turner free(isrc, M_INTRNG); 13643fc155dcSAndrew Turner return (err); 13653fc155dcSAndrew Turner } 13663fc155dcSAndrew Turner 13673fc155dcSAndrew Turner int 13683fc155dcSAndrew Turner intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) 13693fc155dcSAndrew Turner { 13703fc155dcSAndrew Turner struct intr_irqsrc *isrc; 13713fc155dcSAndrew Turner struct intr_pic *pic; 13723fc155dcSAndrew Turner device_t pdev; 1373895c8b1cSMichal Meloun struct intr_map_data_msi *msi; 13743fc155dcSAndrew Turner int err; 13753fc155dcSAndrew Turner 1376c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 13773fc155dcSAndrew Turner if (pic == NULL) 13783fc155dcSAndrew Turner return (ESRCH); 13793fc155dcSAndrew Turner 1380c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 13813fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 13823fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 13833fc155dcSAndrew Turner 13843fc155dcSAndrew Turner err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); 13853fc155dcSAndrew Turner if (err != 0) 13863fc155dcSAndrew Turner return (err); 13873fc155dcSAndrew Turner 1388895c8b1cSMichal Meloun msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1389895c8b1cSMichal Meloun INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1390895c8b1cSMichal Meloun msi->isrc = isrc; 1391895c8b1cSMichal Meloun *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); 13923fc155dcSAndrew Turner return (0); 13933fc155dcSAndrew Turner } 13943fc155dcSAndrew Turner 13953fc155dcSAndrew Turner int 13963fc155dcSAndrew Turner intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) 13973fc155dcSAndrew Turner { 13983fc155dcSAndrew Turner struct intr_irqsrc *isrc; 13993fc155dcSAndrew Turner struct intr_pic *pic; 1400609b0fe9SOleksandr Tymoshenko struct intr_map_data_msi *msi; 14013fc155dcSAndrew Turner int err; 14023fc155dcSAndrew Turner 1403c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 14043fc155dcSAndrew Turner if (pic == NULL) 14053fc155dcSAndrew Turner return (ESRCH); 14063fc155dcSAndrew Turner 1407c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 14083fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 14093fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 14103fc155dcSAndrew Turner 1411609b0fe9SOleksandr Tymoshenko msi = (struct intr_map_data_msi *) 1412609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(irq); 1413609b0fe9SOleksandr Tymoshenko KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1414609b0fe9SOleksandr Tymoshenko ("%s: irq %d map data is not MSI", __func__, 1415609b0fe9SOleksandr Tymoshenko irq)); 1416609b0fe9SOleksandr Tymoshenko isrc = msi->isrc; 1417895c8b1cSMichal Meloun if (isrc == NULL) { 1418895c8b1cSMichal Meloun intr_unmap_irq(irq); 14193fc155dcSAndrew Turner return (EINVAL); 1420895c8b1cSMichal Meloun } 14213fc155dcSAndrew Turner 14223fc155dcSAndrew Turner err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); 1423895c8b1cSMichal Meloun intr_unmap_irq(irq); 1424895c8b1cSMichal Meloun 14253fc155dcSAndrew Turner return (err); 14263fc155dcSAndrew Turner } 14273fc155dcSAndrew Turner 14283fc155dcSAndrew Turner int 14293fc155dcSAndrew Turner intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, 14303fc155dcSAndrew Turner uint64_t *addr, uint32_t *data) 14313fc155dcSAndrew Turner { 14323fc155dcSAndrew Turner struct intr_irqsrc *isrc; 14333fc155dcSAndrew Turner struct intr_pic *pic; 14343fc155dcSAndrew Turner int err; 14353fc155dcSAndrew Turner 1436c0d52370SAndrew Turner pic = pic_lookup(NULL, xref, FLAG_MSI); 14373fc155dcSAndrew Turner if (pic == NULL) 14383fc155dcSAndrew Turner return (ESRCH); 14393fc155dcSAndrew Turner 1440c0d52370SAndrew Turner KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 14413fc155dcSAndrew Turner ("%s: Found a non-MSI controller: %s", __func__, 14423fc155dcSAndrew Turner device_get_name(pic->pic_dev))); 14433fc155dcSAndrew Turner 1444895c8b1cSMichal Meloun isrc = intr_map_get_isrc(irq); 14453fc155dcSAndrew Turner if (isrc == NULL) 14463fc155dcSAndrew Turner return (EINVAL); 14473fc155dcSAndrew Turner 14483fc155dcSAndrew Turner err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); 14493fc155dcSAndrew Turner return (err); 14503fc155dcSAndrew Turner } 14513fc155dcSAndrew Turner 14522b3ad188SAdrian Chadd void dosoftints(void); 14532b3ad188SAdrian Chadd void 14542b3ad188SAdrian Chadd dosoftints(void) 14552b3ad188SAdrian Chadd { 14562b3ad188SAdrian Chadd } 14572b3ad188SAdrian Chadd 14582b3ad188SAdrian Chadd #ifdef SMP 14592b3ad188SAdrian Chadd /* 14602b3ad188SAdrian Chadd * Init interrupt controller on another CPU. 14612b3ad188SAdrian Chadd */ 14622b3ad188SAdrian Chadd void 14632b3ad188SAdrian Chadd intr_pic_init_secondary(void) 14642b3ad188SAdrian Chadd { 14652b3ad188SAdrian Chadd 14662b3ad188SAdrian Chadd /* 14672b3ad188SAdrian Chadd * QQQ: Only root PIC is aware of other CPUs ??? 14682b3ad188SAdrian Chadd */ 14695b70c08cSSvatopluk Kraus KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 14702b3ad188SAdrian Chadd 14712b3ad188SAdrian Chadd //mtx_lock(&isrc_table_lock); 14725b70c08cSSvatopluk Kraus PIC_INIT_SECONDARY(intr_irq_root_dev); 14732b3ad188SAdrian Chadd //mtx_unlock(&isrc_table_lock); 14742b3ad188SAdrian Chadd } 14752b3ad188SAdrian Chadd #endif 14762b3ad188SAdrian Chadd 14772b3ad188SAdrian Chadd #ifdef DDB 14782b3ad188SAdrian Chadd DB_SHOW_COMMAND(irqs, db_show_irqs) 14792b3ad188SAdrian Chadd { 14802b3ad188SAdrian Chadd u_int i, irqsum; 1481bff6be3eSSvatopluk Kraus u_long num; 14822b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 14832b3ad188SAdrian Chadd 14842b3ad188SAdrian Chadd for (irqsum = 0, i = 0; i < NIRQ; i++) { 14852b3ad188SAdrian Chadd isrc = irq_sources[i]; 14862b3ad188SAdrian Chadd if (isrc == NULL) 14872b3ad188SAdrian Chadd continue; 14882b3ad188SAdrian Chadd 1489bff6be3eSSvatopluk Kraus num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; 14902b3ad188SAdrian Chadd db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 14912b3ad188SAdrian Chadd isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1492bff6be3eSSvatopluk Kraus isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); 1493bff6be3eSSvatopluk Kraus irqsum += num; 14942b3ad188SAdrian Chadd } 14952b3ad188SAdrian Chadd db_printf("irq total %u\n", irqsum); 14962b3ad188SAdrian Chadd } 14972b3ad188SAdrian Chadd #endif 1498895c8b1cSMichal Meloun 1499895c8b1cSMichal Meloun /* 1500895c8b1cSMichal Meloun * Interrupt mapping table functions. 1501895c8b1cSMichal Meloun * 1502895c8b1cSMichal Meloun * Please, keep this part separately, it can be transformed to 1503895c8b1cSMichal Meloun * extension of standard resources. 1504895c8b1cSMichal Meloun */ 1505895c8b1cSMichal Meloun struct intr_map_entry 1506895c8b1cSMichal Meloun { 1507895c8b1cSMichal Meloun device_t dev; 1508895c8b1cSMichal Meloun intptr_t xref; 1509895c8b1cSMichal Meloun struct intr_map_data *map_data; 1510895c8b1cSMichal Meloun struct intr_irqsrc *isrc; 1511895c8b1cSMichal Meloun /* XXX TODO DISCONECTED PICs */ 1512895c8b1cSMichal Meloun /*int flags */ 1513895c8b1cSMichal Meloun }; 1514895c8b1cSMichal Meloun 1515895c8b1cSMichal Meloun /* XXX Convert irq_map[] to dynamicaly expandable one. */ 1516895c8b1cSMichal Meloun static struct intr_map_entry *irq_map[2 * NIRQ]; 1517895c8b1cSMichal Meloun static int irq_map_count = nitems(irq_map); 1518895c8b1cSMichal Meloun static int irq_map_first_free_idx; 1519895c8b1cSMichal Meloun static struct mtx irq_map_lock; 1520895c8b1cSMichal Meloun 1521895c8b1cSMichal Meloun static struct intr_irqsrc * 1522895c8b1cSMichal Meloun intr_map_get_isrc(u_int res_id) 1523895c8b1cSMichal Meloun { 1524895c8b1cSMichal Meloun struct intr_irqsrc *isrc; 1525895c8b1cSMichal Meloun 1526ecc8ccb4SAndrew Turner isrc = NULL; 1527895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1528ecc8ccb4SAndrew Turner if (res_id < irq_map_count && irq_map[res_id] != NULL) 1529895c8b1cSMichal Meloun isrc = irq_map[res_id]->isrc; 1530895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1531ecc8ccb4SAndrew Turner 1532895c8b1cSMichal Meloun return (isrc); 1533895c8b1cSMichal Meloun } 1534895c8b1cSMichal Meloun 1535895c8b1cSMichal Meloun static void 1536895c8b1cSMichal Meloun intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) 1537895c8b1cSMichal Meloun { 1538895c8b1cSMichal Meloun 1539895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1540ecc8ccb4SAndrew Turner if (res_id < irq_map_count && irq_map[res_id] != NULL) 1541895c8b1cSMichal Meloun irq_map[res_id]->isrc = isrc; 1542895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1543895c8b1cSMichal Meloun } 1544895c8b1cSMichal Meloun 1545895c8b1cSMichal Meloun /* 1546895c8b1cSMichal Meloun * Get a copy of intr_map_entry data 1547895c8b1cSMichal Meloun */ 1548609b0fe9SOleksandr Tymoshenko static struct intr_map_data * 1549609b0fe9SOleksandr Tymoshenko intr_map_get_map_data(u_int res_id) 1550609b0fe9SOleksandr Tymoshenko { 1551609b0fe9SOleksandr Tymoshenko struct intr_map_data *data; 1552609b0fe9SOleksandr Tymoshenko 1553609b0fe9SOleksandr Tymoshenko data = NULL; 1554609b0fe9SOleksandr Tymoshenko mtx_lock(&irq_map_lock); 1555609b0fe9SOleksandr Tymoshenko if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1556609b0fe9SOleksandr Tymoshenko panic("Attempt to copy invalid resource id: %u\n", res_id); 1557609b0fe9SOleksandr Tymoshenko data = irq_map[res_id]->map_data; 1558609b0fe9SOleksandr Tymoshenko mtx_unlock(&irq_map_lock); 1559609b0fe9SOleksandr Tymoshenko 1560609b0fe9SOleksandr Tymoshenko return (data); 1561609b0fe9SOleksandr Tymoshenko } 1562609b0fe9SOleksandr Tymoshenko 1563609b0fe9SOleksandr Tymoshenko /* 1564609b0fe9SOleksandr Tymoshenko * Get a copy of intr_map_entry data 1565609b0fe9SOleksandr Tymoshenko */ 1566895c8b1cSMichal Meloun static void 1567895c8b1cSMichal Meloun intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, 1568895c8b1cSMichal Meloun struct intr_map_data **data) 1569895c8b1cSMichal Meloun { 1570895c8b1cSMichal Meloun size_t len; 1571895c8b1cSMichal Meloun 1572895c8b1cSMichal Meloun len = 0; 1573895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1574895c8b1cSMichal Meloun if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1575895c8b1cSMichal Meloun panic("Attempt to copy invalid resource id: %u\n", res_id); 1576895c8b1cSMichal Meloun if (irq_map[res_id]->map_data != NULL) 1577895c8b1cSMichal Meloun len = irq_map[res_id]->map_data->len; 1578895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1579895c8b1cSMichal Meloun 1580895c8b1cSMichal Meloun if (len == 0) 1581895c8b1cSMichal Meloun *data = NULL; 1582895c8b1cSMichal Meloun else 1583895c8b1cSMichal Meloun *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); 1584895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1585895c8b1cSMichal Meloun if (irq_map[res_id] == NULL) 1586895c8b1cSMichal Meloun panic("Attempt to copy invalid resource id: %u\n", res_id); 1587895c8b1cSMichal Meloun if (len != 0) { 1588895c8b1cSMichal Meloun if (len != irq_map[res_id]->map_data->len) 1589895c8b1cSMichal Meloun panic("Resource id: %u has changed.\n", res_id); 1590895c8b1cSMichal Meloun memcpy(*data, irq_map[res_id]->map_data, len); 1591895c8b1cSMichal Meloun } 1592895c8b1cSMichal Meloun *map_dev = irq_map[res_id]->dev; 1593895c8b1cSMichal Meloun *map_xref = irq_map[res_id]->xref; 1594895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1595895c8b1cSMichal Meloun } 1596895c8b1cSMichal Meloun 1597895c8b1cSMichal Meloun /* 1598895c8b1cSMichal Meloun * Allocate and fill new entry in irq_map table. 1599895c8b1cSMichal Meloun */ 1600895c8b1cSMichal Meloun u_int 1601895c8b1cSMichal Meloun intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) 1602895c8b1cSMichal Meloun { 1603895c8b1cSMichal Meloun u_int i; 1604895c8b1cSMichal Meloun struct intr_map_entry *entry; 1605895c8b1cSMichal Meloun 1606895c8b1cSMichal Meloun /* Prepare new entry first. */ 1607895c8b1cSMichal Meloun entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); 1608895c8b1cSMichal Meloun 1609895c8b1cSMichal Meloun entry->dev = dev; 1610895c8b1cSMichal Meloun entry->xref = xref; 1611895c8b1cSMichal Meloun entry->map_data = data; 1612895c8b1cSMichal Meloun entry->isrc = NULL; 1613895c8b1cSMichal Meloun 1614895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1615895c8b1cSMichal Meloun for (i = irq_map_first_free_idx; i < irq_map_count; i++) { 1616895c8b1cSMichal Meloun if (irq_map[i] == NULL) { 1617895c8b1cSMichal Meloun irq_map[i] = entry; 1618895c8b1cSMichal Meloun irq_map_first_free_idx = i + 1; 1619895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1620895c8b1cSMichal Meloun return (i); 1621895c8b1cSMichal Meloun } 1622895c8b1cSMichal Meloun } 1623895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1624895c8b1cSMichal Meloun 1625895c8b1cSMichal Meloun /* XXX Expand irq_map table */ 1626895c8b1cSMichal Meloun panic("IRQ mapping table is full."); 1627895c8b1cSMichal Meloun } 1628895c8b1cSMichal Meloun 1629895c8b1cSMichal Meloun /* 1630895c8b1cSMichal Meloun * Remove and free mapping entry. 1631895c8b1cSMichal Meloun */ 1632895c8b1cSMichal Meloun void 1633895c8b1cSMichal Meloun intr_unmap_irq(u_int res_id) 1634895c8b1cSMichal Meloun { 1635895c8b1cSMichal Meloun struct intr_map_entry *entry; 1636895c8b1cSMichal Meloun 1637895c8b1cSMichal Meloun mtx_lock(&irq_map_lock); 1638895c8b1cSMichal Meloun if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) 1639895c8b1cSMichal Meloun panic("Attempt to unmap invalid resource id: %u\n", res_id); 1640895c8b1cSMichal Meloun entry = irq_map[res_id]; 1641895c8b1cSMichal Meloun irq_map[res_id] = NULL; 1642895c8b1cSMichal Meloun irq_map_first_free_idx = res_id; 1643895c8b1cSMichal Meloun mtx_unlock(&irq_map_lock); 1644895c8b1cSMichal Meloun intr_free_intr_map_data(entry->map_data); 1645895c8b1cSMichal Meloun free(entry, M_INTRNG); 1646895c8b1cSMichal Meloun } 1647895c8b1cSMichal Meloun 1648895c8b1cSMichal Meloun /* 1649895c8b1cSMichal Meloun * Clone mapping entry. 1650895c8b1cSMichal Meloun */ 1651895c8b1cSMichal Meloun u_int 1652895c8b1cSMichal Meloun intr_map_clone_irq(u_int old_res_id) 1653895c8b1cSMichal Meloun { 1654895c8b1cSMichal Meloun device_t map_dev; 1655895c8b1cSMichal Meloun intptr_t map_xref; 1656895c8b1cSMichal Meloun struct intr_map_data *data; 1657895c8b1cSMichal Meloun 1658895c8b1cSMichal Meloun intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); 1659895c8b1cSMichal Meloun return (intr_map_irq(map_dev, map_xref, data)); 1660895c8b1cSMichal Meloun } 1661895c8b1cSMichal Meloun 1662895c8b1cSMichal Meloun static void 1663895c8b1cSMichal Meloun intr_map_init(void *dummy __unused) 1664895c8b1cSMichal Meloun { 1665895c8b1cSMichal Meloun 1666895c8b1cSMichal Meloun mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); 1667895c8b1cSMichal Meloun } 1668895c8b1cSMichal Meloun SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); 1669