1*2b3ad188SAdrian Chadd /*- 2*2b3ad188SAdrian Chadd * Copyright (c) 2012-2014 Jakub Wojciech Klama <jceel@FreeBSD.org>. 3*2b3ad188SAdrian Chadd * Copyright (c) 2015 Svatopluk Kraus 4*2b3ad188SAdrian Chadd * Copyright (c) 2015 Michal Meloun 5*2b3ad188SAdrian Chadd * All rights reserved. 6*2b3ad188SAdrian Chadd * 7*2b3ad188SAdrian Chadd * Redistribution and use in source and binary forms, with or without 8*2b3ad188SAdrian Chadd * modification, are permitted provided that the following conditions 9*2b3ad188SAdrian Chadd * are met: 10*2b3ad188SAdrian Chadd * 1. Redistributions of source code must retain the above copyright 11*2b3ad188SAdrian Chadd * notice, this list of conditions and the following disclaimer. 12*2b3ad188SAdrian Chadd * 2. Redistributions in binary form must reproduce the above copyright 13*2b3ad188SAdrian Chadd * notice, this list of conditions and the following disclaimer in the 14*2b3ad188SAdrian Chadd * documentation and/or other materials provided with the distribution. 15*2b3ad188SAdrian Chadd * 16*2b3ad188SAdrian Chadd * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17*2b3ad188SAdrian Chadd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18*2b3ad188SAdrian Chadd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19*2b3ad188SAdrian Chadd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20*2b3ad188SAdrian Chadd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21*2b3ad188SAdrian Chadd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22*2b3ad188SAdrian Chadd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23*2b3ad188SAdrian Chadd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24*2b3ad188SAdrian Chadd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25*2b3ad188SAdrian Chadd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26*2b3ad188SAdrian Chadd * SUCH DAMAGE. 27*2b3ad188SAdrian Chadd * 28*2b3ad188SAdrian Chadd * $FreeBSD$ 29*2b3ad188SAdrian Chadd */ 30*2b3ad188SAdrian Chadd 31*2b3ad188SAdrian Chadd #include <sys/cdefs.h> 32*2b3ad188SAdrian Chadd __FBSDID("$FreeBSD$"); 33*2b3ad188SAdrian Chadd 34*2b3ad188SAdrian Chadd /* 35*2b3ad188SAdrian Chadd * New-style Interrupt Framework 36*2b3ad188SAdrian Chadd * 37*2b3ad188SAdrian Chadd * TODO: - to support IPI (PPI) enabling on other CPUs if already started 38*2b3ad188SAdrian Chadd * - to complete things for removable PICs 39*2b3ad188SAdrian Chadd */ 40*2b3ad188SAdrian Chadd 41*2b3ad188SAdrian Chadd #include "opt_ddb.h" 42*2b3ad188SAdrian Chadd #include "opt_platform.h" 43*2b3ad188SAdrian Chadd 44*2b3ad188SAdrian Chadd #include <sys/param.h> 45*2b3ad188SAdrian Chadd #include <sys/systm.h> 46*2b3ad188SAdrian Chadd #include <sys/kernel.h> 47*2b3ad188SAdrian Chadd #include <sys/syslog.h> 48*2b3ad188SAdrian Chadd #include <sys/malloc.h> 49*2b3ad188SAdrian Chadd #include <sys/proc.h> 50*2b3ad188SAdrian Chadd #include <sys/queue.h> 51*2b3ad188SAdrian Chadd #include <sys/bus.h> 52*2b3ad188SAdrian Chadd #include <sys/interrupt.h> 53*2b3ad188SAdrian Chadd #include <sys/conf.h> 54*2b3ad188SAdrian Chadd #include <sys/cpuset.h> 55*2b3ad188SAdrian Chadd #include <sys/sched.h> 56*2b3ad188SAdrian Chadd #include <sys/smp.h> 57*2b3ad188SAdrian Chadd #include <machine/atomic.h> 58*2b3ad188SAdrian Chadd #include <machine/intr.h> 59*2b3ad188SAdrian Chadd #include <machine/cpu.h> 60*2b3ad188SAdrian Chadd #include <machine/smp.h> 61*2b3ad188SAdrian Chadd #include <machine/stdarg.h> 62*2b3ad188SAdrian Chadd 63*2b3ad188SAdrian Chadd #include <dev/ofw/openfirm.h> 64*2b3ad188SAdrian Chadd #include <dev/ofw/ofw_bus.h> 65*2b3ad188SAdrian Chadd #include <dev/ofw/ofw_bus_subr.h> 66*2b3ad188SAdrian Chadd 67*2b3ad188SAdrian Chadd #include <dev/fdt/fdt_common.h> 68*2b3ad188SAdrian Chadd 69*2b3ad188SAdrian Chadd #ifdef DDB 70*2b3ad188SAdrian Chadd #include <ddb/ddb.h> 71*2b3ad188SAdrian Chadd #endif 72*2b3ad188SAdrian Chadd 73*2b3ad188SAdrian Chadd #include "pic_if.h" 74*2b3ad188SAdrian Chadd 75*2b3ad188SAdrian Chadd #define INTRNAME_LEN (2*MAXCOMLEN + 1) 76*2b3ad188SAdrian Chadd 77*2b3ad188SAdrian Chadd #ifdef DEBUG 78*2b3ad188SAdrian Chadd #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 79*2b3ad188SAdrian Chadd printf(fmt,##args); } while (0) 80*2b3ad188SAdrian Chadd #else 81*2b3ad188SAdrian Chadd #define debugf(fmt, args...) 82*2b3ad188SAdrian Chadd #endif 83*2b3ad188SAdrian Chadd 84*2b3ad188SAdrian Chadd MALLOC_DECLARE(M_INTRNG); 85*2b3ad188SAdrian Chadd MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 86*2b3ad188SAdrian Chadd 87*2b3ad188SAdrian Chadd /* Main interrupt handler called from assembler -> 'hidden' for C code. */ 88*2b3ad188SAdrian Chadd void intr_irq_handler(struct trapframe *tf); 89*2b3ad188SAdrian Chadd 90*2b3ad188SAdrian Chadd /* Root interrupt controller stuff. */ 91*2b3ad188SAdrian Chadd static struct intr_irqsrc *irq_root_isrc; 92*2b3ad188SAdrian Chadd static device_t irq_root_dev; 93*2b3ad188SAdrian Chadd static intr_irq_filter_t *irq_root_filter; 94*2b3ad188SAdrian Chadd static void *irq_root_arg; 95*2b3ad188SAdrian Chadd static u_int irq_root_ipicount; 96*2b3ad188SAdrian Chadd 97*2b3ad188SAdrian Chadd /* Interrupt controller definition. */ 98*2b3ad188SAdrian Chadd struct intr_pic { 99*2b3ad188SAdrian Chadd SLIST_ENTRY(intr_pic) pic_next; 100*2b3ad188SAdrian Chadd intptr_t pic_xref; /* hardware identification */ 101*2b3ad188SAdrian Chadd device_t pic_dev; 102*2b3ad188SAdrian Chadd }; 103*2b3ad188SAdrian Chadd 104*2b3ad188SAdrian Chadd static struct mtx pic_list_lock; 105*2b3ad188SAdrian Chadd static SLIST_HEAD(, intr_pic) pic_list; 106*2b3ad188SAdrian Chadd 107*2b3ad188SAdrian Chadd static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); 108*2b3ad188SAdrian Chadd 109*2b3ad188SAdrian Chadd /* Interrupt source definition. */ 110*2b3ad188SAdrian Chadd static struct mtx isrc_table_lock; 111*2b3ad188SAdrian Chadd static struct intr_irqsrc *irq_sources[NIRQ]; 112*2b3ad188SAdrian Chadd u_int irq_next_free; 113*2b3ad188SAdrian Chadd 114*2b3ad188SAdrian Chadd #define IRQ_INVALID nitems(irq_sources) 115*2b3ad188SAdrian Chadd 116*2b3ad188SAdrian Chadd #ifdef SMP 117*2b3ad188SAdrian Chadd static boolean_t irq_assign_cpu = FALSE; 118*2b3ad188SAdrian Chadd 119*2b3ad188SAdrian Chadd static struct intr_irqsrc ipi_sources[INTR_IPI_COUNT]; 120*2b3ad188SAdrian Chadd static u_int ipi_next_num; 121*2b3ad188SAdrian Chadd #endif 122*2b3ad188SAdrian Chadd 123*2b3ad188SAdrian Chadd /* 124*2b3ad188SAdrian Chadd * - 2 counters for each I/O interrupt. 125*2b3ad188SAdrian Chadd * - MAXCPU counters for each IPI counters for SMP. 126*2b3ad188SAdrian Chadd */ 127*2b3ad188SAdrian Chadd #ifdef SMP 128*2b3ad188SAdrian Chadd #define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) 129*2b3ad188SAdrian Chadd #else 130*2b3ad188SAdrian Chadd #define INTRCNT_COUNT (NIRQ * 2) 131*2b3ad188SAdrian Chadd #endif 132*2b3ad188SAdrian Chadd 133*2b3ad188SAdrian Chadd /* Data for MI statistics reporting. */ 134*2b3ad188SAdrian Chadd u_long intrcnt[INTRCNT_COUNT]; 135*2b3ad188SAdrian Chadd char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; 136*2b3ad188SAdrian Chadd size_t sintrcnt = sizeof(intrcnt); 137*2b3ad188SAdrian Chadd size_t sintrnames = sizeof(intrnames); 138*2b3ad188SAdrian Chadd static u_int intrcnt_index; 139*2b3ad188SAdrian Chadd 140*2b3ad188SAdrian Chadd /* 141*2b3ad188SAdrian Chadd * Interrupt framework initialization routine. 142*2b3ad188SAdrian Chadd */ 143*2b3ad188SAdrian Chadd static void 144*2b3ad188SAdrian Chadd intr_irq_init(void *dummy __unused) 145*2b3ad188SAdrian Chadd { 146*2b3ad188SAdrian Chadd 147*2b3ad188SAdrian Chadd SLIST_INIT(&pic_list); 148*2b3ad188SAdrian Chadd mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 149*2b3ad188SAdrian Chadd mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 150*2b3ad188SAdrian Chadd } 151*2b3ad188SAdrian Chadd SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 152*2b3ad188SAdrian Chadd 153*2b3ad188SAdrian Chadd static void 154*2b3ad188SAdrian Chadd intrcnt_setname(const char *name, int index) 155*2b3ad188SAdrian Chadd { 156*2b3ad188SAdrian Chadd 157*2b3ad188SAdrian Chadd snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 158*2b3ad188SAdrian Chadd INTRNAME_LEN - 1, name); 159*2b3ad188SAdrian Chadd } 160*2b3ad188SAdrian Chadd 161*2b3ad188SAdrian Chadd /* 162*2b3ad188SAdrian Chadd * Update name for interrupt source with interrupt event. 163*2b3ad188SAdrian Chadd */ 164*2b3ad188SAdrian Chadd static void 165*2b3ad188SAdrian Chadd intrcnt_updatename(struct intr_irqsrc *isrc) 166*2b3ad188SAdrian Chadd { 167*2b3ad188SAdrian Chadd 168*2b3ad188SAdrian Chadd /* QQQ: What about stray counter name? */ 169*2b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 170*2b3ad188SAdrian Chadd intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 171*2b3ad188SAdrian Chadd } 172*2b3ad188SAdrian Chadd 173*2b3ad188SAdrian Chadd /* 174*2b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt counter increment. 175*2b3ad188SAdrian Chadd */ 176*2b3ad188SAdrian Chadd static inline void 177*2b3ad188SAdrian Chadd isrc_increment_count(struct intr_irqsrc *isrc) 178*2b3ad188SAdrian Chadd { 179*2b3ad188SAdrian Chadd 180*2b3ad188SAdrian Chadd /* 181*2b3ad188SAdrian Chadd * XXX - It should be atomic for PPI interrupts. It was proven that 182*2b3ad188SAdrian Chadd * the lost is measurable easily for timer PPI interrupts. 183*2b3ad188SAdrian Chadd */ 184*2b3ad188SAdrian Chadd isrc->isrc_count[0]++; 185*2b3ad188SAdrian Chadd /*atomic_add_long(&isrc->isrc_count[0], 1);*/ 186*2b3ad188SAdrian Chadd } 187*2b3ad188SAdrian Chadd 188*2b3ad188SAdrian Chadd /* 189*2b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt stray counter increment. 190*2b3ad188SAdrian Chadd */ 191*2b3ad188SAdrian Chadd static inline void 192*2b3ad188SAdrian Chadd isrc_increment_straycount(struct intr_irqsrc *isrc) 193*2b3ad188SAdrian Chadd { 194*2b3ad188SAdrian Chadd 195*2b3ad188SAdrian Chadd isrc->isrc_count[1]++; 196*2b3ad188SAdrian Chadd } 197*2b3ad188SAdrian Chadd 198*2b3ad188SAdrian Chadd /* 199*2b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt name update. 200*2b3ad188SAdrian Chadd */ 201*2b3ad188SAdrian Chadd static void 202*2b3ad188SAdrian Chadd isrc_update_name(struct intr_irqsrc *isrc, const char *name) 203*2b3ad188SAdrian Chadd { 204*2b3ad188SAdrian Chadd char str[INTRNAME_LEN]; 205*2b3ad188SAdrian Chadd 206*2b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 207*2b3ad188SAdrian Chadd 208*2b3ad188SAdrian Chadd if (name != NULL) { 209*2b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 210*2b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index); 211*2b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 212*2b3ad188SAdrian Chadd name); 213*2b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index + 1); 214*2b3ad188SAdrian Chadd } else { 215*2b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 216*2b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index); 217*2b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 218*2b3ad188SAdrian Chadd intrcnt_setname(str, isrc->isrc_index + 1); 219*2b3ad188SAdrian Chadd } 220*2b3ad188SAdrian Chadd } 221*2b3ad188SAdrian Chadd 222*2b3ad188SAdrian Chadd /* 223*2b3ad188SAdrian Chadd * Virtualization for interrupt source interrupt counters setup. 224*2b3ad188SAdrian Chadd */ 225*2b3ad188SAdrian Chadd static void 226*2b3ad188SAdrian Chadd isrc_setup_counters(struct intr_irqsrc *isrc) 227*2b3ad188SAdrian Chadd { 228*2b3ad188SAdrian Chadd u_int index; 229*2b3ad188SAdrian Chadd 230*2b3ad188SAdrian Chadd /* 231*2b3ad188SAdrian Chadd * XXX - it does not work well with removable controllers and 232*2b3ad188SAdrian Chadd * interrupt sources !!! 233*2b3ad188SAdrian Chadd */ 234*2b3ad188SAdrian Chadd index = atomic_fetchadd_int(&intrcnt_index, 2); 235*2b3ad188SAdrian Chadd isrc->isrc_index = index; 236*2b3ad188SAdrian Chadd isrc->isrc_count = &intrcnt[index]; 237*2b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 238*2b3ad188SAdrian Chadd } 239*2b3ad188SAdrian Chadd 240*2b3ad188SAdrian Chadd #ifdef SMP 241*2b3ad188SAdrian Chadd /* 242*2b3ad188SAdrian Chadd * Virtualization for interrupt source IPI counter increment. 243*2b3ad188SAdrian Chadd */ 244*2b3ad188SAdrian Chadd static inline void 245*2b3ad188SAdrian Chadd isrc_increment_ipi_count(struct intr_irqsrc *isrc, u_int cpu) 246*2b3ad188SAdrian Chadd { 247*2b3ad188SAdrian Chadd 248*2b3ad188SAdrian Chadd isrc->isrc_count[cpu]++; 249*2b3ad188SAdrian Chadd } 250*2b3ad188SAdrian Chadd 251*2b3ad188SAdrian Chadd /* 252*2b3ad188SAdrian Chadd * Virtualization for interrupt source IPI counters setup. 253*2b3ad188SAdrian Chadd */ 254*2b3ad188SAdrian Chadd static void 255*2b3ad188SAdrian Chadd isrc_setup_ipi_counters(struct intr_irqsrc *isrc, const char *name) 256*2b3ad188SAdrian Chadd { 257*2b3ad188SAdrian Chadd u_int index, i; 258*2b3ad188SAdrian Chadd char str[INTRNAME_LEN]; 259*2b3ad188SAdrian Chadd 260*2b3ad188SAdrian Chadd index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); 261*2b3ad188SAdrian Chadd isrc->isrc_index = index; 262*2b3ad188SAdrian Chadd isrc->isrc_count = &intrcnt[index]; 263*2b3ad188SAdrian Chadd 264*2b3ad188SAdrian Chadd for (i = 0; i < MAXCPU; i++) { 265*2b3ad188SAdrian Chadd /* 266*2b3ad188SAdrian Chadd * We do not expect any race in IPI case here, 267*2b3ad188SAdrian Chadd * so locking is not needed. 268*2b3ad188SAdrian Chadd */ 269*2b3ad188SAdrian Chadd snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 270*2b3ad188SAdrian Chadd intrcnt_setname(str, index + i); 271*2b3ad188SAdrian Chadd } 272*2b3ad188SAdrian Chadd } 273*2b3ad188SAdrian Chadd #endif 274*2b3ad188SAdrian Chadd 275*2b3ad188SAdrian Chadd /* 276*2b3ad188SAdrian Chadd * Main interrupt dispatch handler. It's called straight 277*2b3ad188SAdrian Chadd * from the assembler, where CPU interrupt is served. 278*2b3ad188SAdrian Chadd */ 279*2b3ad188SAdrian Chadd void 280*2b3ad188SAdrian Chadd intr_irq_handler(struct trapframe *tf) 281*2b3ad188SAdrian Chadd { 282*2b3ad188SAdrian Chadd struct trapframe * oldframe; 283*2b3ad188SAdrian Chadd struct thread * td; 284*2b3ad188SAdrian Chadd 285*2b3ad188SAdrian Chadd KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); 286*2b3ad188SAdrian Chadd 287*2b3ad188SAdrian Chadd PCPU_INC(cnt.v_intr); 288*2b3ad188SAdrian Chadd critical_enter(); 289*2b3ad188SAdrian Chadd td = curthread; 290*2b3ad188SAdrian Chadd oldframe = td->td_intr_frame; 291*2b3ad188SAdrian Chadd td->td_intr_frame = tf; 292*2b3ad188SAdrian Chadd irq_root_filter(irq_root_arg); 293*2b3ad188SAdrian Chadd td->td_intr_frame = oldframe; 294*2b3ad188SAdrian Chadd critical_exit(); 295*2b3ad188SAdrian Chadd } 296*2b3ad188SAdrian Chadd 297*2b3ad188SAdrian Chadd /* 298*2b3ad188SAdrian Chadd * interrupt controller dispatch function for interrupts. It should 299*2b3ad188SAdrian Chadd * be called straight from the interrupt controller, when associated interrupt 300*2b3ad188SAdrian Chadd * source is learned. 301*2b3ad188SAdrian Chadd */ 302*2b3ad188SAdrian Chadd void 303*2b3ad188SAdrian Chadd intr_irq_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 304*2b3ad188SAdrian Chadd { 305*2b3ad188SAdrian Chadd 306*2b3ad188SAdrian Chadd KASSERT(isrc != NULL, ("%s: no source", __func__)); 307*2b3ad188SAdrian Chadd 308*2b3ad188SAdrian Chadd isrc_increment_count(isrc); 309*2b3ad188SAdrian Chadd 310*2b3ad188SAdrian Chadd #ifdef INTR_SOLO 311*2b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 312*2b3ad188SAdrian Chadd int error; 313*2b3ad188SAdrian Chadd error = isrc->isrc_filter(isrc->isrc_arg, tf); 314*2b3ad188SAdrian Chadd PIC_POST_FILTER(isrc->isrc_dev, isrc); 315*2b3ad188SAdrian Chadd if (error == FILTER_HANDLED) 316*2b3ad188SAdrian Chadd return; 317*2b3ad188SAdrian Chadd } else 318*2b3ad188SAdrian Chadd #endif 319*2b3ad188SAdrian Chadd if (isrc->isrc_event != NULL) { 320*2b3ad188SAdrian Chadd if (intr_event_handle(isrc->isrc_event, tf) == 0) 321*2b3ad188SAdrian Chadd return; 322*2b3ad188SAdrian Chadd } 323*2b3ad188SAdrian Chadd 324*2b3ad188SAdrian Chadd isrc_increment_straycount(isrc); 325*2b3ad188SAdrian Chadd PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc); 326*2b3ad188SAdrian Chadd 327*2b3ad188SAdrian Chadd device_printf(isrc->isrc_dev, "stray irq <%s> disabled", 328*2b3ad188SAdrian Chadd isrc->isrc_name); 329*2b3ad188SAdrian Chadd } 330*2b3ad188SAdrian Chadd 331*2b3ad188SAdrian Chadd /* 332*2b3ad188SAdrian Chadd * Allocate interrupt source. 333*2b3ad188SAdrian Chadd */ 334*2b3ad188SAdrian Chadd static struct intr_irqsrc * 335*2b3ad188SAdrian Chadd isrc_alloc(u_int type, u_int extsize) 336*2b3ad188SAdrian Chadd { 337*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 338*2b3ad188SAdrian Chadd 339*2b3ad188SAdrian Chadd isrc = malloc(sizeof(*isrc) + extsize, M_INTRNG, M_WAITOK | M_ZERO); 340*2b3ad188SAdrian Chadd isrc->isrc_irq = IRQ_INVALID; /* just to be safe */ 341*2b3ad188SAdrian Chadd isrc->isrc_type = type; 342*2b3ad188SAdrian Chadd isrc->isrc_nspc_type = INTR_IRQ_NSPC_NONE; 343*2b3ad188SAdrian Chadd isrc->isrc_trig = INTR_TRIGGER_CONFORM; 344*2b3ad188SAdrian Chadd isrc->isrc_pol = INTR_POLARITY_CONFORM; 345*2b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 346*2b3ad188SAdrian Chadd return (isrc); 347*2b3ad188SAdrian Chadd } 348*2b3ad188SAdrian Chadd 349*2b3ad188SAdrian Chadd /* 350*2b3ad188SAdrian Chadd * Free interrupt source. 351*2b3ad188SAdrian Chadd */ 352*2b3ad188SAdrian Chadd static void 353*2b3ad188SAdrian Chadd isrc_free(struct intr_irqsrc *isrc) 354*2b3ad188SAdrian Chadd { 355*2b3ad188SAdrian Chadd 356*2b3ad188SAdrian Chadd free(isrc, M_INTRNG); 357*2b3ad188SAdrian Chadd } 358*2b3ad188SAdrian Chadd 359*2b3ad188SAdrian Chadd void 360*2b3ad188SAdrian Chadd intr_irq_set_name(struct intr_irqsrc *isrc, const char *fmt, ...) 361*2b3ad188SAdrian Chadd { 362*2b3ad188SAdrian Chadd va_list ap; 363*2b3ad188SAdrian Chadd 364*2b3ad188SAdrian Chadd va_start(ap, fmt); 365*2b3ad188SAdrian Chadd vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 366*2b3ad188SAdrian Chadd va_end(ap); 367*2b3ad188SAdrian Chadd } 368*2b3ad188SAdrian Chadd 369*2b3ad188SAdrian Chadd /* 370*2b3ad188SAdrian Chadd * Alloc unique interrupt number (resource handle) for interrupt source. 371*2b3ad188SAdrian Chadd * 372*2b3ad188SAdrian Chadd * There could be various strategies how to allocate free interrupt number 373*2b3ad188SAdrian Chadd * (resource handle) for new interrupt source. 374*2b3ad188SAdrian Chadd * 375*2b3ad188SAdrian Chadd * 1. Handles are always allocated forward, so handles are not recycled 376*2b3ad188SAdrian Chadd * immediately. However, if only one free handle left which is reused 377*2b3ad188SAdrian Chadd * constantly... 378*2b3ad188SAdrian Chadd */ 379*2b3ad188SAdrian Chadd static int 380*2b3ad188SAdrian Chadd isrc_alloc_irq_locked(struct intr_irqsrc *isrc) 381*2b3ad188SAdrian Chadd { 382*2b3ad188SAdrian Chadd u_int maxirqs, irq; 383*2b3ad188SAdrian Chadd 384*2b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 385*2b3ad188SAdrian Chadd 386*2b3ad188SAdrian Chadd maxirqs = nitems(irq_sources); 387*2b3ad188SAdrian Chadd if (irq_next_free >= maxirqs) 388*2b3ad188SAdrian Chadd return (ENOSPC); 389*2b3ad188SAdrian Chadd 390*2b3ad188SAdrian Chadd for (irq = irq_next_free; irq < maxirqs; irq++) { 391*2b3ad188SAdrian Chadd if (irq_sources[irq] == NULL) 392*2b3ad188SAdrian Chadd goto found; 393*2b3ad188SAdrian Chadd } 394*2b3ad188SAdrian Chadd for (irq = 0; irq < irq_next_free; irq++) { 395*2b3ad188SAdrian Chadd if (irq_sources[irq] == NULL) 396*2b3ad188SAdrian Chadd goto found; 397*2b3ad188SAdrian Chadd } 398*2b3ad188SAdrian Chadd 399*2b3ad188SAdrian Chadd irq_next_free = maxirqs; 400*2b3ad188SAdrian Chadd return (ENOSPC); 401*2b3ad188SAdrian Chadd 402*2b3ad188SAdrian Chadd found: 403*2b3ad188SAdrian Chadd isrc->isrc_irq = irq; 404*2b3ad188SAdrian Chadd irq_sources[irq] = isrc; 405*2b3ad188SAdrian Chadd 406*2b3ad188SAdrian Chadd intr_irq_set_name(isrc, "irq%u", irq); 407*2b3ad188SAdrian Chadd isrc_setup_counters(isrc); 408*2b3ad188SAdrian Chadd 409*2b3ad188SAdrian Chadd irq_next_free = irq + 1; 410*2b3ad188SAdrian Chadd if (irq_next_free >= maxirqs) 411*2b3ad188SAdrian Chadd irq_next_free = 0; 412*2b3ad188SAdrian Chadd return (0); 413*2b3ad188SAdrian Chadd } 414*2b3ad188SAdrian Chadd #ifdef notyet 415*2b3ad188SAdrian Chadd /* 416*2b3ad188SAdrian Chadd * Free unique interrupt number (resource handle) from interrupt source. 417*2b3ad188SAdrian Chadd */ 418*2b3ad188SAdrian Chadd static int 419*2b3ad188SAdrian Chadd isrc_free_irq(struct intr_irqsrc *isrc) 420*2b3ad188SAdrian Chadd { 421*2b3ad188SAdrian Chadd u_int maxirqs; 422*2b3ad188SAdrian Chadd 423*2b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_NOTOWNED); 424*2b3ad188SAdrian Chadd 425*2b3ad188SAdrian Chadd maxirqs = nitems(irq_sources); 426*2b3ad188SAdrian Chadd if (isrc->isrc_irq >= maxirqs) 427*2b3ad188SAdrian Chadd return (EINVAL); 428*2b3ad188SAdrian Chadd 429*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 430*2b3ad188SAdrian Chadd if (irq_sources[isrc->isrc_irq] != isrc) { 431*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 432*2b3ad188SAdrian Chadd return (EINVAL); 433*2b3ad188SAdrian Chadd } 434*2b3ad188SAdrian Chadd 435*2b3ad188SAdrian Chadd irq_sources[isrc->isrc_irq] = NULL; 436*2b3ad188SAdrian Chadd isrc->isrc_irq = IRQ_INVALID; /* just to be safe */ 437*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 438*2b3ad188SAdrian Chadd 439*2b3ad188SAdrian Chadd return (0); 440*2b3ad188SAdrian Chadd } 441*2b3ad188SAdrian Chadd #endif 442*2b3ad188SAdrian Chadd /* 443*2b3ad188SAdrian Chadd * Lookup interrupt source by interrupt number (resource handle). 444*2b3ad188SAdrian Chadd */ 445*2b3ad188SAdrian Chadd static struct intr_irqsrc * 446*2b3ad188SAdrian Chadd isrc_lookup(u_int irq) 447*2b3ad188SAdrian Chadd { 448*2b3ad188SAdrian Chadd 449*2b3ad188SAdrian Chadd if (irq < nitems(irq_sources)) 450*2b3ad188SAdrian Chadd return (irq_sources[irq]); 451*2b3ad188SAdrian Chadd return (NULL); 452*2b3ad188SAdrian Chadd } 453*2b3ad188SAdrian Chadd 454*2b3ad188SAdrian Chadd /* 455*2b3ad188SAdrian Chadd * Lookup interrupt source by namespace description. 456*2b3ad188SAdrian Chadd */ 457*2b3ad188SAdrian Chadd static struct intr_irqsrc * 458*2b3ad188SAdrian Chadd isrc_namespace_lookup(device_t dev, uint16_t type, uint16_t num) 459*2b3ad188SAdrian Chadd { 460*2b3ad188SAdrian Chadd u_int irq; 461*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 462*2b3ad188SAdrian Chadd 463*2b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 464*2b3ad188SAdrian Chadd 465*2b3ad188SAdrian Chadd for (irq = 0; irq < nitems(irq_sources); irq++) { 466*2b3ad188SAdrian Chadd isrc = irq_sources[irq]; 467*2b3ad188SAdrian Chadd if (isrc != NULL && isrc->isrc_dev == dev && 468*2b3ad188SAdrian Chadd isrc->isrc_nspc_type == type && isrc->isrc_nspc_num == num) 469*2b3ad188SAdrian Chadd return (isrc); 470*2b3ad188SAdrian Chadd } 471*2b3ad188SAdrian Chadd return (NULL); 472*2b3ad188SAdrian Chadd } 473*2b3ad188SAdrian Chadd 474*2b3ad188SAdrian Chadd /* 475*2b3ad188SAdrian Chadd * Map interrupt source according to namespace into framework. If such mapping 476*2b3ad188SAdrian Chadd * does not exist, create it. Return unique interrupt number (resource handle) 477*2b3ad188SAdrian Chadd * associated with mapped interrupt source. 478*2b3ad188SAdrian Chadd */ 479*2b3ad188SAdrian Chadd u_int 480*2b3ad188SAdrian Chadd intr_namespace_map_irq(device_t dev, uint16_t type, uint16_t num) 481*2b3ad188SAdrian Chadd { 482*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc, *new_isrc; 483*2b3ad188SAdrian Chadd int error; 484*2b3ad188SAdrian Chadd 485*2b3ad188SAdrian Chadd new_isrc = isrc_alloc(INTR_ISRCT_NAMESPACE, 0); 486*2b3ad188SAdrian Chadd 487*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 488*2b3ad188SAdrian Chadd isrc = isrc_namespace_lookup(dev, type, num); 489*2b3ad188SAdrian Chadd if (isrc != NULL) { 490*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 491*2b3ad188SAdrian Chadd isrc_free(new_isrc); 492*2b3ad188SAdrian Chadd return (isrc->isrc_irq); /* already mapped */ 493*2b3ad188SAdrian Chadd } 494*2b3ad188SAdrian Chadd 495*2b3ad188SAdrian Chadd error = isrc_alloc_irq_locked(new_isrc); 496*2b3ad188SAdrian Chadd if (error != 0) { 497*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 498*2b3ad188SAdrian Chadd isrc_free(new_isrc); 499*2b3ad188SAdrian Chadd return (IRQ_INVALID); /* no space left */ 500*2b3ad188SAdrian Chadd } 501*2b3ad188SAdrian Chadd 502*2b3ad188SAdrian Chadd new_isrc->isrc_dev = dev; 503*2b3ad188SAdrian Chadd new_isrc->isrc_nspc_type = type; 504*2b3ad188SAdrian Chadd new_isrc->isrc_nspc_num = num; 505*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 506*2b3ad188SAdrian Chadd 507*2b3ad188SAdrian Chadd return (new_isrc->isrc_irq); 508*2b3ad188SAdrian Chadd } 509*2b3ad188SAdrian Chadd 510*2b3ad188SAdrian Chadd #ifdef FDT 511*2b3ad188SAdrian Chadd /* 512*2b3ad188SAdrian Chadd * Lookup interrupt source by FDT description. 513*2b3ad188SAdrian Chadd */ 514*2b3ad188SAdrian Chadd static struct intr_irqsrc * 515*2b3ad188SAdrian Chadd isrc_fdt_lookup(intptr_t xref, pcell_t *cells, u_int ncells) 516*2b3ad188SAdrian Chadd { 517*2b3ad188SAdrian Chadd u_int irq, cellsize; 518*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 519*2b3ad188SAdrian Chadd 520*2b3ad188SAdrian Chadd mtx_assert(&isrc_table_lock, MA_OWNED); 521*2b3ad188SAdrian Chadd 522*2b3ad188SAdrian Chadd cellsize = ncells * sizeof(*cells); 523*2b3ad188SAdrian Chadd for (irq = 0; irq < nitems(irq_sources); irq++) { 524*2b3ad188SAdrian Chadd isrc = irq_sources[irq]; 525*2b3ad188SAdrian Chadd if (isrc != NULL && isrc->isrc_type == INTR_ISRCT_FDT && 526*2b3ad188SAdrian Chadd isrc->isrc_xref == xref && isrc->isrc_ncells == ncells && 527*2b3ad188SAdrian Chadd memcmp(isrc->isrc_cells, cells, cellsize) == 0) 528*2b3ad188SAdrian Chadd return (isrc); 529*2b3ad188SAdrian Chadd } 530*2b3ad188SAdrian Chadd return (NULL); 531*2b3ad188SAdrian Chadd } 532*2b3ad188SAdrian Chadd 533*2b3ad188SAdrian Chadd /* 534*2b3ad188SAdrian Chadd * Map interrupt source according to FDT data into framework. If such mapping 535*2b3ad188SAdrian Chadd * does not exist, create it. Return unique interrupt number (resource handle) 536*2b3ad188SAdrian Chadd * associated with mapped interrupt source. 537*2b3ad188SAdrian Chadd */ 538*2b3ad188SAdrian Chadd u_int 539*2b3ad188SAdrian Chadd intr_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) 540*2b3ad188SAdrian Chadd { 541*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc, *new_isrc; 542*2b3ad188SAdrian Chadd u_int cellsize; 543*2b3ad188SAdrian Chadd intptr_t xref; 544*2b3ad188SAdrian Chadd int error; 545*2b3ad188SAdrian Chadd 546*2b3ad188SAdrian Chadd xref = (intptr_t)node; /* It's so simple for now. */ 547*2b3ad188SAdrian Chadd 548*2b3ad188SAdrian Chadd cellsize = ncells * sizeof(*cells); 549*2b3ad188SAdrian Chadd new_isrc = isrc_alloc(INTR_ISRCT_FDT, cellsize); 550*2b3ad188SAdrian Chadd 551*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 552*2b3ad188SAdrian Chadd isrc = isrc_fdt_lookup(xref, cells, ncells); 553*2b3ad188SAdrian Chadd if (isrc != NULL) { 554*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 555*2b3ad188SAdrian Chadd isrc_free(new_isrc); 556*2b3ad188SAdrian Chadd return (isrc->isrc_irq); /* already mapped */ 557*2b3ad188SAdrian Chadd } 558*2b3ad188SAdrian Chadd 559*2b3ad188SAdrian Chadd error = isrc_alloc_irq_locked(new_isrc); 560*2b3ad188SAdrian Chadd if (error != 0) { 561*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 562*2b3ad188SAdrian Chadd isrc_free(new_isrc); 563*2b3ad188SAdrian Chadd return (IRQ_INVALID); /* no space left */ 564*2b3ad188SAdrian Chadd } 565*2b3ad188SAdrian Chadd 566*2b3ad188SAdrian Chadd new_isrc->isrc_xref = xref; 567*2b3ad188SAdrian Chadd new_isrc->isrc_ncells = ncells; 568*2b3ad188SAdrian Chadd memcpy(new_isrc->isrc_cells, cells, cellsize); 569*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 570*2b3ad188SAdrian Chadd 571*2b3ad188SAdrian Chadd return (new_isrc->isrc_irq); 572*2b3ad188SAdrian Chadd } 573*2b3ad188SAdrian Chadd #endif 574*2b3ad188SAdrian Chadd 575*2b3ad188SAdrian Chadd /* 576*2b3ad188SAdrian Chadd * Register interrupt source into interrupt controller. 577*2b3ad188SAdrian Chadd */ 578*2b3ad188SAdrian Chadd static int 579*2b3ad188SAdrian Chadd isrc_register(struct intr_irqsrc *isrc) 580*2b3ad188SAdrian Chadd { 581*2b3ad188SAdrian Chadd struct intr_pic *pic; 582*2b3ad188SAdrian Chadd boolean_t is_percpu; 583*2b3ad188SAdrian Chadd int error; 584*2b3ad188SAdrian Chadd 585*2b3ad188SAdrian Chadd if (isrc->isrc_flags & INTR_ISRCF_REGISTERED) 586*2b3ad188SAdrian Chadd return (0); 587*2b3ad188SAdrian Chadd 588*2b3ad188SAdrian Chadd if (isrc->isrc_dev == NULL) { 589*2b3ad188SAdrian Chadd pic = pic_lookup(NULL, isrc->isrc_xref); 590*2b3ad188SAdrian Chadd if (pic == NULL || pic->pic_dev == NULL) 591*2b3ad188SAdrian Chadd return (ESRCH); 592*2b3ad188SAdrian Chadd isrc->isrc_dev = pic->pic_dev; 593*2b3ad188SAdrian Chadd } 594*2b3ad188SAdrian Chadd 595*2b3ad188SAdrian Chadd error = PIC_REGISTER(isrc->isrc_dev, isrc, &is_percpu); 596*2b3ad188SAdrian Chadd if (error != 0) 597*2b3ad188SAdrian Chadd return (error); 598*2b3ad188SAdrian Chadd 599*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 600*2b3ad188SAdrian Chadd isrc->isrc_flags |= INTR_ISRCF_REGISTERED; 601*2b3ad188SAdrian Chadd if (is_percpu) 602*2b3ad188SAdrian Chadd isrc->isrc_flags |= INTR_ISRCF_PERCPU; 603*2b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 604*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 605*2b3ad188SAdrian Chadd return (0); 606*2b3ad188SAdrian Chadd } 607*2b3ad188SAdrian Chadd 608*2b3ad188SAdrian Chadd #ifdef INTR_SOLO 609*2b3ad188SAdrian Chadd /* 610*2b3ad188SAdrian Chadd * Setup filter into interrupt source. 611*2b3ad188SAdrian Chadd */ 612*2b3ad188SAdrian Chadd static int 613*2b3ad188SAdrian Chadd iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 614*2b3ad188SAdrian Chadd intr_irq_filter_t *filter, void *arg, void **cookiep) 615*2b3ad188SAdrian Chadd { 616*2b3ad188SAdrian Chadd 617*2b3ad188SAdrian Chadd if (filter == NULL) 618*2b3ad188SAdrian Chadd return (EINVAL); 619*2b3ad188SAdrian Chadd 620*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 621*2b3ad188SAdrian Chadd /* 622*2b3ad188SAdrian Chadd * Make sure that we do not mix the two ways 623*2b3ad188SAdrian Chadd * how we handle interrupt sources. 624*2b3ad188SAdrian Chadd */ 625*2b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 626*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 627*2b3ad188SAdrian Chadd return (EBUSY); 628*2b3ad188SAdrian Chadd } 629*2b3ad188SAdrian Chadd isrc->isrc_filter = filter; 630*2b3ad188SAdrian Chadd isrc->isrc_arg = arg; 631*2b3ad188SAdrian Chadd isrc_update_name(isrc, name); 632*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 633*2b3ad188SAdrian Chadd 634*2b3ad188SAdrian Chadd *cookiep = isrc; 635*2b3ad188SAdrian Chadd return (0); 636*2b3ad188SAdrian Chadd } 637*2b3ad188SAdrian Chadd #endif 638*2b3ad188SAdrian Chadd 639*2b3ad188SAdrian Chadd /* 640*2b3ad188SAdrian Chadd * Interrupt source pre_ithread method for MI interrupt framework. 641*2b3ad188SAdrian Chadd */ 642*2b3ad188SAdrian Chadd static void 643*2b3ad188SAdrian Chadd intr_isrc_pre_ithread(void *arg) 644*2b3ad188SAdrian Chadd { 645*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 646*2b3ad188SAdrian Chadd 647*2b3ad188SAdrian Chadd PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 648*2b3ad188SAdrian Chadd } 649*2b3ad188SAdrian Chadd 650*2b3ad188SAdrian Chadd /* 651*2b3ad188SAdrian Chadd * Interrupt source post_ithread method for MI interrupt framework. 652*2b3ad188SAdrian Chadd */ 653*2b3ad188SAdrian Chadd static void 654*2b3ad188SAdrian Chadd intr_isrc_post_ithread(void *arg) 655*2b3ad188SAdrian Chadd { 656*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 657*2b3ad188SAdrian Chadd 658*2b3ad188SAdrian Chadd PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 659*2b3ad188SAdrian Chadd } 660*2b3ad188SAdrian Chadd 661*2b3ad188SAdrian Chadd /* 662*2b3ad188SAdrian Chadd * Interrupt source post_filter method for MI interrupt framework. 663*2b3ad188SAdrian Chadd */ 664*2b3ad188SAdrian Chadd static void 665*2b3ad188SAdrian Chadd intr_isrc_post_filter(void *arg) 666*2b3ad188SAdrian Chadd { 667*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 668*2b3ad188SAdrian Chadd 669*2b3ad188SAdrian Chadd PIC_POST_FILTER(isrc->isrc_dev, isrc); 670*2b3ad188SAdrian Chadd } 671*2b3ad188SAdrian Chadd 672*2b3ad188SAdrian Chadd /* 673*2b3ad188SAdrian Chadd * Interrupt source assign_cpu method for MI interrupt framework. 674*2b3ad188SAdrian Chadd */ 675*2b3ad188SAdrian Chadd static int 676*2b3ad188SAdrian Chadd intr_isrc_assign_cpu(void *arg, int cpu) 677*2b3ad188SAdrian Chadd { 678*2b3ad188SAdrian Chadd #ifdef SMP 679*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc = arg; 680*2b3ad188SAdrian Chadd int error; 681*2b3ad188SAdrian Chadd 682*2b3ad188SAdrian Chadd if (isrc->isrc_dev != irq_root_dev) 683*2b3ad188SAdrian Chadd return (EINVAL); 684*2b3ad188SAdrian Chadd 685*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 686*2b3ad188SAdrian Chadd if (cpu == NOCPU) { 687*2b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 688*2b3ad188SAdrian Chadd isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 689*2b3ad188SAdrian Chadd } else { 690*2b3ad188SAdrian Chadd CPU_SETOF(cpu, &isrc->isrc_cpu); 691*2b3ad188SAdrian Chadd isrc->isrc_flags |= INTR_ISRCF_BOUND; 692*2b3ad188SAdrian Chadd } 693*2b3ad188SAdrian Chadd 694*2b3ad188SAdrian Chadd /* 695*2b3ad188SAdrian Chadd * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 696*2b3ad188SAdrian Chadd * re-balance it to another CPU or enable it on more CPUs. However, 697*2b3ad188SAdrian Chadd * PIC is expected to change isrc_cpu appropriately to keep us well 698*2b3ad188SAdrian Chadd * informed if the call is successfull. 699*2b3ad188SAdrian Chadd */ 700*2b3ad188SAdrian Chadd if (irq_assign_cpu) { 701*2b3ad188SAdrian Chadd error = PIC_BIND(isrc->isrc_dev, isrc); 702*2b3ad188SAdrian Chadd if (error) { 703*2b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 704*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 705*2b3ad188SAdrian Chadd return (error); 706*2b3ad188SAdrian Chadd } 707*2b3ad188SAdrian Chadd } 708*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 709*2b3ad188SAdrian Chadd return (0); 710*2b3ad188SAdrian Chadd #else 711*2b3ad188SAdrian Chadd return (EOPNOTSUPP); 712*2b3ad188SAdrian Chadd #endif 713*2b3ad188SAdrian Chadd } 714*2b3ad188SAdrian Chadd 715*2b3ad188SAdrian Chadd /* 716*2b3ad188SAdrian Chadd * Create interrupt event for interrupt source. 717*2b3ad188SAdrian Chadd */ 718*2b3ad188SAdrian Chadd static int 719*2b3ad188SAdrian Chadd isrc_event_create(struct intr_irqsrc *isrc) 720*2b3ad188SAdrian Chadd { 721*2b3ad188SAdrian Chadd struct intr_event *ie; 722*2b3ad188SAdrian Chadd int error; 723*2b3ad188SAdrian Chadd 724*2b3ad188SAdrian Chadd error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 725*2b3ad188SAdrian Chadd intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 726*2b3ad188SAdrian Chadd intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 727*2b3ad188SAdrian Chadd if (error) 728*2b3ad188SAdrian Chadd return (error); 729*2b3ad188SAdrian Chadd 730*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 731*2b3ad188SAdrian Chadd /* 732*2b3ad188SAdrian Chadd * Make sure that we do not mix the two ways 733*2b3ad188SAdrian Chadd * how we handle interrupt sources. Let contested event wins. 734*2b3ad188SAdrian Chadd */ 735*2b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 736*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 737*2b3ad188SAdrian Chadd intr_event_destroy(ie); 738*2b3ad188SAdrian Chadd return (isrc->isrc_event != NULL ? EBUSY : 0); 739*2b3ad188SAdrian Chadd } 740*2b3ad188SAdrian Chadd isrc->isrc_event = ie; 741*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 742*2b3ad188SAdrian Chadd 743*2b3ad188SAdrian Chadd return (0); 744*2b3ad188SAdrian Chadd } 745*2b3ad188SAdrian Chadd #ifdef notyet 746*2b3ad188SAdrian Chadd /* 747*2b3ad188SAdrian Chadd * Destroy interrupt event for interrupt source. 748*2b3ad188SAdrian Chadd */ 749*2b3ad188SAdrian Chadd static void 750*2b3ad188SAdrian Chadd isrc_event_destroy(struct intr_irqsrc *isrc) 751*2b3ad188SAdrian Chadd { 752*2b3ad188SAdrian Chadd struct intr_event *ie; 753*2b3ad188SAdrian Chadd 754*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 755*2b3ad188SAdrian Chadd ie = isrc->isrc_event; 756*2b3ad188SAdrian Chadd isrc->isrc_event = NULL; 757*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 758*2b3ad188SAdrian Chadd 759*2b3ad188SAdrian Chadd if (ie != NULL) 760*2b3ad188SAdrian Chadd intr_event_destroy(ie); 761*2b3ad188SAdrian Chadd } 762*2b3ad188SAdrian Chadd #endif 763*2b3ad188SAdrian Chadd /* 764*2b3ad188SAdrian Chadd * Add handler to interrupt source. 765*2b3ad188SAdrian Chadd */ 766*2b3ad188SAdrian Chadd static int 767*2b3ad188SAdrian Chadd isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 768*2b3ad188SAdrian Chadd driver_filter_t filter, driver_intr_t handler, void *arg, 769*2b3ad188SAdrian Chadd enum intr_type flags, void **cookiep) 770*2b3ad188SAdrian Chadd { 771*2b3ad188SAdrian Chadd int error; 772*2b3ad188SAdrian Chadd 773*2b3ad188SAdrian Chadd if (isrc->isrc_event == NULL) { 774*2b3ad188SAdrian Chadd error = isrc_event_create(isrc); 775*2b3ad188SAdrian Chadd if (error) 776*2b3ad188SAdrian Chadd return (error); 777*2b3ad188SAdrian Chadd } 778*2b3ad188SAdrian Chadd 779*2b3ad188SAdrian Chadd error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 780*2b3ad188SAdrian Chadd arg, intr_priority(flags), flags, cookiep); 781*2b3ad188SAdrian Chadd if (error == 0) { 782*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 783*2b3ad188SAdrian Chadd intrcnt_updatename(isrc); 784*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 785*2b3ad188SAdrian Chadd } 786*2b3ad188SAdrian Chadd 787*2b3ad188SAdrian Chadd return (error); 788*2b3ad188SAdrian Chadd } 789*2b3ad188SAdrian Chadd 790*2b3ad188SAdrian Chadd /* 791*2b3ad188SAdrian Chadd * Lookup interrupt controller locked. 792*2b3ad188SAdrian Chadd */ 793*2b3ad188SAdrian Chadd static struct intr_pic * 794*2b3ad188SAdrian Chadd pic_lookup_locked(device_t dev, intptr_t xref) 795*2b3ad188SAdrian Chadd { 796*2b3ad188SAdrian Chadd struct intr_pic *pic; 797*2b3ad188SAdrian Chadd 798*2b3ad188SAdrian Chadd mtx_assert(&pic_list_lock, MA_OWNED); 799*2b3ad188SAdrian Chadd 800*2b3ad188SAdrian Chadd SLIST_FOREACH(pic, &pic_list, pic_next) { 801*2b3ad188SAdrian Chadd if (pic->pic_xref != xref) 802*2b3ad188SAdrian Chadd continue; 803*2b3ad188SAdrian Chadd if (pic->pic_xref != 0 || pic->pic_dev == dev) 804*2b3ad188SAdrian Chadd return (pic); 805*2b3ad188SAdrian Chadd } 806*2b3ad188SAdrian Chadd return (NULL); 807*2b3ad188SAdrian Chadd } 808*2b3ad188SAdrian Chadd 809*2b3ad188SAdrian Chadd /* 810*2b3ad188SAdrian Chadd * Lookup interrupt controller. 811*2b3ad188SAdrian Chadd */ 812*2b3ad188SAdrian Chadd static struct intr_pic * 813*2b3ad188SAdrian Chadd pic_lookup(device_t dev, intptr_t xref) 814*2b3ad188SAdrian Chadd { 815*2b3ad188SAdrian Chadd struct intr_pic *pic; 816*2b3ad188SAdrian Chadd 817*2b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 818*2b3ad188SAdrian Chadd pic = pic_lookup_locked(dev, xref); 819*2b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 820*2b3ad188SAdrian Chadd 821*2b3ad188SAdrian Chadd return (pic); 822*2b3ad188SAdrian Chadd } 823*2b3ad188SAdrian Chadd 824*2b3ad188SAdrian Chadd /* 825*2b3ad188SAdrian Chadd * Create interrupt controller. 826*2b3ad188SAdrian Chadd */ 827*2b3ad188SAdrian Chadd static struct intr_pic * 828*2b3ad188SAdrian Chadd pic_create(device_t dev, intptr_t xref) 829*2b3ad188SAdrian Chadd { 830*2b3ad188SAdrian Chadd struct intr_pic *pic; 831*2b3ad188SAdrian Chadd 832*2b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 833*2b3ad188SAdrian Chadd pic = pic_lookup_locked(dev, xref); 834*2b3ad188SAdrian Chadd if (pic != NULL) { 835*2b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 836*2b3ad188SAdrian Chadd return (pic); 837*2b3ad188SAdrian Chadd } 838*2b3ad188SAdrian Chadd pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 839*2b3ad188SAdrian Chadd pic->pic_xref = xref; 840*2b3ad188SAdrian Chadd pic->pic_dev = dev; 841*2b3ad188SAdrian Chadd SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 842*2b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 843*2b3ad188SAdrian Chadd 844*2b3ad188SAdrian Chadd return (pic); 845*2b3ad188SAdrian Chadd } 846*2b3ad188SAdrian Chadd #ifdef notyet 847*2b3ad188SAdrian Chadd /* 848*2b3ad188SAdrian Chadd * Destroy interrupt controller. 849*2b3ad188SAdrian Chadd */ 850*2b3ad188SAdrian Chadd static void 851*2b3ad188SAdrian Chadd pic_destroy(device_t dev, intptr_t xref) 852*2b3ad188SAdrian Chadd { 853*2b3ad188SAdrian Chadd struct intr_pic *pic; 854*2b3ad188SAdrian Chadd 855*2b3ad188SAdrian Chadd mtx_lock(&pic_list_lock); 856*2b3ad188SAdrian Chadd pic = pic_lookup_locked(dev, xref); 857*2b3ad188SAdrian Chadd if (pic == NULL) { 858*2b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 859*2b3ad188SAdrian Chadd return; 860*2b3ad188SAdrian Chadd } 861*2b3ad188SAdrian Chadd SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 862*2b3ad188SAdrian Chadd mtx_unlock(&pic_list_lock); 863*2b3ad188SAdrian Chadd 864*2b3ad188SAdrian Chadd free(pic, M_INTRNG); 865*2b3ad188SAdrian Chadd } 866*2b3ad188SAdrian Chadd #endif 867*2b3ad188SAdrian Chadd /* 868*2b3ad188SAdrian Chadd * Register interrupt controller. 869*2b3ad188SAdrian Chadd */ 870*2b3ad188SAdrian Chadd int 871*2b3ad188SAdrian Chadd intr_pic_register(device_t dev, intptr_t xref) 872*2b3ad188SAdrian Chadd { 873*2b3ad188SAdrian Chadd struct intr_pic *pic; 874*2b3ad188SAdrian Chadd 875*2b3ad188SAdrian Chadd pic = pic_create(dev, xref); 876*2b3ad188SAdrian Chadd if (pic == NULL) 877*2b3ad188SAdrian Chadd return (ENOMEM); 878*2b3ad188SAdrian Chadd if (pic->pic_dev != dev) 879*2b3ad188SAdrian Chadd return (EINVAL); /* XXX it could be many things. */ 880*2b3ad188SAdrian Chadd 881*2b3ad188SAdrian Chadd debugf("PIC %p registered for %s <xref %x>\n", pic, 882*2b3ad188SAdrian Chadd device_get_nameunit(dev), xref); 883*2b3ad188SAdrian Chadd return (0); 884*2b3ad188SAdrian Chadd } 885*2b3ad188SAdrian Chadd 886*2b3ad188SAdrian Chadd /* 887*2b3ad188SAdrian Chadd * Unregister interrupt controller. 888*2b3ad188SAdrian Chadd */ 889*2b3ad188SAdrian Chadd int 890*2b3ad188SAdrian Chadd intr_pic_unregister(device_t dev, intptr_t xref) 891*2b3ad188SAdrian Chadd { 892*2b3ad188SAdrian Chadd 893*2b3ad188SAdrian Chadd panic("%s: not implemented", __func__); 894*2b3ad188SAdrian Chadd } 895*2b3ad188SAdrian Chadd 896*2b3ad188SAdrian Chadd /* 897*2b3ad188SAdrian Chadd * Mark interrupt controller (itself) as a root one. 898*2b3ad188SAdrian Chadd * 899*2b3ad188SAdrian Chadd * Note that only an interrupt controller can really know its position 900*2b3ad188SAdrian Chadd * in interrupt controller's tree. So root PIC must claim itself as a root. 901*2b3ad188SAdrian Chadd * 902*2b3ad188SAdrian Chadd * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 903*2b3ad188SAdrian Chadd * page 30: 904*2b3ad188SAdrian Chadd * "The root of the interrupt tree is determined when traversal 905*2b3ad188SAdrian Chadd * of the interrupt tree reaches an interrupt controller node without 906*2b3ad188SAdrian Chadd * an interrupts property and thus no explicit interrupt parent." 907*2b3ad188SAdrian Chadd */ 908*2b3ad188SAdrian Chadd int 909*2b3ad188SAdrian Chadd intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 910*2b3ad188SAdrian Chadd void *arg, u_int ipicount) 911*2b3ad188SAdrian Chadd { 912*2b3ad188SAdrian Chadd int error; 913*2b3ad188SAdrian Chadd u_int rootirq; 914*2b3ad188SAdrian Chadd 915*2b3ad188SAdrian Chadd if (pic_lookup(dev, xref) == NULL) { 916*2b3ad188SAdrian Chadd device_printf(dev, "not registered\n"); 917*2b3ad188SAdrian Chadd return (EINVAL); 918*2b3ad188SAdrian Chadd } 919*2b3ad188SAdrian Chadd if (filter == NULL) { 920*2b3ad188SAdrian Chadd device_printf(dev, "filter missing\n"); 921*2b3ad188SAdrian Chadd return (EINVAL); 922*2b3ad188SAdrian Chadd } 923*2b3ad188SAdrian Chadd 924*2b3ad188SAdrian Chadd /* 925*2b3ad188SAdrian Chadd * Only one interrupt controllers could be on the root for now. 926*2b3ad188SAdrian Chadd * Note that we further suppose that there is not threaded interrupt 927*2b3ad188SAdrian Chadd * routine (handler) on the root. See intr_irq_handler(). 928*2b3ad188SAdrian Chadd */ 929*2b3ad188SAdrian Chadd if (irq_root_dev != NULL) { 930*2b3ad188SAdrian Chadd device_printf(dev, "another root already set\n"); 931*2b3ad188SAdrian Chadd return (EBUSY); 932*2b3ad188SAdrian Chadd } 933*2b3ad188SAdrian Chadd 934*2b3ad188SAdrian Chadd rootirq = intr_namespace_map_irq(device_get_parent(dev), 0, 0); 935*2b3ad188SAdrian Chadd if (rootirq == IRQ_INVALID) { 936*2b3ad188SAdrian Chadd device_printf(dev, "failed to map an irq for the root pic\n"); 937*2b3ad188SAdrian Chadd return (ENOMEM); 938*2b3ad188SAdrian Chadd } 939*2b3ad188SAdrian Chadd 940*2b3ad188SAdrian Chadd /* Create the isrc. */ 941*2b3ad188SAdrian Chadd irq_root_isrc = isrc_lookup(rootirq); 942*2b3ad188SAdrian Chadd 943*2b3ad188SAdrian Chadd /* XXX "register" with the PIC. We are the "pic" here, so fake it. */ 944*2b3ad188SAdrian Chadd irq_root_isrc->isrc_flags |= INTR_ISRCF_REGISTERED; 945*2b3ad188SAdrian Chadd 946*2b3ad188SAdrian Chadd error = intr_irq_add_handler(device_get_parent(dev), 947*2b3ad188SAdrian Chadd (void*)filter, NULL, arg, rootirq, INTR_TYPE_CLK, NULL); 948*2b3ad188SAdrian Chadd if (error != 0) { 949*2b3ad188SAdrian Chadd device_printf(dev, "failed to install root pic handler\n"); 950*2b3ad188SAdrian Chadd return (error); 951*2b3ad188SAdrian Chadd } 952*2b3ad188SAdrian Chadd irq_root_dev = dev; 953*2b3ad188SAdrian Chadd irq_root_filter = filter; 954*2b3ad188SAdrian Chadd irq_root_arg = arg; 955*2b3ad188SAdrian Chadd irq_root_ipicount = ipicount; 956*2b3ad188SAdrian Chadd 957*2b3ad188SAdrian Chadd debugf("irq root set to %s\n", device_get_nameunit(dev)); 958*2b3ad188SAdrian Chadd return (0); 959*2b3ad188SAdrian Chadd } 960*2b3ad188SAdrian Chadd 961*2b3ad188SAdrian Chadd int 962*2b3ad188SAdrian Chadd intr_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, 963*2b3ad188SAdrian Chadd void *arg, u_int irq, int flags, void **cookiep) 964*2b3ad188SAdrian Chadd { 965*2b3ad188SAdrian Chadd const char *name; 966*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 967*2b3ad188SAdrian Chadd int error; 968*2b3ad188SAdrian Chadd 969*2b3ad188SAdrian Chadd name = device_get_nameunit(dev); 970*2b3ad188SAdrian Chadd 971*2b3ad188SAdrian Chadd #ifdef INTR_SOLO 972*2b3ad188SAdrian Chadd /* 973*2b3ad188SAdrian Chadd * Standard handling is done thru MI interrupt framework. However, 974*2b3ad188SAdrian Chadd * some interrupts could request solely own special handling. This 975*2b3ad188SAdrian Chadd * non standard handling can be used for interrupt controllers without 976*2b3ad188SAdrian Chadd * handler (filter only), so in case that interrupt controllers are 977*2b3ad188SAdrian Chadd * chained, MI interrupt framework is called only in leaf controller. 978*2b3ad188SAdrian Chadd * 979*2b3ad188SAdrian Chadd * Note that root interrupt controller routine is served as well, 980*2b3ad188SAdrian Chadd * however in intr_irq_handler(), i.e. main system dispatch routine. 981*2b3ad188SAdrian Chadd */ 982*2b3ad188SAdrian Chadd if (flags & INTR_SOLO && hand != NULL) { 983*2b3ad188SAdrian Chadd debugf("irq %u cannot solo on %s\n", irq, name); 984*2b3ad188SAdrian Chadd return (EINVAL); 985*2b3ad188SAdrian Chadd } 986*2b3ad188SAdrian Chadd #endif 987*2b3ad188SAdrian Chadd 988*2b3ad188SAdrian Chadd isrc = isrc_lookup(irq); 989*2b3ad188SAdrian Chadd if (isrc == NULL) { 990*2b3ad188SAdrian Chadd debugf("irq %u without source on %s\n", irq, name); 991*2b3ad188SAdrian Chadd return (EINVAL); 992*2b3ad188SAdrian Chadd } 993*2b3ad188SAdrian Chadd 994*2b3ad188SAdrian Chadd error = isrc_register(isrc); 995*2b3ad188SAdrian Chadd if (error != 0) { 996*2b3ad188SAdrian Chadd debugf("irq %u map error %d on %s\n", irq, error, name); 997*2b3ad188SAdrian Chadd return (error); 998*2b3ad188SAdrian Chadd } 999*2b3ad188SAdrian Chadd 1000*2b3ad188SAdrian Chadd #ifdef INTR_SOLO 1001*2b3ad188SAdrian Chadd if (flags & INTR_SOLO) { 1002*2b3ad188SAdrian Chadd error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 1003*2b3ad188SAdrian Chadd arg, cookiep); 1004*2b3ad188SAdrian Chadd debugf("irq %u setup filter error %d on %s\n", irq, error, 1005*2b3ad188SAdrian Chadd name); 1006*2b3ad188SAdrian Chadd } else 1007*2b3ad188SAdrian Chadd #endif 1008*2b3ad188SAdrian Chadd { 1009*2b3ad188SAdrian Chadd error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 1010*2b3ad188SAdrian Chadd cookiep); 1011*2b3ad188SAdrian Chadd debugf("irq %u add handler error %d on %s\n", irq, error, name); 1012*2b3ad188SAdrian Chadd } 1013*2b3ad188SAdrian Chadd if (error != 0) 1014*2b3ad188SAdrian Chadd return (error); 1015*2b3ad188SAdrian Chadd 1016*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1017*2b3ad188SAdrian Chadd isrc->isrc_handlers++; 1018*2b3ad188SAdrian Chadd if (isrc->isrc_handlers == 1) { 1019*2b3ad188SAdrian Chadd PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 1020*2b3ad188SAdrian Chadd PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); 1021*2b3ad188SAdrian Chadd } 1022*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1023*2b3ad188SAdrian Chadd return (0); 1024*2b3ad188SAdrian Chadd } 1025*2b3ad188SAdrian Chadd 1026*2b3ad188SAdrian Chadd int 1027*2b3ad188SAdrian Chadd intr_irq_remove_handler(device_t dev, u_int irq, void *cookie) 1028*2b3ad188SAdrian Chadd { 1029*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1030*2b3ad188SAdrian Chadd int error; 1031*2b3ad188SAdrian Chadd 1032*2b3ad188SAdrian Chadd isrc = isrc_lookup(irq); 1033*2b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 1034*2b3ad188SAdrian Chadd return (EINVAL); 1035*2b3ad188SAdrian Chadd 1036*2b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 1037*2b3ad188SAdrian Chadd if (isrc != cookie) 1038*2b3ad188SAdrian Chadd return (EINVAL); 1039*2b3ad188SAdrian Chadd 1040*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1041*2b3ad188SAdrian Chadd isrc->isrc_filter = NULL; 1042*2b3ad188SAdrian Chadd isrc->isrc_arg = NULL; 1043*2b3ad188SAdrian Chadd isrc->isrc_handlers = 0; 1044*2b3ad188SAdrian Chadd PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc); 1045*2b3ad188SAdrian Chadd PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1046*2b3ad188SAdrian Chadd isrc_update_name(isrc, NULL); 1047*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1048*2b3ad188SAdrian Chadd return (0); 1049*2b3ad188SAdrian Chadd } 1050*2b3ad188SAdrian Chadd 1051*2b3ad188SAdrian Chadd if (isrc != intr_handler_source(cookie)) 1052*2b3ad188SAdrian Chadd return (EINVAL); 1053*2b3ad188SAdrian Chadd 1054*2b3ad188SAdrian Chadd error = intr_event_remove_handler(cookie); 1055*2b3ad188SAdrian Chadd if (error == 0) { 1056*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1057*2b3ad188SAdrian Chadd isrc->isrc_handlers--; 1058*2b3ad188SAdrian Chadd if (isrc->isrc_handlers == 0) { 1059*2b3ad188SAdrian Chadd PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc); 1060*2b3ad188SAdrian Chadd PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1061*2b3ad188SAdrian Chadd } 1062*2b3ad188SAdrian Chadd intrcnt_updatename(isrc); 1063*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1064*2b3ad188SAdrian Chadd } 1065*2b3ad188SAdrian Chadd return (error); 1066*2b3ad188SAdrian Chadd } 1067*2b3ad188SAdrian Chadd 1068*2b3ad188SAdrian Chadd int 1069*2b3ad188SAdrian Chadd intr_irq_config(u_int irq, enum intr_trigger trig, enum intr_polarity pol) 1070*2b3ad188SAdrian Chadd { 1071*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1072*2b3ad188SAdrian Chadd 1073*2b3ad188SAdrian Chadd isrc = isrc_lookup(irq); 1074*2b3ad188SAdrian Chadd if (isrc == NULL) 1075*2b3ad188SAdrian Chadd return (EINVAL); 1076*2b3ad188SAdrian Chadd 1077*2b3ad188SAdrian Chadd if (isrc->isrc_handlers != 0) 1078*2b3ad188SAdrian Chadd return (EBUSY); /* interrrupt is enabled (active) */ 1079*2b3ad188SAdrian Chadd 1080*2b3ad188SAdrian Chadd /* 1081*2b3ad188SAdrian Chadd * Once an interrupt is enabled, we do not change its configuration. 1082*2b3ad188SAdrian Chadd * A controller PIC_ENABLE_INTR() method is called when an interrupt 1083*2b3ad188SAdrian Chadd * is going to be enabled. In this method, a controller should setup 1084*2b3ad188SAdrian Chadd * the interrupt according to saved configuration parameters. 1085*2b3ad188SAdrian Chadd */ 1086*2b3ad188SAdrian Chadd isrc->isrc_trig = trig; 1087*2b3ad188SAdrian Chadd isrc->isrc_pol = pol; 1088*2b3ad188SAdrian Chadd 1089*2b3ad188SAdrian Chadd return (0); 1090*2b3ad188SAdrian Chadd } 1091*2b3ad188SAdrian Chadd 1092*2b3ad188SAdrian Chadd int 1093*2b3ad188SAdrian Chadd intr_irq_describe(u_int irq, void *cookie, const char *descr) 1094*2b3ad188SAdrian Chadd { 1095*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1096*2b3ad188SAdrian Chadd int error; 1097*2b3ad188SAdrian Chadd 1098*2b3ad188SAdrian Chadd isrc = isrc_lookup(irq); 1099*2b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 1100*2b3ad188SAdrian Chadd return (EINVAL); 1101*2b3ad188SAdrian Chadd 1102*2b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) { 1103*2b3ad188SAdrian Chadd if (isrc != cookie) 1104*2b3ad188SAdrian Chadd return (EINVAL); 1105*2b3ad188SAdrian Chadd 1106*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1107*2b3ad188SAdrian Chadd isrc_update_name(isrc, descr); 1108*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1109*2b3ad188SAdrian Chadd return (0); 1110*2b3ad188SAdrian Chadd } 1111*2b3ad188SAdrian Chadd 1112*2b3ad188SAdrian Chadd error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 1113*2b3ad188SAdrian Chadd if (error == 0) { 1114*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1115*2b3ad188SAdrian Chadd intrcnt_updatename(isrc); 1116*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1117*2b3ad188SAdrian Chadd } 1118*2b3ad188SAdrian Chadd return (error); 1119*2b3ad188SAdrian Chadd } 1120*2b3ad188SAdrian Chadd 1121*2b3ad188SAdrian Chadd #ifdef SMP 1122*2b3ad188SAdrian Chadd int 1123*2b3ad188SAdrian Chadd intr_irq_bind(u_int irq, int cpu) 1124*2b3ad188SAdrian Chadd { 1125*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1126*2b3ad188SAdrian Chadd 1127*2b3ad188SAdrian Chadd isrc = isrc_lookup(irq); 1128*2b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0) 1129*2b3ad188SAdrian Chadd return (EINVAL); 1130*2b3ad188SAdrian Chadd 1131*2b3ad188SAdrian Chadd if (isrc->isrc_filter != NULL) 1132*2b3ad188SAdrian Chadd return (intr_isrc_assign_cpu(isrc, cpu)); 1133*2b3ad188SAdrian Chadd 1134*2b3ad188SAdrian Chadd return (intr_event_bind(isrc->isrc_event, cpu)); 1135*2b3ad188SAdrian Chadd } 1136*2b3ad188SAdrian Chadd 1137*2b3ad188SAdrian Chadd /* 1138*2b3ad188SAdrian Chadd * Return the CPU that the next interrupt source should use. 1139*2b3ad188SAdrian Chadd * For now just returns the next CPU according to round-robin. 1140*2b3ad188SAdrian Chadd */ 1141*2b3ad188SAdrian Chadd u_int 1142*2b3ad188SAdrian Chadd intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 1143*2b3ad188SAdrian Chadd { 1144*2b3ad188SAdrian Chadd 1145*2b3ad188SAdrian Chadd if (!irq_assign_cpu || mp_ncpus == 1) 1146*2b3ad188SAdrian Chadd return (PCPU_GET(cpuid)); 1147*2b3ad188SAdrian Chadd 1148*2b3ad188SAdrian Chadd do { 1149*2b3ad188SAdrian Chadd last_cpu++; 1150*2b3ad188SAdrian Chadd if (last_cpu > mp_maxid) 1151*2b3ad188SAdrian Chadd last_cpu = 0; 1152*2b3ad188SAdrian Chadd } while (!CPU_ISSET(last_cpu, cpumask)); 1153*2b3ad188SAdrian Chadd return (last_cpu); 1154*2b3ad188SAdrian Chadd } 1155*2b3ad188SAdrian Chadd 1156*2b3ad188SAdrian Chadd /* 1157*2b3ad188SAdrian Chadd * Distribute all the interrupt sources among the available 1158*2b3ad188SAdrian Chadd * CPUs once the AP's have been launched. 1159*2b3ad188SAdrian Chadd */ 1160*2b3ad188SAdrian Chadd static void 1161*2b3ad188SAdrian Chadd intr_irq_shuffle(void *arg __unused) 1162*2b3ad188SAdrian Chadd { 1163*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1164*2b3ad188SAdrian Chadd u_int i; 1165*2b3ad188SAdrian Chadd 1166*2b3ad188SAdrian Chadd if (mp_ncpus == 1) 1167*2b3ad188SAdrian Chadd return; 1168*2b3ad188SAdrian Chadd 1169*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1170*2b3ad188SAdrian Chadd irq_assign_cpu = TRUE; 1171*2b3ad188SAdrian Chadd for (i = 0; i < NIRQ; i++) { 1172*2b3ad188SAdrian Chadd isrc = irq_sources[i]; 1173*2b3ad188SAdrian Chadd if (isrc == NULL || isrc->isrc_handlers == 0 || 1174*2b3ad188SAdrian Chadd isrc->isrc_flags & INTR_ISRCF_PERCPU) 1175*2b3ad188SAdrian Chadd continue; 1176*2b3ad188SAdrian Chadd 1177*2b3ad188SAdrian Chadd if (isrc->isrc_event != NULL && 1178*2b3ad188SAdrian Chadd isrc->isrc_flags & INTR_ISRCF_BOUND && 1179*2b3ad188SAdrian Chadd isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 1180*2b3ad188SAdrian Chadd panic("%s: CPU inconsistency", __func__); 1181*2b3ad188SAdrian Chadd 1182*2b3ad188SAdrian Chadd if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 1183*2b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); /* start again */ 1184*2b3ad188SAdrian Chadd 1185*2b3ad188SAdrian Chadd /* 1186*2b3ad188SAdrian Chadd * We are in wicked position here if the following call fails 1187*2b3ad188SAdrian Chadd * for bound ISRC. The best thing we can do is to clear 1188*2b3ad188SAdrian Chadd * isrc_cpu so inconsistency with ie_cpu will be detectable. 1189*2b3ad188SAdrian Chadd */ 1190*2b3ad188SAdrian Chadd if (PIC_BIND(isrc->isrc_dev, isrc) != 0) 1191*2b3ad188SAdrian Chadd CPU_ZERO(&isrc->isrc_cpu); 1192*2b3ad188SAdrian Chadd } 1193*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1194*2b3ad188SAdrian Chadd } 1195*2b3ad188SAdrian Chadd SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1196*2b3ad188SAdrian Chadd 1197*2b3ad188SAdrian Chadd #else 1198*2b3ad188SAdrian Chadd u_int 1199*2b3ad188SAdrian Chadd intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 1200*2b3ad188SAdrian Chadd { 1201*2b3ad188SAdrian Chadd 1202*2b3ad188SAdrian Chadd return (PCPU_GET(cpuid)); 1203*2b3ad188SAdrian Chadd } 1204*2b3ad188SAdrian Chadd #endif 1205*2b3ad188SAdrian Chadd 1206*2b3ad188SAdrian Chadd void dosoftints(void); 1207*2b3ad188SAdrian Chadd void 1208*2b3ad188SAdrian Chadd dosoftints(void) 1209*2b3ad188SAdrian Chadd { 1210*2b3ad188SAdrian Chadd } 1211*2b3ad188SAdrian Chadd 1212*2b3ad188SAdrian Chadd #ifdef SMP 1213*2b3ad188SAdrian Chadd /* 1214*2b3ad188SAdrian Chadd * Lookup IPI source. 1215*2b3ad188SAdrian Chadd */ 1216*2b3ad188SAdrian Chadd static struct intr_irqsrc * 1217*2b3ad188SAdrian Chadd intr_ipi_lookup(u_int ipi) 1218*2b3ad188SAdrian Chadd { 1219*2b3ad188SAdrian Chadd 1220*2b3ad188SAdrian Chadd if (ipi >= INTR_IPI_COUNT) 1221*2b3ad188SAdrian Chadd panic("%s: no such IPI %u", __func__, ipi); 1222*2b3ad188SAdrian Chadd 1223*2b3ad188SAdrian Chadd return (&ipi_sources[ipi]); 1224*2b3ad188SAdrian Chadd } 1225*2b3ad188SAdrian Chadd 1226*2b3ad188SAdrian Chadd /* 1227*2b3ad188SAdrian Chadd * interrupt controller dispatch function for IPIs. It should 1228*2b3ad188SAdrian Chadd * be called straight from the interrupt controller, when associated 1229*2b3ad188SAdrian Chadd * interrupt source is learned. Or from anybody who has an interrupt 1230*2b3ad188SAdrian Chadd * source mapped. 1231*2b3ad188SAdrian Chadd */ 1232*2b3ad188SAdrian Chadd void 1233*2b3ad188SAdrian Chadd intr_ipi_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 1234*2b3ad188SAdrian Chadd { 1235*2b3ad188SAdrian Chadd void *arg; 1236*2b3ad188SAdrian Chadd 1237*2b3ad188SAdrian Chadd KASSERT(isrc != NULL, ("%s: no source", __func__)); 1238*2b3ad188SAdrian Chadd 1239*2b3ad188SAdrian Chadd isrc_increment_ipi_count(isrc, PCPU_GET(cpuid)); 1240*2b3ad188SAdrian Chadd 1241*2b3ad188SAdrian Chadd /* 1242*2b3ad188SAdrian Chadd * Supply ipi filter with trapframe argument 1243*2b3ad188SAdrian Chadd * if none is registered. 1244*2b3ad188SAdrian Chadd */ 1245*2b3ad188SAdrian Chadd arg = isrc->isrc_arg != NULL ? isrc->isrc_arg : tf; 1246*2b3ad188SAdrian Chadd isrc->isrc_ipifilter(arg); 1247*2b3ad188SAdrian Chadd } 1248*2b3ad188SAdrian Chadd 1249*2b3ad188SAdrian Chadd /* 1250*2b3ad188SAdrian Chadd * Map IPI into interrupt controller. 1251*2b3ad188SAdrian Chadd * 1252*2b3ad188SAdrian Chadd * Not SMP coherent. 1253*2b3ad188SAdrian Chadd */ 1254*2b3ad188SAdrian Chadd static int 1255*2b3ad188SAdrian Chadd ipi_map(struct intr_irqsrc *isrc, u_int ipi) 1256*2b3ad188SAdrian Chadd { 1257*2b3ad188SAdrian Chadd boolean_t is_percpu; 1258*2b3ad188SAdrian Chadd int error; 1259*2b3ad188SAdrian Chadd 1260*2b3ad188SAdrian Chadd if (ipi >= INTR_IPI_COUNT) 1261*2b3ad188SAdrian Chadd panic("%s: no such IPI %u", __func__, ipi); 1262*2b3ad188SAdrian Chadd 1263*2b3ad188SAdrian Chadd KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__)); 1264*2b3ad188SAdrian Chadd 1265*2b3ad188SAdrian Chadd isrc->isrc_type = INTR_ISRCT_NAMESPACE; 1266*2b3ad188SAdrian Chadd isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI; 1267*2b3ad188SAdrian Chadd isrc->isrc_nspc_num = ipi_next_num; 1268*2b3ad188SAdrian Chadd 1269*2b3ad188SAdrian Chadd error = PIC_REGISTER(irq_root_dev, isrc, &is_percpu); 1270*2b3ad188SAdrian Chadd 1271*2b3ad188SAdrian Chadd debugf("ipi %u mapped to %u on %s - error %d\n", ipi, ipi_next_num, 1272*2b3ad188SAdrian Chadd device_get_nameunit(irq_root_dev), error); 1273*2b3ad188SAdrian Chadd 1274*2b3ad188SAdrian Chadd if (error == 0) { 1275*2b3ad188SAdrian Chadd isrc->isrc_dev = irq_root_dev; 1276*2b3ad188SAdrian Chadd ipi_next_num++; 1277*2b3ad188SAdrian Chadd } 1278*2b3ad188SAdrian Chadd return (error); 1279*2b3ad188SAdrian Chadd } 1280*2b3ad188SAdrian Chadd 1281*2b3ad188SAdrian Chadd /* 1282*2b3ad188SAdrian Chadd * Setup IPI handler to interrupt source. 1283*2b3ad188SAdrian Chadd * 1284*2b3ad188SAdrian Chadd * Note that there could be more ways how to send and receive IPIs 1285*2b3ad188SAdrian Chadd * on a platform like fast interrupts for example. In that case, 1286*2b3ad188SAdrian Chadd * one can call this function with ASIF_NOALLOC flag set and then 1287*2b3ad188SAdrian Chadd * call intr_ipi_dispatch() when appropriate. 1288*2b3ad188SAdrian Chadd * 1289*2b3ad188SAdrian Chadd * Not SMP coherent. 1290*2b3ad188SAdrian Chadd */ 1291*2b3ad188SAdrian Chadd int 1292*2b3ad188SAdrian Chadd intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, 1293*2b3ad188SAdrian Chadd void *arg, u_int flags) 1294*2b3ad188SAdrian Chadd { 1295*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1296*2b3ad188SAdrian Chadd int error; 1297*2b3ad188SAdrian Chadd 1298*2b3ad188SAdrian Chadd if (filter == NULL) 1299*2b3ad188SAdrian Chadd return(EINVAL); 1300*2b3ad188SAdrian Chadd 1301*2b3ad188SAdrian Chadd isrc = intr_ipi_lookup(ipi); 1302*2b3ad188SAdrian Chadd if (isrc->isrc_ipifilter != NULL) 1303*2b3ad188SAdrian Chadd return (EEXIST); 1304*2b3ad188SAdrian Chadd 1305*2b3ad188SAdrian Chadd if ((flags & AISHF_NOALLOC) == 0) { 1306*2b3ad188SAdrian Chadd error = ipi_map(isrc, ipi); 1307*2b3ad188SAdrian Chadd if (error != 0) 1308*2b3ad188SAdrian Chadd return (error); 1309*2b3ad188SAdrian Chadd } 1310*2b3ad188SAdrian Chadd 1311*2b3ad188SAdrian Chadd isrc->isrc_ipifilter = filter; 1312*2b3ad188SAdrian Chadd isrc->isrc_arg = arg; 1313*2b3ad188SAdrian Chadd isrc->isrc_handlers = 1; 1314*2b3ad188SAdrian Chadd isrc_setup_ipi_counters(isrc, name); 1315*2b3ad188SAdrian Chadd 1316*2b3ad188SAdrian Chadd if (isrc->isrc_dev != NULL) { 1317*2b3ad188SAdrian Chadd mtx_lock(&isrc_table_lock); 1318*2b3ad188SAdrian Chadd PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 1319*2b3ad188SAdrian Chadd PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); 1320*2b3ad188SAdrian Chadd mtx_unlock(&isrc_table_lock); 1321*2b3ad188SAdrian Chadd } 1322*2b3ad188SAdrian Chadd return (0); 1323*2b3ad188SAdrian Chadd } 1324*2b3ad188SAdrian Chadd 1325*2b3ad188SAdrian Chadd /* 1326*2b3ad188SAdrian Chadd * Send IPI thru interrupt controller. 1327*2b3ad188SAdrian Chadd */ 1328*2b3ad188SAdrian Chadd void 1329*2b3ad188SAdrian Chadd pic_ipi_send(cpuset_t cpus, u_int ipi) 1330*2b3ad188SAdrian Chadd { 1331*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1332*2b3ad188SAdrian Chadd 1333*2b3ad188SAdrian Chadd isrc = intr_ipi_lookup(ipi); 1334*2b3ad188SAdrian Chadd 1335*2b3ad188SAdrian Chadd KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__)); 1336*2b3ad188SAdrian Chadd PIC_IPI_SEND(irq_root_dev, isrc, cpus); 1337*2b3ad188SAdrian Chadd } 1338*2b3ad188SAdrian Chadd 1339*2b3ad188SAdrian Chadd /* 1340*2b3ad188SAdrian Chadd * Init interrupt controller on another CPU. 1341*2b3ad188SAdrian Chadd */ 1342*2b3ad188SAdrian Chadd void 1343*2b3ad188SAdrian Chadd intr_pic_init_secondary(void) 1344*2b3ad188SAdrian Chadd { 1345*2b3ad188SAdrian Chadd 1346*2b3ad188SAdrian Chadd /* 1347*2b3ad188SAdrian Chadd * QQQ: Only root PIC is aware of other CPUs ??? 1348*2b3ad188SAdrian Chadd */ 1349*2b3ad188SAdrian Chadd KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__)); 1350*2b3ad188SAdrian Chadd 1351*2b3ad188SAdrian Chadd //mtx_lock(&isrc_table_lock); 1352*2b3ad188SAdrian Chadd PIC_INIT_SECONDARY(irq_root_dev); 1353*2b3ad188SAdrian Chadd //mtx_unlock(&isrc_table_lock); 1354*2b3ad188SAdrian Chadd } 1355*2b3ad188SAdrian Chadd #endif 1356*2b3ad188SAdrian Chadd 1357*2b3ad188SAdrian Chadd #ifdef DDB 1358*2b3ad188SAdrian Chadd DB_SHOW_COMMAND(irqs, db_show_irqs) 1359*2b3ad188SAdrian Chadd { 1360*2b3ad188SAdrian Chadd u_int i, irqsum; 1361*2b3ad188SAdrian Chadd struct intr_irqsrc *isrc; 1362*2b3ad188SAdrian Chadd 1363*2b3ad188SAdrian Chadd #ifdef SMP 1364*2b3ad188SAdrian Chadd for (i = 0; i <= mp_maxid; i++) { 1365*2b3ad188SAdrian Chadd struct pcpu *pc; 1366*2b3ad188SAdrian Chadd u_int ipi, ipisum; 1367*2b3ad188SAdrian Chadd 1368*2b3ad188SAdrian Chadd pc = pcpu_find(i); 1369*2b3ad188SAdrian Chadd if (pc != NULL) { 1370*2b3ad188SAdrian Chadd for (ipisum = 0, ipi = 0; ipi < INTR_IPI_COUNT; ipi++) { 1371*2b3ad188SAdrian Chadd isrc = intr_ipi_lookup(ipi); 1372*2b3ad188SAdrian Chadd if (isrc->isrc_count != NULL) 1373*2b3ad188SAdrian Chadd ipisum += isrc->isrc_count[i]; 1374*2b3ad188SAdrian Chadd } 1375*2b3ad188SAdrian Chadd printf ("cpu%u: total %u ipis %u\n", i, 1376*2b3ad188SAdrian Chadd pc->pc_cnt.v_intr, ipisum); 1377*2b3ad188SAdrian Chadd } 1378*2b3ad188SAdrian Chadd } 1379*2b3ad188SAdrian Chadd db_printf("\n"); 1380*2b3ad188SAdrian Chadd #endif 1381*2b3ad188SAdrian Chadd 1382*2b3ad188SAdrian Chadd for (irqsum = 0, i = 0; i < NIRQ; i++) { 1383*2b3ad188SAdrian Chadd isrc = irq_sources[i]; 1384*2b3ad188SAdrian Chadd if (isrc == NULL) 1385*2b3ad188SAdrian Chadd continue; 1386*2b3ad188SAdrian Chadd 1387*2b3ad188SAdrian Chadd db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 1388*2b3ad188SAdrian Chadd isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1389*2b3ad188SAdrian Chadd isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", 1390*2b3ad188SAdrian Chadd isrc->isrc_count[0]); 1391*2b3ad188SAdrian Chadd irqsum += isrc->isrc_count[0]; 1392*2b3ad188SAdrian Chadd } 1393*2b3ad188SAdrian Chadd db_printf("irq total %u\n", irqsum); 1394*2b3ad188SAdrian Chadd } 1395*2b3ad188SAdrian Chadd #endif 1396