1 /*- 2 * Copyright (c) 2012-2014 Jakub Wojciech Klama <jceel@FreeBSD.org>. 3 * Copyright (c) 2015 Svatopluk Kraus 4 * Copyright (c) 2015 Michal Meloun 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 /* 35 * New-style Interrupt Framework 36 * 37 * TODO: - to support IPI (PPI) enabling on other CPUs if already started 38 * - to complete things for removable PICs 39 */ 40 41 #include "opt_ddb.h" 42 #include "opt_platform.h" 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/syslog.h> 48 #include <sys/malloc.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/bus.h> 52 #include <sys/interrupt.h> 53 #include <sys/conf.h> 54 #include <sys/cpuset.h> 55 #include <sys/sched.h> 56 #include <sys/smp.h> 57 #include <machine/atomic.h> 58 #include <machine/intr.h> 59 #include <machine/cpu.h> 60 #include <machine/smp.h> 61 #include <machine/stdarg.h> 62 63 #ifdef FDT 64 #include <dev/ofw/openfirm.h> 65 #include <dev/ofw/ofw_bus.h> 66 #include <dev/ofw/ofw_bus_subr.h> 67 #endif 68 69 #ifdef DDB 70 #include <ddb/ddb.h> 71 #endif 72 73 #include "pic_if.h" 74 75 #define INTRNAME_LEN (2*MAXCOMLEN + 1) 76 77 #ifdef DEBUG 78 #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 79 printf(fmt,##args); } while (0) 80 #else 81 #define debugf(fmt, args...) 82 #endif 83 84 MALLOC_DECLARE(M_INTRNG); 85 MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 86 87 /* Main interrupt handler called from assembler -> 'hidden' for C code. */ 88 void intr_irq_handler(struct trapframe *tf); 89 90 /* Root interrupt controller stuff. */ 91 device_t intr_irq_root_dev; 92 static intr_irq_filter_t *irq_root_filter; 93 static void *irq_root_arg; 94 static u_int irq_root_ipicount; 95 96 /* Interrupt controller definition. */ 97 struct intr_pic { 98 SLIST_ENTRY(intr_pic) pic_next; 99 intptr_t pic_xref; /* hardware identification */ 100 device_t pic_dev; 101 }; 102 103 static struct mtx pic_list_lock; 104 static SLIST_HEAD(, intr_pic) pic_list; 105 106 static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); 107 108 /* Interrupt source definition. */ 109 static struct mtx isrc_table_lock; 110 static struct intr_irqsrc *irq_sources[NIRQ]; 111 u_int irq_next_free; 112 113 #define IRQ_INVALID nitems(irq_sources) 114 115 #ifdef SMP 116 static boolean_t irq_assign_cpu = FALSE; 117 #endif 118 119 /* 120 * - 2 counters for each I/O interrupt. 121 * - MAXCPU counters for each IPI counters for SMP. 122 */ 123 #ifdef SMP 124 #define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) 125 #else 126 #define INTRCNT_COUNT (NIRQ * 2) 127 #endif 128 129 /* Data for MI statistics reporting. */ 130 u_long intrcnt[INTRCNT_COUNT]; 131 char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; 132 size_t sintrcnt = sizeof(intrcnt); 133 size_t sintrnames = sizeof(intrnames); 134 static u_int intrcnt_index; 135 136 /* 137 * Interrupt framework initialization routine. 138 */ 139 static void 140 intr_irq_init(void *dummy __unused) 141 { 142 143 SLIST_INIT(&pic_list); 144 mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 145 mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 146 } 147 SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 148 149 static void 150 intrcnt_setname(const char *name, int index) 151 { 152 153 snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 154 INTRNAME_LEN - 1, name); 155 } 156 157 /* 158 * Update name for interrupt source with interrupt event. 159 */ 160 static void 161 intrcnt_updatename(struct intr_irqsrc *isrc) 162 { 163 164 /* QQQ: What about stray counter name? */ 165 mtx_assert(&isrc_table_lock, MA_OWNED); 166 intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 167 } 168 169 /* 170 * Virtualization for interrupt source interrupt counter increment. 171 */ 172 static inline void 173 isrc_increment_count(struct intr_irqsrc *isrc) 174 { 175 176 /* 177 * XXX - It should be atomic for PPI interrupts. It was proven that 178 * the lost is measurable easily for timer PPI interrupts. 179 */ 180 isrc->isrc_count[0]++; 181 /*atomic_add_long(&isrc->isrc_count[0], 1);*/ 182 } 183 184 /* 185 * Virtualization for interrupt source interrupt stray counter increment. 186 */ 187 static inline void 188 isrc_increment_straycount(struct intr_irqsrc *isrc) 189 { 190 191 isrc->isrc_count[1]++; 192 } 193 194 /* 195 * Virtualization for interrupt source interrupt name update. 196 */ 197 static void 198 isrc_update_name(struct intr_irqsrc *isrc, const char *name) 199 { 200 char str[INTRNAME_LEN]; 201 202 mtx_assert(&isrc_table_lock, MA_OWNED); 203 204 if (name != NULL) { 205 snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 206 intrcnt_setname(str, isrc->isrc_index); 207 snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 208 name); 209 intrcnt_setname(str, isrc->isrc_index + 1); 210 } else { 211 snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 212 intrcnt_setname(str, isrc->isrc_index); 213 snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 214 intrcnt_setname(str, isrc->isrc_index + 1); 215 } 216 } 217 218 /* 219 * Virtualization for interrupt source interrupt counters setup. 220 */ 221 static void 222 isrc_setup_counters(struct intr_irqsrc *isrc) 223 { 224 u_int index; 225 226 /* 227 * XXX - it does not work well with removable controllers and 228 * interrupt sources !!! 229 */ 230 index = atomic_fetchadd_int(&intrcnt_index, 2); 231 isrc->isrc_index = index; 232 isrc->isrc_count = &intrcnt[index]; 233 isrc_update_name(isrc, NULL); 234 } 235 236 #ifdef SMP 237 /* 238 * Virtualization for interrupt source IPI counters setup. 239 */ 240 u_long * 241 intr_ipi_setup_counters(const char *name) 242 { 243 u_int index, i; 244 char str[INTRNAME_LEN]; 245 246 index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); 247 for (i = 0; i < MAXCPU; i++) { 248 snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 249 intrcnt_setname(str, index + i); 250 } 251 return (&intrcnt[index]); 252 } 253 #endif 254 255 /* 256 * Main interrupt dispatch handler. It's called straight 257 * from the assembler, where CPU interrupt is served. 258 */ 259 void 260 intr_irq_handler(struct trapframe *tf) 261 { 262 struct trapframe * oldframe; 263 struct thread * td; 264 265 KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); 266 267 PCPU_INC(cnt.v_intr); 268 critical_enter(); 269 td = curthread; 270 oldframe = td->td_intr_frame; 271 td->td_intr_frame = tf; 272 irq_root_filter(irq_root_arg); 273 td->td_intr_frame = oldframe; 274 critical_exit(); 275 } 276 277 /* 278 * interrupt controller dispatch function for interrupts. It should 279 * be called straight from the interrupt controller, when associated interrupt 280 * source is learned. 281 */ 282 void 283 intr_irq_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 284 { 285 286 KASSERT(isrc != NULL, ("%s: no source", __func__)); 287 288 isrc_increment_count(isrc); 289 290 #ifdef INTR_SOLO 291 if (isrc->isrc_filter != NULL) { 292 int error; 293 error = isrc->isrc_filter(isrc->isrc_arg, tf); 294 PIC_POST_FILTER(isrc->isrc_dev, isrc); 295 if (error == FILTER_HANDLED) 296 return; 297 } else 298 #endif 299 if (isrc->isrc_event != NULL) { 300 if (intr_event_handle(isrc->isrc_event, tf) == 0) 301 return; 302 } 303 304 isrc_increment_straycount(isrc); 305 PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc); 306 307 device_printf(isrc->isrc_dev, "stray irq <%s> disabled", 308 isrc->isrc_name); 309 } 310 311 /* 312 * Allocate interrupt source. 313 */ 314 static struct intr_irqsrc * 315 isrc_alloc(u_int type, u_int extsize) 316 { 317 struct intr_irqsrc *isrc; 318 319 isrc = malloc(sizeof(*isrc) + extsize, M_INTRNG, M_WAITOK | M_ZERO); 320 isrc->isrc_irq = IRQ_INVALID; /* just to be safe */ 321 isrc->isrc_type = type; 322 isrc->isrc_nspc_type = INTR_IRQ_NSPC_NONE; 323 isrc->isrc_trig = INTR_TRIGGER_CONFORM; 324 isrc->isrc_pol = INTR_POLARITY_CONFORM; 325 CPU_ZERO(&isrc->isrc_cpu); 326 return (isrc); 327 } 328 329 /* 330 * Free interrupt source. 331 */ 332 static void 333 isrc_free(struct intr_irqsrc *isrc) 334 { 335 336 free(isrc, M_INTRNG); 337 } 338 339 void 340 intr_irq_set_name(struct intr_irqsrc *isrc, const char *fmt, ...) 341 { 342 va_list ap; 343 344 va_start(ap, fmt); 345 vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 346 va_end(ap); 347 } 348 349 /* 350 * Alloc unique interrupt number (resource handle) for interrupt source. 351 * 352 * There could be various strategies how to allocate free interrupt number 353 * (resource handle) for new interrupt source. 354 * 355 * 1. Handles are always allocated forward, so handles are not recycled 356 * immediately. However, if only one free handle left which is reused 357 * constantly... 358 */ 359 static int 360 isrc_alloc_irq_locked(struct intr_irqsrc *isrc) 361 { 362 u_int maxirqs, irq; 363 364 mtx_assert(&isrc_table_lock, MA_OWNED); 365 366 maxirqs = nitems(irq_sources); 367 if (irq_next_free >= maxirqs) 368 return (ENOSPC); 369 370 for (irq = irq_next_free; irq < maxirqs; irq++) { 371 if (irq_sources[irq] == NULL) 372 goto found; 373 } 374 for (irq = 0; irq < irq_next_free; irq++) { 375 if (irq_sources[irq] == NULL) 376 goto found; 377 } 378 379 irq_next_free = maxirqs; 380 return (ENOSPC); 381 382 found: 383 isrc->isrc_irq = irq; 384 irq_sources[irq] = isrc; 385 386 intr_irq_set_name(isrc, "irq%u", irq); 387 isrc_setup_counters(isrc); 388 389 irq_next_free = irq + 1; 390 if (irq_next_free >= maxirqs) 391 irq_next_free = 0; 392 return (0); 393 } 394 #ifdef notyet 395 /* 396 * Free unique interrupt number (resource handle) from interrupt source. 397 */ 398 static int 399 isrc_free_irq(struct intr_irqsrc *isrc) 400 { 401 u_int maxirqs; 402 403 mtx_assert(&isrc_table_lock, MA_NOTOWNED); 404 405 maxirqs = nitems(irq_sources); 406 if (isrc->isrc_irq >= maxirqs) 407 return (EINVAL); 408 409 mtx_lock(&isrc_table_lock); 410 if (irq_sources[isrc->isrc_irq] != isrc) { 411 mtx_unlock(&isrc_table_lock); 412 return (EINVAL); 413 } 414 415 irq_sources[isrc->isrc_irq] = NULL; 416 isrc->isrc_irq = IRQ_INVALID; /* just to be safe */ 417 mtx_unlock(&isrc_table_lock); 418 419 return (0); 420 } 421 #endif 422 /* 423 * Lookup interrupt source by interrupt number (resource handle). 424 */ 425 static struct intr_irqsrc * 426 isrc_lookup(u_int irq) 427 { 428 429 if (irq < nitems(irq_sources)) 430 return (irq_sources[irq]); 431 return (NULL); 432 } 433 434 /* 435 * Lookup interrupt source by namespace description. 436 */ 437 static struct intr_irqsrc * 438 isrc_namespace_lookup(device_t dev, uint16_t type, uint16_t num) 439 { 440 u_int irq; 441 struct intr_irqsrc *isrc; 442 443 mtx_assert(&isrc_table_lock, MA_OWNED); 444 445 for (irq = 0; irq < nitems(irq_sources); irq++) { 446 isrc = irq_sources[irq]; 447 if (isrc != NULL && isrc->isrc_dev == dev && 448 isrc->isrc_nspc_type == type && isrc->isrc_nspc_num == num) 449 return (isrc); 450 } 451 return (NULL); 452 } 453 454 /* 455 * Map interrupt source according to namespace into framework. If such mapping 456 * does not exist, create it. Return unique interrupt number (resource handle) 457 * associated with mapped interrupt source. 458 */ 459 u_int 460 intr_namespace_map_irq(device_t dev, uint16_t type, uint16_t num) 461 { 462 struct intr_irqsrc *isrc, *new_isrc; 463 int error; 464 465 new_isrc = isrc_alloc(INTR_ISRCT_NAMESPACE, 0); 466 467 mtx_lock(&isrc_table_lock); 468 isrc = isrc_namespace_lookup(dev, type, num); 469 if (isrc != NULL) { 470 mtx_unlock(&isrc_table_lock); 471 isrc_free(new_isrc); 472 return (isrc->isrc_irq); /* already mapped */ 473 } 474 475 error = isrc_alloc_irq_locked(new_isrc); 476 if (error != 0) { 477 mtx_unlock(&isrc_table_lock); 478 isrc_free(new_isrc); 479 return (IRQ_INVALID); /* no space left */ 480 } 481 482 new_isrc->isrc_dev = dev; 483 new_isrc->isrc_nspc_type = type; 484 new_isrc->isrc_nspc_num = num; 485 mtx_unlock(&isrc_table_lock); 486 487 return (new_isrc->isrc_irq); 488 } 489 490 #ifdef FDT 491 /* 492 * Lookup interrupt source by FDT description. 493 */ 494 static struct intr_irqsrc * 495 isrc_fdt_lookup(intptr_t xref, pcell_t *cells, u_int ncells) 496 { 497 u_int irq, cellsize; 498 struct intr_irqsrc *isrc; 499 500 mtx_assert(&isrc_table_lock, MA_OWNED); 501 502 cellsize = ncells * sizeof(*cells); 503 for (irq = 0; irq < nitems(irq_sources); irq++) { 504 isrc = irq_sources[irq]; 505 if (isrc != NULL && isrc->isrc_type == INTR_ISRCT_FDT && 506 isrc->isrc_xref == xref && isrc->isrc_ncells == ncells && 507 memcmp(isrc->isrc_cells, cells, cellsize) == 0) 508 return (isrc); 509 } 510 return (NULL); 511 } 512 513 /* 514 * Map interrupt source according to FDT data into framework. If such mapping 515 * does not exist, create it. Return unique interrupt number (resource handle) 516 * associated with mapped interrupt source. 517 */ 518 u_int 519 intr_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) 520 { 521 struct intr_irqsrc *isrc, *new_isrc; 522 u_int cellsize; 523 intptr_t xref; 524 int error; 525 526 xref = (intptr_t)node; /* It's so simple for now. */ 527 528 cellsize = ncells * sizeof(*cells); 529 new_isrc = isrc_alloc(INTR_ISRCT_FDT, cellsize); 530 531 mtx_lock(&isrc_table_lock); 532 isrc = isrc_fdt_lookup(xref, cells, ncells); 533 if (isrc != NULL) { 534 mtx_unlock(&isrc_table_lock); 535 isrc_free(new_isrc); 536 return (isrc->isrc_irq); /* already mapped */ 537 } 538 539 error = isrc_alloc_irq_locked(new_isrc); 540 if (error != 0) { 541 mtx_unlock(&isrc_table_lock); 542 isrc_free(new_isrc); 543 return (IRQ_INVALID); /* no space left */ 544 } 545 546 new_isrc->isrc_xref = xref; 547 new_isrc->isrc_ncells = ncells; 548 memcpy(new_isrc->isrc_cells, cells, cellsize); 549 mtx_unlock(&isrc_table_lock); 550 551 return (new_isrc->isrc_irq); 552 } 553 #endif 554 555 /* 556 * Register interrupt source into interrupt controller. 557 */ 558 static int 559 isrc_register(struct intr_irqsrc *isrc) 560 { 561 struct intr_pic *pic; 562 boolean_t is_percpu; 563 int error; 564 565 if (isrc->isrc_flags & INTR_ISRCF_REGISTERED) 566 return (0); 567 568 if (isrc->isrc_dev == NULL) { 569 pic = pic_lookup(NULL, isrc->isrc_xref); 570 if (pic == NULL || pic->pic_dev == NULL) 571 return (ESRCH); 572 isrc->isrc_dev = pic->pic_dev; 573 } 574 575 error = PIC_REGISTER(isrc->isrc_dev, isrc, &is_percpu); 576 if (error != 0) 577 return (error); 578 579 mtx_lock(&isrc_table_lock); 580 isrc->isrc_flags |= INTR_ISRCF_REGISTERED; 581 if (is_percpu) 582 isrc->isrc_flags |= INTR_ISRCF_PERCPU; 583 isrc_update_name(isrc, NULL); 584 mtx_unlock(&isrc_table_lock); 585 return (0); 586 } 587 588 #ifdef INTR_SOLO 589 /* 590 * Setup filter into interrupt source. 591 */ 592 static int 593 iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 594 intr_irq_filter_t *filter, void *arg, void **cookiep) 595 { 596 597 if (filter == NULL) 598 return (EINVAL); 599 600 mtx_lock(&isrc_table_lock); 601 /* 602 * Make sure that we do not mix the two ways 603 * how we handle interrupt sources. 604 */ 605 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 606 mtx_unlock(&isrc_table_lock); 607 return (EBUSY); 608 } 609 isrc->isrc_filter = filter; 610 isrc->isrc_arg = arg; 611 isrc_update_name(isrc, name); 612 mtx_unlock(&isrc_table_lock); 613 614 *cookiep = isrc; 615 return (0); 616 } 617 #endif 618 619 /* 620 * Interrupt source pre_ithread method for MI interrupt framework. 621 */ 622 static void 623 intr_isrc_pre_ithread(void *arg) 624 { 625 struct intr_irqsrc *isrc = arg; 626 627 PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 628 } 629 630 /* 631 * Interrupt source post_ithread method for MI interrupt framework. 632 */ 633 static void 634 intr_isrc_post_ithread(void *arg) 635 { 636 struct intr_irqsrc *isrc = arg; 637 638 PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 639 } 640 641 /* 642 * Interrupt source post_filter method for MI interrupt framework. 643 */ 644 static void 645 intr_isrc_post_filter(void *arg) 646 { 647 struct intr_irqsrc *isrc = arg; 648 649 PIC_POST_FILTER(isrc->isrc_dev, isrc); 650 } 651 652 /* 653 * Interrupt source assign_cpu method for MI interrupt framework. 654 */ 655 static int 656 intr_isrc_assign_cpu(void *arg, int cpu) 657 { 658 #ifdef SMP 659 struct intr_irqsrc *isrc = arg; 660 int error; 661 662 if (isrc->isrc_dev != intr_irq_root_dev) 663 return (EINVAL); 664 665 mtx_lock(&isrc_table_lock); 666 if (cpu == NOCPU) { 667 CPU_ZERO(&isrc->isrc_cpu); 668 isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 669 } else { 670 CPU_SETOF(cpu, &isrc->isrc_cpu); 671 isrc->isrc_flags |= INTR_ISRCF_BOUND; 672 } 673 674 /* 675 * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 676 * re-balance it to another CPU or enable it on more CPUs. However, 677 * PIC is expected to change isrc_cpu appropriately to keep us well 678 * informed if the call is successfull. 679 */ 680 if (irq_assign_cpu) { 681 error = PIC_BIND(isrc->isrc_dev, isrc); 682 if (error) { 683 CPU_ZERO(&isrc->isrc_cpu); 684 mtx_unlock(&isrc_table_lock); 685 return (error); 686 } 687 } 688 mtx_unlock(&isrc_table_lock); 689 return (0); 690 #else 691 return (EOPNOTSUPP); 692 #endif 693 } 694 695 /* 696 * Create interrupt event for interrupt source. 697 */ 698 static int 699 isrc_event_create(struct intr_irqsrc *isrc) 700 { 701 struct intr_event *ie; 702 int error; 703 704 error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 705 intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 706 intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 707 if (error) 708 return (error); 709 710 mtx_lock(&isrc_table_lock); 711 /* 712 * Make sure that we do not mix the two ways 713 * how we handle interrupt sources. Let contested event wins. 714 */ 715 #ifdef INTR_SOLO 716 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 717 #else 718 if (isrc->isrc_event != NULL) { 719 #endif 720 mtx_unlock(&isrc_table_lock); 721 intr_event_destroy(ie); 722 return (isrc->isrc_event != NULL ? EBUSY : 0); 723 } 724 isrc->isrc_event = ie; 725 mtx_unlock(&isrc_table_lock); 726 727 return (0); 728 } 729 #ifdef notyet 730 /* 731 * Destroy interrupt event for interrupt source. 732 */ 733 static void 734 isrc_event_destroy(struct intr_irqsrc *isrc) 735 { 736 struct intr_event *ie; 737 738 mtx_lock(&isrc_table_lock); 739 ie = isrc->isrc_event; 740 isrc->isrc_event = NULL; 741 mtx_unlock(&isrc_table_lock); 742 743 if (ie != NULL) 744 intr_event_destroy(ie); 745 } 746 #endif 747 /* 748 * Add handler to interrupt source. 749 */ 750 static int 751 isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 752 driver_filter_t filter, driver_intr_t handler, void *arg, 753 enum intr_type flags, void **cookiep) 754 { 755 int error; 756 757 if (isrc->isrc_event == NULL) { 758 error = isrc_event_create(isrc); 759 if (error) 760 return (error); 761 } 762 763 error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 764 arg, intr_priority(flags), flags, cookiep); 765 if (error == 0) { 766 mtx_lock(&isrc_table_lock); 767 intrcnt_updatename(isrc); 768 mtx_unlock(&isrc_table_lock); 769 } 770 771 return (error); 772 } 773 774 /* 775 * Lookup interrupt controller locked. 776 */ 777 static struct intr_pic * 778 pic_lookup_locked(device_t dev, intptr_t xref) 779 { 780 struct intr_pic *pic; 781 782 mtx_assert(&pic_list_lock, MA_OWNED); 783 784 SLIST_FOREACH(pic, &pic_list, pic_next) { 785 if (pic->pic_xref != xref) 786 continue; 787 if (pic->pic_xref != 0 || pic->pic_dev == dev) 788 return (pic); 789 } 790 return (NULL); 791 } 792 793 /* 794 * Lookup interrupt controller. 795 */ 796 static struct intr_pic * 797 pic_lookup(device_t dev, intptr_t xref) 798 { 799 struct intr_pic *pic; 800 801 mtx_lock(&pic_list_lock); 802 pic = pic_lookup_locked(dev, xref); 803 mtx_unlock(&pic_list_lock); 804 805 return (pic); 806 } 807 808 /* 809 * Create interrupt controller. 810 */ 811 static struct intr_pic * 812 pic_create(device_t dev, intptr_t xref) 813 { 814 struct intr_pic *pic; 815 816 mtx_lock(&pic_list_lock); 817 pic = pic_lookup_locked(dev, xref); 818 if (pic != NULL) { 819 mtx_unlock(&pic_list_lock); 820 return (pic); 821 } 822 pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 823 pic->pic_xref = xref; 824 pic->pic_dev = dev; 825 SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 826 mtx_unlock(&pic_list_lock); 827 828 return (pic); 829 } 830 #ifdef notyet 831 /* 832 * Destroy interrupt controller. 833 */ 834 static void 835 pic_destroy(device_t dev, intptr_t xref) 836 { 837 struct intr_pic *pic; 838 839 mtx_lock(&pic_list_lock); 840 pic = pic_lookup_locked(dev, xref); 841 if (pic == NULL) { 842 mtx_unlock(&pic_list_lock); 843 return; 844 } 845 SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 846 mtx_unlock(&pic_list_lock); 847 848 free(pic, M_INTRNG); 849 } 850 #endif 851 /* 852 * Register interrupt controller. 853 */ 854 int 855 intr_pic_register(device_t dev, intptr_t xref) 856 { 857 struct intr_pic *pic; 858 859 pic = pic_create(dev, xref); 860 if (pic == NULL) 861 return (ENOMEM); 862 if (pic->pic_dev != dev) 863 return (EINVAL); /* XXX it could be many things. */ 864 865 debugf("PIC %p registered for %s <xref %x>\n", pic, 866 device_get_nameunit(dev), xref); 867 return (0); 868 } 869 870 /* 871 * Unregister interrupt controller. 872 */ 873 int 874 intr_pic_unregister(device_t dev, intptr_t xref) 875 { 876 877 panic("%s: not implemented", __func__); 878 } 879 880 /* 881 * Mark interrupt controller (itself) as a root one. 882 * 883 * Note that only an interrupt controller can really know its position 884 * in interrupt controller's tree. So root PIC must claim itself as a root. 885 * 886 * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 887 * page 30: 888 * "The root of the interrupt tree is determined when traversal 889 * of the interrupt tree reaches an interrupt controller node without 890 * an interrupts property and thus no explicit interrupt parent." 891 */ 892 int 893 intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 894 void *arg, u_int ipicount) 895 { 896 897 if (pic_lookup(dev, xref) == NULL) { 898 device_printf(dev, "not registered\n"); 899 return (EINVAL); 900 } 901 if (filter == NULL) { 902 device_printf(dev, "filter missing\n"); 903 return (EINVAL); 904 } 905 906 /* 907 * Only one interrupt controllers could be on the root for now. 908 * Note that we further suppose that there is not threaded interrupt 909 * routine (handler) on the root. See intr_irq_handler(). 910 */ 911 if (intr_irq_root_dev != NULL) { 912 device_printf(dev, "another root already set\n"); 913 return (EBUSY); 914 } 915 916 intr_irq_root_dev = dev; 917 irq_root_filter = filter; 918 irq_root_arg = arg; 919 irq_root_ipicount = ipicount; 920 921 debugf("irq root set to %s\n", device_get_nameunit(dev)); 922 return (0); 923 } 924 925 int 926 intr_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, 927 void *arg, u_int irq, int flags, void **cookiep) 928 { 929 const char *name; 930 struct intr_irqsrc *isrc; 931 int error; 932 933 name = device_get_nameunit(dev); 934 935 #ifdef INTR_SOLO 936 /* 937 * Standard handling is done thru MI interrupt framework. However, 938 * some interrupts could request solely own special handling. This 939 * non standard handling can be used for interrupt controllers without 940 * handler (filter only), so in case that interrupt controllers are 941 * chained, MI interrupt framework is called only in leaf controller. 942 * 943 * Note that root interrupt controller routine is served as well, 944 * however in intr_irq_handler(), i.e. main system dispatch routine. 945 */ 946 if (flags & INTR_SOLO && hand != NULL) { 947 debugf("irq %u cannot solo on %s\n", irq, name); 948 return (EINVAL); 949 } 950 #endif 951 952 isrc = isrc_lookup(irq); 953 if (isrc == NULL) { 954 debugf("irq %u without source on %s\n", irq, name); 955 return (EINVAL); 956 } 957 958 error = isrc_register(isrc); 959 if (error != 0) { 960 debugf("irq %u map error %d on %s\n", irq, error, name); 961 return (error); 962 } 963 964 #ifdef INTR_SOLO 965 if (flags & INTR_SOLO) { 966 error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 967 arg, cookiep); 968 debugf("irq %u setup filter error %d on %s\n", irq, error, 969 name); 970 } else 971 #endif 972 { 973 error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 974 cookiep); 975 debugf("irq %u add handler error %d on %s\n", irq, error, name); 976 } 977 if (error != 0) 978 return (error); 979 980 mtx_lock(&isrc_table_lock); 981 isrc->isrc_handlers++; 982 if (isrc->isrc_handlers == 1) { 983 PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 984 PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); 985 } 986 mtx_unlock(&isrc_table_lock); 987 return (0); 988 } 989 990 int 991 intr_irq_remove_handler(device_t dev, u_int irq, void *cookie) 992 { 993 struct intr_irqsrc *isrc; 994 int error; 995 996 isrc = isrc_lookup(irq); 997 if (isrc == NULL || isrc->isrc_handlers == 0) 998 return (EINVAL); 999 #ifdef INTR_SOLO 1000 if (isrc->isrc_filter != NULL) { 1001 if (isrc != cookie) 1002 return (EINVAL); 1003 1004 mtx_lock(&isrc_table_lock); 1005 isrc->isrc_filter = NULL; 1006 isrc->isrc_arg = NULL; 1007 isrc->isrc_handlers = 0; 1008 PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc); 1009 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1010 isrc_update_name(isrc, NULL); 1011 mtx_unlock(&isrc_table_lock); 1012 return (0); 1013 } 1014 #endif 1015 if (isrc != intr_handler_source(cookie)) 1016 return (EINVAL); 1017 1018 error = intr_event_remove_handler(cookie); 1019 if (error == 0) { 1020 mtx_lock(&isrc_table_lock); 1021 isrc->isrc_handlers--; 1022 if (isrc->isrc_handlers == 0) { 1023 PIC_DISABLE_SOURCE(isrc->isrc_dev, isrc); 1024 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1025 } 1026 intrcnt_updatename(isrc); 1027 mtx_unlock(&isrc_table_lock); 1028 } 1029 return (error); 1030 } 1031 1032 int 1033 intr_irq_config(u_int irq, enum intr_trigger trig, enum intr_polarity pol) 1034 { 1035 struct intr_irqsrc *isrc; 1036 1037 isrc = isrc_lookup(irq); 1038 if (isrc == NULL) 1039 return (EINVAL); 1040 1041 if (isrc->isrc_handlers != 0) 1042 return (EBUSY); /* interrrupt is enabled (active) */ 1043 1044 /* 1045 * Once an interrupt is enabled, we do not change its configuration. 1046 * A controller PIC_ENABLE_INTR() method is called when an interrupt 1047 * is going to be enabled. In this method, a controller should setup 1048 * the interrupt according to saved configuration parameters. 1049 */ 1050 isrc->isrc_trig = trig; 1051 isrc->isrc_pol = pol; 1052 1053 return (0); 1054 } 1055 1056 int 1057 intr_irq_describe(u_int irq, void *cookie, const char *descr) 1058 { 1059 struct intr_irqsrc *isrc; 1060 int error; 1061 1062 isrc = isrc_lookup(irq); 1063 if (isrc == NULL || isrc->isrc_handlers == 0) 1064 return (EINVAL); 1065 #ifdef INTR_SOLO 1066 if (isrc->isrc_filter != NULL) { 1067 if (isrc != cookie) 1068 return (EINVAL); 1069 1070 mtx_lock(&isrc_table_lock); 1071 isrc_update_name(isrc, descr); 1072 mtx_unlock(&isrc_table_lock); 1073 return (0); 1074 } 1075 #endif 1076 error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 1077 if (error == 0) { 1078 mtx_lock(&isrc_table_lock); 1079 intrcnt_updatename(isrc); 1080 mtx_unlock(&isrc_table_lock); 1081 } 1082 return (error); 1083 } 1084 1085 #ifdef SMP 1086 int 1087 intr_irq_bind(u_int irq, int cpu) 1088 { 1089 struct intr_irqsrc *isrc; 1090 1091 isrc = isrc_lookup(irq); 1092 if (isrc == NULL || isrc->isrc_handlers == 0) 1093 return (EINVAL); 1094 #ifdef INTR_SOLO 1095 if (isrc->isrc_filter != NULL) 1096 return (intr_isrc_assign_cpu(isrc, cpu)); 1097 #endif 1098 return (intr_event_bind(isrc->isrc_event, cpu)); 1099 } 1100 1101 /* 1102 * Return the CPU that the next interrupt source should use. 1103 * For now just returns the next CPU according to round-robin. 1104 */ 1105 u_int 1106 intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 1107 { 1108 1109 if (!irq_assign_cpu || mp_ncpus == 1) 1110 return (PCPU_GET(cpuid)); 1111 1112 do { 1113 last_cpu++; 1114 if (last_cpu > mp_maxid) 1115 last_cpu = 0; 1116 } while (!CPU_ISSET(last_cpu, cpumask)); 1117 return (last_cpu); 1118 } 1119 1120 /* 1121 * Distribute all the interrupt sources among the available 1122 * CPUs once the AP's have been launched. 1123 */ 1124 static void 1125 intr_irq_shuffle(void *arg __unused) 1126 { 1127 struct intr_irqsrc *isrc; 1128 u_int i; 1129 1130 if (mp_ncpus == 1) 1131 return; 1132 1133 mtx_lock(&isrc_table_lock); 1134 irq_assign_cpu = TRUE; 1135 for (i = 0; i < NIRQ; i++) { 1136 isrc = irq_sources[i]; 1137 if (isrc == NULL || isrc->isrc_handlers == 0 || 1138 isrc->isrc_flags & INTR_ISRCF_PERCPU) 1139 continue; 1140 1141 if (isrc->isrc_event != NULL && 1142 isrc->isrc_flags & INTR_ISRCF_BOUND && 1143 isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 1144 panic("%s: CPU inconsistency", __func__); 1145 1146 if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 1147 CPU_ZERO(&isrc->isrc_cpu); /* start again */ 1148 1149 /* 1150 * We are in wicked position here if the following call fails 1151 * for bound ISRC. The best thing we can do is to clear 1152 * isrc_cpu so inconsistency with ie_cpu will be detectable. 1153 */ 1154 if (PIC_BIND(isrc->isrc_dev, isrc) != 0) 1155 CPU_ZERO(&isrc->isrc_cpu); 1156 } 1157 mtx_unlock(&isrc_table_lock); 1158 } 1159 SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1160 1161 #else 1162 u_int 1163 intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 1164 { 1165 1166 return (PCPU_GET(cpuid)); 1167 } 1168 #endif 1169 1170 void dosoftints(void); 1171 void 1172 dosoftints(void) 1173 { 1174 } 1175 1176 #ifdef SMP 1177 /* 1178 * Init interrupt controller on another CPU. 1179 */ 1180 void 1181 intr_pic_init_secondary(void) 1182 { 1183 1184 /* 1185 * QQQ: Only root PIC is aware of other CPUs ??? 1186 */ 1187 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 1188 1189 //mtx_lock(&isrc_table_lock); 1190 PIC_INIT_SECONDARY(intr_irq_root_dev); 1191 //mtx_unlock(&isrc_table_lock); 1192 } 1193 #endif 1194 1195 #ifdef DDB 1196 DB_SHOW_COMMAND(irqs, db_show_irqs) 1197 { 1198 u_int i, irqsum; 1199 struct intr_irqsrc *isrc; 1200 1201 for (irqsum = 0, i = 0; i < NIRQ; i++) { 1202 isrc = irq_sources[i]; 1203 if (isrc == NULL) 1204 continue; 1205 1206 db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 1207 isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1208 isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", 1209 isrc->isrc_count[0]); 1210 irqsum += isrc->isrc_count[0]; 1211 } 1212 db_printf("irq total %u\n", irqsum); 1213 } 1214 #endif 1215