1 /*- 2 * Copyright (c) 2015-2016 Svatopluk Kraus 3 * Copyright (c) 2015-2016 Michal Meloun 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 /* 32 * New-style Interrupt Framework 33 * 34 * TODO: - add support for disconnected PICs. 35 * - to support IPI (PPI) enabling on other CPUs if already started. 36 * - to complete things for removable PICs. 37 */ 38 39 #include "opt_ddb.h" 40 #include "opt_hwpmc_hooks.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/syslog.h> 46 #include <sys/malloc.h> 47 #include <sys/proc.h> 48 #include <sys/queue.h> 49 #include <sys/bus.h> 50 #include <sys/interrupt.h> 51 #include <sys/conf.h> 52 #include <sys/cpuset.h> 53 #include <sys/rman.h> 54 #include <sys/sched.h> 55 #include <sys/smp.h> 56 #ifdef HWPMC_HOOKS 57 #include <sys/pmckern.h> 58 #endif 59 60 #include <machine/atomic.h> 61 #include <machine/intr.h> 62 #include <machine/cpu.h> 63 #include <machine/smp.h> 64 #include <machine/stdarg.h> 65 66 #ifdef DDB 67 #include <ddb/ddb.h> 68 #endif 69 70 #include "pic_if.h" 71 #include "msi_if.h" 72 73 #define INTRNAME_LEN (2*MAXCOMLEN + 1) 74 75 #ifdef DEBUG 76 #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 77 printf(fmt,##args); } while (0) 78 #else 79 #define debugf(fmt, args...) 80 #endif 81 82 MALLOC_DECLARE(M_INTRNG); 83 MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 84 85 /* Main interrupt handler called from assembler -> 'hidden' for C code. */ 86 void intr_irq_handler(struct trapframe *tf); 87 88 /* Root interrupt controller stuff. */ 89 device_t intr_irq_root_dev; 90 static intr_irq_filter_t *irq_root_filter; 91 static void *irq_root_arg; 92 static u_int irq_root_ipicount; 93 94 struct intr_pic_child { 95 SLIST_ENTRY(intr_pic_child) pc_next; 96 struct intr_pic *pc_pic; 97 intr_child_irq_filter_t *pc_filter; 98 void *pc_filter_arg; 99 uintptr_t pc_start; 100 uintptr_t pc_length; 101 }; 102 103 /* Interrupt controller definition. */ 104 struct intr_pic { 105 SLIST_ENTRY(intr_pic) pic_next; 106 intptr_t pic_xref; /* hardware identification */ 107 device_t pic_dev; 108 #define FLAG_PIC (1 << 0) 109 #define FLAG_MSI (1 << 1) 110 u_int pic_flags; 111 struct mtx pic_child_lock; 112 SLIST_HEAD(, intr_pic_child) pic_children; 113 }; 114 115 static struct mtx pic_list_lock; 116 static SLIST_HEAD(, intr_pic) pic_list; 117 118 static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); 119 120 /* Interrupt source definition. */ 121 static struct mtx isrc_table_lock; 122 static struct intr_irqsrc *irq_sources[NIRQ]; 123 u_int irq_next_free; 124 125 #ifdef SMP 126 static boolean_t irq_assign_cpu = FALSE; 127 #endif 128 129 /* 130 * - 2 counters for each I/O interrupt. 131 * - MAXCPU counters for each IPI counters for SMP. 132 */ 133 #ifdef SMP 134 #define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) 135 #else 136 #define INTRCNT_COUNT (NIRQ * 2) 137 #endif 138 139 /* Data for MI statistics reporting. */ 140 u_long intrcnt[INTRCNT_COUNT]; 141 char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; 142 size_t sintrcnt = sizeof(intrcnt); 143 size_t sintrnames = sizeof(intrnames); 144 static u_int intrcnt_index; 145 146 static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); 147 static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); 148 static struct intr_map_data * intr_map_get_map_data(u_int res_id); 149 static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, 150 struct intr_map_data **data); 151 152 /* 153 * Interrupt framework initialization routine. 154 */ 155 static void 156 intr_irq_init(void *dummy __unused) 157 { 158 159 SLIST_INIT(&pic_list); 160 mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 161 162 mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 163 } 164 SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 165 166 static void 167 intrcnt_setname(const char *name, int index) 168 { 169 170 snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 171 INTRNAME_LEN - 1, name); 172 } 173 174 /* 175 * Update name for interrupt source with interrupt event. 176 */ 177 static void 178 intrcnt_updatename(struct intr_irqsrc *isrc) 179 { 180 181 /* QQQ: What about stray counter name? */ 182 mtx_assert(&isrc_table_lock, MA_OWNED); 183 intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 184 } 185 186 /* 187 * Virtualization for interrupt source interrupt counter increment. 188 */ 189 static inline void 190 isrc_increment_count(struct intr_irqsrc *isrc) 191 { 192 193 if (isrc->isrc_flags & INTR_ISRCF_PPI) 194 atomic_add_long(&isrc->isrc_count[0], 1); 195 else 196 isrc->isrc_count[0]++; 197 } 198 199 /* 200 * Virtualization for interrupt source interrupt stray counter increment. 201 */ 202 static inline void 203 isrc_increment_straycount(struct intr_irqsrc *isrc) 204 { 205 206 isrc->isrc_count[1]++; 207 } 208 209 /* 210 * Virtualization for interrupt source interrupt name update. 211 */ 212 static void 213 isrc_update_name(struct intr_irqsrc *isrc, const char *name) 214 { 215 char str[INTRNAME_LEN]; 216 217 mtx_assert(&isrc_table_lock, MA_OWNED); 218 219 if (name != NULL) { 220 snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 221 intrcnt_setname(str, isrc->isrc_index); 222 snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 223 name); 224 intrcnt_setname(str, isrc->isrc_index + 1); 225 } else { 226 snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 227 intrcnt_setname(str, isrc->isrc_index); 228 snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 229 intrcnt_setname(str, isrc->isrc_index + 1); 230 } 231 } 232 233 /* 234 * Virtualization for interrupt source interrupt counters setup. 235 */ 236 static void 237 isrc_setup_counters(struct intr_irqsrc *isrc) 238 { 239 u_int index; 240 241 /* 242 * XXX - it does not work well with removable controllers and 243 * interrupt sources !!! 244 */ 245 index = atomic_fetchadd_int(&intrcnt_index, 2); 246 isrc->isrc_index = index; 247 isrc->isrc_count = &intrcnt[index]; 248 isrc_update_name(isrc, NULL); 249 } 250 251 /* 252 * Virtualization for interrupt source interrupt counters release. 253 */ 254 static void 255 isrc_release_counters(struct intr_irqsrc *isrc) 256 { 257 258 panic("%s: not implemented", __func__); 259 } 260 261 #ifdef SMP 262 /* 263 * Virtualization for interrupt source IPI counters setup. 264 */ 265 u_long * 266 intr_ipi_setup_counters(const char *name) 267 { 268 u_int index, i; 269 char str[INTRNAME_LEN]; 270 271 index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); 272 for (i = 0; i < MAXCPU; i++) { 273 snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 274 intrcnt_setname(str, index + i); 275 } 276 return (&intrcnt[index]); 277 } 278 #endif 279 280 /* 281 * Main interrupt dispatch handler. It's called straight 282 * from the assembler, where CPU interrupt is served. 283 */ 284 void 285 intr_irq_handler(struct trapframe *tf) 286 { 287 struct trapframe * oldframe; 288 struct thread * td; 289 290 KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); 291 292 PCPU_INC(cnt.v_intr); 293 critical_enter(); 294 td = curthread; 295 oldframe = td->td_intr_frame; 296 td->td_intr_frame = tf; 297 irq_root_filter(irq_root_arg); 298 td->td_intr_frame = oldframe; 299 critical_exit(); 300 #ifdef HWPMC_HOOKS 301 if (pmc_hook && TRAPF_USERMODE(tf) && 302 (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) 303 pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); 304 #endif 305 } 306 307 int 308 intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) 309 { 310 struct intr_pic_child *child; 311 bool found; 312 313 found = false; 314 mtx_lock_spin(&parent->pic_child_lock); 315 SLIST_FOREACH(child, &parent->pic_children, pc_next) { 316 if (child->pc_start <= irq && 317 irq < (child->pc_start + child->pc_length)) { 318 found = true; 319 break; 320 } 321 } 322 mtx_unlock_spin(&parent->pic_child_lock); 323 324 if (found) 325 return (child->pc_filter(child->pc_filter_arg, irq)); 326 327 return (FILTER_STRAY); 328 } 329 330 /* 331 * interrupt controller dispatch function for interrupts. It should 332 * be called straight from the interrupt controller, when associated interrupt 333 * source is learned. 334 */ 335 int 336 intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 337 { 338 339 KASSERT(isrc != NULL, ("%s: no source", __func__)); 340 341 isrc_increment_count(isrc); 342 343 #ifdef INTR_SOLO 344 if (isrc->isrc_filter != NULL) { 345 int error; 346 error = isrc->isrc_filter(isrc->isrc_arg, tf); 347 PIC_POST_FILTER(isrc->isrc_dev, isrc); 348 if (error == FILTER_HANDLED) 349 return (0); 350 } else 351 #endif 352 if (isrc->isrc_event != NULL) { 353 if (intr_event_handle(isrc->isrc_event, tf) == 0) 354 return (0); 355 } 356 357 isrc_increment_straycount(isrc); 358 return (EINVAL); 359 } 360 361 /* 362 * Alloc unique interrupt number (resource handle) for interrupt source. 363 * 364 * There could be various strategies how to allocate free interrupt number 365 * (resource handle) for new interrupt source. 366 * 367 * 1. Handles are always allocated forward, so handles are not recycled 368 * immediately. However, if only one free handle left which is reused 369 * constantly... 370 */ 371 static inline int 372 isrc_alloc_irq(struct intr_irqsrc *isrc) 373 { 374 u_int maxirqs, irq; 375 376 mtx_assert(&isrc_table_lock, MA_OWNED); 377 378 maxirqs = nitems(irq_sources); 379 if (irq_next_free >= maxirqs) 380 return (ENOSPC); 381 382 for (irq = irq_next_free; irq < maxirqs; irq++) { 383 if (irq_sources[irq] == NULL) 384 goto found; 385 } 386 for (irq = 0; irq < irq_next_free; irq++) { 387 if (irq_sources[irq] == NULL) 388 goto found; 389 } 390 391 irq_next_free = maxirqs; 392 return (ENOSPC); 393 394 found: 395 isrc->isrc_irq = irq; 396 irq_sources[irq] = isrc; 397 398 irq_next_free = irq + 1; 399 if (irq_next_free >= maxirqs) 400 irq_next_free = 0; 401 return (0); 402 } 403 404 /* 405 * Free unique interrupt number (resource handle) from interrupt source. 406 */ 407 static inline int 408 isrc_free_irq(struct intr_irqsrc *isrc) 409 { 410 411 mtx_assert(&isrc_table_lock, MA_OWNED); 412 413 if (isrc->isrc_irq >= nitems(irq_sources)) 414 return (EINVAL); 415 if (irq_sources[isrc->isrc_irq] != isrc) 416 return (EINVAL); 417 418 irq_sources[isrc->isrc_irq] = NULL; 419 isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 420 return (0); 421 } 422 423 /* 424 * Initialize interrupt source and register it into global interrupt table. 425 */ 426 int 427 intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, 428 const char *fmt, ...) 429 { 430 int error; 431 va_list ap; 432 433 bzero(isrc, sizeof(struct intr_irqsrc)); 434 isrc->isrc_dev = dev; 435 isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 436 isrc->isrc_flags = flags; 437 438 va_start(ap, fmt); 439 vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 440 va_end(ap); 441 442 mtx_lock(&isrc_table_lock); 443 error = isrc_alloc_irq(isrc); 444 if (error != 0) { 445 mtx_unlock(&isrc_table_lock); 446 return (error); 447 } 448 /* 449 * Setup interrupt counters, but not for IPI sources. Those are setup 450 * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust 451 * our counter pool. 452 */ 453 if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 454 isrc_setup_counters(isrc); 455 mtx_unlock(&isrc_table_lock); 456 return (0); 457 } 458 459 /* 460 * Deregister interrupt source from global interrupt table. 461 */ 462 int 463 intr_isrc_deregister(struct intr_irqsrc *isrc) 464 { 465 int error; 466 467 mtx_lock(&isrc_table_lock); 468 if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 469 isrc_release_counters(isrc); 470 error = isrc_free_irq(isrc); 471 mtx_unlock(&isrc_table_lock); 472 return (error); 473 } 474 475 #ifdef SMP 476 /* 477 * A support function for a PIC to decide if provided ISRC should be inited 478 * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of 479 * struct intr_irqsrc is the following: 480 * 481 * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus 482 * set in isrc_cpu. If not, the ISRC should be inited on every cpu and 483 * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. 484 */ 485 bool 486 intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) 487 { 488 489 if (isrc->isrc_handlers == 0) 490 return (false); 491 if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) 492 return (false); 493 if (isrc->isrc_flags & INTR_ISRCF_BOUND) 494 return (CPU_ISSET(cpu, &isrc->isrc_cpu)); 495 496 CPU_SET(cpu, &isrc->isrc_cpu); 497 return (true); 498 } 499 #endif 500 501 #ifdef INTR_SOLO 502 /* 503 * Setup filter into interrupt source. 504 */ 505 static int 506 iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 507 intr_irq_filter_t *filter, void *arg, void **cookiep) 508 { 509 510 if (filter == NULL) 511 return (EINVAL); 512 513 mtx_lock(&isrc_table_lock); 514 /* 515 * Make sure that we do not mix the two ways 516 * how we handle interrupt sources. 517 */ 518 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 519 mtx_unlock(&isrc_table_lock); 520 return (EBUSY); 521 } 522 isrc->isrc_filter = filter; 523 isrc->isrc_arg = arg; 524 isrc_update_name(isrc, name); 525 mtx_unlock(&isrc_table_lock); 526 527 *cookiep = isrc; 528 return (0); 529 } 530 #endif 531 532 /* 533 * Interrupt source pre_ithread method for MI interrupt framework. 534 */ 535 static void 536 intr_isrc_pre_ithread(void *arg) 537 { 538 struct intr_irqsrc *isrc = arg; 539 540 PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 541 } 542 543 /* 544 * Interrupt source post_ithread method for MI interrupt framework. 545 */ 546 static void 547 intr_isrc_post_ithread(void *arg) 548 { 549 struct intr_irqsrc *isrc = arg; 550 551 PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 552 } 553 554 /* 555 * Interrupt source post_filter method for MI interrupt framework. 556 */ 557 static void 558 intr_isrc_post_filter(void *arg) 559 { 560 struct intr_irqsrc *isrc = arg; 561 562 PIC_POST_FILTER(isrc->isrc_dev, isrc); 563 } 564 565 /* 566 * Interrupt source assign_cpu method for MI interrupt framework. 567 */ 568 static int 569 intr_isrc_assign_cpu(void *arg, int cpu) 570 { 571 #ifdef SMP 572 struct intr_irqsrc *isrc = arg; 573 int error; 574 575 if (isrc->isrc_dev != intr_irq_root_dev) 576 return (EINVAL); 577 578 mtx_lock(&isrc_table_lock); 579 if (cpu == NOCPU) { 580 CPU_ZERO(&isrc->isrc_cpu); 581 isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 582 } else { 583 CPU_SETOF(cpu, &isrc->isrc_cpu); 584 isrc->isrc_flags |= INTR_ISRCF_BOUND; 585 } 586 587 /* 588 * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 589 * re-balance it to another CPU or enable it on more CPUs. However, 590 * PIC is expected to change isrc_cpu appropriately to keep us well 591 * informed if the call is successful. 592 */ 593 if (irq_assign_cpu) { 594 error = PIC_BIND_INTR(isrc->isrc_dev, isrc); 595 if (error) { 596 CPU_ZERO(&isrc->isrc_cpu); 597 mtx_unlock(&isrc_table_lock); 598 return (error); 599 } 600 } 601 mtx_unlock(&isrc_table_lock); 602 return (0); 603 #else 604 return (EOPNOTSUPP); 605 #endif 606 } 607 608 /* 609 * Create interrupt event for interrupt source. 610 */ 611 static int 612 isrc_event_create(struct intr_irqsrc *isrc) 613 { 614 struct intr_event *ie; 615 int error; 616 617 error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 618 intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 619 intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 620 if (error) 621 return (error); 622 623 mtx_lock(&isrc_table_lock); 624 /* 625 * Make sure that we do not mix the two ways 626 * how we handle interrupt sources. Let contested event wins. 627 */ 628 #ifdef INTR_SOLO 629 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 630 #else 631 if (isrc->isrc_event != NULL) { 632 #endif 633 mtx_unlock(&isrc_table_lock); 634 intr_event_destroy(ie); 635 return (isrc->isrc_event != NULL ? EBUSY : 0); 636 } 637 isrc->isrc_event = ie; 638 mtx_unlock(&isrc_table_lock); 639 640 return (0); 641 } 642 #ifdef notyet 643 /* 644 * Destroy interrupt event for interrupt source. 645 */ 646 static void 647 isrc_event_destroy(struct intr_irqsrc *isrc) 648 { 649 struct intr_event *ie; 650 651 mtx_lock(&isrc_table_lock); 652 ie = isrc->isrc_event; 653 isrc->isrc_event = NULL; 654 mtx_unlock(&isrc_table_lock); 655 656 if (ie != NULL) 657 intr_event_destroy(ie); 658 } 659 #endif 660 /* 661 * Add handler to interrupt source. 662 */ 663 static int 664 isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 665 driver_filter_t filter, driver_intr_t handler, void *arg, 666 enum intr_type flags, void **cookiep) 667 { 668 int error; 669 670 if (isrc->isrc_event == NULL) { 671 error = isrc_event_create(isrc); 672 if (error) 673 return (error); 674 } 675 676 error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 677 arg, intr_priority(flags), flags, cookiep); 678 if (error == 0) { 679 mtx_lock(&isrc_table_lock); 680 intrcnt_updatename(isrc); 681 mtx_unlock(&isrc_table_lock); 682 } 683 684 return (error); 685 } 686 687 /* 688 * Lookup interrupt controller locked. 689 */ 690 static inline struct intr_pic * 691 pic_lookup_locked(device_t dev, intptr_t xref) 692 { 693 struct intr_pic *pic; 694 695 mtx_assert(&pic_list_lock, MA_OWNED); 696 697 if (dev == NULL && xref == 0) 698 return (NULL); 699 700 /* Note that pic->pic_dev is never NULL on registered PIC. */ 701 SLIST_FOREACH(pic, &pic_list, pic_next) { 702 if (dev == NULL) { 703 if (xref == pic->pic_xref) 704 return (pic); 705 } else if (xref == 0 || pic->pic_xref == 0) { 706 if (dev == pic->pic_dev) 707 return (pic); 708 } else if (xref == pic->pic_xref && dev == pic->pic_dev) 709 return (pic); 710 } 711 return (NULL); 712 } 713 714 /* 715 * Lookup interrupt controller. 716 */ 717 static struct intr_pic * 718 pic_lookup(device_t dev, intptr_t xref) 719 { 720 struct intr_pic *pic; 721 722 mtx_lock(&pic_list_lock); 723 pic = pic_lookup_locked(dev, xref); 724 mtx_unlock(&pic_list_lock); 725 return (pic); 726 } 727 728 /* 729 * Create interrupt controller. 730 */ 731 static struct intr_pic * 732 pic_create(device_t dev, intptr_t xref) 733 { 734 struct intr_pic *pic; 735 736 mtx_lock(&pic_list_lock); 737 pic = pic_lookup_locked(dev, xref); 738 if (pic != NULL) { 739 mtx_unlock(&pic_list_lock); 740 return (pic); 741 } 742 pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 743 if (pic == NULL) { 744 mtx_unlock(&pic_list_lock); 745 return (NULL); 746 } 747 pic->pic_xref = xref; 748 pic->pic_dev = dev; 749 mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); 750 SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 751 mtx_unlock(&pic_list_lock); 752 753 return (pic); 754 } 755 #ifdef notyet 756 /* 757 * Destroy interrupt controller. 758 */ 759 static void 760 pic_destroy(device_t dev, intptr_t xref) 761 { 762 struct intr_pic *pic; 763 764 mtx_lock(&pic_list_lock); 765 pic = pic_lookup_locked(dev, xref); 766 if (pic == NULL) { 767 mtx_unlock(&pic_list_lock); 768 return; 769 } 770 SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 771 mtx_unlock(&pic_list_lock); 772 773 free(pic, M_INTRNG); 774 } 775 #endif 776 /* 777 * Register interrupt controller. 778 */ 779 struct intr_pic * 780 intr_pic_register(device_t dev, intptr_t xref) 781 { 782 struct intr_pic *pic; 783 784 if (dev == NULL) 785 return (NULL); 786 pic = pic_create(dev, xref); 787 if (pic == NULL) 788 return (NULL); 789 790 pic->pic_flags |= FLAG_PIC; 791 792 debugf("PIC %p registered for %s <dev %p, xref %x>\n", pic, 793 device_get_nameunit(dev), dev, xref); 794 return (pic); 795 } 796 797 /* 798 * Unregister interrupt controller. 799 */ 800 int 801 intr_pic_deregister(device_t dev, intptr_t xref) 802 { 803 804 panic("%s: not implemented", __func__); 805 } 806 807 /* 808 * Mark interrupt controller (itself) as a root one. 809 * 810 * Note that only an interrupt controller can really know its position 811 * in interrupt controller's tree. So root PIC must claim itself as a root. 812 * 813 * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 814 * page 30: 815 * "The root of the interrupt tree is determined when traversal 816 * of the interrupt tree reaches an interrupt controller node without 817 * an interrupts property and thus no explicit interrupt parent." 818 */ 819 int 820 intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 821 void *arg, u_int ipicount) 822 { 823 struct intr_pic *pic; 824 825 pic = pic_lookup(dev, xref); 826 if (pic == NULL) { 827 device_printf(dev, "not registered\n"); 828 return (EINVAL); 829 } 830 831 KASSERT((pic->pic_flags & FLAG_PIC) != 0, 832 ("%s: Found a non-PIC controller: %s", __func__, 833 device_get_name(pic->pic_dev))); 834 835 if (filter == NULL) { 836 device_printf(dev, "filter missing\n"); 837 return (EINVAL); 838 } 839 840 /* 841 * Only one interrupt controllers could be on the root for now. 842 * Note that we further suppose that there is not threaded interrupt 843 * routine (handler) on the root. See intr_irq_handler(). 844 */ 845 if (intr_irq_root_dev != NULL) { 846 device_printf(dev, "another root already set\n"); 847 return (EBUSY); 848 } 849 850 intr_irq_root_dev = dev; 851 irq_root_filter = filter; 852 irq_root_arg = arg; 853 irq_root_ipicount = ipicount; 854 855 debugf("irq root set to %s\n", device_get_nameunit(dev)); 856 return (0); 857 } 858 859 /* 860 * Add a handler to manage a sub range of a parents interrupts. 861 */ 862 struct intr_pic * 863 intr_pic_add_handler(device_t parent, struct intr_pic *pic, 864 intr_child_irq_filter_t *filter, void *arg, uintptr_t start, 865 uintptr_t length) 866 { 867 struct intr_pic *parent_pic; 868 struct intr_pic_child *newchild; 869 #ifdef INVARIANTS 870 struct intr_pic_child *child; 871 #endif 872 873 parent_pic = pic_lookup(parent, 0); 874 if (parent_pic == NULL) 875 return (NULL); 876 877 newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); 878 newchild->pc_pic = pic; 879 newchild->pc_filter = filter; 880 newchild->pc_filter_arg = arg; 881 newchild->pc_start = start; 882 newchild->pc_length = length; 883 884 mtx_lock_spin(&parent_pic->pic_child_lock); 885 #ifdef INVARIANTS 886 SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { 887 KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", 888 __func__)); 889 } 890 #endif 891 SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); 892 mtx_unlock_spin(&parent_pic->pic_child_lock); 893 894 return (pic); 895 } 896 897 static int 898 intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, 899 struct intr_irqsrc **isrc) 900 { 901 struct intr_pic *pic; 902 struct intr_map_data_msi *msi; 903 904 if (data == NULL) 905 return (EINVAL); 906 907 pic = pic_lookup(dev, xref); 908 if (pic == NULL) 909 return (ESRCH); 910 911 switch (data->type) { 912 case INTR_MAP_DATA_MSI: 913 KASSERT((pic->pic_flags & FLAG_MSI) != 0, 914 ("%s: Found a non-MSI controller: %s", __func__, 915 device_get_name(pic->pic_dev))); 916 msi = (struct intr_map_data_msi *)data; 917 *isrc = msi->isrc; 918 return (0); 919 920 default: 921 KASSERT((pic->pic_flags & FLAG_PIC) != 0, 922 ("%s: Found a non-PIC controller: %s", __func__, 923 device_get_name(pic->pic_dev))); 924 return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); 925 926 } 927 } 928 929 int 930 intr_activate_irq(device_t dev, struct resource *res) 931 { 932 device_t map_dev; 933 intptr_t map_xref; 934 struct intr_map_data *data; 935 struct intr_irqsrc *isrc; 936 u_int res_id; 937 int error; 938 939 KASSERT(rman_get_start(res) == rman_get_end(res), 940 ("%s: more interrupts in resource", __func__)); 941 942 res_id = (u_int)rman_get_start(res); 943 if (intr_map_get_isrc(res_id) != NULL) 944 panic("Attempt to double activation of resource id: %u\n", 945 res_id); 946 intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); 947 error = intr_resolve_irq(map_dev, map_xref, data, &isrc); 948 if (error != 0) { 949 free(data, M_INTRNG); 950 /* XXX TODO DISCONECTED PICs */ 951 /* if (error == EINVAL) return(0); */ 952 return (error); 953 } 954 intr_map_set_isrc(res_id, isrc); 955 rman_set_virtual(res, data); 956 return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); 957 } 958 959 int 960 intr_deactivate_irq(device_t dev, struct resource *res) 961 { 962 struct intr_map_data *data; 963 struct intr_irqsrc *isrc; 964 u_int res_id; 965 int error; 966 967 KASSERT(rman_get_start(res) == rman_get_end(res), 968 ("%s: more interrupts in resource", __func__)); 969 970 res_id = (u_int)rman_get_start(res); 971 isrc = intr_map_get_isrc(res_id); 972 if (isrc == NULL) 973 panic("Attempt to deactivate non-active resource id: %u\n", 974 res_id); 975 976 data = rman_get_virtual(res); 977 error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); 978 intr_map_set_isrc(res_id, NULL); 979 rman_set_virtual(res, NULL); 980 free(data, M_INTRNG); 981 return (error); 982 } 983 984 int 985 intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, 986 driver_intr_t hand, void *arg, int flags, void **cookiep) 987 { 988 int error; 989 struct intr_map_data *data; 990 struct intr_irqsrc *isrc; 991 const char *name; 992 u_int res_id; 993 994 KASSERT(rman_get_start(res) == rman_get_end(res), 995 ("%s: more interrupts in resource", __func__)); 996 997 res_id = (u_int)rman_get_start(res); 998 isrc = intr_map_get_isrc(res_id); 999 if (isrc == NULL) { 1000 /* XXX TODO DISCONECTED PICs */ 1001 return (EINVAL); 1002 } 1003 1004 data = rman_get_virtual(res); 1005 name = device_get_nameunit(dev); 1006 1007 #ifdef INTR_SOLO 1008 /* 1009 * Standard handling is done through MI interrupt framework. However, 1010 * some interrupts could request solely own special handling. This 1011 * non standard handling can be used for interrupt controllers without 1012 * handler (filter only), so in case that interrupt controllers are 1013 * chained, MI interrupt framework is called only in leaf controller. 1014 * 1015 * Note that root interrupt controller routine is served as well, 1016 * however in intr_irq_handler(), i.e. main system dispatch routine. 1017 */ 1018 if (flags & INTR_SOLO && hand != NULL) { 1019 debugf("irq %u cannot solo on %s\n", irq, name); 1020 return (EINVAL); 1021 } 1022 1023 if (flags & INTR_SOLO) { 1024 error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 1025 arg, cookiep); 1026 debugf("irq %u setup filter error %d on %s\n", irq, error, 1027 name); 1028 } else 1029 #endif 1030 { 1031 error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 1032 cookiep); 1033 debugf("irq %u add handler error %d on %s\n", irq, error, name); 1034 } 1035 if (error != 0) 1036 return (error); 1037 1038 mtx_lock(&isrc_table_lock); 1039 error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); 1040 if (error == 0) { 1041 isrc->isrc_handlers++; 1042 if (isrc->isrc_handlers == 1) 1043 PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 1044 } 1045 mtx_unlock(&isrc_table_lock); 1046 if (error != 0) 1047 intr_event_remove_handler(*cookiep); 1048 return (error); 1049 } 1050 1051 int 1052 intr_teardown_irq(device_t dev, struct resource *res, void *cookie) 1053 { 1054 int error; 1055 struct intr_map_data *data; 1056 struct intr_irqsrc *isrc; 1057 u_int res_id; 1058 1059 KASSERT(rman_get_start(res) == rman_get_end(res), 1060 ("%s: more interrupts in resource", __func__)); 1061 1062 res_id = (u_int)rman_get_start(res); 1063 isrc = intr_map_get_isrc(res_id); 1064 if (isrc == NULL || isrc->isrc_handlers == 0) 1065 return (EINVAL); 1066 1067 data = rman_get_virtual(res); 1068 1069 #ifdef INTR_SOLO 1070 if (isrc->isrc_filter != NULL) { 1071 if (isrc != cookie) 1072 return (EINVAL); 1073 1074 mtx_lock(&isrc_table_lock); 1075 isrc->isrc_filter = NULL; 1076 isrc->isrc_arg = NULL; 1077 isrc->isrc_handlers = 0; 1078 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1079 PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 1080 isrc_update_name(isrc, NULL); 1081 mtx_unlock(&isrc_table_lock); 1082 return (0); 1083 } 1084 #endif 1085 if (isrc != intr_handler_source(cookie)) 1086 return (EINVAL); 1087 1088 error = intr_event_remove_handler(cookie); 1089 if (error == 0) { 1090 mtx_lock(&isrc_table_lock); 1091 isrc->isrc_handlers--; 1092 if (isrc->isrc_handlers == 0) 1093 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1094 PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 1095 intrcnt_updatename(isrc); 1096 mtx_unlock(&isrc_table_lock); 1097 } 1098 return (error); 1099 } 1100 1101 int 1102 intr_describe_irq(device_t dev, struct resource *res, void *cookie, 1103 const char *descr) 1104 { 1105 int error; 1106 struct intr_irqsrc *isrc; 1107 u_int res_id; 1108 1109 KASSERT(rman_get_start(res) == rman_get_end(res), 1110 ("%s: more interrupts in resource", __func__)); 1111 1112 res_id = (u_int)rman_get_start(res); 1113 isrc = intr_map_get_isrc(res_id); 1114 if (isrc == NULL || isrc->isrc_handlers == 0) 1115 return (EINVAL); 1116 #ifdef INTR_SOLO 1117 if (isrc->isrc_filter != NULL) { 1118 if (isrc != cookie) 1119 return (EINVAL); 1120 1121 mtx_lock(&isrc_table_lock); 1122 isrc_update_name(isrc, descr); 1123 mtx_unlock(&isrc_table_lock); 1124 return (0); 1125 } 1126 #endif 1127 error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 1128 if (error == 0) { 1129 mtx_lock(&isrc_table_lock); 1130 intrcnt_updatename(isrc); 1131 mtx_unlock(&isrc_table_lock); 1132 } 1133 return (error); 1134 } 1135 1136 #ifdef SMP 1137 int 1138 intr_bind_irq(device_t dev, struct resource *res, int cpu) 1139 { 1140 struct intr_irqsrc *isrc; 1141 u_int res_id; 1142 1143 KASSERT(rman_get_start(res) == rman_get_end(res), 1144 ("%s: more interrupts in resource", __func__)); 1145 1146 res_id = (u_int)rman_get_start(res); 1147 isrc = intr_map_get_isrc(res_id); 1148 if (isrc == NULL || isrc->isrc_handlers == 0) 1149 return (EINVAL); 1150 #ifdef INTR_SOLO 1151 if (isrc->isrc_filter != NULL) 1152 return (intr_isrc_assign_cpu(isrc, cpu)); 1153 #endif 1154 return (intr_event_bind(isrc->isrc_event, cpu)); 1155 } 1156 1157 /* 1158 * Return the CPU that the next interrupt source should use. 1159 * For now just returns the next CPU according to round-robin. 1160 */ 1161 u_int 1162 intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 1163 { 1164 1165 if (!irq_assign_cpu || mp_ncpus == 1) 1166 return (PCPU_GET(cpuid)); 1167 1168 do { 1169 last_cpu++; 1170 if (last_cpu > mp_maxid) 1171 last_cpu = 0; 1172 } while (!CPU_ISSET(last_cpu, cpumask)); 1173 return (last_cpu); 1174 } 1175 1176 /* 1177 * Distribute all the interrupt sources among the available 1178 * CPUs once the AP's have been launched. 1179 */ 1180 static void 1181 intr_irq_shuffle(void *arg __unused) 1182 { 1183 struct intr_irqsrc *isrc; 1184 u_int i; 1185 1186 if (mp_ncpus == 1) 1187 return; 1188 1189 mtx_lock(&isrc_table_lock); 1190 irq_assign_cpu = TRUE; 1191 for (i = 0; i < NIRQ; i++) { 1192 isrc = irq_sources[i]; 1193 if (isrc == NULL || isrc->isrc_handlers == 0 || 1194 isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) 1195 continue; 1196 1197 if (isrc->isrc_event != NULL && 1198 isrc->isrc_flags & INTR_ISRCF_BOUND && 1199 isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 1200 panic("%s: CPU inconsistency", __func__); 1201 1202 if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 1203 CPU_ZERO(&isrc->isrc_cpu); /* start again */ 1204 1205 /* 1206 * We are in wicked position here if the following call fails 1207 * for bound ISRC. The best thing we can do is to clear 1208 * isrc_cpu so inconsistency with ie_cpu will be detectable. 1209 */ 1210 if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) 1211 CPU_ZERO(&isrc->isrc_cpu); 1212 } 1213 mtx_unlock(&isrc_table_lock); 1214 } 1215 SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1216 1217 #else 1218 u_int 1219 intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 1220 { 1221 1222 return (PCPU_GET(cpuid)); 1223 } 1224 #endif 1225 1226 /* 1227 * Allocate memory for new intr_map_data structure. 1228 * Initialize common fields. 1229 */ 1230 struct intr_map_data * 1231 intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) 1232 { 1233 struct intr_map_data *data; 1234 1235 data = malloc(len, M_INTRNG, flags); 1236 data->type = type; 1237 data->len = len; 1238 return (data); 1239 } 1240 1241 void intr_free_intr_map_data(struct intr_map_data *data) 1242 { 1243 1244 free(data, M_INTRNG); 1245 } 1246 1247 1248 /* 1249 * Register a MSI/MSI-X interrupt controller 1250 */ 1251 int 1252 intr_msi_register(device_t dev, intptr_t xref) 1253 { 1254 struct intr_pic *pic; 1255 1256 if (dev == NULL) 1257 return (EINVAL); 1258 pic = pic_create(dev, xref); 1259 if (pic == NULL) 1260 return (ENOMEM); 1261 1262 pic->pic_flags |= FLAG_MSI; 1263 1264 debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 1265 device_get_nameunit(dev), dev, (uintmax_t)xref); 1266 return (0); 1267 } 1268 1269 int 1270 intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, 1271 int maxcount, int *irqs) 1272 { 1273 struct intr_irqsrc **isrc; 1274 struct intr_pic *pic; 1275 device_t pdev; 1276 struct intr_map_data_msi *msi; 1277 int err, i; 1278 1279 pic = pic_lookup(NULL, xref); 1280 if (pic == NULL) 1281 return (ESRCH); 1282 1283 KASSERT((pic->pic_flags & FLAG_MSI) != 0, 1284 ("%s: Found a non-MSI controller: %s", __func__, 1285 device_get_name(pic->pic_dev))); 1286 1287 isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 1288 err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); 1289 if (err != 0) { 1290 free(isrc, M_INTRNG); 1291 return (err); 1292 } 1293 1294 for (i = 0; i < count; i++) { 1295 msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1296 INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1297 msi-> isrc = isrc[i]; 1298 irqs[i] = intr_map_irq(pic->pic_dev, xref, 1299 (struct intr_map_data *)msi); 1300 1301 } 1302 free(isrc, M_INTRNG); 1303 1304 return (err); 1305 } 1306 1307 int 1308 intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, 1309 int *irqs) 1310 { 1311 struct intr_irqsrc **isrc; 1312 struct intr_pic *pic; 1313 struct intr_map_data_msi *msi; 1314 int i, err; 1315 1316 pic = pic_lookup(NULL, xref); 1317 if (pic == NULL) 1318 return (ESRCH); 1319 1320 KASSERT((pic->pic_flags & FLAG_MSI) != 0, 1321 ("%s: Found a non-MSI controller: %s", __func__, 1322 device_get_name(pic->pic_dev))); 1323 1324 isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 1325 1326 for (i = 0; i < count; i++) { 1327 msi = (struct intr_map_data_msi *) 1328 intr_map_get_map_data(irqs[i]); 1329 KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1330 ("%s: irq %d map data is not MSI", __func__, 1331 irqs[i])); 1332 isrc[i] = msi->isrc; 1333 } 1334 1335 err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); 1336 1337 for (i = 0; i < count; i++) { 1338 if (isrc[i] != NULL) 1339 intr_unmap_irq(irqs[i]); 1340 } 1341 1342 free(isrc, M_INTRNG); 1343 return (err); 1344 } 1345 1346 int 1347 intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) 1348 { 1349 struct intr_irqsrc *isrc; 1350 struct intr_pic *pic; 1351 device_t pdev; 1352 struct intr_map_data_msi *msi; 1353 int err; 1354 1355 pic = pic_lookup(NULL, xref); 1356 if (pic == NULL) 1357 return (ESRCH); 1358 1359 KASSERT((pic->pic_flags & FLAG_MSI) != 0, 1360 ("%s: Found a non-MSI controller: %s", __func__, 1361 device_get_name(pic->pic_dev))); 1362 1363 1364 err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); 1365 if (err != 0) 1366 return (err); 1367 1368 msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1369 INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1370 msi->isrc = isrc; 1371 *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); 1372 return (0); 1373 } 1374 1375 int 1376 intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) 1377 { 1378 struct intr_irqsrc *isrc; 1379 struct intr_pic *pic; 1380 struct intr_map_data_msi *msi; 1381 int err; 1382 1383 pic = pic_lookup(NULL, xref); 1384 if (pic == NULL) 1385 return (ESRCH); 1386 1387 KASSERT((pic->pic_flags & FLAG_MSI) != 0, 1388 ("%s: Found a non-MSI controller: %s", __func__, 1389 device_get_name(pic->pic_dev))); 1390 1391 msi = (struct intr_map_data_msi *) 1392 intr_map_get_map_data(irq); 1393 KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1394 ("%s: irq %d map data is not MSI", __func__, 1395 irq)); 1396 isrc = msi->isrc; 1397 if (isrc == NULL) { 1398 intr_unmap_irq(irq); 1399 return (EINVAL); 1400 } 1401 1402 err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); 1403 intr_unmap_irq(irq); 1404 1405 return (err); 1406 } 1407 1408 int 1409 intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, 1410 uint64_t *addr, uint32_t *data) 1411 { 1412 struct intr_irqsrc *isrc; 1413 struct intr_pic *pic; 1414 int err; 1415 1416 pic = pic_lookup(NULL, xref); 1417 if (pic == NULL) 1418 return (ESRCH); 1419 1420 KASSERT((pic->pic_flags & FLAG_MSI) != 0, 1421 ("%s: Found a non-MSI controller: %s", __func__, 1422 device_get_name(pic->pic_dev))); 1423 1424 isrc = intr_map_get_isrc(irq); 1425 if (isrc == NULL) 1426 return (EINVAL); 1427 1428 err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); 1429 return (err); 1430 } 1431 1432 1433 void dosoftints(void); 1434 void 1435 dosoftints(void) 1436 { 1437 } 1438 1439 #ifdef SMP 1440 /* 1441 * Init interrupt controller on another CPU. 1442 */ 1443 void 1444 intr_pic_init_secondary(void) 1445 { 1446 1447 /* 1448 * QQQ: Only root PIC is aware of other CPUs ??? 1449 */ 1450 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 1451 1452 //mtx_lock(&isrc_table_lock); 1453 PIC_INIT_SECONDARY(intr_irq_root_dev); 1454 //mtx_unlock(&isrc_table_lock); 1455 } 1456 #endif 1457 1458 #ifdef DDB 1459 DB_SHOW_COMMAND(irqs, db_show_irqs) 1460 { 1461 u_int i, irqsum; 1462 u_long num; 1463 struct intr_irqsrc *isrc; 1464 1465 for (irqsum = 0, i = 0; i < NIRQ; i++) { 1466 isrc = irq_sources[i]; 1467 if (isrc == NULL) 1468 continue; 1469 1470 num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; 1471 db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 1472 isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1473 isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); 1474 irqsum += num; 1475 } 1476 db_printf("irq total %u\n", irqsum); 1477 } 1478 #endif 1479 1480 /* 1481 * Interrupt mapping table functions. 1482 * 1483 * Please, keep this part separately, it can be transformed to 1484 * extension of standard resources. 1485 */ 1486 struct intr_map_entry 1487 { 1488 device_t dev; 1489 intptr_t xref; 1490 struct intr_map_data *map_data; 1491 struct intr_irqsrc *isrc; 1492 /* XXX TODO DISCONECTED PICs */ 1493 /*int flags */ 1494 }; 1495 1496 /* XXX Convert irq_map[] to dynamicaly expandable one. */ 1497 static struct intr_map_entry *irq_map[2 * NIRQ]; 1498 static int irq_map_count = nitems(irq_map); 1499 static int irq_map_first_free_idx; 1500 static struct mtx irq_map_lock; 1501 1502 static struct intr_irqsrc * 1503 intr_map_get_isrc(u_int res_id) 1504 { 1505 struct intr_irqsrc *isrc; 1506 1507 mtx_lock(&irq_map_lock); 1508 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) { 1509 mtx_unlock(&irq_map_lock); 1510 return (NULL); 1511 } 1512 isrc = irq_map[res_id]->isrc; 1513 mtx_unlock(&irq_map_lock); 1514 return (isrc); 1515 } 1516 1517 static void 1518 intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) 1519 { 1520 1521 mtx_lock(&irq_map_lock); 1522 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) { 1523 mtx_unlock(&irq_map_lock); 1524 return; 1525 } 1526 irq_map[res_id]->isrc = isrc; 1527 mtx_unlock(&irq_map_lock); 1528 } 1529 1530 /* 1531 * Get a copy of intr_map_entry data 1532 */ 1533 static struct intr_map_data * 1534 intr_map_get_map_data(u_int res_id) 1535 { 1536 struct intr_map_data *data; 1537 1538 data = NULL; 1539 mtx_lock(&irq_map_lock); 1540 if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1541 panic("Attempt to copy invalid resource id: %u\n", res_id); 1542 data = irq_map[res_id]->map_data; 1543 mtx_unlock(&irq_map_lock); 1544 1545 return (data); 1546 } 1547 1548 /* 1549 * Get a copy of intr_map_entry data 1550 */ 1551 static void 1552 intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, 1553 struct intr_map_data **data) 1554 { 1555 size_t len; 1556 1557 len = 0; 1558 mtx_lock(&irq_map_lock); 1559 if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1560 panic("Attempt to copy invalid resource id: %u\n", res_id); 1561 if (irq_map[res_id]->map_data != NULL) 1562 len = irq_map[res_id]->map_data->len; 1563 mtx_unlock(&irq_map_lock); 1564 1565 if (len == 0) 1566 *data = NULL; 1567 else 1568 *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); 1569 mtx_lock(&irq_map_lock); 1570 if (irq_map[res_id] == NULL) 1571 panic("Attempt to copy invalid resource id: %u\n", res_id); 1572 if (len != 0) { 1573 if (len != irq_map[res_id]->map_data->len) 1574 panic("Resource id: %u has changed.\n", res_id); 1575 memcpy(*data, irq_map[res_id]->map_data, len); 1576 } 1577 *map_dev = irq_map[res_id]->dev; 1578 *map_xref = irq_map[res_id]->xref; 1579 mtx_unlock(&irq_map_lock); 1580 } 1581 1582 1583 /* 1584 * Allocate and fill new entry in irq_map table. 1585 */ 1586 u_int 1587 intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) 1588 { 1589 u_int i; 1590 struct intr_map_entry *entry; 1591 1592 /* Prepare new entry first. */ 1593 entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); 1594 1595 entry->dev = dev; 1596 entry->xref = xref; 1597 entry->map_data = data; 1598 entry->isrc = NULL; 1599 1600 mtx_lock(&irq_map_lock); 1601 for (i = irq_map_first_free_idx; i < irq_map_count; i++) { 1602 if (irq_map[i] == NULL) { 1603 irq_map[i] = entry; 1604 irq_map_first_free_idx = i + 1; 1605 mtx_unlock(&irq_map_lock); 1606 return (i); 1607 } 1608 } 1609 mtx_unlock(&irq_map_lock); 1610 1611 /* XXX Expand irq_map table */ 1612 panic("IRQ mapping table is full."); 1613 } 1614 1615 /* 1616 * Remove and free mapping entry. 1617 */ 1618 void 1619 intr_unmap_irq(u_int res_id) 1620 { 1621 struct intr_map_entry *entry; 1622 1623 mtx_lock(&irq_map_lock); 1624 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) 1625 panic("Attempt to unmap invalid resource id: %u\n", res_id); 1626 entry = irq_map[res_id]; 1627 irq_map[res_id] = NULL; 1628 irq_map_first_free_idx = res_id; 1629 mtx_unlock(&irq_map_lock); 1630 intr_free_intr_map_data(entry->map_data); 1631 free(entry, M_INTRNG); 1632 } 1633 1634 /* 1635 * Clone mapping entry. 1636 */ 1637 u_int 1638 intr_map_clone_irq(u_int old_res_id) 1639 { 1640 device_t map_dev; 1641 intptr_t map_xref; 1642 struct intr_map_data *data; 1643 1644 intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); 1645 return (intr_map_irq(map_dev, map_xref, data)); 1646 } 1647 1648 static void 1649 intr_map_init(void *dummy __unused) 1650 { 1651 1652 mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); 1653 } 1654 SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); 1655