1 /*- 2 * Copyright (c) 2015-2016 Svatopluk Kraus 3 * Copyright (c) 2015-2016 Michal Meloun 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 /* 32 * New-style Interrupt Framework 33 * 34 * TODO: - add support for disconnected PICs. 35 * - to support IPI (PPI) enabling on other CPUs if already started. 36 * - to complete things for removable PICs. 37 */ 38 39 #include "opt_ddb.h" 40 #include "opt_hwpmc_hooks.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/syslog.h> 46 #include <sys/malloc.h> 47 #include <sys/proc.h> 48 #include <sys/queue.h> 49 #include <sys/bus.h> 50 #include <sys/interrupt.h> 51 #include <sys/conf.h> 52 #include <sys/cpuset.h> 53 #include <sys/rman.h> 54 #include <sys/sched.h> 55 #include <sys/smp.h> 56 #ifdef HWPMC_HOOKS 57 #include <sys/pmckern.h> 58 #endif 59 60 #include <machine/atomic.h> 61 #include <machine/intr.h> 62 #include <machine/cpu.h> 63 #include <machine/smp.h> 64 #include <machine/stdarg.h> 65 66 #ifdef DDB 67 #include <ddb/ddb.h> 68 #endif 69 70 #include "pic_if.h" 71 #include "msi_if.h" 72 73 #define INTRNAME_LEN (2*MAXCOMLEN + 1) 74 75 #ifdef DEBUG 76 #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 77 printf(fmt,##args); } while (0) 78 #else 79 #define debugf(fmt, args...) 80 #endif 81 82 MALLOC_DECLARE(M_INTRNG); 83 MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 84 85 /* Main interrupt handler called from assembler -> 'hidden' for C code. */ 86 void intr_irq_handler(struct trapframe *tf); 87 88 /* Root interrupt controller stuff. */ 89 device_t intr_irq_root_dev; 90 static intr_irq_filter_t *irq_root_filter; 91 static void *irq_root_arg; 92 static u_int irq_root_ipicount; 93 94 struct intr_pic_child { 95 SLIST_ENTRY(intr_pic_child) pc_next; 96 struct intr_pic *pc_pic; 97 intr_child_irq_filter_t *pc_filter; 98 void *pc_filter_arg; 99 uintptr_t pc_start; 100 uintptr_t pc_length; 101 }; 102 103 /* Interrupt controller definition. */ 104 struct intr_pic { 105 SLIST_ENTRY(intr_pic) pic_next; 106 intptr_t pic_xref; /* hardware identification */ 107 device_t pic_dev; 108 /* Only one of FLAG_PIC or FLAG_MSI may be set */ 109 #define FLAG_PIC (1 << 0) 110 #define FLAG_MSI (1 << 1) 111 #define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) 112 u_int pic_flags; 113 struct mtx pic_child_lock; 114 SLIST_HEAD(, intr_pic_child) pic_children; 115 }; 116 117 static struct mtx pic_list_lock; 118 static SLIST_HEAD(, intr_pic) pic_list; 119 120 static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, int flags); 121 122 /* Interrupt source definition. */ 123 static struct mtx isrc_table_lock; 124 static struct intr_irqsrc *irq_sources[NIRQ]; 125 u_int irq_next_free; 126 127 #ifdef SMP 128 static boolean_t irq_assign_cpu = FALSE; 129 #endif 130 131 /* 132 * - 2 counters for each I/O interrupt. 133 * - MAXCPU counters for each IPI counters for SMP. 134 */ 135 #ifdef SMP 136 #define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) 137 #else 138 #define INTRCNT_COUNT (NIRQ * 2) 139 #endif 140 141 /* Data for MI statistics reporting. */ 142 u_long intrcnt[INTRCNT_COUNT]; 143 char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; 144 size_t sintrcnt = sizeof(intrcnt); 145 size_t sintrnames = sizeof(intrnames); 146 static u_int intrcnt_index; 147 148 static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); 149 static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); 150 static struct intr_map_data * intr_map_get_map_data(u_int res_id); 151 static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, 152 struct intr_map_data **data); 153 154 /* 155 * Interrupt framework initialization routine. 156 */ 157 static void 158 intr_irq_init(void *dummy __unused) 159 { 160 161 SLIST_INIT(&pic_list); 162 mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 163 164 mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 165 } 166 SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 167 168 static void 169 intrcnt_setname(const char *name, int index) 170 { 171 172 snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 173 INTRNAME_LEN - 1, name); 174 } 175 176 /* 177 * Update name for interrupt source with interrupt event. 178 */ 179 static void 180 intrcnt_updatename(struct intr_irqsrc *isrc) 181 { 182 183 /* QQQ: What about stray counter name? */ 184 mtx_assert(&isrc_table_lock, MA_OWNED); 185 intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 186 } 187 188 /* 189 * Virtualization for interrupt source interrupt counter increment. 190 */ 191 static inline void 192 isrc_increment_count(struct intr_irqsrc *isrc) 193 { 194 195 if (isrc->isrc_flags & INTR_ISRCF_PPI) 196 atomic_add_long(&isrc->isrc_count[0], 1); 197 else 198 isrc->isrc_count[0]++; 199 } 200 201 /* 202 * Virtualization for interrupt source interrupt stray counter increment. 203 */ 204 static inline void 205 isrc_increment_straycount(struct intr_irqsrc *isrc) 206 { 207 208 isrc->isrc_count[1]++; 209 } 210 211 /* 212 * Virtualization for interrupt source interrupt name update. 213 */ 214 static void 215 isrc_update_name(struct intr_irqsrc *isrc, const char *name) 216 { 217 char str[INTRNAME_LEN]; 218 219 mtx_assert(&isrc_table_lock, MA_OWNED); 220 221 if (name != NULL) { 222 snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 223 intrcnt_setname(str, isrc->isrc_index); 224 snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 225 name); 226 intrcnt_setname(str, isrc->isrc_index + 1); 227 } else { 228 snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 229 intrcnt_setname(str, isrc->isrc_index); 230 snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 231 intrcnt_setname(str, isrc->isrc_index + 1); 232 } 233 } 234 235 /* 236 * Virtualization for interrupt source interrupt counters setup. 237 */ 238 static void 239 isrc_setup_counters(struct intr_irqsrc *isrc) 240 { 241 u_int index; 242 243 /* 244 * XXX - it does not work well with removable controllers and 245 * interrupt sources !!! 246 */ 247 index = atomic_fetchadd_int(&intrcnt_index, 2); 248 isrc->isrc_index = index; 249 isrc->isrc_count = &intrcnt[index]; 250 isrc_update_name(isrc, NULL); 251 } 252 253 /* 254 * Virtualization for interrupt source interrupt counters release. 255 */ 256 static void 257 isrc_release_counters(struct intr_irqsrc *isrc) 258 { 259 260 panic("%s: not implemented", __func__); 261 } 262 263 #ifdef SMP 264 /* 265 * Virtualization for interrupt source IPI counters setup. 266 */ 267 u_long * 268 intr_ipi_setup_counters(const char *name) 269 { 270 u_int index, i; 271 char str[INTRNAME_LEN]; 272 273 index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); 274 for (i = 0; i < MAXCPU; i++) { 275 snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 276 intrcnt_setname(str, index + i); 277 } 278 return (&intrcnt[index]); 279 } 280 #endif 281 282 /* 283 * Main interrupt dispatch handler. It's called straight 284 * from the assembler, where CPU interrupt is served. 285 */ 286 void 287 intr_irq_handler(struct trapframe *tf) 288 { 289 struct trapframe * oldframe; 290 struct thread * td; 291 292 KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); 293 294 PCPU_INC(cnt.v_intr); 295 critical_enter(); 296 td = curthread; 297 oldframe = td->td_intr_frame; 298 td->td_intr_frame = tf; 299 irq_root_filter(irq_root_arg); 300 td->td_intr_frame = oldframe; 301 critical_exit(); 302 #ifdef HWPMC_HOOKS 303 if (pmc_hook && TRAPF_USERMODE(tf) && 304 (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) 305 pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); 306 #endif 307 } 308 309 int 310 intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) 311 { 312 struct intr_pic_child *child; 313 bool found; 314 315 found = false; 316 mtx_lock_spin(&parent->pic_child_lock); 317 SLIST_FOREACH(child, &parent->pic_children, pc_next) { 318 if (child->pc_start <= irq && 319 irq < (child->pc_start + child->pc_length)) { 320 found = true; 321 break; 322 } 323 } 324 mtx_unlock_spin(&parent->pic_child_lock); 325 326 if (found) 327 return (child->pc_filter(child->pc_filter_arg, irq)); 328 329 return (FILTER_STRAY); 330 } 331 332 /* 333 * interrupt controller dispatch function for interrupts. It should 334 * be called straight from the interrupt controller, when associated interrupt 335 * source is learned. 336 */ 337 int 338 intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 339 { 340 341 KASSERT(isrc != NULL, ("%s: no source", __func__)); 342 343 isrc_increment_count(isrc); 344 345 #ifdef INTR_SOLO 346 if (isrc->isrc_filter != NULL) { 347 int error; 348 error = isrc->isrc_filter(isrc->isrc_arg, tf); 349 PIC_POST_FILTER(isrc->isrc_dev, isrc); 350 if (error == FILTER_HANDLED) 351 return (0); 352 } else 353 #endif 354 if (isrc->isrc_event != NULL) { 355 if (intr_event_handle(isrc->isrc_event, tf) == 0) 356 return (0); 357 } 358 359 isrc_increment_straycount(isrc); 360 return (EINVAL); 361 } 362 363 /* 364 * Alloc unique interrupt number (resource handle) for interrupt source. 365 * 366 * There could be various strategies how to allocate free interrupt number 367 * (resource handle) for new interrupt source. 368 * 369 * 1. Handles are always allocated forward, so handles are not recycled 370 * immediately. However, if only one free handle left which is reused 371 * constantly... 372 */ 373 static inline int 374 isrc_alloc_irq(struct intr_irqsrc *isrc) 375 { 376 u_int maxirqs, irq; 377 378 mtx_assert(&isrc_table_lock, MA_OWNED); 379 380 maxirqs = nitems(irq_sources); 381 if (irq_next_free >= maxirqs) 382 return (ENOSPC); 383 384 for (irq = irq_next_free; irq < maxirqs; irq++) { 385 if (irq_sources[irq] == NULL) 386 goto found; 387 } 388 for (irq = 0; irq < irq_next_free; irq++) { 389 if (irq_sources[irq] == NULL) 390 goto found; 391 } 392 393 irq_next_free = maxirqs; 394 return (ENOSPC); 395 396 found: 397 isrc->isrc_irq = irq; 398 irq_sources[irq] = isrc; 399 400 irq_next_free = irq + 1; 401 if (irq_next_free >= maxirqs) 402 irq_next_free = 0; 403 return (0); 404 } 405 406 /* 407 * Free unique interrupt number (resource handle) from interrupt source. 408 */ 409 static inline int 410 isrc_free_irq(struct intr_irqsrc *isrc) 411 { 412 413 mtx_assert(&isrc_table_lock, MA_OWNED); 414 415 if (isrc->isrc_irq >= nitems(irq_sources)) 416 return (EINVAL); 417 if (irq_sources[isrc->isrc_irq] != isrc) 418 return (EINVAL); 419 420 irq_sources[isrc->isrc_irq] = NULL; 421 isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 422 return (0); 423 } 424 425 /* 426 * Initialize interrupt source and register it into global interrupt table. 427 */ 428 int 429 intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, 430 const char *fmt, ...) 431 { 432 int error; 433 va_list ap; 434 435 bzero(isrc, sizeof(struct intr_irqsrc)); 436 isrc->isrc_dev = dev; 437 isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 438 isrc->isrc_flags = flags; 439 440 va_start(ap, fmt); 441 vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 442 va_end(ap); 443 444 mtx_lock(&isrc_table_lock); 445 error = isrc_alloc_irq(isrc); 446 if (error != 0) { 447 mtx_unlock(&isrc_table_lock); 448 return (error); 449 } 450 /* 451 * Setup interrupt counters, but not for IPI sources. Those are setup 452 * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust 453 * our counter pool. 454 */ 455 if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 456 isrc_setup_counters(isrc); 457 mtx_unlock(&isrc_table_lock); 458 return (0); 459 } 460 461 /* 462 * Deregister interrupt source from global interrupt table. 463 */ 464 int 465 intr_isrc_deregister(struct intr_irqsrc *isrc) 466 { 467 int error; 468 469 mtx_lock(&isrc_table_lock); 470 if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 471 isrc_release_counters(isrc); 472 error = isrc_free_irq(isrc); 473 mtx_unlock(&isrc_table_lock); 474 return (error); 475 } 476 477 #ifdef SMP 478 /* 479 * A support function for a PIC to decide if provided ISRC should be inited 480 * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of 481 * struct intr_irqsrc is the following: 482 * 483 * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus 484 * set in isrc_cpu. If not, the ISRC should be inited on every cpu and 485 * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. 486 */ 487 bool 488 intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) 489 { 490 491 if (isrc->isrc_handlers == 0) 492 return (false); 493 if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) 494 return (false); 495 if (isrc->isrc_flags & INTR_ISRCF_BOUND) 496 return (CPU_ISSET(cpu, &isrc->isrc_cpu)); 497 498 CPU_SET(cpu, &isrc->isrc_cpu); 499 return (true); 500 } 501 #endif 502 503 #ifdef INTR_SOLO 504 /* 505 * Setup filter into interrupt source. 506 */ 507 static int 508 iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 509 intr_irq_filter_t *filter, void *arg, void **cookiep) 510 { 511 512 if (filter == NULL) 513 return (EINVAL); 514 515 mtx_lock(&isrc_table_lock); 516 /* 517 * Make sure that we do not mix the two ways 518 * how we handle interrupt sources. 519 */ 520 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 521 mtx_unlock(&isrc_table_lock); 522 return (EBUSY); 523 } 524 isrc->isrc_filter = filter; 525 isrc->isrc_arg = arg; 526 isrc_update_name(isrc, name); 527 mtx_unlock(&isrc_table_lock); 528 529 *cookiep = isrc; 530 return (0); 531 } 532 #endif 533 534 /* 535 * Interrupt source pre_ithread method for MI interrupt framework. 536 */ 537 static void 538 intr_isrc_pre_ithread(void *arg) 539 { 540 struct intr_irqsrc *isrc = arg; 541 542 PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 543 } 544 545 /* 546 * Interrupt source post_ithread method for MI interrupt framework. 547 */ 548 static void 549 intr_isrc_post_ithread(void *arg) 550 { 551 struct intr_irqsrc *isrc = arg; 552 553 PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 554 } 555 556 /* 557 * Interrupt source post_filter method for MI interrupt framework. 558 */ 559 static void 560 intr_isrc_post_filter(void *arg) 561 { 562 struct intr_irqsrc *isrc = arg; 563 564 PIC_POST_FILTER(isrc->isrc_dev, isrc); 565 } 566 567 /* 568 * Interrupt source assign_cpu method for MI interrupt framework. 569 */ 570 static int 571 intr_isrc_assign_cpu(void *arg, int cpu) 572 { 573 #ifdef SMP 574 struct intr_irqsrc *isrc = arg; 575 int error; 576 577 if (isrc->isrc_dev != intr_irq_root_dev) 578 return (EINVAL); 579 580 mtx_lock(&isrc_table_lock); 581 if (cpu == NOCPU) { 582 CPU_ZERO(&isrc->isrc_cpu); 583 isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 584 } else { 585 CPU_SETOF(cpu, &isrc->isrc_cpu); 586 isrc->isrc_flags |= INTR_ISRCF_BOUND; 587 } 588 589 /* 590 * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 591 * re-balance it to another CPU or enable it on more CPUs. However, 592 * PIC is expected to change isrc_cpu appropriately to keep us well 593 * informed if the call is successful. 594 */ 595 if (irq_assign_cpu) { 596 error = PIC_BIND_INTR(isrc->isrc_dev, isrc); 597 if (error) { 598 CPU_ZERO(&isrc->isrc_cpu); 599 mtx_unlock(&isrc_table_lock); 600 return (error); 601 } 602 } 603 mtx_unlock(&isrc_table_lock); 604 return (0); 605 #else 606 return (EOPNOTSUPP); 607 #endif 608 } 609 610 /* 611 * Create interrupt event for interrupt source. 612 */ 613 static int 614 isrc_event_create(struct intr_irqsrc *isrc) 615 { 616 struct intr_event *ie; 617 int error; 618 619 error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 620 intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 621 intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 622 if (error) 623 return (error); 624 625 mtx_lock(&isrc_table_lock); 626 /* 627 * Make sure that we do not mix the two ways 628 * how we handle interrupt sources. Let contested event wins. 629 */ 630 #ifdef INTR_SOLO 631 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 632 #else 633 if (isrc->isrc_event != NULL) { 634 #endif 635 mtx_unlock(&isrc_table_lock); 636 intr_event_destroy(ie); 637 return (isrc->isrc_event != NULL ? EBUSY : 0); 638 } 639 isrc->isrc_event = ie; 640 mtx_unlock(&isrc_table_lock); 641 642 return (0); 643 } 644 #ifdef notyet 645 /* 646 * Destroy interrupt event for interrupt source. 647 */ 648 static void 649 isrc_event_destroy(struct intr_irqsrc *isrc) 650 { 651 struct intr_event *ie; 652 653 mtx_lock(&isrc_table_lock); 654 ie = isrc->isrc_event; 655 isrc->isrc_event = NULL; 656 mtx_unlock(&isrc_table_lock); 657 658 if (ie != NULL) 659 intr_event_destroy(ie); 660 } 661 #endif 662 /* 663 * Add handler to interrupt source. 664 */ 665 static int 666 isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 667 driver_filter_t filter, driver_intr_t handler, void *arg, 668 enum intr_type flags, void **cookiep) 669 { 670 int error; 671 672 if (isrc->isrc_event == NULL) { 673 error = isrc_event_create(isrc); 674 if (error) 675 return (error); 676 } 677 678 error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 679 arg, intr_priority(flags), flags, cookiep); 680 if (error == 0) { 681 mtx_lock(&isrc_table_lock); 682 intrcnt_updatename(isrc); 683 mtx_unlock(&isrc_table_lock); 684 } 685 686 return (error); 687 } 688 689 /* 690 * Lookup interrupt controller locked. 691 */ 692 static inline struct intr_pic * 693 pic_lookup_locked(device_t dev, intptr_t xref, int flags) 694 { 695 struct intr_pic *pic; 696 697 mtx_assert(&pic_list_lock, MA_OWNED); 698 699 if (dev == NULL && xref == 0) 700 return (NULL); 701 702 /* Note that pic->pic_dev is never NULL on registered PIC. */ 703 SLIST_FOREACH(pic, &pic_list, pic_next) { 704 if ((pic->pic_flags & FLAG_TYPE_MASK) != 705 (flags & FLAG_TYPE_MASK)) 706 continue; 707 708 if (dev == NULL) { 709 if (xref == pic->pic_xref) 710 return (pic); 711 } else if (xref == 0 || pic->pic_xref == 0) { 712 if (dev == pic->pic_dev) 713 return (pic); 714 } else if (xref == pic->pic_xref && dev == pic->pic_dev) 715 return (pic); 716 } 717 return (NULL); 718 } 719 720 /* 721 * Lookup interrupt controller. 722 */ 723 static struct intr_pic * 724 pic_lookup(device_t dev, intptr_t xref, int flags) 725 { 726 struct intr_pic *pic; 727 728 mtx_lock(&pic_list_lock); 729 pic = pic_lookup_locked(dev, xref, flags); 730 mtx_unlock(&pic_list_lock); 731 return (pic); 732 } 733 734 /* 735 * Create interrupt controller. 736 */ 737 static struct intr_pic * 738 pic_create(device_t dev, intptr_t xref, int flags) 739 { 740 struct intr_pic *pic; 741 742 mtx_lock(&pic_list_lock); 743 pic = pic_lookup_locked(dev, xref, flags); 744 if (pic != NULL) { 745 mtx_unlock(&pic_list_lock); 746 return (pic); 747 } 748 pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 749 if (pic == NULL) { 750 mtx_unlock(&pic_list_lock); 751 return (NULL); 752 } 753 pic->pic_xref = xref; 754 pic->pic_dev = dev; 755 pic->pic_flags = flags; 756 mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); 757 SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 758 mtx_unlock(&pic_list_lock); 759 760 return (pic); 761 } 762 #ifdef notyet 763 /* 764 * Destroy interrupt controller. 765 */ 766 static void 767 pic_destroy(device_t dev, intptr_t xref, int flags) 768 { 769 struct intr_pic *pic; 770 771 mtx_lock(&pic_list_lock); 772 pic = pic_lookup_locked(dev, xref, flags); 773 if (pic == NULL) { 774 mtx_unlock(&pic_list_lock); 775 return; 776 } 777 SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 778 mtx_unlock(&pic_list_lock); 779 780 free(pic, M_INTRNG); 781 } 782 #endif 783 /* 784 * Register interrupt controller. 785 */ 786 struct intr_pic * 787 intr_pic_register(device_t dev, intptr_t xref) 788 { 789 struct intr_pic *pic; 790 791 if (dev == NULL) 792 return (NULL); 793 pic = pic_create(dev, xref, FLAG_PIC); 794 if (pic == NULL) 795 return (NULL); 796 797 debugf("PIC %p registered for %s <dev %p, xref %x>\n", pic, 798 device_get_nameunit(dev), dev, xref); 799 return (pic); 800 } 801 802 /* 803 * Unregister interrupt controller. 804 */ 805 int 806 intr_pic_deregister(device_t dev, intptr_t xref) 807 { 808 809 panic("%s: not implemented", __func__); 810 } 811 812 /* 813 * Mark interrupt controller (itself) as a root one. 814 * 815 * Note that only an interrupt controller can really know its position 816 * in interrupt controller's tree. So root PIC must claim itself as a root. 817 * 818 * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 819 * page 30: 820 * "The root of the interrupt tree is determined when traversal 821 * of the interrupt tree reaches an interrupt controller node without 822 * an interrupts property and thus no explicit interrupt parent." 823 */ 824 int 825 intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 826 void *arg, u_int ipicount) 827 { 828 struct intr_pic *pic; 829 830 pic = pic_lookup(dev, xref, FLAG_PIC); 831 if (pic == NULL) { 832 device_printf(dev, "not registered\n"); 833 return (EINVAL); 834 } 835 836 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 837 ("%s: Found a non-PIC controller: %s", __func__, 838 device_get_name(pic->pic_dev))); 839 840 if (filter == NULL) { 841 device_printf(dev, "filter missing\n"); 842 return (EINVAL); 843 } 844 845 /* 846 * Only one interrupt controllers could be on the root for now. 847 * Note that we further suppose that there is not threaded interrupt 848 * routine (handler) on the root. See intr_irq_handler(). 849 */ 850 if (intr_irq_root_dev != NULL) { 851 device_printf(dev, "another root already set\n"); 852 return (EBUSY); 853 } 854 855 intr_irq_root_dev = dev; 856 irq_root_filter = filter; 857 irq_root_arg = arg; 858 irq_root_ipicount = ipicount; 859 860 debugf("irq root set to %s\n", device_get_nameunit(dev)); 861 return (0); 862 } 863 864 /* 865 * Add a handler to manage a sub range of a parents interrupts. 866 */ 867 struct intr_pic * 868 intr_pic_add_handler(device_t parent, struct intr_pic *pic, 869 intr_child_irq_filter_t *filter, void *arg, uintptr_t start, 870 uintptr_t length) 871 { 872 struct intr_pic *parent_pic; 873 struct intr_pic_child *newchild; 874 #ifdef INVARIANTS 875 struct intr_pic_child *child; 876 #endif 877 878 /* Find the parent PIC */ 879 parent_pic = pic_lookup(parent, 0, FLAG_PIC); 880 if (parent_pic == NULL) 881 return (NULL); 882 883 newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); 884 newchild->pc_pic = pic; 885 newchild->pc_filter = filter; 886 newchild->pc_filter_arg = arg; 887 newchild->pc_start = start; 888 newchild->pc_length = length; 889 890 mtx_lock_spin(&parent_pic->pic_child_lock); 891 #ifdef INVARIANTS 892 SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { 893 KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", 894 __func__)); 895 } 896 #endif 897 SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); 898 mtx_unlock_spin(&parent_pic->pic_child_lock); 899 900 return (pic); 901 } 902 903 static int 904 intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, 905 struct intr_irqsrc **isrc) 906 { 907 struct intr_pic *pic; 908 struct intr_map_data_msi *msi; 909 910 if (data == NULL) 911 return (EINVAL); 912 913 pic = pic_lookup(dev, xref, 914 (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); 915 if (pic == NULL) 916 return (ESRCH); 917 918 switch (data->type) { 919 case INTR_MAP_DATA_MSI: 920 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 921 ("%s: Found a non-MSI controller: %s", __func__, 922 device_get_name(pic->pic_dev))); 923 msi = (struct intr_map_data_msi *)data; 924 *isrc = msi->isrc; 925 return (0); 926 927 default: 928 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 929 ("%s: Found a non-PIC controller: %s", __func__, 930 device_get_name(pic->pic_dev))); 931 return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); 932 933 } 934 } 935 936 int 937 intr_activate_irq(device_t dev, struct resource *res) 938 { 939 device_t map_dev; 940 intptr_t map_xref; 941 struct intr_map_data *data; 942 struct intr_irqsrc *isrc; 943 u_int res_id; 944 int error; 945 946 KASSERT(rman_get_start(res) == rman_get_end(res), 947 ("%s: more interrupts in resource", __func__)); 948 949 res_id = (u_int)rman_get_start(res); 950 if (intr_map_get_isrc(res_id) != NULL) 951 panic("Attempt to double activation of resource id: %u\n", 952 res_id); 953 intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); 954 error = intr_resolve_irq(map_dev, map_xref, data, &isrc); 955 if (error != 0) { 956 free(data, M_INTRNG); 957 /* XXX TODO DISCONECTED PICs */ 958 /* if (error == EINVAL) return(0); */ 959 return (error); 960 } 961 intr_map_set_isrc(res_id, isrc); 962 rman_set_virtual(res, data); 963 return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); 964 } 965 966 int 967 intr_deactivate_irq(device_t dev, struct resource *res) 968 { 969 struct intr_map_data *data; 970 struct intr_irqsrc *isrc; 971 u_int res_id; 972 int error; 973 974 KASSERT(rman_get_start(res) == rman_get_end(res), 975 ("%s: more interrupts in resource", __func__)); 976 977 res_id = (u_int)rman_get_start(res); 978 isrc = intr_map_get_isrc(res_id); 979 if (isrc == NULL) 980 panic("Attempt to deactivate non-active resource id: %u\n", 981 res_id); 982 983 data = rman_get_virtual(res); 984 error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); 985 intr_map_set_isrc(res_id, NULL); 986 rman_set_virtual(res, NULL); 987 free(data, M_INTRNG); 988 return (error); 989 } 990 991 int 992 intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, 993 driver_intr_t hand, void *arg, int flags, void **cookiep) 994 { 995 int error; 996 struct intr_map_data *data; 997 struct intr_irqsrc *isrc; 998 const char *name; 999 u_int res_id; 1000 1001 KASSERT(rman_get_start(res) == rman_get_end(res), 1002 ("%s: more interrupts in resource", __func__)); 1003 1004 res_id = (u_int)rman_get_start(res); 1005 isrc = intr_map_get_isrc(res_id); 1006 if (isrc == NULL) { 1007 /* XXX TODO DISCONECTED PICs */ 1008 return (EINVAL); 1009 } 1010 1011 data = rman_get_virtual(res); 1012 name = device_get_nameunit(dev); 1013 1014 #ifdef INTR_SOLO 1015 /* 1016 * Standard handling is done through MI interrupt framework. However, 1017 * some interrupts could request solely own special handling. This 1018 * non standard handling can be used for interrupt controllers without 1019 * handler (filter only), so in case that interrupt controllers are 1020 * chained, MI interrupt framework is called only in leaf controller. 1021 * 1022 * Note that root interrupt controller routine is served as well, 1023 * however in intr_irq_handler(), i.e. main system dispatch routine. 1024 */ 1025 if (flags & INTR_SOLO && hand != NULL) { 1026 debugf("irq %u cannot solo on %s\n", irq, name); 1027 return (EINVAL); 1028 } 1029 1030 if (flags & INTR_SOLO) { 1031 error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 1032 arg, cookiep); 1033 debugf("irq %u setup filter error %d on %s\n", irq, error, 1034 name); 1035 } else 1036 #endif 1037 { 1038 error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 1039 cookiep); 1040 debugf("irq %u add handler error %d on %s\n", irq, error, name); 1041 } 1042 if (error != 0) 1043 return (error); 1044 1045 mtx_lock(&isrc_table_lock); 1046 error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); 1047 if (error == 0) { 1048 isrc->isrc_handlers++; 1049 if (isrc->isrc_handlers == 1) 1050 PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 1051 } 1052 mtx_unlock(&isrc_table_lock); 1053 if (error != 0) 1054 intr_event_remove_handler(*cookiep); 1055 return (error); 1056 } 1057 1058 int 1059 intr_teardown_irq(device_t dev, struct resource *res, void *cookie) 1060 { 1061 int error; 1062 struct intr_map_data *data; 1063 struct intr_irqsrc *isrc; 1064 u_int res_id; 1065 1066 KASSERT(rman_get_start(res) == rman_get_end(res), 1067 ("%s: more interrupts in resource", __func__)); 1068 1069 res_id = (u_int)rman_get_start(res); 1070 isrc = intr_map_get_isrc(res_id); 1071 if (isrc == NULL || isrc->isrc_handlers == 0) 1072 return (EINVAL); 1073 1074 data = rman_get_virtual(res); 1075 1076 #ifdef INTR_SOLO 1077 if (isrc->isrc_filter != NULL) { 1078 if (isrc != cookie) 1079 return (EINVAL); 1080 1081 mtx_lock(&isrc_table_lock); 1082 isrc->isrc_filter = NULL; 1083 isrc->isrc_arg = NULL; 1084 isrc->isrc_handlers = 0; 1085 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1086 PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 1087 isrc_update_name(isrc, NULL); 1088 mtx_unlock(&isrc_table_lock); 1089 return (0); 1090 } 1091 #endif 1092 if (isrc != intr_handler_source(cookie)) 1093 return (EINVAL); 1094 1095 error = intr_event_remove_handler(cookie); 1096 if (error == 0) { 1097 mtx_lock(&isrc_table_lock); 1098 isrc->isrc_handlers--; 1099 if (isrc->isrc_handlers == 0) 1100 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1101 PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 1102 intrcnt_updatename(isrc); 1103 mtx_unlock(&isrc_table_lock); 1104 } 1105 return (error); 1106 } 1107 1108 int 1109 intr_describe_irq(device_t dev, struct resource *res, void *cookie, 1110 const char *descr) 1111 { 1112 int error; 1113 struct intr_irqsrc *isrc; 1114 u_int res_id; 1115 1116 KASSERT(rman_get_start(res) == rman_get_end(res), 1117 ("%s: more interrupts in resource", __func__)); 1118 1119 res_id = (u_int)rman_get_start(res); 1120 isrc = intr_map_get_isrc(res_id); 1121 if (isrc == NULL || isrc->isrc_handlers == 0) 1122 return (EINVAL); 1123 #ifdef INTR_SOLO 1124 if (isrc->isrc_filter != NULL) { 1125 if (isrc != cookie) 1126 return (EINVAL); 1127 1128 mtx_lock(&isrc_table_lock); 1129 isrc_update_name(isrc, descr); 1130 mtx_unlock(&isrc_table_lock); 1131 return (0); 1132 } 1133 #endif 1134 error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 1135 if (error == 0) { 1136 mtx_lock(&isrc_table_lock); 1137 intrcnt_updatename(isrc); 1138 mtx_unlock(&isrc_table_lock); 1139 } 1140 return (error); 1141 } 1142 1143 #ifdef SMP 1144 int 1145 intr_bind_irq(device_t dev, struct resource *res, int cpu) 1146 { 1147 struct intr_irqsrc *isrc; 1148 u_int res_id; 1149 1150 KASSERT(rman_get_start(res) == rman_get_end(res), 1151 ("%s: more interrupts in resource", __func__)); 1152 1153 res_id = (u_int)rman_get_start(res); 1154 isrc = intr_map_get_isrc(res_id); 1155 if (isrc == NULL || isrc->isrc_handlers == 0) 1156 return (EINVAL); 1157 #ifdef INTR_SOLO 1158 if (isrc->isrc_filter != NULL) 1159 return (intr_isrc_assign_cpu(isrc, cpu)); 1160 #endif 1161 return (intr_event_bind(isrc->isrc_event, cpu)); 1162 } 1163 1164 /* 1165 * Return the CPU that the next interrupt source should use. 1166 * For now just returns the next CPU according to round-robin. 1167 */ 1168 u_int 1169 intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 1170 { 1171 1172 if (!irq_assign_cpu || mp_ncpus == 1) 1173 return (PCPU_GET(cpuid)); 1174 1175 do { 1176 last_cpu++; 1177 if (last_cpu > mp_maxid) 1178 last_cpu = 0; 1179 } while (!CPU_ISSET(last_cpu, cpumask)); 1180 return (last_cpu); 1181 } 1182 1183 /* 1184 * Distribute all the interrupt sources among the available 1185 * CPUs once the AP's have been launched. 1186 */ 1187 static void 1188 intr_irq_shuffle(void *arg __unused) 1189 { 1190 struct intr_irqsrc *isrc; 1191 u_int i; 1192 1193 if (mp_ncpus == 1) 1194 return; 1195 1196 mtx_lock(&isrc_table_lock); 1197 irq_assign_cpu = TRUE; 1198 for (i = 0; i < NIRQ; i++) { 1199 isrc = irq_sources[i]; 1200 if (isrc == NULL || isrc->isrc_handlers == 0 || 1201 isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) 1202 continue; 1203 1204 if (isrc->isrc_event != NULL && 1205 isrc->isrc_flags & INTR_ISRCF_BOUND && 1206 isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 1207 panic("%s: CPU inconsistency", __func__); 1208 1209 if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 1210 CPU_ZERO(&isrc->isrc_cpu); /* start again */ 1211 1212 /* 1213 * We are in wicked position here if the following call fails 1214 * for bound ISRC. The best thing we can do is to clear 1215 * isrc_cpu so inconsistency with ie_cpu will be detectable. 1216 */ 1217 if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) 1218 CPU_ZERO(&isrc->isrc_cpu); 1219 } 1220 mtx_unlock(&isrc_table_lock); 1221 } 1222 SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1223 1224 #else 1225 u_int 1226 intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 1227 { 1228 1229 return (PCPU_GET(cpuid)); 1230 } 1231 #endif 1232 1233 /* 1234 * Allocate memory for new intr_map_data structure. 1235 * Initialize common fields. 1236 */ 1237 struct intr_map_data * 1238 intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) 1239 { 1240 struct intr_map_data *data; 1241 1242 data = malloc(len, M_INTRNG, flags); 1243 data->type = type; 1244 data->len = len; 1245 return (data); 1246 } 1247 1248 void intr_free_intr_map_data(struct intr_map_data *data) 1249 { 1250 1251 free(data, M_INTRNG); 1252 } 1253 1254 1255 /* 1256 * Register a MSI/MSI-X interrupt controller 1257 */ 1258 int 1259 intr_msi_register(device_t dev, intptr_t xref) 1260 { 1261 struct intr_pic *pic; 1262 1263 if (dev == NULL) 1264 return (EINVAL); 1265 pic = pic_create(dev, xref, FLAG_MSI); 1266 if (pic == NULL) 1267 return (ENOMEM); 1268 1269 debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 1270 device_get_nameunit(dev), dev, (uintmax_t)xref); 1271 return (0); 1272 } 1273 1274 int 1275 intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, 1276 int maxcount, int *irqs) 1277 { 1278 struct intr_irqsrc **isrc; 1279 struct intr_pic *pic; 1280 device_t pdev; 1281 struct intr_map_data_msi *msi; 1282 int err, i; 1283 1284 pic = pic_lookup(NULL, xref, FLAG_MSI); 1285 if (pic == NULL) 1286 return (ESRCH); 1287 1288 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1289 ("%s: Found a non-MSI controller: %s", __func__, 1290 device_get_name(pic->pic_dev))); 1291 1292 isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 1293 err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); 1294 if (err != 0) { 1295 free(isrc, M_INTRNG); 1296 return (err); 1297 } 1298 1299 for (i = 0; i < count; i++) { 1300 msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1301 INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1302 msi-> isrc = isrc[i]; 1303 irqs[i] = intr_map_irq(pic->pic_dev, xref, 1304 (struct intr_map_data *)msi); 1305 1306 } 1307 free(isrc, M_INTRNG); 1308 1309 return (err); 1310 } 1311 1312 int 1313 intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, 1314 int *irqs) 1315 { 1316 struct intr_irqsrc **isrc; 1317 struct intr_pic *pic; 1318 struct intr_map_data_msi *msi; 1319 int i, err; 1320 1321 pic = pic_lookup(NULL, xref, FLAG_MSI); 1322 if (pic == NULL) 1323 return (ESRCH); 1324 1325 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1326 ("%s: Found a non-MSI controller: %s", __func__, 1327 device_get_name(pic->pic_dev))); 1328 1329 isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 1330 1331 for (i = 0; i < count; i++) { 1332 msi = (struct intr_map_data_msi *) 1333 intr_map_get_map_data(irqs[i]); 1334 KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1335 ("%s: irq %d map data is not MSI", __func__, 1336 irqs[i])); 1337 isrc[i] = msi->isrc; 1338 } 1339 1340 err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); 1341 1342 for (i = 0; i < count; i++) { 1343 if (isrc[i] != NULL) 1344 intr_unmap_irq(irqs[i]); 1345 } 1346 1347 free(isrc, M_INTRNG); 1348 return (err); 1349 } 1350 1351 int 1352 intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) 1353 { 1354 struct intr_irqsrc *isrc; 1355 struct intr_pic *pic; 1356 device_t pdev; 1357 struct intr_map_data_msi *msi; 1358 int err; 1359 1360 pic = pic_lookup(NULL, xref, FLAG_MSI); 1361 if (pic == NULL) 1362 return (ESRCH); 1363 1364 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1365 ("%s: Found a non-MSI controller: %s", __func__, 1366 device_get_name(pic->pic_dev))); 1367 1368 1369 err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); 1370 if (err != 0) 1371 return (err); 1372 1373 msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1374 INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1375 msi->isrc = isrc; 1376 *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); 1377 return (0); 1378 } 1379 1380 int 1381 intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) 1382 { 1383 struct intr_irqsrc *isrc; 1384 struct intr_pic *pic; 1385 struct intr_map_data_msi *msi; 1386 int err; 1387 1388 pic = pic_lookup(NULL, xref, FLAG_MSI); 1389 if (pic == NULL) 1390 return (ESRCH); 1391 1392 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1393 ("%s: Found a non-MSI controller: %s", __func__, 1394 device_get_name(pic->pic_dev))); 1395 1396 msi = (struct intr_map_data_msi *) 1397 intr_map_get_map_data(irq); 1398 KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1399 ("%s: irq %d map data is not MSI", __func__, 1400 irq)); 1401 isrc = msi->isrc; 1402 if (isrc == NULL) { 1403 intr_unmap_irq(irq); 1404 return (EINVAL); 1405 } 1406 1407 err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); 1408 intr_unmap_irq(irq); 1409 1410 return (err); 1411 } 1412 1413 int 1414 intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, 1415 uint64_t *addr, uint32_t *data) 1416 { 1417 struct intr_irqsrc *isrc; 1418 struct intr_pic *pic; 1419 int err; 1420 1421 pic = pic_lookup(NULL, xref, FLAG_MSI); 1422 if (pic == NULL) 1423 return (ESRCH); 1424 1425 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1426 ("%s: Found a non-MSI controller: %s", __func__, 1427 device_get_name(pic->pic_dev))); 1428 1429 isrc = intr_map_get_isrc(irq); 1430 if (isrc == NULL) 1431 return (EINVAL); 1432 1433 err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); 1434 return (err); 1435 } 1436 1437 1438 void dosoftints(void); 1439 void 1440 dosoftints(void) 1441 { 1442 } 1443 1444 #ifdef SMP 1445 /* 1446 * Init interrupt controller on another CPU. 1447 */ 1448 void 1449 intr_pic_init_secondary(void) 1450 { 1451 1452 /* 1453 * QQQ: Only root PIC is aware of other CPUs ??? 1454 */ 1455 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 1456 1457 //mtx_lock(&isrc_table_lock); 1458 PIC_INIT_SECONDARY(intr_irq_root_dev); 1459 //mtx_unlock(&isrc_table_lock); 1460 } 1461 #endif 1462 1463 #ifdef DDB 1464 DB_SHOW_COMMAND(irqs, db_show_irqs) 1465 { 1466 u_int i, irqsum; 1467 u_long num; 1468 struct intr_irqsrc *isrc; 1469 1470 for (irqsum = 0, i = 0; i < NIRQ; i++) { 1471 isrc = irq_sources[i]; 1472 if (isrc == NULL) 1473 continue; 1474 1475 num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; 1476 db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 1477 isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1478 isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); 1479 irqsum += num; 1480 } 1481 db_printf("irq total %u\n", irqsum); 1482 } 1483 #endif 1484 1485 /* 1486 * Interrupt mapping table functions. 1487 * 1488 * Please, keep this part separately, it can be transformed to 1489 * extension of standard resources. 1490 */ 1491 struct intr_map_entry 1492 { 1493 device_t dev; 1494 intptr_t xref; 1495 struct intr_map_data *map_data; 1496 struct intr_irqsrc *isrc; 1497 /* XXX TODO DISCONECTED PICs */ 1498 /*int flags */ 1499 }; 1500 1501 /* XXX Convert irq_map[] to dynamicaly expandable one. */ 1502 static struct intr_map_entry *irq_map[2 * NIRQ]; 1503 static int irq_map_count = nitems(irq_map); 1504 static int irq_map_first_free_idx; 1505 static struct mtx irq_map_lock; 1506 1507 static struct intr_irqsrc * 1508 intr_map_get_isrc(u_int res_id) 1509 { 1510 struct intr_irqsrc *isrc; 1511 1512 mtx_lock(&irq_map_lock); 1513 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) { 1514 mtx_unlock(&irq_map_lock); 1515 return (NULL); 1516 } 1517 isrc = irq_map[res_id]->isrc; 1518 mtx_unlock(&irq_map_lock); 1519 return (isrc); 1520 } 1521 1522 static void 1523 intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) 1524 { 1525 1526 mtx_lock(&irq_map_lock); 1527 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) { 1528 mtx_unlock(&irq_map_lock); 1529 return; 1530 } 1531 irq_map[res_id]->isrc = isrc; 1532 mtx_unlock(&irq_map_lock); 1533 } 1534 1535 /* 1536 * Get a copy of intr_map_entry data 1537 */ 1538 static struct intr_map_data * 1539 intr_map_get_map_data(u_int res_id) 1540 { 1541 struct intr_map_data *data; 1542 1543 data = NULL; 1544 mtx_lock(&irq_map_lock); 1545 if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1546 panic("Attempt to copy invalid resource id: %u\n", res_id); 1547 data = irq_map[res_id]->map_data; 1548 mtx_unlock(&irq_map_lock); 1549 1550 return (data); 1551 } 1552 1553 /* 1554 * Get a copy of intr_map_entry data 1555 */ 1556 static void 1557 intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, 1558 struct intr_map_data **data) 1559 { 1560 size_t len; 1561 1562 len = 0; 1563 mtx_lock(&irq_map_lock); 1564 if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1565 panic("Attempt to copy invalid resource id: %u\n", res_id); 1566 if (irq_map[res_id]->map_data != NULL) 1567 len = irq_map[res_id]->map_data->len; 1568 mtx_unlock(&irq_map_lock); 1569 1570 if (len == 0) 1571 *data = NULL; 1572 else 1573 *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); 1574 mtx_lock(&irq_map_lock); 1575 if (irq_map[res_id] == NULL) 1576 panic("Attempt to copy invalid resource id: %u\n", res_id); 1577 if (len != 0) { 1578 if (len != irq_map[res_id]->map_data->len) 1579 panic("Resource id: %u has changed.\n", res_id); 1580 memcpy(*data, irq_map[res_id]->map_data, len); 1581 } 1582 *map_dev = irq_map[res_id]->dev; 1583 *map_xref = irq_map[res_id]->xref; 1584 mtx_unlock(&irq_map_lock); 1585 } 1586 1587 1588 /* 1589 * Allocate and fill new entry in irq_map table. 1590 */ 1591 u_int 1592 intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) 1593 { 1594 u_int i; 1595 struct intr_map_entry *entry; 1596 1597 /* Prepare new entry first. */ 1598 entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); 1599 1600 entry->dev = dev; 1601 entry->xref = xref; 1602 entry->map_data = data; 1603 entry->isrc = NULL; 1604 1605 mtx_lock(&irq_map_lock); 1606 for (i = irq_map_first_free_idx; i < irq_map_count; i++) { 1607 if (irq_map[i] == NULL) { 1608 irq_map[i] = entry; 1609 irq_map_first_free_idx = i + 1; 1610 mtx_unlock(&irq_map_lock); 1611 return (i); 1612 } 1613 } 1614 mtx_unlock(&irq_map_lock); 1615 1616 /* XXX Expand irq_map table */ 1617 panic("IRQ mapping table is full."); 1618 } 1619 1620 /* 1621 * Remove and free mapping entry. 1622 */ 1623 void 1624 intr_unmap_irq(u_int res_id) 1625 { 1626 struct intr_map_entry *entry; 1627 1628 mtx_lock(&irq_map_lock); 1629 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) 1630 panic("Attempt to unmap invalid resource id: %u\n", res_id); 1631 entry = irq_map[res_id]; 1632 irq_map[res_id] = NULL; 1633 irq_map_first_free_idx = res_id; 1634 mtx_unlock(&irq_map_lock); 1635 intr_free_intr_map_data(entry->map_data); 1636 free(entry, M_INTRNG); 1637 } 1638 1639 /* 1640 * Clone mapping entry. 1641 */ 1642 u_int 1643 intr_map_clone_irq(u_int old_res_id) 1644 { 1645 device_t map_dev; 1646 intptr_t map_xref; 1647 struct intr_map_data *data; 1648 1649 intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); 1650 return (intr_map_irq(map_dev, map_xref, data)); 1651 } 1652 1653 static void 1654 intr_map_init(void *dummy __unused) 1655 { 1656 1657 mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); 1658 } 1659 SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); 1660