1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include "opt_ddb.h" 33 #include "opt_kstack_usage_prof.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/conf.h> 38 #include <sys/cpuset.h> 39 #include <sys/rtprio.h> 40 #include <sys/systm.h> 41 #include <sys/interrupt.h> 42 #include <sys/kernel.h> 43 #include <sys/kthread.h> 44 #include <sys/ktr.h> 45 #include <sys/limits.h> 46 #include <sys/lock.h> 47 #include <sys/malloc.h> 48 #include <sys/mutex.h> 49 #include <sys/priv.h> 50 #include <sys/proc.h> 51 #include <sys/random.h> 52 #include <sys/resourcevar.h> 53 #include <sys/sched.h> 54 #include <sys/smp.h> 55 #include <sys/sysctl.h> 56 #include <sys/syslog.h> 57 #include <sys/unistd.h> 58 #include <sys/vmmeter.h> 59 #include <machine/atomic.h> 60 #include <machine/cpu.h> 61 #include <machine/md_var.h> 62 #include <machine/stdarg.h> 63 #ifdef DDB 64 #include <ddb/ddb.h> 65 #include <ddb/db_sym.h> 66 #endif 67 68 /* 69 * Describe an interrupt thread. There is one of these per interrupt event. 70 */ 71 struct intr_thread { 72 struct intr_event *it_event; 73 struct thread *it_thread; /* Kernel thread. */ 74 int it_flags; /* (j) IT_* flags. */ 75 int it_need; /* Needs service. */ 76 }; 77 78 /* Interrupt thread flags kept in it_flags */ 79 #define IT_DEAD 0x000001 /* Thread is waiting to exit. */ 80 #define IT_WAIT 0x000002 /* Thread is waiting for completion. */ 81 82 struct intr_entropy { 83 struct thread *td; 84 uintptr_t event; 85 }; 86 87 struct intr_event *clk_intr_event; 88 struct intr_event *tty_intr_event; 89 void *vm_ih; 90 struct proc *intrproc; 91 92 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 93 94 static int intr_storm_threshold = 1000; 95 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN, 96 &intr_storm_threshold, 0, 97 "Number of consecutive interrupts before storm protection is enabled"); 98 static TAILQ_HEAD(, intr_event) event_list = 99 TAILQ_HEAD_INITIALIZER(event_list); 100 static struct mtx event_lock; 101 MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); 102 103 static void intr_event_update(struct intr_event *ie); 104 #ifdef INTR_FILTER 105 static int intr_event_schedule_thread(struct intr_event *ie, 106 struct intr_thread *ithd); 107 static int intr_filter_loop(struct intr_event *ie, 108 struct trapframe *frame, struct intr_thread **ithd); 109 static struct intr_thread *ithread_create(const char *name, 110 struct intr_handler *ih); 111 #else 112 static int intr_event_schedule_thread(struct intr_event *ie); 113 static struct intr_thread *ithread_create(const char *name); 114 #endif 115 static void ithread_destroy(struct intr_thread *ithread); 116 static void ithread_execute_handlers(struct proc *p, 117 struct intr_event *ie); 118 #ifdef INTR_FILTER 119 static void priv_ithread_execute_handler(struct proc *p, 120 struct intr_handler *ih); 121 #endif 122 static void ithread_loop(void *); 123 static void ithread_update(struct intr_thread *ithd); 124 static void start_softintr(void *); 125 126 /* Map an interrupt type to an ithread priority. */ 127 u_char 128 intr_priority(enum intr_type flags) 129 { 130 u_char pri; 131 132 flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 133 INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 134 switch (flags) { 135 case INTR_TYPE_TTY: 136 pri = PI_TTY; 137 break; 138 case INTR_TYPE_BIO: 139 pri = PI_DISK; 140 break; 141 case INTR_TYPE_NET: 142 pri = PI_NET; 143 break; 144 case INTR_TYPE_CAM: 145 pri = PI_DISK; 146 break; 147 case INTR_TYPE_AV: 148 pri = PI_AV; 149 break; 150 case INTR_TYPE_CLK: 151 pri = PI_REALTIME; 152 break; 153 case INTR_TYPE_MISC: 154 pri = PI_DULL; /* don't care */ 155 break; 156 default: 157 /* We didn't specify an interrupt level. */ 158 panic("intr_priority: no interrupt type in flags"); 159 } 160 161 return pri; 162 } 163 164 /* 165 * Update an ithread based on the associated intr_event. 166 */ 167 static void 168 ithread_update(struct intr_thread *ithd) 169 { 170 struct intr_event *ie; 171 struct thread *td; 172 u_char pri; 173 174 ie = ithd->it_event; 175 td = ithd->it_thread; 176 177 /* Determine the overall priority of this event. */ 178 if (TAILQ_EMPTY(&ie->ie_handlers)) 179 pri = PRI_MAX_ITHD; 180 else 181 pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; 182 183 /* Update name and priority. */ 184 strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name)); 185 #ifdef KTR 186 sched_clear_tdname(td); 187 #endif 188 thread_lock(td); 189 sched_prio(td, pri); 190 thread_unlock(td); 191 } 192 193 /* 194 * Regenerate the full name of an interrupt event and update its priority. 195 */ 196 static void 197 intr_event_update(struct intr_event *ie) 198 { 199 struct intr_handler *ih; 200 char *last; 201 int missed, space; 202 203 /* Start off with no entropy and just the name of the event. */ 204 mtx_assert(&ie->ie_lock, MA_OWNED); 205 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 206 ie->ie_flags &= ~IE_ENTROPY; 207 missed = 0; 208 space = 1; 209 210 /* Run through all the handlers updating values. */ 211 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 212 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < 213 sizeof(ie->ie_fullname)) { 214 strcat(ie->ie_fullname, " "); 215 strcat(ie->ie_fullname, ih->ih_name); 216 space = 0; 217 } else 218 missed++; 219 if (ih->ih_flags & IH_ENTROPY) 220 ie->ie_flags |= IE_ENTROPY; 221 } 222 223 /* 224 * If the handler names were too long, add +'s to indicate missing 225 * names. If we run out of room and still have +'s to add, change 226 * the last character from a + to a *. 227 */ 228 last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; 229 while (missed-- > 0) { 230 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { 231 if (*last == '+') { 232 *last = '*'; 233 break; 234 } else 235 *last = '+'; 236 } else if (space) { 237 strcat(ie->ie_fullname, " +"); 238 space = 0; 239 } else 240 strcat(ie->ie_fullname, "+"); 241 } 242 243 /* 244 * If this event has an ithread, update it's priority and 245 * name. 246 */ 247 if (ie->ie_thread != NULL) 248 ithread_update(ie->ie_thread); 249 CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); 250 } 251 252 int 253 intr_event_create(struct intr_event **event, void *source, int flags, int irq, 254 void (*pre_ithread)(void *), void (*post_ithread)(void *), 255 void (*post_filter)(void *), int (*assign_cpu)(void *, int), 256 const char *fmt, ...) 257 { 258 struct intr_event *ie; 259 va_list ap; 260 261 /* The only valid flag during creation is IE_SOFT. */ 262 if ((flags & ~IE_SOFT) != 0) 263 return (EINVAL); 264 ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 265 ie->ie_source = source; 266 ie->ie_pre_ithread = pre_ithread; 267 ie->ie_post_ithread = post_ithread; 268 ie->ie_post_filter = post_filter; 269 ie->ie_assign_cpu = assign_cpu; 270 ie->ie_flags = flags; 271 ie->ie_irq = irq; 272 ie->ie_cpu = NOCPU; 273 TAILQ_INIT(&ie->ie_handlers); 274 mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 275 276 va_start(ap, fmt); 277 vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 278 va_end(ap); 279 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 280 mtx_lock(&event_lock); 281 TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 282 mtx_unlock(&event_lock); 283 if (event != NULL) 284 *event = ie; 285 CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 286 return (0); 287 } 288 289 /* 290 * Bind an interrupt event to the specified CPU. Note that not all 291 * platforms support binding an interrupt to a CPU. For those 292 * platforms this request will fail. Using a cpu id of NOCPU unbinds 293 * the interrupt event. 294 */ 295 static int 296 _intr_event_bind(struct intr_event *ie, int cpu, bool bindirq, bool bindithread) 297 { 298 lwpid_t id; 299 int error; 300 301 /* Need a CPU to bind to. */ 302 if (cpu != NOCPU && CPU_ABSENT(cpu)) 303 return (EINVAL); 304 305 if (ie->ie_assign_cpu == NULL) 306 return (EOPNOTSUPP); 307 308 error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR); 309 if (error) 310 return (error); 311 312 /* 313 * If we have any ithreads try to set their mask first to verify 314 * permissions, etc. 315 */ 316 if (bindithread) { 317 mtx_lock(&ie->ie_lock); 318 if (ie->ie_thread != NULL) { 319 id = ie->ie_thread->it_thread->td_tid; 320 mtx_unlock(&ie->ie_lock); 321 error = cpuset_setithread(id, cpu); 322 if (error) 323 return (error); 324 } else 325 mtx_unlock(&ie->ie_lock); 326 } 327 if (bindirq) 328 error = ie->ie_assign_cpu(ie->ie_source, cpu); 329 if (error) { 330 if (bindithread) { 331 mtx_lock(&ie->ie_lock); 332 if (ie->ie_thread != NULL) { 333 cpu = ie->ie_cpu; 334 id = ie->ie_thread->it_thread->td_tid; 335 mtx_unlock(&ie->ie_lock); 336 (void)cpuset_setithread(id, cpu); 337 } else 338 mtx_unlock(&ie->ie_lock); 339 } 340 return (error); 341 } 342 343 if (bindirq) { 344 mtx_lock(&ie->ie_lock); 345 ie->ie_cpu = cpu; 346 mtx_unlock(&ie->ie_lock); 347 } 348 349 return (error); 350 } 351 352 /* 353 * Bind an interrupt event to the specified CPU. For supported platforms, any 354 * associated ithreads as well as the primary interrupt context will be bound 355 * to the specificed CPU. 356 */ 357 int 358 intr_event_bind(struct intr_event *ie, int cpu) 359 { 360 361 return (_intr_event_bind(ie, cpu, true, true)); 362 } 363 364 /* 365 * Bind an interrupt event to the specified CPU, but do not bind associated 366 * ithreads. 367 */ 368 int 369 intr_event_bind_irqonly(struct intr_event *ie, int cpu) 370 { 371 372 return (_intr_event_bind(ie, cpu, true, false)); 373 } 374 375 /* 376 * Bind an interrupt event's ithread to the specified CPU. 377 */ 378 int 379 intr_event_bind_ithread(struct intr_event *ie, int cpu) 380 { 381 382 return (_intr_event_bind(ie, cpu, false, true)); 383 } 384 385 static struct intr_event * 386 intr_lookup(int irq) 387 { 388 struct intr_event *ie; 389 390 mtx_lock(&event_lock); 391 TAILQ_FOREACH(ie, &event_list, ie_list) 392 if (ie->ie_irq == irq && 393 (ie->ie_flags & IE_SOFT) == 0 && 394 TAILQ_FIRST(&ie->ie_handlers) != NULL) 395 break; 396 mtx_unlock(&event_lock); 397 return (ie); 398 } 399 400 int 401 intr_setaffinity(int irq, int mode, void *m) 402 { 403 struct intr_event *ie; 404 cpuset_t *mask; 405 int cpu, n; 406 407 mask = m; 408 cpu = NOCPU; 409 /* 410 * If we're setting all cpus we can unbind. Otherwise make sure 411 * only one cpu is in the set. 412 */ 413 if (CPU_CMP(cpuset_root, mask)) { 414 for (n = 0; n < CPU_SETSIZE; n++) { 415 if (!CPU_ISSET(n, mask)) 416 continue; 417 if (cpu != NOCPU) 418 return (EINVAL); 419 cpu = n; 420 } 421 } 422 ie = intr_lookup(irq); 423 if (ie == NULL) 424 return (ESRCH); 425 switch (mode) { 426 case CPU_WHICH_IRQ: 427 return (intr_event_bind(ie, cpu)); 428 case CPU_WHICH_INTRHANDLER: 429 return (intr_event_bind_irqonly(ie, cpu)); 430 case CPU_WHICH_ITHREAD: 431 return (intr_event_bind_ithread(ie, cpu)); 432 default: 433 return (EINVAL); 434 } 435 } 436 437 int 438 intr_getaffinity(int irq, int mode, void *m) 439 { 440 struct intr_event *ie; 441 struct thread *td; 442 struct proc *p; 443 cpuset_t *mask; 444 lwpid_t id; 445 int error; 446 447 mask = m; 448 ie = intr_lookup(irq); 449 if (ie == NULL) 450 return (ESRCH); 451 452 error = 0; 453 CPU_ZERO(mask); 454 switch (mode) { 455 case CPU_WHICH_IRQ: 456 case CPU_WHICH_INTRHANDLER: 457 mtx_lock(&ie->ie_lock); 458 if (ie->ie_cpu == NOCPU) 459 CPU_COPY(cpuset_root, mask); 460 else 461 CPU_SET(ie->ie_cpu, mask); 462 mtx_unlock(&ie->ie_lock); 463 break; 464 case CPU_WHICH_ITHREAD: 465 mtx_lock(&ie->ie_lock); 466 if (ie->ie_thread == NULL) { 467 mtx_unlock(&ie->ie_lock); 468 CPU_COPY(cpuset_root, mask); 469 } else { 470 id = ie->ie_thread->it_thread->td_tid; 471 mtx_unlock(&ie->ie_lock); 472 error = cpuset_which(CPU_WHICH_TID, id, &p, &td, NULL); 473 if (error != 0) 474 return (error); 475 CPU_COPY(&td->td_cpuset->cs_mask, mask); 476 PROC_UNLOCK(p); 477 } 478 default: 479 return (EINVAL); 480 } 481 return (0); 482 } 483 484 int 485 intr_event_destroy(struct intr_event *ie) 486 { 487 488 mtx_lock(&event_lock); 489 mtx_lock(&ie->ie_lock); 490 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 491 mtx_unlock(&ie->ie_lock); 492 mtx_unlock(&event_lock); 493 return (EBUSY); 494 } 495 TAILQ_REMOVE(&event_list, ie, ie_list); 496 #ifndef notyet 497 if (ie->ie_thread != NULL) { 498 ithread_destroy(ie->ie_thread); 499 ie->ie_thread = NULL; 500 } 501 #endif 502 mtx_unlock(&ie->ie_lock); 503 mtx_unlock(&event_lock); 504 mtx_destroy(&ie->ie_lock); 505 free(ie, M_ITHREAD); 506 return (0); 507 } 508 509 #ifndef INTR_FILTER 510 static struct intr_thread * 511 ithread_create(const char *name) 512 { 513 struct intr_thread *ithd; 514 struct thread *td; 515 int error; 516 517 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 518 519 error = kproc_kthread_add(ithread_loop, ithd, &intrproc, 520 &td, RFSTOPPED | RFHIGHPID, 521 0, "intr", "%s", name); 522 if (error) 523 panic("kproc_create() failed with %d", error); 524 thread_lock(td); 525 sched_class(td, PRI_ITHD); 526 TD_SET_IWAIT(td); 527 thread_unlock(td); 528 td->td_pflags |= TDP_ITHREAD; 529 ithd->it_thread = td; 530 CTR2(KTR_INTR, "%s: created %s", __func__, name); 531 return (ithd); 532 } 533 #else 534 static struct intr_thread * 535 ithread_create(const char *name, struct intr_handler *ih) 536 { 537 struct intr_thread *ithd; 538 struct thread *td; 539 int error; 540 541 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 542 543 error = kproc_kthread_add(ithread_loop, ih, &intrproc, 544 &td, RFSTOPPED | RFHIGHPID, 545 0, "intr", "%s", name); 546 if (error) 547 panic("kproc_create() failed with %d", error); 548 thread_lock(td); 549 sched_class(td, PRI_ITHD); 550 TD_SET_IWAIT(td); 551 thread_unlock(td); 552 td->td_pflags |= TDP_ITHREAD; 553 ithd->it_thread = td; 554 CTR2(KTR_INTR, "%s: created %s", __func__, name); 555 return (ithd); 556 } 557 #endif 558 559 static void 560 ithread_destroy(struct intr_thread *ithread) 561 { 562 struct thread *td; 563 564 CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name); 565 td = ithread->it_thread; 566 thread_lock(td); 567 ithread->it_flags |= IT_DEAD; 568 if (TD_AWAITING_INTR(td)) { 569 TD_CLR_IWAIT(td); 570 sched_add(td, SRQ_INTR); 571 } 572 thread_unlock(td); 573 } 574 575 #ifndef INTR_FILTER 576 int 577 intr_event_add_handler(struct intr_event *ie, const char *name, 578 driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 579 enum intr_type flags, void **cookiep) 580 { 581 struct intr_handler *ih, *temp_ih; 582 struct intr_thread *it; 583 584 if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 585 return (EINVAL); 586 587 /* Allocate and populate an interrupt handler structure. */ 588 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 589 ih->ih_filter = filter; 590 ih->ih_handler = handler; 591 ih->ih_argument = arg; 592 strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 593 ih->ih_event = ie; 594 ih->ih_pri = pri; 595 if (flags & INTR_EXCL) 596 ih->ih_flags = IH_EXCLUSIVE; 597 if (flags & INTR_MPSAFE) 598 ih->ih_flags |= IH_MPSAFE; 599 if (flags & INTR_ENTROPY) 600 ih->ih_flags |= IH_ENTROPY; 601 602 /* We can only have one exclusive handler in a event. */ 603 mtx_lock(&ie->ie_lock); 604 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 605 if ((flags & INTR_EXCL) || 606 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 607 mtx_unlock(&ie->ie_lock); 608 free(ih, M_ITHREAD); 609 return (EINVAL); 610 } 611 } 612 613 /* Create a thread if we need one. */ 614 while (ie->ie_thread == NULL && handler != NULL) { 615 if (ie->ie_flags & IE_ADDING_THREAD) 616 msleep(ie, &ie->ie_lock, 0, "ithread", 0); 617 else { 618 ie->ie_flags |= IE_ADDING_THREAD; 619 mtx_unlock(&ie->ie_lock); 620 it = ithread_create("intr: newborn"); 621 mtx_lock(&ie->ie_lock); 622 ie->ie_flags &= ~IE_ADDING_THREAD; 623 ie->ie_thread = it; 624 it->it_event = ie; 625 ithread_update(it); 626 wakeup(ie); 627 } 628 } 629 630 /* Add the new handler to the event in priority order. */ 631 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 632 if (temp_ih->ih_pri > ih->ih_pri) 633 break; 634 } 635 if (temp_ih == NULL) 636 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 637 else 638 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 639 intr_event_update(ie); 640 641 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 642 ie->ie_name); 643 mtx_unlock(&ie->ie_lock); 644 645 if (cookiep != NULL) 646 *cookiep = ih; 647 return (0); 648 } 649 #else 650 int 651 intr_event_add_handler(struct intr_event *ie, const char *name, 652 driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 653 enum intr_type flags, void **cookiep) 654 { 655 struct intr_handler *ih, *temp_ih; 656 struct intr_thread *it; 657 658 if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 659 return (EINVAL); 660 661 /* Allocate and populate an interrupt handler structure. */ 662 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 663 ih->ih_filter = filter; 664 ih->ih_handler = handler; 665 ih->ih_argument = arg; 666 strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 667 ih->ih_event = ie; 668 ih->ih_pri = pri; 669 if (flags & INTR_EXCL) 670 ih->ih_flags = IH_EXCLUSIVE; 671 if (flags & INTR_MPSAFE) 672 ih->ih_flags |= IH_MPSAFE; 673 if (flags & INTR_ENTROPY) 674 ih->ih_flags |= IH_ENTROPY; 675 676 /* We can only have one exclusive handler in a event. */ 677 mtx_lock(&ie->ie_lock); 678 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 679 if ((flags & INTR_EXCL) || 680 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 681 mtx_unlock(&ie->ie_lock); 682 free(ih, M_ITHREAD); 683 return (EINVAL); 684 } 685 } 686 687 /* For filtered handlers, create a private ithread to run on. */ 688 if (filter != NULL && handler != NULL) { 689 mtx_unlock(&ie->ie_lock); 690 it = ithread_create("intr: newborn", ih); 691 mtx_lock(&ie->ie_lock); 692 it->it_event = ie; 693 ih->ih_thread = it; 694 ithread_update(it); /* XXX - do we really need this?!?!? */ 695 } else { /* Create the global per-event thread if we need one. */ 696 while (ie->ie_thread == NULL && handler != NULL) { 697 if (ie->ie_flags & IE_ADDING_THREAD) 698 msleep(ie, &ie->ie_lock, 0, "ithread", 0); 699 else { 700 ie->ie_flags |= IE_ADDING_THREAD; 701 mtx_unlock(&ie->ie_lock); 702 it = ithread_create("intr: newborn", ih); 703 mtx_lock(&ie->ie_lock); 704 ie->ie_flags &= ~IE_ADDING_THREAD; 705 ie->ie_thread = it; 706 it->it_event = ie; 707 ithread_update(it); 708 wakeup(ie); 709 } 710 } 711 } 712 713 /* Add the new handler to the event in priority order. */ 714 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 715 if (temp_ih->ih_pri > ih->ih_pri) 716 break; 717 } 718 if (temp_ih == NULL) 719 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 720 else 721 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 722 intr_event_update(ie); 723 724 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 725 ie->ie_name); 726 mtx_unlock(&ie->ie_lock); 727 728 if (cookiep != NULL) 729 *cookiep = ih; 730 return (0); 731 } 732 #endif 733 734 /* 735 * Append a description preceded by a ':' to the name of the specified 736 * interrupt handler. 737 */ 738 int 739 intr_event_describe_handler(struct intr_event *ie, void *cookie, 740 const char *descr) 741 { 742 struct intr_handler *ih; 743 size_t space; 744 char *start; 745 746 mtx_lock(&ie->ie_lock); 747 #ifdef INVARIANTS 748 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 749 if (ih == cookie) 750 break; 751 } 752 if (ih == NULL) { 753 mtx_unlock(&ie->ie_lock); 754 panic("handler %p not found in interrupt event %p", cookie, ie); 755 } 756 #endif 757 ih = cookie; 758 759 /* 760 * Look for an existing description by checking for an 761 * existing ":". This assumes device names do not include 762 * colons. If one is found, prepare to insert the new 763 * description at that point. If one is not found, find the 764 * end of the name to use as the insertion point. 765 */ 766 start = strchr(ih->ih_name, ':'); 767 if (start == NULL) 768 start = strchr(ih->ih_name, 0); 769 770 /* 771 * See if there is enough remaining room in the string for the 772 * description + ":". The "- 1" leaves room for the trailing 773 * '\0'. The "+ 1" accounts for the colon. 774 */ 775 space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1; 776 if (strlen(descr) + 1 > space) { 777 mtx_unlock(&ie->ie_lock); 778 return (ENOSPC); 779 } 780 781 /* Append a colon followed by the description. */ 782 *start = ':'; 783 strcpy(start + 1, descr); 784 intr_event_update(ie); 785 mtx_unlock(&ie->ie_lock); 786 return (0); 787 } 788 789 /* 790 * Return the ie_source field from the intr_event an intr_handler is 791 * associated with. 792 */ 793 void * 794 intr_handler_source(void *cookie) 795 { 796 struct intr_handler *ih; 797 struct intr_event *ie; 798 799 ih = (struct intr_handler *)cookie; 800 if (ih == NULL) 801 return (NULL); 802 ie = ih->ih_event; 803 KASSERT(ie != NULL, 804 ("interrupt handler \"%s\" has a NULL interrupt event", 805 ih->ih_name)); 806 return (ie->ie_source); 807 } 808 809 /* 810 * Sleep until an ithread finishes executing an interrupt handler. 811 * 812 * XXX Doesn't currently handle interrupt filters or fast interrupt 813 * handlers. This is intended for compatibility with linux drivers 814 * only. Do not use in BSD code. 815 */ 816 void 817 _intr_drain(int irq) 818 { 819 struct intr_event *ie; 820 struct intr_thread *ithd; 821 struct thread *td; 822 823 ie = intr_lookup(irq); 824 if (ie == NULL) 825 return; 826 if (ie->ie_thread == NULL) 827 return; 828 ithd = ie->ie_thread; 829 td = ithd->it_thread; 830 /* 831 * We set the flag and wait for it to be cleared to avoid 832 * long delays with potentially busy interrupt handlers 833 * were we to only sample TD_AWAITING_INTR() every tick. 834 */ 835 thread_lock(td); 836 if (!TD_AWAITING_INTR(td)) { 837 ithd->it_flags |= IT_WAIT; 838 while (ithd->it_flags & IT_WAIT) { 839 thread_unlock(td); 840 pause("idrain", 1); 841 thread_lock(td); 842 } 843 } 844 thread_unlock(td); 845 return; 846 } 847 848 849 #ifndef INTR_FILTER 850 int 851 intr_event_remove_handler(void *cookie) 852 { 853 struct intr_handler *handler = (struct intr_handler *)cookie; 854 struct intr_event *ie; 855 #ifdef INVARIANTS 856 struct intr_handler *ih; 857 #endif 858 #ifdef notyet 859 int dead; 860 #endif 861 862 if (handler == NULL) 863 return (EINVAL); 864 ie = handler->ih_event; 865 KASSERT(ie != NULL, 866 ("interrupt handler \"%s\" has a NULL interrupt event", 867 handler->ih_name)); 868 mtx_lock(&ie->ie_lock); 869 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 870 ie->ie_name); 871 #ifdef INVARIANTS 872 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 873 if (ih == handler) 874 goto ok; 875 mtx_unlock(&ie->ie_lock); 876 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 877 ih->ih_name, ie->ie_name); 878 ok: 879 #endif 880 /* 881 * If there is no ithread, then just remove the handler and return. 882 * XXX: Note that an INTR_FAST handler might be running on another 883 * CPU! 884 */ 885 if (ie->ie_thread == NULL) { 886 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 887 mtx_unlock(&ie->ie_lock); 888 free(handler, M_ITHREAD); 889 return (0); 890 } 891 892 /* 893 * If the interrupt thread is already running, then just mark this 894 * handler as being dead and let the ithread do the actual removal. 895 * 896 * During a cold boot while cold is set, msleep() does not sleep, 897 * so we have to remove the handler here rather than letting the 898 * thread do it. 899 */ 900 thread_lock(ie->ie_thread->it_thread); 901 if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { 902 handler->ih_flags |= IH_DEAD; 903 904 /* 905 * Ensure that the thread will process the handler list 906 * again and remove this handler if it has already passed 907 * it on the list. 908 * 909 * The release part of the following store ensures 910 * that the update of ih_flags is ordered before the 911 * it_need setting. See the comment before 912 * atomic_cmpset_acq(&ithd->it_need, ...) operation in 913 * the ithread_execute_handlers(). 914 */ 915 atomic_store_rel_int(&ie->ie_thread->it_need, 1); 916 } else 917 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 918 thread_unlock(ie->ie_thread->it_thread); 919 while (handler->ih_flags & IH_DEAD) 920 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 921 intr_event_update(ie); 922 #ifdef notyet 923 /* 924 * XXX: This could be bad in the case of ppbus(8). Also, I think 925 * this could lead to races of stale data when servicing an 926 * interrupt. 927 */ 928 dead = 1; 929 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 930 if (!(ih->ih_flags & IH_FAST)) { 931 dead = 0; 932 break; 933 } 934 } 935 if (dead) { 936 ithread_destroy(ie->ie_thread); 937 ie->ie_thread = NULL; 938 } 939 #endif 940 mtx_unlock(&ie->ie_lock); 941 free(handler, M_ITHREAD); 942 return (0); 943 } 944 945 static int 946 intr_event_schedule_thread(struct intr_event *ie) 947 { 948 struct intr_entropy entropy; 949 struct intr_thread *it; 950 struct thread *td; 951 struct thread *ctd; 952 struct proc *p; 953 954 /* 955 * If no ithread or no handlers, then we have a stray interrupt. 956 */ 957 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || 958 ie->ie_thread == NULL) 959 return (EINVAL); 960 961 ctd = curthread; 962 it = ie->ie_thread; 963 td = it->it_thread; 964 p = td->td_proc; 965 966 /* 967 * If any of the handlers for this ithread claim to be good 968 * sources of entropy, then gather some. 969 */ 970 if (ie->ie_flags & IE_ENTROPY) { 971 entropy.event = (uintptr_t)ie; 972 entropy.td = ctd; 973 random_harvest_queue(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT); 974 } 975 976 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 977 978 /* 979 * Set it_need to tell the thread to keep running if it is already 980 * running. Then, lock the thread and see if we actually need to 981 * put it on the runqueue. 982 * 983 * Use store_rel to arrange that the store to ih_need in 984 * swi_sched() is before the store to it_need and prepare for 985 * transfer of this order to loads in the ithread. 986 */ 987 atomic_store_rel_int(&it->it_need, 1); 988 thread_lock(td); 989 if (TD_AWAITING_INTR(td)) { 990 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 991 td->td_name); 992 TD_CLR_IWAIT(td); 993 sched_add(td, SRQ_INTR); 994 } else { 995 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 996 __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 997 } 998 thread_unlock(td); 999 1000 return (0); 1001 } 1002 #else 1003 int 1004 intr_event_remove_handler(void *cookie) 1005 { 1006 struct intr_handler *handler = (struct intr_handler *)cookie; 1007 struct intr_event *ie; 1008 struct intr_thread *it; 1009 #ifdef INVARIANTS 1010 struct intr_handler *ih; 1011 #endif 1012 #ifdef notyet 1013 int dead; 1014 #endif 1015 1016 if (handler == NULL) 1017 return (EINVAL); 1018 ie = handler->ih_event; 1019 KASSERT(ie != NULL, 1020 ("interrupt handler \"%s\" has a NULL interrupt event", 1021 handler->ih_name)); 1022 mtx_lock(&ie->ie_lock); 1023 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 1024 ie->ie_name); 1025 #ifdef INVARIANTS 1026 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 1027 if (ih == handler) 1028 goto ok; 1029 mtx_unlock(&ie->ie_lock); 1030 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 1031 ih->ih_name, ie->ie_name); 1032 ok: 1033 #endif 1034 /* 1035 * If there are no ithreads (per event and per handler), then 1036 * just remove the handler and return. 1037 * XXX: Note that an INTR_FAST handler might be running on another CPU! 1038 */ 1039 if (ie->ie_thread == NULL && handler->ih_thread == NULL) { 1040 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 1041 mtx_unlock(&ie->ie_lock); 1042 free(handler, M_ITHREAD); 1043 return (0); 1044 } 1045 1046 /* Private or global ithread? */ 1047 it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; 1048 /* 1049 * If the interrupt thread is already running, then just mark this 1050 * handler as being dead and let the ithread do the actual removal. 1051 * 1052 * During a cold boot while cold is set, msleep() does not sleep, 1053 * so we have to remove the handler here rather than letting the 1054 * thread do it. 1055 */ 1056 thread_lock(it->it_thread); 1057 if (!TD_AWAITING_INTR(it->it_thread) && !cold) { 1058 handler->ih_flags |= IH_DEAD; 1059 1060 /* 1061 * Ensure that the thread will process the handler list 1062 * again and remove this handler if it has already passed 1063 * it on the list. 1064 * 1065 * The release part of the following store ensures 1066 * that the update of ih_flags is ordered before the 1067 * it_need setting. See the comment before 1068 * atomic_cmpset_acq(&ithd->it_need, ...) operation in 1069 * the ithread_execute_handlers(). 1070 */ 1071 atomic_store_rel_int(&it->it_need, 1); 1072 } else 1073 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 1074 thread_unlock(it->it_thread); 1075 while (handler->ih_flags & IH_DEAD) 1076 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 1077 /* 1078 * At this point, the handler has been disconnected from the event, 1079 * so we can kill the private ithread if any. 1080 */ 1081 if (handler->ih_thread) { 1082 ithread_destroy(handler->ih_thread); 1083 handler->ih_thread = NULL; 1084 } 1085 intr_event_update(ie); 1086 #ifdef notyet 1087 /* 1088 * XXX: This could be bad in the case of ppbus(8). Also, I think 1089 * this could lead to races of stale data when servicing an 1090 * interrupt. 1091 */ 1092 dead = 1; 1093 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1094 if (handler != NULL) { 1095 dead = 0; 1096 break; 1097 } 1098 } 1099 if (dead) { 1100 ithread_destroy(ie->ie_thread); 1101 ie->ie_thread = NULL; 1102 } 1103 #endif 1104 mtx_unlock(&ie->ie_lock); 1105 free(handler, M_ITHREAD); 1106 return (0); 1107 } 1108 1109 static int 1110 intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) 1111 { 1112 struct intr_entropy entropy; 1113 struct thread *td; 1114 struct thread *ctd; 1115 struct proc *p; 1116 1117 /* 1118 * If no ithread or no handlers, then we have a stray interrupt. 1119 */ 1120 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) 1121 return (EINVAL); 1122 1123 ctd = curthread; 1124 td = it->it_thread; 1125 p = td->td_proc; 1126 1127 /* 1128 * If any of the handlers for this ithread claim to be good 1129 * sources of entropy, then gather some. 1130 */ 1131 if (ie->ie_flags & IE_ENTROPY) { 1132 entropy.event = (uintptr_t)ie; 1133 entropy.td = ctd; 1134 random_harvest_queue(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT); 1135 } 1136 1137 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 1138 1139 /* 1140 * Set it_need to tell the thread to keep running if it is already 1141 * running. Then, lock the thread and see if we actually need to 1142 * put it on the runqueue. 1143 * 1144 * Use store_rel to arrange that the store to ih_need in 1145 * swi_sched() is before the store to it_need and prepare for 1146 * transfer of this order to loads in the ithread. 1147 */ 1148 atomic_store_rel_int(&it->it_need, 1); 1149 thread_lock(td); 1150 if (TD_AWAITING_INTR(td)) { 1151 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 1152 td->td_name); 1153 TD_CLR_IWAIT(td); 1154 sched_add(td, SRQ_INTR); 1155 } else { 1156 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 1157 __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 1158 } 1159 thread_unlock(td); 1160 1161 return (0); 1162 } 1163 #endif 1164 1165 /* 1166 * Allow interrupt event binding for software interrupt handlers -- a no-op, 1167 * since interrupts are generated in software rather than being directed by 1168 * a PIC. 1169 */ 1170 static int 1171 swi_assign_cpu(void *arg, int cpu) 1172 { 1173 1174 return (0); 1175 } 1176 1177 /* 1178 * Add a software interrupt handler to a specified event. If a given event 1179 * is not specified, then a new event is created. 1180 */ 1181 int 1182 swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, 1183 void *arg, int pri, enum intr_type flags, void **cookiep) 1184 { 1185 struct intr_event *ie; 1186 int error; 1187 1188 if (flags & INTR_ENTROPY) 1189 return (EINVAL); 1190 1191 ie = (eventp != NULL) ? *eventp : NULL; 1192 1193 if (ie != NULL) { 1194 if (!(ie->ie_flags & IE_SOFT)) 1195 return (EINVAL); 1196 } else { 1197 error = intr_event_create(&ie, NULL, IE_SOFT, 0, 1198 NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri); 1199 if (error) 1200 return (error); 1201 if (eventp != NULL) 1202 *eventp = ie; 1203 } 1204 error = intr_event_add_handler(ie, name, NULL, handler, arg, 1205 PI_SWI(pri), flags, cookiep); 1206 return (error); 1207 } 1208 1209 /* 1210 * Schedule a software interrupt thread. 1211 */ 1212 void 1213 swi_sched(void *cookie, int flags) 1214 { 1215 struct intr_handler *ih = (struct intr_handler *)cookie; 1216 struct intr_event *ie = ih->ih_event; 1217 struct intr_entropy entropy; 1218 int error; 1219 1220 CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, 1221 ih->ih_need); 1222 1223 entropy.event = (uintptr_t)ih; 1224 entropy.td = curthread; 1225 random_harvest_queue(&entropy, sizeof(entropy), 1, RANDOM_SWI); 1226 1227 /* 1228 * Set ih_need for this handler so that if the ithread is already 1229 * running it will execute this handler on the next pass. Otherwise, 1230 * it will execute it the next time it runs. 1231 */ 1232 ih->ih_need = 1; 1233 1234 if (!(flags & SWI_DELAY)) { 1235 VM_CNT_INC(v_soft); 1236 #ifdef INTR_FILTER 1237 error = intr_event_schedule_thread(ie, ie->ie_thread); 1238 #else 1239 error = intr_event_schedule_thread(ie); 1240 #endif 1241 KASSERT(error == 0, ("stray software interrupt")); 1242 } 1243 } 1244 1245 /* 1246 * Remove a software interrupt handler. Currently this code does not 1247 * remove the associated interrupt event if it becomes empty. Calling code 1248 * may do so manually via intr_event_destroy(), but that's not really 1249 * an optimal interface. 1250 */ 1251 int 1252 swi_remove(void *cookie) 1253 { 1254 1255 return (intr_event_remove_handler(cookie)); 1256 } 1257 1258 #ifdef INTR_FILTER 1259 static void 1260 priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) 1261 { 1262 struct intr_event *ie; 1263 1264 ie = ih->ih_event; 1265 /* 1266 * If this handler is marked for death, remove it from 1267 * the list of handlers and wake up the sleeper. 1268 */ 1269 if (ih->ih_flags & IH_DEAD) { 1270 mtx_lock(&ie->ie_lock); 1271 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1272 ih->ih_flags &= ~IH_DEAD; 1273 wakeup(ih); 1274 mtx_unlock(&ie->ie_lock); 1275 return; 1276 } 1277 1278 /* Execute this handler. */ 1279 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1280 __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, 1281 ih->ih_name, ih->ih_flags); 1282 1283 if (!(ih->ih_flags & IH_MPSAFE)) 1284 mtx_lock(&Giant); 1285 ih->ih_handler(ih->ih_argument); 1286 if (!(ih->ih_flags & IH_MPSAFE)) 1287 mtx_unlock(&Giant); 1288 } 1289 #endif 1290 1291 /* 1292 * This is a public function for use by drivers that mux interrupt 1293 * handlers for child devices from their interrupt handler. 1294 */ 1295 void 1296 intr_event_execute_handlers(struct proc *p, struct intr_event *ie) 1297 { 1298 struct intr_handler *ih, *ihn; 1299 1300 TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { 1301 /* 1302 * If this handler is marked for death, remove it from 1303 * the list of handlers and wake up the sleeper. 1304 */ 1305 if (ih->ih_flags & IH_DEAD) { 1306 mtx_lock(&ie->ie_lock); 1307 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1308 ih->ih_flags &= ~IH_DEAD; 1309 wakeup(ih); 1310 mtx_unlock(&ie->ie_lock); 1311 continue; 1312 } 1313 1314 /* Skip filter only handlers */ 1315 if (ih->ih_handler == NULL) 1316 continue; 1317 1318 /* 1319 * For software interrupt threads, we only execute 1320 * handlers that have their need flag set. Hardware 1321 * interrupt threads always invoke all of their handlers. 1322 * 1323 * ih_need can only be 0 or 1. Failed cmpset below 1324 * means that there is no request to execute handlers, 1325 * so a retry of the cmpset is not needed. 1326 */ 1327 if ((ie->ie_flags & IE_SOFT) != 0 && 1328 atomic_cmpset_int(&ih->ih_need, 1, 0) == 0) 1329 continue; 1330 1331 /* Execute this handler. */ 1332 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1333 __func__, p->p_pid, (void *)ih->ih_handler, 1334 ih->ih_argument, ih->ih_name, ih->ih_flags); 1335 1336 if (!(ih->ih_flags & IH_MPSAFE)) 1337 mtx_lock(&Giant); 1338 ih->ih_handler(ih->ih_argument); 1339 if (!(ih->ih_flags & IH_MPSAFE)) 1340 mtx_unlock(&Giant); 1341 } 1342 } 1343 1344 static void 1345 ithread_execute_handlers(struct proc *p, struct intr_event *ie) 1346 { 1347 1348 /* Interrupt handlers should not sleep. */ 1349 if (!(ie->ie_flags & IE_SOFT)) 1350 THREAD_NO_SLEEPING(); 1351 intr_event_execute_handlers(p, ie); 1352 if (!(ie->ie_flags & IE_SOFT)) 1353 THREAD_SLEEPING_OK(); 1354 1355 /* 1356 * Interrupt storm handling: 1357 * 1358 * If this interrupt source is currently storming, then throttle 1359 * it to only fire the handler once per clock tick. 1360 * 1361 * If this interrupt source is not currently storming, but the 1362 * number of back to back interrupts exceeds the storm threshold, 1363 * then enter storming mode. 1364 */ 1365 if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold && 1366 !(ie->ie_flags & IE_SOFT)) { 1367 /* Report the message only once every second. */ 1368 if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) { 1369 printf( 1370 "interrupt storm detected on \"%s\"; throttling interrupt source\n", 1371 ie->ie_name); 1372 } 1373 pause("istorm", 1); 1374 } else 1375 ie->ie_count++; 1376 1377 /* 1378 * Now that all the handlers have had a chance to run, reenable 1379 * the interrupt source. 1380 */ 1381 if (ie->ie_post_ithread != NULL) 1382 ie->ie_post_ithread(ie->ie_source); 1383 } 1384 1385 #ifndef INTR_FILTER 1386 /* 1387 * This is the main code for interrupt threads. 1388 */ 1389 static void 1390 ithread_loop(void *arg) 1391 { 1392 struct intr_thread *ithd; 1393 struct intr_event *ie; 1394 struct thread *td; 1395 struct proc *p; 1396 int wake; 1397 1398 td = curthread; 1399 p = td->td_proc; 1400 ithd = (struct intr_thread *)arg; 1401 KASSERT(ithd->it_thread == td, 1402 ("%s: ithread and proc linkage out of sync", __func__)); 1403 ie = ithd->it_event; 1404 ie->ie_count = 0; 1405 wake = 0; 1406 1407 /* 1408 * As long as we have interrupts outstanding, go through the 1409 * list of handlers, giving each one a go at it. 1410 */ 1411 for (;;) { 1412 /* 1413 * If we are an orphaned thread, then just die. 1414 */ 1415 if (ithd->it_flags & IT_DEAD) { 1416 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1417 p->p_pid, td->td_name); 1418 free(ithd, M_ITHREAD); 1419 kthread_exit(); 1420 } 1421 1422 /* 1423 * Service interrupts. If another interrupt arrives while 1424 * we are running, it will set it_need to note that we 1425 * should make another pass. 1426 * 1427 * The load_acq part of the following cmpset ensures 1428 * that the load of ih_need in ithread_execute_handlers() 1429 * is ordered after the load of it_need here. 1430 */ 1431 while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) 1432 ithread_execute_handlers(p, ie); 1433 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1434 mtx_assert(&Giant, MA_NOTOWNED); 1435 1436 /* 1437 * Processed all our interrupts. Now get the sched 1438 * lock. This may take a while and it_need may get 1439 * set again, so we have to check it again. 1440 */ 1441 thread_lock(td); 1442 if (atomic_load_acq_int(&ithd->it_need) == 0 && 1443 (ithd->it_flags & (IT_DEAD | IT_WAIT)) == 0) { 1444 TD_SET_IWAIT(td); 1445 ie->ie_count = 0; 1446 mi_switch(SW_VOL | SWT_IWAIT, NULL); 1447 } 1448 if (ithd->it_flags & IT_WAIT) { 1449 wake = 1; 1450 ithd->it_flags &= ~IT_WAIT; 1451 } 1452 thread_unlock(td); 1453 if (wake) { 1454 wakeup(ithd); 1455 wake = 0; 1456 } 1457 } 1458 } 1459 1460 /* 1461 * Main interrupt handling body. 1462 * 1463 * Input: 1464 * o ie: the event connected to this interrupt. 1465 * o frame: some archs (i.e. i386) pass a frame to some. 1466 * handlers as their main argument. 1467 * Return value: 1468 * o 0: everything ok. 1469 * o EINVAL: stray interrupt. 1470 */ 1471 int 1472 intr_event_handle(struct intr_event *ie, struct trapframe *frame) 1473 { 1474 struct intr_handler *ih; 1475 struct trapframe *oldframe; 1476 struct thread *td; 1477 int error, ret, thread; 1478 1479 td = curthread; 1480 1481 #ifdef KSTACK_USAGE_PROF 1482 intr_prof_stack_use(td, frame); 1483 #endif 1484 1485 /* An interrupt with no event or handlers is a stray interrupt. */ 1486 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1487 return (EINVAL); 1488 1489 /* 1490 * Execute fast interrupt handlers directly. 1491 * To support clock handlers, if a handler registers 1492 * with a NULL argument, then we pass it a pointer to 1493 * a trapframe as its argument. 1494 */ 1495 td->td_intr_nesting_level++; 1496 thread = 0; 1497 ret = 0; 1498 critical_enter(); 1499 oldframe = td->td_intr_frame; 1500 td->td_intr_frame = frame; 1501 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1502 if (ih->ih_filter == NULL) { 1503 thread = 1; 1504 continue; 1505 } 1506 CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, 1507 ih->ih_filter, ih->ih_argument == NULL ? frame : 1508 ih->ih_argument, ih->ih_name); 1509 if (ih->ih_argument == NULL) 1510 ret = ih->ih_filter(frame); 1511 else 1512 ret = ih->ih_filter(ih->ih_argument); 1513 KASSERT(ret == FILTER_STRAY || 1514 ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1515 (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1516 ("%s: incorrect return value %#x from %s", __func__, ret, 1517 ih->ih_name)); 1518 1519 /* 1520 * Wrapper handler special handling: 1521 * 1522 * in some particular cases (like pccard and pccbb), 1523 * the _real_ device handler is wrapped in a couple of 1524 * functions - a filter wrapper and an ithread wrapper. 1525 * In this case (and just in this case), the filter wrapper 1526 * could ask the system to schedule the ithread and mask 1527 * the interrupt source if the wrapped handler is composed 1528 * of just an ithread handler. 1529 * 1530 * TODO: write a generic wrapper to avoid people rolling 1531 * their own 1532 */ 1533 if (!thread) { 1534 if (ret == FILTER_SCHEDULE_THREAD) 1535 thread = 1; 1536 } 1537 } 1538 td->td_intr_frame = oldframe; 1539 1540 if (thread) { 1541 if (ie->ie_pre_ithread != NULL) 1542 ie->ie_pre_ithread(ie->ie_source); 1543 } else { 1544 if (ie->ie_post_filter != NULL) 1545 ie->ie_post_filter(ie->ie_source); 1546 } 1547 1548 /* Schedule the ithread if needed. */ 1549 if (thread) { 1550 error = intr_event_schedule_thread(ie); 1551 KASSERT(error == 0, ("bad stray interrupt")); 1552 } 1553 critical_exit(); 1554 td->td_intr_nesting_level--; 1555 return (0); 1556 } 1557 #else 1558 /* 1559 * This is the main code for interrupt threads. 1560 */ 1561 static void 1562 ithread_loop(void *arg) 1563 { 1564 struct intr_thread *ithd; 1565 struct intr_handler *ih; 1566 struct intr_event *ie; 1567 struct thread *td; 1568 struct proc *p; 1569 int priv; 1570 int wake; 1571 1572 td = curthread; 1573 p = td->td_proc; 1574 ih = (struct intr_handler *)arg; 1575 priv = (ih->ih_thread != NULL) ? 1 : 0; 1576 ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; 1577 KASSERT(ithd->it_thread == td, 1578 ("%s: ithread and proc linkage out of sync", __func__)); 1579 ie = ithd->it_event; 1580 ie->ie_count = 0; 1581 wake = 0; 1582 1583 /* 1584 * As long as we have interrupts outstanding, go through the 1585 * list of handlers, giving each one a go at it. 1586 */ 1587 for (;;) { 1588 /* 1589 * If we are an orphaned thread, then just die. 1590 */ 1591 if (ithd->it_flags & IT_DEAD) { 1592 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1593 p->p_pid, td->td_name); 1594 free(ithd, M_ITHREAD); 1595 kthread_exit(); 1596 } 1597 1598 /* 1599 * Service interrupts. If another interrupt arrives while 1600 * we are running, it will set it_need to note that we 1601 * should make another pass. 1602 * 1603 * The load_acq part of the following cmpset ensures 1604 * that the load of ih_need in ithread_execute_handlers() 1605 * is ordered after the load of it_need here. 1606 */ 1607 while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) { 1608 if (priv) 1609 priv_ithread_execute_handler(p, ih); 1610 else 1611 ithread_execute_handlers(p, ie); 1612 } 1613 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1614 mtx_assert(&Giant, MA_NOTOWNED); 1615 1616 /* 1617 * Processed all our interrupts. Now get the sched 1618 * lock. This may take a while and it_need may get 1619 * set again, so we have to check it again. 1620 */ 1621 thread_lock(td); 1622 if (atomic_load_acq_int(&ithd->it_need) == 0 && 1623 (ithd->it_flags & (IT_DEAD | IT_WAIT)) == 0) { 1624 TD_SET_IWAIT(td); 1625 ie->ie_count = 0; 1626 mi_switch(SW_VOL | SWT_IWAIT, NULL); 1627 } 1628 if (ithd->it_flags & IT_WAIT) { 1629 wake = 1; 1630 ithd->it_flags &= ~IT_WAIT; 1631 } 1632 thread_unlock(td); 1633 if (wake) { 1634 wakeup(ithd); 1635 wake = 0; 1636 } 1637 } 1638 } 1639 1640 /* 1641 * Main loop for interrupt filter. 1642 * 1643 * Some architectures (i386, amd64 and arm) require the optional frame 1644 * parameter, and use it as the main argument for fast handler execution 1645 * when ih_argument == NULL. 1646 * 1647 * Return value: 1648 * o FILTER_STRAY: No filter recognized the event, and no 1649 * filter-less handler is registered on this 1650 * line. 1651 * o FILTER_HANDLED: A filter claimed the event and served it. 1652 * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at 1653 * least one filter-less handler on this line. 1654 * o FILTER_HANDLED | 1655 * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for 1656 * scheduling the per-handler ithread. 1657 * 1658 * In case an ithread has to be scheduled, in *ithd there will be a 1659 * pointer to a struct intr_thread containing the thread to be 1660 * scheduled. 1661 */ 1662 1663 static int 1664 intr_filter_loop(struct intr_event *ie, struct trapframe *frame, 1665 struct intr_thread **ithd) 1666 { 1667 struct intr_handler *ih; 1668 void *arg; 1669 int ret, thread_only; 1670 1671 ret = 0; 1672 thread_only = 0; 1673 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1674 /* 1675 * Execute fast interrupt handlers directly. 1676 * To support clock handlers, if a handler registers 1677 * with a NULL argument, then we pass it a pointer to 1678 * a trapframe as its argument. 1679 */ 1680 arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); 1681 1682 CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, 1683 ih->ih_filter, ih->ih_handler, arg, ih->ih_name); 1684 1685 if (ih->ih_filter != NULL) 1686 ret = ih->ih_filter(arg); 1687 else { 1688 thread_only = 1; 1689 continue; 1690 } 1691 KASSERT(ret == FILTER_STRAY || 1692 ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1693 (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1694 ("%s: incorrect return value %#x from %s", __func__, ret, 1695 ih->ih_name)); 1696 if (ret & FILTER_STRAY) 1697 continue; 1698 else { 1699 *ithd = ih->ih_thread; 1700 return (ret); 1701 } 1702 } 1703 1704 /* 1705 * No filters handled the interrupt and we have at least 1706 * one handler without a filter. In this case, we schedule 1707 * all of the filter-less handlers to run in the ithread. 1708 */ 1709 if (thread_only) { 1710 *ithd = ie->ie_thread; 1711 return (FILTER_SCHEDULE_THREAD); 1712 } 1713 return (FILTER_STRAY); 1714 } 1715 1716 /* 1717 * Main interrupt handling body. 1718 * 1719 * Input: 1720 * o ie: the event connected to this interrupt. 1721 * o frame: some archs (i.e. i386) pass a frame to some. 1722 * handlers as their main argument. 1723 * Return value: 1724 * o 0: everything ok. 1725 * o EINVAL: stray interrupt. 1726 */ 1727 int 1728 intr_event_handle(struct intr_event *ie, struct trapframe *frame) 1729 { 1730 struct intr_thread *ithd; 1731 struct trapframe *oldframe; 1732 struct thread *td; 1733 int thread; 1734 1735 ithd = NULL; 1736 td = curthread; 1737 1738 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1739 return (EINVAL); 1740 1741 td->td_intr_nesting_level++; 1742 thread = 0; 1743 critical_enter(); 1744 oldframe = td->td_intr_frame; 1745 td->td_intr_frame = frame; 1746 thread = intr_filter_loop(ie, frame, &ithd); 1747 if (thread & FILTER_HANDLED) { 1748 if (ie->ie_post_filter != NULL) 1749 ie->ie_post_filter(ie->ie_source); 1750 } else { 1751 if (ie->ie_pre_ithread != NULL) 1752 ie->ie_pre_ithread(ie->ie_source); 1753 } 1754 td->td_intr_frame = oldframe; 1755 critical_exit(); 1756 1757 /* Interrupt storm logic */ 1758 if (thread & FILTER_STRAY) { 1759 ie->ie_count++; 1760 if (ie->ie_count < intr_storm_threshold) 1761 printf("Interrupt stray detection not present\n"); 1762 } 1763 1764 /* Schedule an ithread if needed. */ 1765 if (thread & FILTER_SCHEDULE_THREAD) { 1766 if (intr_event_schedule_thread(ie, ithd) != 0) 1767 panic("%s: impossible stray interrupt", __func__); 1768 } 1769 td->td_intr_nesting_level--; 1770 return (0); 1771 } 1772 #endif 1773 1774 #ifdef DDB 1775 /* 1776 * Dump details about an interrupt handler 1777 */ 1778 static void 1779 db_dump_intrhand(struct intr_handler *ih) 1780 { 1781 int comma; 1782 1783 db_printf("\t%-10s ", ih->ih_name); 1784 switch (ih->ih_pri) { 1785 case PI_REALTIME: 1786 db_printf("CLK "); 1787 break; 1788 case PI_AV: 1789 db_printf("AV "); 1790 break; 1791 case PI_TTY: 1792 db_printf("TTY "); 1793 break; 1794 case PI_NET: 1795 db_printf("NET "); 1796 break; 1797 case PI_DISK: 1798 db_printf("DISK"); 1799 break; 1800 case PI_DULL: 1801 db_printf("DULL"); 1802 break; 1803 default: 1804 if (ih->ih_pri >= PI_SOFT) 1805 db_printf("SWI "); 1806 else 1807 db_printf("%4u", ih->ih_pri); 1808 break; 1809 } 1810 db_printf(" "); 1811 if (ih->ih_filter != NULL) { 1812 db_printf("[F]"); 1813 db_printsym((uintptr_t)ih->ih_filter, DB_STGY_PROC); 1814 } 1815 if (ih->ih_handler != NULL) { 1816 if (ih->ih_filter != NULL) 1817 db_printf(","); 1818 db_printf("[H]"); 1819 db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 1820 } 1821 db_printf("(%p)", ih->ih_argument); 1822 if (ih->ih_need || 1823 (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 1824 IH_MPSAFE)) != 0) { 1825 db_printf(" {"); 1826 comma = 0; 1827 if (ih->ih_flags & IH_EXCLUSIVE) { 1828 if (comma) 1829 db_printf(", "); 1830 db_printf("EXCL"); 1831 comma = 1; 1832 } 1833 if (ih->ih_flags & IH_ENTROPY) { 1834 if (comma) 1835 db_printf(", "); 1836 db_printf("ENTROPY"); 1837 comma = 1; 1838 } 1839 if (ih->ih_flags & IH_DEAD) { 1840 if (comma) 1841 db_printf(", "); 1842 db_printf("DEAD"); 1843 comma = 1; 1844 } 1845 if (ih->ih_flags & IH_MPSAFE) { 1846 if (comma) 1847 db_printf(", "); 1848 db_printf("MPSAFE"); 1849 comma = 1; 1850 } 1851 if (ih->ih_need) { 1852 if (comma) 1853 db_printf(", "); 1854 db_printf("NEED"); 1855 } 1856 db_printf("}"); 1857 } 1858 db_printf("\n"); 1859 } 1860 1861 /* 1862 * Dump details about a event. 1863 */ 1864 void 1865 db_dump_intr_event(struct intr_event *ie, int handlers) 1866 { 1867 struct intr_handler *ih; 1868 struct intr_thread *it; 1869 int comma; 1870 1871 db_printf("%s ", ie->ie_fullname); 1872 it = ie->ie_thread; 1873 if (it != NULL) 1874 db_printf("(pid %d)", it->it_thread->td_proc->p_pid); 1875 else 1876 db_printf("(no thread)"); 1877 if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || 1878 (it != NULL && it->it_need)) { 1879 db_printf(" {"); 1880 comma = 0; 1881 if (ie->ie_flags & IE_SOFT) { 1882 db_printf("SOFT"); 1883 comma = 1; 1884 } 1885 if (ie->ie_flags & IE_ENTROPY) { 1886 if (comma) 1887 db_printf(", "); 1888 db_printf("ENTROPY"); 1889 comma = 1; 1890 } 1891 if (ie->ie_flags & IE_ADDING_THREAD) { 1892 if (comma) 1893 db_printf(", "); 1894 db_printf("ADDING_THREAD"); 1895 comma = 1; 1896 } 1897 if (it != NULL && it->it_need) { 1898 if (comma) 1899 db_printf(", "); 1900 db_printf("NEED"); 1901 } 1902 db_printf("}"); 1903 } 1904 db_printf("\n"); 1905 1906 if (handlers) 1907 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 1908 db_dump_intrhand(ih); 1909 } 1910 1911 /* 1912 * Dump data about interrupt handlers 1913 */ 1914 DB_SHOW_COMMAND(intr, db_show_intr) 1915 { 1916 struct intr_event *ie; 1917 int all, verbose; 1918 1919 verbose = strchr(modif, 'v') != NULL; 1920 all = strchr(modif, 'a') != NULL; 1921 TAILQ_FOREACH(ie, &event_list, ie_list) { 1922 if (!all && TAILQ_EMPTY(&ie->ie_handlers)) 1923 continue; 1924 db_dump_intr_event(ie, verbose); 1925 if (db_pager_quit) 1926 break; 1927 } 1928 } 1929 #endif /* DDB */ 1930 1931 /* 1932 * Start standard software interrupt threads 1933 */ 1934 static void 1935 start_softintr(void *dummy) 1936 { 1937 1938 if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 1939 panic("died while creating vm swi ithread"); 1940 } 1941 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, 1942 NULL); 1943 1944 /* 1945 * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 1946 * The data for this machine dependent, and the declarations are in machine 1947 * dependent code. The layout of intrnames and intrcnt however is machine 1948 * independent. 1949 * 1950 * We do not know the length of intrcnt and intrnames at compile time, so 1951 * calculate things at run time. 1952 */ 1953 static int 1954 sysctl_intrnames(SYSCTL_HANDLER_ARGS) 1955 { 1956 return (sysctl_handle_opaque(oidp, intrnames, sintrnames, req)); 1957 } 1958 1959 SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 1960 NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 1961 1962 static int 1963 sysctl_intrcnt(SYSCTL_HANDLER_ARGS) 1964 { 1965 #ifdef SCTL_MASK32 1966 uint32_t *intrcnt32; 1967 unsigned i; 1968 int error; 1969 1970 if (req->flags & SCTL_MASK32) { 1971 if (!req->oldptr) 1972 return (sysctl_handle_opaque(oidp, NULL, sintrcnt / 2, req)); 1973 intrcnt32 = malloc(sintrcnt / 2, M_TEMP, M_NOWAIT); 1974 if (intrcnt32 == NULL) 1975 return (ENOMEM); 1976 for (i = 0; i < sintrcnt / sizeof (u_long); i++) 1977 intrcnt32[i] = intrcnt[i]; 1978 error = sysctl_handle_opaque(oidp, intrcnt32, sintrcnt / 2, req); 1979 free(intrcnt32, M_TEMP); 1980 return (error); 1981 } 1982 #endif 1983 return (sysctl_handle_opaque(oidp, intrcnt, sintrcnt, req)); 1984 } 1985 1986 SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 1987 NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 1988 1989 #ifdef DDB 1990 /* 1991 * DDB command to dump the interrupt statistics. 1992 */ 1993 DB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 1994 { 1995 u_long *i; 1996 char *cp; 1997 u_int j; 1998 1999 cp = intrnames; 2000 j = 0; 2001 for (i = intrcnt; j < (sintrcnt / sizeof(u_long)) && !db_pager_quit; 2002 i++, j++) { 2003 if (*cp == '\0') 2004 break; 2005 if (*i != 0) 2006 db_printf("%s\t%lu\n", cp, *i); 2007 cp += strlen(cp) + 1; 2008 } 2009 } 2010 #endif 2011