1 /*- 2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_ddb.h" 31 32 #include <sys/param.h> 33 #include <sys/bus.h> 34 #include <sys/conf.h> 35 #include <sys/rtprio.h> 36 #include <sys/systm.h> 37 #include <sys/interrupt.h> 38 #include <sys/kernel.h> 39 #include <sys/kthread.h> 40 #include <sys/ktr.h> 41 #include <sys/limits.h> 42 #include <sys/lock.h> 43 #include <sys/malloc.h> 44 #include <sys/mutex.h> 45 #include <sys/proc.h> 46 #include <sys/random.h> 47 #include <sys/resourcevar.h> 48 #include <sys/sched.h> 49 #include <sys/sysctl.h> 50 #include <sys/unistd.h> 51 #include <sys/vmmeter.h> 52 #include <machine/atomic.h> 53 #include <machine/cpu.h> 54 #include <machine/md_var.h> 55 #include <machine/stdarg.h> 56 #ifdef DDB 57 #include <ddb/ddb.h> 58 #include <ddb/db_sym.h> 59 #endif 60 61 /* 62 * Describe an interrupt thread. There is one of these per interrupt event. 63 */ 64 struct intr_thread { 65 struct intr_event *it_event; 66 struct thread *it_thread; /* Kernel thread. */ 67 int it_flags; /* (j) IT_* flags. */ 68 int it_need; /* Needs service. */ 69 }; 70 71 /* Interrupt thread flags kept in it_flags */ 72 #define IT_DEAD 0x000001 /* Thread is waiting to exit. */ 73 74 struct intr_entropy { 75 struct thread *td; 76 uintptr_t event; 77 }; 78 79 struct intr_event *clk_intr_event; 80 struct intr_event *tty_intr_event; 81 void *softclock_ih; 82 void *vm_ih; 83 84 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 85 86 static int intr_storm_threshold = 500; 87 TUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold); 88 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW, 89 &intr_storm_threshold, 0, 90 "Number of consecutive interrupts before storm protection is enabled"); 91 static TAILQ_HEAD(, intr_event) event_list = 92 TAILQ_HEAD_INITIALIZER(event_list); 93 94 static void intr_event_update(struct intr_event *ie); 95 static struct intr_thread *ithread_create(const char *name); 96 #ifdef notyet 97 static void ithread_destroy(struct intr_thread *ithread); 98 #endif 99 static void ithread_execute_handlers(struct proc *p, struct intr_event *ie); 100 static void ithread_loop(void *); 101 static void ithread_update(struct intr_thread *ithd); 102 static void start_softintr(void *); 103 104 u_char 105 intr_priority(enum intr_type flags) 106 { 107 u_char pri; 108 109 flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 110 INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 111 switch (flags) { 112 case INTR_TYPE_TTY: 113 pri = PI_TTYLOW; 114 break; 115 case INTR_TYPE_BIO: 116 /* 117 * XXX We need to refine this. BSD/OS distinguishes 118 * between tape and disk priorities. 119 */ 120 pri = PI_DISK; 121 break; 122 case INTR_TYPE_NET: 123 pri = PI_NET; 124 break; 125 case INTR_TYPE_CAM: 126 pri = PI_DISK; /* XXX or PI_CAM? */ 127 break; 128 case INTR_TYPE_AV: /* Audio/video */ 129 pri = PI_AV; 130 break; 131 case INTR_TYPE_CLK: 132 pri = PI_REALTIME; 133 break; 134 case INTR_TYPE_MISC: 135 pri = PI_DULL; /* don't care */ 136 break; 137 default: 138 /* We didn't specify an interrupt level. */ 139 panic("intr_priority: no interrupt type in flags"); 140 } 141 142 return pri; 143 } 144 145 /* 146 * Update an ithread based on the associated intr_event. 147 */ 148 static void 149 ithread_update(struct intr_thread *ithd) 150 { 151 struct intr_event *ie; 152 struct thread *td; 153 u_char pri; 154 155 ie = ithd->it_event; 156 td = ithd->it_thread; 157 158 /* Determine the overall priority of this event. */ 159 if (TAILQ_EMPTY(&ie->ie_handlers)) 160 pri = PRI_MAX_ITHD; 161 else 162 pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; 163 164 /* Update name and priority. */ 165 strlcpy(td->td_proc->p_comm, ie->ie_fullname, 166 sizeof(td->td_proc->p_comm)); 167 mtx_lock_spin(&sched_lock); 168 sched_prio(td, pri); 169 mtx_unlock_spin(&sched_lock); 170 } 171 172 /* 173 * Regenerate the full name of an interrupt event and update its priority. 174 */ 175 static void 176 intr_event_update(struct intr_event *ie) 177 { 178 struct intr_handler *ih; 179 char *last; 180 int missed, space; 181 182 /* Start off with no entropy and just the name of the event. */ 183 mtx_assert(&ie->ie_lock, MA_OWNED); 184 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 185 ie->ie_flags &= ~IE_ENTROPY; 186 missed = 0; 187 space = 1; 188 189 /* Run through all the handlers updating values. */ 190 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 191 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < 192 sizeof(ie->ie_fullname)) { 193 strcat(ie->ie_fullname, " "); 194 strcat(ie->ie_fullname, ih->ih_name); 195 space = 0; 196 } else 197 missed++; 198 if (ih->ih_flags & IH_ENTROPY) 199 ie->ie_flags |= IE_ENTROPY; 200 } 201 202 /* 203 * If the handler names were too long, add +'s to indicate missing 204 * names. If we run out of room and still have +'s to add, change 205 * the last character from a + to a *. 206 */ 207 last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; 208 while (missed-- > 0) { 209 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { 210 if (*last == '+') { 211 *last = '*'; 212 break; 213 } else 214 *last = '+'; 215 } else if (space) { 216 strcat(ie->ie_fullname, " +"); 217 space = 0; 218 } else 219 strcat(ie->ie_fullname, "+"); 220 } 221 222 /* 223 * If this event has an ithread, update it's priority and 224 * name. 225 */ 226 if (ie->ie_thread != NULL) 227 ithread_update(ie->ie_thread); 228 CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); 229 } 230 231 int 232 intr_event_create(struct intr_event **event, void *source, int flags, 233 void (*enable)(void *), const char *fmt, ...) 234 { 235 struct intr_event *ie; 236 va_list ap; 237 238 /* The only valid flag during creation is IE_SOFT. */ 239 if ((flags & ~IE_SOFT) != 0) 240 return (EINVAL); 241 ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 242 ie->ie_source = source; 243 ie->ie_enable = enable; 244 ie->ie_flags = flags; 245 TAILQ_INIT(&ie->ie_handlers); 246 mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 247 248 va_start(ap, fmt); 249 vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 250 va_end(ap); 251 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 252 mtx_pool_lock(mtxpool_sleep, &event_list); 253 TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 254 mtx_pool_unlock(mtxpool_sleep, &event_list); 255 if (event != NULL) 256 *event = ie; 257 CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 258 return (0); 259 } 260 261 int 262 intr_event_destroy(struct intr_event *ie) 263 { 264 265 mtx_lock(&ie->ie_lock); 266 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 267 mtx_unlock(&ie->ie_lock); 268 return (EBUSY); 269 } 270 mtx_pool_lock(mtxpool_sleep, &event_list); 271 TAILQ_REMOVE(&event_list, ie, ie_list); 272 mtx_pool_unlock(mtxpool_sleep, &event_list); 273 mtx_unlock(&ie->ie_lock); 274 mtx_destroy(&ie->ie_lock); 275 free(ie, M_ITHREAD); 276 return (0); 277 } 278 279 static struct intr_thread * 280 ithread_create(const char *name) 281 { 282 struct intr_thread *ithd; 283 struct thread *td; 284 struct proc *p; 285 int error; 286 287 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 288 289 error = kthread_create(ithread_loop, ithd, &p, RFSTOPPED | RFHIGHPID, 290 0, "%s", name); 291 if (error) 292 panic("kthread_create() failed with %d", error); 293 td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */ 294 mtx_lock_spin(&sched_lock); 295 td->td_ksegrp->kg_pri_class = PRI_ITHD; 296 TD_SET_IWAIT(td); 297 mtx_unlock_spin(&sched_lock); 298 td->td_pflags |= TDP_ITHREAD; 299 ithd->it_thread = td; 300 CTR2(KTR_INTR, "%s: created %s", __func__, name); 301 return (ithd); 302 } 303 304 #ifdef notyet 305 static void 306 ithread_destroy(struct intr_thread *ithread) 307 { 308 struct thread *td; 309 310 td = ithread->it_thread; 311 mtx_lock_spin(&sched_lock); 312 ithread->it_flags |= IT_DEAD; 313 if (TD_AWAITING_INTR(td)) { 314 TD_CLR_IWAIT(td); 315 setrunqueue(td, SRQ_INTR); 316 } 317 mtx_unlock_spin(&sched_lock); 318 CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_name); 319 } 320 #endif 321 322 int 323 intr_event_add_handler(struct intr_event *ie, const char *name, 324 driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, 325 void **cookiep) 326 { 327 struct intr_handler *ih, *temp_ih; 328 struct intr_thread *it; 329 330 if (ie == NULL || name == NULL || handler == NULL) 331 return (EINVAL); 332 333 /* Allocate and populate an interrupt handler structure. */ 334 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 335 ih->ih_handler = handler; 336 ih->ih_argument = arg; 337 ih->ih_name = name; 338 ih->ih_event = ie; 339 ih->ih_pri = pri; 340 if (flags & INTR_FAST) 341 ih->ih_flags = IH_FAST; 342 else if (flags & INTR_EXCL) 343 ih->ih_flags = IH_EXCLUSIVE; 344 if (flags & INTR_MPSAFE) 345 ih->ih_flags |= IH_MPSAFE; 346 if (flags & INTR_ENTROPY) 347 ih->ih_flags |= IH_ENTROPY; 348 349 /* We can only have one exclusive handler in a event. */ 350 mtx_lock(&ie->ie_lock); 351 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 352 if ((flags & INTR_EXCL) || 353 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 354 mtx_unlock(&ie->ie_lock); 355 free(ih, M_ITHREAD); 356 return (EINVAL); 357 } 358 } 359 360 /* Add the new handler to the event in priority order. */ 361 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 362 if (temp_ih->ih_pri > ih->ih_pri) 363 break; 364 } 365 if (temp_ih == NULL) 366 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 367 else 368 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 369 intr_event_update(ie); 370 371 /* Create a thread if we need one. */ 372 while (ie->ie_thread == NULL && !(flags & INTR_FAST)) { 373 if (ie->ie_flags & IE_ADDING_THREAD) 374 msleep(ie, &ie->ie_lock, curthread->td_priority, 375 "ithread", 0); 376 else { 377 ie->ie_flags |= IE_ADDING_THREAD; 378 mtx_unlock(&ie->ie_lock); 379 it = ithread_create("intr: newborn"); 380 mtx_lock(&ie->ie_lock); 381 ie->ie_flags &= ~IE_ADDING_THREAD; 382 ie->ie_thread = it; 383 it->it_event = ie; 384 ithread_update(it); 385 wakeup(ie); 386 } 387 } 388 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 389 ie->ie_name); 390 mtx_unlock(&ie->ie_lock); 391 392 if (cookiep != NULL) 393 *cookiep = ih; 394 return (0); 395 } 396 397 int 398 intr_event_remove_handler(void *cookie) 399 { 400 struct intr_handler *handler = (struct intr_handler *)cookie; 401 struct intr_event *ie; 402 #ifdef INVARIANTS 403 struct intr_handler *ih; 404 #endif 405 #ifdef notyet 406 int dead; 407 #endif 408 409 if (handler == NULL) 410 return (EINVAL); 411 ie = handler->ih_event; 412 KASSERT(ie != NULL, 413 ("interrupt handler \"%s\" has a NULL interrupt event", 414 handler->ih_name)); 415 mtx_lock(&ie->ie_lock); 416 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 417 ie->ie_name); 418 #ifdef INVARIANTS 419 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 420 if (ih == handler) 421 goto ok; 422 mtx_unlock(&ie->ie_lock); 423 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 424 ih->ih_name, ie->ie_name); 425 ok: 426 #endif 427 /* 428 * If there is no ithread, then just remove the handler and return. 429 * XXX: Note that an INTR_FAST handler might be running on another 430 * CPU! 431 */ 432 if (ie->ie_thread == NULL) { 433 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 434 mtx_unlock(&ie->ie_lock); 435 free(handler, M_ITHREAD); 436 return (0); 437 } 438 439 /* 440 * If the interrupt thread is already running, then just mark this 441 * handler as being dead and let the ithread do the actual removal. 442 * 443 * During a cold boot while cold is set, msleep() does not sleep, 444 * so we have to remove the handler here rather than letting the 445 * thread do it. 446 */ 447 mtx_lock_spin(&sched_lock); 448 if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { 449 handler->ih_flags |= IH_DEAD; 450 451 /* 452 * Ensure that the thread will process the handler list 453 * again and remove this handler if it has already passed 454 * it on the list. 455 */ 456 ie->ie_thread->it_need = 1; 457 } else 458 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 459 mtx_unlock_spin(&sched_lock); 460 while (handler->ih_flags & IH_DEAD) 461 msleep(handler, &ie->ie_lock, curthread->td_priority, "iev_rmh", 462 0); 463 intr_event_update(ie); 464 #ifdef notyet 465 /* 466 * XXX: This could be bad in the case of ppbus(8). Also, I think 467 * this could lead to races of stale data when servicing an 468 * interrupt. 469 */ 470 dead = 1; 471 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 472 if (!(ih->ih_flags & IH_FAST)) { 473 dead = 0; 474 break; 475 } 476 } 477 if (dead) { 478 ithread_destroy(ie->ie_thread); 479 ie->ie_thread = NULL; 480 } 481 #endif 482 mtx_unlock(&ie->ie_lock); 483 free(handler, M_ITHREAD); 484 return (0); 485 } 486 487 int 488 intr_event_schedule_thread(struct intr_event *ie) 489 { 490 struct intr_entropy entropy; 491 struct intr_thread *it; 492 struct thread *td; 493 struct thread *ctd; 494 struct proc *p; 495 496 /* 497 * If no ithread or no handlers, then we have a stray interrupt. 498 */ 499 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || 500 ie->ie_thread == NULL) 501 return (EINVAL); 502 503 ctd = curthread; 504 it = ie->ie_thread; 505 td = it->it_thread; 506 p = td->td_proc; 507 508 /* 509 * If any of the handlers for this ithread claim to be good 510 * sources of entropy, then gather some. 511 */ 512 if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 513 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 514 p->p_pid, p->p_comm); 515 entropy.event = (uintptr_t)ie; 516 entropy.td = ctd; 517 random_harvest(&entropy, sizeof(entropy), 2, 0, 518 RANDOM_INTERRUPT); 519 } 520 521 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 522 523 /* 524 * Set it_need to tell the thread to keep running if it is already 525 * running. Then, grab sched_lock and see if we actually need to 526 * put this thread on the runqueue. 527 */ 528 it->it_need = 1; 529 mtx_lock_spin(&sched_lock); 530 if (TD_AWAITING_INTR(td)) { 531 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 532 p->p_comm); 533 TD_CLR_IWAIT(td); 534 setrunqueue(td, SRQ_INTR); 535 } else { 536 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 537 __func__, p->p_pid, p->p_comm, it->it_need, td->td_state); 538 } 539 mtx_unlock_spin(&sched_lock); 540 541 return (0); 542 } 543 544 /* 545 * Add a software interrupt handler to a specified event. If a given event 546 * is not specified, then a new event is created. 547 */ 548 int 549 swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, 550 void *arg, int pri, enum intr_type flags, void **cookiep) 551 { 552 struct intr_event *ie; 553 int error; 554 555 if (flags & (INTR_FAST | INTR_ENTROPY)) 556 return (EINVAL); 557 558 ie = (eventp != NULL) ? *eventp : NULL; 559 560 if (ie != NULL) { 561 if (!(ie->ie_flags & IE_SOFT)) 562 return (EINVAL); 563 } else { 564 error = intr_event_create(&ie, NULL, IE_SOFT, NULL, 565 "swi%d:", pri); 566 if (error) 567 return (error); 568 if (eventp != NULL) 569 *eventp = ie; 570 } 571 return (intr_event_add_handler(ie, name, handler, arg, 572 (pri * RQ_PPQ) + PI_SOFT, flags, cookiep)); 573 /* XXKSE.. think of a better way to get separate queues */ 574 } 575 576 /* 577 * Schedule a software interrupt thread. 578 */ 579 void 580 swi_sched(void *cookie, int flags) 581 { 582 struct intr_handler *ih = (struct intr_handler *)cookie; 583 struct intr_event *ie = ih->ih_event; 584 int error; 585 586 PCPU_LAZY_INC(cnt.v_intr); 587 588 CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, 589 ih->ih_need); 590 591 /* 592 * Set ih_need for this handler so that if the ithread is already 593 * running it will execute this handler on the next pass. Otherwise, 594 * it will execute it the next time it runs. 595 */ 596 atomic_store_rel_int(&ih->ih_need, 1); 597 if (!(flags & SWI_DELAY)) { 598 error = intr_event_schedule_thread(ie); 599 KASSERT(error == 0, ("stray software interrupt")); 600 } 601 } 602 603 /* 604 * Remove a software interrupt handler. Currently this code does not 605 * remove the associated interrupt event if it becomes empty. Calling code 606 * may do so manually via intr_event_destroy(), but that's not really 607 * an optimal interface. 608 */ 609 int 610 swi_remove(void *cookie) 611 { 612 613 return (intr_event_remove_handler(cookie)); 614 } 615 616 static void 617 ithread_execute_handlers(struct proc *p, struct intr_event *ie) 618 { 619 struct intr_handler *ih, *ihn; 620 621 /* Interrupt handlers should not sleep. */ 622 if (!(ie->ie_flags & IE_SOFT)) 623 THREAD_NO_SLEEPING(); 624 TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { 625 626 /* 627 * If this handler is marked for death, remove it from 628 * the list of handlers and wake up the sleeper. 629 */ 630 if (ih->ih_flags & IH_DEAD) { 631 mtx_lock(&ie->ie_lock); 632 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 633 ih->ih_flags &= ~IH_DEAD; 634 wakeup(ih); 635 mtx_unlock(&ie->ie_lock); 636 continue; 637 } 638 639 /* 640 * For software interrupt threads, we only execute 641 * handlers that have their need flag set. Hardware 642 * interrupt threads always invoke all of their handlers. 643 */ 644 if (ie->ie_flags & IE_SOFT) { 645 if (!ih->ih_need) 646 continue; 647 else 648 atomic_store_rel_int(&ih->ih_need, 0); 649 } 650 651 /* Fast handlers are handled in primary interrupt context. */ 652 if (ih->ih_flags & IH_FAST) 653 continue; 654 655 /* Execute this handler. */ 656 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 657 __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, 658 ih->ih_name, ih->ih_flags); 659 660 if (!(ih->ih_flags & IH_MPSAFE)) 661 mtx_lock(&Giant); 662 ih->ih_handler(ih->ih_argument); 663 if (!(ih->ih_flags & IH_MPSAFE)) 664 mtx_unlock(&Giant); 665 } 666 if (!(ie->ie_flags & IE_SOFT)) 667 THREAD_SLEEPING_OK(); 668 669 /* 670 * Interrupt storm handling: 671 * 672 * If this interrupt source is currently storming, then throttle 673 * it to only fire the handler once per clock tick. 674 * 675 * If this interrupt source is not currently storming, but the 676 * number of back to back interrupts exceeds the storm threshold, 677 * then enter storming mode. 678 */ 679 if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold) { 680 if (ie->ie_warned == 0) { 681 printf( 682 "Interrupt storm detected on \"%s\"; throttling interrupt source\n", 683 ie->ie_name); 684 ie->ie_warned = 1; 685 } 686 tsleep(&ie->ie_count, curthread->td_priority, "istorm", 1); 687 } else 688 ie->ie_count++; 689 690 /* 691 * Now that all the handlers have had a chance to run, reenable 692 * the interrupt source. 693 */ 694 if (ie->ie_enable != NULL) 695 ie->ie_enable(ie->ie_source); 696 } 697 698 /* 699 * This is the main code for interrupt threads. 700 */ 701 static void 702 ithread_loop(void *arg) 703 { 704 struct intr_thread *ithd; 705 struct intr_event *ie; 706 struct thread *td; 707 struct proc *p; 708 709 td = curthread; 710 p = td->td_proc; 711 ithd = (struct intr_thread *)arg; 712 KASSERT(ithd->it_thread == td, 713 ("%s: ithread and proc linkage out of sync", __func__)); 714 ie = ithd->it_event; 715 ie->ie_count = 0; 716 717 /* 718 * As long as we have interrupts outstanding, go through the 719 * list of handlers, giving each one a go at it. 720 */ 721 for (;;) { 722 /* 723 * If we are an orphaned thread, then just die. 724 */ 725 if (ithd->it_flags & IT_DEAD) { 726 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 727 p->p_pid, p->p_comm); 728 free(ithd, M_ITHREAD); 729 kthread_exit(0); 730 } 731 732 /* 733 * Service interrupts. If another interrupt arrives while 734 * we are running, it will set it_need to note that we 735 * should make another pass. 736 */ 737 while (ithd->it_need) { 738 /* 739 * This might need a full read and write barrier 740 * to make sure that this write posts before any 741 * of the memory or device accesses in the 742 * handlers. 743 */ 744 atomic_store_rel_int(&ithd->it_need, 0); 745 ithread_execute_handlers(p, ie); 746 } 747 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 748 mtx_assert(&Giant, MA_NOTOWNED); 749 750 /* 751 * Processed all our interrupts. Now get the sched 752 * lock. This may take a while and it_need may get 753 * set again, so we have to check it again. 754 */ 755 mtx_lock_spin(&sched_lock); 756 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) { 757 TD_SET_IWAIT(td); 758 ie->ie_count = 0; 759 mi_switch(SW_VOL, NULL); 760 } 761 mtx_unlock_spin(&sched_lock); 762 } 763 } 764 765 #ifdef DDB 766 /* 767 * Dump details about an interrupt handler 768 */ 769 static void 770 db_dump_intrhand(struct intr_handler *ih) 771 { 772 int comma; 773 774 db_printf("\t%-10s ", ih->ih_name); 775 switch (ih->ih_pri) { 776 case PI_REALTIME: 777 db_printf("CLK "); 778 break; 779 case PI_AV: 780 db_printf("AV "); 781 break; 782 case PI_TTYHIGH: 783 case PI_TTYLOW: 784 db_printf("TTY "); 785 break; 786 case PI_TAPE: 787 db_printf("TAPE"); 788 break; 789 case PI_NET: 790 db_printf("NET "); 791 break; 792 case PI_DISK: 793 case PI_DISKLOW: 794 db_printf("DISK"); 795 break; 796 case PI_DULL: 797 db_printf("DULL"); 798 break; 799 default: 800 if (ih->ih_pri >= PI_SOFT) 801 db_printf("SWI "); 802 else 803 db_printf("%4u", ih->ih_pri); 804 break; 805 } 806 db_printf(" "); 807 db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 808 db_printf("(%p)", ih->ih_argument); 809 if (ih->ih_need || 810 (ih->ih_flags & (IH_FAST | IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 811 IH_MPSAFE)) != 0) { 812 db_printf(" {"); 813 comma = 0; 814 if (ih->ih_flags & IH_FAST) { 815 db_printf("FAST"); 816 comma = 1; 817 } 818 if (ih->ih_flags & IH_EXCLUSIVE) { 819 if (comma) 820 db_printf(", "); 821 db_printf("EXCL"); 822 comma = 1; 823 } 824 if (ih->ih_flags & IH_ENTROPY) { 825 if (comma) 826 db_printf(", "); 827 db_printf("ENTROPY"); 828 comma = 1; 829 } 830 if (ih->ih_flags & IH_DEAD) { 831 if (comma) 832 db_printf(", "); 833 db_printf("DEAD"); 834 comma = 1; 835 } 836 if (ih->ih_flags & IH_MPSAFE) { 837 if (comma) 838 db_printf(", "); 839 db_printf("MPSAFE"); 840 comma = 1; 841 } 842 if (ih->ih_need) { 843 if (comma) 844 db_printf(", "); 845 db_printf("NEED"); 846 } 847 db_printf("}"); 848 } 849 db_printf("\n"); 850 } 851 852 /* 853 * Dump details about a event. 854 */ 855 void 856 db_dump_intr_event(struct intr_event *ie, int handlers) 857 { 858 struct intr_handler *ih; 859 struct intr_thread *it; 860 int comma; 861 862 db_printf("%s ", ie->ie_fullname); 863 it = ie->ie_thread; 864 if (it != NULL) 865 db_printf("(pid %d)", it->it_thread->td_proc->p_pid); 866 else 867 db_printf("(no thread)"); 868 if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || 869 (it != NULL && it->it_need)) { 870 db_printf(" {"); 871 comma = 0; 872 if (ie->ie_flags & IE_SOFT) { 873 db_printf("SOFT"); 874 comma = 1; 875 } 876 if (ie->ie_flags & IE_ENTROPY) { 877 if (comma) 878 db_printf(", "); 879 db_printf("ENTROPY"); 880 comma = 1; 881 } 882 if (ie->ie_flags & IE_ADDING_THREAD) { 883 if (comma) 884 db_printf(", "); 885 db_printf("ADDING_THREAD"); 886 comma = 1; 887 } 888 if (it != NULL && it->it_need) { 889 if (comma) 890 db_printf(", "); 891 db_printf("NEED"); 892 } 893 db_printf("}"); 894 } 895 db_printf("\n"); 896 897 if (handlers) 898 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 899 db_dump_intrhand(ih); 900 } 901 902 /* 903 * Dump data about interrupt handlers 904 */ 905 DB_SHOW_COMMAND(intr, db_show_intr) 906 { 907 struct intr_event *ie; 908 int quit, all, verbose; 909 910 quit = 0; 911 verbose = index(modif, 'v') != NULL; 912 all = index(modif, 'a') != NULL; 913 db_setup_paging(db_simple_pager, &quit, db_lines_per_page); 914 TAILQ_FOREACH(ie, &event_list, ie_list) { 915 if (!all && TAILQ_EMPTY(&ie->ie_handlers)) 916 continue; 917 db_dump_intr_event(ie, verbose); 918 } 919 } 920 #endif /* DDB */ 921 922 /* 923 * Start standard software interrupt threads 924 */ 925 static void 926 start_softintr(void *dummy) 927 { 928 struct proc *p; 929 930 if (swi_add(&clk_intr_event, "clock", softclock, NULL, SWI_CLOCK, 931 INTR_MPSAFE, &softclock_ih) || 932 swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 933 panic("died while creating standard software ithreads"); 934 935 p = clk_intr_event->ie_thread->it_thread->td_proc; 936 PROC_LOCK(p); 937 p->p_flag |= P_NOLOAD; 938 PROC_UNLOCK(p); 939 } 940 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL) 941 942 /* 943 * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 944 * The data for this machine dependent, and the declarations are in machine 945 * dependent code. The layout of intrnames and intrcnt however is machine 946 * independent. 947 * 948 * We do not know the length of intrcnt and intrnames at compile time, so 949 * calculate things at run time. 950 */ 951 static int 952 sysctl_intrnames(SYSCTL_HANDLER_ARGS) 953 { 954 return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames, 955 req)); 956 } 957 958 SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 959 NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 960 961 static int 962 sysctl_intrcnt(SYSCTL_HANDLER_ARGS) 963 { 964 return (sysctl_handle_opaque(oidp, intrcnt, 965 (char *)eintrcnt - (char *)intrcnt, req)); 966 } 967 968 SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 969 NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 970 971 #ifdef DDB 972 /* 973 * DDB command to dump the interrupt statistics. 974 */ 975 DB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 976 { 977 u_long *i; 978 char *cp; 979 int quit; 980 981 cp = intrnames; 982 db_setup_paging(db_simple_pager, &quit, db_lines_per_page); 983 for (i = intrcnt, quit = 0; i != eintrcnt && !quit; i++) { 984 if (*cp == '\0') 985 break; 986 if (*i != 0) 987 db_printf("%s\t%lu\n", cp, *i); 988 cp += strlen(cp) + 1; 989 } 990 } 991 #endif 992