1 /* 2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 30 31 #include <sys/param.h> 32 #include <sys/bus.h> 33 #include <sys/rtprio.h> 34 #include <sys/systm.h> 35 #include <sys/ipl.h> 36 #include <sys/interrupt.h> 37 #include <sys/kernel.h> 38 #include <sys/kthread.h> 39 #include <sys/ktr.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/proc.h> 43 #include <sys/random.h> 44 #include <sys/resourcevar.h> 45 #include <sys/unistd.h> 46 #include <sys/vmmeter.h> 47 #include <machine/atomic.h> 48 #include <machine/cpu.h> 49 #include <machine/md_var.h> 50 #include <machine/stdarg.h> 51 52 #include <net/netisr.h> /* prototype for legacy_setsoftnet */ 53 54 struct int_entropy { 55 struct proc *proc; 56 int vector; 57 }; 58 59 void *net_ih; 60 void *vm_ih; 61 void *softclock_ih; 62 struct ithd *clk_ithd; 63 struct ithd *tty_ithd; 64 65 static struct mtx ithread_list_lock; 66 67 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 68 69 static void ithread_update(struct ithd *); 70 static void ithread_loop(void *); 71 static void ithread_init(void *); 72 static void start_softintr(void *); 73 static void swi_net(void *); 74 75 u_char 76 ithread_priority(enum intr_type flags) 77 { 78 u_char pri; 79 80 flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 81 INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK); 82 switch (flags) { 83 case INTR_TYPE_TTY: 84 pri = PI_TTYLOW; 85 break; 86 case INTR_TYPE_BIO: 87 /* 88 * XXX We need to refine this. BSD/OS distinguishes 89 * between tape and disk priorities. 90 */ 91 pri = PI_DISK; 92 break; 93 case INTR_TYPE_NET: 94 pri = PI_NET; 95 break; 96 case INTR_TYPE_CAM: 97 pri = PI_DISK; /* XXX or PI_CAM? */ 98 break; 99 case INTR_TYPE_CLK: 100 pri = PI_REALTIME; 101 break; 102 case INTR_TYPE_MISC: 103 pri = PI_DULL; /* don't care */ 104 break; 105 default: 106 /* We didn't specify an interrupt level. */ 107 panic("ithread_priority: no interrupt type in flags"); 108 } 109 110 return pri; 111 } 112 113 /* 114 * Regenerate the name (p_comm) and priority for a threaded interrupt thread. 115 */ 116 static void 117 ithread_update(struct ithd *ithd) 118 { 119 struct intrhand *ih; 120 struct proc *p; 121 int entropy; 122 123 p = ithd->it_proc; 124 if (p == NULL) 125 return; 126 127 strncpy(p->p_comm, ithd->it_name, sizeof(ithd->it_name)); 128 ih = TAILQ_FIRST(&ithd->it_handlers); 129 if (ih == NULL) { 130 p->p_pri.pri_level = PRI_MAX_ITHD; 131 ithd->it_flags &= ~IT_ENTROPY; 132 return; 133 } 134 135 entropy = 0; 136 p->p_pri.pri_level = ih->ih_pri; 137 TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { 138 if (strlen(p->p_comm) + strlen(ih->ih_name) + 1 < 139 sizeof(p->p_comm)) { 140 strcat(p->p_comm, " "); 141 strcat(p->p_comm, ih->ih_name); 142 } else if (strlen(p->p_comm) + 1 == sizeof(p->p_comm)) { 143 if (p->p_comm[sizeof(p->p_comm) - 2] == '+') 144 p->p_comm[sizeof(p->p_comm) - 2] = '*'; 145 else 146 p->p_comm[sizeof(p->p_comm) - 2] = '+'; 147 } else 148 strcat(p->p_comm, "+"); 149 if (ih->ih_flags & IH_ENTROPY) 150 entropy++; 151 } 152 153 if (entropy) 154 ithd->it_flags |= IT_ENTROPY; 155 else 156 ithd->it_flags &= ~IT_ENTROPY; 157 158 CTR1(KTR_INTR, __func__ ": updated %s\n", p->p_comm); 159 } 160 161 int 162 ithread_create(struct ithd **ithread, int vector, int flags, 163 void (*disable)(int), void (*enable)(int), const char *fmt, ...) 164 { 165 struct ithd *ithd; 166 struct proc *p; 167 int error; 168 va_list ap; 169 170 /* The only valid flag during creation is IT_SOFT. */ 171 if ((flags & ~IT_SOFT) != 0) 172 return (EINVAL); 173 174 ithd = malloc(sizeof(struct ithd), M_ITHREAD, M_WAITOK | M_ZERO); 175 ithd->it_vector = vector; 176 ithd->it_disable = disable; 177 ithd->it_enable = enable; 178 ithd->it_flags = flags; 179 TAILQ_INIT(&ithd->it_handlers); 180 181 va_start(ap, fmt); 182 vsnprintf(ithd->it_name, sizeof(ithd->it_name), fmt, ap); 183 va_end(ap); 184 185 error = kthread_create(ithread_loop, ithd, &p, RFSTOPPED | RFHIGHPID, 186 ithd->it_name); 187 if (error) { 188 free(ithd, M_ITHREAD); 189 return (error); 190 } 191 p->p_pri.pri_class = PRI_ITHD; 192 p->p_pri.pri_level = PRI_MAX_ITHD; 193 p->p_stat = SWAIT; 194 ithd->it_proc = p; 195 p->p_ithd = ithd; 196 if (ithread != NULL) 197 *ithread = ithd; 198 199 CTR1(KTR_INTR, __func__ ": created %s", ithd->it_name); 200 return (0); 201 } 202 203 int 204 ithread_destroy(struct ithd *ithread) 205 { 206 207 if (ithread == NULL || !TAILQ_EMPTY(&ithread->it_handlers)) 208 return (EINVAL); 209 210 mtx_lock_spin(&sched_lock); 211 ithread->it_flags |= IT_DEAD; 212 if (ithread->it_proc->p_stat == SWAIT) { 213 ithread->it_proc->p_stat = SRUN; 214 setrunqueue(ithread->it_proc); 215 } 216 mtx_unlock_spin(&sched_lock); 217 CTR1(KTR_INTR, __func__ ": killing %s", ithread->it_name); 218 return (0); 219 } 220 221 int 222 ithread_add_handler(struct ithd* ithread, const char *name, 223 driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, 224 void **cookiep) 225 { 226 struct intrhand *ih, *temp_ih; 227 228 if (ithread == NULL || name == NULL || handler == NULL) 229 return (EINVAL); 230 if ((flags & INTR_FAST) !=0) 231 flags |= INTR_EXCL; 232 233 ih = malloc(sizeof(struct intrhand), M_ITHREAD, M_WAITOK | M_ZERO); 234 ih->ih_handler = handler; 235 ih->ih_argument = arg; 236 ih->ih_name = name; 237 ih->ih_ithread = ithread; 238 ih->ih_pri = pri; 239 if (flags & INTR_FAST) 240 ih->ih_flags = IH_FAST | IH_EXCLUSIVE; 241 else if (flags & INTR_EXCL) 242 ih->ih_flags = IH_EXCLUSIVE; 243 if (flags & INTR_MPSAFE) 244 ih->ih_flags |= IH_MPSAFE; 245 if (flags & INTR_ENTROPY) 246 ih->ih_flags |= IH_ENTROPY; 247 248 mtx_lock_spin(&ithread_list_lock); 249 if ((flags & INTR_EXCL) !=0 && !TAILQ_EMPTY(&ithread->it_handlers)) 250 goto fail; 251 if (!TAILQ_EMPTY(&ithread->it_handlers) && 252 (TAILQ_FIRST(&ithread->it_handlers)->ih_flags & IH_EXCLUSIVE) != 0) 253 goto fail; 254 255 TAILQ_FOREACH(temp_ih, &ithread->it_handlers, ih_next) 256 if (temp_ih->ih_pri > ih->ih_pri) 257 break; 258 if (temp_ih == NULL) 259 TAILQ_INSERT_TAIL(&ithread->it_handlers, ih, ih_next); 260 else 261 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 262 ithread_update(ithread); 263 mtx_unlock_spin(&ithread_list_lock); 264 265 if (cookiep != NULL) 266 *cookiep = ih; 267 CTR2(KTR_INTR, __func__ ": added %s to %s", ih->ih_name, 268 ithread->it_name); 269 return (0); 270 271 fail: 272 mtx_unlock_spin(&ithread_list_lock); 273 free(ih, M_ITHREAD); 274 return (EINVAL); 275 } 276 277 int 278 ithread_remove_handler(void *cookie) 279 { 280 struct intrhand *handler = (struct intrhand *)cookie; 281 struct ithd *ithread; 282 #ifdef INVARIANTS 283 struct intrhand *ih; 284 #endif 285 286 if (handler == NULL) 287 return (EINVAL); 288 ithread = handler->ih_ithread; 289 KASSERT(ithread != NULL, 290 ("interrupt handler \"%s\" has a NULL interrupt thread", 291 handler->ih_name)); 292 CTR2(KTR_INTR, __func__ ": removing %s from %s", handler->ih_name, 293 ithread->it_name); 294 mtx_lock_spin(&ithread_list_lock); 295 #ifdef INVARIANTS 296 TAILQ_FOREACH(ih, &ithread->it_handlers, ih_next) 297 if (ih == handler) 298 goto ok; 299 mtx_unlock_spin(&ithread_list_lock); 300 panic("interrupt handler \"%s\" not found in interrupt thread \"%s\"", 301 ih->ih_name, ithread->it_name); 302 ok: 303 #endif 304 /* 305 * If the interrupt thread is already running, then just mark this 306 * handler as being dead and let the ithread do the actual removal. 307 */ 308 mtx_lock_spin(&sched_lock); 309 if (ithread->it_proc->p_stat != SWAIT) { 310 handler->ih_flags |= IH_DEAD; 311 312 /* 313 * Ensure that the thread will process the handler list 314 * again and remove this handler if it has already passed 315 * it on the list. 316 */ 317 ithread->it_need = 1; 318 } else { 319 TAILQ_REMOVE(&ithread->it_handlers, handler, ih_next); 320 ithread_update(ithread); 321 } 322 mtx_unlock_spin(&sched_lock); 323 mtx_unlock_spin(&ithread_list_lock); 324 325 if ((handler->ih_flags & IH_DEAD) == 0) 326 free(handler, M_ITHREAD); 327 return (0); 328 } 329 330 int 331 ithread_schedule(struct ithd *ithread, int do_switch) 332 { 333 struct int_entropy entropy; 334 struct proc *p; 335 intrmask_t saveintr; 336 337 /* 338 * If no ithread or no handlers, then we have a stray interrupt. 339 */ 340 if ((ithread == NULL) || TAILQ_EMPTY(&ithread->it_handlers)) 341 return (EINVAL); 342 343 /* 344 * If any of the handlers for this ithread claim to be good 345 * sources of entropy, then gather some. 346 */ 347 if (harvest.interrupt && ithread->it_flags & IT_ENTROPY) { 348 entropy.vector = ithread->it_vector; 349 entropy.proc = CURPROC; 350 random_harvest(&entropy, sizeof(entropy), 2, 0, 351 RANDOM_INTERRUPT); 352 } 353 354 p = ithread->it_proc; 355 CTR3(KTR_INTR, __func__ ": pid %d: (%s) need = %d", p->p_pid, p->p_comm, 356 ithread->it_need); 357 358 /* 359 * Set it_need to tell the thread to keep running if it is already 360 * running. Then, grab sched_lock and see if we actually need to 361 * put this thread on the runqueue. If so and the do_switch flag is 362 * true, then switch to the ithread immediately. Otherwise, use 363 * need_resched() to guarantee that this ithread will run before any 364 * userland processes. 365 */ 366 ithread->it_need = 1; 367 mtx_lock_spin(&sched_lock); 368 if (p->p_stat == SWAIT) { 369 CTR1(KTR_INTR, __func__ ": setrunqueue %d", p->p_pid); 370 p->p_stat = SRUN; 371 setrunqueue(p); 372 if (do_switch) { 373 saveintr = sched_lock.mtx_saveintr; 374 mtx_intr_enable(&sched_lock); 375 if (curproc != PCPU_GET(idleproc)) 376 setrunqueue(curproc); 377 curproc->p_stats->p_ru.ru_nvcsw++; 378 mi_switch(); 379 sched_lock.mtx_saveintr = saveintr; 380 } else 381 need_resched(); 382 } else { 383 CTR3(KTR_INTR, __func__ ": pid %d: it_need %d, state %d", 384 p->p_pid, ithread->it_need, p->p_stat); 385 } 386 mtx_unlock_spin(&sched_lock); 387 388 return (0); 389 } 390 391 int 392 swi_add(struct ithd **ithdp, const char *name, driver_intr_t handler, 393 void *arg, int pri, enum intr_type flags, void **cookiep) 394 { 395 struct ithd *ithd; 396 int error; 397 398 if (flags & (INTR_FAST | INTR_ENTROPY)) 399 return (EINVAL); 400 401 ithd = (ithdp != NULL) ? *ithdp : NULL; 402 403 if (ithd != NULL) { 404 if ((ithd->it_flags & IT_SOFT) == 0) 405 return(EINVAL); 406 } else { 407 error = ithread_create(&ithd, pri, IT_SOFT, NULL, NULL, 408 "swi%d:", pri); 409 if (error) 410 return (error); 411 412 if (ithdp != NULL) 413 *ithdp = ithd; 414 } 415 return (ithread_add_handler(ithd, name, handler, arg, 416 (pri * RQ_PPQ) + PI_SOFT, flags, cookiep)); 417 } 418 419 420 /* 421 * Schedule a heavyweight software interrupt process. 422 */ 423 void 424 swi_sched(void *cookie, int flags) 425 { 426 struct intrhand *ih = (struct intrhand *)cookie; 427 struct ithd *it = ih->ih_ithread; 428 int error; 429 430 atomic_add_int(&cnt.v_intr, 1); /* one more global interrupt */ 431 432 CTR3(KTR_INTR, "swi_sched pid %d(%s) need=%d", 433 it->it_proc->p_pid, it->it_proc->p_comm, it->it_need); 434 435 /* 436 * Set ih_need for this handler so that if the ithread is already 437 * running it will execute this handler on the next pass. Otherwise, 438 * it will execute it the next time it runs. 439 */ 440 atomic_store_rel_int(&ih->ih_need, 1); 441 if (!(flags & SWI_DELAY)) { 442 error = ithread_schedule(it, !cold && flags & SWI_SWITCH); 443 KASSERT(error == 0, ("stray software interrupt")); 444 } 445 } 446 447 /* 448 * This is the main code for interrupt threads. 449 */ 450 void 451 ithread_loop(void *arg) 452 { 453 struct ithd *ithd; /* our thread context */ 454 struct intrhand *ih; /* and our interrupt handler chain */ 455 struct proc *p; 456 457 p = curproc; 458 ithd = (struct ithd *)arg; /* point to myself */ 459 KASSERT(ithd->it_proc == p && p->p_ithd == ithd, 460 (__func__ ": ithread and proc linkage out of sync")); 461 462 /* 463 * As long as we have interrupts outstanding, go through the 464 * list of handlers, giving each one a go at it. 465 */ 466 for (;;) { 467 /* 468 * If we are an orphaned thread, then just die. 469 */ 470 if (ithd->it_flags & IT_DEAD) { 471 CTR2(KTR_INTR, __func__ ": pid %d: (%s) exiting", 472 p->p_pid, p->p_comm); 473 p->p_ithd = NULL; 474 mtx_lock(&Giant); 475 free(ithd, M_ITHREAD); 476 kthread_exit(0); 477 } 478 479 CTR3(KTR_INTR, __func__ ": pid %d: (%s) need=%d", 480 p->p_pid, p->p_comm, ithd->it_need); 481 while (ithd->it_need) { 482 /* 483 * Service interrupts. If another interrupt 484 * arrives while we are running, they will set 485 * it_need to denote that we should make 486 * another pass. 487 */ 488 atomic_store_rel_int(&ithd->it_need, 0); 489 restart: 490 TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { 491 if (ithd->it_flags & IT_SOFT && !ih->ih_need) 492 continue; 493 atomic_store_rel_int(&ih->ih_need, 0); 494 CTR5(KTR_INTR, 495 __func__ ": pid %d ih=%p: %p(%p) flg=%x", 496 p->p_pid, (void *)ih, 497 (void *)ih->ih_handler, ih->ih_argument, 498 ih->ih_flags); 499 500 if ((ih->ih_flags & IH_MPSAFE) == 0) 501 mtx_lock(&Giant); 502 if ((ih->ih_flags & IH_DEAD) != 0) { 503 mtx_lock_spin(&ithread_list_lock); 504 TAILQ_REMOVE(&ithd->it_handlers, ih, 505 ih_next); 506 ithread_update(ithd); 507 mtx_unlock_spin(&ithread_list_lock); 508 if (!mtx_owned(&Giant)) 509 mtx_lock(&Giant); 510 free(ih, M_ITHREAD); 511 mtx_unlock(&Giant); 512 goto restart; 513 } 514 ih->ih_handler(ih->ih_argument); 515 if ((ih->ih_flags & IH_MPSAFE) == 0) 516 mtx_unlock(&Giant); 517 } 518 } 519 520 /* 521 * Processed all our interrupts. Now get the sched 522 * lock. This may take a while and it_need may get 523 * set again, so we have to check it again. 524 */ 525 mtx_assert(&Giant, MA_NOTOWNED); 526 mtx_lock_spin(&sched_lock); 527 if (!ithd->it_need) { 528 /* 529 * Should we call this earlier in the loop above? 530 */ 531 if (ithd->it_enable != NULL) 532 ithd->it_enable(ithd->it_vector); 533 p->p_stat = SWAIT; /* we're idle */ 534 CTR1(KTR_INTR, __func__ ": pid %d: done", p->p_pid); 535 mi_switch(); 536 CTR1(KTR_INTR, __func__ ": pid %d: resumed", p->p_pid); 537 } 538 mtx_unlock_spin(&sched_lock); 539 } 540 } 541 542 /* 543 * Initialize mutex used to protect ithread handler lists. 544 */ 545 static void 546 ithread_init(void *dummy) 547 { 548 549 mtx_init(&ithread_list_lock, "ithread list lock", MTX_SPIN); 550 } 551 SYSINIT(ithread_init, SI_SUB_INTR, SI_ORDER_FIRST, ithread_init, NULL); 552 553 /* 554 * Start standard software interrupt threads 555 */ 556 static void 557 start_softintr(void *dummy) 558 { 559 560 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, 0, &net_ih) || 561 swi_add(&clk_ithd, "clock", softclock, NULL, SWI_CLOCK, 562 INTR_MPSAFE, &softclock_ih) || 563 swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, 0, &vm_ih)) 564 panic("died while creating standard software ithreads"); 565 566 PROC_LOCK(clk_ithd->it_proc); 567 clk_ithd->it_proc->p_flag |= P_NOLOAD; 568 PROC_UNLOCK(clk_ithd->it_proc); 569 } 570 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL) 571 572 void 573 legacy_setsoftnet(void) 574 { 575 swi_sched(net_ih, SWI_NOSWITCH); 576 } 577 578 /* 579 * XXX: This should really be in the network code somewhere and installed 580 * via a SI_SUB_SOFINTR, SI_ORDER_MIDDLE sysinit. 581 */ 582 void (*netisrs[32]) __P((void)); 583 u_int netisr; 584 585 int 586 register_netisr(num, handler) 587 int num; 588 netisr_t *handler; 589 { 590 591 if (num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs)) ) { 592 printf("register_netisr: bad isr number: %d\n", num); 593 return (EINVAL); 594 } 595 netisrs[num] = handler; 596 return (0); 597 } 598 599 int 600 unregister_netisr(num) 601 int num; 602 { 603 604 if (num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs)) ) { 605 printf("unregister_netisr: bad isr number: %d\n", num); 606 return (EINVAL); 607 } 608 netisrs[num] = NULL; 609 return (0); 610 } 611 612 static void 613 swi_net(void *dummy) 614 { 615 u_int bits; 616 int i; 617 618 bits = atomic_readandclear_int(&netisr); 619 while ((i = ffs(bits)) != 0) { 620 i--; 621 if (netisrs[i] != NULL) 622 netisrs[i](); 623 else 624 printf("swi_net: unregistered isr number: %d.\n", i); 625 bits &= ~(1 << i); 626 } 627 } 628