1 /* 2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 30 31 #include <sys/param.h> 32 #include <sys/bus.h> 33 #include <sys/rtprio.h> 34 #include <sys/systm.h> 35 #include <sys/ipl.h> 36 #include <sys/interrupt.h> 37 #include <sys/kernel.h> 38 #include <sys/kthread.h> 39 #include <sys/ktr.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/proc.h> 43 #include <sys/resourcevar.h> 44 #include <sys/unistd.h> 45 #include <sys/vmmeter.h> 46 #include <machine/atomic.h> 47 #include <machine/cpu.h> 48 #include <machine/md_var.h> 49 #include <machine/stdarg.h> 50 51 #include <net/netisr.h> /* prototype for legacy_setsoftnet */ 52 53 void *net_ih; 54 void *vm_ih; 55 void *softclock_ih; 56 struct ithd *clk_ithd; 57 struct ithd *tty_ithd; 58 59 static struct mtx ithread_list_lock; 60 61 static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 62 63 static void ithread_update(struct ithd *); 64 static void ithread_loop(void *); 65 static void ithread_init(void *); 66 static void start_softintr(void *); 67 static void swi_net(void *); 68 69 u_char 70 ithread_priority(enum intr_type flags) 71 { 72 u_char pri; 73 74 flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 75 INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK); 76 switch (flags) { 77 case INTR_TYPE_TTY: 78 pri = PI_TTYLOW; 79 break; 80 case INTR_TYPE_BIO: 81 /* 82 * XXX We need to refine this. BSD/OS distinguishes 83 * between tape and disk priorities. 84 */ 85 pri = PI_DISK; 86 break; 87 case INTR_TYPE_NET: 88 pri = PI_NET; 89 break; 90 case INTR_TYPE_CAM: 91 pri = PI_DISK; /* XXX or PI_CAM? */ 92 break; 93 case INTR_TYPE_CLK: 94 pri = PI_REALTIME; 95 break; 96 case INTR_TYPE_MISC: 97 pri = PI_DULL; /* don't care */ 98 break; 99 default: 100 /* We didn't specify an interrupt level. */ 101 panic("ithread_priority: no interrupt type in flags"); 102 } 103 104 return pri; 105 } 106 107 /* 108 * Regenerate the name (p_comm) and priority for a threaded interrupt thread. 109 */ 110 static void 111 ithread_update(struct ithd *ithd) 112 { 113 struct intrhand *ih; 114 struct proc *p; 115 int entropy; 116 117 p = ithd->it_proc; 118 if (p == NULL) 119 return; 120 121 strncpy(p->p_comm, ithd->it_name, sizeof(ithd->it_name)); 122 ih = TAILQ_FIRST(&ithd->it_handlers); 123 if (ih == NULL) { 124 p->p_pri.pri_level = PRI_MAX_ITHD; 125 ithd->it_flags &= ~IT_ENTROPY; 126 return; 127 } 128 129 entropy = 0; 130 p->p_pri.pri_level = ih->ih_pri; 131 TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { 132 if (strlen(p->p_comm) + strlen(ih->ih_name) + 1 < 133 sizeof(p->p_comm)) { 134 strcat(p->p_comm, " "); 135 strcat(p->p_comm, ih->ih_name); 136 } else if (strlen(p->p_comm) + 1 == sizeof(p->p_comm)) { 137 if (p->p_comm[sizeof(p->p_comm) - 2] == '+') 138 p->p_comm[sizeof(p->p_comm) - 2] = '*'; 139 else 140 p->p_comm[sizeof(p->p_comm) - 2] = '+'; 141 } else 142 strcat(p->p_comm, "+"); 143 if (ih->ih_flags & IH_ENTROPY) 144 entropy++; 145 } 146 147 if (entropy) { 148 printf("Warning, ithread (%d, %s) is an entropy source.\n", 149 p->p_pid, p->p_comm); 150 ithd->it_flags |= IT_ENTROPY; 151 } 152 else 153 ithd->it_flags &= ~IT_ENTROPY; 154 } 155 156 int 157 ithread_create(struct ithd **ithread, int vector, int flags, 158 void (*disable)(int), void (*enable)(int), const char *fmt, ...) 159 { 160 struct ithd *ithd; 161 struct proc *p; 162 int error; 163 va_list ap; 164 165 ithd = malloc(sizeof(struct ithd), M_ITHREAD, M_WAITOK | M_ZERO); 166 ithd->it_vector = vector; 167 ithd->it_disable = disable; 168 ithd->it_enable = enable; 169 ithd->it_flags = flags; 170 TAILQ_INIT(&ithd->it_handlers); 171 172 va_start(ap, fmt); 173 vsnprintf(ithd->it_name, sizeof(ithd->it_name), fmt, ap); 174 va_end(ap); 175 176 error = kthread_create(ithread_loop, ithd, &p, RFSTOPPED | RFHIGHPID, 177 ithd->it_name); 178 if (error) { 179 free(ithd, M_ITHREAD); 180 return (error); 181 } 182 p->p_pri.pri_class = PRI_ITHD; 183 p->p_pri.pri_level = PRI_MAX_ITHD; 184 p->p_stat = SWAIT; 185 ithd->it_proc = p; 186 p->p_ithd = ithd; 187 if (ithread != NULL) 188 *ithread = ithd; 189 190 return (0); 191 } 192 193 int 194 ithread_destroy(struct ithd *ithread) 195 { 196 197 if (ithread == NULL || !TAILQ_EMPTY(&ithread->it_handlers)) 198 return (EINVAL); 199 200 mtx_lock_spin(&sched_lock); 201 ithread->it_flags |= IT_DEAD; 202 if (ithread->it_proc->p_stat == SWAIT) { 203 ithread->it_proc->p_stat = SRUN; 204 setrunqueue(ithread->it_proc); 205 } 206 mtx_unlock_spin(&sched_lock); 207 return (0); 208 } 209 210 int 211 ithread_add_handler(struct ithd* ithread, const char *name, 212 driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, 213 void **cookiep) 214 { 215 struct intrhand *ih, *temp_ih; 216 217 if (ithread == NULL || name == NULL || handler == NULL) 218 return (EINVAL); 219 if ((flags & INTR_FAST) !=0) 220 flags |= INTR_EXCL; 221 222 ih = malloc(sizeof(struct intrhand), M_ITHREAD, M_WAITOK | M_ZERO); 223 ih->ih_handler = handler; 224 ih->ih_argument = arg; 225 ih->ih_name = name; 226 ih->ih_ithread = ithread; 227 ih->ih_pri = pri; 228 if (flags & INTR_FAST) 229 ih->ih_flags = IH_FAST | IH_EXCLUSIVE; 230 else if (flags & INTR_EXCL) 231 ih->ih_flags = IH_EXCLUSIVE; 232 if (flags & INTR_MPSAFE) 233 ih->ih_flags |= IH_MPSAFE; 234 if (flags & INTR_ENTROPY) 235 ih->ih_flags |= IH_ENTROPY; 236 237 mtx_lock_spin(&ithread_list_lock); 238 if ((flags & INTR_EXCL) !=0 && !TAILQ_EMPTY(&ithread->it_handlers)) 239 goto fail; 240 if (!TAILQ_EMPTY(&ithread->it_handlers) && 241 (TAILQ_FIRST(&ithread->it_handlers)->ih_flags & IH_EXCLUSIVE) != 0) 242 goto fail; 243 244 TAILQ_FOREACH(temp_ih, &ithread->it_handlers, ih_next) 245 if (temp_ih->ih_pri > ih->ih_pri) 246 break; 247 if (temp_ih == NULL) 248 TAILQ_INSERT_TAIL(&ithread->it_handlers, ih, ih_next); 249 else 250 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 251 ithread_update(ithread); 252 mtx_unlock_spin(&ithread_list_lock); 253 254 if (cookiep != NULL) 255 *cookiep = ih; 256 return (0); 257 258 fail: 259 mtx_unlock_spin(&ithread_list_lock); 260 free(ih, M_ITHREAD); 261 return (EINVAL); 262 } 263 264 int 265 ithread_remove_handler(void *cookie) 266 { 267 struct intrhand *handler = (struct intrhand *)cookie; 268 struct ithd *ithread; 269 #ifdef INVARIANTS 270 struct intrhand *ih; 271 int found; 272 #endif 273 274 if (handler == NULL || (ithread = handler->ih_ithread) == NULL) 275 return (EINVAL); 276 277 mtx_lock_spin(&ithread_list_lock); 278 #ifdef INVARIANTS 279 found = 0; 280 TAILQ_FOREACH(ih, &ithread->it_handlers, ih_next) 281 if (ih == handler) { 282 found++; 283 break; 284 } 285 if (found == 0) { 286 mtx_unlock_spin(&ithread_list_lock); 287 return (EINVAL); 288 } 289 #endif 290 TAILQ_REMOVE(&ithread->it_handlers, handler, ih_next); 291 ithread_update(ithread); 292 mtx_unlock_spin(&ithread_list_lock); 293 294 free(handler, M_ITHREAD); 295 return (0); 296 } 297 298 int 299 swi_add(struct ithd **ithdp, const char *name, driver_intr_t handler, 300 void *arg, int pri, enum intr_type flags, void **cookiep) 301 { 302 struct proc *p; 303 struct ithd *ithd; 304 int error; 305 306 ithd = (ithdp != NULL) ? *ithdp : NULL; 307 308 if (ithd == NULL) { 309 error = ithread_create(&ithd, pri, IT_SOFT, NULL, NULL, 310 "swi%d:", pri); 311 if (error) 312 return (error); 313 314 /* XXX - some hacks are _really_ gross */ 315 p = ithd->it_proc; 316 PROC_LOCK(p); 317 if (pri == SWI_CLOCK) 318 p->p_flag |= P_NOLOAD; 319 PROC_UNLOCK(p); 320 if (ithdp != NULL) 321 *ithdp = ithd; 322 } 323 return (ithread_add_handler(ithd, name, handler, arg, 324 (pri * RQ_PPQ) + PI_SOFT, flags, cookiep)); 325 } 326 327 328 /* 329 * Schedule a heavyweight software interrupt process. 330 */ 331 void 332 swi_sched(void *cookie, int flags) 333 { 334 struct intrhand *ih = (struct intrhand *)cookie; 335 struct ithd *it = ih->ih_ithread; 336 struct proc *p = it->it_proc; 337 338 atomic_add_int(&cnt.v_intr, 1); /* one more global interrupt */ 339 340 CTR3(KTR_INTR, "swi_sched pid %d(%s) need=%d", 341 p->p_pid, p->p_comm, it->it_need); 342 343 /* 344 * Set it_need so that if the thread is already running but close 345 * to done, it will do another go-round. Then get the sched lock 346 * and see if the thread is on whichkqs yet. If not, put it on 347 * there. In any case, kick everyone so that if the new thread 348 * is higher priority than their current thread, it gets run now. 349 */ 350 atomic_store_rel_int(&ih->ih_need, 1); 351 if (!(flags & SWI_DELAY)) { 352 it->it_need = 1; 353 mtx_lock_spin(&sched_lock); 354 if (p->p_stat == SWAIT) { /* not on run queue */ 355 CTR1(KTR_INTR, "swi_sched: setrunqueue %d", p->p_pid); 356 p->p_stat = SRUN; 357 setrunqueue(p); 358 if (!cold && flags & SWI_SWITCH) { 359 if (curproc != PCPU_GET(idleproc)) 360 setrunqueue(curproc); 361 curproc->p_stats->p_ru.ru_nvcsw++; 362 mi_switch(); 363 } else 364 need_resched(); 365 } 366 else { 367 CTR3(KTR_INTR, "swi_sched %d: it_need %d, state %d", 368 p->p_pid, it->it_need, p->p_stat ); 369 } 370 mtx_unlock_spin(&sched_lock); 371 } 372 } 373 374 /* 375 * This is the main code for interrupt threads. 376 */ 377 void 378 ithread_loop(void *arg) 379 { 380 struct ithd *ithd; /* our thread context */ 381 struct intrhand *ih; /* and our interrupt handler chain */ 382 struct proc *p; 383 384 p = curproc; 385 ithd = (struct ithd *)arg; /* point to myself */ 386 KASSERT(ithd->it_proc == p && p->p_ithd == ithd, 387 (__func__ ": ithread and proc linkage out of sync")); 388 389 /* 390 * As long as we have interrupts outstanding, go through the 391 * list of handlers, giving each one a go at it. 392 */ 393 for (;;) { 394 /* 395 * If we are an orphaned thread, then just die. 396 */ 397 if (ithd->it_flags & IT_DEAD) { 398 CTR2(KTR_INTR, __func__ ": pid %d: (%s) exiting", 399 p->p_pid, p->p_comm); 400 p->p_ithd = NULL; 401 mtx_lock(&Giant); 402 free(ithd, M_ITHREAD); 403 kthread_exit(0); 404 } 405 406 CTR3(KTR_INTR, __func__ ": pid %d: (%s) need=%d", 407 p->p_pid, p->p_comm, ithd->it_need); 408 while (ithd->it_need) { 409 /* 410 * Service interrupts. If another interrupt 411 * arrives while we are running, they will set 412 * it_need to denote that we should make 413 * another pass. 414 */ 415 atomic_store_rel_int(&ithd->it_need, 0); 416 TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { 417 if (ithd->it_flags & IT_SOFT && !ih->ih_need) 418 continue; 419 atomic_store_rel_int(&ih->ih_need, 0); 420 CTR5(KTR_INTR, 421 __func__ ": pid %d ih=%p: %p(%p) flg=%x", 422 p->p_pid, (void *)ih, 423 (void *)ih->ih_handler, ih->ih_argument, 424 ih->ih_flags); 425 426 if ((ih->ih_flags & IH_MPSAFE) == 0) 427 mtx_lock(&Giant); 428 ih->ih_handler(ih->ih_argument); 429 if ((ih->ih_flags & IH_MPSAFE) == 0) 430 mtx_unlock(&Giant); 431 } 432 } 433 434 /* 435 * Processed all our interrupts. Now get the sched 436 * lock. This may take a while and it_need may get 437 * set again, so we have to check it again. 438 */ 439 mtx_assert(&Giant, MA_NOTOWNED); 440 mtx_lock_spin(&sched_lock); 441 if (!ithd->it_need) { 442 /* 443 * Should we call this earlier in the loop above? 444 */ 445 if (ithd->it_enable != NULL) 446 ithd->it_enable(ithd->it_vector); 447 p->p_stat = SWAIT; /* we're idle */ 448 CTR1(KTR_INTR, __func__ ": pid %d: done", p->p_pid); 449 mi_switch(); 450 CTR1(KTR_INTR, __func__ ": pid %d: resumed", p->p_pid); 451 } 452 mtx_unlock_spin(&sched_lock); 453 } 454 } 455 456 /* 457 * Initialize mutex used to protect ithread handler lists. 458 */ 459 static void 460 ithread_init(void *dummy) 461 { 462 463 mtx_init(&ithread_list_lock, "ithread list lock", MTX_SPIN); 464 } 465 SYSINIT(ithread_init, SI_SUB_INTR, SI_ORDER_FIRST, ithread_init, NULL); 466 467 /* 468 * Start standard software interrupt threads 469 */ 470 static void 471 start_softintr(void *dummy) 472 { 473 474 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, 0, &net_ih) || 475 swi_add(&clk_ithd, "clock", softclock, NULL, SWI_CLOCK, 476 INTR_MPSAFE, &softclock_ih) || 477 swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, 0, &vm_ih)) 478 panic("died while creating standard software ithreads"); 479 } 480 SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL) 481 482 void 483 legacy_setsoftnet(void) 484 { 485 swi_sched(net_ih, SWI_NOSWITCH); 486 } 487 488 /* 489 * XXX: This should really be in the network code somewhere and installed 490 * via a SI_SUB_SOFINTR, SI_ORDER_MIDDLE sysinit. 491 */ 492 void (*netisrs[32]) __P((void)); 493 u_int netisr; 494 495 int 496 register_netisr(num, handler) 497 int num; 498 netisr_t *handler; 499 { 500 501 if (num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs)) ) { 502 printf("register_netisr: bad isr number: %d\n", num); 503 return (EINVAL); 504 } 505 netisrs[num] = handler; 506 return (0); 507 } 508 509 int 510 unregister_netisr(num) 511 int num; 512 { 513 514 if (num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs)) ) { 515 printf("unregister_netisr: bad isr number: %d\n", num); 516 return (EINVAL); 517 } 518 netisrs[num] = NULL; 519 return (0); 520 } 521 522 static void 523 swi_net(void *dummy) 524 { 525 u_int bits; 526 int i; 527 528 bits = atomic_readandclear_int(&netisr); 529 while ((i = ffs(bits)) != 0) { 530 i--; 531 if (netisrs[i] != NULL) 532 netisrs[i](); 533 else 534 printf("swi_net: unregistered isr number: %d.\n", i); 535 bits &= ~(1 << i); 536 } 537 } 538