1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2018 Western Digital Corporation. All rights reserved. 25 */ 26 27 #include <sys/cpuvar.h> 28 #include <sys/cpu_event.h> 29 #include <sys/param.h> 30 #include <sys/cmn_err.h> 31 #include <sys/t_lock.h> 32 #include <sys/kmem.h> 33 #include <sys/machlock.h> 34 #include <sys/systm.h> 35 #include <sys/archsystm.h> 36 #include <sys/atomic.h> 37 #include <sys/sdt.h> 38 #include <sys/processor.h> 39 #include <sys/time.h> 40 #include <sys/psm.h> 41 #include <sys/smp_impldefs.h> 42 #include <sys/cram.h> 43 #include <sys/apic.h> 44 #include <sys/pit.h> 45 #include <sys/ddi.h> 46 #include <sys/sunddi.h> 47 #include <sys/ddi_impldefs.h> 48 #include <sys/pci.h> 49 #include <sys/promif.h> 50 #include <sys/x86_archext.h> 51 #include <sys/cpc_impl.h> 52 #include <sys/uadmin.h> 53 #include <sys/panic.h> 54 #include <sys/debug.h> 55 #include <sys/trap.h> 56 #include <sys/machsystm.h> 57 #include <sys/sysmacros.h> 58 #include <sys/rm_platter.h> 59 #include <sys/privregs.h> 60 #include <sys/note.h> 61 #include <sys/pci_intr_lib.h> 62 #include <sys/spl.h> 63 #include <sys/clock.h> 64 #include <sys/dditypes.h> 65 #include <sys/sunddi.h> 66 #include <sys/x_call.h> 67 #include <sys/reboot.h> 68 #include <vm/hat_i86.h> 69 #include <sys/stack.h> 70 #include <sys/apix.h> 71 72 static void apix_post_hardint(int); 73 74 /* 75 * Insert an vector into the tail of the interrupt pending list 76 */ 77 static __inline__ void 78 apix_insert_pending_av(apix_impl_t *apixp, struct autovec *avp, int ipl) 79 { 80 struct autovec **head = apixp->x_intr_head; 81 struct autovec **tail = apixp->x_intr_tail; 82 83 avp->av_ipl_link = NULL; 84 if (tail[ipl] == NULL) { 85 head[ipl] = tail[ipl] = avp; 86 return; 87 } 88 89 tail[ipl]->av_ipl_link = avp; 90 tail[ipl] = avp; 91 } 92 93 /* 94 * Remove and return an vector from the head of hardware interrupt 95 * pending list. 96 */ 97 static __inline__ struct autovec * 98 apix_remove_pending_av(apix_impl_t *apixp, int ipl) 99 { 100 struct cpu *cpu = CPU; 101 struct autovec **head = apixp->x_intr_head; 102 struct autovec **tail = apixp->x_intr_tail; 103 struct autovec *avp = head[ipl]; 104 105 if (avp == NULL) 106 return (NULL); 107 108 if (avp->av_vector != NULL && avp->av_prilevel < cpu->cpu_base_spl) { 109 /* 110 * If there is blocked higher level interrupts, return 111 * NULL to quit handling of current IPL level. 112 */ 113 apixp->x_intr_pending |= (1 << avp->av_prilevel); 114 return (NULL); 115 } 116 117 avp->av_flags &= ~AV_PENTRY_PEND; 118 avp->av_flags |= AV_PENTRY_ONPROC; 119 head[ipl] = avp->av_ipl_link; 120 avp->av_ipl_link = NULL; 121 122 if (head[ipl] == NULL) 123 tail[ipl] = NULL; 124 125 return (avp); 126 } 127 128 /* 129 * add_pending_hardint: 130 * 131 * Add hardware interrupts to the interrupt pending list. 132 */ 133 static void 134 apix_add_pending_hardint(int vector) 135 { 136 uint32_t cpuid = psm_get_cpu_id(); 137 apix_impl_t *apixp = apixs[cpuid]; 138 apix_vector_t *vecp = apixp->x_vectbl[vector]; 139 struct autovec *p, *prevp = NULL; 140 int ipl; 141 142 /* 143 * The MSI interrupt not supporting per-vector masking could 144 * be triggered on a false vector as a result of rebinding 145 * operation cannot programme MSI address & data atomically. 146 * Add ISR of this interrupt to the pending list for such 147 * suspicious interrupt. 148 */ 149 APIX_DO_FAKE_INTR(cpuid, vector); 150 if (vecp == NULL) 151 return; 152 153 for (p = vecp->v_autovect; p != NULL; p = p->av_link) { 154 if (p->av_vector == NULL) 155 continue; /* skip freed entry */ 156 157 ipl = p->av_prilevel; 158 prevp = p; 159 160 /* set pending at specified priority level */ 161 apixp->x_intr_pending |= (1 << ipl); 162 163 if (p->av_flags & AV_PENTRY_PEND) 164 continue; /* already in the pending list */ 165 p->av_flags |= AV_PENTRY_PEND; 166 167 /* insert into pending list by it original IPL */ 168 apix_insert_pending_av(apixp, p, ipl); 169 } 170 171 /* last one of the linked list */ 172 if (prevp && ((prevp->av_flags & AV_PENTRY_LEVEL) != 0)) 173 prevp->av_flags |= (vector & AV_PENTRY_VECTMASK); 174 } 175 176 /* 177 * Walk pending hardware interrupts at given priority level, invoking 178 * each interrupt handler as we go. 179 */ 180 extern uint64_t intr_get_time(void); 181 182 static void 183 apix_dispatch_pending_autovect(uint_t ipl) 184 { 185 uint32_t cpuid = psm_get_cpu_id(); 186 apix_impl_t *apixp = apixs[cpuid]; 187 struct autovec *av; 188 189 while ((av = apix_remove_pending_av(apixp, ipl)) != NULL) { 190 uint_t r; 191 uint_t (*intr)() = av->av_vector; 192 caddr_t arg1 = av->av_intarg1; 193 caddr_t arg2 = av->av_intarg2; 194 dev_info_t *dip = av->av_dip; 195 uchar_t vector = av->av_flags & AV_PENTRY_VECTMASK; 196 197 if (intr == NULL) 198 continue; 199 200 /* Don't enable interrupts during x-calls */ 201 if (ipl != XC_HI_PIL) 202 sti(); 203 204 DTRACE_PROBE4(interrupt__start, dev_info_t *, dip, 205 void *, intr, caddr_t, arg1, caddr_t, arg2); 206 r = (*intr)(arg1, arg2); 207 DTRACE_PROBE4(interrupt__complete, dev_info_t *, dip, 208 void *, intr, caddr_t, arg1, uint_t, r); 209 210 if (av->av_ticksp && av->av_prilevel <= LOCK_LEVEL) 211 atomic_add_64(av->av_ticksp, intr_get_time()); 212 213 cli(); 214 215 if (vector) { 216 if ((av->av_flags & AV_PENTRY_PEND) == 0) 217 av->av_flags &= ~AV_PENTRY_VECTMASK; 218 219 apix_post_hardint(vector); 220 } 221 222 /* mark it as idle */ 223 av->av_flags &= ~AV_PENTRY_ONPROC; 224 } 225 } 226 227 static caddr_t 228 apix_do_softint_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, 229 caddr_t stackptr) 230 { 231 kthread_t *t, *volatile it; 232 struct machcpu *mcpu = &cpu->cpu_m; 233 hrtime_t now; 234 235 UNREFERENCED_1PARAMETER(oldpil); 236 ASSERT(pil > mcpu->mcpu_pri && pil > cpu->cpu_base_spl); 237 238 atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, ~(1 << pil)); 239 240 mcpu->mcpu_pri = pil; 241 242 now = tsc_read(); 243 244 /* 245 * Get set to run interrupt thread. 246 * There should always be an interrupt thread since we 247 * allocate one for each level on the CPU. 248 */ 249 it = cpu->cpu_intr_thread; 250 ASSERT(it != NULL); 251 cpu->cpu_intr_thread = it->t_link; 252 253 /* t_intr_start could be zero due to cpu_intr_swtch_enter. */ 254 t = cpu->cpu_thread; 255 if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { 256 hrtime_t intrtime = now - t->t_intr_start; 257 mcpu->intrstat[pil][0] += intrtime; 258 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 259 t->t_intr_start = 0; 260 } 261 262 /* 263 * Note that the code in kcpc_overflow_intr -relies- on the 264 * ordering of events here - in particular that t->t_lwp of 265 * the interrupt thread is set to the pinned thread *before* 266 * curthread is changed. 267 */ 268 it->t_lwp = t->t_lwp; 269 it->t_state = TS_ONPROC; 270 271 /* 272 * Push interrupted thread onto list from new thread. 273 * Set the new thread as the current one. 274 * Set interrupted thread's T_SP because if it is the idle thread, 275 * resume() may use that stack between threads. 276 */ 277 278 ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); 279 t->t_sp = (uintptr_t)stackptr; 280 281 it->t_intr = t; 282 cpu->cpu_thread = it; 283 284 /* 285 * Set bit for this pil in CPU's interrupt active bitmask. 286 */ 287 ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 288 cpu->cpu_intr_actv |= (1 << pil); 289 290 /* 291 * Initialize thread priority level from intr_pri 292 */ 293 it->t_pil = (uchar_t)pil; 294 it->t_pri = (pri_t)pil + intr_pri; 295 it->t_intr_start = now; 296 297 return (it->t_stk); 298 } 299 300 static void 301 apix_do_softint_epilog(struct cpu *cpu, uint_t oldpil) 302 { 303 struct machcpu *mcpu = &cpu->cpu_m; 304 kthread_t *t, *it; 305 uint_t pil, basespl; 306 hrtime_t intrtime; 307 hrtime_t now = tsc_read(); 308 309 it = cpu->cpu_thread; 310 pil = it->t_pil; 311 312 cpu->cpu_stats.sys.intr[pil - 1]++; 313 314 ASSERT(cpu->cpu_intr_actv & (1 << pil)); 315 cpu->cpu_intr_actv &= ~(1 << pil); 316 317 intrtime = now - it->t_intr_start; 318 mcpu->intrstat[pil][0] += intrtime; 319 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 320 321 /* 322 * If there is still an interrupted thread underneath this one 323 * then the interrupt was never blocked and the return is 324 * fairly simple. Otherwise it isn't. 325 */ 326 if ((t = it->t_intr) == NULL) { 327 /* 328 * Put thread back on the interrupt thread list. 329 * This was an interrupt thread, so set CPU's base SPL. 330 */ 331 set_base_spl(); 332 /* mcpu->mcpu_pri = cpu->cpu_base_spl; */ 333 334 /* 335 * If there are pending interrupts, send a softint to 336 * re-enter apix_do_interrupt() and get them processed. 337 */ 338 if (apixs[cpu->cpu_id]->x_intr_pending) 339 siron(); 340 341 it->t_state = TS_FREE; 342 it->t_link = cpu->cpu_intr_thread; 343 cpu->cpu_intr_thread = it; 344 (void) splhigh(); 345 sti(); 346 swtch(); 347 /*NOTREACHED*/ 348 panic("dosoftint_epilog: swtch returned"); 349 } 350 it->t_link = cpu->cpu_intr_thread; 351 cpu->cpu_intr_thread = it; 352 it->t_state = TS_FREE; 353 cpu->cpu_thread = t; 354 if (t->t_flag & T_INTR_THREAD) 355 t->t_intr_start = now; 356 basespl = cpu->cpu_base_spl; 357 pil = MAX(oldpil, basespl); 358 mcpu->mcpu_pri = pil; 359 } 360 361 /* 362 * Dispatch a soft interrupt 363 */ 364 static void 365 apix_dispatch_softint(uint_t oldpil, uint_t arg2) 366 { 367 struct cpu *cpu = CPU; 368 369 UNREFERENCED_1PARAMETER(arg2); 370 371 sti(); 372 av_dispatch_softvect((int)cpu->cpu_thread->t_pil); 373 cli(); 374 375 /* 376 * Must run softint_epilog() on the interrupt thread stack, since 377 * there may not be a return from it if the interrupt thread blocked. 378 */ 379 apix_do_softint_epilog(cpu, oldpil); 380 } 381 382 /* 383 * Deliver any softints the current interrupt priority allows. 384 * Called with interrupts disabled. 385 */ 386 int 387 apix_do_softint(struct regs *regs) 388 { 389 struct cpu *cpu = CPU; 390 int oldipl; 391 int newipl; 392 volatile uint16_t pending; 393 caddr_t newsp; 394 395 while ((pending = cpu->cpu_softinfo.st_pending) != 0) { 396 newipl = bsrw_insn(pending); 397 oldipl = cpu->cpu_pri; 398 if (newipl <= oldipl || newipl <= cpu->cpu_base_spl) 399 return (-1); 400 401 newsp = apix_do_softint_prolog(cpu, newipl, oldipl, 402 (caddr_t)regs); 403 ASSERT(newsp != NULL); 404 switch_sp_and_call(newsp, apix_dispatch_softint, oldipl, 0); 405 } 406 407 return (0); 408 } 409 410 static int 411 apix_hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, 412 struct regs *rp) 413 { 414 struct machcpu *mcpu = &cpu->cpu_m; 415 hrtime_t intrtime; 416 hrtime_t now = tsc_read(); 417 apix_impl_t *apixp = apixs[cpu->cpu_id]; 418 uint_t mask; 419 420 ASSERT(pil > mcpu->mcpu_pri && pil > cpu->cpu_base_spl); 421 422 if (pil == CBE_HIGH_PIL) { /* 14 */ 423 cpu->cpu_profile_pil = oldpil; 424 if (USERMODE(rp->r_cs)) { 425 cpu->cpu_profile_pc = 0; 426 cpu->cpu_profile_upc = rp->r_pc; 427 cpu->cpu_cpcprofile_pc = 0; 428 cpu->cpu_cpcprofile_upc = rp->r_pc; 429 } else { 430 cpu->cpu_profile_pc = rp->r_pc; 431 cpu->cpu_profile_upc = 0; 432 cpu->cpu_cpcprofile_pc = rp->r_pc; 433 cpu->cpu_cpcprofile_upc = 0; 434 } 435 } 436 437 mcpu->mcpu_pri = pil; 438 439 mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; 440 if (mask != 0) { 441 int nestpil; 442 443 /* 444 * We have interrupted another high-level interrupt. 445 * Load starting timestamp, compute interval, update 446 * cumulative counter. 447 */ 448 nestpil = bsrw_insn((uint16_t)mask); 449 intrtime = now - 450 mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)]; 451 mcpu->intrstat[nestpil][0] += intrtime; 452 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 453 } else { 454 kthread_t *t = cpu->cpu_thread; 455 456 /* 457 * See if we are interrupting a low-level interrupt thread. 458 * If so, account for its time slice only if its time stamp 459 * is non-zero. 460 */ 461 if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) { 462 intrtime = now - t->t_intr_start; 463 mcpu->intrstat[t->t_pil][0] += intrtime; 464 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 465 t->t_intr_start = 0; 466 } 467 } 468 469 /* store starting timestamp in CPu structure for this IPL */ 470 mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now; 471 472 if (pil == 15) { 473 /* 474 * To support reentrant level 15 interrupts, we maintain a 475 * recursion count in the top half of cpu_intr_actv. Only 476 * when this count hits zero do we clear the PIL 15 bit from 477 * the lower half of cpu_intr_actv. 478 */ 479 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; 480 (*refcntp)++; 481 } 482 483 cpu->cpu_intr_actv |= (1 << pil); 484 /* clear pending ipl level bit */ 485 apixp->x_intr_pending &= ~(1 << pil); 486 487 return (mask); 488 } 489 490 static int 491 apix_hilevel_intr_epilog(struct cpu *cpu, uint_t oldpil) 492 { 493 struct machcpu *mcpu = &cpu->cpu_m; 494 uint_t mask, pil; 495 hrtime_t intrtime; 496 hrtime_t now = tsc_read(); 497 498 pil = mcpu->mcpu_pri; 499 cpu->cpu_stats.sys.intr[pil - 1]++; 500 501 ASSERT(cpu->cpu_intr_actv & (1 << pil)); 502 503 if (pil == 15) { 504 /* 505 * To support reentrant level 15 interrupts, we maintain a 506 * recursion count in the top half of cpu_intr_actv. Only 507 * when this count hits zero do we clear the PIL 15 bit from 508 * the lower half of cpu_intr_actv. 509 */ 510 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; 511 512 ASSERT(*refcntp > 0); 513 514 if (--(*refcntp) == 0) 515 cpu->cpu_intr_actv &= ~(1 << pil); 516 } else { 517 cpu->cpu_intr_actv &= ~(1 << pil); 518 } 519 520 ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0); 521 522 intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)]; 523 mcpu->intrstat[pil][0] += intrtime; 524 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 525 526 /* 527 * Check for lower-pil nested high-level interrupt beneath 528 * current one. If so, place a starting timestamp in its 529 * pil_high_start entry. 530 */ 531 mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; 532 if (mask != 0) { 533 int nestpil; 534 535 /* 536 * find PIL of nested interrupt 537 */ 538 nestpil = bsrw_insn((uint16_t)mask); 539 ASSERT(nestpil < pil); 540 mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now; 541 /* 542 * (Another high-level interrupt is active below this one, 543 * so there is no need to check for an interrupt 544 * thread. That will be done by the lowest priority 545 * high-level interrupt active.) 546 */ 547 } else { 548 /* 549 * Check to see if there is a low-level interrupt active. 550 * If so, place a starting timestamp in the thread 551 * structure. 552 */ 553 kthread_t *t = cpu->cpu_thread; 554 555 if (t->t_flag & T_INTR_THREAD) 556 t->t_intr_start = now; 557 } 558 559 mcpu->mcpu_pri = oldpil; 560 if (pil < CBE_HIGH_PIL) 561 (void) (*setlvlx)(oldpil, 0); 562 563 return (mask); 564 } 565 566 /* 567 * Dispatch a hilevel interrupt (one above LOCK_LEVEL) 568 */ 569 static void 570 apix_dispatch_pending_hilevel(uint_t ipl, uint_t arg2) 571 { 572 UNREFERENCED_1PARAMETER(arg2); 573 574 apix_dispatch_pending_autovect(ipl); 575 } 576 577 static __inline__ int 578 apix_do_pending_hilevel(struct cpu *cpu, struct regs *rp) 579 { 580 volatile uint16_t pending; 581 uint_t newipl, oldipl; 582 caddr_t newsp; 583 584 while ((pending = HILEVEL_PENDING(cpu)) != 0) { 585 newipl = bsrw_insn(pending); 586 ASSERT(newipl > LOCK_LEVEL && newipl > cpu->cpu_base_spl); 587 oldipl = cpu->cpu_pri; 588 if (newipl <= oldipl) 589 return (-1); 590 591 /* 592 * High priority interrupts run on this cpu's interrupt stack. 593 */ 594 if (apix_hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) { 595 newsp = cpu->cpu_intr_stack; 596 switch_sp_and_call(newsp, apix_dispatch_pending_hilevel, 597 newipl, 0); 598 } else { /* already on the interrupt stack */ 599 apix_dispatch_pending_hilevel(newipl, 0); 600 } 601 (void) apix_hilevel_intr_epilog(cpu, oldipl); 602 } 603 604 return (0); 605 } 606 607 /* 608 * Get an interrupt thread and swith to it. It's called from do_interrupt(). 609 * The IF flag is cleared and thus all maskable interrupts are blocked at 610 * the time of calling. 611 */ 612 static caddr_t 613 apix_intr_thread_prolog(struct cpu *cpu, uint_t pil, caddr_t stackptr) 614 { 615 apix_impl_t *apixp = apixs[cpu->cpu_id]; 616 struct machcpu *mcpu = &cpu->cpu_m; 617 hrtime_t now = tsc_read(); 618 kthread_t *t, *volatile it; 619 620 ASSERT(pil > mcpu->mcpu_pri && pil > cpu->cpu_base_spl); 621 622 apixp->x_intr_pending &= ~(1 << pil); 623 ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 624 cpu->cpu_intr_actv |= (1 << pil); 625 mcpu->mcpu_pri = pil; 626 627 /* 628 * Get set to run interrupt thread. 629 * There should always be an interrupt thread since we 630 * allocate one for each level on the CPU. 631 */ 632 /* t_intr_start could be zero due to cpu_intr_swtch_enter. */ 633 t = cpu->cpu_thread; 634 if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { 635 hrtime_t intrtime = now - t->t_intr_start; 636 mcpu->intrstat[pil][0] += intrtime; 637 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 638 t->t_intr_start = 0; 639 } 640 641 /* 642 * Push interrupted thread onto list from new thread. 643 * Set the new thread as the current one. 644 * Set interrupted thread's T_SP because if it is the idle thread, 645 * resume() may use that stack between threads. 646 */ 647 648 ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); 649 650 t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */ 651 652 /* 653 * Note that the code in kcpc_overflow_intr -relies- on the 654 * ordering of events here - in particular that t->t_lwp of 655 * the interrupt thread is set to the pinned thread *before* 656 * curthread is changed. 657 */ 658 it = cpu->cpu_intr_thread; 659 cpu->cpu_intr_thread = it->t_link; 660 it->t_intr = t; 661 it->t_lwp = t->t_lwp; 662 663 /* 664 * (threads on the interrupt thread free list could have state 665 * preset to TS_ONPROC, but it helps in debugging if 666 * they're TS_FREE.) 667 */ 668 it->t_state = TS_ONPROC; 669 670 cpu->cpu_thread = it; 671 672 /* 673 * Initialize thread priority level from intr_pri 674 */ 675 it->t_pil = (uchar_t)pil; 676 it->t_pri = (pri_t)pil + intr_pri; 677 it->t_intr_start = now; 678 679 return (it->t_stk); 680 } 681 682 static void 683 apix_intr_thread_epilog(struct cpu *cpu, uint_t oldpil) 684 { 685 struct machcpu *mcpu = &cpu->cpu_m; 686 kthread_t *t, *it = cpu->cpu_thread; 687 uint_t pil, basespl; 688 hrtime_t intrtime; 689 hrtime_t now = tsc_read(); 690 691 pil = it->t_pil; 692 cpu->cpu_stats.sys.intr[pil - 1]++; 693 694 ASSERT(cpu->cpu_intr_actv & (1 << pil)); 695 cpu->cpu_intr_actv &= ~(1 << pil); 696 697 ASSERT(it->t_intr_start != 0); 698 intrtime = now - it->t_intr_start; 699 mcpu->intrstat[pil][0] += intrtime; 700 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 701 702 /* 703 * If there is still an interrupted thread underneath this one 704 * then the interrupt was never blocked and the return is 705 * fairly simple. Otherwise it isn't. 706 */ 707 if ((t = it->t_intr) == NULL) { 708 /* 709 * The interrupted thread is no longer pinned underneath 710 * the interrupt thread. This means the interrupt must 711 * have blocked, and the interrupted thread has been 712 * unpinned, and has probably been running around the 713 * system for a while. 714 * 715 * Since there is no longer a thread under this one, put 716 * this interrupt thread back on the CPU's free list and 717 * resume the idle thread which will dispatch the next 718 * thread to run. 719 */ 720 cpu->cpu_stats.sys.intrblk++; 721 722 /* 723 * Put thread back on the interrupt thread list. 724 * This was an interrupt thread, so set CPU's base SPL. 725 */ 726 set_base_spl(); 727 basespl = cpu->cpu_base_spl; 728 mcpu->mcpu_pri = basespl; 729 (*setlvlx)(basespl, 0); 730 731 /* 732 * If there are pending interrupts, send a softint to 733 * re-enter apix_do_interrupt() and get them processed. 734 */ 735 if (apixs[cpu->cpu_id]->x_intr_pending) 736 siron(); 737 738 it->t_state = TS_FREE; 739 /* 740 * Return interrupt thread to pool 741 */ 742 it->t_link = cpu->cpu_intr_thread; 743 cpu->cpu_intr_thread = it; 744 745 (void) splhigh(); 746 sti(); 747 swtch(); 748 /*NOTREACHED*/ 749 panic("dosoftint_epilog: swtch returned"); 750 } 751 752 /* 753 * Return interrupt thread to the pool 754 */ 755 it->t_link = cpu->cpu_intr_thread; 756 cpu->cpu_intr_thread = it; 757 it->t_state = TS_FREE; 758 759 cpu->cpu_thread = t; 760 if (t->t_flag & T_INTR_THREAD) 761 t->t_intr_start = now; 762 basespl = cpu->cpu_base_spl; 763 mcpu->mcpu_pri = MAX(oldpil, basespl); 764 (*setlvlx)(mcpu->mcpu_pri, 0); 765 } 766 767 768 static void 769 apix_dispatch_pending_hardint(uint_t oldpil, uint_t arg2) 770 { 771 struct cpu *cpu = CPU; 772 773 UNREFERENCED_1PARAMETER(arg2); 774 775 apix_dispatch_pending_autovect((int)cpu->cpu_thread->t_pil); 776 777 /* 778 * Must run intr_thread_epilog() on the interrupt thread stack, since 779 * there may not be a return from it if the interrupt thread blocked. 780 */ 781 apix_intr_thread_epilog(cpu, oldpil); 782 } 783 784 static __inline__ int 785 apix_do_pending_hardint(struct cpu *cpu, struct regs *rp) 786 { 787 volatile uint16_t pending; 788 uint_t newipl, oldipl; 789 caddr_t newsp; 790 791 while ((pending = LOWLEVEL_PENDING(cpu)) != 0) { 792 newipl = bsrw_insn(pending); 793 ASSERT(newipl <= LOCK_LEVEL); 794 oldipl = cpu->cpu_pri; 795 if (newipl <= oldipl || newipl <= cpu->cpu_base_spl) 796 return (-1); 797 798 /* 799 * Run this interrupt in a separate thread. 800 */ 801 newsp = apix_intr_thread_prolog(cpu, newipl, (caddr_t)rp); 802 ASSERT(newsp != NULL); 803 switch_sp_and_call(newsp, apix_dispatch_pending_hardint, 804 oldipl, 0); 805 } 806 807 return (0); 808 } 809 810 /* 811 * Unmask level triggered interrupts 812 */ 813 static void 814 apix_post_hardint(int vector) 815 { 816 apix_vector_t *vecp = xv_vector(psm_get_cpu_id(), vector); 817 int irqno = vecp->v_inum; 818 819 ASSERT(vecp->v_type == APIX_TYPE_FIXED && apic_level_intr[irqno]); 820 821 apix_level_intr_post_dispatch(irqno); 822 } 823 824 static void 825 apix_dispatch_by_vector(uint_t vector) 826 { 827 struct cpu *cpu = CPU; 828 apix_vector_t *vecp = xv_vector(cpu->cpu_id, vector); 829 struct autovec *avp; 830 uint_t r, (*intr)(); 831 caddr_t arg1, arg2; 832 dev_info_t *dip; 833 834 if (vecp == NULL || 835 (avp = vecp->v_autovect) == NULL || avp->av_vector == NULL) 836 return; 837 838 avp->av_flags |= AV_PENTRY_ONPROC; 839 intr = avp->av_vector; 840 arg1 = avp->av_intarg1; 841 arg2 = avp->av_intarg2; 842 dip = avp->av_dip; 843 844 if (avp->av_prilevel != XC_HI_PIL) 845 sti(); 846 847 DTRACE_PROBE4(interrupt__start, dev_info_t *, dip, 848 void *, intr, caddr_t, arg1, caddr_t, arg2); 849 r = (*intr)(arg1, arg2); 850 DTRACE_PROBE4(interrupt__complete, dev_info_t *, dip, 851 void *, intr, caddr_t, arg1, uint_t, r); 852 853 cli(); 854 avp->av_flags &= ~AV_PENTRY_ONPROC; 855 } 856 857 858 static void 859 apix_dispatch_hilevel(uint_t vector, uint_t arg2) 860 { 861 UNREFERENCED_1PARAMETER(arg2); 862 863 apix_dispatch_by_vector(vector); 864 } 865 866 static void 867 apix_dispatch_lowlevel(uint_t vector, uint_t oldipl) 868 { 869 struct cpu *cpu = CPU; 870 871 apix_dispatch_by_vector(vector); 872 873 /* 874 * Must run intr_thread_epilog() on the interrupt thread stack, since 875 * there may not be a return from it if the interrupt thread blocked. 876 */ 877 apix_intr_thread_epilog(cpu, oldipl); 878 } 879 880 /* 881 * Interrupt service routine, called with interrupts disabled. 882 */ 883 void 884 apix_do_interrupt(struct regs *rp, trap_trace_rec_t *ttp) 885 { 886 struct cpu *cpu = CPU; 887 int vector = rp->r_trapno, newipl, oldipl = cpu->cpu_pri, ret; 888 apix_vector_t *vecp = NULL; 889 890 #ifdef TRAPTRACE 891 ttp->ttr_marker = TT_INTERRUPT; 892 ttp->ttr_cpuid = cpu->cpu_id; 893 ttp->ttr_ipl = 0xff; 894 ttp->ttr_pri = (uchar_t)oldipl; 895 ttp->ttr_spl = cpu->cpu_base_spl; 896 ttp->ttr_vector = 0xff; 897 #endif /* TRAPTRACE */ 898 899 cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR); 900 901 ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp; 902 903 /* 904 * If it's a softint go do it now. 905 */ 906 if (rp->r_trapno == T_SOFTINT) { 907 /* 908 * It might be the case that when an interrupt is triggered, 909 * the spl is raised to high by splhigh(). Later when do_splx() 910 * is called to restore the spl, both hardware and software 911 * interrupt pending flags are check and an SOFTINT is faked 912 * accordingly. 913 */ 914 (void) apix_do_pending_hilevel(cpu, rp); 915 (void) apix_do_pending_hardint(cpu, rp); 916 (void) apix_do_softint(rp); 917 ASSERT(!interrupts_enabled()); 918 #ifdef TRAPTRACE 919 ttp->ttr_vector = T_SOFTINT; 920 #endif 921 /* 922 * We need to check again for pending interrupts that may have 923 * arrived while the softint was running. 924 */ 925 goto do_pending; 926 } 927 928 /* 929 * Send EOI to local APIC 930 */ 931 newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno); 932 #ifdef TRAPTRACE 933 ttp->ttr_ipl = (uchar_t)newipl; 934 #endif /* TRAPTRACE */ 935 936 /* 937 * Bail if it is a spurious interrupt 938 */ 939 if (newipl == -1) 940 return; 941 942 vector = rp->r_trapno; 943 vecp = xv_vector(cpu->cpu_id, vector); 944 #ifdef TRAPTRACE 945 ttp->ttr_vector = (short)vector; 946 #endif /* TRAPTRACE */ 947 948 /* 949 * Direct dispatch for IPI, MSI, MSI-X 950 */ 951 if (vecp && vecp->v_type != APIX_TYPE_FIXED && 952 newipl > MAX(oldipl, cpu->cpu_base_spl)) { 953 caddr_t newsp; 954 955 if (INTR_PENDING(apixs[cpu->cpu_id], newipl)) { 956 /* 957 * There are already vectors pending at newipl, 958 * queue this one and fall through to process 959 * all pending. 960 */ 961 apix_add_pending_hardint(vector); 962 } else if (newipl > LOCK_LEVEL) { 963 if (apix_hilevel_intr_prolog(cpu, newipl, oldipl, rp) 964 == 0) { 965 newsp = cpu->cpu_intr_stack; 966 switch_sp_and_call(newsp, apix_dispatch_hilevel, 967 vector, 0); 968 } else { 969 apix_dispatch_hilevel(vector, 0); 970 } 971 (void) apix_hilevel_intr_epilog(cpu, oldipl); 972 } else { 973 newsp = apix_intr_thread_prolog(cpu, newipl, 974 (caddr_t)rp); 975 switch_sp_and_call(newsp, apix_dispatch_lowlevel, 976 vector, oldipl); 977 } 978 } else { 979 /* Add to per-pil pending queue */ 980 apix_add_pending_hardint(vector); 981 if (newipl <= MAX(oldipl, cpu->cpu_base_spl) || 982 !apixs[cpu->cpu_id]->x_intr_pending) 983 return; 984 } 985 986 do_pending: 987 if (apix_do_pending_hilevel(cpu, rp) < 0) 988 return; 989 990 do { 991 ret = apix_do_pending_hardint(cpu, rp); 992 993 /* 994 * Deliver any pending soft interrupts. 995 */ 996 (void) apix_do_softint(rp); 997 } while (!ret && LOWLEVEL_PENDING(cpu)); 998 } 999