17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5100b72f4Sandrei * Common Development and Distribution License (the "License"). 6100b72f4Sandrei * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21843e1988Sjohnlev 227c478bd9Sstevel@tonic-gate /* 23b9e93c10SJonathan Haslam * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 28*fb2caebeSRandy Fishel #include <sys/cpu_event.h> 297c478bd9Sstevel@tonic-gate #include <sys/regset.h> 307c478bd9Sstevel@tonic-gate #include <sys/psw.h> 317c478bd9Sstevel@tonic-gate #include <sys/types.h> 327c478bd9Sstevel@tonic-gate #include <sys/thread.h> 337c478bd9Sstevel@tonic-gate #include <sys/systm.h> 347c478bd9Sstevel@tonic-gate #include <sys/segments.h> 357c478bd9Sstevel@tonic-gate #include <sys/pcb.h> 367c478bd9Sstevel@tonic-gate #include <sys/trap.h> 377c478bd9Sstevel@tonic-gate #include <sys/ftrace.h> 387c478bd9Sstevel@tonic-gate #include <sys/traptrace.h> 397c478bd9Sstevel@tonic-gate #include <sys/clock.h> 407c478bd9Sstevel@tonic-gate #include <sys/panic.h> 417c478bd9Sstevel@tonic-gate #include <sys/disp.h> 427c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 437c478bd9Sstevel@tonic-gate #include <sys/stack.h> 447c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 457c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 467c478bd9Sstevel@tonic-gate #include <sys/kstat.h> 477c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h> 487c478bd9Sstevel@tonic-gate #include <sys/pool_pset.h> 497c478bd9Sstevel@tonic-gate #include <sys/zone.h> 507c478bd9Sstevel@tonic-gate #include <sys/bitmap.h> 51ae115bc7Smrj #include <sys/archsystm.h> 52ae115bc7Smrj #include <sys/machsystm.h> 53ae115bc7Smrj #include <sys/ontrap.h> 54ae115bc7Smrj #include <sys/x86_archext.h> 55ae115bc7Smrj #include <sys/promif.h> 5695c0a3c8Sjosephb #include <vm/hat_i86.h> 57843e1988Sjohnlev #if defined(__xpv) 58843e1988Sjohnlev #include <sys/hypervisor.h> 59843e1988Sjohnlev #endif 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate 62843e1988Sjohnlev #if defined(__xpv) && defined(DEBUG) 63843e1988Sjohnlev 64843e1988Sjohnlev /* 65843e1988Sjohnlev * This panic message is intended as an aid to interrupt debugging. 66843e1988Sjohnlev * 67843e1988Sjohnlev * The associated assertion tests the condition of enabling 68843e1988Sjohnlev * events when events are already enabled. The implication 69843e1988Sjohnlev * being that whatever code the programmer thought was 70843e1988Sjohnlev * protected by having events disabled until the second 71843e1988Sjohnlev * enable happened really wasn't protected at all .. 72843e1988Sjohnlev */ 73843e1988Sjohnlev 74843e1988Sjohnlev int stistipanic = 1; /* controls the debug panic check */ 75843e1988Sjohnlev const char *stistimsg = "stisti"; 76843e1988Sjohnlev ulong_t laststi[NCPU]; 77843e1988Sjohnlev 78843e1988Sjohnlev /* 79843e1988Sjohnlev * This variable tracks the last place events were disabled on each cpu 80*fb2caebeSRandy Fishel * it assists in debugging when asserts that interrupts are enabled trip. 81843e1988Sjohnlev */ 82843e1988Sjohnlev ulong_t lastcli[NCPU]; 83843e1988Sjohnlev 84843e1988Sjohnlev #endif 85843e1988Sjohnlev 867c478bd9Sstevel@tonic-gate /* 87ae115bc7Smrj * Set cpu's base SPL level to the highest active interrupt level 887c478bd9Sstevel@tonic-gate */ 89ae115bc7Smrj void 90ae115bc7Smrj set_base_spl(void) 917c478bd9Sstevel@tonic-gate { 92ae115bc7Smrj struct cpu *cpu = CPU; 93ae115bc7Smrj uint16_t active = (uint16_t)cpu->cpu_intr_actv; 947c478bd9Sstevel@tonic-gate 95ae115bc7Smrj cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active); 967c478bd9Sstevel@tonic-gate } 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate /* 997c478bd9Sstevel@tonic-gate * Do all the work necessary to set up the cpu and thread structures 1007c478bd9Sstevel@tonic-gate * to dispatch a high-level interrupt. 1017c478bd9Sstevel@tonic-gate * 1027c478bd9Sstevel@tonic-gate * Returns 0 if we're -not- already on the high-level interrupt stack, 1037c478bd9Sstevel@tonic-gate * (and *must* switch to it), non-zero if we are already on that stack. 1047c478bd9Sstevel@tonic-gate * 1057c478bd9Sstevel@tonic-gate * Called with interrupts masked. 1067c478bd9Sstevel@tonic-gate * The 'pil' is already set to the appropriate level for rp->r_trapno. 1077c478bd9Sstevel@tonic-gate */ 108ae115bc7Smrj static int 1097c478bd9Sstevel@tonic-gate hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp) 1107c478bd9Sstevel@tonic-gate { 1117c478bd9Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 1127c478bd9Sstevel@tonic-gate uint_t mask; 113eda89462Sesolom hrtime_t intrtime; 114ae115bc7Smrj hrtime_t now = tsc_read(); 1157c478bd9Sstevel@tonic-gate 1167c478bd9Sstevel@tonic-gate ASSERT(pil > LOCK_LEVEL); 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate if (pil == CBE_HIGH_PIL) { 1197c478bd9Sstevel@tonic-gate cpu->cpu_profile_pil = oldpil; 1207c478bd9Sstevel@tonic-gate if (USERMODE(rp->r_cs)) { 1217c478bd9Sstevel@tonic-gate cpu->cpu_profile_pc = 0; 1227c478bd9Sstevel@tonic-gate cpu->cpu_profile_upc = rp->r_pc; 123b9e93c10SJonathan Haslam cpu->cpu_cpcprofile_pc = 0; 124b9e93c10SJonathan Haslam cpu->cpu_cpcprofile_upc = rp->r_pc; 1257c478bd9Sstevel@tonic-gate } else { 1267c478bd9Sstevel@tonic-gate cpu->cpu_profile_pc = rp->r_pc; 1277c478bd9Sstevel@tonic-gate cpu->cpu_profile_upc = 0; 128b9e93c10SJonathan Haslam cpu->cpu_cpcprofile_pc = rp->r_pc; 129b9e93c10SJonathan Haslam cpu->cpu_cpcprofile_upc = 0; 1307c478bd9Sstevel@tonic-gate } 1317c478bd9Sstevel@tonic-gate } 1327c478bd9Sstevel@tonic-gate 1337c478bd9Sstevel@tonic-gate mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; 1347c478bd9Sstevel@tonic-gate if (mask != 0) { 1357c478bd9Sstevel@tonic-gate int nestpil; 1367c478bd9Sstevel@tonic-gate 1377c478bd9Sstevel@tonic-gate /* 1387c478bd9Sstevel@tonic-gate * We have interrupted another high-level interrupt. 1397c478bd9Sstevel@tonic-gate * Load starting timestamp, compute interval, update 1407c478bd9Sstevel@tonic-gate * cumulative counter. 1417c478bd9Sstevel@tonic-gate */ 1427c478bd9Sstevel@tonic-gate nestpil = bsrw_insn((uint16_t)mask); 1437c478bd9Sstevel@tonic-gate ASSERT(nestpil < pil); 144ae115bc7Smrj intrtime = now - 1457c478bd9Sstevel@tonic-gate mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)]; 1467a364d25Sschwartz mcpu->intrstat[nestpil][0] += intrtime; 147eda89462Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 1487c478bd9Sstevel@tonic-gate /* 1497c478bd9Sstevel@tonic-gate * Another high-level interrupt is active below this one, so 1507c478bd9Sstevel@tonic-gate * there is no need to check for an interrupt thread. That 1517c478bd9Sstevel@tonic-gate * will be done by the lowest priority high-level interrupt 1527c478bd9Sstevel@tonic-gate * active. 1537c478bd9Sstevel@tonic-gate */ 1547c478bd9Sstevel@tonic-gate } else { 1557c478bd9Sstevel@tonic-gate kthread_t *t = cpu->cpu_thread; 1567c478bd9Sstevel@tonic-gate 1577c478bd9Sstevel@tonic-gate /* 1587c478bd9Sstevel@tonic-gate * See if we are interrupting a low-level interrupt thread. 1597c478bd9Sstevel@tonic-gate * If so, account for its time slice only if its time stamp 1607c478bd9Sstevel@tonic-gate * is non-zero. 1617c478bd9Sstevel@tonic-gate */ 1627c478bd9Sstevel@tonic-gate if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) { 163ae115bc7Smrj intrtime = now - t->t_intr_start; 1647a364d25Sschwartz mcpu->intrstat[t->t_pil][0] += intrtime; 165eda89462Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 1667c478bd9Sstevel@tonic-gate t->t_intr_start = 0; 1677c478bd9Sstevel@tonic-gate } 1687c478bd9Sstevel@tonic-gate } 1697c478bd9Sstevel@tonic-gate 1707c478bd9Sstevel@tonic-gate /* 1717c478bd9Sstevel@tonic-gate * Store starting timestamp in CPU structure for this PIL. 1727c478bd9Sstevel@tonic-gate */ 173ae115bc7Smrj mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now; 1747c478bd9Sstevel@tonic-gate 1757c478bd9Sstevel@tonic-gate ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate if (pil == 15) { 1787c478bd9Sstevel@tonic-gate /* 1797c478bd9Sstevel@tonic-gate * To support reentrant level 15 interrupts, we maintain a 1807c478bd9Sstevel@tonic-gate * recursion count in the top half of cpu_intr_actv. Only 1817c478bd9Sstevel@tonic-gate * when this count hits zero do we clear the PIL 15 bit from 1827c478bd9Sstevel@tonic-gate * the lower half of cpu_intr_actv. 1837c478bd9Sstevel@tonic-gate */ 1847c478bd9Sstevel@tonic-gate uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; 1857c478bd9Sstevel@tonic-gate (*refcntp)++; 1867c478bd9Sstevel@tonic-gate } 1877c478bd9Sstevel@tonic-gate 1887c478bd9Sstevel@tonic-gate mask = cpu->cpu_intr_actv; 1897c478bd9Sstevel@tonic-gate 1907c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv |= (1 << pil); 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK); 1937c478bd9Sstevel@tonic-gate } 1947c478bd9Sstevel@tonic-gate 1957c478bd9Sstevel@tonic-gate /* 1967c478bd9Sstevel@tonic-gate * Does most of the work of returning from a high level interrupt. 1977c478bd9Sstevel@tonic-gate * 1987c478bd9Sstevel@tonic-gate * Returns 0 if there are no more high level interrupts (in which 1997c478bd9Sstevel@tonic-gate * case we must switch back to the interrupted thread stack) or 2007c478bd9Sstevel@tonic-gate * non-zero if there are more (in which case we should stay on it). 2017c478bd9Sstevel@tonic-gate * 2027c478bd9Sstevel@tonic-gate * Called with interrupts masked 2037c478bd9Sstevel@tonic-gate */ 204ae115bc7Smrj static int 2057c478bd9Sstevel@tonic-gate hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum) 2067c478bd9Sstevel@tonic-gate { 2077c478bd9Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 2087c478bd9Sstevel@tonic-gate uint_t mask; 209eda89462Sesolom hrtime_t intrtime; 210ae115bc7Smrj hrtime_t now = tsc_read(); 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate ASSERT(mcpu->mcpu_pri == pil); 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate cpu->cpu_stats.sys.intr[pil - 1]++; 2157c478bd9Sstevel@tonic-gate 2167c478bd9Sstevel@tonic-gate ASSERT(cpu->cpu_intr_actv & (1 << pil)); 2177c478bd9Sstevel@tonic-gate 2187c478bd9Sstevel@tonic-gate if (pil == 15) { 2197c478bd9Sstevel@tonic-gate /* 2207c478bd9Sstevel@tonic-gate * To support reentrant level 15 interrupts, we maintain a 2217c478bd9Sstevel@tonic-gate * recursion count in the top half of cpu_intr_actv. Only 2227c478bd9Sstevel@tonic-gate * when this count hits zero do we clear the PIL 15 bit from 2237c478bd9Sstevel@tonic-gate * the lower half of cpu_intr_actv. 2247c478bd9Sstevel@tonic-gate */ 2257c478bd9Sstevel@tonic-gate uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; 2267c478bd9Sstevel@tonic-gate 2277c478bd9Sstevel@tonic-gate ASSERT(*refcntp > 0); 2287c478bd9Sstevel@tonic-gate 2297c478bd9Sstevel@tonic-gate if (--(*refcntp) == 0) 2307c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 2317c478bd9Sstevel@tonic-gate } else { 2327c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 2337c478bd9Sstevel@tonic-gate } 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0); 2367c478bd9Sstevel@tonic-gate 237ae115bc7Smrj intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)]; 2387a364d25Sschwartz mcpu->intrstat[pil][0] += intrtime; 239eda89462Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate /* 2427c478bd9Sstevel@tonic-gate * Check for lower-pil nested high-level interrupt beneath 2437c478bd9Sstevel@tonic-gate * current one. If so, place a starting timestamp in its 2447c478bd9Sstevel@tonic-gate * pil_high_start entry. 2457c478bd9Sstevel@tonic-gate */ 2467c478bd9Sstevel@tonic-gate mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; 2477c478bd9Sstevel@tonic-gate if (mask != 0) { 2487c478bd9Sstevel@tonic-gate int nestpil; 2497c478bd9Sstevel@tonic-gate 2507c478bd9Sstevel@tonic-gate /* 2517c478bd9Sstevel@tonic-gate * find PIL of nested interrupt 2527c478bd9Sstevel@tonic-gate */ 2537c478bd9Sstevel@tonic-gate nestpil = bsrw_insn((uint16_t)mask); 2547c478bd9Sstevel@tonic-gate ASSERT(nestpil < pil); 255ae115bc7Smrj mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now; 2567c478bd9Sstevel@tonic-gate /* 2577c478bd9Sstevel@tonic-gate * (Another high-level interrupt is active below this one, 2587c478bd9Sstevel@tonic-gate * so there is no need to check for an interrupt 2597c478bd9Sstevel@tonic-gate * thread. That will be done by the lowest priority 2607c478bd9Sstevel@tonic-gate * high-level interrupt active.) 2617c478bd9Sstevel@tonic-gate */ 2627c478bd9Sstevel@tonic-gate } else { 2637c478bd9Sstevel@tonic-gate /* 2647c478bd9Sstevel@tonic-gate * Check to see if there is a low-level interrupt active. 2657c478bd9Sstevel@tonic-gate * If so, place a starting timestamp in the thread 2667c478bd9Sstevel@tonic-gate * structure. 2677c478bd9Sstevel@tonic-gate */ 2687c478bd9Sstevel@tonic-gate kthread_t *t = cpu->cpu_thread; 2697c478bd9Sstevel@tonic-gate 2707c478bd9Sstevel@tonic-gate if (t->t_flag & T_INTR_THREAD) 271ae115bc7Smrj t->t_intr_start = now; 2727c478bd9Sstevel@tonic-gate } 2737c478bd9Sstevel@tonic-gate 2747c478bd9Sstevel@tonic-gate mcpu->mcpu_pri = oldpil; 2757c478bd9Sstevel@tonic-gate (void) (*setlvlx)(oldpil, vecnum); 2767c478bd9Sstevel@tonic-gate 2777c478bd9Sstevel@tonic-gate return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK); 2787c478bd9Sstevel@tonic-gate } 2797c478bd9Sstevel@tonic-gate 2807c478bd9Sstevel@tonic-gate /* 2817c478bd9Sstevel@tonic-gate * Set up the cpu, thread and interrupt thread structures for 2827c478bd9Sstevel@tonic-gate * executing an interrupt thread. The new stack pointer of the 2837c478bd9Sstevel@tonic-gate * interrupt thread (which *must* be switched to) is returned. 2847c478bd9Sstevel@tonic-gate */ 285ae115bc7Smrj static caddr_t 2867c478bd9Sstevel@tonic-gate intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil) 2877c478bd9Sstevel@tonic-gate { 2887c478bd9Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 2897c478bd9Sstevel@tonic-gate kthread_t *t, *volatile it; 290ae115bc7Smrj hrtime_t now = tsc_read(); 2917c478bd9Sstevel@tonic-gate 2927c478bd9Sstevel@tonic-gate ASSERT(pil > 0); 2937c478bd9Sstevel@tonic-gate ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 2947c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv |= (1 << pil); 2957c478bd9Sstevel@tonic-gate 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * Get set to run an interrupt thread. 2987c478bd9Sstevel@tonic-gate * There should always be an interrupt thread, since we 2997c478bd9Sstevel@tonic-gate * allocate one for each level on each CPU. 3007c478bd9Sstevel@tonic-gate * 301fd71cd2fSesolom * t_intr_start could be zero due to cpu_intr_swtch_enter. 3027c478bd9Sstevel@tonic-gate */ 3037c478bd9Sstevel@tonic-gate t = cpu->cpu_thread; 304fd71cd2fSesolom if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { 305ae115bc7Smrj hrtime_t intrtime = now - t->t_intr_start; 3067a364d25Sschwartz mcpu->intrstat[t->t_pil][0] += intrtime; 307eda89462Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 3087c478bd9Sstevel@tonic-gate t->t_intr_start = 0; 3097c478bd9Sstevel@tonic-gate } 3107c478bd9Sstevel@tonic-gate 3117c478bd9Sstevel@tonic-gate ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); 3127c478bd9Sstevel@tonic-gate 3137c478bd9Sstevel@tonic-gate t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */ 3147c478bd9Sstevel@tonic-gate 3157c478bd9Sstevel@tonic-gate /* 3167c478bd9Sstevel@tonic-gate * unlink the interrupt thread off the cpu 317fd71cd2fSesolom * 318fd71cd2fSesolom * Note that the code in kcpc_overflow_intr -relies- on the 319fd71cd2fSesolom * ordering of events here - in particular that t->t_lwp of 320fd71cd2fSesolom * the interrupt thread is set to the pinned thread *before* 321fd71cd2fSesolom * curthread is changed. 3227c478bd9Sstevel@tonic-gate */ 3237c478bd9Sstevel@tonic-gate it = cpu->cpu_intr_thread; 3247c478bd9Sstevel@tonic-gate cpu->cpu_intr_thread = it->t_link; 3257c478bd9Sstevel@tonic-gate it->t_intr = t; 3267c478bd9Sstevel@tonic-gate it->t_lwp = t->t_lwp; 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate /* 3297c478bd9Sstevel@tonic-gate * (threads on the interrupt thread free list could have state 3307c478bd9Sstevel@tonic-gate * preset to TS_ONPROC, but it helps in debugging if 3317c478bd9Sstevel@tonic-gate * they're TS_FREE.) 3327c478bd9Sstevel@tonic-gate */ 3337c478bd9Sstevel@tonic-gate it->t_state = TS_ONPROC; 3347c478bd9Sstevel@tonic-gate 3357c478bd9Sstevel@tonic-gate cpu->cpu_thread = it; /* new curthread on this cpu */ 3367c478bd9Sstevel@tonic-gate it->t_pil = (uchar_t)pil; 3377c478bd9Sstevel@tonic-gate it->t_pri = intr_pri + (pri_t)pil; 338ae115bc7Smrj it->t_intr_start = now; 3397c478bd9Sstevel@tonic-gate 3407c478bd9Sstevel@tonic-gate return (it->t_stk); 3417c478bd9Sstevel@tonic-gate } 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate 3447c478bd9Sstevel@tonic-gate #ifdef DEBUG 3457c478bd9Sstevel@tonic-gate int intr_thread_cnt; 3467c478bd9Sstevel@tonic-gate #endif 3477c478bd9Sstevel@tonic-gate 3487c478bd9Sstevel@tonic-gate /* 3497c478bd9Sstevel@tonic-gate * Called with interrupts disabled 3507c478bd9Sstevel@tonic-gate */ 351ae115bc7Smrj static void 3527c478bd9Sstevel@tonic-gate intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil) 3537c478bd9Sstevel@tonic-gate { 3547c478bd9Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 3557c478bd9Sstevel@tonic-gate kthread_t *t; 3567c478bd9Sstevel@tonic-gate kthread_t *it = cpu->cpu_thread; /* curthread */ 3577c478bd9Sstevel@tonic-gate uint_t pil, basespl; 358eda89462Sesolom hrtime_t intrtime; 359ae115bc7Smrj hrtime_t now = tsc_read(); 3607c478bd9Sstevel@tonic-gate 3617c478bd9Sstevel@tonic-gate pil = it->t_pil; 3627c478bd9Sstevel@tonic-gate cpu->cpu_stats.sys.intr[pil - 1]++; 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate ASSERT(it->t_intr_start != 0); 365ae115bc7Smrj intrtime = now - it->t_intr_start; 3667a364d25Sschwartz mcpu->intrstat[pil][0] += intrtime; 367eda89462Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 3687c478bd9Sstevel@tonic-gate 3697c478bd9Sstevel@tonic-gate ASSERT(cpu->cpu_intr_actv & (1 << pil)); 3707c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate /* 3737c478bd9Sstevel@tonic-gate * If there is still an interrupted thread underneath this one 3747c478bd9Sstevel@tonic-gate * then the interrupt was never blocked and the return is 3757c478bd9Sstevel@tonic-gate * fairly simple. Otherwise it isn't. 3767c478bd9Sstevel@tonic-gate */ 3777c478bd9Sstevel@tonic-gate if ((t = it->t_intr) == NULL) { 3787c478bd9Sstevel@tonic-gate /* 3797c478bd9Sstevel@tonic-gate * The interrupted thread is no longer pinned underneath 3807c478bd9Sstevel@tonic-gate * the interrupt thread. This means the interrupt must 3817c478bd9Sstevel@tonic-gate * have blocked, and the interrupted thread has been 3827c478bd9Sstevel@tonic-gate * unpinned, and has probably been running around the 3837c478bd9Sstevel@tonic-gate * system for a while. 3847c478bd9Sstevel@tonic-gate * 3857c478bd9Sstevel@tonic-gate * Since there is no longer a thread under this one, put 3867c478bd9Sstevel@tonic-gate * this interrupt thread back on the CPU's free list and 3877c478bd9Sstevel@tonic-gate * resume the idle thread which will dispatch the next 3887c478bd9Sstevel@tonic-gate * thread to run. 3897c478bd9Sstevel@tonic-gate */ 3907c478bd9Sstevel@tonic-gate #ifdef DEBUG 3917c478bd9Sstevel@tonic-gate intr_thread_cnt++; 3927c478bd9Sstevel@tonic-gate #endif 3937c478bd9Sstevel@tonic-gate cpu->cpu_stats.sys.intrblk++; 3947c478bd9Sstevel@tonic-gate /* 3957c478bd9Sstevel@tonic-gate * Set CPU's base SPL based on active interrupts bitmask 3967c478bd9Sstevel@tonic-gate */ 3977c478bd9Sstevel@tonic-gate set_base_spl(); 3987c478bd9Sstevel@tonic-gate basespl = cpu->cpu_base_spl; 3997c478bd9Sstevel@tonic-gate mcpu->mcpu_pri = basespl; 4007c478bd9Sstevel@tonic-gate (*setlvlx)(basespl, vec); 4017c478bd9Sstevel@tonic-gate (void) splhigh(); 402ae115bc7Smrj sti(); 4037c478bd9Sstevel@tonic-gate it->t_state = TS_FREE; 4047c478bd9Sstevel@tonic-gate /* 4057c478bd9Sstevel@tonic-gate * Return interrupt thread to pool 4067c478bd9Sstevel@tonic-gate */ 4077c478bd9Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 4087c478bd9Sstevel@tonic-gate cpu->cpu_intr_thread = it; 4097c478bd9Sstevel@tonic-gate swtch(); 410ae115bc7Smrj panic("intr_thread_epilog: swtch returned"); 4117c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 4127c478bd9Sstevel@tonic-gate } 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate /* 4157c478bd9Sstevel@tonic-gate * Return interrupt thread to the pool 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 4187c478bd9Sstevel@tonic-gate cpu->cpu_intr_thread = it; 4197c478bd9Sstevel@tonic-gate it->t_state = TS_FREE; 4207c478bd9Sstevel@tonic-gate 4217c478bd9Sstevel@tonic-gate basespl = cpu->cpu_base_spl; 4227c478bd9Sstevel@tonic-gate pil = MAX(oldpil, basespl); 4237c478bd9Sstevel@tonic-gate mcpu->mcpu_pri = pil; 4247c478bd9Sstevel@tonic-gate (*setlvlx)(pil, vec); 425ae115bc7Smrj t->t_intr_start = now; 4267c478bd9Sstevel@tonic-gate cpu->cpu_thread = t; 4277c478bd9Sstevel@tonic-gate } 4287c478bd9Sstevel@tonic-gate 4297a364d25Sschwartz /* 430ae115bc7Smrj * intr_get_time() is a resource for interrupt handlers to determine how 431ae115bc7Smrj * much time has been spent handling the current interrupt. Such a function 432ae115bc7Smrj * is needed because higher level interrupts can arrive during the 433ae115bc7Smrj * processing of an interrupt. intr_get_time() only returns time spent in the 434ae115bc7Smrj * current interrupt handler. 435ae115bc7Smrj * 436ae115bc7Smrj * The caller must be calling from an interrupt handler running at a pil 437ae115bc7Smrj * below or at lock level. Timings are not provided for high-level 438ae115bc7Smrj * interrupts. 439ae115bc7Smrj * 440ae115bc7Smrj * The first time intr_get_time() is called while handling an interrupt, 441ae115bc7Smrj * it returns the time since the interrupt handler was invoked. Subsequent 442ae115bc7Smrj * calls will return the time since the prior call to intr_get_time(). Time 443843e1988Sjohnlev * is returned as ticks. Use scalehrtimef() to convert ticks to nsec. 444ae115bc7Smrj * 445ae115bc7Smrj * Theory Of Intrstat[][]: 446ae115bc7Smrj * 447ae115bc7Smrj * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two 448ae115bc7Smrj * uint64_ts per pil. 449ae115bc7Smrj * 450ae115bc7Smrj * intrstat[pil][0] is a cumulative count of the number of ticks spent 451ae115bc7Smrj * handling all interrupts at the specified pil on this CPU. It is 452ae115bc7Smrj * exported via kstats to the user. 453ae115bc7Smrj * 454ae115bc7Smrj * intrstat[pil][1] is always a count of ticks less than or equal to the 455ae115bc7Smrj * value in [0]. The difference between [1] and [0] is the value returned 456ae115bc7Smrj * by a call to intr_get_time(). At the start of interrupt processing, 457ae115bc7Smrj * [0] and [1] will be equal (or nearly so). As the interrupt consumes 458ae115bc7Smrj * time, [0] will increase, but [1] will remain the same. A call to 459ae115bc7Smrj * intr_get_time() will return the difference, then update [1] to be the 460ae115bc7Smrj * same as [0]. Future calls will return the time since the last call. 461ae115bc7Smrj * Finally, when the interrupt completes, [1] is updated to the same as [0]. 462ae115bc7Smrj * 463ae115bc7Smrj * Implementation: 464ae115bc7Smrj * 465ae115bc7Smrj * intr_get_time() works much like a higher level interrupt arriving. It 466ae115bc7Smrj * "checkpoints" the timing information by incrementing intrstat[pil][0] 467ae115bc7Smrj * to include elapsed running time, and by setting t_intr_start to rdtsc. 468ae115bc7Smrj * It then sets the return value to intrstat[pil][0] - intrstat[pil][1], 469ae115bc7Smrj * and updates intrstat[pil][1] to be the same as the new value of 470ae115bc7Smrj * intrstat[pil][0]. 471ae115bc7Smrj * 472ae115bc7Smrj * In the normal handling of interrupts, after an interrupt handler returns 473ae115bc7Smrj * and the code in intr_thread() updates intrstat[pil][0], it then sets 474ae115bc7Smrj * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1], 475ae115bc7Smrj * the timings are reset, i.e. intr_get_time() will return [0] - [1] which 476ae115bc7Smrj * is 0. 477ae115bc7Smrj * 478ae115bc7Smrj * Whenever interrupts arrive on a CPU which is handling a lower pil 479ae115bc7Smrj * interrupt, they update the lower pil's [0] to show time spent in the 480ae115bc7Smrj * handler that they've interrupted. This results in a growing discrepancy 481ae115bc7Smrj * between [0] and [1], which is returned the next time intr_get_time() is 482ae115bc7Smrj * called. Time spent in the higher-pil interrupt will not be returned in 483ae115bc7Smrj * the next intr_get_time() call from the original interrupt, because 484ae115bc7Smrj * the higher-pil interrupt's time is accumulated in intrstat[higherpil][]. 4857a364d25Sschwartz */ 4867a364d25Sschwartz uint64_t 487ae115bc7Smrj intr_get_time(void) 4887a364d25Sschwartz { 489ae115bc7Smrj struct cpu *cpu; 490ae115bc7Smrj struct machcpu *mcpu; 491ae115bc7Smrj kthread_t *t; 4927a364d25Sschwartz uint64_t time, delta, ret; 493ae115bc7Smrj uint_t pil; 4947a364d25Sschwartz 495ae115bc7Smrj cli(); 496ae115bc7Smrj cpu = CPU; 497ae115bc7Smrj mcpu = &cpu->cpu_m; 498ae115bc7Smrj t = cpu->cpu_thread; 499ae115bc7Smrj pil = t->t_pil; 5007a364d25Sschwartz ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0); 5017a364d25Sschwartz ASSERT(t->t_flag & T_INTR_THREAD); 5027a364d25Sschwartz ASSERT(pil != 0); 5037a364d25Sschwartz ASSERT(t->t_intr_start != 0); 5047a364d25Sschwartz 5057a364d25Sschwartz time = tsc_read(); 5067a364d25Sschwartz delta = time - t->t_intr_start; 5077a364d25Sschwartz t->t_intr_start = time; 5087a364d25Sschwartz 5097a364d25Sschwartz time = mcpu->intrstat[pil][0] + delta; 5107a364d25Sschwartz ret = time - mcpu->intrstat[pil][1]; 5117a364d25Sschwartz mcpu->intrstat[pil][0] = time; 5127a364d25Sschwartz mcpu->intrstat[pil][1] = time; 513c81508f4Sjhaslam cpu->cpu_intracct[cpu->cpu_mstate] += delta; 5147a364d25Sschwartz 515ae115bc7Smrj sti(); 5167a364d25Sschwartz return (ret); 5177a364d25Sschwartz } 5187a364d25Sschwartz 519ae115bc7Smrj static caddr_t 5207c478bd9Sstevel@tonic-gate dosoftint_prolog( 5217c478bd9Sstevel@tonic-gate struct cpu *cpu, 5227c478bd9Sstevel@tonic-gate caddr_t stackptr, 5237c478bd9Sstevel@tonic-gate uint32_t st_pending, 5247c478bd9Sstevel@tonic-gate uint_t oldpil) 5257c478bd9Sstevel@tonic-gate { 5267c478bd9Sstevel@tonic-gate kthread_t *t, *volatile it; 5277c478bd9Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 5287c478bd9Sstevel@tonic-gate uint_t pil; 529ae115bc7Smrj hrtime_t now; 5307c478bd9Sstevel@tonic-gate 5317c478bd9Sstevel@tonic-gate top: 5327c478bd9Sstevel@tonic-gate ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending); 5337c478bd9Sstevel@tonic-gate 5347c478bd9Sstevel@tonic-gate pil = bsrw_insn((uint16_t)st_pending); 5357c478bd9Sstevel@tonic-gate if (pil <= oldpil || pil <= cpu->cpu_base_spl) 5367c478bd9Sstevel@tonic-gate return (0); 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate /* 5397c478bd9Sstevel@tonic-gate * XX64 Sigh. 5407c478bd9Sstevel@tonic-gate * 5417c478bd9Sstevel@tonic-gate * This is a transliteration of the i386 assembler code for 5427c478bd9Sstevel@tonic-gate * soft interrupts. One question is "why does this need 5437c478bd9Sstevel@tonic-gate * to be atomic?" One possible race is -other- processors 5447c478bd9Sstevel@tonic-gate * posting soft interrupts to us in set_pending() i.e. the 5457c478bd9Sstevel@tonic-gate * CPU might get preempted just after the address computation, 5467c478bd9Sstevel@tonic-gate * but just before the atomic transaction, so another CPU would 5477c478bd9Sstevel@tonic-gate * actually set the original CPU's st_pending bit. However, 5487c478bd9Sstevel@tonic-gate * it looks like it would be simpler to disable preemption there. 5497c478bd9Sstevel@tonic-gate * Are there other races for which preemption control doesn't work? 5507c478bd9Sstevel@tonic-gate * 5517c478bd9Sstevel@tonic-gate * The i386 assembler version -also- checks to see if the bit 5527c478bd9Sstevel@tonic-gate * being cleared was actually set; if it wasn't, it rechecks 5537c478bd9Sstevel@tonic-gate * for more. This seems a bit strange, as the only code that 5547c478bd9Sstevel@tonic-gate * ever clears the bit is -this- code running with interrupts 5557c478bd9Sstevel@tonic-gate * disabled on -this- CPU. This code would probably be cheaper: 5567c478bd9Sstevel@tonic-gate * 5577c478bd9Sstevel@tonic-gate * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, 5587c478bd9Sstevel@tonic-gate * ~(1 << pil)); 5597c478bd9Sstevel@tonic-gate * 5607c478bd9Sstevel@tonic-gate * and t->t_preempt--/++ around set_pending() even cheaper, 5617c478bd9Sstevel@tonic-gate * but at this point, correctness is critical, so we slavishly 5627c478bd9Sstevel@tonic-gate * emulate the i386 port. 5637c478bd9Sstevel@tonic-gate */ 564ae115bc7Smrj if (atomic_btr32((uint32_t *) 565ae115bc7Smrj &mcpu->mcpu_softinfo.st_pending, pil) == 0) { 5667c478bd9Sstevel@tonic-gate st_pending = mcpu->mcpu_softinfo.st_pending; 5677c478bd9Sstevel@tonic-gate goto top; 5687c478bd9Sstevel@tonic-gate } 5697c478bd9Sstevel@tonic-gate 5707c478bd9Sstevel@tonic-gate mcpu->mcpu_pri = pil; 5717c478bd9Sstevel@tonic-gate (*setspl)(pil); 5727c478bd9Sstevel@tonic-gate 573ae115bc7Smrj now = tsc_read(); 574ae115bc7Smrj 5757c478bd9Sstevel@tonic-gate /* 5767c478bd9Sstevel@tonic-gate * Get set to run interrupt thread. 5777c478bd9Sstevel@tonic-gate * There should always be an interrupt thread since we 5787c478bd9Sstevel@tonic-gate * allocate one for each level on the CPU. 5797c478bd9Sstevel@tonic-gate */ 5807c478bd9Sstevel@tonic-gate it = cpu->cpu_intr_thread; 5817c478bd9Sstevel@tonic-gate cpu->cpu_intr_thread = it->t_link; 5827c478bd9Sstevel@tonic-gate 583fd71cd2fSesolom /* t_intr_start could be zero due to cpu_intr_swtch_enter. */ 584fd71cd2fSesolom t = cpu->cpu_thread; 585fd71cd2fSesolom if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { 586ae115bc7Smrj hrtime_t intrtime = now - t->t_intr_start; 587fd71cd2fSesolom mcpu->intrstat[pil][0] += intrtime; 588fd71cd2fSesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 589fd71cd2fSesolom t->t_intr_start = 0; 590fd71cd2fSesolom } 591fd71cd2fSesolom 5927c478bd9Sstevel@tonic-gate /* 5937c478bd9Sstevel@tonic-gate * Note that the code in kcpc_overflow_intr -relies- on the 5947c478bd9Sstevel@tonic-gate * ordering of events here - in particular that t->t_lwp of 5957c478bd9Sstevel@tonic-gate * the interrupt thread is set to the pinned thread *before* 596fd71cd2fSesolom * curthread is changed. 5977c478bd9Sstevel@tonic-gate */ 5987c478bd9Sstevel@tonic-gate it->t_lwp = t->t_lwp; 5997c478bd9Sstevel@tonic-gate it->t_state = TS_ONPROC; 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate /* 6027c478bd9Sstevel@tonic-gate * Push interrupted thread onto list from new thread. 6037c478bd9Sstevel@tonic-gate * Set the new thread as the current one. 6047c478bd9Sstevel@tonic-gate * Set interrupted thread's T_SP because if it is the idle thread, 6057c478bd9Sstevel@tonic-gate * resume() may use that stack between threads. 6067c478bd9Sstevel@tonic-gate */ 6077c478bd9Sstevel@tonic-gate 6087c478bd9Sstevel@tonic-gate ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); 6097c478bd9Sstevel@tonic-gate t->t_sp = (uintptr_t)stackptr; 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate it->t_intr = t; 6127c478bd9Sstevel@tonic-gate cpu->cpu_thread = it; 6137c478bd9Sstevel@tonic-gate 6147c478bd9Sstevel@tonic-gate /* 6157c478bd9Sstevel@tonic-gate * Set bit for this pil in CPU's interrupt active bitmask. 6167c478bd9Sstevel@tonic-gate */ 6177c478bd9Sstevel@tonic-gate ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 6187c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv |= (1 << pil); 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate /* 6217c478bd9Sstevel@tonic-gate * Initialize thread priority level from intr_pri 6227c478bd9Sstevel@tonic-gate */ 6237c478bd9Sstevel@tonic-gate it->t_pil = (uchar_t)pil; 6247c478bd9Sstevel@tonic-gate it->t_pri = (pri_t)pil + intr_pri; 625ae115bc7Smrj it->t_intr_start = now; 6267c478bd9Sstevel@tonic-gate 6277c478bd9Sstevel@tonic-gate return (it->t_stk); 6287c478bd9Sstevel@tonic-gate } 6297c478bd9Sstevel@tonic-gate 630ae115bc7Smrj static void 6317c478bd9Sstevel@tonic-gate dosoftint_epilog(struct cpu *cpu, uint_t oldpil) 6327c478bd9Sstevel@tonic-gate { 6337c478bd9Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 6347c478bd9Sstevel@tonic-gate kthread_t *t, *it; 6357c478bd9Sstevel@tonic-gate uint_t pil, basespl; 636eda89462Sesolom hrtime_t intrtime; 637ae115bc7Smrj hrtime_t now = tsc_read(); 6387c478bd9Sstevel@tonic-gate 6397c478bd9Sstevel@tonic-gate it = cpu->cpu_thread; 6407c478bd9Sstevel@tonic-gate pil = it->t_pil; 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate cpu->cpu_stats.sys.intr[pil - 1]++; 6437c478bd9Sstevel@tonic-gate 6447c478bd9Sstevel@tonic-gate ASSERT(cpu->cpu_intr_actv & (1 << pil)); 6457c478bd9Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 646ae115bc7Smrj intrtime = now - it->t_intr_start; 6477a364d25Sschwartz mcpu->intrstat[pil][0] += intrtime; 648eda89462Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate /* 6517c478bd9Sstevel@tonic-gate * If there is still an interrupted thread underneath this one 6527c478bd9Sstevel@tonic-gate * then the interrupt was never blocked and the return is 6537c478bd9Sstevel@tonic-gate * fairly simple. Otherwise it isn't. 6547c478bd9Sstevel@tonic-gate */ 6557c478bd9Sstevel@tonic-gate if ((t = it->t_intr) == NULL) { 6567c478bd9Sstevel@tonic-gate /* 6577c478bd9Sstevel@tonic-gate * Put thread back on the interrupt thread list. 6587c478bd9Sstevel@tonic-gate * This was an interrupt thread, so set CPU's base SPL. 6597c478bd9Sstevel@tonic-gate */ 6607c478bd9Sstevel@tonic-gate set_base_spl(); 6617c478bd9Sstevel@tonic-gate it->t_state = TS_FREE; 6627c478bd9Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 6637c478bd9Sstevel@tonic-gate cpu->cpu_intr_thread = it; 6647c478bd9Sstevel@tonic-gate (void) splhigh(); 665ae115bc7Smrj sti(); 6667c478bd9Sstevel@tonic-gate swtch(); 6677c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 668ae115bc7Smrj panic("dosoftint_epilog: swtch returned"); 6697c478bd9Sstevel@tonic-gate } 6707c478bd9Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 6717c478bd9Sstevel@tonic-gate cpu->cpu_intr_thread = it; 6727c478bd9Sstevel@tonic-gate it->t_state = TS_FREE; 6737c478bd9Sstevel@tonic-gate cpu->cpu_thread = t; 6747c478bd9Sstevel@tonic-gate if (t->t_flag & T_INTR_THREAD) 675ae115bc7Smrj t->t_intr_start = now; 6767c478bd9Sstevel@tonic-gate basespl = cpu->cpu_base_spl; 6777c478bd9Sstevel@tonic-gate pil = MAX(oldpil, basespl); 6787c478bd9Sstevel@tonic-gate mcpu->mcpu_pri = pil; 6797c478bd9Sstevel@tonic-gate (*setspl)(pil); 6807c478bd9Sstevel@tonic-gate } 6817c478bd9Sstevel@tonic-gate 682ae115bc7Smrj 6837c478bd9Sstevel@tonic-gate /* 6847c478bd9Sstevel@tonic-gate * Make the interrupted thread 'to' be runnable. 6857c478bd9Sstevel@tonic-gate * 6867c478bd9Sstevel@tonic-gate * Since t->t_sp has already been saved, t->t_pc is all 6877c478bd9Sstevel@tonic-gate * that needs to be set in this function. 6887c478bd9Sstevel@tonic-gate * 6897c478bd9Sstevel@tonic-gate * Returns the interrupt level of the interrupt thread. 6907c478bd9Sstevel@tonic-gate */ 6917c478bd9Sstevel@tonic-gate int 6927c478bd9Sstevel@tonic-gate intr_passivate( 6937c478bd9Sstevel@tonic-gate kthread_t *it, /* interrupt thread */ 6947c478bd9Sstevel@tonic-gate kthread_t *t) /* interrupted thread */ 6957c478bd9Sstevel@tonic-gate { 6967c478bd9Sstevel@tonic-gate extern void _sys_rtt(); 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate ASSERT(it->t_flag & T_INTR_THREAD); 6997c478bd9Sstevel@tonic-gate ASSERT(SA(t->t_sp) == t->t_sp); 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate t->t_pc = (uintptr_t)_sys_rtt; 7027c478bd9Sstevel@tonic-gate return (it->t_pil); 7037c478bd9Sstevel@tonic-gate } 7047c478bd9Sstevel@tonic-gate 7057c478bd9Sstevel@tonic-gate /* 7067c478bd9Sstevel@tonic-gate * Create interrupt kstats for this CPU. 7077c478bd9Sstevel@tonic-gate */ 7087c478bd9Sstevel@tonic-gate void 7097c478bd9Sstevel@tonic-gate cpu_create_intrstat(cpu_t *cp) 7107c478bd9Sstevel@tonic-gate { 7117c478bd9Sstevel@tonic-gate int i; 7127c478bd9Sstevel@tonic-gate kstat_t *intr_ksp; 7137c478bd9Sstevel@tonic-gate kstat_named_t *knp; 7147c478bd9Sstevel@tonic-gate char name[KSTAT_STRLEN]; 7157c478bd9Sstevel@tonic-gate zoneid_t zoneid; 7167c478bd9Sstevel@tonic-gate 7177c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 7187c478bd9Sstevel@tonic-gate 7197c478bd9Sstevel@tonic-gate if (pool_pset_enabled()) 7207c478bd9Sstevel@tonic-gate zoneid = GLOBAL_ZONEID; 7217c478bd9Sstevel@tonic-gate else 7227c478bd9Sstevel@tonic-gate zoneid = ALL_ZONES; 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc", 7257c478bd9Sstevel@tonic-gate KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid); 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate /* 7287c478bd9Sstevel@tonic-gate * Initialize each PIL's named kstat 7297c478bd9Sstevel@tonic-gate */ 7307c478bd9Sstevel@tonic-gate if (intr_ksp != NULL) { 7317c478bd9Sstevel@tonic-gate intr_ksp->ks_update = cpu_kstat_intrstat_update; 7327c478bd9Sstevel@tonic-gate knp = (kstat_named_t *)intr_ksp->ks_data; 7337c478bd9Sstevel@tonic-gate intr_ksp->ks_private = cp; 7347c478bd9Sstevel@tonic-gate for (i = 0; i < PIL_MAX; i++) { 7357c478bd9Sstevel@tonic-gate (void) snprintf(name, KSTAT_STRLEN, "level-%d-time", 7367c478bd9Sstevel@tonic-gate i + 1); 7377c478bd9Sstevel@tonic-gate kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64); 7387c478bd9Sstevel@tonic-gate (void) snprintf(name, KSTAT_STRLEN, "level-%d-count", 7397c478bd9Sstevel@tonic-gate i + 1); 7407c478bd9Sstevel@tonic-gate kstat_named_init(&knp[(i * 2) + 1], name, 7417c478bd9Sstevel@tonic-gate KSTAT_DATA_UINT64); 7427c478bd9Sstevel@tonic-gate } 7437c478bd9Sstevel@tonic-gate kstat_install(intr_ksp); 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate } 7467c478bd9Sstevel@tonic-gate 7477c478bd9Sstevel@tonic-gate /* 7487c478bd9Sstevel@tonic-gate * Delete interrupt kstats for this CPU. 7497c478bd9Sstevel@tonic-gate */ 7507c478bd9Sstevel@tonic-gate void 7517c478bd9Sstevel@tonic-gate cpu_delete_intrstat(cpu_t *cp) 7527c478bd9Sstevel@tonic-gate { 7537c478bd9Sstevel@tonic-gate kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES); 7547c478bd9Sstevel@tonic-gate } 7557c478bd9Sstevel@tonic-gate 7567c478bd9Sstevel@tonic-gate /* 7577c478bd9Sstevel@tonic-gate * Convert interrupt statistics from CPU ticks to nanoseconds and 7587c478bd9Sstevel@tonic-gate * update kstat. 7597c478bd9Sstevel@tonic-gate */ 7607c478bd9Sstevel@tonic-gate int 7617c478bd9Sstevel@tonic-gate cpu_kstat_intrstat_update(kstat_t *ksp, int rw) 7627c478bd9Sstevel@tonic-gate { 7637c478bd9Sstevel@tonic-gate kstat_named_t *knp = ksp->ks_data; 7647c478bd9Sstevel@tonic-gate cpu_t *cpup = (cpu_t *)ksp->ks_private; 7657c478bd9Sstevel@tonic-gate int i; 7667c478bd9Sstevel@tonic-gate hrtime_t hrt; 7677c478bd9Sstevel@tonic-gate 7687c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 7697c478bd9Sstevel@tonic-gate return (EACCES); 7707c478bd9Sstevel@tonic-gate 7717c478bd9Sstevel@tonic-gate for (i = 0; i < PIL_MAX; i++) { 7727a364d25Sschwartz hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0]; 773843e1988Sjohnlev scalehrtimef(&hrt); 7747c478bd9Sstevel@tonic-gate knp[i * 2].value.ui64 = (uint64_t)hrt; 7757c478bd9Sstevel@tonic-gate knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i]; 7767c478bd9Sstevel@tonic-gate } 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate return (0); 7797c478bd9Sstevel@tonic-gate } 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate /* 7827c478bd9Sstevel@tonic-gate * An interrupt thread is ending a time slice, so compute the interval it 7837c478bd9Sstevel@tonic-gate * ran for and update the statistic for its PIL. 7847c478bd9Sstevel@tonic-gate */ 7857c478bd9Sstevel@tonic-gate void 7867c478bd9Sstevel@tonic-gate cpu_intr_swtch_enter(kthread_id_t t) 7877c478bd9Sstevel@tonic-gate { 7887c478bd9Sstevel@tonic-gate uint64_t interval; 7897c478bd9Sstevel@tonic-gate uint64_t start; 790eda89462Sesolom cpu_t *cpu; 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate ASSERT((t->t_flag & T_INTR_THREAD) != 0); 7937c478bd9Sstevel@tonic-gate ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate /* 7967c478bd9Sstevel@tonic-gate * We could be here with a zero timestamp. This could happen if: 7977c478bd9Sstevel@tonic-gate * an interrupt thread which no longer has a pinned thread underneath 7987c478bd9Sstevel@tonic-gate * it (i.e. it blocked at some point in its past) has finished running 7997c478bd9Sstevel@tonic-gate * its handler. intr_thread() updated the interrupt statistic for its 8007c478bd9Sstevel@tonic-gate * PIL and zeroed its timestamp. Since there was no pinned thread to 8017c478bd9Sstevel@tonic-gate * return to, swtch() gets called and we end up here. 802eda89462Sesolom * 803eda89462Sesolom * Note that we use atomic ops below (cas64 and atomic_add_64), which 804eda89462Sesolom * we don't use in the functions above, because we're not called 805eda89462Sesolom * with interrupts blocked, but the epilog/prolog functions are. 8067c478bd9Sstevel@tonic-gate */ 8077c478bd9Sstevel@tonic-gate if (t->t_intr_start) { 8087c478bd9Sstevel@tonic-gate do { 8097c478bd9Sstevel@tonic-gate start = t->t_intr_start; 8107c478bd9Sstevel@tonic-gate interval = tsc_read() - start; 8117c478bd9Sstevel@tonic-gate } while (cas64(&t->t_intr_start, start, 0) != start); 812eda89462Sesolom cpu = CPU; 8137a364d25Sschwartz cpu->cpu_m.intrstat[t->t_pil][0] += interval; 814eda89462Sesolom 815eda89462Sesolom atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate], 816eda89462Sesolom interval); 8177c478bd9Sstevel@tonic-gate } else 8187c478bd9Sstevel@tonic-gate ASSERT(t->t_intr == NULL); 8197c478bd9Sstevel@tonic-gate } 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate /* 8227c478bd9Sstevel@tonic-gate * An interrupt thread is returning from swtch(). Place a starting timestamp 8237c478bd9Sstevel@tonic-gate * in its thread structure. 8247c478bd9Sstevel@tonic-gate */ 8257c478bd9Sstevel@tonic-gate void 8267c478bd9Sstevel@tonic-gate cpu_intr_swtch_exit(kthread_id_t t) 8277c478bd9Sstevel@tonic-gate { 8287c478bd9Sstevel@tonic-gate uint64_t ts; 8297c478bd9Sstevel@tonic-gate 8307c478bd9Sstevel@tonic-gate ASSERT((t->t_flag & T_INTR_THREAD) != 0); 8317c478bd9Sstevel@tonic-gate ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); 8327c478bd9Sstevel@tonic-gate 8337c478bd9Sstevel@tonic-gate do { 8347c478bd9Sstevel@tonic-gate ts = t->t_intr_start; 8357c478bd9Sstevel@tonic-gate } while (cas64(&t->t_intr_start, ts, tsc_read()) != ts); 8367c478bd9Sstevel@tonic-gate } 837ae115bc7Smrj 838ae115bc7Smrj /* 839ae115bc7Smrj * Dispatch a hilevel interrupt (one above LOCK_LEVEL) 840ae115bc7Smrj */ 841ae115bc7Smrj /*ARGSUSED*/ 842ae115bc7Smrj static void 843ae115bc7Smrj dispatch_hilevel(uint_t vector, uint_t arg2) 844ae115bc7Smrj { 845ae115bc7Smrj sti(); 846ae115bc7Smrj av_dispatch_autovect(vector); 847ae115bc7Smrj cli(); 848ae115bc7Smrj } 849ae115bc7Smrj 850ae115bc7Smrj /* 851ae115bc7Smrj * Dispatch a soft interrupt 852ae115bc7Smrj */ 853ae115bc7Smrj /*ARGSUSED*/ 854ae115bc7Smrj static void 855ae115bc7Smrj dispatch_softint(uint_t oldpil, uint_t arg2) 856ae115bc7Smrj { 857ae115bc7Smrj struct cpu *cpu = CPU; 858ae115bc7Smrj 859ae115bc7Smrj sti(); 860ae115bc7Smrj av_dispatch_softvect((int)cpu->cpu_thread->t_pil); 861ae115bc7Smrj cli(); 862ae115bc7Smrj 863ae115bc7Smrj /* 864ae115bc7Smrj * Must run softint_epilog() on the interrupt thread stack, since 865ae115bc7Smrj * there may not be a return from it if the interrupt thread blocked. 866ae115bc7Smrj */ 867ae115bc7Smrj dosoftint_epilog(cpu, oldpil); 868ae115bc7Smrj } 869ae115bc7Smrj 870ae115bc7Smrj /* 871ae115bc7Smrj * Dispatch a normal interrupt 872ae115bc7Smrj */ 873ae115bc7Smrj static void 874ae115bc7Smrj dispatch_hardint(uint_t vector, uint_t oldipl) 875ae115bc7Smrj { 876ae115bc7Smrj struct cpu *cpu = CPU; 877ae115bc7Smrj 878ae115bc7Smrj sti(); 879ae115bc7Smrj av_dispatch_autovect(vector); 880ae115bc7Smrj cli(); 881ae115bc7Smrj 882ae115bc7Smrj /* 883ae115bc7Smrj * Must run intr_thread_epilog() on the interrupt thread stack, since 884ae115bc7Smrj * there may not be a return from it if the interrupt thread blocked. 885ae115bc7Smrj */ 886ae115bc7Smrj intr_thread_epilog(cpu, vector, oldipl); 887ae115bc7Smrj } 888ae115bc7Smrj 889ae115bc7Smrj /* 890ae115bc7Smrj * Deliver any softints the current interrupt priority allows. 891ae115bc7Smrj * Called with interrupts disabled. 892ae115bc7Smrj */ 893ae115bc7Smrj void 894ae115bc7Smrj dosoftint(struct regs *regs) 895ae115bc7Smrj { 896ae115bc7Smrj struct cpu *cpu = CPU; 897ae115bc7Smrj int oldipl; 898ae115bc7Smrj caddr_t newsp; 899ae115bc7Smrj 900ae115bc7Smrj while (cpu->cpu_softinfo.st_pending) { 901ae115bc7Smrj oldipl = cpu->cpu_pri; 902ae115bc7Smrj newsp = dosoftint_prolog(cpu, (caddr_t)regs, 903ae115bc7Smrj cpu->cpu_softinfo.st_pending, oldipl); 904ae115bc7Smrj /* 905ae115bc7Smrj * If returned stack pointer is NULL, priority is too high 906ae115bc7Smrj * to run any of the pending softints now. 907ae115bc7Smrj * Break out and they will be run later. 908ae115bc7Smrj */ 909ae115bc7Smrj if (newsp == NULL) 910ae115bc7Smrj break; 911ae115bc7Smrj switch_sp_and_call(newsp, dispatch_softint, oldipl, 0); 912ae115bc7Smrj } 913ae115bc7Smrj } 914ae115bc7Smrj 915ae115bc7Smrj /* 916ae115bc7Smrj * Interrupt service routine, called with interrupts disabled. 917ae115bc7Smrj */ 918ae115bc7Smrj /*ARGSUSED*/ 919ae115bc7Smrj void 920ae115bc7Smrj do_interrupt(struct regs *rp, trap_trace_rec_t *ttp) 921ae115bc7Smrj { 922ae115bc7Smrj struct cpu *cpu = CPU; 923ae115bc7Smrj int newipl, oldipl = cpu->cpu_pri; 924ae115bc7Smrj uint_t vector; 925ae115bc7Smrj caddr_t newsp; 926ae115bc7Smrj 927ae115bc7Smrj #ifdef TRAPTRACE 928ae115bc7Smrj ttp->ttr_marker = TT_INTERRUPT; 929ae115bc7Smrj ttp->ttr_ipl = 0xff; 930ae115bc7Smrj ttp->ttr_pri = oldipl; 931ae115bc7Smrj ttp->ttr_spl = cpu->cpu_base_spl; 932ae115bc7Smrj ttp->ttr_vector = 0xff; 933ae115bc7Smrj #endif /* TRAPTRACE */ 934ae115bc7Smrj 935*fb2caebeSRandy Fishel cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR); 93695c0a3c8Sjosephb 93795c0a3c8Sjosephb /* 938ae115bc7Smrj * If it's a softint go do it now. 939ae115bc7Smrj */ 940ae115bc7Smrj if (rp->r_trapno == T_SOFTINT) { 941ae115bc7Smrj dosoftint(rp); 942ae115bc7Smrj ASSERT(!interrupts_enabled()); 943ae115bc7Smrj return; 944ae115bc7Smrj } 945ae115bc7Smrj 946ae115bc7Smrj /* 947ae115bc7Smrj * Raise the interrupt priority. 948ae115bc7Smrj */ 949ae115bc7Smrj newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno); 950ae115bc7Smrj #ifdef TRAPTRACE 951ae115bc7Smrj ttp->ttr_ipl = newipl; 952ae115bc7Smrj #endif /* TRAPTRACE */ 953ae115bc7Smrj 954ae115bc7Smrj /* 955ae115bc7Smrj * Bail if it is a spurious interrupt 956ae115bc7Smrj */ 957ae115bc7Smrj if (newipl == -1) 958ae115bc7Smrj return; 959ae115bc7Smrj cpu->cpu_pri = newipl; 960ae115bc7Smrj vector = rp->r_trapno; 961ae115bc7Smrj #ifdef TRAPTRACE 962ae115bc7Smrj ttp->ttr_vector = vector; 963ae115bc7Smrj #endif /* TRAPTRACE */ 964ae115bc7Smrj if (newipl > LOCK_LEVEL) { 965ae115bc7Smrj /* 966ae115bc7Smrj * High priority interrupts run on this cpu's interrupt stack. 967ae115bc7Smrj */ 968ae115bc7Smrj if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) { 969ae115bc7Smrj newsp = cpu->cpu_intr_stack; 970ae115bc7Smrj switch_sp_and_call(newsp, dispatch_hilevel, vector, 0); 971ae115bc7Smrj } else { /* already on the interrupt stack */ 972ae115bc7Smrj dispatch_hilevel(vector, 0); 973ae115bc7Smrj } 974ae115bc7Smrj (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector); 975ae115bc7Smrj } else { 976ae115bc7Smrj /* 977ae115bc7Smrj * Run this interrupt in a separate thread. 978ae115bc7Smrj */ 979ae115bc7Smrj newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl); 980ae115bc7Smrj switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl); 981ae115bc7Smrj } 982ae115bc7Smrj 983ae115bc7Smrj /* 984ae115bc7Smrj * Deliver any pending soft interrupts. 985ae115bc7Smrj */ 986ae115bc7Smrj if (cpu->cpu_softinfo.st_pending) 987ae115bc7Smrj dosoftint(rp); 988ae115bc7Smrj } 989ae115bc7Smrj 990ae115bc7Smrj /* 991ae115bc7Smrj * Common tasks always done by _sys_rtt, called with interrupts disabled. 992ae115bc7Smrj * Returns 1 if returning to userland, 0 if returning to system mode. 993ae115bc7Smrj */ 994ae115bc7Smrj int 995ae115bc7Smrj sys_rtt_common(struct regs *rp) 996ae115bc7Smrj { 997ae115bc7Smrj kthread_t *tp; 998ae115bc7Smrj extern void mutex_exit_critical_start(); 999ae115bc7Smrj extern long mutex_exit_critical_size; 1000575a7426Spt157919 extern void mutex_owner_running_critical_start(); 1001575a7426Spt157919 extern long mutex_owner_running_critical_size; 1002ae115bc7Smrj 1003ae115bc7Smrj loop: 1004ae115bc7Smrj 1005ae115bc7Smrj /* 1006ae115bc7Smrj * Check if returning to user 1007ae115bc7Smrj */ 1008ae115bc7Smrj tp = CPU->cpu_thread; 1009ae115bc7Smrj if (USERMODE(rp->r_cs)) { 1010ae115bc7Smrj /* 1011ae115bc7Smrj * Check if AST pending. 1012ae115bc7Smrj */ 1013ae115bc7Smrj if (tp->t_astflag) { 1014ae115bc7Smrj /* 1015ae115bc7Smrj * Let trap() handle the AST 1016ae115bc7Smrj */ 1017ae115bc7Smrj sti(); 1018ae115bc7Smrj rp->r_trapno = T_AST; 1019ae115bc7Smrj trap(rp, (caddr_t)0, CPU->cpu_id); 1020ae115bc7Smrj cli(); 1021ae115bc7Smrj goto loop; 1022ae115bc7Smrj } 1023ae115bc7Smrj 1024ae115bc7Smrj #if defined(__amd64) 1025ae115bc7Smrj /* 1026ae115bc7Smrj * We are done if segment registers do not need updating. 1027ae115bc7Smrj */ 10287712e92cSsudheer if (tp->t_lwp->lwp_pcb.pcb_rupdate == 0) 1029ae115bc7Smrj return (1); 1030ae115bc7Smrj 1031ae115bc7Smrj if (update_sregs(rp, tp->t_lwp)) { 1032ae115bc7Smrj /* 1033ae115bc7Smrj * 1 or more of the selectors is bad. 1034ae115bc7Smrj * Deliver a SIGSEGV. 1035ae115bc7Smrj */ 1036ae115bc7Smrj proc_t *p = ttoproc(tp); 1037ae115bc7Smrj 1038ae115bc7Smrj sti(); 1039ae115bc7Smrj mutex_enter(&p->p_lock); 1040ae115bc7Smrj tp->t_lwp->lwp_cursig = SIGSEGV; 1041ae115bc7Smrj mutex_exit(&p->p_lock); 1042ae115bc7Smrj psig(); 1043ae115bc7Smrj tp->t_sig_check = 1; 1044ae115bc7Smrj cli(); 1045ae115bc7Smrj } 10467712e92cSsudheer tp->t_lwp->lwp_pcb.pcb_rupdate = 0; 1047ae115bc7Smrj 1048ae115bc7Smrj #endif /* __amd64 */ 1049ae115bc7Smrj return (1); 1050ae115bc7Smrj } 1051ae115bc7Smrj 1052ae115bc7Smrj /* 1053ae115bc7Smrj * Here if we are returning to supervisor mode. 1054ae115bc7Smrj * Check for a kernel preemption request. 1055ae115bc7Smrj */ 1056ae115bc7Smrj if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) { 1057ae115bc7Smrj 1058ae115bc7Smrj /* 1059ae115bc7Smrj * Do nothing if already in kpreempt 1060ae115bc7Smrj */ 1061ae115bc7Smrj if (!tp->t_preempt_lk) { 1062ae115bc7Smrj tp->t_preempt_lk = 1; 1063ae115bc7Smrj sti(); 1064ae115bc7Smrj kpreempt(1); /* asynchronous kpreempt call */ 1065ae115bc7Smrj cli(); 1066ae115bc7Smrj tp->t_preempt_lk = 0; 1067ae115bc7Smrj } 1068ae115bc7Smrj } 1069ae115bc7Smrj 1070ae115bc7Smrj /* 1071ae115bc7Smrj * If we interrupted the mutex_exit() critical region we must 1072ae115bc7Smrj * reset the PC back to the beginning to prevent missed wakeups 1073ae115bc7Smrj * See the comments in mutex_exit() for details. 1074ae115bc7Smrj */ 1075ae115bc7Smrj if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start < 1076ae115bc7Smrj mutex_exit_critical_size) { 1077ae115bc7Smrj rp->r_pc = (greg_t)mutex_exit_critical_start; 1078ae115bc7Smrj } 1079575a7426Spt157919 1080575a7426Spt157919 /* 1081575a7426Spt157919 * If we interrupted the mutex_owner_running() critical region we 1082575a7426Spt157919 * must reset the PC back to the beginning to prevent dereferencing 1083575a7426Spt157919 * of a freed thread pointer. See the comments in mutex_owner_running 1084575a7426Spt157919 * for details. 1085575a7426Spt157919 */ 1086575a7426Spt157919 if ((uintptr_t)rp->r_pc - 1087575a7426Spt157919 (uintptr_t)mutex_owner_running_critical_start < 1088575a7426Spt157919 mutex_owner_running_critical_size) { 1089575a7426Spt157919 rp->r_pc = (greg_t)mutex_owner_running_critical_start; 1090575a7426Spt157919 } 1091575a7426Spt157919 1092ae115bc7Smrj return (0); 1093ae115bc7Smrj } 1094ae115bc7Smrj 1095ae115bc7Smrj void 1096ae115bc7Smrj send_dirint(int cpuid, int int_level) 1097ae115bc7Smrj { 1098ae115bc7Smrj (*send_dirintf)(cpuid, int_level); 1099ae115bc7Smrj } 1100ae115bc7Smrj 1101ae115bc7Smrj /* 1102ae115bc7Smrj * do_splx routine, takes new ipl to set 1103ae115bc7Smrj * returns the old ipl. 1104ae115bc7Smrj * We are careful not to set priority lower than CPU->cpu_base_pri, 1105ae115bc7Smrj * even though it seems we're raising the priority, it could be set 1106ae115bc7Smrj * higher at any time by an interrupt routine, so we must block interrupts 1107ae115bc7Smrj * and look at CPU->cpu_base_pri 1108ae115bc7Smrj */ 1109ae115bc7Smrj int 1110ae115bc7Smrj do_splx(int newpri) 1111ae115bc7Smrj { 1112ae115bc7Smrj ulong_t flag; 1113ae115bc7Smrj cpu_t *cpu; 1114ae115bc7Smrj int curpri, basepri; 1115ae115bc7Smrj 1116ae115bc7Smrj flag = intr_clear(); 1117ae115bc7Smrj cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */ 1118ae115bc7Smrj curpri = cpu->cpu_m.mcpu_pri; 1119ae115bc7Smrj basepri = cpu->cpu_base_spl; 1120ae115bc7Smrj if (newpri < basepri) 1121ae115bc7Smrj newpri = basepri; 1122ae115bc7Smrj cpu->cpu_m.mcpu_pri = newpri; 1123ae115bc7Smrj (*setspl)(newpri); 1124ae115bc7Smrj /* 1125ae115bc7Smrj * If we are going to reenable interrupts see if new priority level 1126ae115bc7Smrj * allows pending softint delivery. 1127ae115bc7Smrj */ 1128ae115bc7Smrj if ((flag & PS_IE) && 1129ae115bc7Smrj bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > newpri) 1130ae115bc7Smrj fakesoftint(); 1131ae115bc7Smrj ASSERT(!interrupts_enabled()); 1132ae115bc7Smrj intr_restore(flag); 1133ae115bc7Smrj return (curpri); 1134ae115bc7Smrj } 1135ae115bc7Smrj 1136ae115bc7Smrj /* 1137ae115bc7Smrj * Common spl raise routine, takes new ipl to set 1138ae115bc7Smrj * returns the old ipl, will not lower ipl. 1139ae115bc7Smrj */ 1140ae115bc7Smrj int 1141ae115bc7Smrj splr(int newpri) 1142ae115bc7Smrj { 1143ae115bc7Smrj ulong_t flag; 1144ae115bc7Smrj cpu_t *cpu; 1145ae115bc7Smrj int curpri, basepri; 1146ae115bc7Smrj 1147ae115bc7Smrj flag = intr_clear(); 1148ae115bc7Smrj cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */ 1149ae115bc7Smrj curpri = cpu->cpu_m.mcpu_pri; 1150ae115bc7Smrj /* 1151ae115bc7Smrj * Only do something if new priority is larger 1152ae115bc7Smrj */ 1153ae115bc7Smrj if (newpri > curpri) { 1154ae115bc7Smrj basepri = cpu->cpu_base_spl; 1155ae115bc7Smrj if (newpri < basepri) 1156ae115bc7Smrj newpri = basepri; 1157ae115bc7Smrj cpu->cpu_m.mcpu_pri = newpri; 1158ae115bc7Smrj (*setspl)(newpri); 1159ae115bc7Smrj /* 1160ae115bc7Smrj * See if new priority level allows pending softint delivery 1161ae115bc7Smrj */ 1162ae115bc7Smrj if ((flag & PS_IE) && 1163ae115bc7Smrj bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > newpri) 1164ae115bc7Smrj fakesoftint(); 1165ae115bc7Smrj } 1166ae115bc7Smrj intr_restore(flag); 1167ae115bc7Smrj return (curpri); 1168ae115bc7Smrj } 1169ae115bc7Smrj 1170ae115bc7Smrj int 1171ae115bc7Smrj getpil(void) 1172ae115bc7Smrj { 1173ae115bc7Smrj return (CPU->cpu_m.mcpu_pri); 1174ae115bc7Smrj } 1175ae115bc7Smrj 1176ae115bc7Smrj int 1177ae115bc7Smrj interrupts_enabled(void) 1178ae115bc7Smrj { 1179ae115bc7Smrj ulong_t flag; 1180ae115bc7Smrj 1181ae115bc7Smrj flag = getflags(); 1182ae115bc7Smrj return ((flag & PS_IE) == PS_IE); 1183ae115bc7Smrj } 1184ae115bc7Smrj 1185ae115bc7Smrj #ifdef DEBUG 1186ae115bc7Smrj void 1187ae115bc7Smrj assert_ints_enabled(void) 1188ae115bc7Smrj { 1189ae115bc7Smrj ASSERT(!interrupts_unleashed || interrupts_enabled()); 1190ae115bc7Smrj } 1191ae115bc7Smrj #endif /* DEBUG */ 1192