xref: /titanic_44/usr/src/uts/i86pc/os/intr.c (revision fb2caebe9e38ee2e6e469d5136fb247faaa7299b)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5100b72f4Sandrei  * Common Development and Distribution License (the "License").
6100b72f4Sandrei  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21843e1988Sjohnlev 
227c478bd9Sstevel@tonic-gate /*
23b9e93c10SJonathan Haslam  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
28*fb2caebeSRandy Fishel #include <sys/cpu_event.h>
297c478bd9Sstevel@tonic-gate #include <sys/regset.h>
307c478bd9Sstevel@tonic-gate #include <sys/psw.h>
317c478bd9Sstevel@tonic-gate #include <sys/types.h>
327c478bd9Sstevel@tonic-gate #include <sys/thread.h>
337c478bd9Sstevel@tonic-gate #include <sys/systm.h>
347c478bd9Sstevel@tonic-gate #include <sys/segments.h>
357c478bd9Sstevel@tonic-gate #include <sys/pcb.h>
367c478bd9Sstevel@tonic-gate #include <sys/trap.h>
377c478bd9Sstevel@tonic-gate #include <sys/ftrace.h>
387c478bd9Sstevel@tonic-gate #include <sys/traptrace.h>
397c478bd9Sstevel@tonic-gate #include <sys/clock.h>
407c478bd9Sstevel@tonic-gate #include <sys/panic.h>
417c478bd9Sstevel@tonic-gate #include <sys/disp.h>
427c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
437c478bd9Sstevel@tonic-gate #include <sys/stack.h>
447c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
457c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
467c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
477c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h>
487c478bd9Sstevel@tonic-gate #include <sys/pool_pset.h>
497c478bd9Sstevel@tonic-gate #include <sys/zone.h>
507c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
51ae115bc7Smrj #include <sys/archsystm.h>
52ae115bc7Smrj #include <sys/machsystm.h>
53ae115bc7Smrj #include <sys/ontrap.h>
54ae115bc7Smrj #include <sys/x86_archext.h>
55ae115bc7Smrj #include <sys/promif.h>
5695c0a3c8Sjosephb #include <vm/hat_i86.h>
57843e1988Sjohnlev #if defined(__xpv)
58843e1988Sjohnlev #include <sys/hypervisor.h>
59843e1988Sjohnlev #endif
607c478bd9Sstevel@tonic-gate 
617c478bd9Sstevel@tonic-gate 
62843e1988Sjohnlev #if defined(__xpv) && defined(DEBUG)
63843e1988Sjohnlev 
64843e1988Sjohnlev /*
65843e1988Sjohnlev  * This panic message is intended as an aid to interrupt debugging.
66843e1988Sjohnlev  *
67843e1988Sjohnlev  * The associated assertion tests the condition of enabling
68843e1988Sjohnlev  * events when events are already enabled.  The implication
69843e1988Sjohnlev  * being that whatever code the programmer thought was
70843e1988Sjohnlev  * protected by having events disabled until the second
71843e1988Sjohnlev  * enable happened really wasn't protected at all ..
72843e1988Sjohnlev  */
73843e1988Sjohnlev 
74843e1988Sjohnlev int stistipanic = 1;	/* controls the debug panic check */
75843e1988Sjohnlev const char *stistimsg = "stisti";
76843e1988Sjohnlev ulong_t laststi[NCPU];
77843e1988Sjohnlev 
78843e1988Sjohnlev /*
79843e1988Sjohnlev  * This variable tracks the last place events were disabled on each cpu
80*fb2caebeSRandy Fishel  * it assists in debugging when asserts that interrupts are enabled trip.
81843e1988Sjohnlev  */
82843e1988Sjohnlev ulong_t lastcli[NCPU];
83843e1988Sjohnlev 
84843e1988Sjohnlev #endif
85843e1988Sjohnlev 
867c478bd9Sstevel@tonic-gate /*
87ae115bc7Smrj  * Set cpu's base SPL level to the highest active interrupt level
887c478bd9Sstevel@tonic-gate  */
89ae115bc7Smrj void
90ae115bc7Smrj set_base_spl(void)
917c478bd9Sstevel@tonic-gate {
92ae115bc7Smrj 	struct cpu *cpu = CPU;
93ae115bc7Smrj 	uint16_t active = (uint16_t)cpu->cpu_intr_actv;
947c478bd9Sstevel@tonic-gate 
95ae115bc7Smrj 	cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);
967c478bd9Sstevel@tonic-gate }
977c478bd9Sstevel@tonic-gate 
987c478bd9Sstevel@tonic-gate /*
997c478bd9Sstevel@tonic-gate  * Do all the work necessary to set up the cpu and thread structures
1007c478bd9Sstevel@tonic-gate  * to dispatch a high-level interrupt.
1017c478bd9Sstevel@tonic-gate  *
1027c478bd9Sstevel@tonic-gate  * Returns 0 if we're -not- already on the high-level interrupt stack,
1037c478bd9Sstevel@tonic-gate  * (and *must* switch to it), non-zero if we are already on that stack.
1047c478bd9Sstevel@tonic-gate  *
1057c478bd9Sstevel@tonic-gate  * Called with interrupts masked.
1067c478bd9Sstevel@tonic-gate  * The 'pil' is already set to the appropriate level for rp->r_trapno.
1077c478bd9Sstevel@tonic-gate  */
108ae115bc7Smrj static int
1097c478bd9Sstevel@tonic-gate hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
1107c478bd9Sstevel@tonic-gate {
1117c478bd9Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
1127c478bd9Sstevel@tonic-gate 	uint_t mask;
113eda89462Sesolom 	hrtime_t intrtime;
114ae115bc7Smrj 	hrtime_t now = tsc_read();
1157c478bd9Sstevel@tonic-gate 
1167c478bd9Sstevel@tonic-gate 	ASSERT(pil > LOCK_LEVEL);
1177c478bd9Sstevel@tonic-gate 
1187c478bd9Sstevel@tonic-gate 	if (pil == CBE_HIGH_PIL) {
1197c478bd9Sstevel@tonic-gate 		cpu->cpu_profile_pil = oldpil;
1207c478bd9Sstevel@tonic-gate 		if (USERMODE(rp->r_cs)) {
1217c478bd9Sstevel@tonic-gate 			cpu->cpu_profile_pc = 0;
1227c478bd9Sstevel@tonic-gate 			cpu->cpu_profile_upc = rp->r_pc;
123b9e93c10SJonathan Haslam 			cpu->cpu_cpcprofile_pc = 0;
124b9e93c10SJonathan Haslam 			cpu->cpu_cpcprofile_upc = rp->r_pc;
1257c478bd9Sstevel@tonic-gate 		} else {
1267c478bd9Sstevel@tonic-gate 			cpu->cpu_profile_pc = rp->r_pc;
1277c478bd9Sstevel@tonic-gate 			cpu->cpu_profile_upc = 0;
128b9e93c10SJonathan Haslam 			cpu->cpu_cpcprofile_pc = rp->r_pc;
129b9e93c10SJonathan Haslam 			cpu->cpu_cpcprofile_upc = 0;
1307c478bd9Sstevel@tonic-gate 		}
1317c478bd9Sstevel@tonic-gate 	}
1327c478bd9Sstevel@tonic-gate 
1337c478bd9Sstevel@tonic-gate 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
1347c478bd9Sstevel@tonic-gate 	if (mask != 0) {
1357c478bd9Sstevel@tonic-gate 		int nestpil;
1367c478bd9Sstevel@tonic-gate 
1377c478bd9Sstevel@tonic-gate 		/*
1387c478bd9Sstevel@tonic-gate 		 * We have interrupted another high-level interrupt.
1397c478bd9Sstevel@tonic-gate 		 * Load starting timestamp, compute interval, update
1407c478bd9Sstevel@tonic-gate 		 * cumulative counter.
1417c478bd9Sstevel@tonic-gate 		 */
1427c478bd9Sstevel@tonic-gate 		nestpil = bsrw_insn((uint16_t)mask);
1437c478bd9Sstevel@tonic-gate 		ASSERT(nestpil < pil);
144ae115bc7Smrj 		intrtime = now -
1457c478bd9Sstevel@tonic-gate 		    mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
1467a364d25Sschwartz 		mcpu->intrstat[nestpil][0] += intrtime;
147eda89462Sesolom 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1487c478bd9Sstevel@tonic-gate 		/*
1497c478bd9Sstevel@tonic-gate 		 * Another high-level interrupt is active below this one, so
1507c478bd9Sstevel@tonic-gate 		 * there is no need to check for an interrupt thread.  That
1517c478bd9Sstevel@tonic-gate 		 * will be done by the lowest priority high-level interrupt
1527c478bd9Sstevel@tonic-gate 		 * active.
1537c478bd9Sstevel@tonic-gate 		 */
1547c478bd9Sstevel@tonic-gate 	} else {
1557c478bd9Sstevel@tonic-gate 		kthread_t *t = cpu->cpu_thread;
1567c478bd9Sstevel@tonic-gate 
1577c478bd9Sstevel@tonic-gate 		/*
1587c478bd9Sstevel@tonic-gate 		 * See if we are interrupting a low-level interrupt thread.
1597c478bd9Sstevel@tonic-gate 		 * If so, account for its time slice only if its time stamp
1607c478bd9Sstevel@tonic-gate 		 * is non-zero.
1617c478bd9Sstevel@tonic-gate 		 */
1627c478bd9Sstevel@tonic-gate 		if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
163ae115bc7Smrj 			intrtime = now - t->t_intr_start;
1647a364d25Sschwartz 			mcpu->intrstat[t->t_pil][0] += intrtime;
165eda89462Sesolom 			cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1667c478bd9Sstevel@tonic-gate 			t->t_intr_start = 0;
1677c478bd9Sstevel@tonic-gate 		}
1687c478bd9Sstevel@tonic-gate 	}
1697c478bd9Sstevel@tonic-gate 
1707c478bd9Sstevel@tonic-gate 	/*
1717c478bd9Sstevel@tonic-gate 	 * Store starting timestamp in CPU structure for this PIL.
1727c478bd9Sstevel@tonic-gate 	 */
173ae115bc7Smrj 	mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
1747c478bd9Sstevel@tonic-gate 
1757c478bd9Sstevel@tonic-gate 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
1767c478bd9Sstevel@tonic-gate 
1777c478bd9Sstevel@tonic-gate 	if (pil == 15) {
1787c478bd9Sstevel@tonic-gate 		/*
1797c478bd9Sstevel@tonic-gate 		 * To support reentrant level 15 interrupts, we maintain a
1807c478bd9Sstevel@tonic-gate 		 * recursion count in the top half of cpu_intr_actv.  Only
1817c478bd9Sstevel@tonic-gate 		 * when this count hits zero do we clear the PIL 15 bit from
1827c478bd9Sstevel@tonic-gate 		 * the lower half of cpu_intr_actv.
1837c478bd9Sstevel@tonic-gate 		 */
1847c478bd9Sstevel@tonic-gate 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
1857c478bd9Sstevel@tonic-gate 		(*refcntp)++;
1867c478bd9Sstevel@tonic-gate 	}
1877c478bd9Sstevel@tonic-gate 
1887c478bd9Sstevel@tonic-gate 	mask = cpu->cpu_intr_actv;
1897c478bd9Sstevel@tonic-gate 
1907c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_actv |= (1 << pil);
1917c478bd9Sstevel@tonic-gate 
1927c478bd9Sstevel@tonic-gate 	return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
1937c478bd9Sstevel@tonic-gate }
1947c478bd9Sstevel@tonic-gate 
1957c478bd9Sstevel@tonic-gate /*
1967c478bd9Sstevel@tonic-gate  * Does most of the work of returning from a high level interrupt.
1977c478bd9Sstevel@tonic-gate  *
1987c478bd9Sstevel@tonic-gate  * Returns 0 if there are no more high level interrupts (in which
1997c478bd9Sstevel@tonic-gate  * case we must switch back to the interrupted thread stack) or
2007c478bd9Sstevel@tonic-gate  * non-zero if there are more (in which case we should stay on it).
2017c478bd9Sstevel@tonic-gate  *
2027c478bd9Sstevel@tonic-gate  * Called with interrupts masked
2037c478bd9Sstevel@tonic-gate  */
204ae115bc7Smrj static int
2057c478bd9Sstevel@tonic-gate hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
2067c478bd9Sstevel@tonic-gate {
2077c478bd9Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
2087c478bd9Sstevel@tonic-gate 	uint_t mask;
209eda89462Sesolom 	hrtime_t intrtime;
210ae115bc7Smrj 	hrtime_t now = tsc_read();
2117c478bd9Sstevel@tonic-gate 
2127c478bd9Sstevel@tonic-gate 	ASSERT(mcpu->mcpu_pri == pil);
2137c478bd9Sstevel@tonic-gate 
2147c478bd9Sstevel@tonic-gate 	cpu->cpu_stats.sys.intr[pil - 1]++;
2157c478bd9Sstevel@tonic-gate 
2167c478bd9Sstevel@tonic-gate 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
2177c478bd9Sstevel@tonic-gate 
2187c478bd9Sstevel@tonic-gate 	if (pil == 15) {
2197c478bd9Sstevel@tonic-gate 		/*
2207c478bd9Sstevel@tonic-gate 		 * To support reentrant level 15 interrupts, we maintain a
2217c478bd9Sstevel@tonic-gate 		 * recursion count in the top half of cpu_intr_actv.  Only
2227c478bd9Sstevel@tonic-gate 		 * when this count hits zero do we clear the PIL 15 bit from
2237c478bd9Sstevel@tonic-gate 		 * the lower half of cpu_intr_actv.
2247c478bd9Sstevel@tonic-gate 		 */
2257c478bd9Sstevel@tonic-gate 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
2267c478bd9Sstevel@tonic-gate 
2277c478bd9Sstevel@tonic-gate 		ASSERT(*refcntp > 0);
2287c478bd9Sstevel@tonic-gate 
2297c478bd9Sstevel@tonic-gate 		if (--(*refcntp) == 0)
2307c478bd9Sstevel@tonic-gate 			cpu->cpu_intr_actv &= ~(1 << pil);
2317c478bd9Sstevel@tonic-gate 	} else {
2327c478bd9Sstevel@tonic-gate 		cpu->cpu_intr_actv &= ~(1 << pil);
2337c478bd9Sstevel@tonic-gate 	}
2347c478bd9Sstevel@tonic-gate 
2357c478bd9Sstevel@tonic-gate 	ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
2367c478bd9Sstevel@tonic-gate 
237ae115bc7Smrj 	intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
2387a364d25Sschwartz 	mcpu->intrstat[pil][0] += intrtime;
239eda89462Sesolom 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate 	/*
2427c478bd9Sstevel@tonic-gate 	 * Check for lower-pil nested high-level interrupt beneath
2437c478bd9Sstevel@tonic-gate 	 * current one.  If so, place a starting timestamp in its
2447c478bd9Sstevel@tonic-gate 	 * pil_high_start entry.
2457c478bd9Sstevel@tonic-gate 	 */
2467c478bd9Sstevel@tonic-gate 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
2477c478bd9Sstevel@tonic-gate 	if (mask != 0) {
2487c478bd9Sstevel@tonic-gate 		int nestpil;
2497c478bd9Sstevel@tonic-gate 
2507c478bd9Sstevel@tonic-gate 		/*
2517c478bd9Sstevel@tonic-gate 		 * find PIL of nested interrupt
2527c478bd9Sstevel@tonic-gate 		 */
2537c478bd9Sstevel@tonic-gate 		nestpil = bsrw_insn((uint16_t)mask);
2547c478bd9Sstevel@tonic-gate 		ASSERT(nestpil < pil);
255ae115bc7Smrj 		mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
2567c478bd9Sstevel@tonic-gate 		/*
2577c478bd9Sstevel@tonic-gate 		 * (Another high-level interrupt is active below this one,
2587c478bd9Sstevel@tonic-gate 		 * so there is no need to check for an interrupt
2597c478bd9Sstevel@tonic-gate 		 * thread.  That will be done by the lowest priority
2607c478bd9Sstevel@tonic-gate 		 * high-level interrupt active.)
2617c478bd9Sstevel@tonic-gate 		 */
2627c478bd9Sstevel@tonic-gate 	} else {
2637c478bd9Sstevel@tonic-gate 		/*
2647c478bd9Sstevel@tonic-gate 		 * Check to see if there is a low-level interrupt active.
2657c478bd9Sstevel@tonic-gate 		 * If so, place a starting timestamp in the thread
2667c478bd9Sstevel@tonic-gate 		 * structure.
2677c478bd9Sstevel@tonic-gate 		 */
2687c478bd9Sstevel@tonic-gate 		kthread_t *t = cpu->cpu_thread;
2697c478bd9Sstevel@tonic-gate 
2707c478bd9Sstevel@tonic-gate 		if (t->t_flag & T_INTR_THREAD)
271ae115bc7Smrj 			t->t_intr_start = now;
2727c478bd9Sstevel@tonic-gate 	}
2737c478bd9Sstevel@tonic-gate 
2747c478bd9Sstevel@tonic-gate 	mcpu->mcpu_pri = oldpil;
2757c478bd9Sstevel@tonic-gate 	(void) (*setlvlx)(oldpil, vecnum);
2767c478bd9Sstevel@tonic-gate 
2777c478bd9Sstevel@tonic-gate 	return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
2787c478bd9Sstevel@tonic-gate }
2797c478bd9Sstevel@tonic-gate 
2807c478bd9Sstevel@tonic-gate /*
2817c478bd9Sstevel@tonic-gate  * Set up the cpu, thread and interrupt thread structures for
2827c478bd9Sstevel@tonic-gate  * executing an interrupt thread.  The new stack pointer of the
2837c478bd9Sstevel@tonic-gate  * interrupt thread (which *must* be switched to) is returned.
2847c478bd9Sstevel@tonic-gate  */
285ae115bc7Smrj static caddr_t
2867c478bd9Sstevel@tonic-gate intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
2877c478bd9Sstevel@tonic-gate {
2887c478bd9Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
2897c478bd9Sstevel@tonic-gate 	kthread_t *t, *volatile it;
290ae115bc7Smrj 	hrtime_t now = tsc_read();
2917c478bd9Sstevel@tonic-gate 
2927c478bd9Sstevel@tonic-gate 	ASSERT(pil > 0);
2937c478bd9Sstevel@tonic-gate 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
2947c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_actv |= (1 << pil);
2957c478bd9Sstevel@tonic-gate 
2967c478bd9Sstevel@tonic-gate 	/*
2977c478bd9Sstevel@tonic-gate 	 * Get set to run an interrupt thread.
2987c478bd9Sstevel@tonic-gate 	 * There should always be an interrupt thread, since we
2997c478bd9Sstevel@tonic-gate 	 * allocate one for each level on each CPU.
3007c478bd9Sstevel@tonic-gate 	 *
301fd71cd2fSesolom 	 * t_intr_start could be zero due to cpu_intr_swtch_enter.
3027c478bd9Sstevel@tonic-gate 	 */
3037c478bd9Sstevel@tonic-gate 	t = cpu->cpu_thread;
304fd71cd2fSesolom 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
305ae115bc7Smrj 		hrtime_t intrtime = now - t->t_intr_start;
3067a364d25Sschwartz 		mcpu->intrstat[t->t_pil][0] += intrtime;
307eda89462Sesolom 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
3087c478bd9Sstevel@tonic-gate 		t->t_intr_start = 0;
3097c478bd9Sstevel@tonic-gate 	}
3107c478bd9Sstevel@tonic-gate 
3117c478bd9Sstevel@tonic-gate 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
3127c478bd9Sstevel@tonic-gate 
3137c478bd9Sstevel@tonic-gate 	t->t_sp = (uintptr_t)stackptr;	/* mark stack in curthread for resume */
3147c478bd9Sstevel@tonic-gate 
3157c478bd9Sstevel@tonic-gate 	/*
3167c478bd9Sstevel@tonic-gate 	 * unlink the interrupt thread off the cpu
317fd71cd2fSesolom 	 *
318fd71cd2fSesolom 	 * Note that the code in kcpc_overflow_intr -relies- on the
319fd71cd2fSesolom 	 * ordering of events here - in particular that t->t_lwp of
320fd71cd2fSesolom 	 * the interrupt thread is set to the pinned thread *before*
321fd71cd2fSesolom 	 * curthread is changed.
3227c478bd9Sstevel@tonic-gate 	 */
3237c478bd9Sstevel@tonic-gate 	it = cpu->cpu_intr_thread;
3247c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_thread = it->t_link;
3257c478bd9Sstevel@tonic-gate 	it->t_intr = t;
3267c478bd9Sstevel@tonic-gate 	it->t_lwp = t->t_lwp;
3277c478bd9Sstevel@tonic-gate 
3287c478bd9Sstevel@tonic-gate 	/*
3297c478bd9Sstevel@tonic-gate 	 * (threads on the interrupt thread free list could have state
3307c478bd9Sstevel@tonic-gate 	 * preset to TS_ONPROC, but it helps in debugging if
3317c478bd9Sstevel@tonic-gate 	 * they're TS_FREE.)
3327c478bd9Sstevel@tonic-gate 	 */
3337c478bd9Sstevel@tonic-gate 	it->t_state = TS_ONPROC;
3347c478bd9Sstevel@tonic-gate 
3357c478bd9Sstevel@tonic-gate 	cpu->cpu_thread = it;		/* new curthread on this cpu */
3367c478bd9Sstevel@tonic-gate 	it->t_pil = (uchar_t)pil;
3377c478bd9Sstevel@tonic-gate 	it->t_pri = intr_pri + (pri_t)pil;
338ae115bc7Smrj 	it->t_intr_start = now;
3397c478bd9Sstevel@tonic-gate 
3407c478bd9Sstevel@tonic-gate 	return (it->t_stk);
3417c478bd9Sstevel@tonic-gate }
3427c478bd9Sstevel@tonic-gate 
3437c478bd9Sstevel@tonic-gate 
3447c478bd9Sstevel@tonic-gate #ifdef DEBUG
3457c478bd9Sstevel@tonic-gate int intr_thread_cnt;
3467c478bd9Sstevel@tonic-gate #endif
3477c478bd9Sstevel@tonic-gate 
3487c478bd9Sstevel@tonic-gate /*
3497c478bd9Sstevel@tonic-gate  * Called with interrupts disabled
3507c478bd9Sstevel@tonic-gate  */
351ae115bc7Smrj static void
3527c478bd9Sstevel@tonic-gate intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
3537c478bd9Sstevel@tonic-gate {
3547c478bd9Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
3557c478bd9Sstevel@tonic-gate 	kthread_t *t;
3567c478bd9Sstevel@tonic-gate 	kthread_t *it = cpu->cpu_thread;	/* curthread */
3577c478bd9Sstevel@tonic-gate 	uint_t pil, basespl;
358eda89462Sesolom 	hrtime_t intrtime;
359ae115bc7Smrj 	hrtime_t now = tsc_read();
3607c478bd9Sstevel@tonic-gate 
3617c478bd9Sstevel@tonic-gate 	pil = it->t_pil;
3627c478bd9Sstevel@tonic-gate 	cpu->cpu_stats.sys.intr[pil - 1]++;
3637c478bd9Sstevel@tonic-gate 
3647c478bd9Sstevel@tonic-gate 	ASSERT(it->t_intr_start != 0);
365ae115bc7Smrj 	intrtime = now - it->t_intr_start;
3667a364d25Sschwartz 	mcpu->intrstat[pil][0] += intrtime;
367eda89462Sesolom 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
3687c478bd9Sstevel@tonic-gate 
3697c478bd9Sstevel@tonic-gate 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
3707c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_actv &= ~(1 << pil);
3717c478bd9Sstevel@tonic-gate 
3727c478bd9Sstevel@tonic-gate 	/*
3737c478bd9Sstevel@tonic-gate 	 * If there is still an interrupted thread underneath this one
3747c478bd9Sstevel@tonic-gate 	 * then the interrupt was never blocked and the return is
3757c478bd9Sstevel@tonic-gate 	 * fairly simple.  Otherwise it isn't.
3767c478bd9Sstevel@tonic-gate 	 */
3777c478bd9Sstevel@tonic-gate 	if ((t = it->t_intr) == NULL) {
3787c478bd9Sstevel@tonic-gate 		/*
3797c478bd9Sstevel@tonic-gate 		 * The interrupted thread is no longer pinned underneath
3807c478bd9Sstevel@tonic-gate 		 * the interrupt thread.  This means the interrupt must
3817c478bd9Sstevel@tonic-gate 		 * have blocked, and the interrupted thread has been
3827c478bd9Sstevel@tonic-gate 		 * unpinned, and has probably been running around the
3837c478bd9Sstevel@tonic-gate 		 * system for a while.
3847c478bd9Sstevel@tonic-gate 		 *
3857c478bd9Sstevel@tonic-gate 		 * Since there is no longer a thread under this one, put
3867c478bd9Sstevel@tonic-gate 		 * this interrupt thread back on the CPU's free list and
3877c478bd9Sstevel@tonic-gate 		 * resume the idle thread which will dispatch the next
3887c478bd9Sstevel@tonic-gate 		 * thread to run.
3897c478bd9Sstevel@tonic-gate 		 */
3907c478bd9Sstevel@tonic-gate #ifdef DEBUG
3917c478bd9Sstevel@tonic-gate 		intr_thread_cnt++;
3927c478bd9Sstevel@tonic-gate #endif
3937c478bd9Sstevel@tonic-gate 		cpu->cpu_stats.sys.intrblk++;
3947c478bd9Sstevel@tonic-gate 		/*
3957c478bd9Sstevel@tonic-gate 		 * Set CPU's base SPL based on active interrupts bitmask
3967c478bd9Sstevel@tonic-gate 		 */
3977c478bd9Sstevel@tonic-gate 		set_base_spl();
3987c478bd9Sstevel@tonic-gate 		basespl = cpu->cpu_base_spl;
3997c478bd9Sstevel@tonic-gate 		mcpu->mcpu_pri = basespl;
4007c478bd9Sstevel@tonic-gate 		(*setlvlx)(basespl, vec);
4017c478bd9Sstevel@tonic-gate 		(void) splhigh();
402ae115bc7Smrj 		sti();
4037c478bd9Sstevel@tonic-gate 		it->t_state = TS_FREE;
4047c478bd9Sstevel@tonic-gate 		/*
4057c478bd9Sstevel@tonic-gate 		 * Return interrupt thread to pool
4067c478bd9Sstevel@tonic-gate 		 */
4077c478bd9Sstevel@tonic-gate 		it->t_link = cpu->cpu_intr_thread;
4087c478bd9Sstevel@tonic-gate 		cpu->cpu_intr_thread = it;
4097c478bd9Sstevel@tonic-gate 		swtch();
410ae115bc7Smrj 		panic("intr_thread_epilog: swtch returned");
4117c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
4127c478bd9Sstevel@tonic-gate 	}
4137c478bd9Sstevel@tonic-gate 
4147c478bd9Sstevel@tonic-gate 	/*
4157c478bd9Sstevel@tonic-gate 	 * Return interrupt thread to the pool
4167c478bd9Sstevel@tonic-gate 	 */
4177c478bd9Sstevel@tonic-gate 	it->t_link = cpu->cpu_intr_thread;
4187c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_thread = it;
4197c478bd9Sstevel@tonic-gate 	it->t_state = TS_FREE;
4207c478bd9Sstevel@tonic-gate 
4217c478bd9Sstevel@tonic-gate 	basespl = cpu->cpu_base_spl;
4227c478bd9Sstevel@tonic-gate 	pil = MAX(oldpil, basespl);
4237c478bd9Sstevel@tonic-gate 	mcpu->mcpu_pri = pil;
4247c478bd9Sstevel@tonic-gate 	(*setlvlx)(pil, vec);
425ae115bc7Smrj 	t->t_intr_start = now;
4267c478bd9Sstevel@tonic-gate 	cpu->cpu_thread = t;
4277c478bd9Sstevel@tonic-gate }
4287c478bd9Sstevel@tonic-gate 
4297a364d25Sschwartz /*
430ae115bc7Smrj  * intr_get_time() is a resource for interrupt handlers to determine how
431ae115bc7Smrj  * much time has been spent handling the current interrupt. Such a function
432ae115bc7Smrj  * is needed because higher level interrupts can arrive during the
433ae115bc7Smrj  * processing of an interrupt.  intr_get_time() only returns time spent in the
434ae115bc7Smrj  * current interrupt handler.
435ae115bc7Smrj  *
436ae115bc7Smrj  * The caller must be calling from an interrupt handler running at a pil
437ae115bc7Smrj  * below or at lock level. Timings are not provided for high-level
438ae115bc7Smrj  * interrupts.
439ae115bc7Smrj  *
440ae115bc7Smrj  * The first time intr_get_time() is called while handling an interrupt,
441ae115bc7Smrj  * it returns the time since the interrupt handler was invoked. Subsequent
442ae115bc7Smrj  * calls will return the time since the prior call to intr_get_time(). Time
443843e1988Sjohnlev  * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
444ae115bc7Smrj  *
445ae115bc7Smrj  * Theory Of Intrstat[][]:
446ae115bc7Smrj  *
447ae115bc7Smrj  * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
448ae115bc7Smrj  * uint64_ts per pil.
449ae115bc7Smrj  *
450ae115bc7Smrj  * intrstat[pil][0] is a cumulative count of the number of ticks spent
451ae115bc7Smrj  * handling all interrupts at the specified pil on this CPU. It is
452ae115bc7Smrj  * exported via kstats to the user.
453ae115bc7Smrj  *
454ae115bc7Smrj  * intrstat[pil][1] is always a count of ticks less than or equal to the
455ae115bc7Smrj  * value in [0]. The difference between [1] and [0] is the value returned
456ae115bc7Smrj  * by a call to intr_get_time(). At the start of interrupt processing,
457ae115bc7Smrj  * [0] and [1] will be equal (or nearly so). As the interrupt consumes
458ae115bc7Smrj  * time, [0] will increase, but [1] will remain the same. A call to
459ae115bc7Smrj  * intr_get_time() will return the difference, then update [1] to be the
460ae115bc7Smrj  * same as [0]. Future calls will return the time since the last call.
461ae115bc7Smrj  * Finally, when the interrupt completes, [1] is updated to the same as [0].
462ae115bc7Smrj  *
463ae115bc7Smrj  * Implementation:
464ae115bc7Smrj  *
465ae115bc7Smrj  * intr_get_time() works much like a higher level interrupt arriving. It
466ae115bc7Smrj  * "checkpoints" the timing information by incrementing intrstat[pil][0]
467ae115bc7Smrj  * to include elapsed running time, and by setting t_intr_start to rdtsc.
468ae115bc7Smrj  * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
469ae115bc7Smrj  * and updates intrstat[pil][1] to be the same as the new value of
470ae115bc7Smrj  * intrstat[pil][0].
471ae115bc7Smrj  *
472ae115bc7Smrj  * In the normal handling of interrupts, after an interrupt handler returns
473ae115bc7Smrj  * and the code in intr_thread() updates intrstat[pil][0], it then sets
474ae115bc7Smrj  * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
475ae115bc7Smrj  * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
476ae115bc7Smrj  * is 0.
477ae115bc7Smrj  *
478ae115bc7Smrj  * Whenever interrupts arrive on a CPU which is handling a lower pil
479ae115bc7Smrj  * interrupt, they update the lower pil's [0] to show time spent in the
480ae115bc7Smrj  * handler that they've interrupted. This results in a growing discrepancy
481ae115bc7Smrj  * between [0] and [1], which is returned the next time intr_get_time() is
482ae115bc7Smrj  * called. Time spent in the higher-pil interrupt will not be returned in
483ae115bc7Smrj  * the next intr_get_time() call from the original interrupt, because
484ae115bc7Smrj  * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
4857a364d25Sschwartz  */
4867a364d25Sschwartz uint64_t
487ae115bc7Smrj intr_get_time(void)
4887a364d25Sschwartz {
489ae115bc7Smrj 	struct cpu *cpu;
490ae115bc7Smrj 	struct machcpu *mcpu;
491ae115bc7Smrj 	kthread_t *t;
4927a364d25Sschwartz 	uint64_t time, delta, ret;
493ae115bc7Smrj 	uint_t pil;
4947a364d25Sschwartz 
495ae115bc7Smrj 	cli();
496ae115bc7Smrj 	cpu = CPU;
497ae115bc7Smrj 	mcpu = &cpu->cpu_m;
498ae115bc7Smrj 	t = cpu->cpu_thread;
499ae115bc7Smrj 	pil = t->t_pil;
5007a364d25Sschwartz 	ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
5017a364d25Sschwartz 	ASSERT(t->t_flag & T_INTR_THREAD);
5027a364d25Sschwartz 	ASSERT(pil != 0);
5037a364d25Sschwartz 	ASSERT(t->t_intr_start != 0);
5047a364d25Sschwartz 
5057a364d25Sschwartz 	time = tsc_read();
5067a364d25Sschwartz 	delta = time - t->t_intr_start;
5077a364d25Sschwartz 	t->t_intr_start = time;
5087a364d25Sschwartz 
5097a364d25Sschwartz 	time = mcpu->intrstat[pil][0] + delta;
5107a364d25Sschwartz 	ret = time - mcpu->intrstat[pil][1];
5117a364d25Sschwartz 	mcpu->intrstat[pil][0] = time;
5127a364d25Sschwartz 	mcpu->intrstat[pil][1] = time;
513c81508f4Sjhaslam 	cpu->cpu_intracct[cpu->cpu_mstate] += delta;
5147a364d25Sschwartz 
515ae115bc7Smrj 	sti();
5167a364d25Sschwartz 	return (ret);
5177a364d25Sschwartz }
5187a364d25Sschwartz 
519ae115bc7Smrj static caddr_t
5207c478bd9Sstevel@tonic-gate dosoftint_prolog(
5217c478bd9Sstevel@tonic-gate 	struct cpu *cpu,
5227c478bd9Sstevel@tonic-gate 	caddr_t stackptr,
5237c478bd9Sstevel@tonic-gate 	uint32_t st_pending,
5247c478bd9Sstevel@tonic-gate 	uint_t oldpil)
5257c478bd9Sstevel@tonic-gate {
5267c478bd9Sstevel@tonic-gate 	kthread_t *t, *volatile it;
5277c478bd9Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
5287c478bd9Sstevel@tonic-gate 	uint_t pil;
529ae115bc7Smrj 	hrtime_t now;
5307c478bd9Sstevel@tonic-gate 
5317c478bd9Sstevel@tonic-gate top:
5327c478bd9Sstevel@tonic-gate 	ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
5337c478bd9Sstevel@tonic-gate 
5347c478bd9Sstevel@tonic-gate 	pil = bsrw_insn((uint16_t)st_pending);
5357c478bd9Sstevel@tonic-gate 	if (pil <= oldpil || pil <= cpu->cpu_base_spl)
5367c478bd9Sstevel@tonic-gate 		return (0);
5377c478bd9Sstevel@tonic-gate 
5387c478bd9Sstevel@tonic-gate 	/*
5397c478bd9Sstevel@tonic-gate 	 * XX64	Sigh.
5407c478bd9Sstevel@tonic-gate 	 *
5417c478bd9Sstevel@tonic-gate 	 * This is a transliteration of the i386 assembler code for
5427c478bd9Sstevel@tonic-gate 	 * soft interrupts.  One question is "why does this need
5437c478bd9Sstevel@tonic-gate 	 * to be atomic?"  One possible race is -other- processors
5447c478bd9Sstevel@tonic-gate 	 * posting soft interrupts to us in set_pending() i.e. the
5457c478bd9Sstevel@tonic-gate 	 * CPU might get preempted just after the address computation,
5467c478bd9Sstevel@tonic-gate 	 * but just before the atomic transaction, so another CPU would
5477c478bd9Sstevel@tonic-gate 	 * actually set the original CPU's st_pending bit.  However,
5487c478bd9Sstevel@tonic-gate 	 * it looks like it would be simpler to disable preemption there.
5497c478bd9Sstevel@tonic-gate 	 * Are there other races for which preemption control doesn't work?
5507c478bd9Sstevel@tonic-gate 	 *
5517c478bd9Sstevel@tonic-gate 	 * The i386 assembler version -also- checks to see if the bit
5527c478bd9Sstevel@tonic-gate 	 * being cleared was actually set; if it wasn't, it rechecks
5537c478bd9Sstevel@tonic-gate 	 * for more.  This seems a bit strange, as the only code that
5547c478bd9Sstevel@tonic-gate 	 * ever clears the bit is -this- code running with interrupts
5557c478bd9Sstevel@tonic-gate 	 * disabled on -this- CPU.  This code would probably be cheaper:
5567c478bd9Sstevel@tonic-gate 	 *
5577c478bd9Sstevel@tonic-gate 	 * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
5587c478bd9Sstevel@tonic-gate 	 *   ~(1 << pil));
5597c478bd9Sstevel@tonic-gate 	 *
5607c478bd9Sstevel@tonic-gate 	 * and t->t_preempt--/++ around set_pending() even cheaper,
5617c478bd9Sstevel@tonic-gate 	 * but at this point, correctness is critical, so we slavishly
5627c478bd9Sstevel@tonic-gate 	 * emulate the i386 port.
5637c478bd9Sstevel@tonic-gate 	 */
564ae115bc7Smrj 	if (atomic_btr32((uint32_t *)
565ae115bc7Smrj 	    &mcpu->mcpu_softinfo.st_pending, pil) == 0) {
5667c478bd9Sstevel@tonic-gate 		st_pending = mcpu->mcpu_softinfo.st_pending;
5677c478bd9Sstevel@tonic-gate 		goto top;
5687c478bd9Sstevel@tonic-gate 	}
5697c478bd9Sstevel@tonic-gate 
5707c478bd9Sstevel@tonic-gate 	mcpu->mcpu_pri = pil;
5717c478bd9Sstevel@tonic-gate 	(*setspl)(pil);
5727c478bd9Sstevel@tonic-gate 
573ae115bc7Smrj 	now = tsc_read();
574ae115bc7Smrj 
5757c478bd9Sstevel@tonic-gate 	/*
5767c478bd9Sstevel@tonic-gate 	 * Get set to run interrupt thread.
5777c478bd9Sstevel@tonic-gate 	 * There should always be an interrupt thread since we
5787c478bd9Sstevel@tonic-gate 	 * allocate one for each level on the CPU.
5797c478bd9Sstevel@tonic-gate 	 */
5807c478bd9Sstevel@tonic-gate 	it = cpu->cpu_intr_thread;
5817c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_thread = it->t_link;
5827c478bd9Sstevel@tonic-gate 
583fd71cd2fSesolom 	/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
584fd71cd2fSesolom 	t = cpu->cpu_thread;
585fd71cd2fSesolom 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
586ae115bc7Smrj 		hrtime_t intrtime = now - t->t_intr_start;
587fd71cd2fSesolom 		mcpu->intrstat[pil][0] += intrtime;
588fd71cd2fSesolom 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
589fd71cd2fSesolom 		t->t_intr_start = 0;
590fd71cd2fSesolom 	}
591fd71cd2fSesolom 
5927c478bd9Sstevel@tonic-gate 	/*
5937c478bd9Sstevel@tonic-gate 	 * Note that the code in kcpc_overflow_intr -relies- on the
5947c478bd9Sstevel@tonic-gate 	 * ordering of events here - in particular that t->t_lwp of
5957c478bd9Sstevel@tonic-gate 	 * the interrupt thread is set to the pinned thread *before*
596fd71cd2fSesolom 	 * curthread is changed.
5977c478bd9Sstevel@tonic-gate 	 */
5987c478bd9Sstevel@tonic-gate 	it->t_lwp = t->t_lwp;
5997c478bd9Sstevel@tonic-gate 	it->t_state = TS_ONPROC;
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate 	/*
6027c478bd9Sstevel@tonic-gate 	 * Push interrupted thread onto list from new thread.
6037c478bd9Sstevel@tonic-gate 	 * Set the new thread as the current one.
6047c478bd9Sstevel@tonic-gate 	 * Set interrupted thread's T_SP because if it is the idle thread,
6057c478bd9Sstevel@tonic-gate 	 * resume() may use that stack between threads.
6067c478bd9Sstevel@tonic-gate 	 */
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
6097c478bd9Sstevel@tonic-gate 	t->t_sp = (uintptr_t)stackptr;
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate 	it->t_intr = t;
6127c478bd9Sstevel@tonic-gate 	cpu->cpu_thread = it;
6137c478bd9Sstevel@tonic-gate 
6147c478bd9Sstevel@tonic-gate 	/*
6157c478bd9Sstevel@tonic-gate 	 * Set bit for this pil in CPU's interrupt active bitmask.
6167c478bd9Sstevel@tonic-gate 	 */
6177c478bd9Sstevel@tonic-gate 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
6187c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_actv |= (1 << pil);
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate 	/*
6217c478bd9Sstevel@tonic-gate 	 * Initialize thread priority level from intr_pri
6227c478bd9Sstevel@tonic-gate 	 */
6237c478bd9Sstevel@tonic-gate 	it->t_pil = (uchar_t)pil;
6247c478bd9Sstevel@tonic-gate 	it->t_pri = (pri_t)pil + intr_pri;
625ae115bc7Smrj 	it->t_intr_start = now;
6267c478bd9Sstevel@tonic-gate 
6277c478bd9Sstevel@tonic-gate 	return (it->t_stk);
6287c478bd9Sstevel@tonic-gate }
6297c478bd9Sstevel@tonic-gate 
630ae115bc7Smrj static void
6317c478bd9Sstevel@tonic-gate dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
6327c478bd9Sstevel@tonic-gate {
6337c478bd9Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
6347c478bd9Sstevel@tonic-gate 	kthread_t *t, *it;
6357c478bd9Sstevel@tonic-gate 	uint_t pil, basespl;
636eda89462Sesolom 	hrtime_t intrtime;
637ae115bc7Smrj 	hrtime_t now = tsc_read();
6387c478bd9Sstevel@tonic-gate 
6397c478bd9Sstevel@tonic-gate 	it = cpu->cpu_thread;
6407c478bd9Sstevel@tonic-gate 	pil = it->t_pil;
6417c478bd9Sstevel@tonic-gate 
6427c478bd9Sstevel@tonic-gate 	cpu->cpu_stats.sys.intr[pil - 1]++;
6437c478bd9Sstevel@tonic-gate 
6447c478bd9Sstevel@tonic-gate 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
6457c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_actv &= ~(1 << pil);
646ae115bc7Smrj 	intrtime = now - it->t_intr_start;
6477a364d25Sschwartz 	mcpu->intrstat[pil][0] += intrtime;
648eda89462Sesolom 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
6497c478bd9Sstevel@tonic-gate 
6507c478bd9Sstevel@tonic-gate 	/*
6517c478bd9Sstevel@tonic-gate 	 * If there is still an interrupted thread underneath this one
6527c478bd9Sstevel@tonic-gate 	 * then the interrupt was never blocked and the return is
6537c478bd9Sstevel@tonic-gate 	 * fairly simple.  Otherwise it isn't.
6547c478bd9Sstevel@tonic-gate 	 */
6557c478bd9Sstevel@tonic-gate 	if ((t = it->t_intr) == NULL) {
6567c478bd9Sstevel@tonic-gate 		/*
6577c478bd9Sstevel@tonic-gate 		 * Put thread back on the interrupt thread list.
6587c478bd9Sstevel@tonic-gate 		 * This was an interrupt thread, so set CPU's base SPL.
6597c478bd9Sstevel@tonic-gate 		 */
6607c478bd9Sstevel@tonic-gate 		set_base_spl();
6617c478bd9Sstevel@tonic-gate 		it->t_state = TS_FREE;
6627c478bd9Sstevel@tonic-gate 		it->t_link = cpu->cpu_intr_thread;
6637c478bd9Sstevel@tonic-gate 		cpu->cpu_intr_thread = it;
6647c478bd9Sstevel@tonic-gate 		(void) splhigh();
665ae115bc7Smrj 		sti();
6667c478bd9Sstevel@tonic-gate 		swtch();
6677c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
668ae115bc7Smrj 		panic("dosoftint_epilog: swtch returned");
6697c478bd9Sstevel@tonic-gate 	}
6707c478bd9Sstevel@tonic-gate 	it->t_link = cpu->cpu_intr_thread;
6717c478bd9Sstevel@tonic-gate 	cpu->cpu_intr_thread = it;
6727c478bd9Sstevel@tonic-gate 	it->t_state = TS_FREE;
6737c478bd9Sstevel@tonic-gate 	cpu->cpu_thread = t;
6747c478bd9Sstevel@tonic-gate 	if (t->t_flag & T_INTR_THREAD)
675ae115bc7Smrj 		t->t_intr_start = now;
6767c478bd9Sstevel@tonic-gate 	basespl = cpu->cpu_base_spl;
6777c478bd9Sstevel@tonic-gate 	pil = MAX(oldpil, basespl);
6787c478bd9Sstevel@tonic-gate 	mcpu->mcpu_pri = pil;
6797c478bd9Sstevel@tonic-gate 	(*setspl)(pil);
6807c478bd9Sstevel@tonic-gate }
6817c478bd9Sstevel@tonic-gate 
682ae115bc7Smrj 
6837c478bd9Sstevel@tonic-gate /*
6847c478bd9Sstevel@tonic-gate  * Make the interrupted thread 'to' be runnable.
6857c478bd9Sstevel@tonic-gate  *
6867c478bd9Sstevel@tonic-gate  * Since t->t_sp has already been saved, t->t_pc is all
6877c478bd9Sstevel@tonic-gate  * that needs to be set in this function.
6887c478bd9Sstevel@tonic-gate  *
6897c478bd9Sstevel@tonic-gate  * Returns the interrupt level of the interrupt thread.
6907c478bd9Sstevel@tonic-gate  */
6917c478bd9Sstevel@tonic-gate int
6927c478bd9Sstevel@tonic-gate intr_passivate(
6937c478bd9Sstevel@tonic-gate 	kthread_t *it,		/* interrupt thread */
6947c478bd9Sstevel@tonic-gate 	kthread_t *t)		/* interrupted thread */
6957c478bd9Sstevel@tonic-gate {
6967c478bd9Sstevel@tonic-gate 	extern void _sys_rtt();
6977c478bd9Sstevel@tonic-gate 
6987c478bd9Sstevel@tonic-gate 	ASSERT(it->t_flag & T_INTR_THREAD);
6997c478bd9Sstevel@tonic-gate 	ASSERT(SA(t->t_sp) == t->t_sp);
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	t->t_pc = (uintptr_t)_sys_rtt;
7027c478bd9Sstevel@tonic-gate 	return (it->t_pil);
7037c478bd9Sstevel@tonic-gate }
7047c478bd9Sstevel@tonic-gate 
7057c478bd9Sstevel@tonic-gate /*
7067c478bd9Sstevel@tonic-gate  * Create interrupt kstats for this CPU.
7077c478bd9Sstevel@tonic-gate  */
7087c478bd9Sstevel@tonic-gate void
7097c478bd9Sstevel@tonic-gate cpu_create_intrstat(cpu_t *cp)
7107c478bd9Sstevel@tonic-gate {
7117c478bd9Sstevel@tonic-gate 	int		i;
7127c478bd9Sstevel@tonic-gate 	kstat_t		*intr_ksp;
7137c478bd9Sstevel@tonic-gate 	kstat_named_t	*knp;
7147c478bd9Sstevel@tonic-gate 	char		name[KSTAT_STRLEN];
7157c478bd9Sstevel@tonic-gate 	zoneid_t	zoneid;
7167c478bd9Sstevel@tonic-gate 
7177c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
7187c478bd9Sstevel@tonic-gate 
7197c478bd9Sstevel@tonic-gate 	if (pool_pset_enabled())
7207c478bd9Sstevel@tonic-gate 		zoneid = GLOBAL_ZONEID;
7217c478bd9Sstevel@tonic-gate 	else
7227c478bd9Sstevel@tonic-gate 		zoneid = ALL_ZONES;
7237c478bd9Sstevel@tonic-gate 
7247c478bd9Sstevel@tonic-gate 	intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
7257c478bd9Sstevel@tonic-gate 	    KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid);
7267c478bd9Sstevel@tonic-gate 
7277c478bd9Sstevel@tonic-gate 	/*
7287c478bd9Sstevel@tonic-gate 	 * Initialize each PIL's named kstat
7297c478bd9Sstevel@tonic-gate 	 */
7307c478bd9Sstevel@tonic-gate 	if (intr_ksp != NULL) {
7317c478bd9Sstevel@tonic-gate 		intr_ksp->ks_update = cpu_kstat_intrstat_update;
7327c478bd9Sstevel@tonic-gate 		knp = (kstat_named_t *)intr_ksp->ks_data;
7337c478bd9Sstevel@tonic-gate 		intr_ksp->ks_private = cp;
7347c478bd9Sstevel@tonic-gate 		for (i = 0; i < PIL_MAX; i++) {
7357c478bd9Sstevel@tonic-gate 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
7367c478bd9Sstevel@tonic-gate 			    i + 1);
7377c478bd9Sstevel@tonic-gate 			kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
7387c478bd9Sstevel@tonic-gate 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
7397c478bd9Sstevel@tonic-gate 			    i + 1);
7407c478bd9Sstevel@tonic-gate 			kstat_named_init(&knp[(i * 2) + 1], name,
7417c478bd9Sstevel@tonic-gate 			    KSTAT_DATA_UINT64);
7427c478bd9Sstevel@tonic-gate 		}
7437c478bd9Sstevel@tonic-gate 		kstat_install(intr_ksp);
7447c478bd9Sstevel@tonic-gate 	}
7457c478bd9Sstevel@tonic-gate }
7467c478bd9Sstevel@tonic-gate 
7477c478bd9Sstevel@tonic-gate /*
7487c478bd9Sstevel@tonic-gate  * Delete interrupt kstats for this CPU.
7497c478bd9Sstevel@tonic-gate  */
7507c478bd9Sstevel@tonic-gate void
7517c478bd9Sstevel@tonic-gate cpu_delete_intrstat(cpu_t *cp)
7527c478bd9Sstevel@tonic-gate {
7537c478bd9Sstevel@tonic-gate 	kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
7547c478bd9Sstevel@tonic-gate }
7557c478bd9Sstevel@tonic-gate 
7567c478bd9Sstevel@tonic-gate /*
7577c478bd9Sstevel@tonic-gate  * Convert interrupt statistics from CPU ticks to nanoseconds and
7587c478bd9Sstevel@tonic-gate  * update kstat.
7597c478bd9Sstevel@tonic-gate  */
7607c478bd9Sstevel@tonic-gate int
7617c478bd9Sstevel@tonic-gate cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
7627c478bd9Sstevel@tonic-gate {
7637c478bd9Sstevel@tonic-gate 	kstat_named_t	*knp = ksp->ks_data;
7647c478bd9Sstevel@tonic-gate 	cpu_t		*cpup = (cpu_t *)ksp->ks_private;
7657c478bd9Sstevel@tonic-gate 	int		i;
7667c478bd9Sstevel@tonic-gate 	hrtime_t	hrt;
7677c478bd9Sstevel@tonic-gate 
7687c478bd9Sstevel@tonic-gate 	if (rw == KSTAT_WRITE)
7697c478bd9Sstevel@tonic-gate 		return (EACCES);
7707c478bd9Sstevel@tonic-gate 
7717c478bd9Sstevel@tonic-gate 	for (i = 0; i < PIL_MAX; i++) {
7727a364d25Sschwartz 		hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
773843e1988Sjohnlev 		scalehrtimef(&hrt);
7747c478bd9Sstevel@tonic-gate 		knp[i * 2].value.ui64 = (uint64_t)hrt;
7757c478bd9Sstevel@tonic-gate 		knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
7767c478bd9Sstevel@tonic-gate 	}
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate 	return (0);
7797c478bd9Sstevel@tonic-gate }
7807c478bd9Sstevel@tonic-gate 
7817c478bd9Sstevel@tonic-gate /*
7827c478bd9Sstevel@tonic-gate  * An interrupt thread is ending a time slice, so compute the interval it
7837c478bd9Sstevel@tonic-gate  * ran for and update the statistic for its PIL.
7847c478bd9Sstevel@tonic-gate  */
7857c478bd9Sstevel@tonic-gate void
7867c478bd9Sstevel@tonic-gate cpu_intr_swtch_enter(kthread_id_t t)
7877c478bd9Sstevel@tonic-gate {
7887c478bd9Sstevel@tonic-gate 	uint64_t	interval;
7897c478bd9Sstevel@tonic-gate 	uint64_t	start;
790eda89462Sesolom 	cpu_t		*cpu;
7917c478bd9Sstevel@tonic-gate 
7927c478bd9Sstevel@tonic-gate 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
7937c478bd9Sstevel@tonic-gate 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
7947c478bd9Sstevel@tonic-gate 
7957c478bd9Sstevel@tonic-gate 	/*
7967c478bd9Sstevel@tonic-gate 	 * We could be here with a zero timestamp. This could happen if:
7977c478bd9Sstevel@tonic-gate 	 * an interrupt thread which no longer has a pinned thread underneath
7987c478bd9Sstevel@tonic-gate 	 * it (i.e. it blocked at some point in its past) has finished running
7997c478bd9Sstevel@tonic-gate 	 * its handler. intr_thread() updated the interrupt statistic for its
8007c478bd9Sstevel@tonic-gate 	 * PIL and zeroed its timestamp. Since there was no pinned thread to
8017c478bd9Sstevel@tonic-gate 	 * return to, swtch() gets called and we end up here.
802eda89462Sesolom 	 *
803eda89462Sesolom 	 * Note that we use atomic ops below (cas64 and atomic_add_64), which
804eda89462Sesolom 	 * we don't use in the functions above, because we're not called
805eda89462Sesolom 	 * with interrupts blocked, but the epilog/prolog functions are.
8067c478bd9Sstevel@tonic-gate 	 */
8077c478bd9Sstevel@tonic-gate 	if (t->t_intr_start) {
8087c478bd9Sstevel@tonic-gate 		do {
8097c478bd9Sstevel@tonic-gate 			start = t->t_intr_start;
8107c478bd9Sstevel@tonic-gate 			interval = tsc_read() - start;
8117c478bd9Sstevel@tonic-gate 		} while (cas64(&t->t_intr_start, start, 0) != start);
812eda89462Sesolom 		cpu = CPU;
8137a364d25Sschwartz 		cpu->cpu_m.intrstat[t->t_pil][0] += interval;
814eda89462Sesolom 
815eda89462Sesolom 		atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
816eda89462Sesolom 		    interval);
8177c478bd9Sstevel@tonic-gate 	} else
8187c478bd9Sstevel@tonic-gate 		ASSERT(t->t_intr == NULL);
8197c478bd9Sstevel@tonic-gate }
8207c478bd9Sstevel@tonic-gate 
8217c478bd9Sstevel@tonic-gate /*
8227c478bd9Sstevel@tonic-gate  * An interrupt thread is returning from swtch(). Place a starting timestamp
8237c478bd9Sstevel@tonic-gate  * in its thread structure.
8247c478bd9Sstevel@tonic-gate  */
8257c478bd9Sstevel@tonic-gate void
8267c478bd9Sstevel@tonic-gate cpu_intr_swtch_exit(kthread_id_t t)
8277c478bd9Sstevel@tonic-gate {
8287c478bd9Sstevel@tonic-gate 	uint64_t ts;
8297c478bd9Sstevel@tonic-gate 
8307c478bd9Sstevel@tonic-gate 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
8317c478bd9Sstevel@tonic-gate 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
8327c478bd9Sstevel@tonic-gate 
8337c478bd9Sstevel@tonic-gate 	do {
8347c478bd9Sstevel@tonic-gate 		ts = t->t_intr_start;
8357c478bd9Sstevel@tonic-gate 	} while (cas64(&t->t_intr_start, ts, tsc_read()) != ts);
8367c478bd9Sstevel@tonic-gate }
837ae115bc7Smrj 
838ae115bc7Smrj /*
839ae115bc7Smrj  * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
840ae115bc7Smrj  */
841ae115bc7Smrj /*ARGSUSED*/
842ae115bc7Smrj static void
843ae115bc7Smrj dispatch_hilevel(uint_t vector, uint_t arg2)
844ae115bc7Smrj {
845ae115bc7Smrj 	sti();
846ae115bc7Smrj 	av_dispatch_autovect(vector);
847ae115bc7Smrj 	cli();
848ae115bc7Smrj }
849ae115bc7Smrj 
850ae115bc7Smrj /*
851ae115bc7Smrj  * Dispatch a soft interrupt
852ae115bc7Smrj  */
853ae115bc7Smrj /*ARGSUSED*/
854ae115bc7Smrj static void
855ae115bc7Smrj dispatch_softint(uint_t oldpil, uint_t arg2)
856ae115bc7Smrj {
857ae115bc7Smrj 	struct cpu *cpu = CPU;
858ae115bc7Smrj 
859ae115bc7Smrj 	sti();
860ae115bc7Smrj 	av_dispatch_softvect((int)cpu->cpu_thread->t_pil);
861ae115bc7Smrj 	cli();
862ae115bc7Smrj 
863ae115bc7Smrj 	/*
864ae115bc7Smrj 	 * Must run softint_epilog() on the interrupt thread stack, since
865ae115bc7Smrj 	 * there may not be a return from it if the interrupt thread blocked.
866ae115bc7Smrj 	 */
867ae115bc7Smrj 	dosoftint_epilog(cpu, oldpil);
868ae115bc7Smrj }
869ae115bc7Smrj 
870ae115bc7Smrj /*
871ae115bc7Smrj  * Dispatch a normal interrupt
872ae115bc7Smrj  */
873ae115bc7Smrj static void
874ae115bc7Smrj dispatch_hardint(uint_t vector, uint_t oldipl)
875ae115bc7Smrj {
876ae115bc7Smrj 	struct cpu *cpu = CPU;
877ae115bc7Smrj 
878ae115bc7Smrj 	sti();
879ae115bc7Smrj 	av_dispatch_autovect(vector);
880ae115bc7Smrj 	cli();
881ae115bc7Smrj 
882ae115bc7Smrj 	/*
883ae115bc7Smrj 	 * Must run intr_thread_epilog() on the interrupt thread stack, since
884ae115bc7Smrj 	 * there may not be a return from it if the interrupt thread blocked.
885ae115bc7Smrj 	 */
886ae115bc7Smrj 	intr_thread_epilog(cpu, vector, oldipl);
887ae115bc7Smrj }
888ae115bc7Smrj 
889ae115bc7Smrj /*
890ae115bc7Smrj  * Deliver any softints the current interrupt priority allows.
891ae115bc7Smrj  * Called with interrupts disabled.
892ae115bc7Smrj  */
893ae115bc7Smrj void
894ae115bc7Smrj dosoftint(struct regs *regs)
895ae115bc7Smrj {
896ae115bc7Smrj 	struct cpu *cpu = CPU;
897ae115bc7Smrj 	int oldipl;
898ae115bc7Smrj 	caddr_t newsp;
899ae115bc7Smrj 
900ae115bc7Smrj 	while (cpu->cpu_softinfo.st_pending) {
901ae115bc7Smrj 		oldipl = cpu->cpu_pri;
902ae115bc7Smrj 		newsp = dosoftint_prolog(cpu, (caddr_t)regs,
903ae115bc7Smrj 		    cpu->cpu_softinfo.st_pending, oldipl);
904ae115bc7Smrj 		/*
905ae115bc7Smrj 		 * If returned stack pointer is NULL, priority is too high
906ae115bc7Smrj 		 * to run any of the pending softints now.
907ae115bc7Smrj 		 * Break out and they will be run later.
908ae115bc7Smrj 		 */
909ae115bc7Smrj 		if (newsp == NULL)
910ae115bc7Smrj 			break;
911ae115bc7Smrj 		switch_sp_and_call(newsp, dispatch_softint, oldipl, 0);
912ae115bc7Smrj 	}
913ae115bc7Smrj }
914ae115bc7Smrj 
915ae115bc7Smrj /*
916ae115bc7Smrj  * Interrupt service routine, called with interrupts disabled.
917ae115bc7Smrj  */
918ae115bc7Smrj /*ARGSUSED*/
919ae115bc7Smrj void
920ae115bc7Smrj do_interrupt(struct regs *rp, trap_trace_rec_t *ttp)
921ae115bc7Smrj {
922ae115bc7Smrj 	struct cpu *cpu = CPU;
923ae115bc7Smrj 	int newipl, oldipl = cpu->cpu_pri;
924ae115bc7Smrj 	uint_t vector;
925ae115bc7Smrj 	caddr_t newsp;
926ae115bc7Smrj 
927ae115bc7Smrj #ifdef TRAPTRACE
928ae115bc7Smrj 	ttp->ttr_marker = TT_INTERRUPT;
929ae115bc7Smrj 	ttp->ttr_ipl = 0xff;
930ae115bc7Smrj 	ttp->ttr_pri = oldipl;
931ae115bc7Smrj 	ttp->ttr_spl = cpu->cpu_base_spl;
932ae115bc7Smrj 	ttp->ttr_vector = 0xff;
933ae115bc7Smrj #endif	/* TRAPTRACE */
934ae115bc7Smrj 
935*fb2caebeSRandy Fishel 	cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR);
93695c0a3c8Sjosephb 
93795c0a3c8Sjosephb 	/*
938ae115bc7Smrj 	 * If it's a softint go do it now.
939ae115bc7Smrj 	 */
940ae115bc7Smrj 	if (rp->r_trapno == T_SOFTINT) {
941ae115bc7Smrj 		dosoftint(rp);
942ae115bc7Smrj 		ASSERT(!interrupts_enabled());
943ae115bc7Smrj 		return;
944ae115bc7Smrj 	}
945ae115bc7Smrj 
946ae115bc7Smrj 	/*
947ae115bc7Smrj 	 * Raise the interrupt priority.
948ae115bc7Smrj 	 */
949ae115bc7Smrj 	newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno);
950ae115bc7Smrj #ifdef TRAPTRACE
951ae115bc7Smrj 	ttp->ttr_ipl = newipl;
952ae115bc7Smrj #endif	/* TRAPTRACE */
953ae115bc7Smrj 
954ae115bc7Smrj 	/*
955ae115bc7Smrj 	 * Bail if it is a spurious interrupt
956ae115bc7Smrj 	 */
957ae115bc7Smrj 	if (newipl == -1)
958ae115bc7Smrj 		return;
959ae115bc7Smrj 	cpu->cpu_pri = newipl;
960ae115bc7Smrj 	vector = rp->r_trapno;
961ae115bc7Smrj #ifdef TRAPTRACE
962ae115bc7Smrj 	ttp->ttr_vector = vector;
963ae115bc7Smrj #endif	/* TRAPTRACE */
964ae115bc7Smrj 	if (newipl > LOCK_LEVEL) {
965ae115bc7Smrj 		/*
966ae115bc7Smrj 		 * High priority interrupts run on this cpu's interrupt stack.
967ae115bc7Smrj 		 */
968ae115bc7Smrj 		if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {
969ae115bc7Smrj 			newsp = cpu->cpu_intr_stack;
970ae115bc7Smrj 			switch_sp_and_call(newsp, dispatch_hilevel, vector, 0);
971ae115bc7Smrj 		} else { /* already on the interrupt stack */
972ae115bc7Smrj 			dispatch_hilevel(vector, 0);
973ae115bc7Smrj 		}
974ae115bc7Smrj 		(void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);
975ae115bc7Smrj 	} else {
976ae115bc7Smrj 		/*
977ae115bc7Smrj 		 * Run this interrupt in a separate thread.
978ae115bc7Smrj 		 */
979ae115bc7Smrj 		newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);
980ae115bc7Smrj 		switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl);
981ae115bc7Smrj 	}
982ae115bc7Smrj 
983ae115bc7Smrj 	/*
984ae115bc7Smrj 	 * Deliver any pending soft interrupts.
985ae115bc7Smrj 	 */
986ae115bc7Smrj 	if (cpu->cpu_softinfo.st_pending)
987ae115bc7Smrj 		dosoftint(rp);
988ae115bc7Smrj }
989ae115bc7Smrj 
990ae115bc7Smrj /*
991ae115bc7Smrj  * Common tasks always done by _sys_rtt, called with interrupts disabled.
992ae115bc7Smrj  * Returns 1 if returning to userland, 0 if returning to system mode.
993ae115bc7Smrj  */
994ae115bc7Smrj int
995ae115bc7Smrj sys_rtt_common(struct regs *rp)
996ae115bc7Smrj {
997ae115bc7Smrj 	kthread_t *tp;
998ae115bc7Smrj 	extern void mutex_exit_critical_start();
999ae115bc7Smrj 	extern long mutex_exit_critical_size;
1000575a7426Spt157919 	extern void mutex_owner_running_critical_start();
1001575a7426Spt157919 	extern long mutex_owner_running_critical_size;
1002ae115bc7Smrj 
1003ae115bc7Smrj loop:
1004ae115bc7Smrj 
1005ae115bc7Smrj 	/*
1006ae115bc7Smrj 	 * Check if returning to user
1007ae115bc7Smrj 	 */
1008ae115bc7Smrj 	tp = CPU->cpu_thread;
1009ae115bc7Smrj 	if (USERMODE(rp->r_cs)) {
1010ae115bc7Smrj 		/*
1011ae115bc7Smrj 		 * Check if AST pending.
1012ae115bc7Smrj 		 */
1013ae115bc7Smrj 		if (tp->t_astflag) {
1014ae115bc7Smrj 			/*
1015ae115bc7Smrj 			 * Let trap() handle the AST
1016ae115bc7Smrj 			 */
1017ae115bc7Smrj 			sti();
1018ae115bc7Smrj 			rp->r_trapno = T_AST;
1019ae115bc7Smrj 			trap(rp, (caddr_t)0, CPU->cpu_id);
1020ae115bc7Smrj 			cli();
1021ae115bc7Smrj 			goto loop;
1022ae115bc7Smrj 		}
1023ae115bc7Smrj 
1024ae115bc7Smrj #if defined(__amd64)
1025ae115bc7Smrj 		/*
1026ae115bc7Smrj 		 * We are done if segment registers do not need updating.
1027ae115bc7Smrj 		 */
10287712e92cSsudheer 		if (tp->t_lwp->lwp_pcb.pcb_rupdate == 0)
1029ae115bc7Smrj 			return (1);
1030ae115bc7Smrj 
1031ae115bc7Smrj 		if (update_sregs(rp, tp->t_lwp)) {
1032ae115bc7Smrj 			/*
1033ae115bc7Smrj 			 * 1 or more of the selectors is bad.
1034ae115bc7Smrj 			 * Deliver a SIGSEGV.
1035ae115bc7Smrj 			 */
1036ae115bc7Smrj 			proc_t *p = ttoproc(tp);
1037ae115bc7Smrj 
1038ae115bc7Smrj 			sti();
1039ae115bc7Smrj 			mutex_enter(&p->p_lock);
1040ae115bc7Smrj 			tp->t_lwp->lwp_cursig = SIGSEGV;
1041ae115bc7Smrj 			mutex_exit(&p->p_lock);
1042ae115bc7Smrj 			psig();
1043ae115bc7Smrj 			tp->t_sig_check = 1;
1044ae115bc7Smrj 			cli();
1045ae115bc7Smrj 		}
10467712e92cSsudheer 		tp->t_lwp->lwp_pcb.pcb_rupdate = 0;
1047ae115bc7Smrj 
1048ae115bc7Smrj #endif	/* __amd64 */
1049ae115bc7Smrj 		return (1);
1050ae115bc7Smrj 	}
1051ae115bc7Smrj 
1052ae115bc7Smrj 	/*
1053ae115bc7Smrj 	 * Here if we are returning to supervisor mode.
1054ae115bc7Smrj 	 * Check for a kernel preemption request.
1055ae115bc7Smrj 	 */
1056ae115bc7Smrj 	if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
1057ae115bc7Smrj 
1058ae115bc7Smrj 		/*
1059ae115bc7Smrj 		 * Do nothing if already in kpreempt
1060ae115bc7Smrj 		 */
1061ae115bc7Smrj 		if (!tp->t_preempt_lk) {
1062ae115bc7Smrj 			tp->t_preempt_lk = 1;
1063ae115bc7Smrj 			sti();
1064ae115bc7Smrj 			kpreempt(1); /* asynchronous kpreempt call */
1065ae115bc7Smrj 			cli();
1066ae115bc7Smrj 			tp->t_preempt_lk = 0;
1067ae115bc7Smrj 		}
1068ae115bc7Smrj 	}
1069ae115bc7Smrj 
1070ae115bc7Smrj 	/*
1071ae115bc7Smrj 	 * If we interrupted the mutex_exit() critical region we must
1072ae115bc7Smrj 	 * reset the PC back to the beginning to prevent missed wakeups
1073ae115bc7Smrj 	 * See the comments in mutex_exit() for details.
1074ae115bc7Smrj 	 */
1075ae115bc7Smrj 	if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
1076ae115bc7Smrj 	    mutex_exit_critical_size) {
1077ae115bc7Smrj 		rp->r_pc = (greg_t)mutex_exit_critical_start;
1078ae115bc7Smrj 	}
1079575a7426Spt157919 
1080575a7426Spt157919 	/*
1081575a7426Spt157919 	 * If we interrupted the mutex_owner_running() critical region we
1082575a7426Spt157919 	 * must reset the PC back to the beginning to prevent dereferencing
1083575a7426Spt157919 	 * of a freed thread pointer. See the comments in mutex_owner_running
1084575a7426Spt157919 	 * for details.
1085575a7426Spt157919 	 */
1086575a7426Spt157919 	if ((uintptr_t)rp->r_pc -
1087575a7426Spt157919 	    (uintptr_t)mutex_owner_running_critical_start <
1088575a7426Spt157919 	    mutex_owner_running_critical_size) {
1089575a7426Spt157919 		rp->r_pc = (greg_t)mutex_owner_running_critical_start;
1090575a7426Spt157919 	}
1091575a7426Spt157919 
1092ae115bc7Smrj 	return (0);
1093ae115bc7Smrj }
1094ae115bc7Smrj 
1095ae115bc7Smrj void
1096ae115bc7Smrj send_dirint(int cpuid, int int_level)
1097ae115bc7Smrj {
1098ae115bc7Smrj 	(*send_dirintf)(cpuid, int_level);
1099ae115bc7Smrj }
1100ae115bc7Smrj 
1101ae115bc7Smrj /*
1102ae115bc7Smrj  * do_splx routine, takes new ipl to set
1103ae115bc7Smrj  * returns the old ipl.
1104ae115bc7Smrj  * We are careful not to set priority lower than CPU->cpu_base_pri,
1105ae115bc7Smrj  * even though it seems we're raising the priority, it could be set
1106ae115bc7Smrj  * higher at any time by an interrupt routine, so we must block interrupts
1107ae115bc7Smrj  * and look at CPU->cpu_base_pri
1108ae115bc7Smrj  */
1109ae115bc7Smrj int
1110ae115bc7Smrj do_splx(int newpri)
1111ae115bc7Smrj {
1112ae115bc7Smrj 	ulong_t	flag;
1113ae115bc7Smrj 	cpu_t	*cpu;
1114ae115bc7Smrj 	int	curpri, basepri;
1115ae115bc7Smrj 
1116ae115bc7Smrj 	flag = intr_clear();
1117ae115bc7Smrj 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1118ae115bc7Smrj 	curpri = cpu->cpu_m.mcpu_pri;
1119ae115bc7Smrj 	basepri = cpu->cpu_base_spl;
1120ae115bc7Smrj 	if (newpri < basepri)
1121ae115bc7Smrj 		newpri = basepri;
1122ae115bc7Smrj 	cpu->cpu_m.mcpu_pri = newpri;
1123ae115bc7Smrj 	(*setspl)(newpri);
1124ae115bc7Smrj 	/*
1125ae115bc7Smrj 	 * If we are going to reenable interrupts see if new priority level
1126ae115bc7Smrj 	 * allows pending softint delivery.
1127ae115bc7Smrj 	 */
1128ae115bc7Smrj 	if ((flag & PS_IE) &&
1129ae115bc7Smrj 	    bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > newpri)
1130ae115bc7Smrj 		fakesoftint();
1131ae115bc7Smrj 	ASSERT(!interrupts_enabled());
1132ae115bc7Smrj 	intr_restore(flag);
1133ae115bc7Smrj 	return (curpri);
1134ae115bc7Smrj }
1135ae115bc7Smrj 
1136ae115bc7Smrj /*
1137ae115bc7Smrj  * Common spl raise routine, takes new ipl to set
1138ae115bc7Smrj  * returns the old ipl, will not lower ipl.
1139ae115bc7Smrj  */
1140ae115bc7Smrj int
1141ae115bc7Smrj splr(int newpri)
1142ae115bc7Smrj {
1143ae115bc7Smrj 	ulong_t	flag;
1144ae115bc7Smrj 	cpu_t	*cpu;
1145ae115bc7Smrj 	int	curpri, basepri;
1146ae115bc7Smrj 
1147ae115bc7Smrj 	flag = intr_clear();
1148ae115bc7Smrj 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1149ae115bc7Smrj 	curpri = cpu->cpu_m.mcpu_pri;
1150ae115bc7Smrj 	/*
1151ae115bc7Smrj 	 * Only do something if new priority is larger
1152ae115bc7Smrj 	 */
1153ae115bc7Smrj 	if (newpri > curpri) {
1154ae115bc7Smrj 		basepri = cpu->cpu_base_spl;
1155ae115bc7Smrj 		if (newpri < basepri)
1156ae115bc7Smrj 			newpri = basepri;
1157ae115bc7Smrj 		cpu->cpu_m.mcpu_pri = newpri;
1158ae115bc7Smrj 		(*setspl)(newpri);
1159ae115bc7Smrj 		/*
1160ae115bc7Smrj 		 * See if new priority level allows pending softint delivery
1161ae115bc7Smrj 		 */
1162ae115bc7Smrj 		if ((flag & PS_IE) &&
1163ae115bc7Smrj 		    bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > newpri)
1164ae115bc7Smrj 			fakesoftint();
1165ae115bc7Smrj 	}
1166ae115bc7Smrj 	intr_restore(flag);
1167ae115bc7Smrj 	return (curpri);
1168ae115bc7Smrj }
1169ae115bc7Smrj 
1170ae115bc7Smrj int
1171ae115bc7Smrj getpil(void)
1172ae115bc7Smrj {
1173ae115bc7Smrj 	return (CPU->cpu_m.mcpu_pri);
1174ae115bc7Smrj }
1175ae115bc7Smrj 
1176ae115bc7Smrj int
1177ae115bc7Smrj interrupts_enabled(void)
1178ae115bc7Smrj {
1179ae115bc7Smrj 	ulong_t	flag;
1180ae115bc7Smrj 
1181ae115bc7Smrj 	flag = getflags();
1182ae115bc7Smrj 	return ((flag & PS_IE) == PS_IE);
1183ae115bc7Smrj }
1184ae115bc7Smrj 
1185ae115bc7Smrj #ifdef DEBUG
1186ae115bc7Smrj void
1187ae115bc7Smrj assert_ints_enabled(void)
1188ae115bc7Smrj {
1189ae115bc7Smrj 	ASSERT(!interrupts_unleashed || interrupts_enabled());
1190ae115bc7Smrj }
1191ae115bc7Smrj #endif	/* DEBUG */
1192