xref: /titanic_53/usr/src/uts/i86pc/ml/interrupt.s (revision 7a364d25fde47aa82704b12b5251bf7fac37f02e)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T	*/
/*	  All Rights Reserved					*/

/*	Copyright (c) 1987, 1988 Microsoft Corporation		*/
/*	  All Rights Reserved					*/

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/psw.h>
#include <sys/x86_archext.h>

#if defined(__lint)

#include <sys/types.h>
#include <sys/thread.h>
#include <sys/systm.h>

#else   /* __lint */

#include <sys/segments.h>
#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/panic.h>
#include "assym.h"

_ftrace_intr_thread_fmt:
	.string	"intr_thread(): regs=0x%lx, int=0x%x, pil=0x%x"

#endif	/* lint */

#if defined(__i386)

#if defined(__lint)

void
patch_tsc(void)
{}

#else	/* __lint */

/*
 * To cope with processors that do not implement the rdtsc instruction,
 * we patch the kernel to use rdtsc if that feature is detected on the CPU.
 * On an unpatched kernel, all locations requiring rdtsc are nop's.
 *
 * This function patches the nop's to rdtsc.
 */
	ENTRY_NP(patch_tsc)
	movw	_rdtsc_insn, %cx
	movw	%cx, _tsc_patch1
	movw	%cx, _tsc_patch2
	movw	%cx, _tsc_patch3
	movw	%cx, _tsc_patch4
	movw	%cx, _tsc_patch5
	movw	%cx, _tsc_patch6
	movw	%cx, _tsc_patch7
	movw	%cx, _tsc_patch8
	movw	%cx, _tsc_patch9
	movw	%cx, _tsc_patch10
	movw	%cx, _tsc_patch11
	movw	%cx, _tsc_patch12
	movw	%cx, _tsc_patch13
	movw	%cx, _tsc_patch14
	movw	%cx, _tsc_patch15
	movw	%cx, _tsc_patch16
	movw	%cx, _tsc_patch17
	ret
_rdtsc_insn:
	rdtsc
	SET_SIZE(patch_tsc)

#endif	/* __lint */

#endif	/* __i386 */


#if defined(__lint)

void
_interrupt(void)
{}

#else	/* __lint */

#if defined(__amd64)

	/*
	 * Common register usage:
	 *
	 * %rbx		cpu pointer
	 * %r12		trap trace pointer -and- stash of
	 *		vec across intr_thread dispatch.
	 * %r13d	ipl of isr
	 * %r14d	old ipl (ipl level we entered on)
	 * %r15		interrupted thread stack pointer
	 */
	ENTRY_NP2(cmnint, _interrupt)

	INTR_PUSH

	/*
	 * At the end of TRACE_PTR %r12 points to the current TRAPTRACE entry
	 */
	TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_INTERRUPT)
						/* Uses labels 8 and 9 */
	TRACE_REGS(%r12, %rsp, %rax, %rbx)	/* Uses label 9 */
	TRACE_STAMP(%r12)		/* Clobbers %eax, %edx, uses 9 */

	DISABLE_INTR_FLAGS		/* (and set kernel flag values) */

	movq	%rsp, %rbp

	TRACE_STACK(%r12)

	LOADCPU(%rbx)				/* &cpu */
	leaq	REGOFF_TRAPNO(%rbp), %rsi	/* &vector */
	movl	CPU_PRI(%rbx), %r14d		/* old ipl */
	movl	CPU_SOFTINFO(%rbx), %edx

#ifdef TRAPTRACE
	movl	$255, TTR_IPL(%r12)
	movl	%r14d, %edi
	movb	%dil, TTR_PRI(%r12)
	movl	CPU_BASE_SPL(%rbx), %edi
	movb	%dil, TTR_SPL(%r12)
	movb	$255, TTR_VECTOR(%r12)
#endif

	/*
	 * Check to see if the trap number is T_SOFTINT; if it is,
	 * jump straight to dosoftint now.
	 */
	cmpq	$T_SOFTINT, (%rsi)
	je	dosoftint

	/*
	 * Raise the interrupt priority level, returns newpil.
	 * (The vector address is in %rsi so setlvl can update it.)
	 */
	movl	%r14d, %edi			/* old ipl */
						/* &vector */
	call	*setlvl(%rip)

#ifdef TRAPTRACE
	movb	%al, TTR_IPL(%r12)
#endif
	/*
	 * check for spurious interrupt
	 */
	cmpl	$-1, %eax
	je	_sys_rtt

#ifdef TRAPTRACE
	movl	%r14d, %edx
	movb	%dl, TTR_PRI(%r12)
	movl	CPU_BASE_SPL(%rbx), %edx
	movb	%dl, TTR_SPL(%r12)
#endif
	movl	%eax, CPU_PRI(%rbx)		/* update ipl */

#ifdef TRAPTRACE
	movl	REGOFF_TRAPNO(%rbp), %edx
	movb	%dl, TTR_VECTOR(%r12)
#endif
	movl	%eax, %r13d			/* ipl of isr */

	/*
	 * At this point we can take one of two paths.
	 * If the new level is at or below lock level, we will
	 * run this interrupt in a separate thread.
	 */
	cmpl	$LOCK_LEVEL, %eax
	jbe	intr_thread

	movq	%rbx, %rdi		/* &cpu */
	movl	%r13d, %esi		/* ipl */
	movl	%r14d, %edx		/* old ipl */
	movq	%rbp, %rcx		/* &regs */
	call	hilevel_intr_prolog
	orl	%eax, %eax		/* zero if need to switch stack */
	jnz	1f

	/*
	 * Save the thread stack and get on the cpu's interrupt stack
	 */
	movq	%rsp, %r15
	movq	CPU_INTR_STACK(%rbx), %rsp
1:

	sti

	/*
	 * Walk the list of handlers for this vector, calling
	 * them as we go until no more interrupts are claimed.
	 */
	movl	REGOFF_TRAPNO(%rbp), %edi
	call	av_dispatch_autovect

	cli

	movq	%rbx, %rdi			/* &cpu */
	movl	%r13d, %esi			/* ipl */
	movl	%r14d, %edx			/* oldipl */
	movl	REGOFF_TRAPNO(%rbp), %ecx	/* vec */
	call	hilevel_intr_epilog
	orl	%eax, %eax		/* zero if need to switch stack */
	jnz	2f
	movq	%r15, %rsp
2:	/*
	 * Check for, and execute, softints before we iret.
	 *
	 * (dosoftint expects oldipl in %r14d (which is where it is)
	 * the cpu pointer in %rbx (which is where it is) and the
	 * softinfo in %edx (which is where we'll put it right now))
	 */
	movl	CPU_SOFTINFO(%rbx), %edx
	orl	%edx, %edx
	jz	_sys_rtt
	jmp	dosoftint
	/*NOTREACHED*/

	SET_SIZE(cmnint)
	SET_SIZE(_interrupt)

/*
 * Handle an interrupt in a new thread
 *
 * As we branch here, interrupts are still masked,
 * %rbx still contains the cpu pointer,
 * %r14d contains the old ipl that we came in on, and
 * %eax contains the new ipl that we got from the setlvl routine
 */

	ENTRY_NP(intr_thread)

	movq	%rbx, %rdi	/* &cpu */
	movq	%rbp, %rsi	/* &regs = stack pointer for _sys_rtt */
	movl	REGOFF_TRAPNO(%rbp), %r12d	/* stash the vec */
	movl	%eax, %edx	/* new pil from setlvlx() */
	call	intr_thread_prolog
	movq	%rsp, %r15
	movq	%rax, %rsp	/* t_stk from interrupt thread */
	movq	%rsp, %rbp

	sti

	testl	$FTRACE_ENABLED, CPU_FTRACE_STATE(%rbx)
	jz	1f
	/*
	 * ftracing support. do we need this on x86?
	 */
	leaq	_ftrace_intr_thread_fmt(%rip), %rdi
	movq	%rbp, %rsi			/* &regs */
	movl	%r12d, %edx			/* vec */
	movq	CPU_THREAD(%rbx), %r11		/* (the interrupt thread) */
	movzbl	T_PIL(%r11), %ecx		/* newipl */
	call	ftrace_3_notick
1:
	movl	%r12d, %edi			/* vec */
	call	av_dispatch_autovect

	cli

	movq	%rbx, %rdi			/* &cpu */
	movl	%r12d, %esi			/* vec */
	movl	%r14d, %edx			/* oldpil */
	call	intr_thread_epilog
	/*
	 * If we return from here (we might not if the interrupted thread
	 * has exited or blocked, in which case we'll have quietly swtch()ed
	 * away) then we need to switch back to our old %rsp
	 */
	movq	%r15, %rsp
	movq	%rsp, %rbp
	/*
	 * Check for, and execute, softints before we iret.
	 *
	 * (dosoftint expects oldpil in %r14d, the cpu pointer in %rbx and
	 * the mcpu_softinfo.st_pending field in %edx.
	 */
	movl	CPU_SOFTINFO(%rbx), %edx
	orl	%edx, %edx
	jz	_sys_rtt
	/*FALLTHROUGH*/

/*
 * Process soft interrupts.
 * Interrupts are masked, and we have a minimal frame on the stack.
 * %edx should contain the mcpu_softinfo.st_pending field
 */

	ALTENTRY(dosoftint)

	movq	%rbx, %rdi	/* &cpu */
	movq	%rbp, %rsi	/* &regs = stack pointer for _sys_rtt */
				/* cpu->cpu_m.mcpu_softinfo.st_pending */
	movl	%r14d, %ecx	/* oldipl */
	call	dosoftint_prolog
	/*
	 * dosoftint_prolog() usually returns a stack pointer for the
	 * interrupt thread that we must switch to.  However, if the
	 * returned stack pointer is NULL, then the software interrupt was
	 * too low in priority to run now; we'll catch it another time.
	 */
	orq	%rax, %rax
	jz	_sys_rtt
	movq	%rsp, %r15
	movq	%rax, %rsp	/* t_stk from interrupt thread */
	movq	%rsp, %rbp

	sti

	/*
	 * Enabling interrupts (above) could raise the current ipl
	 * and base spl.  But, we continue processing the current soft
	 * interrupt and we will check the base spl next time around
	 * so that blocked interrupt threads get a chance to run.
	 */
	movq	CPU_THREAD(%rbx), %r11	/* now an interrupt thread */
	movzbl	T_PIL(%r11), %edi
	call	av_dispatch_softvect

	cli

	movq	%rbx, %rdi		/* &cpu */
	movl	%r14d, %esi		/* oldpil */
	call	dosoftint_epilog
	movq	%r15, %rsp		/* back on old stack pointer */
	movq	%rsp, %rbp
	movl	CPU_SOFTINFO(%rbx), %edx
	orl	%edx, %edx
	jz	_sys_rtt
	jmp	dosoftint

	SET_SIZE(dosoftint)
	SET_SIZE(intr_thread)

#elif defined(__i386)

/*
 * One day, this should just invoke the C routines that know how to
 * do all the interrupt bookkeeping.  In the meantime, try
 * and make the assembler a little more comprehensible.
 */

#define	INC64(basereg, offset)			\
	addl	$1, offset(basereg);		\
	adcl	$0, offset + 4(basereg)

#define	TSC_CLR(basereg, offset)		\
	movl	$0, offset(basereg);		\
	movl	$0, offset + 4(basereg)

/*
 * The following macros assume the time value is in %edx:%eax
 * e.g. from a rdtsc instruction.
 */
#define	TSC_STORE(reg, offset)		\
	movl	%eax, offset(reg);	\
	movl	%edx, offset + 4(reg)

#define	TSC_LOAD(reg, offset)	\
	movl	offset(reg), %eax;	\
	movl	offset + 4(reg), %edx

#define	TSC_ADD_TO(reg, offset)		\
	addl	%eax, offset(reg);	\
	adcl	%edx, offset + 4(reg)

#define	TSC_SUB_FROM(reg, offset)	\
	subl	offset(reg), %eax;	\
	sbbl	offset + 4(reg), %edx	/* interval in edx:eax */

/*
 * basereg   - pointer to cpu struct
 * pilreg    - pil or converted pil (pil - (LOCK_LEVEL + 1))
 *
 * Returns (base + pil * 8) in pilreg
 */
#define	PILBASE(basereg, pilreg)	\
	lea	(basereg, pilreg, 8), pilreg

/*
 * Returns (base + (pil - (LOCK_LEVEL + 1)) * 8) in pilreg
 */
#define	HIGHPILBASE(basereg, pilreg)		\
	subl	$LOCK_LEVEL + 1, pilreg;	\
	PILBASE(basereg, pilreg)

/*
 * Returns (base + pil * 16) in pilreg
 */
#define	PILBASE_INTRSTAT(basereg, pilreg)	\
	shl	$4, pilreg;			\
	addl	basereg, pilreg;

/*
 * Returns (cpu + cpu_mstate * 8) in tgt
 */
#define	INTRACCTBASE(cpureg, tgtreg)		\
	movzwl	CPU_MSTATE(cpureg), tgtreg;	\
	lea	(cpureg, tgtreg, 8), tgtreg

/*
 * cpu_stats.sys.intr[PIL]++
 */
#define	INC_CPU_STATS_INTR(pilreg, tmpreg, tmpreg_32, basereg)	\
	movl	pilreg, tmpreg_32;				\
	PILBASE(basereg, tmpreg);				\
	INC64(tmpreg, _CONST(CPU_STATS_SYS_INTR - 8))

/*
 * Unlink thread from CPU's list
 */
#define	UNLINK_INTR_THREAD(cpureg, ithread, tmpreg)	\
	mov	CPU_INTR_THREAD(cpureg), ithread;	\
	mov	T_LINK(ithread), tmpreg;		\
	mov	tmpreg, CPU_INTR_THREAD(cpureg)

/*
 * Link a thread into CPU's list
 */
#define	LINK_INTR_THREAD(cpureg, ithread, tmpreg)	\
	mov	CPU_INTR_THREAD(cpureg), tmpreg;	\
	mov	tmpreg, T_LINK(ithread);		\
	mov	ithread, CPU_INTR_THREAD(cpureg)

#if defined(DEBUG)

/*
 * Do not call panic, if panic is already in progress.
 */
#define	__PANIC(msg, label)		\
	cmpl	$0, panic_quiesce;		\
	jne	label;				\
	pushl	$msg;				\
	call	panic

#define	__CMP64_JNE(basereg, offset, label)	\
	cmpl	$0, offset(basereg);		\
	jne	label;				\
	cmpl	$0, offset + 4(basereg);	\
	jne	label

/*
 * ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
 */
#define	ASSERT_NOT_CPU_INTR_ACTV(pilreg, basereg, msg)	\
	btl	pilreg, CPU_INTR_ACTV(basereg);		\
	jnc	4f;					\
	__PANIC(msg, 4f);				\
4:

/*
 * ASSERT(CPU->cpu_intr_actv & (1 << PIL))
 */
#define	ASSERT_CPU_INTR_ACTV(pilreg, basereg, msg)	\
	btl	pilreg, CPU_INTR_ACTV(basereg);		\
	jc	5f;					\
	__PANIC(msg, 5f);				\
5:

/*
 * ASSERT(CPU->cpu_pil_high_start != 0)
 */
#define	ASSERT_CPU_PIL_HIGH_START_NZ(basereg)			\
	__CMP64_JNE(basereg, CPU_PIL_HIGH_START, 6f);		\
	__PANIC(_interrupt_timestamp_zero, 6f);		\
6:

/*
 * ASSERT(t->t_intr_start != 0)
 */
#define	ASSERT_T_INTR_START_NZ(basereg)				\
	__CMP64_JNE(basereg, T_INTR_START, 7f);			\
	__PANIC(_intr_thread_t_intr_start_zero, 7f);	\
7:

_interrupt_actv_bit_set:
	.string	"_interrupt(): cpu_intr_actv bit already set for PIL"
_interrupt_actv_bit_not_set:
	.string	"_interrupt(): cpu_intr_actv bit not set for PIL"
_interrupt_timestamp_zero:
	.string "_interrupt(): timestamp zero upon handler return"
_intr_thread_actv_bit_not_set:
	.string	"intr_thread():	cpu_intr_actv bit not set for PIL"
_intr_thread_t_intr_start_zero:
	.string	"intr_thread():	t_intr_start zero upon handler return"
_dosoftint_actv_bit_set:
	.string	"dosoftint(): cpu_intr_actv bit already set for PIL"
_dosoftint_actv_bit_not_set:
	.string	"dosoftint(): cpu_intr_actv bit not set for PIL"

	DGDEF(intr_thread_cnt)

#else
#define	ASSERT_NOT_CPU_INTR_ACTV(pilreg, basereg, msg)
#define	ASSERT_CPU_INTR_ACTV(pilreg, basereg, msg)
#define	ASSERT_CPU_PIL_HIGH_START_NZ(basereg)
#define	ASSERT_T_INTR_START_NZ(basereg)
#endif

	ENTRY_NP2(cmnint, _interrupt)

	INTR_PUSH

	/*
	 * At the end of TRACE_PTR %esi points to the current TRAPTRACE entry
	 */
	TRACE_PTR(%esi, %eax, %eax, %edx, $TT_INTERRUPT)
						/* Uses labels 8 and 9 */
	TRACE_REGS(%esi, %esp, %eax, %ebx)	/* Uses label 9 */
	TRACE_STAMP(%esi)		/* Clobbers %eax, %edx, uses 9 */

	movl	%esp, %ebp
	DISABLE_INTR_FLAGS
	LOADCPU(%ebx)		/* get pointer to CPU struct. Avoid gs refs */
	leal    REGOFF_TRAPNO(%ebp), %ecx	/* get address of vector */
	movl	CPU_PRI(%ebx), %edi		/* get ipl */
	movl	CPU_SOFTINFO(%ebx), %edx

	/
	/ Check to see if the trap number is T_SOFTINT; if it is, we'll
	/ jump straight to dosoftint now.
	/
	cmpl	$T_SOFTINT, (%ecx)
	je	dosoftint

	/ raise interrupt priority level
	/ oldipl is in %edi, vectorp is in %ecx
	/ newipl is returned in %eax
	pushl	%ecx
	pushl	%edi
	call    *setlvl
	popl	%edi			/* save oldpil in %edi */
	popl	%ecx

#ifdef TRAPTRACE
	movb	%al, TTR_IPL(%esi)
#endif

	/ check for spurious interrupt
	cmp	$-1, %eax
	je	_sys_rtt

#ifdef TRAPTRACE
	movl	CPU_PRI(%ebx), %edx
	movb	%dl, TTR_PRI(%esi)
	movl	CPU_BASE_SPL(%ebx), %edx
	movb	%dl, TTR_SPL(%esi)
#endif

	movl	%eax, CPU_PRI(%ebx) /* update ipl */
	movl	REGOFF_TRAPNO(%ebp), %ecx /* reload the interrupt vector */

#ifdef TRAPTRACE
	movb	%cl, TTR_VECTOR(%esi)
#endif

	/ At this point we can take one of two paths.  If the new priority
	/ level is less than or equal to LOCK LEVEL then we jump to code that
	/ will run this interrupt as a separate thread.  Otherwise the
	/ interrupt is NOT run as a separate thread.

	/ %edi - old priority level
	/ %ebp - pointer to REGS
	/ %ecx - translated vector
	/ %eax - ipl of isr
	/ %ebx - cpu pointer

	cmpl 	$LOCK_LEVEL, %eax	/* compare to highest thread level */
	jbe	intr_thread		/* process as a separate thread */

	cmpl	$CBE_HIGH_PIL, %eax	/* Is this a CY_HIGH_LEVEL interrupt? */
	jne	2f

	movl	REGOFF_PC(%ebp), %esi
	movl	%edi, CPU_PROFILE_PIL(%ebx)	/* record interrupted PIL */
	testw	$CPL_MASK, REGOFF_CS(%ebp)	/* trap from supervisor mode? */
	jz	1f
	movl	%esi, CPU_PROFILE_UPC(%ebx)	/* record user PC */
	movl	$0, CPU_PROFILE_PC(%ebx)	/* zero kernel PC */
	jmp	2f

1:
	movl	%esi, CPU_PROFILE_PC(%ebx)	/* record kernel PC */
	movl	$0, CPU_PROFILE_UPC(%ebx)	/* zero user PC */

2:
	pushl	%ecx				/* vec */
	pushl	%eax				/* newpil */

	/
	/ See if we are interrupting another high-level interrupt.
	/
	movl	CPU_INTR_ACTV(%ebx), %eax
	andl	$CPU_INTR_ACTV_HIGH_LEVEL_MASK, %eax
	jz	0f
	/
	/ We have interrupted another high-level interrupt.
	/ Load starting timestamp, compute interval, update cumulative counter.
	/
	bsrl	%eax, %ecx		/* find PIL of interrupted handler */
	movl	%ecx, %esi		/* save PIL for later */
	HIGHPILBASE(%ebx, %ecx)
_tsc_patch1:
	nop; nop			/* patched to rdtsc if available */
	TSC_SUB_FROM(%ecx, CPU_PIL_HIGH_START)

	PILBASE_INTRSTAT(%ebx, %esi)
	TSC_ADD_TO(%esi, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %ecx)
	TSC_ADD_TO(%ecx, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */
	/
	/ Another high-level interrupt is active below this one, so
	/ there is no need to check for an interrupt thread. That will be
	/ done by the lowest priority high-level interrupt active.
	/
	jmp	1f
0:
	/
	/ See if we are interrupting a low-level interrupt thread.
	/
	movl	CPU_THREAD(%ebx), %esi
	testw	$T_INTR_THREAD, T_FLAGS(%esi)
	jz	1f
	/
	/ We have interrupted an interrupt thread. Account for its time slice
	/ only if its time stamp is non-zero.
	/
	cmpl	$0, T_INTR_START+4(%esi)
	jne	0f
	cmpl	$0, T_INTR_START(%esi)
	je	1f
0:
	movzbl	T_PIL(%esi), %ecx /* %ecx has PIL of interrupted handler */
	PILBASE_INTRSTAT(%ebx, %ecx)
_tsc_patch2:
	nop; nop			/* patched to rdtsc if available */
	TSC_SUB_FROM(%esi, T_INTR_START)
	TSC_CLR(%esi, T_INTR_START)
	TSC_ADD_TO(%ecx, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %ecx)
	TSC_ADD_TO(%ecx, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */
1:
	/ Store starting timestamp in CPU structure for this PIL.
	popl	%ecx			/* restore new PIL */
	pushl	%ecx
	HIGHPILBASE(%ebx, %ecx)
_tsc_patch3:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%ecx, CPU_PIL_HIGH_START)

	popl	%eax			/* restore new pil */
	popl	%ecx			/* vec */
	/
	/ Set bit for this PIL in CPU's interrupt active bitmask.
	/

	ASSERT_NOT_CPU_INTR_ACTV(%eax, %ebx, _interrupt_actv_bit_set)

	/ Save old CPU_INTR_ACTV
	movl	CPU_INTR_ACTV(%ebx), %esi

	cmpl	$15, %eax
	jne	0f
	/ PIL-15 interrupt. Increment nest-count in upper 16 bits of intr_actv
	incw	CPU_INTR_ACTV_REF(%ebx)	/* increment ref count */
0:
	btsl	%eax, CPU_INTR_ACTV(%ebx)
	/
	/ Handle high-level nested interrupt on separate interrupt stack
	/
	testl	$CPU_INTR_ACTV_HIGH_LEVEL_MASK, %esi
	jnz	onstack			/* already on interrupt stack */
	movl	%esp, %eax
	movl	CPU_INTR_STACK(%ebx), %esp	/* get on interrupt stack */
	pushl	%eax			/* save the thread stack pointer */
onstack:
	movl	$autovect, %esi		/* get autovect structure before */
					/* sti to save on AGI later */
	sti				/* enable interrupts */
	pushl	%ecx			/* save interrupt vector */
	/
	/ Get handler address
	/
pre_loop1:
	movl	AVH_LINK(%esi, %ecx, 8), %esi
	xorl	%ebx, %ebx	/* bh is no. of intpts in chain */
				/* bl is DDI_INTR_CLAIMED status of chain */
	testl	%esi, %esi		/* if pointer is null */
	jz	.intr_ret		/* then skip */
loop1:
	incb	%bh
	movl	AV_VECTOR(%esi), %edx	/* get the interrupt routine */
	testl	%edx, %edx		/* if func is null */
	jz	.intr_ret		/* then skip */
	pushl	$0
	pushl	AV_INTARG2(%esi)
	pushl	AV_INTARG1(%esi)
	pushl	AV_VECTOR(%esi)
	pushl	AV_DIP(%esi)
	call	__dtrace_probe_interrupt__start
	pushl	AV_INTARG2(%esi)	/* get 2nd arg to interrupt routine */
	pushl	AV_INTARG1(%esi)	/* get first arg to interrupt routine */
	call	*%edx			/* call interrupt routine with arg */
	addl	$8, %esp
	movl	%eax, 16(%esp)
	call	__dtrace_probe_interrupt__complete
	addl	$20, %esp
	orb	%al, %bl		/* see if anyone claims intpt. */
	movl	AV_LINK(%esi), %esi	/* get next routine on list */
	testl	%esi, %esi		/* if pointer is non-null */
	jnz	loop1			/* then continue */

.intr_ret:
	cmpb	$1, %bh		/* if only 1 intpt in chain, it is OK */
	je	.intr_ret1
	orb	%bl, %bl	/* If no one claims intpt, then it is OK */
	jz	.intr_ret1
	movl	(%esp), %ecx		/* else restore intr vector */
	movl	$autovect, %esi		/* get autovect structure */
	jmp	pre_loop1		/* and try again. */

.intr_ret1:
	LOADCPU(%ebx)			/* get pointer to cpu struct */

	cli
	movl	CPU_PRI(%ebx), %esi

	/ cpu_stats.sys.intr[PIL]++
	INC_CPU_STATS_INTR(%esi, %eax, %eax, %ebx)

	/
	/ Clear bit for this PIL in CPU's interrupt active bitmask.
	/

	ASSERT_CPU_INTR_ACTV(%esi, %ebx, _interrupt_actv_bit_not_set)

	cmpl	$15, %esi
	jne	0f
	/ Only clear bit if reference count is now zero.
	decw	CPU_INTR_ACTV_REF(%ebx)
	jnz	1f
0:
	btrl	%esi, CPU_INTR_ACTV(%ebx)
1:
	/
	/ Take timestamp, compute interval, update cumulative counter.
	/ esi = PIL
_tsc_patch4:
	nop; nop			/* patched to rdtsc if available */
	movl	%esi, %ecx		/* save for later */
	HIGHPILBASE(%ebx, %esi)

	ASSERT_CPU_PIL_HIGH_START_NZ(%esi)

	TSC_SUB_FROM(%esi, CPU_PIL_HIGH_START)

	PILBASE_INTRSTAT(%ebx, %ecx)
	TSC_ADD_TO(%ecx, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %esi)
	TSC_ADD_TO(%esi, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */
	/
	/ Check for lower-PIL nested high-level interrupt beneath current one
	/ If so, place a starting timestamp in its pil_high_start entry.
	/
	movl	CPU_INTR_ACTV(%ebx), %eax
	movl	%eax, %esi
	andl	$CPU_INTR_ACTV_HIGH_LEVEL_MASK, %eax
	jz	0f
	bsrl	%eax, %ecx		/* find PIL of nested interrupt */
	HIGHPILBASE(%ebx, %ecx)
_tsc_patch5:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%ecx, CPU_PIL_HIGH_START)
	/
	/ Another high-level interrupt is active below this one, so
	/ there is no need to check for an interrupt thread. That will be
	/ done by the lowest priority high-level interrupt active.
	/
	jmp	1f
0:
	/ Check to see if there is a low-level interrupt active. If so,
	/ place a starting timestamp in the thread structure.
	movl	CPU_THREAD(%ebx), %esi
	testw	$T_INTR_THREAD, T_FLAGS(%esi)
	jz	1f
_tsc_patch6:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%esi, T_INTR_START)
1:
	movl	%edi, CPU_PRI(%ebx)
				/* interrupt vector already on stack */
	pushl	%edi			/* old ipl */
	call	*setlvlx
	addl	$8, %esp		/* eax contains the current ipl */

	movl	CPU_INTR_ACTV(%ebx), %esi /* reset stack pointer if no more */
	shrl	$LOCK_LEVEL + 1, %esi	/* HI PRI intrs. */
	jnz	.intr_ret2
	popl	%esp			/* restore the thread stack pointer */
.intr_ret2:
	movl	CPU_SOFTINFO(%ebx), %edx /* any pending software interrupts */
	orl	%edx, %edx
	jz	_sys_rtt
	jmp	dosoftint	/* check for softints before we return. */
	SET_SIZE(cmnint)
	SET_SIZE(_interrupt)

#endif	/* __i386 */

/*
 * Declare a uintptr_t which has the size of _interrupt to enable stack
 * traceback code to know when a regs structure is on the stack.
 */
	.globl	_interrupt_size
	.align	CLONGSIZE
_interrupt_size:
	.NWORD	. - _interrupt
	.type	_interrupt_size, @object

#endif	/* __lint */

#if defined(__i386)

/*
 * Handle an interrupt in a new thread.
 *	Entry:  traps disabled.
 *		%edi - old priority level
 *		%ebp - pointer to REGS
 *		%ecx - translated vector
 *		%eax - ipl of isr.
 *		%ebx - pointer to CPU struct
 *	Uses:
 */

#if !defined(__lint)

	ENTRY_NP(intr_thread)
	/
	/ Set bit for this PIL in CPU's interrupt active bitmask.
	/

	ASSERT_NOT_CPU_INTR_ACTV(%eax, %ebx, _interrupt_actv_bit_set)

	btsl	%eax, CPU_INTR_ACTV(%ebx)

	/ Get set to run interrupt thread.
	/ There should always be an interrupt thread since we allocate one
	/ for each level on the CPU.
	/
	/ Note that the code in kcpc_overflow_intr -relies- on the ordering
	/ of events here - in particular that t->t_lwp of the interrupt
	/ thread is set to the pinned thread *before* curthread is changed
	/
	movl	CPU_THREAD(%ebx), %edx		/* cur thread in edx */

	/
	/ Are we interrupting an interrupt thread? If so, account for it.
	/
	testw	$T_INTR_THREAD, T_FLAGS(%edx)
	jz	0f
	pushl	%ecx
	pushl	%eax
	movl	%edx, %esi
_tsc_patch7:
	nop; nop			/* patched to rdtsc if available */
	TSC_SUB_FROM(%esi, T_INTR_START)
	TSC_CLR(%esi, T_INTR_START)
	movzbl	T_PIL(%esi), %ecx
	PILBASE_INTRSTAT(%ebx, %ecx)
	TSC_ADD_TO(%ecx, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %ecx)
	TSC_ADD_TO(%ecx, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */
	movl	%esi, %edx
	popl	%eax
	popl	%ecx
0:
	movl	%esp, T_SP(%edx)	/* mark stack in curthread for resume */
	pushl	%edi			/* get a temporary register */
	UNLINK_INTR_THREAD(%ebx, %esi, %edi)

	movl	T_LWP(%edx), %edi
	movl	%edx, T_INTR(%esi)		/* push old thread */
	movl	%edi, T_LWP(%esi)
	/
	/ Threads on the interrupt thread free list could have state already
	/ set to TS_ONPROC, but it helps in debugging if they're TS_FREE
	/
	movl	$ONPROC_THREAD, T_STATE(%esi)
	/
	/ chain the interrupted thread onto list from the interrupt thread.
	/ Set the new interrupt thread as the current one.
	/
	popl	%edi			/* Don't need a temp reg anymore */
	movl	T_STACK(%esi), %esp		/* interrupt stack pointer */
	movl	%esp, %ebp
	movl	%esi, CPU_THREAD(%ebx)		/* set new thread */
	pushl	%eax				/* save the ipl */
	/
	/ Initialize thread priority level from intr_pri
	/
	movb	%al, T_PIL(%esi)	/* store pil */
	movzwl	intr_pri, %ebx		/* XXX Can cause probs if new class */
					/* is loaded on some other cpu. */
	addl	%ebx, %eax		/* convert level to dispatch priority */
	movw	%ax, T_PRI(%esi)

	/
	/ Take timestamp and store it in the thread structure.
	/
	movl	%eax, %ebx		/* save priority over rdtsc */
_tsc_patch8:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%esi, T_INTR_START)
	movl	%ebx, %eax		/* restore priority */

	/ The following 3 instructions need not be in cli.
	/ Putting them here only to avoid the AGI penalty on Pentiums.

	pushl	%ecx			/* save interrupt vector. */
	pushl	%esi			/* save interrupt thread */
	movl	$autovect, %esi		/* get autovect structure */
	sti				/* enable interrupts */

	/ Fast event tracing.
	LOADCPU(%ebx)
	movl	CPU_FTRACE_STATE(%ebx), %ebx
	testl	$FTRACE_ENABLED, %ebx
	jz	1f

	movl	8(%esp), %ebx
	pushl	%ebx			/* ipl */
	pushl	%ecx			/* int vector */
	movl	T_SP(%edx), %ebx
	pushl	%ebx			/* &regs */
	pushl	$_ftrace_intr_thread_fmt
	call	ftrace_3_notick
	addl	$8, %esp
	popl	%ecx			/* restore int vector */
	addl	$4, %esp
1:
pre_loop2:
	movl	AVH_LINK(%esi, %ecx, 8), %esi
	xorl	%ebx, %ebx	/* bh is cno. of intpts in chain */
				/* bl is DDI_INTR_CLAIMED status of * chain */
	testl	%esi, %esi	/* if pointer is null */
	jz	loop_done2	/* we're done */
loop2:
	movl	AV_VECTOR(%esi), %edx	/* get the interrupt routine */
	testl	%edx, %edx		/* if pointer is null */
	jz	loop_done2		/* we're done */
	incb	%bh
	pushl	$0
	pushl	AV_INTARG2(%esi)
	pushl	AV_INTARG1(%esi)
	pushl	AV_VECTOR(%esi)
	pushl	AV_DIP(%esi)
	call	__dtrace_probe_interrupt__start
	pushl	AV_INTARG2(%esi)	/* get 2nd arg to interrupt routine */
	pushl	AV_INTARG1(%esi)	/* get first arg to interrupt routine */
	call	*%edx			/* call interrupt routine with arg */
	addl	$8, %esp
	movl	%eax, 16(%esp)
	call	__dtrace_probe_interrupt__complete
	addl	$20, %esp
	orb	%al, %bl		/* see if anyone claims intpt. */
	movl	AV_TICKSP(%esi), %ecx
	testl	%ecx, %ecx
	jz	no_time
	call	intr_get_time
	movl	AV_TICKSP(%esi), %ecx
	TSC_ADD_TO(%ecx, 0)
no_time:
	movl	AV_LINK(%esi), %esi	/* get next routine on list */
	testl	%esi, %esi		/* if pointer is non-null */
	jnz	loop2			/* continue */
loop_done2:
	cmpb	$1, %bh		/* if only 1 intpt in chain, it is OK */
	je	.loop_done2_1
	orb	%bl, %bl	/* If no one claims intpt, then it is OK */
	jz	.loop_done2_1
	movl	$autovect, %esi		/* else get autovect structure */
	movl	4(%esp), %ecx		/* restore intr vector */
	jmp	pre_loop2		/* and try again. */
.loop_done2_1:
	popl	%esi			/* restore intr thread pointer */

	LOADCPU(%ebx)

	cli		/* protect interrupt thread pool and intr_actv */
	movzbl	T_PIL(%esi), %eax

	/ Save value in regs
	pushl	%eax			/* current pil */
	pushl	%edx			/* (huh?) */
	pushl	%edi			/* old pil */

	/ cpu_stats.sys.intr[PIL]++
	INC_CPU_STATS_INTR(%eax, %edx, %edx, %ebx)

	/
	/ Take timestamp, compute interval, and update cumulative counter.
	/ esi = thread pointer, ebx = cpu pointer, eax = PIL
	/
	movl	%eax, %edi

	ASSERT_T_INTR_START_NZ(%esi)

_tsc_patch9:
	nop; nop			/* patched to rdtsc if available */
	TSC_SUB_FROM(%esi, T_INTR_START)
	PILBASE_INTRSTAT(%ebx, %edi)
	TSC_ADD_TO(%edi, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %edi)
	TSC_ADD_TO(%edi, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */
	popl	%edi
	popl	%edx
	popl	%eax

	/
	/ Clear bit for this PIL in CPU's interrupt active bitmask.
	/

	ASSERT_CPU_INTR_ACTV(%eax, %ebx, _intr_thread_actv_bit_not_set)

	btrl	%eax, CPU_INTR_ACTV(%ebx)

	/ if there is still an interrupted thread underneath this one
	/ then the interrupt was never blocked and the return is fairly
	/ simple.  Otherwise jump to intr_thread_exit
	cmpl	$0, T_INTR(%esi)
	je	intr_thread_exit

	/
	/ link the thread back onto the interrupt thread pool
	LINK_INTR_THREAD(%ebx, %esi, %edx)

	movl	CPU_BASE_SPL(%ebx), %eax	/* used below. */
	/ set the thread state to free so kmdb doesn't see it
	movl	$FREE_THREAD, T_STATE(%esi)

	cmpl	%eax, %edi		/* if (oldipl >= basespl) */
	jae	intr_restore_ipl	/* then use oldipl */
	movl	%eax, %edi		/* else use basespl */
intr_restore_ipl:
	movl	%edi, CPU_PRI(%ebx)
					/* intr vector already on stack */
	pushl	%edi			/* old ipl */
	call	*setlvlx		/* eax contains the current ipl */
	/
	/ Switch back to the interrupted thread
	movl	T_INTR(%esi), %ecx

	/ Place starting timestamp in interrupted thread's thread structure.
_tsc_patch10:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%ecx, T_INTR_START)

	movl	T_SP(%ecx), %esp	/* restore stack pointer */
	movl	%esp, %ebp
	movl	%ecx, CPU_THREAD(%ebx)

	movl	CPU_SOFTINFO(%ebx), %edx /* any pending software interrupts */
	orl	%edx, %edx
	jz	_sys_rtt
	jmp	dosoftint	/* check for softints before we return. */

	/
	/ An interrupt returned on what was once (and still might be)
	/ an interrupt thread stack, but the interrupted process is no longer
	/ there.  This means the interrupt must have blocked.
	/
	/ There is no longer a thread under this one, so put this thread back
	/ on the CPU's free list and resume the idle thread which will dispatch
	/ the next thread to run.
	/
	/ All interrupts are disabled here
	/

intr_thread_exit:
#ifdef DEBUG
	incl	intr_thread_cnt
#endif
	INC64(%ebx, CPU_STATS_SYS_INTRBLK)	/* cpu_stats.sys.intrblk++ */
	/
	/ Put thread back on the interrupt thread list.
	/ As a reminder, the regs at this point are
	/	esi	interrupt thread
	/	edi	old ipl
	/	ebx	ptr to CPU struct

	/ Set CPU's base SPL level based on active interrupts bitmask
	call	set_base_spl

	movl	CPU_BASE_SPL(%ebx), %edi
	movl	%edi, CPU_PRI(%ebx)
					/* interrupt vector already on stack */
	pushl	%edi
	call	*setlvlx
	addl	$8, %esp		/* XXX - don't need to pop since */
					/* we are ready to switch */
	call	splhigh			/* block all intrs below lock level */
	/
	/ Set the thread state to free so kmdb doesn't see it
	/
	movl	$FREE_THREAD, T_STATE(%esi)
	/
	/ Put thread on either the interrupt pool or the free pool and
	/ call swtch() to resume another thread.
	/
	LINK_INTR_THREAD(%ebx, %esi, %edx)
	call 	swtch
	/ swtch() shouldn't return

	SET_SIZE(intr_thread)

#endif	/* __lint */
#endif	/* __i386 */

/*
 * Set Cpu's base SPL level, base on which interrupt levels are active
 *	Called at spl7 or above.
 */

#if defined(__lint)

void
set_base_spl(void)
{}

#else	/* __lint */

	ENTRY_NP(set_base_spl)
	movl	%gs:CPU_INTR_ACTV, %eax	/* load active interrupts mask */
	testl	%eax, %eax		/* is it zero? */
	jz	setbase
	testl	$0xff00, %eax
	jnz	ah_set
	shl	$24, %eax		/* shift 'em over so we can find */
					/* the 1st bit faster */
	bsrl	%eax, %eax
	subl	$24, %eax
setbase:
	movl	%eax, %gs:CPU_BASE_SPL	/* store base priority */
	ret
ah_set:
	shl	$16, %eax
	bsrl	%eax, %eax
	subl	$16, %eax
	jmp	setbase
	SET_SIZE(set_base_spl)

#endif	/* __lint */

#if defined(__i386)

/*
 * int
 * intr_passivate(from, to)
 *      thread_id_t     from;           interrupt thread
 *      thread_id_t     to;             interrupted thread
 *
 *	intr_passivate(t, itp) makes the interrupted thread "t" runnable.
 *
 *	Since t->t_sp has already been saved, t->t_pc is all that needs
 *	set in this function.
 *
 *	Returns interrupt level of the thread.
 */

#if defined(__lint)

/* ARGSUSED */
int
intr_passivate(kthread_id_t from, kthread_id_t to)
{ return (0); }

#else	/* __lint */

	ENTRY(intr_passivate)
	movl	8(%esp), %eax		/* interrupted thread  */
	movl	$_sys_rtt, T_PC(%eax)	/* set T_PC for interrupted thread */

	movl	4(%esp), %eax		/* interrupt thread */
	movl	T_STACK(%eax), %eax	/* get the pointer to the start of */
					/* of the interrupt thread stack */
	movl	-4(%eax), %eax		/* interrupt level was the first */
					/* thing pushed onto the stack */
	ret
	SET_SIZE(intr_passivate)

#endif	/* __lint */
#endif	/* __i386 */

#if defined(__lint)

void
fakesoftint(void)
{}

#else	/* __lint */

	/
	/ If we're here, we're being called from splx() to fake a soft
	/ interrupt (note that interrupts are still disabled from splx()).
	/ We execute this code when a soft interrupt is posted at
	/ level higher than the CPU's current spl; when spl is lowered in
	/ splx(), it will see the softint and jump here.  We'll do exactly
	/ what a trap would do:  push our flags, %cs, %eip, error code
	/ and trap number (T_SOFTINT).  The cmnint() code will see T_SOFTINT
	/ and branch to the dosoftint() code.
	/
#if defined(__amd64)

	/*
	 * In 64-bit mode, iretq -always- pops all five regs
	 * Imitate the 16-byte auto-align of the stack, and the
	 * zero-ed out %ss value.
	 */
	ENTRY_NP(fakesoftint)
	movq	%rsp, %r11
	andq	$-16, %rsp
	pushq	$KDS_SEL	/* %ss */
	pushq	%r11		/* %rsp */
	pushf			/* rflags */
	pushq	$KCS_SEL	/* %cs */
	leaq	fakesoftint_return(%rip), %r11
	pushq	%r11		/* %rip */
	pushq	$0		/* err */
	pushq	$T_SOFTINT	/* trap */
	jmp	cmnint
	SET_SIZE(fakesoftint)

#elif defined(__i386)

	ENTRY_NP(fakesoftint)
	pushf
	push	%cs
	push	$fakesoftint_return
	push	$0
	push	$T_SOFTINT
	jmp	cmnint
	SET_SIZE(fakesoftint)

#endif	/* __i386 */

	.align	CPTRSIZE
	.globl	_fakesoftint_size
	.type	_fakesoftint_size, @object
_fakesoftint_size:
	.NWORD	. - fakesoftint
	SET_SIZE(_fakesoftint_size)

/*
 * dosoftint(old_pil in %edi, softinfo in %edx, CPU pointer in %ebx)
 * Process software interrupts
 * Interrupts are disabled here.
 */
#if defined(__i386)

	ENTRY_NP(dosoftint)

	bsrl	%edx, %edx		/* find highest pending interrupt */
	cmpl 	%edx, %edi		/* if curipl >= pri soft pending intr */
	jae	_sys_rtt		/* skip */

	movl	%gs:CPU_BASE_SPL, %eax	/* check for blocked intr threads */
	cmpl	%edx, %eax		/* if basespl >= pri soft pending */
	jae	_sys_rtt		/* skip */

	lock				/* MP protect */
	btrl	%edx, CPU_SOFTINFO(%ebx) /* clear the selected interrupt bit */
	jnc	dosoftint_again

	movl	%edx, CPU_PRI(%ebx) /* set IPL to sofint level */
	pushl	%edx
	call	*setspl			/* mask levels upto the softint level */
	popl	%eax			/* priority we are at in %eax */

	/ Get set to run interrupt thread.
	/ There should always be an interrupt thread since we allocate one
	/ for each level on the CPU.
	UNLINK_INTR_THREAD(%ebx, %esi, %edx)

	/
	/ Note that the code in kcpc_overflow_intr -relies- on the ordering
	/ of events here - in particular that t->t_lwp of the interrupt
	/ thread is set to the pinned thread *before* curthread is changed
	/
	movl	CPU_THREAD(%ebx), %ecx

	/ If we are interrupting an interrupt thread, account for it.
	testw	$T_INTR_THREAD, T_FLAGS(%ecx)
	jz	0f
	pushl	%eax
	movl	%eax, %ebp
_tsc_patch11:
	nop; nop			/* patched to rdtsc if available */
	PILBASE_INTRSTAT(%ebx, %ebp)
	TSC_SUB_FROM(%ecx, T_INTR_START)
	TSC_ADD_TO(%ebp, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %ebp)
	TSC_ADD_TO(%ebp, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */
	popl	%eax
0:
	movl	T_LWP(%ecx), %ebp
	movl	%ebp, T_LWP(%esi)
	/
	/ Threads on the interrupt thread free list could have state already
	/ set to TS_ONPROC, but it helps in debugging if they're TS_FREE
	/ Could eliminate the next two instructions with a little work.
	/
	movl	$ONPROC_THREAD, T_STATE(%esi)
	/
	/ Push interrupted thread onto list from new thread.
	/ Set the new thread as the current one.
	/ Set interrupted thread's T_SP because if it is the idle thread,
	/ Resume() may use that stack between threads.
	/
	movl	%esp, T_SP(%ecx)		/* mark stack for resume */
	movl	%ecx, T_INTR(%esi)		/* push old thread */
	movl	%esi, CPU_THREAD(%ebx)		/* set new thread */
	movl	T_STACK(%esi), %esp		/* interrupt stack pointer */
	movl	%esp, %ebp

	pushl	%eax			/* push ipl as first element in stack */
					/* see intr_passivate() */
	/
	/ Set bit for this PIL in CPU's interrupt active bitmask.
	/

	ASSERT_NOT_CPU_INTR_ACTV(%eax, %ebx, _dosoftint_actv_bit_set)

	btsl	%eax, CPU_INTR_ACTV(%ebx)

	/
	/ Initialize thread priority level from intr_pri
	/
	movb	%al, T_PIL(%esi)	/* store pil */
	movzwl	intr_pri, %ecx
	addl	%eax, %ecx		/* convert level to dispatch priority */
	movw	%cx, T_PRI(%esi)

	/
	/ Store starting timestamp in thread structure.
	/ esi = thread, ebx = cpu pointer, eax = PIL
	/
	movl	%eax, %ecx		/* save PIL from rdtsc clobber */
_tsc_patch12:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%esi, T_INTR_START)

	sti				/* enable interrupts */

	/
	/ Enabling interrupts (above) could raise the current
	/ IPL and base SPL. But, we continue processing the current soft
	/ interrupt and we will check the base SPL next time in the loop
	/ so that blocked interrupt thread would get a chance to run.
	/

	/
	/ dispatch soft interrupts
	/
	pushl	%ecx
	call	av_dispatch_softvect
	addl	$4, %esp

	cli				/* protect interrupt thread pool */
					/* and softinfo & sysinfo */
	movl	CPU_THREAD(%ebx), %esi	/* restore thread pointer */
	movzbl	T_PIL(%esi), %ecx

	/ cpu_stats.sys.intr[PIL]++
	INC_CPU_STATS_INTR(%ecx, %edx, %edx, %ebx)

	/
	/ Clear bit for this PIL in CPU's interrupt active bitmask.
	/

	ASSERT_CPU_INTR_ACTV(%ecx, %ebx, _dosoftint_actv_bit_not_set)

	btrl	%ecx, CPU_INTR_ACTV(%ebx)

	/
	/ Take timestamp, compute interval, update cumulative counter.
	/ esi = thread, ebx = cpu, ecx = PIL
	/
	PILBASE_INTRSTAT(%ebx, %ecx)
_tsc_patch13:
	nop; nop		/* patched to rdtsc if available */
	TSC_SUB_FROM(%esi, T_INTR_START)
	TSC_ADD_TO(%ecx, CPU_INTRSTAT)
	INTRACCTBASE(%ebx, %ecx)
	TSC_ADD_TO(%ecx, CPU_INTRACCT)	/* cpu_intracct[cpu_mstate] += tsc */

	/ if there is still an interrupt thread underneath this one
	/ then the interrupt was never blocked and the return is fairly
	/ simple.  Otherwise jump to softintr_thread_exit.
	/ softintr_thread_exit expect esi to be curthread & ebx to be ipl.
	cmpl	$0, T_INTR(%esi)
	je	softintr_thread_exit

	/
	/ link the thread back onto the interrupt thread pool
	LINK_INTR_THREAD(%ebx, %esi, %edx)

	/ set the thread state to free so kmdb doesn't see it
	movl	$FREE_THREAD, T_STATE(%esi)
	/
	/ Switch back to the interrupted thread
	movl	T_INTR(%esi), %ecx
	movl	%ecx, CPU_THREAD(%ebx)
	movl	T_SP(%ecx), %esp	/* restore stack pointer */
	movl	%esp, %ebp

	/ If we are returning to an interrupt thread, store a starting
	/ timestamp in the thread structure.
	testw	$T_INTR_THREAD, T_FLAGS(%ecx)
	jz	0f
_tsc_patch14:
	nop; nop			/* patched to rdtsc if available */
	TSC_STORE(%ecx, T_INTR_START)
0:
	movl	CPU_BASE_SPL(%ebx), %eax
	cmpl	%eax, %edi		/* if (oldipl >= basespl) */
	jae	softintr_restore_ipl	/* then use oldipl */
	movl	%eax, %edi		/* else use basespl */
softintr_restore_ipl:
	movl	%edi, CPU_PRI(%ebx) /* set IPL to old level */
	pushl	%edi
	call	*setspl
	popl	%eax
dosoftint_again:
	movl	CPU_SOFTINFO(%ebx), %edx /* any pending software interrupts */
	orl	%edx, %edx
	jz	_sys_rtt
	jmp	dosoftint		/* process more software interrupts */

softintr_thread_exit:
	/
	/ Put thread back on the interrupt thread list.
	/ As a reminder, the regs at this point are
	/	%esi	interrupt thread

	/
	/ This was an interrupt thread, so set CPU's base SPL level
	/ set_base_spl only uses %eax.
	/
	call	set_base_spl		/* interrupt vector already on stack */
	/
	/ Set the thread state to free so kmdb doesn't see it
	/
	movl	$FREE_THREAD, T_STATE(%esi)
	/
	/ Put thread on either the interrupt pool or the free pool and
	/ call swtch() to resume another thread.
	/
	LOADCPU(%ebx)
	LINK_INTR_THREAD(%ebx, %esi, %edx)
	call	splhigh			/* block all intrs below lock lvl */
	call	swtch
	/ swtch() shouldn't return
	SET_SIZE(dosoftint)

#endif	/* __i386 */
#endif	/* __lint */

#if defined(lint)

/*
 * intr_get_time() is a resource for interrupt handlers to determine how
 * much time has been spent handling the current interrupt. Such a function
 * is needed because higher level interrupts can arrive during the
 * processing of an interrupt, thus making direct comparisons of %tick by
 * the handler inaccurate. intr_get_time() only returns time spent in the
 * current interrupt handler.
 *
 * The caller must be calling from an interrupt handler running at a pil
 * below or at lock level. Timings are not provided for high-level
 * interrupts.
 *
 * The first time intr_get_time() is called while handling an interrupt,
 * it returns the time since the interrupt handler was invoked. Subsequent
 * calls will return the time since the prior call to intr_get_time(). Time
 * is returned as ticks. Use tsc_scalehrtime() to convert ticks to nsec.
 *
 * Theory Of Intrstat[][]:
 *
 * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
 * uint64_ts per pil.
 *
 * intrstat[pil][0] is a cumulative count of the number of ticks spent
 * handling all interrupts at the specified pil on this CPU. It is
 * exported via kstats to the user.
 *
 * intrstat[pil][1] is always a count of ticks less than or equal to the
 * value in [0]. The difference between [1] and [0] is the value returned
 * by a call to intr_get_time(). At the start of interrupt processing,
 * [0] and [1] will be equal (or nearly so). As the interrupt consumes
 * time, [0] will increase, but [1] will remain the same. A call to
 * intr_get_time() will return the difference, then update [1] to be the
 * same as [0]. Future calls will return the time since the last call.
 * Finally, when the interrupt completes, [1] is updated to the same as [0].
 *
 * Implementation:
 *
 * intr_get_time() works much like a higher level interrupt arriving. It
 * "checkpoints" the timing information by incrementing intrstat[pil][0]
 * to include elapsed running time, and by setting t_intr_start to rdtsc.
 * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
 * and updates intrstat[pil][1] to be the same as the new value of
 * intrstat[pil][0].
 *
 * In the normal handling of interrupts, after an interrupt handler returns
 * and the code in intr_thread() updates intrstat[pil][0], it then sets
 * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
 * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
 * is 0.
 *
 * Whenever interrupts arrive on a CPU which is handling a lower pil
 * interrupt, they update the lower pil's [0] to show time spent in the
 * handler that they've interrupted. This results in a growing discrepancy
 * between [0] and [1], which is returned the next time intr_get_time() is
 * called. Time spent in the higher-pil interrupt will not be returned in
 * the next intr_get_time() call from the original interrupt, because
 * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
 */

/*ARGSUSED*/
uint64_t
intr_get_time(void)
{ return 0; }
#else	/* lint */


#if defined(__amd64)
	ENTRY_NP(intr_get_time)
	cli				/* make this easy -- block intrs */
	LOADCPU(%rdi)
	call	intr_thread_get_time
	sti
	ret
	SET_SIZE(intr_get_time)

#elif defined(__i386)

#ifdef DEBUG


_intr_get_time_high_pil:
	.string	"intr_get_time(): %pil > LOCK_LEVEL"
_intr_get_time_not_intr:
	.string	"intr_get_time(): not called from an interrupt thread"
_intr_get_time_no_start_time:
	.string	"intr_get_time(): t_intr_start == 0"

/*
 * ASSERT(%pil <= LOCK_LEVEL)
 */
#define	ASSERT_PIL_BELOW_LOCK_LEVEL(cpureg)				\
	testl	$CPU_INTR_ACTV_HIGH_LEVEL_MASK, CPU_INTR_ACTV(cpureg);	\
	jz	0f;							\
	__PANIC(_intr_get_time_high_pil, 0f);				\
0:

/*
 * ASSERT((t_flags & T_INTR_THREAD) != 0 && t_pil > 0)
 */
#define	ASSERT_NO_PIL_0_INTRS(thrreg)			\
	testw	$T_INTR_THREAD, T_FLAGS(thrreg);	\
	jz	1f;					\
	cmpb	$0, T_PIL(thrreg);			\
	jne	0f;					\
1:							\
	__PANIC(_intr_get_time_not_intr, 0f);		\
0:

/*
 * ASSERT(t_intr_start != 0)
 */
#define	ASSERT_INTR_START_NOT_0(thrreg)			\
	cmpl	$0, T_INTR_START(thrreg);		\
	jnz	0f;					\
	cmpl	$0, T_INTR_START+4(thrreg);		\
	jnz	0f;					\
	__PANIC(_intr_get_time_no_start_time, 0f);	\
0:

#endif /* DEBUG */

	ENTRY_NP(intr_get_time)

	cli				/* make this easy -- block intrs */
	pushl	%esi			/* and free up some registers */

	LOADCPU(%esi)
	movl	CPU_THREAD(%esi), %ecx

#ifdef DEBUG
	ASSERT_PIL_BELOW_LOCK_LEVEL(%esi)
	ASSERT_NO_PIL_0_INTRS(%ecx)
	ASSERT_INTR_START_NOT_0(%ecx)
#endif /* DEBUG */

_tsc_patch17:
	nop; nop			/* patched to rdtsc if available */
	TSC_SUB_FROM(%ecx, T_INTR_START)	/* get elapsed time */
	TSC_ADD_TO(%ecx, T_INTR_START)		/* T_INTR_START = rdtsc */

	movzbl	T_PIL(%ecx), %ecx		/* %ecx = pil */
	PILBASE_INTRSTAT(%esi, %ecx)		/* %ecx = CPU + pil*16 */
	TSC_ADD_TO(%ecx, CPU_INTRSTAT)		/* intrstat[0] += elapsed */
	TSC_LOAD(%ecx, CPU_INTRSTAT)		/* get new intrstat[0] */
	TSC_SUB_FROM(%ecx, CPU_INTRSTAT+8)	/* diff with intrstat[1] */
	TSC_ADD_TO(%ecx, CPU_INTRSTAT+8)	/* intrstat[1] = intrstat[0] */

	/* %edx/%eax contain difference between old and new intrstat[1] */

	popl	%esi
	sti
	ret
	SET_SIZE(intr_get_time)
#endif	/* __i386 */

#endif  /* lint */