sfmmu/ml/sfmmu_asm.s

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 * SFMMU primitives.  These primitives should only be used by sfmmu
 * routines.
 */

#if defined(lint)
#include <sys/types.h>
#else	/* lint */
#include "assym.h"
#endif	/* lint */

#include <sys/asm_linkage.h>
#include <sys/machtrap.h>
#include <sys/machasi.h>
#include <sys/sun4asi.h>
#include <sys/pte.h>
#include <sys/mmu.h>
#include <vm/hat_sfmmu.h>
#include <vm/seg_spt.h>
#include <sys/machparam.h>
#include <sys/privregs.h>
#include <sys/scb.h>
#include <sys/intreg.h>
#include <sys/machthread.h>
#include <sys/intr.h>
#include <sys/clock.h>
#include <sys/trapstat.h>

#ifdef TRAPTRACE
#include <sys/traptrace.h>

/*
 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
 */
#define	TT_TRACE(label)		\
	ba	label		;\
	rd	%pc, %g7
#else

#define	TT_TRACE(label)

#endif /* TRAPTRACE */

#ifndef	lint

#if (TTE_SUSPEND_SHIFT > 0)
#define	TTE_SUSPEND_INT_SHIFT(reg)				\
	sllx	reg, TTE_SUSPEND_SHIFT, reg
#else
#define	TTE_SUSPEND_INT_SHIFT(reg)
#endif

#endif /* lint */

#ifndef	lint

/*
 * Assumes TSBE_TAG is 0
 * Assumes TSBE_INTHI is 0
 * Assumes TSBREG.split is 0
 */

#if TSBE_TAG != 0
#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
#endif

#if TSBTAG_INTHI != 0
#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
#endif

/*
 * The following code assumes the tsb is not split.
 *
 * With TSBs no longer shared between processes, it's no longer
 * necessary to hash the context bits into the tsb index to get
 * tsb coloring; the new implementation treats the TSB as a
 * direct-mapped, virtually-addressed cache.
 *
 * In:
 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
 *    tsbbase = base address of TSB (clobbered)
 *    tagacc = tag access register (clobbered)
 *    szc = size code of TSB (ro)
 *    tmp = scratch reg
 * Out:
 *    tsbbase = pointer to entry in TSB
 */
#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
	srlx	tagacc, vpshift, tagacc 				;\
	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */

/*
 * When the kpm TSB is used it is assumed that it is direct mapped
 * using (vaddr>>vpshift)%tsbsz as the index.
 *
 * Note that, for now, the kpm TSB and kernel TSB are the same for
 * each mapping size.  However that need not always be the case.  If
 * the trap handlers are updated to search a different TSB for kpm
 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
 *
 * In:
 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
 *    vaddr = virtual address (clobbered)
 *    tsbp, szc, tmp = scratch
 * Out:
 *    tsbp = pointer to entry in TSB
 */
#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
	cmp	vpshift, MMU_PAGESHIFT					;\
	bne,pn	%icc, 1f		/* branch if large case */	;\
	  sethi	%hi(kpmsm_tsbsz), szc					;\
	sethi	%hi(kpmsm_tsbbase), tsbp				;\
	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
	ba,pt	%icc, 2f						;\
	  nop								;\
1:	sethi	%hi(kpm_tsbsz), szc					;\
	sethi	%hi(kpm_tsbbase), tsbp					;\
	ld	[szc + %lo(kpm_tsbsz)], szc				;\
	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
2:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)

/*
 * Lock the TSBE at virtual address tsbep.
 *
 * tsbep = TSBE va (ro)
 * tmp1, tmp2 = scratch registers (clobbered)
 * label = label to use for branches (text)
 * %asi = ASI to use for TSB access
 *
 * NOTE that we flush the TSB using fast VIS instructions that
 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
 * not be treated as a locked entry or we'll get stuck spinning on
 * an entry that isn't locked but really invalid.
 */

#if defined(UTSB_PHYS)

#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
	lda	[tsbep]ASI_MEM, tmp1					;\
label:									;\
	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
	cmp	tmp1, tmp2 						;\
	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
	  lda	[tsbep]ASI_MEM, tmp1					;\
	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
	cmp	tmp1, tmp2 						;\
	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
	  lda	[tsbep]ASI_MEM, tmp1					;\
	/* tsbe lock acquired */					;\
	membar #StoreStore

#else /* UTSB_PHYS */

#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
	lda	[tsbep]%asi, tmp1					;\
label:									;\
	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
	cmp	tmp1, tmp2 						;\
	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
	  lda	[tsbep]%asi, tmp1					;\
	casa	[tsbep]%asi, tmp1, tmp2					;\
	cmp	tmp1, tmp2 						;\
	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
	  lda	[tsbep]%asi, tmp1					;\
	/* tsbe lock acquired */					;\
	membar #StoreStore

#endif /* UTSB_PHYS */

/*
 * Atomically write TSBE at virtual address tsbep.
 *
 * tsbep = TSBE va (ro)
 * tte = TSBE TTE (ro)
 * tagtarget = TSBE tag (ro)
 * %asi = ASI to use for TSB access
 */

#if defined(UTSB_PHYS)

#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
	add	tsbep, TSBE_TTE, tmp1					;\
	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
	membar #StoreStore						;\
	add	tsbep, TSBE_TAG, tmp1					;\
	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */

#else /* UTSB_PHYS */

#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
	membar #StoreStore						;\
	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */

#endif /* UTSB_PHYS */

/*
 * Load an entry into the TSB at TL > 0.
 *
 * tsbep = pointer to the TSBE to load as va (ro)
 * tte = value of the TTE retrieved and loaded (wo)
 * tagtarget = tag target register.  To get TSBE tag to load,
 *   we need to mask off the context and leave only the va (clobbered)
 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
 * tmp1, tmp2 = scratch registers
 * label = label to use for branches (text)
 * %asi = ASI to use for TSB access
 */

#if defined(UTSB_PHYS)

#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
	/*								;\
	 * I don't need to update the TSB then check for the valid tte.	;\
	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
	 * we always invalidate the hash table before we unload the TSB.;\
	 */								;\
	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
	ldxa	[ttepa]ASI_MEM, tte					;\
	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
	sethi	%hi(TSBTAG_INVALID), tmp2				;\
	add	tsbep, TSBE_TAG, tmp1					;\
	brgez,a,pn tte, label/**/f					;\
	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
label:

#else /* UTSB_PHYS */

#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
	/*								;\
	 * I don't need to update the TSB then check for the valid tte.	;\
	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
	 * we always invalidate the hash table before we unload the TSB.;\
	 */								;\
	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
	ldxa	[ttepa]ASI_MEM, tte					;\
	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
	sethi	%hi(TSBTAG_INVALID), tmp2				;\
	brgez,a,pn tte, label/**/f					;\
	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
label:

#endif /* UTSB_PHYS */

/*
 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
 *   for ITLB synthesis.
 *
 * tsbep = pointer to the TSBE to load as va (ro)
 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
 *   with exec_perm turned off and exec_synth turned on
 * tagtarget = tag target register.  To get TSBE tag to load,
 *   we need to mask off the context and leave only the va (clobbered)
 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
 * tmp1, tmp2 = scratch registers
 * label = label to use for branch (text)
 * %asi = ASI to use for TSB access
 */

#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
	/*								;\
	 * I don't need to update the TSB then check for the valid tte.	;\
	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
	 * we always invalidate the hash table before we unload the TSB.;\
	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
	 * and exec_synth bit to 1.					;\
	 */								;\
	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
	mov	tte, tmp1						;\
	ldxa	[ttepa]ASI_MEM, tte					;\
	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
	sethi	%hi(TSBTAG_INVALID), tmp2				;\
	brgez,a,pn tte, label/**/f					;\
	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
	or	tte, tmp1, tte						;\
	andn	tte, TTE_EXECPRM_INT, tte				;\
	or	tte, TTE_E_SYNTH_INT, tte				;\
	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
label:

/*
 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
 *
 * tte = value of the TTE, used to get tte_size bits (ro)
 * tagaccess = tag access register, used to get 4M pfn bits (ro)
 * pfn = 4M pfn bits shifted to offset for tte (out)
 * tmp1 = scratch register
 * label = label to use for branch (text)
 */

#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
	/*								;\
	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
	 * Return them, shifted, in pfn.				;\
	 */								;\
	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
label:									;\
	sllx	tagaccess, MMU_PAGESHIFT4M, pfn

/*
 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
 * for ITLB synthesis.
 *
 * tte = value of the TTE, used to get tte_size bits (rw)
 * tmp1 = scratch register
 */

#define	SET_TTE4M_PN(tte, tmp)						\
	/*								;\
	 * Set 4M pagesize tte bits. 					;\
	 */								;\
	set	TTE4M, tmp						;\
	sllx	tmp, TTE_SZ_SHFT, tmp					;\
	or	tte, tmp, tte

/*
 * Load an entry into the TSB at TL=0.
 *
 * tsbep = pointer to the TSBE to load as va (ro)
 * tteva = pointer to the TTE to load as va (ro)
 * tagtarget = TSBE tag to load (which contains no context), synthesized
 * to match va of MMU tag target register only (ro)
 * tmp1, tmp2 = scratch registers (clobbered)
 * label = label to use for branches (text)
 * %asi = ASI to use for TSB access
 */

#if defined(UTSB_PHYS)

#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
	/* can't rd tteva after locking tsb because it can tlb miss */	;\
	ldx	[tteva], tteva			/* load tte */		;\
	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
	sethi	%hi(TSBTAG_INVALID), tmp2				;\
	add	tsbep, TSBE_TAG, tmp1					;\
	brgez,a,pn tteva, label/**/f					;\
	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
label:

#else /* UTSB_PHYS */

#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
	/* can't rd tteva after locking tsb because it can tlb miss */	;\
	ldx	[tteva], tteva			/* load tte */		;\
	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
	sethi	%hi(TSBTAG_INVALID), tmp2				;\
	brgez,a,pn tteva, label/**/f					;\
	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
label:

#endif /* UTSB_PHYS */

/*
 * Invalidate a TSB entry in the TSB.
 *
 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
 *	 about this earlier to ensure this is true.  Thus when we are
 *	 directly referencing tsbep below, we are referencing the tte_tag
 *	 field of the TSBE.  If this  offset ever changes, the code below
 *	 will need to be modified.
 *
 * tsbep = pointer to TSBE as va (ro)
 * tag = invalidation is done if this matches the TSBE tag (ro)
 * tmp1 - tmp3 = scratch registers (clobbered)
 * label = label name to use for branches (text)
 * %asi = ASI to use for TSB access
 */

#if defined(UTSB_PHYS)

#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
label/**/1:								;\
	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
	cmp	tag, tmp3		/* compare tags */		;\
	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
	cmp	tmp1, tmp3		/* if not successful */		;\
	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
label/**/2:

#else /* UTSB_PHYS */

#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
label/**/1:								;\
	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
	cmp	tag, tmp3		/* compare tags */		;\
	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
	cmp	tmp1, tmp3		/* if not successful */		;\
	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
label/**/2:

#endif /* UTSB_PHYS */

#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
#error	- TSB_SOFTSZ_MASK too small
#endif


/*
 * An implementation of setx which will be hot patched at run time.
 * since it is being hot patched, there is no value passed in.
 * Thus, essentially we are implementing
 *	setx value, tmp, dest
 * where value is RUNTIME_PATCH (aka 0) in this case.
 */
#define	RUNTIME_PATCH_SETX(dest, tmp)					\
	sethi	%hh(RUNTIME_PATCH), tmp					;\
	sethi	%lm(RUNTIME_PATCH), dest				;\
	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
	or	dest, %lo(RUNTIME_PATCH), dest				;\
	sllx	tmp, 32, tmp						;\
	nop				/* for perf reasons */		;\
	or	tmp, dest, dest		/* contents of patched value */


#endif (lint)


#if defined (lint)

/*
 * sfmmu related subroutines
 */
uint_t
sfmmu_disable_intrs()
{ return(0); }

/* ARGSUSED */
void
sfmmu_enable_intrs(uint_t pstate_save)
{}

/* ARGSUSED */
void
sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp)
{}

/*
 * Use cas, if tte has changed underneath us then reread and try again.
 * In the case of a retry, it will update sttep with the new original.
 */
/* ARGSUSED */
int
sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
{ return(0); }

/*
 * Use cas, if tte has changed underneath us then return 1, else return 0
 */
/* ARGSUSED */
int
sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
{ return(0); }

/* ARGSUSED */
void
sfmmu_copytte(tte_t *sttep, tte_t *dttep)
{}

/*ARGSUSED*/
struct tsbe *
sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
{ return(0); }

/*ARGSUSED*/
uint64_t
sfmmu_make_tsbtag(caddr_t va)
{ return(0); }

#else	/* lint */

	.seg	".data"
	.global	sfmmu_panic1
sfmmu_panic1:
	.asciz	"sfmmu_asm: interrupts already disabled"

	.global	sfmmu_panic3
sfmmu_panic3:
	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"

	.global	sfmmu_panic4
sfmmu_panic4:
	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"

	.global	sfmmu_panic5
sfmmu_panic5:
	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"

	.global	sfmmu_panic6
sfmmu_panic6:
	.asciz	"sfmmu_asm: interrupts not disabled"

	.global	sfmmu_panic7
sfmmu_panic7:
	.asciz	"sfmmu_asm: kernel as"

	.global	sfmmu_panic8
sfmmu_panic8:
	.asciz	"sfmmu_asm: gnum is zero"

	.global	sfmmu_panic9
sfmmu_panic9:
	.asciz	"sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"

        ENTRY(sfmmu_disable_intrs)
        rdpr    %pstate, %o0
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
#endif /* DEBUG */
        retl
          wrpr   %o0, PSTATE_IE, %pstate
        SET_SIZE(sfmmu_disable_intrs)

	ENTRY(sfmmu_enable_intrs)
        retl
          wrpr    %g0, %o0, %pstate
        SET_SIZE(sfmmu_enable_intrs)

/*
 * This routine is called both by resume() and sfmmu_get_ctx() to
 * allocate a new context for the process on a MMU.
 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
 * is the case when sfmmu_alloc_ctx is called from resume().
 *
 * The caller must disable interrupts before entering this routine.
 * To reduce ctx switch overhead, the code contains both 'fast path' and
 * 'slow path' code. The fast path code covers the common case where only
 * a quick check is needed and the real ctx allocation is not required.
 * It can be done without holding the per-process (PP) lock.
 * The 'slow path' code must be protected by the PP Lock and performs ctx
 * allocation.
 * Hardware context register and HAT mmu cnum are updated accordingly.
 *
 * %o0 - sfmmup
 * %o1 - allocflag
 * %o2 - CPU
 */
        ENTRY_NP(sfmmu_alloc_ctx)

#ifdef DEBUG
	sethi   %hi(ksfmmup), %o3
	ldx     [%o3 + %lo(ksfmmup)], %o3
	cmp     %o3, %o0
	bne,pt   %xcc, 0f
	  nop

	sethi   %hi(panicstr), %g1		! if kernel as, panic
        ldx     [%g1 + %lo(panicstr)], %g1
        tst     %g1
        bnz,pn  %icc, 7f
          nop

	sethi	%hi(sfmmu_panic7), %o0
	call	panic
	  or	%o0, %lo(sfmmu_panic7), %o0

7:
	retl
	  nop

0:
	PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
#endif /* DEBUG */

	! load global mmu_ctxp info
	ldx	[%o2 + CPU_MMU_CTXP], %o3		! %o3 = mmu_ctx_t ptr
        lduw	[%o2 + CPU_MMU_IDX], %g2		! %g2 = mmu index

	! load global mmu_ctxp gnum
	ldx	[%o3 + MMU_CTX_GNUM], %o4		! %o4 = mmu_ctxp->gnum

#ifdef DEBUG
	cmp	%o4, %g0		! mmu_ctxp->gnum should never be 0
	bne,pt	%xcc, 3f
	  nop

	sethi   %hi(panicstr), %g1	! test if panicstr is already set
        ldx     [%g1 + %lo(panicstr)], %g1
        tst     %g1
        bnz,pn  %icc, 3f
          nop

	sethi	%hi(sfmmu_panic8), %o0
	call	panic
	  or	%o0, %lo(sfmmu_panic8), %o0
3:
#endif

	! load HAT sfmmu_ctxs[mmuid] gnum, cnum

	sllx	%g2, SFMMU_MMU_CTX_SHIFT, %g2
	add	%o0, %g2, %g2		! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS

	/*
	 * %g5 = sfmmu gnum returned
	 * %g6 = sfmmu cnum returned
	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
	 * %g4 = scratch
	 *
	 * Fast path code, do a quick check.
	 */
	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)

	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
	bne,pt	%icc, 1f			! valid hat cnum, check gnum
	  nop

	! cnum == INVALID, check allocflag
	brz,pt  %o1, 8f		! allocflag == 0, skip ctx allocation, bail
	  mov	%g6, %o1

	! (invalid HAT cnum) && (allocflag == 1)
	ba,pt	%icc, 2f
	  nop
1:
	! valid HAT cnum, check gnum
	cmp	%g5, %o4
	be,a,pt	%icc, 8f			! gnum unchanged, go to done
	  mov	%g6, %o1

2:
	/*
	 * Grab per process (PP) sfmmu_ctx_lock spinlock,
	 * followed by the 'slow path' code.
	 */
	ldstub	[%o0 + SFMMU_CTX_LOCK], %g3	! %g3 = per process (PP) lock
3:
	brz	%g3, 5f
	  nop
4:
	brnz,a,pt       %g3, 4b				! spin if lock is 1
	  ldub	[%o0 + SFMMU_CTX_LOCK], %g3
	ba	%xcc, 3b				! retry the lock
	  ldstub	[%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock

5:
	membar  #LoadLoad
	/*
	 * %g5 = sfmmu gnum returned
	 * %g6 = sfmmu cnum returned
	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
	 * %g4 = scratch
	 */
	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)

	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
	bne,pt	%icc, 1f			! valid hat cnum, check gnum
	  nop

	! cnum == INVALID, check allocflag
	brz,pt	%o1, 2f		! allocflag == 0, called from resume, set hw
	  mov	%g6, %o1

	! (invalid HAT cnum) && (allocflag == 1)
	ba,pt	%icc, 6f
	  nop
1:
	! valid HAT cnum, check gnum
	cmp	%g5, %o4
	be,a,pt	%icc, 2f			! gnum unchanged, go to done
	  mov	%g6, %o1

	ba,pt	%icc, 6f
	  nop
2:
	membar  #LoadStore|#StoreStore
	ba,pt %icc, 8f
	  clrb  [%o0 + SFMMU_CTX_LOCK]
6:
	/*
	 * We get here if we do not have a valid context, or
	 * the HAT gnum does not match global gnum. We hold
	 * sfmmu_ctx_lock spinlock. Allocate that context.
	 *
	 * %o3 = mmu_ctxp
	 */
	add	%o3, MMU_CTX_CNUM, %g3
	ld	[%o3 + MMU_CTX_NCTXS], %g4

	/*
         * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
         * %g3 = mmu cnum address
	 * %g4 = mmu nctxs
	 *
	 * %o0 = sfmmup
	 * %o1 = mmu current cnum value (used as new cnum)
	 * %o4 = mmu gnum
	 *
	 * %o5 = scratch
	 */
	ld	[%g3], %o1
0:
	cmp	%o1, %g4
	bl,a,pt %icc, 1f
	  add	%o1, 1, %o5		! %o5 = mmu_ctxp->cnum + 1

	/*
	 * cnum reachs max, update HAT with INVALID
	 */
	set	INVALID_CONTEXT, %o1

	/*
	 * update hat cnum to INVALID, sun4v sfmmu_load_mmustate checks
	 * hat cnum to determine if set the number of TSBs to 0.
	 */
	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
	or	%o4, %o1, %o4
	stx	%o4, [%g2 + SFMMU_CTXS]

	membar  #LoadStore|#StoreStore
	ba,pt	%icc, 8f
	  clrb	[%o0 + SFMMU_CTX_LOCK]
1:
	! %g3 = addr of mmu_ctxp->cnum
	! %o5 = mmu_ctxp->cnum + 1
	cas	[%g3], %o1, %o5
	cmp	%o1, %o5
	bne,a,pn %xcc, 0b	! cas failed
	  ld	[%g3], %o1

#ifdef DEBUG
        set	MAX_SFMMU_CTX_VAL, %o5
	cmp	%o1, %o5
	ble,pt %icc, 2f
	  nop

	sethi	%hi(sfmmu_panic9), %o0
	call	panic
	  or	%o0, %lo(sfmmu_panic9), %o0
2:
#endif
	! update hat gnum and cnum
	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
	or	%o4, %o1, %o4
	stx	%o4, [%g2 + SFMMU_CTXS]

	membar  #LoadStore|#StoreStore
	clrb	[%o0 + SFMMU_CTX_LOCK]

8:
	/*
	 * program the secondary context register
	 *
	 * %o1 = cnum
	 */
#ifdef	sun4u
	ldub	[%o0 + SFMMU_CEXT], %o2
	sll	%o2, CTXREG_EXT_SHIFT, %o2
	or	%o1, %o2, %o1
#endif

	mov	MMU_SCONTEXT, %o4
	sethi	%hi(FLUSH_ADDR), %o5
	stxa	%o1, [%o4]ASI_MMU_CTX	         ! set 2nd context reg.
	flush	%o5

	retl
	nop

	SET_SIZE(sfmmu_alloc_ctx)


	ENTRY_NP(sfmmu_modifytte)
	ldx	[%o2], %g3			/* current */
	ldx	[%o0], %g1			/* original */
2:
	ldx	[%o1], %g2			/* modified */
	cmp	%g2, %g3			/* is modified = current? */
	be,a,pt	%xcc,1f				/* yes, don't write */
	stx	%g3, [%o0]			/* update new original */
	casx	[%o2], %g1, %g2
	cmp	%g1, %g2
	be,pt	%xcc, 1f			/* cas succeeded - return */
	  nop
	ldx	[%o2], %g3			/* new current */
	stx	%g3, [%o0]			/* save as new original */
	ba,pt	%xcc, 2b
	  mov	%g3, %g1
1:	retl
	membar	#StoreLoad
	SET_SIZE(sfmmu_modifytte)

	ENTRY_NP(sfmmu_modifytte_try)
	ldx	[%o1], %g2			/* modified */
	ldx	[%o2], %g3			/* current */
	ldx	[%o0], %g1			/* original */
	cmp	%g3, %g2			/* is modified = current? */
	be,a,pn %xcc,1f				/* yes, don't write */
	mov	0, %o1				/* as if cas failed. */

	casx	[%o2], %g1, %g2
	membar	#StoreLoad
	cmp	%g1, %g2
	movne	%xcc, -1, %o1			/* cas failed. */
	move	%xcc, 1, %o1			/* cas succeeded. */
1:
	stx	%g2, [%o0]			/* report "current" value */
	retl
	mov	%o1, %o0
	SET_SIZE(sfmmu_modifytte_try)

	ENTRY_NP(sfmmu_copytte)
	ldx	[%o0], %g1
	retl
	stx	%g1, [%o1]
	SET_SIZE(sfmmu_copytte)


	/*
	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
	 * %o0 = TSB base address (in), pointer to TSB entry (out)
	 * %o1 = vaddr (in)
	 * %o2 = vpshift (in)
	 * %o3 = tsb size code (in)
	 * %o4 = scratch register
	 */
	ENTRY_NP(sfmmu_get_tsbe)
	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
	retl
	nop
	SET_SIZE(sfmmu_get_tsbe)

	/*
	 * Return a TSB tag for the given va.
	 * %o0 = va (in/clobbered)
	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
	 */
	ENTRY_NP(sfmmu_make_tsbtag)
	retl
	srln	%o0, TTARGET_VA_SHIFT, %o0
	SET_SIZE(sfmmu_make_tsbtag)

#endif /* lint */

/*
 * Other sfmmu primitives
 */


#if defined (lint)
void
sfmmu_patch_ktsb(void)
{
}

void
sfmmu_kpm_patch_tlbm(void)
{
}

void
sfmmu_kpm_patch_tsbm(void)
{
}

/* ARGSUSED */
void
sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
{
}

/* ARGSUSED */
void
sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
{
}

/* ARGSUSED */
void
sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
{
}

/* ARGSUSED */
void
sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
{
}

#else /* lint */

#define	I_SIZE		4

	ENTRY_NP(sfmmu_fix_ktlb_traptable)
	/*
	 * %o0 = start of patch area
	 * %o1 = size code of TSB to patch
	 * %o3 = scratch
	 */
	/* fix sll */
	ld	[%o0], %o3			/* get sll */
	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
	st	%o3, [%o0]			/* write sll */
	flush	%o0
	/* fix srl */
	add	%o0, I_SIZE, %o0		/* goto next instr. */
	ld	[%o0], %o3			/* get srl */
	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
	st	%o3, [%o0]			/* write srl */
	retl
	flush	%o0
	SET_SIZE(sfmmu_fix_ktlb_traptable)

	ENTRY_NP(sfmmu_fixup_ktsbbase)
	/*
	 * %o0 = start of patch area
	 * %o5 = kernel virtual or physical tsb base address
	 * %o2, %o3 are used as scratch registers.
	 */
	/* fixup sethi instruction */
	ld	[%o0], %o3
	srl	%o5, 10, %o2			! offset is bits 32:10
	or	%o3, %o2, %o3			! set imm22
	st	%o3, [%o0]
	/* fixup offset of lduw/ldx */
	add	%o0, I_SIZE, %o0		! next instr
	ld	[%o0], %o3
	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
	or	%o3, %o2, %o3
	st	%o3, [%o0]
	retl
	flush	%o0
	SET_SIZE(sfmmu_fixup_ktsbbase)

	ENTRY_NP(sfmmu_fixup_setx)
	/*
	 * %o0 = start of patch area
	 * %o4 = 64 bit value to patch
	 * %o2, %o3 are used as scratch registers.
	 *
	 * Note: Assuming that all parts of the instructions which need to be
	 *	 patched correspond to RUNTIME_PATCH (aka 0)
	 *
	 * Note the implementation of setx which is being patched is as follows:
	 *
	 * sethi   %hh(RUNTIME_PATCH), tmp
	 * sethi   %lm(RUNTIME_PATCH), dest
	 * or      tmp, %hm(RUNTIME_PATCH), tmp
	 * or      dest, %lo(RUNTIME_PATCH), dest
	 * sllx    tmp, 32, tmp
	 * nop
	 * or      tmp, dest, dest
	 *
	 * which differs from the implementation in the
	 * "SPARC Architecture Manual"
	 */
	/* fixup sethi instruction */
	ld	[%o0], %o3
	srlx	%o4, 42, %o2			! bits [63:42]
	or	%o3, %o2, %o3			! set imm22
	st	%o3, [%o0]
	/* fixup sethi instruction */
	add	%o0, I_SIZE, %o0		! next instr
	ld	[%o0], %o3
	sllx	%o4, 32, %o2			! clear upper bits
	srlx	%o2, 42, %o2			! bits [31:10]
	or	%o3, %o2, %o3			! set imm22
	st	%o3, [%o0]
	/* fixup or instruction */
	add	%o0, I_SIZE, %o0		! next instr
	ld	[%o0], %o3
	srlx	%o4, 32, %o2			! bits [63:32]
	and	%o2, 0x3ff, %o2			! bits [41:32]
	or	%o3, %o2, %o3			! set imm
	st	%o3, [%o0]
	/* fixup or instruction */
	add	%o0, I_SIZE, %o0		! next instr
	ld	[%o0], %o3
	and	%o4, 0x3ff, %o2			! bits [9:0]
	or	%o3, %o2, %o3			! set imm
	st	%o3, [%o0]
	retl
	flush	%o0
	SET_SIZE(sfmmu_fixup_setx)

	ENTRY_NP(sfmmu_fixup_or)
	/*
	 * %o0 = start of patch area
	 * %o4 = 32 bit value to patch
	 * %o2, %o3 are used as scratch registers.
	 * Note: Assuming that all parts of the instructions which need to be
	 *	 patched correspond to RUNTIME_PATCH (aka 0)
	 */
	ld	[%o0], %o3
	and	%o4, 0x3ff, %o2			! bits [9:0]
	or	%o3, %o2, %o3			! set imm
	st	%o3, [%o0]
	retl
	flush	%o0
	SET_SIZE(sfmmu_fixup_or)

	ENTRY_NP(sfmmu_fixup_shiftx)
	/*
	 * %o0 = start of patch area
	 * %o4 = signed int immediate value to add to sllx/srlx imm field
	 * %o2, %o3 are used as scratch registers.
	 *
	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
	 * so we do a simple add.  The caller must be careful to prevent
	 * overflow, which could easily occur if the initial value is nonzero!
	 */
	ld	[%o0], %o3			! %o3 = instruction to patch
	and	%o3, 0x3f, %o2			! %o2 = existing imm value
	add	%o2, %o4, %o2			! %o2 = new imm value
	andn	%o3, 0x3f, %o3			! clear old imm value
	and	%o2, 0x3f, %o2			! truncate new imm value
	or	%o3, %o2, %o3			! set new imm value
	st	%o3, [%o0]			! store updated instruction
	retl
	flush	%o0
	SET_SIZE(sfmmu_fixup_shiftx)

	ENTRY_NP(sfmmu_fixup_mmu_asi)
	/*
	 * Patch imm_asi of all ldda instructions in the MMU
	 * trap handlers.  We search MMU_PATCH_INSTR instructions
	 * starting from the itlb miss handler (trap 0x64).
	 * %o0 = address of tt[0,1]_itlbmiss
	 * %o1 = imm_asi to setup, shifted by appropriate offset.
	 * %o3 = number of instructions to search
	 * %o4 = reserved by caller: called from leaf routine
	 */
1:	ldsw	[%o0], %o2			! load instruction to %o2
	brgez,pt %o2, 2f
	  srl	%o2, 30, %o5
	btst	1, %o5				! test bit 30; skip if not set
	bz,pt	%icc, 2f
	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
	srlx	%o5, 58, %o5			! isolate op3 part of opcode
	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
	brnz,pt	%o5, 2f				! skip if not a match
	  or	%o2, %o1, %o2			! or in imm_asi
	st	%o2, [%o0]			! write patched instruction
2:	dec	%o3
	brnz,a,pt %o3, 1b			! loop until we're done
	  add	%o0, I_SIZE, %o0
	retl
	flush	%o0
	SET_SIZE(sfmmu_fixup_mmu_asi)

	/*
	 * Patch immediate ASI used to access the TSB in the
	 * trap table.
	 * inputs: %o0 = value of ktsb_phys
	 */
	ENTRY_NP(sfmmu_patch_mmu_asi)
	mov	%o7, %o4			! save return pc in %o4
	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
	movrz	%o0, ASI_NQUAD_LD, %o3
	sll	%o3, 5, %o1			! imm_asi offset
	mov	6, %o3				! number of instructions
	sethi	%hi(dktsb), %o0			! to search
	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
	  or	%o0, %lo(dktsb), %o0
	mov	6, %o3				! number of instructions
	sethi	%hi(dktsb4m), %o0		! to search
	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
	  or	%o0, %lo(dktsb4m), %o0
	mov	6, %o3				! number of instructions
	sethi	%hi(iktsb), %o0			! to search
	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
	  or	%o0, %lo(iktsb), %o0
	mov	%o4, %o7			! retore return pc -- leaf
	retl
	nop
	SET_SIZE(sfmmu_patch_mmu_asi)

	ENTRY_NP(sfmmu_patch_ktsb)
	/*
	 * We need to fix iktsb, dktsb, et. al.
	 */
	save	%sp, -SA(MINFRAME), %sp
	set	ktsb_phys, %o1
	ld	[%o1], %o4
	set	ktsb_base, %o5
	set	ktsb4m_base, %l1
	brz,pt	%o4, 1f
	  nop
	set	ktsb_pbase, %o5
	set	ktsb4m_pbase, %l1
1:
	sethi	%hi(ktsb_szcode), %o1
	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */

	sethi	%hi(iktsb), %o0
	call	sfmmu_fix_ktlb_traptable
	  or	%o0, %lo(iktsb), %o0

	sethi	%hi(dktsb), %o0
	call	sfmmu_fix_ktlb_traptable
	  or	%o0, %lo(dktsb), %o0

	sethi	%hi(ktsb4m_szcode), %o1
	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */

	sethi	%hi(dktsb4m), %o0
	call	sfmmu_fix_ktlb_traptable
	  or	%o0, %lo(dktsb4m), %o0

#ifndef sun4v
	mov	ASI_N, %o2
	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
	sethi	%hi(tsb_kernel_patch_asi), %o0
	call	sfmmu_fixup_or
	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
#endif

	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)

	sethi	%hi(dktsbbase), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb base addr
	  or	%o0, %lo(dktsbbase), %o0

	sethi	%hi(iktsbbase), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb base addr
	  or	%o0, %lo(iktsbbase), %o0

	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb base addr
	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0

#ifdef sun4v
	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb base addr
	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
#endif /* sun4v */

	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)

	sethi	%hi(dktsb4mbase), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
	  or	%o0, %lo(dktsb4mbase), %o0

	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0

#ifdef sun4v
	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
#endif /* sun4v */

	set	ktsb_szcode, %o4
	ld	[%o4], %o4
	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
	call	sfmmu_fixup_or		! patch value of ktsb_szcode
	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0

#ifdef sun4v
	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
	call	sfmmu_fixup_or		! patch value of ktsb_szcode
	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
#endif /* sun4v */

	set	ktsb4m_szcode, %o4
	ld	[%o4], %o4
	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0

#ifdef sun4v
	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
#endif /* sun4v */

	ret
	restore
	SET_SIZE(sfmmu_patch_ktsb)

	ENTRY_NP(sfmmu_kpm_patch_tlbm)
	/*
	 * Fixup trap handlers in common segkpm case.  This is reserved
	 * for future use should kpm TSB be changed to be other than the
	 * kernel TSB.
	 */
	retl
	nop
	SET_SIZE(sfmmu_kpm_patch_tlbm)

	ENTRY_NP(sfmmu_kpm_patch_tsbm)
	/*
	 * nop the branch to sfmmu_kpm_dtsb_miss_small
	 * in the case where we are using large pages for
	 * seg_kpm (and hence must probe the second TSB for
	 * seg_kpm VAs)
	 */
	set	dktsb4m_kpmcheck_small, %o0
	MAKE_NOP_INSTR(%o1)
	st	%o1, [%o0]
	flush	%o0
	retl
	nop
	SET_SIZE(sfmmu_kpm_patch_tsbm)

	ENTRY_NP(sfmmu_patch_utsb)
#ifdef UTSB_PHYS
	retl
	nop
#else /* UTSB_PHYS */
	/*
	 * We need to hot patch utsb_vabase and utsb4m_vabase
	 */
	save	%sp, -SA(MINFRAME), %sp

	/* patch value of utsb_vabase */
	set	utsb_vabase, %o1
	ldx	[%o1], %o4
	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
	call	sfmmu_fixup_setx
	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
	call	sfmmu_fixup_setx
	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
	call	sfmmu_fixup_setx
	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0

	/* patch value of utsb4m_vabase */
	set	utsb4m_vabase, %o1
	ldx	[%o1], %o4
	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
	call	sfmmu_fixup_setx
	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
	call	sfmmu_fixup_setx
	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
	call	sfmmu_fixup_setx
	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0

	/*
	 * Patch TSB base register masks and shifts if needed.
	 * By default the TSB base register contents are set up for 4M slab.
	 * If we're using a smaller slab size and reserved VA range we need
	 * to patch up those values here.
	 */
	set	tsb_slab_shift, %o1
	set	MMU_PAGESHIFT4M, %o4
	ldsw	[%o1], %o3
	subcc	%o4, %o3, %o4
	bz,pt	%icc, 1f
	  /* delay slot safe */

	/* patch reserved VA range size if needed. */
	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
	call	sfmmu_fixup_shiftx
	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
	call	sfmmu_fixup_shiftx
	  add	%o0, I_SIZE, %o0
	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
	call	sfmmu_fixup_shiftx
	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
	call	sfmmu_fixup_shiftx
	  add	%o0, I_SIZE, %o0
1:
	/* patch TSBREG_VAMASK used to set up TSB base register */
	set	tsb_slab_mask, %o1
	lduw	[%o1], %o4
	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
	call	sfmmu_fixup_or
	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
	call	sfmmu_fixup_or
	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0

	ret
	restore
#endif /* UTSB_PHYS */
	SET_SIZE(sfmmu_patch_utsb)


	/*
	 * Routine that loads an entry into a tsb using virtual addresses.
	 * Locking is required since all cpus can use the same TSB.
	 * Note that it is no longer required to have a valid context
	 * when calling this function.
	 */
	ENTRY_NP(sfmmu_load_tsbe)
	/*
	 * %o0 = pointer to tsbe to load
	 * %o1 = tsb tag
	 * %o2 = virtual pointer to TTE
	 * %o3 = 1 if physical address in %o0 else 0
	 */
	rdpr	%pstate, %o5
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
#endif /* DEBUG */

	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */

	SETUP_TSB_ASI(%o3, %g3)
	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)

	wrpr	%g0, %o5, %pstate		/* enable interrupts */

	retl
	membar	#StoreStore|#StoreLoad
	SET_SIZE(sfmmu_load_tsbe)

	/*
	 * Flush TSB of a given entry if the tag matches.
	 */
	ENTRY(sfmmu_unload_tsbe)
	/*
	 * %o0 = pointer to tsbe to be flushed
	 * %o1 = tag to match
	 * %o2 = 1 if physical address in %o0 else 0
	 */
	SETUP_TSB_ASI(%o2, %g1)
	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
	retl
	membar	#StoreStore|#StoreLoad
	SET_SIZE(sfmmu_unload_tsbe)

	/*
	 * Routine that loads a TTE into the kpm TSB from C code.
	 * Locking is required since kpm TSB is shared among all CPUs.
	 */
	ENTRY_NP(sfmmu_kpm_load_tsb)
	/*
	 * %o0 = vaddr
	 * %o1 = ttep
	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
	 */
	rdpr	%pstate, %o5			! %o5 = saved pstate
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
#endif /* DEBUG */
	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts

#ifndef sun4v
	sethi	%hi(ktsb_phys), %o4
	mov	ASI_N, %o3
	ld	[%o4 + %lo(ktsb_phys)], %o4
	movrnz	%o4, ASI_MEM, %o3
	mov	%o3, %asi
#endif
	mov	%o0, %g1			! %g1 = vaddr

	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
	/* %g2 = tsbep, %g1 clobbered */

	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)

	wrpr	%g0, %o5, %pstate		! enable interrupts
	retl
	  membar #StoreStore|#StoreLoad
	SET_SIZE(sfmmu_kpm_load_tsb)

	/*
	 * Routine that shoots down a TTE in the kpm TSB or in the
	 * kernel TSB depending on virtpg. Locking is required since
	 * kpm/kernel TSB is shared among all CPUs.
	 */
	ENTRY_NP(sfmmu_kpm_unload_tsb)
	/*
	 * %o0 = vaddr
	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
	 */
#ifndef sun4v
	sethi	%hi(ktsb_phys), %o4
	mov	ASI_N, %o3
	ld	[%o4 + %lo(ktsb_phys)], %o4
	movrnz	%o4, ASI_MEM, %o3
	mov	%o3, %asi
#endif
	mov	%o0, %g1			! %g1 = vaddr

	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
	/* %g2 = tsbep, %g1 clobbered */

	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)

	retl
	  membar	#StoreStore|#StoreLoad
	SET_SIZE(sfmmu_kpm_unload_tsb)

#endif /* lint */


#if defined (lint)

/*ARGSUSED*/
pfn_t
sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
{ return(0); }

#else /* lint */

	ENTRY_NP(sfmmu_ttetopfn)
	ldx	[%o0], %g1			/* read tte */
	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
	/*
	 * g1 = pfn
	 */
	retl
	mov	%g1, %o0
	SET_SIZE(sfmmu_ttetopfn)

#endif /* !lint */


#if defined (lint)
/*
 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
 * the hash list.
 */
/* ARGSUSED */
void
sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
	uint64_t hblkpa)
{
}

/*
 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
 * hash list.
 */
/* ARGSUSED */
void
sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
	uint64_t hblkpa, struct hme_blk *prev_hblkp)
{
}
#else /* lint */

/*
 * Functions to grab/release hme bucket list lock.  I only use a byte
 * instead of the whole int because eventually we might want to
 * put some counters on the other bytes (of course, these routines would
 * have to change).  The code that grab this lock should execute
 * with interrupts disabled and hold the lock for the least amount of time
 * possible.
 */

/*
 * Even though hmeh_listlock is updated using pa there's no need to flush
 * dcache since hmeh_listlock will be restored to the original value (0)
 * before interrupts are reenabled.
 */

/*
 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
 * routines still use virtual addresses to update the bucket fields. But they
 * must not cause a TLB miss after grabbing the low level bucket lock. To
 * achieve this we must make sure the bucket structure is completely within an
 * 8K page.
 */

#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
#error - the size of hmehash_bucket structure is not power of 2
#endif

#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
	mov     0xff, tmp2                                      ;\
	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
label1:                                                         ;\
	casa    [tmp1]asi, %g0, tmp2                            ;\
	brnz,pn tmp2, label1                                    ;\
	mov     0xff, tmp2                                      ;\
	membar  #LoadLoad

#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
	membar  #LoadStore|#StoreStore                          ;\
	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
	sta     %g0, [tmp1]asi

	.seg	".data"
hblk_add_panic1:
	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
	.byte	0
hblk_add_panic2:
	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
	.byte	0
	.align	4
	.seg	".text"

	ENTRY_NP(sfmmu_hblk_hash_add)
	/*
	 * %o0 = hmebp
	 * %o1 = hmeblkp
	 * %o2 = hblkpa
	 */
	rdpr	%pstate, %o5
#ifdef DEBUG
	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
	bnz,pt %icc, 3f				/* disabled, panic	 */
	  nop
	save	%sp, -SA(MINFRAME), %sp
	sethi	%hi(hblk_add_panic1), %o0
	call	panic
	 or	%o0, %lo(hblk_add_panic1), %o0
	ret
	restore

3:
#endif /* DEBUG */
	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
	mov	%o2, %g1

	/*
	 * g1 = hblkpa
	 */
	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
#ifdef	DEBUG
	cmp	%o4, %g0
	bne,pt %xcc, 1f
	 nop
	brz,pt %g2, 1f
	 nop
	wrpr	%g0, %o5, %pstate		/* enable interrupts */
	save	%sp, -SA(MINFRAME), %sp
	sethi	%hi(hblk_add_panic2), %o0
	call	panic
	  or	%o0, %lo(hblk_add_panic2), %o0
	ret
	restore
1:
#endif /* DEBUG */
	/*
	 * We update hmeblks entries before grabbing lock because the stores
	 * could take a tlb miss and require the hash lock.  The buckets
	 * are part of the nucleus so we are cool with those stores.
	 *
	 * if buckets are not part of the nucleus our game is to
	 * not touch any other page via va until we drop the lock.
	 * This guarantees we won't get a tlb miss before the lock release
	 * since interrupts are disabled.
	 */
	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
	HMELOCK_EXIT(%o0, %g2, ASI_N)
	retl
	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
	SET_SIZE(sfmmu_hblk_hash_add)

	ENTRY_NP(sfmmu_hblk_hash_rm)
	/*
	 * This function removes an hmeblk from the hash chain.
	 * It is written to guarantee we don't take a tlb miss
	 * by using physical addresses to update the list.
	 *
	 * %o0 = hmebp
	 * %o1 = hmeblkp
	 * %o2 = hmeblkp previous pa
	 * %o3 = hmeblkp previous
	 */

	mov	%o3, %o4			/* o4 = hmeblkp previous */

	rdpr	%pstate, %o5
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1)
#endif /* DEBUG */
	/*
	 * disable interrupts, clear Address Mask to access 64 bit physaddr
	 */
	andn    %o5, PSTATE_IE, %g1
	wrpr    %g1, 0, %pstate

#ifndef sun4v
	sethi   %hi(dcache_line_mask), %g4
	ld      [%g4 + %lo(dcache_line_mask)], %g4
#endif /* sun4v */

	/*
	 * if buckets are not part of the nucleus our game is to
	 * not touch any other page via va until we drop the lock.
	 * This guarantees we won't get a tlb miss before the lock release
	 * since interrupts are disabled.
	 */
	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
	cmp	%g2, %o1
	bne,pt	%ncc,1f
	 mov	ASI_MEM, %asi
	/*
	 * hmeblk is first on list
	 */
	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
	ba,pt	%xcc, 2f
	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
1:
	/* hmeblk is not first on list */

	mov	%o2, %g3
#ifndef sun4v
	GET_CPU_IMPL(%g2)
	cmp 	%g2, CHEETAH_IMPL
	bge,a,pt %icc, hblk_hash_rm_1
	  and	%o4, %g4, %g2
	cmp	%g2, SPITFIRE_IMPL
	blt	%icc, hblk_hash_rm_2		/* no flushing needed for OPL */
	  and	%o4, %g4, %g2
	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
	add	%o4, HMEBLK_NEXT, %o4
	and	%o4, %g4, %g2
	ba	hblk_hash_rm_2
	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
hblk_hash_rm_1:

	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
	membar	#Sync
	add     %g3, HMEBLK_NEXT, %g2
	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
hblk_hash_rm_2:
	membar	#Sync
#endif /* sun4v */
	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
2:
	HMELOCK_EXIT(%o0, %g2, ASI_N)
	retl
	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
	SET_SIZE(sfmmu_hblk_hash_rm)

#endif /* lint */

/*
 * These macros are used to update global sfmmu hme hash statistics
 * in perf critical paths. It is only enabled in debug kernels or
 * if SFMMU_STAT_GATHER is defined
 */
#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
	cmp	tmp1, hatid						;\
	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
	set	sfmmu_global_stat, tmp1					;\
	add	tmp1, tmp2, tmp1					;\
	ld	[tmp1], tmp2						;\
	inc	tmp2							;\
	st	tmp2, [tmp1]

#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
	mov	HATSTAT_KHASH_LINKS, tmp2				;\
	cmp	tmp1, hatid						;\
	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
	set	sfmmu_global_stat, tmp1					;\
	add	tmp1, tmp2, tmp1					;\
	ld	[tmp1], tmp2						;\
	inc	tmp2							;\
	st	tmp2, [tmp1]


#else /* DEBUG || SFMMU_STAT_GATHER */

#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)

#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)

#endif  /* DEBUG || SFMMU_STAT_GATHER */

/*
 * This macro is used to update global sfmmu kstas in non
 * perf critical areas so they are enabled all the time
 */
#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
	sethi	%hi(sfmmu_global_stat), tmp1				;\
	add	tmp1, statname, tmp1					;\
	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
	inc	tmp2							;\
	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]

/*
 * These macros are used to update per cpu stats in non perf
 * critical areas so they are enabled all the time
 */
#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
	ld	[tsbarea + stat], tmp1					;\
	inc	tmp1							;\
	st	tmp1, [tsbarea + stat]

/*
 * These macros are used to update per cpu stats in non perf
 * critical areas so they are enabled all the time
 */
#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
	lduh	[tsbarea + stat], tmp1					;\
	inc	tmp1							;\
	stuh	tmp1, [tsbarea + stat]

#if defined(KPM_TLBMISS_STATS_GATHER)
	/*
	 * Count kpm dtlb misses separately to allow a different
	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
	 */
#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
	brgez	tagacc, label	/* KPM VA? */				;\
	nop								;\
	CPU_INDEX(tmp1, tsbma)						;\
	sethi	%hi(kpmtsbm_area), tsbma				;\
	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
	/* VA range check */						;\
	ldx	[tsbma + KPMTSBM_VBASE], val				;\
	cmp	tagacc, val						;\
	blu,pn	%xcc, label						;\
	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
	cmp	tagacc, tmp1						;\
	bgeu,pn	%xcc, label						;\
	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
	inc	val							;\
	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
label:
#else
#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
#endif	/* KPM_TLBMISS_STATS_GATHER */

#if defined (lint)
/*
 * The following routines are jumped to from the mmu trap handlers to do
 * the setting up to call systrap.  They are separate routines instead of
 * being part of the handlers because the handlers would exceed 32
 * instructions and since this is part of the slow path the jump
 * cost is irrelevant.
 */
void
sfmmu_pagefault(void)
{
}

void
sfmmu_mmu_trap(void)
{
}

void
sfmmu_window_trap(void)
{
}

void
sfmmu_kpm_exception(void)
{
}

#else /* lint */

#ifdef	PTL1_PANIC_DEBUG
	.seg	".data"
	.global	test_ptl1_panic
test_ptl1_panic:
	.word	0
	.align	8

	.seg	".text"
	.align	4
#endif	/* PTL1_PANIC_DEBUG */


	ENTRY_NP(sfmmu_pagefault)
	SET_GL_REG(1)
	USE_ALTERNATE_GLOBALS(%g5)
	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
	rdpr	%tt, %g6
	cmp	%g6, FAST_IMMU_MISS_TT
	be,a,pn	%icc, 1f
	  mov	T_INSTR_MMU_MISS, %g3
	cmp	%g6, T_INSTR_MMU_MISS
	be,a,pn	%icc, 1f
	  mov	T_INSTR_MMU_MISS, %g3
	mov	%g5, %g2
	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
	cmp	%g6, FAST_DMMU_MISS_TT
	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
	cmp	%g6, T_DATA_MMU_MISS
	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */

#ifdef  PTL1_PANIC_DEBUG
	/* check if we want to test the tl1 panic */
	sethi	%hi(test_ptl1_panic), %g4
	ld	[%g4 + %lo(test_ptl1_panic)], %g1
	st	%g0, [%g4 + %lo(test_ptl1_panic)]
	cmp	%g1, %g0
	bne,a,pn %icc, ptl1_panic
	  or	%g0, PTL1_BAD_DEBUG, %g1
#endif	/* PTL1_PANIC_DEBUG */
1:
	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
	/*
	 * g2 = tag access reg
	 * g3.l = type
	 * g3.h = 0
	 */
	sethi	%hi(trap), %g1
	or	%g1, %lo(trap), %g1
2:
	ba,pt	%xcc, sys_trap
	  mov	-1, %g4
	SET_SIZE(sfmmu_pagefault)

	ENTRY_NP(sfmmu_mmu_trap)
	SET_GL_REG(1)
	USE_ALTERNATE_GLOBALS(%g5)
	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
	rdpr	%tt, %g6
	cmp	%g6, FAST_IMMU_MISS_TT
	be,a,pn	%icc, 1f
	  mov	T_INSTR_MMU_MISS, %g3
	cmp	%g6, T_INSTR_MMU_MISS
	be,a,pn	%icc, 1f
	  mov	T_INSTR_MMU_MISS, %g3
	mov	%g5, %g2
	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
	cmp	%g6, FAST_DMMU_MISS_TT
	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
	cmp	%g6, T_DATA_MMU_MISS
	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1:
	/*
	 * g2 = tag access reg
	 * g3 = type
	 */
	sethi	%hi(sfmmu_tsbmiss_exception), %g1
	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
	ba,pt	%xcc, sys_trap
	  mov	-1, %g4
	/*NOTREACHED*/
	SET_SIZE(sfmmu_mmu_trap)

	ENTRY_NP(sfmmu_suspend_tl)
	SET_GL_REG(1)
	USE_ALTERNATE_GLOBALS(%g5)
	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
	rdpr	%tt, %g6
	cmp	%g6, FAST_IMMU_MISS_TT
	be,a,pn	%icc, 1f
	  mov	T_INSTR_MMU_MISS, %g3
	mov	%g5, %g2
	cmp	%g6, FAST_DMMU_MISS_TT
	move	%icc, T_DATA_MMU_MISS, %g3
	movne	%icc, T_DATA_PROT, %g3
1:
	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
	ba,pt	%xcc, sys_trap
	  mov	PIL_15, %g4
	/*NOTREACHED*/
	SET_SIZE(sfmmu_suspend_tl)

	/*
	 * No %g registers in use at this point.
	 */
	ENTRY_NP(sfmmu_window_trap)
	rdpr	%tpc, %g1
#ifdef sun4v
#ifdef DEBUG
	/* We assume previous %gl was 1 */
	rdpr	%tstate, %g4
	srlx	%g4, TSTATE_GL_SHIFT, %g4
	and	%g4, TSTATE_GL_MASK, %g4
	cmp	%g4, 1
	bne,a,pn %icc, ptl1_panic
	  mov	PTL1_BAD_WTRAP, %g1
#endif /* DEBUG */
	/* user miss at tl>1. better be the window handler or user_rtt */
	/* in user_rtt? */
	set	rtt_fill_start, %g4
	cmp	%g1, %g4
	blu,pn %xcc, 6f
	 .empty
	set	rtt_fill_end, %g4
	cmp	%g1, %g4
	bgeu,pn %xcc, 6f
	 nop
	set	fault_rtt_fn1, %g1
	wrpr	%g0, %g1, %tnpc
	ba,a	7f
6:
	! must save this trap level before descending trap stack
	! no need to save %tnpc, either overwritten or discarded
	! already got it: rdpr	%tpc, %g1
	rdpr	%tstate, %g6
	rdpr	%tt, %g7
	! trap level saved, go get underlying trap type
	rdpr	%tl, %g5
	sub	%g5, 1, %g3
	wrpr	%g3, %tl
	rdpr	%tt, %g2
	wrpr	%g5, %tl
	! restore saved trap level
	wrpr	%g1, %tpc
	wrpr	%g6, %tstate
	wrpr	%g7, %tt
#else /* sun4v */
	/* user miss at tl>1. better be the window handler */
	rdpr	%tl, %g5
	sub	%g5, 1, %g3
	wrpr	%g3, %tl
	rdpr	%tt, %g2
	wrpr	%g5, %tl
#endif /* sun4v */
	and	%g2, WTRAP_TTMASK, %g4
	cmp	%g4, WTRAP_TYPE
	bne,pn	%xcc, 1f
	 nop
	/* tpc should be in the trap table */
	set	trap_table, %g4
	cmp	%g1, %g4
	blt,pn %xcc, 1f
	 .empty
	set	etrap_table, %g4
	cmp	%g1, %g4
	bge,pn %xcc, 1f
	 .empty
	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
	add	%g1, WTRAP_FAULTOFF, %g1
	wrpr	%g0, %g1, %tnpc
7:
	/*
	 * some wbuf handlers will call systrap to resolve the fault
	 * we pass the trap type so they figure out the correct parameters.
	 * g5 = trap type, g6 = tag access reg
	 */

	/*
	 * only use g5, g6, g7 registers after we have switched to alternate
	 * globals.
	 */
	SET_GL_REG(1)
	USE_ALTERNATE_GLOBALS(%g5)
	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
	rdpr	%tt, %g7
	cmp	%g7, FAST_IMMU_MISS_TT
	be,a,pn	%icc, ptl1_panic
	  mov	PTL1_BAD_WTRAP, %g1
	cmp	%g7, T_INSTR_MMU_MISS
	be,a,pn	%icc, ptl1_panic
	  mov	PTL1_BAD_WTRAP, %g1
	mov	T_DATA_PROT, %g5
	cmp	%g7, FAST_DMMU_MISS_TT
	move	%icc, T_DATA_MMU_MISS, %g5
	cmp	%g7, T_DATA_MMU_MISS
	move	%icc, T_DATA_MMU_MISS, %g5
	! XXXQ AGS re-check out this one
	done
1:
	CPU_ADDR(%g1, %g4)
	ld	[%g1 + CPU_TL1_HDLR], %g4
	brnz,a,pt %g4, sfmmu_mmu_trap
	  st	%g0, [%g1 + CPU_TL1_HDLR]
	ba,pt	%icc, ptl1_panic
	  mov	PTL1_BAD_TRAP, %g1
	SET_SIZE(sfmmu_window_trap)

	ENTRY_NP(sfmmu_kpm_exception)
	/*
	 * We have accessed an unmapped segkpm address or a legal segkpm
	 * address which is involved in a VAC alias conflict prevention.
	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
	 * set. If it is, we will instead note that a fault has occurred
	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
	 * a "retry"). This will step over the faulting instruction.
	 * Note that this means that a legal segkpm address involved in
	 * a VAC alias conflict prevention (a rare case to begin with)
	 * cannot be used in DTrace.
	 */
	CPU_INDEX(%g1, %g2)
	set	cpu_core, %g2
	sllx	%g1, CPU_CORE_SHIFT, %g1
	add	%g1, %g2, %g1
	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
	bz	0f
	or	%g2, CPU_DTRACE_BADADDR, %g2
	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
	done
0:
	TSTAT_CHECK_TL1(1f, %g1, %g2)
1:
	SET_GL_REG(1)
	USE_ALTERNATE_GLOBALS(%g5)
	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
	/*
	 * g2=tagacc g3.l=type g3.h=0
	 */
	sethi	%hi(trap), %g1
	or	%g1, %lo(trap), %g1
	ba,pt	%xcc, sys_trap
	mov	-1, %g4
	SET_SIZE(sfmmu_kpm_exception)

#endif /* lint */

#if defined (lint)

void
sfmmu_tsb_miss(void)
{
}

void
sfmmu_kpm_dtsb_miss(void)
{
}

void
sfmmu_kpm_dtsb_miss_small(void)
{
}

#else /* lint */

#if (IMAP_SEG != 0)
#error - ism_map->ism_seg offset is not zero
#endif

/*
 * Copies ism mapping for this ctx in param "ism" if this is a ISM
 * tlb miss and branches to label "ismhit". If this is not an ISM
 * process or an ISM tlb miss it falls thru.
 *
 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
 * this process.
 * If so, it will branch to label "ismhit".  If not, it will fall through.
 *
 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
 * so that any other threads of this process will not try and walk the ism
 * maps while they are being changed.
 *
 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
 *       will make sure of that. This means we can terminate our search on
 *       the first zero mapping we find.
 *
 * Parameters:
 * tagacc	= tag access register (vaddr + ctx) (in)
 * tsbmiss	= address of tsb miss area (in)
 * ismseg	= contents of ism_seg for this ism map (out)
 * ismhat	= physical address of imap_ismhat for this ism map (out)
 * tmp1		= scratch reg (CLOBBERED)
 * tmp2		= scratch reg (CLOBBERED)
 * tmp3		= scratch reg (CLOBBERED)
 * label:    temporary labels
 * ismhit:   label where to jump to if an ism dtlb miss
 * exitlabel:label where to jump if hat is busy due to hat_unshare.
 */
#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
	label, ismhit)							\
	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
label/**/1:								;\
	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
label/**/2:								;\
	brz,pt  ismseg, label/**/3		/* no mapping */	;\
	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
	lduha	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
									;\
	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
	cmp	ismhat, tmp1						;\
	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
									;\
	add	tmp3, IBLK_NEXTPA, tmp1					;\
	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
label/**/3:

/*
 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
 * Parameters:
 * vaddr = reg containing virtual address
 * hatid = reg containing sfmmu pointer
 * hmeshift = constant/register to shift vaddr to obtain vapg
 * hmebp = register where bucket pointer will be stored
 * vapg = register where virtual page will be stored
 * tmp1, tmp2 = tmp registers
 */


#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
	vapg, label, tmp1, tmp2)					\
	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
	brnz,a,pt tmp1, label/**/1					;\
	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
	ba,pt	%xcc, label/**/2					;\
	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
label/**/1:								;\
	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
label/**/2:								;\
	srlx	tagacc, hmeshift, vapg					;\
	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
	add	hmebp, tmp1, hmebp

/*
 * hashtag includes bspage + hashno (64 bits).
 */

#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
	sllx	vapg, hmeshift, vapg					;\
	or	vapg, hashno, hblktag

/*
 * Function to traverse hmeblk hash link list and find corresponding match.
 * The search is done using physical pointers. It returns the physical address
 * and virtual address pointers to the hmeblk that matches with the tag
 * provided.
 * Parameters:
 * hmebp	= register that points to hme hash bucket, also used as
 *		  tmp reg (clobbered)
 * hmeblktag	= register with hmeblk tag match
 * hatid	= register with hatid
 * hmeblkpa	= register where physical ptr will be stored
 * hmeblkva	= register where virtual ptr will be stored
 * tmp1		= tmp reg
 * label: temporary label
 */

#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
	tsbarea, tmp1, label)					 	\
	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
label/**/1:								;\
	brz,pn	hmeblkva, label/**/2					;\
	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
	add	hmebp, CLONGSIZE, hmebp					;\
	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
	xor	tmp1, hmeblktag, tmp1					;\
	xor	hmebp, hatid, hmebp					;\
	or	hmebp, tmp1, hmebp					;\
	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
	ba,pt	%xcc, label/**/1					;\
	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
label/**/2:


#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
#endif

/*
 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
 * he offset for the corresponding hment.
 * Parameters:
 * vaddr = register with virtual address
 * hmeblkpa = physical pointer to hme_blk
 * hment = register where address of hment will be stored
 * hmentoff = register where hment offset will be stored
 * label1 = temporary label
 */
#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1)	\
	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
	lda	[hmentoff]ASI_MEM, tmp1 				;\
	andcc	tmp1, HBLK_SZMASK, %g0	 /* tmp1 = get_hblk_sz(%g5) */	;\
	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
	  or	%g0, HMEBLK_HME1, hmentoff				;\
	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
	sllx	tmp1, SFHME_SHIFT, tmp1					;\
	add	tmp1, HMEBLK_HME1, hmentoff				;\
label1:

/*
 * GET_TTE is a macro that returns a TTE given a tag and hatid.
 *
 * tagacc	= tag access register (vaddr + ctx) (in)
 * hatid	= sfmmu pointer for TSB miss (in)
 * tte		= tte for TLB miss if found, otherwise clobbered (out)
 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
 * hmentoff	= temporarily stores hment offset (clobbered)
 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
 *		  for this page size.
 * hashno	= constant/register hash number
 * label	= temporary label for branching within macro.
 * foundlabel	= label to jump to when tte is found.
 * suspendlabel= label to jump to when tte is suspended.
 * exitlabel	= label to jump to when tte is not found.  The hmebp lock
 *		  is still held at this time.
 *
 * The caller should set up the tsbmiss->scratch[2] field correctly before
 * calling this funciton  (aka TSBMISS_SCRATCH + TSBMISS_HATID)
 */
#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \
		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
									;\
	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
		hmeblkpa, label/**/5, hmentoff, hmeblkva)		;\
									;\
	/*								;\
	 * tagacc = tagacc						;\
	 * hatid = hatid						;\
	 * tsbarea = tsbarea						;\
	 * tte   = hmebp (hme bucket pointer)				;\
	 * hmeblkpa  = vapg  (virtual page)				;\
	 * hmentoff, hmeblkva = scratch					;\
	 */								;\
	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff)	;\
									;\
	/*								;\
	 * tagacc = tagacc						;\
	 * hatid = hatid						;\
	 * tte   = hmebp						;\
	 * hmeblkpa  = CLOBBERED					;\
	 * hmentoff  = htag_bspage & hashno				;\
	 * hmeblkva  = scratch						;\
	 */								;\
	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
	HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, 	\
		tsbarea, tagacc, label/**/1)				;\
	/*								;\
	 * tagacc = CLOBBERED						;\
	 * tte = CLOBBERED						;\
	 * hmeblkpa = hmeblkpa						;\
	 * hmeblkva = hmeblkva						;\
	 */								;\
	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
	  nop								;\
label/**/4:								;\
	/*								;\
	 * We have found the hmeblk containing the hment.		;\
	 * Now we calculate the corresponding tte.			;\
	 *								;\
	 * tagacc = tagacc						;\
	 * hatid = clobbered						;\
	 * tte   = hmebp						;\
	 * hmeblkpa  = hmeblkpa						;\
	 * hmentoff  = hblktag						;\
	 * hmeblkva  = hmeblkva 					;\
	 */								;\
	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2)	;\
									;\
	add	hmentoff, SFHME_TTE, hmentoff				;\
	add	hmeblkpa, hmentoff, hmeblkpa				;\
	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
	add	hmeblkva, hmentoff, hmeblkva				;\
	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\
	HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM)	/* drop lock */	;\
	set	TTE_SUSPEND, hmentoff					;\
	TTE_SUSPEND_INT_SHIFT(hmentoff)					;\
	btst	tte, hmentoff						;\
	bz,pt	%xcc, foundlabel					;\
	  nop								;\
									;\
	/*								;\
	 * Mapping is suspended, so goto suspend label.			;\
	 */								;\
	ba,pt	%xcc, suspendlabel					;\
	  nop

	/*
	 * KERNEL PROTECTION HANDLER
	 *
	 * g1 = tsb8k pointer register (clobbered)
	 * g2 = tag access register (ro)
	 * g3 - g7 = scratch registers
	 *
	 * Note: This function is patched at runtime for performance reasons.
	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
	 */
	ENTRY_NP(sfmmu_kprot_trap)
	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
sfmmu_kprot_patch_ktsb_base:
	RUNTIME_PATCH_SETX(%g1, %g6)
	/* %g1 = contents of ktsb_base or ktsb_pbase */
sfmmu_kprot_patch_ktsb_szcode:
	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)

	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
	! %g1 = First TSB entry pointer, as TSB miss handler expects

	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
sfmmu_kprot_patch_ktsb4m_base:
	RUNTIME_PATCH_SETX(%g3, %g6)
	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
sfmmu_kprot_patch_ktsb4m_szcode:
	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)

	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
	! %g3 = 4M tsb entry pointer, as TSB miss handler expects

	CPU_TSBMISS_AREA(%g6, %g7)
	HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
	ba,pt	%xcc, sfmmu_tsb_miss_tt
	  nop

	/*
	 * USER PROTECTION HANDLER
	 *
	 * g1 = tsb8k pointer register (ro)
	 * g2 = tag access register (ro)
	 * g3 = faulting context (clobbered, currently not used)
	 * g4 - g7 = scratch registers
	 */
	ALTENTRY(sfmmu_uprot_trap)
#ifdef sun4v
	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
	/* %g1 = first TSB entry ptr now, %g2 preserved */

	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
	brlz,pt %g3, 9f			/* check for 2nd TSB */
	  mov	%g0, %g3		/* clear second tsbe ptr */

	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
	/* %g3 = second TSB entry ptr now, %g2 preserved */

#else /* sun4v */
#ifdef UTSB_PHYS
	/* g1 = first TSB entry ptr */
	GET_2ND_TSBREG(%g3)
	brlz,a,pt %g3, 9f		/* check for 2nd TSB */
	  mov	%g0, %g3		/* clear second tsbe ptr */

	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
	/* %g3 = second TSB entry ptr now, %g2 preserved */
#else /* UTSB_PHYS */
	brgez,pt %g1, 9f		/* check for 2nd TSB */
	  mov	%g0, %g3		/* clear second tsbe ptr */

	mov	%g2, %g7
	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
	/* %g3 = second TSB entry ptr now, %g7 clobbered */
	mov	%g1, %g7
	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
#endif /* UTSB_PHYS */
#endif /* sun4v */
9:
	CPU_TSBMISS_AREA(%g6, %g7)
	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
	  nop

	/*
	 * Kernel 8K page iTLB miss.  We also get here if we took a
	 * fast instruction access mmu miss trap while running in
	 * invalid context.
	 *
	 * %g1 = 8K TSB pointer register (not used, clobbered)
	 * %g2 = tag access register (used)
	 * %g3 = faulting context id (used)
	 * %g7 = 4M virtual page number for tag matching  (used)
	 */
	.align	64
	ALTENTRY(sfmmu_kitlb_miss)
	brnz,pn %g3, tsb_tl0_noctxt
	  nop

	/* kernel miss */
	/* get kernel tsb pointer */
	/* we patch the next set of instructions at run time */
	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
iktsbbase:
	RUNTIME_PATCH_SETX(%g4, %g5)
	/* %g4 = contents of ktsb_base or ktsb_pbase */

iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
	or	%g4, %g1, %g1			! form tsb ptr
	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
	cmp	%g4, %g7
	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
	  andcc %g5, TTE_EXECPRM_INT, %g0	! check exec bit
	bz,pn	%icc, exec_fault
	  nop
	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	retry

	/*
	 * Kernel dTLB miss.  We also get here if we took a fast data
	 * access mmu miss trap while running in invalid context.
	 *
	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
	 *	We select the TSB miss handler to branch to depending on
	 *	the virtual address of the access.  In the future it may
	 *	be desirable to separate kpm TTEs into their own TSB,
	 *	in which case all that needs to be done is to set
	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
	 *	early in the miss if we detect a kpm VA to a new handler.
	 *
	 * %g1 = 8K TSB pointer register (not used, clobbered)
	 * %g2 = tag access register (used)
	 * %g3 = faulting context id (used)
	 */
	.align	64
	ALTENTRY(sfmmu_kdtlb_miss)
	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
	  nop

	/* Gather some stats for kpm misses in the TLB. */
	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)

	/*
	 * Get first TSB offset and look for 8K/64K/512K mapping
	 * using the 8K virtual page as the index.
	 *
	 * We patch the next set of instructions at run time;
	 * any changes here require sfmmu_patch_ktsb changes too.
	 */
dktsbbase:
	RUNTIME_PATCH_SETX(%g7, %g6)
	/* %g7 = contents of ktsb_base or ktsb_pbase */

dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1

	/*
	 * At this point %g1 is our index into the TSB.
	 * We just masked off enough bits of the VA depending
	 * on our TSB size code.
	 */
	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
	cmp	%g6, %g4			! compare tag
	bne,pn	%xcc, dktsb4m_kpmcheck_small
	  add	%g7, %g1, %g1			/* form tsb ptr */
	TT_TRACE(trace_tsbhit)
	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	/* trapstat expects tte in %g5 */
	retry

	/*
	 * If kpm is using large pages, the following instruction needs
	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
	 * so that we will probe the 4M TSB regardless of the VA.  In
	 * the case kpm is using small pages, we know no large kernel
	 * mappings are located above 0x80000000.00000000 so we skip the
	 * probe as an optimization.
	 */
dktsb4m_kpmcheck_small:
	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
	  /* delay slot safe, below */

	/*
	 * Get second TSB offset and look for 4M mapping
	 * using 4M virtual page as the TSB index.
	 *
	 * Here:
	 * %g1 = 8K TSB pointer.  Don't squash it.
	 * %g2 = tag access register (we still need it)
	 */
	srlx	%g2, MMU_PAGESHIFT4M, %g3

	/*
	 * We patch the next set of instructions at run time;
	 * any changes here require sfmmu_patch_ktsb changes too.
	 */
dktsb4mbase:
	RUNTIME_PATCH_SETX(%g7, %g6)
	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
dktsb4m:
	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3

	/*
	 * At this point %g3 is our index into the TSB.
	 * We just masked off enough bits of the VA depending
	 * on our TSB size code.
	 */
	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
	cmp	%g6, %g4			! compare tag

dktsb4m_tsbmiss:
	bne,pn	%xcc, dktsb4m_kpmcheck
	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
	TT_TRACE(trace_tsbhit)
	/* we don't check TTE size here since we assume 4M TSB is separate */
	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	/* trapstat expects tte in %g5 */
	retry

	/*
	 * So, we failed to find a valid TTE to match the faulting
	 * address in either TSB.  There are a few cases that could land
	 * us here:
	 *
	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
	 *    to sfmmu_tsb_miss_tt to handle the miss.
	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
	 *    4M TSB.  Let segkpm handle it.
	 *
	 * Note that we shouldn't land here in the case of a kpm VA when
	 * kpm_smallpages is active -- we handled that case earlier at
	 * dktsb4m_kpmcheck_small.
	 *
	 * At this point:
	 *  g1 = 8K-indexed primary TSB pointer
	 *  g2 = tag access register
	 *  g3 = 4M-indexed secondary TSB pointer
	 */
dktsb4m_kpmcheck:
	cmp	%g2, %g0
	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
	  nop
	ba,a,pt	%icc, sfmmu_tsb_miss_tt
	  nop

#ifdef sun4v
	/*
	 * User instruction miss w/ single TSB.
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.
	 *
	 * g1 = tsb8k pointer register
	 * g2 = tag access register
	 * g3 - g6 = scratch registers
	 * g7 = TSB tag to match
	 */
	.align	64
	ALTENTRY(sfmmu_uitlb_fastpath)

	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
	ba,pn	%xcc, sfmmu_tsb_miss_tt
	  mov	%g0, %g3

	/*
	 * User data miss w/ single TSB.
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.
	 *
	 * g1 = tsb8k pointer register
	 * g2 = tag access register
	 * g3 - g6 = scratch registers
	 * g7 = TSB tag to match
	 */
	.align 64
	ALTENTRY(sfmmu_udtlb_fastpath)

	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
	ba,pn	%xcc, sfmmu_tsb_miss_tt
	  mov	%g0, %g3

	/*
	 * User instruction miss w/ multiple TSBs (sun4v).
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.  Second probe covers 4M page size only.
	 *
	 * Just like sfmmu_udtlb_slowpath, except:
	 *   o Uses ASI_ITLB_IN
	 *   o checks for execute permission
	 *   o No ISM prediction.
	 *
	 * g1 = tsb8k pointer register
	 * g2 = tag access register
	 * g3 - g6 = scratch registers
	 * g7 = TSB tag to match
	 */
	.align	64
	ALTENTRY(sfmmu_uitlb_slowpath)

	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
	/* g4 - g5 = clobbered here */

	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
	/* g1 = first TSB pointer, g3 = second TSB pointer */
	srlx	%g2, TAG_VALO_SHIFT, %g7
	PROBE_2ND_ITSB(%g3, %g7)
	/* NOT REACHED */

#else /* sun4v */

	/*
	 * User instruction miss w/ multiple TSBs (sun4u).
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.  Second probe covers 4M page size only.
	 *
	 * Just like sfmmu_udtlb_slowpath, except:
	 *   o Uses ASI_ITLB_IN
	 *   o checks for execute permission
	 *   o No ISM prediction.
	 *
	 * g1 = tsb8k pointer register
	 * g2 = tag access register
	 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch
	 * g4 - g6 = scratch registers
	 * g7 = TSB tag to match
	 */
	.align	64
	ALTENTRY(sfmmu_uitlb_slowpath)

#ifdef UTSB_PHYS
	/*
	 * g1 = 1st TSB entry pointer
	 * g3 = 2nd TSB base register
	 * Need 2nd TSB entry pointer for 2nd probe.
	 */
	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)

	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
#else /* UTSB_PHYS */
	mov	%g1, %g3	/* save tsb8k reg in %g3 */
	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)

	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
#endif /* UTSB_PHYS */
	/* g1 = first TSB pointer, g3 = second TSB pointer */
	srlx	%g2, TAG_VALO_SHIFT, %g7
	PROBE_2ND_ITSB(%g3, %g7, isynth)
	/* NOT REACHED */
#endif /* sun4v */

	/*
	 * User data miss w/ multiple TSBs.
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.  Second probe covers 4M page size only.
	 *
	 * We consider probing for 4M pages first if the VA falls
	 * in a range that's likely to be ISM.
	 *
	 * g1 = tsb8k pointer register
	 * g2 = tag access register
	 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch
	 * g4 - g6 = scratch registers
	 * g7 = TSB tag to match
	 */
	.align 64
	ALTENTRY(sfmmu_udtlb_slowpath)

	/*
	 * Check for ISM.  If it exists, look for 4M mappings in the second TSB
	 * first, then probe for other mappings in the first TSB if that fails.
	 */
	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
	  mov	%g1, %g3

udtlb_miss_probefirst:
	/*
	 * g1 = 8K TSB pointer register
	 * g2 = tag access register
	 * g3 = (potentially) second TSB entry ptr
	 * g6 = ism pred.
	 * g7 = vpg_4m
	 */
#ifdef sun4v
	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)

	/*
	 * Here:
	 *   g1 = first TSB pointer
	 *   g2 = tag access reg
	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
	 */
	brgz,pn	%g6, sfmmu_tsb_miss_tt
	  nop
#else /* sun4v */
#ifndef UTSB_PHYS
	mov	%g1, %g4
	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
#endif UTSB_PHYS
	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)

	/*
	 * Here:
	 *   g1 = first TSB pointer
	 *   g2 = tag access reg
	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
	 */
	brgz,pn	%g6, sfmmu_tsb_miss_tt
	  nop
#ifndef UTSB_PHYS
	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
#endif UTSB_PHYS
	/* fall through in 8K->4M probe order */
#endif /* sun4v */

udtlb_miss_probesecond:
	/*
	 * Look in the second TSB for the TTE
	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
	 * g2 = tag access reg
	 * g3 = 8K TSB pointer register
	 * g6 = ism pred.
	 * g7 = vpg_4m
	 */
#ifdef sun4v
	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
#else /* sun4v */
#ifdef UTSB_PHYS
	GET_2ND_TSBREG(%g3)
	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
	/* tagacc (%g2) is okay, no need to reload, %g3 = second tsbe ptr */
#else /* UTSB_PHYS */
	mov	%g3, %g7
	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
	/* %g2 clobbered, %g3 =second tsbe ptr */
	mov	MMU_TAG_ACCESS, %g2
	ldxa	[%g2]ASI_DMMU, %g2
#endif /* UTSB_PHYS */
#endif /* sun4v */

	srlx	%g2, TAG_VALO_SHIFT, %g7
	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
	brgz,pn	%g6, udtlb_miss_probefirst
	  nop

	/* fall through to sfmmu_tsb_miss_tt */

	ALTENTRY(sfmmu_tsb_miss_tt)
	TT_TRACE(trace_tsbmiss)
	/*
	 * We get here if there is a TSB miss OR a write protect trap.
	 *
	 * g1 = First TSB entry pointer
	 * g2 = tag access register
	 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB
	 * g4 - g7 = scratch registers
	 */

	ALTENTRY(sfmmu_tsb_miss)

	/*
	 * If trapstat is running, we need to shift the %tpc and %tnpc to
	 * point to trapstat's TSB miss return code (note that trapstat
	 * itself will patch the correct offset to add).
	 */
	rdpr	%tl, %g7
	cmp	%g7, 1
	ble,pt	%xcc, 0f
	  sethi	%hi(KERNELBASE), %g6
	rdpr	%tpc, %g7
	or	%g6, %lo(KERNELBASE), %g6
	cmp	%g7, %g6
	bgeu,pt	%xcc, 0f
	/* delay slot safe */

	ALTENTRY(tsbmiss_trapstat_patch_point)
	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
	wrpr	%g7, %tpc
	add	%g7, 4, %g7
	wrpr	%g7, %tnpc
0:
	CPU_TSBMISS_AREA(%g6, %g7)

	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save first tsb pointer */
	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save second tsb pointer */

	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
	brz,a,pn %g3, 1f			/* skip ahead if kernel */
	  ldn	[%g6 + TSBMISS_KHATID], %g7
	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */

	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)

	cmp	%g3, INVALID_CONTEXT
	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]

	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
	/*
	 * The miss wasn't in an ISM segment.
	 *
	 * %g1 %g3, %g4, %g5, %g7 all clobbered
	 * %g2 = tag access (vaddr + ctx)
	 */

	ba,pt	%icc, 2f
	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7

1:
	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
	/*
	 * 8K and 64K hash.
	 */
2:

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
		sfmmu_suspend_tl, tsb_512K)
	/* NOT REACHED */

tsb_512K:
	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
	brz,pn	%g5, 3f
	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
	and	%g4, HAT_512K_FLAG, %g5

	/*
	 * Note that there is a small window here where we may have
	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
	 * flag yet, so we will skip searching the 512k hash list.
	 * In this case we will end up in pagefault which will find
	 * the mapping and return.  So, in this instance we will end up
	 * spending a bit more time resolving this TSB miss, but it can
	 * only happen once per process and even then, the chances of that
	 * are very small, so it's not worth the extra overhead it would
	 * take to close this window.
	 */
	brz,pn	%g5, tsb_4M
	  nop
3:
	/*
	 * 512K hash
	 */

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
		sfmmu_suspend_tl, tsb_4M)
	/* NOT REACHED */

tsb_4M:
	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
	brz,pn	%g5, 4f
	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
	and	%g4, HAT_4M_FLAG, %g5
	brz,pn	%g5, tsb_32M
	  nop
4:
	/*
	 * 4M hash
	 */

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
		sfmmu_suspend_tl, tsb_32M)
	/* NOT REACHED */

tsb_32M:
#ifndef sun4v
	GET_CPU_IMPL(%g5)
	cmp	%g5, OLYMPUS_C_IMPL
	be,pn	%xcc, 0f
	  nop
	cmp	%g5, PANTHER_IMPL
	bne,pt	%xcc, tsb_pagefault
	  nop
#endif

0:
	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
#ifdef sun4v
        brz,pn	%g5, 6f
#else
	brz,pn	%g5, tsb_pagefault
#endif
	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
	and	%g4, HAT_32M_FLAG, %g5
	brz,pn	%g5, tsb_256M
	  nop
5:
	/*
	 * 32M hash
	 */

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
		sfmmu_suspend_tl, tsb_256M)
	/* NOT REACHED */

tsb_256M:
	lduh	[%g6 + TSBMISS_HATFLAGS], %g4
	and	%g4, HAT_256M_FLAG, %g5
	brz,pn	%g5, tsb_pagefault
	  nop
6:
	/*
	 * 256M hash
	 */

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
	    sfmmu_suspend_tl, tsb_pagefault)
	/* NOT REACHED */

tsb_checktte:
	/*
	 * g3 = tte
	 * g4 = tte pa
	 * g5 = tte va
	 * g6 = tsbmiss area
	 */
	brgez,pn %g3, tsb_pagefault	/* if tte invalid branch */
	  nop

tsb_validtte:
	/*
	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
	 */
	rdpr	%tt, %g7
	cmp	%g7, FAST_PROT_TT
	bne,pt	%icc, 4f
	  nop

	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
	    tsb_protfault)

	rdpr	%tt, %g5
	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
	ba,pt	%xcc, tsb_update_tl1
	  nop

4:
	/*
	 * If ITLB miss check exec bit.
	 * If not set treat as invalid TTE.
	 */
	cmp     %g7, T_INSTR_MMU_MISS
	be,pn	%icc, 5f
	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
	cmp     %g7, FAST_IMMU_MISS_TT
	bne,pt %icc, 3f
	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
5:
	bz,pn %icc, tsb_protfault
	  nop

3:
	/*
	 * Set reference bit if not already set
	 */
	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)

	/*
	 * Now, load into TSB/TLB.  At this point:
	 * g3 = tte
	 * g4 = patte
	 * g6 = tsbmiss area
	 */
	rdpr	%tt, %g5
#ifdef sun4v
	MMU_FAULT_STATUS_AREA(%g2)
	cmp	%g5, T_INSTR_MMU_MISS
	be,a,pt	%icc, 9f
	  nop
	cmp	%g5, FAST_IMMU_MISS_TT
	be,a,pt	%icc, 9f
	  nop
	add	%g2, MMFSA_D_, %g2
9:
	ldx	[%g2 + MMFSA_CTX_], %g7
	sllx	%g7, TTARGET_CTX_SHIFT, %g7
	ldx	[%g2 + MMFSA_ADDR_], %g2
	srlx	%g2, TTARGET_VA_SHIFT, %g2
	or	%g2, %g7, %g2
#else
	cmp	%g5, FAST_IMMU_MISS_TT
	be,a,pt	%icc, tsb_update_tl1
	  ldxa	[%g0]ASI_IMMU, %g2
	ldxa	[%g0]ASI_DMMU, %g2
#endif
tsb_update_tl1:
	srlx	%g2, TTARGET_CTX_SHIFT, %g7
	brz,pn	%g7, tsb_kernel
#ifdef sun4v
	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
#else
	  srlx	%g3, TTE_SZ_SHFT, %g7
#endif

tsb_user:
#ifdef sun4v
	cmp	%g7, TTE4M
	bge,pn	%icc, tsb_user4m
	  nop
#else /* sun4v */
	cmp	%g7, TTESZ_VALID | TTE4M
	be,pn	%icc, tsb_user4m
	  srlx	%g3, TTE_SZ2_SHFT, %g7
	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
#ifdef ITLB_32M_256M_SUPPORT
	bnz,pn	%icc, tsb_user4m
	  nop
#else /* ITLB_32M_256M_SUPPORT */
	bnz,a,pn %icc, tsb_user_pn_synth
	 cmp	%g5, FAST_IMMU_MISS_TT
#endif /* ITLB_32M_256M_SUPPORT */
#endif /* sun4v */

tsb_user8k:
	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr

#ifndef UTSB_PHYS
	mov	ASI_N, %g7	! user TSBs accessed by VA
	mov	%g7, %asi
#endif /* UTSB_PHYS */

	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)

#ifdef sun4v
	cmp	%g5, T_INSTR_MMU_MISS
	be,a,pn	%xcc, 9f
	  mov	%g3, %g5
#endif /* sun4v */
	cmp	%g5, FAST_IMMU_MISS_TT
	be,pn	%xcc, 9f
	  mov	%g3, %g5

	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	! trapstat wants TTE in %g5
	retry
9:
	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	! trapstat wants TTE in %g5
	retry

tsb_user4m:
	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
4:
	brz,pn	%g1, 5f	/* Check to see if we have 2nd TSB programmed */
	  nop

#ifndef UTSB_PHYS
	mov	ASI_N, %g7	! user TSBs accessed by VA
	mov	%g7, %asi
#endif /* UTSB_PHYS */

        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)

5:
#ifdef sun4v
        cmp     %g5, T_INSTR_MMU_MISS
        be,a,pn %xcc, 9f
          mov   %g3, %g5
#endif /* sun4v */
        cmp     %g5, FAST_IMMU_MISS_TT
        be,pn   %xcc, 9f
        mov     %g3, %g5

        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
        ! trapstat wants TTE in %g5
        retry
9:
        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
        ! trapstat wants TTE in %g5
        retry

#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
	/*
	 * Panther ITLB synthesis.
	 * The Panther 32M and 256M ITLB code simulates these two large page
	 * sizes with 4M pages, to provide support for programs, for example
	 * Java, that may copy instructions into a 32M or 256M data page and
	 * then execute them. The code below generates the 4M pfn bits and
	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
	 * are ignored by the hardware.
	 *
	 * Now, load into TSB/TLB.  At this point:
	 * g2 = tagtarget
	 * g3 = tte
	 * g4 = patte
	 * g5 = tt
	 * g6 = tsbmiss area
	 */
tsb_user_pn_synth:
	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
	bz,pn %icc, 4b				/* if not, been here before */
	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
	brz,a,pn %g1, 5f			/* no 2nd tsb */
	  mov	%g3, %g5

	mov	MMU_TAG_ACCESS, %g7
	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */

	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
	mov	%g7, %asi
	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
5:
        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
        retry

tsb_user_itlb_synth:
	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */

	mov	MMU_TAG_ACCESS, %g7
	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
	brz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
	  or	%g5, %g3, %g5			/* add 4M bits to TTE */

	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
	mov	%g7, %asi
	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
7:
	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
        retry
#endif /* sun4v && ITLB_32M_256M_SUPPORT */

tsb_kernel:
#ifdef sun4v
	cmp	%g7, TTE4M
	bge,pn	%icc, 5f
#else
	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
	be,pn	%icc, 5f
#endif
	  nop
	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8k tsbptr
	ba,pt	%xcc, 6f
	  nop
5:
	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4m tsbptr
	brz,pn	%g1, 3f		/* skip programming if 4m TSB ptr is NULL */
	  nop
6:
#ifndef sun4v
tsb_kernel_patch_asi:
	or	%g0, RUNTIME_PATCH, %g6
	mov	%g6, %asi	! XXX avoid writing to %asi !!
#endif
	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
3:
#ifdef sun4v
	cmp	%g5, T_INSTR_MMU_MISS
	be,a,pn	%icc, 1f
	  mov	%g3, %g5			! trapstat wants TTE in %g5
#endif /* sun4v */
	cmp	%g5, FAST_IMMU_MISS_TT
	be,pn	%icc, 1f
	  mov	%g3, %g5			! trapstat wants TTE in %g5
	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	! trapstat wants TTE in %g5
	retry
1:
	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
	! trapstat wants TTE in %g5
	retry

tsb_ism:
	/*
	 * This is an ISM [i|d]tlb miss.  We optimize for largest
	 * page size down to smallest.
	 *
	 * g2 = vaddr + ctx	aka tag access register
	 * g3 = ismmap->ism_seg
	 * g4 = physical address of ismmap->ism_sfmmu
	 * g6 = tsbmiss area
	 */
	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
	  mov	PTL1_BAD_ISM, %g1
						/* g5 = pa of imap_vb_shift */
	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
	lduha	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
	srlx	%g3, %g4, %g3			/* clr size field */
	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
	sllx	%g3, %g4, %g3			/* g3 = ism vbase */
	and	%g2, %g1, %g4			/* g4 = ctx number */
	andn	%g2, %g1, %g1			/* g1 = tlb miss vaddr */
	sub	%g1, %g3, %g2			/* g2 = offset in ISM seg */
	or	%g2, %g4, %g2			/* g2 = tagacc (vaddr + ctx) */

	/*
	 * ISM pages are always locked down.
	 * If we can't find the tte then pagefault
	 * and let the spt segment driver resovle it.
	 *
	 * g2 = ISM vaddr (offset in ISM seg)
	 * g6 = tsb miss area
	 * g7 = ISM hatid
	 */
	sub	%g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
	lduha	[%g5]ASI_MEM, %g4		/* g5 = pa of imap_hatflags */
	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
	  nop

tsb_ism_32M:
	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
	brz,pn	%g5, tsb_ism_256M
	  nop

	/*
	 * 32M hash.
	 */

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
	    tsb_ism_4M)
	/* NOT REACHED */

tsb_ism_32M_found:
	brlz,pt %g3, tsb_validtte
	  nop
	ba,pt	%xcc, tsb_ism_4M
	  nop

tsb_ism_256M:
	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
	brz,a,pn %g5, ptl1_panic
	  mov	PTL1_BAD_ISM, %g1

	/*
	 * 256M hash.
	 */
	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
	    tsb_ism_4M)

tsb_ism_256M_found:
	brlz,pt %g3, tsb_validtte
	  nop

tsb_ism_4M:
	/*
	 * 4M hash.
	 */
	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
	    tsb_ism_8K)
	/* NOT REACHED */

tsb_ism_4M_found:
	brlz,pt %g3, tsb_validtte
	  nop

tsb_ism_8K:
	/*
	 * 8K and 64K hash.
	 */

	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
	    tsb_pagefault)
	/* NOT REACHED */

tsb_ism_8K_found:
	brlz,pt	%g3, tsb_validtte
	  nop

tsb_pagefault:
	rdpr	%tt, %g7
	cmp	%g7, FAST_PROT_TT
	be,a,pn	%icc, tsb_protfault
	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt

tsb_protfault:
	/*
	 * we get here if we couldn't find a valid tte in the hash.
	 *
	 * If user and we are at tl>1 we go to window handling code.
	 *
	 * If kernel and the fault is on the same page as our stack
	 * pointer, then we know the stack is bad and the trap handler
	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
	 *
	 * If this is a kernel trap and tl>1, panic.
	 *
	 * Otherwise we call pagefault.
	 */
	cmp	%g7, FAST_IMMU_MISS_TT
#ifdef sun4v
	MMU_FAULT_STATUS_AREA(%g4)
	ldx	[%g4 + MMFSA_I_CTX], %g5
	ldx	[%g4 + MMFSA_D_CTX], %g4
	move	%icc, %g5, %g4
	cmp	%g7, T_INSTR_MMU_MISS
	move	%icc, %g5, %g4
#else
	mov	MMU_TAG_ACCESS, %g4
	ldxa	[%g4]ASI_DMMU, %g2
	ldxa	[%g4]ASI_IMMU, %g5
	move	%icc, %g5, %g2
	cmp	%g7, T_INSTR_MMU_MISS
	move	%icc, %g5, %g2
	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
#endif
	brnz,pn	%g4, 3f				/* skip if not kernel */
	  rdpr	%tl, %g5

	add	%sp, STACK_BIAS, %g3
	srlx	%g3, MMU_PAGESHIFT, %g3
	srlx	%g2, MMU_PAGESHIFT, %g4
	cmp	%g3, %g4
	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
	  mov	PTL1_BAD_STACK, %g1

	cmp	%g5, 1
	ble,pt	%icc, 2f
	  nop
	TSTAT_CHECK_TL1(2f, %g1, %g2)
	rdpr	%tt, %g2
	cmp	%g2, FAST_PROT_TT
	mov	PTL1_BAD_KPROT_FAULT, %g1
	movne	%icc, PTL1_BAD_KMISS, %g1
	ba,pt	%icc, ptl1_panic
	  nop

2:
	/*
	 * We are taking a pagefault in the kernel on a kernel address.  If
	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
	 * want to call sfmmu_pagefault -- we will instead note that a fault
	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
	 * (instead of a "retry").  This will step over the faulting
	 * instruction.
	 */
	CPU_INDEX(%g1, %g2)
	set	cpu_core, %g2
	sllx	%g1, CPU_CORE_SHIFT, %g1
	add	%g1, %g2, %g1
	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
	bz	sfmmu_pagefault
	or	%g2, CPU_DTRACE_BADADDR, %g2
	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
	GET_MMU_D_ADDR(%g3, %g4)
	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
	done

3:
	cmp	%g5, 1
	ble,pt	%icc, 4f
	  nop
	TSTAT_CHECK_TL1(4f, %g1, %g2)
	ba,pt	%icc, sfmmu_window_trap
	  nop

4:
	/*
	 * We are taking a pagefault on a non-kernel address.  If we are in
	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
	 */
	CPU_INDEX(%g1, %g2)
	set	cpu_core, %g2
	sllx	%g1, CPU_CORE_SHIFT, %g1
	add	%g1, %g2, %g1
	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
	bz	sfmmu_pagefault
	or	%g2, CPU_DTRACE_BADADDR, %g2
	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
	GET_MMU_D_ADDR(%g3, %g4)
	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]

	/*
	 * Be sure that we're actually taking this miss from the kernel --
	 * otherwise we have managed to return to user-level with
	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
	 */
	rdpr	%tstate, %g2
	btst	TSTATE_PRIV, %g2
	bz,a	ptl1_panic
	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
	done

	ALTENTRY(tsb_tl0_noctxt)
	/*
	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
	 * if it is, indicated that we have faulted and issue a done.
	 */
	CPU_INDEX(%g5, %g6)
	set	cpu_core, %g6
	sllx	%g5, CPU_CORE_SHIFT, %g5
	add	%g5, %g6, %g5
	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
	bz	1f
	or	%g6, CPU_DTRACE_BADADDR, %g6
	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
	GET_MMU_D_ADDR(%g3, %g4)
	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]

	/*
	 * Be sure that we're actually taking this miss from the kernel --
	 * otherwise we have managed to return to user-level with
	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
	 */
	rdpr	%tstate, %g5
	btst	TSTATE_PRIV, %g5
	bz,a	ptl1_panic
	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
	done

1:
	rdpr	%tt, %g5
	cmp	%g5, FAST_IMMU_MISS_TT
#ifdef sun4v
	MMU_FAULT_STATUS_AREA(%g2)
	be,a,pt	%icc, 2f
	  ldx	[%g2 + MMFSA_I_CTX], %g3
	cmp	%g5, T_INSTR_MMU_MISS
	be,a,pt	%icc, 2f
	  ldx	[%g2 + MMFSA_I_CTX], %g3
	ldx	[%g2 + MMFSA_D_CTX], %g3
2:
#else
	mov	MMU_TAG_ACCESS, %g2
	be,a,pt	%icc, 2f
	  ldxa	[%g2]ASI_IMMU, %g3
	ldxa	[%g2]ASI_DMMU, %g3
2:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
#endif
	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
	rdpr	%tl, %g5
	cmp	%g5, 1
	ble,pt	%icc, sfmmu_mmu_trap
	  nop
	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
	ba,pt	%icc, sfmmu_window_trap
	  nop
	SET_SIZE(sfmmu_tsb_miss)

#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE
#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct
#endif

#endif /* lint */

#if defined (lint)
/*
 * This routine will look for a user or kernel vaddr in the hash
 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
 * grab any locks.  It should only be used by other sfmmu routines.
 */
/* ARGSUSED */
pfn_t
sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
{
	return(0);
}

#else /* lint */

	ENTRY_NP(sfmmu_vatopfn)
 	/*
 	 * disable interrupts
 	 */
 	rdpr	%pstate, %o3
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
#endif
	/*
	 * disable interrupts to protect the TSBMISS area
	 */
	andn    %o3, PSTATE_IE, %o5
	wrpr    %o5, 0, %pstate

	/*
	 * o0 = vaddr
	 * o1 = sfmmup
	 * o2 = ttep
	 */
	CPU_TSBMISS_AREA(%g1, %o5)
	ldn	[%g1 + TSBMISS_KHATID], %o4
	cmp	%o4, %o1
	bne,pn	%ncc, vatopfn_nokernel
	  mov	TTE64K, %g5			/* g5 = rehash # */
	mov %g1,%o5				/* o5 = tsbmiss_area */
	/*
	 * o0 = vaddr
	 * o1 & o4 = hatid
	 * o2 = ttep
	 * o5 = tsbmiss area
	 */
	mov	HBLK_RANGE_SHIFT, %g6
1:

	/*
	 * o0 = vaddr
	 * o1 = sfmmup
	 * o2 = ttep
	 * o3 = old %pstate
	 * o4 = hatid
	 * o5 = tsbmiss
	 * g5 = rehash #
	 * g6 = hmeshift
	 *
	 * The first arg to GET_TTE is actually tagaccess register
	 * not just vaddr. Since this call is for kernel we need to clear
	 * any lower vaddr bits that would be interpreted as ctx bits.
	 */
	set     TAGACC_CTX_MASK, %g1
	andn    %o0, %g1, %o0
	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)

kvtop_hblk_found:
	/*
	 * o0 = vaddr
	 * o1 = sfmmup
	 * o2 = ttep
	 * g1 = tte
	 * g2 = tte pa
	 * g3 = tte va
	 * o2 = tsbmiss area
	 * o1 = hat id
	 */
	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
	stx %g1,[%o2]				/* put tte into *ttep */
	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
	/*
	 * o0 = vaddr
	 * o1 = sfmmup
	 * o2 = ttep
	 * g1 = pfn
	 */
	ba,pt	%xcc, 6f
	  mov	%g1, %o0

kvtop_nohblk:
	/*
	 * we get here if we couldn't find valid hblk in hash.  We rehash
	 * if neccesary.
	 */
	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
#ifdef sun4v
	cmp	%g5, MAX_HASHCNT
#else
	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
#endif
	be,a,pn	%icc, 6f
	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
	mov	%o1, %o4			/* restore hatid */
#ifdef sun4v
        add	%g5, 2, %g5
	cmp	%g5, 3
	move	%icc, MMU_PAGESHIFT4M, %g6
	ba,pt	%icc, 1b
	movne	%icc, MMU_PAGESHIFT256M, %g6
#else
        inc	%g5
	cmp	%g5, 2
	move	%icc, MMU_PAGESHIFT512K, %g6
	ba,pt	%icc, 1b
	movne	%icc, MMU_PAGESHIFT4M, %g6
#endif
6:
	retl
 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */

tsb_suspend:
	/*
	 * o0 = vaddr
	 * o1 = sfmmup
	 * o2 = ttep
	 * g1 = tte
	 * g2 = tte pa
	 * g3 = tte va
	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
	 */
	stx %g1,[%o2]				/* put tte into *ttep */
	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
	  sub	%g0, 1, %o0			/* output = -1 (PFN_INVALID) */
	TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4)
	/*
	 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn#
	 * o1 = sfmmup
	 * o2 = ttep
	 * g1 = pfn
	 */
	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
8:
	retl
	 wrpr	%g0, %o3, %pstate		/* enable interrupts */

vatopfn_nokernel:
	/*
	 * This routine does NOT support user addresses
	 * There is a routine in C that supports this.
	 * The only reason why we don't have the C routine
	 * support kernel addresses as well is because
	 * we do va_to_pa while holding the hashlock.
	 */
 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
	save	%sp, -SA(MINFRAME), %sp
	sethi	%hi(sfmmu_panic3), %o0
	call	panic
	 or	%o0, %lo(sfmmu_panic3), %o0

	SET_SIZE(sfmmu_vatopfn)
#endif /* lint */


#if !defined(lint)

/*
 * kpm lock used between trap level tsbmiss handler and kpm C level.
 */
#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
	mov     0xff, tmp1						;\
label1:									;\
	casa    [kpmlckp]asi, %g0, tmp1					;\
	brnz,pn tmp1, label1						;\
	mov     0xff, tmp1						;\
	membar  #LoadLoad

#define KPMLOCK_EXIT(kpmlckp, asi)					\
	membar  #LoadStore|#StoreStore					;\
	sta     %g0, [kpmlckp]asi

/*
 * Lookup a memseg for a given pfn and if found, return the physical
 * address of the corresponding struct memseg in mseg, otherwise
 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
 * tsbmp, %asi is assumed to be ASI_MEM.
 * This lookup is done by strictly traversing only the physical memseg
 * linkage. The more generic approach, to check the virtual linkage
 * before using the physical (used e.g. with hmehash buckets), cannot
 * be used here. Memory DR operations can run in parallel to this
 * lookup w/o any locks and updates of the physical and virtual linkage
 * cannot be done atomically wrt. to each other. Because physical
 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
 * as "physical NULL" pointer.
 */
#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
	udivx	pfn, mseg, mseg						;\
	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
	add	tmp1, mseg, tmp1					;\
	ldxa	[tmp1]%asi, mseg					;\
	cmp	mseg, MSEG_NULLPTR_PA					;\
	be,pn	%xcc, label/**/1		/* if not found */	;\
	  nop								;\
	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
	cmp	pfn, tmp1			/* pfn - pages_base */	;\
	blu,pn	%xcc, label/**/1					;\
	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
	cmp	pfn, tmp2			/* pfn - pages_end */	;\
	bgeu,pn	%xcc, label/**/1					;\
	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
	mulx	tmp1, PAGE_SIZE, tmp1					;\
	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
	add	tmp2, tmp1, tmp1			/* pp */	;\
	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
	cmp	tmp2, pfn						;\
	be,pt	%xcc, label/**/_ok			/* found */	;\
label/**/1:								;\
	/* brute force lookup */					;\
	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
label/**/2:								;\
	cmp	mseg, MSEG_NULLPTR_PA					;\
	be,pn	%xcc, label/**/_ok		/* if not found */	;\
	  nop								;\
	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
	cmp	pfn, tmp1			/* pfn - pages_base */	;\
	blu,a,pt %xcc, label/**/2					;\
	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
	cmp	pfn, tmp2			/* pfn - pages_end */	;\
	bgeu,a,pt %xcc, label/**/2					;\
	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
label/**/_ok:

	/*
	 * kpm tsb miss handler large pages
	 * g1 = 8K kpm TSB entry pointer
	 * g2 = tag access register
	 * g3 = 4M kpm TSB entry pointer
	 */
	ALTENTRY(sfmmu_kpm_dtsb_miss)
	TT_TRACE(trace_tsbmiss)

	CPU_INDEX(%g7, %g6)
	sethi	%hi(kpmtsbm_area), %g6
	sllx	%g7, KPMTSBM_SHIFT, %g7
	or	%g6, %lo(kpmtsbm_area), %g6
	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */

	/* check enable flag */
	ldub	[%g6 + KPMTSBM_FLAGS], %g4
	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
	  nop

	/* VA range check */
	ldx	[%g6 + KPMTSBM_VBASE], %g7
	cmp	%g2, %g7
	blu,pn	%xcc, sfmmu_tsb_miss
	  ldx	[%g6 + KPMTSBM_VEND], %g5
	cmp	%g2, %g5
	bgeu,pn	%xcc, sfmmu_tsb_miss
	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]

	/*
	 * check TL tsbmiss handling flag
	 * bump tsbmiss counter
	 */
	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
#ifdef	DEBUG
	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
	inc	%g5
	brz,pn	%g3, sfmmu_kpm_exception
	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
#else
	inc	%g5
	st	%g5, [%g6 + KPMTSBM_TSBMISS]
#endif
	/*
	 * At this point:
	 *  g1 = 8K kpm TSB pointer (not used)
	 *  g2 = tag access register
	 *  g3 = clobbered
	 *  g6 = per-CPU kpm tsbmiss area
	 *  g7 = kpm_vbase
	 */

	/* vaddr2pfn */
	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
	srax    %g4, %g3, %g2			/* which alias range (r) */
	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */

	/*
	 * Setup %asi
	 * mseg_pa = page_numtomemseg_nolock(pfn)
	 * if (mseg_pa == NULL) sfmmu_kpm_exception
	 * g2=pfn
	 */
	mov	ASI_MEM, %asi
	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
	cmp	%g3, MSEG_NULLPTR_PA
	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
	  nop

	/*
	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
	 * g2=pfn g3=mseg_pa
	 */
	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
	srlx	%g2, %g5, %g4
	sllx	%g4, %g5, %g4
	sub	%g4, %g7, %g4
	srlx	%g4, %g5, %g4

	/*
	 * Validate inx value
	 * g2=pfn g3=mseg_pa g4=inx
	 */
#ifdef	DEBUG
	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
	cmp	%g4, %g5			/* inx - nkpmpgs */
	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
#else
	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
#endif
	/*
	 * kp = &mseg_pa->kpm_pages[inx]
	 */
	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
	add	%g5, %g4, %g5			/* kp */

	/*
	 * KPMP_HASH(kp)
	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
	 */
	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
	sub	%g7, 1, %g7			/* mask */
	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
	add	%g5, %g1, %g5			/* y = ksp + x */
	and 	%g5, %g7, %g5			/* hashinx = y & mask */

	/*
	 * Calculate physical kpm_page pointer
	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
	 */
	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
	add	%g1, %g4, %g1			/* kp_pa */

	/*
	 * Calculate physical hash lock address
	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
	 */
	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
	sllx	%g5, KPMHLK_SHIFT, %g5
	add	%g4, %g5, %g3
	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */

	/*
	 * Assemble tte
	 * g1=kp_pa g2=pfn g3=hlck_pa
	 */
#ifdef sun4v
	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
	sllx	%g5, 32, %g5
	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
	or	%g4, TTE4M, %g4
	or	%g5, %g4, %g5
#else
	sethi	%hi(TTE_VALID_INT), %g4
	mov	TTE4M, %g5
	sllx	%g5, TTE_SZ_SHFT_INT, %g5
	or	%g5, %g4, %g5			/* upper part */
	sllx	%g5, 32, %g5
	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
	or	%g5, %g4, %g5
#endif
	sllx	%g2, MMU_PAGESHIFT, %g4
	or	%g5, %g4, %g5			/* tte */
	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */

	/*
	 * tsb dropin
	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
	 */

	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)

	/* use C-handler if there's no go for dropin */
	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
	cmp	%g7, -1
	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
	  nop

#ifdef	DEBUG
	/* double check refcnt */
	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
	brz,pn	%g7, 5f			/* let C-handler deal with this */
	  nop
#endif

#ifndef sun4v
	ldub	[%g6 + KPMTSBM_FLAGS], %g7
	mov	ASI_N, %g1
	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
	movnz	%icc, ASI_MEM, %g1
	mov	%g1, %asi
#endif

	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)

	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)

	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)

	/* KPMLOCK_EXIT(kpmlckp, asi) */
	KPMLOCK_EXIT(%g3, ASI_MEM)

	/*
	 * If trapstat is running, we need to shift the %tpc and %tnpc to
	 * point to trapstat's TSB miss return code (note that trapstat
	 * itself will patch the correct offset to add).
	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
	 */
	rdpr	%tl, %g7
	cmp	%g7, 1
	ble	%icc, 0f
	sethi	%hi(KERNELBASE), %g6
	rdpr	%tpc, %g7
	or	%g6, %lo(KERNELBASE), %g6
	cmp	%g7, %g6
	bgeu	%xcc, 0f
	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
	wrpr	%g7, %tpc
	add	%g7, 4, %g7
	wrpr	%g7, %tnpc
0:
	retry
5:
	/* g3=hlck_pa */
	KPMLOCK_EXIT(%g3, ASI_MEM)
	ba,pt	%icc, sfmmu_kpm_exception
	  nop
	SET_SIZE(sfmmu_kpm_dtsb_miss)

	/*
	 * kpm tsbmiss handler for smallpages
	 * g1 = 8K kpm TSB pointer
	 * g2 = tag access register
	 * g3 = 4M kpm TSB pointer
	 */
	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
	TT_TRACE(trace_tsbmiss)
	CPU_INDEX(%g7, %g6)
	sethi	%hi(kpmtsbm_area), %g6
	sllx	%g7, KPMTSBM_SHIFT, %g7
	or	%g6, %lo(kpmtsbm_area), %g6
	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */

	/* check enable flag */
	ldub	[%g6 + KPMTSBM_FLAGS], %g4
	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
	  nop

	/*
	 * VA range check
	 * On fail: goto sfmmu_tsb_miss
	 */
	ldx	[%g6 + KPMTSBM_VBASE], %g7
	cmp	%g2, %g7
	blu,pn	%xcc, sfmmu_tsb_miss
	  ldx	[%g6 + KPMTSBM_VEND], %g5
	cmp	%g2, %g5
	bgeu,pn	%xcc, sfmmu_tsb_miss
	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */

	/*
	 * check TL tsbmiss handling flag
	 * bump tsbmiss counter
	 */
	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
#ifdef	DEBUG
	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
	inc	%g5
	brz,pn	%g1, sfmmu_kpm_exception
	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
#else
	inc	%g5
	st	%g5, [%g6 + KPMTSBM_TSBMISS]
#endif
	/*
	 * At this point:
	 *  g1 = clobbered
	 *  g2 = tag access register
	 *  g3 = 4M kpm TSB pointer (not used)
	 *  g6 = per-CPU kpm tsbmiss area
	 *  g7 = kpm_vbase
	 */

	/* vaddr2pfn */
	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
	srax    %g4, %g3, %g2			/* which alias range (r) */
	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */

	/*
	 * Setup %asi
	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
	 * if (mseg not found) sfmmu_kpm_exception
	 * g2=pfn
	 */
	mov	ASI_MEM, %asi
	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
	cmp	%g3, MSEG_NULLPTR_PA
	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
	  nop

	/*
	 * inx = pfn - mseg_pa->kpm_pbase
	 * g2=pfn g3=mseg_pa
	 */
	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
	sub	%g2, %g7, %g4

#ifdef	DEBUG
	/*
	 * Validate inx value
	 * g2=pfn g3=mseg_pa g4=inx
	 */
	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
	cmp	%g4, %g5			/* inx - nkpmpgs */
	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
#else
	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
#endif
	/* ksp = &mseg_pa->kpm_spages[inx] */
	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
	add	%g5, %g4, %g5			/* ksp */

	/*
	 * KPMP_SHASH(kp)
	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
	 */
	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
	sub	%g7, 1, %g7			/* mask */
	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
	add	%g5, %g1, %g5			/* y = ksp + x */
	and 	%g5, %g7, %g5			/* hashinx = y & mask */

	/*
	 * Calculate physical kpm_spage pointer
	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
	 */
	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
	add	%g1, %g4, %g1			/* ksp_pa */

	/*
	 * Calculate physical hash lock address.
	 * Note: Changes in kpm_shlk_t must be reflected here.
	 * g1=ksp_pa g2=pfn g5=hashinx
	 */
	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
	sllx	%g5, KPMSHLK_SHIFT, %g5
	add	%g4, %g5, %g3			/* hlck_pa */

	/*
	 * Assemble tte
	 * g1=ksp_pa g2=pfn g3=hlck_pa
	 */
	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
	sllx	%g5, 32, %g5
	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
	or	%g5, %g4, %g5
	sllx	%g2, MMU_PAGESHIFT, %g4
	or	%g5, %g4, %g5			/* tte */
	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */

	/*
	 * tsb dropin
	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
	 */

	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)

	/* use C-handler if there's no go for dropin */
	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
	cmp	%g7, -1
	bne,pn	%xcc, 5f
	  nop

#ifndef sun4v
	ldub	[%g6 + KPMTSBM_FLAGS], %g7
	mov	ASI_N, %g1
	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
	movnz	%icc, ASI_MEM, %g1
	mov	%g1, %asi
#endif

	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)

	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)

	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)

	/* KPMLOCK_EXIT(kpmlckp, asi) */
	KPMLOCK_EXIT(%g3, ASI_MEM)

	/*
	 * If trapstat is running, we need to shift the %tpc and %tnpc to
	 * point to trapstat's TSB miss return code (note that trapstat
	 * itself will patch the correct offset to add).
	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
	 */
	rdpr	%tl, %g7
	cmp	%g7, 1
	ble	%icc, 0f
	sethi	%hi(KERNELBASE), %g6
	rdpr	%tpc, %g7
	or	%g6, %lo(KERNELBASE), %g6
	cmp	%g7, %g6
	bgeu	%xcc, 0f
	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
	wrpr	%g7, %tpc
	add	%g7, 4, %g7
	wrpr	%g7, %tnpc
0:
	retry
5:
	/* g3=hlck_pa */
	KPMLOCK_EXIT(%g3, ASI_MEM)
	ba,pt	%icc, sfmmu_kpm_exception
	  nop
	SET_SIZE(sfmmu_kpm_dtsb_miss_small)

#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
#endif

#endif /* lint */

#ifdef	lint
/*
 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
 * Called from C-level, sets/clears "go" indication for trap level handler.
 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
 * Assumes khl_mutex is held when called from C-level.
 */
/* ARGSUSED */
void
sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
{
}

/*
 * kpm_smallpages: stores val to byte at address mapped within
 * low level lock brackets. The old value is returned.
 * Called from C-level.
 */
/* ARGSUSED */
int
sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
{
	return (0);
}

#else /* lint */

	.seg	".data"
sfmmu_kpm_tsbmtl_panic:
	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
	.byte	0
sfmmu_kpm_stsbmtl_panic:
	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
	.byte	0
	.align	4
	.seg	".text"

	ENTRY_NP(sfmmu_kpm_tsbmtl)
	rdpr	%pstate, %o3
	/*
	 * %o0 = &kp_refcntc
	 * %o1 = &khl_lock
	 * %o2 = 0/1 (off/on)
	 * %o3 = pstate save
	 */
#ifdef DEBUG
	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
	bnz,pt %icc, 1f				/* disabled, panic	 */
	  nop
	save	%sp, -SA(MINFRAME), %sp
	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
	call	panic
	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
	ret
	restore
1:
#endif /* DEBUG */
	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */

	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
	mov	-1, %o5
	brz,a	%o2, 2f
	  mov	0, %o5
2:
	sth	%o5, [%o0]
	KPMLOCK_EXIT(%o1, ASI_N)

	retl
	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
	SET_SIZE(sfmmu_kpm_tsbmtl)

	ENTRY_NP(sfmmu_kpm_stsbmtl)
	rdpr	%pstate, %o3
	/*
	 * %o0 = &mapped
	 * %o1 = &kshl_lock
	 * %o2 = val
	 * %o3 = pstate save
	 */
#ifdef DEBUG
	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
	bnz,pt %icc, 1f				/* disabled, panic	 */
	  nop
	save	%sp, -SA(MINFRAME), %sp
	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
	call	panic
	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
	ret
	restore
1:
#endif /* DEBUG */
	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */

	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
	ldsb	[%o0], %o5
	stb	%o2, [%o0]
	KPMLOCK_EXIT(%o1, ASI_N)

	mov	%o5, %o0			/* return old val */
	retl
	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
	SET_SIZE(sfmmu_kpm_stsbmtl)

#endif /* lint */

#ifndef lint
#ifdef sun4v
	/*
	 * User/kernel data miss w// multiple TSBs
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.  Second probe covers 4M page size only.
	 *
	 * MMU fault area contains miss address and context.
	 */
	ALTENTRY(sfmmu_slow_dmmu_miss)
	GET_MMU_D_TAGACC_CTX(%g2, %g3)	! %g2 = tagacc, %g3 = ctx

slow_miss_common:
	/*
	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
	 */
	brnz,pt	%g3, 8f			! check for user context
	  nop

	/*
	 * Kernel miss
	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
	 * branch to sfmmu_tsb_miss_tt to handle it.
	 */
	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
sfmmu_dslow_patch_ktsb_base:
	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
sfmmu_dslow_patch_ktsb_szcode:
	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)

	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
	! %g1 = First TSB entry pointer, as TSB miss handler expects

	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
sfmmu_dslow_patch_ktsb4m_base:
	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
sfmmu_dslow_patch_ktsb4m_szcode:
	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)

	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
	.empty

8:
	/*
	 * User miss
	 * Get first TSB pointer in %g1
	 * Get second TSB pointer (or NULL if no second TSB) in %g3
	 * Branch to sfmmu_tsb_miss_tt to handle it
	 */
	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
	/* %g1 = first TSB entry ptr now, %g2 preserved */

	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
	brlz,a,pt %g3, sfmmu_tsb_miss_tt	/* done if no 2nd TSB */
	  mov	%g0, %g3

	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
	/* %g3 = second TSB entry ptr now, %g2 preserved */
9:
	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
	.empty
	SET_SIZE(sfmmu_slow_dmmu_miss)


	/*
	 * User/kernel instruction miss w/ multiple TSBs
	 * The first probe covers 8K, 64K, and 512K page sizes,
	 * because 64K and 512K mappings are replicated off 8K
	 * pointer.  Second probe covers 4M page size only.
	 *
	 * MMU fault area contains miss address and context.
	 */
	ALTENTRY(sfmmu_slow_immu_miss)
	MMU_FAULT_STATUS_AREA(%g2)
	ldx	[%g2 + MMFSA_I_CTX], %g3
	ldx	[%g2 + MMFSA_I_ADDR], %g2
	srlx	%g2, MMU_PAGESHIFT, %g2	! align address to page boundry
	sllx	%g2, MMU_PAGESHIFT, %g2
	ba,pt	%xcc, slow_miss_common
	or	%g2, %g3, %g2
	SET_SIZE(sfmmu_slow_immu_miss)

#endif /* sun4v */
#endif	/* lint */

#ifndef lint

/*
 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
 */
	.seg	".data"
	.align	64
	.global tsbmiss_area
tsbmiss_area:
	.skip	(TSBMISS_SIZE * NCPU)

	.align	64
	.global kpmtsbm_area
kpmtsbm_area:
	.skip	(KPMTSBM_SIZE * NCPU)
#endif	/* lint */