sun4u/vm/zulu_hat.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/mman.h>
#include <sys/sunddi.h>
#include <sys/tnf_probe.h>
#include <vm/hat_sfmmu.h>
#include <vm/as.h>
#include <vm/xhat.h>
#include <vm/xhat_sfmmu.h>
#include <sys/zulu_hat.h>
#include <sys/zulumod.h>

/*
 * This file contains the implementation of zulu_hat: an XHAT provider
 * to support the MMU for the XVR-4000 graphics accelerator (code name zulu).
 *
 * The zulu hat is linked into the kernel misc module zuluvm.
 * zuluvm provides services that the zulu device driver module requires
 * that are not part of the standard ddi. See PSARC 2002/231.
 *
 * The zulu driver is delivered by the graphics consolidation.
 * zuluvm is in ON workspace.
 *
 * There are two types of interfaces provided by zulu_hat
 *   1.	The set of functions and data structures used by zuluvm to obtain
 * 	tte entries for the zulu MMU and to manage the association between
 *	user process's address spaces and zulu graphics contexts.
 *
 *   2.	The entry points required for an XHAT provider: zulu_hat_ops
 */

/*
 * zulu_ctx_tab contains an array of pointers to the zulu_hats.
 *
 * During zulu graphics context switch, the zulu MMU's current context register
 * is set to the index of the process's zulu hat's location in the array
 * zulu_ctx_tab.
 *
 * This allows the TL=1 TLB miss handler to quickly find the zulu hat and
 * lookup a tte in the zulu hat's TSB.
 *
 * To synchronize with the trap handler we use bit zero of
 * the pointer as a lock bit. See the function zulu_ctx_tsb_lock_enter().
 *
 * If the trap handler finds the ctx locked it doesn't wait, it
 * posts a soft interrupt which is handled at TL=0.
 */

#define		ZULU_HAT_MAX_CTX 32
struct zulu_hat *zulu_ctx_tab[ZULU_HAT_MAX_CTX];

/*
 * To avoid searching through the whole zulu_ctx_tab for a free slot,
 * we maintain the value of zulu_ctx_search_start.
 *
 * This value is a guess as to where a free slot in the context table might be.
 * All slots < zulu_ctx_search_start are definitely occupied.
 */
static int zulu_ctx_search_start = 0;


/*
 * this mutex protects the zulu_ctx_tab and zulu_ctx_search_start
 */
static kmutex_t	zulu_ctx_lock;


uint64_t	zulu_tsb_hit = 0;	/* assembly code increments this */
static uint64_t	zulu_tsb_miss = 0;
static uint64_t	zulu_as_fault = 0;

/*
 * The zulu device has two zulu data mmus.
 * We use the base pagesize for one of them and the and 4M for the other.
 */
extern int zuluvm_base_pgsize;


/*
 * call zuluvm to remove translations for a page
 */
static void
zulu_hat_demap_page(struct zulu_hat *zhat, caddr_t vaddr, int size)
{
	if (zhat->zulu_ctx < 0) {
		/* context has been stolen, so page is already demapped */
		return;
	}
	zuluvm_demap_page(zhat->zdev, NULL, zhat->zulu_ctx, vaddr, size);
}

static void
zulu_hat_demap_ctx(void *zdev, int zulu_ctx)
{
	if (zulu_ctx < 0) {
		/* context has been stolen */
		return;
	}
	zuluvm_demap_ctx(zdev, zulu_ctx);
}


/*
 * steal the least recently used context slot.
 */
static int
zulu_hat_steal_ctx()
{
	int		ctx;
	hrtime_t	delta = INT64_MAX;
	struct zulu_hat *zhat_oldest = NULL;

	ASSERT(mutex_owned(&zulu_ctx_lock));

	for (ctx = 0; ctx < ZULU_HAT_MAX_CTX; ctx++) {
		struct zulu_hat *zhat = ZULU_CTX_GET_HAT(ctx);

		/*
		 * we shouldn't be here unless all slots are occupied
		 */
		ASSERT(zhat != NULL);

		TNF_PROBE_3(steal_ctx_loop, "zulu_hat", /* CSTYLED */,
		    tnf_int, ctx, ctx,
		    tnf_long, last_used, zhat->last_used,
		    tnf_long, oldest, delta);

		if (zhat->last_used <  delta) {
			zhat_oldest = zhat;
			delta  = zhat->last_used;
		}
	}

	ASSERT(zhat_oldest != NULL);

	mutex_enter(&zhat_oldest->lock);

	/* Nobody should have the tsb lock bit set here */
	ASSERT(((uint64_t)zulu_ctx_tab[zhat_oldest->zulu_ctx] & ZULU_CTX_LOCK)
	    == 0);

	ctx = zhat_oldest->zulu_ctx;
	zhat_oldest->zulu_ctx = -1;

	ZULU_CTX_SET_HAT(ctx, NULL);

	zulu_hat_demap_ctx(zhat_oldest->zdev, ctx);

	mutex_exit(&zhat_oldest->lock);

	TNF_PROBE_1(zulu_hat_steal_ctx, "zulu_hat", /* CSTYLED */,
		tnf_int, ctx, ctx);

	return (ctx);
}

/*
 * find a slot in the context table for a zulu_hat
 */
static void
zulu_hat_ctx_alloc(struct zulu_hat *zhat)
{
	int 		ctx;

	mutex_enter(&zulu_ctx_lock);

	for (ctx = zulu_ctx_search_start; ctx < ZULU_HAT_MAX_CTX; ctx++) {
		if (ZULU_CTX_IS_FREE(ctx)) {
			zulu_ctx_search_start = ctx + 1;
			break;
		}
	}

	if (ctx == ZULU_HAT_MAX_CTX) {
		/* table is full need to steal an entry */
		zulu_ctx_search_start = ZULU_HAT_MAX_CTX;
		ctx = zulu_hat_steal_ctx();
	}

	mutex_enter(&zhat->lock);

	ZULU_CTX_SET_HAT(ctx, zhat);
	zhat->zulu_ctx = ctx;

	mutex_exit(&zhat->lock);

	mutex_exit(&zulu_ctx_lock);

	TNF_PROBE_2(zulu_hat_ctx_alloc, "zulu_hat", /* CSTYLED */,
		tnf_opaque, zhat, zhat, tnf_int, ctx, ctx);
}

/*
 * zulu_hat_validate_ctx: Called before the graphics context associated
 * with a given zulu hat becomes the current zulu graphics context.
 * Make sure that the hat has a slot in zulu_ctx_tab.
 */
void
zulu_hat_validate_ctx(struct zulu_hat *zhat)
{
	if (zhat->zulu_ctx < 0)  {
		zulu_hat_ctx_alloc(zhat);
	}
	zhat->last_used = gethrtime();
}


static void
zulu_hat_ctx_free(struct zulu_hat *zhat)
{
	TNF_PROBE_1(zulu_hat_ctx_free, "zulu_hat", /* CSTYLED */,
		tnf_int, ctx, zhat->zulu_ctx);

	mutex_enter(&zulu_ctx_lock);

	mutex_enter(&zhat->lock);
	if (zhat->zulu_ctx >= 0) {
		ZULU_CTX_SET_HAT(zhat->zulu_ctx, NULL);

		if (zulu_ctx_search_start > zhat->zulu_ctx) {
			zulu_ctx_search_start = zhat->zulu_ctx;
		}
	}
	mutex_exit(&zhat->lock);
	mutex_exit(&zulu_ctx_lock);
}

/*
 * Lock the zulu tsb for a given zulu_hat.
 *
 * We're just protecting against the TLB trap handler here. Other operations
 * on the zulu_hat require entering the zhat's lock.
 */
static void
zulu_ctx_tsb_lock_enter(struct zulu_hat *zhat)
{
	uint64_t	lck;
	uint64_t    	*plck;

	ASSERT(mutex_owned(&zhat->lock));

	if (zhat->zulu_ctx < 0) {
		return;
	}
	plck = (uint64_t *)&zulu_ctx_tab[zhat->zulu_ctx];

	for (; ; ) {
		lck = *plck;
		if (!(lck & ZULU_CTX_LOCK)) {
			uint64_t old_lck, new_lck;

			new_lck = lck | ZULU_CTX_LOCK;

			old_lck = cas64(plck, lck, new_lck);

			if (old_lck == lck) {
				/*
				 * success
				 */
				break;
			}
		}
	}
}

static void
zulu_ctx_tsb_lock_exit(struct zulu_hat *zhat)
{
	uint64_t	lck;
	int		zulu_ctx = zhat->zulu_ctx;

	if (zulu_ctx < 0) {
		return;
	}
	lck = (uint64_t)zulu_ctx_tab[zulu_ctx];
	ASSERT(lck & ZULU_CTX_LOCK);
	lck &= ~ZULU_CTX_LOCK;
	zulu_ctx_tab[zulu_ctx] = (struct zulu_hat *)lck;
}

/*
 * Each zulu hat has a "shadow tree" which is a table of 4MB address regions
 * for which the zhat has mappings.
 *
 * This table is maintained in an avl tree.
 * Nodes in the tree are called shadow blocks (or sblks)
 *
 * This data structure allows unload operations by (address, range) to be
 * much more efficent.
 *
 * We get called a lot for address ranges that have never been supplied
 * to zulu.
 */

/*
 * compare the base address of two nodes in the shadow tree
 */
static int
zulu_shadow_tree_compare(const void *a, const void *b)
{
	struct zulu_shadow_blk *zba = (struct zulu_shadow_blk *)a;
	struct zulu_shadow_blk *zbb = (struct zulu_shadow_blk *)b;
	uint64_t		addr_a = zba->ivaddr;
	uint64_t		addr_b = zbb->ivaddr;

	TNF_PROBE_2(zulu_shadow_tree_compare, "zulu_shadow_tree", /* CSTYLED */,
		tnf_opaque, addr_a, addr_a, tnf_opaque, addr_b, addr_b);

	if (addr_a < addr_b) {
		return (-1);
	} else if (addr_a > addr_b) {
		return (1);
	} else {
		return (0);
	}
}

/*
 * lookup the entry in the shadow tree for a given virtual address
 */
static struct zulu_shadow_blk *
zulu_shadow_tree_lookup(struct zulu_hat *zhat, uint64_t ivaddr,
	avl_index_t *where)
{
	struct zulu_shadow_blk proto;
	struct zulu_shadow_blk *sblk;

	proto.ivaddr = ivaddr & ZULU_SHADOW_BLK_MASK;

	/*
	 * pages typically fault in in order so we cache the last shadow
	 * block that was referenced so we usually get to reduce calls to
	 * avl_find.
	 */
	if ((zhat->sblk_last != NULL) &&
	    (proto.ivaddr == zhat->sblk_last->ivaddr)) {
		sblk = zhat->sblk_last;
	} else {
		sblk = (struct zulu_shadow_blk *)avl_find(&zhat->shadow_tree,
		    &proto, where);
		zhat->sblk_last = sblk;
	}

	TNF_PROBE_2(zulu_shadow_tree_lookup, "zulu_shadow_tree", /* CSTYLED */,
	    tnf_opaque, ivaddr, proto.ivaddr,
	    tnf_opaque, where, where ? *where : ~0);

	return (sblk);
}

/*
 * insert a sblk into the shadow tree for a given zblk.
 * If a sblk already exists, just increment it's refcount.
 */
static void
zulu_shadow_tree_insert(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)
{
	avl_index_t		where;
	struct zulu_shadow_blk 	*sblk  = NULL;
	uint64_t		ivaddr;
	uint64_t		end;

	ivaddr = zblk->zulu_hat_blk_vaddr & ZULU_SHADOW_BLK_MASK;

	end = zblk->zulu_hat_blk_vaddr + ZULU_HAT_PGSZ(zblk->zulu_hat_blk_size);

	sblk = zulu_shadow_tree_lookup(zhat, ivaddr, &where);
	if (sblk != NULL) {
		sblk->ref_count++;

		end = zblk->zulu_hat_blk_vaddr +
		    ZULU_HAT_PGSZ(zblk->zulu_hat_blk_size);
		if (zblk->zulu_hat_blk_vaddr < sblk->min_addr) {
			sblk->min_addr = zblk->zulu_hat_blk_vaddr;
		}
		/*
		 * a blk can set both the minimum and maximum when it
		 * is the first zblk added to a previously emptied sblk
		 */
		if (end > sblk->max_addr) {
			sblk->max_addr = end;
		}
	} else {
		sblk = kmem_zalloc(sizeof (*sblk), KM_SLEEP);
		sblk->ref_count = 1;
		sblk->ivaddr = ivaddr;
		sblk->min_addr = zblk->zulu_hat_blk_vaddr;
		sblk->max_addr = end;
		zhat->sblk_last = sblk;

		avl_insert(&zhat->shadow_tree, sblk, where);
	}
	zblk->zulu_shadow_blk = sblk;
	TNF_PROBE_2(zulu_shadow_tree_insert, "zulu_shadow_tree", /* CSTYLED */,
	    tnf_opaque, vaddr, ivaddr,
	    tnf_opaque, ref_count, sblk->ref_count);
}

/*
 * decrement the ref_count for the sblk that corresponds to a given zblk.
 * When the ref_count goes to zero remove the sblk from the tree and free it.
 */

static void
zulu_shadow_tree_delete(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)
{
	struct zulu_shadow_blk 	*sblk;

	ASSERT(zblk->zulu_shadow_blk != NULL);

	sblk = zblk->zulu_shadow_blk;

	TNF_PROBE_2(zulu_shadow_tree_delete, "zulu_shadow_tree", /* CSTYLED */,
	    tnf_opaque, vaddr, sblk->ivaddr,
	    tnf_opaque, ref_count, sblk->ref_count-1);

	if (--sblk->ref_count == 0) {
		if (zhat->sblk_last == sblk) {
			zhat->sblk_last = NULL;
		}
		sblk->min_addr = sblk->ivaddr + ZULU_SHADOW_BLK_RANGE;
		sblk->max_addr = sblk->ivaddr;
	} else {
		/*
		 * Update the high and low water marks for this sblk.
		 * These are estimates, because we don't know if the previous
		 * or next region are actually occupied, but we can tell
		 * whether the previous values have become invalid.
		 *
		 * In the most often applied case a segment is being
		 * unloaded, and the min_addr will be kept up to date as
		 * the zblks are deleted in order.
		 */
		uint64_t end = zblk->zulu_hat_blk_vaddr +
		    ZULU_HAT_PGSZ(zblk->zulu_hat_blk_size);

		if (zblk->zulu_hat_blk_vaddr == sblk->min_addr) {
			sblk->min_addr = end;
		}
		if (end == sblk->max_addr) {
			sblk->max_addr = zblk->zulu_hat_blk_vaddr;
		}
	}

	zblk->zulu_shadow_blk = NULL;
}

static void
zulu_shadow_tree_destroy(struct zulu_hat *zhat)
{
	struct zulu_shadow_blk *sblk;
	void	*cookie = NULL;

	while ((sblk = (struct zulu_shadow_blk *)avl_destroy_nodes(
	    &zhat->shadow_tree, &cookie)) != NULL) {
		TNF_PROBE_2(shadow_tree_destroy, "zulu_hat", /* CSTYLED */,
		    tnf_opaque, vaddr, sblk->ivaddr,
		    tnf_opaque, ref_count, sblk->ref_count);
		kmem_free(sblk, sizeof (*sblk));
	}
	avl_destroy(&zhat->shadow_tree);
}

/*
 * zulu_hat_insert_map:
 *
 * Add a zulu_hat_blk to the a zhat's mappings list.
 *
 * Several data stuctures are used
 *	tsb: for simple fast lookups by the trap handler
 *	hash table: for efficent lookups by address, range
 *	An shadow tree of 4MB ranges with mappings for unloading big regions.
 */
static void
zulu_hat_insert_map(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)
{
	int tsb_hash;

	tsb_hash = ZULU_TSB_HASH(zblk->zulu_hat_blk_vaddr,
	    zblk->zulu_hat_blk_size, zhat->zulu_tsb_size);

	TNF_PROBE_3(zulu_hat_insert_map, "zulu_hat", /* CSTYLED */,
	    tnf_opaque, zblkp, zblk,
	    tnf_opaque, vaddr, zblk->zulu_hat_blk_vaddr,
	    tnf_opaque, hash, tsb_hash);

	ASSERT(tsb_hash < zhat->zulu_tsb_size);

	zulu_shadow_tree_insert(zhat, zblk);

	/*
	 * The hash table is an array of buckets. Each bucket is the
	 * head of a linked list of mappings who's address hashess to the bucket
	 * New entries go to the head of the list.
	 */
	zblk->zulu_hash_prev = NULL;
	zblk->zulu_hash_next = ZULU_MAP_HASH_HEAD(zhat,
	    zblk->zulu_hat_blk_vaddr, zblk->zulu_hat_blk_size);
	if (zblk->zulu_hash_next) {
		zblk->zulu_hash_next->zulu_hash_prev = zblk;
	}
	ZULU_MAP_HASH_HEAD(zhat, zblk->zulu_hat_blk_vaddr,
	    zblk->zulu_hat_blk_size) = zblk;

	zulu_ctx_tsb_lock_enter(zhat);
	zhat->zulu_tsb[tsb_hash] = zblk->zulu_hat_blk_tte;
	zulu_ctx_tsb_lock_exit(zhat);
}

/*
 * remove a block from a zhat
 */
static void
zulu_hat_remove_map(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)
{
	int tsb_hash = ZULU_TSB_HASH(zblk->zulu_hat_blk_vaddr,
	    zblk->zulu_hat_blk_size, zhat->zulu_tsb_size);

	TNF_PROBE_2(zulu_hat_remove_map, "zulu_hat", /* CSTYLED */,
	    tnf_opaque, vaddr, zblk->zulu_hat_blk_vaddr,
	    tnf_opaque, hash, tsb_hash);

	ASSERT(tsb_hash < zhat->zulu_tsb_size);
	ASSERT(mutex_owned(&zhat->lock));

	zulu_shadow_tree_delete(zhat, zblk);

	/*
	 * first remove zblk from hash table
	 */
	if (zblk->zulu_hash_prev) {
		zblk->zulu_hash_prev->zulu_hash_next = zblk->zulu_hash_next;
	} else {
		ZULU_MAP_HASH_HEAD(zhat, zblk->zulu_hat_blk_vaddr,
		    zblk->zulu_hat_blk_size) = NULL;
	}
	if (zblk->zulu_hash_next) {
		zblk->zulu_hash_next->zulu_hash_prev = zblk->zulu_hash_prev;
	}
	zblk->zulu_hash_next = NULL;
	zblk->zulu_hash_prev = NULL;

	/*
	 * then remove the tsb entry
	 */
	zulu_ctx_tsb_lock_enter(zhat);
	if (zhat->zulu_tsb[tsb_hash].un.zulu_tte_addr ==
	    zblk->zulu_hat_blk_vaddr) {
		zhat->zulu_tsb[tsb_hash].zulu_tte_valid = 0;
	}
	zulu_ctx_tsb_lock_exit(zhat);
}

/*
 * look for a mapping to a given vaddr and page size
 */
static struct zulu_hat_blk *
zulu_lookup_map_bysize(struct zulu_hat *zhat, caddr_t vaddr, int page_sz)
{
	struct  	zulu_hat_blk *zblkp;
	uint64_t	ivaddr = (uint64_t)vaddr;
	int		blks_checked = 0;

	ASSERT(mutex_owned(&zhat->lock));

	for (zblkp = ZULU_MAP_HASH_HEAD(zhat, ivaddr, page_sz); zblkp != NULL;
	    zblkp = zblkp->zulu_hash_next) {
		uint64_t	size;
		uint64_t	iaddr;

		blks_checked++;

		size = ZULU_HAT_PGSZ(zblkp->zulu_hat_blk_size);
		iaddr = ZULU_VADDR((uint64_t)zblkp->zulu_hat_blk_vaddr);

		if (iaddr <= ivaddr && (iaddr + size) > ivaddr) {
			int tsb_hash;

			tsb_hash = ZULU_TSB_HASH(zblkp->zulu_hat_blk_vaddr,
			    zblkp->zulu_hat_blk_size,
			    zhat->zulu_tsb_size);
			ASSERT(tsb_hash < zhat->zulu_tsb_size);

			zulu_ctx_tsb_lock_enter(zhat);
			zhat->zulu_tsb[tsb_hash] = zblkp->zulu_hat_blk_tte;
			zulu_ctx_tsb_lock_exit(zhat);
			break;
		}

	}

	TNF_PROBE_3(zulu_hat_lookup_map_bysz, "zulu_hat", /* CSTYLED */,
	    tnf_opaque, zblkp, zblkp,
	    tnf_int, blks_checked, blks_checked,
	    tnf_int, page_sz, page_sz);

	return (zblkp);
}

/*
 * Lookup a zblk for a given virtual address.
 */
static struct zulu_hat_blk *
zulu_lookup_map(struct zulu_hat *zhat, caddr_t vaddr)
{
	struct  	zulu_hat_blk *zblkp = NULL;

	/*
	 * if the hat is using 4M pages, look first for a 4M page
	 */
	if (zhat->map4m) {
		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE4M);
		if (zblkp != NULL) {
			return (zblkp);
		}
	}
	/*
	 * Otherwise look for a 8k page
	 * Note: if base pagesize gets increased to 64K remove this test
	 */
	if (zhat->map8k) {
		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE8K);
		if (zblkp != NULL) {
			return (zblkp);
		}
	}
	/*
	 * only if the page isn't found in the sizes that match the zulu mmus
	 * look for the inefficient 64K or 512K page sizes
	 */
	if (zhat->map64k) {
		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE64K);
		if (zblkp != NULL) {
			return (zblkp);
		}
	}
	if (zhat->map512k) {
		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE512K);
	}

	return (zblkp);
}

/*
 * zulu_hat_load: Load translation for given vaddr
 */
int
zulu_hat_load(struct zulu_hat *zhat, caddr_t vaddr,
		enum seg_rw rw, int *ppg_size)
{
	faultcode_t 		as_err;
	struct zulu_hat_blk 	*zblkp;
	int			rval;
	uint64_t 		flags_pfn;
	struct zulu_tte		tte;

	TNF_PROBE_2(zulu_hat_load, "zulu_hat", /* CSTYLED */,
	    tnf_int, zulu_ctx, zhat->zulu_ctx,
	    tnf_opaque, vaddr, vaddr);

	mutex_enter(&zhat->lock);
	ASSERT(zhat->zulu_ctx >= 0);
	/*
	 * lookup in our tsb first
	 */
	zulu_ctx_tsb_lock_enter(zhat);
	flags_pfn = zulu_hat_tsb_lookup_tl0(zhat, vaddr);
	zulu_ctx_tsb_lock_exit(zhat);

	if (flags_pfn) {
		uint64_t *p = (uint64_t *)&tte;

		p++; 			/* ignore the tag */
		*p = flags_pfn;		/* load the flags */

		zuluvm_load_tte(zhat, vaddr, flags_pfn, tte.zulu_tte_perm,
		    tte.zulu_tte_size);
		if (ppg_size != NULL) {
			*ppg_size = tte.zulu_tte_size;
		}

		zulu_tsb_hit++;
		mutex_exit(&zhat->lock);
		return (0);
	}

	zulu_tsb_miss++;

	zblkp = zulu_lookup_map(zhat, vaddr);
	if (zblkp) {
		tte = zblkp->zulu_hat_blk_tte;
		tte.zulu_tte_pfn = ZULU_HAT_ADJ_PFN((&tte), vaddr);
		zuluvm_load_tte(zhat, vaddr,  tte.zulu_tte_pfn,
		    tte.zulu_tte_perm, tte.zulu_tte_size);
		if (ppg_size != NULL) {
			*ppg_size = tte.zulu_tte_size;
		}
		mutex_exit(&zhat->lock);
		return (0);
	}

	/*
	 * Set a flag indicating that we're processing a fault.
	 * See comments in zulu_hat_unload_region.
	 */
	zhat->in_fault = 1;
	mutex_exit(&zhat->lock);

	zulu_as_fault++;
	TNF_PROBE_0(calling_as_fault, "zulu_hat", /* CSTYLED */);

	as_err = as_fault((struct hat *)zhat, zhat->zulu_xhat.xhat_as,
	    (caddr_t)(ZULU_VADDR((uint64_t)vaddr) & PAGEMASK),
	    PAGESIZE, F_INVAL, rw);

	mutex_enter(&zhat->lock);
	zhat->in_fault = 0;
	if (ppg_size != NULL) {
		/*
		 * caller wants to know the page size (used by preload)
		 */
		zblkp = zulu_lookup_map(zhat, vaddr);
		if (zblkp != NULL) {
			*ppg_size = zblkp->zulu_hat_blk_size;
		} else {
			*ppg_size = -1;
		}
	}
	mutex_exit(&zhat->lock);

	TNF_PROBE_1(as_fault_returned, "zulu_hat", /* CSTYLED */,
		tnf_int, as_err, as_err);

	if (as_err != 0) {
		printf("as_fault returned %d\n", as_err);
		rval = as_err;
	} else if (zhat->freed) {
		rval = -1;
	} else {
		rval = 0;
	}

	return (rval);
}

static struct xhat *
zulu_hat_alloc(void *arg)
{
	struct zulu_hat *zhat = kmem_zalloc(sizeof (struct zulu_hat), KM_SLEEP);

	(void) arg;

	zulu_hat_ctx_alloc(zhat);

	mutex_init(&zhat->lock, NULL, MUTEX_DEFAULT, NULL);

	zhat->zulu_tsb = kmem_zalloc(ZULU_TSB_SZ, KM_SLEEP);
	zhat->zulu_tsb_size = ZULU_TSB_NUM;
	zhat->hash_tbl = kmem_zalloc(ZULU_HASH_TBL_SZ, KM_SLEEP);
	avl_create(&zhat->shadow_tree, zulu_shadow_tree_compare,
	    sizeof (zhat->shadow_tree), ZULU_SHADOW_BLK_LINK_OFFSET);
	/*
	 * The zulu hat has a few opaque data structs embedded in it.
	 * This tag makes finding the our data easier with a debugger.
	 */
	zhat->magic = 0x42;

	zhat->freed = 0;
	TNF_PROBE_1(zulu_hat_alloc, "zulu_hat", /* CSTYLED */,
		tnf_int, zulu_ctx, zhat->zulu_ctx);
	return ((struct xhat *)zhat);
}

static void
zulu_hat_free(struct xhat *xhat)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;

	TNF_PROBE_1(zulu_hat_free, "zulu_hat", /* CSTYLED */,
		tnf_int, zulu_ctx, zhat->zulu_ctx);

	zulu_shadow_tree_destroy(zhat);
	kmem_free(zhat->hash_tbl, ZULU_HASH_TBL_SZ);
	kmem_free(zhat->zulu_tsb, ZULU_TSB_SZ);
	mutex_destroy(&zhat->lock);
	kmem_free(xhat, sizeof (struct zulu_hat));
}

static void
zulu_hat_free_start(struct xhat *xhat)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;

	TNF_PROBE_1(zulu_hat_free_start, "zulu_hat", /* CSTYLED */,
		tnf_int, zulu_ctx, zhat->zulu_ctx);
	(void) xhat;
}

/*
 * zulu_hat_memload: This is the callback where the vm system gives us our
 * translations
 */
static void
zulu_do_hat_memload(struct xhat *xhat, caddr_t vaddr, struct page *page,
    uint_t attr, uint_t flags, int use_pszc)
{
	void *blk;
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
	struct zulu_hat_blk *zblk;
	pfn_t pfn;

	TNF_PROBE_4(zulu_hat_memload, "zulu_hat", /* CSTYLED */,
	    tnf_int, zulu_ctx, zhat->zulu_ctx,
	    tnf_opaque, vaddr, vaddr, tnf_opaque, attr, attr,
	    tnf_opaque, flags, flags);

	/*
	 * keep track of the highest address that this zhat has had
	 * a mapping for.
	 * We use this in unload to avoid searching for regions that
	 * we've never seen.
	 *
	 * This is particularly useful avoiding repeated searches for
	 * for the process's mappings to the zulu hardware. These mappings
	 * are explicitly unloaded at each graphics context switch..
	 *
	 * This takes advantage of the fact that the device addresses
	 * are always above than the heap where most DMA data is stored.
	 */
	if (vaddr > zhat->vaddr_max) {
		zhat->vaddr_max = vaddr;
	}

	pfn = xhat_insert_xhatblk(page, xhat, &blk);
	zblk = (struct zulu_hat_blk *)blk;
	zblk->zulu_hat_blk_vaddr = (uintptr_t)vaddr;
	zblk->zulu_hat_blk_pfn = (uint_t)pfn;
	/*
	 * The perm bit is actually in the tte which gets copied to the TSB
	 */
	zblk->zulu_hat_blk_perm = (attr & PROT_WRITE) ? 1 : 0;
	zblk->zulu_hat_blk_size = use_pszc ? page->p_szc : 0;
	zblk->zulu_hat_blk_valid = 1;

	switch (zblk->zulu_hat_blk_size) {
	case	ZULU_TTE8K:
		zhat->map8k = 1;
		break;
	case	ZULU_TTE64K:
		zhat->map64k = 1;
		break;
	case	ZULU_TTE512K:
		zhat->map512k = 1;
		break;
	case	ZULU_TTE4M:
		zhat->map4m = 1;
		break;
	default:
		panic("zulu_hat illegal page size\n");
	}

	mutex_enter(&zhat->lock);

	zulu_hat_insert_map(zhat, zblk);
	if (!zhat->freed) {
		zuluvm_load_tte(zhat, vaddr, zblk->zulu_hat_blk_pfn,
		    zblk->zulu_hat_blk_perm, zblk->zulu_hat_blk_size);
	}
	zhat->fault_ivaddr_last =
	    ZULU_VADDR((uint64_t)zblk->zulu_hat_blk_vaddr);

	mutex_exit(&zhat->lock);
}

static void
zulu_hat_memload(struct xhat *xhat, caddr_t vaddr, struct page *page,
    uint_t attr, uint_t flags)
{
	zulu_do_hat_memload(xhat, vaddr, page, attr, flags, 0);
}

static void
zulu_hat_devload(struct xhat *xhat, caddr_t vaddr, size_t size, pfn_t pfn,
	uint_t attr, int flags)
{
	struct page *pp = page_numtopp_nolock(pfn);
	(void) size;
	zulu_do_hat_memload(xhat, vaddr, pp, attr, (uint_t)flags, 1);
}

static void
zulu_hat_memload_array(struct xhat *xhat, caddr_t addr, size_t len,
    struct page **gen_pps, uint_t attr, uint_t flags)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;

	TNF_PROBE_3(zulu_hat_memload_array, "zulu_hat", /* CSTYLED */,
	    tnf_int, zulu_ctx, zhat->zulu_ctx,
	    tnf_opaque, addr, addr,
	    tnf_opaque, len, len);

	for (; len > 0; len -= ZULU_HAT_PGSZ((*gen_pps)->p_szc),
	    gen_pps += ZULU_HAT_NUM_PGS((*gen_pps)->p_szc)) {
		zulu_do_hat_memload(xhat, addr, *gen_pps, attr, flags, 1);

		addr += ZULU_HAT_PGSZ((*gen_pps)->p_szc);
	}
}

static void
free_zblks(struct zulu_hat_blk *free_list)
{
	struct zulu_hat_blk *zblkp;
	struct zulu_hat_blk *next;

	for (zblkp = free_list; zblkp != NULL; zblkp = next) {
		next = zblkp->zulu_hash_next;
		(void) xhat_delete_xhatblk((struct xhat_hme_blk *)zblkp, 0);
	}
}

static void
add_to_free_list(struct zulu_hat_blk **pfree_list, struct zulu_hat_blk *zblk)
{
	zblk->zulu_hash_next = *pfree_list;
	*pfree_list = zblk;
}

static void
zulu_hat_unload_region(struct zulu_hat *zhat, uint64_t ivaddr, size_t size,
		struct zulu_shadow_blk *sblk, struct zulu_hat_blk **pfree_list)
{
	uint64_t	end = ivaddr + size;
	int		found = 0;

	TNF_PROBE_2(zulu_hat_unload_region, "zulu_hat", /* CSTYLED */,
		tnf_opaque, vaddr, ivaddr, tnf_opaque, size, size);

	/*
	 * check address against the low and highwater marks for mappings
	 * in this sblk
	 */
	if (ivaddr < sblk->min_addr) {
		ivaddr = sblk->min_addr;
		TNF_PROBE_1(zulu_hat_unload_skip, "zulu_hat", /* CSTYLED */,
			tnf_opaque, ivaddr, ivaddr);
	}
	if (end > sblk->max_addr) {
		end = sblk->max_addr;
		TNF_PROBE_1(zulu_hat_unload_reg_skip, "zulu_hat", /* CSTYLED */,
			tnf_opaque, end, end);
	}
	/*
	 * REMIND: It's not safe to touch the sblk after we enter this loop
	 * because it may get deleted.
	 */

	while (ivaddr < end) {
		uint64_t iaddr;
		size_t  pg_sz;
		struct zulu_hat_blk *zblkp;

		zblkp = zulu_lookup_map(zhat, (caddr_t)ivaddr);
		if (zblkp == NULL) {
			ivaddr += PAGESIZE;
			continue;
		}

		iaddr = ZULU_VADDR((uint64_t)zblkp->zulu_hat_blk_vaddr);
		pg_sz = ZULU_HAT_PGSZ(zblkp->zulu_hat_blk_size);

		found++;

		zulu_hat_remove_map(zhat, zblkp);
		/*
		 * skip demap page if as_free has already been entered
		 * zuluvm demapped the context already
		 */
		if (!zhat->freed) {
			if ((zhat->in_fault) &&
			    (iaddr == zhat->fault_ivaddr_last)) {
				/*
				 * We're being called from within as_fault to
				 * unload the last translation we loaded.
				 *
				 * This is probably due to watchpoint handling.
				 * Delay the demap for a millisecond
				 * to allow zulu to make some progress.
				 */
				drv_usecwait(1000);
				zhat->fault_ivaddr_last = 0;
			}
			zulu_hat_demap_page(zhat, (caddr_t)iaddr,
			    zblkp->zulu_hat_blk_size);
		}

		add_to_free_list(pfree_list, zblkp);

		if ((iaddr + pg_sz) >= end) {
			break;
		}

		ivaddr += pg_sz;
	}
	TNF_PROBE_1(zulu_hat_unload_region_done, "zulu_hat", /* CSTYLED */,
		tnf_opaque, found, found);
}

static void
zulu_hat_unload(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
	uint64_t	ivaddr;
	uint64_t	end;
	int		found = 0;
	struct zulu_hat_blk *free_list = NULL;

	(void) flags;

	TNF_PROBE_4(zulu_hat_unload, "zulu_hat", /* CSTYLED */,
	    tnf_int, zulu_ctx, zhat->zulu_ctx,
	    tnf_opaque, vaddr, vaddr,
	    tnf_opaque, vaddr_max, zhat->vaddr_max,
	    tnf_opaque, size, size);

	mutex_enter(&zhat->lock);

	/*
	 * The following test prevents us from searching for the user's
	 * mappings to the zulu device registers. Those mappings get unloaded
	 * every time a graphics context switch away from a given context
	 * occurs.
	 *
	 * Since the heap is located at smaller virtual addresses than the
	 * registers, this simple test avoids quite a bit of useless work.
	 */
	if (vaddr > zhat->vaddr_max) {
		/*
		 * all existing mappings have lower addresses than vaddr
		 * no need to search further.
		 */
		mutex_exit(&zhat->lock);
		return;
	}

	ivaddr = (uint64_t)vaddr;
	end = ivaddr + size;

	do {
		struct zulu_shadow_blk *sblk;

		sblk = zulu_shadow_tree_lookup(zhat, ivaddr, NULL);
		if (sblk != NULL) {
			uint64_t 	sblk_end;
			size_t		region_size;

			found++;

			sblk_end = (ivaddr + ZULU_SHADOW_BLK_RANGE) &
			    ZULU_SHADOW_BLK_MASK;

			if (sblk_end < end) {
				region_size = sblk_end - ivaddr;
			} else {
				region_size = end - ivaddr;
			}
			zulu_hat_unload_region(zhat, ivaddr, region_size, sblk,
			    &free_list);

		}
		ivaddr += ZULU_SHADOW_BLK_RANGE;
	} while (ivaddr < end);

	mutex_exit(&zhat->lock);

	free_zblks(free_list);

	TNF_PROBE_1(zulu_hat_unload_done, "zulu_hat", /* CSTYLED */,
		tnf_int, found, found);
}

static void
zulu_hat_unload_callback(struct xhat *xhat, caddr_t vaddr, size_t size,
	uint_t flags, hat_callback_t *pcb)
{
	(void) size;
	(void) pcb;
	zulu_hat_unload(xhat, vaddr, size, flags);
}


/*
 * unload one page
 */
static int
zulu_hat_pageunload(struct xhat *xhat, struct page *pp, uint_t flags,
    void *xblk)
{
	struct zulu_hat_blk *zblk = (struct zulu_hat_blk *)xblk;
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
	int	do_delete;

	(void) pp;
	(void) flags;

	TNF_PROBE_3(zulu_hat_pageunload, "zulu_hat", /* CSTYLED */,
	    tnf_int, zulu_ctx, zhat->zulu_ctx,
	    tnf_opaque, vaddr, zblk->zulu_hat_blk_vaddr,
	    tnf_int, pg_size, zblk->zulu_hat_blk_size);

	mutex_enter(&zhat->lock);
	if (zblk->zulu_shadow_blk != NULL) {

		do_delete = 1;

		zulu_hat_remove_map(zhat, zblk);

		/*
		 * now that the entry is removed from the TSB, remove the
		 * translation from the zulu hardware.
		 *
		 * Skip the demap if this as is in the process of being freed.
		 * The zuluvm as callback has demapped the whole context.
		 */
		if (!zhat->freed) {
			zulu_hat_demap_page(zhat,
			    (caddr_t)(uintptr_t)(zblk->zulu_hat_blk_page <<
			    ZULU_HAT_BP_SHIFT),
			    zblk->zulu_hat_blk_size);
		}
	} else {
		/*
		 * This block has already been removed from the zulu_hat,
		 * it's on a free list waiting for our thread to release
		 * a mutex so it can be freed
		 */
		do_delete = 0;

		TNF_PROBE_0(zulu_hat_pageunload_skip, "zulu_hat",
		    /* CSTYLED */);
	}
	mutex_exit(&zhat->lock);

	if (do_delete) {
		(void) xhat_delete_xhatblk(xblk, 1);
	}

	return (0);
}

static void
zulu_hat_swapout(struct xhat *xhat)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
	struct zulu_hat_blk *zblk;
	struct zulu_hat_blk *free_list = NULL;
	int	i;
	int	nblks = 0;

	TNF_PROBE_1(zulu_hat_swapout, "zulu_hat", /* CSTYLED */,
		tnf_int, zulu_ctx, zhat->zulu_ctx);

	mutex_enter(&zhat->lock);

	/*
	 * real swapout calls are rare so we don't do anything in
	 * particular to optimize them.
	 *
	 * Just loop over all buckets in the hash table and free each
	 * zblk.
	 */
	for (i = 0; i < ZULU_HASH_TBL_NUM; i++) {
		struct zulu_hat_blk *next;
		for (zblk = zhat->hash_tbl[i]; zblk != NULL; zblk = next) {
			next = zblk->zulu_hash_next;
			zulu_hat_remove_map(zhat, zblk);
			add_to_free_list(&free_list, zblk);
			nblks++;
		}
	}

	/*
	 * remove all mappings for this context from zulu hardware.
	 */
	zulu_hat_demap_ctx(zhat->zdev, zhat->zulu_ctx);

	mutex_exit(&zhat->lock);

	free_zblks(free_list);

	TNF_PROBE_1(zulu_hat_swapout_done, "zulu_hat", /* CSTYLED */,
		tnf_int, nblks, nblks);
}


static void
zulu_hat_unshare(struct xhat *xhat, caddr_t vaddr, size_t size)
{
	TNF_PROBE_0(zulu_hat_unshare, "zulu_hat", /* CSTYLED */);

	zulu_hat_unload(xhat, vaddr, size, 0);
}

/*
 * Functions to manage changes in protections for mappings.
 *
 * These are rarely called in normal operation so for now just unload
 * the region.
 * If the mapping is still needed, it will fault in later with the new
 * attrributes.
 */
typedef enum {
	ZULU_HAT_CHGATTR,
	ZULU_HAT_SETATTR,
	ZULU_HAT_CLRATTR
} zulu_hat_prot_op;

static void
zulu_hat_update_attr(struct xhat *xhat, caddr_t vaddr, size_t size,
	uint_t flags, zulu_hat_prot_op op)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;

	TNF_PROBE_5(zulu_hat_changeprot, "zulu_hat", /* CSTYLED */,
	    tnf_int, ctx, zhat->zulu_ctx,
	    tnf_opaque, vaddr, vaddr, tnf_opaque, size, size,
	    tnf_uint, flags, flags, tnf_int, op, op);

	zulu_hat_unload(xhat, vaddr, size, 0);
}

static void
zulu_hat_chgprot(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
#ifdef DEBUG
	printf("zulu_hat_chgprot: ctx: %d addr: %lx, size: %lx flags: %x\n",
	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);
#endif
	zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_CHGATTR);
}


static void
zulu_hat_setattr(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
#ifdef DEBUG
	printf("zulu_hat_setattr: ctx: %d addr: %lx, size: %lx flags: %x\n",
	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);
#endif
	zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_SETATTR);
}

static void
zulu_hat_clrattr(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
#ifdef DEBUG
	printf("zulu_hat_clrattr: ctx: %d addr: %lx, size: %lx flags: %x\n",
	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);
#endif
	zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_CLRATTR);
}

static void
zulu_hat_chgattr(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags)
{
	struct zulu_hat *zhat = (struct zulu_hat *)xhat;
	TNF_PROBE_3(zulu_hat_chgattr, "zulu_hat", /* CSTYLED */,
	    tnf_int, ctx, zhat->zulu_ctx,
	    tnf_opaque, vaddr, vaddr,
	    tnf_opaque, flags, flags);
#ifdef DEBUG
	printf("zulu_hat_chgattr: ctx: %d addr: %lx, size: %lx flags: %x\n",
	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);
#endif
	zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_CHGATTR);
}


struct xhat_ops zulu_hat_ops = {
	zulu_hat_alloc,		/* xhat_alloc */
	zulu_hat_free,		/* xhat_free */
	zulu_hat_free_start,	/* xhat_free_start */
	NULL,			/* xhat_free_end */
	NULL,			/* xhat_dup */
	NULL,			/* xhat_swapin */
	zulu_hat_swapout,	/* xhat_swapout */
	zulu_hat_memload,	/* xhat_memload */
	zulu_hat_memload_array,	/* xhat_memload_array */
	zulu_hat_devload,	/* xhat_devload */
	zulu_hat_unload,	/* xhat_unload */
	zulu_hat_unload_callback, /* xhat_unload_callback */
	zulu_hat_setattr,	/* xhat_setattr */
	zulu_hat_clrattr,	/* xhat_clrattr */
	zulu_hat_chgattr,	/* xhat_chgattr */
	zulu_hat_unshare,	/* xhat_unshare */
	zulu_hat_chgprot,	/* xhat_chgprot */
	zulu_hat_pageunload,	/* xhat_pageunload */
};

xblk_cache_t zulu_xblk_cache = {
    NULL,
    NULL,
    NULL,
    xhat_xblkcache_reclaim
};

xhat_provider_t zulu_hat_provider = {
	XHAT_PROVIDER_VERSION,
	0,
	NULL,
	NULL,
	"zulu_hat_provider",
	&zulu_xblk_cache,
	&zulu_hat_ops,
	sizeof (struct zulu_hat_blk) + sizeof (struct xhat_hme_blk)
};

/*
 * The following functions are the entry points that zuluvm uses.
 */

/*
 * initialize this module. Called from zuluvm's _init function
 */
int
zulu_hat_init()
{
	int 	c;
	int	rval;
	mutex_init(&zulu_ctx_lock, NULL, MUTEX_DEFAULT, NULL);

	for (c = 0; c < ZULU_HAT_MAX_CTX; c++) {
		ZULU_CTX_LOCK_INIT(c);
	}
	zulu_ctx_search_start = 0;
	rval = xhat_provider_register(&zulu_hat_provider);
	if (rval != 0) {
		mutex_destroy(&zulu_ctx_lock);
	}
	return (rval);
}

/*
 * un-initialize this module. Called from zuluvm's _fini function
 */
int
zulu_hat_destroy()
{
	if (xhat_provider_unregister(&zulu_hat_provider) != 0) {
		return (-1);
	}
	mutex_destroy(&zulu_ctx_lock);
	return (0);
}

int
zulu_hat_attach(void *arg)
{
	(void) arg;
	return (0);
}

int
zulu_hat_detach(void *arg)
{
	(void) arg;
	return (0);
}

/*
 * create a zulu hat for this address space.
 */
struct zulu_hat *
zulu_hat_proc_attach(struct as *as, void *zdev)
{
	struct zulu_hat *zhat;
	int		xhat_rval;

	xhat_rval = xhat_attach_xhat(&zulu_hat_provider, as,
	    (struct xhat **)&zhat, NULL);
	if ((xhat_rval == 0) && (zhat != NULL)) {
		mutex_enter(&zhat->lock);
		ZULU_HAT2AS(zhat) = as;
		zhat->zdev = zdev;
		mutex_exit(&zhat->lock);
	}

	TNF_PROBE_3(zulu_hat_proc_attach, "zulu_hat", /* CSTYLED */,
	    tnf_int, xhat_rval, xhat_rval, tnf_opaque, as, as,
	    tnf_opaque, zhat, zhat);

	return (zhat);
}

void
zulu_hat_proc_detach(struct zulu_hat *zhat)
{
	struct  as *as = ZULU_HAT2AS(zhat);

	zulu_hat_ctx_free(zhat);

	(void) xhat_detach_xhat(&zulu_hat_provider, ZULU_HAT2AS(zhat));

	TNF_PROBE_1(zulu_hat_proc_detach, "zulu_hat", /* CSTYLED */,
			tnf_opaque, as, as);
}

/*
 * zulu_hat_terminate
 *
 * Disables any further TLB miss processing for this hat
 * Called by zuluvm's as_free callback. The primary purpose of this
 * function is to cause any pending zulu DMA to abort quickly.
 */
void
zulu_hat_terminate(struct zulu_hat *zhat)
{
	int	ctx = zhat->zulu_ctx;

	TNF_PROBE_1(zulu_hat_terminate, "zulu_hat", /* CSTYLED */,
		tnf_int, ctx, ctx);

	mutex_enter(&zhat->lock);

	zhat->freed = 1;

	zulu_ctx_tsb_lock_enter(zhat);
	/*
	 * zap the tsb
	 */
	bzero(zhat->zulu_tsb, ZULU_TSB_SZ);
	zulu_ctx_tsb_lock_exit(zhat);

	zulu_hat_demap_ctx(zhat->zdev, zhat->zulu_ctx);

	mutex_exit(&zhat->lock);

	TNF_PROBE_0(zulu_hat_terminate_done, "zulu_hat", /* CSTYLED */);
}