110 zulu_hat_demap_page(struct zulu_hat *zhat, caddr_t vaddr, int size)  in zulu_hat_demap_page()  argument
112 	if (zhat->zulu_ctx < 0) {  in zulu_hat_demap_page()
116 	zuluvm_demap_page(zhat->zdev, NULL, zhat->zulu_ctx, vaddr, size);  in zulu_hat_demap_page()
143 		struct zulu_hat *zhat = ZULU_CTX_GET_HAT(ctx);  in zulu_hat_steal_ctx()  local
148 		ASSERT(zhat != NULL);  in zulu_hat_steal_ctx()
152 		    tnf_long, last_used, zhat->last_used,  in zulu_hat_steal_ctx()
155 		if (zhat->last_used <  delta) {  in zulu_hat_steal_ctx()
156 			zhat_oldest = zhat;  in zulu_hat_steal_ctx()
157 			delta  = zhat->last_used;  in zulu_hat_steal_ctx()
188 zulu_hat_ctx_alloc(struct zulu_hat *zhat)  in zulu_hat_ctx_alloc()  argument
207 	mutex_enter(&zhat->lock);  in zulu_hat_ctx_alloc()
209 	ZULU_CTX_SET_HAT(ctx, zhat);  in zulu_hat_ctx_alloc()
210 	zhat->zulu_ctx = ctx;  in zulu_hat_ctx_alloc()
212 	mutex_exit(&zhat->lock);  in zulu_hat_ctx_alloc()
217 		tnf_opaque, zhat, zhat, tnf_int, ctx, ctx);  in zulu_hat_ctx_alloc()
226 zulu_hat_validate_ctx(struct zulu_hat *zhat)  in zulu_hat_validate_ctx()  argument
228 	if (zhat->zulu_ctx < 0)  {  in zulu_hat_validate_ctx()
229 		zulu_hat_ctx_alloc(zhat);  in zulu_hat_validate_ctx()
231 	zhat->last_used = gethrtime();  in zulu_hat_validate_ctx()
236 zulu_hat_ctx_free(struct zulu_hat *zhat)  in zulu_hat_ctx_free()  argument
239 		tnf_int, ctx, zhat->zulu_ctx);  in zulu_hat_ctx_free()
243 	mutex_enter(&zhat->lock);  in zulu_hat_ctx_free()
244 	if (zhat->zulu_ctx >= 0) {  in zulu_hat_ctx_free()
245 		ZULU_CTX_SET_HAT(zhat->zulu_ctx, NULL);  in zulu_hat_ctx_free()
247 		if (zulu_ctx_search_start > zhat->zulu_ctx) {  in zulu_hat_ctx_free()
248 			zulu_ctx_search_start = zhat->zulu_ctx;  in zulu_hat_ctx_free()
251 	mutex_exit(&zhat->lock);  in zulu_hat_ctx_free()
262 zulu_ctx_tsb_lock_enter(struct zulu_hat *zhat)  in zulu_ctx_tsb_lock_enter()  argument
267 	ASSERT(mutex_owned(&zhat->lock));  in zulu_ctx_tsb_lock_enter()
269 	if (zhat->zulu_ctx < 0) {  in zulu_ctx_tsb_lock_enter()
272 	plck = (uint64_t *)&zulu_ctx_tab[zhat->zulu_ctx];  in zulu_ctx_tsb_lock_enter()
294 zulu_ctx_tsb_lock_exit(struct zulu_hat *zhat)  in zulu_ctx_tsb_lock_exit()  argument
297 	int		zulu_ctx = zhat->zulu_ctx;  in zulu_ctx_tsb_lock_exit()
349 zulu_shadow_tree_lookup(struct zulu_hat *zhat, uint64_t ivaddr,  in zulu_shadow_tree_lookup()  argument
362 	if ((zhat->sblk_last != NULL) &&  in zulu_shadow_tree_lookup()
363 	    (proto.ivaddr == zhat->sblk_last->ivaddr)) {  in zulu_shadow_tree_lookup()
364 		sblk = zhat->sblk_last;  in zulu_shadow_tree_lookup()
366 		sblk = (struct zulu_shadow_blk *)avl_find(&zhat->shadow_tree,  in zulu_shadow_tree_lookup()
368 		zhat->sblk_last = sblk;  in zulu_shadow_tree_lookup()
383 zulu_shadow_tree_insert(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)  in zulu_shadow_tree_insert()  argument
394 	sblk = zulu_shadow_tree_lookup(zhat, ivaddr, &where);  in zulu_shadow_tree_insert()
416 		zhat->sblk_last = sblk;  in zulu_shadow_tree_insert()
418 		avl_insert(&zhat->shadow_tree, sblk, where);  in zulu_shadow_tree_insert()
432 zulu_shadow_tree_delete(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)  in zulu_shadow_tree_delete()  argument
445 		if (zhat->sblk_last == sblk) {  in zulu_shadow_tree_delete()
446 			zhat->sblk_last = NULL;  in zulu_shadow_tree_delete()
476 zulu_shadow_tree_destroy(struct zulu_hat *zhat)  in zulu_shadow_tree_destroy()  argument
482 	    &zhat->shadow_tree, &cookie)) != NULL) {  in zulu_shadow_tree_destroy()
488 	avl_destroy(&zhat->shadow_tree);  in zulu_shadow_tree_destroy()
502 zulu_hat_insert_map(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)  in zulu_hat_insert_map()  argument
507 	    zblk->zulu_hat_blk_size, zhat->zulu_tsb_size);  in zulu_hat_insert_map()
514 	ASSERT(tsb_hash < zhat->zulu_tsb_size);  in zulu_hat_insert_map()
516 	zulu_shadow_tree_insert(zhat, zblk);  in zulu_hat_insert_map()
524 	zblk->zulu_hash_next = ZULU_MAP_HASH_HEAD(zhat,  in zulu_hat_insert_map()
529 	ZULU_MAP_HASH_HEAD(zhat, zblk->zulu_hat_blk_vaddr,  in zulu_hat_insert_map()
532 	zulu_ctx_tsb_lock_enter(zhat);  in zulu_hat_insert_map()
533 	zhat->zulu_tsb[tsb_hash] = zblk->zulu_hat_blk_tte;  in zulu_hat_insert_map()
534 	zulu_ctx_tsb_lock_exit(zhat);  in zulu_hat_insert_map()
541 zulu_hat_remove_map(struct zulu_hat *zhat, struct zulu_hat_blk *zblk)  in zulu_hat_remove_map()  argument
544 	    zblk->zulu_hat_blk_size, zhat->zulu_tsb_size);  in zulu_hat_remove_map()
550 	ASSERT(tsb_hash < zhat->zulu_tsb_size);  in zulu_hat_remove_map()
551 	ASSERT(mutex_owned(&zhat->lock));  in zulu_hat_remove_map()
553 	zulu_shadow_tree_delete(zhat, zblk);  in zulu_hat_remove_map()
561 		ZULU_MAP_HASH_HEAD(zhat, zblk->zulu_hat_blk_vaddr,  in zulu_hat_remove_map()
573 	zulu_ctx_tsb_lock_enter(zhat);  in zulu_hat_remove_map()
574 	if (zhat->zulu_tsb[tsb_hash].un.zulu_tte_addr ==  in zulu_hat_remove_map()
576 		zhat->zulu_tsb[tsb_hash].zulu_tte_valid = 0;  in zulu_hat_remove_map()
578 	zulu_ctx_tsb_lock_exit(zhat);  in zulu_hat_remove_map()
585 zulu_lookup_map_bysize(struct zulu_hat *zhat, caddr_t vaddr, int page_sz)  in zulu_lookup_map_bysize()  argument
591 	ASSERT(mutex_owned(&zhat->lock));  in zulu_lookup_map_bysize()
593 	for (zblkp = ZULU_MAP_HASH_HEAD(zhat, ivaddr, page_sz); zblkp != NULL;  in zulu_lookup_map_bysize()
608 			    zhat->zulu_tsb_size);  in zulu_lookup_map_bysize()
609 			ASSERT(tsb_hash < zhat->zulu_tsb_size);  in zulu_lookup_map_bysize()
611 			zulu_ctx_tsb_lock_enter(zhat);  in zulu_lookup_map_bysize()
612 			zhat->zulu_tsb[tsb_hash] = zblkp->zulu_hat_blk_tte;  in zulu_lookup_map_bysize()
613 			zulu_ctx_tsb_lock_exit(zhat);  in zulu_lookup_map_bysize()
631 zulu_lookup_map(struct zulu_hat *zhat, caddr_t vaddr)  in zulu_lookup_map()  argument
638 	if (zhat->map4m) {  in zulu_lookup_map()
639 		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE4M);  in zulu_lookup_map()
648 	if (zhat->map8k) {  in zulu_lookup_map()
649 		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE8K);  in zulu_lookup_map()
658 	if (zhat->map64k) {  in zulu_lookup_map()
659 		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE64K);  in zulu_lookup_map()
664 	if (zhat->map512k) {  in zulu_lookup_map()
665 		zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE512K);  in zulu_lookup_map()
675 zulu_hat_load(struct zulu_hat *zhat, caddr_t vaddr,  in zulu_hat_load()  argument
685 	    tnf_int, zulu_ctx, zhat->zulu_ctx,  in zulu_hat_load()
688 	mutex_enter(&zhat->lock);  in zulu_hat_load()
689 	ASSERT(zhat->zulu_ctx >= 0);  in zulu_hat_load()
693 	zulu_ctx_tsb_lock_enter(zhat);  in zulu_hat_load()
694 	flags_pfn = zulu_hat_tsb_lookup_tl0(zhat, vaddr);  in zulu_hat_load()
695 	zulu_ctx_tsb_lock_exit(zhat);  in zulu_hat_load()
703 		zuluvm_load_tte(zhat, vaddr, flags_pfn, tte.zulu_tte_perm,  in zulu_hat_load()
710 		mutex_exit(&zhat->lock);  in zulu_hat_load()
716 	zblkp = zulu_lookup_map(zhat, vaddr);  in zulu_hat_load()
720 		zuluvm_load_tte(zhat, vaddr,  tte.zulu_tte_pfn,  in zulu_hat_load()
725 		mutex_exit(&zhat->lock);  in zulu_hat_load()
733 	zhat->in_fault = 1;  in zulu_hat_load()
734 	mutex_exit(&zhat->lock);  in zulu_hat_load()
739 	as_err = as_fault((struct hat *)zhat, zhat->zulu_xhat.xhat_as,  in zulu_hat_load()
743 	mutex_enter(&zhat->lock);  in zulu_hat_load()
744 	zhat->in_fault = 0;  in zulu_hat_load()
749 		zblkp = zulu_lookup_map(zhat, vaddr);  in zulu_hat_load()
756 	mutex_exit(&zhat->lock);  in zulu_hat_load()
764 	} else if (zhat->freed) {  in zulu_hat_load()
776 	struct zulu_hat *zhat = kmem_zalloc(sizeof (struct zulu_hat), KM_SLEEP);  in zulu_hat_alloc()  local
780 	zulu_hat_ctx_alloc(zhat);  in zulu_hat_alloc()
782 	mutex_init(&zhat->lock, NULL, MUTEX_DEFAULT, NULL);  in zulu_hat_alloc()
784 	zhat->zulu_tsb = kmem_zalloc(ZULU_TSB_SZ, KM_SLEEP);  in zulu_hat_alloc()
785 	zhat->zulu_tsb_size = ZULU_TSB_NUM;  in zulu_hat_alloc()
786 	zhat->hash_tbl = kmem_zalloc(ZULU_HASH_TBL_SZ, KM_SLEEP);  in zulu_hat_alloc()
787 	avl_create(&zhat->shadow_tree, zulu_shadow_tree_compare,  in zulu_hat_alloc()
788 	    sizeof (zhat->shadow_tree), ZULU_SHADOW_BLK_LINK_OFFSET);  in zulu_hat_alloc()
793 	zhat->magic = 0x42;  in zulu_hat_alloc()
795 	zhat->freed = 0;  in zulu_hat_alloc()
797 		tnf_int, zulu_ctx, zhat->zulu_ctx);  in zulu_hat_alloc()
798 	return ((struct xhat *)zhat);  in zulu_hat_alloc()
804 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_free()  local
807 		tnf_int, zulu_ctx, zhat->zulu_ctx);  in zulu_hat_free()
809 	zulu_shadow_tree_destroy(zhat);  in zulu_hat_free()
810 	kmem_free(zhat->hash_tbl, ZULU_HASH_TBL_SZ);  in zulu_hat_free()
811 	kmem_free(zhat->zulu_tsb, ZULU_TSB_SZ);  in zulu_hat_free()
812 	mutex_destroy(&zhat->lock);  in zulu_hat_free()
819 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_free_start()  local
822 		tnf_int, zulu_ctx, zhat->zulu_ctx);  in zulu_hat_free_start()
835 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_do_hat_memload()  local
840 	    tnf_int, zulu_ctx, zhat->zulu_ctx,  in zulu_do_hat_memload()
857 	if (vaddr > zhat->vaddr_max) {  in zulu_do_hat_memload()
858 		zhat->vaddr_max = vaddr;  in zulu_do_hat_memload()
874 		zhat->map8k = 1;  in zulu_do_hat_memload()
877 		zhat->map64k = 1;  in zulu_do_hat_memload()
880 		zhat->map512k = 1;  in zulu_do_hat_memload()
883 		zhat->map4m = 1;  in zulu_do_hat_memload()
889 	mutex_enter(&zhat->lock);  in zulu_do_hat_memload()
891 	zulu_hat_insert_map(zhat, zblk);  in zulu_do_hat_memload()
892 	if (!zhat->freed) {  in zulu_do_hat_memload()
893 		zuluvm_load_tte(zhat, vaddr, zblk->zulu_hat_blk_pfn,  in zulu_do_hat_memload()
896 	zhat->fault_ivaddr_last =  in zulu_do_hat_memload()
899 	mutex_exit(&zhat->lock);  in zulu_do_hat_memload()
922 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_memload_array()  local
925 	    tnf_int, zulu_ctx, zhat->zulu_ctx,  in zulu_hat_memload_array()
957 zulu_hat_unload_region(struct zulu_hat *zhat, uint64_t ivaddr, size_t size,  in zulu_hat_unload_region()  argument
990 		zblkp = zulu_lookup_map(zhat, (caddr_t)ivaddr);  in zulu_hat_unload_region()
1001 		zulu_hat_remove_map(zhat, zblkp);  in zulu_hat_unload_region()
1006 		if (!zhat->freed) {  in zulu_hat_unload_region()
1007 			if ((zhat->in_fault) &&  in zulu_hat_unload_region()
1008 			    (iaddr == zhat->fault_ivaddr_last)) {  in zulu_hat_unload_region()
1018 				zhat->fault_ivaddr_last = 0;  in zulu_hat_unload_region()
1020 			zulu_hat_demap_page(zhat, (caddr_t)iaddr,  in zulu_hat_unload_region()
1039 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_unload()  local
1048 	    tnf_int, zulu_ctx, zhat->zulu_ctx,  in zulu_hat_unload()
1050 	    tnf_opaque, vaddr_max, zhat->vaddr_max,  in zulu_hat_unload()
1053 	mutex_enter(&zhat->lock);  in zulu_hat_unload()
1064 	if (vaddr > zhat->vaddr_max) {  in zulu_hat_unload()
1069 		mutex_exit(&zhat->lock);  in zulu_hat_unload()
1079 		sblk = zulu_shadow_tree_lookup(zhat, ivaddr, NULL);  in zulu_hat_unload()
1094 			zulu_hat_unload_region(zhat, ivaddr, region_size, sblk,  in zulu_hat_unload()
1101 	mutex_exit(&zhat->lock);  in zulu_hat_unload()
1127 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_pageunload()  local
1134 	    tnf_int, zulu_ctx, zhat->zulu_ctx,  in zulu_hat_pageunload()
1138 	mutex_enter(&zhat->lock);  in zulu_hat_pageunload()
1143 		zulu_hat_remove_map(zhat, zblk);  in zulu_hat_pageunload()
1152 		if (!zhat->freed) {  in zulu_hat_pageunload()
1153 			zulu_hat_demap_page(zhat,  in zulu_hat_pageunload()
1169 	mutex_exit(&zhat->lock);  in zulu_hat_pageunload()
1181 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_swapout()  local
1188 		tnf_int, zulu_ctx, zhat->zulu_ctx);  in zulu_hat_swapout()
1190 	mutex_enter(&zhat->lock);  in zulu_hat_swapout()
1201 		for (zblk = zhat->hash_tbl[i]; zblk != NULL; zblk = next) {  in zulu_hat_swapout()
1203 			zulu_hat_remove_map(zhat, zblk);  in zulu_hat_swapout()
1212 	zulu_hat_demap_ctx(zhat->zdev, zhat->zulu_ctx);  in zulu_hat_swapout()
1214 	mutex_exit(&zhat->lock);  in zulu_hat_swapout()
1249 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_update_attr()  local
1252 	    tnf_int, ctx, zhat->zulu_ctx,  in zulu_hat_update_attr()
1262 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_chgprot()  local
1265 	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);  in zulu_hat_chgprot()
1274 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_setattr()  local
1277 	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);  in zulu_hat_setattr()
1285 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_clrattr()  local
1288 	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);  in zulu_hat_clrattr()
1296 	struct zulu_hat *zhat = (struct zulu_hat *)xhat;  in zulu_hat_chgattr()  local
1298 	    tnf_int, ctx, zhat->zulu_ctx,  in zulu_hat_chgattr()
1303 	    zhat->zulu_ctx, (uint64_t)vaddr, size, flags);  in zulu_hat_chgattr()
1406 	struct zulu_hat *zhat;  in zulu_hat_proc_attach()  local
1410 	    (struct xhat **)&zhat, NULL);  in zulu_hat_proc_attach()
1411 	if ((xhat_rval == 0) && (zhat != NULL)) {  in zulu_hat_proc_attach()
1412 		mutex_enter(&zhat->lock);  in zulu_hat_proc_attach()
1413 		ZULU_HAT2AS(zhat) = as;  in zulu_hat_proc_attach()
1414 		zhat->zdev = zdev;  in zulu_hat_proc_attach()
1415 		mutex_exit(&zhat->lock);  in zulu_hat_proc_attach()
1420 	    tnf_opaque, zhat, zhat);  in zulu_hat_proc_attach()
1422 	return (zhat);  in zulu_hat_proc_attach()
1426 zulu_hat_proc_detach(struct zulu_hat *zhat)  in zulu_hat_proc_detach()  argument
1428 	struct  as *as = ZULU_HAT2AS(zhat);  in zulu_hat_proc_detach()
1430 	zulu_hat_ctx_free(zhat);  in zulu_hat_proc_detach()
1432 	(void) xhat_detach_xhat(&zulu_hat_provider, ZULU_HAT2AS(zhat));  in zulu_hat_proc_detach()
1446 zulu_hat_terminate(struct zulu_hat *zhat)  in zulu_hat_terminate()  argument
1448 	int	ctx = zhat->zulu_ctx;  in zulu_hat_terminate()
1453 	mutex_enter(&zhat->lock);  in zulu_hat_terminate()
1455 	zhat->freed = 1;  in zulu_hat_terminate()
1457 	zulu_ctx_tsb_lock_enter(zhat);  in zulu_hat_terminate()
1461 	bzero(zhat->zulu_tsb, ZULU_TSB_SZ);  in zulu_hat_terminate()
1462 	zulu_ctx_tsb_lock_exit(zhat);  in zulu_hat_terminate()
1464 	zulu_hat_demap_ctx(zhat->zdev, zhat->zulu_ctx);  in zulu_hat_terminate()
1466 	mutex_exit(&zhat->lock);  in zulu_hat_terminate()