436 htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold,  in htable_steal_active()  argument
445 	h = h_start = h_seed++ % hat->hat_num_hash;  in htable_steal_active()
449 		for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {  in htable_steal_active()
511 				ASSERT(hat->hat_ht_hash[h] == ht);  in htable_steal_active()
512 				hat->hat_ht_hash[h] = ht->ht_next;  in htable_steal_active()
529 		if (++h == hat->hat_num_hash)  in htable_steal_active()
538 move_victim(hat_t *hat)  in move_victim()  argument
543 	if (hat->hat_prev)  in move_victim()
544 		hat->hat_prev->hat_next = hat->hat_next;  in move_victim()
546 		kas.a_hat->hat_next = hat->hat_next;  in move_victim()
548 	if (hat->hat_next)  in move_victim()
549 		hat->hat_next->hat_prev = hat->hat_prev;  in move_victim()
551 		kas.a_hat->hat_prev = hat->hat_prev;  in move_victim()
553 	hat->hat_next = NULL;  in move_victim()
554 	hat->hat_prev = kas.a_hat->hat_prev;  in move_victim()
555 	if (hat->hat_prev)  in move_victim()
556 		hat->hat_prev->hat_next = hat;  in move_victim()
558 		kas.a_hat->hat_next = hat;  in move_victim()
560 	kas.a_hat->hat_prev = hat;  in move_victim()
570 	hat_t		*hat = kas.a_hat;	/* list starts with khat */  in htable_steal()  local
611 		hat = kas.a_hat->hat_next;  in htable_steal()
628 			while (hat != NULL && (hat->hat_flags &  in htable_steal()
631 				hat = hat->hat_next;  in htable_steal()
634 			if (hat == NULL)  in htable_steal()
641 			hat->hat_flags |= HAT_VICTIM;  in htable_steal()
647 			hat_enter(hat);  in htable_steal()
648 			while ((ht = hat->hat_ht_cached) != NULL &&  in htable_steal()
650 				hat->hat_ht_cached = ht->ht_next;  in htable_steal()
655 			hat_exit(hat);  in htable_steal()
661 				htable_steal_active(hat, cnt, threshold,  in htable_steal()
673 				ASSERT(ht->ht_hat == hat);  in htable_steal()
677 					ptable_free(hat->hat_user_ptable);  in htable_steal()
678 					hat->hat_user_ptable = PFN_INVALID;  in htable_steal()
698 				    kas.a_hat->hat_prev != hat)  in htable_steal()
699 					move_victim(hat);  in htable_steal()
709 			if (hat->hat_flags & HAT_VICTIM) {  in htable_steal()
710 				ASSERT(hat != kas.a_hat);  in htable_steal()
711 				hat->hat_flags &= ~HAT_VICTIM;  in htable_steal()
716 			hat = hat->hat_next;  in htable_steal()
780 	hat_t		*hat,  in htable_alloc()  argument
791 	if (level < 0 || level > TOP_LEVEL(hat))  in htable_alloc()
794 	is_copied = (hat->hat_flags & HAT_COPIED) &&  in htable_alloc()
795 	    level == hat->hat_max_level;  in htable_alloc()
803 	if (hat->hat_ht_cached != NULL && !is_bare) {  in htable_alloc()
804 		hat_enter(hat);  in htable_alloc()
805 		ht = hat->hat_ht_cached;  in htable_alloc()
807 			hat->hat_ht_cached = ht->ht_next;  in htable_alloc()
812 		hat_exit(hat);  in htable_alloc()
841 			ht->ht_hat = hat;  in htable_alloc()
905 			hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1);  in htable_alloc()
906 			if (hat->hat_user_ptable != PFN_INVALID)  in htable_alloc()
913 		block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable),  in htable_alloc()
949 	ht->ht_hat = hat;  in htable_alloc()
967 			(void) xen_kpm_page(hat->hat_user_ptable, PT_VALID);  in htable_alloc()
981 	hat_t *hat = ht->ht_hat;  in htable_free()  local
988 	if (hat != NULL &&  in htable_free()
991 	    (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) {  in htable_free()
994 		hat_enter(hat);  in htable_free()
995 		ht->ht_next = hat->hat_ht_cached;  in htable_free()
996 		hat->hat_ht_cached = ht;  in htable_free()
997 		hat_exit(hat);  in htable_free()
1010 		if (ht->ht_level == mmu.max_level && hat != NULL) {  in htable_free()
1011 			ptable_free(hat->hat_user_ptable);  in htable_free()
1012 			hat->hat_user_ptable = PFN_INVALID;  in htable_free()
1038 htable_purge_hat(hat_t *hat)  in htable_purge_hat()  argument
1046 	if (!(hat->hat_flags & HAT_FREEING)) {  in htable_purge_hat()
1049 			hat_enter(hat);  in htable_purge_hat()
1050 			ht = hat->hat_ht_cached;  in htable_purge_hat()
1052 				hat_exit(hat);  in htable_purge_hat()
1055 			hat->hat_ht_cached = ht->ht_next;  in htable_purge_hat()
1056 			hat_exit(hat);  in htable_purge_hat()
1066 	while ((ht = hat->hat_ht_cached) != NULL) {  in htable_purge_hat()
1067 		hat->hat_ht_cached = ht->ht_next;  in htable_purge_hat()
1074 	for (h = 0; h < hat->hat_num_hash; ++h) {  in htable_purge_hat()
1075 		while ((ht = hat->hat_ht_hash[h]) != NULL) {  in htable_purge_hat()
1082 				ASSERT(hat->hat_ht_hash[h] == ht);  in htable_purge_hat()
1083 				hat->hat_ht_hash[h] = ht->ht_next;  in htable_purge_hat()
1100 	hat_t		*hat = old->ht_hat;  in unlink_ptp()  local
1126 	if (!(hat->hat_flags & HAT_FREEING)) {  in unlink_ptp()
1127 		hat_tlb_inval(hat, (higher->ht_flags & HTABLE_COPIED) ?  in unlink_ptp()
1182 	hat_t		*hat;  in htable_release()  local
1189 			hat = ht->ht_hat;  in htable_release()
1192 			hashval = HTABLE_HASH(hat, va, level);  in htable_release()
1216 				if (hat->hat_flags & HAT_FREEING)  in htable_release()
1224 				    (hat != kas.a_hat || va >= kernelbase))  in htable_release()
1260 				ASSERT(hat->hat_ht_hash[hashval] == ht);  in htable_release()
1261 				hat->hat_ht_hash[hashval] = ht->ht_next;  in htable_release()
1285 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level)  in htable_lookup()  argument
1292 	ASSERT(level <= TOP_LEVEL(hat));  in htable_lookup()
1294 	if (level == TOP_LEVEL(hat)) {  in htable_lookup()
1299 		if ((hat->hat_flags & HAT_COPIED_32) &&  in htable_lookup()
1307 	hashval = HTABLE_HASH(hat, base, level);  in htable_lookup()
1309 	for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) {  in htable_lookup()
1310 		if (ht->ht_hat == hat &&  in htable_lookup()
1328 	hat_t		*hat = ht->ht_hat;  in htable_acquire()  local
1331 	uint_t		hashval = HTABLE_HASH(hat, base, level);  in htable_acquire()
1341 		for (h = hat->hat_ht_hash[hashval];  in htable_acquire()
1363 	hat_t		*hat,  in htable_create()  argument
1375 	if (level < 0 || level > TOP_LEVEL(hat))  in htable_create()
1382 	for (l = TOP_LEVEL(hat); l >= level; --l) {  in htable_create()
1384 		if (l == TOP_LEVEL(hat))  in htable_create()
1389 		h = HTABLE_HASH(hat, base, l);  in htable_create()
1395 		if (l == TOP_LEVEL(hat)) {  in htable_create()
1396 			ht = hat->hat_htable;  in htable_create()
1398 			for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {  in htable_create()
1399 				ASSERT(ht->ht_hat == hat);  in htable_create()
1435 			new = htable_alloc(hat, base, l,  in htable_create()
1449 			ht->ht_next = hat->hat_ht_hash[h];  in htable_create()
1451 			if (hat->hat_ht_hash[h])  in htable_create()
1452 				hat->hat_ht_hash[h]->ht_prev = ht;  in htable_create()
1453 			hat->hat_ht_hash[h] = ht;  in htable_create()
1486 	hat_t *hat,  in htable_attach()  argument
1503 	ht->ht_hat = hat;  in htable_attach()
1517 	h = HTABLE_HASH(hat, base, level);  in htable_attach()
1519 	ht->ht_next = hat->hat_ht_hash[h];  in htable_attach()
1521 	if (hat->hat_ht_hash[h])  in htable_attach()
1522 		hat->hat_ht_hash[h]->ht_prev = ht;  in htable_attach()
1523 	hat->hat_ht_hash[h] = ht;  in htable_attach()
1571 				htable_attach(hat, base, level - 1,  in htable_attach()
1668 	struct hat *hat,  in htable_walk()  argument
1686 	ASSERT(hat == kas.a_hat || eaddr <= kernelbase ||  in htable_walk()
1688 	if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END)  in htable_walk()
1719 	if (hat->hat_ism_pgcnt > 0) {  in htable_walk()
1724 			if (hat->hat_pages_mapped[l] != 0)  in htable_walk()
1732 		for (l = 0; l <= TOP_LEVEL(hat); ++l) {  in htable_walk()
1733 			ht = htable_lookup(hat, va, l);  in htable_walk()
1751 			ASSERT(l < TOP_LEVEL(hat));  in htable_walk()
1773 	struct hat *hat,  in htable_getpte()  argument
1786 		ht = htable_lookup(hat, vaddr, l);  in htable_getpte()
1806 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry)  in htable_getpage()  argument
1812 	ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level);  in htable_getpage()
2075 	hat_t		*hat = ht->ht_hat;  in x86pte_set()  local
2135 		hat_tlb_inval(hat, addr);  in x86pte_set()
2462 	hat_t *hat;  in hat_dump()  local
2469 	for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) {  in hat_dump()
2470 		for (h = 0; h < hat->hat_num_hash; ++h) {  in hat_dump()
2471 			for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {  in hat_dump()