460 	uintptr_t		va;  in hat_alloc()  local
556 		for (va = rp->hkr_start_va; va != rp->hkr_end_va;  in hat_alloc()
557 		    va += cnt * LEVEL_SIZE(rp->hkr_level)) {  in hat_alloc()
562 				ht = htable_create(hat, va, rp->hkr_level,  in hat_alloc()
565 			start = htable_va2entry(va, ht);  in hat_alloc()
567 			eva = va +  in hat_alloc()
574 			src = htable_lookup(kas.a_hat, va, rp->hkr_level);  in hat_alloc()
734 hat_kernelbase(uintptr_t va)  in hat_kernelbase()  argument
736 	if (IN_VA_HOLE(va))  in hat_kernelbase()
737 		panic("_userlimit %p will fall in VA hole\n", (void *)va);  in hat_kernelbase()
738 	return (va);  in hat_kernelbase()
1080 	uintptr_t va;  in hat_pcp_setup()  local
1158 	for (va = (uintptr_t)&kpti_tramp_start;  in hat_pcp_setup()
1159 	    va < (uintptr_t)&kpti_tramp_end; va += MMU_PAGESIZE) {  in hat_pcp_setup()
1160 		hati_cpu_punchin(cpu, va, PROT_READ | PROT_EXEC);  in hat_pcp_setup()
1164 	for (va = (uintptr_t)cpu->cpu_m.mcpu_ldt, len = LDT_CPU_SIZE;  in hat_pcp_setup()
1165 	    len >= MMU_PAGESIZE; va += MMU_PAGESIZE, len -= MMU_PAGESIZE) {  in hat_pcp_setup()
1166 		hati_cpu_punchin(cpu, va, PROT_READ);  in hat_pcp_setup()
1184 		for (va = (uintptr_t)dblfault_stack0,  in hat_pcp_setup()
1186 		    va += MMU_PAGESIZE, len -= MMU_PAGESIZE) {  in hat_pcp_setup()
1187 			hati_cpu_punchin(cpu, va, PROT_READ | PROT_WRITE);  in hat_pcp_setup()
1193 	for (va = (uintptr_t)&kdi_isr_start;  in hat_pcp_setup()
1194 	    va < (uintptr_t)&kdi_isr_end; va += MMU_PAGESIZE) {  in hat_pcp_setup()
1195 		hati_cpu_punchin(cpu, va, PROT_READ | PROT_EXEC);  in hat_pcp_setup()
1240 	uintptr_t	va;  in hat_init_finish()  local
1271 		for (va = rp->hkr_start_va; va != rp->hkr_end_va;  in hat_init_finish()
1272 		    va += LEVEL_SIZE(rp->hkr_level)) {  in hat_init_finish()
1275 			if (IN_HYPERVISOR_VA(va))  in hat_init_finish()
1280 			    (ht = htable_getpage(kas.a_hat, va, NULL)) !=  in hat_init_finish()
1286 			(void) htable_create(kas.a_hat, va, rp->hkr_level - 1,  in hat_init_finish()
1928 	uintptr_t	va,  in hati_load_common()  argument
1956 	ht = htable_lookup(hat, va, level);  in hati_load_common()
1965 		ht = htable_create(hat, va, level, NULL);  in hati_load_common()
1973 	if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht)) {  in hati_load_common()
1975 		    "ht->ht_vaddr=%p, ht->ht_level=%d", (void *)va,  in hati_load_common()
1979 	entry = htable_va2entry(va, ht);  in hati_load_common()
1993 	if (hat == kas.a_hat && va >= kernelbase)  in hati_load_common()
2019 	uintptr_t	va = (uintptr_t)addr;  in hat_kmap_load()  local
2022 	pgcnt_t		pg_off = mmu_btop(va - mmu.kmap_addr);  in hat_kmap_load()
2042 	ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >>  in hat_kmap_load()
2044 	entry = htable_va2entry(va, ht);  in hat_kmap_load()
2097 	uintptr_t	va = (uintptr_t)addr;  in hat_memload()  local
2102 	ASSERT(IS_PAGEALIGNED(va));  in hat_memload()
2103 	ASSERT(hat == kas.a_hat || va < _userlimit);  in hat_memload()
2107 	ASSERT(!IN_VA_HOLE(va));  in hat_memload()
2113 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {  in hat_memload()
2125 	if (hati_load_common(hat, va, pp, attr, flags, level, pfn) != 0)  in hat_memload()
2150 	uintptr_t	va = (uintptr_t)addr;  in hat_memload_array()  local
2151 	uintptr_t	eaddr = va + len;  in hat_memload_array()
2159 	ASSERT(IS_PAGEALIGNED(va));  in hat_memload_array()
2160 	ASSERT(hat == kas.a_hat || va + len <= _userlimit);  in hat_memload_array()
2173 	while (va < eaddr) {  in hat_memload_array()
2183 			if (!IS_P2ALIGNED(va, pgsize) ||  in hat_memload_array()
2184 			    (eaddr - va) < pgsize ||  in hat_memload_array()
2219 		ASSERT(!IN_VA_HOLE(va));  in hat_memload_array()
2220 		while (hati_load_common(hat, va, pages[pgindx], attr,  in hat_memload_array()
2231 		va += pgsize;  in hat_memload_array()
2285 	uintptr_t	va = ALIGN2PAGE(addr);  in hat_devload()  local
2286 	uintptr_t	eva = va + len;  in hat_devload()
2294 	ASSERT(IS_PAGEALIGNED(va));  in hat_devload()
2302 	while (va < eva) {  in hat_devload()
2311 			if (IS_P2ALIGNED(va, pgsize) &&  in hat_devload()
2312 			    (eva - va) >= pgsize &&  in hat_devload()
2362 		ASSERT(!IN_VA_HOLE(va));  in hat_devload()
2363 		while (hati_load_common(hat, va, pp, a, f, level, pfn) != 0) {  in hat_devload()
2373 		va += pgsize;  in hat_devload()
2650 hat_tlb_inval(hat_t *hat, uintptr_t va)  in hat_tlb_inval()  argument
2656 	range.tr_va = va;  in hat_tlb_inval()
2772 	uintptr_t	va = (uintptr_t)addr;  in hat_kmap_unload()  local
2773 	uintptr_t	eva = va + len;  in hat_kmap_unload()
2780 	for (; va < eva; va += MMU_PAGESIZE) {  in hat_kmap_unload()
2784 		pg_index = mmu_btop(va - mmu.kmap_addr);  in hat_kmap_unload()
2791 		ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr)  in hat_kmap_unload()
2793 		entry = htable_va2entry(va, ht);  in hat_kmap_unload()
2809 	uintptr_t va = (uintptr_t)addr;  in hat_unload()  local
2812 	ASSERT(hat == kas.a_hat || va + len <= _userlimit);  in hat_unload()
2817 	if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {  in hat_unload()
2954 hat_flush_range(hat_t *hat, caddr_t va, size_t size)  in hat_flush_range()  argument
2957 	caddr_t endva = va + size;  in hat_flush_range()
2959 	while (va < endva) {  in hat_flush_range()
2960 		sz = hat_getpagesize(hat, va);  in hat_flush_range()
2970 		xen_flush_va(va);  in hat_flush_range()
2972 		mmu_flush_tlb_kpage((uintptr_t)va);  in hat_flush_range()
2974 		va += sz;  in hat_flush_range()
3365 is_it_dism(hat_t *hat, caddr_t va)  in is_it_dism()  argument
3371 	seg = as_findseg(hat->hat_as, va, 0);  in is_it_dism()
3373 	ASSERT(seg->s_base <= va);  in is_it_dism()
4353 	uintptr_t	va = (uintptr_t)addr;  in hat_mempte_setup()  local
4359 	ASSERT(IS_PAGEALIGNED(va));  in hat_mempte_setup()
4360 	ASSERT(!IN_VA_HOLE(va));  in hat_mempte_setup()
4363 	ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0);  in hat_mempte_setup()
4365 		ht = htable_create(kas.a_hat, va, 0, NULL);  in hat_mempte_setup()
4366 		entry = htable_va2entry(va, ht);  in hat_mempte_setup()
4443 	uintptr_t	va = (uintptr_t)addr;  in hat_mempte_remap()  local
4454 	ASSERT(IS_PAGEALIGNED(va));  in hat_mempte_remap()
4455 	ASSERT(!IN_VA_HOLE(va));  in hat_mempte_remap()
4456 	ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0);  in hat_mempte_remap()
4466 	if (HYPERVISOR_update_va_mapping(va, pte, UVMF_INVLPG | UVMF_LOCAL))  in hat_mempte_remap()
4980 hati_cpu_punchin(cpu_t *cpu, uintptr_t va, uint_t attrs)  in hati_cpu_punchin()  argument
4989 	ASSERT3U(va & MMU_PAGEOFFSET, ==, 0);  in hati_cpu_punchin()
4991 	pfn = hat_getpfnum(kas.a_hat, (caddr_t)va);  in hati_cpu_punchin()
5000 	ret = hati_load_common(cpu_hat, va, NULL, attrs, 0, 0, pfn);  in hati_cpu_punchin()