294 static void		moea64_syncicache(pmap_t pmap, vm_offset_t va,
362 static int moea64_sp_enter(pmap_t pmap, vm_offset_t va,
367 static void moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m);
566 init_pvo_entry(struct pvo_entry *pvo, pmap_t pmap, vm_offset_t va)  in init_pvo_entry()  argument
572 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);  in init_pvo_entry()
574 	pvo->pvo_pmap = pmap;  in init_pvo_entry()
577 	vsid = va_to_vsid(pmap, va);  in init_pvo_entry()
581 	if (pmap == kernel_pmap && (pvo->pvo_vaddr & PVO_LARGE) != 0)  in init_pvo_entry()
1388 moea64_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)  in moea64_mincore()  argument
1396 	PMAP_LOCK(pmap);  in moea64_mincore()
1398 	pvo = moea64_pvo_find_va(pmap, addr);  in moea64_mincore()
1408 		PMAP_UNLOCK(pmap);  in moea64_mincore()
1412 	PMAP_UNLOCK(pmap);  in moea64_mincore()
1645 moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,  in moea64_enter()  argument
1662 		return (moea64_sp_enter(pmap, va, m, prot, flags, psind));  in moea64_enter()
1685 	PMAP_LOCK(pmap);  in moea64_enter()
1687 		init_pvo_entry(pvo, pmap, va);  in moea64_enter()
1689 	if (moea64_ps_enabled(pmap) &&  in moea64_enter()
1690 	    (tpvo = moea64_pvo_find_va(pmap, va & ~HPT_SP_MASK)) != NULL &&  in moea64_enter()
1718 			PMAP_UNLOCK(pmap);  in moea64_enter()
1725 			KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old "  in moea64_enter()
1731 	PMAP_UNLOCK(pmap);  in moea64_enter()
1745 	if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 &&  in moea64_enter()
1748 		moea64_syncicache(pmap, va, pa, PAGE_SIZE);  in moea64_enter()
1761 	if (moea64_ps_enabled(pmap) && pmap != kernel_pmap && pvo != NULL &&  in moea64_enter()
1768 		moea64_sp_promote(pmap, va, m);  in moea64_enter()
1775 moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,  in moea64_syncicache()  argument
1794 	} else if (pmap == kernel_pmap) {  in moea64_syncicache()
1885 moea64_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)  in moea64_extract_and_hold()  argument
1891 	PMAP_LOCK(pmap);  in moea64_extract_and_hold()
1892 	pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);  in moea64_extract_and_hold()
1898 	PMAP_UNLOCK(pmap);  in moea64_extract_and_hold()
2022 moea64_is_prefaultable(pmap_t pmap, vm_offset_t va)  in moea64_is_prefaultable()  argument
2027 	PMAP_LOCK(pmap);  in moea64_is_prefaultable()
2028 	pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);  in moea64_is_prefaultable()
2031 	PMAP_UNLOCK(pmap);  in moea64_is_prefaultable()
2056 	pmap_t	pmap;  in moea64_remove_write()  local
2069 		pmap = pvo->pvo_pmap;  in moea64_remove_write()
2070 		PMAP_LOCK(pmap);  in moea64_remove_write()
2086 		PMAP_UNLOCK(pmap);  in moea64_remove_write()
2123 	pmap_t	pmap;  in moea64_page_set_memattr()  local
2138 		pmap = pvo->pvo_pmap;  in moea64_page_set_memattr()
2139 		PMAP_LOCK(pmap);  in moea64_page_set_memattr()
2164 		PMAP_UNLOCK(pmap);  in moea64_page_set_memattr()
2385 moea64_page_exists_quick(pmap_t pmap, vm_page_t m)  in moea64_page_exists_quick()  argument
2397 		if (!(pvo->pvo_vaddr & PVO_DEAD) && pvo->pvo_pmap == pmap) {  in moea64_page_exists_quick()
2494 moea64_pinit(pmap_t pmap)  in moea64_pinit()  argument
2497 	RB_INIT(&pmap->pmap_pvo);  in moea64_pinit()
2499 	pmap->pm_slb_tree_root = slb_alloc_tree();  in moea64_pinit()
2500 	pmap->pm_slb = slb_alloc_user_cache();  in moea64_pinit()
2501 	pmap->pm_slb_len = 0;  in moea64_pinit()
2507 moea64_pinit(pmap_t pmap)  in moea64_pinit()  argument
2512 	RB_INIT(&pmap->pmap_pvo);  in moea64_pinit()
2515 		pmap->pmap_phys = (pmap_t)moea64_kextract((vm_offset_t)pmap);  in moea64_pinit()
2517 		pmap->pmap_phys = pmap;  in moea64_pinit()
2525 		pmap->pm_sr[i] = VSID_MAKE(i, hash);  in moea64_pinit()
2527 	KASSERT(pmap->pm_sr[0] != 0, ("moea64_pinit: pm_sr[0] = 0"));  in moea64_pinit()
2675 moea64_release(pmap_t pmap)  in moea64_release()  argument
2682 	slb_free_tree(pmap);  in moea64_release()
2683 	slb_free_user_cache(pmap->pm_slb);  in moea64_release()
2685 	KASSERT(pmap->pm_sr[0] != 0, ("moea64_release: pm_sr[0] = 0"));  in moea64_release()
2687 	moea64_release_vsid(VSID_TO_HASH(pmap->pm_sr[0]));  in moea64_release()
2796 	pmap_t	pmap;  in moea64_remove_all()  local
2802 		pmap = pvo->pvo_pmap;  in moea64_remove_all()
2803 		PMAP_LOCK(pmap);  in moea64_remove_all()
2816 		PMAP_UNLOCK(pmap);  in moea64_remove_all()
3571 moea64_ps_enabled(pmap_t pmap)  in moea64_ps_enabled()  argument
3680 moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,  in moea64_sp_enter()  argument
3698 	KASSERT(pmap != kernel_pmap,  in moea64_sp_enter()
3731 	PMAP_LOCK(pmap);  in moea64_sp_enter()
3734 	moea64_remove_locked(pmap, va, va + HPT_SP_SIZE, &tofree);  in moea64_sp_enter()
3756 		init_pvo_entry(pvo, pmap, va);  in moea64_sp_enter()
3768 	PMAP_UNLOCK(pmap);  in moea64_sp_enter()
3787 		moea64_syncicache(pmap, sva, spa, HPT_SP_SIZE);  in moea64_sp_enter()
3791 	    __func__, (uintmax_t)sva, pmap);  in moea64_sp_enter()
3798 moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m)  in moea64_sp_promote()  argument
3814 	PMAP_LOCK(pmap);  in moea64_sp_promote()
3832 	first = pvo = moea64_pvo_find_va(pmap, sva);  in moea64_sp_promote()
3838 			    __func__, pmap, (uintmax_t)va);  in moea64_sp_promote()
3844 			    __func__, pmap, (uintmax_t)va,  in moea64_sp_promote()
3853 			    __func__, pmap, (uintmax_t)va,  in moea64_sp_promote()
3862 			    __func__, pmap, (uintmax_t)va,  in moea64_sp_promote()
3871 			    __func__, pmap, (uintmax_t)va,  in moea64_sp_promote()
3878 		pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);  in moea64_sp_promote()
3898 	    pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {  in moea64_sp_promote()
3911 	PMAP_UNLOCK(pmap);  in moea64_sp_promote()
3916 	    __func__, (uintmax_t)sva, pmap);  in moea64_sp_promote()
3921 	PMAP_UNLOCK(pmap);  in moea64_sp_promote()
3931 	pmap_t pmap __diagused;  in moea64_sp_demote_aligned()
3936 	pmap = sp->pvo_pmap;  in moea64_sp_demote_aligned()
3937 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);  in moea64_sp_demote_aligned()
3949 	    pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo),  in moea64_sp_demote_aligned()
3979 	    __func__, (uintmax_t)PVO_VADDR(sp), pmap);  in moea64_sp_demote_aligned()
4129 	pmap_t pmap;  in moea64_sp_query_locked()  local
4132 	pmap = pvo->pvo_pmap;  in moea64_sp_query_locked()
4133 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);  in moea64_sp_query_locked()
4137 		sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);  in moea64_sp_query_locked()
4146 	    pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {  in moea64_sp_query_locked()
4168 	pmap_t pmap;  in moea64_sp_query()  local
4170 	pmap = pvo->pvo_pmap;  in moea64_sp_query()
4171 	PMAP_LOCK(pmap);  in moea64_sp_query()
4179 		PMAP_UNLOCK(pmap);  in moea64_sp_query()
4184 	PMAP_UNLOCK(pmap);  in moea64_sp_query()
4197 	pmap_t pmap;  in moea64_sp_pvo_clear()  local
4202 	pmap = pvo->pvo_pmap;  in moea64_sp_pvo_clear()
4203 	PMAP_LOCK(pmap);  in moea64_sp_pvo_clear()
4211 		PMAP_UNLOCK(pmap);  in moea64_sp_pvo_clear()
4217 		sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);  in moea64_sp_pvo_clear()
4226 	    pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {  in moea64_sp_pvo_clear()
4234 	PMAP_UNLOCK(pmap);  in moea64_sp_pvo_clear()
4247 	pmap_t pmap;  in moea64_sp_clear()  local
4250 	pmap = pvo->pvo_pmap;  in moea64_sp_clear()
4267 	    (PVO_VADDR(pvo) >> HPT_SP_SHIFT) ^ (uintptr_t)pmap) &  in moea64_sp_clear()
4292 		PMAP_LOCK(pmap);  in moea64_sp_clear()
4301 			PMAP_UNLOCK(pmap);  in moea64_sp_clear()
4309 			PMAP_UNLOCK(pmap);  in moea64_sp_clear()
4321 			moea64_pvo_protect(pmap, pvo,  in moea64_sp_clear()
4324 		PMAP_UNLOCK(pmap);  in moea64_sp_clear()