587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
589 	vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
2728 	vm_page_t mpte;  in pmap_promote_l3e()  local
2794 	mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);  in pmap_promote_l3e()
2795 	KASSERT(mpte >= vm_page_array &&  in pmap_promote_l3e()
2796 	    mpte < &vm_page_array[vm_page_array_size],  in pmap_promote_l3e()
2798 	KASSERT(mpte->pindex == pmap_l3e_pindex(va),  in pmap_promote_l3e()
2800 	if (pmap_insert_pt_page(pmap, mpte)) {  in pmap_promote_l3e()
2835 	vm_page_t mpte, om;  in mmu_radix_enter()  local
2892 	mpte = NULL;  in mmu_radix_enter()
2903 		if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {  in mmu_radix_enter()
2904 			mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);  in mmu_radix_enter()
2905 			mpte->ref_count++;  in mmu_radix_enter()
2913 		mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),  in mmu_radix_enter()
2915 		if (mpte == NULL && nosleep) {  in mmu_radix_enter()
2958 		if (mpte != NULL) {  in mmu_radix_enter()
2959 			mpte->ref_count--;  in mmu_radix_enter()
2960 			KASSERT(mpte->ref_count > 0,  in mmu_radix_enter()
3119 	if ((mpte == NULL || mpte->ref_count == NPTEPG) &&  in mmu_radix_enter()
3340 	vm_page_t m, mpte;  in mmu_radix_enter_object()  local
3348 	mpte = NULL;  in mmu_radix_enter_object()
3361 			mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,  in mmu_radix_enter_object()
3362 			    mpte, &lock, &invalidate);  in mmu_radix_enter_object()
3376     vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate)  in mmu_radix_enter_quick_locked()  argument
3399 		if (mpte && (mpte->pindex == ptepindex)) {  in mmu_radix_enter_quick_locked()
3400 			mpte->ref_count++;  in mmu_radix_enter_quick_locked()
3416 				mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);  in mmu_radix_enter_quick_locked()
3417 				mpte->ref_count++;  in mmu_radix_enter_quick_locked()
3423 				mpte = _pmap_allocpte(pmap, ptepindex, NULL);  in mmu_radix_enter_quick_locked()
3424 				if (mpte == NULL)  in mmu_radix_enter_quick_locked()
3425 					return (mpte);  in mmu_radix_enter_quick_locked()
3428 		pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));  in mmu_radix_enter_quick_locked()
3431 		mpte = NULL;  in mmu_radix_enter_quick_locked()
3435 		if (mpte != NULL) {  in mmu_radix_enter_quick_locked()
3436 			mpte->ref_count--;  in mmu_radix_enter_quick_locked()
3437 			mpte = NULL;  in mmu_radix_enter_quick_locked()
3439 		return (mpte);  in mmu_radix_enter_quick_locked()
3447 		if (mpte != NULL) {  in mmu_radix_enter_quick_locked()
3449 			if (pmap_unwire_ptp(pmap, va, mpte, &free)) {  in mmu_radix_enter_quick_locked()
3459 			mpte = NULL;  in mmu_radix_enter_quick_locked()
3461 		return (mpte);  in mmu_radix_enter_quick_locked()
3478 	return (mpte);  in mmu_radix_enter_quick_locked()
3667 	vm_page_t mpte;  in mmu_radix_init()  local
3691 		mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));  in mmu_radix_init()
3692 		KASSERT(mpte >= vm_page_array &&  in mmu_radix_init()
3693 		    mpte < &vm_page_array[vm_page_array_size],  in mmu_radix_init()
3696 		mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;  in mmu_radix_init()
3697 		mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);  in mmu_radix_init()
3698 		MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);  in mmu_radix_init()
3700 		mpte->ref_count = 1;  in mmu_radix_init()
4757 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)  in pmap_insert_pt_page()  argument
4761 	return (vm_radix_insert(&pmap->pm_radix, mpte));  in pmap_insert_pt_page()
4851 	vm_page_t mpte;  in pmap_unuse_pt()  local
4856 	mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);  in pmap_unuse_pt()
4857 	return (pmap_unwire_ptp(pmap, va, mpte, free));  in pmap_unuse_pt()
4938 	vm_page_t mpte;  in pmap_demote_l3e_locked()  local
4947 	if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==  in pmap_demote_l3e_locked()
4965 		if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj(  in pmap_demote_l3e_locked()
4977 		mpte->pindex = pmap_l3e_pindex(va);  in pmap_demote_l3e_locked()
4981 	mptepa = VM_PAGE_TO_PHYS(mpte);  in pmap_demote_l3e_locked()
4991 	if (mpte->ref_count == 1) {  in pmap_demote_l3e_locked()
4992 		mpte->ref_count = NPTEPG;  in pmap_demote_l3e_locked()
5046 	vm_page_t mpte;  in pmap_remove_kernel_l3e()  local
5050 	mpte = pmap_remove_pt_page(pmap, va);  in pmap_remove_kernel_l3e()
5051 	if (mpte == NULL)  in pmap_remove_kernel_l3e()
5054 	mptepa = VM_PAGE_TO_PHYS(mpte);  in pmap_remove_kernel_l3e()
5078 	vm_page_t m, mpte;  in pmap_remove_l3e()  local
5106 		mpte = pmap_remove_pt_page(pmap, sva);  in pmap_remove_l3e()
5107 		if (mpte != NULL) {  in pmap_remove_l3e()
5109 			KASSERT(mpte->ref_count == NPTEPG,  in pmap_remove_l3e()
5111 			mpte->ref_count = 0;  in pmap_remove_l3e()
5112 			pmap_add_delayed_free_list(mpte, free, false);  in pmap_remove_l3e()
5453 	vm_page_t m, mpte, mt;  in mmu_radix_remove_pages()  local
5576 					mpte = pmap_remove_pt_page(pmap, pv->pv_va);  in mmu_radix_remove_pages()
5577 					if (mpte != NULL) {  in mmu_radix_remove_pages()
5579 						KASSERT(mpte->ref_count == NPTEPG,  in mmu_radix_remove_pages()
5581 						mpte->ref_count = 0;  in mmu_radix_remove_pages()
5582 						pmap_add_delayed_free_list(mpte, &free, false);  in mmu_radix_remove_pages()