333 	ptepa = (be64toh(*l3e) & NLB_MASK);  in pmap_l3e_to_pte()
345 	l3pa = (be64toh(*l2e) & NLB_MASK);  in pmap_l2e_to_l3e()
357 	l2pa = (be64toh(*l1e) & NLB_MASK);  in pmap_l1e_to_l2e()
376 	if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0)  in pmap_pml2e()
387 	if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0)  in pmap_pml3e()
398 	if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)  in pmap_pte()
911 	if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)  in kvtopte()
941 	if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0)  in pmap_nofault_pte()
944 	if (be64toh(*l3e) & RPTE_LEAF) {  in pmap_nofault_pte()
951 	if (pte == NULL || (be64toh(*pte) & PG_V) == 0)  in pmap_nofault_pte()
968 	origpte = newpte = be64toh(*pte);  in pmap_nofault()
989 	    be64toh(*pte) != origpte) {  in pmap_nofault()
1458 				if ((be64toh(*l3e) & RPTE_LEAF) != 0)  in reclaim_pv_chunk()
1461 				if ((be64toh(*pte) & PG_W) != 0)  in reclaim_pv_chunk()
1463 				tpte = be64toh(pte_load_clear(pte));  in reclaim_pv_chunk()
1482 				pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);  in reclaim_pv_chunk()
1792 		if ((be64toh(*pte) & RPTE_VALID) == 0) {  in mmu_radix_dmap_range()
1801 		} else if ((be64toh(*pte) & RPTE_VALID) == 0) {  in mmu_radix_dmap_range()
1811 		} else if ((be64toh(*pte) & RPTE_VALID) == 0) {  in mmu_radix_dmap_range()
2140 	prev = be64toh(isa3_parttab[lpid].pagetab);  in mmu_parttab_update()
2172 	pagetab = be64toh(isa3_parttab[0].pagetab);  in mmu_radix_proctab_register()
2236 		if ((be64toh(*l1e) & PG_V) == 0) {  in mmu_radix_advise()
2243 		if ((be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_advise()
2253 		oldl3e = be64toh(*l3e);  in mmu_radix_advise()
2287 				KASSERT((be64toh(*pte) & PG_V) != 0,  in mmu_radix_advise()
2289 				pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,  in mmu_radix_advise()
2303 			if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))  in mmu_radix_advise()
2305 			else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {  in mmu_radix_advise()
2312 					m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME);  in mmu_radix_advise()
2316 			} else if ((be64toh(*pte) & PG_A) != 0)  in mmu_radix_advise()
2467 		oldl3e = be64toh(*l3e);  in mmu_radix_clear_modify()
2479 			oldpte = be64toh(*pte);  in mmu_radix_clear_modify()
2483 				   oldpte = be64toh(*pte);  in mmu_radix_clear_modify()
2503 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found"  in mmu_radix_clear_modify()
2506 		if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {  in mmu_radix_clear_modify()
2550 		if ((be64toh(*l1e) & PG_V) == 0) {  in mmu_radix_copy()
2558 		if ((be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_copy()
2570 		srcptepaddr = be64toh(*l3e);  in mmu_radix_copy()
2584 			if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||  in mmu_radix_copy()
2609 			ptetemp = be64toh(*src_pte);  in mmu_radix_copy()
2623 				if (be64toh(*dst_pte) == 0 &&  in mmu_radix_copy()
2737 	firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME);  in pmap_promote_l3e()
2739 	newpde = be64toh(*firstpte);  in pmap_promote_l3e()
2763 		oldpte = be64toh(*pte);  in pmap_promote_l3e()
2794 	mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);  in pmap_promote_l3e()
2900 	if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 ||  in mmu_radix_enter()
2904 			mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);  in mmu_radix_enter()
2926 	origpte = be64toh(*pte);  in mmu_radix_enter()
3003 		origpte = be64toh(pte_load_clear(pte));  in mmu_radix_enter()
3085 		origpte = be64toh(pte_load_store(pte, htobe64(newpte)));  in mmu_radix_enter()
3223 	oldl3e = be64toh(*l3e);  in pmap_enter_l3e()
3251 			mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);  in pmap_enter_l3e()
3261 			KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",  in pmap_enter_l3e()
3412 			if (ptepa && (be64toh(*ptepa) & PG_V) != 0) {  in mmu_radix_enter_quick_locked()
3413 				if (be64toh(*ptepa) & RPTE_LEAF)  in mmu_radix_enter_quick_locked()
3415 				mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);  in mmu_radix_enter_quick_locked()
3433 	if (be64toh(*pte)) {  in mmu_radix_enter_quick_locked()
3510 	if (be64toh(*l3e) & RPTE_LEAF) {  in mmu_radix_extract()
3511 		pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);  in mmu_radix_extract()
3525 		pa = be64toh(*pte);  in mmu_radix_extract()
3543 	if (l3ep != NULL && (l3e = be64toh(*l3ep))) {  in mmu_radix_extract_and_hold()
3550 			pte = be64toh(*pmap_l3e_to_pte(l3ep, va));  in mmu_radix_extract_and_hold()
3580 		if ((be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_growkernel()
3592 		if ((be64toh(*l3e) & PG_V) != 0) {  in mmu_radix_growkernel()
3786 		rv = (be64toh(*pte) & mask) == mask;  in pmap_page_test_mappings()
3813 			rv = (be64toh(*pte) & mask) == mask;  in pmap_page_test_mappings()
3857 	if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) {  in mmu_radix_is_prefaultable()
3859 		rv = (be64toh(*pte) & PG_V) == 0;  in mmu_radix_is_prefaultable()
3936 		oldl3e = be64toh(*l3e);  in mmu_radix_ts_referenced()
4006 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,  in mmu_radix_ts_referenced()
4010 		if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW))  in mmu_radix_ts_referenced()
4012 		if ((be64toh(*pte) & PG_A) != 0) {  in mmu_radix_ts_referenced()
4108 			if ((be64toh(*l3e) & PG_V) == 0) {  in mmu_radix_object_init_pt()
4206 		if ((be64toh(*pte) & PG_W) != 0)  in mmu_radix_page_wired_mappings()
4227 			if ((be64toh(*pte) & PG_W) != 0)  in mmu_radix_page_wired_mappings()
4319 		KASSERT((be64toh(*l1e) & PG_V) == 0,  in _pmap_allocpte()
4333 		if ((be64toh(*l1e) & PG_V) == 0) {  in _pmap_allocpte()
4343 			pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME);  in _pmap_allocpte()
4346 		l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);  in _pmap_allocpte()
4350 		KASSERT((be64toh(*l2e) & PG_V) == 0,  in _pmap_allocpte()
4366 		if ((be64toh(*l1e) & PG_V) == 0) {  in _pmap_allocpte()
4374 			l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);  in _pmap_allocpte()
4377 			l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);  in _pmap_allocpte()
4379 			if ((be64toh(*l2e) & PG_V) == 0) {  in _pmap_allocpte()
4389 				pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME);  in _pmap_allocpte()
4393 		l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME);  in _pmap_allocpte()
4397 		KASSERT((be64toh(*l3e) & PG_V) == 0,  in _pmap_allocpte()
4414 	if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) {  in pmap_allocl3e()
4416 		pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME);  in pmap_allocl3e()
4450 	if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) {  in pmap_allocpte()
4464 	if (pd != NULL && (be64toh(*pd) & PG_V) != 0) {  in pmap_allocpte()
4465 		m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME);  in pmap_allocpte()
4510 	oldpde = newpde = be64toh(*l3e);  in pmap_protect_l3e()
4571 		if ((be64toh(*l1e) & PG_V) == 0) {  in mmu_radix_protect()
4579 		if ((be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_protect()
4591 		ptpaddr = be64toh(*l3e);  in mmu_radix_protect()
4629 			obits = pbits = be64toh(*pte);  in mmu_radix_protect()
4692 		if (be64toh(*pte) != pa) {  in mmu_radix_qenter()
4693 			oldpte |= be64toh(*pte);  in mmu_radix_qenter()
4821 		pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);  in _pmap_unwire_ptp()
4828 		pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);  in _pmap_unwire_ptp()
4939 	oldpde = be64toh(*l3e);  in pmap_demote_l3e_locked()
4992 	KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME),  in pmap_demote_l3e_locked()
5000 	if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE))  in pmap_demote_l3e_locked()
5079 	oldpde = be64toh(pte_load_clear(pdq));  in pmap_remove_l3e()
5111 	return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));  in pmap_remove_l3e()
5126 	oldpte = be64toh(pte_load_clear(ptq));  in pmap_remove_pte()
5160 	if ((be64toh(*l3e) & RPTE_VALID) == 0) {  in pmap_remove_page()
5164 	if ((be64toh(*pte) & RPTE_VALID) == 0) {  in pmap_remove_page()
5169 	invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);  in pmap_remove_page()
5203 		if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {  in pmap_remove_ptes()
5251 		if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) {  in mmu_radix_remove()
5262 		if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) {  in mmu_radix_remove()
5270 		if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_remove()
5285 		ptpaddr = be64toh(*l3e);  in mmu_radix_remove()
5310 				ptpaddr = be64toh(*l3e);  in mmu_radix_remove()
5389 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found"  in mmu_radix_remove_all()
5392 		tpte = be64toh(pte_load_clear(pte));  in mmu_radix_remove_all()
5403 		pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);  in mmu_radix_remove_all()
5491 				ptel3e = be64toh(*pte);  in mmu_radix_remove_pages()
5493 				tpte = be64toh(*pte);  in mmu_radix_remove_pages()
5500 					tpte = be64toh(*pte);  in mmu_radix_remove_pages()
5654 		if ((be64toh(*l3e) & PG_RW) != 0)  in mmu_radix_remove_write()
5677 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,  in mmu_radix_remove_write()
5682 		oldpte = be64toh(*pte);  in mmu_radix_remove_write()
5722 		if ((be64toh(*l1e) & PG_V) == 0) {  in mmu_radix_unwire()
5729 		if ((be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_unwire()
5739 		if ((be64toh(*l3e) & PG_V) == 0)  in mmu_radix_unwire()
5741 		if ((be64toh(*l3e) & RPTE_LEAF) != 0) {  in mmu_radix_unwire()
5742 			if ((be64toh(*l3e) & PG_W) == 0)  in mmu_radix_unwire()
5744 				    (uintmax_t)(be64toh(*l3e)));  in mmu_radix_unwire()
5763 			if ((be64toh(*pte) & PG_V) == 0)  in mmu_radix_unwire()
5765 			if ((be64toh(*pte) & PG_W) == 0)  in mmu_radix_unwire()
5767 				    (uintmax_t)(be64toh(*pte)));  in mmu_radix_unwire()
5814 	if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) {  in mmu_radix_mincore()
5815 		if (be64toh(*l3ep) & RPTE_LEAF) {  in mmu_radix_mincore()
5816 			pte = be64toh(*l3ep);  in mmu_radix_mincore()
5818 			pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) &  in mmu_radix_mincore()
5823 			pte = be64toh(*pmap_l3e_to_pte(l3ep, addr));  in mmu_radix_mincore()
6003 		opte = be64toh(*pte);  in pmap_pte_attr()
6021 	oldpdpe = be64toh(*l2e);  in pmap_demote_l2e()
6075 		if (be64toh(l3e) & RPTE_LEAF) {  in mmu_radix_kextract()
6076 			pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);  in mmu_radix_kextract()
6087 			pa = be64toh(*pmap_l3e_to_pte(&l3e, va));  in mmu_radix_kextract()
6244 		if (be64toh(*l2e) & RPTE_LEAF) {  in pmap_change_attr_locked()
6250 			if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) {  in pmap_change_attr_locked()
6273 		if (be64toh(*l3e) & RPTE_LEAF) {  in pmap_change_attr_locked()
6279 			if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) {  in pmap_change_attr_locked()
6311 		if (be64toh(*l2e) & RPTE_LEAF) {  in pmap_change_attr_locked()
6312 			if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) {  in pmap_change_attr_locked()
6321 					pa_start = be64toh(*l2e) & PG_PS_FRAME;  in pmap_change_attr_locked()
6323 				} else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME))  in pmap_change_attr_locked()
6333 					pa_start = be64toh(*l2e) & PG_PS_FRAME;  in pmap_change_attr_locked()
6341 		if (be64toh(*l3e) & RPTE_LEAF) {  in pmap_change_attr_locked()
6342 			if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) {  in pmap_change_attr_locked()
6348 			    (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) {  in pmap_change_attr_locked()
6351 					pa_start = be64toh(*l3e) & PG_PS_FRAME;  in pmap_change_attr_locked()
6353 				} else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME))  in pmap_change_attr_locked()
6363 					pa_start = be64toh(*l3e) & PG_PS_FRAME;  in pmap_change_attr_locked()
6370 			if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) {  in pmap_change_attr_locked()
6376 			    (be64toh(*pte) & PG_FRAME) < dmaplimit) {  in pmap_change_attr_locked()
6379 					pa_start = be64toh(*pte) & PG_FRAME;  in pmap_change_attr_locked()
6381 				} else if (pa_end == (be64toh(*pte) & PG_FRAME))  in pmap_change_attr_locked()
6391 					pa_start = be64toh(*pte) & PG_FRAME;  in pmap_change_attr_locked()
6450 		if ((be64toh(*l2e) & PG_V) == 0) {  in mmu_radix_page_array_startup()
6457 		if ((be64toh(*pde) & PG_V) != 0)  in mmu_radix_page_array_startup()
6482 	db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e));  in pmap_pte_walk()
6483 	if ((be64toh(*l1e) & PG_V) == 0) {  in pmap_pte_walk()
6488 	db_printf(" l2e %#016lx", be64toh(*l2e));  in pmap_pte_walk()
6489 	if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) {  in pmap_pte_walk()
6494 	db_printf(" l3e %#016lx", be64toh(*l3e));  in pmap_pte_walk()
6495 	if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) {  in pmap_pte_walk()
6500 	db_printf(" pte %#016lx\n", be64toh(*pte));  in pmap_pte_walk()