129 #define	pa_index(pa)	((pa) >> PV_LOCK_SHIFT)  argument
137 #define PV_LOCK_IDX(pa)	((pa_index(pa) * (((pa) >> 45) + 1)) % PV_LOCK_COUNT)  argument
139 #define PV_LOCK_IDX(pa)	(pa_index(pa) % PV_LOCK_COUNT)  argument
141 #define PV_LOCKPTR(pa)	((struct mtx *)(&pv_lock[PV_LOCK_IDX(pa)]))  argument
142 #define PV_LOCK(pa)		mtx_lock(PV_LOCKPTR(pa))  argument
143 #define PV_UNLOCK(pa)		mtx_unlock(PV_LOCKPTR(pa))  argument
144 #define PV_LOCKASSERT(pa) 	mtx_assert(PV_LOCKPTR(pa), MA_OWNED)  argument
154 moea64_sp_pv_lock(vm_paddr_t pa)  in moea64_sp_pv_lock()  argument
159 	pa_end = pa + (HPT_SP_SIZE - PV_LOCK_SIZE);  in moea64_sp_pv_lock()
161 		mtx_lock_flags(PV_LOCKPTR(pa), MTX_DUPOK);  in moea64_sp_pv_lock()
162 		if (pa == pa_end)  in moea64_sp_pv_lock()
164 		pa += PV_LOCK_SIZE;  in moea64_sp_pv_lock()
169 moea64_sp_pv_unlock(vm_paddr_t pa)  in moea64_sp_pv_unlock()  argument
174 	pa_end = pa;  in moea64_sp_pv_unlock()
175 	pa += HPT_SP_SIZE - PV_LOCK_SIZE;  in moea64_sp_pv_unlock()
177 		mtx_unlock_flags(PV_LOCKPTR(pa), MTX_DUPOK);  in moea64_sp_pv_unlock()
178 		if (pa == pa_end)  in moea64_sp_pv_unlock()
180 		pa -= PV_LOCK_SIZE;  in moea64_sp_pv_unlock()
184 #define	SP_PV_LOCK_ALIGNED(pa)		moea64_sp_pv_lock(pa)  argument
185 #define	SP_PV_UNLOCK_ALIGNED(pa)	moea64_sp_pv_unlock(pa)  argument
186 #define	SP_PV_LOCK(pa)			moea64_sp_pv_lock((pa) & ~HPT_SP_MASK)  argument
187 #define	SP_PV_UNLOCK(pa)		moea64_sp_pv_unlock((pa) & ~HPT_SP_MASK)  argument
299 			    vm_paddr_t pa, vm_size_t sz);
439 void moea64_dumpsys_map(vm_paddr_t pa, size_t sz,
523  * For superpages, the lower bits are not stored on pvo_pte.pa and must be
529 	vm_paddr_t pa;  in moea64_pvo_paddr()  local
531 	pa = (pvo)->pvo_pte.pa & LPTE_RPGN;  in moea64_pvo_paddr()
534 		pa &= ~HPT_SP_MASK; /* This is needed to clear LPTE_LP bits. */  in moea64_pvo_paddr()
535 		pa |= PVO_VADDR(pvo) & HPT_SP_MASK;  in moea64_pvo_paddr()
537 	return (pa);  in moea64_pvo_paddr()
617 	lpte->pte_lo = pvo->pvo_pte.pa; /* Includes WIMG bits */  in moea64_pte_from_pvo()
628 moea64_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)  in moea64_calc_wimg()  argument
654 		if ((pa >= pregions[i].mr_start) &&  in moea64_calc_wimg()
655 		    (pa < (pregions[i].mr_start + pregions[i].mr_size))) {  in moea64_calc_wimg()
733 		CTR3(KTR_PMAP, "translation: pa=%#zx va=%#x len=%#x",  in moea64_add_ofw_mappings()
813 moea64_kenter_large(vm_offset_t va, vm_paddr_t pa, uint64_t attr, int bootstrap)  in moea64_kenter_large()  argument
828 	pvo->pvo_pte.pa = pa | pte_lo;  in moea64_kenter_large()
840 	vm_paddr_t pa, pkernelstart, pkernelend;  in moea64_setup_direct_map()  local
852 		  for (pa = pregions[i].mr_start; pa < pregions[i].mr_start +  in moea64_setup_direct_map()
853 		     pregions[i].mr_size; pa += moea64_large_page_size) {  in moea64_setup_direct_map()
855 			if (pa & moea64_large_page_mask) {  in moea64_setup_direct_map()
856 				pa &= moea64_large_page_mask;  in moea64_setup_direct_map()
859 			if (pa + moea64_large_page_size >  in moea64_setup_direct_map()
863 			moea64_kenter_large(PHYS_TO_DMAP(pa), pa, pte_lo, 1);  in moea64_setup_direct_map()
880 		for (pa = kernelstart & ~PAGE_MASK; pa < kernelend;  in moea64_setup_direct_map()
881 		    pa += PAGE_SIZE)  in moea64_setup_direct_map()
882 			moea64_kenter(pa, pa);  in moea64_setup_direct_map()
886 		for (pa = pkernelstart & ~PAGE_MASK; pa < pkernelend;  in moea64_setup_direct_map()
887 		    pa += PAGE_SIZE)  in moea64_setup_direct_map()
888 			moea64_kenter(pa | DMAP_BASE_ADDRESS, pa);  in moea64_setup_direct_map()
894 		for (pa = off; pa < off + size; pa += PAGE_SIZE)  in moea64_setup_direct_map()
895 			moea64_kenter(pa, pa);  in moea64_setup_direct_map()
898 		for (pa = EXC_RSVD; pa < EXC_LAST; pa += PAGE_SIZE)  in moea64_setup_direct_map()
899 			moea64_kenter(pa | DMAP_BASE_ADDRESS, pa);  in moea64_setup_direct_map()
916 	const vm_paddr_t *pa = a, *pb = b;  in pa_cmp()  local
918 	if (*pa < *pb)  in pa_cmp()
920 	else if (*pa > *pb)  in pa_cmp()
1144 	vm_offset_t	pa, va;  in moea64_late_bootstrap()  local
1218 	pa = moea64_bootstrap_alloc(kstack_pages * PAGE_SIZE, PAGE_SIZE);  in moea64_late_bootstrap()
1221 	CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);  in moea64_late_bootstrap()
1225 		moea64_kenter(va, pa);  in moea64_late_bootstrap()
1226 		pa += PAGE_SIZE;  in moea64_late_bootstrap()
1233 	pa = msgbuf_phys = moea64_bootstrap_alloc(msgbufsize, PAGE_SIZE);  in moea64_late_bootstrap()
1238 		moea64_kenter(va, pa);  in moea64_late_bootstrap()
1239 		pa += PAGE_SIZE;  in moea64_late_bootstrap()
1246 	pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE);  in moea64_late_bootstrap()
1251 		moea64_kenter(va, pa);  in moea64_late_bootstrap()
1252 		pa += PAGE_SIZE;  in moea64_late_bootstrap()
1397 	vm_paddr_t pa;  in moea64_mincore()  local
1406 		pa = PVO_PADDR(pvo);  in moea64_mincore()
1407 		m = PHYS_TO_VM_PAGE(pa);  in moea64_mincore()
1434 		*pap = pa;  in moea64_mincore()
1442  * special scratch PTE to the PA we want to zero or copy. Because
1448 void moea64_set_scratchpage_pa(int which, vm_paddr_t pa)  in moea64_set_scratchpage_pa()  argument
1457 	pvo->pvo_pte.pa =  in moea64_set_scratchpage_pa()
1458 	    moea64_calc_wimg(pa, VM_MEMATTR_DEFAULT) | (uint64_t)pa;  in moea64_set_scratchpage_pa()
1548 	vm_paddr_t pa = VM_PAGE_TO_PHYS(m);  in moea64_zero_page_area()  local
1554 		bzero((caddr_t)(uintptr_t)PHYS_TO_DMAP(pa) + off, size);  in moea64_zero_page_area()
1557 		moea64_set_scratchpage_pa(0, pa);  in moea64_zero_page_area()
1569 	vm_paddr_t pa = VM_PAGE_TO_PHYS(m);  in moea64_zero_page()  local
1574 	moea64_set_scratchpage_pa(0, pa);  in moea64_zero_page()
1586 	vm_paddr_t pa = VM_PAGE_TO_PHYS(m);  in moea64_zero_page_dmap()  local
1589 	va = PHYS_TO_DMAP(pa);  in moea64_zero_page_dmap()
1598 	vm_paddr_t pa = VM_PAGE_TO_PHYS(m);  in moea64_quick_enter_page()  local
1612 	pvo->pvo_pte.pa = moea64_calc_wimg(pa, pmap_page_get_memattr(m)) |  in moea64_quick_enter_page()
1613 	    (uint64_t)pa;  in moea64_quick_enter_page()
1658 	vm_paddr_t	pa;  in moea64_enter()  local
1676 	pa = VM_PAGE_TO_PHYS(m);  in moea64_enter()
1677 	pte_lo = moea64_calc_wimg(pa, pmap_page_get_memattr(m));  in moea64_enter()
1678 	pvo->pvo_pte.pa = pa | pte_lo;  in moea64_enter()
1690 	PV_LOCK(pa);  in moea64_enter()
1712 		    oldpvo->pvo_pte.pa == pvo->pvo_pte.pa &&  in moea64_enter()
1725 			PV_UNLOCK(pa);  in moea64_enter()
1738 	PV_UNLOCK(pa);  in moea64_enter()
1754 		moea64_syncicache(pmap, va, pa, PAGE_SIZE);  in moea64_enter()
1761 	 * If the VA of the entered page is not aligned with its PA,  in moea64_enter()
1769 	    (va & HPT_SP_MASK) == (pa & HPT_SP_MASK) &&  in moea64_enter()
1781 moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,  in moea64_syncicache()  argument
1799 		__syncicache((void *)(uintptr_t)pa, sz);  in moea64_syncicache()
1803 		__syncicache((void *)(uintptr_t)PHYS_TO_DMAP(pa), sz);  in moea64_syncicache()
1809 		moea64_set_scratchpage_pa(1, pa & ~ADDR_POFF);  in moea64_syncicache()
1874 	vm_paddr_t pa;  in moea64_extract()  local
1879 		pa = 0;  in moea64_extract()
1881 		pa = PVO_PADDR(pvo) | (va - PVO_VADDR(pvo));  in moea64_extract()
1884 	return (pa);  in moea64_extract()
1939 	pvo->pvo_pte.pa = VM_PAGE_TO_PHYS(m) | LPTE_M;  in moea64_uma_page_alloc()
2134 	CTR3(KTR_PMAP, "%s: pa=%#jx, ma=%#x",  in moea64_page_set_memattr()
2157 			pvo->pvo_pte.pa &= ~LPTE_WIMG;  in moea64_page_set_memattr()
2158 			pvo->pvo_pte.pa |= lo;  in moea64_page_set_memattr()
2185 moea64_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)  in moea64_kenter_attr()  argument
2196 	pvo->pvo_pte.pa = (pa & ~ADDR_POFF) | moea64_calc_wimg(pa, ma);  in moea64_kenter_attr()
2214 		panic("moea64_kenter: failed to enter va %#zx pa %#jx: %d", va,  in moea64_kenter_attr()
2215 		    (uintmax_t)pa, error);  in moea64_kenter_attr()
2219 moea64_kenter(vm_offset_t va, vm_paddr_t pa)  in moea64_kenter()  argument
2222 	moea64_kenter_attr(va, pa, VM_MEMATTR_DEFAULT);  in moea64_kenter()
2233 	vm_paddr_t pa;  in moea64_kextract()  local
2247 	pa = PVO_PADDR(pvo) | (va - PVO_VADDR(pvo));  in moea64_kextract()
2249 	return (pa);  in moea64_kextract()
2584 	    (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {  in moea64_pvo_protect()
3160 moea64_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)  in moea64_dev_direct_mapped()  argument
3166 	if (hw_direct_map && mem_valid(pa, size) == 0)  in moea64_dev_direct_mapped()
3170 	ppa = pa & ~ADDR_POFF;  in moea64_dev_direct_mapped()
3173 	    ppa < pa + size; ppa += PAGE_SIZE,  in moea64_dev_direct_mapped()
3192 moea64_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)  in moea64_mapdev_attr()  argument
3196 	ppa = trunc_page(pa);  in moea64_mapdev_attr()
3197 	offset = pa & PAGE_MASK;  in moea64_mapdev_attr()
3216 moea64_mapdev(vm_paddr_t pa, vm_size_t size)  in moea64_mapdev()  argument
3219 	return moea64_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT);  in moea64_mapdev()
3241 	vm_paddr_t pa;  in moea64_sync_icache()  local
3252 		if (pvo != NULL && !(pvo->pvo_pte.pa & LPTE_I)) {  in moea64_sync_icache()
3253 			pa = PVO_PADDR(pvo) | (va & ADDR_POFF);  in moea64_sync_icache()
3254 			moea64_syncicache(pm, va, pa, len);  in moea64_sync_icache()
3263 moea64_dumpsys_map(vm_paddr_t pa, size_t sz, void **va)  in moea64_dumpsys_map()  argument
3266 	*va = (void *)(uintptr_t)pa;  in moea64_dumpsys_map()
3338 	vm_paddr_t pa, pa_end;  in moea64_scan_pmap()  local
3372 		pa = PVO_PADDR(pvo);  in moea64_scan_pmap()
3375 			pa_end = pa + lpsize;  in moea64_scan_pmap()
3376 			for (; pa < pa_end; pa += PAGE_SIZE) {  in moea64_scan_pmap()
3377 				if (vm_phys_is_dumpable(pa))  in moea64_scan_pmap()
3378 					vm_page_dump_add(dump_bitset, pa);  in moea64_scan_pmap()
3381 			if (vm_phys_is_dumpable(pa))  in moea64_scan_pmap()
3382 				vm_page_dump_add(dump_bitset, pa);  in moea64_scan_pmap()
3419 moea64_map_range(vm_offset_t va, vm_paddr_t pa, vm_size_t npages)  in moea64_map_range()  argument
3424 		    (pa & moea64_large_page_mask) == 0 &&  in moea64_map_range()
3428 			moea64_kenter_large(va, pa, 0, 0);  in moea64_map_range()
3430 			pa += moea64_large_page_size;  in moea64_map_range()
3434 			moea64_kenter(va, pa);  in moea64_map_range()
3435 			pa += PAGE_SIZE;  in moea64_map_range()
3445 	vm_paddr_t pa;  in moea64_page_array_startup()  local
3456 		pa = vm_phys_early_alloc(0, size);  in moea64_page_array_startup()
3458 		    pa, pa + size, VM_PROT_READ | VM_PROT_WRITE);  in moea64_page_array_startup()
3498 		pa = vm_phys_early_alloc(i, size);  in moea64_page_array_startup()
3500 		moea64_map_range(va, pa, size >> PAGE_SHIFT);  in moea64_page_array_startup()
3698 	vm_paddr_t pa, spa;  in moea64_sp_enter()  local
3712 	CTR5(KTR_PMAP, "%s: va=%#jx, pa=%#jx, prot=%#x, flags=%#x, psind=1",  in moea64_sp_enter()
3720 	spa = pa = VM_PAGE_TO_PHYS(sm);  in moea64_sp_enter()
3749 	    i++, va += PAGE_SIZE, pa += PAGE_SIZE, m++) {  in moea64_sp_enter()
3753 		pvo->pvo_pte.pa = (pa & ~HPT_SP_MASK) | LPTE_LP_4K_16M |  in moea64_sp_enter()
3754 		    moea64_calc_wimg(pa, pmap_page_get_memattr(m));  in moea64_sp_enter()
3797 	if (sync && (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)  in moea64_sp_enter()
3813 	vm_paddr_t pa, pa_end;  in moea64_sp_promote()  local
3823 	pa = VM_PAGE_TO_PHYS(m) & ~HPT_SP_MASK;  in moea64_sp_promote()
3824 	m = PHYS_TO_VM_PAGE(pa);  in moea64_sp_promote()
3845 	for (pa_end = pa + HPT_SP_SIZE;  in moea64_sp_promote()
3846 	    pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) {  in moea64_sp_promote()
3853 		if (PVO_PADDR(pvo) != pa) {  in moea64_sp_promote()
3857 			    (uintmax_t)PVO_PADDR(pvo), (uintmax_t)pa);  in moea64_sp_promote()
3879 		if ((first->pvo_pte.pa & LPTE_WIMG) !=  in moea64_sp_promote()
3880 		    (pvo->pvo_pte.pa & LPTE_WIMG)) {  in moea64_sp_promote()
3884 			    (uintmax_t)(pvo->pvo_pte.pa & LPTE_WIMG),  in moea64_sp_promote()
3885 			    (uintmax_t)(first->pvo_pte.pa & LPTE_WIMG));  in moea64_sp_promote()
3911 		pvo->pvo_pte.pa &= ADDR_POFF | ~HPT_SP_MASK;  in moea64_sp_promote()
3912 		pvo->pvo_pte.pa |= LPTE_LP_4K_16M;  in moea64_sp_promote()
3942 	vm_paddr_t pa;  in moea64_sp_demote_aligned()  local
3957 	pa = PVO_PADDR(pvo);  in moea64_sp_demote_aligned()
3958 	m = PHYS_TO_VM_PAGE(pa);  in moea64_sp_demote_aligned()
3963 	    va += PAGE_SIZE, pa += PAGE_SIZE) {  in moea64_sp_demote_aligned()
3968 		pvo->pvo_pte.pa &= ~LPTE_RPGN;  in moea64_sp_demote_aligned()
3969 		pvo->pvo_pte.pa |= pa;  in moea64_sp_demote_aligned()
4065 	KASSERT(m != NULL, ("%s: missing vm page for pa %#jx",  in moea64_sp_protect()
4088 	    (sp->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {  in moea64_sp_protect()
4190 		CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",  in moea64_sp_query()
4199 	CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",  in moea64_sp_query()
4222 		CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",  in moea64_sp_pvo_clear()
4249 	CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",  in moea64_sp_pvo_clear()
4301 		CTR4(KTR_PMAP, "%s: ptebit=%#jx, va=%#jx, pa=%#jx",  in moea64_sp_clear()
4312 			CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",  in moea64_sp_clear()