Lines Matching refs:pvo
161 #define PVO_PADDR(pvo) ((pvo)->pvo_pte.pte.pte_lo & PTE_RPGN) argument
1035 struct pvo_entry key, *pvo; in moea_unwire() local
1039 for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); in moea_unwire()
1040 pvo != NULL && PVO_VADDR(pvo) < eva; in moea_unwire()
1041 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { in moea_unwire()
1042 if ((pvo->pvo_vaddr & PVO_WIRED) == 0) in moea_unwire()
1043 panic("moea_unwire: pvo %p is missing PVO_WIRED", pvo); in moea_unwire()
1044 pvo->pvo_vaddr &= ~PVO_WIRED; in moea_unwire()
1273 struct pvo_entry *pvo; in moea_extract() local
1277 pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); in moea_extract()
1278 if (pvo == NULL) in moea_extract()
1281 pa = PVO_PADDR(pvo) | (va & ADDR_POFF); in moea_extract()
1294 struct pvo_entry *pvo; in moea_extract_and_hold() local
1299 pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); in moea_extract_and_hold()
1300 if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID) && in moea_extract_and_hold()
1301 ((pvo->pvo_pte.pte.pte_lo & PTE_PP) == PTE_RW || in moea_extract_and_hold()
1303 m = PHYS_TO_VM_PAGE(PVO_PADDR(pvo)); in moea_extract_and_hold()
1360 struct pvo_entry *pvo; in moea_is_prefaultable() local
1364 pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); in moea_is_prefaultable()
1365 rv = pvo == NULL || (pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0; in moea_is_prefaultable()
1391 struct pvo_entry *pvo; in moea_remove_write() local
1405 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { in moea_remove_write()
1406 pmap = pvo->pvo_pmap; in moea_remove_write()
1408 if ((pvo->pvo_pte.pte.pte_lo & PTE_PP) != PTE_BR) { in moea_remove_write()
1409 pt = moea_pvo_to_pte(pvo, -1); in moea_remove_write()
1410 pvo->pvo_pte.pte.pte_lo &= ~PTE_PP; in moea_remove_write()
1411 pvo->pvo_pte.pte.pte_lo |= PTE_BR; in moea_remove_write()
1413 moea_pte_synch(pt, &pvo->pvo_pte.pte); in moea_remove_write()
1414 lo |= pvo->pvo_pte.pte.pte_lo; in moea_remove_write()
1415 pvo->pvo_pte.pte.pte_lo &= ~PTE_CHG; in moea_remove_write()
1416 moea_pte_change(pt, &pvo->pvo_pte.pte, in moea_remove_write()
1417 pvo->pvo_vaddr); in moea_remove_write()
1462 struct pvo_entry *pvo; in moea_page_set_memattr() local
1477 LIST_FOREACH(pvo, pvo_head, pvo_vlink) { in moea_page_set_memattr()
1478 pmap = pvo->pvo_pmap; in moea_page_set_memattr()
1480 pt = moea_pvo_to_pte(pvo, -1); in moea_page_set_memattr()
1481 pvo->pvo_pte.pte.pte_lo &= ~PTE_WIMG; in moea_page_set_memattr()
1482 pvo->pvo_pte.pte.pte_lo |= lo; in moea_page_set_memattr()
1484 moea_pte_change(pt, &pvo->pvo_pte.pte, in moea_page_set_memattr()
1485 pvo->pvo_vaddr); in moea_page_set_memattr()
1486 if (pvo->pvo_pmap == kernel_pmap) in moea_page_set_memattr()
1538 struct pvo_entry *pvo; in moea_kextract() local
1549 pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); in moea_kextract()
1550 KASSERT(pvo != NULL, ("moea_kextract: no addr found")); in moea_kextract()
1551 pa = PVO_PADDR(pvo) | (va & ADDR_POFF); in moea_kextract()
1664 struct pvo_entry *pvo; in moea_page_exists_quick() local
1672 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { in moea_page_exists_quick()
1673 if (pvo->pvo_pmap == pmap) { in moea_page_exists_quick()
1700 struct pvo_entry *pvo; in moea_page_wired_mappings() local
1707 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) in moea_page_wired_mappings()
1708 if ((pvo->pvo_vaddr & PVO_WIRED) != 0) in moea_page_wired_mappings()
1796 struct pvo_entry *pvo, *tpvo, key; in moea_protect() local
1810 for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); in moea_protect()
1811 pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { in moea_protect()
1812 tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); in moea_protect()
1818 pt = moea_pvo_to_pte(pvo, -1); in moea_protect()
1822 pvo->pvo_pte.pte.pte_lo &= ~PTE_PP; in moea_protect()
1823 pvo->pvo_pte.pte.pte_lo |= PTE_BR; in moea_protect()
1829 moea_pte_change(pt, &pvo->pvo_pte.pte, pvo->pvo_vaddr); in moea_protect()
1896 struct pvo_entry *pvo, *tpvo, key; in moea_remove() local
1901 for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); in moea_remove()
1902 pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { in moea_remove()
1903 tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); in moea_remove()
1904 moea_pvo_remove(pvo, -1); in moea_remove()
1918 struct pvo_entry *pvo, *next_pvo; in moea_remove_all() local
1923 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { in moea_remove_all()
1924 next_pvo = LIST_NEXT(pvo, pvo_vlink); in moea_remove_all()
1926 pmap = pvo->pvo_pmap; in moea_remove_all()
1928 moea_pvo_remove(pvo, -1); in moea_remove_all()
1942 struct pvo_entry *pvo; in moea_mincore() local
1950 pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); in moea_mincore()
1951 if (pvo != NULL) { in moea_mincore()
1952 pa = PVO_PADDR(pvo); in moea_mincore()
1954 managed = (pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED; in moea_mincore()
2036 struct pvo_entry *pvo; in moea_pvo_enter() local
2059 LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { in moea_pvo_enter()
2060 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { in moea_pvo_enter()
2061 if (PVO_PADDR(pvo) == pa && in moea_pvo_enter()
2062 (pvo->pvo_pte.pte.pte_lo & PTE_PP) == in moea_pvo_enter()
2071 (pvo->pvo_vaddr & PVO_WIRED) == 0) { in moea_pvo_enter()
2072 pvo->pvo_vaddr |= PVO_WIRED; in moea_pvo_enter()
2075 (pvo->pvo_vaddr & PVO_WIRED) != 0) { in moea_pvo_enter()
2076 pvo->pvo_vaddr &= ~PVO_WIRED; in moea_pvo_enter()
2081 moea_pvo_remove(pvo, -1); in moea_pvo_enter()
2090 pvo = uma_zalloc(zone, M_NOWAIT); in moea_pvo_enter()
2097 pvo = &moea_bpvo_pool[moea_bpvo_pool_index]; in moea_pvo_enter()
2102 if (pvo == NULL) { in moea_pvo_enter()
2108 pvo->pvo_vaddr = va; in moea_pvo_enter()
2109 pvo->pvo_pmap = pm; in moea_pvo_enter()
2110 LIST_INSERT_HEAD(&moea_pvo_table[ptegidx], pvo, pvo_olink); in moea_pvo_enter()
2111 pvo->pvo_vaddr &= ~ADDR_POFF; in moea_pvo_enter()
2113 pvo->pvo_vaddr |= PVO_WIRED; in moea_pvo_enter()
2115 pvo->pvo_vaddr |= PVO_MANAGED; in moea_pvo_enter()
2117 pvo->pvo_vaddr |= PVO_BOOTSTRAP; in moea_pvo_enter()
2119 moea_pte_create(&pvo->pvo_pte.pte, sr, va, pa | pte_lo); in moea_pvo_enter()
2124 RB_INSERT(pvo_tree, &pm->pmap_pvo, pvo); in moea_pvo_enter()
2132 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); in moea_pvo_enter()
2134 if (pvo->pvo_vaddr & PVO_WIRED) in moea_pvo_enter()
2138 i = moea_pte_insert(ptegidx, &pvo->pvo_pte.pte); in moea_pvo_enter()
2141 PVO_PTEGIDX_SET(pvo, i); in moea_pvo_enter()
2152 moea_pvo_remove(struct pvo_entry *pvo, int pteidx) in moea_pvo_remove() argument
2160 pt = moea_pvo_to_pte(pvo, pteidx); in moea_pvo_remove()
2162 moea_pte_unset(pt, &pvo->pvo_pte.pte, pvo->pvo_vaddr); in moea_pvo_remove()
2164 PVO_PTEGIDX_CLR(pvo); in moea_pvo_remove()
2172 pvo->pvo_pmap->pm_stats.resident_count--; in moea_pvo_remove()
2173 if (pvo->pvo_vaddr & PVO_WIRED) in moea_pvo_remove()
2174 pvo->pvo_pmap->pm_stats.wired_count--; in moea_pvo_remove()
2179 LIST_REMOVE(pvo, pvo_vlink); in moea_pvo_remove()
2180 RB_REMOVE(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo); in moea_pvo_remove()
2186 if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED) { in moea_pvo_remove()
2189 pg = PHYS_TO_VM_PAGE(PVO_PADDR(pvo)); in moea_pvo_remove()
2191 moea_attr_save(pg, pvo->pvo_pte.pte.pte_lo & in moea_pvo_remove()
2202 LIST_REMOVE(pvo, pvo_olink); in moea_pvo_remove()
2203 if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) in moea_pvo_remove()
2204 uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea_mpvo_zone : in moea_pvo_remove()
2205 moea_upvo_zone, pvo); in moea_pvo_remove()
2211 moea_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) in moea_pvo_pte_index() argument
2220 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); in moea_pvo_pte_index()
2221 if (pvo->pvo_pte.pte.pte_hi & PTE_HID) in moea_pvo_pte_index()
2230 struct pvo_entry *pvo; in moea_pvo_find_va() local
2239 LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { in moea_pvo_find_va()
2240 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { in moea_pvo_find_va()
2242 *pteidx_p = moea_pvo_pte_index(pvo, ptegidx); in moea_pvo_find_va()
2248 return (pvo); in moea_pvo_find_va()
2252 moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) in moea_pvo_to_pte() argument
2263 sr = va_to_sr(pvo->pvo_pmap->pm_sr, pvo->pvo_vaddr); in moea_pvo_to_pte()
2264 ptegidx = va_to_pteg(sr, pvo->pvo_vaddr); in moea_pvo_to_pte()
2265 pteidx = moea_pvo_pte_index(pvo, ptegidx); in moea_pvo_to_pte()
2271 if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { in moea_pvo_to_pte()
2273 "valid pte index", pvo); in moea_pvo_to_pte()
2276 if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { in moea_pvo_to_pte()
2278 "pvo but no valid pte", pvo); in moea_pvo_to_pte()
2281 if ((pt->pte_hi ^ (pvo->pvo_pte.pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { in moea_pvo_to_pte()
2282 if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0) { in moea_pvo_to_pte()
2284 "moea_pteg_table %p but invalid in pvo", pvo, pt); in moea_pvo_to_pte()
2287 if (((pt->pte_lo ^ pvo->pvo_pte.pte.pte_lo) & ~(PTE_CHG|PTE_REF)) in moea_pvo_to_pte()
2290 "pte %p in moea_pteg_table", pvo, pt); in moea_pvo_to_pte()
2297 if (pvo->pvo_pte.pte.pte_hi & PTE_VALID) { in moea_pvo_to_pte()
2299 "moea_pteg_table but valid in pvo: %8x, %8x", pvo, pt, pvo->pvo_pte.pte.pte_hi, pt->pte_hi); in moea_pvo_to_pte()
2313 struct pvo_entry *pvo; in moea_pte_spill() local
2336 LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { in moea_pte_spill()
2341 moea_pte_match(&pvo->pvo_pte.pte, sr, addr, in moea_pte_spill()
2342 pvo->pvo_pte.pte.pte_hi & PTE_HID)) { in moea_pte_spill()
2347 j = moea_pte_insert(ptegidx, &pvo->pvo_pte.pte); in moea_pte_spill()
2350 PVO_PTEGIDX_SET(pvo, j); in moea_pte_spill()
2356 source_pvo = pvo; in moea_pte_spill()
2367 moea_pte_compare(pt, &pvo->pvo_pte.pte)) { in moea_pte_spill()
2368 victim_pvo = pvo; in moea_pte_spill()
2388 LIST_FOREACH(pvo, &moea_pvo_table[ptegidx ^ moea_pteg_mask], in moea_pte_spill()
2394 if (moea_pte_compare(pt, &pvo->pvo_pte.pte)) { in moea_pte_spill()
2395 victim_pvo = pvo; in moea_pte_spill()
2427 struct pvo_entry *pvo_walk, *pvo = NULL; in moea_pte_spillable_ident() local
2441 pvo = pvo_walk; in moea_pte_spillable_ident()
2448 return (pvo); in moea_pte_spillable_ident()
2531 struct pvo_entry *pvo; in moea_query_bit() local
2538 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { in moea_query_bit()
2543 if (pvo->pvo_pte.pte.pte_lo & ptebit) { in moea_query_bit()
2555 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { in moea_query_bit()
2561 pt = moea_pvo_to_pte(pvo, -1); in moea_query_bit()
2563 moea_pte_synch(pt, &pvo->pvo_pte.pte); in moea_query_bit()
2565 if (pvo->pvo_pte.pte.pte_lo & ptebit) { in moea_query_bit()
2579 struct pvo_entry *pvo; in moea_clear_bit() local
2603 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { in moea_clear_bit()
2604 pt = moea_pvo_to_pte(pvo, -1); in moea_clear_bit()
2606 moea_pte_synch(pt, &pvo->pvo_pte.pte); in moea_clear_bit()
2607 if (pvo->pvo_pte.pte.pte_lo & ptebit) { in moea_clear_bit()
2609 moea_pte_clear(pt, PVO_VADDR(pvo), ptebit); in moea_clear_bit()
2613 pvo->pvo_pte.pte.pte_lo &= ~ptebit; in moea_clear_bit()
2746 struct pvo_entry *pvo; in moea_sync_icache() local
2755 pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); in moea_sync_icache()
2756 if (pvo != NULL) { in moea_sync_icache()
2757 pa = PVO_PADDR(pvo) | (va & ADDR_POFF); in moea_sync_icache()
2778 struct pvo_entry *pvo; in moea_scan_init() local
2814 pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); in moea_scan_init()
2815 if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID)) in moea_scan_init()
2827 pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, in moea_scan_init()
2829 if (pvo == NULL || in moea_scan_init()
2830 !(pvo->pvo_pte.pte.pte_hi & PTE_VALID)) in moea_scan_init()