Lines Matching refs:ht
111 static void htable_free(htable_t *ht);
112 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
113 static void x86pte_release_pagetable(htable_t *ht);
114 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
357 htable_put_reserve(htable_t *ht) in htable_put_reserve() argument
359 ht->ht_hat = NULL; /* no longer tied to a hat */ in htable_put_reserve()
360 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_put_reserve()
363 ht->ht_next = htable_reserve_pool; in htable_put_reserve()
364 htable_reserve_pool = ht; in htable_put_reserve()
375 htable_t *ht = NULL; in htable_get_reserve() local
379 ht = htable_reserve_pool; in htable_get_reserve()
380 ASSERT(ht != NULL); in htable_get_reserve()
381 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_get_reserve()
382 htable_reserve_pool = ht->ht_next; in htable_get_reserve()
387 return (ht); in htable_get_reserve()
396 htable_t *ht; in htable_initial_reserve() local
400 ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP); in htable_initial_reserve()
401 ASSERT(ht != NULL); in htable_initial_reserve()
404 ht->ht_pfn = PFN_INVALID; in htable_initial_reserve()
405 htable_put_reserve(ht); in htable_initial_reserve()
416 htable_t *ht; in htable_adjust_reserve() local
423 ht = htable_get_reserve(); in htable_adjust_reserve()
424 if (ht == NULL) in htable_adjust_reserve()
426 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_adjust_reserve()
427 kmem_cache_free(htable_cache, ht); in htable_adjust_reserve()
440 htable_t *higher, *ht; in htable_steal_active() local
449 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_steal_active()
454 if (ht->ht_busy != 0 || in htable_steal_active()
455 (ht->ht_flags & HTABLE_SHARED_PFN) || in htable_steal_active()
456 ht->ht_level > 0 || ht->ht_valid_cnt > threshold || in htable_steal_active()
457 ht->ht_lock_cnt != 0) in htable_steal_active()
466 ++ht->ht_busy; in htable_steal_active()
473 for (e = 0, va = ht->ht_vaddr; in htable_steal_active()
474 e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 && in htable_steal_active()
475 ht->ht_busy == 1 && ht->ht_lock_cnt == 0; in htable_steal_active()
477 pte = x86pte_get(ht, e); in htable_steal_active()
480 hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL, in htable_steal_active()
490 if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 || in htable_steal_active()
491 ht->ht_lock_cnt != 0) { in htable_steal_active()
492 --ht->ht_busy; in htable_steal_active()
499 higher = ht->ht_parent; in htable_steal_active()
500 unlink_ptp(higher, ht, ht->ht_vaddr); in htable_steal_active()
505 if (ht->ht_next) in htable_steal_active()
506 ht->ht_next->ht_prev = ht->ht_prev; in htable_steal_active()
508 if (ht->ht_prev) { in htable_steal_active()
509 ht->ht_prev->ht_next = ht->ht_next; in htable_steal_active()
511 ASSERT(hat->hat_ht_hash[h] == ht); in htable_steal_active()
512 hat->hat_ht_hash[h] = ht->ht_next; in htable_steal_active()
521 ht->ht_next = *list; in htable_steal_active()
522 *list = ht; in htable_steal_active()
572 htable_t *ht; in htable_steal() local
648 while ((ht = hat->hat_ht_cached) != NULL && in htable_steal()
650 hat->hat_ht_cached = ht->ht_next; in htable_steal()
651 ht->ht_next = list; in htable_steal()
652 list = ht; in htable_steal()
670 for (ht = list; (ht) && (reap); ht = ht->ht_next) { in htable_steal()
671 if (ht->ht_hat == NULL) in htable_steal()
673 ASSERT(ht->ht_hat == hat); in htable_steal()
675 ASSERT(!(ht->ht_flags & HTABLE_COPIED)); in htable_steal()
676 if (ht->ht_level == mmu.max_level) { in htable_steal()
684 ht->ht_hat = NULL; in htable_steal()
738 htable_t *ht; in htable_reap() local
761 while ((ht = list) != NULL) { in htable_reap()
762 list = ht->ht_next; in htable_reap()
764 htable_free(ht); in htable_reap()
785 htable_t *ht = NULL; in htable_alloc() local
805 ht = hat->hat_ht_cached; in htable_alloc()
806 if (ht != NULL) { in htable_alloc()
807 hat->hat_ht_cached = ht->ht_next; in htable_alloc()
810 ASSERT(ht->ht_pfn != PFN_INVALID); in htable_alloc()
815 if (ht == NULL) { in htable_alloc()
820 ht = htable_get_reserve(); in htable_alloc()
826 ht = kmem_cache_alloc(htable_cache, kmflags); in htable_alloc()
827 if (ht == NULL) in htable_alloc()
829 ht->ht_pfn = PFN_INVALID; in htable_alloc()
833 htable_put_reserve(ht); in htable_alloc()
840 if (ht != NULL && !is_bare) { in htable_alloc()
841 ht->ht_hat = hat; in htable_alloc()
842 ht->ht_pfn = ptable_alloc((uintptr_t)ht); in htable_alloc()
843 if (ht->ht_pfn == PFN_INVALID) { in htable_alloc()
845 htable_put_reserve(ht); in htable_alloc()
847 kmem_cache_free(htable_cache, ht); in htable_alloc()
848 ht = NULL; in htable_alloc()
861 while (ht == NULL && can_steal_post_boot) { in htable_alloc()
863 ht = htable_steal(1, B_FALSE); in htable_alloc()
869 if (ht != NULL) { in htable_alloc()
871 ptable_free(ht->ht_pfn); in htable_alloc()
872 ht->ht_pfn = PFN_INVALID; in htable_alloc()
877 } else if (kpm_vbase && xen_kpm_page(ht->ht_pfn, in htable_alloc()
880 ht->ht_pfn); in htable_alloc()
891 if (ht == NULL) in htable_alloc()
905 hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1); in htable_alloc()
922 ht->ht_flags = 0; in htable_alloc()
925 ht->ht_flags |= HTABLE_SHARED_PFN; in htable_alloc()
926 ht->ht_pfn = shared->ht_pfn; in htable_alloc()
927 ht->ht_lock_cnt = 0; in htable_alloc()
928 ht->ht_valid_cnt = 0; /* updated in hat_share() */ in htable_alloc()
929 ht->ht_shares = shared; in htable_alloc()
932 ht->ht_shares = NULL; in htable_alloc()
933 ht->ht_lock_cnt = 0; in htable_alloc()
934 ht->ht_valid_cnt = 0; in htable_alloc()
941 ht->ht_flags |= HTABLE_COPIED; in htable_alloc()
942 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_alloc()
949 ht->ht_hat = hat; in htable_alloc()
950 ht->ht_parent = NULL; in htable_alloc()
951 ht->ht_vaddr = vaddr; in htable_alloc()
952 ht->ht_level = level; in htable_alloc()
953 ht->ht_busy = 1; in htable_alloc()
954 ht->ht_next = NULL; in htable_alloc()
955 ht->ht_prev = NULL; in htable_alloc()
961 x86pte_zero(ht, 0, mmu.ptes_per_table); in htable_alloc()
965 (void) xen_kpm_page(ht->ht_pfn, PT_VALID); in htable_alloc()
971 return (ht); in htable_alloc()
979 htable_free(htable_t *ht) in htable_free() argument
981 hat_t *hat = ht->ht_hat; in htable_free()
989 !(ht->ht_flags & HTABLE_SHARED_PFN) && in htable_free()
992 ASSERT((ht->ht_flags & HTABLE_COPIED) == 0); in htable_free()
993 ASSERT(ht->ht_pfn != PFN_INVALID); in htable_free()
995 ht->ht_next = hat->hat_ht_cached; in htable_free()
996 hat->hat_ht_cached = ht; in htable_free()
1005 if (ht->ht_flags & HTABLE_SHARED_PFN) { in htable_free()
1006 ASSERT(ht->ht_pfn != PFN_INVALID); in htable_free()
1007 } else if (!(ht->ht_flags & HTABLE_COPIED)) { in htable_free()
1008 ptable_free(ht->ht_pfn); in htable_free()
1010 if (ht->ht_level == mmu.max_level && hat != NULL) { in htable_free()
1016 ht->ht_pfn = PFN_INVALID; in htable_free()
1022 htable_put_reserve(ht); in htable_free()
1024 kmem_cache_free(htable_cache, ht); in htable_free()
1040 htable_t *ht; in htable_purge_hat() local
1050 ht = hat->hat_ht_cached; in htable_purge_hat()
1051 if (ht == NULL) { in htable_purge_hat()
1055 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1057 htable_free(ht); in htable_purge_hat()
1066 while ((ht = hat->hat_ht_cached) != NULL) { in htable_purge_hat()
1067 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1068 htable_free(ht); in htable_purge_hat()
1075 while ((ht = hat->hat_ht_hash[h]) != NULL) { in htable_purge_hat()
1076 if (ht->ht_next) in htable_purge_hat()
1077 ht->ht_next->ht_prev = ht->ht_prev; in htable_purge_hat()
1079 if (ht->ht_prev) { in htable_purge_hat()
1080 ht->ht_prev->ht_next = ht->ht_next; in htable_purge_hat()
1082 ASSERT(hat->hat_ht_hash[h] == ht); in htable_purge_hat()
1083 hat->hat_ht_hash[h] = ht->ht_next; in htable_purge_hat()
1085 htable_free(ht); in htable_purge_hat()
1177 htable_release(htable_t *ht) in htable_release() argument
1186 while (ht != NULL) { in htable_release()
1189 hat = ht->ht_hat; in htable_release()
1190 va = ht->ht_vaddr; in htable_release()
1191 level = ht->ht_level; in htable_release()
1199 ASSERT(ht->ht_valid_cnt >= 0); in htable_release()
1200 ASSERT(ht->ht_busy > 0); in htable_release()
1201 if (ht->ht_valid_cnt > 0) in htable_release()
1203 if (ht->ht_busy > 1) in htable_release()
1205 ASSERT(ht->ht_lock_cnt == 0); in htable_release()
1211 if (!(ht->ht_flags & HTABLE_SHARED_PFN)) { in htable_release()
1233 if (ht->ht_flags & HTABLE_SHARED_PFN) { in htable_release()
1235 shared = ht->ht_shares; in htable_release()
1243 higher = ht->ht_parent; in htable_release()
1249 unlink_ptp(higher, ht, va); in htable_release()
1254 if (ht->ht_next) in htable_release()
1255 ht->ht_next->ht_prev = ht->ht_prev; in htable_release()
1257 if (ht->ht_prev) { in htable_release()
1258 ht->ht_prev->ht_next = ht->ht_next; in htable_release()
1260 ASSERT(hat->hat_ht_hash[hashval] == ht); in htable_release()
1261 hat->hat_ht_hash[hashval] = ht->ht_next; in htable_release()
1264 htable_free(ht); in htable_release()
1265 ht = higher; in htable_release()
1268 ASSERT(ht->ht_busy >= 1); in htable_release()
1269 --ht->ht_busy; in htable_release()
1276 ht = shared; in htable_release()
1289 htable_t *ht = NULL; in htable_lookup() local
1309 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) { in htable_lookup()
1310 if (ht->ht_hat == hat && in htable_lookup()
1311 ht->ht_vaddr == base && in htable_lookup()
1312 ht->ht_level == level) in htable_lookup()
1315 if (ht) in htable_lookup()
1316 ++ht->ht_busy; in htable_lookup()
1319 return (ht); in htable_lookup()
1326 htable_acquire(htable_t *ht) in htable_acquire() argument
1328 hat_t *hat = ht->ht_hat; in htable_acquire()
1329 level_t level = ht->ht_level; in htable_acquire()
1330 uintptr_t base = ht->ht_vaddr; in htable_acquire()
1342 h && h != ht; in htable_acquire()
1345 ASSERT(h == ht); in htable_acquire()
1348 ++ht->ht_busy; in htable_acquire()
1371 htable_t *ht; in htable_create() local
1378 ht = NULL; in htable_create()
1396 ht = hat->hat_htable; in htable_create()
1398 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_create()
1399 ASSERT(ht->ht_hat == hat); in htable_create()
1400 if (ht->ht_vaddr == base && in htable_create()
1401 ht->ht_level == l) in htable_create()
1410 if (ht != NULL) { in htable_create()
1415 if (l == level && shared && ht->ht_shares && in htable_create()
1416 ht->ht_shares != shared) { in htable_create()
1419 (void *)ht, (void *)shared); in htable_create()
1421 ++ht->ht_busy; in htable_create()
1427 higher = ht; in htable_create()
1444 ht = new; in htable_create()
1446 link_ptp(higher, ht, base); in htable_create()
1447 ht->ht_parent = higher; in htable_create()
1449 ht->ht_next = hat->hat_ht_hash[h]; in htable_create()
1450 ASSERT(ht->ht_prev == NULL); in htable_create()
1452 hat->hat_ht_hash[h]->ht_prev = ht; in htable_create()
1453 hat->hat_ht_hash[h] = ht; in htable_create()
1461 higher = ht; in htable_create()
1476 return (ht); in htable_create()
1492 htable_t *ht; in htable_attach() local
1500 ht = htable_get_reserve(); in htable_attach()
1502 kas.a_hat->hat_htable = ht; in htable_attach()
1503 ht->ht_hat = hat; in htable_attach()
1504 ht->ht_parent = parent; in htable_attach()
1505 ht->ht_vaddr = base; in htable_attach()
1506 ht->ht_level = level; in htable_attach()
1507 ht->ht_busy = 1; in htable_attach()
1508 ht->ht_next = NULL; in htable_attach()
1509 ht->ht_prev = NULL; in htable_attach()
1510 ht->ht_flags = 0; in htable_attach()
1511 ht->ht_pfn = pfn; in htable_attach()
1512 ht->ht_lock_cnt = 0; in htable_attach()
1513 ht->ht_valid_cnt = 0; in htable_attach()
1519 ht->ht_next = hat->hat_ht_hash[h]; in htable_attach()
1520 ASSERT(ht->ht_prev == NULL); in htable_attach()
1522 hat->hat_ht_hash[h]->ht_prev = ht; in htable_attach()
1523 hat->hat_ht_hash[h] = ht; in htable_attach()
1542 u_offset_t offset = (uintptr_t)ht; in htable_attach()
1563 for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) { in htable_attach()
1569 ++ht->ht_valid_cnt; in htable_attach()
1572 ht, PTE2PFN(pte, level)); in htable_attach()
1586 htable_release(ht); in htable_attach()
1605 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr) in htable_scan() argument
1611 int l = ht->ht_level; in htable_scan()
1615 ASSERT(va >= ht->ht_vaddr); in htable_scan()
1616 ASSERT(va <= HTABLE_LAST_PAGE(ht)); in htable_scan()
1621 e = htable_va2entry(va, ht); in htable_scan()
1627 pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0); in htable_scan()
1628 end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht)); in htable_scan()
1645 x86pte_release_pagetable(ht); in htable_scan()
1674 htable_t *ht; in htable_walk() local
1733 ht = htable_lookup(hat, va, l); in htable_walk()
1734 if (ht != NULL) { in htable_walk()
1735 pte = htable_scan(ht, &va, eaddr); in htable_walk()
1739 *htp = ht; in htable_walk()
1742 htable_release(ht); in htable_walk()
1779 htable_t *ht; in htable_getpte() local
1786 ht = htable_lookup(hat, vaddr, l); in htable_getpte()
1787 if (ht == NULL) in htable_getpte()
1789 e = htable_va2entry(vaddr, ht); in htable_getpte()
1793 *pte = x86pte_get(ht, e); in htable_getpte()
1794 return (ht); in htable_getpte()
1808 htable_t *ht; in htable_getpage() local
1812 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level); in htable_getpage()
1813 if (ht == NULL) in htable_getpage()
1819 if (PTE_ISPAGE(pte, ht->ht_level)) in htable_getpage()
1820 return (ht); in htable_getpage()
1821 htable_release(ht); in htable_getpage()
1847 htable_va2entry(uintptr_t va, htable_t *ht) in htable_va2entry() argument
1849 level_t l = ht->ht_level; in htable_va2entry()
1851 ASSERT(va >= ht->ht_vaddr); in htable_va2entry()
1852 ASSERT(va <= HTABLE_LAST_PAGE(ht)); in htable_va2entry()
1853 return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1)); in htable_va2entry()
1861 htable_e2va(htable_t *ht, uint_t entry) in htable_e2va() argument
1863 level_t l = ht->ht_level; in htable_e2va()
1866 ASSERT(entry < HTABLE_NUM_PTES(ht)); in htable_e2va()
1867 va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l)); in htable_e2va()
1872 if (ht->ht_level == mmu.max_level && va >= mmu.hole_start) in htable_e2va()
1913 x86pte_access_pagetable(htable_t *ht, uint_t index) in x86pte_access_pagetable() argument
1918 if (ht->ht_flags & HTABLE_COPIED) { in x86pte_access_pagetable()
1919 ASSERT3U(index, <, ht->ht_hat->hat_num_copied); in x86pte_access_pagetable()
1920 return (PT_INDEX_PTR(ht->ht_hat->hat_copied_ptes, index)); in x86pte_access_pagetable()
1922 return (x86pte_mapin(ht->ht_pfn, index, ht)); in x86pte_access_pagetable()
1930 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht) in x86pte_mapin() argument
2002 x86pte_release_pagetable(htable_t *ht) in x86pte_release_pagetable() argument
2004 if (ht->ht_flags & HTABLE_COPIED) in x86pte_release_pagetable()
2040 x86pte_get(htable_t *ht, uint_t entry) in x86pte_get() argument
2049 ptep = x86pte_access_pagetable(ht, entry); in x86pte_get()
2051 x86pte_release_pagetable(ht); in x86pte_get()
2066 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr) in x86pte_set() argument
2071 level_t l = ht->ht_level; in x86pte_set()
2074 uintptr_t addr = htable_e2va(ht, entry); in x86pte_set()
2075 hat_t *hat = ht->ht_hat; in x86pte_set()
2078 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); in x86pte_set()
2080 ptep = x86pte_access_pagetable(ht, entry); in x86pte_set()
2139 x86pte_release_pagetable(ht); in x86pte_set()
2150 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new) in x86pte_cas() argument
2164 ASSERT(!(ht->ht_flags & HTABLE_COPIED)); in x86pte_cas()
2165 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry)); in x86pte_cas()
2173 if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) { in x86pte_cas()
2175 ht->ht_hat->hat_user_ptable), entry)); in x86pte_cas()
2187 ptep = x86pte_access_pagetable(ht, entry); in x86pte_cas()
2191 x86pte_release_pagetable(ht); in x86pte_cas()
2205 htable_t *ht, in x86pte_inval() argument
2215 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); in x86pte_inval()
2216 ASSERT(ht->ht_level <= mmu.max_page_level); in x86pte_inval()
2221 ptep = x86pte_access_pagetable(ht, entry); in x86pte_inval()
2228 if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) { in x86pte_inval()
2236 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry)); in x86pte_inval()
2259 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry)); in x86pte_inval()
2263 x86pte_release_pagetable(ht); in x86pte_inval()
2272 htable_t *ht, in x86pte_update() argument
2281 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); in x86pte_update()
2282 ASSERT(ht->ht_level <= mmu.max_page_level); in x86pte_update()
2284 ptep = x86pte_access_pagetable(ht, entry); in x86pte_update()
2289 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry)); in x86pte_update()
2311 x86pte_release_pagetable(ht); in x86pte_update()
2464 htable_t *ht; in hat_dump() local
2471 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in hat_dump()
2472 if ((ht->ht_flags & HTABLE_COPIED) == 0) in hat_dump()
2473 dump_page(ht->ht_pfn); in hat_dump()