Lines Matching refs:ht
111 static void htable_free(htable_t *ht);
112 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
113 static void x86pte_release_pagetable(htable_t *ht);
114 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
357 htable_put_reserve(htable_t *ht) in htable_put_reserve() argument
359 ht->ht_hat = NULL; /* no longer tied to a hat */ in htable_put_reserve()
360 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_put_reserve()
363 ht->ht_next = htable_reserve_pool; in htable_put_reserve()
364 htable_reserve_pool = ht; in htable_put_reserve()
375 htable_t *ht = NULL; in htable_get_reserve() local
379 ht = htable_reserve_pool; in htable_get_reserve()
380 ASSERT(ht != NULL); in htable_get_reserve()
381 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_get_reserve()
382 htable_reserve_pool = ht->ht_next; in htable_get_reserve()
387 return (ht); in htable_get_reserve()
396 htable_t *ht; in htable_initial_reserve() local
400 ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP); in htable_initial_reserve()
401 ASSERT(ht != NULL); in htable_initial_reserve()
404 ht->ht_pfn = PFN_INVALID; in htable_initial_reserve()
405 htable_put_reserve(ht); in htable_initial_reserve()
416 htable_t *ht; in htable_adjust_reserve() local
423 ht = htable_get_reserve(); in htable_adjust_reserve()
424 if (ht == NULL) in htable_adjust_reserve()
426 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_adjust_reserve()
427 kmem_cache_free(htable_cache, ht); in htable_adjust_reserve()
440 htable_t *higher, *ht; in htable_steal_active() local
449 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_steal_active()
454 if (ht->ht_busy != 0 || in htable_steal_active()
455 (ht->ht_flags & HTABLE_SHARED_PFN) || in htable_steal_active()
456 ht->ht_level > 0 || ht->ht_valid_cnt > threshold || in htable_steal_active()
457 ht->ht_lock_cnt != 0) in htable_steal_active()
466 ++ht->ht_busy; in htable_steal_active()
473 for (e = 0, va = ht->ht_vaddr; in htable_steal_active()
474 e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 && in htable_steal_active()
475 ht->ht_busy == 1 && ht->ht_lock_cnt == 0; in htable_steal_active()
477 pte = x86pte_get(ht, e); in htable_steal_active()
480 hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL, in htable_steal_active()
490 if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 || in htable_steal_active()
491 ht->ht_lock_cnt != 0) { in htable_steal_active()
492 --ht->ht_busy; in htable_steal_active()
499 higher = ht->ht_parent; in htable_steal_active()
500 unlink_ptp(higher, ht, ht->ht_vaddr); in htable_steal_active()
505 if (ht->ht_next) in htable_steal_active()
506 ht->ht_next->ht_prev = ht->ht_prev; in htable_steal_active()
508 if (ht->ht_prev) { in htable_steal_active()
509 ht->ht_prev->ht_next = ht->ht_next; in htable_steal_active()
511 ASSERT(hat->hat_ht_hash[h] == ht); in htable_steal_active()
512 hat->hat_ht_hash[h] = ht->ht_next; in htable_steal_active()
521 ht->ht_next = *list; in htable_steal_active()
522 *list = ht; in htable_steal_active()
572 htable_t *ht; in htable_steal() local
629 while ((ht = hat->hat_ht_cached) != NULL && in htable_steal()
631 hat->hat_ht_cached = ht->ht_next; in htable_steal()
632 ht->ht_next = list; in htable_steal()
633 list = ht; in htable_steal()
651 for (ht = list; (ht) && (reap); ht = ht->ht_next) { in htable_steal()
652 if (ht->ht_hat == NULL) in htable_steal()
654 ASSERT(ht->ht_hat == hat); in htable_steal()
656 if (!(ht->ht_flags & HTABLE_VLP) && in htable_steal()
657 ht->ht_level == mmu.max_level) { in htable_steal()
665 ht->ht_hat = NULL; in htable_steal()
719 htable_t *ht; in htable_reap() local
742 while ((ht = list) != NULL) { in htable_reap()
743 list = ht->ht_next; in htable_reap()
745 htable_free(ht); in htable_reap()
766 htable_t *ht = NULL; in htable_alloc() local
785 ht = hat->hat_ht_cached; in htable_alloc()
786 if (ht != NULL) { in htable_alloc()
787 hat->hat_ht_cached = ht->ht_next; in htable_alloc()
790 ASSERT(ht->ht_pfn != PFN_INVALID); in htable_alloc()
795 if (ht == NULL) { in htable_alloc()
800 ht = htable_get_reserve(); in htable_alloc()
806 ht = kmem_cache_alloc(htable_cache, kmflags); in htable_alloc()
807 if (ht == NULL) in htable_alloc()
809 ht->ht_pfn = PFN_INVALID; in htable_alloc()
813 htable_put_reserve(ht); in htable_alloc()
820 if (ht != NULL && !is_bare) { in htable_alloc()
821 ht->ht_hat = hat; in htable_alloc()
822 ht->ht_pfn = ptable_alloc((uintptr_t)ht); in htable_alloc()
823 if (ht->ht_pfn == PFN_INVALID) { in htable_alloc()
825 htable_put_reserve(ht); in htable_alloc()
827 kmem_cache_free(htable_cache, ht); in htable_alloc()
828 ht = NULL; in htable_alloc()
841 while (ht == NULL && can_steal_post_boot) { in htable_alloc()
843 ht = htable_steal(1, B_FALSE); in htable_alloc()
849 if (ht != NULL) { in htable_alloc()
851 ptable_free(ht->ht_pfn); in htable_alloc()
852 ht->ht_pfn = PFN_INVALID; in htable_alloc()
857 } else if (kpm_vbase && xen_kpm_page(ht->ht_pfn, in htable_alloc()
860 ht->ht_pfn); in htable_alloc()
871 if (ht == NULL) in htable_alloc()
885 hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1); in htable_alloc()
902 ht->ht_flags = 0; in htable_alloc()
905 ht->ht_flags |= HTABLE_SHARED_PFN; in htable_alloc()
906 ht->ht_pfn = shared->ht_pfn; in htable_alloc()
907 ht->ht_lock_cnt = 0; in htable_alloc()
908 ht->ht_valid_cnt = 0; /* updated in hat_share() */ in htable_alloc()
909 ht->ht_shares = shared; in htable_alloc()
912 ht->ht_shares = NULL; in htable_alloc()
913 ht->ht_lock_cnt = 0; in htable_alloc()
914 ht->ht_valid_cnt = 0; in htable_alloc()
921 ht->ht_flags |= HTABLE_VLP; in htable_alloc()
922 ASSERT(ht->ht_pfn == PFN_INVALID); in htable_alloc()
929 ht->ht_hat = hat; in htable_alloc()
930 ht->ht_parent = NULL; in htable_alloc()
931 ht->ht_vaddr = vaddr; in htable_alloc()
932 ht->ht_level = level; in htable_alloc()
933 ht->ht_busy = 1; in htable_alloc()
934 ht->ht_next = NULL; in htable_alloc()
935 ht->ht_prev = NULL; in htable_alloc()
941 x86pte_zero(ht, 0, mmu.ptes_per_table); in htable_alloc()
945 (void) xen_kpm_page(ht->ht_pfn, PT_VALID); in htable_alloc()
951 return (ht); in htable_alloc()
959 htable_free(htable_t *ht) in htable_free() argument
961 hat_t *hat = ht->ht_hat; in htable_free()
969 !(ht->ht_flags & HTABLE_SHARED_PFN) && in htable_free()
972 ASSERT((ht->ht_flags & HTABLE_VLP) == 0); in htable_free()
973 ASSERT(ht->ht_pfn != PFN_INVALID); in htable_free()
975 ht->ht_next = hat->hat_ht_cached; in htable_free()
976 hat->hat_ht_cached = ht; in htable_free()
985 if (ht->ht_flags & HTABLE_SHARED_PFN) { in htable_free()
986 ASSERT(ht->ht_pfn != PFN_INVALID); in htable_free()
987 } else if (!(ht->ht_flags & HTABLE_VLP)) { in htable_free()
988 ptable_free(ht->ht_pfn); in htable_free()
990 if (ht->ht_level == mmu.max_level && hat != NULL) { in htable_free()
996 ht->ht_pfn = PFN_INVALID; in htable_free()
1002 htable_put_reserve(ht); in htable_free()
1004 kmem_cache_free(htable_cache, ht); in htable_free()
1020 htable_t *ht; in htable_purge_hat() local
1030 ht = hat->hat_ht_cached; in htable_purge_hat()
1031 if (ht == NULL) { in htable_purge_hat()
1035 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1037 htable_free(ht); in htable_purge_hat()
1046 while ((ht = hat->hat_ht_cached) != NULL) { in htable_purge_hat()
1047 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1048 htable_free(ht); in htable_purge_hat()
1055 while ((ht = hat->hat_ht_hash[h]) != NULL) { in htable_purge_hat()
1056 if (ht->ht_next) in htable_purge_hat()
1057 ht->ht_next->ht_prev = ht->ht_prev; in htable_purge_hat()
1059 if (ht->ht_prev) { in htable_purge_hat()
1060 ht->ht_prev->ht_next = ht->ht_next; in htable_purge_hat()
1062 ASSERT(hat->hat_ht_hash[h] == ht); in htable_purge_hat()
1063 hat->hat_ht_hash[h] = ht->ht_next; in htable_purge_hat()
1065 htable_free(ht); in htable_purge_hat()
1160 htable_release(htable_t *ht) in htable_release() argument
1169 while (ht != NULL) { in htable_release()
1172 hat = ht->ht_hat; in htable_release()
1173 va = ht->ht_vaddr; in htable_release()
1174 level = ht->ht_level; in htable_release()
1182 ASSERT(ht->ht_valid_cnt >= 0); in htable_release()
1183 ASSERT(ht->ht_busy > 0); in htable_release()
1184 if (ht->ht_valid_cnt > 0) in htable_release()
1186 if (ht->ht_busy > 1) in htable_release()
1188 ASSERT(ht->ht_lock_cnt == 0); in htable_release()
1194 if (!(ht->ht_flags & HTABLE_SHARED_PFN)) { in htable_release()
1216 if (ht->ht_flags & HTABLE_SHARED_PFN) { in htable_release()
1218 shared = ht->ht_shares; in htable_release()
1226 higher = ht->ht_parent; in htable_release()
1232 unlink_ptp(higher, ht, va); in htable_release()
1237 if (ht->ht_next) in htable_release()
1238 ht->ht_next->ht_prev = ht->ht_prev; in htable_release()
1240 if (ht->ht_prev) { in htable_release()
1241 ht->ht_prev->ht_next = ht->ht_next; in htable_release()
1243 ASSERT(hat->hat_ht_hash[hashval] == ht); in htable_release()
1244 hat->hat_ht_hash[hashval] = ht->ht_next; in htable_release()
1247 htable_free(ht); in htable_release()
1248 ht = higher; in htable_release()
1251 ASSERT(ht->ht_busy >= 1); in htable_release()
1252 --ht->ht_busy; in htable_release()
1259 ht = shared; in htable_release()
1272 htable_t *ht = NULL; in htable_lookup() local
1293 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) { in htable_lookup()
1294 if (ht->ht_hat == hat && in htable_lookup()
1295 ht->ht_vaddr == base && in htable_lookup()
1296 ht->ht_level == level) in htable_lookup()
1299 if (ht) in htable_lookup()
1300 ++ht->ht_busy; in htable_lookup()
1303 return (ht); in htable_lookup()
1310 htable_acquire(htable_t *ht) in htable_acquire() argument
1312 hat_t *hat = ht->ht_hat; in htable_acquire()
1313 level_t level = ht->ht_level; in htable_acquire()
1314 uintptr_t base = ht->ht_vaddr; in htable_acquire()
1326 h && h != ht; in htable_acquire()
1329 ASSERT(h == ht); in htable_acquire()
1332 ++ht->ht_busy; in htable_acquire()
1355 htable_t *ht; in htable_create() local
1379 ht = hat->hat_htable; in htable_create()
1381 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_create()
1382 ASSERT(ht->ht_hat == hat); in htable_create()
1383 if (ht->ht_vaddr == base && in htable_create()
1384 ht->ht_level == l) in htable_create()
1393 if (ht != NULL) { in htable_create()
1398 if (l == level && shared && ht->ht_shares && in htable_create()
1399 ht->ht_shares != shared) { in htable_create()
1402 (void *)ht, (void *)shared); in htable_create()
1404 ++ht->ht_busy; in htable_create()
1410 higher = ht; in htable_create()
1427 ht = new; in htable_create()
1429 link_ptp(higher, ht, base); in htable_create()
1430 ht->ht_parent = higher; in htable_create()
1432 ht->ht_next = hat->hat_ht_hash[h]; in htable_create()
1433 ASSERT(ht->ht_prev == NULL); in htable_create()
1435 hat->hat_ht_hash[h]->ht_prev = ht; in htable_create()
1436 hat->hat_ht_hash[h] = ht; in htable_create()
1444 higher = ht; in htable_create()
1459 return (ht); in htable_create()
1475 htable_t *ht; in htable_attach() local
1483 ht = htable_get_reserve(); in htable_attach()
1485 kas.a_hat->hat_htable = ht; in htable_attach()
1486 ht->ht_hat = hat; in htable_attach()
1487 ht->ht_parent = parent; in htable_attach()
1488 ht->ht_vaddr = base; in htable_attach()
1489 ht->ht_level = level; in htable_attach()
1490 ht->ht_busy = 1; in htable_attach()
1491 ht->ht_next = NULL; in htable_attach()
1492 ht->ht_prev = NULL; in htable_attach()
1493 ht->ht_flags = 0; in htable_attach()
1494 ht->ht_pfn = pfn; in htable_attach()
1495 ht->ht_lock_cnt = 0; in htable_attach()
1496 ht->ht_valid_cnt = 0; in htable_attach()
1502 ht->ht_next = hat->hat_ht_hash[h]; in htable_attach()
1503 ASSERT(ht->ht_prev == NULL); in htable_attach()
1505 hat->hat_ht_hash[h]->ht_prev = ht; in htable_attach()
1506 hat->hat_ht_hash[h] = ht; in htable_attach()
1525 u_offset_t offset = (uintptr_t)ht; in htable_attach()
1550 for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) { in htable_attach()
1556 ++ht->ht_valid_cnt; in htable_attach()
1559 ht, PTE2PFN(pte, level)); in htable_attach()
1573 htable_release(ht); in htable_attach()
1592 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr) in htable_scan() argument
1598 int l = ht->ht_level; in htable_scan()
1602 ASSERT(va >= ht->ht_vaddr); in htable_scan()
1603 ASSERT(va <= HTABLE_LAST_PAGE(ht)); in htable_scan()
1608 e = htable_va2entry(va, ht); in htable_scan()
1614 pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0); in htable_scan()
1615 end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht)); in htable_scan()
1632 x86pte_release_pagetable(ht); in htable_scan()
1663 htable_t *ht; in htable_walk() local
1722 ht = htable_lookup(hat, va, l); in htable_walk()
1723 if (ht != NULL) { in htable_walk()
1724 pte = htable_scan(ht, &va, eaddr); in htable_walk()
1728 *htp = ht; in htable_walk()
1731 htable_release(ht); in htable_walk()
1768 htable_t *ht; in htable_getpte() local
1775 ht = htable_lookup(hat, vaddr, l); in htable_getpte()
1776 if (ht == NULL) in htable_getpte()
1778 e = htable_va2entry(vaddr, ht); in htable_getpte()
1782 *pte = x86pte_get(ht, e); in htable_getpte()
1783 return (ht); in htable_getpte()
1797 htable_t *ht; in htable_getpage() local
1801 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level); in htable_getpage()
1802 if (ht == NULL) in htable_getpage()
1808 if (PTE_ISPAGE(pte, ht->ht_level)) in htable_getpage()
1809 return (ht); in htable_getpage()
1810 htable_release(ht); in htable_getpage()
1840 htable_va2entry(uintptr_t va, htable_t *ht) in htable_va2entry() argument
1842 level_t l = ht->ht_level; in htable_va2entry()
1844 ASSERT(va >= ht->ht_vaddr); in htable_va2entry()
1845 ASSERT(va <= HTABLE_LAST_PAGE(ht)); in htable_va2entry()
1846 return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1)); in htable_va2entry()
1854 htable_e2va(htable_t *ht, uint_t entry) in htable_e2va() argument
1856 level_t l = ht->ht_level; in htable_e2va()
1859 ASSERT(entry < HTABLE_NUM_PTES(ht)); in htable_e2va()
1860 va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l)); in htable_e2va()
1866 if (ht->ht_level == mmu.max_level && va >= mmu.hole_start) in htable_e2va()
1928 x86pte_access_pagetable(htable_t *ht, uint_t index) in x86pte_access_pagetable() argument
1933 if (ht->ht_flags & HTABLE_VLP) in x86pte_access_pagetable()
1934 return (PT_INDEX_PTR(ht->ht_hat->hat_vlp_ptes, index)); in x86pte_access_pagetable()
1935 return (x86pte_mapin(ht->ht_pfn, index, ht)); in x86pte_access_pagetable()
1943 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht) in x86pte_mapin() argument
2012 x86pte_release_pagetable(htable_t *ht) in x86pte_release_pagetable() argument
2017 if (ht->ht_flags & HTABLE_VLP) in x86pte_release_pagetable()
2053 x86pte_get(htable_t *ht, uint_t entry) in x86pte_get() argument
2062 ptep = x86pte_access_pagetable(ht, entry); in x86pte_get()
2064 x86pte_release_pagetable(ht); in x86pte_get()
2079 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr) in x86pte_set() argument
2084 level_t l = ht->ht_level; in x86pte_set()
2087 uintptr_t addr = htable_e2va(ht, entry); in x86pte_set()
2088 hat_t *hat = ht->ht_hat; in x86pte_set()
2091 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); in x86pte_set()
2093 ptep = x86pte_access_pagetable(ht, entry); in x86pte_set()
2152 x86pte_release_pagetable(ht); in x86pte_set()
2163 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new) in x86pte_cas() argument
2177 ASSERT(!(ht->ht_flags & HTABLE_VLP)); /* no VLP yet */ in x86pte_cas()
2178 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry)); in x86pte_cas()
2187 if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) { in x86pte_cas()
2189 ht->ht_hat->hat_user_ptable), entry)); in x86pte_cas()
2202 ptep = x86pte_access_pagetable(ht, entry); in x86pte_cas()
2206 x86pte_release_pagetable(ht); in x86pte_cas()
2220 htable_t *ht, in x86pte_inval() argument
2230 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); in x86pte_inval()
2231 ASSERT(ht->ht_level <= mmu.max_page_level); in x86pte_inval()
2236 ptep = x86pte_access_pagetable(ht, entry); in x86pte_inval()
2243 if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) { in x86pte_inval()
2251 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry)); in x86pte_inval()
2274 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry)); in x86pte_inval()
2278 x86pte_release_pagetable(ht); in x86pte_inval()
2287 htable_t *ht, in x86pte_update() argument
2296 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN)); in x86pte_update()
2297 ASSERT(ht->ht_level <= mmu.max_page_level); in x86pte_update()
2299 ptep = x86pte_access_pagetable(ht, entry); in x86pte_update()
2304 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry)); in x86pte_update()
2326 x86pte_release_pagetable(ht); in x86pte_update()
2484 htable_t *ht; in hat_dump() local
2491 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in hat_dump()
2492 if ((ht->ht_flags & HTABLE_VLP) == 0) in hat_dump()
2493 dump_page(ht->ht_pfn); in hat_dump()