Lines Matching defs:ht
111 static void htable_free(htable_t *ht);
112 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
113 static void x86pte_release_pagetable(htable_t *ht);
114 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
357 htable_put_reserve(htable_t *ht)
359 ht->ht_hat = NULL; /* no longer tied to a hat */
360 ASSERT(ht->ht_pfn == PFN_INVALID);
363 ht->ht_next = htable_reserve_pool;
364 htable_reserve_pool = ht;
375 htable_t *ht = NULL;
379 ht = htable_reserve_pool;
380 ASSERT(ht != NULL);
381 ASSERT(ht->ht_pfn == PFN_INVALID);
382 htable_reserve_pool = ht->ht_next;
387 return (ht);
396 htable_t *ht;
400 ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP);
401 ASSERT(ht != NULL);
404 ht->ht_pfn = PFN_INVALID;
405 htable_put_reserve(ht);
416 htable_t *ht;
423 ht = htable_get_reserve();
424 if (ht == NULL)
426 ASSERT(ht->ht_pfn == PFN_INVALID);
427 kmem_cache_free(htable_cache, ht);
440 htable_t *higher, *ht;
449 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
454 if (ht->ht_busy != 0 ||
455 (ht->ht_flags & HTABLE_SHARED_PFN) ||
456 ht->ht_level > 0 || ht->ht_valid_cnt > threshold ||
457 ht->ht_lock_cnt != 0)
466 ++ht->ht_busy;
473 for (e = 0, va = ht->ht_vaddr;
474 e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 &&
475 ht->ht_busy == 1 && ht->ht_lock_cnt == 0;
477 pte = x86pte_get(ht, e);
480 hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL,
490 if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 ||
491 ht->ht_lock_cnt != 0) {
492 --ht->ht_busy;
499 higher = ht->ht_parent;
500 unlink_ptp(higher, ht, ht->ht_vaddr);
505 if (ht->ht_next)
506 ht->ht_next->ht_prev = ht->ht_prev;
508 if (ht->ht_prev) {
509 ht->ht_prev->ht_next = ht->ht_next;
511 ASSERT(hat->hat_ht_hash[h] == ht);
512 hat->hat_ht_hash[h] = ht->ht_next;
521 ht->ht_next = *list;
522 *list = ht;
572 htable_t *ht;
644 while ((ht = hat->hat_ht_cached) != NULL &&
646 hat->hat_ht_cached = ht->ht_next;
647 ht->ht_next = list;
648 list = ht;
666 for (ht = list; (ht) && (reap); ht = ht->ht_next) {
667 if (ht->ht_hat == NULL)
669 ASSERT(ht->ht_hat == hat);
671 if (!(ht->ht_flags & HTABLE_VLP) &&
672 ht->ht_level == mmu.max_level) {
680 ht->ht_hat = NULL;
734 htable_t *ht;
757 while ((ht = list) != NULL) {
758 list = ht->ht_next;
760 htable_free(ht);
781 htable_t *ht = NULL;
800 ht = hat->hat_ht_cached;
801 if (ht != NULL) {
802 hat->hat_ht_cached = ht->ht_next;
805 ASSERT(ht->ht_pfn != PFN_INVALID);
810 if (ht == NULL) {
815 ht = htable_get_reserve();
821 ht = kmem_cache_alloc(htable_cache, kmflags);
822 if (ht == NULL)
824 ht->ht_pfn = PFN_INVALID;
828 htable_put_reserve(ht);
835 if (ht != NULL && !is_bare) {
836 ht->ht_hat = hat;
837 ht->ht_pfn = ptable_alloc((uintptr_t)ht);
838 if (ht->ht_pfn == PFN_INVALID) {
840 htable_put_reserve(ht);
842 kmem_cache_free(htable_cache, ht);
843 ht = NULL;
856 while (ht == NULL && can_steal_post_boot) {
858 ht = htable_steal(1, B_FALSE);
864 if (ht != NULL) {
866 ptable_free(ht->ht_pfn);
867 ht->ht_pfn = PFN_INVALID;
872 } else if (kpm_vbase && xen_kpm_page(ht->ht_pfn,
875 ht->ht_pfn);
886 if (ht == NULL)
900 hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1);
917 ht->ht_flags = 0;
920 ht->ht_flags |= HTABLE_SHARED_PFN;
921 ht->ht_pfn = shared->ht_pfn;
922 ht->ht_lock_cnt = 0;
923 ht->ht_valid_cnt = 0; /* updated in hat_share() */
924 ht->ht_shares = shared;
927 ht->ht_shares = NULL;
928 ht->ht_lock_cnt = 0;
929 ht->ht_valid_cnt = 0;
936 ht->ht_flags |= HTABLE_VLP;
937 ASSERT(ht->ht_pfn == PFN_INVALID);
944 ht->ht_hat = hat;
945 ht->ht_parent = NULL;
946 ht->ht_vaddr = vaddr;
947 ht->ht_level = level;
948 ht->ht_busy = 1;
949 ht->ht_next = NULL;
950 ht->ht_prev = NULL;
956 x86pte_zero(ht, 0, mmu.ptes_per_table);
960 (void) xen_kpm_page(ht->ht_pfn, PT_VALID);
966 return (ht);
974 htable_free(htable_t *ht)
976 hat_t *hat = ht->ht_hat;
984 !(ht->ht_flags & HTABLE_SHARED_PFN) &&
987 ASSERT((ht->ht_flags & HTABLE_VLP) == 0);
988 ASSERT(ht->ht_pfn != PFN_INVALID);
990 ht->ht_next = hat->hat_ht_cached;
991 hat->hat_ht_cached = ht;
1000 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1001 ASSERT(ht->ht_pfn != PFN_INVALID);
1002 } else if (!(ht->ht_flags & HTABLE_VLP)) {
1003 ptable_free(ht->ht_pfn);
1005 if (ht->ht_level == mmu.max_level && hat != NULL) {
1011 ht->ht_pfn = PFN_INVALID;
1017 htable_put_reserve(ht);
1019 kmem_cache_free(htable_cache, ht);
1035 htable_t *ht;
1045 ht = hat->hat_ht_cached;
1046 if (ht == NULL) {
1050 hat->hat_ht_cached = ht->ht_next;
1052 htable_free(ht);
1061 while ((ht = hat->hat_ht_cached) != NULL) {
1062 hat->hat_ht_cached = ht->ht_next;
1063 htable_free(ht);
1070 while ((ht = hat->hat_ht_hash[h]) != NULL) {
1071 if (ht->ht_next)
1072 ht->ht_next->ht_prev = ht->ht_prev;
1074 if (ht->ht_prev) {
1075 ht->ht_prev->ht_next = ht->ht_next;
1077 ASSERT(hat->hat_ht_hash[h] == ht);
1078 hat->hat_ht_hash[h] = ht->ht_next;
1080 htable_free(ht);
1175 htable_release(htable_t *ht)
1184 while (ht != NULL) {
1187 hat = ht->ht_hat;
1188 va = ht->ht_vaddr;
1189 level = ht->ht_level;
1197 ASSERT(ht->ht_valid_cnt >= 0);
1198 ASSERT(ht->ht_busy > 0);
1199 if (ht->ht_valid_cnt > 0)
1201 if (ht->ht_busy > 1)
1203 ASSERT(ht->ht_lock_cnt == 0);
1209 if (!(ht->ht_flags & HTABLE_SHARED_PFN)) {
1231 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1233 shared = ht->ht_shares;
1241 higher = ht->ht_parent;
1247 unlink_ptp(higher, ht, va);
1252 if (ht->ht_next)
1253 ht->ht_next->ht_prev = ht->ht_prev;
1255 if (ht->ht_prev) {
1256 ht->ht_prev->ht_next = ht->ht_next;
1258 ASSERT(hat->hat_ht_hash[hashval] == ht);
1259 hat->hat_ht_hash[hashval] = ht->ht_next;
1262 htable_free(ht);
1263 ht = higher;
1266 ASSERT(ht->ht_busy >= 1);
1267 --ht->ht_busy;
1274 ht = shared;
1287 htable_t *ht = NULL;
1308 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) {
1309 if (ht->ht_hat == hat &&
1310 ht->ht_vaddr == base &&
1311 ht->ht_level == level)
1314 if (ht)
1315 ++ht->ht_busy;
1318 return (ht);
1325 htable_acquire(htable_t *ht)
1327 hat_t *hat = ht->ht_hat;
1328 level_t level = ht->ht_level;
1329 uintptr_t base = ht->ht_vaddr;
1341 h && h != ht;
1344 ASSERT(h == ht);
1347 ++ht->ht_busy;
1370 htable_t *ht;
1394 ht = hat->hat_htable;
1396 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
1397 ASSERT(ht->ht_hat == hat);
1398 if (ht->ht_vaddr == base &&
1399 ht->ht_level == l)
1408 if (ht != NULL) {
1413 if (l == level && shared && ht->ht_shares &&
1414 ht->ht_shares != shared) {
1417 (void *)ht, (void *)shared);
1419 ++ht->ht_busy;
1425 higher = ht;
1442 ht = new;
1444 link_ptp(higher, ht, base);
1445 ht->ht_parent = higher;
1447 ht->ht_next = hat->hat_ht_hash[h];
1448 ASSERT(ht->ht_prev == NULL);
1450 hat->hat_ht_hash[h]->ht_prev = ht;
1451 hat->hat_ht_hash[h] = ht;
1459 higher = ht;
1474 return (ht);
1490 htable_t *ht;
1498 ht = htable_get_reserve();
1500 kas.a_hat->hat_htable = ht;
1501 ht->ht_hat = hat;
1502 ht->ht_parent = parent;
1503 ht->ht_vaddr = base;
1504 ht->ht_level = level;
1505 ht->ht_busy = 1;
1506 ht->ht_next = NULL;
1507 ht->ht_prev = NULL;
1508 ht->ht_flags = 0;
1509 ht->ht_pfn = pfn;
1510 ht->ht_lock_cnt = 0;
1511 ht->ht_valid_cnt = 0;
1517 ht->ht_next = hat->hat_ht_hash[h];
1518 ASSERT(ht->ht_prev == NULL);
1520 hat->hat_ht_hash[h]->ht_prev = ht;
1521 hat->hat_ht_hash[h] = ht;
1540 u_offset_t offset = (uintptr_t)ht;
1565 for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) {
1571 ++ht->ht_valid_cnt;
1574 ht, PTE2PFN(pte, level));
1588 htable_release(ht);
1607 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr)
1613 int l = ht->ht_level;
1617 ASSERT(va >= ht->ht_vaddr);
1618 ASSERT(va <= HTABLE_LAST_PAGE(ht));
1623 e = htable_va2entry(va, ht);
1629 pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0);
1630 end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht));
1647 x86pte_release_pagetable(ht);
1678 htable_t *ht;
1737 ht = htable_lookup(hat, va, l);
1738 if (ht != NULL) {
1739 pte = htable_scan(ht, &va, eaddr);
1743 *htp = ht;
1746 htable_release(ht);
1783 htable_t *ht;
1790 ht = htable_lookup(hat, vaddr, l);
1791 if (ht == NULL)
1793 e = htable_va2entry(vaddr, ht);
1797 *pte = x86pte_get(ht, e);
1798 return (ht);
1812 htable_t *ht;
1816 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level);
1817 if (ht == NULL)
1823 if (PTE_ISPAGE(pte, ht->ht_level))
1824 return (ht);
1825 htable_release(ht);
1855 htable_va2entry(uintptr_t va, htable_t *ht)
1857 level_t l = ht->ht_level;
1859 ASSERT(va >= ht->ht_vaddr);
1860 ASSERT(va <= HTABLE_LAST_PAGE(ht));
1861 return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1));
1869 htable_e2va(htable_t *ht, uint_t entry)
1871 level_t l = ht->ht_level;
1874 ASSERT(entry < HTABLE_NUM_PTES(ht));
1875 va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l));
1881 if (ht->ht_level == mmu.max_level && va >= mmu.hole_start)
1943 x86pte_access_pagetable(htable_t *ht, uint_t index)
1948 if (ht->ht_flags & HTABLE_VLP)
1949 return (PT_INDEX_PTR(ht->ht_hat->hat_vlp_ptes, index));
1950 return (x86pte_mapin(ht->ht_pfn, index, ht));
1958 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht)
2027 x86pte_release_pagetable(htable_t *ht)
2032 if (ht->ht_flags & HTABLE_VLP)
2068 x86pte_get(htable_t *ht, uint_t entry)
2077 ptep = x86pte_access_pagetable(ht, entry);
2079 x86pte_release_pagetable(ht);
2094 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr)
2099 level_t l = ht->ht_level;
2102 uintptr_t addr = htable_e2va(ht, entry);
2103 hat_t *hat = ht->ht_hat;
2106 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2108 ptep = x86pte_access_pagetable(ht, entry);
2167 x86pte_release_pagetable(ht);
2178 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new)
2192 ASSERT(!(ht->ht_flags & HTABLE_VLP)); /* no VLP yet */
2193 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2202 if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) {
2204 ht->ht_hat->hat_user_ptable), entry));
2217 ptep = x86pte_access_pagetable(ht, entry);
2221 x86pte_release_pagetable(ht);
2235 htable_t *ht,
2245 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2246 ASSERT(ht->ht_level <= mmu.max_page_level);
2251 ptep = x86pte_access_pagetable(ht, entry);
2258 if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) {
2266 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2289 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2293 x86pte_release_pagetable(ht);
2302 htable_t *ht,
2311 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2312 ASSERT(ht->ht_level <= mmu.max_page_level);
2314 ptep = x86pte_access_pagetable(ht, entry);
2319 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2341 x86pte_release_pagetable(ht);
2499 htable_t *ht;
2506 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
2507 if ((ht->ht_flags & HTABLE_VLP) == 0)
2508 dump_page(ht->ht_pfn);