Lines Matching refs:ht
111 static x86pte_t hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected,
246 htable_t *ht; /* top level htable */ in hat_alloc() local
308 ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL); in hat_alloc()
309 hat->hat_htable = ht; in hat_alloc()
322 ht = hat->hat_htable; in hat_alloc()
324 ht = htable_create(hat, va, rp->hkr_level, in hat_alloc()
327 start = htable_va2entry(va, ht); in hat_alloc()
328 cnt = HTABLE_NUM_PTES(ht) - start; in hat_alloc()
333 cnt = htable_va2entry(rp->hkr_end_va, ht) - in hat_alloc()
337 if (ht->ht_flags & HTABLE_VLP) { in hat_alloc()
346 x86pte_copy(src, ht, start, cnt); in hat_alloc()
896 htable_t *ht; in hat_init_finish() local
903 (ht = htable_getpage(kas.a_hat, va, NULL)) != in hat_init_finish()
905 htable_release(ht); in hat_init_finish()
1146 htable_t *ht = NULL; in hat_swapout() local
1165 (void) htable_walk(hat, &ht, &vaddr, eaddr); in hat_swapout()
1166 if (ht == NULL) in hat_swapout()
1174 l = ht->ht_level; in hat_swapout()
1175 if (ht->ht_flags & HTABLE_SHARED_PFN) { in hat_swapout()
1176 vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1); in hat_swapout()
1177 htable_release(ht); in hat_swapout()
1178 ht = NULL; in hat_swapout()
1185 if (ht->ht_lock_cnt == 0) in hat_swapout()
1193 if (ht->ht_lock_cnt > 0 && l == 0) in hat_swapout()
1194 vaddr = ht->ht_vaddr + LEVEL_SIZE(1); in hat_swapout()
1198 if (ht) in hat_swapout()
1199 htable_release(ht); in hat_swapout()
1297 htable_t *ht, in hati_pte_map() argument
1304 hat_t *hat = ht->ht_hat; in hati_pte_map()
1306 level_t l = ht->ht_level; in hati_pte_map()
1323 HTABLE_LOCK_INC(ht); in hati_pte_map()
1331 hm = hment_prepare(ht, entry, pp); in hati_pte_map()
1337 old_pte = x86pte_set(ht, entry, pte, pte_ptr); in hati_pte_map()
1344 HTABLE_LOCK_DEC(ht); in hati_pte_map()
1360 hment_assign(ht, entry, pp, hm); in hati_pte_map()
1366 if (ht->ht_flags & HTABLE_VLP) { in hati_pte_map()
1372 HTABLE_INC(ht->ht_valid_cnt); in hati_pte_map()
1429 htable_t *ht; in hati_load_common() local
1449 ht = htable_lookup(hat, va, level); in hati_load_common()
1457 if (ht == NULL) { in hati_load_common()
1458 ht = htable_create(hat, va, level, NULL); in hati_load_common()
1459 ASSERT(ht != NULL); in hati_load_common()
1461 entry = htable_va2entry(va, ht); in hati_load_common()
1466 ASSERT(ht->ht_busy > 0); in hati_load_common()
1467 if (ht->ht_vaddr > va || va > HTABLE_LAST_PAGE(ht)) in hati_load_common()
1469 (void *)ht, (void *)va); in hati_load_common()
1470 ASSERT(ht->ht_level == level); in hati_load_common()
1484 rv = hati_pte_map(ht, entry, pp, pte, flags, NULL); in hati_load_common()
1489 htable_release(ht); in hati_load_common()
1508 htable_t *ht; in hat_kmap_load() local
1527 ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) >> in hat_kmap_load()
1529 entry = htable_va2entry(va, ht); in hat_kmap_load()
1532 (void) hati_pte_map(ht, entry, pp, pte, flags, pte_ptr); in hat_kmap_load()
1875 htable_t *ht = NULL; in hat_unlock() local
1891 (void) htable_walk(hat, &ht, &vaddr, eaddr); in hat_unlock()
1892 if (ht == NULL) in hat_unlock()
1897 if (ht->ht_lock_cnt < 1) in hat_unlock()
1899 "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr); in hat_unlock()
1900 HTABLE_LOCK_DEC(ht); in hat_unlock()
1902 vaddr += LEVEL_SIZE(ht->ht_level); in hat_unlock()
1904 if (ht) in hat_unlock()
1905 htable_release(ht); in hat_unlock()
2174 htable_t *ht, in hat_pte_unmap() argument
2181 hat_t *hat = ht->ht_hat; in hat_pte_unmap()
2184 level_t l = ht->ht_level; in hat_pte_unmap()
2191 ASSERT(ht->ht_lock_cnt > 0); in hat_pte_unmap()
2192 HTABLE_LOCK_DEC(ht); in hat_pte_unmap()
2201 ASSERT(ht->ht_busy > 0); in hat_pte_unmap()
2215 old_pte, (uintptr_t)ht, entry, in hat_pte_unmap()
2221 old_pte = x86pte_inval(ht, entry, old_pte, pte_ptr, tlb); in hat_pte_unmap()
2256 hm = hment_remove(pp, ht, entry); in hat_pte_unmap()
2265 ASSERT(ht->ht_valid_cnt > 0); in hat_pte_unmap()
2266 HTABLE_DEC(ht->ht_valid_cnt); in hat_pte_unmap()
2279 htable_t *ht; in hat_kmap_unload() local
2295 ht = mmu.kmap_htables[(va - mmu.kmap_htables[0]->ht_vaddr) in hat_kmap_unload()
2297 entry = htable_va2entry(va, ht); in hat_kmap_unload()
2302 hat_pte_unmap(ht, entry, flags, old_pte, pte_ptr, B_TRUE); in hat_kmap_unload()
2383 htable_t *ht = NULL; in hat_unload_callback() local
2400 ht = htable_getpte(hat, vaddr, &entry, &old_pte, 0); in hat_unload_callback()
2401 if (ht != NULL) { in hat_unload_callback()
2403 hat_pte_unmap(ht, entry, flags, old_pte, in hat_unload_callback()
2406 htable_release(ht); in hat_unload_callback()
2413 old_pte = htable_walk(hat, &ht, &vaddr, eaddr); in hat_unload_callback()
2414 if (ht == NULL) in hat_unload_callback()
2426 (r_cnt > 0 && r[r_cnt - 1].rng_level != ht->ht_level)) { in hat_unload_callback()
2433 r[r_cnt].rng_level = ht->ht_level; in hat_unload_callback()
2446 entry = htable_va2entry(vaddr, ht); in hat_unload_callback()
2447 hat_pte_unmap(ht, entry, flags, old_pte, NULL, B_FALSE); in hat_unload_callback()
2448 ASSERT(ht->ht_level <= mmu.max_page_level); in hat_unload_callback()
2449 vaddr += LEVEL_SIZE(ht->ht_level); in hat_unload_callback()
2453 if (ht) in hat_unload_callback()
2454 htable_release(ht); in hat_unload_callback()
2505 htable_t *ht = NULL; in hat_sync() local
2518 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) { in hat_sync()
2520 pte = htable_walk(hat, &ht, &vaddr, eaddr); in hat_sync()
2521 if (ht == NULL) in hat_sync()
2523 entry = htable_va2entry(vaddr, ht); in hat_sync()
2533 pp = page_numtopp_nolock(PTE2PFN(pte, ht->ht_level)); in hat_sync()
2538 pte = x86pte_get(ht, entry); in hat_sync()
2556 pte = hati_update_pte(ht, entry, pte, new); in hat_sync()
2565 hati_sync_pte_to_page(pp, save_pte, ht->ht_level); in hat_sync()
2569 if (ht) in hat_sync()
2570 htable_release(ht); in hat_sync()
2594 htable_t *ht = NULL; in hat_getattr() local
2602 ht = htable_getpte(hat, vaddr, NULL, &pte, mmu.max_page_level); in hat_getattr()
2603 if (ht == NULL) in hat_getattr()
2606 if (!PTE_ISVALID(pte) || !PTE_ISPAGE(pte, ht->ht_level)) { in hat_getattr()
2607 htable_release(ht); in hat_getattr()
2620 htable_release(ht); in hat_getattr()
2636 htable_t *ht = NULL; in hat_updateattr() local
2645 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) { in hat_updateattr()
2647 oldpte = htable_walk(hat, &ht, &vaddr, eaddr); in hat_updateattr()
2648 if (ht == NULL) in hat_updateattr()
2653 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level)); in hat_updateattr()
2717 entry = htable_va2entry(vaddr, ht); in hat_updateattr()
2718 oldpte = hati_update_pte(ht, entry, oldpte, newpte); in hat_updateattr()
2726 if (ht) in hat_updateattr()
2727 htable_release(ht); in hat_updateattr()
2771 htable_t *ht; in hat_getpagesize() local
2777 ht = htable_getpage(hat, vaddr, NULL); in hat_getpagesize()
2778 if (ht == NULL) in hat_getpagesize()
2780 pagesize = LEVEL_SIZE(ht->ht_level); in hat_getpagesize()
2781 htable_release(ht); in hat_getpagesize()
2795 htable_t *ht; in hat_getpfnum() local
2825 ht = htable_getpage(hat, vaddr, &entry); in hat_getpfnum()
2826 if (ht == NULL) { in hat_getpfnum()
2830 ASSERT(vaddr >= ht->ht_vaddr); in hat_getpfnum()
2831 ASSERT(vaddr <= HTABLE_LAST_PAGE(ht)); in hat_getpfnum()
2832 pfn = PTE2PFN(x86pte_get(ht, entry), ht->ht_level); in hat_getpfnum()
2833 if (ht->ht_level > 0) in hat_getpfnum()
2834 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level)); in hat_getpfnum()
2835 htable_release(ht); in hat_getpfnum()
2850 htable_t *ht; in hat_probe() local
2871 ht = htable_getpage(hat, vaddr, &entry); in hat_probe()
2872 htable_release(ht); in hat_probe()
2873 return (ht != NULL); in hat_probe()
2921 htable_t *ht; in hat_share() local
2964 ht = htable_lookup(hat, vaddr, l); in hat_share()
2965 if (ht != NULL) { in hat_share()
2966 if (ht->ht_flags & HTABLE_SHARED_PFN) in hat_share()
2968 htable_release(ht); in hat_share()
3019 ht = htable_create(hat, vaddr, l, ism_ht); in hat_share()
3021 ASSERT(ht->ht_flags & HTABLE_SHARED_PFN); in hat_share()
3022 ASSERT(ht->ht_shares == ism_ht); in hat_share()
3024 (ism_ht->ht_valid_cnt - ht->ht_valid_cnt) << in hat_share()
3025 (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT); in hat_share()
3026 ht->ht_valid_cnt = ism_ht->ht_valid_cnt; in hat_share()
3027 htable_release(ht); in hat_share()
3106 htable_t *ht = NULL; in hat_unshare() local
3134 ht = htable_lookup(hat, vaddr, l); in hat_unshare()
3135 if (ht == NULL) in hat_unshare()
3137 if (ht->ht_flags & HTABLE_SHARED_PFN) { in hat_unshare()
3142 hat->hat_ism_pgcnt -= ht->ht_valid_cnt << in hat_unshare()
3143 (LEVEL_SHIFT(ht->ht_level) - MMU_PAGESHIFT); in hat_unshare()
3144 ht->ht_valid_cnt = 0; in hat_unshare()
3147 htable_release(ht); in hat_unshare()
3187 htable_t *ht; in hati_page_clrwrt() local
3199 while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) { in hati_page_clrwrt()
3200 if (ht->ht_level < pszc) in hati_page_clrwrt()
3202 old = x86pte_get(ht, entry); in hati_page_clrwrt()
3208 if (PTE2PFN(old, ht->ht_level) != pp->p_pagenum || in hati_page_clrwrt()
3218 old = hati_update_pte(ht, entry, old, new); in hati_page_clrwrt()
3329 hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry) in hati_page_unmap() argument
3341 htable_acquire(ht); in hati_page_unmap()
3346 old_pte = x86pte_inval(ht, entry, 0, NULL, B_TRUE); in hati_page_unmap()
3347 if (PTE2PFN(old_pte, ht->ht_level) != pfn) { in hati_page_unmap()
3350 old_pte, pfn, (uintptr_t)ht, entry); in hati_page_unmap()
3356 ASSERT(ht->ht_valid_cnt > 0); in hati_page_unmap()
3357 HTABLE_DEC(ht->ht_valid_cnt); in hati_page_unmap()
3358 PGCNT_DEC(ht->ht_hat, ht->ht_level); in hati_page_unmap()
3364 hati_sync_pte_to_page(pp, old_pte, ht->ht_level); in hati_page_unmap()
3369 hm = hment_remove(pp, ht, entry); in hati_page_unmap()
3376 htable_release(ht); in hati_page_unmap()
3395 htable_t *ht; in hati_pageunload() local
3426 hm = hment_walk(cur_pp, &ht, &entry, prev); in hati_pageunload()
3455 level = ht->ht_level; in hati_pageunload()
3464 hm = hati_page_unmap(cur_pp, ht, entry); in hati_pageunload()
3666 htable_t *ht; in hat_pagesync() local
3702 while ((hm = hment_walk(pp, &ht, &entry, hm)) != NULL) { in hat_pagesync()
3703 if (ht->ht_level < pszc) in hat_pagesync()
3705 old = x86pte_get(ht, entry); in hat_pagesync()
3708 ASSERT(PTE2PFN(old, ht->ht_level) == pp->p_pagenum); in hat_pagesync()
3722 old = hati_update_pte(ht, entry, old, new); in hat_pagesync()
3734 hati_sync_pte_to_page(pp, old, ht->ht_level); in hat_pagesync()
3873 htable_t *ht; in hat_mempte_setup() local
3882 ht = htable_getpte(kas.a_hat, va, &entry, &oldpte, 0); in hat_mempte_setup()
3883 if (ht == NULL) { in hat_mempte_setup()
3884 ht = htable_create(kas.a_hat, va, 0, NULL); in hat_mempte_setup()
3885 entry = htable_va2entry(va, ht); in hat_mempte_setup()
3886 ASSERT(ht->ht_level == 0); in hat_mempte_setup()
3887 oldpte = x86pte_get(ht, entry); in hat_mempte_setup()
3891 "ht=%p, entry=%d, pte=" FMT_PTE, (void *)ht, entry, oldpte); in hat_mempte_setup()
3896 HTABLE_INC(ht->ht_valid_cnt); in hat_mempte_setup()
3901 htable_release(ht); in hat_mempte_setup()
3903 p = PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry); in hat_mempte_setup()
3916 htable_t *ht; in hat_mempte_release() local
3941 ht = htable_getpte(kas.a_hat, ALIGN2PAGE(addr), NULL, NULL, 0); in hat_mempte_release()
3942 if (ht == NULL) in hat_mempte_release()
3944 ASSERT(ht->ht_level == 0); in hat_mempte_release()
3945 HTABLE_DEC(ht->ht_valid_cnt); in hat_mempte_release()
3946 htable_release(ht); in hat_mempte_release()
3970 htable_t *ht; in hat_mempte_remap() local
3975 ht = htable_getpte(kas.a_hat, va, &entry, NULL, 0); in hat_mempte_remap()
3976 ASSERT(ht != NULL); in hat_mempte_remap()
3977 ASSERT(ht->ht_level == 0); in hat_mempte_remap()
3978 ASSERT(ht->ht_valid_cnt > 0); in hat_mempte_remap()
3979 ASSERT(ht->ht_pfn == mmu_btop(pte_pa)); in hat_mempte_remap()
3980 htable_release(ht); in hat_mempte_remap()
4058 htable_t *ht = NULL; in clear_boot_mappings() local
4068 pte = htable_walk(kas.a_hat, &ht, &vaddr, high); in clear_boot_mappings()
4069 if (ht == NULL) in clear_boot_mappings()
4072 level = ht->ht_level; in clear_boot_mappings()
4073 entry = htable_va2entry(vaddr, ht); in clear_boot_mappings()
4080 (void) x86pte_inval(ht, entry, 0, NULL, B_TRUE); in clear_boot_mappings()
4081 ASSERT(ht->ht_valid_cnt > 0); in clear_boot_mappings()
4082 HTABLE_DEC(ht->ht_valid_cnt); in clear_boot_mappings()
4083 PGCNT_DEC(ht->ht_hat, ht->ht_level); in clear_boot_mappings()
4085 vaddr += LEVEL_SIZE(ht->ht_level); in clear_boot_mappings()
4087 if (ht) in clear_boot_mappings()
4088 htable_release(ht); in clear_boot_mappings()
4100 hati_update_pte(htable_t *ht, uint_t entry, x86pte_t expected, x86pte_t new) in hati_update_pte() argument
4111 ASSERT(!pfn_is_foreign(PTE2PFN(expected, ht->ht_level))); in hati_update_pte()
4112 pp = page_numtopp_nolock(PTE2PFN(expected, ht->ht_level)); in hati_update_pte()
4121 replaced = x86pte_update(ht, entry, expected, new); in hati_update_pte()
4129 pgcnt_t pgcnt = page_get_pagecnt(ht->ht_level); in hati_update_pte()
4138 ASSERT(pp->p_szc >= ht->ht_level); in hati_update_pte()
4458 htable_t *ht; in hat_prepare_mapping() local
4463 ht = htable_create(hat, (uintptr_t)addr, 0, NULL); in hat_prepare_mapping()
4471 entry = htable_va2entry((uintptr_t)addr, ht); in hat_prepare_mapping()
4472 base_ma = pa_to_ma(ptob(ht->ht_pfn)); in hat_prepare_mapping()
4481 htable_t *ht; in hat_release_mapping() local
4485 ht = htable_lookup(hat, (uintptr_t)addr, 0); in hat_release_mapping()
4486 ASSERT(ht != NULL); in hat_release_mapping()
4487 ASSERT(ht->ht_busy >= 2); in hat_release_mapping()
4488 htable_release(ht); in hat_release_mapping()
4489 htable_release(ht); in hat_release_mapping()