Lines Matching refs:hat
436 htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold, in htable_steal_active() argument
445 h = h_start = h_seed++ % hat->hat_num_hash; in htable_steal_active()
449 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_steal_active()
511 ASSERT(hat->hat_ht_hash[h] == ht); in htable_steal_active()
512 hat->hat_ht_hash[h] = ht->ht_next; in htable_steal_active()
529 if (++h == hat->hat_num_hash) in htable_steal_active()
538 move_victim(hat_t *hat) in move_victim() argument
543 if (hat->hat_prev) in move_victim()
544 hat->hat_prev->hat_next = hat->hat_next; in move_victim()
546 kas.a_hat->hat_next = hat->hat_next; in move_victim()
548 if (hat->hat_next) in move_victim()
549 hat->hat_next->hat_prev = hat->hat_prev; in move_victim()
551 kas.a_hat->hat_prev = hat->hat_prev; in move_victim()
553 hat->hat_next = NULL; in move_victim()
554 hat->hat_prev = kas.a_hat->hat_prev; in move_victim()
555 if (hat->hat_prev) in move_victim()
556 hat->hat_prev->hat_next = hat; in move_victim()
558 kas.a_hat->hat_next = hat; in move_victim()
560 kas.a_hat->hat_prev = hat; in move_victim()
570 hat_t *hat = kas.a_hat; /* list starts with khat */ in htable_steal() local
596 hat = kas.a_hat->hat_next; in htable_steal()
610 while (hat != NULL && in htable_steal()
611 (hat->hat_flags & in htable_steal()
613 hat = hat->hat_next; in htable_steal()
615 if (hat == NULL) in htable_steal()
622 hat->hat_flags |= HAT_VICTIM; in htable_steal()
628 hat_enter(hat); in htable_steal()
629 while ((ht = hat->hat_ht_cached) != NULL && in htable_steal()
631 hat->hat_ht_cached = ht->ht_next; in htable_steal()
636 hat_exit(hat); in htable_steal()
642 htable_steal_active(hat, cnt, threshold, in htable_steal()
654 ASSERT(ht->ht_hat == hat); in htable_steal()
658 ptable_free(hat->hat_user_ptable); in htable_steal()
659 hat->hat_user_ptable = PFN_INVALID; in htable_steal()
679 kas.a_hat->hat_prev != hat) in htable_steal()
680 move_victim(hat); in htable_steal()
690 if (hat->hat_flags & HAT_VICTIM) { in htable_steal()
691 ASSERT(hat != kas.a_hat); in htable_steal()
692 hat->hat_flags &= ~HAT_VICTIM; in htable_steal()
697 hat = hat->hat_next; in htable_steal()
761 hat_t *hat, in htable_alloc() argument
772 if (level < 0 || level > TOP_LEVEL(hat)) in htable_alloc()
775 is_vlp = (hat->hat_flags & HAT_VLP) && level == VLP_LEVEL; in htable_alloc()
783 if (hat->hat_ht_cached != NULL && !is_bare) { in htable_alloc()
784 hat_enter(hat); in htable_alloc()
785 ht = hat->hat_ht_cached; in htable_alloc()
787 hat->hat_ht_cached = ht->ht_next; in htable_alloc()
792 hat_exit(hat); in htable_alloc()
821 ht->ht_hat = hat; in htable_alloc()
885 hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1); in htable_alloc()
886 if (hat->hat_user_ptable != PFN_INVALID) in htable_alloc()
893 block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable), in htable_alloc()
929 ht->ht_hat = hat; in htable_alloc()
947 (void) xen_kpm_page(hat->hat_user_ptable, PT_VALID); in htable_alloc()
961 hat_t *hat = ht->ht_hat; in htable_free() local
968 if (hat != NULL && in htable_free()
971 (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) { in htable_free()
974 hat_enter(hat); in htable_free()
975 ht->ht_next = hat->hat_ht_cached; in htable_free()
976 hat->hat_ht_cached = ht; in htable_free()
977 hat_exit(hat); in htable_free()
990 if (ht->ht_level == mmu.max_level && hat != NULL) { in htable_free()
991 ptable_free(hat->hat_user_ptable); in htable_free()
992 hat->hat_user_ptable = PFN_INVALID; in htable_free()
1018 htable_purge_hat(hat_t *hat) in htable_purge_hat() argument
1026 if (!(hat->hat_flags & HAT_FREEING)) { in htable_purge_hat()
1029 hat_enter(hat); in htable_purge_hat()
1030 ht = hat->hat_ht_cached; in htable_purge_hat()
1032 hat_exit(hat); in htable_purge_hat()
1035 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1036 hat_exit(hat); in htable_purge_hat()
1046 while ((ht = hat->hat_ht_cached) != NULL) { in htable_purge_hat()
1047 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1054 for (h = 0; h < hat->hat_num_hash; ++h) { in htable_purge_hat()
1055 while ((ht = hat->hat_ht_hash[h]) != NULL) { in htable_purge_hat()
1062 ASSERT(hat->hat_ht_hash[h] == ht); in htable_purge_hat()
1063 hat->hat_ht_hash[h] = ht->ht_next; in htable_purge_hat()
1080 hat_t *hat = old->ht_hat; in unlink_ptp() local
1106 if (!(hat->hat_flags & HAT_FREEING)) { in unlink_ptp()
1107 hat_tlb_inval(hat, (higher->ht_flags & HTABLE_VLP) ? in unlink_ptp()
1165 hat_t *hat; in htable_release() local
1172 hat = ht->ht_hat; in htable_release()
1175 hashval = HTABLE_HASH(hat, va, level); in htable_release()
1199 if (hat->hat_flags & HAT_FREEING) in htable_release()
1207 (hat != kas.a_hat || va >= kernelbase)) in htable_release()
1243 ASSERT(hat->hat_ht_hash[hashval] == ht); in htable_release()
1244 hat->hat_ht_hash[hashval] = ht->ht_next; in htable_release()
1268 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level) in htable_lookup() argument
1275 ASSERT(level <= TOP_LEVEL(hat)); in htable_lookup()
1277 if (level == TOP_LEVEL(hat)) { in htable_lookup()
1283 if ((hat->hat_flags & HAT_VLP) && vaddr >= ((uint64_t)1 << 32)) in htable_lookup()
1291 hashval = HTABLE_HASH(hat, base, level); in htable_lookup()
1293 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) { in htable_lookup()
1294 if (ht->ht_hat == hat && in htable_lookup()
1312 hat_t *hat = ht->ht_hat; in htable_acquire() local
1315 uint_t hashval = HTABLE_HASH(hat, base, level); in htable_acquire()
1325 for (h = hat->hat_ht_hash[hashval]; in htable_acquire()
1347 hat_t *hat, in htable_create() argument
1359 if (level < 0 || level > TOP_LEVEL(hat)) in htable_create()
1365 for (l = TOP_LEVEL(hat); l >= level; --l) { in htable_create()
1367 if (l == TOP_LEVEL(hat)) in htable_create()
1372 h = HTABLE_HASH(hat, base, l); in htable_create()
1378 if (l == TOP_LEVEL(hat)) { in htable_create()
1379 ht = hat->hat_htable; in htable_create()
1381 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_create()
1382 ASSERT(ht->ht_hat == hat); in htable_create()
1418 new = htable_alloc(hat, base, l, in htable_create()
1432 ht->ht_next = hat->hat_ht_hash[h]; in htable_create()
1434 if (hat->hat_ht_hash[h]) in htable_create()
1435 hat->hat_ht_hash[h]->ht_prev = ht; in htable_create()
1436 hat->hat_ht_hash[h] = ht; in htable_create()
1469 hat_t *hat, in htable_attach() argument
1486 ht->ht_hat = hat; in htable_attach()
1500 h = HTABLE_HASH(hat, base, level); in htable_attach()
1502 ht->ht_next = hat->hat_ht_hash[h]; in htable_attach()
1504 if (hat->hat_ht_hash[h]) in htable_attach()
1505 hat->hat_ht_hash[h]->ht_prev = ht; in htable_attach()
1506 hat->hat_ht_hash[h] = ht; in htable_attach()
1558 htable_attach(hat, base, level - 1, in htable_attach()
1657 struct hat *hat, in htable_walk() argument
1675 ASSERT(hat == kas.a_hat || eaddr <= kernelbase || in htable_walk()
1677 if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END) in htable_walk()
1708 if (hat->hat_ism_pgcnt > 0) { in htable_walk()
1713 if (hat->hat_pages_mapped[l] != 0) in htable_walk()
1721 for (l = 0; l <= TOP_LEVEL(hat); ++l) { in htable_walk()
1722 ht = htable_lookup(hat, va, l); in htable_walk()
1740 ASSERT(l < TOP_LEVEL(hat)); in htable_walk()
1762 struct hat *hat, in htable_getpte() argument
1775 ht = htable_lookup(hat, vaddr, l); in htable_getpte()
1795 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry) in htable_getpage() argument
1801 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level); in htable_getpage()
2088 hat_t *hat = ht->ht_hat; in x86pte_set() local
2148 hat_tlb_inval(hat, addr); in x86pte_set()
2482 hat_t *hat; in hat_dump() local
2489 for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) { in hat_dump()
2490 for (h = 0; h < hat->hat_num_hash; ++h) { in hat_dump()
2491 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in hat_dump()