Lines Matching refs:hat
436 htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold, in htable_steal_active() argument
445 h = h_start = h_seed++ % hat->hat_num_hash; in htable_steal_active()
449 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_steal_active()
511 ASSERT(hat->hat_ht_hash[h] == ht); in htable_steal_active()
512 hat->hat_ht_hash[h] = ht->ht_next; in htable_steal_active()
529 if (++h == hat->hat_num_hash) in htable_steal_active()
538 move_victim(hat_t *hat) in move_victim() argument
543 if (hat->hat_prev) in move_victim()
544 hat->hat_prev->hat_next = hat->hat_next; in move_victim()
546 kas.a_hat->hat_next = hat->hat_next; in move_victim()
548 if (hat->hat_next) in move_victim()
549 hat->hat_next->hat_prev = hat->hat_prev; in move_victim()
551 kas.a_hat->hat_prev = hat->hat_prev; in move_victim()
553 hat->hat_next = NULL; in move_victim()
554 hat->hat_prev = kas.a_hat->hat_prev; in move_victim()
555 if (hat->hat_prev) in move_victim()
556 hat->hat_prev->hat_next = hat; in move_victim()
558 kas.a_hat->hat_next = hat; in move_victim()
560 kas.a_hat->hat_prev = hat; in move_victim()
570 hat_t *hat = kas.a_hat; /* list starts with khat */ in htable_steal() local
611 hat = kas.a_hat->hat_next; in htable_steal()
625 while (hat != NULL && in htable_steal()
626 (hat->hat_flags & in htable_steal()
628 hat = hat->hat_next; in htable_steal()
630 if (hat == NULL) in htable_steal()
637 hat->hat_flags |= HAT_VICTIM; in htable_steal()
643 hat_enter(hat); in htable_steal()
644 while ((ht = hat->hat_ht_cached) != NULL && in htable_steal()
646 hat->hat_ht_cached = ht->ht_next; in htable_steal()
651 hat_exit(hat); in htable_steal()
657 htable_steal_active(hat, cnt, threshold, in htable_steal()
669 ASSERT(ht->ht_hat == hat); in htable_steal()
673 ptable_free(hat->hat_user_ptable); in htable_steal()
674 hat->hat_user_ptable = PFN_INVALID; in htable_steal()
694 kas.a_hat->hat_prev != hat) in htable_steal()
695 move_victim(hat); in htable_steal()
705 if (hat->hat_flags & HAT_VICTIM) { in htable_steal()
706 ASSERT(hat != kas.a_hat); in htable_steal()
707 hat->hat_flags &= ~HAT_VICTIM; in htable_steal()
712 hat = hat->hat_next; in htable_steal()
776 hat_t *hat, in htable_alloc() argument
787 if (level < 0 || level > TOP_LEVEL(hat)) in htable_alloc()
790 is_vlp = (hat->hat_flags & HAT_VLP) && level == VLP_LEVEL; in htable_alloc()
798 if (hat->hat_ht_cached != NULL && !is_bare) { in htable_alloc()
799 hat_enter(hat); in htable_alloc()
800 ht = hat->hat_ht_cached; in htable_alloc()
802 hat->hat_ht_cached = ht->ht_next; in htable_alloc()
807 hat_exit(hat); in htable_alloc()
836 ht->ht_hat = hat; in htable_alloc()
900 hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1); in htable_alloc()
901 if (hat->hat_user_ptable != PFN_INVALID) in htable_alloc()
908 block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable), in htable_alloc()
944 ht->ht_hat = hat; in htable_alloc()
962 (void) xen_kpm_page(hat->hat_user_ptable, PT_VALID); in htable_alloc()
976 hat_t *hat = ht->ht_hat; in htable_free() local
983 if (hat != NULL && in htable_free()
986 (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) { in htable_free()
989 hat_enter(hat); in htable_free()
990 ht->ht_next = hat->hat_ht_cached; in htable_free()
991 hat->hat_ht_cached = ht; in htable_free()
992 hat_exit(hat); in htable_free()
1005 if (ht->ht_level == mmu.max_level && hat != NULL) { in htable_free()
1006 ptable_free(hat->hat_user_ptable); in htable_free()
1007 hat->hat_user_ptable = PFN_INVALID; in htable_free()
1033 htable_purge_hat(hat_t *hat) in htable_purge_hat() argument
1041 if (!(hat->hat_flags & HAT_FREEING)) { in htable_purge_hat()
1044 hat_enter(hat); in htable_purge_hat()
1045 ht = hat->hat_ht_cached; in htable_purge_hat()
1047 hat_exit(hat); in htable_purge_hat()
1050 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1051 hat_exit(hat); in htable_purge_hat()
1061 while ((ht = hat->hat_ht_cached) != NULL) { in htable_purge_hat()
1062 hat->hat_ht_cached = ht->ht_next; in htable_purge_hat()
1069 for (h = 0; h < hat->hat_num_hash; ++h) { in htable_purge_hat()
1070 while ((ht = hat->hat_ht_hash[h]) != NULL) { in htable_purge_hat()
1077 ASSERT(hat->hat_ht_hash[h] == ht); in htable_purge_hat()
1078 hat->hat_ht_hash[h] = ht->ht_next; in htable_purge_hat()
1095 hat_t *hat = old->ht_hat; in unlink_ptp() local
1121 if (!(hat->hat_flags & HAT_FREEING)) { in unlink_ptp()
1122 hat_tlb_inval(hat, (higher->ht_flags & HTABLE_VLP) ? in unlink_ptp()
1180 hat_t *hat; in htable_release() local
1187 hat = ht->ht_hat; in htable_release()
1190 hashval = HTABLE_HASH(hat, va, level); in htable_release()
1214 if (hat->hat_flags & HAT_FREEING) in htable_release()
1222 (hat != kas.a_hat || va >= kernelbase)) in htable_release()
1258 ASSERT(hat->hat_ht_hash[hashval] == ht); in htable_release()
1259 hat->hat_ht_hash[hashval] = ht->ht_next; in htable_release()
1283 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level) in htable_lookup() argument
1290 ASSERT(level <= TOP_LEVEL(hat)); in htable_lookup()
1292 if (level == TOP_LEVEL(hat)) { in htable_lookup()
1298 if ((hat->hat_flags & HAT_VLP) && vaddr >= ((uint64_t)1 << 32)) in htable_lookup()
1306 hashval = HTABLE_HASH(hat, base, level); in htable_lookup()
1308 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) { in htable_lookup()
1309 if (ht->ht_hat == hat && in htable_lookup()
1327 hat_t *hat = ht->ht_hat; in htable_acquire() local
1330 uint_t hashval = HTABLE_HASH(hat, base, level); in htable_acquire()
1340 for (h = hat->hat_ht_hash[hashval]; in htable_acquire()
1362 hat_t *hat, in htable_create() argument
1374 if (level < 0 || level > TOP_LEVEL(hat)) in htable_create()
1380 for (l = TOP_LEVEL(hat); l >= level; --l) { in htable_create()
1382 if (l == TOP_LEVEL(hat)) in htable_create()
1387 h = HTABLE_HASH(hat, base, l); in htable_create()
1393 if (l == TOP_LEVEL(hat)) { in htable_create()
1394 ht = hat->hat_htable; in htable_create()
1396 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in htable_create()
1397 ASSERT(ht->ht_hat == hat); in htable_create()
1433 new = htable_alloc(hat, base, l, in htable_create()
1447 ht->ht_next = hat->hat_ht_hash[h]; in htable_create()
1449 if (hat->hat_ht_hash[h]) in htable_create()
1450 hat->hat_ht_hash[h]->ht_prev = ht; in htable_create()
1451 hat->hat_ht_hash[h] = ht; in htable_create()
1484 hat_t *hat, in htable_attach() argument
1501 ht->ht_hat = hat; in htable_attach()
1515 h = HTABLE_HASH(hat, base, level); in htable_attach()
1517 ht->ht_next = hat->hat_ht_hash[h]; in htable_attach()
1519 if (hat->hat_ht_hash[h]) in htable_attach()
1520 hat->hat_ht_hash[h]->ht_prev = ht; in htable_attach()
1521 hat->hat_ht_hash[h] = ht; in htable_attach()
1573 htable_attach(hat, base, level - 1, in htable_attach()
1672 struct hat *hat, in htable_walk() argument
1690 ASSERT(hat == kas.a_hat || eaddr <= kernelbase || in htable_walk()
1692 if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END) in htable_walk()
1723 if (hat->hat_ism_pgcnt > 0) { in htable_walk()
1728 if (hat->hat_pages_mapped[l] != 0) in htable_walk()
1736 for (l = 0; l <= TOP_LEVEL(hat); ++l) { in htable_walk()
1737 ht = htable_lookup(hat, va, l); in htable_walk()
1755 ASSERT(l < TOP_LEVEL(hat)); in htable_walk()
1777 struct hat *hat, in htable_getpte() argument
1790 ht = htable_lookup(hat, vaddr, l); in htable_getpte()
1810 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry) in htable_getpage() argument
1816 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level); in htable_getpage()
2103 hat_t *hat = ht->ht_hat; in x86pte_set() local
2163 hat_tlb_inval(hat, addr); in x86pte_set()
2497 hat_t *hat; in hat_dump() local
2504 for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) { in hat_dump()
2505 for (h = 0; h < hat->hat_num_hash; ++h) { in hat_dump()
2506 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) { in hat_dump()