Lines Matching refs:hat
224 hat_t *hat = buf; in hati_constructor() local
226 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); in hati_constructor()
227 bzero(hat->hat_pages_mapped, in hati_constructor()
229 hat->hat_ism_pgcnt = 0; in hati_constructor()
230 hat->hat_stats = 0; in hati_constructor()
231 hat->hat_flags = 0; in hati_constructor()
232 CPUSET_ZERO(hat->hat_cpus); in hati_constructor()
233 hat->hat_htable = NULL; in hati_constructor()
234 hat->hat_ht_hash = NULL; in hati_constructor()
245 hat_t *hat; in hat_alloc() local
264 hat = kmem_cache_alloc(hat_cache, KM_SLEEP); in hat_alloc()
265 hat->hat_as = as; in hat_alloc()
266 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL); in hat_alloc()
267 ASSERT(hat->hat_flags == 0); in hat_alloc()
285 hat->hat_flags = HAT_VLP; in hat_alloc()
286 bzero(hat->hat_vlp_ptes, VLP_SIZE); in hat_alloc()
292 if ((hat->hat_flags & HAT_VLP)) { in hat_alloc()
293 hat->hat_num_hash = mmu.vlp_hash_cnt; in hat_alloc()
294 hat->hat_ht_hash = kmem_cache_alloc(vlp_hash_cache, KM_SLEEP); in hat_alloc()
296 hat->hat_num_hash = mmu.hash_cnt; in hat_alloc()
297 hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_SLEEP); in hat_alloc()
299 bzero(hat->hat_ht_hash, hat->hat_num_hash * sizeof (htable_t *)); in hat_alloc()
305 hat->hat_htable = NULL; in hat_alloc()
306 hat->hat_ht_cached = NULL; in hat_alloc()
308 ht = htable_create(hat, (uintptr_t)0, TOP_LEVEL(hat), NULL); in hat_alloc()
309 hat->hat_htable = ht; in hat_alloc()
312 if (hat->hat_flags & HAT_VLP) in hat_alloc()
321 if (rp->hkr_level == TOP_LEVEL(hat)) in hat_alloc()
322 ht = hat->hat_htable; in hat_alloc()
324 ht = htable_create(hat, va, rp->hkr_level, in hat_alloc()
339 &hat->hat_vlp_ptes[start], in hat_alloc()
357 xen_pin(hat->hat_htable->ht_pfn, mmu.max_level); in hat_alloc()
359 xen_pin(hat->hat_user_ptable, mmu.max_level); in hat_alloc()
377 hat->hat_prev = NULL; in hat_alloc()
378 hat->hat_next = kas.a_hat->hat_next; in hat_alloc()
379 if (hat->hat_next) in hat_alloc()
380 hat->hat_next->hat_prev = hat; in hat_alloc()
382 kas.a_hat->hat_prev = hat; in hat_alloc()
383 kas.a_hat->hat_next = hat; in hat_alloc()
386 return (hat); in hat_alloc()
394 hat_free_start(hat_t *hat) in hat_free_start() argument
396 ASSERT(AS_WRITE_HELD(hat->hat_as)); in hat_free_start()
404 while (hat->hat_flags & HAT_VICTIM) in hat_free_start()
406 hat->hat_flags |= HAT_FREEING; in hat_free_start()
414 hat_free_end(hat_t *hat) in hat_free_end() argument
418 ASSERT(hat->hat_flags & HAT_FREEING); in hat_free_end()
423 ASSERT(CPU->cpu_current_hat != hat); in hat_free_end()
429 if (hat->hat_prev) in hat_free_end()
430 hat->hat_prev->hat_next = hat->hat_next; in hat_free_end()
432 kas.a_hat->hat_next = hat->hat_next; in hat_free_end()
433 if (hat->hat_next) in hat_free_end()
434 hat->hat_next->hat_prev = hat->hat_prev; in hat_free_end()
436 kas.a_hat->hat_prev = hat->hat_prev; in hat_free_end()
438 hat->hat_next = hat->hat_prev = NULL; in hat_free_end()
444 xen_unpin(hat->hat_htable->ht_pfn); in hat_free_end()
446 xen_unpin(hat->hat_user_ptable); in hat_free_end()
453 htable_purge_hat(hat); in hat_free_end()
458 if (hat->hat_flags & HAT_VLP) in hat_free_end()
462 kmem_cache_free(cache, hat->hat_ht_hash); in hat_free_end()
463 hat->hat_ht_hash = NULL; in hat_free_end()
465 hat->hat_flags = 0; in hat_free_end()
466 kmem_cache_free(hat_cache, hat); in hat_free_end()
952 reload_pae32(hat_t *hat, cpu_t *cpu) in reload_pae32() argument
964 src = hat->hat_vlp_ptes; in reload_pae32()
985 hat_switch(hat_t *hat) in hat_switch() argument
995 if (old == hat) in hat_switch()
1004 if (hat != kas.a_hat) { in hat_switch()
1005 CPUSET_ATOMIC_ADD(hat->hat_cpus, cpu->cpu_id); in hat_switch()
1007 cpu->cpu_current_hat = hat; in hat_switch()
1012 if (hat->hat_flags & HAT_VLP) { in hat_switch()
1016 VLP_COPY(hat->hat_vlp_ptes, vlpptep); in hat_switch()
1019 reload_pae32(hat, cpu); in hat_switch()
1024 newcr3 = MAKECR3((uint64_t)hat->hat_htable->ht_pfn); in hat_switch()
1041 if (hat == kas.a_hat) in hat_switch()
1044 t[1].arg1.mfn = pfn_to_mfn(hat->hat_user_ptable); in hat_switch()
1132 hat_swapin(hat_t *hat) in hat_swapin() argument
1142 hat_swapout(hat_t *hat) in hat_swapout() argument
1160 ASSERT(AS_LOCK_HELD(hat->hat_as)); in hat_swapout()
1161 if ((uintptr_t)hat->hat_as->a_userlimit < eaddr) in hat_swapout()
1162 eaddr = (uintptr_t)hat->hat_as->a_userlimit; in hat_swapout()
1165 (void) htable_walk(hat, &ht, &vaddr, eaddr); in hat_swapout()
1186 hat_unload(hat, (caddr_t)vaddr, LEVEL_SIZE(l), in hat_swapout()
1205 htable_purge_hat(hat); in hat_swapout()
1213 hat_get_mapped_size(hat_t *hat) in hat_get_mapped_size() argument
1219 total += (hat->hat_pages_mapped[l] << LEVEL_SHIFT(l)); in hat_get_mapped_size()
1220 total += hat->hat_ism_pgcnt; in hat_get_mapped_size()
1229 hat_stats_enable(hat_t *hat) in hat_stats_enable() argument
1231 atomic_inc_32(&hat->hat_stats); in hat_stats_enable()
1236 hat_stats_disable(hat_t *hat) in hat_stats_disable() argument
1238 atomic_dec_32(&hat->hat_stats); in hat_stats_disable()
1304 hat_t *hat = ht->ht_hat; in hati_pte_map() local
1321 is_locked = (flags & HAT_LOAD_LOCK) != 0 && hat != kas.a_hat; in hati_pte_map()
1369 VLP_COPY(hat->hat_vlp_ptes, vlpptep); in hati_pte_map()
1373 PGCNT_INC(hat, l); in hati_pte_map()
1421 hat_t *hat, in hati_load_common() argument
1441 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as)); in hati_load_common()
1444 hat->hat_flags |= HAT_SHARED; in hati_load_common()
1449 ht = htable_lookup(hat, va, level); in hati_load_common()
1458 ht = htable_create(hat, va, level, NULL); in hati_load_common()
1475 if (hat == kas.a_hat) in hati_load_common()
1478 if (hat == kas.a_hat && va >= kernelbase) in hati_load_common()
1576 hat_t *hat, in hat_memload() argument
1588 ASSERT(hat == kas.a_hat || va < _userlimit); in hat_memload()
1589 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as)); in hat_memload()
1599 ASSERT(hat == kas.a_hat); in hat_memload()
1610 if (hati_load_common(hat, va, pp, attr, flags, level, pfn) != 0) in hat_memload()
1617 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp, in hat_memload_region() argument
1620 hat_memload(hat, addr, pp, attr, flags); in hat_memload_region()
1628 hat_t *hat, in hat_memload_array() argument
1645 ASSERT(hat == kas.a_hat || va + len <= _userlimit); in hat_memload_array()
1646 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as)); in hat_memload_array()
1705 while (hati_load_common(hat, va, pages[pgindx], attr, in hat_memload_array()
1724 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len, in hat_memload_array_region() argument
1728 hat_memload_array(hat, addr, len, pps, attr, flags); in hat_memload_array_region()
1763 hat_t *hat, in hat_devload() argument
1780 ASSERT(hat == kas.a_hat || eva <= _userlimit); in hat_devload()
1781 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as)); in hat_devload()
1848 while (hati_load_common(hat, va, pp, a, f, level, pfn) != 0) { in hat_devload()
1871 hat_unlock(hat_t *hat, caddr_t addr, size_t len) in hat_unlock() argument
1880 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); in hat_unlock()
1883 if (hat == kas.a_hat) in hat_unlock()
1889 ASSERT(AS_LOCK_HELD(hat->hat_as)); in hat_unlock()
1891 (void) htable_walk(hat, &ht, &vaddr, eaddr); in hat_unlock()
1911 hat_unlock_region(struct hat *hat, caddr_t addr, size_t len, in hat_unlock_region() argument
1926 hat_t *hat = (hat_t *)a1; in hati_demap_func() local
1934 if (hat != kas.a_hat && hat != CPU->cpu_current_hat) in hati_demap_func()
1952 if (hat->hat_flags & HAT_VLP) { in hati_demap_func()
1956 VLP_COPY(hat->hat_vlp_ptes, vlpptep); in hati_demap_func()
1958 reload_pae32(hat, CPU); in hati_demap_func()
2037 hat_tlb_inval_range(hat_t *hat, uintptr_t va, size_t len) in hat_tlb_inval_range() argument
2052 if (hat->hat_flags & HAT_FREEING) in hat_tlb_inval_range()
2060 if (hat->hat_flags & HAT_SHARED) { in hat_tlb_inval_range()
2061 hat = kas.a_hat; in hat_tlb_inval_range()
2077 (void) hati_demap_func((xc_arg_t)hat, in hat_tlb_inval_range()
2090 if (hat == kas.a_hat) in hat_tlb_inval_range()
2093 cpus_to_shootdown = hat->hat_cpus; in hat_tlb_inval_range()
2136 (void) hati_demap_func((xc_arg_t)hat, in hat_tlb_inval_range()
2153 xc_call((xc_arg_t)hat, (xc_arg_t)va, (xc_arg_t)len, in hat_tlb_inval_range()
2162 hat_tlb_inval(hat_t *hat, uintptr_t va) in hat_tlb_inval() argument
2164 hat_tlb_inval_range(hat, va, MMU_PAGESIZE); in hat_tlb_inval()
2181 hat_t *hat = ht->ht_hat; in hat_pte_unmap() local
2190 if ((flags & HAT_UNLOAD_UNLOCK) != 0 && hat != kas.a_hat) { in hat_pte_unmap()
2267 PGCNT_DEC(hat, l); in hat_pte_unmap()
2311 hat_unload(hat_t *hat, caddr_t addr, size_t len, uint_t flags) in hat_unload() argument
2316 ASSERT(hat == kas.a_hat || va + len <= _userlimit); in hat_unload()
2322 ASSERT(hat == kas.a_hat); in hat_unload()
2325 hat_unload_callback(hat, addr, len, flags, NULL); in hat_unload()
2344 handle_ranges(hat_t *hat, hat_callback_t *cb, uint_t cnt, range_info_t *range) in handle_ranges() argument
2351 hat_tlb_inval_range(hat, (uintptr_t)range[cnt].rng_va, len); in handle_ranges()
2375 hat_t *hat, in hat_unload_callback() argument
2391 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); in hat_unload_callback()
2400 ht = htable_getpte(hat, vaddr, &entry, &old_pte, 0); in hat_unload_callback()
2413 old_pte = htable_walk(hat, &ht, &vaddr, eaddr); in hat_unload_callback()
2428 handle_ranges(hat, cb, r_cnt, r); in hat_unload_callback()
2460 handle_ranges(hat, cb, r_cnt, r); in hat_unload_callback()
2469 hat_flush_range(hat_t *hat, caddr_t va, size_t size) in hat_flush_range() argument
2475 sz = hat_getpagesize(hat, va); in hat_flush_range()
2501 hat_sync(hat_t *hat, caddr_t addr, size_t len, uint_t flags) in hat_sync() argument
2515 ASSERT(hat == kas.a_hat || eaddr <= _userlimit); in hat_sync()
2520 pte = htable_walk(hat, &ht, &vaddr, eaddr); in hat_sync()
2579 hat_map(hat_t *hat, caddr_t addr, size_t len, uint_t flags) in hat_map() argument
2591 hat_getattr(hat_t *hat, caddr_t addr, uint_t *attr) in hat_getattr() argument
2597 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); in hat_getattr()
2602 ht = htable_getpte(hat, vaddr, NULL, &pte, mmu.max_page_level); in hat_getattr()
2632 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what) in hat_updateattr() argument
2644 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as)); in hat_updateattr()
2647 oldpte = htable_walk(hat, &ht, &vaddr, eaddr); in hat_updateattr()
2735 hat_setattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) in hat_setattr() argument
2737 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); in hat_setattr()
2738 hat_updateattr(hat, addr, len, attr, HAT_SET_ATTR); in hat_setattr()
2742 hat_clrattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) in hat_clrattr() argument
2744 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); in hat_clrattr()
2745 hat_updateattr(hat, addr, len, attr, HAT_CLR_ATTR); in hat_clrattr()
2749 hat_chgattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr) in hat_chgattr() argument
2751 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); in hat_chgattr()
2752 hat_updateattr(hat, addr, len, attr, HAT_LOAD_ATTR); in hat_chgattr()
2756 hat_chgprot(hat_t *hat, caddr_t addr, size_t len, uint_t vprot) in hat_chgprot() argument
2758 ASSERT(hat == kas.a_hat || (uintptr_t)addr + len <= _userlimit); in hat_chgprot()
2759 hat_updateattr(hat, addr, len, vprot & HAT_PROT_MASK, HAT_LOAD_ATTR); in hat_chgprot()
2768 hat_getpagesize(hat_t *hat, caddr_t addr) in hat_getpagesize() argument
2774 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); in hat_getpagesize()
2777 ht = htable_getpage(hat, vaddr, NULL); in hat_getpagesize()
2792 hat_getpfnum(hat_t *hat, caddr_t addr) in hat_getpfnum() argument
2799 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); in hat_getpfnum()
2825 ht = htable_getpage(hat, vaddr, &entry); in hat_getpfnum()
2846 hat_probe(hat_t *hat, caddr_t addr) in hat_probe() argument
2853 ASSERT(hat == kas.a_hat || vaddr <= _userlimit); in hat_probe()
2854 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as)); in hat_probe()
2871 ht = htable_getpage(hat, vaddr, &entry); in hat_probe()
2880 is_it_dism(hat_t *hat, caddr_t va) in is_it_dism() argument
2886 seg = as_findseg(hat->hat_as, va, 0); in is_it_dism()
2907 hat_t *hat, in hat_share() argument
2934 ASSERT(hat != kas.a_hat); in hat_share()
2950 is_dism = is_it_dism(hat, addr); in hat_share()
2964 ht = htable_lookup(hat, vaddr, l); in hat_share()
3019 ht = htable_create(hat, vaddr, l, ism_ht); in hat_share()
3023 hat->hat_ism_pgcnt += in hat_share()
3075 while (hati_load_common(hat, vaddr, pp, prot, flags, in hat_share()
3102 hat_unshare(hat_t *hat, caddr_t addr, size_t len, uint_t ismszc) in hat_unshare() argument
3111 ASSERT(hat != kas.a_hat); in hat_unshare()
3134 ht = htable_lookup(hat, vaddr, l); in hat_unshare()
3142 hat->hat_ism_pgcnt -= ht->ht_valid_cnt << in hat_unshare()
3155 if (!(hat->hat_flags & HAT_FREEING) && need_demaps) in hat_unshare()
3156 hat_tlb_inval(hat, DEMAP_ALL_ADDR); in hat_unshare()
3162 if (!is_it_dism(hat, addr)) in hat_unshare()
3164 hat_unload(hat, addr, len, flags); in hat_unshare()
3793 hat_t *hat, in hat_softlock() argument
3848 hat_setup(hat_t *hat, int flags) in hat_setup() argument
3853 hat_switch(hat); in hat_setup()
4012 hat_enter(hat_t *hat) in hat_enter() argument
4014 mutex_enter(&hat->hat_mutex); in hat_enter()
4018 hat_exit(hat_t *hat) in hat_exit() argument
4020 mutex_exit(&hat->hat_mutex); in hat_exit()
4149 hat_join_srd(struct hat *hat, vnode_t *evp) in hat_join_srd() argument
4155 hat_join_region(struct hat *hat, in hat_join_region() argument
4171 hat_leave_region(struct hat *hat, hat_region_cookie_t rcookie, uint_t flags) in hat_leave_region() argument
4178 hat_dup_region(struct hat *hat, hat_region_cookie_t rcookie) in hat_dup_region() argument
4341 hat_kpm_fault(hat_t *hat, caddr_t vaddr) in hat_kpm_fault() argument
4344 (void *)hat, (void *)vaddr); in hat_kpm_fault()
4455 hat_prepare_mapping(hat_t *hat, caddr_t addr, uint64_t *pte_ma) in hat_prepare_mapping() argument
4463 ht = htable_create(hat, (uintptr_t)addr, 0, NULL); in hat_prepare_mapping()
4479 hat_release_mapping(hat_t *hat, caddr_t addr) in hat_release_mapping() argument
4485 ht = htable_lookup(hat, (uintptr_t)addr, 0); in hat_release_mapping()