Lines Matching refs:hmeblkp
113 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid) \ argument
119 int _ttesz = get_hblk_ttesz(hmeblkp); \
128 _hsva = (caddr_t)get_hblk_base(hmeblkp); \
129 _heva = get_hblk_endaddr(hmeblkp); \
146 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid) argument
852 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, ismhat) \ argument
854 int ttesz = get_hblk_ttesz(hmeblkp); \
859 (caddr_t)get_hblk_base(hmeblkp); \
860 caddr_t eva = sva + get_hblk_span(hmeblkp); \
991 #define HBLKTOHME(hment, hmeblkp, addr) \ argument
994 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \
1001 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ argument
1003 ASSERT(in_hblk_range((hmeblkp), (addr))); \
1005 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \
1010 (hment) = &(hmeblkp)->hblk_hme[idx]; \
1992 struct hme_blk *hmeblkp; in hat_swapout() local
2024 hmeblkp = hmebp->hmeblkp; in hat_swapout()
2026 while (hmeblkp) { in hat_swapout()
2028 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && in hat_swapout()
2029 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { in hat_swapout()
2030 ASSERT(!hmeblkp->hblk_shared); in hat_swapout()
2031 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in hat_swapout()
2032 (caddr_t)get_hblk_base(hmeblkp), in hat_swapout()
2033 get_hblk_endaddr(hmeblkp), in hat_swapout()
2036 nx_hblk = hmeblkp->hblk_next; in hat_swapout()
2037 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in hat_swapout()
2038 ASSERT(!hmeblkp->hblk_lckcnt); in hat_swapout()
2039 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in hat_swapout()
2042 pr_hblk = hmeblkp; in hat_swapout()
2044 hmeblkp = nx_hblk; in hat_swapout()
2563 struct hme_blk *hmeblkp; in sfmmu_memload_batchsmall() local
2577 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, in sfmmu_memload_batchsmall()
2579 ASSERT(hmeblkp); in sfmmu_memload_batchsmall()
2591 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, in sfmmu_memload_batchsmall()
2880 struct hme_blk *hmeblkp; in sfmmu_tteload_array() local
2899 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags, in sfmmu_tteload_array()
2901 ASSERT(hmeblkp); in sfmmu_tteload_array()
2906 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags, in sfmmu_tteload_array()
2950 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; in sfmmu_tteload_find_hmeblk() local
2963 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in sfmmu_tteload_find_hmeblk()
2970 if (hmeblkp == (struct hme_blk *)hblk_reserve && in sfmmu_tteload_find_hmeblk()
2980 if (hmeblkp == NULL) { in sfmmu_tteload_find_hmeblk()
2981 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, in sfmmu_tteload_find_hmeblk()
2983 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
2984 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
2992 if (get_hblk_ttesz(hmeblkp) != size) { in sfmmu_tteload_find_hmeblk()
2993 ASSERT(!hmeblkp->hblk_vcnt); in sfmmu_tteload_find_hmeblk()
2994 ASSERT(!hmeblkp->hblk_hmecnt); in sfmmu_tteload_find_hmeblk()
2995 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_tteload_find_hmeblk()
2999 if (hmeblkp->hblk_shw_bit) { in sfmmu_tteload_find_hmeblk()
3004 ASSERT(!hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
3005 if (hmeblkp->hblk_shw_mask) { in sfmmu_tteload_find_hmeblk()
3006 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); in sfmmu_tteload_find_hmeblk()
3010 hmeblkp->hblk_shw_bit = 0; in sfmmu_tteload_find_hmeblk()
3024 ASSERT(get_hblk_ttesz(hmeblkp) == size); in sfmmu_tteload_find_hmeblk()
3025 ASSERT(!hmeblkp->hblk_shw_bit); in sfmmu_tteload_find_hmeblk()
3026 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
3027 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
3028 ASSERT(hmeblkp->hblk_tag.htag_rid == rid); in sfmmu_tteload_find_hmeblk()
3030 return (hmeblkp); in sfmmu_tteload_find_hmeblk()
3038 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, in sfmmu_tteload_addentry() argument
3111 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared); in sfmmu_tteload_addentry()
3112 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared); in sfmmu_tteload_addentry()
3114 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); in sfmmu_tteload_addentry()
3147 (void *)hmeblkp); in sfmmu_tteload_addentry()
3194 if ((hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { in sfmmu_tteload_addentry()
3196 (void *)hmeblkp); in sfmmu_tteload_addentry()
3198 atomic_inc_32(&hmeblkp->hblk_lckcnt); in sfmmu_tteload_addentry()
3200 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); in sfmmu_tteload_addentry()
3226 chk_tte(&orig_old, &tteold, ttep, hmeblkp); in sfmmu_tteload_addentry()
3233 atomic_inc_16(&hmeblkp->hblk_vcnt); in sfmmu_tteload_addentry()
3316 if (hmeblkp->hblk_shared) { in sfmmu_tteload_addentry()
3318 sfmmup->sfmmu_srdp->srd_hmergnp[rid], hmeblkp, 1); in sfmmu_tteload_addentry()
3322 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); in sfmmu_tteload_addentry()
3352 atomic_inc_16(&hmeblkp->hblk_hmecnt); in sfmmu_tteload_addentry()
3353 ASSERT(hmeblkp->hblk_hmecnt > 0); in sfmmu_tteload_addentry()
3594 struct hme_blk *hmeblkp; in sfmmu_shadow_hcreate() local
3620 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in sfmmu_shadow_hcreate()
3621 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); in sfmmu_shadow_hcreate()
3622 if (hmeblkp == NULL) { in sfmmu_shadow_hcreate()
3623 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, in sfmmu_shadow_hcreate()
3626 ASSERT(hmeblkp); in sfmmu_shadow_hcreate()
3627 if (!hmeblkp->hblk_shw_mask) { in sfmmu_shadow_hcreate()
3633 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); in sfmmu_shadow_hcreate()
3634 hmeblkp->hblk_shw_bit = 1; in sfmmu_shadow_hcreate()
3635 } else if (hmeblkp->hblk_shw_bit == 0) { in sfmmu_shadow_hcreate()
3637 (void *)hmeblkp); in sfmmu_shadow_hcreate()
3639 ASSERT(hmeblkp->hblk_shw_bit == 1); in sfmmu_shadow_hcreate()
3640 ASSERT(!hmeblkp->hblk_shared); in sfmmu_shadow_hcreate()
3647 shw_mask = hmeblkp->hblk_shw_mask; in sfmmu_shadow_hcreate()
3649 newshw_mask = atomic_cas_32(&hmeblkp->hblk_shw_mask, shw_mask, in sfmmu_shadow_hcreate()
3655 return (hmeblkp); in sfmmu_shadow_hcreate()
3668 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_shadow_hcleanup() argument
3674 ASSERT(hmeblkp->hblk_shw_bit); in sfmmu_shadow_hcleanup()
3675 ASSERT(!hmeblkp->hblk_shared); in sfmmu_shadow_hcleanup()
3679 if (!hmeblkp->hblk_shw_mask) { in sfmmu_shadow_hcleanup()
3680 hmeblkp->hblk_shw_bit = 0; in sfmmu_shadow_hcleanup()
3683 addr = (caddr_t)get_hblk_base(hmeblkp); in sfmmu_shadow_hcleanup()
3684 endaddr = get_hblk_endaddr(hmeblkp); in sfmmu_shadow_hcleanup()
3685 size = get_hblk_ttesz(hmeblkp); in sfmmu_shadow_hcleanup()
3702 struct hme_blk *hmeblkp; in sfmmu_free_hblks() local
3717 hmeblkp = hmebp->hmeblkp; in sfmmu_free_hblks()
3719 while (hmeblkp) { in sfmmu_free_hblks()
3720 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { in sfmmu_free_hblks()
3722 ASSERT(!hmeblkp->hblk_shared); in sfmmu_free_hblks()
3723 if (hmeblkp->hblk_shw_bit) { in sfmmu_free_hblks()
3724 if (hmeblkp->hblk_shw_mask) { in sfmmu_free_hblks()
3727 hmeblkp, hmebp); in sfmmu_free_hblks()
3730 hmeblkp->hblk_shw_bit = 0; in sfmmu_free_hblks()
3743 nx_hblk = hmeblkp->hblk_next; in sfmmu_free_hblks()
3744 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in sfmmu_free_hblks()
3745 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_free_hblks()
3748 pr_hblk = hmeblkp; in sfmmu_free_hblks()
3750 hmeblkp = nx_hblk; in sfmmu_free_hblks()
3781 struct hme_blk *hmeblkp; in sfmmu_cleanup_rhblk() local
3796 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in sfmmu_cleanup_rhblk()
3797 if (hmeblkp != NULL) { in sfmmu_cleanup_rhblk()
3798 ASSERT(hmeblkp->hblk_shared); in sfmmu_cleanup_rhblk()
3799 ASSERT(!hmeblkp->hblk_shw_bit); in sfmmu_cleanup_rhblk()
3800 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_cleanup_rhblk()
3803 ASSERT(!hmeblkp->hblk_lckcnt); in sfmmu_cleanup_rhblk()
3804 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_cleanup_rhblk()
3829 struct hme_blk *hmeblkp; in sfmmu_unload_hmeregion_va() local
3845 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in sfmmu_unload_hmeregion_va()
3846 if (hmeblkp != NULL) { in sfmmu_unload_hmeregion_va()
3847 ASSERT(hmeblkp->hblk_shared); in sfmmu_unload_hmeregion_va()
3848 ASSERT(!hmeblkp->hblk_lckcnt); in sfmmu_unload_hmeregion_va()
3849 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_unload_hmeregion_va()
3850 *eaddrp = sfmmu_hblk_unload(NULL, hmeblkp, addr, in sfmmu_unload_hmeregion_va()
3854 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); in sfmmu_unload_hmeregion_va()
3855 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_unload_hmeregion_va()
3938 struct hme_blk *hmeblkp, *list = NULL; in hat_unlock() local
3964 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in hat_unlock()
3965 if (hmeblkp != NULL) { in hat_unlock()
3966 ASSERT(!hmeblkp->hblk_shared); in hat_unlock()
3974 if (hmeblkp->hblk_shw_bit) { in hat_unlock()
3977 addr = sfmmu_hblk_unlock(hmeblkp, addr, in hat_unlock()
4015 struct hme_blk *hmeblkp; in hat_unlock_region() local
4060 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, in hat_unlock_region()
4062 if (hmeblkp == NULL) { in hat_unlock_region()
4067 ASSERT(hmeblkp->hblk_shared); in hat_unlock_region()
4068 va = sfmmu_hblk_unlock(hmeblkp, va, eaddr); in hat_unlock_region()
4088 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) in sfmmu_hblk_unlock() argument
4094 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_unlock()
4095 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_unlock()
4097 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_unlock()
4098 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_unlock()
4100 HBLKTOHME(sfhme, hmeblkp, addr); in sfmmu_hblk_unlock()
4114 if (hmeblkp->hblk_lckcnt == 0) in sfmmu_hblk_unlock()
4121 ASSERT(hmeblkp->hblk_lckcnt > 0); in sfmmu_hblk_unlock()
4122 atomic_dec_32(&hmeblkp->hblk_lckcnt); in sfmmu_hblk_unlock()
4123 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); in sfmmu_hblk_unlock()
4291 struct hme_blk *hmeblkp; in hat_add_callback() local
4335 for (hashno = TTE64K, hmeblkp = NULL; in hat_add_callback()
4336 hmeblkp == NULL && hashno <= mmu_hashcnt; in hat_add_callback()
4347 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in hat_add_callback()
4349 if (hmeblkp == NULL) in hat_add_callback()
4353 if (hmeblkp == NULL) { in hat_add_callback()
4359 ASSERT(!hmeblkp->hblk_shared); in hat_add_callback()
4361 HBLKTOHME(osfhmep, hmeblkp, saddr); in hat_add_callback()
4375 baseaddr = (caddr_t)get_hblk_base(hmeblkp); in hat_add_callback()
4520 struct hme_blk *hmeblkp; in hat_delete_callback() local
4543 for (hashno = TTE64K, hmeblkp = NULL; in hat_delete_callback()
4544 hmeblkp == NULL && hashno <= mmu_hashcnt; in hat_delete_callback()
4555 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in hat_delete_callback()
4557 if (hmeblkp == NULL) in hat_delete_callback()
4561 if (hmeblkp == NULL) in hat_delete_callback()
4564 ASSERT(!hmeblkp->hblk_shared); in hat_delete_callback()
4566 HBLKTOHME(osfhmep, hmeblkp, saddr); in hat_delete_callback()
4839 struct hme_blk *hmeblkp, *list = NULL; in sfmmu_chgattr() local
4869 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in sfmmu_chgattr()
4870 if (hmeblkp != NULL) { in sfmmu_chgattr()
4871 ASSERT(!hmeblkp->hblk_shared); in sfmmu_chgattr()
4876 if (hmeblkp->hblk_shw_bit) { in sfmmu_chgattr()
4883 hmeblkp, addr, endaddr, &dmr, attr, mode); in sfmmu_chgattr()
4929 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_chgattr() argument
4943 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_chgattr()
4944 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_chgattr()
4945 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_chgattr()
4947 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_chgattr()
4948 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_chgattr()
4967 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_chgattr()
5040 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_hblk_chgattr()
5168 struct hme_blk *hmeblkp, *list = NULL; in hat_chgprot() local
5198 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in hat_chgprot()
5199 if (hmeblkp != NULL) { in hat_chgprot()
5200 ASSERT(!hmeblkp->hblk_shared); in hat_chgprot()
5205 if (hmeblkp->hblk_shw_bit) { in hat_chgprot()
5211 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, in hat_chgprot()
5257 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_chgprot() argument
5273 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_chgprot()
5274 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_chgprot()
5275 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_chgprot()
5278 if (get_hblk_ttesz(hmeblkp) != TTE8K && in sfmmu_hblk_chgprot()
5279 (endaddr < get_hblk_endaddr(hmeblkp))) { in sfmmu_hblk_chgprot()
5284 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_chgprot()
5285 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_chgprot()
5293 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_chgprot()
5368 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_hblk_chgprot()
5452 struct hme_blk *hmeblkp; in hat_unload_large_virtual() local
5480 hmeblkp = hmebp->hmeblkp; in hat_unload_large_virtual()
5482 while (hmeblkp) { in hat_unload_large_virtual()
5483 nx_hblk = hmeblkp->hblk_next; in hat_unload_large_virtual()
5489 if (hmeblkp->hblk_tag.htag_id != sfmmup || in hat_unload_large_virtual()
5490 hmeblkp->hblk_shw_bit || in hat_unload_large_virtual()
5491 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || in hat_unload_large_virtual()
5492 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { in hat_unload_large_virtual()
5493 pr_hblk = hmeblkp; in hat_unload_large_virtual()
5497 ASSERT(!hmeblkp->hblk_shared); in hat_unload_large_virtual()
5501 if (hmeblkp->hblk_vcnt != 0 || in hat_unload_large_virtual()
5502 hmeblkp->hblk_hmecnt != 0) in hat_unload_large_virtual()
5503 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in hat_unload_large_virtual()
5511 !hmeblkp->hblk_vcnt && in hat_unload_large_virtual()
5512 !hmeblkp->hblk_hmecnt) { in hat_unload_large_virtual()
5513 ASSERT(!hmeblkp->hblk_lckcnt); in hat_unload_large_virtual()
5514 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in hat_unload_large_virtual()
5517 pr_hblk = hmeblkp; in hat_unload_large_virtual()
5551 hmeblkp = nx_hblk; in hat_unload_large_virtual()
5593 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; in hat_unload_callback() local
5678 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in hat_unload_callback()
5679 if (hmeblkp == NULL) { in hat_unload_callback()
5722 ASSERT(hmeblkp); in hat_unload_callback()
5723 ASSERT(!hmeblkp->hblk_shared); in hat_unload_callback()
5724 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in hat_unload_callback()
5734 get_hblk_span(hmeblkp)); in hat_unload_callback()
5737 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in hat_unload_callback()
5770 if (hmeblkp->hblk_shw_bit) { in hat_unload_callback()
5794 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, in hat_unload_callback()
5801 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in hat_unload_callback()
5802 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 0); in hat_unload_callback()
5910 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_unload() argument
5922 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_unload()
5923 ASSERT(!hmeblkp->hblk_shw_bit); in sfmmu_hblk_unload()
5924 ASSERT(sfmmup != NULL || hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5925 ASSERT(sfmmup == NULL || !hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5926 ASSERT(dmrp == NULL || !hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5929 if (get_hblk_ttesz(hmeblkp) != TTE8K && in sfmmu_hblk_unload()
5930 (endaddr < get_hblk_endaddr(hmeblkp))) { in sfmmu_hblk_unload()
5935 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_unload()
5936 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_unload()
5947 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_unload()
6020 ASSERT(hmeblkp->hblk_lckcnt > 0); in sfmmu_hblk_unload()
6021 atomic_dec_32(&hmeblkp->hblk_lckcnt); in sfmmu_hblk_unload()
6022 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); in sfmmu_hblk_unload()
6058 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_unload()
6059 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, in sfmmu_hblk_unload()
6067 ASSERT(hmeblkp->hblk_hmecnt > 0); in sfmmu_hblk_unload()
6075 atomic_dec_16(&hmeblkp->hblk_hmecnt); in sfmmu_hblk_unload()
6078 ASSERT(hmeblkp->hblk_vcnt > 0); in sfmmu_hblk_unload()
6079 atomic_dec_16(&hmeblkp->hblk_vcnt); in sfmmu_hblk_unload()
6081 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || in sfmmu_hblk_unload()
6082 !hmeblkp->hblk_lckcnt); in sfmmu_hblk_unload()
6120 } else if (hmeblkp->hblk_hmecnt != 0) { in sfmmu_hblk_unload()
6215 struct hme_blk *hmeblkp, *list = NULL; in hat_sync() local
6246 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in hat_sync()
6247 if (hmeblkp != NULL) { in hat_sync()
6248 ASSERT(!hmeblkp->hblk_shared); in hat_sync()
6253 if (hmeblkp->hblk_shw_bit) { in hat_sync()
6259 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, in hat_sync()
6294 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_sync() argument
6304 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_sync()
6305 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_sync()
6307 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_sync()
6309 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_sync()
6310 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_sync()
6346 hmeblkp, 0, 0); in sfmmu_hblk_sync()
6606 struct hme_blk *hmeblkp; in hat_pagesuspend() local
6653 hmeblkp = sfmmu_hmetohblk(sfhmep); in hat_pagesuspend()
6655 sfmmup = hblktosfmmu(hmeblkp); in hat_pagesuspend()
6657 ASSERT(!hmeblkp->hblk_shared); in hat_pagesuspend()
6659 addr = tte_to_vaddr(hmeblkp, tte); in hat_pagesuspend()
6666 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in hat_pagesuspend()
7017 struct hme_blk *hmeblkp; in hat_pageunload() local
7059 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_pageunload()
7065 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && in hat_pageunload()
7066 hmeblkp->hblk_tag.htag_id == ksfmmup) in hat_pageunload()
7142 struct hme_blk *hmeblkp; in sfmmu_pageunload() local
7159 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_pageunload()
7164 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pageunload()
7165 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_pageunload()
7188 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); in sfmmu_pageunload()
7197 addr = tte_to_vaddr(hmeblkp, tte); in sfmmu_pageunload()
7199 if (hmeblkp->hblk_shared) { in sfmmu_pageunload()
7201 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_pageunload()
7207 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_pageunload()
7208 cpuset = sfmmu_rgntlb_demap(addr, rgnp, hmeblkp, 1); in sfmmu_pageunload()
7243 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, in sfmmu_pageunload()
7250 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_pageunload()
7273 ASSERT(hmeblkp->hblk_hmecnt > 0); in sfmmu_pageunload()
7274 ASSERT(hmeblkp->hblk_vcnt > 0); in sfmmu_pageunload()
7275 atomic_dec_16(&hmeblkp->hblk_vcnt); in sfmmu_pageunload()
7276 atomic_dec_16(&hmeblkp->hblk_hmecnt); in sfmmu_pageunload()
7365 struct hme_blk *hmeblkp; in hat_pagesync() local
7415 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_pagesync()
7421 if (hmeblkp->hblk_shared) { in hat_pagesync()
7422 sf_srd_t *srdp = hblktosrd(hmeblkp); in hat_pagesync()
7423 uint_t rid = hmeblkp->hblk_tag.htag_rid; in hat_pagesync()
7429 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, in hat_pagesync()
7485 struct hme_blk *hmeblkp; in sfmmu_pagesync() local
7503 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_pagesync()
7504 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pagesync()
7505 addr = tte_to_vaddr(hmeblkp, tte); in sfmmu_pagesync()
7521 if (hmeblkp->hblk_shared) { in sfmmu_pagesync()
7524 hmeblkp->hblk_tag.htag_rid; in sfmmu_pagesync()
7530 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_pagesync()
7533 rgnp, hmeblkp, 1); in sfmmu_pagesync()
7535 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, in sfmmu_pagesync()
7541 sfmmu_ttesync(hmeblkp->hblk_shared ? NULL : sfmmup, addr, in sfmmu_pagesync()
7559 struct hme_blk *hmeblkp; in sfmmu_pageclrwrt() local
7574 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_pageclrwrt()
7575 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pageclrwrt()
7576 addr = tte_to_vaddr(hmeblkp, tte); in sfmmu_pageclrwrt()
7592 if (hmeblkp->hblk_shared) { in sfmmu_pageclrwrt()
7594 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_pageclrwrt()
7600 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_pageclrwrt()
7603 rgnp, hmeblkp, 1); in sfmmu_pageclrwrt()
7605 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_pageclrwrt()
7870 struct hme_blk *hmeblkp = NULL; in sfmmu_uvatopfn() local
7929 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in sfmmu_uvatopfn()
7930 if (hmeblkp != NULL) { in sfmmu_uvatopfn()
7931 ASSERT(!hmeblkp->hblk_shared); in sfmmu_uvatopfn()
7932 HBLKTOHME(sfhmep, hmeblkp, vaddr); in sfmmu_uvatopfn()
7960 for (hmeblkp = hmebp->hmeblkp; hmeblkp != NULL; in sfmmu_uvatopfn()
7961 hmeblkp = hmeblkp->hblk_next) { in sfmmu_uvatopfn()
7967 if (!HTAGS_EQ_SHME(hmeblkp->hblk_tag, hblktag, in sfmmu_uvatopfn()
7971 ASSERT(hmeblkp->hblk_shared); in sfmmu_uvatopfn()
7972 rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_uvatopfn()
7976 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_uvatopfn()
7977 HBLKTOHME(sfhmep, hmeblkp, sv_vaddr); in sfmmu_uvatopfn()
7983 get_hblk_ttesz(hmeblkp) > TTE8K) { in sfmmu_uvatopfn()
7984 caddr_t eva = tte_to_evaddr(hmeblkp, ttep); in sfmmu_uvatopfn()
8000 } else if (get_hblk_ttesz(hmeblkp) > TTE8K || in sfmmu_uvatopfn()
8086 struct hme_blk *hmeblkp; in hat_page_checkshare() local
8113 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_page_checkshare()
8118 if (hmeblkp->hblk_shared) { in hat_page_checkshare()
8119 sf_srd_t *srdp = hblktosrd(hmeblkp); in hat_page_checkshare()
8120 uint_t rid = hmeblkp->hblk_tag.htag_rid; in hat_page_checkshare()
8126 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, in hat_page_checkshare()
8186 struct hme_blk *hmeblkp; in hat_page_demote() local
8224 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_page_demote()
8883 struct hme_blk *hmeblkp; in sfmmu_hblkcache_constructor() local
8886 hmeblkp = (struct hme_blk *)buf; in sfmmu_hblkcache_constructor()
8887 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); in sfmmu_hblkcache_constructor()
8890 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); in sfmmu_hblkcache_constructor()
8903 struct hme_blk *hmeblkp; in sfmmu_hblkcache_destructor() local
8905 hmeblkp = (struct hme_blk *)buf; in sfmmu_hblkcache_destructor()
8906 mutex_destroy(&hmeblkp->hblk_audit_lock); in sfmmu_hblkcache_destructor()
8925 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; in sfmmu_hblkcache_reclaim() local
8971 hmeblkp = hmebp->hmeblkp; in sfmmu_hblkcache_reclaim()
8973 while (hmeblkp) { in sfmmu_hblkcache_reclaim()
8974 nx_hblk = hmeblkp->hblk_next; in sfmmu_hblkcache_reclaim()
8975 if (!hmeblkp->hblk_vcnt && in sfmmu_hblkcache_reclaim()
8976 !hmeblkp->hblk_hmecnt) { in sfmmu_hblkcache_reclaim()
8977 sfmmu_hblk_hash_rm(hmebp, hmeblkp, in sfmmu_hblkcache_reclaim()
8980 pr_hblk = hmeblkp; in sfmmu_hblkcache_reclaim()
8982 hmeblkp = nx_hblk; in sfmmu_hblkcache_reclaim()
8997 hmeblkp = hmebp->hmeblkp; in sfmmu_hblkcache_reclaim()
8999 while (hmeblkp) { in sfmmu_hblkcache_reclaim()
9000 nx_hblk = hmeblkp->hblk_next; in sfmmu_hblkcache_reclaim()
9001 if (!hmeblkp->hblk_vcnt && in sfmmu_hblkcache_reclaim()
9002 !hmeblkp->hblk_hmecnt) { in sfmmu_hblkcache_reclaim()
9003 sfmmu_hblk_hash_rm(hmebp, hmeblkp, in sfmmu_hblkcache_reclaim()
9006 pr_hblk = hmeblkp; in sfmmu_hblkcache_reclaim()
9008 hmeblkp = nx_hblk; in sfmmu_hblkcache_reclaim()
9078 struct hme_blk *hmeblkp; in sfmmu_vac_conflict() local
9150 hmeblkp = sfmmu_hmetohblk(sfhmep); in sfmmu_vac_conflict()
9151 tmphat = hblktosfmmu(hmeblkp); in sfmmu_vac_conflict()
9154 if (hmeblkp->hblk_shared || tmphat == hat || in sfmmu_vac_conflict()
9155 hmeblkp->hblk_lckcnt) { in sfmmu_vac_conflict()
9176 hmeblkp = sfmmu_hmetohblk(sfhmep); in sfmmu_vac_conflict()
9177 ASSERT(!hmeblkp->hblk_shared); in sfmmu_vac_conflict()
9286 struct hme_blk *hmeblkp; in tst_tnc() local
9323 hmeblkp = sfmmu_hmetohblk(sfhme); in tst_tnc()
9328 vaddr = tte_to_vaddr(hmeblkp, tte); in tst_tnc()
9455 struct hme_blk *hmeblkp; in sfmmu_page_cache() local
9469 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_page_cache()
9473 vaddr = tte_to_vaddr(hmeblkp, tte); in sfmmu_page_cache()
9499 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_page_cache()
9504 if (hmeblkp->hblk_shared) { in sfmmu_page_cache()
9506 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_page_cache()
9512 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_page_cache()
9515 hmeblkp, 0); in sfmmu_page_cache()
9523 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9526 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9539 if (hmeblkp->hblk_shared) { in sfmmu_page_cache()
9541 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_page_cache()
9547 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_page_cache()
9550 hmeblkp, 0); in sfmmu_page_cache()
9557 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9560 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); in sfmmu_page_cache()
10566 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) in sfmmu_put_free_hblk() argument
10570 ASSERT(hmeblkp->hblk_hmecnt == 0); in sfmmu_put_free_hblk()
10571 ASSERT(hmeblkp->hblk_vcnt == 0); in sfmmu_put_free_hblk()
10572 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); in sfmmu_put_free_hblk()
10590 hmeblkp->hblk_next = freehblkp; in sfmmu_put_free_hblk()
10591 freehblkp = hmeblkp; in sfmmu_put_free_hblk()
10672 hblkp = hmebp->hmeblkp; hblkp != NULL && hblkp != old; in sfmmu_hblk_swap()
10925 struct hme_blk *hmeblkp = NULL; in sfmmu_hblk_alloc() local
10963 hmeblkp = in sfmmu_hblk_alloc()
10978 hmeblkp = in sfmmu_hblk_alloc()
11053 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) in sfmmu_hblk_alloc()
11066 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { in sfmmu_hblk_alloc()
11067 hmeblkp = sfmmu_hblk_steal(size); in sfmmu_hblk_alloc()
11077 sfmmu_hblk_swap(hmeblkp); in sfmmu_hblk_alloc()
11087 if (sfmmu_put_free_hblk(hmeblkp, 0)) in sfmmu_hblk_alloc()
11098 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { in sfmmu_hblk_alloc()
11105 hmeblkp = HBLK_RESERVE; in sfmmu_hblk_alloc()
11111 ASSERT(hmeblkp != NULL); in sfmmu_hblk_alloc()
11112 set_hblk_sz(hmeblkp, size); in sfmmu_hblk_alloc()
11113 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); in sfmmu_hblk_alloc()
11118 if (hmeblkp != HBLK_RESERVE) { in sfmmu_hblk_alloc()
11153 sfmmu_put_free_hblk(hmeblkp, forcefree)) { in sfmmu_hblk_alloc()
11157 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); in sfmmu_hblk_alloc()
11217 hmeblkp->hblk_shared = 1; in sfmmu_hblk_alloc()
11219 hmeblkp->hblk_shared = 0; in sfmmu_hblk_alloc()
11221 set_hblk_sz(hmeblkp, size); in sfmmu_hblk_alloc()
11223 hmeblkp->hblk_next = (struct hme_blk *)NULL; in sfmmu_hblk_alloc()
11224 hmeblkp->hblk_tag = hblktag; in sfmmu_hblk_alloc()
11225 hmeblkp->hblk_shadow = shw_hblkp; in sfmmu_hblk_alloc()
11226 hblkpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_alloc()
11227 hmeblkp->hblk_nextpa = HMEBLK_ENDPA; in sfmmu_hblk_alloc()
11229 ASSERT(get_hblk_ttesz(hmeblkp) == size); in sfmmu_hblk_alloc()
11230 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); in sfmmu_hblk_alloc()
11231 ASSERT(hmeblkp->hblk_hmecnt == 0); in sfmmu_hblk_alloc()
11232 ASSERT(hmeblkp->hblk_vcnt == 0); in sfmmu_hblk_alloc()
11233 ASSERT(hmeblkp->hblk_lckcnt == 0); in sfmmu_hblk_alloc()
11234 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); in sfmmu_hblk_alloc()
11235 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); in sfmmu_hblk_alloc()
11236 return (hmeblkp); in sfmmu_hblk_alloc()
11246 struct hme_blk *hmeblkp, *next_hmeblkp; in sfmmu_hblk_free() local
11253 hmeblkp = *listp; in sfmmu_hblk_free()
11254 while (hmeblkp != NULL) { in sfmmu_hblk_free()
11255 next_hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_free()
11256 ASSERT(!hmeblkp->hblk_hmecnt); in sfmmu_hblk_free()
11257 ASSERT(!hmeblkp->hblk_vcnt); in sfmmu_hblk_free()
11258 ASSERT(!hmeblkp->hblk_lckcnt); in sfmmu_hblk_free()
11259 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); in sfmmu_hblk_free()
11260 ASSERT(hmeblkp->hblk_shared == 0); in sfmmu_hblk_free()
11261 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_free()
11262 ASSERT(hmeblkp->hblk_shadow == NULL); in sfmmu_hblk_free()
11264 hblkpa = va_to_pa((caddr_t)hmeblkp); in sfmmu_hblk_free()
11266 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; in sfmmu_hblk_free()
11268 size = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_free()
11269 hmeblkp->hblk_next = NULL; in sfmmu_hblk_free()
11270 hmeblkp->hblk_nextpa = hblkpa; in sfmmu_hblk_free()
11272 if (hmeblkp->hblk_nuc_bit == 0) { in sfmmu_hblk_free()
11275 !sfmmu_put_free_hblk(hmeblkp, critical)) in sfmmu_hblk_free()
11276 kmem_cache_free(get_hblk_cache(hmeblkp), in sfmmu_hblk_free()
11277 hmeblkp); in sfmmu_hblk_free()
11279 hmeblkp = next_hmeblkp; in sfmmu_hblk_free()
11301 struct hme_blk *hmeblkp = NULL, *pr_hblk; in sfmmu_hblk_steal() local
11308 if ((hmeblkp = sfmmu_check_pending_hblks(size)) != NULL) { in sfmmu_hblk_steal()
11309 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); in sfmmu_hblk_steal()
11310 ASSERT(hmeblkp->hblk_hmecnt == 0); in sfmmu_hblk_steal()
11311 ASSERT(hmeblkp->hblk_vcnt == 0); in sfmmu_hblk_steal()
11312 return (hmeblkp); in sfmmu_hblk_steal()
11318 if (sfmmu_get_free_hblk(&hmeblkp, critical)) in sfmmu_hblk_steal()
11319 return (hmeblkp); in sfmmu_hblk_steal()
11326 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + in sfmmu_hblk_steal()
11329 hmeblkp = hmebp->hmeblkp; in sfmmu_hblk_steal()
11332 while (hmeblkp) { in sfmmu_hblk_steal()
11338 if ((get_hblk_ttesz(hmeblkp) == size) && in sfmmu_hblk_steal()
11339 (hmeblkp->hblk_shw_bit == 0 || in sfmmu_hblk_steal()
11340 hmeblkp->hblk_vcnt == 0) && in sfmmu_hblk_steal()
11341 (hmeblkp->hblk_lckcnt == 0)) { in sfmmu_hblk_steal()
11348 if ((hmeblkp->hblk_vcnt == 0 && in sfmmu_hblk_steal()
11349 hmeblkp->hblk_hmecnt == 0) || (i >= in sfmmu_hblk_steal()
11352 hmeblkp, hblkpa, pr_hblk)) { in sfmmu_hblk_steal()
11361 pr_hblk = hmeblkp; in sfmmu_hblk_steal()
11362 hblkpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_steal()
11363 hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_steal()
11372 if (hmeblkp != NULL) in sfmmu_hblk_steal()
11381 hmeblkp = hmebp->hmeblkp; in sfmmu_hblk_steal()
11384 while (hmeblkp) { in sfmmu_hblk_steal()
11388 if ((get_hblk_ttesz(hmeblkp) == size) && in sfmmu_hblk_steal()
11389 (hmeblkp->hblk_lckcnt == 0) && in sfmmu_hblk_steal()
11390 (hmeblkp->hblk_vcnt == 0) && in sfmmu_hblk_steal()
11391 (hmeblkp->hblk_hmecnt == 0)) { in sfmmu_hblk_steal()
11393 hmeblkp, hblkpa, pr_hblk)) { in sfmmu_hblk_steal()
11404 pr_hblk = hmeblkp; in sfmmu_hblk_steal()
11405 hblkpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_steal()
11406 hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_steal()
11414 if (hmeblkp != NULL) in sfmmu_hblk_steal()
11418 return (hmeblkp); in sfmmu_hblk_steal()
11429 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, in sfmmu_steal_this_hblk() argument
11443 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_steal_this_hblk()
11447 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_steal_this_hblk()
11448 if (hmeblkp->hblk_shared || sfmmup->sfmmu_ismhat) { in sfmmu_steal_this_hblk()
11452 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in sfmmu_steal_this_hblk()
11453 (caddr_t)get_hblk_base(hmeblkp), in sfmmu_steal_this_hblk()
11454 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); in sfmmu_steal_this_hblk()
11456 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_steal_this_hblk()
11466 ASSERT(hmeblkp->hblk_lckcnt == 0); in sfmmu_steal_this_hblk()
11467 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); in sfmmu_steal_this_hblk()
11469 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 1); in sfmmu_steal_this_hblk()
11470 hmeblkp->hblk_nextpa = hblkpa; in sfmmu_steal_this_hblk()
11472 shw_hblkp = hmeblkp->hblk_shadow; in sfmmu_steal_this_hblk()
11474 ASSERT(!hmeblkp->hblk_shared); in sfmmu_steal_this_hblk()
11476 vaddr = (caddr_t)get_hblk_base(hmeblkp); in sfmmu_steal_this_hblk()
11489 hmeblkp->hblk_shadow = NULL; in sfmmu_steal_this_hblk()
11497 hmeblkp->hblk_shw_bit = 0; in sfmmu_steal_this_hblk()
11499 if (hmeblkp->hblk_shared) { in sfmmu_steal_this_hblk()
11504 srdp = hblktosrd(hmeblkp); in sfmmu_steal_this_hblk()
11506 rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_steal_this_hblk()
11511 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_steal_this_hblk()
11512 hmeblkp->hblk_shared = 0; in sfmmu_steal_this_hblk()
11524 struct hme_blk *hmeblkp; in sfmmu_hmetohblk() local
11534 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - in sfmmu_hmetohblk()
11537 return (hmeblkp); in sfmmu_hmetohblk()
11937 struct hme_blk *hmeblkp, int uselocks) in sfmmu_rgntlb_demap() argument
11947 ASSERT(hmeblkp->hblk_shared); in sfmmu_rgntlb_demap()
11980 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_rgntlb_demap()
12055 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) in sfmmu_ismtlbcache_demap() argument
12068 ASSERT(!hmeblkp->hblk_shared); in sfmmu_ismtlbcache_demap()
12115 SFMMU_UNLOAD_TSB(va, sfmmup, hmeblkp, 1); in sfmmu_ismtlbcache_demap()
12151 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_tlbcache_demap() argument
12161 ASSERT(!hmeblkp->hblk_shared); in sfmmu_tlbcache_demap()
12194 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_tlbcache_demap()
12240 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_tlb_demap() argument
12246 ASSERT(!hmeblkp->hblk_shared); in sfmmu_tlb_demap()
12259 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_tlb_demap()
13162 struct hme_blk *hmeblkp; in sfmmu_init_nucleus_hblks() local
13185 hmeblkp = (struct hme_blk *)addr; in sfmmu_init_nucleus_hblks()
13187 hmeblkp->hblk_nuc_bit = 1; in sfmmu_init_nucleus_hblks()
13188 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); in sfmmu_init_nucleus_hblks()
13197 hmeblkp = (struct hme_blk *)addr; in sfmmu_init_nucleus_hblks()
13199 hmeblkp->hblk_nuc_bit = 1; in sfmmu_init_nucleus_hblks()
13200 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); in sfmmu_init_nucleus_hblks()
13230 struct hme_blk *hmeblkp; in sfmmu_rm_large_mappings() local
13249 hmeblkp = sfmmu_hmetohblk(sfhmep); in sfmmu_rm_large_mappings()
13398 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) in chk_tte() argument
13408 hmeblkp = hmeblkp; in chk_tte()
15446 sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, in sfmmu_hblk_hash_add() argument
15451 if (hmebp->hmeblkp == NULL) { in sfmmu_hblk_hash_add()
15456 hmeblkp->hblk_nextpa = hmebp->hmeh_nextpa; in sfmmu_hblk_hash_add()
15465 hmeblkp->hblk_next = hmebp->hmeblkp; in sfmmu_hblk_hash_add()
15466 hmebp->hmeblkp = hmeblkp; in sfmmu_hblk_hash_add()
15502 sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, in sfmmu_hblk_hash_rm() argument
15514 if (hmebp->hmeblkp == hmeblkp) { in sfmmu_hblk_hash_rm()
15515 hmebp->hmeh_nextpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_hash_rm()
15516 hmebp->hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_hash_rm()
15518 pr_hblk->hblk_nextpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_hash_rm()
15519 pr_hblk->hblk_next = hmeblkp->hblk_next; in sfmmu_hblk_hash_rm()
15522 size = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_hash_rm()
15523 shw_hblkp = hmeblkp->hblk_shadow; in sfmmu_hblk_hash_rm()
15525 ASSERT(hblktosfmmu(hmeblkp) != KHATID); in sfmmu_hblk_hash_rm()
15526 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_hash_rm()
15536 vaddr = (caddr_t)get_hblk_base(hmeblkp); in sfmmu_hblk_hash_rm()
15549 hmeblkp->hblk_shadow = NULL; in sfmmu_hblk_hash_rm()
15551 hmeblkp->hblk_shw_bit = 0; in sfmmu_hblk_hash_rm()
15553 if (hmeblkp->hblk_shared) { in sfmmu_hblk_hash_rm()
15559 srdp = hblktosrd(hmeblkp); in sfmmu_hblk_hash_rm()
15561 rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_hblk_hash_rm()
15566 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_hblk_hash_rm()
15568 hmeblkp->hblk_shared = 0; in sfmmu_hblk_hash_rm()
15577 hmeblkp->hblk_nextpa = HMEBLK_ENDPA; in sfmmu_hblk_hash_rm()
15578 hmeblkp->hblk_next = NULL; in sfmmu_hblk_hash_rm()
15581 hmeblkp->hblk_next = *listp; in sfmmu_hblk_hash_rm()
15582 *listp = hmeblkp; in sfmmu_hblk_hash_rm()
15594 struct hme_blk *hmeblkp = NULL, *last_hmeblkp; in sfmmu_check_pending_hblks() local
15610 for (hmeblkp = cpuhp->chp_listp; hmeblkp != NULL; in sfmmu_check_pending_hblks()
15611 hmeblkp = hmeblkp->hblk_next) { in sfmmu_check_pending_hblks()
15612 if (get_hblk_ttesz(hmeblkp) == size) { in sfmmu_check_pending_hblks()
15615 hmeblkp->hblk_next; in sfmmu_check_pending_hblks()
15618 hmeblkp->hblk_next; in sfmmu_check_pending_hblks()
15625 last_hmeblkp = hmeblkp; in sfmmu_check_pending_hblks()
15636 return (hmeblkp); in sfmmu_check_pending_hblks()