Lines Matching refs:hmeblkp
114 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid) \ argument
120 int _ttesz = get_hblk_ttesz(hmeblkp); \
129 _hsva = (caddr_t)get_hblk_base(hmeblkp); \
130 _heva = get_hblk_endaddr(hmeblkp); \
147 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid) argument
853 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, ismhat) \ argument
855 int ttesz = get_hblk_ttesz(hmeblkp); \
860 (caddr_t)get_hblk_base(hmeblkp); \
861 caddr_t eva = sva + get_hblk_span(hmeblkp); \
992 #define HBLKTOHME(hment, hmeblkp, addr) \ argument
995 HBLKTOHME_IDX(hment, hmeblkp, addr, index) \
1002 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx) \ argument
1004 ASSERT(in_hblk_range((hmeblkp), (addr))); \
1006 if (get_hblk_ttesz(hmeblkp) == TTE8K) { \
1011 (hment) = &(hmeblkp)->hblk_hme[idx]; \
1993 struct hme_blk *hmeblkp; in hat_swapout() local
2025 hmeblkp = hmebp->hmeblkp; in hat_swapout()
2027 while (hmeblkp) { in hat_swapout()
2029 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && in hat_swapout()
2030 !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) { in hat_swapout()
2031 ASSERT(!hmeblkp->hblk_shared); in hat_swapout()
2032 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in hat_swapout()
2033 (caddr_t)get_hblk_base(hmeblkp), in hat_swapout()
2034 get_hblk_endaddr(hmeblkp), in hat_swapout()
2037 nx_hblk = hmeblkp->hblk_next; in hat_swapout()
2038 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in hat_swapout()
2039 ASSERT(!hmeblkp->hblk_lckcnt); in hat_swapout()
2040 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in hat_swapout()
2043 pr_hblk = hmeblkp; in hat_swapout()
2045 hmeblkp = nx_hblk; in hat_swapout()
2564 struct hme_blk *hmeblkp; in sfmmu_memload_batchsmall() local
2578 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr, in sfmmu_memload_batchsmall()
2580 ASSERT(hmeblkp); in sfmmu_memload_batchsmall()
2592 (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte, in sfmmu_memload_batchsmall()
2881 struct hme_blk *hmeblkp; in sfmmu_tteload_array() local
2900 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags, in sfmmu_tteload_array()
2902 ASSERT(hmeblkp); in sfmmu_tteload_array()
2907 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags, in sfmmu_tteload_array()
2951 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; in sfmmu_tteload_find_hmeblk() local
2964 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in sfmmu_tteload_find_hmeblk()
2971 if (hmeblkp == (struct hme_blk *)hblk_reserve && in sfmmu_tteload_find_hmeblk()
2981 if (hmeblkp == NULL) { in sfmmu_tteload_find_hmeblk()
2982 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, in sfmmu_tteload_find_hmeblk()
2984 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
2985 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
2993 if (get_hblk_ttesz(hmeblkp) != size) { in sfmmu_tteload_find_hmeblk()
2994 ASSERT(!hmeblkp->hblk_vcnt); in sfmmu_tteload_find_hmeblk()
2995 ASSERT(!hmeblkp->hblk_hmecnt); in sfmmu_tteload_find_hmeblk()
2996 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_tteload_find_hmeblk()
3000 if (hmeblkp->hblk_shw_bit) { in sfmmu_tteload_find_hmeblk()
3005 ASSERT(!hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
3006 if (hmeblkp->hblk_shw_mask) { in sfmmu_tteload_find_hmeblk()
3007 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); in sfmmu_tteload_find_hmeblk()
3011 hmeblkp->hblk_shw_bit = 0; in sfmmu_tteload_find_hmeblk()
3025 ASSERT(get_hblk_ttesz(hmeblkp) == size); in sfmmu_tteload_find_hmeblk()
3026 ASSERT(!hmeblkp->hblk_shw_bit); in sfmmu_tteload_find_hmeblk()
3027 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
3028 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared); in sfmmu_tteload_find_hmeblk()
3029 ASSERT(hmeblkp->hblk_tag.htag_rid == rid); in sfmmu_tteload_find_hmeblk()
3031 return (hmeblkp); in sfmmu_tteload_find_hmeblk()
3039 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, in sfmmu_tteload_addentry() argument
3112 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared); in sfmmu_tteload_addentry()
3113 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared); in sfmmu_tteload_addentry()
3115 HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum); in sfmmu_tteload_addentry()
3148 (void *)hmeblkp); in sfmmu_tteload_addentry()
3195 if ((hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) { in sfmmu_tteload_addentry()
3197 (void *)hmeblkp); in sfmmu_tteload_addentry()
3199 atomic_inc_32(&hmeblkp->hblk_lckcnt); in sfmmu_tteload_addentry()
3201 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK); in sfmmu_tteload_addentry()
3227 chk_tte(&orig_old, &tteold, ttep, hmeblkp); in sfmmu_tteload_addentry()
3234 atomic_inc_16(&hmeblkp->hblk_vcnt); in sfmmu_tteload_addentry()
3317 if (hmeblkp->hblk_shared) { in sfmmu_tteload_addentry()
3319 sfmmup->sfmmu_srdp->srd_hmergnp[rid], hmeblkp, 1); in sfmmu_tteload_addentry()
3323 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); in sfmmu_tteload_addentry()
3353 atomic_inc_16(&hmeblkp->hblk_hmecnt); in sfmmu_tteload_addentry()
3354 ASSERT(hmeblkp->hblk_hmecnt > 0); in sfmmu_tteload_addentry()
3595 struct hme_blk *hmeblkp; in sfmmu_shadow_hcreate() local
3621 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in sfmmu_shadow_hcreate()
3622 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); in sfmmu_shadow_hcreate()
3623 if (hmeblkp == NULL) { in sfmmu_shadow_hcreate()
3624 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, in sfmmu_shadow_hcreate()
3627 ASSERT(hmeblkp); in sfmmu_shadow_hcreate()
3628 if (!hmeblkp->hblk_shw_mask) { in sfmmu_shadow_hcreate()
3634 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); in sfmmu_shadow_hcreate()
3635 hmeblkp->hblk_shw_bit = 1; in sfmmu_shadow_hcreate()
3636 } else if (hmeblkp->hblk_shw_bit == 0) { in sfmmu_shadow_hcreate()
3638 (void *)hmeblkp); in sfmmu_shadow_hcreate()
3640 ASSERT(hmeblkp->hblk_shw_bit == 1); in sfmmu_shadow_hcreate()
3641 ASSERT(!hmeblkp->hblk_shared); in sfmmu_shadow_hcreate()
3648 shw_mask = hmeblkp->hblk_shw_mask; in sfmmu_shadow_hcreate()
3650 newshw_mask = atomic_cas_32(&hmeblkp->hblk_shw_mask, shw_mask, in sfmmu_shadow_hcreate()
3656 return (hmeblkp); in sfmmu_shadow_hcreate()
3669 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_shadow_hcleanup() argument
3675 ASSERT(hmeblkp->hblk_shw_bit); in sfmmu_shadow_hcleanup()
3676 ASSERT(!hmeblkp->hblk_shared); in sfmmu_shadow_hcleanup()
3680 if (!hmeblkp->hblk_shw_mask) { in sfmmu_shadow_hcleanup()
3681 hmeblkp->hblk_shw_bit = 0; in sfmmu_shadow_hcleanup()
3684 addr = (caddr_t)get_hblk_base(hmeblkp); in sfmmu_shadow_hcleanup()
3685 endaddr = get_hblk_endaddr(hmeblkp); in sfmmu_shadow_hcleanup()
3686 size = get_hblk_ttesz(hmeblkp); in sfmmu_shadow_hcleanup()
3703 struct hme_blk *hmeblkp; in sfmmu_free_hblks() local
3718 hmeblkp = hmebp->hmeblkp; in sfmmu_free_hblks()
3720 while (hmeblkp) { in sfmmu_free_hblks()
3721 if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) { in sfmmu_free_hblks()
3723 ASSERT(!hmeblkp->hblk_shared); in sfmmu_free_hblks()
3724 if (hmeblkp->hblk_shw_bit) { in sfmmu_free_hblks()
3725 if (hmeblkp->hblk_shw_mask) { in sfmmu_free_hblks()
3728 hmeblkp, hmebp); in sfmmu_free_hblks()
3731 hmeblkp->hblk_shw_bit = 0; in sfmmu_free_hblks()
3744 nx_hblk = hmeblkp->hblk_next; in sfmmu_free_hblks()
3745 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in sfmmu_free_hblks()
3746 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_free_hblks()
3749 pr_hblk = hmeblkp; in sfmmu_free_hblks()
3751 hmeblkp = nx_hblk; in sfmmu_free_hblks()
3782 struct hme_blk *hmeblkp; in sfmmu_cleanup_rhblk() local
3797 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in sfmmu_cleanup_rhblk()
3798 if (hmeblkp != NULL) { in sfmmu_cleanup_rhblk()
3799 ASSERT(hmeblkp->hblk_shared); in sfmmu_cleanup_rhblk()
3800 ASSERT(!hmeblkp->hblk_shw_bit); in sfmmu_cleanup_rhblk()
3801 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_cleanup_rhblk()
3804 ASSERT(!hmeblkp->hblk_lckcnt); in sfmmu_cleanup_rhblk()
3805 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_cleanup_rhblk()
3830 struct hme_blk *hmeblkp; in sfmmu_unload_hmeregion_va() local
3846 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in sfmmu_unload_hmeregion_va()
3847 if (hmeblkp != NULL) { in sfmmu_unload_hmeregion_va()
3848 ASSERT(hmeblkp->hblk_shared); in sfmmu_unload_hmeregion_va()
3849 ASSERT(!hmeblkp->hblk_lckcnt); in sfmmu_unload_hmeregion_va()
3850 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_unload_hmeregion_va()
3851 *eaddrp = sfmmu_hblk_unload(NULL, hmeblkp, addr, in sfmmu_unload_hmeregion_va()
3855 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt); in sfmmu_unload_hmeregion_va()
3856 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in sfmmu_unload_hmeregion_va()
3939 struct hme_blk *hmeblkp, *list = NULL; in hat_unlock() local
3965 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in hat_unlock()
3966 if (hmeblkp != NULL) { in hat_unlock()
3967 ASSERT(!hmeblkp->hblk_shared); in hat_unlock()
3975 if (hmeblkp->hblk_shw_bit) { in hat_unlock()
3978 addr = sfmmu_hblk_unlock(hmeblkp, addr, in hat_unlock()
4016 struct hme_blk *hmeblkp; in hat_unlock_region() local
4061 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, in hat_unlock_region()
4063 if (hmeblkp == NULL) { in hat_unlock_region()
4068 ASSERT(hmeblkp->hblk_shared); in hat_unlock_region()
4069 va = sfmmu_hblk_unlock(hmeblkp, va, eaddr); in hat_unlock_region()
4089 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr) in sfmmu_hblk_unlock() argument
4095 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_unlock()
4096 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_unlock()
4098 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_unlock()
4099 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_unlock()
4101 HBLKTOHME(sfhme, hmeblkp, addr); in sfmmu_hblk_unlock()
4115 if (hmeblkp->hblk_lckcnt == 0) in sfmmu_hblk_unlock()
4122 ASSERT(hmeblkp->hblk_lckcnt > 0); in sfmmu_hblk_unlock()
4123 atomic_dec_32(&hmeblkp->hblk_lckcnt); in sfmmu_hblk_unlock()
4124 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); in sfmmu_hblk_unlock()
4292 struct hme_blk *hmeblkp; in hat_add_callback() local
4336 for (hashno = TTE64K, hmeblkp = NULL; in hat_add_callback()
4337 hmeblkp == NULL && hashno <= mmu_hashcnt; in hat_add_callback()
4348 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in hat_add_callback()
4350 if (hmeblkp == NULL) in hat_add_callback()
4354 if (hmeblkp == NULL) { in hat_add_callback()
4360 ASSERT(!hmeblkp->hblk_shared); in hat_add_callback()
4362 HBLKTOHME(osfhmep, hmeblkp, saddr); in hat_add_callback()
4376 baseaddr = (caddr_t)get_hblk_base(hmeblkp); in hat_add_callback()
4522 struct hme_blk *hmeblkp; in hat_delete_callback() local
4545 for (hashno = TTE64K, hmeblkp = NULL; in hat_delete_callback()
4546 hmeblkp == NULL && hashno <= mmu_hashcnt; in hat_delete_callback()
4557 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in hat_delete_callback()
4559 if (hmeblkp == NULL) in hat_delete_callback()
4563 if (hmeblkp == NULL) in hat_delete_callback()
4566 ASSERT(!hmeblkp->hblk_shared); in hat_delete_callback()
4568 HBLKTOHME(osfhmep, hmeblkp, saddr); in hat_delete_callback()
4843 struct hme_blk *hmeblkp, *list = NULL; in sfmmu_chgattr() local
4873 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in sfmmu_chgattr()
4874 if (hmeblkp != NULL) { in sfmmu_chgattr()
4875 ASSERT(!hmeblkp->hblk_shared); in sfmmu_chgattr()
4880 if (hmeblkp->hblk_shw_bit) { in sfmmu_chgattr()
4887 hmeblkp, addr, endaddr, &dmr, attr, mode); in sfmmu_chgattr()
4933 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_chgattr() argument
4947 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_chgattr()
4948 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_chgattr()
4949 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_chgattr()
4951 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_chgattr()
4952 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_chgattr()
4971 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_chgattr()
5044 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_hblk_chgattr()
5172 struct hme_blk *hmeblkp, *list = NULL; in hat_chgprot() local
5202 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in hat_chgprot()
5203 if (hmeblkp != NULL) { in hat_chgprot()
5204 ASSERT(!hmeblkp->hblk_shared); in hat_chgprot()
5209 if (hmeblkp->hblk_shw_bit) { in hat_chgprot()
5215 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, in hat_chgprot()
5261 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_chgprot() argument
5277 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_chgprot()
5278 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_chgprot()
5279 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_chgprot()
5282 if (get_hblk_ttesz(hmeblkp) != TTE8K && in sfmmu_hblk_chgprot()
5283 (endaddr < get_hblk_endaddr(hmeblkp))) { in sfmmu_hblk_chgprot()
5288 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_chgprot()
5289 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_chgprot()
5297 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_chgprot()
5372 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_hblk_chgprot()
5456 struct hme_blk *hmeblkp; in hat_unload_large_virtual() local
5484 hmeblkp = hmebp->hmeblkp; in hat_unload_large_virtual()
5486 while (hmeblkp) { in hat_unload_large_virtual()
5487 nx_hblk = hmeblkp->hblk_next; in hat_unload_large_virtual()
5493 if (hmeblkp->hblk_tag.htag_id != sfmmup || in hat_unload_large_virtual()
5494 hmeblkp->hblk_shw_bit || in hat_unload_large_virtual()
5495 (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr || in hat_unload_large_virtual()
5496 (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) { in hat_unload_large_virtual()
5497 pr_hblk = hmeblkp; in hat_unload_large_virtual()
5501 ASSERT(!hmeblkp->hblk_shared); in hat_unload_large_virtual()
5505 if (hmeblkp->hblk_vcnt != 0 || in hat_unload_large_virtual()
5506 hmeblkp->hblk_hmecnt != 0) in hat_unload_large_virtual()
5507 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in hat_unload_large_virtual()
5515 !hmeblkp->hblk_vcnt && in hat_unload_large_virtual()
5516 !hmeblkp->hblk_hmecnt) { in hat_unload_large_virtual()
5517 ASSERT(!hmeblkp->hblk_lckcnt); in hat_unload_large_virtual()
5518 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in hat_unload_large_virtual()
5521 pr_hblk = hmeblkp; in hat_unload_large_virtual()
5555 hmeblkp = nx_hblk; in hat_unload_large_virtual()
5597 struct hme_blk *hmeblkp, *pr_hblk, *list = NULL; in hat_unload_callback() local
5682 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list); in hat_unload_callback()
5683 if (hmeblkp == NULL) { in hat_unload_callback()
5726 ASSERT(hmeblkp); in hat_unload_callback()
5727 ASSERT(!hmeblkp->hblk_shared); in hat_unload_callback()
5728 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in hat_unload_callback()
5738 get_hblk_span(hmeblkp)); in hat_unload_callback()
5741 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, in hat_unload_callback()
5774 if (hmeblkp->hblk_shw_bit) { in hat_unload_callback()
5798 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, in hat_unload_callback()
5805 !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) { in hat_unload_callback()
5806 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 0); in hat_unload_callback()
5914 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_unload() argument
5926 ASSERT(in_hblk_range(hmeblkp, addr)); in sfmmu_hblk_unload()
5927 ASSERT(!hmeblkp->hblk_shw_bit); in sfmmu_hblk_unload()
5928 ASSERT(sfmmup != NULL || hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5929 ASSERT(sfmmup == NULL || !hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5930 ASSERT(dmrp == NULL || !hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5933 if (get_hblk_ttesz(hmeblkp) != TTE8K && in sfmmu_hblk_unload()
5934 (endaddr < get_hblk_endaddr(hmeblkp))) { in sfmmu_hblk_unload()
5939 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_unload()
5940 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_unload()
5951 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_unload()
6024 ASSERT(hmeblkp->hblk_lckcnt > 0); in sfmmu_hblk_unload()
6025 atomic_dec_32(&hmeblkp->hblk_lckcnt); in sfmmu_hblk_unload()
6026 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK); in sfmmu_hblk_unload()
6062 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_unload()
6063 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, in sfmmu_hblk_unload()
6071 ASSERT(hmeblkp->hblk_hmecnt > 0); in sfmmu_hblk_unload()
6079 atomic_dec_16(&hmeblkp->hblk_hmecnt); in sfmmu_hblk_unload()
6082 ASSERT(hmeblkp->hblk_vcnt > 0); in sfmmu_hblk_unload()
6083 atomic_dec_16(&hmeblkp->hblk_vcnt); in sfmmu_hblk_unload()
6085 ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt || in sfmmu_hblk_unload()
6086 !hmeblkp->hblk_lckcnt); in sfmmu_hblk_unload()
6124 } else if (hmeblkp->hblk_hmecnt != 0) { in sfmmu_hblk_unload()
6219 struct hme_blk *hmeblkp, *list = NULL; in hat_sync() local
6250 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list); in hat_sync()
6251 if (hmeblkp != NULL) { in hat_sync()
6252 ASSERT(!hmeblkp->hblk_shared); in hat_sync()
6257 if (hmeblkp->hblk_shw_bit) { in hat_sync()
6263 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, in hat_sync()
6298 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_sync() argument
6308 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_sync()
6309 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_sync()
6311 endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp)); in sfmmu_hblk_sync()
6313 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_sync()
6314 HBLKTOHME(sfhmep, hmeblkp, addr); in sfmmu_hblk_sync()
6350 hmeblkp, 0, 0); in sfmmu_hblk_sync()
6610 struct hme_blk *hmeblkp; in hat_pagesuspend() local
6657 hmeblkp = sfmmu_hmetohblk(sfhmep); in hat_pagesuspend()
6659 sfmmup = hblktosfmmu(hmeblkp); in hat_pagesuspend()
6661 ASSERT(!hmeblkp->hblk_shared); in hat_pagesuspend()
6663 addr = tte_to_vaddr(hmeblkp, tte); in hat_pagesuspend()
6670 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in hat_pagesuspend()
7021 struct hme_blk *hmeblkp; in hat_pageunload() local
7063 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_pageunload()
7069 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt && in hat_pageunload()
7070 hmeblkp->hblk_tag.htag_id == ksfmmup) in hat_pageunload()
7146 struct hme_blk *hmeblkp; in sfmmu_pageunload() local
7163 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_pageunload()
7168 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pageunload()
7169 ttesz = get_hblk_ttesz(hmeblkp); in sfmmu_pageunload()
7192 chk_tte(&orig_old, &tte, &ttemod, hmeblkp); in sfmmu_pageunload()
7201 addr = tte_to_vaddr(hmeblkp, tte); in sfmmu_pageunload()
7203 if (hmeblkp->hblk_shared) { in sfmmu_pageunload()
7205 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_pageunload()
7211 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_pageunload()
7212 cpuset = sfmmu_rgntlb_demap(addr, rgnp, hmeblkp, 1); in sfmmu_pageunload()
7247 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, in sfmmu_pageunload()
7254 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_pageunload()
7277 ASSERT(hmeblkp->hblk_hmecnt > 0); in sfmmu_pageunload()
7278 ASSERT(hmeblkp->hblk_vcnt > 0); in sfmmu_pageunload()
7279 atomic_dec_16(&hmeblkp->hblk_vcnt); in sfmmu_pageunload()
7280 atomic_dec_16(&hmeblkp->hblk_hmecnt); in sfmmu_pageunload()
7369 struct hme_blk *hmeblkp; in hat_pagesync() local
7419 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_pagesync()
7425 if (hmeblkp->hblk_shared) { in hat_pagesync()
7426 sf_srd_t *srdp = hblktosrd(hmeblkp); in hat_pagesync()
7427 uint_t rid = hmeblkp->hblk_tag.htag_rid; in hat_pagesync()
7433 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, in hat_pagesync()
7489 struct hme_blk *hmeblkp; in sfmmu_pagesync() local
7507 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_pagesync()
7508 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pagesync()
7509 addr = tte_to_vaddr(hmeblkp, tte); in sfmmu_pagesync()
7525 if (hmeblkp->hblk_shared) { in sfmmu_pagesync()
7528 hmeblkp->hblk_tag.htag_rid; in sfmmu_pagesync()
7534 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_pagesync()
7537 rgnp, hmeblkp, 1); in sfmmu_pagesync()
7539 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, in sfmmu_pagesync()
7545 sfmmu_ttesync(hmeblkp->hblk_shared ? NULL : sfmmup, addr, in sfmmu_pagesync()
7563 struct hme_blk *hmeblkp; in sfmmu_pageclrwrt() local
7578 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_pageclrwrt()
7579 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pageclrwrt()
7580 addr = tte_to_vaddr(hmeblkp, tte); in sfmmu_pageclrwrt()
7596 if (hmeblkp->hblk_shared) { in sfmmu_pageclrwrt()
7598 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_pageclrwrt()
7604 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_pageclrwrt()
7607 rgnp, hmeblkp, 1); in sfmmu_pageclrwrt()
7609 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_pageclrwrt()
7874 struct hme_blk *hmeblkp = NULL; in sfmmu_uvatopfn() local
7933 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp); in sfmmu_uvatopfn()
7934 if (hmeblkp != NULL) { in sfmmu_uvatopfn()
7935 ASSERT(!hmeblkp->hblk_shared); in sfmmu_uvatopfn()
7936 HBLKTOHME(sfhmep, hmeblkp, vaddr); in sfmmu_uvatopfn()
7964 for (hmeblkp = hmebp->hmeblkp; hmeblkp != NULL; in sfmmu_uvatopfn()
7965 hmeblkp = hmeblkp->hblk_next) { in sfmmu_uvatopfn()
7971 if (!HTAGS_EQ_SHME(hmeblkp->hblk_tag, hblktag, in sfmmu_uvatopfn()
7975 ASSERT(hmeblkp->hblk_shared); in sfmmu_uvatopfn()
7976 rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_uvatopfn()
7980 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_uvatopfn()
7981 HBLKTOHME(sfhmep, hmeblkp, sv_vaddr); in sfmmu_uvatopfn()
7987 get_hblk_ttesz(hmeblkp) > TTE8K) { in sfmmu_uvatopfn()
7988 caddr_t eva = tte_to_evaddr(hmeblkp, ttep); in sfmmu_uvatopfn()
8004 } else if (get_hblk_ttesz(hmeblkp) > TTE8K || in sfmmu_uvatopfn()
8090 struct hme_blk *hmeblkp; in hat_page_checkshare() local
8117 hmeblkp = sfmmu_hmetohblk(sfhme); in hat_page_checkshare()
8122 if (hmeblkp->hblk_shared) { in hat_page_checkshare()
8123 sf_srd_t *srdp = hblktosrd(hmeblkp); in hat_page_checkshare()
8124 uint_t rid = hmeblkp->hblk_tag.htag_rid; in hat_page_checkshare()
8130 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, in hat_page_checkshare()
8885 struct hme_blk *hmeblkp; in sfmmu_hblkcache_constructor() local
8888 hmeblkp = (struct hme_blk *)buf; in sfmmu_hblkcache_constructor()
8889 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); in sfmmu_hblkcache_constructor()
8892 mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL); in sfmmu_hblkcache_constructor()
8905 struct hme_blk *hmeblkp; in sfmmu_hblkcache_destructor() local
8907 hmeblkp = (struct hme_blk *)buf; in sfmmu_hblkcache_destructor()
8908 mutex_destroy(&hmeblkp->hblk_audit_lock); in sfmmu_hblkcache_destructor()
8927 struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL; in sfmmu_hblkcache_reclaim() local
8973 hmeblkp = hmebp->hmeblkp; in sfmmu_hblkcache_reclaim()
8975 while (hmeblkp) { in sfmmu_hblkcache_reclaim()
8976 nx_hblk = hmeblkp->hblk_next; in sfmmu_hblkcache_reclaim()
8977 if (!hmeblkp->hblk_vcnt && in sfmmu_hblkcache_reclaim()
8978 !hmeblkp->hblk_hmecnt) { in sfmmu_hblkcache_reclaim()
8979 sfmmu_hblk_hash_rm(hmebp, hmeblkp, in sfmmu_hblkcache_reclaim()
8982 pr_hblk = hmeblkp; in sfmmu_hblkcache_reclaim()
8984 hmeblkp = nx_hblk; in sfmmu_hblkcache_reclaim()
8999 hmeblkp = hmebp->hmeblkp; in sfmmu_hblkcache_reclaim()
9001 while (hmeblkp) { in sfmmu_hblkcache_reclaim()
9002 nx_hblk = hmeblkp->hblk_next; in sfmmu_hblkcache_reclaim()
9003 if (!hmeblkp->hblk_vcnt && in sfmmu_hblkcache_reclaim()
9004 !hmeblkp->hblk_hmecnt) { in sfmmu_hblkcache_reclaim()
9005 sfmmu_hblk_hash_rm(hmebp, hmeblkp, in sfmmu_hblkcache_reclaim()
9008 pr_hblk = hmeblkp; in sfmmu_hblkcache_reclaim()
9010 hmeblkp = nx_hblk; in sfmmu_hblkcache_reclaim()
9080 struct hme_blk *hmeblkp; in sfmmu_vac_conflict() local
9152 hmeblkp = sfmmu_hmetohblk(sfhmep); in sfmmu_vac_conflict()
9153 tmphat = hblktosfmmu(hmeblkp); in sfmmu_vac_conflict()
9156 if (hmeblkp->hblk_shared || tmphat == hat || in sfmmu_vac_conflict()
9157 hmeblkp->hblk_lckcnt) { in sfmmu_vac_conflict()
9178 hmeblkp = sfmmu_hmetohblk(sfhmep); in sfmmu_vac_conflict()
9179 ASSERT(!hmeblkp->hblk_shared); in sfmmu_vac_conflict()
9288 struct hme_blk *hmeblkp; in tst_tnc() local
9325 hmeblkp = sfmmu_hmetohblk(sfhme); in tst_tnc()
9330 vaddr = tte_to_vaddr(hmeblkp, tte); in tst_tnc()
9457 struct hme_blk *hmeblkp; in sfmmu_page_cache() local
9471 hmeblkp = sfmmu_hmetohblk(sfhme); in sfmmu_page_cache()
9475 vaddr = tte_to_vaddr(hmeblkp, tte); in sfmmu_page_cache()
9501 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_page_cache()
9506 if (hmeblkp->hblk_shared) { in sfmmu_page_cache()
9508 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_page_cache()
9514 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_page_cache()
9517 hmeblkp, 0); in sfmmu_page_cache()
9525 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9528 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9541 if (hmeblkp->hblk_shared) { in sfmmu_page_cache()
9543 uint_t rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_page_cache()
9549 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, in sfmmu_page_cache()
9552 hmeblkp, 0); in sfmmu_page_cache()
9559 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9562 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); in sfmmu_page_cache()
10568 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical) in sfmmu_put_free_hblk() argument
10572 ASSERT(hmeblkp->hblk_hmecnt == 0); in sfmmu_put_free_hblk()
10573 ASSERT(hmeblkp->hblk_vcnt == 0); in sfmmu_put_free_hblk()
10574 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); in sfmmu_put_free_hblk()
10592 hmeblkp->hblk_next = freehblkp; in sfmmu_put_free_hblk()
10593 freehblkp = hmeblkp; in sfmmu_put_free_hblk()
10674 hblkp = hmebp->hmeblkp; hblkp != NULL && hblkp != old; in sfmmu_hblk_swap()
10927 struct hme_blk *hmeblkp = NULL; in sfmmu_hblk_alloc() local
10965 hmeblkp = in sfmmu_hblk_alloc()
10980 hmeblkp = in sfmmu_hblk_alloc()
11055 if (!sfmmu_get_free_hblk(&hmeblkp, 1)) in sfmmu_hblk_alloc()
11068 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) { in sfmmu_hblk_alloc()
11069 hmeblkp = sfmmu_hblk_steal(size); in sfmmu_hblk_alloc()
11079 sfmmu_hblk_swap(hmeblkp); in sfmmu_hblk_alloc()
11089 if (sfmmu_put_free_hblk(hmeblkp, 0)) in sfmmu_hblk_alloc()
11100 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) { in sfmmu_hblk_alloc()
11107 hmeblkp = HBLK_RESERVE; in sfmmu_hblk_alloc()
11113 ASSERT(hmeblkp != NULL); in sfmmu_hblk_alloc()
11114 set_hblk_sz(hmeblkp, size); in sfmmu_hblk_alloc()
11115 ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp)); in sfmmu_hblk_alloc()
11120 if (hmeblkp != HBLK_RESERVE) { in sfmmu_hblk_alloc()
11155 sfmmu_put_free_hblk(hmeblkp, forcefree)) { in sfmmu_hblk_alloc()
11159 kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp); in sfmmu_hblk_alloc()
11219 hmeblkp->hblk_shared = 1; in sfmmu_hblk_alloc()
11221 hmeblkp->hblk_shared = 0; in sfmmu_hblk_alloc()
11223 set_hblk_sz(hmeblkp, size); in sfmmu_hblk_alloc()
11225 hmeblkp->hblk_next = (struct hme_blk *)NULL; in sfmmu_hblk_alloc()
11226 hmeblkp->hblk_tag = hblktag; in sfmmu_hblk_alloc()
11227 hmeblkp->hblk_shadow = shw_hblkp; in sfmmu_hblk_alloc()
11228 hblkpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_alloc()
11229 hmeblkp->hblk_nextpa = HMEBLK_ENDPA; in sfmmu_hblk_alloc()
11231 ASSERT(get_hblk_ttesz(hmeblkp) == size); in sfmmu_hblk_alloc()
11232 ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size)); in sfmmu_hblk_alloc()
11233 ASSERT(hmeblkp->hblk_hmecnt == 0); in sfmmu_hblk_alloc()
11234 ASSERT(hmeblkp->hblk_vcnt == 0); in sfmmu_hblk_alloc()
11235 ASSERT(hmeblkp->hblk_lckcnt == 0); in sfmmu_hblk_alloc()
11236 ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp)); in sfmmu_hblk_alloc()
11237 sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa); in sfmmu_hblk_alloc()
11238 return (hmeblkp); in sfmmu_hblk_alloc()
11248 struct hme_blk *hmeblkp, *next_hmeblkp; in sfmmu_hblk_free() local
11255 hmeblkp = *listp; in sfmmu_hblk_free()
11256 while (hmeblkp != NULL) { in sfmmu_hblk_free()
11257 next_hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_free()
11258 ASSERT(!hmeblkp->hblk_hmecnt); in sfmmu_hblk_free()
11259 ASSERT(!hmeblkp->hblk_vcnt); in sfmmu_hblk_free()
11260 ASSERT(!hmeblkp->hblk_lckcnt); in sfmmu_hblk_free()
11261 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve); in sfmmu_hblk_free()
11262 ASSERT(hmeblkp->hblk_shared == 0); in sfmmu_hblk_free()
11263 ASSERT(hmeblkp->hblk_shw_bit == 0); in sfmmu_hblk_free()
11264 ASSERT(hmeblkp->hblk_shadow == NULL); in sfmmu_hblk_free()
11266 hblkpa = va_to_pa((caddr_t)hmeblkp); in sfmmu_hblk_free()
11268 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0; in sfmmu_hblk_free()
11270 size = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_free()
11271 hmeblkp->hblk_next = NULL; in sfmmu_hblk_free()
11272 hmeblkp->hblk_nextpa = hblkpa; in sfmmu_hblk_free()
11274 if (hmeblkp->hblk_nuc_bit == 0) { in sfmmu_hblk_free()
11277 !sfmmu_put_free_hblk(hmeblkp, critical)) in sfmmu_hblk_free()
11278 kmem_cache_free(get_hblk_cache(hmeblkp), in sfmmu_hblk_free()
11279 hmeblkp); in sfmmu_hblk_free()
11281 hmeblkp = next_hmeblkp; in sfmmu_hblk_free()
11303 struct hme_blk *hmeblkp = NULL, *pr_hblk; in sfmmu_hblk_steal() local
11310 if ((hmeblkp = sfmmu_check_pending_hblks(size)) != NULL) { in sfmmu_hblk_steal()
11311 hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp); in sfmmu_hblk_steal()
11312 ASSERT(hmeblkp->hblk_hmecnt == 0); in sfmmu_hblk_steal()
11313 ASSERT(hmeblkp->hblk_vcnt == 0); in sfmmu_hblk_steal()
11314 return (hmeblkp); in sfmmu_hblk_steal()
11320 if (sfmmu_get_free_hblk(&hmeblkp, critical)) in sfmmu_hblk_steal()
11321 return (hmeblkp); in sfmmu_hblk_steal()
11328 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ + in sfmmu_hblk_steal()
11331 hmeblkp = hmebp->hmeblkp; in sfmmu_hblk_steal()
11334 while (hmeblkp) { in sfmmu_hblk_steal()
11340 if ((get_hblk_ttesz(hmeblkp) == size) && in sfmmu_hblk_steal()
11341 (hmeblkp->hblk_shw_bit == 0 || in sfmmu_hblk_steal()
11342 hmeblkp->hblk_vcnt == 0) && in sfmmu_hblk_steal()
11343 (hmeblkp->hblk_lckcnt == 0)) { in sfmmu_hblk_steal()
11350 if ((hmeblkp->hblk_vcnt == 0 && in sfmmu_hblk_steal()
11351 hmeblkp->hblk_hmecnt == 0) || (i >= in sfmmu_hblk_steal()
11354 hmeblkp, hblkpa, pr_hblk)) { in sfmmu_hblk_steal()
11363 pr_hblk = hmeblkp; in sfmmu_hblk_steal()
11364 hblkpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_steal()
11365 hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_steal()
11374 if (hmeblkp != NULL) in sfmmu_hblk_steal()
11383 hmeblkp = hmebp->hmeblkp; in sfmmu_hblk_steal()
11386 while (hmeblkp) { in sfmmu_hblk_steal()
11390 if ((get_hblk_ttesz(hmeblkp) == size) && in sfmmu_hblk_steal()
11391 (hmeblkp->hblk_lckcnt == 0) && in sfmmu_hblk_steal()
11392 (hmeblkp->hblk_vcnt == 0) && in sfmmu_hblk_steal()
11393 (hmeblkp->hblk_hmecnt == 0)) { in sfmmu_hblk_steal()
11395 hmeblkp, hblkpa, pr_hblk)) { in sfmmu_hblk_steal()
11406 pr_hblk = hmeblkp; in sfmmu_hblk_steal()
11407 hblkpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_steal()
11408 hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_steal()
11416 if (hmeblkp != NULL) in sfmmu_hblk_steal()
11420 return (hmeblkp); in sfmmu_hblk_steal()
11431 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, in sfmmu_steal_this_hblk() argument
11445 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_steal_this_hblk()
11449 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_steal_this_hblk()
11450 if (hmeblkp->hblk_shared || sfmmup->sfmmu_ismhat) { in sfmmu_steal_this_hblk()
11454 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in sfmmu_steal_this_hblk()
11455 (caddr_t)get_hblk_base(hmeblkp), in sfmmu_steal_this_hblk()
11456 get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD); in sfmmu_steal_this_hblk()
11458 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) { in sfmmu_steal_this_hblk()
11468 ASSERT(hmeblkp->hblk_lckcnt == 0); in sfmmu_steal_this_hblk()
11469 ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0); in sfmmu_steal_this_hblk()
11471 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 1); in sfmmu_steal_this_hblk()
11472 hmeblkp->hblk_nextpa = hblkpa; in sfmmu_steal_this_hblk()
11474 shw_hblkp = hmeblkp->hblk_shadow; in sfmmu_steal_this_hblk()
11476 ASSERT(!hmeblkp->hblk_shared); in sfmmu_steal_this_hblk()
11478 vaddr = (caddr_t)get_hblk_base(hmeblkp); in sfmmu_steal_this_hblk()
11491 hmeblkp->hblk_shadow = NULL; in sfmmu_steal_this_hblk()
11499 hmeblkp->hblk_shw_bit = 0; in sfmmu_steal_this_hblk()
11501 if (hmeblkp->hblk_shared) { in sfmmu_steal_this_hblk()
11506 srdp = hblktosrd(hmeblkp); in sfmmu_steal_this_hblk()
11508 rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_steal_this_hblk()
11513 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_steal_this_hblk()
11514 hmeblkp->hblk_shared = 0; in sfmmu_steal_this_hblk()
11526 struct hme_blk *hmeblkp; in sfmmu_hmetohblk() local
11536 hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 - in sfmmu_hmetohblk()
11539 return (hmeblkp); in sfmmu_hmetohblk()
11939 struct hme_blk *hmeblkp, int uselocks) in sfmmu_rgntlb_demap() argument
11949 ASSERT(hmeblkp->hblk_shared); in sfmmu_rgntlb_demap()
11982 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_rgntlb_demap()
12057 struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag) in sfmmu_ismtlbcache_demap() argument
12070 ASSERT(!hmeblkp->hblk_shared); in sfmmu_ismtlbcache_demap()
12117 SFMMU_UNLOAD_TSB(va, sfmmup, hmeblkp, 1); in sfmmu_ismtlbcache_demap()
12153 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_tlbcache_demap() argument
12163 ASSERT(!hmeblkp->hblk_shared); in sfmmu_tlbcache_demap()
12196 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_tlbcache_demap()
12242 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_tlb_demap() argument
12248 ASSERT(!hmeblkp->hblk_shared); in sfmmu_tlb_demap()
12261 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_tlb_demap()
13164 struct hme_blk *hmeblkp; in sfmmu_init_nucleus_hblks() local
13187 hmeblkp = (struct hme_blk *)addr; in sfmmu_init_nucleus_hblks()
13189 hmeblkp->hblk_nuc_bit = 1; in sfmmu_init_nucleus_hblks()
13190 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); in sfmmu_init_nucleus_hblks()
13199 hmeblkp = (struct hme_blk *)addr; in sfmmu_init_nucleus_hblks()
13201 hmeblkp->hblk_nuc_bit = 1; in sfmmu_init_nucleus_hblks()
13202 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp); in sfmmu_init_nucleus_hblks()
13398 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp) in chk_tte() argument
13408 hmeblkp = hmeblkp; in chk_tte()
15446 sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, in sfmmu_hblk_hash_add() argument
15451 if (hmebp->hmeblkp == NULL) { in sfmmu_hblk_hash_add()
15456 hmeblkp->hblk_nextpa = hmebp->hmeh_nextpa; in sfmmu_hblk_hash_add()
15465 hmeblkp->hblk_next = hmebp->hmeblkp; in sfmmu_hblk_hash_add()
15466 hmebp->hmeblkp = hmeblkp; in sfmmu_hblk_hash_add()
15502 sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp, in sfmmu_hblk_hash_rm() argument
15514 if (hmebp->hmeblkp == hmeblkp) { in sfmmu_hblk_hash_rm()
15515 hmebp->hmeh_nextpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_hash_rm()
15516 hmebp->hmeblkp = hmeblkp->hblk_next; in sfmmu_hblk_hash_rm()
15518 pr_hblk->hblk_nextpa = hmeblkp->hblk_nextpa; in sfmmu_hblk_hash_rm()
15519 pr_hblk->hblk_next = hmeblkp->hblk_next; in sfmmu_hblk_hash_rm()
15522 size = get_hblk_ttesz(hmeblkp); in sfmmu_hblk_hash_rm()
15523 shw_hblkp = hmeblkp->hblk_shadow; in sfmmu_hblk_hash_rm()
15525 ASSERT(hblktosfmmu(hmeblkp) != KHATID); in sfmmu_hblk_hash_rm()
15526 ASSERT(!hmeblkp->hblk_shared); in sfmmu_hblk_hash_rm()
15536 vaddr = (caddr_t)get_hblk_base(hmeblkp); in sfmmu_hblk_hash_rm()
15549 hmeblkp->hblk_shadow = NULL; in sfmmu_hblk_hash_rm()
15551 hmeblkp->hblk_shw_bit = 0; in sfmmu_hblk_hash_rm()
15553 if (hmeblkp->hblk_shared) { in sfmmu_hblk_hash_rm()
15559 srdp = hblktosrd(hmeblkp); in sfmmu_hblk_hash_rm()
15561 rid = hmeblkp->hblk_tag.htag_rid; in sfmmu_hblk_hash_rm()
15566 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid); in sfmmu_hblk_hash_rm()
15568 hmeblkp->hblk_shared = 0; in sfmmu_hblk_hash_rm()
15577 hmeblkp->hblk_nextpa = HMEBLK_ENDPA; in sfmmu_hblk_hash_rm()
15578 hmeblkp->hblk_next = NULL; in sfmmu_hblk_hash_rm()
15581 hmeblkp->hblk_next = *listp; in sfmmu_hblk_hash_rm()
15582 *listp = hmeblkp; in sfmmu_hblk_hash_rm()
15594 struct hme_blk *hmeblkp = NULL, *last_hmeblkp; in sfmmu_check_pending_hblks() local
15610 for (hmeblkp = cpuhp->chp_listp; hmeblkp != NULL; in sfmmu_check_pending_hblks()
15611 hmeblkp = hmeblkp->hblk_next) { in sfmmu_check_pending_hblks()
15612 if (get_hblk_ttesz(hmeblkp) == size) { in sfmmu_check_pending_hblks()
15615 hmeblkp->hblk_next; in sfmmu_check_pending_hblks()
15618 hmeblkp->hblk_next; in sfmmu_check_pending_hblks()
15625 last_hmeblkp = hmeblkp; in sfmmu_check_pending_hblks()
15636 return (hmeblkp); in sfmmu_check_pending_hblks()