Lines Matching refs:sfmmup
344 #define TSB_HASH(sfmmup) \ argument
346 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)])
348 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)] argument
476 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \ argument
477 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc)
478 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \ argument
479 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc)
838 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) { \ argument
839 (tsbinfop) = (sfmmup)->sfmmu_tsb; \
841 sfmmu_hat_lock_held(sfmmup)); \
853 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, ismhat) \ argument
857 sfmmu_unload_tsb(sfmmup, addr, ttesz); \
863 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \
1460 sfmmu_t *sfmmup; in hat_alloc() local
1466 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); in hat_alloc()
1467 sfmmup->sfmmu_as = as; in hat_alloc()
1468 sfmmup->sfmmu_flags = 0; in hat_alloc()
1469 sfmmup->sfmmu_tteflags = 0; in hat_alloc()
1470 sfmmup->sfmmu_rtteflags = 0; in hat_alloc()
1471 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); in hat_alloc()
1474 ksfmmup = sfmmup; in hat_alloc()
1475 sfmmup->sfmmu_cext = 0; in hat_alloc()
1478 sfmmup->sfmmu_clrstart = 0; in hat_alloc()
1479 sfmmup->sfmmu_tsb = NULL; in hat_alloc()
1493 sfmmup->sfmmu_cext = 0; in hat_alloc()
1497 sfmmup->sfmmu_clrstart = get_color_start(as); in hat_alloc()
1504 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size, in hat_alloc()
1505 TSB8K|TSB64K|TSB512K, 0, sfmmup); in hat_alloc()
1508 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, in hat_alloc()
1510 TSB8K|TSB64K|TSB512K, 0, sfmmup); in hat_alloc()
1511 sfmmup->sfmmu_flags = HAT_SWAPPED | HAT_ALLCTX_INVALID; in hat_alloc()
1512 ASSERT(sfmmup->sfmmu_tsb != NULL); in hat_alloc()
1517 sfmmup->sfmmu_ctxs[i].cnum = cnum; in hat_alloc()
1518 sfmmup->sfmmu_ctxs[i].gnum = 0; in hat_alloc()
1522 sfmmup->sfmmu_ttecnt[i] = 0; in hat_alloc()
1523 sfmmup->sfmmu_scdrttecnt[i] = 0; in hat_alloc()
1524 sfmmup->sfmmu_ismttecnt[i] = 0; in hat_alloc()
1525 sfmmup->sfmmu_scdismttecnt[i] = 0; in hat_alloc()
1526 sfmmup->sfmmu_pgsz[i] = TTE8K; in hat_alloc()
1528 sfmmup->sfmmu_tsb0_4minflcnt = 0; in hat_alloc()
1529 sfmmup->sfmmu_iblk = NULL; in hat_alloc()
1530 sfmmup->sfmmu_ismhat = 0; in hat_alloc()
1531 sfmmup->sfmmu_scdhat = 0; in hat_alloc()
1532 sfmmup->sfmmu_ismblkpa = (uint64_t)-1; in hat_alloc()
1533 if (sfmmup == ksfmmup) { in hat_alloc()
1534 CPUSET_ALL(sfmmup->sfmmu_cpusran); in hat_alloc()
1536 CPUSET_ZERO(sfmmup->sfmmu_cpusran); in hat_alloc()
1538 sfmmup->sfmmu_free = 0; in hat_alloc()
1539 sfmmup->sfmmu_rmstat = 0; in hat_alloc()
1540 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart; in hat_alloc()
1541 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL); in hat_alloc()
1542 sfmmup->sfmmu_srdp = NULL; in hat_alloc()
1543 SF_RGNMAP_ZERO(sfmmup->sfmmu_region_map); in hat_alloc()
1544 bzero(sfmmup->sfmmu_hmeregion_links, SFMMU_L1_HMERLINKS_SIZE); in hat_alloc()
1545 sfmmup->sfmmu_scdp = NULL; in hat_alloc()
1546 sfmmup->sfmmu_scd_link.next = NULL; in hat_alloc()
1547 sfmmup->sfmmu_scd_link.prev = NULL; in hat_alloc()
1548 return (sfmmup); in hat_alloc()
1866 hat_setup(struct hat *sfmmup, int allocflag) in hat_setup() argument
1877 hatlockp = sfmmu_hat_enter(sfmmup); in hat_setup()
1885 sfmmu_tsb_swapin(sfmmup, hatlockp); in hat_setup()
1887 sfmmu_get_ctx(sfmmup); in hat_setup()
1893 hatlockp = sfmmu_hat_enter(sfmmup); in hat_setup()
1896 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); in hat_setup()
1916 hat_free_start(struct hat *sfmmup) in hat_free_start() argument
1918 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as)); in hat_free_start()
1919 ASSERT(sfmmup != ksfmmup); in hat_free_start()
1921 sfmmup->sfmmu_free = 1; in hat_free_start()
1922 if (sfmmup->sfmmu_scdp != NULL) { in hat_free_start()
1923 sfmmu_leave_scd(sfmmup, 0); in hat_free_start()
1926 ASSERT(sfmmup->sfmmu_scdp == NULL); in hat_free_start()
1930 hat_free_end(struct hat *sfmmup) in hat_free_end() argument
1934 ASSERT(sfmmup->sfmmu_free == 1); in hat_free_end()
1935 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); in hat_free_end()
1936 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); in hat_free_end()
1937 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); in hat_free_end()
1938 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); in hat_free_end()
1939 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); in hat_free_end()
1940 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); in hat_free_end()
1942 if (sfmmup->sfmmu_rmstat) { in hat_free_end()
1943 hat_freestat(sfmmup->sfmmu_as, 0); in hat_free_end()
1946 while (sfmmup->sfmmu_tsb != NULL) { in hat_free_end()
1947 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; in hat_free_end()
1948 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); in hat_free_end()
1949 sfmmup->sfmmu_tsb = next; in hat_free_end()
1952 if (sfmmup->sfmmu_srdp != NULL) { in hat_free_end()
1953 sfmmu_leave_srd(sfmmup); in hat_free_end()
1954 ASSERT(sfmmup->sfmmu_srdp == NULL); in hat_free_end()
1956 if (sfmmup->sfmmu_hmeregion_links[i] != NULL) { in hat_free_end()
1957 kmem_free(sfmmup->sfmmu_hmeregion_links[i], in hat_free_end()
1959 sfmmup->sfmmu_hmeregion_links[i] = NULL; in hat_free_end()
1963 sfmmu_free_sfmmu(sfmmup); in hat_free_end()
1967 ASSERT(sfmmup->sfmmu_hmeregion_links[i] == NULL); in hat_free_end()
1971 kmem_cache_free(sfmmuid_cache, sfmmup); in hat_free_end()
1990 hat_swapout(struct hat *sfmmup) in hat_swapout() argument
2021 ASSERT(sfmmup != KHATID); in hat_swapout()
2029 if ((hmeblkp->hblk_tag.htag_id == sfmmup) && in hat_swapout()
2032 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in hat_swapout()
2055 hatlockp = sfmmu_hat_enter(sfmmup); in hat_swapout()
2057 sfmmu_invalidate_ctx(sfmmup); in hat_swapout()
2073 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { in hat_swapout()
2078 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED); in hat_swapout()
2080 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; in hat_swapout()
2674 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp, in sfmmu_tteload() argument
2677 ASSERT(sfmmup == ksfmmup); in sfmmu_tteload()
2678 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags, in sfmmu_tteload()
2687 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz) in sfmmu_mod_tsb() argument
2697 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */ in sfmmu_mod_tsb()
2710 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); in sfmmu_mod_tsb()
2768 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz) in sfmmu_unload_tsb_range() argument
2787 if (sfmmup == ksfmmup) { in sfmmu_unload_tsb_range()
2800 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz); in sfmmu_unload_tsb_range()
2877 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr, in sfmmu_tteload_array() argument
2894 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size, rid); in sfmmu_tteload_array()
2900 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags, in sfmmu_tteload_array()
2907 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags, in sfmmu_tteload_array()
2922 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size, in sfmmu_tteload_acquire_hashbucket() argument
2927 void *htagid = sfmmutohtagid(sfmmup, rid); in sfmmu_tteload_acquire_hashbucket()
2946 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp, in sfmmu_tteload_find_hmeblk() argument
2953 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size)); in sfmmu_tteload_find_hmeblk()
2955 hblktag.htag_id = sfmmutohtagid(sfmmup, rid); in sfmmu_tteload_find_hmeblk()
2982 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, in sfmmu_tteload_find_hmeblk()
3007 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp); in sfmmu_tteload_find_hmeblk()
3039 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep, in sfmmu_tteload_addentry() argument
3111 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size)); in sfmmu_tteload_addentry()
3160 sfmmu_vac_conflict(sfmmup, vaddr, pp); in sfmmu_tteload_addentry()
3222 if ((sfmmup == KHATID) && in sfmmu_tteload_addentry()
3236 atomic_inc_ulong(&sfmmup->sfmmu_ttecnt[size]); in sfmmu_tteload_addentry()
3238 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in sfmmu_tteload_addentry()
3249 myflt = (astosfmmu(curthread->t_procp->p_as) == sfmmup); in sfmmu_tteload_addentry()
3251 sfmmup != ksfmmup) { in sfmmu_tteload_addentry()
3254 if (!(sfmmup->sfmmu_tteflags & tteflag)) { in sfmmu_tteload_addentry()
3255 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tteload_addentry()
3256 sfmmup->sfmmu_tteflags |= tteflag; in sfmmu_tteload_addentry()
3259 } else if (!(sfmmup->sfmmu_rtteflags & tteflag)) { in sfmmu_tteload_addentry()
3260 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tteload_addentry()
3261 sfmmup->sfmmu_rtteflags |= tteflag; in sfmmu_tteload_addentry()
3289 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { in sfmmu_tteload_addentry()
3290 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tteload_addentry()
3291 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); in sfmmu_tteload_addentry()
3309 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp); in sfmmu_tteload_addentry()
3319 sfmmup->sfmmu_srdp->srd_hmergnp[rid], hmeblkp, 1); in sfmmu_tteload_addentry()
3323 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0); in sfmmu_tteload_addentry()
3324 xt_sync(sfmmup->sfmmu_cpusran); in sfmmu_tteload_addentry()
3336 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tteload_addentry()
3341 scdp = sfmmup->sfmmu_scdp; in sfmmu_tteload_addentry()
3344 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte, in sfmmu_tteload_addentry()
3589 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags) in sfmmu_shadow_hcreate() argument
3597 ASSERT(sfmmup != KHATID); in sfmmu_shadow_hcreate()
3602 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); in sfmmu_shadow_hcreate()
3603 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); in sfmmu_shadow_hcreate()
3612 hblktag.htag_id = sfmmup; in sfmmu_shadow_hcreate()
3617 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); in sfmmu_shadow_hcreate()
3624 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size, in sfmmu_shadow_hcreate()
3669 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_shadow_hcleanup() argument
3691 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno); in sfmmu_shadow_hcleanup()
3697 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr, in sfmmu_free_hblks() argument
3707 hblktag.htag_id = sfmmup; in sfmmu_free_hblks()
3715 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); in sfmmu_free_hblks()
3727 sfmmu_shadow_hcleanup(sfmmup, in sfmmu_free_hblks()
3934 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len) in hat_unlock() argument
3942 ASSERT(sfmmup != NULL); in hat_unlock()
3944 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as)); in hat_unlock()
3947 hblktag.htag_id = sfmmup; in hat_unlock()
3961 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); in hat_unlock()
3987 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { in hat_unlock()
3994 "addr %p hat %p", (void *)addr, (void *)sfmmup); in hat_unlock()
4004 hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len, in hat_unlock_region() argument
4021 hat_unlock(sfmmup, addr, len); in hat_unlock_region()
4025 ASSERT(sfmmup != NULL); in hat_unlock_region()
4026 ASSERT(sfmmup != ksfmmup); in hat_unlock_region()
4028 srdp = sfmmup->sfmmu_srdp; in hat_unlock_region()
4035 SFMMU_VALIDATE_HMERID(sfmmup, rid, addr, len); in hat_unlock_region()
4077 "addr %p hat %p", (void *)va, (void *)sfmmup); in hat_unlock_region()
4734 hat_probe(struct hat *sfmmup, caddr_t addr) in hat_probe() argument
4739 ASSERT(sfmmup != NULL); in hat_probe()
4741 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as)); in hat_probe()
4743 if (sfmmup == ksfmmup) { in hat_probe()
4744 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte)) in hat_probe()
4746 sfmmu_vatopfn_suspended(addr, sfmmup, &tte); in hat_probe()
4749 pfn = sfmmu_uvatopfn(addr, sfmmup, NULL); in hat_probe()
4759 hat_getpagesize(struct hat *sfmmup, caddr_t addr) in hat_getpagesize() argument
4763 if (sfmmup == ksfmmup) { in hat_getpagesize()
4764 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) { in hat_getpagesize()
4768 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) { in hat_getpagesize()
4778 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr) in hat_getattr() argument
4782 if (sfmmup == ksfmmup) { in hat_getattr()
4783 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) { in hat_getattr()
4787 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) { in hat_getattr()
4837 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr, in sfmmu_chgattr() argument
4850 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as)); in sfmmu_chgattr()
4861 hblktag.htag_id = sfmmup; in sfmmu_chgattr()
4863 DEMAP_RANGE_INIT(sfmmup, &dmr); in sfmmu_chgattr()
4869 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); in sfmmu_chgattr()
4881 ASSERT(sfmmup != ksfmmup); in sfmmu_chgattr()
4886 addr = sfmmu_hblk_chgattr(sfmmup, in sfmmu_chgattr()
4895 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { in sfmmu_chgattr()
4905 if (sfmmup == ksfmmup) in sfmmu_chgattr()
4919 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_chgattr()
4933 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_chgattr() argument
4967 check_exec = (sfmmup != ksfmmup) && in sfmmu_hblk_chgattr()
4968 AS_TYPE_64BIT(sfmmup->sfmmu_as) && in sfmmu_hblk_chgattr()
5030 sfmmu_ttesync(sfmmup, addr, &tte, pp); in sfmmu_hblk_chgattr()
5044 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_hblk_chgattr()
5167 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot) in hat_chgprot() argument
5180 ASSERT(sfmmup->sfmmu_as != NULL); in hat_chgprot()
5190 hblktag.htag_id = sfmmup; in hat_chgprot()
5192 DEMAP_RANGE_INIT(sfmmup, &dmr); in hat_chgprot()
5198 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); in hat_chgprot()
5210 ASSERT(sfmmup != ksfmmup); in hat_chgprot()
5215 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp, in hat_chgprot()
5224 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { in hat_chgprot()
5233 if (sfmmup == ksfmmup) in hat_chgprot()
5247 cpuset = sfmmup->sfmmu_cpusran; in hat_chgprot()
5261 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_chgprot() argument
5293 check_exec = (sfmmup != ksfmmup) && in sfmmu_hblk_chgprot()
5294 AS_TYPE_64BIT(sfmmup->sfmmu_as) && in sfmmu_hblk_chgprot()
5358 sfmmu_ttesync(sfmmup, addr, &tte, pp); in sfmmu_hblk_chgprot()
5372 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_hblk_chgprot()
5452 hat_unload_large_virtual(struct hat *sfmmup, caddr_t startaddr, size_t len, in hat_unload_large_virtual() argument
5471 if (sfmmup->sfmmu_free) { in hat_unload_large_virtual()
5475 DEMAP_RANGE_INIT(sfmmup, dmrp); in hat_unload_large_virtual()
5493 if (hmeblkp->hblk_tag.htag_id != sfmmup || in hat_unload_large_virtual()
5507 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in hat_unload_large_virtual()
5542 cpuset = sfmmup->sfmmu_cpusran; in hat_unload_large_virtual()
5563 cpuset = sfmmup->sfmmu_cpusran; in hat_unload_large_virtual()
5576 if (!sfmmup->sfmmu_free) in hat_unload_large_virtual()
5577 sfmmu_check_page_sizes(sfmmup, 0); in hat_unload_large_virtual()
5586 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \ argument
5591 hat_unload_callback(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags, in hat_unload_callback() argument
5604 int issegkmap = ISSEGKMAP(sfmmup, addr); in hat_unload_callback()
5607 ASSERT(sfmmup->sfmmu_as != NULL); in hat_unload_callback()
5609 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \ in hat_unload_callback()
5610 AS_LOCK_HELD(sfmmup->sfmmu_as)); in hat_unload_callback()
5612 ASSERT(sfmmup != NULL); in hat_unload_callback()
5626 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) { in hat_unload_callback()
5627 hat_unload_large_virtual(sfmmup, addr, len, flags, callback); in hat_unload_callback()
5637 if (sfmmup->sfmmu_free) { in hat_unload_callback()
5641 DEMAP_RANGE_INIT(sfmmup, dmrp); in hat_unload_callback()
5645 hblktag.htag_id = sfmmup; in hat_unload_callback()
5663 if (sfmmup == KHATID) { in hat_unload_callback()
5678 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); in hat_unload_callback()
5780 ASSERT(sfmmup != KHATID); in hat_unload_callback()
5798 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr, in hat_unload_callback()
5818 cpuset = sfmmup->sfmmu_cpusran; in hat_unload_callback()
5859 cpuset = sfmmup->sfmmu_cpusran; in hat_unload_callback()
5873 if (!sfmmup->sfmmu_free) in hat_unload_callback()
5874 sfmmu_check_page_sizes(sfmmup, 0); in hat_unload_callback()
5882 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags) in hat_unload() argument
5884 hat_unload_callback(sfmmup, addr, len, flags, NULL); in hat_unload()
5914 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_unload() argument
5928 ASSERT(sfmmup != NULL || hmeblkp->hblk_shared); in sfmmu_hblk_unload()
5929 ASSERT(sfmmup == NULL || !hmeblkp->hblk_shared); in sfmmu_hblk_unload()
6015 sfmmu_ttesync(sfmmup, addr, &tte, pp); in sfmmu_hblk_unload()
6061 ASSERT(sfmmup != NULL); in sfmmu_hblk_unload()
6063 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, in sfmmu_hblk_unload()
6064 sfmmup->sfmmu_free, 0); in sfmmu_hblk_unload()
6179 if (ttecnt > 0 && sfmmup != NULL) { in sfmmu_hblk_unload()
6180 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt); in sfmmu_hblk_unload()
6191 hat_flush_range(struct hat *sfmmup, caddr_t va, size_t size) in hat_flush_range() argument
6197 sz = hat_getpagesize(sfmmup, va); in hat_flush_range()
6202 vtag_flushpage(va, (uint64_t)sfmmup); in hat_flush_range()
6214 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag) in hat_sync() argument
6223 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as)); in hat_sync()
6231 hblktag.htag_id = sfmmup; in hat_sync()
6246 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift); in hat_sync()
6258 ASSERT(sfmmup != ksfmmup); in hat_sync()
6263 addr = sfmmu_hblk_sync(sfmmup, hmeblkp, in hat_sync()
6272 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) { in hat_sync()
6281 if (sfmmup == ksfmmup) in hat_sync()
6293 cpuset = sfmmup->sfmmu_cpusran; in hat_sync()
6298 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, in sfmmu_hblk_sync() argument
6349 sfmmu_tlb_demap(addr, sfmmup, in sfmmu_hblk_sync()
6353 sfmmu_ttesync(sfmmup, addr, &tte, pp); in sfmmu_hblk_sync()
6371 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp) in sfmmu_ttesync() argument
6395 if (sfmmup != NULL && sfmmup->sfmmu_rmstat) { in sfmmu_ttesync()
6400 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm); in sfmmu_ttesync()
6608 sfmmu_t *sfmmup; in hat_pagesuspend() local
6659 sfmmup = hblktosfmmu(hmeblkp); in hat_pagesuspend()
6660 ASSERT(sfmmup == ksfmmup); in hat_pagesuspend()
6670 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in hat_pagesuspend()
7147 sfmmu_t *sfmmup; in sfmmu_pageunload() local
7168 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pageunload()
7204 sf_srd_t *srdp = (sf_srd_t *)sfmmup; in sfmmu_pageunload()
7217 sfmmu_ttesync(sfmmup, addr, &tte, pp); in sfmmu_pageunload()
7218 atomic_dec_ulong(&sfmmup->sfmmu_ttecnt[ttesz]); in sfmmu_pageunload()
7238 if (sfmmup->sfmmu_ismhat) { in sfmmu_pageunload()
7247 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp, in sfmmu_pageunload()
7254 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_pageunload()
7255 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_pageunload()
7491 sfmmu_t *sfmmup; in sfmmu_pagesync() local
7508 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pagesync()
7526 sf_srd_t *srdp = (sf_srd_t *)sfmmup; in sfmmu_pagesync()
7539 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, in sfmmu_pagesync()
7541 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_pagesync()
7545 sfmmu_ttesync(hmeblkp->hblk_shared ? NULL : sfmmup, addr, in sfmmu_pagesync()
7565 sfmmu_t *sfmmup; in sfmmu_pageclrwrt() local
7579 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_pageclrwrt()
7597 sf_srd_t *srdp = (sf_srd_t *)sfmmup; in sfmmu_pageclrwrt()
7609 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0); in sfmmu_pageclrwrt()
7610 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_pageclrwrt()
7869 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup, tte_t *ttep) in sfmmu_uvatopfn() argument
7886 sfmmu_t *sv_sfmmup = sfmmup; in sfmmu_uvatopfn()
7896 ASSERT(sfmmup != ksfmmup); in sfmmu_uvatopfn()
7901 ism_blkp = sfmmup->sfmmu_iblk; in sfmmu_uvatopfn()
7903 sfmmu_ismhat_enter(sfmmup, 0); in sfmmu_uvatopfn()
7904 locked_hatid = sfmmup; in sfmmu_uvatopfn()
7911 sfmmup = ism_hatid = ism_map[i].imap_ismhat; in sfmmu_uvatopfn()
7923 hblktag.htag_id = sfmmup; in sfmmu_uvatopfn()
7929 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift); in sfmmu_uvatopfn()
7947 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt)); in sfmmu_uvatopfn()
8297 ism_tsb_entries(sfmmu_t *sfmmup, int szc) in ism_tsb_entries() argument
8299 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk; in ism_tsb_entries()
8307 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); in ism_tsb_entries()
8308 scdp = sfmmup->sfmmu_scdp; in ism_tsb_entries()
8315 rid < sfmmup->sfmmu_srdp->srd_next_ismrid); in ism_tsb_entries()
8329 sfmmup->sfmmu_ismttecnt[szc] = npgs; in ism_tsb_entries()
8330 sfmmup->sfmmu_scdismttecnt[szc] = npgs_scd; in ism_tsb_entries()
8452 hat_share(struct hat *sfmmup, caddr_t addr, struct hat *ism_hatid, in hat_share() argument
8474 ASSERT(ism_hatid != NULL && sfmmup != NULL); in hat_share()
8501 sfmmu_ismhat_enter(sfmmup, 0); in hat_share()
8506 if (sfmmup->sfmmu_iblk == NULL) { in hat_share()
8507 sfmmup->sfmmu_iblk = new_iblk; in hat_share()
8511 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk); in hat_share()
8520 ism_blkp = sfmmup->sfmmu_iblk; in hat_share()
8546 ism_blkp = sfmmup->sfmmu_iblk; in hat_share()
8570 ism_ment->iment_hat = sfmmup; in hat_share()
8599 old_scdp = sfmmup->sfmmu_scdp; in hat_share()
8601 rcookie = hat_join_region(sfmmup, addr, len, (void *)ism_hatid, 0, in hat_share()
8611 (void) ism_tsb_entries(sfmmup, i); in hat_share()
8626 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_ISM)) { in hat_share()
8627 hatlockp = sfmmu_hat_enter(sfmmup); in hat_share()
8628 SFMMU_FLAGS_SET(sfmmup, HAT_256M_ISM); in hat_share()
8633 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_ISM)) { in hat_share()
8634 hatlockp = sfmmu_hat_enter(sfmmup); in hat_share()
8635 SFMMU_FLAGS_SET(sfmmup, HAT_32M_ISM); in hat_share()
8650 hatlockp = sfmmu_hat_enter(sfmmup); in hat_share()
8651 sfmmu_sync_mmustate(sfmmup); in hat_share()
8655 sfmmu_ismhat_exit(sfmmup, 0); in hat_share()
8666 if (sfmmup->sfmmu_scdp != NULL && old_scdp != sfmmup->sfmmu_scdp) { in hat_share()
8667 sfmmu_check_page_sizes(sfmmup, 0); in hat_share()
8669 sfmmu_check_page_sizes(sfmmup, 1); in hat_share()
8680 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) in hat_unshare() argument
8696 ASSERT(sfmmup != NULL); in hat_unshare()
8697 ASSERT(sfmmup != ksfmmup); in hat_unshare()
8699 ASSERT(sfmmup->sfmmu_as != NULL); in hat_unshare()
8709 sfmmu_ismhat_enter(sfmmup, 0); in hat_unshare()
8724 ism_blkp = sfmmup->sfmmu_iblk; in hat_unshare()
8749 old_scdp = sfmmup->sfmmu_scdp; in hat_unshare()
8754 hat_leave_region(sfmmup, (void *)((uint64_t)ism_rid), in hat_unshare()
8763 hatlockp = sfmmu_hat_enter(sfmmup); in hat_unshare()
8764 sfmmu_invalidate_ctx(sfmmup); in hat_unshare()
8811 if (!sfmmup->sfmmu_free) { in hat_unshare()
8812 hatlockp = sfmmu_hat_enter(sfmmup); in hat_unshare()
8813 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL; in hat_unshare()
8835 (void) ism_tsb_entries(sfmmup, i); in hat_unshare()
8838 sfmmu_ismhat_exit(sfmmup, 0); in hat_unshare()
8851 if (!sfmmup->sfmmu_free) { in hat_unshare()
8852 if (found && old_scdp != NULL && sfmmup->sfmmu_scdp == NULL) { in hat_unshare()
8853 sfmmu_check_page_sizes(sfmmup, 1); in hat_unshare()
8855 sfmmu_check_page_sizes(sfmmup, 0); in hat_unshare()
9458 sfmmu_t *sfmmup; in sfmmu_page_cache() local
9501 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_page_cache()
9507 sf_srd_t *srdp = (sf_srd_t *)sfmmup; in sfmmu_page_cache()
9519 } else if (sfmmup->sfmmu_ismhat) { in sfmmu_page_cache()
9525 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9528 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9542 sf_srd_t *srdp = (sf_srd_t *)sfmmup; in sfmmu_page_cache()
9553 } else if (sfmmup->sfmmu_ismhat) { in sfmmu_page_cache()
9559 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp, in sfmmu_page_cache()
9562 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1); in sfmmu_page_cache()
9606 sfmmu_get_ctx(sfmmu_t *sfmmup) in sfmmu_get_ctx() argument
9612 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_get_ctx()
9613 ASSERT(sfmmup != ksfmmup); in sfmmu_get_ctx()
9615 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID)) { in sfmmu_get_ctx()
9616 sfmmu_setup_tsbinfo(sfmmup); in sfmmu_get_ctx()
9617 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ALLCTX_INVALID); in sfmmu_get_ctx()
9637 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { in sfmmu_get_ctx()
9638 mmu_set_ctx_page_sizes(sfmmup); in sfmmu_get_ctx()
9651 if (sfmmu_alloc_ctx(sfmmup, 1, CPU, SFMMU_PRIVATE) && in sfmmu_get_ctx()
9652 sfmmup->sfmmu_scdp != NULL) { in sfmmu_get_ctx()
9653 sf_scd_t *scdp = sfmmup->sfmmu_scdp; in sfmmu_get_ctx()
9660 sfmmu_load_mmustate(sfmmup); in sfmmu_get_ctx()
9759 sfmmu_sync_mmustate(sfmmu_t *sfmmup) in sfmmu_sync_mmustate() argument
9764 ASSERT(sfmmup != ksfmmup); in sfmmu_sync_mmustate()
9765 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_sync_mmustate()
9774 if ((sfmmup == curthread->t_procp->p_as->a_hat) && in sfmmu_sync_mmustate()
9777 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; in sfmmu_sync_mmustate()
9790 sfmmu_load_mmustate(sfmmup); in sfmmu_sync_mmustate()
9799 sfmmu_invalidate_ctx(sfmmup); in sfmmu_sync_mmustate()
9822 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, in sfmmu_replace_tsb() argument
9830 ASSERT(sfmmup != ksfmmup); in sfmmu_replace_tsb()
9831 ASSERT(sfmmup->sfmmu_ismhat == 0); in sfmmu_replace_tsb()
9832 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_replace_tsb()
9835 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY)) in sfmmu_replace_tsb()
9843 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; in sfmmu_replace_tsb()
9849 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { in sfmmu_replace_tsb()
9858 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY); in sfmmu_replace_tsb()
9874 tte_sz_mask, flags, sfmmup) && in sfmmu_replace_tsb()
9879 tte_sz_mask, flags, sfmmup))) { in sfmmu_replace_tsb()
9880 (void) sfmmu_hat_enter(sfmmup); in sfmmu_replace_tsb()
9883 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); in sfmmu_replace_tsb()
9886 (void) sfmmu_hat_enter(sfmmup); in sfmmu_replace_tsb()
9894 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { in sfmmu_replace_tsb()
9896 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); in sfmmu_replace_tsb()
9899 (void) sfmmu_hat_enter(sfmmup); in sfmmu_replace_tsb()
9905 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb; in sfmmu_replace_tsb()
9930 sfmmu_invalidate_ctx(sfmmup); in sfmmu_replace_tsb()
9939 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); in sfmmu_replace_tsb()
9950 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); in sfmmu_replace_tsb()
9959 sfmmup->sfmmu_tsb = new_tsbinfo; in sfmmu_replace_tsb()
9969 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); in sfmmu_replace_tsb()
9984 (void) sfmmu_hat_enter(sfmmup); in sfmmu_replace_tsb()
9995 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) in sfmmu_reprog_pgsz_arr() argument
10000 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_reprog_pgsz_arr()
10004 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; in sfmmu_reprog_pgsz_arr()
10008 sfmmu_invalidate_ctx(sfmmup); in sfmmu_reprog_pgsz_arr()
10080 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing) in sfmmu_check_page_sizes() argument
10091 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != 0) in sfmmu_check_page_sizes()
10094 if (!SFMMU_LGPGS_INUSE(sfmmup) && in sfmmu_check_page_sizes()
10095 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor) in sfmmu_check_page_sizes()
10099 ttecnt[i] = sfmmup->sfmmu_ttecnt[i] + in sfmmu_check_page_sizes()
10100 sfmmup->sfmmu_ismttecnt[i]; in sfmmu_check_page_sizes()
10105 mmu_check_page_sizes(sfmmup, ttecnt); in sfmmu_check_page_sizes()
10125 tte8k_cnt += sfmmup->sfmmu_tsb0_4minflcnt; in sfmmu_check_page_sizes()
10142 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) { in sfmmu_check_page_sizes()
10156 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh); in sfmmu_check_page_sizes()
10160 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt, in sfmmu_size_tsb() argument
10168 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_size_tsb()
10170 tsbinfop = sfmmup->sfmmu_tsb; in sfmmu_size_tsb()
10185 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, in sfmmu_size_tsb()
10188 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc, in sfmmu_size_tsb()
10191 tsbinfop = sfmmup->sfmmu_tsb; in sfmmu_size_tsb()
10216 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)? in sfmmu_size_tsb()
10230 allocflags, sfmmup)) && in sfmmu_size_tsb()
10233 tsb_bits, allocflags, sfmmup)) && in sfmmu_size_tsb()
10235 tsb_bits, allocflags, sfmmup)) { in sfmmu_size_tsb()
10239 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_size_tsb()
10241 sfmmu_invalidate_ctx(sfmmup); in sfmmu_size_tsb()
10243 if (sfmmup->sfmmu_tsb->tsb_next == NULL) { in sfmmu_size_tsb()
10244 sfmmup->sfmmu_tsb->tsb_next = newtsb; in sfmmu_size_tsb()
10279 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, in sfmmu_size_tsb()
10283 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, in sfmmu_size_tsb()
10297 sfmmu_free_sfmmu(sfmmu_t *sfmmup) in sfmmu_free_sfmmu() argument
10305 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0); in sfmmu_free_sfmmu()
10306 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0); in sfmmu_free_sfmmu()
10307 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0); in sfmmu_free_sfmmu()
10308 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); in sfmmu_free_sfmmu()
10309 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); in sfmmu_free_sfmmu()
10310 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); in sfmmu_free_sfmmu()
10311 ASSERT(SF_RGNMAP_ISNULL(sfmmup)); in sfmmu_free_sfmmu()
10313 sfmmup->sfmmu_free = 0; in sfmmu_free_sfmmu()
10314 sfmmup->sfmmu_ismhat = 0; in sfmmu_free_sfmmu()
10316 blkp = sfmmup->sfmmu_iblk; in sfmmu_free_sfmmu()
10317 sfmmup->sfmmu_iblk = NULL; in sfmmu_free_sfmmu()
10803 sfmmu_hat_enter(sfmmu_t *sfmmup) in sfmmu_hat_enter() argument
10807 if (sfmmup != ksfmmup) { in sfmmu_hat_enter()
10808 hatlockp = TSB_HASH(sfmmup); in sfmmu_hat_enter()
10816 sfmmu_hat_tryenter(sfmmu_t *sfmmup) in sfmmu_hat_tryenter() argument
10820 if (sfmmup != ksfmmup) { in sfmmu_hat_tryenter()
10821 hatlockp = TSB_HASH(sfmmup); in sfmmu_hat_tryenter()
10853 sfmmu_hat_lock_held(sfmmu_t *sfmmup) in sfmmu_hat_lock_held() argument
10855 ASSERT(sfmmup != ksfmmup); in sfmmu_hat_lock_held()
10856 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup)))); in sfmmu_hat_lock_held()
10871 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held) in sfmmu_ismhat_enter() argument
10876 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_ismhat_enter()
10877 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) in sfmmu_ismhat_enter()
10878 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); in sfmmu_ismhat_enter()
10879 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); in sfmmu_ismhat_enter()
10885 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held) in sfmmu_ismhat_exit() argument
10890 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_ismhat_exit()
10891 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); in sfmmu_ismhat_exit()
10892 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); in sfmmu_ismhat_exit()
10893 cv_broadcast(&sfmmup->sfmmu_tsb_cv); in sfmmu_ismhat_exit()
10923 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr, in sfmmu_hblk_alloc() argument
10941 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size)); in sfmmu_hblk_alloc()
10991 if (sfmmup != KHATID && !SFMMU_IS_SHMERID_VALID(rid)) { in sfmmu_hblk_alloc()
10994 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, in sfmmu_hblk_alloc()
10998 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr, in sfmmu_hblk_alloc()
11013 ASSERT(sfmmup != KHATID); in sfmmu_hblk_alloc()
11015 srdp = sfmmup->sfmmu_srdp; in sfmmu_hblk_alloc()
11066 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP); in sfmmu_hblk_alloc()
11088 if (size == TTE8K && sfmmup != KHATID) { in sfmmu_hblk_alloc()
11153 forcefree = (sfmmup == KHATID) ? 1 : 0; in sfmmu_hblk_alloc()
11158 ASSERT(sfmmup != KHATID); in sfmmu_hblk_alloc()
11446 sfmmu_t *sfmmup; in sfmmu_steal_this_hblk() local
11449 sfmmup = hblktosfmmu(hmeblkp); in sfmmu_steal_this_hblk()
11450 if (hmeblkp->hblk_shared || sfmmup->sfmmu_ismhat) { in sfmmu_steal_this_hblk()
11453 DEMAP_RANGE_INIT(sfmmup, &dmr); in sfmmu_steal_this_hblk()
11454 (void) sfmmu_hblk_unload(sfmmup, hmeblkp, in sfmmu_steal_this_hblk()
11550 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) in sfmmu_tsb_swapin() argument
11556 ASSERT(sfmmup != ksfmmup); in sfmmu_tsb_swapin()
11557 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_tsb_swapin()
11559 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) { in sfmmu_tsb_swapin()
11560 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp)); in sfmmu_tsb_swapin()
11563 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { in sfmmu_tsb_swapin()
11564 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN); in sfmmu_tsb_swapin()
11569 ASSERT(sfmmup->sfmmu_tsb != NULL); in sfmmu_tsb_swapin()
11575 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) { in sfmmu_tsb_swapin()
11578 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc, in sfmmu_tsb_swapin()
11588 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); in sfmmu_tsb_swapin()
11589 cv_broadcast(&sfmmup->sfmmu_tsb_cv); in sfmmu_tsb_swapin()
11607 tsbinfop = sfmmup->sfmmu_tsb; in sfmmu_tsb_swapin()
11616 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsb_swapin()
11623 tsbinfop = sfmmup->sfmmu_tsb; in sfmmu_tsb_swapin()
11624 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE, in sfmmu_tsb_swapin()
11629 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN); in sfmmu_tsb_swapin()
11630 cv_broadcast(&sfmmup->sfmmu_tsb_cv); in sfmmu_tsb_swapin()
11689 sfmmu_t *sfmmup, *shsfmmup; in sfmmu_tsbmiss_exception() local
11700 sfmmup = astosfmmu(curthread->t_procp->p_as); in sfmmu_tsbmiss_exception()
11707 ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT); in sfmmu_tsbmiss_exception()
11708 ASSERT(sfmmup->sfmmu_ismhat == 0); in sfmmu_tsbmiss_exception()
11709 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || in sfmmu_tsbmiss_exception()
11733 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsbmiss_exception()
11736 ASSERT(sfmmup == tsbmp->usfmmup); in sfmmu_tsbmiss_exception()
11737 if (((tsbmp->uhat_tteflags ^ sfmmup->sfmmu_tteflags) & in sfmmu_tsbmiss_exception()
11739 ((tsbmp->uhat_rtteflags ^ sfmmup->sfmmu_rtteflags) & in sfmmu_tsbmiss_exception()
11741 tsbmp->uhat_tteflags = sfmmup->sfmmu_tteflags; in sfmmu_tsbmiss_exception()
11742 tsbmp->uhat_rtteflags = sfmmup->sfmmu_rtteflags; in sfmmu_tsbmiss_exception()
11745 if (sfmmup->sfmmu_srdp != NULL) { in sfmmu_tsbmiss_exception()
11746 ulong_t *sm = sfmmup->sfmmu_hmeregion_map.bitmap; in sfmmu_tsbmiss_exception()
11754 sfmmup->sfmmu_srdp, in sfmmu_tsbmiss_exception()
11791 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsbmiss_exception()
11793 if ((scdp = sfmmup->sfmmu_scdp) != NULL) { in sfmmu_tsbmiss_exception()
11808 if (sfmmup->sfmmu_scdp == scdp && in sfmmu_tsbmiss_exception()
11814 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsbmiss_exception()
11820 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; in sfmmu_tsbmiss_exception()
11823 cv_wait(&sfmmup->sfmmu_tsb_cv, in sfmmu_tsbmiss_exception()
11832 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { in sfmmu_tsbmiss_exception()
11833 cv_wait(&sfmmup->sfmmu_tsb_cv, in sfmmu_tsbmiss_exception()
11839 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) { in sfmmu_tsbmiss_exception()
11845 sfmmu_finish_join_scd(sfmmup); in sfmmu_tsbmiss_exception()
11846 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD); in sfmmu_tsbmiss_exception()
11855 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { in sfmmu_tsbmiss_exception()
11856 sfmmu_tsb_swapin(sfmmup, hatlockp); in sfmmu_tsbmiss_exception()
11860 sfmmu_get_ctx(sfmmup); in sfmmu_tsbmiss_exception()
11875 sfmmu_tsb_chk_reloc(sfmmu_t *sfmmup, hatlock_t *hatlockp) in sfmmu_tsb_chk_reloc() argument
11879 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_tsb_chk_reloc()
11881 for (tp = sfmmup->sfmmu_tsb; tp != NULL; tp = tp->tsb_next) { in sfmmu_tsb_chk_reloc()
11883 cv_wait(&sfmmup->sfmmu_tsb_cv, in sfmmu_tsb_chk_reloc()
11941 sfmmu_t *sfmmup; in sfmmu_rgntlb_demap() local
11957 sfmmup = rgnp->rgn_sfmmu_head; in sfmmu_rgntlb_demap()
11958 while (sfmmup != NULL) { in sfmmu_rgntlb_demap()
11960 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_rgntlb_demap()
11971 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL && in sfmmu_rgntlb_demap()
11972 !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) { in sfmmu_rgntlb_demap()
11973 scdp = sfmmup->sfmmu_scdp; in sfmmu_rgntlb_demap()
11982 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_rgntlb_demap()
11985 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_rgntlb_demap()
11988 SFMMU_XCALL_STATS(sfmmup); in sfmmu_rgntlb_demap()
11990 (uint64_t)addr, (uint64_t)sfmmup); in sfmmu_rgntlb_demap()
11991 vtag_flushpage(addr, (uint64_t)sfmmup); in sfmmu_rgntlb_demap()
12000 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0); in sfmmu_rgntlb_demap()
12002 sfmmup = rlink->next; in sfmmu_rgntlb_demap()
12017 find_ism_rid(sfmmu_t *sfmmup, sfmmu_t *ism_sfmmup, caddr_t va, uint_t *ism_rid) in find_ism_rid() argument
12025 ASSERT(sfmmu_hat_lock_held(sfmmup)); in find_ism_rid()
12027 ism_blkp = sfmmup->sfmmu_iblk; in find_ism_rid()
12062 sfmmu_t *sfmmup; in sfmmu_ismtlbcache_demap() local
12087 sfmmup = ment->iment_hat; in sfmmu_ismtlbcache_demap()
12101 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL && in sfmmu_ismtlbcache_demap()
12102 !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD) && in sfmmu_ismtlbcache_demap()
12103 !SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { in sfmmu_ismtlbcache_demap()
12104 if (!find_ism_rid(sfmmup, ism_sfmmup, va, in sfmmu_ismtlbcache_demap()
12110 scdp = sfmmup->sfmmu_scdp; in sfmmu_ismtlbcache_demap()
12117 SFMMU_UNLOAD_TSB(va, sfmmup, hmeblkp, 1); in sfmmu_ismtlbcache_demap()
12119 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_ismtlbcache_demap()
12122 SFMMU_XCALL_STATS(sfmmup); in sfmmu_ismtlbcache_demap()
12124 (uint64_t)sfmmup); in sfmmu_ismtlbcache_demap()
12125 vtag_flushpage(va, (uint64_t)sfmmup); in sfmmu_ismtlbcache_demap()
12137 SFMMU_XCALL_STATS(sfmmup); in sfmmu_ismtlbcache_demap()
12153 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_tlbcache_demap() argument
12189 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tlbcache_demap()
12196 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_tlbcache_demap()
12198 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_tlbcache_demap()
12202 SFMMU_XCALL_STATS(sfmmup); in sfmmu_tlbcache_demap()
12205 (uint64_t)sfmmup); in sfmmu_tlbcache_demap()
12207 vtag_flushpage(addr, (uint64_t)sfmmup); in sfmmu_tlbcache_demap()
12224 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_tlbcache_demap()
12228 SFMMU_XCALL_STATS(sfmmup); in sfmmu_tlbcache_demap()
12242 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, in sfmmu_tlb_demap() argument
12260 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tlb_demap()
12261 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0); in sfmmu_tlb_demap()
12265 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_tlb_demap()
12269 SFMMU_XCALL_STATS(sfmmup); in sfmmu_tlb_demap()
12270 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); in sfmmu_tlb_demap()
12272 vtag_flushpage(addr, (uint64_t)sfmmup); in sfmmu_tlb_demap()
12293 sfmmu_t *sfmmup = dmrp->dmr_sfmmup; in sfmmu_tlb_range_demap() local
12326 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tlb_range_demap()
12327 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K); in sfmmu_tlb_range_demap()
12335 if (sfmmup->sfmmu_free == 0) { in sfmmu_tlb_range_demap()
12351 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); in sfmmu_tlb_range_demap()
12352 sfmmu_pgcnt = (uint64_t)sfmmup | in sfmmu_tlb_range_demap()
12364 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tlb_range_demap()
12367 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_tlb_range_demap()
12371 SFMMU_XCALL_STATS(sfmmup); in sfmmu_tlb_range_demap()
12377 vtag_flushpage(addr, (uint64_t)sfmmup); in sfmmu_tlb_range_demap()
12412 sfmmu_invalidate_ctx(sfmmu_t *sfmmup) in sfmmu_invalidate_ctx() argument
12422 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_invalidate_ctx()
12423 ASSERT(sfmmup != ksfmmup); in sfmmu_invalidate_ctx()
12432 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; in sfmmu_invalidate_ctx()
12436 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ in sfmmu_invalidate_ctx()
12440 cnum = sfmmup->sfmmu_ctxs[i].cnum; in sfmmu_invalidate_ctx()
12445 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; in sfmmu_invalidate_ctx()
12448 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ in sfmmu_invalidate_ctx()
12452 cpuset = sfmmup->sfmmu_cpusran; in sfmmu_invalidate_ctx()
12456 SFMMU_XCALL_STATS(sfmmup); in sfmmu_invalidate_ctx()
12458 (uint64_t)sfmmup, INVALID_CONTEXT); in sfmmu_invalidate_ctx()
12476 SFMMU_FLAGS_SET(sfmmup, HAT_ALLCTX_INVALID); in sfmmu_invalidate_ctx()
12484 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); in sfmmu_invalidate_ctx()
12540 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; in sfmmu_tsb_pre_relocator() local
12553 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsb_pre_relocator()
12557 sfmmu_tsb_inv_ctx(sfmmup); in sfmmu_tsb_pre_relocator()
12563 ASSERT(sfmmup->sfmmu_scdhat); in sfmmu_tsb_pre_relocator()
12564 scdp = sfmmup->sfmmu_scdp; in sfmmu_tsb_pre_relocator()
12568 sfmmup = scdp->scd_sf_list; in sfmmu_tsb_pre_relocator()
12569 while (sfmmup != NULL) { in sfmmu_tsb_pre_relocator()
12570 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsb_pre_relocator()
12576 sfmmu_invalidate_ctx(sfmmup); in sfmmu_tsb_pre_relocator()
12578 sfmmup = sfmmup->sfmmu_scd_link.next; in sfmmu_tsb_pre_relocator()
12587 sfmmu_tsb_inv_ctx(sfmmu_t *sfmmup) in sfmmu_tsb_inv_ctx() argument
12591 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_tsb_inv_ctx()
12623 sfmmu_invalidate_ctx(sfmmup); in sfmmu_tsb_inv_ctx()
12633 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; in sfmmu_tsb_post_relocator() local
12638 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_tsb_post_relocator()
12660 cv_broadcast(&sfmmup->sfmmu_tsb_cv); in sfmmu_tsb_post_relocator()
12673 uint_t flags, sfmmu_t *sfmmup) in sfmmu_tsbinfo_alloc() argument
12681 tsb_szc, flags, sfmmup)) != 0) { in sfmmu_tsbinfo_alloc()
12797 int tsbcode, uint_t flags, sfmmu_t *sfmmup) in sfmmu_init_tsbinfo() argument
12823 tsbinfo->tsb_sfmmu = sfmmup; in sfmmu_init_tsbinfo()
13596 hat_join_srd(struct hat *sfmmup, vnode_t *evp) in hat_join_srd() argument
13602 ASSERT(sfmmup != ksfmmup); in hat_join_srd()
13603 ASSERT(sfmmup->sfmmu_srdp == NULL); in hat_join_srd()
13617 sfmmup->sfmmu_srdp = srdp; in hat_join_srd()
13639 sfmmup->sfmmu_srdp = srdp; in hat_join_srd()
13648 sfmmup->sfmmu_srdp = newsrdp; in hat_join_srd()
13655 sfmmu_leave_srd(sfmmu_t *sfmmup) in sfmmu_leave_srd() argument
13658 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in sfmmu_leave_srd()
13668 ASSERT(sfmmup != ksfmmup); in sfmmu_leave_srd()
13671 ASSERT(sfmmup->sfmmu_scdp == NULL); in sfmmu_leave_srd()
13672 ASSERT(sfmmup->sfmmu_free == 1); in sfmmu_leave_srd()
13674 sfmmup->sfmmu_srdp = NULL; in sfmmu_leave_srd()
13803 hat_join_region(struct hat *sfmmup, caddr_t r_saddr, size_t r_size, in hat_join_region() argument
13807 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in hat_join_region()
13828 ASSERT(sfmmup != ksfmmup); in hat_join_region()
13829 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as)); in hat_join_region()
13894 (sfmmup == astosfmmu(curthread->t_procp->p_as)); in hat_join_region()
13896 sfmmu_link_to_hmeregion(sfmmup, rgnp); in hat_join_region()
13921 if (tteflag && !(sfmmup->sfmmu_rtteflags & tteflag)) { in hat_join_region()
13922 hatlockp = sfmmu_hat_enter(sfmmup); in hat_join_region()
13923 sfmmup->sfmmu_rtteflags |= tteflag; in hat_join_region()
13926 hatlockp = sfmmu_hat_enter(sfmmup); in hat_join_region()
13933 sfmmup->sfmmu_tsb0_4minflcnt += in hat_join_region()
13939 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc], in hat_join_region()
13945 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) { in hat_join_region()
13946 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG); in hat_join_region()
13957 sfmmu_check_page_sizes(sfmmup, 1); in hat_join_region()
13958 hatlockp = sfmmu_hat_enter(sfmmup); in hat_join_region()
13959 SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid); in hat_join_region()
13968 (sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum in hat_join_region()
13973 ASSERT(sfmmup == tsbmp->usfmmup); in hat_join_region()
13986 hatlockp = sfmmu_hat_enter(sfmmup); in hat_join_region()
13987 SF_RGNMAP_ADD(sfmmup->sfmmu_ismregion_map, rid); in hat_join_region()
13993 sfmmu_find_scd(sfmmup); in hat_join_region()
14091 hat_leave_region(struct hat *sfmmup, hat_region_cookie_t rcookie, uint_t flags) in hat_leave_region() argument
14093 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in hat_leave_region()
14109 ASSERT(sfmmup != ksfmmup); in hat_leave_region()
14114 ASSERT(!sfmmup->sfmmu_free || sfmmup->sfmmu_scdp == NULL); in hat_leave_region()
14132 ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as)); in hat_leave_region()
14134 if (sfmmup->sfmmu_free) { in hat_leave_region()
14139 ASSERT(sfmmup->sfmmu_scdp == NULL); in hat_leave_region()
14141 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid); in hat_leave_region()
14145 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt); in hat_leave_region()
14147 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc], in hat_leave_region()
14150 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid); in hat_leave_region()
14153 hatlockp = sfmmu_hat_enter(sfmmup); in hat_leave_region()
14155 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid); in hat_leave_region()
14156 scdp = sfmmup->sfmmu_scdp; in hat_leave_region()
14159 sfmmu_leave_scd(sfmmup, r_type); in hat_leave_region()
14160 ASSERT(sfmmu_hat_lock_held(sfmmup)); in hat_leave_region()
14171 hatlockp = sfmmu_hat_enter(sfmmup); in hat_leave_region()
14173 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid); in hat_leave_region()
14182 scdp = sfmmup->sfmmu_scdp; in hat_leave_region()
14185 sfmmu_leave_scd(sfmmup, r_type); in hat_leave_region()
14186 ASSERT(sfmmu_hat_lock_held(sfmmup)); in hat_leave_region()
14188 sfmmu_invalidate_ctx(sfmmup); in hat_leave_region()
14193 sfmmu_unload_tsb_range(sfmmup, r_saddr, in hat_leave_region()
14207 ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >= in hat_leave_region()
14209 sfmmup->sfmmu_tsb0_4minflcnt -= rttecnt; in hat_leave_region()
14214 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt); in hat_leave_region()
14215 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc], -rttecnt); in hat_leave_region()
14218 if (scdp != NULL && sfmmup->sfmmu_scdp == NULL) { in hat_leave_region()
14220 sfmmu_check_page_sizes(sfmmup, 1); in hat_leave_region()
14222 sfmmu_check_page_sizes(sfmmup, 0); in hat_leave_region()
14227 sfmmu_unlink_from_hmeregion(sfmmup, rgnp); in hat_leave_region()
14293 hat_dup_region(struct hat *sfmmup, hat_region_cookie_t rcookie) in hat_dup_region() argument
14295 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in hat_dup_region()
14302 ASSERT(sfmmup != ksfmmup); in hat_dup_region()
14319 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 0); in hat_dup_region()
14329 hrlink->prev = sfmmup; in hat_dup_region()
14335 rgnp->rgn_sfmmu_head = sfmmup; in hat_dup_region()
14340 atomic_add_long(&sfmmup->sfmmu_ttecnt[rgnp->rgn_pgszc], rttecnt); in hat_dup_region()
14343 sfmmup->sfmmu_tsb0_4minflcnt += in hat_dup_region()
14350 SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid); in hat_dup_region()
14401 check_scd_sfmmu_list(sfmmu_t **headp, sfmmu_t *sfmmup, int onlist) in check_scd_sfmmu_list() argument
14404 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in check_scd_sfmmu_list()
14408 if (sp == sfmmup) { in check_scd_sfmmu_list()
14413 "list 0x%p", (void *)sfmmup, in check_scd_sfmmu_list()
14420 (void *)sfmmup, (void *)*headp); in check_scd_sfmmu_list()
14426 #define check_scd_sfmmu_list(headp, sfmmup, onlist) argument
14433 sfmmu_from_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup) in sfmmu_from_scd_list() argument
14435 ASSERT(sfmmup->sfmmu_srdp != NULL); in sfmmu_from_scd_list()
14436 check_scd_sfmmu_list(headp, sfmmup, 1); in sfmmu_from_scd_list()
14437 if (sfmmup->sfmmu_scd_link.prev != NULL) { in sfmmu_from_scd_list()
14438 ASSERT(*headp != sfmmup); in sfmmu_from_scd_list()
14439 sfmmup->sfmmu_scd_link.prev->sfmmu_scd_link.next = in sfmmu_from_scd_list()
14440 sfmmup->sfmmu_scd_link.next; in sfmmu_from_scd_list()
14442 ASSERT(*headp == sfmmup); in sfmmu_from_scd_list()
14443 *headp = sfmmup->sfmmu_scd_link.next; in sfmmu_from_scd_list()
14445 if (sfmmup->sfmmu_scd_link.next != NULL) { in sfmmu_from_scd_list()
14446 sfmmup->sfmmu_scd_link.next->sfmmu_scd_link.prev = in sfmmu_from_scd_list()
14447 sfmmup->sfmmu_scd_link.prev; in sfmmu_from_scd_list()
14456 sfmmu_to_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup) in sfmmu_to_scd_list() argument
14458 check_scd_sfmmu_list(headp, sfmmup, 0); in sfmmu_to_scd_list()
14459 sfmmup->sfmmu_scd_link.prev = NULL; in sfmmu_to_scd_list()
14460 sfmmup->sfmmu_scd_link.next = *headp; in sfmmu_to_scd_list()
14462 (*headp)->sfmmu_scd_link.prev = sfmmup; in sfmmu_to_scd_list()
14463 *headp = sfmmup; in sfmmu_to_scd_list()
14632 sfmmu_link_to_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp) in sfmmu_link_to_hmeregion() argument
14643 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 1); in sfmmu_link_to_hmeregion()
14654 rgnp->rgn_sfmmu_head = sfmmup; in sfmmu_link_to_hmeregion()
14662 hrlink->prev = sfmmup; in sfmmu_link_to_hmeregion()
14668 rgnp->rgn_sfmmu_head = sfmmup; in sfmmu_link_to_hmeregion()
14677 sfmmu_unlink_from_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp) in sfmmu_unlink_from_hmeregion() argument
14686 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0); in sfmmu_unlink_from_hmeregion()
14689 if (rgnp->rgn_sfmmu_head == sfmmup) { in sfmmu_unlink_from_hmeregion()
14706 ASSERT(nrlink->prev == sfmmup); in sfmmu_unlink_from_hmeregion()
14718 ASSERT(prlink->next == sfmmup); in sfmmu_unlink_from_hmeregion()
14734 ASSERT(nrlink->prev == sfmmup); in sfmmu_unlink_from_hmeregion()
14912 sfmmu_join_scd(sf_scd_t *scdp, sfmmu_t *sfmmup) in sfmmu_join_scd() argument
14915 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in sfmmu_join_scd()
14922 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as)); in sfmmu_join_scd()
14924 if ((old_scdp = sfmmup->sfmmu_scdp) != NULL) { in sfmmu_join_scd()
14928 sfmmu_from_scd_list(&old_scdp->scd_sf_list, sfmmup); in sfmmu_join_scd()
14936 ASSERT(sfmmup->sfmmu_scdrttecnt[i] == in sfmmu_join_scd()
14938 atomic_add_long(&sfmmup->sfmmu_ttecnt[i], in sfmmu_join_scd()
14939 sfmmup->sfmmu_scdrttecnt[i]); in sfmmu_join_scd()
14947 sfmmu_to_scd_list(&scdp->scd_sf_list, sfmmup); in sfmmu_join_scd()
14951 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_join_scd()
14957 SFMMU_FLAGS_SET(sfmmup, HAT_JOIN_SCD); in sfmmu_join_scd()
14959 sfmmu_invalidate_ctx(sfmmup); in sfmmu_join_scd()
14960 sfmmup->sfmmu_scdp = scdp; in sfmmu_join_scd()
14967 sfmmup->sfmmu_scdrttecnt[i] = scdp->scd_rttecnt[i]; in sfmmu_join_scd()
14968 ASSERT(sfmmup->sfmmu_ttecnt[i] >= scdp->scd_rttecnt[i]); in sfmmu_join_scd()
14969 atomic_add_long(&sfmmup->sfmmu_ttecnt[i], in sfmmu_join_scd()
14970 -sfmmup->sfmmu_scdrttecnt[i]); in sfmmu_join_scd()
14974 sfmmup->sfmmu_tsb0_4minflcnt += in sfmmu_join_scd()
14977 ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >= in sfmmu_join_scd()
14979 sfmmup->sfmmu_tsb0_4minflcnt -= scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt; in sfmmu_join_scd()
14995 sfmmu_finish_join_scd(sfmmu_t *sfmmup) in sfmmu_finish_join_scd() argument
14999 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_finish_join_scd()
15000 ASSERT(sfmmup->sfmmu_scdp != NULL); in sfmmu_finish_join_scd()
15001 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)); in sfmmu_finish_join_scd()
15002 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); in sfmmu_finish_join_scd()
15003 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID)); in sfmmu_finish_join_scd()
15005 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; in sfmmu_finish_join_scd()
15017 sfmmu_ism_hatflags(sfmmup, 1); in sfmmu_finish_join_scd()
15027 sfmmu_find_scd(sfmmu_t *sfmmup) in sfmmu_find_scd() argument
15029 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in sfmmu_find_scd()
15034 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as)); in sfmmu_find_scd()
15040 &sfmmup->sfmmu_region_map, ret); in sfmmu_find_scd()
15044 sfmmu_join_scd(scdp, sfmmup); in sfmmu_find_scd()
15056 &sfmmup->sfmmu_region_map, ret); in sfmmu_find_scd()
15068 if ((new_scdp = sfmmu_alloc_scd(srdp, &sfmmup->sfmmu_region_map)) == in sfmmu_find_scd()
15089 sfmmu_join_scd(new_scdp, sfmmup); in sfmmu_find_scd()
15100 sfmmu_leave_scd(sfmmu_t *sfmmup, uchar_t r_type) in sfmmu_leave_scd() argument
15102 sf_scd_t *scdp = sfmmup->sfmmu_scdp; in sfmmu_leave_scd()
15103 sf_srd_t *srdp = sfmmup->sfmmu_srdp; in sfmmu_leave_scd()
15104 hatlock_t *hatlockp = TSB_HASH(sfmmup); in sfmmu_leave_scd()
15110 if (sfmmup->sfmmu_free) { in sfmmu_leave_scd()
15116 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup); in sfmmu_leave_scd()
15123 ASSERT(sfmmup->sfmmu_scdrttecnt[i] == in sfmmu_leave_scd()
15125 atomic_add_long(&sfmmup->sfmmu_ttecnt[i], in sfmmu_leave_scd()
15126 sfmmup->sfmmu_scdrttecnt[i]); in sfmmu_leave_scd()
15127 sfmmup->sfmmu_scdrttecnt[i] = 0; in sfmmu_leave_scd()
15129 sfmmup->sfmmu_scdp = NULL; in sfmmu_leave_scd()
15136 SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); in sfmmu_leave_scd()
15138 ASSERT(!sfmmup->sfmmu_free); in sfmmu_leave_scd()
15139 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_leave_scd()
15140 ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as)); in sfmmu_leave_scd()
15146 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY) && in sfmmu_leave_scd()
15147 sfmmup->sfmmu_scdp != NULL) { in sfmmu_leave_scd()
15148 cv_wait(&sfmmup->sfmmu_tsb_cv, in sfmmu_leave_scd()
15152 if (sfmmup->sfmmu_scdp == NULL) { in sfmmu_leave_scd()
15156 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY); in sfmmu_leave_scd()
15159 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) { in sfmmu_leave_scd()
15160 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD); in sfmmu_leave_scd()
15171 sfmmu_invalidate_ctx(sfmmup); in sfmmu_leave_scd()
15175 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)); in sfmmu_leave_scd()
15176 sfmmu_ism_hatflags(sfmmup, 0); in sfmmu_leave_scd()
15183 ASSERT(sfmmup->sfmmu_scdrttecnt[i] == in sfmmu_leave_scd()
15185 atomic_add_long(&sfmmup->sfmmu_ttecnt[i], in sfmmu_leave_scd()
15186 sfmmup->sfmmu_scdrttecnt[i]); in sfmmu_leave_scd()
15187 sfmmup->sfmmu_scdrttecnt[i] = 0; in sfmmu_leave_scd()
15189 sfmmup->sfmmu_ismttecnt[i] += sfmmup->sfmmu_scdismttecnt[i]; in sfmmu_leave_scd()
15190 sfmmup->sfmmu_scdismttecnt[i] = 0; in sfmmu_leave_scd()
15193 sfmmup->sfmmu_tsb0_4minflcnt += scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt; in sfmmu_leave_scd()
15196 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY); in sfmmu_leave_scd()
15198 sfmmup->sfmmu_scdp = NULL; in sfmmu_leave_scd()
15211 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup); in sfmmu_leave_scd()
15216 hatlockp = sfmmu_hat_enter(sfmmup); in sfmmu_leave_scd()
15293 sfmmu_ism_hatflags(sfmmu_t *sfmmup, int addflag) in sfmmu_ism_hatflags() argument
15295 sf_scd_t *scdp = sfmmup->sfmmu_scdp; in sfmmu_ism_hatflags()
15300 ASSERT(sfmmup->sfmmu_iblk != NULL); in sfmmu_ism_hatflags()
15306 ASSERT(sfmmu_hat_lock_held(sfmmup)); in sfmmu_ism_hatflags()
15308 ism_blkp = sfmmup->sfmmu_iblk; in sfmmu_ism_hatflags()