Lines Matching defs:sfmmup
318 * The lock is hashed on the sfmmup since the case where we need to lock
343 #define TSB_HASH(sfmmup) \
345 &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)])
347 #define TSB_HASH(sfmmup) &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]
475 #define sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \
476 sfmmu_mod_tsb(sfmmup, vaddr, tte, szc)
477 #define sfmmu_unload_tsb(sfmmup, vaddr, szc) \
478 sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc)
837 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) { \
838 (tsbinfop) = (sfmmup)->sfmmu_tsb; \
840 sfmmu_hat_lock_held(sfmmup)); \
852 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, ismhat) \
856 sfmmu_unload_tsb(sfmmup, addr, ttesz); \
862 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); \
1459 sfmmu_t *sfmmup;
1465 sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP);
1466 sfmmup->sfmmu_as = as;
1467 sfmmup->sfmmu_flags = 0;
1468 sfmmup->sfmmu_tteflags = 0;
1469 sfmmup->sfmmu_rtteflags = 0;
1470 LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock);
1473 ksfmmup = sfmmup;
1474 sfmmup->sfmmu_cext = 0;
1477 sfmmup->sfmmu_clrstart = 0;
1478 sfmmup->sfmmu_tsb = NULL;
1492 sfmmup->sfmmu_cext = 0;
1496 sfmmup->sfmmu_clrstart = get_color_start(as);
1503 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size,
1504 TSB8K|TSB64K|TSB512K, 0, sfmmup);
1507 (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb,
1509 TSB8K|TSB64K|TSB512K, 0, sfmmup);
1510 sfmmup->sfmmu_flags = HAT_SWAPPED | HAT_ALLCTX_INVALID;
1511 ASSERT(sfmmup->sfmmu_tsb != NULL);
1516 sfmmup->sfmmu_ctxs[i].cnum = cnum;
1517 sfmmup->sfmmu_ctxs[i].gnum = 0;
1521 sfmmup->sfmmu_ttecnt[i] = 0;
1522 sfmmup->sfmmu_scdrttecnt[i] = 0;
1523 sfmmup->sfmmu_ismttecnt[i] = 0;
1524 sfmmup->sfmmu_scdismttecnt[i] = 0;
1525 sfmmup->sfmmu_pgsz[i] = TTE8K;
1527 sfmmup->sfmmu_tsb0_4minflcnt = 0;
1528 sfmmup->sfmmu_iblk = NULL;
1529 sfmmup->sfmmu_ismhat = 0;
1530 sfmmup->sfmmu_scdhat = 0;
1531 sfmmup->sfmmu_ismblkpa = (uint64_t)-1;
1532 if (sfmmup == ksfmmup) {
1533 CPUSET_ALL(sfmmup->sfmmu_cpusran);
1535 CPUSET_ZERO(sfmmup->sfmmu_cpusran);
1537 sfmmup->sfmmu_free = 0;
1538 sfmmup->sfmmu_rmstat = 0;
1539 sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart;
1540 cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL);
1541 sfmmup->sfmmu_srdp = NULL;
1542 SF_RGNMAP_ZERO(sfmmup->sfmmu_region_map);
1543 bzero(sfmmup->sfmmu_hmeregion_links, SFMMU_L1_HMERLINKS_SIZE);
1544 sfmmup->sfmmu_scdp = NULL;
1545 sfmmup->sfmmu_scd_link.next = NULL;
1546 sfmmup->sfmmu_scd_link.prev = NULL;
1547 return (sfmmup);
1865 hat_setup(struct hat *sfmmup, int allocflag)
1876 hatlockp = sfmmu_hat_enter(sfmmup);
1884 sfmmu_tsb_swapin(sfmmup, hatlockp);
1886 sfmmu_get_ctx(sfmmup);
1892 hatlockp = sfmmu_hat_enter(sfmmup);
1895 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id);
1915 hat_free_start(struct hat *sfmmup)
1917 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
1918 ASSERT(sfmmup != ksfmmup);
1920 sfmmup->sfmmu_free = 1;
1921 if (sfmmup->sfmmu_scdp != NULL) {
1922 sfmmu_leave_scd(sfmmup, 0);
1925 ASSERT(sfmmup->sfmmu_scdp == NULL);
1929 hat_free_end(struct hat *sfmmup)
1933 ASSERT(sfmmup->sfmmu_free == 1);
1934 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0);
1935 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0);
1936 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0);
1937 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0);
1938 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
1939 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
1941 if (sfmmup->sfmmu_rmstat) {
1942 hat_freestat(sfmmup->sfmmu_as, NULL);
1945 while (sfmmup->sfmmu_tsb != NULL) {
1946 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next;
1947 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb);
1948 sfmmup->sfmmu_tsb = next;
1951 if (sfmmup->sfmmu_srdp != NULL) {
1952 sfmmu_leave_srd(sfmmup);
1953 ASSERT(sfmmup->sfmmu_srdp == NULL);
1955 if (sfmmup->sfmmu_hmeregion_links[i] != NULL) {
1956 kmem_free(sfmmup->sfmmu_hmeregion_links[i],
1958 sfmmup->sfmmu_hmeregion_links[i] = NULL;
1962 sfmmu_free_sfmmu(sfmmup);
1966 ASSERT(sfmmup->sfmmu_hmeregion_links[i] == NULL);
1970 kmem_cache_free(sfmmuid_cache, sfmmup);
1989 hat_swapout(struct hat *sfmmup)
2020 ASSERT(sfmmup != KHATID);
2028 if ((hmeblkp->hblk_tag.htag_id == sfmmup) &&
2031 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
2054 hatlockp = sfmmu_hat_enter(sfmmup);
2056 sfmmu_invalidate_ctx(sfmmup);
2072 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
2077 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED);
2079 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
2673 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp,
2676 ASSERT(sfmmup == ksfmmup);
2677 (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags,
2686 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz)
2696 if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */
2709 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz);
2767 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz)
2786 if (sfmmup == ksfmmup) {
2799 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz);
2876 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr,
2893 hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size, rid);
2899 hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags,
2906 ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags,
2921 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size,
2926 void *htagid = sfmmutohtagid(sfmmup, rid);
2941 * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is
2945 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp,
2952 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
2954 hblktag.htag_id = sfmmutohtagid(sfmmup, rid);
2981 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size,
3006 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp);
3038 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep,
3110 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
3159 sfmmu_vac_conflict(sfmmup, vaddr, pp);
3221 if ((sfmmup == KHATID) &&
3235 atomic_inc_ulong(&sfmmup->sfmmu_ttecnt[size]);
3237 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
3248 myflt = (astosfmmu(curthread->t_procp->p_as) == sfmmup);
3250 sfmmup != ksfmmup) {
3253 if (!(sfmmup->sfmmu_tteflags & tteflag)) {
3254 hatlockp = sfmmu_hat_enter(sfmmup);
3255 sfmmup->sfmmu_tteflags |= tteflag;
3258 } else if (!(sfmmup->sfmmu_rtteflags & tteflag)) {
3259 hatlockp = sfmmu_hat_enter(sfmmup);
3260 sfmmup->sfmmu_rtteflags |= tteflag;
3288 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) {
3289 hatlockp = sfmmu_hat_enter(sfmmup);
3290 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG);
3308 sfmmu_ttesync(sfmmup, vaddr, &tteold, pp);
3318 sfmmup->sfmmu_srdp->srd_hmergnp[rid], hmeblkp, 1);
3322 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0);
3323 xt_sync(sfmmup->sfmmu_cpusran);
3335 hatlockp = sfmmu_hat_enter(sfmmup);
3340 scdp = sfmmup->sfmmu_scdp;
3343 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte,
3588 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags)
3596 ASSERT(sfmmup != KHATID);
3601 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
3602 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
3611 hblktag.htag_id = sfmmup;
3616 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift);
3623 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size,
3668 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
3690 sfmmu_free_hblks(sfmmup, addr, endaddr, hashno);
3696 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
3706 hblktag.htag_id = sfmmup;
3714 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
3726 sfmmu_shadow_hcleanup(sfmmup,
3933 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len)
3941 ASSERT(sfmmup != NULL);
3943 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
3946 hblktag.htag_id = sfmmup;
3960 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
3986 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
3993 "addr %p hat %p", (void *)addr, (void *)sfmmup);
4003 hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,
4020 hat_unlock(sfmmup, addr, len);
4024 ASSERT(sfmmup != NULL);
4025 ASSERT(sfmmup != ksfmmup);
4027 srdp = sfmmup->sfmmu_srdp;
4034 SFMMU_VALIDATE_HMERID(sfmmup, rid, addr, len);
4076 "addr %p hat %p", (void *)va, (void *)sfmmup);
4730 hat_probe(struct hat *sfmmup, caddr_t addr)
4735 ASSERT(sfmmup != NULL);
4737 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
4739 if (sfmmup == ksfmmup) {
4740 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte))
4742 sfmmu_vatopfn_suspended(addr, sfmmup, &tte);
4745 pfn = sfmmu_uvatopfn(addr, sfmmup, NULL);
4755 hat_getpagesize(struct hat *sfmmup, caddr_t addr)
4759 if (sfmmup == ksfmmup) {
4760 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4764 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4774 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr)
4778 if (sfmmup == ksfmmup) {
4779 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4783 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4833 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr,
4846 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
4857 hblktag.htag_id = sfmmup;
4859 DEMAP_RANGE_INIT(sfmmup, &dmr);
4865 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
4877 ASSERT(sfmmup != ksfmmup);
4882 addr = sfmmu_hblk_chgattr(sfmmup,
4891 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
4901 if (sfmmup == ksfmmup)
4915 cpuset = sfmmup->sfmmu_cpusran;
4929 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
4963 check_exec = (sfmmup != ksfmmup) &&
4964 AS_TYPE_64BIT(sfmmup->sfmmu_as) &&
5026 sfmmu_ttesync(sfmmup, addr, &tte, pp);
5040 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
5163 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot)
5176 ASSERT(sfmmup->sfmmu_as != NULL);
5186 hblktag.htag_id = sfmmup;
5188 DEMAP_RANGE_INIT(sfmmup, &dmr);
5194 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
5206 ASSERT(sfmmup != ksfmmup);
5211 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp,
5220 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
5229 if (sfmmup == ksfmmup)
5243 cpuset = sfmmup->sfmmu_cpusran;
5257 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
5289 check_exec = (sfmmup != ksfmmup) &&
5290 AS_TYPE_64BIT(sfmmup->sfmmu_as) &&
5354 sfmmu_ttesync(sfmmup, addr, &tte, pp);
5368 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
5448 hat_unload_large_virtual(struct hat *sfmmup, caddr_t startaddr, size_t len,
5467 if (sfmmup->sfmmu_free) {
5471 DEMAP_RANGE_INIT(sfmmup, dmrp);
5489 if (hmeblkp->hblk_tag.htag_id != sfmmup ||
5503 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
5538 cpuset = sfmmup->sfmmu_cpusran;
5559 cpuset = sfmmup->sfmmu_cpusran;
5572 if (!sfmmup->sfmmu_free)
5573 sfmmu_check_page_sizes(sfmmup, 0);
5582 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \
5587 hat_unload_callback(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags,
5600 int issegkmap = ISSEGKMAP(sfmmup, addr);
5603 ASSERT(sfmmup->sfmmu_as != NULL);
5605 ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \
5606 AS_LOCK_HELD(sfmmup->sfmmu_as));
5608 ASSERT(sfmmup != NULL);
5622 if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) {
5623 hat_unload_large_virtual(sfmmup, addr, len, flags, callback);
5633 if (sfmmup->sfmmu_free) {
5637 DEMAP_RANGE_INIT(sfmmup, dmrp);
5641 hblktag.htag_id = sfmmup;
5659 if (sfmmup == KHATID) {
5674 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
5776 ASSERT(sfmmup != KHATID);
5794 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr,
5814 cpuset = sfmmup->sfmmu_cpusran;
5855 cpuset = sfmmup->sfmmu_cpusran;
5869 if (!sfmmup->sfmmu_free)
5870 sfmmu_check_page_sizes(sfmmup, 0);
5878 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags)
5880 hat_unload_callback(sfmmup, addr, len, flags, NULL);
5910 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
5924 ASSERT(sfmmup != NULL || hmeblkp->hblk_shared);
5925 ASSERT(sfmmup == NULL || !hmeblkp->hblk_shared);
6011 sfmmu_ttesync(sfmmup, addr, &tte, pp);
6057 ASSERT(sfmmup != NULL);
6059 sfmmu_tlb_demap(addr, sfmmup, hmeblkp,
6060 sfmmup->sfmmu_free, 0);
6175 if (ttecnt > 0 && sfmmup != NULL) {
6176 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt);
6187 hat_flush_range(struct hat *sfmmup, caddr_t va, size_t size)
6193 sz = hat_getpagesize(sfmmup, va);
6198 vtag_flushpage(va, (uint64_t)sfmmup);
6210 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)
6219 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
6227 hblktag.htag_id = sfmmup;
6242 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
6254 ASSERT(sfmmup != ksfmmup);
6259 addr = sfmmu_hblk_sync(sfmmup, hmeblkp,
6268 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
6277 if (sfmmup == ksfmmup)
6289 cpuset = sfmmup->sfmmu_cpusran;
6294 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
6345 sfmmu_tlb_demap(addr, sfmmup,
6349 sfmmu_ttesync(sfmmup, addr, &tte, pp);
6367 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp)
6391 if (sfmmup != NULL && sfmmup->sfmmu_rmstat) {
6396 hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm);
6604 sfmmu_t *sfmmup;
6655 sfmmup = hblktosfmmu(hmeblkp);
6656 ASSERT(sfmmup == ksfmmup);
6666 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
7143 sfmmu_t *sfmmup;
7164 sfmmup = hblktosfmmu(hmeblkp);
7200 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7213 sfmmu_ttesync(sfmmup, addr, &tte, pp);
7214 atomic_dec_ulong(&sfmmup->sfmmu_ttecnt[ttesz]);
7234 if (sfmmup->sfmmu_ismhat) {
7243 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp,
7250 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
7251 cpuset = sfmmup->sfmmu_cpusran;
7487 sfmmu_t *sfmmup;
7504 sfmmup = hblktosfmmu(hmeblkp);
7522 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7535 sfmmu_tlb_demap(addr, sfmmup, hmeblkp,
7537 cpuset = sfmmup->sfmmu_cpusran;
7541 sfmmu_ttesync(hmeblkp->hblk_shared ? NULL : sfmmup, addr,
7561 sfmmu_t *sfmmup;
7575 sfmmup = hblktosfmmu(hmeblkp);
7593 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7605 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
7606 cpuset = sfmmup->sfmmu_cpusran;
7865 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup, tte_t *ttep)
7882 sfmmu_t *sv_sfmmup = sfmmup;
7892 ASSERT(sfmmup != ksfmmup);
7897 ism_blkp = sfmmup->sfmmu_iblk;
7899 sfmmu_ismhat_enter(sfmmup, 0);
7900 locked_hatid = sfmmup;
7907 sfmmup = ism_hatid = ism_map[i].imap_ismhat;
7919 hblktag.htag_id = sfmmup;
7925 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift);
7943 } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt));
8295 ism_tsb_entries(sfmmu_t *sfmmup, int szc)
8297 ism_blk_t *ism_blkp = sfmmup->sfmmu_iblk;
8305 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
8306 scdp = sfmmup->sfmmu_scdp;
8313 rid < sfmmup->sfmmu_srdp->srd_next_ismrid);
8317 /* ISM is in sfmmup's SCD */
8327 sfmmup->sfmmu_ismttecnt[szc] = npgs;
8328 sfmmup->sfmmu_scdismttecnt[szc] = npgs_scd;
8450 hat_share(struct hat *sfmmup, caddr_t addr, struct hat *ism_hatid,
8472 ASSERT(ism_hatid != NULL && sfmmup != NULL);
8499 sfmmu_ismhat_enter(sfmmup, 0);
8504 if (sfmmup->sfmmu_iblk == NULL) {
8505 sfmmup->sfmmu_iblk = new_iblk;
8509 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk);
8518 ism_blkp = sfmmup->sfmmu_iblk;
8544 ism_blkp = sfmmup->sfmmu_iblk;
8568 ism_ment->iment_hat = sfmmup;
8592 * After calling hat_join_region, sfmmup may join a new SCD or
8594 * shrink the sfmmup's private tsb size, i.e., pass shrink to
8597 old_scdp = sfmmup->sfmmu_scdp;
8599 rcookie = hat_join_region(sfmmup, addr, len, (void *)ism_hatid, 0,
8605 * Update our counters for this sfmmup's ism mappings.
8609 (void) ism_tsb_entries(sfmmup, i);
8624 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_ISM)) {
8625 hatlockp = sfmmu_hat_enter(sfmmup);
8626 SFMMU_FLAGS_SET(sfmmup, HAT_256M_ISM);
8631 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_ISM)) {
8632 hatlockp = sfmmu_hat_enter(sfmmup);
8633 SFMMU_FLAGS_SET(sfmmup, HAT_32M_ISM);
8648 hatlockp = sfmmu_hat_enter(sfmmup);
8649 sfmmu_sync_mmustate(sfmmup);
8653 sfmmu_ismhat_exit(sfmmup, 0);
8664 if (sfmmup->sfmmu_scdp != NULL && old_scdp != sfmmup->sfmmu_scdp) {
8665 sfmmu_check_page_sizes(sfmmup, 0);
8667 sfmmu_check_page_sizes(sfmmup, 1);
8678 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc)
8694 ASSERT(sfmmup != NULL);
8695 ASSERT(sfmmup != ksfmmup);
8697 ASSERT(sfmmup->sfmmu_as != NULL);
8707 sfmmu_ismhat_enter(sfmmup, 0);
8722 ism_blkp = sfmmup->sfmmu_iblk;
8743 * After hat_leave_region, the sfmmup may leave SCD,
8747 old_scdp = sfmmup->sfmmu_scdp;
8752 hat_leave_region(sfmmup, (void *)((uint64_t)ism_rid),
8761 hatlockp = sfmmu_hat_enter(sfmmup);
8762 sfmmu_invalidate_ctx(sfmmup);
8809 if (!sfmmup->sfmmu_free) {
8810 hatlockp = sfmmu_hat_enter(sfmmup);
8811 for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL;
8829 * Update our counters for this sfmmup's ism mappings.
8833 (void) ism_tsb_entries(sfmmup, i);
8836 sfmmu_ismhat_exit(sfmmup, 0);
8849 if (!sfmmup->sfmmu_free) {
8850 if (found && old_scdp != NULL && sfmmup->sfmmu_scdp == NULL) {
8851 sfmmu_check_page_sizes(sfmmup, 1);
8853 sfmmu_check_page_sizes(sfmmup, 0);
9456 sfmmu_t *sfmmup;
9499 sfmmup = hblktosfmmu(hmeblkp);
9505 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
9517 } else if (sfmmup->sfmmu_ismhat) {
9523 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp,
9526 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp,
9540 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
9551 } else if (sfmmup->sfmmu_ismhat) {
9557 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp,
9560 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1);
9604 sfmmu_get_ctx(sfmmu_t *sfmmup)
9610 ASSERT(sfmmu_hat_lock_held(sfmmup));
9611 ASSERT(sfmmup != ksfmmup);
9613 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID)) {
9614 sfmmu_setup_tsbinfo(sfmmup);
9615 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ALLCTX_INVALID);
9635 if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) {
9636 mmu_set_ctx_page_sizes(sfmmup);
9649 if (sfmmu_alloc_ctx(sfmmup, 1, CPU, SFMMU_PRIVATE) &&
9650 sfmmup->sfmmu_scdp != NULL) {
9651 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
9658 sfmmu_load_mmustate(sfmmup);
9757 sfmmu_sync_mmustate(sfmmu_t *sfmmup)
9762 ASSERT(sfmmup != ksfmmup);
9763 ASSERT(sfmmu_hat_lock_held(sfmmup));
9768 * We check whether the pass'ed-in sfmmup is the same as the
9772 if ((sfmmup == curthread->t_procp->p_as->a_hat) &&
9775 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum;
9788 sfmmu_load_mmustate(sfmmup);
9795 * or when sfmmup is not the same as the curproc.
9797 sfmmu_invalidate_ctx(sfmmup);
9820 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc,
9828 ASSERT(sfmmup != ksfmmup);
9829 ASSERT(sfmmup->sfmmu_ismhat == 0);
9830 ASSERT(sfmmu_hat_lock_held(sfmmup));
9833 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY))
9841 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb;
9847 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
9856 SFMMU_FLAGS_SET(sfmmup, HAT_BUSY);
9872 tte_sz_mask, flags, sfmmup) &&
9877 tte_sz_mask, flags, sfmmup))) {
9878 (void) sfmmu_hat_enter(sfmmup);
9881 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
9884 (void) sfmmu_hat_enter(sfmmup);
9892 if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
9894 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
9897 (void) sfmmu_hat_enter(sfmmup);
9903 for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb;
9928 sfmmu_invalidate_ctx(sfmmup);
9937 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup);
9948 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT);
9957 sfmmup->sfmmu_tsb = new_tsbinfo;
9967 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
9982 (void) sfmmu_hat_enter(sfmmup);
9993 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz)
9998 hatlockp = sfmmu_hat_enter(sfmmup);
10002 sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i];
10006 sfmmu_invalidate_ctx(sfmmup);
10078 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing)
10089 if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL)
10092 if (!SFMMU_LGPGS_INUSE(sfmmup) &&
10093 sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor)
10097 ttecnt[i] = sfmmup->sfmmu_ttecnt[i] +
10098 sfmmup->sfmmu_ismttecnt[i];
10103 mmu_check_page_sizes(sfmmup, ttecnt);
10123 tte8k_cnt += sfmmup->sfmmu_tsb0_4minflcnt;
10140 if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) {
10154 sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh);
10158 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt,
10166 hatlockp = sfmmu_hat_enter(sfmmup);
10168 tsbinfop = sfmmup->sfmmu_tsb;
10183 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc,
10186 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc,
10189 tsbinfop = sfmmup->sfmmu_tsb;
10214 int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)?
10228 allocflags, sfmmup)) &&
10231 tsb_bits, allocflags, sfmmup)) &&
10233 tsb_bits, allocflags, sfmmup)) {
10237 hatlockp = sfmmu_hat_enter(sfmmup);
10239 sfmmu_invalidate_ctx(sfmmup);
10241 if (sfmmup->sfmmu_tsb->tsb_next == NULL) {
10242 sfmmup->sfmmu_tsb->tsb_next = newtsb;
10277 (void) sfmmu_replace_tsb(sfmmup, tsbinfop,
10281 (void) sfmmu_replace_tsb(sfmmup, tsbinfop,
10295 sfmmu_free_sfmmu(sfmmu_t *sfmmup)
10303 ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0);
10304 ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0);
10305 ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0);
10306 ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0);
10307 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
10308 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
10309 ASSERT(SF_RGNMAP_ISNULL(sfmmup));
10311 sfmmup->sfmmu_free = 0;
10312 sfmmup->sfmmu_ismhat = 0;
10314 blkp = sfmmup->sfmmu_iblk;
10315 sfmmup->sfmmu_iblk = NULL;
10801 sfmmu_hat_enter(sfmmu_t *sfmmup)
10805 if (sfmmup != ksfmmup) {
10806 hatlockp = TSB_HASH(sfmmup);
10814 sfmmu_hat_tryenter(sfmmu_t *sfmmup)
10818 if (sfmmup != ksfmmup) {
10819 hatlockp = TSB_HASH(sfmmup);
10851 sfmmu_hat_lock_held(sfmmu_t *sfmmup)
10853 ASSERT(sfmmup != ksfmmup);
10854 return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup))));
10869 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held)
10874 hatlockp = sfmmu_hat_enter(sfmmup);
10875 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY))
10876 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp));
10877 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY);
10883 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held)
10888 hatlockp = sfmmu_hat_enter(sfmmup);
10889 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
10890 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY);
10891 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
10921 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr,
10939 SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
10989 if (sfmmup != KHATID && !SFMMU_IS_SHMERID_VALID(rid)) {
10992 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr,
10996 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr,
11011 ASSERT(sfmmup != KHATID);
11013 srdp = sfmmup->sfmmu_srdp;
11064 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP);
11086 if (size == TTE8K && sfmmup != KHATID) {
11151 forcefree = (sfmmup == KHATID) ? 1 : 0;
11156 ASSERT(sfmmup != KHATID);
11444 sfmmu_t *sfmmup;
11447 sfmmup = hblktosfmmu(hmeblkp);
11448 if (hmeblkp->hblk_shared || sfmmup->sfmmu_ismhat) {
11451 DEMAP_RANGE_INIT(sfmmup, &dmr);
11452 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
11548 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp)
11554 ASSERT(sfmmup != ksfmmup);
11555 ASSERT(sfmmu_hat_lock_held(sfmmup));
11557 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) {
11558 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp));
11561 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
11562 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN);
11567 ASSERT(sfmmup->sfmmu_tsb != NULL);
11573 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) {
11576 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc,
11586 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN);
11587 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11605 tsbinfop = sfmmup->sfmmu_tsb;
11614 hatlockp = sfmmu_hat_enter(sfmmup);
11621 tsbinfop = sfmmup->sfmmu_tsb;
11622 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE,
11627 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN);
11628 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11687 sfmmu_t *sfmmup, *shsfmmup;
11698 sfmmup = astosfmmu(curthread->t_procp->p_as);
11705 ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT);
11706 ASSERT(sfmmup->sfmmu_ismhat == 0);
11707 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) ||
11731 hatlockp = sfmmu_hat_enter(sfmmup);
11734 ASSERT(sfmmup == tsbmp->usfmmup);
11735 if (((tsbmp->uhat_tteflags ^ sfmmup->sfmmu_tteflags) &
11737 ((tsbmp->uhat_rtteflags ^ sfmmup->sfmmu_rtteflags) &
11739 tsbmp->uhat_tteflags = sfmmup->sfmmu_tteflags;
11740 tsbmp->uhat_rtteflags = sfmmup->sfmmu_rtteflags;
11743 if (sfmmup->sfmmu_srdp != NULL) {
11744 ulong_t *sm = sfmmup->sfmmu_hmeregion_map.bitmap;
11752 sfmmup->sfmmu_srdp,
11789 hatlockp = sfmmu_hat_enter(sfmmup);
11791 if ((scdp = sfmmup->sfmmu_scdp) != NULL) {
11806 if (sfmmup->sfmmu_scdp == scdp &&
11812 hatlockp = sfmmu_hat_enter(sfmmup);
11818 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
11821 cv_wait(&sfmmup->sfmmu_tsb_cv,
11830 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) {
11831 cv_wait(&sfmmup->sfmmu_tsb_cv,
11837 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
11843 sfmmu_finish_join_scd(sfmmup);
11844 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD);
11853 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
11854 sfmmu_tsb_swapin(sfmmup, hatlockp);
11858 sfmmu_get_ctx(sfmmup);
11873 sfmmu_tsb_chk_reloc(sfmmu_t *sfmmup, hatlock_t *hatlockp)
11877 ASSERT(sfmmu_hat_lock_held(sfmmup));
11879 for (tp = sfmmup->sfmmu_tsb; tp != NULL; tp = tp->tsb_next) {
11881 cv_wait(&sfmmup->sfmmu_tsb_cv,
11939 sfmmu_t *sfmmup;
11955 sfmmup = rgnp->rgn_sfmmu_head;
11956 while (sfmmup != NULL) {
11958 hatlockp = sfmmu_hat_enter(sfmmup);
11969 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL &&
11970 !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
11971 scdp = sfmmup->sfmmu_scdp;
11980 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
11983 cpuset = sfmmup->sfmmu_cpusran;
11986 SFMMU_XCALL_STATS(sfmmup);
11988 (uint64_t)addr, (uint64_t)sfmmup);
11989 vtag_flushpage(addr, (uint64_t)sfmmup);
11998 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0);
12000 sfmmup = rlink->next;
12015 find_ism_rid(sfmmu_t *sfmmup, sfmmu_t *ism_sfmmup, caddr_t va, uint_t *ism_rid)
12023 ASSERT(sfmmu_hat_lock_held(sfmmup));
12025 ism_blkp = sfmmup->sfmmu_iblk;
12060 sfmmu_t *sfmmup;
12085 sfmmup = ment->iment_hat;
12099 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL &&
12100 !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD) &&
12101 !SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) {
12102 if (!find_ism_rid(sfmmup, ism_sfmmup, va,
12108 scdp = sfmmup->sfmmu_scdp;
12115 SFMMU_UNLOAD_TSB(va, sfmmup, hmeblkp, 1);
12117 cpuset = sfmmup->sfmmu_cpusran;
12120 SFMMU_XCALL_STATS(sfmmup);
12122 (uint64_t)sfmmup);
12123 vtag_flushpage(va, (uint64_t)sfmmup);
12135 SFMMU_XCALL_STATS(sfmmup);
12151 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
12187 hatlockp = sfmmu_hat_enter(sfmmup);
12194 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12196 cpuset = sfmmup->sfmmu_cpusran;
12200 SFMMU_XCALL_STATS(sfmmup);
12203 (uint64_t)sfmmup);
12205 vtag_flushpage(addr, (uint64_t)sfmmup);
12222 cpuset = sfmmup->sfmmu_cpusran;
12226 SFMMU_XCALL_STATS(sfmmup);
12240 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
12258 hatlockp = sfmmu_hat_enter(sfmmup);
12259 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12263 cpuset = sfmmup->sfmmu_cpusran;
12267 SFMMU_XCALL_STATS(sfmmup);
12268 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup);
12270 vtag_flushpage(addr, (uint64_t)sfmmup);
12291 sfmmu_t *sfmmup = dmrp->dmr_sfmmup;
12324 hatlockp = sfmmu_hat_enter(sfmmup);
12325 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K);
12333 if (sfmmup->sfmmu_free == 0) {
12346 * the low 6 bits of sfmmup. This is doable since pgcnt
12349 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK));
12350 sfmmu_pgcnt = (uint64_t)sfmmup |
12362 hatlockp = sfmmu_hat_enter(sfmmup);
12365 cpuset = sfmmup->sfmmu_cpusran;
12369 SFMMU_XCALL_STATS(sfmmup);
12375 vtag_flushpage(addr, (uint64_t)sfmmup);
12410 sfmmu_invalidate_ctx(sfmmu_t *sfmmup)
12420 ASSERT(sfmmu_hat_lock_held(sfmmup));
12421 ASSERT(sfmmup != ksfmmup);
12430 currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum;
12434 lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */
12438 cnum = sfmmup->sfmmu_ctxs[i].cnum;
12443 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT;
12446 lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */
12450 cpuset = sfmmup->sfmmu_cpusran;
12454 SFMMU_XCALL_STATS(sfmmup);
12456 (uint64_t)sfmmup, INVALID_CONTEXT);
12474 SFMMU_FLAGS_SET(sfmmup, HAT_ALLCTX_INVALID);
12482 ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT);
12538 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu;
12546 * If tsb is a shared TSB with TSB_SHAREDCTX set, sfmmup must
12548 * If tsb is not shared, sfmmup is a private hat, then set
12551 hatlockp = sfmmu_hat_enter(sfmmup);
12555 sfmmu_tsb_inv_ctx(sfmmup);
12560 /* sfmmup is a shared hat */
12561 ASSERT(sfmmup->sfmmu_scdhat);
12562 scdp = sfmmup->sfmmu_scdp;
12566 sfmmup = scdp->scd_sf_list;
12567 while (sfmmup != NULL) {
12568 hatlockp = sfmmu_hat_enter(sfmmup);
12574 sfmmu_invalidate_ctx(sfmmup);
12576 sfmmup = sfmmup->sfmmu_scd_link.next;
12585 sfmmu_tsb_inv_ctx(sfmmu_t *sfmmup)
12589 ASSERT(sfmmu_hat_lock_held(sfmmup));
12621 sfmmu_invalidate_ctx(sfmmup);
12631 sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu;
12636 hatlockp = sfmmu_hat_enter(sfmmup);
12658 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
12671 uint_t flags, sfmmu_t *sfmmup)
12679 tsb_szc, flags, sfmmup)) != 0) {
12795 int tsbcode, uint_t flags, sfmmu_t *sfmmup)
12821 tsbinfo->tsb_sfmmu = sfmmup;
13596 hat_join_srd(struct hat *sfmmup, vnode_t *evp)
13602 ASSERT(sfmmup != ksfmmup);
13603 ASSERT(sfmmup->sfmmu_srdp == NULL);
13617 sfmmup->sfmmu_srdp = srdp;
13639 sfmmup->sfmmu_srdp = srdp;
13648 sfmmup->sfmmu_srdp = newsrdp;
13655 sfmmu_leave_srd(sfmmu_t *sfmmup)
13658 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
13668 ASSERT(sfmmup != ksfmmup);
13671 ASSERT(sfmmup->sfmmu_scdp == NULL);
13672 ASSERT(sfmmup->sfmmu_free == 1);
13674 sfmmup->sfmmu_srdp = NULL;
13803 hat_join_region(struct hat *sfmmup, caddr_t r_saddr, size_t r_size,
13807 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
13828 ASSERT(sfmmup != ksfmmup);
13829 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
13894 (sfmmup == astosfmmu(curthread->t_procp->p_as));
13896 sfmmu_link_to_hmeregion(sfmmup, rgnp);
13921 if (tteflag && !(sfmmup->sfmmu_rtteflags & tteflag)) {
13922 hatlockp = sfmmu_hat_enter(sfmmup);
13923 sfmmup->sfmmu_rtteflags |= tteflag;
13926 hatlockp = sfmmu_hat_enter(sfmmup);
13933 sfmmup->sfmmu_tsb0_4minflcnt +=
13939 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc],
13945 !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) {
13946 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG);
13957 sfmmu_check_page_sizes(sfmmup, 1);
13958 hatlockp = sfmmu_hat_enter(sfmmup);
13959 SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid);
13968 (sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum
13973 ASSERT(sfmmup == tsbmp->usfmmup);
13986 hatlockp = sfmmu_hat_enter(sfmmup);
13987 SF_RGNMAP_ADD(sfmmup->sfmmu_ismregion_map, rid);
13993 sfmmu_find_scd(sfmmup);
14091 hat_leave_region(struct hat *sfmmup, hat_region_cookie_t rcookie, uint_t flags)
14093 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14109 ASSERT(sfmmup != ksfmmup);
14114 ASSERT(!sfmmup->sfmmu_free || sfmmup->sfmmu_scdp == NULL);
14132 ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as));
14134 if (sfmmup->sfmmu_free) {
14139 ASSERT(sfmmup->sfmmu_scdp == NULL);
14141 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid);
14145 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt);
14147 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc],
14150 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid);
14153 hatlockp = sfmmu_hat_enter(sfmmup);
14155 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid);
14156 scdp = sfmmup->sfmmu_scdp;
14159 sfmmu_leave_scd(sfmmup, r_type);
14160 ASSERT(sfmmu_hat_lock_held(sfmmup));
14171 hatlockp = sfmmu_hat_enter(sfmmup);
14173 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid);
14182 scdp = sfmmup->sfmmu_scdp;
14185 sfmmu_leave_scd(sfmmup, r_type);
14186 ASSERT(sfmmu_hat_lock_held(sfmmup));
14188 sfmmu_invalidate_ctx(sfmmup);
14193 sfmmu_unload_tsb_range(sfmmup, r_saddr,
14207 ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >=
14209 sfmmup->sfmmu_tsb0_4minflcnt -= rttecnt;
14214 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt);
14215 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc], -rttecnt);
14218 if (scdp != NULL && sfmmup->sfmmu_scdp == NULL) {
14219 /* sfmmup left the scd, grow private tsb */
14220 sfmmu_check_page_sizes(sfmmup, 1);
14222 sfmmu_check_page_sizes(sfmmup, 0);
14227 sfmmu_unlink_from_hmeregion(sfmmup, rgnp);
14293 hat_dup_region(struct hat *sfmmup, hat_region_cookie_t rcookie)
14295 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14302 ASSERT(sfmmup != ksfmmup);
14319 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 0);
14329 hrlink->prev = sfmmup;
14335 rgnp->rgn_sfmmu_head = sfmmup;
14340 atomic_add_long(&sfmmup->sfmmu_ttecnt[rgnp->rgn_pgszc], rttecnt);
14343 sfmmup->sfmmu_tsb0_4minflcnt +=
14350 SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid);
14401 check_scd_sfmmu_list(sfmmu_t **headp, sfmmu_t *sfmmup, int onlist)
14404 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14408 if (sp == sfmmup) {
14413 "list 0x%p", (void *)sfmmup,
14420 (void *)sfmmup, (void *)*headp);
14426 #define check_scd_sfmmu_list(headp, sfmmup, onlist)
14433 sfmmu_from_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup)
14435 ASSERT(sfmmup->sfmmu_srdp != NULL);
14436 check_scd_sfmmu_list(headp, sfmmup, 1);
14437 if (sfmmup->sfmmu_scd_link.prev != NULL) {
14438 ASSERT(*headp != sfmmup);
14439 sfmmup->sfmmu_scd_link.prev->sfmmu_scd_link.next =
14440 sfmmup->sfmmu_scd_link.next;
14442 ASSERT(*headp == sfmmup);
14443 *headp = sfmmup->sfmmu_scd_link.next;
14445 if (sfmmup->sfmmu_scd_link.next != NULL) {
14446 sfmmup->sfmmu_scd_link.next->sfmmu_scd_link.prev =
14447 sfmmup->sfmmu_scd_link.prev;
14456 sfmmu_to_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup)
14458 check_scd_sfmmu_list(headp, sfmmup, 0);
14459 sfmmup->sfmmu_scd_link.prev = NULL;
14460 sfmmup->sfmmu_scd_link.next = *headp;
14462 (*headp)->sfmmu_scd_link.prev = sfmmup;
14463 *headp = sfmmup;
14632 sfmmu_link_to_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14643 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 1);
14654 rgnp->rgn_sfmmu_head = sfmmup;
14662 hrlink->prev = sfmmup;
14668 rgnp->rgn_sfmmu_head = sfmmup;
14677 sfmmu_unlink_from_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14686 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0);
14689 if (rgnp->rgn_sfmmu_head == sfmmup) {
14706 ASSERT(nrlink->prev == sfmmup);
14718 ASSERT(prlink->next == sfmmup);
14734 ASSERT(nrlink->prev == sfmmup);
14912 sfmmu_join_scd(sf_scd_t *scdp, sfmmu_t *sfmmup)
14915 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14922 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
14924 if ((old_scdp = sfmmup->sfmmu_scdp) != NULL) {
14928 sfmmu_from_scd_list(&old_scdp->scd_sf_list, sfmmup);
14931 * sfmmup leaves the old scd. Update sfmmu_ttecnt to
14936 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
14938 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
14939 sfmmup->sfmmu_scdrttecnt[i]);
14947 sfmmu_to_scd_list(&scdp->scd_sf_list, sfmmup);
14951 hatlockp = sfmmu_hat_enter(sfmmup);
14957 SFMMU_FLAGS_SET(sfmmup, HAT_JOIN_SCD);
14959 sfmmu_invalidate_ctx(sfmmup);
14960 sfmmup->sfmmu_scdp = scdp;
14963 * Copy scd_rttecnt into sfmmup's sfmmu_scdrttecnt, and update
14967 sfmmup->sfmmu_scdrttecnt[i] = scdp->scd_rttecnt[i];
14968 ASSERT(sfmmup->sfmmu_ttecnt[i] >= scdp->scd_rttecnt[i]);
14969 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
14970 -sfmmup->sfmmu_scdrttecnt[i]);
14974 sfmmup->sfmmu_tsb0_4minflcnt +=
14977 ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >=
14979 sfmmup->sfmmu_tsb0_4minflcnt -= scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
14995 sfmmu_finish_join_scd(sfmmu_t *sfmmup)
14999 ASSERT(sfmmu_hat_lock_held(sfmmup));
15000 ASSERT(sfmmup->sfmmu_scdp != NULL);
15001 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD));
15002 ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15003 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID));
15005 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
15017 sfmmu_ism_hatflags(sfmmup, 1);
15027 sfmmu_find_scd(sfmmu_t *sfmmup)
15029 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
15034 ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as));
15040 &sfmmup->sfmmu_region_map, ret);
15044 sfmmu_join_scd(scdp, sfmmup);
15056 &sfmmup->sfmmu_region_map, ret);
15068 if ((new_scdp = sfmmu_alloc_scd(srdp, &sfmmup->sfmmu_region_map)) ==
15089 sfmmu_join_scd(new_scdp, sfmmup);
15100 sfmmu_leave_scd(sfmmu_t *sfmmup, uchar_t r_type)
15102 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
15103 sf_srd_t *srdp = sfmmup->sfmmu_srdp;
15104 hatlock_t *hatlockp = TSB_HASH(sfmmup);
15110 if (sfmmup->sfmmu_free) {
15116 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup);
15123 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15125 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15126 sfmmup->sfmmu_scdrttecnt[i]);
15127 sfmmup->sfmmu_scdrttecnt[i] = 0;
15129 sfmmup->sfmmu_scdp = NULL;
15136 SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15138 ASSERT(!sfmmup->sfmmu_free);
15139 ASSERT(sfmmu_hat_lock_held(sfmmup));
15140 ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as));
15146 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY) &&
15147 sfmmup->sfmmu_scdp != NULL) {
15148 cv_wait(&sfmmup->sfmmu_tsb_cv,
15152 if (sfmmup->sfmmu_scdp == NULL) {
15156 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY);
15159 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
15160 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD);
15171 sfmmu_invalidate_ctx(sfmmup);
15175 ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15176 sfmmu_ism_hatflags(sfmmup, 0);
15180 * are in SCD before this sfmmup leaves the SCD.
15183 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15185 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15186 sfmmup->sfmmu_scdrttecnt[i]);
15187 sfmmup->sfmmu_scdrttecnt[i] = 0;
15189 sfmmup->sfmmu_ismttecnt[i] += sfmmup->sfmmu_scdismttecnt[i];
15190 sfmmup->sfmmu_scdismttecnt[i] = 0;
15193 sfmmup->sfmmu_tsb0_4minflcnt += scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
15196 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY);
15198 sfmmup->sfmmu_scdp = NULL;
15211 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup);
15216 hatlockp = sfmmu_hat_enter(sfmmup);
15293 sfmmu_ism_hatflags(sfmmu_t *sfmmup, int addflag)
15295 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
15300 ASSERT(sfmmup->sfmmu_iblk != NULL);
15306 ASSERT(sfmmu_hat_lock_held(sfmmup));
15308 ism_blkp = sfmmup->sfmmu_iblk;