Lines Matching defs:rgnp
113 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid) \
123 ASSERT((rgnp)->rgn_id == rid); \
124 ASSERT(!((rgnp)->rgn_flags & SFMMU_REGION_FREE)); \
125 ASSERT(((rgnp)->rgn_flags & SFMMU_REGION_TYPE_MASK) == \
127 ASSERT(_ttesz <= (rgnp)->rgn_pgszc); \
131 (uintptr_t)(rgnp)->rgn_saddr, HBLK_MIN_BYTES); \
133 (uintptr_t)((rgnp)->rgn_saddr + (rgnp)->rgn_size), \
141 ASSERT(rgnp->rgn_hmeflags & (0x1 << _flagtte)); \
146 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid)
3238 sf_region_t *rgnp = srdp->srd_hmergnp[rid];
3244 atomic_inc_ulong(&rgnp->rgn_ttecnt[size]);
3863 sfmmu_unload_hmeregion(sf_srd_t *srdp, sf_region_t *rgnp)
3865 int ttesz = rgnp->rgn_pgszc;
3866 size_t rsz = rgnp->rgn_size;
3867 caddr_t rsaddr = rgnp->rgn_saddr;
3871 uint_t rid = rgnp->rgn_id;
3889 if ((rcbfunc = rgnp->rgn_cb_function) == NULL) {
3896 if (!(rgnp->rgn_hmeflags & (1 << ttesz))) {
3908 rsaddr, rsz, rgnp->rgn_obj,
3909 rgnp->rgn_objoff);
3922 rsz, rgnp->rgn_obj,
3923 rgnp->rgn_objoff);
4007 sf_region_t *rgnp;
4033 rgnp = srdp->srd_hmergnp[rid];
4036 ASSERT(IS_P2ALIGNED(addr, TTEBYTES(rgnp->rgn_pgszc)));
4037 ASSERT(IS_P2ALIGNED(len, TTEBYTES(rgnp->rgn_pgszc)));
4038 if (rgnp->rgn_pgszc < HBLK_MIN_TTESZ) {
4041 ttesz = rgnp->rgn_pgszc;
4044 while (ttesz < rgnp->rgn_pgszc &&
4049 if (!(rgnp->rgn_hmeflags & (1 << ttesz))) {
7202 sf_region_t *rgnp;
7206 rgnp = srdp->srd_hmergnp[rid];
7207 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
7208 cpuset = sfmmu_rgntlb_demap(addr, rgnp, hmeblkp, 1);
7210 ASSERT(rgnp->rgn_ttecnt[ttesz] > 0);
7211 atomic_dec_ulong(&rgnp->rgn_ttecnt[ttesz]);
7424 sf_region_t *rgnp;
7428 rgnp = srdp->srd_hmergnp[rid];
7430 rgnp, rid);
7431 shcnt += rgnp->rgn_refcnt;
7525 sf_region_t *rgnp;
7529 rgnp = srdp->srd_hmergnp[rid];
7531 srdp, rgnp, rid);
7533 rgnp, hmeblkp, 1);
7595 sf_region_t *rgnp;
7599 rgnp = srdp->srd_hmergnp[rid];
7601 srdp, rgnp, rid);
7603 rgnp, hmeblkp, 1);
7963 sf_region_t *rgnp;
7975 rgnp = srdp->srd_hmergnp[rid];
7976 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
7979 rsaddr = rgnp->rgn_saddr;
7980 readdr = rsaddr + rgnp->rgn_size;
8121 sf_region_t *rgnp;
8125 rgnp = srdp->srd_hmergnp[rid];
8127 rgnp, rid);
8128 cnt += rgnp->rgn_refcnt;
9507 sf_region_t *rgnp;
9511 rgnp = srdp->srd_hmergnp[rid];
9513 srdp, rgnp, rid);
9514 (void) sfmmu_rgntlb_demap(vaddr, rgnp,
9542 sf_region_t *rgnp;
9546 rgnp = srdp->srd_hmergnp[rid];
9548 srdp, rgnp, rid);
9549 (void) sfmmu_rgntlb_demap(vaddr, rgnp,
10021 sf_region_t *rgnp;
10043 rgnp = srdp->srd_hmergnp[rid];
10044 ASSERT(rgnp->rgn_refcnt > 0);
10045 ASSERT(rgnp->rgn_id == rid);
10047 scdp->scd_rttecnt[rgnp->rgn_pgszc] +=
10048 rgnp->rgn_size >> TTE_PAGE_SHIFT(rgnp->rgn_pgszc);
10054 if (rgnp->rgn_pgszc >= TTE4M) {
10056 rgnp->rgn_size >>
10935 sf_region_t *rgnp;
11015 rgnp = srdp->srd_hmergnp[rid];
11016 ASSERT(rgnp != NULL && rgnp->rgn_id == rid);
11017 ASSERT(rgnp->rgn_refcnt != 0);
11018 ASSERT(size <= rgnp->rgn_pgszc);
11022 if (!(rgnp->rgn_hmeflags & (0x1 << ttesz))) {
11035 } while (++ttesz <= rgnp->rgn_pgszc);
11214 if (!(rgnp->rgn_hmeflags & tteflag)) {
11215 atomic_or_16(&rgnp->rgn_hmeflags, tteflag);
11501 sf_region_t *rgnp;
11509 rgnp = srdp->srd_hmergnp[rid];
11510 ASSERT(rgnp != NULL);
11511 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
11636 sf_region_t *rgnp;
11649 rgnp = srdp->srd_hmergnp[rid];
11650 ASSERT(rgnp->rgn_refcnt > 0);
11651 ASSERT(rgnp->rgn_id == rid);
11652 if (addr < rgnp->rgn_saddr ||
11653 addr >= (rgnp->rgn_saddr + rgnp->rgn_size)) {
11936 sfmmu_rgntlb_demap(caddr_t addr, sf_region_t *rgnp,
11943 uint_t rid = rgnp->rgn_id;
11953 mutex_enter(&rgnp->rgn_mutex);
11955 sfmmup = rgnp->rgn_sfmmu_head;
12003 mutex_exit(&rgnp->rgn_mutex);
13661 sf_region_t *rgnp;
13708 for (rgnp = srdp->srd_hmergnfree; rgnp != NULL; rgnp = nrgnp) {
13709 nrgnp = rgnp->rgn_next;
13710 ASSERT(rgnp->rgn_id < srdp->srd_next_hmerid);
13711 ASSERT(rgnp->rgn_refcnt == 0);
13712 ASSERT(rgnp->rgn_sfmmu_head == NULL);
13713 ASSERT(rgnp->rgn_flags & SFMMU_REGION_FREE);
13714 ASSERT(rgnp->rgn_hmeflags == 0);
13715 ASSERT(srdp->srd_hmergnp[rgnp->rgn_id] == rgnp);
13718 ASSERT(rgnp->rgn_ttecnt[i] == 0);
13722 kmem_cache_free(region_cache, rgnp);
13730 for (rgnp = srdp->srd_ismrgnfree; rgnp != NULL; rgnp = nrgnp) {
13731 nrgnp = rgnp->rgn_next;
13732 ASSERT(rgnp->rgn_id < srdp->srd_next_ismrid);
13733 ASSERT(rgnp->rgn_refcnt == 0);
13734 ASSERT(rgnp->rgn_sfmmu_head == NULL);
13735 ASSERT(rgnp->rgn_flags & SFMMU_REGION_FREE);
13736 ASSERT(srdp->srd_ismrgnp[rgnp->rgn_id] == rgnp);
13739 ASSERT(rgnp->rgn_ttecnt[i] == 0);
13743 kmem_cache_free(region_cache, rgnp);
13811 sf_region_t *rgnp;
13869 for (rgnp = srdp->srd_rgnhash[rhash]; rgnp != NULL;
13870 rgnp = rgnp->rgn_hash) {
13871 if (rgnp->rgn_saddr == r_saddr && rgnp->rgn_size == r_size &&
13872 rgnp->rgn_obj == r_obj && rgnp->rgn_objoff == r_objoff &&
13873 rgnp->rgn_perm == r_perm && rgnp->rgn_pgszc == r_pgszc) {
13879 if (rgnp != NULL) {
13880 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == r_type);
13881 ASSERT(rgnp->rgn_cb_function == r_cb_function);
13882 ASSERT(rgnp->rgn_refcnt >= 0);
13883 rid = rgnp->rgn_id;
13885 ASSERT(rarrp[rid] == rgnp);
13887 atomic_inc_32((volatile uint_t *)&rgnp->rgn_refcnt);
13896 sfmmu_link_to_hmeregion(sfmmup, rgnp);
14007 rgnp = *freelistp;
14008 *freelistp = rgnp->rgn_next;
14009 ASSERT(rgnp->rgn_id < *nextidp);
14010 ASSERT(rgnp->rgn_id < maxids);
14011 ASSERT(rgnp->rgn_flags & SFMMU_REGION_FREE);
14012 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK)
14014 ASSERT(rarrp[rgnp->rgn_id] == rgnp);
14015 ASSERT(rgnp->rgn_hmeflags == 0);
14025 for (rgnp = srdp->srd_rgnhash[rhash]; rgnp != NULL;
14026 rgnp = rgnp->rgn_hash) {
14027 if (rgnp->rgn_saddr == r_saddr &&
14028 rgnp->rgn_size == r_size &&
14029 rgnp->rgn_obj == r_obj &&
14030 rgnp->rgn_objoff == r_objoff &&
14031 rgnp->rgn_perm == r_perm &&
14032 rgnp->rgn_pgszc == r_pgszc) {
14036 if (rgnp != NULL) {
14044 rgnp = new_rgnp;
14046 rgnp->rgn_id = (*nextidp)++;
14047 ASSERT(rgnp->rgn_id < maxids);
14048 ASSERT(rarrp[rgnp->rgn_id] == NULL);
14049 rarrp[rgnp->rgn_id] = rgnp;
14052 ASSERT(rgnp->rgn_sfmmu_head == NULL);
14053 ASSERT(rgnp->rgn_hmeflags == 0);
14056 ASSERT(rgnp->rgn_ttecnt[i] == 0);
14059 rgnp->rgn_saddr = r_saddr;
14060 rgnp->rgn_size = r_size;
14061 rgnp->rgn_obj = r_obj;
14062 rgnp->rgn_objoff = r_objoff;
14063 rgnp->rgn_perm = r_perm;
14064 rgnp->rgn_pgszc = r_pgszc;
14065 rgnp->rgn_flags = r_type;
14066 rgnp->rgn_refcnt = 0;
14067 rgnp->rgn_cb_function = r_cb_function;
14068 rgnp->rgn_hash = srdp->srd_rgnhash[rhash];
14069 srdp->srd_rgnhash[rhash] = rgnp;
14098 sf_region_t *rgnp;
14122 rgnp = srdp->srd_ismrgnp[rid];
14126 rgnp = srdp->srd_hmergnp[rid];
14128 ASSERT(rgnp != NULL);
14129 ASSERT(rgnp->rgn_id == rid);
14130 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == r_type);
14131 ASSERT(!(rgnp->rgn_flags & SFMMU_REGION_FREE));
14136 r_pgszc = rgnp->rgn_pgszc;
14137 r_size = rgnp->rgn_size;
14165 r_pgszc = rgnp->rgn_pgszc;
14166 r_saddr = rgnp->rgn_saddr;
14167 r_size = rgnp->rgn_size;
14192 if (rgnp->rgn_ttecnt[i] != 0) {
14227 sfmmu_unlink_from_hmeregion(sfmmup, rgnp);
14230 r_obj = rgnp->rgn_obj;
14231 if (atomic_dec_32_nv((volatile uint_t *)&rgnp->rgn_refcnt)) {
14243 if (cur_rgnp == rgnp && cur_rgnp->rgn_refcnt == 0) {
14253 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == r_type);
14254 *prev_rgnpp = rgnp->rgn_hash;
14256 rgnp->rgn_flags |= SFMMU_REGION_FREE;
14258 rgnp->rgn_next = srdp->srd_ismrgnfree;
14259 srdp->srd_ismrgnfree = rgnp;
14270 sfmmu_unload_hmeregion(srdp, rgnp);
14272 rgnp->rgn_hmeflags = 0;
14274 ASSERT(rgnp->rgn_sfmmu_head == NULL);
14275 ASSERT(rgnp->rgn_id == rid);
14277 rgnp->rgn_ttecnt[i] = 0;
14279 rgnp->rgn_flags |= SFMMU_REGION_FREE;
14282 rgnp->rgn_next = srdp->srd_hmergnfree;
14283 srdp->srd_hmergnfree = rgnp;
14297 sf_region_t *rgnp;
14310 rgnp = srdp->srd_hmergnp[rid];
14311 ASSERT(rgnp->rgn_refcnt > 0);
14312 ASSERT(rgnp->rgn_id == rid);
14313 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == SFMMU_REGION_HME);
14314 ASSERT(!(rgnp->rgn_flags & SFMMU_REGION_FREE));
14316 atomic_inc_32((volatile uint_t *)&rgnp->rgn_refcnt);
14321 mutex_enter(&rgnp->rgn_mutex);
14322 ASSERT(rgnp->rgn_sfmmu_head != NULL);
14324 SFMMU_HMERID2RLINKP(rgnp->rgn_sfmmu_head, rid, hrlink, 0, 0);
14327 rlink->next = rgnp->rgn_sfmmu_head;
14335 rgnp->rgn_sfmmu_head = sfmmup;
14336 mutex_exit(&rgnp->rgn_mutex);
14339 rttecnt = rgnp->rgn_size >> TTE_PAGE_SHIFT(rgnp->rgn_pgszc);
14340 atomic_add_long(&sfmmup->sfmmu_ttecnt[rgnp->rgn_pgszc], rttecnt);
14342 if (rgnp->rgn_pgszc >= TTE4M) {
14344 rgnp->rgn_size >> (TTE_PAGE_SHIFT(TTE8K) + 2);
14357 sf_region_t *rgnp = (sf_region_t *)buf;
14358 bzero(buf, sizeof (*rgnp));
14360 mutex_init(&rgnp->rgn_mutex, NULL, MUTEX_DEFAULT, NULL);
14369 sf_region_t *rgnp = (sf_region_t *)buf;
14370 mutex_destroy(&rgnp->rgn_mutex);
14506 sf_region_t *rgnp;
14533 rgnp = srdp->srd_hmergnp[rid];
14534 ASSERT(rgnp->rgn_id == rid);
14535 ASSERT(rgnp->rgn_refcnt > 0);
14537 if (rgnp->rgn_pgszc < TTE4M) {
14538 tte8k_cnt += rgnp->rgn_size >>
14541 ASSERT(rgnp->rgn_pgszc >= TTE4M);
14542 tte4m_cnt += rgnp->rgn_size >>
14549 tte8k_cnt += rgnp->rgn_size >>
14554 rgnp = srdp->srd_ismrgnp[rid];
14555 ASSERT(rgnp->rgn_id == rid);
14556 ASSERT(rgnp->rgn_refcnt > 0);
14558 ism_hatid = (sfmmu_t *)rgnp->rgn_obj;
14567 ASSERT(rgnp->rgn_pgszc >= TTE4M);
14568 if (rgnp->rgn_pgszc >= TTE4M) {
14569 tte4m_cnt += rgnp->rgn_size >>
14632 sfmmu_link_to_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14639 rid = rgnp->rgn_id;
14645 mutex_enter(&rgnp->rgn_mutex);
14646 if ((head = rgnp->rgn_sfmmu_head) == NULL) {
14654 rgnp->rgn_sfmmu_head = sfmmup;
14668 rgnp->rgn_sfmmu_head = sfmmup;
14670 mutex_exit(&rgnp->rgn_mutex);
14677 sfmmu_unlink_from_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14682 rid = rgnp->rgn_id;
14688 mutex_enter(&rgnp->rgn_mutex);
14689 if (rgnp->rgn_sfmmu_head == sfmmup) {
14691 rgnp->rgn_sfmmu_head = next;
14738 mutex_exit(&rgnp->rgn_mutex);
14752 sf_region_t *rgnp;
14773 rgnp = srdp->srd_hmergnp[rid];
14774 ASSERT(rgnp->rgn_id == rid);
14775 ASSERT(rgnp->rgn_refcnt > 0);
14776 sfmmu_link_to_hmeregion(scsfmmup, rgnp);
14781 rgnp = srdp->srd_ismrgnp[rid];
14782 ASSERT(rgnp->rgn_id == rid);
14783 ASSERT(rgnp->rgn_refcnt > 0);
14785 ism_hatid = (sfmmu_t *)rgnp->rgn_obj;
14789 ism_ment->iment_base_va = rgnp->rgn_saddr;
14809 sf_region_t *rgnp;
14829 rgnp = srdp->srd_hmergnp[rid];
14830 ASSERT(rgnp->rgn_id == rid);
14831 ASSERT(rgnp->rgn_refcnt > 0);
14833 rgnp);
14839 rgnp = srdp->srd_ismrgnp[rid];
14840 ASSERT(rgnp->rgn_id == rid);
14841 ASSERT(rgnp->rgn_refcnt > 0);
14843 ism_hatid = (sfmmu_t *)rgnp->rgn_obj;
14848 rgnp->rgn_saddr);
15556 sf_region_t *rgnp;
15564 rgnp = srdp->srd_hmergnp[rid];
15565 ASSERT(rgnp != NULL);
15566 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);