Lines Matching refs:svd
266 struct segvn_data *svd = buf; in segvn_cache_constructor() local
268 rw_init(&svd->lock, NULL, RW_DEFAULT, NULL); in segvn_cache_constructor()
269 mutex_init(&svd->segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL); in segvn_cache_constructor()
270 svd->svn_trnext = svd->svn_trprev = NULL; in segvn_cache_constructor()
278 struct segvn_data *svd = buf; in segvn_cache_destructor() local
280 rw_destroy(&svd->lock); in segvn_cache_destructor()
281 mutex_destroy(&svd->segfree_syncmtx); in segvn_cache_destructor()
544 struct segvn_data *svd; in segvn_create() local
810 svd = kmem_cache_alloc(segvn_cache, KM_SLEEP); in segvn_create()
813 seg->s_data = (void *)svd; in segvn_create()
816 svd->seg = seg; in segvn_create()
817 svd->vp = a->vp; in segvn_create()
821 svd->offset = a->vp ? (a->offset & PAGEMASK) : 0; in segvn_create()
822 svd->prot = a->prot; in segvn_create()
823 svd->maxprot = a->maxprot; in segvn_create()
824 svd->pageprot = 0; in segvn_create()
825 svd->type = a->type; in segvn_create()
826 svd->vpage = NULL; in segvn_create()
827 svd->cred = cred; in segvn_create()
828 svd->advice = MADV_NORMAL; in segvn_create()
829 svd->pageadvice = 0; in segvn_create()
830 svd->flags = (ushort_t)a->flags; in segvn_create()
831 svd->softlockcnt = 0; in segvn_create()
832 svd->softlockcnt_sbase = 0; in segvn_create()
833 svd->softlockcnt_send = 0; in segvn_create()
834 svd->svn_inz = 0; in segvn_create()
835 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_create()
836 svd->pageswap = 0; in segvn_create()
841 if (svd->type == MAP_SHARED && svd->vp != NULL && in segvn_create()
842 (svd->vp->v_flag & VVMEXEC) && (svd->prot & PROT_WRITE)) { in segvn_create()
843 ASSERT(vn_is_mapped(svd->vp, V_WRITE)); in segvn_create()
844 segvn_inval_trcache(svd->vp); in segvn_create()
848 if ((svd->amp = amp) == NULL) { in segvn_create()
849 svd->anon_index = 0; in segvn_create()
850 if (svd->type == MAP_SHARED) { in segvn_create()
851 svd->swresv = 0; in segvn_create()
860 svd->amp = anonmap_alloc(seg->s_size, swresv, in segvn_create()
862 svd->amp->a_szc = seg->s_szc; in segvn_create()
869 svd->swresv = swresv; in segvn_create()
901 svd->anon_index = anon_num; in segvn_create()
902 svd->swresv = 0; in segvn_create()
919 if (svd->flags & MAP_TEXT) { in segvn_create()
923 svd->amp = anonmap_alloc(seg->s_size, 0, ANON_SLEEP); in segvn_create()
924 svd->amp->a_szc = seg->s_szc; in segvn_create()
925 svd->anon_index = 0; in segvn_create()
926 svd->swresv = swresv; in segvn_create()
968 svd->prot & ~PROT_WRITE, hat_flag); in segvn_create()
973 anon_dup(amp->ahp, anon_num, svd->amp->ahp, in segvn_create()
985 (void) lgrp_privm_policy_set(mpolicy, &svd->policy_info, seg->s_size); in segvn_create()
987 if (svd->type == MAP_SHARED) in segvn_create()
988 (void) lgrp_shm_policy_set(mpolicy, svd->amp, svd->anon_index, in segvn_create()
989 svd->vp, svd->offset, seg->s_size); in segvn_create()
993 ASSERT(svd->amp == NULL); in segvn_create()
994 svd->rcookie = hat_join_region(seg->s_as->a_hat, seg->s_base, in segvn_create()
995 seg->s_size, (void *)svd->vp, svd->offset, svd->prot, in segvn_create()
1000 ASSERT(!trok || !(svd->prot & PROT_WRITE)); in segvn_create()
1001 svd->tr_state = trok ? SEGVN_TR_INIT : SEGVN_TR_OFF; in segvn_create()
1508 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_dup_pages() local
1510 ulong_t old_idx = svd->anon_index; in segvn_dup_pages()
1522 if ((ap = anon_get_ptr(svd->amp->ahp, old_idx)) != NULL) { in segvn_dup_pages()
1525 vpp = &svd->vpage[seg_page(seg, addr)]; in segvn_dup_pages()
1532 prot = svd->pageprot ? VPP_PROT(vpp) : svd->prot; in segvn_dup_pages()
1537 if (svd->svn_inz == SEGVN_INZ_ALL || in segvn_dup_pages()
1538 (svd->svn_inz == SEGVN_INZ_VPP && in segvn_dup_pages()
1546 PAGESIZE, seg, addr, S_READ, svd->cred); in segvn_dup_pages()
1571 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_dup() local
1589 if ((len = svd->swresv) != 0) { in segvn_dup()
1590 if (anon_resv(svd->swresv) == 0) in segvn_dup()
1604 if ((newsvd->vp = svd->vp) != NULL) { in segvn_dup()
1605 VN_HOLD(svd->vp); in segvn_dup()
1606 if (svd->type == MAP_SHARED) in segvn_dup()
1607 lgrp_shm_policy_init(NULL, svd->vp); in segvn_dup()
1609 newsvd->offset = svd->offset; in segvn_dup()
1610 newsvd->prot = svd->prot; in segvn_dup()
1611 newsvd->maxprot = svd->maxprot; in segvn_dup()
1612 newsvd->pageprot = svd->pageprot; in segvn_dup()
1613 newsvd->type = svd->type; in segvn_dup()
1614 newsvd->cred = svd->cred; in segvn_dup()
1616 newsvd->advice = svd->advice; in segvn_dup()
1617 newsvd->pageadvice = svd->pageadvice; in segvn_dup()
1618 newsvd->svn_inz = svd->svn_inz; in segvn_dup()
1619 newsvd->swresv = svd->swresv; in segvn_dup()
1620 newsvd->pageswap = svd->pageswap; in segvn_dup()
1621 newsvd->flags = svd->flags; in segvn_dup()
1625 newsvd->policy_info = svd->policy_info; in segvn_dup()
1628 if ((amp = svd->amp) == NULL || svd->tr_state == SEGVN_TR_ON) { in segvn_dup()
1632 ASSERT(!HAT_IS_REGION_COOKIE_VALID(svd->rcookie) || in segvn_dup()
1633 svd->tr_state == SEGVN_TR_OFF); in segvn_dup()
1634 if (svd->tr_state == SEGVN_TR_ON) { in segvn_dup()
1638 newsvd->tr_state = svd->tr_state; in segvn_dup()
1644 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_dup()
1645 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_dup()
1647 if (svd->type == MAP_SHARED) { in segvn_dup()
1648 ASSERT(svd->svn_inz == SEGVN_INZ_NONE); in segvn_dup()
1653 newsvd->anon_index = svd->anon_index; in segvn_dup()
1664 ASSERT(svd->svn_inz == SEGVN_INZ_NONE || in segvn_dup()
1665 svd->svn_inz == SEGVN_INZ_ALL || in segvn_dup()
1666 svd->svn_inz == SEGVN_INZ_VPP); in segvn_dup()
1698 if (svd->softlockcnt || in segvn_dup()
1699 svd->svn_inz != SEGVN_INZ_NONE) { in segvn_dup()
1709 if (svd->softlockcnt && reclaim == 1) { in segvn_dup()
1730 svd->anon_index, newsvd->amp->ahp, in segvn_dup()
1732 svd->vp != NULL); in segvn_dup()
1734 anon_dup(amp->ahp, svd->anon_index, in segvn_dup()
1747 if (svd->vpage != NULL) { in segvn_dup()
1749 struct vpage *ovp = svd->vpage; in segvn_dup()
1768 if (error == 0 && HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_dup()
1771 newsvd->rcookie = svd->rcookie; in segvn_dup()
1816 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_hat_unload_callback() local
1820 ASSERT(svd->vp != NULL); in segvn_hat_unload_callback()
1826 free_vp_pages(svd->vp, svd->offset + off, len); in segvn_hat_unload_callback()
1837 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_count_swap_by_vpages() local
1841 ASSERT(svd->pageswap); in segvn_count_swap_by_vpages()
1842 ASSERT(svd->vpage != NULL); in segvn_count_swap_by_vpages()
1844 evp = &svd->vpage[seg_page(seg, seg->s_base + seg->s_size)]; in segvn_count_swap_by_vpages()
1846 for (vp = svd->vpage; vp < evp; vp++) { in segvn_count_swap_by_vpages()
1857 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_unmap() local
1882 if (svd->softlockcnt > 0) { in segvn_unmap()
1883 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_unmap()
1889 if (svd->type == MAP_SHARED) { in segvn_unmap()
1920 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_unmap()
1921 ASSERT(svd->amp == NULL); in segvn_unmap()
1922 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_unmap()
1924 svd->rcookie, HAT_REGION_TEXT); in segvn_unmap()
1925 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_unmap()
1932 } else if (svd->tr_state == SEGVN_TR_INIT) { in segvn_unmap()
1933 svd->tr_state = SEGVN_TR_OFF; in segvn_unmap()
1934 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_unmap()
1935 ASSERT(svd->amp != NULL); in segvn_unmap()
1937 ASSERT(svd->amp == NULL); in segvn_unmap()
1938 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_unmap()
1950 if (svd->vp) { in segvn_unmap()
1953 error = VOP_DELMAP(svd->vp, in segvn_unmap()
1954 (offset_t)svd->offset + (uintptr_t)(addr - seg->s_base), in segvn_unmap()
1955 seg->s_as, addr, len, svd->prot, svd->maxprot, in segvn_unmap()
1956 svd->type, svd->cred, NULL); in segvn_unmap()
1967 if (svd->tr_state == SEGVN_TR_OFF) { in segvn_unmap()
1971 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_unmap()
1972 ASSERT(svd->amp == NULL); in segvn_unmap()
1973 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_unmap()
1974 ASSERT(svd->type == MAP_PRIVATE); in segvn_unmap()
1975 hat_leave_region(seg->s_as->a_hat, svd->rcookie, in segvn_unmap()
1977 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_unmap()
1978 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_unmap()
1979 ASSERT(svd->amp != NULL); in segvn_unmap()
1980 ASSERT(svd->pageprot == 0 && !(svd->prot & PROT_WRITE)); in segvn_unmap()
1982 ASSERT(svd->amp == NULL && svd->tr_state == SEGVN_TR_OFF); in segvn_unmap()
1984 if (svd->tr_state != SEGVN_TR_OFF) { in segvn_unmap()
1985 ASSERT(svd->tr_state == SEGVN_TR_INIT); in segvn_unmap()
1986 svd->tr_state = SEGVN_TR_OFF; in segvn_unmap()
1992 if (svd->vp != NULL && free_pages != 0) { in segvn_unmap()
2000 if (svd->type == MAP_SHARED && svd->vp != NULL && in segvn_unmap()
2001 (svd->vp->v_flag & VVMEXEC) && in segvn_unmap()
2002 ((svd->prot & PROT_WRITE) || svd->pageprot)) { in segvn_unmap()
2003 segvn_inval_trcache(svd->vp); in segvn_unmap()
2018 amp = svd->amp; in segvn_unmap()
2025 if (svd->vpage != NULL) { in segvn_unmap()
2029 ovpage = svd->vpage; /* keep pointer to vpage */ in segvn_unmap()
2032 svd->vpage = kmem_alloc(nbytes, KM_SLEEP); in segvn_unmap()
2033 bcopy(&ovpage[dpages], svd->vpage, nbytes); in segvn_unmap()
2040 if (amp->refcnt == 1 || svd->type == MAP_PRIVATE) { in segvn_unmap()
2046 if (svd->type == MAP_SHARED) { in segvn_unmap()
2048 ASSERT(svd->softlockcnt == 0); in segvn_unmap()
2057 svd->anon_index, len, in segvn_unmap()
2061 svd->anon_index, in segvn_unmap()
2065 ASSERT(svd->type == MAP_SHARED); in segvn_unmap()
2068 svd->anon_index, len); in segvn_unmap()
2076 if (svd->type == MAP_SHARED) { in segvn_unmap()
2083 svd->anon_index += dpages; in segvn_unmap()
2085 if (svd->vp != NULL) in segvn_unmap()
2086 svd->offset += len; in segvn_unmap()
2091 if (svd->swresv) { in segvn_unmap()
2092 if (svd->flags & MAP_NORESERVE) { in segvn_unmap()
2094 oswresv = svd->swresv; in segvn_unmap()
2096 svd->swresv = ptob(anon_pages(amp->ahp, in segvn_unmap()
2097 svd->anon_index, npages)); in segvn_unmap()
2098 anon_unresv_zone(oswresv - svd->swresv, in segvn_unmap()
2102 svd->swresv; in segvn_unmap()
2106 if (svd->pageswap) { in segvn_unmap()
2107 oswresv = svd->swresv; in segvn_unmap()
2108 svd->swresv = in segvn_unmap()
2110 ASSERT(oswresv >= svd->swresv); in segvn_unmap()
2111 unlen = oswresv - svd->swresv; in segvn_unmap()
2113 svd->swresv -= len; in segvn_unmap()
2114 ASSERT(svd->swresv == seg->s_size); in segvn_unmap()
2131 if (svd->vpage != NULL) { in segvn_unmap()
2135 ovpage = svd->vpage; /* keep pointer to vpage */ in segvn_unmap()
2138 svd->vpage = kmem_alloc(nbytes, KM_SLEEP); in segvn_unmap()
2139 bcopy(ovpage, svd->vpage, nbytes); in segvn_unmap()
2147 if (amp->refcnt == 1 || svd->type == MAP_PRIVATE) { in segvn_unmap()
2151 ulong_t an_idx = svd->anon_index + npages; in segvn_unmap()
2158 if (svd->type == MAP_SHARED) { in segvn_unmap()
2160 ASSERT(svd->softlockcnt == 0); in segvn_unmap()
2174 ASSERT(svd->type == MAP_SHARED); in segvn_unmap()
2185 if (svd->type == MAP_SHARED) { in segvn_unmap()
2196 if (svd->swresv) { in segvn_unmap()
2197 if (svd->flags & MAP_NORESERVE) { in segvn_unmap()
2199 oswresv = svd->swresv; in segvn_unmap()
2200 svd->swresv = ptob(anon_pages(amp->ahp, in segvn_unmap()
2201 svd->anon_index, npages)); in segvn_unmap()
2202 anon_unresv_zone(oswresv - svd->swresv, in segvn_unmap()
2206 svd->swresv; in segvn_unmap()
2210 if (svd->pageswap) { in segvn_unmap()
2211 oswresv = svd->swresv; in segvn_unmap()
2212 svd->swresv = in segvn_unmap()
2214 ASSERT(oswresv >= svd->swresv); in segvn_unmap()
2215 unlen = oswresv - svd->swresv; in segvn_unmap()
2217 svd->swresv -= len; in segvn_unmap()
2218 ASSERT(svd->swresv == seg->s_size); in segvn_unmap()
2248 *nsvd = *svd; in segvn_unmap()
2250 nsvd->offset = svd->offset + (uintptr_t)(nseg->s_base - seg->s_base); in segvn_unmap()
2255 nsvd->svn_inz = svd->svn_inz; in segvn_unmap()
2258 if (svd->vp != NULL) { in segvn_unmap()
2263 crhold(svd->cred); in segvn_unmap()
2265 if (svd->vpage == NULL) { in segvn_unmap()
2272 ovpage = svd->vpage; /* keep pointer to vpage */ in segvn_unmap()
2276 svd->vpage = kmem_alloc(nbytes, KM_SLEEP); in segvn_unmap()
2278 bcopy(ovpage, svd->vpage, nbytes); in segvn_unmap()
2301 if (amp->refcnt == 1 || svd->type == MAP_PRIVATE) { in segvn_unmap()
2305 ulong_t an_idx = svd->anon_index + opages; in segvn_unmap()
2312 if (svd->type == MAP_SHARED) { in segvn_unmap()
2314 ASSERT(svd->softlockcnt == 0); in segvn_unmap()
2327 ASSERT(svd->type == MAP_SHARED); in segvn_unmap()
2337 if (svd->type == MAP_SHARED) { in segvn_unmap()
2343 nsvd->anon_index = svd->anon_index + in segvn_unmap()
2345 if (svd->type == MAP_SHARED) { in segvn_unmap()
2352 ASSERT(svd->type == MAP_PRIVATE); in segvn_unmap()
2356 (void) anon_copy_ptr(amp->ahp, svd->anon_index, nahp, in segvn_unmap()
2361 svd->anon_index = 0; in segvn_unmap()
2369 if (svd->swresv) { in segvn_unmap()
2370 if (svd->flags & MAP_NORESERVE) { in segvn_unmap()
2372 oswresv = svd->swresv; in segvn_unmap()
2373 svd->swresv = ptob(anon_pages(amp->ahp, in segvn_unmap()
2374 svd->anon_index, btop(seg->s_size))); in segvn_unmap()
2377 ASSERT(oswresv >= (svd->swresv + nsvd->swresv)); in segvn_unmap()
2378 anon_unresv_zone(oswresv - (svd->swresv + nsvd->swresv), in segvn_unmap()
2382 (svd->swresv + nsvd->swresv); in segvn_unmap()
2386 if (svd->pageswap) { in segvn_unmap()
2387 oswresv = svd->swresv; in segvn_unmap()
2388 svd->swresv = segvn_count_swap_by_vpages(seg); in segvn_unmap()
2390 ASSERT(oswresv >= (svd->swresv + nsvd->swresv)); in segvn_unmap()
2391 unlen = oswresv - (svd->swresv + nsvd->swresv); in segvn_unmap()
2394 svd->swresv) { in segvn_unmap()
2399 svd->swresv = seg->s_size; in segvn_unmap()
2416 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_free() local
2426 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_free()
2428 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_free()
2440 if (svd->vpage != NULL) { in segvn_free()
2441 kmem_free(svd->vpage, vpgtob(npages)); in segvn_free()
2442 svd->vpage = NULL; in segvn_free()
2444 if ((amp = svd->amp) != NULL) { in segvn_free()
2453 if (svd->type == MAP_PRIVATE) { in segvn_free()
2460 svd->anon_index, seg->s_size, in segvn_free()
2463 anon_free(amp->ahp, svd->anon_index, in segvn_free()
2473 ASSERT(svd->softlockcnt == 0); in segvn_free()
2494 svd->amp = NULL; in segvn_free()
2497 } else if (svd->type == MAP_PRIVATE) { in segvn_free()
2504 anon_free_pages(amp->ahp, svd->anon_index, in segvn_free()
2507 anon_free(amp->ahp, svd->anon_index, in segvn_free()
2519 if ((len = svd->swresv) != 0) { in segvn_free()
2520 anon_unresv_zone(svd->swresv, in segvn_free()
2525 seg->s_as->a_resvsize -= svd->swresv; in segvn_free()
2526 svd->swresv = 0; in segvn_free()
2532 if (svd->vp != NULL) { in segvn_free()
2533 if (svd->type == MAP_SHARED) in segvn_free()
2534 lgrp_shm_policy_fini(NULL, svd->vp); in segvn_free()
2535 VN_RELE(svd->vp); in segvn_free()
2536 svd->vp = NULL; in segvn_free()
2538 crfree(svd->cred); in segvn_free()
2539 svd->pageprot = 0; in segvn_free()
2540 svd->pageadvice = 0; in segvn_free()
2541 svd->pageswap = 0; in segvn_free()
2542 svd->cred = NULL; in segvn_free()
2549 ASSERT(svd->softlockcnt == 0); in segvn_free()
2550 mutex_enter(&svd->segfree_syncmtx); in segvn_free()
2551 mutex_exit(&svd->segfree_syncmtx); in segvn_free()
2554 kmem_cache_free(segvn_cache, svd); in segvn_free()
2566 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_softunlock() local
2576 ASSERT(SEGVN_LOCK_HELD(seg->s_as, &svd->lock)); in segvn_softunlock()
2578 if ((amp = svd->amp) != NULL) in segvn_softunlock()
2579 anon_index = svd->anon_index + seg_page(seg, addr); in segvn_softunlock()
2581 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_softunlock()
2582 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_softunlock()
2583 hat_unlock_region(seg->s_as->a_hat, addr, len, svd->rcookie); in segvn_softunlock()
2594 vp = svd->vp; in segvn_softunlock()
2595 offset = svd->offset + in segvn_softunlock()
2600 vp = svd->vp; in segvn_softunlock()
2601 offset = svd->offset + in segvn_softunlock()
2631 ASSERT(svd->softlockcnt >= btop(len)); in segvn_softunlock()
2632 if (!atomic_add_long_nv((ulong_t *)&svd->softlockcnt, -btop(len))) { in segvn_softunlock()
2712 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_faultpage() local
2729 if (svd->flags & MAP_TEXT) { in segvn_faultpage()
2733 ASSERT(SEGVN_READ_HELD(seg->s_as, &svd->lock)); in segvn_faultpage()
2735 ASSERT(svd->tr_state != SEGVN_TR_INIT); in segvn_faultpage()
2741 if (svd->pageprot) { in segvn_faultpage()
2764 prot = svd->prot; in segvn_faultpage()
2768 atomic_inc_ulong((ulong_t *)&svd->softlockcnt); in segvn_faultpage()
2776 if ((amp = svd->amp) != NULL) { in segvn_faultpage()
2778 anon_index = svd->anon_index + seg_page(seg, addr); in segvn_faultpage()
2783 if (svd->vp == NULL && amp != NULL) { in segvn_faultpage()
2789 if (svd->flags & MAP_NORESERVE) { in segvn_faultpage()
2792 atomic_add_long(&svd->swresv, ptob(1)); in segvn_faultpage()
2801 svd->cred)) == NULL) { in segvn_faultpage()
2861 (svd->flags & MAP_NORESERVE) && in segvn_faultpage()
2864 ASSERT(svd->type == MAP_PRIVATE); in segvn_faultpage()
2879 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_faultpage()
2898 seg, addr, rw, svd->cred); in segvn_faultpage()
2902 if (svd->type == MAP_SHARED) { in segvn_faultpage()
2928 ASSERT(opp->p_vnode == svd->vp); /* XXX */ in segvn_faultpage()
2953 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_faultpage()
2955 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_faultpage()
2960 ASSERT(svd->vp != NULL); in segvn_faultpage()
2986 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE || in segvn_faultpage()
2987 (!svd->pageprot && svd->prot == (prot & vpprot))); in segvn_faultpage()
2989 svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_faultpage()
2991 svd->rcookie); in segvn_faultpage()
3002 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_faultpage()
3037 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_faultpage()
3068 if ((svd->flags & MAP_NORESERVE) && (ap == NULL)) { in segvn_faultpage()
3070 atomic_add_long(&svd->swresv, ptob(1)); in segvn_faultpage()
3079 pp = anon_private(&ap, seg, addr, prot, opp, pageflags, svd->cred); in segvn_faultpage()
3115 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_faultpage()
3129 atomic_dec_ulong((ulong_t *)&svd->softlockcnt); in segvn_faultpage()
3357 segvn_fill_vp_pages(struct segvn_data *svd, vnode_t *vp, u_offset_t off, in segvn_fill_vp_pages() argument
3488 if (VOP_GETATTR(vp, &va, ATTR_HINT, svd->cred, NULL)) { in segvn_fill_vp_pages()
3501 B_READ, svd->cred, NULL); in segvn_fill_vp_pages()
3558 if (VOP_GETATTR(vp, &va, ATTR_HINT, svd->cred, NULL) != 0) { in segvn_fill_vp_pages()
3569 B_READ, svd->cred, NULL); in segvn_fill_vp_pages()
3634 if (svd->type == MAP_PRIVATE) { in segvn_fill_vp_pages()
3787 atomic_add_long((ulong_t *)&(svd)->softlockcnt, \
3828 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_fault_vnodepages() local
3829 struct anon_map *amp = svd->amp; in segvn_fault_vnodepages()
3830 uchar_t segtype = svd->type; in segvn_fault_vnodepages()
3839 u_offset_t off = svd->offset + (uintptr_t)(a - seg->s_base); in segvn_fault_vnodepages()
3840 ulong_t aindx = svd->anon_index + seg_page(seg, a); in segvn_fault_vnodepages()
3841 struct vpage *vpage = (svd->vpage != NULL) ? in segvn_fault_vnodepages()
3842 &svd->vpage[seg_page(seg, a)] : NULL; in segvn_fault_vnodepages()
3843 vnode_t *vp = svd->vp; in segvn_fault_vnodepages()
3865 int tron = (svd->tr_state == SEGVN_TR_ON); in segvn_fault_vnodepages()
3872 ASSERT(!(svd->flags & MAP_NORESERVE)); in segvn_fault_vnodepages()
3876 ASSERT(SEGVN_LOCK_HELD(seg->s_as, &svd->lock)); in segvn_fault_vnodepages()
3879 ASSERT(svd->tr_state != SEGVN_TR_INIT); in segvn_fault_vnodepages()
3884 if (svd->flags & MAP_TEXT) { in segvn_fault_vnodepages()
3888 if (svd->pageprot) { in segvn_fault_vnodepages()
3905 prot = svd->prot; in segvn_fault_vnodepages()
3984 if (svd->pageprot != 0 && IS_P2ALIGNED(a, maxpgsz)) { in segvn_fault_vnodepages()
3995 atomic_add_long((ulong_t *)&svd->softlockcnt, in segvn_fault_vnodepages()
4018 physcontig = segvn_fill_vp_pages(svd, in segvn_fault_vnodepages()
4047 svd->cred, NULL); in segvn_fault_vnodepages()
4084 if (VOP_GETATTR(vp, &va, 0, svd->cred, NULL)) { in segvn_fault_vnodepages()
4175 tron ? PG_LOCAL : 0, svd->cred); in segvn_fault_vnodepages()
4190 ASSERT(svd->rcookie == in segvn_fault_vnodepages()
4206 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE || in segvn_fault_vnodepages()
4207 (!svd->pageprot && svd->prot == (prot & vpprot))); in segvn_fault_vnodepages()
4271 svd->rcookie); in segvn_fault_vnodepages()
4386 svd->rcookie); in segvn_fault_vnodepages()
4392 hat_flag, svd->rcookie); in segvn_fault_vnodepages()
4451 prot & vpprot, hat_flag, svd->rcookie); in segvn_fault_vnodepages()
4506 svd->rcookie); in segvn_fault_vnodepages()
4511 prot & vpprot, hat_flag, svd->rcookie); in segvn_fault_vnodepages()
4575 off = svd->offset + (uintptr_t)(a - seg->s_base); in segvn_fault_vnodepages()
4576 aindx = svd->anon_index + seg_page(seg, a); in segvn_fault_vnodepages()
4577 vpage = (svd->vpage != NULL) ? in segvn_fault_vnodepages()
4578 &svd->vpage[seg_page(seg, a)] : NULL; in segvn_fault_vnodepages()
4598 off = svd->offset + in segvn_fault_vnodepages()
4600 aindx = svd->anon_index + seg_page(seg, a); in segvn_fault_vnodepages()
4601 vpage = (svd->vpage != NULL) ? in segvn_fault_vnodepages()
4602 &svd->vpage[seg_page(seg, a)] : NULL; in segvn_fault_vnodepages()
4631 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault_vnodepages()
4632 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_fault_vnodepages()
4636 ASSERT(svd->softlockcnt == 0); in segvn_fault_vnodepages()
4643 SEGVN_LOCK_DOWNGRADE(seg->s_as, &svd->lock); in segvn_fault_vnodepages()
4658 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_fault_anonpages() local
4659 struct anon_map *amp = svd->amp; in segvn_fault_anonpages()
4660 uchar_t segtype = svd->type; in segvn_fault_anonpages()
4667 ulong_t aindx = svd->anon_index + seg_page(seg, a); in segvn_fault_anonpages()
4668 struct vpage *vpage = (svd->vpage != NULL) ? in segvn_fault_anonpages()
4669 &svd->vpage[seg_page(seg, a)] : NULL; in segvn_fault_anonpages()
4679 int pgflags = (svd->tr_state == SEGVN_TR_ON) ? PG_LOCAL : 0; in segvn_fault_anonpages()
4684 ASSERT(!(svd->flags & MAP_NORESERVE)); in segvn_fault_anonpages()
4687 ASSERT(!brkcow || svd->tr_state == SEGVN_TR_OFF); in segvn_fault_anonpages()
4688 ASSERT(svd->tr_state != SEGVN_TR_INIT); in segvn_fault_anonpages()
4690 ASSERT(SEGVN_LOCK_HELD(seg->s_as, &svd->lock)); in segvn_fault_anonpages()
4695 if (svd->flags & MAP_TEXT) { in segvn_fault_anonpages()
4699 if (svd->pageprot) { in segvn_fault_anonpages()
4717 prot = svd->prot; in segvn_fault_anonpages()
4726 if (svd->pageprot != 0 && IS_P2ALIGNED(a, maxpgsz)) { in segvn_fault_anonpages()
4747 atomic_add_long((ulong_t *)&svd->softlockcnt, in segvn_fault_anonpages()
4754 segvn_anypgsz, pgflags, svd->cred); in segvn_fault_anonpages()
4760 (ulong_t *)&svd->softlockcnt, in segvn_fault_anonpages()
4784 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_fault_anonpages()
4875 aindx = svd->anon_index + seg_page(seg, a); in segvn_fault_anonpages()
4876 vpage = (svd->vpage != NULL) ? in segvn_fault_anonpages()
4877 &svd->vpage[seg_page(seg, a)] : NULL; in segvn_fault_anonpages()
4898 aindx = svd->anon_index + seg_page(seg, a); in segvn_fault_anonpages()
4899 vpage = (svd->vpage != NULL) ? in segvn_fault_anonpages()
4900 &svd->vpage[seg_page(seg, a)] : NULL; in segvn_fault_anonpages()
4946 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_fault() local
4962 int brkcow = BREAK_COW_SHARE(rw, type, svd->type); in segvn_fault()
4965 ASSERT(svd->amp == NULL || svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_fault()
4975 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_fault()
4981 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
4985 ASSERT(svd->tr_state == SEGVN_TR_OFF || in segvn_fault()
4986 !HAT_IS_REGION_COOKIE_VALID(svd->rcookie)); in segvn_fault()
4988 if (svd->tr_state == SEGVN_TR_INIT) { in segvn_fault()
4989 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_fault()
4990 if (svd->tr_state == SEGVN_TR_INIT) { in segvn_fault()
4991 ASSERT(svd->vp != NULL && svd->amp == NULL); in segvn_fault()
4992 ASSERT(svd->flags & MAP_TEXT); in segvn_fault()
4993 ASSERT(svd->type == MAP_PRIVATE); in segvn_fault()
4995 ASSERT(svd->tr_state != SEGVN_TR_INIT); in segvn_fault()
4996 ASSERT(svd->tr_state != SEGVN_TR_ON || in segvn_fault()
4997 svd->amp != NULL); in segvn_fault()
4999 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5001 } else if (svd->tr_state != SEGVN_TR_OFF) { in segvn_fault()
5002 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_fault()
5004 if (rw == S_WRITE && svd->tr_state != SEGVN_TR_OFF) { in segvn_fault()
5005 ASSERT(!svd->pageprot && !(svd->prot & PROT_WRITE)); in segvn_fault()
5006 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5010 if (svd->tr_state == SEGVN_TR_ON) { in segvn_fault()
5011 ASSERT(svd->vp != NULL && svd->amp != NULL); in segvn_fault()
5013 ASSERT(svd->amp == NULL && in segvn_fault()
5014 svd->tr_state == SEGVN_TR_OFF); in segvn_fault()
5015 } else if (svd->tr_state != SEGVN_TR_OFF) { in segvn_fault()
5016 svd->tr_state = SEGVN_TR_OFF; in segvn_fault()
5018 ASSERT(svd->amp == NULL && svd->tr_state == SEGVN_TR_OFF); in segvn_fault()
5019 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5023 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_fault()
5030 if (svd->pageprot == 0) { in segvn_fault()
5050 if ((svd->prot & protchk) == 0) { in segvn_fault()
5051 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5056 if (brkcow && HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_fault()
5058 ASSERT(svd->amp == NULL); in segvn_fault()
5059 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_fault()
5060 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5061 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_fault()
5062 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_fault()
5067 ASSERT(svd->softlockcnt == 0); in segvn_fault()
5068 hat_leave_region(seg->s_as->a_hat, svd->rcookie, in segvn_fault()
5070 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_fault()
5072 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5103 if (type == F_SOFTLOCK && svd->vp != NULL && seg->s_szc != 0) { in segvn_fault()
5121 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5122 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_fault()
5125 ASSERT(svd->softlockcnt == 0); in segvn_fault()
5129 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5134 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5142 if (svd->amp == NULL && (svd->vp == NULL || brkcow)) { in segvn_fault()
5143 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_fault()
5149 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5150 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_fault()
5152 if (svd->amp == NULL) { in segvn_fault()
5153 svd->amp = anonmap_alloc(seg->s_size, 0, ANON_SLEEP); in segvn_fault()
5154 svd->amp->a_szc = seg->s_szc; in segvn_fault()
5156 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5176 amp = svd->amp; in segvn_fault()
5183 ASSERT(SEGVN_LOCK_HELD(seg->s_as, &svd->lock)); in segvn_fault()
5185 if (svd->vp == NULL) { in segvn_fault()
5193 ASSERT(SEGVN_READ_HELD(seg->s_as, &svd->lock)); in segvn_fault()
5194 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5198 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5204 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_fault()
5205 anon_index = svd->anon_index + page; in segvn_fault()
5208 svd->tr_state == SEGVN_TR_OFF && in segvn_fault()
5209 svd->type == MAP_PRIVATE && svd->pageprot == 0) { in segvn_fault()
5232 hat_chgprot(seg->s_as->a_hat, addr, len, svd->prot); in segvn_fault()
5234 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5240 if (svd->vpage == NULL) in segvn_fault()
5243 vpage = &svd->vpage[page]; in segvn_fault()
5245 off = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_fault()
5253 if ((page != 0) && fltadvice && svd->tr_state != SEGVN_TR_ON) { in segvn_fault()
5261 if (svd->advice == MADV_SEQUENTIAL || in segvn_fault()
5262 (svd->pageadvice && in segvn_fault()
5267 vpp = &svd->vpage[fpage]; in segvn_fault()
5269 fanon_index = svd->anon_index + fpage; in segvn_fault()
5271 while (pgoff > svd->offset) { in segvn_fault()
5272 if (svd->advice != MADV_SEQUENTIAL && in segvn_fault()
5273 (!svd->pageadvice || (vpage && in segvn_fault()
5293 fvp = svd->vp; in segvn_fault()
5299 fvp = svd->vp; in segvn_fault()
5333 svd->cred, NULL); in segvn_fault()
5359 if (svd->vp != NULL) { in segvn_fault()
5402 } else if (rw == S_WRITE && svd->type == MAP_PRIVATE || in segvn_fault()
5403 svd->tr_state == SEGVN_TR_ON || rw == S_OTHER || in segvn_fault()
5438 if (rw == S_WRITE && svd->type == MAP_PRIVATE) { in segvn_fault()
5443 vp = svd->vp; in segvn_fault()
5449 svd->cred, NULL); in segvn_fault()
5451 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5457 if (svd->type == MAP_PRIVATE) in segvn_fault()
5515 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5523 } else if (svd->vpage) { in segvn_fault()
5525 vpage = &svd->vpage[++page]; in segvn_fault()
5540 if (svd->pageprot == 0) in segvn_fault()
5541 prot = svd->prot & vpprot; in segvn_fault()
5558 if (svd->flags & MAP_TEXT) { in segvn_fault()
5565 if (svd->tr_state != SEGVN_TR_ON && in segvn_fault()
5566 pp->p_offset >= svd->offset && in segvn_fault()
5567 pp->p_offset < svd->offset + seg->s_size) { in segvn_fault()
5569 diff = pp->p_offset - svd->offset; in segvn_fault()
5575 ASSERT(svd->vp == pp->p_vnode); in segvn_fault()
5578 if (svd->pageprot) in segvn_fault()
5579 prot = VPP_PROT(&svd->vpage[page]) & vpprot; in segvn_fault()
5589 anon_index = svd->anon_index + page; in segvn_fault()
5608 svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_fault()
5613 svd->rcookie); in segvn_fault()
5624 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_fault()
5638 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_faulta() local
5645 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_faulta()
5646 if ((amp = svd->amp) != NULL) { in segvn_faulta()
5656 svd->anon_index + seg_page(seg, addr))) != NULL) { in segvn_faulta()
5659 0, seg, addr, S_READ, svd->cred); in segvn_faulta()
5662 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_faulta()
5670 if (svd->vp == NULL) { in segvn_faulta()
5671 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_faulta()
5675 vp = svd->vp; in segvn_faulta()
5679 (offset_t)(svd->offset + (uintptr_t)(addr - seg->s_base)), in segvn_faulta()
5681 S_OTHER, svd->cred, NULL); in segvn_faulta()
5683 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_faulta()
5692 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_setprot() local
5702 if ((svd->maxprot & prot) != prot) in segvn_setprot()
5705 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_setprot()
5708 if (!svd->pageprot && svd->prot == prot) { in segvn_setprot()
5709 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5718 if (svd->softlockcnt > 0) { in segvn_setprot()
5719 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_setprot()
5725 if (svd->type == MAP_SHARED) { in segvn_setprot()
5726 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5737 if (svd->softlockcnt > 0) { in segvn_setprot()
5738 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5743 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_setprot()
5744 ASSERT(svd->amp == NULL); in segvn_setprot()
5745 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_setprot()
5746 hat_leave_region(seg->s_as->a_hat, svd->rcookie, in segvn_setprot()
5748 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_setprot()
5750 } else if (svd->tr_state == SEGVN_TR_INIT) { in segvn_setprot()
5751 svd->tr_state = SEGVN_TR_OFF; in segvn_setprot()
5752 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_setprot()
5753 ASSERT(svd->amp != NULL); in segvn_setprot()
5755 ASSERT(svd->amp == NULL && svd->tr_state == SEGVN_TR_OFF); in segvn_setprot()
5759 if ((prot & PROT_WRITE) && svd->type == MAP_SHARED && in segvn_setprot()
5760 svd->vp != NULL && (svd->vp->v_flag & VVMEXEC)) { in segvn_setprot()
5761 ASSERT(vn_is_mapped(svd->vp, V_WRITE)); in segvn_setprot()
5762 segvn_inval_trcache(svd->vp); in segvn_setprot()
5770 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5780 if (svd->type == MAP_PRIVATE || svd->vp != NULL) { in segvn_setprot()
5786 (svd->flags & MAP_TEXT), MAPPGSZC_SHM, 0); in segvn_setprot()
5811 if (svd->type == MAP_PRIVATE) { in segvn_setprot()
5813 if (!(svd->flags & MAP_NORESERVE) && in segvn_setprot()
5814 !(svd->swresv && svd->pageswap == 0)) { in segvn_setprot()
5823 svd->pageswap == 0) { in segvn_setprot()
5834 if (svd->vpage == NULL) { in segvn_setprot()
5836 &svd->lock); in segvn_setprot()
5839 svp = &svd->vpage[seg_page(seg, addr)]; in segvn_setprot()
5840 evp = &svd->vpage[seg_page(seg, in segvn_setprot()
5843 if (svd->pageswap == 0) { in segvn_setprot()
5869 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5877 if (svd->pageswap == 0 && sz == seg->s_size) { in segvn_setprot()
5878 svd->swresv = sz; in segvn_setprot()
5880 ASSERT(svd->vpage != NULL); in segvn_setprot()
5881 svd->swresv += sz; in segvn_setprot()
5882 svd->pageswap = 1; in segvn_setprot()
5896 if (svd->swresv != 0 && svd->vp != NULL && in segvn_setprot()
5897 svd->amp == NULL && addr == seg->s_base && in segvn_setprot()
5898 len == seg->s_size && svd->pageprot == 0) { in segvn_setprot()
5899 ASSERT(svd->pageswap == 0); in segvn_setprot()
5900 anon_unresv_zone(svd->swresv, in segvn_setprot()
5902 svd->swresv = 0; in segvn_setprot()
5909 if (addr == seg->s_base && len == seg->s_size && svd->vpage == NULL) { in segvn_setprot()
5910 if (svd->prot == prot) { in segvn_setprot()
5911 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5914 svd->prot = (uchar_t)prot; in segvn_setprot()
5915 } else if (svd->type == MAP_PRIVATE) { in segvn_setprot()
5932 if (svd->vpage == NULL) { in segvn_setprot()
5933 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
5936 svd->pageprot = 1; in segvn_setprot()
5937 if ((amp = svd->amp) != NULL) { in segvn_setprot()
5938 anon_idx = svd->anon_index + seg_page(seg, addr); in segvn_setprot()
5944 offset = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_setprot()
5945 evp = &svd->vpage[seg_page(seg, addr + len)]; in segvn_setprot()
5951 for (svp = &svd->vpage[seg_page(seg, addr)]; svp < evp; svp++) { in segvn_setprot()
5981 vp = svd->vp; in segvn_setprot()
6026 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
6029 len = (svp - &svd->vpage[seg_page(seg, addr)]) * in segvn_setprot()
6035 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
6040 if (svd->vpage == NULL) { in segvn_setprot()
6041 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
6044 svd->pageprot = 1; in segvn_setprot()
6045 evp = &svd->vpage[seg_page(seg, addr + len)]; in segvn_setprot()
6046 for (svp = &svd->vpage[seg_page(seg, addr)]; svp < evp; svp++) { in segvn_setprot()
6052 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
6057 (svd->vp != NULL || svd->type == MAP_PRIVATE)) || in segvn_setprot()
6081 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_setprot()
6093 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_setpagesize() local
6095 struct anon_map *amp = svd->amp; in segvn_setpagesize()
6101 u_offset_t off = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_setpagesize()
6125 if (amp != NULL && svd->type == MAP_SHARED) { in segvn_setpagesize()
6126 ulong_t an_idx = svd->anon_index + seg_page(seg, addr); in segvn_setpagesize()
6134 if ((svd->flags & MAP_NORESERVE) || seg->s_as == &kas || in segvn_setpagesize()
6140 if (svd->vp != NULL && in segvn_setpagesize()
6141 (IS_SWAPFSVP(svd->vp) || VN_ISKAS(svd->vp))) { in segvn_setpagesize()
6145 if (seg->s_szc == 0 && svd->vp != NULL && in segvn_setpagesize()
6154 if (svd->pageprot) { in segvn_setpagesize()
6173 if (svd->softlockcnt > 0) { in segvn_setpagesize()
6174 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_setpagesize()
6180 if (svd->type == MAP_SHARED) { in segvn_setpagesize()
6191 if (svd->softlockcnt > 0) { in segvn_setpagesize()
6196 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_setpagesize()
6197 ASSERT(svd->amp == NULL); in segvn_setpagesize()
6198 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_setpagesize()
6199 hat_leave_region(seg->s_as->a_hat, svd->rcookie, in segvn_setpagesize()
6201 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_setpagesize()
6202 } else if (svd->tr_state == SEGVN_TR_INIT) { in segvn_setpagesize()
6203 svd->tr_state = SEGVN_TR_OFF; in segvn_setpagesize()
6204 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_setpagesize()
6205 ASSERT(svd->amp != NULL); in segvn_setpagesize()
6207 ASSERT(svd->amp == NULL && svd->tr_state == SEGVN_TR_OFF); in segvn_setpagesize()
6301 if (!IS_P2ALIGNED(svd->anon_index, pgcnt)) { in segvn_setpagesize()
6304 ASSERT(svd->type == MAP_PRIVATE); in segvn_setpagesize()
6313 if (anon_copy_ptr(amp->ahp, svd->anon_index, in segvn_setpagesize()
6321 svd->anon_index = 0; in segvn_setpagesize()
6325 if (svd->vp != NULL && szc != 0) { in segvn_setpagesize()
6327 u_offset_t eoffpage = svd->offset; in segvn_setpagesize()
6331 if (VOP_GETATTR(svd->vp, &va, 0, svd->cred, NULL) != 0) { in segvn_setpagesize()
6349 amp->ahp, svd->anon_index, svd->vp, svd->offset, in segvn_setpagesize()
6350 seg->s_size, szc, svd->prot, svd->vpage, in segvn_setpagesize()
6351 svd->cred)) != 0) { in segvn_setpagesize()
6355 segvn_setvnode_mpss(svd->vp); in segvn_setpagesize()
6360 if (svd->type == MAP_PRIVATE) { in segvn_setpagesize()
6376 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_clrszc() local
6377 struct anon_map *amp = svd->amp; in segvn_clrszc()
6383 ulong_t an_idx = svd->anon_index; in segvn_clrszc()
6384 vnode_t *vp = svd->vp; in segvn_clrszc()
6385 struct vpage *vpage = svd->vpage; in segvn_clrszc()
6388 uint_t prot = svd->prot, vpprot; in segvn_clrszc()
6392 SEGVN_WRITE_HELD(seg->s_as, &svd->lock)); in segvn_clrszc()
6393 ASSERT(svd->softlockcnt == 0); in segvn_clrszc()
6396 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_clrszc()
6401 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_clrszc()
6402 ASSERT(svd->amp == NULL); in segvn_clrszc()
6403 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_clrszc()
6404 hat_leave_region(seg->s_as->a_hat, svd->rcookie, in segvn_clrszc()
6406 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_clrszc()
6407 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_clrszc()
6408 ASSERT(svd->amp != NULL); in segvn_clrszc()
6410 ASSERT(svd->amp == NULL && svd->tr_state == SEGVN_TR_OFF); in segvn_clrszc()
6413 if (svd->tr_state != SEGVN_TR_OFF) { in segvn_clrszc()
6414 ASSERT(svd->tr_state == SEGVN_TR_INIT); in segvn_clrszc()
6415 svd->tr_state = SEGVN_TR_OFF; in segvn_clrszc()
6427 if (amp == NULL || svd->type == MAP_SHARED) { in segvn_clrszc()
6443 ASSERT(vpage != NULL || svd->pageprot == 0); in segvn_clrszc()
6453 seg, a, prot, vpage, svd->cred)) != 0) { in segvn_clrszc()
6462 svd->cred))) { in segvn_clrszc()
6466 anon_pl[0], pageflag, svd->cred)) == NULL) { in segvn_clrszc()
6497 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_claim_pages() local
6498 struct anon_map *amp = svd->amp; in segvn_claim_pages()
6500 caddr_t addr = ((uintptr_t)(svp - svd->vpage) << PAGESHIFT) in segvn_claim_pages()
6503 struct vnode *vp = svd->vp; in segvn_claim_pages()
6510 ASSERT(svd->type == MAP_PRIVATE); in segvn_claim_pages()
6511 ASSERT(svd->vpage != NULL); in segvn_claim_pages()
6598 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_split_seg() local
6604 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_split_seg()
6608 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_split_seg()
6621 *nsvd = *svd; in segvn_split_seg()
6628 nsvd->offset = svd->offset + in segvn_split_seg()
6647 ASSERT(svd->softlockcnt == 0); in segvn_split_seg()
6648 ASSERT(svd->softlockcnt_sbase == 0); in segvn_split_seg()
6649 ASSERT(svd->softlockcnt_send == 0); in segvn_split_seg()
6650 crhold(svd->cred); in segvn_split_seg()
6652 if (svd->vpage != NULL) { in segvn_split_seg()
6655 struct vpage *ovpage = svd->vpage; in segvn_split_seg()
6657 svd->vpage = kmem_alloc(bytes, KM_SLEEP); in segvn_split_seg()
6658 bcopy(ovpage, svd->vpage, bytes); in segvn_split_seg()
6663 if (svd->amp != NULL && svd->type == MAP_PRIVATE) { in segvn_split_seg()
6664 struct anon_map *oamp = svd->amp, *namp; in segvn_split_seg()
6670 (void) anon_copy_ptr(oamp->ahp, svd->anon_index, in segvn_split_seg()
6676 svd->anon_index + btop(seg->s_size), in segvn_split_seg()
6681 svd->anon_index = 0; in segvn_split_seg()
6685 } else if (svd->amp != NULL) { in segvn_split_seg()
6687 ASSERT(svd->amp == nsvd->amp); in segvn_split_seg()
6688 ASSERT(seg->s_szc <= svd->amp->a_szc); in segvn_split_seg()
6689 nsvd->anon_index = svd->anon_index + seg_pages(seg); in segvn_split_seg()
6691 ANON_LOCK_ENTER(&svd->amp->a_rwlock, RW_WRITER); in segvn_split_seg()
6692 svd->amp->refcnt++; in segvn_split_seg()
6693 ANON_LOCK_EXIT(&svd->amp->a_rwlock); in segvn_split_seg()
6699 if (svd->swresv) { in segvn_split_seg()
6705 if (svd->flags & MAP_NORESERVE) { in segvn_split_seg()
6708 ASSERT(svd->amp); in segvn_split_seg()
6709 oswresv = svd->swresv; in segvn_split_seg()
6710 svd->swresv = ptob(anon_pages(svd->amp->ahp, in segvn_split_seg()
6711 svd->anon_index, btop(seg->s_size))); in segvn_split_seg()
6714 ASSERT(oswresv >= (svd->swresv + nsvd->swresv)); in segvn_split_seg()
6716 if (svd->pageswap) { in segvn_split_seg()
6717 svd->swresv = segvn_count_swap_by_vpages(seg); in segvn_split_seg()
6718 ASSERT(nsvd->swresv >= svd->swresv); in segvn_split_seg()
6719 nsvd->swresv -= svd->swresv; in segvn_split_seg()
6721 ASSERT(svd->swresv == seg->s_size + in segvn_split_seg()
6723 svd->swresv = seg->s_size; in segvn_split_seg()
6753 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_demote_range() local
6759 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_demote_range()
6764 ASSERT(svd->softlockcnt == 0); in segvn_demote_range()
6765 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_demote_range()
6766 ASSERT(szcvec == 0 || (flag == SDR_END && svd->type == MAP_SHARED)); in segvn_demote_range()
6824 ASSERT(svd->type == MAP_SHARED); in segvn_demote_range()
6864 ASSERT(svd->type == MAP_SHARED); in segvn_demote_range()
6884 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_checkprot() local
6889 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_checkprot()
6893 if (svd->pageprot == 0) { in segvn_checkprot()
6896 err = ((svd->prot & prot) != prot) ? EACCES : 0; in segvn_checkprot()
6897 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_checkprot()
6904 evp = &svd->vpage[seg_page(seg, addr + len)]; in segvn_checkprot()
6905 for (vp = &svd->vpage[seg_page(seg, addr)]; vp < evp; vp++) { in segvn_checkprot()
6907 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_checkprot()
6911 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_checkprot()
6918 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_getprot() local
6924 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_getprot()
6925 if (svd->pageprot == 0) { in segvn_getprot()
6927 protv[--pgno] = svd->prot; in segvn_getprot()
6934 protv[pgno] = VPP_PROT(&svd->vpage[pgno+pgoff]); in segvn_getprot()
6937 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_getprot()
6945 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_getoffset() local
6949 return (svd->offset + (uintptr_t)(addr - seg->s_base)); in segvn_getoffset()
6956 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_gettype() local
6960 return (svd->type | (svd->flags & (MAP_NORESERVE | MAP_TEXT | in segvn_gettype()
6968 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_getvp() local
6972 *vpp = svd->vp; in segvn_getvp()
6989 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_kluster() local
6999 SEGVN_LOCK_HELD(seg->s_as, &svd->lock)); in segvn_kluster()
7014 if (svd->advice == MADV_RANDOM || in segvn_kluster()
7015 svd->advice == MADV_SEQUENTIAL && delta < 0) in segvn_kluster()
7017 else if (svd->pageadvice && svd->vpage) { in segvn_kluster()
7020 bvpp = &svd->vpage[page]; in segvn_kluster()
7021 evpp = &svd->vpage[page + pd]; in segvn_kluster()
7030 if (svd->type == MAP_SHARED) in segvn_kluster()
7033 if ((amp = svd->amp) == NULL) in segvn_kluster()
7036 page += svd->anon_index; in segvn_kluster()
7096 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_swapout() local
7105 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_swapout()
7110 if ((amp = svd->amp) != NULL) in segvn_swapout()
7111 anon_index = svd->anon_index; in segvn_swapout()
7138 vp = svd->vp; in segvn_swapout()
7139 off = svd->offset + ptob(page); in segvn_swapout()
7144 vp = svd->vp; in segvn_swapout()
7145 off = svd->offset + ptob(page); in segvn_swapout()
7248 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_swapout()
7265 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_sync() local
7284 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_sync()
7286 if (svd->softlockcnt > 0) { in segvn_sync()
7291 if (svd->type == MAP_SHARED) { in segvn_sync()
7292 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7311 if (svd->softlockcnt > 0) { in segvn_sync()
7312 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7315 } else if (svd->type == MAP_SHARED && svd->amp != NULL && in segvn_sync()
7316 svd->amp->a_softlockcnt > 0) { in segvn_sync()
7323 if (svd->amp->a_softlockcnt > 0 || svd->softlockcnt > 0) { in segvn_sync()
7324 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7329 vpp = svd->vpage; in segvn_sync()
7330 offset = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_sync()
7343 if (svd->type != segtype) { in segvn_sync()
7344 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7348 if (svd->prot != pageprot) { in segvn_sync()
7349 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7352 prot = svd->prot; in segvn_sync()
7354 vpp = &svd->vpage[seg_page(seg, addr)]; in segvn_sync()
7356 } else if (svd->vp && svd->amp == NULL && in segvn_sync()
7363 err = VOP_PUTPAGE(svd->vp, (offset_t)offset, len, in segvn_sync()
7364 bflags, svd->cred, NULL); in segvn_sync()
7365 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7369 if ((amp = svd->amp) != NULL) in segvn_sync()
7370 anon_index = svd->anon_index + seg_page(seg, addr); in segvn_sync()
7381 vp = svd->vp; in segvn_sync()
7387 vp = svd->vp; in segvn_sync()
7417 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7438 } else if (svd->type == MAP_SHARED && amp != NULL) { in segvn_sync()
7445 ASSERT(svd->vp == NULL); in segvn_sync()
7474 svd->cred, NULL); in segvn_sync()
7480 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_sync()
7491 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_incore() local
7506 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_incore()
7507 if (svd->amp == NULL && svd->vp == NULL) { in segvn_incore()
7508 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_incore()
7515 start = svd->vp ? SEG_PAGE_VNODEBACKED : 0; in segvn_incore()
7517 amp = svd->amp; in segvn_incore()
7519 vpp = (svd->vpage) ? &svd->vpage[p]: NULL; in segvn_incore()
7526 anon_array_enter(amp, svd->anon_index + p, &cookie); in segvn_incore()
7527 ap = anon_get_ptr(amp->ahp, svd->anon_index + p); in segvn_incore()
7561 vp = svd->vp; in segvn_incore()
7562 offset = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_incore()
7598 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_incore()
7653 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_lockop() local
7688 if (svd->amp != NULL && svd->amp->a_sp != NULL) { in segvn_lockop()
7689 ASSERT(svd->type == MAP_SHARED); in segvn_lockop()
7690 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_lockop()
7691 sp = svd->amp->a_sp; in segvn_lockop()
7696 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_lockop()
7706 if (svd->type != segtype) { in segvn_lockop()
7707 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_lockop()
7710 if (svd->pageprot == 0 && svd->prot != pageprot) { in segvn_lockop()
7711 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_lockop()
7717 if (svd->tr_state == SEGVN_TR_INIT) { in segvn_lockop()
7718 svd->tr_state = SEGVN_TR_OFF; in segvn_lockop()
7719 } else if (svd->tr_state == SEGVN_TR_ON) { in segvn_lockop()
7720 ASSERT(svd->amp != NULL); in segvn_lockop()
7722 ASSERT(svd->amp == NULL && in segvn_lockop()
7723 svd->tr_state == SEGVN_TR_OFF); in segvn_lockop()
7733 if ((vpp = svd->vpage) == NULL) { in segvn_lockop()
7736 if (svd->vpage == NULL) { in segvn_lockop()
7737 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_lockop()
7741 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_lockop()
7751 if (op == MC_LOCK && svd->amp == NULL && svd->vp == NULL) { in segvn_lockop()
7752 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_lockop()
7753 svd->amp = anonmap_alloc(seg->s_size, 0, ANON_SLEEP); in segvn_lockop()
7754 svd->amp->a_szc = seg->s_szc; in segvn_lockop()
7757 if ((amp = svd->amp) != NULL) { in segvn_lockop()
7758 anon_index = svd->anon_index + seg_page(seg, addr); in segvn_lockop()
7761 offset = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_lockop()
7762 evp = &svd->vpage[seg_page(seg, addr + len)]; in segvn_lockop()
7771 for (vpp = &svd->vpage[seg_page(seg, addr)]; vpp < evp; in segvn_lockop()
7784 i_edx = svd->anon_index + seg_page(seg, addr + len); in segvn_lockop()
7815 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_lockop()
7824 for (vpp = &svd->vpage[seg_page(seg, addr)]; vpp < evp; in segvn_lockop()
7837 if (op == MC_LOCK && svd->vp == NULL && in segvn_lockop()
7838 ((svd->flags & MAP_NORESERVE) == 0) && in segvn_lockop()
7847 svd->cred); in segvn_lockop()
7874 if (svd->vp == NULL && in segvn_lockop()
7875 (svd->flags & MAP_NORESERVE)) { in segvn_lockop()
7880 vp = svd->vp; in segvn_lockop()
7888 vp = svd->vp; in segvn_lockop()
7912 S_OTHER, svd->cred, NULL); in segvn_lockop()
7939 if (error && svd->vp) { in segvn_lockop()
7941 if (VOP_GETATTR(svd->vp, &va, 0, in segvn_lockop()
7942 svd->cred, NULL) != 0) { in segvn_lockop()
7974 (svd->type == MAP_PRIVATE)); in segvn_lockop()
8060 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_lockop()
8086 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_advise() local
8104 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_advise()
8105 if (svd->tr_state != SEGVN_TR_OFF) { in segvn_advise()
8106 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8110 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_advise()
8122 (seg->s_szc != 0 || HAT_IS_REGION_COOKIE_VALID(svd->rcookie))) || in segvn_advise()
8125 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8138 if (svd->softlockcnt > 0) { in segvn_advise()
8143 if (svd->type == MAP_SHARED) { in segvn_advise()
8144 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8156 if (svd->softlockcnt > 0) { in segvn_advise()
8163 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8166 } else if (svd->type == MAP_SHARED && svd->amp != NULL && in segvn_advise()
8167 svd->amp->a_softlockcnt > 0) { in segvn_advise()
8177 amp = svd->amp; in segvn_advise()
8178 vp = svd->vp; in segvn_advise()
8190 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8201 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8210 svd->anon_index + page, len, behav, &purged); in segvn_advise()
8212 if (purged != 0 && (svd->flags & MAP_NORESERVE)) { in segvn_advise()
8224 atomic_add_long(&svd->swresv, -bytes); in segvn_advise()
8229 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8253 policy = lgrp_madv_to_policy(behav, len, svd->type); in segvn_advise()
8254 if (svd->type == MAP_SHARED) in segvn_advise()
8256 svd->anon_index, vp, svd->offset, len); in segvn_advise()
8264 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8269 &svd->policy_info, len); in segvn_advise()
8284 page_mark_migrate(seg, addr, len, amp, svd->anon_index, in segvn_advise()
8285 vp, svd->offset, 1); in segvn_advise()
8292 if (already_set || svd->type == MAP_SHARED) in segvn_advise()
8316 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8330 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_advise()
8336 svd->advice = (uchar_t)behav; in segvn_advise()
8337 svd->pageadvice = 0; in segvn_advise()
8357 if (svd->vpage == NULL) { in segvn_advise()
8358 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8388 policy = lgrp_madv_to_policy(behav, len, svd->type); in segvn_advise()
8390 anon_index = svd->anon_index + page; in segvn_advise()
8391 off = svd->offset + (uintptr_t)(addr - seg->s_base); in segvn_advise()
8393 if (svd->type == MAP_SHARED) in segvn_advise()
8398 (policy == svd->policy_info.mem_policy); in segvn_advise()
8413 if (svd->type == MAP_PRIVATE && in segvn_advise()
8415 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8423 page_mark_migrate(seg, addr, len, amp, svd->anon_index, in segvn_advise()
8424 vp, svd->offset, 1); in segvn_advise()
8431 if (already_set || svd->type == MAP_SHARED) in segvn_advise()
8434 if (HAT_IS_REGION_COOKIE_VALID(svd->rcookie)) { in segvn_advise()
8435 ASSERT(svd->amp == NULL); in segvn_advise()
8436 ASSERT(svd->tr_state == SEGVN_TR_OFF); in segvn_advise()
8437 ASSERT(svd->softlockcnt == 0); in segvn_advise()
8438 hat_leave_region(seg->s_as->a_hat, svd->rcookie, in segvn_advise()
8440 svd->rcookie = HAT_INVALID_REGION_COOKIE; in segvn_advise()
8455 if (svd->softlockcnt > 0) in segvn_advise()
8500 if (svd->softlockcnt > 0) in segvn_advise()
8530 &svd->policy_info, seg->s_size); in segvn_advise()
8553 &svd->lock); in segvn_advise()
8564 ASSERT(svd->rcookie == HAT_INVALID_REGION_COOKIE); in segvn_advise()
8569 bvpp = &svd->vpage[page]; in segvn_advise()
8570 evpp = &svd->vpage[page + (len >> PAGESHIFT)]; in segvn_advise()
8573 svd->advice = MADV_NORMAL; in segvn_advise()
8584 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_advise()
8596 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_inherit() local
8607 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_WRITER); in segvn_inherit()
8613 if (svd->tr_state != SEGVN_TR_OFF || in segvn_inherit()
8614 svd->type != MAP_PRIVATE || in segvn_inherit()
8615 svd->vp != NULL) { in segvn_inherit()
8624 if (svd->svn_inz == SEGVN_INZ_ALL) { in segvn_inherit()
8633 svd->svn_inz = SEGVN_INZ_ALL; in segvn_inherit()
8642 if (svd->vpage == NULL) { in segvn_inherit()
8644 if (svd->vpage == NULL) { in segvn_inherit()
8650 svd->svn_inz = SEGVN_INZ_VPP; in segvn_inherit()
8652 bvpp = &svd->vpage[page]; in segvn_inherit()
8653 evpp = &svd->vpage[page + (len >> PAGESHIFT)]; in segvn_inherit()
8659 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_inherit()
8669 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_vpage() local
8673 ASSERT(SEGVN_WRITE_HELD(seg->s_as, &svd->lock)); in segvn_vpage()
8679 if (svd->vpage == NULL) { in segvn_vpage()
8704 svd->pageadvice = 1; in segvn_vpage()
8705 svd->vpage = kmem_zalloc(mem_needed, KM_SLEEP); in segvn_vpage()
8706 evp = &svd->vpage[seg_page(seg, seg->s_base + seg->s_size)]; in segvn_vpage()
8707 for (vp = svd->vpage; vp < evp; vp++) { in segvn_vpage()
8708 VPP_SETPROT(vp, svd->prot); in segvn_vpage()
8709 VPP_SETADVICE(vp, svd->advice); in segvn_vpage()
8720 struct segvn_data *svd; in segvn_dump() local
8731 svd = (struct segvn_data *)seg->s_data; in segvn_dump()
8732 vp = svd->vp; in segvn_dump()
8733 off = offset = svd->offset; in segvn_dump()
8736 if ((amp = svd->amp) != NULL) { in segvn_dump()
8737 anon_index = svd->anon_index; in segvn_dump()
8745 if (amp && (ap = anon_get_ptr(svd->amp->ahp, anon_index++))) { in segvn_dump()
8748 vp = svd->vp; in segvn_dump()
8830 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_pagelock() local
8873 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_pagelock()
8881 if (svd->vp != NULL) { in segvn_pagelock()
8888 if ((amp = svd->amp) == NULL) { in segvn_pagelock()
8966 if (svd->type == MAP_PRIVATE) { in segvn_pagelock()
8974 ulong_t aix = svd->anon_index + seg_page(seg, addr); in segvn_pagelock()
8976 if (aaix < svd->anon_index) { in segvn_pagelock()
8984 if (svd->pageprot && lpgaddr != addr) { in segvn_pagelock()
8985 struct vpage *vp = &svd->vpage[seg_page(seg, lpgaddr)]; in segvn_pagelock()
8986 struct vpage *evp = &svd->vpage[seg_page(seg, addr)]; in segvn_pagelock()
9000 if (svd->type == MAP_PRIVATE) { in segvn_pagelock()
9005 ulong_t aix = svd->anon_index + in segvn_pagelock()
9021 if (svd->pageprot && lpgeaddr != addr + len) { in segvn_pagelock()
9025 vp = &svd->vpage[seg_page(seg, addr + len)]; in segvn_pagelock()
9026 evp = &svd->vpage[seg_page(seg, lpgeaddr)]; in segvn_pagelock()
9047 if (svd->type == MAP_SHARED) { in segvn_pagelock()
9050 ptob(svd->anon_index)); in segvn_pagelock()
9091 ASSERT(svd->type == MAP_SHARED); in segvn_pagelock()
9094 ptob(svd->anon_index)); in segvn_pagelock()
9110 ASSERT(svd->type == MAP_SHARED); in segvn_pagelock()
9111 ASSERT(svd->softlockcnt >= npages); in segvn_pagelock()
9112 atomic_add_long((ulong_t *)&svd->softlockcnt, -npages); in segvn_pagelock()
9116 ASSERT(svd->softlockcnt_sbase > 0); in segvn_pagelock()
9117 atomic_dec_ulong((ulong_t *)&svd->softlockcnt_sbase); in segvn_pagelock()
9120 ASSERT(svd->softlockcnt_send > 0); in segvn_pagelock()
9121 atomic_dec_ulong((ulong_t *)&svd->softlockcnt_send); in segvn_pagelock()
9133 if (svd->softlockcnt == 0) { in segvn_pagelock()
9153 ASSERT(svd->type == MAP_PRIVATE); in segvn_pagelock()
9157 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_pagelock()
9173 ASSERT(svd->type == MAP_SHARED); in segvn_pagelock()
9174 if (svd->pageprot == 0) { in segvn_pagelock()
9175 if ((svd->prot & protchk) == 0) { in segvn_pagelock()
9197 vp = &svd->vpage[seg_page(seg, a)]; in segvn_pagelock()
9213 ASSERT(svd->type == MAP_SHARED); in segvn_pagelock()
9214 atomic_add_long((ulong_t *)&svd->softlockcnt, in segvn_pagelock()
9218 atomic_inc_ulong((ulong_t *)&svd->softlockcnt_sbase); in segvn_pagelock()
9221 atomic_inc_ulong((ulong_t *)&svd->softlockcnt_send); in segvn_pagelock()
9223 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_pagelock()
9235 ASSERT(svd->type == MAP_PRIVATE); in segvn_pagelock()
9236 if (svd->pageprot == 0) { in segvn_pagelock()
9237 if ((svd->prot & protchk) == 0) { in segvn_pagelock()
9241 if (svd->prot & PROT_WRITE) { in segvn_pagelock()
9257 vp = &svd->vpage[seg_page(seg, a)]; in segvn_pagelock()
9313 anon_index = svd->anon_index + page; in segvn_pagelock()
9355 if (svd->vpage != NULL) { in segvn_pagelock()
9356 vpage = &svd->vpage[seg_page(seg, a)]; in segvn_pagelock()
9403 atomic_add_long((ulong_t *)&svd->softlockcnt, npages); in segvn_pagelock()
9405 ASSERT(svd->type == MAP_SHARED); in segvn_pagelock()
9411 atomic_inc_ulong((ulong_t *)&svd->softlockcnt_sbase); in segvn_pagelock()
9414 atomic_inc_ulong((ulong_t *)&svd->softlockcnt_send); in segvn_pagelock()
9420 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_pagelock()
9436 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_pagelock()
9449 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_purge() local
9454 if (svd->amp == NULL || svd->vp != NULL) { in segvn_purge()
9463 if (svd->type == MAP_PRIVATE) { in segvn_purge()
9464 if (svd->softlockcnt) { in segvn_purge()
9467 } else if (svd->softlockcnt == 0 && svd->amp->a_softlockcnt != 0) { in segvn_purge()
9468 seg_ppurge(seg, svd->amp, 0); in segvn_purge()
9483 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_reclaim() local
9490 ASSERT(svd->vp == NULL && svd->amp != NULL); in segvn_reclaim()
9491 ASSERT(svd->softlockcnt >= npages); in segvn_reclaim()
9531 mutex_enter(&svd->segfree_syncmtx); in segvn_reclaim()
9534 if (!atomic_add_long_nv((ulong_t *)&svd->softlockcnt, -npages)) { in segvn_reclaim()
9549 mutex_exit(&svd->segfree_syncmtx); in segvn_reclaim()
9609 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_getmemid() local
9615 if (svd->type == MAP_PRIVATE) { in segvn_getmemid()
9621 if (svd->type == MAP_SHARED) { in segvn_getmemid()
9622 if (svd->vp) { in segvn_getmemid()
9623 memidp->val[0] = (uintptr_t)svd->vp; in segvn_getmemid()
9624 memidp->val[1] = (u_longlong_t)svd->offset + in segvn_getmemid()
9629 SEGVN_LOCK_ENTER(seg->s_as, &svd->lock, RW_READER); in segvn_getmemid()
9630 if ((amp = svd->amp) != NULL) { in segvn_getmemid()
9631 anon_index = svd->anon_index + in segvn_getmemid()
9634 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_getmemid()
9644 pp = anon_zero(seg, addr, &ap, svd->cred); in segvn_getmemid()
9671 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in sameprot() local
9676 if (svd->pageprot == 0) in sameprot()
9679 ASSERT(svd->vpage != NULL); in sameprot()
9681 vpage = &svd->vpage[seg_page(seg, a)]; in sameprot()
9755 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_textrepl() local
9756 vnode_t *vp = svd->vp; in segvn_textrepl()
9757 u_offset_t off = svd->offset; in segvn_textrepl()
9771 ASSERT(SEGVN_WRITE_HELD(seg->s_as, &svd->lock)); in segvn_textrepl()
9773 ASSERT(svd->tr_state == SEGVN_TR_INIT); in segvn_textrepl()
9774 ASSERT(!HAT_IS_REGION_COOKIE_VALID(svd->rcookie)); in segvn_textrepl()
9775 ASSERT(svd->flags & MAP_TEXT); in segvn_textrepl()
9776 ASSERT(svd->type == MAP_PRIVATE); in segvn_textrepl()
9777 ASSERT(vp != NULL && svd->amp == NULL); in segvn_textrepl()
9778 ASSERT(!svd->pageprot && !(svd->prot & PROT_WRITE)); in segvn_textrepl()
9779 ASSERT(!(svd->flags & MAP_NORESERVE) && svd->swresv == 0); in segvn_textrepl()
9788 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9797 if (VOP_GETATTR(vp, &va, 0, svd->cred, NULL) != 0) { in segvn_textrepl()
9798 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9803 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9828 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9857 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9877 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9889 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
9994 ASSERT(svd->svn_trnext == NULL); in segvn_textrepl()
9995 ASSERT(svd->svn_trprev == NULL); in segvn_textrepl()
9996 svd->svn_trnext = svntrp->tr_svnhead; in segvn_textrepl()
9997 svd->svn_trprev = NULL; in segvn_textrepl()
9999 svntrp->tr_svnhead->svn_trprev = svd; in segvn_textrepl()
10001 svntrp->tr_svnhead = svd; in segvn_textrepl()
10004 svd->amp = amp; in segvn_textrepl()
10005 svd->anon_index = 0; in segvn_textrepl()
10006 svd->tr_policy_info.mem_policy = LGRP_MEM_POLICY_NEXT_SEG; in segvn_textrepl()
10007 svd->tr_policy_info.mem_lgrpid = lgrp_id; in segvn_textrepl()
10008 svd->tr_state = SEGVN_TR_ON; in segvn_textrepl()
10025 svd->tr_state = SEGVN_TR_OFF; in segvn_textrepl()
10039 struct segvn_data *svd = (struct segvn_data *)seg->s_data; in segvn_textunrepl() local
10040 vnode_t *vp = svd->vp; in segvn_textunrepl()
10041 u_offset_t off = svd->offset; in segvn_textunrepl()
10048 lgrp_id_t lgrp_id = svd->tr_policy_info.mem_lgrpid; in segvn_textunrepl()
10053 SEGVN_WRITE_HELD(seg->s_as, &svd->lock)); in segvn_textunrepl()
10054 ASSERT(svd->tr_state == SEGVN_TR_ON); in segvn_textunrepl()
10055 ASSERT(!HAT_IS_REGION_COOKIE_VALID(svd->rcookie)); in segvn_textunrepl()
10056 ASSERT(svd->amp != NULL); in segvn_textunrepl()
10057 ASSERT(svd->amp->refcnt >= 1); in segvn_textunrepl()
10058 ASSERT(svd->anon_index == 0); in segvn_textunrepl()
10074 if (svntrp->tr_amp[lgrp_id] != svd->amp) { in segvn_textunrepl()
10077 svd->tr_state = SEGVN_TR_OFF; in segvn_textunrepl()
10078 svd->amp = NULL; in segvn_textunrepl()
10079 if (svd->svn_trprev == NULL) { in segvn_textunrepl()
10080 ASSERT(svntrp->tr_svnhead == svd); in segvn_textunrepl()
10081 svntrp->tr_svnhead = svd->svn_trnext; in segvn_textunrepl()
10085 svd->svn_trnext = NULL; in segvn_textunrepl()
10087 svd->svn_trprev->svn_trnext = svd->svn_trnext; in segvn_textunrepl()
10088 if (svd->svn_trnext != NULL) { in segvn_textunrepl()
10089 svd->svn_trnext->svn_trprev = svd->svn_trprev; in segvn_textunrepl()
10090 svd->svn_trnext = NULL; in segvn_textunrepl()
10092 svd->svn_trprev = NULL; in segvn_textunrepl()
10210 segvn_data_t *svd; in segvn_trupdate() local
10219 svd = svntrp->tr_svnhead; in segvn_trupdate()
10220 for (; svd != NULL; svd = svd->svn_trnext) { in segvn_trupdate()
10221 segvn_trupdate_seg(svd->seg, svd, svntrp, in segvn_trupdate()
10231 segvn_data_t *svd, in segvn_trupdate_seg() argument
10241 ASSERT(svd->vp != NULL); in segvn_trupdate_seg()
10242 ASSERT(svd->vp == svntrp->tr_vp); in segvn_trupdate_seg()
10243 ASSERT(svd->offset == svntrp->tr_off); in segvn_trupdate_seg()
10244 ASSERT(svd->offset + seg->s_size == svntrp->tr_eoff); in segvn_trupdate_seg()
10246 ASSERT(svd->seg == seg); in segvn_trupdate_seg()
10247 ASSERT(seg->s_data == (void *)svd); in segvn_trupdate_seg()
10249 ASSERT(svd->tr_state == SEGVN_TR_ON); in segvn_trupdate_seg()
10250 ASSERT(!HAT_IS_REGION_COOKIE_VALID(svd->rcookie)); in segvn_trupdate_seg()
10251 ASSERT(svd->amp != NULL); in segvn_trupdate_seg()
10252 ASSERT(svd->tr_policy_info.mem_policy == LGRP_MEM_POLICY_NEXT_SEG); in segvn_trupdate_seg()
10253 ASSERT(svd->tr_policy_info.mem_lgrpid != LGRP_NONE); in segvn_trupdate_seg()
10254 ASSERT(svd->tr_policy_info.mem_lgrpid < NLGRPS_MAX); in segvn_trupdate_seg()
10255 ASSERT(svntrp->tr_amp[svd->tr_policy_info.mem_lgrpid] == svd->amp); in segvn_trupdate_seg()
10269 if (svd->tr_policy_info.mem_lgrpid == lgrp_id) { in segvn_trupdate_seg()
10284 if (!SEGVN_LOCK_TRYENTER(seg->s_as, &svd->lock, RW_WRITER)) { in segvn_trupdate_seg()
10296 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_trupdate_seg()
10303 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_trupdate_seg()
10311 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_trupdate_seg()
10334 ASSERT(svd->tr_state == SEGVN_TR_ON); in segvn_trupdate_seg()
10335 ASSERT(svd->amp != NULL); in segvn_trupdate_seg()
10336 ASSERT(svd->tr_policy_info.mem_policy == LGRP_MEM_POLICY_NEXT_SEG); in segvn_trupdate_seg()
10337 ASSERT(svd->tr_policy_info.mem_lgrpid != lgrp_id); in segvn_trupdate_seg()
10338 ASSERT(svd->amp != svntrp->tr_amp[lgrp_id]); in segvn_trupdate_seg()
10340 svd->tr_policy_info.mem_lgrpid = lgrp_id; in segvn_trupdate_seg()
10341 svd->amp = svntrp->tr_amp[lgrp_id]; in segvn_trupdate_seg()
10343 SEGVN_LOCK_EXIT(seg->s_as, &svd->lock); in segvn_trupdate_seg()
10347 ASSERT(svd->vp == svntrp->tr_vp); in segvn_trupdate_seg()
10348 ASSERT(svd->tr_policy_info.mem_lgrpid == lgrp_id); in segvn_trupdate_seg()
10349 ASSERT(svd->amp != NULL && svd->amp == svntrp->tr_amp[lgrp_id]); in segvn_trupdate_seg()
10350 ASSERT(svd->seg == seg); in segvn_trupdate_seg()
10351 ASSERT(svd->tr_state == SEGVN_TR_ON); in segvn_trupdate_seg()