Lines Matching refs:pp
575 page_t *pp, in add_physmem() argument
585 "add_physmem:pp %p num %lu", pp, num); in add_physmem()
603 for (; num; pp++, pnum++, num--) { in add_physmem()
609 add_physmem_cb(pp, pnum); in add_physmem()
611 pp->p_lckcnt = 0; in add_physmem()
612 pp->p_cowcnt = 0; in add_physmem()
613 pp->p_slckcnt = 0; in add_physmem()
619 pp->p_selock = 0; in add_physmem()
624 page_iolock_init(pp); in add_physmem()
629 PP_SETFREE(pp); in add_physmem()
630 page_clr_all_props(pp); in add_physmem()
631 PP_SETAGED(pp); in add_physmem()
632 pp->p_offset = (u_offset_t)-1; in add_physmem()
633 pp->p_next = pp; in add_physmem()
634 pp->p_prev = pp; in add_physmem()
640 pp->p_szc = 0; in add_physmem()
641 page_free_at_startup(pp); in add_physmem()
656 pp->p_szc = 0; in add_physmem()
657 page_free_at_startup(pp); in add_physmem()
664 pp->p_szc = szc; in add_physmem()
665 page_list_concat(&root, &pp); in add_physmem()
692 pp->p_szc = 0; in add_physmem()
693 page_free_at_startup(pp); in add_physmem()
700 pp->p_szc = szc; in add_physmem()
702 root = pp; in add_physmem()
743 page_t *pp; in page_lookup_create() local
763 pp = page_hash_search(index, vp, off); in page_lookup_create()
764 if (pp != NULL) { in page_lookup_create()
770 if (!page_try_reclaim_lock(pp, se, es)) { in page_lookup_create()
786 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { in page_lookup_create()
807 if (((volatile struct vnode *)(pp->p_vnode) != vp) || in page_lookup_create()
808 ((volatile u_offset_t)(pp->p_offset) != off)) { in page_lookup_create()
812 (void *)pp); in page_lookup_create()
815 page_unlock(pp); in page_lookup_create()
835 if ((!hash_locked) && (PP_ISFREE(pp))) { in page_lookup_create()
836 ASSERT(PP_ISAGED(pp) == 0); in page_lookup_create()
844 if (!page_reclaim(pp, NULL)) { in page_lookup_create()
856 page_downgrade(pp); in page_lookup_create()
864 if (newpp != NULL && pp->p_szc < newpp->p_szc && in page_lookup_create()
865 PAGE_EXCL(pp) && nrelocp != NULL) { in page_lookup_create()
867 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, in page_lookup_create()
874 pp = newpp; in page_lookup_create()
878 page_downgrade(pp); in page_lookup_create()
883 if (PAGE_EXCL(pp) && se == SE_SHARED) { in page_lookup_create()
884 page_downgrade(pp); in page_lookup_create()
886 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, in page_lookup_create()
888 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, in page_lookup_create()
890 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, in page_lookup_create()
892 } else if (newpp != NULL && PAGE_EXCL(pp)) { in page_lookup_create()
929 pp = newpp; in page_lookup_create()
936 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); in page_lookup_create()
938 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); in page_lookup_create()
940 return (pp); in page_lookup_create()
952 page_t *pp; in page_lookup_nowait() local
961 pp = page_hash_search(index, vp, off); in page_lookup_nowait()
963 if (pp == NULL) { in page_lookup_nowait()
969 pp = page_hash_search(index, vp, off); in page_lookup_nowait()
972 if (pp == NULL || PP_ISFREE(pp)) { in page_lookup_nowait()
974 pp = NULL; in page_lookup_nowait()
976 if (!page_trylock(pp, se)) { in page_lookup_nowait()
978 pp = NULL; in page_lookup_nowait()
984 if (((volatile struct vnode *)(pp->p_vnode) != vp) || in page_lookup_nowait()
985 ((u_offset_t)(pp->p_offset) != off)) { in page_lookup_nowait()
989 (void *)pp); in page_lookup_nowait()
992 page_unlock(pp); in page_lookup_nowait()
995 if (PP_ISFREE(pp)) { in page_lookup_nowait()
997 page_unlock(pp); in page_lookup_nowait()
998 pp = NULL; in page_lookup_nowait()
1007 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); in page_lookup_nowait()
1009 return (pp); in page_lookup_nowait()
1020 page_t *pp; in page_find() local
1031 pp = page_hash_search(index, vp, off); in page_find()
1034 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); in page_find()
1035 return (pp); in page_find()
1084 page_t *pp; in page_exists_physcontig() local
1103 pp = page_hash_search(index, vp, off); in page_exists_physcontig()
1108 if (pp == NULL) { in page_exists_physcontig()
1114 rootpp = pp; in page_exists_physcontig()
1117 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { in page_exists_physcontig()
1119 if (!page_trylock(pp, SE_SHARED)) { in page_exists_physcontig()
1126 if (pp->p_szc != pszc || pp->p_vnode != vp || in page_exists_physcontig()
1127 pp->p_offset != off || pp->p_pagenum != pfn) { in page_exists_physcontig()
1129 page_unlock(pp); in page_exists_physcontig()
1137 ASSERT(!PP_ISFREE(pp)); in page_exists_physcontig()
1139 page_unlock(pp); in page_exists_physcontig()
1142 ppa[0] = pp; in page_exists_physcontig()
1143 pp++; in page_exists_physcontig()
1146 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { in page_exists_physcontig()
1147 if (!page_trylock(pp, SE_SHARED)) { in page_exists_physcontig()
1149 pp--; in page_exists_physcontig()
1151 page_unlock(pp); in page_exists_physcontig()
1152 pp--; in page_exists_physcontig()
1157 if (pp->p_szc != pszc) { in page_exists_physcontig()
1159 page_unlock(pp); in page_exists_physcontig()
1160 pp--; in page_exists_physcontig()
1162 page_unlock(pp); in page_exists_physcontig()
1163 pp--; in page_exists_physcontig()
1175 ASSERT(!PP_ISFREE(pp)); in page_exists_physcontig()
1176 if (pp->p_vnode != vp || in page_exists_physcontig()
1177 pp->p_offset != off) { in page_exists_physcontig()
1181 ppa[i] = pp; in page_exists_physcontig()
1182 ASSERT(pp->p_pagenum == pfn); in page_exists_physcontig()
1213 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { in page_exists_physcontig()
1214 if (!page_trylock(pp, SE_EXCL)) { in page_exists_physcontig()
1221 if (pp->p_pagenum != pfn) { in page_exists_physcontig()
1222 page_unlock(pp); in page_exists_physcontig()
1225 if (pp->p_vnode != vp || in page_exists_physcontig()
1226 pp->p_offset != off) { in page_exists_physcontig()
1228 page_unlock(pp); in page_exists_physcontig()
1231 if (pp->p_szc >= szc) { in page_exists_physcontig()
1233 page_unlock(pp); in page_exists_physcontig()
1241 --pp; in page_exists_physcontig()
1243 page_unlock(pp); in page_exists_physcontig()
1244 --pp; in page_exists_physcontig()
1249 pp = rootpp; in page_exists_physcontig()
1250 for (i = 0; i < pages; i++, pp++) { in page_exists_physcontig()
1251 if (PP_ISFREE(pp)) { in page_exists_physcontig()
1253 ASSERT(!PP_ISAGED(pp)); in page_exists_physcontig()
1254 ASSERT(pp->p_szc == 0); in page_exists_physcontig()
1255 if (!page_reclaim(pp, NULL)) { in page_exists_physcontig()
1259 ASSERT(pp->p_szc < szc); in page_exists_physcontig()
1261 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_exists_physcontig()
1271 pp = rootpp; in page_exists_physcontig()
1272 for (j = 0; j < pages; j++, pp++) { in page_exists_physcontig()
1274 page_unlock(pp); in page_exists_physcontig()
1281 pp = rootpp; in page_exists_physcontig()
1282 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { in page_exists_physcontig()
1283 ASSERT(PAGE_EXCL(pp)); in page_exists_physcontig()
1284 ASSERT(!PP_ISFREE(pp)); in page_exists_physcontig()
1285 ASSERT(!hat_page_is_mapped(pp)); in page_exists_physcontig()
1286 ASSERT(pp->p_vnode == vp); in page_exists_physcontig()
1287 ASSERT(pp->p_offset == off); in page_exists_physcontig()
1288 pp->p_szc = szc; in page_exists_physcontig()
1290 pp = rootpp; in page_exists_physcontig()
1291 for (i = 0; i < pages; i++, pp++) { in page_exists_physcontig()
1293 page_unlock(pp); in page_exists_physcontig()
1295 ppa[i] = pp; in page_exists_physcontig()
1317 page_t *pp; in page_exists_forreal() local
1330 pp = page_hash_search(index, vp, off); in page_exists_forreal()
1331 if (pp != NULL) { in page_exists_forreal()
1332 *szc = pp->p_szc; in page_exists_forreal()
1778 page_t *pp; in page_create_get_something() local
1870 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, in page_create_get_something()
1872 if (pp == NULL) { in page_create_get_something()
1873 pp = page_get_cachelist(vp, off, seg, vaddr, in page_create_get_something()
1876 if (pp == NULL) { in page_create_get_something()
1935 return (pp); in page_create_get_something()
2011 page_t *pplist = NULL, *pp; in page_alloc_pages() local
2059 pp = page_get_freelist(vp, 0, seg, addr, pgsz, in page_alloc_pages()
2061 if (pp == NULL) { in page_alloc_pages()
2062 pp = page_get_freelist(vp, 0, seg, addr, pgsz, in page_alloc_pages()
2066 pp = page_get_freelist(vp, 0, seg, addr, pgsz, in page_alloc_pages()
2069 if (pp != NULL) { in page_alloc_pages()
2071 page_list_concat(&pplist, &pp); in page_alloc_pages()
2098 pp = pplist; in page_alloc_pages()
2106 ASSERT(PP_ISFREE(pp)); in page_alloc_pages()
2107 ASSERT(PP_ISAGED(pp)); in page_alloc_pages()
2111 PP_CLRFREE(pp); in page_alloc_pages()
2112 PP_CLRAGED(pp); in page_alloc_pages()
2113 page_sub(&pplist, pp); in page_alloc_pages()
2114 *ppa++ = pp; in page_alloc_pages()
2118 ASSERT(pp->p_szc != 0); in page_alloc_pages()
2119 curnpgs = page_get_pagecnt(pp->p_szc); in page_alloc_pages()
2120 page_list_break(&pp, &pplist, curnpgs); in page_alloc_pages()
2121 page_list_add_pages(pp, 0); in page_alloc_pages()
2126 pp = pplist; in page_alloc_pages()
2129 PP_CLRFREE(pp); in page_alloc_pages()
2130 PP_CLRAGED(pp); in page_alloc_pages()
2131 pp = pp->p_next; in page_alloc_pages()
2152 page_t *pp; in page_create_va_large() local
2253 pp = rootpp; in page_create_va_large()
2255 ASSERT(PAGE_EXCL(pp)); in page_create_va_large()
2256 ASSERT(pp->p_vnode == NULL); in page_create_va_large()
2257 ASSERT(!hat_page_is_mapped(pp)); in page_create_va_large()
2258 PP_CLRFREE(pp); in page_create_va_large()
2259 PP_CLRAGED(pp); in page_create_va_large()
2260 if (!page_hashin(pp, vp, off, NULL)) in page_create_va_large()
2262 (void *)pp); in page_create_va_large()
2263 page_io_lock(pp); in page_create_va_large()
2265 pp = pp->p_next; in page_create_va_large()
2391 page_t *pp; in page_create_va() local
2460 pp = page_hash_search(index, vp, off); in page_create_va()
2461 if (pp == NULL) { in page_create_va()
2463 pp = npp; in page_create_va()
2465 if (!page_hashin(pp, vp, off, phm)) { in page_create_va()
2478 (void *)pp, (void *)vp, off, (void *)phm); in page_create_va()
2496 page_set_props(pp, P_REF); in page_create_va()
2509 pp = plist; in page_create_va()
2510 page_sub(&plist, pp); in page_create_va()
2511 page_io_unlock(pp); in page_create_va()
2513 ASSERT(pp->p_szc == 0); in page_create_va()
2515 VN_DISPOSE(pp, B_INVAL, 0, kcred); in page_create_va()
2521 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { in page_create_va()
2534 if (PP_ISFREE(pp)) { in page_create_va()
2535 ASSERT(PP_ISAGED(pp) == 0); in page_create_va()
2537 page_list_sub(pp, PG_CACHE_LIST); in page_create_va()
2538 PP_CLRFREE(pp); in page_create_va()
2548 page_io_lock(pp); in page_create_va()
2549 page_add(&plist, pp); in page_create_va()
2641 page_free(page_t *pp, int dontneed) in page_free() argument
2646 ASSERT((PAGE_EXCL(pp) && in page_free()
2647 !page_iolock_assert(pp)) || panicstr); in page_free()
2649 if (PP_ISFREE(pp)) { in page_free()
2650 panic("page_free: page %p is free", (void *)pp); in page_free()
2653 if (pp->p_szc != 0) { in page_free()
2654 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || in page_free()
2655 PP_ISKAS(pp)) { in page_free()
2657 "or no vnode large page %p", (void *)pp); in page_free()
2659 page_demote_vp_pages(pp); in page_free()
2660 ASSERT(pp->p_szc == 0); in page_free()
2667 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || in page_free()
2668 pp->p_slckcnt != 0) { in page_free()
2670 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt, in page_free()
2671 pp->p_cowcnt, pp->p_slckcnt); in page_free()
2675 ASSERT(!hat_page_getshare(pp)); in page_free()
2677 PP_SETFREE(pp); in page_free()
2678 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || in page_free()
2679 !hat_ismod(pp)); in page_free()
2680 page_clr_all_props(pp); in page_free()
2681 ASSERT(!hat_page_getshare(pp)); in page_free()
2689 if (pp->p_vnode == NULL) { in page_free()
2693 PP_SETAGED(pp); in page_free()
2694 pp->p_offset = (u_offset_t)-1; in page_free()
2695 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_free()
2698 "page_free_free:pp %p", pp); in page_free()
2700 PP_CLRAGED(pp); in page_free()
2704 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); in page_free()
2708 "page_free_cache_tail:pp %p", pp); in page_free()
2710 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); in page_free()
2714 "page_free_cache_head:pp %p", pp); in page_free()
2717 page_unlock(pp); in page_free()
2763 page_free_at_startup(page_t *pp) in page_free_at_startup() argument
2768 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); in page_free_at_startup()
2786 page_free_pages(page_t *pp) in page_free_pages() argument
2789 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); in page_free_pages()
2791 uint_t szc = pp->p_szc; in page_free_pages()
2795 "page_free_free:pp %p", pp); in page_free_pages()
2797 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); in page_free_pages()
2798 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { in page_free_pages()
2799 panic("page_free_pages: not root page %p", (void *)pp); in page_free_pages()
2803 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { in page_free_pages()
2828 ASSERT(rootpp == pp); in page_free_pages()
2849 page_t *pp; in free_vp_pages() local
2866 pp = page_exists(vp, off); in free_vp_pages()
2873 if (pp == NULL || in free_vp_pages()
2874 PP_ISFREE(pp) || in free_vp_pages()
2875 page_share_cnt(pp) > 0 || in free_vp_pages()
2876 !page_trylock(pp, SE_EXCL)) in free_vp_pages()
2883 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); in free_vp_pages()
2884 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { in free_vp_pages()
2885 page_unlock(pp); in free_vp_pages()
2892 (void) page_release(pp, 1); in free_vp_pages()
2913 page_reclaim(page_t *pp, kmutex_t *lock) in page_reclaim() argument
2921 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); in page_reclaim()
2994 page_unlock(pp); in page_reclaim()
3038 if (PP_ISAGED(pp)) { in page_reclaim()
3039 page_list_sub(pp, PG_FREE_LIST); in page_reclaim()
3041 "page_reclaim_free:pp %p", pp); in page_reclaim()
3043 page_list_sub(pp, PG_CACHE_LIST); in page_reclaim()
3045 "page_reclaim_cache:pp %p", pp); in page_reclaim()
3055 PP_CLRFREE(pp); in page_reclaim()
3056 PP_CLRAGED(pp); in page_reclaim()
3057 page_set_props(pp, P_REF); in page_reclaim()
3064 ASSERT(pp->p_szc == 0); in page_reclaim()
3075 page_destroy(page_t *pp, int dontfree) in page_destroy() argument
3077 ASSERT((PAGE_EXCL(pp) && in page_destroy()
3078 !page_iolock_assert(pp)) || panicstr); in page_destroy()
3079 ASSERT(pp->p_slckcnt == 0 || panicstr); in page_destroy()
3081 if (pp->p_szc != 0) { in page_destroy()
3082 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || in page_destroy()
3083 PP_ISKAS(pp)) { in page_destroy()
3085 "large page %p", (void *)pp); in page_destroy()
3087 page_demote_vp_pages(pp); in page_destroy()
3088 ASSERT(pp->p_szc == 0); in page_destroy()
3091 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); in page_destroy()
3097 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_destroy()
3098 page_hashout(pp, NULL); in page_destroy()
3107 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { in page_destroy()
3109 if (pp->p_lckcnt != 0) { in page_destroy()
3112 pp->p_lckcnt = 0; in page_destroy()
3114 if (pp->p_cowcnt != 0) { in page_destroy()
3115 availrmem += pp->p_cowcnt; in page_destroy()
3116 pages_locked -= pp->p_cowcnt; in page_destroy()
3117 pp->p_cowcnt = 0; in page_destroy()
3124 page_free(pp, 0); in page_destroy()
3129 page_destroy_pages(page_t *pp) in page_destroy_pages() argument
3133 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); in page_destroy_pages()
3135 uint_t szc = pp->p_szc; in page_destroy_pages()
3137 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); in page_destroy_pages()
3141 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); in page_destroy_pages()
3143 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { in page_destroy_pages()
3144 panic("page_destroy_pages: not root page %p", (void *)pp); in page_destroy_pages()
3148 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { in page_destroy_pages()
3174 ASSERT(rootpp == pp); in page_destroy_pages()
3194 page_destroy_free(page_t *pp) in page_destroy_free() argument
3196 ASSERT(PAGE_EXCL(pp)); in page_destroy_free()
3197 ASSERT(PP_ISFREE(pp)); in page_destroy_free()
3198 ASSERT(pp->p_vnode); in page_destroy_free()
3199 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); in page_destroy_free()
3200 ASSERT(!hat_page_is_mapped(pp)); in page_destroy_free()
3201 ASSERT(PP_ISAGED(pp) == 0); in page_destroy_free()
3202 ASSERT(pp->p_szc == 0); in page_destroy_free()
3205 page_list_sub(pp, PG_CACHE_LIST); in page_destroy_free()
3207 page_hashout(pp, NULL); in page_destroy_free()
3208 ASSERT(pp->p_vnode == NULL); in page_destroy_free()
3209 ASSERT(pp->p_offset == (u_offset_t)-1); in page_destroy_free()
3210 ASSERT(pp->p_hash == NULL); in page_destroy_free()
3212 PP_SETAGED(pp); in page_destroy_free()
3213 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_destroy_free()
3214 page_unlock(pp); in page_destroy_free()
3239 page_t *pp; in page_rename() local
3290 pp = page_hash_search(index, vp, off); in page_rename()
3291 if (pp != NULL) { in page_rename()
3302 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { in page_rename()
3320 if (hat_page_is_mapped(pp)) { in page_rename()
3329 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_rename()
3330 if (pp->p_szc != 0) { in page_rename()
3333 page_demote_vp_pages(pp); in page_rename()
3334 ASSERT(pp->p_szc == 0); in page_rename()
3337 } else if (pp->p_szc != 0) { in page_rename()
3341 page_demote_vp_pages(pp); in page_rename()
3342 ASSERT(pp->p_szc == 0); in page_rename()
3345 page_hashout(pp, phm); in page_rename()
3357 panic("page_rename: Can't hash in page: %p", (void *)pp); in page_rename()
3368 if (pp != NULL) { in page_rename()
3369 ASSERT(!hat_page_is_mapped(pp)); in page_rename()
3371 ASSERT(pp->p_szc == 0); in page_rename()
3378 olckcnt = pp->p_lckcnt; in page_rename()
3379 ocowcnt = pp->p_cowcnt; in page_rename()
3380 pp->p_lckcnt = pp->p_cowcnt = 0; in page_rename()
3387 VN_DISPOSE(pp, B_FREE, 0, kcred); in page_rename()
3409 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) in page_do_hashin() argument
3415 ASSERT(PAGE_EXCL(pp)); in page_do_hashin()
3425 pp->p_vnode = vp; in page_do_hashin()
3426 pp->p_offset = offset; in page_do_hashin()
3432 PP_SETSWAP(pp); in page_do_hashin()
3443 pp->p_vnode = NULL; in page_do_hashin()
3444 pp->p_offset = (u_offset_t)(-1); in page_do_hashin()
3448 pp->p_hash = *listp; in page_do_hashin()
3449 *listp = pp; in page_do_hashin()
3454 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) in page_do_hashin()
3459 page_vpadd(listp, pp); in page_do_hashin()
3471 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) in page_hashin() argument
3478 ASSERT(pp->p_fsdata == 0 || panicstr); in page_hashin()
3482 pp, vp, offset); in page_hashin()
3496 rc = page_do_hashin(pp, vp, offset); in page_hashin()
3510 page_do_hashout(page_t *pp) in page_do_hashout() argument
3514 vnode_t *vp = pp->p_vnode; in page_do_hashout()
3522 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; in page_do_hashout()
3526 if (hp == pp) in page_do_hashout()
3534 *hpp = pp->p_hash; in page_do_hashout()
3540 page_vpsub(&vp->v_pages, pp); in page_do_hashout()
3542 pp->p_hash = NULL; in page_do_hashout()
3543 page_clr_all_props(pp); in page_do_hashout()
3544 PP_CLRSWAP(pp); in page_do_hashout()
3545 pp->p_vnode = NULL; in page_do_hashout()
3546 pp->p_offset = (u_offset_t)-1; in page_do_hashout()
3547 pp->p_fsdata = 0; in page_do_hashout()
3557 page_hashout(page_t *pp, kmutex_t *phm) in page_hashout() argument
3566 ASSERT(pp->p_vnode != NULL); in page_hashout()
3567 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); in page_hashout()
3568 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); in page_hashout()
3570 vp = pp->p_vnode; in page_hashout()
3573 "page_hashout:pp %p vp %p", pp, vp); in page_hashout()
3579 index = PAGE_HASH_FUNC(vp, pp->p_offset); in page_hashout()
3594 page_do_hashout(pp); in page_hashout()
3605 sep = page_se_mutex(pp); in page_hashout()
3607 pp->p_selock &= ~SE_EWANTED; in page_hashout()
3608 if (CV_HAS_WAITERS(&pp->p_cv)) in page_hashout()
3609 cv_broadcast(&pp->p_cv); in page_hashout()
3619 page_add(page_t **ppp, page_t *pp) in page_add() argument
3621 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); in page_add()
3623 page_add_common(ppp, pp); in page_add()
3632 page_add_common(page_t **ppp, page_t *pp) in page_add_common() argument
3635 pp->p_next = pp->p_prev = pp; in page_add_common()
3637 pp->p_next = *ppp; in page_add_common()
3638 pp->p_prev = (*ppp)->p_prev; in page_add_common()
3639 (*ppp)->p_prev = pp; in page_add_common()
3640 pp->p_prev->p_next = pp; in page_add_common()
3642 *ppp = pp; in page_add_common()
3653 page_sub(page_t **ppp, page_t *pp) in page_sub() argument
3655 ASSERT(pp != NULL && (PP_ISFREE(pp)) ? 1 : in page_sub()
3656 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); in page_sub()
3658 if (*ppp == NULL || pp == NULL) { in page_sub()
3660 (void *)pp, (void *)(*ppp)); in page_sub()
3664 page_sub_common(ppp, pp); in page_sub()
3672 page_sub_common(page_t **ppp, page_t *pp) in page_sub_common() argument
3674 if (*ppp == pp) in page_sub_common()
3675 *ppp = pp->p_next; /* go to next page */ in page_sub_common()
3677 if (*ppp == pp) in page_sub_common()
3680 pp->p_prev->p_next = pp->p_next; in page_sub_common()
3681 pp->p_next->p_prev = pp->p_prev; in page_sub_common()
3683 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ in page_sub_common()
3758 page_list_next(page_t *pp) in page_list_next() argument
3760 return (pp->p_next); in page_list_next()
3771 page_vpadd(page_t **ppp, page_t *pp) in page_vpadd() argument
3774 pp->p_vpnext = pp->p_vpprev = pp; in page_vpadd()
3776 pp->p_vpnext = *ppp; in page_vpadd()
3777 pp->p_vpprev = (*ppp)->p_vpprev; in page_vpadd()
3778 (*ppp)->p_vpprev = pp; in page_vpadd()
3779 pp->p_vpprev->p_vpnext = pp; in page_vpadd()
3781 *ppp = pp; in page_vpadd()
3791 page_vpsub(page_t **ppp, page_t *pp) in page_vpsub() argument
3793 if (*ppp == NULL || pp == NULL) { in page_vpsub()
3795 (void *)pp, (void *)(*ppp)); in page_vpsub()
3799 if (*ppp == pp) in page_vpsub()
3800 *ppp = pp->p_vpnext; /* go to next page */ in page_vpsub()
3802 if (*ppp == pp) in page_vpsub()
3805 pp->p_vpprev->p_vpnext = pp->p_vpnext; in page_vpsub()
3806 pp->p_vpnext->p_vpprev = pp->p_vpprev; in page_vpsub()
3808 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ in page_vpsub()
3819 page_t *pp, /* page to be locked */ in page_pp_lock() argument
3825 ASSERT(PAGE_LOCKED(pp)); in page_pp_lock()
3827 page_struct_lock(pp); in page_pp_lock()
3834 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { in page_pp_lock()
3839 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_pp_lock()
3842 page_pptonum(pp)); in page_pp_lock()
3847 if (pp->p_lckcnt) { in page_pp_lock()
3848 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { in page_pp_lock()
3850 if (++pp->p_lckcnt == in page_pp_lock()
3854 page_pptonum(pp)); in page_pp_lock()
3860 ++pp->p_lckcnt; in page_pp_lock()
3867 ++pp->p_lckcnt; in page_pp_lock()
3874 page_struct_unlock(pp); in page_pp_lock()
3884 page_t *pp, /* page to be unlocked */ in page_pp_unlock() argument
3888 ASSERT(PAGE_LOCKED(pp)); in page_pp_unlock()
3890 page_struct_lock(pp); in page_pp_unlock()
3900 if (pp->p_cowcnt) { in page_pp_unlock()
3902 pp->p_cowcnt--; in page_pp_unlock()
3908 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { in page_pp_unlock()
3917 page_struct_unlock(pp); in page_pp_unlock()
4095 page_addclaim(page_t *pp) in page_addclaim() argument
4099 ASSERT(PAGE_LOCKED(pp)); in page_addclaim()
4101 page_struct_lock(pp); in page_addclaim()
4102 ASSERT(pp->p_lckcnt != 0); in page_addclaim()
4104 if (pp->p_lckcnt == 1) { in page_addclaim()
4105 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { in page_addclaim()
4106 --pp->p_lckcnt; in page_addclaim()
4108 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_addclaim()
4111 page_pptonum(pp)); in page_addclaim()
4117 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { in page_addclaim()
4121 --pp->p_lckcnt; in page_addclaim()
4123 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_addclaim()
4126 page_pptonum(pp)); in page_addclaim()
4131 page_struct_unlock(pp); in page_addclaim()
4136 page_subclaim(page_t *pp) in page_subclaim() argument
4140 ASSERT(PAGE_LOCKED(pp)); in page_subclaim()
4142 page_struct_lock(pp); in page_subclaim()
4143 ASSERT(pp->p_cowcnt != 0); in page_subclaim()
4145 if (pp->p_lckcnt) { in page_subclaim()
4146 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { in page_subclaim()
4156 pp->p_cowcnt--; in page_subclaim()
4158 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_subclaim()
4161 page_pptonum(pp)); in page_subclaim()
4166 pp->p_cowcnt--; in page_subclaim()
4167 pp->p_lckcnt++; in page_subclaim()
4169 page_struct_unlock(pp); in page_subclaim()
4267 page_t *pp; in page_numtopp() local
4270 pp = page_numtopp_nolock(pfnum); in page_numtopp()
4271 if (pp == NULL) { in page_numtopp()
4278 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { in page_numtopp()
4279 if (page_pptonum(pp) != pfnum) in page_numtopp()
4284 if (page_pptonum(pp) != pfnum) { in page_numtopp()
4285 page_unlock(pp); in page_numtopp()
4289 return (pp); in page_numtopp()
4295 page_t *pp; in page_numtopp_noreclaim() local
4298 pp = page_numtopp_nolock(pfnum); in page_numtopp_noreclaim()
4299 if (pp == NULL) { in page_numtopp_noreclaim()
4306 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { in page_numtopp_noreclaim()
4307 if (page_pptonum(pp) != pfnum) in page_numtopp_noreclaim()
4312 if (page_pptonum(pp) != pfnum) { in page_numtopp_noreclaim()
4313 page_unlock(pp); in page_numtopp_noreclaim()
4317 return (pp); in page_numtopp_noreclaim()
4327 page_t *pp; in page_numtopp_nowait() local
4330 pp = page_numtopp_nolock(pfnum); in page_numtopp_nowait()
4331 if (pp == NULL) { in page_numtopp_nowait()
4338 if (PP_ISFREE(pp)) in page_numtopp_nowait()
4339 pp = NULL; in page_numtopp_nowait()
4341 if (!page_trylock(pp, se)) in page_numtopp_nowait()
4342 pp = NULL; in page_numtopp_nowait()
4344 if (page_pptonum(pp) != pfnum) { in page_numtopp_nowait()
4345 page_unlock(pp); in page_numtopp_nowait()
4348 if (PP_ISFREE(pp)) { in page_numtopp_nowait()
4349 page_unlock(pp); in page_numtopp_nowait()
4350 pp = NULL; in page_numtopp_nowait()
4354 return (pp); in page_numtopp_nowait()
4365 page_t *pp = page0; in page_busy() local
4370 vnode_t *vp = pp->p_vnode; in page_busy()
4384 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && in page_busy()
4385 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && in page_busy()
4391 if (!page_trylock(pp, SE_EXCL)) in page_busy()
4394 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || in page_busy()
4395 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || in page_busy()
4396 !(hat_pagesync(pp, in page_busy()
4398 page_unlock(pp); in page_busy()
4401 off = pp->p_offset; in page_busy()
4403 page_unlock(pp); in page_busy()
4408 } while ((pp = page_next(pp)) != page0); in page_busy()
4436 page_t *pp; in page_invalidate_pages() local
4447 pp = page0 = page_first(); in page_invalidate_pages()
4457 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) in page_invalidate_pages()
4463 if (PP_ISFREE(pp) && PP_ISAGED(pp)) in page_invalidate_pages()
4475 if (!page_trylock(pp, SE_EXCL)) { in page_invalidate_pages()
4478 } else if (PP_ISFREE(pp)) { in page_invalidate_pages()
4479 if (!PP_ISAGED(pp)) { in page_invalidate_pages()
4480 page_destroy_free(pp); in page_invalidate_pages()
4482 page_unlock(pp); in page_invalidate_pages()
4493 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { in page_invalidate_pages()
4494 page_unlock(pp); in page_invalidate_pages()
4503 if (!page_try_demote_pages(pp)) { in page_invalidate_pages()
4504 page_unlock(pp); in page_invalidate_pages()
4512 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) in page_invalidate_pages()
4515 offset = pp->p_offset; in page_invalidate_pages()
4522 page_unlock(pp); in page_invalidate_pages()
4533 VN_DISPOSE(pp, B_INVAL, 0, kcred); in page_invalidate_pages()
4535 } while ((pp = page_next(pp)) != page0); in page_invalidate_pages()
4709 group_page_trylock(page_t *pp, se_t se) in group_page_trylock() argument
4713 uint_t pszc = pp->p_szc; in group_page_trylock()
4721 if (pp != PP_GROUPLEADER(pp, pszc)) { in group_page_trylock()
4726 ASSERT(PAGE_LOCKED_SE(pp, se)); in group_page_trylock()
4727 ASSERT(!PP_ISFREE(pp)); in group_page_trylock()
4732 tpp = pp + 1; in group_page_trylock()
4735 tpp = pp + 1; in group_page_trylock()
4742 if (pp->p_szc != pszc) { in group_page_trylock()
4743 ASSERT(pp->p_szc < pszc); in group_page_trylock()
4744 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && in group_page_trylock()
4745 !IS_SWAPFSVP(pp->p_vnode)); in group_page_trylock()
4746 tpp = pp + 1; in group_page_trylock()
4750 pszc = pp->p_szc; in group_page_trylock()
4757 group_page_unlock(page_t *pp) in group_page_unlock() argument
4762 ASSERT(PAGE_LOCKED(pp)); in group_page_unlock()
4763 ASSERT(!PP_ISFREE(pp)); in group_page_unlock()
4764 ASSERT(pp == PP_PAGEROOT(pp)); in group_page_unlock()
4765 npgs = page_get_pagecnt(pp->p_szc); in group_page_unlock()
4766 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { in group_page_unlock()
5049 page_t *pp; in page_free_replacement_page() local
5055 pp = pplist; in page_free_replacement_page()
5056 if (pp->p_szc == 0) { in page_free_replacement_page()
5057 page_sub(&pplist, pp); in page_free_replacement_page()
5058 page_clr_all_props(pp); in page_free_replacement_page()
5059 PP_SETFREE(pp); in page_free_replacement_page()
5060 PP_SETAGED(pp); in page_free_replacement_page()
5061 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_free_replacement_page()
5062 page_unlock(pp); in page_free_replacement_page()
5065 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); in page_free_replacement_page()
5067 page_list_break(&pp, &pplist, curnpgs); in page_free_replacement_page()
5068 tpp = pp; in page_free_replacement_page()
5075 } while ((tpp = tpp->p_next) != pp); in page_free_replacement_page()
5076 page_list_add_pages(pp, 0); in page_free_replacement_page()
5127 page_release(page_t *pp, int checkmod) in page_release() argument
5131 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && in page_release()
5132 (pp->p_vnode != NULL)); in page_release()
5134 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && in page_release()
5135 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && in page_release()
5136 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && in page_release()
5137 !hat_page_is_mapped(pp)) { in page_release()
5147 if (checkmod && hat_ismod(pp)) { in page_release()
5148 page_unlock(pp); in page_release()
5152 VN_DISPOSE(pp, B_FREE, 0, kcred); in page_release()
5156 page_unlock(pp); in page_release()
5169 page_try_demote_free_pages(page_t *pp) in page_try_demote_free_pages() argument
5171 page_t *rootpp = pp; in page_try_demote_free_pages()
5172 pfn_t pfn = page_pptonum(pp); in page_try_demote_free_pages()
5174 uint_t szc = pp->p_szc; in page_try_demote_free_pages()
5176 ASSERT(PP_ISFREE(pp)); in page_try_demote_free_pages()
5177 ASSERT(PAGE_EXCL(pp)); in page_try_demote_free_pages()
5183 npgs = page_get_pagecnt(pp->p_szc); in page_try_demote_free_pages()
5193 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { in page_try_demote_free_pages()
5198 if (pp != rootpp) in page_try_demote_free_pages()
5205 if (pp != rootpp) in page_try_demote_free_pages()
5208 ASSERT(PP_ISFREE(pp)); in page_try_demote_free_pages()
5209 ASSERT(PAGE_EXCL(pp)); in page_try_demote_free_pages()
5220 page_try_demote_pages(page_t *pp) in page_try_demote_pages() argument
5222 page_t *tpp, *rootpp = pp; in page_try_demote_pages()
5223 pfn_t pfn = page_pptonum(pp); in page_try_demote_pages()
5225 uint_t szc = pp->p_szc; in page_try_demote_pages()
5226 vnode_t *vp = pp->p_vnode; in page_try_demote_pages()
5228 ASSERT(PAGE_EXCL(pp)); in page_try_demote_pages()
5232 if (pp->p_szc == 0) { in page_try_demote_pages()
5239 page_demote_vp_pages(pp); in page_try_demote_pages()
5240 ASSERT(pp->p_szc == 0); in page_try_demote_pages()
5248 npgs = page_get_pagecnt(pp->p_szc); in page_try_demote_pages()
5273 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) in page_try_demote_pages()
5286 if (tpp != pp) in page_try_demote_pages()
5306 if (tpp != pp) in page_try_demote_pages()
5378 page_demote_vp_pages(page_t *pp) in page_demote_vp_pages() argument
5382 ASSERT(PAGE_EXCL(pp)); in page_demote_vp_pages()
5383 ASSERT(!PP_ISFREE(pp)); in page_demote_vp_pages()
5384 ASSERT(pp->p_vnode != NULL); in page_demote_vp_pages()
5385 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); in page_demote_vp_pages()
5386 ASSERT(!PP_ISKAS(pp)); in page_demote_vp_pages()
5390 mtx = page_szc_lock(pp); in page_demote_vp_pages()
5392 hat_page_demote(pp); in page_demote_vp_pages()
5395 ASSERT(pp->p_szc == 0); in page_demote_vp_pages()
5416 page_t *pp0, *pp; in page_mark_migrate() local
5463 pp = NULL; in page_mark_migrate()
5465 pp = page_lookup(curvp, off, SE_SHARED); in page_mark_migrate()
5471 if (pp == NULL) { in page_mark_migrate()
5479 pfn = page_pptonum(pp); in page_mark_migrate()
5486 pszc = pp->p_szc; in page_mark_migrate()
5493 page_unlock(pp); in page_mark_migrate()
5504 if (!page_tryupgrade(pp)) { in page_mark_migrate()
5505 page_unlock(pp); in page_mark_migrate()
5512 pp0 = pp++; in page_mark_migrate()
5524 if (!page_trylock(pp, SE_EXCL)) { in page_mark_migrate()
5527 if (PP_ISFREE(pp) || in page_mark_migrate()
5528 pp->p_szc != pszc) { in page_mark_migrate()
5533 page_unlock(pp); in page_mark_migrate()
5536 pp++; in page_mark_migrate()
5545 while (pp0 < pp) { in page_mark_migrate()
5568 while (pp0 < pp) { in page_mark_migrate()
5592 page_t *pp; in page_migrate() local
5602 pp = *ppa; in page_migrate()
5603 pszc = pp->p_szc; in page_migrate()
5625 if (!PP_ISMIGRATE(pp)) in page_migrate()
5631 pfn = page_pptonum(pp); in page_migrate()
5679 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); in page_migrate()
5693 PP_CLRMIGRATE(pp); in page_migrate()
5694 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { in page_migrate()
5709 for (i = 0; i < page_cnt; ++i, ++pp) { in page_migrate()
5713 pp = newpp; in page_migrate()
5714 page_sub(&newpp, pp); in page_migrate()
5715 page_downgrade(pp); in page_migrate()
5796 page_t *pp; in page_numtopp_nolock() local
5820 pp = seg->pages + (pfnum - seg->pages_base); in page_numtopp_nolock()
5821 if (pp->p_pagenum == pfnum) { in page_numtopp_nolock()
5823 return ((page_t *)pp); in page_numtopp_nolock()
5832 pp = seg->pages + (pfnum - seg->pages_base); in page_numtopp_nolock()
5833 if (pp->p_pagenum == pfnum) { in page_numtopp_nolock()
5835 return ((page_t *)pp); in page_numtopp_nolock()
5843 pp = seg->pages + (pfnum - seg->pages_base); in page_numtopp_nolock()
5844 if (pp->p_pagenum == pfnum) { in page_numtopp_nolock()
5846 return ((page_t *)pp); in page_numtopp_nolock()
5861 page_t *pp; in page_numtomemseg_nolock() local
5872 pp = seg->pages + (pfnum - seg->pages_base); in page_numtomemseg_nolock()
5873 if (pp->p_pagenum == pfnum) { in page_numtomemseg_nolock()
5882 pp = seg->pages + (pfnum - seg->pages_base); in page_numtomemseg_nolock()
5883 if (pp->p_pagenum == pfnum) { in page_numtomemseg_nolock()
5902 page_nextn(page_t *pp, ulong_t n) in page_nextn() argument
5926 !(pp >= seg->pages && pp < seg->epages)) { in page_nextn()
5929 if (pp >= seg->pages && pp < seg->epages) in page_nextn()
5937 pp = seg->pages; in page_nextn()
5942 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { in page_nextn()
5943 n -= seg->epages - pp; in page_nextn()
5947 pp = seg->pages; in page_nextn()
5973 page_t *pp, in page_next_scan_large() argument
5986 ASSERT(pp != NULL); in page_next_scan_large()
5987 if (pp->p_szc == 0) { in page_next_scan_large()
5990 pfn = page_pptonum(pp); in page_next_scan_large()
5991 cnt = page_get_pagecnt(pp->p_szc); in page_next_scan_large()
5995 new_pp = pp + cnt; in page_next_scan_large()
6024 page_next(page_t *pp) in page_next() argument
6026 return (page_nextn(pp, 1)); in page_next()
6090 page_pptonum(page_t *pp) in page_pptonum() argument
6092 return (pp->p_pagenum); in page_pptonum()
6101 page_set_props(page_t *pp, uint_t flags) in page_set_props() argument
6104 pp->p_nrm |= (uchar_t)flags; in page_set_props()
6108 page_clr_all_props(page_t *pp) in page_clr_all_props() argument
6110 pp->p_nrm = 0; in page_clr_all_props()
6117 page_clear_lck_cow(page_t *pp, int adjust) in page_clear_lck_cow() argument
6121 ASSERT(PAGE_EXCL(pp)); in page_clear_lck_cow()
6128 if (pp->p_lckcnt) { in page_clear_lck_cow()
6130 pp->p_lckcnt = 0; in page_clear_lck_cow()
6132 if (pp->p_cowcnt) { in page_clear_lck_cow()
6133 f_amount += pp->p_cowcnt; in page_clear_lck_cow()
6134 pp->p_cowcnt = 0; in page_clear_lck_cow()
6151 page_share_cnt(page_t *pp) in page_share_cnt() argument
6153 return (hat_page_getshare(pp)); in page_share_cnt()
6157 page_isshared(page_t *pp) in page_isshared() argument
6159 return (hat_page_checkshare(pp, 1)); in page_isshared()
6163 page_isfree(page_t *pp) in page_isfree() argument
6165 return (PP_ISFREE(pp)); in page_isfree()
6169 page_isref(page_t *pp) in page_isref() argument
6171 return (hat_page_getattr(pp, P_REF)); in page_isref()
6175 page_ismod(page_t *pp) in page_ismod() argument
6177 return (hat_page_getattr(pp, P_MOD)); in page_ismod()
6222 page_t *pp; member
6236 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI) argument
6272 #define PAGE_CAPTURE_HASH(pp) \ argument
6273 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
6277 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
6334 page_clrtoxic(head->pp, PR_CAPTURE); in page_capture_unregister_callback()
6362 page_capture_move_to_walked(page_t *pp) in page_capture_move_to_walked() argument
6367 index = PAGE_CAPTURE_HASH(pp); in page_capture_move_to_walked()
6372 if (bp->pp == pp) { in page_capture_move_to_walked()
6393 bp->pri = PAGE_CAPTURE_PRIO(pp); in page_capture_move_to_walked()
6412 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) in page_capture_add_hash() argument
6429 bp1->pp = pp; in page_capture_add_hash()
6457 index = PAGE_CAPTURE_HASH(pp); in page_capture_add_hash()
6468 if (!(pp->p_toxic & PR_CAPTURE)) { in page_capture_add_hash()
6474 if (tp1->pp == pp) { in page_capture_add_hash()
6477 (void *)pp, (void *)tp1); in page_capture_add_hash()
6484 page_settoxic(pp, PR_CAPTURE); in page_capture_add_hash()
6485 pri = PAGE_CAPTURE_PRIO(pp); in page_capture_add_hash()
6516 if (bp2->pp == pp) { in page_capture_add_hash()
6548 (void *)pp); in page_capture_add_hash()
6559 page_capture_clean_page(page_t *pp) in page_capture_clean_page() argument
6568 ASSERT(PAGE_EXCL(pp)); in page_capture_clean_page()
6569 ASSERT(!PP_RETIRED(pp)); in page_capture_clean_page()
6572 if (PP_ISFREE(pp)) { in page_capture_clean_page()
6573 if (!page_reclaim(pp, NULL)) { in page_capture_clean_page()
6578 ASSERT(pp->p_szc == 0); in page_capture_clean_page()
6579 if (pp->p_vnode != NULL) { in page_capture_clean_page()
6585 page_hashout(pp, NULL); in page_capture_clean_page()
6595 if (pp->p_toxic & PR_UE) { in page_capture_clean_page()
6603 if (pp->p_vnode == NULL) { in page_capture_clean_page()
6614 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { in page_capture_clean_page()
6615 if (page_try_demote_pages(pp) == 0) { in page_capture_clean_page()
6620 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); in page_capture_clean_page()
6635 if (pp->p_szc > 0) { in page_capture_clean_page()
6637 extra = page_get_pagecnt(pp->p_szc) - 1; in page_capture_clean_page()
6639 tpp = pp->p_next; in page_capture_clean_page()
6640 page_sub(&pp, tpp); in page_capture_clean_page()
6646 ASSERT(pp->p_next == pp && pp->p_prev == pp); in page_capture_clean_page()
6647 pp->p_szc = 0; in page_capture_clean_page()
6664 if (pp->p_szc > 0) { in page_capture_clean_page()
6665 if (page_try_demote_pages(pp) == 0) { in page_capture_clean_page()
6671 ASSERT(pp->p_szc == 0); in page_capture_clean_page()
6673 if (hat_ismod(pp)) { in page_capture_clean_page()
6677 if (PP_ISKAS(pp)) { in page_capture_clean_page()
6681 if (pp->p_lckcnt || pp->p_cowcnt) { in page_capture_clean_page()
6686 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_capture_clean_page()
6687 ASSERT(!hat_page_is_mapped(pp)); in page_capture_clean_page()
6689 if (hat_ismod(pp)) { in page_capture_clean_page()
6697 if (pp->p_vnode != NULL) { in page_capture_clean_page()
6698 page_hashout(pp, NULL); in page_capture_clean_page()
6709 page_unlock(pp); in page_capture_clean_page()
6712 ASSERT(pp->p_szc == 0); in page_capture_clean_page()
6713 ASSERT(PAGE_EXCL(pp)); in page_capture_clean_page()
6715 pp->p_next = pp; in page_capture_clean_page()
6716 pp->p_prev = pp; in page_capture_clean_page()
6733 page_capture_pre_checks(page_t *pp, uint_t flags) in page_capture_pre_checks() argument
6735 ASSERT(pp != NULL); in page_capture_pre_checks()
6738 if (pp->p_vnode == &promvp) { in page_capture_pre_checks()
6742 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) && in page_capture_pre_checks()
6747 if (PP_ISNORELOCKERNEL(pp)) { in page_capture_pre_checks()
6751 if (PP_ISKAS(pp)) { in page_capture_pre_checks()
6780 page_capture_take_action(page_t *pp, uint_t flags, void *datap) in page_capture_take_action() argument
6790 ASSERT(PAGE_EXCL(pp)); in page_capture_take_action()
6810 index = PAGE_CAPTURE_HASH(pp); in page_capture_take_action()
6816 if (bp1->pp == pp) { in page_capture_take_action()
6821 page_clrtoxic(pp, PR_CAPTURE); in page_capture_take_action()
6834 page_free(pp, 1); in page_capture_take_action()
6853 ret = pc_cb[cb_index].cb_func(pp, datap, flags); in page_capture_take_action()
6895 if (!(pp->p_toxic & PR_CAPTURE)) { in page_capture_take_action()
6897 page_settoxic(pp, PR_CAPTURE); in page_capture_take_action()
6901 bp1->pri = PAGE_CAPTURE_PRIO(pp); in page_capture_take_action()
6916 if (bp2->pp == pp) { in page_capture_take_action()
6934 bp2->pri = PAGE_CAPTURE_PRIO(pp); in page_capture_take_action()
6944 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp); in page_capture_take_action()
6970 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) in page_itrycapture() argument
6976 ASSERT(PAGE_EXCL(pp)); in page_itrycapture()
6981 ret = page_capture_pre_checks(pp, flags); in page_itrycapture()
6986 if (!page_trylock(pp, SE_EXCL)) { in page_itrycapture()
6995 if (PP_RETIRED(pp)) { in page_itrycapture()
6997 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { in page_itrycapture()
7003 page_capture_add_hash(pp, szc, in page_itrycapture()
7012 page_capture_add_hash(pp, szc, flags, datap); in page_itrycapture()
7017 ASSERT(PAGE_EXCL(pp)); in page_itrycapture()
7023 page_unlock(pp); in page_itrycapture()
7027 ret = page_capture_clean_page(pp); in page_itrycapture()
7032 page_capture_add_hash(pp, szc, flags, datap); in page_itrycapture()
7038 ASSERT(PAGE_EXCL(pp)); in page_itrycapture()
7039 ASSERT(pp->p_szc == 0); in page_itrycapture()
7042 ret = page_capture_take_action(pp, flags, datap); in page_itrycapture()
7053 page_capture_add_hash(pp, szc, flags, datap); in page_itrycapture()
7059 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) in page_trycapture() argument
7064 ret = page_itrycapture(pp, szc, flags, datap); in page_trycapture()
7074 page_unlock_capture(page_t *pp) in page_unlock_capture() argument
7097 page_unlock_nocapture(pp); in page_unlock_capture()
7100 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { in page_unlock_capture()
7101 page_unlock_nocapture(pp); in page_unlock_capture()
7105 index = PAGE_CAPTURE_HASH(pp); in page_unlock_capture()
7112 if (bp->pp == pp) { in page_unlock_capture()
7117 (void) page_trycapture(pp, szc, flags, datap); in page_unlock_capture()
7125 page_clrtoxic(pp, PR_CAPTURE); in page_unlock_capture()
7126 page_unlock(pp); in page_unlock_capture()
7162 page_t *pp; in page_retire_mdboot() local
7182 pp = bp->pp; in page_retire_mdboot()
7183 if (PP_TOXIC(pp)) { in page_retire_mdboot()
7184 if (page_trylock(pp, SE_EXCL)) { in page_retire_mdboot()
7185 PP_CLRFREE(pp); in page_retire_mdboot()
7186 pagescrub(pp, 0, PAGESIZE); in page_retire_mdboot()
7187 page_unlock(pp); in page_retire_mdboot()
7204 page_t *pp; in page_capture_async() local
7242 page_deleted(bp1->pp)) { in page_capture_async()
7255 page_clrtoxic(bp1->pp, PR_CAPTURE); in page_capture_async()
7262 pp = bp1->pp; in page_capture_async()
7267 if (page_trylock(pp, SE_EXCL)) { in page_capture_async()
7268 ret = page_trycapture(pp, szc, in page_capture_async()
7276 (void) page_capture_move_to_walked(pp); in page_capture_async()