Lines Matching +full:0 +full:m
127 static SYSCTL_NODE(_vm_stats, OID_AUTO, page, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
161 CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");
167 static void vm_page_alloc_check(vm_page_t m);
169 static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m,
171 static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
172 static void vm_page_enqueue(vm_page_t m, uint8_t queue);
173 static bool vm_page_free_prep(vm_page_t m);
174 static void vm_page_free_toq(vm_page_t m);
176 static int vm_page_insert_after(vm_page_t m, vm_object_t object,
178 static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
180 static void vm_page_mvqueue(vm_page_t m, const uint8_t queue,
184 static void vm_page_release_toq(vm_page_t m, uint8_t nqueue, bool noreuse);
204 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &pgcache_zone_max_pcpu, 0,
220 for (domain = 0; domain < vm_ndomains; domain++) { in vm_page_init_cache_zones()
222 for (pool = 0; pool < VM_NFREEPOOL; pool++) { in vm_page_init_cache_zones()
235 cache = maxcache != 0 ? maxcache : in vm_page_init_cache_zones()
260 if (vm_cnt.v_page_size == 0) in vm_set_page_size()
262 if (((vm_cnt.v_page_size - 1) & vm_cnt.v_page_size) != 0) in vm_set_page_size()
282 return (0); in vm_page_blacklist_next()
283 if (**list =='\0') { in vm_page_blacklist_next()
285 return (0); in vm_page_blacklist_next()
296 if (*end != '\0') { in vm_page_blacklist_next()
298 *end = '\0'; in vm_page_blacklist_next()
302 return (0); in vm_page_blacklist_next()
306 for (pos = *list; *pos != '\0'; pos = cp) { in vm_page_blacklist_next()
307 bad = strtoq(pos, &cp, 0); in vm_page_blacklist_next()
308 if (*cp == '\0' || *cp == ' ' || *cp == ',' || *cp == '\n') { in vm_page_blacklist_next()
309 if (bad == 0) { in vm_page_blacklist_next()
317 if (*cp == '\0' || ++cp >= end) in vm_page_blacklist_next()
325 return (0); in vm_page_blacklist_next()
332 vm_page_t m; in vm_page_blacklist_add() local
335 m = vm_phys_paddr_to_vm_page(pa); in vm_page_blacklist_add()
336 if (m == NULL) in vm_page_blacklist_add()
345 TAILQ_INSERT_TAIL(&blacklist_head, m, listq); in vm_page_blacklist_add()
347 printf("Skipping page with pa 0x%jx\n", (uintmax_t)pa); in vm_page_blacklist_add()
367 if ((pa = vm_page_blacklist_next(&next, end)) == 0) in vm_page_blacklist_check()
406 vm_page_t m; in sysctl_vm_page_blacklist() local
411 error = sysctl_wire_old_buffer(req, 0); in sysctl_vm_page_blacklist()
412 if (error != 0) in sysctl_vm_page_blacklist()
415 TAILQ_FOREACH(m, &blacklist_head, listq) { in sysctl_vm_page_blacklist()
417 (uintmax_t)m->phys_addr); in sysctl_vm_page_blacklist()
418 first = 0; in sysctl_vm_page_blacklist()
460 vmd->vmd_page_count = 0; in vm_page_domain_init()
461 vmd->vmd_free_count = 0; in vm_page_domain_init()
462 vmd->vmd_segs = 0; in vm_page_domain_init()
465 for (i = 0; i < PQ_COUNT; i++) { in vm_page_domain_init()
470 pq->pq_pdpages = 0; in vm_page_domain_init()
471 vm_page_init_marker(&vmd->vmd_markers[i], i, 0); in vm_page_domain_init()
487 * requeues. Scans start at clock[0], which is advanced after the scan in vm_page_domain_init()
491 vm_page_init_marker(&vmd->vmd_clock[0], PQ_ACTIVE, PGA_ENQUEUED); in vm_page_domain_init()
494 &vmd->vmd_clock[0], plinks.q); in vm_page_domain_init()
504 vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind, int pool) in vm_page_init_page() argument
506 m->object = NULL; in vm_page_init_page()
507 m->ref_count = 0; in vm_page_init_page()
508 m->busy_lock = VPB_FREED; in vm_page_init_page()
509 m->flags = m->a.flags = 0; in vm_page_init_page()
510 m->phys_addr = pa; in vm_page_init_page()
511 m->a.queue = PQ_NONE; in vm_page_init_page()
512 m->psind = 0; in vm_page_init_page()
513 m->segind = segind; in vm_page_init_page()
514 m->order = VM_NFREEORDER; in vm_page_init_page()
515 m->pool = pool; in vm_page_init_page()
516 m->valid = m->dirty = 0; in vm_page_init_page()
517 pmap_page_init(m); in vm_page_init_page()
558 vm_page_t m; in vm_page_startup() local
589 for (i = 0; i < PA_LOCK_COUNT; i++) in vm_page_startup()
591 for (i = 0; i < vm_ndomains; i++) in vm_page_startup()
616 last_pa = 0; in vm_page_startup()
617 vm_page_dump_pages = 0; in vm_page_startup()
618 for (i = 0; dump_avail[i + 1] != 0; i += 2) { in vm_page_startup()
663 low_avail = phys_avail[0]; in vm_page_startup()
665 for (i = 0; i < vm_phys_nsegs; i++) { in vm_page_startup()
672 for (i = 2; phys_avail[i + 1] != 0; i += 2) { in vm_page_startup()
680 size = 0; in vm_page_startup()
681 for (i = 0; i < vm_phys_nsegs; i++) in vm_page_startup()
683 for (i = 0; phys_avail[i + 1] != 0; i += 2) in vm_page_startup()
729 #if VM_NRESERVLEVEL > 0 in vm_page_startup()
749 for (i = 0; phys_avail[i + 1] != 0; i += 2) in vm_page_startup()
750 if (vm_phys_avail_size(i) != 0) in vm_page_startup()
768 for (ii = 0; ii < vm_page_array_size; ii++) { in vm_page_startup()
769 m = &vm_page_array[ii]; in vm_page_startup()
770 vm_page_init_page(m, (first_page + ii) << PAGE_SHIFT, 0, in vm_page_startup()
772 m->flags = PG_FICTITIOUS; in vm_page_startup()
775 vm_cnt.v_page_count = 0; in vm_page_startup()
776 for (segind = 0; segind < vm_phys_nsegs; segind++) { in vm_page_startup()
797 for (i = 0; phys_avail[i + 1] != 0; i += 2) { in vm_page_startup()
808 m = vm_phys_seg_paddr_to_vm_page(seg, startp); in vm_page_startup()
810 startp < endp; startp += PAGE_SIZE, m++) { in vm_page_startup()
811 vm_page_init_page(m, startp, segind, in vm_page_startup()
817 for (m = seg->first_page, pa = seg->start; in vm_page_startup()
818 pa < seg->end; m++, pa += PAGE_SIZE) { in vm_page_startup()
819 vm_page_init_page(m, pa, segind, in vm_page_startup()
827 for (i = 0; phys_avail[i + 1] != 0; i += 2) { in vm_page_startup()
835 if (pagecount == 0) in vm_page_startup()
838 m = vm_phys_seg_paddr_to_vm_page(seg, startp); in vm_page_startup()
841 vm_page_init_page(m, startp, segind, in vm_page_startup()
847 vm_phys_enqueue_contig(m, pagecount); in vm_page_startup()
867 #if VM_NRESERVLEVEL > 0 in vm_page_startup()
878 vm_page_reference(vm_page_t m) in vm_page_reference() argument
881 vm_page_aflag_set(m, PGA_REFERENCED); in vm_page_reference()
892 vm_page_trybusy(vm_page_t m, int allocflags) in vm_page_trybusy() argument
895 if ((allocflags & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0) in vm_page_trybusy()
896 return (vm_page_trysbusy(m)); in vm_page_trybusy()
898 return (vm_page_tryxbusy(m)); in vm_page_trybusy()
909 vm_page_tryacquire(vm_page_t m, int allocflags) in vm_page_tryacquire() argument
913 locked = vm_page_trybusy(m, allocflags); in vm_page_tryacquire()
914 if (locked && (allocflags & VM_ALLOC_WIRED) != 0) in vm_page_tryacquire()
915 vm_page_wire(m); in vm_page_tryacquire()
926 vm_page_busy_acquire(vm_page_t m, int allocflags) in vm_page_busy_acquire() argument
938 obj = atomic_load_ptr(&m->object); in vm_page_busy_acquire()
940 if (vm_page_tryacquire(m, allocflags)) in vm_page_busy_acquire()
942 if ((allocflags & VM_ALLOC_NOWAIT) != 0) in vm_page_busy_acquire()
948 MPASS(locked || vm_page_wired(m)); in vm_page_busy_acquire()
949 if (_vm_page_busy_sleep(obj, m, m->pindex, "vmpba", allocflags, in vm_page_busy_acquire()
952 if ((allocflags & VM_ALLOC_WAITFAIL) != 0) in vm_page_busy_acquire()
954 KASSERT(m->object == obj || m->object == NULL, in vm_page_busy_acquire()
956 m, obj)); in vm_page_busy_acquire()
966 vm_page_busy_downgrade(vm_page_t m) in vm_page_busy_downgrade() argument
970 vm_page_assert_xbusied(m); in vm_page_busy_downgrade()
972 x = vm_page_busy_fetch(m); in vm_page_busy_downgrade()
974 if (atomic_fcmpset_rel_int(&m->busy_lock, in vm_page_busy_downgrade()
978 if ((x & VPB_BIT_WAITERS) != 0) in vm_page_busy_downgrade()
979 wakeup(m); in vm_page_busy_downgrade()
989 vm_page_busy_tryupgrade(vm_page_t m) in vm_page_busy_tryupgrade() argument
993 vm_page_assert_sbusied(m); in vm_page_busy_tryupgrade()
995 x = vm_page_busy_fetch(m); in vm_page_busy_tryupgrade()
999 return (0); in vm_page_busy_tryupgrade()
1002 if (!atomic_fcmpset_acq_int(&m->busy_lock, &x, in vm_page_busy_tryupgrade()
1012 * Return a positive value if the page is shared busied, 0 otherwise.
1015 vm_page_sbusied(vm_page_t m) in vm_page_sbusied() argument
1019 x = vm_page_busy_fetch(m); in vm_page_sbusied()
1020 return ((x & VPB_BIT_SHARED) != 0 && x != VPB_UNBUSIED); in vm_page_sbusied()
1029 vm_page_sunbusy(vm_page_t m) in vm_page_sunbusy() argument
1033 vm_page_assert_sbusied(m); in vm_page_sunbusy()
1035 x = vm_page_busy_fetch(m); in vm_page_sunbusy()
1040 if (atomic_fcmpset_int(&m->busy_lock, &x, in vm_page_sunbusy()
1047 if (!atomic_fcmpset_rel_int(&m->busy_lock, &x, VPB_UNBUSIED)) in vm_page_sunbusy()
1049 if ((x & VPB_BIT_WAITERS) == 0) in vm_page_sunbusy()
1051 wakeup(m); in vm_page_sunbusy()
1071 vm_page_busy_sleep(vm_page_t m, const char *wmesg, int allocflags) in vm_page_busy_sleep() argument
1075 obj = m->object; in vm_page_busy_sleep()
1078 return (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, allocflags, in vm_page_busy_sleep()
1095 vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex, in vm_page_busy_sleep_unlocked() argument
1100 (void)_vm_page_busy_sleep(obj, m, pindex, wmesg, allocflags, false); in vm_page_busy_sleep_unlocked()
1116 _vm_page_busy_sleep(vm_object_t obj, vm_page_t m, vm_pindex_t pindex, in _vm_page_busy_sleep() argument
1133 if (!vm_page_busied(m)) in _vm_page_busy_sleep()
1136 xsleep = (allocflags & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0; in _vm_page_busy_sleep()
1137 sleepq_lock(m); in _vm_page_busy_sleep()
1138 x = vm_page_busy_fetch(m); in _vm_page_busy_sleep()
1145 (xsleep && (x & VPB_BIT_SHARED) != 0) || in _vm_page_busy_sleep()
1146 m->object != obj || m->pindex != pindex) { in _vm_page_busy_sleep()
1147 sleepq_release(m); in _vm_page_busy_sleep()
1150 if ((x & VPB_BIT_WAITERS) != 0) in _vm_page_busy_sleep()
1152 } while (!atomic_fcmpset_int(&m->busy_lock, &x, x | VPB_BIT_WAITERS)); in _vm_page_busy_sleep()
1156 sleepq_add(m, NULL, wmesg, 0, 0); in _vm_page_busy_sleep()
1157 sleepq_wait(m, PVM); in _vm_page_busy_sleep()
1166 * If the operation succeeds 1 is returned otherwise 0.
1170 vm_page_trysbusy(vm_page_t m) in vm_page_trysbusy() argument
1175 obj = m->object; in vm_page_trysbusy()
1176 x = vm_page_busy_fetch(m); in vm_page_trysbusy()
1178 if ((x & VPB_BIT_SHARED) == 0) in vm_page_trysbusy()
1179 return (0); in vm_page_trysbusy()
1185 return (0); in vm_page_trysbusy()
1186 if (atomic_fcmpset_acq_int(&m->busy_lock, &x, in vm_page_trysbusy()
1192 obj = m->object; in vm_page_trysbusy()
1194 vm_page_sunbusy(m); in vm_page_trysbusy()
1195 return (0); in vm_page_trysbusy()
1204 * If the operation succeeds 1 is returned otherwise 0.
1208 vm_page_tryxbusy(vm_page_t m) in vm_page_tryxbusy() argument
1212 if (atomic_cmpset_acq_int(&m->busy_lock, VPB_UNBUSIED, in vm_page_tryxbusy()
1213 VPB_CURTHREAD_EXCLUSIVE) == 0) in vm_page_tryxbusy()
1214 return (0); in vm_page_tryxbusy()
1216 obj = m->object; in vm_page_tryxbusy()
1218 vm_page_xunbusy(m); in vm_page_tryxbusy()
1219 return (0); in vm_page_tryxbusy()
1225 vm_page_xunbusy_hard_tail(vm_page_t m) in vm_page_xunbusy_hard_tail() argument
1227 atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED); in vm_page_xunbusy_hard_tail()
1229 wakeup(m); in vm_page_xunbusy_hard_tail()
1238 vm_page_xunbusy_hard(vm_page_t m) in vm_page_xunbusy_hard() argument
1240 vm_page_assert_xbusied(m); in vm_page_xunbusy_hard()
1241 vm_page_xunbusy_hard_tail(m); in vm_page_xunbusy_hard()
1245 vm_page_xunbusy_hard_unchecked(vm_page_t m) in vm_page_xunbusy_hard_unchecked() argument
1247 vm_page_assert_xbusied_unchecked(m); in vm_page_xunbusy_hard_unchecked()
1248 vm_page_xunbusy_hard_tail(m); in vm_page_xunbusy_hard_unchecked()
1252 vm_page_busy_free(vm_page_t m) in vm_page_busy_free() argument
1257 x = atomic_swap_int(&m->busy_lock, VPB_FREED); in vm_page_busy_free()
1258 if ((x & VPB_BIT_WAITERS) != 0) in vm_page_busy_free()
1259 wakeup(m); in vm_page_busy_free()
1271 for (; count != 0; count--) { in vm_page_unhold_pages()
1280 vm_page_t m; in PHYS_TO_VM_PAGE() local
1283 m = vm_phys_paddr_to_vm_page(pa); in PHYS_TO_VM_PAGE()
1284 if (m == NULL) in PHYS_TO_VM_PAGE()
1285 m = vm_phys_fictitious_to_vm_page(pa); in PHYS_TO_VM_PAGE()
1286 return (m); in PHYS_TO_VM_PAGE()
1292 m = &vm_page_array[pi - first_page]; in PHYS_TO_VM_PAGE()
1293 return (m); in PHYS_TO_VM_PAGE()
1311 vm_page_t m; in vm_page_getfake() local
1313 m = uma_zalloc(fakepg_zone, M_WAITOK | M_ZERO); in vm_page_getfake()
1314 vm_page_initfake(m, paddr, memattr); in vm_page_getfake()
1315 return (m); in vm_page_getfake()
1319 vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr) in vm_page_initfake() argument
1322 if ((m->flags & PG_FICTITIOUS) != 0) { in vm_page_initfake()
1330 m->phys_addr = paddr; in vm_page_initfake()
1331 m->a.queue = PQ_NONE; in vm_page_initfake()
1333 m->flags = PG_FICTITIOUS; in vm_page_initfake()
1335 m->oflags = VPO_UNMANAGED; in vm_page_initfake()
1336 m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; in vm_page_initfake()
1338 m->ref_count = 1; in vm_page_initfake()
1339 pmap_page_init(m); in vm_page_initfake()
1341 pmap_page_set_memattr(m, memattr); in vm_page_initfake()
1350 vm_page_putfake(vm_page_t m) in vm_page_putfake() argument
1353 KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed %p", m)); in vm_page_putfake()
1354 KASSERT((m->flags & PG_FICTITIOUS) != 0, in vm_page_putfake()
1355 ("vm_page_putfake: bad page %p", m)); in vm_page_putfake()
1356 vm_page_assert_xbusied(m); in vm_page_putfake()
1357 vm_page_busy_free(m); in vm_page_putfake()
1358 uma_zfree(fakepg_zone, m); in vm_page_putfake()
1368 vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr) in vm_page_updatefake() argument
1371 KASSERT((m->flags & PG_FICTITIOUS) != 0, in vm_page_updatefake()
1372 ("vm_page_updatefake: bad page %p", m)); in vm_page_updatefake()
1373 m->phys_addr = paddr; in vm_page_updatefake()
1374 pmap_page_set_memattr(m, memattr); in vm_page_updatefake()
1383 vm_page_free(vm_page_t m) in vm_page_free() argument
1386 m->flags &= ~PG_ZERO; in vm_page_free()
1387 vm_page_free_toq(m); in vm_page_free()
1396 vm_page_free_zero(vm_page_t m) in vm_page_free_zero() argument
1399 m->flags |= PG_ZERO; in vm_page_free_zero()
1400 vm_page_free_toq(m); in vm_page_free_zero()
1408 vm_page_readahead_finish(vm_page_t m) in vm_page_readahead_finish() argument
1412 KASSERT(!vm_page_none_valid(m), ("%s: %p is invalid", __func__, m)); in vm_page_readahead_finish()
1420 if ((vm_page_busy_fetch(m) & VPB_BIT_WAITERS) != 0) in vm_page_readahead_finish()
1421 vm_page_activate(m); in vm_page_readahead_finish()
1423 vm_page_deactivate(m); in vm_page_readahead_finish()
1424 vm_page_xunbusy_unchecked(m); in vm_page_readahead_finish()
1432 vm_page_free_invalid(vm_page_t m) in vm_page_free_invalid() argument
1435 KASSERT(vm_page_none_valid(m), ("page %p is valid", m)); in vm_page_free_invalid()
1436 KASSERT(!pmap_page_is_mapped(m), ("page %p is mapped", m)); in vm_page_free_invalid()
1437 KASSERT(m->object != NULL, ("page %p has no object", m)); in vm_page_free_invalid()
1438 VM_OBJECT_ASSERT_WLOCKED(m->object); in vm_page_free_invalid()
1444 vm_page_xbusy_claim(m); in vm_page_free_invalid()
1453 if (vm_page_remove(m)) in vm_page_free_invalid()
1454 vm_page_free(m); in vm_page_free_invalid()
1470 vm_page_dirty_KBI(vm_page_t m) in vm_page_dirty_KBI() argument
1474 KASSERT(vm_page_all_valid(m), ("vm_page_dirty: page is invalid!")); in vm_page_dirty_KBI()
1475 m->dirty = VM_PAGE_BITS_ALL; in vm_page_dirty_KBI()
1490 vm_page_insert_lookup(vm_page_t m, vm_object_t object, vm_pindex_t pindex, in vm_page_insert_lookup() argument
1496 KASSERT(m->object == NULL, in vm_page_insert_lookup()
1497 ("vm_page_insert: page %p already inserted", m)); in vm_page_insert_lookup()
1502 m->object = object; in vm_page_insert_lookup()
1503 m->pindex = pindex; in vm_page_insert_lookup()
1504 m->ref_count |= VPRC_OBJREF; in vm_page_insert_lookup()
1512 error = vm_radix_iter_insert(pages, m); in vm_page_insert_lookup()
1514 error = vm_radix_insert_lookup_lt(&object->rtree, m, &mpred); in vm_page_insert_lookup()
1516 error = vm_radix_insert(&object->rtree, m); in vm_page_insert_lookup()
1517 if (__predict_false(error != 0)) { in vm_page_insert_lookup()
1518 m->object = NULL; in vm_page_insert_lookup()
1519 m->pindex = 0; in vm_page_insert_lookup()
1520 m->ref_count &= ~VPRC_OBJREF; in vm_page_insert_lookup()
1527 vm_page_insert_radixdone(m, object, mpred); in vm_page_insert_lookup()
1528 vm_pager_page_inserted(object, m); in vm_page_insert_lookup()
1529 return (0); in vm_page_insert_lookup()
1540 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) in vm_page_insert() argument
1542 return (vm_page_insert_lookup(m, object, pindex, NULL, false, NULL, in vm_page_insert()
1549 * Inserts the page "m" into the specified object at offset "pindex".
1557 vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex, in vm_page_insert_after() argument
1560 return (vm_page_insert_lookup(m, object, pindex, NULL, false, mpred, in vm_page_insert_after()
1567 * Tries to insert the page "m" into the specified object at offset
1568 * "pindex" using the iterator "pages". Returns 0 if the insertion was
1577 vm_page_iter_insert(struct pctrie_iter *pages, vm_page_t m, vm_object_t object, in vm_page_iter_insert() argument
1580 return (vm_page_insert_lookup(m, object, pindex, pages, true, mpred, in vm_page_iter_insert()
1587 * Complete page "m" insertion into the specified object after the
1590 * The page "mpred" must precede the offset "m->pindex" within the
1596 vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred) in vm_page_insert_radixdone() argument
1600 KASSERT(object != NULL && m->object == object, in vm_page_insert_radixdone()
1601 ("vm_page_insert_radixdone: page %p has inconsistent object", m)); in vm_page_insert_radixdone()
1602 KASSERT((m->ref_count & VPRC_OBJREF) != 0, in vm_page_insert_radixdone()
1603 ("vm_page_insert_radixdone: page %p is missing object ref", m)); in vm_page_insert_radixdone()
1607 KASSERT(mpred->pindex < m->pindex, in vm_page_insert_radixdone()
1610 m->pindex < TAILQ_NEXT(mpred, listq)->pindex, in vm_page_insert_radixdone()
1614 m->pindex < TAILQ_FIRST(&object->memq)->pindex, in vm_page_insert_radixdone()
1619 TAILQ_INSERT_AFTER(&object->memq, mpred, m, listq); in vm_page_insert_radixdone()
1621 TAILQ_INSERT_HEAD(&object->memq, m, listq); in vm_page_insert_radixdone()
1638 if (pmap_page_is_write_mapped(m)) in vm_page_insert_radixdone()
1645 * Complete page "m" removal from the specified object after the radix trie
1652 vm_page_remove_radixdone(vm_page_t m) in vm_page_remove_radixdone() argument
1656 vm_page_assert_xbusied(m); in vm_page_remove_radixdone()
1657 object = m->object; in vm_page_remove_radixdone()
1659 KASSERT((m->ref_count & VPRC_OBJREF) != 0, in vm_page_remove_radixdone()
1660 ("page %p is missing its object ref", m)); in vm_page_remove_radixdone()
1663 if ((m->a.flags & PGA_SWAP_FREE) != 0) in vm_page_remove_radixdone()
1664 vm_pager_page_unswapped(m); in vm_page_remove_radixdone()
1666 vm_pager_page_removed(object, m); in vm_page_remove_radixdone()
1667 m->object = NULL; in vm_page_remove_radixdone()
1672 TAILQ_REMOVE(&object->memq, m, listq); in vm_page_remove_radixdone()
1682 if (object->resident_page_count == 0 && object->type == OBJT_VNODE) in vm_page_remove_radixdone()
1694 vm_page_free_object_prep(vm_page_t m) in vm_page_free_object_prep() argument
1696 KASSERT(((m->oflags & VPO_UNMANAGED) != 0) == in vm_page_free_object_prep()
1697 ((m->object->flags & OBJ_UNMANAGED) != 0), in vm_page_free_object_prep()
1699 __func__, m)); in vm_page_free_object_prep()
1700 vm_page_assert_xbusied(m); in vm_page_free_object_prep()
1706 KASSERT((m->flags & PG_FICTITIOUS) != 0 || in vm_page_free_object_prep()
1707 m->ref_count == VPRC_OBJREF, in vm_page_free_object_prep()
1709 __func__, m, m->ref_count)); in vm_page_free_object_prep()
1710 vm_page_remove_radixdone(m); in vm_page_free_object_prep()
1711 m->ref_count -= VPRC_OBJREF; in vm_page_free_object_prep()
1721 vm_page_iter_free(struct pctrie_iter *pages, vm_page_t m) in vm_page_iter_free() argument
1724 vm_page_free_object_prep(m); in vm_page_iter_free()
1725 vm_page_xunbusy(m); in vm_page_iter_free()
1726 m->flags &= ~PG_ZERO; in vm_page_iter_free()
1727 vm_page_free_toq(m); in vm_page_iter_free()
1743 vm_page_remove(vm_page_t m) in vm_page_remove() argument
1747 dropped = vm_page_remove_xbusy(m); in vm_page_remove()
1748 vm_page_xunbusy(m); in vm_page_remove()
1760 vm_page_iter_remove(struct pctrie_iter *pages, vm_page_t m) in vm_page_iter_remove() argument
1765 vm_page_remove_radixdone(m); in vm_page_iter_remove()
1766 dropped = (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF); in vm_page_iter_remove()
1767 vm_page_xunbusy(m); in vm_page_iter_remove()
1778 vm_page_radix_remove(vm_page_t m) in vm_page_radix_remove() argument
1782 mrem = vm_radix_remove(&m->object->rtree, m->pindex); in vm_page_radix_remove()
1783 KASSERT(mrem == m, in vm_page_radix_remove()
1784 ("removed page %p, expected page %p", mrem, m)); in vm_page_radix_remove()
1794 vm_page_remove_xbusy(vm_page_t m) in vm_page_remove_xbusy() argument
1797 vm_page_radix_remove(m); in vm_page_remove_xbusy()
1798 vm_page_remove_radixdone(m); in vm_page_remove_xbusy()
1799 return (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF); in vm_page_remove_xbusy()
1882 vm_page_t m; in vm_page_relookup() local
1884 m = vm_page_lookup_unlocked(object, pindex); in vm_page_relookup()
1885 KASSERT(m != NULL && (vm_page_busied(m) || vm_page_wired(m)) && in vm_page_relookup()
1886 m->object == object && m->pindex == pindex, in vm_page_relookup()
1887 ("vm_page_relookup: Invalid page %p", m)); in vm_page_relookup()
1888 return (m); in vm_page_relookup()
1898 vm_page_busy_release(vm_page_t m) in vm_page_busy_release() argument
1902 x = vm_page_busy_fetch(m); in vm_page_busy_release()
1906 if ((x & VPB_BIT_SHARED) != 0 && VPB_SHARERS(x) > 1) { in vm_page_busy_release()
1907 if (atomic_fcmpset_int(&m->busy_lock, &x, in vm_page_busy_release()
1912 KASSERT((x & VPB_BIT_SHARED) != 0 || in vm_page_busy_release()
1914 ("vm_page_busy_release: %p xbusy not owned.", m)); in vm_page_busy_release()
1915 if (!atomic_fcmpset_rel_int(&m->busy_lock, &x, VPB_UNBUSIED)) in vm_page_busy_release()
1917 if ((x & VPB_BIT_WAITERS) != 0) in vm_page_busy_release()
1918 wakeup(m); in vm_page_busy_release()
1934 vm_page_t m; in vm_page_find_least() local
1937 if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex) in vm_page_find_least()
1938 m = vm_radix_lookup_ge(&object->rtree, pindex); in vm_page_find_least()
1939 return (m); in vm_page_find_least()
1965 vm_page_next(vm_page_t m) in vm_page_next() argument
1969 VM_OBJECT_ASSERT_LOCKED(m->object); in vm_page_next()
1970 if ((next = TAILQ_NEXT(m, listq)) != NULL) { in vm_page_next()
1971 MPASS(next->object == m->object); in vm_page_next()
1972 if (next->pindex != m->pindex + 1) in vm_page_next()
1985 vm_page_prev(vm_page_t m) in vm_page_prev() argument
1989 VM_OBJECT_ASSERT_LOCKED(m->object); in vm_page_prev()
1990 if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL) { in vm_page_prev()
1991 MPASS(prev->object == m->object); in vm_page_prev()
1992 if (prev->pindex != m->pindex - 1) in vm_page_prev()
2018 KASSERT(mnew->object == NULL && (mnew->ref_count & VPRC_OBJREF) == 0, in vm_page_replace_hold()
2089 vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m, in vm_page_iter_rename() argument
2095 KASSERT((m->ref_count & VPRC_OBJREF) != 0, in vm_page_iter_rename()
2096 ("%s: page %p is missing object ref", __func__, m)); in vm_page_iter_rename()
2097 VM_OBJECT_ASSERT_WLOCKED(m->object); in vm_page_iter_rename()
2105 opidx = m->pindex; in vm_page_iter_rename()
2106 m->pindex = new_pindex; in vm_page_iter_rename()
2107 if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) { in vm_page_iter_rename()
2108 m->pindex = opidx; in vm_page_iter_rename()
2116 m->pindex = opidx; in vm_page_iter_rename()
2118 vm_page_remove_radixdone(m); in vm_page_iter_rename()
2121 m->pindex = new_pindex; in vm_page_iter_rename()
2122 m->object = new_object; in vm_page_iter_rename()
2124 vm_page_insert_radixdone(m, new_object, mpred); in vm_page_iter_rename()
2125 vm_page_dirty(m); in vm_page_iter_rename()
2126 vm_pager_page_inserted(new_object, m); in vm_page_iter_rename()
2183 vm_page_t m; in vm_page_alloc_after() local
2188 m = vm_page_alloc_domain_after(object, pindex, domain, req, in vm_page_alloc_after()
2190 if (m != NULL) in vm_page_alloc_after()
2192 } while (vm_domainset_iter_page(&di, object, &domain) == 0); in vm_page_alloc_after()
2194 return (m); in vm_page_alloc_after()
2207 limit = 0; in _vm_domain_allocate()
2220 return (0); in _vm_domain_allocate()
2222 } while (atomic_fcmpset_int(&vmd->vmd_free_count, &old, new) == 0); in _vm_domain_allocate()
2255 vm_page_t m; in vm_page_alloc_domain_after() local
2263 KASSERT((req & ~VPA_FLAGS) == 0, in vm_page_alloc_domain_after()
2269 ("mpred %p doesn't precede pindex 0x%jx", mpred, in vm_page_alloc_domain_after()
2273 flags = 0; in vm_page_alloc_domain_after()
2274 m = NULL; in vm_page_alloc_domain_after()
2278 if (__predict_false((req & VM_ALLOC_NOFREE) != 0)) { in vm_page_alloc_domain_after()
2279 m = vm_page_alloc_nofree_domain(domain, req); in vm_page_alloc_domain_after()
2280 if (m != NULL) in vm_page_alloc_domain_after()
2283 #if VM_NRESERVLEVEL > 0 in vm_page_alloc_domain_after()
2288 (m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) != in vm_page_alloc_domain_after()
2295 m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DEFAULT].zone, in vm_page_alloc_domain_after()
2297 if (m != NULL) { in vm_page_alloc_domain_after()
2307 m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, 0); in vm_page_alloc_domain_after()
2309 if (m == NULL) { in vm_page_alloc_domain_after()
2311 #if VM_NRESERVLEVEL > 0 in vm_page_alloc_domain_after()
2317 if (m == NULL) { in vm_page_alloc_domain_after()
2330 vm_page_dequeue(m); in vm_page_alloc_domain_after()
2331 vm_page_alloc_check(m); in vm_page_alloc_domain_after()
2336 flags |= m->flags & PG_ZERO; in vm_page_alloc_domain_after()
2337 if ((req & VM_ALLOC_NODUMP) != 0) in vm_page_alloc_domain_after()
2339 if ((req & VM_ALLOC_NOFREE) != 0) in vm_page_alloc_domain_after()
2341 m->flags = flags; in vm_page_alloc_domain_after()
2342 m->a.flags = 0; in vm_page_alloc_domain_after()
2343 m->oflags = (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; in vm_page_alloc_domain_after()
2344 if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) in vm_page_alloc_domain_after()
2345 m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; in vm_page_alloc_domain_after()
2346 else if ((req & VM_ALLOC_SBUSY) != 0) in vm_page_alloc_domain_after()
2347 m->busy_lock = VPB_SHARERS_WORD(1); in vm_page_alloc_domain_after()
2349 m->busy_lock = VPB_UNBUSIED; in vm_page_alloc_domain_after()
2352 m->ref_count = 1; in vm_page_alloc_domain_after()
2354 m->a.act_count = 0; in vm_page_alloc_domain_after()
2356 if (vm_page_insert_after(m, object, pindex, mpred)) { in vm_page_alloc_domain_after()
2359 m->ref_count = 0; in vm_page_alloc_domain_after()
2361 KASSERT(m->object == NULL, ("page %p has object", m)); in vm_page_alloc_domain_after()
2362 m->oflags = VPO_UNMANAGED; in vm_page_alloc_domain_after()
2363 m->busy_lock = VPB_UNBUSIED; in vm_page_alloc_domain_after()
2365 vm_page_free_toq(m); in vm_page_alloc_domain_after()
2376 (object->flags & OBJ_FICTITIOUS) == 0) in vm_page_alloc_domain_after()
2377 pmap_page_set_memattr(m, object->memattr); in vm_page_alloc_domain_after()
2379 return (m); in vm_page_alloc_domain_after()
2426 vm_page_t m; in vm_page_alloc_contig() local
2434 m = vm_page_alloc_contig_domain(object, pindex, domain, req, in vm_page_alloc_contig()
2436 if (m != NULL) in vm_page_alloc_contig()
2444 } while (vm_domainset_iter_page(&di, object, &domain) == 0); in vm_page_alloc_contig()
2446 return (m); in vm_page_alloc_contig()
2472 #if VM_NRESERVLEVEL > 0 in vm_page_find_contig_domain()
2476 if ((req & VM_ALLOC_NORECLAIM) == 0) { in vm_page_find_contig_domain()
2493 vm_page_t m, m_ret, mpred; in vm_page_alloc_contig_domain() local
2497 KASSERT((req & ~VPAC_FLAGS) == 0, in vm_page_alloc_contig_domain()
2506 KASSERT((object->flags & OBJ_FICTITIOUS) == 0, in vm_page_alloc_contig_domain()
2509 KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); in vm_page_alloc_contig_domain()
2516 #if VM_NRESERVLEVEL > 0 in vm_page_alloc_contig_domain()
2537 if ((req & VM_ALLOC_NODUMP) != 0) in vm_page_alloc_contig_domain()
2539 oflags = (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; in vm_page_alloc_contig_domain()
2540 if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) in vm_page_alloc_contig_domain()
2542 else if ((req & VM_ALLOC_SBUSY) != 0) in vm_page_alloc_contig_domain()
2546 if ((req & VM_ALLOC_WIRED) != 0) in vm_page_alloc_contig_domain()
2551 for (m = m_ret; m < &m_ret[npages]; m++) { in vm_page_alloc_contig_domain()
2552 vm_page_dequeue(m); in vm_page_alloc_contig_domain()
2553 vm_page_alloc_check(m); in vm_page_alloc_contig_domain()
2554 m->a.flags = 0; in vm_page_alloc_contig_domain()
2555 m->flags = (m->flags | PG_NODUMP) & flags; in vm_page_alloc_contig_domain()
2556 m->busy_lock = busy_lock; in vm_page_alloc_contig_domain()
2557 if ((req & VM_ALLOC_WIRED) != 0) in vm_page_alloc_contig_domain()
2558 m->ref_count = 1; in vm_page_alloc_contig_domain()
2559 m->a.act_count = 0; in vm_page_alloc_contig_domain()
2560 m->oflags = oflags; in vm_page_alloc_contig_domain()
2561 if (vm_page_iter_insert(&pages, m, object, pindex, mpred)) { in vm_page_alloc_contig_domain()
2562 if ((req & VM_ALLOC_WIRED) != 0) in vm_page_alloc_contig_domain()
2564 KASSERT(m->object == NULL, in vm_page_alloc_contig_domain()
2565 ("page %p has object", m)); in vm_page_alloc_contig_domain()
2566 mpred = m; in vm_page_alloc_contig_domain()
2567 for (m = m_ret; m < &m_ret[npages]; m++) { in vm_page_alloc_contig_domain()
2568 if (m <= mpred && in vm_page_alloc_contig_domain()
2569 (req & VM_ALLOC_WIRED) != 0) in vm_page_alloc_contig_domain()
2570 m->ref_count = 0; in vm_page_alloc_contig_domain()
2571 m->oflags = VPO_UNMANAGED; in vm_page_alloc_contig_domain()
2572 m->busy_lock = VPB_UNBUSIED; in vm_page_alloc_contig_domain()
2574 vm_page_free_toq(m); in vm_page_alloc_contig_domain()
2583 mpred = m; in vm_page_alloc_contig_domain()
2585 pmap_page_set_memattr(m, memattr); in vm_page_alloc_contig_domain()
2599 vm_page_t m; in vm_page_alloc_noobj_domain() local
2607 KASSERT((req & ~VPAN_FLAGS) == 0, in vm_page_alloc_noobj_domain()
2610 flags = ((req & VM_ALLOC_NODUMP) != 0 ? PG_NODUMP : 0) | in vm_page_alloc_noobj_domain()
2611 ((req & VM_ALLOC_NOFREE) != 0 ? PG_NOFREE : 0); in vm_page_alloc_noobj_domain()
2614 if (__predict_false((req & VM_ALLOC_NOFREE) != 0)) { in vm_page_alloc_noobj_domain()
2615 m = vm_page_alloc_nofree_domain(domain, req); in vm_page_alloc_noobj_domain()
2616 if (m != NULL) in vm_page_alloc_noobj_domain()
2621 m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone, in vm_page_alloc_noobj_domain()
2623 if (m != NULL) { in vm_page_alloc_noobj_domain()
2631 m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DIRECT, 0); in vm_page_alloc_noobj_domain()
2633 if (m == NULL) { in vm_page_alloc_noobj_domain()
2635 #if VM_NRESERVLEVEL > 0 in vm_page_alloc_noobj_domain()
2641 if (m == NULL) { in vm_page_alloc_noobj_domain()
2648 vm_page_dequeue(m); in vm_page_alloc_noobj_domain()
2649 vm_page_alloc_check(m); in vm_page_alloc_noobj_domain()
2654 m->pindex = 0xdeadc0dedeadc0de; in vm_page_alloc_noobj_domain()
2655 m->flags = (m->flags & PG_ZERO) | flags; in vm_page_alloc_noobj_domain()
2656 m->a.flags = 0; in vm_page_alloc_noobj_domain()
2657 m->oflags = VPO_UNMANAGED; in vm_page_alloc_noobj_domain()
2658 m->busy_lock = VPB_UNBUSIED; in vm_page_alloc_noobj_domain()
2659 if ((req & VM_ALLOC_WIRED) != 0) { in vm_page_alloc_noobj_domain()
2661 m->ref_count = 1; in vm_page_alloc_noobj_domain()
2664 if ((req & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) in vm_page_alloc_noobj_domain()
2665 pmap_zero_page(m); in vm_page_alloc_noobj_domain()
2667 return (m); in vm_page_alloc_noobj_domain()
2672 #elif VM_NRESERVLEVEL > 0
2690 vm_page_t m; in vm_page_alloc_nofree_domain() local
2694 KASSERT((req & VM_ALLOC_NOFREE) != 0, ("invalid request %#x", req)); in vm_page_alloc_nofree_domain()
2712 nqp->offs = 0; in vm_page_alloc_nofree_domain()
2714 m = &nqp->ma[nqp->offs++]; in vm_page_alloc_nofree_domain()
2718 return (m); in vm_page_alloc_nofree_domain()
2725 vm_page_t m; in vm_page_alloc_noobj() local
2728 vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); in vm_page_alloc_noobj()
2730 m = vm_page_alloc_noobj_domain(domain, req); in vm_page_alloc_noobj()
2731 if (m != NULL) in vm_page_alloc_noobj()
2733 } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); in vm_page_alloc_noobj()
2735 return (m); in vm_page_alloc_noobj()
2744 vm_page_t m; in vm_page_alloc_noobj_contig() local
2747 vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); in vm_page_alloc_noobj_contig()
2749 m = vm_page_alloc_noobj_contig_domain(domain, req, npages, low, in vm_page_alloc_noobj_contig()
2751 if (m != NULL) in vm_page_alloc_noobj_contig()
2753 } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); in vm_page_alloc_noobj_contig()
2755 return (m); in vm_page_alloc_noobj_contig()
2763 vm_page_t m, m_ret; in vm_page_alloc_noobj_contig_domain() local
2767 KASSERT((req & ~VPANC_FLAGS) == 0, in vm_page_alloc_noobj_contig_domain()
2775 KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); in vm_page_alloc_noobj_contig_domain()
2787 if ((req & VM_ALLOC_NODUMP) != 0) in vm_page_alloc_noobj_contig_domain()
2789 if ((req & VM_ALLOC_WIRED) != 0) in vm_page_alloc_noobj_contig_domain()
2791 for (m = m_ret; m < &m_ret[npages]; m++) { in vm_page_alloc_noobj_contig_domain()
2792 vm_page_dequeue(m); in vm_page_alloc_noobj_contig_domain()
2793 vm_page_alloc_check(m); in vm_page_alloc_noobj_contig_domain()
2798 m->pindex = 0xdeadc0dedeadc0de; in vm_page_alloc_noobj_contig_domain()
2799 m->a.flags = 0; in vm_page_alloc_noobj_contig_domain()
2800 m->flags = (m->flags | PG_NODUMP) & flags; in vm_page_alloc_noobj_contig_domain()
2801 m->busy_lock = VPB_UNBUSIED; in vm_page_alloc_noobj_contig_domain()
2802 if ((req & VM_ALLOC_WIRED) != 0) in vm_page_alloc_noobj_contig_domain()
2803 m->ref_count = 1; in vm_page_alloc_noobj_contig_domain()
2804 m->a.act_count = 0; in vm_page_alloc_noobj_contig_domain()
2805 m->oflags = VPO_UNMANAGED; in vm_page_alloc_noobj_contig_domain()
2813 if ((req & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) in vm_page_alloc_noobj_contig_domain()
2814 pmap_zero_page(m); in vm_page_alloc_noobj_contig_domain()
2816 pmap_page_set_memattr(m, memattr); in vm_page_alloc_noobj_contig_domain()
2825 vm_page_alloc_check(vm_page_t m) in vm_page_alloc_check() argument
2828 KASSERT(m->object == NULL, ("page %p has object", m)); in vm_page_alloc_check()
2829 KASSERT(m->a.queue == PQ_NONE && in vm_page_alloc_check()
2830 (m->a.flags & PGA_QUEUE_STATE_MASK) == 0, in vm_page_alloc_check()
2832 m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK))); in vm_page_alloc_check()
2833 KASSERT(m->ref_count == 0, ("page %p has references", m)); in vm_page_alloc_check()
2834 KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m)); in vm_page_alloc_check()
2835 KASSERT(m->dirty == 0, ("page %p is dirty", m)); in vm_page_alloc_check()
2836 KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, in vm_page_alloc_check()
2838 m, pmap_page_get_memattr(m))); in vm_page_alloc_check()
2839 KASSERT(vm_page_none_valid(m), ("free page %p is valid", m)); in vm_page_alloc_check()
2840 pmap_vm_page_alloc_check(m); in vm_page_alloc_check()
2859 return (0); in vm_page_zone_import()
2876 vm_page_t m; in vm_page_zone_release() local
2882 for (i = 0; i < cnt; i++) { in vm_page_zone_release()
2883 m = (vm_page_t)store[i]; in vm_page_zone_release()
2884 vm_phys_free_pages(m, 0); in vm_page_zone_release()
2890 #define VPSC_ANY 0 /* No restrictions. */
2918 vm_page_t m, m_run; in vm_page_scan_contig() local
2919 #if VM_NRESERVLEVEL > 0 in vm_page_scan_contig()
2924 KASSERT(npages > 0, ("npages is 0")); in vm_page_scan_contig()
2928 run_len = 0; in vm_page_scan_contig()
2929 for (m = m_start; m < m_end && run_len < npages; m += m_inc) { in vm_page_scan_contig()
2930 KASSERT((m->flags & PG_MARKER) == 0, in vm_page_scan_contig()
2931 ("page %p is PG_MARKER", m)); in vm_page_scan_contig()
2932 KASSERT((m->flags & PG_FICTITIOUS) == 0 || m->ref_count >= 1, in vm_page_scan_contig()
2933 ("fictitious page %p has invalid ref count", m)); in vm_page_scan_contig()
2942 if (run_len == 0) { in vm_page_scan_contig()
2944 if (m + npages > m_end) in vm_page_scan_contig()
2946 pa = VM_PAGE_TO_PHYS(m); in vm_page_scan_contig()
2960 if (vm_page_wired(m)) in vm_page_scan_contig()
2961 run_ext = 0; in vm_page_scan_contig()
2962 #if VM_NRESERVLEVEL > 0 in vm_page_scan_contig()
2963 else if ((level = vm_reserv_level(m)) >= 0 && in vm_page_scan_contig()
2964 (options & VPSC_NORESERV) != 0) { in vm_page_scan_contig()
2965 run_ext = 0; in vm_page_scan_contig()
2967 pa = VM_PAGE_TO_PHYS(m); in vm_page_scan_contig()
2972 else if ((object = atomic_load_ptr(&m->object)) != NULL) { in vm_page_scan_contig()
2979 if (object != m->object) { in vm_page_scan_contig()
2984 if ((object->flags & OBJ_SWAP) == 0 && in vm_page_scan_contig()
2986 run_ext = 0; in vm_page_scan_contig()
2987 #if VM_NRESERVLEVEL > 0 in vm_page_scan_contig()
2988 } else if ((options & VPSC_NOSUPER) != 0 && in vm_page_scan_contig()
2989 (level = vm_reserv_level_iffullpop(m)) >= 0) { in vm_page_scan_contig()
2990 run_ext = 0; in vm_page_scan_contig()
2992 pa = VM_PAGE_TO_PHYS(m); in vm_page_scan_contig()
2997 vm_page_queue(m) != PQ_NONE && !vm_page_busied(m)) { in vm_page_scan_contig()
3003 KASSERT(pmap_page_get_memattr(m) == in vm_page_scan_contig()
3005 ("page %p has an unexpected memattr", m)); in vm_page_scan_contig()
3006 KASSERT((m->oflags & (VPO_SWAPINPROG | in vm_page_scan_contig()
3007 VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0, in vm_page_scan_contig()
3008 ("page %p has unexpected oflags", m)); in vm_page_scan_contig()
3012 run_ext = 0; in vm_page_scan_contig()
3014 #if VM_NRESERVLEVEL > 0 in vm_page_scan_contig()
3015 } else if (level >= 0) { in vm_page_scan_contig()
3023 } else if ((order = m->order) < VM_NFREEORDER) { in vm_page_scan_contig()
3047 run_ext = 0; in vm_page_scan_contig()
3053 if (run_ext > 0) { in vm_page_scan_contig()
3054 if (run_len == 0) in vm_page_scan_contig()
3055 m_run = m; in vm_page_scan_contig()
3058 if (run_len > 0) { in vm_page_scan_contig()
3060 run_len = 0; in vm_page_scan_contig()
3079 * Returns 0 if every physical page within the run was already free or
3094 vm_page_t m, m_end, m_new; in vm_page_reclaim_run() local
3100 error = 0; in vm_page_reclaim_run()
3101 m = m_run; in vm_page_reclaim_run()
3103 for (; error == 0 && m < m_end; m++) { in vm_page_reclaim_run()
3104 KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, in vm_page_reclaim_run()
3105 ("page %p is PG_FICTITIOUS or PG_MARKER", m)); in vm_page_reclaim_run()
3111 if (vm_page_wired(m)) in vm_page_reclaim_run()
3113 else if ((object = atomic_load_ptr(&m->object)) != NULL) { in vm_page_reclaim_run()
3120 if (m->object != object || in vm_page_reclaim_run()
3121 ((object->flags & OBJ_SWAP) == 0 && in vm_page_reclaim_run()
3126 else if (vm_page_queue(m) != PQ_NONE && in vm_page_reclaim_run()
3127 vm_page_tryxbusy(m) != 0) { in vm_page_reclaim_run()
3128 if (vm_page_wired(m)) { in vm_page_reclaim_run()
3129 vm_page_xunbusy(m); in vm_page_reclaim_run()
3133 KASSERT(pmap_page_get_memattr(m) == in vm_page_reclaim_run()
3135 ("page %p has an unexpected memattr", m)); in vm_page_reclaim_run()
3136 KASSERT(m->oflags == 0, in vm_page_reclaim_run()
3137 ("page %p has unexpected oflags", m)); in vm_page_reclaim_run()
3139 if (!vm_page_none_valid(m)) { in vm_page_reclaim_run()
3150 if ((m->flags & PG_NODUMP) != 0) in vm_page_reclaim_run()
3157 ~(vm_paddr_t)0, PAGE_SIZE, in vm_page_reclaim_run()
3158 0, VM_MEMATTR_DEFAULT); in vm_page_reclaim_run()
3165 req, 1, 0, pa - 1, in vm_page_reclaim_run()
3166 PAGE_SIZE, 0, in vm_page_reclaim_run()
3174 0, VM_MEMATTR_DEFAULT); in vm_page_reclaim_run()
3177 vm_page_xunbusy(m); in vm_page_reclaim_run()
3187 if (object->ref_count != 0 && in vm_page_reclaim_run()
3188 !vm_page_try_remove_all(m)) { in vm_page_reclaim_run()
3189 vm_page_xunbusy(m); in vm_page_reclaim_run()
3196 * Replace "m" with the new page. For in vm_page_reclaim_run()
3197 * vm_page_replace(), "m" must be busy in vm_page_reclaim_run()
3198 * and dequeued. Finally, change "m" in vm_page_reclaim_run()
3201 m_new->a.flags = m->a.flags & in vm_page_reclaim_run()
3205 m_new->oflags = 0; in vm_page_reclaim_run()
3206 pmap_copy_page(m, m_new); in vm_page_reclaim_run()
3207 m_new->valid = m->valid; in vm_page_reclaim_run()
3208 m_new->dirty = m->dirty; in vm_page_reclaim_run()
3209 m->flags &= ~PG_ZERO; in vm_page_reclaim_run()
3210 vm_page_dequeue(m); in vm_page_reclaim_run()
3212 m->pindex, m) && in vm_page_reclaim_run()
3213 vm_page_free_prep(m)) in vm_page_reclaim_run()
3214 SLIST_INSERT_HEAD(&free, m, in vm_page_reclaim_run()
3223 m->flags &= ~PG_ZERO; in vm_page_reclaim_run()
3224 vm_page_dequeue(m); in vm_page_reclaim_run()
3225 if (vm_page_free_prep(m)) in vm_page_reclaim_run()
3226 SLIST_INSERT_HEAD(&free, m, in vm_page_reclaim_run()
3228 KASSERT(m->dirty == 0, in vm_page_reclaim_run()
3229 ("page %p is dirty", m)); in vm_page_reclaim_run()
3236 MPASS(vm_page_domain(m) == domain); in vm_page_reclaim_run()
3239 order = m->order; in vm_page_reclaim_run()
3249 m += (1 << order) - 1; in vm_page_reclaim_run()
3251 #if VM_NRESERVLEVEL > 0 in vm_page_reclaim_run()
3252 else if (vm_reserv_is_page_free(m)) in vm_page_reclaim_run()
3253 order = 0; in vm_page_reclaim_run()
3260 if ((m = SLIST_FIRST(&free)) != NULL) { in vm_page_reclaim_run()
3264 cnt = 0; in vm_page_reclaim_run()
3267 MPASS(vm_page_domain(m) == domain); in vm_page_reclaim_run()
3269 vm_phys_free_pages(m, 0); in vm_page_reclaim_run()
3271 } while ((m = SLIST_FIRST(&free)) != NULL); in vm_page_reclaim_run()
3289 * Returns 0 if reclamation is successful, ERANGE if the specified domain
3321 KASSERT(npages > 0, ("npages is 0")); in vm_page_reclaim_contig_domain_ext()
3389 count = 0; in vm_page_reclaim_contig_domain_ext()
3394 while ((m_run = vm_page_scan_contig(npages, bounds[0], in vm_page_reclaim_contig_domain_ext()
3396 bounds[0] = m_run + npages; in vm_page_reclaim_contig_domain_ext()
3415 reclaimed = 0; in vm_page_reclaim_contig_domain_ext()
3416 for (i = 0; count > 0 && i < nruns; i++) { in vm_page_reclaim_contig_domain_ext()
3421 if (error == 0) { in vm_page_reclaim_contig_domain_ext()
3424 ret = 0; in vm_page_reclaim_contig_domain_ext()
3439 if (reclaimed != 0) in vm_page_reclaim_contig_domain_ext()
3440 ret = 0; in vm_page_reclaim_contig_domain_ext()
3467 vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); in vm_page_reclaim_contig()
3471 if (status == 0) in vm_page_reclaim_contig()
3472 return (0); in vm_page_reclaim_contig()
3480 } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); in vm_page_reclaim_contig()
3513 vmd->vmd_minset = 0; in vm_domain_clear()
3515 if (vm_min_waiters != 0) { in vm_domain_clear()
3516 vm_min_waiters = 0; in vm_domain_clear()
3521 vmd->vmd_severeset = 0; in vm_domain_clear()
3523 if (vm_severe_waiters != 0) { in vm_domain_clear()
3524 vm_severe_waiters = 0; in vm_domain_clear()
3536 vmd->vmd_pageout_pages_needed = 0; in vm_domain_clear()
3541 vm_pageproc_waiters = 0; in vm_domain_clear()
3557 msleep(&vm_min_domains, &vm_domainset_lock, PVM, "vmwait", 0); in vm_wait_min()
3573 "vmwait", 0); in vm_wait_severe()
3590 error = 0; in vm_wait_doms()
3617 PVM | PDROP | mflags, "vmwait", 0); in vm_wait_doms()
3644 &vm_domainset_lock, PDROP | PSWP, "VMWait", 0); in vm_wait_domain()
3650 vm_wait_doms(&wdom, 0); in vm_wait_domain()
3684 (void)vm_wait_flags(obj, 0); in vm_wait()
3721 return (0); in vm_domain_alloc_fail()
3753 _vm_page_pagequeue(vm_page_t m, uint8_t queue) in _vm_page_pagequeue() argument
3756 return (&vm_pagequeue_domain(m)->vmd_pagequeues[queue]); in _vm_page_pagequeue()
3761 vm_page_pagequeue(vm_page_t m) in vm_page_pagequeue() argument
3764 return (_vm_page_pagequeue(m, vm_page_astate_load(m).queue)); in vm_page_pagequeue()
3769 vm_page_pqstate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new) in vm_page_pqstate_fcmpset() argument
3775 if (__predict_true(vm_page_astate_fcmpset(m, old, new))) in vm_page_pqstate_fcmpset()
3788 _vm_page_pqstate_commit_dequeue(struct vm_pagequeue *pq, vm_page_t m, in _vm_page_pqstate_commit_dequeue() argument
3794 KASSERT(vm_page_pagequeue(m) == pq, in _vm_page_pqstate_commit_dequeue()
3795 ("%s: queue %p does not match page %p", __func__, pq, m)); in _vm_page_pqstate_commit_dequeue()
3808 if ((old->flags & PGA_ENQUEUED) != 0) { in _vm_page_pqstate_commit_dequeue()
3810 next = TAILQ_NEXT(m, plinks.q); in _vm_page_pqstate_commit_dequeue()
3811 TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); in _vm_page_pqstate_commit_dequeue()
3813 if (!vm_page_pqstate_fcmpset(m, old, new)) { in _vm_page_pqstate_commit_dequeue()
3815 TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); in _vm_page_pqstate_commit_dequeue()
3817 TAILQ_INSERT_BEFORE(next, m, plinks.q); in _vm_page_pqstate_commit_dequeue()
3824 return (vm_page_pqstate_fcmpset(m, old, new)); in _vm_page_pqstate_commit_dequeue()
3829 vm_page_pqstate_commit_dequeue(vm_page_t m, vm_page_astate_t *old, in vm_page_pqstate_commit_dequeue() argument
3836 pq = _vm_page_pagequeue(m, old->queue); in vm_page_pqstate_commit_dequeue()
3843 as = vm_page_astate_load(m); in vm_page_pqstate_commit_dequeue()
3848 ret = _vm_page_pqstate_commit_dequeue(pq, m, old, new); in vm_page_pqstate_commit_dequeue()
3858 _vm_page_pqstate_commit_requeue(struct vm_pagequeue *pq, vm_page_t m, in _vm_page_pqstate_commit_requeue() argument
3869 if (!vm_page_pqstate_fcmpset(m, old, new)) in _vm_page_pqstate_commit_requeue()
3872 if ((old->flags & PGA_ENQUEUED) != 0) in _vm_page_pqstate_commit_requeue()
3873 TAILQ_REMOVE(&pq->pq_pl, m, plinks.q); in _vm_page_pqstate_commit_requeue()
3882 if ((old->flags & PGA_REQUEUE_HEAD) != 0) { in _vm_page_pqstate_commit_requeue()
3883 vmd = vm_pagequeue_domain(m); in _vm_page_pqstate_commit_requeue()
3885 ("%s: invalid page queue for page %p", __func__, m)); in _vm_page_pqstate_commit_requeue()
3886 TAILQ_INSERT_BEFORE(&vmd->vmd_inacthead, m, plinks.q); in _vm_page_pqstate_commit_requeue()
3888 TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q); in _vm_page_pqstate_commit_requeue()
3898 vm_page_pqstate_commit_request(vm_page_t m, vm_page_astate_t *old, in vm_page_pqstate_commit_request() argument
3907 !vm_page_pqstate_fcmpset(m, old, new)) in vm_page_pqstate_commit_request()
3909 vm_page_pqbatch_submit(m, new.queue); in vm_page_pqstate_commit_request()
3918 vm_page_pqstate_commit(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new) in vm_page_pqstate_commit() argument
3925 if (!vm_page_pqstate_commit_dequeue(m, old, new)) in vm_page_pqstate_commit()
3928 vm_page_pqbatch_submit(m, new.queue); in vm_page_pqstate_commit()
3930 if (!vm_page_pqstate_fcmpset(m, old, new)) in vm_page_pqstate_commit()
3933 ((new.flags & ~old->flags) & PGA_QUEUE_OP_MASK) != 0) in vm_page_pqstate_commit()
3934 vm_page_pqbatch_submit(m, new.queue); in vm_page_pqstate_commit()
3943 vm_pqbatch_process_page(struct vm_pagequeue *pq, vm_page_t m, uint8_t queue) in vm_pqbatch_process_page() argument
3951 KASSERT(pq == _vm_page_pagequeue(m, queue), in vm_pqbatch_process_page()
3952 ("%s: page %p does not belong to queue %p", __func__, m, pq)); in vm_pqbatch_process_page()
3954 for (old = vm_page_astate_load(m);;) { in vm_pqbatch_process_page()
3956 (old.flags & PGA_QUEUE_OP_MASK) == 0)) { in vm_pqbatch_process_page()
3960 KASSERT((m->oflags & VPO_UNMANAGED) == 0, in vm_pqbatch_process_page()
3961 ("%s: page %p is unmanaged", __func__, m)); in vm_pqbatch_process_page()
3964 if ((old.flags & PGA_DEQUEUE) != 0) { in vm_pqbatch_process_page()
3968 m, &old, new))) { in vm_pqbatch_process_page()
3975 m, &old, new))) { in vm_pqbatch_process_page()
3989 for (i = 0; i < bq->bq_cnt; i++) in vm_pqbatch_process()
4002 vm_page_pqbatch_submit(vm_page_t m, uint8_t queue) in vm_page_pqbatch_submit() argument
4010 domain = vm_page_domain(m); in vm_page_pqbatch_submit()
4013 slots_remaining = vm_batchqueue_insert(bq, m); in vm_page_pqbatch_submit()
4018 } else if (slots_remaining > 0 ) { in vm_page_pqbatch_submit()
4037 vm_pqbatch_process_page(pq, m, queue); in vm_page_pqbatch_submit()
4063 for (domain = 0; domain < vm_ndomains; domain++) { in vm_page_pqbatch_drain()
4065 for (queue = 0; queue < PQ_COUNT; queue++) { in vm_page_pqbatch_drain()
4089 vm_page_dequeue_deferred(vm_page_t m) in vm_page_dequeue_deferred() argument
4093 old = vm_page_astate_load(m); in vm_page_dequeue_deferred()
4096 KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0, in vm_page_dequeue_deferred()
4098 __func__, m)); in vm_page_dequeue_deferred()
4103 } while (!vm_page_pqstate_commit_request(m, &old, new)); in vm_page_dequeue_deferred()
4113 vm_page_dequeue(vm_page_t m) in vm_page_dequeue() argument
4117 old = vm_page_astate_load(m); in vm_page_dequeue()
4120 KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0, in vm_page_dequeue()
4122 __func__, m)); in vm_page_dequeue()
4128 } while (!vm_page_pqstate_commit_dequeue(m, &old, new)); in vm_page_dequeue()
4137 vm_page_enqueue(vm_page_t m, uint8_t queue) in vm_page_enqueue() argument
4140 KASSERT(m->a.queue == PQ_NONE && in vm_page_enqueue()
4141 (m->a.flags & PGA_QUEUE_STATE_MASK) == 0, in vm_page_enqueue()
4142 ("%s: page %p is already enqueued", __func__, m)); in vm_page_enqueue()
4143 KASSERT(m->ref_count > 0, in vm_page_enqueue()
4144 ("%s: page %p does not carry any references", __func__, m)); in vm_page_enqueue()
4146 m->a.queue = queue; in vm_page_enqueue()
4147 if ((m->a.flags & PGA_REQUEUE) == 0) in vm_page_enqueue()
4148 vm_page_aflag_set(m, PGA_REQUEUE); in vm_page_enqueue()
4149 vm_page_pqbatch_submit(m, queue); in vm_page_enqueue()
4164 vm_page_free_prep(vm_page_t m) in vm_page_free_prep() argument
4174 if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) { in vm_page_free_prep()
4177 p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); in vm_page_free_prep()
4178 for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++) in vm_page_free_prep()
4179 KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx", in vm_page_free_prep()
4180 m, i, (uintmax_t)*p)); in vm_page_free_prep()
4183 KASSERT((m->flags & PG_NOFREE) == 0, in vm_page_free_prep()
4185 if ((m->oflags & VPO_UNMANAGED) == 0) { in vm_page_free_prep()
4186 KASSERT(!pmap_page_is_mapped(m), in vm_page_free_prep()
4187 ("vm_page_free_prep: freeing mapped page %p", m)); in vm_page_free_prep()
4188 KASSERT((m->a.flags & (PGA_EXECUTABLE | PGA_WRITEABLE)) == 0, in vm_page_free_prep()
4189 ("vm_page_free_prep: mapping flags set in page %p", m)); in vm_page_free_prep()
4191 KASSERT(m->a.queue == PQ_NONE, in vm_page_free_prep()
4192 ("vm_page_free_prep: unmanaged page %p is queued", m)); in vm_page_free_prep()
4196 if (m->object != NULL) { in vm_page_free_prep()
4197 vm_page_radix_remove(m); in vm_page_free_prep()
4198 vm_page_free_object_prep(m); in vm_page_free_prep()
4200 vm_page_assert_unbusied(m); in vm_page_free_prep()
4202 vm_page_busy_free(m); in vm_page_free_prep()
4208 if ((m->flags & PG_FICTITIOUS) != 0) { in vm_page_free_prep()
4209 KASSERT(m->ref_count == 1, in vm_page_free_prep()
4210 ("fictitious page %p is referenced", m)); in vm_page_free_prep()
4211 KASSERT(m->a.queue == PQ_NONE, in vm_page_free_prep()
4212 ("fictitious page %p is queued", m)); in vm_page_free_prep()
4221 if ((m->oflags & VPO_UNMANAGED) == 0) in vm_page_free_prep()
4222 vm_page_dequeue_deferred(m); in vm_page_free_prep()
4224 m->valid = 0; in vm_page_free_prep()
4225 vm_page_undirty(m); in vm_page_free_prep()
4227 if (m->ref_count != 0) in vm_page_free_prep()
4228 panic("vm_page_free_prep: page %p has references", m); in vm_page_free_prep()
4233 if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT) in vm_page_free_prep()
4234 pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT); in vm_page_free_prep()
4236 #if VM_NRESERVLEVEL > 0 in vm_page_free_prep()
4242 if ((m->flags & PG_PCPU_CACHE) == 0 && vm_reserv_free_page(m)) in vm_page_free_prep()
4259 vm_page_free_toq(vm_page_t m) in vm_page_free_toq() argument
4264 if (!vm_page_free_prep(m)) in vm_page_free_toq()
4267 vmd = vm_pagequeue_domain(m); in vm_page_free_toq()
4268 zone = vmd->vmd_pgcache[m->pool].zone; in vm_page_free_toq()
4269 if ((m->flags & PG_PCPU_CACHE) != 0 && zone != NULL) { in vm_page_free_toq()
4270 uma_zfree(zone, m); in vm_page_free_toq()
4274 vm_phys_free_pages(m, 0); in vm_page_free_toq()
4289 vm_page_t m; in vm_page_free_pages_toq() local
4293 return (0); in vm_page_free_pages_toq()
4295 count = 0; in vm_page_free_pages_toq()
4296 while ((m = SLIST_FIRST(free)) != NULL) { in vm_page_free_pages_toq()
4299 vm_page_free_toq(m); in vm_page_free_pages_toq()
4312 vm_page_wire(vm_page_t m) in vm_page_wire() argument
4317 if (m->object != NULL && !vm_page_busied(m) && in vm_page_wire()
4318 !vm_object_busied(m->object)) in vm_page_wire()
4319 VM_OBJECT_ASSERT_LOCKED(m->object); in vm_page_wire()
4321 KASSERT((m->flags & PG_FICTITIOUS) == 0 || in vm_page_wire()
4322 VPRC_WIRE_COUNT(m->ref_count) >= 1, in vm_page_wire()
4323 ("vm_page_wire: fictitious page %p has zero wirings", m)); in vm_page_wire()
4325 old = atomic_fetchadd_int(&m->ref_count, 1); in vm_page_wire()
4327 ("vm_page_wire: counter overflow for page %p", m)); in vm_page_wire()
4328 if (VPRC_WIRE_COUNT(old) == 0) { in vm_page_wire()
4329 if ((m->oflags & VPO_UNMANAGED) == 0) in vm_page_wire()
4330 vm_page_aflag_set(m, PGA_DEQUEUE); in vm_page_wire()
4343 vm_page_wire_mapped(vm_page_t m) in vm_page_wire_mapped() argument
4347 old = atomic_load_int(&m->ref_count); in vm_page_wire_mapped()
4349 KASSERT(old > 0, in vm_page_wire_mapped()
4350 ("vm_page_wire_mapped: wiring unreferenced page %p", m)); in vm_page_wire_mapped()
4351 if ((old & VPRC_BLOCKED) != 0) in vm_page_wire_mapped()
4353 } while (!atomic_fcmpset_int(&m->ref_count, &old, old + 1)); in vm_page_wire_mapped()
4355 if (VPRC_WIRE_COUNT(old) == 0) { in vm_page_wire_mapped()
4356 if ((m->oflags & VPO_UNMANAGED) == 0) in vm_page_wire_mapped()
4357 vm_page_aflag_set(m, PGA_DEQUEUE); in vm_page_wire_mapped()
4369 vm_page_unwire_managed(vm_page_t m, uint8_t nqueue, bool noreuse) in vm_page_unwire_managed() argument
4373 KASSERT((m->oflags & VPO_UNMANAGED) == 0, in vm_page_unwire_managed()
4374 ("%s: page %p is unmanaged", __func__, m)); in vm_page_unwire_managed()
4381 old = atomic_load_int(&m->ref_count); in vm_page_unwire_managed()
4385 KASSERT(VPRC_WIRE_COUNT(old) > 0, in vm_page_unwire_managed()
4386 ("vm_page_unwire: wire count underflow for page %p", m)); in vm_page_unwire_managed()
4396 if ((vm_page_astate_load(m).flags & PGA_DEQUEUE) == 0) in vm_page_unwire_managed()
4397 vm_page_aflag_set(m, PGA_DEQUEUE); in vm_page_unwire_managed()
4406 vm_page_release_toq(m, nqueue, noreuse); in vm_page_unwire_managed()
4408 vm_page_aflag_clear(m, PGA_DEQUEUE); in vm_page_unwire_managed()
4410 } while (!atomic_fcmpset_rel_int(&m->ref_count, &old, old - 1)); in vm_page_unwire_managed()
4415 vm_page_free(m); in vm_page_unwire_managed()
4429 vm_page_unwire(vm_page_t m, uint8_t nqueue) in vm_page_unwire() argument
4434 nqueue, m)); in vm_page_unwire()
4436 if ((m->oflags & VPO_UNMANAGED) != 0) { in vm_page_unwire()
4437 if (vm_page_unwire_noq(m) && m->ref_count == 0) in vm_page_unwire()
4438 vm_page_free(m); in vm_page_unwire()
4441 vm_page_unwire_managed(m, nqueue, false); in vm_page_unwire()
4451 vm_page_unwire_noq(vm_page_t m) in vm_page_unwire_noq() argument
4455 old = vm_page_drop(m, 1); in vm_page_unwire_noq()
4456 KASSERT(VPRC_WIRE_COUNT(old) != 0, in vm_page_unwire_noq()
4457 ("%s: counter underflow for page %p", __func__, m)); in vm_page_unwire_noq()
4458 KASSERT((m->flags & PG_FICTITIOUS) == 0 || VPRC_WIRE_COUNT(old) > 1, in vm_page_unwire_noq()
4459 ("%s: missing ref on fictitious page %p", __func__, m)); in vm_page_unwire_noq()
4463 if ((m->oflags & VPO_UNMANAGED) == 0) in vm_page_unwire_noq()
4464 vm_page_aflag_clear(m, PGA_DEQUEUE); in vm_page_unwire_noq()
4475 vm_page_mvqueue(vm_page_t m, const uint8_t nqueue, const uint16_t nflag) in vm_page_mvqueue() argument
4479 KASSERT(m->ref_count > 0, in vm_page_mvqueue()
4480 ("%s: page %p does not carry any references", __func__, m)); in vm_page_mvqueue()
4484 if ((m->oflags & VPO_UNMANAGED) != 0 || vm_page_wired(m)) in vm_page_mvqueue()
4487 old = vm_page_astate_load(m); in vm_page_mvqueue()
4489 if ((old.flags & PGA_DEQUEUE) != 0) in vm_page_mvqueue()
4501 (old.flags & PGA_ENQUEUED) == 0) in vm_page_mvqueue()
4507 } while (!vm_page_pqstate_commit(m, &old, new)); in vm_page_mvqueue()
4514 vm_page_activate(vm_page_t m) in vm_page_activate() argument
4517 vm_page_mvqueue(m, PQ_ACTIVE, PGA_REQUEUE); in vm_page_activate()
4525 vm_page_deactivate(vm_page_t m) in vm_page_deactivate() argument
4528 vm_page_mvqueue(m, PQ_INACTIVE, PGA_REQUEUE); in vm_page_deactivate()
4532 vm_page_deactivate_noreuse(vm_page_t m) in vm_page_deactivate_noreuse() argument
4535 vm_page_mvqueue(m, PQ_INACTIVE, PGA_REQUEUE_HEAD); in vm_page_deactivate_noreuse()
4542 vm_page_launder(vm_page_t m) in vm_page_launder() argument
4545 vm_page_mvqueue(m, PQ_LAUNDRY, PGA_REQUEUE); in vm_page_launder()
4552 vm_page_unswappable(vm_page_t m) in vm_page_unswappable() argument
4555 VM_OBJECT_ASSERT_LOCKED(m->object); in vm_page_unswappable()
4556 KASSERT((m->oflags & VPO_UNMANAGED) == 0, in vm_page_unswappable()
4557 ("page %p already unswappable", m)); in vm_page_unswappable()
4559 vm_page_dequeue(m); in vm_page_unswappable()
4560 vm_page_enqueue(m, PQ_UNSWAPPABLE); in vm_page_unswappable()
4567 vm_page_release_toq(vm_page_t m, uint8_t nqueue, const bool noreuse) in vm_page_release_toq() argument
4582 if (noreuse || vm_page_none_valid(m)) { in vm_page_release_toq()
4589 old = vm_page_astate_load(m); in vm_page_release_toq()
4600 (old.flags & PGA_ENQUEUED) != 0) in vm_page_release_toq()
4606 } while (!vm_page_pqstate_commit(m, &old, new)); in vm_page_release_toq()
4613 vm_page_release(vm_page_t m, int flags) in vm_page_release() argument
4617 KASSERT((m->oflags & VPO_UNMANAGED) == 0, in vm_page_release()
4618 ("vm_page_release: page %p is unmanaged", m)); in vm_page_release()
4620 if ((flags & VPR_TRYFREE) != 0) { in vm_page_release()
4622 object = atomic_load_ptr(&m->object); in vm_page_release()
4626 if (vm_page_busied(m) || !VM_OBJECT_TRYWLOCK(object)) in vm_page_release()
4628 if (object == m->object) { in vm_page_release()
4629 vm_page_release_locked(m, flags); in vm_page_release()
4636 vm_page_unwire_managed(m, PQ_INACTIVE, flags != 0); in vm_page_release()
4641 vm_page_release_locked(vm_page_t m, int flags) in vm_page_release_locked() argument
4644 VM_OBJECT_ASSERT_WLOCKED(m->object); in vm_page_release_locked()
4645 KASSERT((m->oflags & VPO_UNMANAGED) == 0, in vm_page_release_locked()
4646 ("vm_page_release_locked: page %p is unmanaged", m)); in vm_page_release_locked()
4648 if (vm_page_unwire_noq(m)) { in vm_page_release_locked()
4649 if ((flags & VPR_TRYFREE) != 0 && in vm_page_release_locked()
4650 (m->object->ref_count == 0 || !pmap_page_is_mapped(m)) && in vm_page_release_locked()
4651 m->dirty == 0 && vm_page_tryxbusy(m)) { in vm_page_release_locked()
4657 if (__predict_true(!vm_page_wired(m))) { in vm_page_release_locked()
4658 vm_page_free(m); in vm_page_release_locked()
4661 vm_page_xunbusy(m); in vm_page_release_locked()
4663 vm_page_release_toq(m, PQ_INACTIVE, flags != 0); in vm_page_release_locked()
4669 vm_page_try_blocked_op(vm_page_t m, void (*op)(vm_page_t)) in vm_page_try_blocked_op() argument
4673 KASSERT(m->object != NULL && (m->oflags & VPO_UNMANAGED) == 0, in vm_page_try_blocked_op()
4674 ("vm_page_try_blocked_op: page %p has no object", m)); in vm_page_try_blocked_op()
4675 KASSERT(vm_page_busied(m), in vm_page_try_blocked_op()
4676 ("vm_page_try_blocked_op: page %p is not busy", m)); in vm_page_try_blocked_op()
4677 VM_OBJECT_ASSERT_LOCKED(m->object); in vm_page_try_blocked_op()
4679 old = atomic_load_int(&m->ref_count); in vm_page_try_blocked_op()
4681 KASSERT(old != 0, in vm_page_try_blocked_op()
4682 ("vm_page_try_blocked_op: page %p has no references", m)); in vm_page_try_blocked_op()
4683 KASSERT((old & VPRC_BLOCKED) == 0, in vm_page_try_blocked_op()
4684 ("vm_page_try_blocked_op: page %p blocks wirings", m)); in vm_page_try_blocked_op()
4685 if (VPRC_WIRE_COUNT(old) != 0) in vm_page_try_blocked_op()
4687 } while (!atomic_fcmpset_int(&m->ref_count, &old, old | VPRC_BLOCKED)); in vm_page_try_blocked_op()
4689 (op)(m); in vm_page_try_blocked_op()
4695 old = vm_page_drop(m, VPRC_BLOCKED); in vm_page_try_blocked_op()
4696 KASSERT(!VM_OBJECT_WOWNED(m->object) || in vm_page_try_blocked_op()
4699 old, m)); in vm_page_try_blocked_op()
4707 vm_page_try_remove_all(vm_page_t m) in vm_page_try_remove_all() argument
4710 return (vm_page_try_blocked_op(m, pmap_remove_all)); in vm_page_try_remove_all()
4717 vm_page_try_remove_write(vm_page_t m) in vm_page_try_remove_write() argument
4720 return (vm_page_try_blocked_op(m, pmap_remove_write)); in vm_page_try_remove_write()
4729 vm_page_advise(vm_page_t m, int advice) in vm_page_advise() argument
4732 VM_OBJECT_ASSERT_WLOCKED(m->object); in vm_page_advise()
4733 vm_page_assert_xbusied(m); in vm_page_advise()
4742 vm_page_undirty(m); in vm_page_advise()
4745 vm_page_activate(m); in vm_page_advise()
4749 if (advice != MADV_FREE && m->dirty == 0 && pmap_is_modified(m)) in vm_page_advise()
4750 vm_page_dirty(m); in vm_page_advise()
4756 vm_page_aflag_clear(m, PGA_REFERENCED); in vm_page_advise()
4764 if (m->dirty == 0) in vm_page_advise()
4765 vm_page_deactivate_noreuse(m); in vm_page_advise()
4766 else if (!vm_page_in_laundry(m)) in vm_page_advise()
4767 vm_page_launder(m); in vm_page_advise()
4776 vm_page_grab_release(vm_page_t m, int allocflags) in vm_page_grab_release() argument
4779 if ((allocflags & VM_ALLOC_NOBUSY) != 0) { in vm_page_grab_release()
4780 if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) in vm_page_grab_release()
4781 vm_page_sunbusy(m); in vm_page_grab_release()
4783 vm_page_xunbusy(m); in vm_page_grab_release()
4798 vm_page_grab_sleep(vm_object_t object, vm_page_t m, vm_pindex_t pindex, in vm_page_grab_sleep() argument
4802 if ((allocflags & VM_ALLOC_NOWAIT) != 0) in vm_page_grab_sleep()
4809 if (locked && (allocflags & VM_ALLOC_NOCREAT) == 0) in vm_page_grab_sleep()
4810 vm_page_reference(m); in vm_page_grab_sleep()
4812 if (_vm_page_busy_sleep(object, m, pindex, wmesg, allocflags, locked) && in vm_page_grab_sleep()
4815 if ((allocflags & VM_ALLOC_WAITFAIL) != 0) in vm_page_grab_sleep()
4828 KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || in vm_page_grab_check()
4829 (allocflags & VM_ALLOC_WIRED) != 0, in vm_page_grab_check()
4832 KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || in vm_page_grab_check()
4833 (allocflags & VM_ALLOC_IGN_SBUSY) != 0, in vm_page_grab_check()
4848 if ((allocflags & VM_ALLOC_NOWAIT) == 0) in vm_page_grab_pflags()
4850 if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) in vm_page_grab_pflags()
4870 vm_page_t m; in vm_page_grab() local
4876 if ((m = vm_page_lookup(object, pindex)) != NULL) { in vm_page_grab()
4877 if (!vm_page_tryacquire(m, allocflags)) { in vm_page_grab()
4878 if (vm_page_grab_sleep(object, m, pindex, "pgrbwt", in vm_page_grab()
4885 if ((allocflags & VM_ALLOC_NOCREAT) != 0) in vm_page_grab()
4887 m = vm_page_alloc(object, pindex, vm_page_grab_pflags(allocflags)); in vm_page_grab()
4888 if (m == NULL) { in vm_page_grab()
4889 if ((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0) in vm_page_grab()
4893 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) in vm_page_grab()
4894 pmap_zero_page(m); in vm_page_grab()
4897 vm_page_grab_release(m, allocflags); in vm_page_grab()
4899 return (m); in vm_page_grab()
4917 vm_page_acquire_unlocked(vm_object_t object, vm_pindex_t pindex, vm_page_t m, in vm_page_acquire_unlocked() argument
4920 if (m == NULL) in vm_page_acquire_unlocked()
4921 m = vm_page_lookup_unlocked(object, pindex); in vm_page_acquire_unlocked()
4922 for (; m != NULL; m = vm_page_lookup_unlocked(object, pindex)) { in vm_page_acquire_unlocked()
4923 if (vm_page_trybusy(m, allocflags)) { in vm_page_acquire_unlocked()
4924 if (m->object == object && m->pindex == pindex) { in vm_page_acquire_unlocked()
4925 if ((allocflags & VM_ALLOC_WIRED) != 0) in vm_page_acquire_unlocked()
4926 vm_page_wire(m); in vm_page_acquire_unlocked()
4927 vm_page_grab_release(m, allocflags); in vm_page_acquire_unlocked()
4931 vm_page_busy_release(m); in vm_page_acquire_unlocked()
4935 if (!vm_page_grab_sleep(object, m, pindex, "pgnslp", in vm_page_acquire_unlocked()
4939 return (m); in vm_page_acquire_unlocked()
4949 vm_page_t m; in vm_page_grab_unlocked() local
4952 m = vm_page_acquire_unlocked(object, pindex, NULL, allocflags); in vm_page_grab_unlocked()
4953 if (m == PAGE_NOT_ACQUIRED) in vm_page_grab_unlocked()
4955 if (m != NULL) in vm_page_grab_unlocked()
4956 return (m); in vm_page_grab_unlocked()
4964 if ((allocflags & VM_ALLOC_NOCREAT) != 0) in vm_page_grab_unlocked()
4968 m = vm_page_grab(object, pindex, allocflags); in vm_page_grab_unlocked()
4971 return (m); in vm_page_grab_unlocked()
4984 vm_page_t m; in vm_page_grab_valid() local
4988 KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || in vm_page_grab_valid()
4989 (allocflags & VM_ALLOC_IGN_SBUSY) != 0, in vm_page_grab_valid()
4992 (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | VM_ALLOC_ZERO)) == 0, in vm_page_grab_valid()
4993 ("vm_page_grab_valid: Invalid flags 0x%X", allocflags)); in vm_page_grab_valid()
5000 if ((m = vm_page_lookup(object, pindex)) != NULL) { in vm_page_grab_valid()
5010 if (!vm_page_trybusy(m, in vm_page_grab_valid()
5011 vm_page_all_valid(m) ? allocflags : 0)) { in vm_page_grab_valid()
5012 (void)vm_page_grab_sleep(object, m, pindex, "pgrbwt", in vm_page_grab_valid()
5016 if (vm_page_all_valid(m)) in vm_page_grab_valid()
5018 if ((allocflags & VM_ALLOC_NOCREAT) != 0) { in vm_page_grab_valid()
5019 vm_page_busy_release(m); in vm_page_grab_valid()
5023 } else if ((allocflags & VM_ALLOC_NOCREAT) != 0) { in vm_page_grab_valid()
5026 } else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) { in vm_page_grab_valid()
5034 vm_page_assert_xbusied(m); in vm_page_grab_valid()
5039 ma[0] = m; in vm_page_grab_valid()
5046 ma[i] = vm_page_alloc(object, m->pindex + i, in vm_page_grab_valid()
5059 m = ma[0]; in vm_page_grab_valid()
5061 for (i = 0; i < after; i++) { in vm_page_grab_valid()
5072 MPASS(vm_page_all_valid(m)); in vm_page_grab_valid()
5074 vm_page_zero_invalid(m, TRUE); in vm_page_grab_valid()
5077 if ((allocflags & VM_ALLOC_WIRED) != 0) in vm_page_grab_valid()
5078 vm_page_wire(m); in vm_page_grab_valid()
5079 if ((allocflags & VM_ALLOC_SBUSY) != 0 && vm_page_xbusied(m)) in vm_page_grab_valid()
5080 vm_page_busy_downgrade(m); in vm_page_grab_valid()
5081 else if ((allocflags & VM_ALLOC_NOBUSY) != 0) in vm_page_grab_valid()
5082 vm_page_busy_release(m); in vm_page_grab_valid()
5083 *mp = m; in vm_page_grab_valid()
5095 vm_page_t m; in vm_page_grab_valid_unlocked() local
5099 KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || in vm_page_grab_valid_unlocked()
5100 (allocflags & VM_ALLOC_IGN_SBUSY) != 0, in vm_page_grab_valid_unlocked()
5104 (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | VM_ALLOC_ZERO)) == 0, in vm_page_grab_valid_unlocked()
5105 ("vm_page_grab_valid_unlocked: Invalid flags 0x%X", allocflags)); in vm_page_grab_valid_unlocked()
5113 m = vm_page_acquire_unlocked(object, pindex, NULL, flags); in vm_page_grab_valid_unlocked()
5114 if (m == PAGE_NOT_ACQUIRED) in vm_page_grab_valid_unlocked()
5116 if (m != NULL) { in vm_page_grab_valid_unlocked()
5117 if (vm_page_all_valid(m)) { in vm_page_grab_valid_unlocked()
5118 if ((allocflags & VM_ALLOC_WIRED) != 0) in vm_page_grab_valid_unlocked()
5119 vm_page_wire(m); in vm_page_grab_valid_unlocked()
5120 vm_page_grab_release(m, allocflags); in vm_page_grab_valid_unlocked()
5121 *mp = m; in vm_page_grab_valid_unlocked()
5124 vm_page_busy_release(m); in vm_page_grab_valid_unlocked()
5126 if ((allocflags & VM_ALLOC_NOCREAT) != 0) { in vm_page_grab_valid_unlocked()
5167 vm_page_t m, mpred; in vm_page_grab_pages() local
5172 KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0, in vm_page_grab_pages()
5174 KASSERT(count > 0, in vm_page_grab_pages()
5179 i = 0; in vm_page_grab_pages()
5181 m = vm_page_mpred(object, pindex + i); in vm_page_grab_pages()
5182 if (m == NULL || m->pindex != pindex + i) { in vm_page_grab_pages()
5183 mpred = m; in vm_page_grab_pages()
5184 m = NULL; in vm_page_grab_pages()
5186 mpred = TAILQ_PREV(m, pglist, listq); in vm_page_grab_pages()
5188 if (m != NULL) { in vm_page_grab_pages()
5189 if (!vm_page_tryacquire(m, allocflags)) { in vm_page_grab_pages()
5190 if (vm_page_grab_sleep(object, m, pindex + i, in vm_page_grab_pages()
5196 if ((allocflags & VM_ALLOC_NOCREAT) != 0) in vm_page_grab_pages()
5198 m = vm_page_alloc_after(object, pindex + i, in vm_page_grab_pages()
5200 if (m == NULL) { in vm_page_grab_pages()
5202 VM_ALLOC_WAITFAIL)) != 0) in vm_page_grab_pages()
5207 if (vm_page_none_valid(m) && in vm_page_grab_pages()
5208 (allocflags & VM_ALLOC_ZERO) != 0) { in vm_page_grab_pages()
5209 if ((m->flags & PG_ZERO) == 0) in vm_page_grab_pages()
5210 pmap_zero_page(m); in vm_page_grab_pages()
5211 vm_page_valid(m); in vm_page_grab_pages()
5213 vm_page_grab_release(m, allocflags); in vm_page_grab_pages()
5214 ma[i] = mpred = m; in vm_page_grab_pages()
5215 m = vm_page_next(m); in vm_page_grab_pages()
5228 vm_page_t m; in vm_page_grab_pages_unlocked() local
5232 KASSERT(count > 0, in vm_page_grab_pages_unlocked()
5242 m = NULL; in vm_page_grab_pages_unlocked()
5243 for (i = 0; i < count; i++, pindex++) { in vm_page_grab_pages_unlocked()
5249 if (m == NULL || QMD_IS_TRASHED(m) || m->pindex != pindex || in vm_page_grab_pages_unlocked()
5250 atomic_load_ptr(&m->object) != object) { in vm_page_grab_pages_unlocked()
5255 m = NULL; in vm_page_grab_pages_unlocked()
5257 m = vm_page_acquire_unlocked(object, pindex, m, flags); in vm_page_grab_pages_unlocked()
5258 if (m == PAGE_NOT_ACQUIRED) in vm_page_grab_pages_unlocked()
5260 if (m == NULL) in vm_page_grab_pages_unlocked()
5262 if ((flags & VM_ALLOC_ZERO) != 0 && vm_page_none_valid(m)) { in vm_page_grab_pages_unlocked()
5263 if ((m->flags & PG_ZERO) == 0) in vm_page_grab_pages_unlocked()
5264 pmap_zero_page(m); in vm_page_grab_pages_unlocked()
5265 vm_page_valid(m); in vm_page_grab_pages_unlocked()
5267 /* m will still be wired or busy according to flags. */ in vm_page_grab_pages_unlocked()
5268 vm_page_grab_release(m, allocflags); in vm_page_grab_pages_unlocked()
5269 ma[i] = m; in vm_page_grab_pages_unlocked()
5270 m = TAILQ_NEXT(m, listq); in vm_page_grab_pages_unlocked()
5272 if (i == count || (allocflags & VM_ALLOC_NOCREAT) != 0) in vm_page_grab_pages_unlocked()
5298 if (size == 0) /* handle degenerate case */ in vm_page_bits()
5299 return (0); in vm_page_bits()
5309 vm_page_bits_set(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t set) in vm_page_bits_set() argument
5342 vm_page_bits_clear(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t clear) in vm_page_bits_clear() argument
5375 vm_page_bits_swap(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t newbits) in vm_page_bits_swap() argument
5381 while (atomic_fcmpset_64(bits, &old, newbits) == 0); in vm_page_bits_swap()
5387 while (atomic_fcmpset_32(bits, &old, newbits) == 0); in vm_page_bits_swap()
5393 while (atomic_fcmpset_16(bits, &old, newbits) == 0); in vm_page_bits_swap()
5399 while (atomic_fcmpset_8(bits, &old, newbits) == 0); in vm_page_bits_swap()
5425 } while (atomic_fcmpset_32((uint32_t *)addr, &old, new) == 0); in vm_page_bits_swap()
5441 vm_page_set_valid_range(vm_page_t m, int base, int size) in vm_page_set_valid_range() argument
5446 vm_page_assert_busied(m); in vm_page_set_valid_range()
5447 if (size == 0) /* handle degenerate case */ in vm_page_set_valid_range()
5456 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0) in vm_page_set_valid_range()
5457 pmap_zero_page_area(m, frag, base - frag); in vm_page_set_valid_range()
5466 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0) in vm_page_set_valid_range()
5467 pmap_zero_page_area(m, endoff, in vm_page_set_valid_range()
5474 KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0, in vm_page_set_valid_range()
5475 ("vm_page_set_valid_range: page %p is dirty", m)); in vm_page_set_valid_range()
5481 if (vm_page_xbusied(m)) in vm_page_set_valid_range()
5482 m->valid |= pagebits; in vm_page_set_valid_range()
5484 vm_page_bits_set(m, &m->valid, pagebits); in vm_page_set_valid_range()
5492 vm_page_set_dirty(vm_page_t m) in vm_page_set_dirty() argument
5496 VM_PAGE_OBJECT_BUSY_ASSERT(m); in vm_page_set_dirty()
5498 if (vm_page_xbusied(m) && !pmap_page_is_write_mapped(m)) { in vm_page_set_dirty()
5499 old = m->dirty; in vm_page_set_dirty()
5500 m->dirty = VM_PAGE_BITS_ALL; in vm_page_set_dirty()
5502 old = vm_page_bits_swap(m, &m->dirty, VM_PAGE_BITS_ALL); in vm_page_set_dirty()
5503 if (old == 0 && (m->a.flags & PGA_SWAP_SPACE) != 0) in vm_page_set_dirty()
5504 vm_pager_page_unswapped(m); in vm_page_set_dirty()
5513 vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits) in vm_page_clear_dirty_mask() argument
5516 vm_page_assert_busied(m); in vm_page_clear_dirty_mask()
5525 if (vm_page_xbusied(m) && !pmap_page_is_write_mapped(m)) in vm_page_clear_dirty_mask()
5526 m->dirty &= ~pagebits; in vm_page_clear_dirty_mask()
5528 vm_page_bits_clear(m, &m->dirty, pagebits); in vm_page_clear_dirty_mask()
5542 vm_page_set_validclean(vm_page_t m, int base, int size) in vm_page_set_validclean() argument
5547 vm_page_assert_busied(m); in vm_page_set_validclean()
5548 if (size == 0) /* handle degenerate case */ in vm_page_set_validclean()
5557 (m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0) in vm_page_set_validclean()
5558 pmap_zero_page_area(m, frag, base - frag); in vm_page_set_validclean()
5567 (m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0) in vm_page_set_validclean()
5568 pmap_zero_page_area(m, endoff, in vm_page_set_validclean()
5582 oldvalid = m->valid; in vm_page_set_validclean()
5584 if (vm_page_xbusied(m)) in vm_page_set_validclean()
5585 m->valid |= pagebits; in vm_page_set_validclean()
5587 vm_page_bits_set(m, &m->valid, pagebits); in vm_page_set_validclean()
5588 #if 0 /* NOT YET */ in vm_page_set_validclean()
5589 if ((frag = base & (DEV_BSIZE - 1)) != 0) { in vm_page_set_validclean()
5593 if (size < 0) in vm_page_set_validclean()
5594 size = 0; in vm_page_set_validclean()
5598 if (base == 0 && size == PAGE_SIZE) { in vm_page_set_validclean()
5613 pmap_clear_modify(m); in vm_page_set_validclean()
5614 m->dirty = 0; in vm_page_set_validclean()
5615 vm_page_aflag_clear(m, PGA_NOSYNC); in vm_page_set_validclean()
5616 } else if (oldvalid != VM_PAGE_BITS_ALL && vm_page_xbusied(m)) in vm_page_set_validclean()
5617 m->dirty &= ~pagebits; in vm_page_set_validclean()
5619 vm_page_clear_dirty_mask(m, pagebits); in vm_page_set_validclean()
5623 vm_page_clear_dirty(vm_page_t m, int base, int size) in vm_page_clear_dirty() argument
5626 vm_page_clear_dirty_mask(m, vm_page_bits(base, size)); in vm_page_clear_dirty()
5636 vm_page_set_invalid(vm_page_t m, int base, int size) in vm_page_set_invalid() argument
5645 object = m->object; in vm_page_set_invalid()
5647 vm_page_assert_busied(m); in vm_page_set_invalid()
5649 if (object->type == OBJT_VNODE && base == 0 && IDX_TO_OFF(m->pindex) + in vm_page_set_invalid()
5654 if (object->ref_count != 0 && vm_page_all_valid(m) && bits != 0) in vm_page_set_invalid()
5655 pmap_remove_all(m); in vm_page_set_invalid()
5656 KASSERT((bits == 0 && vm_page_all_valid(m)) || in vm_page_set_invalid()
5657 !pmap_page_is_mapped(m), in vm_page_set_invalid()
5658 ("vm_page_set_invalid: page %p is mapped", m)); in vm_page_set_invalid()
5659 if (vm_page_xbusied(m)) { in vm_page_set_invalid()
5660 m->valid &= ~bits; in vm_page_set_invalid()
5661 m->dirty &= ~bits; in vm_page_set_invalid()
5663 vm_page_bits_clear(m, &m->valid, bits); in vm_page_set_invalid()
5664 vm_page_bits_clear(m, &m->dirty, bits); in vm_page_set_invalid()
5677 vm_page_invalid(vm_page_t m) in vm_page_invalid() argument
5680 vm_page_assert_busied(m); in vm_page_invalid()
5681 VM_OBJECT_ASSERT_WLOCKED(m->object); in vm_page_invalid()
5682 MPASS(!pmap_page_is_mapped(m)); in vm_page_invalid()
5684 if (vm_page_xbusied(m)) in vm_page_invalid()
5685 m->valid = 0; in vm_page_invalid()
5687 vm_page_bits_clear(m, &m->valid, VM_PAGE_BITS_ALL); in vm_page_invalid()
5702 vm_page_zero_invalid(vm_page_t m, boolean_t setvalid) in vm_page_zero_invalid() argument
5713 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) { in vm_page_zero_invalid()
5715 (m->valid & ((vm_page_bits_t)1 << i))) { in vm_page_zero_invalid()
5717 pmap_zero_page_area(m, in vm_page_zero_invalid()
5730 vm_page_valid(m); in vm_page_zero_invalid()
5736 * Is (partial) page valid? Note that the case where size == 0
5745 vm_page_is_valid(vm_page_t m, int base, int size) in vm_page_is_valid() argument
5750 return (vm_page_any_valid(m) && (m->valid & bits) == bits); in vm_page_is_valid()
5758 vm_page_ps_test(vm_page_t m, int psind, int flags, vm_page_t skip_m) in vm_page_ps_test() argument
5763 object = m->object; in vm_page_ps_test()
5767 KASSERT(psind <= m->psind, in vm_page_ps_test()
5768 ("psind %d > psind %d of m %p", psind, m->psind, m)); in vm_page_ps_test()
5776 for (i = 0; i < npages; i++) { in vm_page_ps_test()
5778 if (m[i].object != object) in vm_page_ps_test()
5780 if (&m[i] == skip_m) in vm_page_ps_test()
5782 if ((flags & PS_NONE_BUSY) != 0 && vm_page_busied(&m[i])) in vm_page_ps_test()
5784 if ((flags & PS_ALL_DIRTY) != 0) { in vm_page_ps_test()
5789 * on the object containing "m[i]". in vm_page_ps_test()
5791 if (m[i].dirty != VM_PAGE_BITS_ALL) in vm_page_ps_test()
5794 if ((flags & PS_ALL_VALID) != 0 && in vm_page_ps_test()
5795 m[i].valid != VM_PAGE_BITS_ALL) in vm_page_ps_test()
5805 vm_page_test_dirty(vm_page_t m) in vm_page_test_dirty() argument
5808 vm_page_assert_busied(m); in vm_page_test_dirty()
5809 if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m)) in vm_page_test_dirty()
5810 vm_page_dirty(m); in vm_page_test_dirty()
5814 vm_page_valid(vm_page_t m) in vm_page_valid() argument
5817 vm_page_assert_busied(m); in vm_page_valid()
5818 if (vm_page_xbusied(m)) in vm_page_valid()
5819 m->valid = VM_PAGE_BITS_ALL; in vm_page_valid()
5821 vm_page_bits_set(m, &m->valid, VM_PAGE_BITS_ALL); in vm_page_valid()
5825 vm_page_lock_KBI(vm_page_t m, const char *file, int line) in vm_page_lock_KBI() argument
5828 mtx_lock_flags_(vm_page_lockptr(m), 0, file, line); in vm_page_lock_KBI()
5832 vm_page_unlock_KBI(vm_page_t m, const char *file, int line) in vm_page_unlock_KBI() argument
5835 mtx_unlock_flags_(vm_page_lockptr(m), 0, file, line); in vm_page_unlock_KBI()
5839 vm_page_trylock_KBI(vm_page_t m, const char *file, int line) in vm_page_trylock_KBI() argument
5842 return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line)); in vm_page_trylock_KBI()
5847 vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line) in vm_page_assert_locked_KBI() argument
5850 vm_page_lock_assert_KBI(m, MA_OWNED, file, line); in vm_page_assert_locked_KBI()
5854 vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line) in vm_page_lock_assert_KBI() argument
5857 mtx_assert_(vm_page_lockptr(m), a, file, line); in vm_page_lock_assert_KBI()
5863 vm_page_object_busy_assert(vm_page_t m) in vm_page_object_busy_assert() argument
5870 if (m->object != NULL && !vm_page_busied(m)) in vm_page_object_busy_assert()
5871 VM_OBJECT_ASSERT_BUSY(m->object); in vm_page_object_busy_assert()
5875 vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits) in vm_page_assert_pga_writeable() argument
5878 if ((bits & PGA_WRITEABLE) == 0) in vm_page_assert_pga_writeable()
5886 KASSERT((m->oflags & VPO_UNMANAGED) == 0, in vm_page_assert_pga_writeable()
5888 if (!vm_page_xbusied(m)) in vm_page_assert_pga_writeable()
5889 VM_OBJECT_ASSERT_BUSY(m->object); in vm_page_assert_pga_writeable()
5918 for (dom = 0; dom < vm_ndomains; dom++) { in DB_SHOW_COMMAND_FLAGS()
5933 vm_page_t m; in DB_SHOW_COMMAND() local
5944 m = PHYS_TO_VM_PAGE(pmap_kextract(addr)); in DB_SHOW_COMMAND()
5946 m = PHYS_TO_VM_PAGE(addr); in DB_SHOW_COMMAND()
5948 m = (vm_page_t)addr; in DB_SHOW_COMMAND()
5950 "page %p obj %p pidx 0x%jx phys 0x%jx q %d ref 0x%x\n" in DB_SHOW_COMMAND()
5951 " af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n", in DB_SHOW_COMMAND()
5952 m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr, in DB_SHOW_COMMAND()
5953 m->a.queue, m->ref_count, m->a.flags, m->oflags, in DB_SHOW_COMMAND()
5954 m->flags, m->a.act_count, m->busy_lock, m->valid, m->dirty); in DB_SHOW_COMMAND()