89 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m);
90 static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
152 	vm_object_t object;  in vnode_create_vobject_any()  local
156 	object = vp->v_object;  in vnode_create_vobject_any()
157 	if (object != NULL)  in vnode_create_vobject_any()
167 	object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred);  in vnode_create_vobject_any()
174 	VM_OBJECT_RLOCK(object);  in vnode_create_vobject_any()
175 	last = refcount_release(&object->ref_count);  in vnode_create_vobject_any()
176 	VM_OBJECT_RUNLOCK(object);  in vnode_create_vobject_any()
256 	vm_object_t object;  in vnode_pager_alloc()  local
269 	object = vp->v_object;  in vnode_pager_alloc()
271 	if (object == NULL) {  in vnode_pager_alloc()
275 		object = vm_object_allocate(OBJT_VNODE,  in vnode_pager_alloc()
278 		object->un_pager.vnp.vnp_size = size;  in vnode_pager_alloc()
279 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_alloc()
280 		object->domain.dr_policy = vnode_domainset;  in vnode_pager_alloc()
281 		object->handle = handle;  in vnode_pager_alloc()
283 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
284 			vm_object_set_flag(object, OBJ_SIZEVNLOCK);  in vnode_pager_alloc()
285 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
293 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
294 			KASSERT(object->ref_count == 1,  in vnode_pager_alloc()
295 			    ("leaked ref %p %d", object, object->ref_count));  in vnode_pager_alloc()
296 			object->type = OBJT_DEAD;  in vnode_pager_alloc()
297 			refcount_init(&object->ref_count, 0);  in vnode_pager_alloc()
298 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
299 			vm_object_destroy(object);  in vnode_pager_alloc()
302 		vp->v_object = object;  in vnode_pager_alloc()
306 		vm_object_reference(object);  in vnode_pager_alloc()
308 		if ((object->flags & OBJ_COLORED) == 0) {  in vnode_pager_alloc()
309 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
310 			vm_object_color(object, 0);  in vnode_pager_alloc()
311 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
315 	return (object);  in vnode_pager_alloc()
322 vnode_pager_dealloc(vm_object_t object)  in vnode_pager_dealloc()  argument
327 	vp = object->handle;  in vnode_pager_dealloc()
331 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_dealloc()
332 	vm_object_pip_wait(object, "vnpdea");  in vnode_pager_dealloc()
333 	refs = object->ref_count;  in vnode_pager_dealloc()
335 	object->handle = NULL;  in vnode_pager_dealloc()
336 	object->type = OBJT_DEAD;  in vnode_pager_dealloc()
338 	if (object->un_pager.vnp.writemappings > 0) {  in vnode_pager_dealloc()
339 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_dealloc()
357 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_dealloc()
360 	VM_OBJECT_WLOCK(object);  in vnode_pager_dealloc()
364 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in vnode_pager_haspage()  argument
367 	struct vnode *vp = object->handle;  in vnode_pager_haspage()
376 	VM_OBJECT_ASSERT_LOCKED(object);  in vnode_pager_haspage()
387 	if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)  in vnode_pager_haspage()
399 	lockstate = VM_OBJECT_DROP(object);  in vnode_pager_haspage()
401 	VM_OBJECT_PICKUP(object, lockstate);  in vnode_pager_haspage()
420 			    roundup2(object->size, pagesperblock),  in vnode_pager_haspage()
423 			    (uintmax_t )object->size));  in vnode_pager_haspage()
426 			if (pindex + *after >= object->size)  in vnode_pager_haspage()
427 				*after = object->size - 1 - pindex;  in vnode_pager_haspage()
499 	vm_object_t object;  in vnode_pager_setsize()  local
503 	if ((object = vp->v_object) == NULL)  in vnode_pager_setsize()
515 	VM_OBJECT_WLOCK(object);  in vnode_pager_setsize()
516 	if (object->type == OBJT_DEAD) {  in vnode_pager_setsize()
517 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
520 	KASSERT(object->type == OBJT_VNODE,  in vnode_pager_setsize()
521 	    ("not vnode-backed object %p", object));  in vnode_pager_setsize()
522 	if (nsize == object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
526 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
530 	if (nsize < object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
534 		if (nobjsize < object->size)  in vnode_pager_setsize()
535 			vm_object_page_remove(object, nobjsize, object->size,  in vnode_pager_setsize()
547 		m = vm_page_grab(object, OFF_TO_IDX(nsize), VM_ALLOC_NOCREAT);  in vnode_pager_setsize()
557 	object->un_pager.vnp.vnp_size = nsize;  in vnode_pager_setsize()
559 	atomic_store_64(&object->un_pager.vnp.vnp_size, nsize);  in vnode_pager_setsize()
561 	object->size = nobjsize;  in vnode_pager_setsize()
562 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
575 	struct vm_object *object;  in vnode_pager_purge_range()  local
582 	object = vp->v_object;  in vnode_pager_purge_range()
588 	if ((end != 0 && end <= start) || object == NULL)  in vnode_pager_purge_range()
591 	VM_OBJECT_WLOCK(object);  in vnode_pager_purge_range()
594 		vm_object_page_remove(object, pi, piend, 0);  in vnode_pager_purge_range()
599 		m = vm_page_grab(object, pistart, VM_ALLOC_NOCREAT);  in vnode_pager_purge_range()
611 		m = vm_page_grab(object, piend, VM_ALLOC_NOCREAT);  in vnode_pager_purge_range()
619 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_purge_range()
667 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)  in vnode_pager_input_smlfs()  argument
679 	vp = object->handle;  in vnode_pager_input_smlfs()
697 		if (address >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_smlfs()
759 vnode_pager_input_old(vm_object_t object, vm_page_t m)  in vnode_pager_input_old()  argument
768 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_input_old()
774 	if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_old()
778 		if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size)  in vnode_pager_input_old()
779 			size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex);  in vnode_pager_input_old()
780 		vp = object->handle;  in vnode_pager_input_old()
781 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_input_old()
811 		VM_OBJECT_WLOCK(object);  in vnode_pager_input_old()
832 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,  in vnode_pager_getpages()  argument
839 	vp = object->handle;  in vnode_pager_getpages()
847 vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_getpages_async()  argument
853 	vp = object->handle;  in vnode_pager_getpages_async()
893 	vm_object_t object;  in vnode_pager_generic_getpages()  local
909 	object = vp->v_object;  in vnode_pager_generic_getpages()
914 	KASSERT(foff < object->un_pager.vnp.vnp_size,  in vnode_pager_generic_getpages()
941 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
945 			error = vnode_pager_input_old(object, m[i]);  in vnode_pager_generic_getpages()
949 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
965 			error = vnode_pager_input_smlfs(object, m[i]);  in vnode_pager_generic_getpages()
999 	if (m[0]->pindex + after >= object->size)  in vnode_pager_generic_getpages()
1000 		after = object->size - 1 - m[0]->pindex;  in vnode_pager_generic_getpages()
1041 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
1042 		vm_object_prepare_buf_pages(object, bp->b_pages, count,  in vnode_pager_generic_getpages()
1044 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
1080 	if ((foff + bytecount) > object->un_pager.vnp.vnp_size)  in vnode_pager_generic_getpages()
1081 		bytecount = object->un_pager.vnp.vnp_size - foff;  in vnode_pager_generic_getpages()
1164 	vm_object_t object;  in vnode_pager_generic_getpages_done()  local
1171 	object = bp->b_vp->v_object;  in vnode_pager_generic_getpages_done()
1195 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages_done()
1200 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages_done()
1205 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_getpages_done()
1215 		if (nextoff <= object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_getpages_done()
1233 			    object->un_pager.vnp.vnp_size - tfoff);  in vnode_pager_generic_getpages_done()
1235 			    object->un_pager.vnp.vnp_size - tfoff)) == 0,  in vnode_pager_generic_getpages_done()
1242 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_getpages_done()
1256 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_putpages()  argument
1281 	vp = object->handle;  in vnode_pager_putpages()
1282 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_putpages()
1286 	VM_OBJECT_WLOCK(object);  in vnode_pager_putpages()
1320 	vm_object_t object;  in vnode_pager_generic_putpages()  local
1331 	object = vp->v_object;  in vnode_pager_generic_putpages()
1362 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_putpages()
1363 	if (maxsize + poffset > object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_putpages()
1364 		if (object->un_pager.vnp.vnp_size > poffset) {  in vnode_pager_generic_putpages()
1365 			maxsize = object->un_pager.vnp.vnp_size - poffset;  in vnode_pager_generic_putpages()
1391 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_putpages()
1573 vnode_pager_update_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_update_writecount()  argument
1579 	VM_OBJECT_WLOCK(object);  in vnode_pager_update_writecount()
1580 	if (object->type != OBJT_VNODE) {  in vnode_pager_update_writecount()
1581 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1584 	old_wm = object->un_pager.vnp.writemappings;  in vnode_pager_update_writecount()
1585 	object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start;  in vnode_pager_update_writecount()
1586 	vp = object->handle;  in vnode_pager_update_writecount()
1587 	if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) {  in vnode_pager_update_writecount()
1592 	} else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) {  in vnode_pager_update_writecount()
1598 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1602 vnode_pager_release_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_release_writecount()  argument
1609 	VM_OBJECT_WLOCK(object);  in vnode_pager_release_writecount()
1615 	if (object->type != OBJT_VNODE) {  in vnode_pager_release_writecount()
1616 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1625 	if (object->un_pager.vnp.writemappings != inc) {  in vnode_pager_release_writecount()
1626 		object->un_pager.vnp.writemappings -= inc;  in vnode_pager_release_writecount()
1627 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1631 	vp = object->handle;  in vnode_pager_release_writecount()
1633 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1644 	vnode_pager_update_writecount(object, end, start);  in vnode_pager_release_writecount()
1652 vnode_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp)  in vnode_pager_getvp()  argument
1654 	*vpp = object->handle;  in vnode_pager_getvp()