1df8bae1dSRodney W. Grimes /* 2df8bae1dSRodney W. Grimes * Copyright (c) 1990 University of Utah. 326f9a767SRodney W. Grimes * Copyright (c) 1991 The Regents of the University of California. 426f9a767SRodney W. Grimes * All rights reserved. 526f9a767SRodney W. Grimes * Copyright (c) 1993, 1994 John S. Dyson 624a1cce3SDavid Greenman * Copyright (c) 1995, David Greenman 7df8bae1dSRodney W. Grimes * 8df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 9df8bae1dSRodney W. Grimes * the Systems Programming Group of the University of Utah Computer 10df8bae1dSRodney W. Grimes * Science Department. 11df8bae1dSRodney W. Grimes * 12df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 13df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 14df8bae1dSRodney W. Grimes * are met: 15df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 16df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 17df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 18df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 19df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 20df8bae1dSRodney W. Grimes * 3. All advertising materials mentioning features or use of this software 215929bcfaSPhilippe Charnier * must display the following acknowledgement: 22df8bae1dSRodney W. Grimes * This product includes software developed by the University of 23df8bae1dSRodney W. Grimes * California, Berkeley and its contributors. 24df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 25df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 26df8bae1dSRodney W. Grimes * without specific prior written permission. 27df8bae1dSRodney W. Grimes * 28df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38df8bae1dSRodney W. Grimes * SUCH DAMAGE. 39df8bae1dSRodney W. Grimes * 4026f9a767SRodney W. Grimes * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41df8bae1dSRodney W. Grimes */ 42df8bae1dSRodney W. Grimes 43df8bae1dSRodney W. Grimes /* 44df8bae1dSRodney W. Grimes * Page to/from files (vnodes). 45df8bae1dSRodney W. Grimes */ 46df8bae1dSRodney W. Grimes 4726f9a767SRodney W. Grimes /* 4826f9a767SRodney W. Grimes * TODO: 4924a1cce3SDavid Greenman * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 50f6b04d2bSDavid Greenman * greatly re-simplify the vnode_pager. 5126f9a767SRodney W. Grimes */ 5226f9a767SRodney W. Grimes 53874651b1SDavid E. O'Brien #include <sys/cdefs.h> 54874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 55874651b1SDavid E. O'Brien 56df8bae1dSRodney W. Grimes #include <sys/param.h> 57df8bae1dSRodney W. Grimes #include <sys/systm.h> 58df8bae1dSRodney W. Grimes #include <sys/proc.h> 59df8bae1dSRodney W. Grimes #include <sys/vnode.h> 60df8bae1dSRodney W. Grimes #include <sys/mount.h> 619626b608SPoul-Henning Kamp #include <sys/bio.h> 6224a1cce3SDavid Greenman #include <sys/buf.h> 63efeaf95aSDavid Greenman #include <sys/vmmeter.h> 6424579ca1SMatthew Dillon #include <sys/conf.h> 659e0ddbd0SAlan Cox #include <sys/sf_buf.h> 66df8bae1dSRodney W. Grimes 67df8bae1dSRodney W. Grimes #include <vm/vm.h> 68efeaf95aSDavid Greenman #include <vm/vm_object.h> 69df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 7024a1cce3SDavid Greenman #include <vm/vm_pager.h> 711efb74fbSJohn Dyson #include <vm/vm_map.h> 72df8bae1dSRodney W. Grimes #include <vm/vnode_pager.h> 73efeaf95aSDavid Greenman #include <vm/vm_extern.h> 74df8bae1dSRodney W. Grimes 7511caded3SAlfred Perlstein static void vnode_pager_init(void); 7611caded3SAlfred Perlstein static vm_offset_t vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, 7711caded3SAlfred Perlstein int *run); 7811caded3SAlfred Perlstein static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); 7911caded3SAlfred Perlstein static int vnode_pager_input_old(vm_object_t object, vm_page_t m); 8011caded3SAlfred Perlstein static void vnode_pager_dealloc(vm_object_t); 8111caded3SAlfred Perlstein static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); 8211caded3SAlfred Perlstein static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); 8311caded3SAlfred Perlstein static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); 840b8253a7SBruce Evans 85df8bae1dSRodney W. Grimes struct pagerops vnodepagerops = { 864e658600SPoul-Henning Kamp .pgo_init = vnode_pager_init, 874e658600SPoul-Henning Kamp .pgo_alloc = vnode_pager_alloc, 884e658600SPoul-Henning Kamp .pgo_dealloc = vnode_pager_dealloc, 894e658600SPoul-Henning Kamp .pgo_getpages = vnode_pager_getpages, 904e658600SPoul-Henning Kamp .pgo_putpages = vnode_pager_putpages, 914e658600SPoul-Henning Kamp .pgo_haspage = vnode_pager_haspage, 92df8bae1dSRodney W. Grimes }; 93df8bae1dSRodney W. Grimes 94b62b9b64SJohn Baldwin int vnode_pbuf_freecnt; 951c7c3c6aSMatthew Dillon 9637c84183SPoul-Henning Kamp static void 97b62b9b64SJohn Baldwin vnode_pager_init(void) 98b62b9b64SJohn Baldwin { 99b62b9b64SJohn Baldwin 100b62b9b64SJohn Baldwin vnode_pbuf_freecnt = nswbuf / 2 + 1; 101b62b9b64SJohn Baldwin } 102170db9c6SJohn Dyson 103df8bae1dSRodney W. Grimes /* 104df8bae1dSRodney W. Grimes * Allocate (or lookup) pager for a vnode. 105df8bae1dSRodney W. Grimes * Handle is a vnode pointer. 106990ab7adSAlan Cox * 107990ab7adSAlan Cox * MPSAFE 108df8bae1dSRodney W. Grimes */ 10924a1cce3SDavid Greenman vm_object_t 1106cde7a16SDavid Greenman vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 111b9dcd593SBruce Evans vm_ooffset_t offset) 112df8bae1dSRodney W. Grimes { 11306cb7259SDavid Greenman vm_object_t object; 114df8bae1dSRodney W. Grimes struct vnode *vp; 115df8bae1dSRodney W. Grimes 116df8bae1dSRodney W. Grimes /* 117df8bae1dSRodney W. Grimes * Pageout to vnode, no can do yet. 118df8bae1dSRodney W. Grimes */ 119df8bae1dSRodney W. Grimes if (handle == NULL) 120df8bae1dSRodney W. Grimes return (NULL); 121df8bae1dSRodney W. Grimes 122df8bae1dSRodney W. Grimes vp = (struct vnode *) handle; 12339d38f93SDavid Greenman 12463e7e60dSJeff Roberson ASSERT_VOP_LOCKED(vp, "vnode_pager_alloc"); 12563e7e60dSJeff Roberson 12639d38f93SDavid Greenman /* 12739d38f93SDavid Greenman * Prevent race condition when allocating the object. This 12839d38f93SDavid Greenman * can happen with NFS vnodes since the nfsnode isn't locked. 12939d38f93SDavid Greenman */ 130e6e370a7SJeff Roberson VI_LOCK(vp); 131e6e370a7SJeff Roberson while (vp->v_iflag & VI_OLOCK) { 132e6e370a7SJeff Roberson vp->v_iflag |= VI_OWANT; 133e6e370a7SJeff Roberson msleep(vp, VI_MTX(vp), PVM, "vnpobj", 0); 13439d38f93SDavid Greenman } 135e6e370a7SJeff Roberson vp->v_iflag |= VI_OLOCK; 136e6e370a7SJeff Roberson VI_UNLOCK(vp); 13739d38f93SDavid Greenman 13839d38f93SDavid Greenman /* 13939d38f93SDavid Greenman * If the object is being terminated, wait for it to 14039d38f93SDavid Greenman * go away. 14139d38f93SDavid Greenman */ 1421ca58953SAlan Cox while ((object = vp->v_object) != NULL) { 1431ca58953SAlan Cox VM_OBJECT_LOCK(object); 1441ca58953SAlan Cox if ((object->flags & OBJ_DEAD) == 0) 1451ca58953SAlan Cox break; 14619187819SAlan Cox vm_object_set_flag(object, OBJ_DISCONNECTWNT); 1471ca58953SAlan Cox msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vadead", 0); 14824a1cce3SDavid Greenman } 1490d94caffSDavid Greenman 1502be70f79SJohn Dyson if (vp->v_usecount == 0) 1512be70f79SJohn Dyson panic("vnode_pager_alloc: no vnode reference"); 1522be70f79SJohn Dyson 15324a1cce3SDavid Greenman if (object == NULL) { 154df8bae1dSRodney W. Grimes /* 155df8bae1dSRodney W. Grimes * And an object of the appropriate size 156df8bae1dSRodney W. Grimes */ 1576cde7a16SDavid Greenman object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 158bbc0ec52SDavid Greenman 1596cde7a16SDavid Greenman object->un_pager.vnp.vnp_size = size; 16026f9a767SRodney W. Grimes 16124a1cce3SDavid Greenman object->handle = handle; 16224a1cce3SDavid Greenman vp->v_object = object; 163df8bae1dSRodney W. Grimes } else { 16495e5e988SJohn Dyson object->ref_count++; 1651ca58953SAlan Cox VM_OBJECT_UNLOCK(object); 166df8bae1dSRodney W. Grimes } 167e6e370a7SJeff Roberson VI_LOCK(vp); 168990ab7adSAlan Cox vp->v_usecount++; 169e6e370a7SJeff Roberson vp->v_iflag &= ~VI_OLOCK; 170e6e370a7SJeff Roberson if (vp->v_iflag & VI_OWANT) { 171e6e370a7SJeff Roberson vp->v_iflag &= ~VI_OWANT; 17239d38f93SDavid Greenman wakeup(vp); 17339d38f93SDavid Greenman } 174e6e370a7SJeff Roberson VI_UNLOCK(vp); 17524a1cce3SDavid Greenman return (object); 176df8bae1dSRodney W. Grimes } 177df8bae1dSRodney W. Grimes 178658ad5ffSAlan Cox /* 179658ad5ffSAlan Cox * The object must be locked. 180658ad5ffSAlan Cox */ 181f708ef1bSPoul-Henning Kamp static void 18224a1cce3SDavid Greenman vnode_pager_dealloc(object) 1830d94caffSDavid Greenman vm_object_t object; 18424a1cce3SDavid Greenman { 18554d92145SMatthew Dillon struct vnode *vp = object->handle; 186df8bae1dSRodney W. Grimes 18724a1cce3SDavid Greenman if (vp == NULL) 18824a1cce3SDavid Greenman panic("vnode_pager_dealloc: pager already dealloced"); 18924a1cce3SDavid Greenman 190658ad5ffSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 19166095752SJohn Dyson vm_object_pip_wait(object, "vnpdea"); 19224a1cce3SDavid Greenman 19324a1cce3SDavid Greenman object->handle = NULL; 19495461b45SJohn Dyson object->type = OBJT_DEAD; 19519187819SAlan Cox if (object->flags & OBJ_DISCONNECTWNT) { 19619187819SAlan Cox vm_object_clear_flag(object, OBJ_DISCONNECTWNT); 19719187819SAlan Cox wakeup(object); 19819187819SAlan Cox } 199e6e370a7SJeff Roberson ASSERT_VOP_LOCKED(vp, "vnode_pager_dealloc"); 200aa2cabb9SDavid Greenman vp->v_object = NULL; 201e6e370a7SJeff Roberson vp->v_vflag &= ~(VV_TEXT | VV_OBJBUF); 202df8bae1dSRodney W. Grimes } 20326f9a767SRodney W. Grimes 204f708ef1bSPoul-Henning Kamp static boolean_t 205a316d390SJohn Dyson vnode_pager_haspage(object, pindex, before, after) 20624a1cce3SDavid Greenman vm_object_t object; 207a316d390SJohn Dyson vm_pindex_t pindex; 20824a1cce3SDavid Greenman int *before; 20924a1cce3SDavid Greenman int *after; 210df8bae1dSRodney W. Grimes { 21124a1cce3SDavid Greenman struct vnode *vp = object->handle; 21298b0c789SPoul-Henning Kamp daddr_t bn; 2133af76890SPoul-Henning Kamp int err; 214170db9c6SJohn Dyson daddr_t reqblock; 2152c4488fcSJohn Dyson int poff; 2162c4488fcSJohn Dyson int bsize; 217d63596ceSJohn Dyson int pagesperblock, blocksperpage; 218df8bae1dSRodney W. Grimes 219f29ba63eSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 22024579ca1SMatthew Dillon /* 22124579ca1SMatthew Dillon * If no vp or vp is doomed or marked transparent to VM, we do not 22224579ca1SMatthew Dillon * have the page. 22324579ca1SMatthew Dillon */ 224e6e370a7SJeff Roberson if (vp == NULL) 22547221757SJohn Dyson return FALSE; 22647221757SJohn Dyson 22763e7e60dSJeff Roberson VI_LOCK(vp); 22863e7e60dSJeff Roberson if (vp->v_iflag & VI_DOOMED) { 22963e7e60dSJeff Roberson VI_UNLOCK(vp); 230e6e370a7SJeff Roberson return FALSE; 23163e7e60dSJeff Roberson } 23263e7e60dSJeff Roberson VI_UNLOCK(vp); 233df8bae1dSRodney W. Grimes /* 2340d94caffSDavid Greenman * If filesystem no longer mounted or offset beyond end of file we do 2350d94caffSDavid Greenman * not have the page. 236df8bae1dSRodney W. Grimes */ 237a316d390SJohn Dyson if ((vp->v_mount == NULL) || 238a316d390SJohn Dyson (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)) 2394abc71c0SDavid Greenman return FALSE; 240df8bae1dSRodney W. Grimes 241eed2d59bSDavid Greenman bsize = vp->v_mount->mnt_stat.f_iosize; 242170db9c6SJohn Dyson pagesperblock = bsize / PAGE_SIZE; 243d63596ceSJohn Dyson blocksperpage = 0; 244d63596ceSJohn Dyson if (pagesperblock > 0) { 245a316d390SJohn Dyson reqblock = pindex / pagesperblock; 246d63596ceSJohn Dyson } else { 247d63596ceSJohn Dyson blocksperpage = (PAGE_SIZE / bsize); 248d63596ceSJohn Dyson reqblock = pindex * blocksperpage; 249d63596ceSJohn Dyson } 250f29ba63eSAlan Cox VM_OBJECT_UNLOCK(object); 2510cb507cbSAlan Cox mtx_lock(&Giant); 252ec794849SPoul-Henning Kamp err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before); 2530cb507cbSAlan Cox mtx_unlock(&Giant); 254f29ba63eSAlan Cox VM_OBJECT_LOCK(object); 2550d94caffSDavid Greenman if (err) 25624a1cce3SDavid Greenman return TRUE; 2576eab77f2SJohn Dyson if (bn == -1) 258ced399eeSJohn Dyson return FALSE; 259d63596ceSJohn Dyson if (pagesperblock > 0) { 260a316d390SJohn Dyson poff = pindex - (reqblock * pagesperblock); 261170db9c6SJohn Dyson if (before) { 262170db9c6SJohn Dyson *before *= pagesperblock; 263170db9c6SJohn Dyson *before += poff; 264170db9c6SJohn Dyson } 265170db9c6SJohn Dyson if (after) { 266b1fc01b7SJohn Dyson int numafter; 267170db9c6SJohn Dyson *after *= pagesperblock; 268b1fc01b7SJohn Dyson numafter = pagesperblock - (poff + 1); 26947e151ddSRobert Drehmel if (IDX_TO_OFF(pindex + numafter) > 27047e151ddSRobert Drehmel object->un_pager.vnp.vnp_size) { 27147e151ddSRobert Drehmel numafter = 27247e151ddSRobert Drehmel OFF_TO_IDX(object->un_pager.vnp.vnp_size) - 27347e151ddSRobert Drehmel pindex; 274b1fc01b7SJohn Dyson } 275b1fc01b7SJohn Dyson *after += numafter; 276170db9c6SJohn Dyson } 277d63596ceSJohn Dyson } else { 278d63596ceSJohn Dyson if (before) { 279d63596ceSJohn Dyson *before /= blocksperpage; 280d63596ceSJohn Dyson } 281d63596ceSJohn Dyson 282d63596ceSJohn Dyson if (after) { 283d63596ceSJohn Dyson *after /= blocksperpage; 284d63596ceSJohn Dyson } 285d63596ceSJohn Dyson } 286ced399eeSJohn Dyson return TRUE; 287df8bae1dSRodney W. Grimes } 288df8bae1dSRodney W. Grimes 289df8bae1dSRodney W. Grimes /* 290df8bae1dSRodney W. Grimes * Lets the VM system know about a change in size for a file. 29124a1cce3SDavid Greenman * We adjust our own internal size and flush any cached pages in 292df8bae1dSRodney W. Grimes * the associated object that are affected by the size change. 293df8bae1dSRodney W. Grimes * 294df8bae1dSRodney W. Grimes * Note: this routine may be invoked as a result of a pager put 295df8bae1dSRodney W. Grimes * operation (possibly at object termination time), so we must be careful. 296df8bae1dSRodney W. Grimes */ 297df8bae1dSRodney W. Grimes void 298df8bae1dSRodney W. Grimes vnode_pager_setsize(vp, nsize) 299df8bae1dSRodney W. Grimes struct vnode *vp; 300a316d390SJohn Dyson vm_ooffset_t nsize; 301df8bae1dSRodney W. Grimes { 3022a8f9ab5SAlan Cox vm_object_t object; 3032a8f9ab5SAlan Cox vm_page_t m; 304c576d121SLuoqi Chen vm_pindex_t nobjsize; 305df8bae1dSRodney W. Grimes 3062a8f9ab5SAlan Cox if ((object = vp->v_object) == NULL) 307df8bae1dSRodney W. Grimes return; 3082a8f9ab5SAlan Cox VM_OBJECT_LOCK(object); 3092a8f9ab5SAlan Cox if (nsize == object->un_pager.vnp.vnp_size) { 310df8bae1dSRodney W. Grimes /* 311df8bae1dSRodney W. Grimes * Hasn't changed size 312df8bae1dSRodney W. Grimes */ 3132a8f9ab5SAlan Cox VM_OBJECT_UNLOCK(object); 314df8bae1dSRodney W. Grimes return; 3152a8f9ab5SAlan Cox } 316c576d121SLuoqi Chen nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 3172a8f9ab5SAlan Cox if (nsize < object->un_pager.vnp.vnp_size) { 318df8bae1dSRodney W. Grimes /* 319bbc0ec52SDavid Greenman * File has shrunk. Toss any cached pages beyond the new EOF. 320df8bae1dSRodney W. Grimes */ 3212a8f9ab5SAlan Cox if (nobjsize < object->size) 322c576d121SLuoqi Chen vm_object_page_remove(object, nobjsize, object->size, 323c576d121SLuoqi Chen FALSE); 324bbc0ec52SDavid Greenman /* 325bbc0ec52SDavid Greenman * this gets rid of garbage at the end of a page that is now 3263ebeaf59SMatthew Dillon * only partially backed by the vnode. 3273ebeaf59SMatthew Dillon * 3283ebeaf59SMatthew Dillon * XXX for some reason (I don't know yet), if we take a 3293ebeaf59SMatthew Dillon * completely invalid page and mark it partially valid 3303ebeaf59SMatthew Dillon * it can screw up NFS reads, so we don't allow the case. 331bbc0ec52SDavid Greenman */ 3322a8f9ab5SAlan Cox if ((nsize & PAGE_MASK) && 3331b26eb10SAlan Cox (m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL && 3341b26eb10SAlan Cox m->valid != 0) { 3352b6b0df7SMatthew Dillon int base = (int)nsize & PAGE_MASK; 3362b6b0df7SMatthew Dillon int size = PAGE_SIZE - base; 3372b6b0df7SMatthew Dillon 3382b6b0df7SMatthew Dillon /* 3392b6b0df7SMatthew Dillon * Clear out partial-page garbage in case 3402b6b0df7SMatthew Dillon * the page has been mapped. 3412b6b0df7SMatthew Dillon */ 342fff6062aSAlan Cox pmap_zero_page_area(m, base, size); 3432b6b0df7SMatthew Dillon 3442b6b0df7SMatthew Dillon /* 3453ebeaf59SMatthew Dillon * XXX work around SMP data integrity race 3463ebeaf59SMatthew Dillon * by unmapping the page from user processes. 3473ebeaf59SMatthew Dillon * The garbage we just cleared may be mapped 3483ebeaf59SMatthew Dillon * to a user process running on another cpu 3493ebeaf59SMatthew Dillon * and this code is not running through normal 3503ebeaf59SMatthew Dillon * I/O channels which handle SMP issues for 3513ebeaf59SMatthew Dillon * us, so unmap page to synchronize all cpus. 3523ebeaf59SMatthew Dillon * 3533ebeaf59SMatthew Dillon * XXX should vm_pager_unmap_page() have 3543ebeaf59SMatthew Dillon * dealt with this? 3553ebeaf59SMatthew Dillon */ 3561b26eb10SAlan Cox vm_page_lock_queues(); 3574fec79beSAlan Cox pmap_remove_all(m); 3583ebeaf59SMatthew Dillon 3593ebeaf59SMatthew Dillon /* 3602b6b0df7SMatthew Dillon * Clear out partial-page dirty bits. This 3612b6b0df7SMatthew Dillon * has the side effect of setting the valid 3622b6b0df7SMatthew Dillon * bits, but that is ok. There are a bunch 3632b6b0df7SMatthew Dillon * of places in the VM system where we expected 3642b6b0df7SMatthew Dillon * m->dirty == VM_PAGE_BITS_ALL. The file EOF 3652b6b0df7SMatthew Dillon * case is one of them. If the page is still 3662b6b0df7SMatthew Dillon * partially dirty, make it fully dirty. 3673ebeaf59SMatthew Dillon * 3683ebeaf59SMatthew Dillon * note that we do not clear out the valid 3693ebeaf59SMatthew Dillon * bits. This would prevent bogus_page 3703ebeaf59SMatthew Dillon * replacement from working properly. 3712b6b0df7SMatthew Dillon */ 3722b6b0df7SMatthew Dillon vm_page_set_validclean(m, base, size); 3732b6b0df7SMatthew Dillon if (m->dirty != 0) 3742b6b0df7SMatthew Dillon m->dirty = VM_PAGE_BITS_ALL; 3752a8f9ab5SAlan Cox vm_page_unlock_queues(); 376bbc0ec52SDavid Greenman } 377bbc0ec52SDavid Greenman } 378a316d390SJohn Dyson object->un_pager.vnp.vnp_size = nsize; 379c576d121SLuoqi Chen object->size = nobjsize; 3802a8f9ab5SAlan Cox VM_OBJECT_UNLOCK(object); 381df8bae1dSRodney W. Grimes } 382df8bae1dSRodney W. Grimes 38326f9a767SRodney W. Grimes /* 38426f9a767SRodney W. Grimes * calculate the linear (byte) disk address of specified virtual 38526f9a767SRodney W. Grimes * file address 38626f9a767SRodney W. Grimes */ 387f708ef1bSPoul-Henning Kamp static vm_offset_t 388efc68ce1SDavid Greenman vnode_pager_addr(vp, address, run) 38926f9a767SRodney W. Grimes struct vnode *vp; 390a316d390SJohn Dyson vm_ooffset_t address; 391efc68ce1SDavid Greenman int *run; 39226f9a767SRodney W. Grimes { 39326f9a767SRodney W. Grimes int rtaddress; 39426f9a767SRodney W. Grimes int bsize; 39598b0c789SPoul-Henning Kamp daddr_t block; 39626f9a767SRodney W. Grimes int err; 397a316d390SJohn Dyson daddr_t vblock; 398a316d390SJohn Dyson int voffset; 39926f9a767SRodney W. Grimes 4000cddd8f0SMatthew Dillon GIANT_REQUIRED; 4012ad036b6SAlan Cox if (address < 0) 4020d94caffSDavid Greenman return -1; 4030d94caffSDavid Greenman 4042c4488fcSJohn Dyson if (vp->v_mount == NULL) 4052c4488fcSJohn Dyson return -1; 4062c4488fcSJohn Dyson 40726f9a767SRodney W. Grimes bsize = vp->v_mount->mnt_stat.f_iosize; 40826f9a767SRodney W. Grimes vblock = address / bsize; 40926f9a767SRodney W. Grimes voffset = address % bsize; 41026f9a767SRodney W. Grimes 411ec794849SPoul-Henning Kamp err = VOP_BMAP(vp, vblock, NULL, &block, run, NULL); 41226f9a767SRodney W. Grimes 413efc68ce1SDavid Greenman if (err || (block == -1)) 41426f9a767SRodney W. Grimes rtaddress = -1; 415efc68ce1SDavid Greenman else { 416187f0071SDavid Greenman rtaddress = block + voffset / DEV_BSIZE; 417efc68ce1SDavid Greenman if (run) { 418efc68ce1SDavid Greenman *run += 1; 419efc68ce1SDavid Greenman *run *= bsize/PAGE_SIZE; 420efc68ce1SDavid Greenman *run -= voffset/PAGE_SIZE; 421efc68ce1SDavid Greenman } 422efc68ce1SDavid Greenman } 42326f9a767SRodney W. Grimes 42426f9a767SRodney W. Grimes return rtaddress; 42526f9a767SRodney W. Grimes } 42626f9a767SRodney W. Grimes 42726f9a767SRodney W. Grimes /* 42826f9a767SRodney W. Grimes * small block filesystem vnode pager input 42926f9a767SRodney W. Grimes */ 430f708ef1bSPoul-Henning Kamp static int 43124a1cce3SDavid Greenman vnode_pager_input_smlfs(object, m) 43224a1cce3SDavid Greenman vm_object_t object; 43326f9a767SRodney W. Grimes vm_page_t m; 43426f9a767SRodney W. Grimes { 43526f9a767SRodney W. Grimes int i; 4369c83534dSPoul-Henning Kamp struct vnode *vp; 4379c83534dSPoul-Henning Kamp struct bufobj *bo; 43826f9a767SRodney W. Grimes struct buf *bp; 4399e0ddbd0SAlan Cox struct sf_buf *sf; 44026f9a767SRodney W. Grimes int fileaddr; 44126f9a767SRodney W. Grimes vm_offset_t bsize; 44226f9a767SRodney W. Grimes int error = 0; 44326f9a767SRodney W. Grimes 4440cddd8f0SMatthew Dillon GIANT_REQUIRED; 4450cddd8f0SMatthew Dillon 44624a1cce3SDavid Greenman vp = object->handle; 4472c4488fcSJohn Dyson if (vp->v_mount == NULL) 4482c4488fcSJohn Dyson return VM_PAGER_BAD; 4492c4488fcSJohn Dyson 45026f9a767SRodney W. Grimes bsize = vp->v_mount->mnt_stat.f_iosize; 4510bdb7528SDavid Greenman 4529c83534dSPoul-Henning Kamp VOP_BMAP(vp, 0, &bo, 0, NULL, NULL); 45326f9a767SRodney W. Grimes 4549e0ddbd0SAlan Cox sf = sf_buf_alloc(m, 0); 45526f9a767SRodney W. Grimes 45626f9a767SRodney W. Grimes for (i = 0; i < PAGE_SIZE / bsize; i++) { 45733c67741SMatthew Dillon vm_ooffset_t address; 458bbc0ec52SDavid Greenman 459897a45efSDmitrij Tejblum if (vm_page_bits(i * bsize, bsize) & m->valid) 46026f9a767SRodney W. Grimes continue; 46126f9a767SRodney W. Grimes 46233c67741SMatthew Dillon address = IDX_TO_OFF(m->pindex) + i * bsize; 46333c67741SMatthew Dillon if (address >= object->un_pager.vnp.vnp_size) { 46433c67741SMatthew Dillon fileaddr = -1; 46533c67741SMatthew Dillon } else { 46633c67741SMatthew Dillon fileaddr = vnode_pager_addr(vp, address, NULL); 46733c67741SMatthew Dillon } 46826f9a767SRodney W. Grimes if (fileaddr != -1) { 4691c7c3c6aSMatthew Dillon bp = getpbuf(&vnode_pbuf_freecnt); 47026f9a767SRodney W. Grimes 47126f9a767SRodney W. Grimes /* build a minimal buffer header */ 47221144e3bSPoul-Henning Kamp bp->b_iocmd = BIO_READ; 4736a4b5823SPoul-Henning Kamp bp->b_iodone = bdone; 474bd78ceceSJohn Baldwin KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 475bd78ceceSJohn Baldwin KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 476a854ed98SJohn Baldwin bp->b_rcred = crhold(curthread->td_ucred); 477a854ed98SJohn Baldwin bp->b_wcred = crhold(curthread->td_ucred); 4789e0ddbd0SAlan Cox bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize; 479187f0071SDavid Greenman bp->b_blkno = fileaddr; 4809c83534dSPoul-Henning Kamp pbgetbo(bo, bp); 48126f9a767SRodney W. Grimes bp->b_bcount = bsize; 48226f9a767SRodney W. Grimes bp->b_bufsize = bsize; 4832b6b0df7SMatthew Dillon bp->b_runningbufspace = bp->b_bufsize; 4842b6b0df7SMatthew Dillon runningbufspace += bp->b_runningbufspace; 48526f9a767SRodney W. Grimes 48626f9a767SRodney W. Grimes /* do the input */ 4872c18019fSPoul-Henning Kamp bp->b_iooffset = dbtob(bp->b_blkno); 488b792bebeSPoul-Henning Kamp bstrategy(bp); 48926f9a767SRodney W. Grimes 490e47ed70bSJohn Dyson /* we definitely need to be at splvm here */ 49126f9a767SRodney W. Grimes 4926a4b5823SPoul-Henning Kamp bwait(bp, PVM, "vnsrd"); 4936a4b5823SPoul-Henning Kamp 494c244d2deSPoul-Henning Kamp if ((bp->b_ioflags & BIO_ERROR) != 0) 49526f9a767SRodney W. Grimes error = EIO; 49626f9a767SRodney W. Grimes 49726f9a767SRodney W. Grimes /* 49826f9a767SRodney W. Grimes * free the buffer header back to the swap buffer pool 49926f9a767SRodney W. Grimes */ 5009c83534dSPoul-Henning Kamp pbrelbo(bp); 5011c7c3c6aSMatthew Dillon relpbuf(bp, &vnode_pbuf_freecnt); 50226f9a767SRodney W. Grimes if (error) 50326f9a767SRodney W. Grimes break; 5040d94caffSDavid Greenman 5052bf43e43SAlan Cox VM_OBJECT_LOCK(object); 506178949e0SAlan Cox vm_page_lock_queues(); 507aa8de40aSPoul-Henning Kamp vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 508178949e0SAlan Cox vm_page_unlock_queues(); 5092bf43e43SAlan Cox VM_OBJECT_UNLOCK(object); 51026f9a767SRodney W. Grimes } else { 5112bf43e43SAlan Cox VM_OBJECT_LOCK(object); 512178949e0SAlan Cox vm_page_lock_queues(); 513aa8de40aSPoul-Henning Kamp vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 514178949e0SAlan Cox vm_page_unlock_queues(); 5152bf43e43SAlan Cox VM_OBJECT_UNLOCK(object); 5169e0ddbd0SAlan Cox bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize); 51726f9a767SRodney W. Grimes } 51826f9a767SRodney W. Grimes } 5199e0ddbd0SAlan Cox sf_buf_free(sf); 52085e01243SAlan Cox vm_page_lock_queues(); 5210385347cSPeter Wemm pmap_clear_modify(m); 52285e01243SAlan Cox vm_page_unlock_queues(); 52326f9a767SRodney W. Grimes if (error) { 524a83c285cSDavid Greenman return VM_PAGER_ERROR; 52526f9a767SRodney W. Grimes } 52626f9a767SRodney W. Grimes return VM_PAGER_OK; 52726f9a767SRodney W. Grimes 52826f9a767SRodney W. Grimes } 52926f9a767SRodney W. Grimes 53026f9a767SRodney W. Grimes 53126f9a767SRodney W. Grimes /* 532475e8cc3SPoul-Henning Kamp * old style vnode pager input routine 53326f9a767SRodney W. Grimes */ 534f708ef1bSPoul-Henning Kamp static int 53524a1cce3SDavid Greenman vnode_pager_input_old(object, m) 53624a1cce3SDavid Greenman vm_object_t object; 53726f9a767SRodney W. Grimes vm_page_t m; 53826f9a767SRodney W. Grimes { 539df8bae1dSRodney W. Grimes struct uio auio; 540df8bae1dSRodney W. Grimes struct iovec aiov; 54126f9a767SRodney W. Grimes int error; 54226f9a767SRodney W. Grimes int size; 5439e0ddbd0SAlan Cox struct sf_buf *sf; 544342a1480SJohn Baldwin struct vnode *vp; 545df8bae1dSRodney W. Grimes 54652051abcSAlan Cox VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 54726f9a767SRodney W. Grimes error = 0; 548bbc0ec52SDavid Greenman 549df8bae1dSRodney W. Grimes /* 55026f9a767SRodney W. Grimes * Return failure if beyond current EOF 55126f9a767SRodney W. Grimes */ 552a316d390SJohn Dyson if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { 55326f9a767SRodney W. Grimes return VM_PAGER_BAD; 55426f9a767SRodney W. Grimes } else { 55526f9a767SRodney W. Grimes size = PAGE_SIZE; 556a316d390SJohn Dyson if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) 557a316d390SJohn Dyson size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); 55852051abcSAlan Cox vp = object->handle; 55952051abcSAlan Cox VM_OBJECT_UNLOCK(object); 5600bdb7528SDavid Greenman 56126f9a767SRodney W. Grimes /* 562df8bae1dSRodney W. Grimes * Allocate a kernel virtual address and initialize so that 563df8bae1dSRodney W. Grimes * we can use VOP_READ/WRITE routines. 564df8bae1dSRodney W. Grimes */ 5659e0ddbd0SAlan Cox sf = sf_buf_alloc(m, 0); 5660bdb7528SDavid Greenman 5679e0ddbd0SAlan Cox aiov.iov_base = (caddr_t)sf_buf_kva(sf); 568df8bae1dSRodney W. Grimes aiov.iov_len = size; 569df8bae1dSRodney W. Grimes auio.uio_iov = &aiov; 570df8bae1dSRodney W. Grimes auio.uio_iovcnt = 1; 571a316d390SJohn Dyson auio.uio_offset = IDX_TO_OFF(m->pindex); 572df8bae1dSRodney W. Grimes auio.uio_segflg = UIO_SYSSPACE; 57326f9a767SRodney W. Grimes auio.uio_rw = UIO_READ; 574df8bae1dSRodney W. Grimes auio.uio_resid = size; 575b40ce416SJulian Elischer auio.uio_td = curthread; 57626f9a767SRodney W. Grimes 577a854ed98SJohn Baldwin error = VOP_READ(vp, &auio, 0, curthread->td_ucred); 578df8bae1dSRodney W. Grimes if (!error) { 57954d92145SMatthew Dillon int count = size - auio.uio_resid; 580df8bae1dSRodney W. Grimes 581df8bae1dSRodney W. Grimes if (count == 0) 582df8bae1dSRodney W. Grimes error = EINVAL; 58326f9a767SRodney W. Grimes else if (count != PAGE_SIZE) 5849e0ddbd0SAlan Cox bzero((caddr_t)sf_buf_kva(sf) + count, 5859e0ddbd0SAlan Cox PAGE_SIZE - count); 586df8bae1dSRodney W. Grimes } 5879e0ddbd0SAlan Cox sf_buf_free(sf); 5881b26eb10SAlan Cox 5891b26eb10SAlan Cox VM_OBJECT_LOCK(object); 590df8bae1dSRodney W. Grimes } 59185e01243SAlan Cox vm_page_lock_queues(); 5920385347cSPeter Wemm pmap_clear_modify(m); 5932c28a105SAlan Cox vm_page_undirty(m); 5941b26eb10SAlan Cox vm_page_unlock_queues(); 5956e3a3f38SRobert V. Baron if (!error) 5966e3a3f38SRobert V. Baron m->valid = VM_PAGE_BITS_ALL; 597a83c285cSDavid Greenman return error ? VM_PAGER_ERROR : VM_PAGER_OK; 59826f9a767SRodney W. Grimes } 59926f9a767SRodney W. Grimes 60026f9a767SRodney W. Grimes /* 60126f9a767SRodney W. Grimes * generic vnode pager input routine 60226f9a767SRodney W. Grimes */ 603170db9c6SJohn Dyson 604ce75f2c3SMike Smith /* 60523955314SAlfred Perlstein * Local media VFS's that do not implement their own VOP_GETPAGES 60647e151ddSRobert Drehmel * should have their VOP_GETPAGES call to vnode_pager_generic_getpages() 60747e151ddSRobert Drehmel * to implement the previous behaviour. 608ce75f2c3SMike Smith * 609ce75f2c3SMike Smith * All other FS's should use the bypass to get to the local media 610ce75f2c3SMike Smith * backing vp's VOP_GETPAGES. 611ce75f2c3SMike Smith */ 612f708ef1bSPoul-Henning Kamp static int 61324a1cce3SDavid Greenman vnode_pager_getpages(object, m, count, reqpage) 61426f9a767SRodney W. Grimes vm_object_t object; 61524a1cce3SDavid Greenman vm_page_t *m; 61624a1cce3SDavid Greenman int count; 61724a1cce3SDavid Greenman int reqpage; 61824a1cce3SDavid Greenman { 619170db9c6SJohn Dyson int rtval; 620170db9c6SJohn Dyson struct vnode *vp; 62186ffbd76SMike Smith int bytes = count * PAGE_SIZE; 62295e5e988SJohn Dyson 623170db9c6SJohn Dyson vp = object->handle; 6248630c117SAlan Cox VM_OBJECT_UNLOCK(object); 62587aefa49SAlan Cox mtx_lock(&Giant); 62686ffbd76SMike Smith rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0); 62723955314SAlfred Perlstein KASSERT(rtval != EOPNOTSUPP, 62823955314SAlfred Perlstein ("vnode_pager: FS getpages not implemented\n")); 62987aefa49SAlan Cox mtx_unlock(&Giant); 6308630c117SAlan Cox VM_OBJECT_LOCK(object); 631170db9c6SJohn Dyson return rtval; 632170db9c6SJohn Dyson } 633170db9c6SJohn Dyson 634ce75f2c3SMike Smith /* 635ce75f2c3SMike Smith * This is now called from local media FS's to operate against their 636ce75f2c3SMike Smith * own vnodes if they fail to implement VOP_GETPAGES. 637ce75f2c3SMike Smith */ 638ce75f2c3SMike Smith int 639ce75f2c3SMike Smith vnode_pager_generic_getpages(vp, m, bytecount, reqpage) 640ce75f2c3SMike Smith struct vnode *vp; 641170db9c6SJohn Dyson vm_page_t *m; 642ce75f2c3SMike Smith int bytecount; 643170db9c6SJohn Dyson int reqpage; 644170db9c6SJohn Dyson { 645ce75f2c3SMike Smith vm_object_t object; 646a316d390SJohn Dyson vm_offset_t kva; 6478f9110f6SJohn Dyson off_t foff, tfoff, nextoff; 648e43c2eabSAlan Cox int i, j, size, bsize, first, firstaddr; 6499c83534dSPoul-Henning Kamp struct bufobj *bo; 650efc68ce1SDavid Greenman int runpg; 651efc68ce1SDavid Greenman int runend; 6520bdb7528SDavid Greenman struct buf *bp; 653ce75f2c3SMike Smith int count; 65426f9a767SRodney W. Grimes int error = 0; 65526f9a767SRodney W. Grimes 6560cddd8f0SMatthew Dillon GIANT_REQUIRED; 657ce75f2c3SMike Smith object = vp->v_object; 658ce75f2c3SMike Smith count = bytecount / PAGE_SIZE; 659ce75f2c3SMike Smith 6609c83534dSPoul-Henning Kamp KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, 6619c83534dSPoul-Henning Kamp ("vnode_pager_generic_getpages does not support devices")); 6622c4488fcSJohn Dyson if (vp->v_mount == NULL) 6632c4488fcSJohn Dyson return VM_PAGER_BAD; 6642c4488fcSJohn Dyson 66526f9a767SRodney W. Grimes bsize = vp->v_mount->mnt_stat.f_iosize; 66626f9a767SRodney W. Grimes 66726f9a767SRodney W. Grimes /* get the UNDERLYING device for the file with VOP_BMAP() */ 668bbc0ec52SDavid Greenman 66926f9a767SRodney W. Grimes /* 670bbc0ec52SDavid Greenman * originally, we did not check for an error return value -- assuming 671bbc0ec52SDavid Greenman * an fs always has a bmap entry point -- that assumption is wrong!!! 67226f9a767SRodney W. Grimes */ 673a316d390SJohn Dyson foff = IDX_TO_OFF(m[reqpage]->pindex); 674bbc0ec52SDavid Greenman 67526f9a767SRodney W. Grimes /* 67616f62314SDavid Greenman * if we can't bmap, use old VOP code 67726f9a767SRodney W. Grimes */ 6789c83534dSPoul-Henning Kamp if (VOP_BMAP(vp, 0, &bo, 0, NULL, NULL)) { 67931953be9SAlan Cox VM_OBJECT_LOCK(object); 680e43c2eabSAlan Cox vm_page_lock_queues(); 681e43c2eabSAlan Cox for (i = 0; i < count; i++) 682e43c2eabSAlan Cox if (i != reqpage) 683d8d5fa88SAlfred Perlstein vm_page_free(m[i]); 684e43c2eabSAlan Cox vm_page_unlock_queues(); 685976e77fcSDavid Greenman cnt.v_vnodein++; 686976e77fcSDavid Greenman cnt.v_vnodepgsin++; 68752051abcSAlan Cox error = vnode_pager_input_old(object, m[reqpage]); 68852051abcSAlan Cox VM_OBJECT_UNLOCK(object); 68952051abcSAlan Cox return (error); 690bbc0ec52SDavid Greenman 69126f9a767SRodney W. Grimes /* 69226f9a767SRodney W. Grimes * if the blocksize is smaller than a page size, then use 69326f9a767SRodney W. Grimes * special small filesystem code. NFS sometimes has a small 69426f9a767SRodney W. Grimes * blocksize, but it can handle large reads itself. 69526f9a767SRodney W. Grimes */ 69626f9a767SRodney W. Grimes } else if ((PAGE_SIZE / bsize) > 1 && 697500b04a2SBruce Evans (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 69831953be9SAlan Cox VM_OBJECT_LOCK(object); 699e43c2eabSAlan Cox vm_page_lock_queues(); 700e43c2eabSAlan Cox for (i = 0; i < count; i++) 701e43c2eabSAlan Cox if (i != reqpage) 702d8d5fa88SAlfred Perlstein vm_page_free(m[i]); 703e43c2eabSAlan Cox vm_page_unlock_queues(); 70431953be9SAlan Cox VM_OBJECT_UNLOCK(object); 705976e77fcSDavid Greenman cnt.v_vnodein++; 706976e77fcSDavid Greenman cnt.v_vnodepgsin++; 70724a1cce3SDavid Greenman return vnode_pager_input_smlfs(object, m[reqpage]); 70826f9a767SRodney W. Grimes } 7098d17e694SJulian Elischer 71026f9a767SRodney W. Grimes /* 7118d17e694SJulian Elischer * If we have a completely valid page available to us, we can 7128d17e694SJulian Elischer * clean up and return. Otherwise we have to re-read the 7138d17e694SJulian Elischer * media. 7140d94caffSDavid Greenman */ 71531953be9SAlan Cox VM_OBJECT_LOCK(object); 7168b575f6cSAlan Cox if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 717e43c2eabSAlan Cox vm_page_lock_queues(); 718e43c2eabSAlan Cox for (i = 0; i < count; i++) 7190d94caffSDavid Greenman if (i != reqpage) 720d8d5fa88SAlfred Perlstein vm_page_free(m[i]); 721e43c2eabSAlan Cox vm_page_unlock_queues(); 72231953be9SAlan Cox VM_OBJECT_UNLOCK(object); 7230d94caffSDavid Greenman return VM_PAGER_OK; 7240d94caffSDavid Greenman } 7258d17e694SJulian Elischer m[reqpage]->valid = 0; 7268b575f6cSAlan Cox VM_OBJECT_UNLOCK(object); 7270bdb7528SDavid Greenman 7280d94caffSDavid Greenman /* 72926f9a767SRodney W. Grimes * here on direct device I/O 73026f9a767SRodney W. Grimes */ 731efc68ce1SDavid Greenman firstaddr = -1; 732a1287949SEivind Eklund 73326f9a767SRodney W. Grimes /* 734efc68ce1SDavid Greenman * calculate the run that includes the required page 73526f9a767SRodney W. Grimes */ 736efc68ce1SDavid Greenman for (first = 0, i = 0; i < count; i = runend) { 737a316d390SJohn Dyson firstaddr = vnode_pager_addr(vp, 738a316d390SJohn Dyson IDX_TO_OFF(m[i]->pindex), &runpg); 739efc68ce1SDavid Greenman if (firstaddr == -1) { 74031953be9SAlan Cox VM_OBJECT_LOCK(object); 74124a1cce3SDavid Greenman if (i == reqpage && foff < object->un_pager.vnp.vnp_size) { 742bf1001faSMaxime Henrion panic("vnode_pager_getpages: unexpected missing page: firstaddr: %d, foff: 0x%jx%08jx, vnp_size: 0x%jx%08jx", 743bf1001faSMaxime Henrion firstaddr, (uintmax_t)(foff >> 32), 744bf1001faSMaxime Henrion (uintmax_t)foff, 745bf1001faSMaxime Henrion (uintmax_t) 746fc62ef1fSBruce Evans (object->un_pager.vnp.vnp_size >> 32), 747bf1001faSMaxime Henrion (uintmax_t)object->un_pager.vnp.vnp_size); 748efc68ce1SDavid Greenman } 749e43c2eabSAlan Cox vm_page_lock_queues(); 750d8d5fa88SAlfred Perlstein vm_page_free(m[i]); 751e43c2eabSAlan Cox vm_page_unlock_queues(); 75231953be9SAlan Cox VM_OBJECT_UNLOCK(object); 753efc68ce1SDavid Greenman runend = i + 1; 754efc68ce1SDavid Greenman first = runend; 755efc68ce1SDavid Greenman continue; 756efc68ce1SDavid Greenman } 757efc68ce1SDavid Greenman runend = i + runpg; 758efc68ce1SDavid Greenman if (runend <= reqpage) { 75931953be9SAlan Cox VM_OBJECT_LOCK(object); 760e43c2eabSAlan Cox vm_page_lock_queues(); 761e43c2eabSAlan Cox for (j = i; j < runend; j++) 762d8d5fa88SAlfred Perlstein vm_page_free(m[j]); 763e43c2eabSAlan Cox vm_page_unlock_queues(); 76431953be9SAlan Cox VM_OBJECT_UNLOCK(object); 76526f9a767SRodney W. Grimes } else { 766efc68ce1SDavid Greenman if (runpg < (count - first)) { 76731953be9SAlan Cox VM_OBJECT_LOCK(object); 768e43c2eabSAlan Cox vm_page_lock_queues(); 769efc68ce1SDavid Greenman for (i = first + runpg; i < count; i++) 770d8d5fa88SAlfred Perlstein vm_page_free(m[i]); 771e43c2eabSAlan Cox vm_page_unlock_queues(); 77231953be9SAlan Cox VM_OBJECT_UNLOCK(object); 773efc68ce1SDavid Greenman count = first + runpg; 77426f9a767SRodney W. Grimes } 775efc68ce1SDavid Greenman break; 77626f9a767SRodney W. Grimes } 777efc68ce1SDavid Greenman first = runend; 778efc68ce1SDavid Greenman } 77926f9a767SRodney W. Grimes 78026f9a767SRodney W. Grimes /* 781bbc0ec52SDavid Greenman * the first and last page have been calculated now, move input pages 782bbc0ec52SDavid Greenman * to be zero based... 78326f9a767SRodney W. Grimes */ 78426f9a767SRodney W. Grimes if (first != 0) { 78526f9a767SRodney W. Grimes for (i = first; i < count; i++) { 78626f9a767SRodney W. Grimes m[i - first] = m[i]; 78726f9a767SRodney W. Grimes } 78826f9a767SRodney W. Grimes count -= first; 78926f9a767SRodney W. Grimes reqpage -= first; 79026f9a767SRodney W. Grimes } 791efc68ce1SDavid Greenman 79226f9a767SRodney W. Grimes /* 79326f9a767SRodney W. Grimes * calculate the file virtual address for the transfer 79426f9a767SRodney W. Grimes */ 795a316d390SJohn Dyson foff = IDX_TO_OFF(m[0]->pindex); 79626f9a767SRodney W. Grimes 79726f9a767SRodney W. Grimes /* 79826f9a767SRodney W. Grimes * calculate the size of the transfer 79926f9a767SRodney W. Grimes */ 80026f9a767SRodney W. Grimes size = count * PAGE_SIZE; 8011a31a6c3SPoul-Henning Kamp KASSERT(count > 0, ("zero count")); 80224a1cce3SDavid Greenman if ((foff + size) > object->un_pager.vnp.vnp_size) 80324a1cce3SDavid Greenman size = object->un_pager.vnp.vnp_size - foff; 8041a31a6c3SPoul-Henning Kamp KASSERT(size > 0, ("zero size")); 80526f9a767SRodney W. Grimes 80626f9a767SRodney W. Grimes /* 80724579ca1SMatthew Dillon * round up physical size for real devices. 80826f9a767SRodney W. Grimes */ 8099c83534dSPoul-Henning Kamp if (1) { 8109c83534dSPoul-Henning Kamp int secmask = bo->bo_bsize - 1; 8116229cc50SPoul-Henning Kamp KASSERT(secmask < PAGE_SIZE && secmask > 0, 8126229cc50SPoul-Henning Kamp ("vnode_pager_generic_getpages: sector size %d too large", 8136229cc50SPoul-Henning Kamp secmask + 1)); 81424579ca1SMatthew Dillon size = (size + secmask) & ~secmask; 81524579ca1SMatthew Dillon } 81626f9a767SRodney W. Grimes 8171c7c3c6aSMatthew Dillon bp = getpbuf(&vnode_pbuf_freecnt); 81816f62314SDavid Greenman kva = (vm_offset_t) bp->b_data; 81916f62314SDavid Greenman 82026f9a767SRodney W. Grimes /* 82126f9a767SRodney W. Grimes * and map the pages to be read into the kva 82226f9a767SRodney W. Grimes */ 82316f62314SDavid Greenman pmap_qenter(kva, m, count); 82426f9a767SRodney W. Grimes 82526f9a767SRodney W. Grimes /* build a minimal buffer header */ 82621144e3bSPoul-Henning Kamp bp->b_iocmd = BIO_READ; 8276a4b5823SPoul-Henning Kamp bp->b_iodone = bdone; 828bd78ceceSJohn Baldwin KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 829bd78ceceSJohn Baldwin KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 830a854ed98SJohn Baldwin bp->b_rcred = crhold(curthread->td_ucred); 831a854ed98SJohn Baldwin bp->b_wcred = crhold(curthread->td_ucred); 832187f0071SDavid Greenman bp->b_blkno = firstaddr; 8339c83534dSPoul-Henning Kamp pbgetbo(bo, bp); 83426f9a767SRodney W. Grimes bp->b_bcount = size; 83526f9a767SRodney W. Grimes bp->b_bufsize = size; 8362b6b0df7SMatthew Dillon bp->b_runningbufspace = bp->b_bufsize; 8372b6b0df7SMatthew Dillon runningbufspace += bp->b_runningbufspace; 83826f9a767SRodney W. Grimes 839976e77fcSDavid Greenman cnt.v_vnodein++; 840976e77fcSDavid Greenman cnt.v_vnodepgsin += count; 841976e77fcSDavid Greenman 84226f9a767SRodney W. Grimes /* do the input */ 8432c18019fSPoul-Henning Kamp bp->b_iooffset = dbtob(bp->b_blkno); 844b792bebeSPoul-Henning Kamp bstrategy(bp); 845976e77fcSDavid Greenman 8466a4b5823SPoul-Henning Kamp bwait(bp, PVM, "vnread"); 84726f9a767SRodney W. Grimes 848c244d2deSPoul-Henning Kamp if ((bp->b_ioflags & BIO_ERROR) != 0) 84926f9a767SRodney W. Grimes error = EIO; 85026f9a767SRodney W. Grimes 85126f9a767SRodney W. Grimes if (!error) { 85226f9a767SRodney W. Grimes if (size != count * PAGE_SIZE) 85326f9a767SRodney W. Grimes bzero((caddr_t) kva + size, PAGE_SIZE * count - size); 85426f9a767SRodney W. Grimes } 85516f62314SDavid Greenman pmap_qremove(kva, count); 85626f9a767SRodney W. Grimes 85726f9a767SRodney W. Grimes /* 85826f9a767SRodney W. Grimes * free the buffer header back to the swap buffer pool 85926f9a767SRodney W. Grimes */ 8609c83534dSPoul-Henning Kamp pbrelbo(bp); 8611c7c3c6aSMatthew Dillon relpbuf(bp, &vnode_pbuf_freecnt); 86226f9a767SRodney W. Grimes 86331953be9SAlan Cox VM_OBJECT_LOCK(object); 8649d522888SAlan Cox vm_page_lock_queues(); 8658f9110f6SJohn Dyson for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { 8668f9110f6SJohn Dyson vm_page_t mt; 8678f9110f6SJohn Dyson 8688f9110f6SJohn Dyson nextoff = tfoff + PAGE_SIZE; 8698f9110f6SJohn Dyson mt = m[i]; 8708f9110f6SJohn Dyson 87154746b67SDmitrij Tejblum if (nextoff <= object->un_pager.vnp.vnp_size) { 8728d17e694SJulian Elischer /* 8738d17e694SJulian Elischer * Read filled up entire page. 8748d17e694SJulian Elischer */ 8758f9110f6SJohn Dyson mt->valid = VM_PAGE_BITS_ALL; 8762c28a105SAlan Cox vm_page_undirty(mt); /* should be an assert? XXX */ 8770385347cSPeter Wemm pmap_clear_modify(mt); 8788f9110f6SJohn Dyson } else { 8798d17e694SJulian Elischer /* 8808d17e694SJulian Elischer * Read did not fill up entire page. Since this 8818d17e694SJulian Elischer * is getpages, the page may be mapped, so we have 8828d17e694SJulian Elischer * to zero the invalid portions of the page even 8838d17e694SJulian Elischer * though we aren't setting them valid. 8848d17e694SJulian Elischer * 8858d17e694SJulian Elischer * Currently we do not set the entire page valid, 8868d17e694SJulian Elischer * we just try to clear the piece that we couldn't 8878d17e694SJulian Elischer * read. 8888d17e694SJulian Elischer */ 88954746b67SDmitrij Tejblum vm_page_set_validclean(mt, 0, 89054746b67SDmitrij Tejblum object->un_pager.vnp.vnp_size - tfoff); 8914221e284SAlan Cox /* handled by vm_fault now */ 8924221e284SAlan Cox /* vm_page_zero_invalid(mt, FALSE); */ 8938f9110f6SJohn Dyson } 8948f9110f6SJohn Dyson 89526f9a767SRodney W. Grimes if (i != reqpage) { 896bbc0ec52SDavid Greenman 89726f9a767SRodney W. Grimes /* 898bbc0ec52SDavid Greenman * whether or not to leave the page activated is up in 899bbc0ec52SDavid Greenman * the air, but we should put the page on a page queue 900bbc0ec52SDavid Greenman * somewhere. (it already is in the object). Result: 901956f3135SPhilippe Charnier * It appears that empirical results show that 902bbc0ec52SDavid Greenman * deactivating pages is best. 90326f9a767SRodney W. Grimes */ 904bbc0ec52SDavid Greenman 90526f9a767SRodney W. Grimes /* 906bbc0ec52SDavid Greenman * just in case someone was asking for this page we 907bbc0ec52SDavid Greenman * now tell them that it is ok to use 90826f9a767SRodney W. Grimes */ 90926f9a767SRodney W. Grimes if (!error) { 9108f9110f6SJohn Dyson if (mt->flags & PG_WANTED) 9118f9110f6SJohn Dyson vm_page_activate(mt); 91295461b45SJohn Dyson else 9138f9110f6SJohn Dyson vm_page_deactivate(mt); 914e69763a3SDoug Rabson vm_page_wakeup(mt); 91526f9a767SRodney W. Grimes } else { 916d8d5fa88SAlfred Perlstein vm_page_free(mt); 91726f9a767SRodney W. Grimes } 91826f9a767SRodney W. Grimes } 91926f9a767SRodney W. Grimes } 9209d522888SAlan Cox vm_page_unlock_queues(); 92131953be9SAlan Cox VM_OBJECT_UNLOCK(object); 92226f9a767SRodney W. Grimes if (error) { 92324a1cce3SDavid Greenman printf("vnode_pager_getpages: I/O read error\n"); 92426f9a767SRodney W. Grimes } 925a83c285cSDavid Greenman return (error ? VM_PAGER_ERROR : VM_PAGER_OK); 92626f9a767SRodney W. Grimes } 92726f9a767SRodney W. Grimes 928ce75f2c3SMike Smith /* 929ce75f2c3SMike Smith * EOPNOTSUPP is no longer legal. For local media VFS's that do not 930ce75f2c3SMike Smith * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 931ce75f2c3SMike Smith * vnode_pager_generic_putpages() to implement the previous behaviour. 932ce75f2c3SMike Smith * 933ce75f2c3SMike Smith * All other FS's should use the bypass to get to the local media 934ce75f2c3SMike Smith * backing vp's VOP_PUTPAGES. 935ce75f2c3SMike Smith */ 936e4542174SMatthew Dillon static void 937170db9c6SJohn Dyson vnode_pager_putpages(object, m, count, sync, rtvals) 938170db9c6SJohn Dyson vm_object_t object; 939170db9c6SJohn Dyson vm_page_t *m; 940170db9c6SJohn Dyson int count; 941170db9c6SJohn Dyson boolean_t sync; 942170db9c6SJohn Dyson int *rtvals; 943170db9c6SJohn Dyson { 944170db9c6SJohn Dyson int rtval; 945170db9c6SJohn Dyson struct vnode *vp; 946f2a2857bSKirk McKusick struct mount *mp; 94786ffbd76SMike Smith int bytes = count * PAGE_SIZE; 948ad980522SJohn Dyson 9490cddd8f0SMatthew Dillon GIANT_REQUIRED; 9500e3cdf2cSAlan Cox /* 9510e3cdf2cSAlan Cox * Force synchronous operation if we are extremely low on memory 9520e3cdf2cSAlan Cox * to prevent a low-memory deadlock. VOP operations often need to 9530e3cdf2cSAlan Cox * allocate more memory to initiate the I/O ( i.e. do a BMAP 9540e3cdf2cSAlan Cox * operation ). The swapper handles the case by limiting the amount 9550e3cdf2cSAlan Cox * of asynchronous I/O, but that sort of solution doesn't scale well 9560e3cdf2cSAlan Cox * for the vnode pager without a lot of work. 9570e3cdf2cSAlan Cox * 9580e3cdf2cSAlan Cox * Also, the backing vnode's iodone routine may not wake the pageout 9590e3cdf2cSAlan Cox * daemon up. This should be probably be addressed XXX. 9600e3cdf2cSAlan Cox */ 9610e3cdf2cSAlan Cox 9620e3cdf2cSAlan Cox if ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_pageout_free_min) 9630e3cdf2cSAlan Cox sync |= OBJPC_SYNC; 9640e3cdf2cSAlan Cox 9650e3cdf2cSAlan Cox /* 9660e3cdf2cSAlan Cox * Call device-specific putpages function 9670e3cdf2cSAlan Cox */ 968170db9c6SJohn Dyson vp = object->handle; 9692e3b314dSAlan Cox VM_OBJECT_UNLOCK(object); 970f2a2857bSKirk McKusick if (vp->v_type != VREG) 971f2a2857bSKirk McKusick mp = NULL; 972f2a2857bSKirk McKusick (void)vn_start_write(vp, &mp, V_WAIT); 97386ffbd76SMike Smith rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0); 97423955314SAlfred Perlstein KASSERT(rtval != EOPNOTSUPP, 97523955314SAlfred Perlstein ("vnode_pager: stale FS putpages\n")); 976f2a2857bSKirk McKusick vn_finished_write(mp); 9772e3b314dSAlan Cox VM_OBJECT_LOCK(object); 978170db9c6SJohn Dyson } 979170db9c6SJohn Dyson 980ce75f2c3SMike Smith 98126f9a767SRodney W. Grimes /* 982ce75f2c3SMike Smith * This is now called from local media FS's to operate against their 9834491ea91SEivind Eklund * own vnodes if they fail to implement VOP_PUTPAGES. 9842b6b0df7SMatthew Dillon * 9852b6b0df7SMatthew Dillon * This is typically called indirectly via the pageout daemon and 9862b6b0df7SMatthew Dillon * clustering has already typically occured, so in general we ask the 9872b6b0df7SMatthew Dillon * underlying filesystem to write the data out asynchronously rather 9882b6b0df7SMatthew Dillon * then delayed. 98926f9a767SRodney W. Grimes */ 990ce75f2c3SMike Smith int 9918f9110f6SJohn Dyson vnode_pager_generic_putpages(vp, m, bytecount, flags, rtvals) 992ce75f2c3SMike Smith struct vnode *vp; 99326f9a767SRodney W. Grimes vm_page_t *m; 994ce75f2c3SMike Smith int bytecount; 9958f9110f6SJohn Dyson int flags; 99626f9a767SRodney W. Grimes int *rtvals; 99726f9a767SRodney W. Grimes { 998f6b04d2bSDavid Greenman int i; 999ce75f2c3SMike Smith vm_object_t object; 1000ce75f2c3SMike Smith int count; 100126f9a767SRodney W. Grimes 1002f6b04d2bSDavid Greenman int maxsize, ncount; 1003a316d390SJohn Dyson vm_ooffset_t poffset; 1004f6b04d2bSDavid Greenman struct uio auio; 1005f6b04d2bSDavid Greenman struct iovec aiov; 1006f6b04d2bSDavid Greenman int error; 10078f9110f6SJohn Dyson int ioflags; 100826f9a767SRodney W. Grimes 10090cddd8f0SMatthew Dillon GIANT_REQUIRED; 1010ce75f2c3SMike Smith object = vp->v_object; 1011ce75f2c3SMike Smith count = bytecount / PAGE_SIZE; 1012ce75f2c3SMike Smith 101326f9a767SRodney W. Grimes for (i = 0; i < count; i++) 101426f9a767SRodney W. Grimes rtvals[i] = VM_PAGER_AGAIN; 101526f9a767SRodney W. Grimes 1016d8fed1d0SAlan Cox if ((int64_t)m[0]->pindex < 0) { 101723562e4bSMarcel Moolenaar printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%lx)\n", 101823562e4bSMarcel Moolenaar (long)m[0]->pindex, (u_long)m[0]->dirty); 1019f6b04d2bSDavid Greenman rtvals[0] = VM_PAGER_BAD; 1020f6b04d2bSDavid Greenman return VM_PAGER_BAD; 10210d94caffSDavid Greenman } 10220bdb7528SDavid Greenman 1023f6b04d2bSDavid Greenman maxsize = count * PAGE_SIZE; 1024f6b04d2bSDavid Greenman ncount = count; 102526f9a767SRodney W. Grimes 1026a316d390SJohn Dyson poffset = IDX_TO_OFF(m[0]->pindex); 102700a6f47fSMatthew Dillon 102800a6f47fSMatthew Dillon /* 102900a6f47fSMatthew Dillon * If the page-aligned write is larger then the actual file we 103000a6f47fSMatthew Dillon * have to invalidate pages occuring beyond the file EOF. However, 103100a6f47fSMatthew Dillon * there is an edge case where a file may not be page-aligned where 103200a6f47fSMatthew Dillon * the last page is partially invalid. In this case the filesystem 103300a6f47fSMatthew Dillon * may not properly clear the dirty bits for the entire page (which 103400a6f47fSMatthew Dillon * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 103500a6f47fSMatthew Dillon * With the page locked we are free to fix-up the dirty bits here. 10363ebeaf59SMatthew Dillon * 10373ebeaf59SMatthew Dillon * We do not under any circumstances truncate the valid bits, as 10383ebeaf59SMatthew Dillon * this will screw up bogus page replacement. 103900a6f47fSMatthew Dillon */ 1040a316d390SJohn Dyson if (maxsize + poffset > object->un_pager.vnp.vnp_size) { 104100a6f47fSMatthew Dillon if (object->un_pager.vnp.vnp_size > poffset) { 104200a6f47fSMatthew Dillon int pgoff; 104300a6f47fSMatthew Dillon 1044a316d390SJohn Dyson maxsize = object->un_pager.vnp.vnp_size - poffset; 1045aa8de40aSPoul-Henning Kamp ncount = btoc(maxsize); 104600a6f47fSMatthew Dillon if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 1047b7ad744dSAlan Cox vm_page_lock_queues(); 104800a6f47fSMatthew Dillon vm_page_clear_dirty(m[ncount - 1], pgoff, 104900a6f47fSMatthew Dillon PAGE_SIZE - pgoff); 1050b7ad744dSAlan Cox vm_page_unlock_queues(); 105100a6f47fSMatthew Dillon } 105200a6f47fSMatthew Dillon } else { 105300a6f47fSMatthew Dillon maxsize = 0; 105400a6f47fSMatthew Dillon ncount = 0; 105500a6f47fSMatthew Dillon } 1056f6b04d2bSDavid Greenman if (ncount < count) { 1057f6b04d2bSDavid Greenman for (i = ncount; i < count; i++) { 1058f6b04d2bSDavid Greenman rtvals[i] = VM_PAGER_BAD; 1059f6b04d2bSDavid Greenman } 1060f6b04d2bSDavid Greenman } 1061f6b04d2bSDavid Greenman } 106226f9a767SRodney W. Grimes 10632b6b0df7SMatthew Dillon /* 10642b6b0df7SMatthew Dillon * pageouts are already clustered, use IO_ASYNC t o force a bawrite() 10652b6b0df7SMatthew Dillon * rather then a bdwrite() to prevent paging I/O from saturating 106643b7990eSMatthew Dillon * the buffer cache. Dummy-up the sequential heuristic to cause 106743b7990eSMatthew Dillon * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, 106843b7990eSMatthew Dillon * the system decides how to cluster. 10692b6b0df7SMatthew Dillon */ 10708f9110f6SJohn Dyson ioflags = IO_VMIO; 107143b7990eSMatthew Dillon if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) 107243b7990eSMatthew Dillon ioflags |= IO_SYNC; 107343b7990eSMatthew Dillon else if ((flags & VM_PAGER_CLUSTER_OK) == 0) 107443b7990eSMatthew Dillon ioflags |= IO_ASYNC; 10758f9110f6SJohn Dyson ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 107643b7990eSMatthew Dillon ioflags |= IO_SEQMAX << IO_SEQSHIFT; 1077f6b04d2bSDavid Greenman 1078f6b04d2bSDavid Greenman aiov.iov_base = (caddr_t) 0; 1079f6b04d2bSDavid Greenman aiov.iov_len = maxsize; 1080f6b04d2bSDavid Greenman auio.uio_iov = &aiov; 1081f6b04d2bSDavid Greenman auio.uio_iovcnt = 1; 1082a316d390SJohn Dyson auio.uio_offset = poffset; 1083f6b04d2bSDavid Greenman auio.uio_segflg = UIO_NOCOPY; 1084f6b04d2bSDavid Greenman auio.uio_rw = UIO_WRITE; 1085f6b04d2bSDavid Greenman auio.uio_resid = maxsize; 1086b40ce416SJulian Elischer auio.uio_td = (struct thread *) 0; 1087a854ed98SJohn Baldwin error = VOP_WRITE(vp, &auio, ioflags, curthread->td_ucred); 1088976e77fcSDavid Greenman cnt.v_vnodeout++; 1089f6b04d2bSDavid Greenman cnt.v_vnodepgsout += ncount; 1090f6b04d2bSDavid Greenman 1091f6b04d2bSDavid Greenman if (error) { 109224a1cce3SDavid Greenman printf("vnode_pager_putpages: I/O error %d\n", error); 1093f6b04d2bSDavid Greenman } 1094f6b04d2bSDavid Greenman if (auio.uio_resid) { 1095ac1e407bSBruce Evans printf("vnode_pager_putpages: residual I/O %d at %lu\n", 1096ac1e407bSBruce Evans auio.uio_resid, (u_long)m[0]->pindex); 109726f9a767SRodney W. Grimes } 1098ffc82b0aSJohn Dyson for (i = 0; i < ncount; i++) { 109926f9a767SRodney W. Grimes rtvals[i] = VM_PAGER_OK; 110026f9a767SRodney W. Grimes } 1101f6b04d2bSDavid Greenman return rtvals[0]; 110226f9a767SRodney W. Grimes } 1103f6b04d2bSDavid Greenman 1104f6b04d2bSDavid Greenman struct vnode * 1105417a26a1SAlan Cox vnode_pager_lock(vm_object_t first_object) 110624a1cce3SDavid Greenman { 1107417a26a1SAlan Cox struct vnode *vp; 1108417a26a1SAlan Cox vm_object_t backing_object, object; 1109996c772fSJohn Dyson 1110417a26a1SAlan Cox VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); 1111417a26a1SAlan Cox for (object = first_object; object != NULL; object = backing_object) { 1112417a26a1SAlan Cox if (object->type != OBJT_VNODE) { 1113417a26a1SAlan Cox if ((backing_object = object->backing_object) != NULL) 1114417a26a1SAlan Cox VM_OBJECT_LOCK(backing_object); 1115417a26a1SAlan Cox if (object != first_object) 1116417a26a1SAlan Cox VM_OBJECT_UNLOCK(object); 1117f6b04d2bSDavid Greenman continue; 1118417a26a1SAlan Cox } 1119417a26a1SAlan Cox retry: 1120e6b961ffSJohn Baldwin if (object->flags & OBJ_DEAD) { 1121417a26a1SAlan Cox if (object != first_object) 1122417a26a1SAlan Cox VM_OBJECT_UNLOCK(object); 112347221757SJohn Dyson return NULL; 1124e6b961ffSJohn Baldwin } 1125417a26a1SAlan Cox vp = object->handle; 1126417a26a1SAlan Cox VI_LOCK(vp); 1127417a26a1SAlan Cox VM_OBJECT_UNLOCK(object); 1128417a26a1SAlan Cox if (first_object != object) 1129417a26a1SAlan Cox VM_OBJECT_UNLOCK(first_object); 1130417a26a1SAlan Cox if (vget(vp, LK_CANRECURSE | LK_INTERLOCK | LK_NOPAUSE | 1131417a26a1SAlan Cox LK_RETRY | LK_SHARED, curthread)) { 1132417a26a1SAlan Cox VM_OBJECT_LOCK(first_object); 1133417a26a1SAlan Cox if (object != first_object) 1134417a26a1SAlan Cox VM_OBJECT_LOCK(object); 1135417a26a1SAlan Cox if (object->type != OBJT_VNODE) { 1136417a26a1SAlan Cox if (object != first_object) 1137417a26a1SAlan Cox VM_OBJECT_UNLOCK(object); 1138bef608bdSJohn Dyson return NULL; 1139417a26a1SAlan Cox } 114047221757SJohn Dyson printf("vnode_pager_lock: retrying\n"); 1141417a26a1SAlan Cox goto retry; 114247221757SJohn Dyson } 1143417a26a1SAlan Cox VM_OBJECT_LOCK(first_object); 1144417a26a1SAlan Cox return (vp); 114526f9a767SRodney W. Grimes } 114624a1cce3SDavid Greenman return NULL; 1147f6b04d2bSDavid Greenman } 1148