1 /*- 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 1993, 1994 John S. Dyson 6 * Copyright (c) 1995, David Greenman 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41 */ 42 43 /* 44 * Page to/from files (vnodes). 45 */ 46 47 /* 48 * TODO: 49 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 50 * greatly re-simplify the vnode_pager. 51 */ 52 53 #include <sys/cdefs.h> 54 __FBSDID("$FreeBSD$"); 55 56 #include <sys/param.h> 57 #include <sys/systm.h> 58 #include <sys/proc.h> 59 #include <sys/vnode.h> 60 #include <sys/mount.h> 61 #include <sys/bio.h> 62 #include <sys/buf.h> 63 #include <sys/vmmeter.h> 64 #include <sys/limits.h> 65 #include <sys/conf.h> 66 #include <sys/rwlock.h> 67 #include <sys/sf_buf.h> 68 69 #include <machine/atomic.h> 70 71 #include <vm/vm.h> 72 #include <vm/vm_param.h> 73 #include <vm/vm_object.h> 74 #include <vm/vm_page.h> 75 #include <vm/vm_pager.h> 76 #include <vm/vm_map.h> 77 #include <vm/vnode_pager.h> 78 #include <vm/vm_extern.h> 79 80 static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, 81 daddr_t *rtaddress, int *run); 82 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); 83 static int vnode_pager_input_old(vm_object_t object, vm_page_t m); 84 static void vnode_pager_dealloc(vm_object_t); 85 static int vnode_pager_local_getpages0(struct vnode *, vm_page_t *, int, int, 86 vop_getpages_iodone_t, void *); 87 static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); 88 static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int, 89 vop_getpages_iodone_t, void *); 90 static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, int, int *); 91 static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); 92 static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 93 vm_ooffset_t, struct ucred *cred); 94 static int vnode_pager_generic_getpages_done(struct buf *); 95 static void vnode_pager_generic_getpages_done_async(struct buf *); 96 97 struct pagerops vnodepagerops = { 98 .pgo_alloc = vnode_pager_alloc, 99 .pgo_dealloc = vnode_pager_dealloc, 100 .pgo_getpages = vnode_pager_getpages, 101 .pgo_getpages_async = vnode_pager_getpages_async, 102 .pgo_putpages = vnode_pager_putpages, 103 .pgo_haspage = vnode_pager_haspage, 104 }; 105 106 int vnode_pbuf_freecnt; 107 int vnode_async_pbuf_freecnt; 108 109 /* Create the VM system backing object for this vnode */ 110 int 111 vnode_create_vobject(struct vnode *vp, off_t isize, struct thread *td) 112 { 113 vm_object_t object; 114 vm_ooffset_t size = isize; 115 struct vattr va; 116 117 if (!vn_isdisk(vp, NULL) && vn_canvmio(vp) == FALSE) 118 return (0); 119 120 while ((object = vp->v_object) != NULL) { 121 VM_OBJECT_WLOCK(object); 122 if (!(object->flags & OBJ_DEAD)) { 123 VM_OBJECT_WUNLOCK(object); 124 return (0); 125 } 126 VOP_UNLOCK(vp, 0); 127 vm_object_set_flag(object, OBJ_DISCONNECTWNT); 128 VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead", 0); 129 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 130 } 131 132 if (size == 0) { 133 if (vn_isdisk(vp, NULL)) { 134 size = IDX_TO_OFF(INT_MAX); 135 } else { 136 if (VOP_GETATTR(vp, &va, td->td_ucred)) 137 return (0); 138 size = va.va_size; 139 } 140 } 141 142 object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred); 143 /* 144 * Dereference the reference we just created. This assumes 145 * that the object is associated with the vp. 146 */ 147 VM_OBJECT_WLOCK(object); 148 object->ref_count--; 149 VM_OBJECT_WUNLOCK(object); 150 vrele(vp); 151 152 KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object")); 153 154 return (0); 155 } 156 157 void 158 vnode_destroy_vobject(struct vnode *vp) 159 { 160 struct vm_object *obj; 161 162 obj = vp->v_object; 163 if (obj == NULL) 164 return; 165 ASSERT_VOP_ELOCKED(vp, "vnode_destroy_vobject"); 166 VM_OBJECT_WLOCK(obj); 167 if (obj->ref_count == 0) { 168 /* 169 * don't double-terminate the object 170 */ 171 if ((obj->flags & OBJ_DEAD) == 0) 172 vm_object_terminate(obj); 173 else 174 VM_OBJECT_WUNLOCK(obj); 175 } else { 176 /* 177 * Woe to the process that tries to page now :-). 178 */ 179 vm_pager_deallocate(obj); 180 VM_OBJECT_WUNLOCK(obj); 181 } 182 vp->v_object = NULL; 183 } 184 185 186 /* 187 * Allocate (or lookup) pager for a vnode. 188 * Handle is a vnode pointer. 189 * 190 * MPSAFE 191 */ 192 vm_object_t 193 vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 194 vm_ooffset_t offset, struct ucred *cred) 195 { 196 vm_object_t object; 197 struct vnode *vp; 198 199 /* 200 * Pageout to vnode, no can do yet. 201 */ 202 if (handle == NULL) 203 return (NULL); 204 205 vp = (struct vnode *) handle; 206 207 /* 208 * If the object is being terminated, wait for it to 209 * go away. 210 */ 211 retry: 212 while ((object = vp->v_object) != NULL) { 213 VM_OBJECT_WLOCK(object); 214 if ((object->flags & OBJ_DEAD) == 0) 215 break; 216 vm_object_set_flag(object, OBJ_DISCONNECTWNT); 217 VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0); 218 } 219 220 KASSERT(vp->v_usecount != 0, ("vnode_pager_alloc: no vnode reference")); 221 222 if (object == NULL) { 223 /* 224 * Add an object of the appropriate size 225 */ 226 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 227 228 object->un_pager.vnp.vnp_size = size; 229 object->un_pager.vnp.writemappings = 0; 230 231 object->handle = handle; 232 VI_LOCK(vp); 233 if (vp->v_object != NULL) { 234 /* 235 * Object has been created while we were sleeping 236 */ 237 VI_UNLOCK(vp); 238 vm_object_destroy(object); 239 goto retry; 240 } 241 vp->v_object = object; 242 VI_UNLOCK(vp); 243 } else { 244 object->ref_count++; 245 VM_OBJECT_WUNLOCK(object); 246 } 247 vref(vp); 248 return (object); 249 } 250 251 /* 252 * The object must be locked. 253 */ 254 static void 255 vnode_pager_dealloc(vm_object_t object) 256 { 257 struct vnode *vp; 258 int refs; 259 260 vp = object->handle; 261 if (vp == NULL) 262 panic("vnode_pager_dealloc: pager already dealloced"); 263 264 VM_OBJECT_ASSERT_WLOCKED(object); 265 vm_object_pip_wait(object, "vnpdea"); 266 refs = object->ref_count; 267 268 object->handle = NULL; 269 object->type = OBJT_DEAD; 270 if (object->flags & OBJ_DISCONNECTWNT) { 271 vm_object_clear_flag(object, OBJ_DISCONNECTWNT); 272 wakeup(object); 273 } 274 ASSERT_VOP_ELOCKED(vp, "vnode_pager_dealloc"); 275 if (object->un_pager.vnp.writemappings > 0) { 276 object->un_pager.vnp.writemappings = 0; 277 VOP_ADD_WRITECOUNT(vp, -1); 278 CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d", 279 __func__, vp, vp->v_writecount); 280 } 281 vp->v_object = NULL; 282 VOP_UNSET_TEXT(vp); 283 VM_OBJECT_WUNLOCK(object); 284 while (refs-- > 0) 285 vunref(vp); 286 VM_OBJECT_WLOCK(object); 287 } 288 289 static boolean_t 290 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 291 int *after) 292 { 293 struct vnode *vp = object->handle; 294 daddr_t bn; 295 int err; 296 daddr_t reqblock; 297 int poff; 298 int bsize; 299 int pagesperblock, blocksperpage; 300 301 VM_OBJECT_ASSERT_WLOCKED(object); 302 /* 303 * If no vp or vp is doomed or marked transparent to VM, we do not 304 * have the page. 305 */ 306 if (vp == NULL || vp->v_iflag & VI_DOOMED) 307 return FALSE; 308 /* 309 * If the offset is beyond end of file we do 310 * not have the page. 311 */ 312 if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size) 313 return FALSE; 314 315 bsize = vp->v_mount->mnt_stat.f_iosize; 316 pagesperblock = bsize / PAGE_SIZE; 317 blocksperpage = 0; 318 if (pagesperblock > 0) { 319 reqblock = pindex / pagesperblock; 320 } else { 321 blocksperpage = (PAGE_SIZE / bsize); 322 reqblock = pindex * blocksperpage; 323 } 324 VM_OBJECT_WUNLOCK(object); 325 err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before); 326 VM_OBJECT_WLOCK(object); 327 if (err) 328 return TRUE; 329 if (bn == -1) 330 return FALSE; 331 if (pagesperblock > 0) { 332 poff = pindex - (reqblock * pagesperblock); 333 if (before) { 334 *before *= pagesperblock; 335 *before += poff; 336 } 337 if (after) { 338 int numafter; 339 *after *= pagesperblock; 340 numafter = pagesperblock - (poff + 1); 341 if (IDX_TO_OFF(pindex + numafter) > 342 object->un_pager.vnp.vnp_size) { 343 numafter = 344 OFF_TO_IDX(object->un_pager.vnp.vnp_size) - 345 pindex; 346 } 347 *after += numafter; 348 } 349 } else { 350 if (before) { 351 *before /= blocksperpage; 352 } 353 354 if (after) { 355 *after /= blocksperpage; 356 } 357 } 358 return TRUE; 359 } 360 361 /* 362 * Lets the VM system know about a change in size for a file. 363 * We adjust our own internal size and flush any cached pages in 364 * the associated object that are affected by the size change. 365 * 366 * Note: this routine may be invoked as a result of a pager put 367 * operation (possibly at object termination time), so we must be careful. 368 */ 369 void 370 vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize) 371 { 372 vm_object_t object; 373 vm_page_t m; 374 vm_pindex_t nobjsize; 375 376 if ((object = vp->v_object) == NULL) 377 return; 378 /* ASSERT_VOP_ELOCKED(vp, "vnode_pager_setsize and not locked vnode"); */ 379 VM_OBJECT_WLOCK(object); 380 if (object->type == OBJT_DEAD) { 381 VM_OBJECT_WUNLOCK(object); 382 return; 383 } 384 KASSERT(object->type == OBJT_VNODE, 385 ("not vnode-backed object %p", object)); 386 if (nsize == object->un_pager.vnp.vnp_size) { 387 /* 388 * Hasn't changed size 389 */ 390 VM_OBJECT_WUNLOCK(object); 391 return; 392 } 393 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 394 if (nsize < object->un_pager.vnp.vnp_size) { 395 /* 396 * File has shrunk. Toss any cached pages beyond the new EOF. 397 */ 398 if (nobjsize < object->size) 399 vm_object_page_remove(object, nobjsize, object->size, 400 0); 401 /* 402 * this gets rid of garbage at the end of a page that is now 403 * only partially backed by the vnode. 404 * 405 * XXX for some reason (I don't know yet), if we take a 406 * completely invalid page and mark it partially valid 407 * it can screw up NFS reads, so we don't allow the case. 408 */ 409 if ((nsize & PAGE_MASK) && 410 (m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL && 411 m->valid != 0) { 412 int base = (int)nsize & PAGE_MASK; 413 int size = PAGE_SIZE - base; 414 415 /* 416 * Clear out partial-page garbage in case 417 * the page has been mapped. 418 */ 419 pmap_zero_page_area(m, base, size); 420 421 /* 422 * Update the valid bits to reflect the blocks that 423 * have been zeroed. Some of these valid bits may 424 * have already been set. 425 */ 426 vm_page_set_valid_range(m, base, size); 427 428 /* 429 * Round "base" to the next block boundary so that the 430 * dirty bit for a partially zeroed block is not 431 * cleared. 432 */ 433 base = roundup2(base, DEV_BSIZE); 434 435 /* 436 * Clear out partial-page dirty bits. 437 * 438 * note that we do not clear out the valid 439 * bits. This would prevent bogus_page 440 * replacement from working properly. 441 */ 442 vm_page_clear_dirty(m, base, PAGE_SIZE - base); 443 } else if ((nsize & PAGE_MASK) && 444 vm_page_is_cached(object, OFF_TO_IDX(nsize))) { 445 vm_page_cache_free(object, OFF_TO_IDX(nsize), 446 nobjsize); 447 } 448 } 449 object->un_pager.vnp.vnp_size = nsize; 450 object->size = nobjsize; 451 VM_OBJECT_WUNLOCK(object); 452 } 453 454 /* 455 * calculate the linear (byte) disk address of specified virtual 456 * file address 457 */ 458 static int 459 vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, daddr_t *rtaddress, 460 int *run) 461 { 462 int bsize; 463 int err; 464 daddr_t vblock; 465 daddr_t voffset; 466 467 if (address < 0) 468 return -1; 469 470 if (vp->v_iflag & VI_DOOMED) 471 return -1; 472 473 bsize = vp->v_mount->mnt_stat.f_iosize; 474 vblock = address / bsize; 475 voffset = address % bsize; 476 477 err = VOP_BMAP(vp, vblock, NULL, rtaddress, run, NULL); 478 if (err == 0) { 479 if (*rtaddress != -1) 480 *rtaddress += voffset / DEV_BSIZE; 481 if (run) { 482 *run += 1; 483 *run *= bsize/PAGE_SIZE; 484 *run -= voffset/PAGE_SIZE; 485 } 486 } 487 488 return (err); 489 } 490 491 /* 492 * small block filesystem vnode pager input 493 */ 494 static int 495 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m) 496 { 497 struct vnode *vp; 498 struct bufobj *bo; 499 struct buf *bp; 500 struct sf_buf *sf; 501 daddr_t fileaddr; 502 vm_offset_t bsize; 503 vm_page_bits_t bits; 504 int error, i; 505 506 error = 0; 507 vp = object->handle; 508 if (vp->v_iflag & VI_DOOMED) 509 return VM_PAGER_BAD; 510 511 bsize = vp->v_mount->mnt_stat.f_iosize; 512 513 VOP_BMAP(vp, 0, &bo, 0, NULL, NULL); 514 515 sf = sf_buf_alloc(m, 0); 516 517 for (i = 0; i < PAGE_SIZE / bsize; i++) { 518 vm_ooffset_t address; 519 520 bits = vm_page_bits(i * bsize, bsize); 521 if (m->valid & bits) 522 continue; 523 524 address = IDX_TO_OFF(m->pindex) + i * bsize; 525 if (address >= object->un_pager.vnp.vnp_size) { 526 fileaddr = -1; 527 } else { 528 error = vnode_pager_addr(vp, address, &fileaddr, NULL); 529 if (error) 530 break; 531 } 532 if (fileaddr != -1) { 533 bp = getpbuf(&vnode_pbuf_freecnt); 534 535 /* build a minimal buffer header */ 536 bp->b_iocmd = BIO_READ; 537 bp->b_iodone = bdone; 538 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 539 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 540 bp->b_rcred = crhold(curthread->td_ucred); 541 bp->b_wcred = crhold(curthread->td_ucred); 542 bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize; 543 bp->b_blkno = fileaddr; 544 pbgetbo(bo, bp); 545 bp->b_vp = vp; 546 bp->b_bcount = bsize; 547 bp->b_bufsize = bsize; 548 bp->b_runningbufspace = bp->b_bufsize; 549 atomic_add_long(&runningbufspace, bp->b_runningbufspace); 550 551 /* do the input */ 552 bp->b_iooffset = dbtob(bp->b_blkno); 553 bstrategy(bp); 554 555 bwait(bp, PVM, "vnsrd"); 556 557 if ((bp->b_ioflags & BIO_ERROR) != 0) 558 error = EIO; 559 560 /* 561 * free the buffer header back to the swap buffer pool 562 */ 563 bp->b_vp = NULL; 564 pbrelbo(bp); 565 relpbuf(bp, &vnode_pbuf_freecnt); 566 if (error) 567 break; 568 } else 569 bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize); 570 KASSERT((m->dirty & bits) == 0, 571 ("vnode_pager_input_smlfs: page %p is dirty", m)); 572 VM_OBJECT_WLOCK(object); 573 m->valid |= bits; 574 VM_OBJECT_WUNLOCK(object); 575 } 576 sf_buf_free(sf); 577 if (error) { 578 return VM_PAGER_ERROR; 579 } 580 return VM_PAGER_OK; 581 } 582 583 /* 584 * old style vnode pager input routine 585 */ 586 static int 587 vnode_pager_input_old(vm_object_t object, vm_page_t m) 588 { 589 struct uio auio; 590 struct iovec aiov; 591 int error; 592 int size; 593 struct sf_buf *sf; 594 struct vnode *vp; 595 596 VM_OBJECT_ASSERT_WLOCKED(object); 597 error = 0; 598 599 /* 600 * Return failure if beyond current EOF 601 */ 602 if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { 603 return VM_PAGER_BAD; 604 } else { 605 size = PAGE_SIZE; 606 if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) 607 size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); 608 vp = object->handle; 609 VM_OBJECT_WUNLOCK(object); 610 611 /* 612 * Allocate a kernel virtual address and initialize so that 613 * we can use VOP_READ/WRITE routines. 614 */ 615 sf = sf_buf_alloc(m, 0); 616 617 aiov.iov_base = (caddr_t)sf_buf_kva(sf); 618 aiov.iov_len = size; 619 auio.uio_iov = &aiov; 620 auio.uio_iovcnt = 1; 621 auio.uio_offset = IDX_TO_OFF(m->pindex); 622 auio.uio_segflg = UIO_SYSSPACE; 623 auio.uio_rw = UIO_READ; 624 auio.uio_resid = size; 625 auio.uio_td = curthread; 626 627 error = VOP_READ(vp, &auio, 0, curthread->td_ucred); 628 if (!error) { 629 int count = size - auio.uio_resid; 630 631 if (count == 0) 632 error = EINVAL; 633 else if (count != PAGE_SIZE) 634 bzero((caddr_t)sf_buf_kva(sf) + count, 635 PAGE_SIZE - count); 636 } 637 sf_buf_free(sf); 638 639 VM_OBJECT_WLOCK(object); 640 } 641 KASSERT(m->dirty == 0, ("vnode_pager_input_old: page %p is dirty", m)); 642 if (!error) 643 m->valid = VM_PAGE_BITS_ALL; 644 return error ? VM_PAGER_ERROR : VM_PAGER_OK; 645 } 646 647 /* 648 * generic vnode pager input routine 649 */ 650 651 /* 652 * Local media VFS's that do not implement their own VOP_GETPAGES 653 * should have their VOP_GETPAGES call to vnode_pager_generic_getpages() 654 * to implement the previous behaviour. 655 * 656 * All other FS's should use the bypass to get to the local media 657 * backing vp's VOP_GETPAGES. 658 */ 659 static int 660 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) 661 { 662 int rtval; 663 struct vnode *vp; 664 int bytes = count * PAGE_SIZE; 665 666 vp = object->handle; 667 VM_OBJECT_WUNLOCK(object); 668 rtval = VOP_GETPAGES(vp, m, bytes, reqpage); 669 KASSERT(rtval != EOPNOTSUPP, 670 ("vnode_pager: FS getpages not implemented\n")); 671 VM_OBJECT_WLOCK(object); 672 return rtval; 673 } 674 675 static int 676 vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, 677 int reqpage, vop_getpages_iodone_t iodone, void *arg) 678 { 679 struct vnode *vp; 680 int rtval; 681 682 vp = object->handle; 683 VM_OBJECT_WUNLOCK(object); 684 rtval = VOP_GETPAGES_ASYNC(vp, m, count * PAGE_SIZE, reqpage, 0, 685 iodone, arg); 686 KASSERT(rtval != EOPNOTSUPP, 687 ("vnode_pager: FS getpages_async not implemented\n")); 688 VM_OBJECT_WLOCK(object); 689 return (rtval); 690 } 691 692 /* 693 * The implementation of VOP_GETPAGES() and VOP_GETPAGES_ASYNC() for 694 * local filesystems, where partially valid pages can only occur at 695 * the end of file. 696 */ 697 int 698 vnode_pager_local_getpages(struct vop_getpages_args *ap) 699 { 700 701 return (vnode_pager_local_getpages0(ap->a_vp, ap->a_m, ap->a_count, 702 ap->a_reqpage, NULL, NULL)); 703 } 704 705 int 706 vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap) 707 { 708 709 return (vnode_pager_local_getpages0(ap->a_vp, ap->a_m, ap->a_count, 710 ap->a_reqpage, ap->a_iodone, ap->a_arg)); 711 } 712 713 static int 714 vnode_pager_local_getpages0(struct vnode *vp, vm_page_t *m, int bytecount, 715 int reqpage, vop_getpages_iodone_t iodone, void *arg) 716 { 717 vm_page_t mreq; 718 719 mreq = m[reqpage]; 720 721 /* 722 * Since the caller has busied the requested page, that page's valid 723 * field will not be changed by other threads. 724 */ 725 vm_page_assert_xbusied(mreq); 726 727 /* 728 * The requested page has valid blocks. Invalid part can only 729 * exist at the end of file, and the page is made fully valid 730 * by zeroing in vm_pager_get_pages(). Free non-requested 731 * pages, since no i/o is done to read its content. 732 */ 733 if (mreq->valid != 0) { 734 vm_pager_free_nonreq(mreq->object, m, reqpage, 735 round_page(bytecount) / PAGE_SIZE); 736 if (iodone != NULL) 737 iodone(arg, m, reqpage, 0); 738 return (VM_PAGER_OK); 739 } 740 741 return (vnode_pager_generic_getpages(vp, m, bytecount, reqpage, 742 iodone, arg)); 743 } 744 745 /* 746 * This is now called from local media FS's to operate against their 747 * own vnodes if they fail to implement VOP_GETPAGES. 748 */ 749 int 750 vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, 751 int reqpage, vop_getpages_iodone_t iodone, void *arg) 752 { 753 vm_object_t object; 754 off_t foff; 755 int i, j, size, bsize, first, *freecnt; 756 daddr_t firstaddr, reqblock; 757 struct bufobj *bo; 758 int runpg; 759 int runend; 760 struct buf *bp; 761 int count; 762 int error; 763 764 object = vp->v_object; 765 count = bytecount / PAGE_SIZE; 766 767 KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, 768 ("vnode_pager_generic_getpages does not support devices")); 769 if (vp->v_iflag & VI_DOOMED) 770 return VM_PAGER_BAD; 771 772 bsize = vp->v_mount->mnt_stat.f_iosize; 773 foff = IDX_TO_OFF(m[reqpage]->pindex); 774 775 freecnt = iodone != NULL ? 776 &vnode_async_pbuf_freecnt : &vnode_pbuf_freecnt; 777 bp = getpbuf(freecnt); 778 779 /* 780 * Get the underlying device blocks for the file with VOP_BMAP(). 781 * If the file system doesn't support VOP_BMAP, use old way of 782 * getting pages via VOP_READ. 783 */ 784 error = VOP_BMAP(vp, foff / bsize, &bo, &reqblock, NULL, NULL); 785 if (error == EOPNOTSUPP) { 786 relpbuf(bp, freecnt); 787 VM_OBJECT_WLOCK(object); 788 for (i = 0; i < count; i++) 789 if (i != reqpage) { 790 vm_page_lock(m[i]); 791 vm_page_free(m[i]); 792 vm_page_unlock(m[i]); 793 } 794 PCPU_INC(cnt.v_vnodein); 795 PCPU_INC(cnt.v_vnodepgsin); 796 error = vnode_pager_input_old(object, m[reqpage]); 797 VM_OBJECT_WUNLOCK(object); 798 return (error); 799 } else if (error != 0) { 800 relpbuf(bp, freecnt); 801 vm_pager_free_nonreq(object, m, reqpage, count); 802 return (VM_PAGER_ERROR); 803 804 /* 805 * if the blocksize is smaller than a page size, then use 806 * special small filesystem code. NFS sometimes has a small 807 * blocksize, but it can handle large reads itself. 808 */ 809 } else if ((PAGE_SIZE / bsize) > 1 && 810 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 811 relpbuf(bp, freecnt); 812 vm_pager_free_nonreq(object, m, reqpage, count); 813 PCPU_INC(cnt.v_vnodein); 814 PCPU_INC(cnt.v_vnodepgsin); 815 return vnode_pager_input_smlfs(object, m[reqpage]); 816 } 817 818 /* 819 * Since the caller has busied the requested page, that page's valid 820 * field will not be changed by other threads. 821 */ 822 vm_page_assert_xbusied(m[reqpage]); 823 824 /* 825 * If we have a completely valid page available to us, we can 826 * clean up and return. Otherwise we have to re-read the 827 * media. 828 */ 829 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 830 relpbuf(bp, freecnt); 831 vm_pager_free_nonreq(object, m, reqpage, count); 832 return (VM_PAGER_OK); 833 } else if (reqblock == -1) { 834 relpbuf(bp, freecnt); 835 pmap_zero_page(m[reqpage]); 836 KASSERT(m[reqpage]->dirty == 0, 837 ("vnode_pager_generic_getpages: page %p is dirty", m)); 838 VM_OBJECT_WLOCK(object); 839 m[reqpage]->valid = VM_PAGE_BITS_ALL; 840 for (i = 0; i < count; i++) 841 if (i != reqpage) { 842 vm_page_lock(m[i]); 843 vm_page_free(m[i]); 844 vm_page_unlock(m[i]); 845 } 846 VM_OBJECT_WUNLOCK(object); 847 return (VM_PAGER_OK); 848 } else if (m[reqpage]->valid != 0) { 849 VM_OBJECT_WLOCK(object); 850 m[reqpage]->valid = 0; 851 VM_OBJECT_WUNLOCK(object); 852 } 853 854 /* 855 * here on direct device I/O 856 */ 857 firstaddr = -1; 858 859 /* 860 * calculate the run that includes the required page 861 */ 862 for (first = 0, i = 0; i < count; i = runend) { 863 if (vnode_pager_addr(vp, IDX_TO_OFF(m[i]->pindex), &firstaddr, 864 &runpg) != 0) { 865 relpbuf(bp, freecnt); 866 VM_OBJECT_WLOCK(object); 867 for (; i < count; i++) 868 if (i != reqpage) { 869 vm_page_lock(m[i]); 870 vm_page_free(m[i]); 871 vm_page_unlock(m[i]); 872 } 873 VM_OBJECT_WUNLOCK(object); 874 return (VM_PAGER_ERROR); 875 } 876 if (firstaddr == -1) { 877 VM_OBJECT_WLOCK(object); 878 if (i == reqpage && foff < object->un_pager.vnp.vnp_size) { 879 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %jd, foff: 0x%jx%08jx, vnp_size: 0x%jx%08jx", 880 (intmax_t)firstaddr, (uintmax_t)(foff >> 32), 881 (uintmax_t)foff, 882 (uintmax_t) 883 (object->un_pager.vnp.vnp_size >> 32), 884 (uintmax_t)object->un_pager.vnp.vnp_size); 885 } 886 vm_page_lock(m[i]); 887 vm_page_free(m[i]); 888 vm_page_unlock(m[i]); 889 VM_OBJECT_WUNLOCK(object); 890 runend = i + 1; 891 first = runend; 892 continue; 893 } 894 runend = i + runpg; 895 if (runend <= reqpage) { 896 VM_OBJECT_WLOCK(object); 897 for (j = i; j < runend; j++) { 898 vm_page_lock(m[j]); 899 vm_page_free(m[j]); 900 vm_page_unlock(m[j]); 901 } 902 VM_OBJECT_WUNLOCK(object); 903 } else { 904 if (runpg < (count - first)) { 905 VM_OBJECT_WLOCK(object); 906 for (i = first + runpg; i < count; i++) { 907 vm_page_lock(m[i]); 908 vm_page_free(m[i]); 909 vm_page_unlock(m[i]); 910 } 911 VM_OBJECT_WUNLOCK(object); 912 count = first + runpg; 913 } 914 break; 915 } 916 first = runend; 917 } 918 919 /* 920 * the first and last page have been calculated now, move input pages 921 * to be zero based... 922 */ 923 if (first != 0) { 924 m += first; 925 count -= first; 926 reqpage -= first; 927 } 928 929 /* 930 * calculate the file virtual address for the transfer 931 */ 932 foff = IDX_TO_OFF(m[0]->pindex); 933 934 /* 935 * calculate the size of the transfer 936 */ 937 size = count * PAGE_SIZE; 938 KASSERT(count > 0, ("zero count")); 939 if ((foff + size) > object->un_pager.vnp.vnp_size) 940 size = object->un_pager.vnp.vnp_size - foff; 941 KASSERT(size > 0, ("zero size")); 942 943 /* 944 * round up physical size for real devices. 945 */ 946 if (1) { 947 int secmask = bo->bo_bsize - 1; 948 KASSERT(secmask < PAGE_SIZE && secmask > 0, 949 ("vnode_pager_generic_getpages: sector size %d too large", 950 secmask + 1)); 951 size = (size + secmask) & ~secmask; 952 } 953 954 bp->b_kvaalloc = bp->b_data; 955 956 /* 957 * and map the pages to be read into the kva, if the filesystem 958 * requires mapped buffers. 959 */ 960 if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 && 961 unmapped_buf_allowed) { 962 bp->b_data = unmapped_buf; 963 bp->b_kvabase = unmapped_buf; 964 bp->b_offset = 0; 965 bp->b_flags |= B_UNMAPPED; 966 } else 967 pmap_qenter((vm_offset_t)bp->b_kvaalloc, m, count); 968 969 /* build a minimal buffer header */ 970 bp->b_iocmd = BIO_READ; 971 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 972 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 973 bp->b_rcred = crhold(curthread->td_ucred); 974 bp->b_wcred = crhold(curthread->td_ucred); 975 bp->b_blkno = firstaddr; 976 pbgetbo(bo, bp); 977 bp->b_vp = vp; 978 bp->b_bcount = size; 979 bp->b_bufsize = size; 980 bp->b_runningbufspace = bp->b_bufsize; 981 for (i = 0; i < count; i++) 982 bp->b_pages[i] = m[i]; 983 bp->b_npages = count; 984 bp->b_pager.pg_reqpage = reqpage; 985 atomic_add_long(&runningbufspace, bp->b_runningbufspace); 986 987 PCPU_INC(cnt.v_vnodein); 988 PCPU_ADD(cnt.v_vnodepgsin, count); 989 990 /* do the input */ 991 bp->b_iooffset = dbtob(bp->b_blkno); 992 993 if (iodone != NULL) { /* async */ 994 bp->b_pager.pg_iodone = iodone; 995 bp->b_caller1 = arg; 996 bp->b_iodone = vnode_pager_generic_getpages_done_async; 997 bp->b_flags |= B_ASYNC; 998 BUF_KERNPROC(bp); 999 bstrategy(bp); 1000 /* Good bye! */ 1001 } else { 1002 bp->b_iodone = bdone; 1003 bstrategy(bp); 1004 bwait(bp, PVM, "vnread"); 1005 error = vnode_pager_generic_getpages_done(bp); 1006 for (i = 0; i < bp->b_npages; i++) 1007 bp->b_pages[i] = NULL; 1008 bp->b_vp = NULL; 1009 pbrelbo(bp); 1010 relpbuf(bp, &vnode_pbuf_freecnt); 1011 } 1012 1013 return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK); 1014 } 1015 1016 static void 1017 vnode_pager_generic_getpages_done_async(struct buf *bp) 1018 { 1019 int error; 1020 1021 error = vnode_pager_generic_getpages_done(bp); 1022 bp->b_pager.pg_iodone(bp->b_caller1, bp->b_pages, 1023 bp->b_pager.pg_reqpage, error); 1024 for (int i = 0; i < bp->b_npages; i++) 1025 bp->b_pages[i] = NULL; 1026 bp->b_vp = NULL; 1027 pbrelbo(bp); 1028 relpbuf(bp, &vnode_async_pbuf_freecnt); 1029 } 1030 1031 static int 1032 vnode_pager_generic_getpages_done(struct buf *bp) 1033 { 1034 vm_object_t object; 1035 off_t tfoff, nextoff; 1036 int i, error; 1037 1038 error = (bp->b_ioflags & BIO_ERROR) != 0 ? EIO : 0; 1039 object = bp->b_vp->v_object; 1040 1041 if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) { 1042 if ((bp->b_flags & B_UNMAPPED) != 0) { 1043 bp->b_flags &= ~B_UNMAPPED; 1044 pmap_qenter((vm_offset_t)bp->b_kvaalloc, bp->b_pages, 1045 bp->b_npages); 1046 } 1047 bzero(bp->b_kvaalloc + bp->b_bcount, 1048 PAGE_SIZE * bp->b_npages - bp->b_bcount); 1049 } 1050 if ((bp->b_flags & B_UNMAPPED) == 0) 1051 pmap_qremove((vm_offset_t)bp->b_kvaalloc, bp->b_npages); 1052 if ((bp->b_vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) { 1053 bp->b_data = bp->b_kvaalloc; 1054 bp->b_kvabase = bp->b_kvaalloc; 1055 bp->b_flags &= ~B_UNMAPPED; 1056 } 1057 1058 VM_OBJECT_WLOCK(object); 1059 for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex); 1060 i < bp->b_npages; i++, tfoff = nextoff) { 1061 vm_page_t mt; 1062 1063 nextoff = tfoff + PAGE_SIZE; 1064 mt = bp->b_pages[i]; 1065 1066 if (nextoff <= object->un_pager.vnp.vnp_size) { 1067 /* 1068 * Read filled up entire page. 1069 */ 1070 mt->valid = VM_PAGE_BITS_ALL; 1071 KASSERT(mt->dirty == 0, 1072 ("%s: page %p is dirty", __func__, mt)); 1073 KASSERT(!pmap_page_is_mapped(mt), 1074 ("%s: page %p is mapped", __func__, mt)); 1075 } else { 1076 /* 1077 * Read did not fill up entire page. 1078 * 1079 * Currently we do not set the entire page valid, 1080 * we just try to clear the piece that we couldn't 1081 * read. 1082 */ 1083 vm_page_set_valid_range(mt, 0, 1084 object->un_pager.vnp.vnp_size - tfoff); 1085 KASSERT((mt->dirty & vm_page_bits(0, 1086 object->un_pager.vnp.vnp_size - tfoff)) == 0, 1087 ("%s: page %p is dirty", __func__, mt)); 1088 } 1089 1090 if (i != bp->b_pager.pg_reqpage) 1091 vm_page_readahead_finish(mt); 1092 } 1093 VM_OBJECT_WUNLOCK(object); 1094 if (error != 0) 1095 printf("%s: I/O read error %d\n", __func__, error); 1096 1097 return (error); 1098 } 1099 1100 /* 1101 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 1102 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 1103 * vnode_pager_generic_putpages() to implement the previous behaviour. 1104 * 1105 * All other FS's should use the bypass to get to the local media 1106 * backing vp's VOP_PUTPAGES. 1107 */ 1108 static void 1109 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count, 1110 int flags, int *rtvals) 1111 { 1112 int rtval; 1113 struct vnode *vp; 1114 int bytes = count * PAGE_SIZE; 1115 1116 /* 1117 * Force synchronous operation if we are extremely low on memory 1118 * to prevent a low-memory deadlock. VOP operations often need to 1119 * allocate more memory to initiate the I/O ( i.e. do a BMAP 1120 * operation ). The swapper handles the case by limiting the amount 1121 * of asynchronous I/O, but that sort of solution doesn't scale well 1122 * for the vnode pager without a lot of work. 1123 * 1124 * Also, the backing vnode's iodone routine may not wake the pageout 1125 * daemon up. This should be probably be addressed XXX. 1126 */ 1127 1128 if (vm_cnt.v_free_count + vm_cnt.v_cache_count < 1129 vm_cnt.v_pageout_free_min) 1130 flags |= VM_PAGER_PUT_SYNC; 1131 1132 /* 1133 * Call device-specific putpages function 1134 */ 1135 vp = object->handle; 1136 VM_OBJECT_WUNLOCK(object); 1137 rtval = VOP_PUTPAGES(vp, m, bytes, flags, rtvals); 1138 KASSERT(rtval != EOPNOTSUPP, 1139 ("vnode_pager: stale FS putpages\n")); 1140 VM_OBJECT_WLOCK(object); 1141 } 1142 1143 1144 /* 1145 * This is now called from local media FS's to operate against their 1146 * own vnodes if they fail to implement VOP_PUTPAGES. 1147 * 1148 * This is typically called indirectly via the pageout daemon and 1149 * clustering has already typically occured, so in general we ask the 1150 * underlying filesystem to write the data out asynchronously rather 1151 * then delayed. 1152 */ 1153 int 1154 vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount, 1155 int flags, int *rtvals) 1156 { 1157 int i; 1158 vm_object_t object; 1159 vm_page_t m; 1160 int count; 1161 1162 int maxsize, ncount; 1163 vm_ooffset_t poffset; 1164 struct uio auio; 1165 struct iovec aiov; 1166 int error; 1167 int ioflags; 1168 int ppscheck = 0; 1169 static struct timeval lastfail; 1170 static int curfail; 1171 1172 object = vp->v_object; 1173 count = bytecount / PAGE_SIZE; 1174 1175 for (i = 0; i < count; i++) 1176 rtvals[i] = VM_PAGER_ERROR; 1177 1178 if ((int64_t)ma[0]->pindex < 0) { 1179 printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%lx)\n", 1180 (long)ma[0]->pindex, (u_long)ma[0]->dirty); 1181 rtvals[0] = VM_PAGER_BAD; 1182 return VM_PAGER_BAD; 1183 } 1184 1185 maxsize = count * PAGE_SIZE; 1186 ncount = count; 1187 1188 poffset = IDX_TO_OFF(ma[0]->pindex); 1189 1190 /* 1191 * If the page-aligned write is larger then the actual file we 1192 * have to invalidate pages occuring beyond the file EOF. However, 1193 * there is an edge case where a file may not be page-aligned where 1194 * the last page is partially invalid. In this case the filesystem 1195 * may not properly clear the dirty bits for the entire page (which 1196 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 1197 * With the page locked we are free to fix-up the dirty bits here. 1198 * 1199 * We do not under any circumstances truncate the valid bits, as 1200 * this will screw up bogus page replacement. 1201 */ 1202 VM_OBJECT_WLOCK(object); 1203 if (maxsize + poffset > object->un_pager.vnp.vnp_size) { 1204 if (object->un_pager.vnp.vnp_size > poffset) { 1205 int pgoff; 1206 1207 maxsize = object->un_pager.vnp.vnp_size - poffset; 1208 ncount = btoc(maxsize); 1209 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 1210 /* 1211 * If the object is locked and the following 1212 * conditions hold, then the page's dirty 1213 * field cannot be concurrently changed by a 1214 * pmap operation. 1215 */ 1216 m = ma[ncount - 1]; 1217 vm_page_assert_sbusied(m); 1218 KASSERT(!pmap_page_is_write_mapped(m), 1219 ("vnode_pager_generic_putpages: page %p is not read-only", m)); 1220 vm_page_clear_dirty(m, pgoff, PAGE_SIZE - 1221 pgoff); 1222 } 1223 } else { 1224 maxsize = 0; 1225 ncount = 0; 1226 } 1227 if (ncount < count) { 1228 for (i = ncount; i < count; i++) { 1229 rtvals[i] = VM_PAGER_BAD; 1230 } 1231 } 1232 } 1233 VM_OBJECT_WUNLOCK(object); 1234 1235 /* 1236 * pageouts are already clustered, use IO_ASYNC to force a bawrite() 1237 * rather then a bdwrite() to prevent paging I/O from saturating 1238 * the buffer cache. Dummy-up the sequential heuristic to cause 1239 * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, 1240 * the system decides how to cluster. 1241 */ 1242 ioflags = IO_VMIO; 1243 if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) 1244 ioflags |= IO_SYNC; 1245 else if ((flags & VM_PAGER_CLUSTER_OK) == 0) 1246 ioflags |= IO_ASYNC; 1247 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 1248 ioflags |= IO_SEQMAX << IO_SEQSHIFT; 1249 1250 aiov.iov_base = (caddr_t) 0; 1251 aiov.iov_len = maxsize; 1252 auio.uio_iov = &aiov; 1253 auio.uio_iovcnt = 1; 1254 auio.uio_offset = poffset; 1255 auio.uio_segflg = UIO_NOCOPY; 1256 auio.uio_rw = UIO_WRITE; 1257 auio.uio_resid = maxsize; 1258 auio.uio_td = (struct thread *) 0; 1259 error = VOP_WRITE(vp, &auio, ioflags, curthread->td_ucred); 1260 PCPU_INC(cnt.v_vnodeout); 1261 PCPU_ADD(cnt.v_vnodepgsout, ncount); 1262 1263 if (error) { 1264 if ((ppscheck = ppsratecheck(&lastfail, &curfail, 1))) 1265 printf("vnode_pager_putpages: I/O error %d\n", error); 1266 } 1267 if (auio.uio_resid) { 1268 if (ppscheck || ppsratecheck(&lastfail, &curfail, 1)) 1269 printf("vnode_pager_putpages: residual I/O %zd at %lu\n", 1270 auio.uio_resid, (u_long)ma[0]->pindex); 1271 } 1272 for (i = 0; i < ncount; i++) { 1273 rtvals[i] = VM_PAGER_OK; 1274 } 1275 return rtvals[0]; 1276 } 1277 1278 void 1279 vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written) 1280 { 1281 vm_object_t obj; 1282 int i, pos; 1283 1284 if (written == 0) 1285 return; 1286 obj = ma[0]->object; 1287 VM_OBJECT_WLOCK(obj); 1288 for (i = 0, pos = 0; pos < written; i++, pos += PAGE_SIZE) { 1289 if (pos < trunc_page(written)) { 1290 rtvals[i] = VM_PAGER_OK; 1291 vm_page_undirty(ma[i]); 1292 } else { 1293 /* Partially written page. */ 1294 rtvals[i] = VM_PAGER_AGAIN; 1295 vm_page_clear_dirty(ma[i], 0, written & PAGE_MASK); 1296 } 1297 } 1298 VM_OBJECT_WUNLOCK(obj); 1299 } 1300 1301 void 1302 vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, 1303 vm_offset_t end) 1304 { 1305 struct vnode *vp; 1306 vm_ooffset_t old_wm; 1307 1308 VM_OBJECT_WLOCK(object); 1309 if (object->type != OBJT_VNODE) { 1310 VM_OBJECT_WUNLOCK(object); 1311 return; 1312 } 1313 old_wm = object->un_pager.vnp.writemappings; 1314 object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start; 1315 vp = object->handle; 1316 if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) { 1317 ASSERT_VOP_ELOCKED(vp, "v_writecount inc"); 1318 VOP_ADD_WRITECOUNT(vp, 1); 1319 CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d", 1320 __func__, vp, vp->v_writecount); 1321 } else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) { 1322 ASSERT_VOP_ELOCKED(vp, "v_writecount dec"); 1323 VOP_ADD_WRITECOUNT(vp, -1); 1324 CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d", 1325 __func__, vp, vp->v_writecount); 1326 } 1327 VM_OBJECT_WUNLOCK(object); 1328 } 1329 1330 void 1331 vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, 1332 vm_offset_t end) 1333 { 1334 struct vnode *vp; 1335 struct mount *mp; 1336 vm_offset_t inc; 1337 1338 VM_OBJECT_WLOCK(object); 1339 1340 /* 1341 * First, recheck the object type to account for the race when 1342 * the vnode is reclaimed. 1343 */ 1344 if (object->type != OBJT_VNODE) { 1345 VM_OBJECT_WUNLOCK(object); 1346 return; 1347 } 1348 1349 /* 1350 * Optimize for the case when writemappings is not going to 1351 * zero. 1352 */ 1353 inc = end - start; 1354 if (object->un_pager.vnp.writemappings != inc) { 1355 object->un_pager.vnp.writemappings -= inc; 1356 VM_OBJECT_WUNLOCK(object); 1357 return; 1358 } 1359 1360 vp = object->handle; 1361 vhold(vp); 1362 VM_OBJECT_WUNLOCK(object); 1363 mp = NULL; 1364 vn_start_write(vp, &mp, V_WAIT); 1365 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1366 1367 /* 1368 * Decrement the object's writemappings, by swapping the start 1369 * and end arguments for vnode_pager_update_writecount(). If 1370 * there was not a race with vnode reclaimation, then the 1371 * vnode's v_writecount is decremented. 1372 */ 1373 vnode_pager_update_writecount(object, end, start); 1374 VOP_UNLOCK(vp, 0); 1375 vdrop(vp); 1376 if (mp != NULL) 1377 vn_finished_write(mp); 1378 } 1379