1 /* 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 1993, 1994 John S. Dyson 6 * Copyright (c) 1995, David Greenman 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41 */ 42 43 /* 44 * Page to/from files (vnodes). 45 */ 46 47 /* 48 * TODO: 49 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 50 * greatly re-simplify the vnode_pager. 51 */ 52 53 #include <sys/cdefs.h> 54 __FBSDID("$FreeBSD$"); 55 56 #include <sys/param.h> 57 #include <sys/systm.h> 58 #include <sys/proc.h> 59 #include <sys/vnode.h> 60 #include <sys/mount.h> 61 #include <sys/bio.h> 62 #include <sys/buf.h> 63 #include <sys/vmmeter.h> 64 #include <sys/conf.h> 65 66 #include <vm/vm.h> 67 #include <vm/vm_object.h> 68 #include <vm/vm_page.h> 69 #include <vm/vm_pager.h> 70 #include <vm/vm_map.h> 71 #include <vm/vnode_pager.h> 72 #include <vm/vm_extern.h> 73 74 static void vnode_pager_init(void); 75 static vm_offset_t vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, 76 int *run); 77 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); 78 static int vnode_pager_input_old(vm_object_t object, vm_page_t m); 79 static void vnode_pager_dealloc(vm_object_t); 80 static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); 81 static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); 82 static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); 83 84 struct pagerops vnodepagerops = { 85 .pgo_init = vnode_pager_init, 86 .pgo_alloc = vnode_pager_alloc, 87 .pgo_dealloc = vnode_pager_dealloc, 88 .pgo_getpages = vnode_pager_getpages, 89 .pgo_putpages = vnode_pager_putpages, 90 .pgo_haspage = vnode_pager_haspage, 91 }; 92 93 int vnode_pbuf_freecnt; 94 95 static void 96 vnode_pager_init(void) 97 { 98 99 vnode_pbuf_freecnt = nswbuf / 2 + 1; 100 } 101 102 /* 103 * Allocate (or lookup) pager for a vnode. 104 * Handle is a vnode pointer. 105 * 106 * MPSAFE 107 */ 108 vm_object_t 109 vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 110 vm_ooffset_t offset) 111 { 112 vm_object_t object; 113 struct vnode *vp; 114 115 /* 116 * Pageout to vnode, no can do yet. 117 */ 118 if (handle == NULL) 119 return (NULL); 120 121 vp = (struct vnode *) handle; 122 123 ASSERT_VOP_LOCKED(vp, "vnode_pager_alloc"); 124 125 /* 126 * Prevent race condition when allocating the object. This 127 * can happen with NFS vnodes since the nfsnode isn't locked. 128 */ 129 VI_LOCK(vp); 130 while (vp->v_iflag & VI_OLOCK) { 131 vp->v_iflag |= VI_OWANT; 132 msleep(vp, VI_MTX(vp), PVM, "vnpobj", 0); 133 } 134 vp->v_iflag |= VI_OLOCK; 135 VI_UNLOCK(vp); 136 137 /* 138 * If the object is being terminated, wait for it to 139 * go away. 140 */ 141 while ((object = vp->v_object) != NULL) { 142 VM_OBJECT_LOCK(object); 143 if ((object->flags & OBJ_DEAD) == 0) 144 break; 145 msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vadead", 0); 146 } 147 148 if (vp->v_usecount == 0) 149 panic("vnode_pager_alloc: no vnode reference"); 150 151 if (object == NULL) { 152 /* 153 * And an object of the appropriate size 154 */ 155 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 156 157 object->un_pager.vnp.vnp_size = size; 158 159 object->handle = handle; 160 vp->v_object = object; 161 } else { 162 object->ref_count++; 163 VM_OBJECT_UNLOCK(object); 164 } 165 VI_LOCK(vp); 166 vp->v_usecount++; 167 vp->v_iflag &= ~VI_OLOCK; 168 if (vp->v_iflag & VI_OWANT) { 169 vp->v_iflag &= ~VI_OWANT; 170 wakeup(vp); 171 } 172 VI_UNLOCK(vp); 173 return (object); 174 } 175 176 /* 177 * The object must be locked. 178 */ 179 static void 180 vnode_pager_dealloc(object) 181 vm_object_t object; 182 { 183 struct vnode *vp = object->handle; 184 185 if (vp == NULL) 186 panic("vnode_pager_dealloc: pager already dealloced"); 187 188 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 189 vm_object_pip_wait(object, "vnpdea"); 190 191 object->handle = NULL; 192 object->type = OBJT_DEAD; 193 ASSERT_VOP_LOCKED(vp, "vnode_pager_dealloc"); 194 vp->v_object = NULL; 195 vp->v_vflag &= ~(VV_TEXT | VV_OBJBUF); 196 } 197 198 static boolean_t 199 vnode_pager_haspage(object, pindex, before, after) 200 vm_object_t object; 201 vm_pindex_t pindex; 202 int *before; 203 int *after; 204 { 205 struct vnode *vp = object->handle; 206 daddr_t bn; 207 int err; 208 daddr_t reqblock; 209 int poff; 210 int bsize; 211 int pagesperblock, blocksperpage; 212 213 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 214 /* 215 * If no vp or vp is doomed or marked transparent to VM, we do not 216 * have the page. 217 */ 218 if (vp == NULL) 219 return FALSE; 220 221 VI_LOCK(vp); 222 if (vp->v_iflag & VI_DOOMED) { 223 VI_UNLOCK(vp); 224 return FALSE; 225 } 226 VI_UNLOCK(vp); 227 /* 228 * If filesystem no longer mounted or offset beyond end of file we do 229 * not have the page. 230 */ 231 if ((vp->v_mount == NULL) || 232 (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)) 233 return FALSE; 234 235 bsize = vp->v_mount->mnt_stat.f_iosize; 236 pagesperblock = bsize / PAGE_SIZE; 237 blocksperpage = 0; 238 if (pagesperblock > 0) { 239 reqblock = pindex / pagesperblock; 240 } else { 241 blocksperpage = (PAGE_SIZE / bsize); 242 reqblock = pindex * blocksperpage; 243 } 244 VM_OBJECT_UNLOCK(object); 245 err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before); 246 VM_OBJECT_LOCK(object); 247 if (err) 248 return TRUE; 249 if (bn == -1) 250 return FALSE; 251 if (pagesperblock > 0) { 252 poff = pindex - (reqblock * pagesperblock); 253 if (before) { 254 *before *= pagesperblock; 255 *before += poff; 256 } 257 if (after) { 258 int numafter; 259 *after *= pagesperblock; 260 numafter = pagesperblock - (poff + 1); 261 if (IDX_TO_OFF(pindex + numafter) > 262 object->un_pager.vnp.vnp_size) { 263 numafter = 264 OFF_TO_IDX(object->un_pager.vnp.vnp_size) - 265 pindex; 266 } 267 *after += numafter; 268 } 269 } else { 270 if (before) { 271 *before /= blocksperpage; 272 } 273 274 if (after) { 275 *after /= blocksperpage; 276 } 277 } 278 return TRUE; 279 } 280 281 /* 282 * Lets the VM system know about a change in size for a file. 283 * We adjust our own internal size and flush any cached pages in 284 * the associated object that are affected by the size change. 285 * 286 * Note: this routine may be invoked as a result of a pager put 287 * operation (possibly at object termination time), so we must be careful. 288 */ 289 void 290 vnode_pager_setsize(vp, nsize) 291 struct vnode *vp; 292 vm_ooffset_t nsize; 293 { 294 vm_object_t object; 295 vm_page_t m; 296 vm_pindex_t nobjsize; 297 298 if ((object = vp->v_object) == NULL) 299 return; 300 VM_OBJECT_LOCK(object); 301 if (nsize == object->un_pager.vnp.vnp_size) { 302 /* 303 * Hasn't changed size 304 */ 305 VM_OBJECT_UNLOCK(object); 306 return; 307 } 308 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 309 if (nsize < object->un_pager.vnp.vnp_size) { 310 /* 311 * File has shrunk. Toss any cached pages beyond the new EOF. 312 */ 313 if (nobjsize < object->size) 314 vm_object_page_remove(object, nobjsize, object->size, 315 FALSE); 316 /* 317 * this gets rid of garbage at the end of a page that is now 318 * only partially backed by the vnode. 319 * 320 * XXX for some reason (I don't know yet), if we take a 321 * completely invalid page and mark it partially valid 322 * it can screw up NFS reads, so we don't allow the case. 323 */ 324 if ((nsize & PAGE_MASK) && 325 (m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL && 326 m->valid != 0) { 327 int base = (int)nsize & PAGE_MASK; 328 int size = PAGE_SIZE - base; 329 330 /* 331 * Clear out partial-page garbage in case 332 * the page has been mapped. 333 */ 334 pmap_zero_page_area(m, base, size); 335 336 /* 337 * XXX work around SMP data integrity race 338 * by unmapping the page from user processes. 339 * The garbage we just cleared may be mapped 340 * to a user process running on another cpu 341 * and this code is not running through normal 342 * I/O channels which handle SMP issues for 343 * us, so unmap page to synchronize all cpus. 344 * 345 * XXX should vm_pager_unmap_page() have 346 * dealt with this? 347 */ 348 vm_page_lock_queues(); 349 pmap_remove_all(m); 350 351 /* 352 * Clear out partial-page dirty bits. This 353 * has the side effect of setting the valid 354 * bits, but that is ok. There are a bunch 355 * of places in the VM system where we expected 356 * m->dirty == VM_PAGE_BITS_ALL. The file EOF 357 * case is one of them. If the page is still 358 * partially dirty, make it fully dirty. 359 * 360 * note that we do not clear out the valid 361 * bits. This would prevent bogus_page 362 * replacement from working properly. 363 */ 364 vm_page_set_validclean(m, base, size); 365 if (m->dirty != 0) 366 m->dirty = VM_PAGE_BITS_ALL; 367 vm_page_unlock_queues(); 368 } 369 } 370 object->un_pager.vnp.vnp_size = nsize; 371 object->size = nobjsize; 372 VM_OBJECT_UNLOCK(object); 373 } 374 375 /* 376 * calculate the linear (byte) disk address of specified virtual 377 * file address 378 */ 379 static vm_offset_t 380 vnode_pager_addr(vp, address, run) 381 struct vnode *vp; 382 vm_ooffset_t address; 383 int *run; 384 { 385 int rtaddress; 386 int bsize; 387 daddr_t block; 388 int err; 389 daddr_t vblock; 390 int voffset; 391 392 GIANT_REQUIRED; 393 if ((int) address < 0) 394 return -1; 395 396 if (vp->v_mount == NULL) 397 return -1; 398 399 bsize = vp->v_mount->mnt_stat.f_iosize; 400 vblock = address / bsize; 401 voffset = address % bsize; 402 403 err = VOP_BMAP(vp, vblock, NULL, &block, run, NULL); 404 405 if (err || (block == -1)) 406 rtaddress = -1; 407 else { 408 rtaddress = block + voffset / DEV_BSIZE; 409 if (run) { 410 *run += 1; 411 *run *= bsize/PAGE_SIZE; 412 *run -= voffset/PAGE_SIZE; 413 } 414 } 415 416 return rtaddress; 417 } 418 419 /* 420 * small block filesystem vnode pager input 421 */ 422 static int 423 vnode_pager_input_smlfs(object, m) 424 vm_object_t object; 425 vm_page_t m; 426 { 427 int i; 428 struct vnode *dp, *vp; 429 struct buf *bp; 430 vm_offset_t kva; 431 int fileaddr; 432 vm_offset_t bsize; 433 int error = 0; 434 435 GIANT_REQUIRED; 436 437 vp = object->handle; 438 if (vp->v_mount == NULL) 439 return VM_PAGER_BAD; 440 441 bsize = vp->v_mount->mnt_stat.f_iosize; 442 443 VOP_BMAP(vp, 0, &dp, 0, NULL, NULL); 444 445 kva = vm_pager_map_page(m); 446 447 for (i = 0; i < PAGE_SIZE / bsize; i++) { 448 vm_ooffset_t address; 449 450 if (vm_page_bits(i * bsize, bsize) & m->valid) 451 continue; 452 453 address = IDX_TO_OFF(m->pindex) + i * bsize; 454 if (address >= object->un_pager.vnp.vnp_size) { 455 fileaddr = -1; 456 } else { 457 fileaddr = vnode_pager_addr(vp, address, NULL); 458 } 459 if (fileaddr != -1) { 460 bp = getpbuf(&vnode_pbuf_freecnt); 461 462 /* build a minimal buffer header */ 463 bp->b_iocmd = BIO_READ; 464 bp->b_iodone = bdone; 465 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 466 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 467 bp->b_rcred = crhold(curthread->td_ucred); 468 bp->b_wcred = crhold(curthread->td_ucred); 469 bp->b_data = (caddr_t) kva + i * bsize; 470 bp->b_blkno = fileaddr; 471 pbgetvp(dp, bp); 472 bp->b_bcount = bsize; 473 bp->b_bufsize = bsize; 474 bp->b_runningbufspace = bp->b_bufsize; 475 runningbufspace += bp->b_runningbufspace; 476 477 /* do the input */ 478 bp->b_iooffset = dbtob(bp->b_blkno); 479 if (dp->v_type == VCHR) 480 VOP_SPECSTRATEGY(bp->b_vp, bp); 481 else 482 VOP_STRATEGY(bp->b_vp, bp); 483 484 /* we definitely need to be at splvm here */ 485 486 bwait(bp, PVM, "vnsrd"); 487 488 if ((bp->b_ioflags & BIO_ERROR) != 0) 489 error = EIO; 490 491 /* 492 * free the buffer header back to the swap buffer pool 493 */ 494 relpbuf(bp, &vnode_pbuf_freecnt); 495 if (error) 496 break; 497 498 VM_OBJECT_LOCK(object); 499 vm_page_lock_queues(); 500 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 501 vm_page_unlock_queues(); 502 VM_OBJECT_UNLOCK(object); 503 } else { 504 VM_OBJECT_LOCK(object); 505 vm_page_lock_queues(); 506 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 507 vm_page_unlock_queues(); 508 VM_OBJECT_UNLOCK(object); 509 bzero((caddr_t) kva + i * bsize, bsize); 510 } 511 } 512 vm_pager_unmap_page(kva); 513 vm_page_lock_queues(); 514 pmap_clear_modify(m); 515 vm_page_flag_clear(m, PG_ZERO); 516 vm_page_unlock_queues(); 517 if (error) { 518 return VM_PAGER_ERROR; 519 } 520 return VM_PAGER_OK; 521 522 } 523 524 525 /* 526 * old style vnode pager output routine 527 */ 528 static int 529 vnode_pager_input_old(object, m) 530 vm_object_t object; 531 vm_page_t m; 532 { 533 struct uio auio; 534 struct iovec aiov; 535 int error; 536 int size; 537 vm_offset_t kva; 538 struct vnode *vp; 539 540 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 541 error = 0; 542 543 /* 544 * Return failure if beyond current EOF 545 */ 546 if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { 547 return VM_PAGER_BAD; 548 } else { 549 size = PAGE_SIZE; 550 if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) 551 size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); 552 vp = object->handle; 553 VM_OBJECT_UNLOCK(object); 554 555 /* 556 * Allocate a kernel virtual address and initialize so that 557 * we can use VOP_READ/WRITE routines. 558 */ 559 kva = vm_pager_map_page(m); 560 561 aiov.iov_base = (caddr_t) kva; 562 aiov.iov_len = size; 563 auio.uio_iov = &aiov; 564 auio.uio_iovcnt = 1; 565 auio.uio_offset = IDX_TO_OFF(m->pindex); 566 auio.uio_segflg = UIO_SYSSPACE; 567 auio.uio_rw = UIO_READ; 568 auio.uio_resid = size; 569 auio.uio_td = curthread; 570 571 error = VOP_READ(vp, &auio, 0, curthread->td_ucred); 572 if (!error) { 573 int count = size - auio.uio_resid; 574 575 if (count == 0) 576 error = EINVAL; 577 else if (count != PAGE_SIZE) 578 bzero((caddr_t) kva + count, PAGE_SIZE - count); 579 } 580 vm_pager_unmap_page(kva); 581 582 VM_OBJECT_LOCK(object); 583 } 584 vm_page_lock_queues(); 585 pmap_clear_modify(m); 586 vm_page_undirty(m); 587 vm_page_flag_clear(m, PG_ZERO); 588 vm_page_unlock_queues(); 589 if (!error) 590 m->valid = VM_PAGE_BITS_ALL; 591 return error ? VM_PAGER_ERROR : VM_PAGER_OK; 592 } 593 594 /* 595 * generic vnode pager input routine 596 */ 597 598 /* 599 * Local media VFS's that do not implement their own VOP_GETPAGES 600 * should have their VOP_GETPAGES call to vnode_pager_generic_getpages() 601 * to implement the previous behaviour. 602 * 603 * All other FS's should use the bypass to get to the local media 604 * backing vp's VOP_GETPAGES. 605 */ 606 static int 607 vnode_pager_getpages(object, m, count, reqpage) 608 vm_object_t object; 609 vm_page_t *m; 610 int count; 611 int reqpage; 612 { 613 int rtval; 614 struct vnode *vp; 615 int bytes = count * PAGE_SIZE; 616 617 vp = object->handle; 618 VM_OBJECT_UNLOCK(object); 619 rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0); 620 KASSERT(rtval != EOPNOTSUPP, 621 ("vnode_pager: FS getpages not implemented\n")); 622 VM_OBJECT_LOCK(object); 623 return rtval; 624 } 625 626 /* 627 * This is now called from local media FS's to operate against their 628 * own vnodes if they fail to implement VOP_GETPAGES. 629 */ 630 int 631 vnode_pager_generic_getpages(vp, m, bytecount, reqpage) 632 struct vnode *vp; 633 vm_page_t *m; 634 int bytecount; 635 int reqpage; 636 { 637 vm_object_t object; 638 vm_offset_t kva; 639 off_t foff, tfoff, nextoff; 640 int i, j, size, bsize, first, firstaddr; 641 struct vnode *dp; 642 int runpg; 643 int runend; 644 struct buf *bp; 645 int count; 646 int error = 0; 647 648 GIANT_REQUIRED; 649 object = vp->v_object; 650 count = bytecount / PAGE_SIZE; 651 652 if (vp->v_mount == NULL) 653 return VM_PAGER_BAD; 654 655 bsize = vp->v_mount->mnt_stat.f_iosize; 656 657 /* get the UNDERLYING device for the file with VOP_BMAP() */ 658 659 /* 660 * originally, we did not check for an error return value -- assuming 661 * an fs always has a bmap entry point -- that assumption is wrong!!! 662 */ 663 foff = IDX_TO_OFF(m[reqpage]->pindex); 664 665 /* 666 * if we can't bmap, use old VOP code 667 */ 668 if (VOP_BMAP(vp, 0, &dp, 0, NULL, NULL)) { 669 VM_OBJECT_LOCK(object); 670 vm_page_lock_queues(); 671 for (i = 0; i < count; i++) 672 if (i != reqpage) 673 vm_page_free(m[i]); 674 vm_page_unlock_queues(); 675 cnt.v_vnodein++; 676 cnt.v_vnodepgsin++; 677 error = vnode_pager_input_old(object, m[reqpage]); 678 VM_OBJECT_UNLOCK(object); 679 return (error); 680 681 /* 682 * if the blocksize is smaller than a page size, then use 683 * special small filesystem code. NFS sometimes has a small 684 * blocksize, but it can handle large reads itself. 685 */ 686 } else if ((PAGE_SIZE / bsize) > 1 && 687 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 688 VM_OBJECT_LOCK(object); 689 vm_page_lock_queues(); 690 for (i = 0; i < count; i++) 691 if (i != reqpage) 692 vm_page_free(m[i]); 693 vm_page_unlock_queues(); 694 VM_OBJECT_UNLOCK(object); 695 cnt.v_vnodein++; 696 cnt.v_vnodepgsin++; 697 return vnode_pager_input_smlfs(object, m[reqpage]); 698 } 699 700 /* 701 * If we have a completely valid page available to us, we can 702 * clean up and return. Otherwise we have to re-read the 703 * media. 704 */ 705 VM_OBJECT_LOCK(object); 706 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 707 vm_page_lock_queues(); 708 for (i = 0; i < count; i++) 709 if (i != reqpage) 710 vm_page_free(m[i]); 711 vm_page_unlock_queues(); 712 VM_OBJECT_UNLOCK(object); 713 return VM_PAGER_OK; 714 } 715 m[reqpage]->valid = 0; 716 VM_OBJECT_UNLOCK(object); 717 718 /* 719 * here on direct device I/O 720 */ 721 firstaddr = -1; 722 723 /* 724 * calculate the run that includes the required page 725 */ 726 for (first = 0, i = 0; i < count; i = runend) { 727 firstaddr = vnode_pager_addr(vp, 728 IDX_TO_OFF(m[i]->pindex), &runpg); 729 if (firstaddr == -1) { 730 VM_OBJECT_LOCK(object); 731 if (i == reqpage && foff < object->un_pager.vnp.vnp_size) { 732 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %d, foff: 0x%jx%08jx, vnp_size: 0x%jx%08jx", 733 firstaddr, (uintmax_t)(foff >> 32), 734 (uintmax_t)foff, 735 (uintmax_t) 736 (object->un_pager.vnp.vnp_size >> 32), 737 (uintmax_t)object->un_pager.vnp.vnp_size); 738 } 739 vm_page_lock_queues(); 740 vm_page_free(m[i]); 741 vm_page_unlock_queues(); 742 VM_OBJECT_UNLOCK(object); 743 runend = i + 1; 744 first = runend; 745 continue; 746 } 747 runend = i + runpg; 748 if (runend <= reqpage) { 749 VM_OBJECT_LOCK(object); 750 vm_page_lock_queues(); 751 for (j = i; j < runend; j++) 752 vm_page_free(m[j]); 753 vm_page_unlock_queues(); 754 VM_OBJECT_UNLOCK(object); 755 } else { 756 if (runpg < (count - first)) { 757 VM_OBJECT_LOCK(object); 758 vm_page_lock_queues(); 759 for (i = first + runpg; i < count; i++) 760 vm_page_free(m[i]); 761 vm_page_unlock_queues(); 762 VM_OBJECT_UNLOCK(object); 763 count = first + runpg; 764 } 765 break; 766 } 767 first = runend; 768 } 769 770 /* 771 * the first and last page have been calculated now, move input pages 772 * to be zero based... 773 */ 774 if (first != 0) { 775 for (i = first; i < count; i++) { 776 m[i - first] = m[i]; 777 } 778 count -= first; 779 reqpage -= first; 780 } 781 782 /* 783 * calculate the file virtual address for the transfer 784 */ 785 foff = IDX_TO_OFF(m[0]->pindex); 786 787 /* 788 * calculate the size of the transfer 789 */ 790 size = count * PAGE_SIZE; 791 if ((foff + size) > object->un_pager.vnp.vnp_size) 792 size = object->un_pager.vnp.vnp_size - foff; 793 794 /* 795 * round up physical size for real devices. 796 */ 797 if (dp->v_type == VBLK || dp->v_type == VCHR) { 798 int secmask = dp->v_rdev->si_bsize_phys - 1; 799 KASSERT(secmask < PAGE_SIZE, ("vnode_pager_generic_getpages: sector size %d too large\n", secmask + 1)); 800 size = (size + secmask) & ~secmask; 801 } 802 803 bp = getpbuf(&vnode_pbuf_freecnt); 804 kva = (vm_offset_t) bp->b_data; 805 806 /* 807 * and map the pages to be read into the kva 808 */ 809 pmap_qenter(kva, m, count); 810 811 /* build a minimal buffer header */ 812 bp->b_iocmd = BIO_READ; 813 bp->b_iodone = bdone; 814 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 815 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 816 bp->b_rcred = crhold(curthread->td_ucred); 817 bp->b_wcred = crhold(curthread->td_ucred); 818 bp->b_blkno = firstaddr; 819 pbgetvp(dp, bp); 820 bp->b_bcount = size; 821 bp->b_bufsize = size; 822 bp->b_runningbufspace = bp->b_bufsize; 823 runningbufspace += bp->b_runningbufspace; 824 825 cnt.v_vnodein++; 826 cnt.v_vnodepgsin += count; 827 828 /* do the input */ 829 bp->b_iooffset = dbtob(bp->b_blkno); 830 if (dp->v_type == VCHR) 831 VOP_SPECSTRATEGY(bp->b_vp, bp); 832 else 833 VOP_STRATEGY(bp->b_vp, bp); 834 835 bwait(bp, PVM, "vnread"); 836 837 if ((bp->b_ioflags & BIO_ERROR) != 0) 838 error = EIO; 839 840 if (!error) { 841 if (size != count * PAGE_SIZE) 842 bzero((caddr_t) kva + size, PAGE_SIZE * count - size); 843 } 844 pmap_qremove(kva, count); 845 846 /* 847 * free the buffer header back to the swap buffer pool 848 */ 849 relpbuf(bp, &vnode_pbuf_freecnt); 850 851 VM_OBJECT_LOCK(object); 852 vm_page_lock_queues(); 853 for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { 854 vm_page_t mt; 855 856 nextoff = tfoff + PAGE_SIZE; 857 mt = m[i]; 858 859 if (nextoff <= object->un_pager.vnp.vnp_size) { 860 /* 861 * Read filled up entire page. 862 */ 863 mt->valid = VM_PAGE_BITS_ALL; 864 vm_page_undirty(mt); /* should be an assert? XXX */ 865 pmap_clear_modify(mt); 866 } else { 867 /* 868 * Read did not fill up entire page. Since this 869 * is getpages, the page may be mapped, so we have 870 * to zero the invalid portions of the page even 871 * though we aren't setting them valid. 872 * 873 * Currently we do not set the entire page valid, 874 * we just try to clear the piece that we couldn't 875 * read. 876 */ 877 vm_page_set_validclean(mt, 0, 878 object->un_pager.vnp.vnp_size - tfoff); 879 /* handled by vm_fault now */ 880 /* vm_page_zero_invalid(mt, FALSE); */ 881 } 882 883 vm_page_flag_clear(mt, PG_ZERO); 884 if (i != reqpage) { 885 886 /* 887 * whether or not to leave the page activated is up in 888 * the air, but we should put the page on a page queue 889 * somewhere. (it already is in the object). Result: 890 * It appears that empirical results show that 891 * deactivating pages is best. 892 */ 893 894 /* 895 * just in case someone was asking for this page we 896 * now tell them that it is ok to use 897 */ 898 if (!error) { 899 if (mt->flags & PG_WANTED) 900 vm_page_activate(mt); 901 else 902 vm_page_deactivate(mt); 903 vm_page_wakeup(mt); 904 } else { 905 vm_page_free(mt); 906 } 907 } 908 } 909 vm_page_unlock_queues(); 910 VM_OBJECT_UNLOCK(object); 911 if (error) { 912 printf("vnode_pager_getpages: I/O read error\n"); 913 } 914 return (error ? VM_PAGER_ERROR : VM_PAGER_OK); 915 } 916 917 /* 918 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 919 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 920 * vnode_pager_generic_putpages() to implement the previous behaviour. 921 * 922 * All other FS's should use the bypass to get to the local media 923 * backing vp's VOP_PUTPAGES. 924 */ 925 static void 926 vnode_pager_putpages(object, m, count, sync, rtvals) 927 vm_object_t object; 928 vm_page_t *m; 929 int count; 930 boolean_t sync; 931 int *rtvals; 932 { 933 int rtval; 934 struct vnode *vp; 935 struct mount *mp; 936 int bytes = count * PAGE_SIZE; 937 938 GIANT_REQUIRED; 939 /* 940 * Force synchronous operation if we are extremely low on memory 941 * to prevent a low-memory deadlock. VOP operations often need to 942 * allocate more memory to initiate the I/O ( i.e. do a BMAP 943 * operation ). The swapper handles the case by limiting the amount 944 * of asynchronous I/O, but that sort of solution doesn't scale well 945 * for the vnode pager without a lot of work. 946 * 947 * Also, the backing vnode's iodone routine may not wake the pageout 948 * daemon up. This should be probably be addressed XXX. 949 */ 950 951 if ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_pageout_free_min) 952 sync |= OBJPC_SYNC; 953 954 /* 955 * Call device-specific putpages function 956 */ 957 vp = object->handle; 958 VM_OBJECT_UNLOCK(object); 959 if (vp->v_type != VREG) 960 mp = NULL; 961 (void)vn_start_write(vp, &mp, V_WAIT); 962 rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0); 963 KASSERT(rtval != EOPNOTSUPP, 964 ("vnode_pager: stale FS putpages\n")); 965 vn_finished_write(mp); 966 VM_OBJECT_LOCK(object); 967 } 968 969 970 /* 971 * This is now called from local media FS's to operate against their 972 * own vnodes if they fail to implement VOP_PUTPAGES. 973 * 974 * This is typically called indirectly via the pageout daemon and 975 * clustering has already typically occured, so in general we ask the 976 * underlying filesystem to write the data out asynchronously rather 977 * then delayed. 978 */ 979 int 980 vnode_pager_generic_putpages(vp, m, bytecount, flags, rtvals) 981 struct vnode *vp; 982 vm_page_t *m; 983 int bytecount; 984 int flags; 985 int *rtvals; 986 { 987 int i; 988 vm_object_t object; 989 int count; 990 991 int maxsize, ncount; 992 vm_ooffset_t poffset; 993 struct uio auio; 994 struct iovec aiov; 995 int error; 996 int ioflags; 997 998 GIANT_REQUIRED; 999 object = vp->v_object; 1000 count = bytecount / PAGE_SIZE; 1001 1002 for (i = 0; i < count; i++) 1003 rtvals[i] = VM_PAGER_AGAIN; 1004 1005 if ((int) m[0]->pindex < 0) { 1006 printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%lx)\n", 1007 (long)m[0]->pindex, (u_long)m[0]->dirty); 1008 rtvals[0] = VM_PAGER_BAD; 1009 return VM_PAGER_BAD; 1010 } 1011 1012 maxsize = count * PAGE_SIZE; 1013 ncount = count; 1014 1015 poffset = IDX_TO_OFF(m[0]->pindex); 1016 1017 /* 1018 * If the page-aligned write is larger then the actual file we 1019 * have to invalidate pages occuring beyond the file EOF. However, 1020 * there is an edge case where a file may not be page-aligned where 1021 * the last page is partially invalid. In this case the filesystem 1022 * may not properly clear the dirty bits for the entire page (which 1023 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 1024 * With the page locked we are free to fix-up the dirty bits here. 1025 * 1026 * We do not under any circumstances truncate the valid bits, as 1027 * this will screw up bogus page replacement. 1028 */ 1029 if (maxsize + poffset > object->un_pager.vnp.vnp_size) { 1030 if (object->un_pager.vnp.vnp_size > poffset) { 1031 int pgoff; 1032 1033 maxsize = object->un_pager.vnp.vnp_size - poffset; 1034 ncount = btoc(maxsize); 1035 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 1036 vm_page_lock_queues(); 1037 vm_page_clear_dirty(m[ncount - 1], pgoff, 1038 PAGE_SIZE - pgoff); 1039 vm_page_unlock_queues(); 1040 } 1041 } else { 1042 maxsize = 0; 1043 ncount = 0; 1044 } 1045 if (ncount < count) { 1046 for (i = ncount; i < count; i++) { 1047 rtvals[i] = VM_PAGER_BAD; 1048 } 1049 } 1050 } 1051 1052 /* 1053 * pageouts are already clustered, use IO_ASYNC t o force a bawrite() 1054 * rather then a bdwrite() to prevent paging I/O from saturating 1055 * the buffer cache. Dummy-up the sequential heuristic to cause 1056 * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, 1057 * the system decides how to cluster. 1058 */ 1059 ioflags = IO_VMIO; 1060 if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) 1061 ioflags |= IO_SYNC; 1062 else if ((flags & VM_PAGER_CLUSTER_OK) == 0) 1063 ioflags |= IO_ASYNC; 1064 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 1065 ioflags |= IO_SEQMAX << IO_SEQSHIFT; 1066 1067 aiov.iov_base = (caddr_t) 0; 1068 aiov.iov_len = maxsize; 1069 auio.uio_iov = &aiov; 1070 auio.uio_iovcnt = 1; 1071 auio.uio_offset = poffset; 1072 auio.uio_segflg = UIO_NOCOPY; 1073 auio.uio_rw = UIO_WRITE; 1074 auio.uio_resid = maxsize; 1075 auio.uio_td = (struct thread *) 0; 1076 error = VOP_WRITE(vp, &auio, ioflags, curthread->td_ucred); 1077 cnt.v_vnodeout++; 1078 cnt.v_vnodepgsout += ncount; 1079 1080 if (error) { 1081 printf("vnode_pager_putpages: I/O error %d\n", error); 1082 } 1083 if (auio.uio_resid) { 1084 printf("vnode_pager_putpages: residual I/O %d at %lu\n", 1085 auio.uio_resid, (u_long)m[0]->pindex); 1086 } 1087 for (i = 0; i < ncount; i++) { 1088 rtvals[i] = VM_PAGER_OK; 1089 } 1090 return rtvals[0]; 1091 } 1092 1093 struct vnode * 1094 vnode_pager_lock(vm_object_t first_object) 1095 { 1096 struct vnode *vp; 1097 vm_object_t backing_object, object; 1098 1099 VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); 1100 for (object = first_object; object != NULL; object = backing_object) { 1101 if (object->type != OBJT_VNODE) { 1102 if ((backing_object = object->backing_object) != NULL) 1103 VM_OBJECT_LOCK(backing_object); 1104 if (object != first_object) 1105 VM_OBJECT_UNLOCK(object); 1106 continue; 1107 } 1108 retry: 1109 if (object->flags & OBJ_DEAD) { 1110 if (object != first_object) 1111 VM_OBJECT_UNLOCK(object); 1112 return NULL; 1113 } 1114 vp = object->handle; 1115 VI_LOCK(vp); 1116 VM_OBJECT_UNLOCK(object); 1117 if (first_object != object) 1118 VM_OBJECT_UNLOCK(first_object); 1119 if (vget(vp, LK_CANRECURSE | LK_INTERLOCK | LK_NOPAUSE | 1120 LK_RETRY | LK_SHARED, curthread)) { 1121 VM_OBJECT_LOCK(first_object); 1122 if (object != first_object) 1123 VM_OBJECT_LOCK(object); 1124 if (object->type != OBJT_VNODE) { 1125 if (object != first_object) 1126 VM_OBJECT_UNLOCK(object); 1127 return NULL; 1128 } 1129 printf("vnode_pager_lock: retrying\n"); 1130 goto retry; 1131 } 1132 VM_OBJECT_LOCK(first_object); 1133 return (vp); 1134 } 1135 return NULL; 1136 } 1137