1 /*- 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 1993, 1994 John S. Dyson 6 * Copyright (c) 1995, David Greenman 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41 */ 42 43 /* 44 * Page to/from files (vnodes). 45 */ 46 47 /* 48 * TODO: 49 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 50 * greatly re-simplify the vnode_pager. 51 */ 52 53 #include <sys/cdefs.h> 54 __FBSDID("$FreeBSD$"); 55 56 #include <sys/param.h> 57 #include <sys/systm.h> 58 #include <sys/proc.h> 59 #include <sys/vnode.h> 60 #include <sys/mount.h> 61 #include <sys/bio.h> 62 #include <sys/buf.h> 63 #include <sys/vmmeter.h> 64 #include <sys/limits.h> 65 #include <sys/conf.h> 66 #include <sys/sf_buf.h> 67 68 #include <vm/vm.h> 69 #include <vm/vm_object.h> 70 #include <vm/vm_page.h> 71 #include <vm/vm_pager.h> 72 #include <vm/vm_map.h> 73 #include <vm/vnode_pager.h> 74 #include <vm/vm_extern.h> 75 76 static void vnode_pager_init(void); 77 static vm_offset_t vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, 78 int *run); 79 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); 80 static int vnode_pager_input_old(vm_object_t object, vm_page_t m); 81 static void vnode_pager_dealloc(vm_object_t); 82 static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); 83 static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); 84 static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); 85 static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t); 86 87 struct pagerops vnodepagerops = { 88 .pgo_init = vnode_pager_init, 89 .pgo_alloc = vnode_pager_alloc, 90 .pgo_dealloc = vnode_pager_dealloc, 91 .pgo_getpages = vnode_pager_getpages, 92 .pgo_putpages = vnode_pager_putpages, 93 .pgo_haspage = vnode_pager_haspage, 94 }; 95 96 int vnode_pbuf_freecnt; 97 98 static void 99 vnode_pager_init(void) 100 { 101 102 vnode_pbuf_freecnt = nswbuf / 2 + 1; 103 } 104 105 /* Create the VM system backing object for this vnode */ 106 int 107 vnode_create_vobject(struct vnode *vp, size_t isize, struct thread *td) 108 { 109 vm_object_t object; 110 vm_ooffset_t size = isize; 111 struct vattr va; 112 113 if (!vn_isdisk(vp, NULL) && vn_canvmio(vp) == FALSE) 114 return (0); 115 116 while ((object = vp->v_object) != NULL) { 117 VM_OBJECT_LOCK(object); 118 if (!(object->flags & OBJ_DEAD)) { 119 VM_OBJECT_UNLOCK(object); 120 return (0); 121 } 122 VOP_UNLOCK(vp, 0, td); 123 vm_object_set_flag(object, OBJ_DISCONNECTWNT); 124 msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vodead", 0); 125 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td); 126 } 127 128 if (size == 0) { 129 if (vn_isdisk(vp, NULL)) { 130 size = IDX_TO_OFF(INT_MAX); 131 } else { 132 if (VOP_GETATTR(vp, &va, td->td_ucred, td) != 0) 133 return (0); 134 size = va.va_size; 135 } 136 } 137 138 object = vnode_pager_alloc(vp, size, 0, 0); 139 /* 140 * Dereference the reference we just created. This assumes 141 * that the object is associated with the vp. 142 */ 143 VM_OBJECT_LOCK(object); 144 object->ref_count--; 145 VM_OBJECT_UNLOCK(object); 146 vrele(vp); 147 148 KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object")); 149 150 return (0); 151 } 152 153 void 154 vnode_destroy_vobject(struct vnode *vp) 155 { 156 struct vm_object *obj; 157 158 obj = vp->v_object; 159 if (obj == NULL) 160 return; 161 VOP_LOCK(vp, LK_EXCLUSIVE, curthread); 162 vp->v_object = NULL; 163 VM_OBJECT_LOCK(obj); 164 if (obj->ref_count == 0) { 165 /* 166 * vclean() may be called twice. The first time 167 * removes the primary reference to the object, 168 * the second time goes one further and is a 169 * special-case to terminate the object. 170 * 171 * don't double-terminate the object 172 */ 173 if ((obj->flags & OBJ_DEAD) == 0) 174 vm_object_terminate(obj); 175 else 176 VM_OBJECT_UNLOCK(obj); 177 } else { 178 /* 179 * Woe to the process that tries to page now :-). 180 */ 181 vm_pager_deallocate(obj); 182 VM_OBJECT_UNLOCK(obj); 183 } 184 VOP_UNLOCK(vp, 0, curthread); 185 } 186 187 188 /* 189 * Allocate (or lookup) pager for a vnode. 190 * Handle is a vnode pointer. 191 * 192 * MPSAFE 193 */ 194 vm_object_t 195 vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 196 vm_ooffset_t offset) 197 { 198 vm_object_t object; 199 struct vnode *vp; 200 201 /* 202 * Pageout to vnode, no can do yet. 203 */ 204 if (handle == NULL) 205 return (NULL); 206 207 vp = (struct vnode *) handle; 208 209 ASSERT_VOP_LOCKED(vp, "vnode_pager_alloc"); 210 211 /* 212 * Prevent race condition when allocating the object. This 213 * can happen with NFS vnodes since the nfsnode isn't locked. 214 */ 215 VI_LOCK(vp); 216 while (vp->v_iflag & VI_OLOCK) { 217 vp->v_iflag |= VI_OWANT; 218 msleep(vp, VI_MTX(vp), PVM, "vnpobj", 0); 219 } 220 vp->v_iflag |= VI_OLOCK; 221 VI_UNLOCK(vp); 222 223 /* 224 * If the object is being terminated, wait for it to 225 * go away. 226 */ 227 while ((object = vp->v_object) != NULL) { 228 VM_OBJECT_LOCK(object); 229 if ((object->flags & OBJ_DEAD) == 0) 230 break; 231 vm_object_set_flag(object, OBJ_DISCONNECTWNT); 232 msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vadead", 0); 233 } 234 235 if (vp->v_usecount == 0) 236 panic("vnode_pager_alloc: no vnode reference"); 237 238 if (object == NULL) { 239 /* 240 * And an object of the appropriate size 241 */ 242 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 243 244 object->un_pager.vnp.vnp_size = size; 245 246 object->handle = handle; 247 vp->v_object = object; 248 } else { 249 object->ref_count++; 250 VM_OBJECT_UNLOCK(object); 251 } 252 VI_LOCK(vp); 253 vp->v_usecount++; 254 vp->v_iflag &= ~VI_OLOCK; 255 if (vp->v_iflag & VI_OWANT) { 256 vp->v_iflag &= ~VI_OWANT; 257 wakeup(vp); 258 } 259 VI_UNLOCK(vp); 260 return (object); 261 } 262 263 /* 264 * The object must be locked. 265 */ 266 static void 267 vnode_pager_dealloc(object) 268 vm_object_t object; 269 { 270 struct vnode *vp = object->handle; 271 272 if (vp == NULL) 273 panic("vnode_pager_dealloc: pager already dealloced"); 274 275 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 276 vm_object_pip_wait(object, "vnpdea"); 277 278 object->handle = NULL; 279 object->type = OBJT_DEAD; 280 if (object->flags & OBJ_DISCONNECTWNT) { 281 vm_object_clear_flag(object, OBJ_DISCONNECTWNT); 282 wakeup(object); 283 } 284 ASSERT_VOP_LOCKED(vp, "vnode_pager_dealloc"); 285 vp->v_object = NULL; 286 vp->v_vflag &= ~VV_TEXT; 287 } 288 289 static boolean_t 290 vnode_pager_haspage(object, pindex, before, after) 291 vm_object_t object; 292 vm_pindex_t pindex; 293 int *before; 294 int *after; 295 { 296 struct vnode *vp = object->handle; 297 daddr_t bn; 298 int err; 299 daddr_t reqblock; 300 int poff; 301 int bsize; 302 int pagesperblock, blocksperpage; 303 int vfslocked; 304 305 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 306 /* 307 * If no vp or vp is doomed or marked transparent to VM, we do not 308 * have the page. 309 */ 310 if (vp == NULL) 311 return FALSE; 312 313 VI_LOCK(vp); 314 if (vp->v_iflag & VI_DOOMED) { 315 VI_UNLOCK(vp); 316 return FALSE; 317 } 318 VI_UNLOCK(vp); 319 /* 320 * If filesystem no longer mounted or offset beyond end of file we do 321 * not have the page. 322 */ 323 if ((vp->v_mount == NULL) || 324 (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)) 325 return FALSE; 326 327 bsize = vp->v_mount->mnt_stat.f_iosize; 328 pagesperblock = bsize / PAGE_SIZE; 329 blocksperpage = 0; 330 if (pagesperblock > 0) { 331 reqblock = pindex / pagesperblock; 332 } else { 333 blocksperpage = (PAGE_SIZE / bsize); 334 reqblock = pindex * blocksperpage; 335 } 336 VM_OBJECT_UNLOCK(object); 337 vfslocked = VFS_LOCK_GIANT(vp->v_mount); 338 err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before); 339 VFS_UNLOCK_GIANT(vfslocked); 340 VM_OBJECT_LOCK(object); 341 if (err) 342 return TRUE; 343 if (bn == -1) 344 return FALSE; 345 if (pagesperblock > 0) { 346 poff = pindex - (reqblock * pagesperblock); 347 if (before) { 348 *before *= pagesperblock; 349 *before += poff; 350 } 351 if (after) { 352 int numafter; 353 *after *= pagesperblock; 354 numafter = pagesperblock - (poff + 1); 355 if (IDX_TO_OFF(pindex + numafter) > 356 object->un_pager.vnp.vnp_size) { 357 numafter = 358 OFF_TO_IDX(object->un_pager.vnp.vnp_size) - 359 pindex; 360 } 361 *after += numafter; 362 } 363 } else { 364 if (before) { 365 *before /= blocksperpage; 366 } 367 368 if (after) { 369 *after /= blocksperpage; 370 } 371 } 372 return TRUE; 373 } 374 375 /* 376 * Lets the VM system know about a change in size for a file. 377 * We adjust our own internal size and flush any cached pages in 378 * the associated object that are affected by the size change. 379 * 380 * Note: this routine may be invoked as a result of a pager put 381 * operation (possibly at object termination time), so we must be careful. 382 */ 383 void 384 vnode_pager_setsize(vp, nsize) 385 struct vnode *vp; 386 vm_ooffset_t nsize; 387 { 388 vm_object_t object; 389 vm_page_t m; 390 vm_pindex_t nobjsize; 391 392 if ((object = vp->v_object) == NULL) 393 return; 394 VM_OBJECT_LOCK(object); 395 if (nsize == object->un_pager.vnp.vnp_size) { 396 /* 397 * Hasn't changed size 398 */ 399 VM_OBJECT_UNLOCK(object); 400 return; 401 } 402 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 403 if (nsize < object->un_pager.vnp.vnp_size) { 404 /* 405 * File has shrunk. Toss any cached pages beyond the new EOF. 406 */ 407 if (nobjsize < object->size) 408 vm_object_page_remove(object, nobjsize, object->size, 409 FALSE); 410 /* 411 * this gets rid of garbage at the end of a page that is now 412 * only partially backed by the vnode. 413 * 414 * XXX for some reason (I don't know yet), if we take a 415 * completely invalid page and mark it partially valid 416 * it can screw up NFS reads, so we don't allow the case. 417 */ 418 if ((nsize & PAGE_MASK) && 419 (m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL && 420 m->valid != 0) { 421 int base = (int)nsize & PAGE_MASK; 422 int size = PAGE_SIZE - base; 423 424 /* 425 * Clear out partial-page garbage in case 426 * the page has been mapped. 427 */ 428 pmap_zero_page_area(m, base, size); 429 430 /* 431 * XXX work around SMP data integrity race 432 * by unmapping the page from user processes. 433 * The garbage we just cleared may be mapped 434 * to a user process running on another cpu 435 * and this code is not running through normal 436 * I/O channels which handle SMP issues for 437 * us, so unmap page to synchronize all cpus. 438 * 439 * XXX should vm_pager_unmap_page() have 440 * dealt with this? 441 */ 442 vm_page_lock_queues(); 443 pmap_remove_all(m); 444 445 /* 446 * Clear out partial-page dirty bits. This 447 * has the side effect of setting the valid 448 * bits, but that is ok. There are a bunch 449 * of places in the VM system where we expected 450 * m->dirty == VM_PAGE_BITS_ALL. The file EOF 451 * case is one of them. If the page is still 452 * partially dirty, make it fully dirty. 453 * 454 * note that we do not clear out the valid 455 * bits. This would prevent bogus_page 456 * replacement from working properly. 457 */ 458 vm_page_set_validclean(m, base, size); 459 if (m->dirty != 0) 460 m->dirty = VM_PAGE_BITS_ALL; 461 vm_page_unlock_queues(); 462 } 463 } 464 object->un_pager.vnp.vnp_size = nsize; 465 object->size = nobjsize; 466 VM_OBJECT_UNLOCK(object); 467 } 468 469 /* 470 * calculate the linear (byte) disk address of specified virtual 471 * file address 472 */ 473 static vm_offset_t 474 vnode_pager_addr(vp, address, run) 475 struct vnode *vp; 476 vm_ooffset_t address; 477 int *run; 478 { 479 int rtaddress; 480 int bsize; 481 daddr_t block; 482 int err; 483 daddr_t vblock; 484 int voffset; 485 486 if (address < 0) 487 return -1; 488 489 if (vp->v_mount == NULL) 490 return -1; 491 492 bsize = vp->v_mount->mnt_stat.f_iosize; 493 vblock = address / bsize; 494 voffset = address % bsize; 495 496 err = VOP_BMAP(vp, vblock, NULL, &block, run, NULL); 497 498 if (err || (block == -1)) 499 rtaddress = -1; 500 else { 501 rtaddress = block + voffset / DEV_BSIZE; 502 if (run) { 503 *run += 1; 504 *run *= bsize/PAGE_SIZE; 505 *run -= voffset/PAGE_SIZE; 506 } 507 } 508 509 return rtaddress; 510 } 511 512 /* 513 * small block filesystem vnode pager input 514 */ 515 static int 516 vnode_pager_input_smlfs(object, m) 517 vm_object_t object; 518 vm_page_t m; 519 { 520 int i; 521 struct vnode *vp; 522 struct bufobj *bo; 523 struct buf *bp; 524 struct sf_buf *sf; 525 int fileaddr; 526 vm_offset_t bsize; 527 int error = 0; 528 529 vp = object->handle; 530 if (vp->v_mount == NULL) 531 return VM_PAGER_BAD; 532 533 bsize = vp->v_mount->mnt_stat.f_iosize; 534 535 VOP_BMAP(vp, 0, &bo, 0, NULL, NULL); 536 537 sf = sf_buf_alloc(m, 0); 538 539 for (i = 0; i < PAGE_SIZE / bsize; i++) { 540 vm_ooffset_t address; 541 542 if (vm_page_bits(i * bsize, bsize) & m->valid) 543 continue; 544 545 address = IDX_TO_OFF(m->pindex) + i * bsize; 546 if (address >= object->un_pager.vnp.vnp_size) { 547 fileaddr = -1; 548 } else { 549 fileaddr = vnode_pager_addr(vp, address, NULL); 550 } 551 if (fileaddr != -1) { 552 bp = getpbuf(&vnode_pbuf_freecnt); 553 554 /* build a minimal buffer header */ 555 bp->b_iocmd = BIO_READ; 556 bp->b_iodone = bdone; 557 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 558 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 559 bp->b_rcred = crhold(curthread->td_ucred); 560 bp->b_wcred = crhold(curthread->td_ucred); 561 bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize; 562 bp->b_blkno = fileaddr; 563 pbgetbo(bo, bp); 564 bp->b_bcount = bsize; 565 bp->b_bufsize = bsize; 566 bp->b_runningbufspace = bp->b_bufsize; 567 runningbufspace += bp->b_runningbufspace; 568 569 /* do the input */ 570 bp->b_iooffset = dbtob(bp->b_blkno); 571 bstrategy(bp); 572 573 /* we definitely need to be at splvm here */ 574 575 bwait(bp, PVM, "vnsrd"); 576 577 if ((bp->b_ioflags & BIO_ERROR) != 0) 578 error = EIO; 579 580 /* 581 * free the buffer header back to the swap buffer pool 582 */ 583 pbrelbo(bp); 584 relpbuf(bp, &vnode_pbuf_freecnt); 585 if (error) 586 break; 587 588 VM_OBJECT_LOCK(object); 589 vm_page_lock_queues(); 590 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 591 vm_page_unlock_queues(); 592 VM_OBJECT_UNLOCK(object); 593 } else { 594 VM_OBJECT_LOCK(object); 595 vm_page_lock_queues(); 596 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 597 vm_page_unlock_queues(); 598 VM_OBJECT_UNLOCK(object); 599 bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize); 600 } 601 } 602 sf_buf_free(sf); 603 vm_page_lock_queues(); 604 pmap_clear_modify(m); 605 vm_page_unlock_queues(); 606 if (error) { 607 return VM_PAGER_ERROR; 608 } 609 return VM_PAGER_OK; 610 611 } 612 613 614 /* 615 * old style vnode pager input routine 616 */ 617 static int 618 vnode_pager_input_old(object, m) 619 vm_object_t object; 620 vm_page_t m; 621 { 622 struct uio auio; 623 struct iovec aiov; 624 int error; 625 int size; 626 struct sf_buf *sf; 627 struct vnode *vp; 628 629 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 630 error = 0; 631 632 /* 633 * Return failure if beyond current EOF 634 */ 635 if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { 636 return VM_PAGER_BAD; 637 } else { 638 size = PAGE_SIZE; 639 if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) 640 size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); 641 vp = object->handle; 642 VM_OBJECT_UNLOCK(object); 643 644 /* 645 * Allocate a kernel virtual address and initialize so that 646 * we can use VOP_READ/WRITE routines. 647 */ 648 sf = sf_buf_alloc(m, 0); 649 650 aiov.iov_base = (caddr_t)sf_buf_kva(sf); 651 aiov.iov_len = size; 652 auio.uio_iov = &aiov; 653 auio.uio_iovcnt = 1; 654 auio.uio_offset = IDX_TO_OFF(m->pindex); 655 auio.uio_segflg = UIO_SYSSPACE; 656 auio.uio_rw = UIO_READ; 657 auio.uio_resid = size; 658 auio.uio_td = curthread; 659 660 error = VOP_READ(vp, &auio, 0, curthread->td_ucred); 661 if (!error) { 662 int count = size - auio.uio_resid; 663 664 if (count == 0) 665 error = EINVAL; 666 else if (count != PAGE_SIZE) 667 bzero((caddr_t)sf_buf_kva(sf) + count, 668 PAGE_SIZE - count); 669 } 670 sf_buf_free(sf); 671 672 VM_OBJECT_LOCK(object); 673 } 674 vm_page_lock_queues(); 675 pmap_clear_modify(m); 676 vm_page_undirty(m); 677 vm_page_unlock_queues(); 678 if (!error) 679 m->valid = VM_PAGE_BITS_ALL; 680 return error ? VM_PAGER_ERROR : VM_PAGER_OK; 681 } 682 683 /* 684 * generic vnode pager input routine 685 */ 686 687 /* 688 * Local media VFS's that do not implement their own VOP_GETPAGES 689 * should have their VOP_GETPAGES call to vnode_pager_generic_getpages() 690 * to implement the previous behaviour. 691 * 692 * All other FS's should use the bypass to get to the local media 693 * backing vp's VOP_GETPAGES. 694 */ 695 static int 696 vnode_pager_getpages(object, m, count, reqpage) 697 vm_object_t object; 698 vm_page_t *m; 699 int count; 700 int reqpage; 701 { 702 int rtval; 703 struct vnode *vp; 704 int bytes = count * PAGE_SIZE; 705 int vfslocked; 706 707 vp = object->handle; 708 VM_OBJECT_UNLOCK(object); 709 vfslocked = VFS_LOCK_GIANT(vp->v_mount); 710 rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0); 711 KASSERT(rtval != EOPNOTSUPP, 712 ("vnode_pager: FS getpages not implemented\n")); 713 VFS_UNLOCK_GIANT(vfslocked); 714 VM_OBJECT_LOCK(object); 715 return rtval; 716 } 717 718 /* 719 * This is now called from local media FS's to operate against their 720 * own vnodes if they fail to implement VOP_GETPAGES. 721 */ 722 int 723 vnode_pager_generic_getpages(vp, m, bytecount, reqpage) 724 struct vnode *vp; 725 vm_page_t *m; 726 int bytecount; 727 int reqpage; 728 { 729 vm_object_t object; 730 vm_offset_t kva; 731 off_t foff, tfoff, nextoff; 732 int i, j, size, bsize, first, firstaddr; 733 struct bufobj *bo; 734 int runpg; 735 int runend; 736 struct buf *bp; 737 int count; 738 int error = 0; 739 740 object = vp->v_object; 741 count = bytecount / PAGE_SIZE; 742 743 KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, 744 ("vnode_pager_generic_getpages does not support devices")); 745 if (vp->v_mount == NULL) 746 return VM_PAGER_BAD; 747 748 bsize = vp->v_mount->mnt_stat.f_iosize; 749 750 /* get the UNDERLYING device for the file with VOP_BMAP() */ 751 752 /* 753 * originally, we did not check for an error return value -- assuming 754 * an fs always has a bmap entry point -- that assumption is wrong!!! 755 */ 756 foff = IDX_TO_OFF(m[reqpage]->pindex); 757 758 /* 759 * if we can't bmap, use old VOP code 760 */ 761 if (VOP_BMAP(vp, 0, &bo, 0, NULL, NULL)) { 762 VM_OBJECT_LOCK(object); 763 vm_page_lock_queues(); 764 for (i = 0; i < count; i++) 765 if (i != reqpage) 766 vm_page_free(m[i]); 767 vm_page_unlock_queues(); 768 cnt.v_vnodein++; 769 cnt.v_vnodepgsin++; 770 error = vnode_pager_input_old(object, m[reqpage]); 771 VM_OBJECT_UNLOCK(object); 772 return (error); 773 774 /* 775 * if the blocksize is smaller than a page size, then use 776 * special small filesystem code. NFS sometimes has a small 777 * blocksize, but it can handle large reads itself. 778 */ 779 } else if ((PAGE_SIZE / bsize) > 1 && 780 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 781 VM_OBJECT_LOCK(object); 782 vm_page_lock_queues(); 783 for (i = 0; i < count; i++) 784 if (i != reqpage) 785 vm_page_free(m[i]); 786 vm_page_unlock_queues(); 787 VM_OBJECT_UNLOCK(object); 788 cnt.v_vnodein++; 789 cnt.v_vnodepgsin++; 790 return vnode_pager_input_smlfs(object, m[reqpage]); 791 } 792 793 /* 794 * If we have a completely valid page available to us, we can 795 * clean up and return. Otherwise we have to re-read the 796 * media. 797 */ 798 VM_OBJECT_LOCK(object); 799 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 800 vm_page_lock_queues(); 801 for (i = 0; i < count; i++) 802 if (i != reqpage) 803 vm_page_free(m[i]); 804 vm_page_unlock_queues(); 805 VM_OBJECT_UNLOCK(object); 806 return VM_PAGER_OK; 807 } 808 m[reqpage]->valid = 0; 809 VM_OBJECT_UNLOCK(object); 810 811 /* 812 * here on direct device I/O 813 */ 814 firstaddr = -1; 815 816 /* 817 * calculate the run that includes the required page 818 */ 819 for (first = 0, i = 0; i < count; i = runend) { 820 firstaddr = vnode_pager_addr(vp, 821 IDX_TO_OFF(m[i]->pindex), &runpg); 822 if (firstaddr == -1) { 823 VM_OBJECT_LOCK(object); 824 if (i == reqpage && foff < object->un_pager.vnp.vnp_size) { 825 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %d, foff: 0x%jx%08jx, vnp_size: 0x%jx%08jx", 826 firstaddr, (uintmax_t)(foff >> 32), 827 (uintmax_t)foff, 828 (uintmax_t) 829 (object->un_pager.vnp.vnp_size >> 32), 830 (uintmax_t)object->un_pager.vnp.vnp_size); 831 } 832 vm_page_lock_queues(); 833 vm_page_free(m[i]); 834 vm_page_unlock_queues(); 835 VM_OBJECT_UNLOCK(object); 836 runend = i + 1; 837 first = runend; 838 continue; 839 } 840 runend = i + runpg; 841 if (runend <= reqpage) { 842 VM_OBJECT_LOCK(object); 843 vm_page_lock_queues(); 844 for (j = i; j < runend; j++) 845 vm_page_free(m[j]); 846 vm_page_unlock_queues(); 847 VM_OBJECT_UNLOCK(object); 848 } else { 849 if (runpg < (count - first)) { 850 VM_OBJECT_LOCK(object); 851 vm_page_lock_queues(); 852 for (i = first + runpg; i < count; i++) 853 vm_page_free(m[i]); 854 vm_page_unlock_queues(); 855 VM_OBJECT_UNLOCK(object); 856 count = first + runpg; 857 } 858 break; 859 } 860 first = runend; 861 } 862 863 /* 864 * the first and last page have been calculated now, move input pages 865 * to be zero based... 866 */ 867 if (first != 0) { 868 for (i = first; i < count; i++) { 869 m[i - first] = m[i]; 870 } 871 count -= first; 872 reqpage -= first; 873 } 874 875 /* 876 * calculate the file virtual address for the transfer 877 */ 878 foff = IDX_TO_OFF(m[0]->pindex); 879 880 /* 881 * calculate the size of the transfer 882 */ 883 size = count * PAGE_SIZE; 884 KASSERT(count > 0, ("zero count")); 885 if ((foff + size) > object->un_pager.vnp.vnp_size) 886 size = object->un_pager.vnp.vnp_size - foff; 887 KASSERT(size > 0, ("zero size")); 888 889 /* 890 * round up physical size for real devices. 891 */ 892 if (1) { 893 int secmask = bo->bo_bsize - 1; 894 KASSERT(secmask < PAGE_SIZE && secmask > 0, 895 ("vnode_pager_generic_getpages: sector size %d too large", 896 secmask + 1)); 897 size = (size + secmask) & ~secmask; 898 } 899 900 bp = getpbuf(&vnode_pbuf_freecnt); 901 kva = (vm_offset_t) bp->b_data; 902 903 /* 904 * and map the pages to be read into the kva 905 */ 906 pmap_qenter(kva, m, count); 907 908 /* build a minimal buffer header */ 909 bp->b_iocmd = BIO_READ; 910 bp->b_iodone = bdone; 911 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 912 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 913 bp->b_rcred = crhold(curthread->td_ucred); 914 bp->b_wcred = crhold(curthread->td_ucred); 915 bp->b_blkno = firstaddr; 916 pbgetbo(bo, bp); 917 bp->b_bcount = size; 918 bp->b_bufsize = size; 919 bp->b_runningbufspace = bp->b_bufsize; 920 runningbufspace += bp->b_runningbufspace; 921 922 cnt.v_vnodein++; 923 cnt.v_vnodepgsin += count; 924 925 /* do the input */ 926 bp->b_iooffset = dbtob(bp->b_blkno); 927 bstrategy(bp); 928 929 bwait(bp, PVM, "vnread"); 930 931 if ((bp->b_ioflags & BIO_ERROR) != 0) 932 error = EIO; 933 934 if (!error) { 935 if (size != count * PAGE_SIZE) 936 bzero((caddr_t) kva + size, PAGE_SIZE * count - size); 937 } 938 pmap_qremove(kva, count); 939 940 /* 941 * free the buffer header back to the swap buffer pool 942 */ 943 pbrelbo(bp); 944 relpbuf(bp, &vnode_pbuf_freecnt); 945 946 VM_OBJECT_LOCK(object); 947 vm_page_lock_queues(); 948 for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { 949 vm_page_t mt; 950 951 nextoff = tfoff + PAGE_SIZE; 952 mt = m[i]; 953 954 if (nextoff <= object->un_pager.vnp.vnp_size) { 955 /* 956 * Read filled up entire page. 957 */ 958 mt->valid = VM_PAGE_BITS_ALL; 959 vm_page_undirty(mt); /* should be an assert? XXX */ 960 pmap_clear_modify(mt); 961 } else { 962 /* 963 * Read did not fill up entire page. Since this 964 * is getpages, the page may be mapped, so we have 965 * to zero the invalid portions of the page even 966 * though we aren't setting them valid. 967 * 968 * Currently we do not set the entire page valid, 969 * we just try to clear the piece that we couldn't 970 * read. 971 */ 972 vm_page_set_validclean(mt, 0, 973 object->un_pager.vnp.vnp_size - tfoff); 974 /* handled by vm_fault now */ 975 /* vm_page_zero_invalid(mt, FALSE); */ 976 } 977 978 if (i != reqpage) { 979 980 /* 981 * whether or not to leave the page activated is up in 982 * the air, but we should put the page on a page queue 983 * somewhere. (it already is in the object). Result: 984 * It appears that empirical results show that 985 * deactivating pages is best. 986 */ 987 988 /* 989 * just in case someone was asking for this page we 990 * now tell them that it is ok to use 991 */ 992 if (!error) { 993 if (mt->flags & PG_WANTED) 994 vm_page_activate(mt); 995 else 996 vm_page_deactivate(mt); 997 vm_page_wakeup(mt); 998 } else { 999 vm_page_free(mt); 1000 } 1001 } 1002 } 1003 vm_page_unlock_queues(); 1004 VM_OBJECT_UNLOCK(object); 1005 if (error) { 1006 printf("vnode_pager_getpages: I/O read error\n"); 1007 } 1008 return (error ? VM_PAGER_ERROR : VM_PAGER_OK); 1009 } 1010 1011 /* 1012 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 1013 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 1014 * vnode_pager_generic_putpages() to implement the previous behaviour. 1015 * 1016 * All other FS's should use the bypass to get to the local media 1017 * backing vp's VOP_PUTPAGES. 1018 */ 1019 static void 1020 vnode_pager_putpages(object, m, count, sync, rtvals) 1021 vm_object_t object; 1022 vm_page_t *m; 1023 int count; 1024 boolean_t sync; 1025 int *rtvals; 1026 { 1027 int rtval; 1028 struct vnode *vp; 1029 struct mount *mp; 1030 int bytes = count * PAGE_SIZE; 1031 1032 /* 1033 * Force synchronous operation if we are extremely low on memory 1034 * to prevent a low-memory deadlock. VOP operations often need to 1035 * allocate more memory to initiate the I/O ( i.e. do a BMAP 1036 * operation ). The swapper handles the case by limiting the amount 1037 * of asynchronous I/O, but that sort of solution doesn't scale well 1038 * for the vnode pager without a lot of work. 1039 * 1040 * Also, the backing vnode's iodone routine may not wake the pageout 1041 * daemon up. This should be probably be addressed XXX. 1042 */ 1043 1044 if ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_pageout_free_min) 1045 sync |= OBJPC_SYNC; 1046 1047 /* 1048 * Call device-specific putpages function 1049 */ 1050 vp = object->handle; 1051 VM_OBJECT_UNLOCK(object); 1052 if (vp->v_type != VREG) 1053 mp = NULL; 1054 (void)vn_start_write(vp, &mp, V_WAIT); 1055 rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0); 1056 KASSERT(rtval != EOPNOTSUPP, 1057 ("vnode_pager: stale FS putpages\n")); 1058 vn_finished_write(mp); 1059 VM_OBJECT_LOCK(object); 1060 } 1061 1062 1063 /* 1064 * This is now called from local media FS's to operate against their 1065 * own vnodes if they fail to implement VOP_PUTPAGES. 1066 * 1067 * This is typically called indirectly via the pageout daemon and 1068 * clustering has already typically occured, so in general we ask the 1069 * underlying filesystem to write the data out asynchronously rather 1070 * then delayed. 1071 */ 1072 int 1073 vnode_pager_generic_putpages(vp, m, bytecount, flags, rtvals) 1074 struct vnode *vp; 1075 vm_page_t *m; 1076 int bytecount; 1077 int flags; 1078 int *rtvals; 1079 { 1080 int i; 1081 vm_object_t object; 1082 int count; 1083 1084 int maxsize, ncount; 1085 vm_ooffset_t poffset; 1086 struct uio auio; 1087 struct iovec aiov; 1088 int error; 1089 int ioflags; 1090 1091 object = vp->v_object; 1092 count = bytecount / PAGE_SIZE; 1093 1094 for (i = 0; i < count; i++) 1095 rtvals[i] = VM_PAGER_AGAIN; 1096 1097 if ((int64_t)m[0]->pindex < 0) { 1098 printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%lx)\n", 1099 (long)m[0]->pindex, (u_long)m[0]->dirty); 1100 rtvals[0] = VM_PAGER_BAD; 1101 return VM_PAGER_BAD; 1102 } 1103 1104 maxsize = count * PAGE_SIZE; 1105 ncount = count; 1106 1107 poffset = IDX_TO_OFF(m[0]->pindex); 1108 1109 /* 1110 * If the page-aligned write is larger then the actual file we 1111 * have to invalidate pages occuring beyond the file EOF. However, 1112 * there is an edge case where a file may not be page-aligned where 1113 * the last page is partially invalid. In this case the filesystem 1114 * may not properly clear the dirty bits for the entire page (which 1115 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 1116 * With the page locked we are free to fix-up the dirty bits here. 1117 * 1118 * We do not under any circumstances truncate the valid bits, as 1119 * this will screw up bogus page replacement. 1120 */ 1121 if (maxsize + poffset > object->un_pager.vnp.vnp_size) { 1122 if (object->un_pager.vnp.vnp_size > poffset) { 1123 int pgoff; 1124 1125 maxsize = object->un_pager.vnp.vnp_size - poffset; 1126 ncount = btoc(maxsize); 1127 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 1128 vm_page_lock_queues(); 1129 vm_page_clear_dirty(m[ncount - 1], pgoff, 1130 PAGE_SIZE - pgoff); 1131 vm_page_unlock_queues(); 1132 } 1133 } else { 1134 maxsize = 0; 1135 ncount = 0; 1136 } 1137 if (ncount < count) { 1138 for (i = ncount; i < count; i++) { 1139 rtvals[i] = VM_PAGER_BAD; 1140 } 1141 } 1142 } 1143 1144 /* 1145 * pageouts are already clustered, use IO_ASYNC t o force a bawrite() 1146 * rather then a bdwrite() to prevent paging I/O from saturating 1147 * the buffer cache. Dummy-up the sequential heuristic to cause 1148 * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, 1149 * the system decides how to cluster. 1150 */ 1151 ioflags = IO_VMIO; 1152 if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) 1153 ioflags |= IO_SYNC; 1154 else if ((flags & VM_PAGER_CLUSTER_OK) == 0) 1155 ioflags |= IO_ASYNC; 1156 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 1157 ioflags |= IO_SEQMAX << IO_SEQSHIFT; 1158 1159 aiov.iov_base = (caddr_t) 0; 1160 aiov.iov_len = maxsize; 1161 auio.uio_iov = &aiov; 1162 auio.uio_iovcnt = 1; 1163 auio.uio_offset = poffset; 1164 auio.uio_segflg = UIO_NOCOPY; 1165 auio.uio_rw = UIO_WRITE; 1166 auio.uio_resid = maxsize; 1167 auio.uio_td = (struct thread *) 0; 1168 error = VOP_WRITE(vp, &auio, ioflags, curthread->td_ucred); 1169 cnt.v_vnodeout++; 1170 cnt.v_vnodepgsout += ncount; 1171 1172 if (error) { 1173 printf("vnode_pager_putpages: I/O error %d\n", error); 1174 } 1175 if (auio.uio_resid) { 1176 printf("vnode_pager_putpages: residual I/O %d at %lu\n", 1177 auio.uio_resid, (u_long)m[0]->pindex); 1178 } 1179 for (i = 0; i < ncount; i++) { 1180 rtvals[i] = VM_PAGER_OK; 1181 } 1182 return rtvals[0]; 1183 } 1184 1185 struct vnode * 1186 vnode_pager_lock(vm_object_t first_object) 1187 { 1188 struct vnode *vp; 1189 vm_object_t backing_object, object; 1190 1191 VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); 1192 for (object = first_object; object != NULL; object = backing_object) { 1193 if (object->type != OBJT_VNODE) { 1194 if ((backing_object = object->backing_object) != NULL) 1195 VM_OBJECT_LOCK(backing_object); 1196 if (object != first_object) 1197 VM_OBJECT_UNLOCK(object); 1198 continue; 1199 } 1200 retry: 1201 if (object->flags & OBJ_DEAD) { 1202 if (object != first_object) 1203 VM_OBJECT_UNLOCK(object); 1204 return NULL; 1205 } 1206 vp = object->handle; 1207 VI_LOCK(vp); 1208 VM_OBJECT_UNLOCK(object); 1209 if (first_object != object) 1210 VM_OBJECT_UNLOCK(first_object); 1211 if (vget(vp, LK_CANRECURSE | LK_INTERLOCK | LK_NOPAUSE | 1212 LK_RETRY | LK_SHARED, curthread)) { 1213 VM_OBJECT_LOCK(first_object); 1214 if (object != first_object) 1215 VM_OBJECT_LOCK(object); 1216 if (object->type != OBJT_VNODE) { 1217 if (object != first_object) 1218 VM_OBJECT_UNLOCK(object); 1219 return NULL; 1220 } 1221 printf("vnode_pager_lock: retrying\n"); 1222 goto retry; 1223 } 1224 VM_OBJECT_LOCK(first_object); 1225 return (vp); 1226 } 1227 return NULL; 1228 } 1229