1 /* 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 1993, 1994 John S. Dyson 6 * Copyright (c) 1995, David Greenman 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41 * $FreeBSD$ 42 */ 43 44 /* 45 * Page to/from files (vnodes). 46 */ 47 48 /* 49 * TODO: 50 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 51 * greatly re-simplify the vnode_pager. 52 */ 53 54 #include <sys/param.h> 55 #include <sys/systm.h> 56 #include <sys/proc.h> 57 #include <sys/vnode.h> 58 #include <sys/mount.h> 59 #include <sys/bio.h> 60 #include <sys/buf.h> 61 #include <sys/vmmeter.h> 62 #include <sys/conf.h> 63 64 #include <vm/vm.h> 65 #include <vm/vm_object.h> 66 #include <vm/vm_page.h> 67 #include <vm/vm_pager.h> 68 #include <vm/vm_map.h> 69 #include <vm/vnode_pager.h> 70 #include <vm/vm_extern.h> 71 72 static void vnode_pager_init __P((void)); 73 static vm_offset_t vnode_pager_addr __P((struct vnode *vp, vm_ooffset_t address, 74 int *run)); 75 static void vnode_pager_iodone __P((struct buf *bp)); 76 static int vnode_pager_input_smlfs __P((vm_object_t object, vm_page_t m)); 77 static int vnode_pager_input_old __P((vm_object_t object, vm_page_t m)); 78 static void vnode_pager_dealloc __P((vm_object_t)); 79 static int vnode_pager_getpages __P((vm_object_t, vm_page_t *, int, int)); 80 static void vnode_pager_putpages __P((vm_object_t, vm_page_t *, int, boolean_t, int *)); 81 static boolean_t vnode_pager_haspage __P((vm_object_t, vm_pindex_t, int *, int *)); 82 83 struct pagerops vnodepagerops = { 84 vnode_pager_init, 85 vnode_pager_alloc, 86 vnode_pager_dealloc, 87 vnode_pager_getpages, 88 vnode_pager_putpages, 89 vnode_pager_haspage, 90 NULL 91 }; 92 93 int vnode_pbuf_freecnt; 94 95 void 96 vnode_pager_init(void) 97 { 98 99 vnode_pbuf_freecnt = nswbuf / 2 + 1; 100 } 101 102 /* 103 * Allocate (or lookup) pager for a vnode. 104 * Handle is a vnode pointer. 105 */ 106 vm_object_t 107 vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 108 vm_ooffset_t offset) 109 { 110 vm_object_t object; 111 struct vnode *vp; 112 113 GIANT_REQUIRED; 114 115 /* 116 * Pageout to vnode, no can do yet. 117 */ 118 if (handle == NULL) 119 return (NULL); 120 121 vp = (struct vnode *) handle; 122 123 /* 124 * Prevent race condition when allocating the object. This 125 * can happen with NFS vnodes since the nfsnode isn't locked. 126 */ 127 while (vp->v_flag & VOLOCK) { 128 vp->v_flag |= VOWANT; 129 tsleep(vp, PVM, "vnpobj", 0); 130 } 131 vp->v_flag |= VOLOCK; 132 133 /* 134 * If the object is being terminated, wait for it to 135 * go away. 136 */ 137 while (((object = vp->v_object) != NULL) && 138 (object->flags & OBJ_DEAD)) { 139 tsleep(object, PVM, "vadead", 0); 140 } 141 142 if (vp->v_usecount == 0) 143 panic("vnode_pager_alloc: no vnode reference"); 144 145 if (object == NULL) { 146 /* 147 * And an object of the appropriate size 148 */ 149 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 150 object->flags = 0; 151 152 object->un_pager.vnp.vnp_size = size; 153 154 object->handle = handle; 155 vp->v_object = object; 156 vp->v_usecount++; 157 } else { 158 object->ref_count++; 159 vp->v_usecount++; 160 } 161 162 vp->v_flag &= ~VOLOCK; 163 if (vp->v_flag & VOWANT) { 164 vp->v_flag &= ~VOWANT; 165 wakeup(vp); 166 } 167 return (object); 168 } 169 170 static void 171 vnode_pager_dealloc(object) 172 vm_object_t object; 173 { 174 struct vnode *vp = object->handle; 175 176 GIANT_REQUIRED; 177 if (vp == NULL) 178 panic("vnode_pager_dealloc: pager already dealloced"); 179 180 vm_object_pip_wait(object, "vnpdea"); 181 182 object->handle = NULL; 183 object->type = OBJT_DEAD; 184 vp->v_object = NULL; 185 vp->v_flag &= ~(VTEXT | VOBJBUF); 186 } 187 188 static boolean_t 189 vnode_pager_haspage(object, pindex, before, after) 190 vm_object_t object; 191 vm_pindex_t pindex; 192 int *before; 193 int *after; 194 { 195 struct vnode *vp = object->handle; 196 daddr_t bn; 197 int err; 198 daddr_t reqblock; 199 int poff; 200 int bsize; 201 int pagesperblock, blocksperpage; 202 203 GIANT_REQUIRED; 204 /* 205 * If no vp or vp is doomed or marked transparent to VM, we do not 206 * have the page. 207 */ 208 if ((vp == NULL) || (vp->v_flag & VDOOMED)) 209 return FALSE; 210 211 /* 212 * If filesystem no longer mounted or offset beyond end of file we do 213 * not have the page. 214 */ 215 if ((vp->v_mount == NULL) || 216 (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)) 217 return FALSE; 218 219 bsize = vp->v_mount->mnt_stat.f_iosize; 220 pagesperblock = bsize / PAGE_SIZE; 221 blocksperpage = 0; 222 if (pagesperblock > 0) { 223 reqblock = pindex / pagesperblock; 224 } else { 225 blocksperpage = (PAGE_SIZE / bsize); 226 reqblock = pindex * blocksperpage; 227 } 228 err = VOP_BMAP(vp, reqblock, (struct vnode **) 0, &bn, 229 after, before); 230 if (err) 231 return TRUE; 232 if ( bn == -1) 233 return FALSE; 234 if (pagesperblock > 0) { 235 poff = pindex - (reqblock * pagesperblock); 236 if (before) { 237 *before *= pagesperblock; 238 *before += poff; 239 } 240 if (after) { 241 int numafter; 242 *after *= pagesperblock; 243 numafter = pagesperblock - (poff + 1); 244 if (IDX_TO_OFF(pindex + numafter) > object->un_pager.vnp.vnp_size) { 245 numafter = OFF_TO_IDX((object->un_pager.vnp.vnp_size - IDX_TO_OFF(pindex))); 246 } 247 *after += numafter; 248 } 249 } else { 250 if (before) { 251 *before /= blocksperpage; 252 } 253 254 if (after) { 255 *after /= blocksperpage; 256 } 257 } 258 return TRUE; 259 } 260 261 /* 262 * Lets the VM system know about a change in size for a file. 263 * We adjust our own internal size and flush any cached pages in 264 * the associated object that are affected by the size change. 265 * 266 * Note: this routine may be invoked as a result of a pager put 267 * operation (possibly at object termination time), so we must be careful. 268 */ 269 void 270 vnode_pager_setsize(vp, nsize) 271 struct vnode *vp; 272 vm_ooffset_t nsize; 273 { 274 vm_pindex_t nobjsize; 275 vm_object_t object = vp->v_object; 276 277 GIANT_REQUIRED; 278 279 if (object == NULL) 280 return; 281 282 /* 283 * Hasn't changed size 284 */ 285 if (nsize == object->un_pager.vnp.vnp_size) 286 return; 287 288 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 289 290 /* 291 * File has shrunk. Toss any cached pages beyond the new EOF. 292 */ 293 if (nsize < object->un_pager.vnp.vnp_size) { 294 vm_freeze_copyopts(object, OFF_TO_IDX(nsize), object->size); 295 if (nobjsize < object->size) { 296 vm_object_page_remove(object, nobjsize, object->size, 297 FALSE); 298 } 299 /* 300 * this gets rid of garbage at the end of a page that is now 301 * only partially backed by the vnode... 302 */ 303 if (nsize & PAGE_MASK) { 304 vm_offset_t kva; 305 vm_page_t m; 306 307 m = vm_page_lookup(object, OFF_TO_IDX(nsize)); 308 if (m) { 309 int base = (int)nsize & PAGE_MASK; 310 int size = PAGE_SIZE - base; 311 312 /* 313 * Clear out partial-page garbage in case 314 * the page has been mapped. 315 */ 316 kva = vm_pager_map_page(m); 317 bzero((caddr_t)kva + base, size); 318 vm_pager_unmap_page(kva); 319 320 /* 321 * Clear out partial-page dirty bits. This 322 * has the side effect of setting the valid 323 * bits, but that is ok. There are a bunch 324 * of places in the VM system where we expected 325 * m->dirty == VM_PAGE_BITS_ALL. The file EOF 326 * case is one of them. If the page is still 327 * partially dirty, make it fully dirty. 328 */ 329 vm_page_set_validclean(m, base, size); 330 if (m->dirty != 0) 331 m->dirty = VM_PAGE_BITS_ALL; 332 } 333 } 334 } 335 object->un_pager.vnp.vnp_size = nsize; 336 object->size = nobjsize; 337 } 338 339 /* 340 * calculate the linear (byte) disk address of specified virtual 341 * file address 342 */ 343 static vm_offset_t 344 vnode_pager_addr(vp, address, run) 345 struct vnode *vp; 346 vm_ooffset_t address; 347 int *run; 348 { 349 int rtaddress; 350 int bsize; 351 daddr_t block; 352 struct vnode *rtvp; 353 int err; 354 daddr_t vblock; 355 int voffset; 356 357 GIANT_REQUIRED; 358 if ((int) address < 0) 359 return -1; 360 361 if (vp->v_mount == NULL) 362 return -1; 363 364 bsize = vp->v_mount->mnt_stat.f_iosize; 365 vblock = address / bsize; 366 voffset = address % bsize; 367 368 err = VOP_BMAP(vp, vblock, &rtvp, &block, run, NULL); 369 370 if (err || (block == -1)) 371 rtaddress = -1; 372 else { 373 rtaddress = block + voffset / DEV_BSIZE; 374 if( run) { 375 *run += 1; 376 *run *= bsize/PAGE_SIZE; 377 *run -= voffset/PAGE_SIZE; 378 } 379 } 380 381 return rtaddress; 382 } 383 384 /* 385 * interrupt routine for I/O completion 386 */ 387 static void 388 vnode_pager_iodone(bp) 389 struct buf *bp; 390 { 391 bp->b_flags |= B_DONE; 392 wakeup(bp); 393 } 394 395 /* 396 * small block file system vnode pager input 397 */ 398 static int 399 vnode_pager_input_smlfs(object, m) 400 vm_object_t object; 401 vm_page_t m; 402 { 403 int i; 404 int s; 405 struct vnode *dp, *vp; 406 struct buf *bp; 407 vm_offset_t kva; 408 int fileaddr; 409 vm_offset_t bsize; 410 int error = 0; 411 412 GIANT_REQUIRED; 413 414 vp = object->handle; 415 if (vp->v_mount == NULL) 416 return VM_PAGER_BAD; 417 418 bsize = vp->v_mount->mnt_stat.f_iosize; 419 420 VOP_BMAP(vp, 0, &dp, 0, NULL, NULL); 421 422 kva = vm_pager_map_page(m); 423 424 for (i = 0; i < PAGE_SIZE / bsize; i++) { 425 426 if (vm_page_bits(i * bsize, bsize) & m->valid) 427 continue; 428 429 fileaddr = vnode_pager_addr(vp, 430 IDX_TO_OFF(m->pindex) + i * bsize, (int *)0); 431 if (fileaddr != -1) { 432 bp = getpbuf(&vnode_pbuf_freecnt); 433 434 /* build a minimal buffer header */ 435 bp->b_iocmd = BIO_READ; 436 bp->b_iodone = vnode_pager_iodone; 437 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 438 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 439 bp->b_rcred = crhold(curthread->td_proc->p_ucred); 440 bp->b_wcred = crhold(curthread->td_proc->p_ucred); 441 bp->b_data = (caddr_t) kva + i * bsize; 442 bp->b_blkno = fileaddr; 443 pbgetvp(dp, bp); 444 bp->b_bcount = bsize; 445 bp->b_bufsize = bsize; 446 bp->b_runningbufspace = bp->b_bufsize; 447 runningbufspace += bp->b_runningbufspace; 448 449 /* do the input */ 450 BUF_STRATEGY(bp); 451 452 /* we definitely need to be at splvm here */ 453 454 s = splvm(); 455 while ((bp->b_flags & B_DONE) == 0) { 456 tsleep(bp, PVM, "vnsrd", 0); 457 } 458 splx(s); 459 if ((bp->b_ioflags & BIO_ERROR) != 0) 460 error = EIO; 461 462 /* 463 * free the buffer header back to the swap buffer pool 464 */ 465 relpbuf(bp, &vnode_pbuf_freecnt); 466 if (error) 467 break; 468 469 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 470 } else { 471 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 472 bzero((caddr_t) kva + i * bsize, bsize); 473 } 474 } 475 vm_pager_unmap_page(kva); 476 pmap_clear_modify(m); 477 vm_page_flag_clear(m, PG_ZERO); 478 if (error) { 479 return VM_PAGER_ERROR; 480 } 481 return VM_PAGER_OK; 482 483 } 484 485 486 /* 487 * old style vnode pager output routine 488 */ 489 static int 490 vnode_pager_input_old(object, m) 491 vm_object_t object; 492 vm_page_t m; 493 { 494 struct uio auio; 495 struct iovec aiov; 496 int error; 497 int size; 498 vm_offset_t kva; 499 struct vnode *vp; 500 501 GIANT_REQUIRED; 502 error = 0; 503 504 /* 505 * Return failure if beyond current EOF 506 */ 507 if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { 508 return VM_PAGER_BAD; 509 } else { 510 size = PAGE_SIZE; 511 if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) 512 size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); 513 514 /* 515 * Allocate a kernel virtual address and initialize so that 516 * we can use VOP_READ/WRITE routines. 517 */ 518 kva = vm_pager_map_page(m); 519 520 vp = object->handle; 521 aiov.iov_base = (caddr_t) kva; 522 aiov.iov_len = size; 523 auio.uio_iov = &aiov; 524 auio.uio_iovcnt = 1; 525 auio.uio_offset = IDX_TO_OFF(m->pindex); 526 auio.uio_segflg = UIO_SYSSPACE; 527 auio.uio_rw = UIO_READ; 528 auio.uio_resid = size; 529 auio.uio_td = curthread; 530 531 error = VOP_READ(vp, &auio, 0, curthread->td_proc->p_ucred); 532 if (!error) { 533 int count = size - auio.uio_resid; 534 535 if (count == 0) 536 error = EINVAL; 537 else if (count != PAGE_SIZE) 538 bzero((caddr_t) kva + count, PAGE_SIZE - count); 539 } 540 vm_pager_unmap_page(kva); 541 } 542 pmap_clear_modify(m); 543 vm_page_undirty(m); 544 vm_page_flag_clear(m, PG_ZERO); 545 if (!error) 546 m->valid = VM_PAGE_BITS_ALL; 547 return error ? VM_PAGER_ERROR : VM_PAGER_OK; 548 } 549 550 /* 551 * generic vnode pager input routine 552 */ 553 554 /* 555 * Local media VFS's that do not implement their own VOP_GETPAGES 556 * should have their VOP_GETPAGES should call to 557 * vnode_pager_generic_getpages() to implement the previous behaviour. 558 * 559 * All other FS's should use the bypass to get to the local media 560 * backing vp's VOP_GETPAGES. 561 */ 562 static int 563 vnode_pager_getpages(object, m, count, reqpage) 564 vm_object_t object; 565 vm_page_t *m; 566 int count; 567 int reqpage; 568 { 569 int rtval; 570 struct vnode *vp; 571 int bytes = count * PAGE_SIZE; 572 573 GIANT_REQUIRED; 574 vp = object->handle; 575 rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0); 576 KASSERT(rtval != EOPNOTSUPP, 577 ("vnode_pager: FS getpages not implemented\n")); 578 return rtval; 579 } 580 581 582 /* 583 * This is now called from local media FS's to operate against their 584 * own vnodes if they fail to implement VOP_GETPAGES. 585 */ 586 int 587 vnode_pager_generic_getpages(vp, m, bytecount, reqpage) 588 struct vnode *vp; 589 vm_page_t *m; 590 int bytecount; 591 int reqpage; 592 { 593 vm_object_t object; 594 vm_offset_t kva; 595 off_t foff, tfoff, nextoff; 596 int i, size, bsize, first, firstaddr; 597 struct vnode *dp; 598 int runpg; 599 int runend; 600 struct buf *bp; 601 int s; 602 int count; 603 int error = 0; 604 605 GIANT_REQUIRED; 606 object = vp->v_object; 607 count = bytecount / PAGE_SIZE; 608 609 if (vp->v_mount == NULL) 610 return VM_PAGER_BAD; 611 612 bsize = vp->v_mount->mnt_stat.f_iosize; 613 614 /* get the UNDERLYING device for the file with VOP_BMAP() */ 615 616 /* 617 * originally, we did not check for an error return value -- assuming 618 * an fs always has a bmap entry point -- that assumption is wrong!!! 619 */ 620 foff = IDX_TO_OFF(m[reqpage]->pindex); 621 622 /* 623 * if we can't bmap, use old VOP code 624 */ 625 if (VOP_BMAP(vp, 0, &dp, 0, NULL, NULL)) { 626 for (i = 0; i < count; i++) { 627 if (i != reqpage) { 628 vm_page_free(m[i]); 629 } 630 } 631 cnt.v_vnodein++; 632 cnt.v_vnodepgsin++; 633 return vnode_pager_input_old(object, m[reqpage]); 634 635 /* 636 * if the blocksize is smaller than a page size, then use 637 * special small filesystem code. NFS sometimes has a small 638 * blocksize, but it can handle large reads itself. 639 */ 640 } else if ((PAGE_SIZE / bsize) > 1 && 641 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 642 for (i = 0; i < count; i++) { 643 if (i != reqpage) { 644 vm_page_free(m[i]); 645 } 646 } 647 cnt.v_vnodein++; 648 cnt.v_vnodepgsin++; 649 return vnode_pager_input_smlfs(object, m[reqpage]); 650 } 651 652 /* 653 * If we have a completely valid page available to us, we can 654 * clean up and return. Otherwise we have to re-read the 655 * media. 656 */ 657 658 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 659 for (i = 0; i < count; i++) { 660 if (i != reqpage) 661 vm_page_free(m[i]); 662 } 663 return VM_PAGER_OK; 664 } 665 m[reqpage]->valid = 0; 666 667 /* 668 * here on direct device I/O 669 */ 670 671 firstaddr = -1; 672 /* 673 * calculate the run that includes the required page 674 */ 675 for(first = 0, i = 0; i < count; i = runend) { 676 firstaddr = vnode_pager_addr(vp, 677 IDX_TO_OFF(m[i]->pindex), &runpg); 678 if (firstaddr == -1) { 679 if (i == reqpage && foff < object->un_pager.vnp.vnp_size) { 680 /* XXX no %qd in kernel. */ 681 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %d, foff: 0x%lx%08lx, vnp_size: 0x%lx%08lx", 682 firstaddr, (u_long)(foff >> 32), 683 (u_long)(u_int32_t)foff, 684 (u_long)(u_int32_t) 685 (object->un_pager.vnp.vnp_size >> 32), 686 (u_long)(u_int32_t) 687 object->un_pager.vnp.vnp_size); 688 } 689 vm_page_free(m[i]); 690 runend = i + 1; 691 first = runend; 692 continue; 693 } 694 runend = i + runpg; 695 if (runend <= reqpage) { 696 int j; 697 for (j = i; j < runend; j++) { 698 vm_page_free(m[j]); 699 } 700 } else { 701 if (runpg < (count - first)) { 702 for (i = first + runpg; i < count; i++) 703 vm_page_free(m[i]); 704 count = first + runpg; 705 } 706 break; 707 } 708 first = runend; 709 } 710 711 /* 712 * the first and last page have been calculated now, move input pages 713 * to be zero based... 714 */ 715 if (first != 0) { 716 for (i = first; i < count; i++) { 717 m[i - first] = m[i]; 718 } 719 count -= first; 720 reqpage -= first; 721 } 722 723 /* 724 * calculate the file virtual address for the transfer 725 */ 726 foff = IDX_TO_OFF(m[0]->pindex); 727 728 /* 729 * calculate the size of the transfer 730 */ 731 size = count * PAGE_SIZE; 732 if ((foff + size) > object->un_pager.vnp.vnp_size) 733 size = object->un_pager.vnp.vnp_size - foff; 734 735 /* 736 * round up physical size for real devices. 737 */ 738 if (dp->v_type == VBLK || dp->v_type == VCHR) { 739 int secmask = dp->v_rdev->si_bsize_phys - 1; 740 KASSERT(secmask < PAGE_SIZE, ("vnode_pager_generic_getpages: sector size %d too large\n", secmask + 1)); 741 size = (size + secmask) & ~secmask; 742 } 743 744 bp = getpbuf(&vnode_pbuf_freecnt); 745 kva = (vm_offset_t) bp->b_data; 746 747 /* 748 * and map the pages to be read into the kva 749 */ 750 pmap_qenter(kva, m, count); 751 752 /* build a minimal buffer header */ 753 bp->b_iocmd = BIO_READ; 754 bp->b_iodone = vnode_pager_iodone; 755 /* B_PHYS is not set, but it is nice to fill this in */ 756 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 757 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 758 bp->b_rcred = crhold(curthread->td_proc->p_ucred); 759 bp->b_wcred = crhold(curthread->td_proc->p_ucred); 760 bp->b_blkno = firstaddr; 761 pbgetvp(dp, bp); 762 bp->b_bcount = size; 763 bp->b_bufsize = size; 764 bp->b_runningbufspace = bp->b_bufsize; 765 runningbufspace += bp->b_runningbufspace; 766 767 cnt.v_vnodein++; 768 cnt.v_vnodepgsin += count; 769 770 /* do the input */ 771 BUF_STRATEGY(bp); 772 773 s = splvm(); 774 /* we definitely need to be at splvm here */ 775 776 while ((bp->b_flags & B_DONE) == 0) { 777 tsleep(bp, PVM, "vnread", 0); 778 } 779 splx(s); 780 if ((bp->b_ioflags & BIO_ERROR) != 0) 781 error = EIO; 782 783 if (!error) { 784 if (size != count * PAGE_SIZE) 785 bzero((caddr_t) kva + size, PAGE_SIZE * count - size); 786 } 787 pmap_qremove(kva, count); 788 789 /* 790 * free the buffer header back to the swap buffer pool 791 */ 792 relpbuf(bp, &vnode_pbuf_freecnt); 793 794 for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { 795 vm_page_t mt; 796 797 nextoff = tfoff + PAGE_SIZE; 798 mt = m[i]; 799 800 if (nextoff <= object->un_pager.vnp.vnp_size) { 801 /* 802 * Read filled up entire page. 803 */ 804 mt->valid = VM_PAGE_BITS_ALL; 805 vm_page_undirty(mt); /* should be an assert? XXX */ 806 pmap_clear_modify(mt); 807 } else { 808 /* 809 * Read did not fill up entire page. Since this 810 * is getpages, the page may be mapped, so we have 811 * to zero the invalid portions of the page even 812 * though we aren't setting them valid. 813 * 814 * Currently we do not set the entire page valid, 815 * we just try to clear the piece that we couldn't 816 * read. 817 */ 818 vm_page_set_validclean(mt, 0, 819 object->un_pager.vnp.vnp_size - tfoff); 820 /* handled by vm_fault now */ 821 /* vm_page_zero_invalid(mt, FALSE); */ 822 } 823 824 vm_page_flag_clear(mt, PG_ZERO); 825 if (i != reqpage) { 826 827 /* 828 * whether or not to leave the page activated is up in 829 * the air, but we should put the page on a page queue 830 * somewhere. (it already is in the object). Result: 831 * It appears that empirical results show that 832 * deactivating pages is best. 833 */ 834 835 /* 836 * just in case someone was asking for this page we 837 * now tell them that it is ok to use 838 */ 839 if (!error) { 840 if (mt->flags & PG_WANTED) 841 vm_page_activate(mt); 842 else 843 vm_page_deactivate(mt); 844 vm_page_wakeup(mt); 845 } else { 846 vm_page_free(mt); 847 } 848 } 849 } 850 if (error) { 851 printf("vnode_pager_getpages: I/O read error\n"); 852 } 853 return (error ? VM_PAGER_ERROR : VM_PAGER_OK); 854 } 855 856 /* 857 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 858 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 859 * vnode_pager_generic_putpages() to implement the previous behaviour. 860 * 861 * All other FS's should use the bypass to get to the local media 862 * backing vp's VOP_PUTPAGES. 863 */ 864 static void 865 vnode_pager_putpages(object, m, count, sync, rtvals) 866 vm_object_t object; 867 vm_page_t *m; 868 int count; 869 boolean_t sync; 870 int *rtvals; 871 { 872 int rtval; 873 struct vnode *vp; 874 struct mount *mp; 875 int bytes = count * PAGE_SIZE; 876 877 GIANT_REQUIRED; 878 /* 879 * Force synchronous operation if we are extremely low on memory 880 * to prevent a low-memory deadlock. VOP operations often need to 881 * allocate more memory to initiate the I/O ( i.e. do a BMAP 882 * operation ). The swapper handles the case by limiting the amount 883 * of asynchronous I/O, but that sort of solution doesn't scale well 884 * for the vnode pager without a lot of work. 885 * 886 * Also, the backing vnode's iodone routine may not wake the pageout 887 * daemon up. This should be probably be addressed XXX. 888 */ 889 890 if ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_pageout_free_min) 891 sync |= OBJPC_SYNC; 892 893 /* 894 * Call device-specific putpages function 895 */ 896 897 vp = object->handle; 898 if (vp->v_type != VREG) 899 mp = NULL; 900 (void)vn_start_write(vp, &mp, V_WAIT); 901 rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0); 902 KASSERT(rtval != EOPNOTSUPP, 903 ("vnode_pager: stale FS putpages\n")); 904 vn_finished_write(mp); 905 } 906 907 908 /* 909 * This is now called from local media FS's to operate against their 910 * own vnodes if they fail to implement VOP_PUTPAGES. 911 * 912 * This is typically called indirectly via the pageout daemon and 913 * clustering has already typically occured, so in general we ask the 914 * underlying filesystem to write the data out asynchronously rather 915 * then delayed. 916 */ 917 int 918 vnode_pager_generic_putpages(vp, m, bytecount, flags, rtvals) 919 struct vnode *vp; 920 vm_page_t *m; 921 int bytecount; 922 int flags; 923 int *rtvals; 924 { 925 int i; 926 vm_object_t object; 927 int count; 928 929 int maxsize, ncount; 930 vm_ooffset_t poffset; 931 struct uio auio; 932 struct iovec aiov; 933 int error; 934 int ioflags; 935 936 GIANT_REQUIRED; 937 object = vp->v_object; 938 count = bytecount / PAGE_SIZE; 939 940 for (i = 0; i < count; i++) 941 rtvals[i] = VM_PAGER_AGAIN; 942 943 if ((int) m[0]->pindex < 0) { 944 printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%x)\n", 945 (long)m[0]->pindex, m[0]->dirty); 946 rtvals[0] = VM_PAGER_BAD; 947 return VM_PAGER_BAD; 948 } 949 950 maxsize = count * PAGE_SIZE; 951 ncount = count; 952 953 poffset = IDX_TO_OFF(m[0]->pindex); 954 955 /* 956 * If the page-aligned write is larger then the actual file we 957 * have to invalidate pages occuring beyond the file EOF. However, 958 * there is an edge case where a file may not be page-aligned where 959 * the last page is partially invalid. In this case the filesystem 960 * may not properly clear the dirty bits for the entire page (which 961 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 962 * With the page locked we are free to fix-up the dirty bits here. 963 */ 964 if (maxsize + poffset > object->un_pager.vnp.vnp_size) { 965 if (object->un_pager.vnp.vnp_size > poffset) { 966 int pgoff; 967 968 maxsize = object->un_pager.vnp.vnp_size - poffset; 969 ncount = btoc(maxsize); 970 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 971 vm_page_clear_dirty(m[ncount - 1], pgoff, 972 PAGE_SIZE - pgoff); 973 } 974 } else { 975 maxsize = 0; 976 ncount = 0; 977 } 978 if (ncount < count) { 979 for (i = ncount; i < count; i++) { 980 rtvals[i] = VM_PAGER_BAD; 981 } 982 } 983 } 984 985 /* 986 * pageouts are already clustered, use IO_ASYNC t o force a bawrite() 987 * rather then a bdwrite() to prevent paging I/O from saturating 988 * the buffer cache. 989 */ 990 ioflags = IO_VMIO; 991 ioflags |= (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) ? IO_SYNC: IO_ASYNC; 992 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 993 994 aiov.iov_base = (caddr_t) 0; 995 aiov.iov_len = maxsize; 996 auio.uio_iov = &aiov; 997 auio.uio_iovcnt = 1; 998 auio.uio_offset = poffset; 999 auio.uio_segflg = UIO_NOCOPY; 1000 auio.uio_rw = UIO_WRITE; 1001 auio.uio_resid = maxsize; 1002 auio.uio_td = (struct thread *) 0; 1003 error = VOP_WRITE(vp, &auio, ioflags, curthread->td_proc->p_ucred); 1004 cnt.v_vnodeout++; 1005 cnt.v_vnodepgsout += ncount; 1006 1007 if (error) { 1008 printf("vnode_pager_putpages: I/O error %d\n", error); 1009 } 1010 if (auio.uio_resid) { 1011 printf("vnode_pager_putpages: residual I/O %d at %lu\n", 1012 auio.uio_resid, (u_long)m[0]->pindex); 1013 } 1014 for (i = 0; i < ncount; i++) { 1015 rtvals[i] = VM_PAGER_OK; 1016 } 1017 return rtvals[0]; 1018 } 1019 1020 struct vnode * 1021 vnode_pager_lock(object) 1022 vm_object_t object; 1023 { 1024 struct thread *td = curthread; /* XXX */ 1025 1026 GIANT_REQUIRED; 1027 1028 for (; object != NULL; object = object->backing_object) { 1029 if (object->type != OBJT_VNODE) 1030 continue; 1031 if (object->flags & OBJ_DEAD) { 1032 return NULL; 1033 } 1034 1035 /* XXX; If object->handle can change, we need to cache it. */ 1036 while (vget(object->handle, 1037 LK_NOPAUSE | LK_SHARED | LK_RETRY | LK_CANRECURSE, td)){ 1038 if ((object->flags & OBJ_DEAD) || (object->type != OBJT_VNODE)) 1039 return NULL; 1040 printf("vnode_pager_lock: retrying\n"); 1041 } 1042 return object->handle; 1043 } 1044 return NULL; 1045 } 1046