1 /* 2 * Copyright (c) 1994 John S. Dyson 3 * Copyright (c) 1990 University of Utah. 4 * Copyright (c) 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * the Systems Programming Group of the University of Utah Computer 9 * Science Department. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 40 * 41 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 42 * $Id: swap_pager.c,v 1.18 1994/11/13 15:36:48 davidg Exp $ 43 */ 44 45 /* 46 * Quick hack to page to dedicated partition(s). 47 * TODO: 48 * Add multiprocessor locks 49 * Deal with async writes in a better fashion 50 */ 51 52 #include <sys/param.h> 53 #include <sys/systm.h> 54 #include <sys/proc.h> 55 #include <sys/buf.h> 56 #include <sys/vnode.h> 57 #include <sys/malloc.h> 58 59 #include <miscfs/specfs/specdev.h> 60 #include <sys/rlist.h> 61 62 #include <vm/vm.h> 63 #include <vm/vm_pager.h> 64 #include <vm/vm_page.h> 65 #include <vm/vm_pageout.h> 66 #include <vm/swap_pager.h> 67 68 #ifndef NPENDINGIO 69 #define NPENDINGIO 16 70 #endif 71 72 int swap_pager_input __P((sw_pager_t, vm_page_t *, int, int)); 73 int swap_pager_output __P((sw_pager_t, vm_page_t *, int, int, int *)); 74 75 int nswiodone; 76 extern int vm_pageout_rate_limit; 77 static int cleandone; 78 extern int hz; 79 int swap_pager_full; 80 extern vm_map_t pager_map; 81 extern int vm_swap_size; 82 int no_swap_space=1; 83 struct rlist *swaplist; 84 int nswaplist; 85 86 #define MAX_PAGEOUT_CLUSTER 8 87 88 TAILQ_HEAD(swpclean, swpagerclean); 89 90 typedef struct swpagerclean *swp_clean_t; 91 92 struct swpagerclean { 93 TAILQ_ENTRY(swpagerclean) spc_list; 94 int spc_flags; 95 struct buf *spc_bp; 96 sw_pager_t spc_swp; 97 vm_offset_t spc_kva; 98 int spc_count; 99 vm_page_t spc_m[MAX_PAGEOUT_CLUSTER]; 100 } swcleanlist [NPENDINGIO] ; 101 102 103 extern vm_map_t kernel_map; 104 105 /* spc_flags values */ 106 #define SPC_ERROR 0x01 107 108 #define SWB_EMPTY (-1) 109 110 struct swpclean swap_pager_done; /* list of compileted page cleans */ 111 struct swpclean swap_pager_inuse; /* list of pending page cleans */ 112 struct swpclean swap_pager_free; /* list of free pager clean structs */ 113 struct pagerlst swap_pager_list; /* list of "named" anon regions */ 114 struct pagerlst swap_pager_un_list; /* list of "unnamed" anon pagers */ 115 116 #define SWAP_FREE_NEEDED 0x1 /* need a swap block */ 117 int swap_pager_needflags; 118 struct rlist *swapfrag; 119 120 struct pagerlst *swp_qs[]={ 121 &swap_pager_list, &swap_pager_un_list, (struct pagerlst *) 0 122 }; 123 124 int swap_pager_putmulti(); 125 126 struct pagerops swappagerops = { 127 swap_pager_init, 128 swap_pager_alloc, 129 swap_pager_dealloc, 130 swap_pager_getpage, 131 swap_pager_getmulti, 132 swap_pager_putpage, 133 swap_pager_putmulti, 134 swap_pager_haspage 135 }; 136 137 int npendingio = NPENDINGIO; 138 int pendingiowait; 139 int require_swap_init; 140 void swap_pager_finish(); 141 int dmmin, dmmax; 142 extern int vm_page_count; 143 144 static inline void swapsizecheck() { 145 if( vm_swap_size < 128*btodb(PAGE_SIZE)) { 146 if( swap_pager_full) 147 printf("swap_pager: out of space\n"); 148 swap_pager_full = 1; 149 } else if( vm_swap_size > 192*btodb(PAGE_SIZE)) 150 swap_pager_full = 0; 151 } 152 153 void 154 swap_pager_init() 155 { 156 dfltpagerops = &swappagerops; 157 158 TAILQ_INIT(&swap_pager_list); 159 TAILQ_INIT(&swap_pager_un_list); 160 161 /* 162 * Initialize clean lists 163 */ 164 TAILQ_INIT(&swap_pager_inuse); 165 TAILQ_INIT(&swap_pager_done); 166 TAILQ_INIT(&swap_pager_free); 167 168 require_swap_init = 1; 169 170 /* 171 * Calculate the swap allocation constants. 172 */ 173 174 dmmin = CLBYTES/DEV_BSIZE; 175 dmmax = btodb(SWB_NPAGES*PAGE_SIZE)*2; 176 177 } 178 179 /* 180 * Allocate a pager structure and associated resources. 181 * Note that if we are called from the pageout daemon (handle == NULL) 182 * we should not wait for memory as it could resulting in deadlock. 183 */ 184 vm_pager_t 185 swap_pager_alloc(handle, size, prot, offset) 186 caddr_t handle; 187 register vm_size_t size; 188 vm_prot_t prot; 189 vm_offset_t offset; 190 { 191 register vm_pager_t pager; 192 register sw_pager_t swp; 193 int waitok; 194 int i,j; 195 196 if (require_swap_init) { 197 swp_clean_t spc; 198 struct buf *bp; 199 /* 200 * kva's are allocated here so that we dont need to keep 201 * doing kmem_alloc pageables at runtime 202 */ 203 for (i = 0, spc = swcleanlist; i < npendingio ; i++, spc++) { 204 spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE*MAX_PAGEOUT_CLUSTER); 205 if (!spc->spc_kva) { 206 break; 207 } 208 spc->spc_bp = malloc( sizeof( *bp), M_TEMP, M_NOWAIT); 209 if (!spc->spc_bp) { 210 kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE); 211 break; 212 } 213 spc->spc_flags = 0; 214 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 215 } 216 require_swap_init = 0; 217 if( size == 0) 218 return(NULL); 219 } 220 221 /* 222 * If this is a "named" anonymous region, look it up and 223 * return the appropriate pager if it exists. 224 */ 225 if (handle) { 226 pager = vm_pager_lookup(&swap_pager_list, handle); 227 if (pager != NULL) { 228 /* 229 * Use vm_object_lookup to gain a reference 230 * to the object and also to remove from the 231 * object cache. 232 */ 233 if (vm_object_lookup(pager) == NULL) 234 panic("swap_pager_alloc: bad object"); 235 return(pager); 236 } 237 } 238 239 /* 240 * Pager doesn't exist, allocate swap management resources 241 * and initialize. 242 */ 243 waitok = handle ? M_WAITOK : M_NOWAIT; 244 pager = (vm_pager_t)malloc(sizeof *pager, M_VMPAGER, waitok); 245 if (pager == NULL) 246 return(NULL); 247 swp = (sw_pager_t)malloc(sizeof *swp, M_VMPGDATA, waitok); 248 if (swp == NULL) { 249 free((caddr_t)pager, M_VMPAGER); 250 return(NULL); 251 } 252 size = round_page(size); 253 swp->sw_osize = size; 254 swp->sw_nblocks = (btodb(size) + btodb(SWB_NPAGES * PAGE_SIZE) - 1) / btodb(SWB_NPAGES*PAGE_SIZE); 255 swp->sw_blocks = (sw_blk_t) 256 malloc(swp->sw_nblocks*sizeof(*swp->sw_blocks), 257 M_VMPGDATA, waitok); 258 if (swp->sw_blocks == NULL) { 259 free((caddr_t)swp, M_VMPGDATA); 260 free((caddr_t)pager, M_VMPAGER); 261 return(NULL); 262 } 263 264 for (i = 0; i < swp->sw_nblocks; i++) { 265 swp->sw_blocks[i].swb_valid = 0; 266 swp->sw_blocks[i].swb_locked = 0; 267 for (j = 0; j < SWB_NPAGES; j++) 268 swp->sw_blocks[i].swb_block[j] = SWB_EMPTY; 269 } 270 271 swp->sw_poip = 0; 272 if (handle) { 273 vm_object_t object; 274 275 swp->sw_flags = SW_NAMED; 276 TAILQ_INSERT_TAIL(&swap_pager_list, pager, pg_list); 277 /* 278 * Consistant with other pagers: return with object 279 * referenced. Can't do this with handle == NULL 280 * since it might be the pageout daemon calling. 281 */ 282 object = vm_object_allocate(size); 283 vm_object_enter(object, pager); 284 vm_object_setpager(object, pager, 0, FALSE); 285 } else { 286 swp->sw_flags = 0; 287 TAILQ_INSERT_TAIL(&swap_pager_un_list, pager, pg_list); 288 } 289 pager->pg_handle = handle; 290 pager->pg_ops = &swappagerops; 291 pager->pg_type = PG_SWAP; 292 pager->pg_data = (caddr_t)swp; 293 294 return(pager); 295 } 296 297 /* 298 * returns disk block associated with pager and offset 299 * additionally, as a side effect returns a flag indicating 300 * if the block has been written 301 */ 302 303 static int * 304 swap_pager_diskaddr(swp, offset, valid) 305 sw_pager_t swp; 306 vm_offset_t offset; 307 int *valid; 308 { 309 register sw_blk_t swb; 310 int ix; 311 312 if (valid) 313 *valid = 0; 314 ix = offset / (SWB_NPAGES*PAGE_SIZE); 315 if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 316 return(FALSE); 317 } 318 swb = &swp->sw_blocks[ix]; 319 ix = (offset % (SWB_NPAGES*PAGE_SIZE)) / PAGE_SIZE; 320 if (valid) 321 *valid = swb->swb_valid & (1<<ix); 322 return &swb->swb_block[ix]; 323 } 324 325 /* 326 * Utility routine to set the valid (written) bit for 327 * a block associated with a pager and offset 328 */ 329 static void 330 swap_pager_setvalid(swp, offset, valid) 331 sw_pager_t swp; 332 vm_offset_t offset; 333 int valid; 334 { 335 register sw_blk_t swb; 336 int ix; 337 338 ix = offset / (SWB_NPAGES*PAGE_SIZE); 339 if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) 340 return; 341 342 swb = &swp->sw_blocks[ix]; 343 ix = (offset % (SWB_NPAGES*PAGE_SIZE)) / PAGE_SIZE; 344 if (valid) 345 swb->swb_valid |= (1 << ix); 346 else 347 swb->swb_valid &= ~(1 << ix); 348 return; 349 } 350 351 /* 352 * this routine allocates swap space with a fragmentation 353 * minimization policy. 354 */ 355 int 356 swap_pager_getswapspace( unsigned amount, unsigned *rtval) { 357 vm_swap_size -= amount; 358 if( !rlist_alloc(&swaplist, amount, rtval)) { 359 vm_swap_size += amount; 360 return 0; 361 } else { 362 swapsizecheck(); 363 return 1; 364 } 365 } 366 367 /* 368 * this routine frees swap space with a fragmentation 369 * minimization policy. 370 */ 371 void 372 swap_pager_freeswapspace( unsigned from, unsigned to) { 373 rlist_free(&swaplist, from, to); 374 vm_swap_size += (to-from)+1; 375 swapsizecheck(); 376 } 377 /* 378 * this routine frees swap blocks from a specified pager 379 */ 380 void 381 _swap_pager_freespace(swp, start, size) 382 sw_pager_t swp; 383 vm_offset_t start; 384 vm_offset_t size; 385 { 386 vm_offset_t i; 387 int s; 388 389 s = splbio(); 390 for (i = start; i < round_page(start + size - 1); i += PAGE_SIZE) { 391 int valid; 392 int *addr = swap_pager_diskaddr(swp, i, &valid); 393 if (addr && *addr != SWB_EMPTY) { 394 swap_pager_freeswapspace(*addr, *addr+btodb(PAGE_SIZE) - 1); 395 if( valid) { 396 swap_pager_setvalid(swp, i, 0); 397 } 398 *addr = SWB_EMPTY; 399 } 400 } 401 splx(s); 402 } 403 404 void 405 swap_pager_freespace(pager, start, size) 406 vm_pager_t pager; 407 vm_offset_t start; 408 vm_offset_t size; 409 { 410 _swap_pager_freespace((sw_pager_t) pager->pg_data, start, size); 411 } 412 413 /* 414 * swap_pager_reclaim frees up over-allocated space from all pagers 415 * this eliminates internal fragmentation due to allocation of space 416 * for segments that are never swapped to. It has been written so that 417 * it does not block until the rlist_free operation occurs; it keeps 418 * the queues consistant. 419 */ 420 421 /* 422 * Maximum number of blocks (pages) to reclaim per pass 423 */ 424 #define MAXRECLAIM 256 425 426 void 427 swap_pager_reclaim() 428 { 429 vm_pager_t p; 430 sw_pager_t swp; 431 int i, j, k; 432 int s; 433 int reclaimcount; 434 static int reclaims[MAXRECLAIM]; 435 static int in_reclaim; 436 437 /* 438 * allow only one process to be in the swap_pager_reclaim subroutine 439 */ 440 s = splbio(); 441 if (in_reclaim) { 442 tsleep((caddr_t) &in_reclaim, PSWP, "swrclm", 0); 443 splx(s); 444 return; 445 } 446 in_reclaim = 1; 447 reclaimcount = 0; 448 449 /* for each pager queue */ 450 for (k = 0; swp_qs[k]; k++) { 451 452 p = swp_qs[k]->tqh_first; 453 while (p && (reclaimcount < MAXRECLAIM)) { 454 455 /* 456 * see if any blocks associated with a pager has been 457 * allocated but not used (written) 458 */ 459 swp = (sw_pager_t) p->pg_data; 460 for (i = 0; i < swp->sw_nblocks; i++) { 461 sw_blk_t swb = &swp->sw_blocks[i]; 462 if( swb->swb_locked) 463 continue; 464 for (j = 0; j < SWB_NPAGES; j++) { 465 if (swb->swb_block[j] != SWB_EMPTY && 466 (swb->swb_valid & (1 << j)) == 0) { 467 reclaims[reclaimcount++] = swb->swb_block[j]; 468 swb->swb_block[j] = SWB_EMPTY; 469 if (reclaimcount >= MAXRECLAIM) 470 goto rfinished; 471 } 472 } 473 } 474 p = p->pg_list.tqe_next; 475 } 476 } 477 478 rfinished: 479 480 /* 481 * free the blocks that have been added to the reclaim list 482 */ 483 for (i = 0; i < reclaimcount; i++) { 484 swap_pager_freeswapspace(reclaims[i], reclaims[i]+btodb(PAGE_SIZE) - 1); 485 wakeup((caddr_t) &in_reclaim); 486 } 487 488 splx(s); 489 in_reclaim = 0; 490 wakeup((caddr_t) &in_reclaim); 491 } 492 493 494 /* 495 * swap_pager_copy copies blocks from one pager to another and 496 * destroys the source pager 497 */ 498 499 void 500 swap_pager_copy(srcpager, srcoffset, dstpager, dstoffset, offset) 501 vm_pager_t srcpager; 502 vm_offset_t srcoffset; 503 vm_pager_t dstpager; 504 vm_offset_t dstoffset; 505 vm_offset_t offset; 506 { 507 sw_pager_t srcswp, dstswp; 508 vm_offset_t i; 509 int s; 510 511 if( vm_swap_size) 512 no_swap_space = 0; 513 514 if( no_swap_space) 515 return; 516 517 srcswp = (sw_pager_t) srcpager->pg_data; 518 dstswp = (sw_pager_t) dstpager->pg_data; 519 520 /* 521 * remove the source pager from the swap_pager internal queue 522 */ 523 s = splbio(); 524 if (srcswp->sw_flags & SW_NAMED) { 525 TAILQ_REMOVE(&swap_pager_list, srcpager, pg_list); 526 srcswp->sw_flags &= ~SW_NAMED; 527 } else { 528 TAILQ_REMOVE(&swap_pager_un_list, srcpager, pg_list); 529 } 530 531 while (srcswp->sw_poip) { 532 tsleep((caddr_t)srcswp, PVM, "spgout", 0); 533 } 534 splx(s); 535 536 /* 537 * clean all of the pages that are currently active and finished 538 */ 539 (void) swap_pager_clean(); 540 541 s = splbio(); 542 /* 543 * clear source block before destination object 544 * (release allocated space) 545 */ 546 for (i = 0; i < offset + srcoffset; i += PAGE_SIZE) { 547 int valid; 548 int *addr = swap_pager_diskaddr(srcswp, i, &valid); 549 if (addr && *addr != SWB_EMPTY) { 550 swap_pager_freeswapspace(*addr, *addr+btodb(PAGE_SIZE) - 1); 551 *addr = SWB_EMPTY; 552 } 553 } 554 /* 555 * transfer source to destination 556 */ 557 for (i = 0; i < dstswp->sw_osize; i += PAGE_SIZE) { 558 int srcvalid, dstvalid; 559 int *srcaddrp = swap_pager_diskaddr(srcswp, i + offset + srcoffset, 560 &srcvalid); 561 int *dstaddrp; 562 /* 563 * see if the source has space allocated 564 */ 565 if (srcaddrp && *srcaddrp != SWB_EMPTY) { 566 /* 567 * if the source is valid and the dest has no space, then 568 * copy the allocation from the srouce to the dest. 569 */ 570 if (srcvalid) { 571 dstaddrp = swap_pager_diskaddr(dstswp, i + dstoffset, &dstvalid); 572 /* 573 * if the dest already has a valid block, deallocate the 574 * source block without copying. 575 */ 576 if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) { 577 swap_pager_freeswapspace(*dstaddrp, *dstaddrp+btodb(PAGE_SIZE) - 1); 578 *dstaddrp = SWB_EMPTY; 579 } 580 if (dstaddrp && *dstaddrp == SWB_EMPTY) { 581 *dstaddrp = *srcaddrp; 582 *srcaddrp = SWB_EMPTY; 583 swap_pager_setvalid(dstswp, i + dstoffset, 1); 584 } 585 } 586 /* 587 * if the source is not empty at this point, then deallocate the space. 588 */ 589 if (*srcaddrp != SWB_EMPTY) { 590 swap_pager_freeswapspace(*srcaddrp, *srcaddrp+btodb(PAGE_SIZE) - 1); 591 *srcaddrp = SWB_EMPTY; 592 } 593 } 594 } 595 596 /* 597 * deallocate the rest of the source object 598 */ 599 for (i = dstswp->sw_osize + offset + srcoffset; i < srcswp->sw_osize; i += PAGE_SIZE) { 600 int valid; 601 int *srcaddrp = swap_pager_diskaddr(srcswp, i, &valid); 602 if (srcaddrp && *srcaddrp != SWB_EMPTY) { 603 swap_pager_freeswapspace(*srcaddrp, *srcaddrp+btodb(PAGE_SIZE) - 1); 604 *srcaddrp = SWB_EMPTY; 605 } 606 } 607 608 splx(s); 609 610 free((caddr_t)srcswp->sw_blocks, M_VMPGDATA); 611 srcswp->sw_blocks = 0; 612 free((caddr_t)srcswp, M_VMPGDATA); 613 srcpager->pg_data = 0; 614 free((caddr_t)srcpager, M_VMPAGER); 615 616 return; 617 } 618 619 620 void 621 swap_pager_dealloc(pager) 622 vm_pager_t pager; 623 { 624 register int i,j; 625 register sw_blk_t bp; 626 register sw_pager_t swp; 627 int s; 628 629 /* 630 * Remove from list right away so lookups will fail if we 631 * block for pageout completion. 632 */ 633 s = splbio(); 634 swp = (sw_pager_t) pager->pg_data; 635 if (swp->sw_flags & SW_NAMED) { 636 TAILQ_REMOVE(&swap_pager_list, pager, pg_list); 637 swp->sw_flags &= ~SW_NAMED; 638 } else { 639 TAILQ_REMOVE(&swap_pager_un_list, pager, pg_list); 640 } 641 /* 642 * Wait for all pageouts to finish and remove 643 * all entries from cleaning list. 644 */ 645 646 while (swp->sw_poip) { 647 tsleep((caddr_t)swp, PVM, "swpout", 0); 648 } 649 splx(s); 650 651 652 (void) swap_pager_clean(); 653 654 /* 655 * Free left over swap blocks 656 */ 657 s = splbio(); 658 for (i = 0, bp = swp->sw_blocks; i < swp->sw_nblocks; i++, bp++) { 659 for (j = 0; j < SWB_NPAGES; j++) 660 if (bp->swb_block[j] != SWB_EMPTY) { 661 swap_pager_freeswapspace((unsigned)bp->swb_block[j], 662 (unsigned)bp->swb_block[j] + btodb(PAGE_SIZE) - 1); 663 bp->swb_block[j] = SWB_EMPTY; 664 } 665 } 666 splx(s); 667 668 /* 669 * Free swap management resources 670 */ 671 free((caddr_t)swp->sw_blocks, M_VMPGDATA); 672 swp->sw_blocks = 0; 673 free((caddr_t)swp, M_VMPGDATA); 674 pager->pg_data = 0; 675 free((caddr_t)pager, M_VMPAGER); 676 } 677 678 /* 679 * swap_pager_getmulti can get multiple pages. 680 */ 681 int 682 swap_pager_getmulti(pager, m, count, reqpage, sync) 683 vm_pager_t pager; 684 vm_page_t *m; 685 int count; 686 int reqpage; 687 boolean_t sync; 688 { 689 if( reqpage >= count) 690 panic("swap_pager_getmulti: reqpage >= count\n"); 691 return swap_pager_input((sw_pager_t) pager->pg_data, m, count, reqpage); 692 } 693 694 /* 695 * swap_pager_getpage gets individual pages 696 */ 697 int 698 swap_pager_getpage(pager, m, sync) 699 vm_pager_t pager; 700 vm_page_t m; 701 boolean_t sync; 702 { 703 vm_page_t marray[1]; 704 705 marray[0] = m; 706 return swap_pager_input((sw_pager_t)pager->pg_data, marray, 1, 0); 707 } 708 709 int 710 swap_pager_putmulti(pager, m, c, sync, rtvals) 711 vm_pager_t pager; 712 vm_page_t *m; 713 int c; 714 boolean_t sync; 715 int *rtvals; 716 { 717 int flags; 718 719 if (pager == NULL) { 720 (void) swap_pager_clean(); 721 return VM_PAGER_OK; 722 } 723 724 flags = B_WRITE; 725 if (!sync) 726 flags |= B_ASYNC; 727 728 return swap_pager_output((sw_pager_t)pager->pg_data, m, c, flags, rtvals); 729 } 730 731 /* 732 * swap_pager_putpage writes individual pages 733 */ 734 int 735 swap_pager_putpage(pager, m, sync) 736 vm_pager_t pager; 737 vm_page_t m; 738 boolean_t sync; 739 { 740 int flags; 741 vm_page_t marray[1]; 742 int rtvals[1]; 743 744 745 if (pager == NULL) { 746 (void) swap_pager_clean(); 747 return VM_PAGER_OK; 748 } 749 750 marray[0] = m; 751 flags = B_WRITE; 752 if (!sync) 753 flags |= B_ASYNC; 754 755 swap_pager_output((sw_pager_t)pager->pg_data, marray, 1, flags, rtvals); 756 757 return rtvals[0]; 758 } 759 760 static inline int 761 const swap_pager_block_index(swp, offset) 762 sw_pager_t swp; 763 vm_offset_t offset; 764 { 765 return (offset / (SWB_NPAGES*PAGE_SIZE)); 766 } 767 768 static inline int 769 const swap_pager_block_offset(swp, offset) 770 sw_pager_t swp; 771 vm_offset_t offset; 772 { 773 return ((offset % (PAGE_SIZE*SWB_NPAGES)) / PAGE_SIZE); 774 } 775 776 /* 777 * _swap_pager_haspage returns TRUE if the pager has data that has 778 * been written out. 779 */ 780 static boolean_t 781 _swap_pager_haspage(swp, offset) 782 sw_pager_t swp; 783 vm_offset_t offset; 784 { 785 register sw_blk_t swb; 786 int ix; 787 788 ix = offset / (SWB_NPAGES*PAGE_SIZE); 789 if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 790 return(FALSE); 791 } 792 swb = &swp->sw_blocks[ix]; 793 ix = (offset % (SWB_NPAGES*PAGE_SIZE)) / PAGE_SIZE; 794 if (swb->swb_block[ix] != SWB_EMPTY) { 795 if (swb->swb_valid & (1 << ix)) 796 return TRUE; 797 } 798 799 return(FALSE); 800 } 801 802 /* 803 * swap_pager_haspage is the externally accessible version of 804 * _swap_pager_haspage above. this routine takes a vm_pager_t 805 * for an argument instead of sw_pager_t. 806 */ 807 boolean_t 808 swap_pager_haspage(pager, offset) 809 vm_pager_t pager; 810 vm_offset_t offset; 811 { 812 return _swap_pager_haspage((sw_pager_t) pager->pg_data, offset); 813 } 814 815 /* 816 * swap_pager_freepage is a convienience routine that clears the busy 817 * bit and deallocates a page. 818 */ 819 static void 820 swap_pager_freepage(m) 821 vm_page_t m; 822 { 823 PAGE_WAKEUP(m); 824 vm_page_free(m); 825 } 826 827 /* 828 * swap_pager_ridpages is a convienience routine that deallocates all 829 * but the required page. this is usually used in error returns that 830 * need to invalidate the "extra" readahead pages. 831 */ 832 static void 833 swap_pager_ridpages(m, count, reqpage) 834 vm_page_t *m; 835 int count; 836 int reqpage; 837 { 838 int i; 839 for (i = 0; i < count; i++) 840 if (i != reqpage) 841 swap_pager_freepage(m[i]); 842 } 843 844 int swapwritecount=0; 845 846 /* 847 * swap_pager_iodone1 is the completion routine for both reads and async writes 848 */ 849 void 850 swap_pager_iodone1(bp) 851 struct buf *bp; 852 { 853 bp->b_flags |= B_DONE; 854 bp->b_flags &= ~B_ASYNC; 855 wakeup((caddr_t)bp); 856 /* 857 if ((bp->b_flags & B_READ) == 0) 858 vwakeup(bp); 859 */ 860 } 861 862 863 int 864 swap_pager_input(swp, m, count, reqpage) 865 register sw_pager_t swp; 866 vm_page_t *m; 867 int count, reqpage; 868 { 869 register struct buf *bp; 870 sw_blk_t swb[count]; 871 register int s; 872 int i; 873 boolean_t rv; 874 vm_offset_t kva, off[count]; 875 swp_clean_t spc; 876 vm_offset_t paging_offset; 877 vm_object_t object; 878 int reqaddr[count]; 879 880 int first, last; 881 int failed; 882 int reqdskregion; 883 884 object = m[reqpage]->object; 885 paging_offset = object->paging_offset; 886 /* 887 * First determine if the page exists in the pager if this is 888 * a sync read. This quickly handles cases where we are 889 * following shadow chains looking for the top level object 890 * with the page. 891 */ 892 if (swp->sw_blocks == NULL) { 893 swap_pager_ridpages(m, count, reqpage); 894 return(VM_PAGER_FAIL); 895 } 896 897 for(i = 0; i < count; i++) { 898 vm_offset_t foff = m[i]->offset + paging_offset; 899 int ix = swap_pager_block_index(swp, foff); 900 if (ix >= swp->sw_nblocks) { 901 int j; 902 if( i <= reqpage) { 903 swap_pager_ridpages(m, count, reqpage); 904 return(VM_PAGER_FAIL); 905 } 906 for(j = i; j < count; j++) { 907 swap_pager_freepage(m[j]); 908 } 909 count = i; 910 break; 911 } 912 913 swb[i] = &swp->sw_blocks[ix]; 914 off[i] = swap_pager_block_offset(swp, foff); 915 reqaddr[i] = swb[i]->swb_block[off[i]]; 916 } 917 918 /* make sure that our required input request is existant */ 919 920 if (reqaddr[reqpage] == SWB_EMPTY || 921 (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) { 922 swap_pager_ridpages(m, count, reqpage); 923 return(VM_PAGER_FAIL); 924 } 925 926 927 reqdskregion = reqaddr[reqpage] / dmmax; 928 929 /* 930 * search backwards for the first contiguous page to transfer 931 */ 932 failed = 0; 933 first = 0; 934 for (i = reqpage - 1; i >= 0; --i) { 935 if ( failed || (reqaddr[i] == SWB_EMPTY) || 936 (swb[i]->swb_valid & (1 << off[i])) == 0 || 937 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 938 ((reqaddr[i] / dmmax) != reqdskregion)) { 939 failed = 1; 940 swap_pager_freepage(m[i]); 941 if (first == 0) 942 first = i + 1; 943 } 944 } 945 /* 946 * search forwards for the last contiguous page to transfer 947 */ 948 failed = 0; 949 last = count; 950 for (i = reqpage + 1; i < count; i++) { 951 if ( failed || (reqaddr[i] == SWB_EMPTY) || 952 (swb[i]->swb_valid & (1 << off[i])) == 0 || 953 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 954 ((reqaddr[i] / dmmax) != reqdskregion)) { 955 failed = 1; 956 swap_pager_freepage(m[i]); 957 if (last == count) 958 last = i; 959 } 960 } 961 962 count = last; 963 if (first != 0) { 964 for (i = first; i < count; i++) { 965 m[i-first] = m[i]; 966 reqaddr[i-first] = reqaddr[i]; 967 off[i-first] = off[i]; 968 } 969 count -= first; 970 reqpage -= first; 971 } 972 973 ++swb[reqpage]->swb_locked; 974 975 /* 976 * at this point: 977 * "m" is a pointer to the array of vm_page_t for paging I/O 978 * "count" is the number of vm_page_t entries represented by "m" 979 * "object" is the vm_object_t for I/O 980 * "reqpage" is the index into "m" for the page actually faulted 981 */ 982 983 spc = NULL; /* we might not use an spc data structure */ 984 985 if (count == 1) { 986 /* 987 * if a kva has not been allocated, we can only do a one page transfer, 988 * so we free the other pages that might have been allocated by 989 * vm_fault. 990 */ 991 swap_pager_ridpages(m, count, reqpage); 992 m[0] = m[reqpage]; 993 reqaddr[0] = reqaddr[reqpage]; 994 995 count = 1; 996 reqpage = 0; 997 /* 998 * get a swap pager clean data structure, block until we get it 999 */ 1000 if (swap_pager_free.tqh_first == NULL) { 1001 s = splbio(); 1002 if( curproc == pageproc) 1003 (void) swap_pager_clean(); 1004 else 1005 wakeup((caddr_t) &vm_pages_needed); 1006 while (swap_pager_free.tqh_first == NULL) { 1007 swap_pager_needflags |= SWAP_FREE_NEEDED; 1008 tsleep((caddr_t)&swap_pager_free, 1009 PVM, "swpfre", 0); 1010 if( curproc == pageproc) 1011 (void) swap_pager_clean(); 1012 else 1013 wakeup((caddr_t) &vm_pages_needed); 1014 } 1015 splx(s); 1016 } 1017 spc = swap_pager_free.tqh_first; 1018 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1019 kva = spc->spc_kva; 1020 bp = spc->spc_bp; 1021 bzero(bp, sizeof *bp); 1022 bp->b_spc = spc; 1023 } else { 1024 /* 1025 * Get a swap buffer header to perform the IO 1026 */ 1027 bp = getpbuf(); 1028 kva = (vm_offset_t) bp->b_data; 1029 } 1030 1031 /* 1032 * map our page(s) into kva for input 1033 */ 1034 pmap_qenter( kva, m, count); 1035 1036 s = splbio(); 1037 bp->b_flags = B_BUSY | B_READ | B_CALL; 1038 bp->b_iodone = swap_pager_iodone1; 1039 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 1040 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1041 crhold(bp->b_rcred); 1042 crhold(bp->b_wcred); 1043 bp->b_un.b_addr = (caddr_t) kva; 1044 bp->b_blkno = reqaddr[0]; 1045 bp->b_bcount = PAGE_SIZE*count; 1046 bp->b_bufsize = PAGE_SIZE*count; 1047 1048 bgetvp( swapdev_vp, bp); 1049 1050 swp->sw_piip++; 1051 1052 cnt.v_swapin++; 1053 cnt.v_swappgsin += count; 1054 /* 1055 * perform the I/O 1056 */ 1057 VOP_STRATEGY(bp); 1058 1059 /* 1060 * wait for the sync I/O to complete 1061 */ 1062 while ((bp->b_flags & B_DONE) == 0) { 1063 tsleep((caddr_t)bp, PVM, "swread", 0); 1064 } 1065 1066 if (bp->b_flags & B_ERROR) { 1067 printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n", 1068 bp->b_blkno, bp->b_bcount, bp->b_error); 1069 rv = VM_PAGER_ERROR; 1070 } else { 1071 rv = VM_PAGER_OK; 1072 } 1073 1074 --swp->sw_piip; 1075 if (swp->sw_piip == 0) 1076 wakeup((caddr_t) swp); 1077 1078 /* 1079 * relpbuf does this, but we maintain our own buffer 1080 * list also... 1081 */ 1082 if (bp->b_vp) 1083 brelvp(bp); 1084 1085 splx(s); 1086 --swb[reqpage]->swb_locked; 1087 1088 /* 1089 * remove the mapping for kernel virtual 1090 */ 1091 pmap_qremove( kva, count); 1092 1093 if (spc) { 1094 /* 1095 * if we have used an spc, we need to free it. 1096 */ 1097 if( bp->b_rcred != NOCRED) 1098 crfree(bp->b_rcred); 1099 if( bp->b_wcred != NOCRED) 1100 crfree(bp->b_wcred); 1101 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1102 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1103 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1104 wakeup((caddr_t)&swap_pager_free); 1105 } 1106 } else { 1107 /* 1108 * release the physical I/O buffer 1109 */ 1110 relpbuf(bp); 1111 /* 1112 * finish up input if everything is ok 1113 */ 1114 if( rv == VM_PAGER_OK) { 1115 for (i = 0; i < count; i++) { 1116 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1117 m[i]->flags |= PG_CLEAN; 1118 m[i]->flags &= ~PG_LAUNDRY; 1119 if (i != reqpage) { 1120 /* 1121 * whether or not to leave the page activated 1122 * is up in the air, but we should put the page 1123 * on a page queue somewhere. (it already is in 1124 * the object). 1125 * After some emperical results, it is best 1126 * to deactivate the readahead pages. 1127 */ 1128 vm_page_deactivate(m[i]); 1129 1130 /* 1131 * just in case someone was asking for this 1132 * page we now tell them that it is ok to use 1133 */ 1134 m[i]->flags &= ~PG_FAKE; 1135 PAGE_WAKEUP(m[i]); 1136 } 1137 } 1138 /* 1139 * If we're out of swap space, then attempt to free 1140 * some whenever pages are brought in. We must clear 1141 * the clean flag so that the page contents will be 1142 * preserved. 1143 */ 1144 if (swap_pager_full) { 1145 for (i = 0; i < count; i++) { 1146 m[i]->flags &= ~PG_CLEAN; 1147 } 1148 _swap_pager_freespace( swp, m[0]->offset+paging_offset, count*PAGE_SIZE); 1149 } 1150 } else { 1151 swap_pager_ridpages(m, count, reqpage); 1152 } 1153 } 1154 return(rv); 1155 } 1156 1157 int 1158 swap_pager_output(swp, m, count, flags, rtvals) 1159 register sw_pager_t swp; 1160 vm_page_t *m; 1161 int count; 1162 int flags; 1163 int *rtvals; 1164 { 1165 register struct buf *bp; 1166 sw_blk_t swb[count]; 1167 register int s; 1168 int i, j, ix; 1169 boolean_t rv; 1170 vm_offset_t kva, off, foff; 1171 swp_clean_t spc; 1172 vm_offset_t paging_offset; 1173 vm_object_t object; 1174 int reqaddr[count]; 1175 int failed; 1176 1177 /* 1178 if( count > 1) 1179 printf("off: 0x%x, count: %d\n", m[0]->offset, count); 1180 */ 1181 if( vm_swap_size) 1182 no_swap_space = 0; 1183 if( no_swap_space) { 1184 for(i=0;i<count;i++) 1185 rtvals[i] = VM_PAGER_FAIL; 1186 return VM_PAGER_FAIL; 1187 } 1188 1189 spc = NULL; 1190 1191 object = m[0]->object; 1192 paging_offset = object->paging_offset; 1193 1194 failed = 0; 1195 for(j=0;j<count;j++) { 1196 foff = m[j]->offset + paging_offset; 1197 ix = swap_pager_block_index(swp, foff); 1198 swb[j] = 0; 1199 if( swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 1200 rtvals[j] = VM_PAGER_FAIL; 1201 failed = 1; 1202 continue; 1203 } else { 1204 rtvals[j] = VM_PAGER_OK; 1205 } 1206 swb[j] = &swp->sw_blocks[ix]; 1207 ++swb[j]->swb_locked; 1208 if( failed) { 1209 rtvals[j] = VM_PAGER_FAIL; 1210 continue; 1211 } 1212 off = swap_pager_block_offset(swp, foff); 1213 reqaddr[j] = swb[j]->swb_block[off]; 1214 if( reqaddr[j] == SWB_EMPTY) { 1215 int blk; 1216 int tries; 1217 int ntoget; 1218 tries = 0; 1219 s = splbio(); 1220 1221 /* 1222 * if any other pages have been allocated in this block, we 1223 * only try to get one page. 1224 */ 1225 for (i = 0; i < SWB_NPAGES; i++) { 1226 if (swb[j]->swb_block[i] != SWB_EMPTY) 1227 break; 1228 } 1229 1230 1231 ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1; 1232 /* 1233 * this code is alittle conservative, but works 1234 * (the intent of this code is to allocate small chunks 1235 * for small objects) 1236 */ 1237 if( (m[j]->offset == 0) && (ntoget*PAGE_SIZE > object->size)) { 1238 ntoget = (object->size + (PAGE_SIZE-1))/PAGE_SIZE; 1239 } 1240 1241 retrygetspace: 1242 if (!swap_pager_full && ntoget > 1 && 1243 swap_pager_getswapspace(ntoget * btodb(PAGE_SIZE), &blk)) { 1244 1245 for (i = 0; i < ntoget; i++) { 1246 swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i; 1247 swb[j]->swb_valid = 0; 1248 } 1249 1250 reqaddr[j] = swb[j]->swb_block[off]; 1251 } else if (!swap_pager_getswapspace(btodb(PAGE_SIZE), 1252 &swb[j]->swb_block[off])) { 1253 /* 1254 * if the allocation has failed, we try to reclaim space and 1255 * retry. 1256 */ 1257 if (++tries == 1) { 1258 swap_pager_reclaim(); 1259 goto retrygetspace; 1260 } 1261 rtvals[j] = VM_PAGER_AGAIN; 1262 failed = 1; 1263 swap_pager_full = 1; 1264 } else { 1265 reqaddr[j] = swb[j]->swb_block[off]; 1266 swb[j]->swb_valid &= ~(1<<off); 1267 } 1268 splx(s); 1269 } 1270 } 1271 1272 /* 1273 * search forwards for the last contiguous page to transfer 1274 */ 1275 failed = 0; 1276 for (i = 0; i < count; i++) { 1277 if( failed || (reqaddr[i] != reqaddr[0] + i*btodb(PAGE_SIZE)) || 1278 (reqaddr[i] / dmmax) != (reqaddr[0] / dmmax) || 1279 (rtvals[i] != VM_PAGER_OK)) { 1280 failed = 1; 1281 if( rtvals[i] == VM_PAGER_OK) 1282 rtvals[i] = VM_PAGER_AGAIN; 1283 } 1284 } 1285 1286 for(i = 0; i < count; i++) { 1287 if( rtvals[i] != VM_PAGER_OK) { 1288 if( swb[i]) 1289 --swb[i]->swb_locked; 1290 } 1291 } 1292 1293 for(i = 0; i < count; i++) 1294 if( rtvals[i] != VM_PAGER_OK) 1295 break; 1296 1297 if( i == 0) { 1298 return VM_PAGER_AGAIN; 1299 } 1300 1301 count = i; 1302 for(i=0;i<count;i++) { 1303 if( reqaddr[i] == SWB_EMPTY) 1304 printf("I/O to empty block????\n"); 1305 } 1306 1307 /* 1308 */ 1309 1310 /* 1311 * For synchronous writes, we clean up 1312 * all completed async pageouts. 1313 */ 1314 if ((flags & B_ASYNC) == 0) { 1315 swap_pager_clean(); 1316 } 1317 1318 kva = 0; 1319 1320 /* 1321 * we allocate a new kva for transfers > 1 page 1322 * but for transfers == 1 page, the swap_pager_free list contains 1323 * entries that have pre-allocated kva's (for efficiency). 1324 * NOTE -- we do not use the physical buffer pool or the 1325 * preallocated associated kva's because of the potential for 1326 * deadlock. This is very subtile -- but deadlocks or resource 1327 * contention must be avoided on pageouts -- or your system will 1328 * sleep (forever) !!! 1329 */ 1330 /* 1331 if ( count > 1) { 1332 kva = kmem_alloc_pageable(pager_map, count*PAGE_SIZE); 1333 if( !kva) { 1334 for (i = 0; i < count; i++) { 1335 if( swb[i]) 1336 --swb[i]->swb_locked; 1337 rtvals[i] = VM_PAGER_AGAIN; 1338 } 1339 return VM_PAGER_AGAIN; 1340 } 1341 } 1342 */ 1343 1344 /* 1345 * get a swap pager clean data structure, block until we get it 1346 */ 1347 if (swap_pager_free.tqh_first == NULL) { 1348 s = splbio(); 1349 if( curproc == pageproc) 1350 (void) swap_pager_clean(); 1351 else 1352 wakeup((caddr_t) &vm_pages_needed); 1353 while (swap_pager_free.tqh_first == NULL) { 1354 swap_pager_needflags |= SWAP_FREE_NEEDED; 1355 tsleep((caddr_t)&swap_pager_free, 1356 PVM, "swpfre", 0); 1357 if( curproc == pageproc) 1358 (void) swap_pager_clean(); 1359 else 1360 wakeup((caddr_t) &vm_pages_needed); 1361 } 1362 splx(s); 1363 } 1364 1365 spc = swap_pager_free.tqh_first; 1366 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1367 1368 kva = spc->spc_kva; 1369 1370 /* 1371 * map our page(s) into kva for I/O 1372 */ 1373 pmap_qenter(kva, m, count); 1374 1375 /* 1376 * get the base I/O offset into the swap file 1377 */ 1378 for(i=0;i<count;i++) { 1379 foff = m[i]->offset + paging_offset; 1380 off = swap_pager_block_offset(swp, foff); 1381 /* 1382 * set the valid bit 1383 */ 1384 swb[i]->swb_valid |= (1 << off); 1385 /* 1386 * and unlock the data structure 1387 */ 1388 --swb[i]->swb_locked; 1389 } 1390 1391 s = splbio(); 1392 /* 1393 * Get a swap buffer header and perform the IO 1394 */ 1395 bp = spc->spc_bp; 1396 bzero(bp, sizeof *bp); 1397 bp->b_spc = spc; 1398 1399 bp->b_flags = B_BUSY; 1400 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 1401 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1402 if( bp->b_rcred != NOCRED) 1403 crhold(bp->b_rcred); 1404 if( bp->b_wcred != NOCRED) 1405 crhold(bp->b_wcred); 1406 bp->b_data = (caddr_t) kva; 1407 bp->b_blkno = reqaddr[0]; 1408 bgetvp( swapdev_vp, bp); 1409 1410 bp->b_bcount = PAGE_SIZE*count; 1411 bp->b_bufsize = PAGE_SIZE*count; 1412 swapdev_vp->v_numoutput++; 1413 1414 /* 1415 * If this is an async write we set up additional buffer fields 1416 * and place a "cleaning" entry on the inuse queue. 1417 */ 1418 if ( flags & B_ASYNC ) { 1419 spc->spc_flags = 0; 1420 spc->spc_swp = swp; 1421 for(i=0;i<count;i++) 1422 spc->spc_m[i] = m[i]; 1423 spc->spc_count = count; 1424 /* 1425 * the completion routine for async writes 1426 */ 1427 bp->b_flags |= B_CALL; 1428 bp->b_iodone = swap_pager_iodone; 1429 bp->b_dirtyoff = 0; 1430 bp->b_dirtyend = bp->b_bcount; 1431 swp->sw_poip++; 1432 TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 1433 } else { 1434 swp->sw_poip++; 1435 bp->b_flags |= B_CALL; 1436 bp->b_iodone = swap_pager_iodone1; 1437 } 1438 1439 cnt.v_swapout++; 1440 cnt.v_swappgsout += count; 1441 /* 1442 * perform the I/O 1443 */ 1444 VOP_STRATEGY(bp); 1445 if ((flags & (B_READ|B_ASYNC)) == B_ASYNC ) { 1446 if ((bp->b_flags & B_DONE) == B_DONE) { 1447 swap_pager_clean(); 1448 } 1449 splx(s); 1450 for(i=0;i<count;i++) { 1451 rtvals[i] = VM_PAGER_PEND; 1452 } 1453 return VM_PAGER_PEND; 1454 } 1455 1456 /* 1457 * wait for the sync I/O to complete 1458 */ 1459 while ((bp->b_flags & B_DONE) == 0) { 1460 tsleep((caddr_t)bp, PVM, "swwrt", 0); 1461 } 1462 if (bp->b_flags & B_ERROR) { 1463 printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n", 1464 bp->b_blkno, bp->b_bcount, bp->b_error); 1465 rv = VM_PAGER_ERROR; 1466 } else { 1467 rv = VM_PAGER_OK; 1468 } 1469 1470 --swp->sw_poip; 1471 if (swp->sw_poip == 0) 1472 wakeup((caddr_t) swp); 1473 1474 if (bp->b_vp) 1475 brelvp(bp); 1476 1477 splx(s); 1478 1479 /* 1480 * remove the mapping for kernel virtual 1481 */ 1482 pmap_qremove( kva, count); 1483 1484 /* 1485 * if we have written the page, then indicate that the page 1486 * is clean. 1487 */ 1488 if (rv == VM_PAGER_OK) { 1489 for(i=0;i<count;i++) { 1490 if( rtvals[i] == VM_PAGER_OK) { 1491 m[i]->flags |= PG_CLEAN; 1492 m[i]->flags &= ~PG_LAUNDRY; 1493 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1494 /* 1495 * optimization, if a page has been read during the 1496 * pageout process, we activate it. 1497 */ 1498 if ( (m[i]->flags & PG_ACTIVE) == 0 && 1499 pmap_is_referenced(VM_PAGE_TO_PHYS(m[i]))) 1500 vm_page_activate(m[i]); 1501 } 1502 } 1503 } else { 1504 for(i=0;i<count;i++) { 1505 rtvals[i] = rv; 1506 m[i]->flags |= PG_LAUNDRY; 1507 } 1508 } 1509 1510 if( bp->b_rcred != NOCRED) 1511 crfree(bp->b_rcred); 1512 if( bp->b_wcred != NOCRED) 1513 crfree(bp->b_wcred); 1514 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1515 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1516 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1517 wakeup((caddr_t)&swap_pager_free); 1518 } 1519 1520 return(rv); 1521 } 1522 1523 boolean_t 1524 swap_pager_clean() 1525 { 1526 register swp_clean_t spc, tspc; 1527 register int s; 1528 1529 tspc = NULL; 1530 if (swap_pager_done.tqh_first == NULL) 1531 return FALSE; 1532 for (;;) { 1533 s = splbio(); 1534 /* 1535 * Look up and removal from done list must be done 1536 * at splbio() to avoid conflicts with swap_pager_iodone. 1537 */ 1538 while ((spc = swap_pager_done.tqh_first) != 0) { 1539 pmap_qremove( spc->spc_kva, spc->spc_count); 1540 swap_pager_finish(spc); 1541 TAILQ_REMOVE(&swap_pager_done, spc, spc_list); 1542 goto doclean; 1543 } 1544 1545 /* 1546 * No operations done, thats all we can do for now. 1547 */ 1548 1549 splx(s); 1550 break; 1551 1552 /* 1553 * The desired page was found to be busy earlier in 1554 * the scan but has since completed. 1555 */ 1556 doclean: 1557 if (tspc && tspc == spc) { 1558 tspc = NULL; 1559 } 1560 spc->spc_flags = 0; 1561 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1562 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1563 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1564 wakeup((caddr_t)&swap_pager_free); 1565 } 1566 ++cleandone; 1567 splx(s); 1568 } 1569 1570 return(tspc ? TRUE : FALSE); 1571 } 1572 1573 void 1574 swap_pager_finish(spc) 1575 register swp_clean_t spc; 1576 { 1577 vm_object_t object = spc->spc_m[0]->object; 1578 int i; 1579 1580 if ((object->paging_in_progress -= spc->spc_count) == 0) 1581 thread_wakeup((int) object); 1582 1583 /* 1584 * If no error mark as clean and inform the pmap system. 1585 * If error, mark as dirty so we will try again. 1586 * (XXX could get stuck doing this, should give up after awhile) 1587 */ 1588 if (spc->spc_flags & SPC_ERROR) { 1589 for(i=0;i<spc->spc_count;i++) { 1590 printf("swap_pager_finish: I/O error, clean of page %lx failed\n", 1591 (u_long)VM_PAGE_TO_PHYS(spc->spc_m[i])); 1592 spc->spc_m[i]->flags |= PG_LAUNDRY; 1593 } 1594 } else { 1595 for(i=0;i<spc->spc_count;i++) { 1596 pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); 1597 spc->spc_m[i]->flags |= PG_CLEAN; 1598 } 1599 } 1600 1601 1602 for(i=0;i<spc->spc_count;i++) { 1603 /* 1604 * we wakeup any processes that are waiting on 1605 * these pages. 1606 */ 1607 PAGE_WAKEUP(spc->spc_m[i]); 1608 } 1609 nswiodone -= spc->spc_count; 1610 1611 return; 1612 } 1613 1614 /* 1615 * swap_pager_iodone 1616 */ 1617 void 1618 swap_pager_iodone(bp) 1619 register struct buf *bp; 1620 { 1621 register swp_clean_t spc; 1622 int s; 1623 1624 s = splbio(); 1625 spc = (swp_clean_t) bp->b_spc; 1626 TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 1627 TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); 1628 if (bp->b_flags & B_ERROR) { 1629 spc->spc_flags |= SPC_ERROR; 1630 printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d", 1631 (bp->b_flags & B_READ) ? "pagein" : "pageout", 1632 bp->b_error, (u_long)bp->b_blkno, bp->b_bcount); 1633 } 1634 1635 /* 1636 if ((bp->b_flags & B_READ) == 0) 1637 vwakeup(bp); 1638 */ 1639 1640 if (bp->b_vp) { 1641 brelvp(bp); 1642 } 1643 if( bp->b_rcred != NOCRED) 1644 crfree(bp->b_rcred); 1645 if( bp->b_wcred != NOCRED) 1646 crfree(bp->b_wcred); 1647 1648 nswiodone += spc->spc_count; 1649 if (--spc->spc_swp->sw_poip == 0) { 1650 wakeup((caddr_t)spc->spc_swp); 1651 } 1652 1653 if ((swap_pager_needflags & SWAP_FREE_NEEDED) || 1654 swap_pager_inuse.tqh_first == 0) { 1655 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1656 wakeup((caddr_t)&swap_pager_free); 1657 wakeup((caddr_t)&vm_pages_needed); 1658 } 1659 1660 if (vm_pageout_pages_needed) { 1661 wakeup((caddr_t)&vm_pageout_pages_needed); 1662 } 1663 1664 if ((swap_pager_inuse.tqh_first == NULL) || 1665 (cnt.v_free_count < cnt.v_free_min && 1666 nswiodone + cnt.v_free_count >= cnt.v_free_min) ) { 1667 wakeup((caddr_t)&vm_pages_needed); 1668 } 1669 splx(s); 1670 } 1671 1672 /* 1673 * return true if any swap control structures can be allocated 1674 */ 1675 int 1676 swap_pager_ready() { 1677 if( swap_pager_free.tqh_first) 1678 return 1; 1679 else 1680 return 0; 1681 } 1682