1 /* 2 * Copyright (c) 1994 John S. Dyson 3 * Copyright (c) 1990 University of Utah. 4 * Copyright (c) 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * the Systems Programming Group of the University of Utah Computer 9 * Science Department. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 40 * 41 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 42 * $Id: swap_pager.c,v 1.46 1995/09/11 00:47:17 dyson Exp $ 43 */ 44 45 /* 46 * Quick hack to page to dedicated partition(s). 47 * TODO: 48 * Add multiprocessor locks 49 * Deal with async writes in a better fashion 50 */ 51 52 #include <sys/param.h> 53 #include <sys/systm.h> 54 #include <sys/kernel.h> 55 #include <sys/proc.h> 56 #include <sys/buf.h> 57 #include <sys/vnode.h> 58 #include <sys/malloc.h> 59 60 #include <miscfs/specfs/specdev.h> 61 #include <sys/rlist.h> 62 63 #include <vm/vm.h> 64 #include <vm/vm_pager.h> 65 #include <vm/vm_page.h> 66 #include <vm/vm_pageout.h> 67 #include <vm/swap_pager.h> 68 #include <vm/vm_kern.h> 69 70 #ifndef NPENDINGIO 71 #define NPENDINGIO 10 72 #endif 73 74 int nswiodone; 75 int swap_pager_full; 76 extern int vm_swap_size; 77 int no_swap_space = 1; 78 struct rlist *swaplist; 79 int nswaplist; 80 81 #define MAX_PAGEOUT_CLUSTER 8 82 83 TAILQ_HEAD(swpclean, swpagerclean); 84 85 typedef struct swpagerclean *swp_clean_t; 86 87 struct swpagerclean { 88 TAILQ_ENTRY(swpagerclean) spc_list; 89 int spc_flags; 90 struct buf *spc_bp; 91 vm_object_t spc_object; 92 vm_offset_t spc_kva; 93 int spc_count; 94 vm_page_t spc_m[MAX_PAGEOUT_CLUSTER]; 95 } swcleanlist[NPENDINGIO]; 96 97 98 /* spc_flags values */ 99 #define SPC_ERROR 0x01 100 101 #define SWB_EMPTY (-1) 102 103 struct swpclean swap_pager_done; /* list of completed page cleans */ 104 struct swpclean swap_pager_inuse; /* list of pending page cleans */ 105 struct swpclean swap_pager_free; /* list of free pager clean structs */ 106 struct pagerlst swap_pager_object_list; /* list of "named" anon region objects */ 107 struct pagerlst swap_pager_un_object_list; /* list of "unnamed" anon region objects */ 108 109 #define SWAP_FREE_NEEDED 0x1 /* need a swap block */ 110 #define SWAP_FREE_NEEDED_BY_PAGEOUT 0x2 111 int swap_pager_needflags; 112 113 struct pagerlst *swp_qs[] = { 114 &swap_pager_object_list, &swap_pager_un_object_list, (struct pagerlst *) 0 115 }; 116 117 /* 118 * pagerops for OBJT_SWAP - "swap pager". 119 */ 120 struct pagerops swappagerops = { 121 swap_pager_init, 122 swap_pager_alloc, 123 swap_pager_dealloc, 124 swap_pager_getpages, 125 swap_pager_putpages, 126 swap_pager_haspage, 127 swap_pager_sync 128 }; 129 130 int npendingio = NPENDINGIO; 131 void swap_pager_finish(); 132 int dmmin, dmmax; 133 134 135 static inline void 136 swapsizecheck() 137 { 138 if (vm_swap_size < 128 * btodb(PAGE_SIZE)) { 139 if (swap_pager_full == 0) 140 printf("swap_pager: out of space\n"); 141 swap_pager_full = 1; 142 } else if (vm_swap_size > 192 * btodb(PAGE_SIZE)) 143 swap_pager_full = 0; 144 } 145 146 void 147 swap_pager_init() 148 { 149 TAILQ_INIT(&swap_pager_object_list); 150 TAILQ_INIT(&swap_pager_un_object_list); 151 152 /* 153 * Initialize clean lists 154 */ 155 TAILQ_INIT(&swap_pager_inuse); 156 TAILQ_INIT(&swap_pager_done); 157 TAILQ_INIT(&swap_pager_free); 158 159 /* 160 * Calculate the swap allocation constants. 161 */ 162 dmmin = CLBYTES / DEV_BSIZE; 163 dmmax = btodb(SWB_NPAGES * PAGE_SIZE) * 2; 164 } 165 166 void 167 swap_pager_swap_init() 168 { 169 swp_clean_t spc; 170 struct buf *bp; 171 int i; 172 173 /* 174 * kva's are allocated here so that we dont need to keep doing 175 * kmem_alloc pageables at runtime 176 */ 177 for (i = 0, spc = swcleanlist; i < npendingio; i++, spc++) { 178 spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE * MAX_PAGEOUT_CLUSTER); 179 if (!spc->spc_kva) { 180 break; 181 } 182 spc->spc_bp = malloc(sizeof(*bp), M_TEMP, M_KERNEL); 183 if (!spc->spc_bp) { 184 kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE); 185 break; 186 } 187 spc->spc_flags = 0; 188 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 189 } 190 } 191 192 int 193 swap_pager_swp_alloc(object, wait) 194 vm_object_t object; 195 int wait; 196 { 197 sw_blk_t swb; 198 int nblocks; 199 int i, j; 200 201 nblocks = (btodb(object->size) + btodb(SWB_NPAGES * PAGE_SIZE) - 1) / 202 btodb(SWB_NPAGES * PAGE_SIZE); 203 204 swb = malloc(nblocks * sizeof(*swb), M_VMPGDATA, wait); 205 if (swb == NULL) 206 return 1; 207 208 for (i = 0; i < nblocks; i++) { 209 swb[i].swb_valid = 0; 210 swb[i].swb_locked = 0; 211 for (j = 0; j < SWB_NPAGES; j++) 212 swb[i].swb_block[j] = SWB_EMPTY; 213 } 214 215 object->un_pager.swp.swp_nblocks = nblocks; 216 object->un_pager.swp.swp_allocsize = 0; 217 object->un_pager.swp.swp_blocks = swb; 218 object->un_pager.swp.swp_poip = 0; 219 220 if (object->handle != NULL) { 221 TAILQ_INSERT_TAIL(&swap_pager_object_list, object, pager_object_list); 222 } else { 223 TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list); 224 } 225 226 return 0; 227 } 228 229 /* 230 * Allocate an object and associated resources. 231 * Note that if we are called from the pageout daemon (handle == NULL) 232 * we should not wait for memory as it could resulting in deadlock. 233 */ 234 vm_object_t 235 swap_pager_alloc(handle, size, prot, offset) 236 void *handle; 237 register vm_size_t size; 238 vm_prot_t prot; 239 vm_offset_t offset; 240 { 241 vm_object_t object; 242 int i; 243 244 /* 245 * If this is a "named" anonymous region, look it up and use the 246 * object if it exists, otherwise allocate a new one. 247 */ 248 if (handle) { 249 object = vm_pager_object_lookup(&swap_pager_object_list, handle); 250 if (object != NULL) { 251 vm_object_reference(object); 252 } else { 253 /* 254 * XXX - there is a race condition here. Two processes 255 * can request the same named object simultaneuously, 256 * and if one blocks for memory, the result is a disaster. 257 * Probably quite rare, but is yet another reason to just 258 * rip support of "named anonymous regions" out altogether. 259 */ 260 object = vm_object_allocate(OBJT_SWAP, offset + size); 261 object->handle = handle; 262 (void) swap_pager_swp_alloc(object, M_WAITOK); 263 } 264 } else { 265 object = vm_object_allocate(OBJT_SWAP, offset + size); 266 (void) swap_pager_swp_alloc(object, M_WAITOK); 267 } 268 269 return (object); 270 } 271 272 /* 273 * returns disk block associated with pager and offset 274 * additionally, as a side effect returns a flag indicating 275 * if the block has been written 276 */ 277 278 inline static int * 279 swap_pager_diskaddr(object, offset, valid) 280 vm_object_t object; 281 vm_offset_t offset; 282 int *valid; 283 { 284 register sw_blk_t swb; 285 int ix; 286 287 if (valid) 288 *valid = 0; 289 ix = offset / (SWB_NPAGES * PAGE_SIZE); 290 if ((ix >= object->un_pager.swp.swp_nblocks) || 291 (offset >= object->size)) { 292 return (FALSE); 293 } 294 swb = &object->un_pager.swp.swp_blocks[ix]; 295 ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 296 if (valid) 297 *valid = swb->swb_valid & (1 << ix); 298 return &swb->swb_block[ix]; 299 } 300 301 /* 302 * Utility routine to set the valid (written) bit for 303 * a block associated with a pager and offset 304 */ 305 static void 306 swap_pager_setvalid(object, offset, valid) 307 vm_object_t object; 308 vm_offset_t offset; 309 int valid; 310 { 311 register sw_blk_t swb; 312 int ix; 313 314 ix = offset / (SWB_NPAGES * PAGE_SIZE); 315 if (ix >= object->un_pager.swp.swp_nblocks) 316 return; 317 318 swb = &object->un_pager.swp.swp_blocks[ix]; 319 ix = (offset % (SWB_NPAGES * PAGE_SIZE)) / PAGE_SIZE; 320 if (valid) 321 swb->swb_valid |= (1 << ix); 322 else 323 swb->swb_valid &= ~(1 << ix); 324 return; 325 } 326 327 /* 328 * this routine allocates swap space with a fragmentation 329 * minimization policy. 330 */ 331 int 332 swap_pager_getswapspace(object, amount, rtval) 333 vm_object_t object; 334 unsigned int amount; 335 unsigned int *rtval; 336 { 337 vm_swap_size -= amount; 338 if (!rlist_alloc(&swaplist, amount, rtval)) { 339 vm_swap_size += amount; 340 return 0; 341 } else { 342 swapsizecheck(); 343 object->un_pager.swp.swp_allocsize += amount; 344 return 1; 345 } 346 } 347 348 /* 349 * this routine frees swap space with a fragmentation 350 * minimization policy. 351 */ 352 void 353 swap_pager_freeswapspace(object, from, to) 354 vm_object_t object; 355 unsigned int from; 356 unsigned int to; 357 { 358 rlist_free(&swaplist, from, to); 359 vm_swap_size += (to - from) + 1; 360 object->un_pager.swp.swp_allocsize -= (to - from) + 1; 361 swapsizecheck(); 362 } 363 /* 364 * this routine frees swap blocks from a specified pager 365 */ 366 void 367 swap_pager_freespace(object, start, size) 368 vm_object_t object; 369 vm_offset_t start; 370 vm_offset_t size; 371 { 372 vm_offset_t i; 373 int s; 374 375 s = splbio(); 376 for (i = start; i < round_page(start + size); i += PAGE_SIZE) { 377 int valid; 378 int *addr = swap_pager_diskaddr(object, i, &valid); 379 380 if (addr && *addr != SWB_EMPTY) { 381 swap_pager_freeswapspace(object, *addr, *addr + btodb(PAGE_SIZE) - 1); 382 if (valid) { 383 swap_pager_setvalid(object, i, 0); 384 } 385 *addr = SWB_EMPTY; 386 } 387 } 388 splx(s); 389 } 390 391 static void 392 swap_pager_free_swap(object) 393 vm_object_t object; 394 { 395 register int i, j; 396 register sw_blk_t swb; 397 int first_block=0, block_count=0; 398 int s; 399 /* 400 * Free left over swap blocks 401 */ 402 s = splbio(); 403 for (i = 0, swb = object->un_pager.swp.swp_blocks; 404 i < object->un_pager.swp.swp_nblocks; i++, swb++) { 405 for (j = 0; j < SWB_NPAGES; j++) { 406 if (swb->swb_block[j] != SWB_EMPTY) { 407 /* 408 * initially the length of the run is zero 409 */ 410 if (block_count == 0) { 411 first_block = swb->swb_block[j]; 412 block_count = btodb(PAGE_SIZE); 413 swb->swb_block[j] = SWB_EMPTY; 414 /* 415 * if the new block can be included into the current run 416 */ 417 } else if (swb->swb_block[j] == first_block + block_count) { 418 block_count += btodb(PAGE_SIZE); 419 swb->swb_block[j] = SWB_EMPTY; 420 /* 421 * terminate the previous run, and start a new one 422 */ 423 } else { 424 swap_pager_freeswapspace(object, first_block, 425 (unsigned) first_block + block_count - 1); 426 first_block = swb->swb_block[j]; 427 block_count = btodb(PAGE_SIZE); 428 swb->swb_block[j] = SWB_EMPTY; 429 } 430 } 431 } 432 } 433 434 if (block_count) { 435 swap_pager_freeswapspace(object, first_block, 436 (unsigned) first_block + block_count - 1); 437 } 438 splx(s); 439 } 440 441 442 /* 443 * swap_pager_reclaim frees up over-allocated space from all pagers 444 * this eliminates internal fragmentation due to allocation of space 445 * for segments that are never swapped to. It has been written so that 446 * it does not block until the rlist_free operation occurs; it keeps 447 * the queues consistant. 448 */ 449 450 /* 451 * Maximum number of blocks (pages) to reclaim per pass 452 */ 453 #define MAXRECLAIM 128 454 455 void 456 swap_pager_reclaim() 457 { 458 vm_object_t object; 459 int i, j, k; 460 int s; 461 int reclaimcount; 462 static struct { 463 int address; 464 vm_object_t object; 465 } reclaims[MAXRECLAIM]; 466 static int in_reclaim; 467 468 /* 469 * allow only one process to be in the swap_pager_reclaim subroutine 470 */ 471 s = splbio(); 472 if (in_reclaim) { 473 tsleep(&in_reclaim, PSWP, "swrclm", 0); 474 splx(s); 475 return; 476 } 477 in_reclaim = 1; 478 reclaimcount = 0; 479 480 /* for each pager queue */ 481 for (k = 0; swp_qs[k]; k++) { 482 483 object = swp_qs[k]->tqh_first; 484 while (object && (reclaimcount < MAXRECLAIM)) { 485 486 /* 487 * see if any blocks associated with a pager has been 488 * allocated but not used (written) 489 */ 490 for (i = 0; i < object->un_pager.swp.swp_nblocks; i++) { 491 sw_blk_t swb = &object->un_pager.swp.swp_blocks[i]; 492 493 if (swb->swb_locked) 494 continue; 495 for (j = 0; j < SWB_NPAGES; j++) { 496 if (swb->swb_block[j] != SWB_EMPTY && 497 (swb->swb_valid & (1 << j)) == 0) { 498 reclaims[reclaimcount].address = swb->swb_block[j]; 499 reclaims[reclaimcount++].object = object; 500 swb->swb_block[j] = SWB_EMPTY; 501 if (reclaimcount >= MAXRECLAIM) 502 goto rfinished; 503 } 504 } 505 } 506 object = object->pager_object_list.tqe_next; 507 } 508 } 509 510 rfinished: 511 512 /* 513 * free the blocks that have been added to the reclaim list 514 */ 515 for (i = 0; i < reclaimcount; i++) { 516 swap_pager_freeswapspace(reclaims[i].object, 517 reclaims[i].address, reclaims[i].address + btodb(PAGE_SIZE) - 1); 518 } 519 splx(s); 520 in_reclaim = 0; 521 wakeup(&in_reclaim); 522 } 523 524 525 /* 526 * swap_pager_copy copies blocks from one pager to another and 527 * destroys the source pager 528 */ 529 530 void 531 swap_pager_copy(srcobject, srcoffset, dstobject, dstoffset, offset) 532 vm_object_t srcobject; 533 vm_offset_t srcoffset; 534 vm_object_t dstobject; 535 vm_offset_t dstoffset; 536 vm_offset_t offset; 537 { 538 vm_offset_t i; 539 int origsize; 540 int s; 541 542 if (vm_swap_size) 543 no_swap_space = 0; 544 545 origsize = srcobject->un_pager.swp.swp_allocsize; 546 547 /* 548 * remove the source object from the swap_pager internal queue 549 */ 550 if (srcobject->handle == NULL) { 551 TAILQ_REMOVE(&swap_pager_un_object_list, srcobject, pager_object_list); 552 } else { 553 TAILQ_REMOVE(&swap_pager_object_list, srcobject, pager_object_list); 554 } 555 556 s = splbio(); 557 while (srcobject->un_pager.swp.swp_poip) { 558 tsleep(srcobject, PVM, "spgout", 0); 559 } 560 splx(s); 561 562 /* 563 * clean all of the pages that are currently active and finished 564 */ 565 swap_pager_sync(); 566 567 s = splbio(); 568 /* 569 * transfer source to destination 570 */ 571 for (i = 0; i < dstobject->size; i += PAGE_SIZE) { 572 int srcvalid, dstvalid; 573 int *srcaddrp = swap_pager_diskaddr(srcobject, i + offset + srcoffset, 574 &srcvalid); 575 int *dstaddrp; 576 577 /* 578 * see if the source has space allocated 579 */ 580 if (srcaddrp && *srcaddrp != SWB_EMPTY) { 581 /* 582 * if the source is valid and the dest has no space, 583 * then copy the allocation from the srouce to the 584 * dest. 585 */ 586 if (srcvalid) { 587 dstaddrp = swap_pager_diskaddr(dstobject, i + dstoffset, 588 &dstvalid); 589 /* 590 * if the dest already has a valid block, 591 * deallocate the source block without 592 * copying. 593 */ 594 if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) { 595 swap_pager_freeswapspace(dstobject, *dstaddrp, 596 *dstaddrp + btodb(PAGE_SIZE) - 1); 597 *dstaddrp = SWB_EMPTY; 598 } 599 if (dstaddrp && *dstaddrp == SWB_EMPTY) { 600 *dstaddrp = *srcaddrp; 601 *srcaddrp = SWB_EMPTY; 602 dstobject->un_pager.swp.swp_allocsize += btodb(PAGE_SIZE); 603 srcobject->un_pager.swp.swp_allocsize -= btodb(PAGE_SIZE); 604 swap_pager_setvalid(dstobject, i + dstoffset, 1); 605 } 606 } 607 /* 608 * if the source is not empty at this point, then 609 * deallocate the space. 610 */ 611 if (*srcaddrp != SWB_EMPTY) { 612 swap_pager_freeswapspace(srcobject, *srcaddrp, 613 *srcaddrp + btodb(PAGE_SIZE) - 1); 614 *srcaddrp = SWB_EMPTY; 615 } 616 } 617 } 618 splx(s); 619 620 /* 621 * Free left over swap blocks 622 */ 623 swap_pager_free_swap(srcobject); 624 625 if (srcobject->un_pager.swp.swp_allocsize) { 626 printf("swap_pager_copy: *warning* pager with %d blocks (orig: %d)\n", 627 srcobject->un_pager.swp.swp_allocsize, origsize); 628 } 629 630 free(srcobject->un_pager.swp.swp_blocks, M_VMPGDATA); 631 srcobject->un_pager.swp.swp_blocks = NULL; 632 633 return; 634 } 635 636 void 637 swap_pager_dealloc(object) 638 vm_object_t object; 639 { 640 int s; 641 642 /* 643 * Remove from list right away so lookups will fail if we block for 644 * pageout completion. 645 */ 646 if (object->handle == NULL) { 647 TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list); 648 } else { 649 TAILQ_REMOVE(&swap_pager_object_list, object, pager_object_list); 650 } 651 652 /* 653 * Wait for all pageouts to finish and remove all entries from 654 * cleaning list. 655 */ 656 657 s = splbio(); 658 while (object->un_pager.swp.swp_poip) { 659 tsleep(object, PVM, "swpout", 0); 660 } 661 splx(s); 662 663 664 swap_pager_sync(); 665 666 /* 667 * Free left over swap blocks 668 */ 669 swap_pager_free_swap(object); 670 671 if (object->un_pager.swp.swp_allocsize) { 672 printf("swap_pager_dealloc: *warning* freeing pager with %d blocks\n", 673 object->un_pager.swp.swp_allocsize); 674 } 675 /* 676 * Free swap management resources 677 */ 678 free(object->un_pager.swp.swp_blocks, M_VMPGDATA); 679 object->un_pager.swp.swp_blocks = NULL; 680 } 681 682 static inline int 683 const 684 swap_pager_block_index(offset) 685 vm_offset_t offset; 686 { 687 return (offset / (SWB_NPAGES * PAGE_SIZE)); 688 } 689 690 static inline int 691 const 692 swap_pager_block_offset(offset) 693 vm_offset_t offset; 694 { 695 return ((offset % (PAGE_SIZE * SWB_NPAGES)) / PAGE_SIZE); 696 } 697 698 /* 699 * swap_pager_haspage returns TRUE if the pager has data that has 700 * been written out. 701 */ 702 boolean_t 703 swap_pager_haspage(object, offset, before, after) 704 vm_object_t object; 705 vm_offset_t offset; 706 int *before; 707 int *after; 708 { 709 register sw_blk_t swb; 710 int ix; 711 int gix; 712 713 if (before != NULL) 714 *before = 0; 715 if (after != NULL) 716 *after = 0; 717 ix = offset / (SWB_NPAGES * PAGE_SIZE); 718 if (ix >= object->un_pager.swp.swp_nblocks) { 719 return (FALSE); 720 } 721 swb = &object->un_pager.swp.swp_blocks[ix]; 722 gix = offset / PAGE_SIZE; 723 ix = gix % SWB_NPAGES; 724 725 if (swb->swb_block[ix] != SWB_EMPTY) { 726 727 if (swb->swb_valid & (1 << ix)) { 728 int tix; 729 if (before) { 730 for(tix = ix - 1; tix >= 0; --tix) { 731 if ((swb->swb_valid & (1 << tix)) == 0) 732 break; 733 if ((swb->swb_block[tix] + 734 (ix - tix) * (PAGE_SIZE/DEV_BSIZE)) != 735 swb->swb_block[ix]) 736 break; 737 (*before)++; 738 } 739 } 740 741 if (after) { 742 for(tix = ix + 1; tix < SWB_NPAGES; tix++) { 743 if ((swb->swb_valid & (1 << tix)) == 0) 744 break; 745 if ((swb->swb_block[tix] - 746 (tix - ix) * (PAGE_SIZE/DEV_BSIZE)) != 747 swb->swb_block[ix]) 748 break; 749 (*after)++; 750 } 751 } 752 753 return TRUE; 754 } 755 } 756 return (FALSE); 757 } 758 759 /* 760 * swap_pager_freepage is a convienience routine that clears the busy 761 * bit and deallocates a page. 762 */ 763 static void 764 swap_pager_freepage(m) 765 vm_page_t m; 766 { 767 PAGE_WAKEUP(m); 768 vm_page_free(m); 769 } 770 771 /* 772 * swap_pager_ridpages is a convienience routine that deallocates all 773 * but the required page. this is usually used in error returns that 774 * need to invalidate the "extra" readahead pages. 775 */ 776 static void 777 swap_pager_ridpages(m, count, reqpage) 778 vm_page_t *m; 779 int count; 780 int reqpage; 781 { 782 int i; 783 784 for (i = 0; i < count; i++) 785 if (i != reqpage) 786 swap_pager_freepage(m[i]); 787 } 788 789 /* 790 * swap_pager_iodone1 is the completion routine for both reads and async writes 791 */ 792 void 793 swap_pager_iodone1(bp) 794 struct buf *bp; 795 { 796 bp->b_flags |= B_DONE; 797 bp->b_flags &= ~B_ASYNC; 798 wakeup(bp); 799 } 800 801 int 802 swap_pager_getpages(object, m, count, reqpage) 803 vm_object_t object; 804 vm_page_t *m; 805 int count, reqpage; 806 { 807 register struct buf *bp; 808 sw_blk_t swb[count]; 809 register int s; 810 int i; 811 boolean_t rv; 812 vm_offset_t kva, off[count]; 813 swp_clean_t spc; 814 vm_offset_t paging_offset; 815 int reqaddr[count]; 816 int sequential; 817 818 int first, last; 819 int failed; 820 int reqdskregion; 821 822 object = m[reqpage]->object; 823 paging_offset = object->paging_offset; 824 sequential = (m[reqpage]->offset == (object->last_read + PAGE_SIZE)); 825 826 for (i = 0; i < count; i++) { 827 vm_offset_t foff = m[i]->offset + paging_offset; 828 int ix = swap_pager_block_index(foff); 829 830 if (ix >= object->un_pager.swp.swp_nblocks) { 831 int j; 832 833 if (i <= reqpage) { 834 swap_pager_ridpages(m, count, reqpage); 835 return (VM_PAGER_FAIL); 836 } 837 for (j = i; j < count; j++) { 838 swap_pager_freepage(m[j]); 839 } 840 count = i; 841 break; 842 } 843 swb[i] = &object->un_pager.swp.swp_blocks[ix]; 844 off[i] = swap_pager_block_offset(foff); 845 reqaddr[i] = swb[i]->swb_block[off[i]]; 846 } 847 848 /* make sure that our required input request is existant */ 849 850 if (reqaddr[reqpage] == SWB_EMPTY || 851 (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) { 852 swap_pager_ridpages(m, count, reqpage); 853 return (VM_PAGER_FAIL); 854 } 855 reqdskregion = reqaddr[reqpage] / dmmax; 856 857 /* 858 * search backwards for the first contiguous page to transfer 859 */ 860 failed = 0; 861 first = 0; 862 for (i = reqpage - 1; i >= 0; --i) { 863 if (sequential || failed || (reqaddr[i] == SWB_EMPTY) || 864 (swb[i]->swb_valid & (1 << off[i])) == 0 || 865 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 866 ((reqaddr[i] / dmmax) != reqdskregion)) { 867 failed = 1; 868 swap_pager_freepage(m[i]); 869 if (first == 0) 870 first = i + 1; 871 } 872 } 873 /* 874 * search forwards for the last contiguous page to transfer 875 */ 876 failed = 0; 877 last = count; 878 for (i = reqpage + 1; i < count; i++) { 879 if (failed || (reqaddr[i] == SWB_EMPTY) || 880 (swb[i]->swb_valid & (1 << off[i])) == 0 || 881 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 882 ((reqaddr[i] / dmmax) != reqdskregion)) { 883 failed = 1; 884 swap_pager_freepage(m[i]); 885 if (last == count) 886 last = i; 887 } 888 } 889 890 count = last; 891 if (first != 0) { 892 for (i = first; i < count; i++) { 893 m[i - first] = m[i]; 894 reqaddr[i - first] = reqaddr[i]; 895 off[i - first] = off[i]; 896 } 897 count -= first; 898 reqpage -= first; 899 } 900 ++swb[reqpage]->swb_locked; 901 902 /* 903 * at this point: "m" is a pointer to the array of vm_page_t for 904 * paging I/O "count" is the number of vm_page_t entries represented 905 * by "m" "object" is the vm_object_t for I/O "reqpage" is the index 906 * into "m" for the page actually faulted 907 */ 908 909 spc = NULL; /* we might not use an spc data structure */ 910 911 if ((count == 1) && (swap_pager_free.tqh_first != NULL)) { 912 /* 913 * if a kva has not been allocated, we can only do a one page 914 * transfer, so we free the other pages that might have been 915 * allocated by vm_fault. 916 */ 917 swap_pager_ridpages(m, count, reqpage); 918 m[0] = m[reqpage]; 919 reqaddr[0] = reqaddr[reqpage]; 920 921 count = 1; 922 reqpage = 0; 923 /* 924 * get a swap pager clean data structure, block until we get 925 * it 926 */ 927 if (swap_pager_free.tqh_first == NULL) { 928 s = splbio(); 929 if (curproc == pageproc) 930 swap_pager_sync(); 931 else 932 pagedaemon_wakeup(); 933 while (swap_pager_free.tqh_first == NULL) { 934 swap_pager_needflags |= SWAP_FREE_NEEDED; 935 if (curproc == pageproc) 936 swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 937 tsleep(&swap_pager_free, 938 PVM, "swpfre", 0); 939 if (curproc == pageproc) 940 swap_pager_sync(); 941 else 942 pagedaemon_wakeup(); 943 } 944 splx(s); 945 } 946 spc = swap_pager_free.tqh_first; 947 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 948 kva = spc->spc_kva; 949 bp = spc->spc_bp; 950 bzero(bp, sizeof *bp); 951 bp->b_spc = spc; 952 bp->b_vnbufs.le_next = NOLIST; 953 } else { 954 /* 955 * Get a swap buffer header to perform the IO 956 */ 957 bp = getpbuf(); 958 kva = (vm_offset_t) bp->b_data; 959 } 960 961 /* 962 * map our page(s) into kva for input 963 */ 964 pmap_qenter(kva, m, count); 965 966 bp->b_flags = B_BUSY | B_READ | B_CALL | B_PAGING; 967 bp->b_iodone = swap_pager_iodone1; 968 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 969 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 970 crhold(bp->b_rcred); 971 crhold(bp->b_wcred); 972 bp->b_un.b_addr = (caddr_t) kva; 973 bp->b_blkno = reqaddr[0]; 974 bp->b_bcount = PAGE_SIZE * count; 975 bp->b_bufsize = PAGE_SIZE * count; 976 977 pbgetvp(swapdev_vp, bp); 978 979 cnt.v_swapin++; 980 cnt.v_swappgsin += count; 981 /* 982 * perform the I/O 983 */ 984 VOP_STRATEGY(bp); 985 986 /* 987 * wait for the sync I/O to complete 988 */ 989 s = splbio(); 990 while ((bp->b_flags & B_DONE) == 0) { 991 tsleep(bp, PVM, "swread", 0); 992 } 993 994 if (bp->b_flags & B_ERROR) { 995 printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n", 996 bp->b_blkno, bp->b_bcount, bp->b_error); 997 rv = VM_PAGER_ERROR; 998 } else { 999 rv = VM_PAGER_OK; 1000 } 1001 1002 /* 1003 * relpbuf does this, but we maintain our own buffer list also... 1004 */ 1005 if (bp->b_vp) 1006 pbrelvp(bp); 1007 1008 splx(s); 1009 swb[reqpage]->swb_locked--; 1010 1011 /* 1012 * remove the mapping for kernel virtual 1013 */ 1014 pmap_qremove(kva, count); 1015 1016 if (spc) { 1017 m[reqpage]->object->last_read = m[reqpage]->offset; 1018 if (bp->b_flags & B_WANTED) 1019 wakeup(bp); 1020 /* 1021 * if we have used an spc, we need to free it. 1022 */ 1023 if (bp->b_rcred != NOCRED) 1024 crfree(bp->b_rcred); 1025 if (bp->b_wcred != NOCRED) 1026 crfree(bp->b_wcred); 1027 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1028 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1029 wakeup(&swap_pager_free); 1030 } 1031 if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1032 pagedaemon_wakeup(); 1033 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1034 } else { 1035 /* 1036 * release the physical I/O buffer 1037 */ 1038 relpbuf(bp); 1039 /* 1040 * finish up input if everything is ok 1041 */ 1042 if (rv == VM_PAGER_OK) { 1043 for (i = 0; i < count; i++) { 1044 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1045 m[i]->dirty = 0; 1046 m[i]->flags &= ~PG_ZERO; 1047 if (i != reqpage) { 1048 /* 1049 * whether or not to leave the page 1050 * activated is up in the air, but we 1051 * should put the page on a page queue 1052 * somewhere. (it already is in the 1053 * object). After some emperical 1054 * results, it is best to deactivate 1055 * the readahead pages. 1056 */ 1057 vm_page_deactivate(m[i]); 1058 1059 /* 1060 * just in case someone was asking for 1061 * this page we now tell them that it 1062 * is ok to use 1063 */ 1064 m[i]->valid = VM_PAGE_BITS_ALL; 1065 PAGE_WAKEUP(m[i]); 1066 } 1067 } 1068 1069 m[reqpage]->object->last_read = m[count-1]->offset; 1070 1071 /* 1072 * If we're out of swap space, then attempt to free 1073 * some whenever pages are brought in. We must clear 1074 * the clean flag so that the page contents will be 1075 * preserved. 1076 */ 1077 if (swap_pager_full) { 1078 for (i = 0; i < count; i++) { 1079 m[i]->dirty = VM_PAGE_BITS_ALL; 1080 } 1081 swap_pager_freespace(object, m[0]->offset + paging_offset, count * PAGE_SIZE); 1082 } 1083 } else { 1084 swap_pager_ridpages(m, count, reqpage); 1085 } 1086 } 1087 return (rv); 1088 } 1089 1090 int 1091 swap_pager_putpages(object, m, count, sync, rtvals) 1092 vm_object_t object; 1093 vm_page_t *m; 1094 int count; 1095 boolean_t sync; 1096 int *rtvals; 1097 { 1098 register struct buf *bp; 1099 sw_blk_t swb[count]; 1100 register int s; 1101 int i, j, ix; 1102 boolean_t rv; 1103 vm_offset_t kva, off, foff; 1104 swp_clean_t spc; 1105 vm_offset_t paging_offset; 1106 int reqaddr[count]; 1107 int failed; 1108 1109 if (vm_swap_size) 1110 no_swap_space = 0; 1111 if (no_swap_space) { 1112 for (i = 0; i < count; i++) 1113 rtvals[i] = VM_PAGER_FAIL; 1114 return VM_PAGER_FAIL; 1115 } 1116 spc = NULL; 1117 1118 object = m[0]->object; 1119 paging_offset = object->paging_offset; 1120 1121 failed = 0; 1122 for (j = 0; j < count; j++) { 1123 foff = m[j]->offset + paging_offset; 1124 ix = swap_pager_block_index(foff); 1125 swb[j] = 0; 1126 if (ix >= object->un_pager.swp.swp_nblocks) { 1127 rtvals[j] = VM_PAGER_FAIL; 1128 failed = 1; 1129 continue; 1130 } else { 1131 rtvals[j] = VM_PAGER_OK; 1132 } 1133 swb[j] = &object->un_pager.swp.swp_blocks[ix]; 1134 swb[j]->swb_locked++; 1135 if (failed) { 1136 rtvals[j] = VM_PAGER_FAIL; 1137 continue; 1138 } 1139 off = swap_pager_block_offset(foff); 1140 reqaddr[j] = swb[j]->swb_block[off]; 1141 if (reqaddr[j] == SWB_EMPTY) { 1142 int blk; 1143 int tries; 1144 int ntoget; 1145 1146 tries = 0; 1147 s = splbio(); 1148 1149 /* 1150 * if any other pages have been allocated in this 1151 * block, we only try to get one page. 1152 */ 1153 for (i = 0; i < SWB_NPAGES; i++) { 1154 if (swb[j]->swb_block[i] != SWB_EMPTY) 1155 break; 1156 } 1157 1158 ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1; 1159 /* 1160 * this code is alittle conservative, but works (the 1161 * intent of this code is to allocate small chunks for 1162 * small objects) 1163 */ 1164 if ((foff == 0) && 1165 ((ntoget * PAGE_SIZE) > object->size)) { 1166 ntoget = (object->size + (PAGE_SIZE - 1)) / PAGE_SIZE; 1167 } 1168 retrygetspace: 1169 if (!swap_pager_full && ntoget > 1 && 1170 swap_pager_getswapspace(object, ntoget * btodb(PAGE_SIZE), &blk)) { 1171 1172 for (i = 0; i < ntoget; i++) { 1173 swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i; 1174 swb[j]->swb_valid = 0; 1175 } 1176 1177 reqaddr[j] = swb[j]->swb_block[off]; 1178 } else if (!swap_pager_getswapspace(object, btodb(PAGE_SIZE), 1179 &swb[j]->swb_block[off])) { 1180 /* 1181 * if the allocation has failed, we try to 1182 * reclaim space and retry. 1183 */ 1184 if (++tries == 1) { 1185 swap_pager_reclaim(); 1186 goto retrygetspace; 1187 } 1188 rtvals[j] = VM_PAGER_AGAIN; 1189 failed = 1; 1190 swap_pager_full = 1; 1191 } else { 1192 reqaddr[j] = swb[j]->swb_block[off]; 1193 swb[j]->swb_valid &= ~(1 << off); 1194 } 1195 splx(s); 1196 } 1197 } 1198 1199 /* 1200 * search forwards for the last contiguous page to transfer 1201 */ 1202 failed = 0; 1203 for (i = 0; i < count; i++) { 1204 if (failed || (reqaddr[i] != reqaddr[0] + i * btodb(PAGE_SIZE)) || 1205 (reqaddr[i] / dmmax) != (reqaddr[0] / dmmax) || 1206 (rtvals[i] != VM_PAGER_OK)) { 1207 failed = 1; 1208 if (rtvals[i] == VM_PAGER_OK) 1209 rtvals[i] = VM_PAGER_AGAIN; 1210 } 1211 } 1212 1213 for (i = 0; i < count; i++) { 1214 if (rtvals[i] != VM_PAGER_OK) { 1215 if (swb[i]) 1216 --swb[i]->swb_locked; 1217 } 1218 } 1219 1220 for (i = 0; i < count; i++) 1221 if (rtvals[i] != VM_PAGER_OK) 1222 break; 1223 1224 if (i == 0) { 1225 return VM_PAGER_AGAIN; 1226 } 1227 count = i; 1228 for (i = 0; i < count; i++) { 1229 if (reqaddr[i] == SWB_EMPTY) 1230 printf("I/O to empty block????\n"); 1231 } 1232 1233 /* 1234 * For synchronous writes, we clean up all completed async pageouts. 1235 */ 1236 if (sync == TRUE) { 1237 swap_pager_sync(); 1238 } 1239 kva = 0; 1240 1241 /* 1242 * get a swap pager clean data structure, block until we get it 1243 */ 1244 if (swap_pager_free.tqh_first == NULL || 1245 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1246 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1247 s = splbio(); 1248 if (curproc == pageproc) { 1249 swap_pager_sync(); 1250 #if 0 1251 splx(s); 1252 return VM_PAGER_AGAIN; 1253 #endif 1254 } else 1255 pagedaemon_wakeup(); 1256 while (swap_pager_free.tqh_first == NULL || 1257 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1258 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1259 if (curproc == pageproc) { 1260 swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 1261 if((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_reserved) 1262 wakeup(&cnt.v_free_count); 1263 } 1264 1265 swap_pager_needflags |= SWAP_FREE_NEEDED; 1266 tsleep(&swap_pager_free, PVM, "swpfre", 0); 1267 if (curproc == pageproc) 1268 swap_pager_sync(); 1269 else 1270 pagedaemon_wakeup(); 1271 } 1272 splx(s); 1273 } 1274 spc = swap_pager_free.tqh_first; 1275 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1276 1277 kva = spc->spc_kva; 1278 1279 /* 1280 * map our page(s) into kva for I/O 1281 */ 1282 pmap_qenter(kva, m, count); 1283 1284 /* 1285 * get the base I/O offset into the swap file 1286 */ 1287 for (i = 0; i < count; i++) { 1288 foff = m[i]->offset + paging_offset; 1289 off = swap_pager_block_offset(foff); 1290 /* 1291 * set the valid bit 1292 */ 1293 swb[i]->swb_valid |= (1 << off); 1294 /* 1295 * and unlock the data structure 1296 */ 1297 swb[i]->swb_locked--; 1298 } 1299 1300 /* 1301 * Get a swap buffer header and perform the IO 1302 */ 1303 bp = spc->spc_bp; 1304 bzero(bp, sizeof *bp); 1305 bp->b_spc = spc; 1306 bp->b_vnbufs.le_next = NOLIST; 1307 1308 bp->b_flags = B_BUSY | B_PAGING; 1309 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 1310 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1311 if (bp->b_rcred != NOCRED) 1312 crhold(bp->b_rcred); 1313 if (bp->b_wcred != NOCRED) 1314 crhold(bp->b_wcred); 1315 bp->b_data = (caddr_t) kva; 1316 bp->b_blkno = reqaddr[0]; 1317 pbgetvp(swapdev_vp, bp); 1318 1319 bp->b_bcount = PAGE_SIZE * count; 1320 bp->b_bufsize = PAGE_SIZE * count; 1321 swapdev_vp->v_numoutput++; 1322 1323 /* 1324 * If this is an async write we set up additional buffer fields and 1325 * place a "cleaning" entry on the inuse queue. 1326 */ 1327 s = splbio(); 1328 if (sync == FALSE) { 1329 spc->spc_flags = 0; 1330 spc->spc_object = object; 1331 for (i = 0; i < count; i++) 1332 spc->spc_m[i] = m[i]; 1333 spc->spc_count = count; 1334 /* 1335 * the completion routine for async writes 1336 */ 1337 bp->b_flags |= B_CALL; 1338 bp->b_iodone = swap_pager_iodone; 1339 bp->b_dirtyoff = 0; 1340 bp->b_dirtyend = bp->b_bcount; 1341 object->un_pager.swp.swp_poip++; 1342 TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 1343 } else { 1344 object->un_pager.swp.swp_poip++; 1345 bp->b_flags |= B_CALL; 1346 bp->b_iodone = swap_pager_iodone1; 1347 } 1348 1349 cnt.v_swapout++; 1350 cnt.v_swappgsout += count; 1351 /* 1352 * perform the I/O 1353 */ 1354 VOP_STRATEGY(bp); 1355 if (sync == FALSE) { 1356 if ((bp->b_flags & B_DONE) == B_DONE) { 1357 swap_pager_sync(); 1358 } 1359 splx(s); 1360 for (i = 0; i < count; i++) { 1361 rtvals[i] = VM_PAGER_PEND; 1362 } 1363 return VM_PAGER_PEND; 1364 } 1365 /* 1366 * wait for the sync I/O to complete 1367 */ 1368 while ((bp->b_flags & B_DONE) == 0) { 1369 tsleep(bp, PVM, "swwrt", 0); 1370 } 1371 if (bp->b_flags & B_ERROR) { 1372 printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n", 1373 bp->b_blkno, bp->b_bcount, bp->b_error); 1374 rv = VM_PAGER_ERROR; 1375 } else { 1376 rv = VM_PAGER_OK; 1377 } 1378 1379 object->un_pager.swp.swp_poip--; 1380 if (object->un_pager.swp.swp_poip == 0) 1381 wakeup(object); 1382 1383 if (bp->b_vp) 1384 pbrelvp(bp); 1385 if (bp->b_flags & B_WANTED) 1386 wakeup(bp); 1387 1388 splx(s); 1389 1390 /* 1391 * remove the mapping for kernel virtual 1392 */ 1393 pmap_qremove(kva, count); 1394 1395 /* 1396 * if we have written the page, then indicate that the page is clean. 1397 */ 1398 if (rv == VM_PAGER_OK) { 1399 for (i = 0; i < count; i++) { 1400 if (rtvals[i] == VM_PAGER_OK) { 1401 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1402 m[i]->dirty = 0; 1403 /* 1404 * optimization, if a page has been read 1405 * during the pageout process, we activate it. 1406 */ 1407 if ((m[i]->flags & PG_ACTIVE) == 0 && 1408 ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) || 1409 pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) { 1410 vm_page_activate(m[i]); 1411 } 1412 } 1413 } 1414 } else { 1415 for (i = 0; i < count; i++) { 1416 rtvals[i] = rv; 1417 } 1418 } 1419 1420 if (bp->b_rcred != NOCRED) 1421 crfree(bp->b_rcred); 1422 if (bp->b_wcred != NOCRED) 1423 crfree(bp->b_wcred); 1424 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1425 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1426 wakeup(&swap_pager_free); 1427 } 1428 if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1429 pagedaemon_wakeup(); 1430 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1431 return (rv); 1432 } 1433 1434 void 1435 swap_pager_sync() 1436 { 1437 register swp_clean_t spc, tspc; 1438 register int s; 1439 1440 tspc = NULL; 1441 if (swap_pager_done.tqh_first == NULL) 1442 return; 1443 for (;;) { 1444 s = splbio(); 1445 /* 1446 * Look up and removal from done list must be done at splbio() 1447 * to avoid conflicts with swap_pager_iodone. 1448 */ 1449 while ((spc = swap_pager_done.tqh_first) != 0) { 1450 pmap_qremove(spc->spc_kva, spc->spc_count); 1451 swap_pager_finish(spc); 1452 TAILQ_REMOVE(&swap_pager_done, spc, spc_list); 1453 goto doclean; 1454 } 1455 1456 /* 1457 * No operations done, thats all we can do for now. 1458 */ 1459 1460 splx(s); 1461 break; 1462 1463 /* 1464 * The desired page was found to be busy earlier in the scan 1465 * but has since completed. 1466 */ 1467 doclean: 1468 if (tspc && tspc == spc) { 1469 tspc = NULL; 1470 } 1471 spc->spc_flags = 0; 1472 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1473 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1474 wakeup(&swap_pager_free); 1475 } 1476 if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1477 pagedaemon_wakeup(); 1478 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1479 splx(s); 1480 } 1481 1482 return; 1483 } 1484 1485 void 1486 swap_pager_finish(spc) 1487 register swp_clean_t spc; 1488 { 1489 vm_object_t object = spc->spc_m[0]->object; 1490 int i; 1491 1492 object->paging_in_progress -= spc->spc_count; 1493 if ((object->paging_in_progress == 0) && 1494 (object->flags & OBJ_PIPWNT)) { 1495 object->flags &= ~OBJ_PIPWNT; 1496 wakeup(object); 1497 } 1498 1499 /* 1500 * If no error, mark as clean and inform the pmap system. If error, 1501 * mark as dirty so we will try again. (XXX could get stuck doing 1502 * this, should give up after awhile) 1503 */ 1504 if (spc->spc_flags & SPC_ERROR) { 1505 for (i = 0; i < spc->spc_count; i++) { 1506 printf("swap_pager_finish: I/O error, clean of page %lx failed\n", 1507 (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i])); 1508 } 1509 } else { 1510 for (i = 0; i < spc->spc_count; i++) { 1511 pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); 1512 spc->spc_m[i]->dirty = 0; 1513 if ((spc->spc_m[i]->flags & PG_ACTIVE) == 0 && 1514 ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i])))) 1515 vm_page_activate(spc->spc_m[i]); 1516 } 1517 } 1518 1519 1520 for (i = 0; i < spc->spc_count; i++) { 1521 /* 1522 * we wakeup any processes that are waiting on these pages. 1523 */ 1524 PAGE_WAKEUP(spc->spc_m[i]); 1525 } 1526 nswiodone -= spc->spc_count; 1527 1528 return; 1529 } 1530 1531 /* 1532 * swap_pager_iodone 1533 */ 1534 void 1535 swap_pager_iodone(bp) 1536 register struct buf *bp; 1537 { 1538 register swp_clean_t spc; 1539 int s; 1540 1541 s = splbio(); 1542 spc = (swp_clean_t) bp->b_spc; 1543 TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 1544 TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); 1545 if (bp->b_flags & B_ERROR) { 1546 spc->spc_flags |= SPC_ERROR; 1547 printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d\n", 1548 (bp->b_flags & B_READ) ? "pagein" : "pageout", 1549 (u_long) bp->b_blkno, bp->b_bcount, bp->b_error); 1550 } 1551 1552 if (bp->b_vp) 1553 pbrelvp(bp); 1554 1555 if (bp->b_flags & B_WANTED) 1556 wakeup(bp); 1557 1558 if (bp->b_rcred != NOCRED) 1559 crfree(bp->b_rcred); 1560 if (bp->b_wcred != NOCRED) 1561 crfree(bp->b_wcred); 1562 1563 nswiodone += spc->spc_count; 1564 if (--spc->spc_object->un_pager.swp.swp_poip == 0) { 1565 wakeup(spc->spc_object); 1566 } 1567 if ((swap_pager_needflags & SWAP_FREE_NEEDED) || 1568 swap_pager_inuse.tqh_first == 0) { 1569 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1570 wakeup(&swap_pager_free); 1571 } 1572 1573 if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) { 1574 swap_pager_needflags &= ~SWAP_FREE_NEEDED_BY_PAGEOUT; 1575 pagedaemon_wakeup(); 1576 } 1577 1578 if (vm_pageout_pages_needed) { 1579 wakeup(&vm_pageout_pages_needed); 1580 vm_pageout_pages_needed = 0; 1581 } 1582 if ((swap_pager_inuse.tqh_first == NULL) || 1583 ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min && 1584 nswiodone + cnt.v_free_count + cnt.v_cache_count >= cnt.v_free_min)) { 1585 pagedaemon_wakeup(); 1586 } 1587 splx(s); 1588 } 1589 1590 /* 1591 * return true if any swap control structures can be allocated 1592 */ 1593 int 1594 swap_pager_ready() 1595 { 1596 if (swap_pager_free.tqh_first) 1597 return 1; 1598 else 1599 return 0; 1600 } 1601