1 /*- 2 * Copyright (c) 1998 Matthew Dillon, 3 * Copyright (c) 1994 John S. Dyson 4 * Copyright (c) 1990 University of Utah. 5 * Copyright (c) 1982, 1986, 1989, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * New Swap System 41 * Matthew Dillon 42 * 43 * Radix Bitmap 'blists'. 44 * 45 * - The new swapper uses the new radix bitmap code. This should scale 46 * to arbitrarily small or arbitrarily large swap spaces and an almost 47 * arbitrary degree of fragmentation. 48 * 49 * Features: 50 * 51 * - on the fly reallocation of swap during putpages. The new system 52 * does not try to keep previously allocated swap blocks for dirty 53 * pages. 54 * 55 * - on the fly deallocation of swap 56 * 57 * - No more garbage collection required. Unnecessarily allocated swap 58 * blocks only exist for dirty vm_page_t's now and these are already 59 * cycled (in a high-load system) by the pager. We also do on-the-fly 60 * removal of invalidated swap blocks when a page is destroyed 61 * or renamed. 62 * 63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 64 * 65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 66 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94 67 */ 68 69 #include <sys/cdefs.h> 70 __FBSDID("$FreeBSD$"); 71 72 #include "opt_compat.h" 73 #include "opt_swap.h" 74 #include "opt_vm.h" 75 76 #include <sys/param.h> 77 #include <sys/systm.h> 78 #include <sys/conf.h> 79 #include <sys/kernel.h> 80 #include <sys/priv.h> 81 #include <sys/proc.h> 82 #include <sys/bio.h> 83 #include <sys/buf.h> 84 #include <sys/disk.h> 85 #include <sys/fcntl.h> 86 #include <sys/mount.h> 87 #include <sys/namei.h> 88 #include <sys/vnode.h> 89 #include <sys/malloc.h> 90 #include <sys/pctrie.h> 91 #include <sys/racct.h> 92 #include <sys/resource.h> 93 #include <sys/resourcevar.h> 94 #include <sys/rwlock.h> 95 #include <sys/sbuf.h> 96 #include <sys/sysctl.h> 97 #include <sys/sysproto.h> 98 #include <sys/blist.h> 99 #include <sys/lock.h> 100 #include <sys/sx.h> 101 #include <sys/vmmeter.h> 102 103 #include <security/mac/mac_framework.h> 104 105 #include <vm/vm.h> 106 #include <vm/pmap.h> 107 #include <vm/vm_map.h> 108 #include <vm/vm_kern.h> 109 #include <vm/vm_object.h> 110 #include <vm/vm_page.h> 111 #include <vm/vm_pager.h> 112 #include <vm/vm_pageout.h> 113 #include <vm/vm_param.h> 114 #include <vm/swap_pager.h> 115 #include <vm/vm_extern.h> 116 #include <vm/uma.h> 117 118 #include <geom/geom.h> 119 120 /* 121 * MAX_PAGEOUT_CLUSTER must be a power of 2 between 1 and 64. 122 * The 64-page limit is due to the radix code (kern/subr_blist.c). 123 */ 124 #ifndef MAX_PAGEOUT_CLUSTER 125 #define MAX_PAGEOUT_CLUSTER 32 126 #endif 127 128 #if !defined(SWB_NPAGES) 129 #define SWB_NPAGES MAX_PAGEOUT_CLUSTER 130 #endif 131 132 #define SWAP_META_PAGES PCTRIE_COUNT 133 134 /* 135 * A swblk structure maps each page index within a 136 * SWAP_META_PAGES-aligned and sized range to the address of an 137 * on-disk swap block (or SWAPBLK_NONE). The collection of these 138 * mappings for an entire vm object is implemented as a pc-trie. 139 */ 140 struct swblk { 141 vm_pindex_t p; 142 daddr_t d[SWAP_META_PAGES]; 143 }; 144 145 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data"); 146 static struct mtx sw_dev_mtx; 147 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq); 148 static struct swdevt *swdevhd; /* Allocate from here next */ 149 static int nswapdev; /* Number of swap devices */ 150 int swap_pager_avail; 151 static struct sx swdev_syscall_lock; /* serialize swap(on|off) */ 152 153 static vm_ooffset_t swap_total; 154 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0, 155 "Total amount of available swap storage."); 156 static vm_ooffset_t swap_reserved; 157 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0, 158 "Amount of swap storage needed to back all allocated anonymous memory."); 159 static int overcommit = 0; 160 SYSCTL_INT(_vm, VM_OVERCOMMIT, overcommit, CTLFLAG_RW, &overcommit, 0, 161 "Configure virtual memory overcommit behavior. See tuning(7) " 162 "for details."); 163 static unsigned long swzone; 164 SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0, 165 "Actual size of swap metadata zone"); 166 static unsigned long swap_maxpages; 167 SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0, 168 "Maximum amount of swap supported"); 169 170 /* bits from overcommit */ 171 #define SWAP_RESERVE_FORCE_ON (1 << 0) 172 #define SWAP_RESERVE_RLIMIT_ON (1 << 1) 173 #define SWAP_RESERVE_ALLOW_NONWIRED (1 << 2) 174 175 int 176 swap_reserve(vm_ooffset_t incr) 177 { 178 179 return (swap_reserve_by_cred(incr, curthread->td_ucred)); 180 } 181 182 int 183 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred) 184 { 185 vm_ooffset_t r, s; 186 int res, error; 187 static int curfail; 188 static struct timeval lastfail; 189 struct uidinfo *uip; 190 191 uip = cred->cr_ruidinfo; 192 193 if (incr & PAGE_MASK) 194 panic("swap_reserve: & PAGE_MASK"); 195 196 #ifdef RACCT 197 if (racct_enable) { 198 PROC_LOCK(curproc); 199 error = racct_add(curproc, RACCT_SWAP, incr); 200 PROC_UNLOCK(curproc); 201 if (error != 0) 202 return (0); 203 } 204 #endif 205 206 res = 0; 207 mtx_lock(&sw_dev_mtx); 208 r = swap_reserved + incr; 209 if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) { 210 s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count; 211 s *= PAGE_SIZE; 212 } else 213 s = 0; 214 s += swap_total; 215 if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s || 216 (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) { 217 res = 1; 218 swap_reserved = r; 219 } 220 mtx_unlock(&sw_dev_mtx); 221 222 if (res) { 223 UIDINFO_VMSIZE_LOCK(uip); 224 if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 && 225 uip->ui_vmsize + incr > lim_cur(curthread, RLIMIT_SWAP) && 226 priv_check(curthread, PRIV_VM_SWAP_NORLIMIT)) 227 res = 0; 228 else 229 uip->ui_vmsize += incr; 230 UIDINFO_VMSIZE_UNLOCK(uip); 231 if (!res) { 232 mtx_lock(&sw_dev_mtx); 233 swap_reserved -= incr; 234 mtx_unlock(&sw_dev_mtx); 235 } 236 } 237 if (!res && ppsratecheck(&lastfail, &curfail, 1)) { 238 printf("uid %d, pid %d: swap reservation for %jd bytes failed\n", 239 uip->ui_uid, curproc->p_pid, incr); 240 } 241 242 #ifdef RACCT 243 if (!res) { 244 PROC_LOCK(curproc); 245 racct_sub(curproc, RACCT_SWAP, incr); 246 PROC_UNLOCK(curproc); 247 } 248 #endif 249 250 return (res); 251 } 252 253 void 254 swap_reserve_force(vm_ooffset_t incr) 255 { 256 struct uidinfo *uip; 257 258 mtx_lock(&sw_dev_mtx); 259 swap_reserved += incr; 260 mtx_unlock(&sw_dev_mtx); 261 262 #ifdef RACCT 263 PROC_LOCK(curproc); 264 racct_add_force(curproc, RACCT_SWAP, incr); 265 PROC_UNLOCK(curproc); 266 #endif 267 268 uip = curthread->td_ucred->cr_ruidinfo; 269 PROC_LOCK(curproc); 270 UIDINFO_VMSIZE_LOCK(uip); 271 uip->ui_vmsize += incr; 272 UIDINFO_VMSIZE_UNLOCK(uip); 273 PROC_UNLOCK(curproc); 274 } 275 276 void 277 swap_release(vm_ooffset_t decr) 278 { 279 struct ucred *cred; 280 281 PROC_LOCK(curproc); 282 cred = curthread->td_ucred; 283 swap_release_by_cred(decr, cred); 284 PROC_UNLOCK(curproc); 285 } 286 287 void 288 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred) 289 { 290 struct uidinfo *uip; 291 292 uip = cred->cr_ruidinfo; 293 294 if (decr & PAGE_MASK) 295 panic("swap_release: & PAGE_MASK"); 296 297 mtx_lock(&sw_dev_mtx); 298 if (swap_reserved < decr) 299 panic("swap_reserved < decr"); 300 swap_reserved -= decr; 301 mtx_unlock(&sw_dev_mtx); 302 303 UIDINFO_VMSIZE_LOCK(uip); 304 if (uip->ui_vmsize < decr) 305 printf("negative vmsize for uid = %d\n", uip->ui_uid); 306 uip->ui_vmsize -= decr; 307 UIDINFO_VMSIZE_UNLOCK(uip); 308 309 racct_sub_cred(cred, RACCT_SWAP, decr); 310 } 311 312 #define SWM_FREE 0x02 /* free, period */ 313 #define SWM_POP 0x04 /* pop out */ 314 315 static int swap_pager_full = 2; /* swap space exhaustion (task killing) */ 316 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ 317 static int nsw_rcount; /* free read buffers */ 318 static int nsw_wcount_sync; /* limit write buffers / synchronous */ 319 static int nsw_wcount_async; /* limit write buffers / asynchronous */ 320 static int nsw_wcount_async_max;/* assigned maximum */ 321 static int nsw_cluster_max; /* maximum VOP I/O allowed */ 322 323 static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); 324 SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW | 325 CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I", 326 "Maximum running async swap ops"); 327 static int sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS); 328 SYSCTL_PROC(_vm, OID_AUTO, swap_fragmentation, CTLTYPE_STRING | CTLFLAG_RD | 329 CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_fragmentation, "A", 330 "Swap Fragmentation Info"); 331 332 static struct sx sw_alloc_sx; 333 334 /* 335 * "named" and "unnamed" anon region objects. Try to reduce the overhead 336 * of searching a named list by hashing it just a little. 337 */ 338 339 #define NOBJLISTS 8 340 341 #define NOBJLIST(handle) \ 342 (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)]) 343 344 static struct pagerlst swap_pager_object_list[NOBJLISTS]; 345 static uma_zone_t swblk_zone; 346 static uma_zone_t swpctrie_zone; 347 348 /* 349 * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure 350 * calls hooked from other parts of the VM system and do not appear here. 351 * (see vm/swap_pager.h). 352 */ 353 static vm_object_t 354 swap_pager_alloc(void *handle, vm_ooffset_t size, 355 vm_prot_t prot, vm_ooffset_t offset, struct ucred *); 356 static void swap_pager_dealloc(vm_object_t object); 357 static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int *, 358 int *); 359 static int swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *, 360 int *, pgo_getpages_iodone_t, void *); 361 static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); 362 static boolean_t 363 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); 364 static void swap_pager_init(void); 365 static void swap_pager_unswapped(vm_page_t); 366 static void swap_pager_swapoff(struct swdevt *sp); 367 368 struct pagerops swappagerops = { 369 .pgo_init = swap_pager_init, /* early system initialization of pager */ 370 .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ 371 .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ 372 .pgo_getpages = swap_pager_getpages, /* pagein */ 373 .pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ 374 .pgo_putpages = swap_pager_putpages, /* pageout */ 375 .pgo_haspage = swap_pager_haspage, /* get backing store status for page */ 376 .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ 377 }; 378 379 /* 380 * swap_*() routines are externally accessible. swp_*() routines are 381 * internal. 382 */ 383 static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ 384 static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ 385 386 SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0, 387 "Maximum size of a swap block in pages"); 388 389 static void swp_sizecheck(void); 390 static void swp_pager_async_iodone(struct buf *bp); 391 static int swapongeom(struct vnode *); 392 static int swaponvp(struct thread *, struct vnode *, u_long); 393 static int swapoff_one(struct swdevt *sp, struct ucred *cred); 394 395 /* 396 * Swap bitmap functions 397 */ 398 static void swp_pager_freeswapspace(daddr_t blk, int npages); 399 static daddr_t swp_pager_getswapspace(int npages); 400 401 /* 402 * Metadata functions 403 */ 404 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); 405 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); 406 static void swp_pager_meta_free_all(vm_object_t); 407 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); 408 409 static void * 410 swblk_trie_alloc(struct pctrie *ptree) 411 { 412 413 return (uma_zalloc(swpctrie_zone, M_NOWAIT | (curproc == pageproc ? 414 M_USE_RESERVE : 0))); 415 } 416 417 static void 418 swblk_trie_free(struct pctrie *ptree, void *node) 419 { 420 421 uma_zfree(swpctrie_zone, node); 422 } 423 424 PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free); 425 426 /* 427 * SWP_SIZECHECK() - update swap_pager_full indication 428 * 429 * update the swap_pager_almost_full indication and warn when we are 430 * about to run out of swap space, using lowat/hiwat hysteresis. 431 * 432 * Clear swap_pager_full ( task killing ) indication when lowat is met. 433 * 434 * No restrictions on call 435 * This routine may not block. 436 */ 437 static void 438 swp_sizecheck(void) 439 { 440 441 if (swap_pager_avail < nswap_lowat) { 442 if (swap_pager_almost_full == 0) { 443 printf("swap_pager: out of swap space\n"); 444 swap_pager_almost_full = 1; 445 } 446 } else { 447 swap_pager_full = 0; 448 if (swap_pager_avail > nswap_hiwat) 449 swap_pager_almost_full = 0; 450 } 451 } 452 453 /* 454 * SWAP_PAGER_INIT() - initialize the swap pager! 455 * 456 * Expected to be started from system init. NOTE: This code is run 457 * before much else so be careful what you depend on. Most of the VM 458 * system has yet to be initialized at this point. 459 */ 460 static void 461 swap_pager_init(void) 462 { 463 /* 464 * Initialize object lists 465 */ 466 int i; 467 468 for (i = 0; i < NOBJLISTS; ++i) 469 TAILQ_INIT(&swap_pager_object_list[i]); 470 mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF); 471 sx_init(&sw_alloc_sx, "swspsx"); 472 sx_init(&swdev_syscall_lock, "swsysc"); 473 } 474 475 /* 476 * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process 477 * 478 * Expected to be started from pageout process once, prior to entering 479 * its main loop. 480 */ 481 void 482 swap_pager_swap_init(void) 483 { 484 unsigned long n, n2; 485 486 /* 487 * Number of in-transit swap bp operations. Don't 488 * exhaust the pbufs completely. Make sure we 489 * initialize workable values (0 will work for hysteresis 490 * but it isn't very efficient). 491 * 492 * The nsw_cluster_max is constrained by the bp->b_pages[] 493 * array (MAXPHYS/PAGE_SIZE) and our locally defined 494 * MAX_PAGEOUT_CLUSTER. Also be aware that swap ops are 495 * constrained by the swap device interleave stripe size. 496 * 497 * Currently we hardwire nsw_wcount_async to 4. This limit is 498 * designed to prevent other I/O from having high latencies due to 499 * our pageout I/O. The value 4 works well for one or two active swap 500 * devices but is probably a little low if you have more. Even so, 501 * a higher value would probably generate only a limited improvement 502 * with three or four active swap devices since the system does not 503 * typically have to pageout at extreme bandwidths. We will want 504 * at least 2 per swap devices, and 4 is a pretty good value if you 505 * have one NFS swap device due to the command/ack latency over NFS. 506 * So it all works out pretty well. 507 */ 508 nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER); 509 510 mtx_lock(&pbuf_mtx); 511 nsw_rcount = (nswbuf + 1) / 2; 512 nsw_wcount_sync = (nswbuf + 3) / 4; 513 nsw_wcount_async = 4; 514 nsw_wcount_async_max = nsw_wcount_async; 515 mtx_unlock(&pbuf_mtx); 516 517 /* 518 * Initialize our zone, guessing on the number we need based 519 * on the number of pages in the system. 520 */ 521 n = vm_cnt.v_page_count / 2; 522 if (maxswzone && n > maxswzone / sizeof(struct swblk)) 523 n = maxswzone / sizeof(struct swblk); 524 swpctrie_zone = uma_zcreate("swpctrie", pctrie_node_size(), NULL, NULL, 525 pctrie_zone_init, NULL, UMA_ALIGN_PTR, 526 UMA_ZONE_NOFREE | UMA_ZONE_VM); 527 if (swpctrie_zone == NULL) 528 panic("failed to create swap pctrie zone."); 529 swblk_zone = uma_zcreate("swblk", sizeof(struct swblk), NULL, NULL, 530 NULL, NULL, _Alignof(struct swblk) - 1, 531 UMA_ZONE_NOFREE | UMA_ZONE_VM); 532 if (swblk_zone == NULL) 533 panic("failed to create swap blk zone."); 534 n2 = n; 535 do { 536 if (uma_zone_reserve_kva(swblk_zone, n)) 537 break; 538 /* 539 * if the allocation failed, try a zone two thirds the 540 * size of the previous attempt. 541 */ 542 n -= ((n + 2) / 3); 543 } while (n > 0); 544 545 /* 546 * Often uma_zone_reserve_kva() cannot reserve exactly the 547 * requested size. Account for the difference when 548 * calculating swap_maxpages. 549 */ 550 n = uma_zone_get_max(swblk_zone); 551 552 if (n < n2) 553 printf("Swap blk zone entries reduced from %lu to %lu.\n", 554 n2, n); 555 swap_maxpages = n * SWAP_META_PAGES; 556 swzone = n * sizeof(struct swblk); 557 if (!uma_zone_reserve_kva(swpctrie_zone, n)) 558 printf("Cannot reserve swap pctrie zone, " 559 "reduce kern.maxswzone.\n"); 560 } 561 562 static vm_object_t 563 swap_pager_alloc_init(void *handle, struct ucred *cred, vm_ooffset_t size, 564 vm_ooffset_t offset) 565 { 566 vm_object_t object; 567 568 if (cred != NULL) { 569 if (!swap_reserve_by_cred(size, cred)) 570 return (NULL); 571 crhold(cred); 572 } 573 574 /* 575 * The un_pager.swp.swp_blks trie is initialized by 576 * vm_object_allocate() to ensure the correct order of 577 * visibility to other threads. 578 */ 579 object = vm_object_allocate(OBJT_SWAP, OFF_TO_IDX(offset + 580 PAGE_MASK + size)); 581 582 object->handle = handle; 583 if (cred != NULL) { 584 object->cred = cred; 585 object->charge = size; 586 } 587 return (object); 588 } 589 590 /* 591 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate 592 * its metadata structures. 593 * 594 * This routine is called from the mmap and fork code to create a new 595 * OBJT_SWAP object. 596 * 597 * This routine must ensure that no live duplicate is created for 598 * the named object request, which is protected against by 599 * holding the sw_alloc_sx lock in case handle != NULL. 600 */ 601 static vm_object_t 602 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 603 vm_ooffset_t offset, struct ucred *cred) 604 { 605 vm_object_t object; 606 607 if (handle != NULL) { 608 /* 609 * Reference existing named region or allocate new one. There 610 * should not be a race here against swp_pager_meta_build() 611 * as called from vm_page_remove() in regards to the lookup 612 * of the handle. 613 */ 614 sx_xlock(&sw_alloc_sx); 615 object = vm_pager_object_lookup(NOBJLIST(handle), handle); 616 if (object == NULL) { 617 object = swap_pager_alloc_init(handle, cred, size, 618 offset); 619 if (object != NULL) { 620 TAILQ_INSERT_TAIL(NOBJLIST(object->handle), 621 object, pager_object_list); 622 } 623 } 624 sx_xunlock(&sw_alloc_sx); 625 } else { 626 object = swap_pager_alloc_init(handle, cred, size, offset); 627 } 628 return (object); 629 } 630 631 /* 632 * SWAP_PAGER_DEALLOC() - remove swap metadata from object 633 * 634 * The swap backing for the object is destroyed. The code is 635 * designed such that we can reinstantiate it later, but this 636 * routine is typically called only when the entire object is 637 * about to be destroyed. 638 * 639 * The object must be locked. 640 */ 641 static void 642 swap_pager_dealloc(vm_object_t object) 643 { 644 645 VM_OBJECT_ASSERT_WLOCKED(object); 646 KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj")); 647 648 /* 649 * Remove from list right away so lookups will fail if we block for 650 * pageout completion. 651 */ 652 if (object->handle != NULL) { 653 VM_OBJECT_WUNLOCK(object); 654 sx_xlock(&sw_alloc_sx); 655 TAILQ_REMOVE(NOBJLIST(object->handle), object, 656 pager_object_list); 657 sx_xunlock(&sw_alloc_sx); 658 VM_OBJECT_WLOCK(object); 659 } 660 661 vm_object_pip_wait(object, "swpdea"); 662 663 /* 664 * Free all remaining metadata. We only bother to free it from 665 * the swap meta data. We do not attempt to free swapblk's still 666 * associated with vm_page_t's for this object. We do not care 667 * if paging is still in progress on some objects. 668 */ 669 swp_pager_meta_free_all(object); 670 object->handle = NULL; 671 object->type = OBJT_DEAD; 672 } 673 674 /************************************************************************ 675 * SWAP PAGER BITMAP ROUTINES * 676 ************************************************************************/ 677 678 /* 679 * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space 680 * 681 * Allocate swap for the requested number of pages. The starting 682 * swap block number (a page index) is returned or SWAPBLK_NONE 683 * if the allocation failed. 684 * 685 * Also has the side effect of advising that somebody made a mistake 686 * when they configured swap and didn't configure enough. 687 * 688 * This routine may not sleep. 689 * 690 * We allocate in round-robin fashion from the configured devices. 691 */ 692 static daddr_t 693 swp_pager_getswapspace(int npages) 694 { 695 daddr_t blk; 696 struct swdevt *sp; 697 int i; 698 699 blk = SWAPBLK_NONE; 700 mtx_lock(&sw_dev_mtx); 701 sp = swdevhd; 702 for (i = 0; i < nswapdev; i++) { 703 if (sp == NULL) 704 sp = TAILQ_FIRST(&swtailq); 705 if (!(sp->sw_flags & SW_CLOSING)) { 706 blk = blist_alloc(sp->sw_blist, npages); 707 if (blk != SWAPBLK_NONE) { 708 blk += sp->sw_first; 709 sp->sw_used += npages; 710 swap_pager_avail -= npages; 711 swp_sizecheck(); 712 swdevhd = TAILQ_NEXT(sp, sw_list); 713 goto done; 714 } 715 } 716 sp = TAILQ_NEXT(sp, sw_list); 717 } 718 if (swap_pager_full != 2) { 719 printf("swap_pager_getswapspace(%d): failed\n", npages); 720 swap_pager_full = 2; 721 swap_pager_almost_full = 1; 722 } 723 swdevhd = NULL; 724 done: 725 mtx_unlock(&sw_dev_mtx); 726 return (blk); 727 } 728 729 static int 730 swp_pager_isondev(daddr_t blk, struct swdevt *sp) 731 { 732 733 return (blk >= sp->sw_first && blk < sp->sw_end); 734 } 735 736 static void 737 swp_pager_strategy(struct buf *bp) 738 { 739 struct swdevt *sp; 740 741 mtx_lock(&sw_dev_mtx); 742 TAILQ_FOREACH(sp, &swtailq, sw_list) { 743 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) { 744 mtx_unlock(&sw_dev_mtx); 745 if ((sp->sw_flags & SW_UNMAPPED) != 0 && 746 unmapped_buf_allowed) { 747 bp->b_data = unmapped_buf; 748 bp->b_offset = 0; 749 } else { 750 pmap_qenter((vm_offset_t)bp->b_data, 751 &bp->b_pages[0], bp->b_bcount / PAGE_SIZE); 752 } 753 sp->sw_strategy(bp, sp); 754 return; 755 } 756 } 757 panic("Swapdev not found"); 758 } 759 760 761 /* 762 * SWP_PAGER_FREESWAPSPACE() - free raw swap space 763 * 764 * This routine returns the specified swap blocks back to the bitmap. 765 * 766 * This routine may not sleep. 767 */ 768 static void 769 swp_pager_freeswapspace(daddr_t blk, int npages) 770 { 771 struct swdevt *sp; 772 773 mtx_lock(&sw_dev_mtx); 774 TAILQ_FOREACH(sp, &swtailq, sw_list) { 775 if (blk >= sp->sw_first && blk < sp->sw_end) { 776 sp->sw_used -= npages; 777 /* 778 * If we are attempting to stop swapping on 779 * this device, we don't want to mark any 780 * blocks free lest they be reused. 781 */ 782 if ((sp->sw_flags & SW_CLOSING) == 0) { 783 blist_free(sp->sw_blist, blk - sp->sw_first, 784 npages); 785 swap_pager_avail += npages; 786 swp_sizecheck(); 787 } 788 mtx_unlock(&sw_dev_mtx); 789 return; 790 } 791 } 792 panic("Swapdev not found"); 793 } 794 795 /* 796 * SYSCTL_SWAP_FRAGMENTATION() - produce raw swap space stats 797 */ 798 static int 799 sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS) 800 { 801 struct sbuf sbuf; 802 struct swdevt *sp; 803 const char *devname; 804 int error; 805 806 error = sysctl_wire_old_buffer(req, 0); 807 if (error != 0) 808 return (error); 809 sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 810 mtx_lock(&sw_dev_mtx); 811 TAILQ_FOREACH(sp, &swtailq, sw_list) { 812 if (vn_isdisk(sp->sw_vp, NULL)) 813 devname = devtoname(sp->sw_vp->v_rdev); 814 else 815 devname = "[file]"; 816 sbuf_printf(&sbuf, "\nFree space on device %s:\n", devname); 817 blist_stats(sp->sw_blist, &sbuf); 818 } 819 mtx_unlock(&sw_dev_mtx); 820 error = sbuf_finish(&sbuf); 821 sbuf_delete(&sbuf); 822 return (error); 823 } 824 825 /* 826 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page 827 * range within an object. 828 * 829 * This is a globally accessible routine. 830 * 831 * This routine removes swapblk assignments from swap metadata. 832 * 833 * The external callers of this routine typically have already destroyed 834 * or renamed vm_page_t's associated with this range in the object so 835 * we should be ok. 836 * 837 * The object must be locked. 838 */ 839 void 840 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size) 841 { 842 843 swp_pager_meta_free(object, start, size); 844 } 845 846 /* 847 * SWAP_PAGER_RESERVE() - reserve swap blocks in object 848 * 849 * Assigns swap blocks to the specified range within the object. The 850 * swap blocks are not zeroed. Any previous swap assignment is destroyed. 851 * 852 * Returns 0 on success, -1 on failure. 853 */ 854 int 855 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) 856 { 857 int n = 0; 858 daddr_t blk = SWAPBLK_NONE; 859 vm_pindex_t beg = start; /* save start index */ 860 861 VM_OBJECT_WLOCK(object); 862 while (size) { 863 if (n == 0) { 864 n = BLIST_MAX_ALLOC; 865 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) { 866 n >>= 1; 867 if (n == 0) { 868 swp_pager_meta_free(object, beg, start - beg); 869 VM_OBJECT_WUNLOCK(object); 870 return (-1); 871 } 872 } 873 } 874 swp_pager_meta_build(object, start, blk); 875 --size; 876 ++start; 877 ++blk; 878 --n; 879 } 880 swp_pager_meta_free(object, start, n); 881 VM_OBJECT_WUNLOCK(object); 882 return (0); 883 } 884 885 /* 886 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager 887 * and destroy the source. 888 * 889 * Copy any valid swapblks from the source to the destination. In 890 * cases where both the source and destination have a valid swapblk, 891 * we keep the destination's. 892 * 893 * This routine is allowed to sleep. It may sleep allocating metadata 894 * indirectly through swp_pager_meta_build() or if paging is still in 895 * progress on the source. 896 * 897 * The source object contains no vm_page_t's (which is just as well) 898 * 899 * The source object is of type OBJT_SWAP. 900 * 901 * The source and destination objects must be locked. 902 * Both object locks may temporarily be released. 903 */ 904 void 905 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, 906 vm_pindex_t offset, int destroysource) 907 { 908 vm_pindex_t i; 909 910 VM_OBJECT_ASSERT_WLOCKED(srcobject); 911 VM_OBJECT_ASSERT_WLOCKED(dstobject); 912 913 /* 914 * If destroysource is set, we remove the source object from the 915 * swap_pager internal queue now. 916 */ 917 if (destroysource && srcobject->handle != NULL) { 918 vm_object_pip_add(srcobject, 1); 919 VM_OBJECT_WUNLOCK(srcobject); 920 vm_object_pip_add(dstobject, 1); 921 VM_OBJECT_WUNLOCK(dstobject); 922 sx_xlock(&sw_alloc_sx); 923 TAILQ_REMOVE(NOBJLIST(srcobject->handle), srcobject, 924 pager_object_list); 925 sx_xunlock(&sw_alloc_sx); 926 VM_OBJECT_WLOCK(dstobject); 927 vm_object_pip_wakeup(dstobject); 928 VM_OBJECT_WLOCK(srcobject); 929 vm_object_pip_wakeup(srcobject); 930 } 931 932 /* 933 * transfer source to destination. 934 */ 935 for (i = 0; i < dstobject->size; ++i) { 936 daddr_t dstaddr; 937 938 /* 939 * Locate (without changing) the swapblk on the destination, 940 * unless it is invalid in which case free it silently, or 941 * if the destination is a resident page, in which case the 942 * source is thrown away. 943 */ 944 dstaddr = swp_pager_meta_ctl(dstobject, i, 0); 945 946 if (dstaddr == SWAPBLK_NONE) { 947 /* 948 * Destination has no swapblk and is not resident, 949 * copy source. 950 */ 951 daddr_t srcaddr; 952 953 srcaddr = swp_pager_meta_ctl( 954 srcobject, 955 i + offset, 956 SWM_POP 957 ); 958 959 if (srcaddr != SWAPBLK_NONE) { 960 /* 961 * swp_pager_meta_build() can sleep. 962 */ 963 vm_object_pip_add(srcobject, 1); 964 VM_OBJECT_WUNLOCK(srcobject); 965 vm_object_pip_add(dstobject, 1); 966 swp_pager_meta_build(dstobject, i, srcaddr); 967 vm_object_pip_wakeup(dstobject); 968 VM_OBJECT_WLOCK(srcobject); 969 vm_object_pip_wakeup(srcobject); 970 } 971 } else { 972 /* 973 * Destination has valid swapblk or it is represented 974 * by a resident page. We destroy the sourceblock. 975 */ 976 977 swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE); 978 } 979 } 980 981 /* 982 * Free left over swap blocks in source. 983 * 984 * We have to revert the type to OBJT_DEFAULT so we do not accidentally 985 * double-remove the object from the swap queues. 986 */ 987 if (destroysource) { 988 swp_pager_meta_free_all(srcobject); 989 /* 990 * Reverting the type is not necessary, the caller is going 991 * to destroy srcobject directly, but I'm doing it here 992 * for consistency since we've removed the object from its 993 * queues. 994 */ 995 srcobject->type = OBJT_DEFAULT; 996 } 997 } 998 999 /* 1000 * SWAP_PAGER_HASPAGE() - determine if we have good backing store for 1001 * the requested page. 1002 * 1003 * We determine whether good backing store exists for the requested 1004 * page and return TRUE if it does, FALSE if it doesn't. 1005 * 1006 * If TRUE, we also try to determine how much valid, contiguous backing 1007 * store exists before and after the requested page. 1008 */ 1009 static boolean_t 1010 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 1011 int *after) 1012 { 1013 daddr_t blk, blk0; 1014 int i; 1015 1016 VM_OBJECT_ASSERT_LOCKED(object); 1017 1018 /* 1019 * do we have good backing store at the requested index ? 1020 */ 1021 blk0 = swp_pager_meta_ctl(object, pindex, 0); 1022 if (blk0 == SWAPBLK_NONE) { 1023 if (before) 1024 *before = 0; 1025 if (after) 1026 *after = 0; 1027 return (FALSE); 1028 } 1029 1030 /* 1031 * find backwards-looking contiguous good backing store 1032 */ 1033 if (before != NULL) { 1034 for (i = 1; i < SWB_NPAGES; i++) { 1035 if (i > pindex) 1036 break; 1037 blk = swp_pager_meta_ctl(object, pindex - i, 0); 1038 if (blk != blk0 - i) 1039 break; 1040 } 1041 *before = i - 1; 1042 } 1043 1044 /* 1045 * find forward-looking contiguous good backing store 1046 */ 1047 if (after != NULL) { 1048 for (i = 1; i < SWB_NPAGES; i++) { 1049 blk = swp_pager_meta_ctl(object, pindex + i, 0); 1050 if (blk != blk0 + i) 1051 break; 1052 } 1053 *after = i - 1; 1054 } 1055 return (TRUE); 1056 } 1057 1058 /* 1059 * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page 1060 * 1061 * This removes any associated swap backing store, whether valid or 1062 * not, from the page. 1063 * 1064 * This routine is typically called when a page is made dirty, at 1065 * which point any associated swap can be freed. MADV_FREE also 1066 * calls us in a special-case situation 1067 * 1068 * NOTE!!! If the page is clean and the swap was valid, the caller 1069 * should make the page dirty before calling this routine. This routine 1070 * does NOT change the m->dirty status of the page. Also: MADV_FREE 1071 * depends on it. 1072 * 1073 * This routine may not sleep. 1074 * 1075 * The object containing the page must be locked. 1076 */ 1077 static void 1078 swap_pager_unswapped(vm_page_t m) 1079 { 1080 1081 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE); 1082 } 1083 1084 /* 1085 * swap_pager_getpages() - bring pages in from swap 1086 * 1087 * Attempt to page in the pages in array "m" of length "count". The caller 1088 * may optionally specify that additional pages preceding and succeeding 1089 * the specified range be paged in. The number of such pages is returned 1090 * in the "rbehind" and "rahead" parameters, and they will be in the 1091 * inactive queue upon return. 1092 * 1093 * The pages in "m" must be busied and will remain busied upon return. 1094 */ 1095 static int 1096 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, 1097 int *rahead) 1098 { 1099 struct buf *bp; 1100 vm_page_t mpred, msucc, p; 1101 vm_pindex_t pindex; 1102 daddr_t blk; 1103 int i, j, maxahead, maxbehind, reqcount, shift; 1104 1105 reqcount = count; 1106 1107 VM_OBJECT_WUNLOCK(object); 1108 bp = getpbuf(&nsw_rcount); 1109 VM_OBJECT_WLOCK(object); 1110 1111 if (!swap_pager_haspage(object, m[0]->pindex, &maxbehind, &maxahead)) { 1112 relpbuf(bp, &nsw_rcount); 1113 return (VM_PAGER_FAIL); 1114 } 1115 1116 /* 1117 * Clip the readahead and readbehind ranges to exclude resident pages. 1118 */ 1119 if (rahead != NULL) { 1120 KASSERT(reqcount - 1 <= maxahead, 1121 ("page count %d extends beyond swap block", reqcount)); 1122 *rahead = imin(*rahead, maxahead - (reqcount - 1)); 1123 pindex = m[reqcount - 1]->pindex; 1124 msucc = TAILQ_NEXT(m[reqcount - 1], listq); 1125 if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead) 1126 *rahead = msucc->pindex - pindex - 1; 1127 } 1128 if (rbehind != NULL) { 1129 *rbehind = imin(*rbehind, maxbehind); 1130 pindex = m[0]->pindex; 1131 mpred = TAILQ_PREV(m[0], pglist, listq); 1132 if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind) 1133 *rbehind = pindex - mpred->pindex - 1; 1134 } 1135 1136 /* 1137 * Allocate readahead and readbehind pages. 1138 */ 1139 shift = rbehind != NULL ? *rbehind : 0; 1140 if (shift != 0) { 1141 for (i = 1; i <= shift; i++) { 1142 p = vm_page_alloc(object, m[0]->pindex - i, 1143 VM_ALLOC_NORMAL); 1144 if (p == NULL) { 1145 /* Shift allocated pages to the left. */ 1146 for (j = 0; j < i - 1; j++) 1147 bp->b_pages[j] = 1148 bp->b_pages[j + shift - i + 1]; 1149 break; 1150 } 1151 bp->b_pages[shift - i] = p; 1152 } 1153 shift = i - 1; 1154 *rbehind = shift; 1155 } 1156 for (i = 0; i < reqcount; i++) 1157 bp->b_pages[i + shift] = m[i]; 1158 if (rahead != NULL) { 1159 for (i = 0; i < *rahead; i++) { 1160 p = vm_page_alloc(object, 1161 m[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL); 1162 if (p == NULL) 1163 break; 1164 bp->b_pages[shift + reqcount + i] = p; 1165 } 1166 *rahead = i; 1167 } 1168 if (rbehind != NULL) 1169 count += *rbehind; 1170 if (rahead != NULL) 1171 count += *rahead; 1172 1173 vm_object_pip_add(object, count); 1174 1175 for (i = 0; i < count; i++) 1176 bp->b_pages[i]->oflags |= VPO_SWAPINPROG; 1177 1178 pindex = bp->b_pages[0]->pindex; 1179 blk = swp_pager_meta_ctl(object, pindex, 0); 1180 KASSERT(blk != SWAPBLK_NONE, 1181 ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); 1182 1183 VM_OBJECT_WUNLOCK(object); 1184 1185 bp->b_flags |= B_PAGING; 1186 bp->b_iocmd = BIO_READ; 1187 bp->b_iodone = swp_pager_async_iodone; 1188 bp->b_rcred = crhold(thread0.td_ucred); 1189 bp->b_wcred = crhold(thread0.td_ucred); 1190 bp->b_blkno = blk; 1191 bp->b_bcount = PAGE_SIZE * count; 1192 bp->b_bufsize = PAGE_SIZE * count; 1193 bp->b_npages = count; 1194 bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; 1195 bp->b_pgafter = rahead != NULL ? *rahead : 0; 1196 1197 VM_CNT_INC(v_swapin); 1198 VM_CNT_ADD(v_swappgsin, count); 1199 1200 /* 1201 * perform the I/O. NOTE!!! bp cannot be considered valid after 1202 * this point because we automatically release it on completion. 1203 * Instead, we look at the one page we are interested in which we 1204 * still hold a lock on even through the I/O completion. 1205 * 1206 * The other pages in our m[] array are also released on completion, 1207 * so we cannot assume they are valid anymore either. 1208 * 1209 * NOTE: b_blkno is destroyed by the call to swapdev_strategy 1210 */ 1211 BUF_KERNPROC(bp); 1212 swp_pager_strategy(bp); 1213 1214 /* 1215 * Wait for the pages we want to complete. VPO_SWAPINPROG is always 1216 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE 1217 * is set in the metadata for each page in the request. 1218 */ 1219 VM_OBJECT_WLOCK(object); 1220 while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { 1221 m[0]->oflags |= VPO_SWAPSLEEP; 1222 VM_CNT_INC(v_intrans); 1223 if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, 1224 "swread", hz * 20)) { 1225 printf( 1226 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", 1227 bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); 1228 } 1229 } 1230 1231 /* 1232 * If we had an unrecoverable read error pages will not be valid. 1233 */ 1234 for (i = 0; i < reqcount; i++) 1235 if (m[i]->valid != VM_PAGE_BITS_ALL) 1236 return (VM_PAGER_ERROR); 1237 1238 return (VM_PAGER_OK); 1239 1240 /* 1241 * A final note: in a low swap situation, we cannot deallocate swap 1242 * and mark a page dirty here because the caller is likely to mark 1243 * the page clean when we return, causing the page to possibly revert 1244 * to all-zero's later. 1245 */ 1246 } 1247 1248 /* 1249 * swap_pager_getpages_async(): 1250 * 1251 * Right now this is emulation of asynchronous operation on top of 1252 * swap_pager_getpages(). 1253 */ 1254 static int 1255 swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, 1256 int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg) 1257 { 1258 int r, error; 1259 1260 r = swap_pager_getpages(object, m, count, rbehind, rahead); 1261 VM_OBJECT_WUNLOCK(object); 1262 switch (r) { 1263 case VM_PAGER_OK: 1264 error = 0; 1265 break; 1266 case VM_PAGER_ERROR: 1267 error = EIO; 1268 break; 1269 case VM_PAGER_FAIL: 1270 error = EINVAL; 1271 break; 1272 default: 1273 panic("unhandled swap_pager_getpages() error %d", r); 1274 } 1275 (iodone)(arg, m, count, error); 1276 VM_OBJECT_WLOCK(object); 1277 1278 return (r); 1279 } 1280 1281 /* 1282 * swap_pager_putpages: 1283 * 1284 * Assign swap (if necessary) and initiate I/O on the specified pages. 1285 * 1286 * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects 1287 * are automatically converted to SWAP objects. 1288 * 1289 * In a low memory situation we may block in VOP_STRATEGY(), but the new 1290 * vm_page reservation system coupled with properly written VFS devices 1291 * should ensure that no low-memory deadlock occurs. This is an area 1292 * which needs work. 1293 * 1294 * The parent has N vm_object_pip_add() references prior to 1295 * calling us and will remove references for rtvals[] that are 1296 * not set to VM_PAGER_PEND. We need to remove the rest on I/O 1297 * completion. 1298 * 1299 * The parent has soft-busy'd the pages it passes us and will unbusy 1300 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return. 1301 * We need to unbusy the rest on I/O completion. 1302 */ 1303 static void 1304 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count, 1305 int flags, int *rtvals) 1306 { 1307 int i, n; 1308 boolean_t sync; 1309 1310 if (count && m[0]->object != object) { 1311 panic("swap_pager_putpages: object mismatch %p/%p", 1312 object, 1313 m[0]->object 1314 ); 1315 } 1316 1317 /* 1318 * Step 1 1319 * 1320 * Turn object into OBJT_SWAP 1321 * check for bogus sysops 1322 * force sync if not pageout process 1323 */ 1324 if (object->type != OBJT_SWAP) 1325 swp_pager_meta_build(object, 0, SWAPBLK_NONE); 1326 VM_OBJECT_WUNLOCK(object); 1327 1328 n = 0; 1329 if (curproc != pageproc) 1330 sync = TRUE; 1331 else 1332 sync = (flags & VM_PAGER_PUT_SYNC) != 0; 1333 1334 /* 1335 * Step 2 1336 * 1337 * Assign swap blocks and issue I/O. We reallocate swap on the fly. 1338 * The page is left dirty until the pageout operation completes 1339 * successfully. 1340 */ 1341 for (i = 0; i < count; i += n) { 1342 int j; 1343 struct buf *bp; 1344 daddr_t blk; 1345 1346 /* 1347 * Maximum I/O size is limited by a number of factors. 1348 */ 1349 n = min(BLIST_MAX_ALLOC, count - i); 1350 n = min(n, nsw_cluster_max); 1351 1352 /* 1353 * Get biggest block of swap we can. If we fail, fall 1354 * back and try to allocate a smaller block. Don't go 1355 * overboard trying to allocate space if it would overly 1356 * fragment swap. 1357 */ 1358 while ( 1359 (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE && 1360 n > 4 1361 ) { 1362 n >>= 1; 1363 } 1364 if (blk == SWAPBLK_NONE) { 1365 for (j = 0; j < n; ++j) 1366 rtvals[i+j] = VM_PAGER_FAIL; 1367 continue; 1368 } 1369 1370 /* 1371 * All I/O parameters have been satisfied, build the I/O 1372 * request and assign the swap space. 1373 */ 1374 if (sync == TRUE) { 1375 bp = getpbuf(&nsw_wcount_sync); 1376 } else { 1377 bp = getpbuf(&nsw_wcount_async); 1378 bp->b_flags = B_ASYNC; 1379 } 1380 bp->b_flags |= B_PAGING; 1381 bp->b_iocmd = BIO_WRITE; 1382 1383 bp->b_rcred = crhold(thread0.td_ucred); 1384 bp->b_wcred = crhold(thread0.td_ucred); 1385 bp->b_bcount = PAGE_SIZE * n; 1386 bp->b_bufsize = PAGE_SIZE * n; 1387 bp->b_blkno = blk; 1388 1389 VM_OBJECT_WLOCK(object); 1390 for (j = 0; j < n; ++j) { 1391 vm_page_t mreq = m[i+j]; 1392 1393 swp_pager_meta_build( 1394 mreq->object, 1395 mreq->pindex, 1396 blk + j 1397 ); 1398 MPASS(mreq->dirty == VM_PAGE_BITS_ALL); 1399 mreq->oflags |= VPO_SWAPINPROG; 1400 bp->b_pages[j] = mreq; 1401 } 1402 VM_OBJECT_WUNLOCK(object); 1403 bp->b_npages = n; 1404 /* 1405 * Must set dirty range for NFS to work. 1406 */ 1407 bp->b_dirtyoff = 0; 1408 bp->b_dirtyend = bp->b_bcount; 1409 1410 VM_CNT_INC(v_swapout); 1411 VM_CNT_ADD(v_swappgsout, bp->b_npages); 1412 1413 /* 1414 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we 1415 * can call the async completion routine at the end of a 1416 * synchronous I/O operation. Otherwise, our caller would 1417 * perform duplicate unbusy and wakeup operations on the page 1418 * and object, respectively. 1419 */ 1420 for (j = 0; j < n; j++) 1421 rtvals[i + j] = VM_PAGER_PEND; 1422 1423 /* 1424 * asynchronous 1425 * 1426 * NOTE: b_blkno is destroyed by the call to swapdev_strategy 1427 */ 1428 if (sync == FALSE) { 1429 bp->b_iodone = swp_pager_async_iodone; 1430 BUF_KERNPROC(bp); 1431 swp_pager_strategy(bp); 1432 continue; 1433 } 1434 1435 /* 1436 * synchronous 1437 * 1438 * NOTE: b_blkno is destroyed by the call to swapdev_strategy 1439 */ 1440 bp->b_iodone = bdone; 1441 swp_pager_strategy(bp); 1442 1443 /* 1444 * Wait for the sync I/O to complete. 1445 */ 1446 bwait(bp, PVM, "swwrt"); 1447 1448 /* 1449 * Now that we are through with the bp, we can call the 1450 * normal async completion, which frees everything up. 1451 */ 1452 swp_pager_async_iodone(bp); 1453 } 1454 VM_OBJECT_WLOCK(object); 1455 } 1456 1457 /* 1458 * swp_pager_async_iodone: 1459 * 1460 * Completion routine for asynchronous reads and writes from/to swap. 1461 * Also called manually by synchronous code to finish up a bp. 1462 * 1463 * This routine may not sleep. 1464 */ 1465 static void 1466 swp_pager_async_iodone(struct buf *bp) 1467 { 1468 int i; 1469 vm_object_t object = NULL; 1470 1471 /* 1472 * report error 1473 */ 1474 if (bp->b_ioflags & BIO_ERROR) { 1475 printf( 1476 "swap_pager: I/O error - %s failed; blkno %ld," 1477 "size %ld, error %d\n", 1478 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"), 1479 (long)bp->b_blkno, 1480 (long)bp->b_bcount, 1481 bp->b_error 1482 ); 1483 } 1484 1485 /* 1486 * remove the mapping for kernel virtual 1487 */ 1488 if (buf_mapped(bp)) 1489 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages); 1490 else 1491 bp->b_data = bp->b_kvabase; 1492 1493 if (bp->b_npages) { 1494 object = bp->b_pages[0]->object; 1495 VM_OBJECT_WLOCK(object); 1496 } 1497 1498 /* 1499 * cleanup pages. If an error occurs writing to swap, we are in 1500 * very serious trouble. If it happens to be a disk error, though, 1501 * we may be able to recover by reassigning the swap later on. So 1502 * in this case we remove the m->swapblk assignment for the page 1503 * but do not free it in the rlist. The errornous block(s) are thus 1504 * never reallocated as swap. Redirty the page and continue. 1505 */ 1506 for (i = 0; i < bp->b_npages; ++i) { 1507 vm_page_t m = bp->b_pages[i]; 1508 1509 m->oflags &= ~VPO_SWAPINPROG; 1510 if (m->oflags & VPO_SWAPSLEEP) { 1511 m->oflags &= ~VPO_SWAPSLEEP; 1512 wakeup(&object->paging_in_progress); 1513 } 1514 1515 if (bp->b_ioflags & BIO_ERROR) { 1516 /* 1517 * If an error occurs I'd love to throw the swapblk 1518 * away without freeing it back to swapspace, so it 1519 * can never be used again. But I can't from an 1520 * interrupt. 1521 */ 1522 if (bp->b_iocmd == BIO_READ) { 1523 /* 1524 * NOTE: for reads, m->dirty will probably 1525 * be overridden by the original caller of 1526 * getpages so don't play cute tricks here. 1527 */ 1528 m->valid = 0; 1529 } else { 1530 /* 1531 * If a write error occurs, reactivate page 1532 * so it doesn't clog the inactive list, 1533 * then finish the I/O. 1534 */ 1535 MPASS(m->dirty == VM_PAGE_BITS_ALL); 1536 vm_page_lock(m); 1537 vm_page_activate(m); 1538 vm_page_unlock(m); 1539 vm_page_sunbusy(m); 1540 } 1541 } else if (bp->b_iocmd == BIO_READ) { 1542 /* 1543 * NOTE: for reads, m->dirty will probably be 1544 * overridden by the original caller of getpages so 1545 * we cannot set them in order to free the underlying 1546 * swap in a low-swap situation. I don't think we'd 1547 * want to do that anyway, but it was an optimization 1548 * that existed in the old swapper for a time before 1549 * it got ripped out due to precisely this problem. 1550 */ 1551 KASSERT(!pmap_page_is_mapped(m), 1552 ("swp_pager_async_iodone: page %p is mapped", m)); 1553 KASSERT(m->dirty == 0, 1554 ("swp_pager_async_iodone: page %p is dirty", m)); 1555 1556 m->valid = VM_PAGE_BITS_ALL; 1557 if (i < bp->b_pgbefore || 1558 i >= bp->b_npages - bp->b_pgafter) 1559 vm_page_readahead_finish(m); 1560 } else { 1561 /* 1562 * For write success, clear the dirty 1563 * status, then finish the I/O ( which decrements the 1564 * busy count and possibly wakes waiter's up ). 1565 * A page is only written to swap after a period of 1566 * inactivity. Therefore, we do not expect it to be 1567 * reused. 1568 */ 1569 KASSERT(!pmap_page_is_write_mapped(m), 1570 ("swp_pager_async_iodone: page %p is not write" 1571 " protected", m)); 1572 vm_page_undirty(m); 1573 vm_page_lock(m); 1574 vm_page_deactivate_noreuse(m); 1575 vm_page_unlock(m); 1576 vm_page_sunbusy(m); 1577 } 1578 } 1579 1580 /* 1581 * adjust pip. NOTE: the original parent may still have its own 1582 * pip refs on the object. 1583 */ 1584 if (object != NULL) { 1585 vm_object_pip_wakeupn(object, bp->b_npages); 1586 VM_OBJECT_WUNLOCK(object); 1587 } 1588 1589 /* 1590 * swapdev_strategy() manually sets b_vp and b_bufobj before calling 1591 * bstrategy(). Set them back to NULL now we're done with it, or we'll 1592 * trigger a KASSERT in relpbuf(). 1593 */ 1594 if (bp->b_vp) { 1595 bp->b_vp = NULL; 1596 bp->b_bufobj = NULL; 1597 } 1598 /* 1599 * release the physical I/O buffer 1600 */ 1601 relpbuf( 1602 bp, 1603 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : 1604 ((bp->b_flags & B_ASYNC) ? 1605 &nsw_wcount_async : 1606 &nsw_wcount_sync 1607 ) 1608 ) 1609 ); 1610 } 1611 1612 int 1613 swap_pager_nswapdev(void) 1614 { 1615 1616 return (nswapdev); 1617 } 1618 1619 /* 1620 * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in 1621 * 1622 * This routine dissociates the page at the given index within an object 1623 * from its backing store, paging it in if it does not reside in memory. 1624 * If the page is paged in, it is marked dirty and placed in the laundry 1625 * queue. The page is marked dirty because it no longer has backing 1626 * store. It is placed in the laundry queue because it has not been 1627 * accessed recently. Otherwise, it would already reside in memory. 1628 * 1629 * We also attempt to swap in all other pages in the swap block. 1630 * However, we only guarantee that the one at the specified index is 1631 * paged in. 1632 * 1633 * XXX - The code to page the whole block in doesn't work, so we 1634 * revert to the one-by-one behavior for now. Sigh. 1635 */ 1636 static inline void 1637 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex) 1638 { 1639 vm_page_t m; 1640 1641 vm_object_pip_add(object, 1); 1642 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 1643 if (m->valid == VM_PAGE_BITS_ALL) { 1644 vm_object_pip_wakeup(object); 1645 vm_page_dirty(m); 1646 #ifdef INVARIANTS 1647 vm_page_lock(m); 1648 if (m->wire_count == 0 && m->queue == PQ_NONE) 1649 panic("page %p is neither wired nor queued", m); 1650 vm_page_unlock(m); 1651 #endif 1652 vm_page_xunbusy(m); 1653 vm_pager_page_unswapped(m); 1654 return; 1655 } 1656 1657 if (swap_pager_getpages(object, &m, 1, NULL, NULL) != VM_PAGER_OK) 1658 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/ 1659 vm_object_pip_wakeup(object); 1660 vm_page_dirty(m); 1661 vm_page_lock(m); 1662 vm_page_launder(m); 1663 vm_page_unlock(m); 1664 vm_page_xunbusy(m); 1665 vm_pager_page_unswapped(m); 1666 } 1667 1668 /* 1669 * swap_pager_swapoff: 1670 * 1671 * Page in all of the pages that have been paged out to the 1672 * given device. The corresponding blocks in the bitmap must be 1673 * marked as allocated and the device must be flagged SW_CLOSING. 1674 * There may be no processes swapped out to the device. 1675 * 1676 * This routine may block. 1677 */ 1678 static void 1679 swap_pager_swapoff(struct swdevt *sp) 1680 { 1681 struct swblk *sb; 1682 vm_object_t object; 1683 vm_pindex_t pi; 1684 int i, retries; 1685 1686 sx_assert(&swdev_syscall_lock, SA_XLOCKED); 1687 1688 retries = 0; 1689 full_rescan: 1690 mtx_lock(&vm_object_list_mtx); 1691 TAILQ_FOREACH(object, &vm_object_list, object_list) { 1692 if (object->type != OBJT_SWAP) 1693 continue; 1694 mtx_unlock(&vm_object_list_mtx); 1695 /* Depends on type-stability. */ 1696 VM_OBJECT_WLOCK(object); 1697 1698 /* 1699 * Dead objects are eventually terminated on their own. 1700 */ 1701 if ((object->flags & OBJ_DEAD) != 0) 1702 goto next_obj; 1703 1704 /* 1705 * Sync with fences placed after pctrie 1706 * initialization. We must not access pctrie below 1707 * unless we checked that our object is swap and not 1708 * dead. 1709 */ 1710 atomic_thread_fence_acq(); 1711 if (object->type != OBJT_SWAP) 1712 goto next_obj; 1713 1714 for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( 1715 &object->un_pager.swp.swp_blks, pi)) != NULL; ) { 1716 pi = sb->p + SWAP_META_PAGES; 1717 for (i = 0; i < SWAP_META_PAGES; i++) { 1718 if (sb->d[i] == SWAPBLK_NONE) 1719 continue; 1720 if (swp_pager_isondev(sb->d[i], sp)) 1721 swp_pager_force_pagein(object, 1722 sb->p + i); 1723 } 1724 } 1725 next_obj: 1726 VM_OBJECT_WUNLOCK(object); 1727 mtx_lock(&vm_object_list_mtx); 1728 } 1729 mtx_unlock(&vm_object_list_mtx); 1730 1731 if (sp->sw_used) { 1732 /* 1733 * Objects may be locked or paging to the device being 1734 * removed, so we will miss their pages and need to 1735 * make another pass. We have marked this device as 1736 * SW_CLOSING, so the activity should finish soon. 1737 */ 1738 retries++; 1739 if (retries > 100) { 1740 panic("swapoff: failed to locate %d swap blocks", 1741 sp->sw_used); 1742 } 1743 pause("swpoff", hz / 20); 1744 goto full_rescan; 1745 } 1746 EVENTHANDLER_INVOKE(swapoff, sp); 1747 } 1748 1749 /************************************************************************ 1750 * SWAP META DATA * 1751 ************************************************************************ 1752 * 1753 * These routines manipulate the swap metadata stored in the 1754 * OBJT_SWAP object. 1755 * 1756 * Swap metadata is implemented with a global hash and not directly 1757 * linked into the object. Instead the object simply contains 1758 * appropriate tracking counters. 1759 */ 1760 1761 /* 1762 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object 1763 * 1764 * We first convert the object to a swap object if it is a default 1765 * object. 1766 * 1767 * The specified swapblk is added to the object's swap metadata. If 1768 * the swapblk is not valid, it is freed instead. Any previously 1769 * assigned swapblk is freed. 1770 */ 1771 static void 1772 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) 1773 { 1774 static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; 1775 struct swblk *sb, *sb1; 1776 vm_pindex_t modpi, rdpi; 1777 int error, i; 1778 1779 VM_OBJECT_ASSERT_WLOCKED(object); 1780 1781 /* 1782 * Convert default object to swap object if necessary 1783 */ 1784 if (object->type != OBJT_SWAP) { 1785 pctrie_init(&object->un_pager.swp.swp_blks); 1786 1787 /* 1788 * Ensure that swap_pager_swapoff()'s iteration over 1789 * object_list does not see a garbage pctrie. 1790 */ 1791 atomic_thread_fence_rel(); 1792 1793 object->type = OBJT_SWAP; 1794 KASSERT(object->handle == NULL, ("default pager with handle")); 1795 } 1796 1797 rdpi = rounddown(pindex, SWAP_META_PAGES); 1798 sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); 1799 if (sb == NULL) { 1800 if (swapblk == SWAPBLK_NONE) 1801 return; 1802 for (;;) { 1803 sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == 1804 pageproc ? M_USE_RESERVE : 0)); 1805 if (sb != NULL) { 1806 sb->p = rdpi; 1807 for (i = 0; i < SWAP_META_PAGES; i++) 1808 sb->d[i] = SWAPBLK_NONE; 1809 if (atomic_cmpset_int(&swblk_zone_exhausted, 1810 1, 0)) 1811 printf("swblk zone ok\n"); 1812 break; 1813 } 1814 VM_OBJECT_WUNLOCK(object); 1815 if (uma_zone_exhausted(swblk_zone)) { 1816 if (atomic_cmpset_int(&swblk_zone_exhausted, 1817 0, 1)) 1818 printf("swap blk zone exhausted, " 1819 "increase kern.maxswzone\n"); 1820 vm_pageout_oom(VM_OOM_SWAPZ); 1821 pause("swzonxb", 10); 1822 } else 1823 uma_zwait(swblk_zone); 1824 VM_OBJECT_WLOCK(object); 1825 sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, 1826 rdpi); 1827 if (sb != NULL) 1828 /* 1829 * Somebody swapped out a nearby page, 1830 * allocating swblk at the rdpi index, 1831 * while we dropped the object lock. 1832 */ 1833 goto allocated; 1834 } 1835 for (;;) { 1836 error = SWAP_PCTRIE_INSERT( 1837 &object->un_pager.swp.swp_blks, sb); 1838 if (error == 0) { 1839 if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1840 1, 0)) 1841 printf("swpctrie zone ok\n"); 1842 break; 1843 } 1844 VM_OBJECT_WUNLOCK(object); 1845 if (uma_zone_exhausted(swpctrie_zone)) { 1846 if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1847 0, 1)) 1848 printf("swap pctrie zone exhausted, " 1849 "increase kern.maxswzone\n"); 1850 vm_pageout_oom(VM_OOM_SWAPZ); 1851 pause("swzonxp", 10); 1852 } else 1853 uma_zwait(swpctrie_zone); 1854 VM_OBJECT_WLOCK(object); 1855 sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, 1856 rdpi); 1857 if (sb1 != NULL) { 1858 uma_zfree(swblk_zone, sb); 1859 sb = sb1; 1860 goto allocated; 1861 } 1862 } 1863 } 1864 allocated: 1865 MPASS(sb->p == rdpi); 1866 1867 modpi = pindex % SWAP_META_PAGES; 1868 /* Delete prior contents of metadata. */ 1869 if (sb->d[modpi] != SWAPBLK_NONE) 1870 swp_pager_freeswapspace(sb->d[modpi], 1); 1871 /* Enter block into metadata. */ 1872 sb->d[modpi] = swapblk; 1873 1874 /* 1875 * Free the swblk if we end up with the empty page run. 1876 */ 1877 if (swapblk == SWAPBLK_NONE) { 1878 for (i = 0; i < SWAP_META_PAGES; i++) { 1879 if (sb->d[i] != SWAPBLK_NONE) 1880 break; 1881 } 1882 if (i == SWAP_META_PAGES) { 1883 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, 1884 rdpi); 1885 uma_zfree(swblk_zone, sb); 1886 } 1887 } 1888 } 1889 1890 /* 1891 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata 1892 * 1893 * The requested range of blocks is freed, with any associated swap 1894 * returned to the swap bitmap. 1895 * 1896 * This routine will free swap metadata structures as they are cleaned 1897 * out. This routine does *NOT* operate on swap metadata associated 1898 * with resident pages. 1899 */ 1900 static void 1901 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) 1902 { 1903 struct swblk *sb; 1904 vm_pindex_t last; 1905 int i; 1906 bool empty; 1907 1908 VM_OBJECT_ASSERT_WLOCKED(object); 1909 if (object->type != OBJT_SWAP || count == 0) 1910 return; 1911 1912 last = pindex + count - 1; 1913 for (;;) { 1914 sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, 1915 rounddown(pindex, SWAP_META_PAGES)); 1916 if (sb == NULL || sb->p > last) 1917 break; 1918 empty = true; 1919 for (i = 0; i < SWAP_META_PAGES; i++) { 1920 if (sb->d[i] == SWAPBLK_NONE) 1921 continue; 1922 if (pindex <= sb->p + i && sb->p + i <= last) { 1923 swp_pager_freeswapspace(sb->d[i], 1); 1924 sb->d[i] = SWAPBLK_NONE; 1925 } else 1926 empty = false; 1927 } 1928 pindex = sb->p + SWAP_META_PAGES; 1929 if (empty) { 1930 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, 1931 sb->p); 1932 uma_zfree(swblk_zone, sb); 1933 } 1934 } 1935 } 1936 1937 /* 1938 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object 1939 * 1940 * This routine locates and destroys all swap metadata associated with 1941 * an object. 1942 */ 1943 static void 1944 swp_pager_meta_free_all(vm_object_t object) 1945 { 1946 struct swblk *sb; 1947 vm_pindex_t pindex; 1948 int i; 1949 1950 VM_OBJECT_ASSERT_WLOCKED(object); 1951 if (object->type != OBJT_SWAP) 1952 return; 1953 1954 for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( 1955 &object->un_pager.swp.swp_blks, pindex)) != NULL;) { 1956 pindex = sb->p + SWAP_META_PAGES; 1957 for (i = 0; i < SWAP_META_PAGES; i++) { 1958 if (sb->d[i] != SWAPBLK_NONE) 1959 swp_pager_freeswapspace(sb->d[i], 1); 1960 } 1961 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); 1962 uma_zfree(swblk_zone, sb); 1963 } 1964 } 1965 1966 /* 1967 * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data. 1968 * 1969 * This routine is capable of looking up, popping, or freeing 1970 * swapblk assignments in the swap meta data or in the vm_page_t. 1971 * The routine typically returns the swapblk being looked-up, or popped, 1972 * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block 1973 * was invalid. This routine will automatically free any invalid 1974 * meta-data swapblks. 1975 * 1976 * When acting on a busy resident page and paging is in progress, we 1977 * have to wait until paging is complete but otherwise can act on the 1978 * busy page. 1979 * 1980 * SWM_FREE remove and free swap block from metadata 1981 * SWM_POP remove from meta data but do not free.. pop it out 1982 */ 1983 static daddr_t 1984 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags) 1985 { 1986 struct swblk *sb; 1987 daddr_t r1; 1988 int i; 1989 1990 if ((flags & (SWM_FREE | SWM_POP)) != 0) 1991 VM_OBJECT_ASSERT_WLOCKED(object); 1992 else 1993 VM_OBJECT_ASSERT_LOCKED(object); 1994 1995 /* 1996 * The meta data only exists if the object is OBJT_SWAP 1997 * and even then might not be allocated yet. 1998 */ 1999 if (object->type != OBJT_SWAP) 2000 return (SWAPBLK_NONE); 2001 2002 sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, 2003 rounddown(pindex, SWAP_META_PAGES)); 2004 if (sb == NULL) 2005 return (SWAPBLK_NONE); 2006 r1 = sb->d[pindex % SWAP_META_PAGES]; 2007 if (r1 == SWAPBLK_NONE) 2008 return (SWAPBLK_NONE); 2009 if ((flags & (SWM_FREE | SWM_POP)) != 0) { 2010 sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; 2011 for (i = 0; i < SWAP_META_PAGES; i++) { 2012 if (sb->d[i] != SWAPBLK_NONE) 2013 break; 2014 } 2015 if (i == SWAP_META_PAGES) { 2016 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, 2017 rounddown(pindex, SWAP_META_PAGES)); 2018 uma_zfree(swblk_zone, sb); 2019 } 2020 } 2021 if ((flags & SWM_FREE) != 0) { 2022 swp_pager_freeswapspace(r1, 1); 2023 r1 = SWAPBLK_NONE; 2024 } 2025 return (r1); 2026 } 2027 2028 /* 2029 * Returns the least page index which is greater than or equal to the 2030 * parameter pindex and for which there is a swap block allocated. 2031 * Returns object's size if the object's type is not swap or if there 2032 * are no allocated swap blocks for the object after the requested 2033 * pindex. 2034 */ 2035 vm_pindex_t 2036 swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) 2037 { 2038 struct swblk *sb; 2039 int i; 2040 2041 VM_OBJECT_ASSERT_LOCKED(object); 2042 if (object->type != OBJT_SWAP) 2043 return (object->size); 2044 2045 sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, 2046 rounddown(pindex, SWAP_META_PAGES)); 2047 if (sb == NULL) 2048 return (object->size); 2049 if (sb->p < pindex) { 2050 for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { 2051 if (sb->d[i] != SWAPBLK_NONE) 2052 return (sb->p + i); 2053 } 2054 sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, 2055 roundup(pindex, SWAP_META_PAGES)); 2056 if (sb == NULL) 2057 return (object->size); 2058 } 2059 for (i = 0; i < SWAP_META_PAGES; i++) { 2060 if (sb->d[i] != SWAPBLK_NONE) 2061 return (sb->p + i); 2062 } 2063 2064 /* 2065 * We get here if a swblk is present in the trie but it 2066 * doesn't map any blocks. 2067 */ 2068 MPASS(0); 2069 return (object->size); 2070 } 2071 2072 /* 2073 * System call swapon(name) enables swapping on device name, 2074 * which must be in the swdevsw. Return EBUSY 2075 * if already swapping on this device. 2076 */ 2077 #ifndef _SYS_SYSPROTO_H_ 2078 struct swapon_args { 2079 char *name; 2080 }; 2081 #endif 2082 2083 /* 2084 * MPSAFE 2085 */ 2086 /* ARGSUSED */ 2087 int 2088 sys_swapon(struct thread *td, struct swapon_args *uap) 2089 { 2090 struct vattr attr; 2091 struct vnode *vp; 2092 struct nameidata nd; 2093 int error; 2094 2095 error = priv_check(td, PRIV_SWAPON); 2096 if (error) 2097 return (error); 2098 2099 sx_xlock(&swdev_syscall_lock); 2100 2101 /* 2102 * Swap metadata may not fit in the KVM if we have physical 2103 * memory of >1GB. 2104 */ 2105 if (swblk_zone == NULL) { 2106 error = ENOMEM; 2107 goto done; 2108 } 2109 2110 NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE, 2111 uap->name, td); 2112 error = namei(&nd); 2113 if (error) 2114 goto done; 2115 2116 NDFREE(&nd, NDF_ONLY_PNBUF); 2117 vp = nd.ni_vp; 2118 2119 if (vn_isdisk(vp, &error)) { 2120 error = swapongeom(vp); 2121 } else if (vp->v_type == VREG && 2122 (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 && 2123 (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) { 2124 /* 2125 * Allow direct swapping to NFS regular files in the same 2126 * way that nfs_mountroot() sets up diskless swapping. 2127 */ 2128 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE); 2129 } 2130 2131 if (error) 2132 vrele(vp); 2133 done: 2134 sx_xunlock(&swdev_syscall_lock); 2135 return (error); 2136 } 2137 2138 /* 2139 * Check that the total amount of swap currently configured does not 2140 * exceed half the theoretical maximum. If it does, print a warning 2141 * message. 2142 */ 2143 static void 2144 swapon_check_swzone(void) 2145 { 2146 unsigned long maxpages, npages; 2147 2148 npages = swap_total / PAGE_SIZE; 2149 /* absolute maximum we can handle assuming 100% efficiency */ 2150 maxpages = uma_zone_get_max(swblk_zone) * SWAP_META_PAGES; 2151 2152 /* recommend using no more than half that amount */ 2153 if (npages > maxpages / 2) { 2154 printf("warning: total configured swap (%lu pages) " 2155 "exceeds maximum recommended amount (%lu pages).\n", 2156 npages, maxpages / 2); 2157 printf("warning: increase kern.maxswzone " 2158 "or reduce amount of swap.\n"); 2159 } 2160 } 2161 2162 static void 2163 swaponsomething(struct vnode *vp, void *id, u_long nblks, 2164 sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags) 2165 { 2166 struct swdevt *sp, *tsp; 2167 swblk_t dvbase; 2168 u_long mblocks; 2169 2170 /* 2171 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. 2172 * First chop nblks off to page-align it, then convert. 2173 * 2174 * sw->sw_nblks is in page-sized chunks now too. 2175 */ 2176 nblks &= ~(ctodb(1) - 1); 2177 nblks = dbtoc(nblks); 2178 2179 /* 2180 * If we go beyond this, we get overflows in the radix 2181 * tree bitmap code. 2182 */ 2183 mblocks = 0x40000000 / BLIST_META_RADIX; 2184 if (nblks > mblocks) { 2185 printf( 2186 "WARNING: reducing swap size to maximum of %luMB per unit\n", 2187 mblocks / 1024 / 1024 * PAGE_SIZE); 2188 nblks = mblocks; 2189 } 2190 2191 sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO); 2192 sp->sw_vp = vp; 2193 sp->sw_id = id; 2194 sp->sw_dev = dev; 2195 sp->sw_flags = 0; 2196 sp->sw_nblks = nblks; 2197 sp->sw_used = 0; 2198 sp->sw_strategy = strategy; 2199 sp->sw_close = close; 2200 sp->sw_flags = flags; 2201 2202 sp->sw_blist = blist_create(nblks, M_WAITOK); 2203 /* 2204 * Do not free the first two block in order to avoid overwriting 2205 * any bsd label at the front of the partition 2206 */ 2207 blist_free(sp->sw_blist, 2, nblks - 2); 2208 2209 dvbase = 0; 2210 mtx_lock(&sw_dev_mtx); 2211 TAILQ_FOREACH(tsp, &swtailq, sw_list) { 2212 if (tsp->sw_end >= dvbase) { 2213 /* 2214 * We put one uncovered page between the devices 2215 * in order to definitively prevent any cross-device 2216 * I/O requests 2217 */ 2218 dvbase = tsp->sw_end + 1; 2219 } 2220 } 2221 sp->sw_first = dvbase; 2222 sp->sw_end = dvbase + nblks; 2223 TAILQ_INSERT_TAIL(&swtailq, sp, sw_list); 2224 nswapdev++; 2225 swap_pager_avail += nblks - 2; 2226 swap_total += (vm_ooffset_t)nblks * PAGE_SIZE; 2227 swapon_check_swzone(); 2228 swp_sizecheck(); 2229 mtx_unlock(&sw_dev_mtx); 2230 EVENTHANDLER_INVOKE(swapon, sp); 2231 } 2232 2233 /* 2234 * SYSCALL: swapoff(devname) 2235 * 2236 * Disable swapping on the given device. 2237 * 2238 * XXX: Badly designed system call: it should use a device index 2239 * rather than filename as specification. We keep sw_vp around 2240 * only to make this work. 2241 */ 2242 #ifndef _SYS_SYSPROTO_H_ 2243 struct swapoff_args { 2244 char *name; 2245 }; 2246 #endif 2247 2248 /* 2249 * MPSAFE 2250 */ 2251 /* ARGSUSED */ 2252 int 2253 sys_swapoff(struct thread *td, struct swapoff_args *uap) 2254 { 2255 struct vnode *vp; 2256 struct nameidata nd; 2257 struct swdevt *sp; 2258 int error; 2259 2260 error = priv_check(td, PRIV_SWAPOFF); 2261 if (error) 2262 return (error); 2263 2264 sx_xlock(&swdev_syscall_lock); 2265 2266 NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name, 2267 td); 2268 error = namei(&nd); 2269 if (error) 2270 goto done; 2271 NDFREE(&nd, NDF_ONLY_PNBUF); 2272 vp = nd.ni_vp; 2273 2274 mtx_lock(&sw_dev_mtx); 2275 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2276 if (sp->sw_vp == vp) 2277 break; 2278 } 2279 mtx_unlock(&sw_dev_mtx); 2280 if (sp == NULL) { 2281 error = EINVAL; 2282 goto done; 2283 } 2284 error = swapoff_one(sp, td->td_ucred); 2285 done: 2286 sx_xunlock(&swdev_syscall_lock); 2287 return (error); 2288 } 2289 2290 static int 2291 swapoff_one(struct swdevt *sp, struct ucred *cred) 2292 { 2293 u_long nblks; 2294 #ifdef MAC 2295 int error; 2296 #endif 2297 2298 sx_assert(&swdev_syscall_lock, SA_XLOCKED); 2299 #ifdef MAC 2300 (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY); 2301 error = mac_system_check_swapoff(cred, sp->sw_vp); 2302 (void) VOP_UNLOCK(sp->sw_vp, 0); 2303 if (error != 0) 2304 return (error); 2305 #endif 2306 nblks = sp->sw_nblks; 2307 2308 /* 2309 * We can turn off this swap device safely only if the 2310 * available virtual memory in the system will fit the amount 2311 * of data we will have to page back in, plus an epsilon so 2312 * the system doesn't become critically low on swap space. 2313 */ 2314 if (vm_cnt.v_free_count + swap_pager_avail < nblks + nswap_lowat) 2315 return (ENOMEM); 2316 2317 /* 2318 * Prevent further allocations on this device. 2319 */ 2320 mtx_lock(&sw_dev_mtx); 2321 sp->sw_flags |= SW_CLOSING; 2322 swap_pager_avail -= blist_fill(sp->sw_blist, 0, nblks); 2323 swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE; 2324 mtx_unlock(&sw_dev_mtx); 2325 2326 /* 2327 * Page in the contents of the device and close it. 2328 */ 2329 swap_pager_swapoff(sp); 2330 2331 sp->sw_close(curthread, sp); 2332 mtx_lock(&sw_dev_mtx); 2333 sp->sw_id = NULL; 2334 TAILQ_REMOVE(&swtailq, sp, sw_list); 2335 nswapdev--; 2336 if (nswapdev == 0) { 2337 swap_pager_full = 2; 2338 swap_pager_almost_full = 1; 2339 } 2340 if (swdevhd == sp) 2341 swdevhd = NULL; 2342 mtx_unlock(&sw_dev_mtx); 2343 blist_destroy(sp->sw_blist); 2344 free(sp, M_VMPGDATA); 2345 return (0); 2346 } 2347 2348 void 2349 swapoff_all(void) 2350 { 2351 struct swdevt *sp, *spt; 2352 const char *devname; 2353 int error; 2354 2355 sx_xlock(&swdev_syscall_lock); 2356 2357 mtx_lock(&sw_dev_mtx); 2358 TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) { 2359 mtx_unlock(&sw_dev_mtx); 2360 if (vn_isdisk(sp->sw_vp, NULL)) 2361 devname = devtoname(sp->sw_vp->v_rdev); 2362 else 2363 devname = "[file]"; 2364 error = swapoff_one(sp, thread0.td_ucred); 2365 if (error != 0) { 2366 printf("Cannot remove swap device %s (error=%d), " 2367 "skipping.\n", devname, error); 2368 } else if (bootverbose) { 2369 printf("Swap device %s removed.\n", devname); 2370 } 2371 mtx_lock(&sw_dev_mtx); 2372 } 2373 mtx_unlock(&sw_dev_mtx); 2374 2375 sx_xunlock(&swdev_syscall_lock); 2376 } 2377 2378 void 2379 swap_pager_status(int *total, int *used) 2380 { 2381 struct swdevt *sp; 2382 2383 *total = 0; 2384 *used = 0; 2385 mtx_lock(&sw_dev_mtx); 2386 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2387 *total += sp->sw_nblks; 2388 *used += sp->sw_used; 2389 } 2390 mtx_unlock(&sw_dev_mtx); 2391 } 2392 2393 int 2394 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len) 2395 { 2396 struct swdevt *sp; 2397 const char *tmp_devname; 2398 int error, n; 2399 2400 n = 0; 2401 error = ENOENT; 2402 mtx_lock(&sw_dev_mtx); 2403 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2404 if (n != name) { 2405 n++; 2406 continue; 2407 } 2408 xs->xsw_version = XSWDEV_VERSION; 2409 xs->xsw_dev = sp->sw_dev; 2410 xs->xsw_flags = sp->sw_flags; 2411 xs->xsw_nblks = sp->sw_nblks; 2412 xs->xsw_used = sp->sw_used; 2413 if (devname != NULL) { 2414 if (vn_isdisk(sp->sw_vp, NULL)) 2415 tmp_devname = devtoname(sp->sw_vp->v_rdev); 2416 else 2417 tmp_devname = "[file]"; 2418 strncpy(devname, tmp_devname, len); 2419 } 2420 error = 0; 2421 break; 2422 } 2423 mtx_unlock(&sw_dev_mtx); 2424 return (error); 2425 } 2426 2427 #if defined(COMPAT_FREEBSD11) 2428 #define XSWDEV_VERSION_11 1 2429 struct xswdev11 { 2430 u_int xsw_version; 2431 uint32_t xsw_dev; 2432 int xsw_flags; 2433 int xsw_nblks; 2434 int xsw_used; 2435 }; 2436 #endif 2437 2438 static int 2439 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS) 2440 { 2441 struct xswdev xs; 2442 #if defined(COMPAT_FREEBSD11) 2443 struct xswdev11 xs11; 2444 #endif 2445 int error; 2446 2447 if (arg2 != 1) /* name length */ 2448 return (EINVAL); 2449 error = swap_dev_info(*(int *)arg1, &xs, NULL, 0); 2450 if (error != 0) 2451 return (error); 2452 #if defined(COMPAT_FREEBSD11) 2453 if (req->oldlen == sizeof(xs11)) { 2454 xs11.xsw_version = XSWDEV_VERSION_11; 2455 xs11.xsw_dev = xs.xsw_dev; /* truncation */ 2456 xs11.xsw_flags = xs.xsw_flags; 2457 xs11.xsw_nblks = xs.xsw_nblks; 2458 xs11.xsw_used = xs.xsw_used; 2459 error = SYSCTL_OUT(req, &xs11, sizeof(xs11)); 2460 } else 2461 #endif 2462 error = SYSCTL_OUT(req, &xs, sizeof(xs)); 2463 return (error); 2464 } 2465 2466 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0, 2467 "Number of swap devices"); 2468 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD | CTLFLAG_MPSAFE, 2469 sysctl_vm_swap_info, 2470 "Swap statistics by device"); 2471 2472 /* 2473 * Count the approximate swap usage in pages for a vmspace. The 2474 * shadowed or not yet copied on write swap blocks are not accounted. 2475 * The map must be locked. 2476 */ 2477 long 2478 vmspace_swap_count(struct vmspace *vmspace) 2479 { 2480 vm_map_t map; 2481 vm_map_entry_t cur; 2482 vm_object_t object; 2483 struct swblk *sb; 2484 vm_pindex_t e, pi; 2485 long count; 2486 int i; 2487 2488 map = &vmspace->vm_map; 2489 count = 0; 2490 2491 for (cur = map->header.next; cur != &map->header; cur = cur->next) { 2492 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) 2493 continue; 2494 object = cur->object.vm_object; 2495 if (object == NULL || object->type != OBJT_SWAP) 2496 continue; 2497 VM_OBJECT_RLOCK(object); 2498 if (object->type != OBJT_SWAP) 2499 goto unlock; 2500 pi = OFF_TO_IDX(cur->offset); 2501 e = pi + OFF_TO_IDX(cur->end - cur->start); 2502 for (;; pi = sb->p + SWAP_META_PAGES) { 2503 sb = SWAP_PCTRIE_LOOKUP_GE( 2504 &object->un_pager.swp.swp_blks, pi); 2505 if (sb == NULL || sb->p >= e) 2506 break; 2507 for (i = 0; i < SWAP_META_PAGES; i++) { 2508 if (sb->p + i < e && 2509 sb->d[i] != SWAPBLK_NONE) 2510 count++; 2511 } 2512 } 2513 unlock: 2514 VM_OBJECT_RUNLOCK(object); 2515 } 2516 return (count); 2517 } 2518 2519 /* 2520 * GEOM backend 2521 * 2522 * Swapping onto disk devices. 2523 * 2524 */ 2525 2526 static g_orphan_t swapgeom_orphan; 2527 2528 static struct g_class g_swap_class = { 2529 .name = "SWAP", 2530 .version = G_VERSION, 2531 .orphan = swapgeom_orphan, 2532 }; 2533 2534 DECLARE_GEOM_CLASS(g_swap_class, g_class); 2535 2536 2537 static void 2538 swapgeom_close_ev(void *arg, int flags) 2539 { 2540 struct g_consumer *cp; 2541 2542 cp = arg; 2543 g_access(cp, -1, -1, 0); 2544 g_detach(cp); 2545 g_destroy_consumer(cp); 2546 } 2547 2548 /* 2549 * Add a reference to the g_consumer for an inflight transaction. 2550 */ 2551 static void 2552 swapgeom_acquire(struct g_consumer *cp) 2553 { 2554 2555 mtx_assert(&sw_dev_mtx, MA_OWNED); 2556 cp->index++; 2557 } 2558 2559 /* 2560 * Remove a reference from the g_consumer. Post a close event if all 2561 * references go away, since the function might be called from the 2562 * biodone context. 2563 */ 2564 static void 2565 swapgeom_release(struct g_consumer *cp, struct swdevt *sp) 2566 { 2567 2568 mtx_assert(&sw_dev_mtx, MA_OWNED); 2569 cp->index--; 2570 if (cp->index == 0) { 2571 if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0) 2572 sp->sw_id = NULL; 2573 } 2574 } 2575 2576 static void 2577 swapgeom_done(struct bio *bp2) 2578 { 2579 struct swdevt *sp; 2580 struct buf *bp; 2581 struct g_consumer *cp; 2582 2583 bp = bp2->bio_caller2; 2584 cp = bp2->bio_from; 2585 bp->b_ioflags = bp2->bio_flags; 2586 if (bp2->bio_error) 2587 bp->b_ioflags |= BIO_ERROR; 2588 bp->b_resid = bp->b_bcount - bp2->bio_completed; 2589 bp->b_error = bp2->bio_error; 2590 bufdone(bp); 2591 sp = bp2->bio_caller1; 2592 mtx_lock(&sw_dev_mtx); 2593 swapgeom_release(cp, sp); 2594 mtx_unlock(&sw_dev_mtx); 2595 g_destroy_bio(bp2); 2596 } 2597 2598 static void 2599 swapgeom_strategy(struct buf *bp, struct swdevt *sp) 2600 { 2601 struct bio *bio; 2602 struct g_consumer *cp; 2603 2604 mtx_lock(&sw_dev_mtx); 2605 cp = sp->sw_id; 2606 if (cp == NULL) { 2607 mtx_unlock(&sw_dev_mtx); 2608 bp->b_error = ENXIO; 2609 bp->b_ioflags |= BIO_ERROR; 2610 bufdone(bp); 2611 return; 2612 } 2613 swapgeom_acquire(cp); 2614 mtx_unlock(&sw_dev_mtx); 2615 if (bp->b_iocmd == BIO_WRITE) 2616 bio = g_new_bio(); 2617 else 2618 bio = g_alloc_bio(); 2619 if (bio == NULL) { 2620 mtx_lock(&sw_dev_mtx); 2621 swapgeom_release(cp, sp); 2622 mtx_unlock(&sw_dev_mtx); 2623 bp->b_error = ENOMEM; 2624 bp->b_ioflags |= BIO_ERROR; 2625 bufdone(bp); 2626 return; 2627 } 2628 2629 bio->bio_caller1 = sp; 2630 bio->bio_caller2 = bp; 2631 bio->bio_cmd = bp->b_iocmd; 2632 bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE; 2633 bio->bio_length = bp->b_bcount; 2634 bio->bio_done = swapgeom_done; 2635 if (!buf_mapped(bp)) { 2636 bio->bio_ma = bp->b_pages; 2637 bio->bio_data = unmapped_buf; 2638 bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK; 2639 bio->bio_ma_n = bp->b_npages; 2640 bio->bio_flags |= BIO_UNMAPPED; 2641 } else { 2642 bio->bio_data = bp->b_data; 2643 bio->bio_ma = NULL; 2644 } 2645 g_io_request(bio, cp); 2646 return; 2647 } 2648 2649 static void 2650 swapgeom_orphan(struct g_consumer *cp) 2651 { 2652 struct swdevt *sp; 2653 int destroy; 2654 2655 mtx_lock(&sw_dev_mtx); 2656 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2657 if (sp->sw_id == cp) { 2658 sp->sw_flags |= SW_CLOSING; 2659 break; 2660 } 2661 } 2662 /* 2663 * Drop reference we were created with. Do directly since we're in a 2664 * special context where we don't have to queue the call to 2665 * swapgeom_close_ev(). 2666 */ 2667 cp->index--; 2668 destroy = ((sp != NULL) && (cp->index == 0)); 2669 if (destroy) 2670 sp->sw_id = NULL; 2671 mtx_unlock(&sw_dev_mtx); 2672 if (destroy) 2673 swapgeom_close_ev(cp, 0); 2674 } 2675 2676 static void 2677 swapgeom_close(struct thread *td, struct swdevt *sw) 2678 { 2679 struct g_consumer *cp; 2680 2681 mtx_lock(&sw_dev_mtx); 2682 cp = sw->sw_id; 2683 sw->sw_id = NULL; 2684 mtx_unlock(&sw_dev_mtx); 2685 2686 /* 2687 * swapgeom_close() may be called from the biodone context, 2688 * where we cannot perform topology changes. Delegate the 2689 * work to the events thread. 2690 */ 2691 if (cp != NULL) 2692 g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL); 2693 } 2694 2695 static int 2696 swapongeom_locked(struct cdev *dev, struct vnode *vp) 2697 { 2698 struct g_provider *pp; 2699 struct g_consumer *cp; 2700 static struct g_geom *gp; 2701 struct swdevt *sp; 2702 u_long nblks; 2703 int error; 2704 2705 pp = g_dev_getprovider(dev); 2706 if (pp == NULL) 2707 return (ENODEV); 2708 mtx_lock(&sw_dev_mtx); 2709 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2710 cp = sp->sw_id; 2711 if (cp != NULL && cp->provider == pp) { 2712 mtx_unlock(&sw_dev_mtx); 2713 return (EBUSY); 2714 } 2715 } 2716 mtx_unlock(&sw_dev_mtx); 2717 if (gp == NULL) 2718 gp = g_new_geomf(&g_swap_class, "swap"); 2719 cp = g_new_consumer(gp); 2720 cp->index = 1; /* Number of active I/Os, plus one for being active. */ 2721 cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; 2722 g_attach(cp, pp); 2723 /* 2724 * XXX: Every time you think you can improve the margin for 2725 * footshooting, somebody depends on the ability to do so: 2726 * savecore(8) wants to write to our swapdev so we cannot 2727 * set an exclusive count :-( 2728 */ 2729 error = g_access(cp, 1, 1, 0); 2730 if (error != 0) { 2731 g_detach(cp); 2732 g_destroy_consumer(cp); 2733 return (error); 2734 } 2735 nblks = pp->mediasize / DEV_BSIZE; 2736 swaponsomething(vp, cp, nblks, swapgeom_strategy, 2737 swapgeom_close, dev2udev(dev), 2738 (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0); 2739 return (0); 2740 } 2741 2742 static int 2743 swapongeom(struct vnode *vp) 2744 { 2745 int error; 2746 2747 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2748 if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) { 2749 error = ENOENT; 2750 } else { 2751 g_topology_lock(); 2752 error = swapongeom_locked(vp->v_rdev, vp); 2753 g_topology_unlock(); 2754 } 2755 VOP_UNLOCK(vp, 0); 2756 return (error); 2757 } 2758 2759 /* 2760 * VNODE backend 2761 * 2762 * This is used mainly for network filesystem (read: probably only tested 2763 * with NFS) swapfiles. 2764 * 2765 */ 2766 2767 static void 2768 swapdev_strategy(struct buf *bp, struct swdevt *sp) 2769 { 2770 struct vnode *vp2; 2771 2772 bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first); 2773 2774 vp2 = sp->sw_id; 2775 vhold(vp2); 2776 if (bp->b_iocmd == BIO_WRITE) { 2777 if (bp->b_bufobj) 2778 bufobj_wdrop(bp->b_bufobj); 2779 bufobj_wref(&vp2->v_bufobj); 2780 } 2781 if (bp->b_bufobj != &vp2->v_bufobj) 2782 bp->b_bufobj = &vp2->v_bufobj; 2783 bp->b_vp = vp2; 2784 bp->b_iooffset = dbtob(bp->b_blkno); 2785 bstrategy(bp); 2786 return; 2787 } 2788 2789 static void 2790 swapdev_close(struct thread *td, struct swdevt *sp) 2791 { 2792 2793 VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td); 2794 vrele(sp->sw_vp); 2795 } 2796 2797 2798 static int 2799 swaponvp(struct thread *td, struct vnode *vp, u_long nblks) 2800 { 2801 struct swdevt *sp; 2802 int error; 2803 2804 if (nblks == 0) 2805 return (ENXIO); 2806 mtx_lock(&sw_dev_mtx); 2807 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2808 if (sp->sw_id == vp) { 2809 mtx_unlock(&sw_dev_mtx); 2810 return (EBUSY); 2811 } 2812 } 2813 mtx_unlock(&sw_dev_mtx); 2814 2815 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2816 #ifdef MAC 2817 error = mac_system_check_swapon(td->td_ucred, vp); 2818 if (error == 0) 2819 #endif 2820 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL); 2821 (void) VOP_UNLOCK(vp, 0); 2822 if (error) 2823 return (error); 2824 2825 swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close, 2826 NODEV, 0); 2827 return (0); 2828 } 2829 2830 static int 2831 sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) 2832 { 2833 int error, new, n; 2834 2835 new = nsw_wcount_async_max; 2836 error = sysctl_handle_int(oidp, &new, 0, req); 2837 if (error != 0 || req->newptr == NULL) 2838 return (error); 2839 2840 if (new > nswbuf / 2 || new < 1) 2841 return (EINVAL); 2842 2843 mtx_lock(&pbuf_mtx); 2844 while (nsw_wcount_async_max != new) { 2845 /* 2846 * Adjust difference. If the current async count is too low, 2847 * we will need to sqeeze our update slowly in. Sleep with a 2848 * higher priority than getpbuf() to finish faster. 2849 */ 2850 n = new - nsw_wcount_async_max; 2851 if (nsw_wcount_async + n >= 0) { 2852 nsw_wcount_async += n; 2853 nsw_wcount_async_max += n; 2854 wakeup(&nsw_wcount_async); 2855 } else { 2856 nsw_wcount_async_max -= nsw_wcount_async; 2857 nsw_wcount_async = 0; 2858 msleep(&nsw_wcount_async, &pbuf_mtx, PSWP, 2859 "swpsysctl", 0); 2860 } 2861 } 2862 mtx_unlock(&pbuf_mtx); 2863 2864 return (0); 2865 } 2866