1 /*- 2 * Copyright (c) 1998 Matthew Dillon, 3 * Copyright (c) 1994 John S. Dyson 4 * Copyright (c) 1990 University of Utah. 5 * Copyright (c) 1982, 1986, 1989, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * New Swap System 41 * Matthew Dillon 42 * 43 * Radix Bitmap 'blists'. 44 * 45 * - The new swapper uses the new radix bitmap code. This should scale 46 * to arbitrarily small or arbitrarily large swap spaces and an almost 47 * arbitrary degree of fragmentation. 48 * 49 * Features: 50 * 51 * - on the fly reallocation of swap during putpages. The new system 52 * does not try to keep previously allocated swap blocks for dirty 53 * pages. 54 * 55 * - on the fly deallocation of swap 56 * 57 * - No more garbage collection required. Unnecessarily allocated swap 58 * blocks only exist for dirty vm_page_t's now and these are already 59 * cycled (in a high-load system) by the pager. We also do on-the-fly 60 * removal of invalidated swap blocks when a page is destroyed 61 * or renamed. 62 * 63 * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 64 * 65 * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 66 * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94 67 */ 68 69 #include <sys/cdefs.h> 70 __FBSDID("$FreeBSD$"); 71 72 #include "opt_compat.h" 73 #include "opt_swap.h" 74 #include "opt_vm.h" 75 76 #include <sys/param.h> 77 #include <sys/systm.h> 78 #include <sys/conf.h> 79 #include <sys/kernel.h> 80 #include <sys/priv.h> 81 #include <sys/proc.h> 82 #include <sys/bio.h> 83 #include <sys/buf.h> 84 #include <sys/disk.h> 85 #include <sys/fcntl.h> 86 #include <sys/mount.h> 87 #include <sys/namei.h> 88 #include <sys/vnode.h> 89 #include <sys/malloc.h> 90 #include <sys/pctrie.h> 91 #include <sys/racct.h> 92 #include <sys/resource.h> 93 #include <sys/resourcevar.h> 94 #include <sys/rwlock.h> 95 #include <sys/sbuf.h> 96 #include <sys/sysctl.h> 97 #include <sys/sysproto.h> 98 #include <sys/blist.h> 99 #include <sys/lock.h> 100 #include <sys/sx.h> 101 #include <sys/vmmeter.h> 102 103 #include <security/mac/mac_framework.h> 104 105 #include <vm/vm.h> 106 #include <vm/pmap.h> 107 #include <vm/vm_map.h> 108 #include <vm/vm_kern.h> 109 #include <vm/vm_object.h> 110 #include <vm/vm_page.h> 111 #include <vm/vm_pager.h> 112 #include <vm/vm_pageout.h> 113 #include <vm/vm_param.h> 114 #include <vm/swap_pager.h> 115 #include <vm/vm_extern.h> 116 #include <vm/uma.h> 117 118 #include <geom/geom.h> 119 120 /* 121 * MAX_PAGEOUT_CLUSTER must be a power of 2 between 1 and 64. 122 * The 64-page limit is due to the radix code (kern/subr_blist.c). 123 */ 124 #ifndef MAX_PAGEOUT_CLUSTER 125 #define MAX_PAGEOUT_CLUSTER 32 126 #endif 127 128 #if !defined(SWB_NPAGES) 129 #define SWB_NPAGES MAX_PAGEOUT_CLUSTER 130 #endif 131 132 #define SWAP_META_PAGES PCTRIE_COUNT 133 134 /* 135 * A swblk structure maps each page index within a 136 * SWAP_META_PAGES-aligned and sized range to the address of an 137 * on-disk swap block (or SWAPBLK_NONE). The collection of these 138 * mappings for an entire vm object is implemented as a pc-trie. 139 */ 140 struct swblk { 141 vm_pindex_t p; 142 daddr_t d[SWAP_META_PAGES]; 143 }; 144 145 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data"); 146 static struct mtx sw_dev_mtx; 147 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq); 148 static struct swdevt *swdevhd; /* Allocate from here next */ 149 static int nswapdev; /* Number of swap devices */ 150 int swap_pager_avail; 151 static struct sx swdev_syscall_lock; /* serialize swap(on|off) */ 152 153 static vm_ooffset_t swap_total; 154 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0, 155 "Total amount of available swap storage."); 156 static vm_ooffset_t swap_reserved; 157 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0, 158 "Amount of swap storage needed to back all allocated anonymous memory."); 159 static int overcommit = 0; 160 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0, 161 "Configure virtual memory overcommit behavior. See tuning(7) " 162 "for details."); 163 static unsigned long swzone; 164 SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0, 165 "Actual size of swap metadata zone"); 166 static unsigned long swap_maxpages; 167 SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0, 168 "Maximum amount of swap supported"); 169 170 /* bits from overcommit */ 171 #define SWAP_RESERVE_FORCE_ON (1 << 0) 172 #define SWAP_RESERVE_RLIMIT_ON (1 << 1) 173 #define SWAP_RESERVE_ALLOW_NONWIRED (1 << 2) 174 175 int 176 swap_reserve(vm_ooffset_t incr) 177 { 178 179 return (swap_reserve_by_cred(incr, curthread->td_ucred)); 180 } 181 182 int 183 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred) 184 { 185 vm_ooffset_t r, s; 186 int res, error; 187 static int curfail; 188 static struct timeval lastfail; 189 struct uidinfo *uip; 190 191 uip = cred->cr_ruidinfo; 192 193 if (incr & PAGE_MASK) 194 panic("swap_reserve: & PAGE_MASK"); 195 196 #ifdef RACCT 197 if (racct_enable) { 198 PROC_LOCK(curproc); 199 error = racct_add(curproc, RACCT_SWAP, incr); 200 PROC_UNLOCK(curproc); 201 if (error != 0) 202 return (0); 203 } 204 #endif 205 206 res = 0; 207 mtx_lock(&sw_dev_mtx); 208 r = swap_reserved + incr; 209 if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) { 210 s = vm_cnt.v_page_count - vm_cnt.v_free_reserved - vm_cnt.v_wire_count; 211 s *= PAGE_SIZE; 212 } else 213 s = 0; 214 s += swap_total; 215 if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s || 216 (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) { 217 res = 1; 218 swap_reserved = r; 219 } 220 mtx_unlock(&sw_dev_mtx); 221 222 if (res) { 223 UIDINFO_VMSIZE_LOCK(uip); 224 if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 && 225 uip->ui_vmsize + incr > lim_cur(curthread, RLIMIT_SWAP) && 226 priv_check(curthread, PRIV_VM_SWAP_NORLIMIT)) 227 res = 0; 228 else 229 uip->ui_vmsize += incr; 230 UIDINFO_VMSIZE_UNLOCK(uip); 231 if (!res) { 232 mtx_lock(&sw_dev_mtx); 233 swap_reserved -= incr; 234 mtx_unlock(&sw_dev_mtx); 235 } 236 } 237 if (!res && ppsratecheck(&lastfail, &curfail, 1)) { 238 printf("uid %d, pid %d: swap reservation for %jd bytes failed\n", 239 uip->ui_uid, curproc->p_pid, incr); 240 } 241 242 #ifdef RACCT 243 if (!res) { 244 PROC_LOCK(curproc); 245 racct_sub(curproc, RACCT_SWAP, incr); 246 PROC_UNLOCK(curproc); 247 } 248 #endif 249 250 return (res); 251 } 252 253 void 254 swap_reserve_force(vm_ooffset_t incr) 255 { 256 struct uidinfo *uip; 257 258 mtx_lock(&sw_dev_mtx); 259 swap_reserved += incr; 260 mtx_unlock(&sw_dev_mtx); 261 262 #ifdef RACCT 263 PROC_LOCK(curproc); 264 racct_add_force(curproc, RACCT_SWAP, incr); 265 PROC_UNLOCK(curproc); 266 #endif 267 268 uip = curthread->td_ucred->cr_ruidinfo; 269 PROC_LOCK(curproc); 270 UIDINFO_VMSIZE_LOCK(uip); 271 uip->ui_vmsize += incr; 272 UIDINFO_VMSIZE_UNLOCK(uip); 273 PROC_UNLOCK(curproc); 274 } 275 276 void 277 swap_release(vm_ooffset_t decr) 278 { 279 struct ucred *cred; 280 281 PROC_LOCK(curproc); 282 cred = curthread->td_ucred; 283 swap_release_by_cred(decr, cred); 284 PROC_UNLOCK(curproc); 285 } 286 287 void 288 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred) 289 { 290 struct uidinfo *uip; 291 292 uip = cred->cr_ruidinfo; 293 294 if (decr & PAGE_MASK) 295 panic("swap_release: & PAGE_MASK"); 296 297 mtx_lock(&sw_dev_mtx); 298 if (swap_reserved < decr) 299 panic("swap_reserved < decr"); 300 swap_reserved -= decr; 301 mtx_unlock(&sw_dev_mtx); 302 303 UIDINFO_VMSIZE_LOCK(uip); 304 if (uip->ui_vmsize < decr) 305 printf("negative vmsize for uid = %d\n", uip->ui_uid); 306 uip->ui_vmsize -= decr; 307 UIDINFO_VMSIZE_UNLOCK(uip); 308 309 racct_sub_cred(cred, RACCT_SWAP, decr); 310 } 311 312 #define SWM_FREE 0x02 /* free, period */ 313 #define SWM_POP 0x04 /* pop out */ 314 315 static int swap_pager_full = 2; /* swap space exhaustion (task killing) */ 316 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ 317 static int nsw_rcount; /* free read buffers */ 318 static int nsw_wcount_sync; /* limit write buffers / synchronous */ 319 static int nsw_wcount_async; /* limit write buffers / asynchronous */ 320 static int nsw_wcount_async_max;/* assigned maximum */ 321 static int nsw_cluster_max; /* maximum VOP I/O allowed */ 322 323 static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); 324 SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW | 325 CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I", 326 "Maximum running async swap ops"); 327 static int sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS); 328 SYSCTL_PROC(_vm, OID_AUTO, swap_fragmentation, CTLTYPE_STRING | CTLFLAG_RD | 329 CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_fragmentation, "A", 330 "Swap Fragmentation Info"); 331 332 static struct sx sw_alloc_sx; 333 334 /* 335 * "named" and "unnamed" anon region objects. Try to reduce the overhead 336 * of searching a named list by hashing it just a little. 337 */ 338 339 #define NOBJLISTS 8 340 341 #define NOBJLIST(handle) \ 342 (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)]) 343 344 static struct pagerlst swap_pager_object_list[NOBJLISTS]; 345 static uma_zone_t swblk_zone; 346 static uma_zone_t swpctrie_zone; 347 348 /* 349 * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure 350 * calls hooked from other parts of the VM system and do not appear here. 351 * (see vm/swap_pager.h). 352 */ 353 static vm_object_t 354 swap_pager_alloc(void *handle, vm_ooffset_t size, 355 vm_prot_t prot, vm_ooffset_t offset, struct ucred *); 356 static void swap_pager_dealloc(vm_object_t object); 357 static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int *, 358 int *); 359 static int swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *, 360 int *, pgo_getpages_iodone_t, void *); 361 static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); 362 static boolean_t 363 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); 364 static void swap_pager_init(void); 365 static void swap_pager_unswapped(vm_page_t); 366 static void swap_pager_swapoff(struct swdevt *sp); 367 368 struct pagerops swappagerops = { 369 .pgo_init = swap_pager_init, /* early system initialization of pager */ 370 .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ 371 .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ 372 .pgo_getpages = swap_pager_getpages, /* pagein */ 373 .pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ 374 .pgo_putpages = swap_pager_putpages, /* pageout */ 375 .pgo_haspage = swap_pager_haspage, /* get backing store status for page */ 376 .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ 377 }; 378 379 /* 380 * swap_*() routines are externally accessible. swp_*() routines are 381 * internal. 382 */ 383 static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ 384 static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ 385 386 SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0, 387 "Maximum size of a swap block in pages"); 388 389 static void swp_sizecheck(void); 390 static void swp_pager_async_iodone(struct buf *bp); 391 static int swapongeom(struct vnode *); 392 static int swaponvp(struct thread *, struct vnode *, u_long); 393 static int swapoff_one(struct swdevt *sp, struct ucred *cred); 394 395 /* 396 * Swap bitmap functions 397 */ 398 static void swp_pager_freeswapspace(daddr_t blk, int npages); 399 static daddr_t swp_pager_getswapspace(int npages); 400 401 /* 402 * Metadata functions 403 */ 404 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); 405 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); 406 static void swp_pager_meta_free_all(vm_object_t); 407 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); 408 409 static void * 410 swblk_trie_alloc(struct pctrie *ptree) 411 { 412 413 return (uma_zalloc(swpctrie_zone, M_NOWAIT | (curproc == pageproc ? 414 M_USE_RESERVE : 0))); 415 } 416 417 static void 418 swblk_trie_free(struct pctrie *ptree, void *node) 419 { 420 421 uma_zfree(swpctrie_zone, node); 422 } 423 424 PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free); 425 426 /* 427 * SWP_SIZECHECK() - update swap_pager_full indication 428 * 429 * update the swap_pager_almost_full indication and warn when we are 430 * about to run out of swap space, using lowat/hiwat hysteresis. 431 * 432 * Clear swap_pager_full ( task killing ) indication when lowat is met. 433 * 434 * No restrictions on call 435 * This routine may not block. 436 */ 437 static void 438 swp_sizecheck(void) 439 { 440 441 if (swap_pager_avail < nswap_lowat) { 442 if (swap_pager_almost_full == 0) { 443 printf("swap_pager: out of swap space\n"); 444 swap_pager_almost_full = 1; 445 } 446 } else { 447 swap_pager_full = 0; 448 if (swap_pager_avail > nswap_hiwat) 449 swap_pager_almost_full = 0; 450 } 451 } 452 453 /* 454 * SWAP_PAGER_INIT() - initialize the swap pager! 455 * 456 * Expected to be started from system init. NOTE: This code is run 457 * before much else so be careful what you depend on. Most of the VM 458 * system has yet to be initialized at this point. 459 */ 460 static void 461 swap_pager_init(void) 462 { 463 /* 464 * Initialize object lists 465 */ 466 int i; 467 468 for (i = 0; i < NOBJLISTS; ++i) 469 TAILQ_INIT(&swap_pager_object_list[i]); 470 mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF); 471 sx_init(&sw_alloc_sx, "swspsx"); 472 sx_init(&swdev_syscall_lock, "swsysc"); 473 } 474 475 /* 476 * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process 477 * 478 * Expected to be started from pageout process once, prior to entering 479 * its main loop. 480 */ 481 void 482 swap_pager_swap_init(void) 483 { 484 unsigned long n, n2; 485 486 /* 487 * Number of in-transit swap bp operations. Don't 488 * exhaust the pbufs completely. Make sure we 489 * initialize workable values (0 will work for hysteresis 490 * but it isn't very efficient). 491 * 492 * The nsw_cluster_max is constrained by the bp->b_pages[] 493 * array (MAXPHYS/PAGE_SIZE) and our locally defined 494 * MAX_PAGEOUT_CLUSTER. Also be aware that swap ops are 495 * constrained by the swap device interleave stripe size. 496 * 497 * Currently we hardwire nsw_wcount_async to 4. This limit is 498 * designed to prevent other I/O from having high latencies due to 499 * our pageout I/O. The value 4 works well for one or two active swap 500 * devices but is probably a little low if you have more. Even so, 501 * a higher value would probably generate only a limited improvement 502 * with three or four active swap devices since the system does not 503 * typically have to pageout at extreme bandwidths. We will want 504 * at least 2 per swap devices, and 4 is a pretty good value if you 505 * have one NFS swap device due to the command/ack latency over NFS. 506 * So it all works out pretty well. 507 */ 508 nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER); 509 510 mtx_lock(&pbuf_mtx); 511 nsw_rcount = (nswbuf + 1) / 2; 512 nsw_wcount_sync = (nswbuf + 3) / 4; 513 nsw_wcount_async = 4; 514 nsw_wcount_async_max = nsw_wcount_async; 515 mtx_unlock(&pbuf_mtx); 516 517 /* 518 * Initialize our zone, guessing on the number we need based 519 * on the number of pages in the system. 520 */ 521 n = vm_cnt.v_page_count / 2; 522 if (maxswzone && n > maxswzone / sizeof(struct swblk)) 523 n = maxswzone / sizeof(struct swblk); 524 swpctrie_zone = uma_zcreate("swpctrie", pctrie_node_size(), NULL, NULL, 525 pctrie_zone_init, NULL, UMA_ALIGN_PTR, 526 UMA_ZONE_NOFREE | UMA_ZONE_VM); 527 if (swpctrie_zone == NULL) 528 panic("failed to create swap pctrie zone."); 529 swblk_zone = uma_zcreate("swblk", sizeof(struct swblk), NULL, NULL, 530 NULL, NULL, _Alignof(struct swblk) - 1, 531 UMA_ZONE_NOFREE | UMA_ZONE_VM); 532 if (swblk_zone == NULL) 533 panic("failed to create swap blk zone."); 534 n2 = n; 535 do { 536 if (uma_zone_reserve_kva(swblk_zone, n)) 537 break; 538 /* 539 * if the allocation failed, try a zone two thirds the 540 * size of the previous attempt. 541 */ 542 n -= ((n + 2) / 3); 543 } while (n > 0); 544 if (n2 != n) 545 printf("Swap blk zone entries reduced from %lu to %lu.\n", 546 n2, n); 547 swap_maxpages = n * SWAP_META_PAGES; 548 swzone = n * sizeof(struct swblk); 549 if (!uma_zone_reserve_kva(swpctrie_zone, n)) 550 printf("Cannot reserve swap pctrie zone, " 551 "reduce kern.maxswzone.\n"); 552 } 553 554 static vm_object_t 555 swap_pager_alloc_init(void *handle, struct ucred *cred, vm_ooffset_t size, 556 vm_ooffset_t offset) 557 { 558 vm_object_t object; 559 560 if (cred != NULL) { 561 if (!swap_reserve_by_cred(size, cred)) 562 return (NULL); 563 crhold(cred); 564 } 565 566 /* 567 * The un_pager.swp.swp_blks trie is initialized by 568 * vm_object_allocate() to ensure the correct order of 569 * visibility to other threads. 570 */ 571 object = vm_object_allocate(OBJT_SWAP, OFF_TO_IDX(offset + 572 PAGE_MASK + size)); 573 574 object->handle = handle; 575 if (cred != NULL) { 576 object->cred = cred; 577 object->charge = size; 578 } 579 return (object); 580 } 581 582 /* 583 * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate 584 * its metadata structures. 585 * 586 * This routine is called from the mmap and fork code to create a new 587 * OBJT_SWAP object. 588 * 589 * This routine must ensure that no live duplicate is created for 590 * the named object request, which is protected against by 591 * holding the sw_alloc_sx lock in case handle != NULL. 592 */ 593 static vm_object_t 594 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 595 vm_ooffset_t offset, struct ucred *cred) 596 { 597 vm_object_t object; 598 599 if (handle != NULL) { 600 /* 601 * Reference existing named region or allocate new one. There 602 * should not be a race here against swp_pager_meta_build() 603 * as called from vm_page_remove() in regards to the lookup 604 * of the handle. 605 */ 606 sx_xlock(&sw_alloc_sx); 607 object = vm_pager_object_lookup(NOBJLIST(handle), handle); 608 if (object == NULL) { 609 object = swap_pager_alloc_init(handle, cred, size, 610 offset); 611 if (object != NULL) { 612 TAILQ_INSERT_TAIL(NOBJLIST(object->handle), 613 object, pager_object_list); 614 } 615 } 616 sx_xunlock(&sw_alloc_sx); 617 } else { 618 object = swap_pager_alloc_init(handle, cred, size, offset); 619 } 620 return (object); 621 } 622 623 /* 624 * SWAP_PAGER_DEALLOC() - remove swap metadata from object 625 * 626 * The swap backing for the object is destroyed. The code is 627 * designed such that we can reinstantiate it later, but this 628 * routine is typically called only when the entire object is 629 * about to be destroyed. 630 * 631 * The object must be locked. 632 */ 633 static void 634 swap_pager_dealloc(vm_object_t object) 635 { 636 637 VM_OBJECT_ASSERT_WLOCKED(object); 638 KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj")); 639 640 /* 641 * Remove from list right away so lookups will fail if we block for 642 * pageout completion. 643 */ 644 if (object->handle != NULL) { 645 VM_OBJECT_WUNLOCK(object); 646 sx_xlock(&sw_alloc_sx); 647 TAILQ_REMOVE(NOBJLIST(object->handle), object, 648 pager_object_list); 649 sx_xunlock(&sw_alloc_sx); 650 VM_OBJECT_WLOCK(object); 651 } 652 653 vm_object_pip_wait(object, "swpdea"); 654 655 /* 656 * Free all remaining metadata. We only bother to free it from 657 * the swap meta data. We do not attempt to free swapblk's still 658 * associated with vm_page_t's for this object. We do not care 659 * if paging is still in progress on some objects. 660 */ 661 swp_pager_meta_free_all(object); 662 object->handle = NULL; 663 object->type = OBJT_DEAD; 664 } 665 666 /************************************************************************ 667 * SWAP PAGER BITMAP ROUTINES * 668 ************************************************************************/ 669 670 /* 671 * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space 672 * 673 * Allocate swap for the requested number of pages. The starting 674 * swap block number (a page index) is returned or SWAPBLK_NONE 675 * if the allocation failed. 676 * 677 * Also has the side effect of advising that somebody made a mistake 678 * when they configured swap and didn't configure enough. 679 * 680 * This routine may not sleep. 681 * 682 * We allocate in round-robin fashion from the configured devices. 683 */ 684 static daddr_t 685 swp_pager_getswapspace(int npages) 686 { 687 daddr_t blk; 688 struct swdevt *sp; 689 int i; 690 691 blk = SWAPBLK_NONE; 692 mtx_lock(&sw_dev_mtx); 693 sp = swdevhd; 694 for (i = 0; i < nswapdev; i++) { 695 if (sp == NULL) 696 sp = TAILQ_FIRST(&swtailq); 697 if (!(sp->sw_flags & SW_CLOSING)) { 698 blk = blist_alloc(sp->sw_blist, npages); 699 if (blk != SWAPBLK_NONE) { 700 blk += sp->sw_first; 701 sp->sw_used += npages; 702 swap_pager_avail -= npages; 703 swp_sizecheck(); 704 swdevhd = TAILQ_NEXT(sp, sw_list); 705 goto done; 706 } 707 } 708 sp = TAILQ_NEXT(sp, sw_list); 709 } 710 if (swap_pager_full != 2) { 711 printf("swap_pager_getswapspace(%d): failed\n", npages); 712 swap_pager_full = 2; 713 swap_pager_almost_full = 1; 714 } 715 swdevhd = NULL; 716 done: 717 mtx_unlock(&sw_dev_mtx); 718 return (blk); 719 } 720 721 static int 722 swp_pager_isondev(daddr_t blk, struct swdevt *sp) 723 { 724 725 return (blk >= sp->sw_first && blk < sp->sw_end); 726 } 727 728 static void 729 swp_pager_strategy(struct buf *bp) 730 { 731 struct swdevt *sp; 732 733 mtx_lock(&sw_dev_mtx); 734 TAILQ_FOREACH(sp, &swtailq, sw_list) { 735 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) { 736 mtx_unlock(&sw_dev_mtx); 737 if ((sp->sw_flags & SW_UNMAPPED) != 0 && 738 unmapped_buf_allowed) { 739 bp->b_data = unmapped_buf; 740 bp->b_offset = 0; 741 } else { 742 pmap_qenter((vm_offset_t)bp->b_data, 743 &bp->b_pages[0], bp->b_bcount / PAGE_SIZE); 744 } 745 sp->sw_strategy(bp, sp); 746 return; 747 } 748 } 749 panic("Swapdev not found"); 750 } 751 752 753 /* 754 * SWP_PAGER_FREESWAPSPACE() - free raw swap space 755 * 756 * This routine returns the specified swap blocks back to the bitmap. 757 * 758 * This routine may not sleep. 759 */ 760 static void 761 swp_pager_freeswapspace(daddr_t blk, int npages) 762 { 763 struct swdevt *sp; 764 765 mtx_lock(&sw_dev_mtx); 766 TAILQ_FOREACH(sp, &swtailq, sw_list) { 767 if (blk >= sp->sw_first && blk < sp->sw_end) { 768 sp->sw_used -= npages; 769 /* 770 * If we are attempting to stop swapping on 771 * this device, we don't want to mark any 772 * blocks free lest they be reused. 773 */ 774 if ((sp->sw_flags & SW_CLOSING) == 0) { 775 blist_free(sp->sw_blist, blk - sp->sw_first, 776 npages); 777 swap_pager_avail += npages; 778 swp_sizecheck(); 779 } 780 mtx_unlock(&sw_dev_mtx); 781 return; 782 } 783 } 784 panic("Swapdev not found"); 785 } 786 787 /* 788 * SYSCTL_SWAP_FRAGMENTATION() - produce raw swap space stats 789 */ 790 static int 791 sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS) 792 { 793 struct sbuf sbuf; 794 struct swdevt *sp; 795 const char *devname; 796 int error; 797 798 error = sysctl_wire_old_buffer(req, 0); 799 if (error != 0) 800 return (error); 801 sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 802 mtx_lock(&sw_dev_mtx); 803 TAILQ_FOREACH(sp, &swtailq, sw_list) { 804 if (vn_isdisk(sp->sw_vp, NULL)) 805 devname = devtoname(sp->sw_vp->v_rdev); 806 else 807 devname = "[file]"; 808 sbuf_printf(&sbuf, "\nFree space on device %s:\n", devname); 809 blist_stats(sp->sw_blist, &sbuf); 810 } 811 mtx_unlock(&sw_dev_mtx); 812 error = sbuf_finish(&sbuf); 813 sbuf_delete(&sbuf); 814 return (error); 815 } 816 817 /* 818 * SWAP_PAGER_FREESPACE() - frees swap blocks associated with a page 819 * range within an object. 820 * 821 * This is a globally accessible routine. 822 * 823 * This routine removes swapblk assignments from swap metadata. 824 * 825 * The external callers of this routine typically have already destroyed 826 * or renamed vm_page_t's associated with this range in the object so 827 * we should be ok. 828 * 829 * The object must be locked. 830 */ 831 void 832 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size) 833 { 834 835 swp_pager_meta_free(object, start, size); 836 } 837 838 /* 839 * SWAP_PAGER_RESERVE() - reserve swap blocks in object 840 * 841 * Assigns swap blocks to the specified range within the object. The 842 * swap blocks are not zeroed. Any previous swap assignment is destroyed. 843 * 844 * Returns 0 on success, -1 on failure. 845 */ 846 int 847 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) 848 { 849 int n = 0; 850 daddr_t blk = SWAPBLK_NONE; 851 vm_pindex_t beg = start; /* save start index */ 852 853 VM_OBJECT_WLOCK(object); 854 while (size) { 855 if (n == 0) { 856 n = BLIST_MAX_ALLOC; 857 while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) { 858 n >>= 1; 859 if (n == 0) { 860 swp_pager_meta_free(object, beg, start - beg); 861 VM_OBJECT_WUNLOCK(object); 862 return (-1); 863 } 864 } 865 } 866 swp_pager_meta_build(object, start, blk); 867 --size; 868 ++start; 869 ++blk; 870 --n; 871 } 872 swp_pager_meta_free(object, start, n); 873 VM_OBJECT_WUNLOCK(object); 874 return (0); 875 } 876 877 /* 878 * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager 879 * and destroy the source. 880 * 881 * Copy any valid swapblks from the source to the destination. In 882 * cases where both the source and destination have a valid swapblk, 883 * we keep the destination's. 884 * 885 * This routine is allowed to sleep. It may sleep allocating metadata 886 * indirectly through swp_pager_meta_build() or if paging is still in 887 * progress on the source. 888 * 889 * The source object contains no vm_page_t's (which is just as well) 890 * 891 * The source object is of type OBJT_SWAP. 892 * 893 * The source and destination objects must be locked. 894 * Both object locks may temporarily be released. 895 */ 896 void 897 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, 898 vm_pindex_t offset, int destroysource) 899 { 900 vm_pindex_t i; 901 902 VM_OBJECT_ASSERT_WLOCKED(srcobject); 903 VM_OBJECT_ASSERT_WLOCKED(dstobject); 904 905 /* 906 * If destroysource is set, we remove the source object from the 907 * swap_pager internal queue now. 908 */ 909 if (destroysource && srcobject->handle != NULL) { 910 vm_object_pip_add(srcobject, 1); 911 VM_OBJECT_WUNLOCK(srcobject); 912 vm_object_pip_add(dstobject, 1); 913 VM_OBJECT_WUNLOCK(dstobject); 914 sx_xlock(&sw_alloc_sx); 915 TAILQ_REMOVE(NOBJLIST(srcobject->handle), srcobject, 916 pager_object_list); 917 sx_xunlock(&sw_alloc_sx); 918 VM_OBJECT_WLOCK(dstobject); 919 vm_object_pip_wakeup(dstobject); 920 VM_OBJECT_WLOCK(srcobject); 921 vm_object_pip_wakeup(srcobject); 922 } 923 924 /* 925 * transfer source to destination. 926 */ 927 for (i = 0; i < dstobject->size; ++i) { 928 daddr_t dstaddr; 929 930 /* 931 * Locate (without changing) the swapblk on the destination, 932 * unless it is invalid in which case free it silently, or 933 * if the destination is a resident page, in which case the 934 * source is thrown away. 935 */ 936 dstaddr = swp_pager_meta_ctl(dstobject, i, 0); 937 938 if (dstaddr == SWAPBLK_NONE) { 939 /* 940 * Destination has no swapblk and is not resident, 941 * copy source. 942 */ 943 daddr_t srcaddr; 944 945 srcaddr = swp_pager_meta_ctl( 946 srcobject, 947 i + offset, 948 SWM_POP 949 ); 950 951 if (srcaddr != SWAPBLK_NONE) { 952 /* 953 * swp_pager_meta_build() can sleep. 954 */ 955 vm_object_pip_add(srcobject, 1); 956 VM_OBJECT_WUNLOCK(srcobject); 957 vm_object_pip_add(dstobject, 1); 958 swp_pager_meta_build(dstobject, i, srcaddr); 959 vm_object_pip_wakeup(dstobject); 960 VM_OBJECT_WLOCK(srcobject); 961 vm_object_pip_wakeup(srcobject); 962 } 963 } else { 964 /* 965 * Destination has valid swapblk or it is represented 966 * by a resident page. We destroy the sourceblock. 967 */ 968 969 swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE); 970 } 971 } 972 973 /* 974 * Free left over swap blocks in source. 975 * 976 * We have to revert the type to OBJT_DEFAULT so we do not accidentally 977 * double-remove the object from the swap queues. 978 */ 979 if (destroysource) { 980 swp_pager_meta_free_all(srcobject); 981 /* 982 * Reverting the type is not necessary, the caller is going 983 * to destroy srcobject directly, but I'm doing it here 984 * for consistency since we've removed the object from its 985 * queues. 986 */ 987 srcobject->type = OBJT_DEFAULT; 988 } 989 } 990 991 /* 992 * SWAP_PAGER_HASPAGE() - determine if we have good backing store for 993 * the requested page. 994 * 995 * We determine whether good backing store exists for the requested 996 * page and return TRUE if it does, FALSE if it doesn't. 997 * 998 * If TRUE, we also try to determine how much valid, contiguous backing 999 * store exists before and after the requested page. 1000 */ 1001 static boolean_t 1002 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 1003 int *after) 1004 { 1005 daddr_t blk, blk0; 1006 int i; 1007 1008 VM_OBJECT_ASSERT_LOCKED(object); 1009 1010 /* 1011 * do we have good backing store at the requested index ? 1012 */ 1013 blk0 = swp_pager_meta_ctl(object, pindex, 0); 1014 if (blk0 == SWAPBLK_NONE) { 1015 if (before) 1016 *before = 0; 1017 if (after) 1018 *after = 0; 1019 return (FALSE); 1020 } 1021 1022 /* 1023 * find backwards-looking contiguous good backing store 1024 */ 1025 if (before != NULL) { 1026 for (i = 1; i < SWB_NPAGES; i++) { 1027 if (i > pindex) 1028 break; 1029 blk = swp_pager_meta_ctl(object, pindex - i, 0); 1030 if (blk != blk0 - i) 1031 break; 1032 } 1033 *before = i - 1; 1034 } 1035 1036 /* 1037 * find forward-looking contiguous good backing store 1038 */ 1039 if (after != NULL) { 1040 for (i = 1; i < SWB_NPAGES; i++) { 1041 blk = swp_pager_meta_ctl(object, pindex + i, 0); 1042 if (blk != blk0 + i) 1043 break; 1044 } 1045 *after = i - 1; 1046 } 1047 return (TRUE); 1048 } 1049 1050 /* 1051 * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page 1052 * 1053 * This removes any associated swap backing store, whether valid or 1054 * not, from the page. 1055 * 1056 * This routine is typically called when a page is made dirty, at 1057 * which point any associated swap can be freed. MADV_FREE also 1058 * calls us in a special-case situation 1059 * 1060 * NOTE!!! If the page is clean and the swap was valid, the caller 1061 * should make the page dirty before calling this routine. This routine 1062 * does NOT change the m->dirty status of the page. Also: MADV_FREE 1063 * depends on it. 1064 * 1065 * This routine may not sleep. 1066 * 1067 * The object containing the page must be locked. 1068 */ 1069 static void 1070 swap_pager_unswapped(vm_page_t m) 1071 { 1072 1073 swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE); 1074 } 1075 1076 /* 1077 * swap_pager_getpages() - bring pages in from swap 1078 * 1079 * Attempt to page in the pages in array "m" of length "count". The caller 1080 * may optionally specify that additional pages preceding and succeeding 1081 * the specified range be paged in. The number of such pages is returned 1082 * in the "rbehind" and "rahead" parameters, and they will be in the 1083 * inactive queue upon return. 1084 * 1085 * The pages in "m" must be busied and will remain busied upon return. 1086 */ 1087 static int 1088 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, 1089 int *rahead) 1090 { 1091 struct buf *bp; 1092 vm_page_t mpred, msucc, p; 1093 vm_pindex_t pindex; 1094 daddr_t blk; 1095 int i, j, maxahead, maxbehind, reqcount, shift; 1096 1097 reqcount = count; 1098 1099 VM_OBJECT_WUNLOCK(object); 1100 bp = getpbuf(&nsw_rcount); 1101 VM_OBJECT_WLOCK(object); 1102 1103 if (!swap_pager_haspage(object, m[0]->pindex, &maxbehind, &maxahead)) { 1104 relpbuf(bp, &nsw_rcount); 1105 return (VM_PAGER_FAIL); 1106 } 1107 1108 /* 1109 * Clip the readahead and readbehind ranges to exclude resident pages. 1110 */ 1111 if (rahead != NULL) { 1112 KASSERT(reqcount - 1 <= maxahead, 1113 ("page count %d extends beyond swap block", reqcount)); 1114 *rahead = imin(*rahead, maxahead - (reqcount - 1)); 1115 pindex = m[reqcount - 1]->pindex; 1116 msucc = TAILQ_NEXT(m[reqcount - 1], listq); 1117 if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead) 1118 *rahead = msucc->pindex - pindex - 1; 1119 } 1120 if (rbehind != NULL) { 1121 *rbehind = imin(*rbehind, maxbehind); 1122 pindex = m[0]->pindex; 1123 mpred = TAILQ_PREV(m[0], pglist, listq); 1124 if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind) 1125 *rbehind = pindex - mpred->pindex - 1; 1126 } 1127 1128 /* 1129 * Allocate readahead and readbehind pages. 1130 */ 1131 shift = rbehind != NULL ? *rbehind : 0; 1132 if (shift != 0) { 1133 for (i = 1; i <= shift; i++) { 1134 p = vm_page_alloc(object, m[0]->pindex - i, 1135 VM_ALLOC_NORMAL); 1136 if (p == NULL) { 1137 /* Shift allocated pages to the left. */ 1138 for (j = 0; j < i - 1; j++) 1139 bp->b_pages[j] = 1140 bp->b_pages[j + shift - i + 1]; 1141 break; 1142 } 1143 bp->b_pages[shift - i] = p; 1144 } 1145 shift = i - 1; 1146 *rbehind = shift; 1147 } 1148 for (i = 0; i < reqcount; i++) 1149 bp->b_pages[i + shift] = m[i]; 1150 if (rahead != NULL) { 1151 for (i = 0; i < *rahead; i++) { 1152 p = vm_page_alloc(object, 1153 m[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL); 1154 if (p == NULL) 1155 break; 1156 bp->b_pages[shift + reqcount + i] = p; 1157 } 1158 *rahead = i; 1159 } 1160 if (rbehind != NULL) 1161 count += *rbehind; 1162 if (rahead != NULL) 1163 count += *rahead; 1164 1165 vm_object_pip_add(object, count); 1166 1167 for (i = 0; i < count; i++) 1168 bp->b_pages[i]->oflags |= VPO_SWAPINPROG; 1169 1170 pindex = bp->b_pages[0]->pindex; 1171 blk = swp_pager_meta_ctl(object, pindex, 0); 1172 KASSERT(blk != SWAPBLK_NONE, 1173 ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); 1174 1175 VM_OBJECT_WUNLOCK(object); 1176 1177 bp->b_flags |= B_PAGING; 1178 bp->b_iocmd = BIO_READ; 1179 bp->b_iodone = swp_pager_async_iodone; 1180 bp->b_rcred = crhold(thread0.td_ucred); 1181 bp->b_wcred = crhold(thread0.td_ucred); 1182 bp->b_blkno = blk; 1183 bp->b_bcount = PAGE_SIZE * count; 1184 bp->b_bufsize = PAGE_SIZE * count; 1185 bp->b_npages = count; 1186 bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; 1187 bp->b_pgafter = rahead != NULL ? *rahead : 0; 1188 1189 VM_CNT_INC(v_swapin); 1190 VM_CNT_ADD(v_swappgsin, count); 1191 1192 /* 1193 * perform the I/O. NOTE!!! bp cannot be considered valid after 1194 * this point because we automatically release it on completion. 1195 * Instead, we look at the one page we are interested in which we 1196 * still hold a lock on even through the I/O completion. 1197 * 1198 * The other pages in our m[] array are also released on completion, 1199 * so we cannot assume they are valid anymore either. 1200 * 1201 * NOTE: b_blkno is destroyed by the call to swapdev_strategy 1202 */ 1203 BUF_KERNPROC(bp); 1204 swp_pager_strategy(bp); 1205 1206 /* 1207 * Wait for the pages we want to complete. VPO_SWAPINPROG is always 1208 * cleared on completion. If an I/O error occurs, SWAPBLK_NONE 1209 * is set in the metadata for each page in the request. 1210 */ 1211 VM_OBJECT_WLOCK(object); 1212 while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { 1213 m[0]->oflags |= VPO_SWAPSLEEP; 1214 VM_CNT_INC(v_intrans); 1215 if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, 1216 "swread", hz * 20)) { 1217 printf( 1218 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", 1219 bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); 1220 } 1221 } 1222 1223 /* 1224 * If we had an unrecoverable read error pages will not be valid. 1225 */ 1226 for (i = 0; i < reqcount; i++) 1227 if (m[i]->valid != VM_PAGE_BITS_ALL) 1228 return (VM_PAGER_ERROR); 1229 1230 return (VM_PAGER_OK); 1231 1232 /* 1233 * A final note: in a low swap situation, we cannot deallocate swap 1234 * and mark a page dirty here because the caller is likely to mark 1235 * the page clean when we return, causing the page to possibly revert 1236 * to all-zero's later. 1237 */ 1238 } 1239 1240 /* 1241 * swap_pager_getpages_async(): 1242 * 1243 * Right now this is emulation of asynchronous operation on top of 1244 * swap_pager_getpages(). 1245 */ 1246 static int 1247 swap_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, 1248 int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg) 1249 { 1250 int r, error; 1251 1252 r = swap_pager_getpages(object, m, count, rbehind, rahead); 1253 VM_OBJECT_WUNLOCK(object); 1254 switch (r) { 1255 case VM_PAGER_OK: 1256 error = 0; 1257 break; 1258 case VM_PAGER_ERROR: 1259 error = EIO; 1260 break; 1261 case VM_PAGER_FAIL: 1262 error = EINVAL; 1263 break; 1264 default: 1265 panic("unhandled swap_pager_getpages() error %d", r); 1266 } 1267 (iodone)(arg, m, count, error); 1268 VM_OBJECT_WLOCK(object); 1269 1270 return (r); 1271 } 1272 1273 /* 1274 * swap_pager_putpages: 1275 * 1276 * Assign swap (if necessary) and initiate I/O on the specified pages. 1277 * 1278 * We support both OBJT_DEFAULT and OBJT_SWAP objects. DEFAULT objects 1279 * are automatically converted to SWAP objects. 1280 * 1281 * In a low memory situation we may block in VOP_STRATEGY(), but the new 1282 * vm_page reservation system coupled with properly written VFS devices 1283 * should ensure that no low-memory deadlock occurs. This is an area 1284 * which needs work. 1285 * 1286 * The parent has N vm_object_pip_add() references prior to 1287 * calling us and will remove references for rtvals[] that are 1288 * not set to VM_PAGER_PEND. We need to remove the rest on I/O 1289 * completion. 1290 * 1291 * The parent has soft-busy'd the pages it passes us and will unbusy 1292 * those whos rtvals[] entry is not set to VM_PAGER_PEND on return. 1293 * We need to unbusy the rest on I/O completion. 1294 */ 1295 static void 1296 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count, 1297 int flags, int *rtvals) 1298 { 1299 int i, n; 1300 boolean_t sync; 1301 1302 if (count && m[0]->object != object) { 1303 panic("swap_pager_putpages: object mismatch %p/%p", 1304 object, 1305 m[0]->object 1306 ); 1307 } 1308 1309 /* 1310 * Step 1 1311 * 1312 * Turn object into OBJT_SWAP 1313 * check for bogus sysops 1314 * force sync if not pageout process 1315 */ 1316 if (object->type != OBJT_SWAP) 1317 swp_pager_meta_build(object, 0, SWAPBLK_NONE); 1318 VM_OBJECT_WUNLOCK(object); 1319 1320 n = 0; 1321 if (curproc != pageproc) 1322 sync = TRUE; 1323 else 1324 sync = (flags & VM_PAGER_PUT_SYNC) != 0; 1325 1326 /* 1327 * Step 2 1328 * 1329 * Assign swap blocks and issue I/O. We reallocate swap on the fly. 1330 * The page is left dirty until the pageout operation completes 1331 * successfully. 1332 */ 1333 for (i = 0; i < count; i += n) { 1334 int j; 1335 struct buf *bp; 1336 daddr_t blk; 1337 1338 /* 1339 * Maximum I/O size is limited by a number of factors. 1340 */ 1341 n = min(BLIST_MAX_ALLOC, count - i); 1342 n = min(n, nsw_cluster_max); 1343 1344 /* 1345 * Get biggest block of swap we can. If we fail, fall 1346 * back and try to allocate a smaller block. Don't go 1347 * overboard trying to allocate space if it would overly 1348 * fragment swap. 1349 */ 1350 while ( 1351 (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE && 1352 n > 4 1353 ) { 1354 n >>= 1; 1355 } 1356 if (blk == SWAPBLK_NONE) { 1357 for (j = 0; j < n; ++j) 1358 rtvals[i+j] = VM_PAGER_FAIL; 1359 continue; 1360 } 1361 1362 /* 1363 * All I/O parameters have been satisfied, build the I/O 1364 * request and assign the swap space. 1365 */ 1366 if (sync == TRUE) { 1367 bp = getpbuf(&nsw_wcount_sync); 1368 } else { 1369 bp = getpbuf(&nsw_wcount_async); 1370 bp->b_flags = B_ASYNC; 1371 } 1372 bp->b_flags |= B_PAGING; 1373 bp->b_iocmd = BIO_WRITE; 1374 1375 bp->b_rcred = crhold(thread0.td_ucred); 1376 bp->b_wcred = crhold(thread0.td_ucred); 1377 bp->b_bcount = PAGE_SIZE * n; 1378 bp->b_bufsize = PAGE_SIZE * n; 1379 bp->b_blkno = blk; 1380 1381 VM_OBJECT_WLOCK(object); 1382 for (j = 0; j < n; ++j) { 1383 vm_page_t mreq = m[i+j]; 1384 1385 swp_pager_meta_build( 1386 mreq->object, 1387 mreq->pindex, 1388 blk + j 1389 ); 1390 MPASS(mreq->dirty == VM_PAGE_BITS_ALL); 1391 mreq->oflags |= VPO_SWAPINPROG; 1392 bp->b_pages[j] = mreq; 1393 } 1394 VM_OBJECT_WUNLOCK(object); 1395 bp->b_npages = n; 1396 /* 1397 * Must set dirty range for NFS to work. 1398 */ 1399 bp->b_dirtyoff = 0; 1400 bp->b_dirtyend = bp->b_bcount; 1401 1402 VM_CNT_INC(v_swapout); 1403 VM_CNT_ADD(v_swappgsout, bp->b_npages); 1404 1405 /* 1406 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we 1407 * can call the async completion routine at the end of a 1408 * synchronous I/O operation. Otherwise, our caller would 1409 * perform duplicate unbusy and wakeup operations on the page 1410 * and object, respectively. 1411 */ 1412 for (j = 0; j < n; j++) 1413 rtvals[i + j] = VM_PAGER_PEND; 1414 1415 /* 1416 * asynchronous 1417 * 1418 * NOTE: b_blkno is destroyed by the call to swapdev_strategy 1419 */ 1420 if (sync == FALSE) { 1421 bp->b_iodone = swp_pager_async_iodone; 1422 BUF_KERNPROC(bp); 1423 swp_pager_strategy(bp); 1424 continue; 1425 } 1426 1427 /* 1428 * synchronous 1429 * 1430 * NOTE: b_blkno is destroyed by the call to swapdev_strategy 1431 */ 1432 bp->b_iodone = bdone; 1433 swp_pager_strategy(bp); 1434 1435 /* 1436 * Wait for the sync I/O to complete. 1437 */ 1438 bwait(bp, PVM, "swwrt"); 1439 1440 /* 1441 * Now that we are through with the bp, we can call the 1442 * normal async completion, which frees everything up. 1443 */ 1444 swp_pager_async_iodone(bp); 1445 } 1446 VM_OBJECT_WLOCK(object); 1447 } 1448 1449 /* 1450 * swp_pager_async_iodone: 1451 * 1452 * Completion routine for asynchronous reads and writes from/to swap. 1453 * Also called manually by synchronous code to finish up a bp. 1454 * 1455 * This routine may not sleep. 1456 */ 1457 static void 1458 swp_pager_async_iodone(struct buf *bp) 1459 { 1460 int i; 1461 vm_object_t object = NULL; 1462 1463 /* 1464 * report error 1465 */ 1466 if (bp->b_ioflags & BIO_ERROR) { 1467 printf( 1468 "swap_pager: I/O error - %s failed; blkno %ld," 1469 "size %ld, error %d\n", 1470 ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"), 1471 (long)bp->b_blkno, 1472 (long)bp->b_bcount, 1473 bp->b_error 1474 ); 1475 } 1476 1477 /* 1478 * remove the mapping for kernel virtual 1479 */ 1480 if (buf_mapped(bp)) 1481 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages); 1482 else 1483 bp->b_data = bp->b_kvabase; 1484 1485 if (bp->b_npages) { 1486 object = bp->b_pages[0]->object; 1487 VM_OBJECT_WLOCK(object); 1488 } 1489 1490 /* 1491 * cleanup pages. If an error occurs writing to swap, we are in 1492 * very serious trouble. If it happens to be a disk error, though, 1493 * we may be able to recover by reassigning the swap later on. So 1494 * in this case we remove the m->swapblk assignment for the page 1495 * but do not free it in the rlist. The errornous block(s) are thus 1496 * never reallocated as swap. Redirty the page and continue. 1497 */ 1498 for (i = 0; i < bp->b_npages; ++i) { 1499 vm_page_t m = bp->b_pages[i]; 1500 1501 m->oflags &= ~VPO_SWAPINPROG; 1502 if (m->oflags & VPO_SWAPSLEEP) { 1503 m->oflags &= ~VPO_SWAPSLEEP; 1504 wakeup(&object->paging_in_progress); 1505 } 1506 1507 if (bp->b_ioflags & BIO_ERROR) { 1508 /* 1509 * If an error occurs I'd love to throw the swapblk 1510 * away without freeing it back to swapspace, so it 1511 * can never be used again. But I can't from an 1512 * interrupt. 1513 */ 1514 if (bp->b_iocmd == BIO_READ) { 1515 /* 1516 * NOTE: for reads, m->dirty will probably 1517 * be overridden by the original caller of 1518 * getpages so don't play cute tricks here. 1519 */ 1520 m->valid = 0; 1521 } else { 1522 /* 1523 * If a write error occurs, reactivate page 1524 * so it doesn't clog the inactive list, 1525 * then finish the I/O. 1526 */ 1527 MPASS(m->dirty == VM_PAGE_BITS_ALL); 1528 vm_page_lock(m); 1529 vm_page_activate(m); 1530 vm_page_unlock(m); 1531 vm_page_sunbusy(m); 1532 } 1533 } else if (bp->b_iocmd == BIO_READ) { 1534 /* 1535 * NOTE: for reads, m->dirty will probably be 1536 * overridden by the original caller of getpages so 1537 * we cannot set them in order to free the underlying 1538 * swap in a low-swap situation. I don't think we'd 1539 * want to do that anyway, but it was an optimization 1540 * that existed in the old swapper for a time before 1541 * it got ripped out due to precisely this problem. 1542 */ 1543 KASSERT(!pmap_page_is_mapped(m), 1544 ("swp_pager_async_iodone: page %p is mapped", m)); 1545 KASSERT(m->dirty == 0, 1546 ("swp_pager_async_iodone: page %p is dirty", m)); 1547 1548 m->valid = VM_PAGE_BITS_ALL; 1549 if (i < bp->b_pgbefore || 1550 i >= bp->b_npages - bp->b_pgafter) 1551 vm_page_readahead_finish(m); 1552 } else { 1553 /* 1554 * For write success, clear the dirty 1555 * status, then finish the I/O ( which decrements the 1556 * busy count and possibly wakes waiter's up ). 1557 * A page is only written to swap after a period of 1558 * inactivity. Therefore, we do not expect it to be 1559 * reused. 1560 */ 1561 KASSERT(!pmap_page_is_write_mapped(m), 1562 ("swp_pager_async_iodone: page %p is not write" 1563 " protected", m)); 1564 vm_page_undirty(m); 1565 vm_page_lock(m); 1566 vm_page_deactivate_noreuse(m); 1567 vm_page_unlock(m); 1568 vm_page_sunbusy(m); 1569 } 1570 } 1571 1572 /* 1573 * adjust pip. NOTE: the original parent may still have its own 1574 * pip refs on the object. 1575 */ 1576 if (object != NULL) { 1577 vm_object_pip_wakeupn(object, bp->b_npages); 1578 VM_OBJECT_WUNLOCK(object); 1579 } 1580 1581 /* 1582 * swapdev_strategy() manually sets b_vp and b_bufobj before calling 1583 * bstrategy(). Set them back to NULL now we're done with it, or we'll 1584 * trigger a KASSERT in relpbuf(). 1585 */ 1586 if (bp->b_vp) { 1587 bp->b_vp = NULL; 1588 bp->b_bufobj = NULL; 1589 } 1590 /* 1591 * release the physical I/O buffer 1592 */ 1593 relpbuf( 1594 bp, 1595 ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : 1596 ((bp->b_flags & B_ASYNC) ? 1597 &nsw_wcount_async : 1598 &nsw_wcount_sync 1599 ) 1600 ) 1601 ); 1602 } 1603 1604 int 1605 swap_pager_nswapdev(void) 1606 { 1607 1608 return (nswapdev); 1609 } 1610 1611 /* 1612 * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in 1613 * 1614 * This routine dissociates the page at the given index within an object 1615 * from its backing store, paging it in if it does not reside in memory. 1616 * If the page is paged in, it is marked dirty and placed in the laundry 1617 * queue. The page is marked dirty because it no longer has backing 1618 * store. It is placed in the laundry queue because it has not been 1619 * accessed recently. Otherwise, it would already reside in memory. 1620 * 1621 * We also attempt to swap in all other pages in the swap block. 1622 * However, we only guarantee that the one at the specified index is 1623 * paged in. 1624 * 1625 * XXX - The code to page the whole block in doesn't work, so we 1626 * revert to the one-by-one behavior for now. Sigh. 1627 */ 1628 static inline void 1629 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex) 1630 { 1631 vm_page_t m; 1632 1633 vm_object_pip_add(object, 1); 1634 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 1635 if (m->valid == VM_PAGE_BITS_ALL) { 1636 vm_object_pip_wakeup(object); 1637 vm_page_dirty(m); 1638 #ifdef INVARIANTS 1639 vm_page_lock(m); 1640 if (m->wire_count == 0 && m->queue == PQ_NONE) 1641 panic("page %p is neither wired nor queued", m); 1642 vm_page_unlock(m); 1643 #endif 1644 vm_page_xunbusy(m); 1645 vm_pager_page_unswapped(m); 1646 return; 1647 } 1648 1649 if (swap_pager_getpages(object, &m, 1, NULL, NULL) != VM_PAGER_OK) 1650 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/ 1651 vm_object_pip_wakeup(object); 1652 vm_page_dirty(m); 1653 vm_page_lock(m); 1654 vm_page_launder(m); 1655 vm_page_unlock(m); 1656 vm_page_xunbusy(m); 1657 vm_pager_page_unswapped(m); 1658 } 1659 1660 /* 1661 * swap_pager_swapoff: 1662 * 1663 * Page in all of the pages that have been paged out to the 1664 * given device. The corresponding blocks in the bitmap must be 1665 * marked as allocated and the device must be flagged SW_CLOSING. 1666 * There may be no processes swapped out to the device. 1667 * 1668 * This routine may block. 1669 */ 1670 static void 1671 swap_pager_swapoff(struct swdevt *sp) 1672 { 1673 struct swblk *sb; 1674 vm_object_t object; 1675 vm_pindex_t pi; 1676 int i, retries; 1677 1678 sx_assert(&swdev_syscall_lock, SA_XLOCKED); 1679 1680 retries = 0; 1681 full_rescan: 1682 mtx_lock(&vm_object_list_mtx); 1683 TAILQ_FOREACH(object, &vm_object_list, object_list) { 1684 if (object->type != OBJT_SWAP) 1685 continue; 1686 mtx_unlock(&vm_object_list_mtx); 1687 /* Depends on type-stability. */ 1688 VM_OBJECT_WLOCK(object); 1689 1690 /* 1691 * Dead objects are eventually terminated on their own. 1692 */ 1693 if ((object->flags & OBJ_DEAD) != 0) 1694 goto next_obj; 1695 1696 /* 1697 * Sync with fences placed after pctrie 1698 * initialization. We must not access pctrie below 1699 * unless we checked that our object is swap and not 1700 * dead. 1701 */ 1702 atomic_thread_fence_acq(); 1703 if (object->type != OBJT_SWAP) 1704 goto next_obj; 1705 1706 for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( 1707 &object->un_pager.swp.swp_blks, pi)) != NULL; ) { 1708 pi = sb->p + SWAP_META_PAGES; 1709 for (i = 0; i < SWAP_META_PAGES; i++) { 1710 if (sb->d[i] == SWAPBLK_NONE) 1711 continue; 1712 if (swp_pager_isondev(sb->d[i], sp)) 1713 swp_pager_force_pagein(object, 1714 sb->p + i); 1715 } 1716 } 1717 next_obj: 1718 VM_OBJECT_WUNLOCK(object); 1719 mtx_lock(&vm_object_list_mtx); 1720 } 1721 mtx_unlock(&vm_object_list_mtx); 1722 1723 if (sp->sw_used) { 1724 /* 1725 * Objects may be locked or paging to the device being 1726 * removed, so we will miss their pages and need to 1727 * make another pass. We have marked this device as 1728 * SW_CLOSING, so the activity should finish soon. 1729 */ 1730 retries++; 1731 if (retries > 100) { 1732 panic("swapoff: failed to locate %d swap blocks", 1733 sp->sw_used); 1734 } 1735 pause("swpoff", hz / 20); 1736 goto full_rescan; 1737 } 1738 EVENTHANDLER_INVOKE(swapoff, sp); 1739 } 1740 1741 /************************************************************************ 1742 * SWAP META DATA * 1743 ************************************************************************ 1744 * 1745 * These routines manipulate the swap metadata stored in the 1746 * OBJT_SWAP object. 1747 * 1748 * Swap metadata is implemented with a global hash and not directly 1749 * linked into the object. Instead the object simply contains 1750 * appropriate tracking counters. 1751 */ 1752 1753 /* 1754 * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object 1755 * 1756 * We first convert the object to a swap object if it is a default 1757 * object. 1758 * 1759 * The specified swapblk is added to the object's swap metadata. If 1760 * the swapblk is not valid, it is freed instead. Any previously 1761 * assigned swapblk is freed. 1762 */ 1763 static void 1764 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) 1765 { 1766 static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; 1767 struct swblk *sb, *sb1; 1768 vm_pindex_t modpi, rdpi; 1769 int error, i; 1770 1771 VM_OBJECT_ASSERT_WLOCKED(object); 1772 1773 /* 1774 * Convert default object to swap object if necessary 1775 */ 1776 if (object->type != OBJT_SWAP) { 1777 pctrie_init(&object->un_pager.swp.swp_blks); 1778 1779 /* 1780 * Ensure that swap_pager_swapoff()'s iteration over 1781 * object_list does not see a garbage pctrie. 1782 */ 1783 atomic_thread_fence_rel(); 1784 1785 object->type = OBJT_SWAP; 1786 KASSERT(object->handle == NULL, ("default pager with handle")); 1787 } 1788 1789 rdpi = rounddown(pindex, SWAP_META_PAGES); 1790 sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); 1791 if (sb == NULL) { 1792 if (swapblk == SWAPBLK_NONE) 1793 return; 1794 for (;;) { 1795 sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == 1796 pageproc ? M_USE_RESERVE : 0)); 1797 if (sb != NULL) { 1798 sb->p = rdpi; 1799 for (i = 0; i < SWAP_META_PAGES; i++) 1800 sb->d[i] = SWAPBLK_NONE; 1801 if (atomic_cmpset_int(&swblk_zone_exhausted, 1802 1, 0)) 1803 printf("swblk zone ok\n"); 1804 break; 1805 } 1806 VM_OBJECT_WUNLOCK(object); 1807 if (uma_zone_exhausted(swblk_zone)) { 1808 if (atomic_cmpset_int(&swblk_zone_exhausted, 1809 0, 1)) 1810 printf("swap blk zone exhausted, " 1811 "increase kern.maxswzone\n"); 1812 vm_pageout_oom(VM_OOM_SWAPZ); 1813 pause("swzonxb", 10); 1814 } else 1815 VM_WAIT; 1816 VM_OBJECT_WLOCK(object); 1817 sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, 1818 rdpi); 1819 if (sb != NULL) 1820 /* 1821 * Somebody swapped out a nearby page, 1822 * allocating swblk at the rdpi index, 1823 * while we dropped the object lock. 1824 */ 1825 goto allocated; 1826 } 1827 for (;;) { 1828 error = SWAP_PCTRIE_INSERT( 1829 &object->un_pager.swp.swp_blks, sb); 1830 if (error == 0) { 1831 if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1832 1, 0)) 1833 printf("swpctrie zone ok\n"); 1834 break; 1835 } 1836 VM_OBJECT_WUNLOCK(object); 1837 if (uma_zone_exhausted(swpctrie_zone)) { 1838 if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1839 0, 1)) 1840 printf("swap pctrie zone exhausted, " 1841 "increase kern.maxswzone\n"); 1842 vm_pageout_oom(VM_OOM_SWAPZ); 1843 pause("swzonxp", 10); 1844 } else 1845 VM_WAIT; 1846 VM_OBJECT_WLOCK(object); 1847 sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, 1848 rdpi); 1849 if (sb1 != NULL) { 1850 uma_zfree(swblk_zone, sb); 1851 sb = sb1; 1852 goto allocated; 1853 } 1854 } 1855 } 1856 allocated: 1857 MPASS(sb->p == rdpi); 1858 1859 modpi = pindex % SWAP_META_PAGES; 1860 /* Delete prior contents of metadata. */ 1861 if (sb->d[modpi] != SWAPBLK_NONE) 1862 swp_pager_freeswapspace(sb->d[modpi], 1); 1863 /* Enter block into metadata. */ 1864 sb->d[modpi] = swapblk; 1865 1866 /* 1867 * Free the swblk if we end up with the empty page run. 1868 */ 1869 if (swapblk == SWAPBLK_NONE) { 1870 for (i = 0; i < SWAP_META_PAGES; i++) { 1871 if (sb->d[i] != SWAPBLK_NONE) 1872 break; 1873 } 1874 if (i == SWAP_META_PAGES) { 1875 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, 1876 rdpi); 1877 uma_zfree(swblk_zone, sb); 1878 } 1879 } 1880 } 1881 1882 /* 1883 * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata 1884 * 1885 * The requested range of blocks is freed, with any associated swap 1886 * returned to the swap bitmap. 1887 * 1888 * This routine will free swap metadata structures as they are cleaned 1889 * out. This routine does *NOT* operate on swap metadata associated 1890 * with resident pages. 1891 */ 1892 static void 1893 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) 1894 { 1895 struct swblk *sb; 1896 vm_pindex_t last; 1897 int i; 1898 bool empty; 1899 1900 VM_OBJECT_ASSERT_WLOCKED(object); 1901 if (object->type != OBJT_SWAP || count == 0) 1902 return; 1903 1904 last = pindex + count - 1; 1905 for (;;) { 1906 sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, 1907 rounddown(pindex, SWAP_META_PAGES)); 1908 if (sb == NULL || sb->p > last) 1909 break; 1910 empty = true; 1911 for (i = 0; i < SWAP_META_PAGES; i++) { 1912 if (sb->d[i] == SWAPBLK_NONE) 1913 continue; 1914 if (pindex <= sb->p + i && sb->p + i <= last) { 1915 swp_pager_freeswapspace(sb->d[i], 1); 1916 sb->d[i] = SWAPBLK_NONE; 1917 } else 1918 empty = false; 1919 } 1920 pindex = sb->p + SWAP_META_PAGES; 1921 if (empty) { 1922 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, 1923 sb->p); 1924 uma_zfree(swblk_zone, sb); 1925 } 1926 } 1927 } 1928 1929 /* 1930 * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object 1931 * 1932 * This routine locates and destroys all swap metadata associated with 1933 * an object. 1934 */ 1935 static void 1936 swp_pager_meta_free_all(vm_object_t object) 1937 { 1938 struct swblk *sb; 1939 vm_pindex_t pindex; 1940 int i; 1941 1942 VM_OBJECT_ASSERT_WLOCKED(object); 1943 if (object->type != OBJT_SWAP) 1944 return; 1945 1946 for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( 1947 &object->un_pager.swp.swp_blks, pindex)) != NULL;) { 1948 pindex = sb->p + SWAP_META_PAGES; 1949 for (i = 0; i < SWAP_META_PAGES; i++) { 1950 if (sb->d[i] != SWAPBLK_NONE) 1951 swp_pager_freeswapspace(sb->d[i], 1); 1952 } 1953 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); 1954 uma_zfree(swblk_zone, sb); 1955 } 1956 } 1957 1958 /* 1959 * SWP_PAGER_METACTL() - misc control of swap and vm_page_t meta data. 1960 * 1961 * This routine is capable of looking up, popping, or freeing 1962 * swapblk assignments in the swap meta data or in the vm_page_t. 1963 * The routine typically returns the swapblk being looked-up, or popped, 1964 * or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block 1965 * was invalid. This routine will automatically free any invalid 1966 * meta-data swapblks. 1967 * 1968 * When acting on a busy resident page and paging is in progress, we 1969 * have to wait until paging is complete but otherwise can act on the 1970 * busy page. 1971 * 1972 * SWM_FREE remove and free swap block from metadata 1973 * SWM_POP remove from meta data but do not free.. pop it out 1974 */ 1975 static daddr_t 1976 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags) 1977 { 1978 struct swblk *sb; 1979 daddr_t r1; 1980 int i; 1981 1982 if ((flags & (SWM_FREE | SWM_POP)) != 0) 1983 VM_OBJECT_ASSERT_WLOCKED(object); 1984 else 1985 VM_OBJECT_ASSERT_LOCKED(object); 1986 1987 /* 1988 * The meta data only exists if the object is OBJT_SWAP 1989 * and even then might not be allocated yet. 1990 */ 1991 if (object->type != OBJT_SWAP) 1992 return (SWAPBLK_NONE); 1993 1994 sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, 1995 rounddown(pindex, SWAP_META_PAGES)); 1996 if (sb == NULL) 1997 return (SWAPBLK_NONE); 1998 r1 = sb->d[pindex % SWAP_META_PAGES]; 1999 if (r1 == SWAPBLK_NONE) 2000 return (SWAPBLK_NONE); 2001 if ((flags & (SWM_FREE | SWM_POP)) != 0) { 2002 sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; 2003 for (i = 0; i < SWAP_META_PAGES; i++) { 2004 if (sb->d[i] != SWAPBLK_NONE) 2005 break; 2006 } 2007 if (i == SWAP_META_PAGES) { 2008 SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, 2009 rounddown(pindex, SWAP_META_PAGES)); 2010 uma_zfree(swblk_zone, sb); 2011 } 2012 } 2013 if ((flags & SWM_FREE) != 0) { 2014 swp_pager_freeswapspace(r1, 1); 2015 r1 = SWAPBLK_NONE; 2016 } 2017 return (r1); 2018 } 2019 2020 /* 2021 * Returns the least page index which is greater than or equal to the 2022 * parameter pindex and for which there is a swap block allocated. 2023 * Returns object's size if the object's type is not swap or if there 2024 * are no allocated swap blocks for the object after the requested 2025 * pindex. 2026 */ 2027 vm_pindex_t 2028 swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) 2029 { 2030 struct swblk *sb; 2031 int i; 2032 2033 VM_OBJECT_ASSERT_LOCKED(object); 2034 if (object->type != OBJT_SWAP) 2035 return (object->size); 2036 2037 sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, 2038 rounddown(pindex, SWAP_META_PAGES)); 2039 if (sb == NULL) 2040 return (object->size); 2041 if (sb->p < pindex) { 2042 for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { 2043 if (sb->d[i] != SWAPBLK_NONE) 2044 return (sb->p + i); 2045 } 2046 sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, 2047 roundup(pindex, SWAP_META_PAGES)); 2048 if (sb == NULL) 2049 return (object->size); 2050 } 2051 for (i = 0; i < SWAP_META_PAGES; i++) { 2052 if (sb->d[i] != SWAPBLK_NONE) 2053 return (sb->p + i); 2054 } 2055 2056 /* 2057 * We get here if a swblk is present in the trie but it 2058 * doesn't map any blocks. 2059 */ 2060 MPASS(0); 2061 return (object->size); 2062 } 2063 2064 /* 2065 * System call swapon(name) enables swapping on device name, 2066 * which must be in the swdevsw. Return EBUSY 2067 * if already swapping on this device. 2068 */ 2069 #ifndef _SYS_SYSPROTO_H_ 2070 struct swapon_args { 2071 char *name; 2072 }; 2073 #endif 2074 2075 /* 2076 * MPSAFE 2077 */ 2078 /* ARGSUSED */ 2079 int 2080 sys_swapon(struct thread *td, struct swapon_args *uap) 2081 { 2082 struct vattr attr; 2083 struct vnode *vp; 2084 struct nameidata nd; 2085 int error; 2086 2087 error = priv_check(td, PRIV_SWAPON); 2088 if (error) 2089 return (error); 2090 2091 sx_xlock(&swdev_syscall_lock); 2092 2093 /* 2094 * Swap metadata may not fit in the KVM if we have physical 2095 * memory of >1GB. 2096 */ 2097 if (swblk_zone == NULL) { 2098 error = ENOMEM; 2099 goto done; 2100 } 2101 2102 NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE, 2103 uap->name, td); 2104 error = namei(&nd); 2105 if (error) 2106 goto done; 2107 2108 NDFREE(&nd, NDF_ONLY_PNBUF); 2109 vp = nd.ni_vp; 2110 2111 if (vn_isdisk(vp, &error)) { 2112 error = swapongeom(vp); 2113 } else if (vp->v_type == VREG && 2114 (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 && 2115 (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) { 2116 /* 2117 * Allow direct swapping to NFS regular files in the same 2118 * way that nfs_mountroot() sets up diskless swapping. 2119 */ 2120 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE); 2121 } 2122 2123 if (error) 2124 vrele(vp); 2125 done: 2126 sx_xunlock(&swdev_syscall_lock); 2127 return (error); 2128 } 2129 2130 /* 2131 * Check that the total amount of swap currently configured does not 2132 * exceed half the theoretical maximum. If it does, print a warning 2133 * message. 2134 */ 2135 static void 2136 swapon_check_swzone(void) 2137 { 2138 unsigned long maxpages, npages; 2139 2140 npages = swap_total / PAGE_SIZE; 2141 /* absolute maximum we can handle assuming 100% efficiency */ 2142 maxpages = uma_zone_get_max(swblk_zone) * SWAP_META_PAGES; 2143 2144 /* recommend using no more than half that amount */ 2145 if (npages > maxpages / 2) { 2146 printf("warning: total configured swap (%lu pages) " 2147 "exceeds maximum recommended amount (%lu pages).\n", 2148 npages, maxpages / 2); 2149 printf("warning: increase kern.maxswzone " 2150 "or reduce amount of swap.\n"); 2151 } 2152 } 2153 2154 static void 2155 swaponsomething(struct vnode *vp, void *id, u_long nblks, 2156 sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags) 2157 { 2158 struct swdevt *sp, *tsp; 2159 swblk_t dvbase; 2160 u_long mblocks; 2161 2162 /* 2163 * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. 2164 * First chop nblks off to page-align it, then convert. 2165 * 2166 * sw->sw_nblks is in page-sized chunks now too. 2167 */ 2168 nblks &= ~(ctodb(1) - 1); 2169 nblks = dbtoc(nblks); 2170 2171 /* 2172 * If we go beyond this, we get overflows in the radix 2173 * tree bitmap code. 2174 */ 2175 mblocks = 0x40000000 / BLIST_META_RADIX; 2176 if (nblks > mblocks) { 2177 printf( 2178 "WARNING: reducing swap size to maximum of %luMB per unit\n", 2179 mblocks / 1024 / 1024 * PAGE_SIZE); 2180 nblks = mblocks; 2181 } 2182 2183 sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO); 2184 sp->sw_vp = vp; 2185 sp->sw_id = id; 2186 sp->sw_dev = dev; 2187 sp->sw_flags = 0; 2188 sp->sw_nblks = nblks; 2189 sp->sw_used = 0; 2190 sp->sw_strategy = strategy; 2191 sp->sw_close = close; 2192 sp->sw_flags = flags; 2193 2194 sp->sw_blist = blist_create(nblks, M_WAITOK); 2195 /* 2196 * Do not free the first two block in order to avoid overwriting 2197 * any bsd label at the front of the partition 2198 */ 2199 blist_free(sp->sw_blist, 2, nblks - 2); 2200 2201 dvbase = 0; 2202 mtx_lock(&sw_dev_mtx); 2203 TAILQ_FOREACH(tsp, &swtailq, sw_list) { 2204 if (tsp->sw_end >= dvbase) { 2205 /* 2206 * We put one uncovered page between the devices 2207 * in order to definitively prevent any cross-device 2208 * I/O requests 2209 */ 2210 dvbase = tsp->sw_end + 1; 2211 } 2212 } 2213 sp->sw_first = dvbase; 2214 sp->sw_end = dvbase + nblks; 2215 TAILQ_INSERT_TAIL(&swtailq, sp, sw_list); 2216 nswapdev++; 2217 swap_pager_avail += nblks - 2; 2218 swap_total += (vm_ooffset_t)nblks * PAGE_SIZE; 2219 swapon_check_swzone(); 2220 swp_sizecheck(); 2221 mtx_unlock(&sw_dev_mtx); 2222 EVENTHANDLER_INVOKE(swapon, sp); 2223 } 2224 2225 /* 2226 * SYSCALL: swapoff(devname) 2227 * 2228 * Disable swapping on the given device. 2229 * 2230 * XXX: Badly designed system call: it should use a device index 2231 * rather than filename as specification. We keep sw_vp around 2232 * only to make this work. 2233 */ 2234 #ifndef _SYS_SYSPROTO_H_ 2235 struct swapoff_args { 2236 char *name; 2237 }; 2238 #endif 2239 2240 /* 2241 * MPSAFE 2242 */ 2243 /* ARGSUSED */ 2244 int 2245 sys_swapoff(struct thread *td, struct swapoff_args *uap) 2246 { 2247 struct vnode *vp; 2248 struct nameidata nd; 2249 struct swdevt *sp; 2250 int error; 2251 2252 error = priv_check(td, PRIV_SWAPOFF); 2253 if (error) 2254 return (error); 2255 2256 sx_xlock(&swdev_syscall_lock); 2257 2258 NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name, 2259 td); 2260 error = namei(&nd); 2261 if (error) 2262 goto done; 2263 NDFREE(&nd, NDF_ONLY_PNBUF); 2264 vp = nd.ni_vp; 2265 2266 mtx_lock(&sw_dev_mtx); 2267 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2268 if (sp->sw_vp == vp) 2269 break; 2270 } 2271 mtx_unlock(&sw_dev_mtx); 2272 if (sp == NULL) { 2273 error = EINVAL; 2274 goto done; 2275 } 2276 error = swapoff_one(sp, td->td_ucred); 2277 done: 2278 sx_xunlock(&swdev_syscall_lock); 2279 return (error); 2280 } 2281 2282 static int 2283 swapoff_one(struct swdevt *sp, struct ucred *cred) 2284 { 2285 u_long nblks; 2286 #ifdef MAC 2287 int error; 2288 #endif 2289 2290 sx_assert(&swdev_syscall_lock, SA_XLOCKED); 2291 #ifdef MAC 2292 (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY); 2293 error = mac_system_check_swapoff(cred, sp->sw_vp); 2294 (void) VOP_UNLOCK(sp->sw_vp, 0); 2295 if (error != 0) 2296 return (error); 2297 #endif 2298 nblks = sp->sw_nblks; 2299 2300 /* 2301 * We can turn off this swap device safely only if the 2302 * available virtual memory in the system will fit the amount 2303 * of data we will have to page back in, plus an epsilon so 2304 * the system doesn't become critically low on swap space. 2305 */ 2306 if (vm_cnt.v_free_count + swap_pager_avail < nblks + nswap_lowat) 2307 return (ENOMEM); 2308 2309 /* 2310 * Prevent further allocations on this device. 2311 */ 2312 mtx_lock(&sw_dev_mtx); 2313 sp->sw_flags |= SW_CLOSING; 2314 swap_pager_avail -= blist_fill(sp->sw_blist, 0, nblks); 2315 swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE; 2316 mtx_unlock(&sw_dev_mtx); 2317 2318 /* 2319 * Page in the contents of the device and close it. 2320 */ 2321 swap_pager_swapoff(sp); 2322 2323 sp->sw_close(curthread, sp); 2324 mtx_lock(&sw_dev_mtx); 2325 sp->sw_id = NULL; 2326 TAILQ_REMOVE(&swtailq, sp, sw_list); 2327 nswapdev--; 2328 if (nswapdev == 0) { 2329 swap_pager_full = 2; 2330 swap_pager_almost_full = 1; 2331 } 2332 if (swdevhd == sp) 2333 swdevhd = NULL; 2334 mtx_unlock(&sw_dev_mtx); 2335 blist_destroy(sp->sw_blist); 2336 free(sp, M_VMPGDATA); 2337 return (0); 2338 } 2339 2340 void 2341 swapoff_all(void) 2342 { 2343 struct swdevt *sp, *spt; 2344 const char *devname; 2345 int error; 2346 2347 sx_xlock(&swdev_syscall_lock); 2348 2349 mtx_lock(&sw_dev_mtx); 2350 TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) { 2351 mtx_unlock(&sw_dev_mtx); 2352 if (vn_isdisk(sp->sw_vp, NULL)) 2353 devname = devtoname(sp->sw_vp->v_rdev); 2354 else 2355 devname = "[file]"; 2356 error = swapoff_one(sp, thread0.td_ucred); 2357 if (error != 0) { 2358 printf("Cannot remove swap device %s (error=%d), " 2359 "skipping.\n", devname, error); 2360 } else if (bootverbose) { 2361 printf("Swap device %s removed.\n", devname); 2362 } 2363 mtx_lock(&sw_dev_mtx); 2364 } 2365 mtx_unlock(&sw_dev_mtx); 2366 2367 sx_xunlock(&swdev_syscall_lock); 2368 } 2369 2370 void 2371 swap_pager_status(int *total, int *used) 2372 { 2373 struct swdevt *sp; 2374 2375 *total = 0; 2376 *used = 0; 2377 mtx_lock(&sw_dev_mtx); 2378 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2379 *total += sp->sw_nblks; 2380 *used += sp->sw_used; 2381 } 2382 mtx_unlock(&sw_dev_mtx); 2383 } 2384 2385 int 2386 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len) 2387 { 2388 struct swdevt *sp; 2389 const char *tmp_devname; 2390 int error, n; 2391 2392 n = 0; 2393 error = ENOENT; 2394 mtx_lock(&sw_dev_mtx); 2395 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2396 if (n != name) { 2397 n++; 2398 continue; 2399 } 2400 xs->xsw_version = XSWDEV_VERSION; 2401 xs->xsw_dev = sp->sw_dev; 2402 xs->xsw_flags = sp->sw_flags; 2403 xs->xsw_nblks = sp->sw_nblks; 2404 xs->xsw_used = sp->sw_used; 2405 if (devname != NULL) { 2406 if (vn_isdisk(sp->sw_vp, NULL)) 2407 tmp_devname = devtoname(sp->sw_vp->v_rdev); 2408 else 2409 tmp_devname = "[file]"; 2410 strncpy(devname, tmp_devname, len); 2411 } 2412 error = 0; 2413 break; 2414 } 2415 mtx_unlock(&sw_dev_mtx); 2416 return (error); 2417 } 2418 2419 #if defined(COMPAT_FREEBSD11) 2420 #define XSWDEV_VERSION_11 1 2421 struct xswdev11 { 2422 u_int xsw_version; 2423 uint32_t xsw_dev; 2424 int xsw_flags; 2425 int xsw_nblks; 2426 int xsw_used; 2427 }; 2428 #endif 2429 2430 static int 2431 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS) 2432 { 2433 struct xswdev xs; 2434 #if defined(COMPAT_FREEBSD11) 2435 struct xswdev11 xs11; 2436 #endif 2437 int error; 2438 2439 if (arg2 != 1) /* name length */ 2440 return (EINVAL); 2441 error = swap_dev_info(*(int *)arg1, &xs, NULL, 0); 2442 if (error != 0) 2443 return (error); 2444 #if defined(COMPAT_FREEBSD11) 2445 if (req->oldlen == sizeof(xs11)) { 2446 xs11.xsw_version = XSWDEV_VERSION_11; 2447 xs11.xsw_dev = xs.xsw_dev; /* truncation */ 2448 xs11.xsw_flags = xs.xsw_flags; 2449 xs11.xsw_nblks = xs.xsw_nblks; 2450 xs11.xsw_used = xs.xsw_used; 2451 error = SYSCTL_OUT(req, &xs11, sizeof(xs11)); 2452 } else 2453 #endif 2454 error = SYSCTL_OUT(req, &xs, sizeof(xs)); 2455 return (error); 2456 } 2457 2458 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0, 2459 "Number of swap devices"); 2460 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD | CTLFLAG_MPSAFE, 2461 sysctl_vm_swap_info, 2462 "Swap statistics by device"); 2463 2464 /* 2465 * Count the approximate swap usage in pages for a vmspace. The 2466 * shadowed or not yet copied on write swap blocks are not accounted. 2467 * The map must be locked. 2468 */ 2469 long 2470 vmspace_swap_count(struct vmspace *vmspace) 2471 { 2472 vm_map_t map; 2473 vm_map_entry_t cur; 2474 vm_object_t object; 2475 struct swblk *sb; 2476 vm_pindex_t e, pi; 2477 long count; 2478 int i; 2479 2480 map = &vmspace->vm_map; 2481 count = 0; 2482 2483 for (cur = map->header.next; cur != &map->header; cur = cur->next) { 2484 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) 2485 continue; 2486 object = cur->object.vm_object; 2487 if (object == NULL || object->type != OBJT_SWAP) 2488 continue; 2489 VM_OBJECT_RLOCK(object); 2490 if (object->type != OBJT_SWAP) 2491 goto unlock; 2492 pi = OFF_TO_IDX(cur->offset); 2493 e = pi + OFF_TO_IDX(cur->end - cur->start); 2494 for (;; pi = sb->p + SWAP_META_PAGES) { 2495 sb = SWAP_PCTRIE_LOOKUP_GE( 2496 &object->un_pager.swp.swp_blks, pi); 2497 if (sb == NULL || sb->p >= e) 2498 break; 2499 for (i = 0; i < SWAP_META_PAGES; i++) { 2500 if (sb->p + i < e && 2501 sb->d[i] != SWAPBLK_NONE) 2502 count++; 2503 } 2504 } 2505 unlock: 2506 VM_OBJECT_RUNLOCK(object); 2507 } 2508 return (count); 2509 } 2510 2511 /* 2512 * GEOM backend 2513 * 2514 * Swapping onto disk devices. 2515 * 2516 */ 2517 2518 static g_orphan_t swapgeom_orphan; 2519 2520 static struct g_class g_swap_class = { 2521 .name = "SWAP", 2522 .version = G_VERSION, 2523 .orphan = swapgeom_orphan, 2524 }; 2525 2526 DECLARE_GEOM_CLASS(g_swap_class, g_class); 2527 2528 2529 static void 2530 swapgeom_close_ev(void *arg, int flags) 2531 { 2532 struct g_consumer *cp; 2533 2534 cp = arg; 2535 g_access(cp, -1, -1, 0); 2536 g_detach(cp); 2537 g_destroy_consumer(cp); 2538 } 2539 2540 /* 2541 * Add a reference to the g_consumer for an inflight transaction. 2542 */ 2543 static void 2544 swapgeom_acquire(struct g_consumer *cp) 2545 { 2546 2547 mtx_assert(&sw_dev_mtx, MA_OWNED); 2548 cp->index++; 2549 } 2550 2551 /* 2552 * Remove a reference from the g_consumer. Post a close event if all 2553 * references go away, since the function might be called from the 2554 * biodone context. 2555 */ 2556 static void 2557 swapgeom_release(struct g_consumer *cp, struct swdevt *sp) 2558 { 2559 2560 mtx_assert(&sw_dev_mtx, MA_OWNED); 2561 cp->index--; 2562 if (cp->index == 0) { 2563 if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0) 2564 sp->sw_id = NULL; 2565 } 2566 } 2567 2568 static void 2569 swapgeom_done(struct bio *bp2) 2570 { 2571 struct swdevt *sp; 2572 struct buf *bp; 2573 struct g_consumer *cp; 2574 2575 bp = bp2->bio_caller2; 2576 cp = bp2->bio_from; 2577 bp->b_ioflags = bp2->bio_flags; 2578 if (bp2->bio_error) 2579 bp->b_ioflags |= BIO_ERROR; 2580 bp->b_resid = bp->b_bcount - bp2->bio_completed; 2581 bp->b_error = bp2->bio_error; 2582 bufdone(bp); 2583 sp = bp2->bio_caller1; 2584 mtx_lock(&sw_dev_mtx); 2585 swapgeom_release(cp, sp); 2586 mtx_unlock(&sw_dev_mtx); 2587 g_destroy_bio(bp2); 2588 } 2589 2590 static void 2591 swapgeom_strategy(struct buf *bp, struct swdevt *sp) 2592 { 2593 struct bio *bio; 2594 struct g_consumer *cp; 2595 2596 mtx_lock(&sw_dev_mtx); 2597 cp = sp->sw_id; 2598 if (cp == NULL) { 2599 mtx_unlock(&sw_dev_mtx); 2600 bp->b_error = ENXIO; 2601 bp->b_ioflags |= BIO_ERROR; 2602 bufdone(bp); 2603 return; 2604 } 2605 swapgeom_acquire(cp); 2606 mtx_unlock(&sw_dev_mtx); 2607 if (bp->b_iocmd == BIO_WRITE) 2608 bio = g_new_bio(); 2609 else 2610 bio = g_alloc_bio(); 2611 if (bio == NULL) { 2612 mtx_lock(&sw_dev_mtx); 2613 swapgeom_release(cp, sp); 2614 mtx_unlock(&sw_dev_mtx); 2615 bp->b_error = ENOMEM; 2616 bp->b_ioflags |= BIO_ERROR; 2617 bufdone(bp); 2618 return; 2619 } 2620 2621 bio->bio_caller1 = sp; 2622 bio->bio_caller2 = bp; 2623 bio->bio_cmd = bp->b_iocmd; 2624 bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE; 2625 bio->bio_length = bp->b_bcount; 2626 bio->bio_done = swapgeom_done; 2627 if (!buf_mapped(bp)) { 2628 bio->bio_ma = bp->b_pages; 2629 bio->bio_data = unmapped_buf; 2630 bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK; 2631 bio->bio_ma_n = bp->b_npages; 2632 bio->bio_flags |= BIO_UNMAPPED; 2633 } else { 2634 bio->bio_data = bp->b_data; 2635 bio->bio_ma = NULL; 2636 } 2637 g_io_request(bio, cp); 2638 return; 2639 } 2640 2641 static void 2642 swapgeom_orphan(struct g_consumer *cp) 2643 { 2644 struct swdevt *sp; 2645 int destroy; 2646 2647 mtx_lock(&sw_dev_mtx); 2648 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2649 if (sp->sw_id == cp) { 2650 sp->sw_flags |= SW_CLOSING; 2651 break; 2652 } 2653 } 2654 /* 2655 * Drop reference we were created with. Do directly since we're in a 2656 * special context where we don't have to queue the call to 2657 * swapgeom_close_ev(). 2658 */ 2659 cp->index--; 2660 destroy = ((sp != NULL) && (cp->index == 0)); 2661 if (destroy) 2662 sp->sw_id = NULL; 2663 mtx_unlock(&sw_dev_mtx); 2664 if (destroy) 2665 swapgeom_close_ev(cp, 0); 2666 } 2667 2668 static void 2669 swapgeom_close(struct thread *td, struct swdevt *sw) 2670 { 2671 struct g_consumer *cp; 2672 2673 mtx_lock(&sw_dev_mtx); 2674 cp = sw->sw_id; 2675 sw->sw_id = NULL; 2676 mtx_unlock(&sw_dev_mtx); 2677 2678 /* 2679 * swapgeom_close() may be called from the biodone context, 2680 * where we cannot perform topology changes. Delegate the 2681 * work to the events thread. 2682 */ 2683 if (cp != NULL) 2684 g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL); 2685 } 2686 2687 static int 2688 swapongeom_locked(struct cdev *dev, struct vnode *vp) 2689 { 2690 struct g_provider *pp; 2691 struct g_consumer *cp; 2692 static struct g_geom *gp; 2693 struct swdevt *sp; 2694 u_long nblks; 2695 int error; 2696 2697 pp = g_dev_getprovider(dev); 2698 if (pp == NULL) 2699 return (ENODEV); 2700 mtx_lock(&sw_dev_mtx); 2701 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2702 cp = sp->sw_id; 2703 if (cp != NULL && cp->provider == pp) { 2704 mtx_unlock(&sw_dev_mtx); 2705 return (EBUSY); 2706 } 2707 } 2708 mtx_unlock(&sw_dev_mtx); 2709 if (gp == NULL) 2710 gp = g_new_geomf(&g_swap_class, "swap"); 2711 cp = g_new_consumer(gp); 2712 cp->index = 1; /* Number of active I/Os, plus one for being active. */ 2713 cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; 2714 g_attach(cp, pp); 2715 /* 2716 * XXX: Every time you think you can improve the margin for 2717 * footshooting, somebody depends on the ability to do so: 2718 * savecore(8) wants to write to our swapdev so we cannot 2719 * set an exclusive count :-( 2720 */ 2721 error = g_access(cp, 1, 1, 0); 2722 if (error != 0) { 2723 g_detach(cp); 2724 g_destroy_consumer(cp); 2725 return (error); 2726 } 2727 nblks = pp->mediasize / DEV_BSIZE; 2728 swaponsomething(vp, cp, nblks, swapgeom_strategy, 2729 swapgeom_close, dev2udev(dev), 2730 (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0); 2731 return (0); 2732 } 2733 2734 static int 2735 swapongeom(struct vnode *vp) 2736 { 2737 int error; 2738 2739 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2740 if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) { 2741 error = ENOENT; 2742 } else { 2743 g_topology_lock(); 2744 error = swapongeom_locked(vp->v_rdev, vp); 2745 g_topology_unlock(); 2746 } 2747 VOP_UNLOCK(vp, 0); 2748 return (error); 2749 } 2750 2751 /* 2752 * VNODE backend 2753 * 2754 * This is used mainly for network filesystem (read: probably only tested 2755 * with NFS) swapfiles. 2756 * 2757 */ 2758 2759 static void 2760 swapdev_strategy(struct buf *bp, struct swdevt *sp) 2761 { 2762 struct vnode *vp2; 2763 2764 bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first); 2765 2766 vp2 = sp->sw_id; 2767 vhold(vp2); 2768 if (bp->b_iocmd == BIO_WRITE) { 2769 if (bp->b_bufobj) 2770 bufobj_wdrop(bp->b_bufobj); 2771 bufobj_wref(&vp2->v_bufobj); 2772 } 2773 if (bp->b_bufobj != &vp2->v_bufobj) 2774 bp->b_bufobj = &vp2->v_bufobj; 2775 bp->b_vp = vp2; 2776 bp->b_iooffset = dbtob(bp->b_blkno); 2777 bstrategy(bp); 2778 return; 2779 } 2780 2781 static void 2782 swapdev_close(struct thread *td, struct swdevt *sp) 2783 { 2784 2785 VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td); 2786 vrele(sp->sw_vp); 2787 } 2788 2789 2790 static int 2791 swaponvp(struct thread *td, struct vnode *vp, u_long nblks) 2792 { 2793 struct swdevt *sp; 2794 int error; 2795 2796 if (nblks == 0) 2797 return (ENXIO); 2798 mtx_lock(&sw_dev_mtx); 2799 TAILQ_FOREACH(sp, &swtailq, sw_list) { 2800 if (sp->sw_id == vp) { 2801 mtx_unlock(&sw_dev_mtx); 2802 return (EBUSY); 2803 } 2804 } 2805 mtx_unlock(&sw_dev_mtx); 2806 2807 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 2808 #ifdef MAC 2809 error = mac_system_check_swapon(td->td_ucred, vp); 2810 if (error == 0) 2811 #endif 2812 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL); 2813 (void) VOP_UNLOCK(vp, 0); 2814 if (error) 2815 return (error); 2816 2817 swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close, 2818 NODEV, 0); 2819 return (0); 2820 } 2821 2822 static int 2823 sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) 2824 { 2825 int error, new, n; 2826 2827 new = nsw_wcount_async_max; 2828 error = sysctl_handle_int(oidp, &new, 0, req); 2829 if (error != 0 || req->newptr == NULL) 2830 return (error); 2831 2832 if (new > nswbuf / 2 || new < 1) 2833 return (EINVAL); 2834 2835 mtx_lock(&pbuf_mtx); 2836 while (nsw_wcount_async_max != new) { 2837 /* 2838 * Adjust difference. If the current async count is too low, 2839 * we will need to sqeeze our update slowly in. Sleep with a 2840 * higher priority than getpbuf() to finish faster. 2841 */ 2842 n = new - nsw_wcount_async_max; 2843 if (nsw_wcount_async + n >= 0) { 2844 nsw_wcount_async += n; 2845 nsw_wcount_async_max += n; 2846 wakeup(&nsw_wcount_async); 2847 } else { 2848 nsw_wcount_async_max -= nsw_wcount_async; 2849 nsw_wcount_async = 0; 2850 msleep(&nsw_wcount_async, &pbuf_mtx, PSWP, 2851 "swpsysctl", 0); 2852 } 2853 } 2854 mtx_unlock(&pbuf_mtx); 2855 2856 return (0); 2857 } 2858