1 /*- 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 33 * 34 * 35 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 36 * All rights reserved. 37 * 38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 39 * 40 * Permission to use, copy, modify and distribute this software and 41 * its documentation is hereby granted, provided that both the copyright 42 * notice and this permission notice appear in all copies of the 43 * software, derivative works or modified versions, and any portions 44 * thereof, and that both notices appear in supporting documentation. 45 * 46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 49 * 50 * Carnegie Mellon requests users of this software to return to 51 * 52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 53 * School of Computer Science 54 * Carnegie Mellon University 55 * Pittsburgh PA 15213-3890 56 * 57 * any improvements or extensions that they make and grant Carnegie the 58 * rights to redistribute these changes. 59 */ 60 61 /* 62 * Virtual memory object module. 63 */ 64 65 #include <sys/cdefs.h> 66 __FBSDID("$FreeBSD$"); 67 68 #include "opt_vm.h" 69 70 #include <sys/param.h> 71 #include <sys/systm.h> 72 #include <sys/lock.h> 73 #include <sys/mman.h> 74 #include <sys/mount.h> 75 #include <sys/kernel.h> 76 #include <sys/sysctl.h> 77 #include <sys/mutex.h> 78 #include <sys/proc.h> /* for curproc, pageproc */ 79 #include <sys/socket.h> 80 #include <sys/resourcevar.h> 81 #include <sys/vnode.h> 82 #include <sys/vmmeter.h> 83 #include <sys/sx.h> 84 85 #include <vm/vm.h> 86 #include <vm/vm_param.h> 87 #include <vm/pmap.h> 88 #include <vm/vm_map.h> 89 #include <vm/vm_object.h> 90 #include <vm/vm_page.h> 91 #include <vm/vm_pageout.h> 92 #include <vm/vm_pager.h> 93 #include <vm/swap_pager.h> 94 #include <vm/vm_kern.h> 95 #include <vm/vm_extern.h> 96 #include <vm/vm_reserv.h> 97 #include <vm/uma.h> 98 99 static int old_msync; 100 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0, 101 "Use old (insecure) msync behavior"); 102 103 static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, 104 int pagerflags); 105 static void vm_object_qcollapse(vm_object_t object); 106 static void vm_object_vndeallocate(vm_object_t object); 107 108 /* 109 * Virtual memory objects maintain the actual data 110 * associated with allocated virtual memory. A given 111 * page of memory exists within exactly one object. 112 * 113 * An object is only deallocated when all "references" 114 * are given up. Only one "reference" to a given 115 * region of an object should be writeable. 116 * 117 * Associated with each object is a list of all resident 118 * memory pages belonging to that object; this list is 119 * maintained by the "vm_page" module, and locked by the object's 120 * lock. 121 * 122 * Each object also records a "pager" routine which is 123 * used to retrieve (and store) pages to the proper backing 124 * storage. In addition, objects may be backed by other 125 * objects from which they were virtual-copied. 126 * 127 * The only items within the object structure which are 128 * modified after time of creation are: 129 * reference count locked by object's lock 130 * pager routine locked by object's lock 131 * 132 */ 133 134 struct object_q vm_object_list; 135 struct mtx vm_object_list_mtx; /* lock for object list and count */ 136 137 struct vm_object kernel_object_store; 138 struct vm_object kmem_object_store; 139 140 SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0, "VM object stats"); 141 142 static long object_collapses; 143 SYSCTL_LONG(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD, 144 &object_collapses, 0, "VM object collapses"); 145 146 static long object_bypasses; 147 SYSCTL_LONG(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD, 148 &object_bypasses, 0, "VM object bypasses"); 149 150 static uma_zone_t obj_zone; 151 152 static int vm_object_zinit(void *mem, int size, int flags); 153 154 #ifdef INVARIANTS 155 static void vm_object_zdtor(void *mem, int size, void *arg); 156 157 static void 158 vm_object_zdtor(void *mem, int size, void *arg) 159 { 160 vm_object_t object; 161 162 object = (vm_object_t)mem; 163 KASSERT(TAILQ_EMPTY(&object->memq), 164 ("object %p has resident pages", 165 object)); 166 #if VM_NRESERVLEVEL > 0 167 KASSERT(LIST_EMPTY(&object->rvq), 168 ("object %p has reservations", 169 object)); 170 #endif 171 KASSERT(object->cache == NULL, 172 ("object %p has cached pages", 173 object)); 174 KASSERT(object->paging_in_progress == 0, 175 ("object %p paging_in_progress = %d", 176 object, object->paging_in_progress)); 177 KASSERT(object->resident_page_count == 0, 178 ("object %p resident_page_count = %d", 179 object, object->resident_page_count)); 180 KASSERT(object->shadow_count == 0, 181 ("object %p shadow_count = %d", 182 object, object->shadow_count)); 183 } 184 #endif 185 186 static int 187 vm_object_zinit(void *mem, int size, int flags) 188 { 189 vm_object_t object; 190 191 object = (vm_object_t)mem; 192 bzero(&object->mtx, sizeof(object->mtx)); 193 VM_OBJECT_LOCK_INIT(object, "standard object"); 194 195 /* These are true for any object that has been freed */ 196 object->paging_in_progress = 0; 197 object->resident_page_count = 0; 198 object->shadow_count = 0; 199 return (0); 200 } 201 202 void 203 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 204 { 205 206 TAILQ_INIT(&object->memq); 207 LIST_INIT(&object->shadow_head); 208 209 object->root = NULL; 210 object->type = type; 211 object->size = size; 212 object->generation = 1; 213 object->ref_count = 1; 214 object->memattr = VM_MEMATTR_DEFAULT; 215 object->flags = 0; 216 object->uip = NULL; 217 object->charge = 0; 218 if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP)) 219 object->flags = OBJ_ONEMAPPING; 220 object->pg_color = 0; 221 object->handle = NULL; 222 object->backing_object = NULL; 223 object->backing_object_offset = (vm_ooffset_t) 0; 224 #if VM_NRESERVLEVEL > 0 225 LIST_INIT(&object->rvq); 226 #endif 227 object->cache = NULL; 228 229 mtx_lock(&vm_object_list_mtx); 230 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 231 mtx_unlock(&vm_object_list_mtx); 232 } 233 234 /* 235 * vm_object_init: 236 * 237 * Initialize the VM objects module. 238 */ 239 void 240 vm_object_init(void) 241 { 242 TAILQ_INIT(&vm_object_list); 243 mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); 244 245 VM_OBJECT_LOCK_INIT(&kernel_object_store, "kernel object"); 246 _vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), 247 kernel_object); 248 #if VM_NRESERVLEVEL > 0 249 kernel_object->flags |= OBJ_COLORED; 250 kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); 251 #endif 252 253 VM_OBJECT_LOCK_INIT(&kmem_object_store, "kmem object"); 254 _vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), 255 kmem_object); 256 #if VM_NRESERVLEVEL > 0 257 kmem_object->flags |= OBJ_COLORED; 258 kmem_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); 259 #endif 260 261 /* 262 * The lock portion of struct vm_object must be type stable due 263 * to vm_pageout_fallback_object_lock locking a vm object 264 * without holding any references to it. 265 */ 266 obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL, 267 #ifdef INVARIANTS 268 vm_object_zdtor, 269 #else 270 NULL, 271 #endif 272 vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); 273 } 274 275 void 276 vm_object_clear_flag(vm_object_t object, u_short bits) 277 { 278 279 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 280 object->flags &= ~bits; 281 } 282 283 /* 284 * Sets the default memory attribute for the specified object. Pages 285 * that are allocated to this object are by default assigned this memory 286 * attribute. 287 * 288 * Presently, this function must be called before any pages are allocated 289 * to the object. In the future, this requirement may be relaxed for 290 * "default" and "swap" objects. 291 */ 292 int 293 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr) 294 { 295 296 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 297 switch (object->type) { 298 case OBJT_DEFAULT: 299 case OBJT_DEVICE: 300 case OBJT_PHYS: 301 case OBJT_SG: 302 case OBJT_SWAP: 303 case OBJT_VNODE: 304 if (!TAILQ_EMPTY(&object->memq)) 305 return (KERN_FAILURE); 306 break; 307 case OBJT_DEAD: 308 return (KERN_INVALID_ARGUMENT); 309 } 310 object->memattr = memattr; 311 return (KERN_SUCCESS); 312 } 313 314 void 315 vm_object_pip_add(vm_object_t object, short i) 316 { 317 318 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 319 object->paging_in_progress += i; 320 } 321 322 void 323 vm_object_pip_subtract(vm_object_t object, short i) 324 { 325 326 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 327 object->paging_in_progress -= i; 328 } 329 330 void 331 vm_object_pip_wakeup(vm_object_t object) 332 { 333 334 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 335 object->paging_in_progress--; 336 if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { 337 vm_object_clear_flag(object, OBJ_PIPWNT); 338 wakeup(object); 339 } 340 } 341 342 void 343 vm_object_pip_wakeupn(vm_object_t object, short i) 344 { 345 346 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 347 if (i) 348 object->paging_in_progress -= i; 349 if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { 350 vm_object_clear_flag(object, OBJ_PIPWNT); 351 wakeup(object); 352 } 353 } 354 355 void 356 vm_object_pip_wait(vm_object_t object, char *waitid) 357 { 358 359 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 360 while (object->paging_in_progress) { 361 object->flags |= OBJ_PIPWNT; 362 msleep(object, VM_OBJECT_MTX(object), PVM, waitid, 0); 363 } 364 } 365 366 /* 367 * vm_object_allocate: 368 * 369 * Returns a new object with the given size. 370 */ 371 vm_object_t 372 vm_object_allocate(objtype_t type, vm_pindex_t size) 373 { 374 vm_object_t object; 375 376 object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK); 377 _vm_object_allocate(type, size, object); 378 return (object); 379 } 380 381 382 /* 383 * vm_object_reference: 384 * 385 * Gets another reference to the given object. Note: OBJ_DEAD 386 * objects can be referenced during final cleaning. 387 */ 388 void 389 vm_object_reference(vm_object_t object) 390 { 391 if (object == NULL) 392 return; 393 VM_OBJECT_LOCK(object); 394 vm_object_reference_locked(object); 395 VM_OBJECT_UNLOCK(object); 396 } 397 398 /* 399 * vm_object_reference_locked: 400 * 401 * Gets another reference to the given object. 402 * 403 * The object must be locked. 404 */ 405 void 406 vm_object_reference_locked(vm_object_t object) 407 { 408 struct vnode *vp; 409 410 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 411 object->ref_count++; 412 if (object->type == OBJT_VNODE) { 413 vp = object->handle; 414 vref(vp); 415 } 416 } 417 418 /* 419 * Handle deallocating an object of type OBJT_VNODE. 420 */ 421 static void 422 vm_object_vndeallocate(vm_object_t object) 423 { 424 struct vnode *vp = (struct vnode *) object->handle; 425 426 VFS_ASSERT_GIANT(vp->v_mount); 427 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 428 KASSERT(object->type == OBJT_VNODE, 429 ("vm_object_vndeallocate: not a vnode object")); 430 KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 431 #ifdef INVARIANTS 432 if (object->ref_count == 0) { 433 vprint("vm_object_vndeallocate", vp); 434 panic("vm_object_vndeallocate: bad object reference count"); 435 } 436 #endif 437 438 object->ref_count--; 439 if (object->ref_count == 0) { 440 mp_fixme("Unlocked vflag access."); 441 vp->v_vflag &= ~VV_TEXT; 442 } 443 VM_OBJECT_UNLOCK(object); 444 /* 445 * vrele may need a vop lock 446 */ 447 vrele(vp); 448 } 449 450 /* 451 * vm_object_deallocate: 452 * 453 * Release a reference to the specified object, 454 * gained either through a vm_object_allocate 455 * or a vm_object_reference call. When all references 456 * are gone, storage associated with this object 457 * may be relinquished. 458 * 459 * No object may be locked. 460 */ 461 void 462 vm_object_deallocate(vm_object_t object) 463 { 464 vm_object_t temp; 465 466 while (object != NULL) { 467 int vfslocked; 468 469 vfslocked = 0; 470 restart: 471 VM_OBJECT_LOCK(object); 472 if (object->type == OBJT_VNODE) { 473 struct vnode *vp = (struct vnode *) object->handle; 474 475 /* 476 * Conditionally acquire Giant for a vnode-backed 477 * object. We have to be careful since the type of 478 * a vnode object can change while the object is 479 * unlocked. 480 */ 481 if (VFS_NEEDSGIANT(vp->v_mount) && !vfslocked) { 482 vfslocked = 1; 483 if (!mtx_trylock(&Giant)) { 484 VM_OBJECT_UNLOCK(object); 485 mtx_lock(&Giant); 486 goto restart; 487 } 488 } 489 vm_object_vndeallocate(object); 490 VFS_UNLOCK_GIANT(vfslocked); 491 return; 492 } else 493 /* 494 * This is to handle the case that the object 495 * changed type while we dropped its lock to 496 * obtain Giant. 497 */ 498 VFS_UNLOCK_GIANT(vfslocked); 499 500 KASSERT(object->ref_count != 0, 501 ("vm_object_deallocate: object deallocated too many times: %d", object->type)); 502 503 /* 504 * If the reference count goes to 0 we start calling 505 * vm_object_terminate() on the object chain. 506 * A ref count of 1 may be a special case depending on the 507 * shadow count being 0 or 1. 508 */ 509 object->ref_count--; 510 if (object->ref_count > 1) { 511 VM_OBJECT_UNLOCK(object); 512 return; 513 } else if (object->ref_count == 1) { 514 if (object->shadow_count == 0 && 515 object->handle == NULL && 516 (object->type == OBJT_DEFAULT || 517 object->type == OBJT_SWAP)) { 518 vm_object_set_flag(object, OBJ_ONEMAPPING); 519 } else if ((object->shadow_count == 1) && 520 (object->handle == NULL) && 521 (object->type == OBJT_DEFAULT || 522 object->type == OBJT_SWAP)) { 523 vm_object_t robject; 524 525 robject = LIST_FIRST(&object->shadow_head); 526 KASSERT(robject != NULL, 527 ("vm_object_deallocate: ref_count: %d, shadow_count: %d", 528 object->ref_count, 529 object->shadow_count)); 530 if (!VM_OBJECT_TRYLOCK(robject)) { 531 /* 532 * Avoid a potential deadlock. 533 */ 534 object->ref_count++; 535 VM_OBJECT_UNLOCK(object); 536 /* 537 * More likely than not the thread 538 * holding robject's lock has lower 539 * priority than the current thread. 540 * Let the lower priority thread run. 541 */ 542 pause("vmo_de", 1); 543 continue; 544 } 545 /* 546 * Collapse object into its shadow unless its 547 * shadow is dead. In that case, object will 548 * be deallocated by the thread that is 549 * deallocating its shadow. 550 */ 551 if ((robject->flags & OBJ_DEAD) == 0 && 552 (robject->handle == NULL) && 553 (robject->type == OBJT_DEFAULT || 554 robject->type == OBJT_SWAP)) { 555 556 robject->ref_count++; 557 retry: 558 if (robject->paging_in_progress) { 559 VM_OBJECT_UNLOCK(object); 560 vm_object_pip_wait(robject, 561 "objde1"); 562 temp = robject->backing_object; 563 if (object == temp) { 564 VM_OBJECT_LOCK(object); 565 goto retry; 566 } 567 } else if (object->paging_in_progress) { 568 VM_OBJECT_UNLOCK(robject); 569 object->flags |= OBJ_PIPWNT; 570 msleep(object, 571 VM_OBJECT_MTX(object), 572 PDROP | PVM, "objde2", 0); 573 VM_OBJECT_LOCK(robject); 574 temp = robject->backing_object; 575 if (object == temp) { 576 VM_OBJECT_LOCK(object); 577 goto retry; 578 } 579 } else 580 VM_OBJECT_UNLOCK(object); 581 582 if (robject->ref_count == 1) { 583 robject->ref_count--; 584 object = robject; 585 goto doterm; 586 } 587 object = robject; 588 vm_object_collapse(object); 589 VM_OBJECT_UNLOCK(object); 590 continue; 591 } 592 VM_OBJECT_UNLOCK(robject); 593 } 594 VM_OBJECT_UNLOCK(object); 595 return; 596 } 597 doterm: 598 temp = object->backing_object; 599 if (temp != NULL) { 600 VM_OBJECT_LOCK(temp); 601 LIST_REMOVE(object, shadow_list); 602 temp->shadow_count--; 603 temp->generation++; 604 VM_OBJECT_UNLOCK(temp); 605 object->backing_object = NULL; 606 } 607 /* 608 * Don't double-terminate, we could be in a termination 609 * recursion due to the terminate having to sync data 610 * to disk. 611 */ 612 if ((object->flags & OBJ_DEAD) == 0) 613 vm_object_terminate(object); 614 else 615 VM_OBJECT_UNLOCK(object); 616 object = temp; 617 } 618 } 619 620 /* 621 * vm_object_destroy removes the object from the global object list 622 * and frees the space for the object. 623 */ 624 void 625 vm_object_destroy(vm_object_t object) 626 { 627 628 /* 629 * Remove the object from the global object list. 630 */ 631 mtx_lock(&vm_object_list_mtx); 632 TAILQ_REMOVE(&vm_object_list, object, object_list); 633 mtx_unlock(&vm_object_list_mtx); 634 635 /* 636 * Release the allocation charge. 637 */ 638 if (object->uip != NULL) { 639 KASSERT(object->type == OBJT_DEFAULT || 640 object->type == OBJT_SWAP, 641 ("vm_object_terminate: non-swap obj %p has uip", 642 object)); 643 swap_release_by_uid(object->charge, object->uip); 644 object->charge = 0; 645 uifree(object->uip); 646 object->uip = NULL; 647 } 648 649 /* 650 * Free the space for the object. 651 */ 652 uma_zfree(obj_zone, object); 653 } 654 655 /* 656 * vm_object_terminate actually destroys the specified object, freeing 657 * up all previously used resources. 658 * 659 * The object must be locked. 660 * This routine may block. 661 */ 662 void 663 vm_object_terminate(vm_object_t object) 664 { 665 vm_page_t p; 666 667 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 668 669 /* 670 * Make sure no one uses us. 671 */ 672 vm_object_set_flag(object, OBJ_DEAD); 673 674 /* 675 * wait for the pageout daemon to be done with the object 676 */ 677 vm_object_pip_wait(object, "objtrm"); 678 679 KASSERT(!object->paging_in_progress, 680 ("vm_object_terminate: pageout in progress")); 681 682 /* 683 * Clean and free the pages, as appropriate. All references to the 684 * object are gone, so we don't need to lock it. 685 */ 686 if (object->type == OBJT_VNODE) { 687 struct vnode *vp = (struct vnode *)object->handle; 688 689 /* 690 * Clean pages and flush buffers. 691 */ 692 vm_object_page_clean(object, 0, 0, OBJPC_SYNC); 693 VM_OBJECT_UNLOCK(object); 694 695 vinvalbuf(vp, V_SAVE, 0, 0); 696 697 VM_OBJECT_LOCK(object); 698 } 699 700 KASSERT(object->ref_count == 0, 701 ("vm_object_terminate: object with references, ref_count=%d", 702 object->ref_count)); 703 704 /* 705 * Now free any remaining pages. For internal objects, this also 706 * removes them from paging queues. Don't free wired pages, just 707 * remove them from the object. 708 */ 709 while ((p = TAILQ_FIRST(&object->memq)) != NULL) { 710 KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0, 711 ("vm_object_terminate: freeing busy page %p " 712 "p->busy = %d, p->oflags %x\n", p, p->busy, p->oflags)); 713 vm_page_lock(p); 714 if (p->wire_count == 0) { 715 vm_page_free(p); 716 PCPU_INC(cnt.v_pfree); 717 } else 718 vm_page_remove(p); 719 vm_page_unlock(p); 720 } 721 722 #if VM_NRESERVLEVEL > 0 723 if (__predict_false(!LIST_EMPTY(&object->rvq))) 724 vm_reserv_break_all(object); 725 #endif 726 if (__predict_false(object->cache != NULL)) 727 vm_page_cache_free(object, 0, 0); 728 729 /* 730 * Let the pager know object is dead. 731 */ 732 vm_pager_deallocate(object); 733 VM_OBJECT_UNLOCK(object); 734 735 vm_object_destroy(object); 736 } 737 738 /* 739 * vm_object_page_clean 740 * 741 * Clean all dirty pages in the specified range of object. Leaves page 742 * on whatever queue it is currently on. If NOSYNC is set then do not 743 * write out pages with VPO_NOSYNC set (originally comes from MAP_NOSYNC), 744 * leaving the object dirty. 745 * 746 * When stuffing pages asynchronously, allow clustering. XXX we need a 747 * synchronous clustering mode implementation. 748 * 749 * Odd semantics: if start == end, we clean everything. 750 * 751 * The object must be locked. 752 */ 753 void 754 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 755 int flags) 756 { 757 vm_page_t np, p; 758 vm_pindex_t pi, tend; 759 int clearobjflags, curgeneration, n, pagerflags; 760 761 mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED); 762 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 763 KASSERT(object->type == OBJT_VNODE, ("Not a vnode object")); 764 if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || 765 object->resident_page_count == 0) 766 return; 767 768 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ? 769 VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 770 pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0; 771 772 tend = (end == 0) ? object->size : end; 773 774 vm_object_set_flag(object, OBJ_CLEANING); 775 776 /* 777 * Make the page read-only so we can then clear the object flags. 778 * 779 * However, if this is a nosync mmap then the object is likely to 780 * stay dirty so do not mess with the page and do not clear the 781 * object flags. 782 */ 783 clearobjflags = 1; 784 for (p = vm_page_find_least(object, start); 785 p != NULL && p->pindex < tend; p = TAILQ_NEXT(p, listq)) { 786 if ((flags & OBJPC_NOSYNC) != 0 && 787 (p->oflags & VPO_NOSYNC) != 0) 788 clearobjflags = 0; 789 else 790 pmap_remove_write(p); 791 } 792 793 if (clearobjflags && (start == 0) && (tend == object->size)) 794 vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY); 795 796 rescan: 797 curgeneration = object->generation; 798 799 for (p = vm_page_find_least(object, start); p != NULL; p = np) { 800 pi = p->pindex; 801 if (pi >= tend) 802 break; 803 np = TAILQ_NEXT(p, listq); 804 if (p->valid == 0) 805 continue; 806 while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { 807 if (object->generation != curgeneration) 808 goto rescan; 809 } 810 vm_page_test_dirty(p); 811 if (p->dirty == 0) 812 continue; 813 814 /* 815 * If we have been asked to skip nosync pages and this is a 816 * nosync page, skip it. Note that the object flags were 817 * not cleared in this case so we do not have to set them. 818 */ 819 if ((flags & OBJPC_NOSYNC) != 0 && 820 (p->oflags & VPO_NOSYNC) != 0) 821 continue; 822 823 n = vm_object_page_collect_flush(object, p, pagerflags); 824 KASSERT(n > 0, ("vm_object_page_collect_flush failed")); 825 if (object->generation != curgeneration) 826 goto rescan; 827 np = vm_page_find_least(object, pi + n); 828 } 829 #if 0 830 VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0); 831 #endif 832 833 vm_object_clear_flag(object, OBJ_CLEANING); 834 } 835 836 static int 837 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags) 838 { 839 int runlen; 840 int maxf; 841 int chkb; 842 int maxb; 843 int i, index; 844 vm_pindex_t pi; 845 vm_page_t maf[vm_pageout_page_count]; 846 vm_page_t mab[vm_pageout_page_count]; 847 vm_page_t ma[vm_pageout_page_count]; 848 vm_page_t tp, p1; 849 850 mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED); 851 vm_page_lock_assert(p, MA_NOTOWNED); 852 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 853 pi = p->pindex; 854 maxf = 0; 855 for (i = 1, p1 = p; i < vm_pageout_page_count; i++) { 856 tp = vm_page_next(p1); 857 if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0) 858 break; 859 vm_page_test_dirty(tp); 860 if (tp->dirty == 0) 861 break; 862 maf[i - 1] = p1 = tp; 863 maxf++; 864 } 865 866 maxb = 0; 867 chkb = vm_pageout_page_count - maxf; 868 for (i = 1, p1 = p; i < chkb; i++) { 869 tp = vm_page_prev(p1); 870 if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0) 871 break; 872 vm_page_test_dirty(tp); 873 if (tp->dirty == 0) 874 break; 875 mab[i - 1] = p1 = tp; 876 maxb++; 877 } 878 879 for (i = 0; i < maxb; i++) { 880 index = (maxb - i) - 1; 881 ma[index] = mab[i]; 882 } 883 ma[maxb] = p; 884 for (i = 0; i < maxf; i++) { 885 index = (maxb + i) + 1; 886 ma[index] = maf[i]; 887 } 888 runlen = maxb + maxf + 1; 889 890 vm_pageout_flush(ma, runlen, pagerflags); 891 for (i = 0; i < runlen; i++) { 892 if (ma[i]->dirty != 0) { 893 KASSERT((ma[i]->flags & PG_WRITEABLE) == 0, 894 ("vm_object_page_collect_flush: page %p is not write protected", 895 ma[i])); 896 } 897 } 898 for (i = 0; i < maxf; i++) { 899 if (ma[i + maxb + 1]->dirty != 0) { 900 /* 901 * maxf will end up being the actual number of pages 902 * we wrote out contiguously, non-inclusive of the 903 * first page. We do not count look-behind pages. 904 */ 905 if (maxf > i) { 906 maxf = i; 907 break; 908 } 909 } 910 } 911 return (maxf + 1); 912 } 913 914 /* 915 * Note that there is absolutely no sense in writing out 916 * anonymous objects, so we track down the vnode object 917 * to write out. 918 * We invalidate (remove) all pages from the address space 919 * for semantic correctness. 920 * 921 * Note: certain anonymous maps, such as MAP_NOSYNC maps, 922 * may start out with a NULL object. 923 */ 924 void 925 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size, 926 boolean_t syncio, boolean_t invalidate) 927 { 928 vm_object_t backing_object; 929 struct vnode *vp; 930 struct mount *mp; 931 int flags; 932 933 if (object == NULL) 934 return; 935 VM_OBJECT_LOCK(object); 936 while ((backing_object = object->backing_object) != NULL) { 937 VM_OBJECT_LOCK(backing_object); 938 offset += object->backing_object_offset; 939 VM_OBJECT_UNLOCK(object); 940 object = backing_object; 941 if (object->size < OFF_TO_IDX(offset + size)) 942 size = IDX_TO_OFF(object->size) - offset; 943 } 944 /* 945 * Flush pages if writing is allowed, invalidate them 946 * if invalidation requested. Pages undergoing I/O 947 * will be ignored by vm_object_page_remove(). 948 * 949 * We cannot lock the vnode and then wait for paging 950 * to complete without deadlocking against vm_fault. 951 * Instead we simply call vm_object_page_remove() and 952 * allow it to block internally on a page-by-page 953 * basis when it encounters pages undergoing async 954 * I/O. 955 */ 956 if (object->type == OBJT_VNODE && 957 (object->flags & OBJ_MIGHTBEDIRTY) != 0) { 958 int vfslocked; 959 vp = object->handle; 960 VM_OBJECT_UNLOCK(object); 961 (void) vn_start_write(vp, &mp, V_WAIT); 962 vfslocked = VFS_LOCK_GIANT(vp->v_mount); 963 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 964 flags = (syncio || invalidate) ? OBJPC_SYNC : 0; 965 flags |= invalidate ? OBJPC_INVAL : 0; 966 VM_OBJECT_LOCK(object); 967 vm_object_page_clean(object, 968 OFF_TO_IDX(offset), 969 OFF_TO_IDX(offset + size + PAGE_MASK), 970 flags); 971 VM_OBJECT_UNLOCK(object); 972 VOP_UNLOCK(vp, 0); 973 VFS_UNLOCK_GIANT(vfslocked); 974 vn_finished_write(mp); 975 VM_OBJECT_LOCK(object); 976 } 977 if ((object->type == OBJT_VNODE || 978 object->type == OBJT_DEVICE) && invalidate) { 979 boolean_t purge; 980 purge = old_msync || (object->type == OBJT_DEVICE); 981 vm_object_page_remove(object, 982 OFF_TO_IDX(offset), 983 OFF_TO_IDX(offset + size + PAGE_MASK), 984 purge ? FALSE : TRUE); 985 } 986 VM_OBJECT_UNLOCK(object); 987 } 988 989 /* 990 * vm_object_madvise: 991 * 992 * Implements the madvise function at the object/page level. 993 * 994 * MADV_WILLNEED (any object) 995 * 996 * Activate the specified pages if they are resident. 997 * 998 * MADV_DONTNEED (any object) 999 * 1000 * Deactivate the specified pages if they are resident. 1001 * 1002 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, 1003 * OBJ_ONEMAPPING only) 1004 * 1005 * Deactivate and clean the specified pages if they are 1006 * resident. This permits the process to reuse the pages 1007 * without faulting or the kernel to reclaim the pages 1008 * without I/O. 1009 */ 1010 void 1011 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise) 1012 { 1013 vm_pindex_t end, tpindex; 1014 vm_object_t backing_object, tobject; 1015 vm_page_t m; 1016 1017 if (object == NULL) 1018 return; 1019 VM_OBJECT_LOCK(object); 1020 end = pindex + count; 1021 /* 1022 * Locate and adjust resident pages 1023 */ 1024 for (; pindex < end; pindex += 1) { 1025 relookup: 1026 tobject = object; 1027 tpindex = pindex; 1028 shadowlookup: 1029 /* 1030 * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages 1031 * and those pages must be OBJ_ONEMAPPING. 1032 */ 1033 if (advise == MADV_FREE) { 1034 if ((tobject->type != OBJT_DEFAULT && 1035 tobject->type != OBJT_SWAP) || 1036 (tobject->flags & OBJ_ONEMAPPING) == 0) { 1037 goto unlock_tobject; 1038 } 1039 } else if (tobject->type == OBJT_PHYS) 1040 goto unlock_tobject; 1041 m = vm_page_lookup(tobject, tpindex); 1042 if (m == NULL && advise == MADV_WILLNEED) { 1043 /* 1044 * If the page is cached, reactivate it. 1045 */ 1046 m = vm_page_alloc(tobject, tpindex, VM_ALLOC_IFCACHED | 1047 VM_ALLOC_NOBUSY); 1048 } 1049 if (m == NULL) { 1050 /* 1051 * There may be swap even if there is no backing page 1052 */ 1053 if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 1054 swap_pager_freespace(tobject, tpindex, 1); 1055 /* 1056 * next object 1057 */ 1058 backing_object = tobject->backing_object; 1059 if (backing_object == NULL) 1060 goto unlock_tobject; 1061 VM_OBJECT_LOCK(backing_object); 1062 tpindex += OFF_TO_IDX(tobject->backing_object_offset); 1063 if (tobject != object) 1064 VM_OBJECT_UNLOCK(tobject); 1065 tobject = backing_object; 1066 goto shadowlookup; 1067 } else if (m->valid != VM_PAGE_BITS_ALL) 1068 goto unlock_tobject; 1069 /* 1070 * If the page is not in a normal state, skip it. 1071 */ 1072 vm_page_lock(m); 1073 if (m->hold_count != 0 || m->wire_count != 0) { 1074 vm_page_unlock(m); 1075 goto unlock_tobject; 1076 } 1077 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 1078 ("vm_object_madvise: page %p is not managed", m)); 1079 if ((m->oflags & VPO_BUSY) || m->busy) { 1080 if (advise == MADV_WILLNEED) { 1081 /* 1082 * Reference the page before unlocking and 1083 * sleeping so that the page daemon is less 1084 * likely to reclaim it. 1085 */ 1086 vm_page_lock_queues(); 1087 vm_page_flag_set(m, PG_REFERENCED); 1088 vm_page_unlock_queues(); 1089 } 1090 vm_page_unlock(m); 1091 if (object != tobject) 1092 VM_OBJECT_UNLOCK(object); 1093 m->oflags |= VPO_WANTED; 1094 msleep(m, VM_OBJECT_MTX(tobject), PDROP | PVM, "madvpo", 1095 0); 1096 VM_OBJECT_LOCK(object); 1097 goto relookup; 1098 } 1099 if (advise == MADV_WILLNEED) { 1100 vm_page_activate(m); 1101 } else if (advise == MADV_DONTNEED) { 1102 vm_page_dontneed(m); 1103 } else if (advise == MADV_FREE) { 1104 /* 1105 * Mark the page clean. This will allow the page 1106 * to be freed up by the system. However, such pages 1107 * are often reused quickly by malloc()/free() 1108 * so we do not do anything that would cause 1109 * a page fault if we can help it. 1110 * 1111 * Specifically, we do not try to actually free 1112 * the page now nor do we try to put it in the 1113 * cache (which would cause a page fault on reuse). 1114 * 1115 * But we do make the page is freeable as we 1116 * can without actually taking the step of unmapping 1117 * it. 1118 */ 1119 pmap_clear_modify(m); 1120 m->dirty = 0; 1121 m->act_count = 0; 1122 vm_page_dontneed(m); 1123 } 1124 vm_page_unlock(m); 1125 if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 1126 swap_pager_freespace(tobject, tpindex, 1); 1127 unlock_tobject: 1128 if (tobject != object) 1129 VM_OBJECT_UNLOCK(tobject); 1130 } 1131 VM_OBJECT_UNLOCK(object); 1132 } 1133 1134 /* 1135 * vm_object_shadow: 1136 * 1137 * Create a new object which is backed by the 1138 * specified existing object range. The source 1139 * object reference is deallocated. 1140 * 1141 * The new object and offset into that object 1142 * are returned in the source parameters. 1143 */ 1144 void 1145 vm_object_shadow( 1146 vm_object_t *object, /* IN/OUT */ 1147 vm_ooffset_t *offset, /* IN/OUT */ 1148 vm_size_t length) 1149 { 1150 vm_object_t source; 1151 vm_object_t result; 1152 1153 source = *object; 1154 1155 /* 1156 * Don't create the new object if the old object isn't shared. 1157 */ 1158 if (source != NULL) { 1159 VM_OBJECT_LOCK(source); 1160 if (source->ref_count == 1 && 1161 source->handle == NULL && 1162 (source->type == OBJT_DEFAULT || 1163 source->type == OBJT_SWAP)) { 1164 VM_OBJECT_UNLOCK(source); 1165 return; 1166 } 1167 VM_OBJECT_UNLOCK(source); 1168 } 1169 1170 /* 1171 * Allocate a new object with the given length. 1172 */ 1173 result = vm_object_allocate(OBJT_DEFAULT, length); 1174 1175 /* 1176 * The new object shadows the source object, adding a reference to it. 1177 * Our caller changes his reference to point to the new object, 1178 * removing a reference to the source object. Net result: no change 1179 * of reference count. 1180 * 1181 * Try to optimize the result object's page color when shadowing 1182 * in order to maintain page coloring consistency in the combined 1183 * shadowed object. 1184 */ 1185 result->backing_object = source; 1186 /* 1187 * Store the offset into the source object, and fix up the offset into 1188 * the new object. 1189 */ 1190 result->backing_object_offset = *offset; 1191 if (source != NULL) { 1192 VM_OBJECT_LOCK(source); 1193 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list); 1194 source->shadow_count++; 1195 source->generation++; 1196 #if VM_NRESERVLEVEL > 0 1197 result->flags |= source->flags & OBJ_COLORED; 1198 result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & 1199 ((1 << (VM_NFREEORDER - 1)) - 1); 1200 #endif 1201 VM_OBJECT_UNLOCK(source); 1202 } 1203 1204 1205 /* 1206 * Return the new things 1207 */ 1208 *offset = 0; 1209 *object = result; 1210 } 1211 1212 /* 1213 * vm_object_split: 1214 * 1215 * Split the pages in a map entry into a new object. This affords 1216 * easier removal of unused pages, and keeps object inheritance from 1217 * being a negative impact on memory usage. 1218 */ 1219 void 1220 vm_object_split(vm_map_entry_t entry) 1221 { 1222 vm_page_t m, m_next; 1223 vm_object_t orig_object, new_object, source; 1224 vm_pindex_t idx, offidxstart; 1225 vm_size_t size; 1226 1227 orig_object = entry->object.vm_object; 1228 if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) 1229 return; 1230 if (orig_object->ref_count <= 1) 1231 return; 1232 VM_OBJECT_UNLOCK(orig_object); 1233 1234 offidxstart = OFF_TO_IDX(entry->offset); 1235 size = atop(entry->end - entry->start); 1236 1237 /* 1238 * If swap_pager_copy() is later called, it will convert new_object 1239 * into a swap object. 1240 */ 1241 new_object = vm_object_allocate(OBJT_DEFAULT, size); 1242 1243 /* 1244 * At this point, the new object is still private, so the order in 1245 * which the original and new objects are locked does not matter. 1246 */ 1247 VM_OBJECT_LOCK(new_object); 1248 VM_OBJECT_LOCK(orig_object); 1249 source = orig_object->backing_object; 1250 if (source != NULL) { 1251 VM_OBJECT_LOCK(source); 1252 if ((source->flags & OBJ_DEAD) != 0) { 1253 VM_OBJECT_UNLOCK(source); 1254 VM_OBJECT_UNLOCK(orig_object); 1255 VM_OBJECT_UNLOCK(new_object); 1256 vm_object_deallocate(new_object); 1257 VM_OBJECT_LOCK(orig_object); 1258 return; 1259 } 1260 LIST_INSERT_HEAD(&source->shadow_head, 1261 new_object, shadow_list); 1262 source->shadow_count++; 1263 source->generation++; 1264 vm_object_reference_locked(source); /* for new_object */ 1265 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1266 VM_OBJECT_UNLOCK(source); 1267 new_object->backing_object_offset = 1268 orig_object->backing_object_offset + entry->offset; 1269 new_object->backing_object = source; 1270 } 1271 if (orig_object->uip != NULL) { 1272 new_object->uip = orig_object->uip; 1273 uihold(orig_object->uip); 1274 new_object->charge = ptoa(size); 1275 KASSERT(orig_object->charge >= ptoa(size), 1276 ("orig_object->charge < 0")); 1277 orig_object->charge -= ptoa(size); 1278 } 1279 retry: 1280 m = vm_page_find_least(orig_object, offidxstart); 1281 for (; m != NULL && (idx = m->pindex - offidxstart) < size; 1282 m = m_next) { 1283 m_next = TAILQ_NEXT(m, listq); 1284 1285 /* 1286 * We must wait for pending I/O to complete before we can 1287 * rename the page. 1288 * 1289 * We do not have to VM_PROT_NONE the page as mappings should 1290 * not be changed by this operation. 1291 */ 1292 if ((m->oflags & VPO_BUSY) || m->busy) { 1293 VM_OBJECT_UNLOCK(new_object); 1294 m->oflags |= VPO_WANTED; 1295 msleep(m, VM_OBJECT_MTX(orig_object), PVM, "spltwt", 0); 1296 VM_OBJECT_LOCK(new_object); 1297 goto retry; 1298 } 1299 vm_page_lock(m); 1300 vm_page_rename(m, new_object, idx); 1301 vm_page_unlock(m); 1302 /* page automatically made dirty by rename and cache handled */ 1303 vm_page_busy(m); 1304 } 1305 if (orig_object->type == OBJT_SWAP) { 1306 /* 1307 * swap_pager_copy() can sleep, in which case the orig_object's 1308 * and new_object's locks are released and reacquired. 1309 */ 1310 swap_pager_copy(orig_object, new_object, offidxstart, 0); 1311 1312 /* 1313 * Transfer any cached pages from orig_object to new_object. 1314 */ 1315 if (__predict_false(orig_object->cache != NULL)) 1316 vm_page_cache_transfer(orig_object, offidxstart, 1317 new_object); 1318 } 1319 VM_OBJECT_UNLOCK(orig_object); 1320 TAILQ_FOREACH(m, &new_object->memq, listq) 1321 vm_page_wakeup(m); 1322 VM_OBJECT_UNLOCK(new_object); 1323 entry->object.vm_object = new_object; 1324 entry->offset = 0LL; 1325 vm_object_deallocate(orig_object); 1326 VM_OBJECT_LOCK(new_object); 1327 } 1328 1329 #define OBSC_TEST_ALL_SHADOWED 0x0001 1330 #define OBSC_COLLAPSE_NOWAIT 0x0002 1331 #define OBSC_COLLAPSE_WAIT 0x0004 1332 1333 static int 1334 vm_object_backing_scan(vm_object_t object, int op) 1335 { 1336 int r = 1; 1337 vm_page_t p; 1338 vm_object_t backing_object; 1339 vm_pindex_t backing_offset_index; 1340 1341 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1342 VM_OBJECT_LOCK_ASSERT(object->backing_object, MA_OWNED); 1343 1344 backing_object = object->backing_object; 1345 backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 1346 1347 /* 1348 * Initial conditions 1349 */ 1350 if (op & OBSC_TEST_ALL_SHADOWED) { 1351 /* 1352 * We do not want to have to test for the existence of cache 1353 * or swap pages in the backing object. XXX but with the 1354 * new swapper this would be pretty easy to do. 1355 * 1356 * XXX what about anonymous MAP_SHARED memory that hasn't 1357 * been ZFOD faulted yet? If we do not test for this, the 1358 * shadow test may succeed! XXX 1359 */ 1360 if (backing_object->type != OBJT_DEFAULT) { 1361 return (0); 1362 } 1363 } 1364 if (op & OBSC_COLLAPSE_WAIT) { 1365 vm_object_set_flag(backing_object, OBJ_DEAD); 1366 } 1367 1368 /* 1369 * Our scan 1370 */ 1371 p = TAILQ_FIRST(&backing_object->memq); 1372 while (p) { 1373 vm_page_t next = TAILQ_NEXT(p, listq); 1374 vm_pindex_t new_pindex = p->pindex - backing_offset_index; 1375 1376 if (op & OBSC_TEST_ALL_SHADOWED) { 1377 vm_page_t pp; 1378 1379 /* 1380 * Ignore pages outside the parent object's range 1381 * and outside the parent object's mapping of the 1382 * backing object. 1383 * 1384 * note that we do not busy the backing object's 1385 * page. 1386 */ 1387 if ( 1388 p->pindex < backing_offset_index || 1389 new_pindex >= object->size 1390 ) { 1391 p = next; 1392 continue; 1393 } 1394 1395 /* 1396 * See if the parent has the page or if the parent's 1397 * object pager has the page. If the parent has the 1398 * page but the page is not valid, the parent's 1399 * object pager must have the page. 1400 * 1401 * If this fails, the parent does not completely shadow 1402 * the object and we might as well give up now. 1403 */ 1404 1405 pp = vm_page_lookup(object, new_pindex); 1406 if ( 1407 (pp == NULL || pp->valid == 0) && 1408 !vm_pager_has_page(object, new_pindex, NULL, NULL) 1409 ) { 1410 r = 0; 1411 break; 1412 } 1413 } 1414 1415 /* 1416 * Check for busy page 1417 */ 1418 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) { 1419 vm_page_t pp; 1420 1421 if (op & OBSC_COLLAPSE_NOWAIT) { 1422 if ((p->oflags & VPO_BUSY) || 1423 !p->valid || 1424 p->busy) { 1425 p = next; 1426 continue; 1427 } 1428 } else if (op & OBSC_COLLAPSE_WAIT) { 1429 if ((p->oflags & VPO_BUSY) || p->busy) { 1430 VM_OBJECT_UNLOCK(object); 1431 p->oflags |= VPO_WANTED; 1432 msleep(p, VM_OBJECT_MTX(backing_object), 1433 PDROP | PVM, "vmocol", 0); 1434 VM_OBJECT_LOCK(object); 1435 VM_OBJECT_LOCK(backing_object); 1436 /* 1437 * If we slept, anything could have 1438 * happened. Since the object is 1439 * marked dead, the backing offset 1440 * should not have changed so we 1441 * just restart our scan. 1442 */ 1443 p = TAILQ_FIRST(&backing_object->memq); 1444 continue; 1445 } 1446 } 1447 1448 KASSERT( 1449 p->object == backing_object, 1450 ("vm_object_backing_scan: object mismatch") 1451 ); 1452 1453 /* 1454 * Destroy any associated swap 1455 */ 1456 if (backing_object->type == OBJT_SWAP) { 1457 swap_pager_freespace( 1458 backing_object, 1459 p->pindex, 1460 1 1461 ); 1462 } 1463 1464 if ( 1465 p->pindex < backing_offset_index || 1466 new_pindex >= object->size 1467 ) { 1468 /* 1469 * Page is out of the parent object's range, we 1470 * can simply destroy it. 1471 */ 1472 vm_page_lock(p); 1473 KASSERT(!pmap_page_is_mapped(p), 1474 ("freeing mapped page %p", p)); 1475 if (p->wire_count == 0) 1476 vm_page_free(p); 1477 else 1478 vm_page_remove(p); 1479 vm_page_unlock(p); 1480 p = next; 1481 continue; 1482 } 1483 1484 pp = vm_page_lookup(object, new_pindex); 1485 if ( 1486 pp != NULL || 1487 vm_pager_has_page(object, new_pindex, NULL, NULL) 1488 ) { 1489 /* 1490 * page already exists in parent OR swap exists 1491 * for this location in the parent. Destroy 1492 * the original page from the backing object. 1493 * 1494 * Leave the parent's page alone 1495 */ 1496 vm_page_lock(p); 1497 KASSERT(!pmap_page_is_mapped(p), 1498 ("freeing mapped page %p", p)); 1499 if (p->wire_count == 0) 1500 vm_page_free(p); 1501 else 1502 vm_page_remove(p); 1503 vm_page_unlock(p); 1504 p = next; 1505 continue; 1506 } 1507 1508 #if VM_NRESERVLEVEL > 0 1509 /* 1510 * Rename the reservation. 1511 */ 1512 vm_reserv_rename(p, object, backing_object, 1513 backing_offset_index); 1514 #endif 1515 1516 /* 1517 * Page does not exist in parent, rename the 1518 * page from the backing object to the main object. 1519 * 1520 * If the page was mapped to a process, it can remain 1521 * mapped through the rename. 1522 */ 1523 vm_page_lock(p); 1524 vm_page_rename(p, object, new_pindex); 1525 vm_page_unlock(p); 1526 /* page automatically made dirty by rename */ 1527 } 1528 p = next; 1529 } 1530 return (r); 1531 } 1532 1533 1534 /* 1535 * this version of collapse allows the operation to occur earlier and 1536 * when paging_in_progress is true for an object... This is not a complete 1537 * operation, but should plug 99.9% of the rest of the leaks. 1538 */ 1539 static void 1540 vm_object_qcollapse(vm_object_t object) 1541 { 1542 vm_object_t backing_object = object->backing_object; 1543 1544 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1545 VM_OBJECT_LOCK_ASSERT(backing_object, MA_OWNED); 1546 1547 if (backing_object->ref_count != 1) 1548 return; 1549 1550 vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT); 1551 } 1552 1553 /* 1554 * vm_object_collapse: 1555 * 1556 * Collapse an object with the object backing it. 1557 * Pages in the backing object are moved into the 1558 * parent, and the backing object is deallocated. 1559 */ 1560 void 1561 vm_object_collapse(vm_object_t object) 1562 { 1563 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1564 1565 while (TRUE) { 1566 vm_object_t backing_object; 1567 1568 /* 1569 * Verify that the conditions are right for collapse: 1570 * 1571 * The object exists and the backing object exists. 1572 */ 1573 if ((backing_object = object->backing_object) == NULL) 1574 break; 1575 1576 /* 1577 * we check the backing object first, because it is most likely 1578 * not collapsable. 1579 */ 1580 VM_OBJECT_LOCK(backing_object); 1581 if (backing_object->handle != NULL || 1582 (backing_object->type != OBJT_DEFAULT && 1583 backing_object->type != OBJT_SWAP) || 1584 (backing_object->flags & OBJ_DEAD) || 1585 object->handle != NULL || 1586 (object->type != OBJT_DEFAULT && 1587 object->type != OBJT_SWAP) || 1588 (object->flags & OBJ_DEAD)) { 1589 VM_OBJECT_UNLOCK(backing_object); 1590 break; 1591 } 1592 1593 if ( 1594 object->paging_in_progress != 0 || 1595 backing_object->paging_in_progress != 0 1596 ) { 1597 vm_object_qcollapse(object); 1598 VM_OBJECT_UNLOCK(backing_object); 1599 break; 1600 } 1601 /* 1602 * We know that we can either collapse the backing object (if 1603 * the parent is the only reference to it) or (perhaps) have 1604 * the parent bypass the object if the parent happens to shadow 1605 * all the resident pages in the entire backing object. 1606 * 1607 * This is ignoring pager-backed pages such as swap pages. 1608 * vm_object_backing_scan fails the shadowing test in this 1609 * case. 1610 */ 1611 if (backing_object->ref_count == 1) { 1612 /* 1613 * If there is exactly one reference to the backing 1614 * object, we can collapse it into the parent. 1615 */ 1616 vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT); 1617 1618 #if VM_NRESERVLEVEL > 0 1619 /* 1620 * Break any reservations from backing_object. 1621 */ 1622 if (__predict_false(!LIST_EMPTY(&backing_object->rvq))) 1623 vm_reserv_break_all(backing_object); 1624 #endif 1625 1626 /* 1627 * Move the pager from backing_object to object. 1628 */ 1629 if (backing_object->type == OBJT_SWAP) { 1630 /* 1631 * swap_pager_copy() can sleep, in which case 1632 * the backing_object's and object's locks are 1633 * released and reacquired. 1634 */ 1635 swap_pager_copy( 1636 backing_object, 1637 object, 1638 OFF_TO_IDX(object->backing_object_offset), TRUE); 1639 1640 /* 1641 * Free any cached pages from backing_object. 1642 */ 1643 if (__predict_false(backing_object->cache != NULL)) 1644 vm_page_cache_free(backing_object, 0, 0); 1645 } 1646 /* 1647 * Object now shadows whatever backing_object did. 1648 * Note that the reference to 1649 * backing_object->backing_object moves from within 1650 * backing_object to within object. 1651 */ 1652 LIST_REMOVE(object, shadow_list); 1653 backing_object->shadow_count--; 1654 backing_object->generation++; 1655 if (backing_object->backing_object) { 1656 VM_OBJECT_LOCK(backing_object->backing_object); 1657 LIST_REMOVE(backing_object, shadow_list); 1658 LIST_INSERT_HEAD( 1659 &backing_object->backing_object->shadow_head, 1660 object, shadow_list); 1661 /* 1662 * The shadow_count has not changed. 1663 */ 1664 backing_object->backing_object->generation++; 1665 VM_OBJECT_UNLOCK(backing_object->backing_object); 1666 } 1667 object->backing_object = backing_object->backing_object; 1668 object->backing_object_offset += 1669 backing_object->backing_object_offset; 1670 1671 /* 1672 * Discard backing_object. 1673 * 1674 * Since the backing object has no pages, no pager left, 1675 * and no object references within it, all that is 1676 * necessary is to dispose of it. 1677 */ 1678 KASSERT(backing_object->ref_count == 1, ( 1679 "backing_object %p was somehow re-referenced during collapse!", 1680 backing_object)); 1681 VM_OBJECT_UNLOCK(backing_object); 1682 vm_object_destroy(backing_object); 1683 1684 object_collapses++; 1685 } else { 1686 vm_object_t new_backing_object; 1687 1688 /* 1689 * If we do not entirely shadow the backing object, 1690 * there is nothing we can do so we give up. 1691 */ 1692 if (object->resident_page_count != object->size && 1693 vm_object_backing_scan(object, 1694 OBSC_TEST_ALL_SHADOWED) == 0) { 1695 VM_OBJECT_UNLOCK(backing_object); 1696 break; 1697 } 1698 1699 /* 1700 * Make the parent shadow the next object in the 1701 * chain. Deallocating backing_object will not remove 1702 * it, since its reference count is at least 2. 1703 */ 1704 LIST_REMOVE(object, shadow_list); 1705 backing_object->shadow_count--; 1706 backing_object->generation++; 1707 1708 new_backing_object = backing_object->backing_object; 1709 if ((object->backing_object = new_backing_object) != NULL) { 1710 VM_OBJECT_LOCK(new_backing_object); 1711 LIST_INSERT_HEAD( 1712 &new_backing_object->shadow_head, 1713 object, 1714 shadow_list 1715 ); 1716 new_backing_object->shadow_count++; 1717 new_backing_object->generation++; 1718 vm_object_reference_locked(new_backing_object); 1719 VM_OBJECT_UNLOCK(new_backing_object); 1720 object->backing_object_offset += 1721 backing_object->backing_object_offset; 1722 } 1723 1724 /* 1725 * Drop the reference count on backing_object. Since 1726 * its ref_count was at least 2, it will not vanish. 1727 */ 1728 backing_object->ref_count--; 1729 VM_OBJECT_UNLOCK(backing_object); 1730 object_bypasses++; 1731 } 1732 1733 /* 1734 * Try again with this object's new backing object. 1735 */ 1736 } 1737 } 1738 1739 /* 1740 * vm_object_page_remove: 1741 * 1742 * For the given object, either frees or invalidates each of the 1743 * specified pages. In general, a page is freed. However, if a 1744 * page is wired for any reason other than the existence of a 1745 * managed, wired mapping, then it may be invalidated but not 1746 * removed from the object. Pages are specified by the given 1747 * range ["start", "end") and Boolean "clean_only". As a 1748 * special case, if "end" is zero, then the range extends from 1749 * "start" to the end of the object. If "clean_only" is TRUE, 1750 * then only the non-dirty pages within the specified range are 1751 * affected. 1752 * 1753 * In general, this operation should only be performed on objects 1754 * that contain managed pages. There are two exceptions. First, 1755 * it may be performed on the kernel and kmem objects. Second, 1756 * it may be used by msync(..., MS_INVALIDATE) to invalidate 1757 * device-backed pages. In both of these cases, "clean_only" 1758 * must be FALSE. 1759 * 1760 * The object must be locked. 1761 */ 1762 void 1763 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 1764 boolean_t clean_only) 1765 { 1766 vm_page_t p, next; 1767 int wirings; 1768 1769 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1770 if (object->resident_page_count == 0) 1771 goto skipmemq; 1772 1773 /* 1774 * Since physically-backed objects do not use managed pages, we can't 1775 * remove pages from the object (we must instead remove the page 1776 * references, and then destroy the object). 1777 */ 1778 KASSERT(object->type != OBJT_PHYS || object == kernel_object || 1779 object == kmem_object, 1780 ("attempt to remove pages from a physical object")); 1781 1782 vm_object_pip_add(object, 1); 1783 again: 1784 p = vm_page_find_least(object, start); 1785 1786 /* 1787 * Assert: the variable p is either (1) the page with the 1788 * least pindex greater than or equal to the parameter pindex 1789 * or (2) NULL. 1790 */ 1791 for (; 1792 p != NULL && (p->pindex < end || end == 0); 1793 p = next) { 1794 next = TAILQ_NEXT(p, listq); 1795 1796 /* 1797 * If the page is wired for any reason besides the 1798 * existence of managed, wired mappings, then it cannot 1799 * be freed. For example, fictitious pages, which 1800 * represent device memory, are inherently wired and 1801 * cannot be freed. They can, however, be invalidated 1802 * if "clean_only" is FALSE. 1803 */ 1804 vm_page_lock(p); 1805 if ((wirings = p->wire_count) != 0 && 1806 (wirings = pmap_page_wired_mappings(p)) != p->wire_count) { 1807 /* Fictitious pages do not have managed mappings. */ 1808 if ((p->flags & PG_FICTITIOUS) == 0) 1809 pmap_remove_all(p); 1810 /* Account for removal of managed, wired mappings. */ 1811 p->wire_count -= wirings; 1812 if (!clean_only) { 1813 p->valid = 0; 1814 vm_page_undirty(p); 1815 } 1816 vm_page_unlock(p); 1817 continue; 1818 } 1819 if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) 1820 goto again; 1821 KASSERT((p->flags & PG_FICTITIOUS) == 0, 1822 ("vm_object_page_remove: page %p is fictitious", p)); 1823 if (clean_only && p->valid) { 1824 pmap_remove_write(p); 1825 if (p->dirty) { 1826 vm_page_unlock(p); 1827 continue; 1828 } 1829 } 1830 pmap_remove_all(p); 1831 /* Account for removal of managed, wired mappings. */ 1832 if (wirings != 0) 1833 p->wire_count -= wirings; 1834 vm_page_free(p); 1835 vm_page_unlock(p); 1836 } 1837 vm_object_pip_wakeup(object); 1838 skipmemq: 1839 if (__predict_false(object->cache != NULL)) 1840 vm_page_cache_free(object, start, end); 1841 } 1842 1843 /* 1844 * Populate the specified range of the object with valid pages. Returns 1845 * TRUE if the range is successfully populated and FALSE otherwise. 1846 * 1847 * Note: This function should be optimized to pass a larger array of 1848 * pages to vm_pager_get_pages() before it is applied to a non- 1849 * OBJT_DEVICE object. 1850 * 1851 * The object must be locked. 1852 */ 1853 boolean_t 1854 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end) 1855 { 1856 vm_page_t m, ma[1]; 1857 vm_pindex_t pindex; 1858 int rv; 1859 1860 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1861 for (pindex = start; pindex < end; pindex++) { 1862 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | 1863 VM_ALLOC_RETRY); 1864 if (m->valid != VM_PAGE_BITS_ALL) { 1865 ma[0] = m; 1866 rv = vm_pager_get_pages(object, ma, 1, 0); 1867 m = vm_page_lookup(object, pindex); 1868 if (m == NULL) 1869 break; 1870 if (rv != VM_PAGER_OK) { 1871 vm_page_lock(m); 1872 vm_page_free(m); 1873 vm_page_unlock(m); 1874 break; 1875 } 1876 } 1877 /* 1878 * Keep "m" busy because a subsequent iteration may unlock 1879 * the object. 1880 */ 1881 } 1882 if (pindex > start) { 1883 m = vm_page_lookup(object, start); 1884 while (m != NULL && m->pindex < pindex) { 1885 vm_page_wakeup(m); 1886 m = TAILQ_NEXT(m, listq); 1887 } 1888 } 1889 return (pindex == end); 1890 } 1891 1892 /* 1893 * Routine: vm_object_coalesce 1894 * Function: Coalesces two objects backing up adjoining 1895 * regions of memory into a single object. 1896 * 1897 * returns TRUE if objects were combined. 1898 * 1899 * NOTE: Only works at the moment if the second object is NULL - 1900 * if it's not, which object do we lock first? 1901 * 1902 * Parameters: 1903 * prev_object First object to coalesce 1904 * prev_offset Offset into prev_object 1905 * prev_size Size of reference to prev_object 1906 * next_size Size of reference to the second object 1907 * reserved Indicator that extension region has 1908 * swap accounted for 1909 * 1910 * Conditions: 1911 * The object must *not* be locked. 1912 */ 1913 boolean_t 1914 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset, 1915 vm_size_t prev_size, vm_size_t next_size, boolean_t reserved) 1916 { 1917 vm_pindex_t next_pindex; 1918 1919 if (prev_object == NULL) 1920 return (TRUE); 1921 VM_OBJECT_LOCK(prev_object); 1922 if (prev_object->type != OBJT_DEFAULT && 1923 prev_object->type != OBJT_SWAP) { 1924 VM_OBJECT_UNLOCK(prev_object); 1925 return (FALSE); 1926 } 1927 1928 /* 1929 * Try to collapse the object first 1930 */ 1931 vm_object_collapse(prev_object); 1932 1933 /* 1934 * Can't coalesce if: . more than one reference . paged out . shadows 1935 * another object . has a copy elsewhere (any of which mean that the 1936 * pages not mapped to prev_entry may be in use anyway) 1937 */ 1938 if (prev_object->backing_object != NULL) { 1939 VM_OBJECT_UNLOCK(prev_object); 1940 return (FALSE); 1941 } 1942 1943 prev_size >>= PAGE_SHIFT; 1944 next_size >>= PAGE_SHIFT; 1945 next_pindex = OFF_TO_IDX(prev_offset) + prev_size; 1946 1947 if ((prev_object->ref_count > 1) && 1948 (prev_object->size != next_pindex)) { 1949 VM_OBJECT_UNLOCK(prev_object); 1950 return (FALSE); 1951 } 1952 1953 /* 1954 * Account for the charge. 1955 */ 1956 if (prev_object->uip != NULL) { 1957 1958 /* 1959 * If prev_object was charged, then this mapping, 1960 * althought not charged now, may become writable 1961 * later. Non-NULL uip in the object would prevent 1962 * swap reservation during enabling of the write 1963 * access, so reserve swap now. Failed reservation 1964 * cause allocation of the separate object for the map 1965 * entry, and swap reservation for this entry is 1966 * managed in appropriate time. 1967 */ 1968 if (!reserved && !swap_reserve_by_uid(ptoa(next_size), 1969 prev_object->uip)) { 1970 return (FALSE); 1971 } 1972 prev_object->charge += ptoa(next_size); 1973 } 1974 1975 /* 1976 * Remove any pages that may still be in the object from a previous 1977 * deallocation. 1978 */ 1979 if (next_pindex < prev_object->size) { 1980 vm_object_page_remove(prev_object, 1981 next_pindex, 1982 next_pindex + next_size, FALSE); 1983 if (prev_object->type == OBJT_SWAP) 1984 swap_pager_freespace(prev_object, 1985 next_pindex, next_size); 1986 #if 0 1987 if (prev_object->uip != NULL) { 1988 KASSERT(prev_object->charge >= 1989 ptoa(prev_object->size - next_pindex), 1990 ("object %p overcharged 1 %jx %jx", prev_object, 1991 (uintmax_t)next_pindex, (uintmax_t)next_size)); 1992 prev_object->charge -= ptoa(prev_object->size - 1993 next_pindex); 1994 } 1995 #endif 1996 } 1997 1998 /* 1999 * Extend the object if necessary. 2000 */ 2001 if (next_pindex + next_size > prev_object->size) 2002 prev_object->size = next_pindex + next_size; 2003 2004 VM_OBJECT_UNLOCK(prev_object); 2005 return (TRUE); 2006 } 2007 2008 void 2009 vm_object_set_writeable_dirty(vm_object_t object) 2010 { 2011 2012 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2013 if (object->type != OBJT_VNODE || 2014 (object->flags & OBJ_MIGHTBEDIRTY) != 0) 2015 return; 2016 vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); 2017 } 2018 2019 #include "opt_ddb.h" 2020 #ifdef DDB 2021 #include <sys/kernel.h> 2022 2023 #include <sys/cons.h> 2024 2025 #include <ddb/ddb.h> 2026 2027 static int 2028 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 2029 { 2030 vm_map_t tmpm; 2031 vm_map_entry_t tmpe; 2032 vm_object_t obj; 2033 int entcount; 2034 2035 if (map == 0) 2036 return 0; 2037 2038 if (entry == 0) { 2039 tmpe = map->header.next; 2040 entcount = map->nentries; 2041 while (entcount-- && (tmpe != &map->header)) { 2042 if (_vm_object_in_map(map, object, tmpe)) { 2043 return 1; 2044 } 2045 tmpe = tmpe->next; 2046 } 2047 } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 2048 tmpm = entry->object.sub_map; 2049 tmpe = tmpm->header.next; 2050 entcount = tmpm->nentries; 2051 while (entcount-- && tmpe != &tmpm->header) { 2052 if (_vm_object_in_map(tmpm, object, tmpe)) { 2053 return 1; 2054 } 2055 tmpe = tmpe->next; 2056 } 2057 } else if ((obj = entry->object.vm_object) != NULL) { 2058 for (; obj; obj = obj->backing_object) 2059 if (obj == object) { 2060 return 1; 2061 } 2062 } 2063 return 0; 2064 } 2065 2066 static int 2067 vm_object_in_map(vm_object_t object) 2068 { 2069 struct proc *p; 2070 2071 /* sx_slock(&allproc_lock); */ 2072 FOREACH_PROC_IN_SYSTEM(p) { 2073 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 2074 continue; 2075 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) { 2076 /* sx_sunlock(&allproc_lock); */ 2077 return 1; 2078 } 2079 } 2080 /* sx_sunlock(&allproc_lock); */ 2081 if (_vm_object_in_map(kernel_map, object, 0)) 2082 return 1; 2083 if (_vm_object_in_map(kmem_map, object, 0)) 2084 return 1; 2085 if (_vm_object_in_map(pager_map, object, 0)) 2086 return 1; 2087 if (_vm_object_in_map(buffer_map, object, 0)) 2088 return 1; 2089 return 0; 2090 } 2091 2092 DB_SHOW_COMMAND(vmochk, vm_object_check) 2093 { 2094 vm_object_t object; 2095 2096 /* 2097 * make sure that internal objs are in a map somewhere 2098 * and none have zero ref counts. 2099 */ 2100 TAILQ_FOREACH(object, &vm_object_list, object_list) { 2101 if (object->handle == NULL && 2102 (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) { 2103 if (object->ref_count == 0) { 2104 db_printf("vmochk: internal obj has zero ref count: %ld\n", 2105 (long)object->size); 2106 } 2107 if (!vm_object_in_map(object)) { 2108 db_printf( 2109 "vmochk: internal obj is not in a map: " 2110 "ref: %d, size: %lu: 0x%lx, backing_object: %p\n", 2111 object->ref_count, (u_long)object->size, 2112 (u_long)object->size, 2113 (void *)object->backing_object); 2114 } 2115 } 2116 } 2117 } 2118 2119 /* 2120 * vm_object_print: [ debug ] 2121 */ 2122 DB_SHOW_COMMAND(object, vm_object_print_static) 2123 { 2124 /* XXX convert args. */ 2125 vm_object_t object = (vm_object_t)addr; 2126 boolean_t full = have_addr; 2127 2128 vm_page_t p; 2129 2130 /* XXX count is an (unused) arg. Avoid shadowing it. */ 2131 #define count was_count 2132 2133 int count; 2134 2135 if (object == NULL) 2136 return; 2137 2138 db_iprintf( 2139 "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x uip %d charge %jx\n", 2140 object, (int)object->type, (uintmax_t)object->size, 2141 object->resident_page_count, object->ref_count, object->flags, 2142 object->uip ? object->uip->ui_uid : -1, (uintmax_t)object->charge); 2143 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n", 2144 object->shadow_count, 2145 object->backing_object ? object->backing_object->ref_count : 0, 2146 object->backing_object, (uintmax_t)object->backing_object_offset); 2147 2148 if (!full) 2149 return; 2150 2151 db_indent += 2; 2152 count = 0; 2153 TAILQ_FOREACH(p, &object->memq, listq) { 2154 if (count == 0) 2155 db_iprintf("memory:="); 2156 else if (count == 6) { 2157 db_printf("\n"); 2158 db_iprintf(" ..."); 2159 count = 0; 2160 } else 2161 db_printf(","); 2162 count++; 2163 2164 db_printf("(off=0x%jx,page=0x%jx)", 2165 (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p)); 2166 } 2167 if (count != 0) 2168 db_printf("\n"); 2169 db_indent -= 2; 2170 } 2171 2172 /* XXX. */ 2173 #undef count 2174 2175 /* XXX need this non-static entry for calling from vm_map_print. */ 2176 void 2177 vm_object_print( 2178 /* db_expr_t */ long addr, 2179 boolean_t have_addr, 2180 /* db_expr_t */ long count, 2181 char *modif) 2182 { 2183 vm_object_print_static(addr, have_addr, count, modif); 2184 } 2185 2186 DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 2187 { 2188 vm_object_t object; 2189 vm_pindex_t fidx; 2190 vm_paddr_t pa; 2191 vm_page_t m, prev_m; 2192 int rcount, nl, c; 2193 2194 nl = 0; 2195 TAILQ_FOREACH(object, &vm_object_list, object_list) { 2196 db_printf("new object: %p\n", (void *)object); 2197 if (nl > 18) { 2198 c = cngetc(); 2199 if (c != ' ') 2200 return; 2201 nl = 0; 2202 } 2203 nl++; 2204 rcount = 0; 2205 fidx = 0; 2206 pa = -1; 2207 TAILQ_FOREACH(m, &object->memq, listq) { 2208 if (m->pindex > 128) 2209 break; 2210 if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL && 2211 prev_m->pindex + 1 != m->pindex) { 2212 if (rcount) { 2213 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2214 (long)fidx, rcount, (long)pa); 2215 if (nl > 18) { 2216 c = cngetc(); 2217 if (c != ' ') 2218 return; 2219 nl = 0; 2220 } 2221 nl++; 2222 rcount = 0; 2223 } 2224 } 2225 if (rcount && 2226 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 2227 ++rcount; 2228 continue; 2229 } 2230 if (rcount) { 2231 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2232 (long)fidx, rcount, (long)pa); 2233 if (nl > 18) { 2234 c = cngetc(); 2235 if (c != ' ') 2236 return; 2237 nl = 0; 2238 } 2239 nl++; 2240 } 2241 fidx = m->pindex; 2242 pa = VM_PAGE_TO_PHYS(m); 2243 rcount = 1; 2244 } 2245 if (rcount) { 2246 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2247 (long)fidx, rcount, (long)pa); 2248 if (nl > 18) { 2249 c = cngetc(); 2250 if (c != ' ') 2251 return; 2252 nl = 0; 2253 } 2254 nl++; 2255 } 2256 } 2257 } 2258 #endif /* DDB */ 2259