1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 37 * 38 * 39 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 40 * All rights reserved. 41 * 42 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 43 * 44 * Permission to use, copy, modify and distribute this software and 45 * its documentation is hereby granted, provided that both the copyright 46 * notice and this permission notice appear in all copies of the 47 * software, derivative works or modified versions, and any portions 48 * thereof, and that both notices appear in supporting documentation. 49 * 50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 51 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 53 * 54 * Carnegie Mellon requests users of this software to return to 55 * 56 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 57 * School of Computer Science 58 * Carnegie Mellon University 59 * Pittsburgh PA 15213-3890 60 * 61 * any improvements or extensions that they make and grant Carnegie the 62 * rights to redistribute these changes. 63 * 64 * $Id: vm_object.c,v 1.14 1995/01/05 04:30:40 davidg Exp $ 65 */ 66 67 /* 68 * Virtual memory object module. 69 */ 70 71 #include <sys/param.h> 72 #include <sys/systm.h> 73 #include <sys/kernel.h> 74 #include <sys/proc.h> /* for curproc, pageproc */ 75 #include <sys/malloc.h> 76 #include <sys/vnode.h> 77 #include <sys/mount.h> 78 79 #include <vm/vm.h> 80 #include <vm/vm_page.h> 81 #include <vm/vm_pageout.h> 82 #include <vm/vm_pager.h> 83 #include <vm/swap_pager.h> 84 #include <vm/vnode_pager.h> 85 86 static void _vm_object_allocate(vm_size_t, vm_object_t); 87 static void vm_object_rcollapse(vm_object_t, vm_object_t); 88 89 /* 90 * Virtual memory objects maintain the actual data 91 * associated with allocated virtual memory. A given 92 * page of memory exists within exactly one object. 93 * 94 * An object is only deallocated when all "references" 95 * are given up. Only one "reference" to a given 96 * region of an object should be writeable. 97 * 98 * Associated with each object is a list of all resident 99 * memory pages belonging to that object; this list is 100 * maintained by the "vm_page" module, and locked by the object's 101 * lock. 102 * 103 * Each object also records a "pager" routine which is 104 * used to retrieve (and store) pages to the proper backing 105 * storage. In addition, objects may be backed by other 106 * objects from which they were virtual-copied. 107 * 108 * The only items within the object structure which are 109 * modified after time of creation are: 110 * reference count locked by object's lock 111 * pager routine locked by object's lock 112 * 113 */ 114 115 116 struct vm_object kernel_object_store; 117 struct vm_object kmem_object_store; 118 119 int vm_object_cache_max; 120 121 #define VM_OBJECT_HASH_COUNT 509 122 123 struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; 124 125 long object_collapses = 0; 126 long object_bypasses = 0; 127 128 static void 129 _vm_object_allocate(size, object) 130 vm_size_t size; 131 register vm_object_t object; 132 { 133 bzero(object, sizeof *object); 134 TAILQ_INIT(&object->memq); 135 TAILQ_INIT(&object->reverse_shadow_head); 136 vm_object_lock_init(object); 137 object->ref_count = 1; 138 object->resident_page_count = 0; 139 object->size = size; 140 object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ 141 object->paging_in_progress = 0; 142 object->copy = NULL; 143 144 /* 145 * Object starts out read-write, with no pager. 146 */ 147 148 object->pager = NULL; 149 object->paging_offset = 0; 150 object->shadow = NULL; 151 object->shadow_offset = (vm_offset_t) 0; 152 153 simple_lock(&vm_object_list_lock); 154 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 155 vm_object_count++; 156 cnt.v_nzfod += atop(size); 157 simple_unlock(&vm_object_list_lock); 158 } 159 160 /* 161 * vm_object_init: 162 * 163 * Initialize the VM objects module. 164 */ 165 void 166 vm_object_init(vm_offset_t nothing) 167 { 168 register int i; 169 170 TAILQ_INIT(&vm_object_cached_list); 171 TAILQ_INIT(&vm_object_list); 172 vm_object_count = 0; 173 simple_lock_init(&vm_cache_lock); 174 simple_lock_init(&vm_object_list_lock); 175 vm_object_cache_max = (cnt.v_page_count - 500) / 8; 176 177 for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) 178 TAILQ_INIT(&vm_object_hashtable[i]); 179 180 kernel_object = &kernel_object_store; 181 _vm_object_allocate(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 182 kernel_object); 183 184 kmem_object = &kmem_object_store; 185 _vm_object_allocate(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 186 kmem_object); 187 } 188 189 /* 190 * vm_object_allocate: 191 * 192 * Returns a new object with the given size. 193 */ 194 195 vm_object_t 196 vm_object_allocate(size) 197 vm_size_t size; 198 { 199 register vm_object_t result; 200 201 result = (vm_object_t) 202 malloc((u_long) sizeof *result, M_VMOBJ, M_WAITOK); 203 204 205 _vm_object_allocate(size, result); 206 207 return (result); 208 } 209 210 211 /* 212 * vm_object_reference: 213 * 214 * Gets another reference to the given object. 215 */ 216 inline void 217 vm_object_reference(object) 218 register vm_object_t object; 219 { 220 if (object == NULL) 221 return; 222 223 vm_object_lock(object); 224 object->ref_count++; 225 vm_object_unlock(object); 226 } 227 228 /* 229 * vm_object_deallocate: 230 * 231 * Release a reference to the specified object, 232 * gained either through a vm_object_allocate 233 * or a vm_object_reference call. When all references 234 * are gone, storage associated with this object 235 * may be relinquished. 236 * 237 * No object may be locked. 238 */ 239 void 240 vm_object_deallocate(object) 241 vm_object_t object; 242 { 243 vm_object_t temp; 244 245 while (object != NULL) { 246 247 /* 248 * The cache holds a reference (uncounted) to the object; we 249 * must lock it before removing the object. 250 */ 251 252 vm_object_cache_lock(); 253 254 /* 255 * Lose the reference 256 */ 257 vm_object_lock(object); 258 if (--(object->ref_count) != 0) { 259 if (object->ref_count == 1) { 260 if (object->reverse_shadow_head.tqh_first) { 261 ++object->reverse_shadow_head.tqh_first->ref_count; 262 if (vm_object_lock_try(object->reverse_shadow_head.tqh_first)) { 263 vm_object_rcollapse(object->reverse_shadow_head.tqh_first, object); 264 vm_object_unlock(object->reverse_shadow_head.tqh_first); 265 } 266 vm_object_deallocate(object->reverse_shadow_head.tqh_first); 267 } 268 } 269 vm_object_unlock(object); 270 /* 271 * If there are still references, then we are done. 272 */ 273 vm_object_cache_unlock(); 274 return; 275 } 276 /* 277 * See if this object can persist. If so, enter it in the 278 * cache, then deactivate all of its pages. 279 */ 280 281 if (object->flags & OBJ_CANPERSIST) { 282 283 TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 284 cached_list); 285 vm_object_cached++; 286 vm_object_cache_unlock(); 287 288 vm_object_unlock(object); 289 290 vm_object_cache_trim(); 291 return; 292 } 293 /* 294 * Make sure no one can look us up now. 295 */ 296 object->flags |= OBJ_DEAD; 297 vm_object_remove(object->pager); 298 vm_object_cache_unlock(); 299 300 temp = object->shadow; 301 if (temp) 302 TAILQ_REMOVE(&temp->reverse_shadow_head, object, reverse_shadow_list); 303 vm_object_terminate(object); 304 /* unlocks and deallocates object */ 305 object = temp; 306 } 307 } 308 309 /* 310 * vm_object_terminate actually destroys the specified object, freeing 311 * up all previously used resources. 312 * 313 * The object must be locked. 314 */ 315 void 316 vm_object_terminate(object) 317 register vm_object_t object; 318 { 319 register vm_page_t p, next; 320 vm_object_t shadow_object; 321 int s; 322 struct vnode *vp = NULL; 323 324 /* 325 * Detach the object from its shadow if we are the shadow's copy. 326 */ 327 if ((shadow_object = object->shadow) != NULL) { 328 vm_object_lock(shadow_object); 329 if (shadow_object->copy == object) 330 shadow_object->copy = NULL; 331 /* 332 else if (shadow_object->copy != NULL) 333 panic("vm_object_terminate: copy/shadow inconsistency"); 334 */ 335 vm_object_unlock(shadow_object); 336 } 337 if (object->pager && (object->pager->pg_type == PG_VNODE)) { 338 vn_pager_t vnp = object->pager->pg_data; 339 340 vp = vnp->vnp_vp; 341 VOP_FSYNC(vp, NOCRED, MNT_WAIT, NULL); 342 vinvalbuf(vp, 0, NOCRED, NULL, 0, 0); 343 } 344 /* 345 * Wait until the pageout daemon is through with the object. 346 */ 347 348 s = splhigh(); 349 while (object->paging_in_progress) { 350 vm_object_unlock(object); 351 tsleep((caddr_t) object, PVM, "objtrm", 0); 352 vm_object_lock(object); 353 } 354 splx(s); 355 356 /* 357 * While the paging system is locked, pull the object's pages off the 358 * active and inactive queues. This keeps the pageout daemon from 359 * playing with them during vm_pager_deallocate. 360 * 361 * We can't free the pages yet, because the object's pager may have to 362 * write them out before deallocating the paging space. 363 */ 364 365 for (p = object->memq.tqh_first; p; p = next) { 366 VM_PAGE_CHECK(p); 367 next = p->listq.tqe_next; 368 369 vm_page_lock_queues(); 370 if (p->flags & PG_CACHE) 371 vm_page_free(p); 372 else 373 vm_page_unqueue(p); 374 vm_page_unlock_queues(); 375 p = next; 376 } 377 378 if (object->paging_in_progress != 0) 379 panic("vm_object_deallocate: pageout in progress"); 380 381 /* 382 * Clean and free the pages, as appropriate. All references to the 383 * object are gone, so we don't need to lock it. 384 */ 385 386 if ((object->flags & OBJ_INTERNAL) == 0) { 387 (void) vm_object_page_clean(object, 0, 0, TRUE, TRUE); 388 } 389 /* 390 * one last time -- get rid of buffers that might have been created 391 * for the vm_object_page_clean 392 */ 393 if (vp != NULL) { 394 vm_object_unlock(object); 395 vinvalbuf(vp, 0, NOCRED, NULL, 0, 0); 396 vm_object_lock(object); 397 } 398 /* 399 * Now free the pages. For internal objects, this also removes them 400 * from paging queues. 401 */ 402 while ((p = object->memq.tqh_first) != NULL) { 403 VM_PAGE_CHECK(p); 404 vm_page_lock_queues(); 405 PAGE_WAKEUP(p); 406 vm_page_free(p); 407 cnt.v_pfree++; 408 vm_page_unlock_queues(); 409 } 410 vm_object_unlock(object); 411 412 /* 413 * Let the pager know object is dead. 414 */ 415 if (object->pager != NULL) 416 vm_pager_deallocate(object->pager); 417 418 simple_lock(&vm_object_list_lock); 419 TAILQ_REMOVE(&vm_object_list, object, object_list); 420 vm_object_count--; 421 simple_unlock(&vm_object_list_lock); 422 423 /* 424 * Free the space for the object. 425 */ 426 free((caddr_t) object, M_VMOBJ); 427 } 428 429 /* 430 * vm_object_page_clean 431 * 432 * Clean all dirty pages in the specified range of object. 433 * Leaves page on whatever queue it is currently on. 434 * 435 * Odd semantics: if start == end, we clean everything. 436 * 437 * The object must be locked. 438 */ 439 #if 1 440 boolean_t 441 vm_object_page_clean(object, start, end, syncio, de_queue) 442 register vm_object_t object; 443 register vm_offset_t start; 444 register vm_offset_t end; 445 boolean_t syncio; 446 boolean_t de_queue; 447 { 448 register vm_page_t p, nextp; 449 int size; 450 451 if (object->pager == NULL) 452 return 1; 453 454 if (start != end) { 455 start = trunc_page(start); 456 end = round_page(end); 457 } 458 size = end - start; 459 460 again: 461 /* 462 * Wait until the pageout daemon is through with the object. 463 */ 464 while (object->paging_in_progress) { 465 tsleep(object, PVM, "objpcw", 0); 466 } 467 468 nextp = object->memq.tqh_first; 469 while ((p = nextp) && ((start == end) || (size != 0))) { 470 nextp = p->listq.tqe_next; 471 if (start == end || (p->offset >= start && p->offset < end)) { 472 if ((p->flags & PG_BUSY) || p->busy) { 473 int s = splhigh(); 474 475 p->flags |= PG_WANTED; 476 tsleep(p, PVM, "objpcn", 0); 477 splx(s); 478 goto again; 479 } 480 size -= PAGE_SIZE; 481 482 vm_page_test_dirty(p); 483 484 if ((p->dirty & p->valid) != 0) { 485 vm_pageout_clean(p, VM_PAGEOUT_FORCE); 486 goto again; 487 } 488 } 489 } 490 wakeup((caddr_t) object); 491 return 1; 492 } 493 #endif 494 /* 495 * vm_object_page_clean 496 * 497 * Clean all dirty pages in the specified range of object. 498 * If syncio is TRUE, page cleaning is done synchronously. 499 * If de_queue is TRUE, pages are removed from any paging queue 500 * they were on, otherwise they are left on whatever queue they 501 * were on before the cleaning operation began. 502 * 503 * Odd semantics: if start == end, we clean everything. 504 * 505 * The object must be locked. 506 * 507 * Returns TRUE if all was well, FALSE if there was a pager error 508 * somewhere. We attempt to clean (and dequeue) all pages regardless 509 * of where an error occurs. 510 */ 511 #if 0 512 boolean_t 513 vm_object_page_clean(object, start, end, syncio, de_queue) 514 register vm_object_t object; 515 register vm_offset_t start; 516 register vm_offset_t end; 517 boolean_t syncio; 518 boolean_t de_queue; 519 { 520 register vm_page_t p; 521 int onqueue; 522 boolean_t noerror = TRUE; 523 524 if (object == NULL) 525 return (TRUE); 526 527 /* 528 * If it is an internal object and there is no pager, attempt to 529 * allocate one. Note that vm_object_collapse may relocate one from a 530 * collapsed object so we must recheck afterward. 531 */ 532 if ((object->flags & OBJ_INTERNAL) && object->pager == NULL) { 533 vm_object_collapse(object); 534 if (object->pager == NULL) { 535 vm_pager_t pager; 536 537 vm_object_unlock(object); 538 pager = vm_pager_allocate(PG_DFLT, (caddr_t) 0, 539 object->size, VM_PROT_ALL, 540 (vm_offset_t) 0); 541 if (pager) 542 vm_object_setpager(object, pager, 0, FALSE); 543 vm_object_lock(object); 544 } 545 } 546 if (object->pager == NULL) 547 return (FALSE); 548 549 again: 550 /* 551 * Wait until the pageout daemon is through with the object. 552 */ 553 while (object->paging_in_progress) { 554 vm_object_sleep((int) object, object, FALSE); 555 vm_object_lock(object); 556 } 557 /* 558 * Loop through the object page list cleaning as necessary. 559 */ 560 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 561 onqueue = 0; 562 if ((start == end || p->offset >= start && p->offset < end) && 563 !(p->flags & PG_FICTITIOUS)) { 564 vm_page_test_dirty(p); 565 /* 566 * Remove the page from any paging queue. This needs 567 * to be done if either we have been explicitly asked 568 * to do so or it is about to be cleaned (see comment 569 * below). 570 */ 571 if (de_queue || (p->dirty & p->valid)) { 572 vm_page_lock_queues(); 573 if (p->flags & PG_ACTIVE) { 574 TAILQ_REMOVE(&vm_page_queue_active, 575 p, pageq); 576 p->flags &= ~PG_ACTIVE; 577 cnt.v_active_count--; 578 onqueue = 1; 579 } else if (p->flags & PG_INACTIVE) { 580 TAILQ_REMOVE(&vm_page_queue_inactive, 581 p, pageq); 582 p->flags &= ~PG_INACTIVE; 583 cnt.v_inactive_count--; 584 onqueue = -1; 585 } else 586 onqueue = 0; 587 vm_page_unlock_queues(); 588 } 589 /* 590 * To ensure the state of the page doesn't change 591 * during the clean operation we do two things. First 592 * we set the busy bit and write-protect all mappings 593 * to ensure that write accesses to the page block (in 594 * vm_fault). Second, we remove the page from any 595 * paging queue to foil the pageout daemon 596 * (vm_pageout_scan). 597 */ 598 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 599 if (p->dirty & p->valid) { 600 p->flags |= PG_BUSY; 601 object->paging_in_progress++; 602 vm_object_unlock(object); 603 /* 604 * XXX if put fails we mark the page as clean 605 * to avoid an infinite loop. Will loose 606 * changes to the page. 607 */ 608 if (vm_pager_put(object->pager, p, syncio)) { 609 printf("%s: pager_put error\n", 610 "vm_object_page_clean"); 611 p->dirty = 0; 612 noerror = FALSE; 613 } 614 vm_object_lock(object); 615 object->paging_in_progress--; 616 if (!de_queue && onqueue) { 617 vm_page_lock_queues(); 618 if (onqueue > 0) 619 vm_page_activate(p); 620 else 621 vm_page_deactivate(p); 622 vm_page_unlock_queues(); 623 } 624 PAGE_WAKEUP(p); 625 goto again; 626 } 627 } 628 } 629 return (noerror); 630 } 631 #endif 632 633 /* 634 * vm_object_deactivate_pages 635 * 636 * Deactivate all pages in the specified object. (Keep its pages 637 * in memory even though it is no longer referenced.) 638 * 639 * The object must be locked. 640 */ 641 void 642 vm_object_deactivate_pages(object) 643 register vm_object_t object; 644 { 645 register vm_page_t p, next; 646 647 for (p = object->memq.tqh_first; p != NULL; p = next) { 648 next = p->listq.tqe_next; 649 vm_page_lock_queues(); 650 vm_page_deactivate(p); 651 vm_page_unlock_queues(); 652 } 653 } 654 655 /* 656 * Trim the object cache to size. 657 */ 658 void 659 vm_object_cache_trim() 660 { 661 register vm_object_t object; 662 663 vm_object_cache_lock(); 664 while (vm_object_cached > vm_object_cache_max) { 665 object = vm_object_cached_list.tqh_first; 666 vm_object_cache_unlock(); 667 668 if (object != vm_object_lookup(object->pager)) 669 panic("vm_object_cache_trim: I'm sooo confused."); 670 671 pager_cache(object, FALSE); 672 673 vm_object_cache_lock(); 674 } 675 vm_object_cache_unlock(); 676 } 677 678 679 /* 680 * vm_object_pmap_copy: 681 * 682 * Makes all physical pages in the specified 683 * object range copy-on-write. No writeable 684 * references to these pages should remain. 685 * 686 * The object must *not* be locked. 687 */ 688 void 689 vm_object_pmap_copy(object, start, end) 690 register vm_object_t object; 691 register vm_offset_t start; 692 register vm_offset_t end; 693 { 694 register vm_page_t p; 695 696 if (object == NULL) 697 return; 698 699 vm_object_lock(object); 700 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 701 if ((start <= p->offset) && (p->offset < end)) { 702 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); 703 p->flags |= PG_COPYONWRITE; 704 } 705 } 706 vm_object_unlock(object); 707 } 708 709 /* 710 * vm_object_pmap_remove: 711 * 712 * Removes all physical pages in the specified 713 * object range from all physical maps. 714 * 715 * The object must *not* be locked. 716 */ 717 void 718 vm_object_pmap_remove(object, start, end) 719 register vm_object_t object; 720 register vm_offset_t start; 721 register vm_offset_t end; 722 { 723 register vm_page_t p; 724 int s; 725 726 if (object == NULL) 727 return; 728 ++object->paging_in_progress; 729 730 vm_object_lock(object); 731 again: 732 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 733 if ((start <= p->offset) && (p->offset < end)) { 734 s = splhigh(); 735 if ((p->flags & PG_BUSY) || p->busy) { 736 p->flags |= PG_WANTED; 737 tsleep((caddr_t) p, PVM, "vmopmr", 0); 738 splx(s); 739 goto again; 740 } 741 splx(s); 742 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 743 } 744 } 745 vm_object_unlock(object); 746 --object->paging_in_progress; 747 if (object->paging_in_progress == 0) 748 wakeup((caddr_t) object); 749 } 750 751 /* 752 * vm_object_copy: 753 * 754 * Create a new object which is a copy of an existing 755 * object, and mark all of the pages in the existing 756 * object 'copy-on-write'. The new object has one reference. 757 * Returns the new object. 758 * 759 * May defer the copy until later if the object is not backed 760 * up by a non-default pager. 761 */ 762 void 763 vm_object_copy(src_object, src_offset, size, 764 dst_object, dst_offset, src_needs_copy) 765 register vm_object_t src_object; 766 vm_offset_t src_offset; 767 vm_size_t size; 768 vm_object_t *dst_object;/* OUT */ 769 vm_offset_t *dst_offset;/* OUT */ 770 boolean_t *src_needs_copy; /* OUT */ 771 { 772 register vm_object_t new_copy; 773 register vm_object_t old_copy; 774 vm_offset_t new_start, new_end; 775 776 register vm_page_t p; 777 778 if (src_object == NULL) { 779 /* 780 * Nothing to copy 781 */ 782 *dst_object = NULL; 783 *dst_offset = 0; 784 *src_needs_copy = FALSE; 785 return; 786 } 787 /* 788 * If the object's pager is null_pager or the default pager, we don't 789 * have to make a copy of it. Instead, we set the needs copy flag and 790 * make a shadow later. 791 */ 792 793 vm_object_lock(src_object); 794 795 /* 796 * Try to collapse the object before copying it. 797 */ 798 799 vm_object_collapse(src_object); 800 801 if (src_object->pager == NULL || 802 src_object->pager->pg_type == PG_SWAP || 803 (src_object->flags & OBJ_INTERNAL)) { 804 805 /* 806 * Make another reference to the object 807 */ 808 src_object->ref_count++; 809 810 /* 811 * Mark all of the pages copy-on-write. 812 */ 813 for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) 814 if (src_offset <= p->offset && 815 p->offset < src_offset + size) 816 p->flags |= PG_COPYONWRITE; 817 vm_object_unlock(src_object); 818 819 *dst_object = src_object; 820 *dst_offset = src_offset; 821 822 /* 823 * Must make a shadow when write is desired 824 */ 825 *src_needs_copy = TRUE; 826 return; 827 } 828 /* 829 * If the object has a pager, the pager wants to see all of the 830 * changes. We need a copy-object for the changed pages. 831 * 832 * If there is a copy-object, and it is empty, no changes have been made 833 * to the object since the copy-object was made. We can use the same 834 * copy- object. 835 */ 836 837 Retry1: 838 old_copy = src_object->copy; 839 if (old_copy != NULL) { 840 /* 841 * Try to get the locks (out of order) 842 */ 843 if (!vm_object_lock_try(old_copy)) { 844 vm_object_unlock(src_object); 845 846 /* should spin a bit here... */ 847 tsleep((caddr_t) old_copy, PVM, "cpylck", 1); 848 vm_object_lock(src_object); 849 goto Retry1; 850 } 851 if (old_copy->resident_page_count == 0 && 852 old_copy->pager == NULL) { 853 /* 854 * Return another reference to the existing 855 * copy-object. 856 */ 857 old_copy->ref_count++; 858 vm_object_unlock(old_copy); 859 vm_object_unlock(src_object); 860 *dst_object = old_copy; 861 *dst_offset = src_offset; 862 *src_needs_copy = FALSE; 863 return; 864 } 865 vm_object_unlock(old_copy); 866 } 867 vm_object_unlock(src_object); 868 869 /* 870 * If the object has a pager, the pager wants to see all of the 871 * changes. We must make a copy-object and put the changed pages 872 * there. 873 * 874 * The copy-object is always made large enough to completely shadow the 875 * original object, since it may have several users who want to shadow 876 * the original object at different points. 877 */ 878 879 new_copy = vm_object_allocate(src_object->size); 880 881 Retry2: 882 vm_object_lock(src_object); 883 /* 884 * Copy object may have changed while we were unlocked 885 */ 886 old_copy = src_object->copy; 887 if (old_copy != NULL) { 888 /* 889 * Try to get the locks (out of order) 890 */ 891 if (!vm_object_lock_try(old_copy)) { 892 vm_object_unlock(src_object); 893 tsleep((caddr_t) old_copy, PVM, "cpylck", 1); 894 goto Retry2; 895 } 896 /* 897 * Consistency check 898 */ 899 if (old_copy->shadow != src_object || 900 old_copy->shadow_offset != (vm_offset_t) 0) 901 panic("vm_object_copy: copy/shadow inconsistency"); 902 903 /* 904 * Make the old copy-object shadow the new one. It will 905 * receive no more pages from the original object. 906 */ 907 908 src_object->ref_count--; /* remove ref. from old_copy */ 909 if (old_copy->shadow) 910 TAILQ_REMOVE(&old_copy->shadow->reverse_shadow_head, old_copy, reverse_shadow_list); 911 old_copy->shadow = new_copy; 912 TAILQ_INSERT_TAIL(&old_copy->shadow->reverse_shadow_head, old_copy, reverse_shadow_list); 913 new_copy->ref_count++; /* locking not needed - we have the 914 * only pointer */ 915 vm_object_unlock(old_copy); /* done with old_copy */ 916 } 917 new_start = (vm_offset_t) 0; /* always shadow original at 0 */ 918 new_end = (vm_offset_t) new_copy->size; /* for the whole object */ 919 920 /* 921 * Point the new copy at the existing object. 922 */ 923 924 new_copy->shadow = src_object; 925 TAILQ_INSERT_TAIL(&new_copy->shadow->reverse_shadow_head, new_copy, reverse_shadow_list); 926 new_copy->shadow_offset = new_start; 927 src_object->ref_count++; 928 src_object->copy = new_copy; 929 930 /* 931 * Mark all the affected pages of the existing object copy-on-write. 932 */ 933 for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) 934 if ((new_start <= p->offset) && (p->offset < new_end)) 935 p->flags |= PG_COPYONWRITE; 936 937 vm_object_unlock(src_object); 938 939 *dst_object = new_copy; 940 *dst_offset = src_offset - new_start; 941 *src_needs_copy = FALSE; 942 } 943 944 /* 945 * vm_object_shadow: 946 * 947 * Create a new object which is backed by the 948 * specified existing object range. The source 949 * object reference is deallocated. 950 * 951 * The new object and offset into that object 952 * are returned in the source parameters. 953 */ 954 955 void 956 vm_object_shadow(object, offset, length) 957 vm_object_t *object; /* IN/OUT */ 958 vm_offset_t *offset; /* IN/OUT */ 959 vm_size_t length; 960 { 961 register vm_object_t source; 962 register vm_object_t result; 963 964 source = *object; 965 966 /* 967 * Allocate a new object with the given length 968 */ 969 970 if ((result = vm_object_allocate(length)) == NULL) 971 panic("vm_object_shadow: no object for shadowing"); 972 973 /* 974 * The new object shadows the source object, adding a reference to it. 975 * Our caller changes his reference to point to the new object, 976 * removing a reference to the source object. Net result: no change 977 * of reference count. 978 */ 979 result->shadow = source; 980 if (source) 981 TAILQ_INSERT_TAIL(&result->shadow->reverse_shadow_head, result, reverse_shadow_list); 982 983 /* 984 * Store the offset into the source object, and fix up the offset into 985 * the new object. 986 */ 987 988 result->shadow_offset = *offset; 989 990 /* 991 * Return the new things 992 */ 993 994 *offset = 0; 995 *object = result; 996 } 997 998 /* 999 * Set the specified object's pager to the specified pager. 1000 */ 1001 1002 void 1003 vm_object_setpager(object, pager, paging_offset, 1004 read_only) 1005 vm_object_t object; 1006 vm_pager_t pager; 1007 vm_offset_t paging_offset; 1008 boolean_t read_only; 1009 { 1010 vm_object_lock(object); /* XXX ? */ 1011 if (object->pager && object->pager != pager) { 1012 panic("!!!pager already allocated!!!\n"); 1013 } 1014 object->pager = pager; 1015 object->paging_offset = paging_offset; 1016 vm_object_unlock(object); /* XXX ? */ 1017 } 1018 1019 /* 1020 * vm_object_hash hashes the pager/id pair. 1021 */ 1022 1023 #define vm_object_hash(pager) \ 1024 (((unsigned)pager >> 5)%VM_OBJECT_HASH_COUNT) 1025 1026 /* 1027 * vm_object_lookup looks in the object cache for an object with the 1028 * specified pager and paging id. 1029 */ 1030 1031 vm_object_t 1032 vm_object_lookup(pager) 1033 vm_pager_t pager; 1034 { 1035 register vm_object_hash_entry_t entry; 1036 vm_object_t object; 1037 1038 cnt.v_lookups++; 1039 vm_object_cache_lock(); 1040 1041 for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; 1042 entry != NULL; 1043 entry = entry->hash_links.tqe_next) { 1044 object = entry->object; 1045 if (object->pager == pager) { 1046 vm_object_lock(object); 1047 if (object->ref_count == 0) { 1048 TAILQ_REMOVE(&vm_object_cached_list, object, 1049 cached_list); 1050 vm_object_cached--; 1051 } 1052 object->ref_count++; 1053 vm_object_unlock(object); 1054 vm_object_cache_unlock(); 1055 cnt.v_hits++; 1056 return (object); 1057 } 1058 } 1059 1060 vm_object_cache_unlock(); 1061 return (NULL); 1062 } 1063 1064 /* 1065 * vm_object_enter enters the specified object/pager/id into 1066 * the hash table. 1067 */ 1068 1069 void 1070 vm_object_enter(object, pager) 1071 vm_object_t object; 1072 vm_pager_t pager; 1073 { 1074 struct vm_object_hash_head *bucket; 1075 register vm_object_hash_entry_t entry; 1076 1077 /* 1078 * We don't cache null objects, and we can't cache objects with the 1079 * null pager. 1080 */ 1081 1082 if (object == NULL) 1083 return; 1084 if (pager == NULL) 1085 return; 1086 1087 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 1088 entry = (vm_object_hash_entry_t) 1089 malloc((u_long) sizeof *entry, M_VMOBJHASH, M_WAITOK); 1090 entry->object = object; 1091 object->flags |= OBJ_CANPERSIST; 1092 1093 vm_object_cache_lock(); 1094 TAILQ_INSERT_TAIL(bucket, entry, hash_links); 1095 vm_object_cache_unlock(); 1096 } 1097 1098 /* 1099 * vm_object_remove: 1100 * 1101 * Remove the pager from the hash table. 1102 * Note: This assumes that the object cache 1103 * is locked. XXX this should be fixed 1104 * by reorganizing vm_object_deallocate. 1105 */ 1106 void 1107 vm_object_remove(pager) 1108 register vm_pager_t pager; 1109 { 1110 struct vm_object_hash_head *bucket; 1111 register vm_object_hash_entry_t entry; 1112 register vm_object_t object; 1113 1114 bucket = &vm_object_hashtable[vm_object_hash(pager)]; 1115 1116 for (entry = bucket->tqh_first; 1117 entry != NULL; 1118 entry = entry->hash_links.tqe_next) { 1119 object = entry->object; 1120 if (object->pager == pager) { 1121 TAILQ_REMOVE(bucket, entry, hash_links); 1122 free((caddr_t) entry, M_VMOBJHASH); 1123 break; 1124 } 1125 } 1126 } 1127 1128 static void 1129 vm_object_rcollapse(object, sobject) 1130 register vm_object_t object, sobject; 1131 { 1132 register vm_object_t backing_object; 1133 register vm_offset_t backing_offset, new_offset; 1134 register vm_page_t p, pp; 1135 register vm_size_t size; 1136 int s; 1137 1138 if (!object) 1139 return; 1140 backing_object = object->shadow; 1141 if (backing_object != sobject) { 1142 printf("backing obj != sobject!!!\n"); 1143 return; 1144 } 1145 if (!backing_object) 1146 return; 1147 if ((backing_object->flags & OBJ_INTERNAL) == 0) 1148 return; 1149 if (backing_object->shadow != NULL && 1150 backing_object->shadow->copy == backing_object) 1151 return; 1152 if (backing_object->ref_count != 1) 1153 return; 1154 1155 backing_object->ref_count += 2; 1156 s = splbio(); 1157 while (backing_object->paging_in_progress) { 1158 tsleep(backing_object, PVM, "rcolow", 0); 1159 } 1160 splx(s); 1161 1162 backing_offset = object->shadow_offset; 1163 size = object->size; 1164 p = backing_object->memq.tqh_first; 1165 while (p) { 1166 vm_page_t next; 1167 1168 next = p->listq.tqe_next; 1169 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1170 new_offset = (p->offset - backing_offset); 1171 if (p->offset < backing_offset || 1172 new_offset >= size) { 1173 if (backing_object->pager) 1174 swap_pager_freespace(backing_object->pager, 1175 backing_object->paging_offset + p->offset, PAGE_SIZE); 1176 vm_page_lock_queues(); 1177 vm_page_free(p); 1178 vm_page_unlock_queues(); 1179 } else { 1180 pp = vm_page_lookup(object, new_offset); 1181 if (pp != NULL || 1182 (object->pager && 1183 vm_pager_has_page(object->pager, object->paging_offset + new_offset))) { 1184 if (backing_object->pager) 1185 swap_pager_freespace(backing_object->pager, 1186 backing_object->paging_offset + p->offset, PAGE_SIZE); 1187 vm_page_lock_queues(); 1188 vm_page_free(p); 1189 vm_page_unlock_queues(); 1190 } else { 1191 if (!backing_object->pager || 1192 !vm_pager_has_page(backing_object->pager, backing_object->paging_offset + p->offset)) 1193 vm_page_rename(p, object, new_offset); 1194 } 1195 } 1196 p = next; 1197 } 1198 backing_object->ref_count -= 2; 1199 } 1200 1201 /* 1202 * this version of collapse allows the operation to occur earlier and 1203 * when paging_in_progress is true for an object... This is not a complete 1204 * operation, but should plug 99.9% of the rest of the leaks. 1205 */ 1206 static void 1207 vm_object_qcollapse(object) 1208 register vm_object_t object; 1209 { 1210 register vm_object_t backing_object; 1211 register vm_offset_t backing_offset, new_offset; 1212 register vm_page_t p, pp; 1213 register vm_size_t size; 1214 1215 backing_object = object->shadow; 1216 if (!backing_object) 1217 return; 1218 if ((backing_object->flags & OBJ_INTERNAL) == 0) 1219 return; 1220 if (backing_object->shadow != NULL && 1221 backing_object->shadow->copy == backing_object) 1222 return; 1223 if (backing_object->ref_count != 1) 1224 return; 1225 1226 backing_object->ref_count += 2; 1227 1228 backing_offset = object->shadow_offset; 1229 size = object->size; 1230 p = backing_object->memq.tqh_first; 1231 while (p) { 1232 vm_page_t next; 1233 1234 next = p->listq.tqe_next; 1235 if ((p->flags & (PG_BUSY | PG_FICTITIOUS | PG_CACHE)) || 1236 !p->valid || p->hold_count || p->wire_count || p->busy || p->bmapped) { 1237 p = next; 1238 continue; 1239 } 1240 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1241 new_offset = (p->offset - backing_offset); 1242 if (p->offset < backing_offset || 1243 new_offset >= size) { 1244 if (backing_object->pager) 1245 swap_pager_freespace(backing_object->pager, 1246 backing_object->paging_offset + p->offset, PAGE_SIZE); 1247 vm_page_lock_queues(); 1248 vm_page_free(p); 1249 vm_page_unlock_queues(); 1250 } else { 1251 pp = vm_page_lookup(object, new_offset); 1252 if (pp != NULL || (object->pager && vm_pager_has_page(object->pager, 1253 object->paging_offset + new_offset))) { 1254 if (backing_object->pager) 1255 swap_pager_freespace(backing_object->pager, 1256 backing_object->paging_offset + p->offset, PAGE_SIZE); 1257 vm_page_lock_queues(); 1258 vm_page_free(p); 1259 vm_page_unlock_queues(); 1260 } else { 1261 if (!backing_object->pager || 1262 !vm_pager_has_page(backing_object->pager, backing_object->paging_offset + p->offset)) 1263 vm_page_rename(p, object, new_offset); 1264 } 1265 } 1266 p = next; 1267 } 1268 backing_object->ref_count -= 2; 1269 } 1270 1271 boolean_t vm_object_collapse_allowed = TRUE; 1272 1273 /* 1274 * vm_object_collapse: 1275 * 1276 * Collapse an object with the object backing it. 1277 * Pages in the backing object are moved into the 1278 * parent, and the backing object is deallocated. 1279 * 1280 * Requires that the object be locked and the page 1281 * queues be unlocked. 1282 * 1283 * This routine has significant changes by John S. Dyson 1284 * to fix some swap memory leaks. 18 Dec 93 1285 * 1286 */ 1287 void 1288 vm_object_collapse(object) 1289 register vm_object_t object; 1290 1291 { 1292 register vm_object_t backing_object; 1293 register vm_offset_t backing_offset; 1294 register vm_size_t size; 1295 register vm_offset_t new_offset; 1296 register vm_page_t p, pp; 1297 1298 if (!vm_object_collapse_allowed) 1299 return; 1300 1301 while (TRUE) { 1302 /* 1303 * Verify that the conditions are right for collapse: 1304 * 1305 * The object exists and no pages in it are currently being paged 1306 * out. 1307 */ 1308 if (object == NULL) 1309 return; 1310 if (object->paging_in_progress != 0) { 1311 if (object->shadow) 1312 vm_object_qcollapse(object); 1313 return; 1314 } 1315 /* 1316 * There is a backing object, and 1317 */ 1318 1319 if ((backing_object = object->shadow) == NULL) 1320 return; 1321 1322 vm_object_lock(backing_object); 1323 /* 1324 * ... The backing object is not read_only, and no pages in 1325 * the backing object are currently being paged out. The 1326 * backing object is internal. 1327 */ 1328 1329 if ((backing_object->flags & OBJ_INTERNAL) == 0 || 1330 backing_object->paging_in_progress != 0) { 1331 vm_object_unlock(backing_object); 1332 vm_object_qcollapse(object); 1333 return; 1334 } 1335 /* 1336 * The backing object can't be a copy-object: the 1337 * shadow_offset for the copy-object must stay as 0. 1338 * Furthermore (for the 'we have all the pages' case), if we 1339 * bypass backing_object and just shadow the next object in 1340 * the chain, old pages from that object would then have to be 1341 * copied BOTH into the (former) backing_object and into the 1342 * parent object. 1343 */ 1344 if (backing_object->shadow != NULL && 1345 backing_object->shadow->copy == backing_object) { 1346 vm_object_unlock(backing_object); 1347 return; 1348 } 1349 /* 1350 * we can deal only with the swap pager 1351 */ 1352 if ((object->pager && 1353 object->pager->pg_type != PG_SWAP) || 1354 (backing_object->pager && 1355 backing_object->pager->pg_type != PG_SWAP)) { 1356 vm_object_unlock(backing_object); 1357 return; 1358 } 1359 /* 1360 * We know that we can either collapse the backing object (if 1361 * the parent is the only reference to it) or (perhaps) remove 1362 * the parent's reference to it. 1363 */ 1364 1365 backing_offset = object->shadow_offset; 1366 size = object->size; 1367 1368 /* 1369 * If there is exactly one reference to the backing object, we 1370 * can collapse it into the parent. 1371 */ 1372 1373 if (backing_object->ref_count == 1) { 1374 1375 /* 1376 * We can collapse the backing object. 1377 * 1378 * Move all in-memory pages from backing_object to the 1379 * parent. Pages that have been paged out will be 1380 * overwritten by any of the parent's pages that 1381 * shadow them. 1382 */ 1383 1384 while ((p = backing_object->memq.tqh_first) != 0) { 1385 1386 new_offset = (p->offset - backing_offset); 1387 1388 /* 1389 * If the parent has a page here, or if this 1390 * page falls outside the parent, dispose of 1391 * it. 1392 * 1393 * Otherwise, move it as planned. 1394 */ 1395 1396 if (p->offset < backing_offset || 1397 new_offset >= size) { 1398 vm_page_lock_queues(); 1399 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1400 PAGE_WAKEUP(p); 1401 vm_page_free(p); 1402 vm_page_unlock_queues(); 1403 } else { 1404 pp = vm_page_lookup(object, new_offset); 1405 if (pp != NULL || (object->pager && vm_pager_has_page(object->pager, 1406 object->paging_offset + new_offset))) { 1407 vm_page_lock_queues(); 1408 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1409 PAGE_WAKEUP(p); 1410 vm_page_free(p); 1411 vm_page_unlock_queues(); 1412 } else { 1413 vm_page_rename(p, object, new_offset); 1414 } 1415 } 1416 } 1417 1418 /* 1419 * Move the pager from backing_object to object. 1420 */ 1421 1422 if (backing_object->pager) { 1423 backing_object->paging_in_progress++; 1424 if (object->pager) { 1425 vm_pager_t bopager; 1426 1427 object->paging_in_progress++; 1428 /* 1429 * copy shadow object pages into ours 1430 * and destroy unneeded pages in 1431 * shadow object. 1432 */ 1433 bopager = backing_object->pager; 1434 backing_object->pager = NULL; 1435 vm_object_remove(backing_object->pager); 1436 swap_pager_copy( 1437 bopager, backing_object->paging_offset, 1438 object->pager, object->paging_offset, 1439 object->shadow_offset); 1440 object->paging_in_progress--; 1441 if (object->paging_in_progress == 0) 1442 wakeup((caddr_t) object); 1443 } else { 1444 object->paging_in_progress++; 1445 /* 1446 * grab the shadow objects pager 1447 */ 1448 object->pager = backing_object->pager; 1449 object->paging_offset = backing_object->paging_offset + backing_offset; 1450 vm_object_remove(backing_object->pager); 1451 backing_object->pager = NULL; 1452 /* 1453 * free unnecessary blocks 1454 */ 1455 swap_pager_freespace(object->pager, 0, object->paging_offset); 1456 object->paging_in_progress--; 1457 if (object->paging_in_progress == 0) 1458 wakeup((caddr_t) object); 1459 } 1460 backing_object->paging_in_progress--; 1461 if (backing_object->paging_in_progress == 0) 1462 wakeup((caddr_t) backing_object); 1463 } 1464 /* 1465 * Object now shadows whatever backing_object did. 1466 * Note that the reference to backing_object->shadow 1467 * moves from within backing_object to within object. 1468 */ 1469 1470 TAILQ_REMOVE(&object->shadow->reverse_shadow_head, object, 1471 reverse_shadow_list); 1472 if (backing_object->shadow) 1473 TAILQ_REMOVE(&backing_object->shadow->reverse_shadow_head, 1474 backing_object, reverse_shadow_list); 1475 object->shadow = backing_object->shadow; 1476 if (object->shadow) 1477 TAILQ_INSERT_TAIL(&object->shadow->reverse_shadow_head, 1478 object, reverse_shadow_list); 1479 1480 object->shadow_offset += backing_object->shadow_offset; 1481 if (object->shadow != NULL && 1482 object->shadow->copy != NULL) { 1483 panic("vm_object_collapse: we collapsed a copy-object!"); 1484 } 1485 /* 1486 * Discard backing_object. 1487 * 1488 * Since the backing object has no pages, no pager left, 1489 * and no object references within it, all that is 1490 * necessary is to dispose of it. 1491 */ 1492 1493 vm_object_unlock(backing_object); 1494 1495 simple_lock(&vm_object_list_lock); 1496 TAILQ_REMOVE(&vm_object_list, backing_object, 1497 object_list); 1498 vm_object_count--; 1499 simple_unlock(&vm_object_list_lock); 1500 1501 free((caddr_t) backing_object, M_VMOBJ); 1502 1503 object_collapses++; 1504 } else { 1505 /* 1506 * If all of the pages in the backing object are 1507 * shadowed by the parent object, the parent object no 1508 * longer has to shadow the backing object; it can 1509 * shadow the next one in the chain. 1510 * 1511 * The backing object must not be paged out - we'd have 1512 * to check all of the paged-out pages, as well. 1513 */ 1514 1515 if (backing_object->pager != NULL) { 1516 vm_object_unlock(backing_object); 1517 return; 1518 } 1519 /* 1520 * Should have a check for a 'small' number of pages 1521 * here. 1522 */ 1523 1524 for (p = backing_object->memq.tqh_first; p; p = p->listq.tqe_next) { 1525 new_offset = (p->offset - backing_offset); 1526 1527 /* 1528 * If the parent has a page here, or if this 1529 * page falls outside the parent, keep going. 1530 * 1531 * Otherwise, the backing_object must be left in 1532 * the chain. 1533 */ 1534 1535 if (p->offset >= backing_offset && 1536 new_offset <= size && 1537 ((pp = vm_page_lookup(object, new_offset)) == NULL || 1538 !pp->valid) && 1539 (!object->pager || !vm_pager_has_page(object->pager, object->paging_offset + new_offset))) { 1540 /* 1541 * Page still needed. Can't go any 1542 * further. 1543 */ 1544 vm_object_unlock(backing_object); 1545 return; 1546 } 1547 } 1548 1549 /* 1550 * Make the parent shadow the next object in the 1551 * chain. Deallocating backing_object will not remove 1552 * it, since its reference count is at least 2. 1553 */ 1554 1555 TAILQ_REMOVE(&object->shadow->reverse_shadow_head, 1556 object, reverse_shadow_list); 1557 vm_object_reference(object->shadow = backing_object->shadow); 1558 if (object->shadow) 1559 TAILQ_INSERT_TAIL(&object->shadow->reverse_shadow_head, 1560 object, reverse_shadow_list); 1561 object->shadow_offset += backing_object->shadow_offset; 1562 1563 /* 1564 * Backing object might have had a copy pointer to us. 1565 * If it did, clear it. 1566 */ 1567 if (backing_object->copy == object) { 1568 backing_object->copy = NULL; 1569 } 1570 /* 1571 * Drop the reference count on backing_object. Since 1572 * its ref_count was at least 2, it will not vanish; 1573 * so we don't need to call vm_object_deallocate. 1574 */ 1575 if (backing_object->ref_count == 1) 1576 printf("should have called obj deallocate\n"); 1577 backing_object->ref_count--; 1578 vm_object_unlock(backing_object); 1579 1580 object_bypasses++; 1581 1582 } 1583 1584 /* 1585 * Try again with this object's new backing object. 1586 */ 1587 } 1588 } 1589 1590 /* 1591 * vm_object_page_remove: [internal] 1592 * 1593 * Removes all physical pages in the specified 1594 * object range from the object's list of pages. 1595 * 1596 * The object must be locked. 1597 */ 1598 void 1599 vm_object_page_remove(object, start, end) 1600 register vm_object_t object; 1601 register vm_offset_t start; 1602 register vm_offset_t end; 1603 { 1604 register vm_page_t p, next; 1605 vm_offset_t size; 1606 int s; 1607 1608 if (object == NULL) 1609 return; 1610 1611 object->paging_in_progress++; 1612 start = trunc_page(start); 1613 end = round_page(end); 1614 again: 1615 size = end - start; 1616 if (size > 4 * PAGE_SIZE || size >= object->size / 4) { 1617 for (p = object->memq.tqh_first; p != NULL; p = next) { 1618 next = p->listq.tqe_next; 1619 if ((start <= p->offset) && (p->offset < end)) { 1620 s = splhigh(); 1621 if (p->bmapped) { 1622 splx(s); 1623 continue; 1624 } 1625 if ((p->flags & PG_BUSY) || p->busy) { 1626 p->flags |= PG_WANTED; 1627 tsleep((caddr_t) p, PVM, "vmopar", 0); 1628 splx(s); 1629 goto again; 1630 } 1631 splx(s); 1632 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1633 vm_page_lock_queues(); 1634 PAGE_WAKEUP(p); 1635 vm_page_free(p); 1636 vm_page_unlock_queues(); 1637 } 1638 } 1639 } else { 1640 while (size > 0) { 1641 while ((p = vm_page_lookup(object, start)) != 0) { 1642 s = splhigh(); 1643 if (p->bmapped) { 1644 splx(s); 1645 break; 1646 } 1647 if ((p->flags & PG_BUSY) || p->busy) { 1648 p->flags |= PG_WANTED; 1649 tsleep((caddr_t) p, PVM, "vmopar", 0); 1650 splx(s); 1651 goto again; 1652 } 1653 splx(s); 1654 pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); 1655 vm_page_lock_queues(); 1656 PAGE_WAKEUP(p); 1657 vm_page_free(p); 1658 vm_page_unlock_queues(); 1659 } 1660 start += PAGE_SIZE; 1661 size -= PAGE_SIZE; 1662 } 1663 } 1664 --object->paging_in_progress; 1665 if (object->paging_in_progress == 0) 1666 wakeup((caddr_t) object); 1667 } 1668 1669 /* 1670 * Routine: vm_object_coalesce 1671 * Function: Coalesces two objects backing up adjoining 1672 * regions of memory into a single object. 1673 * 1674 * returns TRUE if objects were combined. 1675 * 1676 * NOTE: Only works at the moment if the second object is NULL - 1677 * if it's not, which object do we lock first? 1678 * 1679 * Parameters: 1680 * prev_object First object to coalesce 1681 * prev_offset Offset into prev_object 1682 * next_object Second object into coalesce 1683 * next_offset Offset into next_object 1684 * 1685 * prev_size Size of reference to prev_object 1686 * next_size Size of reference to next_object 1687 * 1688 * Conditions: 1689 * The object must *not* be locked. 1690 */ 1691 boolean_t 1692 vm_object_coalesce(prev_object, next_object, 1693 prev_offset, next_offset, 1694 prev_size, next_size) 1695 register vm_object_t prev_object; 1696 vm_object_t next_object; 1697 vm_offset_t prev_offset, next_offset; 1698 vm_size_t prev_size, next_size; 1699 { 1700 vm_size_t newsize; 1701 1702 if (next_object != NULL) { 1703 return (FALSE); 1704 } 1705 if (prev_object == NULL) { 1706 return (TRUE); 1707 } 1708 vm_object_lock(prev_object); 1709 1710 /* 1711 * Try to collapse the object first 1712 */ 1713 vm_object_collapse(prev_object); 1714 1715 /* 1716 * Can't coalesce if: . more than one reference . paged out . shadows 1717 * another object . has a copy elsewhere (any of which mean that the 1718 * pages not mapped to prev_entry may be in use anyway) 1719 */ 1720 1721 if (prev_object->ref_count > 1 || 1722 prev_object->pager != NULL || 1723 prev_object->shadow != NULL || 1724 prev_object->copy != NULL) { 1725 vm_object_unlock(prev_object); 1726 return (FALSE); 1727 } 1728 /* 1729 * Remove any pages that may still be in the object from a previous 1730 * deallocation. 1731 */ 1732 1733 vm_object_page_remove(prev_object, 1734 prev_offset + prev_size, 1735 prev_offset + prev_size + next_size); 1736 1737 /* 1738 * Extend the object if necessary. 1739 */ 1740 newsize = prev_offset + prev_size + next_size; 1741 if (newsize > prev_object->size) 1742 prev_object->size = newsize; 1743 1744 vm_object_unlock(prev_object); 1745 return (TRUE); 1746 } 1747 1748 /* 1749 * returns page after looking up in shadow chain 1750 */ 1751 1752 vm_page_t 1753 vm_object_page_lookup(object, offset) 1754 vm_object_t object; 1755 vm_offset_t offset; 1756 { 1757 vm_page_t m; 1758 1759 if (!(m = vm_page_lookup(object, offset))) { 1760 if (!object->shadow) 1761 return 0; 1762 else 1763 return vm_object_page_lookup(object->shadow, offset + object->shadow_offset); 1764 } 1765 return m; 1766 } 1767 1768 #define DEBUG 1769 #if defined(DEBUG) || defined(DDB) 1770 /* 1771 * vm_object_print: [ debug ] 1772 */ 1773 void 1774 vm_object_print(object, full) 1775 vm_object_t object; 1776 boolean_t full; 1777 { 1778 register vm_page_t p; 1779 extern indent; 1780 1781 register int count; 1782 1783 if (object == NULL) 1784 return; 1785 1786 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1787 (int) object, (int) object->size, 1788 object->resident_page_count, object->ref_count); 1789 printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n", 1790 (int) object->pager, (int) object->paging_offset, 1791 (int) object->shadow, (int) object->shadow_offset); 1792 printf("cache: next=%p, prev=%p\n", 1793 object->cached_list.tqe_next, object->cached_list.tqe_prev); 1794 1795 if (!full) 1796 return; 1797 1798 indent += 2; 1799 count = 0; 1800 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1801 if (count == 0) 1802 iprintf("memory:="); 1803 else if (count == 6) { 1804 printf("\n"); 1805 iprintf(" ..."); 1806 count = 0; 1807 } else 1808 printf(","); 1809 count++; 1810 1811 printf("(off=0x%lx,page=0x%lx)", 1812 (u_long) p->offset, (u_long) VM_PAGE_TO_PHYS(p)); 1813 } 1814 if (count != 0) 1815 printf("\n"); 1816 indent -= 2; 1817 } 1818 #endif /* defined(DEBUG) || defined(DDB) */ 1819