1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 37 * 38 * 39 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 40 * All rights reserved. 41 * 42 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 43 * 44 * Permission to use, copy, modify and distribute this software and 45 * its documentation is hereby granted, provided that both the copyright 46 * notice and this permission notice appear in all copies of the 47 * software, derivative works or modified versions, and any portions 48 * thereof, and that both notices appear in supporting documentation. 49 * 50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 51 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 53 * 54 * Carnegie Mellon requests users of this software to return to 55 * 56 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 57 * School of Computer Science 58 * Carnegie Mellon University 59 * Pittsburgh PA 15213-3890 60 * 61 * any improvements or extensions that they make and grant Carnegie the 62 * rights to redistribute these changes. 63 * 64 * $Id: vm_object.c,v 1.55 1995/11/05 20:46:00 dyson Exp $ 65 */ 66 67 /* 68 * Virtual memory object module. 69 */ 70 71 #include <sys/param.h> 72 #include <sys/systm.h> 73 #include <sys/kernel.h> 74 #include <sys/proc.h> /* for curproc, pageproc */ 75 #include <sys/malloc.h> 76 #include <sys/vnode.h> 77 #include <sys/mount.h> 78 79 #include <vm/vm.h> 80 #include <vm/vm_page.h> 81 #include <vm/vm_pageout.h> 82 #include <vm/vm_pager.h> 83 #include <vm/swap_pager.h> 84 #include <vm/vm_kern.h> 85 86 static void _vm_object_allocate(objtype_t, vm_size_t, vm_object_t); 87 88 89 /* 90 * Virtual memory objects maintain the actual data 91 * associated with allocated virtual memory. A given 92 * page of memory exists within exactly one object. 93 * 94 * An object is only deallocated when all "references" 95 * are given up. Only one "reference" to a given 96 * region of an object should be writeable. 97 * 98 * Associated with each object is a list of all resident 99 * memory pages belonging to that object; this list is 100 * maintained by the "vm_page" module, and locked by the object's 101 * lock. 102 * 103 * Each object also records a "pager" routine which is 104 * used to retrieve (and store) pages to the proper backing 105 * storage. In addition, objects may be backed by other 106 * objects from which they were virtual-copied. 107 * 108 * The only items within the object structure which are 109 * modified after time of creation are: 110 * reference count locked by object's lock 111 * pager routine locked by object's lock 112 * 113 */ 114 115 int vm_object_cache_max; 116 struct object_q vm_object_cached_list; 117 int vm_object_cached; 118 struct object_q vm_object_list; 119 long vm_object_count; 120 vm_object_t kernel_object; 121 vm_object_t kmem_object; 122 struct vm_object kernel_object_store; 123 struct vm_object kmem_object_store; 124 extern int vm_pageout_page_count; 125 126 long object_collapses; 127 long object_bypasses; 128 129 static void 130 _vm_object_allocate(type, size, object) 131 objtype_t type; 132 vm_size_t size; 133 register vm_object_t object; 134 { 135 TAILQ_INIT(&object->memq); 136 TAILQ_INIT(&object->shadow_head); 137 138 object->type = type; 139 object->size = size; 140 object->ref_count = 1; 141 object->flags = 0; 142 object->paging_in_progress = 0; 143 object->resident_page_count = 0; 144 object->handle = NULL; 145 object->paging_offset = 0; 146 object->backing_object = NULL; 147 object->backing_object_offset = (vm_offset_t) 0; 148 149 object->last_read = 0; 150 151 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 152 vm_object_count++; 153 } 154 155 /* 156 * vm_object_init: 157 * 158 * Initialize the VM objects module. 159 */ 160 void 161 vm_object_init(vm_offset_t nothing) 162 { 163 TAILQ_INIT(&vm_object_cached_list); 164 TAILQ_INIT(&vm_object_list); 165 vm_object_count = 0; 166 167 vm_object_cache_max = 84; 168 if (cnt.v_page_count > 1000) 169 vm_object_cache_max += (cnt.v_page_count - 1000) / 4; 170 171 kernel_object = &kernel_object_store; 172 _vm_object_allocate(OBJT_DEFAULT, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 173 kernel_object); 174 175 kmem_object = &kmem_object_store; 176 _vm_object_allocate(OBJT_DEFAULT, VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, 177 kmem_object); 178 } 179 180 /* 181 * vm_object_allocate: 182 * 183 * Returns a new object with the given size. 184 */ 185 186 vm_object_t 187 vm_object_allocate(type, size) 188 objtype_t type; 189 vm_size_t size; 190 { 191 register vm_object_t result; 192 193 result = (vm_object_t) 194 malloc((u_long) sizeof *result, M_VMOBJ, M_WAITOK); 195 196 197 _vm_object_allocate(type, size, result); 198 199 return (result); 200 } 201 202 203 /* 204 * vm_object_reference: 205 * 206 * Gets another reference to the given object. 207 */ 208 inline void 209 vm_object_reference(object) 210 register vm_object_t object; 211 { 212 if (object == NULL) 213 return; 214 215 if (object->ref_count == 0) { 216 if ((object->flags & OBJ_CANPERSIST) == 0) 217 panic("vm_object_reference: non-persistent object with 0 ref_count"); 218 TAILQ_REMOVE(&vm_object_cached_list, object, cached_list); 219 vm_object_cached--; 220 } 221 object->ref_count++; 222 } 223 224 /* 225 * vm_object_deallocate: 226 * 227 * Release a reference to the specified object, 228 * gained either through a vm_object_allocate 229 * or a vm_object_reference call. When all references 230 * are gone, storage associated with this object 231 * may be relinquished. 232 * 233 * No object may be locked. 234 */ 235 void 236 vm_object_deallocate(object) 237 vm_object_t object; 238 { 239 vm_object_t temp; 240 241 while (object != NULL) { 242 243 if (object->ref_count == 0) 244 panic("vm_object_deallocate: object deallocated too many times"); 245 246 /* 247 * Lose the reference 248 */ 249 object->ref_count--; 250 251 if (object->ref_count != 0) { 252 if ((object->ref_count == 1) && 253 (object->handle == NULL) && 254 (object->type == OBJT_DEFAULT || 255 object->type == OBJT_SWAP)) { 256 vm_object_t robject; 257 robject = object->shadow_head.tqh_first; 258 if ((robject != NULL) && 259 (robject->handle == NULL) && 260 (robject->type == OBJT_DEFAULT || 261 robject->type == OBJT_SWAP)) { 262 int s; 263 robject->ref_count += 2; 264 object->ref_count += 2; 265 266 do { 267 s = splhigh(); 268 while (robject->paging_in_progress) { 269 robject->flags |= OBJ_PIPWNT; 270 tsleep(robject, PVM, "objde1", 0); 271 } 272 273 while (object->paging_in_progress) { 274 object->flags |= OBJ_PIPWNT; 275 tsleep(object, PVM, "objde2", 0); 276 } 277 splx(s); 278 279 } while( object->paging_in_progress || robject->paging_in_progress); 280 281 object->ref_count -= 2; 282 robject->ref_count -= 2; 283 if( robject->ref_count == 0) { 284 robject->ref_count += 1; 285 object = robject; 286 continue; 287 } 288 vm_object_collapse(robject); 289 return; 290 } 291 } 292 /* 293 * If there are still references, then we are done. 294 */ 295 return; 296 } 297 298 if (object->type == OBJT_VNODE) { 299 struct vnode *vp = object->handle; 300 301 vp->v_flag &= ~VTEXT; 302 } 303 304 /* 305 * See if this object can persist and has some resident 306 * pages. If so, enter it in the cache. 307 */ 308 if (object->flags & OBJ_CANPERSIST) { 309 if (object->resident_page_count != 0) { 310 vm_object_page_clean(object, 0, 0 ,TRUE, TRUE); 311 TAILQ_INSERT_TAIL(&vm_object_cached_list, object, 312 cached_list); 313 vm_object_cached++; 314 315 vm_object_cache_trim(); 316 return; 317 } else { 318 object->flags &= ~OBJ_CANPERSIST; 319 } 320 } 321 322 /* 323 * Make sure no one uses us. 324 */ 325 object->flags |= OBJ_DEAD; 326 327 temp = object->backing_object; 328 if (temp) 329 TAILQ_REMOVE(&temp->shadow_head, object, shadow_list); 330 vm_object_terminate(object); 331 /* unlocks and deallocates object */ 332 object = temp; 333 } 334 } 335 336 /* 337 * vm_object_terminate actually destroys the specified object, freeing 338 * up all previously used resources. 339 * 340 * The object must be locked. 341 */ 342 void 343 vm_object_terminate(object) 344 register vm_object_t object; 345 { 346 register vm_page_t p; 347 int s; 348 349 /* 350 * wait for the pageout daemon to be done with the object 351 */ 352 s = splhigh(); 353 while (object->paging_in_progress) { 354 object->flags |= OBJ_PIPWNT; 355 tsleep(object, PVM, "objtrm", 0); 356 } 357 splx(s); 358 359 if (object->paging_in_progress != 0) 360 panic("vm_object_deallocate: pageout in progress"); 361 362 /* 363 * Clean and free the pages, as appropriate. All references to the 364 * object are gone, so we don't need to lock it. 365 */ 366 if (object->type == OBJT_VNODE) { 367 struct vnode *vp = object->handle; 368 369 VOP_LOCK(vp); 370 vm_object_page_clean(object, 0, 0, TRUE, FALSE); 371 vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0); 372 VOP_UNLOCK(vp); 373 } 374 375 /* 376 * Now free the pages. For internal objects, this also removes them 377 * from paging queues. 378 */ 379 while ((p = object->memq.tqh_first) != NULL) { 380 if (p->flags & PG_BUSY) 381 printf("vm_object_terminate: freeing busy page\n"); 382 PAGE_WAKEUP(p); 383 vm_page_free(p); 384 cnt.v_pfree++; 385 } 386 387 /* 388 * Let the pager know object is dead. 389 */ 390 vm_pager_deallocate(object); 391 392 TAILQ_REMOVE(&vm_object_list, object, object_list); 393 vm_object_count--; 394 395 wakeup(object); 396 397 /* 398 * Free the space for the object. 399 */ 400 free((caddr_t) object, M_VMOBJ); 401 } 402 403 /* 404 * vm_object_page_clean 405 * 406 * Clean all dirty pages in the specified range of object. 407 * Leaves page on whatever queue it is currently on. 408 * 409 * Odd semantics: if start == end, we clean everything. 410 * 411 * The object must be locked. 412 */ 413 414 void 415 vm_object_page_clean(object, start, end, syncio, lockflag) 416 vm_object_t object; 417 vm_offset_t start; 418 vm_offset_t end; 419 boolean_t syncio; 420 boolean_t lockflag; 421 { 422 register vm_page_t p; 423 register vm_offset_t tstart, tend; 424 int s; 425 struct vnode *vp; 426 int runlen; 427 vm_page_t ma[vm_pageout_page_count]; 428 429 if (object->type != OBJT_VNODE || 430 (object->flags & OBJ_MIGHTBEDIRTY) == 0) 431 return; 432 433 vp = object->handle; 434 435 if (lockflag) 436 VOP_LOCK(vp); 437 object->flags |= OBJ_CLEANING; 438 439 if (start != end) { 440 start = trunc_page(start); 441 end = round_page(end); 442 } 443 444 tstart = start; 445 if (end == 0) { 446 tend = object->size; 447 } else { 448 tend = end; 449 } 450 if (tstart == 0 && tend == object->size) { 451 object->flags &= ~(OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 452 } 453 454 runlen = 0; 455 for(;tstart < tend; tstart += PAGE_SIZE) { 456 relookup: 457 p = vm_page_lookup(object, tstart); 458 if (!p) { 459 if (runlen > 0) { 460 vm_pageout_flush(ma, runlen, syncio); 461 runlen = 0; 462 } 463 continue; 464 } 465 if (p->valid == 0 || (p->flags & PG_CACHE)) { 466 if (runlen > 0) { 467 vm_pageout_flush(ma, runlen, syncio); 468 runlen = 0; 469 } 470 continue; 471 } 472 473 vm_page_protect(p, VM_PROT_READ); 474 475 s = splhigh(); 476 while ((p->flags & PG_BUSY) || p->busy) { 477 if (runlen > 0) { 478 splx(s); 479 vm_pageout_flush(ma, runlen, syncio); 480 runlen = 0; 481 goto relookup; 482 } 483 p->flags |= PG_WANTED|PG_REFERENCED; 484 tsleep(p, PVM, "vpcwai", 0); 485 splx(s); 486 goto relookup; 487 } 488 splx(s); 489 490 if (p->dirty == 0) 491 vm_page_test_dirty(p); 492 493 if ((p->valid & p->dirty) != 0) { 494 ma[runlen] = p; 495 p->flags |= PG_BUSY; 496 runlen++; 497 if (runlen >= vm_pageout_page_count) { 498 vm_pageout_flush(ma, runlen, syncio); 499 runlen = 0; 500 } 501 } else if (runlen > 0) { 502 vm_pageout_flush(ma, runlen, syncio); 503 runlen = 0; 504 } 505 506 } 507 if (runlen > 0) { 508 vm_pageout_flush(ma, runlen, syncio); 509 } 510 511 VOP_FSYNC(vp, NULL, syncio, curproc); 512 513 if (lockflag) 514 VOP_UNLOCK(vp); 515 object->flags &= ~OBJ_CLEANING; 516 return; 517 } 518 519 /* 520 * vm_object_deactivate_pages 521 * 522 * Deactivate all pages in the specified object. (Keep its pages 523 * in memory even though it is no longer referenced.) 524 * 525 * The object must be locked. 526 */ 527 void 528 vm_object_deactivate_pages(object) 529 register vm_object_t object; 530 { 531 register vm_page_t p, next; 532 533 for (p = object->memq.tqh_first; p != NULL; p = next) { 534 next = p->listq.tqe_next; 535 vm_page_deactivate(p); 536 } 537 } 538 539 /* 540 * Trim the object cache to size. 541 */ 542 void 543 vm_object_cache_trim() 544 { 545 register vm_object_t object; 546 547 while (vm_object_cached > vm_object_cache_max) { 548 object = vm_object_cached_list.tqh_first; 549 550 vm_object_reference(object); 551 pager_cache(object, FALSE); 552 } 553 } 554 555 556 /* 557 * vm_object_pmap_copy: 558 * 559 * Makes all physical pages in the specified 560 * object range copy-on-write. No writeable 561 * references to these pages should remain. 562 * 563 * The object must *not* be locked. 564 */ 565 void 566 vm_object_pmap_copy(object, start, end) 567 register vm_object_t object; 568 register vm_offset_t start; 569 register vm_offset_t end; 570 { 571 register vm_page_t p; 572 573 if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0) 574 return; 575 576 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 577 vm_page_protect(p, VM_PROT_READ); 578 } 579 580 object->flags &= ~OBJ_WRITEABLE; 581 } 582 583 /* 584 * vm_object_pmap_remove: 585 * 586 * Removes all physical pages in the specified 587 * object range from all physical maps. 588 * 589 * The object must *not* be locked. 590 */ 591 void 592 vm_object_pmap_remove(object, start, end) 593 register vm_object_t object; 594 register vm_offset_t start; 595 register vm_offset_t end; 596 { 597 register vm_page_t p; 598 599 if (object == NULL) 600 return; 601 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 602 vm_page_protect(p, VM_PROT_NONE); 603 } 604 } 605 606 /* 607 * vm_object_copy: 608 * 609 * Create a new object which is a copy of an existing 610 * object, and mark all of the pages in the existing 611 * object 'copy-on-write'. The new object has one reference. 612 * Returns the new object. 613 * 614 * May defer the copy until later if the object is not backed 615 * up by a non-default pager. 616 */ 617 void 618 vm_object_copy(src_object, src_offset, size, 619 dst_object, dst_offset, src_needs_copy) 620 register vm_object_t src_object; 621 vm_offset_t src_offset; 622 vm_size_t size; 623 vm_object_t *dst_object;/* OUT */ 624 vm_offset_t *dst_offset;/* OUT */ 625 boolean_t *src_needs_copy; /* OUT */ 626 { 627 if (src_object == NULL) { 628 /* 629 * Nothing to copy 630 */ 631 *dst_object = NULL; 632 *dst_offset = 0; 633 *src_needs_copy = FALSE; 634 return; 635 } 636 637 /* 638 * Try to collapse the object before copying it. 639 */ 640 if (src_object->handle == NULL && 641 (src_object->type == OBJT_DEFAULT || 642 src_object->type == OBJT_SWAP)) 643 vm_object_collapse(src_object); 644 645 646 /* 647 * Make another reference to the object 648 */ 649 src_object->ref_count++; 650 651 *dst_object = src_object; 652 *dst_offset = src_offset; 653 654 /* 655 * Must make a shadow when write is desired 656 */ 657 *src_needs_copy = TRUE; 658 return; 659 } 660 661 /* 662 * vm_object_shadow: 663 * 664 * Create a new object which is backed by the 665 * specified existing object range. The source 666 * object reference is deallocated. 667 * 668 * The new object and offset into that object 669 * are returned in the source parameters. 670 */ 671 672 void 673 vm_object_shadow(object, offset, length) 674 vm_object_t *object; /* IN/OUT */ 675 vm_offset_t *offset; /* IN/OUT */ 676 vm_size_t length; 677 { 678 register vm_object_t source; 679 register vm_object_t result; 680 681 source = *object; 682 683 /* 684 * Allocate a new object with the given length 685 */ 686 687 if ((result = vm_object_allocate(OBJT_DEFAULT, length)) == NULL) 688 panic("vm_object_shadow: no object for shadowing"); 689 690 /* 691 * The new object shadows the source object, adding a reference to it. 692 * Our caller changes his reference to point to the new object, 693 * removing a reference to the source object. Net result: no change 694 * of reference count. 695 */ 696 result->backing_object = source; 697 if (source) 698 TAILQ_INSERT_TAIL(&result->backing_object->shadow_head, result, shadow_list); 699 700 /* 701 * Store the offset into the source object, and fix up the offset into 702 * the new object. 703 */ 704 705 result->backing_object_offset = *offset; 706 707 /* 708 * Return the new things 709 */ 710 711 *offset = 0; 712 *object = result; 713 } 714 715 716 /* 717 * this version of collapse allows the operation to occur earlier and 718 * when paging_in_progress is true for an object... This is not a complete 719 * operation, but should plug 99.9% of the rest of the leaks. 720 */ 721 static void 722 vm_object_qcollapse(object) 723 register vm_object_t object; 724 { 725 register vm_object_t backing_object; 726 register vm_offset_t backing_offset, new_offset; 727 register vm_page_t p, pp; 728 register vm_size_t size; 729 730 backing_object = object->backing_object; 731 if (backing_object->ref_count != 1) 732 return; 733 734 backing_object->ref_count += 2; 735 736 backing_offset = object->backing_object_offset; 737 size = object->size; 738 p = backing_object->memq.tqh_first; 739 while (p) { 740 vm_page_t next; 741 742 next = p->listq.tqe_next; 743 if ((p->flags & (PG_BUSY | PG_FICTITIOUS | PG_CACHE)) || 744 !p->valid || p->hold_count || p->wire_count || p->busy) { 745 p = next; 746 continue; 747 } 748 vm_page_protect(p, VM_PROT_NONE); 749 new_offset = (p->offset - backing_offset); 750 if (p->offset < backing_offset || 751 new_offset >= size) { 752 if (backing_object->type == OBJT_SWAP) 753 swap_pager_freespace(backing_object, 754 backing_object->paging_offset + p->offset, PAGE_SIZE); 755 vm_page_free(p); 756 } else { 757 pp = vm_page_lookup(object, new_offset); 758 if (pp != NULL || (object->type == OBJT_SWAP && vm_pager_has_page(object, 759 object->paging_offset + new_offset, NULL, NULL))) { 760 if (backing_object->type == OBJT_SWAP) 761 swap_pager_freespace(backing_object, 762 backing_object->paging_offset + p->offset, PAGE_SIZE); 763 vm_page_free(p); 764 } else { 765 if (backing_object->type == OBJT_SWAP) 766 swap_pager_freespace(backing_object, 767 backing_object->paging_offset + p->offset, PAGE_SIZE); 768 vm_page_rename(p, object, new_offset); 769 p->dirty = VM_PAGE_BITS_ALL; 770 } 771 } 772 p = next; 773 } 774 backing_object->ref_count -= 2; 775 } 776 777 /* 778 * vm_object_collapse: 779 * 780 * Collapse an object with the object backing it. 781 * Pages in the backing object are moved into the 782 * parent, and the backing object is deallocated. 783 */ 784 void 785 vm_object_collapse(object) 786 vm_object_t object; 787 788 { 789 vm_object_t backing_object; 790 vm_offset_t backing_offset; 791 vm_size_t size; 792 vm_offset_t new_offset; 793 vm_page_t p, pp; 794 795 while (TRUE) { 796 /* 797 * Verify that the conditions are right for collapse: 798 * 799 * The object exists and no pages in it are currently being paged 800 * out. 801 */ 802 if (object == NULL) 803 return; 804 805 /* 806 * Make sure there is a backing object. 807 */ 808 if ((backing_object = object->backing_object) == NULL) 809 return; 810 811 /* 812 * we check the backing object first, because it is most likely 813 * not collapsable. 814 */ 815 if (backing_object->handle != NULL || 816 (backing_object->type != OBJT_DEFAULT && 817 backing_object->type != OBJT_SWAP) || 818 (backing_object->flags & OBJ_DEAD) || 819 object->handle != NULL || 820 (object->type != OBJT_DEFAULT && 821 object->type != OBJT_SWAP) || 822 (object->flags & OBJ_DEAD)) { 823 return; 824 } 825 826 if (object->paging_in_progress != 0 || 827 backing_object->paging_in_progress != 0) { 828 vm_object_qcollapse(object); 829 return; 830 } 831 832 /* 833 * We know that we can either collapse the backing object (if 834 * the parent is the only reference to it) or (perhaps) remove 835 * the parent's reference to it. 836 */ 837 838 backing_offset = object->backing_object_offset; 839 size = object->size; 840 841 /* 842 * If there is exactly one reference to the backing object, we 843 * can collapse it into the parent. 844 */ 845 846 if (backing_object->ref_count == 1) { 847 848 backing_object->flags |= OBJ_DEAD; 849 /* 850 * We can collapse the backing object. 851 * 852 * Move all in-memory pages from backing_object to the 853 * parent. Pages that have been paged out will be 854 * overwritten by any of the parent's pages that 855 * shadow them. 856 */ 857 858 while ((p = backing_object->memq.tqh_first) != 0) { 859 860 new_offset = (p->offset - backing_offset); 861 862 /* 863 * If the parent has a page here, or if this 864 * page falls outside the parent, dispose of 865 * it. 866 * 867 * Otherwise, move it as planned. 868 */ 869 870 if (p->offset < backing_offset || 871 new_offset >= size) { 872 vm_page_protect(p, VM_PROT_NONE); 873 PAGE_WAKEUP(p); 874 vm_page_free(p); 875 } else { 876 pp = vm_page_lookup(object, new_offset); 877 if (pp != NULL || (object->type == OBJT_SWAP && vm_pager_has_page(object, 878 object->paging_offset + new_offset, NULL, NULL))) { 879 vm_page_protect(p, VM_PROT_NONE); 880 PAGE_WAKEUP(p); 881 vm_page_free(p); 882 } else { 883 vm_page_rename(p, object, new_offset); 884 } 885 } 886 } 887 888 /* 889 * Move the pager from backing_object to object. 890 */ 891 892 if (backing_object->type == OBJT_SWAP) { 893 backing_object->paging_in_progress++; 894 if (object->type == OBJT_SWAP) { 895 object->paging_in_progress++; 896 /* 897 * copy shadow object pages into ours 898 * and destroy unneeded pages in 899 * shadow object. 900 */ 901 swap_pager_copy( 902 backing_object, backing_object->paging_offset, 903 object, object->paging_offset, 904 object->backing_object_offset); 905 vm_object_pip_wakeup(object); 906 } else { 907 object->paging_in_progress++; 908 /* 909 * move the shadow backing_object's pager data to 910 * "object" and convert "object" type to OBJT_SWAP. 911 */ 912 object->type = OBJT_SWAP; 913 object->un_pager.swp.swp_nblocks = 914 backing_object->un_pager.swp.swp_nblocks; 915 object->un_pager.swp.swp_allocsize = 916 backing_object->un_pager.swp.swp_allocsize; 917 object->un_pager.swp.swp_blocks = 918 backing_object->un_pager.swp.swp_blocks; 919 object->un_pager.swp.swp_poip = /* XXX */ 920 backing_object->un_pager.swp.swp_poip; 921 object->paging_offset = backing_object->paging_offset + backing_offset; 922 TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list); 923 924 /* 925 * Convert backing object from OBJT_SWAP to 926 * OBJT_DEFAULT. XXX - only the TAILQ_REMOVE is 927 * actually necessary. 928 */ 929 backing_object->type = OBJT_DEFAULT; 930 TAILQ_REMOVE(&swap_pager_un_object_list, backing_object, pager_object_list); 931 /* 932 * free unnecessary blocks 933 */ 934 swap_pager_freespace(object, 0, object->paging_offset); 935 vm_object_pip_wakeup(object); 936 } 937 938 vm_object_pip_wakeup(backing_object); 939 } 940 /* 941 * Object now shadows whatever backing_object did. 942 * Note that the reference to backing_object->backing_object 943 * moves from within backing_object to within object. 944 */ 945 946 TAILQ_REMOVE(&object->backing_object->shadow_head, object, 947 shadow_list); 948 if (backing_object->backing_object) 949 TAILQ_REMOVE(&backing_object->backing_object->shadow_head, 950 backing_object, shadow_list); 951 object->backing_object = backing_object->backing_object; 952 if (object->backing_object) 953 TAILQ_INSERT_TAIL(&object->backing_object->shadow_head, 954 object, shadow_list); 955 956 object->backing_object_offset += backing_object->backing_object_offset; 957 /* 958 * Discard backing_object. 959 * 960 * Since the backing object has no pages, no pager left, 961 * and no object references within it, all that is 962 * necessary is to dispose of it. 963 */ 964 965 TAILQ_REMOVE(&vm_object_list, backing_object, 966 object_list); 967 vm_object_count--; 968 969 free((caddr_t) backing_object, M_VMOBJ); 970 971 object_collapses++; 972 } else { 973 /* 974 * If all of the pages in the backing object are 975 * shadowed by the parent object, the parent object no 976 * longer has to shadow the backing object; it can 977 * shadow the next one in the chain. 978 * 979 * The backing object must not be paged out - we'd have 980 * to check all of the paged-out pages, as well. 981 */ 982 983 if (backing_object->type != OBJT_DEFAULT) { 984 return; 985 } 986 /* 987 * Should have a check for a 'small' number of pages 988 * here. 989 */ 990 991 for (p = backing_object->memq.tqh_first; p; p = p->listq.tqe_next) { 992 new_offset = (p->offset - backing_offset); 993 994 /* 995 * If the parent has a page here, or if this 996 * page falls outside the parent, keep going. 997 * 998 * Otherwise, the backing_object must be left in 999 * the chain. 1000 */ 1001 1002 if (p->offset >= backing_offset && new_offset <= size) { 1003 1004 pp = vm_page_lookup(object, new_offset); 1005 1006 if ((pp == NULL || pp->valid == 0) && 1007 !vm_pager_has_page(object, object->paging_offset + new_offset, NULL, NULL)) { 1008 1009 /* 1010 * Page still needed. Can't go any 1011 * further. 1012 */ 1013 return; 1014 } 1015 } 1016 } 1017 1018 /* 1019 * Make the parent shadow the next object in the 1020 * chain. Deallocating backing_object will not remove 1021 * it, since its reference count is at least 2. 1022 */ 1023 1024 TAILQ_REMOVE(&object->backing_object->shadow_head, 1025 object, shadow_list); 1026 vm_object_reference(object->backing_object = backing_object->backing_object); 1027 if (object->backing_object) 1028 TAILQ_INSERT_TAIL(&object->backing_object->shadow_head, 1029 object, shadow_list); 1030 object->backing_object_offset += backing_object->backing_object_offset; 1031 1032 /* 1033 * Drop the reference count on backing_object. Since 1034 * its ref_count was at least 2, it will not vanish; 1035 * so we don't need to call vm_object_deallocate. 1036 */ 1037 if (backing_object->ref_count == 1) 1038 printf("should have called obj deallocate\n"); 1039 backing_object->ref_count--; 1040 1041 object_bypasses++; 1042 1043 } 1044 1045 /* 1046 * Try again with this object's new backing object. 1047 */ 1048 } 1049 } 1050 1051 /* 1052 * vm_object_page_remove: [internal] 1053 * 1054 * Removes all physical pages in the specified 1055 * object range from the object's list of pages. 1056 * 1057 * The object must be locked. 1058 */ 1059 void 1060 vm_object_page_remove(object, start, end, clean_only) 1061 register vm_object_t object; 1062 register vm_offset_t start; 1063 register vm_offset_t end; 1064 boolean_t clean_only; 1065 { 1066 register vm_page_t p, next; 1067 vm_offset_t size; 1068 int s; 1069 1070 if (object == NULL) 1071 return; 1072 1073 object->paging_in_progress++; 1074 start = trunc_page(start); 1075 end = round_page(end); 1076 again: 1077 size = end - start; 1078 if (size > 4 * PAGE_SIZE || size >= object->size / 4) { 1079 for (p = object->memq.tqh_first; p != NULL; p = next) { 1080 next = p->listq.tqe_next; 1081 if ((start <= p->offset) && (p->offset < end)) { 1082 s = splhigh(); 1083 if (p->bmapped) { 1084 splx(s); 1085 continue; 1086 } 1087 if ((p->flags & PG_BUSY) || p->busy) { 1088 p->flags |= PG_WANTED; 1089 tsleep(p, PVM, "vmopar", 0); 1090 splx(s); 1091 goto again; 1092 } 1093 splx(s); 1094 if (clean_only) { 1095 vm_page_test_dirty(p); 1096 if (p->valid & p->dirty) 1097 continue; 1098 } 1099 vm_page_protect(p, VM_PROT_NONE); 1100 PAGE_WAKEUP(p); 1101 vm_page_free(p); 1102 } 1103 } 1104 } else { 1105 while (size > 0) { 1106 while ((p = vm_page_lookup(object, start)) != 0) { 1107 s = splhigh(); 1108 if (p->bmapped) { 1109 splx(s); 1110 break; 1111 } 1112 if ((p->flags & PG_BUSY) || p->busy) { 1113 p->flags |= PG_WANTED; 1114 tsleep(p, PVM, "vmopar", 0); 1115 splx(s); 1116 goto again; 1117 } 1118 splx(s); 1119 if (clean_only) { 1120 vm_page_test_dirty(p); 1121 if (p->valid & p->dirty) 1122 continue; 1123 } 1124 vm_page_protect(p, VM_PROT_NONE); 1125 PAGE_WAKEUP(p); 1126 vm_page_free(p); 1127 } 1128 start += PAGE_SIZE; 1129 size -= PAGE_SIZE; 1130 } 1131 } 1132 vm_object_pip_wakeup(object); 1133 } 1134 1135 /* 1136 * Routine: vm_object_coalesce 1137 * Function: Coalesces two objects backing up adjoining 1138 * regions of memory into a single object. 1139 * 1140 * returns TRUE if objects were combined. 1141 * 1142 * NOTE: Only works at the moment if the second object is NULL - 1143 * if it's not, which object do we lock first? 1144 * 1145 * Parameters: 1146 * prev_object First object to coalesce 1147 * prev_offset Offset into prev_object 1148 * next_object Second object into coalesce 1149 * next_offset Offset into next_object 1150 * 1151 * prev_size Size of reference to prev_object 1152 * next_size Size of reference to next_object 1153 * 1154 * Conditions: 1155 * The object must *not* be locked. 1156 */ 1157 boolean_t 1158 vm_object_coalesce(prev_object, next_object, 1159 prev_offset, next_offset, 1160 prev_size, next_size) 1161 register vm_object_t prev_object; 1162 vm_object_t next_object; 1163 vm_offset_t prev_offset, next_offset; 1164 vm_size_t prev_size, next_size; 1165 { 1166 vm_size_t newsize; 1167 1168 if (next_object != NULL) { 1169 return (FALSE); 1170 } 1171 if (prev_object == NULL) { 1172 return (TRUE); 1173 } 1174 1175 /* 1176 * Try to collapse the object first 1177 */ 1178 vm_object_collapse(prev_object); 1179 1180 /* 1181 * Can't coalesce if: . more than one reference . paged out . shadows 1182 * another object . has a copy elsewhere (any of which mean that the 1183 * pages not mapped to prev_entry may be in use anyway) 1184 */ 1185 1186 if (prev_object->ref_count > 1 || 1187 prev_object->type != OBJT_DEFAULT || 1188 prev_object->backing_object != NULL) { 1189 return (FALSE); 1190 } 1191 /* 1192 * Remove any pages that may still be in the object from a previous 1193 * deallocation. 1194 */ 1195 1196 vm_object_page_remove(prev_object, 1197 prev_offset + prev_size, 1198 prev_offset + prev_size + next_size, FALSE); 1199 1200 /* 1201 * Extend the object if necessary. 1202 */ 1203 newsize = prev_offset + prev_size + next_size; 1204 if (newsize > prev_object->size) 1205 prev_object->size = newsize; 1206 1207 return (TRUE); 1208 } 1209 1210 /* 1211 * returns page after looking up in shadow chain 1212 */ 1213 1214 vm_page_t 1215 vm_object_page_lookup(object, offset) 1216 vm_object_t object; 1217 vm_offset_t offset; 1218 { 1219 vm_page_t m; 1220 1221 if (!(m = vm_page_lookup(object, offset))) { 1222 if (!object->backing_object) 1223 return 0; 1224 else 1225 return vm_object_page_lookup(object->backing_object, offset + object->backing_object_offset); 1226 } 1227 return m; 1228 } 1229 1230 #ifdef DDB 1231 1232 int 1233 _vm_object_in_map(map, object, entry) 1234 vm_map_t map; 1235 vm_object_t object; 1236 vm_map_entry_t entry; 1237 { 1238 vm_map_t tmpm; 1239 vm_map_entry_t tmpe; 1240 vm_object_t obj; 1241 int entcount; 1242 1243 if (map == 0) 1244 return 0; 1245 1246 if (entry == 0) { 1247 tmpe = map->header.next; 1248 entcount = map->nentries; 1249 while (entcount-- && (tmpe != &map->header)) { 1250 if( _vm_object_in_map(map, object, tmpe)) { 1251 return 1; 1252 } 1253 tmpe = tmpe->next; 1254 } 1255 } else if (entry->is_sub_map || entry->is_a_map) { 1256 tmpm = entry->object.share_map; 1257 tmpe = tmpm->header.next; 1258 entcount = tmpm->nentries; 1259 while (entcount-- && tmpe != &tmpm->header) { 1260 if( _vm_object_in_map(tmpm, object, tmpe)) { 1261 return 1; 1262 } 1263 tmpe = tmpe->next; 1264 } 1265 } else if (obj = entry->object.vm_object) { 1266 for(; obj; obj=obj->backing_object) 1267 if( obj == object) { 1268 return 1; 1269 } 1270 } 1271 return 0; 1272 } 1273 1274 int 1275 vm_object_in_map( object) 1276 vm_object_t object; 1277 { 1278 struct proc *p; 1279 for (p = (struct proc *) allproc; p != NULL; p = p->p_next) { 1280 if( !p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 1281 continue; 1282 /* 1283 if (p->p_stat != SRUN && p->p_stat != SSLEEP) { 1284 continue; 1285 } 1286 */ 1287 if( _vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) 1288 return 1; 1289 } 1290 if( _vm_object_in_map( kernel_map, object, 0)) 1291 return 1; 1292 if( _vm_object_in_map( kmem_map, object, 0)) 1293 return 1; 1294 if( _vm_object_in_map( pager_map, object, 0)) 1295 return 1; 1296 if( _vm_object_in_map( buffer_map, object, 0)) 1297 return 1; 1298 if( _vm_object_in_map( io_map, object, 0)) 1299 return 1; 1300 if( _vm_object_in_map( phys_map, object, 0)) 1301 return 1; 1302 if( _vm_object_in_map( mb_map, object, 0)) 1303 return 1; 1304 if( _vm_object_in_map( u_map, object, 0)) 1305 return 1; 1306 return 0; 1307 } 1308 1309 1310 void 1311 vm_object_check() { 1312 int i; 1313 int maxhash = 0; 1314 vm_object_t object; 1315 1316 /* 1317 * make sure that internal objs are in a map somewhere 1318 * and none have zero ref counts. 1319 */ 1320 for (object = vm_object_list.tqh_first; 1321 object != NULL; 1322 object = object->object_list.tqe_next) { 1323 if (object->handle == NULL && 1324 (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) { 1325 if (object->ref_count == 0) { 1326 printf("vmochk: internal obj has zero ref count: %d\n", 1327 object->size); 1328 } 1329 if (!vm_object_in_map(object)) { 1330 printf("vmochk: internal obj is not in a map: " 1331 "ref: %d, size: %d: 0x%x, backing_object: 0x%x\n", 1332 object->ref_count, object->size, 1333 object->size, object->backing_object); 1334 } 1335 } 1336 } 1337 } 1338 1339 /* 1340 * vm_object_print: [ debug ] 1341 */ 1342 void 1343 vm_object_print(iobject, full, dummy3, dummy4) 1344 /* db_expr_t */ int iobject; 1345 boolean_t full; 1346 /* db_expr_t */ int dummy3; 1347 char *dummy4; 1348 { 1349 vm_object_t object = (vm_object_t)iobject; /* XXX */ 1350 register vm_page_t p; 1351 1352 register int count; 1353 1354 if (object == NULL) 1355 return; 1356 1357 iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ", 1358 (int) object, (int) object->size, 1359 object->resident_page_count, object->ref_count); 1360 printf("offset=0x%x, backing_object=(0x%x)+0x%x\n", 1361 (int) object->paging_offset, 1362 (int) object->backing_object, (int) object->backing_object_offset); 1363 printf("cache: next=%p, prev=%p\n", 1364 object->cached_list.tqe_next, object->cached_list.tqe_prev); 1365 1366 if (!full) 1367 return; 1368 1369 indent += 2; 1370 count = 0; 1371 for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { 1372 if (count == 0) 1373 iprintf("memory:="); 1374 else if (count == 6) { 1375 printf("\n"); 1376 iprintf(" ..."); 1377 count = 0; 1378 } else 1379 printf(","); 1380 count++; 1381 1382 printf("(off=0x%lx,page=0x%lx)", 1383 (u_long) p->offset, (u_long) VM_PAGE_TO_PHYS(p)); 1384 } 1385 if (count != 0) 1386 printf("\n"); 1387 indent -= 2; 1388 } 1389 #endif /* DDB */ 1390