1 /* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94 33 * 34 * 35 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 36 * All rights reserved. 37 * 38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 39 * 40 * Permission to use, copy, modify and distribute this software and 41 * its documentation is hereby granted, provided that both the copyright 42 * notice and this permission notice appear in all copies of the 43 * software, derivative works or modified versions, and any portions 44 * thereof, and that both notices appear in supporting documentation. 45 * 46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 49 * 50 * Carnegie Mellon requests users of this software to return to 51 * 52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 53 * School of Computer Science 54 * Carnegie Mellon University 55 * Pittsburgh PA 15213-3890 56 * 57 * any improvements or extensions that they make and grant Carnegie the 58 * rights to redistribute these changes. 59 */ 60 61 /* 62 * Virtual memory object module. 63 */ 64 65 #include <sys/cdefs.h> 66 __FBSDID("$FreeBSD$"); 67 68 #include <sys/param.h> 69 #include <sys/systm.h> 70 #include <sys/lock.h> 71 #include <sys/mman.h> 72 #include <sys/mount.h> 73 #include <sys/kernel.h> 74 #include <sys/sysctl.h> 75 #include <sys/mutex.h> 76 #include <sys/proc.h> /* for curproc, pageproc */ 77 #include <sys/socket.h> 78 #include <sys/vnode.h> 79 #include <sys/vmmeter.h> 80 #include <sys/sx.h> 81 82 #include <vm/vm.h> 83 #include <vm/vm_param.h> 84 #include <vm/pmap.h> 85 #include <vm/vm_map.h> 86 #include <vm/vm_object.h> 87 #include <vm/vm_page.h> 88 #include <vm/vm_pageout.h> 89 #include <vm/vm_pager.h> 90 #include <vm/swap_pager.h> 91 #include <vm/vm_kern.h> 92 #include <vm/vm_extern.h> 93 #include <vm/uma.h> 94 95 #define EASY_SCAN_FACTOR 8 96 97 #define MSYNC_FLUSH_HARDSEQ 0x01 98 #define MSYNC_FLUSH_SOFTSEQ 0x02 99 100 /* 101 * msync / VM object flushing optimizations 102 */ 103 static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ; 104 SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags, 105 CTLFLAG_RW, &msync_flush_flags, 0, ""); 106 107 static void vm_object_qcollapse(vm_object_t object); 108 static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags); 109 110 /* 111 * Virtual memory objects maintain the actual data 112 * associated with allocated virtual memory. A given 113 * page of memory exists within exactly one object. 114 * 115 * An object is only deallocated when all "references" 116 * are given up. Only one "reference" to a given 117 * region of an object should be writeable. 118 * 119 * Associated with each object is a list of all resident 120 * memory pages belonging to that object; this list is 121 * maintained by the "vm_page" module, and locked by the object's 122 * lock. 123 * 124 * Each object also records a "pager" routine which is 125 * used to retrieve (and store) pages to the proper backing 126 * storage. In addition, objects may be backed by other 127 * objects from which they were virtual-copied. 128 * 129 * The only items within the object structure which are 130 * modified after time of creation are: 131 * reference count locked by object's lock 132 * pager routine locked by object's lock 133 * 134 */ 135 136 struct object_q vm_object_list; 137 struct mtx vm_object_list_mtx; /* lock for object list and count */ 138 139 struct vm_object kernel_object_store; 140 struct vm_object kmem_object_store; 141 142 static long object_collapses; 143 static long object_bypasses; 144 static int next_index; 145 static uma_zone_t obj_zone; 146 #define VM_OBJECTS_INIT 256 147 148 static void vm_object_zinit(void *mem, int size); 149 150 #ifdef INVARIANTS 151 static void vm_object_zdtor(void *mem, int size, void *arg); 152 153 static void 154 vm_object_zdtor(void *mem, int size, void *arg) 155 { 156 vm_object_t object; 157 158 object = (vm_object_t)mem; 159 KASSERT(TAILQ_EMPTY(&object->memq), 160 ("object %p has resident pages", 161 object)); 162 KASSERT(object->paging_in_progress == 0, 163 ("object %p paging_in_progress = %d", 164 object, object->paging_in_progress)); 165 KASSERT(object->resident_page_count == 0, 166 ("object %p resident_page_count = %d", 167 object, object->resident_page_count)); 168 KASSERT(object->shadow_count == 0, 169 ("object %p shadow_count = %d", 170 object, object->shadow_count)); 171 } 172 #endif 173 174 static void 175 vm_object_zinit(void *mem, int size) 176 { 177 vm_object_t object; 178 179 object = (vm_object_t)mem; 180 bzero(&object->mtx, sizeof(object->mtx)); 181 VM_OBJECT_LOCK_INIT(object); 182 183 /* These are true for any object that has been freed */ 184 object->paging_in_progress = 0; 185 object->resident_page_count = 0; 186 object->shadow_count = 0; 187 } 188 189 void 190 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object) 191 { 192 int incr; 193 194 TAILQ_INIT(&object->memq); 195 LIST_INIT(&object->shadow_head); 196 197 object->root = NULL; 198 object->type = type; 199 object->size = size; 200 object->generation = 1; 201 object->ref_count = 1; 202 object->flags = 0; 203 if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP)) 204 object->flags = OBJ_ONEMAPPING; 205 if (size > (PQ_L2_SIZE / 3 + PQ_PRIME1)) 206 incr = PQ_L2_SIZE / 3 + PQ_PRIME1; 207 else 208 incr = size; 209 do 210 object->pg_color = next_index; 211 while (!atomic_cmpset_int(&next_index, object->pg_color, 212 (object->pg_color + incr) & PQ_L2_MASK)); 213 object->handle = NULL; 214 object->backing_object = NULL; 215 object->backing_object_offset = (vm_ooffset_t) 0; 216 217 mtx_lock(&vm_object_list_mtx); 218 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); 219 mtx_unlock(&vm_object_list_mtx); 220 } 221 222 /* 223 * vm_object_init: 224 * 225 * Initialize the VM objects module. 226 */ 227 void 228 vm_object_init(void) 229 { 230 TAILQ_INIT(&vm_object_list); 231 mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); 232 233 VM_OBJECT_LOCK_INIT(&kernel_object_store); 234 _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), 235 kernel_object); 236 237 /* 238 * The kmem object's mutex is given a unique name, instead of 239 * "vm object", to avoid false reports of lock-order reversal 240 * with a system map mutex. 241 */ 242 mtx_init(VM_OBJECT_MTX(kmem_object), "kmem object", NULL, MTX_DEF); 243 _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), 244 kmem_object); 245 246 obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL, 247 #ifdef INVARIANTS 248 vm_object_zdtor, 249 #else 250 NULL, 251 #endif 252 vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); 253 uma_prealloc(obj_zone, VM_OBJECTS_INIT); 254 } 255 256 void 257 vm_object_clear_flag(vm_object_t object, u_short bits) 258 { 259 260 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 261 object->flags &= ~bits; 262 } 263 264 void 265 vm_object_pip_add(vm_object_t object, short i) 266 { 267 268 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 269 object->paging_in_progress += i; 270 } 271 272 void 273 vm_object_pip_subtract(vm_object_t object, short i) 274 { 275 276 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 277 object->paging_in_progress -= i; 278 } 279 280 void 281 vm_object_pip_wakeup(vm_object_t object) 282 { 283 284 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 285 object->paging_in_progress--; 286 if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { 287 vm_object_clear_flag(object, OBJ_PIPWNT); 288 wakeup(object); 289 } 290 } 291 292 void 293 vm_object_pip_wakeupn(vm_object_t object, short i) 294 { 295 296 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 297 if (i) 298 object->paging_in_progress -= i; 299 if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { 300 vm_object_clear_flag(object, OBJ_PIPWNT); 301 wakeup(object); 302 } 303 } 304 305 void 306 vm_object_pip_wait(vm_object_t object, char *waitid) 307 { 308 309 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 310 while (object->paging_in_progress) { 311 object->flags |= OBJ_PIPWNT; 312 msleep(object, VM_OBJECT_MTX(object), PVM, waitid, 0); 313 } 314 } 315 316 /* 317 * vm_object_allocate_wait 318 * 319 * Return a new object with the given size, and give the user the 320 * option of waiting for it to complete or failing if the needed 321 * memory isn't available. 322 */ 323 vm_object_t 324 vm_object_allocate_wait(objtype_t type, vm_pindex_t size, int flags) 325 { 326 vm_object_t result; 327 328 result = (vm_object_t) uma_zalloc(obj_zone, flags); 329 330 if (result != NULL) 331 _vm_object_allocate(type, size, result); 332 333 return (result); 334 } 335 336 /* 337 * vm_object_allocate: 338 * 339 * Returns a new object with the given size. 340 */ 341 vm_object_t 342 vm_object_allocate(objtype_t type, vm_pindex_t size) 343 { 344 return(vm_object_allocate_wait(type, size, M_WAITOK)); 345 } 346 347 348 /* 349 * vm_object_reference: 350 * 351 * Gets another reference to the given object. Note: OBJ_DEAD 352 * objects can be referenced during final cleaning. 353 */ 354 void 355 vm_object_reference(vm_object_t object) 356 { 357 struct vnode *vp; 358 int flags; 359 360 if (object == NULL) 361 return; 362 VM_OBJECT_LOCK(object); 363 object->ref_count++; 364 if (object->type == OBJT_VNODE) { 365 vp = object->handle; 366 VI_LOCK(vp); 367 VM_OBJECT_UNLOCK(object); 368 for (flags = LK_INTERLOCK; vget(vp, flags, curthread); 369 flags = 0) 370 printf("vm_object_reference: delay in vget\n"); 371 } else 372 VM_OBJECT_UNLOCK(object); 373 } 374 375 /* 376 * vm_object_reference_locked: 377 * 378 * Gets another reference to the given object. 379 * 380 * The object must be locked. 381 */ 382 void 383 vm_object_reference_locked(vm_object_t object) 384 { 385 struct vnode *vp; 386 387 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 388 KASSERT((object->flags & OBJ_DEAD) == 0, 389 ("vm_object_reference_locked: dead object referenced")); 390 object->ref_count++; 391 if (object->type == OBJT_VNODE) { 392 vp = object->handle; 393 vref(vp); 394 } 395 } 396 397 /* 398 * Handle deallocating an object of type OBJT_VNODE. 399 */ 400 void 401 vm_object_vndeallocate(vm_object_t object) 402 { 403 struct vnode *vp = (struct vnode *) object->handle; 404 405 GIANT_REQUIRED; 406 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 407 KASSERT(object->type == OBJT_VNODE, 408 ("vm_object_vndeallocate: not a vnode object")); 409 KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); 410 #ifdef INVARIANTS 411 if (object->ref_count == 0) { 412 vprint("vm_object_vndeallocate", vp); 413 panic("vm_object_vndeallocate: bad object reference count"); 414 } 415 #endif 416 417 object->ref_count--; 418 if (object->ref_count == 0) { 419 mp_fixme("Unlocked vflag access."); 420 vp->v_vflag &= ~VV_TEXT; 421 } 422 VM_OBJECT_UNLOCK(object); 423 /* 424 * vrele may need a vop lock 425 */ 426 vrele(vp); 427 } 428 429 /* 430 * vm_object_deallocate: 431 * 432 * Release a reference to the specified object, 433 * gained either through a vm_object_allocate 434 * or a vm_object_reference call. When all references 435 * are gone, storage associated with this object 436 * may be relinquished. 437 * 438 * No object may be locked. 439 */ 440 void 441 vm_object_deallocate(vm_object_t object) 442 { 443 vm_object_t temp; 444 445 while (object != NULL) { 446 /* 447 * In general, the object should be locked when working with 448 * its type. In this case, in order to maintain proper lock 449 * ordering, an exception is possible because a vnode-backed 450 * object never changes its type. 451 */ 452 if (object->type == OBJT_VNODE) 453 mtx_lock(&Giant); 454 VM_OBJECT_LOCK(object); 455 if (object->type == OBJT_VNODE) { 456 vm_object_vndeallocate(object); 457 mtx_unlock(&Giant); 458 return; 459 } 460 461 KASSERT(object->ref_count != 0, 462 ("vm_object_deallocate: object deallocated too many times: %d", object->type)); 463 464 /* 465 * If the reference count goes to 0 we start calling 466 * vm_object_terminate() on the object chain. 467 * A ref count of 1 may be a special case depending on the 468 * shadow count being 0 or 1. 469 */ 470 object->ref_count--; 471 if (object->ref_count > 1) { 472 VM_OBJECT_UNLOCK(object); 473 return; 474 } else if (object->ref_count == 1) { 475 if (object->shadow_count == 0) { 476 vm_object_set_flag(object, OBJ_ONEMAPPING); 477 } else if ((object->shadow_count == 1) && 478 (object->handle == NULL) && 479 (object->type == OBJT_DEFAULT || 480 object->type == OBJT_SWAP)) { 481 vm_object_t robject; 482 483 robject = LIST_FIRST(&object->shadow_head); 484 KASSERT(robject != NULL, 485 ("vm_object_deallocate: ref_count: %d, shadow_count: %d", 486 object->ref_count, 487 object->shadow_count)); 488 if (!VM_OBJECT_TRYLOCK(robject)) { 489 /* 490 * Avoid a potential deadlock. 491 */ 492 object->ref_count++; 493 VM_OBJECT_UNLOCK(object); 494 /* 495 * More likely than not the thread 496 * holding robject's lock has lower 497 * priority than the current thread. 498 * Let the lower priority thread run. 499 */ 500 tsleep(&proc0, PVM, "vmo_de", 1); 501 continue; 502 } 503 if ((robject->handle == NULL) && 504 (robject->type == OBJT_DEFAULT || 505 robject->type == OBJT_SWAP)) { 506 507 robject->ref_count++; 508 retry: 509 if (robject->paging_in_progress) { 510 VM_OBJECT_UNLOCK(object); 511 vm_object_pip_wait(robject, 512 "objde1"); 513 VM_OBJECT_LOCK(object); 514 goto retry; 515 } else if (object->paging_in_progress) { 516 VM_OBJECT_UNLOCK(robject); 517 object->flags |= OBJ_PIPWNT; 518 msleep(object, 519 VM_OBJECT_MTX(object), 520 PDROP | PVM, "objde2", 0); 521 VM_OBJECT_LOCK(robject); 522 VM_OBJECT_LOCK(object); 523 goto retry; 524 } 525 VM_OBJECT_UNLOCK(object); 526 if (robject->ref_count == 1) { 527 robject->ref_count--; 528 object = robject; 529 goto doterm; 530 } 531 object = robject; 532 vm_object_collapse(object); 533 VM_OBJECT_UNLOCK(object); 534 continue; 535 } 536 VM_OBJECT_UNLOCK(robject); 537 } 538 VM_OBJECT_UNLOCK(object); 539 return; 540 } 541 doterm: 542 temp = object->backing_object; 543 if (temp != NULL) { 544 VM_OBJECT_LOCK(temp); 545 LIST_REMOVE(object, shadow_list); 546 temp->shadow_count--; 547 temp->generation++; 548 VM_OBJECT_UNLOCK(temp); 549 object->backing_object = NULL; 550 } 551 /* 552 * Don't double-terminate, we could be in a termination 553 * recursion due to the terminate having to sync data 554 * to disk. 555 */ 556 if ((object->flags & OBJ_DEAD) == 0) 557 vm_object_terminate(object); 558 else 559 VM_OBJECT_UNLOCK(object); 560 object = temp; 561 } 562 } 563 564 /* 565 * vm_object_terminate actually destroys the specified object, freeing 566 * up all previously used resources. 567 * 568 * The object must be locked. 569 * This routine may block. 570 */ 571 void 572 vm_object_terminate(vm_object_t object) 573 { 574 vm_page_t p; 575 int s; 576 577 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 578 579 /* 580 * Make sure no one uses us. 581 */ 582 vm_object_set_flag(object, OBJ_DEAD); 583 584 /* 585 * wait for the pageout daemon to be done with the object 586 */ 587 vm_object_pip_wait(object, "objtrm"); 588 589 KASSERT(!object->paging_in_progress, 590 ("vm_object_terminate: pageout in progress")); 591 592 /* 593 * Clean and free the pages, as appropriate. All references to the 594 * object are gone, so we don't need to lock it. 595 */ 596 if (object->type == OBJT_VNODE) { 597 struct vnode *vp = (struct vnode *)object->handle; 598 599 /* 600 * Clean pages and flush buffers. 601 */ 602 vm_object_page_clean(object, 0, 0, OBJPC_SYNC); 603 VM_OBJECT_UNLOCK(object); 604 605 vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0); 606 607 VM_OBJECT_LOCK(object); 608 } 609 610 KASSERT(object->ref_count == 0, 611 ("vm_object_terminate: object with references, ref_count=%d", 612 object->ref_count)); 613 614 /* 615 * Now free any remaining pages. For internal objects, this also 616 * removes them from paging queues. Don't free wired pages, just 617 * remove them from the object. 618 */ 619 s = splvm(); 620 vm_page_lock_queues(); 621 while ((p = TAILQ_FIRST(&object->memq)) != NULL) { 622 KASSERT(!p->busy && (p->flags & PG_BUSY) == 0, 623 ("vm_object_terminate: freeing busy page %p " 624 "p->busy = %d, p->flags %x\n", p, p->busy, p->flags)); 625 if (p->wire_count == 0) { 626 vm_page_busy(p); 627 vm_page_free(p); 628 cnt.v_pfree++; 629 } else { 630 vm_page_busy(p); 631 vm_page_remove(p); 632 } 633 } 634 vm_page_unlock_queues(); 635 splx(s); 636 637 /* 638 * Let the pager know object is dead. 639 */ 640 vm_pager_deallocate(object); 641 VM_OBJECT_UNLOCK(object); 642 643 /* 644 * Remove the object from the global object list. 645 */ 646 mtx_lock(&vm_object_list_mtx); 647 TAILQ_REMOVE(&vm_object_list, object, object_list); 648 mtx_unlock(&vm_object_list_mtx); 649 650 wakeup(object); 651 652 /* 653 * Free the space for the object. 654 */ 655 uma_zfree(obj_zone, object); 656 } 657 658 /* 659 * vm_object_page_clean 660 * 661 * Clean all dirty pages in the specified range of object. Leaves page 662 * on whatever queue it is currently on. If NOSYNC is set then do not 663 * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC), 664 * leaving the object dirty. 665 * 666 * When stuffing pages asynchronously, allow clustering. XXX we need a 667 * synchronous clustering mode implementation. 668 * 669 * Odd semantics: if start == end, we clean everything. 670 * 671 * The object must be locked. 672 */ 673 void 674 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int flags) 675 { 676 vm_page_t p, np; 677 vm_pindex_t tstart, tend; 678 vm_pindex_t pi; 679 int clearobjflags; 680 int pagerflags; 681 int curgeneration; 682 683 GIANT_REQUIRED; 684 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 685 if (object->type != OBJT_VNODE || 686 (object->flags & OBJ_MIGHTBEDIRTY) == 0) 687 return; 688 689 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; 690 pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0; 691 692 vm_object_set_flag(object, OBJ_CLEANING); 693 694 tstart = start; 695 if (end == 0) { 696 tend = object->size; 697 } else { 698 tend = end; 699 } 700 701 vm_page_lock_queues(); 702 /* 703 * If the caller is smart and only msync()s a range he knows is 704 * dirty, we may be able to avoid an object scan. This results in 705 * a phenominal improvement in performance. We cannot do this 706 * as a matter of course because the object may be huge - e.g. 707 * the size might be in the gigabytes or terrabytes. 708 */ 709 if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) { 710 vm_pindex_t tscan; 711 int scanlimit; 712 int scanreset; 713 714 scanreset = object->resident_page_count / EASY_SCAN_FACTOR; 715 if (scanreset < 16) 716 scanreset = 16; 717 pagerflags |= VM_PAGER_IGNORE_CLEANCHK; 718 719 scanlimit = scanreset; 720 tscan = tstart; 721 while (tscan < tend) { 722 curgeneration = object->generation; 723 p = vm_page_lookup(object, tscan); 724 if (p == NULL || p->valid == 0 || 725 (p->queue - p->pc) == PQ_CACHE) { 726 if (--scanlimit == 0) 727 break; 728 ++tscan; 729 continue; 730 } 731 vm_page_test_dirty(p); 732 if ((p->dirty & p->valid) == 0) { 733 if (--scanlimit == 0) 734 break; 735 ++tscan; 736 continue; 737 } 738 /* 739 * If we have been asked to skip nosync pages and 740 * this is a nosync page, we can't continue. 741 */ 742 if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 743 if (--scanlimit == 0) 744 break; 745 ++tscan; 746 continue; 747 } 748 scanlimit = scanreset; 749 750 /* 751 * This returns 0 if it was unable to busy the first 752 * page (i.e. had to sleep). 753 */ 754 tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags); 755 } 756 757 /* 758 * If everything was dirty and we flushed it successfully, 759 * and the requested range is not the entire object, we 760 * don't have to mess with CLEANCHK or MIGHTBEDIRTY and can 761 * return immediately. 762 */ 763 if (tscan >= tend && (tstart || tend < object->size)) { 764 vm_page_unlock_queues(); 765 vm_object_clear_flag(object, OBJ_CLEANING); 766 return; 767 } 768 pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK; 769 } 770 771 /* 772 * Generally set CLEANCHK interlock and make the page read-only so 773 * we can then clear the object flags. 774 * 775 * However, if this is a nosync mmap then the object is likely to 776 * stay dirty so do not mess with the page and do not clear the 777 * object flags. 778 */ 779 clearobjflags = 1; 780 TAILQ_FOREACH(p, &object->memq, listq) { 781 vm_page_flag_set(p, PG_CLEANCHK); 782 if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) 783 clearobjflags = 0; 784 else 785 pmap_page_protect(p, VM_PROT_READ); 786 } 787 788 if (clearobjflags && (tstart == 0) && (tend == object->size)) { 789 struct vnode *vp; 790 791 vm_object_clear_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 792 if (object->type == OBJT_VNODE && 793 (vp = (struct vnode *)object->handle) != NULL) { 794 VI_LOCK(vp); 795 if (vp->v_iflag & VI_OBJDIRTY) 796 vp->v_iflag &= ~VI_OBJDIRTY; 797 VI_UNLOCK(vp); 798 } 799 } 800 801 rescan: 802 curgeneration = object->generation; 803 804 for (p = TAILQ_FIRST(&object->memq); p; p = np) { 805 int n; 806 807 np = TAILQ_NEXT(p, listq); 808 809 again: 810 pi = p->pindex; 811 if (((p->flags & PG_CLEANCHK) == 0) || 812 (pi < tstart) || (pi >= tend) || 813 (p->valid == 0) || 814 ((p->queue - p->pc) == PQ_CACHE)) { 815 vm_page_flag_clear(p, PG_CLEANCHK); 816 continue; 817 } 818 819 vm_page_test_dirty(p); 820 if ((p->dirty & p->valid) == 0) { 821 vm_page_flag_clear(p, PG_CLEANCHK); 822 continue; 823 } 824 825 /* 826 * If we have been asked to skip nosync pages and this is a 827 * nosync page, skip it. Note that the object flags were 828 * not cleared in this case so we do not have to set them. 829 */ 830 if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { 831 vm_page_flag_clear(p, PG_CLEANCHK); 832 continue; 833 } 834 835 n = vm_object_page_collect_flush(object, p, 836 curgeneration, pagerflags); 837 if (n == 0) 838 goto rescan; 839 840 if (object->generation != curgeneration) 841 goto rescan; 842 843 /* 844 * Try to optimize the next page. If we can't we pick up 845 * our (random) scan where we left off. 846 */ 847 if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ) { 848 if ((p = vm_page_lookup(object, pi + n)) != NULL) 849 goto again; 850 } 851 } 852 vm_page_unlock_queues(); 853 #if 0 854 VOP_FSYNC(vp, NULL, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc); 855 #endif 856 857 vm_object_clear_flag(object, OBJ_CLEANING); 858 return; 859 } 860 861 static int 862 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags) 863 { 864 int runlen; 865 int s; 866 int maxf; 867 int chkb; 868 int maxb; 869 int i; 870 vm_pindex_t pi; 871 vm_page_t maf[vm_pageout_page_count]; 872 vm_page_t mab[vm_pageout_page_count]; 873 vm_page_t ma[vm_pageout_page_count]; 874 875 s = splvm(); 876 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 877 pi = p->pindex; 878 while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { 879 vm_page_lock_queues(); 880 if (object->generation != curgeneration) { 881 splx(s); 882 return(0); 883 } 884 } 885 maxf = 0; 886 for(i = 1; i < vm_pageout_page_count; i++) { 887 vm_page_t tp; 888 889 if ((tp = vm_page_lookup(object, pi + i)) != NULL) { 890 if ((tp->flags & PG_BUSY) || 891 ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 892 (tp->flags & PG_CLEANCHK) == 0) || 893 (tp->busy != 0)) 894 break; 895 if((tp->queue - tp->pc) == PQ_CACHE) { 896 vm_page_flag_clear(tp, PG_CLEANCHK); 897 break; 898 } 899 vm_page_test_dirty(tp); 900 if ((tp->dirty & tp->valid) == 0) { 901 vm_page_flag_clear(tp, PG_CLEANCHK); 902 break; 903 } 904 maf[ i - 1 ] = tp; 905 maxf++; 906 continue; 907 } 908 break; 909 } 910 911 maxb = 0; 912 chkb = vm_pageout_page_count - maxf; 913 if (chkb) { 914 for(i = 1; i < chkb;i++) { 915 vm_page_t tp; 916 917 if ((tp = vm_page_lookup(object, pi - i)) != NULL) { 918 if ((tp->flags & PG_BUSY) || 919 ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 920 (tp->flags & PG_CLEANCHK) == 0) || 921 (tp->busy != 0)) 922 break; 923 if ((tp->queue - tp->pc) == PQ_CACHE) { 924 vm_page_flag_clear(tp, PG_CLEANCHK); 925 break; 926 } 927 vm_page_test_dirty(tp); 928 if ((tp->dirty & tp->valid) == 0) { 929 vm_page_flag_clear(tp, PG_CLEANCHK); 930 break; 931 } 932 mab[ i - 1 ] = tp; 933 maxb++; 934 continue; 935 } 936 break; 937 } 938 } 939 940 for(i = 0; i < maxb; i++) { 941 int index = (maxb - i) - 1; 942 ma[index] = mab[i]; 943 vm_page_flag_clear(ma[index], PG_CLEANCHK); 944 } 945 vm_page_flag_clear(p, PG_CLEANCHK); 946 ma[maxb] = p; 947 for(i = 0; i < maxf; i++) { 948 int index = (maxb + i) + 1; 949 ma[index] = maf[i]; 950 vm_page_flag_clear(ma[index], PG_CLEANCHK); 951 } 952 runlen = maxb + maxf + 1; 953 954 splx(s); 955 vm_pageout_flush(ma, runlen, pagerflags); 956 for (i = 0; i < runlen; i++) { 957 if (ma[i]->valid & ma[i]->dirty) { 958 pmap_page_protect(ma[i], VM_PROT_READ); 959 vm_page_flag_set(ma[i], PG_CLEANCHK); 960 961 /* 962 * maxf will end up being the actual number of pages 963 * we wrote out contiguously, non-inclusive of the 964 * first page. We do not count look-behind pages. 965 */ 966 if (i >= maxb + 1 && (maxf > i - maxb - 1)) 967 maxf = i - maxb - 1; 968 } 969 } 970 return(maxf + 1); 971 } 972 973 /* 974 * Note that there is absolutely no sense in writing out 975 * anonymous objects, so we track down the vnode object 976 * to write out. 977 * We invalidate (remove) all pages from the address space 978 * for semantic correctness. 979 * 980 * Note: certain anonymous maps, such as MAP_NOSYNC maps, 981 * may start out with a NULL object. 982 */ 983 void 984 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size, 985 boolean_t syncio, boolean_t invalidate) 986 { 987 vm_object_t backing_object; 988 struct vnode *vp; 989 int flags; 990 991 if (object == NULL) 992 return; 993 VM_OBJECT_LOCK(object); 994 while ((backing_object = object->backing_object) != NULL) { 995 VM_OBJECT_LOCK(backing_object); 996 VM_OBJECT_UNLOCK(object); 997 object = backing_object; 998 offset += object->backing_object_offset; 999 if (object->size < OFF_TO_IDX(offset + size)) 1000 size = IDX_TO_OFF(object->size) - offset; 1001 } 1002 /* 1003 * Flush pages if writing is allowed, invalidate them 1004 * if invalidation requested. Pages undergoing I/O 1005 * will be ignored by vm_object_page_remove(). 1006 * 1007 * We cannot lock the vnode and then wait for paging 1008 * to complete without deadlocking against vm_fault. 1009 * Instead we simply call vm_object_page_remove() and 1010 * allow it to block internally on a page-by-page 1011 * basis when it encounters pages undergoing async 1012 * I/O. 1013 */ 1014 if (object->type == OBJT_VNODE && 1015 (object->flags & OBJ_MIGHTBEDIRTY) != 0) { 1016 vp = object->handle; 1017 VM_OBJECT_UNLOCK(object); 1018 mtx_lock(&Giant); 1019 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curthread); 1020 flags = (syncio || invalidate) ? OBJPC_SYNC : 0; 1021 flags |= invalidate ? OBJPC_INVAL : 0; 1022 VM_OBJECT_LOCK(object); 1023 vm_object_page_clean(object, 1024 OFF_TO_IDX(offset), 1025 OFF_TO_IDX(offset + size + PAGE_MASK), 1026 flags); 1027 VM_OBJECT_UNLOCK(object); 1028 VOP_UNLOCK(vp, 0, curthread); 1029 mtx_unlock(&Giant); 1030 VM_OBJECT_LOCK(object); 1031 } 1032 if ((object->type == OBJT_VNODE || 1033 object->type == OBJT_DEVICE) && invalidate) { 1034 vm_object_page_remove(object, 1035 OFF_TO_IDX(offset), 1036 OFF_TO_IDX(offset + size + PAGE_MASK), 1037 FALSE); 1038 } 1039 VM_OBJECT_UNLOCK(object); 1040 } 1041 1042 /* 1043 * vm_object_madvise: 1044 * 1045 * Implements the madvise function at the object/page level. 1046 * 1047 * MADV_WILLNEED (any object) 1048 * 1049 * Activate the specified pages if they are resident. 1050 * 1051 * MADV_DONTNEED (any object) 1052 * 1053 * Deactivate the specified pages if they are resident. 1054 * 1055 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, 1056 * OBJ_ONEMAPPING only) 1057 * 1058 * Deactivate and clean the specified pages if they are 1059 * resident. This permits the process to reuse the pages 1060 * without faulting or the kernel to reclaim the pages 1061 * without I/O. 1062 */ 1063 void 1064 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise) 1065 { 1066 vm_pindex_t end, tpindex; 1067 vm_object_t backing_object, tobject; 1068 vm_page_t m; 1069 1070 if (object == NULL) 1071 return; 1072 end = pindex + count; 1073 /* 1074 * Locate and adjust resident pages 1075 */ 1076 for (; pindex < end; pindex += 1) { 1077 relookup: 1078 tobject = object; 1079 tpindex = pindex; 1080 VM_OBJECT_LOCK(tobject); 1081 shadowlookup: 1082 /* 1083 * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages 1084 * and those pages must be OBJ_ONEMAPPING. 1085 */ 1086 if (advise == MADV_FREE) { 1087 if ((tobject->type != OBJT_DEFAULT && 1088 tobject->type != OBJT_SWAP) || 1089 (tobject->flags & OBJ_ONEMAPPING) == 0) { 1090 goto unlock_tobject; 1091 } 1092 } 1093 m = vm_page_lookup(tobject, tpindex); 1094 if (m == NULL) { 1095 /* 1096 * There may be swap even if there is no backing page 1097 */ 1098 if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 1099 swap_pager_freespace(tobject, tpindex, 1); 1100 /* 1101 * next object 1102 */ 1103 backing_object = tobject->backing_object; 1104 if (backing_object == NULL) 1105 goto unlock_tobject; 1106 VM_OBJECT_LOCK(backing_object); 1107 VM_OBJECT_UNLOCK(tobject); 1108 tobject = backing_object; 1109 tpindex += OFF_TO_IDX(tobject->backing_object_offset); 1110 goto shadowlookup; 1111 } 1112 /* 1113 * If the page is busy or not in a normal active state, 1114 * we skip it. If the page is not managed there are no 1115 * page queues to mess with. Things can break if we mess 1116 * with pages in any of the below states. 1117 */ 1118 vm_page_lock_queues(); 1119 if (m->hold_count || 1120 m->wire_count || 1121 (m->flags & PG_UNMANAGED) || 1122 m->valid != VM_PAGE_BITS_ALL) { 1123 vm_page_unlock_queues(); 1124 goto unlock_tobject; 1125 } 1126 if (vm_page_sleep_if_busy(m, TRUE, "madvpo")) { 1127 VM_OBJECT_UNLOCK(tobject); 1128 goto relookup; 1129 } 1130 if (advise == MADV_WILLNEED) { 1131 vm_page_activate(m); 1132 } else if (advise == MADV_DONTNEED) { 1133 vm_page_dontneed(m); 1134 } else if (advise == MADV_FREE) { 1135 /* 1136 * Mark the page clean. This will allow the page 1137 * to be freed up by the system. However, such pages 1138 * are often reused quickly by malloc()/free() 1139 * so we do not do anything that would cause 1140 * a page fault if we can help it. 1141 * 1142 * Specifically, we do not try to actually free 1143 * the page now nor do we try to put it in the 1144 * cache (which would cause a page fault on reuse). 1145 * 1146 * But we do make the page is freeable as we 1147 * can without actually taking the step of unmapping 1148 * it. 1149 */ 1150 pmap_clear_modify(m); 1151 m->dirty = 0; 1152 m->act_count = 0; 1153 vm_page_dontneed(m); 1154 } 1155 vm_page_unlock_queues(); 1156 if (advise == MADV_FREE && tobject->type == OBJT_SWAP) 1157 swap_pager_freespace(tobject, tpindex, 1); 1158 unlock_tobject: 1159 VM_OBJECT_UNLOCK(tobject); 1160 } 1161 } 1162 1163 /* 1164 * vm_object_shadow: 1165 * 1166 * Create a new object which is backed by the 1167 * specified existing object range. The source 1168 * object reference is deallocated. 1169 * 1170 * The new object and offset into that object 1171 * are returned in the source parameters. 1172 */ 1173 void 1174 vm_object_shadow( 1175 vm_object_t *object, /* IN/OUT */ 1176 vm_ooffset_t *offset, /* IN/OUT */ 1177 vm_size_t length) 1178 { 1179 vm_object_t source; 1180 vm_object_t result; 1181 1182 source = *object; 1183 1184 /* 1185 * Don't create the new object if the old object isn't shared. 1186 */ 1187 if (source != NULL) { 1188 VM_OBJECT_LOCK(source); 1189 if (source->ref_count == 1 && 1190 source->handle == NULL && 1191 (source->type == OBJT_DEFAULT || 1192 source->type == OBJT_SWAP)) { 1193 VM_OBJECT_UNLOCK(source); 1194 return; 1195 } 1196 VM_OBJECT_UNLOCK(source); 1197 } 1198 1199 /* 1200 * Allocate a new object with the given length. 1201 */ 1202 result = vm_object_allocate(OBJT_DEFAULT, length); 1203 1204 /* 1205 * The new object shadows the source object, adding a reference to it. 1206 * Our caller changes his reference to point to the new object, 1207 * removing a reference to the source object. Net result: no change 1208 * of reference count. 1209 * 1210 * Try to optimize the result object's page color when shadowing 1211 * in order to maintain page coloring consistency in the combined 1212 * shadowed object. 1213 */ 1214 result->backing_object = source; 1215 if (source != NULL) { 1216 VM_OBJECT_LOCK(source); 1217 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list); 1218 source->shadow_count++; 1219 source->generation++; 1220 if (length < source->size) 1221 length = source->size; 1222 if (length > PQ_L2_SIZE / 3 + PQ_PRIME1 || 1223 source->generation > 1) 1224 length = PQ_L2_SIZE / 3 + PQ_PRIME1; 1225 result->pg_color = (source->pg_color + 1226 length * source->generation) & PQ_L2_MASK; 1227 VM_OBJECT_UNLOCK(source); 1228 next_index = (result->pg_color + PQ_L2_SIZE / 3 + PQ_PRIME1) & 1229 PQ_L2_MASK; 1230 } 1231 1232 /* 1233 * Store the offset into the source object, and fix up the offset into 1234 * the new object. 1235 */ 1236 result->backing_object_offset = *offset; 1237 1238 /* 1239 * Return the new things 1240 */ 1241 *offset = 0; 1242 *object = result; 1243 } 1244 1245 /* 1246 * vm_object_split: 1247 * 1248 * Split the pages in a map entry into a new object. This affords 1249 * easier removal of unused pages, and keeps object inheritance from 1250 * being a negative impact on memory usage. 1251 */ 1252 void 1253 vm_object_split(vm_map_entry_t entry) 1254 { 1255 vm_page_t m; 1256 vm_object_t orig_object, new_object, source; 1257 vm_pindex_t offidxstart, offidxend; 1258 vm_size_t idx, size; 1259 1260 orig_object = entry->object.vm_object; 1261 if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) 1262 return; 1263 if (orig_object->ref_count <= 1) 1264 return; 1265 VM_OBJECT_UNLOCK(orig_object); 1266 1267 offidxstart = OFF_TO_IDX(entry->offset); 1268 offidxend = offidxstart + OFF_TO_IDX(entry->end - entry->start); 1269 size = offidxend - offidxstart; 1270 1271 /* 1272 * If swap_pager_copy() is later called, it will convert new_object 1273 * into a swap object. 1274 */ 1275 new_object = vm_object_allocate(OBJT_DEFAULT, size); 1276 1277 VM_OBJECT_LOCK(new_object); 1278 VM_OBJECT_LOCK(orig_object); 1279 source = orig_object->backing_object; 1280 if (source != NULL) { 1281 VM_OBJECT_LOCK(source); 1282 LIST_INSERT_HEAD(&source->shadow_head, 1283 new_object, shadow_list); 1284 source->shadow_count++; 1285 source->generation++; 1286 vm_object_reference_locked(source); /* for new_object */ 1287 vm_object_clear_flag(source, OBJ_ONEMAPPING); 1288 VM_OBJECT_UNLOCK(source); 1289 new_object->backing_object_offset = 1290 orig_object->backing_object_offset + entry->offset; 1291 new_object->backing_object = source; 1292 } 1293 for (idx = 0; idx < size; idx++) { 1294 retry: 1295 m = vm_page_lookup(orig_object, offidxstart + idx); 1296 if (m == NULL) 1297 continue; 1298 1299 /* 1300 * We must wait for pending I/O to complete before we can 1301 * rename the page. 1302 * 1303 * We do not have to VM_PROT_NONE the page as mappings should 1304 * not be changed by this operation. 1305 */ 1306 vm_page_lock_queues(); 1307 if ((m->flags & PG_BUSY) || m->busy) { 1308 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED); 1309 VM_OBJECT_UNLOCK(orig_object); 1310 VM_OBJECT_UNLOCK(new_object); 1311 msleep(m, &vm_page_queue_mtx, PDROP | PVM, "spltwt", 0); 1312 VM_OBJECT_LOCK(new_object); 1313 VM_OBJECT_LOCK(orig_object); 1314 goto retry; 1315 } 1316 vm_page_busy(m); 1317 vm_page_rename(m, new_object, idx); 1318 /* page automatically made dirty by rename and cache handled */ 1319 vm_page_busy(m); 1320 vm_page_unlock_queues(); 1321 } 1322 if (orig_object->type == OBJT_SWAP) { 1323 /* 1324 * swap_pager_copy() can sleep, in which case the orig_object's 1325 * and new_object's locks are released and reacquired. 1326 */ 1327 swap_pager_copy(orig_object, new_object, offidxstart, 0); 1328 } 1329 VM_OBJECT_UNLOCK(orig_object); 1330 vm_page_lock_queues(); 1331 TAILQ_FOREACH(m, &new_object->memq, listq) 1332 vm_page_wakeup(m); 1333 vm_page_unlock_queues(); 1334 VM_OBJECT_UNLOCK(new_object); 1335 entry->object.vm_object = new_object; 1336 entry->offset = 0LL; 1337 vm_object_deallocate(orig_object); 1338 VM_OBJECT_LOCK(new_object); 1339 } 1340 1341 #define OBSC_TEST_ALL_SHADOWED 0x0001 1342 #define OBSC_COLLAPSE_NOWAIT 0x0002 1343 #define OBSC_COLLAPSE_WAIT 0x0004 1344 1345 static int 1346 vm_object_backing_scan(vm_object_t object, int op) 1347 { 1348 int s; 1349 int r = 1; 1350 vm_page_t p; 1351 vm_object_t backing_object; 1352 vm_pindex_t backing_offset_index; 1353 1354 s = splvm(); 1355 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1356 VM_OBJECT_LOCK_ASSERT(object->backing_object, MA_OWNED); 1357 1358 backing_object = object->backing_object; 1359 backing_offset_index = OFF_TO_IDX(object->backing_object_offset); 1360 1361 /* 1362 * Initial conditions 1363 */ 1364 if (op & OBSC_TEST_ALL_SHADOWED) { 1365 /* 1366 * We do not want to have to test for the existence of 1367 * swap pages in the backing object. XXX but with the 1368 * new swapper this would be pretty easy to do. 1369 * 1370 * XXX what about anonymous MAP_SHARED memory that hasn't 1371 * been ZFOD faulted yet? If we do not test for this, the 1372 * shadow test may succeed! XXX 1373 */ 1374 if (backing_object->type != OBJT_DEFAULT) { 1375 splx(s); 1376 return (0); 1377 } 1378 } 1379 if (op & OBSC_COLLAPSE_WAIT) { 1380 vm_object_set_flag(backing_object, OBJ_DEAD); 1381 } 1382 1383 /* 1384 * Our scan 1385 */ 1386 p = TAILQ_FIRST(&backing_object->memq); 1387 while (p) { 1388 vm_page_t next = TAILQ_NEXT(p, listq); 1389 vm_pindex_t new_pindex = p->pindex - backing_offset_index; 1390 1391 if (op & OBSC_TEST_ALL_SHADOWED) { 1392 vm_page_t pp; 1393 1394 /* 1395 * Ignore pages outside the parent object's range 1396 * and outside the parent object's mapping of the 1397 * backing object. 1398 * 1399 * note that we do not busy the backing object's 1400 * page. 1401 */ 1402 if ( 1403 p->pindex < backing_offset_index || 1404 new_pindex >= object->size 1405 ) { 1406 p = next; 1407 continue; 1408 } 1409 1410 /* 1411 * See if the parent has the page or if the parent's 1412 * object pager has the page. If the parent has the 1413 * page but the page is not valid, the parent's 1414 * object pager must have the page. 1415 * 1416 * If this fails, the parent does not completely shadow 1417 * the object and we might as well give up now. 1418 */ 1419 1420 pp = vm_page_lookup(object, new_pindex); 1421 if ( 1422 (pp == NULL || pp->valid == 0) && 1423 !vm_pager_has_page(object, new_pindex, NULL, NULL) 1424 ) { 1425 r = 0; 1426 break; 1427 } 1428 } 1429 1430 /* 1431 * Check for busy page 1432 */ 1433 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) { 1434 vm_page_t pp; 1435 1436 vm_page_lock_queues(); 1437 if (op & OBSC_COLLAPSE_NOWAIT) { 1438 if ((p->flags & PG_BUSY) || 1439 !p->valid || 1440 p->hold_count || 1441 p->wire_count || 1442 p->busy) { 1443 vm_page_unlock_queues(); 1444 p = next; 1445 continue; 1446 } 1447 } else if (op & OBSC_COLLAPSE_WAIT) { 1448 if ((p->flags & PG_BUSY) || p->busy) { 1449 vm_page_flag_set(p, 1450 PG_WANTED | PG_REFERENCED); 1451 VM_OBJECT_UNLOCK(backing_object); 1452 VM_OBJECT_UNLOCK(object); 1453 msleep(p, &vm_page_queue_mtx, 1454 PDROP | PVM, "vmocol", 0); 1455 VM_OBJECT_LOCK(object); 1456 VM_OBJECT_LOCK(backing_object); 1457 /* 1458 * If we slept, anything could have 1459 * happened. Since the object is 1460 * marked dead, the backing offset 1461 * should not have changed so we 1462 * just restart our scan. 1463 */ 1464 p = TAILQ_FIRST(&backing_object->memq); 1465 continue; 1466 } 1467 } 1468 1469 /* 1470 * Busy the page 1471 */ 1472 vm_page_busy(p); 1473 vm_page_unlock_queues(); 1474 1475 KASSERT( 1476 p->object == backing_object, 1477 ("vm_object_qcollapse(): object mismatch") 1478 ); 1479 1480 /* 1481 * Destroy any associated swap 1482 */ 1483 if (backing_object->type == OBJT_SWAP) { 1484 swap_pager_freespace( 1485 backing_object, 1486 p->pindex, 1487 1 1488 ); 1489 } 1490 1491 if ( 1492 p->pindex < backing_offset_index || 1493 new_pindex >= object->size 1494 ) { 1495 /* 1496 * Page is out of the parent object's range, we 1497 * can simply destroy it. 1498 */ 1499 vm_page_lock_queues(); 1500 pmap_remove_all(p); 1501 vm_page_free(p); 1502 vm_page_unlock_queues(); 1503 p = next; 1504 continue; 1505 } 1506 1507 pp = vm_page_lookup(object, new_pindex); 1508 if ( 1509 pp != NULL || 1510 vm_pager_has_page(object, new_pindex, NULL, NULL) 1511 ) { 1512 /* 1513 * page already exists in parent OR swap exists 1514 * for this location in the parent. Destroy 1515 * the original page from the backing object. 1516 * 1517 * Leave the parent's page alone 1518 */ 1519 vm_page_lock_queues(); 1520 pmap_remove_all(p); 1521 vm_page_free(p); 1522 vm_page_unlock_queues(); 1523 p = next; 1524 continue; 1525 } 1526 1527 /* 1528 * Page does not exist in parent, rename the 1529 * page from the backing object to the main object. 1530 * 1531 * If the page was mapped to a process, it can remain 1532 * mapped through the rename. 1533 */ 1534 vm_page_lock_queues(); 1535 vm_page_rename(p, object, new_pindex); 1536 vm_page_unlock_queues(); 1537 /* page automatically made dirty by rename */ 1538 } 1539 p = next; 1540 } 1541 splx(s); 1542 return (r); 1543 } 1544 1545 1546 /* 1547 * this version of collapse allows the operation to occur earlier and 1548 * when paging_in_progress is true for an object... This is not a complete 1549 * operation, but should plug 99.9% of the rest of the leaks. 1550 */ 1551 static void 1552 vm_object_qcollapse(vm_object_t object) 1553 { 1554 vm_object_t backing_object = object->backing_object; 1555 1556 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1557 VM_OBJECT_LOCK_ASSERT(backing_object, MA_OWNED); 1558 1559 if (backing_object->ref_count != 1) 1560 return; 1561 1562 backing_object->ref_count += 2; 1563 1564 vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT); 1565 1566 backing_object->ref_count -= 2; 1567 } 1568 1569 /* 1570 * vm_object_collapse: 1571 * 1572 * Collapse an object with the object backing it. 1573 * Pages in the backing object are moved into the 1574 * parent, and the backing object is deallocated. 1575 */ 1576 void 1577 vm_object_collapse(vm_object_t object) 1578 { 1579 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1580 1581 while (TRUE) { 1582 vm_object_t backing_object; 1583 1584 /* 1585 * Verify that the conditions are right for collapse: 1586 * 1587 * The object exists and the backing object exists. 1588 */ 1589 if ((backing_object = object->backing_object) == NULL) 1590 break; 1591 1592 /* 1593 * we check the backing object first, because it is most likely 1594 * not collapsable. 1595 */ 1596 VM_OBJECT_LOCK(backing_object); 1597 if (backing_object->handle != NULL || 1598 (backing_object->type != OBJT_DEFAULT && 1599 backing_object->type != OBJT_SWAP) || 1600 (backing_object->flags & OBJ_DEAD) || 1601 object->handle != NULL || 1602 (object->type != OBJT_DEFAULT && 1603 object->type != OBJT_SWAP) || 1604 (object->flags & OBJ_DEAD)) { 1605 VM_OBJECT_UNLOCK(backing_object); 1606 break; 1607 } 1608 1609 if ( 1610 object->paging_in_progress != 0 || 1611 backing_object->paging_in_progress != 0 1612 ) { 1613 vm_object_qcollapse(object); 1614 VM_OBJECT_UNLOCK(backing_object); 1615 break; 1616 } 1617 /* 1618 * We know that we can either collapse the backing object (if 1619 * the parent is the only reference to it) or (perhaps) have 1620 * the parent bypass the object if the parent happens to shadow 1621 * all the resident pages in the entire backing object. 1622 * 1623 * This is ignoring pager-backed pages such as swap pages. 1624 * vm_object_backing_scan fails the shadowing test in this 1625 * case. 1626 */ 1627 if (backing_object->ref_count == 1) { 1628 /* 1629 * If there is exactly one reference to the backing 1630 * object, we can collapse it into the parent. 1631 */ 1632 vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT); 1633 1634 /* 1635 * Move the pager from backing_object to object. 1636 */ 1637 if (backing_object->type == OBJT_SWAP) { 1638 /* 1639 * swap_pager_copy() can sleep, in which case 1640 * the backing_object's and object's locks are 1641 * released and reacquired. 1642 */ 1643 swap_pager_copy( 1644 backing_object, 1645 object, 1646 OFF_TO_IDX(object->backing_object_offset), TRUE); 1647 } 1648 /* 1649 * Object now shadows whatever backing_object did. 1650 * Note that the reference to 1651 * backing_object->backing_object moves from within 1652 * backing_object to within object. 1653 */ 1654 LIST_REMOVE(object, shadow_list); 1655 backing_object->shadow_count--; 1656 backing_object->generation++; 1657 if (backing_object->backing_object) { 1658 VM_OBJECT_LOCK(backing_object->backing_object); 1659 LIST_REMOVE(backing_object, shadow_list); 1660 LIST_INSERT_HEAD( 1661 &backing_object->backing_object->shadow_head, 1662 object, shadow_list); 1663 /* 1664 * The shadow_count has not changed. 1665 */ 1666 backing_object->backing_object->generation++; 1667 VM_OBJECT_UNLOCK(backing_object->backing_object); 1668 } 1669 object->backing_object = backing_object->backing_object; 1670 object->backing_object_offset += 1671 backing_object->backing_object_offset; 1672 1673 /* 1674 * Discard backing_object. 1675 * 1676 * Since the backing object has no pages, no pager left, 1677 * and no object references within it, all that is 1678 * necessary is to dispose of it. 1679 */ 1680 KASSERT(backing_object->ref_count == 1, ("backing_object %p was somehow re-referenced during collapse!", backing_object)); 1681 VM_OBJECT_UNLOCK(backing_object); 1682 1683 mtx_lock(&vm_object_list_mtx); 1684 TAILQ_REMOVE( 1685 &vm_object_list, 1686 backing_object, 1687 object_list 1688 ); 1689 mtx_unlock(&vm_object_list_mtx); 1690 1691 uma_zfree(obj_zone, backing_object); 1692 1693 object_collapses++; 1694 } else { 1695 vm_object_t new_backing_object; 1696 1697 /* 1698 * If we do not entirely shadow the backing object, 1699 * there is nothing we can do so we give up. 1700 */ 1701 if (vm_object_backing_scan(object, OBSC_TEST_ALL_SHADOWED) == 0) { 1702 VM_OBJECT_UNLOCK(backing_object); 1703 break; 1704 } 1705 1706 /* 1707 * Make the parent shadow the next object in the 1708 * chain. Deallocating backing_object will not remove 1709 * it, since its reference count is at least 2. 1710 */ 1711 LIST_REMOVE(object, shadow_list); 1712 backing_object->shadow_count--; 1713 backing_object->generation++; 1714 1715 new_backing_object = backing_object->backing_object; 1716 if ((object->backing_object = new_backing_object) != NULL) { 1717 VM_OBJECT_LOCK(new_backing_object); 1718 LIST_INSERT_HEAD( 1719 &new_backing_object->shadow_head, 1720 object, 1721 shadow_list 1722 ); 1723 new_backing_object->shadow_count++; 1724 new_backing_object->generation++; 1725 vm_object_reference_locked(new_backing_object); 1726 VM_OBJECT_UNLOCK(new_backing_object); 1727 object->backing_object_offset += 1728 backing_object->backing_object_offset; 1729 } 1730 1731 /* 1732 * Drop the reference count on backing_object. Since 1733 * its ref_count was at least 2, it will not vanish. 1734 */ 1735 backing_object->ref_count--; 1736 VM_OBJECT_UNLOCK(backing_object); 1737 object_bypasses++; 1738 } 1739 1740 /* 1741 * Try again with this object's new backing object. 1742 */ 1743 } 1744 } 1745 1746 /* 1747 * vm_object_page_remove: 1748 * 1749 * Removes all physical pages in the given range from the 1750 * object's list of pages. If the range's end is zero, all 1751 * physical pages from the range's start to the end of the object 1752 * are deleted. 1753 * 1754 * The object must be locked. 1755 */ 1756 void 1757 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, 1758 boolean_t clean_only) 1759 { 1760 vm_page_t p, next; 1761 1762 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1763 if (object->resident_page_count == 0) 1764 return; 1765 1766 /* 1767 * Since physically-backed objects do not use managed pages, we can't 1768 * remove pages from the object (we must instead remove the page 1769 * references, and then destroy the object). 1770 */ 1771 KASSERT(object->type != OBJT_PHYS, 1772 ("attempt to remove pages from a physical object")); 1773 1774 vm_object_pip_add(object, 1); 1775 again: 1776 vm_page_lock_queues(); 1777 if ((p = TAILQ_FIRST(&object->memq)) != NULL) { 1778 if (p->pindex < start) { 1779 p = vm_page_splay(start, object->root); 1780 if ((object->root = p)->pindex < start) 1781 p = TAILQ_NEXT(p, listq); 1782 } 1783 } 1784 /* 1785 * Assert: the variable p is either (1) the page with the 1786 * least pindex greater than or equal to the parameter pindex 1787 * or (2) NULL. 1788 */ 1789 for (; 1790 p != NULL && (p->pindex < end || end == 0); 1791 p = next) { 1792 next = TAILQ_NEXT(p, listq); 1793 1794 if (p->wire_count != 0) { 1795 pmap_remove_all(p); 1796 if (!clean_only) 1797 p->valid = 0; 1798 continue; 1799 } 1800 if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) 1801 goto again; 1802 if (clean_only && p->valid) { 1803 pmap_page_protect(p, VM_PROT_READ | VM_PROT_EXECUTE); 1804 if (p->valid & p->dirty) 1805 continue; 1806 } 1807 vm_page_busy(p); 1808 pmap_remove_all(p); 1809 vm_page_free(p); 1810 } 1811 vm_page_unlock_queues(); 1812 vm_object_pip_wakeup(object); 1813 } 1814 1815 /* 1816 * Routine: vm_object_coalesce 1817 * Function: Coalesces two objects backing up adjoining 1818 * regions of memory into a single object. 1819 * 1820 * returns TRUE if objects were combined. 1821 * 1822 * NOTE: Only works at the moment if the second object is NULL - 1823 * if it's not, which object do we lock first? 1824 * 1825 * Parameters: 1826 * prev_object First object to coalesce 1827 * prev_offset Offset into prev_object 1828 * next_object Second object into coalesce 1829 * next_offset Offset into next_object 1830 * 1831 * prev_size Size of reference to prev_object 1832 * next_size Size of reference to next_object 1833 * 1834 * Conditions: 1835 * The object must *not* be locked. 1836 */ 1837 boolean_t 1838 vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex, 1839 vm_size_t prev_size, vm_size_t next_size) 1840 { 1841 vm_pindex_t next_pindex; 1842 1843 if (prev_object == NULL) 1844 return (TRUE); 1845 VM_OBJECT_LOCK(prev_object); 1846 if (prev_object->type != OBJT_DEFAULT && 1847 prev_object->type != OBJT_SWAP) { 1848 VM_OBJECT_UNLOCK(prev_object); 1849 return (FALSE); 1850 } 1851 1852 /* 1853 * Try to collapse the object first 1854 */ 1855 vm_object_collapse(prev_object); 1856 1857 /* 1858 * Can't coalesce if: . more than one reference . paged out . shadows 1859 * another object . has a copy elsewhere (any of which mean that the 1860 * pages not mapped to prev_entry may be in use anyway) 1861 */ 1862 if (prev_object->backing_object != NULL) { 1863 VM_OBJECT_UNLOCK(prev_object); 1864 return (FALSE); 1865 } 1866 1867 prev_size >>= PAGE_SHIFT; 1868 next_size >>= PAGE_SHIFT; 1869 next_pindex = prev_pindex + prev_size; 1870 1871 if ((prev_object->ref_count > 1) && 1872 (prev_object->size != next_pindex)) { 1873 VM_OBJECT_UNLOCK(prev_object); 1874 return (FALSE); 1875 } 1876 1877 /* 1878 * Remove any pages that may still be in the object from a previous 1879 * deallocation. 1880 */ 1881 if (next_pindex < prev_object->size) { 1882 vm_object_page_remove(prev_object, 1883 next_pindex, 1884 next_pindex + next_size, FALSE); 1885 if (prev_object->type == OBJT_SWAP) 1886 swap_pager_freespace(prev_object, 1887 next_pindex, next_size); 1888 } 1889 1890 /* 1891 * Extend the object if necessary. 1892 */ 1893 if (next_pindex + next_size > prev_object->size) 1894 prev_object->size = next_pindex + next_size; 1895 1896 VM_OBJECT_UNLOCK(prev_object); 1897 return (TRUE); 1898 } 1899 1900 void 1901 vm_object_set_writeable_dirty(vm_object_t object) 1902 { 1903 struct vnode *vp; 1904 1905 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1906 vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY); 1907 if (object->type == OBJT_VNODE && 1908 (vp = (struct vnode *)object->handle) != NULL) { 1909 VI_LOCK(vp); 1910 if ((vp->v_iflag & VI_OBJDIRTY) == 0) 1911 vp->v_iflag |= VI_OBJDIRTY; 1912 VI_UNLOCK(vp); 1913 } 1914 } 1915 1916 #include "opt_ddb.h" 1917 #ifdef DDB 1918 #include <sys/kernel.h> 1919 1920 #include <sys/cons.h> 1921 1922 #include <ddb/ddb.h> 1923 1924 static int 1925 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry) 1926 { 1927 vm_map_t tmpm; 1928 vm_map_entry_t tmpe; 1929 vm_object_t obj; 1930 int entcount; 1931 1932 if (map == 0) 1933 return 0; 1934 1935 if (entry == 0) { 1936 tmpe = map->header.next; 1937 entcount = map->nentries; 1938 while (entcount-- && (tmpe != &map->header)) { 1939 if (_vm_object_in_map(map, object, tmpe)) { 1940 return 1; 1941 } 1942 tmpe = tmpe->next; 1943 } 1944 } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { 1945 tmpm = entry->object.sub_map; 1946 tmpe = tmpm->header.next; 1947 entcount = tmpm->nentries; 1948 while (entcount-- && tmpe != &tmpm->header) { 1949 if (_vm_object_in_map(tmpm, object, tmpe)) { 1950 return 1; 1951 } 1952 tmpe = tmpe->next; 1953 } 1954 } else if ((obj = entry->object.vm_object) != NULL) { 1955 for (; obj; obj = obj->backing_object) 1956 if (obj == object) { 1957 return 1; 1958 } 1959 } 1960 return 0; 1961 } 1962 1963 static int 1964 vm_object_in_map(vm_object_t object) 1965 { 1966 struct proc *p; 1967 1968 /* sx_slock(&allproc_lock); */ 1969 LIST_FOREACH(p, &allproc, p_list) { 1970 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */) 1971 continue; 1972 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) { 1973 /* sx_sunlock(&allproc_lock); */ 1974 return 1; 1975 } 1976 } 1977 /* sx_sunlock(&allproc_lock); */ 1978 if (_vm_object_in_map(kernel_map, object, 0)) 1979 return 1; 1980 if (_vm_object_in_map(kmem_map, object, 0)) 1981 return 1; 1982 if (_vm_object_in_map(pager_map, object, 0)) 1983 return 1; 1984 if (_vm_object_in_map(buffer_map, object, 0)) 1985 return 1; 1986 return 0; 1987 } 1988 1989 DB_SHOW_COMMAND(vmochk, vm_object_check) 1990 { 1991 vm_object_t object; 1992 1993 /* 1994 * make sure that internal objs are in a map somewhere 1995 * and none have zero ref counts. 1996 */ 1997 TAILQ_FOREACH(object, &vm_object_list, object_list) { 1998 if (object->handle == NULL && 1999 (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) { 2000 if (object->ref_count == 0) { 2001 db_printf("vmochk: internal obj has zero ref count: %ld\n", 2002 (long)object->size); 2003 } 2004 if (!vm_object_in_map(object)) { 2005 db_printf( 2006 "vmochk: internal obj is not in a map: " 2007 "ref: %d, size: %lu: 0x%lx, backing_object: %p\n", 2008 object->ref_count, (u_long)object->size, 2009 (u_long)object->size, 2010 (void *)object->backing_object); 2011 } 2012 } 2013 } 2014 } 2015 2016 /* 2017 * vm_object_print: [ debug ] 2018 */ 2019 DB_SHOW_COMMAND(object, vm_object_print_static) 2020 { 2021 /* XXX convert args. */ 2022 vm_object_t object = (vm_object_t)addr; 2023 boolean_t full = have_addr; 2024 2025 vm_page_t p; 2026 2027 /* XXX count is an (unused) arg. Avoid shadowing it. */ 2028 #define count was_count 2029 2030 int count; 2031 2032 if (object == NULL) 2033 return; 2034 2035 db_iprintf( 2036 "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x\n", 2037 object, (int)object->type, (uintmax_t)object->size, 2038 object->resident_page_count, object->ref_count, object->flags); 2039 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n", 2040 object->shadow_count, 2041 object->backing_object ? object->backing_object->ref_count : 0, 2042 object->backing_object, (uintmax_t)object->backing_object_offset); 2043 2044 if (!full) 2045 return; 2046 2047 db_indent += 2; 2048 count = 0; 2049 TAILQ_FOREACH(p, &object->memq, listq) { 2050 if (count == 0) 2051 db_iprintf("memory:="); 2052 else if (count == 6) { 2053 db_printf("\n"); 2054 db_iprintf(" ..."); 2055 count = 0; 2056 } else 2057 db_printf(","); 2058 count++; 2059 2060 db_printf("(off=0x%jx,page=0x%jx)", 2061 (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p)); 2062 } 2063 if (count != 0) 2064 db_printf("\n"); 2065 db_indent -= 2; 2066 } 2067 2068 /* XXX. */ 2069 #undef count 2070 2071 /* XXX need this non-static entry for calling from vm_map_print. */ 2072 void 2073 vm_object_print( 2074 /* db_expr_t */ long addr, 2075 boolean_t have_addr, 2076 /* db_expr_t */ long count, 2077 char *modif) 2078 { 2079 vm_object_print_static(addr, have_addr, count, modif); 2080 } 2081 2082 DB_SHOW_COMMAND(vmopag, vm_object_print_pages) 2083 { 2084 vm_object_t object; 2085 int nl = 0; 2086 int c; 2087 2088 TAILQ_FOREACH(object, &vm_object_list, object_list) { 2089 vm_pindex_t idx, fidx; 2090 vm_pindex_t osize; 2091 vm_paddr_t pa = -1, padiff; 2092 int rcount; 2093 vm_page_t m; 2094 2095 db_printf("new object: %p\n", (void *)object); 2096 if (nl > 18) { 2097 c = cngetc(); 2098 if (c != ' ') 2099 return; 2100 nl = 0; 2101 } 2102 nl++; 2103 rcount = 0; 2104 fidx = 0; 2105 osize = object->size; 2106 if (osize > 128) 2107 osize = 128; 2108 for (idx = 0; idx < osize; idx++) { 2109 m = vm_page_lookup(object, idx); 2110 if (m == NULL) { 2111 if (rcount) { 2112 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2113 (long)fidx, rcount, (long)pa); 2114 if (nl > 18) { 2115 c = cngetc(); 2116 if (c != ' ') 2117 return; 2118 nl = 0; 2119 } 2120 nl++; 2121 rcount = 0; 2122 } 2123 continue; 2124 } 2125 2126 2127 if (rcount && 2128 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) { 2129 ++rcount; 2130 continue; 2131 } 2132 if (rcount) { 2133 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m); 2134 padiff >>= PAGE_SHIFT; 2135 padiff &= PQ_L2_MASK; 2136 if (padiff == 0) { 2137 pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE; 2138 ++rcount; 2139 continue; 2140 } 2141 db_printf(" index(%ld)run(%d)pa(0x%lx)", 2142 (long)fidx, rcount, (long)pa); 2143 db_printf("pd(%ld)\n", (long)padiff); 2144 if (nl > 18) { 2145 c = cngetc(); 2146 if (c != ' ') 2147 return; 2148 nl = 0; 2149 } 2150 nl++; 2151 } 2152 fidx = idx; 2153 pa = VM_PAGE_TO_PHYS(m); 2154 rcount = 1; 2155 } 2156 if (rcount) { 2157 db_printf(" index(%ld)run(%d)pa(0x%lx)\n", 2158 (long)fidx, rcount, (long)pa); 2159 if (nl > 18) { 2160 c = cngetc(); 2161 if (c != ' ') 2162 return; 2163 nl = 0; 2164 } 2165 nl++; 2166 } 2167 } 2168 } 2169 #endif /* DDB */ 2170