1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #pragma ident "%Z%%M% %I% %E% SMI" 40 41 /* 42 * VM - physical page management. 43 */ 44 45 #include <sys/types.h> 46 #include <sys/t_lock.h> 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/errno.h> 50 #include <sys/time.h> 51 #include <sys/vnode.h> 52 #include <sys/vm.h> 53 #include <sys/vtrace.h> 54 #include <sys/swap.h> 55 #include <sys/cmn_err.h> 56 #include <sys/tuneable.h> 57 #include <sys/sysmacros.h> 58 #include <sys/cpuvar.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/tnf_probe.h> 62 #include <sys/condvar_impl.h> 63 #include <sys/mem_config.h> 64 #include <sys/mem_cage.h> 65 #include <sys/kmem.h> 66 #include <sys/atomic.h> 67 #include <sys/strlog.h> 68 #include <sys/mman.h> 69 #include <sys/ontrap.h> 70 #include <sys/lgrp.h> 71 #include <sys/vfs.h> 72 73 #include <vm/hat.h> 74 #include <vm/anon.h> 75 #include <vm/page.h> 76 #include <vm/seg.h> 77 #include <vm/pvn.h> 78 #include <vm/seg_kmem.h> 79 #include <vm/vm_dep.h> 80 81 #include <fs/fs_subr.h> 82 83 static int nopageage = 0; 84 85 static pgcnt_t max_page_get; /* max page_get request size in pages */ 86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 87 88 /* 89 * freemem_lock protects all freemem variables: 90 * availrmem. Also this lock protects the globals which track the 91 * availrmem changes for accurate kernel footprint calculation. 92 * See below for an explanation of these 93 * globals. 94 */ 95 kmutex_t freemem_lock; 96 pgcnt_t availrmem; 97 pgcnt_t availrmem_initial; 98 99 /* 100 * These globals track availrmem changes to get a more accurate 101 * estimate of tke kernel size. Historically pp_kernel is used for 102 * kernel size and is based on availrmem. But availrmem is adjusted for 103 * locked pages in the system not just for kernel locked pages. 104 * These new counters will track the pages locked through segvn and 105 * by explicit user locking. 106 * 107 * segvn_pages_locked : This keeps track on a global basis how many pages 108 * are currently locked because of I/O. 109 * 110 * pages_locked : How many pages are locked becuase of user specified 111 * locking through mlock or plock. 112 * 113 * pages_useclaim,pages_claimed : These two variables track the 114 * cliam adjustments because of the protection changes on a segvn segment. 115 * 116 * All these globals are protected by the same lock which protects availrmem. 117 */ 118 pgcnt_t segvn_pages_locked; 119 pgcnt_t pages_locked; 120 pgcnt_t pages_useclaim; 121 pgcnt_t pages_claimed; 122 123 124 /* 125 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 126 */ 127 static kmutex_t new_freemem_lock; 128 static uint_t freemem_wait; /* someone waiting for freemem */ 129 static kcondvar_t freemem_cv; 130 131 /* 132 * The logical page free list is maintained as two lists, the 'free' 133 * and the 'cache' lists. 134 * The free list contains those pages that should be reused first. 135 * 136 * The implementation of the lists is machine dependent. 137 * page_get_freelist(), page_get_cachelist(), 138 * page_list_sub(), and page_list_add() 139 * form the interface to the machine dependent implementation. 140 * 141 * Pages with p_free set are on the cache list. 142 * Pages with p_free and p_age set are on the free list, 143 * 144 * A page may be locked while on either list. 145 */ 146 147 /* 148 * free list accounting stuff. 149 * 150 * 151 * Spread out the value for the number of pages on the 152 * page free and page cache lists. If there is just one 153 * value, then it must be under just one lock. 154 * The lock contention and cache traffic are a real bother. 155 * 156 * When we acquire and then drop a single pcf lock 157 * we can start in the middle of the array of pcf structures. 158 * If we acquire more than one pcf lock at a time, we need to 159 * start at the front to avoid deadlocking. 160 * 161 * pcf_count holds the number of pages in each pool. 162 * 163 * pcf_block is set when page_create_get_something() has asked the 164 * PSM page freelist and page cachelist routines without specifying 165 * a color and nothing came back. This is used to block anything 166 * else from moving pages from one list to the other while the 167 * lists are searched again. If a page is freeed while pcf_block is 168 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 169 * of clearning pcf_block, doing the wakeups, etc. 170 */ 171 172 #if NCPU <= 4 173 #define PAD 2 174 #define PCF_FANOUT 4 175 static uint_t pcf_mask = PCF_FANOUT - 1; 176 #else 177 #define PAD 10 178 #ifdef sun4v 179 #define PCF_FANOUT 32 180 #else 181 #define PCF_FANOUT 128 182 #endif 183 static uint_t pcf_mask = PCF_FANOUT - 1; 184 #endif 185 186 struct pcf { 187 kmutex_t pcf_lock; /* protects the structure */ 188 uint_t pcf_count; /* page count */ 189 uint_t pcf_wait; /* number of waiters */ 190 uint_t pcf_block; /* pcgs flag to page_free() */ 191 uint_t pcf_reserve; /* pages freed after pcf_block set */ 192 uint_t pcf_fill[PAD]; /* to line up on the caches */ 193 }; 194 195 static struct pcf pcf[PCF_FANOUT]; 196 #define PCF_INDEX() ((CPU->cpu_id) & (pcf_mask)) 197 198 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 199 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 200 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 201 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 202 203 #ifdef VM_STATS 204 205 /* 206 * No locks, but so what, they are only statistics. 207 */ 208 209 static struct page_tcnt { 210 int pc_free_cache; /* free's into cache list */ 211 int pc_free_dontneed; /* free's with dontneed */ 212 int pc_free_pageout; /* free's from pageout */ 213 int pc_free_free; /* free's into free list */ 214 int pc_free_pages; /* free's into large page free list */ 215 int pc_destroy_pages; /* large page destroy's */ 216 int pc_get_cache; /* get's from cache list */ 217 int pc_get_free; /* get's from free list */ 218 int pc_reclaim; /* reclaim's */ 219 int pc_abortfree; /* abort's of free pages */ 220 int pc_find_hit; /* find's that find page */ 221 int pc_find_miss; /* find's that don't find page */ 222 int pc_destroy_free; /* # of free pages destroyed */ 223 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 224 int pc_find_hashlen[PC_HASH_CNT+1]; 225 int pc_addclaim_pages; 226 int pc_subclaim_pages; 227 int pc_free_replacement_page[2]; 228 int pc_try_demote_pages[6]; 229 int pc_demote_pages[2]; 230 } pagecnt; 231 232 uint_t hashin_count; 233 uint_t hashin_not_held; 234 uint_t hashin_already; 235 236 uint_t hashout_count; 237 uint_t hashout_not_held; 238 239 uint_t page_create_count; 240 uint_t page_create_not_enough; 241 uint_t page_create_not_enough_again; 242 uint_t page_create_zero; 243 uint_t page_create_hashout; 244 uint_t page_create_page_lock_failed; 245 uint_t page_create_trylock_failed; 246 uint_t page_create_found_one; 247 uint_t page_create_hashin_failed; 248 uint_t page_create_dropped_phm; 249 250 uint_t page_create_new; 251 uint_t page_create_exists; 252 uint_t page_create_putbacks; 253 uint_t page_create_overshoot; 254 255 uint_t page_reclaim_zero; 256 uint_t page_reclaim_zero_locked; 257 258 uint_t page_rename_exists; 259 uint_t page_rename_count; 260 261 uint_t page_lookup_cnt[20]; 262 uint_t page_lookup_nowait_cnt[10]; 263 uint_t page_find_cnt; 264 uint_t page_exists_cnt; 265 uint_t page_exists_forreal_cnt; 266 uint_t page_lookup_dev_cnt; 267 uint_t get_cachelist_cnt; 268 uint_t page_create_cnt[10]; 269 uint_t alloc_pages[8]; 270 uint_t page_exphcontg[19]; 271 uint_t page_create_large_cnt[10]; 272 273 /* 274 * Collects statistics. 275 */ 276 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 277 uint_t mylen = 0; \ 278 \ 279 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 280 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 281 break; \ 282 } \ 283 if ((pp) != NULL) \ 284 pagecnt.pc_find_hit++; \ 285 else \ 286 pagecnt.pc_find_miss++; \ 287 if (mylen > PC_HASH_CNT) \ 288 mylen = PC_HASH_CNT; \ 289 pagecnt.pc_find_hashlen[mylen]++; \ 290 } 291 292 #else /* VM_STATS */ 293 294 /* 295 * Don't collect statistics 296 */ 297 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 298 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 299 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 300 break; \ 301 } \ 302 } 303 304 #endif /* VM_STATS */ 305 306 307 308 #ifdef DEBUG 309 #define MEMSEG_SEARCH_STATS 310 #endif 311 312 #ifdef MEMSEG_SEARCH_STATS 313 struct memseg_stats { 314 uint_t nsearch; 315 uint_t nlastwon; 316 uint_t nhashwon; 317 uint_t nnotfound; 318 } memseg_stats; 319 320 #define MEMSEG_STAT_INCR(v) \ 321 atomic_add_32(&memseg_stats.v, 1) 322 #else 323 #define MEMSEG_STAT_INCR(x) 324 #endif 325 326 struct memseg *memsegs; /* list of memory segments */ 327 328 329 static void page_init_mem_config(void); 330 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 331 static void page_do_hashout(page_t *); 332 333 static void page_demote_vp_pages(page_t *); 334 335 /* 336 * vm subsystem related initialization 337 */ 338 void 339 vm_init(void) 340 { 341 boolean_t callb_vm_cpr(void *, int); 342 343 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 344 page_init_mem_config(); 345 page_retire_init(); 346 } 347 348 /* 349 * This function is called at startup and when memory is added or deleted. 350 */ 351 void 352 init_pages_pp_maximum() 353 { 354 static pgcnt_t p_min; 355 static pgcnt_t pages_pp_maximum_startup; 356 static pgcnt_t avrmem_delta; 357 static int init_done; 358 static int user_set; /* true if set in /etc/system */ 359 360 if (init_done == 0) { 361 362 /* If the user specified a value, save it */ 363 if (pages_pp_maximum != 0) { 364 user_set = 1; 365 pages_pp_maximum_startup = pages_pp_maximum; 366 } 367 368 /* 369 * Setting of pages_pp_maximum is based first time 370 * on the value of availrmem just after the start-up 371 * allocations. To preserve this relationship at run 372 * time, use a delta from availrmem_initial. 373 */ 374 ASSERT(availrmem_initial >= availrmem); 375 avrmem_delta = availrmem_initial - availrmem; 376 377 /* The allowable floor of pages_pp_maximum */ 378 p_min = tune.t_minarmem + 100; 379 380 /* Make sure we don't come through here again. */ 381 init_done = 1; 382 } 383 /* 384 * Determine pages_pp_maximum, the number of currently available 385 * pages (availrmem) that can't be `locked'. If not set by 386 * the user, we set it to 4% of the currently available memory 387 * plus 4MB. 388 * But we also insist that it be greater than tune.t_minarmem; 389 * otherwise a process could lock down a lot of memory, get swapped 390 * out, and never have enough to get swapped back in. 391 */ 392 if (user_set) 393 pages_pp_maximum = pages_pp_maximum_startup; 394 else 395 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 396 + btop(4 * 1024 * 1024); 397 398 if (pages_pp_maximum <= p_min) { 399 pages_pp_maximum = p_min; 400 } 401 } 402 403 void 404 set_max_page_get(pgcnt_t target_total_pages) 405 { 406 max_page_get = target_total_pages / 2; 407 } 408 409 static pgcnt_t pending_delete; 410 411 /*ARGSUSED*/ 412 static void 413 page_mem_config_post_add( 414 void *arg, 415 pgcnt_t delta_pages) 416 { 417 set_max_page_get(total_pages - pending_delete); 418 init_pages_pp_maximum(); 419 } 420 421 /*ARGSUSED*/ 422 static int 423 page_mem_config_pre_del( 424 void *arg, 425 pgcnt_t delta_pages) 426 { 427 pgcnt_t nv; 428 429 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 430 set_max_page_get(total_pages - nv); 431 return (0); 432 } 433 434 /*ARGSUSED*/ 435 static void 436 page_mem_config_post_del( 437 void *arg, 438 pgcnt_t delta_pages, 439 int cancelled) 440 { 441 pgcnt_t nv; 442 443 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 444 set_max_page_get(total_pages - nv); 445 if (!cancelled) 446 init_pages_pp_maximum(); 447 } 448 449 static kphysm_setup_vector_t page_mem_config_vec = { 450 KPHYSM_SETUP_VECTOR_VERSION, 451 page_mem_config_post_add, 452 page_mem_config_pre_del, 453 page_mem_config_post_del, 454 }; 455 456 static void 457 page_init_mem_config(void) 458 { 459 int ret; 460 461 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 462 ASSERT(ret == 0); 463 } 464 465 /* 466 * Evenly spread out the PCF counters for large free pages 467 */ 468 static void 469 page_free_large_ctr(pgcnt_t npages) 470 { 471 static struct pcf *p = pcf; 472 pgcnt_t lump; 473 474 freemem += npages; 475 476 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 477 478 while (npages > 0) { 479 480 ASSERT(!p->pcf_block); 481 482 if (lump < npages) { 483 p->pcf_count += (uint_t)lump; 484 npages -= lump; 485 } else { 486 p->pcf_count += (uint_t)npages; 487 npages = 0; 488 } 489 490 ASSERT(!p->pcf_wait); 491 492 if (++p > &pcf[PCF_FANOUT - 1]) 493 p = pcf; 494 } 495 496 ASSERT(npages == 0); 497 } 498 499 /* 500 * Add a physical chunk of memory to the system freee lists during startup. 501 * Platform specific startup() allocates the memory for the page structs. 502 * 503 * num - number of page structures 504 * base - page number (pfn) to be associated with the first page. 505 * 506 * Since we are doing this during startup (ie. single threaded), we will 507 * use shortcut routines to avoid any locking overhead while putting all 508 * these pages on the freelists. 509 * 510 * NOTE: Any changes performed to page_free(), must also be performed to 511 * add_physmem() since this is how we initialize all page_t's at 512 * boot time. 513 */ 514 void 515 add_physmem( 516 page_t *pp, 517 pgcnt_t num, 518 pfn_t pnum) 519 { 520 page_t *root = NULL; 521 uint_t szc = page_num_pagesizes() - 1; 522 pgcnt_t large = page_get_pagecnt(szc); 523 pgcnt_t cnt = 0; 524 525 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 526 "add_physmem:pp %p num %lu", pp, num); 527 528 /* 529 * Arbitrarily limit the max page_get request 530 * to 1/2 of the page structs we have. 531 */ 532 total_pages += num; 533 set_max_page_get(total_pages); 534 535 PLCNT_MODIFY_MAX(pnum, (long)num); 536 537 /* 538 * The physical space for the pages array 539 * representing ram pages has already been 540 * allocated. Here we initialize each lock 541 * in the page structure, and put each on 542 * the free list 543 */ 544 for (; num; pp++, pnum++, num--) { 545 546 /* 547 * this needs to fill in the page number 548 * and do any other arch specific initialization 549 */ 550 add_physmem_cb(pp, pnum); 551 552 pp->p_lckcnt = 0; 553 pp->p_cowcnt = 0; 554 pp->p_slckcnt = 0; 555 556 /* 557 * Initialize the page lock as unlocked, since nobody 558 * can see or access this page yet. 559 */ 560 pp->p_selock = 0; 561 562 /* 563 * Initialize IO lock 564 */ 565 page_iolock_init(pp); 566 567 /* 568 * initialize other fields in the page_t 569 */ 570 PP_SETFREE(pp); 571 page_clr_all_props(pp); 572 PP_SETAGED(pp); 573 pp->p_offset = (u_offset_t)-1; 574 pp->p_next = pp; 575 pp->p_prev = pp; 576 577 /* 578 * Simple case: System doesn't support large pages. 579 */ 580 if (szc == 0) { 581 pp->p_szc = 0; 582 page_free_at_startup(pp); 583 continue; 584 } 585 586 /* 587 * Handle unaligned pages, we collect them up onto 588 * the root page until we have a full large page. 589 */ 590 if (!IS_P2ALIGNED(pnum, large)) { 591 592 /* 593 * If not in a large page, 594 * just free as small page. 595 */ 596 if (root == NULL) { 597 pp->p_szc = 0; 598 page_free_at_startup(pp); 599 continue; 600 } 601 602 /* 603 * Link a constituent page into the large page. 604 */ 605 pp->p_szc = szc; 606 page_list_concat(&root, &pp); 607 608 /* 609 * When large page is fully formed, free it. 610 */ 611 if (++cnt == large) { 612 page_free_large_ctr(cnt); 613 page_list_add_pages(root, PG_LIST_ISINIT); 614 root = NULL; 615 cnt = 0; 616 } 617 continue; 618 } 619 620 /* 621 * At this point we have a page number which 622 * is aligned. We assert that we aren't already 623 * in a different large page. 624 */ 625 ASSERT(IS_P2ALIGNED(pnum, large)); 626 ASSERT(root == NULL && cnt == 0); 627 628 /* 629 * If insufficient number of pages left to form 630 * a large page, just free the small page. 631 */ 632 if (num < large) { 633 pp->p_szc = 0; 634 page_free_at_startup(pp); 635 continue; 636 } 637 638 /* 639 * Otherwise start a new large page. 640 */ 641 pp->p_szc = szc; 642 cnt++; 643 root = pp; 644 } 645 ASSERT(root == NULL && cnt == 0); 646 } 647 648 /* 649 * Find a page representing the specified [vp, offset]. 650 * If we find the page but it is intransit coming in, 651 * it will have an "exclusive" lock and we wait for 652 * the i/o to complete. A page found on the free list 653 * is always reclaimed and then locked. On success, the page 654 * is locked, its data is valid and it isn't on the free 655 * list, while a NULL is returned if the page doesn't exist. 656 */ 657 page_t * 658 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 659 { 660 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 661 } 662 663 /* 664 * Find a page representing the specified [vp, offset]. 665 * We either return the one we found or, if passed in, 666 * create one with identity of [vp, offset] of the 667 * pre-allocated page. If we find exsisting page but it is 668 * intransit coming in, it will have an "exclusive" lock 669 * and we wait for the i/o to complete. A page found on 670 * the free list is always reclaimed and then locked. 671 * On success, the page is locked, its data is valid and 672 * it isn't on the free list, while a NULL is returned 673 * if the page doesn't exist and newpp is NULL; 674 */ 675 page_t * 676 page_lookup_create( 677 vnode_t *vp, 678 u_offset_t off, 679 se_t se, 680 page_t *newpp, 681 spgcnt_t *nrelocp, 682 int flags) 683 { 684 page_t *pp; 685 kmutex_t *phm; 686 ulong_t index; 687 uint_t hash_locked; 688 uint_t es; 689 690 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 691 VM_STAT_ADD(page_lookup_cnt[0]); 692 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 693 694 /* 695 * Acquire the appropriate page hash lock since 696 * we have to search the hash list. Pages that 697 * hash to this list can't change identity while 698 * this lock is held. 699 */ 700 hash_locked = 0; 701 index = PAGE_HASH_FUNC(vp, off); 702 phm = NULL; 703 top: 704 PAGE_HASH_SEARCH(index, pp, vp, off); 705 if (pp != NULL) { 706 VM_STAT_ADD(page_lookup_cnt[1]); 707 es = (newpp != NULL) ? 1 : 0; 708 es |= flags; 709 if (!hash_locked) { 710 VM_STAT_ADD(page_lookup_cnt[2]); 711 if (!page_try_reclaim_lock(pp, se, es)) { 712 /* 713 * On a miss, acquire the phm. Then 714 * next time, page_lock() will be called, 715 * causing a wait if the page is busy. 716 * just looping with page_trylock() would 717 * get pretty boring. 718 */ 719 VM_STAT_ADD(page_lookup_cnt[3]); 720 phm = PAGE_HASH_MUTEX(index); 721 mutex_enter(phm); 722 hash_locked = 1; 723 goto top; 724 } 725 } else { 726 VM_STAT_ADD(page_lookup_cnt[4]); 727 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 728 VM_STAT_ADD(page_lookup_cnt[5]); 729 goto top; 730 } 731 } 732 733 /* 734 * Since `pp' is locked it can not change identity now. 735 * Reconfirm we locked the correct page. 736 * 737 * Both the p_vnode and p_offset *must* be cast volatile 738 * to force a reload of their values: The PAGE_HASH_SEARCH 739 * macro will have stuffed p_vnode and p_offset into 740 * registers before calling page_trylock(); another thread, 741 * actually holding the hash lock, could have changed the 742 * page's identity in memory, but our registers would not 743 * be changed, fooling the reconfirmation. If the hash 744 * lock was held during the search, the casting would 745 * not be needed. 746 */ 747 VM_STAT_ADD(page_lookup_cnt[6]); 748 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 749 ((volatile u_offset_t)(pp->p_offset) != off)) { 750 VM_STAT_ADD(page_lookup_cnt[7]); 751 if (hash_locked) { 752 panic("page_lookup_create: lost page %p", 753 (void *)pp); 754 /*NOTREACHED*/ 755 } 756 page_unlock(pp); 757 phm = PAGE_HASH_MUTEX(index); 758 mutex_enter(phm); 759 hash_locked = 1; 760 goto top; 761 } 762 763 /* 764 * If page_trylock() was called, then pp may still be on 765 * the cachelist (can't be on the free list, it would not 766 * have been found in the search). If it is on the 767 * cachelist it must be pulled now. To pull the page from 768 * the cachelist, it must be exclusively locked. 769 * 770 * The other big difference between page_trylock() and 771 * page_lock(), is that page_lock() will pull the 772 * page from whatever free list (the cache list in this 773 * case) the page is on. If page_trylock() was used 774 * above, then we have to do the reclaim ourselves. 775 */ 776 if ((!hash_locked) && (PP_ISFREE(pp))) { 777 ASSERT(PP_ISAGED(pp) == 0); 778 VM_STAT_ADD(page_lookup_cnt[8]); 779 780 /* 781 * page_relcaim will insure that we 782 * have this page exclusively 783 */ 784 785 if (!page_reclaim(pp, NULL)) { 786 /* 787 * Page_reclaim dropped whatever lock 788 * we held. 789 */ 790 VM_STAT_ADD(page_lookup_cnt[9]); 791 phm = PAGE_HASH_MUTEX(index); 792 mutex_enter(phm); 793 hash_locked = 1; 794 goto top; 795 } else if (se == SE_SHARED && newpp == NULL) { 796 VM_STAT_ADD(page_lookup_cnt[10]); 797 page_downgrade(pp); 798 } 799 } 800 801 if (hash_locked) { 802 mutex_exit(phm); 803 } 804 805 if (newpp != NULL && pp->p_szc < newpp->p_szc && 806 PAGE_EXCL(pp) && nrelocp != NULL) { 807 ASSERT(nrelocp != NULL); 808 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 809 NULL); 810 if (*nrelocp > 0) { 811 VM_STAT_COND_ADD(*nrelocp == 1, 812 page_lookup_cnt[11]); 813 VM_STAT_COND_ADD(*nrelocp > 1, 814 page_lookup_cnt[12]); 815 pp = newpp; 816 se = SE_EXCL; 817 } else { 818 if (se == SE_SHARED) { 819 page_downgrade(pp); 820 } 821 VM_STAT_ADD(page_lookup_cnt[13]); 822 } 823 } else if (newpp != NULL && nrelocp != NULL) { 824 if (PAGE_EXCL(pp) && se == SE_SHARED) { 825 page_downgrade(pp); 826 } 827 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 828 page_lookup_cnt[14]); 829 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 830 page_lookup_cnt[15]); 831 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 832 page_lookup_cnt[16]); 833 } else if (newpp != NULL && PAGE_EXCL(pp)) { 834 se = SE_EXCL; 835 } 836 } else if (!hash_locked) { 837 VM_STAT_ADD(page_lookup_cnt[17]); 838 phm = PAGE_HASH_MUTEX(index); 839 mutex_enter(phm); 840 hash_locked = 1; 841 goto top; 842 } else if (newpp != NULL) { 843 /* 844 * If we have a preallocated page then 845 * insert it now and basically behave like 846 * page_create. 847 */ 848 VM_STAT_ADD(page_lookup_cnt[18]); 849 /* 850 * Since we hold the page hash mutex and 851 * just searched for this page, page_hashin 852 * had better not fail. If it does, that 853 * means some thread did not follow the 854 * page hash mutex rules. Panic now and 855 * get it over with. As usual, go down 856 * holding all the locks. 857 */ 858 ASSERT(MUTEX_HELD(phm)); 859 if (!page_hashin(newpp, vp, off, phm)) { 860 ASSERT(MUTEX_HELD(phm)); 861 panic("page_lookup_create: hashin failed %p %p %llx %p", 862 (void *)newpp, (void *)vp, off, (void *)phm); 863 /*NOTREACHED*/ 864 } 865 ASSERT(MUTEX_HELD(phm)); 866 mutex_exit(phm); 867 phm = NULL; 868 page_set_props(newpp, P_REF); 869 page_io_lock(newpp); 870 pp = newpp; 871 se = SE_EXCL; 872 } else { 873 VM_STAT_ADD(page_lookup_cnt[19]); 874 mutex_exit(phm); 875 } 876 877 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 878 879 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 880 881 return (pp); 882 } 883 884 /* 885 * Search the hash list for the page representing the 886 * specified [vp, offset] and return it locked. Skip 887 * free pages and pages that cannot be locked as requested. 888 * Used while attempting to kluster pages. 889 */ 890 page_t * 891 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 892 { 893 page_t *pp; 894 kmutex_t *phm; 895 ulong_t index; 896 uint_t locked; 897 898 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 899 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 900 901 index = PAGE_HASH_FUNC(vp, off); 902 PAGE_HASH_SEARCH(index, pp, vp, off); 903 locked = 0; 904 if (pp == NULL) { 905 top: 906 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 907 locked = 1; 908 phm = PAGE_HASH_MUTEX(index); 909 mutex_enter(phm); 910 PAGE_HASH_SEARCH(index, pp, vp, off); 911 } 912 913 if (pp == NULL || PP_ISFREE(pp)) { 914 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 915 pp = NULL; 916 } else { 917 if (!page_trylock(pp, se)) { 918 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 919 pp = NULL; 920 } else { 921 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 922 /* 923 * See the comment in page_lookup() 924 */ 925 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 926 ((u_offset_t)(pp->p_offset) != off)) { 927 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 928 if (locked) { 929 panic("page_lookup_nowait %p", 930 (void *)pp); 931 /*NOTREACHED*/ 932 } 933 page_unlock(pp); 934 goto top; 935 } 936 if (PP_ISFREE(pp)) { 937 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 938 page_unlock(pp); 939 pp = NULL; 940 } 941 } 942 } 943 if (locked) { 944 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 945 mutex_exit(phm); 946 } 947 948 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 949 950 return (pp); 951 } 952 953 /* 954 * Search the hash list for a page with the specified [vp, off] 955 * that is known to exist and is already locked. This routine 956 * is typically used by segment SOFTUNLOCK routines. 957 */ 958 page_t * 959 page_find(vnode_t *vp, u_offset_t off) 960 { 961 page_t *pp; 962 kmutex_t *phm; 963 ulong_t index; 964 965 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 966 VM_STAT_ADD(page_find_cnt); 967 968 index = PAGE_HASH_FUNC(vp, off); 969 phm = PAGE_HASH_MUTEX(index); 970 971 mutex_enter(phm); 972 PAGE_HASH_SEARCH(index, pp, vp, off); 973 mutex_exit(phm); 974 975 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 976 return (pp); 977 } 978 979 /* 980 * Determine whether a page with the specified [vp, off] 981 * currently exists in the system. Obviously this should 982 * only be considered as a hint since nothing prevents the 983 * page from disappearing or appearing immediately after 984 * the return from this routine. Subsequently, we don't 985 * even bother to lock the list. 986 */ 987 page_t * 988 page_exists(vnode_t *vp, u_offset_t off) 989 { 990 page_t *pp; 991 ulong_t index; 992 993 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 994 VM_STAT_ADD(page_exists_cnt); 995 996 index = PAGE_HASH_FUNC(vp, off); 997 PAGE_HASH_SEARCH(index, pp, vp, off); 998 999 return (pp); 1000 } 1001 1002 /* 1003 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1004 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1005 * with these pages locked SHARED. If necessary reclaim pages from 1006 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1007 * 1008 * If we fail to lock pages still return 1 if pages exist and contiguous. 1009 * But in this case return value is just a hint. ppa array won't be filled. 1010 * Caller should initialize ppa[0] as NULL to distinguish return value. 1011 * 1012 * Returns 0 if pages don't exist or not physically contiguous. 1013 * 1014 * This routine doesn't work for anonymous(swapfs) pages. 1015 */ 1016 int 1017 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1018 { 1019 pgcnt_t pages; 1020 pfn_t pfn; 1021 page_t *rootpp; 1022 pgcnt_t i; 1023 pgcnt_t j; 1024 u_offset_t save_off = off; 1025 ulong_t index; 1026 kmutex_t *phm; 1027 page_t *pp; 1028 uint_t pszc; 1029 int loopcnt = 0; 1030 1031 ASSERT(szc != 0); 1032 ASSERT(vp != NULL); 1033 ASSERT(!IS_SWAPFSVP(vp)); 1034 ASSERT(vp != &kvp); 1035 1036 again: 1037 if (++loopcnt > 3) { 1038 VM_STAT_ADD(page_exphcontg[0]); 1039 return (0); 1040 } 1041 1042 index = PAGE_HASH_FUNC(vp, off); 1043 phm = PAGE_HASH_MUTEX(index); 1044 1045 mutex_enter(phm); 1046 PAGE_HASH_SEARCH(index, pp, vp, off); 1047 mutex_exit(phm); 1048 1049 VM_STAT_ADD(page_exphcontg[1]); 1050 1051 if (pp == NULL) { 1052 VM_STAT_ADD(page_exphcontg[2]); 1053 return (0); 1054 } 1055 1056 pages = page_get_pagecnt(szc); 1057 rootpp = pp; 1058 pfn = rootpp->p_pagenum; 1059 1060 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1061 VM_STAT_ADD(page_exphcontg[3]); 1062 if (!page_trylock(pp, SE_SHARED)) { 1063 VM_STAT_ADD(page_exphcontg[4]); 1064 return (1); 1065 } 1066 if (pp->p_szc != pszc || pp->p_vnode != vp || 1067 pp->p_offset != off) { 1068 VM_STAT_ADD(page_exphcontg[5]); 1069 page_unlock(pp); 1070 off = save_off; 1071 goto again; 1072 } 1073 /* 1074 * szc was non zero and vnode and offset matched after we 1075 * locked the page it means it can't become free on us. 1076 */ 1077 ASSERT(!PP_ISFREE(pp)); 1078 if (!IS_P2ALIGNED(pfn, pages)) { 1079 page_unlock(pp); 1080 return (0); 1081 } 1082 ppa[0] = pp; 1083 pp++; 1084 off += PAGESIZE; 1085 pfn++; 1086 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1087 if (!page_trylock(pp, SE_SHARED)) { 1088 VM_STAT_ADD(page_exphcontg[6]); 1089 pp--; 1090 while (i-- > 0) { 1091 page_unlock(pp); 1092 pp--; 1093 } 1094 ppa[0] = NULL; 1095 return (1); 1096 } 1097 if (pp->p_szc != pszc) { 1098 VM_STAT_ADD(page_exphcontg[7]); 1099 page_unlock(pp); 1100 pp--; 1101 while (i-- > 0) { 1102 page_unlock(pp); 1103 pp--; 1104 } 1105 ppa[0] = NULL; 1106 off = save_off; 1107 goto again; 1108 } 1109 /* 1110 * szc the same as for previous already locked pages 1111 * with right identity. Since this page had correct 1112 * szc after we locked it can't get freed or destroyed 1113 * and therefore must have the expected identity. 1114 */ 1115 ASSERT(!PP_ISFREE(pp)); 1116 if (pp->p_vnode != vp || 1117 pp->p_offset != off) { 1118 panic("page_exists_physcontig: " 1119 "large page identity doesn't match"); 1120 } 1121 ppa[i] = pp; 1122 ASSERT(pp->p_pagenum == pfn); 1123 } 1124 VM_STAT_ADD(page_exphcontg[8]); 1125 ppa[pages] = NULL; 1126 return (1); 1127 } else if (pszc >= szc) { 1128 VM_STAT_ADD(page_exphcontg[9]); 1129 if (!IS_P2ALIGNED(pfn, pages)) { 1130 return (0); 1131 } 1132 return (1); 1133 } 1134 1135 if (!IS_P2ALIGNED(pfn, pages)) { 1136 VM_STAT_ADD(page_exphcontg[10]); 1137 return (0); 1138 } 1139 1140 if (page_numtomemseg_nolock(pfn) != 1141 page_numtomemseg_nolock(pfn + pages - 1)) { 1142 VM_STAT_ADD(page_exphcontg[11]); 1143 return (0); 1144 } 1145 1146 /* 1147 * We loop up 4 times across pages to promote page size. 1148 * We're extra cautious to promote page size atomically with respect 1149 * to everybody else. But we can probably optimize into 1 loop if 1150 * this becomes an issue. 1151 */ 1152 1153 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1154 ASSERT(pp->p_pagenum == pfn); 1155 if (!page_trylock(pp, SE_EXCL)) { 1156 VM_STAT_ADD(page_exphcontg[12]); 1157 break; 1158 } 1159 if (pp->p_vnode != vp || 1160 pp->p_offset != off) { 1161 VM_STAT_ADD(page_exphcontg[13]); 1162 page_unlock(pp); 1163 break; 1164 } 1165 if (pp->p_szc >= szc) { 1166 ASSERT(i == 0); 1167 page_unlock(pp); 1168 off = save_off; 1169 goto again; 1170 } 1171 } 1172 1173 if (i != pages) { 1174 VM_STAT_ADD(page_exphcontg[14]); 1175 --pp; 1176 while (i-- > 0) { 1177 page_unlock(pp); 1178 --pp; 1179 } 1180 return (0); 1181 } 1182 1183 pp = rootpp; 1184 for (i = 0; i < pages; i++, pp++) { 1185 if (PP_ISFREE(pp)) { 1186 VM_STAT_ADD(page_exphcontg[15]); 1187 ASSERT(!PP_ISAGED(pp)); 1188 ASSERT(pp->p_szc == 0); 1189 if (!page_reclaim(pp, NULL)) { 1190 break; 1191 } 1192 } else { 1193 ASSERT(pp->p_szc < szc); 1194 VM_STAT_ADD(page_exphcontg[16]); 1195 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1196 } 1197 } 1198 if (i < pages) { 1199 VM_STAT_ADD(page_exphcontg[17]); 1200 /* 1201 * page_reclaim failed because we were out of memory. 1202 * drop the rest of the locks and return because this page 1203 * must be already reallocated anyway. 1204 */ 1205 pp = rootpp; 1206 for (j = 0; j < pages; j++, pp++) { 1207 if (j != i) { 1208 page_unlock(pp); 1209 } 1210 } 1211 return (0); 1212 } 1213 1214 off = save_off; 1215 pp = rootpp; 1216 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1217 ASSERT(PAGE_EXCL(pp)); 1218 ASSERT(!PP_ISFREE(pp)); 1219 ASSERT(!hat_page_is_mapped(pp)); 1220 ASSERT(pp->p_vnode == vp); 1221 ASSERT(pp->p_offset == off); 1222 pp->p_szc = szc; 1223 } 1224 pp = rootpp; 1225 for (i = 0; i < pages; i++, pp++) { 1226 if (ppa == NULL) { 1227 page_unlock(pp); 1228 } else { 1229 ppa[i] = pp; 1230 page_downgrade(ppa[i]); 1231 } 1232 } 1233 if (ppa != NULL) { 1234 ppa[pages] = NULL; 1235 } 1236 VM_STAT_ADD(page_exphcontg[18]); 1237 ASSERT(vp->v_pages != NULL); 1238 return (1); 1239 } 1240 1241 /* 1242 * Determine whether a page with the specified [vp, off] 1243 * currently exists in the system and if so return its 1244 * size code. Obviously this should only be considered as 1245 * a hint since nothing prevents the page from disappearing 1246 * or appearing immediately after the return from this routine. 1247 */ 1248 int 1249 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1250 { 1251 page_t *pp; 1252 kmutex_t *phm; 1253 ulong_t index; 1254 int rc = 0; 1255 1256 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1257 ASSERT(szc != NULL); 1258 VM_STAT_ADD(page_exists_forreal_cnt); 1259 1260 index = PAGE_HASH_FUNC(vp, off); 1261 phm = PAGE_HASH_MUTEX(index); 1262 1263 mutex_enter(phm); 1264 PAGE_HASH_SEARCH(index, pp, vp, off); 1265 if (pp != NULL) { 1266 *szc = pp->p_szc; 1267 rc = 1; 1268 } 1269 mutex_exit(phm); 1270 return (rc); 1271 } 1272 1273 /* wakeup threads waiting for pages in page_create_get_something() */ 1274 void 1275 wakeup_pcgs(void) 1276 { 1277 if (!CV_HAS_WAITERS(&pcgs_cv)) 1278 return; 1279 cv_broadcast(&pcgs_cv); 1280 } 1281 1282 /* 1283 * 'freemem' is used all over the kernel as an indication of how many 1284 * pages are free (either on the cache list or on the free page list) 1285 * in the system. In very few places is a really accurate 'freemem' 1286 * needed. To avoid contention of the lock protecting a the 1287 * single freemem, it was spread out into NCPU buckets. Set_freemem 1288 * sets freemem to the total of all NCPU buckets. It is called from 1289 * clock() on each TICK. 1290 */ 1291 void 1292 set_freemem() 1293 { 1294 struct pcf *p; 1295 ulong_t t; 1296 uint_t i; 1297 1298 t = 0; 1299 p = pcf; 1300 for (i = 0; i < PCF_FANOUT; i++) { 1301 t += p->pcf_count; 1302 p++; 1303 } 1304 freemem = t; 1305 1306 /* 1307 * Don't worry about grabbing mutex. It's not that 1308 * critical if we miss a tick or two. This is 1309 * where we wakeup possible delayers in 1310 * page_create_get_something(). 1311 */ 1312 wakeup_pcgs(); 1313 } 1314 1315 ulong_t 1316 get_freemem() 1317 { 1318 struct pcf *p; 1319 ulong_t t; 1320 uint_t i; 1321 1322 t = 0; 1323 p = pcf; 1324 for (i = 0; i < PCF_FANOUT; i++) { 1325 t += p->pcf_count; 1326 p++; 1327 } 1328 /* 1329 * We just calculated it, might as well set it. 1330 */ 1331 freemem = t; 1332 return (t); 1333 } 1334 1335 /* 1336 * Acquire all of the page cache & free (pcf) locks. 1337 */ 1338 void 1339 pcf_acquire_all() 1340 { 1341 struct pcf *p; 1342 uint_t i; 1343 1344 p = pcf; 1345 for (i = 0; i < PCF_FANOUT; i++) { 1346 mutex_enter(&p->pcf_lock); 1347 p++; 1348 } 1349 } 1350 1351 /* 1352 * Release all the pcf_locks. 1353 */ 1354 void 1355 pcf_release_all() 1356 { 1357 struct pcf *p; 1358 uint_t i; 1359 1360 p = pcf; 1361 for (i = 0; i < PCF_FANOUT; i++) { 1362 mutex_exit(&p->pcf_lock); 1363 p++; 1364 } 1365 } 1366 1367 /* 1368 * Inform the VM system that we need some pages freed up. 1369 * Calls must be symmetric, e.g.: 1370 * 1371 * page_needfree(100); 1372 * wait a bit; 1373 * page_needfree(-100); 1374 */ 1375 void 1376 page_needfree(spgcnt_t npages) 1377 { 1378 mutex_enter(&new_freemem_lock); 1379 needfree += npages; 1380 mutex_exit(&new_freemem_lock); 1381 } 1382 1383 /* 1384 * Throttle for page_create(): try to prevent freemem from dropping 1385 * below throttlefree. We can't provide a 100% guarantee because 1386 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1387 * nibble away at the freelist. However, we can block all PG_WAIT 1388 * allocations until memory becomes available. The motivation is 1389 * that several things can fall apart when there's no free memory: 1390 * 1391 * (1) If pageout() needs memory to push a page, the system deadlocks. 1392 * 1393 * (2) By (broken) specification, timeout(9F) can neither fail nor 1394 * block, so it has no choice but to panic the system if it 1395 * cannot allocate a callout structure. 1396 * 1397 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1398 * it panics if it cannot allocate a callback structure. 1399 * 1400 * (4) Untold numbers of third-party drivers have not yet been hardened 1401 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1402 * success and panic the system with a data fault on failure. 1403 * (The long-term solution to this particular problem is to ship 1404 * hostile fault-injecting DEBUG kernels with the DDK.) 1405 * 1406 * It is theoretically impossible to guarantee success of non-blocking 1407 * allocations, but in practice, this throttle is very hard to break. 1408 */ 1409 static int 1410 page_create_throttle(pgcnt_t npages, int flags) 1411 { 1412 ulong_t fm; 1413 uint_t i; 1414 pgcnt_t tf; /* effective value of throttlefree */ 1415 1416 /* 1417 * Never deny pages when: 1418 * - it's a thread that cannot block [NOMEMWAIT()] 1419 * - the allocation cannot block and must not fail 1420 * - the allocation cannot block and is pageout dispensated 1421 */ 1422 if (NOMEMWAIT() || 1423 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1424 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1425 return (1); 1426 1427 /* 1428 * If the allocation can't block, we look favorably upon it 1429 * unless we're below pageout_reserve. In that case we fail 1430 * the allocation because we want to make sure there are a few 1431 * pages available for pageout. 1432 */ 1433 if ((flags & PG_WAIT) == 0) 1434 return (freemem >= npages + pageout_reserve); 1435 1436 /* Calculate the effective throttlefree value */ 1437 tf = throttlefree - 1438 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1439 1440 cv_signal(&proc_pageout->p_cv); 1441 1442 while (freemem < npages + tf) { 1443 pcf_acquire_all(); 1444 mutex_enter(&new_freemem_lock); 1445 fm = 0; 1446 for (i = 0; i < PCF_FANOUT; i++) { 1447 fm += pcf[i].pcf_count; 1448 pcf[i].pcf_wait++; 1449 mutex_exit(&pcf[i].pcf_lock); 1450 } 1451 freemem = fm; 1452 needfree += npages; 1453 freemem_wait++; 1454 cv_wait(&freemem_cv, &new_freemem_lock); 1455 freemem_wait--; 1456 needfree -= npages; 1457 mutex_exit(&new_freemem_lock); 1458 } 1459 return (1); 1460 } 1461 1462 /* 1463 * page_create_wait() is called to either coalecse pages from the 1464 * different pcf buckets or to wait because there simply are not 1465 * enough pages to satisfy the caller's request. 1466 * 1467 * Sadly, this is called from platform/vm/vm_machdep.c 1468 */ 1469 int 1470 page_create_wait(size_t npages, uint_t flags) 1471 { 1472 pgcnt_t total; 1473 uint_t i; 1474 struct pcf *p; 1475 1476 /* 1477 * Wait until there are enough free pages to satisfy our 1478 * entire request. 1479 * We set needfree += npages before prodding pageout, to make sure 1480 * it does real work when npages > lotsfree > freemem. 1481 */ 1482 VM_STAT_ADD(page_create_not_enough); 1483 1484 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1485 checkagain: 1486 if ((flags & PG_NORELOC) && 1487 kcage_freemem < kcage_throttlefree + npages) 1488 (void) kcage_create_throttle(npages, flags); 1489 1490 if (freemem < npages + throttlefree) 1491 if (!page_create_throttle(npages, flags)) 1492 return (0); 1493 1494 /* 1495 * Since page_create_va() looked at every 1496 * bucket, assume we are going to have to wait. 1497 * Get all of the pcf locks. 1498 */ 1499 total = 0; 1500 p = pcf; 1501 for (i = 0; i < PCF_FANOUT; i++) { 1502 mutex_enter(&p->pcf_lock); 1503 total += p->pcf_count; 1504 if (total >= npages) { 1505 /* 1506 * Wow! There are enough pages laying around 1507 * to satisfy the request. Do the accounting, 1508 * drop the locks we acquired, and go back. 1509 * 1510 * freemem is not protected by any lock. So, 1511 * we cannot have any assertion containing 1512 * freemem. 1513 */ 1514 freemem -= npages; 1515 1516 while (p >= pcf) { 1517 if (p->pcf_count <= npages) { 1518 npages -= p->pcf_count; 1519 p->pcf_count = 0; 1520 } else { 1521 p->pcf_count -= (uint_t)npages; 1522 npages = 0; 1523 } 1524 mutex_exit(&p->pcf_lock); 1525 p--; 1526 } 1527 ASSERT(npages == 0); 1528 return (1); 1529 } 1530 p++; 1531 } 1532 1533 /* 1534 * All of the pcf locks are held, there are not enough pages 1535 * to satisfy the request (npages < total). 1536 * Be sure to acquire the new_freemem_lock before dropping 1537 * the pcf locks. This prevents dropping wakeups in page_free(). 1538 * The order is always pcf_lock then new_freemem_lock. 1539 * 1540 * Since we hold all the pcf locks, it is a good time to set freemem. 1541 * 1542 * If the caller does not want to wait, return now. 1543 * Else turn the pageout daemon loose to find something 1544 * and wait till it does. 1545 * 1546 */ 1547 freemem = total; 1548 1549 if ((flags & PG_WAIT) == 0) { 1550 pcf_release_all(); 1551 1552 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1553 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1554 return (0); 1555 } 1556 1557 ASSERT(proc_pageout != NULL); 1558 cv_signal(&proc_pageout->p_cv); 1559 1560 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1561 "page_create_sleep_start: freemem %ld needfree %ld", 1562 freemem, needfree); 1563 1564 /* 1565 * We are going to wait. 1566 * We currently hold all of the pcf_locks, 1567 * get the new_freemem_lock (it protects freemem_wait), 1568 * before dropping the pcf_locks. 1569 */ 1570 mutex_enter(&new_freemem_lock); 1571 1572 p = pcf; 1573 for (i = 0; i < PCF_FANOUT; i++) { 1574 p->pcf_wait++; 1575 mutex_exit(&p->pcf_lock); 1576 p++; 1577 } 1578 1579 needfree += npages; 1580 freemem_wait++; 1581 1582 cv_wait(&freemem_cv, &new_freemem_lock); 1583 1584 freemem_wait--; 1585 needfree -= npages; 1586 1587 mutex_exit(&new_freemem_lock); 1588 1589 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1590 "page_create_sleep_end: freemem %ld needfree %ld", 1591 freemem, needfree); 1592 1593 VM_STAT_ADD(page_create_not_enough_again); 1594 goto checkagain; 1595 } 1596 1597 /* 1598 * A routine to do the opposite of page_create_wait(). 1599 */ 1600 void 1601 page_create_putback(spgcnt_t npages) 1602 { 1603 struct pcf *p; 1604 pgcnt_t lump; 1605 uint_t *which; 1606 1607 /* 1608 * When a contiguous lump is broken up, we have to 1609 * deal with lots of pages (min 64) so lets spread 1610 * the wealth around. 1611 */ 1612 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 1613 freemem += npages; 1614 1615 for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) { 1616 which = &p->pcf_count; 1617 1618 mutex_enter(&p->pcf_lock); 1619 1620 if (p->pcf_block) { 1621 which = &p->pcf_reserve; 1622 } 1623 1624 if (lump < npages) { 1625 *which += (uint_t)lump; 1626 npages -= lump; 1627 } else { 1628 *which += (uint_t)npages; 1629 npages = 0; 1630 } 1631 1632 if (p->pcf_wait) { 1633 mutex_enter(&new_freemem_lock); 1634 /* 1635 * Check to see if some other thread 1636 * is actually waiting. Another bucket 1637 * may have woken it up by now. If there 1638 * are no waiters, then set our pcf_wait 1639 * count to zero to avoid coming in here 1640 * next time. 1641 */ 1642 if (freemem_wait) { 1643 if (npages > 1) { 1644 cv_broadcast(&freemem_cv); 1645 } else { 1646 cv_signal(&freemem_cv); 1647 } 1648 p->pcf_wait--; 1649 } else { 1650 p->pcf_wait = 0; 1651 } 1652 mutex_exit(&new_freemem_lock); 1653 } 1654 mutex_exit(&p->pcf_lock); 1655 } 1656 ASSERT(npages == 0); 1657 } 1658 1659 /* 1660 * A helper routine for page_create_get_something. 1661 * The indenting got to deep down there. 1662 * Unblock the pcf counters. Any pages freed after 1663 * pcf_block got set are moved to pcf_count and 1664 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1665 */ 1666 static void 1667 pcgs_unblock(void) 1668 { 1669 int i; 1670 struct pcf *p; 1671 1672 /* Update freemem while we're here. */ 1673 freemem = 0; 1674 p = pcf; 1675 for (i = 0; i < PCF_FANOUT; i++) { 1676 mutex_enter(&p->pcf_lock); 1677 ASSERT(p->pcf_count == 0); 1678 p->pcf_count = p->pcf_reserve; 1679 p->pcf_block = 0; 1680 freemem += p->pcf_count; 1681 if (p->pcf_wait) { 1682 mutex_enter(&new_freemem_lock); 1683 if (freemem_wait) { 1684 if (p->pcf_reserve > 1) { 1685 cv_broadcast(&freemem_cv); 1686 p->pcf_wait = 0; 1687 } else { 1688 cv_signal(&freemem_cv); 1689 p->pcf_wait--; 1690 } 1691 } else { 1692 p->pcf_wait = 0; 1693 } 1694 mutex_exit(&new_freemem_lock); 1695 } 1696 p->pcf_reserve = 0; 1697 mutex_exit(&p->pcf_lock); 1698 p++; 1699 } 1700 } 1701 1702 /* 1703 * Called from page_create_va() when both the cache and free lists 1704 * have been checked once. 1705 * 1706 * Either returns a page or panics since the accounting was done 1707 * way before we got here. 1708 * 1709 * We don't come here often, so leave the accounting on permanently. 1710 */ 1711 1712 #define MAX_PCGS 100 1713 1714 #ifdef DEBUG 1715 #define PCGS_TRIES 100 1716 #else /* DEBUG */ 1717 #define PCGS_TRIES 10 1718 #endif /* DEBUG */ 1719 1720 #ifdef VM_STATS 1721 uint_t pcgs_counts[PCGS_TRIES]; 1722 uint_t pcgs_too_many; 1723 uint_t pcgs_entered; 1724 uint_t pcgs_entered_noreloc; 1725 uint_t pcgs_locked; 1726 uint_t pcgs_cagelocked; 1727 #endif /* VM_STATS */ 1728 1729 static page_t * 1730 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1731 caddr_t vaddr, uint_t flags) 1732 { 1733 uint_t count; 1734 page_t *pp; 1735 uint_t locked, i; 1736 struct pcf *p; 1737 lgrp_t *lgrp; 1738 int cagelocked = 0; 1739 1740 VM_STAT_ADD(pcgs_entered); 1741 1742 /* 1743 * Tap any reserve freelists: if we fail now, we'll die 1744 * since the page(s) we're looking for have already been 1745 * accounted for. 1746 */ 1747 flags |= PG_PANIC; 1748 1749 if ((flags & PG_NORELOC) != 0) { 1750 VM_STAT_ADD(pcgs_entered_noreloc); 1751 /* 1752 * Requests for free pages from critical threads 1753 * such as pageout still won't throttle here, but 1754 * we must try again, to give the cageout thread 1755 * another chance to catch up. Since we already 1756 * accounted for the pages, we had better get them 1757 * this time. 1758 * 1759 * N.B. All non-critical threads acquire the pcgs_cagelock 1760 * to serialize access to the freelists. This implements a 1761 * turnstile-type synchornization to avoid starvation of 1762 * critical requests for PG_NORELOC memory by non-critical 1763 * threads: all non-critical threads must acquire a 'ticket' 1764 * before passing through, which entails making sure 1765 * kcage_freemem won't fall below minfree prior to grabbing 1766 * pages from the freelists. 1767 */ 1768 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1769 mutex_enter(&pcgs_cagelock); 1770 cagelocked = 1; 1771 VM_STAT_ADD(pcgs_cagelocked); 1772 } 1773 } 1774 1775 /* 1776 * Time to get serious. 1777 * We failed to get a `correctly colored' page from both the 1778 * free and cache lists. 1779 * We escalate in stage. 1780 * 1781 * First try both lists without worring about color. 1782 * 1783 * Then, grab all page accounting locks (ie. pcf[]) and 1784 * steal any pages that they have and set the pcf_block flag to 1785 * stop deletions from the lists. This will help because 1786 * a page can get added to the free list while we are looking 1787 * at the cache list, then another page could be added to the cache 1788 * list allowing the page on the free list to be removed as we 1789 * move from looking at the cache list to the free list. This 1790 * could happen over and over. We would never find the page 1791 * we have accounted for. 1792 * 1793 * Noreloc pages are a subset of the global (relocatable) page pool. 1794 * They are not tracked separately in the pcf bins, so it is 1795 * impossible to know when doing pcf accounting if the available 1796 * page(s) are noreloc pages or not. When looking for a noreloc page 1797 * it is quite easy to end up here even if the global (relocatable) 1798 * page pool has plenty of free pages but the noreloc pool is empty. 1799 * 1800 * When the noreloc pool is empty (or low), additional noreloc pages 1801 * are created by converting pages from the global page pool. This 1802 * process will stall during pcf accounting if the pcf bins are 1803 * already locked. Such is the case when a noreloc allocation is 1804 * looping here in page_create_get_something waiting for more noreloc 1805 * pages to appear. 1806 * 1807 * Short of adding a new field to the pcf bins to accurately track 1808 * the number of free noreloc pages, we instead do not grab the 1809 * pcgs_lock, do not set the pcf blocks and do not timeout when 1810 * allocating a noreloc page. This allows noreloc allocations to 1811 * loop without blocking global page pool allocations. 1812 * 1813 * NOTE: the behaviour of page_create_get_something has not changed 1814 * for the case of global page pool allocations. 1815 */ 1816 1817 flags &= ~PG_MATCH_COLOR; 1818 locked = 0; 1819 #if defined(__i386) || defined(__amd64) 1820 /* 1821 * page_create_get_something may be called because 4g memory may be 1822 * depleted. Set flags to allow for relocation of base page below 1823 * 4g if necessary. 1824 */ 1825 if (physmax4g) 1826 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI); 1827 #endif 1828 1829 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1830 1831 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1832 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1833 flags, lgrp); 1834 if (pp == NULL) { 1835 pp = page_get_cachelist(vp, off, seg, vaddr, 1836 flags, lgrp); 1837 } 1838 if (pp == NULL) { 1839 /* 1840 * Serialize. Don't fight with other pcgs(). 1841 */ 1842 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1843 mutex_enter(&pcgs_lock); 1844 VM_STAT_ADD(pcgs_locked); 1845 locked = 1; 1846 p = pcf; 1847 for (i = 0; i < PCF_FANOUT; i++) { 1848 mutex_enter(&p->pcf_lock); 1849 ASSERT(p->pcf_block == 0); 1850 p->pcf_block = 1; 1851 p->pcf_reserve = p->pcf_count; 1852 p->pcf_count = 0; 1853 mutex_exit(&p->pcf_lock); 1854 p++; 1855 } 1856 freemem = 0; 1857 } 1858 1859 if (count) { 1860 /* 1861 * Since page_free() puts pages on 1862 * a list then accounts for it, we 1863 * just have to wait for page_free() 1864 * to unlock any page it was working 1865 * with. The page_lock()-page_reclaim() 1866 * path falls in the same boat. 1867 * 1868 * We don't need to check on the 1869 * PG_WAIT flag, we have already 1870 * accounted for the page we are 1871 * looking for in page_create_va(). 1872 * 1873 * We just wait a moment to let any 1874 * locked pages on the lists free up, 1875 * then continue around and try again. 1876 * 1877 * Will be awakened by set_freemem(). 1878 */ 1879 mutex_enter(&pcgs_wait_lock); 1880 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1881 mutex_exit(&pcgs_wait_lock); 1882 } 1883 } else { 1884 #ifdef VM_STATS 1885 if (count >= PCGS_TRIES) { 1886 VM_STAT_ADD(pcgs_too_many); 1887 } else { 1888 VM_STAT_ADD(pcgs_counts[count]); 1889 } 1890 #endif 1891 if (locked) { 1892 pcgs_unblock(); 1893 mutex_exit(&pcgs_lock); 1894 } 1895 if (cagelocked) 1896 mutex_exit(&pcgs_cagelock); 1897 return (pp); 1898 } 1899 } 1900 /* 1901 * we go down holding the pcf locks. 1902 */ 1903 panic("no %spage found %d", 1904 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1905 /*NOTREACHED*/ 1906 } 1907 1908 /* 1909 * Create enough pages for "bytes" worth of data starting at 1910 * "off" in "vp". 1911 * 1912 * Where flag must be one of: 1913 * 1914 * PG_EXCL: Exclusive create (fail if any page already 1915 * exists in the page cache) which does not 1916 * wait for memory to become available. 1917 * 1918 * PG_WAIT: Non-exclusive create which can wait for 1919 * memory to become available. 1920 * 1921 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1922 * (Not Supported) 1923 * 1924 * A doubly linked list of pages is returned to the caller. Each page 1925 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1926 * lock. 1927 * 1928 * Unable to change the parameters to page_create() in a minor release, 1929 * we renamed page_create() to page_create_va(), changed all known calls 1930 * from page_create() to page_create_va(), and created this wrapper. 1931 * 1932 * Upon a major release, we should break compatibility by deleting this 1933 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1934 * 1935 * NOTE: There is a copy of this interface as page_create_io() in 1936 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1937 * there. 1938 */ 1939 page_t * 1940 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1941 { 1942 caddr_t random_vaddr; 1943 struct seg kseg; 1944 1945 #ifdef DEBUG 1946 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1947 (void *)caller()); 1948 #endif 1949 1950 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1951 (uintptr_t)(off >> PAGESHIFT)); 1952 kseg.s_as = &kas; 1953 1954 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1955 } 1956 1957 #ifdef DEBUG 1958 uint32_t pg_alloc_pgs_mtbf = 0; 1959 #endif 1960 1961 /* 1962 * Used for large page support. It will attempt to allocate 1963 * a large page(s) off the freelist. 1964 * 1965 * Returns non zero on failure. 1966 */ 1967 int 1968 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1969 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz) 1970 { 1971 pgcnt_t npgs, curnpgs, totpgs; 1972 size_t pgsz; 1973 page_t *pplist = NULL, *pp; 1974 int err = 0; 1975 lgrp_t *lgrp; 1976 1977 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 1978 1979 VM_STAT_ADD(alloc_pages[0]); 1980 1981 #ifdef DEBUG 1982 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 1983 return (ENOMEM); 1984 } 1985 #endif 1986 1987 pgsz = page_get_pagesize(szc); 1988 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 1989 1990 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 1991 /* 1992 * One must be NULL but not both. 1993 * And one must be non NULL but not both. 1994 */ 1995 ASSERT(basepp != NULL || ppa != NULL); 1996 ASSERT(basepp == NULL || ppa == NULL); 1997 1998 (void) page_create_wait(npgs, PG_WAIT); 1999 2000 while (npgs && szc) { 2001 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2002 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 0, lgrp); 2003 if (pp != NULL) { 2004 VM_STAT_ADD(alloc_pages[1]); 2005 page_list_concat(&pplist, &pp); 2006 ASSERT(npgs >= curnpgs); 2007 npgs -= curnpgs; 2008 } else if (anypgsz) { 2009 VM_STAT_ADD(alloc_pages[2]); 2010 szc--; 2011 pgsz = page_get_pagesize(szc); 2012 curnpgs = pgsz >> PAGESHIFT; 2013 } else { 2014 VM_STAT_ADD(alloc_pages[3]); 2015 ASSERT(npgs == totpgs); 2016 page_create_putback(npgs); 2017 return (ENOMEM); 2018 } 2019 } 2020 if (szc == 0) { 2021 VM_STAT_ADD(alloc_pages[4]); 2022 ASSERT(npgs != 0); 2023 page_create_putback(npgs); 2024 err = ENOMEM; 2025 } else if (basepp != NULL) { 2026 ASSERT(npgs == 0); 2027 ASSERT(ppa == NULL); 2028 *basepp = pplist; 2029 } 2030 2031 npgs = totpgs - npgs; 2032 pp = pplist; 2033 2034 /* 2035 * Clear the free and age bits. Also if we were passed in a ppa then 2036 * fill it in with all the constituent pages from the large page. But 2037 * if we failed to allocate all the pages just free what we got. 2038 */ 2039 while (npgs != 0) { 2040 ASSERT(PP_ISFREE(pp)); 2041 ASSERT(PP_ISAGED(pp)); 2042 if (ppa != NULL || err != 0) { 2043 if (err == 0) { 2044 VM_STAT_ADD(alloc_pages[5]); 2045 PP_CLRFREE(pp); 2046 PP_CLRAGED(pp); 2047 page_sub(&pplist, pp); 2048 *ppa++ = pp; 2049 npgs--; 2050 } else { 2051 VM_STAT_ADD(alloc_pages[6]); 2052 ASSERT(pp->p_szc != 0); 2053 curnpgs = page_get_pagecnt(pp->p_szc); 2054 page_list_break(&pp, &pplist, curnpgs); 2055 page_list_add_pages(pp, 0); 2056 page_create_putback(curnpgs); 2057 ASSERT(npgs >= curnpgs); 2058 npgs -= curnpgs; 2059 } 2060 pp = pplist; 2061 } else { 2062 VM_STAT_ADD(alloc_pages[7]); 2063 PP_CLRFREE(pp); 2064 PP_CLRAGED(pp); 2065 pp = pp->p_next; 2066 npgs--; 2067 } 2068 } 2069 return (err); 2070 } 2071 2072 /* 2073 * Get a single large page off of the freelists, and set it up for use. 2074 * Number of bytes requested must be a supported page size. 2075 * 2076 * Note that this call may fail even if there is sufficient 2077 * memory available or PG_WAIT is set, so the caller must 2078 * be willing to fallback on page_create_va(), block and retry, 2079 * or fail the requester. 2080 */ 2081 page_t * 2082 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2083 struct seg *seg, caddr_t vaddr, void *arg) 2084 { 2085 pgcnt_t npages, pcftotal; 2086 page_t *pp; 2087 page_t *rootpp; 2088 lgrp_t *lgrp; 2089 uint_t enough; 2090 uint_t pcf_index; 2091 uint_t i; 2092 struct pcf *p; 2093 struct pcf *q; 2094 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2095 2096 ASSERT(vp != NULL); 2097 2098 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2099 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2100 /* but no others */ 2101 2102 ASSERT((flags & PG_EXCL) == PG_EXCL); 2103 2104 npages = btop(bytes); 2105 2106 if (!kcage_on || panicstr) { 2107 /* 2108 * Cage is OFF, or we are single threaded in 2109 * panic, so make everything a RELOC request. 2110 */ 2111 flags &= ~PG_NORELOC; 2112 } 2113 2114 /* 2115 * Make sure there's adequate physical memory available. 2116 * Note: PG_WAIT is ignored here. 2117 */ 2118 if (freemem <= throttlefree + npages) { 2119 VM_STAT_ADD(page_create_large_cnt[1]); 2120 return (NULL); 2121 } 2122 2123 /* 2124 * If cage is on, dampen draw from cage when available 2125 * cage space is low. 2126 */ 2127 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2128 kcage_freemem < kcage_throttlefree + npages) { 2129 2130 /* 2131 * The cage is on, the caller wants PG_NORELOC 2132 * pages and available cage memory is very low. 2133 * Call kcage_create_throttle() to attempt to 2134 * control demand on the cage. 2135 */ 2136 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2137 VM_STAT_ADD(page_create_large_cnt[2]); 2138 return (NULL); 2139 } 2140 } 2141 2142 enough = 0; 2143 pcf_index = PCF_INDEX(); 2144 p = &pcf[pcf_index]; 2145 q = &pcf[PCF_FANOUT]; 2146 for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2147 if (p->pcf_count > npages) { 2148 /* 2149 * a good one to try. 2150 */ 2151 mutex_enter(&p->pcf_lock); 2152 if (p->pcf_count > npages) { 2153 p->pcf_count -= (uint_t)npages; 2154 /* 2155 * freemem is not protected by any lock. 2156 * Thus, we cannot have any assertion 2157 * containing freemem here. 2158 */ 2159 freemem -= npages; 2160 enough = 1; 2161 mutex_exit(&p->pcf_lock); 2162 break; 2163 } 2164 mutex_exit(&p->pcf_lock); 2165 } 2166 pcftotal += p->pcf_count; 2167 p++; 2168 if (p >= q) { 2169 p = pcf; 2170 } 2171 } 2172 2173 if (!enough) { 2174 /* If there isn't enough memory available, give up. */ 2175 if (pcftotal < npages) { 2176 VM_STAT_ADD(page_create_large_cnt[3]); 2177 return (NULL); 2178 } 2179 2180 /* try to collect pages from several pcf bins */ 2181 for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2182 mutex_enter(&p->pcf_lock); 2183 pcftotal += p->pcf_count; 2184 if (pcftotal >= npages) { 2185 /* 2186 * Wow! There are enough pages laying around 2187 * to satisfy the request. Do the accounting, 2188 * drop the locks we acquired, and go back. 2189 * 2190 * freemem is not protected by any lock. So, 2191 * we cannot have any assertion containing 2192 * freemem. 2193 */ 2194 pgcnt_t tpages = npages; 2195 freemem -= npages; 2196 while (p >= pcf) { 2197 if (p->pcf_count <= tpages) { 2198 tpages -= p->pcf_count; 2199 p->pcf_count = 0; 2200 } else { 2201 p->pcf_count -= (uint_t)tpages; 2202 tpages = 0; 2203 } 2204 mutex_exit(&p->pcf_lock); 2205 p--; 2206 } 2207 ASSERT(tpages == 0); 2208 break; 2209 } 2210 p++; 2211 } 2212 if (i == PCF_FANOUT) { 2213 /* failed to collect pages - release the locks */ 2214 while (--p >= pcf) { 2215 mutex_exit(&p->pcf_lock); 2216 } 2217 VM_STAT_ADD(page_create_large_cnt[4]); 2218 return (NULL); 2219 } 2220 } 2221 2222 /* 2223 * This is where this function behaves fundamentally differently 2224 * than page_create_va(); since we're intending to map the page 2225 * with a single TTE, we have to get it as a physically contiguous 2226 * hardware pagesize chunk. If we can't, we fail. 2227 */ 2228 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2229 LGRP_EXISTS(lgrp_table[*lgrpid])) 2230 lgrp = lgrp_table[*lgrpid]; 2231 else 2232 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2233 2234 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2235 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2236 page_create_putback(npages); 2237 VM_STAT_ADD(page_create_large_cnt[5]); 2238 return (NULL); 2239 } 2240 2241 /* 2242 * if we got the page with the wrong mtype give it back this is a 2243 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2244 * inside "if" and the workaround becomes just a nop 2245 */ 2246 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2247 page_list_add_pages(rootpp, 0); 2248 page_create_putback(npages); 2249 VM_STAT_ADD(page_create_large_cnt[6]); 2250 return (NULL); 2251 } 2252 2253 /* 2254 * If satisfying this request has left us with too little 2255 * memory, start the wheels turning to get some back. The 2256 * first clause of the test prevents waking up the pageout 2257 * daemon in situations where it would decide that there's 2258 * nothing to do. 2259 */ 2260 if (nscan < desscan && freemem < minfree) { 2261 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2262 "pageout_cv_signal:freemem %ld", freemem); 2263 cv_signal(&proc_pageout->p_cv); 2264 } 2265 2266 pp = rootpp; 2267 while (npages--) { 2268 ASSERT(PAGE_EXCL(pp)); 2269 ASSERT(pp->p_vnode == NULL); 2270 ASSERT(!hat_page_is_mapped(pp)); 2271 PP_CLRFREE(pp); 2272 PP_CLRAGED(pp); 2273 if (!page_hashin(pp, vp, off, NULL)) 2274 panic("page_create_large: hashin failed: page %p", 2275 (void *)pp); 2276 page_io_lock(pp); 2277 off += PAGESIZE; 2278 pp = pp->p_next; 2279 } 2280 2281 VM_STAT_ADD(page_create_large_cnt[0]); 2282 return (rootpp); 2283 } 2284 2285 page_t * 2286 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2287 struct seg *seg, caddr_t vaddr) 2288 { 2289 page_t *plist = NULL; 2290 pgcnt_t npages; 2291 pgcnt_t found_on_free = 0; 2292 pgcnt_t pages_req; 2293 page_t *npp = NULL; 2294 uint_t enough; 2295 uint_t i; 2296 uint_t pcf_index; 2297 struct pcf *p; 2298 struct pcf *q; 2299 lgrp_t *lgrp; 2300 2301 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2302 "page_create_start:vp %p off %llx bytes %lu flags %x", 2303 vp, off, bytes, flags); 2304 2305 ASSERT(bytes != 0 && vp != NULL); 2306 2307 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2308 panic("page_create: invalid flags"); 2309 /*NOTREACHED*/ 2310 } 2311 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2312 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2313 /* but no others */ 2314 2315 pages_req = npages = btopr(bytes); 2316 /* 2317 * Try to see whether request is too large to *ever* be 2318 * satisfied, in order to prevent deadlock. We arbitrarily 2319 * decide to limit maximum size requests to max_page_get. 2320 */ 2321 if (npages >= max_page_get) { 2322 if ((flags & PG_WAIT) == 0) { 2323 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2324 "page_create_toobig:vp %p off %llx npages " 2325 "%lu max_page_get %lu", 2326 vp, off, npages, max_page_get); 2327 return (NULL); 2328 } else { 2329 cmn_err(CE_WARN, 2330 "Request for too much kernel memory " 2331 "(%lu bytes), will hang forever", bytes); 2332 for (;;) 2333 delay(1000000000); 2334 } 2335 } 2336 2337 if (!kcage_on || panicstr) { 2338 /* 2339 * Cage is OFF, or we are single threaded in 2340 * panic, so make everything a RELOC request. 2341 */ 2342 flags &= ~PG_NORELOC; 2343 } 2344 2345 if (freemem <= throttlefree + npages) 2346 if (!page_create_throttle(npages, flags)) 2347 return (NULL); 2348 2349 /* 2350 * If cage is on, dampen draw from cage when available 2351 * cage space is low. 2352 */ 2353 if ((flags & PG_NORELOC) && 2354 kcage_freemem < kcage_throttlefree + npages) { 2355 2356 /* 2357 * The cage is on, the caller wants PG_NORELOC 2358 * pages and available cage memory is very low. 2359 * Call kcage_create_throttle() to attempt to 2360 * control demand on the cage. 2361 */ 2362 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2363 return (NULL); 2364 } 2365 2366 VM_STAT_ADD(page_create_cnt[0]); 2367 2368 enough = 0; 2369 pcf_index = PCF_INDEX(); 2370 2371 p = &pcf[pcf_index]; 2372 q = &pcf[PCF_FANOUT]; 2373 for (i = 0; i < PCF_FANOUT; i++) { 2374 if (p->pcf_count > npages) { 2375 /* 2376 * a good one to try. 2377 */ 2378 mutex_enter(&p->pcf_lock); 2379 if (p->pcf_count > npages) { 2380 p->pcf_count -= (uint_t)npages; 2381 /* 2382 * freemem is not protected by any lock. 2383 * Thus, we cannot have any assertion 2384 * containing freemem here. 2385 */ 2386 freemem -= npages; 2387 enough = 1; 2388 mutex_exit(&p->pcf_lock); 2389 break; 2390 } 2391 mutex_exit(&p->pcf_lock); 2392 } 2393 p++; 2394 if (p >= q) { 2395 p = pcf; 2396 } 2397 } 2398 2399 if (!enough) { 2400 /* 2401 * Have to look harder. If npages is greater than 2402 * one, then we might have to coalecse the counters. 2403 * 2404 * Go wait. We come back having accounted 2405 * for the memory. 2406 */ 2407 VM_STAT_ADD(page_create_cnt[1]); 2408 if (!page_create_wait(npages, flags)) { 2409 VM_STAT_ADD(page_create_cnt[2]); 2410 return (NULL); 2411 } 2412 } 2413 2414 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2415 "page_create_success:vp %p off %llx", vp, off); 2416 2417 /* 2418 * If satisfying this request has left us with too little 2419 * memory, start the wheels turning to get some back. The 2420 * first clause of the test prevents waking up the pageout 2421 * daemon in situations where it would decide that there's 2422 * nothing to do. 2423 */ 2424 if (nscan < desscan && freemem < minfree) { 2425 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2426 "pageout_cv_signal:freemem %ld", freemem); 2427 cv_signal(&proc_pageout->p_cv); 2428 } 2429 2430 /* 2431 * Loop around collecting the requested number of pages. 2432 * Most of the time, we have to `create' a new page. With 2433 * this in mind, pull the page off the free list before 2434 * getting the hash lock. This will minimize the hash 2435 * lock hold time, nesting, and the like. If it turns 2436 * out we don't need the page, we put it back at the end. 2437 */ 2438 while (npages--) { 2439 page_t *pp; 2440 kmutex_t *phm = NULL; 2441 ulong_t index; 2442 2443 index = PAGE_HASH_FUNC(vp, off); 2444 top: 2445 ASSERT(phm == NULL); 2446 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2447 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2448 2449 if (npp == NULL) { 2450 /* 2451 * Try to get a page from the freelist (ie, 2452 * a page with no [vp, off] tag). If that 2453 * fails, use the cachelist. 2454 * 2455 * During the first attempt at both the free 2456 * and cache lists we try for the correct color. 2457 */ 2458 /* 2459 * XXXX-how do we deal with virtual indexed 2460 * caches and and colors? 2461 */ 2462 VM_STAT_ADD(page_create_cnt[4]); 2463 /* 2464 * Get lgroup to allocate next page of shared memory 2465 * from and use it to specify where to allocate 2466 * the physical memory 2467 */ 2468 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2469 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2470 flags | PG_MATCH_COLOR, lgrp); 2471 if (npp == NULL) { 2472 npp = page_get_cachelist(vp, off, seg, 2473 vaddr, flags | PG_MATCH_COLOR, lgrp); 2474 if (npp == NULL) { 2475 npp = page_create_get_something(vp, 2476 off, seg, vaddr, 2477 flags & ~PG_MATCH_COLOR); 2478 } 2479 2480 if (PP_ISAGED(npp) == 0) { 2481 /* 2482 * Since this page came from the 2483 * cachelist, we must destroy the 2484 * old vnode association. 2485 */ 2486 page_hashout(npp, NULL); 2487 } 2488 } 2489 } 2490 2491 /* 2492 * We own this page! 2493 */ 2494 ASSERT(PAGE_EXCL(npp)); 2495 ASSERT(npp->p_vnode == NULL); 2496 ASSERT(!hat_page_is_mapped(npp)); 2497 PP_CLRFREE(npp); 2498 PP_CLRAGED(npp); 2499 2500 /* 2501 * Here we have a page in our hot little mits and are 2502 * just waiting to stuff it on the appropriate lists. 2503 * Get the mutex and check to see if it really does 2504 * not exist. 2505 */ 2506 phm = PAGE_HASH_MUTEX(index); 2507 mutex_enter(phm); 2508 PAGE_HASH_SEARCH(index, pp, vp, off); 2509 if (pp == NULL) { 2510 VM_STAT_ADD(page_create_new); 2511 pp = npp; 2512 npp = NULL; 2513 if (!page_hashin(pp, vp, off, phm)) { 2514 /* 2515 * Since we hold the page hash mutex and 2516 * just searched for this page, page_hashin 2517 * had better not fail. If it does, that 2518 * means somethread did not follow the 2519 * page hash mutex rules. Panic now and 2520 * get it over with. As usual, go down 2521 * holding all the locks. 2522 */ 2523 ASSERT(MUTEX_HELD(phm)); 2524 panic("page_create: " 2525 "hashin failed %p %p %llx %p", 2526 (void *)pp, (void *)vp, off, (void *)phm); 2527 /*NOTREACHED*/ 2528 } 2529 ASSERT(MUTEX_HELD(phm)); 2530 mutex_exit(phm); 2531 phm = NULL; 2532 2533 /* 2534 * Hat layer locking need not be done to set 2535 * the following bits since the page is not hashed 2536 * and was on the free list (i.e., had no mappings). 2537 * 2538 * Set the reference bit to protect 2539 * against immediate pageout 2540 * 2541 * XXXmh modify freelist code to set reference 2542 * bit so we don't have to do it here. 2543 */ 2544 page_set_props(pp, P_REF); 2545 found_on_free++; 2546 } else { 2547 VM_STAT_ADD(page_create_exists); 2548 if (flags & PG_EXCL) { 2549 /* 2550 * Found an existing page, and the caller 2551 * wanted all new pages. Undo all of the work 2552 * we have done. 2553 */ 2554 mutex_exit(phm); 2555 phm = NULL; 2556 while (plist != NULL) { 2557 pp = plist; 2558 page_sub(&plist, pp); 2559 page_io_unlock(pp); 2560 /* large pages should not end up here */ 2561 ASSERT(pp->p_szc == 0); 2562 /*LINTED: constant in conditional ctx*/ 2563 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2564 } 2565 VM_STAT_ADD(page_create_found_one); 2566 goto fail; 2567 } 2568 ASSERT(flags & PG_WAIT); 2569 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2570 /* 2571 * Start all over again if we blocked trying 2572 * to lock the page. 2573 */ 2574 mutex_exit(phm); 2575 VM_STAT_ADD(page_create_page_lock_failed); 2576 phm = NULL; 2577 goto top; 2578 } 2579 mutex_exit(phm); 2580 phm = NULL; 2581 2582 if (PP_ISFREE(pp)) { 2583 ASSERT(PP_ISAGED(pp) == 0); 2584 VM_STAT_ADD(pagecnt.pc_get_cache); 2585 page_list_sub(pp, PG_CACHE_LIST); 2586 PP_CLRFREE(pp); 2587 found_on_free++; 2588 } 2589 } 2590 2591 /* 2592 * Got a page! It is locked. Acquire the i/o 2593 * lock since we are going to use the p_next and 2594 * p_prev fields to link the requested pages together. 2595 */ 2596 page_io_lock(pp); 2597 page_add(&plist, pp); 2598 plist = plist->p_next; 2599 off += PAGESIZE; 2600 vaddr += PAGESIZE; 2601 } 2602 2603 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2604 fail: 2605 if (npp != NULL) { 2606 /* 2607 * Did not need this page after all. 2608 * Put it back on the free list. 2609 */ 2610 VM_STAT_ADD(page_create_putbacks); 2611 PP_SETFREE(npp); 2612 PP_SETAGED(npp); 2613 npp->p_offset = (u_offset_t)-1; 2614 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2615 page_unlock(npp); 2616 2617 } 2618 2619 ASSERT(pages_req >= found_on_free); 2620 2621 { 2622 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2623 2624 if (overshoot) { 2625 VM_STAT_ADD(page_create_overshoot); 2626 p = &pcf[pcf_index]; 2627 mutex_enter(&p->pcf_lock); 2628 if (p->pcf_block) { 2629 p->pcf_reserve += overshoot; 2630 } else { 2631 p->pcf_count += overshoot; 2632 if (p->pcf_wait) { 2633 mutex_enter(&new_freemem_lock); 2634 if (freemem_wait) { 2635 cv_signal(&freemem_cv); 2636 p->pcf_wait--; 2637 } else { 2638 p->pcf_wait = 0; 2639 } 2640 mutex_exit(&new_freemem_lock); 2641 } 2642 } 2643 mutex_exit(&p->pcf_lock); 2644 /* freemem is approximate, so this test OK */ 2645 if (!p->pcf_block) 2646 freemem += overshoot; 2647 } 2648 } 2649 2650 return (plist); 2651 } 2652 2653 /* 2654 * One or more constituent pages of this large page has been marked 2655 * toxic. Simply demote the large page to PAGESIZE pages and let 2656 * page_free() handle it. This routine should only be called by 2657 * large page free routines (page_free_pages() and page_destroy_pages(). 2658 * All pages are locked SE_EXCL and have already been marked free. 2659 */ 2660 static void 2661 page_free_toxic_pages(page_t *rootpp) 2662 { 2663 page_t *tpp; 2664 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2665 uint_t szc = rootpp->p_szc; 2666 2667 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2668 ASSERT(tpp->p_szc == szc); 2669 ASSERT((PAGE_EXCL(tpp) && 2670 !page_iolock_assert(tpp)) || panicstr); 2671 tpp->p_szc = 0; 2672 } 2673 2674 while (rootpp != NULL) { 2675 tpp = rootpp; 2676 page_sub(&rootpp, tpp); 2677 ASSERT(PP_ISFREE(tpp)); 2678 PP_CLRFREE(tpp); 2679 page_free(tpp, 1); 2680 } 2681 } 2682 2683 /* 2684 * Put page on the "free" list. 2685 * The free list is really two lists maintained by 2686 * the PSM of whatever machine we happen to be on. 2687 */ 2688 void 2689 page_free(page_t *pp, int dontneed) 2690 { 2691 struct pcf *p; 2692 uint_t pcf_index; 2693 2694 ASSERT((PAGE_EXCL(pp) && 2695 !page_iolock_assert(pp)) || panicstr); 2696 2697 if (PP_ISFREE(pp)) { 2698 panic("page_free: page %p is free", (void *)pp); 2699 } 2700 2701 if (pp->p_szc != 0) { 2702 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2703 pp->p_vnode == &kvp) { 2704 panic("page_free: anon or kernel " 2705 "or no vnode large page %p", (void *)pp); 2706 } 2707 page_demote_vp_pages(pp); 2708 ASSERT(pp->p_szc == 0); 2709 } 2710 2711 /* 2712 * The page_struct_lock need not be acquired to examine these 2713 * fields since the page has an "exclusive" lock. 2714 */ 2715 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 2716 pp->p_slckcnt != 0) { 2717 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 2718 "slckcnt = %d", pp, page_pptonum(pp), pp->p_lckcnt, 2719 pp->p_cowcnt, pp->p_slckcnt); 2720 /*NOTREACHED*/ 2721 } 2722 2723 ASSERT(!hat_page_getshare(pp)); 2724 2725 PP_SETFREE(pp); 2726 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2727 !hat_ismod(pp)); 2728 page_clr_all_props(pp); 2729 ASSERT(!hat_page_getshare(pp)); 2730 2731 /* 2732 * Now we add the page to the head of the free list. 2733 * But if this page is associated with a paged vnode 2734 * then we adjust the head forward so that the page is 2735 * effectively at the end of the list. 2736 */ 2737 if (pp->p_vnode == NULL) { 2738 /* 2739 * Page has no identity, put it on the free list. 2740 */ 2741 PP_SETAGED(pp); 2742 pp->p_offset = (u_offset_t)-1; 2743 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2744 VM_STAT_ADD(pagecnt.pc_free_free); 2745 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2746 "page_free_free:pp %p", pp); 2747 } else { 2748 PP_CLRAGED(pp); 2749 2750 if (!dontneed || nopageage) { 2751 /* move it to the tail of the list */ 2752 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2753 2754 VM_STAT_ADD(pagecnt.pc_free_cache); 2755 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2756 "page_free_cache_tail:pp %p", pp); 2757 } else { 2758 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2759 2760 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2761 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2762 "page_free_cache_head:pp %p", pp); 2763 } 2764 } 2765 page_unlock(pp); 2766 2767 /* 2768 * Now do the `freemem' accounting. 2769 */ 2770 pcf_index = PCF_INDEX(); 2771 p = &pcf[pcf_index]; 2772 2773 mutex_enter(&p->pcf_lock); 2774 if (p->pcf_block) { 2775 p->pcf_reserve += 1; 2776 } else { 2777 p->pcf_count += 1; 2778 if (p->pcf_wait) { 2779 mutex_enter(&new_freemem_lock); 2780 /* 2781 * Check to see if some other thread 2782 * is actually waiting. Another bucket 2783 * may have woken it up by now. If there 2784 * are no waiters, then set our pcf_wait 2785 * count to zero to avoid coming in here 2786 * next time. Also, since only one page 2787 * was put on the free list, just wake 2788 * up one waiter. 2789 */ 2790 if (freemem_wait) { 2791 cv_signal(&freemem_cv); 2792 p->pcf_wait--; 2793 } else { 2794 p->pcf_wait = 0; 2795 } 2796 mutex_exit(&new_freemem_lock); 2797 } 2798 } 2799 mutex_exit(&p->pcf_lock); 2800 2801 /* freemem is approximate, so this test OK */ 2802 if (!p->pcf_block) 2803 freemem += 1; 2804 } 2805 2806 /* 2807 * Put page on the "free" list during intial startup. 2808 * This happens during initial single threaded execution. 2809 */ 2810 void 2811 page_free_at_startup(page_t *pp) 2812 { 2813 struct pcf *p; 2814 uint_t pcf_index; 2815 2816 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2817 VM_STAT_ADD(pagecnt.pc_free_free); 2818 2819 /* 2820 * Now do the `freemem' accounting. 2821 */ 2822 pcf_index = PCF_INDEX(); 2823 p = &pcf[pcf_index]; 2824 2825 ASSERT(p->pcf_block == 0); 2826 ASSERT(p->pcf_wait == 0); 2827 p->pcf_count += 1; 2828 2829 /* freemem is approximate, so this is OK */ 2830 freemem += 1; 2831 } 2832 2833 void 2834 page_free_pages(page_t *pp) 2835 { 2836 page_t *tpp, *rootpp = NULL; 2837 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2838 pgcnt_t i; 2839 uint_t szc = pp->p_szc; 2840 2841 VM_STAT_ADD(pagecnt.pc_free_pages); 2842 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2843 "page_free_free:pp %p", pp); 2844 2845 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2846 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2847 panic("page_free_pages: not root page %p", (void *)pp); 2848 /*NOTREACHED*/ 2849 } 2850 2851 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2852 ASSERT((PAGE_EXCL(tpp) && 2853 !page_iolock_assert(tpp)) || panicstr); 2854 if (PP_ISFREE(tpp)) { 2855 panic("page_free_pages: page %p is free", (void *)tpp); 2856 /*NOTREACHED*/ 2857 } 2858 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2859 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 2860 panic("page_free_pages %p", (void *)tpp); 2861 /*NOTREACHED*/ 2862 } 2863 2864 ASSERT(!hat_page_getshare(tpp)); 2865 ASSERT(tpp->p_vnode == NULL); 2866 ASSERT(tpp->p_szc == szc); 2867 2868 PP_SETFREE(tpp); 2869 page_clr_all_props(tpp); 2870 PP_SETAGED(tpp); 2871 tpp->p_offset = (u_offset_t)-1; 2872 ASSERT(tpp->p_next == tpp); 2873 ASSERT(tpp->p_prev == tpp); 2874 page_list_concat(&rootpp, &tpp); 2875 } 2876 ASSERT(rootpp == pp); 2877 2878 page_list_add_pages(rootpp, 0); 2879 page_create_putback(pgcnt); 2880 } 2881 2882 int free_pages = 1; 2883 2884 /* 2885 * This routine attempts to return pages to the cachelist via page_release(). 2886 * It does not *have* to be successful in all cases, since the pageout scanner 2887 * will catch any pages it misses. It does need to be fast and not introduce 2888 * too much overhead. 2889 * 2890 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2891 * don't lock and retry. This is ok, since the page scanner will eventually 2892 * find any page we miss in free_vp_pages(). 2893 */ 2894 void 2895 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2896 { 2897 page_t *pp; 2898 u_offset_t eoff; 2899 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2900 2901 eoff = off + len; 2902 2903 if (free_pages == 0) 2904 return; 2905 if (swap_in_range(vp, off, len)) 2906 return; 2907 2908 for (; off < eoff; off += PAGESIZE) { 2909 2910 /* 2911 * find the page using a fast, but inexact search. It'll be OK 2912 * if a few pages slip through the cracks here. 2913 */ 2914 pp = page_exists(vp, off); 2915 2916 /* 2917 * If we didn't find the page (it may not exist), the page 2918 * is free, looks still in use (shared), or we can't lock it, 2919 * just give up. 2920 */ 2921 if (pp == NULL || 2922 PP_ISFREE(pp) || 2923 page_share_cnt(pp) > 0 || 2924 !page_trylock(pp, SE_EXCL)) 2925 continue; 2926 2927 /* 2928 * Once we have locked pp, verify that it's still the 2929 * correct page and not already free 2930 */ 2931 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2932 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2933 page_unlock(pp); 2934 continue; 2935 } 2936 2937 /* 2938 * try to release the page... 2939 */ 2940 (void) page_release(pp, 1); 2941 } 2942 } 2943 2944 /* 2945 * Reclaim the given page from the free list. 2946 * Returns 1 on success or 0 on failure. 2947 * 2948 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2949 * If `lock' is non-null, it will be dropped and re-acquired if 2950 * the routine must wait while freemem is 0. 2951 * 2952 * As it turns out, boot_getpages() does this. It picks a page, 2953 * based on where OBP mapped in some address, gets its pfn, searches 2954 * the memsegs, locks the page, then pulls it off the free list! 2955 */ 2956 int 2957 page_reclaim(page_t *pp, kmutex_t *lock) 2958 { 2959 struct pcf *p; 2960 uint_t pcf_index; 2961 struct cpu *cpup; 2962 uint_t i; 2963 pgcnt_t npgs, need; 2964 pgcnt_t collected = 0; 2965 2966 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2967 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2968 2969 npgs = page_get_pagecnt(pp->p_szc); 2970 2971 /* 2972 * If `freemem' is 0, we cannot reclaim this page from the 2973 * freelist, so release every lock we might hold: the page, 2974 * and the `lock' before blocking. 2975 * 2976 * The only way `freemem' can become 0 while there are pages 2977 * marked free (have their p->p_free bit set) is when the 2978 * system is low on memory and doing a page_create(). In 2979 * order to guarantee that once page_create() starts acquiring 2980 * pages it will be able to get all that it needs since `freemem' 2981 * was decreased by the requested amount. So, we need to release 2982 * this page, and let page_create() have it. 2983 * 2984 * Since `freemem' being zero is not supposed to happen, just 2985 * use the usual hash stuff as a starting point. If that bucket 2986 * is empty, then assume the worst, and start at the beginning 2987 * of the pcf array. If we always start at the beginning 2988 * when acquiring more than one pcf lock, there won't be any 2989 * deadlock problems. 2990 */ 2991 2992 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 2993 2994 if (freemem <= throttlefree && !page_create_throttle(npgs, 0)) { 2995 pcf_acquire_all(); 2996 goto page_reclaim_nomem; 2997 } 2998 2999 pcf_index = PCF_INDEX(); 3000 p = &pcf[pcf_index]; 3001 mutex_enter(&p->pcf_lock); 3002 if (p->pcf_count >= npgs) { 3003 collected = npgs; 3004 p->pcf_count -= npgs; 3005 } 3006 mutex_exit(&p->pcf_lock); 3007 need = npgs - collected; 3008 3009 if (need > 0) { 3010 VM_STAT_ADD(page_reclaim_zero); 3011 /* 3012 * Check again. Its possible that some other thread 3013 * could have been right behind us, and added one 3014 * to a list somewhere. Acquire each of the pcf locks 3015 * until we find a page. 3016 */ 3017 p = pcf; 3018 for (i = 0; i < PCF_FANOUT; i++) { 3019 mutex_enter(&p->pcf_lock); 3020 if (p->pcf_count) { 3021 if (p->pcf_count >= need) { 3022 p->pcf_count -= need; 3023 collected += need; 3024 need = 0; 3025 break; 3026 } else if (p->pcf_count) { 3027 collected += p->pcf_count; 3028 need -= p->pcf_count; 3029 p->pcf_count = 0; 3030 } 3031 } 3032 p++; 3033 } 3034 3035 if (need > 0) { 3036 page_reclaim_nomem: 3037 /* 3038 * We really can't have page `pp'. 3039 * Time for the no-memory dance with 3040 * page_free(). This is just like 3041 * page_create_wait(). Plus the added 3042 * attraction of releasing whatever mutex 3043 * we held when we were called with in `lock'. 3044 * Page_unlock() will wakeup any thread 3045 * waiting around for this page. 3046 */ 3047 if (lock) { 3048 VM_STAT_ADD(page_reclaim_zero_locked); 3049 mutex_exit(lock); 3050 } 3051 page_unlock(pp); 3052 3053 /* 3054 * get this before we drop all the pcf locks. 3055 */ 3056 mutex_enter(&new_freemem_lock); 3057 3058 p = pcf; 3059 p->pcf_count += collected; 3060 for (i = 0; i < PCF_FANOUT; i++) { 3061 p->pcf_wait++; 3062 mutex_exit(&p->pcf_lock); 3063 p++; 3064 } 3065 3066 freemem_wait++; 3067 cv_wait(&freemem_cv, &new_freemem_lock); 3068 freemem_wait--; 3069 3070 mutex_exit(&new_freemem_lock); 3071 3072 if (lock) { 3073 mutex_enter(lock); 3074 } 3075 return (0); 3076 } 3077 3078 /* 3079 * We beat the PCF bins over the head until 3080 * we got the memory that we wanted. 3081 * The pcf accounting has been done, 3082 * though none of the pcf_wait flags have been set, 3083 * drop the locks and continue on. 3084 */ 3085 ASSERT(collected == npgs); 3086 while (p >= pcf) { 3087 mutex_exit(&p->pcf_lock); 3088 p--; 3089 } 3090 } 3091 3092 /* 3093 * freemem is not protected by any lock. Thus, we cannot 3094 * have any assertion containing freemem here. 3095 */ 3096 freemem -= npgs; 3097 3098 VM_STAT_ADD(pagecnt.pc_reclaim); 3099 if (PP_ISAGED(pp)) { 3100 if (npgs > 1) { 3101 page_list_sub_pages(pp, pp->p_szc); 3102 } else { 3103 page_list_sub(pp, PG_FREE_LIST); 3104 } 3105 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3106 "page_reclaim_free:pp %p", pp); 3107 } else { 3108 ASSERT(npgs == 1); 3109 page_list_sub(pp, PG_CACHE_LIST); 3110 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3111 "page_reclaim_cache:pp %p", pp); 3112 } 3113 3114 /* 3115 * clear the p_free & p_age bits since this page is no longer 3116 * on the free list. Notice that there was a brief time where 3117 * a page is marked as free, but is not on the list. 3118 * 3119 * Set the reference bit to protect against immediate pageout. 3120 */ 3121 for (i = 0; i < npgs; i++, pp++) { 3122 PP_CLRFREE(pp); 3123 PP_CLRAGED(pp); 3124 page_set_props(pp, P_REF); 3125 } 3126 3127 CPU_STATS_ENTER_K(); 3128 cpup = CPU; /* get cpup now that CPU cannot change */ 3129 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3130 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3131 CPU_STATS_EXIT_K(); 3132 3133 return (1); 3134 } 3135 3136 3137 3138 /* 3139 * Destroy identity of the page and put it back on 3140 * the page free list. Assumes that the caller has 3141 * acquired the "exclusive" lock on the page. 3142 */ 3143 void 3144 page_destroy(page_t *pp, int dontfree) 3145 { 3146 ASSERT((PAGE_EXCL(pp) && 3147 !page_iolock_assert(pp)) || panicstr); 3148 ASSERT(pp->p_slckcnt == 0 || panicstr); 3149 3150 if (pp->p_szc != 0) { 3151 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3152 pp->p_vnode == &kvp) { 3153 panic("page_destroy: anon or kernel or no vnode " 3154 "large page %p", (void *)pp); 3155 } 3156 page_demote_vp_pages(pp); 3157 ASSERT(pp->p_szc == 0); 3158 } 3159 3160 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3161 3162 /* 3163 * Unload translations, if any, then hash out the 3164 * page to erase its identity. 3165 */ 3166 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3167 page_hashout(pp, NULL); 3168 3169 if (!dontfree) { 3170 /* 3171 * Acquire the "freemem_lock" for availrmem. 3172 * The page_struct_lock need not be acquired for lckcnt 3173 * and cowcnt since the page has an "exclusive" lock. 3174 */ 3175 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3176 mutex_enter(&freemem_lock); 3177 if (pp->p_lckcnt != 0) { 3178 availrmem++; 3179 pp->p_lckcnt = 0; 3180 } 3181 if (pp->p_cowcnt != 0) { 3182 availrmem += pp->p_cowcnt; 3183 pp->p_cowcnt = 0; 3184 } 3185 mutex_exit(&freemem_lock); 3186 } 3187 /* 3188 * Put the page on the "free" list. 3189 */ 3190 page_free(pp, 0); 3191 } 3192 } 3193 3194 void 3195 page_destroy_pages(page_t *pp) 3196 { 3197 3198 page_t *tpp, *rootpp = NULL; 3199 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3200 pgcnt_t i, pglcks = 0; 3201 uint_t szc = pp->p_szc; 3202 3203 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3204 3205 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3206 3207 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3208 3209 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3210 panic("page_destroy_pages: not root page %p", (void *)pp); 3211 /*NOTREACHED*/ 3212 } 3213 3214 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3215 ASSERT((PAGE_EXCL(tpp) && 3216 !page_iolock_assert(tpp)) || panicstr); 3217 ASSERT(tpp->p_slckcnt == 0 || panicstr); 3218 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3219 page_hashout(tpp, NULL); 3220 ASSERT(tpp->p_offset == (u_offset_t)-1); 3221 if (tpp->p_lckcnt != 0) { 3222 pglcks++; 3223 tpp->p_lckcnt = 0; 3224 } else if (tpp->p_cowcnt != 0) { 3225 pglcks += tpp->p_cowcnt; 3226 tpp->p_cowcnt = 0; 3227 } 3228 ASSERT(!hat_page_getshare(tpp)); 3229 ASSERT(tpp->p_vnode == NULL); 3230 ASSERT(tpp->p_szc == szc); 3231 3232 PP_SETFREE(tpp); 3233 page_clr_all_props(tpp); 3234 PP_SETAGED(tpp); 3235 ASSERT(tpp->p_next == tpp); 3236 ASSERT(tpp->p_prev == tpp); 3237 page_list_concat(&rootpp, &tpp); 3238 } 3239 3240 ASSERT(rootpp == pp); 3241 if (pglcks != 0) { 3242 mutex_enter(&freemem_lock); 3243 availrmem += pglcks; 3244 mutex_exit(&freemem_lock); 3245 } 3246 3247 page_list_add_pages(rootpp, 0); 3248 page_create_putback(pgcnt); 3249 } 3250 3251 /* 3252 * Similar to page_destroy(), but destroys pages which are 3253 * locked and known to be on the page free list. Since 3254 * the page is known to be free and locked, no one can access 3255 * it. 3256 * 3257 * Also, the number of free pages does not change. 3258 */ 3259 void 3260 page_destroy_free(page_t *pp) 3261 { 3262 ASSERT(PAGE_EXCL(pp)); 3263 ASSERT(PP_ISFREE(pp)); 3264 ASSERT(pp->p_vnode); 3265 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3266 ASSERT(!hat_page_is_mapped(pp)); 3267 ASSERT(PP_ISAGED(pp) == 0); 3268 ASSERT(pp->p_szc == 0); 3269 3270 VM_STAT_ADD(pagecnt.pc_destroy_free); 3271 page_list_sub(pp, PG_CACHE_LIST); 3272 3273 page_hashout(pp, NULL); 3274 ASSERT(pp->p_vnode == NULL); 3275 ASSERT(pp->p_offset == (u_offset_t)-1); 3276 ASSERT(pp->p_hash == NULL); 3277 3278 PP_SETAGED(pp); 3279 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3280 page_unlock(pp); 3281 3282 mutex_enter(&new_freemem_lock); 3283 if (freemem_wait) { 3284 cv_signal(&freemem_cv); 3285 } 3286 mutex_exit(&new_freemem_lock); 3287 } 3288 3289 /* 3290 * Rename the page "opp" to have an identity specified 3291 * by [vp, off]. If a page already exists with this name 3292 * it is locked and destroyed. Note that the page's 3293 * translations are not unloaded during the rename. 3294 * 3295 * This routine is used by the anon layer to "steal" the 3296 * original page and is not unlike destroying a page and 3297 * creating a new page using the same page frame. 3298 * 3299 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3300 * caller 2 tries to rename B to A. 3301 */ 3302 void 3303 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3304 { 3305 page_t *pp; 3306 int olckcnt = 0; 3307 int ocowcnt = 0; 3308 kmutex_t *phm; 3309 ulong_t index; 3310 3311 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3312 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3313 ASSERT(PP_ISFREE(opp) == 0); 3314 3315 VM_STAT_ADD(page_rename_count); 3316 3317 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3318 "page rename:pp %p vp %p off %llx", opp, vp, off); 3319 3320 /* 3321 * CacheFS may call page_rename for a large NFS page 3322 * when both CacheFS and NFS mount points are used 3323 * by applications. Demote this large page before 3324 * renaming it, to ensure that there are no "partial" 3325 * large pages left lying around. 3326 */ 3327 if (opp->p_szc != 0) { 3328 vnode_t *ovp = opp->p_vnode; 3329 ASSERT(ovp != NULL); 3330 ASSERT(!IS_SWAPFSVP(ovp)); 3331 ASSERT(ovp != &kvp); 3332 page_demote_vp_pages(opp); 3333 ASSERT(opp->p_szc == 0); 3334 } 3335 3336 page_hashout(opp, NULL); 3337 PP_CLRAGED(opp); 3338 3339 /* 3340 * Acquire the appropriate page hash lock, since 3341 * we're going to rename the page. 3342 */ 3343 index = PAGE_HASH_FUNC(vp, off); 3344 phm = PAGE_HASH_MUTEX(index); 3345 mutex_enter(phm); 3346 top: 3347 /* 3348 * Look for an existing page with this name and destroy it if found. 3349 * By holding the page hash lock all the way to the page_hashin() 3350 * call, we are assured that no page can be created with this 3351 * identity. In the case when the phm lock is dropped to undo any 3352 * hat layer mappings, the existing page is held with an "exclusive" 3353 * lock, again preventing another page from being created with 3354 * this identity. 3355 */ 3356 PAGE_HASH_SEARCH(index, pp, vp, off); 3357 if (pp != NULL) { 3358 VM_STAT_ADD(page_rename_exists); 3359 3360 /* 3361 * As it turns out, this is one of only two places where 3362 * page_lock() needs to hold the passed in lock in the 3363 * successful case. In all of the others, the lock could 3364 * be dropped as soon as the attempt is made to lock 3365 * the page. It is tempting to add yet another arguement, 3366 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3367 */ 3368 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3369 /* 3370 * Went to sleep because the page could not 3371 * be locked. We were woken up when the page 3372 * was unlocked, or when the page was destroyed. 3373 * In either case, `phm' was dropped while we 3374 * slept. Hence we should not just roar through 3375 * this loop. 3376 */ 3377 goto top; 3378 } 3379 3380 /* 3381 * If an existing page is a large page, then demote 3382 * it to ensure that no "partial" large pages are 3383 * "created" after page_rename. An existing page 3384 * can be a CacheFS page, and can't belong to swapfs. 3385 */ 3386 if (hat_page_is_mapped(pp)) { 3387 /* 3388 * Unload translations. Since we hold the 3389 * exclusive lock on this page, the page 3390 * can not be changed while we drop phm. 3391 * This is also not a lock protocol violation, 3392 * but rather the proper way to do things. 3393 */ 3394 mutex_exit(phm); 3395 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3396 if (pp->p_szc != 0) { 3397 ASSERT(!IS_SWAPFSVP(vp)); 3398 ASSERT(vp != &kvp); 3399 page_demote_vp_pages(pp); 3400 ASSERT(pp->p_szc == 0); 3401 } 3402 mutex_enter(phm); 3403 } else if (pp->p_szc != 0) { 3404 ASSERT(!IS_SWAPFSVP(vp)); 3405 ASSERT(vp != &kvp); 3406 mutex_exit(phm); 3407 page_demote_vp_pages(pp); 3408 ASSERT(pp->p_szc == 0); 3409 mutex_enter(phm); 3410 } 3411 page_hashout(pp, phm); 3412 } 3413 /* 3414 * Hash in the page with the new identity. 3415 */ 3416 if (!page_hashin(opp, vp, off, phm)) { 3417 /* 3418 * We were holding phm while we searched for [vp, off] 3419 * and only dropped phm if we found and locked a page. 3420 * If we can't create this page now, then some thing 3421 * is really broken. 3422 */ 3423 panic("page_rename: Can't hash in page: %p", (void *)pp); 3424 /*NOTREACHED*/ 3425 } 3426 3427 ASSERT(MUTEX_HELD(phm)); 3428 mutex_exit(phm); 3429 3430 /* 3431 * Now that we have dropped phm, lets get around to finishing up 3432 * with pp. 3433 */ 3434 if (pp != NULL) { 3435 ASSERT(!hat_page_is_mapped(pp)); 3436 /* for now large pages should not end up here */ 3437 ASSERT(pp->p_szc == 0); 3438 /* 3439 * Save the locks for transfer to the new page and then 3440 * clear them so page_free doesn't think they're important. 3441 * The page_struct_lock need not be acquired for lckcnt and 3442 * cowcnt since the page has an "exclusive" lock. 3443 */ 3444 olckcnt = pp->p_lckcnt; 3445 ocowcnt = pp->p_cowcnt; 3446 pp->p_lckcnt = pp->p_cowcnt = 0; 3447 3448 /* 3449 * Put the page on the "free" list after we drop 3450 * the lock. The less work under the lock the better. 3451 */ 3452 /*LINTED: constant in conditional context*/ 3453 VN_DISPOSE(pp, B_FREE, 0, kcred); 3454 } 3455 3456 /* 3457 * Transfer the lock count from the old page (if any). 3458 * The page_struct_lock need not be acquired for lckcnt and 3459 * cowcnt since the page has an "exclusive" lock. 3460 */ 3461 opp->p_lckcnt += olckcnt; 3462 opp->p_cowcnt += ocowcnt; 3463 } 3464 3465 /* 3466 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3467 * 3468 * Pages are normally inserted at the start of a vnode's v_pages list. 3469 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3470 * This can happen when a modified page is relocated for DR. 3471 * 3472 * Returns 1 on success and 0 on failure. 3473 */ 3474 static int 3475 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3476 { 3477 page_t **listp; 3478 page_t *tp; 3479 ulong_t index; 3480 3481 ASSERT(PAGE_EXCL(pp)); 3482 ASSERT(vp != NULL); 3483 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3484 3485 /* 3486 * Be sure to set these up before the page is inserted on the hash 3487 * list. As soon as the page is placed on the list some other 3488 * thread might get confused and wonder how this page could 3489 * possibly hash to this list. 3490 */ 3491 pp->p_vnode = vp; 3492 pp->p_offset = offset; 3493 3494 /* 3495 * record if this page is on a swap vnode 3496 */ 3497 if ((vp->v_flag & VISSWAP) != 0) 3498 PP_SETSWAP(pp); 3499 3500 index = PAGE_HASH_FUNC(vp, offset); 3501 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3502 listp = &page_hash[index]; 3503 3504 /* 3505 * If this page is already hashed in, fail this attempt to add it. 3506 */ 3507 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3508 if (tp->p_vnode == vp && tp->p_offset == offset) { 3509 pp->p_vnode = NULL; 3510 pp->p_offset = (u_offset_t)(-1); 3511 return (0); 3512 } 3513 } 3514 pp->p_hash = *listp; 3515 *listp = pp; 3516 3517 /* 3518 * Add the page to the vnode's list of pages 3519 */ 3520 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3521 listp = &vp->v_pages->p_vpprev->p_vpnext; 3522 else 3523 listp = &vp->v_pages; 3524 3525 page_vpadd(listp, pp); 3526 3527 return (1); 3528 } 3529 3530 /* 3531 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3532 * 3533 * Returns 1 on success and 0 on failure. 3534 * If hold is passed in, it is not dropped. 3535 */ 3536 int 3537 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3538 { 3539 kmutex_t *phm = NULL; 3540 kmutex_t *vphm; 3541 int rc; 3542 3543 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3544 3545 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3546 "page_hashin:pp %p vp %p offset %llx", 3547 pp, vp, offset); 3548 3549 VM_STAT_ADD(hashin_count); 3550 3551 if (hold != NULL) 3552 phm = hold; 3553 else { 3554 VM_STAT_ADD(hashin_not_held); 3555 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3556 mutex_enter(phm); 3557 } 3558 3559 vphm = page_vnode_mutex(vp); 3560 mutex_enter(vphm); 3561 rc = page_do_hashin(pp, vp, offset); 3562 mutex_exit(vphm); 3563 if (hold == NULL) 3564 mutex_exit(phm); 3565 if (rc == 0) 3566 VM_STAT_ADD(hashin_already); 3567 return (rc); 3568 } 3569 3570 /* 3571 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3572 * All mutexes must be held 3573 */ 3574 static void 3575 page_do_hashout(page_t *pp) 3576 { 3577 page_t **hpp; 3578 page_t *hp; 3579 vnode_t *vp = pp->p_vnode; 3580 3581 ASSERT(vp != NULL); 3582 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3583 3584 /* 3585 * First, take pp off of its hash chain. 3586 */ 3587 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3588 3589 for (;;) { 3590 hp = *hpp; 3591 if (hp == pp) 3592 break; 3593 if (hp == NULL) { 3594 panic("page_do_hashout"); 3595 /*NOTREACHED*/ 3596 } 3597 hpp = &hp->p_hash; 3598 } 3599 *hpp = pp->p_hash; 3600 3601 /* 3602 * Now remove it from its associated vnode. 3603 */ 3604 if (vp->v_pages) 3605 page_vpsub(&vp->v_pages, pp); 3606 3607 pp->p_hash = NULL; 3608 page_clr_all_props(pp); 3609 PP_CLRSWAP(pp); 3610 pp->p_vnode = NULL; 3611 pp->p_offset = (u_offset_t)-1; 3612 } 3613 3614 /* 3615 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3616 * 3617 * When `phm' is non-NULL it contains the address of the mutex protecting the 3618 * hash list pp is on. It is not dropped. 3619 */ 3620 void 3621 page_hashout(page_t *pp, kmutex_t *phm) 3622 { 3623 vnode_t *vp; 3624 ulong_t index; 3625 kmutex_t *nphm; 3626 kmutex_t *vphm; 3627 kmutex_t *sep; 3628 3629 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3630 ASSERT(pp->p_vnode != NULL); 3631 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3632 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3633 3634 vp = pp->p_vnode; 3635 3636 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3637 "page_hashout:pp %p vp %p", pp, vp); 3638 3639 /* Kernel probe */ 3640 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3641 tnf_opaque, vnode, vp, 3642 tnf_offset, offset, pp->p_offset); 3643 3644 /* 3645 * 3646 */ 3647 VM_STAT_ADD(hashout_count); 3648 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3649 if (phm == NULL) { 3650 VM_STAT_ADD(hashout_not_held); 3651 nphm = PAGE_HASH_MUTEX(index); 3652 mutex_enter(nphm); 3653 } 3654 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3655 3656 3657 /* 3658 * grab page vnode mutex and remove it... 3659 */ 3660 vphm = page_vnode_mutex(vp); 3661 mutex_enter(vphm); 3662 3663 page_do_hashout(pp); 3664 3665 mutex_exit(vphm); 3666 if (phm == NULL) 3667 mutex_exit(nphm); 3668 3669 /* 3670 * Wake up processes waiting for this page. The page's 3671 * identity has been changed, and is probably not the 3672 * desired page any longer. 3673 */ 3674 sep = page_se_mutex(pp); 3675 mutex_enter(sep); 3676 pp->p_selock &= ~SE_EWANTED; 3677 if (CV_HAS_WAITERS(&pp->p_cv)) 3678 cv_broadcast(&pp->p_cv); 3679 mutex_exit(sep); 3680 } 3681 3682 /* 3683 * Add the page to the front of a linked list of pages 3684 * using the p_next & p_prev pointers for the list. 3685 * The caller is responsible for protecting the list pointers. 3686 */ 3687 void 3688 page_add(page_t **ppp, page_t *pp) 3689 { 3690 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3691 3692 page_add_common(ppp, pp); 3693 } 3694 3695 3696 3697 /* 3698 * Common code for page_add() and mach_page_add() 3699 */ 3700 void 3701 page_add_common(page_t **ppp, page_t *pp) 3702 { 3703 if (*ppp == NULL) { 3704 pp->p_next = pp->p_prev = pp; 3705 } else { 3706 pp->p_next = *ppp; 3707 pp->p_prev = (*ppp)->p_prev; 3708 (*ppp)->p_prev = pp; 3709 pp->p_prev->p_next = pp; 3710 } 3711 *ppp = pp; 3712 } 3713 3714 3715 /* 3716 * Remove this page from a linked list of pages 3717 * using the p_next & p_prev pointers for the list. 3718 * 3719 * The caller is responsible for protecting the list pointers. 3720 */ 3721 void 3722 page_sub(page_t **ppp, page_t *pp) 3723 { 3724 ASSERT((PP_ISFREE(pp)) ? 1 : 3725 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3726 3727 if (*ppp == NULL || pp == NULL) { 3728 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3729 (void *)pp, (void *)(*ppp)); 3730 /*NOTREACHED*/ 3731 } 3732 3733 page_sub_common(ppp, pp); 3734 } 3735 3736 3737 /* 3738 * Common code for page_sub() and mach_page_sub() 3739 */ 3740 void 3741 page_sub_common(page_t **ppp, page_t *pp) 3742 { 3743 if (*ppp == pp) 3744 *ppp = pp->p_next; /* go to next page */ 3745 3746 if (*ppp == pp) 3747 *ppp = NULL; /* page list is gone */ 3748 else { 3749 pp->p_prev->p_next = pp->p_next; 3750 pp->p_next->p_prev = pp->p_prev; 3751 } 3752 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3753 } 3754 3755 3756 /* 3757 * Break page list cppp into two lists with npages in the first list. 3758 * The tail is returned in nppp. 3759 */ 3760 void 3761 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3762 { 3763 page_t *s1pp = *oppp; 3764 page_t *s2pp; 3765 page_t *e1pp, *e2pp; 3766 long n = 0; 3767 3768 if (s1pp == NULL) { 3769 *nppp = NULL; 3770 return; 3771 } 3772 if (npages == 0) { 3773 *nppp = s1pp; 3774 *oppp = NULL; 3775 return; 3776 } 3777 for (n = 0, s2pp = *oppp; n < npages; n++) { 3778 s2pp = s2pp->p_next; 3779 } 3780 /* Fix head and tail of new lists */ 3781 e1pp = s2pp->p_prev; 3782 e2pp = s1pp->p_prev; 3783 s1pp->p_prev = e1pp; 3784 e1pp->p_next = s1pp; 3785 s2pp->p_prev = e2pp; 3786 e2pp->p_next = s2pp; 3787 3788 /* second list empty */ 3789 if (s2pp == s1pp) { 3790 *oppp = s1pp; 3791 *nppp = NULL; 3792 } else { 3793 *oppp = s1pp; 3794 *nppp = s2pp; 3795 } 3796 } 3797 3798 /* 3799 * Concatenate page list nppp onto the end of list ppp. 3800 */ 3801 void 3802 page_list_concat(page_t **ppp, page_t **nppp) 3803 { 3804 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3805 3806 if (*nppp == NULL) { 3807 return; 3808 } 3809 if (*ppp == NULL) { 3810 *ppp = *nppp; 3811 return; 3812 } 3813 s1pp = *ppp; 3814 e1pp = s1pp->p_prev; 3815 s2pp = *nppp; 3816 e2pp = s2pp->p_prev; 3817 s1pp->p_prev = e2pp; 3818 e2pp->p_next = s1pp; 3819 e1pp->p_next = s2pp; 3820 s2pp->p_prev = e1pp; 3821 } 3822 3823 /* 3824 * return the next page in the page list 3825 */ 3826 page_t * 3827 page_list_next(page_t *pp) 3828 { 3829 return (pp->p_next); 3830 } 3831 3832 3833 /* 3834 * Add the page to the front of the linked list of pages 3835 * using p_vpnext/p_vpprev pointers for the list. 3836 * 3837 * The caller is responsible for protecting the lists. 3838 */ 3839 void 3840 page_vpadd(page_t **ppp, page_t *pp) 3841 { 3842 if (*ppp == NULL) { 3843 pp->p_vpnext = pp->p_vpprev = pp; 3844 } else { 3845 pp->p_vpnext = *ppp; 3846 pp->p_vpprev = (*ppp)->p_vpprev; 3847 (*ppp)->p_vpprev = pp; 3848 pp->p_vpprev->p_vpnext = pp; 3849 } 3850 *ppp = pp; 3851 } 3852 3853 /* 3854 * Remove this page from the linked list of pages 3855 * using p_vpnext/p_vpprev pointers for the list. 3856 * 3857 * The caller is responsible for protecting the lists. 3858 */ 3859 void 3860 page_vpsub(page_t **ppp, page_t *pp) 3861 { 3862 if (*ppp == NULL || pp == NULL) { 3863 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3864 (void *)pp, (void *)(*ppp)); 3865 /*NOTREACHED*/ 3866 } 3867 3868 if (*ppp == pp) 3869 *ppp = pp->p_vpnext; /* go to next page */ 3870 3871 if (*ppp == pp) 3872 *ppp = NULL; /* page list is gone */ 3873 else { 3874 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3875 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3876 } 3877 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3878 } 3879 3880 /* 3881 * Lock a physical page into memory "long term". Used to support "lock 3882 * in memory" functions. Accepts the page to be locked, and a cow variable 3883 * to indicate whether a the lock will travel to the new page during 3884 * a potential copy-on-write. 3885 */ 3886 int 3887 page_pp_lock( 3888 page_t *pp, /* page to be locked */ 3889 int cow, /* cow lock */ 3890 int kernel) /* must succeed -- ignore checking */ 3891 { 3892 int r = 0; /* result -- assume failure */ 3893 3894 ASSERT(PAGE_LOCKED(pp)); 3895 3896 page_struct_lock(pp); 3897 /* 3898 * Acquire the "freemem_lock" for availrmem. 3899 */ 3900 if (cow) { 3901 mutex_enter(&freemem_lock); 3902 if ((availrmem > pages_pp_maximum) && 3903 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3904 availrmem--; 3905 pages_locked++; 3906 mutex_exit(&freemem_lock); 3907 r = 1; 3908 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3909 cmn_err(CE_WARN, 3910 "COW lock limit reached on pfn 0x%lx", 3911 page_pptonum(pp)); 3912 } 3913 } else 3914 mutex_exit(&freemem_lock); 3915 } else { 3916 if (pp->p_lckcnt) { 3917 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3918 r = 1; 3919 if (++pp->p_lckcnt == 3920 (ushort_t)PAGE_LOCK_MAXIMUM) { 3921 cmn_err(CE_WARN, "Page lock limit " 3922 "reached on pfn 0x%lx", 3923 page_pptonum(pp)); 3924 } 3925 } 3926 } else { 3927 if (kernel) { 3928 /* availrmem accounting done by caller */ 3929 ++pp->p_lckcnt; 3930 r = 1; 3931 } else { 3932 mutex_enter(&freemem_lock); 3933 if (availrmem > pages_pp_maximum) { 3934 availrmem--; 3935 pages_locked++; 3936 ++pp->p_lckcnt; 3937 r = 1; 3938 } 3939 mutex_exit(&freemem_lock); 3940 } 3941 } 3942 } 3943 page_struct_unlock(pp); 3944 return (r); 3945 } 3946 3947 /* 3948 * Decommit a lock on a physical page frame. Account for cow locks if 3949 * appropriate. 3950 */ 3951 void 3952 page_pp_unlock( 3953 page_t *pp, /* page to be unlocked */ 3954 int cow, /* expect cow lock */ 3955 int kernel) /* this was a kernel lock */ 3956 { 3957 ASSERT(PAGE_LOCKED(pp)); 3958 3959 page_struct_lock(pp); 3960 /* 3961 * Acquire the "freemem_lock" for availrmem. 3962 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3963 * could be called to unlock even if nothing is locked. This could 3964 * happen if locked file pages were truncated (removing the lock) 3965 * and the file was grown again and new pages faulted in; the new 3966 * pages are unlocked but the segment still thinks they're locked. 3967 */ 3968 if (cow) { 3969 if (pp->p_cowcnt) { 3970 mutex_enter(&freemem_lock); 3971 pp->p_cowcnt--; 3972 availrmem++; 3973 pages_locked--; 3974 mutex_exit(&freemem_lock); 3975 } 3976 } else { 3977 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3978 if (!kernel) { 3979 mutex_enter(&freemem_lock); 3980 availrmem++; 3981 pages_locked--; 3982 mutex_exit(&freemem_lock); 3983 } 3984 } 3985 } 3986 page_struct_unlock(pp); 3987 } 3988 3989 /* 3990 * This routine reserves availrmem for npages; 3991 * flags: KM_NOSLEEP or KM_SLEEP 3992 * returns 1 on success or 0 on failure 3993 */ 3994 int 3995 page_resv(pgcnt_t npages, uint_t flags) 3996 { 3997 mutex_enter(&freemem_lock); 3998 while (availrmem < tune.t_minarmem + npages) { 3999 if (flags & KM_NOSLEEP) { 4000 mutex_exit(&freemem_lock); 4001 return (0); 4002 } 4003 mutex_exit(&freemem_lock); 4004 page_needfree(npages); 4005 kmem_reap(); 4006 delay(hz >> 2); 4007 page_needfree(-(spgcnt_t)npages); 4008 mutex_enter(&freemem_lock); 4009 } 4010 availrmem -= npages; 4011 mutex_exit(&freemem_lock); 4012 return (1); 4013 } 4014 4015 /* 4016 * This routine unreserves availrmem for npages; 4017 */ 4018 void 4019 page_unresv(pgcnt_t npages) 4020 { 4021 mutex_enter(&freemem_lock); 4022 availrmem += npages; 4023 mutex_exit(&freemem_lock); 4024 } 4025 4026 /* 4027 * See Statement at the beginning of segvn_lockop() regarding 4028 * the way we handle cowcnts and lckcnts. 4029 * 4030 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 4031 * that breaks COW has PROT_WRITE. 4032 * 4033 * Note that, we may also break COW in case we are softlocking 4034 * on read access during physio; 4035 * in this softlock case, the vpage may not have PROT_WRITE. 4036 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 4037 * if the vpage doesn't have PROT_WRITE. 4038 * 4039 * This routine is never called if we are stealing a page 4040 * in anon_private. 4041 * 4042 * The caller subtracted from availrmem for read only mapping. 4043 * if lckcnt is 1 increment availrmem. 4044 */ 4045 void 4046 page_pp_useclaim( 4047 page_t *opp, /* original page frame losing lock */ 4048 page_t *npp, /* new page frame gaining lock */ 4049 uint_t write_perm) /* set if vpage has PROT_WRITE */ 4050 { 4051 int payback = 0; 4052 4053 ASSERT(PAGE_LOCKED(opp)); 4054 ASSERT(PAGE_LOCKED(npp)); 4055 4056 page_struct_lock(opp); 4057 4058 ASSERT(npp->p_cowcnt == 0); 4059 ASSERT(npp->p_lckcnt == 0); 4060 4061 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4062 if ((write_perm && opp->p_cowcnt != 0) || 4063 (!write_perm && opp->p_lckcnt != 0)) { 4064 4065 if (write_perm) { 4066 npp->p_cowcnt++; 4067 ASSERT(opp->p_cowcnt != 0); 4068 opp->p_cowcnt--; 4069 } else { 4070 4071 ASSERT(opp->p_lckcnt != 0); 4072 4073 /* 4074 * We didn't need availrmem decremented if p_lckcnt on 4075 * original page is 1. Here, we are unlocking 4076 * read-only copy belonging to original page and 4077 * are locking a copy belonging to new page. 4078 */ 4079 if (opp->p_lckcnt == 1) 4080 payback = 1; 4081 4082 npp->p_lckcnt++; 4083 opp->p_lckcnt--; 4084 } 4085 } 4086 if (payback) { 4087 mutex_enter(&freemem_lock); 4088 availrmem++; 4089 pages_useclaim--; 4090 mutex_exit(&freemem_lock); 4091 } 4092 page_struct_unlock(opp); 4093 } 4094 4095 /* 4096 * Simple claim adjust functions -- used to support changes in 4097 * claims due to changes in access permissions. Used by segvn_setprot(). 4098 */ 4099 int 4100 page_addclaim(page_t *pp) 4101 { 4102 int r = 0; /* result */ 4103 4104 ASSERT(PAGE_LOCKED(pp)); 4105 4106 page_struct_lock(pp); 4107 ASSERT(pp->p_lckcnt != 0); 4108 4109 if (pp->p_lckcnt == 1) { 4110 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4111 --pp->p_lckcnt; 4112 r = 1; 4113 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4114 cmn_err(CE_WARN, 4115 "COW lock limit reached on pfn 0x%lx", 4116 page_pptonum(pp)); 4117 } 4118 } 4119 } else { 4120 mutex_enter(&freemem_lock); 4121 if ((availrmem > pages_pp_maximum) && 4122 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4123 --availrmem; 4124 ++pages_claimed; 4125 mutex_exit(&freemem_lock); 4126 --pp->p_lckcnt; 4127 r = 1; 4128 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4129 cmn_err(CE_WARN, 4130 "COW lock limit reached on pfn 0x%lx", 4131 page_pptonum(pp)); 4132 } 4133 } else 4134 mutex_exit(&freemem_lock); 4135 } 4136 page_struct_unlock(pp); 4137 return (r); 4138 } 4139 4140 int 4141 page_subclaim(page_t *pp) 4142 { 4143 int r = 0; 4144 4145 ASSERT(PAGE_LOCKED(pp)); 4146 4147 page_struct_lock(pp); 4148 ASSERT(pp->p_cowcnt != 0); 4149 4150 if (pp->p_lckcnt) { 4151 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4152 r = 1; 4153 /* 4154 * for availrmem 4155 */ 4156 mutex_enter(&freemem_lock); 4157 availrmem++; 4158 pages_claimed--; 4159 mutex_exit(&freemem_lock); 4160 4161 pp->p_cowcnt--; 4162 4163 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4164 cmn_err(CE_WARN, 4165 "Page lock limit reached on pfn 0x%lx", 4166 page_pptonum(pp)); 4167 } 4168 } 4169 } else { 4170 r = 1; 4171 pp->p_cowcnt--; 4172 pp->p_lckcnt++; 4173 } 4174 page_struct_unlock(pp); 4175 return (r); 4176 } 4177 4178 int 4179 page_addclaim_pages(page_t **ppa) 4180 { 4181 4182 pgcnt_t lckpgs = 0, pg_idx; 4183 4184 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4185 4186 mutex_enter(&page_llock); 4187 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4188 4189 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4190 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4191 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4192 mutex_exit(&page_llock); 4193 return (0); 4194 } 4195 if (ppa[pg_idx]->p_lckcnt > 1) 4196 lckpgs++; 4197 } 4198 4199 if (lckpgs != 0) { 4200 mutex_enter(&freemem_lock); 4201 if (availrmem >= pages_pp_maximum + lckpgs) { 4202 availrmem -= lckpgs; 4203 pages_claimed += lckpgs; 4204 } else { 4205 mutex_exit(&freemem_lock); 4206 mutex_exit(&page_llock); 4207 return (0); 4208 } 4209 mutex_exit(&freemem_lock); 4210 } 4211 4212 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4213 ppa[pg_idx]->p_lckcnt--; 4214 ppa[pg_idx]->p_cowcnt++; 4215 } 4216 mutex_exit(&page_llock); 4217 return (1); 4218 } 4219 4220 int 4221 page_subclaim_pages(page_t **ppa) 4222 { 4223 pgcnt_t ulckpgs = 0, pg_idx; 4224 4225 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4226 4227 mutex_enter(&page_llock); 4228 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4229 4230 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4231 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4232 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4233 mutex_exit(&page_llock); 4234 return (0); 4235 } 4236 if (ppa[pg_idx]->p_lckcnt != 0) 4237 ulckpgs++; 4238 } 4239 4240 if (ulckpgs != 0) { 4241 mutex_enter(&freemem_lock); 4242 availrmem += ulckpgs; 4243 pages_claimed -= ulckpgs; 4244 mutex_exit(&freemem_lock); 4245 } 4246 4247 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4248 ppa[pg_idx]->p_cowcnt--; 4249 ppa[pg_idx]->p_lckcnt++; 4250 4251 } 4252 mutex_exit(&page_llock); 4253 return (1); 4254 } 4255 4256 page_t * 4257 page_numtopp(pfn_t pfnum, se_t se) 4258 { 4259 page_t *pp; 4260 4261 retry: 4262 pp = page_numtopp_nolock(pfnum); 4263 if (pp == NULL) { 4264 return ((page_t *)NULL); 4265 } 4266 4267 /* 4268 * Acquire the appropriate lock on the page. 4269 */ 4270 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4271 if (page_pptonum(pp) != pfnum) 4272 goto retry; 4273 continue; 4274 } 4275 4276 if (page_pptonum(pp) != pfnum) { 4277 page_unlock(pp); 4278 goto retry; 4279 } 4280 4281 return (pp); 4282 } 4283 4284 page_t * 4285 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4286 { 4287 page_t *pp; 4288 4289 retry: 4290 pp = page_numtopp_nolock(pfnum); 4291 if (pp == NULL) { 4292 return ((page_t *)NULL); 4293 } 4294 4295 /* 4296 * Acquire the appropriate lock on the page. 4297 */ 4298 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4299 if (page_pptonum(pp) != pfnum) 4300 goto retry; 4301 continue; 4302 } 4303 4304 if (page_pptonum(pp) != pfnum) { 4305 page_unlock(pp); 4306 goto retry; 4307 } 4308 4309 return (pp); 4310 } 4311 4312 /* 4313 * This routine is like page_numtopp, but will only return page structs 4314 * for pages which are ok for loading into hardware using the page struct. 4315 */ 4316 page_t * 4317 page_numtopp_nowait(pfn_t pfnum, se_t se) 4318 { 4319 page_t *pp; 4320 4321 retry: 4322 pp = page_numtopp_nolock(pfnum); 4323 if (pp == NULL) { 4324 return ((page_t *)NULL); 4325 } 4326 4327 /* 4328 * Try to acquire the appropriate lock on the page. 4329 */ 4330 if (PP_ISFREE(pp)) 4331 pp = NULL; 4332 else { 4333 if (!page_trylock(pp, se)) 4334 pp = NULL; 4335 else { 4336 if (page_pptonum(pp) != pfnum) { 4337 page_unlock(pp); 4338 goto retry; 4339 } 4340 if (PP_ISFREE(pp)) { 4341 page_unlock(pp); 4342 pp = NULL; 4343 } 4344 } 4345 } 4346 return (pp); 4347 } 4348 4349 /* 4350 * Returns a count of dirty pages that are in the process 4351 * of being written out. If 'cleanit' is set, try to push the page. 4352 */ 4353 pgcnt_t 4354 page_busy(int cleanit) 4355 { 4356 page_t *page0 = page_first(); 4357 page_t *pp = page0; 4358 pgcnt_t nppbusy = 0; 4359 u_offset_t off; 4360 4361 do { 4362 vnode_t *vp = pp->p_vnode; 4363 4364 /* 4365 * A page is a candidate for syncing if it is: 4366 * 4367 * (a) On neither the freelist nor the cachelist 4368 * (b) Hashed onto a vnode 4369 * (c) Not a kernel page 4370 * (d) Dirty 4371 * (e) Not part of a swapfile 4372 * (f) a page which belongs to a real vnode; eg has a non-null 4373 * v_vfsp pointer. 4374 * (g) Backed by a filesystem which doesn't have a 4375 * stubbed-out sync operation 4376 */ 4377 if (!PP_ISFREE(pp) && vp != NULL && vp != &kvp && 4378 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4379 vfs_can_sync(vp->v_vfsp)) { 4380 nppbusy++; 4381 vfs_syncprogress(); 4382 4383 if (!cleanit) 4384 continue; 4385 if (!page_trylock(pp, SE_EXCL)) 4386 continue; 4387 4388 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4389 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4390 !(hat_pagesync(pp, 4391 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4392 page_unlock(pp); 4393 continue; 4394 } 4395 off = pp->p_offset; 4396 VN_HOLD(vp); 4397 page_unlock(pp); 4398 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4399 B_ASYNC | B_FREE, kcred); 4400 VN_RELE(vp); 4401 } 4402 } while ((pp = page_next(pp)) != page0); 4403 4404 return (nppbusy); 4405 } 4406 4407 void page_invalidate_pages(void); 4408 4409 /* 4410 * callback handler to vm sub-system 4411 * 4412 * callers make sure no recursive entries to this func. 4413 */ 4414 /*ARGSUSED*/ 4415 boolean_t 4416 callb_vm_cpr(void *arg, int code) 4417 { 4418 if (code == CB_CODE_CPR_CHKPT) 4419 page_invalidate_pages(); 4420 return (B_TRUE); 4421 } 4422 4423 /* 4424 * Invalidate all pages of the system. 4425 * It shouldn't be called until all user page activities are all stopped. 4426 */ 4427 void 4428 page_invalidate_pages() 4429 { 4430 page_t *pp; 4431 page_t *page0; 4432 pgcnt_t nbusypages; 4433 int retry = 0; 4434 const int MAXRETRIES = 4; 4435 #if defined(__sparc) 4436 extern struct vnode prom_ppages; 4437 #endif /* __sparc */ 4438 4439 top: 4440 /* 4441 * Flush dirty pages and destory the clean ones. 4442 */ 4443 nbusypages = 0; 4444 4445 pp = page0 = page_first(); 4446 do { 4447 struct vnode *vp; 4448 u_offset_t offset; 4449 int mod; 4450 4451 /* 4452 * skip the page if it has no vnode or the page associated 4453 * with the kernel vnode or prom allocated kernel mem. 4454 */ 4455 #if defined(__sparc) 4456 if ((vp = pp->p_vnode) == NULL || vp == &kvp || 4457 vp == &prom_ppages) 4458 #else /* x86 doesn't have prom or prom_ppage */ 4459 if ((vp = pp->p_vnode) == NULL || vp == &kvp) 4460 #endif /* __sparc */ 4461 continue; 4462 4463 /* 4464 * skip the page which is already free invalidated. 4465 */ 4466 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4467 continue; 4468 4469 /* 4470 * skip pages that are already locked or can't be "exclusively" 4471 * locked or are already free. After we lock the page, check 4472 * the free and age bits again to be sure it's not destroied 4473 * yet. 4474 * To achieve max. parallelization, we use page_trylock instead 4475 * of page_lock so that we don't get block on individual pages 4476 * while we have thousands of other pages to process. 4477 */ 4478 if (!page_trylock(pp, SE_EXCL)) { 4479 nbusypages++; 4480 continue; 4481 } else if (PP_ISFREE(pp)) { 4482 if (!PP_ISAGED(pp)) { 4483 page_destroy_free(pp); 4484 } else { 4485 page_unlock(pp); 4486 } 4487 continue; 4488 } 4489 /* 4490 * Is this page involved in some I/O? shared? 4491 * 4492 * The page_struct_lock need not be acquired to 4493 * examine these fields since the page has an 4494 * "exclusive" lock. 4495 */ 4496 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4497 page_unlock(pp); 4498 continue; 4499 } 4500 4501 if (vp->v_type == VCHR) { 4502 panic("vp->v_type == VCHR"); 4503 /*NOTREACHED*/ 4504 } 4505 4506 if (!page_try_demote_pages(pp)) { 4507 page_unlock(pp); 4508 continue; 4509 } 4510 4511 /* 4512 * Check the modified bit. Leave the bits alone in hardware 4513 * (they will be modified if we do the putpage). 4514 */ 4515 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4516 & P_MOD); 4517 if (mod) { 4518 offset = pp->p_offset; 4519 /* 4520 * Hold the vnode before releasing the page lock 4521 * to prevent it from being freed and re-used by 4522 * some other thread. 4523 */ 4524 VN_HOLD(vp); 4525 page_unlock(pp); 4526 /* 4527 * No error return is checked here. Callers such as 4528 * cpr deals with the dirty pages at the dump time 4529 * if this putpage fails. 4530 */ 4531 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4532 kcred); 4533 VN_RELE(vp); 4534 } else { 4535 page_destroy(pp, 0); 4536 } 4537 } while ((pp = page_next(pp)) != page0); 4538 if (nbusypages && retry++ < MAXRETRIES) { 4539 delay(1); 4540 goto top; 4541 } 4542 } 4543 4544 /* 4545 * Replace the page "old" with the page "new" on the page hash and vnode lists 4546 * 4547 * the replacemnt must be done in place, ie the equivalent sequence: 4548 * 4549 * vp = old->p_vnode; 4550 * off = old->p_offset; 4551 * page_do_hashout(old) 4552 * page_do_hashin(new, vp, off) 4553 * 4554 * doesn't work, since 4555 * 1) if old is the only page on the vnode, the v_pages list has a window 4556 * where it looks empty. This will break file system assumptions. 4557 * and 4558 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4559 */ 4560 static void 4561 page_do_relocate_hash(page_t *new, page_t *old) 4562 { 4563 page_t **hash_list; 4564 vnode_t *vp = old->p_vnode; 4565 kmutex_t *sep; 4566 4567 ASSERT(PAGE_EXCL(old)); 4568 ASSERT(PAGE_EXCL(new)); 4569 ASSERT(vp != NULL); 4570 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4571 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4572 4573 /* 4574 * First find old page on the page hash list 4575 */ 4576 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4577 4578 for (;;) { 4579 if (*hash_list == old) 4580 break; 4581 if (*hash_list == NULL) { 4582 panic("page_do_hashout"); 4583 /*NOTREACHED*/ 4584 } 4585 hash_list = &(*hash_list)->p_hash; 4586 } 4587 4588 /* 4589 * update new and replace old with new on the page hash list 4590 */ 4591 new->p_vnode = old->p_vnode; 4592 new->p_offset = old->p_offset; 4593 new->p_hash = old->p_hash; 4594 *hash_list = new; 4595 4596 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4597 PP_SETSWAP(new); 4598 4599 /* 4600 * replace old with new on the vnode's page list 4601 */ 4602 if (old->p_vpnext == old) { 4603 new->p_vpnext = new; 4604 new->p_vpprev = new; 4605 } else { 4606 new->p_vpnext = old->p_vpnext; 4607 new->p_vpprev = old->p_vpprev; 4608 new->p_vpnext->p_vpprev = new; 4609 new->p_vpprev->p_vpnext = new; 4610 } 4611 if (vp->v_pages == old) 4612 vp->v_pages = new; 4613 4614 /* 4615 * clear out the old page 4616 */ 4617 old->p_hash = NULL; 4618 old->p_vpnext = NULL; 4619 old->p_vpprev = NULL; 4620 old->p_vnode = NULL; 4621 PP_CLRSWAP(old); 4622 old->p_offset = (u_offset_t)-1; 4623 page_clr_all_props(old); 4624 4625 /* 4626 * Wake up processes waiting for this page. The page's 4627 * identity has been changed, and is probably not the 4628 * desired page any longer. 4629 */ 4630 sep = page_se_mutex(old); 4631 mutex_enter(sep); 4632 old->p_selock &= ~SE_EWANTED; 4633 if (CV_HAS_WAITERS(&old->p_cv)) 4634 cv_broadcast(&old->p_cv); 4635 mutex_exit(sep); 4636 } 4637 4638 /* 4639 * This function moves the identity of page "pp_old" to page "pp_new". 4640 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4641 * and need not be hashed out from anywhere. 4642 */ 4643 void 4644 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4645 { 4646 vnode_t *vp = pp_old->p_vnode; 4647 u_offset_t off = pp_old->p_offset; 4648 kmutex_t *phm, *vphm; 4649 4650 /* 4651 * Rehash two pages 4652 */ 4653 ASSERT(PAGE_EXCL(pp_old)); 4654 ASSERT(PAGE_EXCL(pp_new)); 4655 ASSERT(vp != NULL); 4656 ASSERT(pp_new->p_vnode == NULL); 4657 4658 /* 4659 * hashout then hashin while holding the mutexes 4660 */ 4661 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4662 mutex_enter(phm); 4663 vphm = page_vnode_mutex(vp); 4664 mutex_enter(vphm); 4665 4666 page_do_relocate_hash(pp_new, pp_old); 4667 4668 mutex_exit(vphm); 4669 mutex_exit(phm); 4670 4671 /* 4672 * The page_struct_lock need not be acquired for lckcnt and 4673 * cowcnt since the page has an "exclusive" lock. 4674 */ 4675 ASSERT(pp_new->p_lckcnt == 0); 4676 ASSERT(pp_new->p_cowcnt == 0); 4677 pp_new->p_lckcnt = pp_old->p_lckcnt; 4678 pp_new->p_cowcnt = pp_old->p_cowcnt; 4679 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4680 4681 /* The following comment preserved from page_flip(). */ 4682 /* XXX - Do we need to protect fsdata? */ 4683 pp_new->p_fsdata = pp_old->p_fsdata; 4684 } 4685 4686 /* 4687 * Helper routine used to lock all remaining members of a 4688 * large page. The caller is responsible for passing in a locked 4689 * pp. If pp is a large page, then it succeeds in locking all the 4690 * remaining constituent pages or it returns with only the 4691 * original page locked. 4692 * 4693 * Returns 1 on success, 0 on failure. 4694 * 4695 * If success is returned this routine gurantees p_szc for all constituent 4696 * pages of a large page pp belongs to can't change. To achieve this we 4697 * recheck szc of pp after locking all constituent pages and retry if szc 4698 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4699 * lock on one of constituent pages it can't be running after all constituent 4700 * pages are locked. hat_page_demote() with a lock on a constituent page 4701 * outside of this large page (i.e. pp belonged to a larger large page) is 4702 * already done with all constituent pages of pp since the root's p_szc is 4703 * changed last. Thefore no need to synchronize with hat_page_demote() that 4704 * locked a constituent page outside of pp's current large page. 4705 */ 4706 #ifdef DEBUG 4707 uint32_t gpg_trylock_mtbf = 0; 4708 #endif 4709 4710 int 4711 group_page_trylock(page_t *pp, se_t se) 4712 { 4713 page_t *tpp; 4714 pgcnt_t npgs, i, j; 4715 uint_t pszc = pp->p_szc; 4716 4717 #ifdef DEBUG 4718 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4719 return (0); 4720 } 4721 #endif 4722 4723 if (pp != PP_GROUPLEADER(pp, pszc)) { 4724 return (0); 4725 } 4726 4727 retry: 4728 ASSERT(PAGE_LOCKED_SE(pp, se)); 4729 ASSERT(!PP_ISFREE(pp)); 4730 if (pszc == 0) { 4731 return (1); 4732 } 4733 npgs = page_get_pagecnt(pszc); 4734 tpp = pp + 1; 4735 for (i = 1; i < npgs; i++, tpp++) { 4736 if (!page_trylock(tpp, se)) { 4737 tpp = pp + 1; 4738 for (j = 1; j < i; j++, tpp++) { 4739 page_unlock(tpp); 4740 } 4741 return (0); 4742 } 4743 } 4744 if (pp->p_szc != pszc) { 4745 ASSERT(pp->p_szc < pszc); 4746 ASSERT(pp->p_vnode != NULL && pp->p_vnode != &kvp && 4747 !IS_SWAPFSVP(pp->p_vnode)); 4748 tpp = pp + 1; 4749 for (i = 1; i < npgs; i++, tpp++) { 4750 page_unlock(tpp); 4751 } 4752 pszc = pp->p_szc; 4753 goto retry; 4754 } 4755 return (1); 4756 } 4757 4758 void 4759 group_page_unlock(page_t *pp) 4760 { 4761 page_t *tpp; 4762 pgcnt_t npgs, i; 4763 4764 ASSERT(PAGE_LOCKED(pp)); 4765 ASSERT(!PP_ISFREE(pp)); 4766 ASSERT(pp == PP_PAGEROOT(pp)); 4767 npgs = page_get_pagecnt(pp->p_szc); 4768 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4769 page_unlock(tpp); 4770 } 4771 } 4772 4773 /* 4774 * returns 4775 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4776 * ERANGE : this is not a base page 4777 * EBUSY : failure to get locks on the page/pages 4778 * ENOMEM : failure to obtain replacement pages 4779 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4780 * 4781 * Return with all constituent members of target and replacement 4782 * SE_EXCL locked. It is the callers responsibility to drop the 4783 * locks. 4784 */ 4785 int 4786 do_page_relocate( 4787 page_t **target, 4788 page_t **replacement, 4789 int grouplock, 4790 spgcnt_t *nrelocp, 4791 lgrp_t *lgrp) 4792 { 4793 #ifdef DEBUG 4794 page_t *first_repl; 4795 #endif /* DEBUG */ 4796 page_t *repl; 4797 page_t *targ; 4798 page_t *pl = NULL; 4799 uint_t ppattr; 4800 pfn_t pfn, repl_pfn; 4801 uint_t szc; 4802 spgcnt_t npgs, i; 4803 int repl_contig = 0; 4804 uint_t flags = 0; 4805 spgcnt_t dofree = 0; 4806 4807 *nrelocp = 0; 4808 4809 #if defined(__sparc) 4810 /* 4811 * We need to wait till OBP has completed 4812 * its boot-time handoff of its resources to the kernel 4813 * before we allow page relocation 4814 */ 4815 if (page_relocate_ready == 0) { 4816 return (EAGAIN); 4817 } 4818 #endif 4819 4820 /* 4821 * If this is not a base page, 4822 * just return with 0x0 pages relocated. 4823 */ 4824 targ = *target; 4825 ASSERT(PAGE_EXCL(targ)); 4826 ASSERT(!PP_ISFREE(targ)); 4827 szc = targ->p_szc; 4828 ASSERT(szc < mmu_page_sizes); 4829 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4830 pfn = targ->p_pagenum; 4831 if (pfn != PFN_BASE(pfn, szc)) { 4832 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4833 return (ERANGE); 4834 } 4835 4836 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4837 repl_pfn = repl->p_pagenum; 4838 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4839 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4840 return (ERANGE); 4841 } 4842 repl_contig = 1; 4843 } 4844 4845 /* 4846 * We must lock all members of this large page or we cannot 4847 * relocate any part of it. 4848 */ 4849 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4850 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4851 return (EBUSY); 4852 } 4853 4854 /* 4855 * reread szc it could have been decreased before 4856 * group_page_trylock() was done. 4857 */ 4858 szc = targ->p_szc; 4859 ASSERT(szc < mmu_page_sizes); 4860 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4861 ASSERT(pfn == PFN_BASE(pfn, szc)); 4862 4863 npgs = page_get_pagecnt(targ->p_szc); 4864 4865 if (repl == NULL) { 4866 dofree = npgs; /* Size of target page in MMU pages */ 4867 if (!page_create_wait(dofree, 0)) { 4868 if (grouplock != 0) { 4869 group_page_unlock(targ); 4870 } 4871 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4872 return (ENOMEM); 4873 } 4874 4875 /* 4876 * seg kmem pages require that the target and replacement 4877 * page be the same pagesize. 4878 */ 4879 flags = (targ->p_vnode == &kvp) ? PGR_SAMESZC : 0; 4880 repl = page_get_replacement_page(targ, lgrp, flags); 4881 if (repl == NULL) { 4882 if (grouplock != 0) { 4883 group_page_unlock(targ); 4884 } 4885 page_create_putback(dofree); 4886 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4887 return (ENOMEM); 4888 } 4889 } 4890 #ifdef DEBUG 4891 else { 4892 ASSERT(PAGE_LOCKED(repl)); 4893 } 4894 #endif /* DEBUG */ 4895 4896 #if defined(__sparc) 4897 /* 4898 * Let hat_page_relocate() complete the relocation if it's kernel page 4899 */ 4900 if (targ->p_vnode == &kvp) { 4901 *replacement = repl; 4902 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4903 if (grouplock != 0) { 4904 group_page_unlock(targ); 4905 } 4906 if (dofree) { 4907 *replacement = NULL; 4908 page_free_replacement_page(repl); 4909 page_create_putback(dofree); 4910 } 4911 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4912 return (EAGAIN); 4913 } 4914 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4915 return (0); 4916 } 4917 #else 4918 #if defined(lint) 4919 dofree = dofree; 4920 #endif 4921 #endif 4922 4923 #ifdef DEBUG 4924 first_repl = repl; 4925 #endif /* DEBUG */ 4926 4927 for (i = 0; i < npgs; i++) { 4928 ASSERT(PAGE_EXCL(targ)); 4929 ASSERT(targ->p_slckcnt == 0); 4930 ASSERT(repl->p_slckcnt == 0); 4931 4932 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4933 4934 ASSERT(hat_page_getshare(targ) == 0); 4935 ASSERT(!PP_ISFREE(targ)); 4936 ASSERT(targ->p_pagenum == (pfn + i)); 4937 ASSERT(repl_contig == 0 || 4938 repl->p_pagenum == (repl_pfn + i)); 4939 4940 /* 4941 * Copy the page contents and attributes then 4942 * relocate the page in the page hash. 4943 */ 4944 ppcopy(targ, repl); 4945 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4946 page_clr_all_props(repl); 4947 page_set_props(repl, ppattr); 4948 page_relocate_hash(repl, targ); 4949 4950 ASSERT(hat_page_getshare(targ) == 0); 4951 ASSERT(hat_page_getshare(repl) == 0); 4952 /* 4953 * Now clear the props on targ, after the 4954 * page_relocate_hash(), they no longer 4955 * have any meaning. 4956 */ 4957 page_clr_all_props(targ); 4958 ASSERT(targ->p_next == targ); 4959 ASSERT(targ->p_prev == targ); 4960 page_list_concat(&pl, &targ); 4961 4962 targ++; 4963 if (repl_contig != 0) { 4964 repl++; 4965 } else { 4966 repl = repl->p_next; 4967 } 4968 } 4969 /* assert that we have come full circle with repl */ 4970 ASSERT(repl_contig == 1 || first_repl == repl); 4971 4972 *target = pl; 4973 if (*replacement == NULL) { 4974 ASSERT(first_repl == repl); 4975 *replacement = repl; 4976 } 4977 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4978 *nrelocp = npgs; 4979 return (0); 4980 } 4981 /* 4982 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 4983 */ 4984 int 4985 page_relocate( 4986 page_t **target, 4987 page_t **replacement, 4988 int grouplock, 4989 int freetarget, 4990 spgcnt_t *nrelocp, 4991 lgrp_t *lgrp) 4992 { 4993 spgcnt_t ret; 4994 4995 /* do_page_relocate returns 0 on success or errno value */ 4996 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 4997 4998 if (ret != 0 || freetarget == 0) { 4999 return (ret); 5000 } 5001 if (*nrelocp == 1) { 5002 ASSERT(*target != NULL); 5003 page_free(*target, 1); 5004 } else { 5005 page_t *tpp = *target; 5006 uint_t szc = tpp->p_szc; 5007 pgcnt_t npgs = page_get_pagecnt(szc); 5008 ASSERT(npgs > 1); 5009 ASSERT(szc != 0); 5010 do { 5011 ASSERT(PAGE_EXCL(tpp)); 5012 ASSERT(!hat_page_is_mapped(tpp)); 5013 ASSERT(tpp->p_szc == szc); 5014 PP_SETFREE(tpp); 5015 PP_SETAGED(tpp); 5016 npgs--; 5017 } while ((tpp = tpp->p_next) != *target); 5018 ASSERT(npgs == 0); 5019 page_list_add_pages(*target, 0); 5020 npgs = page_get_pagecnt(szc); 5021 page_create_putback(npgs); 5022 } 5023 return (ret); 5024 } 5025 5026 /* 5027 * it is up to the caller to deal with pcf accounting. 5028 */ 5029 void 5030 page_free_replacement_page(page_t *pplist) 5031 { 5032 page_t *pp; 5033 5034 while (pplist != NULL) { 5035 /* 5036 * pp_targ is a linked list. 5037 */ 5038 pp = pplist; 5039 if (pp->p_szc == 0) { 5040 page_sub(&pplist, pp); 5041 page_clr_all_props(pp); 5042 PP_SETFREE(pp); 5043 PP_SETAGED(pp); 5044 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5045 page_unlock(pp); 5046 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5047 } else { 5048 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5049 page_t *tpp; 5050 page_list_break(&pp, &pplist, curnpgs); 5051 tpp = pp; 5052 do { 5053 ASSERT(PAGE_EXCL(tpp)); 5054 ASSERT(!hat_page_is_mapped(tpp)); 5055 page_clr_all_props(pp); 5056 PP_SETFREE(tpp); 5057 PP_SETAGED(tpp); 5058 } while ((tpp = tpp->p_next) != pp); 5059 page_list_add_pages(pp, 0); 5060 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5061 } 5062 } 5063 } 5064 5065 /* 5066 * Relocate target to non-relocatable replacement page. 5067 */ 5068 int 5069 page_relocate_cage(page_t **target, page_t **replacement) 5070 { 5071 page_t *tpp, *rpp; 5072 spgcnt_t pgcnt, npgs; 5073 int result; 5074 5075 tpp = *target; 5076 5077 ASSERT(PAGE_EXCL(tpp)); 5078 ASSERT(tpp->p_szc == 0); 5079 5080 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5081 5082 do { 5083 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5084 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5085 if (rpp == NULL) { 5086 page_create_putback(pgcnt); 5087 kcage_cageout_wakeup(); 5088 } 5089 } while (rpp == NULL); 5090 5091 ASSERT(PP_ISNORELOC(rpp)); 5092 5093 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5094 5095 if (result == 0) { 5096 *replacement = rpp; 5097 if (pgcnt != npgs) 5098 panic("page_relocate_cage: partial relocation"); 5099 } 5100 5101 return (result); 5102 } 5103 5104 /* 5105 * Release the page lock on a page, place on cachelist 5106 * tail if no longer mapped. Caller can let us know if 5107 * the page is known to be clean. 5108 */ 5109 int 5110 page_release(page_t *pp, int checkmod) 5111 { 5112 int status; 5113 5114 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5115 (pp->p_vnode != NULL)); 5116 5117 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5118 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5119 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5120 !hat_page_is_mapped(pp)) { 5121 5122 /* 5123 * If page is modified, unlock it 5124 * 5125 * (p_nrm & P_MOD) bit has the latest stuff because: 5126 * (1) We found that this page doesn't have any mappings 5127 * _after_ holding SE_EXCL and 5128 * (2) We didn't drop SE_EXCL lock after the check in (1) 5129 */ 5130 if (checkmod && hat_ismod(pp)) { 5131 page_unlock(pp); 5132 status = PGREL_MOD; 5133 } else { 5134 /*LINTED: constant in conditional context*/ 5135 VN_DISPOSE(pp, B_FREE, 0, kcred); 5136 status = PGREL_CLEAN; 5137 } 5138 } else { 5139 page_unlock(pp); 5140 status = PGREL_NOTREL; 5141 } 5142 return (status); 5143 } 5144 5145 /* 5146 * Given a constituent page, try to demote the large page on the freelist. 5147 * 5148 * Returns nonzero if the page could be demoted successfully. Returns with 5149 * the constituent page still locked. 5150 */ 5151 int 5152 page_try_demote_free_pages(page_t *pp) 5153 { 5154 page_t *rootpp = pp; 5155 pfn_t pfn = page_pptonum(pp); 5156 spgcnt_t npgs; 5157 uint_t szc = pp->p_szc; 5158 5159 ASSERT(PP_ISFREE(pp)); 5160 ASSERT(PAGE_EXCL(pp)); 5161 5162 /* 5163 * Adjust rootpp and lock it, if `pp' is not the base 5164 * constituent page. 5165 */ 5166 npgs = page_get_pagecnt(pp->p_szc); 5167 if (npgs == 1) { 5168 return (0); 5169 } 5170 5171 if (!IS_P2ALIGNED(pfn, npgs)) { 5172 pfn = P2ALIGN(pfn, npgs); 5173 rootpp = page_numtopp_nolock(pfn); 5174 } 5175 5176 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5177 return (0); 5178 } 5179 5180 if (rootpp->p_szc != szc) { 5181 if (pp != rootpp) 5182 page_unlock(rootpp); 5183 return (0); 5184 } 5185 5186 page_demote_free_pages(rootpp); 5187 5188 if (pp != rootpp) 5189 page_unlock(rootpp); 5190 5191 ASSERT(PP_ISFREE(pp)); 5192 ASSERT(PAGE_EXCL(pp)); 5193 return (1); 5194 } 5195 5196 /* 5197 * Given a constituent page, try to demote the large page. 5198 * 5199 * Returns nonzero if the page could be demoted successfully. Returns with 5200 * the constituent page still locked. 5201 */ 5202 int 5203 page_try_demote_pages(page_t *pp) 5204 { 5205 page_t *tpp, *rootpp = pp; 5206 pfn_t pfn = page_pptonum(pp); 5207 spgcnt_t i, npgs; 5208 uint_t szc = pp->p_szc; 5209 vnode_t *vp = pp->p_vnode; 5210 5211 ASSERT(PAGE_EXCL(pp)); 5212 5213 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5214 5215 if (pp->p_szc == 0) { 5216 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5217 return (1); 5218 } 5219 5220 if (vp != NULL && !IS_SWAPFSVP(vp) && vp != &kvp) { 5221 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5222 page_demote_vp_pages(pp); 5223 ASSERT(pp->p_szc == 0); 5224 return (1); 5225 } 5226 5227 /* 5228 * Adjust rootpp if passed in is not the base 5229 * constituent page. 5230 */ 5231 npgs = page_get_pagecnt(pp->p_szc); 5232 ASSERT(npgs > 1); 5233 if (!IS_P2ALIGNED(pfn, npgs)) { 5234 pfn = P2ALIGN(pfn, npgs); 5235 rootpp = page_numtopp_nolock(pfn); 5236 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5237 ASSERT(rootpp->p_vnode != NULL); 5238 ASSERT(rootpp->p_szc == szc); 5239 } 5240 5241 /* 5242 * We can't demote kernel pages since we can't hat_unload() 5243 * the mappings. 5244 */ 5245 if (rootpp->p_vnode == &kvp) 5246 return (0); 5247 5248 /* 5249 * Attempt to lock all constituent pages except the page passed 5250 * in since it's already locked. 5251 */ 5252 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5253 ASSERT(!PP_ISFREE(tpp)); 5254 ASSERT(tpp->p_vnode != NULL); 5255 5256 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5257 break; 5258 ASSERT(tpp->p_szc == rootpp->p_szc); 5259 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5260 } 5261 5262 /* 5263 * If we failed to lock them all then unlock what we have 5264 * locked so far and bail. 5265 */ 5266 if (i < npgs) { 5267 tpp = rootpp; 5268 while (i-- > 0) { 5269 if (tpp != pp) 5270 page_unlock(tpp); 5271 tpp++; 5272 } 5273 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5274 return (0); 5275 } 5276 5277 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5278 ASSERT(PAGE_EXCL(tpp)); 5279 ASSERT(tpp->p_slckcnt == 0); 5280 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5281 tpp->p_szc = 0; 5282 } 5283 5284 /* 5285 * Unlock all pages except the page passed in. 5286 */ 5287 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5288 ASSERT(!hat_page_is_mapped(tpp)); 5289 if (tpp != pp) 5290 page_unlock(tpp); 5291 } 5292 5293 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5294 return (1); 5295 } 5296 5297 /* 5298 * Called by page_free() and page_destroy() to demote the page size code 5299 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5300 * p_szc on free list, neither can we just clear p_szc of a single page_t 5301 * within a large page since it will break other code that relies on p_szc 5302 * being the same for all page_t's of a large page). Anonymous pages should 5303 * never end up here because anon_map_getpages() cannot deal with p_szc 5304 * changes after a single constituent page is locked. While anonymous or 5305 * kernel large pages are demoted or freed the entire large page at a time 5306 * with all constituent pages locked EXCL for the file system pages we 5307 * have to be able to demote a large page (i.e. decrease all constituent pages 5308 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5309 * we can easily deal with anonymous page demotion the entire large page at a 5310 * time is that those operation originate at address space level and concern 5311 * the entire large page region with actual demotion only done when pages are 5312 * not shared with any other processes (therefore we can always get EXCL lock 5313 * on all anonymous constituent pages after clearing segment page 5314 * cache). However file system pages can be truncated or invalidated at a 5315 * PAGESIZE level from the file system side and end up in page_free() or 5316 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5317 * and therfore pageout should be able to demote a large page by EXCL locking 5318 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5319 * rely on being able to lock EXCL all constituent pages. 5320 * 5321 * To prevent szc changes on file system pages one has to lock all constituent 5322 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5323 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5324 * prevent szc changes is hat layer that uses its own page level mlist 5325 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5326 * taken. Therefore we need to change szc under hat level locks if we only 5327 * have an EXCL lock on a single constituent page and hat still references any 5328 * of constituent pages. (Note we can't "ignore" hat layer by simply 5329 * hat_pageunload() all constituent pages without having EXCL locks on all of 5330 * constituent pages). We use hat_page_demote() call to safely demote szc of 5331 * all constituent pages under hat locks when we only have an EXCL lock on one 5332 * of constituent pages. 5333 * 5334 * This routine calls page_szc_lock() before calling hat_page_demote() to 5335 * allow segvn in one special case not to lock all constituent pages SHARED 5336 * before calling hat_memload_array() that relies on p_szc not changeing even 5337 * before hat level mlist lock is taken. In that case segvn uses 5338 * page_szc_lock() to prevent hat_page_demote() changeing p_szc values. 5339 * 5340 * Anonymous or kernel page demotion still has to lock all pages exclusively 5341 * and do hat_pageunload() on all constituent pages before demoting the page 5342 * therefore there's no need for anonymous or kernel page demotion to use 5343 * hat_page_demote() mechanism. 5344 * 5345 * hat_page_demote() removes all large mappings that map pp and then decreases 5346 * p_szc starting from the last constituent page of the large page. By working 5347 * from the tail of a large page in pfn decreasing order allows one looking at 5348 * the root page to know that hat_page_demote() is done for root's szc area. 5349 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5350 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5351 * that started on this page when it had szc > 1 is done for this szc 1 area. 5352 * 5353 * We are guranteed that all constituent pages of pp's large page belong to 5354 * the same vnode with the consecutive offsets increasing in the direction of 5355 * the pfn i.e. the identity of constituent pages can't change until their 5356 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5357 * large mappings to pp even though we don't lock any constituent page except 5358 * pp (i.e. we won't unload e.g. kernel locked page). 5359 */ 5360 static void 5361 page_demote_vp_pages(page_t *pp) 5362 { 5363 kmutex_t *mtx; 5364 5365 ASSERT(PAGE_EXCL(pp)); 5366 ASSERT(!PP_ISFREE(pp)); 5367 ASSERT(pp->p_vnode != NULL); 5368 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5369 ASSERT(pp->p_vnode != &kvp); 5370 5371 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5372 5373 mtx = page_szc_lock(pp); 5374 if (mtx != NULL) { 5375 hat_page_demote(pp); 5376 mutex_exit(mtx); 5377 } 5378 ASSERT(pp->p_szc == 0); 5379 } 5380 5381 /* 5382 * Mark any existing pages for migration in the given range 5383 */ 5384 void 5385 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5386 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5387 u_offset_t vnoff, int rflag) 5388 { 5389 struct anon *ap; 5390 vnode_t *curvp; 5391 lgrp_t *from; 5392 pgcnt_t i; 5393 pgcnt_t nlocked; 5394 u_offset_t off; 5395 pfn_t pfn; 5396 size_t pgsz; 5397 size_t segpgsz; 5398 pgcnt_t pages; 5399 uint_t pszc; 5400 page_t **ppa; 5401 pgcnt_t ppa_nentries; 5402 page_t *pp; 5403 caddr_t va; 5404 ulong_t an_idx; 5405 anon_sync_obj_t cookie; 5406 5407 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5408 5409 /* 5410 * Don't do anything if don't need to do lgroup optimizations 5411 * on this system 5412 */ 5413 if (!lgrp_optimizations()) 5414 return; 5415 5416 /* 5417 * Align address and length to (potentially large) page boundary 5418 */ 5419 segpgsz = page_get_pagesize(seg->s_szc); 5420 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5421 if (rflag) 5422 len = P2ROUNDUP(len, segpgsz); 5423 5424 /* 5425 * Allocate page array to accomodate largest page size 5426 */ 5427 pgsz = page_get_pagesize(page_num_pagesizes() - 1); 5428 ppa_nentries = btop(pgsz); 5429 ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP); 5430 5431 /* 5432 * Do one (large) page at a time 5433 */ 5434 va = addr; 5435 while (va < addr + len) { 5436 /* 5437 * Lookup (root) page for vnode and offset corresponding to 5438 * this virtual address 5439 * Try anonmap first since there may be copy-on-write 5440 * pages, but initialize vnode pointer and offset using 5441 * vnode arguments just in case there isn't an amp. 5442 */ 5443 curvp = vp; 5444 off = vnoff + va - seg->s_base; 5445 if (amp) { 5446 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5447 an_idx = anon_index + seg_page(seg, va); 5448 anon_array_enter(amp, an_idx, &cookie); 5449 ap = anon_get_ptr(amp->ahp, an_idx); 5450 if (ap) 5451 swap_xlate(ap, &curvp, &off); 5452 anon_array_exit(&cookie); 5453 ANON_LOCK_EXIT(&->a_rwlock); 5454 } 5455 5456 pp = NULL; 5457 if (curvp) 5458 pp = page_lookup(curvp, off, SE_SHARED); 5459 5460 /* 5461 * If there isn't a page at this virtual address, 5462 * skip to next page 5463 */ 5464 if (pp == NULL) { 5465 va += PAGESIZE; 5466 continue; 5467 } 5468 5469 /* 5470 * Figure out which lgroup this page is in for kstats 5471 */ 5472 pfn = page_pptonum(pp); 5473 from = lgrp_pfn_to_lgrp(pfn); 5474 5475 /* 5476 * Get page size, and round up and skip to next page boundary 5477 * if unaligned address 5478 */ 5479 pszc = pp->p_szc; 5480 pgsz = page_get_pagesize(pszc); 5481 pages = btop(pgsz); 5482 if (!IS_P2ALIGNED(va, pgsz) || 5483 !IS_P2ALIGNED(pfn, pages) || 5484 pgsz > segpgsz) { 5485 pgsz = MIN(pgsz, segpgsz); 5486 page_unlock(pp); 5487 i = btop(P2END((uintptr_t)va, pgsz) - 5488 (uintptr_t)va); 5489 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5490 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i); 5491 continue; 5492 } 5493 5494 /* 5495 * Upgrade to exclusive lock on page 5496 */ 5497 if (!page_tryupgrade(pp)) { 5498 page_unlock(pp); 5499 va += pgsz; 5500 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5501 btop(pgsz)); 5502 continue; 5503 } 5504 5505 /* 5506 * Remember pages locked exclusively and how many 5507 */ 5508 ppa[0] = pp; 5509 nlocked = 1; 5510 5511 /* 5512 * Lock constituent pages if this is large page 5513 */ 5514 if (pages > 1) { 5515 /* 5516 * Lock all constituents except root page, since it 5517 * should be locked already. 5518 */ 5519 for (i = 1; i < pages; i++) { 5520 pp++; 5521 if (!page_trylock(pp, SE_EXCL)) { 5522 break; 5523 } 5524 if (PP_ISFREE(pp) || 5525 pp->p_szc != pszc) { 5526 /* 5527 * hat_page_demote() raced in with us. 5528 */ 5529 ASSERT(!IS_SWAPFSVP(curvp)); 5530 page_unlock(pp); 5531 break; 5532 } 5533 ppa[nlocked] = pp; 5534 nlocked++; 5535 } 5536 } 5537 5538 /* 5539 * If all constituent pages couldn't be locked, 5540 * unlock pages locked so far and skip to next page. 5541 */ 5542 if (nlocked != pages) { 5543 for (i = 0; i < nlocked; i++) 5544 page_unlock(ppa[i]); 5545 va += pgsz; 5546 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5547 btop(pgsz)); 5548 continue; 5549 } 5550 5551 /* 5552 * hat_page_demote() can no longer happen 5553 * since last cons page had the right p_szc after 5554 * all cons pages were locked. all cons pages 5555 * should now have the same p_szc. 5556 */ 5557 5558 /* 5559 * All constituent pages locked successfully, so mark 5560 * large page for migration and unload the mappings of 5561 * constituent pages, so a fault will occur on any part of the 5562 * large page 5563 */ 5564 PP_SETMIGRATE(ppa[0]); 5565 for (i = 0; i < nlocked; i++) { 5566 pp = ppa[i]; 5567 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 5568 ASSERT(hat_page_getshare(pp) == 0); 5569 page_unlock(pp); 5570 } 5571 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5572 5573 va += pgsz; 5574 } 5575 kmem_free(ppa, ppa_nentries * sizeof (page_t *)); 5576 } 5577 5578 /* 5579 * Migrate any pages that have been marked for migration in the given range 5580 */ 5581 void 5582 page_migrate( 5583 struct seg *seg, 5584 caddr_t addr, 5585 page_t **ppa, 5586 pgcnt_t npages) 5587 { 5588 lgrp_t *from; 5589 lgrp_t *to; 5590 page_t *newpp; 5591 page_t *pp; 5592 pfn_t pfn; 5593 size_t pgsz; 5594 spgcnt_t page_cnt; 5595 spgcnt_t i; 5596 uint_t pszc; 5597 5598 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5599 5600 while (npages > 0) { 5601 pp = *ppa; 5602 pszc = pp->p_szc; 5603 pgsz = page_get_pagesize(pszc); 5604 page_cnt = btop(pgsz); 5605 5606 /* 5607 * Check to see whether this page is marked for migration 5608 * 5609 * Assume that root page of large page is marked for 5610 * migration and none of the other constituent pages 5611 * are marked. This really simplifies clearing the 5612 * migrate bit by not having to clear it from each 5613 * constituent page. 5614 * 5615 * note we don't want to relocate an entire large page if 5616 * someone is only using one subpage. 5617 */ 5618 if (npages < page_cnt) 5619 break; 5620 5621 /* 5622 * Is it marked for migration? 5623 */ 5624 if (!PP_ISMIGRATE(pp)) 5625 goto next; 5626 5627 /* 5628 * Determine lgroups that page is being migrated between 5629 */ 5630 pfn = page_pptonum(pp); 5631 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5632 break; 5633 } 5634 from = lgrp_pfn_to_lgrp(pfn); 5635 to = lgrp_mem_choose(seg, addr, pgsz); 5636 5637 /* 5638 * Check to see whether we are trying to migrate page to lgroup 5639 * where it is allocated already 5640 */ 5641 if (to == from) { 5642 PP_CLRMIGRATE(pp); 5643 goto next; 5644 } 5645 5646 /* 5647 * Need to get exclusive lock's to migrate 5648 */ 5649 for (i = 0; i < page_cnt; i++) { 5650 ASSERT(PAGE_LOCKED(ppa[i])); 5651 if (page_pptonum(ppa[i]) != pfn + i || 5652 ppa[i]->p_szc != pszc) { 5653 break; 5654 } 5655 if (!page_tryupgrade(ppa[i])) { 5656 lgrp_stat_add(from->lgrp_id, 5657 LGRP_PM_FAIL_LOCK_PGS, 5658 page_cnt); 5659 break; 5660 } 5661 } 5662 if (i != page_cnt) { 5663 while (--i != -1) { 5664 page_downgrade(ppa[i]); 5665 } 5666 goto next; 5667 } 5668 5669 (void) page_create_wait(page_cnt, PG_WAIT); 5670 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5671 if (newpp == NULL) { 5672 page_create_putback(page_cnt); 5673 for (i = 0; i < page_cnt; i++) { 5674 page_downgrade(ppa[i]); 5675 } 5676 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5677 page_cnt); 5678 goto next; 5679 } 5680 ASSERT(newpp->p_szc == pszc); 5681 /* 5682 * Clear migrate bit and relocate page 5683 */ 5684 PP_CLRMIGRATE(pp); 5685 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5686 panic("page_migrate: page_relocate failed"); 5687 } 5688 ASSERT(page_cnt * PAGESIZE == pgsz); 5689 5690 /* 5691 * Keep stats for number of pages migrated from and to 5692 * each lgroup 5693 */ 5694 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5695 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5696 /* 5697 * update the page_t array we were passed in and 5698 * unlink constituent pages of a large page. 5699 */ 5700 for (i = 0; i < page_cnt; ++i, ++pp) { 5701 ASSERT(PAGE_EXCL(newpp)); 5702 ASSERT(newpp->p_szc == pszc); 5703 ppa[i] = newpp; 5704 pp = newpp; 5705 page_sub(&newpp, pp); 5706 page_downgrade(pp); 5707 } 5708 ASSERT(newpp == NULL); 5709 next: 5710 addr += pgsz; 5711 ppa += page_cnt; 5712 npages -= page_cnt; 5713 } 5714 } 5715 5716 ulong_t mem_waiters = 0; 5717 ulong_t max_count = 20; 5718 #define MAX_DELAY 0x1ff 5719 5720 /* 5721 * Check if enough memory is available to proceed. 5722 * Depending on system configuration and how much memory is 5723 * reserved for swap we need to check against two variables. 5724 * e.g. on systems with little physical swap availrmem can be 5725 * more reliable indicator of how much memory is available. 5726 * On systems with large phys swap freemem can be better indicator. 5727 * If freemem drops below threshold level don't return an error 5728 * immediately but wake up pageout to free memory and block. 5729 * This is done number of times. If pageout is not able to free 5730 * memory within certain time return an error. 5731 * The same applies for availrmem but kmem_reap is used to 5732 * free memory. 5733 */ 5734 int 5735 page_mem_avail(pgcnt_t npages) 5736 { 5737 ulong_t count; 5738 5739 #if defined(__i386) 5740 if (freemem > desfree + npages && 5741 availrmem > swapfs_reserve + npages && 5742 btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem + 5743 npages) 5744 return (1); 5745 #else 5746 if (freemem > desfree + npages && 5747 availrmem > swapfs_reserve + npages) 5748 return (1); 5749 #endif 5750 5751 count = max_count; 5752 atomic_add_long(&mem_waiters, 1); 5753 5754 while (freemem < desfree + npages && --count) { 5755 cv_signal(&proc_pageout->p_cv); 5756 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5757 atomic_add_long(&mem_waiters, -1); 5758 return (0); 5759 } 5760 } 5761 if (count == 0) { 5762 atomic_add_long(&mem_waiters, -1); 5763 return (0); 5764 } 5765 5766 count = max_count; 5767 while (availrmem < swapfs_reserve + npages && --count) { 5768 kmem_reap(); 5769 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5770 atomic_add_long(&mem_waiters, -1); 5771 return (0); 5772 } 5773 } 5774 atomic_add_long(&mem_waiters, -1); 5775 if (count == 0) 5776 return (0); 5777 5778 #if defined(__i386) 5779 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 5780 tune.t_minarmem + npages) 5781 return (0); 5782 #endif 5783 return (1); 5784 } 5785 5786 #define MAX_CNT 60 /* max num of iterations */ 5787 /* 5788 * Reclaim/reserve availrmem for npages. 5789 * If there is not enough memory start reaping seg, kmem caches. 5790 * Start pageout scanner (via page_needfree()). 5791 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5792 * Note: There is no guarantee that any availrmem will be freed as 5793 * this memory typically is locked (kernel heap) or reserved for swap. 5794 * Also due to memory fragmentation kmem allocator may not be able 5795 * to free any memory (single user allocated buffer will prevent 5796 * freeing slab or a page). 5797 */ 5798 int 5799 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5800 { 5801 int i = 0; 5802 int ret = 0; 5803 pgcnt_t deficit; 5804 pgcnt_t old_availrmem; 5805 5806 mutex_enter(&freemem_lock); 5807 old_availrmem = availrmem - 1; 5808 while ((availrmem < tune.t_minarmem + npages + epages) && 5809 (old_availrmem < availrmem) && (i++ < MAX_CNT)) { 5810 old_availrmem = availrmem; 5811 deficit = tune.t_minarmem + npages + epages - availrmem; 5812 mutex_exit(&freemem_lock); 5813 page_needfree(deficit); 5814 seg_preap(); 5815 kmem_reap(); 5816 delay(hz); 5817 page_needfree(-(spgcnt_t)deficit); 5818 mutex_enter(&freemem_lock); 5819 } 5820 5821 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5822 availrmem -= npages; 5823 ret = 1; 5824 } 5825 5826 mutex_exit(&freemem_lock); 5827 5828 return (ret); 5829 } 5830 5831 /* 5832 * Search the memory segments to locate the desired page. Within a 5833 * segment, pages increase linearly with one page structure per 5834 * physical page frame (size PAGESIZE). The search begins 5835 * with the segment that was accessed last, to take advantage of locality. 5836 * If the hint misses, we start from the beginning of the sorted memseg list 5837 */ 5838 5839 5840 /* 5841 * Some data structures for pfn to pp lookup. 5842 */ 5843 ulong_t mhash_per_slot; 5844 struct memseg *memseg_hash[N_MEM_SLOTS]; 5845 5846 page_t * 5847 page_numtopp_nolock(pfn_t pfnum) 5848 { 5849 struct memseg *seg; 5850 page_t *pp; 5851 vm_cpu_data_t *vc = CPU->cpu_vm_data; 5852 5853 ASSERT(vc != NULL); 5854 5855 MEMSEG_STAT_INCR(nsearch); 5856 5857 /* Try last winner first */ 5858 if (((seg = vc->vc_pnum_memseg) != NULL) && 5859 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5860 MEMSEG_STAT_INCR(nlastwon); 5861 pp = seg->pages + (pfnum - seg->pages_base); 5862 if (pp->p_pagenum == pfnum) 5863 return ((page_t *)pp); 5864 } 5865 5866 /* Else Try hash */ 5867 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5868 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5869 MEMSEG_STAT_INCR(nhashwon); 5870 vc->vc_pnum_memseg = seg; 5871 pp = seg->pages + (pfnum - seg->pages_base); 5872 if (pp->p_pagenum == pfnum) 5873 return ((page_t *)pp); 5874 } 5875 5876 /* Else Brute force */ 5877 for (seg = memsegs; seg != NULL; seg = seg->next) { 5878 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5879 vc->vc_pnum_memseg = seg; 5880 pp = seg->pages + (pfnum - seg->pages_base); 5881 return ((page_t *)pp); 5882 } 5883 } 5884 vc->vc_pnum_memseg = NULL; 5885 MEMSEG_STAT_INCR(nnotfound); 5886 return ((page_t *)NULL); 5887 5888 } 5889 5890 struct memseg * 5891 page_numtomemseg_nolock(pfn_t pfnum) 5892 { 5893 struct memseg *seg; 5894 page_t *pp; 5895 5896 /* Try hash */ 5897 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5898 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5899 pp = seg->pages + (pfnum - seg->pages_base); 5900 if (pp->p_pagenum == pfnum) 5901 return (seg); 5902 } 5903 5904 /* Else Brute force */ 5905 for (seg = memsegs; seg != NULL; seg = seg->next) { 5906 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5907 return (seg); 5908 } 5909 } 5910 return ((struct memseg *)NULL); 5911 } 5912 5913 /* 5914 * Given a page and a count return the page struct that is 5915 * n structs away from the current one in the global page 5916 * list. 5917 * 5918 * This function wraps to the first page upon 5919 * reaching the end of the memseg list. 5920 */ 5921 page_t * 5922 page_nextn(page_t *pp, ulong_t n) 5923 { 5924 struct memseg *seg; 5925 page_t *ppn; 5926 vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5927 5928 ASSERT(vc != NULL); 5929 5930 if (((seg = vc->vc_pnext_memseg) == NULL) || 5931 (seg->pages_base == seg->pages_end) || 5932 !(pp >= seg->pages && pp < seg->epages)) { 5933 5934 for (seg = memsegs; seg; seg = seg->next) { 5935 if (pp >= seg->pages && pp < seg->epages) 5936 break; 5937 } 5938 5939 if (seg == NULL) { 5940 /* Memory delete got in, return something valid. */ 5941 /* TODO: fix me. */ 5942 seg = memsegs; 5943 pp = seg->pages; 5944 } 5945 } 5946 5947 /* check for wraparound - possible if n is large */ 5948 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5949 n -= seg->epages - pp; 5950 seg = seg->next; 5951 if (seg == NULL) 5952 seg = memsegs; 5953 pp = seg->pages; 5954 } 5955 vc->vc_pnext_memseg = seg; 5956 return (ppn); 5957 } 5958 5959 /* 5960 * Initialize for a loop using page_next_scan_large(). 5961 */ 5962 page_t * 5963 page_next_scan_init(void **cookie) 5964 { 5965 ASSERT(cookie != NULL); 5966 *cookie = (void *)memsegs; 5967 return ((page_t *)memsegs->pages); 5968 } 5969 5970 /* 5971 * Return the next page in a scan of page_t's, assuming we want 5972 * to skip over sub-pages within larger page sizes. 5973 * 5974 * The cookie is used to keep track of the current memseg. 5975 */ 5976 page_t * 5977 page_next_scan_large( 5978 page_t *pp, 5979 ulong_t *n, 5980 void **cookie) 5981 { 5982 struct memseg *seg = (struct memseg *)*cookie; 5983 page_t *new_pp; 5984 ulong_t cnt; 5985 pfn_t pfn; 5986 5987 5988 /* 5989 * get the count of page_t's to skip based on the page size 5990 */ 5991 ASSERT(pp != NULL); 5992 if (pp->p_szc == 0) { 5993 cnt = 1; 5994 } else { 5995 pfn = page_pptonum(pp); 5996 cnt = page_get_pagecnt(pp->p_szc); 5997 cnt -= pfn & (cnt - 1); 5998 } 5999 *n += cnt; 6000 new_pp = pp + cnt; 6001 6002 /* 6003 * Catch if we went past the end of the current memory segment. If so, 6004 * just move to the next segment with pages. 6005 */ 6006 if (new_pp >= seg->epages) { 6007 do { 6008 seg = seg->next; 6009 if (seg == NULL) 6010 seg = memsegs; 6011 } while (seg->pages == seg->epages); 6012 new_pp = seg->pages; 6013 *cookie = (void *)seg; 6014 } 6015 6016 return (new_pp); 6017 } 6018 6019 6020 /* 6021 * Returns next page in list. Note: this function wraps 6022 * to the first page in the list upon reaching the end 6023 * of the list. Callers should be aware of this fact. 6024 */ 6025 6026 /* We should change this be a #define */ 6027 6028 page_t * 6029 page_next(page_t *pp) 6030 { 6031 return (page_nextn(pp, 1)); 6032 } 6033 6034 page_t * 6035 page_first() 6036 { 6037 return ((page_t *)memsegs->pages); 6038 } 6039 6040 6041 /* 6042 * This routine is called at boot with the initial memory configuration 6043 * and when memory is added or removed. 6044 */ 6045 void 6046 build_pfn_hash() 6047 { 6048 pfn_t cur; 6049 pgcnt_t index; 6050 struct memseg *pseg; 6051 int i; 6052 6053 /* 6054 * Clear memseg_hash array. 6055 * Since memory add/delete is designed to operate concurrently 6056 * with normal operation, the hash rebuild must be able to run 6057 * concurrently with page_numtopp_nolock(). To support this 6058 * functionality, assignments to memseg_hash array members must 6059 * be done atomically. 6060 * 6061 * NOTE: bzero() does not currently guarantee this for kernel 6062 * threads, and cannot be used here. 6063 */ 6064 for (i = 0; i < N_MEM_SLOTS; i++) 6065 memseg_hash[i] = NULL; 6066 6067 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6068 6069 /* 6070 * Physmax is the last valid pfn. 6071 */ 6072 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6073 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6074 index = MEMSEG_PFN_HASH(pseg->pages_base); 6075 cur = pseg->pages_base; 6076 do { 6077 if (index >= N_MEM_SLOTS) 6078 index = MEMSEG_PFN_HASH(cur); 6079 6080 if (memseg_hash[index] == NULL || 6081 memseg_hash[index]->pages_base > pseg->pages_base) { 6082 memseg_hash[index] = pseg; 6083 hat_kpm_mseghash_update(index, pseg); 6084 } 6085 cur += mhash_per_slot; 6086 index++; 6087 } while (cur < pseg->pages_end); 6088 } 6089 } 6090 6091 /* 6092 * Return the pagenum for the pp 6093 */ 6094 pfn_t 6095 page_pptonum(page_t *pp) 6096 { 6097 return (pp->p_pagenum); 6098 } 6099 6100 /* 6101 * interface to the referenced and modified etc bits 6102 * in the PSM part of the page struct 6103 * when no locking is desired. 6104 */ 6105 void 6106 page_set_props(page_t *pp, uint_t flags) 6107 { 6108 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6109 pp->p_nrm |= (uchar_t)flags; 6110 } 6111 6112 void 6113 page_clr_all_props(page_t *pp) 6114 { 6115 pp->p_nrm = 0; 6116 } 6117 6118 /* 6119 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6120 */ 6121 int 6122 page_clear_lck_cow(page_t *pp, int adjust) 6123 { 6124 int f_amount; 6125 6126 ASSERT(PAGE_EXCL(pp)); 6127 6128 /* 6129 * The page_struct_lock need not be acquired here since 6130 * we require the caller hold the page exclusively locked. 6131 */ 6132 f_amount = 0; 6133 if (pp->p_lckcnt) { 6134 f_amount = 1; 6135 pp->p_lckcnt = 0; 6136 } 6137 if (pp->p_cowcnt) { 6138 f_amount += pp->p_cowcnt; 6139 pp->p_cowcnt = 0; 6140 } 6141 6142 if (adjust && f_amount) { 6143 mutex_enter(&freemem_lock); 6144 availrmem += f_amount; 6145 mutex_exit(&freemem_lock); 6146 } 6147 6148 return (f_amount); 6149 } 6150 6151 /* 6152 * The following functions is called from free_vp_pages() 6153 * for an inexact estimate of a newly free'd page... 6154 */ 6155 ulong_t 6156 page_share_cnt(page_t *pp) 6157 { 6158 return (hat_page_getshare(pp)); 6159 } 6160 6161 int 6162 page_isshared(page_t *pp) 6163 { 6164 return (hat_page_getshare(pp) > 1); 6165 } 6166 6167 int 6168 page_isfree(page_t *pp) 6169 { 6170 return (PP_ISFREE(pp)); 6171 } 6172 6173 int 6174 page_isref(page_t *pp) 6175 { 6176 return (hat_page_getattr(pp, P_REF)); 6177 } 6178 6179 int 6180 page_ismod(page_t *pp) 6181 { 6182 return (hat_page_getattr(pp, P_MOD)); 6183 } 6184