1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #pragma ident "%Z%%M% %I% %E% SMI" 40 41 /* 42 * VM - physical page management. 43 */ 44 45 #include <sys/types.h> 46 #include <sys/t_lock.h> 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/errno.h> 50 #include <sys/time.h> 51 #include <sys/vnode.h> 52 #include <sys/vm.h> 53 #include <sys/vtrace.h> 54 #include <sys/swap.h> 55 #include <sys/cmn_err.h> 56 #include <sys/tuneable.h> 57 #include <sys/sysmacros.h> 58 #include <sys/cpuvar.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/tnf_probe.h> 62 #include <sys/condvar_impl.h> 63 #include <sys/mem_config.h> 64 #include <sys/mem_cage.h> 65 #include <sys/kmem.h> 66 #include <sys/atomic.h> 67 #include <sys/strlog.h> 68 #include <sys/mman.h> 69 #include <sys/ontrap.h> 70 #include <sys/lgrp.h> 71 #include <sys/vfs.h> 72 73 #include <vm/hat.h> 74 #include <vm/anon.h> 75 #include <vm/page.h> 76 #include <vm/seg.h> 77 #include <vm/pvn.h> 78 #include <vm/seg_kmem.h> 79 #include <vm/vm_dep.h> 80 81 #include <fs/fs_subr.h> 82 83 static int nopageage = 0; 84 85 static pgcnt_t max_page_get; /* max page_get request size in pages */ 86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 87 88 /* 89 * freemem_lock protects all freemem variables: 90 * availrmem. Also this lock protects the globals which track the 91 * availrmem changes for accurate kernel footprint calculation. 92 * See below for an explanation of these 93 * globals. 94 */ 95 kmutex_t freemem_lock; 96 pgcnt_t availrmem; 97 pgcnt_t availrmem_initial; 98 99 /* 100 * These globals track availrmem changes to get a more accurate 101 * estimate of tke kernel size. Historically pp_kernel is used for 102 * kernel size and is based on availrmem. But availrmem is adjusted for 103 * locked pages in the system not just for kernel locked pages. 104 * These new counters will track the pages locked through segvn and 105 * by explicit user locking. 106 * 107 * segvn_pages_locked : This keeps track on a global basis how many pages 108 * are currently locked because of I/O. 109 * 110 * pages_locked : How many pages are locked becuase of user specified 111 * locking through mlock or plock. 112 * 113 * pages_useclaim,pages_claimed : These two variables track the 114 * cliam adjustments because of the protection changes on a segvn segment. 115 * 116 * All these globals are protected by the same lock which protects availrmem. 117 */ 118 pgcnt_t segvn_pages_locked; 119 pgcnt_t pages_locked; 120 pgcnt_t pages_useclaim; 121 pgcnt_t pages_claimed; 122 123 124 /* 125 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 126 */ 127 static kmutex_t new_freemem_lock; 128 static uint_t freemem_wait; /* someone waiting for freemem */ 129 static kcondvar_t freemem_cv; 130 131 /* 132 * The logical page free list is maintained as two lists, the 'free' 133 * and the 'cache' lists. 134 * The free list contains those pages that should be reused first. 135 * 136 * The implementation of the lists is machine dependent. 137 * page_get_freelist(), page_get_cachelist(), 138 * page_list_sub(), and page_list_add() 139 * form the interface to the machine dependent implementation. 140 * 141 * Pages with p_free set are on the cache list. 142 * Pages with p_free and p_age set are on the free list, 143 * 144 * A page may be locked while on either list. 145 */ 146 147 /* 148 * free list accounting stuff. 149 * 150 * 151 * Spread out the value for the number of pages on the 152 * page free and page cache lists. If there is just one 153 * value, then it must be under just one lock. 154 * The lock contention and cache traffic are a real bother. 155 * 156 * When we acquire and then drop a single pcf lock 157 * we can start in the middle of the array of pcf structures. 158 * If we acquire more than one pcf lock at a time, we need to 159 * start at the front to avoid deadlocking. 160 * 161 * pcf_count holds the number of pages in each pool. 162 * 163 * pcf_block is set when page_create_get_something() has asked the 164 * PSM page freelist and page cachelist routines without specifying 165 * a color and nothing came back. This is used to block anything 166 * else from moving pages from one list to the other while the 167 * lists are searched again. If a page is freeed while pcf_block is 168 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 169 * of clearning pcf_block, doing the wakeups, etc. 170 */ 171 172 #if NCPU <= 4 173 #define PAD 1 174 #define PCF_FANOUT 4 175 static uint_t pcf_mask = PCF_FANOUT - 1; 176 #else 177 #define PAD 9 178 #ifdef sun4v 179 #define PCF_FANOUT 32 180 #else 181 #define PCF_FANOUT 128 182 #endif 183 static uint_t pcf_mask = PCF_FANOUT - 1; 184 #endif 185 186 struct pcf { 187 uint_t pcf_touch; /* just to help the cache */ 188 uint_t pcf_count; /* page count */ 189 kmutex_t pcf_lock; /* protects the structure */ 190 uint_t pcf_wait; /* number of waiters */ 191 uint_t pcf_block; /* pcgs flag to page_free() */ 192 uint_t pcf_reserve; /* pages freed after pcf_block set */ 193 uint_t pcf_fill[PAD]; /* to line up on the caches */ 194 }; 195 196 static struct pcf pcf[PCF_FANOUT]; 197 #define PCF_INDEX() ((CPU->cpu_id) & (pcf_mask)) 198 199 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 200 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 201 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 202 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 203 204 #define PAGE_LOCK_MAXIMUM \ 205 ((1 << (sizeof (((page_t *)0)->p_lckcnt) * NBBY)) - 1) 206 207 #ifdef VM_STATS 208 209 /* 210 * No locks, but so what, they are only statistics. 211 */ 212 213 static struct page_tcnt { 214 int pc_free_cache; /* free's into cache list */ 215 int pc_free_dontneed; /* free's with dontneed */ 216 int pc_free_pageout; /* free's from pageout */ 217 int pc_free_free; /* free's into free list */ 218 int pc_free_pages; /* free's into large page free list */ 219 int pc_destroy_pages; /* large page destroy's */ 220 int pc_get_cache; /* get's from cache list */ 221 int pc_get_free; /* get's from free list */ 222 int pc_reclaim; /* reclaim's */ 223 int pc_abortfree; /* abort's of free pages */ 224 int pc_find_hit; /* find's that find page */ 225 int pc_find_miss; /* find's that don't find page */ 226 int pc_destroy_free; /* # of free pages destroyed */ 227 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 228 int pc_find_hashlen[PC_HASH_CNT+1]; 229 int pc_addclaim_pages; 230 int pc_subclaim_pages; 231 int pc_free_replacement_page[2]; 232 int pc_try_demote_pages[6]; 233 int pc_demote_pages[2]; 234 } pagecnt; 235 236 uint_t hashin_count; 237 uint_t hashin_not_held; 238 uint_t hashin_already; 239 240 uint_t hashout_count; 241 uint_t hashout_not_held; 242 243 uint_t page_create_count; 244 uint_t page_create_not_enough; 245 uint_t page_create_not_enough_again; 246 uint_t page_create_zero; 247 uint_t page_create_hashout; 248 uint_t page_create_page_lock_failed; 249 uint_t page_create_trylock_failed; 250 uint_t page_create_found_one; 251 uint_t page_create_hashin_failed; 252 uint_t page_create_dropped_phm; 253 254 uint_t page_create_new; 255 uint_t page_create_exists; 256 uint_t page_create_putbacks; 257 uint_t page_create_overshoot; 258 259 uint_t page_reclaim_zero; 260 uint_t page_reclaim_zero_locked; 261 262 uint_t page_rename_exists; 263 uint_t page_rename_count; 264 265 uint_t page_lookup_cnt[20]; 266 uint_t page_lookup_nowait_cnt[10]; 267 uint_t page_find_cnt; 268 uint_t page_exists_cnt; 269 uint_t page_exists_forreal_cnt; 270 uint_t page_lookup_dev_cnt; 271 uint_t get_cachelist_cnt; 272 uint_t page_create_cnt[10]; 273 uint_t alloc_pages[8]; 274 uint_t page_exphcontg[19]; 275 uint_t page_create_large_cnt[10]; 276 277 /* 278 * Collects statistics. 279 */ 280 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 281 uint_t mylen = 0; \ 282 \ 283 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 284 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 285 break; \ 286 } \ 287 if ((pp) != NULL) \ 288 pagecnt.pc_find_hit++; \ 289 else \ 290 pagecnt.pc_find_miss++; \ 291 if (mylen > PC_HASH_CNT) \ 292 mylen = PC_HASH_CNT; \ 293 pagecnt.pc_find_hashlen[mylen]++; \ 294 } 295 296 #else /* VM_STATS */ 297 298 /* 299 * Don't collect statistics 300 */ 301 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 302 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 303 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 304 break; \ 305 } \ 306 } 307 308 #endif /* VM_STATS */ 309 310 311 312 #ifdef DEBUG 313 #define MEMSEG_SEARCH_STATS 314 #endif 315 316 #ifdef MEMSEG_SEARCH_STATS 317 struct memseg_stats { 318 uint_t nsearch; 319 uint_t nlastwon; 320 uint_t nhashwon; 321 uint_t nnotfound; 322 } memseg_stats; 323 324 #define MEMSEG_STAT_INCR(v) \ 325 atomic_add_32(&memseg_stats.v, 1) 326 #else 327 #define MEMSEG_STAT_INCR(x) 328 #endif 329 330 struct memseg *memsegs; /* list of memory segments */ 331 332 333 static void page_init_mem_config(void); 334 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 335 static void page_do_hashout(page_t *); 336 337 static void page_demote_vp_pages(page_t *); 338 339 /* 340 * vm subsystem related initialization 341 */ 342 void 343 vm_init(void) 344 { 345 boolean_t callb_vm_cpr(void *, int); 346 347 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 348 page_init_mem_config(); 349 page_retire_init(); 350 } 351 352 /* 353 * This function is called at startup and when memory is added or deleted. 354 */ 355 void 356 init_pages_pp_maximum() 357 { 358 static pgcnt_t p_min; 359 static pgcnt_t pages_pp_maximum_startup; 360 static pgcnt_t avrmem_delta; 361 static int init_done; 362 static int user_set; /* true if set in /etc/system */ 363 364 if (init_done == 0) { 365 366 /* If the user specified a value, save it */ 367 if (pages_pp_maximum != 0) { 368 user_set = 1; 369 pages_pp_maximum_startup = pages_pp_maximum; 370 } 371 372 /* 373 * Setting of pages_pp_maximum is based first time 374 * on the value of availrmem just after the start-up 375 * allocations. To preserve this relationship at run 376 * time, use a delta from availrmem_initial. 377 */ 378 ASSERT(availrmem_initial >= availrmem); 379 avrmem_delta = availrmem_initial - availrmem; 380 381 /* The allowable floor of pages_pp_maximum */ 382 p_min = tune.t_minarmem + 100; 383 384 /* Make sure we don't come through here again. */ 385 init_done = 1; 386 } 387 /* 388 * Determine pages_pp_maximum, the number of currently available 389 * pages (availrmem) that can't be `locked'. If not set by 390 * the user, we set it to 4% of the currently available memory 391 * plus 4MB. 392 * But we also insist that it be greater than tune.t_minarmem; 393 * otherwise a process could lock down a lot of memory, get swapped 394 * out, and never have enough to get swapped back in. 395 */ 396 if (user_set) 397 pages_pp_maximum = pages_pp_maximum_startup; 398 else 399 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 400 + btop(4 * 1024 * 1024); 401 402 if (pages_pp_maximum <= p_min) { 403 pages_pp_maximum = p_min; 404 } 405 } 406 407 void 408 set_max_page_get(pgcnt_t target_total_pages) 409 { 410 max_page_get = target_total_pages / 2; 411 } 412 413 static pgcnt_t pending_delete; 414 415 /*ARGSUSED*/ 416 static void 417 page_mem_config_post_add( 418 void *arg, 419 pgcnt_t delta_pages) 420 { 421 set_max_page_get(total_pages - pending_delete); 422 init_pages_pp_maximum(); 423 } 424 425 /*ARGSUSED*/ 426 static int 427 page_mem_config_pre_del( 428 void *arg, 429 pgcnt_t delta_pages) 430 { 431 pgcnt_t nv; 432 433 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 434 set_max_page_get(total_pages - nv); 435 return (0); 436 } 437 438 /*ARGSUSED*/ 439 static void 440 page_mem_config_post_del( 441 void *arg, 442 pgcnt_t delta_pages, 443 int cancelled) 444 { 445 pgcnt_t nv; 446 447 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 448 set_max_page_get(total_pages - nv); 449 if (!cancelled) 450 init_pages_pp_maximum(); 451 } 452 453 static kphysm_setup_vector_t page_mem_config_vec = { 454 KPHYSM_SETUP_VECTOR_VERSION, 455 page_mem_config_post_add, 456 page_mem_config_pre_del, 457 page_mem_config_post_del, 458 }; 459 460 static void 461 page_init_mem_config(void) 462 { 463 int ret; 464 465 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 466 ASSERT(ret == 0); 467 } 468 469 /* 470 * Evenly spread out the PCF counters for large free pages 471 */ 472 static void 473 page_free_large_ctr(pgcnt_t npages) 474 { 475 static struct pcf *p = pcf; 476 pgcnt_t lump; 477 478 freemem += npages; 479 480 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 481 482 while (npages > 0) { 483 484 ASSERT(!p->pcf_block); 485 486 if (lump < npages) { 487 p->pcf_count += (uint_t)lump; 488 npages -= lump; 489 } else { 490 p->pcf_count += (uint_t)npages; 491 npages = 0; 492 } 493 494 ASSERT(!p->pcf_wait); 495 496 if (++p > &pcf[PCF_FANOUT - 1]) 497 p = pcf; 498 } 499 500 ASSERT(npages == 0); 501 } 502 503 /* 504 * Add a physical chunk of memory to the system freee lists during startup. 505 * Platform specific startup() allocates the memory for the page structs. 506 * 507 * num - number of page structures 508 * base - page number (pfn) to be associated with the first page. 509 * 510 * Since we are doing this during startup (ie. single threaded), we will 511 * use shortcut routines to avoid any locking overhead while putting all 512 * these pages on the freelists. 513 * 514 * NOTE: Any changes performed to page_free(), must also be performed to 515 * add_physmem() since this is how we initialize all page_t's at 516 * boot time. 517 */ 518 void 519 add_physmem( 520 page_t *pp, 521 pgcnt_t num, 522 pfn_t pnum) 523 { 524 page_t *root = NULL; 525 uint_t szc = page_num_pagesizes() - 1; 526 pgcnt_t large = page_get_pagecnt(szc); 527 pgcnt_t cnt = 0; 528 529 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 530 "add_physmem:pp %p num %lu", pp, num); 531 532 /* 533 * Arbitrarily limit the max page_get request 534 * to 1/2 of the page structs we have. 535 */ 536 total_pages += num; 537 set_max_page_get(total_pages); 538 539 PLCNT_MODIFY_MAX(pnum, (long)num); 540 541 /* 542 * The physical space for the pages array 543 * representing ram pages has already been 544 * allocated. Here we initialize each lock 545 * in the page structure, and put each on 546 * the free list 547 */ 548 for (; num; pp++, pnum++, num--) { 549 550 /* 551 * this needs to fill in the page number 552 * and do any other arch specific initialization 553 */ 554 add_physmem_cb(pp, pnum); 555 556 /* 557 * Initialize the page lock as unlocked, since nobody 558 * can see or access this page yet. 559 */ 560 pp->p_selock = 0; 561 562 /* 563 * Initialize IO lock 564 */ 565 page_iolock_init(pp); 566 567 /* 568 * initialize other fields in the page_t 569 */ 570 PP_SETFREE(pp); 571 page_clr_all_props(pp); 572 PP_SETAGED(pp); 573 pp->p_offset = (u_offset_t)-1; 574 pp->p_next = pp; 575 pp->p_prev = pp; 576 577 /* 578 * Simple case: System doesn't support large pages. 579 */ 580 if (szc == 0) { 581 pp->p_szc = 0; 582 page_free_at_startup(pp); 583 continue; 584 } 585 586 /* 587 * Handle unaligned pages, we collect them up onto 588 * the root page until we have a full large page. 589 */ 590 if (!IS_P2ALIGNED(pnum, large)) { 591 592 /* 593 * If not in a large page, 594 * just free as small page. 595 */ 596 if (root == NULL) { 597 pp->p_szc = 0; 598 page_free_at_startup(pp); 599 continue; 600 } 601 602 /* 603 * Link a constituent page into the large page. 604 */ 605 pp->p_szc = szc; 606 page_list_concat(&root, &pp); 607 608 /* 609 * When large page is fully formed, free it. 610 */ 611 if (++cnt == large) { 612 page_free_large_ctr(cnt); 613 page_list_add_pages(root, PG_LIST_ISINIT); 614 root = NULL; 615 cnt = 0; 616 } 617 continue; 618 } 619 620 /* 621 * At this point we have a page number which 622 * is aligned. We assert that we aren't already 623 * in a different large page. 624 */ 625 ASSERT(IS_P2ALIGNED(pnum, large)); 626 ASSERT(root == NULL && cnt == 0); 627 628 /* 629 * If insufficient number of pages left to form 630 * a large page, just free the small page. 631 */ 632 if (num < large) { 633 pp->p_szc = 0; 634 page_free_at_startup(pp); 635 continue; 636 } 637 638 /* 639 * Otherwise start a new large page. 640 */ 641 pp->p_szc = szc; 642 cnt++; 643 root = pp; 644 } 645 ASSERT(root == NULL && cnt == 0); 646 } 647 648 /* 649 * Find a page representing the specified [vp, offset]. 650 * If we find the page but it is intransit coming in, 651 * it will have an "exclusive" lock and we wait for 652 * the i/o to complete. A page found on the free list 653 * is always reclaimed and then locked. On success, the page 654 * is locked, its data is valid and it isn't on the free 655 * list, while a NULL is returned if the page doesn't exist. 656 */ 657 page_t * 658 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 659 { 660 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 661 } 662 663 /* 664 * Find a page representing the specified [vp, offset]. 665 * We either return the one we found or, if passed in, 666 * create one with identity of [vp, offset] of the 667 * pre-allocated page. If we find exsisting page but it is 668 * intransit coming in, it will have an "exclusive" lock 669 * and we wait for the i/o to complete. A page found on 670 * the free list is always reclaimed and then locked. 671 * On success, the page is locked, its data is valid and 672 * it isn't on the free list, while a NULL is returned 673 * if the page doesn't exist and newpp is NULL; 674 */ 675 page_t * 676 page_lookup_create( 677 vnode_t *vp, 678 u_offset_t off, 679 se_t se, 680 page_t *newpp, 681 spgcnt_t *nrelocp, 682 int flags) 683 { 684 page_t *pp; 685 kmutex_t *phm; 686 ulong_t index; 687 uint_t hash_locked; 688 uint_t es; 689 690 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 691 VM_STAT_ADD(page_lookup_cnt[0]); 692 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 693 694 /* 695 * Acquire the appropriate page hash lock since 696 * we have to search the hash list. Pages that 697 * hash to this list can't change identity while 698 * this lock is held. 699 */ 700 hash_locked = 0; 701 index = PAGE_HASH_FUNC(vp, off); 702 phm = NULL; 703 top: 704 PAGE_HASH_SEARCH(index, pp, vp, off); 705 if (pp != NULL) { 706 VM_STAT_ADD(page_lookup_cnt[1]); 707 es = (newpp != NULL) ? 1 : 0; 708 es |= flags; 709 if (!hash_locked) { 710 VM_STAT_ADD(page_lookup_cnt[2]); 711 if (!page_try_reclaim_lock(pp, se, es)) { 712 /* 713 * On a miss, acquire the phm. Then 714 * next time, page_lock() will be called, 715 * causing a wait if the page is busy. 716 * just looping with page_trylock() would 717 * get pretty boring. 718 */ 719 VM_STAT_ADD(page_lookup_cnt[3]); 720 phm = PAGE_HASH_MUTEX(index); 721 mutex_enter(phm); 722 hash_locked = 1; 723 goto top; 724 } 725 } else { 726 VM_STAT_ADD(page_lookup_cnt[4]); 727 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 728 VM_STAT_ADD(page_lookup_cnt[5]); 729 goto top; 730 } 731 } 732 733 /* 734 * Since `pp' is locked it can not change identity now. 735 * Reconfirm we locked the correct page. 736 * 737 * Both the p_vnode and p_offset *must* be cast volatile 738 * to force a reload of their values: The PAGE_HASH_SEARCH 739 * macro will have stuffed p_vnode and p_offset into 740 * registers before calling page_trylock(); another thread, 741 * actually holding the hash lock, could have changed the 742 * page's identity in memory, but our registers would not 743 * be changed, fooling the reconfirmation. If the hash 744 * lock was held during the search, the casting would 745 * not be needed. 746 */ 747 VM_STAT_ADD(page_lookup_cnt[6]); 748 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 749 ((volatile u_offset_t)(pp->p_offset) != off)) { 750 VM_STAT_ADD(page_lookup_cnt[7]); 751 if (hash_locked) { 752 panic("page_lookup_create: lost page %p", 753 (void *)pp); 754 /*NOTREACHED*/ 755 } 756 page_unlock(pp); 757 phm = PAGE_HASH_MUTEX(index); 758 mutex_enter(phm); 759 hash_locked = 1; 760 goto top; 761 } 762 763 /* 764 * If page_trylock() was called, then pp may still be on 765 * the cachelist (can't be on the free list, it would not 766 * have been found in the search). If it is on the 767 * cachelist it must be pulled now. To pull the page from 768 * the cachelist, it must be exclusively locked. 769 * 770 * The other big difference between page_trylock() and 771 * page_lock(), is that page_lock() will pull the 772 * page from whatever free list (the cache list in this 773 * case) the page is on. If page_trylock() was used 774 * above, then we have to do the reclaim ourselves. 775 */ 776 if ((!hash_locked) && (PP_ISFREE(pp))) { 777 ASSERT(PP_ISAGED(pp) == 0); 778 VM_STAT_ADD(page_lookup_cnt[8]); 779 780 /* 781 * page_relcaim will insure that we 782 * have this page exclusively 783 */ 784 785 if (!page_reclaim(pp, NULL)) { 786 /* 787 * Page_reclaim dropped whatever lock 788 * we held. 789 */ 790 VM_STAT_ADD(page_lookup_cnt[9]); 791 phm = PAGE_HASH_MUTEX(index); 792 mutex_enter(phm); 793 hash_locked = 1; 794 goto top; 795 } else if (se == SE_SHARED && newpp == NULL) { 796 VM_STAT_ADD(page_lookup_cnt[10]); 797 page_downgrade(pp); 798 } 799 } 800 801 if (hash_locked) { 802 mutex_exit(phm); 803 } 804 805 if (newpp != NULL && pp->p_szc < newpp->p_szc && 806 PAGE_EXCL(pp) && nrelocp != NULL) { 807 ASSERT(nrelocp != NULL); 808 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 809 NULL); 810 if (*nrelocp > 0) { 811 VM_STAT_COND_ADD(*nrelocp == 1, 812 page_lookup_cnt[11]); 813 VM_STAT_COND_ADD(*nrelocp > 1, 814 page_lookup_cnt[12]); 815 pp = newpp; 816 se = SE_EXCL; 817 } else { 818 if (se == SE_SHARED) { 819 page_downgrade(pp); 820 } 821 VM_STAT_ADD(page_lookup_cnt[13]); 822 } 823 } else if (newpp != NULL && nrelocp != NULL) { 824 if (PAGE_EXCL(pp) && se == SE_SHARED) { 825 page_downgrade(pp); 826 } 827 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 828 page_lookup_cnt[14]); 829 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 830 page_lookup_cnt[15]); 831 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 832 page_lookup_cnt[16]); 833 } else if (newpp != NULL && PAGE_EXCL(pp)) { 834 se = SE_EXCL; 835 } 836 } else if (!hash_locked) { 837 VM_STAT_ADD(page_lookup_cnt[17]); 838 phm = PAGE_HASH_MUTEX(index); 839 mutex_enter(phm); 840 hash_locked = 1; 841 goto top; 842 } else if (newpp != NULL) { 843 /* 844 * If we have a preallocated page then 845 * insert it now and basically behave like 846 * page_create. 847 */ 848 VM_STAT_ADD(page_lookup_cnt[18]); 849 /* 850 * Since we hold the page hash mutex and 851 * just searched for this page, page_hashin 852 * had better not fail. If it does, that 853 * means some thread did not follow the 854 * page hash mutex rules. Panic now and 855 * get it over with. As usual, go down 856 * holding all the locks. 857 */ 858 ASSERT(MUTEX_HELD(phm)); 859 if (!page_hashin(newpp, vp, off, phm)) { 860 ASSERT(MUTEX_HELD(phm)); 861 panic("page_lookup_create: hashin failed %p %p %llx %p", 862 (void *)newpp, (void *)vp, off, (void *)phm); 863 /*NOTREACHED*/ 864 } 865 ASSERT(MUTEX_HELD(phm)); 866 mutex_exit(phm); 867 phm = NULL; 868 page_set_props(newpp, P_REF); 869 page_io_lock(newpp); 870 pp = newpp; 871 se = SE_EXCL; 872 } else { 873 VM_STAT_ADD(page_lookup_cnt[19]); 874 mutex_exit(phm); 875 } 876 877 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 878 879 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 880 881 return (pp); 882 } 883 884 /* 885 * Search the hash list for the page representing the 886 * specified [vp, offset] and return it locked. Skip 887 * free pages and pages that cannot be locked as requested. 888 * Used while attempting to kluster pages. 889 */ 890 page_t * 891 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 892 { 893 page_t *pp; 894 kmutex_t *phm; 895 ulong_t index; 896 uint_t locked; 897 898 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 899 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 900 901 index = PAGE_HASH_FUNC(vp, off); 902 PAGE_HASH_SEARCH(index, pp, vp, off); 903 locked = 0; 904 if (pp == NULL) { 905 top: 906 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 907 locked = 1; 908 phm = PAGE_HASH_MUTEX(index); 909 mutex_enter(phm); 910 PAGE_HASH_SEARCH(index, pp, vp, off); 911 } 912 913 if (pp == NULL || PP_ISFREE(pp)) { 914 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 915 pp = NULL; 916 } else { 917 if (!page_trylock(pp, se)) { 918 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 919 pp = NULL; 920 } else { 921 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 922 /* 923 * See the comment in page_lookup() 924 */ 925 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 926 ((u_offset_t)(pp->p_offset) != off)) { 927 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 928 if (locked) { 929 panic("page_lookup_nowait %p", 930 (void *)pp); 931 /*NOTREACHED*/ 932 } 933 page_unlock(pp); 934 goto top; 935 } 936 if (PP_ISFREE(pp)) { 937 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 938 page_unlock(pp); 939 pp = NULL; 940 } 941 } 942 } 943 if (locked) { 944 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 945 mutex_exit(phm); 946 } 947 948 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 949 950 return (pp); 951 } 952 953 /* 954 * Search the hash list for a page with the specified [vp, off] 955 * that is known to exist and is already locked. This routine 956 * is typically used by segment SOFTUNLOCK routines. 957 */ 958 page_t * 959 page_find(vnode_t *vp, u_offset_t off) 960 { 961 page_t *pp; 962 kmutex_t *phm; 963 ulong_t index; 964 965 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 966 VM_STAT_ADD(page_find_cnt); 967 968 index = PAGE_HASH_FUNC(vp, off); 969 phm = PAGE_HASH_MUTEX(index); 970 971 mutex_enter(phm); 972 PAGE_HASH_SEARCH(index, pp, vp, off); 973 mutex_exit(phm); 974 975 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 976 return (pp); 977 } 978 979 /* 980 * Determine whether a page with the specified [vp, off] 981 * currently exists in the system. Obviously this should 982 * only be considered as a hint since nothing prevents the 983 * page from disappearing or appearing immediately after 984 * the return from this routine. Subsequently, we don't 985 * even bother to lock the list. 986 */ 987 page_t * 988 page_exists(vnode_t *vp, u_offset_t off) 989 { 990 page_t *pp; 991 ulong_t index; 992 993 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 994 VM_STAT_ADD(page_exists_cnt); 995 996 index = PAGE_HASH_FUNC(vp, off); 997 PAGE_HASH_SEARCH(index, pp, vp, off); 998 999 return (pp); 1000 } 1001 1002 /* 1003 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1004 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1005 * with these pages locked SHARED. If necessary reclaim pages from 1006 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1007 * 1008 * If we fail to lock pages still return 1 if pages exist and contiguous. 1009 * But in this case return value is just a hint. ppa array won't be filled. 1010 * Caller should initialize ppa[0] as NULL to distinguish return value. 1011 * 1012 * Returns 0 if pages don't exist or not physically contiguous. 1013 * 1014 * This routine doesn't work for anonymous(swapfs) pages. 1015 */ 1016 int 1017 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1018 { 1019 pgcnt_t pages; 1020 pfn_t pfn; 1021 page_t *rootpp; 1022 pgcnt_t i; 1023 pgcnt_t j; 1024 u_offset_t save_off = off; 1025 ulong_t index; 1026 kmutex_t *phm; 1027 page_t *pp; 1028 uint_t pszc; 1029 int loopcnt = 0; 1030 1031 ASSERT(szc != 0); 1032 ASSERT(vp != NULL); 1033 ASSERT(!IS_SWAPFSVP(vp)); 1034 ASSERT(vp != &kvp); 1035 1036 again: 1037 if (++loopcnt > 3) { 1038 VM_STAT_ADD(page_exphcontg[0]); 1039 return (0); 1040 } 1041 1042 index = PAGE_HASH_FUNC(vp, off); 1043 phm = PAGE_HASH_MUTEX(index); 1044 1045 mutex_enter(phm); 1046 PAGE_HASH_SEARCH(index, pp, vp, off); 1047 mutex_exit(phm); 1048 1049 VM_STAT_ADD(page_exphcontg[1]); 1050 1051 if (pp == NULL) { 1052 VM_STAT_ADD(page_exphcontg[2]); 1053 return (0); 1054 } 1055 1056 pages = page_get_pagecnt(szc); 1057 rootpp = pp; 1058 pfn = rootpp->p_pagenum; 1059 1060 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1061 VM_STAT_ADD(page_exphcontg[3]); 1062 if (!page_trylock(pp, SE_SHARED)) { 1063 VM_STAT_ADD(page_exphcontg[4]); 1064 return (1); 1065 } 1066 if (pp->p_szc != pszc || pp->p_vnode != vp || 1067 pp->p_offset != off) { 1068 VM_STAT_ADD(page_exphcontg[5]); 1069 page_unlock(pp); 1070 off = save_off; 1071 goto again; 1072 } 1073 /* 1074 * szc was non zero and vnode and offset matched after we 1075 * locked the page it means it can't become free on us. 1076 */ 1077 ASSERT(!PP_ISFREE(pp)); 1078 if (!IS_P2ALIGNED(pfn, pages)) { 1079 page_unlock(pp); 1080 return (0); 1081 } 1082 ppa[0] = pp; 1083 pp++; 1084 off += PAGESIZE; 1085 pfn++; 1086 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1087 if (!page_trylock(pp, SE_SHARED)) { 1088 VM_STAT_ADD(page_exphcontg[6]); 1089 pp--; 1090 while (i-- > 0) { 1091 page_unlock(pp); 1092 pp--; 1093 } 1094 ppa[0] = NULL; 1095 return (1); 1096 } 1097 if (pp->p_szc != pszc) { 1098 VM_STAT_ADD(page_exphcontg[7]); 1099 page_unlock(pp); 1100 pp--; 1101 while (i-- > 0) { 1102 page_unlock(pp); 1103 pp--; 1104 } 1105 ppa[0] = NULL; 1106 off = save_off; 1107 goto again; 1108 } 1109 /* 1110 * szc the same as for previous already locked pages 1111 * with right identity. Since this page had correct 1112 * szc after we locked it can't get freed or destroyed 1113 * and therefore must have the expected identity. 1114 */ 1115 ASSERT(!PP_ISFREE(pp)); 1116 if (pp->p_vnode != vp || 1117 pp->p_offset != off) { 1118 panic("page_exists_physcontig: " 1119 "large page identity doesn't match"); 1120 } 1121 ppa[i] = pp; 1122 ASSERT(pp->p_pagenum == pfn); 1123 } 1124 VM_STAT_ADD(page_exphcontg[8]); 1125 ppa[pages] = NULL; 1126 return (1); 1127 } else if (pszc >= szc) { 1128 VM_STAT_ADD(page_exphcontg[9]); 1129 if (!IS_P2ALIGNED(pfn, pages)) { 1130 return (0); 1131 } 1132 return (1); 1133 } 1134 1135 if (!IS_P2ALIGNED(pfn, pages)) { 1136 VM_STAT_ADD(page_exphcontg[10]); 1137 return (0); 1138 } 1139 1140 if (page_numtomemseg_nolock(pfn) != 1141 page_numtomemseg_nolock(pfn + pages - 1)) { 1142 VM_STAT_ADD(page_exphcontg[11]); 1143 return (0); 1144 } 1145 1146 /* 1147 * We loop up 4 times across pages to promote page size. 1148 * We're extra cautious to promote page size atomically with respect 1149 * to everybody else. But we can probably optimize into 1 loop if 1150 * this becomes an issue. 1151 */ 1152 1153 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1154 ASSERT(pp->p_pagenum == pfn); 1155 if (!page_trylock(pp, SE_EXCL)) { 1156 VM_STAT_ADD(page_exphcontg[12]); 1157 break; 1158 } 1159 if (pp->p_vnode != vp || 1160 pp->p_offset != off) { 1161 VM_STAT_ADD(page_exphcontg[13]); 1162 page_unlock(pp); 1163 break; 1164 } 1165 if (pp->p_szc >= szc) { 1166 ASSERT(i == 0); 1167 page_unlock(pp); 1168 off = save_off; 1169 goto again; 1170 } 1171 } 1172 1173 if (i != pages) { 1174 VM_STAT_ADD(page_exphcontg[14]); 1175 --pp; 1176 while (i-- > 0) { 1177 page_unlock(pp); 1178 --pp; 1179 } 1180 return (0); 1181 } 1182 1183 pp = rootpp; 1184 for (i = 0; i < pages; i++, pp++) { 1185 if (PP_ISFREE(pp)) { 1186 VM_STAT_ADD(page_exphcontg[15]); 1187 ASSERT(!PP_ISAGED(pp)); 1188 ASSERT(pp->p_szc == 0); 1189 if (!page_reclaim(pp, NULL)) { 1190 break; 1191 } 1192 } else { 1193 ASSERT(pp->p_szc < szc); 1194 VM_STAT_ADD(page_exphcontg[16]); 1195 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1196 } 1197 } 1198 if (i < pages) { 1199 VM_STAT_ADD(page_exphcontg[17]); 1200 /* 1201 * page_reclaim failed because we were out of memory. 1202 * drop the rest of the locks and return because this page 1203 * must be already reallocated anyway. 1204 */ 1205 pp = rootpp; 1206 for (j = 0; j < pages; j++, pp++) { 1207 if (j != i) { 1208 page_unlock(pp); 1209 } 1210 } 1211 return (0); 1212 } 1213 1214 off = save_off; 1215 pp = rootpp; 1216 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1217 ASSERT(PAGE_EXCL(pp)); 1218 ASSERT(!PP_ISFREE(pp)); 1219 ASSERT(!hat_page_is_mapped(pp)); 1220 ASSERT(pp->p_vnode == vp); 1221 ASSERT(pp->p_offset == off); 1222 pp->p_szc = szc; 1223 } 1224 pp = rootpp; 1225 for (i = 0; i < pages; i++, pp++) { 1226 if (ppa == NULL) { 1227 page_unlock(pp); 1228 } else { 1229 ppa[i] = pp; 1230 page_downgrade(ppa[i]); 1231 } 1232 } 1233 if (ppa != NULL) { 1234 ppa[pages] = NULL; 1235 } 1236 VM_STAT_ADD(page_exphcontg[18]); 1237 ASSERT(vp->v_pages != NULL); 1238 return (1); 1239 } 1240 1241 /* 1242 * Determine whether a page with the specified [vp, off] 1243 * currently exists in the system and if so return its 1244 * size code. Obviously this should only be considered as 1245 * a hint since nothing prevents the page from disappearing 1246 * or appearing immediately after the return from this routine. 1247 */ 1248 int 1249 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1250 { 1251 page_t *pp; 1252 kmutex_t *phm; 1253 ulong_t index; 1254 int rc = 0; 1255 1256 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1257 ASSERT(szc != NULL); 1258 VM_STAT_ADD(page_exists_forreal_cnt); 1259 1260 index = PAGE_HASH_FUNC(vp, off); 1261 phm = PAGE_HASH_MUTEX(index); 1262 1263 mutex_enter(phm); 1264 PAGE_HASH_SEARCH(index, pp, vp, off); 1265 if (pp != NULL) { 1266 *szc = pp->p_szc; 1267 rc = 1; 1268 } 1269 mutex_exit(phm); 1270 return (rc); 1271 } 1272 1273 /* wakeup threads waiting for pages in page_create_get_something() */ 1274 void 1275 wakeup_pcgs(void) 1276 { 1277 if (!CV_HAS_WAITERS(&pcgs_cv)) 1278 return; 1279 cv_broadcast(&pcgs_cv); 1280 } 1281 1282 /* 1283 * 'freemem' is used all over the kernel as an indication of how many 1284 * pages are free (either on the cache list or on the free page list) 1285 * in the system. In very few places is a really accurate 'freemem' 1286 * needed. To avoid contention of the lock protecting a the 1287 * single freemem, it was spread out into NCPU buckets. Set_freemem 1288 * sets freemem to the total of all NCPU buckets. It is called from 1289 * clock() on each TICK. 1290 */ 1291 void 1292 set_freemem() 1293 { 1294 struct pcf *p; 1295 ulong_t t; 1296 uint_t i; 1297 1298 t = 0; 1299 p = pcf; 1300 for (i = 0; i < PCF_FANOUT; i++) { 1301 t += p->pcf_count; 1302 p++; 1303 } 1304 freemem = t; 1305 1306 /* 1307 * Don't worry about grabbing mutex. It's not that 1308 * critical if we miss a tick or two. This is 1309 * where we wakeup possible delayers in 1310 * page_create_get_something(). 1311 */ 1312 wakeup_pcgs(); 1313 } 1314 1315 ulong_t 1316 get_freemem() 1317 { 1318 struct pcf *p; 1319 ulong_t t; 1320 uint_t i; 1321 1322 t = 0; 1323 p = pcf; 1324 for (i = 0; i < PCF_FANOUT; i++) { 1325 t += p->pcf_count; 1326 p++; 1327 } 1328 /* 1329 * We just calculated it, might as well set it. 1330 */ 1331 freemem = t; 1332 return (t); 1333 } 1334 1335 /* 1336 * Acquire all of the page cache & free (pcf) locks. 1337 */ 1338 void 1339 pcf_acquire_all() 1340 { 1341 struct pcf *p; 1342 uint_t i; 1343 1344 p = pcf; 1345 for (i = 0; i < PCF_FANOUT; i++) { 1346 p->pcf_touch = 1; 1347 mutex_enter(&p->pcf_lock); 1348 p++; 1349 } 1350 } 1351 1352 /* 1353 * Release all the pcf_locks. 1354 */ 1355 void 1356 pcf_release_all() 1357 { 1358 struct pcf *p; 1359 uint_t i; 1360 1361 p = pcf; 1362 for (i = 0; i < PCF_FANOUT; i++) { 1363 mutex_exit(&p->pcf_lock); 1364 p++; 1365 } 1366 } 1367 1368 /* 1369 * Inform the VM system that we need some pages freed up. 1370 * Calls must be symmetric, e.g.: 1371 * 1372 * page_needfree(100); 1373 * wait a bit; 1374 * page_needfree(-100); 1375 */ 1376 void 1377 page_needfree(spgcnt_t npages) 1378 { 1379 mutex_enter(&new_freemem_lock); 1380 needfree += npages; 1381 mutex_exit(&new_freemem_lock); 1382 } 1383 1384 /* 1385 * Throttle for page_create(): try to prevent freemem from dropping 1386 * below throttlefree. We can't provide a 100% guarantee because 1387 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1388 * nibble away at the freelist. However, we can block all PG_WAIT 1389 * allocations until memory becomes available. The motivation is 1390 * that several things can fall apart when there's no free memory: 1391 * 1392 * (1) If pageout() needs memory to push a page, the system deadlocks. 1393 * 1394 * (2) By (broken) specification, timeout(9F) can neither fail nor 1395 * block, so it has no choice but to panic the system if it 1396 * cannot allocate a callout structure. 1397 * 1398 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1399 * it panics if it cannot allocate a callback structure. 1400 * 1401 * (4) Untold numbers of third-party drivers have not yet been hardened 1402 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1403 * success and panic the system with a data fault on failure. 1404 * (The long-term solution to this particular problem is to ship 1405 * hostile fault-injecting DEBUG kernels with the DDK.) 1406 * 1407 * It is theoretically impossible to guarantee success of non-blocking 1408 * allocations, but in practice, this throttle is very hard to break. 1409 */ 1410 static int 1411 page_create_throttle(pgcnt_t npages, int flags) 1412 { 1413 ulong_t fm; 1414 uint_t i; 1415 pgcnt_t tf; /* effective value of throttlefree */ 1416 1417 /* 1418 * Never deny pages when: 1419 * - it's a thread that cannot block [NOMEMWAIT()] 1420 * - the allocation cannot block and must not fail 1421 * - the allocation cannot block and is pageout dispensated 1422 */ 1423 if (NOMEMWAIT() || 1424 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1425 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1426 return (1); 1427 1428 /* 1429 * If the allocation can't block, we look favorably upon it 1430 * unless we're below pageout_reserve. In that case we fail 1431 * the allocation because we want to make sure there are a few 1432 * pages available for pageout. 1433 */ 1434 if ((flags & PG_WAIT) == 0) 1435 return (freemem >= npages + pageout_reserve); 1436 1437 /* Calculate the effective throttlefree value */ 1438 tf = throttlefree - 1439 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1440 1441 cv_signal(&proc_pageout->p_cv); 1442 1443 while (freemem < npages + tf) { 1444 pcf_acquire_all(); 1445 mutex_enter(&new_freemem_lock); 1446 fm = 0; 1447 for (i = 0; i < PCF_FANOUT; i++) { 1448 fm += pcf[i].pcf_count; 1449 pcf[i].pcf_wait++; 1450 mutex_exit(&pcf[i].pcf_lock); 1451 } 1452 freemem = fm; 1453 needfree += npages; 1454 freemem_wait++; 1455 cv_wait(&freemem_cv, &new_freemem_lock); 1456 freemem_wait--; 1457 needfree -= npages; 1458 mutex_exit(&new_freemem_lock); 1459 } 1460 return (1); 1461 } 1462 1463 /* 1464 * page_create_wait() is called to either coalecse pages from the 1465 * different pcf buckets or to wait because there simply are not 1466 * enough pages to satisfy the caller's request. 1467 * 1468 * Sadly, this is called from platform/vm/vm_machdep.c 1469 */ 1470 int 1471 page_create_wait(size_t npages, uint_t flags) 1472 { 1473 pgcnt_t total; 1474 uint_t i; 1475 struct pcf *p; 1476 1477 /* 1478 * Wait until there are enough free pages to satisfy our 1479 * entire request. 1480 * We set needfree += npages before prodding pageout, to make sure 1481 * it does real work when npages > lotsfree > freemem. 1482 */ 1483 VM_STAT_ADD(page_create_not_enough); 1484 1485 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1486 checkagain: 1487 if ((flags & PG_NORELOC) && 1488 kcage_freemem < kcage_throttlefree + npages) 1489 (void) kcage_create_throttle(npages, flags); 1490 1491 if (freemem < npages + throttlefree) 1492 if (!page_create_throttle(npages, flags)) 1493 return (0); 1494 1495 /* 1496 * Since page_create_va() looked at every 1497 * bucket, assume we are going to have to wait. 1498 * Get all of the pcf locks. 1499 */ 1500 total = 0; 1501 p = pcf; 1502 for (i = 0; i < PCF_FANOUT; i++) { 1503 p->pcf_touch = 1; 1504 mutex_enter(&p->pcf_lock); 1505 total += p->pcf_count; 1506 if (total >= npages) { 1507 /* 1508 * Wow! There are enough pages laying around 1509 * to satisfy the request. Do the accounting, 1510 * drop the locks we acquired, and go back. 1511 * 1512 * freemem is not protected by any lock. So, 1513 * we cannot have any assertion containing 1514 * freemem. 1515 */ 1516 freemem -= npages; 1517 1518 while (p >= pcf) { 1519 if (p->pcf_count <= npages) { 1520 npages -= p->pcf_count; 1521 p->pcf_count = 0; 1522 } else { 1523 p->pcf_count -= (uint_t)npages; 1524 npages = 0; 1525 } 1526 mutex_exit(&p->pcf_lock); 1527 p--; 1528 } 1529 ASSERT(npages == 0); 1530 return (1); 1531 } 1532 p++; 1533 } 1534 1535 /* 1536 * All of the pcf locks are held, there are not enough pages 1537 * to satisfy the request (npages < total). 1538 * Be sure to acquire the new_freemem_lock before dropping 1539 * the pcf locks. This prevents dropping wakeups in page_free(). 1540 * The order is always pcf_lock then new_freemem_lock. 1541 * 1542 * Since we hold all the pcf locks, it is a good time to set freemem. 1543 * 1544 * If the caller does not want to wait, return now. 1545 * Else turn the pageout daemon loose to find something 1546 * and wait till it does. 1547 * 1548 */ 1549 freemem = total; 1550 1551 if ((flags & PG_WAIT) == 0) { 1552 pcf_release_all(); 1553 1554 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1555 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1556 return (0); 1557 } 1558 1559 ASSERT(proc_pageout != NULL); 1560 cv_signal(&proc_pageout->p_cv); 1561 1562 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1563 "page_create_sleep_start: freemem %ld needfree %ld", 1564 freemem, needfree); 1565 1566 /* 1567 * We are going to wait. 1568 * We currently hold all of the pcf_locks, 1569 * get the new_freemem_lock (it protects freemem_wait), 1570 * before dropping the pcf_locks. 1571 */ 1572 mutex_enter(&new_freemem_lock); 1573 1574 p = pcf; 1575 for (i = 0; i < PCF_FANOUT; i++) { 1576 p->pcf_wait++; 1577 mutex_exit(&p->pcf_lock); 1578 p++; 1579 } 1580 1581 needfree += npages; 1582 freemem_wait++; 1583 1584 cv_wait(&freemem_cv, &new_freemem_lock); 1585 1586 freemem_wait--; 1587 needfree -= npages; 1588 1589 mutex_exit(&new_freemem_lock); 1590 1591 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1592 "page_create_sleep_end: freemem %ld needfree %ld", 1593 freemem, needfree); 1594 1595 VM_STAT_ADD(page_create_not_enough_again); 1596 goto checkagain; 1597 } 1598 1599 /* 1600 * A routine to do the opposite of page_create_wait(). 1601 */ 1602 void 1603 page_create_putback(spgcnt_t npages) 1604 { 1605 struct pcf *p; 1606 pgcnt_t lump; 1607 uint_t *which; 1608 1609 /* 1610 * When a contiguous lump is broken up, we have to 1611 * deal with lots of pages (min 64) so lets spread 1612 * the wealth around. 1613 */ 1614 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 1615 freemem += npages; 1616 1617 for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) { 1618 which = &p->pcf_count; 1619 1620 mutex_enter(&p->pcf_lock); 1621 1622 if (p->pcf_block) { 1623 which = &p->pcf_reserve; 1624 } 1625 1626 if (lump < npages) { 1627 *which += (uint_t)lump; 1628 npages -= lump; 1629 } else { 1630 *which += (uint_t)npages; 1631 npages = 0; 1632 } 1633 1634 if (p->pcf_wait) { 1635 mutex_enter(&new_freemem_lock); 1636 /* 1637 * Check to see if some other thread 1638 * is actually waiting. Another bucket 1639 * may have woken it up by now. If there 1640 * are no waiters, then set our pcf_wait 1641 * count to zero to avoid coming in here 1642 * next time. 1643 */ 1644 if (freemem_wait) { 1645 if (npages > 1) { 1646 cv_broadcast(&freemem_cv); 1647 } else { 1648 cv_signal(&freemem_cv); 1649 } 1650 p->pcf_wait--; 1651 } else { 1652 p->pcf_wait = 0; 1653 } 1654 mutex_exit(&new_freemem_lock); 1655 } 1656 mutex_exit(&p->pcf_lock); 1657 } 1658 ASSERT(npages == 0); 1659 } 1660 1661 /* 1662 * A helper routine for page_create_get_something. 1663 * The indenting got to deep down there. 1664 * Unblock the pcf counters. Any pages freed after 1665 * pcf_block got set are moved to pcf_count and 1666 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1667 */ 1668 static void 1669 pcgs_unblock(void) 1670 { 1671 int i; 1672 struct pcf *p; 1673 1674 /* Update freemem while we're here. */ 1675 freemem = 0; 1676 p = pcf; 1677 for (i = 0; i < PCF_FANOUT; i++) { 1678 mutex_enter(&p->pcf_lock); 1679 ASSERT(p->pcf_count == 0); 1680 p->pcf_count = p->pcf_reserve; 1681 p->pcf_block = 0; 1682 freemem += p->pcf_count; 1683 if (p->pcf_wait) { 1684 mutex_enter(&new_freemem_lock); 1685 if (freemem_wait) { 1686 if (p->pcf_reserve > 1) { 1687 cv_broadcast(&freemem_cv); 1688 p->pcf_wait = 0; 1689 } else { 1690 cv_signal(&freemem_cv); 1691 p->pcf_wait--; 1692 } 1693 } else { 1694 p->pcf_wait = 0; 1695 } 1696 mutex_exit(&new_freemem_lock); 1697 } 1698 p->pcf_reserve = 0; 1699 mutex_exit(&p->pcf_lock); 1700 p++; 1701 } 1702 } 1703 1704 /* 1705 * Called from page_create_va() when both the cache and free lists 1706 * have been checked once. 1707 * 1708 * Either returns a page or panics since the accounting was done 1709 * way before we got here. 1710 * 1711 * We don't come here often, so leave the accounting on permanently. 1712 */ 1713 1714 #define MAX_PCGS 100 1715 1716 #ifdef DEBUG 1717 #define PCGS_TRIES 100 1718 #else /* DEBUG */ 1719 #define PCGS_TRIES 10 1720 #endif /* DEBUG */ 1721 1722 #ifdef VM_STATS 1723 uint_t pcgs_counts[PCGS_TRIES]; 1724 uint_t pcgs_too_many; 1725 uint_t pcgs_entered; 1726 uint_t pcgs_entered_noreloc; 1727 uint_t pcgs_locked; 1728 uint_t pcgs_cagelocked; 1729 #endif /* VM_STATS */ 1730 1731 static page_t * 1732 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1733 caddr_t vaddr, uint_t flags) 1734 { 1735 uint_t count; 1736 page_t *pp; 1737 uint_t locked, i; 1738 struct pcf *p; 1739 lgrp_t *lgrp; 1740 int cagelocked = 0; 1741 1742 VM_STAT_ADD(pcgs_entered); 1743 1744 /* 1745 * Tap any reserve freelists: if we fail now, we'll die 1746 * since the page(s) we're looking for have already been 1747 * accounted for. 1748 */ 1749 flags |= PG_PANIC; 1750 1751 if ((flags & PG_NORELOC) != 0) { 1752 VM_STAT_ADD(pcgs_entered_noreloc); 1753 /* 1754 * Requests for free pages from critical threads 1755 * such as pageout still won't throttle here, but 1756 * we must try again, to give the cageout thread 1757 * another chance to catch up. Since we already 1758 * accounted for the pages, we had better get them 1759 * this time. 1760 * 1761 * N.B. All non-critical threads acquire the pcgs_cagelock 1762 * to serialize access to the freelists. This implements a 1763 * turnstile-type synchornization to avoid starvation of 1764 * critical requests for PG_NORELOC memory by non-critical 1765 * threads: all non-critical threads must acquire a 'ticket' 1766 * before passing through, which entails making sure 1767 * kcage_freemem won't fall below minfree prior to grabbing 1768 * pages from the freelists. 1769 */ 1770 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1771 mutex_enter(&pcgs_cagelock); 1772 cagelocked = 1; 1773 VM_STAT_ADD(pcgs_cagelocked); 1774 } 1775 } 1776 1777 /* 1778 * Time to get serious. 1779 * We failed to get a `correctly colored' page from both the 1780 * free and cache lists. 1781 * We escalate in stage. 1782 * 1783 * First try both lists without worring about color. 1784 * 1785 * Then, grab all page accounting locks (ie. pcf[]) and 1786 * steal any pages that they have and set the pcf_block flag to 1787 * stop deletions from the lists. This will help because 1788 * a page can get added to the free list while we are looking 1789 * at the cache list, then another page could be added to the cache 1790 * list allowing the page on the free list to be removed as we 1791 * move from looking at the cache list to the free list. This 1792 * could happen over and over. We would never find the page 1793 * we have accounted for. 1794 * 1795 * Noreloc pages are a subset of the global (relocatable) page pool. 1796 * They are not tracked separately in the pcf bins, so it is 1797 * impossible to know when doing pcf accounting if the available 1798 * page(s) are noreloc pages or not. When looking for a noreloc page 1799 * it is quite easy to end up here even if the global (relocatable) 1800 * page pool has plenty of free pages but the noreloc pool is empty. 1801 * 1802 * When the noreloc pool is empty (or low), additional noreloc pages 1803 * are created by converting pages from the global page pool. This 1804 * process will stall during pcf accounting if the pcf bins are 1805 * already locked. Such is the case when a noreloc allocation is 1806 * looping here in page_create_get_something waiting for more noreloc 1807 * pages to appear. 1808 * 1809 * Short of adding a new field to the pcf bins to accurately track 1810 * the number of free noreloc pages, we instead do not grab the 1811 * pcgs_lock, do not set the pcf blocks and do not timeout when 1812 * allocating a noreloc page. This allows noreloc allocations to 1813 * loop without blocking global page pool allocations. 1814 * 1815 * NOTE: the behaviour of page_create_get_something has not changed 1816 * for the case of global page pool allocations. 1817 */ 1818 1819 flags &= ~PG_MATCH_COLOR; 1820 locked = 0; 1821 #if defined(__i386) || defined(__amd64) 1822 /* 1823 * page_create_get_something may be called because 4g memory may be 1824 * depleted. Set flags to allow for relocation of base page below 1825 * 4g if necessary. 1826 */ 1827 if (physmax4g) 1828 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI); 1829 #endif 1830 1831 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1832 1833 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1834 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1835 flags, lgrp); 1836 if (pp == NULL) { 1837 pp = page_get_cachelist(vp, off, seg, vaddr, 1838 flags, lgrp); 1839 } 1840 if (pp == NULL) { 1841 /* 1842 * Serialize. Don't fight with other pcgs(). 1843 */ 1844 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1845 mutex_enter(&pcgs_lock); 1846 VM_STAT_ADD(pcgs_locked); 1847 locked = 1; 1848 p = pcf; 1849 for (i = 0; i < PCF_FANOUT; i++) { 1850 mutex_enter(&p->pcf_lock); 1851 ASSERT(p->pcf_block == 0); 1852 p->pcf_block = 1; 1853 p->pcf_reserve = p->pcf_count; 1854 p->pcf_count = 0; 1855 mutex_exit(&p->pcf_lock); 1856 p++; 1857 } 1858 freemem = 0; 1859 } 1860 1861 if (count) { 1862 /* 1863 * Since page_free() puts pages on 1864 * a list then accounts for it, we 1865 * just have to wait for page_free() 1866 * to unlock any page it was working 1867 * with. The page_lock()-page_reclaim() 1868 * path falls in the same boat. 1869 * 1870 * We don't need to check on the 1871 * PG_WAIT flag, we have already 1872 * accounted for the page we are 1873 * looking for in page_create_va(). 1874 * 1875 * We just wait a moment to let any 1876 * locked pages on the lists free up, 1877 * then continue around and try again. 1878 * 1879 * Will be awakened by set_freemem(). 1880 */ 1881 mutex_enter(&pcgs_wait_lock); 1882 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1883 mutex_exit(&pcgs_wait_lock); 1884 } 1885 } else { 1886 #ifdef VM_STATS 1887 if (count >= PCGS_TRIES) { 1888 VM_STAT_ADD(pcgs_too_many); 1889 } else { 1890 VM_STAT_ADD(pcgs_counts[count]); 1891 } 1892 #endif 1893 if (locked) { 1894 pcgs_unblock(); 1895 mutex_exit(&pcgs_lock); 1896 } 1897 if (cagelocked) 1898 mutex_exit(&pcgs_cagelock); 1899 return (pp); 1900 } 1901 } 1902 /* 1903 * we go down holding the pcf locks. 1904 */ 1905 panic("no %spage found %d", 1906 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1907 /*NOTREACHED*/ 1908 } 1909 1910 /* 1911 * Create enough pages for "bytes" worth of data starting at 1912 * "off" in "vp". 1913 * 1914 * Where flag must be one of: 1915 * 1916 * PG_EXCL: Exclusive create (fail if any page already 1917 * exists in the page cache) which does not 1918 * wait for memory to become available. 1919 * 1920 * PG_WAIT: Non-exclusive create which can wait for 1921 * memory to become available. 1922 * 1923 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1924 * (Not Supported) 1925 * 1926 * A doubly linked list of pages is returned to the caller. Each page 1927 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1928 * lock. 1929 * 1930 * Unable to change the parameters to page_create() in a minor release, 1931 * we renamed page_create() to page_create_va(), changed all known calls 1932 * from page_create() to page_create_va(), and created this wrapper. 1933 * 1934 * Upon a major release, we should break compatibility by deleting this 1935 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1936 * 1937 * NOTE: There is a copy of this interface as page_create_io() in 1938 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1939 * there. 1940 */ 1941 page_t * 1942 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1943 { 1944 caddr_t random_vaddr; 1945 struct seg kseg; 1946 1947 #ifdef DEBUG 1948 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1949 (void *)caller()); 1950 #endif 1951 1952 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1953 (uintptr_t)(off >> PAGESHIFT)); 1954 kseg.s_as = &kas; 1955 1956 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1957 } 1958 1959 #ifdef DEBUG 1960 uint32_t pg_alloc_pgs_mtbf = 0; 1961 #endif 1962 1963 /* 1964 * Used for large page support. It will attempt to allocate 1965 * a large page(s) off the freelist. 1966 * 1967 * Returns non zero on failure. 1968 */ 1969 int 1970 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1971 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz) 1972 { 1973 pgcnt_t npgs, curnpgs, totpgs; 1974 size_t pgsz; 1975 page_t *pplist = NULL, *pp; 1976 int err = 0; 1977 lgrp_t *lgrp; 1978 1979 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 1980 1981 VM_STAT_ADD(alloc_pages[0]); 1982 1983 #ifdef DEBUG 1984 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 1985 return (ENOMEM); 1986 } 1987 #endif 1988 1989 pgsz = page_get_pagesize(szc); 1990 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 1991 1992 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 1993 /* 1994 * One must be NULL but not both. 1995 * And one must be non NULL but not both. 1996 */ 1997 ASSERT(basepp != NULL || ppa != NULL); 1998 ASSERT(basepp == NULL || ppa == NULL); 1999 2000 (void) page_create_wait(npgs, PG_WAIT); 2001 2002 while (npgs && szc) { 2003 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2004 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 0, lgrp); 2005 if (pp != NULL) { 2006 VM_STAT_ADD(alloc_pages[1]); 2007 page_list_concat(&pplist, &pp); 2008 ASSERT(npgs >= curnpgs); 2009 npgs -= curnpgs; 2010 } else if (anypgsz) { 2011 VM_STAT_ADD(alloc_pages[2]); 2012 szc--; 2013 pgsz = page_get_pagesize(szc); 2014 curnpgs = pgsz >> PAGESHIFT; 2015 } else { 2016 VM_STAT_ADD(alloc_pages[3]); 2017 ASSERT(npgs == totpgs); 2018 page_create_putback(npgs); 2019 return (ENOMEM); 2020 } 2021 } 2022 if (szc == 0) { 2023 VM_STAT_ADD(alloc_pages[4]); 2024 ASSERT(npgs != 0); 2025 page_create_putback(npgs); 2026 err = ENOMEM; 2027 } else if (basepp != NULL) { 2028 ASSERT(npgs == 0); 2029 ASSERT(ppa == NULL); 2030 *basepp = pplist; 2031 } 2032 2033 npgs = totpgs - npgs; 2034 pp = pplist; 2035 2036 /* 2037 * Clear the free and age bits. Also if we were passed in a ppa then 2038 * fill it in with all the constituent pages from the large page. But 2039 * if we failed to allocate all the pages just free what we got. 2040 */ 2041 while (npgs != 0) { 2042 ASSERT(PP_ISFREE(pp)); 2043 ASSERT(PP_ISAGED(pp)); 2044 if (ppa != NULL || err != 0) { 2045 if (err == 0) { 2046 VM_STAT_ADD(alloc_pages[5]); 2047 PP_CLRFREE(pp); 2048 PP_CLRAGED(pp); 2049 page_sub(&pplist, pp); 2050 *ppa++ = pp; 2051 npgs--; 2052 } else { 2053 VM_STAT_ADD(alloc_pages[6]); 2054 ASSERT(pp->p_szc != 0); 2055 curnpgs = page_get_pagecnt(pp->p_szc); 2056 page_list_break(&pp, &pplist, curnpgs); 2057 page_list_add_pages(pp, 0); 2058 page_create_putback(curnpgs); 2059 ASSERT(npgs >= curnpgs); 2060 npgs -= curnpgs; 2061 } 2062 pp = pplist; 2063 } else { 2064 VM_STAT_ADD(alloc_pages[7]); 2065 PP_CLRFREE(pp); 2066 PP_CLRAGED(pp); 2067 pp = pp->p_next; 2068 npgs--; 2069 } 2070 } 2071 return (err); 2072 } 2073 2074 /* 2075 * Get a single large page off of the freelists, and set it up for use. 2076 * Number of bytes requested must be a supported page size. 2077 * 2078 * Note that this call may fail even if there is sufficient 2079 * memory available or PG_WAIT is set, so the caller must 2080 * be willing to fallback on page_create_va(), block and retry, 2081 * or fail the requester. 2082 */ 2083 page_t * 2084 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2085 struct seg *seg, caddr_t vaddr, void *arg) 2086 { 2087 pgcnt_t npages, pcftotal; 2088 page_t *pp; 2089 page_t *rootpp; 2090 lgrp_t *lgrp; 2091 uint_t enough; 2092 uint_t pcf_index; 2093 uint_t i; 2094 struct pcf *p; 2095 struct pcf *q; 2096 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2097 2098 ASSERT(vp != NULL); 2099 2100 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2101 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2102 /* but no others */ 2103 2104 ASSERT((flags & PG_EXCL) == PG_EXCL); 2105 2106 npages = btop(bytes); 2107 2108 if (!kcage_on || panicstr) { 2109 /* 2110 * Cage is OFF, or we are single threaded in 2111 * panic, so make everything a RELOC request. 2112 */ 2113 flags &= ~PG_NORELOC; 2114 } 2115 2116 /* 2117 * Make sure there's adequate physical memory available. 2118 * Note: PG_WAIT is ignored here. 2119 */ 2120 if (freemem <= throttlefree + npages) { 2121 VM_STAT_ADD(page_create_large_cnt[1]); 2122 return (NULL); 2123 } 2124 2125 /* 2126 * If cage is on, dampen draw from cage when available 2127 * cage space is low. 2128 */ 2129 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2130 kcage_freemem < kcage_throttlefree + npages) { 2131 2132 /* 2133 * The cage is on, the caller wants PG_NORELOC 2134 * pages and available cage memory is very low. 2135 * Call kcage_create_throttle() to attempt to 2136 * control demand on the cage. 2137 */ 2138 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2139 VM_STAT_ADD(page_create_large_cnt[2]); 2140 return (NULL); 2141 } 2142 } 2143 2144 enough = 0; 2145 pcf_index = PCF_INDEX(); 2146 p = &pcf[pcf_index]; 2147 p->pcf_touch = 1; 2148 q = &pcf[PCF_FANOUT]; 2149 for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2150 if (p->pcf_count > npages) { 2151 /* 2152 * a good one to try. 2153 */ 2154 mutex_enter(&p->pcf_lock); 2155 if (p->pcf_count > npages) { 2156 p->pcf_count -= (uint_t)npages; 2157 /* 2158 * freemem is not protected by any lock. 2159 * Thus, we cannot have any assertion 2160 * containing freemem here. 2161 */ 2162 freemem -= npages; 2163 enough = 1; 2164 mutex_exit(&p->pcf_lock); 2165 break; 2166 } 2167 mutex_exit(&p->pcf_lock); 2168 } 2169 pcftotal += p->pcf_count; 2170 p++; 2171 if (p >= q) { 2172 p = pcf; 2173 } 2174 p->pcf_touch = 1; 2175 } 2176 2177 if (!enough) { 2178 /* If there isn't enough memory available, give up. */ 2179 if (pcftotal < npages) { 2180 VM_STAT_ADD(page_create_large_cnt[3]); 2181 return (NULL); 2182 } 2183 2184 /* try to collect pages from several pcf bins */ 2185 for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2186 p->pcf_touch = 1; 2187 mutex_enter(&p->pcf_lock); 2188 pcftotal += p->pcf_count; 2189 if (pcftotal >= npages) { 2190 /* 2191 * Wow! There are enough pages laying around 2192 * to satisfy the request. Do the accounting, 2193 * drop the locks we acquired, and go back. 2194 * 2195 * freemem is not protected by any lock. So, 2196 * we cannot have any assertion containing 2197 * freemem. 2198 */ 2199 pgcnt_t tpages = npages; 2200 freemem -= npages; 2201 while (p >= pcf) { 2202 if (p->pcf_count <= tpages) { 2203 tpages -= p->pcf_count; 2204 p->pcf_count = 0; 2205 } else { 2206 p->pcf_count -= (uint_t)tpages; 2207 tpages = 0; 2208 } 2209 mutex_exit(&p->pcf_lock); 2210 p--; 2211 } 2212 ASSERT(tpages == 0); 2213 break; 2214 } 2215 p++; 2216 } 2217 if (i == PCF_FANOUT) { 2218 /* failed to collect pages - release the locks */ 2219 while (--p >= pcf) { 2220 mutex_exit(&p->pcf_lock); 2221 } 2222 VM_STAT_ADD(page_create_large_cnt[4]); 2223 return (NULL); 2224 } 2225 } 2226 2227 /* 2228 * This is where this function behaves fundamentally differently 2229 * than page_create_va(); since we're intending to map the page 2230 * with a single TTE, we have to get it as a physically contiguous 2231 * hardware pagesize chunk. If we can't, we fail. 2232 */ 2233 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2234 LGRP_EXISTS(lgrp_table[*lgrpid])) 2235 lgrp = lgrp_table[*lgrpid]; 2236 else 2237 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2238 2239 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2240 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2241 page_create_putback(npages); 2242 VM_STAT_ADD(page_create_large_cnt[5]); 2243 return (NULL); 2244 } 2245 2246 /* 2247 * if we got the page with the wrong mtype give it back this is a 2248 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2249 * inside "if" and the workaround becomes just a nop 2250 */ 2251 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2252 page_list_add_pages(rootpp, 0); 2253 page_create_putback(npages); 2254 VM_STAT_ADD(page_create_large_cnt[6]); 2255 return (NULL); 2256 } 2257 2258 /* 2259 * If satisfying this request has left us with too little 2260 * memory, start the wheels turning to get some back. The 2261 * first clause of the test prevents waking up the pageout 2262 * daemon in situations where it would decide that there's 2263 * nothing to do. 2264 */ 2265 if (nscan < desscan && freemem < minfree) { 2266 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2267 "pageout_cv_signal:freemem %ld", freemem); 2268 cv_signal(&proc_pageout->p_cv); 2269 } 2270 2271 pp = rootpp; 2272 while (npages--) { 2273 ASSERT(PAGE_EXCL(pp)); 2274 ASSERT(pp->p_vnode == NULL); 2275 ASSERT(!hat_page_is_mapped(pp)); 2276 PP_CLRFREE(pp); 2277 PP_CLRAGED(pp); 2278 if (!page_hashin(pp, vp, off, NULL)) 2279 panic("page_create_large: hashin failed: page %p", 2280 (void *)pp); 2281 page_io_lock(pp); 2282 off += PAGESIZE; 2283 pp = pp->p_next; 2284 } 2285 2286 VM_STAT_ADD(page_create_large_cnt[0]); 2287 return (rootpp); 2288 } 2289 2290 page_t * 2291 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2292 struct seg *seg, caddr_t vaddr) 2293 { 2294 page_t *plist = NULL; 2295 pgcnt_t npages; 2296 pgcnt_t found_on_free = 0; 2297 pgcnt_t pages_req; 2298 page_t *npp = NULL; 2299 uint_t enough; 2300 uint_t i; 2301 uint_t pcf_index; 2302 struct pcf *p; 2303 struct pcf *q; 2304 lgrp_t *lgrp; 2305 2306 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2307 "page_create_start:vp %p off %llx bytes %lu flags %x", 2308 vp, off, bytes, flags); 2309 2310 ASSERT(bytes != 0 && vp != NULL); 2311 2312 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2313 panic("page_create: invalid flags"); 2314 /*NOTREACHED*/ 2315 } 2316 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2317 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2318 /* but no others */ 2319 2320 pages_req = npages = btopr(bytes); 2321 /* 2322 * Try to see whether request is too large to *ever* be 2323 * satisfied, in order to prevent deadlock. We arbitrarily 2324 * decide to limit maximum size requests to max_page_get. 2325 */ 2326 if (npages >= max_page_get) { 2327 if ((flags & PG_WAIT) == 0) { 2328 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2329 "page_create_toobig:vp %p off %llx npages " 2330 "%lu max_page_get %lu", 2331 vp, off, npages, max_page_get); 2332 return (NULL); 2333 } else { 2334 cmn_err(CE_WARN, 2335 "Request for too much kernel memory " 2336 "(%lu bytes), will hang forever", bytes); 2337 for (;;) 2338 delay(1000000000); 2339 } 2340 } 2341 2342 if (!kcage_on || panicstr) { 2343 /* 2344 * Cage is OFF, or we are single threaded in 2345 * panic, so make everything a RELOC request. 2346 */ 2347 flags &= ~PG_NORELOC; 2348 } 2349 2350 if (freemem <= throttlefree + npages) 2351 if (!page_create_throttle(npages, flags)) 2352 return (NULL); 2353 2354 /* 2355 * If cage is on, dampen draw from cage when available 2356 * cage space is low. 2357 */ 2358 if ((flags & PG_NORELOC) && 2359 kcage_freemem < kcage_throttlefree + npages) { 2360 2361 /* 2362 * The cage is on, the caller wants PG_NORELOC 2363 * pages and available cage memory is very low. 2364 * Call kcage_create_throttle() to attempt to 2365 * control demand on the cage. 2366 */ 2367 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2368 return (NULL); 2369 } 2370 2371 VM_STAT_ADD(page_create_cnt[0]); 2372 2373 enough = 0; 2374 pcf_index = PCF_INDEX(); 2375 2376 p = &pcf[pcf_index]; 2377 p->pcf_touch = 1; 2378 q = &pcf[PCF_FANOUT]; 2379 for (i = 0; i < PCF_FANOUT; i++) { 2380 if (p->pcf_count > npages) { 2381 /* 2382 * a good one to try. 2383 */ 2384 mutex_enter(&p->pcf_lock); 2385 if (p->pcf_count > npages) { 2386 p->pcf_count -= (uint_t)npages; 2387 /* 2388 * freemem is not protected by any lock. 2389 * Thus, we cannot have any assertion 2390 * containing freemem here. 2391 */ 2392 freemem -= npages; 2393 enough = 1; 2394 mutex_exit(&p->pcf_lock); 2395 break; 2396 } 2397 mutex_exit(&p->pcf_lock); 2398 } 2399 p++; 2400 if (p >= q) { 2401 p = pcf; 2402 } 2403 p->pcf_touch = 1; 2404 } 2405 2406 if (!enough) { 2407 /* 2408 * Have to look harder. If npages is greater than 2409 * one, then we might have to coalecse the counters. 2410 * 2411 * Go wait. We come back having accounted 2412 * for the memory. 2413 */ 2414 VM_STAT_ADD(page_create_cnt[1]); 2415 if (!page_create_wait(npages, flags)) { 2416 VM_STAT_ADD(page_create_cnt[2]); 2417 return (NULL); 2418 } 2419 } 2420 2421 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2422 "page_create_success:vp %p off %llx", vp, off); 2423 2424 /* 2425 * If satisfying this request has left us with too little 2426 * memory, start the wheels turning to get some back. The 2427 * first clause of the test prevents waking up the pageout 2428 * daemon in situations where it would decide that there's 2429 * nothing to do. 2430 */ 2431 if (nscan < desscan && freemem < minfree) { 2432 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2433 "pageout_cv_signal:freemem %ld", freemem); 2434 cv_signal(&proc_pageout->p_cv); 2435 } 2436 2437 /* 2438 * Loop around collecting the requested number of pages. 2439 * Most of the time, we have to `create' a new page. With 2440 * this in mind, pull the page off the free list before 2441 * getting the hash lock. This will minimize the hash 2442 * lock hold time, nesting, and the like. If it turns 2443 * out we don't need the page, we put it back at the end. 2444 */ 2445 while (npages--) { 2446 page_t *pp; 2447 kmutex_t *phm = NULL; 2448 ulong_t index; 2449 2450 index = PAGE_HASH_FUNC(vp, off); 2451 top: 2452 ASSERT(phm == NULL); 2453 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2454 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2455 2456 if (npp == NULL) { 2457 /* 2458 * Try to get a page from the freelist (ie, 2459 * a page with no [vp, off] tag). If that 2460 * fails, use the cachelist. 2461 * 2462 * During the first attempt at both the free 2463 * and cache lists we try for the correct color. 2464 */ 2465 /* 2466 * XXXX-how do we deal with virtual indexed 2467 * caches and and colors? 2468 */ 2469 VM_STAT_ADD(page_create_cnt[4]); 2470 /* 2471 * Get lgroup to allocate next page of shared memory 2472 * from and use it to specify where to allocate 2473 * the physical memory 2474 */ 2475 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2476 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2477 flags | PG_MATCH_COLOR, lgrp); 2478 if (npp == NULL) { 2479 npp = page_get_cachelist(vp, off, seg, 2480 vaddr, flags | PG_MATCH_COLOR, lgrp); 2481 if (npp == NULL) { 2482 npp = page_create_get_something(vp, 2483 off, seg, vaddr, 2484 flags & ~PG_MATCH_COLOR); 2485 } 2486 2487 if (PP_ISAGED(npp) == 0) { 2488 /* 2489 * Since this page came from the 2490 * cachelist, we must destroy the 2491 * old vnode association. 2492 */ 2493 page_hashout(npp, NULL); 2494 } 2495 } 2496 } 2497 2498 /* 2499 * We own this page! 2500 */ 2501 ASSERT(PAGE_EXCL(npp)); 2502 ASSERT(npp->p_vnode == NULL); 2503 ASSERT(!hat_page_is_mapped(npp)); 2504 PP_CLRFREE(npp); 2505 PP_CLRAGED(npp); 2506 2507 /* 2508 * Here we have a page in our hot little mits and are 2509 * just waiting to stuff it on the appropriate lists. 2510 * Get the mutex and check to see if it really does 2511 * not exist. 2512 */ 2513 phm = PAGE_HASH_MUTEX(index); 2514 mutex_enter(phm); 2515 PAGE_HASH_SEARCH(index, pp, vp, off); 2516 if (pp == NULL) { 2517 VM_STAT_ADD(page_create_new); 2518 pp = npp; 2519 npp = NULL; 2520 if (!page_hashin(pp, vp, off, phm)) { 2521 /* 2522 * Since we hold the page hash mutex and 2523 * just searched for this page, page_hashin 2524 * had better not fail. If it does, that 2525 * means somethread did not follow the 2526 * page hash mutex rules. Panic now and 2527 * get it over with. As usual, go down 2528 * holding all the locks. 2529 */ 2530 ASSERT(MUTEX_HELD(phm)); 2531 panic("page_create: " 2532 "hashin failed %p %p %llx %p", 2533 (void *)pp, (void *)vp, off, (void *)phm); 2534 /*NOTREACHED*/ 2535 } 2536 ASSERT(MUTEX_HELD(phm)); 2537 mutex_exit(phm); 2538 phm = NULL; 2539 2540 /* 2541 * Hat layer locking need not be done to set 2542 * the following bits since the page is not hashed 2543 * and was on the free list (i.e., had no mappings). 2544 * 2545 * Set the reference bit to protect 2546 * against immediate pageout 2547 * 2548 * XXXmh modify freelist code to set reference 2549 * bit so we don't have to do it here. 2550 */ 2551 page_set_props(pp, P_REF); 2552 found_on_free++; 2553 } else { 2554 VM_STAT_ADD(page_create_exists); 2555 if (flags & PG_EXCL) { 2556 /* 2557 * Found an existing page, and the caller 2558 * wanted all new pages. Undo all of the work 2559 * we have done. 2560 */ 2561 mutex_exit(phm); 2562 phm = NULL; 2563 while (plist != NULL) { 2564 pp = plist; 2565 page_sub(&plist, pp); 2566 page_io_unlock(pp); 2567 /* large pages should not end up here */ 2568 ASSERT(pp->p_szc == 0); 2569 /*LINTED: constant in conditional ctx*/ 2570 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2571 } 2572 VM_STAT_ADD(page_create_found_one); 2573 goto fail; 2574 } 2575 ASSERT(flags & PG_WAIT); 2576 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2577 /* 2578 * Start all over again if we blocked trying 2579 * to lock the page. 2580 */ 2581 mutex_exit(phm); 2582 VM_STAT_ADD(page_create_page_lock_failed); 2583 phm = NULL; 2584 goto top; 2585 } 2586 mutex_exit(phm); 2587 phm = NULL; 2588 2589 if (PP_ISFREE(pp)) { 2590 ASSERT(PP_ISAGED(pp) == 0); 2591 VM_STAT_ADD(pagecnt.pc_get_cache); 2592 page_list_sub(pp, PG_CACHE_LIST); 2593 PP_CLRFREE(pp); 2594 found_on_free++; 2595 } 2596 } 2597 2598 /* 2599 * Got a page! It is locked. Acquire the i/o 2600 * lock since we are going to use the p_next and 2601 * p_prev fields to link the requested pages together. 2602 */ 2603 page_io_lock(pp); 2604 page_add(&plist, pp); 2605 plist = plist->p_next; 2606 off += PAGESIZE; 2607 vaddr += PAGESIZE; 2608 } 2609 2610 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2611 fail: 2612 if (npp != NULL) { 2613 /* 2614 * Did not need this page after all. 2615 * Put it back on the free list. 2616 */ 2617 VM_STAT_ADD(page_create_putbacks); 2618 PP_SETFREE(npp); 2619 PP_SETAGED(npp); 2620 npp->p_offset = (u_offset_t)-1; 2621 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2622 page_unlock(npp); 2623 2624 } 2625 2626 ASSERT(pages_req >= found_on_free); 2627 2628 { 2629 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2630 2631 if (overshoot) { 2632 VM_STAT_ADD(page_create_overshoot); 2633 p = &pcf[pcf_index]; 2634 p->pcf_touch = 1; 2635 mutex_enter(&p->pcf_lock); 2636 if (p->pcf_block) { 2637 p->pcf_reserve += overshoot; 2638 } else { 2639 p->pcf_count += overshoot; 2640 if (p->pcf_wait) { 2641 mutex_enter(&new_freemem_lock); 2642 if (freemem_wait) { 2643 cv_signal(&freemem_cv); 2644 p->pcf_wait--; 2645 } else { 2646 p->pcf_wait = 0; 2647 } 2648 mutex_exit(&new_freemem_lock); 2649 } 2650 } 2651 mutex_exit(&p->pcf_lock); 2652 /* freemem is approximate, so this test OK */ 2653 if (!p->pcf_block) 2654 freemem += overshoot; 2655 } 2656 } 2657 2658 return (plist); 2659 } 2660 2661 /* 2662 * One or more constituent pages of this large page has been marked 2663 * toxic. Simply demote the large page to PAGESIZE pages and let 2664 * page_free() handle it. This routine should only be called by 2665 * large page free routines (page_free_pages() and page_destroy_pages(). 2666 * All pages are locked SE_EXCL and have already been marked free. 2667 */ 2668 static void 2669 page_free_toxic_pages(page_t *rootpp) 2670 { 2671 page_t *tpp; 2672 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2673 uint_t szc = rootpp->p_szc; 2674 2675 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2676 ASSERT(tpp->p_szc == szc); 2677 ASSERT((PAGE_EXCL(tpp) && 2678 !page_iolock_assert(tpp)) || panicstr); 2679 tpp->p_szc = 0; 2680 } 2681 2682 while (rootpp != NULL) { 2683 tpp = rootpp; 2684 page_sub(&rootpp, tpp); 2685 ASSERT(PP_ISFREE(tpp)); 2686 PP_CLRFREE(tpp); 2687 page_free(tpp, 1); 2688 } 2689 } 2690 2691 /* 2692 * Put page on the "free" list. 2693 * The free list is really two lists maintained by 2694 * the PSM of whatever machine we happen to be on. 2695 */ 2696 void 2697 page_free(page_t *pp, int dontneed) 2698 { 2699 struct pcf *p; 2700 uint_t pcf_index; 2701 2702 ASSERT((PAGE_EXCL(pp) && 2703 !page_iolock_assert(pp)) || panicstr); 2704 2705 if (PP_ISFREE(pp)) { 2706 panic("page_free: page %p is free", (void *)pp); 2707 } 2708 2709 if (pp->p_szc != 0) { 2710 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2711 pp->p_vnode == &kvp) { 2712 panic("page_free: anon or kernel " 2713 "or no vnode large page %p", (void *)pp); 2714 } 2715 page_demote_vp_pages(pp); 2716 ASSERT(pp->p_szc == 0); 2717 } 2718 2719 /* 2720 * The page_struct_lock need not be acquired to examine these 2721 * fields since the page has an "exclusive" lock. 2722 */ 2723 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 2724 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d", 2725 pp, page_pptonum(pp), pp->p_lckcnt, pp->p_cowcnt); 2726 /*NOTREACHED*/ 2727 } 2728 2729 ASSERT(!hat_page_getshare(pp)); 2730 2731 PP_SETFREE(pp); 2732 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2733 !hat_ismod(pp)); 2734 page_clr_all_props(pp); 2735 ASSERT(!hat_page_getshare(pp)); 2736 2737 /* 2738 * Now we add the page to the head of the free list. 2739 * But if this page is associated with a paged vnode 2740 * then we adjust the head forward so that the page is 2741 * effectively at the end of the list. 2742 */ 2743 if (pp->p_vnode == NULL) { 2744 /* 2745 * Page has no identity, put it on the free list. 2746 */ 2747 PP_SETAGED(pp); 2748 pp->p_offset = (u_offset_t)-1; 2749 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2750 VM_STAT_ADD(pagecnt.pc_free_free); 2751 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2752 "page_free_free:pp %p", pp); 2753 } else { 2754 PP_CLRAGED(pp); 2755 2756 if (!dontneed || nopageage) { 2757 /* move it to the tail of the list */ 2758 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2759 2760 VM_STAT_ADD(pagecnt.pc_free_cache); 2761 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2762 "page_free_cache_tail:pp %p", pp); 2763 } else { 2764 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2765 2766 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2767 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2768 "page_free_cache_head:pp %p", pp); 2769 } 2770 } 2771 page_unlock(pp); 2772 2773 /* 2774 * Now do the `freemem' accounting. 2775 */ 2776 pcf_index = PCF_INDEX(); 2777 p = &pcf[pcf_index]; 2778 p->pcf_touch = 1; 2779 2780 mutex_enter(&p->pcf_lock); 2781 if (p->pcf_block) { 2782 p->pcf_reserve += 1; 2783 } else { 2784 p->pcf_count += 1; 2785 if (p->pcf_wait) { 2786 mutex_enter(&new_freemem_lock); 2787 /* 2788 * Check to see if some other thread 2789 * is actually waiting. Another bucket 2790 * may have woken it up by now. If there 2791 * are no waiters, then set our pcf_wait 2792 * count to zero to avoid coming in here 2793 * next time. Also, since only one page 2794 * was put on the free list, just wake 2795 * up one waiter. 2796 */ 2797 if (freemem_wait) { 2798 cv_signal(&freemem_cv); 2799 p->pcf_wait--; 2800 } else { 2801 p->pcf_wait = 0; 2802 } 2803 mutex_exit(&new_freemem_lock); 2804 } 2805 } 2806 mutex_exit(&p->pcf_lock); 2807 2808 /* freemem is approximate, so this test OK */ 2809 if (!p->pcf_block) 2810 freemem += 1; 2811 } 2812 2813 /* 2814 * Put page on the "free" list during intial startup. 2815 * This happens during initial single threaded execution. 2816 */ 2817 void 2818 page_free_at_startup(page_t *pp) 2819 { 2820 struct pcf *p; 2821 uint_t pcf_index; 2822 2823 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2824 VM_STAT_ADD(pagecnt.pc_free_free); 2825 2826 /* 2827 * Now do the `freemem' accounting. 2828 */ 2829 pcf_index = PCF_INDEX(); 2830 p = &pcf[pcf_index]; 2831 p->pcf_touch = 1; 2832 2833 ASSERT(p->pcf_block == 0); 2834 ASSERT(p->pcf_wait == 0); 2835 p->pcf_count += 1; 2836 2837 /* freemem is approximate, so this is OK */ 2838 freemem += 1; 2839 } 2840 2841 void 2842 page_free_pages(page_t *pp) 2843 { 2844 page_t *tpp, *rootpp = NULL; 2845 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2846 pgcnt_t i; 2847 uint_t szc = pp->p_szc; 2848 2849 VM_STAT_ADD(pagecnt.pc_free_pages); 2850 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2851 "page_free_free:pp %p", pp); 2852 2853 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2854 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2855 panic("page_free_pages: not root page %p", (void *)pp); 2856 /*NOTREACHED*/ 2857 } 2858 2859 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2860 ASSERT((PAGE_EXCL(tpp) && 2861 !page_iolock_assert(tpp)) || panicstr); 2862 if (PP_ISFREE(tpp)) { 2863 panic("page_free_pages: page %p is free", (void *)tpp); 2864 /*NOTREACHED*/ 2865 } 2866 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2867 tpp->p_cowcnt != 0) { 2868 panic("page_free_pages %p", (void *)tpp); 2869 /*NOTREACHED*/ 2870 } 2871 2872 ASSERT(!hat_page_getshare(tpp)); 2873 ASSERT(tpp->p_vnode == NULL); 2874 ASSERT(tpp->p_szc == szc); 2875 2876 PP_SETFREE(tpp); 2877 page_clr_all_props(tpp); 2878 PP_SETAGED(tpp); 2879 tpp->p_offset = (u_offset_t)-1; 2880 ASSERT(tpp->p_next == tpp); 2881 ASSERT(tpp->p_prev == tpp); 2882 page_list_concat(&rootpp, &tpp); 2883 } 2884 ASSERT(rootpp == pp); 2885 2886 page_list_add_pages(rootpp, 0); 2887 page_create_putback(pgcnt); 2888 } 2889 2890 int free_pages = 1; 2891 2892 /* 2893 * This routine attempts to return pages to the cachelist via page_release(). 2894 * It does not *have* to be successful in all cases, since the pageout scanner 2895 * will catch any pages it misses. It does need to be fast and not introduce 2896 * too much overhead. 2897 * 2898 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2899 * don't lock and retry. This is ok, since the page scanner will eventually 2900 * find any page we miss in free_vp_pages(). 2901 */ 2902 void 2903 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2904 { 2905 page_t *pp; 2906 u_offset_t eoff; 2907 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2908 2909 eoff = off + len; 2910 2911 if (free_pages == 0) 2912 return; 2913 if (swap_in_range(vp, off, len)) 2914 return; 2915 2916 for (; off < eoff; off += PAGESIZE) { 2917 2918 /* 2919 * find the page using a fast, but inexact search. It'll be OK 2920 * if a few pages slip through the cracks here. 2921 */ 2922 pp = page_exists(vp, off); 2923 2924 /* 2925 * If we didn't find the page (it may not exist), the page 2926 * is free, looks still in use (shared), or we can't lock it, 2927 * just give up. 2928 */ 2929 if (pp == NULL || 2930 PP_ISFREE(pp) || 2931 page_share_cnt(pp) > 0 || 2932 !page_trylock(pp, SE_EXCL)) 2933 continue; 2934 2935 /* 2936 * Once we have locked pp, verify that it's still the 2937 * correct page and not already free 2938 */ 2939 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2940 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2941 page_unlock(pp); 2942 continue; 2943 } 2944 2945 /* 2946 * try to release the page... 2947 */ 2948 (void) page_release(pp, 1); 2949 } 2950 } 2951 2952 /* 2953 * Reclaim the given page from the free list. 2954 * Returns 1 on success or 0 on failure. 2955 * 2956 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2957 * If `lock' is non-null, it will be dropped and re-acquired if 2958 * the routine must wait while freemem is 0. 2959 * 2960 * As it turns out, boot_getpages() does this. It picks a page, 2961 * based on where OBP mapped in some address, gets its pfn, searches 2962 * the memsegs, locks the page, then pulls it off the free list! 2963 */ 2964 int 2965 page_reclaim(page_t *pp, kmutex_t *lock) 2966 { 2967 struct pcf *p; 2968 uint_t pcf_index; 2969 struct cpu *cpup; 2970 uint_t i; 2971 pgcnt_t npgs, need; 2972 pgcnt_t collected = 0; 2973 2974 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2975 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2976 2977 npgs = page_get_pagecnt(pp->p_szc); 2978 2979 /* 2980 * If `freemem' is 0, we cannot reclaim this page from the 2981 * freelist, so release every lock we might hold: the page, 2982 * and the `lock' before blocking. 2983 * 2984 * The only way `freemem' can become 0 while there are pages 2985 * marked free (have their p->p_free bit set) is when the 2986 * system is low on memory and doing a page_create(). In 2987 * order to guarantee that once page_create() starts acquiring 2988 * pages it will be able to get all that it needs since `freemem' 2989 * was decreased by the requested amount. So, we need to release 2990 * this page, and let page_create() have it. 2991 * 2992 * Since `freemem' being zero is not supposed to happen, just 2993 * use the usual hash stuff as a starting point. If that bucket 2994 * is empty, then assume the worst, and start at the beginning 2995 * of the pcf array. If we always start at the beginning 2996 * when acquiring more than one pcf lock, there won't be any 2997 * deadlock problems. 2998 */ 2999 3000 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 3001 3002 if (freemem <= throttlefree && !page_create_throttle(npgs, 0)) { 3003 pcf_acquire_all(); 3004 goto page_reclaim_nomem; 3005 } 3006 3007 pcf_index = PCF_INDEX(); 3008 p = &pcf[pcf_index]; 3009 p->pcf_touch = 1; 3010 mutex_enter(&p->pcf_lock); 3011 if (p->pcf_count >= npgs) { 3012 collected = npgs; 3013 p->pcf_count -= npgs; 3014 } 3015 mutex_exit(&p->pcf_lock); 3016 need = npgs - collected; 3017 3018 if (need > 0) { 3019 VM_STAT_ADD(page_reclaim_zero); 3020 /* 3021 * Check again. Its possible that some other thread 3022 * could have been right behind us, and added one 3023 * to a list somewhere. Acquire each of the pcf locks 3024 * until we find a page. 3025 */ 3026 p = pcf; 3027 for (i = 0; i < PCF_FANOUT; i++) { 3028 p->pcf_touch = 1; 3029 mutex_enter(&p->pcf_lock); 3030 if (p->pcf_count) { 3031 if (p->pcf_count >= need) { 3032 p->pcf_count -= need; 3033 collected += need; 3034 need = 0; 3035 break; 3036 } else if (p->pcf_count) { 3037 collected += p->pcf_count; 3038 need -= p->pcf_count; 3039 p->pcf_count = 0; 3040 } 3041 } 3042 p++; 3043 } 3044 3045 if (need > 0) { 3046 page_reclaim_nomem: 3047 /* 3048 * We really can't have page `pp'. 3049 * Time for the no-memory dance with 3050 * page_free(). This is just like 3051 * page_create_wait(). Plus the added 3052 * attraction of releasing whatever mutex 3053 * we held when we were called with in `lock'. 3054 * Page_unlock() will wakeup any thread 3055 * waiting around for this page. 3056 */ 3057 if (lock) { 3058 VM_STAT_ADD(page_reclaim_zero_locked); 3059 mutex_exit(lock); 3060 } 3061 page_unlock(pp); 3062 3063 /* 3064 * get this before we drop all the pcf locks. 3065 */ 3066 mutex_enter(&new_freemem_lock); 3067 3068 p = pcf; 3069 p->pcf_count += collected; 3070 for (i = 0; i < PCF_FANOUT; i++) { 3071 p->pcf_wait++; 3072 mutex_exit(&p->pcf_lock); 3073 p++; 3074 } 3075 3076 freemem_wait++; 3077 cv_wait(&freemem_cv, &new_freemem_lock); 3078 freemem_wait--; 3079 3080 mutex_exit(&new_freemem_lock); 3081 3082 if (lock) { 3083 mutex_enter(lock); 3084 } 3085 return (0); 3086 } 3087 3088 /* 3089 * We beat the PCF bins over the head until 3090 * we got the memory that we wanted. 3091 * The pcf accounting has been done, 3092 * though none of the pcf_wait flags have been set, 3093 * drop the locks and continue on. 3094 */ 3095 ASSERT(collected == npgs); 3096 while (p >= pcf) { 3097 mutex_exit(&p->pcf_lock); 3098 p--; 3099 } 3100 } 3101 3102 /* 3103 * freemem is not protected by any lock. Thus, we cannot 3104 * have any assertion containing freemem here. 3105 */ 3106 freemem -= npgs; 3107 3108 VM_STAT_ADD(pagecnt.pc_reclaim); 3109 if (PP_ISAGED(pp)) { 3110 if (npgs > 1) { 3111 page_list_sub_pages(pp, pp->p_szc); 3112 } else { 3113 page_list_sub(pp, PG_FREE_LIST); 3114 } 3115 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3116 "page_reclaim_free:pp %p", pp); 3117 } else { 3118 ASSERT(npgs == 1); 3119 page_list_sub(pp, PG_CACHE_LIST); 3120 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3121 "page_reclaim_cache:pp %p", pp); 3122 } 3123 3124 /* 3125 * clear the p_free & p_age bits since this page is no longer 3126 * on the free list. Notice that there was a brief time where 3127 * a page is marked as free, but is not on the list. 3128 * 3129 * Set the reference bit to protect against immediate pageout. 3130 */ 3131 for (i = 0; i < npgs; i++, pp++) { 3132 PP_CLRFREE(pp); 3133 PP_CLRAGED(pp); 3134 page_set_props(pp, P_REF); 3135 } 3136 3137 CPU_STATS_ENTER_K(); 3138 cpup = CPU; /* get cpup now that CPU cannot change */ 3139 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3140 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3141 CPU_STATS_EXIT_K(); 3142 3143 return (1); 3144 } 3145 3146 3147 3148 /* 3149 * Destroy identity of the page and put it back on 3150 * the page free list. Assumes that the caller has 3151 * acquired the "exclusive" lock on the page. 3152 */ 3153 void 3154 page_destroy(page_t *pp, int dontfree) 3155 { 3156 ASSERT((PAGE_EXCL(pp) && 3157 !page_iolock_assert(pp)) || panicstr); 3158 3159 if (pp->p_szc != 0) { 3160 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3161 pp->p_vnode == &kvp) { 3162 panic("page_destroy: anon or kernel or no vnode " 3163 "large page %p", (void *)pp); 3164 } 3165 page_demote_vp_pages(pp); 3166 ASSERT(pp->p_szc == 0); 3167 } 3168 3169 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3170 3171 /* 3172 * Unload translations, if any, then hash out the 3173 * page to erase its identity. 3174 */ 3175 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3176 page_hashout(pp, NULL); 3177 3178 if (!dontfree) { 3179 /* 3180 * Acquire the "freemem_lock" for availrmem. 3181 * The page_struct_lock need not be acquired for lckcnt 3182 * and cowcnt since the page has an "exclusive" lock. 3183 */ 3184 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3185 mutex_enter(&freemem_lock); 3186 if (pp->p_lckcnt != 0) { 3187 availrmem++; 3188 pp->p_lckcnt = 0; 3189 } 3190 if (pp->p_cowcnt != 0) { 3191 availrmem += pp->p_cowcnt; 3192 pp->p_cowcnt = 0; 3193 } 3194 mutex_exit(&freemem_lock); 3195 } 3196 /* 3197 * Put the page on the "free" list. 3198 */ 3199 page_free(pp, 0); 3200 } 3201 } 3202 3203 void 3204 page_destroy_pages(page_t *pp) 3205 { 3206 3207 page_t *tpp, *rootpp = NULL; 3208 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3209 pgcnt_t i, pglcks = 0; 3210 uint_t szc = pp->p_szc; 3211 3212 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3213 3214 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3215 3216 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3217 3218 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3219 panic("page_destroy_pages: not root page %p", (void *)pp); 3220 /*NOTREACHED*/ 3221 } 3222 3223 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3224 ASSERT((PAGE_EXCL(tpp) && 3225 !page_iolock_assert(tpp)) || panicstr); 3226 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3227 page_hashout(tpp, NULL); 3228 ASSERT(tpp->p_offset == (u_offset_t)-1); 3229 if (tpp->p_lckcnt != 0) { 3230 pglcks++; 3231 tpp->p_lckcnt = 0; 3232 } else if (tpp->p_cowcnt != 0) { 3233 pglcks += tpp->p_cowcnt; 3234 tpp->p_cowcnt = 0; 3235 } 3236 ASSERT(!hat_page_getshare(tpp)); 3237 ASSERT(tpp->p_vnode == NULL); 3238 ASSERT(tpp->p_szc == szc); 3239 3240 PP_SETFREE(tpp); 3241 page_clr_all_props(tpp); 3242 PP_SETAGED(tpp); 3243 ASSERT(tpp->p_next == tpp); 3244 ASSERT(tpp->p_prev == tpp); 3245 page_list_concat(&rootpp, &tpp); 3246 } 3247 3248 ASSERT(rootpp == pp); 3249 if (pglcks != 0) { 3250 mutex_enter(&freemem_lock); 3251 availrmem += pglcks; 3252 mutex_exit(&freemem_lock); 3253 } 3254 3255 page_list_add_pages(rootpp, 0); 3256 page_create_putback(pgcnt); 3257 } 3258 3259 /* 3260 * Similar to page_destroy(), but destroys pages which are 3261 * locked and known to be on the page free list. Since 3262 * the page is known to be free and locked, no one can access 3263 * it. 3264 * 3265 * Also, the number of free pages does not change. 3266 */ 3267 void 3268 page_destroy_free(page_t *pp) 3269 { 3270 ASSERT(PAGE_EXCL(pp)); 3271 ASSERT(PP_ISFREE(pp)); 3272 ASSERT(pp->p_vnode); 3273 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3274 ASSERT(!hat_page_is_mapped(pp)); 3275 ASSERT(PP_ISAGED(pp) == 0); 3276 ASSERT(pp->p_szc == 0); 3277 3278 VM_STAT_ADD(pagecnt.pc_destroy_free); 3279 page_list_sub(pp, PG_CACHE_LIST); 3280 3281 page_hashout(pp, NULL); 3282 ASSERT(pp->p_vnode == NULL); 3283 ASSERT(pp->p_offset == (u_offset_t)-1); 3284 ASSERT(pp->p_hash == NULL); 3285 3286 PP_SETAGED(pp); 3287 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3288 page_unlock(pp); 3289 3290 mutex_enter(&new_freemem_lock); 3291 if (freemem_wait) { 3292 cv_signal(&freemem_cv); 3293 } 3294 mutex_exit(&new_freemem_lock); 3295 } 3296 3297 /* 3298 * Rename the page "opp" to have an identity specified 3299 * by [vp, off]. If a page already exists with this name 3300 * it is locked and destroyed. Note that the page's 3301 * translations are not unloaded during the rename. 3302 * 3303 * This routine is used by the anon layer to "steal" the 3304 * original page and is not unlike destroying a page and 3305 * creating a new page using the same page frame. 3306 * 3307 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3308 * caller 2 tries to rename B to A. 3309 */ 3310 void 3311 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3312 { 3313 page_t *pp; 3314 int olckcnt = 0; 3315 int ocowcnt = 0; 3316 kmutex_t *phm; 3317 ulong_t index; 3318 3319 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3320 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3321 ASSERT(PP_ISFREE(opp) == 0); 3322 3323 VM_STAT_ADD(page_rename_count); 3324 3325 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3326 "page rename:pp %p vp %p off %llx", opp, vp, off); 3327 3328 /* 3329 * CacheFS may call page_rename for a large NFS page 3330 * when both CacheFS and NFS mount points are used 3331 * by applications. Demote this large page before 3332 * renaming it, to ensure that there are no "partial" 3333 * large pages left lying around. 3334 */ 3335 if (opp->p_szc != 0) { 3336 vnode_t *ovp = opp->p_vnode; 3337 ASSERT(ovp != NULL); 3338 ASSERT(!IS_SWAPFSVP(ovp)); 3339 ASSERT(ovp != &kvp); 3340 page_demote_vp_pages(opp); 3341 ASSERT(opp->p_szc == 0); 3342 } 3343 3344 page_hashout(opp, NULL); 3345 PP_CLRAGED(opp); 3346 3347 /* 3348 * Acquire the appropriate page hash lock, since 3349 * we're going to rename the page. 3350 */ 3351 index = PAGE_HASH_FUNC(vp, off); 3352 phm = PAGE_HASH_MUTEX(index); 3353 mutex_enter(phm); 3354 top: 3355 /* 3356 * Look for an existing page with this name and destroy it if found. 3357 * By holding the page hash lock all the way to the page_hashin() 3358 * call, we are assured that no page can be created with this 3359 * identity. In the case when the phm lock is dropped to undo any 3360 * hat layer mappings, the existing page is held with an "exclusive" 3361 * lock, again preventing another page from being created with 3362 * this identity. 3363 */ 3364 PAGE_HASH_SEARCH(index, pp, vp, off); 3365 if (pp != NULL) { 3366 VM_STAT_ADD(page_rename_exists); 3367 3368 /* 3369 * As it turns out, this is one of only two places where 3370 * page_lock() needs to hold the passed in lock in the 3371 * successful case. In all of the others, the lock could 3372 * be dropped as soon as the attempt is made to lock 3373 * the page. It is tempting to add yet another arguement, 3374 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3375 */ 3376 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3377 /* 3378 * Went to sleep because the page could not 3379 * be locked. We were woken up when the page 3380 * was unlocked, or when the page was destroyed. 3381 * In either case, `phm' was dropped while we 3382 * slept. Hence we should not just roar through 3383 * this loop. 3384 */ 3385 goto top; 3386 } 3387 3388 /* 3389 * If an existing page is a large page, then demote 3390 * it to ensure that no "partial" large pages are 3391 * "created" after page_rename. An existing page 3392 * can be a CacheFS page, and can't belong to swapfs. 3393 */ 3394 if (hat_page_is_mapped(pp)) { 3395 /* 3396 * Unload translations. Since we hold the 3397 * exclusive lock on this page, the page 3398 * can not be changed while we drop phm. 3399 * This is also not a lock protocol violation, 3400 * but rather the proper way to do things. 3401 */ 3402 mutex_exit(phm); 3403 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3404 if (pp->p_szc != 0) { 3405 ASSERT(!IS_SWAPFSVP(vp)); 3406 ASSERT(vp != &kvp); 3407 page_demote_vp_pages(pp); 3408 ASSERT(pp->p_szc == 0); 3409 } 3410 mutex_enter(phm); 3411 } else if (pp->p_szc != 0) { 3412 ASSERT(!IS_SWAPFSVP(vp)); 3413 ASSERT(vp != &kvp); 3414 mutex_exit(phm); 3415 page_demote_vp_pages(pp); 3416 ASSERT(pp->p_szc == 0); 3417 mutex_enter(phm); 3418 } 3419 page_hashout(pp, phm); 3420 } 3421 /* 3422 * Hash in the page with the new identity. 3423 */ 3424 if (!page_hashin(opp, vp, off, phm)) { 3425 /* 3426 * We were holding phm while we searched for [vp, off] 3427 * and only dropped phm if we found and locked a page. 3428 * If we can't create this page now, then some thing 3429 * is really broken. 3430 */ 3431 panic("page_rename: Can't hash in page: %p", (void *)pp); 3432 /*NOTREACHED*/ 3433 } 3434 3435 ASSERT(MUTEX_HELD(phm)); 3436 mutex_exit(phm); 3437 3438 /* 3439 * Now that we have dropped phm, lets get around to finishing up 3440 * with pp. 3441 */ 3442 if (pp != NULL) { 3443 ASSERT(!hat_page_is_mapped(pp)); 3444 /* for now large pages should not end up here */ 3445 ASSERT(pp->p_szc == 0); 3446 /* 3447 * Save the locks for transfer to the new page and then 3448 * clear them so page_free doesn't think they're important. 3449 * The page_struct_lock need not be acquired for lckcnt and 3450 * cowcnt since the page has an "exclusive" lock. 3451 */ 3452 olckcnt = pp->p_lckcnt; 3453 ocowcnt = pp->p_cowcnt; 3454 pp->p_lckcnt = pp->p_cowcnt = 0; 3455 3456 /* 3457 * Put the page on the "free" list after we drop 3458 * the lock. The less work under the lock the better. 3459 */ 3460 /*LINTED: constant in conditional context*/ 3461 VN_DISPOSE(pp, B_FREE, 0, kcred); 3462 } 3463 3464 /* 3465 * Transfer the lock count from the old page (if any). 3466 * The page_struct_lock need not be acquired for lckcnt and 3467 * cowcnt since the page has an "exclusive" lock. 3468 */ 3469 opp->p_lckcnt += olckcnt; 3470 opp->p_cowcnt += ocowcnt; 3471 } 3472 3473 /* 3474 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3475 * 3476 * Pages are normally inserted at the start of a vnode's v_pages list. 3477 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3478 * This can happen when a modified page is relocated for DR. 3479 * 3480 * Returns 1 on success and 0 on failure. 3481 */ 3482 static int 3483 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3484 { 3485 page_t **listp; 3486 page_t *tp; 3487 ulong_t index; 3488 3489 ASSERT(PAGE_EXCL(pp)); 3490 ASSERT(vp != NULL); 3491 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3492 3493 /* 3494 * Be sure to set these up before the page is inserted on the hash 3495 * list. As soon as the page is placed on the list some other 3496 * thread might get confused and wonder how this page could 3497 * possibly hash to this list. 3498 */ 3499 pp->p_vnode = vp; 3500 pp->p_offset = offset; 3501 3502 /* 3503 * record if this page is on a swap vnode 3504 */ 3505 if ((vp->v_flag & VISSWAP) != 0) 3506 PP_SETSWAP(pp); 3507 3508 index = PAGE_HASH_FUNC(vp, offset); 3509 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3510 listp = &page_hash[index]; 3511 3512 /* 3513 * If this page is already hashed in, fail this attempt to add it. 3514 */ 3515 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3516 if (tp->p_vnode == vp && tp->p_offset == offset) { 3517 pp->p_vnode = NULL; 3518 pp->p_offset = (u_offset_t)(-1); 3519 return (0); 3520 } 3521 } 3522 pp->p_hash = *listp; 3523 *listp = pp; 3524 3525 /* 3526 * Add the page to the vnode's list of pages 3527 */ 3528 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3529 listp = &vp->v_pages->p_vpprev->p_vpnext; 3530 else 3531 listp = &vp->v_pages; 3532 3533 page_vpadd(listp, pp); 3534 3535 return (1); 3536 } 3537 3538 /* 3539 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3540 * 3541 * Returns 1 on success and 0 on failure. 3542 * If hold is passed in, it is not dropped. 3543 */ 3544 int 3545 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3546 { 3547 kmutex_t *phm = NULL; 3548 kmutex_t *vphm; 3549 int rc; 3550 3551 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3552 3553 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3554 "page_hashin:pp %p vp %p offset %llx", 3555 pp, vp, offset); 3556 3557 VM_STAT_ADD(hashin_count); 3558 3559 if (hold != NULL) 3560 phm = hold; 3561 else { 3562 VM_STAT_ADD(hashin_not_held); 3563 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3564 mutex_enter(phm); 3565 } 3566 3567 vphm = page_vnode_mutex(vp); 3568 mutex_enter(vphm); 3569 rc = page_do_hashin(pp, vp, offset); 3570 mutex_exit(vphm); 3571 if (hold == NULL) 3572 mutex_exit(phm); 3573 if (rc == 0) 3574 VM_STAT_ADD(hashin_already); 3575 return (rc); 3576 } 3577 3578 /* 3579 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3580 * All mutexes must be held 3581 */ 3582 static void 3583 page_do_hashout(page_t *pp) 3584 { 3585 page_t **hpp; 3586 page_t *hp; 3587 vnode_t *vp = pp->p_vnode; 3588 3589 ASSERT(vp != NULL); 3590 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3591 3592 /* 3593 * First, take pp off of its hash chain. 3594 */ 3595 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3596 3597 for (;;) { 3598 hp = *hpp; 3599 if (hp == pp) 3600 break; 3601 if (hp == NULL) { 3602 panic("page_do_hashout"); 3603 /*NOTREACHED*/ 3604 } 3605 hpp = &hp->p_hash; 3606 } 3607 *hpp = pp->p_hash; 3608 3609 /* 3610 * Now remove it from its associated vnode. 3611 */ 3612 if (vp->v_pages) 3613 page_vpsub(&vp->v_pages, pp); 3614 3615 pp->p_hash = NULL; 3616 page_clr_all_props(pp); 3617 PP_CLRSWAP(pp); 3618 pp->p_vnode = NULL; 3619 pp->p_offset = (u_offset_t)-1; 3620 } 3621 3622 /* 3623 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3624 * 3625 * When `phm' is non-NULL it contains the address of the mutex protecting the 3626 * hash list pp is on. It is not dropped. 3627 */ 3628 void 3629 page_hashout(page_t *pp, kmutex_t *phm) 3630 { 3631 vnode_t *vp; 3632 ulong_t index; 3633 kmutex_t *nphm; 3634 kmutex_t *vphm; 3635 kmutex_t *sep; 3636 3637 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3638 ASSERT(pp->p_vnode != NULL); 3639 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3640 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3641 3642 vp = pp->p_vnode; 3643 3644 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3645 "page_hashout:pp %p vp %p", pp, vp); 3646 3647 /* Kernel probe */ 3648 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3649 tnf_opaque, vnode, vp, 3650 tnf_offset, offset, pp->p_offset); 3651 3652 /* 3653 * 3654 */ 3655 VM_STAT_ADD(hashout_count); 3656 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3657 if (phm == NULL) { 3658 VM_STAT_ADD(hashout_not_held); 3659 nphm = PAGE_HASH_MUTEX(index); 3660 mutex_enter(nphm); 3661 } 3662 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3663 3664 3665 /* 3666 * grab page vnode mutex and remove it... 3667 */ 3668 vphm = page_vnode_mutex(vp); 3669 mutex_enter(vphm); 3670 3671 page_do_hashout(pp); 3672 3673 mutex_exit(vphm); 3674 if (phm == NULL) 3675 mutex_exit(nphm); 3676 3677 /* 3678 * Wake up processes waiting for this page. The page's 3679 * identity has been changed, and is probably not the 3680 * desired page any longer. 3681 */ 3682 sep = page_se_mutex(pp); 3683 mutex_enter(sep); 3684 pp->p_selock &= ~SE_EWANTED; 3685 if (CV_HAS_WAITERS(&pp->p_cv)) 3686 cv_broadcast(&pp->p_cv); 3687 mutex_exit(sep); 3688 } 3689 3690 /* 3691 * Add the page to the front of a linked list of pages 3692 * using the p_next & p_prev pointers for the list. 3693 * The caller is responsible for protecting the list pointers. 3694 */ 3695 void 3696 page_add(page_t **ppp, page_t *pp) 3697 { 3698 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3699 3700 page_add_common(ppp, pp); 3701 } 3702 3703 3704 3705 /* 3706 * Common code for page_add() and mach_page_add() 3707 */ 3708 void 3709 page_add_common(page_t **ppp, page_t *pp) 3710 { 3711 if (*ppp == NULL) { 3712 pp->p_next = pp->p_prev = pp; 3713 } else { 3714 pp->p_next = *ppp; 3715 pp->p_prev = (*ppp)->p_prev; 3716 (*ppp)->p_prev = pp; 3717 pp->p_prev->p_next = pp; 3718 } 3719 *ppp = pp; 3720 } 3721 3722 3723 /* 3724 * Remove this page from a linked list of pages 3725 * using the p_next & p_prev pointers for the list. 3726 * 3727 * The caller is responsible for protecting the list pointers. 3728 */ 3729 void 3730 page_sub(page_t **ppp, page_t *pp) 3731 { 3732 ASSERT((PP_ISFREE(pp)) ? 1 : 3733 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3734 3735 if (*ppp == NULL || pp == NULL) { 3736 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3737 (void *)pp, (void *)(*ppp)); 3738 /*NOTREACHED*/ 3739 } 3740 3741 page_sub_common(ppp, pp); 3742 } 3743 3744 3745 /* 3746 * Common code for page_sub() and mach_page_sub() 3747 */ 3748 void 3749 page_sub_common(page_t **ppp, page_t *pp) 3750 { 3751 if (*ppp == pp) 3752 *ppp = pp->p_next; /* go to next page */ 3753 3754 if (*ppp == pp) 3755 *ppp = NULL; /* page list is gone */ 3756 else { 3757 pp->p_prev->p_next = pp->p_next; 3758 pp->p_next->p_prev = pp->p_prev; 3759 } 3760 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3761 } 3762 3763 3764 /* 3765 * Break page list cppp into two lists with npages in the first list. 3766 * The tail is returned in nppp. 3767 */ 3768 void 3769 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3770 { 3771 page_t *s1pp = *oppp; 3772 page_t *s2pp; 3773 page_t *e1pp, *e2pp; 3774 long n = 0; 3775 3776 if (s1pp == NULL) { 3777 *nppp = NULL; 3778 return; 3779 } 3780 if (npages == 0) { 3781 *nppp = s1pp; 3782 *oppp = NULL; 3783 return; 3784 } 3785 for (n = 0, s2pp = *oppp; n < npages; n++) { 3786 s2pp = s2pp->p_next; 3787 } 3788 /* Fix head and tail of new lists */ 3789 e1pp = s2pp->p_prev; 3790 e2pp = s1pp->p_prev; 3791 s1pp->p_prev = e1pp; 3792 e1pp->p_next = s1pp; 3793 s2pp->p_prev = e2pp; 3794 e2pp->p_next = s2pp; 3795 3796 /* second list empty */ 3797 if (s2pp == s1pp) { 3798 *oppp = s1pp; 3799 *nppp = NULL; 3800 } else { 3801 *oppp = s1pp; 3802 *nppp = s2pp; 3803 } 3804 } 3805 3806 /* 3807 * Concatenate page list nppp onto the end of list ppp. 3808 */ 3809 void 3810 page_list_concat(page_t **ppp, page_t **nppp) 3811 { 3812 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3813 3814 if (*nppp == NULL) { 3815 return; 3816 } 3817 if (*ppp == NULL) { 3818 *ppp = *nppp; 3819 return; 3820 } 3821 s1pp = *ppp; 3822 e1pp = s1pp->p_prev; 3823 s2pp = *nppp; 3824 e2pp = s2pp->p_prev; 3825 s1pp->p_prev = e2pp; 3826 e2pp->p_next = s1pp; 3827 e1pp->p_next = s2pp; 3828 s2pp->p_prev = e1pp; 3829 } 3830 3831 /* 3832 * return the next page in the page list 3833 */ 3834 page_t * 3835 page_list_next(page_t *pp) 3836 { 3837 return (pp->p_next); 3838 } 3839 3840 3841 /* 3842 * Add the page to the front of the linked list of pages 3843 * using p_vpnext/p_vpprev pointers for the list. 3844 * 3845 * The caller is responsible for protecting the lists. 3846 */ 3847 void 3848 page_vpadd(page_t **ppp, page_t *pp) 3849 { 3850 if (*ppp == NULL) { 3851 pp->p_vpnext = pp->p_vpprev = pp; 3852 } else { 3853 pp->p_vpnext = *ppp; 3854 pp->p_vpprev = (*ppp)->p_vpprev; 3855 (*ppp)->p_vpprev = pp; 3856 pp->p_vpprev->p_vpnext = pp; 3857 } 3858 *ppp = pp; 3859 } 3860 3861 /* 3862 * Remove this page from the linked list of pages 3863 * using p_vpnext/p_vpprev pointers for the list. 3864 * 3865 * The caller is responsible for protecting the lists. 3866 */ 3867 void 3868 page_vpsub(page_t **ppp, page_t *pp) 3869 { 3870 if (*ppp == NULL || pp == NULL) { 3871 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3872 (void *)pp, (void *)(*ppp)); 3873 /*NOTREACHED*/ 3874 } 3875 3876 if (*ppp == pp) 3877 *ppp = pp->p_vpnext; /* go to next page */ 3878 3879 if (*ppp == pp) 3880 *ppp = NULL; /* page list is gone */ 3881 else { 3882 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3883 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3884 } 3885 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3886 } 3887 3888 /* 3889 * Lock a physical page into memory "long term". Used to support "lock 3890 * in memory" functions. Accepts the page to be locked, and a cow variable 3891 * to indicate whether a the lock will travel to the new page during 3892 * a potential copy-on-write. 3893 */ 3894 int 3895 page_pp_lock( 3896 page_t *pp, /* page to be locked */ 3897 int cow, /* cow lock */ 3898 int kernel) /* must succeed -- ignore checking */ 3899 { 3900 int r = 0; /* result -- assume failure */ 3901 3902 ASSERT(PAGE_LOCKED(pp)); 3903 3904 page_struct_lock(pp); 3905 /* 3906 * Acquire the "freemem_lock" for availrmem. 3907 */ 3908 if (cow) { 3909 mutex_enter(&freemem_lock); 3910 if ((availrmem > pages_pp_maximum) && 3911 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3912 availrmem--; 3913 pages_locked++; 3914 mutex_exit(&freemem_lock); 3915 r = 1; 3916 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3917 cmn_err(CE_WARN, 3918 "COW lock limit reached on pfn 0x%lx", 3919 page_pptonum(pp)); 3920 } 3921 } else 3922 mutex_exit(&freemem_lock); 3923 } else { 3924 if (pp->p_lckcnt) { 3925 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3926 r = 1; 3927 if (++pp->p_lckcnt == 3928 (ushort_t)PAGE_LOCK_MAXIMUM) { 3929 cmn_err(CE_WARN, "Page lock limit " 3930 "reached on pfn 0x%lx", 3931 page_pptonum(pp)); 3932 } 3933 } 3934 } else { 3935 if (kernel) { 3936 /* availrmem accounting done by caller */ 3937 ++pp->p_lckcnt; 3938 r = 1; 3939 } else { 3940 mutex_enter(&freemem_lock); 3941 if (availrmem > pages_pp_maximum) { 3942 availrmem--; 3943 pages_locked++; 3944 ++pp->p_lckcnt; 3945 r = 1; 3946 } 3947 mutex_exit(&freemem_lock); 3948 } 3949 } 3950 } 3951 page_struct_unlock(pp); 3952 return (r); 3953 } 3954 3955 /* 3956 * Decommit a lock on a physical page frame. Account for cow locks if 3957 * appropriate. 3958 */ 3959 void 3960 page_pp_unlock( 3961 page_t *pp, /* page to be unlocked */ 3962 int cow, /* expect cow lock */ 3963 int kernel) /* this was a kernel lock */ 3964 { 3965 ASSERT(PAGE_LOCKED(pp)); 3966 3967 page_struct_lock(pp); 3968 /* 3969 * Acquire the "freemem_lock" for availrmem. 3970 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3971 * could be called to unlock even if nothing is locked. This could 3972 * happen if locked file pages were truncated (removing the lock) 3973 * and the file was grown again and new pages faulted in; the new 3974 * pages are unlocked but the segment still thinks they're locked. 3975 */ 3976 if (cow) { 3977 if (pp->p_cowcnt) { 3978 mutex_enter(&freemem_lock); 3979 pp->p_cowcnt--; 3980 availrmem++; 3981 pages_locked--; 3982 mutex_exit(&freemem_lock); 3983 } 3984 } else { 3985 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3986 if (!kernel) { 3987 mutex_enter(&freemem_lock); 3988 availrmem++; 3989 pages_locked--; 3990 mutex_exit(&freemem_lock); 3991 } 3992 } 3993 } 3994 page_struct_unlock(pp); 3995 } 3996 3997 /* 3998 * This routine reserves availrmem for npages; 3999 * flags: KM_NOSLEEP or KM_SLEEP 4000 * returns 1 on success or 0 on failure 4001 */ 4002 int 4003 page_resv(pgcnt_t npages, uint_t flags) 4004 { 4005 mutex_enter(&freemem_lock); 4006 while (availrmem < tune.t_minarmem + npages) { 4007 if (flags & KM_NOSLEEP) { 4008 mutex_exit(&freemem_lock); 4009 return (0); 4010 } 4011 mutex_exit(&freemem_lock); 4012 page_needfree(npages); 4013 kmem_reap(); 4014 delay(hz >> 2); 4015 page_needfree(-(spgcnt_t)npages); 4016 mutex_enter(&freemem_lock); 4017 } 4018 availrmem -= npages; 4019 mutex_exit(&freemem_lock); 4020 return (1); 4021 } 4022 4023 /* 4024 * This routine unreserves availrmem for npages; 4025 */ 4026 void 4027 page_unresv(pgcnt_t npages) 4028 { 4029 mutex_enter(&freemem_lock); 4030 availrmem += npages; 4031 mutex_exit(&freemem_lock); 4032 } 4033 4034 /* 4035 * See Statement at the beginning of segvn_lockop() regarding 4036 * the way we handle cowcnts and lckcnts. 4037 * 4038 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 4039 * that breaks COW has PROT_WRITE. 4040 * 4041 * Note that, we may also break COW in case we are softlocking 4042 * on read access during physio; 4043 * in this softlock case, the vpage may not have PROT_WRITE. 4044 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 4045 * if the vpage doesn't have PROT_WRITE. 4046 * 4047 * This routine is never called if we are stealing a page 4048 * in anon_private. 4049 * 4050 * The caller subtracted from availrmem for read only mapping. 4051 * if lckcnt is 1 increment availrmem. 4052 */ 4053 void 4054 page_pp_useclaim( 4055 page_t *opp, /* original page frame losing lock */ 4056 page_t *npp, /* new page frame gaining lock */ 4057 uint_t write_perm) /* set if vpage has PROT_WRITE */ 4058 { 4059 int payback = 0; 4060 4061 ASSERT(PAGE_LOCKED(opp)); 4062 ASSERT(PAGE_LOCKED(npp)); 4063 4064 page_struct_lock(opp); 4065 4066 ASSERT(npp->p_cowcnt == 0); 4067 ASSERT(npp->p_lckcnt == 0); 4068 4069 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4070 if ((write_perm && opp->p_cowcnt != 0) || 4071 (!write_perm && opp->p_lckcnt != 0)) { 4072 4073 if (write_perm) { 4074 npp->p_cowcnt++; 4075 ASSERT(opp->p_cowcnt != 0); 4076 opp->p_cowcnt--; 4077 } else { 4078 4079 ASSERT(opp->p_lckcnt != 0); 4080 4081 /* 4082 * We didn't need availrmem decremented if p_lckcnt on 4083 * original page is 1. Here, we are unlocking 4084 * read-only copy belonging to original page and 4085 * are locking a copy belonging to new page. 4086 */ 4087 if (opp->p_lckcnt == 1) 4088 payback = 1; 4089 4090 npp->p_lckcnt++; 4091 opp->p_lckcnt--; 4092 } 4093 } 4094 if (payback) { 4095 mutex_enter(&freemem_lock); 4096 availrmem++; 4097 pages_useclaim--; 4098 mutex_exit(&freemem_lock); 4099 } 4100 page_struct_unlock(opp); 4101 } 4102 4103 /* 4104 * Simple claim adjust functions -- used to support changes in 4105 * claims due to changes in access permissions. Used by segvn_setprot(). 4106 */ 4107 int 4108 page_addclaim(page_t *pp) 4109 { 4110 int r = 0; /* result */ 4111 4112 ASSERT(PAGE_LOCKED(pp)); 4113 4114 page_struct_lock(pp); 4115 ASSERT(pp->p_lckcnt != 0); 4116 4117 if (pp->p_lckcnt == 1) { 4118 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4119 --pp->p_lckcnt; 4120 r = 1; 4121 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4122 cmn_err(CE_WARN, 4123 "COW lock limit reached on pfn 0x%lx", 4124 page_pptonum(pp)); 4125 } 4126 } 4127 } else { 4128 mutex_enter(&freemem_lock); 4129 if ((availrmem > pages_pp_maximum) && 4130 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4131 --availrmem; 4132 ++pages_claimed; 4133 mutex_exit(&freemem_lock); 4134 --pp->p_lckcnt; 4135 r = 1; 4136 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4137 cmn_err(CE_WARN, 4138 "COW lock limit reached on pfn 0x%lx", 4139 page_pptonum(pp)); 4140 } 4141 } else 4142 mutex_exit(&freemem_lock); 4143 } 4144 page_struct_unlock(pp); 4145 return (r); 4146 } 4147 4148 int 4149 page_subclaim(page_t *pp) 4150 { 4151 int r = 0; 4152 4153 ASSERT(PAGE_LOCKED(pp)); 4154 4155 page_struct_lock(pp); 4156 ASSERT(pp->p_cowcnt != 0); 4157 4158 if (pp->p_lckcnt) { 4159 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4160 r = 1; 4161 /* 4162 * for availrmem 4163 */ 4164 mutex_enter(&freemem_lock); 4165 availrmem++; 4166 pages_claimed--; 4167 mutex_exit(&freemem_lock); 4168 4169 pp->p_cowcnt--; 4170 4171 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4172 cmn_err(CE_WARN, 4173 "Page lock limit reached on pfn 0x%lx", 4174 page_pptonum(pp)); 4175 } 4176 } 4177 } else { 4178 r = 1; 4179 pp->p_cowcnt--; 4180 pp->p_lckcnt++; 4181 } 4182 page_struct_unlock(pp); 4183 return (r); 4184 } 4185 4186 int 4187 page_addclaim_pages(page_t **ppa) 4188 { 4189 4190 pgcnt_t lckpgs = 0, pg_idx; 4191 4192 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4193 4194 mutex_enter(&page_llock); 4195 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4196 4197 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4198 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4199 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4200 mutex_exit(&page_llock); 4201 return (0); 4202 } 4203 if (ppa[pg_idx]->p_lckcnt > 1) 4204 lckpgs++; 4205 } 4206 4207 if (lckpgs != 0) { 4208 mutex_enter(&freemem_lock); 4209 if (availrmem >= pages_pp_maximum + lckpgs) { 4210 availrmem -= lckpgs; 4211 pages_claimed += lckpgs; 4212 } else { 4213 mutex_exit(&freemem_lock); 4214 mutex_exit(&page_llock); 4215 return (0); 4216 } 4217 mutex_exit(&freemem_lock); 4218 } 4219 4220 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4221 ppa[pg_idx]->p_lckcnt--; 4222 ppa[pg_idx]->p_cowcnt++; 4223 } 4224 mutex_exit(&page_llock); 4225 return (1); 4226 } 4227 4228 int 4229 page_subclaim_pages(page_t **ppa) 4230 { 4231 pgcnt_t ulckpgs = 0, pg_idx; 4232 4233 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4234 4235 mutex_enter(&page_llock); 4236 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4237 4238 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4239 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4240 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4241 mutex_exit(&page_llock); 4242 return (0); 4243 } 4244 if (ppa[pg_idx]->p_lckcnt != 0) 4245 ulckpgs++; 4246 } 4247 4248 if (ulckpgs != 0) { 4249 mutex_enter(&freemem_lock); 4250 availrmem += ulckpgs; 4251 pages_claimed -= ulckpgs; 4252 mutex_exit(&freemem_lock); 4253 } 4254 4255 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4256 ppa[pg_idx]->p_cowcnt--; 4257 ppa[pg_idx]->p_lckcnt++; 4258 4259 } 4260 mutex_exit(&page_llock); 4261 return (1); 4262 } 4263 4264 page_t * 4265 page_numtopp(pfn_t pfnum, se_t se) 4266 { 4267 page_t *pp; 4268 4269 retry: 4270 pp = page_numtopp_nolock(pfnum); 4271 if (pp == NULL) { 4272 return ((page_t *)NULL); 4273 } 4274 4275 /* 4276 * Acquire the appropriate lock on the page. 4277 */ 4278 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4279 if (page_pptonum(pp) != pfnum) 4280 goto retry; 4281 continue; 4282 } 4283 4284 if (page_pptonum(pp) != pfnum) { 4285 page_unlock(pp); 4286 goto retry; 4287 } 4288 4289 return (pp); 4290 } 4291 4292 page_t * 4293 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4294 { 4295 page_t *pp; 4296 4297 retry: 4298 pp = page_numtopp_nolock(pfnum); 4299 if (pp == NULL) { 4300 return ((page_t *)NULL); 4301 } 4302 4303 /* 4304 * Acquire the appropriate lock on the page. 4305 */ 4306 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4307 if (page_pptonum(pp) != pfnum) 4308 goto retry; 4309 continue; 4310 } 4311 4312 if (page_pptonum(pp) != pfnum) { 4313 page_unlock(pp); 4314 goto retry; 4315 } 4316 4317 return (pp); 4318 } 4319 4320 /* 4321 * This routine is like page_numtopp, but will only return page structs 4322 * for pages which are ok for loading into hardware using the page struct. 4323 */ 4324 page_t * 4325 page_numtopp_nowait(pfn_t pfnum, se_t se) 4326 { 4327 page_t *pp; 4328 4329 retry: 4330 pp = page_numtopp_nolock(pfnum); 4331 if (pp == NULL) { 4332 return ((page_t *)NULL); 4333 } 4334 4335 /* 4336 * Try to acquire the appropriate lock on the page. 4337 */ 4338 if (PP_ISFREE(pp)) 4339 pp = NULL; 4340 else { 4341 if (!page_trylock(pp, se)) 4342 pp = NULL; 4343 else { 4344 if (page_pptonum(pp) != pfnum) { 4345 page_unlock(pp); 4346 goto retry; 4347 } 4348 if (PP_ISFREE(pp)) { 4349 page_unlock(pp); 4350 pp = NULL; 4351 } 4352 } 4353 } 4354 return (pp); 4355 } 4356 4357 /* 4358 * Returns a count of dirty pages that are in the process 4359 * of being written out. If 'cleanit' is set, try to push the page. 4360 */ 4361 pgcnt_t 4362 page_busy(int cleanit) 4363 { 4364 page_t *page0 = page_first(); 4365 page_t *pp = page0; 4366 pgcnt_t nppbusy = 0; 4367 u_offset_t off; 4368 4369 do { 4370 vnode_t *vp = pp->p_vnode; 4371 4372 /* 4373 * A page is a candidate for syncing if it is: 4374 * 4375 * (a) On neither the freelist nor the cachelist 4376 * (b) Hashed onto a vnode 4377 * (c) Not a kernel page 4378 * (d) Dirty 4379 * (e) Not part of a swapfile 4380 * (f) a page which belongs to a real vnode; eg has a non-null 4381 * v_vfsp pointer. 4382 * (g) Backed by a filesystem which doesn't have a 4383 * stubbed-out sync operation 4384 */ 4385 if (!PP_ISFREE(pp) && vp != NULL && vp != &kvp && 4386 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4387 vfs_can_sync(vp->v_vfsp)) { 4388 nppbusy++; 4389 vfs_syncprogress(); 4390 4391 if (!cleanit) 4392 continue; 4393 if (!page_trylock(pp, SE_EXCL)) 4394 continue; 4395 4396 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4397 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4398 !(hat_pagesync(pp, 4399 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4400 page_unlock(pp); 4401 continue; 4402 } 4403 off = pp->p_offset; 4404 VN_HOLD(vp); 4405 page_unlock(pp); 4406 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4407 B_ASYNC | B_FREE, kcred); 4408 VN_RELE(vp); 4409 } 4410 } while ((pp = page_next(pp)) != page0); 4411 4412 return (nppbusy); 4413 } 4414 4415 void page_invalidate_pages(void); 4416 4417 /* 4418 * callback handler to vm sub-system 4419 * 4420 * callers make sure no recursive entries to this func. 4421 */ 4422 /*ARGSUSED*/ 4423 boolean_t 4424 callb_vm_cpr(void *arg, int code) 4425 { 4426 if (code == CB_CODE_CPR_CHKPT) 4427 page_invalidate_pages(); 4428 return (B_TRUE); 4429 } 4430 4431 /* 4432 * Invalidate all pages of the system. 4433 * It shouldn't be called until all user page activities are all stopped. 4434 */ 4435 void 4436 page_invalidate_pages() 4437 { 4438 page_t *pp; 4439 page_t *page0; 4440 pgcnt_t nbusypages; 4441 int retry = 0; 4442 const int MAXRETRIES = 4; 4443 #if defined(__sparc) 4444 extern struct vnode prom_ppages; 4445 #endif /* __sparc */ 4446 4447 top: 4448 /* 4449 * Flush dirty pages and destory the clean ones. 4450 */ 4451 nbusypages = 0; 4452 4453 pp = page0 = page_first(); 4454 do { 4455 struct vnode *vp; 4456 u_offset_t offset; 4457 int mod; 4458 4459 /* 4460 * skip the page if it has no vnode or the page associated 4461 * with the kernel vnode or prom allocated kernel mem. 4462 */ 4463 #if defined(__sparc) 4464 if ((vp = pp->p_vnode) == NULL || vp == &kvp || 4465 vp == &prom_ppages) 4466 #else /* x86 doesn't have prom or prom_ppage */ 4467 if ((vp = pp->p_vnode) == NULL || vp == &kvp) 4468 #endif /* __sparc */ 4469 continue; 4470 4471 /* 4472 * skip the page which is already free invalidated. 4473 */ 4474 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4475 continue; 4476 4477 /* 4478 * skip pages that are already locked or can't be "exclusively" 4479 * locked or are already free. After we lock the page, check 4480 * the free and age bits again to be sure it's not destroied 4481 * yet. 4482 * To achieve max. parallelization, we use page_trylock instead 4483 * of page_lock so that we don't get block on individual pages 4484 * while we have thousands of other pages to process. 4485 */ 4486 if (!page_trylock(pp, SE_EXCL)) { 4487 nbusypages++; 4488 continue; 4489 } else if (PP_ISFREE(pp)) { 4490 if (!PP_ISAGED(pp)) { 4491 page_destroy_free(pp); 4492 } else { 4493 page_unlock(pp); 4494 } 4495 continue; 4496 } 4497 /* 4498 * Is this page involved in some I/O? shared? 4499 * 4500 * The page_struct_lock need not be acquired to 4501 * examine these fields since the page has an 4502 * "exclusive" lock. 4503 */ 4504 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4505 page_unlock(pp); 4506 continue; 4507 } 4508 4509 if (vp->v_type == VCHR) { 4510 panic("vp->v_type == VCHR"); 4511 /*NOTREACHED*/ 4512 } 4513 4514 if (!page_try_demote_pages(pp)) { 4515 page_unlock(pp); 4516 continue; 4517 } 4518 4519 /* 4520 * Check the modified bit. Leave the bits alone in hardware 4521 * (they will be modified if we do the putpage). 4522 */ 4523 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4524 & P_MOD); 4525 if (mod) { 4526 offset = pp->p_offset; 4527 /* 4528 * Hold the vnode before releasing the page lock 4529 * to prevent it from being freed and re-used by 4530 * some other thread. 4531 */ 4532 VN_HOLD(vp); 4533 page_unlock(pp); 4534 /* 4535 * No error return is checked here. Callers such as 4536 * cpr deals with the dirty pages at the dump time 4537 * if this putpage fails. 4538 */ 4539 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4540 kcred); 4541 VN_RELE(vp); 4542 } else { 4543 page_destroy(pp, 0); 4544 } 4545 } while ((pp = page_next(pp)) != page0); 4546 if (nbusypages && retry++ < MAXRETRIES) { 4547 delay(1); 4548 goto top; 4549 } 4550 } 4551 4552 /* 4553 * Replace the page "old" with the page "new" on the page hash and vnode lists 4554 * 4555 * the replacemnt must be done in place, ie the equivalent sequence: 4556 * 4557 * vp = old->p_vnode; 4558 * off = old->p_offset; 4559 * page_do_hashout(old) 4560 * page_do_hashin(new, vp, off) 4561 * 4562 * doesn't work, since 4563 * 1) if old is the only page on the vnode, the v_pages list has a window 4564 * where it looks empty. This will break file system assumptions. 4565 * and 4566 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4567 */ 4568 static void 4569 page_do_relocate_hash(page_t *new, page_t *old) 4570 { 4571 page_t **hash_list; 4572 vnode_t *vp = old->p_vnode; 4573 kmutex_t *sep; 4574 4575 ASSERT(PAGE_EXCL(old)); 4576 ASSERT(PAGE_EXCL(new)); 4577 ASSERT(vp != NULL); 4578 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4579 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4580 4581 /* 4582 * First find old page on the page hash list 4583 */ 4584 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4585 4586 for (;;) { 4587 if (*hash_list == old) 4588 break; 4589 if (*hash_list == NULL) { 4590 panic("page_do_hashout"); 4591 /*NOTREACHED*/ 4592 } 4593 hash_list = &(*hash_list)->p_hash; 4594 } 4595 4596 /* 4597 * update new and replace old with new on the page hash list 4598 */ 4599 new->p_vnode = old->p_vnode; 4600 new->p_offset = old->p_offset; 4601 new->p_hash = old->p_hash; 4602 *hash_list = new; 4603 4604 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4605 PP_SETSWAP(new); 4606 4607 /* 4608 * replace old with new on the vnode's page list 4609 */ 4610 if (old->p_vpnext == old) { 4611 new->p_vpnext = new; 4612 new->p_vpprev = new; 4613 } else { 4614 new->p_vpnext = old->p_vpnext; 4615 new->p_vpprev = old->p_vpprev; 4616 new->p_vpnext->p_vpprev = new; 4617 new->p_vpprev->p_vpnext = new; 4618 } 4619 if (vp->v_pages == old) 4620 vp->v_pages = new; 4621 4622 /* 4623 * clear out the old page 4624 */ 4625 old->p_hash = NULL; 4626 old->p_vpnext = NULL; 4627 old->p_vpprev = NULL; 4628 old->p_vnode = NULL; 4629 PP_CLRSWAP(old); 4630 old->p_offset = (u_offset_t)-1; 4631 page_clr_all_props(old); 4632 4633 /* 4634 * Wake up processes waiting for this page. The page's 4635 * identity has been changed, and is probably not the 4636 * desired page any longer. 4637 */ 4638 sep = page_se_mutex(old); 4639 mutex_enter(sep); 4640 old->p_selock &= ~SE_EWANTED; 4641 if (CV_HAS_WAITERS(&old->p_cv)) 4642 cv_broadcast(&old->p_cv); 4643 mutex_exit(sep); 4644 } 4645 4646 /* 4647 * This function moves the identity of page "pp_old" to page "pp_new". 4648 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4649 * and need not be hashed out from anywhere. 4650 */ 4651 void 4652 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4653 { 4654 vnode_t *vp = pp_old->p_vnode; 4655 u_offset_t off = pp_old->p_offset; 4656 kmutex_t *phm, *vphm; 4657 4658 /* 4659 * Rehash two pages 4660 */ 4661 ASSERT(PAGE_EXCL(pp_old)); 4662 ASSERT(PAGE_EXCL(pp_new)); 4663 ASSERT(vp != NULL); 4664 ASSERT(pp_new->p_vnode == NULL); 4665 4666 /* 4667 * hashout then hashin while holding the mutexes 4668 */ 4669 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4670 mutex_enter(phm); 4671 vphm = page_vnode_mutex(vp); 4672 mutex_enter(vphm); 4673 4674 page_do_relocate_hash(pp_new, pp_old); 4675 4676 mutex_exit(vphm); 4677 mutex_exit(phm); 4678 4679 /* 4680 * The page_struct_lock need not be acquired for lckcnt and 4681 * cowcnt since the page has an "exclusive" lock. 4682 */ 4683 ASSERT(pp_new->p_lckcnt == 0); 4684 ASSERT(pp_new->p_cowcnt == 0); 4685 pp_new->p_lckcnt = pp_old->p_lckcnt; 4686 pp_new->p_cowcnt = pp_old->p_cowcnt; 4687 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4688 4689 /* The following comment preserved from page_flip(). */ 4690 /* XXX - Do we need to protect fsdata? */ 4691 pp_new->p_fsdata = pp_old->p_fsdata; 4692 } 4693 4694 /* 4695 * Helper routine used to lock all remaining members of a 4696 * large page. The caller is responsible for passing in a locked 4697 * pp. If pp is a large page, then it succeeds in locking all the 4698 * remaining constituent pages or it returns with only the 4699 * original page locked. 4700 * 4701 * Returns 1 on success, 0 on failure. 4702 * 4703 * If success is returned this routine gurantees p_szc for all constituent 4704 * pages of a large page pp belongs to can't change. To achieve this we 4705 * recheck szc of pp after locking all constituent pages and retry if szc 4706 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4707 * lock on one of constituent pages it can't be running after all constituent 4708 * pages are locked. hat_page_demote() with a lock on a constituent page 4709 * outside of this large page (i.e. pp belonged to a larger large page) is 4710 * already done with all constituent pages of pp since the root's p_szc is 4711 * changed last. Thefore no need to synchronize with hat_page_demote() that 4712 * locked a constituent page outside of pp's current large page. 4713 */ 4714 #ifdef DEBUG 4715 uint32_t gpg_trylock_mtbf = 0; 4716 #endif 4717 4718 int 4719 group_page_trylock(page_t *pp, se_t se) 4720 { 4721 page_t *tpp; 4722 pgcnt_t npgs, i, j; 4723 uint_t pszc = pp->p_szc; 4724 4725 #ifdef DEBUG 4726 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4727 return (0); 4728 } 4729 #endif 4730 4731 if (pp != PP_GROUPLEADER(pp, pszc)) { 4732 return (0); 4733 } 4734 4735 retry: 4736 ASSERT(PAGE_LOCKED_SE(pp, se)); 4737 ASSERT(!PP_ISFREE(pp)); 4738 if (pszc == 0) { 4739 return (1); 4740 } 4741 npgs = page_get_pagecnt(pszc); 4742 tpp = pp + 1; 4743 for (i = 1; i < npgs; i++, tpp++) { 4744 if (!page_trylock(tpp, se)) { 4745 tpp = pp + 1; 4746 for (j = 1; j < i; j++, tpp++) { 4747 page_unlock(tpp); 4748 } 4749 return (0); 4750 } 4751 } 4752 if (pp->p_szc != pszc) { 4753 ASSERT(pp->p_szc < pszc); 4754 ASSERT(pp->p_vnode != NULL && pp->p_vnode != &kvp && 4755 !IS_SWAPFSVP(pp->p_vnode)); 4756 tpp = pp + 1; 4757 for (i = 1; i < npgs; i++, tpp++) { 4758 page_unlock(tpp); 4759 } 4760 pszc = pp->p_szc; 4761 goto retry; 4762 } 4763 return (1); 4764 } 4765 4766 void 4767 group_page_unlock(page_t *pp) 4768 { 4769 page_t *tpp; 4770 pgcnt_t npgs, i; 4771 4772 ASSERT(PAGE_LOCKED(pp)); 4773 ASSERT(!PP_ISFREE(pp)); 4774 ASSERT(pp == PP_PAGEROOT(pp)); 4775 npgs = page_get_pagecnt(pp->p_szc); 4776 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4777 page_unlock(tpp); 4778 } 4779 } 4780 4781 /* 4782 * returns 4783 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4784 * ERANGE : this is not a base page 4785 * EBUSY : failure to get locks on the page/pages 4786 * ENOMEM : failure to obtain replacement pages 4787 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4788 * 4789 * Return with all constituent members of target and replacement 4790 * SE_EXCL locked. It is the callers responsibility to drop the 4791 * locks. 4792 */ 4793 int 4794 do_page_relocate( 4795 page_t **target, 4796 page_t **replacement, 4797 int grouplock, 4798 spgcnt_t *nrelocp, 4799 lgrp_t *lgrp) 4800 { 4801 #ifdef DEBUG 4802 page_t *first_repl; 4803 #endif /* DEBUG */ 4804 page_t *repl; 4805 page_t *targ; 4806 page_t *pl = NULL; 4807 uint_t ppattr; 4808 pfn_t pfn, repl_pfn; 4809 uint_t szc; 4810 spgcnt_t npgs, i; 4811 int repl_contig = 0; 4812 uint_t flags = 0; 4813 spgcnt_t dofree = 0; 4814 4815 *nrelocp = 0; 4816 4817 #if defined(__sparc) 4818 /* 4819 * We need to wait till OBP has completed 4820 * its boot-time handoff of its resources to the kernel 4821 * before we allow page relocation 4822 */ 4823 if (page_relocate_ready == 0) { 4824 return (EAGAIN); 4825 } 4826 #endif 4827 4828 /* 4829 * If this is not a base page, 4830 * just return with 0x0 pages relocated. 4831 */ 4832 targ = *target; 4833 ASSERT(PAGE_EXCL(targ)); 4834 ASSERT(!PP_ISFREE(targ)); 4835 szc = targ->p_szc; 4836 ASSERT(szc < mmu_page_sizes); 4837 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4838 pfn = targ->p_pagenum; 4839 if (pfn != PFN_BASE(pfn, szc)) { 4840 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4841 return (ERANGE); 4842 } 4843 4844 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4845 repl_pfn = repl->p_pagenum; 4846 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4847 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4848 return (ERANGE); 4849 } 4850 repl_contig = 1; 4851 } 4852 4853 /* 4854 * We must lock all members of this large page or we cannot 4855 * relocate any part of it. 4856 */ 4857 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4858 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4859 return (EBUSY); 4860 } 4861 4862 /* 4863 * reread szc it could have been decreased before 4864 * group_page_trylock() was done. 4865 */ 4866 szc = targ->p_szc; 4867 ASSERT(szc < mmu_page_sizes); 4868 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4869 ASSERT(pfn == PFN_BASE(pfn, szc)); 4870 4871 npgs = page_get_pagecnt(targ->p_szc); 4872 4873 if (repl == NULL) { 4874 dofree = npgs; /* Size of target page in MMU pages */ 4875 if (!page_create_wait(dofree, 0)) { 4876 if (grouplock != 0) { 4877 group_page_unlock(targ); 4878 } 4879 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4880 return (ENOMEM); 4881 } 4882 4883 /* 4884 * seg kmem pages require that the target and replacement 4885 * page be the same pagesize. 4886 */ 4887 flags = (targ->p_vnode == &kvp) ? PGR_SAMESZC : 0; 4888 repl = page_get_replacement_page(targ, lgrp, flags); 4889 if (repl == NULL) { 4890 if (grouplock != 0) { 4891 group_page_unlock(targ); 4892 } 4893 page_create_putback(dofree); 4894 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4895 return (ENOMEM); 4896 } 4897 } 4898 #ifdef DEBUG 4899 else { 4900 ASSERT(PAGE_LOCKED(repl)); 4901 } 4902 #endif /* DEBUG */ 4903 4904 #if defined(__sparc) 4905 /* 4906 * Let hat_page_relocate() complete the relocation if it's kernel page 4907 */ 4908 if (targ->p_vnode == &kvp) { 4909 *replacement = repl; 4910 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4911 if (grouplock != 0) { 4912 group_page_unlock(targ); 4913 } 4914 if (dofree) { 4915 *replacement = NULL; 4916 page_free_replacement_page(repl); 4917 page_create_putback(dofree); 4918 } 4919 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4920 return (EAGAIN); 4921 } 4922 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4923 return (0); 4924 } 4925 #else 4926 #if defined(lint) 4927 dofree = dofree; 4928 #endif 4929 #endif 4930 4931 #ifdef DEBUG 4932 first_repl = repl; 4933 #endif /* DEBUG */ 4934 4935 for (i = 0; i < npgs; i++) { 4936 ASSERT(PAGE_EXCL(targ)); 4937 4938 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4939 4940 ASSERT(hat_page_getshare(targ) == 0); 4941 ASSERT(!PP_ISFREE(targ)); 4942 ASSERT(targ->p_pagenum == (pfn + i)); 4943 ASSERT(repl_contig == 0 || 4944 repl->p_pagenum == (repl_pfn + i)); 4945 4946 /* 4947 * Copy the page contents and attributes then 4948 * relocate the page in the page hash. 4949 */ 4950 ppcopy(targ, repl); 4951 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4952 page_clr_all_props(repl); 4953 page_set_props(repl, ppattr); 4954 page_relocate_hash(repl, targ); 4955 4956 ASSERT(hat_page_getshare(targ) == 0); 4957 ASSERT(hat_page_getshare(repl) == 0); 4958 /* 4959 * Now clear the props on targ, after the 4960 * page_relocate_hash(), they no longer 4961 * have any meaning. 4962 */ 4963 page_clr_all_props(targ); 4964 ASSERT(targ->p_next == targ); 4965 ASSERT(targ->p_prev == targ); 4966 page_list_concat(&pl, &targ); 4967 4968 targ++; 4969 if (repl_contig != 0) { 4970 repl++; 4971 } else { 4972 repl = repl->p_next; 4973 } 4974 } 4975 /* assert that we have come full circle with repl */ 4976 ASSERT(repl_contig == 1 || first_repl == repl); 4977 4978 *target = pl; 4979 if (*replacement == NULL) { 4980 ASSERT(first_repl == repl); 4981 *replacement = repl; 4982 } 4983 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4984 *nrelocp = npgs; 4985 return (0); 4986 } 4987 /* 4988 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 4989 */ 4990 int 4991 page_relocate( 4992 page_t **target, 4993 page_t **replacement, 4994 int grouplock, 4995 int freetarget, 4996 spgcnt_t *nrelocp, 4997 lgrp_t *lgrp) 4998 { 4999 spgcnt_t ret; 5000 5001 /* do_page_relocate returns 0 on success or errno value */ 5002 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 5003 5004 if (ret != 0 || freetarget == 0) { 5005 return (ret); 5006 } 5007 if (*nrelocp == 1) { 5008 ASSERT(*target != NULL); 5009 page_free(*target, 1); 5010 } else { 5011 page_t *tpp = *target; 5012 uint_t szc = tpp->p_szc; 5013 pgcnt_t npgs = page_get_pagecnt(szc); 5014 ASSERT(npgs > 1); 5015 ASSERT(szc != 0); 5016 do { 5017 ASSERT(PAGE_EXCL(tpp)); 5018 ASSERT(!hat_page_is_mapped(tpp)); 5019 ASSERT(tpp->p_szc == szc); 5020 PP_SETFREE(tpp); 5021 PP_SETAGED(tpp); 5022 npgs--; 5023 } while ((tpp = tpp->p_next) != *target); 5024 ASSERT(npgs == 0); 5025 page_list_add_pages(*target, 0); 5026 npgs = page_get_pagecnt(szc); 5027 page_create_putback(npgs); 5028 } 5029 return (ret); 5030 } 5031 5032 /* 5033 * it is up to the caller to deal with pcf accounting. 5034 */ 5035 void 5036 page_free_replacement_page(page_t *pplist) 5037 { 5038 page_t *pp; 5039 5040 while (pplist != NULL) { 5041 /* 5042 * pp_targ is a linked list. 5043 */ 5044 pp = pplist; 5045 if (pp->p_szc == 0) { 5046 page_sub(&pplist, pp); 5047 page_clr_all_props(pp); 5048 PP_SETFREE(pp); 5049 PP_SETAGED(pp); 5050 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5051 page_unlock(pp); 5052 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5053 } else { 5054 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5055 page_t *tpp; 5056 page_list_break(&pp, &pplist, curnpgs); 5057 tpp = pp; 5058 do { 5059 ASSERT(PAGE_EXCL(tpp)); 5060 ASSERT(!hat_page_is_mapped(tpp)); 5061 page_clr_all_props(pp); 5062 PP_SETFREE(tpp); 5063 PP_SETAGED(tpp); 5064 } while ((tpp = tpp->p_next) != pp); 5065 page_list_add_pages(pp, 0); 5066 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5067 } 5068 } 5069 } 5070 5071 /* 5072 * Relocate target to non-relocatable replacement page. 5073 */ 5074 int 5075 page_relocate_cage(page_t **target, page_t **replacement) 5076 { 5077 page_t *tpp, *rpp; 5078 spgcnt_t pgcnt, npgs; 5079 int result; 5080 5081 tpp = *target; 5082 5083 ASSERT(PAGE_EXCL(tpp)); 5084 ASSERT(tpp->p_szc == 0); 5085 5086 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5087 5088 do { 5089 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5090 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5091 if (rpp == NULL) { 5092 page_create_putback(pgcnt); 5093 kcage_cageout_wakeup(); 5094 } 5095 } while (rpp == NULL); 5096 5097 ASSERT(PP_ISNORELOC(rpp)); 5098 5099 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5100 5101 if (result == 0) { 5102 *replacement = rpp; 5103 if (pgcnt != npgs) 5104 panic("page_relocate_cage: partial relocation"); 5105 } 5106 5107 return (result); 5108 } 5109 5110 /* 5111 * Release the page lock on a page, place on cachelist 5112 * tail if no longer mapped. Caller can let us know if 5113 * the page is known to be clean. 5114 */ 5115 int 5116 page_release(page_t *pp, int checkmod) 5117 { 5118 int status; 5119 5120 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5121 (pp->p_vnode != NULL)); 5122 5123 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5124 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5125 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5126 !hat_page_is_mapped(pp)) { 5127 5128 /* 5129 * If page is modified, unlock it 5130 * 5131 * (p_nrm & P_MOD) bit has the latest stuff because: 5132 * (1) We found that this page doesn't have any mappings 5133 * _after_ holding SE_EXCL and 5134 * (2) We didn't drop SE_EXCL lock after the check in (1) 5135 */ 5136 if (checkmod && hat_ismod(pp)) { 5137 page_unlock(pp); 5138 status = PGREL_MOD; 5139 } else { 5140 /*LINTED: constant in conditional context*/ 5141 VN_DISPOSE(pp, B_FREE, 0, kcred); 5142 status = PGREL_CLEAN; 5143 } 5144 } else { 5145 page_unlock(pp); 5146 status = PGREL_NOTREL; 5147 } 5148 return (status); 5149 } 5150 5151 /* 5152 * Given a constituent page, try to demote the large page on the freelist. 5153 * 5154 * Returns nonzero if the page could be demoted successfully. Returns with 5155 * the constituent page still locked. 5156 */ 5157 int 5158 page_try_demote_free_pages(page_t *pp) 5159 { 5160 page_t *rootpp = pp; 5161 pfn_t pfn = page_pptonum(pp); 5162 spgcnt_t npgs; 5163 uint_t szc = pp->p_szc; 5164 5165 ASSERT(PP_ISFREE(pp)); 5166 ASSERT(PAGE_EXCL(pp)); 5167 5168 /* 5169 * Adjust rootpp and lock it, if `pp' is not the base 5170 * constituent page. 5171 */ 5172 npgs = page_get_pagecnt(pp->p_szc); 5173 if (npgs == 1) { 5174 return (0); 5175 } 5176 5177 if (!IS_P2ALIGNED(pfn, npgs)) { 5178 pfn = P2ALIGN(pfn, npgs); 5179 rootpp = page_numtopp_nolock(pfn); 5180 } 5181 5182 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5183 return (0); 5184 } 5185 5186 if (rootpp->p_szc != szc) { 5187 if (pp != rootpp) 5188 page_unlock(rootpp); 5189 return (0); 5190 } 5191 5192 page_demote_free_pages(rootpp); 5193 5194 if (pp != rootpp) 5195 page_unlock(rootpp); 5196 5197 ASSERT(PP_ISFREE(pp)); 5198 ASSERT(PAGE_EXCL(pp)); 5199 return (1); 5200 } 5201 5202 /* 5203 * Given a constituent page, try to demote the large page. 5204 * 5205 * Returns nonzero if the page could be demoted successfully. Returns with 5206 * the constituent page still locked. 5207 */ 5208 int 5209 page_try_demote_pages(page_t *pp) 5210 { 5211 page_t *tpp, *rootpp = pp; 5212 pfn_t pfn = page_pptonum(pp); 5213 spgcnt_t i, npgs; 5214 uint_t szc = pp->p_szc; 5215 vnode_t *vp = pp->p_vnode; 5216 5217 ASSERT(PAGE_EXCL(pp)); 5218 5219 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5220 5221 if (pp->p_szc == 0) { 5222 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5223 return (1); 5224 } 5225 5226 if (vp != NULL && !IS_SWAPFSVP(vp) && vp != &kvp) { 5227 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5228 page_demote_vp_pages(pp); 5229 ASSERT(pp->p_szc == 0); 5230 return (1); 5231 } 5232 5233 /* 5234 * Adjust rootpp if passed in is not the base 5235 * constituent page. 5236 */ 5237 npgs = page_get_pagecnt(pp->p_szc); 5238 ASSERT(npgs > 1); 5239 if (!IS_P2ALIGNED(pfn, npgs)) { 5240 pfn = P2ALIGN(pfn, npgs); 5241 rootpp = page_numtopp_nolock(pfn); 5242 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5243 ASSERT(rootpp->p_vnode != NULL); 5244 ASSERT(rootpp->p_szc == szc); 5245 } 5246 5247 /* 5248 * We can't demote kernel pages since we can't hat_unload() 5249 * the mappings. 5250 */ 5251 if (rootpp->p_vnode == &kvp) 5252 return (0); 5253 5254 /* 5255 * Attempt to lock all constituent pages except the page passed 5256 * in since it's already locked. 5257 */ 5258 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5259 ASSERT(!PP_ISFREE(tpp)); 5260 ASSERT(tpp->p_vnode != NULL); 5261 5262 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5263 break; 5264 ASSERT(tpp->p_szc == rootpp->p_szc); 5265 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5266 } 5267 5268 /* 5269 * If we failed to lock them all then unlock what we have 5270 * locked so far and bail. 5271 */ 5272 if (i < npgs) { 5273 tpp = rootpp; 5274 while (i-- > 0) { 5275 if (tpp != pp) 5276 page_unlock(tpp); 5277 tpp++; 5278 } 5279 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5280 return (0); 5281 } 5282 5283 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5284 ASSERT(PAGE_EXCL(tpp)); 5285 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5286 tpp->p_szc = 0; 5287 } 5288 5289 /* 5290 * Unlock all pages except the page passed in. 5291 */ 5292 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5293 ASSERT(!hat_page_is_mapped(tpp)); 5294 if (tpp != pp) 5295 page_unlock(tpp); 5296 } 5297 5298 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5299 return (1); 5300 } 5301 5302 /* 5303 * Called by page_free() and page_destroy() to demote the page size code 5304 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5305 * p_szc on free list, neither can we just clear p_szc of a single page_t 5306 * within a large page since it will break other code that relies on p_szc 5307 * being the same for all page_t's of a large page). Anonymous pages should 5308 * never end up here because anon_map_getpages() cannot deal with p_szc 5309 * changes after a single constituent page is locked. While anonymous or 5310 * kernel large pages are demoted or freed the entire large page at a time 5311 * with all constituent pages locked EXCL for the file system pages we 5312 * have to be able to demote a large page (i.e. decrease all constituent pages 5313 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5314 * we can easily deal with anonymous page demotion the entire large page at a 5315 * time is that those operation originate at address space level and concern 5316 * the entire large page region with actual demotion only done when pages are 5317 * not shared with any other processes (therefore we can always get EXCL lock 5318 * on all anonymous constituent pages after clearing segment page 5319 * cache). However file system pages can be truncated or invalidated at a 5320 * PAGESIZE level from the file system side and end up in page_free() or 5321 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5322 * and therfore pageout should be able to demote a large page by EXCL locking 5323 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5324 * rely on being able to lock EXCL all constituent pages. 5325 * 5326 * To prevent szc changes on file system pages one has to lock all constituent 5327 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5328 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5329 * prevent szc changes is hat layer that uses its own page level mlist 5330 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5331 * taken. Therefore we need to change szc under hat level locks if we only 5332 * have an EXCL lock on a single constituent page and hat still references any 5333 * of constituent pages. (Note we can't "ignore" hat layer by simply 5334 * hat_pageunload() all constituent pages without having EXCL locks on all of 5335 * constituent pages). We use hat_page_demote() call to safely demote szc of 5336 * all constituent pages under hat locks when we only have an EXCL lock on one 5337 * of constituent pages. 5338 * 5339 * This routine calls page_szc_lock() before calling hat_page_demote() to 5340 * allow segvn in one special case not to lock all constituent pages SHARED 5341 * before calling hat_memload_array() that relies on p_szc not changeing even 5342 * before hat level mlist lock is taken. In that case segvn uses 5343 * page_szc_lock() to prevent hat_page_demote() changeing p_szc values. 5344 * 5345 * Anonymous or kernel page demotion still has to lock all pages exclusively 5346 * and do hat_pageunload() on all constituent pages before demoting the page 5347 * therefore there's no need for anonymous or kernel page demotion to use 5348 * hat_page_demote() mechanism. 5349 * 5350 * hat_page_demote() removes all large mappings that map pp and then decreases 5351 * p_szc starting from the last constituent page of the large page. By working 5352 * from the tail of a large page in pfn decreasing order allows one looking at 5353 * the root page to know that hat_page_demote() is done for root's szc area. 5354 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5355 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5356 * that started on this page when it had szc > 1 is done for this szc 1 area. 5357 * 5358 * We are guranteed that all constituent pages of pp's large page belong to 5359 * the same vnode with the consecutive offsets increasing in the direction of 5360 * the pfn i.e. the identity of constituent pages can't change until their 5361 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5362 * large mappings to pp even though we don't lock any constituent page except 5363 * pp (i.e. we won't unload e.g. kernel locked page). 5364 */ 5365 static void 5366 page_demote_vp_pages(page_t *pp) 5367 { 5368 kmutex_t *mtx; 5369 5370 ASSERT(PAGE_EXCL(pp)); 5371 ASSERT(!PP_ISFREE(pp)); 5372 ASSERT(pp->p_vnode != NULL); 5373 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5374 ASSERT(pp->p_vnode != &kvp); 5375 5376 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5377 5378 mtx = page_szc_lock(pp); 5379 if (mtx != NULL) { 5380 hat_page_demote(pp); 5381 mutex_exit(mtx); 5382 } 5383 ASSERT(pp->p_szc == 0); 5384 } 5385 5386 /* 5387 * Mark any existing pages for migration in the given range 5388 */ 5389 void 5390 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5391 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5392 u_offset_t vnoff, int rflag) 5393 { 5394 struct anon *ap; 5395 vnode_t *curvp; 5396 lgrp_t *from; 5397 pgcnt_t i; 5398 pgcnt_t nlocked; 5399 u_offset_t off; 5400 pfn_t pfn; 5401 size_t pgsz; 5402 size_t segpgsz; 5403 pgcnt_t pages; 5404 uint_t pszc; 5405 page_t **ppa; 5406 pgcnt_t ppa_nentries; 5407 page_t *pp; 5408 caddr_t va; 5409 ulong_t an_idx; 5410 anon_sync_obj_t cookie; 5411 5412 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5413 5414 /* 5415 * Don't do anything if don't need to do lgroup optimizations 5416 * on this system 5417 */ 5418 if (!lgrp_optimizations()) 5419 return; 5420 5421 /* 5422 * Align address and length to (potentially large) page boundary 5423 */ 5424 segpgsz = page_get_pagesize(seg->s_szc); 5425 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5426 if (rflag) 5427 len = P2ROUNDUP(len, segpgsz); 5428 5429 /* 5430 * Allocate page array to accomodate largest page size 5431 */ 5432 pgsz = page_get_pagesize(page_num_pagesizes() - 1); 5433 ppa_nentries = btop(pgsz); 5434 ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP); 5435 5436 /* 5437 * Do one (large) page at a time 5438 */ 5439 va = addr; 5440 while (va < addr + len) { 5441 /* 5442 * Lookup (root) page for vnode and offset corresponding to 5443 * this virtual address 5444 * Try anonmap first since there may be copy-on-write 5445 * pages, but initialize vnode pointer and offset using 5446 * vnode arguments just in case there isn't an amp. 5447 */ 5448 curvp = vp; 5449 off = vnoff + va - seg->s_base; 5450 if (amp) { 5451 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5452 an_idx = anon_index + seg_page(seg, va); 5453 anon_array_enter(amp, an_idx, &cookie); 5454 ap = anon_get_ptr(amp->ahp, an_idx); 5455 if (ap) 5456 swap_xlate(ap, &curvp, &off); 5457 anon_array_exit(&cookie); 5458 ANON_LOCK_EXIT(&->a_rwlock); 5459 } 5460 5461 pp = NULL; 5462 if (curvp) 5463 pp = page_lookup(curvp, off, SE_SHARED); 5464 5465 /* 5466 * If there isn't a page at this virtual address, 5467 * skip to next page 5468 */ 5469 if (pp == NULL) { 5470 va += PAGESIZE; 5471 continue; 5472 } 5473 5474 /* 5475 * Figure out which lgroup this page is in for kstats 5476 */ 5477 pfn = page_pptonum(pp); 5478 from = lgrp_pfn_to_lgrp(pfn); 5479 5480 /* 5481 * Get page size, and round up and skip to next page boundary 5482 * if unaligned address 5483 */ 5484 pszc = pp->p_szc; 5485 pgsz = page_get_pagesize(pszc); 5486 pages = btop(pgsz); 5487 if (!IS_P2ALIGNED(va, pgsz) || 5488 !IS_P2ALIGNED(pfn, pages) || 5489 pgsz > segpgsz) { 5490 pgsz = MIN(pgsz, segpgsz); 5491 page_unlock(pp); 5492 i = btop(P2END((uintptr_t)va, pgsz) - 5493 (uintptr_t)va); 5494 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5495 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i); 5496 continue; 5497 } 5498 5499 /* 5500 * Upgrade to exclusive lock on page 5501 */ 5502 if (!page_tryupgrade(pp)) { 5503 page_unlock(pp); 5504 va += pgsz; 5505 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5506 btop(pgsz)); 5507 continue; 5508 } 5509 5510 /* 5511 * Remember pages locked exclusively and how many 5512 */ 5513 ppa[0] = pp; 5514 nlocked = 1; 5515 5516 /* 5517 * Lock constituent pages if this is large page 5518 */ 5519 if (pages > 1) { 5520 /* 5521 * Lock all constituents except root page, since it 5522 * should be locked already. 5523 */ 5524 for (i = 1; i < pages; i++) { 5525 pp++; 5526 if (!page_trylock(pp, SE_EXCL)) { 5527 break; 5528 } 5529 if (PP_ISFREE(pp) || 5530 pp->p_szc != pszc) { 5531 /* 5532 * hat_page_demote() raced in with us. 5533 */ 5534 ASSERT(!IS_SWAPFSVP(curvp)); 5535 page_unlock(pp); 5536 break; 5537 } 5538 ppa[nlocked] = pp; 5539 nlocked++; 5540 } 5541 } 5542 5543 /* 5544 * If all constituent pages couldn't be locked, 5545 * unlock pages locked so far and skip to next page. 5546 */ 5547 if (nlocked != pages) { 5548 for (i = 0; i < nlocked; i++) 5549 page_unlock(ppa[i]); 5550 va += pgsz; 5551 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5552 btop(pgsz)); 5553 continue; 5554 } 5555 5556 /* 5557 * hat_page_demote() can no longer happen 5558 * since last cons page had the right p_szc after 5559 * all cons pages were locked. all cons pages 5560 * should now have the same p_szc. 5561 */ 5562 5563 /* 5564 * All constituent pages locked successfully, so mark 5565 * large page for migration and unload the mappings of 5566 * constituent pages, so a fault will occur on any part of the 5567 * large page 5568 */ 5569 PP_SETMIGRATE(ppa[0]); 5570 for (i = 0; i < nlocked; i++) { 5571 pp = ppa[i]; 5572 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 5573 ASSERT(hat_page_getshare(pp) == 0); 5574 page_unlock(pp); 5575 } 5576 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5577 5578 va += pgsz; 5579 } 5580 kmem_free(ppa, ppa_nentries * sizeof (page_t *)); 5581 } 5582 5583 /* 5584 * Migrate any pages that have been marked for migration in the given range 5585 */ 5586 void 5587 page_migrate( 5588 struct seg *seg, 5589 caddr_t addr, 5590 page_t **ppa, 5591 pgcnt_t npages) 5592 { 5593 lgrp_t *from; 5594 lgrp_t *to; 5595 page_t *newpp; 5596 page_t *pp; 5597 pfn_t pfn; 5598 size_t pgsz; 5599 spgcnt_t page_cnt; 5600 spgcnt_t i; 5601 uint_t pszc; 5602 5603 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5604 5605 while (npages > 0) { 5606 pp = *ppa; 5607 pszc = pp->p_szc; 5608 pgsz = page_get_pagesize(pszc); 5609 page_cnt = btop(pgsz); 5610 5611 /* 5612 * Check to see whether this page is marked for migration 5613 * 5614 * Assume that root page of large page is marked for 5615 * migration and none of the other constituent pages 5616 * are marked. This really simplifies clearing the 5617 * migrate bit by not having to clear it from each 5618 * constituent page. 5619 * 5620 * note we don't want to relocate an entire large page if 5621 * someone is only using one subpage. 5622 */ 5623 if (npages < page_cnt) 5624 break; 5625 5626 /* 5627 * Is it marked for migration? 5628 */ 5629 if (!PP_ISMIGRATE(pp)) 5630 goto next; 5631 5632 /* 5633 * Determine lgroups that page is being migrated between 5634 */ 5635 pfn = page_pptonum(pp); 5636 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5637 break; 5638 } 5639 from = lgrp_pfn_to_lgrp(pfn); 5640 to = lgrp_mem_choose(seg, addr, pgsz); 5641 5642 /* 5643 * Check to see whether we are trying to migrate page to lgroup 5644 * where it is allocated already 5645 */ 5646 if (to == from) { 5647 PP_CLRMIGRATE(pp); 5648 goto next; 5649 } 5650 5651 /* 5652 * Need to get exclusive lock's to migrate 5653 */ 5654 for (i = 0; i < page_cnt; i++) { 5655 ASSERT(PAGE_LOCKED(ppa[i])); 5656 if (page_pptonum(ppa[i]) != pfn + i || 5657 ppa[i]->p_szc != pszc) { 5658 break; 5659 } 5660 if (!page_tryupgrade(ppa[i])) { 5661 lgrp_stat_add(from->lgrp_id, 5662 LGRP_PM_FAIL_LOCK_PGS, 5663 page_cnt); 5664 break; 5665 } 5666 } 5667 if (i != page_cnt) { 5668 while (--i != -1) { 5669 page_downgrade(ppa[i]); 5670 } 5671 goto next; 5672 } 5673 5674 (void) page_create_wait(page_cnt, PG_WAIT); 5675 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5676 if (newpp == NULL) { 5677 page_create_putback(page_cnt); 5678 for (i = 0; i < page_cnt; i++) { 5679 page_downgrade(ppa[i]); 5680 } 5681 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5682 page_cnt); 5683 goto next; 5684 } 5685 ASSERT(newpp->p_szc == pszc); 5686 /* 5687 * Clear migrate bit and relocate page 5688 */ 5689 PP_CLRMIGRATE(pp); 5690 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5691 panic("page_migrate: page_relocate failed"); 5692 } 5693 ASSERT(page_cnt * PAGESIZE == pgsz); 5694 5695 /* 5696 * Keep stats for number of pages migrated from and to 5697 * each lgroup 5698 */ 5699 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5700 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5701 /* 5702 * update the page_t array we were passed in and 5703 * unlink constituent pages of a large page. 5704 */ 5705 for (i = 0; i < page_cnt; ++i, ++pp) { 5706 ASSERT(PAGE_EXCL(newpp)); 5707 ASSERT(newpp->p_szc == pszc); 5708 ppa[i] = newpp; 5709 pp = newpp; 5710 page_sub(&newpp, pp); 5711 page_downgrade(pp); 5712 } 5713 ASSERT(newpp == NULL); 5714 next: 5715 addr += pgsz; 5716 ppa += page_cnt; 5717 npages -= page_cnt; 5718 } 5719 } 5720 5721 ulong_t mem_waiters = 0; 5722 ulong_t max_count = 20; 5723 #define MAX_DELAY 0x1ff 5724 5725 /* 5726 * Check if enough memory is available to proceed. 5727 * Depending on system configuration and how much memory is 5728 * reserved for swap we need to check against two variables. 5729 * e.g. on systems with little physical swap availrmem can be 5730 * more reliable indicator of how much memory is available. 5731 * On systems with large phys swap freemem can be better indicator. 5732 * If freemem drops below threshold level don't return an error 5733 * immediately but wake up pageout to free memory and block. 5734 * This is done number of times. If pageout is not able to free 5735 * memory within certain time return an error. 5736 * The same applies for availrmem but kmem_reap is used to 5737 * free memory. 5738 */ 5739 int 5740 page_mem_avail(pgcnt_t npages) 5741 { 5742 ulong_t count; 5743 5744 #if defined(__i386) 5745 if (freemem > desfree + npages && 5746 availrmem > swapfs_reserve + npages && 5747 btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem + 5748 npages) 5749 return (1); 5750 #else 5751 if (freemem > desfree + npages && 5752 availrmem > swapfs_reserve + npages) 5753 return (1); 5754 #endif 5755 5756 count = max_count; 5757 atomic_add_long(&mem_waiters, 1); 5758 5759 while (freemem < desfree + npages && --count) { 5760 cv_signal(&proc_pageout->p_cv); 5761 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5762 atomic_add_long(&mem_waiters, -1); 5763 return (0); 5764 } 5765 } 5766 if (count == 0) { 5767 atomic_add_long(&mem_waiters, -1); 5768 return (0); 5769 } 5770 5771 count = max_count; 5772 while (availrmem < swapfs_reserve + npages && --count) { 5773 kmem_reap(); 5774 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5775 atomic_add_long(&mem_waiters, -1); 5776 return (0); 5777 } 5778 } 5779 atomic_add_long(&mem_waiters, -1); 5780 if (count == 0) 5781 return (0); 5782 5783 #if defined(__i386) 5784 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 5785 tune.t_minarmem + npages) 5786 return (0); 5787 #endif 5788 return (1); 5789 } 5790 5791 #define MAX_CNT 60 /* max num of iterations */ 5792 /* 5793 * Reclaim/reserve availrmem for npages. 5794 * If there is not enough memory start reaping seg, kmem caches. 5795 * Start pageout scanner (via page_needfree()). 5796 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5797 * Note: There is no guarantee that any availrmem will be freed as 5798 * this memory typically is locked (kernel heap) or reserved for swap. 5799 * Also due to memory fragmentation kmem allocator may not be able 5800 * to free any memory (single user allocated buffer will prevent 5801 * freeing slab or a page). 5802 */ 5803 int 5804 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5805 { 5806 int i = 0; 5807 int ret = 0; 5808 pgcnt_t deficit; 5809 pgcnt_t old_availrmem; 5810 5811 mutex_enter(&freemem_lock); 5812 old_availrmem = availrmem - 1; 5813 while ((availrmem < tune.t_minarmem + npages + epages) && 5814 (old_availrmem < availrmem) && (i++ < MAX_CNT)) { 5815 old_availrmem = availrmem; 5816 deficit = tune.t_minarmem + npages + epages - availrmem; 5817 mutex_exit(&freemem_lock); 5818 page_needfree(deficit); 5819 seg_preap(); 5820 kmem_reap(); 5821 delay(hz); 5822 page_needfree(-(spgcnt_t)deficit); 5823 mutex_enter(&freemem_lock); 5824 } 5825 5826 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5827 availrmem -= npages; 5828 ret = 1; 5829 } 5830 5831 mutex_exit(&freemem_lock); 5832 5833 return (ret); 5834 } 5835 5836 /* 5837 * Search the memory segments to locate the desired page. Within a 5838 * segment, pages increase linearly with one page structure per 5839 * physical page frame (size PAGESIZE). The search begins 5840 * with the segment that was accessed last, to take advantage of locality. 5841 * If the hint misses, we start from the beginning of the sorted memseg list 5842 */ 5843 5844 5845 /* 5846 * Some data structures for pfn to pp lookup. 5847 */ 5848 ulong_t mhash_per_slot; 5849 struct memseg *memseg_hash[N_MEM_SLOTS]; 5850 5851 page_t * 5852 page_numtopp_nolock(pfn_t pfnum) 5853 { 5854 struct memseg *seg; 5855 page_t *pp; 5856 vm_cpu_data_t *vc = CPU->cpu_vm_data; 5857 5858 ASSERT(vc != NULL); 5859 5860 MEMSEG_STAT_INCR(nsearch); 5861 5862 /* Try last winner first */ 5863 if (((seg = vc->vc_pnum_memseg) != NULL) && 5864 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5865 MEMSEG_STAT_INCR(nlastwon); 5866 pp = seg->pages + (pfnum - seg->pages_base); 5867 if (pp->p_pagenum == pfnum) 5868 return ((page_t *)pp); 5869 } 5870 5871 /* Else Try hash */ 5872 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5873 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5874 MEMSEG_STAT_INCR(nhashwon); 5875 vc->vc_pnum_memseg = seg; 5876 pp = seg->pages + (pfnum - seg->pages_base); 5877 if (pp->p_pagenum == pfnum) 5878 return ((page_t *)pp); 5879 } 5880 5881 /* Else Brute force */ 5882 for (seg = memsegs; seg != NULL; seg = seg->next) { 5883 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5884 vc->vc_pnum_memseg = seg; 5885 pp = seg->pages + (pfnum - seg->pages_base); 5886 return ((page_t *)pp); 5887 } 5888 } 5889 vc->vc_pnum_memseg = NULL; 5890 MEMSEG_STAT_INCR(nnotfound); 5891 return ((page_t *)NULL); 5892 5893 } 5894 5895 struct memseg * 5896 page_numtomemseg_nolock(pfn_t pfnum) 5897 { 5898 struct memseg *seg; 5899 page_t *pp; 5900 5901 /* Try hash */ 5902 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5903 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5904 pp = seg->pages + (pfnum - seg->pages_base); 5905 if (pp->p_pagenum == pfnum) 5906 return (seg); 5907 } 5908 5909 /* Else Brute force */ 5910 for (seg = memsegs; seg != NULL; seg = seg->next) { 5911 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5912 return (seg); 5913 } 5914 } 5915 return ((struct memseg *)NULL); 5916 } 5917 5918 /* 5919 * Given a page and a count return the page struct that is 5920 * n structs away from the current one in the global page 5921 * list. 5922 * 5923 * This function wraps to the first page upon 5924 * reaching the end of the memseg list. 5925 */ 5926 page_t * 5927 page_nextn(page_t *pp, ulong_t n) 5928 { 5929 struct memseg *seg; 5930 page_t *ppn; 5931 vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5932 5933 ASSERT(vc != NULL); 5934 5935 if (((seg = vc->vc_pnext_memseg) == NULL) || 5936 (seg->pages_base == seg->pages_end) || 5937 !(pp >= seg->pages && pp < seg->epages)) { 5938 5939 for (seg = memsegs; seg; seg = seg->next) { 5940 if (pp >= seg->pages && pp < seg->epages) 5941 break; 5942 } 5943 5944 if (seg == NULL) { 5945 /* Memory delete got in, return something valid. */ 5946 /* TODO: fix me. */ 5947 seg = memsegs; 5948 pp = seg->pages; 5949 } 5950 } 5951 5952 /* check for wraparound - possible if n is large */ 5953 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5954 n -= seg->epages - pp; 5955 seg = seg->next; 5956 if (seg == NULL) 5957 seg = memsegs; 5958 pp = seg->pages; 5959 } 5960 vc->vc_pnext_memseg = seg; 5961 return (ppn); 5962 } 5963 5964 /* 5965 * Initialize for a loop using page_next_scan_large(). 5966 */ 5967 page_t * 5968 page_next_scan_init(void **cookie) 5969 { 5970 ASSERT(cookie != NULL); 5971 *cookie = (void *)memsegs; 5972 return ((page_t *)memsegs->pages); 5973 } 5974 5975 /* 5976 * Return the next page in a scan of page_t's, assuming we want 5977 * to skip over sub-pages within larger page sizes. 5978 * 5979 * The cookie is used to keep track of the current memseg. 5980 */ 5981 page_t * 5982 page_next_scan_large( 5983 page_t *pp, 5984 ulong_t *n, 5985 void **cookie) 5986 { 5987 struct memseg *seg = (struct memseg *)*cookie; 5988 page_t *new_pp; 5989 ulong_t cnt; 5990 pfn_t pfn; 5991 5992 5993 /* 5994 * get the count of page_t's to skip based on the page size 5995 */ 5996 ASSERT(pp != NULL); 5997 if (pp->p_szc == 0) { 5998 cnt = 1; 5999 } else { 6000 pfn = page_pptonum(pp); 6001 cnt = page_get_pagecnt(pp->p_szc); 6002 cnt -= pfn & (cnt - 1); 6003 } 6004 *n += cnt; 6005 new_pp = pp + cnt; 6006 6007 /* 6008 * Catch if we went past the end of the current memory segment. If so, 6009 * just move to the next segment with pages. 6010 */ 6011 if (new_pp >= seg->epages) { 6012 do { 6013 seg = seg->next; 6014 if (seg == NULL) 6015 seg = memsegs; 6016 } while (seg->pages == seg->epages); 6017 new_pp = seg->pages; 6018 *cookie = (void *)seg; 6019 } 6020 6021 return (new_pp); 6022 } 6023 6024 6025 /* 6026 * Returns next page in list. Note: this function wraps 6027 * to the first page in the list upon reaching the end 6028 * of the list. Callers should be aware of this fact. 6029 */ 6030 6031 /* We should change this be a #define */ 6032 6033 page_t * 6034 page_next(page_t *pp) 6035 { 6036 return (page_nextn(pp, 1)); 6037 } 6038 6039 page_t * 6040 page_first() 6041 { 6042 return ((page_t *)memsegs->pages); 6043 } 6044 6045 6046 /* 6047 * This routine is called at boot with the initial memory configuration 6048 * and when memory is added or removed. 6049 */ 6050 void 6051 build_pfn_hash() 6052 { 6053 pfn_t cur; 6054 pgcnt_t index; 6055 struct memseg *pseg; 6056 int i; 6057 6058 /* 6059 * Clear memseg_hash array. 6060 * Since memory add/delete is designed to operate concurrently 6061 * with normal operation, the hash rebuild must be able to run 6062 * concurrently with page_numtopp_nolock(). To support this 6063 * functionality, assignments to memseg_hash array members must 6064 * be done atomically. 6065 * 6066 * NOTE: bzero() does not currently guarantee this for kernel 6067 * threads, and cannot be used here. 6068 */ 6069 for (i = 0; i < N_MEM_SLOTS; i++) 6070 memseg_hash[i] = NULL; 6071 6072 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6073 6074 /* 6075 * Physmax is the last valid pfn. 6076 */ 6077 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6078 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6079 index = MEMSEG_PFN_HASH(pseg->pages_base); 6080 cur = pseg->pages_base; 6081 do { 6082 if (index >= N_MEM_SLOTS) 6083 index = MEMSEG_PFN_HASH(cur); 6084 6085 if (memseg_hash[index] == NULL || 6086 memseg_hash[index]->pages_base > pseg->pages_base) { 6087 memseg_hash[index] = pseg; 6088 hat_kpm_mseghash_update(index, pseg); 6089 } 6090 cur += mhash_per_slot; 6091 index++; 6092 } while (cur < pseg->pages_end); 6093 } 6094 } 6095 6096 /* 6097 * Return the pagenum for the pp 6098 */ 6099 pfn_t 6100 page_pptonum(page_t *pp) 6101 { 6102 return (pp->p_pagenum); 6103 } 6104 6105 /* 6106 * interface to the referenced and modified etc bits 6107 * in the PSM part of the page struct 6108 * when no locking is desired. 6109 */ 6110 void 6111 page_set_props(page_t *pp, uint_t flags) 6112 { 6113 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6114 pp->p_nrm |= (uchar_t)flags; 6115 } 6116 6117 void 6118 page_clr_all_props(page_t *pp) 6119 { 6120 pp->p_nrm = 0; 6121 } 6122 6123 /* 6124 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6125 */ 6126 int 6127 page_clear_lck_cow(page_t *pp, int adjust) 6128 { 6129 int f_amount; 6130 6131 ASSERT(PAGE_EXCL(pp)); 6132 6133 /* 6134 * The page_struct_lock need not be acquired here since 6135 * we require the caller hold the page exclusively locked. 6136 */ 6137 f_amount = 0; 6138 if (pp->p_lckcnt) { 6139 f_amount = 1; 6140 pp->p_lckcnt = 0; 6141 } 6142 if (pp->p_cowcnt) { 6143 f_amount += pp->p_cowcnt; 6144 pp->p_cowcnt = 0; 6145 } 6146 6147 if (adjust && f_amount) { 6148 mutex_enter(&freemem_lock); 6149 availrmem += f_amount; 6150 mutex_exit(&freemem_lock); 6151 } 6152 6153 return (f_amount); 6154 } 6155 6156 /* 6157 * The following functions is called from free_vp_pages() 6158 * for an inexact estimate of a newly free'd page... 6159 */ 6160 ulong_t 6161 page_share_cnt(page_t *pp) 6162 { 6163 return (hat_page_getshare(pp)); 6164 } 6165 6166 int 6167 page_isshared(page_t *pp) 6168 { 6169 return (hat_page_getshare(pp) > 1); 6170 } 6171 6172 int 6173 page_isfree(page_t *pp) 6174 { 6175 return (PP_ISFREE(pp)); 6176 } 6177 6178 int 6179 page_isref(page_t *pp) 6180 { 6181 return (hat_page_getattr(pp, P_REF)); 6182 } 6183 6184 int 6185 page_ismod(page_t *pp) 6186 { 6187 return (hat_page_getattr(pp, P_MOD)); 6188 } 6189