1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 28 /* All Rights Reserved */ 29 30 /* 31 * University Copyright- Copyright (c) 1982, 1986, 1988 32 * The Regents of the University of California 33 * All Rights Reserved 34 * 35 * University Acknowledgment- Portions of this document are derived from 36 * software developed by the University of California, Berkeley, and its 37 * contributors. 38 */ 39 40 #pragma ident "%Z%%M% %I% %E% SMI" 41 42 /* 43 * VM - physical page management. 44 */ 45 46 #include <sys/types.h> 47 #include <sys/t_lock.h> 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/errno.h> 51 #include <sys/time.h> 52 #include <sys/vnode.h> 53 #include <sys/vm.h> 54 #include <sys/vtrace.h> 55 #include <sys/swap.h> 56 #include <sys/cmn_err.h> 57 #include <sys/tuneable.h> 58 #include <sys/sysmacros.h> 59 #include <sys/cpuvar.h> 60 #include <sys/callb.h> 61 #include <sys/debug.h> 62 #include <sys/tnf_probe.h> 63 #include <sys/condvar_impl.h> 64 #include <sys/mem_config.h> 65 #include <sys/mem_cage.h> 66 #include <sys/kmem.h> 67 #include <sys/atomic.h> 68 #include <sys/strlog.h> 69 #include <sys/mman.h> 70 #include <sys/ontrap.h> 71 #include <sys/lgrp.h> 72 #include <sys/vfs.h> 73 74 #include <vm/hat.h> 75 #include <vm/anon.h> 76 #include <vm/page.h> 77 #include <vm/seg.h> 78 #include <vm/pvn.h> 79 #include <vm/seg_kmem.h> 80 #include <vm/vm_dep.h> 81 82 #include <fs/fs_subr.h> 83 84 static int nopageage = 0; 85 86 static pgcnt_t max_page_get; /* max page_get request size in pages */ 87 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 88 89 /* 90 * freemem_lock protects all freemem variables: 91 * availrmem. Also this lock protects the globals which track the 92 * availrmem changes for accurate kernel footprint calculation. 93 * See below for an explanation of these 94 * globals. 95 */ 96 kmutex_t freemem_lock; 97 pgcnt_t availrmem; 98 pgcnt_t availrmem_initial; 99 100 /* 101 * These globals track availrmem changes to get a more accurate 102 * estimate of tke kernel size. Historically pp_kernel is used for 103 * kernel size and is based on availrmem. But availrmem is adjusted for 104 * locked pages in the system not just for kernel locked pages. 105 * These new counters will track the pages locked through segvn and 106 * by explicit user locking. 107 * 108 * segvn_pages_locked : This keeps track on a global basis how many pages 109 * are currently locked because of I/O. 110 * 111 * pages_locked : How many pages are locked becuase of user specified 112 * locking through mlock or plock. 113 * 114 * pages_useclaim,pages_claimed : These two variables track the 115 * cliam adjustments because of the protection changes on a segvn segment. 116 * 117 * All these globals are protected by the same lock which protects availrmem. 118 */ 119 pgcnt_t segvn_pages_locked; 120 pgcnt_t pages_locked; 121 pgcnt_t pages_useclaim; 122 pgcnt_t pages_claimed; 123 124 125 /* 126 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 127 */ 128 static kmutex_t new_freemem_lock; 129 static uint_t freemem_wait; /* someone waiting for freemem */ 130 static kcondvar_t freemem_cv; 131 132 /* 133 * The logical page free list is maintained as two lists, the 'free' 134 * and the 'cache' lists. 135 * The free list contains those pages that should be reused first. 136 * 137 * The implementation of the lists is machine dependent. 138 * page_get_freelist(), page_get_cachelist(), 139 * page_list_sub(), and page_list_add() 140 * form the interface to the machine dependent implementation. 141 * 142 * Pages with p_free set are on the cache list. 143 * Pages with p_free and p_age set are on the free list, 144 * 145 * A page may be locked while on either list. 146 */ 147 148 /* 149 * free list accounting stuff. 150 * 151 * 152 * Spread out the value for the number of pages on the 153 * page free and page cache lists. If there is just one 154 * value, then it must be under just one lock. 155 * The lock contention and cache traffic are a real bother. 156 * 157 * When we acquire and then drop a single pcf lock 158 * we can start in the middle of the array of pcf structures. 159 * If we acquire more than one pcf lock at a time, we need to 160 * start at the front to avoid deadlocking. 161 * 162 * pcf_count holds the number of pages in each pool. 163 * 164 * pcf_block is set when page_create_get_something() has asked the 165 * PSM page freelist and page cachelist routines without specifying 166 * a color and nothing came back. This is used to block anything 167 * else from moving pages from one list to the other while the 168 * lists are searched again. If a page is freeed while pcf_block is 169 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 170 * of clearning pcf_block, doing the wakeups, etc. 171 */ 172 173 #if NCPU <= 4 174 #define PAD 1 175 #define PCF_FANOUT 4 176 static uint_t pcf_mask = PCF_FANOUT - 1; 177 #else 178 #define PAD 9 179 #ifdef sun4v 180 #define PCF_FANOUT 32 181 #else 182 #define PCF_FANOUT 128 183 #endif 184 static uint_t pcf_mask = PCF_FANOUT - 1; 185 #endif 186 187 struct pcf { 188 uint_t pcf_touch; /* just to help the cache */ 189 uint_t pcf_count; /* page count */ 190 kmutex_t pcf_lock; /* protects the structure */ 191 uint_t pcf_wait; /* number of waiters */ 192 uint_t pcf_block; /* pcgs flag to page_free() */ 193 uint_t pcf_reserve; /* pages freed after pcf_block set */ 194 uint_t pcf_fill[PAD]; /* to line up on the caches */ 195 }; 196 197 static struct pcf pcf[PCF_FANOUT]; 198 #define PCF_INDEX() ((CPU->cpu_id) & (pcf_mask)) 199 200 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 201 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 202 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 203 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 204 205 #define PAGE_LOCK_MAXIMUM \ 206 ((1 << (sizeof (((page_t *)0)->p_lckcnt) * NBBY)) - 1) 207 208 #ifdef VM_STATS 209 210 /* 211 * No locks, but so what, they are only statistics. 212 */ 213 214 static struct page_tcnt { 215 int pc_free_cache; /* free's into cache list */ 216 int pc_free_dontneed; /* free's with dontneed */ 217 int pc_free_pageout; /* free's from pageout */ 218 int pc_free_free; /* free's into free list */ 219 int pc_free_pages; /* free's into large page free list */ 220 int pc_destroy_pages; /* large page destroy's */ 221 int pc_get_cache; /* get's from cache list */ 222 int pc_get_free; /* get's from free list */ 223 int pc_reclaim; /* reclaim's */ 224 int pc_abortfree; /* abort's of free pages */ 225 int pc_find_hit; /* find's that find page */ 226 int pc_find_miss; /* find's that don't find page */ 227 int pc_destroy_free; /* # of free pages destroyed */ 228 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 229 int pc_find_hashlen[PC_HASH_CNT+1]; 230 int pc_addclaim_pages; 231 int pc_subclaim_pages; 232 int pc_free_replacement_page[2]; 233 int pc_try_demote_pages[6]; 234 int pc_demote_pages[2]; 235 } pagecnt; 236 237 uint_t hashin_count; 238 uint_t hashin_not_held; 239 uint_t hashin_already; 240 241 uint_t hashout_count; 242 uint_t hashout_not_held; 243 244 uint_t page_create_count; 245 uint_t page_create_not_enough; 246 uint_t page_create_not_enough_again; 247 uint_t page_create_zero; 248 uint_t page_create_hashout; 249 uint_t page_create_page_lock_failed; 250 uint_t page_create_trylock_failed; 251 uint_t page_create_found_one; 252 uint_t page_create_hashin_failed; 253 uint_t page_create_dropped_phm; 254 255 uint_t page_create_new; 256 uint_t page_create_exists; 257 uint_t page_create_putbacks; 258 uint_t page_create_overshoot; 259 260 uint_t page_reclaim_zero; 261 uint_t page_reclaim_zero_locked; 262 263 uint_t page_rename_exists; 264 uint_t page_rename_count; 265 266 uint_t page_lookup_cnt[20]; 267 uint_t page_lookup_nowait_cnt[10]; 268 uint_t page_find_cnt; 269 uint_t page_exists_cnt; 270 uint_t page_exists_forreal_cnt; 271 uint_t page_lookup_dev_cnt; 272 uint_t get_cachelist_cnt; 273 uint_t page_create_cnt[10]; 274 uint_t alloc_pages[8]; 275 uint_t page_exphcontg[19]; 276 uint_t page_create_large_cnt[10]; 277 278 /* 279 * Collects statistics. 280 */ 281 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 282 uint_t mylen = 0; \ 283 \ 284 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 285 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 286 break; \ 287 } \ 288 if ((pp) != NULL) \ 289 pagecnt.pc_find_hit++; \ 290 else \ 291 pagecnt.pc_find_miss++; \ 292 if (mylen > PC_HASH_CNT) \ 293 mylen = PC_HASH_CNT; \ 294 pagecnt.pc_find_hashlen[mylen]++; \ 295 } 296 297 #else /* VM_STATS */ 298 299 /* 300 * Don't collect statistics 301 */ 302 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 303 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 304 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 305 break; \ 306 } \ 307 } 308 309 #endif /* VM_STATS */ 310 311 312 313 #ifdef DEBUG 314 #define MEMSEG_SEARCH_STATS 315 #endif 316 317 #ifdef MEMSEG_SEARCH_STATS 318 struct memseg_stats { 319 uint_t nsearch; 320 uint_t nlastwon; 321 uint_t nhashwon; 322 uint_t nnotfound; 323 } memseg_stats; 324 325 #define MEMSEG_STAT_INCR(v) \ 326 atomic_add_32(&memseg_stats.v, 1) 327 #else 328 #define MEMSEG_STAT_INCR(x) 329 #endif 330 331 struct memseg *memsegs; /* list of memory segments */ 332 333 334 static void page_init_mem_config(void); 335 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 336 static void page_do_hashout(page_t *); 337 338 static void page_demote_vp_pages(page_t *); 339 340 /* 341 * vm subsystem related initialization 342 */ 343 void 344 vm_init(void) 345 { 346 boolean_t callb_vm_cpr(void *, int); 347 348 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 349 page_init_mem_config(); 350 page_retire_init(); 351 } 352 353 /* 354 * This function is called at startup and when memory is added or deleted. 355 */ 356 void 357 init_pages_pp_maximum() 358 { 359 static pgcnt_t p_min; 360 static pgcnt_t pages_pp_maximum_startup; 361 static pgcnt_t avrmem_delta; 362 static int init_done; 363 static int user_set; /* true if set in /etc/system */ 364 365 if (init_done == 0) { 366 367 /* If the user specified a value, save it */ 368 if (pages_pp_maximum != 0) { 369 user_set = 1; 370 pages_pp_maximum_startup = pages_pp_maximum; 371 } 372 373 /* 374 * Setting of pages_pp_maximum is based first time 375 * on the value of availrmem just after the start-up 376 * allocations. To preserve this relationship at run 377 * time, use a delta from availrmem_initial. 378 */ 379 ASSERT(availrmem_initial >= availrmem); 380 avrmem_delta = availrmem_initial - availrmem; 381 382 /* The allowable floor of pages_pp_maximum */ 383 p_min = tune.t_minarmem + 100; 384 385 /* Make sure we don't come through here again. */ 386 init_done = 1; 387 } 388 /* 389 * Determine pages_pp_maximum, the number of currently available 390 * pages (availrmem) that can't be `locked'. If not set by 391 * the user, we set it to 4% of the currently available memory 392 * plus 4MB. 393 * But we also insist that it be greater than tune.t_minarmem; 394 * otherwise a process could lock down a lot of memory, get swapped 395 * out, and never have enough to get swapped back in. 396 */ 397 if (user_set) 398 pages_pp_maximum = pages_pp_maximum_startup; 399 else 400 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 401 + btop(4 * 1024 * 1024); 402 403 if (pages_pp_maximum <= p_min) { 404 pages_pp_maximum = p_min; 405 } 406 } 407 408 void 409 set_max_page_get(pgcnt_t target_total_pages) 410 { 411 max_page_get = target_total_pages / 2; 412 } 413 414 static pgcnt_t pending_delete; 415 416 /*ARGSUSED*/ 417 static void 418 page_mem_config_post_add( 419 void *arg, 420 pgcnt_t delta_pages) 421 { 422 set_max_page_get(total_pages - pending_delete); 423 init_pages_pp_maximum(); 424 } 425 426 /*ARGSUSED*/ 427 static int 428 page_mem_config_pre_del( 429 void *arg, 430 pgcnt_t delta_pages) 431 { 432 pgcnt_t nv; 433 434 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 435 set_max_page_get(total_pages - nv); 436 return (0); 437 } 438 439 /*ARGSUSED*/ 440 static void 441 page_mem_config_post_del( 442 void *arg, 443 pgcnt_t delta_pages, 444 int cancelled) 445 { 446 pgcnt_t nv; 447 448 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 449 set_max_page_get(total_pages - nv); 450 if (!cancelled) 451 init_pages_pp_maximum(); 452 } 453 454 static kphysm_setup_vector_t page_mem_config_vec = { 455 KPHYSM_SETUP_VECTOR_VERSION, 456 page_mem_config_post_add, 457 page_mem_config_pre_del, 458 page_mem_config_post_del, 459 }; 460 461 static void 462 page_init_mem_config(void) 463 { 464 int ret; 465 466 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 467 ASSERT(ret == 0); 468 } 469 470 /* 471 * Evenly spread out the PCF counters for large free pages 472 */ 473 static void 474 page_free_large_ctr(pgcnt_t npages) 475 { 476 static struct pcf *p = pcf; 477 pgcnt_t lump; 478 479 freemem += npages; 480 481 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 482 483 while (npages > 0) { 484 485 ASSERT(!p->pcf_block); 486 487 if (lump < npages) { 488 p->pcf_count += (uint_t)lump; 489 npages -= lump; 490 } else { 491 p->pcf_count += (uint_t)npages; 492 npages = 0; 493 } 494 495 ASSERT(!p->pcf_wait); 496 497 if (++p > &pcf[PCF_FANOUT - 1]) 498 p = pcf; 499 } 500 501 ASSERT(npages == 0); 502 } 503 504 /* 505 * Add a physical chunk of memory to the system freee lists during startup. 506 * Platform specific startup() allocates the memory for the page structs. 507 * 508 * num - number of page structures 509 * base - page number (pfn) to be associated with the first page. 510 * 511 * Since we are doing this during startup (ie. single threaded), we will 512 * use shortcut routines to avoid any locking overhead while putting all 513 * these pages on the freelists. 514 * 515 * NOTE: Any changes performed to page_free(), must also be performed to 516 * add_physmem() since this is how we initialize all page_t's at 517 * boot time. 518 */ 519 void 520 add_physmem( 521 page_t *pp, 522 pgcnt_t num, 523 pfn_t pnum) 524 { 525 page_t *root = NULL; 526 uint_t szc = page_num_pagesizes() - 1; 527 pgcnt_t large = page_get_pagecnt(szc); 528 pgcnt_t cnt = 0; 529 530 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 531 "add_physmem:pp %p num %lu", pp, num); 532 533 /* 534 * Arbitrarily limit the max page_get request 535 * to 1/2 of the page structs we have. 536 */ 537 total_pages += num; 538 set_max_page_get(total_pages); 539 540 /* 541 * The physical space for the pages array 542 * representing ram pages has already been 543 * allocated. Here we initialize each lock 544 * in the page structure, and put each on 545 * the free list 546 */ 547 for (; num; pp++, pnum++, num--) { 548 549 /* 550 * this needs to fill in the page number 551 * and do any other arch specific initialization 552 */ 553 add_physmem_cb(pp, pnum); 554 555 /* 556 * Initialize the page lock as unlocked, since nobody 557 * can see or access this page yet. 558 */ 559 pp->p_selock = 0; 560 561 /* 562 * Initialize IO lock 563 */ 564 page_iolock_init(pp); 565 566 /* 567 * initialize other fields in the page_t 568 */ 569 PP_SETFREE(pp); 570 page_clr_all_props(pp); 571 PP_SETAGED(pp); 572 pp->p_offset = (u_offset_t)-1; 573 pp->p_next = pp; 574 pp->p_prev = pp; 575 576 /* 577 * Simple case: System doesn't support large pages. 578 */ 579 if (szc == 0) { 580 pp->p_szc = 0; 581 page_free_at_startup(pp); 582 continue; 583 } 584 585 /* 586 * Handle unaligned pages, we collect them up onto 587 * the root page until we have a full large page. 588 */ 589 if (!IS_P2ALIGNED(pnum, large)) { 590 591 /* 592 * If not in a large page, 593 * just free as small page. 594 */ 595 if (root == NULL) { 596 pp->p_szc = 0; 597 page_free_at_startup(pp); 598 continue; 599 } 600 601 /* 602 * Link a constituent page into the large page. 603 */ 604 pp->p_szc = szc; 605 page_list_concat(&root, &pp); 606 607 /* 608 * When large page is fully formed, free it. 609 */ 610 if (++cnt == large) { 611 page_free_large_ctr(cnt); 612 page_list_add_pages(root, PG_LIST_ISINIT); 613 root = NULL; 614 cnt = 0; 615 } 616 continue; 617 } 618 619 /* 620 * At this point we have a page number which 621 * is aligned. We assert that we aren't already 622 * in a different large page. 623 */ 624 ASSERT(IS_P2ALIGNED(pnum, large)); 625 ASSERT(root == NULL && cnt == 0); 626 627 /* 628 * If insufficient number of pages left to form 629 * a large page, just free the small page. 630 */ 631 if (num < large) { 632 pp->p_szc = 0; 633 page_free_at_startup(pp); 634 continue; 635 } 636 637 /* 638 * Otherwise start a new large page. 639 */ 640 pp->p_szc = szc; 641 cnt++; 642 root = pp; 643 } 644 ASSERT(root == NULL && cnt == 0); 645 } 646 647 /* 648 * Find a page representing the specified [vp, offset]. 649 * If we find the page but it is intransit coming in, 650 * it will have an "exclusive" lock and we wait for 651 * the i/o to complete. A page found on the free list 652 * is always reclaimed and then locked. On success, the page 653 * is locked, its data is valid and it isn't on the free 654 * list, while a NULL is returned if the page doesn't exist. 655 */ 656 page_t * 657 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 658 { 659 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 660 } 661 662 /* 663 * Find a page representing the specified [vp, offset]. 664 * We either return the one we found or, if passed in, 665 * create one with identity of [vp, offset] of the 666 * pre-allocated page. If we find exsisting page but it is 667 * intransit coming in, it will have an "exclusive" lock 668 * and we wait for the i/o to complete. A page found on 669 * the free list is always reclaimed and then locked. 670 * On success, the page is locked, its data is valid and 671 * it isn't on the free list, while a NULL is returned 672 * if the page doesn't exist and newpp is NULL; 673 */ 674 page_t * 675 page_lookup_create( 676 vnode_t *vp, 677 u_offset_t off, 678 se_t se, 679 page_t *newpp, 680 spgcnt_t *nrelocp, 681 int flags) 682 { 683 page_t *pp; 684 kmutex_t *phm; 685 ulong_t index; 686 uint_t hash_locked; 687 uint_t es; 688 689 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 690 VM_STAT_ADD(page_lookup_cnt[0]); 691 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 692 693 /* 694 * Acquire the appropriate page hash lock since 695 * we have to search the hash list. Pages that 696 * hash to this list can't change identity while 697 * this lock is held. 698 */ 699 hash_locked = 0; 700 index = PAGE_HASH_FUNC(vp, off); 701 phm = NULL; 702 top: 703 PAGE_HASH_SEARCH(index, pp, vp, off); 704 if (pp != NULL) { 705 VM_STAT_ADD(page_lookup_cnt[1]); 706 es = (newpp != NULL) ? 1 : 0; 707 es |= flags; 708 if (!hash_locked) { 709 VM_STAT_ADD(page_lookup_cnt[2]); 710 if (!page_try_reclaim_lock(pp, se, es)) { 711 /* 712 * On a miss, acquire the phm. Then 713 * next time, page_lock() will be called, 714 * causing a wait if the page is busy. 715 * just looping with page_trylock() would 716 * get pretty boring. 717 */ 718 VM_STAT_ADD(page_lookup_cnt[3]); 719 phm = PAGE_HASH_MUTEX(index); 720 mutex_enter(phm); 721 hash_locked = 1; 722 goto top; 723 } 724 } else { 725 VM_STAT_ADD(page_lookup_cnt[4]); 726 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 727 VM_STAT_ADD(page_lookup_cnt[5]); 728 goto top; 729 } 730 } 731 732 /* 733 * Since `pp' is locked it can not change identity now. 734 * Reconfirm we locked the correct page. 735 * 736 * Both the p_vnode and p_offset *must* be cast volatile 737 * to force a reload of their values: The PAGE_HASH_SEARCH 738 * macro will have stuffed p_vnode and p_offset into 739 * registers before calling page_trylock(); another thread, 740 * actually holding the hash lock, could have changed the 741 * page's identity in memory, but our registers would not 742 * be changed, fooling the reconfirmation. If the hash 743 * lock was held during the search, the casting would 744 * not be needed. 745 */ 746 VM_STAT_ADD(page_lookup_cnt[6]); 747 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 748 ((volatile u_offset_t)(pp->p_offset) != off)) { 749 VM_STAT_ADD(page_lookup_cnt[7]); 750 if (hash_locked) { 751 panic("page_lookup_create: lost page %p", 752 (void *)pp); 753 /*NOTREACHED*/ 754 } 755 page_unlock(pp); 756 phm = PAGE_HASH_MUTEX(index); 757 mutex_enter(phm); 758 hash_locked = 1; 759 goto top; 760 } 761 762 /* 763 * If page_trylock() was called, then pp may still be on 764 * the cachelist (can't be on the free list, it would not 765 * have been found in the search). If it is on the 766 * cachelist it must be pulled now. To pull the page from 767 * the cachelist, it must be exclusively locked. 768 * 769 * The other big difference between page_trylock() and 770 * page_lock(), is that page_lock() will pull the 771 * page from whatever free list (the cache list in this 772 * case) the page is on. If page_trylock() was used 773 * above, then we have to do the reclaim ourselves. 774 */ 775 if ((!hash_locked) && (PP_ISFREE(pp))) { 776 ASSERT(PP_ISAGED(pp) == 0); 777 VM_STAT_ADD(page_lookup_cnt[8]); 778 779 /* 780 * page_relcaim will insure that we 781 * have this page exclusively 782 */ 783 784 if (!page_reclaim(pp, NULL)) { 785 /* 786 * Page_reclaim dropped whatever lock 787 * we held. 788 */ 789 VM_STAT_ADD(page_lookup_cnt[9]); 790 phm = PAGE_HASH_MUTEX(index); 791 mutex_enter(phm); 792 hash_locked = 1; 793 goto top; 794 } else if (se == SE_SHARED && newpp == NULL) { 795 VM_STAT_ADD(page_lookup_cnt[10]); 796 page_downgrade(pp); 797 } 798 } 799 800 if (hash_locked) { 801 mutex_exit(phm); 802 } 803 804 if (newpp != NULL && pp->p_szc < newpp->p_szc && 805 PAGE_EXCL(pp) && nrelocp != NULL) { 806 ASSERT(nrelocp != NULL); 807 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 808 NULL); 809 if (*nrelocp > 0) { 810 VM_STAT_COND_ADD(*nrelocp == 1, 811 page_lookup_cnt[11]); 812 VM_STAT_COND_ADD(*nrelocp > 1, 813 page_lookup_cnt[12]); 814 pp = newpp; 815 se = SE_EXCL; 816 } else { 817 if (se == SE_SHARED) { 818 page_downgrade(pp); 819 } 820 VM_STAT_ADD(page_lookup_cnt[13]); 821 } 822 } else if (newpp != NULL && nrelocp != NULL) { 823 if (PAGE_EXCL(pp) && se == SE_SHARED) { 824 page_downgrade(pp); 825 } 826 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 827 page_lookup_cnt[14]); 828 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 829 page_lookup_cnt[15]); 830 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 831 page_lookup_cnt[16]); 832 } else if (newpp != NULL && PAGE_EXCL(pp)) { 833 se = SE_EXCL; 834 } 835 } else if (!hash_locked) { 836 VM_STAT_ADD(page_lookup_cnt[17]); 837 phm = PAGE_HASH_MUTEX(index); 838 mutex_enter(phm); 839 hash_locked = 1; 840 goto top; 841 } else if (newpp != NULL) { 842 /* 843 * If we have a preallocated page then 844 * insert it now and basically behave like 845 * page_create. 846 */ 847 VM_STAT_ADD(page_lookup_cnt[18]); 848 /* 849 * Since we hold the page hash mutex and 850 * just searched for this page, page_hashin 851 * had better not fail. If it does, that 852 * means some thread did not follow the 853 * page hash mutex rules. Panic now and 854 * get it over with. As usual, go down 855 * holding all the locks. 856 */ 857 ASSERT(MUTEX_HELD(phm)); 858 if (!page_hashin(newpp, vp, off, phm)) { 859 ASSERT(MUTEX_HELD(phm)); 860 panic("page_lookup_create: hashin failed %p %p %llx %p", 861 (void *)newpp, (void *)vp, off, (void *)phm); 862 /*NOTREACHED*/ 863 } 864 ASSERT(MUTEX_HELD(phm)); 865 mutex_exit(phm); 866 phm = NULL; 867 page_set_props(newpp, P_REF); 868 page_io_lock(newpp); 869 pp = newpp; 870 se = SE_EXCL; 871 } else { 872 VM_STAT_ADD(page_lookup_cnt[19]); 873 mutex_exit(phm); 874 } 875 876 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 877 878 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 879 880 return (pp); 881 } 882 883 /* 884 * Search the hash list for the page representing the 885 * specified [vp, offset] and return it locked. Skip 886 * free pages and pages that cannot be locked as requested. 887 * Used while attempting to kluster pages. 888 */ 889 page_t * 890 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 891 { 892 page_t *pp; 893 kmutex_t *phm; 894 ulong_t index; 895 uint_t locked; 896 897 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 898 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 899 900 index = PAGE_HASH_FUNC(vp, off); 901 PAGE_HASH_SEARCH(index, pp, vp, off); 902 locked = 0; 903 if (pp == NULL) { 904 top: 905 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 906 locked = 1; 907 phm = PAGE_HASH_MUTEX(index); 908 mutex_enter(phm); 909 PAGE_HASH_SEARCH(index, pp, vp, off); 910 } 911 912 if (pp == NULL || PP_ISFREE(pp)) { 913 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 914 pp = NULL; 915 } else { 916 if (!page_trylock(pp, se)) { 917 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 918 pp = NULL; 919 } else { 920 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 921 /* 922 * See the comment in page_lookup() 923 */ 924 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 925 ((u_offset_t)(pp->p_offset) != off)) { 926 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 927 if (locked) { 928 panic("page_lookup_nowait %p", 929 (void *)pp); 930 /*NOTREACHED*/ 931 } 932 page_unlock(pp); 933 goto top; 934 } 935 if (PP_ISFREE(pp)) { 936 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 937 page_unlock(pp); 938 pp = NULL; 939 } 940 } 941 } 942 if (locked) { 943 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 944 mutex_exit(phm); 945 } 946 947 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 948 949 return (pp); 950 } 951 952 /* 953 * Search the hash list for a page with the specified [vp, off] 954 * that is known to exist and is already locked. This routine 955 * is typically used by segment SOFTUNLOCK routines. 956 */ 957 page_t * 958 page_find(vnode_t *vp, u_offset_t off) 959 { 960 page_t *pp; 961 kmutex_t *phm; 962 ulong_t index; 963 964 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 965 VM_STAT_ADD(page_find_cnt); 966 967 index = PAGE_HASH_FUNC(vp, off); 968 phm = PAGE_HASH_MUTEX(index); 969 970 mutex_enter(phm); 971 PAGE_HASH_SEARCH(index, pp, vp, off); 972 mutex_exit(phm); 973 974 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 975 return (pp); 976 } 977 978 /* 979 * Determine whether a page with the specified [vp, off] 980 * currently exists in the system. Obviously this should 981 * only be considered as a hint since nothing prevents the 982 * page from disappearing or appearing immediately after 983 * the return from this routine. Subsequently, we don't 984 * even bother to lock the list. 985 */ 986 page_t * 987 page_exists(vnode_t *vp, u_offset_t off) 988 { 989 page_t *pp; 990 ulong_t index; 991 992 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 993 VM_STAT_ADD(page_exists_cnt); 994 995 index = PAGE_HASH_FUNC(vp, off); 996 PAGE_HASH_SEARCH(index, pp, vp, off); 997 998 return (pp); 999 } 1000 1001 /* 1002 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1003 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1004 * with these pages locked SHARED. If necessary reclaim pages from 1005 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1006 * 1007 * If we fail to lock pages still return 1 if pages exist and contiguous. 1008 * But in this case return value is just a hint. ppa array won't be filled. 1009 * Caller should initialize ppa[0] as NULL to distinguish return value. 1010 * 1011 * Returns 0 if pages don't exist or not physically contiguous. 1012 * 1013 * This routine doesn't work for anonymous(swapfs) pages. 1014 */ 1015 int 1016 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1017 { 1018 pgcnt_t pages; 1019 pfn_t pfn; 1020 page_t *rootpp; 1021 pgcnt_t i; 1022 pgcnt_t j; 1023 u_offset_t save_off = off; 1024 ulong_t index; 1025 kmutex_t *phm; 1026 page_t *pp; 1027 uint_t pszc; 1028 int loopcnt = 0; 1029 1030 ASSERT(szc != 0); 1031 ASSERT(vp != NULL); 1032 ASSERT(!IS_SWAPFSVP(vp)); 1033 ASSERT(vp != &kvp); 1034 1035 again: 1036 if (++loopcnt > 3) { 1037 VM_STAT_ADD(page_exphcontg[0]); 1038 return (0); 1039 } 1040 1041 index = PAGE_HASH_FUNC(vp, off); 1042 phm = PAGE_HASH_MUTEX(index); 1043 1044 mutex_enter(phm); 1045 PAGE_HASH_SEARCH(index, pp, vp, off); 1046 mutex_exit(phm); 1047 1048 VM_STAT_ADD(page_exphcontg[1]); 1049 1050 if (pp == NULL) { 1051 VM_STAT_ADD(page_exphcontg[2]); 1052 return (0); 1053 } 1054 1055 pages = page_get_pagecnt(szc); 1056 rootpp = pp; 1057 pfn = rootpp->p_pagenum; 1058 1059 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1060 VM_STAT_ADD(page_exphcontg[3]); 1061 if (!page_trylock(pp, SE_SHARED)) { 1062 VM_STAT_ADD(page_exphcontg[4]); 1063 return (1); 1064 } 1065 if (pp->p_szc != pszc || pp->p_vnode != vp || 1066 pp->p_offset != off) { 1067 VM_STAT_ADD(page_exphcontg[5]); 1068 page_unlock(pp); 1069 off = save_off; 1070 goto again; 1071 } 1072 /* 1073 * szc was non zero and vnode and offset matched after we 1074 * locked the page it means it can't become free on us. 1075 */ 1076 ASSERT(!PP_ISFREE(pp)); 1077 if (!IS_P2ALIGNED(pfn, pages)) { 1078 page_unlock(pp); 1079 return (0); 1080 } 1081 ppa[0] = pp; 1082 pp++; 1083 off += PAGESIZE; 1084 pfn++; 1085 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1086 if (!page_trylock(pp, SE_SHARED)) { 1087 VM_STAT_ADD(page_exphcontg[6]); 1088 pp--; 1089 while (i-- > 0) { 1090 page_unlock(pp); 1091 pp--; 1092 } 1093 ppa[0] = NULL; 1094 return (1); 1095 } 1096 if (pp->p_szc != pszc) { 1097 VM_STAT_ADD(page_exphcontg[7]); 1098 page_unlock(pp); 1099 pp--; 1100 while (i-- > 0) { 1101 page_unlock(pp); 1102 pp--; 1103 } 1104 ppa[0] = NULL; 1105 off = save_off; 1106 goto again; 1107 } 1108 /* 1109 * szc the same as for previous already locked pages 1110 * with right identity. Since this page had correct 1111 * szc after we locked it can't get freed or destroyed 1112 * and therefore must have the expected identity. 1113 */ 1114 ASSERT(!PP_ISFREE(pp)); 1115 if (pp->p_vnode != vp || 1116 pp->p_offset != off) { 1117 panic("page_exists_physcontig: " 1118 "large page identity doesn't match"); 1119 } 1120 ppa[i] = pp; 1121 ASSERT(pp->p_pagenum == pfn); 1122 } 1123 VM_STAT_ADD(page_exphcontg[8]); 1124 ppa[pages] = NULL; 1125 return (1); 1126 } else if (pszc >= szc) { 1127 VM_STAT_ADD(page_exphcontg[9]); 1128 if (!IS_P2ALIGNED(pfn, pages)) { 1129 return (0); 1130 } 1131 return (1); 1132 } 1133 1134 if (!IS_P2ALIGNED(pfn, pages)) { 1135 VM_STAT_ADD(page_exphcontg[10]); 1136 return (0); 1137 } 1138 1139 if (page_numtomemseg_nolock(pfn) != 1140 page_numtomemseg_nolock(pfn + pages - 1)) { 1141 VM_STAT_ADD(page_exphcontg[11]); 1142 return (0); 1143 } 1144 1145 /* 1146 * We loop up 4 times across pages to promote page size. 1147 * We're extra cautious to promote page size atomically with respect 1148 * to everybody else. But we can probably optimize into 1 loop if 1149 * this becomes an issue. 1150 */ 1151 1152 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1153 ASSERT(pp->p_pagenum == pfn); 1154 if (!page_trylock(pp, SE_EXCL)) { 1155 VM_STAT_ADD(page_exphcontg[12]); 1156 break; 1157 } 1158 if (pp->p_vnode != vp || 1159 pp->p_offset != off) { 1160 VM_STAT_ADD(page_exphcontg[13]); 1161 page_unlock(pp); 1162 break; 1163 } 1164 if (pp->p_szc >= szc) { 1165 ASSERT(i == 0); 1166 page_unlock(pp); 1167 off = save_off; 1168 goto again; 1169 } 1170 } 1171 1172 if (i != pages) { 1173 VM_STAT_ADD(page_exphcontg[14]); 1174 --pp; 1175 while (i-- > 0) { 1176 page_unlock(pp); 1177 --pp; 1178 } 1179 return (0); 1180 } 1181 1182 pp = rootpp; 1183 for (i = 0; i < pages; i++, pp++) { 1184 if (PP_ISFREE(pp)) { 1185 VM_STAT_ADD(page_exphcontg[15]); 1186 ASSERT(!PP_ISAGED(pp)); 1187 ASSERT(pp->p_szc == 0); 1188 if (!page_reclaim(pp, NULL)) { 1189 break; 1190 } 1191 } else { 1192 ASSERT(pp->p_szc < szc); 1193 VM_STAT_ADD(page_exphcontg[16]); 1194 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1195 } 1196 } 1197 if (i < pages) { 1198 VM_STAT_ADD(page_exphcontg[17]); 1199 /* 1200 * page_reclaim failed because we were out of memory. 1201 * drop the rest of the locks and return because this page 1202 * must be already reallocated anyway. 1203 */ 1204 pp = rootpp; 1205 for (j = 0; j < pages; j++, pp++) { 1206 if (j != i) { 1207 page_unlock(pp); 1208 } 1209 } 1210 return (0); 1211 } 1212 1213 off = save_off; 1214 pp = rootpp; 1215 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1216 ASSERT(PAGE_EXCL(pp)); 1217 ASSERT(!PP_ISFREE(pp)); 1218 ASSERT(!hat_page_is_mapped(pp)); 1219 ASSERT(pp->p_vnode == vp); 1220 ASSERT(pp->p_offset == off); 1221 pp->p_szc = szc; 1222 } 1223 pp = rootpp; 1224 for (i = 0; i < pages; i++, pp++) { 1225 if (ppa == NULL) { 1226 page_unlock(pp); 1227 } else { 1228 ppa[i] = pp; 1229 page_downgrade(ppa[i]); 1230 } 1231 } 1232 if (ppa != NULL) { 1233 ppa[pages] = NULL; 1234 } 1235 VM_STAT_ADD(page_exphcontg[18]); 1236 ASSERT(vp->v_pages != NULL); 1237 return (1); 1238 } 1239 1240 /* 1241 * Determine whether a page with the specified [vp, off] 1242 * currently exists in the system and if so return its 1243 * size code. Obviously this should only be considered as 1244 * a hint since nothing prevents the page from disappearing 1245 * or appearing immediately after the return from this routine. 1246 */ 1247 int 1248 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1249 { 1250 page_t *pp; 1251 kmutex_t *phm; 1252 ulong_t index; 1253 int rc = 0; 1254 1255 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1256 ASSERT(szc != NULL); 1257 VM_STAT_ADD(page_exists_forreal_cnt); 1258 1259 index = PAGE_HASH_FUNC(vp, off); 1260 phm = PAGE_HASH_MUTEX(index); 1261 1262 mutex_enter(phm); 1263 PAGE_HASH_SEARCH(index, pp, vp, off); 1264 if (pp != NULL) { 1265 *szc = pp->p_szc; 1266 rc = 1; 1267 } 1268 mutex_exit(phm); 1269 return (rc); 1270 } 1271 1272 /* wakeup threads waiting for pages in page_create_get_something() */ 1273 void 1274 wakeup_pcgs(void) 1275 { 1276 if (!CV_HAS_WAITERS(&pcgs_cv)) 1277 return; 1278 cv_broadcast(&pcgs_cv); 1279 } 1280 1281 /* 1282 * 'freemem' is used all over the kernel as an indication of how many 1283 * pages are free (either on the cache list or on the free page list) 1284 * in the system. In very few places is a really accurate 'freemem' 1285 * needed. To avoid contention of the lock protecting a the 1286 * single freemem, it was spread out into NCPU buckets. Set_freemem 1287 * sets freemem to the total of all NCPU buckets. It is called from 1288 * clock() on each TICK. 1289 */ 1290 void 1291 set_freemem() 1292 { 1293 struct pcf *p; 1294 ulong_t t; 1295 uint_t i; 1296 1297 t = 0; 1298 p = pcf; 1299 for (i = 0; i < PCF_FANOUT; i++) { 1300 t += p->pcf_count; 1301 p++; 1302 } 1303 freemem = t; 1304 1305 /* 1306 * Don't worry about grabbing mutex. It's not that 1307 * critical if we miss a tick or two. This is 1308 * where we wakeup possible delayers in 1309 * page_create_get_something(). 1310 */ 1311 wakeup_pcgs(); 1312 } 1313 1314 ulong_t 1315 get_freemem() 1316 { 1317 struct pcf *p; 1318 ulong_t t; 1319 uint_t i; 1320 1321 t = 0; 1322 p = pcf; 1323 for (i = 0; i < PCF_FANOUT; i++) { 1324 t += p->pcf_count; 1325 p++; 1326 } 1327 /* 1328 * We just calculated it, might as well set it. 1329 */ 1330 freemem = t; 1331 return (t); 1332 } 1333 1334 /* 1335 * Acquire all of the page cache & free (pcf) locks. 1336 */ 1337 void 1338 pcf_acquire_all() 1339 { 1340 struct pcf *p; 1341 uint_t i; 1342 1343 p = pcf; 1344 for (i = 0; i < PCF_FANOUT; i++) { 1345 p->pcf_touch = 1; 1346 mutex_enter(&p->pcf_lock); 1347 p++; 1348 } 1349 } 1350 1351 /* 1352 * Release all the pcf_locks. 1353 */ 1354 void 1355 pcf_release_all() 1356 { 1357 struct pcf *p; 1358 uint_t i; 1359 1360 p = pcf; 1361 for (i = 0; i < PCF_FANOUT; i++) { 1362 mutex_exit(&p->pcf_lock); 1363 p++; 1364 } 1365 } 1366 1367 /* 1368 * Inform the VM system that we need some pages freed up. 1369 * Calls must be symmetric, e.g.: 1370 * 1371 * page_needfree(100); 1372 * wait a bit; 1373 * page_needfree(-100); 1374 */ 1375 void 1376 page_needfree(spgcnt_t npages) 1377 { 1378 mutex_enter(&new_freemem_lock); 1379 needfree += npages; 1380 mutex_exit(&new_freemem_lock); 1381 } 1382 1383 /* 1384 * Throttle for page_create(): try to prevent freemem from dropping 1385 * below throttlefree. We can't provide a 100% guarantee because 1386 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1387 * nibble away at the freelist. However, we can block all PG_WAIT 1388 * allocations until memory becomes available. The motivation is 1389 * that several things can fall apart when there's no free memory: 1390 * 1391 * (1) If pageout() needs memory to push a page, the system deadlocks. 1392 * 1393 * (2) By (broken) specification, timeout(9F) can neither fail nor 1394 * block, so it has no choice but to panic the system if it 1395 * cannot allocate a callout structure. 1396 * 1397 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1398 * it panics if it cannot allocate a callback structure. 1399 * 1400 * (4) Untold numbers of third-party drivers have not yet been hardened 1401 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1402 * success and panic the system with a data fault on failure. 1403 * (The long-term solution to this particular problem is to ship 1404 * hostile fault-injecting DEBUG kernels with the DDK.) 1405 * 1406 * It is theoretically impossible to guarantee success of non-blocking 1407 * allocations, but in practice, this throttle is very hard to break. 1408 */ 1409 static int 1410 page_create_throttle(pgcnt_t npages, int flags) 1411 { 1412 ulong_t fm; 1413 uint_t i; 1414 pgcnt_t tf; /* effective value of throttlefree */ 1415 1416 /* 1417 * Never deny pages when: 1418 * - it's a thread that cannot block [NOMEMWAIT()] 1419 * - the allocation cannot block and must not fail 1420 * - the allocation cannot block and is pageout dispensated 1421 */ 1422 if (NOMEMWAIT() || 1423 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1424 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1425 return (1); 1426 1427 /* 1428 * If the allocation can't block, we look favorably upon it 1429 * unless we're below pageout_reserve. In that case we fail 1430 * the allocation because we want to make sure there are a few 1431 * pages available for pageout. 1432 */ 1433 if ((flags & PG_WAIT) == 0) 1434 return (freemem >= npages + pageout_reserve); 1435 1436 /* Calculate the effective throttlefree value */ 1437 tf = throttlefree - 1438 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1439 1440 cv_signal(&proc_pageout->p_cv); 1441 1442 while (freemem < npages + tf) { 1443 pcf_acquire_all(); 1444 mutex_enter(&new_freemem_lock); 1445 fm = 0; 1446 for (i = 0; i < PCF_FANOUT; i++) { 1447 fm += pcf[i].pcf_count; 1448 pcf[i].pcf_wait++; 1449 mutex_exit(&pcf[i].pcf_lock); 1450 } 1451 freemem = fm; 1452 needfree += npages; 1453 freemem_wait++; 1454 cv_wait(&freemem_cv, &new_freemem_lock); 1455 freemem_wait--; 1456 needfree -= npages; 1457 mutex_exit(&new_freemem_lock); 1458 } 1459 return (1); 1460 } 1461 1462 /* 1463 * page_create_wait() is called to either coalecse pages from the 1464 * different pcf buckets or to wait because there simply are not 1465 * enough pages to satisfy the caller's request. 1466 * 1467 * Sadly, this is called from platform/vm/vm_machdep.c 1468 */ 1469 int 1470 page_create_wait(size_t npages, uint_t flags) 1471 { 1472 pgcnt_t total; 1473 uint_t i; 1474 struct pcf *p; 1475 1476 /* 1477 * Wait until there are enough free pages to satisfy our 1478 * entire request. 1479 * We set needfree += npages before prodding pageout, to make sure 1480 * it does real work when npages > lotsfree > freemem. 1481 */ 1482 VM_STAT_ADD(page_create_not_enough); 1483 1484 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1485 checkagain: 1486 if ((flags & PG_NORELOC) && 1487 kcage_freemem < kcage_throttlefree + npages) 1488 (void) kcage_create_throttle(npages, flags); 1489 1490 if (freemem < npages + throttlefree) 1491 if (!page_create_throttle(npages, flags)) 1492 return (0); 1493 1494 /* 1495 * Since page_create_va() looked at every 1496 * bucket, assume we are going to have to wait. 1497 * Get all of the pcf locks. 1498 */ 1499 total = 0; 1500 p = pcf; 1501 for (i = 0; i < PCF_FANOUT; i++) { 1502 p->pcf_touch = 1; 1503 mutex_enter(&p->pcf_lock); 1504 total += p->pcf_count; 1505 if (total >= npages) { 1506 /* 1507 * Wow! There are enough pages laying around 1508 * to satisfy the request. Do the accounting, 1509 * drop the locks we acquired, and go back. 1510 * 1511 * freemem is not protected by any lock. So, 1512 * we cannot have any assertion containing 1513 * freemem. 1514 */ 1515 freemem -= npages; 1516 1517 while (p >= pcf) { 1518 if (p->pcf_count <= npages) { 1519 npages -= p->pcf_count; 1520 p->pcf_count = 0; 1521 } else { 1522 p->pcf_count -= (uint_t)npages; 1523 npages = 0; 1524 } 1525 mutex_exit(&p->pcf_lock); 1526 p--; 1527 } 1528 ASSERT(npages == 0); 1529 return (1); 1530 } 1531 p++; 1532 } 1533 1534 /* 1535 * All of the pcf locks are held, there are not enough pages 1536 * to satisfy the request (npages < total). 1537 * Be sure to acquire the new_freemem_lock before dropping 1538 * the pcf locks. This prevents dropping wakeups in page_free(). 1539 * The order is always pcf_lock then new_freemem_lock. 1540 * 1541 * Since we hold all the pcf locks, it is a good time to set freemem. 1542 * 1543 * If the caller does not want to wait, return now. 1544 * Else turn the pageout daemon loose to find something 1545 * and wait till it does. 1546 * 1547 */ 1548 freemem = total; 1549 1550 if ((flags & PG_WAIT) == 0) { 1551 pcf_release_all(); 1552 1553 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1554 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1555 return (0); 1556 } 1557 1558 ASSERT(proc_pageout != NULL); 1559 cv_signal(&proc_pageout->p_cv); 1560 1561 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1562 "page_create_sleep_start: freemem %ld needfree %ld", 1563 freemem, needfree); 1564 1565 /* 1566 * We are going to wait. 1567 * We currently hold all of the pcf_locks, 1568 * get the new_freemem_lock (it protects freemem_wait), 1569 * before dropping the pcf_locks. 1570 */ 1571 mutex_enter(&new_freemem_lock); 1572 1573 p = pcf; 1574 for (i = 0; i < PCF_FANOUT; i++) { 1575 p->pcf_wait++; 1576 mutex_exit(&p->pcf_lock); 1577 p++; 1578 } 1579 1580 needfree += npages; 1581 freemem_wait++; 1582 1583 cv_wait(&freemem_cv, &new_freemem_lock); 1584 1585 freemem_wait--; 1586 needfree -= npages; 1587 1588 mutex_exit(&new_freemem_lock); 1589 1590 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1591 "page_create_sleep_end: freemem %ld needfree %ld", 1592 freemem, needfree); 1593 1594 VM_STAT_ADD(page_create_not_enough_again); 1595 goto checkagain; 1596 } 1597 1598 /* 1599 * A routine to do the opposite of page_create_wait(). 1600 */ 1601 void 1602 page_create_putback(spgcnt_t npages) 1603 { 1604 struct pcf *p; 1605 pgcnt_t lump; 1606 uint_t *which; 1607 1608 /* 1609 * When a contiguous lump is broken up, we have to 1610 * deal with lots of pages (min 64) so lets spread 1611 * the wealth around. 1612 */ 1613 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 1614 freemem += npages; 1615 1616 for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) { 1617 which = &p->pcf_count; 1618 1619 mutex_enter(&p->pcf_lock); 1620 1621 if (p->pcf_block) { 1622 which = &p->pcf_reserve; 1623 } 1624 1625 if (lump < npages) { 1626 *which += (uint_t)lump; 1627 npages -= lump; 1628 } else { 1629 *which += (uint_t)npages; 1630 npages = 0; 1631 } 1632 1633 if (p->pcf_wait) { 1634 mutex_enter(&new_freemem_lock); 1635 /* 1636 * Check to see if some other thread 1637 * is actually waiting. Another bucket 1638 * may have woken it up by now. If there 1639 * are no waiters, then set our pcf_wait 1640 * count to zero to avoid coming in here 1641 * next time. 1642 */ 1643 if (freemem_wait) { 1644 if (npages > 1) { 1645 cv_broadcast(&freemem_cv); 1646 } else { 1647 cv_signal(&freemem_cv); 1648 } 1649 p->pcf_wait--; 1650 } else { 1651 p->pcf_wait = 0; 1652 } 1653 mutex_exit(&new_freemem_lock); 1654 } 1655 mutex_exit(&p->pcf_lock); 1656 } 1657 ASSERT(npages == 0); 1658 } 1659 1660 /* 1661 * A helper routine for page_create_get_something. 1662 * The indenting got to deep down there. 1663 * Unblock the pcf counters. Any pages freed after 1664 * pcf_block got set are moved to pcf_count and 1665 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1666 */ 1667 static void 1668 pcgs_unblock(void) 1669 { 1670 int i; 1671 struct pcf *p; 1672 1673 /* Update freemem while we're here. */ 1674 freemem = 0; 1675 p = pcf; 1676 for (i = 0; i < PCF_FANOUT; i++) { 1677 mutex_enter(&p->pcf_lock); 1678 ASSERT(p->pcf_count == 0); 1679 p->pcf_count = p->pcf_reserve; 1680 p->pcf_block = 0; 1681 freemem += p->pcf_count; 1682 if (p->pcf_wait) { 1683 mutex_enter(&new_freemem_lock); 1684 if (freemem_wait) { 1685 if (p->pcf_reserve > 1) { 1686 cv_broadcast(&freemem_cv); 1687 p->pcf_wait = 0; 1688 } else { 1689 cv_signal(&freemem_cv); 1690 p->pcf_wait--; 1691 } 1692 } else { 1693 p->pcf_wait = 0; 1694 } 1695 mutex_exit(&new_freemem_lock); 1696 } 1697 p->pcf_reserve = 0; 1698 mutex_exit(&p->pcf_lock); 1699 p++; 1700 } 1701 } 1702 1703 /* 1704 * Called from page_create_va() when both the cache and free lists 1705 * have been checked once. 1706 * 1707 * Either returns a page or panics since the accounting was done 1708 * way before we got here. 1709 * 1710 * We don't come here often, so leave the accounting on permanently. 1711 */ 1712 1713 #define MAX_PCGS 100 1714 1715 #ifdef DEBUG 1716 #define PCGS_TRIES 100 1717 #else /* DEBUG */ 1718 #define PCGS_TRIES 10 1719 #endif /* DEBUG */ 1720 1721 #ifdef VM_STATS 1722 uint_t pcgs_counts[PCGS_TRIES]; 1723 uint_t pcgs_too_many; 1724 uint_t pcgs_entered; 1725 uint_t pcgs_entered_noreloc; 1726 uint_t pcgs_locked; 1727 uint_t pcgs_cagelocked; 1728 #endif /* VM_STATS */ 1729 1730 static page_t * 1731 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1732 caddr_t vaddr, uint_t flags) 1733 { 1734 uint_t count; 1735 page_t *pp; 1736 uint_t locked, i; 1737 struct pcf *p; 1738 lgrp_t *lgrp; 1739 int cagelocked = 0; 1740 1741 VM_STAT_ADD(pcgs_entered); 1742 1743 /* 1744 * Tap any reserve freelists: if we fail now, we'll die 1745 * since the page(s) we're looking for have already been 1746 * accounted for. 1747 */ 1748 flags |= PG_PANIC; 1749 1750 if ((flags & PG_NORELOC) != 0) { 1751 VM_STAT_ADD(pcgs_entered_noreloc); 1752 /* 1753 * Requests for free pages from critical threads 1754 * such as pageout still won't throttle here, but 1755 * we must try again, to give the cageout thread 1756 * another chance to catch up. Since we already 1757 * accounted for the pages, we had better get them 1758 * this time. 1759 * 1760 * N.B. All non-critical threads acquire the pcgs_cagelock 1761 * to serialize access to the freelists. This implements a 1762 * turnstile-type synchornization to avoid starvation of 1763 * critical requests for PG_NORELOC memory by non-critical 1764 * threads: all non-critical threads must acquire a 'ticket' 1765 * before passing through, which entails making sure 1766 * kcage_freemem won't fall below minfree prior to grabbing 1767 * pages from the freelists. 1768 */ 1769 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1770 mutex_enter(&pcgs_cagelock); 1771 cagelocked = 1; 1772 VM_STAT_ADD(pcgs_cagelocked); 1773 } 1774 } 1775 1776 /* 1777 * Time to get serious. 1778 * We failed to get a `correctly colored' page from both the 1779 * free and cache lists. 1780 * We escalate in stage. 1781 * 1782 * First try both lists without worring about color. 1783 * 1784 * Then, grab all page accounting locks (ie. pcf[]) and 1785 * steal any pages that they have and set the pcf_block flag to 1786 * stop deletions from the lists. This will help because 1787 * a page can get added to the free list while we are looking 1788 * at the cache list, then another page could be added to the cache 1789 * list allowing the page on the free list to be removed as we 1790 * move from looking at the cache list to the free list. This 1791 * could happen over and over. We would never find the page 1792 * we have accounted for. 1793 * 1794 * Noreloc pages are a subset of the global (relocatable) page pool. 1795 * They are not tracked separately in the pcf bins, so it is 1796 * impossible to know when doing pcf accounting if the available 1797 * page(s) are noreloc pages or not. When looking for a noreloc page 1798 * it is quite easy to end up here even if the global (relocatable) 1799 * page pool has plenty of free pages but the noreloc pool is empty. 1800 * 1801 * When the noreloc pool is empty (or low), additional noreloc pages 1802 * are created by converting pages from the global page pool. This 1803 * process will stall during pcf accounting if the pcf bins are 1804 * already locked. Such is the case when a noreloc allocation is 1805 * looping here in page_create_get_something waiting for more noreloc 1806 * pages to appear. 1807 * 1808 * Short of adding a new field to the pcf bins to accurately track 1809 * the number of free noreloc pages, we instead do not grab the 1810 * pcgs_lock, do not set the pcf blocks and do not timeout when 1811 * allocating a noreloc page. This allows noreloc allocations to 1812 * loop without blocking global page pool allocations. 1813 * 1814 * NOTE: the behaviour of page_create_get_something has not changed 1815 * for the case of global page pool allocations. 1816 */ 1817 1818 flags &= ~PG_MATCH_COLOR; 1819 locked = 0; 1820 #ifndef __sparc 1821 /* 1822 * page_create_get_something may be called because 4g memory may be 1823 * depleted. Set flags to allow for relocation of base page below 1824 * 4g if necessary. 1825 */ 1826 if (physmax4g) 1827 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI); 1828 #endif 1829 1830 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1831 1832 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1833 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1834 flags, lgrp); 1835 if (pp == NULL) { 1836 pp = page_get_cachelist(vp, off, seg, vaddr, 1837 flags, lgrp); 1838 } 1839 if (pp == NULL) { 1840 /* 1841 * Serialize. Don't fight with other pcgs(). 1842 */ 1843 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1844 mutex_enter(&pcgs_lock); 1845 VM_STAT_ADD(pcgs_locked); 1846 locked = 1; 1847 p = pcf; 1848 for (i = 0; i < PCF_FANOUT; i++) { 1849 mutex_enter(&p->pcf_lock); 1850 ASSERT(p->pcf_block == 0); 1851 p->pcf_block = 1; 1852 p->pcf_reserve = p->pcf_count; 1853 p->pcf_count = 0; 1854 mutex_exit(&p->pcf_lock); 1855 p++; 1856 } 1857 freemem = 0; 1858 } 1859 1860 if (count) { 1861 /* 1862 * Since page_free() puts pages on 1863 * a list then accounts for it, we 1864 * just have to wait for page_free() 1865 * to unlock any page it was working 1866 * with. The page_lock()-page_reclaim() 1867 * path falls in the same boat. 1868 * 1869 * We don't need to check on the 1870 * PG_WAIT flag, we have already 1871 * accounted for the page we are 1872 * looking for in page_create_va(). 1873 * 1874 * We just wait a moment to let any 1875 * locked pages on the lists free up, 1876 * then continue around and try again. 1877 * 1878 * Will be awakened by set_freemem(). 1879 */ 1880 mutex_enter(&pcgs_wait_lock); 1881 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1882 mutex_exit(&pcgs_wait_lock); 1883 } 1884 } else { 1885 #ifdef VM_STATS 1886 if (count >= PCGS_TRIES) { 1887 VM_STAT_ADD(pcgs_too_many); 1888 } else { 1889 VM_STAT_ADD(pcgs_counts[count]); 1890 } 1891 #endif 1892 if (locked) { 1893 pcgs_unblock(); 1894 mutex_exit(&pcgs_lock); 1895 } 1896 if (cagelocked) 1897 mutex_exit(&pcgs_cagelock); 1898 return (pp); 1899 } 1900 } 1901 /* 1902 * we go down holding the pcf locks. 1903 */ 1904 panic("no %spage found %d", 1905 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1906 /*NOTREACHED*/ 1907 } 1908 1909 /* 1910 * Create enough pages for "bytes" worth of data starting at 1911 * "off" in "vp". 1912 * 1913 * Where flag must be one of: 1914 * 1915 * PG_EXCL: Exclusive create (fail if any page already 1916 * exists in the page cache) which does not 1917 * wait for memory to become available. 1918 * 1919 * PG_WAIT: Non-exclusive create which can wait for 1920 * memory to become available. 1921 * 1922 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1923 * (Not Supported) 1924 * 1925 * A doubly linked list of pages is returned to the caller. Each page 1926 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1927 * lock. 1928 * 1929 * Unable to change the parameters to page_create() in a minor release, 1930 * we renamed page_create() to page_create_va(), changed all known calls 1931 * from page_create() to page_create_va(), and created this wrapper. 1932 * 1933 * Upon a major release, we should break compatibility by deleting this 1934 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1935 * 1936 * NOTE: There is a copy of this interface as page_create_io() in 1937 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1938 * there. 1939 */ 1940 page_t * 1941 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1942 { 1943 caddr_t random_vaddr; 1944 struct seg kseg; 1945 1946 #ifdef DEBUG 1947 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1948 (void *)caller()); 1949 #endif 1950 1951 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1952 (uintptr_t)(off >> PAGESHIFT)); 1953 kseg.s_as = &kas; 1954 1955 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1956 } 1957 1958 #ifdef DEBUG 1959 uint32_t pg_alloc_pgs_mtbf = 0; 1960 #endif 1961 1962 /* 1963 * Used for large page support. It will attempt to allocate 1964 * a large page(s) off the freelist. 1965 * 1966 * Returns non zero on failure. 1967 */ 1968 int 1969 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1970 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz) 1971 { 1972 pgcnt_t npgs, curnpgs, totpgs; 1973 size_t pgsz; 1974 page_t *pplist = NULL, *pp; 1975 int err = 0; 1976 lgrp_t *lgrp; 1977 1978 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 1979 1980 VM_STAT_ADD(alloc_pages[0]); 1981 1982 #ifdef DEBUG 1983 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 1984 return (ENOMEM); 1985 } 1986 #endif 1987 1988 pgsz = page_get_pagesize(szc); 1989 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 1990 1991 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 1992 /* 1993 * One must be NULL but not both. 1994 * And one must be non NULL but not both. 1995 */ 1996 ASSERT(basepp != NULL || ppa != NULL); 1997 ASSERT(basepp == NULL || ppa == NULL); 1998 1999 (void) page_create_wait(npgs, PG_WAIT); 2000 2001 while (npgs && szc) { 2002 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2003 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 0, lgrp); 2004 if (pp != NULL) { 2005 VM_STAT_ADD(alloc_pages[1]); 2006 page_list_concat(&pplist, &pp); 2007 ASSERT(npgs >= curnpgs); 2008 npgs -= curnpgs; 2009 } else if (anypgsz) { 2010 VM_STAT_ADD(alloc_pages[2]); 2011 szc--; 2012 pgsz = page_get_pagesize(szc); 2013 curnpgs = pgsz >> PAGESHIFT; 2014 } else { 2015 VM_STAT_ADD(alloc_pages[3]); 2016 ASSERT(npgs == totpgs); 2017 page_create_putback(npgs); 2018 return (ENOMEM); 2019 } 2020 } 2021 if (szc == 0) { 2022 VM_STAT_ADD(alloc_pages[4]); 2023 ASSERT(npgs != 0); 2024 page_create_putback(npgs); 2025 err = ENOMEM; 2026 } else if (basepp != NULL) { 2027 ASSERT(npgs == 0); 2028 ASSERT(ppa == NULL); 2029 *basepp = pplist; 2030 } 2031 2032 npgs = totpgs - npgs; 2033 pp = pplist; 2034 2035 /* 2036 * Clear the free and age bits. Also if we were passed in a ppa then 2037 * fill it in with all the constituent pages from the large page. But 2038 * if we failed to allocate all the pages just free what we got. 2039 */ 2040 while (npgs != 0) { 2041 ASSERT(PP_ISFREE(pp)); 2042 ASSERT(PP_ISAGED(pp)); 2043 if (ppa != NULL || err != 0) { 2044 if (err == 0) { 2045 VM_STAT_ADD(alloc_pages[5]); 2046 PP_CLRFREE(pp); 2047 PP_CLRAGED(pp); 2048 page_sub(&pplist, pp); 2049 *ppa++ = pp; 2050 npgs--; 2051 } else { 2052 VM_STAT_ADD(alloc_pages[6]); 2053 ASSERT(pp->p_szc != 0); 2054 curnpgs = page_get_pagecnt(pp->p_szc); 2055 page_list_break(&pp, &pplist, curnpgs); 2056 page_list_add_pages(pp, 0); 2057 page_create_putback(curnpgs); 2058 ASSERT(npgs >= curnpgs); 2059 npgs -= curnpgs; 2060 } 2061 pp = pplist; 2062 } else { 2063 VM_STAT_ADD(alloc_pages[7]); 2064 PP_CLRFREE(pp); 2065 PP_CLRAGED(pp); 2066 pp = pp->p_next; 2067 npgs--; 2068 } 2069 } 2070 return (err); 2071 } 2072 2073 /* 2074 * Get a single large page off of the freelists, and set it up for use. 2075 * Number of bytes requested must be a supported page size. 2076 * 2077 * Note that this call may fail even if there is sufficient 2078 * memory available or PG_WAIT is set, so the caller must 2079 * be willing to fallback on page_create_va(), block and retry, 2080 * or fail the requester. 2081 */ 2082 page_t * 2083 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2084 struct seg *seg, caddr_t vaddr, void *arg) 2085 { 2086 pgcnt_t npages, pcftotal; 2087 page_t *pp; 2088 page_t *rootpp; 2089 lgrp_t *lgrp; 2090 uint_t enough; 2091 uint_t pcf_index; 2092 uint_t i; 2093 struct pcf *p; 2094 struct pcf *q; 2095 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2096 2097 ASSERT(vp != NULL); 2098 2099 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2100 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2101 /* but no others */ 2102 2103 ASSERT((flags & PG_EXCL) == PG_EXCL); 2104 2105 npages = btop(bytes); 2106 2107 if (!kcage_on || panicstr) { 2108 /* 2109 * Cage is OFF, or we are single threaded in 2110 * panic, so make everything a RELOC request. 2111 */ 2112 flags &= ~PG_NORELOC; 2113 } 2114 2115 /* 2116 * Make sure there's adequate physical memory available. 2117 * Note: PG_WAIT is ignored here. 2118 */ 2119 if (freemem <= throttlefree + npages) { 2120 VM_STAT_ADD(page_create_large_cnt[1]); 2121 return (NULL); 2122 } 2123 2124 /* 2125 * If cage is on, dampen draw from cage when available 2126 * cage space is low. 2127 */ 2128 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2129 kcage_freemem < kcage_throttlefree + npages) { 2130 2131 /* 2132 * The cage is on, the caller wants PG_NORELOC 2133 * pages and available cage memory is very low. 2134 * Call kcage_create_throttle() to attempt to 2135 * control demand on the cage. 2136 */ 2137 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2138 VM_STAT_ADD(page_create_large_cnt[2]); 2139 return (NULL); 2140 } 2141 } 2142 2143 enough = 0; 2144 pcf_index = PCF_INDEX(); 2145 p = &pcf[pcf_index]; 2146 p->pcf_touch = 1; 2147 q = &pcf[PCF_FANOUT]; 2148 for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2149 if (p->pcf_count > npages) { 2150 /* 2151 * a good one to try. 2152 */ 2153 mutex_enter(&p->pcf_lock); 2154 if (p->pcf_count > npages) { 2155 p->pcf_count -= (uint_t)npages; 2156 /* 2157 * freemem is not protected by any lock. 2158 * Thus, we cannot have any assertion 2159 * containing freemem here. 2160 */ 2161 freemem -= npages; 2162 enough = 1; 2163 mutex_exit(&p->pcf_lock); 2164 break; 2165 } 2166 mutex_exit(&p->pcf_lock); 2167 } 2168 pcftotal += p->pcf_count; 2169 p++; 2170 if (p >= q) { 2171 p = pcf; 2172 } 2173 p->pcf_touch = 1; 2174 } 2175 2176 if (!enough) { 2177 /* If there isn't enough memory available, give up. */ 2178 if (pcftotal < npages) { 2179 VM_STAT_ADD(page_create_large_cnt[3]); 2180 return (NULL); 2181 } 2182 2183 /* try to collect pages from several pcf bins */ 2184 for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2185 p->pcf_touch = 1; 2186 mutex_enter(&p->pcf_lock); 2187 pcftotal += p->pcf_count; 2188 if (pcftotal >= npages) { 2189 /* 2190 * Wow! There are enough pages laying around 2191 * to satisfy the request. Do the accounting, 2192 * drop the locks we acquired, and go back. 2193 * 2194 * freemem is not protected by any lock. So, 2195 * we cannot have any assertion containing 2196 * freemem. 2197 */ 2198 pgcnt_t tpages = npages; 2199 freemem -= npages; 2200 while (p >= pcf) { 2201 if (p->pcf_count <= tpages) { 2202 tpages -= p->pcf_count; 2203 p->pcf_count = 0; 2204 } else { 2205 p->pcf_count -= (uint_t)tpages; 2206 tpages = 0; 2207 } 2208 mutex_exit(&p->pcf_lock); 2209 p--; 2210 } 2211 ASSERT(tpages == 0); 2212 break; 2213 } 2214 p++; 2215 } 2216 if (i == PCF_FANOUT) { 2217 /* failed to collect pages - release the locks */ 2218 while (--p >= pcf) { 2219 mutex_exit(&p->pcf_lock); 2220 } 2221 VM_STAT_ADD(page_create_large_cnt[4]); 2222 return (NULL); 2223 } 2224 } 2225 2226 /* 2227 * This is where this function behaves fundamentally differently 2228 * than page_create_va(); since we're intending to map the page 2229 * with a single TTE, we have to get it as a physically contiguous 2230 * hardware pagesize chunk. If we can't, we fail. 2231 */ 2232 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2233 LGRP_EXISTS(lgrp_table[*lgrpid])) 2234 lgrp = lgrp_table[*lgrpid]; 2235 else 2236 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2237 2238 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2239 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2240 page_create_putback(npages); 2241 VM_STAT_ADD(page_create_large_cnt[5]); 2242 return (NULL); 2243 } 2244 2245 /* 2246 * if we got the page with the wrong mtype give it back this is a 2247 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2248 * inside "if" and the workaround becomes just a nop 2249 */ 2250 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2251 page_list_add_pages(rootpp, 0); 2252 page_create_putback(npages); 2253 VM_STAT_ADD(page_create_large_cnt[6]); 2254 return (NULL); 2255 } 2256 2257 /* 2258 * If satisfying this request has left us with too little 2259 * memory, start the wheels turning to get some back. The 2260 * first clause of the test prevents waking up the pageout 2261 * daemon in situations where it would decide that there's 2262 * nothing to do. 2263 */ 2264 if (nscan < desscan && freemem < minfree) { 2265 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2266 "pageout_cv_signal:freemem %ld", freemem); 2267 cv_signal(&proc_pageout->p_cv); 2268 } 2269 2270 pp = rootpp; 2271 while (npages--) { 2272 ASSERT(PAGE_EXCL(pp)); 2273 ASSERT(pp->p_vnode == NULL); 2274 ASSERT(!hat_page_is_mapped(pp)); 2275 PP_CLRFREE(pp); 2276 PP_CLRAGED(pp); 2277 if (!page_hashin(pp, vp, off, NULL)) 2278 panic("page_create_large: hashin failed: page %p", 2279 (void *)pp); 2280 page_io_lock(pp); 2281 off += PAGESIZE; 2282 pp = pp->p_next; 2283 } 2284 2285 VM_STAT_ADD(page_create_large_cnt[0]); 2286 return (rootpp); 2287 } 2288 2289 page_t * 2290 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2291 struct seg *seg, caddr_t vaddr) 2292 { 2293 page_t *plist = NULL; 2294 pgcnt_t npages; 2295 pgcnt_t found_on_free = 0; 2296 pgcnt_t pages_req; 2297 page_t *npp = NULL; 2298 uint_t enough; 2299 uint_t i; 2300 uint_t pcf_index; 2301 struct pcf *p; 2302 struct pcf *q; 2303 lgrp_t *lgrp; 2304 2305 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2306 "page_create_start:vp %p off %llx bytes %lu flags %x", 2307 vp, off, bytes, flags); 2308 2309 ASSERT(bytes != 0 && vp != NULL); 2310 2311 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2312 panic("page_create: invalid flags"); 2313 /*NOTREACHED*/ 2314 } 2315 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2316 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2317 /* but no others */ 2318 2319 pages_req = npages = btopr(bytes); 2320 /* 2321 * Try to see whether request is too large to *ever* be 2322 * satisfied, in order to prevent deadlock. We arbitrarily 2323 * decide to limit maximum size requests to max_page_get. 2324 */ 2325 if (npages >= max_page_get) { 2326 if ((flags & PG_WAIT) == 0) { 2327 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2328 "page_create_toobig:vp %p off %llx npages " 2329 "%lu max_page_get %lu", 2330 vp, off, npages, max_page_get); 2331 return (NULL); 2332 } else { 2333 cmn_err(CE_WARN, 2334 "Request for too much kernel memory " 2335 "(%lu bytes), will hang forever", bytes); 2336 for (;;) 2337 delay(1000000000); 2338 } 2339 } 2340 2341 if (!kcage_on || panicstr) { 2342 /* 2343 * Cage is OFF, or we are single threaded in 2344 * panic, so make everything a RELOC request. 2345 */ 2346 flags &= ~PG_NORELOC; 2347 } 2348 2349 if (freemem <= throttlefree + npages) 2350 if (!page_create_throttle(npages, flags)) 2351 return (NULL); 2352 2353 /* 2354 * If cage is on, dampen draw from cage when available 2355 * cage space is low. 2356 */ 2357 if ((flags & PG_NORELOC) && 2358 kcage_freemem < kcage_throttlefree + npages) { 2359 2360 /* 2361 * The cage is on, the caller wants PG_NORELOC 2362 * pages and available cage memory is very low. 2363 * Call kcage_create_throttle() to attempt to 2364 * control demand on the cage. 2365 */ 2366 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2367 return (NULL); 2368 } 2369 2370 VM_STAT_ADD(page_create_cnt[0]); 2371 2372 enough = 0; 2373 pcf_index = PCF_INDEX(); 2374 2375 p = &pcf[pcf_index]; 2376 p->pcf_touch = 1; 2377 q = &pcf[PCF_FANOUT]; 2378 for (i = 0; i < PCF_FANOUT; i++) { 2379 if (p->pcf_count > npages) { 2380 /* 2381 * a good one to try. 2382 */ 2383 mutex_enter(&p->pcf_lock); 2384 if (p->pcf_count > npages) { 2385 p->pcf_count -= (uint_t)npages; 2386 /* 2387 * freemem is not protected by any lock. 2388 * Thus, we cannot have any assertion 2389 * containing freemem here. 2390 */ 2391 freemem -= npages; 2392 enough = 1; 2393 mutex_exit(&p->pcf_lock); 2394 break; 2395 } 2396 mutex_exit(&p->pcf_lock); 2397 } 2398 p++; 2399 if (p >= q) { 2400 p = pcf; 2401 } 2402 p->pcf_touch = 1; 2403 } 2404 2405 if (!enough) { 2406 /* 2407 * Have to look harder. If npages is greater than 2408 * one, then we might have to coalecse the counters. 2409 * 2410 * Go wait. We come back having accounted 2411 * for the memory. 2412 */ 2413 VM_STAT_ADD(page_create_cnt[1]); 2414 if (!page_create_wait(npages, flags)) { 2415 VM_STAT_ADD(page_create_cnt[2]); 2416 return (NULL); 2417 } 2418 } 2419 2420 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2421 "page_create_success:vp %p off %llx", vp, off); 2422 2423 /* 2424 * If satisfying this request has left us with too little 2425 * memory, start the wheels turning to get some back. The 2426 * first clause of the test prevents waking up the pageout 2427 * daemon in situations where it would decide that there's 2428 * nothing to do. 2429 */ 2430 if (nscan < desscan && freemem < minfree) { 2431 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2432 "pageout_cv_signal:freemem %ld", freemem); 2433 cv_signal(&proc_pageout->p_cv); 2434 } 2435 2436 /* 2437 * Loop around collecting the requested number of pages. 2438 * Most of the time, we have to `create' a new page. With 2439 * this in mind, pull the page off the free list before 2440 * getting the hash lock. This will minimize the hash 2441 * lock hold time, nesting, and the like. If it turns 2442 * out we don't need the page, we put it back at the end. 2443 */ 2444 while (npages--) { 2445 page_t *pp; 2446 kmutex_t *phm = NULL; 2447 ulong_t index; 2448 2449 index = PAGE_HASH_FUNC(vp, off); 2450 top: 2451 ASSERT(phm == NULL); 2452 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2453 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2454 2455 if (npp == NULL) { 2456 /* 2457 * Try to get a page from the freelist (ie, 2458 * a page with no [vp, off] tag). If that 2459 * fails, use the cachelist. 2460 * 2461 * During the first attempt at both the free 2462 * and cache lists we try for the correct color. 2463 */ 2464 /* 2465 * XXXX-how do we deal with virtual indexed 2466 * caches and and colors? 2467 */ 2468 VM_STAT_ADD(page_create_cnt[4]); 2469 /* 2470 * Get lgroup to allocate next page of shared memory 2471 * from and use it to specify where to allocate 2472 * the physical memory 2473 */ 2474 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2475 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2476 flags | PG_MATCH_COLOR, lgrp); 2477 if (npp == NULL) { 2478 npp = page_get_cachelist(vp, off, seg, 2479 vaddr, flags | PG_MATCH_COLOR, lgrp); 2480 if (npp == NULL) { 2481 npp = page_create_get_something(vp, 2482 off, seg, vaddr, 2483 flags & ~PG_MATCH_COLOR); 2484 } 2485 2486 if (PP_ISAGED(npp) == 0) { 2487 /* 2488 * Since this page came from the 2489 * cachelist, we must destroy the 2490 * old vnode association. 2491 */ 2492 page_hashout(npp, NULL); 2493 } 2494 } 2495 } 2496 2497 /* 2498 * We own this page! 2499 */ 2500 ASSERT(PAGE_EXCL(npp)); 2501 ASSERT(npp->p_vnode == NULL); 2502 ASSERT(!hat_page_is_mapped(npp)); 2503 PP_CLRFREE(npp); 2504 PP_CLRAGED(npp); 2505 2506 /* 2507 * Here we have a page in our hot little mits and are 2508 * just waiting to stuff it on the appropriate lists. 2509 * Get the mutex and check to see if it really does 2510 * not exist. 2511 */ 2512 phm = PAGE_HASH_MUTEX(index); 2513 mutex_enter(phm); 2514 PAGE_HASH_SEARCH(index, pp, vp, off); 2515 if (pp == NULL) { 2516 VM_STAT_ADD(page_create_new); 2517 pp = npp; 2518 npp = NULL; 2519 if (!page_hashin(pp, vp, off, phm)) { 2520 /* 2521 * Since we hold the page hash mutex and 2522 * just searched for this page, page_hashin 2523 * had better not fail. If it does, that 2524 * means somethread did not follow the 2525 * page hash mutex rules. Panic now and 2526 * get it over with. As usual, go down 2527 * holding all the locks. 2528 */ 2529 ASSERT(MUTEX_HELD(phm)); 2530 panic("page_create: " 2531 "hashin failed %p %p %llx %p", 2532 (void *)pp, (void *)vp, off, (void *)phm); 2533 /*NOTREACHED*/ 2534 } 2535 ASSERT(MUTEX_HELD(phm)); 2536 mutex_exit(phm); 2537 phm = NULL; 2538 2539 /* 2540 * Hat layer locking need not be done to set 2541 * the following bits since the page is not hashed 2542 * and was on the free list (i.e., had no mappings). 2543 * 2544 * Set the reference bit to protect 2545 * against immediate pageout 2546 * 2547 * XXXmh modify freelist code to set reference 2548 * bit so we don't have to do it here. 2549 */ 2550 page_set_props(pp, P_REF); 2551 found_on_free++; 2552 } else { 2553 VM_STAT_ADD(page_create_exists); 2554 if (flags & PG_EXCL) { 2555 /* 2556 * Found an existing page, and the caller 2557 * wanted all new pages. Undo all of the work 2558 * we have done. 2559 */ 2560 mutex_exit(phm); 2561 phm = NULL; 2562 while (plist != NULL) { 2563 pp = plist; 2564 page_sub(&plist, pp); 2565 page_io_unlock(pp); 2566 /* large pages should not end up here */ 2567 ASSERT(pp->p_szc == 0); 2568 /*LINTED: constant in conditional ctx*/ 2569 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2570 } 2571 VM_STAT_ADD(page_create_found_one); 2572 goto fail; 2573 } 2574 ASSERT(flags & PG_WAIT); 2575 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2576 /* 2577 * Start all over again if we blocked trying 2578 * to lock the page. 2579 */ 2580 mutex_exit(phm); 2581 VM_STAT_ADD(page_create_page_lock_failed); 2582 phm = NULL; 2583 goto top; 2584 } 2585 mutex_exit(phm); 2586 phm = NULL; 2587 2588 if (PP_ISFREE(pp)) { 2589 ASSERT(PP_ISAGED(pp) == 0); 2590 VM_STAT_ADD(pagecnt.pc_get_cache); 2591 page_list_sub(pp, PG_CACHE_LIST); 2592 PP_CLRFREE(pp); 2593 found_on_free++; 2594 } 2595 } 2596 2597 /* 2598 * Got a page! It is locked. Acquire the i/o 2599 * lock since we are going to use the p_next and 2600 * p_prev fields to link the requested pages together. 2601 */ 2602 page_io_lock(pp); 2603 page_add(&plist, pp); 2604 plist = plist->p_next; 2605 off += PAGESIZE; 2606 vaddr += PAGESIZE; 2607 } 2608 2609 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2610 fail: 2611 if (npp != NULL) { 2612 /* 2613 * Did not need this page after all. 2614 * Put it back on the free list. 2615 */ 2616 VM_STAT_ADD(page_create_putbacks); 2617 PP_SETFREE(npp); 2618 PP_SETAGED(npp); 2619 npp->p_offset = (u_offset_t)-1; 2620 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2621 page_unlock(npp); 2622 2623 } 2624 2625 ASSERT(pages_req >= found_on_free); 2626 2627 { 2628 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2629 2630 if (overshoot) { 2631 VM_STAT_ADD(page_create_overshoot); 2632 p = &pcf[pcf_index]; 2633 p->pcf_touch = 1; 2634 mutex_enter(&p->pcf_lock); 2635 if (p->pcf_block) { 2636 p->pcf_reserve += overshoot; 2637 } else { 2638 p->pcf_count += overshoot; 2639 if (p->pcf_wait) { 2640 mutex_enter(&new_freemem_lock); 2641 if (freemem_wait) { 2642 cv_signal(&freemem_cv); 2643 p->pcf_wait--; 2644 } else { 2645 p->pcf_wait = 0; 2646 } 2647 mutex_exit(&new_freemem_lock); 2648 } 2649 } 2650 mutex_exit(&p->pcf_lock); 2651 /* freemem is approximate, so this test OK */ 2652 if (!p->pcf_block) 2653 freemem += overshoot; 2654 } 2655 } 2656 2657 return (plist); 2658 } 2659 2660 /* 2661 * One or more constituent pages of this large page has been marked 2662 * toxic. Simply demote the large page to PAGESIZE pages and let 2663 * page_free() handle it. This routine should only be called by 2664 * large page free routines (page_free_pages() and page_destroy_pages(). 2665 * All pages are locked SE_EXCL and have already been marked free. 2666 */ 2667 static void 2668 page_free_toxic_pages(page_t *rootpp) 2669 { 2670 page_t *tpp; 2671 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2672 uint_t szc = rootpp->p_szc; 2673 2674 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2675 ASSERT(tpp->p_szc == szc); 2676 ASSERT((PAGE_EXCL(tpp) && 2677 !page_iolock_assert(tpp)) || panicstr); 2678 tpp->p_szc = 0; 2679 } 2680 2681 while (rootpp != NULL) { 2682 tpp = rootpp; 2683 page_sub(&rootpp, tpp); 2684 ASSERT(PP_ISFREE(tpp)); 2685 PP_CLRFREE(tpp); 2686 page_free(tpp, 1); 2687 } 2688 } 2689 2690 /* 2691 * Put page on the "free" list. 2692 * The free list is really two lists maintained by 2693 * the PSM of whatever machine we happen to be on. 2694 */ 2695 void 2696 page_free(page_t *pp, int dontneed) 2697 { 2698 struct pcf *p; 2699 uint_t pcf_index; 2700 2701 ASSERT((PAGE_EXCL(pp) && 2702 !page_iolock_assert(pp)) || panicstr); 2703 2704 if (PP_ISFREE(pp)) { 2705 panic("page_free: page %p is free", (void *)pp); 2706 } 2707 2708 if (pp->p_szc != 0) { 2709 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2710 pp->p_vnode == &kvp) { 2711 panic("page_free: anon or kernel " 2712 "or no vnode large page %p", (void *)pp); 2713 } 2714 page_demote_vp_pages(pp); 2715 ASSERT(pp->p_szc == 0); 2716 } 2717 2718 /* 2719 * The page_struct_lock need not be acquired to examine these 2720 * fields since the page has an "exclusive" lock. 2721 */ 2722 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 2723 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d", 2724 pp, page_pptonum(pp), pp->p_lckcnt, pp->p_cowcnt); 2725 /*NOTREACHED*/ 2726 } 2727 2728 ASSERT(!hat_page_getshare(pp)); 2729 2730 PP_SETFREE(pp); 2731 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2732 !hat_ismod(pp)); 2733 page_clr_all_props(pp); 2734 ASSERT(!hat_page_getshare(pp)); 2735 2736 /* 2737 * Now we add the page to the head of the free list. 2738 * But if this page is associated with a paged vnode 2739 * then we adjust the head forward so that the page is 2740 * effectively at the end of the list. 2741 */ 2742 if (pp->p_vnode == NULL) { 2743 /* 2744 * Page has no identity, put it on the free list. 2745 */ 2746 PP_SETAGED(pp); 2747 pp->p_offset = (u_offset_t)-1; 2748 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2749 VM_STAT_ADD(pagecnt.pc_free_free); 2750 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2751 "page_free_free:pp %p", pp); 2752 } else { 2753 PP_CLRAGED(pp); 2754 2755 if (!dontneed || nopageage) { 2756 /* move it to the tail of the list */ 2757 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2758 2759 VM_STAT_ADD(pagecnt.pc_free_cache); 2760 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2761 "page_free_cache_tail:pp %p", pp); 2762 } else { 2763 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2764 2765 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2766 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2767 "page_free_cache_head:pp %p", pp); 2768 } 2769 } 2770 page_unlock(pp); 2771 2772 /* 2773 * Now do the `freemem' accounting. 2774 */ 2775 pcf_index = PCF_INDEX(); 2776 p = &pcf[pcf_index]; 2777 p->pcf_touch = 1; 2778 2779 mutex_enter(&p->pcf_lock); 2780 if (p->pcf_block) { 2781 p->pcf_reserve += 1; 2782 } else { 2783 p->pcf_count += 1; 2784 if (p->pcf_wait) { 2785 mutex_enter(&new_freemem_lock); 2786 /* 2787 * Check to see if some other thread 2788 * is actually waiting. Another bucket 2789 * may have woken it up by now. If there 2790 * are no waiters, then set our pcf_wait 2791 * count to zero to avoid coming in here 2792 * next time. Also, since only one page 2793 * was put on the free list, just wake 2794 * up one waiter. 2795 */ 2796 if (freemem_wait) { 2797 cv_signal(&freemem_cv); 2798 p->pcf_wait--; 2799 } else { 2800 p->pcf_wait = 0; 2801 } 2802 mutex_exit(&new_freemem_lock); 2803 } 2804 } 2805 mutex_exit(&p->pcf_lock); 2806 2807 /* freemem is approximate, so this test OK */ 2808 if (!p->pcf_block) 2809 freemem += 1; 2810 } 2811 2812 /* 2813 * Put page on the "free" list during intial startup. 2814 * This happens during initial single threaded execution. 2815 */ 2816 void 2817 page_free_at_startup(page_t *pp) 2818 { 2819 struct pcf *p; 2820 uint_t pcf_index; 2821 2822 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2823 VM_STAT_ADD(pagecnt.pc_free_free); 2824 2825 /* 2826 * Now do the `freemem' accounting. 2827 */ 2828 pcf_index = PCF_INDEX(); 2829 p = &pcf[pcf_index]; 2830 p->pcf_touch = 1; 2831 2832 ASSERT(p->pcf_block == 0); 2833 ASSERT(p->pcf_wait == 0); 2834 p->pcf_count += 1; 2835 2836 /* freemem is approximate, so this is OK */ 2837 freemem += 1; 2838 } 2839 2840 void 2841 page_free_pages(page_t *pp) 2842 { 2843 page_t *tpp, *rootpp = NULL; 2844 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2845 pgcnt_t i; 2846 uint_t szc = pp->p_szc; 2847 2848 VM_STAT_ADD(pagecnt.pc_free_pages); 2849 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2850 "page_free_free:pp %p", pp); 2851 2852 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2853 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2854 panic("page_free_pages: not root page %p", (void *)pp); 2855 /*NOTREACHED*/ 2856 } 2857 2858 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2859 ASSERT((PAGE_EXCL(tpp) && 2860 !page_iolock_assert(tpp)) || panicstr); 2861 if (PP_ISFREE(tpp)) { 2862 panic("page_free_pages: page %p is free", (void *)tpp); 2863 /*NOTREACHED*/ 2864 } 2865 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2866 tpp->p_cowcnt != 0) { 2867 panic("page_free_pages %p", (void *)tpp); 2868 /*NOTREACHED*/ 2869 } 2870 2871 ASSERT(!hat_page_getshare(tpp)); 2872 ASSERT(tpp->p_vnode == NULL); 2873 ASSERT(tpp->p_szc == szc); 2874 2875 PP_SETFREE(tpp); 2876 page_clr_all_props(tpp); 2877 PP_SETAGED(tpp); 2878 tpp->p_offset = (u_offset_t)-1; 2879 ASSERT(tpp->p_next == tpp); 2880 ASSERT(tpp->p_prev == tpp); 2881 page_list_concat(&rootpp, &tpp); 2882 } 2883 ASSERT(rootpp == pp); 2884 2885 page_list_add_pages(rootpp, 0); 2886 page_create_putback(pgcnt); 2887 } 2888 2889 int free_pages = 1; 2890 2891 /* 2892 * This routine attempts to return pages to the cachelist via page_release(). 2893 * It does not *have* to be successful in all cases, since the pageout scanner 2894 * will catch any pages it misses. It does need to be fast and not introduce 2895 * too much overhead. 2896 * 2897 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2898 * don't lock and retry. This is ok, since the page scanner will eventually 2899 * find any page we miss in free_vp_pages(). 2900 */ 2901 void 2902 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2903 { 2904 page_t *pp; 2905 u_offset_t eoff; 2906 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2907 2908 eoff = off + len; 2909 2910 if (free_pages == 0) 2911 return; 2912 if (swap_in_range(vp, off, len)) 2913 return; 2914 2915 for (; off < eoff; off += PAGESIZE) { 2916 2917 /* 2918 * find the page using a fast, but inexact search. It'll be OK 2919 * if a few pages slip through the cracks here. 2920 */ 2921 pp = page_exists(vp, off); 2922 2923 /* 2924 * If we didn't find the page (it may not exist), the page 2925 * is free, looks still in use (shared), or we can't lock it, 2926 * just give up. 2927 */ 2928 if (pp == NULL || 2929 PP_ISFREE(pp) || 2930 page_share_cnt(pp) > 0 || 2931 !page_trylock(pp, SE_EXCL)) 2932 continue; 2933 2934 /* 2935 * Once we have locked pp, verify that it's still the 2936 * correct page and not already free 2937 */ 2938 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2939 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2940 page_unlock(pp); 2941 continue; 2942 } 2943 2944 /* 2945 * try to release the page... 2946 */ 2947 (void) page_release(pp, 1); 2948 } 2949 } 2950 2951 /* 2952 * Reclaim the given page from the free list. 2953 * Returns 1 on success or 0 on failure. 2954 * 2955 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2956 * If `lock' is non-null, it will be dropped and re-acquired if 2957 * the routine must wait while freemem is 0. 2958 * 2959 * As it turns out, boot_getpages() does this. It picks a page, 2960 * based on where OBP mapped in some address, gets its pfn, searches 2961 * the memsegs, locks the page, then pulls it off the free list! 2962 */ 2963 int 2964 page_reclaim(page_t *pp, kmutex_t *lock) 2965 { 2966 struct pcf *p; 2967 uint_t pcf_index; 2968 struct cpu *cpup; 2969 uint_t i; 2970 pgcnt_t npgs, need; 2971 pgcnt_t collected = 0; 2972 2973 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2974 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2975 2976 npgs = page_get_pagecnt(pp->p_szc); 2977 2978 /* 2979 * If `freemem' is 0, we cannot reclaim this page from the 2980 * freelist, so release every lock we might hold: the page, 2981 * and the `lock' before blocking. 2982 * 2983 * The only way `freemem' can become 0 while there are pages 2984 * marked free (have their p->p_free bit set) is when the 2985 * system is low on memory and doing a page_create(). In 2986 * order to guarantee that once page_create() starts acquiring 2987 * pages it will be able to get all that it needs since `freemem' 2988 * was decreased by the requested amount. So, we need to release 2989 * this page, and let page_create() have it. 2990 * 2991 * Since `freemem' being zero is not supposed to happen, just 2992 * use the usual hash stuff as a starting point. If that bucket 2993 * is empty, then assume the worst, and start at the beginning 2994 * of the pcf array. If we always start at the beginning 2995 * when acquiring more than one pcf lock, there won't be any 2996 * deadlock problems. 2997 */ 2998 2999 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 3000 3001 if (freemem <= throttlefree && !page_create_throttle(npgs, 0)) { 3002 pcf_acquire_all(); 3003 goto page_reclaim_nomem; 3004 } 3005 3006 pcf_index = PCF_INDEX(); 3007 p = &pcf[pcf_index]; 3008 p->pcf_touch = 1; 3009 mutex_enter(&p->pcf_lock); 3010 if (p->pcf_count >= npgs) { 3011 collected = npgs; 3012 p->pcf_count -= npgs; 3013 } 3014 mutex_exit(&p->pcf_lock); 3015 need = npgs - collected; 3016 3017 if (need > 0) { 3018 VM_STAT_ADD(page_reclaim_zero); 3019 /* 3020 * Check again. Its possible that some other thread 3021 * could have been right behind us, and added one 3022 * to a list somewhere. Acquire each of the pcf locks 3023 * until we find a page. 3024 */ 3025 p = pcf; 3026 for (i = 0; i < PCF_FANOUT; i++) { 3027 p->pcf_touch = 1; 3028 mutex_enter(&p->pcf_lock); 3029 if (p->pcf_count) { 3030 if (p->pcf_count >= need) { 3031 p->pcf_count -= need; 3032 collected += need; 3033 need = 0; 3034 break; 3035 } else if (p->pcf_count) { 3036 collected += p->pcf_count; 3037 need -= p->pcf_count; 3038 p->pcf_count = 0; 3039 } 3040 } 3041 p++; 3042 } 3043 3044 if (need > 0) { 3045 page_reclaim_nomem: 3046 /* 3047 * We really can't have page `pp'. 3048 * Time for the no-memory dance with 3049 * page_free(). This is just like 3050 * page_create_wait(). Plus the added 3051 * attraction of releasing whatever mutex 3052 * we held when we were called with in `lock'. 3053 * Page_unlock() will wakeup any thread 3054 * waiting around for this page. 3055 */ 3056 if (lock) { 3057 VM_STAT_ADD(page_reclaim_zero_locked); 3058 mutex_exit(lock); 3059 } 3060 page_unlock(pp); 3061 3062 /* 3063 * get this before we drop all the pcf locks. 3064 */ 3065 mutex_enter(&new_freemem_lock); 3066 3067 p = pcf; 3068 p->pcf_count += collected; 3069 for (i = 0; i < PCF_FANOUT; i++) { 3070 p->pcf_wait++; 3071 mutex_exit(&p->pcf_lock); 3072 p++; 3073 } 3074 3075 freemem_wait++; 3076 cv_wait(&freemem_cv, &new_freemem_lock); 3077 freemem_wait--; 3078 3079 mutex_exit(&new_freemem_lock); 3080 3081 if (lock) { 3082 mutex_enter(lock); 3083 } 3084 return (0); 3085 } 3086 3087 /* 3088 * We beat the PCF bins over the head until 3089 * we got the memory that we wanted. 3090 * The pcf accounting has been done, 3091 * though none of the pcf_wait flags have been set, 3092 * drop the locks and continue on. 3093 */ 3094 ASSERT(collected == npgs); 3095 while (p >= pcf) { 3096 mutex_exit(&p->pcf_lock); 3097 p--; 3098 } 3099 } 3100 3101 /* 3102 * freemem is not protected by any lock. Thus, we cannot 3103 * have any assertion containing freemem here. 3104 */ 3105 freemem -= npgs; 3106 3107 VM_STAT_ADD(pagecnt.pc_reclaim); 3108 if (PP_ISAGED(pp)) { 3109 if (npgs > 1) { 3110 page_list_sub_pages(pp, pp->p_szc); 3111 } else { 3112 page_list_sub(pp, PG_FREE_LIST); 3113 } 3114 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3115 "page_reclaim_free:pp %p", pp); 3116 } else { 3117 ASSERT(npgs == 1); 3118 page_list_sub(pp, PG_CACHE_LIST); 3119 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3120 "page_reclaim_cache:pp %p", pp); 3121 } 3122 3123 /* 3124 * clear the p_free & p_age bits since this page is no longer 3125 * on the free list. Notice that there was a brief time where 3126 * a page is marked as free, but is not on the list. 3127 * 3128 * Set the reference bit to protect against immediate pageout. 3129 */ 3130 for (i = 0; i < npgs; i++, pp++) { 3131 PP_CLRFREE(pp); 3132 PP_CLRAGED(pp); 3133 page_set_props(pp, P_REF); 3134 } 3135 3136 CPU_STATS_ENTER_K(); 3137 cpup = CPU; /* get cpup now that CPU cannot change */ 3138 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3139 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3140 CPU_STATS_EXIT_K(); 3141 3142 return (1); 3143 } 3144 3145 3146 3147 /* 3148 * Destroy identity of the page and put it back on 3149 * the page free list. Assumes that the caller has 3150 * acquired the "exclusive" lock on the page. 3151 */ 3152 void 3153 page_destroy(page_t *pp, int dontfree) 3154 { 3155 ASSERT((PAGE_EXCL(pp) && 3156 !page_iolock_assert(pp)) || panicstr); 3157 3158 if (pp->p_szc != 0) { 3159 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3160 pp->p_vnode == &kvp) { 3161 panic("page_destroy: anon or kernel or no vnode " 3162 "large page %p", (void *)pp); 3163 } 3164 page_demote_vp_pages(pp); 3165 ASSERT(pp->p_szc == 0); 3166 } 3167 3168 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3169 3170 /* 3171 * Unload translations, if any, then hash out the 3172 * page to erase its identity. 3173 */ 3174 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3175 page_hashout(pp, NULL); 3176 3177 if (!dontfree) { 3178 /* 3179 * Acquire the "freemem_lock" for availrmem. 3180 * The page_struct_lock need not be acquired for lckcnt 3181 * and cowcnt since the page has an "exclusive" lock. 3182 */ 3183 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3184 mutex_enter(&freemem_lock); 3185 if (pp->p_lckcnt != 0) { 3186 availrmem++; 3187 pp->p_lckcnt = 0; 3188 } 3189 if (pp->p_cowcnt != 0) { 3190 availrmem += pp->p_cowcnt; 3191 pp->p_cowcnt = 0; 3192 } 3193 mutex_exit(&freemem_lock); 3194 } 3195 /* 3196 * Put the page on the "free" list. 3197 */ 3198 page_free(pp, 0); 3199 } 3200 } 3201 3202 void 3203 page_destroy_pages(page_t *pp) 3204 { 3205 3206 page_t *tpp, *rootpp = NULL; 3207 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3208 pgcnt_t i, pglcks = 0; 3209 uint_t szc = pp->p_szc; 3210 3211 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3212 3213 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3214 3215 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3216 3217 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3218 panic("page_destroy_pages: not root page %p", (void *)pp); 3219 /*NOTREACHED*/ 3220 } 3221 3222 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3223 ASSERT((PAGE_EXCL(tpp) && 3224 !page_iolock_assert(tpp)) || panicstr); 3225 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3226 page_hashout(tpp, NULL); 3227 ASSERT(tpp->p_offset == (u_offset_t)-1); 3228 if (tpp->p_lckcnt != 0) { 3229 pglcks++; 3230 tpp->p_lckcnt = 0; 3231 } else if (tpp->p_cowcnt != 0) { 3232 pglcks += tpp->p_cowcnt; 3233 tpp->p_cowcnt = 0; 3234 } 3235 ASSERT(!hat_page_getshare(tpp)); 3236 ASSERT(tpp->p_vnode == NULL); 3237 ASSERT(tpp->p_szc == szc); 3238 3239 PP_SETFREE(tpp); 3240 page_clr_all_props(tpp); 3241 PP_SETAGED(tpp); 3242 ASSERT(tpp->p_next == tpp); 3243 ASSERT(tpp->p_prev == tpp); 3244 page_list_concat(&rootpp, &tpp); 3245 } 3246 3247 ASSERT(rootpp == pp); 3248 if (pglcks != 0) { 3249 mutex_enter(&freemem_lock); 3250 availrmem += pglcks; 3251 mutex_exit(&freemem_lock); 3252 } 3253 3254 page_list_add_pages(rootpp, 0); 3255 page_create_putback(pgcnt); 3256 } 3257 3258 /* 3259 * Similar to page_destroy(), but destroys pages which are 3260 * locked and known to be on the page free list. Since 3261 * the page is known to be free and locked, no one can access 3262 * it. 3263 * 3264 * Also, the number of free pages does not change. 3265 */ 3266 void 3267 page_destroy_free(page_t *pp) 3268 { 3269 ASSERT(PAGE_EXCL(pp)); 3270 ASSERT(PP_ISFREE(pp)); 3271 ASSERT(pp->p_vnode); 3272 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3273 ASSERT(!hat_page_is_mapped(pp)); 3274 ASSERT(PP_ISAGED(pp) == 0); 3275 ASSERT(pp->p_szc == 0); 3276 3277 VM_STAT_ADD(pagecnt.pc_destroy_free); 3278 page_list_sub(pp, PG_CACHE_LIST); 3279 3280 page_hashout(pp, NULL); 3281 ASSERT(pp->p_vnode == NULL); 3282 ASSERT(pp->p_offset == (u_offset_t)-1); 3283 ASSERT(pp->p_hash == NULL); 3284 3285 PP_SETAGED(pp); 3286 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3287 page_unlock(pp); 3288 3289 mutex_enter(&new_freemem_lock); 3290 if (freemem_wait) { 3291 cv_signal(&freemem_cv); 3292 } 3293 mutex_exit(&new_freemem_lock); 3294 } 3295 3296 /* 3297 * Rename the page "opp" to have an identity specified 3298 * by [vp, off]. If a page already exists with this name 3299 * it is locked and destroyed. Note that the page's 3300 * translations are not unloaded during the rename. 3301 * 3302 * This routine is used by the anon layer to "steal" the 3303 * original page and is not unlike destroying a page and 3304 * creating a new page using the same page frame. 3305 * 3306 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3307 * caller 2 tries to rename B to A. 3308 */ 3309 void 3310 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3311 { 3312 page_t *pp; 3313 int olckcnt = 0; 3314 int ocowcnt = 0; 3315 kmutex_t *phm; 3316 ulong_t index; 3317 3318 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3319 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3320 ASSERT(PP_ISFREE(opp) == 0); 3321 3322 VM_STAT_ADD(page_rename_count); 3323 3324 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3325 "page rename:pp %p vp %p off %llx", opp, vp, off); 3326 3327 /* 3328 * CacheFS may call page_rename for a large NFS page 3329 * when both CacheFS and NFS mount points are used 3330 * by applications. Demote this large page before 3331 * renaming it, to ensure that there are no "partial" 3332 * large pages left lying around. 3333 */ 3334 if (opp->p_szc != 0) { 3335 vnode_t *ovp = opp->p_vnode; 3336 ASSERT(ovp != NULL); 3337 ASSERT(!IS_SWAPFSVP(ovp)); 3338 ASSERT(ovp != &kvp); 3339 page_demote_vp_pages(opp); 3340 ASSERT(opp->p_szc == 0); 3341 } 3342 3343 page_hashout(opp, NULL); 3344 PP_CLRAGED(opp); 3345 3346 /* 3347 * Acquire the appropriate page hash lock, since 3348 * we're going to rename the page. 3349 */ 3350 index = PAGE_HASH_FUNC(vp, off); 3351 phm = PAGE_HASH_MUTEX(index); 3352 mutex_enter(phm); 3353 top: 3354 /* 3355 * Look for an existing page with this name and destroy it if found. 3356 * By holding the page hash lock all the way to the page_hashin() 3357 * call, we are assured that no page can be created with this 3358 * identity. In the case when the phm lock is dropped to undo any 3359 * hat layer mappings, the existing page is held with an "exclusive" 3360 * lock, again preventing another page from being created with 3361 * this identity. 3362 */ 3363 PAGE_HASH_SEARCH(index, pp, vp, off); 3364 if (pp != NULL) { 3365 VM_STAT_ADD(page_rename_exists); 3366 3367 /* 3368 * As it turns out, this is one of only two places where 3369 * page_lock() needs to hold the passed in lock in the 3370 * successful case. In all of the others, the lock could 3371 * be dropped as soon as the attempt is made to lock 3372 * the page. It is tempting to add yet another arguement, 3373 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3374 */ 3375 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3376 /* 3377 * Went to sleep because the page could not 3378 * be locked. We were woken up when the page 3379 * was unlocked, or when the page was destroyed. 3380 * In either case, `phm' was dropped while we 3381 * slept. Hence we should not just roar through 3382 * this loop. 3383 */ 3384 goto top; 3385 } 3386 3387 /* 3388 * If an existing page is a large page, then demote 3389 * it to ensure that no "partial" large pages are 3390 * "created" after page_rename. An existing page 3391 * can be a CacheFS page, and can't belong to swapfs. 3392 */ 3393 if (hat_page_is_mapped(pp)) { 3394 /* 3395 * Unload translations. Since we hold the 3396 * exclusive lock on this page, the page 3397 * can not be changed while we drop phm. 3398 * This is also not a lock protocol violation, 3399 * but rather the proper way to do things. 3400 */ 3401 mutex_exit(phm); 3402 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3403 if (pp->p_szc != 0) { 3404 ASSERT(!IS_SWAPFSVP(vp)); 3405 ASSERT(vp != &kvp); 3406 page_demote_vp_pages(pp); 3407 ASSERT(pp->p_szc == 0); 3408 } 3409 mutex_enter(phm); 3410 } else if (pp->p_szc != 0) { 3411 ASSERT(!IS_SWAPFSVP(vp)); 3412 ASSERT(vp != &kvp); 3413 mutex_exit(phm); 3414 page_demote_vp_pages(pp); 3415 ASSERT(pp->p_szc == 0); 3416 mutex_enter(phm); 3417 } 3418 page_hashout(pp, phm); 3419 } 3420 /* 3421 * Hash in the page with the new identity. 3422 */ 3423 if (!page_hashin(opp, vp, off, phm)) { 3424 /* 3425 * We were holding phm while we searched for [vp, off] 3426 * and only dropped phm if we found and locked a page. 3427 * If we can't create this page now, then some thing 3428 * is really broken. 3429 */ 3430 panic("page_rename: Can't hash in page: %p", (void *)pp); 3431 /*NOTREACHED*/ 3432 } 3433 3434 ASSERT(MUTEX_HELD(phm)); 3435 mutex_exit(phm); 3436 3437 /* 3438 * Now that we have dropped phm, lets get around to finishing up 3439 * with pp. 3440 */ 3441 if (pp != NULL) { 3442 ASSERT(!hat_page_is_mapped(pp)); 3443 /* for now large pages should not end up here */ 3444 ASSERT(pp->p_szc == 0); 3445 /* 3446 * Save the locks for transfer to the new page and then 3447 * clear them so page_free doesn't think they're important. 3448 * The page_struct_lock need not be acquired for lckcnt and 3449 * cowcnt since the page has an "exclusive" lock. 3450 */ 3451 olckcnt = pp->p_lckcnt; 3452 ocowcnt = pp->p_cowcnt; 3453 pp->p_lckcnt = pp->p_cowcnt = 0; 3454 3455 /* 3456 * Put the page on the "free" list after we drop 3457 * the lock. The less work under the lock the better. 3458 */ 3459 /*LINTED: constant in conditional context*/ 3460 VN_DISPOSE(pp, B_FREE, 0, kcred); 3461 } 3462 3463 /* 3464 * Transfer the lock count from the old page (if any). 3465 * The page_struct_lock need not be acquired for lckcnt and 3466 * cowcnt since the page has an "exclusive" lock. 3467 */ 3468 opp->p_lckcnt += olckcnt; 3469 opp->p_cowcnt += ocowcnt; 3470 } 3471 3472 /* 3473 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3474 * 3475 * Pages are normally inserted at the start of a vnode's v_pages list. 3476 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3477 * This can happen when a modified page is relocated for DR. 3478 * 3479 * Returns 1 on success and 0 on failure. 3480 */ 3481 static int 3482 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3483 { 3484 page_t **listp; 3485 page_t *tp; 3486 ulong_t index; 3487 3488 ASSERT(PAGE_EXCL(pp)); 3489 ASSERT(vp != NULL); 3490 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3491 3492 /* 3493 * Be sure to set these up before the page is inserted on the hash 3494 * list. As soon as the page is placed on the list some other 3495 * thread might get confused and wonder how this page could 3496 * possibly hash to this list. 3497 */ 3498 pp->p_vnode = vp; 3499 pp->p_offset = offset; 3500 3501 /* 3502 * record if this page is on a swap vnode 3503 */ 3504 if ((vp->v_flag & VISSWAP) != 0) 3505 PP_SETSWAP(pp); 3506 3507 index = PAGE_HASH_FUNC(vp, offset); 3508 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3509 listp = &page_hash[index]; 3510 3511 /* 3512 * If this page is already hashed in, fail this attempt to add it. 3513 */ 3514 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3515 if (tp->p_vnode == vp && tp->p_offset == offset) { 3516 pp->p_vnode = NULL; 3517 pp->p_offset = (u_offset_t)(-1); 3518 return (0); 3519 } 3520 } 3521 pp->p_hash = *listp; 3522 *listp = pp; 3523 3524 /* 3525 * Add the page to the vnode's list of pages 3526 */ 3527 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3528 listp = &vp->v_pages->p_vpprev->p_vpnext; 3529 else 3530 listp = &vp->v_pages; 3531 3532 page_vpadd(listp, pp); 3533 3534 return (1); 3535 } 3536 3537 /* 3538 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3539 * 3540 * Returns 1 on success and 0 on failure. 3541 * If hold is passed in, it is not dropped. 3542 */ 3543 int 3544 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3545 { 3546 kmutex_t *phm = NULL; 3547 kmutex_t *vphm; 3548 int rc; 3549 3550 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3551 3552 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3553 "page_hashin:pp %p vp %p offset %llx", 3554 pp, vp, offset); 3555 3556 VM_STAT_ADD(hashin_count); 3557 3558 if (hold != NULL) 3559 phm = hold; 3560 else { 3561 VM_STAT_ADD(hashin_not_held); 3562 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3563 mutex_enter(phm); 3564 } 3565 3566 vphm = page_vnode_mutex(vp); 3567 mutex_enter(vphm); 3568 rc = page_do_hashin(pp, vp, offset); 3569 mutex_exit(vphm); 3570 if (hold == NULL) 3571 mutex_exit(phm); 3572 if (rc == 0) 3573 VM_STAT_ADD(hashin_already); 3574 return (rc); 3575 } 3576 3577 /* 3578 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3579 * All mutexes must be held 3580 */ 3581 static void 3582 page_do_hashout(page_t *pp) 3583 { 3584 page_t **hpp; 3585 page_t *hp; 3586 vnode_t *vp = pp->p_vnode; 3587 3588 ASSERT(vp != NULL); 3589 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3590 3591 /* 3592 * First, take pp off of its hash chain. 3593 */ 3594 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3595 3596 for (;;) { 3597 hp = *hpp; 3598 if (hp == pp) 3599 break; 3600 if (hp == NULL) { 3601 panic("page_do_hashout"); 3602 /*NOTREACHED*/ 3603 } 3604 hpp = &hp->p_hash; 3605 } 3606 *hpp = pp->p_hash; 3607 3608 /* 3609 * Now remove it from its associated vnode. 3610 */ 3611 if (vp->v_pages) 3612 page_vpsub(&vp->v_pages, pp); 3613 3614 pp->p_hash = NULL; 3615 page_clr_all_props(pp); 3616 PP_CLRSWAP(pp); 3617 pp->p_vnode = NULL; 3618 pp->p_offset = (u_offset_t)-1; 3619 } 3620 3621 /* 3622 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3623 * 3624 * When `phm' is non-NULL it contains the address of the mutex protecting the 3625 * hash list pp is on. It is not dropped. 3626 */ 3627 void 3628 page_hashout(page_t *pp, kmutex_t *phm) 3629 { 3630 vnode_t *vp; 3631 ulong_t index; 3632 kmutex_t *nphm; 3633 kmutex_t *vphm; 3634 kmutex_t *sep; 3635 3636 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3637 ASSERT(pp->p_vnode != NULL); 3638 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3639 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3640 3641 vp = pp->p_vnode; 3642 3643 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3644 "page_hashout:pp %p vp %p", pp, vp); 3645 3646 /* Kernel probe */ 3647 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3648 tnf_opaque, vnode, vp, 3649 tnf_offset, offset, pp->p_offset); 3650 3651 /* 3652 * 3653 */ 3654 VM_STAT_ADD(hashout_count); 3655 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3656 if (phm == NULL) { 3657 VM_STAT_ADD(hashout_not_held); 3658 nphm = PAGE_HASH_MUTEX(index); 3659 mutex_enter(nphm); 3660 } 3661 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3662 3663 3664 /* 3665 * grab page vnode mutex and remove it... 3666 */ 3667 vphm = page_vnode_mutex(vp); 3668 mutex_enter(vphm); 3669 3670 page_do_hashout(pp); 3671 3672 mutex_exit(vphm); 3673 if (phm == NULL) 3674 mutex_exit(nphm); 3675 3676 /* 3677 * Wake up processes waiting for this page. The page's 3678 * identity has been changed, and is probably not the 3679 * desired page any longer. 3680 */ 3681 sep = page_se_mutex(pp); 3682 mutex_enter(sep); 3683 pp->p_selock &= ~SE_EWANTED; 3684 if (CV_HAS_WAITERS(&pp->p_cv)) 3685 cv_broadcast(&pp->p_cv); 3686 mutex_exit(sep); 3687 } 3688 3689 /* 3690 * Add the page to the front of a linked list of pages 3691 * using the p_next & p_prev pointers for the list. 3692 * The caller is responsible for protecting the list pointers. 3693 */ 3694 void 3695 page_add(page_t **ppp, page_t *pp) 3696 { 3697 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3698 3699 page_add_common(ppp, pp); 3700 } 3701 3702 3703 3704 /* 3705 * Common code for page_add() and mach_page_add() 3706 */ 3707 void 3708 page_add_common(page_t **ppp, page_t *pp) 3709 { 3710 if (*ppp == NULL) { 3711 pp->p_next = pp->p_prev = pp; 3712 } else { 3713 pp->p_next = *ppp; 3714 pp->p_prev = (*ppp)->p_prev; 3715 (*ppp)->p_prev = pp; 3716 pp->p_prev->p_next = pp; 3717 } 3718 *ppp = pp; 3719 } 3720 3721 3722 /* 3723 * Remove this page from a linked list of pages 3724 * using the p_next & p_prev pointers for the list. 3725 * 3726 * The caller is responsible for protecting the list pointers. 3727 */ 3728 void 3729 page_sub(page_t **ppp, page_t *pp) 3730 { 3731 ASSERT((PP_ISFREE(pp)) ? 1 : 3732 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3733 3734 if (*ppp == NULL || pp == NULL) { 3735 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3736 (void *)pp, (void *)(*ppp)); 3737 /*NOTREACHED*/ 3738 } 3739 3740 page_sub_common(ppp, pp); 3741 } 3742 3743 3744 /* 3745 * Common code for page_sub() and mach_page_sub() 3746 */ 3747 void 3748 page_sub_common(page_t **ppp, page_t *pp) 3749 { 3750 if (*ppp == pp) 3751 *ppp = pp->p_next; /* go to next page */ 3752 3753 if (*ppp == pp) 3754 *ppp = NULL; /* page list is gone */ 3755 else { 3756 pp->p_prev->p_next = pp->p_next; 3757 pp->p_next->p_prev = pp->p_prev; 3758 } 3759 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3760 } 3761 3762 3763 /* 3764 * Break page list cppp into two lists with npages in the first list. 3765 * The tail is returned in nppp. 3766 */ 3767 void 3768 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3769 { 3770 page_t *s1pp = *oppp; 3771 page_t *s2pp; 3772 page_t *e1pp, *e2pp; 3773 long n = 0; 3774 3775 if (s1pp == NULL) { 3776 *nppp = NULL; 3777 return; 3778 } 3779 if (npages == 0) { 3780 *nppp = s1pp; 3781 *oppp = NULL; 3782 return; 3783 } 3784 for (n = 0, s2pp = *oppp; n < npages; n++) { 3785 s2pp = s2pp->p_next; 3786 } 3787 /* Fix head and tail of new lists */ 3788 e1pp = s2pp->p_prev; 3789 e2pp = s1pp->p_prev; 3790 s1pp->p_prev = e1pp; 3791 e1pp->p_next = s1pp; 3792 s2pp->p_prev = e2pp; 3793 e2pp->p_next = s2pp; 3794 3795 /* second list empty */ 3796 if (s2pp == s1pp) { 3797 *oppp = s1pp; 3798 *nppp = NULL; 3799 } else { 3800 *oppp = s1pp; 3801 *nppp = s2pp; 3802 } 3803 } 3804 3805 /* 3806 * Concatenate page list nppp onto the end of list ppp. 3807 */ 3808 void 3809 page_list_concat(page_t **ppp, page_t **nppp) 3810 { 3811 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3812 3813 if (*nppp == NULL) { 3814 return; 3815 } 3816 if (*ppp == NULL) { 3817 *ppp = *nppp; 3818 return; 3819 } 3820 s1pp = *ppp; 3821 e1pp = s1pp->p_prev; 3822 s2pp = *nppp; 3823 e2pp = s2pp->p_prev; 3824 s1pp->p_prev = e2pp; 3825 e2pp->p_next = s1pp; 3826 e1pp->p_next = s2pp; 3827 s2pp->p_prev = e1pp; 3828 } 3829 3830 /* 3831 * return the next page in the page list 3832 */ 3833 page_t * 3834 page_list_next(page_t *pp) 3835 { 3836 return (pp->p_next); 3837 } 3838 3839 3840 /* 3841 * Add the page to the front of the linked list of pages 3842 * using p_vpnext/p_vpprev pointers for the list. 3843 * 3844 * The caller is responsible for protecting the lists. 3845 */ 3846 void 3847 page_vpadd(page_t **ppp, page_t *pp) 3848 { 3849 if (*ppp == NULL) { 3850 pp->p_vpnext = pp->p_vpprev = pp; 3851 } else { 3852 pp->p_vpnext = *ppp; 3853 pp->p_vpprev = (*ppp)->p_vpprev; 3854 (*ppp)->p_vpprev = pp; 3855 pp->p_vpprev->p_vpnext = pp; 3856 } 3857 *ppp = pp; 3858 } 3859 3860 /* 3861 * Remove this page from the linked list of pages 3862 * using p_vpnext/p_vpprev pointers for the list. 3863 * 3864 * The caller is responsible for protecting the lists. 3865 */ 3866 void 3867 page_vpsub(page_t **ppp, page_t *pp) 3868 { 3869 if (*ppp == NULL || pp == NULL) { 3870 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3871 (void *)pp, (void *)(*ppp)); 3872 /*NOTREACHED*/ 3873 } 3874 3875 if (*ppp == pp) 3876 *ppp = pp->p_vpnext; /* go to next page */ 3877 3878 if (*ppp == pp) 3879 *ppp = NULL; /* page list is gone */ 3880 else { 3881 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3882 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3883 } 3884 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3885 } 3886 3887 /* 3888 * Lock a physical page into memory "long term". Used to support "lock 3889 * in memory" functions. Accepts the page to be locked, and a cow variable 3890 * to indicate whether a the lock will travel to the new page during 3891 * a potential copy-on-write. 3892 */ 3893 int 3894 page_pp_lock( 3895 page_t *pp, /* page to be locked */ 3896 int cow, /* cow lock */ 3897 int kernel) /* must succeed -- ignore checking */ 3898 { 3899 int r = 0; /* result -- assume failure */ 3900 3901 ASSERT(PAGE_LOCKED(pp)); 3902 3903 page_struct_lock(pp); 3904 /* 3905 * Acquire the "freemem_lock" for availrmem. 3906 */ 3907 if (cow) { 3908 mutex_enter(&freemem_lock); 3909 if ((availrmem > pages_pp_maximum) && 3910 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3911 availrmem--; 3912 pages_locked++; 3913 mutex_exit(&freemem_lock); 3914 r = 1; 3915 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3916 cmn_err(CE_WARN, 3917 "COW lock limit reached on pfn 0x%lx", 3918 page_pptonum(pp)); 3919 } 3920 } else 3921 mutex_exit(&freemem_lock); 3922 } else { 3923 if (pp->p_lckcnt) { 3924 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3925 r = 1; 3926 if (++pp->p_lckcnt == 3927 (ushort_t)PAGE_LOCK_MAXIMUM) { 3928 cmn_err(CE_WARN, "Page lock limit " 3929 "reached on pfn 0x%lx", 3930 page_pptonum(pp)); 3931 } 3932 } 3933 } else { 3934 if (kernel) { 3935 /* availrmem accounting done by caller */ 3936 ++pp->p_lckcnt; 3937 r = 1; 3938 } else { 3939 mutex_enter(&freemem_lock); 3940 if (availrmem > pages_pp_maximum) { 3941 availrmem--; 3942 pages_locked++; 3943 ++pp->p_lckcnt; 3944 r = 1; 3945 } 3946 mutex_exit(&freemem_lock); 3947 } 3948 } 3949 } 3950 page_struct_unlock(pp); 3951 return (r); 3952 } 3953 3954 /* 3955 * Decommit a lock on a physical page frame. Account for cow locks if 3956 * appropriate. 3957 */ 3958 void 3959 page_pp_unlock( 3960 page_t *pp, /* page to be unlocked */ 3961 int cow, /* expect cow lock */ 3962 int kernel) /* this was a kernel lock */ 3963 { 3964 ASSERT(PAGE_LOCKED(pp)); 3965 3966 page_struct_lock(pp); 3967 /* 3968 * Acquire the "freemem_lock" for availrmem. 3969 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3970 * could be called to unlock even if nothing is locked. This could 3971 * happen if locked file pages were truncated (removing the lock) 3972 * and the file was grown again and new pages faulted in; the new 3973 * pages are unlocked but the segment still thinks they're locked. 3974 */ 3975 if (cow) { 3976 if (pp->p_cowcnt) { 3977 mutex_enter(&freemem_lock); 3978 pp->p_cowcnt--; 3979 availrmem++; 3980 pages_locked--; 3981 mutex_exit(&freemem_lock); 3982 } 3983 } else { 3984 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3985 if (!kernel) { 3986 mutex_enter(&freemem_lock); 3987 availrmem++; 3988 pages_locked--; 3989 mutex_exit(&freemem_lock); 3990 } 3991 } 3992 } 3993 page_struct_unlock(pp); 3994 } 3995 3996 /* 3997 * This routine reserves availrmem for npages; 3998 * flags: KM_NOSLEEP or KM_SLEEP 3999 * returns 1 on success or 0 on failure 4000 */ 4001 int 4002 page_resv(pgcnt_t npages, uint_t flags) 4003 { 4004 mutex_enter(&freemem_lock); 4005 while (availrmem < tune.t_minarmem + npages) { 4006 if (flags & KM_NOSLEEP) { 4007 mutex_exit(&freemem_lock); 4008 return (0); 4009 } 4010 mutex_exit(&freemem_lock); 4011 page_needfree(npages); 4012 kmem_reap(); 4013 delay(hz >> 2); 4014 page_needfree(-(spgcnt_t)npages); 4015 mutex_enter(&freemem_lock); 4016 } 4017 availrmem -= npages; 4018 mutex_exit(&freemem_lock); 4019 return (1); 4020 } 4021 4022 /* 4023 * This routine unreserves availrmem for npages; 4024 */ 4025 void 4026 page_unresv(pgcnt_t npages) 4027 { 4028 mutex_enter(&freemem_lock); 4029 availrmem += npages; 4030 mutex_exit(&freemem_lock); 4031 } 4032 4033 /* 4034 * See Statement at the beginning of segvn_lockop() regarding 4035 * the way we handle cowcnts and lckcnts. 4036 * 4037 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 4038 * that breaks COW has PROT_WRITE. 4039 * 4040 * Note that, we may also break COW in case we are softlocking 4041 * on read access during physio; 4042 * in this softlock case, the vpage may not have PROT_WRITE. 4043 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 4044 * if the vpage doesn't have PROT_WRITE. 4045 * 4046 * This routine is never called if we are stealing a page 4047 * in anon_private. 4048 * 4049 * The caller subtracted from availrmem for read only mapping. 4050 * if lckcnt is 1 increment availrmem. 4051 */ 4052 void 4053 page_pp_useclaim( 4054 page_t *opp, /* original page frame losing lock */ 4055 page_t *npp, /* new page frame gaining lock */ 4056 uint_t write_perm) /* set if vpage has PROT_WRITE */ 4057 { 4058 int payback = 0; 4059 4060 ASSERT(PAGE_LOCKED(opp)); 4061 ASSERT(PAGE_LOCKED(npp)); 4062 4063 page_struct_lock(opp); 4064 4065 ASSERT(npp->p_cowcnt == 0); 4066 ASSERT(npp->p_lckcnt == 0); 4067 4068 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4069 if ((write_perm && opp->p_cowcnt != 0) || 4070 (!write_perm && opp->p_lckcnt != 0)) { 4071 4072 if (write_perm) { 4073 npp->p_cowcnt++; 4074 ASSERT(opp->p_cowcnt != 0); 4075 opp->p_cowcnt--; 4076 } else { 4077 4078 ASSERT(opp->p_lckcnt != 0); 4079 4080 /* 4081 * We didn't need availrmem decremented if p_lckcnt on 4082 * original page is 1. Here, we are unlocking 4083 * read-only copy belonging to original page and 4084 * are locking a copy belonging to new page. 4085 */ 4086 if (opp->p_lckcnt == 1) 4087 payback = 1; 4088 4089 npp->p_lckcnt++; 4090 opp->p_lckcnt--; 4091 } 4092 } 4093 if (payback) { 4094 mutex_enter(&freemem_lock); 4095 availrmem++; 4096 pages_useclaim--; 4097 mutex_exit(&freemem_lock); 4098 } 4099 page_struct_unlock(opp); 4100 } 4101 4102 /* 4103 * Simple claim adjust functions -- used to support changes in 4104 * claims due to changes in access permissions. Used by segvn_setprot(). 4105 */ 4106 int 4107 page_addclaim(page_t *pp) 4108 { 4109 int r = 0; /* result */ 4110 4111 ASSERT(PAGE_LOCKED(pp)); 4112 4113 page_struct_lock(pp); 4114 ASSERT(pp->p_lckcnt != 0); 4115 4116 if (pp->p_lckcnt == 1) { 4117 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4118 --pp->p_lckcnt; 4119 r = 1; 4120 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4121 cmn_err(CE_WARN, 4122 "COW lock limit reached on pfn 0x%lx", 4123 page_pptonum(pp)); 4124 } 4125 } 4126 } else { 4127 mutex_enter(&freemem_lock); 4128 if ((availrmem > pages_pp_maximum) && 4129 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4130 --availrmem; 4131 ++pages_claimed; 4132 mutex_exit(&freemem_lock); 4133 --pp->p_lckcnt; 4134 r = 1; 4135 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4136 cmn_err(CE_WARN, 4137 "COW lock limit reached on pfn 0x%lx", 4138 page_pptonum(pp)); 4139 } 4140 } else 4141 mutex_exit(&freemem_lock); 4142 } 4143 page_struct_unlock(pp); 4144 return (r); 4145 } 4146 4147 int 4148 page_subclaim(page_t *pp) 4149 { 4150 int r = 0; 4151 4152 ASSERT(PAGE_LOCKED(pp)); 4153 4154 page_struct_lock(pp); 4155 ASSERT(pp->p_cowcnt != 0); 4156 4157 if (pp->p_lckcnt) { 4158 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4159 r = 1; 4160 /* 4161 * for availrmem 4162 */ 4163 mutex_enter(&freemem_lock); 4164 availrmem++; 4165 pages_claimed--; 4166 mutex_exit(&freemem_lock); 4167 4168 pp->p_cowcnt--; 4169 4170 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4171 cmn_err(CE_WARN, 4172 "Page lock limit reached on pfn 0x%lx", 4173 page_pptonum(pp)); 4174 } 4175 } 4176 } else { 4177 r = 1; 4178 pp->p_cowcnt--; 4179 pp->p_lckcnt++; 4180 } 4181 page_struct_unlock(pp); 4182 return (r); 4183 } 4184 4185 int 4186 page_addclaim_pages(page_t **ppa) 4187 { 4188 4189 pgcnt_t lckpgs = 0, pg_idx; 4190 4191 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4192 4193 mutex_enter(&page_llock); 4194 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4195 4196 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4197 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4198 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4199 mutex_exit(&page_llock); 4200 return (0); 4201 } 4202 if (ppa[pg_idx]->p_lckcnt > 1) 4203 lckpgs++; 4204 } 4205 4206 if (lckpgs != 0) { 4207 mutex_enter(&freemem_lock); 4208 if (availrmem >= pages_pp_maximum + lckpgs) { 4209 availrmem -= lckpgs; 4210 pages_claimed += lckpgs; 4211 } else { 4212 mutex_exit(&freemem_lock); 4213 mutex_exit(&page_llock); 4214 return (0); 4215 } 4216 mutex_exit(&freemem_lock); 4217 } 4218 4219 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4220 ppa[pg_idx]->p_lckcnt--; 4221 ppa[pg_idx]->p_cowcnt++; 4222 } 4223 mutex_exit(&page_llock); 4224 return (1); 4225 } 4226 4227 int 4228 page_subclaim_pages(page_t **ppa) 4229 { 4230 pgcnt_t ulckpgs = 0, pg_idx; 4231 4232 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4233 4234 mutex_enter(&page_llock); 4235 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4236 4237 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4238 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4239 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4240 mutex_exit(&page_llock); 4241 return (0); 4242 } 4243 if (ppa[pg_idx]->p_lckcnt != 0) 4244 ulckpgs++; 4245 } 4246 4247 if (ulckpgs != 0) { 4248 mutex_enter(&freemem_lock); 4249 availrmem += ulckpgs; 4250 pages_claimed -= ulckpgs; 4251 mutex_exit(&freemem_lock); 4252 } 4253 4254 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4255 ppa[pg_idx]->p_cowcnt--; 4256 ppa[pg_idx]->p_lckcnt++; 4257 4258 } 4259 mutex_exit(&page_llock); 4260 return (1); 4261 } 4262 4263 page_t * 4264 page_numtopp(pfn_t pfnum, se_t se) 4265 { 4266 page_t *pp; 4267 4268 retry: 4269 pp = page_numtopp_nolock(pfnum); 4270 if (pp == NULL) { 4271 return ((page_t *)NULL); 4272 } 4273 4274 /* 4275 * Acquire the appropriate lock on the page. 4276 */ 4277 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4278 if (page_pptonum(pp) != pfnum) 4279 goto retry; 4280 continue; 4281 } 4282 4283 if (page_pptonum(pp) != pfnum) { 4284 page_unlock(pp); 4285 goto retry; 4286 } 4287 4288 return (pp); 4289 } 4290 4291 page_t * 4292 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4293 { 4294 page_t *pp; 4295 4296 retry: 4297 pp = page_numtopp_nolock(pfnum); 4298 if (pp == NULL) { 4299 return ((page_t *)NULL); 4300 } 4301 4302 /* 4303 * Acquire the appropriate lock on the page. 4304 */ 4305 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4306 if (page_pptonum(pp) != pfnum) 4307 goto retry; 4308 continue; 4309 } 4310 4311 if (page_pptonum(pp) != pfnum) { 4312 page_unlock(pp); 4313 goto retry; 4314 } 4315 4316 return (pp); 4317 } 4318 4319 /* 4320 * This routine is like page_numtopp, but will only return page structs 4321 * for pages which are ok for loading into hardware using the page struct. 4322 */ 4323 page_t * 4324 page_numtopp_nowait(pfn_t pfnum, se_t se) 4325 { 4326 page_t *pp; 4327 4328 retry: 4329 pp = page_numtopp_nolock(pfnum); 4330 if (pp == NULL) { 4331 return ((page_t *)NULL); 4332 } 4333 4334 /* 4335 * Try to acquire the appropriate lock on the page. 4336 */ 4337 if (PP_ISFREE(pp)) 4338 pp = NULL; 4339 else { 4340 if (!page_trylock(pp, se)) 4341 pp = NULL; 4342 else { 4343 if (page_pptonum(pp) != pfnum) { 4344 page_unlock(pp); 4345 goto retry; 4346 } 4347 if (PP_ISFREE(pp)) { 4348 page_unlock(pp); 4349 pp = NULL; 4350 } 4351 } 4352 } 4353 return (pp); 4354 } 4355 4356 /* 4357 * Returns a count of dirty pages that are in the process 4358 * of being written out. If 'cleanit' is set, try to push the page. 4359 */ 4360 pgcnt_t 4361 page_busy(int cleanit) 4362 { 4363 page_t *page0 = page_first(); 4364 page_t *pp = page0; 4365 pgcnt_t nppbusy = 0; 4366 u_offset_t off; 4367 4368 do { 4369 vnode_t *vp = pp->p_vnode; 4370 4371 /* 4372 * A page is a candidate for syncing if it is: 4373 * 4374 * (a) On neither the freelist nor the cachelist 4375 * (b) Hashed onto a vnode 4376 * (c) Not a kernel page 4377 * (d) Dirty 4378 * (e) Not part of a swapfile 4379 * (f) a page which belongs to a real vnode; eg has a non-null 4380 * v_vfsp pointer. 4381 * (g) Backed by a filesystem which doesn't have a 4382 * stubbed-out sync operation 4383 */ 4384 if (!PP_ISFREE(pp) && vp != NULL && vp != &kvp && 4385 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4386 vfs_can_sync(vp->v_vfsp)) { 4387 nppbusy++; 4388 vfs_syncprogress(); 4389 4390 if (!cleanit) 4391 continue; 4392 if (!page_trylock(pp, SE_EXCL)) 4393 continue; 4394 4395 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4396 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4397 !(hat_pagesync(pp, 4398 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4399 page_unlock(pp); 4400 continue; 4401 } 4402 off = pp->p_offset; 4403 VN_HOLD(vp); 4404 page_unlock(pp); 4405 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4406 B_ASYNC | B_FREE, kcred); 4407 VN_RELE(vp); 4408 } 4409 } while ((pp = page_next(pp)) != page0); 4410 4411 return (nppbusy); 4412 } 4413 4414 void page_invalidate_pages(void); 4415 4416 /* 4417 * callback handler to vm sub-system 4418 * 4419 * callers make sure no recursive entries to this func. 4420 */ 4421 /*ARGSUSED*/ 4422 boolean_t 4423 callb_vm_cpr(void *arg, int code) 4424 { 4425 if (code == CB_CODE_CPR_CHKPT) 4426 page_invalidate_pages(); 4427 return (B_TRUE); 4428 } 4429 4430 /* 4431 * Invalidate all pages of the system. 4432 * It shouldn't be called until all user page activities are all stopped. 4433 */ 4434 void 4435 page_invalidate_pages() 4436 { 4437 page_t *pp; 4438 page_t *page0; 4439 pgcnt_t nbusypages; 4440 int retry = 0; 4441 const int MAXRETRIES = 4; 4442 #if defined(__sparc) 4443 extern struct vnode prom_ppages; 4444 #endif /* __sparc */ 4445 4446 top: 4447 /* 4448 * Flush dirty pages and destory the clean ones. 4449 */ 4450 nbusypages = 0; 4451 4452 pp = page0 = page_first(); 4453 do { 4454 struct vnode *vp; 4455 u_offset_t offset; 4456 int mod; 4457 4458 /* 4459 * skip the page if it has no vnode or the page associated 4460 * with the kernel vnode or prom allocated kernel mem. 4461 */ 4462 #if defined(__sparc) 4463 if ((vp = pp->p_vnode) == NULL || vp == &kvp || 4464 vp == &prom_ppages) 4465 #else /* x86 doesn't have prom or prom_ppage */ 4466 if ((vp = pp->p_vnode) == NULL || vp == &kvp) 4467 #endif /* __sparc */ 4468 continue; 4469 4470 /* 4471 * skip the page which is already free invalidated. 4472 */ 4473 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4474 continue; 4475 4476 /* 4477 * skip pages that are already locked or can't be "exclusively" 4478 * locked or are already free. After we lock the page, check 4479 * the free and age bits again to be sure it's not destroied 4480 * yet. 4481 * To achieve max. parallelization, we use page_trylock instead 4482 * of page_lock so that we don't get block on individual pages 4483 * while we have thousands of other pages to process. 4484 */ 4485 if (!page_trylock(pp, SE_EXCL)) { 4486 nbusypages++; 4487 continue; 4488 } else if (PP_ISFREE(pp)) { 4489 if (!PP_ISAGED(pp)) { 4490 page_destroy_free(pp); 4491 } else { 4492 page_unlock(pp); 4493 } 4494 continue; 4495 } 4496 /* 4497 * Is this page involved in some I/O? shared? 4498 * 4499 * The page_struct_lock need not be acquired to 4500 * examine these fields since the page has an 4501 * "exclusive" lock. 4502 */ 4503 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4504 page_unlock(pp); 4505 continue; 4506 } 4507 4508 if (vp->v_type == VCHR) { 4509 panic("vp->v_type == VCHR"); 4510 /*NOTREACHED*/ 4511 } 4512 4513 if (!page_try_demote_pages(pp)) { 4514 page_unlock(pp); 4515 continue; 4516 } 4517 4518 /* 4519 * Check the modified bit. Leave the bits alone in hardware 4520 * (they will be modified if we do the putpage). 4521 */ 4522 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4523 & P_MOD); 4524 if (mod) { 4525 offset = pp->p_offset; 4526 /* 4527 * Hold the vnode before releasing the page lock 4528 * to prevent it from being freed and re-used by 4529 * some other thread. 4530 */ 4531 VN_HOLD(vp); 4532 page_unlock(pp); 4533 /* 4534 * No error return is checked here. Callers such as 4535 * cpr deals with the dirty pages at the dump time 4536 * if this putpage fails. 4537 */ 4538 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4539 kcred); 4540 VN_RELE(vp); 4541 } else { 4542 page_destroy(pp, 0); 4543 } 4544 } while ((pp = page_next(pp)) != page0); 4545 if (nbusypages && retry++ < MAXRETRIES) { 4546 delay(1); 4547 goto top; 4548 } 4549 } 4550 4551 /* 4552 * Replace the page "old" with the page "new" on the page hash and vnode lists 4553 * 4554 * the replacemnt must be done in place, ie the equivalent sequence: 4555 * 4556 * vp = old->p_vnode; 4557 * off = old->p_offset; 4558 * page_do_hashout(old) 4559 * page_do_hashin(new, vp, off) 4560 * 4561 * doesn't work, since 4562 * 1) if old is the only page on the vnode, the v_pages list has a window 4563 * where it looks empty. This will break file system assumptions. 4564 * and 4565 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4566 */ 4567 static void 4568 page_do_relocate_hash(page_t *new, page_t *old) 4569 { 4570 page_t **hash_list; 4571 vnode_t *vp = old->p_vnode; 4572 kmutex_t *sep; 4573 4574 ASSERT(PAGE_EXCL(old)); 4575 ASSERT(PAGE_EXCL(new)); 4576 ASSERT(vp != NULL); 4577 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4578 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4579 4580 /* 4581 * First find old page on the page hash list 4582 */ 4583 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4584 4585 for (;;) { 4586 if (*hash_list == old) 4587 break; 4588 if (*hash_list == NULL) { 4589 panic("page_do_hashout"); 4590 /*NOTREACHED*/ 4591 } 4592 hash_list = &(*hash_list)->p_hash; 4593 } 4594 4595 /* 4596 * update new and replace old with new on the page hash list 4597 */ 4598 new->p_vnode = old->p_vnode; 4599 new->p_offset = old->p_offset; 4600 new->p_hash = old->p_hash; 4601 *hash_list = new; 4602 4603 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4604 PP_SETSWAP(new); 4605 4606 /* 4607 * replace old with new on the vnode's page list 4608 */ 4609 if (old->p_vpnext == old) { 4610 new->p_vpnext = new; 4611 new->p_vpprev = new; 4612 } else { 4613 new->p_vpnext = old->p_vpnext; 4614 new->p_vpprev = old->p_vpprev; 4615 new->p_vpnext->p_vpprev = new; 4616 new->p_vpprev->p_vpnext = new; 4617 } 4618 if (vp->v_pages == old) 4619 vp->v_pages = new; 4620 4621 /* 4622 * clear out the old page 4623 */ 4624 old->p_hash = NULL; 4625 old->p_vpnext = NULL; 4626 old->p_vpprev = NULL; 4627 old->p_vnode = NULL; 4628 PP_CLRSWAP(old); 4629 old->p_offset = (u_offset_t)-1; 4630 page_clr_all_props(old); 4631 4632 /* 4633 * Wake up processes waiting for this page. The page's 4634 * identity has been changed, and is probably not the 4635 * desired page any longer. 4636 */ 4637 sep = page_se_mutex(old); 4638 mutex_enter(sep); 4639 old->p_selock &= ~SE_EWANTED; 4640 if (CV_HAS_WAITERS(&old->p_cv)) 4641 cv_broadcast(&old->p_cv); 4642 mutex_exit(sep); 4643 } 4644 4645 /* 4646 * This function moves the identity of page "pp_old" to page "pp_new". 4647 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4648 * and need not be hashed out from anywhere. 4649 */ 4650 void 4651 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4652 { 4653 vnode_t *vp = pp_old->p_vnode; 4654 u_offset_t off = pp_old->p_offset; 4655 kmutex_t *phm, *vphm; 4656 4657 /* 4658 * Rehash two pages 4659 */ 4660 ASSERT(PAGE_EXCL(pp_old)); 4661 ASSERT(PAGE_EXCL(pp_new)); 4662 ASSERT(vp != NULL); 4663 ASSERT(pp_new->p_vnode == NULL); 4664 4665 /* 4666 * hashout then hashin while holding the mutexes 4667 */ 4668 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4669 mutex_enter(phm); 4670 vphm = page_vnode_mutex(vp); 4671 mutex_enter(vphm); 4672 4673 page_do_relocate_hash(pp_new, pp_old); 4674 4675 mutex_exit(vphm); 4676 mutex_exit(phm); 4677 4678 /* 4679 * The page_struct_lock need not be acquired for lckcnt and 4680 * cowcnt since the page has an "exclusive" lock. 4681 */ 4682 ASSERT(pp_new->p_lckcnt == 0); 4683 ASSERT(pp_new->p_cowcnt == 0); 4684 pp_new->p_lckcnt = pp_old->p_lckcnt; 4685 pp_new->p_cowcnt = pp_old->p_cowcnt; 4686 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4687 4688 /* The following comment preserved from page_flip(). */ 4689 /* XXX - Do we need to protect fsdata? */ 4690 pp_new->p_fsdata = pp_old->p_fsdata; 4691 } 4692 4693 /* 4694 * Helper routine used to lock all remaining members of a 4695 * large page. The caller is responsible for passing in a locked 4696 * pp. If pp is a large page, then it succeeds in locking all the 4697 * remaining constituent pages or it returns with only the 4698 * original page locked. 4699 * 4700 * Returns 1 on success, 0 on failure. 4701 * 4702 * If success is returned this routine gurantees p_szc for all constituent 4703 * pages of a large page pp belongs to can't change. To achieve this we 4704 * recheck szc of pp after locking all constituent pages and retry if szc 4705 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4706 * lock on one of constituent pages it can't be running after all constituent 4707 * pages are locked. hat_page_demote() with a lock on a constituent page 4708 * outside of this large page (i.e. pp belonged to a larger large page) is 4709 * already done with all constituent pages of pp since the root's p_szc is 4710 * changed last. Thefore no need to synchronize with hat_page_demote() that 4711 * locked a constituent page outside of pp's current large page. 4712 */ 4713 #ifdef DEBUG 4714 uint32_t gpg_trylock_mtbf = 0; 4715 #endif 4716 4717 int 4718 group_page_trylock(page_t *pp, se_t se) 4719 { 4720 page_t *tpp; 4721 pgcnt_t npgs, i, j; 4722 uint_t pszc = pp->p_szc; 4723 4724 #ifdef DEBUG 4725 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4726 return (0); 4727 } 4728 #endif 4729 4730 if (pp != PP_GROUPLEADER(pp, pszc)) { 4731 return (0); 4732 } 4733 4734 retry: 4735 ASSERT(PAGE_LOCKED_SE(pp, se)); 4736 ASSERT(!PP_ISFREE(pp)); 4737 if (pszc == 0) { 4738 return (1); 4739 } 4740 npgs = page_get_pagecnt(pszc); 4741 tpp = pp + 1; 4742 for (i = 1; i < npgs; i++, tpp++) { 4743 if (!page_trylock(tpp, se)) { 4744 tpp = pp + 1; 4745 for (j = 1; j < i; j++, tpp++) { 4746 page_unlock(tpp); 4747 } 4748 return (0); 4749 } 4750 } 4751 if (pp->p_szc != pszc) { 4752 ASSERT(pp->p_szc < pszc); 4753 ASSERT(pp->p_vnode != NULL && pp->p_vnode != &kvp && 4754 !IS_SWAPFSVP(pp->p_vnode)); 4755 tpp = pp + 1; 4756 for (i = 1; i < npgs; i++, tpp++) { 4757 page_unlock(tpp); 4758 } 4759 pszc = pp->p_szc; 4760 goto retry; 4761 } 4762 return (1); 4763 } 4764 4765 void 4766 group_page_unlock(page_t *pp) 4767 { 4768 page_t *tpp; 4769 pgcnt_t npgs, i; 4770 4771 ASSERT(PAGE_LOCKED(pp)); 4772 ASSERT(!PP_ISFREE(pp)); 4773 ASSERT(pp == PP_PAGEROOT(pp)); 4774 npgs = page_get_pagecnt(pp->p_szc); 4775 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4776 page_unlock(tpp); 4777 } 4778 } 4779 4780 /* 4781 * returns 4782 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4783 * ERANGE : this is not a base page 4784 * EBUSY : failure to get locks on the page/pages 4785 * ENOMEM : failure to obtain replacement pages 4786 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4787 * 4788 * Return with all constituent members of target and replacement 4789 * SE_EXCL locked. It is the callers responsibility to drop the 4790 * locks. 4791 */ 4792 int 4793 do_page_relocate( 4794 page_t **target, 4795 page_t **replacement, 4796 int grouplock, 4797 spgcnt_t *nrelocp, 4798 lgrp_t *lgrp) 4799 { 4800 #ifdef DEBUG 4801 page_t *first_repl; 4802 #endif /* DEBUG */ 4803 page_t *repl; 4804 page_t *targ; 4805 page_t *pl = NULL; 4806 uint_t ppattr; 4807 pfn_t pfn, repl_pfn; 4808 uint_t szc; 4809 spgcnt_t npgs, i; 4810 int repl_contig = 0; 4811 uint_t flags = 0; 4812 spgcnt_t dofree = 0; 4813 4814 *nrelocp = 0; 4815 4816 #if defined(__sparc) 4817 /* 4818 * We need to wait till OBP has completed 4819 * its boot-time handoff of its resources to the kernel 4820 * before we allow page relocation 4821 */ 4822 if (page_relocate_ready == 0) { 4823 return (EAGAIN); 4824 } 4825 #endif 4826 4827 /* 4828 * If this is not a base page, 4829 * just return with 0x0 pages relocated. 4830 */ 4831 targ = *target; 4832 ASSERT(PAGE_EXCL(targ)); 4833 ASSERT(!PP_ISFREE(targ)); 4834 szc = targ->p_szc; 4835 ASSERT(szc < mmu_page_sizes); 4836 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4837 pfn = targ->p_pagenum; 4838 if (pfn != PFN_BASE(pfn, szc)) { 4839 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4840 return (ERANGE); 4841 } 4842 4843 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4844 repl_pfn = repl->p_pagenum; 4845 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4846 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4847 return (ERANGE); 4848 } 4849 repl_contig = 1; 4850 } 4851 4852 /* 4853 * We must lock all members of this large page or we cannot 4854 * relocate any part of it. 4855 */ 4856 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4857 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4858 return (EBUSY); 4859 } 4860 4861 /* 4862 * reread szc it could have been decreased before 4863 * group_page_trylock() was done. 4864 */ 4865 szc = targ->p_szc; 4866 ASSERT(szc < mmu_page_sizes); 4867 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4868 ASSERT(pfn == PFN_BASE(pfn, szc)); 4869 4870 npgs = page_get_pagecnt(targ->p_szc); 4871 4872 if (repl == NULL) { 4873 dofree = npgs; /* Size of target page in MMU pages */ 4874 if (!page_create_wait(dofree, 0)) { 4875 if (grouplock != 0) { 4876 group_page_unlock(targ); 4877 } 4878 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4879 return (ENOMEM); 4880 } 4881 4882 /* 4883 * seg kmem pages require that the target and replacement 4884 * page be the same pagesize. 4885 */ 4886 flags = (targ->p_vnode == &kvp) ? PGR_SAMESZC : 0; 4887 repl = page_get_replacement_page(targ, lgrp, flags); 4888 if (repl == NULL) { 4889 if (grouplock != 0) { 4890 group_page_unlock(targ); 4891 } 4892 page_create_putback(dofree); 4893 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4894 return (ENOMEM); 4895 } 4896 } 4897 #ifdef DEBUG 4898 else { 4899 ASSERT(PAGE_LOCKED(repl)); 4900 } 4901 #endif /* DEBUG */ 4902 4903 #if defined(__sparc) 4904 /* 4905 * Let hat_page_relocate() complete the relocation if it's kernel page 4906 */ 4907 if (targ->p_vnode == &kvp) { 4908 *replacement = repl; 4909 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4910 if (grouplock != 0) { 4911 group_page_unlock(targ); 4912 } 4913 if (dofree) { 4914 *replacement = NULL; 4915 page_free_replacement_page(repl); 4916 page_create_putback(dofree); 4917 } 4918 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4919 return (EAGAIN); 4920 } 4921 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4922 return (0); 4923 } 4924 #else 4925 #if defined(lint) 4926 dofree = dofree; 4927 #endif 4928 #endif 4929 4930 #ifdef DEBUG 4931 first_repl = repl; 4932 #endif /* DEBUG */ 4933 4934 for (i = 0; i < npgs; i++) { 4935 ASSERT(PAGE_EXCL(targ)); 4936 4937 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4938 4939 ASSERT(hat_page_getshare(targ) == 0); 4940 ASSERT(!PP_ISFREE(targ)); 4941 ASSERT(targ->p_pagenum == (pfn + i)); 4942 ASSERT(repl_contig == 0 || 4943 repl->p_pagenum == (repl_pfn + i)); 4944 4945 /* 4946 * Copy the page contents and attributes then 4947 * relocate the page in the page hash. 4948 */ 4949 ppcopy(targ, repl); 4950 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4951 page_clr_all_props(repl); 4952 page_set_props(repl, ppattr); 4953 page_relocate_hash(repl, targ); 4954 4955 ASSERT(hat_page_getshare(targ) == 0); 4956 ASSERT(hat_page_getshare(repl) == 0); 4957 /* 4958 * Now clear the props on targ, after the 4959 * page_relocate_hash(), they no longer 4960 * have any meaning. 4961 */ 4962 page_clr_all_props(targ); 4963 ASSERT(targ->p_next == targ); 4964 ASSERT(targ->p_prev == targ); 4965 page_list_concat(&pl, &targ); 4966 4967 targ++; 4968 if (repl_contig != 0) { 4969 repl++; 4970 } else { 4971 repl = repl->p_next; 4972 } 4973 } 4974 /* assert that we have come full circle with repl */ 4975 ASSERT(repl_contig == 1 || first_repl == repl); 4976 4977 *target = pl; 4978 if (*replacement == NULL) { 4979 ASSERT(first_repl == repl); 4980 *replacement = repl; 4981 } 4982 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4983 *nrelocp = npgs; 4984 return (0); 4985 } 4986 /* 4987 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 4988 */ 4989 int 4990 page_relocate( 4991 page_t **target, 4992 page_t **replacement, 4993 int grouplock, 4994 int freetarget, 4995 spgcnt_t *nrelocp, 4996 lgrp_t *lgrp) 4997 { 4998 spgcnt_t ret; 4999 5000 /* do_page_relocate returns 0 on success or errno value */ 5001 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 5002 5003 if (ret != 0 || freetarget == 0) { 5004 return (ret); 5005 } 5006 if (*nrelocp == 1) { 5007 ASSERT(*target != NULL); 5008 page_free(*target, 1); 5009 } else { 5010 page_t *tpp = *target; 5011 uint_t szc = tpp->p_szc; 5012 pgcnt_t npgs = page_get_pagecnt(szc); 5013 ASSERT(npgs > 1); 5014 ASSERT(szc != 0); 5015 do { 5016 ASSERT(PAGE_EXCL(tpp)); 5017 ASSERT(!hat_page_is_mapped(tpp)); 5018 ASSERT(tpp->p_szc == szc); 5019 PP_SETFREE(tpp); 5020 PP_SETAGED(tpp); 5021 npgs--; 5022 } while ((tpp = tpp->p_next) != *target); 5023 ASSERT(npgs == 0); 5024 page_list_add_pages(*target, 0); 5025 npgs = page_get_pagecnt(szc); 5026 page_create_putback(npgs); 5027 } 5028 return (ret); 5029 } 5030 5031 /* 5032 * it is up to the caller to deal with pcf accounting. 5033 */ 5034 void 5035 page_free_replacement_page(page_t *pplist) 5036 { 5037 page_t *pp; 5038 5039 while (pplist != NULL) { 5040 /* 5041 * pp_targ is a linked list. 5042 */ 5043 pp = pplist; 5044 if (pp->p_szc == 0) { 5045 page_sub(&pplist, pp); 5046 page_clr_all_props(pp); 5047 PP_SETFREE(pp); 5048 PP_SETAGED(pp); 5049 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5050 page_unlock(pp); 5051 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5052 } else { 5053 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5054 page_t *tpp; 5055 page_list_break(&pp, &pplist, curnpgs); 5056 tpp = pp; 5057 do { 5058 ASSERT(PAGE_EXCL(tpp)); 5059 ASSERT(!hat_page_is_mapped(tpp)); 5060 page_clr_all_props(pp); 5061 PP_SETFREE(tpp); 5062 PP_SETAGED(tpp); 5063 } while ((tpp = tpp->p_next) != pp); 5064 page_list_add_pages(pp, 0); 5065 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5066 } 5067 } 5068 } 5069 5070 /* 5071 * Relocate target to non-relocatable replacement page. 5072 */ 5073 int 5074 page_relocate_cage(page_t **target, page_t **replacement) 5075 { 5076 page_t *tpp, *rpp; 5077 spgcnt_t pgcnt, npgs; 5078 int result; 5079 5080 tpp = *target; 5081 5082 ASSERT(PAGE_EXCL(tpp)); 5083 ASSERT(tpp->p_szc == 0); 5084 5085 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5086 5087 do { 5088 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5089 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5090 if (rpp == NULL) { 5091 page_create_putback(pgcnt); 5092 kcage_cageout_wakeup(); 5093 } 5094 } while (rpp == NULL); 5095 5096 ASSERT(PP_ISNORELOC(rpp)); 5097 5098 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5099 5100 if (result == 0) { 5101 *replacement = rpp; 5102 if (pgcnt != npgs) 5103 panic("page_relocate_cage: partial relocation"); 5104 } 5105 5106 return (result); 5107 } 5108 5109 /* 5110 * Release the page lock on a page, place on cachelist 5111 * tail if no longer mapped. Caller can let us know if 5112 * the page is known to be clean. 5113 */ 5114 int 5115 page_release(page_t *pp, int checkmod) 5116 { 5117 int status; 5118 5119 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5120 (pp->p_vnode != NULL)); 5121 5122 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5123 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5124 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5125 !hat_page_is_mapped(pp)) { 5126 5127 /* 5128 * If page is modified, unlock it 5129 * 5130 * (p_nrm & P_MOD) bit has the latest stuff because: 5131 * (1) We found that this page doesn't have any mappings 5132 * _after_ holding SE_EXCL and 5133 * (2) We didn't drop SE_EXCL lock after the check in (1) 5134 */ 5135 if (checkmod && hat_ismod(pp)) { 5136 page_unlock(pp); 5137 status = PGREL_MOD; 5138 } else { 5139 /*LINTED: constant in conditional context*/ 5140 VN_DISPOSE(pp, B_FREE, 0, kcred); 5141 status = PGREL_CLEAN; 5142 } 5143 } else { 5144 page_unlock(pp); 5145 status = PGREL_NOTREL; 5146 } 5147 return (status); 5148 } 5149 5150 /* 5151 * Given a constituent page, try to demote the large page on the freelist. 5152 * 5153 * Returns nonzero if the page could be demoted successfully. Returns with 5154 * the constituent page still locked. 5155 */ 5156 int 5157 page_try_demote_free_pages(page_t *pp) 5158 { 5159 page_t *rootpp = pp; 5160 pfn_t pfn = page_pptonum(pp); 5161 spgcnt_t npgs; 5162 uint_t szc = pp->p_szc; 5163 5164 ASSERT(PP_ISFREE(pp)); 5165 ASSERT(PAGE_EXCL(pp)); 5166 5167 /* 5168 * Adjust rootpp and lock it, if `pp' is not the base 5169 * constituent page. 5170 */ 5171 npgs = page_get_pagecnt(pp->p_szc); 5172 if (npgs == 1) { 5173 return (0); 5174 } 5175 5176 if (!IS_P2ALIGNED(pfn, npgs)) { 5177 pfn = P2ALIGN(pfn, npgs); 5178 rootpp = page_numtopp_nolock(pfn); 5179 } 5180 5181 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5182 return (0); 5183 } 5184 5185 if (rootpp->p_szc != szc) { 5186 if (pp != rootpp) 5187 page_unlock(rootpp); 5188 return (0); 5189 } 5190 5191 page_demote_free_pages(rootpp); 5192 5193 if (pp != rootpp) 5194 page_unlock(rootpp); 5195 5196 ASSERT(PP_ISFREE(pp)); 5197 ASSERT(PAGE_EXCL(pp)); 5198 return (1); 5199 } 5200 5201 /* 5202 * Given a constituent page, try to demote the large page. 5203 * 5204 * Returns nonzero if the page could be demoted successfully. Returns with 5205 * the constituent page still locked. 5206 */ 5207 int 5208 page_try_demote_pages(page_t *pp) 5209 { 5210 page_t *tpp, *rootpp = pp; 5211 pfn_t pfn = page_pptonum(pp); 5212 spgcnt_t i, npgs; 5213 uint_t szc = pp->p_szc; 5214 vnode_t *vp = pp->p_vnode; 5215 5216 ASSERT(PAGE_EXCL(pp)); 5217 5218 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5219 5220 if (pp->p_szc == 0) { 5221 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5222 return (1); 5223 } 5224 5225 if (vp != NULL && !IS_SWAPFSVP(vp) && vp != &kvp) { 5226 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5227 page_demote_vp_pages(pp); 5228 ASSERT(pp->p_szc == 0); 5229 return (1); 5230 } 5231 5232 /* 5233 * Adjust rootpp if passed in is not the base 5234 * constituent page. 5235 */ 5236 npgs = page_get_pagecnt(pp->p_szc); 5237 ASSERT(npgs > 1); 5238 if (!IS_P2ALIGNED(pfn, npgs)) { 5239 pfn = P2ALIGN(pfn, npgs); 5240 rootpp = page_numtopp_nolock(pfn); 5241 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5242 ASSERT(rootpp->p_vnode != NULL); 5243 ASSERT(rootpp->p_szc == szc); 5244 } 5245 5246 /* 5247 * We can't demote kernel pages since we can't hat_unload() 5248 * the mappings. 5249 */ 5250 if (rootpp->p_vnode == &kvp) 5251 return (0); 5252 5253 /* 5254 * Attempt to lock all constituent pages except the page passed 5255 * in since it's already locked. 5256 */ 5257 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5258 ASSERT(!PP_ISFREE(tpp)); 5259 ASSERT(tpp->p_vnode != NULL); 5260 5261 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5262 break; 5263 ASSERT(tpp->p_szc == rootpp->p_szc); 5264 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5265 } 5266 5267 /* 5268 * If we failed to lock them all then unlock what we have 5269 * locked so far and bail. 5270 */ 5271 if (i < npgs) { 5272 tpp = rootpp; 5273 while (i-- > 0) { 5274 if (tpp != pp) 5275 page_unlock(tpp); 5276 tpp++; 5277 } 5278 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5279 return (0); 5280 } 5281 5282 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5283 ASSERT(PAGE_EXCL(tpp)); 5284 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5285 tpp->p_szc = 0; 5286 } 5287 5288 /* 5289 * Unlock all pages except the page passed in. 5290 */ 5291 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5292 ASSERT(!hat_page_is_mapped(tpp)); 5293 if (tpp != pp) 5294 page_unlock(tpp); 5295 } 5296 5297 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5298 return (1); 5299 } 5300 5301 /* 5302 * Called by page_free() and page_destroy() to demote the page size code 5303 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5304 * p_szc on free list, neither can we just clear p_szc of a single page_t 5305 * within a large page since it will break other code that relies on p_szc 5306 * being the same for all page_t's of a large page). Anonymous pages should 5307 * never end up here because anon_map_getpages() cannot deal with p_szc 5308 * changes after a single constituent page is locked. While anonymous or 5309 * kernel large pages are demoted or freed the entire large page at a time 5310 * with all constituent pages locked EXCL for the file system pages we 5311 * have to be able to demote a large page (i.e. decrease all constituent pages 5312 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5313 * we can easily deal with anonymous page demotion the entire large page at a 5314 * time is that those operation originate at address space level and concern 5315 * the entire large page region with actual demotion only done when pages are 5316 * not shared with any other processes (therefore we can always get EXCL lock 5317 * on all anonymous constituent pages after clearing segment page 5318 * cache). However file system pages can be truncated or invalidated at a 5319 * PAGESIZE level from the file system side and end up in page_free() or 5320 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5321 * and therfore pageout should be able to demote a large page by EXCL locking 5322 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5323 * rely on being able to lock EXCL all constituent pages. 5324 * 5325 * To prevent szc changes on file system pages one has to lock all constituent 5326 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5327 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5328 * prevent szc changes is hat layer that uses its own page level mlist 5329 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5330 * taken. Therefore we need to change szc under hat level locks if we only 5331 * have an EXCL lock on a single constituent page and hat still references any 5332 * of constituent pages. (Note we can't "ignore" hat layer by simply 5333 * hat_pageunload() all constituent pages without having EXCL locks on all of 5334 * constituent pages). We use hat_page_demote() call to safely demote szc of 5335 * all constituent pages under hat locks when we only have an EXCL lock on one 5336 * of constituent pages. 5337 * 5338 * This routine calls page_szc_lock() before calling hat_page_demote() to 5339 * allow segvn in one special case not to lock all constituent pages SHARED 5340 * before calling hat_memload_array() that relies on p_szc not changeing even 5341 * before hat level mlist lock is taken. In that case segvn uses 5342 * page_szc_lock() to prevent hat_page_demote() changeing p_szc values. 5343 * 5344 * Anonymous or kernel page demotion still has to lock all pages exclusively 5345 * and do hat_pageunload() on all constituent pages before demoting the page 5346 * therefore there's no need for anonymous or kernel page demotion to use 5347 * hat_page_demote() mechanism. 5348 * 5349 * hat_page_demote() removes all large mappings that map pp and then decreases 5350 * p_szc starting from the last constituent page of the large page. By working 5351 * from the tail of a large page in pfn decreasing order allows one looking at 5352 * the root page to know that hat_page_demote() is done for root's szc area. 5353 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5354 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5355 * that started on this page when it had szc > 1 is done for this szc 1 area. 5356 * 5357 * We are guranteed that all constituent pages of pp's large page belong to 5358 * the same vnode with the consecutive offsets increasing in the direction of 5359 * the pfn i.e. the identity of constituent pages can't change until their 5360 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5361 * large mappings to pp even though we don't lock any constituent page except 5362 * pp (i.e. we won't unload e.g. kernel locked page). 5363 */ 5364 static void 5365 page_demote_vp_pages(page_t *pp) 5366 { 5367 kmutex_t *mtx; 5368 5369 ASSERT(PAGE_EXCL(pp)); 5370 ASSERT(!PP_ISFREE(pp)); 5371 ASSERT(pp->p_vnode != NULL); 5372 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5373 ASSERT(pp->p_vnode != &kvp); 5374 5375 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5376 5377 mtx = page_szc_lock(pp); 5378 if (mtx != NULL) { 5379 hat_page_demote(pp); 5380 mutex_exit(mtx); 5381 } 5382 ASSERT(pp->p_szc == 0); 5383 } 5384 5385 /* 5386 * Mark any existing pages for migration in the given range 5387 */ 5388 void 5389 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5390 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5391 u_offset_t vnoff, int rflag) 5392 { 5393 struct anon *ap; 5394 vnode_t *curvp; 5395 lgrp_t *from; 5396 pgcnt_t i; 5397 pgcnt_t nlocked; 5398 u_offset_t off; 5399 pfn_t pfn; 5400 size_t pgsz; 5401 size_t segpgsz; 5402 pgcnt_t pages; 5403 uint_t pszc; 5404 page_t **ppa; 5405 pgcnt_t ppa_nentries; 5406 page_t *pp; 5407 caddr_t va; 5408 ulong_t an_idx; 5409 anon_sync_obj_t cookie; 5410 5411 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5412 5413 /* 5414 * Don't do anything if don't need to do lgroup optimizations 5415 * on this system 5416 */ 5417 if (!lgrp_optimizations()) 5418 return; 5419 5420 /* 5421 * Align address and length to (potentially large) page boundary 5422 */ 5423 segpgsz = page_get_pagesize(seg->s_szc); 5424 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5425 if (rflag) 5426 len = P2ROUNDUP(len, segpgsz); 5427 5428 /* 5429 * Allocate page array to accomodate largest page size 5430 */ 5431 pgsz = page_get_pagesize(page_num_pagesizes() - 1); 5432 ppa_nentries = btop(pgsz); 5433 ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP); 5434 5435 /* 5436 * Do one (large) page at a time 5437 */ 5438 va = addr; 5439 while (va < addr + len) { 5440 /* 5441 * Lookup (root) page for vnode and offset corresponding to 5442 * this virtual address 5443 * Try anonmap first since there may be copy-on-write 5444 * pages, but initialize vnode pointer and offset using 5445 * vnode arguments just in case there isn't an amp. 5446 */ 5447 curvp = vp; 5448 off = vnoff + va - seg->s_base; 5449 if (amp) { 5450 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5451 an_idx = anon_index + seg_page(seg, va); 5452 anon_array_enter(amp, an_idx, &cookie); 5453 ap = anon_get_ptr(amp->ahp, an_idx); 5454 if (ap) 5455 swap_xlate(ap, &curvp, &off); 5456 anon_array_exit(&cookie); 5457 ANON_LOCK_EXIT(&->a_rwlock); 5458 } 5459 5460 pp = NULL; 5461 if (curvp) 5462 pp = page_lookup(curvp, off, SE_SHARED); 5463 5464 /* 5465 * If there isn't a page at this virtual address, 5466 * skip to next page 5467 */ 5468 if (pp == NULL) { 5469 va += PAGESIZE; 5470 continue; 5471 } 5472 5473 /* 5474 * Figure out which lgroup this page is in for kstats 5475 */ 5476 pfn = page_pptonum(pp); 5477 from = lgrp_pfn_to_lgrp(pfn); 5478 5479 /* 5480 * Get page size, and round up and skip to next page boundary 5481 * if unaligned address 5482 */ 5483 pszc = pp->p_szc; 5484 pgsz = page_get_pagesize(pszc); 5485 pages = btop(pgsz); 5486 if (!IS_P2ALIGNED(va, pgsz) || 5487 !IS_P2ALIGNED(pfn, pages) || 5488 pgsz > segpgsz) { 5489 pgsz = MIN(pgsz, segpgsz); 5490 page_unlock(pp); 5491 i = btop(P2END((uintptr_t)va, pgsz) - 5492 (uintptr_t)va); 5493 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5494 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i); 5495 continue; 5496 } 5497 5498 /* 5499 * Upgrade to exclusive lock on page 5500 */ 5501 if (!page_tryupgrade(pp)) { 5502 page_unlock(pp); 5503 va += pgsz; 5504 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5505 btop(pgsz)); 5506 continue; 5507 } 5508 5509 /* 5510 * Remember pages locked exclusively and how many 5511 */ 5512 ppa[0] = pp; 5513 nlocked = 1; 5514 5515 /* 5516 * Lock constituent pages if this is large page 5517 */ 5518 if (pages > 1) { 5519 /* 5520 * Lock all constituents except root page, since it 5521 * should be locked already. 5522 */ 5523 for (i = 1; i < pages; i++) { 5524 pp++; 5525 if (!page_trylock(pp, SE_EXCL)) { 5526 break; 5527 } 5528 if (PP_ISFREE(pp) || 5529 pp->p_szc != pszc) { 5530 /* 5531 * hat_page_demote() raced in with us. 5532 */ 5533 ASSERT(!IS_SWAPFSVP(curvp)); 5534 page_unlock(pp); 5535 break; 5536 } 5537 ppa[nlocked] = pp; 5538 nlocked++; 5539 } 5540 } 5541 5542 /* 5543 * If all constituent pages couldn't be locked, 5544 * unlock pages locked so far and skip to next page. 5545 */ 5546 if (nlocked != pages) { 5547 for (i = 0; i < nlocked; i++) 5548 page_unlock(ppa[i]); 5549 va += pgsz; 5550 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5551 btop(pgsz)); 5552 continue; 5553 } 5554 5555 /* 5556 * hat_page_demote() can no longer happen 5557 * since last cons page had the right p_szc after 5558 * all cons pages were locked. all cons pages 5559 * should now have the same p_szc. 5560 */ 5561 5562 /* 5563 * All constituent pages locked successfully, so mark 5564 * large page for migration and unload the mappings of 5565 * constituent pages, so a fault will occur on any part of the 5566 * large page 5567 */ 5568 PP_SETMIGRATE(ppa[0]); 5569 for (i = 0; i < nlocked; i++) { 5570 pp = ppa[i]; 5571 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 5572 ASSERT(hat_page_getshare(pp) == 0); 5573 page_unlock(pp); 5574 } 5575 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5576 5577 va += pgsz; 5578 } 5579 kmem_free(ppa, ppa_nentries * sizeof (page_t *)); 5580 } 5581 5582 /* 5583 * Migrate any pages that have been marked for migration in the given range 5584 */ 5585 void 5586 page_migrate( 5587 struct seg *seg, 5588 caddr_t addr, 5589 page_t **ppa, 5590 pgcnt_t npages) 5591 { 5592 lgrp_t *from; 5593 lgrp_t *to; 5594 page_t *newpp; 5595 page_t *pp; 5596 pfn_t pfn; 5597 size_t pgsz; 5598 spgcnt_t page_cnt; 5599 spgcnt_t i; 5600 uint_t pszc; 5601 5602 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5603 5604 while (npages > 0) { 5605 pp = *ppa; 5606 pszc = pp->p_szc; 5607 pgsz = page_get_pagesize(pszc); 5608 page_cnt = btop(pgsz); 5609 5610 /* 5611 * Check to see whether this page is marked for migration 5612 * 5613 * Assume that root page of large page is marked for 5614 * migration and none of the other constituent pages 5615 * are marked. This really simplifies clearing the 5616 * migrate bit by not having to clear it from each 5617 * constituent page. 5618 * 5619 * note we don't want to relocate an entire large page if 5620 * someone is only using one subpage. 5621 */ 5622 if (npages < page_cnt) 5623 break; 5624 5625 /* 5626 * Is it marked for migration? 5627 */ 5628 if (!PP_ISMIGRATE(pp)) 5629 goto next; 5630 5631 /* 5632 * Determine lgroups that page is being migrated between 5633 */ 5634 pfn = page_pptonum(pp); 5635 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5636 break; 5637 } 5638 from = lgrp_pfn_to_lgrp(pfn); 5639 to = lgrp_mem_choose(seg, addr, pgsz); 5640 5641 /* 5642 * Check to see whether we are trying to migrate page to lgroup 5643 * where it is allocated already 5644 */ 5645 if (to == from) { 5646 PP_CLRMIGRATE(pp); 5647 goto next; 5648 } 5649 5650 /* 5651 * Need to get exclusive lock's to migrate 5652 */ 5653 for (i = 0; i < page_cnt; i++) { 5654 ASSERT(PAGE_LOCKED(ppa[i])); 5655 if (page_pptonum(ppa[i]) != pfn + i || 5656 ppa[i]->p_szc != pszc) { 5657 break; 5658 } 5659 if (!page_tryupgrade(ppa[i])) { 5660 lgrp_stat_add(from->lgrp_id, 5661 LGRP_PM_FAIL_LOCK_PGS, 5662 page_cnt); 5663 break; 5664 } 5665 } 5666 if (i != page_cnt) { 5667 while (--i != -1) { 5668 page_downgrade(ppa[i]); 5669 } 5670 goto next; 5671 } 5672 5673 (void) page_create_wait(page_cnt, PG_WAIT); 5674 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5675 if (newpp == NULL) { 5676 page_create_putback(page_cnt); 5677 for (i = 0; i < page_cnt; i++) { 5678 page_downgrade(ppa[i]); 5679 } 5680 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5681 page_cnt); 5682 goto next; 5683 } 5684 ASSERT(newpp->p_szc == pszc); 5685 /* 5686 * Clear migrate bit and relocate page 5687 */ 5688 PP_CLRMIGRATE(pp); 5689 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5690 panic("page_migrate: page_relocate failed"); 5691 } 5692 ASSERT(page_cnt * PAGESIZE == pgsz); 5693 5694 /* 5695 * Keep stats for number of pages migrated from and to 5696 * each lgroup 5697 */ 5698 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5699 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5700 /* 5701 * update the page_t array we were passed in and 5702 * unlink constituent pages of a large page. 5703 */ 5704 for (i = 0; i < page_cnt; ++i, ++pp) { 5705 ASSERT(PAGE_EXCL(newpp)); 5706 ASSERT(newpp->p_szc == pszc); 5707 ppa[i] = newpp; 5708 pp = newpp; 5709 page_sub(&newpp, pp); 5710 page_downgrade(pp); 5711 } 5712 ASSERT(newpp == NULL); 5713 next: 5714 addr += pgsz; 5715 ppa += page_cnt; 5716 npages -= page_cnt; 5717 } 5718 } 5719 5720 ulong_t mem_waiters = 0; 5721 ulong_t max_count = 20; 5722 #define MAX_DELAY 0x1ff 5723 5724 /* 5725 * Check if enough memory is available to proceed. 5726 * Depending on system configuration and how much memory is 5727 * reserved for swap we need to check against two variables. 5728 * e.g. on systems with little physical swap availrmem can be 5729 * more reliable indicator of how much memory is available. 5730 * On systems with large phys swap freemem can be better indicator. 5731 * If freemem drops below threshold level don't return an error 5732 * immediately but wake up pageout to free memory and block. 5733 * This is done number of times. If pageout is not able to free 5734 * memory within certain time return an error. 5735 * The same applies for availrmem but kmem_reap is used to 5736 * free memory. 5737 */ 5738 int 5739 page_mem_avail(pgcnt_t npages) 5740 { 5741 ulong_t count; 5742 5743 #if defined(__i386) 5744 if (freemem > desfree + npages && 5745 availrmem > swapfs_reserve + npages && 5746 btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem + 5747 npages) 5748 return (1); 5749 #else 5750 if (freemem > desfree + npages && 5751 availrmem > swapfs_reserve + npages) 5752 return (1); 5753 #endif 5754 5755 count = max_count; 5756 atomic_add_long(&mem_waiters, 1); 5757 5758 while (freemem < desfree + npages && --count) { 5759 cv_signal(&proc_pageout->p_cv); 5760 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5761 atomic_add_long(&mem_waiters, -1); 5762 return (0); 5763 } 5764 } 5765 if (count == 0) { 5766 atomic_add_long(&mem_waiters, -1); 5767 return (0); 5768 } 5769 5770 count = max_count; 5771 while (availrmem < swapfs_reserve + npages && --count) { 5772 kmem_reap(); 5773 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5774 atomic_add_long(&mem_waiters, -1); 5775 return (0); 5776 } 5777 } 5778 atomic_add_long(&mem_waiters, -1); 5779 if (count == 0) 5780 return (0); 5781 5782 #if defined(__i386) 5783 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 5784 tune.t_minarmem + npages) 5785 return (0); 5786 #endif 5787 return (1); 5788 } 5789 5790 5791 /* 5792 * Search the memory segments to locate the desired page. Within a 5793 * segment, pages increase linearly with one page structure per 5794 * physical page frame (size PAGESIZE). The search begins 5795 * with the segment that was accessed last, to take advantage of locality. 5796 * If the hint misses, we start from the beginning of the sorted memseg list 5797 */ 5798 5799 5800 /* 5801 * Some data structures for pfn to pp lookup. 5802 */ 5803 ulong_t mhash_per_slot; 5804 struct memseg *memseg_hash[N_MEM_SLOTS]; 5805 5806 page_t * 5807 page_numtopp_nolock(pfn_t pfnum) 5808 { 5809 struct memseg *seg; 5810 page_t *pp; 5811 vm_cpu_data_t *vc = CPU->cpu_vm_data; 5812 5813 ASSERT(vc != NULL); 5814 5815 MEMSEG_STAT_INCR(nsearch); 5816 5817 /* Try last winner first */ 5818 if (((seg = vc->vc_pnum_memseg) != NULL) && 5819 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5820 MEMSEG_STAT_INCR(nlastwon); 5821 pp = seg->pages + (pfnum - seg->pages_base); 5822 if (pp->p_pagenum == pfnum) 5823 return ((page_t *)pp); 5824 } 5825 5826 /* Else Try hash */ 5827 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5828 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5829 MEMSEG_STAT_INCR(nhashwon); 5830 vc->vc_pnum_memseg = seg; 5831 pp = seg->pages + (pfnum - seg->pages_base); 5832 if (pp->p_pagenum == pfnum) 5833 return ((page_t *)pp); 5834 } 5835 5836 /* Else Brute force */ 5837 for (seg = memsegs; seg != NULL; seg = seg->next) { 5838 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5839 vc->vc_pnum_memseg = seg; 5840 pp = seg->pages + (pfnum - seg->pages_base); 5841 return ((page_t *)pp); 5842 } 5843 } 5844 vc->vc_pnum_memseg = NULL; 5845 MEMSEG_STAT_INCR(nnotfound); 5846 return ((page_t *)NULL); 5847 5848 } 5849 5850 struct memseg * 5851 page_numtomemseg_nolock(pfn_t pfnum) 5852 { 5853 struct memseg *seg; 5854 page_t *pp; 5855 5856 /* Try hash */ 5857 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5858 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5859 pp = seg->pages + (pfnum - seg->pages_base); 5860 if (pp->p_pagenum == pfnum) 5861 return (seg); 5862 } 5863 5864 /* Else Brute force */ 5865 for (seg = memsegs; seg != NULL; seg = seg->next) { 5866 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5867 return (seg); 5868 } 5869 } 5870 return ((struct memseg *)NULL); 5871 } 5872 5873 /* 5874 * Given a page and a count return the page struct that is 5875 * n structs away from the current one in the global page 5876 * list. 5877 * 5878 * This function wraps to the first page upon 5879 * reaching the end of the memseg list. 5880 */ 5881 page_t * 5882 page_nextn(page_t *pp, ulong_t n) 5883 { 5884 struct memseg *seg; 5885 page_t *ppn; 5886 vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5887 5888 ASSERT(vc != NULL); 5889 5890 if (((seg = vc->vc_pnext_memseg) == NULL) || 5891 (seg->pages_base == seg->pages_end) || 5892 !(pp >= seg->pages && pp < seg->epages)) { 5893 5894 for (seg = memsegs; seg; seg = seg->next) { 5895 if (pp >= seg->pages && pp < seg->epages) 5896 break; 5897 } 5898 5899 if (seg == NULL) { 5900 /* Memory delete got in, return something valid. */ 5901 /* TODO: fix me. */ 5902 seg = memsegs; 5903 pp = seg->pages; 5904 } 5905 } 5906 5907 /* check for wraparound - possible if n is large */ 5908 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5909 n -= seg->epages - pp; 5910 seg = seg->next; 5911 if (seg == NULL) 5912 seg = memsegs; 5913 pp = seg->pages; 5914 } 5915 vc->vc_pnext_memseg = seg; 5916 return (ppn); 5917 } 5918 5919 /* 5920 * Initialize for a loop using page_next_scan_large(). 5921 */ 5922 page_t * 5923 page_next_scan_init(void **cookie) 5924 { 5925 ASSERT(cookie != NULL); 5926 *cookie = (void *)memsegs; 5927 return ((page_t *)memsegs->pages); 5928 } 5929 5930 /* 5931 * Return the next page in a scan of page_t's, assuming we want 5932 * to skip over sub-pages within larger page sizes. 5933 * 5934 * The cookie is used to keep track of the current memseg. 5935 */ 5936 page_t * 5937 page_next_scan_large( 5938 page_t *pp, 5939 ulong_t *n, 5940 void **cookie) 5941 { 5942 struct memseg *seg = (struct memseg *)*cookie; 5943 page_t *new_pp; 5944 ulong_t cnt; 5945 pfn_t pfn; 5946 5947 5948 /* 5949 * get the count of page_t's to skip based on the page size 5950 */ 5951 ASSERT(pp != NULL); 5952 if (pp->p_szc == 0) { 5953 cnt = 1; 5954 } else { 5955 pfn = page_pptonum(pp); 5956 cnt = page_get_pagecnt(pp->p_szc); 5957 cnt -= pfn & (cnt - 1); 5958 } 5959 *n += cnt; 5960 new_pp = pp + cnt; 5961 5962 /* 5963 * Catch if we went past the end of the current memory segment. If so, 5964 * just move to the next segment with pages. 5965 */ 5966 if (new_pp >= seg->epages) { 5967 do { 5968 seg = seg->next; 5969 if (seg == NULL) 5970 seg = memsegs; 5971 } while (seg->pages == seg->epages); 5972 new_pp = seg->pages; 5973 *cookie = (void *)seg; 5974 } 5975 5976 return (new_pp); 5977 } 5978 5979 5980 /* 5981 * Returns next page in list. Note: this function wraps 5982 * to the first page in the list upon reaching the end 5983 * of the list. Callers should be aware of this fact. 5984 */ 5985 5986 /* We should change this be a #define */ 5987 5988 page_t * 5989 page_next(page_t *pp) 5990 { 5991 return (page_nextn(pp, 1)); 5992 } 5993 5994 page_t * 5995 page_first() 5996 { 5997 return ((page_t *)memsegs->pages); 5998 } 5999 6000 6001 /* 6002 * This routine is called at boot with the initial memory configuration 6003 * and when memory is added or removed. 6004 */ 6005 void 6006 build_pfn_hash() 6007 { 6008 pfn_t cur; 6009 pgcnt_t index; 6010 struct memseg *pseg; 6011 int i; 6012 6013 /* 6014 * Clear memseg_hash array. 6015 * Since memory add/delete is designed to operate concurrently 6016 * with normal operation, the hash rebuild must be able to run 6017 * concurrently with page_numtopp_nolock(). To support this 6018 * functionality, assignments to memseg_hash array members must 6019 * be done atomically. 6020 * 6021 * NOTE: bzero() does not currently guarantee this for kernel 6022 * threads, and cannot be used here. 6023 */ 6024 for (i = 0; i < N_MEM_SLOTS; i++) 6025 memseg_hash[i] = NULL; 6026 6027 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6028 6029 /* 6030 * Physmax is the last valid pfn. 6031 */ 6032 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6033 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6034 index = MEMSEG_PFN_HASH(pseg->pages_base); 6035 cur = pseg->pages_base; 6036 do { 6037 if (index >= N_MEM_SLOTS) 6038 index = MEMSEG_PFN_HASH(cur); 6039 6040 if (memseg_hash[index] == NULL || 6041 memseg_hash[index]->pages_base > pseg->pages_base) { 6042 memseg_hash[index] = pseg; 6043 hat_kpm_mseghash_update(index, pseg); 6044 } 6045 cur += mhash_per_slot; 6046 index++; 6047 } while (cur < pseg->pages_end); 6048 } 6049 } 6050 6051 /* 6052 * Return the pagenum for the pp 6053 */ 6054 pfn_t 6055 page_pptonum(page_t *pp) 6056 { 6057 return (pp->p_pagenum); 6058 } 6059 6060 /* 6061 * interface to the referenced and modified etc bits 6062 * in the PSM part of the page struct 6063 * when no locking is desired. 6064 */ 6065 void 6066 page_set_props(page_t *pp, uint_t flags) 6067 { 6068 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6069 pp->p_nrm |= (uchar_t)flags; 6070 } 6071 6072 void 6073 page_clr_all_props(page_t *pp) 6074 { 6075 pp->p_nrm = 0; 6076 } 6077 6078 /* 6079 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6080 */ 6081 int 6082 page_clear_lck_cow(page_t *pp, int adjust) 6083 { 6084 int f_amount; 6085 6086 ASSERT(PAGE_EXCL(pp)); 6087 6088 /* 6089 * The page_struct_lock need not be acquired here since 6090 * we require the caller hold the page exclusively locked. 6091 */ 6092 f_amount = 0; 6093 if (pp->p_lckcnt) { 6094 f_amount = 1; 6095 pp->p_lckcnt = 0; 6096 } 6097 if (pp->p_cowcnt) { 6098 f_amount += pp->p_cowcnt; 6099 pp->p_cowcnt = 0; 6100 } 6101 6102 if (adjust && f_amount) { 6103 mutex_enter(&freemem_lock); 6104 availrmem += f_amount; 6105 mutex_exit(&freemem_lock); 6106 } 6107 6108 return (f_amount); 6109 } 6110 6111 /* 6112 * The following functions is called from free_vp_pages() 6113 * for an inexact estimate of a newly free'd page... 6114 */ 6115 ulong_t 6116 page_share_cnt(page_t *pp) 6117 { 6118 return (hat_page_getshare(pp)); 6119 } 6120 6121 int 6122 page_isshared(page_t *pp) 6123 { 6124 return (hat_page_getshare(pp) > 1); 6125 } 6126 6127 int 6128 page_isfree(page_t *pp) 6129 { 6130 return (PP_ISFREE(pp)); 6131 } 6132 6133 int 6134 page_isref(page_t *pp) 6135 { 6136 return (hat_page_getattr(pp, P_REF)); 6137 } 6138 6139 int 6140 page_ismod(page_t *pp) 6141 { 6142 return (hat_page_getattr(pp, P_MOD)); 6143 } 6144