1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #pragma ident "%Z%%M% %I% %E% SMI" 40 41 /* 42 * VM - physical page management. 43 */ 44 45 #include <sys/types.h> 46 #include <sys/t_lock.h> 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/errno.h> 50 #include <sys/time.h> 51 #include <sys/vnode.h> 52 #include <sys/vm.h> 53 #include <sys/vtrace.h> 54 #include <sys/swap.h> 55 #include <sys/cmn_err.h> 56 #include <sys/tuneable.h> 57 #include <sys/sysmacros.h> 58 #include <sys/cpuvar.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/tnf_probe.h> 62 #include <sys/condvar_impl.h> 63 #include <sys/mem_config.h> 64 #include <sys/mem_cage.h> 65 #include <sys/kmem.h> 66 #include <sys/atomic.h> 67 #include <sys/strlog.h> 68 #include <sys/mman.h> 69 #include <sys/ontrap.h> 70 #include <sys/lgrp.h> 71 #include <sys/vfs.h> 72 73 #include <vm/hat.h> 74 #include <vm/anon.h> 75 #include <vm/page.h> 76 #include <vm/seg.h> 77 #include <vm/pvn.h> 78 #include <vm/seg_kmem.h> 79 #include <vm/vm_dep.h> 80 #include <sys/vm_usage.h> 81 #include <fs/fs_subr.h> 82 #include <sys/ddi.h> 83 #include <sys/modctl.h> 84 85 static int nopageage = 0; 86 87 static pgcnt_t max_page_get; /* max page_get request size in pages */ 88 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 89 90 /* 91 * freemem_lock protects all freemem variables: 92 * availrmem. Also this lock protects the globals which track the 93 * availrmem changes for accurate kernel footprint calculation. 94 * See below for an explanation of these 95 * globals. 96 */ 97 kmutex_t freemem_lock; 98 pgcnt_t availrmem; 99 pgcnt_t availrmem_initial; 100 101 /* 102 * These globals track availrmem changes to get a more accurate 103 * estimate of tke kernel size. Historically pp_kernel is used for 104 * kernel size and is based on availrmem. But availrmem is adjusted for 105 * locked pages in the system not just for kernel locked pages. 106 * These new counters will track the pages locked through segvn and 107 * by explicit user locking. 108 * 109 * segvn_pages_locked : This keeps track on a global basis how many pages 110 * are currently locked because of I/O. 111 * 112 * pages_locked : How many pages are locked because of user specified 113 * locking through mlock or plock. 114 * 115 * pages_useclaim,pages_claimed : These two variables track the 116 * claim adjustments because of the protection changes on a segvn segment. 117 * 118 * All these globals are protected by the same lock which protects availrmem. 119 */ 120 pgcnt_t segvn_pages_locked; 121 pgcnt_t pages_locked; 122 pgcnt_t pages_useclaim; 123 pgcnt_t pages_claimed; 124 125 126 /* 127 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 128 */ 129 static kmutex_t new_freemem_lock; 130 static uint_t freemem_wait; /* someone waiting for freemem */ 131 static kcondvar_t freemem_cv; 132 133 /* 134 * The logical page free list is maintained as two lists, the 'free' 135 * and the 'cache' lists. 136 * The free list contains those pages that should be reused first. 137 * 138 * The implementation of the lists is machine dependent. 139 * page_get_freelist(), page_get_cachelist(), 140 * page_list_sub(), and page_list_add() 141 * form the interface to the machine dependent implementation. 142 * 143 * Pages with p_free set are on the cache list. 144 * Pages with p_free and p_age set are on the free list, 145 * 146 * A page may be locked while on either list. 147 */ 148 149 /* 150 * free list accounting stuff. 151 * 152 * 153 * Spread out the value for the number of pages on the 154 * page free and page cache lists. If there is just one 155 * value, then it must be under just one lock. 156 * The lock contention and cache traffic are a real bother. 157 * 158 * When we acquire and then drop a single pcf lock 159 * we can start in the middle of the array of pcf structures. 160 * If we acquire more than one pcf lock at a time, we need to 161 * start at the front to avoid deadlocking. 162 * 163 * pcf_count holds the number of pages in each pool. 164 * 165 * pcf_block is set when page_create_get_something() has asked the 166 * PSM page freelist and page cachelist routines without specifying 167 * a color and nothing came back. This is used to block anything 168 * else from moving pages from one list to the other while the 169 * lists are searched again. If a page is freeed while pcf_block is 170 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 171 * of clearning pcf_block, doing the wakeups, etc. 172 */ 173 174 #if NCPU <= 4 175 #define PAD 2 176 #define PCF_FANOUT 4 177 static uint_t pcf_mask = PCF_FANOUT - 1; 178 #else 179 #define PAD 10 180 #ifdef sun4v 181 #define PCF_FANOUT 32 182 #else 183 #define PCF_FANOUT 128 184 #endif 185 static uint_t pcf_mask = PCF_FANOUT - 1; 186 #endif 187 188 struct pcf { 189 kmutex_t pcf_lock; /* protects the structure */ 190 uint_t pcf_count; /* page count */ 191 uint_t pcf_wait; /* number of waiters */ 192 uint_t pcf_block; /* pcgs flag to page_free() */ 193 uint_t pcf_reserve; /* pages freed after pcf_block set */ 194 uint_t pcf_fill[PAD]; /* to line up on the caches */ 195 }; 196 197 static struct pcf pcf[PCF_FANOUT]; 198 #define PCF_INDEX() ((CPU->cpu_id) & (pcf_mask)) 199 200 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 201 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 202 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 203 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 204 205 #ifdef VM_STATS 206 207 /* 208 * No locks, but so what, they are only statistics. 209 */ 210 211 static struct page_tcnt { 212 int pc_free_cache; /* free's into cache list */ 213 int pc_free_dontneed; /* free's with dontneed */ 214 int pc_free_pageout; /* free's from pageout */ 215 int pc_free_free; /* free's into free list */ 216 int pc_free_pages; /* free's into large page free list */ 217 int pc_destroy_pages; /* large page destroy's */ 218 int pc_get_cache; /* get's from cache list */ 219 int pc_get_free; /* get's from free list */ 220 int pc_reclaim; /* reclaim's */ 221 int pc_abortfree; /* abort's of free pages */ 222 int pc_find_hit; /* find's that find page */ 223 int pc_find_miss; /* find's that don't find page */ 224 int pc_destroy_free; /* # of free pages destroyed */ 225 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 226 int pc_find_hashlen[PC_HASH_CNT+1]; 227 int pc_addclaim_pages; 228 int pc_subclaim_pages; 229 int pc_free_replacement_page[2]; 230 int pc_try_demote_pages[6]; 231 int pc_demote_pages[2]; 232 } pagecnt; 233 234 uint_t hashin_count; 235 uint_t hashin_not_held; 236 uint_t hashin_already; 237 238 uint_t hashout_count; 239 uint_t hashout_not_held; 240 241 uint_t page_create_count; 242 uint_t page_create_not_enough; 243 uint_t page_create_not_enough_again; 244 uint_t page_create_zero; 245 uint_t page_create_hashout; 246 uint_t page_create_page_lock_failed; 247 uint_t page_create_trylock_failed; 248 uint_t page_create_found_one; 249 uint_t page_create_hashin_failed; 250 uint_t page_create_dropped_phm; 251 252 uint_t page_create_new; 253 uint_t page_create_exists; 254 uint_t page_create_putbacks; 255 uint_t page_create_overshoot; 256 257 uint_t page_reclaim_zero; 258 uint_t page_reclaim_zero_locked; 259 260 uint_t page_rename_exists; 261 uint_t page_rename_count; 262 263 uint_t page_lookup_cnt[20]; 264 uint_t page_lookup_nowait_cnt[10]; 265 uint_t page_find_cnt; 266 uint_t page_exists_cnt; 267 uint_t page_exists_forreal_cnt; 268 uint_t page_lookup_dev_cnt; 269 uint_t get_cachelist_cnt; 270 uint_t page_create_cnt[10]; 271 uint_t alloc_pages[8]; 272 uint_t page_exphcontg[19]; 273 uint_t page_create_large_cnt[10]; 274 275 /* 276 * Collects statistics. 277 */ 278 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 279 uint_t mylen = 0; \ 280 \ 281 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 282 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 283 break; \ 284 } \ 285 if ((pp) != NULL) \ 286 pagecnt.pc_find_hit++; \ 287 else \ 288 pagecnt.pc_find_miss++; \ 289 if (mylen > PC_HASH_CNT) \ 290 mylen = PC_HASH_CNT; \ 291 pagecnt.pc_find_hashlen[mylen]++; \ 292 } 293 294 #else /* VM_STATS */ 295 296 /* 297 * Don't collect statistics 298 */ 299 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 300 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 301 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 302 break; \ 303 } \ 304 } 305 306 #endif /* VM_STATS */ 307 308 309 310 #ifdef DEBUG 311 #define MEMSEG_SEARCH_STATS 312 #endif 313 314 #ifdef MEMSEG_SEARCH_STATS 315 struct memseg_stats { 316 uint_t nsearch; 317 uint_t nlastwon; 318 uint_t nhashwon; 319 uint_t nnotfound; 320 } memseg_stats; 321 322 #define MEMSEG_STAT_INCR(v) \ 323 atomic_add_32(&memseg_stats.v, 1) 324 #else 325 #define MEMSEG_STAT_INCR(x) 326 #endif 327 328 struct memseg *memsegs; /* list of memory segments */ 329 330 331 static void page_init_mem_config(void); 332 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 333 static void page_do_hashout(page_t *); 334 static void page_capture_init(); 335 int page_capture_take_action(page_t *, uint_t, void *); 336 337 static void page_demote_vp_pages(page_t *); 338 339 /* 340 * vm subsystem related initialization 341 */ 342 void 343 vm_init(void) 344 { 345 boolean_t callb_vm_cpr(void *, int); 346 347 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 348 page_init_mem_config(); 349 page_retire_init(); 350 vm_usage_init(); 351 page_capture_init(); 352 } 353 354 /* 355 * This function is called at startup and when memory is added or deleted. 356 */ 357 void 358 init_pages_pp_maximum() 359 { 360 static pgcnt_t p_min; 361 static pgcnt_t pages_pp_maximum_startup; 362 static pgcnt_t avrmem_delta; 363 static int init_done; 364 static int user_set; /* true if set in /etc/system */ 365 366 if (init_done == 0) { 367 368 /* If the user specified a value, save it */ 369 if (pages_pp_maximum != 0) { 370 user_set = 1; 371 pages_pp_maximum_startup = pages_pp_maximum; 372 } 373 374 /* 375 * Setting of pages_pp_maximum is based first time 376 * on the value of availrmem just after the start-up 377 * allocations. To preserve this relationship at run 378 * time, use a delta from availrmem_initial. 379 */ 380 ASSERT(availrmem_initial >= availrmem); 381 avrmem_delta = availrmem_initial - availrmem; 382 383 /* The allowable floor of pages_pp_maximum */ 384 p_min = tune.t_minarmem + 100; 385 386 /* Make sure we don't come through here again. */ 387 init_done = 1; 388 } 389 /* 390 * Determine pages_pp_maximum, the number of currently available 391 * pages (availrmem) that can't be `locked'. If not set by 392 * the user, we set it to 4% of the currently available memory 393 * plus 4MB. 394 * But we also insist that it be greater than tune.t_minarmem; 395 * otherwise a process could lock down a lot of memory, get swapped 396 * out, and never have enough to get swapped back in. 397 */ 398 if (user_set) 399 pages_pp_maximum = pages_pp_maximum_startup; 400 else 401 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 402 + btop(4 * 1024 * 1024); 403 404 if (pages_pp_maximum <= p_min) { 405 pages_pp_maximum = p_min; 406 } 407 } 408 409 void 410 set_max_page_get(pgcnt_t target_total_pages) 411 { 412 max_page_get = target_total_pages / 2; 413 } 414 415 static pgcnt_t pending_delete; 416 417 /*ARGSUSED*/ 418 static void 419 page_mem_config_post_add( 420 void *arg, 421 pgcnt_t delta_pages) 422 { 423 set_max_page_get(total_pages - pending_delete); 424 init_pages_pp_maximum(); 425 } 426 427 /*ARGSUSED*/ 428 static int 429 page_mem_config_pre_del( 430 void *arg, 431 pgcnt_t delta_pages) 432 { 433 pgcnt_t nv; 434 435 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 436 set_max_page_get(total_pages - nv); 437 return (0); 438 } 439 440 /*ARGSUSED*/ 441 static void 442 page_mem_config_post_del( 443 void *arg, 444 pgcnt_t delta_pages, 445 int cancelled) 446 { 447 pgcnt_t nv; 448 449 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 450 set_max_page_get(total_pages - nv); 451 if (!cancelled) 452 init_pages_pp_maximum(); 453 } 454 455 static kphysm_setup_vector_t page_mem_config_vec = { 456 KPHYSM_SETUP_VECTOR_VERSION, 457 page_mem_config_post_add, 458 page_mem_config_pre_del, 459 page_mem_config_post_del, 460 }; 461 462 static void 463 page_init_mem_config(void) 464 { 465 int ret; 466 467 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 468 ASSERT(ret == 0); 469 } 470 471 /* 472 * Evenly spread out the PCF counters for large free pages 473 */ 474 static void 475 page_free_large_ctr(pgcnt_t npages) 476 { 477 static struct pcf *p = pcf; 478 pgcnt_t lump; 479 480 freemem += npages; 481 482 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 483 484 while (npages > 0) { 485 486 ASSERT(!p->pcf_block); 487 488 if (lump < npages) { 489 p->pcf_count += (uint_t)lump; 490 npages -= lump; 491 } else { 492 p->pcf_count += (uint_t)npages; 493 npages = 0; 494 } 495 496 ASSERT(!p->pcf_wait); 497 498 if (++p > &pcf[PCF_FANOUT - 1]) 499 p = pcf; 500 } 501 502 ASSERT(npages == 0); 503 } 504 505 /* 506 * Add a physical chunk of memory to the system free lists during startup. 507 * Platform specific startup() allocates the memory for the page structs. 508 * 509 * num - number of page structures 510 * base - page number (pfn) to be associated with the first page. 511 * 512 * Since we are doing this during startup (ie. single threaded), we will 513 * use shortcut routines to avoid any locking overhead while putting all 514 * these pages on the freelists. 515 * 516 * NOTE: Any changes performed to page_free(), must also be performed to 517 * add_physmem() since this is how we initialize all page_t's at 518 * boot time. 519 */ 520 void 521 add_physmem( 522 page_t *pp, 523 pgcnt_t num, 524 pfn_t pnum) 525 { 526 page_t *root = NULL; 527 uint_t szc = page_num_pagesizes() - 1; 528 pgcnt_t large = page_get_pagecnt(szc); 529 pgcnt_t cnt = 0; 530 531 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 532 "add_physmem:pp %p num %lu", pp, num); 533 534 /* 535 * Arbitrarily limit the max page_get request 536 * to 1/2 of the page structs we have. 537 */ 538 total_pages += num; 539 set_max_page_get(total_pages); 540 541 PLCNT_MODIFY_MAX(pnum, (long)num); 542 543 /* 544 * The physical space for the pages array 545 * representing ram pages has already been 546 * allocated. Here we initialize each lock 547 * in the page structure, and put each on 548 * the free list 549 */ 550 for (; num; pp++, pnum++, num--) { 551 552 /* 553 * this needs to fill in the page number 554 * and do any other arch specific initialization 555 */ 556 add_physmem_cb(pp, pnum); 557 558 pp->p_lckcnt = 0; 559 pp->p_cowcnt = 0; 560 pp->p_slckcnt = 0; 561 562 /* 563 * Initialize the page lock as unlocked, since nobody 564 * can see or access this page yet. 565 */ 566 pp->p_selock = 0; 567 568 /* 569 * Initialize IO lock 570 */ 571 page_iolock_init(pp); 572 573 /* 574 * initialize other fields in the page_t 575 */ 576 PP_SETFREE(pp); 577 page_clr_all_props(pp); 578 PP_SETAGED(pp); 579 pp->p_offset = (u_offset_t)-1; 580 pp->p_next = pp; 581 pp->p_prev = pp; 582 583 /* 584 * Simple case: System doesn't support large pages. 585 */ 586 if (szc == 0) { 587 pp->p_szc = 0; 588 page_free_at_startup(pp); 589 continue; 590 } 591 592 /* 593 * Handle unaligned pages, we collect them up onto 594 * the root page until we have a full large page. 595 */ 596 if (!IS_P2ALIGNED(pnum, large)) { 597 598 /* 599 * If not in a large page, 600 * just free as small page. 601 */ 602 if (root == NULL) { 603 pp->p_szc = 0; 604 page_free_at_startup(pp); 605 continue; 606 } 607 608 /* 609 * Link a constituent page into the large page. 610 */ 611 pp->p_szc = szc; 612 page_list_concat(&root, &pp); 613 614 /* 615 * When large page is fully formed, free it. 616 */ 617 if (++cnt == large) { 618 page_free_large_ctr(cnt); 619 page_list_add_pages(root, PG_LIST_ISINIT); 620 root = NULL; 621 cnt = 0; 622 } 623 continue; 624 } 625 626 /* 627 * At this point we have a page number which 628 * is aligned. We assert that we aren't already 629 * in a different large page. 630 */ 631 ASSERT(IS_P2ALIGNED(pnum, large)); 632 ASSERT(root == NULL && cnt == 0); 633 634 /* 635 * If insufficient number of pages left to form 636 * a large page, just free the small page. 637 */ 638 if (num < large) { 639 pp->p_szc = 0; 640 page_free_at_startup(pp); 641 continue; 642 } 643 644 /* 645 * Otherwise start a new large page. 646 */ 647 pp->p_szc = szc; 648 cnt++; 649 root = pp; 650 } 651 ASSERT(root == NULL && cnt == 0); 652 } 653 654 /* 655 * Find a page representing the specified [vp, offset]. 656 * If we find the page but it is intransit coming in, 657 * it will have an "exclusive" lock and we wait for 658 * the i/o to complete. A page found on the free list 659 * is always reclaimed and then locked. On success, the page 660 * is locked, its data is valid and it isn't on the free 661 * list, while a NULL is returned if the page doesn't exist. 662 */ 663 page_t * 664 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 665 { 666 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 667 } 668 669 /* 670 * Find a page representing the specified [vp, offset]. 671 * We either return the one we found or, if passed in, 672 * create one with identity of [vp, offset] of the 673 * pre-allocated page. If we find existing page but it is 674 * intransit coming in, it will have an "exclusive" lock 675 * and we wait for the i/o to complete. A page found on 676 * the free list is always reclaimed and then locked. 677 * On success, the page is locked, its data is valid and 678 * it isn't on the free list, while a NULL is returned 679 * if the page doesn't exist and newpp is NULL; 680 */ 681 page_t * 682 page_lookup_create( 683 vnode_t *vp, 684 u_offset_t off, 685 se_t se, 686 page_t *newpp, 687 spgcnt_t *nrelocp, 688 int flags) 689 { 690 page_t *pp; 691 kmutex_t *phm; 692 ulong_t index; 693 uint_t hash_locked; 694 uint_t es; 695 696 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 697 VM_STAT_ADD(page_lookup_cnt[0]); 698 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 699 700 /* 701 * Acquire the appropriate page hash lock since 702 * we have to search the hash list. Pages that 703 * hash to this list can't change identity while 704 * this lock is held. 705 */ 706 hash_locked = 0; 707 index = PAGE_HASH_FUNC(vp, off); 708 phm = NULL; 709 top: 710 PAGE_HASH_SEARCH(index, pp, vp, off); 711 if (pp != NULL) { 712 VM_STAT_ADD(page_lookup_cnt[1]); 713 es = (newpp != NULL) ? 1 : 0; 714 es |= flags; 715 if (!hash_locked) { 716 VM_STAT_ADD(page_lookup_cnt[2]); 717 if (!page_try_reclaim_lock(pp, se, es)) { 718 /* 719 * On a miss, acquire the phm. Then 720 * next time, page_lock() will be called, 721 * causing a wait if the page is busy. 722 * just looping with page_trylock() would 723 * get pretty boring. 724 */ 725 VM_STAT_ADD(page_lookup_cnt[3]); 726 phm = PAGE_HASH_MUTEX(index); 727 mutex_enter(phm); 728 hash_locked = 1; 729 goto top; 730 } 731 } else { 732 VM_STAT_ADD(page_lookup_cnt[4]); 733 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 734 VM_STAT_ADD(page_lookup_cnt[5]); 735 goto top; 736 } 737 } 738 739 /* 740 * Since `pp' is locked it can not change identity now. 741 * Reconfirm we locked the correct page. 742 * 743 * Both the p_vnode and p_offset *must* be cast volatile 744 * to force a reload of their values: The PAGE_HASH_SEARCH 745 * macro will have stuffed p_vnode and p_offset into 746 * registers before calling page_trylock(); another thread, 747 * actually holding the hash lock, could have changed the 748 * page's identity in memory, but our registers would not 749 * be changed, fooling the reconfirmation. If the hash 750 * lock was held during the search, the casting would 751 * not be needed. 752 */ 753 VM_STAT_ADD(page_lookup_cnt[6]); 754 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 755 ((volatile u_offset_t)(pp->p_offset) != off)) { 756 VM_STAT_ADD(page_lookup_cnt[7]); 757 if (hash_locked) { 758 panic("page_lookup_create: lost page %p", 759 (void *)pp); 760 /*NOTREACHED*/ 761 } 762 page_unlock(pp); 763 phm = PAGE_HASH_MUTEX(index); 764 mutex_enter(phm); 765 hash_locked = 1; 766 goto top; 767 } 768 769 /* 770 * If page_trylock() was called, then pp may still be on 771 * the cachelist (can't be on the free list, it would not 772 * have been found in the search). If it is on the 773 * cachelist it must be pulled now. To pull the page from 774 * the cachelist, it must be exclusively locked. 775 * 776 * The other big difference between page_trylock() and 777 * page_lock(), is that page_lock() will pull the 778 * page from whatever free list (the cache list in this 779 * case) the page is on. If page_trylock() was used 780 * above, then we have to do the reclaim ourselves. 781 */ 782 if ((!hash_locked) && (PP_ISFREE(pp))) { 783 ASSERT(PP_ISAGED(pp) == 0); 784 VM_STAT_ADD(page_lookup_cnt[8]); 785 786 /* 787 * page_relcaim will insure that we 788 * have this page exclusively 789 */ 790 791 if (!page_reclaim(pp, NULL)) { 792 /* 793 * Page_reclaim dropped whatever lock 794 * we held. 795 */ 796 VM_STAT_ADD(page_lookup_cnt[9]); 797 phm = PAGE_HASH_MUTEX(index); 798 mutex_enter(phm); 799 hash_locked = 1; 800 goto top; 801 } else if (se == SE_SHARED && newpp == NULL) { 802 VM_STAT_ADD(page_lookup_cnt[10]); 803 page_downgrade(pp); 804 } 805 } 806 807 if (hash_locked) { 808 mutex_exit(phm); 809 } 810 811 if (newpp != NULL && pp->p_szc < newpp->p_szc && 812 PAGE_EXCL(pp) && nrelocp != NULL) { 813 ASSERT(nrelocp != NULL); 814 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 815 NULL); 816 if (*nrelocp > 0) { 817 VM_STAT_COND_ADD(*nrelocp == 1, 818 page_lookup_cnt[11]); 819 VM_STAT_COND_ADD(*nrelocp > 1, 820 page_lookup_cnt[12]); 821 pp = newpp; 822 se = SE_EXCL; 823 } else { 824 if (se == SE_SHARED) { 825 page_downgrade(pp); 826 } 827 VM_STAT_ADD(page_lookup_cnt[13]); 828 } 829 } else if (newpp != NULL && nrelocp != NULL) { 830 if (PAGE_EXCL(pp) && se == SE_SHARED) { 831 page_downgrade(pp); 832 } 833 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 834 page_lookup_cnt[14]); 835 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 836 page_lookup_cnt[15]); 837 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 838 page_lookup_cnt[16]); 839 } else if (newpp != NULL && PAGE_EXCL(pp)) { 840 se = SE_EXCL; 841 } 842 } else if (!hash_locked) { 843 VM_STAT_ADD(page_lookup_cnt[17]); 844 phm = PAGE_HASH_MUTEX(index); 845 mutex_enter(phm); 846 hash_locked = 1; 847 goto top; 848 } else if (newpp != NULL) { 849 /* 850 * If we have a preallocated page then 851 * insert it now and basically behave like 852 * page_create. 853 */ 854 VM_STAT_ADD(page_lookup_cnt[18]); 855 /* 856 * Since we hold the page hash mutex and 857 * just searched for this page, page_hashin 858 * had better not fail. If it does, that 859 * means some thread did not follow the 860 * page hash mutex rules. Panic now and 861 * get it over with. As usual, go down 862 * holding all the locks. 863 */ 864 ASSERT(MUTEX_HELD(phm)); 865 if (!page_hashin(newpp, vp, off, phm)) { 866 ASSERT(MUTEX_HELD(phm)); 867 panic("page_lookup_create: hashin failed %p %p %llx %p", 868 (void *)newpp, (void *)vp, off, (void *)phm); 869 /*NOTREACHED*/ 870 } 871 ASSERT(MUTEX_HELD(phm)); 872 mutex_exit(phm); 873 phm = NULL; 874 page_set_props(newpp, P_REF); 875 page_io_lock(newpp); 876 pp = newpp; 877 se = SE_EXCL; 878 } else { 879 VM_STAT_ADD(page_lookup_cnt[19]); 880 mutex_exit(phm); 881 } 882 883 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 884 885 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 886 887 return (pp); 888 } 889 890 /* 891 * Search the hash list for the page representing the 892 * specified [vp, offset] and return it locked. Skip 893 * free pages and pages that cannot be locked as requested. 894 * Used while attempting to kluster pages. 895 */ 896 page_t * 897 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 898 { 899 page_t *pp; 900 kmutex_t *phm; 901 ulong_t index; 902 uint_t locked; 903 904 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 905 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 906 907 index = PAGE_HASH_FUNC(vp, off); 908 PAGE_HASH_SEARCH(index, pp, vp, off); 909 locked = 0; 910 if (pp == NULL) { 911 top: 912 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 913 locked = 1; 914 phm = PAGE_HASH_MUTEX(index); 915 mutex_enter(phm); 916 PAGE_HASH_SEARCH(index, pp, vp, off); 917 } 918 919 if (pp == NULL || PP_ISFREE(pp)) { 920 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 921 pp = NULL; 922 } else { 923 if (!page_trylock(pp, se)) { 924 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 925 pp = NULL; 926 } else { 927 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 928 /* 929 * See the comment in page_lookup() 930 */ 931 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 932 ((u_offset_t)(pp->p_offset) != off)) { 933 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 934 if (locked) { 935 panic("page_lookup_nowait %p", 936 (void *)pp); 937 /*NOTREACHED*/ 938 } 939 page_unlock(pp); 940 goto top; 941 } 942 if (PP_ISFREE(pp)) { 943 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 944 page_unlock(pp); 945 pp = NULL; 946 } 947 } 948 } 949 if (locked) { 950 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 951 mutex_exit(phm); 952 } 953 954 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 955 956 return (pp); 957 } 958 959 /* 960 * Search the hash list for a page with the specified [vp, off] 961 * that is known to exist and is already locked. This routine 962 * is typically used by segment SOFTUNLOCK routines. 963 */ 964 page_t * 965 page_find(vnode_t *vp, u_offset_t off) 966 { 967 page_t *pp; 968 kmutex_t *phm; 969 ulong_t index; 970 971 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 972 VM_STAT_ADD(page_find_cnt); 973 974 index = PAGE_HASH_FUNC(vp, off); 975 phm = PAGE_HASH_MUTEX(index); 976 977 mutex_enter(phm); 978 PAGE_HASH_SEARCH(index, pp, vp, off); 979 mutex_exit(phm); 980 981 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 982 return (pp); 983 } 984 985 /* 986 * Determine whether a page with the specified [vp, off] 987 * currently exists in the system. Obviously this should 988 * only be considered as a hint since nothing prevents the 989 * page from disappearing or appearing immediately after 990 * the return from this routine. Subsequently, we don't 991 * even bother to lock the list. 992 */ 993 page_t * 994 page_exists(vnode_t *vp, u_offset_t off) 995 { 996 page_t *pp; 997 ulong_t index; 998 999 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1000 VM_STAT_ADD(page_exists_cnt); 1001 1002 index = PAGE_HASH_FUNC(vp, off); 1003 PAGE_HASH_SEARCH(index, pp, vp, off); 1004 1005 return (pp); 1006 } 1007 1008 /* 1009 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1010 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1011 * with these pages locked SHARED. If necessary reclaim pages from 1012 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1013 * 1014 * If we fail to lock pages still return 1 if pages exist and contiguous. 1015 * But in this case return value is just a hint. ppa array won't be filled. 1016 * Caller should initialize ppa[0] as NULL to distinguish return value. 1017 * 1018 * Returns 0 if pages don't exist or not physically contiguous. 1019 * 1020 * This routine doesn't work for anonymous(swapfs) pages. 1021 */ 1022 int 1023 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1024 { 1025 pgcnt_t pages; 1026 pfn_t pfn; 1027 page_t *rootpp; 1028 pgcnt_t i; 1029 pgcnt_t j; 1030 u_offset_t save_off = off; 1031 ulong_t index; 1032 kmutex_t *phm; 1033 page_t *pp; 1034 uint_t pszc; 1035 int loopcnt = 0; 1036 1037 ASSERT(szc != 0); 1038 ASSERT(vp != NULL); 1039 ASSERT(!IS_SWAPFSVP(vp)); 1040 ASSERT(!VN_ISKAS(vp)); 1041 1042 again: 1043 if (++loopcnt > 3) { 1044 VM_STAT_ADD(page_exphcontg[0]); 1045 return (0); 1046 } 1047 1048 index = PAGE_HASH_FUNC(vp, off); 1049 phm = PAGE_HASH_MUTEX(index); 1050 1051 mutex_enter(phm); 1052 PAGE_HASH_SEARCH(index, pp, vp, off); 1053 mutex_exit(phm); 1054 1055 VM_STAT_ADD(page_exphcontg[1]); 1056 1057 if (pp == NULL) { 1058 VM_STAT_ADD(page_exphcontg[2]); 1059 return (0); 1060 } 1061 1062 pages = page_get_pagecnt(szc); 1063 rootpp = pp; 1064 pfn = rootpp->p_pagenum; 1065 1066 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1067 VM_STAT_ADD(page_exphcontg[3]); 1068 if (!page_trylock(pp, SE_SHARED)) { 1069 VM_STAT_ADD(page_exphcontg[4]); 1070 return (1); 1071 } 1072 if (pp->p_szc != pszc || pp->p_vnode != vp || 1073 pp->p_offset != off) { 1074 VM_STAT_ADD(page_exphcontg[5]); 1075 page_unlock(pp); 1076 off = save_off; 1077 goto again; 1078 } 1079 /* 1080 * szc was non zero and vnode and offset matched after we 1081 * locked the page it means it can't become free on us. 1082 */ 1083 ASSERT(!PP_ISFREE(pp)); 1084 if (!IS_P2ALIGNED(pfn, pages)) { 1085 page_unlock(pp); 1086 return (0); 1087 } 1088 ppa[0] = pp; 1089 pp++; 1090 off += PAGESIZE; 1091 pfn++; 1092 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1093 if (!page_trylock(pp, SE_SHARED)) { 1094 VM_STAT_ADD(page_exphcontg[6]); 1095 pp--; 1096 while (i-- > 0) { 1097 page_unlock(pp); 1098 pp--; 1099 } 1100 ppa[0] = NULL; 1101 return (1); 1102 } 1103 if (pp->p_szc != pszc) { 1104 VM_STAT_ADD(page_exphcontg[7]); 1105 page_unlock(pp); 1106 pp--; 1107 while (i-- > 0) { 1108 page_unlock(pp); 1109 pp--; 1110 } 1111 ppa[0] = NULL; 1112 off = save_off; 1113 goto again; 1114 } 1115 /* 1116 * szc the same as for previous already locked pages 1117 * with right identity. Since this page had correct 1118 * szc after we locked it can't get freed or destroyed 1119 * and therefore must have the expected identity. 1120 */ 1121 ASSERT(!PP_ISFREE(pp)); 1122 if (pp->p_vnode != vp || 1123 pp->p_offset != off) { 1124 panic("page_exists_physcontig: " 1125 "large page identity doesn't match"); 1126 } 1127 ppa[i] = pp; 1128 ASSERT(pp->p_pagenum == pfn); 1129 } 1130 VM_STAT_ADD(page_exphcontg[8]); 1131 ppa[pages] = NULL; 1132 return (1); 1133 } else if (pszc >= szc) { 1134 VM_STAT_ADD(page_exphcontg[9]); 1135 if (!IS_P2ALIGNED(pfn, pages)) { 1136 return (0); 1137 } 1138 return (1); 1139 } 1140 1141 if (!IS_P2ALIGNED(pfn, pages)) { 1142 VM_STAT_ADD(page_exphcontg[10]); 1143 return (0); 1144 } 1145 1146 if (page_numtomemseg_nolock(pfn) != 1147 page_numtomemseg_nolock(pfn + pages - 1)) { 1148 VM_STAT_ADD(page_exphcontg[11]); 1149 return (0); 1150 } 1151 1152 /* 1153 * We loop up 4 times across pages to promote page size. 1154 * We're extra cautious to promote page size atomically with respect 1155 * to everybody else. But we can probably optimize into 1 loop if 1156 * this becomes an issue. 1157 */ 1158 1159 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1160 ASSERT(pp->p_pagenum == pfn); 1161 if (!page_trylock(pp, SE_EXCL)) { 1162 VM_STAT_ADD(page_exphcontg[12]); 1163 break; 1164 } 1165 if (pp->p_vnode != vp || 1166 pp->p_offset != off) { 1167 VM_STAT_ADD(page_exphcontg[13]); 1168 page_unlock(pp); 1169 break; 1170 } 1171 if (pp->p_szc >= szc) { 1172 ASSERT(i == 0); 1173 page_unlock(pp); 1174 off = save_off; 1175 goto again; 1176 } 1177 } 1178 1179 if (i != pages) { 1180 VM_STAT_ADD(page_exphcontg[14]); 1181 --pp; 1182 while (i-- > 0) { 1183 page_unlock(pp); 1184 --pp; 1185 } 1186 return (0); 1187 } 1188 1189 pp = rootpp; 1190 for (i = 0; i < pages; i++, pp++) { 1191 if (PP_ISFREE(pp)) { 1192 VM_STAT_ADD(page_exphcontg[15]); 1193 ASSERT(!PP_ISAGED(pp)); 1194 ASSERT(pp->p_szc == 0); 1195 if (!page_reclaim(pp, NULL)) { 1196 break; 1197 } 1198 } else { 1199 ASSERT(pp->p_szc < szc); 1200 VM_STAT_ADD(page_exphcontg[16]); 1201 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1202 } 1203 } 1204 if (i < pages) { 1205 VM_STAT_ADD(page_exphcontg[17]); 1206 /* 1207 * page_reclaim failed because we were out of memory. 1208 * drop the rest of the locks and return because this page 1209 * must be already reallocated anyway. 1210 */ 1211 pp = rootpp; 1212 for (j = 0; j < pages; j++, pp++) { 1213 if (j != i) { 1214 page_unlock(pp); 1215 } 1216 } 1217 return (0); 1218 } 1219 1220 off = save_off; 1221 pp = rootpp; 1222 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1223 ASSERT(PAGE_EXCL(pp)); 1224 ASSERT(!PP_ISFREE(pp)); 1225 ASSERT(!hat_page_is_mapped(pp)); 1226 ASSERT(pp->p_vnode == vp); 1227 ASSERT(pp->p_offset == off); 1228 pp->p_szc = szc; 1229 } 1230 pp = rootpp; 1231 for (i = 0; i < pages; i++, pp++) { 1232 if (ppa == NULL) { 1233 page_unlock(pp); 1234 } else { 1235 ppa[i] = pp; 1236 page_downgrade(ppa[i]); 1237 } 1238 } 1239 if (ppa != NULL) { 1240 ppa[pages] = NULL; 1241 } 1242 VM_STAT_ADD(page_exphcontg[18]); 1243 ASSERT(vp->v_pages != NULL); 1244 return (1); 1245 } 1246 1247 /* 1248 * Determine whether a page with the specified [vp, off] 1249 * currently exists in the system and if so return its 1250 * size code. Obviously this should only be considered as 1251 * a hint since nothing prevents the page from disappearing 1252 * or appearing immediately after the return from this routine. 1253 */ 1254 int 1255 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1256 { 1257 page_t *pp; 1258 kmutex_t *phm; 1259 ulong_t index; 1260 int rc = 0; 1261 1262 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1263 ASSERT(szc != NULL); 1264 VM_STAT_ADD(page_exists_forreal_cnt); 1265 1266 index = PAGE_HASH_FUNC(vp, off); 1267 phm = PAGE_HASH_MUTEX(index); 1268 1269 mutex_enter(phm); 1270 PAGE_HASH_SEARCH(index, pp, vp, off); 1271 if (pp != NULL) { 1272 *szc = pp->p_szc; 1273 rc = 1; 1274 } 1275 mutex_exit(phm); 1276 return (rc); 1277 } 1278 1279 /* wakeup threads waiting for pages in page_create_get_something() */ 1280 void 1281 wakeup_pcgs(void) 1282 { 1283 if (!CV_HAS_WAITERS(&pcgs_cv)) 1284 return; 1285 cv_broadcast(&pcgs_cv); 1286 } 1287 1288 /* 1289 * 'freemem' is used all over the kernel as an indication of how many 1290 * pages are free (either on the cache list or on the free page list) 1291 * in the system. In very few places is a really accurate 'freemem' 1292 * needed. To avoid contention of the lock protecting a the 1293 * single freemem, it was spread out into NCPU buckets. Set_freemem 1294 * sets freemem to the total of all NCPU buckets. It is called from 1295 * clock() on each TICK. 1296 */ 1297 void 1298 set_freemem() 1299 { 1300 struct pcf *p; 1301 ulong_t t; 1302 uint_t i; 1303 1304 t = 0; 1305 p = pcf; 1306 for (i = 0; i < PCF_FANOUT; i++) { 1307 t += p->pcf_count; 1308 p++; 1309 } 1310 freemem = t; 1311 1312 /* 1313 * Don't worry about grabbing mutex. It's not that 1314 * critical if we miss a tick or two. This is 1315 * where we wakeup possible delayers in 1316 * page_create_get_something(). 1317 */ 1318 wakeup_pcgs(); 1319 } 1320 1321 ulong_t 1322 get_freemem() 1323 { 1324 struct pcf *p; 1325 ulong_t t; 1326 uint_t i; 1327 1328 t = 0; 1329 p = pcf; 1330 for (i = 0; i < PCF_FANOUT; i++) { 1331 t += p->pcf_count; 1332 p++; 1333 } 1334 /* 1335 * We just calculated it, might as well set it. 1336 */ 1337 freemem = t; 1338 return (t); 1339 } 1340 1341 /* 1342 * Acquire all of the page cache & free (pcf) locks. 1343 */ 1344 void 1345 pcf_acquire_all() 1346 { 1347 struct pcf *p; 1348 uint_t i; 1349 1350 p = pcf; 1351 for (i = 0; i < PCF_FANOUT; i++) { 1352 mutex_enter(&p->pcf_lock); 1353 p++; 1354 } 1355 } 1356 1357 /* 1358 * Release all the pcf_locks. 1359 */ 1360 void 1361 pcf_release_all() 1362 { 1363 struct pcf *p; 1364 uint_t i; 1365 1366 p = pcf; 1367 for (i = 0; i < PCF_FANOUT; i++) { 1368 mutex_exit(&p->pcf_lock); 1369 p++; 1370 } 1371 } 1372 1373 /* 1374 * Inform the VM system that we need some pages freed up. 1375 * Calls must be symmetric, e.g.: 1376 * 1377 * page_needfree(100); 1378 * wait a bit; 1379 * page_needfree(-100); 1380 */ 1381 void 1382 page_needfree(spgcnt_t npages) 1383 { 1384 mutex_enter(&new_freemem_lock); 1385 needfree += npages; 1386 mutex_exit(&new_freemem_lock); 1387 } 1388 1389 /* 1390 * Throttle for page_create(): try to prevent freemem from dropping 1391 * below throttlefree. We can't provide a 100% guarantee because 1392 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1393 * nibble away at the freelist. However, we can block all PG_WAIT 1394 * allocations until memory becomes available. The motivation is 1395 * that several things can fall apart when there's no free memory: 1396 * 1397 * (1) If pageout() needs memory to push a page, the system deadlocks. 1398 * 1399 * (2) By (broken) specification, timeout(9F) can neither fail nor 1400 * block, so it has no choice but to panic the system if it 1401 * cannot allocate a callout structure. 1402 * 1403 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1404 * it panics if it cannot allocate a callback structure. 1405 * 1406 * (4) Untold numbers of third-party drivers have not yet been hardened 1407 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1408 * success and panic the system with a data fault on failure. 1409 * (The long-term solution to this particular problem is to ship 1410 * hostile fault-injecting DEBUG kernels with the DDK.) 1411 * 1412 * It is theoretically impossible to guarantee success of non-blocking 1413 * allocations, but in practice, this throttle is very hard to break. 1414 */ 1415 static int 1416 page_create_throttle(pgcnt_t npages, int flags) 1417 { 1418 ulong_t fm; 1419 uint_t i; 1420 pgcnt_t tf; /* effective value of throttlefree */ 1421 1422 /* 1423 * Never deny pages when: 1424 * - it's a thread that cannot block [NOMEMWAIT()] 1425 * - the allocation cannot block and must not fail 1426 * - the allocation cannot block and is pageout dispensated 1427 */ 1428 if (NOMEMWAIT() || 1429 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1430 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1431 return (1); 1432 1433 /* 1434 * If the allocation can't block, we look favorably upon it 1435 * unless we're below pageout_reserve. In that case we fail 1436 * the allocation because we want to make sure there are a few 1437 * pages available for pageout. 1438 */ 1439 if ((flags & PG_WAIT) == 0) 1440 return (freemem >= npages + pageout_reserve); 1441 1442 /* Calculate the effective throttlefree value */ 1443 tf = throttlefree - 1444 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1445 1446 cv_signal(&proc_pageout->p_cv); 1447 1448 while (freemem < npages + tf) { 1449 pcf_acquire_all(); 1450 mutex_enter(&new_freemem_lock); 1451 fm = 0; 1452 for (i = 0; i < PCF_FANOUT; i++) { 1453 fm += pcf[i].pcf_count; 1454 pcf[i].pcf_wait++; 1455 mutex_exit(&pcf[i].pcf_lock); 1456 } 1457 freemem = fm; 1458 needfree += npages; 1459 freemem_wait++; 1460 cv_wait(&freemem_cv, &new_freemem_lock); 1461 freemem_wait--; 1462 needfree -= npages; 1463 mutex_exit(&new_freemem_lock); 1464 } 1465 return (1); 1466 } 1467 1468 /* 1469 * page_create_wait() is called to either coalesce pages from the 1470 * different pcf buckets or to wait because there simply are not 1471 * enough pages to satisfy the caller's request. 1472 * 1473 * Sadly, this is called from platform/vm/vm_machdep.c 1474 */ 1475 int 1476 page_create_wait(size_t npages, uint_t flags) 1477 { 1478 pgcnt_t total; 1479 uint_t i; 1480 struct pcf *p; 1481 1482 /* 1483 * Wait until there are enough free pages to satisfy our 1484 * entire request. 1485 * We set needfree += npages before prodding pageout, to make sure 1486 * it does real work when npages > lotsfree > freemem. 1487 */ 1488 VM_STAT_ADD(page_create_not_enough); 1489 1490 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1491 checkagain: 1492 if ((flags & PG_NORELOC) && 1493 kcage_freemem < kcage_throttlefree + npages) 1494 (void) kcage_create_throttle(npages, flags); 1495 1496 if (freemem < npages + throttlefree) 1497 if (!page_create_throttle(npages, flags)) 1498 return (0); 1499 1500 /* 1501 * Since page_create_va() looked at every 1502 * bucket, assume we are going to have to wait. 1503 * Get all of the pcf locks. 1504 */ 1505 total = 0; 1506 p = pcf; 1507 for (i = 0; i < PCF_FANOUT; i++) { 1508 mutex_enter(&p->pcf_lock); 1509 total += p->pcf_count; 1510 if (total >= npages) { 1511 /* 1512 * Wow! There are enough pages laying around 1513 * to satisfy the request. Do the accounting, 1514 * drop the locks we acquired, and go back. 1515 * 1516 * freemem is not protected by any lock. So, 1517 * we cannot have any assertion containing 1518 * freemem. 1519 */ 1520 freemem -= npages; 1521 1522 while (p >= pcf) { 1523 if (p->pcf_count <= npages) { 1524 npages -= p->pcf_count; 1525 p->pcf_count = 0; 1526 } else { 1527 p->pcf_count -= (uint_t)npages; 1528 npages = 0; 1529 } 1530 mutex_exit(&p->pcf_lock); 1531 p--; 1532 } 1533 ASSERT(npages == 0); 1534 return (1); 1535 } 1536 p++; 1537 } 1538 1539 /* 1540 * All of the pcf locks are held, there are not enough pages 1541 * to satisfy the request (npages < total). 1542 * Be sure to acquire the new_freemem_lock before dropping 1543 * the pcf locks. This prevents dropping wakeups in page_free(). 1544 * The order is always pcf_lock then new_freemem_lock. 1545 * 1546 * Since we hold all the pcf locks, it is a good time to set freemem. 1547 * 1548 * If the caller does not want to wait, return now. 1549 * Else turn the pageout daemon loose to find something 1550 * and wait till it does. 1551 * 1552 */ 1553 freemem = total; 1554 1555 if ((flags & PG_WAIT) == 0) { 1556 pcf_release_all(); 1557 1558 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1559 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1560 return (0); 1561 } 1562 1563 ASSERT(proc_pageout != NULL); 1564 cv_signal(&proc_pageout->p_cv); 1565 1566 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1567 "page_create_sleep_start: freemem %ld needfree %ld", 1568 freemem, needfree); 1569 1570 /* 1571 * We are going to wait. 1572 * We currently hold all of the pcf_locks, 1573 * get the new_freemem_lock (it protects freemem_wait), 1574 * before dropping the pcf_locks. 1575 */ 1576 mutex_enter(&new_freemem_lock); 1577 1578 p = pcf; 1579 for (i = 0; i < PCF_FANOUT; i++) { 1580 p->pcf_wait++; 1581 mutex_exit(&p->pcf_lock); 1582 p++; 1583 } 1584 1585 needfree += npages; 1586 freemem_wait++; 1587 1588 cv_wait(&freemem_cv, &new_freemem_lock); 1589 1590 freemem_wait--; 1591 needfree -= npages; 1592 1593 mutex_exit(&new_freemem_lock); 1594 1595 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1596 "page_create_sleep_end: freemem %ld needfree %ld", 1597 freemem, needfree); 1598 1599 VM_STAT_ADD(page_create_not_enough_again); 1600 goto checkagain; 1601 } 1602 1603 /* 1604 * A routine to do the opposite of page_create_wait(). 1605 */ 1606 void 1607 page_create_putback(spgcnt_t npages) 1608 { 1609 struct pcf *p; 1610 pgcnt_t lump; 1611 uint_t *which; 1612 1613 /* 1614 * When a contiguous lump is broken up, we have to 1615 * deal with lots of pages (min 64) so lets spread 1616 * the wealth around. 1617 */ 1618 lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 1619 freemem += npages; 1620 1621 for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) { 1622 which = &p->pcf_count; 1623 1624 mutex_enter(&p->pcf_lock); 1625 1626 if (p->pcf_block) { 1627 which = &p->pcf_reserve; 1628 } 1629 1630 if (lump < npages) { 1631 *which += (uint_t)lump; 1632 npages -= lump; 1633 } else { 1634 *which += (uint_t)npages; 1635 npages = 0; 1636 } 1637 1638 if (p->pcf_wait) { 1639 mutex_enter(&new_freemem_lock); 1640 /* 1641 * Check to see if some other thread 1642 * is actually waiting. Another bucket 1643 * may have woken it up by now. If there 1644 * are no waiters, then set our pcf_wait 1645 * count to zero to avoid coming in here 1646 * next time. 1647 */ 1648 if (freemem_wait) { 1649 if (npages > 1) { 1650 cv_broadcast(&freemem_cv); 1651 } else { 1652 cv_signal(&freemem_cv); 1653 } 1654 p->pcf_wait--; 1655 } else { 1656 p->pcf_wait = 0; 1657 } 1658 mutex_exit(&new_freemem_lock); 1659 } 1660 mutex_exit(&p->pcf_lock); 1661 } 1662 ASSERT(npages == 0); 1663 } 1664 1665 /* 1666 * A helper routine for page_create_get_something. 1667 * The indenting got to deep down there. 1668 * Unblock the pcf counters. Any pages freed after 1669 * pcf_block got set are moved to pcf_count and 1670 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1671 */ 1672 static void 1673 pcgs_unblock(void) 1674 { 1675 int i; 1676 struct pcf *p; 1677 1678 /* Update freemem while we're here. */ 1679 freemem = 0; 1680 p = pcf; 1681 for (i = 0; i < PCF_FANOUT; i++) { 1682 mutex_enter(&p->pcf_lock); 1683 ASSERT(p->pcf_count == 0); 1684 p->pcf_count = p->pcf_reserve; 1685 p->pcf_block = 0; 1686 freemem += p->pcf_count; 1687 if (p->pcf_wait) { 1688 mutex_enter(&new_freemem_lock); 1689 if (freemem_wait) { 1690 if (p->pcf_reserve > 1) { 1691 cv_broadcast(&freemem_cv); 1692 p->pcf_wait = 0; 1693 } else { 1694 cv_signal(&freemem_cv); 1695 p->pcf_wait--; 1696 } 1697 } else { 1698 p->pcf_wait = 0; 1699 } 1700 mutex_exit(&new_freemem_lock); 1701 } 1702 p->pcf_reserve = 0; 1703 mutex_exit(&p->pcf_lock); 1704 p++; 1705 } 1706 } 1707 1708 /* 1709 * Called from page_create_va() when both the cache and free lists 1710 * have been checked once. 1711 * 1712 * Either returns a page or panics since the accounting was done 1713 * way before we got here. 1714 * 1715 * We don't come here often, so leave the accounting on permanently. 1716 */ 1717 1718 #define MAX_PCGS 100 1719 1720 #ifdef DEBUG 1721 #define PCGS_TRIES 100 1722 #else /* DEBUG */ 1723 #define PCGS_TRIES 10 1724 #endif /* DEBUG */ 1725 1726 #ifdef VM_STATS 1727 uint_t pcgs_counts[PCGS_TRIES]; 1728 uint_t pcgs_too_many; 1729 uint_t pcgs_entered; 1730 uint_t pcgs_entered_noreloc; 1731 uint_t pcgs_locked; 1732 uint_t pcgs_cagelocked; 1733 #endif /* VM_STATS */ 1734 1735 static page_t * 1736 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1737 caddr_t vaddr, uint_t flags) 1738 { 1739 uint_t count; 1740 page_t *pp; 1741 uint_t locked, i; 1742 struct pcf *p; 1743 lgrp_t *lgrp; 1744 int cagelocked = 0; 1745 1746 VM_STAT_ADD(pcgs_entered); 1747 1748 /* 1749 * Tap any reserve freelists: if we fail now, we'll die 1750 * since the page(s) we're looking for have already been 1751 * accounted for. 1752 */ 1753 flags |= PG_PANIC; 1754 1755 if ((flags & PG_NORELOC) != 0) { 1756 VM_STAT_ADD(pcgs_entered_noreloc); 1757 /* 1758 * Requests for free pages from critical threads 1759 * such as pageout still won't throttle here, but 1760 * we must try again, to give the cageout thread 1761 * another chance to catch up. Since we already 1762 * accounted for the pages, we had better get them 1763 * this time. 1764 * 1765 * N.B. All non-critical threads acquire the pcgs_cagelock 1766 * to serialize access to the freelists. This implements a 1767 * turnstile-type synchornization to avoid starvation of 1768 * critical requests for PG_NORELOC memory by non-critical 1769 * threads: all non-critical threads must acquire a 'ticket' 1770 * before passing through, which entails making sure 1771 * kcage_freemem won't fall below minfree prior to grabbing 1772 * pages from the freelists. 1773 */ 1774 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1775 mutex_enter(&pcgs_cagelock); 1776 cagelocked = 1; 1777 VM_STAT_ADD(pcgs_cagelocked); 1778 } 1779 } 1780 1781 /* 1782 * Time to get serious. 1783 * We failed to get a `correctly colored' page from both the 1784 * free and cache lists. 1785 * We escalate in stage. 1786 * 1787 * First try both lists without worring about color. 1788 * 1789 * Then, grab all page accounting locks (ie. pcf[]) and 1790 * steal any pages that they have and set the pcf_block flag to 1791 * stop deletions from the lists. This will help because 1792 * a page can get added to the free list while we are looking 1793 * at the cache list, then another page could be added to the cache 1794 * list allowing the page on the free list to be removed as we 1795 * move from looking at the cache list to the free list. This 1796 * could happen over and over. We would never find the page 1797 * we have accounted for. 1798 * 1799 * Noreloc pages are a subset of the global (relocatable) page pool. 1800 * They are not tracked separately in the pcf bins, so it is 1801 * impossible to know when doing pcf accounting if the available 1802 * page(s) are noreloc pages or not. When looking for a noreloc page 1803 * it is quite easy to end up here even if the global (relocatable) 1804 * page pool has plenty of free pages but the noreloc pool is empty. 1805 * 1806 * When the noreloc pool is empty (or low), additional noreloc pages 1807 * are created by converting pages from the global page pool. This 1808 * process will stall during pcf accounting if the pcf bins are 1809 * already locked. Such is the case when a noreloc allocation is 1810 * looping here in page_create_get_something waiting for more noreloc 1811 * pages to appear. 1812 * 1813 * Short of adding a new field to the pcf bins to accurately track 1814 * the number of free noreloc pages, we instead do not grab the 1815 * pcgs_lock, do not set the pcf blocks and do not timeout when 1816 * allocating a noreloc page. This allows noreloc allocations to 1817 * loop without blocking global page pool allocations. 1818 * 1819 * NOTE: the behaviour of page_create_get_something has not changed 1820 * for the case of global page pool allocations. 1821 */ 1822 1823 flags &= ~PG_MATCH_COLOR; 1824 locked = 0; 1825 #if defined(__i386) || defined(__amd64) 1826 flags = page_create_update_flags_x86(flags); 1827 #endif 1828 1829 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1830 1831 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1832 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1833 flags, lgrp); 1834 if (pp == NULL) { 1835 pp = page_get_cachelist(vp, off, seg, vaddr, 1836 flags, lgrp); 1837 } 1838 if (pp == NULL) { 1839 /* 1840 * Serialize. Don't fight with other pcgs(). 1841 */ 1842 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1843 mutex_enter(&pcgs_lock); 1844 VM_STAT_ADD(pcgs_locked); 1845 locked = 1; 1846 p = pcf; 1847 for (i = 0; i < PCF_FANOUT; i++) { 1848 mutex_enter(&p->pcf_lock); 1849 ASSERT(p->pcf_block == 0); 1850 p->pcf_block = 1; 1851 p->pcf_reserve = p->pcf_count; 1852 p->pcf_count = 0; 1853 mutex_exit(&p->pcf_lock); 1854 p++; 1855 } 1856 freemem = 0; 1857 } 1858 1859 if (count) { 1860 /* 1861 * Since page_free() puts pages on 1862 * a list then accounts for it, we 1863 * just have to wait for page_free() 1864 * to unlock any page it was working 1865 * with. The page_lock()-page_reclaim() 1866 * path falls in the same boat. 1867 * 1868 * We don't need to check on the 1869 * PG_WAIT flag, we have already 1870 * accounted for the page we are 1871 * looking for in page_create_va(). 1872 * 1873 * We just wait a moment to let any 1874 * locked pages on the lists free up, 1875 * then continue around and try again. 1876 * 1877 * Will be awakened by set_freemem(). 1878 */ 1879 mutex_enter(&pcgs_wait_lock); 1880 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1881 mutex_exit(&pcgs_wait_lock); 1882 } 1883 } else { 1884 #ifdef VM_STATS 1885 if (count >= PCGS_TRIES) { 1886 VM_STAT_ADD(pcgs_too_many); 1887 } else { 1888 VM_STAT_ADD(pcgs_counts[count]); 1889 } 1890 #endif 1891 if (locked) { 1892 pcgs_unblock(); 1893 mutex_exit(&pcgs_lock); 1894 } 1895 if (cagelocked) 1896 mutex_exit(&pcgs_cagelock); 1897 return (pp); 1898 } 1899 } 1900 /* 1901 * we go down holding the pcf locks. 1902 */ 1903 panic("no %spage found %d", 1904 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1905 /*NOTREACHED*/ 1906 } 1907 1908 /* 1909 * Create enough pages for "bytes" worth of data starting at 1910 * "off" in "vp". 1911 * 1912 * Where flag must be one of: 1913 * 1914 * PG_EXCL: Exclusive create (fail if any page already 1915 * exists in the page cache) which does not 1916 * wait for memory to become available. 1917 * 1918 * PG_WAIT: Non-exclusive create which can wait for 1919 * memory to become available. 1920 * 1921 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1922 * (Not Supported) 1923 * 1924 * A doubly linked list of pages is returned to the caller. Each page 1925 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1926 * lock. 1927 * 1928 * Unable to change the parameters to page_create() in a minor release, 1929 * we renamed page_create() to page_create_va(), changed all known calls 1930 * from page_create() to page_create_va(), and created this wrapper. 1931 * 1932 * Upon a major release, we should break compatibility by deleting this 1933 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1934 * 1935 * NOTE: There is a copy of this interface as page_create_io() in 1936 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1937 * there. 1938 */ 1939 page_t * 1940 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1941 { 1942 caddr_t random_vaddr; 1943 struct seg kseg; 1944 1945 #ifdef DEBUG 1946 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1947 (void *)caller()); 1948 #endif 1949 1950 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1951 (uintptr_t)(off >> PAGESHIFT)); 1952 kseg.s_as = &kas; 1953 1954 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1955 } 1956 1957 #ifdef DEBUG 1958 uint32_t pg_alloc_pgs_mtbf = 0; 1959 #endif 1960 1961 /* 1962 * Used for large page support. It will attempt to allocate 1963 * a large page(s) off the freelist. 1964 * 1965 * Returns non zero on failure. 1966 */ 1967 int 1968 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1969 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags) 1970 { 1971 pgcnt_t npgs, curnpgs, totpgs; 1972 size_t pgsz; 1973 page_t *pplist = NULL, *pp; 1974 int err = 0; 1975 lgrp_t *lgrp; 1976 1977 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 1978 ASSERT(pgflags == 0 || pgflags == PG_LOCAL); 1979 1980 VM_STAT_ADD(alloc_pages[0]); 1981 1982 #ifdef DEBUG 1983 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 1984 return (ENOMEM); 1985 } 1986 #endif 1987 1988 pgsz = page_get_pagesize(szc); 1989 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 1990 1991 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 1992 /* 1993 * One must be NULL but not both. 1994 * And one must be non NULL but not both. 1995 */ 1996 ASSERT(basepp != NULL || ppa != NULL); 1997 ASSERT(basepp == NULL || ppa == NULL); 1998 1999 (void) page_create_wait(npgs, PG_WAIT); 2000 2001 while (npgs && szc) { 2002 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2003 if (pgflags == PG_LOCAL) { 2004 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2005 pgflags, lgrp); 2006 if (pp == NULL) { 2007 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2008 0, lgrp); 2009 } 2010 } else { 2011 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2012 0, lgrp); 2013 } 2014 if (pp != NULL) { 2015 VM_STAT_ADD(alloc_pages[1]); 2016 page_list_concat(&pplist, &pp); 2017 ASSERT(npgs >= curnpgs); 2018 npgs -= curnpgs; 2019 } else if (anypgsz) { 2020 VM_STAT_ADD(alloc_pages[2]); 2021 szc--; 2022 pgsz = page_get_pagesize(szc); 2023 curnpgs = pgsz >> PAGESHIFT; 2024 } else { 2025 VM_STAT_ADD(alloc_pages[3]); 2026 ASSERT(npgs == totpgs); 2027 page_create_putback(npgs); 2028 return (ENOMEM); 2029 } 2030 } 2031 if (szc == 0) { 2032 VM_STAT_ADD(alloc_pages[4]); 2033 ASSERT(npgs != 0); 2034 page_create_putback(npgs); 2035 err = ENOMEM; 2036 } else if (basepp != NULL) { 2037 ASSERT(npgs == 0); 2038 ASSERT(ppa == NULL); 2039 *basepp = pplist; 2040 } 2041 2042 npgs = totpgs - npgs; 2043 pp = pplist; 2044 2045 /* 2046 * Clear the free and age bits. Also if we were passed in a ppa then 2047 * fill it in with all the constituent pages from the large page. But 2048 * if we failed to allocate all the pages just free what we got. 2049 */ 2050 while (npgs != 0) { 2051 ASSERT(PP_ISFREE(pp)); 2052 ASSERT(PP_ISAGED(pp)); 2053 if (ppa != NULL || err != 0) { 2054 if (err == 0) { 2055 VM_STAT_ADD(alloc_pages[5]); 2056 PP_CLRFREE(pp); 2057 PP_CLRAGED(pp); 2058 page_sub(&pplist, pp); 2059 *ppa++ = pp; 2060 npgs--; 2061 } else { 2062 VM_STAT_ADD(alloc_pages[6]); 2063 ASSERT(pp->p_szc != 0); 2064 curnpgs = page_get_pagecnt(pp->p_szc); 2065 page_list_break(&pp, &pplist, curnpgs); 2066 page_list_add_pages(pp, 0); 2067 page_create_putback(curnpgs); 2068 ASSERT(npgs >= curnpgs); 2069 npgs -= curnpgs; 2070 } 2071 pp = pplist; 2072 } else { 2073 VM_STAT_ADD(alloc_pages[7]); 2074 PP_CLRFREE(pp); 2075 PP_CLRAGED(pp); 2076 pp = pp->p_next; 2077 npgs--; 2078 } 2079 } 2080 return (err); 2081 } 2082 2083 /* 2084 * Get a single large page off of the freelists, and set it up for use. 2085 * Number of bytes requested must be a supported page size. 2086 * 2087 * Note that this call may fail even if there is sufficient 2088 * memory available or PG_WAIT is set, so the caller must 2089 * be willing to fallback on page_create_va(), block and retry, 2090 * or fail the requester. 2091 */ 2092 page_t * 2093 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2094 struct seg *seg, caddr_t vaddr, void *arg) 2095 { 2096 pgcnt_t npages, pcftotal; 2097 page_t *pp; 2098 page_t *rootpp; 2099 lgrp_t *lgrp; 2100 uint_t enough; 2101 uint_t pcf_index; 2102 uint_t i; 2103 struct pcf *p; 2104 struct pcf *q; 2105 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2106 2107 ASSERT(vp != NULL); 2108 2109 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2110 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2111 /* but no others */ 2112 2113 ASSERT((flags & PG_EXCL) == PG_EXCL); 2114 2115 npages = btop(bytes); 2116 2117 if (!kcage_on || panicstr) { 2118 /* 2119 * Cage is OFF, or we are single threaded in 2120 * panic, so make everything a RELOC request. 2121 */ 2122 flags &= ~PG_NORELOC; 2123 } 2124 2125 /* 2126 * Make sure there's adequate physical memory available. 2127 * Note: PG_WAIT is ignored here. 2128 */ 2129 if (freemem <= throttlefree + npages) { 2130 VM_STAT_ADD(page_create_large_cnt[1]); 2131 return (NULL); 2132 } 2133 2134 /* 2135 * If cage is on, dampen draw from cage when available 2136 * cage space is low. 2137 */ 2138 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2139 kcage_freemem < kcage_throttlefree + npages) { 2140 2141 /* 2142 * The cage is on, the caller wants PG_NORELOC 2143 * pages and available cage memory is very low. 2144 * Call kcage_create_throttle() to attempt to 2145 * control demand on the cage. 2146 */ 2147 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2148 VM_STAT_ADD(page_create_large_cnt[2]); 2149 return (NULL); 2150 } 2151 } 2152 2153 enough = 0; 2154 pcf_index = PCF_INDEX(); 2155 p = &pcf[pcf_index]; 2156 q = &pcf[PCF_FANOUT]; 2157 for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2158 if (p->pcf_count > npages) { 2159 /* 2160 * a good one to try. 2161 */ 2162 mutex_enter(&p->pcf_lock); 2163 if (p->pcf_count > npages) { 2164 p->pcf_count -= (uint_t)npages; 2165 /* 2166 * freemem is not protected by any lock. 2167 * Thus, we cannot have any assertion 2168 * containing freemem here. 2169 */ 2170 freemem -= npages; 2171 enough = 1; 2172 mutex_exit(&p->pcf_lock); 2173 break; 2174 } 2175 mutex_exit(&p->pcf_lock); 2176 } 2177 pcftotal += p->pcf_count; 2178 p++; 2179 if (p >= q) { 2180 p = pcf; 2181 } 2182 } 2183 2184 if (!enough) { 2185 /* If there isn't enough memory available, give up. */ 2186 if (pcftotal < npages) { 2187 VM_STAT_ADD(page_create_large_cnt[3]); 2188 return (NULL); 2189 } 2190 2191 /* try to collect pages from several pcf bins */ 2192 for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 2193 mutex_enter(&p->pcf_lock); 2194 pcftotal += p->pcf_count; 2195 if (pcftotal >= npages) { 2196 /* 2197 * Wow! There are enough pages laying around 2198 * to satisfy the request. Do the accounting, 2199 * drop the locks we acquired, and go back. 2200 * 2201 * freemem is not protected by any lock. So, 2202 * we cannot have any assertion containing 2203 * freemem. 2204 */ 2205 pgcnt_t tpages = npages; 2206 freemem -= npages; 2207 while (p >= pcf) { 2208 if (p->pcf_count <= tpages) { 2209 tpages -= p->pcf_count; 2210 p->pcf_count = 0; 2211 } else { 2212 p->pcf_count -= (uint_t)tpages; 2213 tpages = 0; 2214 } 2215 mutex_exit(&p->pcf_lock); 2216 p--; 2217 } 2218 ASSERT(tpages == 0); 2219 break; 2220 } 2221 p++; 2222 } 2223 if (i == PCF_FANOUT) { 2224 /* failed to collect pages - release the locks */ 2225 while (--p >= pcf) { 2226 mutex_exit(&p->pcf_lock); 2227 } 2228 VM_STAT_ADD(page_create_large_cnt[4]); 2229 return (NULL); 2230 } 2231 } 2232 2233 /* 2234 * This is where this function behaves fundamentally differently 2235 * than page_create_va(); since we're intending to map the page 2236 * with a single TTE, we have to get it as a physically contiguous 2237 * hardware pagesize chunk. If we can't, we fail. 2238 */ 2239 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2240 LGRP_EXISTS(lgrp_table[*lgrpid])) 2241 lgrp = lgrp_table[*lgrpid]; 2242 else 2243 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2244 2245 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2246 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2247 page_create_putback(npages); 2248 VM_STAT_ADD(page_create_large_cnt[5]); 2249 return (NULL); 2250 } 2251 2252 /* 2253 * if we got the page with the wrong mtype give it back this is a 2254 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2255 * inside "if" and the workaround becomes just a nop 2256 */ 2257 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2258 page_list_add_pages(rootpp, 0); 2259 page_create_putback(npages); 2260 VM_STAT_ADD(page_create_large_cnt[6]); 2261 return (NULL); 2262 } 2263 2264 /* 2265 * If satisfying this request has left us with too little 2266 * memory, start the wheels turning to get some back. The 2267 * first clause of the test prevents waking up the pageout 2268 * daemon in situations where it would decide that there's 2269 * nothing to do. 2270 */ 2271 if (nscan < desscan && freemem < minfree) { 2272 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2273 "pageout_cv_signal:freemem %ld", freemem); 2274 cv_signal(&proc_pageout->p_cv); 2275 } 2276 2277 pp = rootpp; 2278 while (npages--) { 2279 ASSERT(PAGE_EXCL(pp)); 2280 ASSERT(pp->p_vnode == NULL); 2281 ASSERT(!hat_page_is_mapped(pp)); 2282 PP_CLRFREE(pp); 2283 PP_CLRAGED(pp); 2284 if (!page_hashin(pp, vp, off, NULL)) 2285 panic("page_create_large: hashin failed: page %p", 2286 (void *)pp); 2287 page_io_lock(pp); 2288 off += PAGESIZE; 2289 pp = pp->p_next; 2290 } 2291 2292 VM_STAT_ADD(page_create_large_cnt[0]); 2293 return (rootpp); 2294 } 2295 2296 page_t * 2297 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2298 struct seg *seg, caddr_t vaddr) 2299 { 2300 page_t *plist = NULL; 2301 pgcnt_t npages; 2302 pgcnt_t found_on_free = 0; 2303 pgcnt_t pages_req; 2304 page_t *npp = NULL; 2305 uint_t enough; 2306 uint_t i; 2307 uint_t pcf_index; 2308 struct pcf *p; 2309 struct pcf *q; 2310 lgrp_t *lgrp; 2311 2312 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2313 "page_create_start:vp %p off %llx bytes %lu flags %x", 2314 vp, off, bytes, flags); 2315 2316 ASSERT(bytes != 0 && vp != NULL); 2317 2318 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2319 panic("page_create: invalid flags"); 2320 /*NOTREACHED*/ 2321 } 2322 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2323 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2324 /* but no others */ 2325 2326 pages_req = npages = btopr(bytes); 2327 /* 2328 * Try to see whether request is too large to *ever* be 2329 * satisfied, in order to prevent deadlock. We arbitrarily 2330 * decide to limit maximum size requests to max_page_get. 2331 */ 2332 if (npages >= max_page_get) { 2333 if ((flags & PG_WAIT) == 0) { 2334 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2335 "page_create_toobig:vp %p off %llx npages " 2336 "%lu max_page_get %lu", 2337 vp, off, npages, max_page_get); 2338 return (NULL); 2339 } else { 2340 cmn_err(CE_WARN, 2341 "Request for too much kernel memory " 2342 "(%lu bytes), will hang forever", bytes); 2343 for (;;) 2344 delay(1000000000); 2345 } 2346 } 2347 2348 if (!kcage_on || panicstr) { 2349 /* 2350 * Cage is OFF, or we are single threaded in 2351 * panic, so make everything a RELOC request. 2352 */ 2353 flags &= ~PG_NORELOC; 2354 } 2355 2356 if (freemem <= throttlefree + npages) 2357 if (!page_create_throttle(npages, flags)) 2358 return (NULL); 2359 2360 /* 2361 * If cage is on, dampen draw from cage when available 2362 * cage space is low. 2363 */ 2364 if ((flags & PG_NORELOC) && 2365 kcage_freemem < kcage_throttlefree + npages) { 2366 2367 /* 2368 * The cage is on, the caller wants PG_NORELOC 2369 * pages and available cage memory is very low. 2370 * Call kcage_create_throttle() to attempt to 2371 * control demand on the cage. 2372 */ 2373 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2374 return (NULL); 2375 } 2376 2377 VM_STAT_ADD(page_create_cnt[0]); 2378 2379 enough = 0; 2380 pcf_index = PCF_INDEX(); 2381 2382 p = &pcf[pcf_index]; 2383 q = &pcf[PCF_FANOUT]; 2384 for (i = 0; i < PCF_FANOUT; i++) { 2385 if (p->pcf_count > npages) { 2386 /* 2387 * a good one to try. 2388 */ 2389 mutex_enter(&p->pcf_lock); 2390 if (p->pcf_count > npages) { 2391 p->pcf_count -= (uint_t)npages; 2392 /* 2393 * freemem is not protected by any lock. 2394 * Thus, we cannot have any assertion 2395 * containing freemem here. 2396 */ 2397 freemem -= npages; 2398 enough = 1; 2399 mutex_exit(&p->pcf_lock); 2400 break; 2401 } 2402 mutex_exit(&p->pcf_lock); 2403 } 2404 p++; 2405 if (p >= q) { 2406 p = pcf; 2407 } 2408 } 2409 2410 if (!enough) { 2411 /* 2412 * Have to look harder. If npages is greater than 2413 * one, then we might have to coalesce the counters. 2414 * 2415 * Go wait. We come back having accounted 2416 * for the memory. 2417 */ 2418 VM_STAT_ADD(page_create_cnt[1]); 2419 if (!page_create_wait(npages, flags)) { 2420 VM_STAT_ADD(page_create_cnt[2]); 2421 return (NULL); 2422 } 2423 } 2424 2425 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2426 "page_create_success:vp %p off %llx", vp, off); 2427 2428 /* 2429 * If satisfying this request has left us with too little 2430 * memory, start the wheels turning to get some back. The 2431 * first clause of the test prevents waking up the pageout 2432 * daemon in situations where it would decide that there's 2433 * nothing to do. 2434 */ 2435 if (nscan < desscan && freemem < minfree) { 2436 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2437 "pageout_cv_signal:freemem %ld", freemem); 2438 cv_signal(&proc_pageout->p_cv); 2439 } 2440 2441 /* 2442 * Loop around collecting the requested number of pages. 2443 * Most of the time, we have to `create' a new page. With 2444 * this in mind, pull the page off the free list before 2445 * getting the hash lock. This will minimize the hash 2446 * lock hold time, nesting, and the like. If it turns 2447 * out we don't need the page, we put it back at the end. 2448 */ 2449 while (npages--) { 2450 page_t *pp; 2451 kmutex_t *phm = NULL; 2452 ulong_t index; 2453 2454 index = PAGE_HASH_FUNC(vp, off); 2455 top: 2456 ASSERT(phm == NULL); 2457 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2458 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2459 2460 if (npp == NULL) { 2461 /* 2462 * Try to get a page from the freelist (ie, 2463 * a page with no [vp, off] tag). If that 2464 * fails, use the cachelist. 2465 * 2466 * During the first attempt at both the free 2467 * and cache lists we try for the correct color. 2468 */ 2469 /* 2470 * XXXX-how do we deal with virtual indexed 2471 * caches and and colors? 2472 */ 2473 VM_STAT_ADD(page_create_cnt[4]); 2474 /* 2475 * Get lgroup to allocate next page of shared memory 2476 * from and use it to specify where to allocate 2477 * the physical memory 2478 */ 2479 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2480 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2481 flags | PG_MATCH_COLOR, lgrp); 2482 if (npp == NULL) { 2483 npp = page_get_cachelist(vp, off, seg, 2484 vaddr, flags | PG_MATCH_COLOR, lgrp); 2485 if (npp == NULL) { 2486 npp = page_create_get_something(vp, 2487 off, seg, vaddr, 2488 flags & ~PG_MATCH_COLOR); 2489 } 2490 2491 if (PP_ISAGED(npp) == 0) { 2492 /* 2493 * Since this page came from the 2494 * cachelist, we must destroy the 2495 * old vnode association. 2496 */ 2497 page_hashout(npp, NULL); 2498 } 2499 } 2500 } 2501 2502 /* 2503 * We own this page! 2504 */ 2505 ASSERT(PAGE_EXCL(npp)); 2506 ASSERT(npp->p_vnode == NULL); 2507 ASSERT(!hat_page_is_mapped(npp)); 2508 PP_CLRFREE(npp); 2509 PP_CLRAGED(npp); 2510 2511 /* 2512 * Here we have a page in our hot little mits and are 2513 * just waiting to stuff it on the appropriate lists. 2514 * Get the mutex and check to see if it really does 2515 * not exist. 2516 */ 2517 phm = PAGE_HASH_MUTEX(index); 2518 mutex_enter(phm); 2519 PAGE_HASH_SEARCH(index, pp, vp, off); 2520 if (pp == NULL) { 2521 VM_STAT_ADD(page_create_new); 2522 pp = npp; 2523 npp = NULL; 2524 if (!page_hashin(pp, vp, off, phm)) { 2525 /* 2526 * Since we hold the page hash mutex and 2527 * just searched for this page, page_hashin 2528 * had better not fail. If it does, that 2529 * means somethread did not follow the 2530 * page hash mutex rules. Panic now and 2531 * get it over with. As usual, go down 2532 * holding all the locks. 2533 */ 2534 ASSERT(MUTEX_HELD(phm)); 2535 panic("page_create: " 2536 "hashin failed %p %p %llx %p", 2537 (void *)pp, (void *)vp, off, (void *)phm); 2538 /*NOTREACHED*/ 2539 } 2540 ASSERT(MUTEX_HELD(phm)); 2541 mutex_exit(phm); 2542 phm = NULL; 2543 2544 /* 2545 * Hat layer locking need not be done to set 2546 * the following bits since the page is not hashed 2547 * and was on the free list (i.e., had no mappings). 2548 * 2549 * Set the reference bit to protect 2550 * against immediate pageout 2551 * 2552 * XXXmh modify freelist code to set reference 2553 * bit so we don't have to do it here. 2554 */ 2555 page_set_props(pp, P_REF); 2556 found_on_free++; 2557 } else { 2558 VM_STAT_ADD(page_create_exists); 2559 if (flags & PG_EXCL) { 2560 /* 2561 * Found an existing page, and the caller 2562 * wanted all new pages. Undo all of the work 2563 * we have done. 2564 */ 2565 mutex_exit(phm); 2566 phm = NULL; 2567 while (plist != NULL) { 2568 pp = plist; 2569 page_sub(&plist, pp); 2570 page_io_unlock(pp); 2571 /* large pages should not end up here */ 2572 ASSERT(pp->p_szc == 0); 2573 /*LINTED: constant in conditional ctx*/ 2574 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2575 } 2576 VM_STAT_ADD(page_create_found_one); 2577 goto fail; 2578 } 2579 ASSERT(flags & PG_WAIT); 2580 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2581 /* 2582 * Start all over again if we blocked trying 2583 * to lock the page. 2584 */ 2585 mutex_exit(phm); 2586 VM_STAT_ADD(page_create_page_lock_failed); 2587 phm = NULL; 2588 goto top; 2589 } 2590 mutex_exit(phm); 2591 phm = NULL; 2592 2593 if (PP_ISFREE(pp)) { 2594 ASSERT(PP_ISAGED(pp) == 0); 2595 VM_STAT_ADD(pagecnt.pc_get_cache); 2596 page_list_sub(pp, PG_CACHE_LIST); 2597 PP_CLRFREE(pp); 2598 found_on_free++; 2599 } 2600 } 2601 2602 /* 2603 * Got a page! It is locked. Acquire the i/o 2604 * lock since we are going to use the p_next and 2605 * p_prev fields to link the requested pages together. 2606 */ 2607 page_io_lock(pp); 2608 page_add(&plist, pp); 2609 plist = plist->p_next; 2610 off += PAGESIZE; 2611 vaddr += PAGESIZE; 2612 } 2613 2614 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2615 fail: 2616 if (npp != NULL) { 2617 /* 2618 * Did not need this page after all. 2619 * Put it back on the free list. 2620 */ 2621 VM_STAT_ADD(page_create_putbacks); 2622 PP_SETFREE(npp); 2623 PP_SETAGED(npp); 2624 npp->p_offset = (u_offset_t)-1; 2625 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2626 page_unlock(npp); 2627 2628 } 2629 2630 ASSERT(pages_req >= found_on_free); 2631 2632 { 2633 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2634 2635 if (overshoot) { 2636 VM_STAT_ADD(page_create_overshoot); 2637 p = &pcf[pcf_index]; 2638 mutex_enter(&p->pcf_lock); 2639 if (p->pcf_block) { 2640 p->pcf_reserve += overshoot; 2641 } else { 2642 p->pcf_count += overshoot; 2643 if (p->pcf_wait) { 2644 mutex_enter(&new_freemem_lock); 2645 if (freemem_wait) { 2646 cv_signal(&freemem_cv); 2647 p->pcf_wait--; 2648 } else { 2649 p->pcf_wait = 0; 2650 } 2651 mutex_exit(&new_freemem_lock); 2652 } 2653 } 2654 mutex_exit(&p->pcf_lock); 2655 /* freemem is approximate, so this test OK */ 2656 if (!p->pcf_block) 2657 freemem += overshoot; 2658 } 2659 } 2660 2661 return (plist); 2662 } 2663 2664 /* 2665 * One or more constituent pages of this large page has been marked 2666 * toxic. Simply demote the large page to PAGESIZE pages and let 2667 * page_free() handle it. This routine should only be called by 2668 * large page free routines (page_free_pages() and page_destroy_pages(). 2669 * All pages are locked SE_EXCL and have already been marked free. 2670 */ 2671 static void 2672 page_free_toxic_pages(page_t *rootpp) 2673 { 2674 page_t *tpp; 2675 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2676 uint_t szc = rootpp->p_szc; 2677 2678 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2679 ASSERT(tpp->p_szc == szc); 2680 ASSERT((PAGE_EXCL(tpp) && 2681 !page_iolock_assert(tpp)) || panicstr); 2682 tpp->p_szc = 0; 2683 } 2684 2685 while (rootpp != NULL) { 2686 tpp = rootpp; 2687 page_sub(&rootpp, tpp); 2688 ASSERT(PP_ISFREE(tpp)); 2689 PP_CLRFREE(tpp); 2690 page_free(tpp, 1); 2691 } 2692 } 2693 2694 /* 2695 * Put page on the "free" list. 2696 * The free list is really two lists maintained by 2697 * the PSM of whatever machine we happen to be on. 2698 */ 2699 void 2700 page_free(page_t *pp, int dontneed) 2701 { 2702 struct pcf *p; 2703 uint_t pcf_index; 2704 2705 ASSERT((PAGE_EXCL(pp) && 2706 !page_iolock_assert(pp)) || panicstr); 2707 2708 if (PP_ISFREE(pp)) { 2709 panic("page_free: page %p is free", (void *)pp); 2710 } 2711 2712 if (pp->p_szc != 0) { 2713 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2714 PP_ISKAS(pp)) { 2715 panic("page_free: anon or kernel " 2716 "or no vnode large page %p", (void *)pp); 2717 } 2718 page_demote_vp_pages(pp); 2719 ASSERT(pp->p_szc == 0); 2720 } 2721 2722 /* 2723 * The page_struct_lock need not be acquired to examine these 2724 * fields since the page has an "exclusive" lock. 2725 */ 2726 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 2727 pp->p_slckcnt != 0) { 2728 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 2729 "slckcnt = %d", pp, page_pptonum(pp), pp->p_lckcnt, 2730 pp->p_cowcnt, pp->p_slckcnt); 2731 /*NOTREACHED*/ 2732 } 2733 2734 ASSERT(!hat_page_getshare(pp)); 2735 2736 PP_SETFREE(pp); 2737 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2738 !hat_ismod(pp)); 2739 page_clr_all_props(pp); 2740 ASSERT(!hat_page_getshare(pp)); 2741 2742 /* 2743 * Now we add the page to the head of the free list. 2744 * But if this page is associated with a paged vnode 2745 * then we adjust the head forward so that the page is 2746 * effectively at the end of the list. 2747 */ 2748 if (pp->p_vnode == NULL) { 2749 /* 2750 * Page has no identity, put it on the free list. 2751 */ 2752 PP_SETAGED(pp); 2753 pp->p_offset = (u_offset_t)-1; 2754 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2755 VM_STAT_ADD(pagecnt.pc_free_free); 2756 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2757 "page_free_free:pp %p", pp); 2758 } else { 2759 PP_CLRAGED(pp); 2760 2761 if (!dontneed || nopageage) { 2762 /* move it to the tail of the list */ 2763 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2764 2765 VM_STAT_ADD(pagecnt.pc_free_cache); 2766 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2767 "page_free_cache_tail:pp %p", pp); 2768 } else { 2769 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2770 2771 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2772 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2773 "page_free_cache_head:pp %p", pp); 2774 } 2775 } 2776 page_unlock(pp); 2777 2778 /* 2779 * Now do the `freemem' accounting. 2780 */ 2781 pcf_index = PCF_INDEX(); 2782 p = &pcf[pcf_index]; 2783 2784 mutex_enter(&p->pcf_lock); 2785 if (p->pcf_block) { 2786 p->pcf_reserve += 1; 2787 } else { 2788 p->pcf_count += 1; 2789 if (p->pcf_wait) { 2790 mutex_enter(&new_freemem_lock); 2791 /* 2792 * Check to see if some other thread 2793 * is actually waiting. Another bucket 2794 * may have woken it up by now. If there 2795 * are no waiters, then set our pcf_wait 2796 * count to zero to avoid coming in here 2797 * next time. Also, since only one page 2798 * was put on the free list, just wake 2799 * up one waiter. 2800 */ 2801 if (freemem_wait) { 2802 cv_signal(&freemem_cv); 2803 p->pcf_wait--; 2804 } else { 2805 p->pcf_wait = 0; 2806 } 2807 mutex_exit(&new_freemem_lock); 2808 } 2809 } 2810 mutex_exit(&p->pcf_lock); 2811 2812 /* freemem is approximate, so this test OK */ 2813 if (!p->pcf_block) 2814 freemem += 1; 2815 } 2816 2817 /* 2818 * Put page on the "free" list during intial startup. 2819 * This happens during initial single threaded execution. 2820 */ 2821 void 2822 page_free_at_startup(page_t *pp) 2823 { 2824 struct pcf *p; 2825 uint_t pcf_index; 2826 2827 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2828 VM_STAT_ADD(pagecnt.pc_free_free); 2829 2830 /* 2831 * Now do the `freemem' accounting. 2832 */ 2833 pcf_index = PCF_INDEX(); 2834 p = &pcf[pcf_index]; 2835 2836 ASSERT(p->pcf_block == 0); 2837 ASSERT(p->pcf_wait == 0); 2838 p->pcf_count += 1; 2839 2840 /* freemem is approximate, so this is OK */ 2841 freemem += 1; 2842 } 2843 2844 void 2845 page_free_pages(page_t *pp) 2846 { 2847 page_t *tpp, *rootpp = NULL; 2848 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2849 pgcnt_t i; 2850 uint_t szc = pp->p_szc; 2851 2852 VM_STAT_ADD(pagecnt.pc_free_pages); 2853 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2854 "page_free_free:pp %p", pp); 2855 2856 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2857 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2858 panic("page_free_pages: not root page %p", (void *)pp); 2859 /*NOTREACHED*/ 2860 } 2861 2862 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2863 ASSERT((PAGE_EXCL(tpp) && 2864 !page_iolock_assert(tpp)) || panicstr); 2865 if (PP_ISFREE(tpp)) { 2866 panic("page_free_pages: page %p is free", (void *)tpp); 2867 /*NOTREACHED*/ 2868 } 2869 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2870 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 2871 panic("page_free_pages %p", (void *)tpp); 2872 /*NOTREACHED*/ 2873 } 2874 2875 ASSERT(!hat_page_getshare(tpp)); 2876 ASSERT(tpp->p_vnode == NULL); 2877 ASSERT(tpp->p_szc == szc); 2878 2879 PP_SETFREE(tpp); 2880 page_clr_all_props(tpp); 2881 PP_SETAGED(tpp); 2882 tpp->p_offset = (u_offset_t)-1; 2883 ASSERT(tpp->p_next == tpp); 2884 ASSERT(tpp->p_prev == tpp); 2885 page_list_concat(&rootpp, &tpp); 2886 } 2887 ASSERT(rootpp == pp); 2888 2889 page_list_add_pages(rootpp, 0); 2890 page_create_putback(pgcnt); 2891 } 2892 2893 int free_pages = 1; 2894 2895 /* 2896 * This routine attempts to return pages to the cachelist via page_release(). 2897 * It does not *have* to be successful in all cases, since the pageout scanner 2898 * will catch any pages it misses. It does need to be fast and not introduce 2899 * too much overhead. 2900 * 2901 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2902 * don't lock and retry. This is ok, since the page scanner will eventually 2903 * find any page we miss in free_vp_pages(). 2904 */ 2905 void 2906 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2907 { 2908 page_t *pp; 2909 u_offset_t eoff; 2910 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2911 2912 eoff = off + len; 2913 2914 if (free_pages == 0) 2915 return; 2916 if (swap_in_range(vp, off, len)) 2917 return; 2918 2919 for (; off < eoff; off += PAGESIZE) { 2920 2921 /* 2922 * find the page using a fast, but inexact search. It'll be OK 2923 * if a few pages slip through the cracks here. 2924 */ 2925 pp = page_exists(vp, off); 2926 2927 /* 2928 * If we didn't find the page (it may not exist), the page 2929 * is free, looks still in use (shared), or we can't lock it, 2930 * just give up. 2931 */ 2932 if (pp == NULL || 2933 PP_ISFREE(pp) || 2934 page_share_cnt(pp) > 0 || 2935 !page_trylock(pp, SE_EXCL)) 2936 continue; 2937 2938 /* 2939 * Once we have locked pp, verify that it's still the 2940 * correct page and not already free 2941 */ 2942 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2943 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2944 page_unlock(pp); 2945 continue; 2946 } 2947 2948 /* 2949 * try to release the page... 2950 */ 2951 (void) page_release(pp, 1); 2952 } 2953 } 2954 2955 /* 2956 * Reclaim the given page from the free list. 2957 * If pp is part of a large pages, only the given constituent page is reclaimed 2958 * and the large page it belonged to will be demoted. This can only happen 2959 * if the page is not on the cachelist. 2960 * 2961 * Returns 1 on success or 0 on failure. 2962 * 2963 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2964 * If `lock' is non-null, it will be dropped and re-acquired if 2965 * the routine must wait while freemem is 0. 2966 * 2967 * As it turns out, boot_getpages() does this. It picks a page, 2968 * based on where OBP mapped in some address, gets its pfn, searches 2969 * the memsegs, locks the page, then pulls it off the free list! 2970 */ 2971 int 2972 page_reclaim(page_t *pp, kmutex_t *lock) 2973 { 2974 struct pcf *p; 2975 uint_t pcf_index; 2976 struct cpu *cpup; 2977 int enough; 2978 uint_t i; 2979 2980 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2981 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2982 2983 /* 2984 * If `freemem' is 0, we cannot reclaim this page from the 2985 * freelist, so release every lock we might hold: the page, 2986 * and the `lock' before blocking. 2987 * 2988 * The only way `freemem' can become 0 while there are pages 2989 * marked free (have their p->p_free bit set) is when the 2990 * system is low on memory and doing a page_create(). In 2991 * order to guarantee that once page_create() starts acquiring 2992 * pages it will be able to get all that it needs since `freemem' 2993 * was decreased by the requested amount. So, we need to release 2994 * this page, and let page_create() have it. 2995 * 2996 * Since `freemem' being zero is not supposed to happen, just 2997 * use the usual hash stuff as a starting point. If that bucket 2998 * is empty, then assume the worst, and start at the beginning 2999 * of the pcf array. If we always start at the beginning 3000 * when acquiring more than one pcf lock, there won't be any 3001 * deadlock problems. 3002 */ 3003 3004 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 3005 3006 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) { 3007 pcf_acquire_all(); 3008 goto page_reclaim_nomem; 3009 } 3010 3011 enough = 0; 3012 pcf_index = PCF_INDEX(); 3013 p = &pcf[pcf_index]; 3014 mutex_enter(&p->pcf_lock); 3015 if (p->pcf_count >= 1) { 3016 enough = 1; 3017 p->pcf_count--; 3018 } 3019 mutex_exit(&p->pcf_lock); 3020 3021 if (!enough) { 3022 VM_STAT_ADD(page_reclaim_zero); 3023 /* 3024 * Check again. Its possible that some other thread 3025 * could have been right behind us, and added one 3026 * to a list somewhere. Acquire each of the pcf locks 3027 * until we find a page. 3028 */ 3029 p = pcf; 3030 for (i = 0; i < PCF_FANOUT; i++) { 3031 mutex_enter(&p->pcf_lock); 3032 if (p->pcf_count >= 1) { 3033 p->pcf_count -= 1; 3034 enough = 1; 3035 break; 3036 } 3037 p++; 3038 } 3039 3040 if (!enough) { 3041 page_reclaim_nomem: 3042 /* 3043 * We really can't have page `pp'. 3044 * Time for the no-memory dance with 3045 * page_free(). This is just like 3046 * page_create_wait(). Plus the added 3047 * attraction of releasing whatever mutex 3048 * we held when we were called with in `lock'. 3049 * Page_unlock() will wakeup any thread 3050 * waiting around for this page. 3051 */ 3052 if (lock) { 3053 VM_STAT_ADD(page_reclaim_zero_locked); 3054 mutex_exit(lock); 3055 } 3056 page_unlock(pp); 3057 3058 /* 3059 * get this before we drop all the pcf locks. 3060 */ 3061 mutex_enter(&new_freemem_lock); 3062 3063 p = pcf; 3064 for (i = 0; i < PCF_FANOUT; i++) { 3065 p->pcf_wait++; 3066 mutex_exit(&p->pcf_lock); 3067 p++; 3068 } 3069 3070 freemem_wait++; 3071 cv_wait(&freemem_cv, &new_freemem_lock); 3072 freemem_wait--; 3073 3074 mutex_exit(&new_freemem_lock); 3075 3076 if (lock) { 3077 mutex_enter(lock); 3078 } 3079 return (0); 3080 } 3081 3082 /* 3083 * The pcf accounting has been done, 3084 * though none of the pcf_wait flags have been set, 3085 * drop the locks and continue on. 3086 */ 3087 while (p >= pcf) { 3088 mutex_exit(&p->pcf_lock); 3089 p--; 3090 } 3091 } 3092 3093 /* 3094 * freemem is not protected by any lock. Thus, we cannot 3095 * have any assertion containing freemem here. 3096 */ 3097 freemem -= 1; 3098 3099 VM_STAT_ADD(pagecnt.pc_reclaim); 3100 3101 /* 3102 * page_list_sub will handle the case where pp is a large page. 3103 * It's possible that the page was promoted while on the freelist 3104 */ 3105 if (PP_ISAGED(pp)) { 3106 page_list_sub(pp, PG_FREE_LIST); 3107 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3108 "page_reclaim_free:pp %p", pp); 3109 } else { 3110 page_list_sub(pp, PG_CACHE_LIST); 3111 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3112 "page_reclaim_cache:pp %p", pp); 3113 } 3114 3115 /* 3116 * clear the p_free & p_age bits since this page is no longer 3117 * on the free list. Notice that there was a brief time where 3118 * a page is marked as free, but is not on the list. 3119 * 3120 * Set the reference bit to protect against immediate pageout. 3121 */ 3122 PP_CLRFREE(pp); 3123 PP_CLRAGED(pp); 3124 page_set_props(pp, P_REF); 3125 3126 CPU_STATS_ENTER_K(); 3127 cpup = CPU; /* get cpup now that CPU cannot change */ 3128 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3129 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3130 CPU_STATS_EXIT_K(); 3131 ASSERT(pp->p_szc == 0); 3132 3133 return (1); 3134 } 3135 3136 /* 3137 * Destroy identity of the page and put it back on 3138 * the page free list. Assumes that the caller has 3139 * acquired the "exclusive" lock on the page. 3140 */ 3141 void 3142 page_destroy(page_t *pp, int dontfree) 3143 { 3144 ASSERT((PAGE_EXCL(pp) && 3145 !page_iolock_assert(pp)) || panicstr); 3146 ASSERT(pp->p_slckcnt == 0 || panicstr); 3147 3148 if (pp->p_szc != 0) { 3149 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3150 PP_ISKAS(pp)) { 3151 panic("page_destroy: anon or kernel or no vnode " 3152 "large page %p", (void *)pp); 3153 } 3154 page_demote_vp_pages(pp); 3155 ASSERT(pp->p_szc == 0); 3156 } 3157 3158 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3159 3160 /* 3161 * Unload translations, if any, then hash out the 3162 * page to erase its identity. 3163 */ 3164 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3165 page_hashout(pp, NULL); 3166 3167 if (!dontfree) { 3168 /* 3169 * Acquire the "freemem_lock" for availrmem. 3170 * The page_struct_lock need not be acquired for lckcnt 3171 * and cowcnt since the page has an "exclusive" lock. 3172 */ 3173 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3174 mutex_enter(&freemem_lock); 3175 if (pp->p_lckcnt != 0) { 3176 availrmem++; 3177 pp->p_lckcnt = 0; 3178 } 3179 if (pp->p_cowcnt != 0) { 3180 availrmem += pp->p_cowcnt; 3181 pp->p_cowcnt = 0; 3182 } 3183 mutex_exit(&freemem_lock); 3184 } 3185 /* 3186 * Put the page on the "free" list. 3187 */ 3188 page_free(pp, 0); 3189 } 3190 } 3191 3192 void 3193 page_destroy_pages(page_t *pp) 3194 { 3195 3196 page_t *tpp, *rootpp = NULL; 3197 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3198 pgcnt_t i, pglcks = 0; 3199 uint_t szc = pp->p_szc; 3200 3201 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3202 3203 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3204 3205 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3206 3207 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3208 panic("page_destroy_pages: not root page %p", (void *)pp); 3209 /*NOTREACHED*/ 3210 } 3211 3212 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3213 ASSERT((PAGE_EXCL(tpp) && 3214 !page_iolock_assert(tpp)) || panicstr); 3215 ASSERT(tpp->p_slckcnt == 0 || panicstr); 3216 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3217 page_hashout(tpp, NULL); 3218 ASSERT(tpp->p_offset == (u_offset_t)-1); 3219 if (tpp->p_lckcnt != 0) { 3220 pglcks++; 3221 tpp->p_lckcnt = 0; 3222 } else if (tpp->p_cowcnt != 0) { 3223 pglcks += tpp->p_cowcnt; 3224 tpp->p_cowcnt = 0; 3225 } 3226 ASSERT(!hat_page_getshare(tpp)); 3227 ASSERT(tpp->p_vnode == NULL); 3228 ASSERT(tpp->p_szc == szc); 3229 3230 PP_SETFREE(tpp); 3231 page_clr_all_props(tpp); 3232 PP_SETAGED(tpp); 3233 ASSERT(tpp->p_next == tpp); 3234 ASSERT(tpp->p_prev == tpp); 3235 page_list_concat(&rootpp, &tpp); 3236 } 3237 3238 ASSERT(rootpp == pp); 3239 if (pglcks != 0) { 3240 mutex_enter(&freemem_lock); 3241 availrmem += pglcks; 3242 mutex_exit(&freemem_lock); 3243 } 3244 3245 page_list_add_pages(rootpp, 0); 3246 page_create_putback(pgcnt); 3247 } 3248 3249 /* 3250 * Similar to page_destroy(), but destroys pages which are 3251 * locked and known to be on the page free list. Since 3252 * the page is known to be free and locked, no one can access 3253 * it. 3254 * 3255 * Also, the number of free pages does not change. 3256 */ 3257 void 3258 page_destroy_free(page_t *pp) 3259 { 3260 ASSERT(PAGE_EXCL(pp)); 3261 ASSERT(PP_ISFREE(pp)); 3262 ASSERT(pp->p_vnode); 3263 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3264 ASSERT(!hat_page_is_mapped(pp)); 3265 ASSERT(PP_ISAGED(pp) == 0); 3266 ASSERT(pp->p_szc == 0); 3267 3268 VM_STAT_ADD(pagecnt.pc_destroy_free); 3269 page_list_sub(pp, PG_CACHE_LIST); 3270 3271 page_hashout(pp, NULL); 3272 ASSERT(pp->p_vnode == NULL); 3273 ASSERT(pp->p_offset == (u_offset_t)-1); 3274 ASSERT(pp->p_hash == NULL); 3275 3276 PP_SETAGED(pp); 3277 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3278 page_unlock(pp); 3279 3280 mutex_enter(&new_freemem_lock); 3281 if (freemem_wait) { 3282 cv_signal(&freemem_cv); 3283 } 3284 mutex_exit(&new_freemem_lock); 3285 } 3286 3287 /* 3288 * Rename the page "opp" to have an identity specified 3289 * by [vp, off]. If a page already exists with this name 3290 * it is locked and destroyed. Note that the page's 3291 * translations are not unloaded during the rename. 3292 * 3293 * This routine is used by the anon layer to "steal" the 3294 * original page and is not unlike destroying a page and 3295 * creating a new page using the same page frame. 3296 * 3297 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3298 * caller 2 tries to rename B to A. 3299 */ 3300 void 3301 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3302 { 3303 page_t *pp; 3304 int olckcnt = 0; 3305 int ocowcnt = 0; 3306 kmutex_t *phm; 3307 ulong_t index; 3308 3309 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3310 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3311 ASSERT(PP_ISFREE(opp) == 0); 3312 3313 VM_STAT_ADD(page_rename_count); 3314 3315 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3316 "page rename:pp %p vp %p off %llx", opp, vp, off); 3317 3318 /* 3319 * CacheFS may call page_rename for a large NFS page 3320 * when both CacheFS and NFS mount points are used 3321 * by applications. Demote this large page before 3322 * renaming it, to ensure that there are no "partial" 3323 * large pages left lying around. 3324 */ 3325 if (opp->p_szc != 0) { 3326 vnode_t *ovp = opp->p_vnode; 3327 ASSERT(ovp != NULL); 3328 ASSERT(!IS_SWAPFSVP(ovp)); 3329 ASSERT(!VN_ISKAS(ovp)); 3330 page_demote_vp_pages(opp); 3331 ASSERT(opp->p_szc == 0); 3332 } 3333 3334 page_hashout(opp, NULL); 3335 PP_CLRAGED(opp); 3336 3337 /* 3338 * Acquire the appropriate page hash lock, since 3339 * we're going to rename the page. 3340 */ 3341 index = PAGE_HASH_FUNC(vp, off); 3342 phm = PAGE_HASH_MUTEX(index); 3343 mutex_enter(phm); 3344 top: 3345 /* 3346 * Look for an existing page with this name and destroy it if found. 3347 * By holding the page hash lock all the way to the page_hashin() 3348 * call, we are assured that no page can be created with this 3349 * identity. In the case when the phm lock is dropped to undo any 3350 * hat layer mappings, the existing page is held with an "exclusive" 3351 * lock, again preventing another page from being created with 3352 * this identity. 3353 */ 3354 PAGE_HASH_SEARCH(index, pp, vp, off); 3355 if (pp != NULL) { 3356 VM_STAT_ADD(page_rename_exists); 3357 3358 /* 3359 * As it turns out, this is one of only two places where 3360 * page_lock() needs to hold the passed in lock in the 3361 * successful case. In all of the others, the lock could 3362 * be dropped as soon as the attempt is made to lock 3363 * the page. It is tempting to add yet another arguement, 3364 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3365 */ 3366 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3367 /* 3368 * Went to sleep because the page could not 3369 * be locked. We were woken up when the page 3370 * was unlocked, or when the page was destroyed. 3371 * In either case, `phm' was dropped while we 3372 * slept. Hence we should not just roar through 3373 * this loop. 3374 */ 3375 goto top; 3376 } 3377 3378 /* 3379 * If an existing page is a large page, then demote 3380 * it to ensure that no "partial" large pages are 3381 * "created" after page_rename. An existing page 3382 * can be a CacheFS page, and can't belong to swapfs. 3383 */ 3384 if (hat_page_is_mapped(pp)) { 3385 /* 3386 * Unload translations. Since we hold the 3387 * exclusive lock on this page, the page 3388 * can not be changed while we drop phm. 3389 * This is also not a lock protocol violation, 3390 * but rather the proper way to do things. 3391 */ 3392 mutex_exit(phm); 3393 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3394 if (pp->p_szc != 0) { 3395 ASSERT(!IS_SWAPFSVP(vp)); 3396 ASSERT(!VN_ISKAS(vp)); 3397 page_demote_vp_pages(pp); 3398 ASSERT(pp->p_szc == 0); 3399 } 3400 mutex_enter(phm); 3401 } else if (pp->p_szc != 0) { 3402 ASSERT(!IS_SWAPFSVP(vp)); 3403 ASSERT(!VN_ISKAS(vp)); 3404 mutex_exit(phm); 3405 page_demote_vp_pages(pp); 3406 ASSERT(pp->p_szc == 0); 3407 mutex_enter(phm); 3408 } 3409 page_hashout(pp, phm); 3410 } 3411 /* 3412 * Hash in the page with the new identity. 3413 */ 3414 if (!page_hashin(opp, vp, off, phm)) { 3415 /* 3416 * We were holding phm while we searched for [vp, off] 3417 * and only dropped phm if we found and locked a page. 3418 * If we can't create this page now, then some thing 3419 * is really broken. 3420 */ 3421 panic("page_rename: Can't hash in page: %p", (void *)pp); 3422 /*NOTREACHED*/ 3423 } 3424 3425 ASSERT(MUTEX_HELD(phm)); 3426 mutex_exit(phm); 3427 3428 /* 3429 * Now that we have dropped phm, lets get around to finishing up 3430 * with pp. 3431 */ 3432 if (pp != NULL) { 3433 ASSERT(!hat_page_is_mapped(pp)); 3434 /* for now large pages should not end up here */ 3435 ASSERT(pp->p_szc == 0); 3436 /* 3437 * Save the locks for transfer to the new page and then 3438 * clear them so page_free doesn't think they're important. 3439 * The page_struct_lock need not be acquired for lckcnt and 3440 * cowcnt since the page has an "exclusive" lock. 3441 */ 3442 olckcnt = pp->p_lckcnt; 3443 ocowcnt = pp->p_cowcnt; 3444 pp->p_lckcnt = pp->p_cowcnt = 0; 3445 3446 /* 3447 * Put the page on the "free" list after we drop 3448 * the lock. The less work under the lock the better. 3449 */ 3450 /*LINTED: constant in conditional context*/ 3451 VN_DISPOSE(pp, B_FREE, 0, kcred); 3452 } 3453 3454 /* 3455 * Transfer the lock count from the old page (if any). 3456 * The page_struct_lock need not be acquired for lckcnt and 3457 * cowcnt since the page has an "exclusive" lock. 3458 */ 3459 opp->p_lckcnt += olckcnt; 3460 opp->p_cowcnt += ocowcnt; 3461 } 3462 3463 /* 3464 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3465 * 3466 * Pages are normally inserted at the start of a vnode's v_pages list. 3467 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3468 * This can happen when a modified page is relocated for DR. 3469 * 3470 * Returns 1 on success and 0 on failure. 3471 */ 3472 static int 3473 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3474 { 3475 page_t **listp; 3476 page_t *tp; 3477 ulong_t index; 3478 3479 ASSERT(PAGE_EXCL(pp)); 3480 ASSERT(vp != NULL); 3481 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3482 3483 /* 3484 * Be sure to set these up before the page is inserted on the hash 3485 * list. As soon as the page is placed on the list some other 3486 * thread might get confused and wonder how this page could 3487 * possibly hash to this list. 3488 */ 3489 pp->p_vnode = vp; 3490 pp->p_offset = offset; 3491 3492 /* 3493 * record if this page is on a swap vnode 3494 */ 3495 if ((vp->v_flag & VISSWAP) != 0) 3496 PP_SETSWAP(pp); 3497 3498 index = PAGE_HASH_FUNC(vp, offset); 3499 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3500 listp = &page_hash[index]; 3501 3502 /* 3503 * If this page is already hashed in, fail this attempt to add it. 3504 */ 3505 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3506 if (tp->p_vnode == vp && tp->p_offset == offset) { 3507 pp->p_vnode = NULL; 3508 pp->p_offset = (u_offset_t)(-1); 3509 return (0); 3510 } 3511 } 3512 pp->p_hash = *listp; 3513 *listp = pp; 3514 3515 /* 3516 * Add the page to the vnode's list of pages 3517 */ 3518 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3519 listp = &vp->v_pages->p_vpprev->p_vpnext; 3520 else 3521 listp = &vp->v_pages; 3522 3523 page_vpadd(listp, pp); 3524 3525 return (1); 3526 } 3527 3528 /* 3529 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3530 * 3531 * Returns 1 on success and 0 on failure. 3532 * If hold is passed in, it is not dropped. 3533 */ 3534 int 3535 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3536 { 3537 kmutex_t *phm = NULL; 3538 kmutex_t *vphm; 3539 int rc; 3540 3541 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3542 3543 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3544 "page_hashin:pp %p vp %p offset %llx", 3545 pp, vp, offset); 3546 3547 VM_STAT_ADD(hashin_count); 3548 3549 if (hold != NULL) 3550 phm = hold; 3551 else { 3552 VM_STAT_ADD(hashin_not_held); 3553 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3554 mutex_enter(phm); 3555 } 3556 3557 vphm = page_vnode_mutex(vp); 3558 mutex_enter(vphm); 3559 rc = page_do_hashin(pp, vp, offset); 3560 mutex_exit(vphm); 3561 if (hold == NULL) 3562 mutex_exit(phm); 3563 if (rc == 0) 3564 VM_STAT_ADD(hashin_already); 3565 return (rc); 3566 } 3567 3568 /* 3569 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3570 * All mutexes must be held 3571 */ 3572 static void 3573 page_do_hashout(page_t *pp) 3574 { 3575 page_t **hpp; 3576 page_t *hp; 3577 vnode_t *vp = pp->p_vnode; 3578 3579 ASSERT(vp != NULL); 3580 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3581 3582 /* 3583 * First, take pp off of its hash chain. 3584 */ 3585 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3586 3587 for (;;) { 3588 hp = *hpp; 3589 if (hp == pp) 3590 break; 3591 if (hp == NULL) { 3592 panic("page_do_hashout"); 3593 /*NOTREACHED*/ 3594 } 3595 hpp = &hp->p_hash; 3596 } 3597 *hpp = pp->p_hash; 3598 3599 /* 3600 * Now remove it from its associated vnode. 3601 */ 3602 if (vp->v_pages) 3603 page_vpsub(&vp->v_pages, pp); 3604 3605 pp->p_hash = NULL; 3606 page_clr_all_props(pp); 3607 PP_CLRSWAP(pp); 3608 pp->p_vnode = NULL; 3609 pp->p_offset = (u_offset_t)-1; 3610 } 3611 3612 /* 3613 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3614 * 3615 * When `phm' is non-NULL it contains the address of the mutex protecting the 3616 * hash list pp is on. It is not dropped. 3617 */ 3618 void 3619 page_hashout(page_t *pp, kmutex_t *phm) 3620 { 3621 vnode_t *vp; 3622 ulong_t index; 3623 kmutex_t *nphm; 3624 kmutex_t *vphm; 3625 kmutex_t *sep; 3626 3627 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3628 ASSERT(pp->p_vnode != NULL); 3629 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3630 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3631 3632 vp = pp->p_vnode; 3633 3634 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3635 "page_hashout:pp %p vp %p", pp, vp); 3636 3637 /* Kernel probe */ 3638 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3639 tnf_opaque, vnode, vp, 3640 tnf_offset, offset, pp->p_offset); 3641 3642 /* 3643 * 3644 */ 3645 VM_STAT_ADD(hashout_count); 3646 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3647 if (phm == NULL) { 3648 VM_STAT_ADD(hashout_not_held); 3649 nphm = PAGE_HASH_MUTEX(index); 3650 mutex_enter(nphm); 3651 } 3652 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3653 3654 3655 /* 3656 * grab page vnode mutex and remove it... 3657 */ 3658 vphm = page_vnode_mutex(vp); 3659 mutex_enter(vphm); 3660 3661 page_do_hashout(pp); 3662 3663 mutex_exit(vphm); 3664 if (phm == NULL) 3665 mutex_exit(nphm); 3666 3667 /* 3668 * Wake up processes waiting for this page. The page's 3669 * identity has been changed, and is probably not the 3670 * desired page any longer. 3671 */ 3672 sep = page_se_mutex(pp); 3673 mutex_enter(sep); 3674 pp->p_selock &= ~SE_EWANTED; 3675 if (CV_HAS_WAITERS(&pp->p_cv)) 3676 cv_broadcast(&pp->p_cv); 3677 mutex_exit(sep); 3678 } 3679 3680 /* 3681 * Add the page to the front of a linked list of pages 3682 * using the p_next & p_prev pointers for the list. 3683 * The caller is responsible for protecting the list pointers. 3684 */ 3685 void 3686 page_add(page_t **ppp, page_t *pp) 3687 { 3688 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3689 3690 page_add_common(ppp, pp); 3691 } 3692 3693 3694 3695 /* 3696 * Common code for page_add() and mach_page_add() 3697 */ 3698 void 3699 page_add_common(page_t **ppp, page_t *pp) 3700 { 3701 if (*ppp == NULL) { 3702 pp->p_next = pp->p_prev = pp; 3703 } else { 3704 pp->p_next = *ppp; 3705 pp->p_prev = (*ppp)->p_prev; 3706 (*ppp)->p_prev = pp; 3707 pp->p_prev->p_next = pp; 3708 } 3709 *ppp = pp; 3710 } 3711 3712 3713 /* 3714 * Remove this page from a linked list of pages 3715 * using the p_next & p_prev pointers for the list. 3716 * 3717 * The caller is responsible for protecting the list pointers. 3718 */ 3719 void 3720 page_sub(page_t **ppp, page_t *pp) 3721 { 3722 ASSERT((PP_ISFREE(pp)) ? 1 : 3723 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3724 3725 if (*ppp == NULL || pp == NULL) { 3726 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3727 (void *)pp, (void *)(*ppp)); 3728 /*NOTREACHED*/ 3729 } 3730 3731 page_sub_common(ppp, pp); 3732 } 3733 3734 3735 /* 3736 * Common code for page_sub() and mach_page_sub() 3737 */ 3738 void 3739 page_sub_common(page_t **ppp, page_t *pp) 3740 { 3741 if (*ppp == pp) 3742 *ppp = pp->p_next; /* go to next page */ 3743 3744 if (*ppp == pp) 3745 *ppp = NULL; /* page list is gone */ 3746 else { 3747 pp->p_prev->p_next = pp->p_next; 3748 pp->p_next->p_prev = pp->p_prev; 3749 } 3750 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3751 } 3752 3753 3754 /* 3755 * Break page list cppp into two lists with npages in the first list. 3756 * The tail is returned in nppp. 3757 */ 3758 void 3759 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3760 { 3761 page_t *s1pp = *oppp; 3762 page_t *s2pp; 3763 page_t *e1pp, *e2pp; 3764 long n = 0; 3765 3766 if (s1pp == NULL) { 3767 *nppp = NULL; 3768 return; 3769 } 3770 if (npages == 0) { 3771 *nppp = s1pp; 3772 *oppp = NULL; 3773 return; 3774 } 3775 for (n = 0, s2pp = *oppp; n < npages; n++) { 3776 s2pp = s2pp->p_next; 3777 } 3778 /* Fix head and tail of new lists */ 3779 e1pp = s2pp->p_prev; 3780 e2pp = s1pp->p_prev; 3781 s1pp->p_prev = e1pp; 3782 e1pp->p_next = s1pp; 3783 s2pp->p_prev = e2pp; 3784 e2pp->p_next = s2pp; 3785 3786 /* second list empty */ 3787 if (s2pp == s1pp) { 3788 *oppp = s1pp; 3789 *nppp = NULL; 3790 } else { 3791 *oppp = s1pp; 3792 *nppp = s2pp; 3793 } 3794 } 3795 3796 /* 3797 * Concatenate page list nppp onto the end of list ppp. 3798 */ 3799 void 3800 page_list_concat(page_t **ppp, page_t **nppp) 3801 { 3802 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3803 3804 if (*nppp == NULL) { 3805 return; 3806 } 3807 if (*ppp == NULL) { 3808 *ppp = *nppp; 3809 return; 3810 } 3811 s1pp = *ppp; 3812 e1pp = s1pp->p_prev; 3813 s2pp = *nppp; 3814 e2pp = s2pp->p_prev; 3815 s1pp->p_prev = e2pp; 3816 e2pp->p_next = s1pp; 3817 e1pp->p_next = s2pp; 3818 s2pp->p_prev = e1pp; 3819 } 3820 3821 /* 3822 * return the next page in the page list 3823 */ 3824 page_t * 3825 page_list_next(page_t *pp) 3826 { 3827 return (pp->p_next); 3828 } 3829 3830 3831 /* 3832 * Add the page to the front of the linked list of pages 3833 * using p_vpnext/p_vpprev pointers for the list. 3834 * 3835 * The caller is responsible for protecting the lists. 3836 */ 3837 void 3838 page_vpadd(page_t **ppp, page_t *pp) 3839 { 3840 if (*ppp == NULL) { 3841 pp->p_vpnext = pp->p_vpprev = pp; 3842 } else { 3843 pp->p_vpnext = *ppp; 3844 pp->p_vpprev = (*ppp)->p_vpprev; 3845 (*ppp)->p_vpprev = pp; 3846 pp->p_vpprev->p_vpnext = pp; 3847 } 3848 *ppp = pp; 3849 } 3850 3851 /* 3852 * Remove this page from the linked list of pages 3853 * using p_vpnext/p_vpprev pointers for the list. 3854 * 3855 * The caller is responsible for protecting the lists. 3856 */ 3857 void 3858 page_vpsub(page_t **ppp, page_t *pp) 3859 { 3860 if (*ppp == NULL || pp == NULL) { 3861 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3862 (void *)pp, (void *)(*ppp)); 3863 /*NOTREACHED*/ 3864 } 3865 3866 if (*ppp == pp) 3867 *ppp = pp->p_vpnext; /* go to next page */ 3868 3869 if (*ppp == pp) 3870 *ppp = NULL; /* page list is gone */ 3871 else { 3872 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3873 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3874 } 3875 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3876 } 3877 3878 /* 3879 * Lock a physical page into memory "long term". Used to support "lock 3880 * in memory" functions. Accepts the page to be locked, and a cow variable 3881 * to indicate whether a the lock will travel to the new page during 3882 * a potential copy-on-write. 3883 */ 3884 int 3885 page_pp_lock( 3886 page_t *pp, /* page to be locked */ 3887 int cow, /* cow lock */ 3888 int kernel) /* must succeed -- ignore checking */ 3889 { 3890 int r = 0; /* result -- assume failure */ 3891 3892 ASSERT(PAGE_LOCKED(pp)); 3893 3894 page_struct_lock(pp); 3895 /* 3896 * Acquire the "freemem_lock" for availrmem. 3897 */ 3898 if (cow) { 3899 mutex_enter(&freemem_lock); 3900 if ((availrmem > pages_pp_maximum) && 3901 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3902 availrmem--; 3903 pages_locked++; 3904 mutex_exit(&freemem_lock); 3905 r = 1; 3906 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3907 cmn_err(CE_WARN, 3908 "COW lock limit reached on pfn 0x%lx", 3909 page_pptonum(pp)); 3910 } 3911 } else 3912 mutex_exit(&freemem_lock); 3913 } else { 3914 if (pp->p_lckcnt) { 3915 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3916 r = 1; 3917 if (++pp->p_lckcnt == 3918 (ushort_t)PAGE_LOCK_MAXIMUM) { 3919 cmn_err(CE_WARN, "Page lock limit " 3920 "reached on pfn 0x%lx", 3921 page_pptonum(pp)); 3922 } 3923 } 3924 } else { 3925 if (kernel) { 3926 /* availrmem accounting done by caller */ 3927 ++pp->p_lckcnt; 3928 r = 1; 3929 } else { 3930 mutex_enter(&freemem_lock); 3931 if (availrmem > pages_pp_maximum) { 3932 availrmem--; 3933 pages_locked++; 3934 ++pp->p_lckcnt; 3935 r = 1; 3936 } 3937 mutex_exit(&freemem_lock); 3938 } 3939 } 3940 } 3941 page_struct_unlock(pp); 3942 return (r); 3943 } 3944 3945 /* 3946 * Decommit a lock on a physical page frame. Account for cow locks if 3947 * appropriate. 3948 */ 3949 void 3950 page_pp_unlock( 3951 page_t *pp, /* page to be unlocked */ 3952 int cow, /* expect cow lock */ 3953 int kernel) /* this was a kernel lock */ 3954 { 3955 ASSERT(PAGE_LOCKED(pp)); 3956 3957 page_struct_lock(pp); 3958 /* 3959 * Acquire the "freemem_lock" for availrmem. 3960 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3961 * could be called to unlock even if nothing is locked. This could 3962 * happen if locked file pages were truncated (removing the lock) 3963 * and the file was grown again and new pages faulted in; the new 3964 * pages are unlocked but the segment still thinks they're locked. 3965 */ 3966 if (cow) { 3967 if (pp->p_cowcnt) { 3968 mutex_enter(&freemem_lock); 3969 pp->p_cowcnt--; 3970 availrmem++; 3971 pages_locked--; 3972 mutex_exit(&freemem_lock); 3973 } 3974 } else { 3975 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3976 if (!kernel) { 3977 mutex_enter(&freemem_lock); 3978 availrmem++; 3979 pages_locked--; 3980 mutex_exit(&freemem_lock); 3981 } 3982 } 3983 } 3984 page_struct_unlock(pp); 3985 } 3986 3987 /* 3988 * This routine reserves availrmem for npages; 3989 * flags: KM_NOSLEEP or KM_SLEEP 3990 * returns 1 on success or 0 on failure 3991 */ 3992 int 3993 page_resv(pgcnt_t npages, uint_t flags) 3994 { 3995 mutex_enter(&freemem_lock); 3996 while (availrmem < tune.t_minarmem + npages) { 3997 if (flags & KM_NOSLEEP) { 3998 mutex_exit(&freemem_lock); 3999 return (0); 4000 } 4001 mutex_exit(&freemem_lock); 4002 page_needfree(npages); 4003 kmem_reap(); 4004 delay(hz >> 2); 4005 page_needfree(-(spgcnt_t)npages); 4006 mutex_enter(&freemem_lock); 4007 } 4008 availrmem -= npages; 4009 mutex_exit(&freemem_lock); 4010 return (1); 4011 } 4012 4013 /* 4014 * This routine unreserves availrmem for npages; 4015 */ 4016 void 4017 page_unresv(pgcnt_t npages) 4018 { 4019 mutex_enter(&freemem_lock); 4020 availrmem += npages; 4021 mutex_exit(&freemem_lock); 4022 } 4023 4024 /* 4025 * See Statement at the beginning of segvn_lockop() regarding 4026 * the way we handle cowcnts and lckcnts. 4027 * 4028 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 4029 * that breaks COW has PROT_WRITE. 4030 * 4031 * Note that, we may also break COW in case we are softlocking 4032 * on read access during physio; 4033 * in this softlock case, the vpage may not have PROT_WRITE. 4034 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 4035 * if the vpage doesn't have PROT_WRITE. 4036 * 4037 * This routine is never called if we are stealing a page 4038 * in anon_private. 4039 * 4040 * The caller subtracted from availrmem for read only mapping. 4041 * if lckcnt is 1 increment availrmem. 4042 */ 4043 void 4044 page_pp_useclaim( 4045 page_t *opp, /* original page frame losing lock */ 4046 page_t *npp, /* new page frame gaining lock */ 4047 uint_t write_perm) /* set if vpage has PROT_WRITE */ 4048 { 4049 int payback = 0; 4050 4051 ASSERT(PAGE_LOCKED(opp)); 4052 ASSERT(PAGE_LOCKED(npp)); 4053 4054 page_struct_lock(opp); 4055 4056 ASSERT(npp->p_cowcnt == 0); 4057 ASSERT(npp->p_lckcnt == 0); 4058 4059 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4060 if ((write_perm && opp->p_cowcnt != 0) || 4061 (!write_perm && opp->p_lckcnt != 0)) { 4062 4063 if (write_perm) { 4064 npp->p_cowcnt++; 4065 ASSERT(opp->p_cowcnt != 0); 4066 opp->p_cowcnt--; 4067 } else { 4068 4069 ASSERT(opp->p_lckcnt != 0); 4070 4071 /* 4072 * We didn't need availrmem decremented if p_lckcnt on 4073 * original page is 1. Here, we are unlocking 4074 * read-only copy belonging to original page and 4075 * are locking a copy belonging to new page. 4076 */ 4077 if (opp->p_lckcnt == 1) 4078 payback = 1; 4079 4080 npp->p_lckcnt++; 4081 opp->p_lckcnt--; 4082 } 4083 } 4084 if (payback) { 4085 mutex_enter(&freemem_lock); 4086 availrmem++; 4087 pages_useclaim--; 4088 mutex_exit(&freemem_lock); 4089 } 4090 page_struct_unlock(opp); 4091 } 4092 4093 /* 4094 * Simple claim adjust functions -- used to support changes in 4095 * claims due to changes in access permissions. Used by segvn_setprot(). 4096 */ 4097 int 4098 page_addclaim(page_t *pp) 4099 { 4100 int r = 0; /* result */ 4101 4102 ASSERT(PAGE_LOCKED(pp)); 4103 4104 page_struct_lock(pp); 4105 ASSERT(pp->p_lckcnt != 0); 4106 4107 if (pp->p_lckcnt == 1) { 4108 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4109 --pp->p_lckcnt; 4110 r = 1; 4111 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4112 cmn_err(CE_WARN, 4113 "COW lock limit reached on pfn 0x%lx", 4114 page_pptonum(pp)); 4115 } 4116 } 4117 } else { 4118 mutex_enter(&freemem_lock); 4119 if ((availrmem > pages_pp_maximum) && 4120 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4121 --availrmem; 4122 ++pages_claimed; 4123 mutex_exit(&freemem_lock); 4124 --pp->p_lckcnt; 4125 r = 1; 4126 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4127 cmn_err(CE_WARN, 4128 "COW lock limit reached on pfn 0x%lx", 4129 page_pptonum(pp)); 4130 } 4131 } else 4132 mutex_exit(&freemem_lock); 4133 } 4134 page_struct_unlock(pp); 4135 return (r); 4136 } 4137 4138 int 4139 page_subclaim(page_t *pp) 4140 { 4141 int r = 0; 4142 4143 ASSERT(PAGE_LOCKED(pp)); 4144 4145 page_struct_lock(pp); 4146 ASSERT(pp->p_cowcnt != 0); 4147 4148 if (pp->p_lckcnt) { 4149 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4150 r = 1; 4151 /* 4152 * for availrmem 4153 */ 4154 mutex_enter(&freemem_lock); 4155 availrmem++; 4156 pages_claimed--; 4157 mutex_exit(&freemem_lock); 4158 4159 pp->p_cowcnt--; 4160 4161 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4162 cmn_err(CE_WARN, 4163 "Page lock limit reached on pfn 0x%lx", 4164 page_pptonum(pp)); 4165 } 4166 } 4167 } else { 4168 r = 1; 4169 pp->p_cowcnt--; 4170 pp->p_lckcnt++; 4171 } 4172 page_struct_unlock(pp); 4173 return (r); 4174 } 4175 4176 int 4177 page_addclaim_pages(page_t **ppa) 4178 { 4179 4180 pgcnt_t lckpgs = 0, pg_idx; 4181 4182 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4183 4184 mutex_enter(&page_llock); 4185 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4186 4187 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4188 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4189 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4190 mutex_exit(&page_llock); 4191 return (0); 4192 } 4193 if (ppa[pg_idx]->p_lckcnt > 1) 4194 lckpgs++; 4195 } 4196 4197 if (lckpgs != 0) { 4198 mutex_enter(&freemem_lock); 4199 if (availrmem >= pages_pp_maximum + lckpgs) { 4200 availrmem -= lckpgs; 4201 pages_claimed += lckpgs; 4202 } else { 4203 mutex_exit(&freemem_lock); 4204 mutex_exit(&page_llock); 4205 return (0); 4206 } 4207 mutex_exit(&freemem_lock); 4208 } 4209 4210 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4211 ppa[pg_idx]->p_lckcnt--; 4212 ppa[pg_idx]->p_cowcnt++; 4213 } 4214 mutex_exit(&page_llock); 4215 return (1); 4216 } 4217 4218 int 4219 page_subclaim_pages(page_t **ppa) 4220 { 4221 pgcnt_t ulckpgs = 0, pg_idx; 4222 4223 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4224 4225 mutex_enter(&page_llock); 4226 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4227 4228 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4229 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4230 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4231 mutex_exit(&page_llock); 4232 return (0); 4233 } 4234 if (ppa[pg_idx]->p_lckcnt != 0) 4235 ulckpgs++; 4236 } 4237 4238 if (ulckpgs != 0) { 4239 mutex_enter(&freemem_lock); 4240 availrmem += ulckpgs; 4241 pages_claimed -= ulckpgs; 4242 mutex_exit(&freemem_lock); 4243 } 4244 4245 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4246 ppa[pg_idx]->p_cowcnt--; 4247 ppa[pg_idx]->p_lckcnt++; 4248 4249 } 4250 mutex_exit(&page_llock); 4251 return (1); 4252 } 4253 4254 page_t * 4255 page_numtopp(pfn_t pfnum, se_t se) 4256 { 4257 page_t *pp; 4258 4259 retry: 4260 pp = page_numtopp_nolock(pfnum); 4261 if (pp == NULL) { 4262 return ((page_t *)NULL); 4263 } 4264 4265 /* 4266 * Acquire the appropriate lock on the page. 4267 */ 4268 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4269 if (page_pptonum(pp) != pfnum) 4270 goto retry; 4271 continue; 4272 } 4273 4274 if (page_pptonum(pp) != pfnum) { 4275 page_unlock(pp); 4276 goto retry; 4277 } 4278 4279 return (pp); 4280 } 4281 4282 page_t * 4283 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4284 { 4285 page_t *pp; 4286 4287 retry: 4288 pp = page_numtopp_nolock(pfnum); 4289 if (pp == NULL) { 4290 return ((page_t *)NULL); 4291 } 4292 4293 /* 4294 * Acquire the appropriate lock on the page. 4295 */ 4296 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4297 if (page_pptonum(pp) != pfnum) 4298 goto retry; 4299 continue; 4300 } 4301 4302 if (page_pptonum(pp) != pfnum) { 4303 page_unlock(pp); 4304 goto retry; 4305 } 4306 4307 return (pp); 4308 } 4309 4310 /* 4311 * This routine is like page_numtopp, but will only return page structs 4312 * for pages which are ok for loading into hardware using the page struct. 4313 */ 4314 page_t * 4315 page_numtopp_nowait(pfn_t pfnum, se_t se) 4316 { 4317 page_t *pp; 4318 4319 retry: 4320 pp = page_numtopp_nolock(pfnum); 4321 if (pp == NULL) { 4322 return ((page_t *)NULL); 4323 } 4324 4325 /* 4326 * Try to acquire the appropriate lock on the page. 4327 */ 4328 if (PP_ISFREE(pp)) 4329 pp = NULL; 4330 else { 4331 if (!page_trylock(pp, se)) 4332 pp = NULL; 4333 else { 4334 if (page_pptonum(pp) != pfnum) { 4335 page_unlock(pp); 4336 goto retry; 4337 } 4338 if (PP_ISFREE(pp)) { 4339 page_unlock(pp); 4340 pp = NULL; 4341 } 4342 } 4343 } 4344 return (pp); 4345 } 4346 4347 /* 4348 * Returns a count of dirty pages that are in the process 4349 * of being written out. If 'cleanit' is set, try to push the page. 4350 */ 4351 pgcnt_t 4352 page_busy(int cleanit) 4353 { 4354 page_t *page0 = page_first(); 4355 page_t *pp = page0; 4356 pgcnt_t nppbusy = 0; 4357 u_offset_t off; 4358 4359 do { 4360 vnode_t *vp = pp->p_vnode; 4361 4362 /* 4363 * A page is a candidate for syncing if it is: 4364 * 4365 * (a) On neither the freelist nor the cachelist 4366 * (b) Hashed onto a vnode 4367 * (c) Not a kernel page 4368 * (d) Dirty 4369 * (e) Not part of a swapfile 4370 * (f) a page which belongs to a real vnode; eg has a non-null 4371 * v_vfsp pointer. 4372 * (g) Backed by a filesystem which doesn't have a 4373 * stubbed-out sync operation 4374 */ 4375 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && 4376 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4377 vfs_can_sync(vp->v_vfsp)) { 4378 nppbusy++; 4379 vfs_syncprogress(); 4380 4381 if (!cleanit) 4382 continue; 4383 if (!page_trylock(pp, SE_EXCL)) 4384 continue; 4385 4386 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4387 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4388 !(hat_pagesync(pp, 4389 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4390 page_unlock(pp); 4391 continue; 4392 } 4393 off = pp->p_offset; 4394 VN_HOLD(vp); 4395 page_unlock(pp); 4396 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4397 B_ASYNC | B_FREE, kcred, NULL); 4398 VN_RELE(vp); 4399 } 4400 } while ((pp = page_next(pp)) != page0); 4401 4402 return (nppbusy); 4403 } 4404 4405 void page_invalidate_pages(void); 4406 4407 /* 4408 * callback handler to vm sub-system 4409 * 4410 * callers make sure no recursive entries to this func. 4411 */ 4412 /*ARGSUSED*/ 4413 boolean_t 4414 callb_vm_cpr(void *arg, int code) 4415 { 4416 if (code == CB_CODE_CPR_CHKPT) 4417 page_invalidate_pages(); 4418 return (B_TRUE); 4419 } 4420 4421 /* 4422 * Invalidate all pages of the system. 4423 * It shouldn't be called until all user page activities are all stopped. 4424 */ 4425 void 4426 page_invalidate_pages() 4427 { 4428 page_t *pp; 4429 page_t *page0; 4430 pgcnt_t nbusypages; 4431 int retry = 0; 4432 const int MAXRETRIES = 4; 4433 #if defined(__sparc) 4434 extern struct vnode prom_ppages; 4435 #endif /* __sparc */ 4436 4437 top: 4438 /* 4439 * Flush dirty pages and destroy the clean ones. 4440 */ 4441 nbusypages = 0; 4442 4443 pp = page0 = page_first(); 4444 do { 4445 struct vnode *vp; 4446 u_offset_t offset; 4447 int mod; 4448 4449 /* 4450 * skip the page if it has no vnode or the page associated 4451 * with the kernel vnode or prom allocated kernel mem. 4452 */ 4453 #if defined(__sparc) 4454 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp) || 4455 vp == &prom_ppages) 4456 #else /* x86 doesn't have prom or prom_ppage */ 4457 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) 4458 #endif /* __sparc */ 4459 continue; 4460 4461 /* 4462 * skip the page which is already free invalidated. 4463 */ 4464 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4465 continue; 4466 4467 /* 4468 * skip pages that are already locked or can't be "exclusively" 4469 * locked or are already free. After we lock the page, check 4470 * the free and age bits again to be sure it's not destroied 4471 * yet. 4472 * To achieve max. parallelization, we use page_trylock instead 4473 * of page_lock so that we don't get block on individual pages 4474 * while we have thousands of other pages to process. 4475 */ 4476 if (!page_trylock(pp, SE_EXCL)) { 4477 nbusypages++; 4478 continue; 4479 } else if (PP_ISFREE(pp)) { 4480 if (!PP_ISAGED(pp)) { 4481 page_destroy_free(pp); 4482 } else { 4483 page_unlock(pp); 4484 } 4485 continue; 4486 } 4487 /* 4488 * Is this page involved in some I/O? shared? 4489 * 4490 * The page_struct_lock need not be acquired to 4491 * examine these fields since the page has an 4492 * "exclusive" lock. 4493 */ 4494 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4495 page_unlock(pp); 4496 continue; 4497 } 4498 4499 if (vp->v_type == VCHR) { 4500 panic("vp->v_type == VCHR"); 4501 /*NOTREACHED*/ 4502 } 4503 4504 if (!page_try_demote_pages(pp)) { 4505 page_unlock(pp); 4506 continue; 4507 } 4508 4509 /* 4510 * Check the modified bit. Leave the bits alone in hardware 4511 * (they will be modified if we do the putpage). 4512 */ 4513 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4514 & P_MOD); 4515 if (mod) { 4516 offset = pp->p_offset; 4517 /* 4518 * Hold the vnode before releasing the page lock 4519 * to prevent it from being freed and re-used by 4520 * some other thread. 4521 */ 4522 VN_HOLD(vp); 4523 page_unlock(pp); 4524 /* 4525 * No error return is checked here. Callers such as 4526 * cpr deals with the dirty pages at the dump time 4527 * if this putpage fails. 4528 */ 4529 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4530 kcred, NULL); 4531 VN_RELE(vp); 4532 } else { 4533 page_destroy(pp, 0); 4534 } 4535 } while ((pp = page_next(pp)) != page0); 4536 if (nbusypages && retry++ < MAXRETRIES) { 4537 delay(1); 4538 goto top; 4539 } 4540 } 4541 4542 /* 4543 * Replace the page "old" with the page "new" on the page hash and vnode lists 4544 * 4545 * the replacement must be done in place, ie the equivalent sequence: 4546 * 4547 * vp = old->p_vnode; 4548 * off = old->p_offset; 4549 * page_do_hashout(old) 4550 * page_do_hashin(new, vp, off) 4551 * 4552 * doesn't work, since 4553 * 1) if old is the only page on the vnode, the v_pages list has a window 4554 * where it looks empty. This will break file system assumptions. 4555 * and 4556 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4557 */ 4558 static void 4559 page_do_relocate_hash(page_t *new, page_t *old) 4560 { 4561 page_t **hash_list; 4562 vnode_t *vp = old->p_vnode; 4563 kmutex_t *sep; 4564 4565 ASSERT(PAGE_EXCL(old)); 4566 ASSERT(PAGE_EXCL(new)); 4567 ASSERT(vp != NULL); 4568 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4569 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4570 4571 /* 4572 * First find old page on the page hash list 4573 */ 4574 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4575 4576 for (;;) { 4577 if (*hash_list == old) 4578 break; 4579 if (*hash_list == NULL) { 4580 panic("page_do_hashout"); 4581 /*NOTREACHED*/ 4582 } 4583 hash_list = &(*hash_list)->p_hash; 4584 } 4585 4586 /* 4587 * update new and replace old with new on the page hash list 4588 */ 4589 new->p_vnode = old->p_vnode; 4590 new->p_offset = old->p_offset; 4591 new->p_hash = old->p_hash; 4592 *hash_list = new; 4593 4594 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4595 PP_SETSWAP(new); 4596 4597 /* 4598 * replace old with new on the vnode's page list 4599 */ 4600 if (old->p_vpnext == old) { 4601 new->p_vpnext = new; 4602 new->p_vpprev = new; 4603 } else { 4604 new->p_vpnext = old->p_vpnext; 4605 new->p_vpprev = old->p_vpprev; 4606 new->p_vpnext->p_vpprev = new; 4607 new->p_vpprev->p_vpnext = new; 4608 } 4609 if (vp->v_pages == old) 4610 vp->v_pages = new; 4611 4612 /* 4613 * clear out the old page 4614 */ 4615 old->p_hash = NULL; 4616 old->p_vpnext = NULL; 4617 old->p_vpprev = NULL; 4618 old->p_vnode = NULL; 4619 PP_CLRSWAP(old); 4620 old->p_offset = (u_offset_t)-1; 4621 page_clr_all_props(old); 4622 4623 /* 4624 * Wake up processes waiting for this page. The page's 4625 * identity has been changed, and is probably not the 4626 * desired page any longer. 4627 */ 4628 sep = page_se_mutex(old); 4629 mutex_enter(sep); 4630 old->p_selock &= ~SE_EWANTED; 4631 if (CV_HAS_WAITERS(&old->p_cv)) 4632 cv_broadcast(&old->p_cv); 4633 mutex_exit(sep); 4634 } 4635 4636 /* 4637 * This function moves the identity of page "pp_old" to page "pp_new". 4638 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4639 * and need not be hashed out from anywhere. 4640 */ 4641 void 4642 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4643 { 4644 vnode_t *vp = pp_old->p_vnode; 4645 u_offset_t off = pp_old->p_offset; 4646 kmutex_t *phm, *vphm; 4647 4648 /* 4649 * Rehash two pages 4650 */ 4651 ASSERT(PAGE_EXCL(pp_old)); 4652 ASSERT(PAGE_EXCL(pp_new)); 4653 ASSERT(vp != NULL); 4654 ASSERT(pp_new->p_vnode == NULL); 4655 4656 /* 4657 * hashout then hashin while holding the mutexes 4658 */ 4659 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4660 mutex_enter(phm); 4661 vphm = page_vnode_mutex(vp); 4662 mutex_enter(vphm); 4663 4664 page_do_relocate_hash(pp_new, pp_old); 4665 4666 mutex_exit(vphm); 4667 mutex_exit(phm); 4668 4669 /* 4670 * The page_struct_lock need not be acquired for lckcnt and 4671 * cowcnt since the page has an "exclusive" lock. 4672 */ 4673 ASSERT(pp_new->p_lckcnt == 0); 4674 ASSERT(pp_new->p_cowcnt == 0); 4675 pp_new->p_lckcnt = pp_old->p_lckcnt; 4676 pp_new->p_cowcnt = pp_old->p_cowcnt; 4677 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4678 4679 /* The following comment preserved from page_flip(). */ 4680 /* XXX - Do we need to protect fsdata? */ 4681 pp_new->p_fsdata = pp_old->p_fsdata; 4682 } 4683 4684 /* 4685 * Helper routine used to lock all remaining members of a 4686 * large page. The caller is responsible for passing in a locked 4687 * pp. If pp is a large page, then it succeeds in locking all the 4688 * remaining constituent pages or it returns with only the 4689 * original page locked. 4690 * 4691 * Returns 1 on success, 0 on failure. 4692 * 4693 * If success is returned this routine guarantees p_szc for all constituent 4694 * pages of a large page pp belongs to can't change. To achieve this we 4695 * recheck szc of pp after locking all constituent pages and retry if szc 4696 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4697 * lock on one of constituent pages it can't be running after all constituent 4698 * pages are locked. hat_page_demote() with a lock on a constituent page 4699 * outside of this large page (i.e. pp belonged to a larger large page) is 4700 * already done with all constituent pages of pp since the root's p_szc is 4701 * changed last. Therefore no need to synchronize with hat_page_demote() that 4702 * locked a constituent page outside of pp's current large page. 4703 */ 4704 #ifdef DEBUG 4705 uint32_t gpg_trylock_mtbf = 0; 4706 #endif 4707 4708 int 4709 group_page_trylock(page_t *pp, se_t se) 4710 { 4711 page_t *tpp; 4712 pgcnt_t npgs, i, j; 4713 uint_t pszc = pp->p_szc; 4714 4715 #ifdef DEBUG 4716 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4717 return (0); 4718 } 4719 #endif 4720 4721 if (pp != PP_GROUPLEADER(pp, pszc)) { 4722 return (0); 4723 } 4724 4725 retry: 4726 ASSERT(PAGE_LOCKED_SE(pp, se)); 4727 ASSERT(!PP_ISFREE(pp)); 4728 if (pszc == 0) { 4729 return (1); 4730 } 4731 npgs = page_get_pagecnt(pszc); 4732 tpp = pp + 1; 4733 for (i = 1; i < npgs; i++, tpp++) { 4734 if (!page_trylock(tpp, se)) { 4735 tpp = pp + 1; 4736 for (j = 1; j < i; j++, tpp++) { 4737 page_unlock(tpp); 4738 } 4739 return (0); 4740 } 4741 } 4742 if (pp->p_szc != pszc) { 4743 ASSERT(pp->p_szc < pszc); 4744 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && 4745 !IS_SWAPFSVP(pp->p_vnode)); 4746 tpp = pp + 1; 4747 for (i = 1; i < npgs; i++, tpp++) { 4748 page_unlock(tpp); 4749 } 4750 pszc = pp->p_szc; 4751 goto retry; 4752 } 4753 return (1); 4754 } 4755 4756 void 4757 group_page_unlock(page_t *pp) 4758 { 4759 page_t *tpp; 4760 pgcnt_t npgs, i; 4761 4762 ASSERT(PAGE_LOCKED(pp)); 4763 ASSERT(!PP_ISFREE(pp)); 4764 ASSERT(pp == PP_PAGEROOT(pp)); 4765 npgs = page_get_pagecnt(pp->p_szc); 4766 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4767 page_unlock(tpp); 4768 } 4769 } 4770 4771 /* 4772 * returns 4773 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4774 * ERANGE : this is not a base page 4775 * EBUSY : failure to get locks on the page/pages 4776 * ENOMEM : failure to obtain replacement pages 4777 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4778 * EIO : An error occurred while trying to copy the page data 4779 * 4780 * Return with all constituent members of target and replacement 4781 * SE_EXCL locked. It is the callers responsibility to drop the 4782 * locks. 4783 */ 4784 int 4785 do_page_relocate( 4786 page_t **target, 4787 page_t **replacement, 4788 int grouplock, 4789 spgcnt_t *nrelocp, 4790 lgrp_t *lgrp) 4791 { 4792 page_t *first_repl; 4793 page_t *repl; 4794 page_t *targ; 4795 page_t *pl = NULL; 4796 uint_t ppattr; 4797 pfn_t pfn, repl_pfn; 4798 uint_t szc; 4799 spgcnt_t npgs, i; 4800 int repl_contig = 0; 4801 uint_t flags = 0; 4802 spgcnt_t dofree = 0; 4803 4804 *nrelocp = 0; 4805 4806 #if defined(__sparc) 4807 /* 4808 * We need to wait till OBP has completed 4809 * its boot-time handoff of its resources to the kernel 4810 * before we allow page relocation 4811 */ 4812 if (page_relocate_ready == 0) { 4813 return (EAGAIN); 4814 } 4815 #endif 4816 4817 /* 4818 * If this is not a base page, 4819 * just return with 0x0 pages relocated. 4820 */ 4821 targ = *target; 4822 ASSERT(PAGE_EXCL(targ)); 4823 ASSERT(!PP_ISFREE(targ)); 4824 szc = targ->p_szc; 4825 ASSERT(szc < mmu_page_sizes); 4826 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4827 pfn = targ->p_pagenum; 4828 if (pfn != PFN_BASE(pfn, szc)) { 4829 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4830 return (ERANGE); 4831 } 4832 4833 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4834 repl_pfn = repl->p_pagenum; 4835 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4836 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4837 return (ERANGE); 4838 } 4839 repl_contig = 1; 4840 } 4841 4842 /* 4843 * We must lock all members of this large page or we cannot 4844 * relocate any part of it. 4845 */ 4846 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4847 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4848 return (EBUSY); 4849 } 4850 4851 /* 4852 * reread szc it could have been decreased before 4853 * group_page_trylock() was done. 4854 */ 4855 szc = targ->p_szc; 4856 ASSERT(szc < mmu_page_sizes); 4857 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4858 ASSERT(pfn == PFN_BASE(pfn, szc)); 4859 4860 npgs = page_get_pagecnt(targ->p_szc); 4861 4862 if (repl == NULL) { 4863 dofree = npgs; /* Size of target page in MMU pages */ 4864 if (!page_create_wait(dofree, 0)) { 4865 if (grouplock != 0) { 4866 group_page_unlock(targ); 4867 } 4868 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4869 return (ENOMEM); 4870 } 4871 4872 /* 4873 * seg kmem pages require that the target and replacement 4874 * page be the same pagesize. 4875 */ 4876 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0; 4877 repl = page_get_replacement_page(targ, lgrp, flags); 4878 if (repl == NULL) { 4879 if (grouplock != 0) { 4880 group_page_unlock(targ); 4881 } 4882 page_create_putback(dofree); 4883 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4884 return (ENOMEM); 4885 } 4886 } 4887 #ifdef DEBUG 4888 else { 4889 ASSERT(PAGE_LOCKED(repl)); 4890 } 4891 #endif /* DEBUG */ 4892 4893 #if defined(__sparc) 4894 /* 4895 * Let hat_page_relocate() complete the relocation if it's kernel page 4896 */ 4897 if (VN_ISKAS(targ->p_vnode)) { 4898 *replacement = repl; 4899 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4900 if (grouplock != 0) { 4901 group_page_unlock(targ); 4902 } 4903 if (dofree) { 4904 *replacement = NULL; 4905 page_free_replacement_page(repl); 4906 page_create_putback(dofree); 4907 } 4908 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4909 return (EAGAIN); 4910 } 4911 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4912 return (0); 4913 } 4914 #else 4915 #if defined(lint) 4916 dofree = dofree; 4917 #endif 4918 #endif 4919 4920 first_repl = repl; 4921 4922 for (i = 0; i < npgs; i++) { 4923 ASSERT(PAGE_EXCL(targ)); 4924 ASSERT(targ->p_slckcnt == 0); 4925 ASSERT(repl->p_slckcnt == 0); 4926 4927 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4928 4929 ASSERT(hat_page_getshare(targ) == 0); 4930 ASSERT(!PP_ISFREE(targ)); 4931 ASSERT(targ->p_pagenum == (pfn + i)); 4932 ASSERT(repl_contig == 0 || 4933 repl->p_pagenum == (repl_pfn + i)); 4934 4935 /* 4936 * Copy the page contents and attributes then 4937 * relocate the page in the page hash. 4938 */ 4939 if (ppcopy(targ, repl) == 0) { 4940 targ = *target; 4941 repl = first_repl; 4942 VM_STAT_ADD(vmm_vmstats.ppr_copyfail); 4943 if (grouplock != 0) { 4944 group_page_unlock(targ); 4945 } 4946 if (dofree) { 4947 *replacement = NULL; 4948 page_free_replacement_page(repl); 4949 page_create_putback(dofree); 4950 } 4951 return (EIO); 4952 } 4953 4954 targ++; 4955 if (repl_contig != 0) { 4956 repl++; 4957 } else { 4958 repl = repl->p_next; 4959 } 4960 } 4961 4962 repl = first_repl; 4963 targ = *target; 4964 4965 for (i = 0; i < npgs; i++) { 4966 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4967 page_clr_all_props(repl); 4968 page_set_props(repl, ppattr); 4969 page_relocate_hash(repl, targ); 4970 4971 ASSERT(hat_page_getshare(targ) == 0); 4972 ASSERT(hat_page_getshare(repl) == 0); 4973 /* 4974 * Now clear the props on targ, after the 4975 * page_relocate_hash(), they no longer 4976 * have any meaning. 4977 */ 4978 page_clr_all_props(targ); 4979 ASSERT(targ->p_next == targ); 4980 ASSERT(targ->p_prev == targ); 4981 page_list_concat(&pl, &targ); 4982 4983 targ++; 4984 if (repl_contig != 0) { 4985 repl++; 4986 } else { 4987 repl = repl->p_next; 4988 } 4989 } 4990 /* assert that we have come full circle with repl */ 4991 ASSERT(repl_contig == 1 || first_repl == repl); 4992 4993 *target = pl; 4994 if (*replacement == NULL) { 4995 ASSERT(first_repl == repl); 4996 *replacement = repl; 4997 } 4998 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4999 *nrelocp = npgs; 5000 return (0); 5001 } 5002 /* 5003 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 5004 */ 5005 int 5006 page_relocate( 5007 page_t **target, 5008 page_t **replacement, 5009 int grouplock, 5010 int freetarget, 5011 spgcnt_t *nrelocp, 5012 lgrp_t *lgrp) 5013 { 5014 spgcnt_t ret; 5015 5016 /* do_page_relocate returns 0 on success or errno value */ 5017 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 5018 5019 if (ret != 0 || freetarget == 0) { 5020 return (ret); 5021 } 5022 if (*nrelocp == 1) { 5023 ASSERT(*target != NULL); 5024 page_free(*target, 1); 5025 } else { 5026 page_t *tpp = *target; 5027 uint_t szc = tpp->p_szc; 5028 pgcnt_t npgs = page_get_pagecnt(szc); 5029 ASSERT(npgs > 1); 5030 ASSERT(szc != 0); 5031 do { 5032 ASSERT(PAGE_EXCL(tpp)); 5033 ASSERT(!hat_page_is_mapped(tpp)); 5034 ASSERT(tpp->p_szc == szc); 5035 PP_SETFREE(tpp); 5036 PP_SETAGED(tpp); 5037 npgs--; 5038 } while ((tpp = tpp->p_next) != *target); 5039 ASSERT(npgs == 0); 5040 page_list_add_pages(*target, 0); 5041 npgs = page_get_pagecnt(szc); 5042 page_create_putback(npgs); 5043 } 5044 return (ret); 5045 } 5046 5047 /* 5048 * it is up to the caller to deal with pcf accounting. 5049 */ 5050 void 5051 page_free_replacement_page(page_t *pplist) 5052 { 5053 page_t *pp; 5054 5055 while (pplist != NULL) { 5056 /* 5057 * pp_targ is a linked list. 5058 */ 5059 pp = pplist; 5060 if (pp->p_szc == 0) { 5061 page_sub(&pplist, pp); 5062 page_clr_all_props(pp); 5063 PP_SETFREE(pp); 5064 PP_SETAGED(pp); 5065 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5066 page_unlock(pp); 5067 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5068 } else { 5069 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5070 page_t *tpp; 5071 page_list_break(&pp, &pplist, curnpgs); 5072 tpp = pp; 5073 do { 5074 ASSERT(PAGE_EXCL(tpp)); 5075 ASSERT(!hat_page_is_mapped(tpp)); 5076 page_clr_all_props(pp); 5077 PP_SETFREE(tpp); 5078 PP_SETAGED(tpp); 5079 } while ((tpp = tpp->p_next) != pp); 5080 page_list_add_pages(pp, 0); 5081 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5082 } 5083 } 5084 } 5085 5086 /* 5087 * Relocate target to non-relocatable replacement page. 5088 */ 5089 int 5090 page_relocate_cage(page_t **target, page_t **replacement) 5091 { 5092 page_t *tpp, *rpp; 5093 spgcnt_t pgcnt, npgs; 5094 int result; 5095 5096 tpp = *target; 5097 5098 ASSERT(PAGE_EXCL(tpp)); 5099 ASSERT(tpp->p_szc == 0); 5100 5101 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5102 5103 do { 5104 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5105 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5106 if (rpp == NULL) { 5107 page_create_putback(pgcnt); 5108 kcage_cageout_wakeup(); 5109 } 5110 } while (rpp == NULL); 5111 5112 ASSERT(PP_ISNORELOC(rpp)); 5113 5114 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5115 5116 if (result == 0) { 5117 *replacement = rpp; 5118 if (pgcnt != npgs) 5119 panic("page_relocate_cage: partial relocation"); 5120 } 5121 5122 return (result); 5123 } 5124 5125 /* 5126 * Release the page lock on a page, place on cachelist 5127 * tail if no longer mapped. Caller can let us know if 5128 * the page is known to be clean. 5129 */ 5130 int 5131 page_release(page_t *pp, int checkmod) 5132 { 5133 int status; 5134 5135 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5136 (pp->p_vnode != NULL)); 5137 5138 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5139 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5140 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5141 !hat_page_is_mapped(pp)) { 5142 5143 /* 5144 * If page is modified, unlock it 5145 * 5146 * (p_nrm & P_MOD) bit has the latest stuff because: 5147 * (1) We found that this page doesn't have any mappings 5148 * _after_ holding SE_EXCL and 5149 * (2) We didn't drop SE_EXCL lock after the check in (1) 5150 */ 5151 if (checkmod && hat_ismod(pp)) { 5152 page_unlock(pp); 5153 status = PGREL_MOD; 5154 } else { 5155 /*LINTED: constant in conditional context*/ 5156 VN_DISPOSE(pp, B_FREE, 0, kcred); 5157 status = PGREL_CLEAN; 5158 } 5159 } else { 5160 page_unlock(pp); 5161 status = PGREL_NOTREL; 5162 } 5163 return (status); 5164 } 5165 5166 /* 5167 * Given a constituent page, try to demote the large page on the freelist. 5168 * 5169 * Returns nonzero if the page could be demoted successfully. Returns with 5170 * the constituent page still locked. 5171 */ 5172 int 5173 page_try_demote_free_pages(page_t *pp) 5174 { 5175 page_t *rootpp = pp; 5176 pfn_t pfn = page_pptonum(pp); 5177 spgcnt_t npgs; 5178 uint_t szc = pp->p_szc; 5179 5180 ASSERT(PP_ISFREE(pp)); 5181 ASSERT(PAGE_EXCL(pp)); 5182 5183 /* 5184 * Adjust rootpp and lock it, if `pp' is not the base 5185 * constituent page. 5186 */ 5187 npgs = page_get_pagecnt(pp->p_szc); 5188 if (npgs == 1) { 5189 return (0); 5190 } 5191 5192 if (!IS_P2ALIGNED(pfn, npgs)) { 5193 pfn = P2ALIGN(pfn, npgs); 5194 rootpp = page_numtopp_nolock(pfn); 5195 } 5196 5197 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5198 return (0); 5199 } 5200 5201 if (rootpp->p_szc != szc) { 5202 if (pp != rootpp) 5203 page_unlock(rootpp); 5204 return (0); 5205 } 5206 5207 page_demote_free_pages(rootpp); 5208 5209 if (pp != rootpp) 5210 page_unlock(rootpp); 5211 5212 ASSERT(PP_ISFREE(pp)); 5213 ASSERT(PAGE_EXCL(pp)); 5214 return (1); 5215 } 5216 5217 /* 5218 * Given a constituent page, try to demote the large page. 5219 * 5220 * Returns nonzero if the page could be demoted successfully. Returns with 5221 * the constituent page still locked. 5222 */ 5223 int 5224 page_try_demote_pages(page_t *pp) 5225 { 5226 page_t *tpp, *rootpp = pp; 5227 pfn_t pfn = page_pptonum(pp); 5228 spgcnt_t i, npgs; 5229 uint_t szc = pp->p_szc; 5230 vnode_t *vp = pp->p_vnode; 5231 5232 ASSERT(PAGE_EXCL(pp)); 5233 5234 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5235 5236 if (pp->p_szc == 0) { 5237 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5238 return (1); 5239 } 5240 5241 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) { 5242 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5243 page_demote_vp_pages(pp); 5244 ASSERT(pp->p_szc == 0); 5245 return (1); 5246 } 5247 5248 /* 5249 * Adjust rootpp if passed in is not the base 5250 * constituent page. 5251 */ 5252 npgs = page_get_pagecnt(pp->p_szc); 5253 ASSERT(npgs > 1); 5254 if (!IS_P2ALIGNED(pfn, npgs)) { 5255 pfn = P2ALIGN(pfn, npgs); 5256 rootpp = page_numtopp_nolock(pfn); 5257 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5258 ASSERT(rootpp->p_vnode != NULL); 5259 ASSERT(rootpp->p_szc == szc); 5260 } 5261 5262 /* 5263 * We can't demote kernel pages since we can't hat_unload() 5264 * the mappings. 5265 */ 5266 if (VN_ISKAS(rootpp->p_vnode)) 5267 return (0); 5268 5269 /* 5270 * Attempt to lock all constituent pages except the page passed 5271 * in since it's already locked. 5272 */ 5273 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5274 ASSERT(!PP_ISFREE(tpp)); 5275 ASSERT(tpp->p_vnode != NULL); 5276 5277 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5278 break; 5279 ASSERT(tpp->p_szc == rootpp->p_szc); 5280 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5281 } 5282 5283 /* 5284 * If we failed to lock them all then unlock what we have 5285 * locked so far and bail. 5286 */ 5287 if (i < npgs) { 5288 tpp = rootpp; 5289 while (i-- > 0) { 5290 if (tpp != pp) 5291 page_unlock(tpp); 5292 tpp++; 5293 } 5294 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5295 return (0); 5296 } 5297 5298 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5299 ASSERT(PAGE_EXCL(tpp)); 5300 ASSERT(tpp->p_slckcnt == 0); 5301 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5302 tpp->p_szc = 0; 5303 } 5304 5305 /* 5306 * Unlock all pages except the page passed in. 5307 */ 5308 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5309 ASSERT(!hat_page_is_mapped(tpp)); 5310 if (tpp != pp) 5311 page_unlock(tpp); 5312 } 5313 5314 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5315 return (1); 5316 } 5317 5318 /* 5319 * Called by page_free() and page_destroy() to demote the page size code 5320 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5321 * p_szc on free list, neither can we just clear p_szc of a single page_t 5322 * within a large page since it will break other code that relies on p_szc 5323 * being the same for all page_t's of a large page). Anonymous pages should 5324 * never end up here because anon_map_getpages() cannot deal with p_szc 5325 * changes after a single constituent page is locked. While anonymous or 5326 * kernel large pages are demoted or freed the entire large page at a time 5327 * with all constituent pages locked EXCL for the file system pages we 5328 * have to be able to demote a large page (i.e. decrease all constituent pages 5329 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5330 * we can easily deal with anonymous page demotion the entire large page at a 5331 * time is that those operation originate at address space level and concern 5332 * the entire large page region with actual demotion only done when pages are 5333 * not shared with any other processes (therefore we can always get EXCL lock 5334 * on all anonymous constituent pages after clearing segment page 5335 * cache). However file system pages can be truncated or invalidated at a 5336 * PAGESIZE level from the file system side and end up in page_free() or 5337 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5338 * and therefore pageout should be able to demote a large page by EXCL locking 5339 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5340 * rely on being able to lock EXCL all constituent pages. 5341 * 5342 * To prevent szc changes on file system pages one has to lock all constituent 5343 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5344 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5345 * prevent szc changes is hat layer that uses its own page level mlist 5346 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5347 * taken. Therefore we need to change szc under hat level locks if we only 5348 * have an EXCL lock on a single constituent page and hat still references any 5349 * of constituent pages. (Note we can't "ignore" hat layer by simply 5350 * hat_pageunload() all constituent pages without having EXCL locks on all of 5351 * constituent pages). We use hat_page_demote() call to safely demote szc of 5352 * all constituent pages under hat locks when we only have an EXCL lock on one 5353 * of constituent pages. 5354 * 5355 * This routine calls page_szc_lock() before calling hat_page_demote() to 5356 * allow segvn in one special case not to lock all constituent pages SHARED 5357 * before calling hat_memload_array() that relies on p_szc not changing even 5358 * before hat level mlist lock is taken. In that case segvn uses 5359 * page_szc_lock() to prevent hat_page_demote() changing p_szc values. 5360 * 5361 * Anonymous or kernel page demotion still has to lock all pages exclusively 5362 * and do hat_pageunload() on all constituent pages before demoting the page 5363 * therefore there's no need for anonymous or kernel page demotion to use 5364 * hat_page_demote() mechanism. 5365 * 5366 * hat_page_demote() removes all large mappings that map pp and then decreases 5367 * p_szc starting from the last constituent page of the large page. By working 5368 * from the tail of a large page in pfn decreasing order allows one looking at 5369 * the root page to know that hat_page_demote() is done for root's szc area. 5370 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5371 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5372 * that started on this page when it had szc > 1 is done for this szc 1 area. 5373 * 5374 * We are guaranteed that all constituent pages of pp's large page belong to 5375 * the same vnode with the consecutive offsets increasing in the direction of 5376 * the pfn i.e. the identity of constituent pages can't change until their 5377 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5378 * large mappings to pp even though we don't lock any constituent page except 5379 * pp (i.e. we won't unload e.g. kernel locked page). 5380 */ 5381 static void 5382 page_demote_vp_pages(page_t *pp) 5383 { 5384 kmutex_t *mtx; 5385 5386 ASSERT(PAGE_EXCL(pp)); 5387 ASSERT(!PP_ISFREE(pp)); 5388 ASSERT(pp->p_vnode != NULL); 5389 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5390 ASSERT(!PP_ISKAS(pp)); 5391 5392 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5393 5394 mtx = page_szc_lock(pp); 5395 if (mtx != NULL) { 5396 hat_page_demote(pp); 5397 mutex_exit(mtx); 5398 } 5399 ASSERT(pp->p_szc == 0); 5400 } 5401 5402 /* 5403 * Mark any existing pages for migration in the given range 5404 */ 5405 void 5406 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5407 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5408 u_offset_t vnoff, int rflag) 5409 { 5410 struct anon *ap; 5411 vnode_t *curvp; 5412 lgrp_t *from; 5413 pgcnt_t i; 5414 pgcnt_t nlocked; 5415 u_offset_t off; 5416 pfn_t pfn; 5417 size_t pgsz; 5418 size_t segpgsz; 5419 pgcnt_t pages; 5420 uint_t pszc; 5421 page_t **ppa; 5422 pgcnt_t ppa_nentries; 5423 page_t *pp; 5424 caddr_t va; 5425 ulong_t an_idx; 5426 anon_sync_obj_t cookie; 5427 5428 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5429 5430 /* 5431 * Don't do anything if don't need to do lgroup optimizations 5432 * on this system 5433 */ 5434 if (!lgrp_optimizations()) 5435 return; 5436 5437 /* 5438 * Align address and length to (potentially large) page boundary 5439 */ 5440 segpgsz = page_get_pagesize(seg->s_szc); 5441 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5442 if (rflag) 5443 len = P2ROUNDUP(len, segpgsz); 5444 5445 /* 5446 * Allocate page array to accommodate largest page size 5447 */ 5448 pgsz = page_get_pagesize(page_num_pagesizes() - 1); 5449 ppa_nentries = btop(pgsz); 5450 ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP); 5451 5452 /* 5453 * Do one (large) page at a time 5454 */ 5455 va = addr; 5456 while (va < addr + len) { 5457 /* 5458 * Lookup (root) page for vnode and offset corresponding to 5459 * this virtual address 5460 * Try anonmap first since there may be copy-on-write 5461 * pages, but initialize vnode pointer and offset using 5462 * vnode arguments just in case there isn't an amp. 5463 */ 5464 curvp = vp; 5465 off = vnoff + va - seg->s_base; 5466 if (amp) { 5467 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5468 an_idx = anon_index + seg_page(seg, va); 5469 anon_array_enter(amp, an_idx, &cookie); 5470 ap = anon_get_ptr(amp->ahp, an_idx); 5471 if (ap) 5472 swap_xlate(ap, &curvp, &off); 5473 anon_array_exit(&cookie); 5474 ANON_LOCK_EXIT(&->a_rwlock); 5475 } 5476 5477 pp = NULL; 5478 if (curvp) 5479 pp = page_lookup(curvp, off, SE_SHARED); 5480 5481 /* 5482 * If there isn't a page at this virtual address, 5483 * skip to next page 5484 */ 5485 if (pp == NULL) { 5486 va += PAGESIZE; 5487 continue; 5488 } 5489 5490 /* 5491 * Figure out which lgroup this page is in for kstats 5492 */ 5493 pfn = page_pptonum(pp); 5494 from = lgrp_pfn_to_lgrp(pfn); 5495 5496 /* 5497 * Get page size, and round up and skip to next page boundary 5498 * if unaligned address 5499 */ 5500 pszc = pp->p_szc; 5501 pgsz = page_get_pagesize(pszc); 5502 pages = btop(pgsz); 5503 if (!IS_P2ALIGNED(va, pgsz) || 5504 !IS_P2ALIGNED(pfn, pages) || 5505 pgsz > segpgsz) { 5506 pgsz = MIN(pgsz, segpgsz); 5507 page_unlock(pp); 5508 i = btop(P2END((uintptr_t)va, pgsz) - 5509 (uintptr_t)va); 5510 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5511 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i); 5512 continue; 5513 } 5514 5515 /* 5516 * Upgrade to exclusive lock on page 5517 */ 5518 if (!page_tryupgrade(pp)) { 5519 page_unlock(pp); 5520 va += pgsz; 5521 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5522 btop(pgsz)); 5523 continue; 5524 } 5525 5526 /* 5527 * Remember pages locked exclusively and how many 5528 */ 5529 ppa[0] = pp; 5530 nlocked = 1; 5531 5532 /* 5533 * Lock constituent pages if this is large page 5534 */ 5535 if (pages > 1) { 5536 /* 5537 * Lock all constituents except root page, since it 5538 * should be locked already. 5539 */ 5540 for (i = 1; i < pages; i++) { 5541 pp++; 5542 if (!page_trylock(pp, SE_EXCL)) { 5543 break; 5544 } 5545 if (PP_ISFREE(pp) || 5546 pp->p_szc != pszc) { 5547 /* 5548 * hat_page_demote() raced in with us. 5549 */ 5550 ASSERT(!IS_SWAPFSVP(curvp)); 5551 page_unlock(pp); 5552 break; 5553 } 5554 ppa[nlocked] = pp; 5555 nlocked++; 5556 } 5557 } 5558 5559 /* 5560 * If all constituent pages couldn't be locked, 5561 * unlock pages locked so far and skip to next page. 5562 */ 5563 if (nlocked != pages) { 5564 for (i = 0; i < nlocked; i++) 5565 page_unlock(ppa[i]); 5566 va += pgsz; 5567 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5568 btop(pgsz)); 5569 continue; 5570 } 5571 5572 /* 5573 * hat_page_demote() can no longer happen 5574 * since last cons page had the right p_szc after 5575 * all cons pages were locked. all cons pages 5576 * should now have the same p_szc. 5577 */ 5578 5579 /* 5580 * All constituent pages locked successfully, so mark 5581 * large page for migration and unload the mappings of 5582 * constituent pages, so a fault will occur on any part of the 5583 * large page 5584 */ 5585 PP_SETMIGRATE(ppa[0]); 5586 for (i = 0; i < nlocked; i++) { 5587 pp = ppa[i]; 5588 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 5589 ASSERT(hat_page_getshare(pp) == 0); 5590 page_unlock(pp); 5591 } 5592 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5593 5594 va += pgsz; 5595 } 5596 kmem_free(ppa, ppa_nentries * sizeof (page_t *)); 5597 } 5598 5599 /* 5600 * Migrate any pages that have been marked for migration in the given range 5601 */ 5602 void 5603 page_migrate( 5604 struct seg *seg, 5605 caddr_t addr, 5606 page_t **ppa, 5607 pgcnt_t npages) 5608 { 5609 lgrp_t *from; 5610 lgrp_t *to; 5611 page_t *newpp; 5612 page_t *pp; 5613 pfn_t pfn; 5614 size_t pgsz; 5615 spgcnt_t page_cnt; 5616 spgcnt_t i; 5617 uint_t pszc; 5618 5619 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5620 5621 while (npages > 0) { 5622 pp = *ppa; 5623 pszc = pp->p_szc; 5624 pgsz = page_get_pagesize(pszc); 5625 page_cnt = btop(pgsz); 5626 5627 /* 5628 * Check to see whether this page is marked for migration 5629 * 5630 * Assume that root page of large page is marked for 5631 * migration and none of the other constituent pages 5632 * are marked. This really simplifies clearing the 5633 * migrate bit by not having to clear it from each 5634 * constituent page. 5635 * 5636 * note we don't want to relocate an entire large page if 5637 * someone is only using one subpage. 5638 */ 5639 if (npages < page_cnt) 5640 break; 5641 5642 /* 5643 * Is it marked for migration? 5644 */ 5645 if (!PP_ISMIGRATE(pp)) 5646 goto next; 5647 5648 /* 5649 * Determine lgroups that page is being migrated between 5650 */ 5651 pfn = page_pptonum(pp); 5652 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5653 break; 5654 } 5655 from = lgrp_pfn_to_lgrp(pfn); 5656 to = lgrp_mem_choose(seg, addr, pgsz); 5657 5658 /* 5659 * Check to see whether we are trying to migrate page to lgroup 5660 * where it is allocated already 5661 */ 5662 if (to == from) { 5663 PP_CLRMIGRATE(pp); 5664 goto next; 5665 } 5666 5667 /* 5668 * Need to get exclusive lock's to migrate 5669 */ 5670 for (i = 0; i < page_cnt; i++) { 5671 ASSERT(PAGE_LOCKED(ppa[i])); 5672 if (page_pptonum(ppa[i]) != pfn + i || 5673 ppa[i]->p_szc != pszc) { 5674 break; 5675 } 5676 if (!page_tryupgrade(ppa[i])) { 5677 lgrp_stat_add(from->lgrp_id, 5678 LGRP_PM_FAIL_LOCK_PGS, 5679 page_cnt); 5680 break; 5681 } 5682 } 5683 if (i != page_cnt) { 5684 while (--i != -1) { 5685 page_downgrade(ppa[i]); 5686 } 5687 goto next; 5688 } 5689 5690 (void) page_create_wait(page_cnt, PG_WAIT); 5691 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5692 if (newpp == NULL) { 5693 page_create_putback(page_cnt); 5694 for (i = 0; i < page_cnt; i++) { 5695 page_downgrade(ppa[i]); 5696 } 5697 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5698 page_cnt); 5699 goto next; 5700 } 5701 ASSERT(newpp->p_szc == pszc); 5702 /* 5703 * Clear migrate bit and relocate page 5704 */ 5705 PP_CLRMIGRATE(pp); 5706 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5707 panic("page_migrate: page_relocate failed"); 5708 } 5709 ASSERT(page_cnt * PAGESIZE == pgsz); 5710 5711 /* 5712 * Keep stats for number of pages migrated from and to 5713 * each lgroup 5714 */ 5715 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5716 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5717 /* 5718 * update the page_t array we were passed in and 5719 * unlink constituent pages of a large page. 5720 */ 5721 for (i = 0; i < page_cnt; ++i, ++pp) { 5722 ASSERT(PAGE_EXCL(newpp)); 5723 ASSERT(newpp->p_szc == pszc); 5724 ppa[i] = newpp; 5725 pp = newpp; 5726 page_sub(&newpp, pp); 5727 page_downgrade(pp); 5728 } 5729 ASSERT(newpp == NULL); 5730 next: 5731 addr += pgsz; 5732 ppa += page_cnt; 5733 npages -= page_cnt; 5734 } 5735 } 5736 5737 ulong_t mem_waiters = 0; 5738 ulong_t max_count = 20; 5739 #define MAX_DELAY 0x1ff 5740 5741 /* 5742 * Check if enough memory is available to proceed. 5743 * Depending on system configuration and how much memory is 5744 * reserved for swap we need to check against two variables. 5745 * e.g. on systems with little physical swap availrmem can be 5746 * more reliable indicator of how much memory is available. 5747 * On systems with large phys swap freemem can be better indicator. 5748 * If freemem drops below threshold level don't return an error 5749 * immediately but wake up pageout to free memory and block. 5750 * This is done number of times. If pageout is not able to free 5751 * memory within certain time return an error. 5752 * The same applies for availrmem but kmem_reap is used to 5753 * free memory. 5754 */ 5755 int 5756 page_mem_avail(pgcnt_t npages) 5757 { 5758 ulong_t count; 5759 5760 #if defined(__i386) 5761 if (freemem > desfree + npages && 5762 availrmem > swapfs_reserve + npages && 5763 btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem + 5764 npages) 5765 return (1); 5766 #else 5767 if (freemem > desfree + npages && 5768 availrmem > swapfs_reserve + npages) 5769 return (1); 5770 #endif 5771 5772 count = max_count; 5773 atomic_add_long(&mem_waiters, 1); 5774 5775 while (freemem < desfree + npages && --count) { 5776 cv_signal(&proc_pageout->p_cv); 5777 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5778 atomic_add_long(&mem_waiters, -1); 5779 return (0); 5780 } 5781 } 5782 if (count == 0) { 5783 atomic_add_long(&mem_waiters, -1); 5784 return (0); 5785 } 5786 5787 count = max_count; 5788 while (availrmem < swapfs_reserve + npages && --count) { 5789 kmem_reap(); 5790 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5791 atomic_add_long(&mem_waiters, -1); 5792 return (0); 5793 } 5794 } 5795 atomic_add_long(&mem_waiters, -1); 5796 if (count == 0) 5797 return (0); 5798 5799 #if defined(__i386) 5800 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 5801 tune.t_minarmem + npages) 5802 return (0); 5803 #endif 5804 return (1); 5805 } 5806 5807 #define MAX_CNT 60 /* max num of iterations */ 5808 /* 5809 * Reclaim/reserve availrmem for npages. 5810 * If there is not enough memory start reaping seg, kmem caches. 5811 * Start pageout scanner (via page_needfree()). 5812 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5813 * Note: There is no guarantee that any availrmem will be freed as 5814 * this memory typically is locked (kernel heap) or reserved for swap. 5815 * Also due to memory fragmentation kmem allocator may not be able 5816 * to free any memory (single user allocated buffer will prevent 5817 * freeing slab or a page). 5818 */ 5819 int 5820 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5821 { 5822 int i = 0; 5823 int ret = 0; 5824 pgcnt_t deficit; 5825 pgcnt_t old_availrmem; 5826 5827 mutex_enter(&freemem_lock); 5828 old_availrmem = availrmem - 1; 5829 while ((availrmem < tune.t_minarmem + npages + epages) && 5830 (old_availrmem < availrmem) && (i++ < MAX_CNT)) { 5831 old_availrmem = availrmem; 5832 deficit = tune.t_minarmem + npages + epages - availrmem; 5833 mutex_exit(&freemem_lock); 5834 page_needfree(deficit); 5835 seg_preap(); 5836 kmem_reap(); 5837 delay(hz); 5838 page_needfree(-(spgcnt_t)deficit); 5839 mutex_enter(&freemem_lock); 5840 } 5841 5842 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5843 availrmem -= npages; 5844 ret = 1; 5845 } 5846 5847 mutex_exit(&freemem_lock); 5848 5849 return (ret); 5850 } 5851 5852 /* 5853 * Search the memory segments to locate the desired page. Within a 5854 * segment, pages increase linearly with one page structure per 5855 * physical page frame (size PAGESIZE). The search begins 5856 * with the segment that was accessed last, to take advantage of locality. 5857 * If the hint misses, we start from the beginning of the sorted memseg list 5858 */ 5859 5860 5861 /* 5862 * Some data structures for pfn to pp lookup. 5863 */ 5864 ulong_t mhash_per_slot; 5865 struct memseg *memseg_hash[N_MEM_SLOTS]; 5866 5867 page_t * 5868 page_numtopp_nolock(pfn_t pfnum) 5869 { 5870 struct memseg *seg; 5871 page_t *pp; 5872 vm_cpu_data_t *vc = CPU->cpu_vm_data; 5873 5874 ASSERT(vc != NULL); 5875 5876 MEMSEG_STAT_INCR(nsearch); 5877 5878 /* Try last winner first */ 5879 if (((seg = vc->vc_pnum_memseg) != NULL) && 5880 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5881 MEMSEG_STAT_INCR(nlastwon); 5882 pp = seg->pages + (pfnum - seg->pages_base); 5883 if (pp->p_pagenum == pfnum) 5884 return ((page_t *)pp); 5885 } 5886 5887 /* Else Try hash */ 5888 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5889 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5890 MEMSEG_STAT_INCR(nhashwon); 5891 vc->vc_pnum_memseg = seg; 5892 pp = seg->pages + (pfnum - seg->pages_base); 5893 if (pp->p_pagenum == pfnum) 5894 return ((page_t *)pp); 5895 } 5896 5897 /* Else Brute force */ 5898 for (seg = memsegs; seg != NULL; seg = seg->next) { 5899 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5900 vc->vc_pnum_memseg = seg; 5901 pp = seg->pages + (pfnum - seg->pages_base); 5902 return ((page_t *)pp); 5903 } 5904 } 5905 vc->vc_pnum_memseg = NULL; 5906 MEMSEG_STAT_INCR(nnotfound); 5907 return ((page_t *)NULL); 5908 5909 } 5910 5911 struct memseg * 5912 page_numtomemseg_nolock(pfn_t pfnum) 5913 { 5914 struct memseg *seg; 5915 page_t *pp; 5916 5917 /* Try hash */ 5918 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5919 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5920 pp = seg->pages + (pfnum - seg->pages_base); 5921 if (pp->p_pagenum == pfnum) 5922 return (seg); 5923 } 5924 5925 /* Else Brute force */ 5926 for (seg = memsegs; seg != NULL; seg = seg->next) { 5927 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5928 return (seg); 5929 } 5930 } 5931 return ((struct memseg *)NULL); 5932 } 5933 5934 /* 5935 * Given a page and a count return the page struct that is 5936 * n structs away from the current one in the global page 5937 * list. 5938 * 5939 * This function wraps to the first page upon 5940 * reaching the end of the memseg list. 5941 */ 5942 page_t * 5943 page_nextn(page_t *pp, ulong_t n) 5944 { 5945 struct memseg *seg; 5946 page_t *ppn; 5947 vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5948 5949 ASSERT(vc != NULL); 5950 5951 if (((seg = vc->vc_pnext_memseg) == NULL) || 5952 (seg->pages_base == seg->pages_end) || 5953 !(pp >= seg->pages && pp < seg->epages)) { 5954 5955 for (seg = memsegs; seg; seg = seg->next) { 5956 if (pp >= seg->pages && pp < seg->epages) 5957 break; 5958 } 5959 5960 if (seg == NULL) { 5961 /* Memory delete got in, return something valid. */ 5962 /* TODO: fix me. */ 5963 seg = memsegs; 5964 pp = seg->pages; 5965 } 5966 } 5967 5968 /* check for wraparound - possible if n is large */ 5969 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5970 n -= seg->epages - pp; 5971 seg = seg->next; 5972 if (seg == NULL) 5973 seg = memsegs; 5974 pp = seg->pages; 5975 } 5976 vc->vc_pnext_memseg = seg; 5977 return (ppn); 5978 } 5979 5980 /* 5981 * Initialize for a loop using page_next_scan_large(). 5982 */ 5983 page_t * 5984 page_next_scan_init(void **cookie) 5985 { 5986 ASSERT(cookie != NULL); 5987 *cookie = (void *)memsegs; 5988 return ((page_t *)memsegs->pages); 5989 } 5990 5991 /* 5992 * Return the next page in a scan of page_t's, assuming we want 5993 * to skip over sub-pages within larger page sizes. 5994 * 5995 * The cookie is used to keep track of the current memseg. 5996 */ 5997 page_t * 5998 page_next_scan_large( 5999 page_t *pp, 6000 ulong_t *n, 6001 void **cookie) 6002 { 6003 struct memseg *seg = (struct memseg *)*cookie; 6004 page_t *new_pp; 6005 ulong_t cnt; 6006 pfn_t pfn; 6007 6008 6009 /* 6010 * get the count of page_t's to skip based on the page size 6011 */ 6012 ASSERT(pp != NULL); 6013 if (pp->p_szc == 0) { 6014 cnt = 1; 6015 } else { 6016 pfn = page_pptonum(pp); 6017 cnt = page_get_pagecnt(pp->p_szc); 6018 cnt -= pfn & (cnt - 1); 6019 } 6020 *n += cnt; 6021 new_pp = pp + cnt; 6022 6023 /* 6024 * Catch if we went past the end of the current memory segment. If so, 6025 * just move to the next segment with pages. 6026 */ 6027 if (new_pp >= seg->epages) { 6028 do { 6029 seg = seg->next; 6030 if (seg == NULL) 6031 seg = memsegs; 6032 } while (seg->pages == seg->epages); 6033 new_pp = seg->pages; 6034 *cookie = (void *)seg; 6035 } 6036 6037 return (new_pp); 6038 } 6039 6040 6041 /* 6042 * Returns next page in list. Note: this function wraps 6043 * to the first page in the list upon reaching the end 6044 * of the list. Callers should be aware of this fact. 6045 */ 6046 6047 /* We should change this be a #define */ 6048 6049 page_t * 6050 page_next(page_t *pp) 6051 { 6052 return (page_nextn(pp, 1)); 6053 } 6054 6055 page_t * 6056 page_first() 6057 { 6058 return ((page_t *)memsegs->pages); 6059 } 6060 6061 6062 /* 6063 * This routine is called at boot with the initial memory configuration 6064 * and when memory is added or removed. 6065 */ 6066 void 6067 build_pfn_hash() 6068 { 6069 pfn_t cur; 6070 pgcnt_t index; 6071 struct memseg *pseg; 6072 int i; 6073 6074 /* 6075 * Clear memseg_hash array. 6076 * Since memory add/delete is designed to operate concurrently 6077 * with normal operation, the hash rebuild must be able to run 6078 * concurrently with page_numtopp_nolock(). To support this 6079 * functionality, assignments to memseg_hash array members must 6080 * be done atomically. 6081 * 6082 * NOTE: bzero() does not currently guarantee this for kernel 6083 * threads, and cannot be used here. 6084 */ 6085 for (i = 0; i < N_MEM_SLOTS; i++) 6086 memseg_hash[i] = NULL; 6087 6088 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6089 6090 /* 6091 * Physmax is the last valid pfn. 6092 */ 6093 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6094 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6095 index = MEMSEG_PFN_HASH(pseg->pages_base); 6096 cur = pseg->pages_base; 6097 do { 6098 if (index >= N_MEM_SLOTS) 6099 index = MEMSEG_PFN_HASH(cur); 6100 6101 if (memseg_hash[index] == NULL || 6102 memseg_hash[index]->pages_base > pseg->pages_base) { 6103 memseg_hash[index] = pseg; 6104 hat_kpm_mseghash_update(index, pseg); 6105 } 6106 cur += mhash_per_slot; 6107 index++; 6108 } while (cur < pseg->pages_end); 6109 } 6110 } 6111 6112 /* 6113 * Return the pagenum for the pp 6114 */ 6115 pfn_t 6116 page_pptonum(page_t *pp) 6117 { 6118 return (pp->p_pagenum); 6119 } 6120 6121 /* 6122 * interface to the referenced and modified etc bits 6123 * in the PSM part of the page struct 6124 * when no locking is desired. 6125 */ 6126 void 6127 page_set_props(page_t *pp, uint_t flags) 6128 { 6129 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6130 pp->p_nrm |= (uchar_t)flags; 6131 } 6132 6133 void 6134 page_clr_all_props(page_t *pp) 6135 { 6136 pp->p_nrm = 0; 6137 } 6138 6139 /* 6140 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6141 */ 6142 int 6143 page_clear_lck_cow(page_t *pp, int adjust) 6144 { 6145 int f_amount; 6146 6147 ASSERT(PAGE_EXCL(pp)); 6148 6149 /* 6150 * The page_struct_lock need not be acquired here since 6151 * we require the caller hold the page exclusively locked. 6152 */ 6153 f_amount = 0; 6154 if (pp->p_lckcnt) { 6155 f_amount = 1; 6156 pp->p_lckcnt = 0; 6157 } 6158 if (pp->p_cowcnt) { 6159 f_amount += pp->p_cowcnt; 6160 pp->p_cowcnt = 0; 6161 } 6162 6163 if (adjust && f_amount) { 6164 mutex_enter(&freemem_lock); 6165 availrmem += f_amount; 6166 mutex_exit(&freemem_lock); 6167 } 6168 6169 return (f_amount); 6170 } 6171 6172 /* 6173 * The following functions is called from free_vp_pages() 6174 * for an inexact estimate of a newly free'd page... 6175 */ 6176 ulong_t 6177 page_share_cnt(page_t *pp) 6178 { 6179 return (hat_page_getshare(pp)); 6180 } 6181 6182 int 6183 page_isshared(page_t *pp) 6184 { 6185 return (hat_page_checkshare(pp, 1)); 6186 } 6187 6188 int 6189 page_isfree(page_t *pp) 6190 { 6191 return (PP_ISFREE(pp)); 6192 } 6193 6194 int 6195 page_isref(page_t *pp) 6196 { 6197 return (hat_page_getattr(pp, P_REF)); 6198 } 6199 6200 int 6201 page_ismod(page_t *pp) 6202 { 6203 return (hat_page_getattr(pp, P_MOD)); 6204 } 6205 6206 /* 6207 * The following code all currently relates to the page capture logic: 6208 * 6209 * This logic is used for cases where there is a desire to claim a certain 6210 * physical page in the system for the caller. As it may not be possible 6211 * to capture the page immediately, the p_toxic bits are used in the page 6212 * structure to indicate that someone wants to capture this page. When the 6213 * page gets unlocked, the toxic flag will be noted and an attempt to capture 6214 * the page will be made. If it is successful, the original callers callback 6215 * will be called with the page to do with it what they please. 6216 * 6217 * There is also an async thread which wakes up to attempt to capture 6218 * pages occasionally which have the capture bit set. All of the pages which 6219 * need to be captured asynchronously have been inserted into the 6220 * page_capture_hash and thus this thread walks that hash list. Items in the 6221 * hash have an expiration time so this thread handles that as well by removing 6222 * the item from the hash if it has expired. 6223 * 6224 * Some important things to note are: 6225 * - if the PR_CAPTURE bit is set on a page, then the page is in the 6226 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed 6227 * to set and clear this bit, and while the lock is held is the only time 6228 * you can add or remove an entry from the hash. 6229 * - the PR_CAPTURE bit can only be set and cleared while holding the 6230 * page_capture_hash_head.pchh_mutex 6231 * - the t_flag field of the thread struct is used with the T_CAPTURING 6232 * flag to prevent recursion while dealing with large pages. 6233 * - pages which need to be retired never expire on the page_capture_hash. 6234 */ 6235 6236 static void page_capture_thread(void); 6237 static kthread_t *pc_thread_id; 6238 kcondvar_t pc_cv; 6239 static kmutex_t pc_thread_mutex; 6240 static clock_t pc_thread_shortwait; 6241 static clock_t pc_thread_longwait; 6242 static int pc_thread_ism_retry; 6243 6244 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS]; 6245 6246 /* Note that this is a circular linked list */ 6247 typedef struct page_capture_hash_bucket { 6248 page_t *pp; 6249 uint_t szc; 6250 uint_t flags; 6251 clock_t expires; /* lbolt at which this request expires. */ 6252 void *datap; /* Cached data passed in for callback */ 6253 struct page_capture_hash_bucket *next; 6254 struct page_capture_hash_bucket *prev; 6255 } page_capture_hash_bucket_t; 6256 6257 /* 6258 * Each hash bucket will have it's own mutex and two lists which are: 6259 * active (0): represents requests which have not been processed by 6260 * the page_capture async thread yet. 6261 * walked (1): represents requests which have been processed by the 6262 * page_capture async thread within it's given walk of this bucket. 6263 * 6264 * These are all needed so that we can synchronize all async page_capture 6265 * events. When the async thread moves to a new bucket, it will append the 6266 * walked list to the active list and walk each item one at a time, moving it 6267 * from the active list to the walked list. Thus if there is an async request 6268 * outstanding for a given page, it will always be in one of the two lists. 6269 * New requests will always be added to the active list. 6270 * If we were not able to capture a page before the request expired, we'd free 6271 * up the request structure which would indicate to page_capture that there is 6272 * no longer a need for the given page, and clear the PR_CAPTURE flag if 6273 * possible. 6274 */ 6275 typedef struct page_capture_hash_head { 6276 kmutex_t pchh_mutex; 6277 uint_t num_pages; 6278 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */ 6279 } page_capture_hash_head_t; 6280 6281 #ifdef DEBUG 6282 #define NUM_PAGE_CAPTURE_BUCKETS 4 6283 #else 6284 #define NUM_PAGE_CAPTURE_BUCKETS 64 6285 #endif 6286 6287 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS]; 6288 6289 /* for now use a very simple hash based upon the size of a page struct */ 6290 #define PAGE_CAPTURE_HASH(pp) \ 6291 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1))) 6292 6293 extern pgcnt_t swapfs_minfree; 6294 6295 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap); 6296 6297 /* 6298 * a callback function is required for page capture requests. 6299 */ 6300 void 6301 page_capture_register_callback(uint_t index, clock_t duration, 6302 int (*cb_func)(page_t *, void *, uint_t)) 6303 { 6304 ASSERT(pc_cb[index].cb_active == 0); 6305 ASSERT(cb_func != NULL); 6306 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6307 pc_cb[index].duration = duration; 6308 pc_cb[index].cb_func = cb_func; 6309 pc_cb[index].cb_active = 1; 6310 rw_exit(&pc_cb[index].cb_rwlock); 6311 } 6312 6313 void 6314 page_capture_unregister_callback(uint_t index) 6315 { 6316 int i, j; 6317 struct page_capture_hash_bucket *bp1; 6318 struct page_capture_hash_bucket *bp2; 6319 struct page_capture_hash_bucket *head = NULL; 6320 uint_t flags = (1 << index); 6321 6322 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6323 ASSERT(pc_cb[index].cb_active == 1); 6324 pc_cb[index].duration = 0; /* Paranoia */ 6325 pc_cb[index].cb_func = NULL; /* Paranoia */ 6326 pc_cb[index].cb_active = 0; 6327 rw_exit(&pc_cb[index].cb_rwlock); 6328 6329 /* 6330 * Just move all the entries to a private list which we can walk 6331 * through without the need to hold any locks. 6332 * No more requests can get added to the hash lists for this consumer 6333 * as the cb_active field for the callback has been cleared. 6334 */ 6335 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 6336 mutex_enter(&page_capture_hash[i].pchh_mutex); 6337 for (j = 0; j < 2; j++) { 6338 bp1 = page_capture_hash[i].lists[j].next; 6339 /* walk through all but first (sentinel) element */ 6340 while (bp1 != &page_capture_hash[i].lists[j]) { 6341 bp2 = bp1; 6342 if (bp2->flags & flags) { 6343 bp1 = bp2->next; 6344 bp1->prev = bp2->prev; 6345 bp2->prev->next = bp1; 6346 bp2->next = head; 6347 head = bp2; 6348 /* 6349 * Clear the PR_CAPTURE bit as we 6350 * hold appropriate locks here. 6351 */ 6352 page_clrtoxic(head->pp, PR_CAPTURE); 6353 page_capture_hash[i].num_pages--; 6354 continue; 6355 } 6356 bp1 = bp1->next; 6357 } 6358 } 6359 mutex_exit(&page_capture_hash[i].pchh_mutex); 6360 } 6361 6362 while (head != NULL) { 6363 bp1 = head; 6364 head = head->next; 6365 kmem_free(bp1, sizeof (*bp1)); 6366 } 6367 } 6368 6369 6370 /* 6371 * Find pp in the active list and move it to the walked list if it 6372 * exists. 6373 * Note that most often pp should be at the front of the active list 6374 * as it is currently used and thus there is no other sort of optimization 6375 * being done here as this is a linked list data structure. 6376 * Returns 1 on successful move or 0 if page could not be found. 6377 */ 6378 static int 6379 page_capture_move_to_walked(page_t *pp) 6380 { 6381 page_capture_hash_bucket_t *bp; 6382 int index; 6383 6384 index = PAGE_CAPTURE_HASH(pp); 6385 6386 mutex_enter(&page_capture_hash[index].pchh_mutex); 6387 bp = page_capture_hash[index].lists[0].next; 6388 while (bp != &page_capture_hash[index].lists[0]) { 6389 if (bp->pp == pp) { 6390 /* Remove from old list */ 6391 bp->next->prev = bp->prev; 6392 bp->prev->next = bp->next; 6393 6394 /* Add to new list */ 6395 bp->next = page_capture_hash[index].lists[1].next; 6396 bp->prev = &page_capture_hash[index].lists[1]; 6397 page_capture_hash[index].lists[1].next = bp; 6398 bp->next->prev = bp; 6399 mutex_exit(&page_capture_hash[index].pchh_mutex); 6400 6401 return (1); 6402 } 6403 bp = bp->next; 6404 } 6405 mutex_exit(&page_capture_hash[index].pchh_mutex); 6406 return (0); 6407 } 6408 6409 /* 6410 * Add a new entry to the page capture hash. The only case where a new 6411 * entry is not added is when the page capture consumer is no longer registered. 6412 * In this case, we'll silently not add the page to the hash. We know that 6413 * page retire will always be registered for the case where we are currently 6414 * unretiring a page and thus there are no conflicts. 6415 */ 6416 static void 6417 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) 6418 { 6419 page_capture_hash_bucket_t *bp1; 6420 page_capture_hash_bucket_t *bp2; 6421 int index; 6422 int cb_index; 6423 int i; 6424 #ifdef DEBUG 6425 page_capture_hash_bucket_t *tp1; 6426 int l; 6427 #endif 6428 6429 ASSERT(!(flags & CAPTURE_ASYNC)); 6430 6431 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP); 6432 6433 bp1->pp = pp; 6434 bp1->szc = szc; 6435 bp1->flags = flags; 6436 bp1->datap = datap; 6437 6438 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6439 if ((flags >> cb_index) & 1) { 6440 break; 6441 } 6442 } 6443 6444 ASSERT(cb_index != PC_NUM_CALLBACKS); 6445 6446 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6447 if (pc_cb[cb_index].cb_active) { 6448 if (pc_cb[cb_index].duration == -1) { 6449 bp1->expires = (clock_t)-1; 6450 } else { 6451 bp1->expires = lbolt + pc_cb[cb_index].duration; 6452 } 6453 } else { 6454 /* There's no callback registered so don't add to the hash */ 6455 rw_exit(&pc_cb[cb_index].cb_rwlock); 6456 kmem_free(bp1, sizeof (*bp1)); 6457 return; 6458 } 6459 6460 index = PAGE_CAPTURE_HASH(pp); 6461 6462 /* 6463 * Only allow capture flag to be modified under this mutex. 6464 * Prevents multiple entries for same page getting added. 6465 */ 6466 mutex_enter(&page_capture_hash[index].pchh_mutex); 6467 6468 /* 6469 * if not already on the hash, set capture bit and add to the hash 6470 */ 6471 if (!(pp->p_toxic & PR_CAPTURE)) { 6472 #ifdef DEBUG 6473 /* Check for duplicate entries */ 6474 for (l = 0; l < 2; l++) { 6475 tp1 = page_capture_hash[index].lists[l].next; 6476 while (tp1 != &page_capture_hash[index].lists[l]) { 6477 if (tp1->pp == pp) { 6478 panic("page pp 0x%p already on hash " 6479 "at 0x%p\n", pp, tp1); 6480 } 6481 tp1 = tp1->next; 6482 } 6483 } 6484 6485 #endif 6486 page_settoxic(pp, PR_CAPTURE); 6487 bp1->next = page_capture_hash[index].lists[0].next; 6488 bp1->prev = &page_capture_hash[index].lists[0]; 6489 bp1->next->prev = bp1; 6490 page_capture_hash[index].lists[0].next = bp1; 6491 page_capture_hash[index].num_pages++; 6492 if (flags & CAPTURE_RETIRE) { 6493 page_retire_incr_pend_count(); 6494 } 6495 mutex_exit(&page_capture_hash[index].pchh_mutex); 6496 rw_exit(&pc_cb[cb_index].cb_rwlock); 6497 cv_signal(&pc_cv); 6498 return; 6499 } 6500 6501 /* 6502 * A page retire request will replace any other request. 6503 * A second physmem request which is for a different process than 6504 * the currently registered one will be dropped as there is 6505 * no way to hold the private data for both calls. 6506 * In the future, once there are more callers, this will have to 6507 * be worked out better as there needs to be private storage for 6508 * at least each type of caller (maybe have datap be an array of 6509 * *void's so that we can index based upon callers index). 6510 */ 6511 6512 /* walk hash list to update expire time */ 6513 for (i = 0; i < 2; i++) { 6514 bp2 = page_capture_hash[index].lists[i].next; 6515 while (bp2 != &page_capture_hash[index].lists[i]) { 6516 if (bp2->pp == pp) { 6517 if (flags & CAPTURE_RETIRE) { 6518 if (!(bp2->flags & CAPTURE_RETIRE)) { 6519 page_retire_incr_pend_count(); 6520 bp2->flags = flags; 6521 bp2->expires = bp1->expires; 6522 bp2->datap = datap; 6523 } 6524 } else { 6525 ASSERT(flags & CAPTURE_PHYSMEM); 6526 if (!(bp2->flags & CAPTURE_RETIRE) && 6527 (datap == bp2->datap)) { 6528 bp2->expires = bp1->expires; 6529 } 6530 } 6531 mutex_exit(&page_capture_hash[index]. 6532 pchh_mutex); 6533 rw_exit(&pc_cb[cb_index].cb_rwlock); 6534 kmem_free(bp1, sizeof (*bp1)); 6535 return; 6536 } 6537 bp2 = bp2->next; 6538 } 6539 } 6540 6541 /* 6542 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes 6543 * and thus it either has to be set or not set and can't change 6544 * while holding the mutex above. 6545 */ 6546 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", pp); 6547 } 6548 6549 /* 6550 * We have a page in our hands, lets try and make it ours by turning 6551 * it into a clean page like it had just come off the freelists. 6552 * 6553 * Returns 0 on success, with the page still EXCL locked. 6554 * On failure, the page will be unlocked, and returns EAGAIN 6555 */ 6556 static int 6557 page_capture_clean_page(page_t *pp) 6558 { 6559 page_t *newpp; 6560 int skip_unlock = 0; 6561 spgcnt_t count; 6562 page_t *tpp; 6563 int ret = 0; 6564 int extra; 6565 6566 ASSERT(PAGE_EXCL(pp)); 6567 ASSERT(!PP_RETIRED(pp)); 6568 ASSERT(curthread->t_flag & T_CAPTURING); 6569 6570 if (PP_ISFREE(pp)) { 6571 if (!page_reclaim(pp, NULL)) { 6572 skip_unlock = 1; 6573 ret = EAGAIN; 6574 goto cleanup; 6575 } 6576 ASSERT(pp->p_szc == 0); 6577 if (pp->p_vnode != NULL) { 6578 /* 6579 * Since this page came from the 6580 * cachelist, we must destroy the 6581 * old vnode association. 6582 */ 6583 page_hashout(pp, NULL); 6584 } 6585 goto cleanup; 6586 } 6587 6588 /* 6589 * If we know page_relocate will fail, skip it 6590 * It could still fail due to a UE on another page but we 6591 * can't do anything about that. 6592 */ 6593 if (pp->p_toxic & PR_UE) { 6594 goto skip_relocate; 6595 } 6596 6597 /* 6598 * It's possible that pages can not have a vnode as fsflush comes 6599 * through and cleans up these pages. It's ugly but that's how it is. 6600 */ 6601 if (pp->p_vnode == NULL) { 6602 goto skip_relocate; 6603 } 6604 6605 /* 6606 * Page was not free, so lets try to relocate it. 6607 * page_relocate only works with root pages, so if this is not a root 6608 * page, we need to demote it to try and relocate it. 6609 * Unfortunately this is the best we can do right now. 6610 */ 6611 newpp = NULL; 6612 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { 6613 if (page_try_demote_pages(pp) == 0) { 6614 ret = EAGAIN; 6615 goto cleanup; 6616 } 6617 } 6618 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); 6619 if (ret == 0) { 6620 page_t *npp; 6621 /* unlock the new page(s) */ 6622 while (count-- > 0) { 6623 ASSERT(newpp != NULL); 6624 npp = newpp; 6625 page_sub(&newpp, npp); 6626 page_unlock(npp); 6627 } 6628 ASSERT(newpp == NULL); 6629 /* 6630 * Check to see if the page we have is too large. 6631 * If so, demote it freeing up the extra pages. 6632 */ 6633 if (pp->p_szc > 0) { 6634 /* For now demote extra pages to szc == 0 */ 6635 extra = page_get_pagecnt(pp->p_szc) - 1; 6636 while (extra > 0) { 6637 tpp = pp->p_next; 6638 page_sub(&pp, tpp); 6639 tpp->p_szc = 0; 6640 page_free(tpp, 1); 6641 extra--; 6642 } 6643 /* Make sure to set our page to szc 0 as well */ 6644 ASSERT(pp->p_next == pp && pp->p_prev == pp); 6645 pp->p_szc = 0; 6646 } 6647 goto cleanup; 6648 } else if (ret == EIO) { 6649 ret = EAGAIN; 6650 goto cleanup; 6651 } else { 6652 /* 6653 * Need to reset return type as we failed to relocate the page 6654 * but that does not mean that some of the next steps will not 6655 * work. 6656 */ 6657 ret = 0; 6658 } 6659 6660 skip_relocate: 6661 6662 if (pp->p_szc > 0) { 6663 if (page_try_demote_pages(pp) == 0) { 6664 ret = EAGAIN; 6665 goto cleanup; 6666 } 6667 } 6668 6669 ASSERT(pp->p_szc == 0); 6670 6671 if (hat_ismod(pp)) { 6672 ret = EAGAIN; 6673 goto cleanup; 6674 } 6675 if (PP_ISKAS(pp)) { 6676 ret = EAGAIN; 6677 goto cleanup; 6678 } 6679 if (pp->p_lckcnt || pp->p_cowcnt) { 6680 ret = EAGAIN; 6681 goto cleanup; 6682 } 6683 6684 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 6685 ASSERT(!hat_page_is_mapped(pp)); 6686 6687 if (hat_ismod(pp)) { 6688 /* 6689 * This is a semi-odd case as the page is now modified but not 6690 * mapped as we just unloaded the mappings above. 6691 */ 6692 ret = EAGAIN; 6693 goto cleanup; 6694 } 6695 if (pp->p_vnode != NULL) { 6696 page_hashout(pp, NULL); 6697 } 6698 6699 /* 6700 * At this point, the page should be in a clean state and 6701 * we can do whatever we want with it. 6702 */ 6703 6704 cleanup: 6705 if (ret != 0) { 6706 if (!skip_unlock) { 6707 page_unlock(pp); 6708 } 6709 } else { 6710 ASSERT(pp->p_szc == 0); 6711 ASSERT(PAGE_EXCL(pp)); 6712 6713 pp->p_next = pp; 6714 pp->p_prev = pp; 6715 } 6716 return (ret); 6717 } 6718 6719 /* 6720 * Various callers of page_trycapture() can have different restrictions upon 6721 * what memory they have access to. 6722 * Returns 0 on success, with the following error codes on failure: 6723 * EPERM - The requested page is long term locked, and thus repeated 6724 * requests to capture this page will likely fail. 6725 * ENOMEM - There was not enough free memory in the system to safely 6726 * map the requested page. 6727 * ENOENT - The requested page was inside the kernel cage, and the 6728 * PHYSMEM_CAGE flag was not set. 6729 */ 6730 int 6731 page_capture_pre_checks(page_t *pp, uint_t flags) 6732 { 6733 #if defined(__sparc) 6734 extern struct vnode prom_ppages; 6735 #endif /* __sparc */ 6736 6737 ASSERT(pp != NULL); 6738 6739 /* only physmem currently has restrictions */ 6740 if (!(flags & CAPTURE_PHYSMEM)) { 6741 return (0); 6742 } 6743 6744 #if defined(__sparc) 6745 if (pp->p_vnode == &prom_ppages) { 6746 return (EPERM); 6747 } 6748 6749 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE)) { 6750 return (ENOENT); 6751 } 6752 6753 if (PP_ISNORELOCKERNEL(pp)) { 6754 return (EPERM); 6755 } 6756 #else 6757 if (PP_ISKAS(pp)) { 6758 return (EPERM); 6759 } 6760 #endif /* __sparc */ 6761 6762 if (availrmem < swapfs_minfree) { 6763 /* 6764 * We won't try to capture this page as we are 6765 * running low on memory. 6766 */ 6767 return (ENOMEM); 6768 } 6769 return (0); 6770 } 6771 6772 /* 6773 * Once we have a page in our mits, go ahead and complete the capture 6774 * operation. 6775 * Returns 1 on failure where page is no longer needed 6776 * Returns 0 on success 6777 * Returns -1 if there was a transient failure. 6778 * Failure cases must release the SE_EXCL lock on pp (usually via page_free). 6779 */ 6780 int 6781 page_capture_take_action(page_t *pp, uint_t flags, void *datap) 6782 { 6783 int cb_index; 6784 int ret = 0; 6785 page_capture_hash_bucket_t *bp1; 6786 page_capture_hash_bucket_t *bp2; 6787 int index; 6788 int found = 0; 6789 int i; 6790 6791 ASSERT(PAGE_EXCL(pp)); 6792 ASSERT(curthread->t_flag & T_CAPTURING); 6793 6794 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6795 if ((flags >> cb_index) & 1) { 6796 break; 6797 } 6798 } 6799 ASSERT(cb_index < PC_NUM_CALLBACKS); 6800 6801 /* 6802 * Remove the entry from the page_capture hash, but don't free it yet 6803 * as we may need to put it back. 6804 * Since we own the page at this point in time, we should find it 6805 * in the hash if this is an ASYNC call. If we don't it's likely 6806 * that the page_capture_async() thread decided that this request 6807 * had expired, in which case we just continue on. 6808 */ 6809 if (flags & CAPTURE_ASYNC) { 6810 6811 index = PAGE_CAPTURE_HASH(pp); 6812 6813 mutex_enter(&page_capture_hash[index].pchh_mutex); 6814 for (i = 0; i < 2 && !found; i++) { 6815 bp1 = page_capture_hash[index].lists[i].next; 6816 while (bp1 != &page_capture_hash[index].lists[i]) { 6817 if (bp1->pp == pp) { 6818 bp1->next->prev = bp1->prev; 6819 bp1->prev->next = bp1->next; 6820 page_capture_hash[index].num_pages--; 6821 page_clrtoxic(pp, PR_CAPTURE); 6822 found = 1; 6823 break; 6824 } 6825 bp1 = bp1->next; 6826 } 6827 } 6828 mutex_exit(&page_capture_hash[index].pchh_mutex); 6829 } 6830 6831 /* Synchronize with the unregister func. */ 6832 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6833 if (!pc_cb[cb_index].cb_active) { 6834 page_free(pp, 1); 6835 rw_exit(&pc_cb[cb_index].cb_rwlock); 6836 if (found) { 6837 kmem_free(bp1, sizeof (*bp1)); 6838 } 6839 return (1); 6840 } 6841 6842 /* 6843 * We need to remove the entry from the page capture hash and turn off 6844 * the PR_CAPTURE bit before calling the callback. We'll need to cache 6845 * the entry here, and then based upon the return value, cleanup 6846 * appropriately or re-add it to the hash, making sure that someone else 6847 * hasn't already done so. 6848 * It should be rare for the callback to fail and thus it's ok for 6849 * the failure path to be a bit complicated as the success path is 6850 * cleaner and the locking rules are easier to follow. 6851 */ 6852 6853 ret = pc_cb[cb_index].cb_func(pp, datap, flags); 6854 6855 rw_exit(&pc_cb[cb_index].cb_rwlock); 6856 6857 /* 6858 * If this was an ASYNC request, we need to cleanup the hash if the 6859 * callback was successful or if the request was no longer valid. 6860 * For non-ASYNC requests, we return failure to map and the caller 6861 * will take care of adding the request to the hash. 6862 * Note also that the callback itself is responsible for the page 6863 * at this point in time in terms of locking ... The most common 6864 * case for the failure path should just be a page_free. 6865 */ 6866 if (ret >= 0) { 6867 if (found) { 6868 if (bp1->flags & CAPTURE_RETIRE) { 6869 page_retire_decr_pend_count(); 6870 } 6871 kmem_free(bp1, sizeof (*bp1)); 6872 } 6873 return (ret); 6874 } 6875 if (!found) { 6876 return (ret); 6877 } 6878 6879 ASSERT(flags & CAPTURE_ASYNC); 6880 6881 /* 6882 * Check for expiration time first as we can just free it up if it's 6883 * expired. 6884 */ 6885 if (lbolt > bp1->expires && bp1->expires != -1) { 6886 kmem_free(bp1, sizeof (*bp1)); 6887 return (ret); 6888 } 6889 6890 /* 6891 * The callback failed and there used to be an entry in the hash for 6892 * this page, so we need to add it back to the hash. 6893 */ 6894 mutex_enter(&page_capture_hash[index].pchh_mutex); 6895 if (!(pp->p_toxic & PR_CAPTURE)) { 6896 /* just add bp1 back to head of walked list */ 6897 page_settoxic(pp, PR_CAPTURE); 6898 bp1->next = page_capture_hash[index].lists[1].next; 6899 bp1->prev = &page_capture_hash[index].lists[1]; 6900 bp1->next->prev = bp1; 6901 page_capture_hash[index].lists[1].next = bp1; 6902 page_capture_hash[index].num_pages++; 6903 mutex_exit(&page_capture_hash[index].pchh_mutex); 6904 return (ret); 6905 } 6906 6907 /* 6908 * Otherwise there was a new capture request added to list 6909 * Need to make sure that our original data is represented if 6910 * appropriate. 6911 */ 6912 for (i = 0; i < 2; i++) { 6913 bp2 = page_capture_hash[index].lists[i].next; 6914 while (bp2 != &page_capture_hash[index].lists[i]) { 6915 if (bp2->pp == pp) { 6916 if (bp1->flags & CAPTURE_RETIRE) { 6917 if (!(bp2->flags & CAPTURE_RETIRE)) { 6918 bp2->szc = bp1->szc; 6919 bp2->flags = bp1->flags; 6920 bp2->expires = bp1->expires; 6921 bp2->datap = bp1->datap; 6922 } 6923 } else { 6924 ASSERT(bp1->flags & CAPTURE_PHYSMEM); 6925 if (!(bp2->flags & CAPTURE_RETIRE)) { 6926 bp2->szc = bp1->szc; 6927 bp2->flags = bp1->flags; 6928 bp2->expires = bp1->expires; 6929 bp2->datap = bp1->datap; 6930 } 6931 } 6932 mutex_exit(&page_capture_hash[index]. 6933 pchh_mutex); 6934 kmem_free(bp1, sizeof (*bp1)); 6935 return (ret); 6936 } 6937 bp2 = bp2->next; 6938 } 6939 } 6940 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", pp); 6941 /*NOTREACHED*/ 6942 } 6943 6944 /* 6945 * Try to capture the given page for the caller specified in the flags 6946 * parameter. The page will either be captured and handed over to the 6947 * appropriate callback, or will be queued up in the page capture hash 6948 * to be captured asynchronously. 6949 * If the current request is due to an async capture, the page must be 6950 * exclusively locked before calling this function. 6951 * Currently szc must be 0 but in the future this should be expandable to 6952 * other page sizes. 6953 * Returns 0 on success, with the following error codes on failure: 6954 * EPERM - The requested page is long term locked, and thus repeated 6955 * requests to capture this page will likely fail. 6956 * ENOMEM - There was not enough free memory in the system to safely 6957 * map the requested page. 6958 * ENOENT - The requested page was inside the kernel cage, and the 6959 * CAPTURE_GET_CAGE flag was not set. 6960 * EAGAIN - The requested page could not be capturead at this point in 6961 * time but future requests will likely work. 6962 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag 6963 * was not set. 6964 */ 6965 int 6966 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 6967 { 6968 int ret; 6969 int cb_index; 6970 6971 if (flags & CAPTURE_ASYNC) { 6972 ASSERT(PAGE_EXCL(pp)); 6973 goto async; 6974 } 6975 6976 /* Make sure there's enough availrmem ... */ 6977 ret = page_capture_pre_checks(pp, flags); 6978 if (ret != 0) { 6979 return (ret); 6980 } 6981 6982 if (!page_trylock(pp, SE_EXCL)) { 6983 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6984 if ((flags >> cb_index) & 1) { 6985 break; 6986 } 6987 } 6988 ASSERT(cb_index < PC_NUM_CALLBACKS); 6989 ret = EAGAIN; 6990 /* Special case for retired pages */ 6991 if (PP_RETIRED(pp)) { 6992 if (flags & CAPTURE_GET_RETIRED) { 6993 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { 6994 /* 6995 * Need to set capture bit and add to 6996 * hash so that the page will be 6997 * retired when freed. 6998 */ 6999 page_capture_add_hash(pp, szc, 7000 CAPTURE_RETIRE, NULL); 7001 ret = 0; 7002 goto own_page; 7003 } 7004 } else { 7005 return (EBUSY); 7006 } 7007 } 7008 page_capture_add_hash(pp, szc, flags, datap); 7009 return (ret); 7010 } 7011 7012 async: 7013 ASSERT(PAGE_EXCL(pp)); 7014 7015 /* Need to check for physmem async requests that availrmem is sane */ 7016 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) == 7017 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) && 7018 (availrmem < swapfs_minfree)) { 7019 page_unlock(pp); 7020 return (ENOMEM); 7021 } 7022 7023 ret = page_capture_clean_page(pp); 7024 7025 if (ret != 0) { 7026 /* We failed to get the page, so lets add it to the hash */ 7027 if (!(flags & CAPTURE_ASYNC)) { 7028 page_capture_add_hash(pp, szc, flags, datap); 7029 } 7030 return (ret); 7031 } 7032 7033 own_page: 7034 ASSERT(PAGE_EXCL(pp)); 7035 ASSERT(pp->p_szc == 0); 7036 7037 /* Call the callback */ 7038 ret = page_capture_take_action(pp, flags, datap); 7039 7040 if (ret == 0) { 7041 return (0); 7042 } 7043 7044 /* 7045 * Note that in the failure cases from page_capture_take_action, the 7046 * EXCL lock will have already been dropped. 7047 */ 7048 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) { 7049 page_capture_add_hash(pp, szc, flags, datap); 7050 } 7051 return (EAGAIN); 7052 } 7053 7054 int 7055 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 7056 { 7057 int ret; 7058 7059 curthread->t_flag |= T_CAPTURING; 7060 ret = page_itrycapture(pp, szc, flags, datap); 7061 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */ 7062 return (ret); 7063 } 7064 7065 /* 7066 * When unlocking a page which has the PR_CAPTURE bit set, this routine 7067 * gets called to try and capture the page. 7068 */ 7069 void 7070 page_unlock_capture(page_t *pp) 7071 { 7072 page_capture_hash_bucket_t *bp; 7073 int index; 7074 int i; 7075 uint_t szc; 7076 uint_t flags = 0; 7077 void *datap; 7078 kmutex_t *mp; 7079 extern vnode_t retired_pages; 7080 7081 /* 7082 * We need to protect against a possible deadlock here where we own 7083 * the vnode page hash mutex and want to acquire it again as there 7084 * are locations in the code, where we unlock a page while holding 7085 * the mutex which can lead to the page being captured and eventually 7086 * end up here. As we may be hashing out the old page and hashing into 7087 * the retire vnode, we need to make sure we don't own them. 7088 * Other callbacks who do hash operations also need to make sure that 7089 * before they hashin to a vnode that they do not currently own the 7090 * vphm mutex otherwise there will be a panic. 7091 */ 7092 if (mutex_owned(page_vnode_mutex(&retired_pages))) { 7093 page_unlock_nocapture(pp); 7094 return; 7095 } 7096 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { 7097 page_unlock_nocapture(pp); 7098 return; 7099 } 7100 7101 index = PAGE_CAPTURE_HASH(pp); 7102 7103 mp = &page_capture_hash[index].pchh_mutex; 7104 mutex_enter(mp); 7105 for (i = 0; i < 2; i++) { 7106 bp = page_capture_hash[index].lists[i].next; 7107 while (bp != &page_capture_hash[index].lists[i]) { 7108 if (bp->pp == pp) { 7109 szc = bp->szc; 7110 flags = bp->flags | CAPTURE_ASYNC; 7111 datap = bp->datap; 7112 mutex_exit(mp); 7113 (void) page_trycapture(pp, szc, flags, datap); 7114 return; 7115 } 7116 bp = bp->next; 7117 } 7118 } 7119 7120 /* Failed to find page in hash so clear flags and unlock it. */ 7121 page_clrtoxic(pp, PR_CAPTURE); 7122 page_unlock(pp); 7123 7124 mutex_exit(mp); 7125 } 7126 7127 void 7128 page_capture_init() 7129 { 7130 int i; 7131 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7132 page_capture_hash[i].lists[0].next = 7133 &page_capture_hash[i].lists[0]; 7134 page_capture_hash[i].lists[0].prev = 7135 &page_capture_hash[i].lists[0]; 7136 page_capture_hash[i].lists[1].next = 7137 &page_capture_hash[i].lists[1]; 7138 page_capture_hash[i].lists[1].prev = 7139 &page_capture_hash[i].lists[1]; 7140 } 7141 7142 pc_thread_shortwait = 23 * hz; 7143 pc_thread_longwait = 1201 * hz; 7144 pc_thread_ism_retry = 3; 7145 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL); 7146 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL); 7147 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0, 7148 TS_RUN, minclsyspri); 7149 } 7150 7151 /* 7152 * It is necessary to scrub any failing pages prior to reboot in order to 7153 * prevent a latent error trap from occurring on the next boot. 7154 */ 7155 void 7156 page_retire_mdboot() 7157 { 7158 page_t *pp; 7159 int i, j; 7160 page_capture_hash_bucket_t *bp; 7161 7162 /* walk lists looking for pages to scrub */ 7163 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7164 if (page_capture_hash[i].num_pages == 0) 7165 continue; 7166 7167 mutex_enter(&page_capture_hash[i].pchh_mutex); 7168 7169 for (j = 0; j < 2; j++) { 7170 bp = page_capture_hash[i].lists[j].next; 7171 while (bp != &page_capture_hash[i].lists[j]) { 7172 pp = bp->pp; 7173 if (!PP_ISKAS(pp) && PP_TOXIC(pp)) { 7174 pp->p_selock = -1; /* pacify ASSERTs */ 7175 PP_CLRFREE(pp); 7176 pagescrub(pp, 0, PAGESIZE); 7177 pp->p_selock = 0; 7178 } 7179 bp = bp->next; 7180 } 7181 } 7182 mutex_exit(&page_capture_hash[i].pchh_mutex); 7183 } 7184 } 7185 7186 /* 7187 * Walk the page_capture_hash trying to capture pages and also cleanup old 7188 * entries which have expired. 7189 */ 7190 void 7191 page_capture_async() 7192 { 7193 page_t *pp; 7194 int i; 7195 int ret; 7196 page_capture_hash_bucket_t *bp1, *bp2; 7197 uint_t szc; 7198 uint_t flags; 7199 void *datap; 7200 7201 /* If there are outstanding pages to be captured, get to work */ 7202 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7203 if (page_capture_hash[i].num_pages == 0) 7204 continue; 7205 /* Append list 1 to list 0 and then walk through list 0 */ 7206 mutex_enter(&page_capture_hash[i].pchh_mutex); 7207 bp1 = &page_capture_hash[i].lists[1]; 7208 bp2 = bp1->next; 7209 if (bp1 != bp2) { 7210 bp1->prev->next = page_capture_hash[i].lists[0].next; 7211 bp2->prev = &page_capture_hash[i].lists[0]; 7212 page_capture_hash[i].lists[0].next->prev = bp1->prev; 7213 page_capture_hash[i].lists[0].next = bp2; 7214 bp1->next = bp1; 7215 bp1->prev = bp1; 7216 } 7217 7218 /* list[1] will be empty now */ 7219 7220 bp1 = page_capture_hash[i].lists[0].next; 7221 while (bp1 != &page_capture_hash[i].lists[0]) { 7222 /* Check expiration time */ 7223 if ((lbolt > bp1->expires && bp1->expires != -1) || 7224 page_deleted(bp1->pp)) { 7225 page_capture_hash[i].lists[0].next = bp1->next; 7226 bp1->next->prev = 7227 &page_capture_hash[i].lists[0]; 7228 page_capture_hash[i].num_pages--; 7229 7230 /* 7231 * We can safely remove the PR_CAPTURE bit 7232 * without holding the EXCL lock on the page 7233 * as the PR_CAPTURE bit requres that the 7234 * page_capture_hash[].pchh_mutex be held 7235 * to modify it. 7236 */ 7237 page_clrtoxic(bp1->pp, PR_CAPTURE); 7238 mutex_exit(&page_capture_hash[i].pchh_mutex); 7239 kmem_free(bp1, sizeof (*bp1)); 7240 mutex_enter(&page_capture_hash[i].pchh_mutex); 7241 bp1 = page_capture_hash[i].lists[0].next; 7242 continue; 7243 } 7244 pp = bp1->pp; 7245 szc = bp1->szc; 7246 flags = bp1->flags; 7247 datap = bp1->datap; 7248 mutex_exit(&page_capture_hash[i].pchh_mutex); 7249 if (page_trylock(pp, SE_EXCL)) { 7250 ret = page_trycapture(pp, szc, 7251 flags | CAPTURE_ASYNC, datap); 7252 } else { 7253 ret = 1; /* move to walked hash */ 7254 } 7255 7256 if (ret != 0) { 7257 /* Move to walked hash */ 7258 (void) page_capture_move_to_walked(pp); 7259 } 7260 mutex_enter(&page_capture_hash[i].pchh_mutex); 7261 bp1 = page_capture_hash[i].lists[0].next; 7262 } 7263 7264 mutex_exit(&page_capture_hash[i].pchh_mutex); 7265 } 7266 } 7267 7268 /* 7269 * This function is called by the page_capture_thread, and is needed in 7270 * in order to initiate aio cleanup, so that pages used in aio 7271 * will be unlocked and subsequently retired by page_capture_thread. 7272 */ 7273 static int 7274 do_aio_cleanup(void) 7275 { 7276 proc_t *procp; 7277 int (*aio_cleanup_dr_delete_memory)(proc_t *); 7278 int cleaned = 0; 7279 7280 if (modload("sys", "kaio") == -1) { 7281 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio"); 7282 return (0); 7283 } 7284 /* 7285 * We use the aio_cleanup_dr_delete_memory function to 7286 * initiate the actual clean up; this function will wake 7287 * up the per-process aio_cleanup_thread. 7288 */ 7289 aio_cleanup_dr_delete_memory = (int (*)(proc_t *)) 7290 modgetsymvalue("aio_cleanup_dr_delete_memory", 0); 7291 if (aio_cleanup_dr_delete_memory == NULL) { 7292 cmn_err(CE_WARN, 7293 "aio_cleanup_dr_delete_memory not found in kaio"); 7294 return (0); 7295 } 7296 mutex_enter(&pidlock); 7297 for (procp = practive; (procp != NULL); procp = procp->p_next) { 7298 mutex_enter(&procp->p_lock); 7299 if (procp->p_aio != NULL) { 7300 /* cleanup proc's outstanding kaio */ 7301 cleaned += (*aio_cleanup_dr_delete_memory)(procp); 7302 } 7303 mutex_exit(&procp->p_lock); 7304 } 7305 mutex_exit(&pidlock); 7306 return (cleaned); 7307 } 7308 7309 /* 7310 * helper function for page_capture_thread 7311 */ 7312 static void 7313 page_capture_handle_outstanding(void) 7314 { 7315 extern size_t spt_used; 7316 int ntry; 7317 7318 if (!page_retire_pend_count()) { 7319 /* 7320 * Do we really want to be this aggressive 7321 * for things other than page_retire? 7322 * Maybe have a counter for each callback 7323 * type to guide how aggressive we should 7324 * be here. Thus if there's at least one 7325 * page for page_retire we go ahead and reap 7326 * like this. 7327 */ 7328 kmem_reap(); 7329 seg_preap(); 7330 page_capture_async(); 7331 } else { 7332 /* 7333 * There are pages pending retirement, so 7334 * we reap prior to attempting to capture. 7335 */ 7336 kmem_reap(); 7337 /* 7338 * When ISM is in use, we need to disable and 7339 * purge the seg_pcache, and initiate aio 7340 * cleanup in order to release page locks and 7341 * subsquently retire pages in need of 7342 * retirement. 7343 */ 7344 if (spt_used) { 7345 /* disable and purge seg_pcache */ 7346 (void) seg_p_disable(); 7347 for (ntry = 0; ntry < pc_thread_ism_retry; ntry++) { 7348 if (!page_retire_pend_count()) 7349 break; 7350 if (do_aio_cleanup()) { 7351 /* 7352 * allow the apps cleanup threads 7353 * to run 7354 */ 7355 delay(pc_thread_shortwait); 7356 } 7357 page_capture_async(); 7358 } 7359 /* reenable seg_pcache */ 7360 seg_p_enable(); 7361 } else { 7362 seg_preap(); 7363 page_capture_async(); 7364 } 7365 } 7366 } 7367 7368 /* 7369 * The page_capture_thread loops forever, looking to see if there are 7370 * pages still waiting to be captured. 7371 */ 7372 static void 7373 page_capture_thread(void) 7374 { 7375 callb_cpr_t c; 7376 int outstanding; 7377 int i; 7378 7379 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture"); 7380 7381 mutex_enter(&pc_thread_mutex); 7382 for (;;) { 7383 outstanding = 0; 7384 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) 7385 outstanding += page_capture_hash[i].num_pages; 7386 if (outstanding) { 7387 page_capture_handle_outstanding(); 7388 CALLB_CPR_SAFE_BEGIN(&c); 7389 (void) cv_timedwait(&pc_cv, &pc_thread_mutex, 7390 lbolt + pc_thread_shortwait); 7391 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7392 } else { 7393 CALLB_CPR_SAFE_BEGIN(&c); 7394 (void) cv_timedwait(&pc_cv, &pc_thread_mutex, 7395 lbolt + pc_thread_longwait); 7396 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7397 } 7398 } 7399 /*NOTREACHED*/ 7400 } 7401