1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 26 /* All Rights Reserved */ 27 28 /* 29 * University Copyright- Copyright (c) 1982, 1986, 1988 30 * The Regents of the University of California 31 * All Rights Reserved 32 * 33 * University Acknowledgment- Portions of this document are derived from 34 * software developed by the University of California, Berkeley, and its 35 * contributors. 36 */ 37 38 /* 39 * VM - physical page management. 40 */ 41 42 #include <sys/types.h> 43 #include <sys/t_lock.h> 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/errno.h> 47 #include <sys/time.h> 48 #include <sys/vnode.h> 49 #include <sys/vm.h> 50 #include <sys/vtrace.h> 51 #include <sys/swap.h> 52 #include <sys/cmn_err.h> 53 #include <sys/tuneable.h> 54 #include <sys/sysmacros.h> 55 #include <sys/cpuvar.h> 56 #include <sys/callb.h> 57 #include <sys/debug.h> 58 #include <sys/tnf_probe.h> 59 #include <sys/condvar_impl.h> 60 #include <sys/mem_config.h> 61 #include <sys/mem_cage.h> 62 #include <sys/kmem.h> 63 #include <sys/atomic.h> 64 #include <sys/strlog.h> 65 #include <sys/mman.h> 66 #include <sys/ontrap.h> 67 #include <sys/lgrp.h> 68 #include <sys/vfs.h> 69 70 #include <vm/hat.h> 71 #include <vm/anon.h> 72 #include <vm/page.h> 73 #include <vm/seg.h> 74 #include <vm/pvn.h> 75 #include <vm/seg_kmem.h> 76 #include <vm/vm_dep.h> 77 #include <sys/vm_usage.h> 78 #include <fs/fs_subr.h> 79 #include <sys/ddi.h> 80 #include <sys/modctl.h> 81 82 static int nopageage = 0; 83 84 static pgcnt_t max_page_get; /* max page_get request size in pages */ 85 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 86 87 /* 88 * freemem_lock protects all freemem variables: 89 * availrmem. Also this lock protects the globals which track the 90 * availrmem changes for accurate kernel footprint calculation. 91 * See below for an explanation of these 92 * globals. 93 */ 94 kmutex_t freemem_lock; 95 pgcnt_t availrmem; 96 pgcnt_t availrmem_initial; 97 98 /* 99 * These globals track availrmem changes to get a more accurate 100 * estimate of tke kernel size. Historically pp_kernel is used for 101 * kernel size and is based on availrmem. But availrmem is adjusted for 102 * locked pages in the system not just for kernel locked pages. 103 * These new counters will track the pages locked through segvn and 104 * by explicit user locking. 105 * 106 * pages_locked : How many pages are locked because of user specified 107 * locking through mlock or plock. 108 * 109 * pages_useclaim,pages_claimed : These two variables track the 110 * claim adjustments because of the protection changes on a segvn segment. 111 * 112 * All these globals are protected by the same lock which protects availrmem. 113 */ 114 pgcnt_t pages_locked = 0; 115 pgcnt_t pages_useclaim = 0; 116 pgcnt_t pages_claimed = 0; 117 118 119 /* 120 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 121 */ 122 static kmutex_t new_freemem_lock; 123 static uint_t freemem_wait; /* someone waiting for freemem */ 124 static kcondvar_t freemem_cv; 125 126 /* 127 * The logical page free list is maintained as two lists, the 'free' 128 * and the 'cache' lists. 129 * The free list contains those pages that should be reused first. 130 * 131 * The implementation of the lists is machine dependent. 132 * page_get_freelist(), page_get_cachelist(), 133 * page_list_sub(), and page_list_add() 134 * form the interface to the machine dependent implementation. 135 * 136 * Pages with p_free set are on the cache list. 137 * Pages with p_free and p_age set are on the free list, 138 * 139 * A page may be locked while on either list. 140 */ 141 142 /* 143 * free list accounting stuff. 144 * 145 * 146 * Spread out the value for the number of pages on the 147 * page free and page cache lists. If there is just one 148 * value, then it must be under just one lock. 149 * The lock contention and cache traffic are a real bother. 150 * 151 * When we acquire and then drop a single pcf lock 152 * we can start in the middle of the array of pcf structures. 153 * If we acquire more than one pcf lock at a time, we need to 154 * start at the front to avoid deadlocking. 155 * 156 * pcf_count holds the number of pages in each pool. 157 * 158 * pcf_block is set when page_create_get_something() has asked the 159 * PSM page freelist and page cachelist routines without specifying 160 * a color and nothing came back. This is used to block anything 161 * else from moving pages from one list to the other while the 162 * lists are searched again. If a page is freeed while pcf_block is 163 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 164 * of clearning pcf_block, doing the wakeups, etc. 165 */ 166 167 #define MAX_PCF_FANOUT NCPU 168 static uint_t pcf_fanout = 1; /* Will get changed at boot time */ 169 static uint_t pcf_fanout_mask = 0; 170 171 struct pcf { 172 kmutex_t pcf_lock; /* protects the structure */ 173 uint_t pcf_count; /* page count */ 174 uint_t pcf_wait; /* number of waiters */ 175 uint_t pcf_block; /* pcgs flag to page_free() */ 176 uint_t pcf_reserve; /* pages freed after pcf_block set */ 177 uint_t pcf_fill[10]; /* to line up on the caches */ 178 }; 179 180 /* 181 * PCF_INDEX hash needs to be dynamic (every so often the hash changes where 182 * it will hash the cpu to). This is done to prevent a drain condition 183 * from happening. This drain condition will occur when pcf_count decrement 184 * occurs on cpu A and the increment of pcf_count always occurs on cpu B. An 185 * example of this shows up with device interrupts. The dma buffer is allocated 186 * by the cpu requesting the IO thus the pcf_count is decremented based on that. 187 * When the memory is returned by the interrupt thread, the pcf_count will be 188 * incremented based on the cpu servicing the interrupt. 189 */ 190 static struct pcf pcf[MAX_PCF_FANOUT]; 191 #define PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \ 192 (randtick() >> 24)) & (pcf_fanout_mask)) 193 194 static int pcf_decrement_bucket(pgcnt_t); 195 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int); 196 197 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 198 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 199 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 200 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 201 202 #ifdef VM_STATS 203 204 /* 205 * No locks, but so what, they are only statistics. 206 */ 207 208 static struct page_tcnt { 209 int pc_free_cache; /* free's into cache list */ 210 int pc_free_dontneed; /* free's with dontneed */ 211 int pc_free_pageout; /* free's from pageout */ 212 int pc_free_free; /* free's into free list */ 213 int pc_free_pages; /* free's into large page free list */ 214 int pc_destroy_pages; /* large page destroy's */ 215 int pc_get_cache; /* get's from cache list */ 216 int pc_get_free; /* get's from free list */ 217 int pc_reclaim; /* reclaim's */ 218 int pc_abortfree; /* abort's of free pages */ 219 int pc_find_hit; /* find's that find page */ 220 int pc_find_miss; /* find's that don't find page */ 221 int pc_destroy_free; /* # of free pages destroyed */ 222 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 223 int pc_find_hashlen[PC_HASH_CNT+1]; 224 int pc_addclaim_pages; 225 int pc_subclaim_pages; 226 int pc_free_replacement_page[2]; 227 int pc_try_demote_pages[6]; 228 int pc_demote_pages[2]; 229 } pagecnt; 230 231 uint_t hashin_count; 232 uint_t hashin_not_held; 233 uint_t hashin_already; 234 235 uint_t hashout_count; 236 uint_t hashout_not_held; 237 238 uint_t page_create_count; 239 uint_t page_create_not_enough; 240 uint_t page_create_not_enough_again; 241 uint_t page_create_zero; 242 uint_t page_create_hashout; 243 uint_t page_create_page_lock_failed; 244 uint_t page_create_trylock_failed; 245 uint_t page_create_found_one; 246 uint_t page_create_hashin_failed; 247 uint_t page_create_dropped_phm; 248 249 uint_t page_create_new; 250 uint_t page_create_exists; 251 uint_t page_create_putbacks; 252 uint_t page_create_overshoot; 253 254 uint_t page_reclaim_zero; 255 uint_t page_reclaim_zero_locked; 256 257 uint_t page_rename_exists; 258 uint_t page_rename_count; 259 260 uint_t page_lookup_cnt[20]; 261 uint_t page_lookup_nowait_cnt[10]; 262 uint_t page_find_cnt; 263 uint_t page_exists_cnt; 264 uint_t page_exists_forreal_cnt; 265 uint_t page_lookup_dev_cnt; 266 uint_t get_cachelist_cnt; 267 uint_t page_create_cnt[10]; 268 uint_t alloc_pages[9]; 269 uint_t page_exphcontg[19]; 270 uint_t page_create_large_cnt[10]; 271 272 /* 273 * Collects statistics. 274 */ 275 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 276 uint_t mylen = 0; \ 277 \ 278 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 279 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 280 break; \ 281 } \ 282 if ((pp) != NULL) \ 283 pagecnt.pc_find_hit++; \ 284 else \ 285 pagecnt.pc_find_miss++; \ 286 if (mylen > PC_HASH_CNT) \ 287 mylen = PC_HASH_CNT; \ 288 pagecnt.pc_find_hashlen[mylen]++; \ 289 } 290 291 #else /* VM_STATS */ 292 293 /* 294 * Don't collect statistics 295 */ 296 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 297 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 298 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 299 break; \ 300 } \ 301 } 302 303 #endif /* VM_STATS */ 304 305 306 307 #ifdef DEBUG 308 #define MEMSEG_SEARCH_STATS 309 #endif 310 311 #ifdef MEMSEG_SEARCH_STATS 312 struct memseg_stats { 313 uint_t nsearch; 314 uint_t nlastwon; 315 uint_t nhashwon; 316 uint_t nnotfound; 317 } memseg_stats; 318 319 #define MEMSEG_STAT_INCR(v) \ 320 atomic_add_32(&memseg_stats.v, 1) 321 #else 322 #define MEMSEG_STAT_INCR(x) 323 #endif 324 325 struct memseg *memsegs; /* list of memory segments */ 326 327 /* 328 * /etc/system tunable to control large page allocation hueristic. 329 * 330 * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup 331 * for large page allocation requests. If a large page is not readily 332 * avaliable on the local freelists we will go through additional effort 333 * to create a large page, potentially moving smaller pages around to coalesce 334 * larger pages in the local lgroup. 335 * Default value of LPAP_DEFAULT will go to remote freelists if large pages 336 * are not readily available in the local lgroup. 337 */ 338 enum lpap { 339 LPAP_DEFAULT, /* default large page allocation policy */ 340 LPAP_LOCAL /* local large page allocation policy */ 341 }; 342 343 enum lpap lpg_alloc_prefer = LPAP_DEFAULT; 344 345 static void page_init_mem_config(void); 346 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 347 static void page_do_hashout(page_t *); 348 static void page_capture_init(); 349 int page_capture_take_action(page_t *, uint_t, void *); 350 351 static void page_demote_vp_pages(page_t *); 352 353 354 void 355 pcf_init(void) 356 357 { 358 if (boot_ncpus != -1) { 359 pcf_fanout = boot_ncpus; 360 } else { 361 pcf_fanout = max_ncpus; 362 } 363 #ifdef sun4v 364 /* 365 * Force at least 4 buckets if possible for sun4v. 366 */ 367 pcf_fanout = MAX(pcf_fanout, 4); 368 #endif /* sun4v */ 369 370 /* 371 * Round up to the nearest power of 2. 372 */ 373 pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT); 374 if (!ISP2(pcf_fanout)) { 375 pcf_fanout = 1 << highbit(pcf_fanout); 376 377 if (pcf_fanout > MAX_PCF_FANOUT) { 378 pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1); 379 } 380 } 381 pcf_fanout_mask = pcf_fanout - 1; 382 } 383 384 /* 385 * vm subsystem related initialization 386 */ 387 void 388 vm_init(void) 389 { 390 boolean_t callb_vm_cpr(void *, int); 391 392 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 393 page_init_mem_config(); 394 page_retire_init(); 395 vm_usage_init(); 396 page_capture_init(); 397 } 398 399 /* 400 * This function is called at startup and when memory is added or deleted. 401 */ 402 void 403 init_pages_pp_maximum() 404 { 405 static pgcnt_t p_min; 406 static pgcnt_t pages_pp_maximum_startup; 407 static pgcnt_t avrmem_delta; 408 static int init_done; 409 static int user_set; /* true if set in /etc/system */ 410 411 if (init_done == 0) { 412 413 /* If the user specified a value, save it */ 414 if (pages_pp_maximum != 0) { 415 user_set = 1; 416 pages_pp_maximum_startup = pages_pp_maximum; 417 } 418 419 /* 420 * Setting of pages_pp_maximum is based first time 421 * on the value of availrmem just after the start-up 422 * allocations. To preserve this relationship at run 423 * time, use a delta from availrmem_initial. 424 */ 425 ASSERT(availrmem_initial >= availrmem); 426 avrmem_delta = availrmem_initial - availrmem; 427 428 /* The allowable floor of pages_pp_maximum */ 429 p_min = tune.t_minarmem + 100; 430 431 /* Make sure we don't come through here again. */ 432 init_done = 1; 433 } 434 /* 435 * Determine pages_pp_maximum, the number of currently available 436 * pages (availrmem) that can't be `locked'. If not set by 437 * the user, we set it to 4% of the currently available memory 438 * plus 4MB. 439 * But we also insist that it be greater than tune.t_minarmem; 440 * otherwise a process could lock down a lot of memory, get swapped 441 * out, and never have enough to get swapped back in. 442 */ 443 if (user_set) 444 pages_pp_maximum = pages_pp_maximum_startup; 445 else 446 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 447 + btop(4 * 1024 * 1024); 448 449 if (pages_pp_maximum <= p_min) { 450 pages_pp_maximum = p_min; 451 } 452 } 453 454 void 455 set_max_page_get(pgcnt_t target_total_pages) 456 { 457 max_page_get = target_total_pages / 2; 458 } 459 460 static pgcnt_t pending_delete; 461 462 /*ARGSUSED*/ 463 static void 464 page_mem_config_post_add( 465 void *arg, 466 pgcnt_t delta_pages) 467 { 468 set_max_page_get(total_pages - pending_delete); 469 init_pages_pp_maximum(); 470 } 471 472 /*ARGSUSED*/ 473 static int 474 page_mem_config_pre_del( 475 void *arg, 476 pgcnt_t delta_pages) 477 { 478 pgcnt_t nv; 479 480 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 481 set_max_page_get(total_pages - nv); 482 return (0); 483 } 484 485 /*ARGSUSED*/ 486 static void 487 page_mem_config_post_del( 488 void *arg, 489 pgcnt_t delta_pages, 490 int cancelled) 491 { 492 pgcnt_t nv; 493 494 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 495 set_max_page_get(total_pages - nv); 496 if (!cancelled) 497 init_pages_pp_maximum(); 498 } 499 500 static kphysm_setup_vector_t page_mem_config_vec = { 501 KPHYSM_SETUP_VECTOR_VERSION, 502 page_mem_config_post_add, 503 page_mem_config_pre_del, 504 page_mem_config_post_del, 505 }; 506 507 static void 508 page_init_mem_config(void) 509 { 510 int ret; 511 512 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 513 ASSERT(ret == 0); 514 } 515 516 /* 517 * Evenly spread out the PCF counters for large free pages 518 */ 519 static void 520 page_free_large_ctr(pgcnt_t npages) 521 { 522 static struct pcf *p = pcf; 523 pgcnt_t lump; 524 525 freemem += npages; 526 527 lump = roundup(npages, pcf_fanout) / pcf_fanout; 528 529 while (npages > 0) { 530 531 ASSERT(!p->pcf_block); 532 533 if (lump < npages) { 534 p->pcf_count += (uint_t)lump; 535 npages -= lump; 536 } else { 537 p->pcf_count += (uint_t)npages; 538 npages = 0; 539 } 540 541 ASSERT(!p->pcf_wait); 542 543 if (++p > &pcf[pcf_fanout - 1]) 544 p = pcf; 545 } 546 547 ASSERT(npages == 0); 548 } 549 550 /* 551 * Add a physical chunk of memory to the system free lists during startup. 552 * Platform specific startup() allocates the memory for the page structs. 553 * 554 * num - number of page structures 555 * base - page number (pfn) to be associated with the first page. 556 * 557 * Since we are doing this during startup (ie. single threaded), we will 558 * use shortcut routines to avoid any locking overhead while putting all 559 * these pages on the freelists. 560 * 561 * NOTE: Any changes performed to page_free(), must also be performed to 562 * add_physmem() since this is how we initialize all page_t's at 563 * boot time. 564 */ 565 void 566 add_physmem( 567 page_t *pp, 568 pgcnt_t num, 569 pfn_t pnum) 570 { 571 page_t *root = NULL; 572 uint_t szc = page_num_pagesizes() - 1; 573 pgcnt_t large = page_get_pagecnt(szc); 574 pgcnt_t cnt = 0; 575 576 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 577 "add_physmem:pp %p num %lu", pp, num); 578 579 /* 580 * Arbitrarily limit the max page_get request 581 * to 1/2 of the page structs we have. 582 */ 583 total_pages += num; 584 set_max_page_get(total_pages); 585 586 PLCNT_MODIFY_MAX(pnum, (long)num); 587 588 /* 589 * The physical space for the pages array 590 * representing ram pages has already been 591 * allocated. Here we initialize each lock 592 * in the page structure, and put each on 593 * the free list 594 */ 595 for (; num; pp++, pnum++, num--) { 596 597 /* 598 * this needs to fill in the page number 599 * and do any other arch specific initialization 600 */ 601 add_physmem_cb(pp, pnum); 602 603 pp->p_lckcnt = 0; 604 pp->p_cowcnt = 0; 605 pp->p_slckcnt = 0; 606 607 /* 608 * Initialize the page lock as unlocked, since nobody 609 * can see or access this page yet. 610 */ 611 pp->p_selock = 0; 612 613 /* 614 * Initialize IO lock 615 */ 616 page_iolock_init(pp); 617 618 /* 619 * initialize other fields in the page_t 620 */ 621 PP_SETFREE(pp); 622 page_clr_all_props(pp); 623 PP_SETAGED(pp); 624 pp->p_offset = (u_offset_t)-1; 625 pp->p_next = pp; 626 pp->p_prev = pp; 627 628 /* 629 * Simple case: System doesn't support large pages. 630 */ 631 if (szc == 0) { 632 pp->p_szc = 0; 633 page_free_at_startup(pp); 634 continue; 635 } 636 637 /* 638 * Handle unaligned pages, we collect them up onto 639 * the root page until we have a full large page. 640 */ 641 if (!IS_P2ALIGNED(pnum, large)) { 642 643 /* 644 * If not in a large page, 645 * just free as small page. 646 */ 647 if (root == NULL) { 648 pp->p_szc = 0; 649 page_free_at_startup(pp); 650 continue; 651 } 652 653 /* 654 * Link a constituent page into the large page. 655 */ 656 pp->p_szc = szc; 657 page_list_concat(&root, &pp); 658 659 /* 660 * When large page is fully formed, free it. 661 */ 662 if (++cnt == large) { 663 page_free_large_ctr(cnt); 664 page_list_add_pages(root, PG_LIST_ISINIT); 665 root = NULL; 666 cnt = 0; 667 } 668 continue; 669 } 670 671 /* 672 * At this point we have a page number which 673 * is aligned. We assert that we aren't already 674 * in a different large page. 675 */ 676 ASSERT(IS_P2ALIGNED(pnum, large)); 677 ASSERT(root == NULL && cnt == 0); 678 679 /* 680 * If insufficient number of pages left to form 681 * a large page, just free the small page. 682 */ 683 if (num < large) { 684 pp->p_szc = 0; 685 page_free_at_startup(pp); 686 continue; 687 } 688 689 /* 690 * Otherwise start a new large page. 691 */ 692 pp->p_szc = szc; 693 cnt++; 694 root = pp; 695 } 696 ASSERT(root == NULL && cnt == 0); 697 } 698 699 /* 700 * Find a page representing the specified [vp, offset]. 701 * If we find the page but it is intransit coming in, 702 * it will have an "exclusive" lock and we wait for 703 * the i/o to complete. A page found on the free list 704 * is always reclaimed and then locked. On success, the page 705 * is locked, its data is valid and it isn't on the free 706 * list, while a NULL is returned if the page doesn't exist. 707 */ 708 page_t * 709 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 710 { 711 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 712 } 713 714 /* 715 * Find a page representing the specified [vp, offset]. 716 * We either return the one we found or, if passed in, 717 * create one with identity of [vp, offset] of the 718 * pre-allocated page. If we find existing page but it is 719 * intransit coming in, it will have an "exclusive" lock 720 * and we wait for the i/o to complete. A page found on 721 * the free list is always reclaimed and then locked. 722 * On success, the page is locked, its data is valid and 723 * it isn't on the free list, while a NULL is returned 724 * if the page doesn't exist and newpp is NULL; 725 */ 726 page_t * 727 page_lookup_create( 728 vnode_t *vp, 729 u_offset_t off, 730 se_t se, 731 page_t *newpp, 732 spgcnt_t *nrelocp, 733 int flags) 734 { 735 page_t *pp; 736 kmutex_t *phm; 737 ulong_t index; 738 uint_t hash_locked; 739 uint_t es; 740 741 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 742 VM_STAT_ADD(page_lookup_cnt[0]); 743 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 744 745 /* 746 * Acquire the appropriate page hash lock since 747 * we have to search the hash list. Pages that 748 * hash to this list can't change identity while 749 * this lock is held. 750 */ 751 hash_locked = 0; 752 index = PAGE_HASH_FUNC(vp, off); 753 phm = NULL; 754 top: 755 PAGE_HASH_SEARCH(index, pp, vp, off); 756 if (pp != NULL) { 757 VM_STAT_ADD(page_lookup_cnt[1]); 758 es = (newpp != NULL) ? 1 : 0; 759 es |= flags; 760 if (!hash_locked) { 761 VM_STAT_ADD(page_lookup_cnt[2]); 762 if (!page_try_reclaim_lock(pp, se, es)) { 763 /* 764 * On a miss, acquire the phm. Then 765 * next time, page_lock() will be called, 766 * causing a wait if the page is busy. 767 * just looping with page_trylock() would 768 * get pretty boring. 769 */ 770 VM_STAT_ADD(page_lookup_cnt[3]); 771 phm = PAGE_HASH_MUTEX(index); 772 mutex_enter(phm); 773 hash_locked = 1; 774 goto top; 775 } 776 } else { 777 VM_STAT_ADD(page_lookup_cnt[4]); 778 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 779 VM_STAT_ADD(page_lookup_cnt[5]); 780 goto top; 781 } 782 } 783 784 /* 785 * Since `pp' is locked it can not change identity now. 786 * Reconfirm we locked the correct page. 787 * 788 * Both the p_vnode and p_offset *must* be cast volatile 789 * to force a reload of their values: The PAGE_HASH_SEARCH 790 * macro will have stuffed p_vnode and p_offset into 791 * registers before calling page_trylock(); another thread, 792 * actually holding the hash lock, could have changed the 793 * page's identity in memory, but our registers would not 794 * be changed, fooling the reconfirmation. If the hash 795 * lock was held during the search, the casting would 796 * not be needed. 797 */ 798 VM_STAT_ADD(page_lookup_cnt[6]); 799 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 800 ((volatile u_offset_t)(pp->p_offset) != off)) { 801 VM_STAT_ADD(page_lookup_cnt[7]); 802 if (hash_locked) { 803 panic("page_lookup_create: lost page %p", 804 (void *)pp); 805 /*NOTREACHED*/ 806 } 807 page_unlock(pp); 808 phm = PAGE_HASH_MUTEX(index); 809 mutex_enter(phm); 810 hash_locked = 1; 811 goto top; 812 } 813 814 /* 815 * If page_trylock() was called, then pp may still be on 816 * the cachelist (can't be on the free list, it would not 817 * have been found in the search). If it is on the 818 * cachelist it must be pulled now. To pull the page from 819 * the cachelist, it must be exclusively locked. 820 * 821 * The other big difference between page_trylock() and 822 * page_lock(), is that page_lock() will pull the 823 * page from whatever free list (the cache list in this 824 * case) the page is on. If page_trylock() was used 825 * above, then we have to do the reclaim ourselves. 826 */ 827 if ((!hash_locked) && (PP_ISFREE(pp))) { 828 ASSERT(PP_ISAGED(pp) == 0); 829 VM_STAT_ADD(page_lookup_cnt[8]); 830 831 /* 832 * page_relcaim will insure that we 833 * have this page exclusively 834 */ 835 836 if (!page_reclaim(pp, NULL)) { 837 /* 838 * Page_reclaim dropped whatever lock 839 * we held. 840 */ 841 VM_STAT_ADD(page_lookup_cnt[9]); 842 phm = PAGE_HASH_MUTEX(index); 843 mutex_enter(phm); 844 hash_locked = 1; 845 goto top; 846 } else if (se == SE_SHARED && newpp == NULL) { 847 VM_STAT_ADD(page_lookup_cnt[10]); 848 page_downgrade(pp); 849 } 850 } 851 852 if (hash_locked) { 853 mutex_exit(phm); 854 } 855 856 if (newpp != NULL && pp->p_szc < newpp->p_szc && 857 PAGE_EXCL(pp) && nrelocp != NULL) { 858 ASSERT(nrelocp != NULL); 859 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 860 NULL); 861 if (*nrelocp > 0) { 862 VM_STAT_COND_ADD(*nrelocp == 1, 863 page_lookup_cnt[11]); 864 VM_STAT_COND_ADD(*nrelocp > 1, 865 page_lookup_cnt[12]); 866 pp = newpp; 867 se = SE_EXCL; 868 } else { 869 if (se == SE_SHARED) { 870 page_downgrade(pp); 871 } 872 VM_STAT_ADD(page_lookup_cnt[13]); 873 } 874 } else if (newpp != NULL && nrelocp != NULL) { 875 if (PAGE_EXCL(pp) && se == SE_SHARED) { 876 page_downgrade(pp); 877 } 878 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 879 page_lookup_cnt[14]); 880 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 881 page_lookup_cnt[15]); 882 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 883 page_lookup_cnt[16]); 884 } else if (newpp != NULL && PAGE_EXCL(pp)) { 885 se = SE_EXCL; 886 } 887 } else if (!hash_locked) { 888 VM_STAT_ADD(page_lookup_cnt[17]); 889 phm = PAGE_HASH_MUTEX(index); 890 mutex_enter(phm); 891 hash_locked = 1; 892 goto top; 893 } else if (newpp != NULL) { 894 /* 895 * If we have a preallocated page then 896 * insert it now and basically behave like 897 * page_create. 898 */ 899 VM_STAT_ADD(page_lookup_cnt[18]); 900 /* 901 * Since we hold the page hash mutex and 902 * just searched for this page, page_hashin 903 * had better not fail. If it does, that 904 * means some thread did not follow the 905 * page hash mutex rules. Panic now and 906 * get it over with. As usual, go down 907 * holding all the locks. 908 */ 909 ASSERT(MUTEX_HELD(phm)); 910 if (!page_hashin(newpp, vp, off, phm)) { 911 ASSERT(MUTEX_HELD(phm)); 912 panic("page_lookup_create: hashin failed %p %p %llx %p", 913 (void *)newpp, (void *)vp, off, (void *)phm); 914 /*NOTREACHED*/ 915 } 916 ASSERT(MUTEX_HELD(phm)); 917 mutex_exit(phm); 918 phm = NULL; 919 page_set_props(newpp, P_REF); 920 page_io_lock(newpp); 921 pp = newpp; 922 se = SE_EXCL; 923 } else { 924 VM_STAT_ADD(page_lookup_cnt[19]); 925 mutex_exit(phm); 926 } 927 928 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 929 930 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 931 932 return (pp); 933 } 934 935 /* 936 * Search the hash list for the page representing the 937 * specified [vp, offset] and return it locked. Skip 938 * free pages and pages that cannot be locked as requested. 939 * Used while attempting to kluster pages. 940 */ 941 page_t * 942 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 943 { 944 page_t *pp; 945 kmutex_t *phm; 946 ulong_t index; 947 uint_t locked; 948 949 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 950 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 951 952 index = PAGE_HASH_FUNC(vp, off); 953 PAGE_HASH_SEARCH(index, pp, vp, off); 954 locked = 0; 955 if (pp == NULL) { 956 top: 957 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 958 locked = 1; 959 phm = PAGE_HASH_MUTEX(index); 960 mutex_enter(phm); 961 PAGE_HASH_SEARCH(index, pp, vp, off); 962 } 963 964 if (pp == NULL || PP_ISFREE(pp)) { 965 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 966 pp = NULL; 967 } else { 968 if (!page_trylock(pp, se)) { 969 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 970 pp = NULL; 971 } else { 972 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 973 /* 974 * See the comment in page_lookup() 975 */ 976 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 977 ((u_offset_t)(pp->p_offset) != off)) { 978 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 979 if (locked) { 980 panic("page_lookup_nowait %p", 981 (void *)pp); 982 /*NOTREACHED*/ 983 } 984 page_unlock(pp); 985 goto top; 986 } 987 if (PP_ISFREE(pp)) { 988 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 989 page_unlock(pp); 990 pp = NULL; 991 } 992 } 993 } 994 if (locked) { 995 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 996 mutex_exit(phm); 997 } 998 999 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 1000 1001 return (pp); 1002 } 1003 1004 /* 1005 * Search the hash list for a page with the specified [vp, off] 1006 * that is known to exist and is already locked. This routine 1007 * is typically used by segment SOFTUNLOCK routines. 1008 */ 1009 page_t * 1010 page_find(vnode_t *vp, u_offset_t off) 1011 { 1012 page_t *pp; 1013 kmutex_t *phm; 1014 ulong_t index; 1015 1016 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1017 VM_STAT_ADD(page_find_cnt); 1018 1019 index = PAGE_HASH_FUNC(vp, off); 1020 phm = PAGE_HASH_MUTEX(index); 1021 1022 mutex_enter(phm); 1023 PAGE_HASH_SEARCH(index, pp, vp, off); 1024 mutex_exit(phm); 1025 1026 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 1027 return (pp); 1028 } 1029 1030 /* 1031 * Determine whether a page with the specified [vp, off] 1032 * currently exists in the system. Obviously this should 1033 * only be considered as a hint since nothing prevents the 1034 * page from disappearing or appearing immediately after 1035 * the return from this routine. Subsequently, we don't 1036 * even bother to lock the list. 1037 */ 1038 page_t * 1039 page_exists(vnode_t *vp, u_offset_t off) 1040 { 1041 page_t *pp; 1042 ulong_t index; 1043 1044 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1045 VM_STAT_ADD(page_exists_cnt); 1046 1047 index = PAGE_HASH_FUNC(vp, off); 1048 PAGE_HASH_SEARCH(index, pp, vp, off); 1049 1050 return (pp); 1051 } 1052 1053 /* 1054 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1055 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1056 * with these pages locked SHARED. If necessary reclaim pages from 1057 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1058 * 1059 * If we fail to lock pages still return 1 if pages exist and contiguous. 1060 * But in this case return value is just a hint. ppa array won't be filled. 1061 * Caller should initialize ppa[0] as NULL to distinguish return value. 1062 * 1063 * Returns 0 if pages don't exist or not physically contiguous. 1064 * 1065 * This routine doesn't work for anonymous(swapfs) pages. 1066 */ 1067 int 1068 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1069 { 1070 pgcnt_t pages; 1071 pfn_t pfn; 1072 page_t *rootpp; 1073 pgcnt_t i; 1074 pgcnt_t j; 1075 u_offset_t save_off = off; 1076 ulong_t index; 1077 kmutex_t *phm; 1078 page_t *pp; 1079 uint_t pszc; 1080 int loopcnt = 0; 1081 1082 ASSERT(szc != 0); 1083 ASSERT(vp != NULL); 1084 ASSERT(!IS_SWAPFSVP(vp)); 1085 ASSERT(!VN_ISKAS(vp)); 1086 1087 again: 1088 if (++loopcnt > 3) { 1089 VM_STAT_ADD(page_exphcontg[0]); 1090 return (0); 1091 } 1092 1093 index = PAGE_HASH_FUNC(vp, off); 1094 phm = PAGE_HASH_MUTEX(index); 1095 1096 mutex_enter(phm); 1097 PAGE_HASH_SEARCH(index, pp, vp, off); 1098 mutex_exit(phm); 1099 1100 VM_STAT_ADD(page_exphcontg[1]); 1101 1102 if (pp == NULL) { 1103 VM_STAT_ADD(page_exphcontg[2]); 1104 return (0); 1105 } 1106 1107 pages = page_get_pagecnt(szc); 1108 rootpp = pp; 1109 pfn = rootpp->p_pagenum; 1110 1111 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1112 VM_STAT_ADD(page_exphcontg[3]); 1113 if (!page_trylock(pp, SE_SHARED)) { 1114 VM_STAT_ADD(page_exphcontg[4]); 1115 return (1); 1116 } 1117 /* 1118 * Also check whether p_pagenum was modified by DR. 1119 */ 1120 if (pp->p_szc != pszc || pp->p_vnode != vp || 1121 pp->p_offset != off || pp->p_pagenum != pfn) { 1122 VM_STAT_ADD(page_exphcontg[5]); 1123 page_unlock(pp); 1124 off = save_off; 1125 goto again; 1126 } 1127 /* 1128 * szc was non zero and vnode and offset matched after we 1129 * locked the page it means it can't become free on us. 1130 */ 1131 ASSERT(!PP_ISFREE(pp)); 1132 if (!IS_P2ALIGNED(pfn, pages)) { 1133 page_unlock(pp); 1134 return (0); 1135 } 1136 ppa[0] = pp; 1137 pp++; 1138 off += PAGESIZE; 1139 pfn++; 1140 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1141 if (!page_trylock(pp, SE_SHARED)) { 1142 VM_STAT_ADD(page_exphcontg[6]); 1143 pp--; 1144 while (i-- > 0) { 1145 page_unlock(pp); 1146 pp--; 1147 } 1148 ppa[0] = NULL; 1149 return (1); 1150 } 1151 if (pp->p_szc != pszc) { 1152 VM_STAT_ADD(page_exphcontg[7]); 1153 page_unlock(pp); 1154 pp--; 1155 while (i-- > 0) { 1156 page_unlock(pp); 1157 pp--; 1158 } 1159 ppa[0] = NULL; 1160 off = save_off; 1161 goto again; 1162 } 1163 /* 1164 * szc the same as for previous already locked pages 1165 * with right identity. Since this page had correct 1166 * szc after we locked it can't get freed or destroyed 1167 * and therefore must have the expected identity. 1168 */ 1169 ASSERT(!PP_ISFREE(pp)); 1170 if (pp->p_vnode != vp || 1171 pp->p_offset != off) { 1172 panic("page_exists_physcontig: " 1173 "large page identity doesn't match"); 1174 } 1175 ppa[i] = pp; 1176 ASSERT(pp->p_pagenum == pfn); 1177 } 1178 VM_STAT_ADD(page_exphcontg[8]); 1179 ppa[pages] = NULL; 1180 return (1); 1181 } else if (pszc >= szc) { 1182 VM_STAT_ADD(page_exphcontg[9]); 1183 if (!IS_P2ALIGNED(pfn, pages)) { 1184 return (0); 1185 } 1186 return (1); 1187 } 1188 1189 if (!IS_P2ALIGNED(pfn, pages)) { 1190 VM_STAT_ADD(page_exphcontg[10]); 1191 return (0); 1192 } 1193 1194 if (page_numtomemseg_nolock(pfn) != 1195 page_numtomemseg_nolock(pfn + pages - 1)) { 1196 VM_STAT_ADD(page_exphcontg[11]); 1197 return (0); 1198 } 1199 1200 /* 1201 * We loop up 4 times across pages to promote page size. 1202 * We're extra cautious to promote page size atomically with respect 1203 * to everybody else. But we can probably optimize into 1 loop if 1204 * this becomes an issue. 1205 */ 1206 1207 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1208 if (!page_trylock(pp, SE_EXCL)) { 1209 VM_STAT_ADD(page_exphcontg[12]); 1210 break; 1211 } 1212 /* 1213 * Check whether p_pagenum was modified by DR. 1214 */ 1215 if (pp->p_pagenum != pfn) { 1216 page_unlock(pp); 1217 break; 1218 } 1219 if (pp->p_vnode != vp || 1220 pp->p_offset != off) { 1221 VM_STAT_ADD(page_exphcontg[13]); 1222 page_unlock(pp); 1223 break; 1224 } 1225 if (pp->p_szc >= szc) { 1226 ASSERT(i == 0); 1227 page_unlock(pp); 1228 off = save_off; 1229 goto again; 1230 } 1231 } 1232 1233 if (i != pages) { 1234 VM_STAT_ADD(page_exphcontg[14]); 1235 --pp; 1236 while (i-- > 0) { 1237 page_unlock(pp); 1238 --pp; 1239 } 1240 return (0); 1241 } 1242 1243 pp = rootpp; 1244 for (i = 0; i < pages; i++, pp++) { 1245 if (PP_ISFREE(pp)) { 1246 VM_STAT_ADD(page_exphcontg[15]); 1247 ASSERT(!PP_ISAGED(pp)); 1248 ASSERT(pp->p_szc == 0); 1249 if (!page_reclaim(pp, NULL)) { 1250 break; 1251 } 1252 } else { 1253 ASSERT(pp->p_szc < szc); 1254 VM_STAT_ADD(page_exphcontg[16]); 1255 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1256 } 1257 } 1258 if (i < pages) { 1259 VM_STAT_ADD(page_exphcontg[17]); 1260 /* 1261 * page_reclaim failed because we were out of memory. 1262 * drop the rest of the locks and return because this page 1263 * must be already reallocated anyway. 1264 */ 1265 pp = rootpp; 1266 for (j = 0; j < pages; j++, pp++) { 1267 if (j != i) { 1268 page_unlock(pp); 1269 } 1270 } 1271 return (0); 1272 } 1273 1274 off = save_off; 1275 pp = rootpp; 1276 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1277 ASSERT(PAGE_EXCL(pp)); 1278 ASSERT(!PP_ISFREE(pp)); 1279 ASSERT(!hat_page_is_mapped(pp)); 1280 ASSERT(pp->p_vnode == vp); 1281 ASSERT(pp->p_offset == off); 1282 pp->p_szc = szc; 1283 } 1284 pp = rootpp; 1285 for (i = 0; i < pages; i++, pp++) { 1286 if (ppa == NULL) { 1287 page_unlock(pp); 1288 } else { 1289 ppa[i] = pp; 1290 page_downgrade(ppa[i]); 1291 } 1292 } 1293 if (ppa != NULL) { 1294 ppa[pages] = NULL; 1295 } 1296 VM_STAT_ADD(page_exphcontg[18]); 1297 ASSERT(vp->v_pages != NULL); 1298 return (1); 1299 } 1300 1301 /* 1302 * Determine whether a page with the specified [vp, off] 1303 * currently exists in the system and if so return its 1304 * size code. Obviously this should only be considered as 1305 * a hint since nothing prevents the page from disappearing 1306 * or appearing immediately after the return from this routine. 1307 */ 1308 int 1309 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1310 { 1311 page_t *pp; 1312 kmutex_t *phm; 1313 ulong_t index; 1314 int rc = 0; 1315 1316 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1317 ASSERT(szc != NULL); 1318 VM_STAT_ADD(page_exists_forreal_cnt); 1319 1320 index = PAGE_HASH_FUNC(vp, off); 1321 phm = PAGE_HASH_MUTEX(index); 1322 1323 mutex_enter(phm); 1324 PAGE_HASH_SEARCH(index, pp, vp, off); 1325 if (pp != NULL) { 1326 *szc = pp->p_szc; 1327 rc = 1; 1328 } 1329 mutex_exit(phm); 1330 return (rc); 1331 } 1332 1333 /* wakeup threads waiting for pages in page_create_get_something() */ 1334 void 1335 wakeup_pcgs(void) 1336 { 1337 if (!CV_HAS_WAITERS(&pcgs_cv)) 1338 return; 1339 cv_broadcast(&pcgs_cv); 1340 } 1341 1342 /* 1343 * 'freemem' is used all over the kernel as an indication of how many 1344 * pages are free (either on the cache list or on the free page list) 1345 * in the system. In very few places is a really accurate 'freemem' 1346 * needed. To avoid contention of the lock protecting a the 1347 * single freemem, it was spread out into NCPU buckets. Set_freemem 1348 * sets freemem to the total of all NCPU buckets. It is called from 1349 * clock() on each TICK. 1350 */ 1351 void 1352 set_freemem() 1353 { 1354 struct pcf *p; 1355 ulong_t t; 1356 uint_t i; 1357 1358 t = 0; 1359 p = pcf; 1360 for (i = 0; i < pcf_fanout; i++) { 1361 t += p->pcf_count; 1362 p++; 1363 } 1364 freemem = t; 1365 1366 /* 1367 * Don't worry about grabbing mutex. It's not that 1368 * critical if we miss a tick or two. This is 1369 * where we wakeup possible delayers in 1370 * page_create_get_something(). 1371 */ 1372 wakeup_pcgs(); 1373 } 1374 1375 ulong_t 1376 get_freemem() 1377 { 1378 struct pcf *p; 1379 ulong_t t; 1380 uint_t i; 1381 1382 t = 0; 1383 p = pcf; 1384 for (i = 0; i < pcf_fanout; i++) { 1385 t += p->pcf_count; 1386 p++; 1387 } 1388 /* 1389 * We just calculated it, might as well set it. 1390 */ 1391 freemem = t; 1392 return (t); 1393 } 1394 1395 /* 1396 * Acquire all of the page cache & free (pcf) locks. 1397 */ 1398 void 1399 pcf_acquire_all() 1400 { 1401 struct pcf *p; 1402 uint_t i; 1403 1404 p = pcf; 1405 for (i = 0; i < pcf_fanout; i++) { 1406 mutex_enter(&p->pcf_lock); 1407 p++; 1408 } 1409 } 1410 1411 /* 1412 * Release all the pcf_locks. 1413 */ 1414 void 1415 pcf_release_all() 1416 { 1417 struct pcf *p; 1418 uint_t i; 1419 1420 p = pcf; 1421 for (i = 0; i < pcf_fanout; i++) { 1422 mutex_exit(&p->pcf_lock); 1423 p++; 1424 } 1425 } 1426 1427 /* 1428 * Inform the VM system that we need some pages freed up. 1429 * Calls must be symmetric, e.g.: 1430 * 1431 * page_needfree(100); 1432 * wait a bit; 1433 * page_needfree(-100); 1434 */ 1435 void 1436 page_needfree(spgcnt_t npages) 1437 { 1438 mutex_enter(&new_freemem_lock); 1439 needfree += npages; 1440 mutex_exit(&new_freemem_lock); 1441 } 1442 1443 /* 1444 * Throttle for page_create(): try to prevent freemem from dropping 1445 * below throttlefree. We can't provide a 100% guarantee because 1446 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1447 * nibble away at the freelist. However, we can block all PG_WAIT 1448 * allocations until memory becomes available. The motivation is 1449 * that several things can fall apart when there's no free memory: 1450 * 1451 * (1) If pageout() needs memory to push a page, the system deadlocks. 1452 * 1453 * (2) By (broken) specification, timeout(9F) can neither fail nor 1454 * block, so it has no choice but to panic the system if it 1455 * cannot allocate a callout structure. 1456 * 1457 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1458 * it panics if it cannot allocate a callback structure. 1459 * 1460 * (4) Untold numbers of third-party drivers have not yet been hardened 1461 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1462 * success and panic the system with a data fault on failure. 1463 * (The long-term solution to this particular problem is to ship 1464 * hostile fault-injecting DEBUG kernels with the DDK.) 1465 * 1466 * It is theoretically impossible to guarantee success of non-blocking 1467 * allocations, but in practice, this throttle is very hard to break. 1468 */ 1469 static int 1470 page_create_throttle(pgcnt_t npages, int flags) 1471 { 1472 ulong_t fm; 1473 uint_t i; 1474 pgcnt_t tf; /* effective value of throttlefree */ 1475 1476 /* 1477 * Normal priority allocations. 1478 */ 1479 if ((flags & (PG_WAIT | PG_NORMALPRI)) == PG_NORMALPRI) { 1480 ASSERT(!(flags & (PG_PANIC | PG_PUSHPAGE))); 1481 return (freemem >= npages + throttlefree); 1482 } 1483 1484 /* 1485 * Never deny pages when: 1486 * - it's a thread that cannot block [NOMEMWAIT()] 1487 * - the allocation cannot block and must not fail 1488 * - the allocation cannot block and is pageout dispensated 1489 */ 1490 if (NOMEMWAIT() || 1491 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1492 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1493 return (1); 1494 1495 /* 1496 * If the allocation can't block, we look favorably upon it 1497 * unless we're below pageout_reserve. In that case we fail 1498 * the allocation because we want to make sure there are a few 1499 * pages available for pageout. 1500 */ 1501 if ((flags & PG_WAIT) == 0) 1502 return (freemem >= npages + pageout_reserve); 1503 1504 /* Calculate the effective throttlefree value */ 1505 tf = throttlefree - 1506 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1507 1508 cv_signal(&proc_pageout->p_cv); 1509 1510 for (;;) { 1511 fm = 0; 1512 pcf_acquire_all(); 1513 mutex_enter(&new_freemem_lock); 1514 for (i = 0; i < pcf_fanout; i++) { 1515 fm += pcf[i].pcf_count; 1516 pcf[i].pcf_wait++; 1517 mutex_exit(&pcf[i].pcf_lock); 1518 } 1519 freemem = fm; 1520 if (freemem >= npages + tf) { 1521 mutex_exit(&new_freemem_lock); 1522 break; 1523 } 1524 needfree += npages; 1525 freemem_wait++; 1526 cv_wait(&freemem_cv, &new_freemem_lock); 1527 freemem_wait--; 1528 needfree -= npages; 1529 mutex_exit(&new_freemem_lock); 1530 } 1531 return (1); 1532 } 1533 1534 /* 1535 * page_create_wait() is called to either coalesce pages from the 1536 * different pcf buckets or to wait because there simply are not 1537 * enough pages to satisfy the caller's request. 1538 * 1539 * Sadly, this is called from platform/vm/vm_machdep.c 1540 */ 1541 int 1542 page_create_wait(pgcnt_t npages, uint_t flags) 1543 { 1544 pgcnt_t total; 1545 uint_t i; 1546 struct pcf *p; 1547 1548 /* 1549 * Wait until there are enough free pages to satisfy our 1550 * entire request. 1551 * We set needfree += npages before prodding pageout, to make sure 1552 * it does real work when npages > lotsfree > freemem. 1553 */ 1554 VM_STAT_ADD(page_create_not_enough); 1555 1556 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1557 checkagain: 1558 if ((flags & PG_NORELOC) && 1559 kcage_freemem < kcage_throttlefree + npages) 1560 (void) kcage_create_throttle(npages, flags); 1561 1562 if (freemem < npages + throttlefree) 1563 if (!page_create_throttle(npages, flags)) 1564 return (0); 1565 1566 if (pcf_decrement_bucket(npages) || 1567 pcf_decrement_multiple(&total, npages, 0)) 1568 return (1); 1569 1570 /* 1571 * All of the pcf locks are held, there are not enough pages 1572 * to satisfy the request (npages < total). 1573 * Be sure to acquire the new_freemem_lock before dropping 1574 * the pcf locks. This prevents dropping wakeups in page_free(). 1575 * The order is always pcf_lock then new_freemem_lock. 1576 * 1577 * Since we hold all the pcf locks, it is a good time to set freemem. 1578 * 1579 * If the caller does not want to wait, return now. 1580 * Else turn the pageout daemon loose to find something 1581 * and wait till it does. 1582 * 1583 */ 1584 freemem = total; 1585 1586 if ((flags & PG_WAIT) == 0) { 1587 pcf_release_all(); 1588 1589 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1590 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1591 return (0); 1592 } 1593 1594 ASSERT(proc_pageout != NULL); 1595 cv_signal(&proc_pageout->p_cv); 1596 1597 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1598 "page_create_sleep_start: freemem %ld needfree %ld", 1599 freemem, needfree); 1600 1601 /* 1602 * We are going to wait. 1603 * We currently hold all of the pcf_locks, 1604 * get the new_freemem_lock (it protects freemem_wait), 1605 * before dropping the pcf_locks. 1606 */ 1607 mutex_enter(&new_freemem_lock); 1608 1609 p = pcf; 1610 for (i = 0; i < pcf_fanout; i++) { 1611 p->pcf_wait++; 1612 mutex_exit(&p->pcf_lock); 1613 p++; 1614 } 1615 1616 needfree += npages; 1617 freemem_wait++; 1618 1619 cv_wait(&freemem_cv, &new_freemem_lock); 1620 1621 freemem_wait--; 1622 needfree -= npages; 1623 1624 mutex_exit(&new_freemem_lock); 1625 1626 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1627 "page_create_sleep_end: freemem %ld needfree %ld", 1628 freemem, needfree); 1629 1630 VM_STAT_ADD(page_create_not_enough_again); 1631 goto checkagain; 1632 } 1633 /* 1634 * A routine to do the opposite of page_create_wait(). 1635 */ 1636 void 1637 page_create_putback(spgcnt_t npages) 1638 { 1639 struct pcf *p; 1640 pgcnt_t lump; 1641 uint_t *which; 1642 1643 /* 1644 * When a contiguous lump is broken up, we have to 1645 * deal with lots of pages (min 64) so lets spread 1646 * the wealth around. 1647 */ 1648 lump = roundup(npages, pcf_fanout) / pcf_fanout; 1649 freemem += npages; 1650 1651 for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) { 1652 which = &p->pcf_count; 1653 1654 mutex_enter(&p->pcf_lock); 1655 1656 if (p->pcf_block) { 1657 which = &p->pcf_reserve; 1658 } 1659 1660 if (lump < npages) { 1661 *which += (uint_t)lump; 1662 npages -= lump; 1663 } else { 1664 *which += (uint_t)npages; 1665 npages = 0; 1666 } 1667 1668 if (p->pcf_wait) { 1669 mutex_enter(&new_freemem_lock); 1670 /* 1671 * Check to see if some other thread 1672 * is actually waiting. Another bucket 1673 * may have woken it up by now. If there 1674 * are no waiters, then set our pcf_wait 1675 * count to zero to avoid coming in here 1676 * next time. 1677 */ 1678 if (freemem_wait) { 1679 if (npages > 1) { 1680 cv_broadcast(&freemem_cv); 1681 } else { 1682 cv_signal(&freemem_cv); 1683 } 1684 p->pcf_wait--; 1685 } else { 1686 p->pcf_wait = 0; 1687 } 1688 mutex_exit(&new_freemem_lock); 1689 } 1690 mutex_exit(&p->pcf_lock); 1691 } 1692 ASSERT(npages == 0); 1693 } 1694 1695 /* 1696 * A helper routine for page_create_get_something. 1697 * The indenting got to deep down there. 1698 * Unblock the pcf counters. Any pages freed after 1699 * pcf_block got set are moved to pcf_count and 1700 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1701 */ 1702 static void 1703 pcgs_unblock(void) 1704 { 1705 int i; 1706 struct pcf *p; 1707 1708 /* Update freemem while we're here. */ 1709 freemem = 0; 1710 p = pcf; 1711 for (i = 0; i < pcf_fanout; i++) { 1712 mutex_enter(&p->pcf_lock); 1713 ASSERT(p->pcf_count == 0); 1714 p->pcf_count = p->pcf_reserve; 1715 p->pcf_block = 0; 1716 freemem += p->pcf_count; 1717 if (p->pcf_wait) { 1718 mutex_enter(&new_freemem_lock); 1719 if (freemem_wait) { 1720 if (p->pcf_reserve > 1) { 1721 cv_broadcast(&freemem_cv); 1722 p->pcf_wait = 0; 1723 } else { 1724 cv_signal(&freemem_cv); 1725 p->pcf_wait--; 1726 } 1727 } else { 1728 p->pcf_wait = 0; 1729 } 1730 mutex_exit(&new_freemem_lock); 1731 } 1732 p->pcf_reserve = 0; 1733 mutex_exit(&p->pcf_lock); 1734 p++; 1735 } 1736 } 1737 1738 /* 1739 * Called from page_create_va() when both the cache and free lists 1740 * have been checked once. 1741 * 1742 * Either returns a page or panics since the accounting was done 1743 * way before we got here. 1744 * 1745 * We don't come here often, so leave the accounting on permanently. 1746 */ 1747 1748 #define MAX_PCGS 100 1749 1750 #ifdef DEBUG 1751 #define PCGS_TRIES 100 1752 #else /* DEBUG */ 1753 #define PCGS_TRIES 10 1754 #endif /* DEBUG */ 1755 1756 #ifdef VM_STATS 1757 uint_t pcgs_counts[PCGS_TRIES]; 1758 uint_t pcgs_too_many; 1759 uint_t pcgs_entered; 1760 uint_t pcgs_entered_noreloc; 1761 uint_t pcgs_locked; 1762 uint_t pcgs_cagelocked; 1763 #endif /* VM_STATS */ 1764 1765 static page_t * 1766 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1767 caddr_t vaddr, uint_t flags) 1768 { 1769 uint_t count; 1770 page_t *pp; 1771 uint_t locked, i; 1772 struct pcf *p; 1773 lgrp_t *lgrp; 1774 int cagelocked = 0; 1775 1776 VM_STAT_ADD(pcgs_entered); 1777 1778 /* 1779 * Tap any reserve freelists: if we fail now, we'll die 1780 * since the page(s) we're looking for have already been 1781 * accounted for. 1782 */ 1783 flags |= PG_PANIC; 1784 1785 if ((flags & PG_NORELOC) != 0) { 1786 VM_STAT_ADD(pcgs_entered_noreloc); 1787 /* 1788 * Requests for free pages from critical threads 1789 * such as pageout still won't throttle here, but 1790 * we must try again, to give the cageout thread 1791 * another chance to catch up. Since we already 1792 * accounted for the pages, we had better get them 1793 * this time. 1794 * 1795 * N.B. All non-critical threads acquire the pcgs_cagelock 1796 * to serialize access to the freelists. This implements a 1797 * turnstile-type synchornization to avoid starvation of 1798 * critical requests for PG_NORELOC memory by non-critical 1799 * threads: all non-critical threads must acquire a 'ticket' 1800 * before passing through, which entails making sure 1801 * kcage_freemem won't fall below minfree prior to grabbing 1802 * pages from the freelists. 1803 */ 1804 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1805 mutex_enter(&pcgs_cagelock); 1806 cagelocked = 1; 1807 VM_STAT_ADD(pcgs_cagelocked); 1808 } 1809 } 1810 1811 /* 1812 * Time to get serious. 1813 * We failed to get a `correctly colored' page from both the 1814 * free and cache lists. 1815 * We escalate in stage. 1816 * 1817 * First try both lists without worring about color. 1818 * 1819 * Then, grab all page accounting locks (ie. pcf[]) and 1820 * steal any pages that they have and set the pcf_block flag to 1821 * stop deletions from the lists. This will help because 1822 * a page can get added to the free list while we are looking 1823 * at the cache list, then another page could be added to the cache 1824 * list allowing the page on the free list to be removed as we 1825 * move from looking at the cache list to the free list. This 1826 * could happen over and over. We would never find the page 1827 * we have accounted for. 1828 * 1829 * Noreloc pages are a subset of the global (relocatable) page pool. 1830 * They are not tracked separately in the pcf bins, so it is 1831 * impossible to know when doing pcf accounting if the available 1832 * page(s) are noreloc pages or not. When looking for a noreloc page 1833 * it is quite easy to end up here even if the global (relocatable) 1834 * page pool has plenty of free pages but the noreloc pool is empty. 1835 * 1836 * When the noreloc pool is empty (or low), additional noreloc pages 1837 * are created by converting pages from the global page pool. This 1838 * process will stall during pcf accounting if the pcf bins are 1839 * already locked. Such is the case when a noreloc allocation is 1840 * looping here in page_create_get_something waiting for more noreloc 1841 * pages to appear. 1842 * 1843 * Short of adding a new field to the pcf bins to accurately track 1844 * the number of free noreloc pages, we instead do not grab the 1845 * pcgs_lock, do not set the pcf blocks and do not timeout when 1846 * allocating a noreloc page. This allows noreloc allocations to 1847 * loop without blocking global page pool allocations. 1848 * 1849 * NOTE: the behaviour of page_create_get_something has not changed 1850 * for the case of global page pool allocations. 1851 */ 1852 1853 flags &= ~PG_MATCH_COLOR; 1854 locked = 0; 1855 #if defined(__i386) || defined(__amd64) 1856 flags = page_create_update_flags_x86(flags); 1857 #endif 1858 1859 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1860 1861 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1862 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1863 flags, lgrp); 1864 if (pp == NULL) { 1865 pp = page_get_cachelist(vp, off, seg, vaddr, 1866 flags, lgrp); 1867 } 1868 if (pp == NULL) { 1869 /* 1870 * Serialize. Don't fight with other pcgs(). 1871 */ 1872 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1873 mutex_enter(&pcgs_lock); 1874 VM_STAT_ADD(pcgs_locked); 1875 locked = 1; 1876 p = pcf; 1877 for (i = 0; i < pcf_fanout; i++) { 1878 mutex_enter(&p->pcf_lock); 1879 ASSERT(p->pcf_block == 0); 1880 p->pcf_block = 1; 1881 p->pcf_reserve = p->pcf_count; 1882 p->pcf_count = 0; 1883 mutex_exit(&p->pcf_lock); 1884 p++; 1885 } 1886 freemem = 0; 1887 } 1888 1889 if (count) { 1890 /* 1891 * Since page_free() puts pages on 1892 * a list then accounts for it, we 1893 * just have to wait for page_free() 1894 * to unlock any page it was working 1895 * with. The page_lock()-page_reclaim() 1896 * path falls in the same boat. 1897 * 1898 * We don't need to check on the 1899 * PG_WAIT flag, we have already 1900 * accounted for the page we are 1901 * looking for in page_create_va(). 1902 * 1903 * We just wait a moment to let any 1904 * locked pages on the lists free up, 1905 * then continue around and try again. 1906 * 1907 * Will be awakened by set_freemem(). 1908 */ 1909 mutex_enter(&pcgs_wait_lock); 1910 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1911 mutex_exit(&pcgs_wait_lock); 1912 } 1913 } else { 1914 #ifdef VM_STATS 1915 if (count >= PCGS_TRIES) { 1916 VM_STAT_ADD(pcgs_too_many); 1917 } else { 1918 VM_STAT_ADD(pcgs_counts[count]); 1919 } 1920 #endif 1921 if (locked) { 1922 pcgs_unblock(); 1923 mutex_exit(&pcgs_lock); 1924 } 1925 if (cagelocked) 1926 mutex_exit(&pcgs_cagelock); 1927 return (pp); 1928 } 1929 } 1930 /* 1931 * we go down holding the pcf locks. 1932 */ 1933 panic("no %spage found %d", 1934 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1935 /*NOTREACHED*/ 1936 } 1937 1938 /* 1939 * Create enough pages for "bytes" worth of data starting at 1940 * "off" in "vp". 1941 * 1942 * Where flag must be one of: 1943 * 1944 * PG_EXCL: Exclusive create (fail if any page already 1945 * exists in the page cache) which does not 1946 * wait for memory to become available. 1947 * 1948 * PG_WAIT: Non-exclusive create which can wait for 1949 * memory to become available. 1950 * 1951 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1952 * (Not Supported) 1953 * 1954 * A doubly linked list of pages is returned to the caller. Each page 1955 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1956 * lock. 1957 * 1958 * Unable to change the parameters to page_create() in a minor release, 1959 * we renamed page_create() to page_create_va(), changed all known calls 1960 * from page_create() to page_create_va(), and created this wrapper. 1961 * 1962 * Upon a major release, we should break compatibility by deleting this 1963 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1964 * 1965 * NOTE: There is a copy of this interface as page_create_io() in 1966 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1967 * there. 1968 */ 1969 page_t * 1970 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1971 { 1972 caddr_t random_vaddr; 1973 struct seg kseg; 1974 1975 #ifdef DEBUG 1976 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1977 (void *)caller()); 1978 #endif 1979 1980 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1981 (uintptr_t)(off >> PAGESHIFT)); 1982 kseg.s_as = &kas; 1983 1984 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1985 } 1986 1987 #ifdef DEBUG 1988 uint32_t pg_alloc_pgs_mtbf = 0; 1989 #endif 1990 1991 /* 1992 * Used for large page support. It will attempt to allocate 1993 * a large page(s) off the freelist. 1994 * 1995 * Returns non zero on failure. 1996 */ 1997 int 1998 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1999 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags) 2000 { 2001 pgcnt_t npgs, curnpgs, totpgs; 2002 size_t pgsz; 2003 page_t *pplist = NULL, *pp; 2004 int err = 0; 2005 lgrp_t *lgrp; 2006 2007 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 2008 ASSERT(pgflags == 0 || pgflags == PG_LOCAL); 2009 2010 /* 2011 * Check if system heavily prefers local large pages over remote 2012 * on systems with multiple lgroups. 2013 */ 2014 if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) { 2015 pgflags = PG_LOCAL; 2016 } 2017 2018 VM_STAT_ADD(alloc_pages[0]); 2019 2020 #ifdef DEBUG 2021 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 2022 return (ENOMEM); 2023 } 2024 #endif 2025 2026 /* 2027 * One must be NULL but not both. 2028 * And one must be non NULL but not both. 2029 */ 2030 ASSERT(basepp != NULL || ppa != NULL); 2031 ASSERT(basepp == NULL || ppa == NULL); 2032 2033 #if defined(__i386) || defined(__amd64) 2034 while (page_chk_freelist(szc) == 0) { 2035 VM_STAT_ADD(alloc_pages[8]); 2036 if (anypgsz == 0 || --szc == 0) 2037 return (ENOMEM); 2038 } 2039 #endif 2040 2041 pgsz = page_get_pagesize(szc); 2042 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 2043 2044 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 2045 2046 (void) page_create_wait(npgs, PG_WAIT); 2047 2048 while (npgs && szc) { 2049 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2050 if (pgflags == PG_LOCAL) { 2051 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2052 pgflags, lgrp); 2053 if (pp == NULL) { 2054 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2055 0, lgrp); 2056 } 2057 } else { 2058 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2059 0, lgrp); 2060 } 2061 if (pp != NULL) { 2062 VM_STAT_ADD(alloc_pages[1]); 2063 page_list_concat(&pplist, &pp); 2064 ASSERT(npgs >= curnpgs); 2065 npgs -= curnpgs; 2066 } else if (anypgsz) { 2067 VM_STAT_ADD(alloc_pages[2]); 2068 szc--; 2069 pgsz = page_get_pagesize(szc); 2070 curnpgs = pgsz >> PAGESHIFT; 2071 } else { 2072 VM_STAT_ADD(alloc_pages[3]); 2073 ASSERT(npgs == totpgs); 2074 page_create_putback(npgs); 2075 return (ENOMEM); 2076 } 2077 } 2078 if (szc == 0) { 2079 VM_STAT_ADD(alloc_pages[4]); 2080 ASSERT(npgs != 0); 2081 page_create_putback(npgs); 2082 err = ENOMEM; 2083 } else if (basepp != NULL) { 2084 ASSERT(npgs == 0); 2085 ASSERT(ppa == NULL); 2086 *basepp = pplist; 2087 } 2088 2089 npgs = totpgs - npgs; 2090 pp = pplist; 2091 2092 /* 2093 * Clear the free and age bits. Also if we were passed in a ppa then 2094 * fill it in with all the constituent pages from the large page. But 2095 * if we failed to allocate all the pages just free what we got. 2096 */ 2097 while (npgs != 0) { 2098 ASSERT(PP_ISFREE(pp)); 2099 ASSERT(PP_ISAGED(pp)); 2100 if (ppa != NULL || err != 0) { 2101 if (err == 0) { 2102 VM_STAT_ADD(alloc_pages[5]); 2103 PP_CLRFREE(pp); 2104 PP_CLRAGED(pp); 2105 page_sub(&pplist, pp); 2106 *ppa++ = pp; 2107 npgs--; 2108 } else { 2109 VM_STAT_ADD(alloc_pages[6]); 2110 ASSERT(pp->p_szc != 0); 2111 curnpgs = page_get_pagecnt(pp->p_szc); 2112 page_list_break(&pp, &pplist, curnpgs); 2113 page_list_add_pages(pp, 0); 2114 page_create_putback(curnpgs); 2115 ASSERT(npgs >= curnpgs); 2116 npgs -= curnpgs; 2117 } 2118 pp = pplist; 2119 } else { 2120 VM_STAT_ADD(alloc_pages[7]); 2121 PP_CLRFREE(pp); 2122 PP_CLRAGED(pp); 2123 pp = pp->p_next; 2124 npgs--; 2125 } 2126 } 2127 return (err); 2128 } 2129 2130 /* 2131 * Get a single large page off of the freelists, and set it up for use. 2132 * Number of bytes requested must be a supported page size. 2133 * 2134 * Note that this call may fail even if there is sufficient 2135 * memory available or PG_WAIT is set, so the caller must 2136 * be willing to fallback on page_create_va(), block and retry, 2137 * or fail the requester. 2138 */ 2139 page_t * 2140 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2141 struct seg *seg, caddr_t vaddr, void *arg) 2142 { 2143 pgcnt_t npages; 2144 page_t *pp; 2145 page_t *rootpp; 2146 lgrp_t *lgrp; 2147 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2148 2149 ASSERT(vp != NULL); 2150 2151 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2152 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0); 2153 /* but no others */ 2154 2155 ASSERT((flags & PG_EXCL) == PG_EXCL); 2156 2157 npages = btop(bytes); 2158 2159 if (!kcage_on || panicstr) { 2160 /* 2161 * Cage is OFF, or we are single threaded in 2162 * panic, so make everything a RELOC request. 2163 */ 2164 flags &= ~PG_NORELOC; 2165 } 2166 2167 /* 2168 * Make sure there's adequate physical memory available. 2169 * Note: PG_WAIT is ignored here. 2170 */ 2171 if (freemem <= throttlefree + npages) { 2172 VM_STAT_ADD(page_create_large_cnt[1]); 2173 return (NULL); 2174 } 2175 2176 /* 2177 * If cage is on, dampen draw from cage when available 2178 * cage space is low. 2179 */ 2180 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2181 kcage_freemem < kcage_throttlefree + npages) { 2182 2183 /* 2184 * The cage is on, the caller wants PG_NORELOC 2185 * pages and available cage memory is very low. 2186 * Call kcage_create_throttle() to attempt to 2187 * control demand on the cage. 2188 */ 2189 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2190 VM_STAT_ADD(page_create_large_cnt[2]); 2191 return (NULL); 2192 } 2193 } 2194 2195 if (!pcf_decrement_bucket(npages) && 2196 !pcf_decrement_multiple(NULL, npages, 1)) { 2197 VM_STAT_ADD(page_create_large_cnt[4]); 2198 return (NULL); 2199 } 2200 2201 /* 2202 * This is where this function behaves fundamentally differently 2203 * than page_create_va(); since we're intending to map the page 2204 * with a single TTE, we have to get it as a physically contiguous 2205 * hardware pagesize chunk. If we can't, we fail. 2206 */ 2207 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2208 LGRP_EXISTS(lgrp_table[*lgrpid])) 2209 lgrp = lgrp_table[*lgrpid]; 2210 else 2211 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2212 2213 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2214 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2215 page_create_putback(npages); 2216 VM_STAT_ADD(page_create_large_cnt[5]); 2217 return (NULL); 2218 } 2219 2220 /* 2221 * if we got the page with the wrong mtype give it back this is a 2222 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2223 * inside "if" and the workaround becomes just a nop 2224 */ 2225 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2226 page_list_add_pages(rootpp, 0); 2227 page_create_putback(npages); 2228 VM_STAT_ADD(page_create_large_cnt[6]); 2229 return (NULL); 2230 } 2231 2232 /* 2233 * If satisfying this request has left us with too little 2234 * memory, start the wheels turning to get some back. The 2235 * first clause of the test prevents waking up the pageout 2236 * daemon in situations where it would decide that there's 2237 * nothing to do. 2238 */ 2239 if (nscan < desscan && freemem < minfree) { 2240 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2241 "pageout_cv_signal:freemem %ld", freemem); 2242 cv_signal(&proc_pageout->p_cv); 2243 } 2244 2245 pp = rootpp; 2246 while (npages--) { 2247 ASSERT(PAGE_EXCL(pp)); 2248 ASSERT(pp->p_vnode == NULL); 2249 ASSERT(!hat_page_is_mapped(pp)); 2250 PP_CLRFREE(pp); 2251 PP_CLRAGED(pp); 2252 if (!page_hashin(pp, vp, off, NULL)) 2253 panic("page_create_large: hashin failed: page %p", 2254 (void *)pp); 2255 page_io_lock(pp); 2256 off += PAGESIZE; 2257 pp = pp->p_next; 2258 } 2259 2260 VM_STAT_ADD(page_create_large_cnt[0]); 2261 return (rootpp); 2262 } 2263 2264 page_t * 2265 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2266 struct seg *seg, caddr_t vaddr) 2267 { 2268 page_t *plist = NULL; 2269 pgcnt_t npages; 2270 pgcnt_t found_on_free = 0; 2271 pgcnt_t pages_req; 2272 page_t *npp = NULL; 2273 struct pcf *p; 2274 lgrp_t *lgrp; 2275 2276 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2277 "page_create_start:vp %p off %llx bytes %lu flags %x", 2278 vp, off, bytes, flags); 2279 2280 ASSERT(bytes != 0 && vp != NULL); 2281 2282 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2283 panic("page_create: invalid flags"); 2284 /*NOTREACHED*/ 2285 } 2286 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2287 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0); 2288 /* but no others */ 2289 2290 pages_req = npages = btopr(bytes); 2291 /* 2292 * Try to see whether request is too large to *ever* be 2293 * satisfied, in order to prevent deadlock. We arbitrarily 2294 * decide to limit maximum size requests to max_page_get. 2295 */ 2296 if (npages >= max_page_get) { 2297 if ((flags & PG_WAIT) == 0) { 2298 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2299 "page_create_toobig:vp %p off %llx npages " 2300 "%lu max_page_get %lu", 2301 vp, off, npages, max_page_get); 2302 return (NULL); 2303 } else { 2304 cmn_err(CE_WARN, 2305 "Request for too much kernel memory " 2306 "(%lu bytes), will hang forever", bytes); 2307 for (;;) 2308 delay(1000000000); 2309 } 2310 } 2311 2312 if (!kcage_on || panicstr) { 2313 /* 2314 * Cage is OFF, or we are single threaded in 2315 * panic, so make everything a RELOC request. 2316 */ 2317 flags &= ~PG_NORELOC; 2318 } 2319 2320 if (freemem <= throttlefree + npages) 2321 if (!page_create_throttle(npages, flags)) 2322 return (NULL); 2323 2324 /* 2325 * If cage is on, dampen draw from cage when available 2326 * cage space is low. 2327 */ 2328 if ((flags & PG_NORELOC) && 2329 kcage_freemem < kcage_throttlefree + npages) { 2330 2331 /* 2332 * The cage is on, the caller wants PG_NORELOC 2333 * pages and available cage memory is very low. 2334 * Call kcage_create_throttle() to attempt to 2335 * control demand on the cage. 2336 */ 2337 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2338 return (NULL); 2339 } 2340 2341 VM_STAT_ADD(page_create_cnt[0]); 2342 2343 if (!pcf_decrement_bucket(npages)) { 2344 /* 2345 * Have to look harder. If npages is greater than 2346 * one, then we might have to coalesce the counters. 2347 * 2348 * Go wait. We come back having accounted 2349 * for the memory. 2350 */ 2351 VM_STAT_ADD(page_create_cnt[1]); 2352 if (!page_create_wait(npages, flags)) { 2353 VM_STAT_ADD(page_create_cnt[2]); 2354 return (NULL); 2355 } 2356 } 2357 2358 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2359 "page_create_success:vp %p off %llx", vp, off); 2360 2361 /* 2362 * If satisfying this request has left us with too little 2363 * memory, start the wheels turning to get some back. The 2364 * first clause of the test prevents waking up the pageout 2365 * daemon in situations where it would decide that there's 2366 * nothing to do. 2367 */ 2368 if (nscan < desscan && freemem < minfree) { 2369 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2370 "pageout_cv_signal:freemem %ld", freemem); 2371 cv_signal(&proc_pageout->p_cv); 2372 } 2373 2374 /* 2375 * Loop around collecting the requested number of pages. 2376 * Most of the time, we have to `create' a new page. With 2377 * this in mind, pull the page off the free list before 2378 * getting the hash lock. This will minimize the hash 2379 * lock hold time, nesting, and the like. If it turns 2380 * out we don't need the page, we put it back at the end. 2381 */ 2382 while (npages--) { 2383 page_t *pp; 2384 kmutex_t *phm = NULL; 2385 ulong_t index; 2386 2387 index = PAGE_HASH_FUNC(vp, off); 2388 top: 2389 ASSERT(phm == NULL); 2390 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2391 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2392 2393 if (npp == NULL) { 2394 /* 2395 * Try to get a page from the freelist (ie, 2396 * a page with no [vp, off] tag). If that 2397 * fails, use the cachelist. 2398 * 2399 * During the first attempt at both the free 2400 * and cache lists we try for the correct color. 2401 */ 2402 /* 2403 * XXXX-how do we deal with virtual indexed 2404 * caches and and colors? 2405 */ 2406 VM_STAT_ADD(page_create_cnt[4]); 2407 /* 2408 * Get lgroup to allocate next page of shared memory 2409 * from and use it to specify where to allocate 2410 * the physical memory 2411 */ 2412 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2413 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2414 flags | PG_MATCH_COLOR, lgrp); 2415 if (npp == NULL) { 2416 npp = page_get_cachelist(vp, off, seg, 2417 vaddr, flags | PG_MATCH_COLOR, lgrp); 2418 if (npp == NULL) { 2419 npp = page_create_get_something(vp, 2420 off, seg, vaddr, 2421 flags & ~PG_MATCH_COLOR); 2422 } 2423 2424 if (PP_ISAGED(npp) == 0) { 2425 /* 2426 * Since this page came from the 2427 * cachelist, we must destroy the 2428 * old vnode association. 2429 */ 2430 page_hashout(npp, NULL); 2431 } 2432 } 2433 } 2434 2435 /* 2436 * We own this page! 2437 */ 2438 ASSERT(PAGE_EXCL(npp)); 2439 ASSERT(npp->p_vnode == NULL); 2440 ASSERT(!hat_page_is_mapped(npp)); 2441 PP_CLRFREE(npp); 2442 PP_CLRAGED(npp); 2443 2444 /* 2445 * Here we have a page in our hot little mits and are 2446 * just waiting to stuff it on the appropriate lists. 2447 * Get the mutex and check to see if it really does 2448 * not exist. 2449 */ 2450 phm = PAGE_HASH_MUTEX(index); 2451 mutex_enter(phm); 2452 PAGE_HASH_SEARCH(index, pp, vp, off); 2453 if (pp == NULL) { 2454 VM_STAT_ADD(page_create_new); 2455 pp = npp; 2456 npp = NULL; 2457 if (!page_hashin(pp, vp, off, phm)) { 2458 /* 2459 * Since we hold the page hash mutex and 2460 * just searched for this page, page_hashin 2461 * had better not fail. If it does, that 2462 * means somethread did not follow the 2463 * page hash mutex rules. Panic now and 2464 * get it over with. As usual, go down 2465 * holding all the locks. 2466 */ 2467 ASSERT(MUTEX_HELD(phm)); 2468 panic("page_create: " 2469 "hashin failed %p %p %llx %p", 2470 (void *)pp, (void *)vp, off, (void *)phm); 2471 /*NOTREACHED*/ 2472 } 2473 ASSERT(MUTEX_HELD(phm)); 2474 mutex_exit(phm); 2475 phm = NULL; 2476 2477 /* 2478 * Hat layer locking need not be done to set 2479 * the following bits since the page is not hashed 2480 * and was on the free list (i.e., had no mappings). 2481 * 2482 * Set the reference bit to protect 2483 * against immediate pageout 2484 * 2485 * XXXmh modify freelist code to set reference 2486 * bit so we don't have to do it here. 2487 */ 2488 page_set_props(pp, P_REF); 2489 found_on_free++; 2490 } else { 2491 VM_STAT_ADD(page_create_exists); 2492 if (flags & PG_EXCL) { 2493 /* 2494 * Found an existing page, and the caller 2495 * wanted all new pages. Undo all of the work 2496 * we have done. 2497 */ 2498 mutex_exit(phm); 2499 phm = NULL; 2500 while (plist != NULL) { 2501 pp = plist; 2502 page_sub(&plist, pp); 2503 page_io_unlock(pp); 2504 /* large pages should not end up here */ 2505 ASSERT(pp->p_szc == 0); 2506 /*LINTED: constant in conditional ctx*/ 2507 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2508 } 2509 VM_STAT_ADD(page_create_found_one); 2510 goto fail; 2511 } 2512 ASSERT(flags & PG_WAIT); 2513 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2514 /* 2515 * Start all over again if we blocked trying 2516 * to lock the page. 2517 */ 2518 mutex_exit(phm); 2519 VM_STAT_ADD(page_create_page_lock_failed); 2520 phm = NULL; 2521 goto top; 2522 } 2523 mutex_exit(phm); 2524 phm = NULL; 2525 2526 if (PP_ISFREE(pp)) { 2527 ASSERT(PP_ISAGED(pp) == 0); 2528 VM_STAT_ADD(pagecnt.pc_get_cache); 2529 page_list_sub(pp, PG_CACHE_LIST); 2530 PP_CLRFREE(pp); 2531 found_on_free++; 2532 } 2533 } 2534 2535 /* 2536 * Got a page! It is locked. Acquire the i/o 2537 * lock since we are going to use the p_next and 2538 * p_prev fields to link the requested pages together. 2539 */ 2540 page_io_lock(pp); 2541 page_add(&plist, pp); 2542 plist = plist->p_next; 2543 off += PAGESIZE; 2544 vaddr += PAGESIZE; 2545 } 2546 2547 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2548 fail: 2549 if (npp != NULL) { 2550 /* 2551 * Did not need this page after all. 2552 * Put it back on the free list. 2553 */ 2554 VM_STAT_ADD(page_create_putbacks); 2555 PP_SETFREE(npp); 2556 PP_SETAGED(npp); 2557 npp->p_offset = (u_offset_t)-1; 2558 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2559 page_unlock(npp); 2560 2561 } 2562 2563 ASSERT(pages_req >= found_on_free); 2564 2565 { 2566 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2567 2568 if (overshoot) { 2569 VM_STAT_ADD(page_create_overshoot); 2570 p = &pcf[PCF_INDEX()]; 2571 mutex_enter(&p->pcf_lock); 2572 if (p->pcf_block) { 2573 p->pcf_reserve += overshoot; 2574 } else { 2575 p->pcf_count += overshoot; 2576 if (p->pcf_wait) { 2577 mutex_enter(&new_freemem_lock); 2578 if (freemem_wait) { 2579 cv_signal(&freemem_cv); 2580 p->pcf_wait--; 2581 } else { 2582 p->pcf_wait = 0; 2583 } 2584 mutex_exit(&new_freemem_lock); 2585 } 2586 } 2587 mutex_exit(&p->pcf_lock); 2588 /* freemem is approximate, so this test OK */ 2589 if (!p->pcf_block) 2590 freemem += overshoot; 2591 } 2592 } 2593 2594 return (plist); 2595 } 2596 2597 /* 2598 * One or more constituent pages of this large page has been marked 2599 * toxic. Simply demote the large page to PAGESIZE pages and let 2600 * page_free() handle it. This routine should only be called by 2601 * large page free routines (page_free_pages() and page_destroy_pages(). 2602 * All pages are locked SE_EXCL and have already been marked free. 2603 */ 2604 static void 2605 page_free_toxic_pages(page_t *rootpp) 2606 { 2607 page_t *tpp; 2608 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2609 uint_t szc = rootpp->p_szc; 2610 2611 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2612 ASSERT(tpp->p_szc == szc); 2613 ASSERT((PAGE_EXCL(tpp) && 2614 !page_iolock_assert(tpp)) || panicstr); 2615 tpp->p_szc = 0; 2616 } 2617 2618 while (rootpp != NULL) { 2619 tpp = rootpp; 2620 page_sub(&rootpp, tpp); 2621 ASSERT(PP_ISFREE(tpp)); 2622 PP_CLRFREE(tpp); 2623 page_free(tpp, 1); 2624 } 2625 } 2626 2627 /* 2628 * Put page on the "free" list. 2629 * The free list is really two lists maintained by 2630 * the PSM of whatever machine we happen to be on. 2631 */ 2632 void 2633 page_free(page_t *pp, int dontneed) 2634 { 2635 struct pcf *p; 2636 uint_t pcf_index; 2637 2638 ASSERT((PAGE_EXCL(pp) && 2639 !page_iolock_assert(pp)) || panicstr); 2640 2641 if (PP_ISFREE(pp)) { 2642 panic("page_free: page %p is free", (void *)pp); 2643 } 2644 2645 if (pp->p_szc != 0) { 2646 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2647 PP_ISKAS(pp)) { 2648 panic("page_free: anon or kernel " 2649 "or no vnode large page %p", (void *)pp); 2650 } 2651 page_demote_vp_pages(pp); 2652 ASSERT(pp->p_szc == 0); 2653 } 2654 2655 /* 2656 * The page_struct_lock need not be acquired to examine these 2657 * fields since the page has an "exclusive" lock. 2658 */ 2659 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 2660 pp->p_slckcnt != 0) { 2661 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 2662 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt, 2663 pp->p_cowcnt, pp->p_slckcnt); 2664 /*NOTREACHED*/ 2665 } 2666 2667 ASSERT(!hat_page_getshare(pp)); 2668 2669 PP_SETFREE(pp); 2670 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2671 !hat_ismod(pp)); 2672 page_clr_all_props(pp); 2673 ASSERT(!hat_page_getshare(pp)); 2674 2675 /* 2676 * Now we add the page to the head of the free list. 2677 * But if this page is associated with a paged vnode 2678 * then we adjust the head forward so that the page is 2679 * effectively at the end of the list. 2680 */ 2681 if (pp->p_vnode == NULL) { 2682 /* 2683 * Page has no identity, put it on the free list. 2684 */ 2685 PP_SETAGED(pp); 2686 pp->p_offset = (u_offset_t)-1; 2687 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2688 VM_STAT_ADD(pagecnt.pc_free_free); 2689 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2690 "page_free_free:pp %p", pp); 2691 } else { 2692 PP_CLRAGED(pp); 2693 2694 if (!dontneed || nopageage) { 2695 /* move it to the tail of the list */ 2696 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2697 2698 VM_STAT_ADD(pagecnt.pc_free_cache); 2699 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2700 "page_free_cache_tail:pp %p", pp); 2701 } else { 2702 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2703 2704 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2705 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2706 "page_free_cache_head:pp %p", pp); 2707 } 2708 } 2709 page_unlock(pp); 2710 2711 /* 2712 * Now do the `freemem' accounting. 2713 */ 2714 pcf_index = PCF_INDEX(); 2715 p = &pcf[pcf_index]; 2716 2717 mutex_enter(&p->pcf_lock); 2718 if (p->pcf_block) { 2719 p->pcf_reserve += 1; 2720 } else { 2721 p->pcf_count += 1; 2722 if (p->pcf_wait) { 2723 mutex_enter(&new_freemem_lock); 2724 /* 2725 * Check to see if some other thread 2726 * is actually waiting. Another bucket 2727 * may have woken it up by now. If there 2728 * are no waiters, then set our pcf_wait 2729 * count to zero to avoid coming in here 2730 * next time. Also, since only one page 2731 * was put on the free list, just wake 2732 * up one waiter. 2733 */ 2734 if (freemem_wait) { 2735 cv_signal(&freemem_cv); 2736 p->pcf_wait--; 2737 } else { 2738 p->pcf_wait = 0; 2739 } 2740 mutex_exit(&new_freemem_lock); 2741 } 2742 } 2743 mutex_exit(&p->pcf_lock); 2744 2745 /* freemem is approximate, so this test OK */ 2746 if (!p->pcf_block) 2747 freemem += 1; 2748 } 2749 2750 /* 2751 * Put page on the "free" list during intial startup. 2752 * This happens during initial single threaded execution. 2753 */ 2754 void 2755 page_free_at_startup(page_t *pp) 2756 { 2757 struct pcf *p; 2758 uint_t pcf_index; 2759 2760 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2761 VM_STAT_ADD(pagecnt.pc_free_free); 2762 2763 /* 2764 * Now do the `freemem' accounting. 2765 */ 2766 pcf_index = PCF_INDEX(); 2767 p = &pcf[pcf_index]; 2768 2769 ASSERT(p->pcf_block == 0); 2770 ASSERT(p->pcf_wait == 0); 2771 p->pcf_count += 1; 2772 2773 /* freemem is approximate, so this is OK */ 2774 freemem += 1; 2775 } 2776 2777 void 2778 page_free_pages(page_t *pp) 2779 { 2780 page_t *tpp, *rootpp = NULL; 2781 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2782 pgcnt_t i; 2783 uint_t szc = pp->p_szc; 2784 2785 VM_STAT_ADD(pagecnt.pc_free_pages); 2786 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2787 "page_free_free:pp %p", pp); 2788 2789 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2790 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2791 panic("page_free_pages: not root page %p", (void *)pp); 2792 /*NOTREACHED*/ 2793 } 2794 2795 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2796 ASSERT((PAGE_EXCL(tpp) && 2797 !page_iolock_assert(tpp)) || panicstr); 2798 if (PP_ISFREE(tpp)) { 2799 panic("page_free_pages: page %p is free", (void *)tpp); 2800 /*NOTREACHED*/ 2801 } 2802 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2803 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 2804 panic("page_free_pages %p", (void *)tpp); 2805 /*NOTREACHED*/ 2806 } 2807 2808 ASSERT(!hat_page_getshare(tpp)); 2809 ASSERT(tpp->p_vnode == NULL); 2810 ASSERT(tpp->p_szc == szc); 2811 2812 PP_SETFREE(tpp); 2813 page_clr_all_props(tpp); 2814 PP_SETAGED(tpp); 2815 tpp->p_offset = (u_offset_t)-1; 2816 ASSERT(tpp->p_next == tpp); 2817 ASSERT(tpp->p_prev == tpp); 2818 page_list_concat(&rootpp, &tpp); 2819 } 2820 ASSERT(rootpp == pp); 2821 2822 page_list_add_pages(rootpp, 0); 2823 page_create_putback(pgcnt); 2824 } 2825 2826 int free_pages = 1; 2827 2828 /* 2829 * This routine attempts to return pages to the cachelist via page_release(). 2830 * It does not *have* to be successful in all cases, since the pageout scanner 2831 * will catch any pages it misses. It does need to be fast and not introduce 2832 * too much overhead. 2833 * 2834 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2835 * don't lock and retry. This is ok, since the page scanner will eventually 2836 * find any page we miss in free_vp_pages(). 2837 */ 2838 void 2839 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2840 { 2841 page_t *pp; 2842 u_offset_t eoff; 2843 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2844 2845 eoff = off + len; 2846 2847 if (free_pages == 0) 2848 return; 2849 if (swap_in_range(vp, off, len)) 2850 return; 2851 2852 for (; off < eoff; off += PAGESIZE) { 2853 2854 /* 2855 * find the page using a fast, but inexact search. It'll be OK 2856 * if a few pages slip through the cracks here. 2857 */ 2858 pp = page_exists(vp, off); 2859 2860 /* 2861 * If we didn't find the page (it may not exist), the page 2862 * is free, looks still in use (shared), or we can't lock it, 2863 * just give up. 2864 */ 2865 if (pp == NULL || 2866 PP_ISFREE(pp) || 2867 page_share_cnt(pp) > 0 || 2868 !page_trylock(pp, SE_EXCL)) 2869 continue; 2870 2871 /* 2872 * Once we have locked pp, verify that it's still the 2873 * correct page and not already free 2874 */ 2875 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2876 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2877 page_unlock(pp); 2878 continue; 2879 } 2880 2881 /* 2882 * try to release the page... 2883 */ 2884 (void) page_release(pp, 1); 2885 } 2886 } 2887 2888 /* 2889 * Reclaim the given page from the free list. 2890 * If pp is part of a large pages, only the given constituent page is reclaimed 2891 * and the large page it belonged to will be demoted. This can only happen 2892 * if the page is not on the cachelist. 2893 * 2894 * Returns 1 on success or 0 on failure. 2895 * 2896 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2897 * If `lock' is non-null, it will be dropped and re-acquired if 2898 * the routine must wait while freemem is 0. 2899 * 2900 * As it turns out, boot_getpages() does this. It picks a page, 2901 * based on where OBP mapped in some address, gets its pfn, searches 2902 * the memsegs, locks the page, then pulls it off the free list! 2903 */ 2904 int 2905 page_reclaim(page_t *pp, kmutex_t *lock) 2906 { 2907 struct pcf *p; 2908 struct cpu *cpup; 2909 int enough; 2910 uint_t i; 2911 2912 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2913 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2914 2915 /* 2916 * If `freemem' is 0, we cannot reclaim this page from the 2917 * freelist, so release every lock we might hold: the page, 2918 * and the `lock' before blocking. 2919 * 2920 * The only way `freemem' can become 0 while there are pages 2921 * marked free (have their p->p_free bit set) is when the 2922 * system is low on memory and doing a page_create(). In 2923 * order to guarantee that once page_create() starts acquiring 2924 * pages it will be able to get all that it needs since `freemem' 2925 * was decreased by the requested amount. So, we need to release 2926 * this page, and let page_create() have it. 2927 * 2928 * Since `freemem' being zero is not supposed to happen, just 2929 * use the usual hash stuff as a starting point. If that bucket 2930 * is empty, then assume the worst, and start at the beginning 2931 * of the pcf array. If we always start at the beginning 2932 * when acquiring more than one pcf lock, there won't be any 2933 * deadlock problems. 2934 */ 2935 2936 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 2937 2938 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) { 2939 pcf_acquire_all(); 2940 goto page_reclaim_nomem; 2941 } 2942 2943 enough = pcf_decrement_bucket(1); 2944 2945 if (!enough) { 2946 VM_STAT_ADD(page_reclaim_zero); 2947 /* 2948 * Check again. Its possible that some other thread 2949 * could have been right behind us, and added one 2950 * to a list somewhere. Acquire each of the pcf locks 2951 * until we find a page. 2952 */ 2953 p = pcf; 2954 for (i = 0; i < pcf_fanout; i++) { 2955 mutex_enter(&p->pcf_lock); 2956 if (p->pcf_count >= 1) { 2957 p->pcf_count -= 1; 2958 /* 2959 * freemem is not protected by any lock. Thus, 2960 * we cannot have any assertion containing 2961 * freemem here. 2962 */ 2963 freemem -= 1; 2964 enough = 1; 2965 break; 2966 } 2967 p++; 2968 } 2969 2970 if (!enough) { 2971 page_reclaim_nomem: 2972 /* 2973 * We really can't have page `pp'. 2974 * Time for the no-memory dance with 2975 * page_free(). This is just like 2976 * page_create_wait(). Plus the added 2977 * attraction of releasing whatever mutex 2978 * we held when we were called with in `lock'. 2979 * Page_unlock() will wakeup any thread 2980 * waiting around for this page. 2981 */ 2982 if (lock) { 2983 VM_STAT_ADD(page_reclaim_zero_locked); 2984 mutex_exit(lock); 2985 } 2986 page_unlock(pp); 2987 2988 /* 2989 * get this before we drop all the pcf locks. 2990 */ 2991 mutex_enter(&new_freemem_lock); 2992 2993 p = pcf; 2994 for (i = 0; i < pcf_fanout; i++) { 2995 p->pcf_wait++; 2996 mutex_exit(&p->pcf_lock); 2997 p++; 2998 } 2999 3000 freemem_wait++; 3001 cv_wait(&freemem_cv, &new_freemem_lock); 3002 freemem_wait--; 3003 3004 mutex_exit(&new_freemem_lock); 3005 3006 if (lock) { 3007 mutex_enter(lock); 3008 } 3009 return (0); 3010 } 3011 3012 /* 3013 * The pcf accounting has been done, 3014 * though none of the pcf_wait flags have been set, 3015 * drop the locks and continue on. 3016 */ 3017 while (p >= pcf) { 3018 mutex_exit(&p->pcf_lock); 3019 p--; 3020 } 3021 } 3022 3023 3024 VM_STAT_ADD(pagecnt.pc_reclaim); 3025 3026 /* 3027 * page_list_sub will handle the case where pp is a large page. 3028 * It's possible that the page was promoted while on the freelist 3029 */ 3030 if (PP_ISAGED(pp)) { 3031 page_list_sub(pp, PG_FREE_LIST); 3032 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3033 "page_reclaim_free:pp %p", pp); 3034 } else { 3035 page_list_sub(pp, PG_CACHE_LIST); 3036 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3037 "page_reclaim_cache:pp %p", pp); 3038 } 3039 3040 /* 3041 * clear the p_free & p_age bits since this page is no longer 3042 * on the free list. Notice that there was a brief time where 3043 * a page is marked as free, but is not on the list. 3044 * 3045 * Set the reference bit to protect against immediate pageout. 3046 */ 3047 PP_CLRFREE(pp); 3048 PP_CLRAGED(pp); 3049 page_set_props(pp, P_REF); 3050 3051 CPU_STATS_ENTER_K(); 3052 cpup = CPU; /* get cpup now that CPU cannot change */ 3053 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3054 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3055 CPU_STATS_EXIT_K(); 3056 ASSERT(pp->p_szc == 0); 3057 3058 return (1); 3059 } 3060 3061 /* 3062 * Destroy identity of the page and put it back on 3063 * the page free list. Assumes that the caller has 3064 * acquired the "exclusive" lock on the page. 3065 */ 3066 void 3067 page_destroy(page_t *pp, int dontfree) 3068 { 3069 ASSERT((PAGE_EXCL(pp) && 3070 !page_iolock_assert(pp)) || panicstr); 3071 ASSERT(pp->p_slckcnt == 0 || panicstr); 3072 3073 if (pp->p_szc != 0) { 3074 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3075 PP_ISKAS(pp)) { 3076 panic("page_destroy: anon or kernel or no vnode " 3077 "large page %p", (void *)pp); 3078 } 3079 page_demote_vp_pages(pp); 3080 ASSERT(pp->p_szc == 0); 3081 } 3082 3083 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3084 3085 /* 3086 * Unload translations, if any, then hash out the 3087 * page to erase its identity. 3088 */ 3089 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3090 page_hashout(pp, NULL); 3091 3092 if (!dontfree) { 3093 /* 3094 * Acquire the "freemem_lock" for availrmem. 3095 * The page_struct_lock need not be acquired for lckcnt 3096 * and cowcnt since the page has an "exclusive" lock. 3097 * We are doing a modified version of page_pp_unlock here. 3098 */ 3099 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3100 mutex_enter(&freemem_lock); 3101 if (pp->p_lckcnt != 0) { 3102 availrmem++; 3103 pages_locked--; 3104 pp->p_lckcnt = 0; 3105 } 3106 if (pp->p_cowcnt != 0) { 3107 availrmem += pp->p_cowcnt; 3108 pages_locked -= pp->p_cowcnt; 3109 pp->p_cowcnt = 0; 3110 } 3111 mutex_exit(&freemem_lock); 3112 } 3113 /* 3114 * Put the page on the "free" list. 3115 */ 3116 page_free(pp, 0); 3117 } 3118 } 3119 3120 void 3121 page_destroy_pages(page_t *pp) 3122 { 3123 3124 page_t *tpp, *rootpp = NULL; 3125 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3126 pgcnt_t i, pglcks = 0; 3127 uint_t szc = pp->p_szc; 3128 3129 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3130 3131 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3132 3133 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3134 3135 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3136 panic("page_destroy_pages: not root page %p", (void *)pp); 3137 /*NOTREACHED*/ 3138 } 3139 3140 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3141 ASSERT((PAGE_EXCL(tpp) && 3142 !page_iolock_assert(tpp)) || panicstr); 3143 ASSERT(tpp->p_slckcnt == 0 || panicstr); 3144 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3145 page_hashout(tpp, NULL); 3146 ASSERT(tpp->p_offset == (u_offset_t)-1); 3147 if (tpp->p_lckcnt != 0) { 3148 pglcks++; 3149 tpp->p_lckcnt = 0; 3150 } else if (tpp->p_cowcnt != 0) { 3151 pglcks += tpp->p_cowcnt; 3152 tpp->p_cowcnt = 0; 3153 } 3154 ASSERT(!hat_page_getshare(tpp)); 3155 ASSERT(tpp->p_vnode == NULL); 3156 ASSERT(tpp->p_szc == szc); 3157 3158 PP_SETFREE(tpp); 3159 page_clr_all_props(tpp); 3160 PP_SETAGED(tpp); 3161 ASSERT(tpp->p_next == tpp); 3162 ASSERT(tpp->p_prev == tpp); 3163 page_list_concat(&rootpp, &tpp); 3164 } 3165 3166 ASSERT(rootpp == pp); 3167 if (pglcks != 0) { 3168 mutex_enter(&freemem_lock); 3169 availrmem += pglcks; 3170 mutex_exit(&freemem_lock); 3171 } 3172 3173 page_list_add_pages(rootpp, 0); 3174 page_create_putback(pgcnt); 3175 } 3176 3177 /* 3178 * Similar to page_destroy(), but destroys pages which are 3179 * locked and known to be on the page free list. Since 3180 * the page is known to be free and locked, no one can access 3181 * it. 3182 * 3183 * Also, the number of free pages does not change. 3184 */ 3185 void 3186 page_destroy_free(page_t *pp) 3187 { 3188 ASSERT(PAGE_EXCL(pp)); 3189 ASSERT(PP_ISFREE(pp)); 3190 ASSERT(pp->p_vnode); 3191 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3192 ASSERT(!hat_page_is_mapped(pp)); 3193 ASSERT(PP_ISAGED(pp) == 0); 3194 ASSERT(pp->p_szc == 0); 3195 3196 VM_STAT_ADD(pagecnt.pc_destroy_free); 3197 page_list_sub(pp, PG_CACHE_LIST); 3198 3199 page_hashout(pp, NULL); 3200 ASSERT(pp->p_vnode == NULL); 3201 ASSERT(pp->p_offset == (u_offset_t)-1); 3202 ASSERT(pp->p_hash == NULL); 3203 3204 PP_SETAGED(pp); 3205 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3206 page_unlock(pp); 3207 3208 mutex_enter(&new_freemem_lock); 3209 if (freemem_wait) { 3210 cv_signal(&freemem_cv); 3211 } 3212 mutex_exit(&new_freemem_lock); 3213 } 3214 3215 /* 3216 * Rename the page "opp" to have an identity specified 3217 * by [vp, off]. If a page already exists with this name 3218 * it is locked and destroyed. Note that the page's 3219 * translations are not unloaded during the rename. 3220 * 3221 * This routine is used by the anon layer to "steal" the 3222 * original page and is not unlike destroying a page and 3223 * creating a new page using the same page frame. 3224 * 3225 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3226 * caller 2 tries to rename B to A. 3227 */ 3228 void 3229 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3230 { 3231 page_t *pp; 3232 int olckcnt = 0; 3233 int ocowcnt = 0; 3234 kmutex_t *phm; 3235 ulong_t index; 3236 3237 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3238 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3239 ASSERT(PP_ISFREE(opp) == 0); 3240 3241 VM_STAT_ADD(page_rename_count); 3242 3243 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3244 "page rename:pp %p vp %p off %llx", opp, vp, off); 3245 3246 /* 3247 * CacheFS may call page_rename for a large NFS page 3248 * when both CacheFS and NFS mount points are used 3249 * by applications. Demote this large page before 3250 * renaming it, to ensure that there are no "partial" 3251 * large pages left lying around. 3252 */ 3253 if (opp->p_szc != 0) { 3254 vnode_t *ovp = opp->p_vnode; 3255 ASSERT(ovp != NULL); 3256 ASSERT(!IS_SWAPFSVP(ovp)); 3257 ASSERT(!VN_ISKAS(ovp)); 3258 page_demote_vp_pages(opp); 3259 ASSERT(opp->p_szc == 0); 3260 } 3261 3262 page_hashout(opp, NULL); 3263 PP_CLRAGED(opp); 3264 3265 /* 3266 * Acquire the appropriate page hash lock, since 3267 * we're going to rename the page. 3268 */ 3269 index = PAGE_HASH_FUNC(vp, off); 3270 phm = PAGE_HASH_MUTEX(index); 3271 mutex_enter(phm); 3272 top: 3273 /* 3274 * Look for an existing page with this name and destroy it if found. 3275 * By holding the page hash lock all the way to the page_hashin() 3276 * call, we are assured that no page can be created with this 3277 * identity. In the case when the phm lock is dropped to undo any 3278 * hat layer mappings, the existing page is held with an "exclusive" 3279 * lock, again preventing another page from being created with 3280 * this identity. 3281 */ 3282 PAGE_HASH_SEARCH(index, pp, vp, off); 3283 if (pp != NULL) { 3284 VM_STAT_ADD(page_rename_exists); 3285 3286 /* 3287 * As it turns out, this is one of only two places where 3288 * page_lock() needs to hold the passed in lock in the 3289 * successful case. In all of the others, the lock could 3290 * be dropped as soon as the attempt is made to lock 3291 * the page. It is tempting to add yet another arguement, 3292 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3293 */ 3294 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3295 /* 3296 * Went to sleep because the page could not 3297 * be locked. We were woken up when the page 3298 * was unlocked, or when the page was destroyed. 3299 * In either case, `phm' was dropped while we 3300 * slept. Hence we should not just roar through 3301 * this loop. 3302 */ 3303 goto top; 3304 } 3305 3306 /* 3307 * If an existing page is a large page, then demote 3308 * it to ensure that no "partial" large pages are 3309 * "created" after page_rename. An existing page 3310 * can be a CacheFS page, and can't belong to swapfs. 3311 */ 3312 if (hat_page_is_mapped(pp)) { 3313 /* 3314 * Unload translations. Since we hold the 3315 * exclusive lock on this page, the page 3316 * can not be changed while we drop phm. 3317 * This is also not a lock protocol violation, 3318 * but rather the proper way to do things. 3319 */ 3320 mutex_exit(phm); 3321 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3322 if (pp->p_szc != 0) { 3323 ASSERT(!IS_SWAPFSVP(vp)); 3324 ASSERT(!VN_ISKAS(vp)); 3325 page_demote_vp_pages(pp); 3326 ASSERT(pp->p_szc == 0); 3327 } 3328 mutex_enter(phm); 3329 } else if (pp->p_szc != 0) { 3330 ASSERT(!IS_SWAPFSVP(vp)); 3331 ASSERT(!VN_ISKAS(vp)); 3332 mutex_exit(phm); 3333 page_demote_vp_pages(pp); 3334 ASSERT(pp->p_szc == 0); 3335 mutex_enter(phm); 3336 } 3337 page_hashout(pp, phm); 3338 } 3339 /* 3340 * Hash in the page with the new identity. 3341 */ 3342 if (!page_hashin(opp, vp, off, phm)) { 3343 /* 3344 * We were holding phm while we searched for [vp, off] 3345 * and only dropped phm if we found and locked a page. 3346 * If we can't create this page now, then some thing 3347 * is really broken. 3348 */ 3349 panic("page_rename: Can't hash in page: %p", (void *)pp); 3350 /*NOTREACHED*/ 3351 } 3352 3353 ASSERT(MUTEX_HELD(phm)); 3354 mutex_exit(phm); 3355 3356 /* 3357 * Now that we have dropped phm, lets get around to finishing up 3358 * with pp. 3359 */ 3360 if (pp != NULL) { 3361 ASSERT(!hat_page_is_mapped(pp)); 3362 /* for now large pages should not end up here */ 3363 ASSERT(pp->p_szc == 0); 3364 /* 3365 * Save the locks for transfer to the new page and then 3366 * clear them so page_free doesn't think they're important. 3367 * The page_struct_lock need not be acquired for lckcnt and 3368 * cowcnt since the page has an "exclusive" lock. 3369 */ 3370 olckcnt = pp->p_lckcnt; 3371 ocowcnt = pp->p_cowcnt; 3372 pp->p_lckcnt = pp->p_cowcnt = 0; 3373 3374 /* 3375 * Put the page on the "free" list after we drop 3376 * the lock. The less work under the lock the better. 3377 */ 3378 /*LINTED: constant in conditional context*/ 3379 VN_DISPOSE(pp, B_FREE, 0, kcred); 3380 } 3381 3382 /* 3383 * Transfer the lock count from the old page (if any). 3384 * The page_struct_lock need not be acquired for lckcnt and 3385 * cowcnt since the page has an "exclusive" lock. 3386 */ 3387 opp->p_lckcnt += olckcnt; 3388 opp->p_cowcnt += ocowcnt; 3389 } 3390 3391 /* 3392 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3393 * 3394 * Pages are normally inserted at the start of a vnode's v_pages list. 3395 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3396 * This can happen when a modified page is relocated for DR. 3397 * 3398 * Returns 1 on success and 0 on failure. 3399 */ 3400 static int 3401 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3402 { 3403 page_t **listp; 3404 page_t *tp; 3405 ulong_t index; 3406 3407 ASSERT(PAGE_EXCL(pp)); 3408 ASSERT(vp != NULL); 3409 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3410 3411 /* 3412 * Be sure to set these up before the page is inserted on the hash 3413 * list. As soon as the page is placed on the list some other 3414 * thread might get confused and wonder how this page could 3415 * possibly hash to this list. 3416 */ 3417 pp->p_vnode = vp; 3418 pp->p_offset = offset; 3419 3420 /* 3421 * record if this page is on a swap vnode 3422 */ 3423 if ((vp->v_flag & VISSWAP) != 0) 3424 PP_SETSWAP(pp); 3425 3426 index = PAGE_HASH_FUNC(vp, offset); 3427 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3428 listp = &page_hash[index]; 3429 3430 /* 3431 * If this page is already hashed in, fail this attempt to add it. 3432 */ 3433 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3434 if (tp->p_vnode == vp && tp->p_offset == offset) { 3435 pp->p_vnode = NULL; 3436 pp->p_offset = (u_offset_t)(-1); 3437 return (0); 3438 } 3439 } 3440 pp->p_hash = *listp; 3441 *listp = pp; 3442 3443 /* 3444 * Add the page to the vnode's list of pages 3445 */ 3446 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3447 listp = &vp->v_pages->p_vpprev->p_vpnext; 3448 else 3449 listp = &vp->v_pages; 3450 3451 page_vpadd(listp, pp); 3452 3453 return (1); 3454 } 3455 3456 /* 3457 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3458 * 3459 * Returns 1 on success and 0 on failure. 3460 * If hold is passed in, it is not dropped. 3461 */ 3462 int 3463 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3464 { 3465 kmutex_t *phm = NULL; 3466 kmutex_t *vphm; 3467 int rc; 3468 3469 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3470 ASSERT(pp->p_fsdata == 0 || panicstr); 3471 3472 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3473 "page_hashin:pp %p vp %p offset %llx", 3474 pp, vp, offset); 3475 3476 VM_STAT_ADD(hashin_count); 3477 3478 if (hold != NULL) 3479 phm = hold; 3480 else { 3481 VM_STAT_ADD(hashin_not_held); 3482 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3483 mutex_enter(phm); 3484 } 3485 3486 vphm = page_vnode_mutex(vp); 3487 mutex_enter(vphm); 3488 rc = page_do_hashin(pp, vp, offset); 3489 mutex_exit(vphm); 3490 if (hold == NULL) 3491 mutex_exit(phm); 3492 if (rc == 0) 3493 VM_STAT_ADD(hashin_already); 3494 return (rc); 3495 } 3496 3497 /* 3498 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3499 * All mutexes must be held 3500 */ 3501 static void 3502 page_do_hashout(page_t *pp) 3503 { 3504 page_t **hpp; 3505 page_t *hp; 3506 vnode_t *vp = pp->p_vnode; 3507 3508 ASSERT(vp != NULL); 3509 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3510 3511 /* 3512 * First, take pp off of its hash chain. 3513 */ 3514 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3515 3516 for (;;) { 3517 hp = *hpp; 3518 if (hp == pp) 3519 break; 3520 if (hp == NULL) { 3521 panic("page_do_hashout"); 3522 /*NOTREACHED*/ 3523 } 3524 hpp = &hp->p_hash; 3525 } 3526 *hpp = pp->p_hash; 3527 3528 /* 3529 * Now remove it from its associated vnode. 3530 */ 3531 if (vp->v_pages) 3532 page_vpsub(&vp->v_pages, pp); 3533 3534 pp->p_hash = NULL; 3535 page_clr_all_props(pp); 3536 PP_CLRSWAP(pp); 3537 pp->p_vnode = NULL; 3538 pp->p_offset = (u_offset_t)-1; 3539 pp->p_fsdata = 0; 3540 } 3541 3542 /* 3543 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3544 * 3545 * When `phm' is non-NULL it contains the address of the mutex protecting the 3546 * hash list pp is on. It is not dropped. 3547 */ 3548 void 3549 page_hashout(page_t *pp, kmutex_t *phm) 3550 { 3551 vnode_t *vp; 3552 ulong_t index; 3553 kmutex_t *nphm; 3554 kmutex_t *vphm; 3555 kmutex_t *sep; 3556 3557 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3558 ASSERT(pp->p_vnode != NULL); 3559 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3560 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3561 3562 vp = pp->p_vnode; 3563 3564 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3565 "page_hashout:pp %p vp %p", pp, vp); 3566 3567 /* Kernel probe */ 3568 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3569 tnf_opaque, vnode, vp, 3570 tnf_offset, offset, pp->p_offset); 3571 3572 /* 3573 * 3574 */ 3575 VM_STAT_ADD(hashout_count); 3576 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3577 if (phm == NULL) { 3578 VM_STAT_ADD(hashout_not_held); 3579 nphm = PAGE_HASH_MUTEX(index); 3580 mutex_enter(nphm); 3581 } 3582 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3583 3584 3585 /* 3586 * grab page vnode mutex and remove it... 3587 */ 3588 vphm = page_vnode_mutex(vp); 3589 mutex_enter(vphm); 3590 3591 page_do_hashout(pp); 3592 3593 mutex_exit(vphm); 3594 if (phm == NULL) 3595 mutex_exit(nphm); 3596 3597 /* 3598 * Wake up processes waiting for this page. The page's 3599 * identity has been changed, and is probably not the 3600 * desired page any longer. 3601 */ 3602 sep = page_se_mutex(pp); 3603 mutex_enter(sep); 3604 pp->p_selock &= ~SE_EWANTED; 3605 if (CV_HAS_WAITERS(&pp->p_cv)) 3606 cv_broadcast(&pp->p_cv); 3607 mutex_exit(sep); 3608 } 3609 3610 /* 3611 * Add the page to the front of a linked list of pages 3612 * using the p_next & p_prev pointers for the list. 3613 * The caller is responsible for protecting the list pointers. 3614 */ 3615 void 3616 page_add(page_t **ppp, page_t *pp) 3617 { 3618 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3619 3620 page_add_common(ppp, pp); 3621 } 3622 3623 3624 3625 /* 3626 * Common code for page_add() and mach_page_add() 3627 */ 3628 void 3629 page_add_common(page_t **ppp, page_t *pp) 3630 { 3631 if (*ppp == NULL) { 3632 pp->p_next = pp->p_prev = pp; 3633 } else { 3634 pp->p_next = *ppp; 3635 pp->p_prev = (*ppp)->p_prev; 3636 (*ppp)->p_prev = pp; 3637 pp->p_prev->p_next = pp; 3638 } 3639 *ppp = pp; 3640 } 3641 3642 3643 /* 3644 * Remove this page from a linked list of pages 3645 * using the p_next & p_prev pointers for the list. 3646 * 3647 * The caller is responsible for protecting the list pointers. 3648 */ 3649 void 3650 page_sub(page_t **ppp, page_t *pp) 3651 { 3652 ASSERT((PP_ISFREE(pp)) ? 1 : 3653 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3654 3655 if (*ppp == NULL || pp == NULL) { 3656 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3657 (void *)pp, (void *)(*ppp)); 3658 /*NOTREACHED*/ 3659 } 3660 3661 page_sub_common(ppp, pp); 3662 } 3663 3664 3665 /* 3666 * Common code for page_sub() and mach_page_sub() 3667 */ 3668 void 3669 page_sub_common(page_t **ppp, page_t *pp) 3670 { 3671 if (*ppp == pp) 3672 *ppp = pp->p_next; /* go to next page */ 3673 3674 if (*ppp == pp) 3675 *ppp = NULL; /* page list is gone */ 3676 else { 3677 pp->p_prev->p_next = pp->p_next; 3678 pp->p_next->p_prev = pp->p_prev; 3679 } 3680 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3681 } 3682 3683 3684 /* 3685 * Break page list cppp into two lists with npages in the first list. 3686 * The tail is returned in nppp. 3687 */ 3688 void 3689 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3690 { 3691 page_t *s1pp = *oppp; 3692 page_t *s2pp; 3693 page_t *e1pp, *e2pp; 3694 long n = 0; 3695 3696 if (s1pp == NULL) { 3697 *nppp = NULL; 3698 return; 3699 } 3700 if (npages == 0) { 3701 *nppp = s1pp; 3702 *oppp = NULL; 3703 return; 3704 } 3705 for (n = 0, s2pp = *oppp; n < npages; n++) { 3706 s2pp = s2pp->p_next; 3707 } 3708 /* Fix head and tail of new lists */ 3709 e1pp = s2pp->p_prev; 3710 e2pp = s1pp->p_prev; 3711 s1pp->p_prev = e1pp; 3712 e1pp->p_next = s1pp; 3713 s2pp->p_prev = e2pp; 3714 e2pp->p_next = s2pp; 3715 3716 /* second list empty */ 3717 if (s2pp == s1pp) { 3718 *oppp = s1pp; 3719 *nppp = NULL; 3720 } else { 3721 *oppp = s1pp; 3722 *nppp = s2pp; 3723 } 3724 } 3725 3726 /* 3727 * Concatenate page list nppp onto the end of list ppp. 3728 */ 3729 void 3730 page_list_concat(page_t **ppp, page_t **nppp) 3731 { 3732 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3733 3734 if (*nppp == NULL) { 3735 return; 3736 } 3737 if (*ppp == NULL) { 3738 *ppp = *nppp; 3739 return; 3740 } 3741 s1pp = *ppp; 3742 e1pp = s1pp->p_prev; 3743 s2pp = *nppp; 3744 e2pp = s2pp->p_prev; 3745 s1pp->p_prev = e2pp; 3746 e2pp->p_next = s1pp; 3747 e1pp->p_next = s2pp; 3748 s2pp->p_prev = e1pp; 3749 } 3750 3751 /* 3752 * return the next page in the page list 3753 */ 3754 page_t * 3755 page_list_next(page_t *pp) 3756 { 3757 return (pp->p_next); 3758 } 3759 3760 3761 /* 3762 * Add the page to the front of the linked list of pages 3763 * using p_vpnext/p_vpprev pointers for the list. 3764 * 3765 * The caller is responsible for protecting the lists. 3766 */ 3767 void 3768 page_vpadd(page_t **ppp, page_t *pp) 3769 { 3770 if (*ppp == NULL) { 3771 pp->p_vpnext = pp->p_vpprev = pp; 3772 } else { 3773 pp->p_vpnext = *ppp; 3774 pp->p_vpprev = (*ppp)->p_vpprev; 3775 (*ppp)->p_vpprev = pp; 3776 pp->p_vpprev->p_vpnext = pp; 3777 } 3778 *ppp = pp; 3779 } 3780 3781 /* 3782 * Remove this page from the linked list of pages 3783 * using p_vpnext/p_vpprev pointers for the list. 3784 * 3785 * The caller is responsible for protecting the lists. 3786 */ 3787 void 3788 page_vpsub(page_t **ppp, page_t *pp) 3789 { 3790 if (*ppp == NULL || pp == NULL) { 3791 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3792 (void *)pp, (void *)(*ppp)); 3793 /*NOTREACHED*/ 3794 } 3795 3796 if (*ppp == pp) 3797 *ppp = pp->p_vpnext; /* go to next page */ 3798 3799 if (*ppp == pp) 3800 *ppp = NULL; /* page list is gone */ 3801 else { 3802 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3803 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3804 } 3805 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3806 } 3807 3808 /* 3809 * Lock a physical page into memory "long term". Used to support "lock 3810 * in memory" functions. Accepts the page to be locked, and a cow variable 3811 * to indicate whether a the lock will travel to the new page during 3812 * a potential copy-on-write. 3813 */ 3814 int 3815 page_pp_lock( 3816 page_t *pp, /* page to be locked */ 3817 int cow, /* cow lock */ 3818 int kernel) /* must succeed -- ignore checking */ 3819 { 3820 int r = 0; /* result -- assume failure */ 3821 3822 ASSERT(PAGE_LOCKED(pp)); 3823 3824 page_struct_lock(pp); 3825 /* 3826 * Acquire the "freemem_lock" for availrmem. 3827 */ 3828 if (cow) { 3829 mutex_enter(&freemem_lock); 3830 if ((availrmem > pages_pp_maximum) && 3831 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3832 availrmem--; 3833 pages_locked++; 3834 mutex_exit(&freemem_lock); 3835 r = 1; 3836 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3837 cmn_err(CE_WARN, 3838 "COW lock limit reached on pfn 0x%lx", 3839 page_pptonum(pp)); 3840 } 3841 } else 3842 mutex_exit(&freemem_lock); 3843 } else { 3844 if (pp->p_lckcnt) { 3845 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3846 r = 1; 3847 if (++pp->p_lckcnt == 3848 (ushort_t)PAGE_LOCK_MAXIMUM) { 3849 cmn_err(CE_WARN, "Page lock limit " 3850 "reached on pfn 0x%lx", 3851 page_pptonum(pp)); 3852 } 3853 } 3854 } else { 3855 if (kernel) { 3856 /* availrmem accounting done by caller */ 3857 ++pp->p_lckcnt; 3858 r = 1; 3859 } else { 3860 mutex_enter(&freemem_lock); 3861 if (availrmem > pages_pp_maximum) { 3862 availrmem--; 3863 pages_locked++; 3864 ++pp->p_lckcnt; 3865 r = 1; 3866 } 3867 mutex_exit(&freemem_lock); 3868 } 3869 } 3870 } 3871 page_struct_unlock(pp); 3872 return (r); 3873 } 3874 3875 /* 3876 * Decommit a lock on a physical page frame. Account for cow locks if 3877 * appropriate. 3878 */ 3879 void 3880 page_pp_unlock( 3881 page_t *pp, /* page to be unlocked */ 3882 int cow, /* expect cow lock */ 3883 int kernel) /* this was a kernel lock */ 3884 { 3885 ASSERT(PAGE_LOCKED(pp)); 3886 3887 page_struct_lock(pp); 3888 /* 3889 * Acquire the "freemem_lock" for availrmem. 3890 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3891 * could be called to unlock even if nothing is locked. This could 3892 * happen if locked file pages were truncated (removing the lock) 3893 * and the file was grown again and new pages faulted in; the new 3894 * pages are unlocked but the segment still thinks they're locked. 3895 */ 3896 if (cow) { 3897 if (pp->p_cowcnt) { 3898 mutex_enter(&freemem_lock); 3899 pp->p_cowcnt--; 3900 availrmem++; 3901 pages_locked--; 3902 mutex_exit(&freemem_lock); 3903 } 3904 } else { 3905 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3906 if (!kernel) { 3907 mutex_enter(&freemem_lock); 3908 availrmem++; 3909 pages_locked--; 3910 mutex_exit(&freemem_lock); 3911 } 3912 } 3913 } 3914 page_struct_unlock(pp); 3915 } 3916 3917 /* 3918 * This routine reserves availrmem for npages; 3919 * flags: KM_NOSLEEP or KM_SLEEP 3920 * returns 1 on success or 0 on failure 3921 */ 3922 int 3923 page_resv(pgcnt_t npages, uint_t flags) 3924 { 3925 mutex_enter(&freemem_lock); 3926 while (availrmem < tune.t_minarmem + npages) { 3927 if (flags & KM_NOSLEEP) { 3928 mutex_exit(&freemem_lock); 3929 return (0); 3930 } 3931 mutex_exit(&freemem_lock); 3932 page_needfree(npages); 3933 kmem_reap(); 3934 delay(hz >> 2); 3935 page_needfree(-(spgcnt_t)npages); 3936 mutex_enter(&freemem_lock); 3937 } 3938 availrmem -= npages; 3939 mutex_exit(&freemem_lock); 3940 return (1); 3941 } 3942 3943 /* 3944 * This routine unreserves availrmem for npages; 3945 */ 3946 void 3947 page_unresv(pgcnt_t npages) 3948 { 3949 mutex_enter(&freemem_lock); 3950 availrmem += npages; 3951 mutex_exit(&freemem_lock); 3952 } 3953 3954 /* 3955 * See Statement at the beginning of segvn_lockop() regarding 3956 * the way we handle cowcnts and lckcnts. 3957 * 3958 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 3959 * that breaks COW has PROT_WRITE. 3960 * 3961 * Note that, we may also break COW in case we are softlocking 3962 * on read access during physio; 3963 * in this softlock case, the vpage may not have PROT_WRITE. 3964 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 3965 * if the vpage doesn't have PROT_WRITE. 3966 * 3967 * This routine is never called if we are stealing a page 3968 * in anon_private. 3969 * 3970 * The caller subtracted from availrmem for read only mapping. 3971 * if lckcnt is 1 increment availrmem. 3972 */ 3973 void 3974 page_pp_useclaim( 3975 page_t *opp, /* original page frame losing lock */ 3976 page_t *npp, /* new page frame gaining lock */ 3977 uint_t write_perm) /* set if vpage has PROT_WRITE */ 3978 { 3979 int payback = 0; 3980 int nidx, oidx; 3981 3982 ASSERT(PAGE_LOCKED(opp)); 3983 ASSERT(PAGE_LOCKED(npp)); 3984 3985 /* 3986 * Since we have two pages we probably have two locks. We need to take 3987 * them in a defined order to avoid deadlocks. It's also possible they 3988 * both hash to the same lock in which case this is a non-issue. 3989 */ 3990 nidx = PAGE_LLOCK_HASH(PP_PAGEROOT(npp)); 3991 oidx = PAGE_LLOCK_HASH(PP_PAGEROOT(opp)); 3992 if (nidx < oidx) { 3993 page_struct_lock(npp); 3994 page_struct_lock(opp); 3995 } else if (oidx < nidx) { 3996 page_struct_lock(opp); 3997 page_struct_lock(npp); 3998 } else { /* The pages hash to the same lock */ 3999 page_struct_lock(npp); 4000 } 4001 4002 ASSERT(npp->p_cowcnt == 0); 4003 ASSERT(npp->p_lckcnt == 0); 4004 4005 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4006 if ((write_perm && opp->p_cowcnt != 0) || 4007 (!write_perm && opp->p_lckcnt != 0)) { 4008 4009 if (write_perm) { 4010 npp->p_cowcnt++; 4011 ASSERT(opp->p_cowcnt != 0); 4012 opp->p_cowcnt--; 4013 } else { 4014 4015 ASSERT(opp->p_lckcnt != 0); 4016 4017 /* 4018 * We didn't need availrmem decremented if p_lckcnt on 4019 * original page is 1. Here, we are unlocking 4020 * read-only copy belonging to original page and 4021 * are locking a copy belonging to new page. 4022 */ 4023 if (opp->p_lckcnt == 1) 4024 payback = 1; 4025 4026 npp->p_lckcnt++; 4027 opp->p_lckcnt--; 4028 } 4029 } 4030 if (payback) { 4031 mutex_enter(&freemem_lock); 4032 availrmem++; 4033 pages_useclaim--; 4034 mutex_exit(&freemem_lock); 4035 } 4036 4037 if (nidx < oidx) { 4038 page_struct_unlock(opp); 4039 page_struct_unlock(npp); 4040 } else if (oidx < nidx) { 4041 page_struct_unlock(npp); 4042 page_struct_unlock(opp); 4043 } else { /* The pages hash to the same lock */ 4044 page_struct_unlock(npp); 4045 } 4046 } 4047 4048 /* 4049 * Simple claim adjust functions -- used to support changes in 4050 * claims due to changes in access permissions. Used by segvn_setprot(). 4051 */ 4052 int 4053 page_addclaim(page_t *pp) 4054 { 4055 int r = 0; /* result */ 4056 4057 ASSERT(PAGE_LOCKED(pp)); 4058 4059 page_struct_lock(pp); 4060 ASSERT(pp->p_lckcnt != 0); 4061 4062 if (pp->p_lckcnt == 1) { 4063 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4064 --pp->p_lckcnt; 4065 r = 1; 4066 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4067 cmn_err(CE_WARN, 4068 "COW lock limit reached on pfn 0x%lx", 4069 page_pptonum(pp)); 4070 } 4071 } 4072 } else { 4073 mutex_enter(&freemem_lock); 4074 if ((availrmem > pages_pp_maximum) && 4075 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4076 --availrmem; 4077 ++pages_claimed; 4078 mutex_exit(&freemem_lock); 4079 --pp->p_lckcnt; 4080 r = 1; 4081 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4082 cmn_err(CE_WARN, 4083 "COW lock limit reached on pfn 0x%lx", 4084 page_pptonum(pp)); 4085 } 4086 } else 4087 mutex_exit(&freemem_lock); 4088 } 4089 page_struct_unlock(pp); 4090 return (r); 4091 } 4092 4093 int 4094 page_subclaim(page_t *pp) 4095 { 4096 int r = 0; 4097 4098 ASSERT(PAGE_LOCKED(pp)); 4099 4100 page_struct_lock(pp); 4101 ASSERT(pp->p_cowcnt != 0); 4102 4103 if (pp->p_lckcnt) { 4104 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4105 r = 1; 4106 /* 4107 * for availrmem 4108 */ 4109 mutex_enter(&freemem_lock); 4110 availrmem++; 4111 pages_claimed--; 4112 mutex_exit(&freemem_lock); 4113 4114 pp->p_cowcnt--; 4115 4116 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4117 cmn_err(CE_WARN, 4118 "Page lock limit reached on pfn 0x%lx", 4119 page_pptonum(pp)); 4120 } 4121 } 4122 } else { 4123 r = 1; 4124 pp->p_cowcnt--; 4125 pp->p_lckcnt++; 4126 } 4127 page_struct_unlock(pp); 4128 return (r); 4129 } 4130 4131 /* 4132 * Variant of page_addclaim(), where ppa[] contains the pages of a single large 4133 * page. 4134 */ 4135 int 4136 page_addclaim_pages(page_t **ppa) 4137 { 4138 pgcnt_t lckpgs = 0, pg_idx; 4139 4140 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4141 4142 /* 4143 * Only need to take the page struct lock on the large page root. 4144 */ 4145 page_struct_lock(ppa[0]); 4146 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4147 4148 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4149 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4150 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4151 page_struct_unlock(ppa[0]); 4152 return (0); 4153 } 4154 if (ppa[pg_idx]->p_lckcnt > 1) 4155 lckpgs++; 4156 } 4157 4158 if (lckpgs != 0) { 4159 mutex_enter(&freemem_lock); 4160 if (availrmem >= pages_pp_maximum + lckpgs) { 4161 availrmem -= lckpgs; 4162 pages_claimed += lckpgs; 4163 } else { 4164 mutex_exit(&freemem_lock); 4165 page_struct_unlock(ppa[0]); 4166 return (0); 4167 } 4168 mutex_exit(&freemem_lock); 4169 } 4170 4171 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4172 ppa[pg_idx]->p_lckcnt--; 4173 ppa[pg_idx]->p_cowcnt++; 4174 } 4175 page_struct_unlock(ppa[0]); 4176 return (1); 4177 } 4178 4179 /* 4180 * Variant of page_subclaim(), where ppa[] contains the pages of a single large 4181 * page. 4182 */ 4183 int 4184 page_subclaim_pages(page_t **ppa) 4185 { 4186 pgcnt_t ulckpgs = 0, pg_idx; 4187 4188 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4189 4190 /* 4191 * Only need to take the page struct lock on the large page root. 4192 */ 4193 page_struct_lock(ppa[0]); 4194 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4195 4196 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4197 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4198 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4199 page_struct_unlock(ppa[0]); 4200 return (0); 4201 } 4202 if (ppa[pg_idx]->p_lckcnt != 0) 4203 ulckpgs++; 4204 } 4205 4206 if (ulckpgs != 0) { 4207 mutex_enter(&freemem_lock); 4208 availrmem += ulckpgs; 4209 pages_claimed -= ulckpgs; 4210 mutex_exit(&freemem_lock); 4211 } 4212 4213 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4214 ppa[pg_idx]->p_cowcnt--; 4215 ppa[pg_idx]->p_lckcnt++; 4216 4217 } 4218 page_struct_unlock(ppa[0]); 4219 return (1); 4220 } 4221 4222 page_t * 4223 page_numtopp(pfn_t pfnum, se_t se) 4224 { 4225 page_t *pp; 4226 4227 retry: 4228 pp = page_numtopp_nolock(pfnum); 4229 if (pp == NULL) { 4230 return ((page_t *)NULL); 4231 } 4232 4233 /* 4234 * Acquire the appropriate lock on the page. 4235 */ 4236 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4237 if (page_pptonum(pp) != pfnum) 4238 goto retry; 4239 continue; 4240 } 4241 4242 if (page_pptonum(pp) != pfnum) { 4243 page_unlock(pp); 4244 goto retry; 4245 } 4246 4247 return (pp); 4248 } 4249 4250 page_t * 4251 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4252 { 4253 page_t *pp; 4254 4255 retry: 4256 pp = page_numtopp_nolock(pfnum); 4257 if (pp == NULL) { 4258 return ((page_t *)NULL); 4259 } 4260 4261 /* 4262 * Acquire the appropriate lock on the page. 4263 */ 4264 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4265 if (page_pptonum(pp) != pfnum) 4266 goto retry; 4267 continue; 4268 } 4269 4270 if (page_pptonum(pp) != pfnum) { 4271 page_unlock(pp); 4272 goto retry; 4273 } 4274 4275 return (pp); 4276 } 4277 4278 /* 4279 * This routine is like page_numtopp, but will only return page structs 4280 * for pages which are ok for loading into hardware using the page struct. 4281 */ 4282 page_t * 4283 page_numtopp_nowait(pfn_t pfnum, se_t se) 4284 { 4285 page_t *pp; 4286 4287 retry: 4288 pp = page_numtopp_nolock(pfnum); 4289 if (pp == NULL) { 4290 return ((page_t *)NULL); 4291 } 4292 4293 /* 4294 * Try to acquire the appropriate lock on the page. 4295 */ 4296 if (PP_ISFREE(pp)) 4297 pp = NULL; 4298 else { 4299 if (!page_trylock(pp, se)) 4300 pp = NULL; 4301 else { 4302 if (page_pptonum(pp) != pfnum) { 4303 page_unlock(pp); 4304 goto retry; 4305 } 4306 if (PP_ISFREE(pp)) { 4307 page_unlock(pp); 4308 pp = NULL; 4309 } 4310 } 4311 } 4312 return (pp); 4313 } 4314 4315 #define SYNC_PROGRESS_NPAGES 1000 4316 4317 /* 4318 * Returns a count of dirty pages that are in the process 4319 * of being written out. If 'cleanit' is set, try to push the page. 4320 */ 4321 pgcnt_t 4322 page_busy(int cleanit) 4323 { 4324 page_t *page0 = page_first(); 4325 page_t *pp = page0; 4326 pgcnt_t nppbusy = 0; 4327 int counter = 0; 4328 u_offset_t off; 4329 4330 do { 4331 vnode_t *vp = pp->p_vnode; 4332 4333 /* 4334 * Reset the sync timeout. The page list is very long 4335 * on large memory systems. 4336 */ 4337 if (++counter > SYNC_PROGRESS_NPAGES) { 4338 counter = 0; 4339 vfs_syncprogress(); 4340 } 4341 4342 /* 4343 * A page is a candidate for syncing if it is: 4344 * 4345 * (a) On neither the freelist nor the cachelist 4346 * (b) Hashed onto a vnode 4347 * (c) Not a kernel page 4348 * (d) Dirty 4349 * (e) Not part of a swapfile 4350 * (f) a page which belongs to a real vnode; eg has a non-null 4351 * v_vfsp pointer. 4352 * (g) Backed by a filesystem which doesn't have a 4353 * stubbed-out sync operation 4354 */ 4355 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && 4356 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4357 vfs_can_sync(vp->v_vfsp)) { 4358 nppbusy++; 4359 4360 if (!cleanit) 4361 continue; 4362 if (!page_trylock(pp, SE_EXCL)) 4363 continue; 4364 4365 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4366 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4367 !(hat_pagesync(pp, 4368 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4369 page_unlock(pp); 4370 continue; 4371 } 4372 off = pp->p_offset; 4373 VN_HOLD(vp); 4374 page_unlock(pp); 4375 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4376 B_ASYNC | B_FREE, kcred, NULL); 4377 VN_RELE(vp); 4378 } 4379 } while ((pp = page_next(pp)) != page0); 4380 4381 vfs_syncprogress(); 4382 return (nppbusy); 4383 } 4384 4385 void page_invalidate_pages(void); 4386 4387 /* 4388 * callback handler to vm sub-system 4389 * 4390 * callers make sure no recursive entries to this func. 4391 */ 4392 /*ARGSUSED*/ 4393 boolean_t 4394 callb_vm_cpr(void *arg, int code) 4395 { 4396 if (code == CB_CODE_CPR_CHKPT) 4397 page_invalidate_pages(); 4398 return (B_TRUE); 4399 } 4400 4401 /* 4402 * Invalidate all pages of the system. 4403 * It shouldn't be called until all user page activities are all stopped. 4404 */ 4405 void 4406 page_invalidate_pages() 4407 { 4408 page_t *pp; 4409 page_t *page0; 4410 pgcnt_t nbusypages; 4411 int retry = 0; 4412 const int MAXRETRIES = 4; 4413 top: 4414 /* 4415 * Flush dirty pages and destroy the clean ones. 4416 */ 4417 nbusypages = 0; 4418 4419 pp = page0 = page_first(); 4420 do { 4421 struct vnode *vp; 4422 u_offset_t offset; 4423 int mod; 4424 4425 /* 4426 * skip the page if it has no vnode or the page associated 4427 * with the kernel vnode or prom allocated kernel mem. 4428 */ 4429 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) 4430 continue; 4431 4432 /* 4433 * skip the page which is already free invalidated. 4434 */ 4435 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4436 continue; 4437 4438 /* 4439 * skip pages that are already locked or can't be "exclusively" 4440 * locked or are already free. After we lock the page, check 4441 * the free and age bits again to be sure it's not destroyed 4442 * yet. 4443 * To achieve max. parallelization, we use page_trylock instead 4444 * of page_lock so that we don't get block on individual pages 4445 * while we have thousands of other pages to process. 4446 */ 4447 if (!page_trylock(pp, SE_EXCL)) { 4448 nbusypages++; 4449 continue; 4450 } else if (PP_ISFREE(pp)) { 4451 if (!PP_ISAGED(pp)) { 4452 page_destroy_free(pp); 4453 } else { 4454 page_unlock(pp); 4455 } 4456 continue; 4457 } 4458 /* 4459 * Is this page involved in some I/O? shared? 4460 * 4461 * The page_struct_lock need not be acquired to 4462 * examine these fields since the page has an 4463 * "exclusive" lock. 4464 */ 4465 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4466 page_unlock(pp); 4467 continue; 4468 } 4469 4470 if (vp->v_type == VCHR) { 4471 panic("vp->v_type == VCHR"); 4472 /*NOTREACHED*/ 4473 } 4474 4475 if (!page_try_demote_pages(pp)) { 4476 page_unlock(pp); 4477 continue; 4478 } 4479 4480 /* 4481 * Check the modified bit. Leave the bits alone in hardware 4482 * (they will be modified if we do the putpage). 4483 */ 4484 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4485 & P_MOD); 4486 if (mod) { 4487 offset = pp->p_offset; 4488 /* 4489 * Hold the vnode before releasing the page lock 4490 * to prevent it from being freed and re-used by 4491 * some other thread. 4492 */ 4493 VN_HOLD(vp); 4494 page_unlock(pp); 4495 /* 4496 * No error return is checked here. Callers such as 4497 * cpr deals with the dirty pages at the dump time 4498 * if this putpage fails. 4499 */ 4500 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4501 kcred, NULL); 4502 VN_RELE(vp); 4503 } else { 4504 /*LINTED: constant in conditional context*/ 4505 VN_DISPOSE(pp, B_INVAL, 0, kcred); 4506 } 4507 } while ((pp = page_next(pp)) != page0); 4508 if (nbusypages && retry++ < MAXRETRIES) { 4509 delay(1); 4510 goto top; 4511 } 4512 } 4513 4514 /* 4515 * Replace the page "old" with the page "new" on the page hash and vnode lists 4516 * 4517 * the replacement must be done in place, ie the equivalent sequence: 4518 * 4519 * vp = old->p_vnode; 4520 * off = old->p_offset; 4521 * page_do_hashout(old) 4522 * page_do_hashin(new, vp, off) 4523 * 4524 * doesn't work, since 4525 * 1) if old is the only page on the vnode, the v_pages list has a window 4526 * where it looks empty. This will break file system assumptions. 4527 * and 4528 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4529 */ 4530 static void 4531 page_do_relocate_hash(page_t *new, page_t *old) 4532 { 4533 page_t **hash_list; 4534 vnode_t *vp = old->p_vnode; 4535 kmutex_t *sep; 4536 4537 ASSERT(PAGE_EXCL(old)); 4538 ASSERT(PAGE_EXCL(new)); 4539 ASSERT(vp != NULL); 4540 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4541 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4542 4543 /* 4544 * First find old page on the page hash list 4545 */ 4546 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4547 4548 for (;;) { 4549 if (*hash_list == old) 4550 break; 4551 if (*hash_list == NULL) { 4552 panic("page_do_hashout"); 4553 /*NOTREACHED*/ 4554 } 4555 hash_list = &(*hash_list)->p_hash; 4556 } 4557 4558 /* 4559 * update new and replace old with new on the page hash list 4560 */ 4561 new->p_vnode = old->p_vnode; 4562 new->p_offset = old->p_offset; 4563 new->p_hash = old->p_hash; 4564 *hash_list = new; 4565 4566 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4567 PP_SETSWAP(new); 4568 4569 /* 4570 * replace old with new on the vnode's page list 4571 */ 4572 if (old->p_vpnext == old) { 4573 new->p_vpnext = new; 4574 new->p_vpprev = new; 4575 } else { 4576 new->p_vpnext = old->p_vpnext; 4577 new->p_vpprev = old->p_vpprev; 4578 new->p_vpnext->p_vpprev = new; 4579 new->p_vpprev->p_vpnext = new; 4580 } 4581 if (vp->v_pages == old) 4582 vp->v_pages = new; 4583 4584 /* 4585 * clear out the old page 4586 */ 4587 old->p_hash = NULL; 4588 old->p_vpnext = NULL; 4589 old->p_vpprev = NULL; 4590 old->p_vnode = NULL; 4591 PP_CLRSWAP(old); 4592 old->p_offset = (u_offset_t)-1; 4593 page_clr_all_props(old); 4594 4595 /* 4596 * Wake up processes waiting for this page. The page's 4597 * identity has been changed, and is probably not the 4598 * desired page any longer. 4599 */ 4600 sep = page_se_mutex(old); 4601 mutex_enter(sep); 4602 old->p_selock &= ~SE_EWANTED; 4603 if (CV_HAS_WAITERS(&old->p_cv)) 4604 cv_broadcast(&old->p_cv); 4605 mutex_exit(sep); 4606 } 4607 4608 /* 4609 * This function moves the identity of page "pp_old" to page "pp_new". 4610 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4611 * and need not be hashed out from anywhere. 4612 */ 4613 void 4614 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4615 { 4616 vnode_t *vp = pp_old->p_vnode; 4617 u_offset_t off = pp_old->p_offset; 4618 kmutex_t *phm, *vphm; 4619 4620 /* 4621 * Rehash two pages 4622 */ 4623 ASSERT(PAGE_EXCL(pp_old)); 4624 ASSERT(PAGE_EXCL(pp_new)); 4625 ASSERT(vp != NULL); 4626 ASSERT(pp_new->p_vnode == NULL); 4627 4628 /* 4629 * hashout then hashin while holding the mutexes 4630 */ 4631 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4632 mutex_enter(phm); 4633 vphm = page_vnode_mutex(vp); 4634 mutex_enter(vphm); 4635 4636 page_do_relocate_hash(pp_new, pp_old); 4637 4638 /* The following comment preserved from page_flip(). */ 4639 pp_new->p_fsdata = pp_old->p_fsdata; 4640 pp_old->p_fsdata = 0; 4641 mutex_exit(vphm); 4642 mutex_exit(phm); 4643 4644 /* 4645 * The page_struct_lock need not be acquired for lckcnt and 4646 * cowcnt since the page has an "exclusive" lock. 4647 */ 4648 ASSERT(pp_new->p_lckcnt == 0); 4649 ASSERT(pp_new->p_cowcnt == 0); 4650 pp_new->p_lckcnt = pp_old->p_lckcnt; 4651 pp_new->p_cowcnt = pp_old->p_cowcnt; 4652 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4653 4654 } 4655 4656 /* 4657 * Helper routine used to lock all remaining members of a 4658 * large page. The caller is responsible for passing in a locked 4659 * pp. If pp is a large page, then it succeeds in locking all the 4660 * remaining constituent pages or it returns with only the 4661 * original page locked. 4662 * 4663 * Returns 1 on success, 0 on failure. 4664 * 4665 * If success is returned this routine guarantees p_szc for all constituent 4666 * pages of a large page pp belongs to can't change. To achieve this we 4667 * recheck szc of pp after locking all constituent pages and retry if szc 4668 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4669 * lock on one of constituent pages it can't be running after all constituent 4670 * pages are locked. hat_page_demote() with a lock on a constituent page 4671 * outside of this large page (i.e. pp belonged to a larger large page) is 4672 * already done with all constituent pages of pp since the root's p_szc is 4673 * changed last. Therefore no need to synchronize with hat_page_demote() that 4674 * locked a constituent page outside of pp's current large page. 4675 */ 4676 #ifdef DEBUG 4677 uint32_t gpg_trylock_mtbf = 0; 4678 #endif 4679 4680 int 4681 group_page_trylock(page_t *pp, se_t se) 4682 { 4683 page_t *tpp; 4684 pgcnt_t npgs, i, j; 4685 uint_t pszc = pp->p_szc; 4686 4687 #ifdef DEBUG 4688 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4689 return (0); 4690 } 4691 #endif 4692 4693 if (pp != PP_GROUPLEADER(pp, pszc)) { 4694 return (0); 4695 } 4696 4697 retry: 4698 ASSERT(PAGE_LOCKED_SE(pp, se)); 4699 ASSERT(!PP_ISFREE(pp)); 4700 if (pszc == 0) { 4701 return (1); 4702 } 4703 npgs = page_get_pagecnt(pszc); 4704 tpp = pp + 1; 4705 for (i = 1; i < npgs; i++, tpp++) { 4706 if (!page_trylock(tpp, se)) { 4707 tpp = pp + 1; 4708 for (j = 1; j < i; j++, tpp++) { 4709 page_unlock(tpp); 4710 } 4711 return (0); 4712 } 4713 } 4714 if (pp->p_szc != pszc) { 4715 ASSERT(pp->p_szc < pszc); 4716 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && 4717 !IS_SWAPFSVP(pp->p_vnode)); 4718 tpp = pp + 1; 4719 for (i = 1; i < npgs; i++, tpp++) { 4720 page_unlock(tpp); 4721 } 4722 pszc = pp->p_szc; 4723 goto retry; 4724 } 4725 return (1); 4726 } 4727 4728 void 4729 group_page_unlock(page_t *pp) 4730 { 4731 page_t *tpp; 4732 pgcnt_t npgs, i; 4733 4734 ASSERT(PAGE_LOCKED(pp)); 4735 ASSERT(!PP_ISFREE(pp)); 4736 ASSERT(pp == PP_PAGEROOT(pp)); 4737 npgs = page_get_pagecnt(pp->p_szc); 4738 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4739 page_unlock(tpp); 4740 } 4741 } 4742 4743 /* 4744 * returns 4745 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4746 * ERANGE : this is not a base page 4747 * EBUSY : failure to get locks on the page/pages 4748 * ENOMEM : failure to obtain replacement pages 4749 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4750 * EIO : An error occurred while trying to copy the page data 4751 * 4752 * Return with all constituent members of target and replacement 4753 * SE_EXCL locked. It is the callers responsibility to drop the 4754 * locks. 4755 */ 4756 int 4757 do_page_relocate( 4758 page_t **target, 4759 page_t **replacement, 4760 int grouplock, 4761 spgcnt_t *nrelocp, 4762 lgrp_t *lgrp) 4763 { 4764 page_t *first_repl; 4765 page_t *repl; 4766 page_t *targ; 4767 page_t *pl = NULL; 4768 uint_t ppattr; 4769 pfn_t pfn, repl_pfn; 4770 uint_t szc; 4771 spgcnt_t npgs, i; 4772 int repl_contig = 0; 4773 uint_t flags = 0; 4774 spgcnt_t dofree = 0; 4775 4776 *nrelocp = 0; 4777 4778 #if defined(__sparc) 4779 /* 4780 * We need to wait till OBP has completed 4781 * its boot-time handoff of its resources to the kernel 4782 * before we allow page relocation 4783 */ 4784 if (page_relocate_ready == 0) { 4785 return (EAGAIN); 4786 } 4787 #endif 4788 4789 /* 4790 * If this is not a base page, 4791 * just return with 0x0 pages relocated. 4792 */ 4793 targ = *target; 4794 ASSERT(PAGE_EXCL(targ)); 4795 ASSERT(!PP_ISFREE(targ)); 4796 szc = targ->p_szc; 4797 ASSERT(szc < mmu_page_sizes); 4798 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4799 pfn = targ->p_pagenum; 4800 if (pfn != PFN_BASE(pfn, szc)) { 4801 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4802 return (ERANGE); 4803 } 4804 4805 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4806 repl_pfn = repl->p_pagenum; 4807 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4808 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4809 return (ERANGE); 4810 } 4811 repl_contig = 1; 4812 } 4813 4814 /* 4815 * We must lock all members of this large page or we cannot 4816 * relocate any part of it. 4817 */ 4818 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4819 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4820 return (EBUSY); 4821 } 4822 4823 /* 4824 * reread szc it could have been decreased before 4825 * group_page_trylock() was done. 4826 */ 4827 szc = targ->p_szc; 4828 ASSERT(szc < mmu_page_sizes); 4829 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4830 ASSERT(pfn == PFN_BASE(pfn, szc)); 4831 4832 npgs = page_get_pagecnt(targ->p_szc); 4833 4834 if (repl == NULL) { 4835 dofree = npgs; /* Size of target page in MMU pages */ 4836 if (!page_create_wait(dofree, 0)) { 4837 if (grouplock != 0) { 4838 group_page_unlock(targ); 4839 } 4840 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4841 return (ENOMEM); 4842 } 4843 4844 /* 4845 * seg kmem pages require that the target and replacement 4846 * page be the same pagesize. 4847 */ 4848 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0; 4849 repl = page_get_replacement_page(targ, lgrp, flags); 4850 if (repl == NULL) { 4851 if (grouplock != 0) { 4852 group_page_unlock(targ); 4853 } 4854 page_create_putback(dofree); 4855 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4856 return (ENOMEM); 4857 } 4858 } 4859 #ifdef DEBUG 4860 else { 4861 ASSERT(PAGE_LOCKED(repl)); 4862 } 4863 #endif /* DEBUG */ 4864 4865 #if defined(__sparc) 4866 /* 4867 * Let hat_page_relocate() complete the relocation if it's kernel page 4868 */ 4869 if (VN_ISKAS(targ->p_vnode)) { 4870 *replacement = repl; 4871 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4872 if (grouplock != 0) { 4873 group_page_unlock(targ); 4874 } 4875 if (dofree) { 4876 *replacement = NULL; 4877 page_free_replacement_page(repl); 4878 page_create_putback(dofree); 4879 } 4880 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4881 return (EAGAIN); 4882 } 4883 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4884 return (0); 4885 } 4886 #else 4887 #if defined(lint) 4888 dofree = dofree; 4889 #endif 4890 #endif 4891 4892 first_repl = repl; 4893 4894 for (i = 0; i < npgs; i++) { 4895 ASSERT(PAGE_EXCL(targ)); 4896 ASSERT(targ->p_slckcnt == 0); 4897 ASSERT(repl->p_slckcnt == 0); 4898 4899 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4900 4901 ASSERT(hat_page_getshare(targ) == 0); 4902 ASSERT(!PP_ISFREE(targ)); 4903 ASSERT(targ->p_pagenum == (pfn + i)); 4904 ASSERT(repl_contig == 0 || 4905 repl->p_pagenum == (repl_pfn + i)); 4906 4907 /* 4908 * Copy the page contents and attributes then 4909 * relocate the page in the page hash. 4910 */ 4911 if (ppcopy(targ, repl) == 0) { 4912 targ = *target; 4913 repl = first_repl; 4914 VM_STAT_ADD(vmm_vmstats.ppr_copyfail); 4915 if (grouplock != 0) { 4916 group_page_unlock(targ); 4917 } 4918 if (dofree) { 4919 *replacement = NULL; 4920 page_free_replacement_page(repl); 4921 page_create_putback(dofree); 4922 } 4923 return (EIO); 4924 } 4925 4926 targ++; 4927 if (repl_contig != 0) { 4928 repl++; 4929 } else { 4930 repl = repl->p_next; 4931 } 4932 } 4933 4934 repl = first_repl; 4935 targ = *target; 4936 4937 for (i = 0; i < npgs; i++) { 4938 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4939 page_clr_all_props(repl); 4940 page_set_props(repl, ppattr); 4941 page_relocate_hash(repl, targ); 4942 4943 ASSERT(hat_page_getshare(targ) == 0); 4944 ASSERT(hat_page_getshare(repl) == 0); 4945 /* 4946 * Now clear the props on targ, after the 4947 * page_relocate_hash(), they no longer 4948 * have any meaning. 4949 */ 4950 page_clr_all_props(targ); 4951 ASSERT(targ->p_next == targ); 4952 ASSERT(targ->p_prev == targ); 4953 page_list_concat(&pl, &targ); 4954 4955 targ++; 4956 if (repl_contig != 0) { 4957 repl++; 4958 } else { 4959 repl = repl->p_next; 4960 } 4961 } 4962 /* assert that we have come full circle with repl */ 4963 ASSERT(repl_contig == 1 || first_repl == repl); 4964 4965 *target = pl; 4966 if (*replacement == NULL) { 4967 ASSERT(first_repl == repl); 4968 *replacement = repl; 4969 } 4970 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4971 *nrelocp = npgs; 4972 return (0); 4973 } 4974 /* 4975 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 4976 */ 4977 int 4978 page_relocate( 4979 page_t **target, 4980 page_t **replacement, 4981 int grouplock, 4982 int freetarget, 4983 spgcnt_t *nrelocp, 4984 lgrp_t *lgrp) 4985 { 4986 spgcnt_t ret; 4987 4988 /* do_page_relocate returns 0 on success or errno value */ 4989 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 4990 4991 if (ret != 0 || freetarget == 0) { 4992 return (ret); 4993 } 4994 if (*nrelocp == 1) { 4995 ASSERT(*target != NULL); 4996 page_free(*target, 1); 4997 } else { 4998 page_t *tpp = *target; 4999 uint_t szc = tpp->p_szc; 5000 pgcnt_t npgs = page_get_pagecnt(szc); 5001 ASSERT(npgs > 1); 5002 ASSERT(szc != 0); 5003 do { 5004 ASSERT(PAGE_EXCL(tpp)); 5005 ASSERT(!hat_page_is_mapped(tpp)); 5006 ASSERT(tpp->p_szc == szc); 5007 PP_SETFREE(tpp); 5008 PP_SETAGED(tpp); 5009 npgs--; 5010 } while ((tpp = tpp->p_next) != *target); 5011 ASSERT(npgs == 0); 5012 page_list_add_pages(*target, 0); 5013 npgs = page_get_pagecnt(szc); 5014 page_create_putback(npgs); 5015 } 5016 return (ret); 5017 } 5018 5019 /* 5020 * it is up to the caller to deal with pcf accounting. 5021 */ 5022 void 5023 page_free_replacement_page(page_t *pplist) 5024 { 5025 page_t *pp; 5026 5027 while (pplist != NULL) { 5028 /* 5029 * pp_targ is a linked list. 5030 */ 5031 pp = pplist; 5032 if (pp->p_szc == 0) { 5033 page_sub(&pplist, pp); 5034 page_clr_all_props(pp); 5035 PP_SETFREE(pp); 5036 PP_SETAGED(pp); 5037 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5038 page_unlock(pp); 5039 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5040 } else { 5041 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5042 page_t *tpp; 5043 page_list_break(&pp, &pplist, curnpgs); 5044 tpp = pp; 5045 do { 5046 ASSERT(PAGE_EXCL(tpp)); 5047 ASSERT(!hat_page_is_mapped(tpp)); 5048 page_clr_all_props(tpp); 5049 PP_SETFREE(tpp); 5050 PP_SETAGED(tpp); 5051 } while ((tpp = tpp->p_next) != pp); 5052 page_list_add_pages(pp, 0); 5053 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5054 } 5055 } 5056 } 5057 5058 /* 5059 * Relocate target to non-relocatable replacement page. 5060 */ 5061 int 5062 page_relocate_cage(page_t **target, page_t **replacement) 5063 { 5064 page_t *tpp, *rpp; 5065 spgcnt_t pgcnt, npgs; 5066 int result; 5067 5068 tpp = *target; 5069 5070 ASSERT(PAGE_EXCL(tpp)); 5071 ASSERT(tpp->p_szc == 0); 5072 5073 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5074 5075 do { 5076 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5077 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5078 if (rpp == NULL) { 5079 page_create_putback(pgcnt); 5080 kcage_cageout_wakeup(); 5081 } 5082 } while (rpp == NULL); 5083 5084 ASSERT(PP_ISNORELOC(rpp)); 5085 5086 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5087 5088 if (result == 0) { 5089 *replacement = rpp; 5090 if (pgcnt != npgs) 5091 panic("page_relocate_cage: partial relocation"); 5092 } 5093 5094 return (result); 5095 } 5096 5097 /* 5098 * Release the page lock on a page, place on cachelist 5099 * tail if no longer mapped. Caller can let us know if 5100 * the page is known to be clean. 5101 */ 5102 int 5103 page_release(page_t *pp, int checkmod) 5104 { 5105 int status; 5106 5107 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5108 (pp->p_vnode != NULL)); 5109 5110 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5111 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5112 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5113 !hat_page_is_mapped(pp)) { 5114 5115 /* 5116 * If page is modified, unlock it 5117 * 5118 * (p_nrm & P_MOD) bit has the latest stuff because: 5119 * (1) We found that this page doesn't have any mappings 5120 * _after_ holding SE_EXCL and 5121 * (2) We didn't drop SE_EXCL lock after the check in (1) 5122 */ 5123 if (checkmod && hat_ismod(pp)) { 5124 page_unlock(pp); 5125 status = PGREL_MOD; 5126 } else { 5127 /*LINTED: constant in conditional context*/ 5128 VN_DISPOSE(pp, B_FREE, 0, kcred); 5129 status = PGREL_CLEAN; 5130 } 5131 } else { 5132 page_unlock(pp); 5133 status = PGREL_NOTREL; 5134 } 5135 return (status); 5136 } 5137 5138 /* 5139 * Given a constituent page, try to demote the large page on the freelist. 5140 * 5141 * Returns nonzero if the page could be demoted successfully. Returns with 5142 * the constituent page still locked. 5143 */ 5144 int 5145 page_try_demote_free_pages(page_t *pp) 5146 { 5147 page_t *rootpp = pp; 5148 pfn_t pfn = page_pptonum(pp); 5149 spgcnt_t npgs; 5150 uint_t szc = pp->p_szc; 5151 5152 ASSERT(PP_ISFREE(pp)); 5153 ASSERT(PAGE_EXCL(pp)); 5154 5155 /* 5156 * Adjust rootpp and lock it, if `pp' is not the base 5157 * constituent page. 5158 */ 5159 npgs = page_get_pagecnt(pp->p_szc); 5160 if (npgs == 1) { 5161 return (0); 5162 } 5163 5164 if (!IS_P2ALIGNED(pfn, npgs)) { 5165 pfn = P2ALIGN(pfn, npgs); 5166 rootpp = page_numtopp_nolock(pfn); 5167 } 5168 5169 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5170 return (0); 5171 } 5172 5173 if (rootpp->p_szc != szc) { 5174 if (pp != rootpp) 5175 page_unlock(rootpp); 5176 return (0); 5177 } 5178 5179 page_demote_free_pages(rootpp); 5180 5181 if (pp != rootpp) 5182 page_unlock(rootpp); 5183 5184 ASSERT(PP_ISFREE(pp)); 5185 ASSERT(PAGE_EXCL(pp)); 5186 return (1); 5187 } 5188 5189 /* 5190 * Given a constituent page, try to demote the large page. 5191 * 5192 * Returns nonzero if the page could be demoted successfully. Returns with 5193 * the constituent page still locked. 5194 */ 5195 int 5196 page_try_demote_pages(page_t *pp) 5197 { 5198 page_t *tpp, *rootpp = pp; 5199 pfn_t pfn = page_pptonum(pp); 5200 spgcnt_t i, npgs; 5201 uint_t szc = pp->p_szc; 5202 vnode_t *vp = pp->p_vnode; 5203 5204 ASSERT(PAGE_EXCL(pp)); 5205 5206 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5207 5208 if (pp->p_szc == 0) { 5209 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5210 return (1); 5211 } 5212 5213 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) { 5214 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5215 page_demote_vp_pages(pp); 5216 ASSERT(pp->p_szc == 0); 5217 return (1); 5218 } 5219 5220 /* 5221 * Adjust rootpp if passed in is not the base 5222 * constituent page. 5223 */ 5224 npgs = page_get_pagecnt(pp->p_szc); 5225 ASSERT(npgs > 1); 5226 if (!IS_P2ALIGNED(pfn, npgs)) { 5227 pfn = P2ALIGN(pfn, npgs); 5228 rootpp = page_numtopp_nolock(pfn); 5229 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5230 ASSERT(rootpp->p_vnode != NULL); 5231 ASSERT(rootpp->p_szc == szc); 5232 } 5233 5234 /* 5235 * We can't demote kernel pages since we can't hat_unload() 5236 * the mappings. 5237 */ 5238 if (VN_ISKAS(rootpp->p_vnode)) 5239 return (0); 5240 5241 /* 5242 * Attempt to lock all constituent pages except the page passed 5243 * in since it's already locked. 5244 */ 5245 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5246 ASSERT(!PP_ISFREE(tpp)); 5247 ASSERT(tpp->p_vnode != NULL); 5248 5249 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5250 break; 5251 ASSERT(tpp->p_szc == rootpp->p_szc); 5252 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5253 } 5254 5255 /* 5256 * If we failed to lock them all then unlock what we have 5257 * locked so far and bail. 5258 */ 5259 if (i < npgs) { 5260 tpp = rootpp; 5261 while (i-- > 0) { 5262 if (tpp != pp) 5263 page_unlock(tpp); 5264 tpp++; 5265 } 5266 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5267 return (0); 5268 } 5269 5270 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5271 ASSERT(PAGE_EXCL(tpp)); 5272 ASSERT(tpp->p_slckcnt == 0); 5273 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5274 tpp->p_szc = 0; 5275 } 5276 5277 /* 5278 * Unlock all pages except the page passed in. 5279 */ 5280 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5281 ASSERT(!hat_page_is_mapped(tpp)); 5282 if (tpp != pp) 5283 page_unlock(tpp); 5284 } 5285 5286 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5287 return (1); 5288 } 5289 5290 /* 5291 * Called by page_free() and page_destroy() to demote the page size code 5292 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5293 * p_szc on free list, neither can we just clear p_szc of a single page_t 5294 * within a large page since it will break other code that relies on p_szc 5295 * being the same for all page_t's of a large page). Anonymous pages should 5296 * never end up here because anon_map_getpages() cannot deal with p_szc 5297 * changes after a single constituent page is locked. While anonymous or 5298 * kernel large pages are demoted or freed the entire large page at a time 5299 * with all constituent pages locked EXCL for the file system pages we 5300 * have to be able to demote a large page (i.e. decrease all constituent pages 5301 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5302 * we can easily deal with anonymous page demotion the entire large page at a 5303 * time is that those operation originate at address space level and concern 5304 * the entire large page region with actual demotion only done when pages are 5305 * not shared with any other processes (therefore we can always get EXCL lock 5306 * on all anonymous constituent pages after clearing segment page 5307 * cache). However file system pages can be truncated or invalidated at a 5308 * PAGESIZE level from the file system side and end up in page_free() or 5309 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5310 * and therefore pageout should be able to demote a large page by EXCL locking 5311 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5312 * rely on being able to lock EXCL all constituent pages. 5313 * 5314 * To prevent szc changes on file system pages one has to lock all constituent 5315 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5316 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5317 * prevent szc changes is hat layer that uses its own page level mlist 5318 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5319 * taken. Therefore we need to change szc under hat level locks if we only 5320 * have an EXCL lock on a single constituent page and hat still references any 5321 * of constituent pages. (Note we can't "ignore" hat layer by simply 5322 * hat_pageunload() all constituent pages without having EXCL locks on all of 5323 * constituent pages). We use hat_page_demote() call to safely demote szc of 5324 * all constituent pages under hat locks when we only have an EXCL lock on one 5325 * of constituent pages. 5326 * 5327 * This routine calls page_szc_lock() before calling hat_page_demote() to 5328 * allow segvn in one special case not to lock all constituent pages SHARED 5329 * before calling hat_memload_array() that relies on p_szc not changing even 5330 * before hat level mlist lock is taken. In that case segvn uses 5331 * page_szc_lock() to prevent hat_page_demote() changing p_szc values. 5332 * 5333 * Anonymous or kernel page demotion still has to lock all pages exclusively 5334 * and do hat_pageunload() on all constituent pages before demoting the page 5335 * therefore there's no need for anonymous or kernel page demotion to use 5336 * hat_page_demote() mechanism. 5337 * 5338 * hat_page_demote() removes all large mappings that map pp and then decreases 5339 * p_szc starting from the last constituent page of the large page. By working 5340 * from the tail of a large page in pfn decreasing order allows one looking at 5341 * the root page to know that hat_page_demote() is done for root's szc area. 5342 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5343 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5344 * that started on this page when it had szc > 1 is done for this szc 1 area. 5345 * 5346 * We are guaranteed that all constituent pages of pp's large page belong to 5347 * the same vnode with the consecutive offsets increasing in the direction of 5348 * the pfn i.e. the identity of constituent pages can't change until their 5349 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5350 * large mappings to pp even though we don't lock any constituent page except 5351 * pp (i.e. we won't unload e.g. kernel locked page). 5352 */ 5353 static void 5354 page_demote_vp_pages(page_t *pp) 5355 { 5356 kmutex_t *mtx; 5357 5358 ASSERT(PAGE_EXCL(pp)); 5359 ASSERT(!PP_ISFREE(pp)); 5360 ASSERT(pp->p_vnode != NULL); 5361 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5362 ASSERT(!PP_ISKAS(pp)); 5363 5364 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5365 5366 mtx = page_szc_lock(pp); 5367 if (mtx != NULL) { 5368 hat_page_demote(pp); 5369 mutex_exit(mtx); 5370 } 5371 ASSERT(pp->p_szc == 0); 5372 } 5373 5374 /* 5375 * Mark any existing pages for migration in the given range 5376 */ 5377 void 5378 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5379 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5380 u_offset_t vnoff, int rflag) 5381 { 5382 struct anon *ap; 5383 vnode_t *curvp; 5384 lgrp_t *from; 5385 pgcnt_t nlocked; 5386 u_offset_t off; 5387 pfn_t pfn; 5388 size_t pgsz; 5389 size_t segpgsz; 5390 pgcnt_t pages; 5391 uint_t pszc; 5392 page_t *pp0, *pp; 5393 caddr_t va; 5394 ulong_t an_idx; 5395 anon_sync_obj_t cookie; 5396 5397 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5398 5399 /* 5400 * Don't do anything if don't need to do lgroup optimizations 5401 * on this system 5402 */ 5403 if (!lgrp_optimizations()) 5404 return; 5405 5406 /* 5407 * Align address and length to (potentially large) page boundary 5408 */ 5409 segpgsz = page_get_pagesize(seg->s_szc); 5410 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5411 if (rflag) 5412 len = P2ROUNDUP(len, segpgsz); 5413 5414 /* 5415 * Do one (large) page at a time 5416 */ 5417 va = addr; 5418 while (va < addr + len) { 5419 /* 5420 * Lookup (root) page for vnode and offset corresponding to 5421 * this virtual address 5422 * Try anonmap first since there may be copy-on-write 5423 * pages, but initialize vnode pointer and offset using 5424 * vnode arguments just in case there isn't an amp. 5425 */ 5426 curvp = vp; 5427 off = vnoff + va - seg->s_base; 5428 if (amp) { 5429 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5430 an_idx = anon_index + seg_page(seg, va); 5431 anon_array_enter(amp, an_idx, &cookie); 5432 ap = anon_get_ptr(amp->ahp, an_idx); 5433 if (ap) 5434 swap_xlate(ap, &curvp, &off); 5435 anon_array_exit(&cookie); 5436 ANON_LOCK_EXIT(&->a_rwlock); 5437 } 5438 5439 pp = NULL; 5440 if (curvp) 5441 pp = page_lookup(curvp, off, SE_SHARED); 5442 5443 /* 5444 * If there isn't a page at this virtual address, 5445 * skip to next page 5446 */ 5447 if (pp == NULL) { 5448 va += PAGESIZE; 5449 continue; 5450 } 5451 5452 /* 5453 * Figure out which lgroup this page is in for kstats 5454 */ 5455 pfn = page_pptonum(pp); 5456 from = lgrp_pfn_to_lgrp(pfn); 5457 5458 /* 5459 * Get page size, and round up and skip to next page boundary 5460 * if unaligned address 5461 */ 5462 pszc = pp->p_szc; 5463 pgsz = page_get_pagesize(pszc); 5464 pages = btop(pgsz); 5465 if (!IS_P2ALIGNED(va, pgsz) || 5466 !IS_P2ALIGNED(pfn, pages) || 5467 pgsz > segpgsz) { 5468 pgsz = MIN(pgsz, segpgsz); 5469 page_unlock(pp); 5470 pages = btop(P2END((uintptr_t)va, pgsz) - 5471 (uintptr_t)va); 5472 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5473 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, pages); 5474 continue; 5475 } 5476 5477 /* 5478 * Upgrade to exclusive lock on page 5479 */ 5480 if (!page_tryupgrade(pp)) { 5481 page_unlock(pp); 5482 va += pgsz; 5483 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5484 btop(pgsz)); 5485 continue; 5486 } 5487 5488 pp0 = pp++; 5489 nlocked = 1; 5490 5491 /* 5492 * Lock constituent pages if this is large page 5493 */ 5494 if (pages > 1) { 5495 /* 5496 * Lock all constituents except root page, since it 5497 * should be locked already. 5498 */ 5499 for (; nlocked < pages; nlocked++) { 5500 if (!page_trylock(pp, SE_EXCL)) { 5501 break; 5502 } 5503 if (PP_ISFREE(pp) || 5504 pp->p_szc != pszc) { 5505 /* 5506 * hat_page_demote() raced in with us. 5507 */ 5508 ASSERT(!IS_SWAPFSVP(curvp)); 5509 page_unlock(pp); 5510 break; 5511 } 5512 pp++; 5513 } 5514 } 5515 5516 /* 5517 * If all constituent pages couldn't be locked, 5518 * unlock pages locked so far and skip to next page. 5519 */ 5520 if (nlocked < pages) { 5521 while (pp0 < pp) { 5522 page_unlock(pp0++); 5523 } 5524 va += pgsz; 5525 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5526 btop(pgsz)); 5527 continue; 5528 } 5529 5530 /* 5531 * hat_page_demote() can no longer happen 5532 * since last cons page had the right p_szc after 5533 * all cons pages were locked. all cons pages 5534 * should now have the same p_szc. 5535 */ 5536 5537 /* 5538 * All constituent pages locked successfully, so mark 5539 * large page for migration and unload the mappings of 5540 * constituent pages, so a fault will occur on any part of the 5541 * large page 5542 */ 5543 PP_SETMIGRATE(pp0); 5544 while (pp0 < pp) { 5545 (void) hat_pageunload(pp0, HAT_FORCE_PGUNLOAD); 5546 ASSERT(hat_page_getshare(pp0) == 0); 5547 page_unlock(pp0++); 5548 } 5549 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5550 5551 va += pgsz; 5552 } 5553 } 5554 5555 /* 5556 * Migrate any pages that have been marked for migration in the given range 5557 */ 5558 void 5559 page_migrate( 5560 struct seg *seg, 5561 caddr_t addr, 5562 page_t **ppa, 5563 pgcnt_t npages) 5564 { 5565 lgrp_t *from; 5566 lgrp_t *to; 5567 page_t *newpp; 5568 page_t *pp; 5569 pfn_t pfn; 5570 size_t pgsz; 5571 spgcnt_t page_cnt; 5572 spgcnt_t i; 5573 uint_t pszc; 5574 5575 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5576 5577 while (npages > 0) { 5578 pp = *ppa; 5579 pszc = pp->p_szc; 5580 pgsz = page_get_pagesize(pszc); 5581 page_cnt = btop(pgsz); 5582 5583 /* 5584 * Check to see whether this page is marked for migration 5585 * 5586 * Assume that root page of large page is marked for 5587 * migration and none of the other constituent pages 5588 * are marked. This really simplifies clearing the 5589 * migrate bit by not having to clear it from each 5590 * constituent page. 5591 * 5592 * note we don't want to relocate an entire large page if 5593 * someone is only using one subpage. 5594 */ 5595 if (npages < page_cnt) 5596 break; 5597 5598 /* 5599 * Is it marked for migration? 5600 */ 5601 if (!PP_ISMIGRATE(pp)) 5602 goto next; 5603 5604 /* 5605 * Determine lgroups that page is being migrated between 5606 */ 5607 pfn = page_pptonum(pp); 5608 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5609 break; 5610 } 5611 from = lgrp_pfn_to_lgrp(pfn); 5612 to = lgrp_mem_choose(seg, addr, pgsz); 5613 5614 /* 5615 * Need to get exclusive lock's to migrate 5616 */ 5617 for (i = 0; i < page_cnt; i++) { 5618 ASSERT(PAGE_LOCKED(ppa[i])); 5619 if (page_pptonum(ppa[i]) != pfn + i || 5620 ppa[i]->p_szc != pszc) { 5621 break; 5622 } 5623 if (!page_tryupgrade(ppa[i])) { 5624 lgrp_stat_add(from->lgrp_id, 5625 LGRP_PM_FAIL_LOCK_PGS, 5626 page_cnt); 5627 break; 5628 } 5629 5630 /* 5631 * Check to see whether we are trying to migrate 5632 * page to lgroup where it is allocated already. 5633 * If so, clear the migrate bit and skip to next 5634 * page. 5635 */ 5636 if (i == 0 && to == from) { 5637 PP_CLRMIGRATE(ppa[0]); 5638 page_downgrade(ppa[0]); 5639 goto next; 5640 } 5641 } 5642 5643 /* 5644 * If all constituent pages couldn't be locked, 5645 * unlock pages locked so far and skip to next page. 5646 */ 5647 if (i != page_cnt) { 5648 while (--i != -1) { 5649 page_downgrade(ppa[i]); 5650 } 5651 goto next; 5652 } 5653 5654 (void) page_create_wait(page_cnt, PG_WAIT); 5655 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5656 if (newpp == NULL) { 5657 page_create_putback(page_cnt); 5658 for (i = 0; i < page_cnt; i++) { 5659 page_downgrade(ppa[i]); 5660 } 5661 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5662 page_cnt); 5663 goto next; 5664 } 5665 ASSERT(newpp->p_szc == pszc); 5666 /* 5667 * Clear migrate bit and relocate page 5668 */ 5669 PP_CLRMIGRATE(pp); 5670 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5671 panic("page_migrate: page_relocate failed"); 5672 } 5673 ASSERT(page_cnt * PAGESIZE == pgsz); 5674 5675 /* 5676 * Keep stats for number of pages migrated from and to 5677 * each lgroup 5678 */ 5679 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5680 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5681 /* 5682 * update the page_t array we were passed in and 5683 * unlink constituent pages of a large page. 5684 */ 5685 for (i = 0; i < page_cnt; ++i, ++pp) { 5686 ASSERT(PAGE_EXCL(newpp)); 5687 ASSERT(newpp->p_szc == pszc); 5688 ppa[i] = newpp; 5689 pp = newpp; 5690 page_sub(&newpp, pp); 5691 page_downgrade(pp); 5692 } 5693 ASSERT(newpp == NULL); 5694 next: 5695 addr += pgsz; 5696 ppa += page_cnt; 5697 npages -= page_cnt; 5698 } 5699 } 5700 5701 #define MAX_CNT 60 /* max num of iterations */ 5702 /* 5703 * Reclaim/reserve availrmem for npages. 5704 * If there is not enough memory start reaping seg, kmem caches. 5705 * Start pageout scanner (via page_needfree()). 5706 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5707 * Note: There is no guarantee that any availrmem will be freed as 5708 * this memory typically is locked (kernel heap) or reserved for swap. 5709 * Also due to memory fragmentation kmem allocator may not be able 5710 * to free any memory (single user allocated buffer will prevent 5711 * freeing slab or a page). 5712 */ 5713 int 5714 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5715 { 5716 int i = 0; 5717 int ret = 0; 5718 pgcnt_t deficit; 5719 pgcnt_t old_availrmem; 5720 5721 mutex_enter(&freemem_lock); 5722 old_availrmem = availrmem - 1; 5723 while ((availrmem < tune.t_minarmem + npages + epages) && 5724 (old_availrmem < availrmem) && (i++ < MAX_CNT)) { 5725 old_availrmem = availrmem; 5726 deficit = tune.t_minarmem + npages + epages - availrmem; 5727 mutex_exit(&freemem_lock); 5728 page_needfree(deficit); 5729 kmem_reap(); 5730 delay(hz); 5731 page_needfree(-(spgcnt_t)deficit); 5732 mutex_enter(&freemem_lock); 5733 } 5734 5735 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5736 availrmem -= npages; 5737 ret = 1; 5738 } 5739 5740 mutex_exit(&freemem_lock); 5741 5742 return (ret); 5743 } 5744 5745 /* 5746 * Search the memory segments to locate the desired page. Within a 5747 * segment, pages increase linearly with one page structure per 5748 * physical page frame (size PAGESIZE). The search begins 5749 * with the segment that was accessed last, to take advantage of locality. 5750 * If the hint misses, we start from the beginning of the sorted memseg list 5751 */ 5752 5753 5754 /* 5755 * Some data structures for pfn to pp lookup. 5756 */ 5757 ulong_t mhash_per_slot; 5758 struct memseg *memseg_hash[N_MEM_SLOTS]; 5759 5760 page_t * 5761 page_numtopp_nolock(pfn_t pfnum) 5762 { 5763 struct memseg *seg; 5764 page_t *pp; 5765 vm_cpu_data_t *vc; 5766 5767 /* 5768 * We need to disable kernel preemption while referencing the 5769 * cpu_vm_data field in order to prevent us from being switched to 5770 * another cpu and trying to reference it after it has been freed. 5771 * This will keep us on cpu and prevent it from being removed while 5772 * we are still on it. 5773 * 5774 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5775 * which is being resued by DR who will flush those references 5776 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5777 */ 5778 kpreempt_disable(); 5779 vc = CPU->cpu_vm_data; 5780 ASSERT(vc != NULL); 5781 5782 MEMSEG_STAT_INCR(nsearch); 5783 5784 /* Try last winner first */ 5785 if (((seg = vc->vc_pnum_memseg) != NULL) && 5786 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5787 MEMSEG_STAT_INCR(nlastwon); 5788 pp = seg->pages + (pfnum - seg->pages_base); 5789 if (pp->p_pagenum == pfnum) { 5790 kpreempt_enable(); 5791 return ((page_t *)pp); 5792 } 5793 } 5794 5795 /* Else Try hash */ 5796 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5797 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5798 MEMSEG_STAT_INCR(nhashwon); 5799 vc->vc_pnum_memseg = seg; 5800 pp = seg->pages + (pfnum - seg->pages_base); 5801 if (pp->p_pagenum == pfnum) { 5802 kpreempt_enable(); 5803 return ((page_t *)pp); 5804 } 5805 } 5806 5807 /* Else Brute force */ 5808 for (seg = memsegs; seg != NULL; seg = seg->next) { 5809 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5810 vc->vc_pnum_memseg = seg; 5811 pp = seg->pages + (pfnum - seg->pages_base); 5812 if (pp->p_pagenum == pfnum) { 5813 kpreempt_enable(); 5814 return ((page_t *)pp); 5815 } 5816 } 5817 } 5818 vc->vc_pnum_memseg = NULL; 5819 kpreempt_enable(); 5820 MEMSEG_STAT_INCR(nnotfound); 5821 return ((page_t *)NULL); 5822 5823 } 5824 5825 struct memseg * 5826 page_numtomemseg_nolock(pfn_t pfnum) 5827 { 5828 struct memseg *seg; 5829 page_t *pp; 5830 5831 /* 5832 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5833 * which is being resued by DR who will flush those references 5834 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5835 */ 5836 kpreempt_disable(); 5837 /* Try hash */ 5838 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5839 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5840 pp = seg->pages + (pfnum - seg->pages_base); 5841 if (pp->p_pagenum == pfnum) { 5842 kpreempt_enable(); 5843 return (seg); 5844 } 5845 } 5846 5847 /* Else Brute force */ 5848 for (seg = memsegs; seg != NULL; seg = seg->next) { 5849 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5850 pp = seg->pages + (pfnum - seg->pages_base); 5851 if (pp->p_pagenum == pfnum) { 5852 kpreempt_enable(); 5853 return (seg); 5854 } 5855 } 5856 } 5857 kpreempt_enable(); 5858 return ((struct memseg *)NULL); 5859 } 5860 5861 /* 5862 * Given a page and a count return the page struct that is 5863 * n structs away from the current one in the global page 5864 * list. 5865 * 5866 * This function wraps to the first page upon 5867 * reaching the end of the memseg list. 5868 */ 5869 page_t * 5870 page_nextn(page_t *pp, ulong_t n) 5871 { 5872 struct memseg *seg; 5873 page_t *ppn; 5874 vm_cpu_data_t *vc; 5875 5876 /* 5877 * We need to disable kernel preemption while referencing the 5878 * cpu_vm_data field in order to prevent us from being switched to 5879 * another cpu and trying to reference it after it has been freed. 5880 * This will keep us on cpu and prevent it from being removed while 5881 * we are still on it. 5882 * 5883 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5884 * which is being resued by DR who will flush those references 5885 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5886 */ 5887 kpreempt_disable(); 5888 vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5889 5890 ASSERT(vc != NULL); 5891 5892 if (((seg = vc->vc_pnext_memseg) == NULL) || 5893 (seg->pages_base == seg->pages_end) || 5894 !(pp >= seg->pages && pp < seg->epages)) { 5895 5896 for (seg = memsegs; seg; seg = seg->next) { 5897 if (pp >= seg->pages && pp < seg->epages) 5898 break; 5899 } 5900 5901 if (seg == NULL) { 5902 /* Memory delete got in, return something valid. */ 5903 /* TODO: fix me. */ 5904 seg = memsegs; 5905 pp = seg->pages; 5906 } 5907 } 5908 5909 /* check for wraparound - possible if n is large */ 5910 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5911 n -= seg->epages - pp; 5912 seg = seg->next; 5913 if (seg == NULL) 5914 seg = memsegs; 5915 pp = seg->pages; 5916 } 5917 vc->vc_pnext_memseg = seg; 5918 kpreempt_enable(); 5919 return (ppn); 5920 } 5921 5922 /* 5923 * Initialize for a loop using page_next_scan_large(). 5924 */ 5925 page_t * 5926 page_next_scan_init(void **cookie) 5927 { 5928 ASSERT(cookie != NULL); 5929 *cookie = (void *)memsegs; 5930 return ((page_t *)memsegs->pages); 5931 } 5932 5933 /* 5934 * Return the next page in a scan of page_t's, assuming we want 5935 * to skip over sub-pages within larger page sizes. 5936 * 5937 * The cookie is used to keep track of the current memseg. 5938 */ 5939 page_t * 5940 page_next_scan_large( 5941 page_t *pp, 5942 ulong_t *n, 5943 void **cookie) 5944 { 5945 struct memseg *seg = (struct memseg *)*cookie; 5946 page_t *new_pp; 5947 ulong_t cnt; 5948 pfn_t pfn; 5949 5950 5951 /* 5952 * get the count of page_t's to skip based on the page size 5953 */ 5954 ASSERT(pp != NULL); 5955 if (pp->p_szc == 0) { 5956 cnt = 1; 5957 } else { 5958 pfn = page_pptonum(pp); 5959 cnt = page_get_pagecnt(pp->p_szc); 5960 cnt -= pfn & (cnt - 1); 5961 } 5962 *n += cnt; 5963 new_pp = pp + cnt; 5964 5965 /* 5966 * Catch if we went past the end of the current memory segment. If so, 5967 * just move to the next segment with pages. 5968 */ 5969 if (new_pp >= seg->epages || seg->pages_base == seg->pages_end) { 5970 do { 5971 seg = seg->next; 5972 if (seg == NULL) 5973 seg = memsegs; 5974 } while (seg->pages_base == seg->pages_end); 5975 new_pp = seg->pages; 5976 *cookie = (void *)seg; 5977 } 5978 5979 return (new_pp); 5980 } 5981 5982 5983 /* 5984 * Returns next page in list. Note: this function wraps 5985 * to the first page in the list upon reaching the end 5986 * of the list. Callers should be aware of this fact. 5987 */ 5988 5989 /* We should change this be a #define */ 5990 5991 page_t * 5992 page_next(page_t *pp) 5993 { 5994 return (page_nextn(pp, 1)); 5995 } 5996 5997 page_t * 5998 page_first() 5999 { 6000 return ((page_t *)memsegs->pages); 6001 } 6002 6003 6004 /* 6005 * This routine is called at boot with the initial memory configuration 6006 * and when memory is added or removed. 6007 */ 6008 void 6009 build_pfn_hash() 6010 { 6011 pfn_t cur; 6012 pgcnt_t index; 6013 struct memseg *pseg; 6014 int i; 6015 6016 /* 6017 * Clear memseg_hash array. 6018 * Since memory add/delete is designed to operate concurrently 6019 * with normal operation, the hash rebuild must be able to run 6020 * concurrently with page_numtopp_nolock(). To support this 6021 * functionality, assignments to memseg_hash array members must 6022 * be done atomically. 6023 * 6024 * NOTE: bzero() does not currently guarantee this for kernel 6025 * threads, and cannot be used here. 6026 */ 6027 for (i = 0; i < N_MEM_SLOTS; i++) 6028 memseg_hash[i] = NULL; 6029 6030 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6031 6032 /* 6033 * Physmax is the last valid pfn. 6034 */ 6035 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6036 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6037 index = MEMSEG_PFN_HASH(pseg->pages_base); 6038 cur = pseg->pages_base; 6039 do { 6040 if (index >= N_MEM_SLOTS) 6041 index = MEMSEG_PFN_HASH(cur); 6042 6043 if (memseg_hash[index] == NULL || 6044 memseg_hash[index]->pages_base > pseg->pages_base) { 6045 memseg_hash[index] = pseg; 6046 hat_kpm_mseghash_update(index, pseg); 6047 } 6048 cur += mhash_per_slot; 6049 index++; 6050 } while (cur < pseg->pages_end); 6051 } 6052 } 6053 6054 /* 6055 * Return the pagenum for the pp 6056 */ 6057 pfn_t 6058 page_pptonum(page_t *pp) 6059 { 6060 return (pp->p_pagenum); 6061 } 6062 6063 /* 6064 * interface to the referenced and modified etc bits 6065 * in the PSM part of the page struct 6066 * when no locking is desired. 6067 */ 6068 void 6069 page_set_props(page_t *pp, uint_t flags) 6070 { 6071 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6072 pp->p_nrm |= (uchar_t)flags; 6073 } 6074 6075 void 6076 page_clr_all_props(page_t *pp) 6077 { 6078 pp->p_nrm = 0; 6079 } 6080 6081 /* 6082 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6083 */ 6084 int 6085 page_clear_lck_cow(page_t *pp, int adjust) 6086 { 6087 int f_amount; 6088 6089 ASSERT(PAGE_EXCL(pp)); 6090 6091 /* 6092 * The page_struct_lock need not be acquired here since 6093 * we require the caller hold the page exclusively locked. 6094 */ 6095 f_amount = 0; 6096 if (pp->p_lckcnt) { 6097 f_amount = 1; 6098 pp->p_lckcnt = 0; 6099 } 6100 if (pp->p_cowcnt) { 6101 f_amount += pp->p_cowcnt; 6102 pp->p_cowcnt = 0; 6103 } 6104 6105 if (adjust && f_amount) { 6106 mutex_enter(&freemem_lock); 6107 availrmem += f_amount; 6108 mutex_exit(&freemem_lock); 6109 } 6110 6111 return (f_amount); 6112 } 6113 6114 /* 6115 * The following functions is called from free_vp_pages() 6116 * for an inexact estimate of a newly free'd page... 6117 */ 6118 ulong_t 6119 page_share_cnt(page_t *pp) 6120 { 6121 return (hat_page_getshare(pp)); 6122 } 6123 6124 int 6125 page_isshared(page_t *pp) 6126 { 6127 return (hat_page_checkshare(pp, 1)); 6128 } 6129 6130 int 6131 page_isfree(page_t *pp) 6132 { 6133 return (PP_ISFREE(pp)); 6134 } 6135 6136 int 6137 page_isref(page_t *pp) 6138 { 6139 return (hat_page_getattr(pp, P_REF)); 6140 } 6141 6142 int 6143 page_ismod(page_t *pp) 6144 { 6145 return (hat_page_getattr(pp, P_MOD)); 6146 } 6147 6148 /* 6149 * The following code all currently relates to the page capture logic: 6150 * 6151 * This logic is used for cases where there is a desire to claim a certain 6152 * physical page in the system for the caller. As it may not be possible 6153 * to capture the page immediately, the p_toxic bits are used in the page 6154 * structure to indicate that someone wants to capture this page. When the 6155 * page gets unlocked, the toxic flag will be noted and an attempt to capture 6156 * the page will be made. If it is successful, the original callers callback 6157 * will be called with the page to do with it what they please. 6158 * 6159 * There is also an async thread which wakes up to attempt to capture 6160 * pages occasionally which have the capture bit set. All of the pages which 6161 * need to be captured asynchronously have been inserted into the 6162 * page_capture_hash and thus this thread walks that hash list. Items in the 6163 * hash have an expiration time so this thread handles that as well by removing 6164 * the item from the hash if it has expired. 6165 * 6166 * Some important things to note are: 6167 * - if the PR_CAPTURE bit is set on a page, then the page is in the 6168 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed 6169 * to set and clear this bit, and while the lock is held is the only time 6170 * you can add or remove an entry from the hash. 6171 * - the PR_CAPTURE bit can only be set and cleared while holding the 6172 * page_capture_hash_head.pchh_mutex 6173 * - the t_flag field of the thread struct is used with the T_CAPTURING 6174 * flag to prevent recursion while dealing with large pages. 6175 * - pages which need to be retired never expire on the page_capture_hash. 6176 */ 6177 6178 static void page_capture_thread(void); 6179 static kthread_t *pc_thread_id; 6180 kcondvar_t pc_cv; 6181 static kmutex_t pc_thread_mutex; 6182 static clock_t pc_thread_shortwait; 6183 static clock_t pc_thread_longwait; 6184 static int pc_thread_retry; 6185 6186 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS]; 6187 6188 /* Note that this is a circular linked list */ 6189 typedef struct page_capture_hash_bucket { 6190 page_t *pp; 6191 uchar_t szc; 6192 uchar_t pri; 6193 uint_t flags; 6194 clock_t expires; /* lbolt at which this request expires. */ 6195 void *datap; /* Cached data passed in for callback */ 6196 struct page_capture_hash_bucket *next; 6197 struct page_capture_hash_bucket *prev; 6198 } page_capture_hash_bucket_t; 6199 6200 #define PC_PRI_HI 0 /* capture now */ 6201 #define PC_PRI_LO 1 /* capture later */ 6202 #define PC_NUM_PRI 2 6203 6204 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI) 6205 6206 6207 /* 6208 * Each hash bucket will have it's own mutex and two lists which are: 6209 * active (0): represents requests which have not been processed by 6210 * the page_capture async thread yet. 6211 * walked (1): represents requests which have been processed by the 6212 * page_capture async thread within it's given walk of this bucket. 6213 * 6214 * These are all needed so that we can synchronize all async page_capture 6215 * events. When the async thread moves to a new bucket, it will append the 6216 * walked list to the active list and walk each item one at a time, moving it 6217 * from the active list to the walked list. Thus if there is an async request 6218 * outstanding for a given page, it will always be in one of the two lists. 6219 * New requests will always be added to the active list. 6220 * If we were not able to capture a page before the request expired, we'd free 6221 * up the request structure which would indicate to page_capture that there is 6222 * no longer a need for the given page, and clear the PR_CAPTURE flag if 6223 * possible. 6224 */ 6225 typedef struct page_capture_hash_head { 6226 kmutex_t pchh_mutex; 6227 uint_t num_pages[PC_NUM_PRI]; 6228 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */ 6229 } page_capture_hash_head_t; 6230 6231 #ifdef DEBUG 6232 #define NUM_PAGE_CAPTURE_BUCKETS 4 6233 #else 6234 #define NUM_PAGE_CAPTURE_BUCKETS 64 6235 #endif 6236 6237 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS]; 6238 6239 /* for now use a very simple hash based upon the size of a page struct */ 6240 #define PAGE_CAPTURE_HASH(pp) \ 6241 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1))) 6242 6243 extern pgcnt_t swapfs_minfree; 6244 6245 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap); 6246 6247 /* 6248 * a callback function is required for page capture requests. 6249 */ 6250 void 6251 page_capture_register_callback(uint_t index, clock_t duration, 6252 int (*cb_func)(page_t *, void *, uint_t)) 6253 { 6254 ASSERT(pc_cb[index].cb_active == 0); 6255 ASSERT(cb_func != NULL); 6256 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6257 pc_cb[index].duration = duration; 6258 pc_cb[index].cb_func = cb_func; 6259 pc_cb[index].cb_active = 1; 6260 rw_exit(&pc_cb[index].cb_rwlock); 6261 } 6262 6263 void 6264 page_capture_unregister_callback(uint_t index) 6265 { 6266 int i, j; 6267 struct page_capture_hash_bucket *bp1; 6268 struct page_capture_hash_bucket *bp2; 6269 struct page_capture_hash_bucket *head = NULL; 6270 uint_t flags = (1 << index); 6271 6272 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6273 ASSERT(pc_cb[index].cb_active == 1); 6274 pc_cb[index].duration = 0; /* Paranoia */ 6275 pc_cb[index].cb_func = NULL; /* Paranoia */ 6276 pc_cb[index].cb_active = 0; 6277 rw_exit(&pc_cb[index].cb_rwlock); 6278 6279 /* 6280 * Just move all the entries to a private list which we can walk 6281 * through without the need to hold any locks. 6282 * No more requests can get added to the hash lists for this consumer 6283 * as the cb_active field for the callback has been cleared. 6284 */ 6285 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 6286 mutex_enter(&page_capture_hash[i].pchh_mutex); 6287 for (j = 0; j < 2; j++) { 6288 bp1 = page_capture_hash[i].lists[j].next; 6289 /* walk through all but first (sentinel) element */ 6290 while (bp1 != &page_capture_hash[i].lists[j]) { 6291 bp2 = bp1; 6292 if (bp2->flags & flags) { 6293 bp1 = bp2->next; 6294 bp1->prev = bp2->prev; 6295 bp2->prev->next = bp1; 6296 bp2->next = head; 6297 head = bp2; 6298 /* 6299 * Clear the PR_CAPTURE bit as we 6300 * hold appropriate locks here. 6301 */ 6302 page_clrtoxic(head->pp, PR_CAPTURE); 6303 page_capture_hash[i]. 6304 num_pages[bp2->pri]--; 6305 continue; 6306 } 6307 bp1 = bp1->next; 6308 } 6309 } 6310 mutex_exit(&page_capture_hash[i].pchh_mutex); 6311 } 6312 6313 while (head != NULL) { 6314 bp1 = head; 6315 head = head->next; 6316 kmem_free(bp1, sizeof (*bp1)); 6317 } 6318 } 6319 6320 6321 /* 6322 * Find pp in the active list and move it to the walked list if it 6323 * exists. 6324 * Note that most often pp should be at the front of the active list 6325 * as it is currently used and thus there is no other sort of optimization 6326 * being done here as this is a linked list data structure. 6327 * Returns 1 on successful move or 0 if page could not be found. 6328 */ 6329 static int 6330 page_capture_move_to_walked(page_t *pp) 6331 { 6332 page_capture_hash_bucket_t *bp; 6333 int index; 6334 6335 index = PAGE_CAPTURE_HASH(pp); 6336 6337 mutex_enter(&page_capture_hash[index].pchh_mutex); 6338 bp = page_capture_hash[index].lists[0].next; 6339 while (bp != &page_capture_hash[index].lists[0]) { 6340 if (bp->pp == pp) { 6341 /* Remove from old list */ 6342 bp->next->prev = bp->prev; 6343 bp->prev->next = bp->next; 6344 6345 /* Add to new list */ 6346 bp->next = page_capture_hash[index].lists[1].next; 6347 bp->prev = &page_capture_hash[index].lists[1]; 6348 page_capture_hash[index].lists[1].next = bp; 6349 bp->next->prev = bp; 6350 6351 /* 6352 * There is a small probability of page on a free 6353 * list being retired while being allocated 6354 * and before P_RAF is set on it. The page may 6355 * end up marked as high priority request instead 6356 * of low priority request. 6357 * If P_RAF page is not marked as low priority request 6358 * change it to low priority request. 6359 */ 6360 page_capture_hash[index].num_pages[bp->pri]--; 6361 bp->pri = PAGE_CAPTURE_PRIO(pp); 6362 page_capture_hash[index].num_pages[bp->pri]++; 6363 mutex_exit(&page_capture_hash[index].pchh_mutex); 6364 return (1); 6365 } 6366 bp = bp->next; 6367 } 6368 mutex_exit(&page_capture_hash[index].pchh_mutex); 6369 return (0); 6370 } 6371 6372 /* 6373 * Add a new entry to the page capture hash. The only case where a new 6374 * entry is not added is when the page capture consumer is no longer registered. 6375 * In this case, we'll silently not add the page to the hash. We know that 6376 * page retire will always be registered for the case where we are currently 6377 * unretiring a page and thus there are no conflicts. 6378 */ 6379 static void 6380 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) 6381 { 6382 page_capture_hash_bucket_t *bp1; 6383 page_capture_hash_bucket_t *bp2; 6384 int index; 6385 int cb_index; 6386 int i; 6387 uchar_t pri; 6388 #ifdef DEBUG 6389 page_capture_hash_bucket_t *tp1; 6390 int l; 6391 #endif 6392 6393 ASSERT(!(flags & CAPTURE_ASYNC)); 6394 6395 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP); 6396 6397 bp1->pp = pp; 6398 bp1->szc = szc; 6399 bp1->flags = flags; 6400 bp1->datap = datap; 6401 6402 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6403 if ((flags >> cb_index) & 1) { 6404 break; 6405 } 6406 } 6407 6408 ASSERT(cb_index != PC_NUM_CALLBACKS); 6409 6410 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6411 if (pc_cb[cb_index].cb_active) { 6412 if (pc_cb[cb_index].duration == -1) { 6413 bp1->expires = (clock_t)-1; 6414 } else { 6415 bp1->expires = ddi_get_lbolt() + 6416 pc_cb[cb_index].duration; 6417 } 6418 } else { 6419 /* There's no callback registered so don't add to the hash */ 6420 rw_exit(&pc_cb[cb_index].cb_rwlock); 6421 kmem_free(bp1, sizeof (*bp1)); 6422 return; 6423 } 6424 6425 index = PAGE_CAPTURE_HASH(pp); 6426 6427 /* 6428 * Only allow capture flag to be modified under this mutex. 6429 * Prevents multiple entries for same page getting added. 6430 */ 6431 mutex_enter(&page_capture_hash[index].pchh_mutex); 6432 6433 /* 6434 * if not already on the hash, set capture bit and add to the hash 6435 */ 6436 if (!(pp->p_toxic & PR_CAPTURE)) { 6437 #ifdef DEBUG 6438 /* Check for duplicate entries */ 6439 for (l = 0; l < 2; l++) { 6440 tp1 = page_capture_hash[index].lists[l].next; 6441 while (tp1 != &page_capture_hash[index].lists[l]) { 6442 if (tp1->pp == pp) { 6443 panic("page pp 0x%p already on hash " 6444 "at 0x%p\n", 6445 (void *)pp, (void *)tp1); 6446 } 6447 tp1 = tp1->next; 6448 } 6449 } 6450 6451 #endif 6452 page_settoxic(pp, PR_CAPTURE); 6453 pri = PAGE_CAPTURE_PRIO(pp); 6454 bp1->pri = pri; 6455 bp1->next = page_capture_hash[index].lists[0].next; 6456 bp1->prev = &page_capture_hash[index].lists[0]; 6457 bp1->next->prev = bp1; 6458 page_capture_hash[index].lists[0].next = bp1; 6459 page_capture_hash[index].num_pages[pri]++; 6460 if (flags & CAPTURE_RETIRE) { 6461 page_retire_incr_pend_count(datap); 6462 } 6463 mutex_exit(&page_capture_hash[index].pchh_mutex); 6464 rw_exit(&pc_cb[cb_index].cb_rwlock); 6465 cv_signal(&pc_cv); 6466 return; 6467 } 6468 6469 /* 6470 * A page retire request will replace any other request. 6471 * A second physmem request which is for a different process than 6472 * the currently registered one will be dropped as there is 6473 * no way to hold the private data for both calls. 6474 * In the future, once there are more callers, this will have to 6475 * be worked out better as there needs to be private storage for 6476 * at least each type of caller (maybe have datap be an array of 6477 * *void's so that we can index based upon callers index). 6478 */ 6479 6480 /* walk hash list to update expire time */ 6481 for (i = 0; i < 2; i++) { 6482 bp2 = page_capture_hash[index].lists[i].next; 6483 while (bp2 != &page_capture_hash[index].lists[i]) { 6484 if (bp2->pp == pp) { 6485 if (flags & CAPTURE_RETIRE) { 6486 if (!(bp2->flags & CAPTURE_RETIRE)) { 6487 page_retire_incr_pend_count( 6488 datap); 6489 bp2->flags = flags; 6490 bp2->expires = bp1->expires; 6491 bp2->datap = datap; 6492 } 6493 } else { 6494 ASSERT(flags & CAPTURE_PHYSMEM); 6495 if (!(bp2->flags & CAPTURE_RETIRE) && 6496 (datap == bp2->datap)) { 6497 bp2->expires = bp1->expires; 6498 } 6499 } 6500 mutex_exit(&page_capture_hash[index]. 6501 pchh_mutex); 6502 rw_exit(&pc_cb[cb_index].cb_rwlock); 6503 kmem_free(bp1, sizeof (*bp1)); 6504 return; 6505 } 6506 bp2 = bp2->next; 6507 } 6508 } 6509 6510 /* 6511 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes 6512 * and thus it either has to be set or not set and can't change 6513 * while holding the mutex above. 6514 */ 6515 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", 6516 (void *)pp); 6517 } 6518 6519 /* 6520 * We have a page in our hands, lets try and make it ours by turning 6521 * it into a clean page like it had just come off the freelists. 6522 * 6523 * Returns 0 on success, with the page still EXCL locked. 6524 * On failure, the page will be unlocked, and returns EAGAIN 6525 */ 6526 static int 6527 page_capture_clean_page(page_t *pp) 6528 { 6529 page_t *newpp; 6530 int skip_unlock = 0; 6531 spgcnt_t count; 6532 page_t *tpp; 6533 int ret = 0; 6534 int extra; 6535 6536 ASSERT(PAGE_EXCL(pp)); 6537 ASSERT(!PP_RETIRED(pp)); 6538 ASSERT(curthread->t_flag & T_CAPTURING); 6539 6540 if (PP_ISFREE(pp)) { 6541 if (!page_reclaim(pp, NULL)) { 6542 skip_unlock = 1; 6543 ret = EAGAIN; 6544 goto cleanup; 6545 } 6546 ASSERT(pp->p_szc == 0); 6547 if (pp->p_vnode != NULL) { 6548 /* 6549 * Since this page came from the 6550 * cachelist, we must destroy the 6551 * old vnode association. 6552 */ 6553 page_hashout(pp, NULL); 6554 } 6555 goto cleanup; 6556 } 6557 6558 /* 6559 * If we know page_relocate will fail, skip it 6560 * It could still fail due to a UE on another page but we 6561 * can't do anything about that. 6562 */ 6563 if (pp->p_toxic & PR_UE) { 6564 goto skip_relocate; 6565 } 6566 6567 /* 6568 * It's possible that pages can not have a vnode as fsflush comes 6569 * through and cleans up these pages. It's ugly but that's how it is. 6570 */ 6571 if (pp->p_vnode == NULL) { 6572 goto skip_relocate; 6573 } 6574 6575 /* 6576 * Page was not free, so lets try to relocate it. 6577 * page_relocate only works with root pages, so if this is not a root 6578 * page, we need to demote it to try and relocate it. 6579 * Unfortunately this is the best we can do right now. 6580 */ 6581 newpp = NULL; 6582 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { 6583 if (page_try_demote_pages(pp) == 0) { 6584 ret = EAGAIN; 6585 goto cleanup; 6586 } 6587 } 6588 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); 6589 if (ret == 0) { 6590 page_t *npp; 6591 /* unlock the new page(s) */ 6592 while (count-- > 0) { 6593 ASSERT(newpp != NULL); 6594 npp = newpp; 6595 page_sub(&newpp, npp); 6596 page_unlock(npp); 6597 } 6598 ASSERT(newpp == NULL); 6599 /* 6600 * Check to see if the page we have is too large. 6601 * If so, demote it freeing up the extra pages. 6602 */ 6603 if (pp->p_szc > 0) { 6604 /* For now demote extra pages to szc == 0 */ 6605 extra = page_get_pagecnt(pp->p_szc) - 1; 6606 while (extra > 0) { 6607 tpp = pp->p_next; 6608 page_sub(&pp, tpp); 6609 tpp->p_szc = 0; 6610 page_free(tpp, 1); 6611 extra--; 6612 } 6613 /* Make sure to set our page to szc 0 as well */ 6614 ASSERT(pp->p_next == pp && pp->p_prev == pp); 6615 pp->p_szc = 0; 6616 } 6617 goto cleanup; 6618 } else if (ret == EIO) { 6619 ret = EAGAIN; 6620 goto cleanup; 6621 } else { 6622 /* 6623 * Need to reset return type as we failed to relocate the page 6624 * but that does not mean that some of the next steps will not 6625 * work. 6626 */ 6627 ret = 0; 6628 } 6629 6630 skip_relocate: 6631 6632 if (pp->p_szc > 0) { 6633 if (page_try_demote_pages(pp) == 0) { 6634 ret = EAGAIN; 6635 goto cleanup; 6636 } 6637 } 6638 6639 ASSERT(pp->p_szc == 0); 6640 6641 if (hat_ismod(pp)) { 6642 ret = EAGAIN; 6643 goto cleanup; 6644 } 6645 if (PP_ISKAS(pp)) { 6646 ret = EAGAIN; 6647 goto cleanup; 6648 } 6649 if (pp->p_lckcnt || pp->p_cowcnt) { 6650 ret = EAGAIN; 6651 goto cleanup; 6652 } 6653 6654 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 6655 ASSERT(!hat_page_is_mapped(pp)); 6656 6657 if (hat_ismod(pp)) { 6658 /* 6659 * This is a semi-odd case as the page is now modified but not 6660 * mapped as we just unloaded the mappings above. 6661 */ 6662 ret = EAGAIN; 6663 goto cleanup; 6664 } 6665 if (pp->p_vnode != NULL) { 6666 page_hashout(pp, NULL); 6667 } 6668 6669 /* 6670 * At this point, the page should be in a clean state and 6671 * we can do whatever we want with it. 6672 */ 6673 6674 cleanup: 6675 if (ret != 0) { 6676 if (!skip_unlock) { 6677 page_unlock(pp); 6678 } 6679 } else { 6680 ASSERT(pp->p_szc == 0); 6681 ASSERT(PAGE_EXCL(pp)); 6682 6683 pp->p_next = pp; 6684 pp->p_prev = pp; 6685 } 6686 return (ret); 6687 } 6688 6689 /* 6690 * Various callers of page_trycapture() can have different restrictions upon 6691 * what memory they have access to. 6692 * Returns 0 on success, with the following error codes on failure: 6693 * EPERM - The requested page is long term locked, and thus repeated 6694 * requests to capture this page will likely fail. 6695 * ENOMEM - There was not enough free memory in the system to safely 6696 * map the requested page. 6697 * ENOENT - The requested page was inside the kernel cage, and the 6698 * PHYSMEM_CAGE flag was not set. 6699 */ 6700 int 6701 page_capture_pre_checks(page_t *pp, uint_t flags) 6702 { 6703 ASSERT(pp != NULL); 6704 6705 #if defined(__sparc) 6706 if (pp->p_vnode == &promvp) { 6707 return (EPERM); 6708 } 6709 6710 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) && 6711 (flags & CAPTURE_PHYSMEM)) { 6712 return (ENOENT); 6713 } 6714 6715 if (PP_ISNORELOCKERNEL(pp)) { 6716 return (EPERM); 6717 } 6718 #else 6719 if (PP_ISKAS(pp)) { 6720 return (EPERM); 6721 } 6722 #endif /* __sparc */ 6723 6724 /* only physmem currently has the restrictions checked below */ 6725 if (!(flags & CAPTURE_PHYSMEM)) { 6726 return (0); 6727 } 6728 6729 if (availrmem < swapfs_minfree) { 6730 /* 6731 * We won't try to capture this page as we are 6732 * running low on memory. 6733 */ 6734 return (ENOMEM); 6735 } 6736 return (0); 6737 } 6738 6739 /* 6740 * Once we have a page in our mits, go ahead and complete the capture 6741 * operation. 6742 * Returns 1 on failure where page is no longer needed 6743 * Returns 0 on success 6744 * Returns -1 if there was a transient failure. 6745 * Failure cases must release the SE_EXCL lock on pp (usually via page_free). 6746 */ 6747 int 6748 page_capture_take_action(page_t *pp, uint_t flags, void *datap) 6749 { 6750 int cb_index; 6751 int ret = 0; 6752 page_capture_hash_bucket_t *bp1; 6753 page_capture_hash_bucket_t *bp2; 6754 int index; 6755 int found = 0; 6756 int i; 6757 6758 ASSERT(PAGE_EXCL(pp)); 6759 ASSERT(curthread->t_flag & T_CAPTURING); 6760 6761 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6762 if ((flags >> cb_index) & 1) { 6763 break; 6764 } 6765 } 6766 ASSERT(cb_index < PC_NUM_CALLBACKS); 6767 6768 /* 6769 * Remove the entry from the page_capture hash, but don't free it yet 6770 * as we may need to put it back. 6771 * Since we own the page at this point in time, we should find it 6772 * in the hash if this is an ASYNC call. If we don't it's likely 6773 * that the page_capture_async() thread decided that this request 6774 * had expired, in which case we just continue on. 6775 */ 6776 if (flags & CAPTURE_ASYNC) { 6777 6778 index = PAGE_CAPTURE_HASH(pp); 6779 6780 mutex_enter(&page_capture_hash[index].pchh_mutex); 6781 for (i = 0; i < 2 && !found; i++) { 6782 bp1 = page_capture_hash[index].lists[i].next; 6783 while (bp1 != &page_capture_hash[index].lists[i]) { 6784 if (bp1->pp == pp) { 6785 bp1->next->prev = bp1->prev; 6786 bp1->prev->next = bp1->next; 6787 page_capture_hash[index]. 6788 num_pages[bp1->pri]--; 6789 page_clrtoxic(pp, PR_CAPTURE); 6790 found = 1; 6791 break; 6792 } 6793 bp1 = bp1->next; 6794 } 6795 } 6796 mutex_exit(&page_capture_hash[index].pchh_mutex); 6797 } 6798 6799 /* Synchronize with the unregister func. */ 6800 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6801 if (!pc_cb[cb_index].cb_active) { 6802 page_free(pp, 1); 6803 rw_exit(&pc_cb[cb_index].cb_rwlock); 6804 if (found) { 6805 kmem_free(bp1, sizeof (*bp1)); 6806 } 6807 return (1); 6808 } 6809 6810 /* 6811 * We need to remove the entry from the page capture hash and turn off 6812 * the PR_CAPTURE bit before calling the callback. We'll need to cache 6813 * the entry here, and then based upon the return value, cleanup 6814 * appropriately or re-add it to the hash, making sure that someone else 6815 * hasn't already done so. 6816 * It should be rare for the callback to fail and thus it's ok for 6817 * the failure path to be a bit complicated as the success path is 6818 * cleaner and the locking rules are easier to follow. 6819 */ 6820 6821 ret = pc_cb[cb_index].cb_func(pp, datap, flags); 6822 6823 rw_exit(&pc_cb[cb_index].cb_rwlock); 6824 6825 /* 6826 * If this was an ASYNC request, we need to cleanup the hash if the 6827 * callback was successful or if the request was no longer valid. 6828 * For non-ASYNC requests, we return failure to map and the caller 6829 * will take care of adding the request to the hash. 6830 * Note also that the callback itself is responsible for the page 6831 * at this point in time in terms of locking ... The most common 6832 * case for the failure path should just be a page_free. 6833 */ 6834 if (ret >= 0) { 6835 if (found) { 6836 if (bp1->flags & CAPTURE_RETIRE) { 6837 page_retire_decr_pend_count(datap); 6838 } 6839 kmem_free(bp1, sizeof (*bp1)); 6840 } 6841 return (ret); 6842 } 6843 if (!found) { 6844 return (ret); 6845 } 6846 6847 ASSERT(flags & CAPTURE_ASYNC); 6848 6849 /* 6850 * Check for expiration time first as we can just free it up if it's 6851 * expired. 6852 */ 6853 if (ddi_get_lbolt() > bp1->expires && bp1->expires != -1) { 6854 kmem_free(bp1, sizeof (*bp1)); 6855 return (ret); 6856 } 6857 6858 /* 6859 * The callback failed and there used to be an entry in the hash for 6860 * this page, so we need to add it back to the hash. 6861 */ 6862 mutex_enter(&page_capture_hash[index].pchh_mutex); 6863 if (!(pp->p_toxic & PR_CAPTURE)) { 6864 /* just add bp1 back to head of walked list */ 6865 page_settoxic(pp, PR_CAPTURE); 6866 bp1->next = page_capture_hash[index].lists[1].next; 6867 bp1->prev = &page_capture_hash[index].lists[1]; 6868 bp1->next->prev = bp1; 6869 bp1->pri = PAGE_CAPTURE_PRIO(pp); 6870 page_capture_hash[index].lists[1].next = bp1; 6871 page_capture_hash[index].num_pages[bp1->pri]++; 6872 mutex_exit(&page_capture_hash[index].pchh_mutex); 6873 return (ret); 6874 } 6875 6876 /* 6877 * Otherwise there was a new capture request added to list 6878 * Need to make sure that our original data is represented if 6879 * appropriate. 6880 */ 6881 for (i = 0; i < 2; i++) { 6882 bp2 = page_capture_hash[index].lists[i].next; 6883 while (bp2 != &page_capture_hash[index].lists[i]) { 6884 if (bp2->pp == pp) { 6885 if (bp1->flags & CAPTURE_RETIRE) { 6886 if (!(bp2->flags & CAPTURE_RETIRE)) { 6887 bp2->szc = bp1->szc; 6888 bp2->flags = bp1->flags; 6889 bp2->expires = bp1->expires; 6890 bp2->datap = bp1->datap; 6891 } 6892 } else { 6893 ASSERT(bp1->flags & CAPTURE_PHYSMEM); 6894 if (!(bp2->flags & CAPTURE_RETIRE)) { 6895 bp2->szc = bp1->szc; 6896 bp2->flags = bp1->flags; 6897 bp2->expires = bp1->expires; 6898 bp2->datap = bp1->datap; 6899 } 6900 } 6901 page_capture_hash[index].num_pages[bp2->pri]--; 6902 bp2->pri = PAGE_CAPTURE_PRIO(pp); 6903 page_capture_hash[index].num_pages[bp2->pri]++; 6904 mutex_exit(&page_capture_hash[index]. 6905 pchh_mutex); 6906 kmem_free(bp1, sizeof (*bp1)); 6907 return (ret); 6908 } 6909 bp2 = bp2->next; 6910 } 6911 } 6912 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp); 6913 /*NOTREACHED*/ 6914 } 6915 6916 /* 6917 * Try to capture the given page for the caller specified in the flags 6918 * parameter. The page will either be captured and handed over to the 6919 * appropriate callback, or will be queued up in the page capture hash 6920 * to be captured asynchronously. 6921 * If the current request is due to an async capture, the page must be 6922 * exclusively locked before calling this function. 6923 * Currently szc must be 0 but in the future this should be expandable to 6924 * other page sizes. 6925 * Returns 0 on success, with the following error codes on failure: 6926 * EPERM - The requested page is long term locked, and thus repeated 6927 * requests to capture this page will likely fail. 6928 * ENOMEM - There was not enough free memory in the system to safely 6929 * map the requested page. 6930 * ENOENT - The requested page was inside the kernel cage, and the 6931 * CAPTURE_GET_CAGE flag was not set. 6932 * EAGAIN - The requested page could not be capturead at this point in 6933 * time but future requests will likely work. 6934 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag 6935 * was not set. 6936 */ 6937 int 6938 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 6939 { 6940 int ret; 6941 int cb_index; 6942 6943 if (flags & CAPTURE_ASYNC) { 6944 ASSERT(PAGE_EXCL(pp)); 6945 goto async; 6946 } 6947 6948 /* Make sure there's enough availrmem ... */ 6949 ret = page_capture_pre_checks(pp, flags); 6950 if (ret != 0) { 6951 return (ret); 6952 } 6953 6954 if (!page_trylock(pp, SE_EXCL)) { 6955 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6956 if ((flags >> cb_index) & 1) { 6957 break; 6958 } 6959 } 6960 ASSERT(cb_index < PC_NUM_CALLBACKS); 6961 ret = EAGAIN; 6962 /* Special case for retired pages */ 6963 if (PP_RETIRED(pp)) { 6964 if (flags & CAPTURE_GET_RETIRED) { 6965 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { 6966 /* 6967 * Need to set capture bit and add to 6968 * hash so that the page will be 6969 * retired when freed. 6970 */ 6971 page_capture_add_hash(pp, szc, 6972 CAPTURE_RETIRE, NULL); 6973 ret = 0; 6974 goto own_page; 6975 } 6976 } else { 6977 return (EBUSY); 6978 } 6979 } 6980 page_capture_add_hash(pp, szc, flags, datap); 6981 return (ret); 6982 } 6983 6984 async: 6985 ASSERT(PAGE_EXCL(pp)); 6986 6987 /* Need to check for physmem async requests that availrmem is sane */ 6988 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) == 6989 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) && 6990 (availrmem < swapfs_minfree)) { 6991 page_unlock(pp); 6992 return (ENOMEM); 6993 } 6994 6995 ret = page_capture_clean_page(pp); 6996 6997 if (ret != 0) { 6998 /* We failed to get the page, so lets add it to the hash */ 6999 if (!(flags & CAPTURE_ASYNC)) { 7000 page_capture_add_hash(pp, szc, flags, datap); 7001 } 7002 return (ret); 7003 } 7004 7005 own_page: 7006 ASSERT(PAGE_EXCL(pp)); 7007 ASSERT(pp->p_szc == 0); 7008 7009 /* Call the callback */ 7010 ret = page_capture_take_action(pp, flags, datap); 7011 7012 if (ret == 0) { 7013 return (0); 7014 } 7015 7016 /* 7017 * Note that in the failure cases from page_capture_take_action, the 7018 * EXCL lock will have already been dropped. 7019 */ 7020 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) { 7021 page_capture_add_hash(pp, szc, flags, datap); 7022 } 7023 return (EAGAIN); 7024 } 7025 7026 int 7027 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 7028 { 7029 int ret; 7030 7031 curthread->t_flag |= T_CAPTURING; 7032 ret = page_itrycapture(pp, szc, flags, datap); 7033 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */ 7034 return (ret); 7035 } 7036 7037 /* 7038 * When unlocking a page which has the PR_CAPTURE bit set, this routine 7039 * gets called to try and capture the page. 7040 */ 7041 void 7042 page_unlock_capture(page_t *pp) 7043 { 7044 page_capture_hash_bucket_t *bp; 7045 int index; 7046 int i; 7047 uint_t szc; 7048 uint_t flags = 0; 7049 void *datap; 7050 kmutex_t *mp; 7051 extern vnode_t retired_pages; 7052 7053 /* 7054 * We need to protect against a possible deadlock here where we own 7055 * the vnode page hash mutex and want to acquire it again as there 7056 * are locations in the code, where we unlock a page while holding 7057 * the mutex which can lead to the page being captured and eventually 7058 * end up here. As we may be hashing out the old page and hashing into 7059 * the retire vnode, we need to make sure we don't own them. 7060 * Other callbacks who do hash operations also need to make sure that 7061 * before they hashin to a vnode that they do not currently own the 7062 * vphm mutex otherwise there will be a panic. 7063 */ 7064 if (mutex_owned(page_vnode_mutex(&retired_pages))) { 7065 page_unlock_nocapture(pp); 7066 return; 7067 } 7068 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { 7069 page_unlock_nocapture(pp); 7070 return; 7071 } 7072 7073 index = PAGE_CAPTURE_HASH(pp); 7074 7075 mp = &page_capture_hash[index].pchh_mutex; 7076 mutex_enter(mp); 7077 for (i = 0; i < 2; i++) { 7078 bp = page_capture_hash[index].lists[i].next; 7079 while (bp != &page_capture_hash[index].lists[i]) { 7080 if (bp->pp == pp) { 7081 szc = bp->szc; 7082 flags = bp->flags | CAPTURE_ASYNC; 7083 datap = bp->datap; 7084 mutex_exit(mp); 7085 (void) page_trycapture(pp, szc, flags, datap); 7086 return; 7087 } 7088 bp = bp->next; 7089 } 7090 } 7091 7092 /* Failed to find page in hash so clear flags and unlock it. */ 7093 page_clrtoxic(pp, PR_CAPTURE); 7094 page_unlock(pp); 7095 7096 mutex_exit(mp); 7097 } 7098 7099 void 7100 page_capture_init() 7101 { 7102 int i; 7103 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7104 page_capture_hash[i].lists[0].next = 7105 &page_capture_hash[i].lists[0]; 7106 page_capture_hash[i].lists[0].prev = 7107 &page_capture_hash[i].lists[0]; 7108 page_capture_hash[i].lists[1].next = 7109 &page_capture_hash[i].lists[1]; 7110 page_capture_hash[i].lists[1].prev = 7111 &page_capture_hash[i].lists[1]; 7112 } 7113 7114 pc_thread_shortwait = 23 * hz; 7115 pc_thread_longwait = 1201 * hz; 7116 pc_thread_retry = 3; 7117 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL); 7118 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL); 7119 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0, 7120 TS_RUN, minclsyspri); 7121 } 7122 7123 /* 7124 * It is necessary to scrub any failing pages prior to reboot in order to 7125 * prevent a latent error trap from occurring on the next boot. 7126 */ 7127 void 7128 page_retire_mdboot() 7129 { 7130 page_t *pp; 7131 int i, j; 7132 page_capture_hash_bucket_t *bp; 7133 uchar_t pri; 7134 7135 /* walk lists looking for pages to scrub */ 7136 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7137 for (pri = 0; pri < PC_NUM_PRI; pri++) { 7138 if (page_capture_hash[i].num_pages[pri] != 0) { 7139 break; 7140 } 7141 } 7142 if (pri == PC_NUM_PRI) 7143 continue; 7144 7145 mutex_enter(&page_capture_hash[i].pchh_mutex); 7146 7147 for (j = 0; j < 2; j++) { 7148 bp = page_capture_hash[i].lists[j].next; 7149 while (bp != &page_capture_hash[i].lists[j]) { 7150 pp = bp->pp; 7151 if (PP_TOXIC(pp)) { 7152 if (page_trylock(pp, SE_EXCL)) { 7153 PP_CLRFREE(pp); 7154 pagescrub(pp, 0, PAGESIZE); 7155 page_unlock(pp); 7156 } 7157 } 7158 bp = bp->next; 7159 } 7160 } 7161 mutex_exit(&page_capture_hash[i].pchh_mutex); 7162 } 7163 } 7164 7165 /* 7166 * Walk the page_capture_hash trying to capture pages and also cleanup old 7167 * entries which have expired. 7168 */ 7169 void 7170 page_capture_async() 7171 { 7172 page_t *pp; 7173 int i; 7174 int ret; 7175 page_capture_hash_bucket_t *bp1, *bp2; 7176 uint_t szc; 7177 uint_t flags; 7178 void *datap; 7179 uchar_t pri; 7180 7181 /* If there are outstanding pages to be captured, get to work */ 7182 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7183 for (pri = 0; pri < PC_NUM_PRI; pri++) { 7184 if (page_capture_hash[i].num_pages[pri] != 0) 7185 break; 7186 } 7187 if (pri == PC_NUM_PRI) 7188 continue; 7189 7190 /* Append list 1 to list 0 and then walk through list 0 */ 7191 mutex_enter(&page_capture_hash[i].pchh_mutex); 7192 bp1 = &page_capture_hash[i].lists[1]; 7193 bp2 = bp1->next; 7194 if (bp1 != bp2) { 7195 bp1->prev->next = page_capture_hash[i].lists[0].next; 7196 bp2->prev = &page_capture_hash[i].lists[0]; 7197 page_capture_hash[i].lists[0].next->prev = bp1->prev; 7198 page_capture_hash[i].lists[0].next = bp2; 7199 bp1->next = bp1; 7200 bp1->prev = bp1; 7201 } 7202 7203 /* list[1] will be empty now */ 7204 7205 bp1 = page_capture_hash[i].lists[0].next; 7206 while (bp1 != &page_capture_hash[i].lists[0]) { 7207 /* Check expiration time */ 7208 if ((ddi_get_lbolt() > bp1->expires && 7209 bp1->expires != -1) || 7210 page_deleted(bp1->pp)) { 7211 page_capture_hash[i].lists[0].next = bp1->next; 7212 bp1->next->prev = 7213 &page_capture_hash[i].lists[0]; 7214 page_capture_hash[i].num_pages[bp1->pri]--; 7215 7216 /* 7217 * We can safely remove the PR_CAPTURE bit 7218 * without holding the EXCL lock on the page 7219 * as the PR_CAPTURE bit requres that the 7220 * page_capture_hash[].pchh_mutex be held 7221 * to modify it. 7222 */ 7223 page_clrtoxic(bp1->pp, PR_CAPTURE); 7224 mutex_exit(&page_capture_hash[i].pchh_mutex); 7225 kmem_free(bp1, sizeof (*bp1)); 7226 mutex_enter(&page_capture_hash[i].pchh_mutex); 7227 bp1 = page_capture_hash[i].lists[0].next; 7228 continue; 7229 } 7230 pp = bp1->pp; 7231 szc = bp1->szc; 7232 flags = bp1->flags; 7233 datap = bp1->datap; 7234 mutex_exit(&page_capture_hash[i].pchh_mutex); 7235 if (page_trylock(pp, SE_EXCL)) { 7236 ret = page_trycapture(pp, szc, 7237 flags | CAPTURE_ASYNC, datap); 7238 } else { 7239 ret = 1; /* move to walked hash */ 7240 } 7241 7242 if (ret != 0) { 7243 /* Move to walked hash */ 7244 (void) page_capture_move_to_walked(pp); 7245 } 7246 mutex_enter(&page_capture_hash[i].pchh_mutex); 7247 bp1 = page_capture_hash[i].lists[0].next; 7248 } 7249 7250 mutex_exit(&page_capture_hash[i].pchh_mutex); 7251 } 7252 } 7253 7254 /* 7255 * This function is called by the page_capture_thread, and is needed in 7256 * in order to initiate aio cleanup, so that pages used in aio 7257 * will be unlocked and subsequently retired by page_capture_thread. 7258 */ 7259 static int 7260 do_aio_cleanup(void) 7261 { 7262 proc_t *procp; 7263 int (*aio_cleanup_dr_delete_memory)(proc_t *); 7264 int cleaned = 0; 7265 7266 if (modload("sys", "kaio") == -1) { 7267 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio"); 7268 return (0); 7269 } 7270 /* 7271 * We use the aio_cleanup_dr_delete_memory function to 7272 * initiate the actual clean up; this function will wake 7273 * up the per-process aio_cleanup_thread. 7274 */ 7275 aio_cleanup_dr_delete_memory = (int (*)(proc_t *)) 7276 modgetsymvalue("aio_cleanup_dr_delete_memory", 0); 7277 if (aio_cleanup_dr_delete_memory == NULL) { 7278 cmn_err(CE_WARN, 7279 "aio_cleanup_dr_delete_memory not found in kaio"); 7280 return (0); 7281 } 7282 mutex_enter(&pidlock); 7283 for (procp = practive; (procp != NULL); procp = procp->p_next) { 7284 mutex_enter(&procp->p_lock); 7285 if (procp->p_aio != NULL) { 7286 /* cleanup proc's outstanding kaio */ 7287 cleaned += (*aio_cleanup_dr_delete_memory)(procp); 7288 } 7289 mutex_exit(&procp->p_lock); 7290 } 7291 mutex_exit(&pidlock); 7292 return (cleaned); 7293 } 7294 7295 /* 7296 * helper function for page_capture_thread 7297 */ 7298 static void 7299 page_capture_handle_outstanding(void) 7300 { 7301 int ntry; 7302 7303 /* Reap pages before attempting capture pages */ 7304 kmem_reap(); 7305 7306 if ((page_retire_pend_count() > page_retire_pend_kas_count()) && 7307 hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 7308 /* 7309 * Note: Purging only for platforms that support 7310 * ISM hat_pageunload() - mainly SPARC. On x86/x64 7311 * platforms ISM pages SE_SHARED locked until destroyed. 7312 */ 7313 7314 /* disable and purge seg_pcache */ 7315 (void) seg_p_disable(); 7316 for (ntry = 0; ntry < pc_thread_retry; ntry++) { 7317 if (!page_retire_pend_count()) 7318 break; 7319 if (do_aio_cleanup()) { 7320 /* 7321 * allow the apps cleanup threads 7322 * to run 7323 */ 7324 delay(pc_thread_shortwait); 7325 } 7326 page_capture_async(); 7327 } 7328 /* reenable seg_pcache */ 7329 seg_p_enable(); 7330 7331 /* completed what can be done. break out */ 7332 return; 7333 } 7334 7335 /* 7336 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap 7337 * and then attempt to capture. 7338 */ 7339 seg_preap(); 7340 page_capture_async(); 7341 } 7342 7343 /* 7344 * The page_capture_thread loops forever, looking to see if there are 7345 * pages still waiting to be captured. 7346 */ 7347 static void 7348 page_capture_thread(void) 7349 { 7350 callb_cpr_t c; 7351 int i; 7352 int high_pri_pages; 7353 int low_pri_pages; 7354 clock_t timeout; 7355 7356 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture"); 7357 7358 mutex_enter(&pc_thread_mutex); 7359 for (;;) { 7360 high_pri_pages = 0; 7361 low_pri_pages = 0; 7362 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7363 high_pri_pages += 7364 page_capture_hash[i].num_pages[PC_PRI_HI]; 7365 low_pri_pages += 7366 page_capture_hash[i].num_pages[PC_PRI_LO]; 7367 } 7368 7369 timeout = pc_thread_longwait; 7370 if (high_pri_pages != 0) { 7371 timeout = pc_thread_shortwait; 7372 page_capture_handle_outstanding(); 7373 } else if (low_pri_pages != 0) { 7374 page_capture_async(); 7375 } 7376 CALLB_CPR_SAFE_BEGIN(&c); 7377 (void) cv_reltimedwait(&pc_cv, &pc_thread_mutex, 7378 timeout, TR_CLOCK_TICK); 7379 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7380 } 7381 /*NOTREACHED*/ 7382 } 7383 /* 7384 * Attempt to locate a bucket that has enough pages to satisfy the request. 7385 * The initial check is done without the lock to avoid unneeded contention. 7386 * The function returns 1 if enough pages were found, else 0 if it could not 7387 * find enough pages in a bucket. 7388 */ 7389 static int 7390 pcf_decrement_bucket(pgcnt_t npages) 7391 { 7392 struct pcf *p; 7393 struct pcf *q; 7394 int i; 7395 7396 p = &pcf[PCF_INDEX()]; 7397 q = &pcf[pcf_fanout]; 7398 for (i = 0; i < pcf_fanout; i++) { 7399 if (p->pcf_count > npages) { 7400 /* 7401 * a good one to try. 7402 */ 7403 mutex_enter(&p->pcf_lock); 7404 if (p->pcf_count > npages) { 7405 p->pcf_count -= (uint_t)npages; 7406 /* 7407 * freemem is not protected by any lock. 7408 * Thus, we cannot have any assertion 7409 * containing freemem here. 7410 */ 7411 freemem -= npages; 7412 mutex_exit(&p->pcf_lock); 7413 return (1); 7414 } 7415 mutex_exit(&p->pcf_lock); 7416 } 7417 p++; 7418 if (p >= q) { 7419 p = pcf; 7420 } 7421 } 7422 return (0); 7423 } 7424 7425 /* 7426 * Arguments: 7427 * pcftotal_ret: If the value is not NULL and we have walked all the 7428 * buckets but did not find enough pages then it will 7429 * be set to the total number of pages in all the pcf 7430 * buckets. 7431 * npages: Is the number of pages we have been requested to 7432 * find. 7433 * unlock: If set to 0 we will leave the buckets locked if the 7434 * requested number of pages are not found. 7435 * 7436 * Go and try to satisfy the page request from any number of buckets. 7437 * This can be a very expensive operation as we have to lock the buckets 7438 * we are checking (and keep them locked), starting at bucket 0. 7439 * 7440 * The function returns 1 if enough pages were found, else 0 if it could not 7441 * find enough pages in the buckets. 7442 * 7443 */ 7444 static int 7445 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock) 7446 { 7447 struct pcf *p; 7448 pgcnt_t pcftotal; 7449 int i; 7450 7451 p = pcf; 7452 /* try to collect pages from several pcf bins */ 7453 for (pcftotal = 0, i = 0; i < pcf_fanout; i++) { 7454 mutex_enter(&p->pcf_lock); 7455 pcftotal += p->pcf_count; 7456 if (pcftotal >= npages) { 7457 /* 7458 * Wow! There are enough pages laying around 7459 * to satisfy the request. Do the accounting, 7460 * drop the locks we acquired, and go back. 7461 * 7462 * freemem is not protected by any lock. So, 7463 * we cannot have any assertion containing 7464 * freemem. 7465 */ 7466 freemem -= npages; 7467 while (p >= pcf) { 7468 if (p->pcf_count <= npages) { 7469 npages -= p->pcf_count; 7470 p->pcf_count = 0; 7471 } else { 7472 p->pcf_count -= (uint_t)npages; 7473 npages = 0; 7474 } 7475 mutex_exit(&p->pcf_lock); 7476 p--; 7477 } 7478 ASSERT(npages == 0); 7479 return (1); 7480 } 7481 p++; 7482 } 7483 if (unlock) { 7484 /* failed to collect pages - release the locks */ 7485 while (--p >= pcf) { 7486 mutex_exit(&p->pcf_lock); 7487 } 7488 } 7489 if (pcftotal_ret != NULL) 7490 *pcftotal_ret = pcftotal; 7491 return (0); 7492 } 7493