1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2015, Josef 'Jeff' Sipek <jeffpc@josefsipek.net> 24 * Copyright (c) 2015, 2016 by Delphix. All rights reserved. 25 * Copyright 2018 Joyent, Inc. 26 * Copyright 2021 Oxide Computer Company 27 */ 28 29 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 30 /* All Rights Reserved */ 31 32 /* 33 * University Copyright- Copyright (c) 1982, 1986, 1988 34 * The Regents of the University of California 35 * All Rights Reserved 36 * 37 * University Acknowledgment- Portions of this document are derived from 38 * software developed by the University of California, Berkeley, and its 39 * contributors. 40 */ 41 42 /* 43 * VM - physical page management. 44 */ 45 46 #include <sys/types.h> 47 #include <sys/t_lock.h> 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/errno.h> 51 #include <sys/time.h> 52 #include <sys/vnode.h> 53 #include <sys/vm.h> 54 #include <sys/vtrace.h> 55 #include <sys/swap.h> 56 #include <sys/cmn_err.h> 57 #include <sys/tuneable.h> 58 #include <sys/sysmacros.h> 59 #include <sys/cpuvar.h> 60 #include <sys/callb.h> 61 #include <sys/debug.h> 62 #include <sys/condvar_impl.h> 63 #include <sys/mem_config.h> 64 #include <sys/mem_cage.h> 65 #include <sys/kmem.h> 66 #include <sys/atomic.h> 67 #include <sys/strlog.h> 68 #include <sys/mman.h> 69 #include <sys/ontrap.h> 70 #include <sys/lgrp.h> 71 #include <sys/vfs.h> 72 73 #include <vm/hat.h> 74 #include <vm/anon.h> 75 #include <vm/page.h> 76 #include <vm/seg.h> 77 #include <vm/pvn.h> 78 #include <vm/seg_kmem.h> 79 #include <vm/vm_dep.h> 80 #include <sys/vm_usage.h> 81 #include <fs/fs_subr.h> 82 #include <sys/ddi.h> 83 #include <sys/modctl.h> 84 85 static pgcnt_t max_page_get; /* max page_get request size in pages */ 86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 87 88 /* 89 * freemem_lock protects all freemem variables: 90 * availrmem. Also this lock protects the globals which track the 91 * availrmem changes for accurate kernel footprint calculation. 92 * See below for an explanation of these 93 * globals. 94 */ 95 kmutex_t freemem_lock; 96 pgcnt_t availrmem; 97 pgcnt_t availrmem_initial; 98 99 /* 100 * These globals track availrmem changes to get a more accurate 101 * estimate of tke kernel size. Historically pp_kernel is used for 102 * kernel size and is based on availrmem. But availrmem is adjusted for 103 * locked pages in the system not just for kernel locked pages. 104 * These new counters will track the pages locked through segvn and 105 * by explicit user locking. 106 * 107 * pages_locked : How many pages are locked because of user specified 108 * locking through mlock or plock. 109 * 110 * pages_useclaim,pages_claimed : These two variables track the 111 * claim adjustments because of the protection changes on a segvn segment. 112 * 113 * All these globals are protected by the same lock which protects availrmem. 114 */ 115 pgcnt_t pages_locked = 0; 116 pgcnt_t pages_useclaim = 0; 117 pgcnt_t pages_claimed = 0; 118 119 120 /* 121 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 122 */ 123 static kmutex_t new_freemem_lock; 124 static uint_t freemem_wait; /* someone waiting for freemem */ 125 static kcondvar_t freemem_cv; 126 127 /* 128 * The logical page free list is maintained as two lists, the 'free' 129 * and the 'cache' lists. 130 * The free list contains those pages that should be reused first. 131 * 132 * The implementation of the lists is machine dependent. 133 * page_get_freelist(), page_get_cachelist(), 134 * page_list_sub(), and page_list_add() 135 * form the interface to the machine dependent implementation. 136 * 137 * Pages with p_free set are on the cache list. 138 * Pages with p_free and p_age set are on the free list, 139 * 140 * A page may be locked while on either list. 141 */ 142 143 /* 144 * free list accounting stuff. 145 * 146 * 147 * Spread out the value for the number of pages on the 148 * page free and page cache lists. If there is just one 149 * value, then it must be under just one lock. 150 * The lock contention and cache traffic are a real bother. 151 * 152 * When we acquire and then drop a single pcf lock 153 * we can start in the middle of the array of pcf structures. 154 * If we acquire more than one pcf lock at a time, we need to 155 * start at the front to avoid deadlocking. 156 * 157 * pcf_count holds the number of pages in each pool. 158 * 159 * pcf_block is set when page_create_get_something() has asked the 160 * PSM page freelist and page cachelist routines without specifying 161 * a color and nothing came back. This is used to block anything 162 * else from moving pages from one list to the other while the 163 * lists are searched again. If a page is freeed while pcf_block is 164 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 165 * of clearning pcf_block, doing the wakeups, etc. 166 */ 167 168 #define MAX_PCF_FANOUT NCPU 169 static uint_t pcf_fanout = 1; /* Will get changed at boot time */ 170 static uint_t pcf_fanout_mask = 0; 171 172 struct pcf { 173 kmutex_t pcf_lock; /* protects the structure */ 174 uint_t pcf_count; /* page count */ 175 uint_t pcf_wait; /* number of waiters */ 176 uint_t pcf_block; /* pcgs flag to page_free() */ 177 uint_t pcf_reserve; /* pages freed after pcf_block set */ 178 uint_t pcf_fill[10]; /* to line up on the caches */ 179 }; 180 181 /* 182 * PCF_INDEX hash needs to be dynamic (every so often the hash changes where 183 * it will hash the cpu to). This is done to prevent a drain condition 184 * from happening. This drain condition will occur when pcf_count decrement 185 * occurs on cpu A and the increment of pcf_count always occurs on cpu B. An 186 * example of this shows up with device interrupts. The dma buffer is allocated 187 * by the cpu requesting the IO thus the pcf_count is decremented based on that. 188 * When the memory is returned by the interrupt thread, the pcf_count will be 189 * incremented based on the cpu servicing the interrupt. 190 */ 191 static struct pcf pcf[MAX_PCF_FANOUT]; 192 #define PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \ 193 (randtick() >> 24)) & (pcf_fanout_mask)) 194 195 static int pcf_decrement_bucket(pgcnt_t); 196 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int); 197 198 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 199 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 200 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 201 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 202 203 #ifdef VM_STATS 204 205 /* 206 * No locks, but so what, they are only statistics. 207 */ 208 209 static struct page_tcnt { 210 int pc_free_cache; /* free's into cache list */ 211 int pc_free_dontneed; /* free's with dontneed */ 212 int pc_free_pageout; /* free's from pageout */ 213 int pc_free_free; /* free's into free list */ 214 int pc_free_pages; /* free's into large page free list */ 215 int pc_destroy_pages; /* large page destroy's */ 216 int pc_get_cache; /* get's from cache list */ 217 int pc_get_free; /* get's from free list */ 218 int pc_reclaim; /* reclaim's */ 219 int pc_abortfree; /* abort's of free pages */ 220 int pc_find_hit; /* find's that find page */ 221 int pc_find_miss; /* find's that don't find page */ 222 int pc_destroy_free; /* # of free pages destroyed */ 223 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 224 int pc_find_hashlen[PC_HASH_CNT+1]; 225 int pc_addclaim_pages; 226 int pc_subclaim_pages; 227 int pc_free_replacement_page[2]; 228 int pc_try_demote_pages[6]; 229 int pc_demote_pages[2]; 230 } pagecnt; 231 232 uint_t hashin_count; 233 uint_t hashin_not_held; 234 uint_t hashin_already; 235 236 uint_t hashout_count; 237 uint_t hashout_not_held; 238 239 uint_t page_create_count; 240 uint_t page_create_not_enough; 241 uint_t page_create_not_enough_again; 242 uint_t page_create_zero; 243 uint_t page_create_hashout; 244 uint_t page_create_page_lock_failed; 245 uint_t page_create_trylock_failed; 246 uint_t page_create_found_one; 247 uint_t page_create_hashin_failed; 248 uint_t page_create_dropped_phm; 249 250 uint_t page_create_new; 251 uint_t page_create_exists; 252 uint_t page_create_putbacks; 253 uint_t page_create_overshoot; 254 255 uint_t page_reclaim_zero; 256 uint_t page_reclaim_zero_locked; 257 258 uint_t page_rename_exists; 259 uint_t page_rename_count; 260 261 uint_t page_lookup_cnt[20]; 262 uint_t page_lookup_nowait_cnt[10]; 263 uint_t page_find_cnt; 264 uint_t page_exists_cnt; 265 uint_t page_exists_forreal_cnt; 266 uint_t page_lookup_dev_cnt; 267 uint_t get_cachelist_cnt; 268 uint_t page_create_cnt[10]; 269 uint_t alloc_pages[9]; 270 uint_t page_exphcontg[19]; 271 uint_t page_create_large_cnt[10]; 272 273 #endif 274 275 static inline page_t * 276 page_hash_search(ulong_t index, vnode_t *vnode, u_offset_t off) 277 { 278 uint_t mylen = 0; 279 page_t *page; 280 281 for (page = page_hash[index]; page; page = page->p_hash, mylen++) 282 if (page->p_vnode == vnode && page->p_offset == off) 283 break; 284 285 #ifdef VM_STATS 286 if (page != NULL) 287 pagecnt.pc_find_hit++; 288 else 289 pagecnt.pc_find_miss++; 290 291 pagecnt.pc_find_hashlen[MIN(mylen, PC_HASH_CNT)]++; 292 #endif 293 294 return (page); 295 } 296 297 298 #ifdef DEBUG 299 #define MEMSEG_SEARCH_STATS 300 #endif 301 302 #ifdef MEMSEG_SEARCH_STATS 303 struct memseg_stats { 304 uint_t nsearch; 305 uint_t nlastwon; 306 uint_t nhashwon; 307 uint_t nnotfound; 308 } memseg_stats; 309 310 #define MEMSEG_STAT_INCR(v) \ 311 atomic_inc_32(&memseg_stats.v) 312 #else 313 #define MEMSEG_STAT_INCR(x) 314 #endif 315 316 struct memseg *memsegs; /* list of memory segments */ 317 318 /* 319 * /etc/system tunable to control large page allocation hueristic. 320 * 321 * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup 322 * for large page allocation requests. If a large page is not readily 323 * avaliable on the local freelists we will go through additional effort 324 * to create a large page, potentially moving smaller pages around to coalesce 325 * larger pages in the local lgroup. 326 * Default value of LPAP_DEFAULT will go to remote freelists if large pages 327 * are not readily available in the local lgroup. 328 */ 329 enum lpap { 330 LPAP_DEFAULT, /* default large page allocation policy */ 331 LPAP_LOCAL /* local large page allocation policy */ 332 }; 333 334 enum lpap lpg_alloc_prefer = LPAP_DEFAULT; 335 336 static void page_init_mem_config(void); 337 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 338 static void page_do_hashout(page_t *); 339 static void page_capture_init(); 340 int page_capture_take_action(page_t *, uint_t, void *); 341 342 static void page_demote_vp_pages(page_t *); 343 344 345 void 346 pcf_init(void) 347 { 348 if (boot_ncpus != -1) { 349 pcf_fanout = boot_ncpus; 350 } else { 351 pcf_fanout = max_ncpus; 352 } 353 #ifdef sun4v 354 /* 355 * Force at least 4 buckets if possible for sun4v. 356 */ 357 pcf_fanout = MAX(pcf_fanout, 4); 358 #endif /* sun4v */ 359 360 /* 361 * Round up to the nearest power of 2. 362 */ 363 pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT); 364 if (!ISP2(pcf_fanout)) { 365 pcf_fanout = 1 << highbit(pcf_fanout); 366 367 if (pcf_fanout > MAX_PCF_FANOUT) { 368 pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1); 369 } 370 } 371 pcf_fanout_mask = pcf_fanout - 1; 372 } 373 374 /* 375 * vm subsystem related initialization 376 */ 377 void 378 vm_init(void) 379 { 380 boolean_t callb_vm_cpr(void *, int); 381 382 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 383 page_init_mem_config(); 384 page_retire_init(); 385 vm_usage_init(); 386 page_capture_init(); 387 } 388 389 /* 390 * This function is called at startup and when memory is added or deleted. 391 */ 392 void 393 init_pages_pp_maximum() 394 { 395 static pgcnt_t p_min; 396 static pgcnt_t pages_pp_maximum_startup; 397 static pgcnt_t avrmem_delta; 398 static int init_done; 399 static int user_set; /* true if set in /etc/system */ 400 401 if (init_done == 0) { 402 403 /* If the user specified a value, save it */ 404 if (pages_pp_maximum != 0) { 405 user_set = 1; 406 pages_pp_maximum_startup = pages_pp_maximum; 407 } 408 409 /* 410 * Setting of pages_pp_maximum is based first time 411 * on the value of availrmem just after the start-up 412 * allocations. To preserve this relationship at run 413 * time, use a delta from availrmem_initial. 414 */ 415 ASSERT(availrmem_initial >= availrmem); 416 avrmem_delta = availrmem_initial - availrmem; 417 418 /* The allowable floor of pages_pp_maximum */ 419 p_min = tune.t_minarmem + 100; 420 421 /* Make sure we don't come through here again. */ 422 init_done = 1; 423 } 424 /* 425 * Determine pages_pp_maximum, the number of currently available 426 * pages (availrmem) that can't be `locked'. If not set by 427 * the user, we set it to 4% of the currently available memory 428 * plus 4MB. 429 * But we also insist that it be greater than tune.t_minarmem; 430 * otherwise a process could lock down a lot of memory, get swapped 431 * out, and never have enough to get swapped back in. 432 */ 433 if (user_set) 434 pages_pp_maximum = pages_pp_maximum_startup; 435 else 436 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 437 + btop(4 * 1024 * 1024); 438 439 if (pages_pp_maximum <= p_min) { 440 pages_pp_maximum = p_min; 441 } 442 } 443 444 /* 445 * In the past, we limited the maximum pages that could be gotten to essentially 446 * 1/2 of the total pages on the system. However, this is too conservative for 447 * some cases. For example, if we want to host a large virtual machine which 448 * needs to use a significant portion of the system's memory. In practice, 449 * allowing more than 1/2 of the total pages is fine, but becomes problematic 450 * as we approach or exceed 75% of the pages on the system. Thus, we limit the 451 * maximum to 23/32 of the total pages, which is ~72%. 452 */ 453 void 454 set_max_page_get(pgcnt_t target_total_pages) 455 { 456 max_page_get = (target_total_pages >> 5) * 23; 457 ASSERT3U(max_page_get, >, 0); 458 } 459 460 pgcnt_t 461 get_max_page_get() 462 { 463 return (max_page_get); 464 } 465 466 static pgcnt_t pending_delete; 467 468 /*ARGSUSED*/ 469 static void 470 page_mem_config_post_add( 471 void *arg, 472 pgcnt_t delta_pages) 473 { 474 set_max_page_get(total_pages - pending_delete); 475 init_pages_pp_maximum(); 476 } 477 478 /*ARGSUSED*/ 479 static int 480 page_mem_config_pre_del( 481 void *arg, 482 pgcnt_t delta_pages) 483 { 484 pgcnt_t nv; 485 486 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 487 set_max_page_get(total_pages - nv); 488 return (0); 489 } 490 491 /*ARGSUSED*/ 492 static void 493 page_mem_config_post_del( 494 void *arg, 495 pgcnt_t delta_pages, 496 int cancelled) 497 { 498 pgcnt_t nv; 499 500 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 501 set_max_page_get(total_pages - nv); 502 if (!cancelled) 503 init_pages_pp_maximum(); 504 } 505 506 static kphysm_setup_vector_t page_mem_config_vec = { 507 KPHYSM_SETUP_VECTOR_VERSION, 508 page_mem_config_post_add, 509 page_mem_config_pre_del, 510 page_mem_config_post_del, 511 }; 512 513 static void 514 page_init_mem_config(void) 515 { 516 int ret; 517 518 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 519 ASSERT(ret == 0); 520 } 521 522 /* 523 * Evenly spread out the PCF counters for large free pages 524 */ 525 static void 526 page_free_large_ctr(pgcnt_t npages) 527 { 528 static struct pcf *p = pcf; 529 pgcnt_t lump; 530 531 freemem += npages; 532 533 lump = roundup(npages, pcf_fanout) / pcf_fanout; 534 535 while (npages > 0) { 536 537 ASSERT(!p->pcf_block); 538 539 if (lump < npages) { 540 p->pcf_count += (uint_t)lump; 541 npages -= lump; 542 } else { 543 p->pcf_count += (uint_t)npages; 544 npages = 0; 545 } 546 547 ASSERT(!p->pcf_wait); 548 549 if (++p > &pcf[pcf_fanout - 1]) 550 p = pcf; 551 } 552 553 ASSERT(npages == 0); 554 } 555 556 /* 557 * Add a physical chunk of memory to the system free lists during startup. 558 * Platform specific startup() allocates the memory for the page structs. 559 * 560 * num - number of page structures 561 * base - page number (pfn) to be associated with the first page. 562 * 563 * Since we are doing this during startup (ie. single threaded), we will 564 * use shortcut routines to avoid any locking overhead while putting all 565 * these pages on the freelists. 566 * 567 * NOTE: Any changes performed to page_free(), must also be performed to 568 * add_physmem() since this is how we initialize all page_t's at 569 * boot time. 570 */ 571 void 572 add_physmem( 573 page_t *pp, 574 pgcnt_t num, 575 pfn_t pnum) 576 { 577 page_t *root = NULL; 578 uint_t szc = page_num_pagesizes() - 1; 579 pgcnt_t large = page_get_pagecnt(szc); 580 pgcnt_t cnt = 0; 581 582 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 583 "add_physmem:pp %p num %lu", pp, num); 584 585 /* 586 * Arbitrarily limit the max page_get request 587 * to 1/2 of the page structs we have. 588 */ 589 total_pages += num; 590 set_max_page_get(total_pages); 591 592 PLCNT_MODIFY_MAX(pnum, (long)num); 593 594 /* 595 * The physical space for the pages array 596 * representing ram pages has already been 597 * allocated. Here we initialize each lock 598 * in the page structure, and put each on 599 * the free list 600 */ 601 for (; num; pp++, pnum++, num--) { 602 603 /* 604 * this needs to fill in the page number 605 * and do any other arch specific initialization 606 */ 607 add_physmem_cb(pp, pnum); 608 609 pp->p_lckcnt = 0; 610 pp->p_cowcnt = 0; 611 pp->p_slckcnt = 0; 612 613 /* 614 * Initialize the page lock as unlocked, since nobody 615 * can see or access this page yet. 616 */ 617 pp->p_selock = 0; 618 619 /* 620 * Initialize IO lock 621 */ 622 page_iolock_init(pp); 623 624 /* 625 * initialize other fields in the page_t 626 */ 627 PP_SETFREE(pp); 628 page_clr_all_props(pp); 629 PP_SETAGED(pp); 630 pp->p_offset = (u_offset_t)-1; 631 pp->p_next = pp; 632 pp->p_prev = pp; 633 634 /* 635 * Simple case: System doesn't support large pages. 636 */ 637 if (szc == 0) { 638 pp->p_szc = 0; 639 page_free_at_startup(pp); 640 continue; 641 } 642 643 /* 644 * Handle unaligned pages, we collect them up onto 645 * the root page until we have a full large page. 646 */ 647 if (!IS_P2ALIGNED(pnum, large)) { 648 649 /* 650 * If not in a large page, 651 * just free as small page. 652 */ 653 if (root == NULL) { 654 pp->p_szc = 0; 655 page_free_at_startup(pp); 656 continue; 657 } 658 659 /* 660 * Link a constituent page into the large page. 661 */ 662 pp->p_szc = szc; 663 page_list_concat(&root, &pp); 664 665 /* 666 * When large page is fully formed, free it. 667 */ 668 if (++cnt == large) { 669 page_free_large_ctr(cnt); 670 page_list_add_pages(root, PG_LIST_ISINIT); 671 root = NULL; 672 cnt = 0; 673 } 674 continue; 675 } 676 677 /* 678 * At this point we have a page number which 679 * is aligned. We assert that we aren't already 680 * in a different large page. 681 */ 682 ASSERT(IS_P2ALIGNED(pnum, large)); 683 ASSERT(root == NULL && cnt == 0); 684 685 /* 686 * If insufficient number of pages left to form 687 * a large page, just free the small page. 688 */ 689 if (num < large) { 690 pp->p_szc = 0; 691 page_free_at_startup(pp); 692 continue; 693 } 694 695 /* 696 * Otherwise start a new large page. 697 */ 698 pp->p_szc = szc; 699 cnt++; 700 root = pp; 701 } 702 ASSERT(root == NULL && cnt == 0); 703 } 704 705 /* 706 * Find a page representing the specified [vp, offset]. 707 * If we find the page but it is intransit coming in, 708 * it will have an "exclusive" lock and we wait for 709 * the i/o to complete. A page found on the free list 710 * is always reclaimed and then locked. On success, the page 711 * is locked, its data is valid and it isn't on the free 712 * list, while a NULL is returned if the page doesn't exist. 713 */ 714 page_t * 715 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 716 { 717 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 718 } 719 720 /* 721 * Find a page representing the specified [vp, offset]. 722 * We either return the one we found or, if passed in, 723 * create one with identity of [vp, offset] of the 724 * pre-allocated page. If we find existing page but it is 725 * intransit coming in, it will have an "exclusive" lock 726 * and we wait for the i/o to complete. A page found on 727 * the free list is always reclaimed and then locked. 728 * On success, the page is locked, its data is valid and 729 * it isn't on the free list, while a NULL is returned 730 * if the page doesn't exist and newpp is NULL; 731 */ 732 page_t * 733 page_lookup_create( 734 vnode_t *vp, 735 u_offset_t off, 736 se_t se, 737 page_t *newpp, 738 spgcnt_t *nrelocp, 739 int flags) 740 { 741 page_t *pp; 742 kmutex_t *phm; 743 ulong_t index; 744 uint_t hash_locked; 745 uint_t es; 746 747 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 748 VM_STAT_ADD(page_lookup_cnt[0]); 749 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 750 751 /* 752 * Acquire the appropriate page hash lock since 753 * we have to search the hash list. Pages that 754 * hash to this list can't change identity while 755 * this lock is held. 756 */ 757 hash_locked = 0; 758 index = PAGE_HASH_FUNC(vp, off); 759 phm = NULL; 760 top: 761 pp = page_hash_search(index, vp, off); 762 if (pp != NULL) { 763 VM_STAT_ADD(page_lookup_cnt[1]); 764 es = (newpp != NULL) ? 1 : 0; 765 es |= flags; 766 if (!hash_locked) { 767 VM_STAT_ADD(page_lookup_cnt[2]); 768 if (!page_try_reclaim_lock(pp, se, es)) { 769 /* 770 * On a miss, acquire the phm. Then 771 * next time, page_lock() will be called, 772 * causing a wait if the page is busy. 773 * just looping with page_trylock() would 774 * get pretty boring. 775 */ 776 VM_STAT_ADD(page_lookup_cnt[3]); 777 phm = PAGE_HASH_MUTEX(index); 778 mutex_enter(phm); 779 hash_locked = 1; 780 goto top; 781 } 782 } else { 783 VM_STAT_ADD(page_lookup_cnt[4]); 784 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 785 VM_STAT_ADD(page_lookup_cnt[5]); 786 goto top; 787 } 788 } 789 790 /* 791 * Since `pp' is locked it can not change identity now. 792 * Reconfirm we locked the correct page. 793 * 794 * Both the p_vnode and p_offset *must* be cast volatile 795 * to force a reload of their values: The page_hash_search 796 * function will have stuffed p_vnode and p_offset into 797 * registers before calling page_trylock(); another thread, 798 * actually holding the hash lock, could have changed the 799 * page's identity in memory, but our registers would not 800 * be changed, fooling the reconfirmation. If the hash 801 * lock was held during the search, the casting would 802 * not be needed. 803 */ 804 VM_STAT_ADD(page_lookup_cnt[6]); 805 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 806 ((volatile u_offset_t)(pp->p_offset) != off)) { 807 VM_STAT_ADD(page_lookup_cnt[7]); 808 if (hash_locked) { 809 panic("page_lookup_create: lost page %p", 810 (void *)pp); 811 /*NOTREACHED*/ 812 } 813 page_unlock(pp); 814 phm = PAGE_HASH_MUTEX(index); 815 mutex_enter(phm); 816 hash_locked = 1; 817 goto top; 818 } 819 820 /* 821 * If page_trylock() was called, then pp may still be on 822 * the cachelist (can't be on the free list, it would not 823 * have been found in the search). If it is on the 824 * cachelist it must be pulled now. To pull the page from 825 * the cachelist, it must be exclusively locked. 826 * 827 * The other big difference between page_trylock() and 828 * page_lock(), is that page_lock() will pull the 829 * page from whatever free list (the cache list in this 830 * case) the page is on. If page_trylock() was used 831 * above, then we have to do the reclaim ourselves. 832 */ 833 if ((!hash_locked) && (PP_ISFREE(pp))) { 834 ASSERT(PP_ISAGED(pp) == 0); 835 VM_STAT_ADD(page_lookup_cnt[8]); 836 837 /* 838 * page_relcaim will insure that we 839 * have this page exclusively 840 */ 841 842 if (!page_reclaim(pp, NULL)) { 843 /* 844 * Page_reclaim dropped whatever lock 845 * we held. 846 */ 847 VM_STAT_ADD(page_lookup_cnt[9]); 848 phm = PAGE_HASH_MUTEX(index); 849 mutex_enter(phm); 850 hash_locked = 1; 851 goto top; 852 } else if (se == SE_SHARED && newpp == NULL) { 853 VM_STAT_ADD(page_lookup_cnt[10]); 854 page_downgrade(pp); 855 } 856 } 857 858 if (hash_locked) { 859 mutex_exit(phm); 860 } 861 862 if (newpp != NULL && pp->p_szc < newpp->p_szc && 863 PAGE_EXCL(pp) && nrelocp != NULL) { 864 ASSERT(nrelocp != NULL); 865 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 866 NULL); 867 if (*nrelocp > 0) { 868 VM_STAT_COND_ADD(*nrelocp == 1, 869 page_lookup_cnt[11]); 870 VM_STAT_COND_ADD(*nrelocp > 1, 871 page_lookup_cnt[12]); 872 pp = newpp; 873 se = SE_EXCL; 874 } else { 875 if (se == SE_SHARED) { 876 page_downgrade(pp); 877 } 878 VM_STAT_ADD(page_lookup_cnt[13]); 879 } 880 } else if (newpp != NULL && nrelocp != NULL) { 881 if (PAGE_EXCL(pp) && se == SE_SHARED) { 882 page_downgrade(pp); 883 } 884 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 885 page_lookup_cnt[14]); 886 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 887 page_lookup_cnt[15]); 888 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 889 page_lookup_cnt[16]); 890 } else if (newpp != NULL && PAGE_EXCL(pp)) { 891 se = SE_EXCL; 892 } 893 } else if (!hash_locked) { 894 VM_STAT_ADD(page_lookup_cnt[17]); 895 phm = PAGE_HASH_MUTEX(index); 896 mutex_enter(phm); 897 hash_locked = 1; 898 goto top; 899 } else if (newpp != NULL) { 900 /* 901 * If we have a preallocated page then 902 * insert it now and basically behave like 903 * page_create. 904 */ 905 VM_STAT_ADD(page_lookup_cnt[18]); 906 /* 907 * Since we hold the page hash mutex and 908 * just searched for this page, page_hashin 909 * had better not fail. If it does, that 910 * means some thread did not follow the 911 * page hash mutex rules. Panic now and 912 * get it over with. As usual, go down 913 * holding all the locks. 914 */ 915 ASSERT(MUTEX_HELD(phm)); 916 if (!page_hashin(newpp, vp, off, phm)) { 917 ASSERT(MUTEX_HELD(phm)); 918 panic("page_lookup_create: hashin failed %p %p %llx %p", 919 (void *)newpp, (void *)vp, off, (void *)phm); 920 /*NOTREACHED*/ 921 } 922 ASSERT(MUTEX_HELD(phm)); 923 mutex_exit(phm); 924 phm = NULL; 925 page_set_props(newpp, P_REF); 926 page_io_lock(newpp); 927 pp = newpp; 928 se = SE_EXCL; 929 } else { 930 VM_STAT_ADD(page_lookup_cnt[19]); 931 mutex_exit(phm); 932 } 933 934 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 935 936 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 937 938 return (pp); 939 } 940 941 /* 942 * Search the hash list for the page representing the 943 * specified [vp, offset] and return it locked. Skip 944 * free pages and pages that cannot be locked as requested. 945 * Used while attempting to kluster pages. 946 */ 947 page_t * 948 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 949 { 950 page_t *pp; 951 kmutex_t *phm; 952 ulong_t index; 953 uint_t locked; 954 955 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 956 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 957 958 index = PAGE_HASH_FUNC(vp, off); 959 pp = page_hash_search(index, vp, off); 960 locked = 0; 961 if (pp == NULL) { 962 top: 963 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 964 locked = 1; 965 phm = PAGE_HASH_MUTEX(index); 966 mutex_enter(phm); 967 pp = page_hash_search(index, vp, off); 968 } 969 970 if (pp == NULL || PP_ISFREE(pp)) { 971 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 972 pp = NULL; 973 } else { 974 if (!page_trylock(pp, se)) { 975 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 976 pp = NULL; 977 } else { 978 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 979 /* 980 * See the comment in page_lookup() 981 */ 982 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 983 ((u_offset_t)(pp->p_offset) != off)) { 984 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 985 if (locked) { 986 panic("page_lookup_nowait %p", 987 (void *)pp); 988 /*NOTREACHED*/ 989 } 990 page_unlock(pp); 991 goto top; 992 } 993 if (PP_ISFREE(pp)) { 994 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 995 page_unlock(pp); 996 pp = NULL; 997 } 998 } 999 } 1000 if (locked) { 1001 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 1002 mutex_exit(phm); 1003 } 1004 1005 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 1006 1007 return (pp); 1008 } 1009 1010 /* 1011 * Search the hash list for a page with the specified [vp, off] 1012 * that is known to exist and is already locked. This routine 1013 * is typically used by segment SOFTUNLOCK routines. 1014 */ 1015 page_t * 1016 page_find(vnode_t *vp, u_offset_t off) 1017 { 1018 page_t *pp; 1019 kmutex_t *phm; 1020 ulong_t index; 1021 1022 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1023 VM_STAT_ADD(page_find_cnt); 1024 1025 index = PAGE_HASH_FUNC(vp, off); 1026 phm = PAGE_HASH_MUTEX(index); 1027 1028 mutex_enter(phm); 1029 pp = page_hash_search(index, vp, off); 1030 mutex_exit(phm); 1031 1032 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 1033 return (pp); 1034 } 1035 1036 /* 1037 * Determine whether a page with the specified [vp, off] 1038 * currently exists in the system. Obviously this should 1039 * only be considered as a hint since nothing prevents the 1040 * page from disappearing or appearing immediately after 1041 * the return from this routine. Subsequently, we don't 1042 * even bother to lock the list. 1043 */ 1044 page_t * 1045 page_exists(vnode_t *vp, u_offset_t off) 1046 { 1047 ulong_t index; 1048 1049 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1050 VM_STAT_ADD(page_exists_cnt); 1051 1052 index = PAGE_HASH_FUNC(vp, off); 1053 1054 return (page_hash_search(index, vp, off)); 1055 } 1056 1057 /* 1058 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1059 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1060 * with these pages locked SHARED. If necessary reclaim pages from 1061 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1062 * 1063 * If we fail to lock pages still return 1 if pages exist and contiguous. 1064 * But in this case return value is just a hint. ppa array won't be filled. 1065 * Caller should initialize ppa[0] as NULL to distinguish return value. 1066 * 1067 * Returns 0 if pages don't exist or not physically contiguous. 1068 * 1069 * This routine doesn't work for anonymous(swapfs) pages. 1070 */ 1071 int 1072 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1073 { 1074 pgcnt_t pages; 1075 pfn_t pfn; 1076 page_t *rootpp; 1077 pgcnt_t i; 1078 pgcnt_t j; 1079 u_offset_t save_off = off; 1080 ulong_t index; 1081 kmutex_t *phm; 1082 page_t *pp; 1083 uint_t pszc; 1084 int loopcnt = 0; 1085 1086 ASSERT(szc != 0); 1087 ASSERT(vp != NULL); 1088 ASSERT(!IS_SWAPFSVP(vp)); 1089 ASSERT(!VN_ISKAS(vp)); 1090 1091 again: 1092 if (++loopcnt > 3) { 1093 VM_STAT_ADD(page_exphcontg[0]); 1094 return (0); 1095 } 1096 1097 index = PAGE_HASH_FUNC(vp, off); 1098 phm = PAGE_HASH_MUTEX(index); 1099 1100 mutex_enter(phm); 1101 pp = page_hash_search(index, vp, off); 1102 mutex_exit(phm); 1103 1104 VM_STAT_ADD(page_exphcontg[1]); 1105 1106 if (pp == NULL) { 1107 VM_STAT_ADD(page_exphcontg[2]); 1108 return (0); 1109 } 1110 1111 pages = page_get_pagecnt(szc); 1112 rootpp = pp; 1113 pfn = rootpp->p_pagenum; 1114 1115 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1116 VM_STAT_ADD(page_exphcontg[3]); 1117 if (!page_trylock(pp, SE_SHARED)) { 1118 VM_STAT_ADD(page_exphcontg[4]); 1119 return (1); 1120 } 1121 /* 1122 * Also check whether p_pagenum was modified by DR. 1123 */ 1124 if (pp->p_szc != pszc || pp->p_vnode != vp || 1125 pp->p_offset != off || pp->p_pagenum != pfn) { 1126 VM_STAT_ADD(page_exphcontg[5]); 1127 page_unlock(pp); 1128 off = save_off; 1129 goto again; 1130 } 1131 /* 1132 * szc was non zero and vnode and offset matched after we 1133 * locked the page it means it can't become free on us. 1134 */ 1135 ASSERT(!PP_ISFREE(pp)); 1136 if (!IS_P2ALIGNED(pfn, pages)) { 1137 page_unlock(pp); 1138 return (0); 1139 } 1140 ppa[0] = pp; 1141 pp++; 1142 off += PAGESIZE; 1143 pfn++; 1144 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1145 if (!page_trylock(pp, SE_SHARED)) { 1146 VM_STAT_ADD(page_exphcontg[6]); 1147 pp--; 1148 while (i-- > 0) { 1149 page_unlock(pp); 1150 pp--; 1151 } 1152 ppa[0] = NULL; 1153 return (1); 1154 } 1155 if (pp->p_szc != pszc) { 1156 VM_STAT_ADD(page_exphcontg[7]); 1157 page_unlock(pp); 1158 pp--; 1159 while (i-- > 0) { 1160 page_unlock(pp); 1161 pp--; 1162 } 1163 ppa[0] = NULL; 1164 off = save_off; 1165 goto again; 1166 } 1167 /* 1168 * szc the same as for previous already locked pages 1169 * with right identity. Since this page had correct 1170 * szc after we locked it can't get freed or destroyed 1171 * and therefore must have the expected identity. 1172 */ 1173 ASSERT(!PP_ISFREE(pp)); 1174 if (pp->p_vnode != vp || 1175 pp->p_offset != off) { 1176 panic("page_exists_physcontig: " 1177 "large page identity doesn't match"); 1178 } 1179 ppa[i] = pp; 1180 ASSERT(pp->p_pagenum == pfn); 1181 } 1182 VM_STAT_ADD(page_exphcontg[8]); 1183 ppa[pages] = NULL; 1184 return (1); 1185 } else if (pszc >= szc) { 1186 VM_STAT_ADD(page_exphcontg[9]); 1187 if (!IS_P2ALIGNED(pfn, pages)) { 1188 return (0); 1189 } 1190 return (1); 1191 } 1192 1193 if (!IS_P2ALIGNED(pfn, pages)) { 1194 VM_STAT_ADD(page_exphcontg[10]); 1195 return (0); 1196 } 1197 1198 if (page_numtomemseg_nolock(pfn) != 1199 page_numtomemseg_nolock(pfn + pages - 1)) { 1200 VM_STAT_ADD(page_exphcontg[11]); 1201 return (0); 1202 } 1203 1204 /* 1205 * We loop up 4 times across pages to promote page size. 1206 * We're extra cautious to promote page size atomically with respect 1207 * to everybody else. But we can probably optimize into 1 loop if 1208 * this becomes an issue. 1209 */ 1210 1211 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1212 if (!page_trylock(pp, SE_EXCL)) { 1213 VM_STAT_ADD(page_exphcontg[12]); 1214 break; 1215 } 1216 /* 1217 * Check whether p_pagenum was modified by DR. 1218 */ 1219 if (pp->p_pagenum != pfn) { 1220 page_unlock(pp); 1221 break; 1222 } 1223 if (pp->p_vnode != vp || 1224 pp->p_offset != off) { 1225 VM_STAT_ADD(page_exphcontg[13]); 1226 page_unlock(pp); 1227 break; 1228 } 1229 if (pp->p_szc >= szc) { 1230 ASSERT(i == 0); 1231 page_unlock(pp); 1232 off = save_off; 1233 goto again; 1234 } 1235 } 1236 1237 if (i != pages) { 1238 VM_STAT_ADD(page_exphcontg[14]); 1239 --pp; 1240 while (i-- > 0) { 1241 page_unlock(pp); 1242 --pp; 1243 } 1244 return (0); 1245 } 1246 1247 pp = rootpp; 1248 for (i = 0; i < pages; i++, pp++) { 1249 if (PP_ISFREE(pp)) { 1250 VM_STAT_ADD(page_exphcontg[15]); 1251 ASSERT(!PP_ISAGED(pp)); 1252 ASSERT(pp->p_szc == 0); 1253 if (!page_reclaim(pp, NULL)) { 1254 break; 1255 } 1256 } else { 1257 ASSERT(pp->p_szc < szc); 1258 VM_STAT_ADD(page_exphcontg[16]); 1259 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1260 } 1261 } 1262 if (i < pages) { 1263 VM_STAT_ADD(page_exphcontg[17]); 1264 /* 1265 * page_reclaim failed because we were out of memory. 1266 * drop the rest of the locks and return because this page 1267 * must be already reallocated anyway. 1268 */ 1269 pp = rootpp; 1270 for (j = 0; j < pages; j++, pp++) { 1271 if (j != i) { 1272 page_unlock(pp); 1273 } 1274 } 1275 return (0); 1276 } 1277 1278 off = save_off; 1279 pp = rootpp; 1280 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1281 ASSERT(PAGE_EXCL(pp)); 1282 ASSERT(!PP_ISFREE(pp)); 1283 ASSERT(!hat_page_is_mapped(pp)); 1284 ASSERT(pp->p_vnode == vp); 1285 ASSERT(pp->p_offset == off); 1286 pp->p_szc = szc; 1287 } 1288 pp = rootpp; 1289 for (i = 0; i < pages; i++, pp++) { 1290 if (ppa == NULL) { 1291 page_unlock(pp); 1292 } else { 1293 ppa[i] = pp; 1294 page_downgrade(ppa[i]); 1295 } 1296 } 1297 if (ppa != NULL) { 1298 ppa[pages] = NULL; 1299 } 1300 VM_STAT_ADD(page_exphcontg[18]); 1301 ASSERT(vp->v_pages != NULL); 1302 return (1); 1303 } 1304 1305 /* 1306 * Determine whether a page with the specified [vp, off] 1307 * currently exists in the system and if so return its 1308 * size code. Obviously this should only be considered as 1309 * a hint since nothing prevents the page from disappearing 1310 * or appearing immediately after the return from this routine. 1311 */ 1312 int 1313 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1314 { 1315 page_t *pp; 1316 kmutex_t *phm; 1317 ulong_t index; 1318 int rc = 0; 1319 1320 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1321 ASSERT(szc != NULL); 1322 VM_STAT_ADD(page_exists_forreal_cnt); 1323 1324 index = PAGE_HASH_FUNC(vp, off); 1325 phm = PAGE_HASH_MUTEX(index); 1326 1327 mutex_enter(phm); 1328 pp = page_hash_search(index, vp, off); 1329 if (pp != NULL) { 1330 *szc = pp->p_szc; 1331 rc = 1; 1332 } 1333 mutex_exit(phm); 1334 return (rc); 1335 } 1336 1337 /* wakeup threads waiting for pages in page_create_get_something() */ 1338 void 1339 wakeup_pcgs(void) 1340 { 1341 if (!CV_HAS_WAITERS(&pcgs_cv)) 1342 return; 1343 cv_broadcast(&pcgs_cv); 1344 } 1345 1346 /* 1347 * 'freemem' is used all over the kernel as an indication of how many 1348 * pages are free (either on the cache list or on the free page list) 1349 * in the system. In very few places is a really accurate 'freemem' 1350 * needed. To avoid contention of the lock protecting a the 1351 * single freemem, it was spread out into NCPU buckets. Set_freemem 1352 * sets freemem to the total of all NCPU buckets. It is called from 1353 * clock() on each TICK. 1354 */ 1355 void 1356 set_freemem(void) 1357 { 1358 struct pcf *p; 1359 ulong_t t; 1360 uint_t i; 1361 1362 t = 0; 1363 p = pcf; 1364 for (i = 0; i < pcf_fanout; i++) { 1365 t += p->pcf_count; 1366 p++; 1367 } 1368 freemem = t; 1369 1370 /* 1371 * Don't worry about grabbing mutex. It's not that 1372 * critical if we miss a tick or two. This is 1373 * where we wakeup possible delayers in 1374 * page_create_get_something(). 1375 */ 1376 wakeup_pcgs(); 1377 } 1378 1379 ulong_t 1380 get_freemem() 1381 { 1382 struct pcf *p; 1383 ulong_t t; 1384 uint_t i; 1385 1386 t = 0; 1387 p = pcf; 1388 for (i = 0; i < pcf_fanout; i++) { 1389 t += p->pcf_count; 1390 p++; 1391 } 1392 /* 1393 * We just calculated it, might as well set it. 1394 */ 1395 freemem = t; 1396 return (t); 1397 } 1398 1399 /* 1400 * Acquire all of the page cache & free (pcf) locks. 1401 */ 1402 void 1403 pcf_acquire_all() 1404 { 1405 struct pcf *p; 1406 uint_t i; 1407 1408 p = pcf; 1409 for (i = 0; i < pcf_fanout; i++) { 1410 mutex_enter(&p->pcf_lock); 1411 p++; 1412 } 1413 } 1414 1415 /* 1416 * Release all the pcf_locks. 1417 */ 1418 void 1419 pcf_release_all() 1420 { 1421 struct pcf *p; 1422 uint_t i; 1423 1424 p = pcf; 1425 for (i = 0; i < pcf_fanout; i++) { 1426 mutex_exit(&p->pcf_lock); 1427 p++; 1428 } 1429 } 1430 1431 /* 1432 * Inform the VM system that we need some pages freed up. 1433 * Calls must be symmetric, e.g.: 1434 * 1435 * page_needfree(100); 1436 * wait a bit; 1437 * page_needfree(-100); 1438 */ 1439 void 1440 page_needfree(spgcnt_t npages) 1441 { 1442 mutex_enter(&new_freemem_lock); 1443 needfree += npages; 1444 mutex_exit(&new_freemem_lock); 1445 } 1446 1447 /* 1448 * Throttle for page_create(): try to prevent freemem from dropping 1449 * below throttlefree. We can't provide a 100% guarantee because 1450 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1451 * nibble away at the freelist. However, we can block all PG_WAIT 1452 * allocations until memory becomes available. The motivation is 1453 * that several things can fall apart when there's no free memory: 1454 * 1455 * (1) If pageout() needs memory to push a page, the system deadlocks. 1456 * 1457 * (2) By (broken) specification, timeout(9F) can neither fail nor 1458 * block, so it has no choice but to panic the system if it 1459 * cannot allocate a callout structure. 1460 * 1461 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1462 * it panics if it cannot allocate a callback structure. 1463 * 1464 * (4) Untold numbers of third-party drivers have not yet been hardened 1465 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1466 * success and panic the system with a data fault on failure. 1467 * (The long-term solution to this particular problem is to ship 1468 * hostile fault-injecting DEBUG kernels with the DDK.) 1469 * 1470 * It is theoretically impossible to guarantee success of non-blocking 1471 * allocations, but in practice, this throttle is very hard to break. 1472 */ 1473 static int 1474 page_create_throttle(pgcnt_t npages, int flags) 1475 { 1476 ulong_t fm; 1477 uint_t i; 1478 pgcnt_t tf; /* effective value of throttlefree */ 1479 1480 /* 1481 * Normal priority allocations. 1482 */ 1483 if ((flags & (PG_WAIT | PG_NORMALPRI)) == PG_NORMALPRI) { 1484 ASSERT(!(flags & (PG_PANIC | PG_PUSHPAGE))); 1485 return (freemem >= npages + throttlefree); 1486 } 1487 1488 /* 1489 * Never deny pages when: 1490 * - it's a thread that cannot block [NOMEMWAIT()] 1491 * - the allocation cannot block and must not fail 1492 * - the allocation cannot block and is pageout dispensated 1493 */ 1494 if (NOMEMWAIT() || 1495 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1496 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1497 return (1); 1498 1499 /* 1500 * If the allocation can't block, we look favorably upon it 1501 * unless we're below pageout_reserve. In that case we fail 1502 * the allocation because we want to make sure there are a few 1503 * pages available for pageout. 1504 */ 1505 if ((flags & PG_WAIT) == 0) 1506 return (freemem >= npages + pageout_reserve); 1507 1508 /* Calculate the effective throttlefree value */ 1509 tf = throttlefree - 1510 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1511 1512 cv_signal(&proc_pageout->p_cv); 1513 1514 for (;;) { 1515 fm = 0; 1516 pcf_acquire_all(); 1517 mutex_enter(&new_freemem_lock); 1518 for (i = 0; i < pcf_fanout; i++) { 1519 fm += pcf[i].pcf_count; 1520 pcf[i].pcf_wait++; 1521 mutex_exit(&pcf[i].pcf_lock); 1522 } 1523 freemem = fm; 1524 if (freemem >= npages + tf) { 1525 mutex_exit(&new_freemem_lock); 1526 break; 1527 } 1528 needfree += npages; 1529 freemem_wait++; 1530 cv_wait(&freemem_cv, &new_freemem_lock); 1531 freemem_wait--; 1532 needfree -= npages; 1533 mutex_exit(&new_freemem_lock); 1534 } 1535 return (1); 1536 } 1537 1538 /* 1539 * page_create_wait() is called to either coalesce pages from the 1540 * different pcf buckets or to wait because there simply are not 1541 * enough pages to satisfy the caller's request. 1542 * 1543 * Sadly, this is called from platform/vm/vm_machdep.c 1544 */ 1545 int 1546 page_create_wait(pgcnt_t npages, uint_t flags) 1547 { 1548 pgcnt_t total; 1549 uint_t i; 1550 struct pcf *p; 1551 1552 /* 1553 * Wait until there are enough free pages to satisfy our 1554 * entire request. 1555 * We set needfree += npages before prodding pageout, to make sure 1556 * it does real work when npages > lotsfree > freemem. 1557 */ 1558 VM_STAT_ADD(page_create_not_enough); 1559 1560 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1561 checkagain: 1562 if ((flags & PG_NORELOC) && 1563 kcage_freemem < kcage_throttlefree + npages) 1564 (void) kcage_create_throttle(npages, flags); 1565 1566 if (freemem < npages + throttlefree) 1567 if (!page_create_throttle(npages, flags)) 1568 return (0); 1569 1570 if (pcf_decrement_bucket(npages) || 1571 pcf_decrement_multiple(&total, npages, 0)) 1572 return (1); 1573 1574 /* 1575 * All of the pcf locks are held, there are not enough pages 1576 * to satisfy the request (npages < total). 1577 * Be sure to acquire the new_freemem_lock before dropping 1578 * the pcf locks. This prevents dropping wakeups in page_free(). 1579 * The order is always pcf_lock then new_freemem_lock. 1580 * 1581 * Since we hold all the pcf locks, it is a good time to set freemem. 1582 * 1583 * If the caller does not want to wait, return now. 1584 * Else turn the pageout daemon loose to find something 1585 * and wait till it does. 1586 * 1587 */ 1588 freemem = total; 1589 1590 if ((flags & PG_WAIT) == 0) { 1591 pcf_release_all(); 1592 1593 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1594 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1595 return (0); 1596 } 1597 1598 ASSERT(proc_pageout != NULL); 1599 cv_signal(&proc_pageout->p_cv); 1600 1601 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1602 "page_create_sleep_start: freemem %ld needfree %ld", 1603 freemem, needfree); 1604 1605 /* 1606 * We are going to wait. 1607 * We currently hold all of the pcf_locks, 1608 * get the new_freemem_lock (it protects freemem_wait), 1609 * before dropping the pcf_locks. 1610 */ 1611 mutex_enter(&new_freemem_lock); 1612 1613 p = pcf; 1614 for (i = 0; i < pcf_fanout; i++) { 1615 p->pcf_wait++; 1616 mutex_exit(&p->pcf_lock); 1617 p++; 1618 } 1619 1620 needfree += npages; 1621 freemem_wait++; 1622 1623 cv_wait(&freemem_cv, &new_freemem_lock); 1624 1625 freemem_wait--; 1626 needfree -= npages; 1627 1628 mutex_exit(&new_freemem_lock); 1629 1630 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1631 "page_create_sleep_end: freemem %ld needfree %ld", 1632 freemem, needfree); 1633 1634 VM_STAT_ADD(page_create_not_enough_again); 1635 goto checkagain; 1636 } 1637 /* 1638 * A routine to do the opposite of page_create_wait(). 1639 */ 1640 void 1641 page_create_putback(spgcnt_t npages) 1642 { 1643 struct pcf *p; 1644 pgcnt_t lump; 1645 uint_t *which; 1646 1647 /* 1648 * When a contiguous lump is broken up, we have to 1649 * deal with lots of pages (min 64) so lets spread 1650 * the wealth around. 1651 */ 1652 lump = roundup(npages, pcf_fanout) / pcf_fanout; 1653 freemem += npages; 1654 1655 for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) { 1656 which = &p->pcf_count; 1657 1658 mutex_enter(&p->pcf_lock); 1659 1660 if (p->pcf_block) { 1661 which = &p->pcf_reserve; 1662 } 1663 1664 if (lump < npages) { 1665 *which += (uint_t)lump; 1666 npages -= lump; 1667 } else { 1668 *which += (uint_t)npages; 1669 npages = 0; 1670 } 1671 1672 if (p->pcf_wait) { 1673 mutex_enter(&new_freemem_lock); 1674 /* 1675 * Check to see if some other thread 1676 * is actually waiting. Another bucket 1677 * may have woken it up by now. If there 1678 * are no waiters, then set our pcf_wait 1679 * count to zero to avoid coming in here 1680 * next time. 1681 */ 1682 if (freemem_wait) { 1683 if (npages > 1) { 1684 cv_broadcast(&freemem_cv); 1685 } else { 1686 cv_signal(&freemem_cv); 1687 } 1688 p->pcf_wait--; 1689 } else { 1690 p->pcf_wait = 0; 1691 } 1692 mutex_exit(&new_freemem_lock); 1693 } 1694 mutex_exit(&p->pcf_lock); 1695 } 1696 ASSERT(npages == 0); 1697 } 1698 1699 /* 1700 * A helper routine for page_create_get_something. 1701 * The indenting got to deep down there. 1702 * Unblock the pcf counters. Any pages freed after 1703 * pcf_block got set are moved to pcf_count and 1704 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1705 */ 1706 static void 1707 pcgs_unblock(void) 1708 { 1709 int i; 1710 struct pcf *p; 1711 1712 /* Update freemem while we're here. */ 1713 freemem = 0; 1714 p = pcf; 1715 for (i = 0; i < pcf_fanout; i++) { 1716 mutex_enter(&p->pcf_lock); 1717 ASSERT(p->pcf_count == 0); 1718 p->pcf_count = p->pcf_reserve; 1719 p->pcf_block = 0; 1720 freemem += p->pcf_count; 1721 if (p->pcf_wait) { 1722 mutex_enter(&new_freemem_lock); 1723 if (freemem_wait) { 1724 if (p->pcf_reserve > 1) { 1725 cv_broadcast(&freemem_cv); 1726 p->pcf_wait = 0; 1727 } else { 1728 cv_signal(&freemem_cv); 1729 p->pcf_wait--; 1730 } 1731 } else { 1732 p->pcf_wait = 0; 1733 } 1734 mutex_exit(&new_freemem_lock); 1735 } 1736 p->pcf_reserve = 0; 1737 mutex_exit(&p->pcf_lock); 1738 p++; 1739 } 1740 } 1741 1742 /* 1743 * Called from page_create_va() when both the cache and free lists 1744 * have been checked once. 1745 * 1746 * Either returns a page or panics since the accounting was done 1747 * way before we got here. 1748 * 1749 * We don't come here often, so leave the accounting on permanently. 1750 */ 1751 1752 #define MAX_PCGS 100 1753 1754 #ifdef DEBUG 1755 #define PCGS_TRIES 100 1756 #else /* DEBUG */ 1757 #define PCGS_TRIES 10 1758 #endif /* DEBUG */ 1759 1760 #ifdef VM_STATS 1761 uint_t pcgs_counts[PCGS_TRIES]; 1762 uint_t pcgs_too_many; 1763 uint_t pcgs_entered; 1764 uint_t pcgs_entered_noreloc; 1765 uint_t pcgs_locked; 1766 uint_t pcgs_cagelocked; 1767 #endif /* VM_STATS */ 1768 1769 static page_t * 1770 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1771 caddr_t vaddr, uint_t flags) 1772 { 1773 uint_t count; 1774 page_t *pp; 1775 uint_t locked, i; 1776 struct pcf *p; 1777 lgrp_t *lgrp; 1778 int cagelocked = 0; 1779 1780 VM_STAT_ADD(pcgs_entered); 1781 1782 /* 1783 * Tap any reserve freelists: if we fail now, we'll die 1784 * since the page(s) we're looking for have already been 1785 * accounted for. 1786 */ 1787 flags |= PG_PANIC; 1788 1789 if ((flags & PG_NORELOC) != 0) { 1790 VM_STAT_ADD(pcgs_entered_noreloc); 1791 /* 1792 * Requests for free pages from critical threads 1793 * such as pageout still won't throttle here, but 1794 * we must try again, to give the cageout thread 1795 * another chance to catch up. Since we already 1796 * accounted for the pages, we had better get them 1797 * this time. 1798 * 1799 * N.B. All non-critical threads acquire the pcgs_cagelock 1800 * to serialize access to the freelists. This implements a 1801 * turnstile-type synchornization to avoid starvation of 1802 * critical requests for PG_NORELOC memory by non-critical 1803 * threads: all non-critical threads must acquire a 'ticket' 1804 * before passing through, which entails making sure 1805 * kcage_freemem won't fall below minfree prior to grabbing 1806 * pages from the freelists. 1807 */ 1808 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1809 mutex_enter(&pcgs_cagelock); 1810 cagelocked = 1; 1811 VM_STAT_ADD(pcgs_cagelocked); 1812 } 1813 } 1814 1815 /* 1816 * Time to get serious. 1817 * We failed to get a `correctly colored' page from both the 1818 * free and cache lists. 1819 * We escalate in stage. 1820 * 1821 * First try both lists without worring about color. 1822 * 1823 * Then, grab all page accounting locks (ie. pcf[]) and 1824 * steal any pages that they have and set the pcf_block flag to 1825 * stop deletions from the lists. This will help because 1826 * a page can get added to the free list while we are looking 1827 * at the cache list, then another page could be added to the cache 1828 * list allowing the page on the free list to be removed as we 1829 * move from looking at the cache list to the free list. This 1830 * could happen over and over. We would never find the page 1831 * we have accounted for. 1832 * 1833 * Noreloc pages are a subset of the global (relocatable) page pool. 1834 * They are not tracked separately in the pcf bins, so it is 1835 * impossible to know when doing pcf accounting if the available 1836 * page(s) are noreloc pages or not. When looking for a noreloc page 1837 * it is quite easy to end up here even if the global (relocatable) 1838 * page pool has plenty of free pages but the noreloc pool is empty. 1839 * 1840 * When the noreloc pool is empty (or low), additional noreloc pages 1841 * are created by converting pages from the global page pool. This 1842 * process will stall during pcf accounting if the pcf bins are 1843 * already locked. Such is the case when a noreloc allocation is 1844 * looping here in page_create_get_something waiting for more noreloc 1845 * pages to appear. 1846 * 1847 * Short of adding a new field to the pcf bins to accurately track 1848 * the number of free noreloc pages, we instead do not grab the 1849 * pcgs_lock, do not set the pcf blocks and do not timeout when 1850 * allocating a noreloc page. This allows noreloc allocations to 1851 * loop without blocking global page pool allocations. 1852 * 1853 * NOTE: the behaviour of page_create_get_something has not changed 1854 * for the case of global page pool allocations. 1855 */ 1856 1857 flags &= ~PG_MATCH_COLOR; 1858 locked = 0; 1859 #if defined(__x86) 1860 flags = page_create_update_flags_x86(flags); 1861 #endif 1862 1863 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1864 1865 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1866 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1867 flags, lgrp); 1868 if (pp == NULL) { 1869 pp = page_get_cachelist(vp, off, seg, vaddr, 1870 flags, lgrp); 1871 } 1872 if (pp == NULL) { 1873 /* 1874 * Serialize. Don't fight with other pcgs(). 1875 */ 1876 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1877 mutex_enter(&pcgs_lock); 1878 VM_STAT_ADD(pcgs_locked); 1879 locked = 1; 1880 p = pcf; 1881 for (i = 0; i < pcf_fanout; i++) { 1882 mutex_enter(&p->pcf_lock); 1883 ASSERT(p->pcf_block == 0); 1884 p->pcf_block = 1; 1885 p->pcf_reserve = p->pcf_count; 1886 p->pcf_count = 0; 1887 mutex_exit(&p->pcf_lock); 1888 p++; 1889 } 1890 freemem = 0; 1891 } 1892 1893 if (count) { 1894 /* 1895 * Since page_free() puts pages on 1896 * a list then accounts for it, we 1897 * just have to wait for page_free() 1898 * to unlock any page it was working 1899 * with. The page_lock()-page_reclaim() 1900 * path falls in the same boat. 1901 * 1902 * We don't need to check on the 1903 * PG_WAIT flag, we have already 1904 * accounted for the page we are 1905 * looking for in page_create_va(). 1906 * 1907 * We just wait a moment to let any 1908 * locked pages on the lists free up, 1909 * then continue around and try again. 1910 * 1911 * Will be awakened by set_freemem(). 1912 */ 1913 mutex_enter(&pcgs_wait_lock); 1914 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1915 mutex_exit(&pcgs_wait_lock); 1916 } 1917 } else { 1918 #ifdef VM_STATS 1919 if (count >= PCGS_TRIES) { 1920 VM_STAT_ADD(pcgs_too_many); 1921 } else { 1922 VM_STAT_ADD(pcgs_counts[count]); 1923 } 1924 #endif 1925 if (locked) { 1926 pcgs_unblock(); 1927 mutex_exit(&pcgs_lock); 1928 } 1929 if (cagelocked) 1930 mutex_exit(&pcgs_cagelock); 1931 return (pp); 1932 } 1933 } 1934 /* 1935 * we go down holding the pcf locks. 1936 */ 1937 panic("no %spage found %d", 1938 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1939 /*NOTREACHED*/ 1940 } 1941 1942 /* 1943 * Create enough pages for "bytes" worth of data starting at 1944 * "off" in "vp". 1945 * 1946 * Where flag must be one of: 1947 * 1948 * PG_EXCL: Exclusive create (fail if any page already 1949 * exists in the page cache) which does not 1950 * wait for memory to become available. 1951 * 1952 * PG_WAIT: Non-exclusive create which can wait for 1953 * memory to become available. 1954 * 1955 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1956 * (Not Supported) 1957 * 1958 * A doubly linked list of pages is returned to the caller. Each page 1959 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1960 * lock. 1961 * 1962 * Unable to change the parameters to page_create() in a minor release, 1963 * we renamed page_create() to page_create_va(), changed all known calls 1964 * from page_create() to page_create_va(), and created this wrapper. 1965 * 1966 * Upon a major release, we should break compatibility by deleting this 1967 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1968 * 1969 * NOTE: There is a copy of this interface as page_create_io() in 1970 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1971 * there. 1972 */ 1973 page_t * 1974 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1975 { 1976 caddr_t random_vaddr; 1977 struct seg kseg; 1978 1979 #ifdef DEBUG 1980 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1981 (void *)caller()); 1982 #endif 1983 1984 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1985 (uintptr_t)(off >> PAGESHIFT)); 1986 kseg.s_as = &kas; 1987 1988 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1989 } 1990 1991 #ifdef DEBUG 1992 uint32_t pg_alloc_pgs_mtbf = 0; 1993 #endif 1994 1995 /* 1996 * Used for large page support. It will attempt to allocate 1997 * a large page(s) off the freelist. 1998 * 1999 * Returns non zero on failure. 2000 */ 2001 int 2002 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 2003 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags) 2004 { 2005 pgcnt_t npgs, curnpgs, totpgs; 2006 size_t pgsz; 2007 page_t *pplist = NULL, *pp; 2008 int err = 0; 2009 lgrp_t *lgrp; 2010 2011 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 2012 ASSERT(pgflags == 0 || pgflags == PG_LOCAL); 2013 2014 /* 2015 * Check if system heavily prefers local large pages over remote 2016 * on systems with multiple lgroups. 2017 */ 2018 if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) { 2019 pgflags = PG_LOCAL; 2020 } 2021 2022 VM_STAT_ADD(alloc_pages[0]); 2023 2024 #ifdef DEBUG 2025 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 2026 return (ENOMEM); 2027 } 2028 #endif 2029 2030 /* 2031 * One must be NULL but not both. 2032 * And one must be non NULL but not both. 2033 */ 2034 ASSERT(basepp != NULL || ppa != NULL); 2035 ASSERT(basepp == NULL || ppa == NULL); 2036 2037 #if defined(__x86) 2038 while (page_chk_freelist(szc) == 0) { 2039 VM_STAT_ADD(alloc_pages[8]); 2040 if (anypgsz == 0 || --szc == 0) 2041 return (ENOMEM); 2042 } 2043 #endif 2044 2045 pgsz = page_get_pagesize(szc); 2046 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 2047 2048 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 2049 2050 (void) page_create_wait(npgs, PG_WAIT); 2051 2052 while (npgs && szc) { 2053 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2054 if (pgflags == PG_LOCAL) { 2055 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2056 pgflags, lgrp); 2057 if (pp == NULL) { 2058 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2059 0, lgrp); 2060 } 2061 } else { 2062 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2063 0, lgrp); 2064 } 2065 if (pp != NULL) { 2066 VM_STAT_ADD(alloc_pages[1]); 2067 page_list_concat(&pplist, &pp); 2068 ASSERT(npgs >= curnpgs); 2069 npgs -= curnpgs; 2070 } else if (anypgsz) { 2071 VM_STAT_ADD(alloc_pages[2]); 2072 szc--; 2073 pgsz = page_get_pagesize(szc); 2074 curnpgs = pgsz >> PAGESHIFT; 2075 } else { 2076 VM_STAT_ADD(alloc_pages[3]); 2077 ASSERT(npgs == totpgs); 2078 page_create_putback(npgs); 2079 return (ENOMEM); 2080 } 2081 } 2082 if (szc == 0) { 2083 VM_STAT_ADD(alloc_pages[4]); 2084 ASSERT(npgs != 0); 2085 page_create_putback(npgs); 2086 err = ENOMEM; 2087 } else if (basepp != NULL) { 2088 ASSERT(npgs == 0); 2089 ASSERT(ppa == NULL); 2090 *basepp = pplist; 2091 } 2092 2093 npgs = totpgs - npgs; 2094 pp = pplist; 2095 2096 /* 2097 * Clear the free and age bits. Also if we were passed in a ppa then 2098 * fill it in with all the constituent pages from the large page. But 2099 * if we failed to allocate all the pages just free what we got. 2100 */ 2101 while (npgs != 0) { 2102 ASSERT(PP_ISFREE(pp)); 2103 ASSERT(PP_ISAGED(pp)); 2104 if (ppa != NULL || err != 0) { 2105 if (err == 0) { 2106 VM_STAT_ADD(alloc_pages[5]); 2107 PP_CLRFREE(pp); 2108 PP_CLRAGED(pp); 2109 page_sub(&pplist, pp); 2110 *ppa++ = pp; 2111 npgs--; 2112 } else { 2113 VM_STAT_ADD(alloc_pages[6]); 2114 ASSERT(pp->p_szc != 0); 2115 curnpgs = page_get_pagecnt(pp->p_szc); 2116 page_list_break(&pp, &pplist, curnpgs); 2117 page_list_add_pages(pp, 0); 2118 page_create_putback(curnpgs); 2119 ASSERT(npgs >= curnpgs); 2120 npgs -= curnpgs; 2121 } 2122 pp = pplist; 2123 } else { 2124 VM_STAT_ADD(alloc_pages[7]); 2125 PP_CLRFREE(pp); 2126 PP_CLRAGED(pp); 2127 pp = pp->p_next; 2128 npgs--; 2129 } 2130 } 2131 return (err); 2132 } 2133 2134 /* 2135 * Get a single large page off of the freelists, and set it up for use. 2136 * Number of bytes requested must be a supported page size. 2137 * 2138 * Note that this call may fail even if there is sufficient 2139 * memory available or PG_WAIT is set, so the caller must 2140 * be willing to fallback on page_create_va(), block and retry, 2141 * or fail the requester. 2142 */ 2143 page_t * 2144 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2145 struct seg *seg, caddr_t vaddr, void *arg) 2146 { 2147 pgcnt_t npages; 2148 page_t *pp; 2149 page_t *rootpp; 2150 lgrp_t *lgrp; 2151 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2152 2153 ASSERT(vp != NULL); 2154 2155 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2156 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0); 2157 /* but no others */ 2158 2159 ASSERT((flags & PG_EXCL) == PG_EXCL); 2160 2161 npages = btop(bytes); 2162 2163 if (!kcage_on || panicstr) { 2164 /* 2165 * Cage is OFF, or we are single threaded in 2166 * panic, so make everything a RELOC request. 2167 */ 2168 flags &= ~PG_NORELOC; 2169 } 2170 2171 /* 2172 * Make sure there's adequate physical memory available. 2173 * Note: PG_WAIT is ignored here. 2174 */ 2175 if (freemem <= throttlefree + npages) { 2176 VM_STAT_ADD(page_create_large_cnt[1]); 2177 return (NULL); 2178 } 2179 2180 /* 2181 * If cage is on, dampen draw from cage when available 2182 * cage space is low. 2183 */ 2184 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2185 kcage_freemem < kcage_throttlefree + npages) { 2186 2187 /* 2188 * The cage is on, the caller wants PG_NORELOC 2189 * pages and available cage memory is very low. 2190 * Call kcage_create_throttle() to attempt to 2191 * control demand on the cage. 2192 */ 2193 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2194 VM_STAT_ADD(page_create_large_cnt[2]); 2195 return (NULL); 2196 } 2197 } 2198 2199 if (!pcf_decrement_bucket(npages) && 2200 !pcf_decrement_multiple(NULL, npages, 1)) { 2201 VM_STAT_ADD(page_create_large_cnt[4]); 2202 return (NULL); 2203 } 2204 2205 /* 2206 * This is where this function behaves fundamentally differently 2207 * than page_create_va(); since we're intending to map the page 2208 * with a single TTE, we have to get it as a physically contiguous 2209 * hardware pagesize chunk. If we can't, we fail. 2210 */ 2211 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2212 LGRP_EXISTS(lgrp_table[*lgrpid])) 2213 lgrp = lgrp_table[*lgrpid]; 2214 else 2215 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2216 2217 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2218 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2219 page_create_putback(npages); 2220 VM_STAT_ADD(page_create_large_cnt[5]); 2221 return (NULL); 2222 } 2223 2224 /* 2225 * if we got the page with the wrong mtype give it back this is a 2226 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2227 * inside "if" and the workaround becomes just a nop 2228 */ 2229 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2230 page_list_add_pages(rootpp, 0); 2231 page_create_putback(npages); 2232 VM_STAT_ADD(page_create_large_cnt[6]); 2233 return (NULL); 2234 } 2235 2236 /* 2237 * If satisfying this request has left us with too little 2238 * memory, start the wheels turning to get some back. The 2239 * first clause of the test prevents waking up the pageout 2240 * daemon in situations where it would decide that there's 2241 * nothing to do. 2242 */ 2243 if (nscan < desscan && freemem < minfree) { 2244 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2245 "pageout_cv_signal:freemem %ld", freemem); 2246 cv_signal(&proc_pageout->p_cv); 2247 } 2248 2249 pp = rootpp; 2250 while (npages--) { 2251 ASSERT(PAGE_EXCL(pp)); 2252 ASSERT(pp->p_vnode == NULL); 2253 ASSERT(!hat_page_is_mapped(pp)); 2254 PP_CLRFREE(pp); 2255 PP_CLRAGED(pp); 2256 if (!page_hashin(pp, vp, off, NULL)) 2257 panic("page_create_large: hashin failed: page %p", 2258 (void *)pp); 2259 page_io_lock(pp); 2260 off += PAGESIZE; 2261 pp = pp->p_next; 2262 } 2263 2264 VM_STAT_ADD(page_create_large_cnt[0]); 2265 return (rootpp); 2266 } 2267 2268 page_t * 2269 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2270 struct seg *seg, caddr_t vaddr) 2271 { 2272 page_t *plist = NULL; 2273 pgcnt_t npages; 2274 pgcnt_t found_on_free = 0; 2275 pgcnt_t pages_req; 2276 page_t *npp = NULL; 2277 struct pcf *p; 2278 lgrp_t *lgrp; 2279 2280 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2281 "page_create_start:vp %p off %llx bytes %lu flags %x", 2282 vp, off, bytes, flags); 2283 2284 ASSERT(bytes != 0 && vp != NULL); 2285 2286 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2287 panic("page_create: invalid flags"); 2288 /*NOTREACHED*/ 2289 } 2290 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2291 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0); 2292 /* but no others */ 2293 2294 pages_req = npages = btopr(bytes); 2295 /* 2296 * Try to see whether request is too large to *ever* be 2297 * satisfied, in order to prevent deadlock. We arbitrarily 2298 * decide to limit maximum size requests to max_page_get. 2299 */ 2300 if (npages >= max_page_get) { 2301 if ((flags & PG_WAIT) == 0) { 2302 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2303 "page_create_toobig:vp %p off %llx npages " 2304 "%lu max_page_get %lu", 2305 vp, off, npages, max_page_get); 2306 return (NULL); 2307 } else { 2308 cmn_err(CE_WARN, 2309 "Request for too much kernel memory " 2310 "(%lu bytes), will hang forever", bytes); 2311 for (;;) 2312 delay(1000000000); 2313 } 2314 } 2315 2316 if (!kcage_on || panicstr) { 2317 /* 2318 * Cage is OFF, or we are single threaded in 2319 * panic, so make everything a RELOC request. 2320 */ 2321 flags &= ~PG_NORELOC; 2322 } 2323 2324 if (freemem <= throttlefree + npages) 2325 if (!page_create_throttle(npages, flags)) 2326 return (NULL); 2327 2328 /* 2329 * If cage is on, dampen draw from cage when available 2330 * cage space is low. 2331 */ 2332 if ((flags & PG_NORELOC) && 2333 kcage_freemem < kcage_throttlefree + npages) { 2334 2335 /* 2336 * The cage is on, the caller wants PG_NORELOC 2337 * pages and available cage memory is very low. 2338 * Call kcage_create_throttle() to attempt to 2339 * control demand on the cage. 2340 */ 2341 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2342 return (NULL); 2343 } 2344 2345 VM_STAT_ADD(page_create_cnt[0]); 2346 2347 if (!pcf_decrement_bucket(npages)) { 2348 /* 2349 * Have to look harder. If npages is greater than 2350 * one, then we might have to coalesce the counters. 2351 * 2352 * Go wait. We come back having accounted 2353 * for the memory. 2354 */ 2355 VM_STAT_ADD(page_create_cnt[1]); 2356 if (!page_create_wait(npages, flags)) { 2357 VM_STAT_ADD(page_create_cnt[2]); 2358 return (NULL); 2359 } 2360 } 2361 2362 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2363 "page_create_success:vp %p off %llx", vp, off); 2364 2365 /* 2366 * If satisfying this request has left us with too little 2367 * memory, start the wheels turning to get some back. The 2368 * first clause of the test prevents waking up the pageout 2369 * daemon in situations where it would decide that there's 2370 * nothing to do. 2371 */ 2372 if (nscan < desscan && freemem < minfree) { 2373 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2374 "pageout_cv_signal:freemem %ld", freemem); 2375 cv_signal(&proc_pageout->p_cv); 2376 } 2377 2378 /* 2379 * Loop around collecting the requested number of pages. 2380 * Most of the time, we have to `create' a new page. With 2381 * this in mind, pull the page off the free list before 2382 * getting the hash lock. This will minimize the hash 2383 * lock hold time, nesting, and the like. If it turns 2384 * out we don't need the page, we put it back at the end. 2385 */ 2386 while (npages--) { 2387 page_t *pp; 2388 kmutex_t *phm = NULL; 2389 ulong_t index; 2390 2391 index = PAGE_HASH_FUNC(vp, off); 2392 top: 2393 ASSERT(phm == NULL); 2394 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2395 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2396 2397 if (npp == NULL) { 2398 /* 2399 * Try to get a page from the freelist (ie, 2400 * a page with no [vp, off] tag). If that 2401 * fails, use the cachelist. 2402 * 2403 * During the first attempt at both the free 2404 * and cache lists we try for the correct color. 2405 */ 2406 /* 2407 * XXXX-how do we deal with virtual indexed 2408 * caches and and colors? 2409 */ 2410 VM_STAT_ADD(page_create_cnt[4]); 2411 /* 2412 * Get lgroup to allocate next page of shared memory 2413 * from and use it to specify where to allocate 2414 * the physical memory 2415 */ 2416 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2417 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2418 flags | PG_MATCH_COLOR, lgrp); 2419 if (npp == NULL) { 2420 npp = page_get_cachelist(vp, off, seg, 2421 vaddr, flags | PG_MATCH_COLOR, lgrp); 2422 if (npp == NULL) { 2423 npp = page_create_get_something(vp, 2424 off, seg, vaddr, 2425 flags & ~PG_MATCH_COLOR); 2426 } 2427 2428 if (PP_ISAGED(npp) == 0) { 2429 /* 2430 * Since this page came from the 2431 * cachelist, we must destroy the 2432 * old vnode association. 2433 */ 2434 page_hashout(npp, NULL); 2435 } 2436 } 2437 } 2438 2439 /* 2440 * We own this page! 2441 */ 2442 ASSERT(PAGE_EXCL(npp)); 2443 ASSERT(npp->p_vnode == NULL); 2444 ASSERT(!hat_page_is_mapped(npp)); 2445 PP_CLRFREE(npp); 2446 PP_CLRAGED(npp); 2447 2448 /* 2449 * Here we have a page in our hot little mits and are 2450 * just waiting to stuff it on the appropriate lists. 2451 * Get the mutex and check to see if it really does 2452 * not exist. 2453 */ 2454 phm = PAGE_HASH_MUTEX(index); 2455 mutex_enter(phm); 2456 pp = page_hash_search(index, vp, off); 2457 if (pp == NULL) { 2458 VM_STAT_ADD(page_create_new); 2459 pp = npp; 2460 npp = NULL; 2461 if (!page_hashin(pp, vp, off, phm)) { 2462 /* 2463 * Since we hold the page hash mutex and 2464 * just searched for this page, page_hashin 2465 * had better not fail. If it does, that 2466 * means somethread did not follow the 2467 * page hash mutex rules. Panic now and 2468 * get it over with. As usual, go down 2469 * holding all the locks. 2470 */ 2471 ASSERT(MUTEX_HELD(phm)); 2472 panic("page_create: " 2473 "hashin failed %p %p %llx %p", 2474 (void *)pp, (void *)vp, off, (void *)phm); 2475 /*NOTREACHED*/ 2476 } 2477 ASSERT(MUTEX_HELD(phm)); 2478 mutex_exit(phm); 2479 phm = NULL; 2480 2481 /* 2482 * Hat layer locking need not be done to set 2483 * the following bits since the page is not hashed 2484 * and was on the free list (i.e., had no mappings). 2485 * 2486 * Set the reference bit to protect 2487 * against immediate pageout 2488 * 2489 * XXXmh modify freelist code to set reference 2490 * bit so we don't have to do it here. 2491 */ 2492 page_set_props(pp, P_REF); 2493 found_on_free++; 2494 } else { 2495 VM_STAT_ADD(page_create_exists); 2496 if (flags & PG_EXCL) { 2497 /* 2498 * Found an existing page, and the caller 2499 * wanted all new pages. Undo all of the work 2500 * we have done. 2501 */ 2502 mutex_exit(phm); 2503 phm = NULL; 2504 while (plist != NULL) { 2505 pp = plist; 2506 page_sub(&plist, pp); 2507 page_io_unlock(pp); 2508 /* large pages should not end up here */ 2509 ASSERT(pp->p_szc == 0); 2510 /*LINTED: constant in conditional ctx*/ 2511 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2512 } 2513 VM_STAT_ADD(page_create_found_one); 2514 goto fail; 2515 } 2516 ASSERT(flags & PG_WAIT); 2517 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2518 /* 2519 * Start all over again if we blocked trying 2520 * to lock the page. 2521 */ 2522 mutex_exit(phm); 2523 VM_STAT_ADD(page_create_page_lock_failed); 2524 phm = NULL; 2525 goto top; 2526 } 2527 mutex_exit(phm); 2528 phm = NULL; 2529 2530 if (PP_ISFREE(pp)) { 2531 ASSERT(PP_ISAGED(pp) == 0); 2532 VM_STAT_ADD(pagecnt.pc_get_cache); 2533 page_list_sub(pp, PG_CACHE_LIST); 2534 PP_CLRFREE(pp); 2535 found_on_free++; 2536 } 2537 } 2538 2539 /* 2540 * Got a page! It is locked. Acquire the i/o 2541 * lock since we are going to use the p_next and 2542 * p_prev fields to link the requested pages together. 2543 */ 2544 page_io_lock(pp); 2545 page_add(&plist, pp); 2546 plist = plist->p_next; 2547 off += PAGESIZE; 2548 vaddr += PAGESIZE; 2549 } 2550 2551 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2552 fail: 2553 if (npp != NULL) { 2554 /* 2555 * Did not need this page after all. 2556 * Put it back on the free list. 2557 */ 2558 VM_STAT_ADD(page_create_putbacks); 2559 PP_SETFREE(npp); 2560 PP_SETAGED(npp); 2561 npp->p_offset = (u_offset_t)-1; 2562 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2563 page_unlock(npp); 2564 2565 } 2566 2567 ASSERT(pages_req >= found_on_free); 2568 2569 { 2570 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2571 2572 if (overshoot) { 2573 VM_STAT_ADD(page_create_overshoot); 2574 p = &pcf[PCF_INDEX()]; 2575 mutex_enter(&p->pcf_lock); 2576 if (p->pcf_block) { 2577 p->pcf_reserve += overshoot; 2578 } else { 2579 p->pcf_count += overshoot; 2580 if (p->pcf_wait) { 2581 mutex_enter(&new_freemem_lock); 2582 if (freemem_wait) { 2583 cv_signal(&freemem_cv); 2584 p->pcf_wait--; 2585 } else { 2586 p->pcf_wait = 0; 2587 } 2588 mutex_exit(&new_freemem_lock); 2589 } 2590 } 2591 mutex_exit(&p->pcf_lock); 2592 /* freemem is approximate, so this test OK */ 2593 if (!p->pcf_block) 2594 freemem += overshoot; 2595 } 2596 } 2597 2598 return (plist); 2599 } 2600 2601 /* 2602 * One or more constituent pages of this large page has been marked 2603 * toxic. Simply demote the large page to PAGESIZE pages and let 2604 * page_free() handle it. This routine should only be called by 2605 * large page free routines (page_free_pages() and page_destroy_pages(). 2606 * All pages are locked SE_EXCL and have already been marked free. 2607 */ 2608 static void 2609 page_free_toxic_pages(page_t *rootpp) 2610 { 2611 page_t *tpp; 2612 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2613 uint_t szc = rootpp->p_szc; 2614 2615 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2616 ASSERT(tpp->p_szc == szc); 2617 ASSERT((PAGE_EXCL(tpp) && 2618 !page_iolock_assert(tpp)) || panicstr); 2619 tpp->p_szc = 0; 2620 } 2621 2622 while (rootpp != NULL) { 2623 tpp = rootpp; 2624 page_sub(&rootpp, tpp); 2625 ASSERT(PP_ISFREE(tpp)); 2626 PP_CLRFREE(tpp); 2627 page_free(tpp, 1); 2628 } 2629 } 2630 2631 /* 2632 * Put page on the "free" list. 2633 * The free list is really two lists maintained by 2634 * the PSM of whatever machine we happen to be on. 2635 */ 2636 void 2637 page_free(page_t *pp, int dontneed) 2638 { 2639 struct pcf *p; 2640 uint_t pcf_index; 2641 2642 ASSERT((PAGE_EXCL(pp) && 2643 !page_iolock_assert(pp)) || panicstr); 2644 2645 if (PP_ISFREE(pp)) { 2646 panic("page_free: page %p is free", (void *)pp); 2647 } 2648 2649 if (pp->p_szc != 0) { 2650 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2651 PP_ISKAS(pp)) { 2652 panic("page_free: anon or kernel " 2653 "or no vnode large page %p", (void *)pp); 2654 } 2655 page_demote_vp_pages(pp); 2656 ASSERT(pp->p_szc == 0); 2657 } 2658 2659 /* 2660 * The page_struct_lock need not be acquired to examine these 2661 * fields since the page has an "exclusive" lock. 2662 */ 2663 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 2664 pp->p_slckcnt != 0) { 2665 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 2666 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt, 2667 pp->p_cowcnt, pp->p_slckcnt); 2668 /*NOTREACHED*/ 2669 } 2670 2671 ASSERT(!hat_page_getshare(pp)); 2672 2673 PP_SETFREE(pp); 2674 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2675 !hat_ismod(pp)); 2676 page_clr_all_props(pp); 2677 ASSERT(!hat_page_getshare(pp)); 2678 2679 /* 2680 * Now we add the page to the head of the free list. 2681 * But if this page is associated with a paged vnode 2682 * then we adjust the head forward so that the page is 2683 * effectively at the end of the list. 2684 */ 2685 if (pp->p_vnode == NULL) { 2686 /* 2687 * Page has no identity, put it on the free list. 2688 */ 2689 PP_SETAGED(pp); 2690 pp->p_offset = (u_offset_t)-1; 2691 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2692 VM_STAT_ADD(pagecnt.pc_free_free); 2693 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2694 "page_free_free:pp %p", pp); 2695 } else { 2696 PP_CLRAGED(pp); 2697 2698 if (!dontneed) { 2699 /* move it to the tail of the list */ 2700 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2701 2702 VM_STAT_ADD(pagecnt.pc_free_cache); 2703 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2704 "page_free_cache_tail:pp %p", pp); 2705 } else { 2706 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2707 2708 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2709 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2710 "page_free_cache_head:pp %p", pp); 2711 } 2712 } 2713 page_unlock(pp); 2714 2715 /* 2716 * Now do the `freemem' accounting. 2717 */ 2718 pcf_index = PCF_INDEX(); 2719 p = &pcf[pcf_index]; 2720 2721 mutex_enter(&p->pcf_lock); 2722 if (p->pcf_block) { 2723 p->pcf_reserve += 1; 2724 } else { 2725 p->pcf_count += 1; 2726 if (p->pcf_wait) { 2727 mutex_enter(&new_freemem_lock); 2728 /* 2729 * Check to see if some other thread 2730 * is actually waiting. Another bucket 2731 * may have woken it up by now. If there 2732 * are no waiters, then set our pcf_wait 2733 * count to zero to avoid coming in here 2734 * next time. Also, since only one page 2735 * was put on the free list, just wake 2736 * up one waiter. 2737 */ 2738 if (freemem_wait) { 2739 cv_signal(&freemem_cv); 2740 p->pcf_wait--; 2741 } else { 2742 p->pcf_wait = 0; 2743 } 2744 mutex_exit(&new_freemem_lock); 2745 } 2746 } 2747 mutex_exit(&p->pcf_lock); 2748 2749 /* freemem is approximate, so this test OK */ 2750 if (!p->pcf_block) 2751 freemem += 1; 2752 } 2753 2754 /* 2755 * Put page on the "free" list during intial startup. 2756 * This happens during initial single threaded execution. 2757 */ 2758 void 2759 page_free_at_startup(page_t *pp) 2760 { 2761 struct pcf *p; 2762 uint_t pcf_index; 2763 2764 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2765 VM_STAT_ADD(pagecnt.pc_free_free); 2766 2767 /* 2768 * Now do the `freemem' accounting. 2769 */ 2770 pcf_index = PCF_INDEX(); 2771 p = &pcf[pcf_index]; 2772 2773 ASSERT(p->pcf_block == 0); 2774 ASSERT(p->pcf_wait == 0); 2775 p->pcf_count += 1; 2776 2777 /* freemem is approximate, so this is OK */ 2778 freemem += 1; 2779 } 2780 2781 void 2782 page_free_pages(page_t *pp) 2783 { 2784 page_t *tpp, *rootpp = NULL; 2785 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2786 pgcnt_t i; 2787 uint_t szc = pp->p_szc; 2788 2789 VM_STAT_ADD(pagecnt.pc_free_pages); 2790 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2791 "page_free_free:pp %p", pp); 2792 2793 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2794 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2795 panic("page_free_pages: not root page %p", (void *)pp); 2796 /*NOTREACHED*/ 2797 } 2798 2799 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2800 ASSERT((PAGE_EXCL(tpp) && 2801 !page_iolock_assert(tpp)) || panicstr); 2802 if (PP_ISFREE(tpp)) { 2803 panic("page_free_pages: page %p is free", (void *)tpp); 2804 /*NOTREACHED*/ 2805 } 2806 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2807 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 2808 panic("page_free_pages %p", (void *)tpp); 2809 /*NOTREACHED*/ 2810 } 2811 2812 ASSERT(!hat_page_getshare(tpp)); 2813 ASSERT(tpp->p_vnode == NULL); 2814 ASSERT(tpp->p_szc == szc); 2815 2816 PP_SETFREE(tpp); 2817 page_clr_all_props(tpp); 2818 PP_SETAGED(tpp); 2819 tpp->p_offset = (u_offset_t)-1; 2820 ASSERT(tpp->p_next == tpp); 2821 ASSERT(tpp->p_prev == tpp); 2822 page_list_concat(&rootpp, &tpp); 2823 } 2824 ASSERT(rootpp == pp); 2825 2826 page_list_add_pages(rootpp, 0); 2827 page_create_putback(pgcnt); 2828 } 2829 2830 int free_pages = 1; 2831 2832 /* 2833 * This routine attempts to return pages to the cachelist via page_release(). 2834 * It does not *have* to be successful in all cases, since the pageout scanner 2835 * will catch any pages it misses. It does need to be fast and not introduce 2836 * too much overhead. 2837 * 2838 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2839 * don't lock and retry. This is ok, since the page scanner will eventually 2840 * find any page we miss in free_vp_pages(). 2841 */ 2842 void 2843 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2844 { 2845 page_t *pp; 2846 u_offset_t eoff; 2847 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2848 2849 eoff = off + len; 2850 2851 if (free_pages == 0) 2852 return; 2853 if (swap_in_range(vp, off, len)) 2854 return; 2855 2856 for (; off < eoff; off += PAGESIZE) { 2857 2858 /* 2859 * find the page using a fast, but inexact search. It'll be OK 2860 * if a few pages slip through the cracks here. 2861 */ 2862 pp = page_exists(vp, off); 2863 2864 /* 2865 * If we didn't find the page (it may not exist), the page 2866 * is free, looks still in use (shared), or we can't lock it, 2867 * just give up. 2868 */ 2869 if (pp == NULL || 2870 PP_ISFREE(pp) || 2871 page_share_cnt(pp) > 0 || 2872 !page_trylock(pp, SE_EXCL)) 2873 continue; 2874 2875 /* 2876 * Once we have locked pp, verify that it's still the 2877 * correct page and not already free 2878 */ 2879 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2880 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2881 page_unlock(pp); 2882 continue; 2883 } 2884 2885 /* 2886 * try to release the page... 2887 */ 2888 (void) page_release(pp, 1); 2889 } 2890 } 2891 2892 /* 2893 * Reclaim the given page from the free list. 2894 * If pp is part of a large pages, only the given constituent page is reclaimed 2895 * and the large page it belonged to will be demoted. This can only happen 2896 * if the page is not on the cachelist. 2897 * 2898 * Returns 1 on success or 0 on failure. 2899 * 2900 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2901 * If `lock' is non-null, it will be dropped and re-acquired if 2902 * the routine must wait while freemem is 0. 2903 * 2904 * As it turns out, boot_getpages() does this. It picks a page, 2905 * based on where OBP mapped in some address, gets its pfn, searches 2906 * the memsegs, locks the page, then pulls it off the free list! 2907 */ 2908 int 2909 page_reclaim(page_t *pp, kmutex_t *lock) 2910 { 2911 struct pcf *p; 2912 struct cpu *cpup; 2913 int enough; 2914 uint_t i; 2915 2916 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2917 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2918 2919 /* 2920 * If `freemem' is 0, we cannot reclaim this page from the 2921 * freelist, so release every lock we might hold: the page, 2922 * and the `lock' before blocking. 2923 * 2924 * The only way `freemem' can become 0 while there are pages 2925 * marked free (have their p->p_free bit set) is when the 2926 * system is low on memory and doing a page_create(). In 2927 * order to guarantee that once page_create() starts acquiring 2928 * pages it will be able to get all that it needs since `freemem' 2929 * was decreased by the requested amount. So, we need to release 2930 * this page, and let page_create() have it. 2931 * 2932 * Since `freemem' being zero is not supposed to happen, just 2933 * use the usual hash stuff as a starting point. If that bucket 2934 * is empty, then assume the worst, and start at the beginning 2935 * of the pcf array. If we always start at the beginning 2936 * when acquiring more than one pcf lock, there won't be any 2937 * deadlock problems. 2938 */ 2939 2940 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 2941 2942 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) { 2943 pcf_acquire_all(); 2944 goto page_reclaim_nomem; 2945 } 2946 2947 enough = pcf_decrement_bucket(1); 2948 2949 if (!enough) { 2950 VM_STAT_ADD(page_reclaim_zero); 2951 /* 2952 * Check again. Its possible that some other thread 2953 * could have been right behind us, and added one 2954 * to a list somewhere. Acquire each of the pcf locks 2955 * until we find a page. 2956 */ 2957 p = pcf; 2958 for (i = 0; i < pcf_fanout; i++) { 2959 mutex_enter(&p->pcf_lock); 2960 if (p->pcf_count >= 1) { 2961 p->pcf_count -= 1; 2962 /* 2963 * freemem is not protected by any lock. Thus, 2964 * we cannot have any assertion containing 2965 * freemem here. 2966 */ 2967 freemem -= 1; 2968 enough = 1; 2969 break; 2970 } 2971 p++; 2972 } 2973 2974 if (!enough) { 2975 page_reclaim_nomem: 2976 /* 2977 * We really can't have page `pp'. 2978 * Time for the no-memory dance with 2979 * page_free(). This is just like 2980 * page_create_wait(). Plus the added 2981 * attraction of releasing whatever mutex 2982 * we held when we were called with in `lock'. 2983 * Page_unlock() will wakeup any thread 2984 * waiting around for this page. 2985 */ 2986 if (lock) { 2987 VM_STAT_ADD(page_reclaim_zero_locked); 2988 mutex_exit(lock); 2989 } 2990 page_unlock(pp); 2991 2992 /* 2993 * get this before we drop all the pcf locks. 2994 */ 2995 mutex_enter(&new_freemem_lock); 2996 2997 p = pcf; 2998 for (i = 0; i < pcf_fanout; i++) { 2999 p->pcf_wait++; 3000 mutex_exit(&p->pcf_lock); 3001 p++; 3002 } 3003 3004 freemem_wait++; 3005 cv_wait(&freemem_cv, &new_freemem_lock); 3006 freemem_wait--; 3007 3008 mutex_exit(&new_freemem_lock); 3009 3010 if (lock) { 3011 mutex_enter(lock); 3012 } 3013 return (0); 3014 } 3015 3016 /* 3017 * The pcf accounting has been done, 3018 * though none of the pcf_wait flags have been set, 3019 * drop the locks and continue on. 3020 */ 3021 while (p >= pcf) { 3022 mutex_exit(&p->pcf_lock); 3023 p--; 3024 } 3025 } 3026 3027 3028 VM_STAT_ADD(pagecnt.pc_reclaim); 3029 3030 /* 3031 * page_list_sub will handle the case where pp is a large page. 3032 * It's possible that the page was promoted while on the freelist 3033 */ 3034 if (PP_ISAGED(pp)) { 3035 page_list_sub(pp, PG_FREE_LIST); 3036 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3037 "page_reclaim_free:pp %p", pp); 3038 } else { 3039 page_list_sub(pp, PG_CACHE_LIST); 3040 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3041 "page_reclaim_cache:pp %p", pp); 3042 } 3043 3044 /* 3045 * clear the p_free & p_age bits since this page is no longer 3046 * on the free list. Notice that there was a brief time where 3047 * a page is marked as free, but is not on the list. 3048 * 3049 * Set the reference bit to protect against immediate pageout. 3050 */ 3051 PP_CLRFREE(pp); 3052 PP_CLRAGED(pp); 3053 page_set_props(pp, P_REF); 3054 3055 CPU_STATS_ENTER_K(); 3056 cpup = CPU; /* get cpup now that CPU cannot change */ 3057 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3058 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3059 CPU_STATS_EXIT_K(); 3060 ASSERT(pp->p_szc == 0); 3061 3062 return (1); 3063 } 3064 3065 /* 3066 * Destroy identity of the page and put it back on 3067 * the page free list. Assumes that the caller has 3068 * acquired the "exclusive" lock on the page. 3069 */ 3070 void 3071 page_destroy(page_t *pp, int dontfree) 3072 { 3073 ASSERT((PAGE_EXCL(pp) && 3074 !page_iolock_assert(pp)) || panicstr); 3075 ASSERT(pp->p_slckcnt == 0 || panicstr); 3076 3077 if (pp->p_szc != 0) { 3078 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3079 PP_ISKAS(pp)) { 3080 panic("page_destroy: anon or kernel or no vnode " 3081 "large page %p", (void *)pp); 3082 } 3083 page_demote_vp_pages(pp); 3084 ASSERT(pp->p_szc == 0); 3085 } 3086 3087 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3088 3089 /* 3090 * Unload translations, if any, then hash out the 3091 * page to erase its identity. 3092 */ 3093 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3094 page_hashout(pp, NULL); 3095 3096 if (!dontfree) { 3097 /* 3098 * Acquire the "freemem_lock" for availrmem. 3099 * The page_struct_lock need not be acquired for lckcnt 3100 * and cowcnt since the page has an "exclusive" lock. 3101 * We are doing a modified version of page_pp_unlock here. 3102 */ 3103 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3104 mutex_enter(&freemem_lock); 3105 if (pp->p_lckcnt != 0) { 3106 availrmem++; 3107 pages_locked--; 3108 pp->p_lckcnt = 0; 3109 } 3110 if (pp->p_cowcnt != 0) { 3111 availrmem += pp->p_cowcnt; 3112 pages_locked -= pp->p_cowcnt; 3113 pp->p_cowcnt = 0; 3114 } 3115 mutex_exit(&freemem_lock); 3116 } 3117 /* 3118 * Put the page on the "free" list. 3119 */ 3120 page_free(pp, 0); 3121 } 3122 } 3123 3124 void 3125 page_destroy_pages(page_t *pp) 3126 { 3127 3128 page_t *tpp, *rootpp = NULL; 3129 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3130 pgcnt_t i, pglcks = 0; 3131 uint_t szc = pp->p_szc; 3132 3133 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3134 3135 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3136 3137 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3138 3139 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3140 panic("page_destroy_pages: not root page %p", (void *)pp); 3141 /*NOTREACHED*/ 3142 } 3143 3144 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3145 ASSERT((PAGE_EXCL(tpp) && 3146 !page_iolock_assert(tpp)) || panicstr); 3147 ASSERT(tpp->p_slckcnt == 0 || panicstr); 3148 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3149 page_hashout(tpp, NULL); 3150 ASSERT(tpp->p_offset == (u_offset_t)-1); 3151 if (tpp->p_lckcnt != 0) { 3152 pglcks++; 3153 tpp->p_lckcnt = 0; 3154 } else if (tpp->p_cowcnt != 0) { 3155 pglcks += tpp->p_cowcnt; 3156 tpp->p_cowcnt = 0; 3157 } 3158 ASSERT(!hat_page_getshare(tpp)); 3159 ASSERT(tpp->p_vnode == NULL); 3160 ASSERT(tpp->p_szc == szc); 3161 3162 PP_SETFREE(tpp); 3163 page_clr_all_props(tpp); 3164 PP_SETAGED(tpp); 3165 ASSERT(tpp->p_next == tpp); 3166 ASSERT(tpp->p_prev == tpp); 3167 page_list_concat(&rootpp, &tpp); 3168 } 3169 3170 ASSERT(rootpp == pp); 3171 if (pglcks != 0) { 3172 mutex_enter(&freemem_lock); 3173 availrmem += pglcks; 3174 mutex_exit(&freemem_lock); 3175 } 3176 3177 page_list_add_pages(rootpp, 0); 3178 page_create_putback(pgcnt); 3179 } 3180 3181 /* 3182 * Similar to page_destroy(), but destroys pages which are 3183 * locked and known to be on the page free list. Since 3184 * the page is known to be free and locked, no one can access 3185 * it. 3186 * 3187 * Also, the number of free pages does not change. 3188 */ 3189 void 3190 page_destroy_free(page_t *pp) 3191 { 3192 ASSERT(PAGE_EXCL(pp)); 3193 ASSERT(PP_ISFREE(pp)); 3194 ASSERT(pp->p_vnode); 3195 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3196 ASSERT(!hat_page_is_mapped(pp)); 3197 ASSERT(PP_ISAGED(pp) == 0); 3198 ASSERT(pp->p_szc == 0); 3199 3200 VM_STAT_ADD(pagecnt.pc_destroy_free); 3201 page_list_sub(pp, PG_CACHE_LIST); 3202 3203 page_hashout(pp, NULL); 3204 ASSERT(pp->p_vnode == NULL); 3205 ASSERT(pp->p_offset == (u_offset_t)-1); 3206 ASSERT(pp->p_hash == NULL); 3207 3208 PP_SETAGED(pp); 3209 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3210 page_unlock(pp); 3211 3212 mutex_enter(&new_freemem_lock); 3213 if (freemem_wait) { 3214 cv_signal(&freemem_cv); 3215 } 3216 mutex_exit(&new_freemem_lock); 3217 } 3218 3219 /* 3220 * Rename the page "opp" to have an identity specified 3221 * by [vp, off]. If a page already exists with this name 3222 * it is locked and destroyed. Note that the page's 3223 * translations are not unloaded during the rename. 3224 * 3225 * This routine is used by the anon layer to "steal" the 3226 * original page and is not unlike destroying a page and 3227 * creating a new page using the same page frame. 3228 * 3229 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3230 * caller 2 tries to rename B to A. 3231 */ 3232 void 3233 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3234 { 3235 page_t *pp; 3236 int olckcnt = 0; 3237 int ocowcnt = 0; 3238 kmutex_t *phm; 3239 ulong_t index; 3240 3241 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3242 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3243 ASSERT(PP_ISFREE(opp) == 0); 3244 3245 VM_STAT_ADD(page_rename_count); 3246 3247 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3248 "page rename:pp %p vp %p off %llx", opp, vp, off); 3249 3250 /* 3251 * CacheFS may call page_rename for a large NFS page 3252 * when both CacheFS and NFS mount points are used 3253 * by applications. Demote this large page before 3254 * renaming it, to ensure that there are no "partial" 3255 * large pages left lying around. 3256 */ 3257 if (opp->p_szc != 0) { 3258 vnode_t *ovp = opp->p_vnode; 3259 ASSERT(ovp != NULL); 3260 ASSERT(!IS_SWAPFSVP(ovp)); 3261 ASSERT(!VN_ISKAS(ovp)); 3262 page_demote_vp_pages(opp); 3263 ASSERT(opp->p_szc == 0); 3264 } 3265 3266 page_hashout(opp, NULL); 3267 PP_CLRAGED(opp); 3268 3269 /* 3270 * Acquire the appropriate page hash lock, since 3271 * we're going to rename the page. 3272 */ 3273 index = PAGE_HASH_FUNC(vp, off); 3274 phm = PAGE_HASH_MUTEX(index); 3275 mutex_enter(phm); 3276 top: 3277 /* 3278 * Look for an existing page with this name and destroy it if found. 3279 * By holding the page hash lock all the way to the page_hashin() 3280 * call, we are assured that no page can be created with this 3281 * identity. In the case when the phm lock is dropped to undo any 3282 * hat layer mappings, the existing page is held with an "exclusive" 3283 * lock, again preventing another page from being created with 3284 * this identity. 3285 */ 3286 pp = page_hash_search(index, vp, off); 3287 if (pp != NULL) { 3288 VM_STAT_ADD(page_rename_exists); 3289 3290 /* 3291 * As it turns out, this is one of only two places where 3292 * page_lock() needs to hold the passed in lock in the 3293 * successful case. In all of the others, the lock could 3294 * be dropped as soon as the attempt is made to lock 3295 * the page. It is tempting to add yet another arguement, 3296 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3297 */ 3298 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3299 /* 3300 * Went to sleep because the page could not 3301 * be locked. We were woken up when the page 3302 * was unlocked, or when the page was destroyed. 3303 * In either case, `phm' was dropped while we 3304 * slept. Hence we should not just roar through 3305 * this loop. 3306 */ 3307 goto top; 3308 } 3309 3310 /* 3311 * If an existing page is a large page, then demote 3312 * it to ensure that no "partial" large pages are 3313 * "created" after page_rename. An existing page 3314 * can be a CacheFS page, and can't belong to swapfs. 3315 */ 3316 if (hat_page_is_mapped(pp)) { 3317 /* 3318 * Unload translations. Since we hold the 3319 * exclusive lock on this page, the page 3320 * can not be changed while we drop phm. 3321 * This is also not a lock protocol violation, 3322 * but rather the proper way to do things. 3323 */ 3324 mutex_exit(phm); 3325 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3326 if (pp->p_szc != 0) { 3327 ASSERT(!IS_SWAPFSVP(vp)); 3328 ASSERT(!VN_ISKAS(vp)); 3329 page_demote_vp_pages(pp); 3330 ASSERT(pp->p_szc == 0); 3331 } 3332 mutex_enter(phm); 3333 } else if (pp->p_szc != 0) { 3334 ASSERT(!IS_SWAPFSVP(vp)); 3335 ASSERT(!VN_ISKAS(vp)); 3336 mutex_exit(phm); 3337 page_demote_vp_pages(pp); 3338 ASSERT(pp->p_szc == 0); 3339 mutex_enter(phm); 3340 } 3341 page_hashout(pp, phm); 3342 } 3343 /* 3344 * Hash in the page with the new identity. 3345 */ 3346 if (!page_hashin(opp, vp, off, phm)) { 3347 /* 3348 * We were holding phm while we searched for [vp, off] 3349 * and only dropped phm if we found and locked a page. 3350 * If we can't create this page now, then some thing 3351 * is really broken. 3352 */ 3353 panic("page_rename: Can't hash in page: %p", (void *)pp); 3354 /*NOTREACHED*/ 3355 } 3356 3357 ASSERT(MUTEX_HELD(phm)); 3358 mutex_exit(phm); 3359 3360 /* 3361 * Now that we have dropped phm, lets get around to finishing up 3362 * with pp. 3363 */ 3364 if (pp != NULL) { 3365 ASSERT(!hat_page_is_mapped(pp)); 3366 /* for now large pages should not end up here */ 3367 ASSERT(pp->p_szc == 0); 3368 /* 3369 * Save the locks for transfer to the new page and then 3370 * clear them so page_free doesn't think they're important. 3371 * The page_struct_lock need not be acquired for lckcnt and 3372 * cowcnt since the page has an "exclusive" lock. 3373 */ 3374 olckcnt = pp->p_lckcnt; 3375 ocowcnt = pp->p_cowcnt; 3376 pp->p_lckcnt = pp->p_cowcnt = 0; 3377 3378 /* 3379 * Put the page on the "free" list after we drop 3380 * the lock. The less work under the lock the better. 3381 */ 3382 /*LINTED: constant in conditional context*/ 3383 VN_DISPOSE(pp, B_FREE, 0, kcred); 3384 } 3385 3386 /* 3387 * Transfer the lock count from the old page (if any). 3388 * The page_struct_lock need not be acquired for lckcnt and 3389 * cowcnt since the page has an "exclusive" lock. 3390 */ 3391 opp->p_lckcnt += olckcnt; 3392 opp->p_cowcnt += ocowcnt; 3393 } 3394 3395 /* 3396 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3397 * 3398 * Pages are normally inserted at the start of a vnode's v_pages list. 3399 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3400 * This can happen when a modified page is relocated for DR. 3401 * 3402 * Returns 1 on success and 0 on failure. 3403 */ 3404 static int 3405 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3406 { 3407 page_t **listp; 3408 page_t *tp; 3409 ulong_t index; 3410 3411 ASSERT(PAGE_EXCL(pp)); 3412 ASSERT(vp != NULL); 3413 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3414 3415 /* 3416 * Be sure to set these up before the page is inserted on the hash 3417 * list. As soon as the page is placed on the list some other 3418 * thread might get confused and wonder how this page could 3419 * possibly hash to this list. 3420 */ 3421 pp->p_vnode = vp; 3422 pp->p_offset = offset; 3423 3424 /* 3425 * record if this page is on a swap vnode 3426 */ 3427 if ((vp->v_flag & VISSWAP) != 0) 3428 PP_SETSWAP(pp); 3429 3430 index = PAGE_HASH_FUNC(vp, offset); 3431 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3432 listp = &page_hash[index]; 3433 3434 /* 3435 * If this page is already hashed in, fail this attempt to add it. 3436 */ 3437 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3438 if (tp->p_vnode == vp && tp->p_offset == offset) { 3439 pp->p_vnode = NULL; 3440 pp->p_offset = (u_offset_t)(-1); 3441 return (0); 3442 } 3443 } 3444 pp->p_hash = *listp; 3445 *listp = pp; 3446 3447 /* 3448 * Add the page to the vnode's list of pages 3449 */ 3450 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3451 listp = &vp->v_pages->p_vpprev->p_vpnext; 3452 else 3453 listp = &vp->v_pages; 3454 3455 page_vpadd(listp, pp); 3456 3457 return (1); 3458 } 3459 3460 /* 3461 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3462 * 3463 * Returns 1 on success and 0 on failure. 3464 * If hold is passed in, it is not dropped. 3465 */ 3466 int 3467 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3468 { 3469 kmutex_t *phm = NULL; 3470 kmutex_t *vphm; 3471 int rc; 3472 3473 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3474 ASSERT(pp->p_fsdata == 0 || panicstr); 3475 3476 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3477 "page_hashin:pp %p vp %p offset %llx", 3478 pp, vp, offset); 3479 3480 VM_STAT_ADD(hashin_count); 3481 3482 if (hold != NULL) 3483 phm = hold; 3484 else { 3485 VM_STAT_ADD(hashin_not_held); 3486 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3487 mutex_enter(phm); 3488 } 3489 3490 vphm = page_vnode_mutex(vp); 3491 mutex_enter(vphm); 3492 rc = page_do_hashin(pp, vp, offset); 3493 mutex_exit(vphm); 3494 if (hold == NULL) 3495 mutex_exit(phm); 3496 if (rc == 0) 3497 VM_STAT_ADD(hashin_already); 3498 return (rc); 3499 } 3500 3501 /* 3502 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3503 * All mutexes must be held 3504 */ 3505 static void 3506 page_do_hashout(page_t *pp) 3507 { 3508 page_t **hpp; 3509 page_t *hp; 3510 vnode_t *vp = pp->p_vnode; 3511 3512 ASSERT(vp != NULL); 3513 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3514 3515 /* 3516 * First, take pp off of its hash chain. 3517 */ 3518 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3519 3520 for (;;) { 3521 hp = *hpp; 3522 if (hp == pp) 3523 break; 3524 if (hp == NULL) { 3525 panic("page_do_hashout"); 3526 /*NOTREACHED*/ 3527 } 3528 hpp = &hp->p_hash; 3529 } 3530 *hpp = pp->p_hash; 3531 3532 /* 3533 * Now remove it from its associated vnode. 3534 */ 3535 if (vp->v_pages) 3536 page_vpsub(&vp->v_pages, pp); 3537 3538 pp->p_hash = NULL; 3539 page_clr_all_props(pp); 3540 PP_CLRSWAP(pp); 3541 pp->p_vnode = NULL; 3542 pp->p_offset = (u_offset_t)-1; 3543 pp->p_fsdata = 0; 3544 } 3545 3546 /* 3547 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3548 * 3549 * When `phm' is non-NULL it contains the address of the mutex protecting the 3550 * hash list pp is on. It is not dropped. 3551 */ 3552 void 3553 page_hashout(page_t *pp, kmutex_t *phm) 3554 { 3555 vnode_t *vp; 3556 ulong_t index; 3557 kmutex_t *nphm; 3558 kmutex_t *vphm; 3559 kmutex_t *sep; 3560 3561 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3562 ASSERT(pp->p_vnode != NULL); 3563 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3564 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3565 3566 vp = pp->p_vnode; 3567 3568 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3569 "page_hashout:pp %p vp %p", pp, vp); 3570 3571 /* 3572 * 3573 */ 3574 VM_STAT_ADD(hashout_count); 3575 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3576 if (phm == NULL) { 3577 VM_STAT_ADD(hashout_not_held); 3578 nphm = PAGE_HASH_MUTEX(index); 3579 mutex_enter(nphm); 3580 } 3581 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3582 3583 3584 /* 3585 * grab page vnode mutex and remove it... 3586 */ 3587 vphm = page_vnode_mutex(vp); 3588 mutex_enter(vphm); 3589 3590 page_do_hashout(pp); 3591 3592 mutex_exit(vphm); 3593 if (phm == NULL) 3594 mutex_exit(nphm); 3595 3596 /* 3597 * Wake up processes waiting for this page. The page's 3598 * identity has been changed, and is probably not the 3599 * desired page any longer. 3600 */ 3601 sep = page_se_mutex(pp); 3602 mutex_enter(sep); 3603 pp->p_selock &= ~SE_EWANTED; 3604 if (CV_HAS_WAITERS(&pp->p_cv)) 3605 cv_broadcast(&pp->p_cv); 3606 mutex_exit(sep); 3607 } 3608 3609 /* 3610 * Add the page to the front of a linked list of pages 3611 * using the p_next & p_prev pointers for the list. 3612 * The caller is responsible for protecting the list pointers. 3613 */ 3614 void 3615 page_add(page_t **ppp, page_t *pp) 3616 { 3617 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3618 3619 page_add_common(ppp, pp); 3620 } 3621 3622 3623 3624 /* 3625 * Common code for page_add() and mach_page_add() 3626 */ 3627 void 3628 page_add_common(page_t **ppp, page_t *pp) 3629 { 3630 if (*ppp == NULL) { 3631 pp->p_next = pp->p_prev = pp; 3632 } else { 3633 pp->p_next = *ppp; 3634 pp->p_prev = (*ppp)->p_prev; 3635 (*ppp)->p_prev = pp; 3636 pp->p_prev->p_next = pp; 3637 } 3638 *ppp = pp; 3639 } 3640 3641 3642 /* 3643 * Remove this page from a linked list of pages 3644 * using the p_next & p_prev pointers for the list. 3645 * 3646 * The caller is responsible for protecting the list pointers. 3647 */ 3648 void 3649 page_sub(page_t **ppp, page_t *pp) 3650 { 3651 ASSERT(pp != NULL && (PP_ISFREE(pp)) ? 1 : 3652 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3653 3654 if (*ppp == NULL || pp == NULL) { 3655 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3656 (void *)pp, (void *)(*ppp)); 3657 /*NOTREACHED*/ 3658 } 3659 3660 page_sub_common(ppp, pp); 3661 } 3662 3663 3664 /* 3665 * Common code for page_sub() and mach_page_sub() 3666 */ 3667 void 3668 page_sub_common(page_t **ppp, page_t *pp) 3669 { 3670 if (*ppp == pp) 3671 *ppp = pp->p_next; /* go to next page */ 3672 3673 if (*ppp == pp) 3674 *ppp = NULL; /* page list is gone */ 3675 else { 3676 pp->p_prev->p_next = pp->p_next; 3677 pp->p_next->p_prev = pp->p_prev; 3678 } 3679 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3680 } 3681 3682 3683 /* 3684 * Break page list cppp into two lists with npages in the first list. 3685 * The tail is returned in nppp. 3686 */ 3687 void 3688 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3689 { 3690 page_t *s1pp = *oppp; 3691 page_t *s2pp; 3692 page_t *e1pp, *e2pp; 3693 long n = 0; 3694 3695 if (s1pp == NULL) { 3696 *nppp = NULL; 3697 return; 3698 } 3699 if (npages == 0) { 3700 *nppp = s1pp; 3701 *oppp = NULL; 3702 return; 3703 } 3704 for (n = 0, s2pp = *oppp; n < npages; n++) { 3705 s2pp = s2pp->p_next; 3706 } 3707 /* Fix head and tail of new lists */ 3708 e1pp = s2pp->p_prev; 3709 e2pp = s1pp->p_prev; 3710 s1pp->p_prev = e1pp; 3711 e1pp->p_next = s1pp; 3712 s2pp->p_prev = e2pp; 3713 e2pp->p_next = s2pp; 3714 3715 /* second list empty */ 3716 if (s2pp == s1pp) { 3717 *oppp = s1pp; 3718 *nppp = NULL; 3719 } else { 3720 *oppp = s1pp; 3721 *nppp = s2pp; 3722 } 3723 } 3724 3725 /* 3726 * Concatenate page list nppp onto the end of list ppp. 3727 */ 3728 void 3729 page_list_concat(page_t **ppp, page_t **nppp) 3730 { 3731 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3732 3733 if (*nppp == NULL) { 3734 return; 3735 } 3736 if (*ppp == NULL) { 3737 *ppp = *nppp; 3738 return; 3739 } 3740 s1pp = *ppp; 3741 e1pp = s1pp->p_prev; 3742 s2pp = *nppp; 3743 e2pp = s2pp->p_prev; 3744 s1pp->p_prev = e2pp; 3745 e2pp->p_next = s1pp; 3746 e1pp->p_next = s2pp; 3747 s2pp->p_prev = e1pp; 3748 } 3749 3750 /* 3751 * return the next page in the page list 3752 */ 3753 page_t * 3754 page_list_next(page_t *pp) 3755 { 3756 return (pp->p_next); 3757 } 3758 3759 3760 /* 3761 * Add the page to the front of the linked list of pages 3762 * using p_vpnext/p_vpprev pointers for the list. 3763 * 3764 * The caller is responsible for protecting the lists. 3765 */ 3766 void 3767 page_vpadd(page_t **ppp, page_t *pp) 3768 { 3769 if (*ppp == NULL) { 3770 pp->p_vpnext = pp->p_vpprev = pp; 3771 } else { 3772 pp->p_vpnext = *ppp; 3773 pp->p_vpprev = (*ppp)->p_vpprev; 3774 (*ppp)->p_vpprev = pp; 3775 pp->p_vpprev->p_vpnext = pp; 3776 } 3777 *ppp = pp; 3778 } 3779 3780 /* 3781 * Remove this page from the linked list of pages 3782 * using p_vpnext/p_vpprev pointers for the list. 3783 * 3784 * The caller is responsible for protecting the lists. 3785 */ 3786 void 3787 page_vpsub(page_t **ppp, page_t *pp) 3788 { 3789 if (*ppp == NULL || pp == NULL) { 3790 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3791 (void *)pp, (void *)(*ppp)); 3792 /*NOTREACHED*/ 3793 } 3794 3795 if (*ppp == pp) 3796 *ppp = pp->p_vpnext; /* go to next page */ 3797 3798 if (*ppp == pp) 3799 *ppp = NULL; /* page list is gone */ 3800 else { 3801 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3802 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3803 } 3804 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3805 } 3806 3807 /* 3808 * Lock a physical page into memory "long term". Used to support "lock 3809 * in memory" functions. Accepts the page to be locked, and a cow variable 3810 * to indicate whether a the lock will travel to the new page during 3811 * a potential copy-on-write. 3812 */ 3813 int 3814 page_pp_lock( 3815 page_t *pp, /* page to be locked */ 3816 int cow, /* cow lock */ 3817 int kernel) /* must succeed -- ignore checking */ 3818 { 3819 int r = 0; /* result -- assume failure */ 3820 3821 ASSERT(PAGE_LOCKED(pp)); 3822 3823 page_struct_lock(pp); 3824 /* 3825 * Acquire the "freemem_lock" for availrmem. 3826 */ 3827 if (cow) { 3828 mutex_enter(&freemem_lock); 3829 if ((availrmem > pages_pp_maximum) && 3830 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3831 availrmem--; 3832 pages_locked++; 3833 mutex_exit(&freemem_lock); 3834 r = 1; 3835 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3836 cmn_err(CE_WARN, 3837 "COW lock limit reached on pfn 0x%lx", 3838 page_pptonum(pp)); 3839 } 3840 } else 3841 mutex_exit(&freemem_lock); 3842 } else { 3843 if (pp->p_lckcnt) { 3844 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3845 r = 1; 3846 if (++pp->p_lckcnt == 3847 (ushort_t)PAGE_LOCK_MAXIMUM) { 3848 cmn_err(CE_WARN, "Page lock limit " 3849 "reached on pfn 0x%lx", 3850 page_pptonum(pp)); 3851 } 3852 } 3853 } else { 3854 if (kernel) { 3855 /* availrmem accounting done by caller */ 3856 ++pp->p_lckcnt; 3857 r = 1; 3858 } else { 3859 mutex_enter(&freemem_lock); 3860 if (availrmem > pages_pp_maximum) { 3861 availrmem--; 3862 pages_locked++; 3863 ++pp->p_lckcnt; 3864 r = 1; 3865 } 3866 mutex_exit(&freemem_lock); 3867 } 3868 } 3869 } 3870 page_struct_unlock(pp); 3871 return (r); 3872 } 3873 3874 /* 3875 * Decommit a lock on a physical page frame. Account for cow locks if 3876 * appropriate. 3877 */ 3878 void 3879 page_pp_unlock( 3880 page_t *pp, /* page to be unlocked */ 3881 int cow, /* expect cow lock */ 3882 int kernel) /* this was a kernel lock */ 3883 { 3884 ASSERT(PAGE_LOCKED(pp)); 3885 3886 page_struct_lock(pp); 3887 /* 3888 * Acquire the "freemem_lock" for availrmem. 3889 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3890 * could be called to unlock even if nothing is locked. This could 3891 * happen if locked file pages were truncated (removing the lock) 3892 * and the file was grown again and new pages faulted in; the new 3893 * pages are unlocked but the segment still thinks they're locked. 3894 */ 3895 if (cow) { 3896 if (pp->p_cowcnt) { 3897 mutex_enter(&freemem_lock); 3898 pp->p_cowcnt--; 3899 availrmem++; 3900 pages_locked--; 3901 mutex_exit(&freemem_lock); 3902 } 3903 } else { 3904 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3905 if (!kernel) { 3906 mutex_enter(&freemem_lock); 3907 availrmem++; 3908 pages_locked--; 3909 mutex_exit(&freemem_lock); 3910 } 3911 } 3912 } 3913 page_struct_unlock(pp); 3914 } 3915 3916 /* 3917 * This routine reserves availrmem for npages. 3918 * It returns 1 on success or 0 on failure. 3919 * 3920 * flags: KM_NOSLEEP or KM_SLEEP 3921 * cb_wait: called to induce delay when KM_SLEEP reservation requires kmem 3922 * reaping to potentially succeed. If the callback returns 0, the 3923 * reservation attempts will cease to repeat and page_xresv() may 3924 * report a failure. If cb_wait is NULL, the traditional delay(hz/2) 3925 * behavior will be used while waiting for a reap. 3926 */ 3927 int 3928 page_xresv(pgcnt_t npages, uint_t flags, int (*cb_wait)(void)) 3929 { 3930 mutex_enter(&freemem_lock); 3931 if (availrmem >= tune.t_minarmem + npages) { 3932 availrmem -= npages; 3933 mutex_exit(&freemem_lock); 3934 return (1); 3935 } else if ((flags & KM_NOSLEEP) != 0) { 3936 mutex_exit(&freemem_lock); 3937 return (0); 3938 } 3939 mutex_exit(&freemem_lock); 3940 3941 /* 3942 * We signal memory pressure to the system by elevating 'needfree'. 3943 * Processes such as kmem reaping, pageout, and ZFS ARC shrinking can 3944 * then respond to said pressure by freeing pages. 3945 */ 3946 page_needfree(npages); 3947 int nobail = 1; 3948 do { 3949 kmem_reap(); 3950 if (cb_wait == NULL) { 3951 delay(hz >> 2); 3952 } else { 3953 nobail = cb_wait(); 3954 } 3955 3956 mutex_enter(&freemem_lock); 3957 if (availrmem >= tune.t_minarmem + npages) { 3958 availrmem -= npages; 3959 mutex_exit(&freemem_lock); 3960 page_needfree(-(spgcnt_t)npages); 3961 return (1); 3962 } 3963 mutex_exit(&freemem_lock); 3964 } while (nobail != 0); 3965 page_needfree(-(spgcnt_t)npages); 3966 3967 return (0); 3968 } 3969 3970 /* 3971 * This routine reserves availrmem for npages; 3972 * flags: KM_NOSLEEP or KM_SLEEP 3973 * returns 1 on success or 0 on failure 3974 */ 3975 int 3976 page_resv(pgcnt_t npages, uint_t flags) 3977 { 3978 return (page_xresv(npages, flags, NULL)); 3979 } 3980 3981 /* 3982 * This routine unreserves availrmem for npages; 3983 */ 3984 void 3985 page_unresv(pgcnt_t npages) 3986 { 3987 mutex_enter(&freemem_lock); 3988 availrmem += npages; 3989 mutex_exit(&freemem_lock); 3990 } 3991 3992 /* 3993 * See Statement at the beginning of segvn_lockop() regarding 3994 * the way we handle cowcnts and lckcnts. 3995 * 3996 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 3997 * that breaks COW has PROT_WRITE. 3998 * 3999 * Note that, we may also break COW in case we are softlocking 4000 * on read access during physio; 4001 * in this softlock case, the vpage may not have PROT_WRITE. 4002 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 4003 * if the vpage doesn't have PROT_WRITE. 4004 * 4005 * This routine is never called if we are stealing a page 4006 * in anon_private. 4007 * 4008 * The caller subtracted from availrmem for read only mapping. 4009 * if lckcnt is 1 increment availrmem. 4010 */ 4011 void 4012 page_pp_useclaim( 4013 page_t *opp, /* original page frame losing lock */ 4014 page_t *npp, /* new page frame gaining lock */ 4015 uint_t write_perm) /* set if vpage has PROT_WRITE */ 4016 { 4017 int payback = 0; 4018 int nidx, oidx; 4019 4020 ASSERT(PAGE_LOCKED(opp)); 4021 ASSERT(PAGE_LOCKED(npp)); 4022 4023 /* 4024 * Since we have two pages we probably have two locks. We need to take 4025 * them in a defined order to avoid deadlocks. It's also possible they 4026 * both hash to the same lock in which case this is a non-issue. 4027 */ 4028 nidx = PAGE_LLOCK_HASH(PP_PAGEROOT(npp)); 4029 oidx = PAGE_LLOCK_HASH(PP_PAGEROOT(opp)); 4030 if (nidx < oidx) { 4031 page_struct_lock(npp); 4032 page_struct_lock(opp); 4033 } else if (oidx < nidx) { 4034 page_struct_lock(opp); 4035 page_struct_lock(npp); 4036 } else { /* The pages hash to the same lock */ 4037 page_struct_lock(npp); 4038 } 4039 4040 ASSERT(npp->p_cowcnt == 0); 4041 ASSERT(npp->p_lckcnt == 0); 4042 4043 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4044 if ((write_perm && opp->p_cowcnt != 0) || 4045 (!write_perm && opp->p_lckcnt != 0)) { 4046 4047 if (write_perm) { 4048 npp->p_cowcnt++; 4049 ASSERT(opp->p_cowcnt != 0); 4050 opp->p_cowcnt--; 4051 } else { 4052 4053 ASSERT(opp->p_lckcnt != 0); 4054 4055 /* 4056 * We didn't need availrmem decremented if p_lckcnt on 4057 * original page is 1. Here, we are unlocking 4058 * read-only copy belonging to original page and 4059 * are locking a copy belonging to new page. 4060 */ 4061 if (opp->p_lckcnt == 1) 4062 payback = 1; 4063 4064 npp->p_lckcnt++; 4065 opp->p_lckcnt--; 4066 } 4067 } 4068 if (payback) { 4069 mutex_enter(&freemem_lock); 4070 availrmem++; 4071 pages_useclaim--; 4072 mutex_exit(&freemem_lock); 4073 } 4074 4075 if (nidx < oidx) { 4076 page_struct_unlock(opp); 4077 page_struct_unlock(npp); 4078 } else if (oidx < nidx) { 4079 page_struct_unlock(npp); 4080 page_struct_unlock(opp); 4081 } else { /* The pages hash to the same lock */ 4082 page_struct_unlock(npp); 4083 } 4084 } 4085 4086 /* 4087 * Simple claim adjust functions -- used to support changes in 4088 * claims due to changes in access permissions. Used by segvn_setprot(). 4089 */ 4090 int 4091 page_addclaim(page_t *pp) 4092 { 4093 int r = 0; /* result */ 4094 4095 ASSERT(PAGE_LOCKED(pp)); 4096 4097 page_struct_lock(pp); 4098 ASSERT(pp->p_lckcnt != 0); 4099 4100 if (pp->p_lckcnt == 1) { 4101 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4102 --pp->p_lckcnt; 4103 r = 1; 4104 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4105 cmn_err(CE_WARN, 4106 "COW lock limit reached on pfn 0x%lx", 4107 page_pptonum(pp)); 4108 } 4109 } 4110 } else { 4111 mutex_enter(&freemem_lock); 4112 if ((availrmem > pages_pp_maximum) && 4113 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4114 --availrmem; 4115 ++pages_claimed; 4116 mutex_exit(&freemem_lock); 4117 --pp->p_lckcnt; 4118 r = 1; 4119 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4120 cmn_err(CE_WARN, 4121 "COW lock limit reached on pfn 0x%lx", 4122 page_pptonum(pp)); 4123 } 4124 } else 4125 mutex_exit(&freemem_lock); 4126 } 4127 page_struct_unlock(pp); 4128 return (r); 4129 } 4130 4131 int 4132 page_subclaim(page_t *pp) 4133 { 4134 int r = 0; 4135 4136 ASSERT(PAGE_LOCKED(pp)); 4137 4138 page_struct_lock(pp); 4139 ASSERT(pp->p_cowcnt != 0); 4140 4141 if (pp->p_lckcnt) { 4142 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4143 r = 1; 4144 /* 4145 * for availrmem 4146 */ 4147 mutex_enter(&freemem_lock); 4148 availrmem++; 4149 pages_claimed--; 4150 mutex_exit(&freemem_lock); 4151 4152 pp->p_cowcnt--; 4153 4154 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4155 cmn_err(CE_WARN, 4156 "Page lock limit reached on pfn 0x%lx", 4157 page_pptonum(pp)); 4158 } 4159 } 4160 } else { 4161 r = 1; 4162 pp->p_cowcnt--; 4163 pp->p_lckcnt++; 4164 } 4165 page_struct_unlock(pp); 4166 return (r); 4167 } 4168 4169 /* 4170 * Variant of page_addclaim(), where ppa[] contains the pages of a single large 4171 * page. 4172 */ 4173 int 4174 page_addclaim_pages(page_t **ppa) 4175 { 4176 pgcnt_t lckpgs = 0, pg_idx; 4177 4178 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4179 4180 /* 4181 * Only need to take the page struct lock on the large page root. 4182 */ 4183 page_struct_lock(ppa[0]); 4184 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4185 4186 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4187 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4188 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4189 page_struct_unlock(ppa[0]); 4190 return (0); 4191 } 4192 if (ppa[pg_idx]->p_lckcnt > 1) 4193 lckpgs++; 4194 } 4195 4196 if (lckpgs != 0) { 4197 mutex_enter(&freemem_lock); 4198 if (availrmem >= pages_pp_maximum + lckpgs) { 4199 availrmem -= lckpgs; 4200 pages_claimed += lckpgs; 4201 } else { 4202 mutex_exit(&freemem_lock); 4203 page_struct_unlock(ppa[0]); 4204 return (0); 4205 } 4206 mutex_exit(&freemem_lock); 4207 } 4208 4209 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4210 ppa[pg_idx]->p_lckcnt--; 4211 ppa[pg_idx]->p_cowcnt++; 4212 } 4213 page_struct_unlock(ppa[0]); 4214 return (1); 4215 } 4216 4217 /* 4218 * Variant of page_subclaim(), where ppa[] contains the pages of a single large 4219 * page. 4220 */ 4221 int 4222 page_subclaim_pages(page_t **ppa) 4223 { 4224 pgcnt_t ulckpgs = 0, pg_idx; 4225 4226 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4227 4228 /* 4229 * Only need to take the page struct lock on the large page root. 4230 */ 4231 page_struct_lock(ppa[0]); 4232 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4233 4234 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4235 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4236 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4237 page_struct_unlock(ppa[0]); 4238 return (0); 4239 } 4240 if (ppa[pg_idx]->p_lckcnt != 0) 4241 ulckpgs++; 4242 } 4243 4244 if (ulckpgs != 0) { 4245 mutex_enter(&freemem_lock); 4246 availrmem += ulckpgs; 4247 pages_claimed -= ulckpgs; 4248 mutex_exit(&freemem_lock); 4249 } 4250 4251 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4252 ppa[pg_idx]->p_cowcnt--; 4253 ppa[pg_idx]->p_lckcnt++; 4254 4255 } 4256 page_struct_unlock(ppa[0]); 4257 return (1); 4258 } 4259 4260 page_t * 4261 page_numtopp(pfn_t pfnum, se_t se) 4262 { 4263 page_t *pp; 4264 4265 retry: 4266 pp = page_numtopp_nolock(pfnum); 4267 if (pp == NULL) { 4268 return ((page_t *)NULL); 4269 } 4270 4271 /* 4272 * Acquire the appropriate lock on the page. 4273 */ 4274 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4275 if (page_pptonum(pp) != pfnum) 4276 goto retry; 4277 continue; 4278 } 4279 4280 if (page_pptonum(pp) != pfnum) { 4281 page_unlock(pp); 4282 goto retry; 4283 } 4284 4285 return (pp); 4286 } 4287 4288 page_t * 4289 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4290 { 4291 page_t *pp; 4292 4293 retry: 4294 pp = page_numtopp_nolock(pfnum); 4295 if (pp == NULL) { 4296 return ((page_t *)NULL); 4297 } 4298 4299 /* 4300 * Acquire the appropriate lock on the page. 4301 */ 4302 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4303 if (page_pptonum(pp) != pfnum) 4304 goto retry; 4305 continue; 4306 } 4307 4308 if (page_pptonum(pp) != pfnum) { 4309 page_unlock(pp); 4310 goto retry; 4311 } 4312 4313 return (pp); 4314 } 4315 4316 /* 4317 * This routine is like page_numtopp, but will only return page structs 4318 * for pages which are ok for loading into hardware using the page struct. 4319 */ 4320 page_t * 4321 page_numtopp_nowait(pfn_t pfnum, se_t se) 4322 { 4323 page_t *pp; 4324 4325 retry: 4326 pp = page_numtopp_nolock(pfnum); 4327 if (pp == NULL) { 4328 return ((page_t *)NULL); 4329 } 4330 4331 /* 4332 * Try to acquire the appropriate lock on the page. 4333 */ 4334 if (PP_ISFREE(pp)) 4335 pp = NULL; 4336 else { 4337 if (!page_trylock(pp, se)) 4338 pp = NULL; 4339 else { 4340 if (page_pptonum(pp) != pfnum) { 4341 page_unlock(pp); 4342 goto retry; 4343 } 4344 if (PP_ISFREE(pp)) { 4345 page_unlock(pp); 4346 pp = NULL; 4347 } 4348 } 4349 } 4350 return (pp); 4351 } 4352 4353 /* 4354 * Returns a count of dirty pages that are in the process 4355 * of being written out. If 'cleanit' is set, try to push the page. 4356 */ 4357 pgcnt_t 4358 page_busy(int cleanit) 4359 { 4360 page_t *page0 = page_first(); 4361 page_t *pp = page0; 4362 pgcnt_t nppbusy = 0; 4363 u_offset_t off; 4364 4365 do { 4366 vnode_t *vp = pp->p_vnode; 4367 /* 4368 * A page is a candidate for syncing if it is: 4369 * 4370 * (a) On neither the freelist nor the cachelist 4371 * (b) Hashed onto a vnode 4372 * (c) Not a kernel page 4373 * (d) Dirty 4374 * (e) Not part of a swapfile 4375 * (f) a page which belongs to a real vnode; eg has a non-null 4376 * v_vfsp pointer. 4377 * (g) Backed by a filesystem which doesn't have a 4378 * stubbed-out sync operation 4379 */ 4380 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && 4381 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4382 vfs_can_sync(vp->v_vfsp)) { 4383 nppbusy++; 4384 4385 if (!cleanit) 4386 continue; 4387 if (!page_trylock(pp, SE_EXCL)) 4388 continue; 4389 4390 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4391 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4392 !(hat_pagesync(pp, 4393 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4394 page_unlock(pp); 4395 continue; 4396 } 4397 off = pp->p_offset; 4398 VN_HOLD(vp); 4399 page_unlock(pp); 4400 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4401 B_ASYNC | B_FREE, kcred, NULL); 4402 VN_RELE(vp); 4403 } 4404 } while ((pp = page_next(pp)) != page0); 4405 4406 return (nppbusy); 4407 } 4408 4409 void page_invalidate_pages(void); 4410 4411 /* 4412 * callback handler to vm sub-system 4413 * 4414 * callers make sure no recursive entries to this func. 4415 */ 4416 /*ARGSUSED*/ 4417 boolean_t 4418 callb_vm_cpr(void *arg, int code) 4419 { 4420 if (code == CB_CODE_CPR_CHKPT) 4421 page_invalidate_pages(); 4422 return (B_TRUE); 4423 } 4424 4425 /* 4426 * Invalidate all pages of the system. 4427 * It shouldn't be called until all user page activities are all stopped. 4428 */ 4429 void 4430 page_invalidate_pages() 4431 { 4432 page_t *pp; 4433 page_t *page0; 4434 pgcnt_t nbusypages; 4435 int retry = 0; 4436 const int MAXRETRIES = 4; 4437 top: 4438 /* 4439 * Flush dirty pages and destroy the clean ones. 4440 */ 4441 nbusypages = 0; 4442 4443 pp = page0 = page_first(); 4444 do { 4445 struct vnode *vp; 4446 u_offset_t offset; 4447 int mod; 4448 4449 /* 4450 * skip the page if it has no vnode or the page associated 4451 * with the kernel vnode or prom allocated kernel mem. 4452 */ 4453 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) 4454 continue; 4455 4456 /* 4457 * skip the page which is already free invalidated. 4458 */ 4459 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4460 continue; 4461 4462 /* 4463 * skip pages that are already locked or can't be "exclusively" 4464 * locked or are already free. After we lock the page, check 4465 * the free and age bits again to be sure it's not destroyed 4466 * yet. 4467 * To achieve max. parallelization, we use page_trylock instead 4468 * of page_lock so that we don't get block on individual pages 4469 * while we have thousands of other pages to process. 4470 */ 4471 if (!page_trylock(pp, SE_EXCL)) { 4472 nbusypages++; 4473 continue; 4474 } else if (PP_ISFREE(pp)) { 4475 if (!PP_ISAGED(pp)) { 4476 page_destroy_free(pp); 4477 } else { 4478 page_unlock(pp); 4479 } 4480 continue; 4481 } 4482 /* 4483 * Is this page involved in some I/O? shared? 4484 * 4485 * The page_struct_lock need not be acquired to 4486 * examine these fields since the page has an 4487 * "exclusive" lock. 4488 */ 4489 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4490 page_unlock(pp); 4491 continue; 4492 } 4493 4494 if (vp->v_type == VCHR) { 4495 panic("vp->v_type == VCHR"); 4496 /*NOTREACHED*/ 4497 } 4498 4499 if (!page_try_demote_pages(pp)) { 4500 page_unlock(pp); 4501 continue; 4502 } 4503 4504 /* 4505 * Check the modified bit. Leave the bits alone in hardware 4506 * (they will be modified if we do the putpage). 4507 */ 4508 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4509 & P_MOD); 4510 if (mod) { 4511 offset = pp->p_offset; 4512 /* 4513 * Hold the vnode before releasing the page lock 4514 * to prevent it from being freed and re-used by 4515 * some other thread. 4516 */ 4517 VN_HOLD(vp); 4518 page_unlock(pp); 4519 /* 4520 * No error return is checked here. Callers such as 4521 * cpr deals with the dirty pages at the dump time 4522 * if this putpage fails. 4523 */ 4524 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4525 kcred, NULL); 4526 VN_RELE(vp); 4527 } else { 4528 /*LINTED: constant in conditional context*/ 4529 VN_DISPOSE(pp, B_INVAL, 0, kcred); 4530 } 4531 } while ((pp = page_next(pp)) != page0); 4532 if (nbusypages && retry++ < MAXRETRIES) { 4533 delay(1); 4534 goto top; 4535 } 4536 } 4537 4538 /* 4539 * Replace the page "old" with the page "new" on the page hash and vnode lists 4540 * 4541 * the replacement must be done in place, ie the equivalent sequence: 4542 * 4543 * vp = old->p_vnode; 4544 * off = old->p_offset; 4545 * page_do_hashout(old) 4546 * page_do_hashin(new, vp, off) 4547 * 4548 * doesn't work, since 4549 * 1) if old is the only page on the vnode, the v_pages list has a window 4550 * where it looks empty. This will break file system assumptions. 4551 * and 4552 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4553 */ 4554 static void 4555 page_do_relocate_hash(page_t *new, page_t *old) 4556 { 4557 page_t **hash_list; 4558 vnode_t *vp = old->p_vnode; 4559 kmutex_t *sep; 4560 4561 ASSERT(PAGE_EXCL(old)); 4562 ASSERT(PAGE_EXCL(new)); 4563 ASSERT(vp != NULL); 4564 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4565 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4566 4567 /* 4568 * First find old page on the page hash list 4569 */ 4570 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4571 4572 for (;;) { 4573 if (*hash_list == old) 4574 break; 4575 if (*hash_list == NULL) { 4576 panic("page_do_hashout"); 4577 /*NOTREACHED*/ 4578 } 4579 hash_list = &(*hash_list)->p_hash; 4580 } 4581 4582 /* 4583 * update new and replace old with new on the page hash list 4584 */ 4585 new->p_vnode = old->p_vnode; 4586 new->p_offset = old->p_offset; 4587 new->p_hash = old->p_hash; 4588 *hash_list = new; 4589 4590 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4591 PP_SETSWAP(new); 4592 4593 /* 4594 * replace old with new on the vnode's page list 4595 */ 4596 if (old->p_vpnext == old) { 4597 new->p_vpnext = new; 4598 new->p_vpprev = new; 4599 } else { 4600 new->p_vpnext = old->p_vpnext; 4601 new->p_vpprev = old->p_vpprev; 4602 new->p_vpnext->p_vpprev = new; 4603 new->p_vpprev->p_vpnext = new; 4604 } 4605 if (vp->v_pages == old) 4606 vp->v_pages = new; 4607 4608 /* 4609 * clear out the old page 4610 */ 4611 old->p_hash = NULL; 4612 old->p_vpnext = NULL; 4613 old->p_vpprev = NULL; 4614 old->p_vnode = NULL; 4615 PP_CLRSWAP(old); 4616 old->p_offset = (u_offset_t)-1; 4617 page_clr_all_props(old); 4618 4619 /* 4620 * Wake up processes waiting for this page. The page's 4621 * identity has been changed, and is probably not the 4622 * desired page any longer. 4623 */ 4624 sep = page_se_mutex(old); 4625 mutex_enter(sep); 4626 old->p_selock &= ~SE_EWANTED; 4627 if (CV_HAS_WAITERS(&old->p_cv)) 4628 cv_broadcast(&old->p_cv); 4629 mutex_exit(sep); 4630 } 4631 4632 /* 4633 * This function moves the identity of page "pp_old" to page "pp_new". 4634 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4635 * and need not be hashed out from anywhere. 4636 */ 4637 void 4638 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4639 { 4640 vnode_t *vp = pp_old->p_vnode; 4641 u_offset_t off = pp_old->p_offset; 4642 kmutex_t *phm, *vphm; 4643 4644 /* 4645 * Rehash two pages 4646 */ 4647 ASSERT(PAGE_EXCL(pp_old)); 4648 ASSERT(PAGE_EXCL(pp_new)); 4649 ASSERT(vp != NULL); 4650 ASSERT(pp_new->p_vnode == NULL); 4651 4652 /* 4653 * hashout then hashin while holding the mutexes 4654 */ 4655 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4656 mutex_enter(phm); 4657 vphm = page_vnode_mutex(vp); 4658 mutex_enter(vphm); 4659 4660 page_do_relocate_hash(pp_new, pp_old); 4661 4662 /* The following comment preserved from page_flip(). */ 4663 pp_new->p_fsdata = pp_old->p_fsdata; 4664 pp_old->p_fsdata = 0; 4665 mutex_exit(vphm); 4666 mutex_exit(phm); 4667 4668 /* 4669 * The page_struct_lock need not be acquired for lckcnt and 4670 * cowcnt since the page has an "exclusive" lock. 4671 */ 4672 ASSERT(pp_new->p_lckcnt == 0); 4673 ASSERT(pp_new->p_cowcnt == 0); 4674 pp_new->p_lckcnt = pp_old->p_lckcnt; 4675 pp_new->p_cowcnt = pp_old->p_cowcnt; 4676 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4677 4678 } 4679 4680 /* 4681 * Helper routine used to lock all remaining members of a 4682 * large page. The caller is responsible for passing in a locked 4683 * pp. If pp is a large page, then it succeeds in locking all the 4684 * remaining constituent pages or it returns with only the 4685 * original page locked. 4686 * 4687 * Returns 1 on success, 0 on failure. 4688 * 4689 * If success is returned this routine guarantees p_szc for all constituent 4690 * pages of a large page pp belongs to can't change. To achieve this we 4691 * recheck szc of pp after locking all constituent pages and retry if szc 4692 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4693 * lock on one of constituent pages it can't be running after all constituent 4694 * pages are locked. hat_page_demote() with a lock on a constituent page 4695 * outside of this large page (i.e. pp belonged to a larger large page) is 4696 * already done with all constituent pages of pp since the root's p_szc is 4697 * changed last. Therefore no need to synchronize with hat_page_demote() that 4698 * locked a constituent page outside of pp's current large page. 4699 */ 4700 #ifdef DEBUG 4701 uint32_t gpg_trylock_mtbf = 0; 4702 #endif 4703 4704 int 4705 group_page_trylock(page_t *pp, se_t se) 4706 { 4707 page_t *tpp; 4708 pgcnt_t npgs, i, j; 4709 uint_t pszc = pp->p_szc; 4710 4711 #ifdef DEBUG 4712 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4713 return (0); 4714 } 4715 #endif 4716 4717 if (pp != PP_GROUPLEADER(pp, pszc)) { 4718 return (0); 4719 } 4720 4721 retry: 4722 ASSERT(PAGE_LOCKED_SE(pp, se)); 4723 ASSERT(!PP_ISFREE(pp)); 4724 if (pszc == 0) { 4725 return (1); 4726 } 4727 npgs = page_get_pagecnt(pszc); 4728 tpp = pp + 1; 4729 for (i = 1; i < npgs; i++, tpp++) { 4730 if (!page_trylock(tpp, se)) { 4731 tpp = pp + 1; 4732 for (j = 1; j < i; j++, tpp++) { 4733 page_unlock(tpp); 4734 } 4735 return (0); 4736 } 4737 } 4738 if (pp->p_szc != pszc) { 4739 ASSERT(pp->p_szc < pszc); 4740 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && 4741 !IS_SWAPFSVP(pp->p_vnode)); 4742 tpp = pp + 1; 4743 for (i = 1; i < npgs; i++, tpp++) { 4744 page_unlock(tpp); 4745 } 4746 pszc = pp->p_szc; 4747 goto retry; 4748 } 4749 return (1); 4750 } 4751 4752 void 4753 group_page_unlock(page_t *pp) 4754 { 4755 page_t *tpp; 4756 pgcnt_t npgs, i; 4757 4758 ASSERT(PAGE_LOCKED(pp)); 4759 ASSERT(!PP_ISFREE(pp)); 4760 ASSERT(pp == PP_PAGEROOT(pp)); 4761 npgs = page_get_pagecnt(pp->p_szc); 4762 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4763 page_unlock(tpp); 4764 } 4765 } 4766 4767 /* 4768 * returns 4769 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4770 * ERANGE : this is not a base page 4771 * EBUSY : failure to get locks on the page/pages 4772 * ENOMEM : failure to obtain replacement pages 4773 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4774 * EIO : An error occurred while trying to copy the page data 4775 * 4776 * Return with all constituent members of target and replacement 4777 * SE_EXCL locked. It is the callers responsibility to drop the 4778 * locks. 4779 */ 4780 int 4781 do_page_relocate( 4782 page_t **target, 4783 page_t **replacement, 4784 int grouplock, 4785 spgcnt_t *nrelocp, 4786 lgrp_t *lgrp) 4787 { 4788 page_t *first_repl; 4789 page_t *repl; 4790 page_t *targ; 4791 page_t *pl = NULL; 4792 uint_t ppattr; 4793 pfn_t pfn, repl_pfn; 4794 uint_t szc; 4795 spgcnt_t npgs, i; 4796 int repl_contig = 0; 4797 uint_t flags = 0; 4798 spgcnt_t dofree = 0; 4799 4800 *nrelocp = 0; 4801 4802 #if defined(__sparc) 4803 /* 4804 * We need to wait till OBP has completed 4805 * its boot-time handoff of its resources to the kernel 4806 * before we allow page relocation 4807 */ 4808 if (page_relocate_ready == 0) { 4809 return (EAGAIN); 4810 } 4811 #endif 4812 4813 /* 4814 * If this is not a base page, 4815 * just return with 0x0 pages relocated. 4816 */ 4817 targ = *target; 4818 ASSERT(PAGE_EXCL(targ)); 4819 ASSERT(!PP_ISFREE(targ)); 4820 szc = targ->p_szc; 4821 ASSERT(szc < mmu_page_sizes); 4822 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4823 pfn = targ->p_pagenum; 4824 if (pfn != PFN_BASE(pfn, szc)) { 4825 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4826 return (ERANGE); 4827 } 4828 4829 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4830 repl_pfn = repl->p_pagenum; 4831 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4832 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4833 return (ERANGE); 4834 } 4835 repl_contig = 1; 4836 } 4837 4838 /* 4839 * We must lock all members of this large page or we cannot 4840 * relocate any part of it. 4841 */ 4842 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4843 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4844 return (EBUSY); 4845 } 4846 4847 /* 4848 * reread szc it could have been decreased before 4849 * group_page_trylock() was done. 4850 */ 4851 szc = targ->p_szc; 4852 ASSERT(szc < mmu_page_sizes); 4853 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4854 ASSERT(pfn == PFN_BASE(pfn, szc)); 4855 4856 npgs = page_get_pagecnt(targ->p_szc); 4857 4858 if (repl == NULL) { 4859 dofree = npgs; /* Size of target page in MMU pages */ 4860 if (!page_create_wait(dofree, 0)) { 4861 if (grouplock != 0) { 4862 group_page_unlock(targ); 4863 } 4864 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4865 return (ENOMEM); 4866 } 4867 4868 /* 4869 * seg kmem pages require that the target and replacement 4870 * page be the same pagesize. 4871 */ 4872 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0; 4873 repl = page_get_replacement_page(targ, lgrp, flags); 4874 if (repl == NULL) { 4875 if (grouplock != 0) { 4876 group_page_unlock(targ); 4877 } 4878 page_create_putback(dofree); 4879 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4880 return (ENOMEM); 4881 } 4882 } 4883 #ifdef DEBUG 4884 else { 4885 ASSERT(PAGE_LOCKED(repl)); 4886 } 4887 #endif /* DEBUG */ 4888 4889 #if defined(__sparc) 4890 /* 4891 * Let hat_page_relocate() complete the relocation if it's kernel page 4892 */ 4893 if (VN_ISKAS(targ->p_vnode)) { 4894 *replacement = repl; 4895 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4896 if (grouplock != 0) { 4897 group_page_unlock(targ); 4898 } 4899 if (dofree) { 4900 *replacement = NULL; 4901 page_free_replacement_page(repl); 4902 page_create_putback(dofree); 4903 } 4904 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4905 return (EAGAIN); 4906 } 4907 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4908 return (0); 4909 } 4910 #else 4911 #if defined(lint) 4912 dofree = dofree; 4913 #endif 4914 #endif 4915 4916 first_repl = repl; 4917 4918 for (i = 0; i < npgs; i++) { 4919 ASSERT(PAGE_EXCL(targ)); 4920 ASSERT(targ->p_slckcnt == 0); 4921 ASSERT(repl->p_slckcnt == 0); 4922 4923 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4924 4925 ASSERT(hat_page_getshare(targ) == 0); 4926 ASSERT(!PP_ISFREE(targ)); 4927 ASSERT(targ->p_pagenum == (pfn + i)); 4928 ASSERT(repl_contig == 0 || 4929 repl->p_pagenum == (repl_pfn + i)); 4930 4931 /* 4932 * Copy the page contents and attributes then 4933 * relocate the page in the page hash. 4934 */ 4935 if (ppcopy(targ, repl) == 0) { 4936 targ = *target; 4937 repl = first_repl; 4938 VM_STAT_ADD(vmm_vmstats.ppr_copyfail); 4939 if (grouplock != 0) { 4940 group_page_unlock(targ); 4941 } 4942 if (dofree) { 4943 *replacement = NULL; 4944 page_free_replacement_page(repl); 4945 page_create_putback(dofree); 4946 } 4947 return (EIO); 4948 } 4949 4950 targ++; 4951 if (repl_contig != 0) { 4952 repl++; 4953 } else { 4954 repl = repl->p_next; 4955 } 4956 } 4957 4958 repl = first_repl; 4959 targ = *target; 4960 4961 for (i = 0; i < npgs; i++) { 4962 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4963 page_clr_all_props(repl); 4964 page_set_props(repl, ppattr); 4965 page_relocate_hash(repl, targ); 4966 4967 ASSERT(hat_page_getshare(targ) == 0); 4968 ASSERT(hat_page_getshare(repl) == 0); 4969 /* 4970 * Now clear the props on targ, after the 4971 * page_relocate_hash(), they no longer 4972 * have any meaning. 4973 */ 4974 page_clr_all_props(targ); 4975 ASSERT(targ->p_next == targ); 4976 ASSERT(targ->p_prev == targ); 4977 page_list_concat(&pl, &targ); 4978 4979 targ++; 4980 if (repl_contig != 0) { 4981 repl++; 4982 } else { 4983 repl = repl->p_next; 4984 } 4985 } 4986 /* assert that we have come full circle with repl */ 4987 ASSERT(repl_contig == 1 || first_repl == repl); 4988 4989 *target = pl; 4990 if (*replacement == NULL) { 4991 ASSERT(first_repl == repl); 4992 *replacement = repl; 4993 } 4994 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4995 *nrelocp = npgs; 4996 return (0); 4997 } 4998 /* 4999 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 5000 */ 5001 int 5002 page_relocate( 5003 page_t **target, 5004 page_t **replacement, 5005 int grouplock, 5006 int freetarget, 5007 spgcnt_t *nrelocp, 5008 lgrp_t *lgrp) 5009 { 5010 spgcnt_t ret; 5011 5012 /* do_page_relocate returns 0 on success or errno value */ 5013 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 5014 5015 if (ret != 0 || freetarget == 0) { 5016 return (ret); 5017 } 5018 if (*nrelocp == 1) { 5019 ASSERT(*target != NULL); 5020 page_free(*target, 1); 5021 } else { 5022 page_t *tpp = *target; 5023 uint_t szc = tpp->p_szc; 5024 pgcnt_t npgs = page_get_pagecnt(szc); 5025 ASSERT(npgs > 1); 5026 ASSERT(szc != 0); 5027 do { 5028 ASSERT(PAGE_EXCL(tpp)); 5029 ASSERT(!hat_page_is_mapped(tpp)); 5030 ASSERT(tpp->p_szc == szc); 5031 PP_SETFREE(tpp); 5032 PP_SETAGED(tpp); 5033 npgs--; 5034 } while ((tpp = tpp->p_next) != *target); 5035 ASSERT(npgs == 0); 5036 page_list_add_pages(*target, 0); 5037 npgs = page_get_pagecnt(szc); 5038 page_create_putback(npgs); 5039 } 5040 return (ret); 5041 } 5042 5043 /* 5044 * it is up to the caller to deal with pcf accounting. 5045 */ 5046 void 5047 page_free_replacement_page(page_t *pplist) 5048 { 5049 page_t *pp; 5050 5051 while (pplist != NULL) { 5052 /* 5053 * pp_targ is a linked list. 5054 */ 5055 pp = pplist; 5056 if (pp->p_szc == 0) { 5057 page_sub(&pplist, pp); 5058 page_clr_all_props(pp); 5059 PP_SETFREE(pp); 5060 PP_SETAGED(pp); 5061 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5062 page_unlock(pp); 5063 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5064 } else { 5065 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5066 page_t *tpp; 5067 page_list_break(&pp, &pplist, curnpgs); 5068 tpp = pp; 5069 do { 5070 ASSERT(PAGE_EXCL(tpp)); 5071 ASSERT(!hat_page_is_mapped(tpp)); 5072 page_clr_all_props(tpp); 5073 PP_SETFREE(tpp); 5074 PP_SETAGED(tpp); 5075 } while ((tpp = tpp->p_next) != pp); 5076 page_list_add_pages(pp, 0); 5077 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5078 } 5079 } 5080 } 5081 5082 /* 5083 * Relocate target to non-relocatable replacement page. 5084 */ 5085 int 5086 page_relocate_cage(page_t **target, page_t **replacement) 5087 { 5088 page_t *tpp, *rpp; 5089 spgcnt_t pgcnt, npgs; 5090 int result; 5091 5092 tpp = *target; 5093 5094 ASSERT(PAGE_EXCL(tpp)); 5095 ASSERT(tpp->p_szc == 0); 5096 5097 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5098 5099 do { 5100 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5101 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5102 if (rpp == NULL) { 5103 page_create_putback(pgcnt); 5104 kcage_cageout_wakeup(); 5105 } 5106 } while (rpp == NULL); 5107 5108 ASSERT(PP_ISNORELOC(rpp)); 5109 5110 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5111 5112 if (result == 0) { 5113 *replacement = rpp; 5114 if (pgcnt != npgs) 5115 panic("page_relocate_cage: partial relocation"); 5116 } 5117 5118 return (result); 5119 } 5120 5121 /* 5122 * Release the page lock on a page, place on cachelist 5123 * tail if no longer mapped. Caller can let us know if 5124 * the page is known to be clean. 5125 */ 5126 int 5127 page_release(page_t *pp, int checkmod) 5128 { 5129 int status; 5130 5131 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5132 (pp->p_vnode != NULL)); 5133 5134 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5135 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5136 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5137 !hat_page_is_mapped(pp)) { 5138 5139 /* 5140 * If page is modified, unlock it 5141 * 5142 * (p_nrm & P_MOD) bit has the latest stuff because: 5143 * (1) We found that this page doesn't have any mappings 5144 * _after_ holding SE_EXCL and 5145 * (2) We didn't drop SE_EXCL lock after the check in (1) 5146 */ 5147 if (checkmod && hat_ismod(pp)) { 5148 page_unlock(pp); 5149 status = PGREL_MOD; 5150 } else { 5151 /*LINTED: constant in conditional context*/ 5152 VN_DISPOSE(pp, B_FREE, 0, kcred); 5153 status = PGREL_CLEAN; 5154 } 5155 } else { 5156 page_unlock(pp); 5157 status = PGREL_NOTREL; 5158 } 5159 return (status); 5160 } 5161 5162 /* 5163 * Given a constituent page, try to demote the large page on the freelist. 5164 * 5165 * Returns nonzero if the page could be demoted successfully. Returns with 5166 * the constituent page still locked. 5167 */ 5168 int 5169 page_try_demote_free_pages(page_t *pp) 5170 { 5171 page_t *rootpp = pp; 5172 pfn_t pfn = page_pptonum(pp); 5173 spgcnt_t npgs; 5174 uint_t szc = pp->p_szc; 5175 5176 ASSERT(PP_ISFREE(pp)); 5177 ASSERT(PAGE_EXCL(pp)); 5178 5179 /* 5180 * Adjust rootpp and lock it, if `pp' is not the base 5181 * constituent page. 5182 */ 5183 npgs = page_get_pagecnt(pp->p_szc); 5184 if (npgs == 1) { 5185 return (0); 5186 } 5187 5188 if (!IS_P2ALIGNED(pfn, npgs)) { 5189 pfn = P2ALIGN(pfn, npgs); 5190 rootpp = page_numtopp_nolock(pfn); 5191 } 5192 5193 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5194 return (0); 5195 } 5196 5197 if (rootpp->p_szc != szc) { 5198 if (pp != rootpp) 5199 page_unlock(rootpp); 5200 return (0); 5201 } 5202 5203 page_demote_free_pages(rootpp); 5204 5205 if (pp != rootpp) 5206 page_unlock(rootpp); 5207 5208 ASSERT(PP_ISFREE(pp)); 5209 ASSERT(PAGE_EXCL(pp)); 5210 return (1); 5211 } 5212 5213 /* 5214 * Given a constituent page, try to demote the large page. 5215 * 5216 * Returns nonzero if the page could be demoted successfully. Returns with 5217 * the constituent page still locked. 5218 */ 5219 int 5220 page_try_demote_pages(page_t *pp) 5221 { 5222 page_t *tpp, *rootpp = pp; 5223 pfn_t pfn = page_pptonum(pp); 5224 spgcnt_t i, npgs; 5225 uint_t szc = pp->p_szc; 5226 vnode_t *vp = pp->p_vnode; 5227 5228 ASSERT(PAGE_EXCL(pp)); 5229 5230 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5231 5232 if (pp->p_szc == 0) { 5233 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5234 return (1); 5235 } 5236 5237 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) { 5238 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5239 page_demote_vp_pages(pp); 5240 ASSERT(pp->p_szc == 0); 5241 return (1); 5242 } 5243 5244 /* 5245 * Adjust rootpp if passed in is not the base 5246 * constituent page. 5247 */ 5248 npgs = page_get_pagecnt(pp->p_szc); 5249 ASSERT(npgs > 1); 5250 if (!IS_P2ALIGNED(pfn, npgs)) { 5251 pfn = P2ALIGN(pfn, npgs); 5252 rootpp = page_numtopp_nolock(pfn); 5253 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5254 ASSERT(rootpp->p_vnode != NULL); 5255 ASSERT(rootpp->p_szc == szc); 5256 } 5257 5258 /* 5259 * We can't demote kernel pages since we can't hat_unload() 5260 * the mappings. 5261 */ 5262 if (VN_ISKAS(rootpp->p_vnode)) 5263 return (0); 5264 5265 /* 5266 * Attempt to lock all constituent pages except the page passed 5267 * in since it's already locked. 5268 */ 5269 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5270 ASSERT(!PP_ISFREE(tpp)); 5271 ASSERT(tpp->p_vnode != NULL); 5272 5273 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5274 break; 5275 ASSERT(tpp->p_szc == rootpp->p_szc); 5276 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5277 } 5278 5279 /* 5280 * If we failed to lock them all then unlock what we have 5281 * locked so far and bail. 5282 */ 5283 if (i < npgs) { 5284 tpp = rootpp; 5285 while (i-- > 0) { 5286 if (tpp != pp) 5287 page_unlock(tpp); 5288 tpp++; 5289 } 5290 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5291 return (0); 5292 } 5293 5294 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5295 ASSERT(PAGE_EXCL(tpp)); 5296 ASSERT(tpp->p_slckcnt == 0); 5297 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5298 tpp->p_szc = 0; 5299 } 5300 5301 /* 5302 * Unlock all pages except the page passed in. 5303 */ 5304 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5305 ASSERT(!hat_page_is_mapped(tpp)); 5306 if (tpp != pp) 5307 page_unlock(tpp); 5308 } 5309 5310 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5311 return (1); 5312 } 5313 5314 /* 5315 * Called by page_free() and page_destroy() to demote the page size code 5316 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5317 * p_szc on free list, neither can we just clear p_szc of a single page_t 5318 * within a large page since it will break other code that relies on p_szc 5319 * being the same for all page_t's of a large page). Anonymous pages should 5320 * never end up here because anon_map_getpages() cannot deal with p_szc 5321 * changes after a single constituent page is locked. While anonymous or 5322 * kernel large pages are demoted or freed the entire large page at a time 5323 * with all constituent pages locked EXCL for the file system pages we 5324 * have to be able to demote a large page (i.e. decrease all constituent pages 5325 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5326 * we can easily deal with anonymous page demotion the entire large page at a 5327 * time is that those operation originate at address space level and concern 5328 * the entire large page region with actual demotion only done when pages are 5329 * not shared with any other processes (therefore we can always get EXCL lock 5330 * on all anonymous constituent pages after clearing segment page 5331 * cache). However file system pages can be truncated or invalidated at a 5332 * PAGESIZE level from the file system side and end up in page_free() or 5333 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5334 * and therefore pageout should be able to demote a large page by EXCL locking 5335 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5336 * rely on being able to lock EXCL all constituent pages. 5337 * 5338 * To prevent szc changes on file system pages one has to lock all constituent 5339 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5340 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5341 * prevent szc changes is hat layer that uses its own page level mlist 5342 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5343 * taken. Therefore we need to change szc under hat level locks if we only 5344 * have an EXCL lock on a single constituent page and hat still references any 5345 * of constituent pages. (Note we can't "ignore" hat layer by simply 5346 * hat_pageunload() all constituent pages without having EXCL locks on all of 5347 * constituent pages). We use hat_page_demote() call to safely demote szc of 5348 * all constituent pages under hat locks when we only have an EXCL lock on one 5349 * of constituent pages. 5350 * 5351 * This routine calls page_szc_lock() before calling hat_page_demote() to 5352 * allow segvn in one special case not to lock all constituent pages SHARED 5353 * before calling hat_memload_array() that relies on p_szc not changing even 5354 * before hat level mlist lock is taken. In that case segvn uses 5355 * page_szc_lock() to prevent hat_page_demote() changing p_szc values. 5356 * 5357 * Anonymous or kernel page demotion still has to lock all pages exclusively 5358 * and do hat_pageunload() on all constituent pages before demoting the page 5359 * therefore there's no need for anonymous or kernel page demotion to use 5360 * hat_page_demote() mechanism. 5361 * 5362 * hat_page_demote() removes all large mappings that map pp and then decreases 5363 * p_szc starting from the last constituent page of the large page. By working 5364 * from the tail of a large page in pfn decreasing order allows one looking at 5365 * the root page to know that hat_page_demote() is done for root's szc area. 5366 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5367 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5368 * that started on this page when it had szc > 1 is done for this szc 1 area. 5369 * 5370 * We are guaranteed that all constituent pages of pp's large page belong to 5371 * the same vnode with the consecutive offsets increasing in the direction of 5372 * the pfn i.e. the identity of constituent pages can't change until their 5373 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5374 * large mappings to pp even though we don't lock any constituent page except 5375 * pp (i.e. we won't unload e.g. kernel locked page). 5376 */ 5377 static void 5378 page_demote_vp_pages(page_t *pp) 5379 { 5380 kmutex_t *mtx; 5381 5382 ASSERT(PAGE_EXCL(pp)); 5383 ASSERT(!PP_ISFREE(pp)); 5384 ASSERT(pp->p_vnode != NULL); 5385 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5386 ASSERT(!PP_ISKAS(pp)); 5387 5388 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5389 5390 mtx = page_szc_lock(pp); 5391 if (mtx != NULL) { 5392 hat_page_demote(pp); 5393 mutex_exit(mtx); 5394 } 5395 ASSERT(pp->p_szc == 0); 5396 } 5397 5398 /* 5399 * Mark any existing pages for migration in the given range 5400 */ 5401 void 5402 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5403 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5404 u_offset_t vnoff, int rflag) 5405 { 5406 struct anon *ap; 5407 vnode_t *curvp; 5408 lgrp_t *from; 5409 pgcnt_t nlocked; 5410 u_offset_t off; 5411 pfn_t pfn; 5412 size_t pgsz; 5413 size_t segpgsz; 5414 pgcnt_t pages; 5415 uint_t pszc; 5416 page_t *pp0, *pp; 5417 caddr_t va; 5418 ulong_t an_idx; 5419 anon_sync_obj_t cookie; 5420 5421 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 5422 5423 /* 5424 * Don't do anything if don't need to do lgroup optimizations 5425 * on this system 5426 */ 5427 if (!lgrp_optimizations()) 5428 return; 5429 5430 /* 5431 * Align address and length to (potentially large) page boundary 5432 */ 5433 segpgsz = page_get_pagesize(seg->s_szc); 5434 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5435 if (rflag) 5436 len = P2ROUNDUP(len, segpgsz); 5437 5438 /* 5439 * Do one (large) page at a time 5440 */ 5441 va = addr; 5442 while (va < addr + len) { 5443 /* 5444 * Lookup (root) page for vnode and offset corresponding to 5445 * this virtual address 5446 * Try anonmap first since there may be copy-on-write 5447 * pages, but initialize vnode pointer and offset using 5448 * vnode arguments just in case there isn't an amp. 5449 */ 5450 curvp = vp; 5451 off = vnoff + va - seg->s_base; 5452 if (amp) { 5453 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5454 an_idx = anon_index + seg_page(seg, va); 5455 anon_array_enter(amp, an_idx, &cookie); 5456 ap = anon_get_ptr(amp->ahp, an_idx); 5457 if (ap) 5458 swap_xlate(ap, &curvp, &off); 5459 anon_array_exit(&cookie); 5460 ANON_LOCK_EXIT(&->a_rwlock); 5461 } 5462 5463 pp = NULL; 5464 if (curvp) 5465 pp = page_lookup(curvp, off, SE_SHARED); 5466 5467 /* 5468 * If there isn't a page at this virtual address, 5469 * skip to next page 5470 */ 5471 if (pp == NULL) { 5472 va += PAGESIZE; 5473 continue; 5474 } 5475 5476 /* 5477 * Figure out which lgroup this page is in for kstats 5478 */ 5479 pfn = page_pptonum(pp); 5480 from = lgrp_pfn_to_lgrp(pfn); 5481 5482 /* 5483 * Get page size, and round up and skip to next page boundary 5484 * if unaligned address 5485 */ 5486 pszc = pp->p_szc; 5487 pgsz = page_get_pagesize(pszc); 5488 pages = btop(pgsz); 5489 if (!IS_P2ALIGNED(va, pgsz) || 5490 !IS_P2ALIGNED(pfn, pages) || 5491 pgsz > segpgsz) { 5492 pgsz = MIN(pgsz, segpgsz); 5493 page_unlock(pp); 5494 pages = btop(P2END((uintptr_t)va, pgsz) - 5495 (uintptr_t)va); 5496 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5497 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, pages); 5498 continue; 5499 } 5500 5501 /* 5502 * Upgrade to exclusive lock on page 5503 */ 5504 if (!page_tryupgrade(pp)) { 5505 page_unlock(pp); 5506 va += pgsz; 5507 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5508 btop(pgsz)); 5509 continue; 5510 } 5511 5512 pp0 = pp++; 5513 nlocked = 1; 5514 5515 /* 5516 * Lock constituent pages if this is large page 5517 */ 5518 if (pages > 1) { 5519 /* 5520 * Lock all constituents except root page, since it 5521 * should be locked already. 5522 */ 5523 for (; nlocked < pages; nlocked++) { 5524 if (!page_trylock(pp, SE_EXCL)) { 5525 break; 5526 } 5527 if (PP_ISFREE(pp) || 5528 pp->p_szc != pszc) { 5529 /* 5530 * hat_page_demote() raced in with us. 5531 */ 5532 ASSERT(!IS_SWAPFSVP(curvp)); 5533 page_unlock(pp); 5534 break; 5535 } 5536 pp++; 5537 } 5538 } 5539 5540 /* 5541 * If all constituent pages couldn't be locked, 5542 * unlock pages locked so far and skip to next page. 5543 */ 5544 if (nlocked < pages) { 5545 while (pp0 < pp) { 5546 page_unlock(pp0++); 5547 } 5548 va += pgsz; 5549 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5550 btop(pgsz)); 5551 continue; 5552 } 5553 5554 /* 5555 * hat_page_demote() can no longer happen 5556 * since last cons page had the right p_szc after 5557 * all cons pages were locked. all cons pages 5558 * should now have the same p_szc. 5559 */ 5560 5561 /* 5562 * All constituent pages locked successfully, so mark 5563 * large page for migration and unload the mappings of 5564 * constituent pages, so a fault will occur on any part of the 5565 * large page 5566 */ 5567 PP_SETMIGRATE(pp0); 5568 while (pp0 < pp) { 5569 (void) hat_pageunload(pp0, HAT_FORCE_PGUNLOAD); 5570 ASSERT(hat_page_getshare(pp0) == 0); 5571 page_unlock(pp0++); 5572 } 5573 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5574 5575 va += pgsz; 5576 } 5577 } 5578 5579 /* 5580 * Migrate any pages that have been marked for migration in the given range 5581 */ 5582 void 5583 page_migrate( 5584 struct seg *seg, 5585 caddr_t addr, 5586 page_t **ppa, 5587 pgcnt_t npages) 5588 { 5589 lgrp_t *from; 5590 lgrp_t *to; 5591 page_t *newpp; 5592 page_t *pp; 5593 pfn_t pfn; 5594 size_t pgsz; 5595 spgcnt_t page_cnt; 5596 spgcnt_t i; 5597 uint_t pszc; 5598 5599 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 5600 5601 while (npages > 0) { 5602 pp = *ppa; 5603 pszc = pp->p_szc; 5604 pgsz = page_get_pagesize(pszc); 5605 page_cnt = btop(pgsz); 5606 5607 /* 5608 * Check to see whether this page is marked for migration 5609 * 5610 * Assume that root page of large page is marked for 5611 * migration and none of the other constituent pages 5612 * are marked. This really simplifies clearing the 5613 * migrate bit by not having to clear it from each 5614 * constituent page. 5615 * 5616 * note we don't want to relocate an entire large page if 5617 * someone is only using one subpage. 5618 */ 5619 if (npages < page_cnt) 5620 break; 5621 5622 /* 5623 * Is it marked for migration? 5624 */ 5625 if (!PP_ISMIGRATE(pp)) 5626 goto next; 5627 5628 /* 5629 * Determine lgroups that page is being migrated between 5630 */ 5631 pfn = page_pptonum(pp); 5632 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5633 break; 5634 } 5635 from = lgrp_pfn_to_lgrp(pfn); 5636 to = lgrp_mem_choose(seg, addr, pgsz); 5637 5638 /* 5639 * Need to get exclusive lock's to migrate 5640 */ 5641 for (i = 0; i < page_cnt; i++) { 5642 ASSERT(PAGE_LOCKED(ppa[i])); 5643 if (page_pptonum(ppa[i]) != pfn + i || 5644 ppa[i]->p_szc != pszc) { 5645 break; 5646 } 5647 if (!page_tryupgrade(ppa[i])) { 5648 lgrp_stat_add(from->lgrp_id, 5649 LGRP_PM_FAIL_LOCK_PGS, 5650 page_cnt); 5651 break; 5652 } 5653 5654 /* 5655 * Check to see whether we are trying to migrate 5656 * page to lgroup where it is allocated already. 5657 * If so, clear the migrate bit and skip to next 5658 * page. 5659 */ 5660 if (i == 0 && to == from) { 5661 PP_CLRMIGRATE(ppa[0]); 5662 page_downgrade(ppa[0]); 5663 goto next; 5664 } 5665 } 5666 5667 /* 5668 * If all constituent pages couldn't be locked, 5669 * unlock pages locked so far and skip to next page. 5670 */ 5671 if (i != page_cnt) { 5672 while (--i != -1) { 5673 page_downgrade(ppa[i]); 5674 } 5675 goto next; 5676 } 5677 5678 (void) page_create_wait(page_cnt, PG_WAIT); 5679 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5680 if (newpp == NULL) { 5681 page_create_putback(page_cnt); 5682 for (i = 0; i < page_cnt; i++) { 5683 page_downgrade(ppa[i]); 5684 } 5685 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5686 page_cnt); 5687 goto next; 5688 } 5689 ASSERT(newpp->p_szc == pszc); 5690 /* 5691 * Clear migrate bit and relocate page 5692 */ 5693 PP_CLRMIGRATE(pp); 5694 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5695 panic("page_migrate: page_relocate failed"); 5696 } 5697 ASSERT(page_cnt * PAGESIZE == pgsz); 5698 5699 /* 5700 * Keep stats for number of pages migrated from and to 5701 * each lgroup 5702 */ 5703 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5704 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5705 /* 5706 * update the page_t array we were passed in and 5707 * unlink constituent pages of a large page. 5708 */ 5709 for (i = 0; i < page_cnt; ++i, ++pp) { 5710 ASSERT(PAGE_EXCL(newpp)); 5711 ASSERT(newpp->p_szc == pszc); 5712 ppa[i] = newpp; 5713 pp = newpp; 5714 page_sub(&newpp, pp); 5715 page_downgrade(pp); 5716 } 5717 ASSERT(newpp == NULL); 5718 next: 5719 addr += pgsz; 5720 ppa += page_cnt; 5721 npages -= page_cnt; 5722 } 5723 } 5724 5725 uint_t page_reclaim_maxcnt = 60; /* max total iterations */ 5726 uint_t page_reclaim_nofree_maxcnt = 3; /* max iterations without progress */ 5727 /* 5728 * Reclaim/reserve availrmem for npages. 5729 * If there is not enough memory start reaping seg, kmem caches. 5730 * Start pageout scanner (via page_needfree()). 5731 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5732 * Note: There is no guarantee that any availrmem will be freed as 5733 * this memory typically is locked (kernel heap) or reserved for swap. 5734 * Also due to memory fragmentation kmem allocator may not be able 5735 * to free any memory (single user allocated buffer will prevent 5736 * freeing slab or a page). 5737 */ 5738 int 5739 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5740 { 5741 int i = 0; 5742 int i_nofree = 0; 5743 int ret = 0; 5744 pgcnt_t deficit; 5745 pgcnt_t old_availrmem = 0; 5746 5747 mutex_enter(&freemem_lock); 5748 while (availrmem < tune.t_minarmem + npages + epages && 5749 i++ < page_reclaim_maxcnt) { 5750 /* ensure we made some progress in the last few iterations */ 5751 if (old_availrmem < availrmem) { 5752 old_availrmem = availrmem; 5753 i_nofree = 0; 5754 } else if (i_nofree++ >= page_reclaim_nofree_maxcnt) { 5755 break; 5756 } 5757 5758 deficit = tune.t_minarmem + npages + epages - availrmem; 5759 mutex_exit(&freemem_lock); 5760 page_needfree(deficit); 5761 kmem_reap(); 5762 delay(hz); 5763 page_needfree(-(spgcnt_t)deficit); 5764 mutex_enter(&freemem_lock); 5765 } 5766 5767 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5768 availrmem -= npages; 5769 ret = 1; 5770 } 5771 5772 mutex_exit(&freemem_lock); 5773 5774 return (ret); 5775 } 5776 5777 /* 5778 * Search the memory segments to locate the desired page. Within a 5779 * segment, pages increase linearly with one page structure per 5780 * physical page frame (size PAGESIZE). The search begins 5781 * with the segment that was accessed last, to take advantage of locality. 5782 * If the hint misses, we start from the beginning of the sorted memseg list 5783 */ 5784 5785 5786 /* 5787 * Some data structures for pfn to pp lookup. 5788 */ 5789 ulong_t mhash_per_slot; 5790 struct memseg *memseg_hash[N_MEM_SLOTS]; 5791 5792 page_t * 5793 page_numtopp_nolock(pfn_t pfnum) 5794 { 5795 struct memseg *seg; 5796 page_t *pp; 5797 vm_cpu_data_t *vc; 5798 5799 /* 5800 * We need to disable kernel preemption while referencing the 5801 * cpu_vm_data field in order to prevent us from being switched to 5802 * another cpu and trying to reference it after it has been freed. 5803 * This will keep us on cpu and prevent it from being removed while 5804 * we are still on it. 5805 * 5806 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5807 * which is being resued by DR who will flush those references 5808 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5809 */ 5810 kpreempt_disable(); 5811 vc = CPU->cpu_vm_data; 5812 ASSERT(vc != NULL); 5813 5814 MEMSEG_STAT_INCR(nsearch); 5815 5816 /* Try last winner first */ 5817 if (((seg = vc->vc_pnum_memseg) != NULL) && 5818 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5819 MEMSEG_STAT_INCR(nlastwon); 5820 pp = seg->pages + (pfnum - seg->pages_base); 5821 if (pp->p_pagenum == pfnum) { 5822 kpreempt_enable(); 5823 return ((page_t *)pp); 5824 } 5825 } 5826 5827 /* Else Try hash */ 5828 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5829 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5830 MEMSEG_STAT_INCR(nhashwon); 5831 vc->vc_pnum_memseg = seg; 5832 pp = seg->pages + (pfnum - seg->pages_base); 5833 if (pp->p_pagenum == pfnum) { 5834 kpreempt_enable(); 5835 return ((page_t *)pp); 5836 } 5837 } 5838 5839 /* Else Brute force */ 5840 for (seg = memsegs; seg != NULL; seg = seg->next) { 5841 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5842 vc->vc_pnum_memseg = seg; 5843 pp = seg->pages + (pfnum - seg->pages_base); 5844 if (pp->p_pagenum == pfnum) { 5845 kpreempt_enable(); 5846 return ((page_t *)pp); 5847 } 5848 } 5849 } 5850 vc->vc_pnum_memseg = NULL; 5851 kpreempt_enable(); 5852 MEMSEG_STAT_INCR(nnotfound); 5853 return ((page_t *)NULL); 5854 5855 } 5856 5857 struct memseg * 5858 page_numtomemseg_nolock(pfn_t pfnum) 5859 { 5860 struct memseg *seg; 5861 page_t *pp; 5862 5863 /* 5864 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5865 * which is being resued by DR who will flush those references 5866 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5867 */ 5868 kpreempt_disable(); 5869 /* Try hash */ 5870 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5871 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5872 pp = seg->pages + (pfnum - seg->pages_base); 5873 if (pp->p_pagenum == pfnum) { 5874 kpreempt_enable(); 5875 return (seg); 5876 } 5877 } 5878 5879 /* Else Brute force */ 5880 for (seg = memsegs; seg != NULL; seg = seg->next) { 5881 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5882 pp = seg->pages + (pfnum - seg->pages_base); 5883 if (pp->p_pagenum == pfnum) { 5884 kpreempt_enable(); 5885 return (seg); 5886 } 5887 } 5888 } 5889 kpreempt_enable(); 5890 return ((struct memseg *)NULL); 5891 } 5892 5893 /* 5894 * Given a page and a count return the page struct that is 5895 * n structs away from the current one in the global page 5896 * list. 5897 * 5898 * This function wraps to the first page upon 5899 * reaching the end of the memseg list. 5900 */ 5901 page_t * 5902 page_nextn(page_t *pp, ulong_t n) 5903 { 5904 struct memseg *seg; 5905 page_t *ppn; 5906 vm_cpu_data_t *vc; 5907 5908 /* 5909 * We need to disable kernel preemption while referencing the 5910 * cpu_vm_data field in order to prevent us from being switched to 5911 * another cpu and trying to reference it after it has been freed. 5912 * This will keep us on cpu and prevent it from being removed while 5913 * we are still on it. 5914 * 5915 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5916 * which is being resued by DR who will flush those references 5917 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5918 */ 5919 kpreempt_disable(); 5920 vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5921 5922 ASSERT(vc != NULL); 5923 5924 if (((seg = vc->vc_pnext_memseg) == NULL) || 5925 (seg->pages_base == seg->pages_end) || 5926 !(pp >= seg->pages && pp < seg->epages)) { 5927 5928 for (seg = memsegs; seg; seg = seg->next) { 5929 if (pp >= seg->pages && pp < seg->epages) 5930 break; 5931 } 5932 5933 if (seg == NULL) { 5934 /* Memory delete got in, return something valid. */ 5935 /* TODO: fix me. */ 5936 seg = memsegs; 5937 pp = seg->pages; 5938 } 5939 } 5940 5941 /* check for wraparound - possible if n is large */ 5942 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5943 n -= seg->epages - pp; 5944 seg = seg->next; 5945 if (seg == NULL) 5946 seg = memsegs; 5947 pp = seg->pages; 5948 } 5949 vc->vc_pnext_memseg = seg; 5950 kpreempt_enable(); 5951 return (ppn); 5952 } 5953 5954 /* 5955 * Initialize for a loop using page_next_scan_large(). 5956 */ 5957 page_t * 5958 page_next_scan_init(void **cookie) 5959 { 5960 ASSERT(cookie != NULL); 5961 *cookie = (void *)memsegs; 5962 return ((page_t *)memsegs->pages); 5963 } 5964 5965 /* 5966 * Return the next page in a scan of page_t's, assuming we want 5967 * to skip over sub-pages within larger page sizes. 5968 * 5969 * The cookie is used to keep track of the current memseg. 5970 */ 5971 page_t * 5972 page_next_scan_large( 5973 page_t *pp, 5974 ulong_t *n, 5975 void **cookie) 5976 { 5977 struct memseg *seg = (struct memseg *)*cookie; 5978 page_t *new_pp; 5979 ulong_t cnt; 5980 pfn_t pfn; 5981 5982 5983 /* 5984 * get the count of page_t's to skip based on the page size 5985 */ 5986 ASSERT(pp != NULL); 5987 if (pp->p_szc == 0) { 5988 cnt = 1; 5989 } else { 5990 pfn = page_pptonum(pp); 5991 cnt = page_get_pagecnt(pp->p_szc); 5992 cnt -= pfn & (cnt - 1); 5993 } 5994 *n += cnt; 5995 new_pp = pp + cnt; 5996 5997 /* 5998 * Catch if we went past the end of the current memory segment. If so, 5999 * just move to the next segment with pages. 6000 */ 6001 if (new_pp >= seg->epages || seg->pages_base == seg->pages_end) { 6002 do { 6003 seg = seg->next; 6004 if (seg == NULL) 6005 seg = memsegs; 6006 } while (seg->pages_base == seg->pages_end); 6007 new_pp = seg->pages; 6008 *cookie = (void *)seg; 6009 } 6010 6011 return (new_pp); 6012 } 6013 6014 6015 /* 6016 * Returns next page in list. Note: this function wraps 6017 * to the first page in the list upon reaching the end 6018 * of the list. Callers should be aware of this fact. 6019 */ 6020 6021 /* We should change this be a #define */ 6022 6023 page_t * 6024 page_next(page_t *pp) 6025 { 6026 return (page_nextn(pp, 1)); 6027 } 6028 6029 page_t * 6030 page_first() 6031 { 6032 return ((page_t *)memsegs->pages); 6033 } 6034 6035 6036 /* 6037 * This routine is called at boot with the initial memory configuration 6038 * and when memory is added or removed. 6039 */ 6040 void 6041 build_pfn_hash() 6042 { 6043 pfn_t cur; 6044 pgcnt_t index; 6045 struct memseg *pseg; 6046 int i; 6047 6048 /* 6049 * Clear memseg_hash array. 6050 * Since memory add/delete is designed to operate concurrently 6051 * with normal operation, the hash rebuild must be able to run 6052 * concurrently with page_numtopp_nolock(). To support this 6053 * functionality, assignments to memseg_hash array members must 6054 * be done atomically. 6055 * 6056 * NOTE: bzero() does not currently guarantee this for kernel 6057 * threads, and cannot be used here. 6058 */ 6059 for (i = 0; i < N_MEM_SLOTS; i++) 6060 memseg_hash[i] = NULL; 6061 6062 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6063 6064 /* 6065 * Physmax is the last valid pfn. 6066 */ 6067 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6068 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6069 index = MEMSEG_PFN_HASH(pseg->pages_base); 6070 cur = pseg->pages_base; 6071 do { 6072 if (index >= N_MEM_SLOTS) 6073 index = MEMSEG_PFN_HASH(cur); 6074 6075 if (memseg_hash[index] == NULL || 6076 memseg_hash[index]->pages_base > pseg->pages_base) { 6077 memseg_hash[index] = pseg; 6078 hat_kpm_mseghash_update(index, pseg); 6079 } 6080 cur += mhash_per_slot; 6081 index++; 6082 } while (cur < pseg->pages_end); 6083 } 6084 } 6085 6086 /* 6087 * Return the pagenum for the pp 6088 */ 6089 pfn_t 6090 page_pptonum(page_t *pp) 6091 { 6092 return (pp->p_pagenum); 6093 } 6094 6095 /* 6096 * interface to the referenced and modified etc bits 6097 * in the PSM part of the page struct 6098 * when no locking is desired. 6099 */ 6100 void 6101 page_set_props(page_t *pp, uint_t flags) 6102 { 6103 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6104 pp->p_nrm |= (uchar_t)flags; 6105 } 6106 6107 void 6108 page_clr_all_props(page_t *pp) 6109 { 6110 pp->p_nrm = 0; 6111 } 6112 6113 /* 6114 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6115 */ 6116 int 6117 page_clear_lck_cow(page_t *pp, int adjust) 6118 { 6119 int f_amount; 6120 6121 ASSERT(PAGE_EXCL(pp)); 6122 6123 /* 6124 * The page_struct_lock need not be acquired here since 6125 * we require the caller hold the page exclusively locked. 6126 */ 6127 f_amount = 0; 6128 if (pp->p_lckcnt) { 6129 f_amount = 1; 6130 pp->p_lckcnt = 0; 6131 } 6132 if (pp->p_cowcnt) { 6133 f_amount += pp->p_cowcnt; 6134 pp->p_cowcnt = 0; 6135 } 6136 6137 if (adjust && f_amount) { 6138 mutex_enter(&freemem_lock); 6139 availrmem += f_amount; 6140 mutex_exit(&freemem_lock); 6141 } 6142 6143 return (f_amount); 6144 } 6145 6146 /* 6147 * The following functions is called from free_vp_pages() 6148 * for an inexact estimate of a newly free'd page... 6149 */ 6150 ulong_t 6151 page_share_cnt(page_t *pp) 6152 { 6153 return (hat_page_getshare(pp)); 6154 } 6155 6156 int 6157 page_isshared(page_t *pp) 6158 { 6159 return (hat_page_checkshare(pp, 1)); 6160 } 6161 6162 int 6163 page_isfree(page_t *pp) 6164 { 6165 return (PP_ISFREE(pp)); 6166 } 6167 6168 int 6169 page_isref(page_t *pp) 6170 { 6171 return (hat_page_getattr(pp, P_REF)); 6172 } 6173 6174 int 6175 page_ismod(page_t *pp) 6176 { 6177 return (hat_page_getattr(pp, P_MOD)); 6178 } 6179 6180 /* 6181 * The following code all currently relates to the page capture logic: 6182 * 6183 * This logic is used for cases where there is a desire to claim a certain 6184 * physical page in the system for the caller. As it may not be possible 6185 * to capture the page immediately, the p_toxic bits are used in the page 6186 * structure to indicate that someone wants to capture this page. When the 6187 * page gets unlocked, the toxic flag will be noted and an attempt to capture 6188 * the page will be made. If it is successful, the original callers callback 6189 * will be called with the page to do with it what they please. 6190 * 6191 * There is also an async thread which wakes up to attempt to capture 6192 * pages occasionally which have the capture bit set. All of the pages which 6193 * need to be captured asynchronously have been inserted into the 6194 * page_capture_hash and thus this thread walks that hash list. Items in the 6195 * hash have an expiration time so this thread handles that as well by removing 6196 * the item from the hash if it has expired. 6197 * 6198 * Some important things to note are: 6199 * - if the PR_CAPTURE bit is set on a page, then the page is in the 6200 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed 6201 * to set and clear this bit, and while the lock is held is the only time 6202 * you can add or remove an entry from the hash. 6203 * - the PR_CAPTURE bit can only be set and cleared while holding the 6204 * page_capture_hash_head.pchh_mutex 6205 * - the t_flag field of the thread struct is used with the T_CAPTURING 6206 * flag to prevent recursion while dealing with large pages. 6207 * - pages which need to be retired never expire on the page_capture_hash. 6208 */ 6209 6210 static void page_capture_thread(void); 6211 static kthread_t *pc_thread_id; 6212 kcondvar_t pc_cv; 6213 static kmutex_t pc_thread_mutex; 6214 static clock_t pc_thread_shortwait; 6215 static clock_t pc_thread_longwait; 6216 static int pc_thread_retry; 6217 6218 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS]; 6219 6220 /* Note that this is a circular linked list */ 6221 typedef struct page_capture_hash_bucket { 6222 page_t *pp; 6223 uchar_t szc; 6224 uchar_t pri; 6225 uint_t flags; 6226 clock_t expires; /* lbolt at which this request expires. */ 6227 void *datap; /* Cached data passed in for callback */ 6228 struct page_capture_hash_bucket *next; 6229 struct page_capture_hash_bucket *prev; 6230 } page_capture_hash_bucket_t; 6231 6232 #define PC_PRI_HI 0 /* capture now */ 6233 #define PC_PRI_LO 1 /* capture later */ 6234 #define PC_NUM_PRI 2 6235 6236 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI) 6237 6238 6239 /* 6240 * Each hash bucket will have it's own mutex and two lists which are: 6241 * active (0): represents requests which have not been processed by 6242 * the page_capture async thread yet. 6243 * walked (1): represents requests which have been processed by the 6244 * page_capture async thread within it's given walk of this bucket. 6245 * 6246 * These are all needed so that we can synchronize all async page_capture 6247 * events. When the async thread moves to a new bucket, it will append the 6248 * walked list to the active list and walk each item one at a time, moving it 6249 * from the active list to the walked list. Thus if there is an async request 6250 * outstanding for a given page, it will always be in one of the two lists. 6251 * New requests will always be added to the active list. 6252 * If we were not able to capture a page before the request expired, we'd free 6253 * up the request structure which would indicate to page_capture that there is 6254 * no longer a need for the given page, and clear the PR_CAPTURE flag if 6255 * possible. 6256 */ 6257 typedef struct page_capture_hash_head { 6258 kmutex_t pchh_mutex; 6259 uint_t num_pages[PC_NUM_PRI]; 6260 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */ 6261 } page_capture_hash_head_t; 6262 6263 #ifdef DEBUG 6264 #define NUM_PAGE_CAPTURE_BUCKETS 4 6265 #else 6266 #define NUM_PAGE_CAPTURE_BUCKETS 64 6267 #endif 6268 6269 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS]; 6270 6271 /* for now use a very simple hash based upon the size of a page struct */ 6272 #define PAGE_CAPTURE_HASH(pp) \ 6273 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1))) 6274 6275 extern pgcnt_t swapfs_minfree; 6276 6277 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap); 6278 6279 /* 6280 * a callback function is required for page capture requests. 6281 */ 6282 void 6283 page_capture_register_callback(uint_t index, clock_t duration, 6284 int (*cb_func)(page_t *, void *, uint_t)) 6285 { 6286 ASSERT(pc_cb[index].cb_active == 0); 6287 ASSERT(cb_func != NULL); 6288 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6289 pc_cb[index].duration = duration; 6290 pc_cb[index].cb_func = cb_func; 6291 pc_cb[index].cb_active = 1; 6292 rw_exit(&pc_cb[index].cb_rwlock); 6293 } 6294 6295 void 6296 page_capture_unregister_callback(uint_t index) 6297 { 6298 int i, j; 6299 struct page_capture_hash_bucket *bp1; 6300 struct page_capture_hash_bucket *bp2; 6301 struct page_capture_hash_bucket *head = NULL; 6302 uint_t flags = (1 << index); 6303 6304 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6305 ASSERT(pc_cb[index].cb_active == 1); 6306 pc_cb[index].duration = 0; /* Paranoia */ 6307 pc_cb[index].cb_func = NULL; /* Paranoia */ 6308 pc_cb[index].cb_active = 0; 6309 rw_exit(&pc_cb[index].cb_rwlock); 6310 6311 /* 6312 * Just move all the entries to a private list which we can walk 6313 * through without the need to hold any locks. 6314 * No more requests can get added to the hash lists for this consumer 6315 * as the cb_active field for the callback has been cleared. 6316 */ 6317 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 6318 mutex_enter(&page_capture_hash[i].pchh_mutex); 6319 for (j = 0; j < 2; j++) { 6320 bp1 = page_capture_hash[i].lists[j].next; 6321 /* walk through all but first (sentinel) element */ 6322 while (bp1 != &page_capture_hash[i].lists[j]) { 6323 bp2 = bp1; 6324 if (bp2->flags & flags) { 6325 bp1 = bp2->next; 6326 bp1->prev = bp2->prev; 6327 bp2->prev->next = bp1; 6328 bp2->next = head; 6329 head = bp2; 6330 /* 6331 * Clear the PR_CAPTURE bit as we 6332 * hold appropriate locks here. 6333 */ 6334 page_clrtoxic(head->pp, PR_CAPTURE); 6335 page_capture_hash[i]. 6336 num_pages[bp2->pri]--; 6337 continue; 6338 } 6339 bp1 = bp1->next; 6340 } 6341 } 6342 mutex_exit(&page_capture_hash[i].pchh_mutex); 6343 } 6344 6345 while (head != NULL) { 6346 bp1 = head; 6347 head = head->next; 6348 kmem_free(bp1, sizeof (*bp1)); 6349 } 6350 } 6351 6352 6353 /* 6354 * Find pp in the active list and move it to the walked list if it 6355 * exists. 6356 * Note that most often pp should be at the front of the active list 6357 * as it is currently used and thus there is no other sort of optimization 6358 * being done here as this is a linked list data structure. 6359 * Returns 1 on successful move or 0 if page could not be found. 6360 */ 6361 static int 6362 page_capture_move_to_walked(page_t *pp) 6363 { 6364 page_capture_hash_bucket_t *bp; 6365 int index; 6366 6367 index = PAGE_CAPTURE_HASH(pp); 6368 6369 mutex_enter(&page_capture_hash[index].pchh_mutex); 6370 bp = page_capture_hash[index].lists[0].next; 6371 while (bp != &page_capture_hash[index].lists[0]) { 6372 if (bp->pp == pp) { 6373 /* Remove from old list */ 6374 bp->next->prev = bp->prev; 6375 bp->prev->next = bp->next; 6376 6377 /* Add to new list */ 6378 bp->next = page_capture_hash[index].lists[1].next; 6379 bp->prev = &page_capture_hash[index].lists[1]; 6380 page_capture_hash[index].lists[1].next = bp; 6381 bp->next->prev = bp; 6382 6383 /* 6384 * There is a small probability of page on a free 6385 * list being retired while being allocated 6386 * and before P_RAF is set on it. The page may 6387 * end up marked as high priority request instead 6388 * of low priority request. 6389 * If P_RAF page is not marked as low priority request 6390 * change it to low priority request. 6391 */ 6392 page_capture_hash[index].num_pages[bp->pri]--; 6393 bp->pri = PAGE_CAPTURE_PRIO(pp); 6394 page_capture_hash[index].num_pages[bp->pri]++; 6395 mutex_exit(&page_capture_hash[index].pchh_mutex); 6396 return (1); 6397 } 6398 bp = bp->next; 6399 } 6400 mutex_exit(&page_capture_hash[index].pchh_mutex); 6401 return (0); 6402 } 6403 6404 /* 6405 * Add a new entry to the page capture hash. The only case where a new 6406 * entry is not added is when the page capture consumer is no longer registered. 6407 * In this case, we'll silently not add the page to the hash. We know that 6408 * page retire will always be registered for the case where we are currently 6409 * unretiring a page and thus there are no conflicts. 6410 */ 6411 static void 6412 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) 6413 { 6414 page_capture_hash_bucket_t *bp1; 6415 page_capture_hash_bucket_t *bp2; 6416 int index; 6417 int cb_index; 6418 int i; 6419 uchar_t pri; 6420 #ifdef DEBUG 6421 page_capture_hash_bucket_t *tp1; 6422 int l; 6423 #endif 6424 6425 ASSERT(!(flags & CAPTURE_ASYNC)); 6426 6427 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP); 6428 6429 bp1->pp = pp; 6430 bp1->szc = szc; 6431 bp1->flags = flags; 6432 bp1->datap = datap; 6433 6434 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6435 if ((flags >> cb_index) & 1) { 6436 break; 6437 } 6438 } 6439 6440 ASSERT(cb_index != PC_NUM_CALLBACKS); 6441 6442 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6443 if (pc_cb[cb_index].cb_active) { 6444 if (pc_cb[cb_index].duration == -1) { 6445 bp1->expires = (clock_t)-1; 6446 } else { 6447 bp1->expires = ddi_get_lbolt() + 6448 pc_cb[cb_index].duration; 6449 } 6450 } else { 6451 /* There's no callback registered so don't add to the hash */ 6452 rw_exit(&pc_cb[cb_index].cb_rwlock); 6453 kmem_free(bp1, sizeof (*bp1)); 6454 return; 6455 } 6456 6457 index = PAGE_CAPTURE_HASH(pp); 6458 6459 /* 6460 * Only allow capture flag to be modified under this mutex. 6461 * Prevents multiple entries for same page getting added. 6462 */ 6463 mutex_enter(&page_capture_hash[index].pchh_mutex); 6464 6465 /* 6466 * if not already on the hash, set capture bit and add to the hash 6467 */ 6468 if (!(pp->p_toxic & PR_CAPTURE)) { 6469 #ifdef DEBUG 6470 /* Check for duplicate entries */ 6471 for (l = 0; l < 2; l++) { 6472 tp1 = page_capture_hash[index].lists[l].next; 6473 while (tp1 != &page_capture_hash[index].lists[l]) { 6474 if (tp1->pp == pp) { 6475 panic("page pp 0x%p already on hash " 6476 "at 0x%p\n", 6477 (void *)pp, (void *)tp1); 6478 } 6479 tp1 = tp1->next; 6480 } 6481 } 6482 6483 #endif 6484 page_settoxic(pp, PR_CAPTURE); 6485 pri = PAGE_CAPTURE_PRIO(pp); 6486 bp1->pri = pri; 6487 bp1->next = page_capture_hash[index].lists[0].next; 6488 bp1->prev = &page_capture_hash[index].lists[0]; 6489 bp1->next->prev = bp1; 6490 page_capture_hash[index].lists[0].next = bp1; 6491 page_capture_hash[index].num_pages[pri]++; 6492 if (flags & CAPTURE_RETIRE) { 6493 page_retire_incr_pend_count(datap); 6494 } 6495 mutex_exit(&page_capture_hash[index].pchh_mutex); 6496 rw_exit(&pc_cb[cb_index].cb_rwlock); 6497 cv_signal(&pc_cv); 6498 return; 6499 } 6500 6501 /* 6502 * A page retire request will replace any other request. 6503 * A second physmem request which is for a different process than 6504 * the currently registered one will be dropped as there is 6505 * no way to hold the private data for both calls. 6506 * In the future, once there are more callers, this will have to 6507 * be worked out better as there needs to be private storage for 6508 * at least each type of caller (maybe have datap be an array of 6509 * *void's so that we can index based upon callers index). 6510 */ 6511 6512 /* walk hash list to update expire time */ 6513 for (i = 0; i < 2; i++) { 6514 bp2 = page_capture_hash[index].lists[i].next; 6515 while (bp2 != &page_capture_hash[index].lists[i]) { 6516 if (bp2->pp == pp) { 6517 if (flags & CAPTURE_RETIRE) { 6518 if (!(bp2->flags & CAPTURE_RETIRE)) { 6519 page_retire_incr_pend_count( 6520 datap); 6521 bp2->flags = flags; 6522 bp2->expires = bp1->expires; 6523 bp2->datap = datap; 6524 } 6525 } else { 6526 ASSERT(flags & CAPTURE_PHYSMEM); 6527 if (!(bp2->flags & CAPTURE_RETIRE) && 6528 (datap == bp2->datap)) { 6529 bp2->expires = bp1->expires; 6530 } 6531 } 6532 mutex_exit(&page_capture_hash[index]. 6533 pchh_mutex); 6534 rw_exit(&pc_cb[cb_index].cb_rwlock); 6535 kmem_free(bp1, sizeof (*bp1)); 6536 return; 6537 } 6538 bp2 = bp2->next; 6539 } 6540 } 6541 6542 /* 6543 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes 6544 * and thus it either has to be set or not set and can't change 6545 * while holding the mutex above. 6546 */ 6547 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", 6548 (void *)pp); 6549 } 6550 6551 /* 6552 * We have a page in our hands, lets try and make it ours by turning 6553 * it into a clean page like it had just come off the freelists. 6554 * 6555 * Returns 0 on success, with the page still EXCL locked. 6556 * On failure, the page will be unlocked, and returns EAGAIN 6557 */ 6558 static int 6559 page_capture_clean_page(page_t *pp) 6560 { 6561 page_t *newpp; 6562 int skip_unlock = 0; 6563 spgcnt_t count; 6564 page_t *tpp; 6565 int ret = 0; 6566 int extra; 6567 6568 ASSERT(PAGE_EXCL(pp)); 6569 ASSERT(!PP_RETIRED(pp)); 6570 ASSERT(curthread->t_flag & T_CAPTURING); 6571 6572 if (PP_ISFREE(pp)) { 6573 if (!page_reclaim(pp, NULL)) { 6574 skip_unlock = 1; 6575 ret = EAGAIN; 6576 goto cleanup; 6577 } 6578 ASSERT(pp->p_szc == 0); 6579 if (pp->p_vnode != NULL) { 6580 /* 6581 * Since this page came from the 6582 * cachelist, we must destroy the 6583 * old vnode association. 6584 */ 6585 page_hashout(pp, NULL); 6586 } 6587 goto cleanup; 6588 } 6589 6590 /* 6591 * If we know page_relocate will fail, skip it 6592 * It could still fail due to a UE on another page but we 6593 * can't do anything about that. 6594 */ 6595 if (pp->p_toxic & PR_UE) { 6596 goto skip_relocate; 6597 } 6598 6599 /* 6600 * It's possible that pages can not have a vnode as fsflush comes 6601 * through and cleans up these pages. It's ugly but that's how it is. 6602 */ 6603 if (pp->p_vnode == NULL) { 6604 goto skip_relocate; 6605 } 6606 6607 /* 6608 * Page was not free, so lets try to relocate it. 6609 * page_relocate only works with root pages, so if this is not a root 6610 * page, we need to demote it to try and relocate it. 6611 * Unfortunately this is the best we can do right now. 6612 */ 6613 newpp = NULL; 6614 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { 6615 if (page_try_demote_pages(pp) == 0) { 6616 ret = EAGAIN; 6617 goto cleanup; 6618 } 6619 } 6620 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); 6621 if (ret == 0) { 6622 page_t *npp; 6623 /* unlock the new page(s) */ 6624 while (count-- > 0) { 6625 ASSERT(newpp != NULL); 6626 npp = newpp; 6627 page_sub(&newpp, npp); 6628 page_unlock(npp); 6629 } 6630 ASSERT(newpp == NULL); 6631 /* 6632 * Check to see if the page we have is too large. 6633 * If so, demote it freeing up the extra pages. 6634 */ 6635 if (pp->p_szc > 0) { 6636 /* For now demote extra pages to szc == 0 */ 6637 extra = page_get_pagecnt(pp->p_szc) - 1; 6638 while (extra > 0) { 6639 tpp = pp->p_next; 6640 page_sub(&pp, tpp); 6641 tpp->p_szc = 0; 6642 page_free(tpp, 1); 6643 extra--; 6644 } 6645 /* Make sure to set our page to szc 0 as well */ 6646 ASSERT(pp->p_next == pp && pp->p_prev == pp); 6647 pp->p_szc = 0; 6648 } 6649 goto cleanup; 6650 } else if (ret == EIO) { 6651 ret = EAGAIN; 6652 goto cleanup; 6653 } else { 6654 /* 6655 * Need to reset return type as we failed to relocate the page 6656 * but that does not mean that some of the next steps will not 6657 * work. 6658 */ 6659 ret = 0; 6660 } 6661 6662 skip_relocate: 6663 6664 if (pp->p_szc > 0) { 6665 if (page_try_demote_pages(pp) == 0) { 6666 ret = EAGAIN; 6667 goto cleanup; 6668 } 6669 } 6670 6671 ASSERT(pp->p_szc == 0); 6672 6673 if (hat_ismod(pp)) { 6674 ret = EAGAIN; 6675 goto cleanup; 6676 } 6677 if (PP_ISKAS(pp)) { 6678 ret = EAGAIN; 6679 goto cleanup; 6680 } 6681 if (pp->p_lckcnt || pp->p_cowcnt) { 6682 ret = EAGAIN; 6683 goto cleanup; 6684 } 6685 6686 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 6687 ASSERT(!hat_page_is_mapped(pp)); 6688 6689 if (hat_ismod(pp)) { 6690 /* 6691 * This is a semi-odd case as the page is now modified but not 6692 * mapped as we just unloaded the mappings above. 6693 */ 6694 ret = EAGAIN; 6695 goto cleanup; 6696 } 6697 if (pp->p_vnode != NULL) { 6698 page_hashout(pp, NULL); 6699 } 6700 6701 /* 6702 * At this point, the page should be in a clean state and 6703 * we can do whatever we want with it. 6704 */ 6705 6706 cleanup: 6707 if (ret != 0) { 6708 if (!skip_unlock) { 6709 page_unlock(pp); 6710 } 6711 } else { 6712 ASSERT(pp->p_szc == 0); 6713 ASSERT(PAGE_EXCL(pp)); 6714 6715 pp->p_next = pp; 6716 pp->p_prev = pp; 6717 } 6718 return (ret); 6719 } 6720 6721 /* 6722 * Various callers of page_trycapture() can have different restrictions upon 6723 * what memory they have access to. 6724 * Returns 0 on success, with the following error codes on failure: 6725 * EPERM - The requested page is long term locked, and thus repeated 6726 * requests to capture this page will likely fail. 6727 * ENOMEM - There was not enough free memory in the system to safely 6728 * map the requested page. 6729 * ENOENT - The requested page was inside the kernel cage, and the 6730 * PHYSMEM_CAGE flag was not set. 6731 */ 6732 int 6733 page_capture_pre_checks(page_t *pp, uint_t flags) 6734 { 6735 ASSERT(pp != NULL); 6736 6737 #if defined(__sparc) 6738 if (pp->p_vnode == &promvp) { 6739 return (EPERM); 6740 } 6741 6742 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) && 6743 (flags & CAPTURE_PHYSMEM)) { 6744 return (ENOENT); 6745 } 6746 6747 if (PP_ISNORELOCKERNEL(pp)) { 6748 return (EPERM); 6749 } 6750 #else 6751 if (PP_ISKAS(pp)) { 6752 return (EPERM); 6753 } 6754 #endif /* __sparc */ 6755 6756 /* only physmem currently has the restrictions checked below */ 6757 if (!(flags & CAPTURE_PHYSMEM)) { 6758 return (0); 6759 } 6760 6761 if (availrmem < swapfs_minfree) { 6762 /* 6763 * We won't try to capture this page as we are 6764 * running low on memory. 6765 */ 6766 return (ENOMEM); 6767 } 6768 return (0); 6769 } 6770 6771 /* 6772 * Once we have a page in our mits, go ahead and complete the capture 6773 * operation. 6774 * Returns 1 on failure where page is no longer needed 6775 * Returns 0 on success 6776 * Returns -1 if there was a transient failure. 6777 * Failure cases must release the SE_EXCL lock on pp (usually via page_free). 6778 */ 6779 int 6780 page_capture_take_action(page_t *pp, uint_t flags, void *datap) 6781 { 6782 int cb_index; 6783 int ret = 0; 6784 page_capture_hash_bucket_t *bp1; 6785 page_capture_hash_bucket_t *bp2; 6786 int index; 6787 int found = 0; 6788 int i; 6789 6790 ASSERT(PAGE_EXCL(pp)); 6791 ASSERT(curthread->t_flag & T_CAPTURING); 6792 6793 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6794 if ((flags >> cb_index) & 1) { 6795 break; 6796 } 6797 } 6798 ASSERT(cb_index < PC_NUM_CALLBACKS); 6799 6800 /* 6801 * Remove the entry from the page_capture hash, but don't free it yet 6802 * as we may need to put it back. 6803 * Since we own the page at this point in time, we should find it 6804 * in the hash if this is an ASYNC call. If we don't it's likely 6805 * that the page_capture_async() thread decided that this request 6806 * had expired, in which case we just continue on. 6807 */ 6808 if (flags & CAPTURE_ASYNC) { 6809 6810 index = PAGE_CAPTURE_HASH(pp); 6811 6812 mutex_enter(&page_capture_hash[index].pchh_mutex); 6813 for (i = 0; i < 2 && !found; i++) { 6814 bp1 = page_capture_hash[index].lists[i].next; 6815 while (bp1 != &page_capture_hash[index].lists[i]) { 6816 if (bp1->pp == pp) { 6817 bp1->next->prev = bp1->prev; 6818 bp1->prev->next = bp1->next; 6819 page_capture_hash[index]. 6820 num_pages[bp1->pri]--; 6821 page_clrtoxic(pp, PR_CAPTURE); 6822 found = 1; 6823 break; 6824 } 6825 bp1 = bp1->next; 6826 } 6827 } 6828 mutex_exit(&page_capture_hash[index].pchh_mutex); 6829 } 6830 6831 /* Synchronize with the unregister func. */ 6832 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6833 if (!pc_cb[cb_index].cb_active) { 6834 page_free(pp, 1); 6835 rw_exit(&pc_cb[cb_index].cb_rwlock); 6836 if (found) { 6837 kmem_free(bp1, sizeof (*bp1)); 6838 } 6839 return (1); 6840 } 6841 6842 /* 6843 * We need to remove the entry from the page capture hash and turn off 6844 * the PR_CAPTURE bit before calling the callback. We'll need to cache 6845 * the entry here, and then based upon the return value, cleanup 6846 * appropriately or re-add it to the hash, making sure that someone else 6847 * hasn't already done so. 6848 * It should be rare for the callback to fail and thus it's ok for 6849 * the failure path to be a bit complicated as the success path is 6850 * cleaner and the locking rules are easier to follow. 6851 */ 6852 6853 ret = pc_cb[cb_index].cb_func(pp, datap, flags); 6854 6855 rw_exit(&pc_cb[cb_index].cb_rwlock); 6856 6857 /* 6858 * If this was an ASYNC request, we need to cleanup the hash if the 6859 * callback was successful or if the request was no longer valid. 6860 * For non-ASYNC requests, we return failure to map and the caller 6861 * will take care of adding the request to the hash. 6862 * Note also that the callback itself is responsible for the page 6863 * at this point in time in terms of locking ... The most common 6864 * case for the failure path should just be a page_free. 6865 */ 6866 if (ret >= 0) { 6867 if (found) { 6868 if (bp1->flags & CAPTURE_RETIRE) { 6869 page_retire_decr_pend_count(datap); 6870 } 6871 kmem_free(bp1, sizeof (*bp1)); 6872 } 6873 return (ret); 6874 } 6875 if (!found) { 6876 return (ret); 6877 } 6878 6879 ASSERT(flags & CAPTURE_ASYNC); 6880 6881 /* 6882 * Check for expiration time first as we can just free it up if it's 6883 * expired. 6884 */ 6885 if (ddi_get_lbolt() > bp1->expires && bp1->expires != -1) { 6886 kmem_free(bp1, sizeof (*bp1)); 6887 return (ret); 6888 } 6889 6890 /* 6891 * The callback failed and there used to be an entry in the hash for 6892 * this page, so we need to add it back to the hash. 6893 */ 6894 mutex_enter(&page_capture_hash[index].pchh_mutex); 6895 if (!(pp->p_toxic & PR_CAPTURE)) { 6896 /* just add bp1 back to head of walked list */ 6897 page_settoxic(pp, PR_CAPTURE); 6898 bp1->next = page_capture_hash[index].lists[1].next; 6899 bp1->prev = &page_capture_hash[index].lists[1]; 6900 bp1->next->prev = bp1; 6901 bp1->pri = PAGE_CAPTURE_PRIO(pp); 6902 page_capture_hash[index].lists[1].next = bp1; 6903 page_capture_hash[index].num_pages[bp1->pri]++; 6904 mutex_exit(&page_capture_hash[index].pchh_mutex); 6905 return (ret); 6906 } 6907 6908 /* 6909 * Otherwise there was a new capture request added to list 6910 * Need to make sure that our original data is represented if 6911 * appropriate. 6912 */ 6913 for (i = 0; i < 2; i++) { 6914 bp2 = page_capture_hash[index].lists[i].next; 6915 while (bp2 != &page_capture_hash[index].lists[i]) { 6916 if (bp2->pp == pp) { 6917 if (bp1->flags & CAPTURE_RETIRE) { 6918 if (!(bp2->flags & CAPTURE_RETIRE)) { 6919 bp2->szc = bp1->szc; 6920 bp2->flags = bp1->flags; 6921 bp2->expires = bp1->expires; 6922 bp2->datap = bp1->datap; 6923 } 6924 } else { 6925 ASSERT(bp1->flags & CAPTURE_PHYSMEM); 6926 if (!(bp2->flags & CAPTURE_RETIRE)) { 6927 bp2->szc = bp1->szc; 6928 bp2->flags = bp1->flags; 6929 bp2->expires = bp1->expires; 6930 bp2->datap = bp1->datap; 6931 } 6932 } 6933 page_capture_hash[index].num_pages[bp2->pri]--; 6934 bp2->pri = PAGE_CAPTURE_PRIO(pp); 6935 page_capture_hash[index].num_pages[bp2->pri]++; 6936 mutex_exit(&page_capture_hash[index]. 6937 pchh_mutex); 6938 kmem_free(bp1, sizeof (*bp1)); 6939 return (ret); 6940 } 6941 bp2 = bp2->next; 6942 } 6943 } 6944 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp); 6945 /*NOTREACHED*/ 6946 } 6947 6948 /* 6949 * Try to capture the given page for the caller specified in the flags 6950 * parameter. The page will either be captured and handed over to the 6951 * appropriate callback, or will be queued up in the page capture hash 6952 * to be captured asynchronously. 6953 * If the current request is due to an async capture, the page must be 6954 * exclusively locked before calling this function. 6955 * Currently szc must be 0 but in the future this should be expandable to 6956 * other page sizes. 6957 * Returns 0 on success, with the following error codes on failure: 6958 * EPERM - The requested page is long term locked, and thus repeated 6959 * requests to capture this page will likely fail. 6960 * ENOMEM - There was not enough free memory in the system to safely 6961 * map the requested page. 6962 * ENOENT - The requested page was inside the kernel cage, and the 6963 * CAPTURE_GET_CAGE flag was not set. 6964 * EAGAIN - The requested page could not be capturead at this point in 6965 * time but future requests will likely work. 6966 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag 6967 * was not set. 6968 */ 6969 int 6970 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 6971 { 6972 int ret; 6973 int cb_index; 6974 6975 if (flags & CAPTURE_ASYNC) { 6976 ASSERT(PAGE_EXCL(pp)); 6977 goto async; 6978 } 6979 6980 /* Make sure there's enough availrmem ... */ 6981 ret = page_capture_pre_checks(pp, flags); 6982 if (ret != 0) { 6983 return (ret); 6984 } 6985 6986 if (!page_trylock(pp, SE_EXCL)) { 6987 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6988 if ((flags >> cb_index) & 1) { 6989 break; 6990 } 6991 } 6992 ASSERT(cb_index < PC_NUM_CALLBACKS); 6993 ret = EAGAIN; 6994 /* Special case for retired pages */ 6995 if (PP_RETIRED(pp)) { 6996 if (flags & CAPTURE_GET_RETIRED) { 6997 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { 6998 /* 6999 * Need to set capture bit and add to 7000 * hash so that the page will be 7001 * retired when freed. 7002 */ 7003 page_capture_add_hash(pp, szc, 7004 CAPTURE_RETIRE, NULL); 7005 ret = 0; 7006 goto own_page; 7007 } 7008 } else { 7009 return (EBUSY); 7010 } 7011 } 7012 page_capture_add_hash(pp, szc, flags, datap); 7013 return (ret); 7014 } 7015 7016 async: 7017 ASSERT(PAGE_EXCL(pp)); 7018 7019 /* Need to check for physmem async requests that availrmem is sane */ 7020 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) == 7021 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) && 7022 (availrmem < swapfs_minfree)) { 7023 page_unlock(pp); 7024 return (ENOMEM); 7025 } 7026 7027 ret = page_capture_clean_page(pp); 7028 7029 if (ret != 0) { 7030 /* We failed to get the page, so lets add it to the hash */ 7031 if (!(flags & CAPTURE_ASYNC)) { 7032 page_capture_add_hash(pp, szc, flags, datap); 7033 } 7034 return (ret); 7035 } 7036 7037 own_page: 7038 ASSERT(PAGE_EXCL(pp)); 7039 ASSERT(pp->p_szc == 0); 7040 7041 /* Call the callback */ 7042 ret = page_capture_take_action(pp, flags, datap); 7043 7044 if (ret == 0) { 7045 return (0); 7046 } 7047 7048 /* 7049 * Note that in the failure cases from page_capture_take_action, the 7050 * EXCL lock will have already been dropped. 7051 */ 7052 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) { 7053 page_capture_add_hash(pp, szc, flags, datap); 7054 } 7055 return (EAGAIN); 7056 } 7057 7058 int 7059 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 7060 { 7061 int ret; 7062 7063 curthread->t_flag |= T_CAPTURING; 7064 ret = page_itrycapture(pp, szc, flags, datap); 7065 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */ 7066 return (ret); 7067 } 7068 7069 /* 7070 * When unlocking a page which has the PR_CAPTURE bit set, this routine 7071 * gets called to try and capture the page. 7072 */ 7073 void 7074 page_unlock_capture(page_t *pp) 7075 { 7076 page_capture_hash_bucket_t *bp; 7077 int index; 7078 int i; 7079 uint_t szc; 7080 uint_t flags = 0; 7081 void *datap; 7082 kmutex_t *mp; 7083 extern vnode_t retired_pages; 7084 7085 /* 7086 * We need to protect against a possible deadlock here where we own 7087 * the vnode page hash mutex and want to acquire it again as there 7088 * are locations in the code, where we unlock a page while holding 7089 * the mutex which can lead to the page being captured and eventually 7090 * end up here. As we may be hashing out the old page and hashing into 7091 * the retire vnode, we need to make sure we don't own them. 7092 * Other callbacks who do hash operations also need to make sure that 7093 * before they hashin to a vnode that they do not currently own the 7094 * vphm mutex otherwise there will be a panic. 7095 */ 7096 if (mutex_owned(page_vnode_mutex(&retired_pages))) { 7097 page_unlock_nocapture(pp); 7098 return; 7099 } 7100 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { 7101 page_unlock_nocapture(pp); 7102 return; 7103 } 7104 7105 index = PAGE_CAPTURE_HASH(pp); 7106 7107 mp = &page_capture_hash[index].pchh_mutex; 7108 mutex_enter(mp); 7109 for (i = 0; i < 2; i++) { 7110 bp = page_capture_hash[index].lists[i].next; 7111 while (bp != &page_capture_hash[index].lists[i]) { 7112 if (bp->pp == pp) { 7113 szc = bp->szc; 7114 flags = bp->flags | CAPTURE_ASYNC; 7115 datap = bp->datap; 7116 mutex_exit(mp); 7117 (void) page_trycapture(pp, szc, flags, datap); 7118 return; 7119 } 7120 bp = bp->next; 7121 } 7122 } 7123 7124 /* Failed to find page in hash so clear flags and unlock it. */ 7125 page_clrtoxic(pp, PR_CAPTURE); 7126 page_unlock(pp); 7127 7128 mutex_exit(mp); 7129 } 7130 7131 void 7132 page_capture_init() 7133 { 7134 int i; 7135 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7136 page_capture_hash[i].lists[0].next = 7137 &page_capture_hash[i].lists[0]; 7138 page_capture_hash[i].lists[0].prev = 7139 &page_capture_hash[i].lists[0]; 7140 page_capture_hash[i].lists[1].next = 7141 &page_capture_hash[i].lists[1]; 7142 page_capture_hash[i].lists[1].prev = 7143 &page_capture_hash[i].lists[1]; 7144 } 7145 7146 pc_thread_shortwait = 23 * hz; 7147 pc_thread_longwait = 1201 * hz; 7148 pc_thread_retry = 3; 7149 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL); 7150 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL); 7151 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0, 7152 TS_RUN, minclsyspri); 7153 } 7154 7155 /* 7156 * It is necessary to scrub any failing pages prior to reboot in order to 7157 * prevent a latent error trap from occurring on the next boot. 7158 */ 7159 void 7160 page_retire_mdboot() 7161 { 7162 page_t *pp; 7163 int i, j; 7164 page_capture_hash_bucket_t *bp; 7165 uchar_t pri; 7166 7167 /* walk lists looking for pages to scrub */ 7168 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7169 for (pri = 0; pri < PC_NUM_PRI; pri++) { 7170 if (page_capture_hash[i].num_pages[pri] != 0) { 7171 break; 7172 } 7173 } 7174 if (pri == PC_NUM_PRI) 7175 continue; 7176 7177 mutex_enter(&page_capture_hash[i].pchh_mutex); 7178 7179 for (j = 0; j < 2; j++) { 7180 bp = page_capture_hash[i].lists[j].next; 7181 while (bp != &page_capture_hash[i].lists[j]) { 7182 pp = bp->pp; 7183 if (PP_TOXIC(pp)) { 7184 if (page_trylock(pp, SE_EXCL)) { 7185 PP_CLRFREE(pp); 7186 pagescrub(pp, 0, PAGESIZE); 7187 page_unlock(pp); 7188 } 7189 } 7190 bp = bp->next; 7191 } 7192 } 7193 mutex_exit(&page_capture_hash[i].pchh_mutex); 7194 } 7195 } 7196 7197 /* 7198 * Walk the page_capture_hash trying to capture pages and also cleanup old 7199 * entries which have expired. 7200 */ 7201 void 7202 page_capture_async() 7203 { 7204 page_t *pp; 7205 int i; 7206 int ret; 7207 page_capture_hash_bucket_t *bp1, *bp2; 7208 uint_t szc; 7209 uint_t flags; 7210 void *datap; 7211 uchar_t pri; 7212 7213 /* If there are outstanding pages to be captured, get to work */ 7214 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7215 for (pri = 0; pri < PC_NUM_PRI; pri++) { 7216 if (page_capture_hash[i].num_pages[pri] != 0) 7217 break; 7218 } 7219 if (pri == PC_NUM_PRI) 7220 continue; 7221 7222 /* Append list 1 to list 0 and then walk through list 0 */ 7223 mutex_enter(&page_capture_hash[i].pchh_mutex); 7224 bp1 = &page_capture_hash[i].lists[1]; 7225 bp2 = bp1->next; 7226 if (bp1 != bp2) { 7227 bp1->prev->next = page_capture_hash[i].lists[0].next; 7228 bp2->prev = &page_capture_hash[i].lists[0]; 7229 page_capture_hash[i].lists[0].next->prev = bp1->prev; 7230 page_capture_hash[i].lists[0].next = bp2; 7231 bp1->next = bp1; 7232 bp1->prev = bp1; 7233 } 7234 7235 /* list[1] will be empty now */ 7236 7237 bp1 = page_capture_hash[i].lists[0].next; 7238 while (bp1 != &page_capture_hash[i].lists[0]) { 7239 /* Check expiration time */ 7240 if ((ddi_get_lbolt() > bp1->expires && 7241 bp1->expires != -1) || 7242 page_deleted(bp1->pp)) { 7243 page_capture_hash[i].lists[0].next = bp1->next; 7244 bp1->next->prev = 7245 &page_capture_hash[i].lists[0]; 7246 page_capture_hash[i].num_pages[bp1->pri]--; 7247 7248 /* 7249 * We can safely remove the PR_CAPTURE bit 7250 * without holding the EXCL lock on the page 7251 * as the PR_CAPTURE bit requres that the 7252 * page_capture_hash[].pchh_mutex be held 7253 * to modify it. 7254 */ 7255 page_clrtoxic(bp1->pp, PR_CAPTURE); 7256 mutex_exit(&page_capture_hash[i].pchh_mutex); 7257 kmem_free(bp1, sizeof (*bp1)); 7258 mutex_enter(&page_capture_hash[i].pchh_mutex); 7259 bp1 = page_capture_hash[i].lists[0].next; 7260 continue; 7261 } 7262 pp = bp1->pp; 7263 szc = bp1->szc; 7264 flags = bp1->flags; 7265 datap = bp1->datap; 7266 mutex_exit(&page_capture_hash[i].pchh_mutex); 7267 if (page_trylock(pp, SE_EXCL)) { 7268 ret = page_trycapture(pp, szc, 7269 flags | CAPTURE_ASYNC, datap); 7270 } else { 7271 ret = 1; /* move to walked hash */ 7272 } 7273 7274 if (ret != 0) { 7275 /* Move to walked hash */ 7276 (void) page_capture_move_to_walked(pp); 7277 } 7278 mutex_enter(&page_capture_hash[i].pchh_mutex); 7279 bp1 = page_capture_hash[i].lists[0].next; 7280 } 7281 7282 mutex_exit(&page_capture_hash[i].pchh_mutex); 7283 } 7284 } 7285 7286 /* 7287 * This function is called by the page_capture_thread, and is needed in 7288 * in order to initiate aio cleanup, so that pages used in aio 7289 * will be unlocked and subsequently retired by page_capture_thread. 7290 */ 7291 static int 7292 do_aio_cleanup(void) 7293 { 7294 proc_t *procp; 7295 int (*aio_cleanup_dr_delete_memory)(proc_t *); 7296 int cleaned = 0; 7297 7298 if (modload("sys", "kaio") == -1) { 7299 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio"); 7300 return (0); 7301 } 7302 /* 7303 * We use the aio_cleanup_dr_delete_memory function to 7304 * initiate the actual clean up; this function will wake 7305 * up the per-process aio_cleanup_thread. 7306 */ 7307 aio_cleanup_dr_delete_memory = (int (*)(proc_t *)) 7308 modgetsymvalue("aio_cleanup_dr_delete_memory", 0); 7309 if (aio_cleanup_dr_delete_memory == NULL) { 7310 cmn_err(CE_WARN, 7311 "aio_cleanup_dr_delete_memory not found in kaio"); 7312 return (0); 7313 } 7314 mutex_enter(&pidlock); 7315 for (procp = practive; (procp != NULL); procp = procp->p_next) { 7316 mutex_enter(&procp->p_lock); 7317 if (procp->p_aio != NULL) { 7318 /* cleanup proc's outstanding kaio */ 7319 cleaned += (*aio_cleanup_dr_delete_memory)(procp); 7320 } 7321 mutex_exit(&procp->p_lock); 7322 } 7323 mutex_exit(&pidlock); 7324 return (cleaned); 7325 } 7326 7327 /* 7328 * helper function for page_capture_thread 7329 */ 7330 static void 7331 page_capture_handle_outstanding(void) 7332 { 7333 int ntry; 7334 7335 /* Reap pages before attempting capture pages */ 7336 kmem_reap(); 7337 7338 if ((page_retire_pend_count() > page_retire_pend_kas_count()) && 7339 hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 7340 /* 7341 * Note: Purging only for platforms that support 7342 * ISM hat_pageunload() - mainly SPARC. On x86/x64 7343 * platforms ISM pages SE_SHARED locked until destroyed. 7344 */ 7345 7346 /* disable and purge seg_pcache */ 7347 (void) seg_p_disable(); 7348 for (ntry = 0; ntry < pc_thread_retry; ntry++) { 7349 if (!page_retire_pend_count()) 7350 break; 7351 if (do_aio_cleanup()) { 7352 /* 7353 * allow the apps cleanup threads 7354 * to run 7355 */ 7356 delay(pc_thread_shortwait); 7357 } 7358 page_capture_async(); 7359 } 7360 /* reenable seg_pcache */ 7361 seg_p_enable(); 7362 7363 /* completed what can be done. break out */ 7364 return; 7365 } 7366 7367 /* 7368 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap 7369 * and then attempt to capture. 7370 */ 7371 seg_preap(); 7372 page_capture_async(); 7373 } 7374 7375 /* 7376 * The page_capture_thread loops forever, looking to see if there are 7377 * pages still waiting to be captured. 7378 */ 7379 static void 7380 page_capture_thread(void) 7381 { 7382 callb_cpr_t c; 7383 int i; 7384 int high_pri_pages; 7385 int low_pri_pages; 7386 clock_t timeout; 7387 7388 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture"); 7389 7390 mutex_enter(&pc_thread_mutex); 7391 for (;;) { 7392 high_pri_pages = 0; 7393 low_pri_pages = 0; 7394 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7395 high_pri_pages += 7396 page_capture_hash[i].num_pages[PC_PRI_HI]; 7397 low_pri_pages += 7398 page_capture_hash[i].num_pages[PC_PRI_LO]; 7399 } 7400 7401 timeout = pc_thread_longwait; 7402 if (high_pri_pages != 0) { 7403 timeout = pc_thread_shortwait; 7404 page_capture_handle_outstanding(); 7405 } else if (low_pri_pages != 0) { 7406 page_capture_async(); 7407 } 7408 CALLB_CPR_SAFE_BEGIN(&c); 7409 (void) cv_reltimedwait(&pc_cv, &pc_thread_mutex, 7410 timeout, TR_CLOCK_TICK); 7411 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7412 } 7413 /*NOTREACHED*/ 7414 } 7415 /* 7416 * Attempt to locate a bucket that has enough pages to satisfy the request. 7417 * The initial check is done without the lock to avoid unneeded contention. 7418 * The function returns 1 if enough pages were found, else 0 if it could not 7419 * find enough pages in a bucket. 7420 */ 7421 static int 7422 pcf_decrement_bucket(pgcnt_t npages) 7423 { 7424 struct pcf *p; 7425 struct pcf *q; 7426 int i; 7427 7428 p = &pcf[PCF_INDEX()]; 7429 q = &pcf[pcf_fanout]; 7430 for (i = 0; i < pcf_fanout; i++) { 7431 if (p->pcf_count > npages) { 7432 /* 7433 * a good one to try. 7434 */ 7435 mutex_enter(&p->pcf_lock); 7436 if (p->pcf_count > npages) { 7437 p->pcf_count -= (uint_t)npages; 7438 /* 7439 * freemem is not protected by any lock. 7440 * Thus, we cannot have any assertion 7441 * containing freemem here. 7442 */ 7443 freemem -= npages; 7444 mutex_exit(&p->pcf_lock); 7445 return (1); 7446 } 7447 mutex_exit(&p->pcf_lock); 7448 } 7449 p++; 7450 if (p >= q) { 7451 p = pcf; 7452 } 7453 } 7454 return (0); 7455 } 7456 7457 /* 7458 * Arguments: 7459 * pcftotal_ret: If the value is not NULL and we have walked all the 7460 * buckets but did not find enough pages then it will 7461 * be set to the total number of pages in all the pcf 7462 * buckets. 7463 * npages: Is the number of pages we have been requested to 7464 * find. 7465 * unlock: If set to 0 we will leave the buckets locked if the 7466 * requested number of pages are not found. 7467 * 7468 * Go and try to satisfy the page request from any number of buckets. 7469 * This can be a very expensive operation as we have to lock the buckets 7470 * we are checking (and keep them locked), starting at bucket 0. 7471 * 7472 * The function returns 1 if enough pages were found, else 0 if it could not 7473 * find enough pages in the buckets. 7474 * 7475 */ 7476 static int 7477 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock) 7478 { 7479 struct pcf *p; 7480 pgcnt_t pcftotal; 7481 int i; 7482 7483 p = pcf; 7484 /* try to collect pages from several pcf bins */ 7485 for (pcftotal = 0, i = 0; i < pcf_fanout; i++) { 7486 mutex_enter(&p->pcf_lock); 7487 pcftotal += p->pcf_count; 7488 if (pcftotal >= npages) { 7489 /* 7490 * Wow! There are enough pages laying around 7491 * to satisfy the request. Do the accounting, 7492 * drop the locks we acquired, and go back. 7493 * 7494 * freemem is not protected by any lock. So, 7495 * we cannot have any assertion containing 7496 * freemem. 7497 */ 7498 freemem -= npages; 7499 while (p >= pcf) { 7500 if (p->pcf_count <= npages) { 7501 npages -= p->pcf_count; 7502 p->pcf_count = 0; 7503 } else { 7504 p->pcf_count -= (uint_t)npages; 7505 npages = 0; 7506 } 7507 mutex_exit(&p->pcf_lock); 7508 p--; 7509 } 7510 ASSERT(npages == 0); 7511 return (1); 7512 } 7513 p++; 7514 } 7515 if (unlock) { 7516 /* failed to collect pages - release the locks */ 7517 while (--p >= pcf) { 7518 mutex_exit(&p->pcf_lock); 7519 } 7520 } 7521 if (pcftotal_ret != NULL) 7522 *pcftotal_ret = pcftotal; 7523 return (0); 7524 } 7525