1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2015, Josef 'Jeff' Sipek <jeffpc@josefsipek.net> 24 * Copyright (c) 2015, 2016 by Delphix. All rights reserved. 25 * Copyright 2018 Joyent, Inc. 26 */ 27 28 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 29 /* All Rights Reserved */ 30 31 /* 32 * University Copyright- Copyright (c) 1982, 1986, 1988 33 * The Regents of the University of California 34 * All Rights Reserved 35 * 36 * University Acknowledgment- Portions of this document are derived from 37 * software developed by the University of California, Berkeley, and its 38 * contributors. 39 */ 40 41 /* 42 * VM - physical page management. 43 */ 44 45 #include <sys/types.h> 46 #include <sys/t_lock.h> 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/errno.h> 50 #include <sys/time.h> 51 #include <sys/vnode.h> 52 #include <sys/vm.h> 53 #include <sys/vtrace.h> 54 #include <sys/swap.h> 55 #include <sys/cmn_err.h> 56 #include <sys/tuneable.h> 57 #include <sys/sysmacros.h> 58 #include <sys/cpuvar.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/tnf_probe.h> 62 #include <sys/condvar_impl.h> 63 #include <sys/mem_config.h> 64 #include <sys/mem_cage.h> 65 #include <sys/kmem.h> 66 #include <sys/atomic.h> 67 #include <sys/strlog.h> 68 #include <sys/mman.h> 69 #include <sys/ontrap.h> 70 #include <sys/lgrp.h> 71 #include <sys/vfs.h> 72 73 #include <vm/hat.h> 74 #include <vm/anon.h> 75 #include <vm/page.h> 76 #include <vm/seg.h> 77 #include <vm/pvn.h> 78 #include <vm/seg_kmem.h> 79 #include <vm/vm_dep.h> 80 #include <sys/vm_usage.h> 81 #include <fs/fs_subr.h> 82 #include <sys/ddi.h> 83 #include <sys/modctl.h> 84 85 static pgcnt_t max_page_get; /* max page_get request size in pages */ 86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 87 88 /* 89 * freemem_lock protects all freemem variables: 90 * availrmem. Also this lock protects the globals which track the 91 * availrmem changes for accurate kernel footprint calculation. 92 * See below for an explanation of these 93 * globals. 94 */ 95 kmutex_t freemem_lock; 96 pgcnt_t availrmem; 97 pgcnt_t availrmem_initial; 98 99 /* 100 * These globals track availrmem changes to get a more accurate 101 * estimate of tke kernel size. Historically pp_kernel is used for 102 * kernel size and is based on availrmem. But availrmem is adjusted for 103 * locked pages in the system not just for kernel locked pages. 104 * These new counters will track the pages locked through segvn and 105 * by explicit user locking. 106 * 107 * pages_locked : How many pages are locked because of user specified 108 * locking through mlock or plock. 109 * 110 * pages_useclaim,pages_claimed : These two variables track the 111 * claim adjustments because of the protection changes on a segvn segment. 112 * 113 * All these globals are protected by the same lock which protects availrmem. 114 */ 115 pgcnt_t pages_locked = 0; 116 pgcnt_t pages_useclaim = 0; 117 pgcnt_t pages_claimed = 0; 118 119 120 /* 121 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 122 */ 123 static kmutex_t new_freemem_lock; 124 static uint_t freemem_wait; /* someone waiting for freemem */ 125 static kcondvar_t freemem_cv; 126 127 /* 128 * The logical page free list is maintained as two lists, the 'free' 129 * and the 'cache' lists. 130 * The free list contains those pages that should be reused first. 131 * 132 * The implementation of the lists is machine dependent. 133 * page_get_freelist(), page_get_cachelist(), 134 * page_list_sub(), and page_list_add() 135 * form the interface to the machine dependent implementation. 136 * 137 * Pages with p_free set are on the cache list. 138 * Pages with p_free and p_age set are on the free list, 139 * 140 * A page may be locked while on either list. 141 */ 142 143 /* 144 * free list accounting stuff. 145 * 146 * 147 * Spread out the value for the number of pages on the 148 * page free and page cache lists. If there is just one 149 * value, then it must be under just one lock. 150 * The lock contention and cache traffic are a real bother. 151 * 152 * When we acquire and then drop a single pcf lock 153 * we can start in the middle of the array of pcf structures. 154 * If we acquire more than one pcf lock at a time, we need to 155 * start at the front to avoid deadlocking. 156 * 157 * pcf_count holds the number of pages in each pool. 158 * 159 * pcf_block is set when page_create_get_something() has asked the 160 * PSM page freelist and page cachelist routines without specifying 161 * a color and nothing came back. This is used to block anything 162 * else from moving pages from one list to the other while the 163 * lists are searched again. If a page is freeed while pcf_block is 164 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 165 * of clearning pcf_block, doing the wakeups, etc. 166 */ 167 168 #define MAX_PCF_FANOUT NCPU 169 static uint_t pcf_fanout = 1; /* Will get changed at boot time */ 170 static uint_t pcf_fanout_mask = 0; 171 172 struct pcf { 173 kmutex_t pcf_lock; /* protects the structure */ 174 uint_t pcf_count; /* page count */ 175 uint_t pcf_wait; /* number of waiters */ 176 uint_t pcf_block; /* pcgs flag to page_free() */ 177 uint_t pcf_reserve; /* pages freed after pcf_block set */ 178 uint_t pcf_fill[10]; /* to line up on the caches */ 179 }; 180 181 /* 182 * PCF_INDEX hash needs to be dynamic (every so often the hash changes where 183 * it will hash the cpu to). This is done to prevent a drain condition 184 * from happening. This drain condition will occur when pcf_count decrement 185 * occurs on cpu A and the increment of pcf_count always occurs on cpu B. An 186 * example of this shows up with device interrupts. The dma buffer is allocated 187 * by the cpu requesting the IO thus the pcf_count is decremented based on that. 188 * When the memory is returned by the interrupt thread, the pcf_count will be 189 * incremented based on the cpu servicing the interrupt. 190 */ 191 static struct pcf pcf[MAX_PCF_FANOUT]; 192 #define PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \ 193 (randtick() >> 24)) & (pcf_fanout_mask)) 194 195 static int pcf_decrement_bucket(pgcnt_t); 196 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int); 197 198 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 199 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 200 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 201 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 202 203 #ifdef VM_STATS 204 205 /* 206 * No locks, but so what, they are only statistics. 207 */ 208 209 static struct page_tcnt { 210 int pc_free_cache; /* free's into cache list */ 211 int pc_free_dontneed; /* free's with dontneed */ 212 int pc_free_pageout; /* free's from pageout */ 213 int pc_free_free; /* free's into free list */ 214 int pc_free_pages; /* free's into large page free list */ 215 int pc_destroy_pages; /* large page destroy's */ 216 int pc_get_cache; /* get's from cache list */ 217 int pc_get_free; /* get's from free list */ 218 int pc_reclaim; /* reclaim's */ 219 int pc_abortfree; /* abort's of free pages */ 220 int pc_find_hit; /* find's that find page */ 221 int pc_find_miss; /* find's that don't find page */ 222 int pc_destroy_free; /* # of free pages destroyed */ 223 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 224 int pc_find_hashlen[PC_HASH_CNT+1]; 225 int pc_addclaim_pages; 226 int pc_subclaim_pages; 227 int pc_free_replacement_page[2]; 228 int pc_try_demote_pages[6]; 229 int pc_demote_pages[2]; 230 } pagecnt; 231 232 uint_t hashin_count; 233 uint_t hashin_not_held; 234 uint_t hashin_already; 235 236 uint_t hashout_count; 237 uint_t hashout_not_held; 238 239 uint_t page_create_count; 240 uint_t page_create_not_enough; 241 uint_t page_create_not_enough_again; 242 uint_t page_create_zero; 243 uint_t page_create_hashout; 244 uint_t page_create_page_lock_failed; 245 uint_t page_create_trylock_failed; 246 uint_t page_create_found_one; 247 uint_t page_create_hashin_failed; 248 uint_t page_create_dropped_phm; 249 250 uint_t page_create_new; 251 uint_t page_create_exists; 252 uint_t page_create_putbacks; 253 uint_t page_create_overshoot; 254 255 uint_t page_reclaim_zero; 256 uint_t page_reclaim_zero_locked; 257 258 uint_t page_rename_exists; 259 uint_t page_rename_count; 260 261 uint_t page_lookup_cnt[20]; 262 uint_t page_lookup_nowait_cnt[10]; 263 uint_t page_find_cnt; 264 uint_t page_exists_cnt; 265 uint_t page_exists_forreal_cnt; 266 uint_t page_lookup_dev_cnt; 267 uint_t get_cachelist_cnt; 268 uint_t page_create_cnt[10]; 269 uint_t alloc_pages[9]; 270 uint_t page_exphcontg[19]; 271 uint_t page_create_large_cnt[10]; 272 273 #endif 274 275 static inline page_t * 276 page_hash_search(ulong_t index, vnode_t *vnode, u_offset_t off) 277 { 278 uint_t mylen = 0; 279 page_t *page; 280 281 for (page = page_hash[index]; page; page = page->p_hash, mylen++) 282 if (page->p_vnode == vnode && page->p_offset == off) 283 break; 284 285 #ifdef VM_STATS 286 if (page != NULL) 287 pagecnt.pc_find_hit++; 288 else 289 pagecnt.pc_find_miss++; 290 291 pagecnt.pc_find_hashlen[MIN(mylen, PC_HASH_CNT)]++; 292 #endif 293 294 return (page); 295 } 296 297 298 #ifdef DEBUG 299 #define MEMSEG_SEARCH_STATS 300 #endif 301 302 #ifdef MEMSEG_SEARCH_STATS 303 struct memseg_stats { 304 uint_t nsearch; 305 uint_t nlastwon; 306 uint_t nhashwon; 307 uint_t nnotfound; 308 } memseg_stats; 309 310 #define MEMSEG_STAT_INCR(v) \ 311 atomic_inc_32(&memseg_stats.v) 312 #else 313 #define MEMSEG_STAT_INCR(x) 314 #endif 315 316 struct memseg *memsegs; /* list of memory segments */ 317 318 /* 319 * /etc/system tunable to control large page allocation hueristic. 320 * 321 * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup 322 * for large page allocation requests. If a large page is not readily 323 * avaliable on the local freelists we will go through additional effort 324 * to create a large page, potentially moving smaller pages around to coalesce 325 * larger pages in the local lgroup. 326 * Default value of LPAP_DEFAULT will go to remote freelists if large pages 327 * are not readily available in the local lgroup. 328 */ 329 enum lpap { 330 LPAP_DEFAULT, /* default large page allocation policy */ 331 LPAP_LOCAL /* local large page allocation policy */ 332 }; 333 334 enum lpap lpg_alloc_prefer = LPAP_DEFAULT; 335 336 static void page_init_mem_config(void); 337 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 338 static void page_do_hashout(page_t *); 339 static void page_capture_init(); 340 int page_capture_take_action(page_t *, uint_t, void *); 341 342 static void page_demote_vp_pages(page_t *); 343 344 345 void 346 pcf_init(void) 347 { 348 if (boot_ncpus != -1) { 349 pcf_fanout = boot_ncpus; 350 } else { 351 pcf_fanout = max_ncpus; 352 } 353 #ifdef sun4v 354 /* 355 * Force at least 4 buckets if possible for sun4v. 356 */ 357 pcf_fanout = MAX(pcf_fanout, 4); 358 #endif /* sun4v */ 359 360 /* 361 * Round up to the nearest power of 2. 362 */ 363 pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT); 364 if (!ISP2(pcf_fanout)) { 365 pcf_fanout = 1 << highbit(pcf_fanout); 366 367 if (pcf_fanout > MAX_PCF_FANOUT) { 368 pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1); 369 } 370 } 371 pcf_fanout_mask = pcf_fanout - 1; 372 } 373 374 /* 375 * vm subsystem related initialization 376 */ 377 void 378 vm_init(void) 379 { 380 boolean_t callb_vm_cpr(void *, int); 381 382 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 383 page_init_mem_config(); 384 page_retire_init(); 385 vm_usage_init(); 386 page_capture_init(); 387 } 388 389 /* 390 * This function is called at startup and when memory is added or deleted. 391 */ 392 void 393 init_pages_pp_maximum() 394 { 395 static pgcnt_t p_min; 396 static pgcnt_t pages_pp_maximum_startup; 397 static pgcnt_t avrmem_delta; 398 static int init_done; 399 static int user_set; /* true if set in /etc/system */ 400 401 if (init_done == 0) { 402 403 /* If the user specified a value, save it */ 404 if (pages_pp_maximum != 0) { 405 user_set = 1; 406 pages_pp_maximum_startup = pages_pp_maximum; 407 } 408 409 /* 410 * Setting of pages_pp_maximum is based first time 411 * on the value of availrmem just after the start-up 412 * allocations. To preserve this relationship at run 413 * time, use a delta from availrmem_initial. 414 */ 415 ASSERT(availrmem_initial >= availrmem); 416 avrmem_delta = availrmem_initial - availrmem; 417 418 /* The allowable floor of pages_pp_maximum */ 419 p_min = tune.t_minarmem + 100; 420 421 /* Make sure we don't come through here again. */ 422 init_done = 1; 423 } 424 /* 425 * Determine pages_pp_maximum, the number of currently available 426 * pages (availrmem) that can't be `locked'. If not set by 427 * the user, we set it to 4% of the currently available memory 428 * plus 4MB. 429 * But we also insist that it be greater than tune.t_minarmem; 430 * otherwise a process could lock down a lot of memory, get swapped 431 * out, and never have enough to get swapped back in. 432 */ 433 if (user_set) 434 pages_pp_maximum = pages_pp_maximum_startup; 435 else 436 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 437 + btop(4 * 1024 * 1024); 438 439 if (pages_pp_maximum <= p_min) { 440 pages_pp_maximum = p_min; 441 } 442 } 443 444 /* 445 * In the past, we limited the maximum pages that could be gotten to essentially 446 * 1/2 of the total pages on the system. However, this is too conservative for 447 * some cases. For example, if we want to host a large virtual machine which 448 * needs to use a significant portion of the system's memory. In practice, 449 * allowing more than 1/2 of the total pages is fine, but becomes problematic 450 * as we approach or exceed 75% of the pages on the system. Thus, we limit the 451 * maximum to 23/32 of the total pages, which is ~72%. 452 */ 453 void 454 set_max_page_get(pgcnt_t target_total_pages) 455 { 456 max_page_get = (target_total_pages >> 5) * 23; 457 ASSERT3U(max_page_get, >, 0); 458 } 459 460 pgcnt_t 461 get_max_page_get() 462 { 463 return (max_page_get); 464 } 465 466 static pgcnt_t pending_delete; 467 468 /*ARGSUSED*/ 469 static void 470 page_mem_config_post_add( 471 void *arg, 472 pgcnt_t delta_pages) 473 { 474 set_max_page_get(total_pages - pending_delete); 475 init_pages_pp_maximum(); 476 } 477 478 /*ARGSUSED*/ 479 static int 480 page_mem_config_pre_del( 481 void *arg, 482 pgcnt_t delta_pages) 483 { 484 pgcnt_t nv; 485 486 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 487 set_max_page_get(total_pages - nv); 488 return (0); 489 } 490 491 /*ARGSUSED*/ 492 static void 493 page_mem_config_post_del( 494 void *arg, 495 pgcnt_t delta_pages, 496 int cancelled) 497 { 498 pgcnt_t nv; 499 500 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 501 set_max_page_get(total_pages - nv); 502 if (!cancelled) 503 init_pages_pp_maximum(); 504 } 505 506 static kphysm_setup_vector_t page_mem_config_vec = { 507 KPHYSM_SETUP_VECTOR_VERSION, 508 page_mem_config_post_add, 509 page_mem_config_pre_del, 510 page_mem_config_post_del, 511 }; 512 513 static void 514 page_init_mem_config(void) 515 { 516 int ret; 517 518 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 519 ASSERT(ret == 0); 520 } 521 522 /* 523 * Evenly spread out the PCF counters for large free pages 524 */ 525 static void 526 page_free_large_ctr(pgcnt_t npages) 527 { 528 static struct pcf *p = pcf; 529 pgcnt_t lump; 530 531 freemem += npages; 532 533 lump = roundup(npages, pcf_fanout) / pcf_fanout; 534 535 while (npages > 0) { 536 537 ASSERT(!p->pcf_block); 538 539 if (lump < npages) { 540 p->pcf_count += (uint_t)lump; 541 npages -= lump; 542 } else { 543 p->pcf_count += (uint_t)npages; 544 npages = 0; 545 } 546 547 ASSERT(!p->pcf_wait); 548 549 if (++p > &pcf[pcf_fanout - 1]) 550 p = pcf; 551 } 552 553 ASSERT(npages == 0); 554 } 555 556 /* 557 * Add a physical chunk of memory to the system free lists during startup. 558 * Platform specific startup() allocates the memory for the page structs. 559 * 560 * num - number of page structures 561 * base - page number (pfn) to be associated with the first page. 562 * 563 * Since we are doing this during startup (ie. single threaded), we will 564 * use shortcut routines to avoid any locking overhead while putting all 565 * these pages on the freelists. 566 * 567 * NOTE: Any changes performed to page_free(), must also be performed to 568 * add_physmem() since this is how we initialize all page_t's at 569 * boot time. 570 */ 571 void 572 add_physmem( 573 page_t *pp, 574 pgcnt_t num, 575 pfn_t pnum) 576 { 577 page_t *root = NULL; 578 uint_t szc = page_num_pagesizes() - 1; 579 pgcnt_t large = page_get_pagecnt(szc); 580 pgcnt_t cnt = 0; 581 582 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 583 "add_physmem:pp %p num %lu", pp, num); 584 585 /* 586 * Arbitrarily limit the max page_get request 587 * to 1/2 of the page structs we have. 588 */ 589 total_pages += num; 590 set_max_page_get(total_pages); 591 592 PLCNT_MODIFY_MAX(pnum, (long)num); 593 594 /* 595 * The physical space for the pages array 596 * representing ram pages has already been 597 * allocated. Here we initialize each lock 598 * in the page structure, and put each on 599 * the free list 600 */ 601 for (; num; pp++, pnum++, num--) { 602 603 /* 604 * this needs to fill in the page number 605 * and do any other arch specific initialization 606 */ 607 add_physmem_cb(pp, pnum); 608 609 pp->p_lckcnt = 0; 610 pp->p_cowcnt = 0; 611 pp->p_slckcnt = 0; 612 613 /* 614 * Initialize the page lock as unlocked, since nobody 615 * can see or access this page yet. 616 */ 617 pp->p_selock = 0; 618 619 /* 620 * Initialize IO lock 621 */ 622 page_iolock_init(pp); 623 624 /* 625 * initialize other fields in the page_t 626 */ 627 PP_SETFREE(pp); 628 page_clr_all_props(pp); 629 PP_SETAGED(pp); 630 pp->p_offset = (u_offset_t)-1; 631 pp->p_next = pp; 632 pp->p_prev = pp; 633 634 /* 635 * Simple case: System doesn't support large pages. 636 */ 637 if (szc == 0) { 638 pp->p_szc = 0; 639 page_free_at_startup(pp); 640 continue; 641 } 642 643 /* 644 * Handle unaligned pages, we collect them up onto 645 * the root page until we have a full large page. 646 */ 647 if (!IS_P2ALIGNED(pnum, large)) { 648 649 /* 650 * If not in a large page, 651 * just free as small page. 652 */ 653 if (root == NULL) { 654 pp->p_szc = 0; 655 page_free_at_startup(pp); 656 continue; 657 } 658 659 /* 660 * Link a constituent page into the large page. 661 */ 662 pp->p_szc = szc; 663 page_list_concat(&root, &pp); 664 665 /* 666 * When large page is fully formed, free it. 667 */ 668 if (++cnt == large) { 669 page_free_large_ctr(cnt); 670 page_list_add_pages(root, PG_LIST_ISINIT); 671 root = NULL; 672 cnt = 0; 673 } 674 continue; 675 } 676 677 /* 678 * At this point we have a page number which 679 * is aligned. We assert that we aren't already 680 * in a different large page. 681 */ 682 ASSERT(IS_P2ALIGNED(pnum, large)); 683 ASSERT(root == NULL && cnt == 0); 684 685 /* 686 * If insufficient number of pages left to form 687 * a large page, just free the small page. 688 */ 689 if (num < large) { 690 pp->p_szc = 0; 691 page_free_at_startup(pp); 692 continue; 693 } 694 695 /* 696 * Otherwise start a new large page. 697 */ 698 pp->p_szc = szc; 699 cnt++; 700 root = pp; 701 } 702 ASSERT(root == NULL && cnt == 0); 703 } 704 705 /* 706 * Find a page representing the specified [vp, offset]. 707 * If we find the page but it is intransit coming in, 708 * it will have an "exclusive" lock and we wait for 709 * the i/o to complete. A page found on the free list 710 * is always reclaimed and then locked. On success, the page 711 * is locked, its data is valid and it isn't on the free 712 * list, while a NULL is returned if the page doesn't exist. 713 */ 714 page_t * 715 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 716 { 717 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 718 } 719 720 /* 721 * Find a page representing the specified [vp, offset]. 722 * We either return the one we found or, if passed in, 723 * create one with identity of [vp, offset] of the 724 * pre-allocated page. If we find existing page but it is 725 * intransit coming in, it will have an "exclusive" lock 726 * and we wait for the i/o to complete. A page found on 727 * the free list is always reclaimed and then locked. 728 * On success, the page is locked, its data is valid and 729 * it isn't on the free list, while a NULL is returned 730 * if the page doesn't exist and newpp is NULL; 731 */ 732 page_t * 733 page_lookup_create( 734 vnode_t *vp, 735 u_offset_t off, 736 se_t se, 737 page_t *newpp, 738 spgcnt_t *nrelocp, 739 int flags) 740 { 741 page_t *pp; 742 kmutex_t *phm; 743 ulong_t index; 744 uint_t hash_locked; 745 uint_t es; 746 747 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 748 VM_STAT_ADD(page_lookup_cnt[0]); 749 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 750 751 /* 752 * Acquire the appropriate page hash lock since 753 * we have to search the hash list. Pages that 754 * hash to this list can't change identity while 755 * this lock is held. 756 */ 757 hash_locked = 0; 758 index = PAGE_HASH_FUNC(vp, off); 759 phm = NULL; 760 top: 761 pp = page_hash_search(index, vp, off); 762 if (pp != NULL) { 763 VM_STAT_ADD(page_lookup_cnt[1]); 764 es = (newpp != NULL) ? 1 : 0; 765 es |= flags; 766 if (!hash_locked) { 767 VM_STAT_ADD(page_lookup_cnt[2]); 768 if (!page_try_reclaim_lock(pp, se, es)) { 769 /* 770 * On a miss, acquire the phm. Then 771 * next time, page_lock() will be called, 772 * causing a wait if the page is busy. 773 * just looping with page_trylock() would 774 * get pretty boring. 775 */ 776 VM_STAT_ADD(page_lookup_cnt[3]); 777 phm = PAGE_HASH_MUTEX(index); 778 mutex_enter(phm); 779 hash_locked = 1; 780 goto top; 781 } 782 } else { 783 VM_STAT_ADD(page_lookup_cnt[4]); 784 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 785 VM_STAT_ADD(page_lookup_cnt[5]); 786 goto top; 787 } 788 } 789 790 /* 791 * Since `pp' is locked it can not change identity now. 792 * Reconfirm we locked the correct page. 793 * 794 * Both the p_vnode and p_offset *must* be cast volatile 795 * to force a reload of their values: The page_hash_search 796 * function will have stuffed p_vnode and p_offset into 797 * registers before calling page_trylock(); another thread, 798 * actually holding the hash lock, could have changed the 799 * page's identity in memory, but our registers would not 800 * be changed, fooling the reconfirmation. If the hash 801 * lock was held during the search, the casting would 802 * not be needed. 803 */ 804 VM_STAT_ADD(page_lookup_cnt[6]); 805 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 806 ((volatile u_offset_t)(pp->p_offset) != off)) { 807 VM_STAT_ADD(page_lookup_cnt[7]); 808 if (hash_locked) { 809 panic("page_lookup_create: lost page %p", 810 (void *)pp); 811 /*NOTREACHED*/ 812 } 813 page_unlock(pp); 814 phm = PAGE_HASH_MUTEX(index); 815 mutex_enter(phm); 816 hash_locked = 1; 817 goto top; 818 } 819 820 /* 821 * If page_trylock() was called, then pp may still be on 822 * the cachelist (can't be on the free list, it would not 823 * have been found in the search). If it is on the 824 * cachelist it must be pulled now. To pull the page from 825 * the cachelist, it must be exclusively locked. 826 * 827 * The other big difference between page_trylock() and 828 * page_lock(), is that page_lock() will pull the 829 * page from whatever free list (the cache list in this 830 * case) the page is on. If page_trylock() was used 831 * above, then we have to do the reclaim ourselves. 832 */ 833 if ((!hash_locked) && (PP_ISFREE(pp))) { 834 ASSERT(PP_ISAGED(pp) == 0); 835 VM_STAT_ADD(page_lookup_cnt[8]); 836 837 /* 838 * page_relcaim will insure that we 839 * have this page exclusively 840 */ 841 842 if (!page_reclaim(pp, NULL)) { 843 /* 844 * Page_reclaim dropped whatever lock 845 * we held. 846 */ 847 VM_STAT_ADD(page_lookup_cnt[9]); 848 phm = PAGE_HASH_MUTEX(index); 849 mutex_enter(phm); 850 hash_locked = 1; 851 goto top; 852 } else if (se == SE_SHARED && newpp == NULL) { 853 VM_STAT_ADD(page_lookup_cnt[10]); 854 page_downgrade(pp); 855 } 856 } 857 858 if (hash_locked) { 859 mutex_exit(phm); 860 } 861 862 if (newpp != NULL && pp->p_szc < newpp->p_szc && 863 PAGE_EXCL(pp) && nrelocp != NULL) { 864 ASSERT(nrelocp != NULL); 865 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 866 NULL); 867 if (*nrelocp > 0) { 868 VM_STAT_COND_ADD(*nrelocp == 1, 869 page_lookup_cnt[11]); 870 VM_STAT_COND_ADD(*nrelocp > 1, 871 page_lookup_cnt[12]); 872 pp = newpp; 873 se = SE_EXCL; 874 } else { 875 if (se == SE_SHARED) { 876 page_downgrade(pp); 877 } 878 VM_STAT_ADD(page_lookup_cnt[13]); 879 } 880 } else if (newpp != NULL && nrelocp != NULL) { 881 if (PAGE_EXCL(pp) && se == SE_SHARED) { 882 page_downgrade(pp); 883 } 884 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 885 page_lookup_cnt[14]); 886 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 887 page_lookup_cnt[15]); 888 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 889 page_lookup_cnt[16]); 890 } else if (newpp != NULL && PAGE_EXCL(pp)) { 891 se = SE_EXCL; 892 } 893 } else if (!hash_locked) { 894 VM_STAT_ADD(page_lookup_cnt[17]); 895 phm = PAGE_HASH_MUTEX(index); 896 mutex_enter(phm); 897 hash_locked = 1; 898 goto top; 899 } else if (newpp != NULL) { 900 /* 901 * If we have a preallocated page then 902 * insert it now and basically behave like 903 * page_create. 904 */ 905 VM_STAT_ADD(page_lookup_cnt[18]); 906 /* 907 * Since we hold the page hash mutex and 908 * just searched for this page, page_hashin 909 * had better not fail. If it does, that 910 * means some thread did not follow the 911 * page hash mutex rules. Panic now and 912 * get it over with. As usual, go down 913 * holding all the locks. 914 */ 915 ASSERT(MUTEX_HELD(phm)); 916 if (!page_hashin(newpp, vp, off, phm)) { 917 ASSERT(MUTEX_HELD(phm)); 918 panic("page_lookup_create: hashin failed %p %p %llx %p", 919 (void *)newpp, (void *)vp, off, (void *)phm); 920 /*NOTREACHED*/ 921 } 922 ASSERT(MUTEX_HELD(phm)); 923 mutex_exit(phm); 924 phm = NULL; 925 page_set_props(newpp, P_REF); 926 page_io_lock(newpp); 927 pp = newpp; 928 se = SE_EXCL; 929 } else { 930 VM_STAT_ADD(page_lookup_cnt[19]); 931 mutex_exit(phm); 932 } 933 934 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 935 936 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 937 938 return (pp); 939 } 940 941 /* 942 * Search the hash list for the page representing the 943 * specified [vp, offset] and return it locked. Skip 944 * free pages and pages that cannot be locked as requested. 945 * Used while attempting to kluster pages. 946 */ 947 page_t * 948 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 949 { 950 page_t *pp; 951 kmutex_t *phm; 952 ulong_t index; 953 uint_t locked; 954 955 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 956 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 957 958 index = PAGE_HASH_FUNC(vp, off); 959 pp = page_hash_search(index, vp, off); 960 locked = 0; 961 if (pp == NULL) { 962 top: 963 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 964 locked = 1; 965 phm = PAGE_HASH_MUTEX(index); 966 mutex_enter(phm); 967 pp = page_hash_search(index, vp, off); 968 } 969 970 if (pp == NULL || PP_ISFREE(pp)) { 971 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 972 pp = NULL; 973 } else { 974 if (!page_trylock(pp, se)) { 975 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 976 pp = NULL; 977 } else { 978 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 979 /* 980 * See the comment in page_lookup() 981 */ 982 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 983 ((u_offset_t)(pp->p_offset) != off)) { 984 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 985 if (locked) { 986 panic("page_lookup_nowait %p", 987 (void *)pp); 988 /*NOTREACHED*/ 989 } 990 page_unlock(pp); 991 goto top; 992 } 993 if (PP_ISFREE(pp)) { 994 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 995 page_unlock(pp); 996 pp = NULL; 997 } 998 } 999 } 1000 if (locked) { 1001 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 1002 mutex_exit(phm); 1003 } 1004 1005 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 1006 1007 return (pp); 1008 } 1009 1010 /* 1011 * Search the hash list for a page with the specified [vp, off] 1012 * that is known to exist and is already locked. This routine 1013 * is typically used by segment SOFTUNLOCK routines. 1014 */ 1015 page_t * 1016 page_find(vnode_t *vp, u_offset_t off) 1017 { 1018 page_t *pp; 1019 kmutex_t *phm; 1020 ulong_t index; 1021 1022 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1023 VM_STAT_ADD(page_find_cnt); 1024 1025 index = PAGE_HASH_FUNC(vp, off); 1026 phm = PAGE_HASH_MUTEX(index); 1027 1028 mutex_enter(phm); 1029 pp = page_hash_search(index, vp, off); 1030 mutex_exit(phm); 1031 1032 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 1033 return (pp); 1034 } 1035 1036 /* 1037 * Determine whether a page with the specified [vp, off] 1038 * currently exists in the system. Obviously this should 1039 * only be considered as a hint since nothing prevents the 1040 * page from disappearing or appearing immediately after 1041 * the return from this routine. Subsequently, we don't 1042 * even bother to lock the list. 1043 */ 1044 page_t * 1045 page_exists(vnode_t *vp, u_offset_t off) 1046 { 1047 ulong_t index; 1048 1049 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1050 VM_STAT_ADD(page_exists_cnt); 1051 1052 index = PAGE_HASH_FUNC(vp, off); 1053 1054 return (page_hash_search(index, vp, off)); 1055 } 1056 1057 /* 1058 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1059 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1060 * with these pages locked SHARED. If necessary reclaim pages from 1061 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1062 * 1063 * If we fail to lock pages still return 1 if pages exist and contiguous. 1064 * But in this case return value is just a hint. ppa array won't be filled. 1065 * Caller should initialize ppa[0] as NULL to distinguish return value. 1066 * 1067 * Returns 0 if pages don't exist or not physically contiguous. 1068 * 1069 * This routine doesn't work for anonymous(swapfs) pages. 1070 */ 1071 int 1072 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1073 { 1074 pgcnt_t pages; 1075 pfn_t pfn; 1076 page_t *rootpp; 1077 pgcnt_t i; 1078 pgcnt_t j; 1079 u_offset_t save_off = off; 1080 ulong_t index; 1081 kmutex_t *phm; 1082 page_t *pp; 1083 uint_t pszc; 1084 int loopcnt = 0; 1085 1086 ASSERT(szc != 0); 1087 ASSERT(vp != NULL); 1088 ASSERT(!IS_SWAPFSVP(vp)); 1089 ASSERT(!VN_ISKAS(vp)); 1090 1091 again: 1092 if (++loopcnt > 3) { 1093 VM_STAT_ADD(page_exphcontg[0]); 1094 return (0); 1095 } 1096 1097 index = PAGE_HASH_FUNC(vp, off); 1098 phm = PAGE_HASH_MUTEX(index); 1099 1100 mutex_enter(phm); 1101 pp = page_hash_search(index, vp, off); 1102 mutex_exit(phm); 1103 1104 VM_STAT_ADD(page_exphcontg[1]); 1105 1106 if (pp == NULL) { 1107 VM_STAT_ADD(page_exphcontg[2]); 1108 return (0); 1109 } 1110 1111 pages = page_get_pagecnt(szc); 1112 rootpp = pp; 1113 pfn = rootpp->p_pagenum; 1114 1115 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1116 VM_STAT_ADD(page_exphcontg[3]); 1117 if (!page_trylock(pp, SE_SHARED)) { 1118 VM_STAT_ADD(page_exphcontg[4]); 1119 return (1); 1120 } 1121 /* 1122 * Also check whether p_pagenum was modified by DR. 1123 */ 1124 if (pp->p_szc != pszc || pp->p_vnode != vp || 1125 pp->p_offset != off || pp->p_pagenum != pfn) { 1126 VM_STAT_ADD(page_exphcontg[5]); 1127 page_unlock(pp); 1128 off = save_off; 1129 goto again; 1130 } 1131 /* 1132 * szc was non zero and vnode and offset matched after we 1133 * locked the page it means it can't become free on us. 1134 */ 1135 ASSERT(!PP_ISFREE(pp)); 1136 if (!IS_P2ALIGNED(pfn, pages)) { 1137 page_unlock(pp); 1138 return (0); 1139 } 1140 ppa[0] = pp; 1141 pp++; 1142 off += PAGESIZE; 1143 pfn++; 1144 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1145 if (!page_trylock(pp, SE_SHARED)) { 1146 VM_STAT_ADD(page_exphcontg[6]); 1147 pp--; 1148 while (i-- > 0) { 1149 page_unlock(pp); 1150 pp--; 1151 } 1152 ppa[0] = NULL; 1153 return (1); 1154 } 1155 if (pp->p_szc != pszc) { 1156 VM_STAT_ADD(page_exphcontg[7]); 1157 page_unlock(pp); 1158 pp--; 1159 while (i-- > 0) { 1160 page_unlock(pp); 1161 pp--; 1162 } 1163 ppa[0] = NULL; 1164 off = save_off; 1165 goto again; 1166 } 1167 /* 1168 * szc the same as for previous already locked pages 1169 * with right identity. Since this page had correct 1170 * szc after we locked it can't get freed or destroyed 1171 * and therefore must have the expected identity. 1172 */ 1173 ASSERT(!PP_ISFREE(pp)); 1174 if (pp->p_vnode != vp || 1175 pp->p_offset != off) { 1176 panic("page_exists_physcontig: " 1177 "large page identity doesn't match"); 1178 } 1179 ppa[i] = pp; 1180 ASSERT(pp->p_pagenum == pfn); 1181 } 1182 VM_STAT_ADD(page_exphcontg[8]); 1183 ppa[pages] = NULL; 1184 return (1); 1185 } else if (pszc >= szc) { 1186 VM_STAT_ADD(page_exphcontg[9]); 1187 if (!IS_P2ALIGNED(pfn, pages)) { 1188 return (0); 1189 } 1190 return (1); 1191 } 1192 1193 if (!IS_P2ALIGNED(pfn, pages)) { 1194 VM_STAT_ADD(page_exphcontg[10]); 1195 return (0); 1196 } 1197 1198 if (page_numtomemseg_nolock(pfn) != 1199 page_numtomemseg_nolock(pfn + pages - 1)) { 1200 VM_STAT_ADD(page_exphcontg[11]); 1201 return (0); 1202 } 1203 1204 /* 1205 * We loop up 4 times across pages to promote page size. 1206 * We're extra cautious to promote page size atomically with respect 1207 * to everybody else. But we can probably optimize into 1 loop if 1208 * this becomes an issue. 1209 */ 1210 1211 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1212 if (!page_trylock(pp, SE_EXCL)) { 1213 VM_STAT_ADD(page_exphcontg[12]); 1214 break; 1215 } 1216 /* 1217 * Check whether p_pagenum was modified by DR. 1218 */ 1219 if (pp->p_pagenum != pfn) { 1220 page_unlock(pp); 1221 break; 1222 } 1223 if (pp->p_vnode != vp || 1224 pp->p_offset != off) { 1225 VM_STAT_ADD(page_exphcontg[13]); 1226 page_unlock(pp); 1227 break; 1228 } 1229 if (pp->p_szc >= szc) { 1230 ASSERT(i == 0); 1231 page_unlock(pp); 1232 off = save_off; 1233 goto again; 1234 } 1235 } 1236 1237 if (i != pages) { 1238 VM_STAT_ADD(page_exphcontg[14]); 1239 --pp; 1240 while (i-- > 0) { 1241 page_unlock(pp); 1242 --pp; 1243 } 1244 return (0); 1245 } 1246 1247 pp = rootpp; 1248 for (i = 0; i < pages; i++, pp++) { 1249 if (PP_ISFREE(pp)) { 1250 VM_STAT_ADD(page_exphcontg[15]); 1251 ASSERT(!PP_ISAGED(pp)); 1252 ASSERT(pp->p_szc == 0); 1253 if (!page_reclaim(pp, NULL)) { 1254 break; 1255 } 1256 } else { 1257 ASSERT(pp->p_szc < szc); 1258 VM_STAT_ADD(page_exphcontg[16]); 1259 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1260 } 1261 } 1262 if (i < pages) { 1263 VM_STAT_ADD(page_exphcontg[17]); 1264 /* 1265 * page_reclaim failed because we were out of memory. 1266 * drop the rest of the locks and return because this page 1267 * must be already reallocated anyway. 1268 */ 1269 pp = rootpp; 1270 for (j = 0; j < pages; j++, pp++) { 1271 if (j != i) { 1272 page_unlock(pp); 1273 } 1274 } 1275 return (0); 1276 } 1277 1278 off = save_off; 1279 pp = rootpp; 1280 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1281 ASSERT(PAGE_EXCL(pp)); 1282 ASSERT(!PP_ISFREE(pp)); 1283 ASSERT(!hat_page_is_mapped(pp)); 1284 ASSERT(pp->p_vnode == vp); 1285 ASSERT(pp->p_offset == off); 1286 pp->p_szc = szc; 1287 } 1288 pp = rootpp; 1289 for (i = 0; i < pages; i++, pp++) { 1290 if (ppa == NULL) { 1291 page_unlock(pp); 1292 } else { 1293 ppa[i] = pp; 1294 page_downgrade(ppa[i]); 1295 } 1296 } 1297 if (ppa != NULL) { 1298 ppa[pages] = NULL; 1299 } 1300 VM_STAT_ADD(page_exphcontg[18]); 1301 ASSERT(vp->v_pages != NULL); 1302 return (1); 1303 } 1304 1305 /* 1306 * Determine whether a page with the specified [vp, off] 1307 * currently exists in the system and if so return its 1308 * size code. Obviously this should only be considered as 1309 * a hint since nothing prevents the page from disappearing 1310 * or appearing immediately after the return from this routine. 1311 */ 1312 int 1313 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1314 { 1315 page_t *pp; 1316 kmutex_t *phm; 1317 ulong_t index; 1318 int rc = 0; 1319 1320 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1321 ASSERT(szc != NULL); 1322 VM_STAT_ADD(page_exists_forreal_cnt); 1323 1324 index = PAGE_HASH_FUNC(vp, off); 1325 phm = PAGE_HASH_MUTEX(index); 1326 1327 mutex_enter(phm); 1328 pp = page_hash_search(index, vp, off); 1329 if (pp != NULL) { 1330 *szc = pp->p_szc; 1331 rc = 1; 1332 } 1333 mutex_exit(phm); 1334 return (rc); 1335 } 1336 1337 /* wakeup threads waiting for pages in page_create_get_something() */ 1338 void 1339 wakeup_pcgs(void) 1340 { 1341 if (!CV_HAS_WAITERS(&pcgs_cv)) 1342 return; 1343 cv_broadcast(&pcgs_cv); 1344 } 1345 1346 /* 1347 * 'freemem' is used all over the kernel as an indication of how many 1348 * pages are free (either on the cache list or on the free page list) 1349 * in the system. In very few places is a really accurate 'freemem' 1350 * needed. To avoid contention of the lock protecting a the 1351 * single freemem, it was spread out into NCPU buckets. Set_freemem 1352 * sets freemem to the total of all NCPU buckets. It is called from 1353 * clock() on each TICK. 1354 */ 1355 void 1356 set_freemem(void) 1357 { 1358 struct pcf *p; 1359 ulong_t t; 1360 uint_t i; 1361 1362 t = 0; 1363 p = pcf; 1364 for (i = 0; i < pcf_fanout; i++) { 1365 t += p->pcf_count; 1366 p++; 1367 } 1368 freemem = t; 1369 1370 /* 1371 * Don't worry about grabbing mutex. It's not that 1372 * critical if we miss a tick or two. This is 1373 * where we wakeup possible delayers in 1374 * page_create_get_something(). 1375 */ 1376 wakeup_pcgs(); 1377 } 1378 1379 ulong_t 1380 get_freemem() 1381 { 1382 struct pcf *p; 1383 ulong_t t; 1384 uint_t i; 1385 1386 t = 0; 1387 p = pcf; 1388 for (i = 0; i < pcf_fanout; i++) { 1389 t += p->pcf_count; 1390 p++; 1391 } 1392 /* 1393 * We just calculated it, might as well set it. 1394 */ 1395 freemem = t; 1396 return (t); 1397 } 1398 1399 /* 1400 * Acquire all of the page cache & free (pcf) locks. 1401 */ 1402 void 1403 pcf_acquire_all() 1404 { 1405 struct pcf *p; 1406 uint_t i; 1407 1408 p = pcf; 1409 for (i = 0; i < pcf_fanout; i++) { 1410 mutex_enter(&p->pcf_lock); 1411 p++; 1412 } 1413 } 1414 1415 /* 1416 * Release all the pcf_locks. 1417 */ 1418 void 1419 pcf_release_all() 1420 { 1421 struct pcf *p; 1422 uint_t i; 1423 1424 p = pcf; 1425 for (i = 0; i < pcf_fanout; i++) { 1426 mutex_exit(&p->pcf_lock); 1427 p++; 1428 } 1429 } 1430 1431 /* 1432 * Inform the VM system that we need some pages freed up. 1433 * Calls must be symmetric, e.g.: 1434 * 1435 * page_needfree(100); 1436 * wait a bit; 1437 * page_needfree(-100); 1438 */ 1439 void 1440 page_needfree(spgcnt_t npages) 1441 { 1442 mutex_enter(&new_freemem_lock); 1443 needfree += npages; 1444 mutex_exit(&new_freemem_lock); 1445 } 1446 1447 /* 1448 * Throttle for page_create(): try to prevent freemem from dropping 1449 * below throttlefree. We can't provide a 100% guarantee because 1450 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1451 * nibble away at the freelist. However, we can block all PG_WAIT 1452 * allocations until memory becomes available. The motivation is 1453 * that several things can fall apart when there's no free memory: 1454 * 1455 * (1) If pageout() needs memory to push a page, the system deadlocks. 1456 * 1457 * (2) By (broken) specification, timeout(9F) can neither fail nor 1458 * block, so it has no choice but to panic the system if it 1459 * cannot allocate a callout structure. 1460 * 1461 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1462 * it panics if it cannot allocate a callback structure. 1463 * 1464 * (4) Untold numbers of third-party drivers have not yet been hardened 1465 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1466 * success and panic the system with a data fault on failure. 1467 * (The long-term solution to this particular problem is to ship 1468 * hostile fault-injecting DEBUG kernels with the DDK.) 1469 * 1470 * It is theoretically impossible to guarantee success of non-blocking 1471 * allocations, but in practice, this throttle is very hard to break. 1472 */ 1473 static int 1474 page_create_throttle(pgcnt_t npages, int flags) 1475 { 1476 ulong_t fm; 1477 uint_t i; 1478 pgcnt_t tf; /* effective value of throttlefree */ 1479 1480 /* 1481 * Normal priority allocations. 1482 */ 1483 if ((flags & (PG_WAIT | PG_NORMALPRI)) == PG_NORMALPRI) { 1484 ASSERT(!(flags & (PG_PANIC | PG_PUSHPAGE))); 1485 return (freemem >= npages + throttlefree); 1486 } 1487 1488 /* 1489 * Never deny pages when: 1490 * - it's a thread that cannot block [NOMEMWAIT()] 1491 * - the allocation cannot block and must not fail 1492 * - the allocation cannot block and is pageout dispensated 1493 */ 1494 if (NOMEMWAIT() || 1495 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1496 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1497 return (1); 1498 1499 /* 1500 * If the allocation can't block, we look favorably upon it 1501 * unless we're below pageout_reserve. In that case we fail 1502 * the allocation because we want to make sure there are a few 1503 * pages available for pageout. 1504 */ 1505 if ((flags & PG_WAIT) == 0) 1506 return (freemem >= npages + pageout_reserve); 1507 1508 /* Calculate the effective throttlefree value */ 1509 tf = throttlefree - 1510 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1511 1512 cv_signal(&proc_pageout->p_cv); 1513 1514 for (;;) { 1515 fm = 0; 1516 pcf_acquire_all(); 1517 mutex_enter(&new_freemem_lock); 1518 for (i = 0; i < pcf_fanout; i++) { 1519 fm += pcf[i].pcf_count; 1520 pcf[i].pcf_wait++; 1521 mutex_exit(&pcf[i].pcf_lock); 1522 } 1523 freemem = fm; 1524 if (freemem >= npages + tf) { 1525 mutex_exit(&new_freemem_lock); 1526 break; 1527 } 1528 needfree += npages; 1529 freemem_wait++; 1530 cv_wait(&freemem_cv, &new_freemem_lock); 1531 freemem_wait--; 1532 needfree -= npages; 1533 mutex_exit(&new_freemem_lock); 1534 } 1535 return (1); 1536 } 1537 1538 /* 1539 * page_create_wait() is called to either coalesce pages from the 1540 * different pcf buckets or to wait because there simply are not 1541 * enough pages to satisfy the caller's request. 1542 * 1543 * Sadly, this is called from platform/vm/vm_machdep.c 1544 */ 1545 int 1546 page_create_wait(pgcnt_t npages, uint_t flags) 1547 { 1548 pgcnt_t total; 1549 uint_t i; 1550 struct pcf *p; 1551 1552 /* 1553 * Wait until there are enough free pages to satisfy our 1554 * entire request. 1555 * We set needfree += npages before prodding pageout, to make sure 1556 * it does real work when npages > lotsfree > freemem. 1557 */ 1558 VM_STAT_ADD(page_create_not_enough); 1559 1560 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1561 checkagain: 1562 if ((flags & PG_NORELOC) && 1563 kcage_freemem < kcage_throttlefree + npages) 1564 (void) kcage_create_throttle(npages, flags); 1565 1566 if (freemem < npages + throttlefree) 1567 if (!page_create_throttle(npages, flags)) 1568 return (0); 1569 1570 if (pcf_decrement_bucket(npages) || 1571 pcf_decrement_multiple(&total, npages, 0)) 1572 return (1); 1573 1574 /* 1575 * All of the pcf locks are held, there are not enough pages 1576 * to satisfy the request (npages < total). 1577 * Be sure to acquire the new_freemem_lock before dropping 1578 * the pcf locks. This prevents dropping wakeups in page_free(). 1579 * The order is always pcf_lock then new_freemem_lock. 1580 * 1581 * Since we hold all the pcf locks, it is a good time to set freemem. 1582 * 1583 * If the caller does not want to wait, return now. 1584 * Else turn the pageout daemon loose to find something 1585 * and wait till it does. 1586 * 1587 */ 1588 freemem = total; 1589 1590 if ((flags & PG_WAIT) == 0) { 1591 pcf_release_all(); 1592 1593 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1594 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1595 return (0); 1596 } 1597 1598 ASSERT(proc_pageout != NULL); 1599 cv_signal(&proc_pageout->p_cv); 1600 1601 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1602 "page_create_sleep_start: freemem %ld needfree %ld", 1603 freemem, needfree); 1604 1605 /* 1606 * We are going to wait. 1607 * We currently hold all of the pcf_locks, 1608 * get the new_freemem_lock (it protects freemem_wait), 1609 * before dropping the pcf_locks. 1610 */ 1611 mutex_enter(&new_freemem_lock); 1612 1613 p = pcf; 1614 for (i = 0; i < pcf_fanout; i++) { 1615 p->pcf_wait++; 1616 mutex_exit(&p->pcf_lock); 1617 p++; 1618 } 1619 1620 needfree += npages; 1621 freemem_wait++; 1622 1623 cv_wait(&freemem_cv, &new_freemem_lock); 1624 1625 freemem_wait--; 1626 needfree -= npages; 1627 1628 mutex_exit(&new_freemem_lock); 1629 1630 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1631 "page_create_sleep_end: freemem %ld needfree %ld", 1632 freemem, needfree); 1633 1634 VM_STAT_ADD(page_create_not_enough_again); 1635 goto checkagain; 1636 } 1637 /* 1638 * A routine to do the opposite of page_create_wait(). 1639 */ 1640 void 1641 page_create_putback(spgcnt_t npages) 1642 { 1643 struct pcf *p; 1644 pgcnt_t lump; 1645 uint_t *which; 1646 1647 /* 1648 * When a contiguous lump is broken up, we have to 1649 * deal with lots of pages (min 64) so lets spread 1650 * the wealth around. 1651 */ 1652 lump = roundup(npages, pcf_fanout) / pcf_fanout; 1653 freemem += npages; 1654 1655 for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) { 1656 which = &p->pcf_count; 1657 1658 mutex_enter(&p->pcf_lock); 1659 1660 if (p->pcf_block) { 1661 which = &p->pcf_reserve; 1662 } 1663 1664 if (lump < npages) { 1665 *which += (uint_t)lump; 1666 npages -= lump; 1667 } else { 1668 *which += (uint_t)npages; 1669 npages = 0; 1670 } 1671 1672 if (p->pcf_wait) { 1673 mutex_enter(&new_freemem_lock); 1674 /* 1675 * Check to see if some other thread 1676 * is actually waiting. Another bucket 1677 * may have woken it up by now. If there 1678 * are no waiters, then set our pcf_wait 1679 * count to zero to avoid coming in here 1680 * next time. 1681 */ 1682 if (freemem_wait) { 1683 if (npages > 1) { 1684 cv_broadcast(&freemem_cv); 1685 } else { 1686 cv_signal(&freemem_cv); 1687 } 1688 p->pcf_wait--; 1689 } else { 1690 p->pcf_wait = 0; 1691 } 1692 mutex_exit(&new_freemem_lock); 1693 } 1694 mutex_exit(&p->pcf_lock); 1695 } 1696 ASSERT(npages == 0); 1697 } 1698 1699 /* 1700 * A helper routine for page_create_get_something. 1701 * The indenting got to deep down there. 1702 * Unblock the pcf counters. Any pages freed after 1703 * pcf_block got set are moved to pcf_count and 1704 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1705 */ 1706 static void 1707 pcgs_unblock(void) 1708 { 1709 int i; 1710 struct pcf *p; 1711 1712 /* Update freemem while we're here. */ 1713 freemem = 0; 1714 p = pcf; 1715 for (i = 0; i < pcf_fanout; i++) { 1716 mutex_enter(&p->pcf_lock); 1717 ASSERT(p->pcf_count == 0); 1718 p->pcf_count = p->pcf_reserve; 1719 p->pcf_block = 0; 1720 freemem += p->pcf_count; 1721 if (p->pcf_wait) { 1722 mutex_enter(&new_freemem_lock); 1723 if (freemem_wait) { 1724 if (p->pcf_reserve > 1) { 1725 cv_broadcast(&freemem_cv); 1726 p->pcf_wait = 0; 1727 } else { 1728 cv_signal(&freemem_cv); 1729 p->pcf_wait--; 1730 } 1731 } else { 1732 p->pcf_wait = 0; 1733 } 1734 mutex_exit(&new_freemem_lock); 1735 } 1736 p->pcf_reserve = 0; 1737 mutex_exit(&p->pcf_lock); 1738 p++; 1739 } 1740 } 1741 1742 /* 1743 * Called from page_create_va() when both the cache and free lists 1744 * have been checked once. 1745 * 1746 * Either returns a page or panics since the accounting was done 1747 * way before we got here. 1748 * 1749 * We don't come here often, so leave the accounting on permanently. 1750 */ 1751 1752 #define MAX_PCGS 100 1753 1754 #ifdef DEBUG 1755 #define PCGS_TRIES 100 1756 #else /* DEBUG */ 1757 #define PCGS_TRIES 10 1758 #endif /* DEBUG */ 1759 1760 #ifdef VM_STATS 1761 uint_t pcgs_counts[PCGS_TRIES]; 1762 uint_t pcgs_too_many; 1763 uint_t pcgs_entered; 1764 uint_t pcgs_entered_noreloc; 1765 uint_t pcgs_locked; 1766 uint_t pcgs_cagelocked; 1767 #endif /* VM_STATS */ 1768 1769 static page_t * 1770 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1771 caddr_t vaddr, uint_t flags) 1772 { 1773 uint_t count; 1774 page_t *pp; 1775 uint_t locked, i; 1776 struct pcf *p; 1777 lgrp_t *lgrp; 1778 int cagelocked = 0; 1779 1780 VM_STAT_ADD(pcgs_entered); 1781 1782 /* 1783 * Tap any reserve freelists: if we fail now, we'll die 1784 * since the page(s) we're looking for have already been 1785 * accounted for. 1786 */ 1787 flags |= PG_PANIC; 1788 1789 if ((flags & PG_NORELOC) != 0) { 1790 VM_STAT_ADD(pcgs_entered_noreloc); 1791 /* 1792 * Requests for free pages from critical threads 1793 * such as pageout still won't throttle here, but 1794 * we must try again, to give the cageout thread 1795 * another chance to catch up. Since we already 1796 * accounted for the pages, we had better get them 1797 * this time. 1798 * 1799 * N.B. All non-critical threads acquire the pcgs_cagelock 1800 * to serialize access to the freelists. This implements a 1801 * turnstile-type synchornization to avoid starvation of 1802 * critical requests for PG_NORELOC memory by non-critical 1803 * threads: all non-critical threads must acquire a 'ticket' 1804 * before passing through, which entails making sure 1805 * kcage_freemem won't fall below minfree prior to grabbing 1806 * pages from the freelists. 1807 */ 1808 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1809 mutex_enter(&pcgs_cagelock); 1810 cagelocked = 1; 1811 VM_STAT_ADD(pcgs_cagelocked); 1812 } 1813 } 1814 1815 /* 1816 * Time to get serious. 1817 * We failed to get a `correctly colored' page from both the 1818 * free and cache lists. 1819 * We escalate in stage. 1820 * 1821 * First try both lists without worring about color. 1822 * 1823 * Then, grab all page accounting locks (ie. pcf[]) and 1824 * steal any pages that they have and set the pcf_block flag to 1825 * stop deletions from the lists. This will help because 1826 * a page can get added to the free list while we are looking 1827 * at the cache list, then another page could be added to the cache 1828 * list allowing the page on the free list to be removed as we 1829 * move from looking at the cache list to the free list. This 1830 * could happen over and over. We would never find the page 1831 * we have accounted for. 1832 * 1833 * Noreloc pages are a subset of the global (relocatable) page pool. 1834 * They are not tracked separately in the pcf bins, so it is 1835 * impossible to know when doing pcf accounting if the available 1836 * page(s) are noreloc pages or not. When looking for a noreloc page 1837 * it is quite easy to end up here even if the global (relocatable) 1838 * page pool has plenty of free pages but the noreloc pool is empty. 1839 * 1840 * When the noreloc pool is empty (or low), additional noreloc pages 1841 * are created by converting pages from the global page pool. This 1842 * process will stall during pcf accounting if the pcf bins are 1843 * already locked. Such is the case when a noreloc allocation is 1844 * looping here in page_create_get_something waiting for more noreloc 1845 * pages to appear. 1846 * 1847 * Short of adding a new field to the pcf bins to accurately track 1848 * the number of free noreloc pages, we instead do not grab the 1849 * pcgs_lock, do not set the pcf blocks and do not timeout when 1850 * allocating a noreloc page. This allows noreloc allocations to 1851 * loop without blocking global page pool allocations. 1852 * 1853 * NOTE: the behaviour of page_create_get_something has not changed 1854 * for the case of global page pool allocations. 1855 */ 1856 1857 flags &= ~PG_MATCH_COLOR; 1858 locked = 0; 1859 #if defined(__i386) || defined(__amd64) 1860 flags = page_create_update_flags_x86(flags); 1861 #endif 1862 1863 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1864 1865 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1866 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1867 flags, lgrp); 1868 if (pp == NULL) { 1869 pp = page_get_cachelist(vp, off, seg, vaddr, 1870 flags, lgrp); 1871 } 1872 if (pp == NULL) { 1873 /* 1874 * Serialize. Don't fight with other pcgs(). 1875 */ 1876 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1877 mutex_enter(&pcgs_lock); 1878 VM_STAT_ADD(pcgs_locked); 1879 locked = 1; 1880 p = pcf; 1881 for (i = 0; i < pcf_fanout; i++) { 1882 mutex_enter(&p->pcf_lock); 1883 ASSERT(p->pcf_block == 0); 1884 p->pcf_block = 1; 1885 p->pcf_reserve = p->pcf_count; 1886 p->pcf_count = 0; 1887 mutex_exit(&p->pcf_lock); 1888 p++; 1889 } 1890 freemem = 0; 1891 } 1892 1893 if (count) { 1894 /* 1895 * Since page_free() puts pages on 1896 * a list then accounts for it, we 1897 * just have to wait for page_free() 1898 * to unlock any page it was working 1899 * with. The page_lock()-page_reclaim() 1900 * path falls in the same boat. 1901 * 1902 * We don't need to check on the 1903 * PG_WAIT flag, we have already 1904 * accounted for the page we are 1905 * looking for in page_create_va(). 1906 * 1907 * We just wait a moment to let any 1908 * locked pages on the lists free up, 1909 * then continue around and try again. 1910 * 1911 * Will be awakened by set_freemem(). 1912 */ 1913 mutex_enter(&pcgs_wait_lock); 1914 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1915 mutex_exit(&pcgs_wait_lock); 1916 } 1917 } else { 1918 #ifdef VM_STATS 1919 if (count >= PCGS_TRIES) { 1920 VM_STAT_ADD(pcgs_too_many); 1921 } else { 1922 VM_STAT_ADD(pcgs_counts[count]); 1923 } 1924 #endif 1925 if (locked) { 1926 pcgs_unblock(); 1927 mutex_exit(&pcgs_lock); 1928 } 1929 if (cagelocked) 1930 mutex_exit(&pcgs_cagelock); 1931 return (pp); 1932 } 1933 } 1934 /* 1935 * we go down holding the pcf locks. 1936 */ 1937 panic("no %spage found %d", 1938 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1939 /*NOTREACHED*/ 1940 } 1941 1942 /* 1943 * Create enough pages for "bytes" worth of data starting at 1944 * "off" in "vp". 1945 * 1946 * Where flag must be one of: 1947 * 1948 * PG_EXCL: Exclusive create (fail if any page already 1949 * exists in the page cache) which does not 1950 * wait for memory to become available. 1951 * 1952 * PG_WAIT: Non-exclusive create which can wait for 1953 * memory to become available. 1954 * 1955 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1956 * (Not Supported) 1957 * 1958 * A doubly linked list of pages is returned to the caller. Each page 1959 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1960 * lock. 1961 * 1962 * Unable to change the parameters to page_create() in a minor release, 1963 * we renamed page_create() to page_create_va(), changed all known calls 1964 * from page_create() to page_create_va(), and created this wrapper. 1965 * 1966 * Upon a major release, we should break compatibility by deleting this 1967 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1968 * 1969 * NOTE: There is a copy of this interface as page_create_io() in 1970 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1971 * there. 1972 */ 1973 page_t * 1974 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1975 { 1976 caddr_t random_vaddr; 1977 struct seg kseg; 1978 1979 #ifdef DEBUG 1980 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1981 (void *)caller()); 1982 #endif 1983 1984 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1985 (uintptr_t)(off >> PAGESHIFT)); 1986 kseg.s_as = &kas; 1987 1988 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1989 } 1990 1991 #ifdef DEBUG 1992 uint32_t pg_alloc_pgs_mtbf = 0; 1993 #endif 1994 1995 /* 1996 * Used for large page support. It will attempt to allocate 1997 * a large page(s) off the freelist. 1998 * 1999 * Returns non zero on failure. 2000 */ 2001 int 2002 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 2003 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags) 2004 { 2005 pgcnt_t npgs, curnpgs, totpgs; 2006 size_t pgsz; 2007 page_t *pplist = NULL, *pp; 2008 int err = 0; 2009 lgrp_t *lgrp; 2010 2011 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 2012 ASSERT(pgflags == 0 || pgflags == PG_LOCAL); 2013 2014 /* 2015 * Check if system heavily prefers local large pages over remote 2016 * on systems with multiple lgroups. 2017 */ 2018 if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) { 2019 pgflags = PG_LOCAL; 2020 } 2021 2022 VM_STAT_ADD(alloc_pages[0]); 2023 2024 #ifdef DEBUG 2025 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 2026 return (ENOMEM); 2027 } 2028 #endif 2029 2030 /* 2031 * One must be NULL but not both. 2032 * And one must be non NULL but not both. 2033 */ 2034 ASSERT(basepp != NULL || ppa != NULL); 2035 ASSERT(basepp == NULL || ppa == NULL); 2036 2037 #if defined(__i386) || defined(__amd64) 2038 while (page_chk_freelist(szc) == 0) { 2039 VM_STAT_ADD(alloc_pages[8]); 2040 if (anypgsz == 0 || --szc == 0) 2041 return (ENOMEM); 2042 } 2043 #endif 2044 2045 pgsz = page_get_pagesize(szc); 2046 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 2047 2048 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 2049 2050 (void) page_create_wait(npgs, PG_WAIT); 2051 2052 while (npgs && szc) { 2053 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2054 if (pgflags == PG_LOCAL) { 2055 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2056 pgflags, lgrp); 2057 if (pp == NULL) { 2058 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2059 0, lgrp); 2060 } 2061 } else { 2062 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2063 0, lgrp); 2064 } 2065 if (pp != NULL) { 2066 VM_STAT_ADD(alloc_pages[1]); 2067 page_list_concat(&pplist, &pp); 2068 ASSERT(npgs >= curnpgs); 2069 npgs -= curnpgs; 2070 } else if (anypgsz) { 2071 VM_STAT_ADD(alloc_pages[2]); 2072 szc--; 2073 pgsz = page_get_pagesize(szc); 2074 curnpgs = pgsz >> PAGESHIFT; 2075 } else { 2076 VM_STAT_ADD(alloc_pages[3]); 2077 ASSERT(npgs == totpgs); 2078 page_create_putback(npgs); 2079 return (ENOMEM); 2080 } 2081 } 2082 if (szc == 0) { 2083 VM_STAT_ADD(alloc_pages[4]); 2084 ASSERT(npgs != 0); 2085 page_create_putback(npgs); 2086 err = ENOMEM; 2087 } else if (basepp != NULL) { 2088 ASSERT(npgs == 0); 2089 ASSERT(ppa == NULL); 2090 *basepp = pplist; 2091 } 2092 2093 npgs = totpgs - npgs; 2094 pp = pplist; 2095 2096 /* 2097 * Clear the free and age bits. Also if we were passed in a ppa then 2098 * fill it in with all the constituent pages from the large page. But 2099 * if we failed to allocate all the pages just free what we got. 2100 */ 2101 while (npgs != 0) { 2102 ASSERT(PP_ISFREE(pp)); 2103 ASSERT(PP_ISAGED(pp)); 2104 if (ppa != NULL || err != 0) { 2105 if (err == 0) { 2106 VM_STAT_ADD(alloc_pages[5]); 2107 PP_CLRFREE(pp); 2108 PP_CLRAGED(pp); 2109 page_sub(&pplist, pp); 2110 *ppa++ = pp; 2111 npgs--; 2112 } else { 2113 VM_STAT_ADD(alloc_pages[6]); 2114 ASSERT(pp->p_szc != 0); 2115 curnpgs = page_get_pagecnt(pp->p_szc); 2116 page_list_break(&pp, &pplist, curnpgs); 2117 page_list_add_pages(pp, 0); 2118 page_create_putback(curnpgs); 2119 ASSERT(npgs >= curnpgs); 2120 npgs -= curnpgs; 2121 } 2122 pp = pplist; 2123 } else { 2124 VM_STAT_ADD(alloc_pages[7]); 2125 PP_CLRFREE(pp); 2126 PP_CLRAGED(pp); 2127 pp = pp->p_next; 2128 npgs--; 2129 } 2130 } 2131 return (err); 2132 } 2133 2134 /* 2135 * Get a single large page off of the freelists, and set it up for use. 2136 * Number of bytes requested must be a supported page size. 2137 * 2138 * Note that this call may fail even if there is sufficient 2139 * memory available or PG_WAIT is set, so the caller must 2140 * be willing to fallback on page_create_va(), block and retry, 2141 * or fail the requester. 2142 */ 2143 page_t * 2144 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2145 struct seg *seg, caddr_t vaddr, void *arg) 2146 { 2147 pgcnt_t npages; 2148 page_t *pp; 2149 page_t *rootpp; 2150 lgrp_t *lgrp; 2151 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2152 2153 ASSERT(vp != NULL); 2154 2155 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2156 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0); 2157 /* but no others */ 2158 2159 ASSERT((flags & PG_EXCL) == PG_EXCL); 2160 2161 npages = btop(bytes); 2162 2163 if (!kcage_on || panicstr) { 2164 /* 2165 * Cage is OFF, or we are single threaded in 2166 * panic, so make everything a RELOC request. 2167 */ 2168 flags &= ~PG_NORELOC; 2169 } 2170 2171 /* 2172 * Make sure there's adequate physical memory available. 2173 * Note: PG_WAIT is ignored here. 2174 */ 2175 if (freemem <= throttlefree + npages) { 2176 VM_STAT_ADD(page_create_large_cnt[1]); 2177 return (NULL); 2178 } 2179 2180 /* 2181 * If cage is on, dampen draw from cage when available 2182 * cage space is low. 2183 */ 2184 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2185 kcage_freemem < kcage_throttlefree + npages) { 2186 2187 /* 2188 * The cage is on, the caller wants PG_NORELOC 2189 * pages and available cage memory is very low. 2190 * Call kcage_create_throttle() to attempt to 2191 * control demand on the cage. 2192 */ 2193 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2194 VM_STAT_ADD(page_create_large_cnt[2]); 2195 return (NULL); 2196 } 2197 } 2198 2199 if (!pcf_decrement_bucket(npages) && 2200 !pcf_decrement_multiple(NULL, npages, 1)) { 2201 VM_STAT_ADD(page_create_large_cnt[4]); 2202 return (NULL); 2203 } 2204 2205 /* 2206 * This is where this function behaves fundamentally differently 2207 * than page_create_va(); since we're intending to map the page 2208 * with a single TTE, we have to get it as a physically contiguous 2209 * hardware pagesize chunk. If we can't, we fail. 2210 */ 2211 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2212 LGRP_EXISTS(lgrp_table[*lgrpid])) 2213 lgrp = lgrp_table[*lgrpid]; 2214 else 2215 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2216 2217 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2218 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2219 page_create_putback(npages); 2220 VM_STAT_ADD(page_create_large_cnt[5]); 2221 return (NULL); 2222 } 2223 2224 /* 2225 * if we got the page with the wrong mtype give it back this is a 2226 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2227 * inside "if" and the workaround becomes just a nop 2228 */ 2229 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2230 page_list_add_pages(rootpp, 0); 2231 page_create_putback(npages); 2232 VM_STAT_ADD(page_create_large_cnt[6]); 2233 return (NULL); 2234 } 2235 2236 /* 2237 * If satisfying this request has left us with too little 2238 * memory, start the wheels turning to get some back. The 2239 * first clause of the test prevents waking up the pageout 2240 * daemon in situations where it would decide that there's 2241 * nothing to do. 2242 */ 2243 if (nscan < desscan && freemem < minfree) { 2244 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2245 "pageout_cv_signal:freemem %ld", freemem); 2246 cv_signal(&proc_pageout->p_cv); 2247 } 2248 2249 pp = rootpp; 2250 while (npages--) { 2251 ASSERT(PAGE_EXCL(pp)); 2252 ASSERT(pp->p_vnode == NULL); 2253 ASSERT(!hat_page_is_mapped(pp)); 2254 PP_CLRFREE(pp); 2255 PP_CLRAGED(pp); 2256 if (!page_hashin(pp, vp, off, NULL)) 2257 panic("page_create_large: hashin failed: page %p", 2258 (void *)pp); 2259 page_io_lock(pp); 2260 off += PAGESIZE; 2261 pp = pp->p_next; 2262 } 2263 2264 VM_STAT_ADD(page_create_large_cnt[0]); 2265 return (rootpp); 2266 } 2267 2268 page_t * 2269 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2270 struct seg *seg, caddr_t vaddr) 2271 { 2272 page_t *plist = NULL; 2273 pgcnt_t npages; 2274 pgcnt_t found_on_free = 0; 2275 pgcnt_t pages_req; 2276 page_t *npp = NULL; 2277 struct pcf *p; 2278 lgrp_t *lgrp; 2279 2280 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2281 "page_create_start:vp %p off %llx bytes %lu flags %x", 2282 vp, off, bytes, flags); 2283 2284 ASSERT(bytes != 0 && vp != NULL); 2285 2286 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2287 panic("page_create: invalid flags"); 2288 /*NOTREACHED*/ 2289 } 2290 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2291 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0); 2292 /* but no others */ 2293 2294 pages_req = npages = btopr(bytes); 2295 /* 2296 * Try to see whether request is too large to *ever* be 2297 * satisfied, in order to prevent deadlock. We arbitrarily 2298 * decide to limit maximum size requests to max_page_get. 2299 */ 2300 if (npages >= max_page_get) { 2301 if ((flags & PG_WAIT) == 0) { 2302 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2303 "page_create_toobig:vp %p off %llx npages " 2304 "%lu max_page_get %lu", 2305 vp, off, npages, max_page_get); 2306 return (NULL); 2307 } else { 2308 cmn_err(CE_WARN, 2309 "Request for too much kernel memory " 2310 "(%lu bytes), will hang forever", bytes); 2311 for (;;) 2312 delay(1000000000); 2313 } 2314 } 2315 2316 if (!kcage_on || panicstr) { 2317 /* 2318 * Cage is OFF, or we are single threaded in 2319 * panic, so make everything a RELOC request. 2320 */ 2321 flags &= ~PG_NORELOC; 2322 } 2323 2324 if (freemem <= throttlefree + npages) 2325 if (!page_create_throttle(npages, flags)) 2326 return (NULL); 2327 2328 /* 2329 * If cage is on, dampen draw from cage when available 2330 * cage space is low. 2331 */ 2332 if ((flags & PG_NORELOC) && 2333 kcage_freemem < kcage_throttlefree + npages) { 2334 2335 /* 2336 * The cage is on, the caller wants PG_NORELOC 2337 * pages and available cage memory is very low. 2338 * Call kcage_create_throttle() to attempt to 2339 * control demand on the cage. 2340 */ 2341 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2342 return (NULL); 2343 } 2344 2345 VM_STAT_ADD(page_create_cnt[0]); 2346 2347 if (!pcf_decrement_bucket(npages)) { 2348 /* 2349 * Have to look harder. If npages is greater than 2350 * one, then we might have to coalesce the counters. 2351 * 2352 * Go wait. We come back having accounted 2353 * for the memory. 2354 */ 2355 VM_STAT_ADD(page_create_cnt[1]); 2356 if (!page_create_wait(npages, flags)) { 2357 VM_STAT_ADD(page_create_cnt[2]); 2358 return (NULL); 2359 } 2360 } 2361 2362 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2363 "page_create_success:vp %p off %llx", vp, off); 2364 2365 /* 2366 * If satisfying this request has left us with too little 2367 * memory, start the wheels turning to get some back. The 2368 * first clause of the test prevents waking up the pageout 2369 * daemon in situations where it would decide that there's 2370 * nothing to do. 2371 */ 2372 if (nscan < desscan && freemem < minfree) { 2373 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2374 "pageout_cv_signal:freemem %ld", freemem); 2375 cv_signal(&proc_pageout->p_cv); 2376 } 2377 2378 /* 2379 * Loop around collecting the requested number of pages. 2380 * Most of the time, we have to `create' a new page. With 2381 * this in mind, pull the page off the free list before 2382 * getting the hash lock. This will minimize the hash 2383 * lock hold time, nesting, and the like. If it turns 2384 * out we don't need the page, we put it back at the end. 2385 */ 2386 while (npages--) { 2387 page_t *pp; 2388 kmutex_t *phm = NULL; 2389 ulong_t index; 2390 2391 index = PAGE_HASH_FUNC(vp, off); 2392 top: 2393 ASSERT(phm == NULL); 2394 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2395 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2396 2397 if (npp == NULL) { 2398 /* 2399 * Try to get a page from the freelist (ie, 2400 * a page with no [vp, off] tag). If that 2401 * fails, use the cachelist. 2402 * 2403 * During the first attempt at both the free 2404 * and cache lists we try for the correct color. 2405 */ 2406 /* 2407 * XXXX-how do we deal with virtual indexed 2408 * caches and and colors? 2409 */ 2410 VM_STAT_ADD(page_create_cnt[4]); 2411 /* 2412 * Get lgroup to allocate next page of shared memory 2413 * from and use it to specify where to allocate 2414 * the physical memory 2415 */ 2416 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2417 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2418 flags | PG_MATCH_COLOR, lgrp); 2419 if (npp == NULL) { 2420 npp = page_get_cachelist(vp, off, seg, 2421 vaddr, flags | PG_MATCH_COLOR, lgrp); 2422 if (npp == NULL) { 2423 npp = page_create_get_something(vp, 2424 off, seg, vaddr, 2425 flags & ~PG_MATCH_COLOR); 2426 } 2427 2428 if (PP_ISAGED(npp) == 0) { 2429 /* 2430 * Since this page came from the 2431 * cachelist, we must destroy the 2432 * old vnode association. 2433 */ 2434 page_hashout(npp, NULL); 2435 } 2436 } 2437 } 2438 2439 /* 2440 * We own this page! 2441 */ 2442 ASSERT(PAGE_EXCL(npp)); 2443 ASSERT(npp->p_vnode == NULL); 2444 ASSERT(!hat_page_is_mapped(npp)); 2445 PP_CLRFREE(npp); 2446 PP_CLRAGED(npp); 2447 2448 /* 2449 * Here we have a page in our hot little mits and are 2450 * just waiting to stuff it on the appropriate lists. 2451 * Get the mutex and check to see if it really does 2452 * not exist. 2453 */ 2454 phm = PAGE_HASH_MUTEX(index); 2455 mutex_enter(phm); 2456 pp = page_hash_search(index, vp, off); 2457 if (pp == NULL) { 2458 VM_STAT_ADD(page_create_new); 2459 pp = npp; 2460 npp = NULL; 2461 if (!page_hashin(pp, vp, off, phm)) { 2462 /* 2463 * Since we hold the page hash mutex and 2464 * just searched for this page, page_hashin 2465 * had better not fail. If it does, that 2466 * means somethread did not follow the 2467 * page hash mutex rules. Panic now and 2468 * get it over with. As usual, go down 2469 * holding all the locks. 2470 */ 2471 ASSERT(MUTEX_HELD(phm)); 2472 panic("page_create: " 2473 "hashin failed %p %p %llx %p", 2474 (void *)pp, (void *)vp, off, (void *)phm); 2475 /*NOTREACHED*/ 2476 } 2477 ASSERT(MUTEX_HELD(phm)); 2478 mutex_exit(phm); 2479 phm = NULL; 2480 2481 /* 2482 * Hat layer locking need not be done to set 2483 * the following bits since the page is not hashed 2484 * and was on the free list (i.e., had no mappings). 2485 * 2486 * Set the reference bit to protect 2487 * against immediate pageout 2488 * 2489 * XXXmh modify freelist code to set reference 2490 * bit so we don't have to do it here. 2491 */ 2492 page_set_props(pp, P_REF); 2493 found_on_free++; 2494 } else { 2495 VM_STAT_ADD(page_create_exists); 2496 if (flags & PG_EXCL) { 2497 /* 2498 * Found an existing page, and the caller 2499 * wanted all new pages. Undo all of the work 2500 * we have done. 2501 */ 2502 mutex_exit(phm); 2503 phm = NULL; 2504 while (plist != NULL) { 2505 pp = plist; 2506 page_sub(&plist, pp); 2507 page_io_unlock(pp); 2508 /* large pages should not end up here */ 2509 ASSERT(pp->p_szc == 0); 2510 /*LINTED: constant in conditional ctx*/ 2511 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2512 } 2513 VM_STAT_ADD(page_create_found_one); 2514 goto fail; 2515 } 2516 ASSERT(flags & PG_WAIT); 2517 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2518 /* 2519 * Start all over again if we blocked trying 2520 * to lock the page. 2521 */ 2522 mutex_exit(phm); 2523 VM_STAT_ADD(page_create_page_lock_failed); 2524 phm = NULL; 2525 goto top; 2526 } 2527 mutex_exit(phm); 2528 phm = NULL; 2529 2530 if (PP_ISFREE(pp)) { 2531 ASSERT(PP_ISAGED(pp) == 0); 2532 VM_STAT_ADD(pagecnt.pc_get_cache); 2533 page_list_sub(pp, PG_CACHE_LIST); 2534 PP_CLRFREE(pp); 2535 found_on_free++; 2536 } 2537 } 2538 2539 /* 2540 * Got a page! It is locked. Acquire the i/o 2541 * lock since we are going to use the p_next and 2542 * p_prev fields to link the requested pages together. 2543 */ 2544 page_io_lock(pp); 2545 page_add(&plist, pp); 2546 plist = plist->p_next; 2547 off += PAGESIZE; 2548 vaddr += PAGESIZE; 2549 } 2550 2551 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2552 fail: 2553 if (npp != NULL) { 2554 /* 2555 * Did not need this page after all. 2556 * Put it back on the free list. 2557 */ 2558 VM_STAT_ADD(page_create_putbacks); 2559 PP_SETFREE(npp); 2560 PP_SETAGED(npp); 2561 npp->p_offset = (u_offset_t)-1; 2562 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2563 page_unlock(npp); 2564 2565 } 2566 2567 ASSERT(pages_req >= found_on_free); 2568 2569 { 2570 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2571 2572 if (overshoot) { 2573 VM_STAT_ADD(page_create_overshoot); 2574 p = &pcf[PCF_INDEX()]; 2575 mutex_enter(&p->pcf_lock); 2576 if (p->pcf_block) { 2577 p->pcf_reserve += overshoot; 2578 } else { 2579 p->pcf_count += overshoot; 2580 if (p->pcf_wait) { 2581 mutex_enter(&new_freemem_lock); 2582 if (freemem_wait) { 2583 cv_signal(&freemem_cv); 2584 p->pcf_wait--; 2585 } else { 2586 p->pcf_wait = 0; 2587 } 2588 mutex_exit(&new_freemem_lock); 2589 } 2590 } 2591 mutex_exit(&p->pcf_lock); 2592 /* freemem is approximate, so this test OK */ 2593 if (!p->pcf_block) 2594 freemem += overshoot; 2595 } 2596 } 2597 2598 return (plist); 2599 } 2600 2601 /* 2602 * One or more constituent pages of this large page has been marked 2603 * toxic. Simply demote the large page to PAGESIZE pages and let 2604 * page_free() handle it. This routine should only be called by 2605 * large page free routines (page_free_pages() and page_destroy_pages(). 2606 * All pages are locked SE_EXCL and have already been marked free. 2607 */ 2608 static void 2609 page_free_toxic_pages(page_t *rootpp) 2610 { 2611 page_t *tpp; 2612 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2613 uint_t szc = rootpp->p_szc; 2614 2615 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2616 ASSERT(tpp->p_szc == szc); 2617 ASSERT((PAGE_EXCL(tpp) && 2618 !page_iolock_assert(tpp)) || panicstr); 2619 tpp->p_szc = 0; 2620 } 2621 2622 while (rootpp != NULL) { 2623 tpp = rootpp; 2624 page_sub(&rootpp, tpp); 2625 ASSERT(PP_ISFREE(tpp)); 2626 PP_CLRFREE(tpp); 2627 page_free(tpp, 1); 2628 } 2629 } 2630 2631 /* 2632 * Put page on the "free" list. 2633 * The free list is really two lists maintained by 2634 * the PSM of whatever machine we happen to be on. 2635 */ 2636 void 2637 page_free(page_t *pp, int dontneed) 2638 { 2639 struct pcf *p; 2640 uint_t pcf_index; 2641 2642 ASSERT((PAGE_EXCL(pp) && 2643 !page_iolock_assert(pp)) || panicstr); 2644 2645 if (PP_ISFREE(pp)) { 2646 panic("page_free: page %p is free", (void *)pp); 2647 } 2648 2649 if (pp->p_szc != 0) { 2650 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2651 PP_ISKAS(pp)) { 2652 panic("page_free: anon or kernel " 2653 "or no vnode large page %p", (void *)pp); 2654 } 2655 page_demote_vp_pages(pp); 2656 ASSERT(pp->p_szc == 0); 2657 } 2658 2659 /* 2660 * The page_struct_lock need not be acquired to examine these 2661 * fields since the page has an "exclusive" lock. 2662 */ 2663 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 2664 pp->p_slckcnt != 0) { 2665 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 2666 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt, 2667 pp->p_cowcnt, pp->p_slckcnt); 2668 /*NOTREACHED*/ 2669 } 2670 2671 ASSERT(!hat_page_getshare(pp)); 2672 2673 PP_SETFREE(pp); 2674 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2675 !hat_ismod(pp)); 2676 page_clr_all_props(pp); 2677 ASSERT(!hat_page_getshare(pp)); 2678 2679 /* 2680 * Now we add the page to the head of the free list. 2681 * But if this page is associated with a paged vnode 2682 * then we adjust the head forward so that the page is 2683 * effectively at the end of the list. 2684 */ 2685 if (pp->p_vnode == NULL) { 2686 /* 2687 * Page has no identity, put it on the free list. 2688 */ 2689 PP_SETAGED(pp); 2690 pp->p_offset = (u_offset_t)-1; 2691 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2692 VM_STAT_ADD(pagecnt.pc_free_free); 2693 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2694 "page_free_free:pp %p", pp); 2695 } else { 2696 PP_CLRAGED(pp); 2697 2698 if (!dontneed) { 2699 /* move it to the tail of the list */ 2700 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2701 2702 VM_STAT_ADD(pagecnt.pc_free_cache); 2703 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2704 "page_free_cache_tail:pp %p", pp); 2705 } else { 2706 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2707 2708 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2709 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2710 "page_free_cache_head:pp %p", pp); 2711 } 2712 } 2713 page_unlock(pp); 2714 2715 /* 2716 * Now do the `freemem' accounting. 2717 */ 2718 pcf_index = PCF_INDEX(); 2719 p = &pcf[pcf_index]; 2720 2721 mutex_enter(&p->pcf_lock); 2722 if (p->pcf_block) { 2723 p->pcf_reserve += 1; 2724 } else { 2725 p->pcf_count += 1; 2726 if (p->pcf_wait) { 2727 mutex_enter(&new_freemem_lock); 2728 /* 2729 * Check to see if some other thread 2730 * is actually waiting. Another bucket 2731 * may have woken it up by now. If there 2732 * are no waiters, then set our pcf_wait 2733 * count to zero to avoid coming in here 2734 * next time. Also, since only one page 2735 * was put on the free list, just wake 2736 * up one waiter. 2737 */ 2738 if (freemem_wait) { 2739 cv_signal(&freemem_cv); 2740 p->pcf_wait--; 2741 } else { 2742 p->pcf_wait = 0; 2743 } 2744 mutex_exit(&new_freemem_lock); 2745 } 2746 } 2747 mutex_exit(&p->pcf_lock); 2748 2749 /* freemem is approximate, so this test OK */ 2750 if (!p->pcf_block) 2751 freemem += 1; 2752 } 2753 2754 /* 2755 * Put page on the "free" list during intial startup. 2756 * This happens during initial single threaded execution. 2757 */ 2758 void 2759 page_free_at_startup(page_t *pp) 2760 { 2761 struct pcf *p; 2762 uint_t pcf_index; 2763 2764 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2765 VM_STAT_ADD(pagecnt.pc_free_free); 2766 2767 /* 2768 * Now do the `freemem' accounting. 2769 */ 2770 pcf_index = PCF_INDEX(); 2771 p = &pcf[pcf_index]; 2772 2773 ASSERT(p->pcf_block == 0); 2774 ASSERT(p->pcf_wait == 0); 2775 p->pcf_count += 1; 2776 2777 /* freemem is approximate, so this is OK */ 2778 freemem += 1; 2779 } 2780 2781 void 2782 page_free_pages(page_t *pp) 2783 { 2784 page_t *tpp, *rootpp = NULL; 2785 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2786 pgcnt_t i; 2787 uint_t szc = pp->p_szc; 2788 2789 VM_STAT_ADD(pagecnt.pc_free_pages); 2790 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2791 "page_free_free:pp %p", pp); 2792 2793 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2794 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2795 panic("page_free_pages: not root page %p", (void *)pp); 2796 /*NOTREACHED*/ 2797 } 2798 2799 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2800 ASSERT((PAGE_EXCL(tpp) && 2801 !page_iolock_assert(tpp)) || panicstr); 2802 if (PP_ISFREE(tpp)) { 2803 panic("page_free_pages: page %p is free", (void *)tpp); 2804 /*NOTREACHED*/ 2805 } 2806 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2807 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 2808 panic("page_free_pages %p", (void *)tpp); 2809 /*NOTREACHED*/ 2810 } 2811 2812 ASSERT(!hat_page_getshare(tpp)); 2813 ASSERT(tpp->p_vnode == NULL); 2814 ASSERT(tpp->p_szc == szc); 2815 2816 PP_SETFREE(tpp); 2817 page_clr_all_props(tpp); 2818 PP_SETAGED(tpp); 2819 tpp->p_offset = (u_offset_t)-1; 2820 ASSERT(tpp->p_next == tpp); 2821 ASSERT(tpp->p_prev == tpp); 2822 page_list_concat(&rootpp, &tpp); 2823 } 2824 ASSERT(rootpp == pp); 2825 2826 page_list_add_pages(rootpp, 0); 2827 page_create_putback(pgcnt); 2828 } 2829 2830 int free_pages = 1; 2831 2832 /* 2833 * This routine attempts to return pages to the cachelist via page_release(). 2834 * It does not *have* to be successful in all cases, since the pageout scanner 2835 * will catch any pages it misses. It does need to be fast and not introduce 2836 * too much overhead. 2837 * 2838 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2839 * don't lock and retry. This is ok, since the page scanner will eventually 2840 * find any page we miss in free_vp_pages(). 2841 */ 2842 void 2843 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2844 { 2845 page_t *pp; 2846 u_offset_t eoff; 2847 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2848 2849 eoff = off + len; 2850 2851 if (free_pages == 0) 2852 return; 2853 if (swap_in_range(vp, off, len)) 2854 return; 2855 2856 for (; off < eoff; off += PAGESIZE) { 2857 2858 /* 2859 * find the page using a fast, but inexact search. It'll be OK 2860 * if a few pages slip through the cracks here. 2861 */ 2862 pp = page_exists(vp, off); 2863 2864 /* 2865 * If we didn't find the page (it may not exist), the page 2866 * is free, looks still in use (shared), or we can't lock it, 2867 * just give up. 2868 */ 2869 if (pp == NULL || 2870 PP_ISFREE(pp) || 2871 page_share_cnt(pp) > 0 || 2872 !page_trylock(pp, SE_EXCL)) 2873 continue; 2874 2875 /* 2876 * Once we have locked pp, verify that it's still the 2877 * correct page and not already free 2878 */ 2879 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2880 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2881 page_unlock(pp); 2882 continue; 2883 } 2884 2885 /* 2886 * try to release the page... 2887 */ 2888 (void) page_release(pp, 1); 2889 } 2890 } 2891 2892 /* 2893 * Reclaim the given page from the free list. 2894 * If pp is part of a large pages, only the given constituent page is reclaimed 2895 * and the large page it belonged to will be demoted. This can only happen 2896 * if the page is not on the cachelist. 2897 * 2898 * Returns 1 on success or 0 on failure. 2899 * 2900 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2901 * If `lock' is non-null, it will be dropped and re-acquired if 2902 * the routine must wait while freemem is 0. 2903 * 2904 * As it turns out, boot_getpages() does this. It picks a page, 2905 * based on where OBP mapped in some address, gets its pfn, searches 2906 * the memsegs, locks the page, then pulls it off the free list! 2907 */ 2908 int 2909 page_reclaim(page_t *pp, kmutex_t *lock) 2910 { 2911 struct pcf *p; 2912 struct cpu *cpup; 2913 int enough; 2914 uint_t i; 2915 2916 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2917 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2918 2919 /* 2920 * If `freemem' is 0, we cannot reclaim this page from the 2921 * freelist, so release every lock we might hold: the page, 2922 * and the `lock' before blocking. 2923 * 2924 * The only way `freemem' can become 0 while there are pages 2925 * marked free (have their p->p_free bit set) is when the 2926 * system is low on memory and doing a page_create(). In 2927 * order to guarantee that once page_create() starts acquiring 2928 * pages it will be able to get all that it needs since `freemem' 2929 * was decreased by the requested amount. So, we need to release 2930 * this page, and let page_create() have it. 2931 * 2932 * Since `freemem' being zero is not supposed to happen, just 2933 * use the usual hash stuff as a starting point. If that bucket 2934 * is empty, then assume the worst, and start at the beginning 2935 * of the pcf array. If we always start at the beginning 2936 * when acquiring more than one pcf lock, there won't be any 2937 * deadlock problems. 2938 */ 2939 2940 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 2941 2942 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) { 2943 pcf_acquire_all(); 2944 goto page_reclaim_nomem; 2945 } 2946 2947 enough = pcf_decrement_bucket(1); 2948 2949 if (!enough) { 2950 VM_STAT_ADD(page_reclaim_zero); 2951 /* 2952 * Check again. Its possible that some other thread 2953 * could have been right behind us, and added one 2954 * to a list somewhere. Acquire each of the pcf locks 2955 * until we find a page. 2956 */ 2957 p = pcf; 2958 for (i = 0; i < pcf_fanout; i++) { 2959 mutex_enter(&p->pcf_lock); 2960 if (p->pcf_count >= 1) { 2961 p->pcf_count -= 1; 2962 /* 2963 * freemem is not protected by any lock. Thus, 2964 * we cannot have any assertion containing 2965 * freemem here. 2966 */ 2967 freemem -= 1; 2968 enough = 1; 2969 break; 2970 } 2971 p++; 2972 } 2973 2974 if (!enough) { 2975 page_reclaim_nomem: 2976 /* 2977 * We really can't have page `pp'. 2978 * Time for the no-memory dance with 2979 * page_free(). This is just like 2980 * page_create_wait(). Plus the added 2981 * attraction of releasing whatever mutex 2982 * we held when we were called with in `lock'. 2983 * Page_unlock() will wakeup any thread 2984 * waiting around for this page. 2985 */ 2986 if (lock) { 2987 VM_STAT_ADD(page_reclaim_zero_locked); 2988 mutex_exit(lock); 2989 } 2990 page_unlock(pp); 2991 2992 /* 2993 * get this before we drop all the pcf locks. 2994 */ 2995 mutex_enter(&new_freemem_lock); 2996 2997 p = pcf; 2998 for (i = 0; i < pcf_fanout; i++) { 2999 p->pcf_wait++; 3000 mutex_exit(&p->pcf_lock); 3001 p++; 3002 } 3003 3004 freemem_wait++; 3005 cv_wait(&freemem_cv, &new_freemem_lock); 3006 freemem_wait--; 3007 3008 mutex_exit(&new_freemem_lock); 3009 3010 if (lock) { 3011 mutex_enter(lock); 3012 } 3013 return (0); 3014 } 3015 3016 /* 3017 * The pcf accounting has been done, 3018 * though none of the pcf_wait flags have been set, 3019 * drop the locks and continue on. 3020 */ 3021 while (p >= pcf) { 3022 mutex_exit(&p->pcf_lock); 3023 p--; 3024 } 3025 } 3026 3027 3028 VM_STAT_ADD(pagecnt.pc_reclaim); 3029 3030 /* 3031 * page_list_sub will handle the case where pp is a large page. 3032 * It's possible that the page was promoted while on the freelist 3033 */ 3034 if (PP_ISAGED(pp)) { 3035 page_list_sub(pp, PG_FREE_LIST); 3036 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3037 "page_reclaim_free:pp %p", pp); 3038 } else { 3039 page_list_sub(pp, PG_CACHE_LIST); 3040 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3041 "page_reclaim_cache:pp %p", pp); 3042 } 3043 3044 /* 3045 * clear the p_free & p_age bits since this page is no longer 3046 * on the free list. Notice that there was a brief time where 3047 * a page is marked as free, but is not on the list. 3048 * 3049 * Set the reference bit to protect against immediate pageout. 3050 */ 3051 PP_CLRFREE(pp); 3052 PP_CLRAGED(pp); 3053 page_set_props(pp, P_REF); 3054 3055 CPU_STATS_ENTER_K(); 3056 cpup = CPU; /* get cpup now that CPU cannot change */ 3057 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3058 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3059 CPU_STATS_EXIT_K(); 3060 ASSERT(pp->p_szc == 0); 3061 3062 return (1); 3063 } 3064 3065 /* 3066 * Destroy identity of the page and put it back on 3067 * the page free list. Assumes that the caller has 3068 * acquired the "exclusive" lock on the page. 3069 */ 3070 void 3071 page_destroy(page_t *pp, int dontfree) 3072 { 3073 ASSERT((PAGE_EXCL(pp) && 3074 !page_iolock_assert(pp)) || panicstr); 3075 ASSERT(pp->p_slckcnt == 0 || panicstr); 3076 3077 if (pp->p_szc != 0) { 3078 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3079 PP_ISKAS(pp)) { 3080 panic("page_destroy: anon or kernel or no vnode " 3081 "large page %p", (void *)pp); 3082 } 3083 page_demote_vp_pages(pp); 3084 ASSERT(pp->p_szc == 0); 3085 } 3086 3087 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3088 3089 /* 3090 * Unload translations, if any, then hash out the 3091 * page to erase its identity. 3092 */ 3093 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3094 page_hashout(pp, NULL); 3095 3096 if (!dontfree) { 3097 /* 3098 * Acquire the "freemem_lock" for availrmem. 3099 * The page_struct_lock need not be acquired for lckcnt 3100 * and cowcnt since the page has an "exclusive" lock. 3101 * We are doing a modified version of page_pp_unlock here. 3102 */ 3103 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3104 mutex_enter(&freemem_lock); 3105 if (pp->p_lckcnt != 0) { 3106 availrmem++; 3107 pages_locked--; 3108 pp->p_lckcnt = 0; 3109 } 3110 if (pp->p_cowcnt != 0) { 3111 availrmem += pp->p_cowcnt; 3112 pages_locked -= pp->p_cowcnt; 3113 pp->p_cowcnt = 0; 3114 } 3115 mutex_exit(&freemem_lock); 3116 } 3117 /* 3118 * Put the page on the "free" list. 3119 */ 3120 page_free(pp, 0); 3121 } 3122 } 3123 3124 void 3125 page_destroy_pages(page_t *pp) 3126 { 3127 3128 page_t *tpp, *rootpp = NULL; 3129 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3130 pgcnt_t i, pglcks = 0; 3131 uint_t szc = pp->p_szc; 3132 3133 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3134 3135 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3136 3137 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3138 3139 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3140 panic("page_destroy_pages: not root page %p", (void *)pp); 3141 /*NOTREACHED*/ 3142 } 3143 3144 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3145 ASSERT((PAGE_EXCL(tpp) && 3146 !page_iolock_assert(tpp)) || panicstr); 3147 ASSERT(tpp->p_slckcnt == 0 || panicstr); 3148 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3149 page_hashout(tpp, NULL); 3150 ASSERT(tpp->p_offset == (u_offset_t)-1); 3151 if (tpp->p_lckcnt != 0) { 3152 pglcks++; 3153 tpp->p_lckcnt = 0; 3154 } else if (tpp->p_cowcnt != 0) { 3155 pglcks += tpp->p_cowcnt; 3156 tpp->p_cowcnt = 0; 3157 } 3158 ASSERT(!hat_page_getshare(tpp)); 3159 ASSERT(tpp->p_vnode == NULL); 3160 ASSERT(tpp->p_szc == szc); 3161 3162 PP_SETFREE(tpp); 3163 page_clr_all_props(tpp); 3164 PP_SETAGED(tpp); 3165 ASSERT(tpp->p_next == tpp); 3166 ASSERT(tpp->p_prev == tpp); 3167 page_list_concat(&rootpp, &tpp); 3168 } 3169 3170 ASSERT(rootpp == pp); 3171 if (pglcks != 0) { 3172 mutex_enter(&freemem_lock); 3173 availrmem += pglcks; 3174 mutex_exit(&freemem_lock); 3175 } 3176 3177 page_list_add_pages(rootpp, 0); 3178 page_create_putback(pgcnt); 3179 } 3180 3181 /* 3182 * Similar to page_destroy(), but destroys pages which are 3183 * locked and known to be on the page free list. Since 3184 * the page is known to be free and locked, no one can access 3185 * it. 3186 * 3187 * Also, the number of free pages does not change. 3188 */ 3189 void 3190 page_destroy_free(page_t *pp) 3191 { 3192 ASSERT(PAGE_EXCL(pp)); 3193 ASSERT(PP_ISFREE(pp)); 3194 ASSERT(pp->p_vnode); 3195 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3196 ASSERT(!hat_page_is_mapped(pp)); 3197 ASSERT(PP_ISAGED(pp) == 0); 3198 ASSERT(pp->p_szc == 0); 3199 3200 VM_STAT_ADD(pagecnt.pc_destroy_free); 3201 page_list_sub(pp, PG_CACHE_LIST); 3202 3203 page_hashout(pp, NULL); 3204 ASSERT(pp->p_vnode == NULL); 3205 ASSERT(pp->p_offset == (u_offset_t)-1); 3206 ASSERT(pp->p_hash == NULL); 3207 3208 PP_SETAGED(pp); 3209 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3210 page_unlock(pp); 3211 3212 mutex_enter(&new_freemem_lock); 3213 if (freemem_wait) { 3214 cv_signal(&freemem_cv); 3215 } 3216 mutex_exit(&new_freemem_lock); 3217 } 3218 3219 /* 3220 * Rename the page "opp" to have an identity specified 3221 * by [vp, off]. If a page already exists with this name 3222 * it is locked and destroyed. Note that the page's 3223 * translations are not unloaded during the rename. 3224 * 3225 * This routine is used by the anon layer to "steal" the 3226 * original page and is not unlike destroying a page and 3227 * creating a new page using the same page frame. 3228 * 3229 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3230 * caller 2 tries to rename B to A. 3231 */ 3232 void 3233 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3234 { 3235 page_t *pp; 3236 int olckcnt = 0; 3237 int ocowcnt = 0; 3238 kmutex_t *phm; 3239 ulong_t index; 3240 3241 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3242 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3243 ASSERT(PP_ISFREE(opp) == 0); 3244 3245 VM_STAT_ADD(page_rename_count); 3246 3247 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3248 "page rename:pp %p vp %p off %llx", opp, vp, off); 3249 3250 /* 3251 * CacheFS may call page_rename for a large NFS page 3252 * when both CacheFS and NFS mount points are used 3253 * by applications. Demote this large page before 3254 * renaming it, to ensure that there are no "partial" 3255 * large pages left lying around. 3256 */ 3257 if (opp->p_szc != 0) { 3258 vnode_t *ovp = opp->p_vnode; 3259 ASSERT(ovp != NULL); 3260 ASSERT(!IS_SWAPFSVP(ovp)); 3261 ASSERT(!VN_ISKAS(ovp)); 3262 page_demote_vp_pages(opp); 3263 ASSERT(opp->p_szc == 0); 3264 } 3265 3266 page_hashout(opp, NULL); 3267 PP_CLRAGED(opp); 3268 3269 /* 3270 * Acquire the appropriate page hash lock, since 3271 * we're going to rename the page. 3272 */ 3273 index = PAGE_HASH_FUNC(vp, off); 3274 phm = PAGE_HASH_MUTEX(index); 3275 mutex_enter(phm); 3276 top: 3277 /* 3278 * Look for an existing page with this name and destroy it if found. 3279 * By holding the page hash lock all the way to the page_hashin() 3280 * call, we are assured that no page can be created with this 3281 * identity. In the case when the phm lock is dropped to undo any 3282 * hat layer mappings, the existing page is held with an "exclusive" 3283 * lock, again preventing another page from being created with 3284 * this identity. 3285 */ 3286 pp = page_hash_search(index, vp, off); 3287 if (pp != NULL) { 3288 VM_STAT_ADD(page_rename_exists); 3289 3290 /* 3291 * As it turns out, this is one of only two places where 3292 * page_lock() needs to hold the passed in lock in the 3293 * successful case. In all of the others, the lock could 3294 * be dropped as soon as the attempt is made to lock 3295 * the page. It is tempting to add yet another arguement, 3296 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3297 */ 3298 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3299 /* 3300 * Went to sleep because the page could not 3301 * be locked. We were woken up when the page 3302 * was unlocked, or when the page was destroyed. 3303 * In either case, `phm' was dropped while we 3304 * slept. Hence we should not just roar through 3305 * this loop. 3306 */ 3307 goto top; 3308 } 3309 3310 /* 3311 * If an existing page is a large page, then demote 3312 * it to ensure that no "partial" large pages are 3313 * "created" after page_rename. An existing page 3314 * can be a CacheFS page, and can't belong to swapfs. 3315 */ 3316 if (hat_page_is_mapped(pp)) { 3317 /* 3318 * Unload translations. Since we hold the 3319 * exclusive lock on this page, the page 3320 * can not be changed while we drop phm. 3321 * This is also not a lock protocol violation, 3322 * but rather the proper way to do things. 3323 */ 3324 mutex_exit(phm); 3325 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3326 if (pp->p_szc != 0) { 3327 ASSERT(!IS_SWAPFSVP(vp)); 3328 ASSERT(!VN_ISKAS(vp)); 3329 page_demote_vp_pages(pp); 3330 ASSERT(pp->p_szc == 0); 3331 } 3332 mutex_enter(phm); 3333 } else if (pp->p_szc != 0) { 3334 ASSERT(!IS_SWAPFSVP(vp)); 3335 ASSERT(!VN_ISKAS(vp)); 3336 mutex_exit(phm); 3337 page_demote_vp_pages(pp); 3338 ASSERT(pp->p_szc == 0); 3339 mutex_enter(phm); 3340 } 3341 page_hashout(pp, phm); 3342 } 3343 /* 3344 * Hash in the page with the new identity. 3345 */ 3346 if (!page_hashin(opp, vp, off, phm)) { 3347 /* 3348 * We were holding phm while we searched for [vp, off] 3349 * and only dropped phm if we found and locked a page. 3350 * If we can't create this page now, then some thing 3351 * is really broken. 3352 */ 3353 panic("page_rename: Can't hash in page: %p", (void *)pp); 3354 /*NOTREACHED*/ 3355 } 3356 3357 ASSERT(MUTEX_HELD(phm)); 3358 mutex_exit(phm); 3359 3360 /* 3361 * Now that we have dropped phm, lets get around to finishing up 3362 * with pp. 3363 */ 3364 if (pp != NULL) { 3365 ASSERT(!hat_page_is_mapped(pp)); 3366 /* for now large pages should not end up here */ 3367 ASSERT(pp->p_szc == 0); 3368 /* 3369 * Save the locks for transfer to the new page and then 3370 * clear them so page_free doesn't think they're important. 3371 * The page_struct_lock need not be acquired for lckcnt and 3372 * cowcnt since the page has an "exclusive" lock. 3373 */ 3374 olckcnt = pp->p_lckcnt; 3375 ocowcnt = pp->p_cowcnt; 3376 pp->p_lckcnt = pp->p_cowcnt = 0; 3377 3378 /* 3379 * Put the page on the "free" list after we drop 3380 * the lock. The less work under the lock the better. 3381 */ 3382 /*LINTED: constant in conditional context*/ 3383 VN_DISPOSE(pp, B_FREE, 0, kcred); 3384 } 3385 3386 /* 3387 * Transfer the lock count from the old page (if any). 3388 * The page_struct_lock need not be acquired for lckcnt and 3389 * cowcnt since the page has an "exclusive" lock. 3390 */ 3391 opp->p_lckcnt += olckcnt; 3392 opp->p_cowcnt += ocowcnt; 3393 } 3394 3395 /* 3396 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3397 * 3398 * Pages are normally inserted at the start of a vnode's v_pages list. 3399 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3400 * This can happen when a modified page is relocated for DR. 3401 * 3402 * Returns 1 on success and 0 on failure. 3403 */ 3404 static int 3405 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3406 { 3407 page_t **listp; 3408 page_t *tp; 3409 ulong_t index; 3410 3411 ASSERT(PAGE_EXCL(pp)); 3412 ASSERT(vp != NULL); 3413 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3414 3415 /* 3416 * Be sure to set these up before the page is inserted on the hash 3417 * list. As soon as the page is placed on the list some other 3418 * thread might get confused and wonder how this page could 3419 * possibly hash to this list. 3420 */ 3421 pp->p_vnode = vp; 3422 pp->p_offset = offset; 3423 3424 /* 3425 * record if this page is on a swap vnode 3426 */ 3427 if ((vp->v_flag & VISSWAP) != 0) 3428 PP_SETSWAP(pp); 3429 3430 index = PAGE_HASH_FUNC(vp, offset); 3431 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3432 listp = &page_hash[index]; 3433 3434 /* 3435 * If this page is already hashed in, fail this attempt to add it. 3436 */ 3437 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3438 if (tp->p_vnode == vp && tp->p_offset == offset) { 3439 pp->p_vnode = NULL; 3440 pp->p_offset = (u_offset_t)(-1); 3441 return (0); 3442 } 3443 } 3444 pp->p_hash = *listp; 3445 *listp = pp; 3446 3447 /* 3448 * Add the page to the vnode's list of pages 3449 */ 3450 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3451 listp = &vp->v_pages->p_vpprev->p_vpnext; 3452 else 3453 listp = &vp->v_pages; 3454 3455 page_vpadd(listp, pp); 3456 3457 return (1); 3458 } 3459 3460 /* 3461 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3462 * 3463 * Returns 1 on success and 0 on failure. 3464 * If hold is passed in, it is not dropped. 3465 */ 3466 int 3467 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3468 { 3469 kmutex_t *phm = NULL; 3470 kmutex_t *vphm; 3471 int rc; 3472 3473 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3474 ASSERT(pp->p_fsdata == 0 || panicstr); 3475 3476 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3477 "page_hashin:pp %p vp %p offset %llx", 3478 pp, vp, offset); 3479 3480 VM_STAT_ADD(hashin_count); 3481 3482 if (hold != NULL) 3483 phm = hold; 3484 else { 3485 VM_STAT_ADD(hashin_not_held); 3486 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3487 mutex_enter(phm); 3488 } 3489 3490 vphm = page_vnode_mutex(vp); 3491 mutex_enter(vphm); 3492 rc = page_do_hashin(pp, vp, offset); 3493 mutex_exit(vphm); 3494 if (hold == NULL) 3495 mutex_exit(phm); 3496 if (rc == 0) 3497 VM_STAT_ADD(hashin_already); 3498 return (rc); 3499 } 3500 3501 /* 3502 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3503 * All mutexes must be held 3504 */ 3505 static void 3506 page_do_hashout(page_t *pp) 3507 { 3508 page_t **hpp; 3509 page_t *hp; 3510 vnode_t *vp = pp->p_vnode; 3511 3512 ASSERT(vp != NULL); 3513 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3514 3515 /* 3516 * First, take pp off of its hash chain. 3517 */ 3518 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3519 3520 for (;;) { 3521 hp = *hpp; 3522 if (hp == pp) 3523 break; 3524 if (hp == NULL) { 3525 panic("page_do_hashout"); 3526 /*NOTREACHED*/ 3527 } 3528 hpp = &hp->p_hash; 3529 } 3530 *hpp = pp->p_hash; 3531 3532 /* 3533 * Now remove it from its associated vnode. 3534 */ 3535 if (vp->v_pages) 3536 page_vpsub(&vp->v_pages, pp); 3537 3538 pp->p_hash = NULL; 3539 page_clr_all_props(pp); 3540 PP_CLRSWAP(pp); 3541 pp->p_vnode = NULL; 3542 pp->p_offset = (u_offset_t)-1; 3543 pp->p_fsdata = 0; 3544 } 3545 3546 /* 3547 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3548 * 3549 * When `phm' is non-NULL it contains the address of the mutex protecting the 3550 * hash list pp is on. It is not dropped. 3551 */ 3552 void 3553 page_hashout(page_t *pp, kmutex_t *phm) 3554 { 3555 vnode_t *vp; 3556 ulong_t index; 3557 kmutex_t *nphm; 3558 kmutex_t *vphm; 3559 kmutex_t *sep; 3560 3561 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3562 ASSERT(pp->p_vnode != NULL); 3563 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3564 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3565 3566 vp = pp->p_vnode; 3567 3568 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3569 "page_hashout:pp %p vp %p", pp, vp); 3570 3571 /* Kernel probe */ 3572 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3573 tnf_opaque, vnode, vp, 3574 tnf_offset, offset, pp->p_offset); 3575 3576 /* 3577 * 3578 */ 3579 VM_STAT_ADD(hashout_count); 3580 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3581 if (phm == NULL) { 3582 VM_STAT_ADD(hashout_not_held); 3583 nphm = PAGE_HASH_MUTEX(index); 3584 mutex_enter(nphm); 3585 } 3586 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3587 3588 3589 /* 3590 * grab page vnode mutex and remove it... 3591 */ 3592 vphm = page_vnode_mutex(vp); 3593 mutex_enter(vphm); 3594 3595 page_do_hashout(pp); 3596 3597 mutex_exit(vphm); 3598 if (phm == NULL) 3599 mutex_exit(nphm); 3600 3601 /* 3602 * Wake up processes waiting for this page. The page's 3603 * identity has been changed, and is probably not the 3604 * desired page any longer. 3605 */ 3606 sep = page_se_mutex(pp); 3607 mutex_enter(sep); 3608 pp->p_selock &= ~SE_EWANTED; 3609 if (CV_HAS_WAITERS(&pp->p_cv)) 3610 cv_broadcast(&pp->p_cv); 3611 mutex_exit(sep); 3612 } 3613 3614 /* 3615 * Add the page to the front of a linked list of pages 3616 * using the p_next & p_prev pointers for the list. 3617 * The caller is responsible for protecting the list pointers. 3618 */ 3619 void 3620 page_add(page_t **ppp, page_t *pp) 3621 { 3622 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3623 3624 page_add_common(ppp, pp); 3625 } 3626 3627 3628 3629 /* 3630 * Common code for page_add() and mach_page_add() 3631 */ 3632 void 3633 page_add_common(page_t **ppp, page_t *pp) 3634 { 3635 if (*ppp == NULL) { 3636 pp->p_next = pp->p_prev = pp; 3637 } else { 3638 pp->p_next = *ppp; 3639 pp->p_prev = (*ppp)->p_prev; 3640 (*ppp)->p_prev = pp; 3641 pp->p_prev->p_next = pp; 3642 } 3643 *ppp = pp; 3644 } 3645 3646 3647 /* 3648 * Remove this page from a linked list of pages 3649 * using the p_next & p_prev pointers for the list. 3650 * 3651 * The caller is responsible for protecting the list pointers. 3652 */ 3653 void 3654 page_sub(page_t **ppp, page_t *pp) 3655 { 3656 ASSERT((PP_ISFREE(pp)) ? 1 : 3657 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3658 3659 if (*ppp == NULL || pp == NULL) { 3660 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3661 (void *)pp, (void *)(*ppp)); 3662 /*NOTREACHED*/ 3663 } 3664 3665 page_sub_common(ppp, pp); 3666 } 3667 3668 3669 /* 3670 * Common code for page_sub() and mach_page_sub() 3671 */ 3672 void 3673 page_sub_common(page_t **ppp, page_t *pp) 3674 { 3675 if (*ppp == pp) 3676 *ppp = pp->p_next; /* go to next page */ 3677 3678 if (*ppp == pp) 3679 *ppp = NULL; /* page list is gone */ 3680 else { 3681 pp->p_prev->p_next = pp->p_next; 3682 pp->p_next->p_prev = pp->p_prev; 3683 } 3684 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3685 } 3686 3687 3688 /* 3689 * Break page list cppp into two lists with npages in the first list. 3690 * The tail is returned in nppp. 3691 */ 3692 void 3693 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3694 { 3695 page_t *s1pp = *oppp; 3696 page_t *s2pp; 3697 page_t *e1pp, *e2pp; 3698 long n = 0; 3699 3700 if (s1pp == NULL) { 3701 *nppp = NULL; 3702 return; 3703 } 3704 if (npages == 0) { 3705 *nppp = s1pp; 3706 *oppp = NULL; 3707 return; 3708 } 3709 for (n = 0, s2pp = *oppp; n < npages; n++) { 3710 s2pp = s2pp->p_next; 3711 } 3712 /* Fix head and tail of new lists */ 3713 e1pp = s2pp->p_prev; 3714 e2pp = s1pp->p_prev; 3715 s1pp->p_prev = e1pp; 3716 e1pp->p_next = s1pp; 3717 s2pp->p_prev = e2pp; 3718 e2pp->p_next = s2pp; 3719 3720 /* second list empty */ 3721 if (s2pp == s1pp) { 3722 *oppp = s1pp; 3723 *nppp = NULL; 3724 } else { 3725 *oppp = s1pp; 3726 *nppp = s2pp; 3727 } 3728 } 3729 3730 /* 3731 * Concatenate page list nppp onto the end of list ppp. 3732 */ 3733 void 3734 page_list_concat(page_t **ppp, page_t **nppp) 3735 { 3736 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3737 3738 if (*nppp == NULL) { 3739 return; 3740 } 3741 if (*ppp == NULL) { 3742 *ppp = *nppp; 3743 return; 3744 } 3745 s1pp = *ppp; 3746 e1pp = s1pp->p_prev; 3747 s2pp = *nppp; 3748 e2pp = s2pp->p_prev; 3749 s1pp->p_prev = e2pp; 3750 e2pp->p_next = s1pp; 3751 e1pp->p_next = s2pp; 3752 s2pp->p_prev = e1pp; 3753 } 3754 3755 /* 3756 * return the next page in the page list 3757 */ 3758 page_t * 3759 page_list_next(page_t *pp) 3760 { 3761 return (pp->p_next); 3762 } 3763 3764 3765 /* 3766 * Add the page to the front of the linked list of pages 3767 * using p_vpnext/p_vpprev pointers for the list. 3768 * 3769 * The caller is responsible for protecting the lists. 3770 */ 3771 void 3772 page_vpadd(page_t **ppp, page_t *pp) 3773 { 3774 if (*ppp == NULL) { 3775 pp->p_vpnext = pp->p_vpprev = pp; 3776 } else { 3777 pp->p_vpnext = *ppp; 3778 pp->p_vpprev = (*ppp)->p_vpprev; 3779 (*ppp)->p_vpprev = pp; 3780 pp->p_vpprev->p_vpnext = pp; 3781 } 3782 *ppp = pp; 3783 } 3784 3785 /* 3786 * Remove this page from the linked list of pages 3787 * using p_vpnext/p_vpprev pointers for the list. 3788 * 3789 * The caller is responsible for protecting the lists. 3790 */ 3791 void 3792 page_vpsub(page_t **ppp, page_t *pp) 3793 { 3794 if (*ppp == NULL || pp == NULL) { 3795 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3796 (void *)pp, (void *)(*ppp)); 3797 /*NOTREACHED*/ 3798 } 3799 3800 if (*ppp == pp) 3801 *ppp = pp->p_vpnext; /* go to next page */ 3802 3803 if (*ppp == pp) 3804 *ppp = NULL; /* page list is gone */ 3805 else { 3806 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3807 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3808 } 3809 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3810 } 3811 3812 /* 3813 * Lock a physical page into memory "long term". Used to support "lock 3814 * in memory" functions. Accepts the page to be locked, and a cow variable 3815 * to indicate whether a the lock will travel to the new page during 3816 * a potential copy-on-write. 3817 */ 3818 int 3819 page_pp_lock( 3820 page_t *pp, /* page to be locked */ 3821 int cow, /* cow lock */ 3822 int kernel) /* must succeed -- ignore checking */ 3823 { 3824 int r = 0; /* result -- assume failure */ 3825 3826 ASSERT(PAGE_LOCKED(pp)); 3827 3828 page_struct_lock(pp); 3829 /* 3830 * Acquire the "freemem_lock" for availrmem. 3831 */ 3832 if (cow) { 3833 mutex_enter(&freemem_lock); 3834 if ((availrmem > pages_pp_maximum) && 3835 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3836 availrmem--; 3837 pages_locked++; 3838 mutex_exit(&freemem_lock); 3839 r = 1; 3840 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3841 cmn_err(CE_WARN, 3842 "COW lock limit reached on pfn 0x%lx", 3843 page_pptonum(pp)); 3844 } 3845 } else 3846 mutex_exit(&freemem_lock); 3847 } else { 3848 if (pp->p_lckcnt) { 3849 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3850 r = 1; 3851 if (++pp->p_lckcnt == 3852 (ushort_t)PAGE_LOCK_MAXIMUM) { 3853 cmn_err(CE_WARN, "Page lock limit " 3854 "reached on pfn 0x%lx", 3855 page_pptonum(pp)); 3856 } 3857 } 3858 } else { 3859 if (kernel) { 3860 /* availrmem accounting done by caller */ 3861 ++pp->p_lckcnt; 3862 r = 1; 3863 } else { 3864 mutex_enter(&freemem_lock); 3865 if (availrmem > pages_pp_maximum) { 3866 availrmem--; 3867 pages_locked++; 3868 ++pp->p_lckcnt; 3869 r = 1; 3870 } 3871 mutex_exit(&freemem_lock); 3872 } 3873 } 3874 } 3875 page_struct_unlock(pp); 3876 return (r); 3877 } 3878 3879 /* 3880 * Decommit a lock on a physical page frame. Account for cow locks if 3881 * appropriate. 3882 */ 3883 void 3884 page_pp_unlock( 3885 page_t *pp, /* page to be unlocked */ 3886 int cow, /* expect cow lock */ 3887 int kernel) /* this was a kernel lock */ 3888 { 3889 ASSERT(PAGE_LOCKED(pp)); 3890 3891 page_struct_lock(pp); 3892 /* 3893 * Acquire the "freemem_lock" for availrmem. 3894 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3895 * could be called to unlock even if nothing is locked. This could 3896 * happen if locked file pages were truncated (removing the lock) 3897 * and the file was grown again and new pages faulted in; the new 3898 * pages are unlocked but the segment still thinks they're locked. 3899 */ 3900 if (cow) { 3901 if (pp->p_cowcnt) { 3902 mutex_enter(&freemem_lock); 3903 pp->p_cowcnt--; 3904 availrmem++; 3905 pages_locked--; 3906 mutex_exit(&freemem_lock); 3907 } 3908 } else { 3909 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3910 if (!kernel) { 3911 mutex_enter(&freemem_lock); 3912 availrmem++; 3913 pages_locked--; 3914 mutex_exit(&freemem_lock); 3915 } 3916 } 3917 } 3918 page_struct_unlock(pp); 3919 } 3920 3921 /* 3922 * This routine reserves availrmem for npages; 3923 * flags: KM_NOSLEEP or KM_SLEEP 3924 * returns 1 on success or 0 on failure 3925 */ 3926 int 3927 page_resv(pgcnt_t npages, uint_t flags) 3928 { 3929 mutex_enter(&freemem_lock); 3930 while (availrmem < tune.t_minarmem + npages) { 3931 if (flags & KM_NOSLEEP) { 3932 mutex_exit(&freemem_lock); 3933 return (0); 3934 } 3935 mutex_exit(&freemem_lock); 3936 page_needfree(npages); 3937 kmem_reap(); 3938 delay(hz >> 2); 3939 page_needfree(-(spgcnt_t)npages); 3940 mutex_enter(&freemem_lock); 3941 } 3942 availrmem -= npages; 3943 mutex_exit(&freemem_lock); 3944 return (1); 3945 } 3946 3947 /* 3948 * This routine unreserves availrmem for npages; 3949 */ 3950 void 3951 page_unresv(pgcnt_t npages) 3952 { 3953 mutex_enter(&freemem_lock); 3954 availrmem += npages; 3955 mutex_exit(&freemem_lock); 3956 } 3957 3958 /* 3959 * See Statement at the beginning of segvn_lockop() regarding 3960 * the way we handle cowcnts and lckcnts. 3961 * 3962 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 3963 * that breaks COW has PROT_WRITE. 3964 * 3965 * Note that, we may also break COW in case we are softlocking 3966 * on read access during physio; 3967 * in this softlock case, the vpage may not have PROT_WRITE. 3968 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 3969 * if the vpage doesn't have PROT_WRITE. 3970 * 3971 * This routine is never called if we are stealing a page 3972 * in anon_private. 3973 * 3974 * The caller subtracted from availrmem for read only mapping. 3975 * if lckcnt is 1 increment availrmem. 3976 */ 3977 void 3978 page_pp_useclaim( 3979 page_t *opp, /* original page frame losing lock */ 3980 page_t *npp, /* new page frame gaining lock */ 3981 uint_t write_perm) /* set if vpage has PROT_WRITE */ 3982 { 3983 int payback = 0; 3984 int nidx, oidx; 3985 3986 ASSERT(PAGE_LOCKED(opp)); 3987 ASSERT(PAGE_LOCKED(npp)); 3988 3989 /* 3990 * Since we have two pages we probably have two locks. We need to take 3991 * them in a defined order to avoid deadlocks. It's also possible they 3992 * both hash to the same lock in which case this is a non-issue. 3993 */ 3994 nidx = PAGE_LLOCK_HASH(PP_PAGEROOT(npp)); 3995 oidx = PAGE_LLOCK_HASH(PP_PAGEROOT(opp)); 3996 if (nidx < oidx) { 3997 page_struct_lock(npp); 3998 page_struct_lock(opp); 3999 } else if (oidx < nidx) { 4000 page_struct_lock(opp); 4001 page_struct_lock(npp); 4002 } else { /* The pages hash to the same lock */ 4003 page_struct_lock(npp); 4004 } 4005 4006 ASSERT(npp->p_cowcnt == 0); 4007 ASSERT(npp->p_lckcnt == 0); 4008 4009 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 4010 if ((write_perm && opp->p_cowcnt != 0) || 4011 (!write_perm && opp->p_lckcnt != 0)) { 4012 4013 if (write_perm) { 4014 npp->p_cowcnt++; 4015 ASSERT(opp->p_cowcnt != 0); 4016 opp->p_cowcnt--; 4017 } else { 4018 4019 ASSERT(opp->p_lckcnt != 0); 4020 4021 /* 4022 * We didn't need availrmem decremented if p_lckcnt on 4023 * original page is 1. Here, we are unlocking 4024 * read-only copy belonging to original page and 4025 * are locking a copy belonging to new page. 4026 */ 4027 if (opp->p_lckcnt == 1) 4028 payback = 1; 4029 4030 npp->p_lckcnt++; 4031 opp->p_lckcnt--; 4032 } 4033 } 4034 if (payback) { 4035 mutex_enter(&freemem_lock); 4036 availrmem++; 4037 pages_useclaim--; 4038 mutex_exit(&freemem_lock); 4039 } 4040 4041 if (nidx < oidx) { 4042 page_struct_unlock(opp); 4043 page_struct_unlock(npp); 4044 } else if (oidx < nidx) { 4045 page_struct_unlock(npp); 4046 page_struct_unlock(opp); 4047 } else { /* The pages hash to the same lock */ 4048 page_struct_unlock(npp); 4049 } 4050 } 4051 4052 /* 4053 * Simple claim adjust functions -- used to support changes in 4054 * claims due to changes in access permissions. Used by segvn_setprot(). 4055 */ 4056 int 4057 page_addclaim(page_t *pp) 4058 { 4059 int r = 0; /* result */ 4060 4061 ASSERT(PAGE_LOCKED(pp)); 4062 4063 page_struct_lock(pp); 4064 ASSERT(pp->p_lckcnt != 0); 4065 4066 if (pp->p_lckcnt == 1) { 4067 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4068 --pp->p_lckcnt; 4069 r = 1; 4070 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4071 cmn_err(CE_WARN, 4072 "COW lock limit reached on pfn 0x%lx", 4073 page_pptonum(pp)); 4074 } 4075 } 4076 } else { 4077 mutex_enter(&freemem_lock); 4078 if ((availrmem > pages_pp_maximum) && 4079 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4080 --availrmem; 4081 ++pages_claimed; 4082 mutex_exit(&freemem_lock); 4083 --pp->p_lckcnt; 4084 r = 1; 4085 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4086 cmn_err(CE_WARN, 4087 "COW lock limit reached on pfn 0x%lx", 4088 page_pptonum(pp)); 4089 } 4090 } else 4091 mutex_exit(&freemem_lock); 4092 } 4093 page_struct_unlock(pp); 4094 return (r); 4095 } 4096 4097 int 4098 page_subclaim(page_t *pp) 4099 { 4100 int r = 0; 4101 4102 ASSERT(PAGE_LOCKED(pp)); 4103 4104 page_struct_lock(pp); 4105 ASSERT(pp->p_cowcnt != 0); 4106 4107 if (pp->p_lckcnt) { 4108 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4109 r = 1; 4110 /* 4111 * for availrmem 4112 */ 4113 mutex_enter(&freemem_lock); 4114 availrmem++; 4115 pages_claimed--; 4116 mutex_exit(&freemem_lock); 4117 4118 pp->p_cowcnt--; 4119 4120 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4121 cmn_err(CE_WARN, 4122 "Page lock limit reached on pfn 0x%lx", 4123 page_pptonum(pp)); 4124 } 4125 } 4126 } else { 4127 r = 1; 4128 pp->p_cowcnt--; 4129 pp->p_lckcnt++; 4130 } 4131 page_struct_unlock(pp); 4132 return (r); 4133 } 4134 4135 /* 4136 * Variant of page_addclaim(), where ppa[] contains the pages of a single large 4137 * page. 4138 */ 4139 int 4140 page_addclaim_pages(page_t **ppa) 4141 { 4142 pgcnt_t lckpgs = 0, pg_idx; 4143 4144 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4145 4146 /* 4147 * Only need to take the page struct lock on the large page root. 4148 */ 4149 page_struct_lock(ppa[0]); 4150 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4151 4152 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4153 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4154 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4155 page_struct_unlock(ppa[0]); 4156 return (0); 4157 } 4158 if (ppa[pg_idx]->p_lckcnt > 1) 4159 lckpgs++; 4160 } 4161 4162 if (lckpgs != 0) { 4163 mutex_enter(&freemem_lock); 4164 if (availrmem >= pages_pp_maximum + lckpgs) { 4165 availrmem -= lckpgs; 4166 pages_claimed += lckpgs; 4167 } else { 4168 mutex_exit(&freemem_lock); 4169 page_struct_unlock(ppa[0]); 4170 return (0); 4171 } 4172 mutex_exit(&freemem_lock); 4173 } 4174 4175 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4176 ppa[pg_idx]->p_lckcnt--; 4177 ppa[pg_idx]->p_cowcnt++; 4178 } 4179 page_struct_unlock(ppa[0]); 4180 return (1); 4181 } 4182 4183 /* 4184 * Variant of page_subclaim(), where ppa[] contains the pages of a single large 4185 * page. 4186 */ 4187 int 4188 page_subclaim_pages(page_t **ppa) 4189 { 4190 pgcnt_t ulckpgs = 0, pg_idx; 4191 4192 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4193 4194 /* 4195 * Only need to take the page struct lock on the large page root. 4196 */ 4197 page_struct_lock(ppa[0]); 4198 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4199 4200 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4201 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4202 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4203 page_struct_unlock(ppa[0]); 4204 return (0); 4205 } 4206 if (ppa[pg_idx]->p_lckcnt != 0) 4207 ulckpgs++; 4208 } 4209 4210 if (ulckpgs != 0) { 4211 mutex_enter(&freemem_lock); 4212 availrmem += ulckpgs; 4213 pages_claimed -= ulckpgs; 4214 mutex_exit(&freemem_lock); 4215 } 4216 4217 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4218 ppa[pg_idx]->p_cowcnt--; 4219 ppa[pg_idx]->p_lckcnt++; 4220 4221 } 4222 page_struct_unlock(ppa[0]); 4223 return (1); 4224 } 4225 4226 page_t * 4227 page_numtopp(pfn_t pfnum, se_t se) 4228 { 4229 page_t *pp; 4230 4231 retry: 4232 pp = page_numtopp_nolock(pfnum); 4233 if (pp == NULL) { 4234 return ((page_t *)NULL); 4235 } 4236 4237 /* 4238 * Acquire the appropriate lock on the page. 4239 */ 4240 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4241 if (page_pptonum(pp) != pfnum) 4242 goto retry; 4243 continue; 4244 } 4245 4246 if (page_pptonum(pp) != pfnum) { 4247 page_unlock(pp); 4248 goto retry; 4249 } 4250 4251 return (pp); 4252 } 4253 4254 page_t * 4255 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4256 { 4257 page_t *pp; 4258 4259 retry: 4260 pp = page_numtopp_nolock(pfnum); 4261 if (pp == NULL) { 4262 return ((page_t *)NULL); 4263 } 4264 4265 /* 4266 * Acquire the appropriate lock on the page. 4267 */ 4268 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4269 if (page_pptonum(pp) != pfnum) 4270 goto retry; 4271 continue; 4272 } 4273 4274 if (page_pptonum(pp) != pfnum) { 4275 page_unlock(pp); 4276 goto retry; 4277 } 4278 4279 return (pp); 4280 } 4281 4282 /* 4283 * This routine is like page_numtopp, but will only return page structs 4284 * for pages which are ok for loading into hardware using the page struct. 4285 */ 4286 page_t * 4287 page_numtopp_nowait(pfn_t pfnum, se_t se) 4288 { 4289 page_t *pp; 4290 4291 retry: 4292 pp = page_numtopp_nolock(pfnum); 4293 if (pp == NULL) { 4294 return ((page_t *)NULL); 4295 } 4296 4297 /* 4298 * Try to acquire the appropriate lock on the page. 4299 */ 4300 if (PP_ISFREE(pp)) 4301 pp = NULL; 4302 else { 4303 if (!page_trylock(pp, se)) 4304 pp = NULL; 4305 else { 4306 if (page_pptonum(pp) != pfnum) { 4307 page_unlock(pp); 4308 goto retry; 4309 } 4310 if (PP_ISFREE(pp)) { 4311 page_unlock(pp); 4312 pp = NULL; 4313 } 4314 } 4315 } 4316 return (pp); 4317 } 4318 4319 /* 4320 * Returns a count of dirty pages that are in the process 4321 * of being written out. If 'cleanit' is set, try to push the page. 4322 */ 4323 pgcnt_t 4324 page_busy(int cleanit) 4325 { 4326 page_t *page0 = page_first(); 4327 page_t *pp = page0; 4328 pgcnt_t nppbusy = 0; 4329 u_offset_t off; 4330 4331 do { 4332 vnode_t *vp = pp->p_vnode; 4333 /* 4334 * A page is a candidate for syncing if it is: 4335 * 4336 * (a) On neither the freelist nor the cachelist 4337 * (b) Hashed onto a vnode 4338 * (c) Not a kernel page 4339 * (d) Dirty 4340 * (e) Not part of a swapfile 4341 * (f) a page which belongs to a real vnode; eg has a non-null 4342 * v_vfsp pointer. 4343 * (g) Backed by a filesystem which doesn't have a 4344 * stubbed-out sync operation 4345 */ 4346 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && 4347 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4348 vfs_can_sync(vp->v_vfsp)) { 4349 nppbusy++; 4350 4351 if (!cleanit) 4352 continue; 4353 if (!page_trylock(pp, SE_EXCL)) 4354 continue; 4355 4356 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4357 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4358 !(hat_pagesync(pp, 4359 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4360 page_unlock(pp); 4361 continue; 4362 } 4363 off = pp->p_offset; 4364 VN_HOLD(vp); 4365 page_unlock(pp); 4366 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4367 B_ASYNC | B_FREE, kcred, NULL); 4368 VN_RELE(vp); 4369 } 4370 } while ((pp = page_next(pp)) != page0); 4371 4372 return (nppbusy); 4373 } 4374 4375 void page_invalidate_pages(void); 4376 4377 /* 4378 * callback handler to vm sub-system 4379 * 4380 * callers make sure no recursive entries to this func. 4381 */ 4382 /*ARGSUSED*/ 4383 boolean_t 4384 callb_vm_cpr(void *arg, int code) 4385 { 4386 if (code == CB_CODE_CPR_CHKPT) 4387 page_invalidate_pages(); 4388 return (B_TRUE); 4389 } 4390 4391 /* 4392 * Invalidate all pages of the system. 4393 * It shouldn't be called until all user page activities are all stopped. 4394 */ 4395 void 4396 page_invalidate_pages() 4397 { 4398 page_t *pp; 4399 page_t *page0; 4400 pgcnt_t nbusypages; 4401 int retry = 0; 4402 const int MAXRETRIES = 4; 4403 top: 4404 /* 4405 * Flush dirty pages and destroy the clean ones. 4406 */ 4407 nbusypages = 0; 4408 4409 pp = page0 = page_first(); 4410 do { 4411 struct vnode *vp; 4412 u_offset_t offset; 4413 int mod; 4414 4415 /* 4416 * skip the page if it has no vnode or the page associated 4417 * with the kernel vnode or prom allocated kernel mem. 4418 */ 4419 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) 4420 continue; 4421 4422 /* 4423 * skip the page which is already free invalidated. 4424 */ 4425 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4426 continue; 4427 4428 /* 4429 * skip pages that are already locked or can't be "exclusively" 4430 * locked or are already free. After we lock the page, check 4431 * the free and age bits again to be sure it's not destroyed 4432 * yet. 4433 * To achieve max. parallelization, we use page_trylock instead 4434 * of page_lock so that we don't get block on individual pages 4435 * while we have thousands of other pages to process. 4436 */ 4437 if (!page_trylock(pp, SE_EXCL)) { 4438 nbusypages++; 4439 continue; 4440 } else if (PP_ISFREE(pp)) { 4441 if (!PP_ISAGED(pp)) { 4442 page_destroy_free(pp); 4443 } else { 4444 page_unlock(pp); 4445 } 4446 continue; 4447 } 4448 /* 4449 * Is this page involved in some I/O? shared? 4450 * 4451 * The page_struct_lock need not be acquired to 4452 * examine these fields since the page has an 4453 * "exclusive" lock. 4454 */ 4455 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4456 page_unlock(pp); 4457 continue; 4458 } 4459 4460 if (vp->v_type == VCHR) { 4461 panic("vp->v_type == VCHR"); 4462 /*NOTREACHED*/ 4463 } 4464 4465 if (!page_try_demote_pages(pp)) { 4466 page_unlock(pp); 4467 continue; 4468 } 4469 4470 /* 4471 * Check the modified bit. Leave the bits alone in hardware 4472 * (they will be modified if we do the putpage). 4473 */ 4474 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4475 & P_MOD); 4476 if (mod) { 4477 offset = pp->p_offset; 4478 /* 4479 * Hold the vnode before releasing the page lock 4480 * to prevent it from being freed and re-used by 4481 * some other thread. 4482 */ 4483 VN_HOLD(vp); 4484 page_unlock(pp); 4485 /* 4486 * No error return is checked here. Callers such as 4487 * cpr deals with the dirty pages at the dump time 4488 * if this putpage fails. 4489 */ 4490 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4491 kcred, NULL); 4492 VN_RELE(vp); 4493 } else { 4494 /*LINTED: constant in conditional context*/ 4495 VN_DISPOSE(pp, B_INVAL, 0, kcred); 4496 } 4497 } while ((pp = page_next(pp)) != page0); 4498 if (nbusypages && retry++ < MAXRETRIES) { 4499 delay(1); 4500 goto top; 4501 } 4502 } 4503 4504 /* 4505 * Replace the page "old" with the page "new" on the page hash and vnode lists 4506 * 4507 * the replacement must be done in place, ie the equivalent sequence: 4508 * 4509 * vp = old->p_vnode; 4510 * off = old->p_offset; 4511 * page_do_hashout(old) 4512 * page_do_hashin(new, vp, off) 4513 * 4514 * doesn't work, since 4515 * 1) if old is the only page on the vnode, the v_pages list has a window 4516 * where it looks empty. This will break file system assumptions. 4517 * and 4518 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4519 */ 4520 static void 4521 page_do_relocate_hash(page_t *new, page_t *old) 4522 { 4523 page_t **hash_list; 4524 vnode_t *vp = old->p_vnode; 4525 kmutex_t *sep; 4526 4527 ASSERT(PAGE_EXCL(old)); 4528 ASSERT(PAGE_EXCL(new)); 4529 ASSERT(vp != NULL); 4530 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4531 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4532 4533 /* 4534 * First find old page on the page hash list 4535 */ 4536 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4537 4538 for (;;) { 4539 if (*hash_list == old) 4540 break; 4541 if (*hash_list == NULL) { 4542 panic("page_do_hashout"); 4543 /*NOTREACHED*/ 4544 } 4545 hash_list = &(*hash_list)->p_hash; 4546 } 4547 4548 /* 4549 * update new and replace old with new on the page hash list 4550 */ 4551 new->p_vnode = old->p_vnode; 4552 new->p_offset = old->p_offset; 4553 new->p_hash = old->p_hash; 4554 *hash_list = new; 4555 4556 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4557 PP_SETSWAP(new); 4558 4559 /* 4560 * replace old with new on the vnode's page list 4561 */ 4562 if (old->p_vpnext == old) { 4563 new->p_vpnext = new; 4564 new->p_vpprev = new; 4565 } else { 4566 new->p_vpnext = old->p_vpnext; 4567 new->p_vpprev = old->p_vpprev; 4568 new->p_vpnext->p_vpprev = new; 4569 new->p_vpprev->p_vpnext = new; 4570 } 4571 if (vp->v_pages == old) 4572 vp->v_pages = new; 4573 4574 /* 4575 * clear out the old page 4576 */ 4577 old->p_hash = NULL; 4578 old->p_vpnext = NULL; 4579 old->p_vpprev = NULL; 4580 old->p_vnode = NULL; 4581 PP_CLRSWAP(old); 4582 old->p_offset = (u_offset_t)-1; 4583 page_clr_all_props(old); 4584 4585 /* 4586 * Wake up processes waiting for this page. The page's 4587 * identity has been changed, and is probably not the 4588 * desired page any longer. 4589 */ 4590 sep = page_se_mutex(old); 4591 mutex_enter(sep); 4592 old->p_selock &= ~SE_EWANTED; 4593 if (CV_HAS_WAITERS(&old->p_cv)) 4594 cv_broadcast(&old->p_cv); 4595 mutex_exit(sep); 4596 } 4597 4598 /* 4599 * This function moves the identity of page "pp_old" to page "pp_new". 4600 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4601 * and need not be hashed out from anywhere. 4602 */ 4603 void 4604 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4605 { 4606 vnode_t *vp = pp_old->p_vnode; 4607 u_offset_t off = pp_old->p_offset; 4608 kmutex_t *phm, *vphm; 4609 4610 /* 4611 * Rehash two pages 4612 */ 4613 ASSERT(PAGE_EXCL(pp_old)); 4614 ASSERT(PAGE_EXCL(pp_new)); 4615 ASSERT(vp != NULL); 4616 ASSERT(pp_new->p_vnode == NULL); 4617 4618 /* 4619 * hashout then hashin while holding the mutexes 4620 */ 4621 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4622 mutex_enter(phm); 4623 vphm = page_vnode_mutex(vp); 4624 mutex_enter(vphm); 4625 4626 page_do_relocate_hash(pp_new, pp_old); 4627 4628 /* The following comment preserved from page_flip(). */ 4629 pp_new->p_fsdata = pp_old->p_fsdata; 4630 pp_old->p_fsdata = 0; 4631 mutex_exit(vphm); 4632 mutex_exit(phm); 4633 4634 /* 4635 * The page_struct_lock need not be acquired for lckcnt and 4636 * cowcnt since the page has an "exclusive" lock. 4637 */ 4638 ASSERT(pp_new->p_lckcnt == 0); 4639 ASSERT(pp_new->p_cowcnt == 0); 4640 pp_new->p_lckcnt = pp_old->p_lckcnt; 4641 pp_new->p_cowcnt = pp_old->p_cowcnt; 4642 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4643 4644 } 4645 4646 /* 4647 * Helper routine used to lock all remaining members of a 4648 * large page. The caller is responsible for passing in a locked 4649 * pp. If pp is a large page, then it succeeds in locking all the 4650 * remaining constituent pages or it returns with only the 4651 * original page locked. 4652 * 4653 * Returns 1 on success, 0 on failure. 4654 * 4655 * If success is returned this routine guarantees p_szc for all constituent 4656 * pages of a large page pp belongs to can't change. To achieve this we 4657 * recheck szc of pp after locking all constituent pages and retry if szc 4658 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4659 * lock on one of constituent pages it can't be running after all constituent 4660 * pages are locked. hat_page_demote() with a lock on a constituent page 4661 * outside of this large page (i.e. pp belonged to a larger large page) is 4662 * already done with all constituent pages of pp since the root's p_szc is 4663 * changed last. Therefore no need to synchronize with hat_page_demote() that 4664 * locked a constituent page outside of pp's current large page. 4665 */ 4666 #ifdef DEBUG 4667 uint32_t gpg_trylock_mtbf = 0; 4668 #endif 4669 4670 int 4671 group_page_trylock(page_t *pp, se_t se) 4672 { 4673 page_t *tpp; 4674 pgcnt_t npgs, i, j; 4675 uint_t pszc = pp->p_szc; 4676 4677 #ifdef DEBUG 4678 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4679 return (0); 4680 } 4681 #endif 4682 4683 if (pp != PP_GROUPLEADER(pp, pszc)) { 4684 return (0); 4685 } 4686 4687 retry: 4688 ASSERT(PAGE_LOCKED_SE(pp, se)); 4689 ASSERT(!PP_ISFREE(pp)); 4690 if (pszc == 0) { 4691 return (1); 4692 } 4693 npgs = page_get_pagecnt(pszc); 4694 tpp = pp + 1; 4695 for (i = 1; i < npgs; i++, tpp++) { 4696 if (!page_trylock(tpp, se)) { 4697 tpp = pp + 1; 4698 for (j = 1; j < i; j++, tpp++) { 4699 page_unlock(tpp); 4700 } 4701 return (0); 4702 } 4703 } 4704 if (pp->p_szc != pszc) { 4705 ASSERT(pp->p_szc < pszc); 4706 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && 4707 !IS_SWAPFSVP(pp->p_vnode)); 4708 tpp = pp + 1; 4709 for (i = 1; i < npgs; i++, tpp++) { 4710 page_unlock(tpp); 4711 } 4712 pszc = pp->p_szc; 4713 goto retry; 4714 } 4715 return (1); 4716 } 4717 4718 void 4719 group_page_unlock(page_t *pp) 4720 { 4721 page_t *tpp; 4722 pgcnt_t npgs, i; 4723 4724 ASSERT(PAGE_LOCKED(pp)); 4725 ASSERT(!PP_ISFREE(pp)); 4726 ASSERT(pp == PP_PAGEROOT(pp)); 4727 npgs = page_get_pagecnt(pp->p_szc); 4728 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4729 page_unlock(tpp); 4730 } 4731 } 4732 4733 /* 4734 * returns 4735 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4736 * ERANGE : this is not a base page 4737 * EBUSY : failure to get locks on the page/pages 4738 * ENOMEM : failure to obtain replacement pages 4739 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4740 * EIO : An error occurred while trying to copy the page data 4741 * 4742 * Return with all constituent members of target and replacement 4743 * SE_EXCL locked. It is the callers responsibility to drop the 4744 * locks. 4745 */ 4746 int 4747 do_page_relocate( 4748 page_t **target, 4749 page_t **replacement, 4750 int grouplock, 4751 spgcnt_t *nrelocp, 4752 lgrp_t *lgrp) 4753 { 4754 page_t *first_repl; 4755 page_t *repl; 4756 page_t *targ; 4757 page_t *pl = NULL; 4758 uint_t ppattr; 4759 pfn_t pfn, repl_pfn; 4760 uint_t szc; 4761 spgcnt_t npgs, i; 4762 int repl_contig = 0; 4763 uint_t flags = 0; 4764 spgcnt_t dofree = 0; 4765 4766 *nrelocp = 0; 4767 4768 #if defined(__sparc) 4769 /* 4770 * We need to wait till OBP has completed 4771 * its boot-time handoff of its resources to the kernel 4772 * before we allow page relocation 4773 */ 4774 if (page_relocate_ready == 0) { 4775 return (EAGAIN); 4776 } 4777 #endif 4778 4779 /* 4780 * If this is not a base page, 4781 * just return with 0x0 pages relocated. 4782 */ 4783 targ = *target; 4784 ASSERT(PAGE_EXCL(targ)); 4785 ASSERT(!PP_ISFREE(targ)); 4786 szc = targ->p_szc; 4787 ASSERT(szc < mmu_page_sizes); 4788 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4789 pfn = targ->p_pagenum; 4790 if (pfn != PFN_BASE(pfn, szc)) { 4791 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4792 return (ERANGE); 4793 } 4794 4795 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4796 repl_pfn = repl->p_pagenum; 4797 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4798 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4799 return (ERANGE); 4800 } 4801 repl_contig = 1; 4802 } 4803 4804 /* 4805 * We must lock all members of this large page or we cannot 4806 * relocate any part of it. 4807 */ 4808 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4809 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4810 return (EBUSY); 4811 } 4812 4813 /* 4814 * reread szc it could have been decreased before 4815 * group_page_trylock() was done. 4816 */ 4817 szc = targ->p_szc; 4818 ASSERT(szc < mmu_page_sizes); 4819 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4820 ASSERT(pfn == PFN_BASE(pfn, szc)); 4821 4822 npgs = page_get_pagecnt(targ->p_szc); 4823 4824 if (repl == NULL) { 4825 dofree = npgs; /* Size of target page in MMU pages */ 4826 if (!page_create_wait(dofree, 0)) { 4827 if (grouplock != 0) { 4828 group_page_unlock(targ); 4829 } 4830 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4831 return (ENOMEM); 4832 } 4833 4834 /* 4835 * seg kmem pages require that the target and replacement 4836 * page be the same pagesize. 4837 */ 4838 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0; 4839 repl = page_get_replacement_page(targ, lgrp, flags); 4840 if (repl == NULL) { 4841 if (grouplock != 0) { 4842 group_page_unlock(targ); 4843 } 4844 page_create_putback(dofree); 4845 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4846 return (ENOMEM); 4847 } 4848 } 4849 #ifdef DEBUG 4850 else { 4851 ASSERT(PAGE_LOCKED(repl)); 4852 } 4853 #endif /* DEBUG */ 4854 4855 #if defined(__sparc) 4856 /* 4857 * Let hat_page_relocate() complete the relocation if it's kernel page 4858 */ 4859 if (VN_ISKAS(targ->p_vnode)) { 4860 *replacement = repl; 4861 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4862 if (grouplock != 0) { 4863 group_page_unlock(targ); 4864 } 4865 if (dofree) { 4866 *replacement = NULL; 4867 page_free_replacement_page(repl); 4868 page_create_putback(dofree); 4869 } 4870 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4871 return (EAGAIN); 4872 } 4873 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4874 return (0); 4875 } 4876 #else 4877 #if defined(lint) 4878 dofree = dofree; 4879 #endif 4880 #endif 4881 4882 first_repl = repl; 4883 4884 for (i = 0; i < npgs; i++) { 4885 ASSERT(PAGE_EXCL(targ)); 4886 ASSERT(targ->p_slckcnt == 0); 4887 ASSERT(repl->p_slckcnt == 0); 4888 4889 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4890 4891 ASSERT(hat_page_getshare(targ) == 0); 4892 ASSERT(!PP_ISFREE(targ)); 4893 ASSERT(targ->p_pagenum == (pfn + i)); 4894 ASSERT(repl_contig == 0 || 4895 repl->p_pagenum == (repl_pfn + i)); 4896 4897 /* 4898 * Copy the page contents and attributes then 4899 * relocate the page in the page hash. 4900 */ 4901 if (ppcopy(targ, repl) == 0) { 4902 targ = *target; 4903 repl = first_repl; 4904 VM_STAT_ADD(vmm_vmstats.ppr_copyfail); 4905 if (grouplock != 0) { 4906 group_page_unlock(targ); 4907 } 4908 if (dofree) { 4909 *replacement = NULL; 4910 page_free_replacement_page(repl); 4911 page_create_putback(dofree); 4912 } 4913 return (EIO); 4914 } 4915 4916 targ++; 4917 if (repl_contig != 0) { 4918 repl++; 4919 } else { 4920 repl = repl->p_next; 4921 } 4922 } 4923 4924 repl = first_repl; 4925 targ = *target; 4926 4927 for (i = 0; i < npgs; i++) { 4928 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4929 page_clr_all_props(repl); 4930 page_set_props(repl, ppattr); 4931 page_relocate_hash(repl, targ); 4932 4933 ASSERT(hat_page_getshare(targ) == 0); 4934 ASSERT(hat_page_getshare(repl) == 0); 4935 /* 4936 * Now clear the props on targ, after the 4937 * page_relocate_hash(), they no longer 4938 * have any meaning. 4939 */ 4940 page_clr_all_props(targ); 4941 ASSERT(targ->p_next == targ); 4942 ASSERT(targ->p_prev == targ); 4943 page_list_concat(&pl, &targ); 4944 4945 targ++; 4946 if (repl_contig != 0) { 4947 repl++; 4948 } else { 4949 repl = repl->p_next; 4950 } 4951 } 4952 /* assert that we have come full circle with repl */ 4953 ASSERT(repl_contig == 1 || first_repl == repl); 4954 4955 *target = pl; 4956 if (*replacement == NULL) { 4957 ASSERT(first_repl == repl); 4958 *replacement = repl; 4959 } 4960 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4961 *nrelocp = npgs; 4962 return (0); 4963 } 4964 /* 4965 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 4966 */ 4967 int 4968 page_relocate( 4969 page_t **target, 4970 page_t **replacement, 4971 int grouplock, 4972 int freetarget, 4973 spgcnt_t *nrelocp, 4974 lgrp_t *lgrp) 4975 { 4976 spgcnt_t ret; 4977 4978 /* do_page_relocate returns 0 on success or errno value */ 4979 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 4980 4981 if (ret != 0 || freetarget == 0) { 4982 return (ret); 4983 } 4984 if (*nrelocp == 1) { 4985 ASSERT(*target != NULL); 4986 page_free(*target, 1); 4987 } else { 4988 page_t *tpp = *target; 4989 uint_t szc = tpp->p_szc; 4990 pgcnt_t npgs = page_get_pagecnt(szc); 4991 ASSERT(npgs > 1); 4992 ASSERT(szc != 0); 4993 do { 4994 ASSERT(PAGE_EXCL(tpp)); 4995 ASSERT(!hat_page_is_mapped(tpp)); 4996 ASSERT(tpp->p_szc == szc); 4997 PP_SETFREE(tpp); 4998 PP_SETAGED(tpp); 4999 npgs--; 5000 } while ((tpp = tpp->p_next) != *target); 5001 ASSERT(npgs == 0); 5002 page_list_add_pages(*target, 0); 5003 npgs = page_get_pagecnt(szc); 5004 page_create_putback(npgs); 5005 } 5006 return (ret); 5007 } 5008 5009 /* 5010 * it is up to the caller to deal with pcf accounting. 5011 */ 5012 void 5013 page_free_replacement_page(page_t *pplist) 5014 { 5015 page_t *pp; 5016 5017 while (pplist != NULL) { 5018 /* 5019 * pp_targ is a linked list. 5020 */ 5021 pp = pplist; 5022 if (pp->p_szc == 0) { 5023 page_sub(&pplist, pp); 5024 page_clr_all_props(pp); 5025 PP_SETFREE(pp); 5026 PP_SETAGED(pp); 5027 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 5028 page_unlock(pp); 5029 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 5030 } else { 5031 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 5032 page_t *tpp; 5033 page_list_break(&pp, &pplist, curnpgs); 5034 tpp = pp; 5035 do { 5036 ASSERT(PAGE_EXCL(tpp)); 5037 ASSERT(!hat_page_is_mapped(tpp)); 5038 page_clr_all_props(tpp); 5039 PP_SETFREE(tpp); 5040 PP_SETAGED(tpp); 5041 } while ((tpp = tpp->p_next) != pp); 5042 page_list_add_pages(pp, 0); 5043 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 5044 } 5045 } 5046 } 5047 5048 /* 5049 * Relocate target to non-relocatable replacement page. 5050 */ 5051 int 5052 page_relocate_cage(page_t **target, page_t **replacement) 5053 { 5054 page_t *tpp, *rpp; 5055 spgcnt_t pgcnt, npgs; 5056 int result; 5057 5058 tpp = *target; 5059 5060 ASSERT(PAGE_EXCL(tpp)); 5061 ASSERT(tpp->p_szc == 0); 5062 5063 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5064 5065 do { 5066 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5067 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5068 if (rpp == NULL) { 5069 page_create_putback(pgcnt); 5070 kcage_cageout_wakeup(); 5071 } 5072 } while (rpp == NULL); 5073 5074 ASSERT(PP_ISNORELOC(rpp)); 5075 5076 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5077 5078 if (result == 0) { 5079 *replacement = rpp; 5080 if (pgcnt != npgs) 5081 panic("page_relocate_cage: partial relocation"); 5082 } 5083 5084 return (result); 5085 } 5086 5087 /* 5088 * Release the page lock on a page, place on cachelist 5089 * tail if no longer mapped. Caller can let us know if 5090 * the page is known to be clean. 5091 */ 5092 int 5093 page_release(page_t *pp, int checkmod) 5094 { 5095 int status; 5096 5097 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5098 (pp->p_vnode != NULL)); 5099 5100 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5101 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5102 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5103 !hat_page_is_mapped(pp)) { 5104 5105 /* 5106 * If page is modified, unlock it 5107 * 5108 * (p_nrm & P_MOD) bit has the latest stuff because: 5109 * (1) We found that this page doesn't have any mappings 5110 * _after_ holding SE_EXCL and 5111 * (2) We didn't drop SE_EXCL lock after the check in (1) 5112 */ 5113 if (checkmod && hat_ismod(pp)) { 5114 page_unlock(pp); 5115 status = PGREL_MOD; 5116 } else { 5117 /*LINTED: constant in conditional context*/ 5118 VN_DISPOSE(pp, B_FREE, 0, kcred); 5119 status = PGREL_CLEAN; 5120 } 5121 } else { 5122 page_unlock(pp); 5123 status = PGREL_NOTREL; 5124 } 5125 return (status); 5126 } 5127 5128 /* 5129 * Given a constituent page, try to demote the large page on the freelist. 5130 * 5131 * Returns nonzero if the page could be demoted successfully. Returns with 5132 * the constituent page still locked. 5133 */ 5134 int 5135 page_try_demote_free_pages(page_t *pp) 5136 { 5137 page_t *rootpp = pp; 5138 pfn_t pfn = page_pptonum(pp); 5139 spgcnt_t npgs; 5140 uint_t szc = pp->p_szc; 5141 5142 ASSERT(PP_ISFREE(pp)); 5143 ASSERT(PAGE_EXCL(pp)); 5144 5145 /* 5146 * Adjust rootpp and lock it, if `pp' is not the base 5147 * constituent page. 5148 */ 5149 npgs = page_get_pagecnt(pp->p_szc); 5150 if (npgs == 1) { 5151 return (0); 5152 } 5153 5154 if (!IS_P2ALIGNED(pfn, npgs)) { 5155 pfn = P2ALIGN(pfn, npgs); 5156 rootpp = page_numtopp_nolock(pfn); 5157 } 5158 5159 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5160 return (0); 5161 } 5162 5163 if (rootpp->p_szc != szc) { 5164 if (pp != rootpp) 5165 page_unlock(rootpp); 5166 return (0); 5167 } 5168 5169 page_demote_free_pages(rootpp); 5170 5171 if (pp != rootpp) 5172 page_unlock(rootpp); 5173 5174 ASSERT(PP_ISFREE(pp)); 5175 ASSERT(PAGE_EXCL(pp)); 5176 return (1); 5177 } 5178 5179 /* 5180 * Given a constituent page, try to demote the large page. 5181 * 5182 * Returns nonzero if the page could be demoted successfully. Returns with 5183 * the constituent page still locked. 5184 */ 5185 int 5186 page_try_demote_pages(page_t *pp) 5187 { 5188 page_t *tpp, *rootpp = pp; 5189 pfn_t pfn = page_pptonum(pp); 5190 spgcnt_t i, npgs; 5191 uint_t szc = pp->p_szc; 5192 vnode_t *vp = pp->p_vnode; 5193 5194 ASSERT(PAGE_EXCL(pp)); 5195 5196 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5197 5198 if (pp->p_szc == 0) { 5199 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5200 return (1); 5201 } 5202 5203 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) { 5204 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5205 page_demote_vp_pages(pp); 5206 ASSERT(pp->p_szc == 0); 5207 return (1); 5208 } 5209 5210 /* 5211 * Adjust rootpp if passed in is not the base 5212 * constituent page. 5213 */ 5214 npgs = page_get_pagecnt(pp->p_szc); 5215 ASSERT(npgs > 1); 5216 if (!IS_P2ALIGNED(pfn, npgs)) { 5217 pfn = P2ALIGN(pfn, npgs); 5218 rootpp = page_numtopp_nolock(pfn); 5219 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5220 ASSERT(rootpp->p_vnode != NULL); 5221 ASSERT(rootpp->p_szc == szc); 5222 } 5223 5224 /* 5225 * We can't demote kernel pages since we can't hat_unload() 5226 * the mappings. 5227 */ 5228 if (VN_ISKAS(rootpp->p_vnode)) 5229 return (0); 5230 5231 /* 5232 * Attempt to lock all constituent pages except the page passed 5233 * in since it's already locked. 5234 */ 5235 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5236 ASSERT(!PP_ISFREE(tpp)); 5237 ASSERT(tpp->p_vnode != NULL); 5238 5239 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5240 break; 5241 ASSERT(tpp->p_szc == rootpp->p_szc); 5242 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5243 } 5244 5245 /* 5246 * If we failed to lock them all then unlock what we have 5247 * locked so far and bail. 5248 */ 5249 if (i < npgs) { 5250 tpp = rootpp; 5251 while (i-- > 0) { 5252 if (tpp != pp) 5253 page_unlock(tpp); 5254 tpp++; 5255 } 5256 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5257 return (0); 5258 } 5259 5260 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5261 ASSERT(PAGE_EXCL(tpp)); 5262 ASSERT(tpp->p_slckcnt == 0); 5263 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5264 tpp->p_szc = 0; 5265 } 5266 5267 /* 5268 * Unlock all pages except the page passed in. 5269 */ 5270 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5271 ASSERT(!hat_page_is_mapped(tpp)); 5272 if (tpp != pp) 5273 page_unlock(tpp); 5274 } 5275 5276 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5277 return (1); 5278 } 5279 5280 /* 5281 * Called by page_free() and page_destroy() to demote the page size code 5282 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5283 * p_szc on free list, neither can we just clear p_szc of a single page_t 5284 * within a large page since it will break other code that relies on p_szc 5285 * being the same for all page_t's of a large page). Anonymous pages should 5286 * never end up here because anon_map_getpages() cannot deal with p_szc 5287 * changes after a single constituent page is locked. While anonymous or 5288 * kernel large pages are demoted or freed the entire large page at a time 5289 * with all constituent pages locked EXCL for the file system pages we 5290 * have to be able to demote a large page (i.e. decrease all constituent pages 5291 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5292 * we can easily deal with anonymous page demotion the entire large page at a 5293 * time is that those operation originate at address space level and concern 5294 * the entire large page region with actual demotion only done when pages are 5295 * not shared with any other processes (therefore we can always get EXCL lock 5296 * on all anonymous constituent pages after clearing segment page 5297 * cache). However file system pages can be truncated or invalidated at a 5298 * PAGESIZE level from the file system side and end up in page_free() or 5299 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5300 * and therefore pageout should be able to demote a large page by EXCL locking 5301 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5302 * rely on being able to lock EXCL all constituent pages. 5303 * 5304 * To prevent szc changes on file system pages one has to lock all constituent 5305 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5306 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5307 * prevent szc changes is hat layer that uses its own page level mlist 5308 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5309 * taken. Therefore we need to change szc under hat level locks if we only 5310 * have an EXCL lock on a single constituent page and hat still references any 5311 * of constituent pages. (Note we can't "ignore" hat layer by simply 5312 * hat_pageunload() all constituent pages without having EXCL locks on all of 5313 * constituent pages). We use hat_page_demote() call to safely demote szc of 5314 * all constituent pages under hat locks when we only have an EXCL lock on one 5315 * of constituent pages. 5316 * 5317 * This routine calls page_szc_lock() before calling hat_page_demote() to 5318 * allow segvn in one special case not to lock all constituent pages SHARED 5319 * before calling hat_memload_array() that relies on p_szc not changing even 5320 * before hat level mlist lock is taken. In that case segvn uses 5321 * page_szc_lock() to prevent hat_page_demote() changing p_szc values. 5322 * 5323 * Anonymous or kernel page demotion still has to lock all pages exclusively 5324 * and do hat_pageunload() on all constituent pages before demoting the page 5325 * therefore there's no need for anonymous or kernel page demotion to use 5326 * hat_page_demote() mechanism. 5327 * 5328 * hat_page_demote() removes all large mappings that map pp and then decreases 5329 * p_szc starting from the last constituent page of the large page. By working 5330 * from the tail of a large page in pfn decreasing order allows one looking at 5331 * the root page to know that hat_page_demote() is done for root's szc area. 5332 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5333 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5334 * that started on this page when it had szc > 1 is done for this szc 1 area. 5335 * 5336 * We are guaranteed that all constituent pages of pp's large page belong to 5337 * the same vnode with the consecutive offsets increasing in the direction of 5338 * the pfn i.e. the identity of constituent pages can't change until their 5339 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5340 * large mappings to pp even though we don't lock any constituent page except 5341 * pp (i.e. we won't unload e.g. kernel locked page). 5342 */ 5343 static void 5344 page_demote_vp_pages(page_t *pp) 5345 { 5346 kmutex_t *mtx; 5347 5348 ASSERT(PAGE_EXCL(pp)); 5349 ASSERT(!PP_ISFREE(pp)); 5350 ASSERT(pp->p_vnode != NULL); 5351 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5352 ASSERT(!PP_ISKAS(pp)); 5353 5354 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5355 5356 mtx = page_szc_lock(pp); 5357 if (mtx != NULL) { 5358 hat_page_demote(pp); 5359 mutex_exit(mtx); 5360 } 5361 ASSERT(pp->p_szc == 0); 5362 } 5363 5364 /* 5365 * Mark any existing pages for migration in the given range 5366 */ 5367 void 5368 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5369 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5370 u_offset_t vnoff, int rflag) 5371 { 5372 struct anon *ap; 5373 vnode_t *curvp; 5374 lgrp_t *from; 5375 pgcnt_t nlocked; 5376 u_offset_t off; 5377 pfn_t pfn; 5378 size_t pgsz; 5379 size_t segpgsz; 5380 pgcnt_t pages; 5381 uint_t pszc; 5382 page_t *pp0, *pp; 5383 caddr_t va; 5384 ulong_t an_idx; 5385 anon_sync_obj_t cookie; 5386 5387 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 5388 5389 /* 5390 * Don't do anything if don't need to do lgroup optimizations 5391 * on this system 5392 */ 5393 if (!lgrp_optimizations()) 5394 return; 5395 5396 /* 5397 * Align address and length to (potentially large) page boundary 5398 */ 5399 segpgsz = page_get_pagesize(seg->s_szc); 5400 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5401 if (rflag) 5402 len = P2ROUNDUP(len, segpgsz); 5403 5404 /* 5405 * Do one (large) page at a time 5406 */ 5407 va = addr; 5408 while (va < addr + len) { 5409 /* 5410 * Lookup (root) page for vnode and offset corresponding to 5411 * this virtual address 5412 * Try anonmap first since there may be copy-on-write 5413 * pages, but initialize vnode pointer and offset using 5414 * vnode arguments just in case there isn't an amp. 5415 */ 5416 curvp = vp; 5417 off = vnoff + va - seg->s_base; 5418 if (amp) { 5419 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5420 an_idx = anon_index + seg_page(seg, va); 5421 anon_array_enter(amp, an_idx, &cookie); 5422 ap = anon_get_ptr(amp->ahp, an_idx); 5423 if (ap) 5424 swap_xlate(ap, &curvp, &off); 5425 anon_array_exit(&cookie); 5426 ANON_LOCK_EXIT(&->a_rwlock); 5427 } 5428 5429 pp = NULL; 5430 if (curvp) 5431 pp = page_lookup(curvp, off, SE_SHARED); 5432 5433 /* 5434 * If there isn't a page at this virtual address, 5435 * skip to next page 5436 */ 5437 if (pp == NULL) { 5438 va += PAGESIZE; 5439 continue; 5440 } 5441 5442 /* 5443 * Figure out which lgroup this page is in for kstats 5444 */ 5445 pfn = page_pptonum(pp); 5446 from = lgrp_pfn_to_lgrp(pfn); 5447 5448 /* 5449 * Get page size, and round up and skip to next page boundary 5450 * if unaligned address 5451 */ 5452 pszc = pp->p_szc; 5453 pgsz = page_get_pagesize(pszc); 5454 pages = btop(pgsz); 5455 if (!IS_P2ALIGNED(va, pgsz) || 5456 !IS_P2ALIGNED(pfn, pages) || 5457 pgsz > segpgsz) { 5458 pgsz = MIN(pgsz, segpgsz); 5459 page_unlock(pp); 5460 pages = btop(P2END((uintptr_t)va, pgsz) - 5461 (uintptr_t)va); 5462 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5463 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, pages); 5464 continue; 5465 } 5466 5467 /* 5468 * Upgrade to exclusive lock on page 5469 */ 5470 if (!page_tryupgrade(pp)) { 5471 page_unlock(pp); 5472 va += pgsz; 5473 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5474 btop(pgsz)); 5475 continue; 5476 } 5477 5478 pp0 = pp++; 5479 nlocked = 1; 5480 5481 /* 5482 * Lock constituent pages if this is large page 5483 */ 5484 if (pages > 1) { 5485 /* 5486 * Lock all constituents except root page, since it 5487 * should be locked already. 5488 */ 5489 for (; nlocked < pages; nlocked++) { 5490 if (!page_trylock(pp, SE_EXCL)) { 5491 break; 5492 } 5493 if (PP_ISFREE(pp) || 5494 pp->p_szc != pszc) { 5495 /* 5496 * hat_page_demote() raced in with us. 5497 */ 5498 ASSERT(!IS_SWAPFSVP(curvp)); 5499 page_unlock(pp); 5500 break; 5501 } 5502 pp++; 5503 } 5504 } 5505 5506 /* 5507 * If all constituent pages couldn't be locked, 5508 * unlock pages locked so far and skip to next page. 5509 */ 5510 if (nlocked < pages) { 5511 while (pp0 < pp) { 5512 page_unlock(pp0++); 5513 } 5514 va += pgsz; 5515 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5516 btop(pgsz)); 5517 continue; 5518 } 5519 5520 /* 5521 * hat_page_demote() can no longer happen 5522 * since last cons page had the right p_szc after 5523 * all cons pages were locked. all cons pages 5524 * should now have the same p_szc. 5525 */ 5526 5527 /* 5528 * All constituent pages locked successfully, so mark 5529 * large page for migration and unload the mappings of 5530 * constituent pages, so a fault will occur on any part of the 5531 * large page 5532 */ 5533 PP_SETMIGRATE(pp0); 5534 while (pp0 < pp) { 5535 (void) hat_pageunload(pp0, HAT_FORCE_PGUNLOAD); 5536 ASSERT(hat_page_getshare(pp0) == 0); 5537 page_unlock(pp0++); 5538 } 5539 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5540 5541 va += pgsz; 5542 } 5543 } 5544 5545 /* 5546 * Migrate any pages that have been marked for migration in the given range 5547 */ 5548 void 5549 page_migrate( 5550 struct seg *seg, 5551 caddr_t addr, 5552 page_t **ppa, 5553 pgcnt_t npages) 5554 { 5555 lgrp_t *from; 5556 lgrp_t *to; 5557 page_t *newpp; 5558 page_t *pp; 5559 pfn_t pfn; 5560 size_t pgsz; 5561 spgcnt_t page_cnt; 5562 spgcnt_t i; 5563 uint_t pszc; 5564 5565 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 5566 5567 while (npages > 0) { 5568 pp = *ppa; 5569 pszc = pp->p_szc; 5570 pgsz = page_get_pagesize(pszc); 5571 page_cnt = btop(pgsz); 5572 5573 /* 5574 * Check to see whether this page is marked for migration 5575 * 5576 * Assume that root page of large page is marked for 5577 * migration and none of the other constituent pages 5578 * are marked. This really simplifies clearing the 5579 * migrate bit by not having to clear it from each 5580 * constituent page. 5581 * 5582 * note we don't want to relocate an entire large page if 5583 * someone is only using one subpage. 5584 */ 5585 if (npages < page_cnt) 5586 break; 5587 5588 /* 5589 * Is it marked for migration? 5590 */ 5591 if (!PP_ISMIGRATE(pp)) 5592 goto next; 5593 5594 /* 5595 * Determine lgroups that page is being migrated between 5596 */ 5597 pfn = page_pptonum(pp); 5598 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5599 break; 5600 } 5601 from = lgrp_pfn_to_lgrp(pfn); 5602 to = lgrp_mem_choose(seg, addr, pgsz); 5603 5604 /* 5605 * Need to get exclusive lock's to migrate 5606 */ 5607 for (i = 0; i < page_cnt; i++) { 5608 ASSERT(PAGE_LOCKED(ppa[i])); 5609 if (page_pptonum(ppa[i]) != pfn + i || 5610 ppa[i]->p_szc != pszc) { 5611 break; 5612 } 5613 if (!page_tryupgrade(ppa[i])) { 5614 lgrp_stat_add(from->lgrp_id, 5615 LGRP_PM_FAIL_LOCK_PGS, 5616 page_cnt); 5617 break; 5618 } 5619 5620 /* 5621 * Check to see whether we are trying to migrate 5622 * page to lgroup where it is allocated already. 5623 * If so, clear the migrate bit and skip to next 5624 * page. 5625 */ 5626 if (i == 0 && to == from) { 5627 PP_CLRMIGRATE(ppa[0]); 5628 page_downgrade(ppa[0]); 5629 goto next; 5630 } 5631 } 5632 5633 /* 5634 * If all constituent pages couldn't be locked, 5635 * unlock pages locked so far and skip to next page. 5636 */ 5637 if (i != page_cnt) { 5638 while (--i != -1) { 5639 page_downgrade(ppa[i]); 5640 } 5641 goto next; 5642 } 5643 5644 (void) page_create_wait(page_cnt, PG_WAIT); 5645 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5646 if (newpp == NULL) { 5647 page_create_putback(page_cnt); 5648 for (i = 0; i < page_cnt; i++) { 5649 page_downgrade(ppa[i]); 5650 } 5651 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5652 page_cnt); 5653 goto next; 5654 } 5655 ASSERT(newpp->p_szc == pszc); 5656 /* 5657 * Clear migrate bit and relocate page 5658 */ 5659 PP_CLRMIGRATE(pp); 5660 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5661 panic("page_migrate: page_relocate failed"); 5662 } 5663 ASSERT(page_cnt * PAGESIZE == pgsz); 5664 5665 /* 5666 * Keep stats for number of pages migrated from and to 5667 * each lgroup 5668 */ 5669 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5670 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5671 /* 5672 * update the page_t array we were passed in and 5673 * unlink constituent pages of a large page. 5674 */ 5675 for (i = 0; i < page_cnt; ++i, ++pp) { 5676 ASSERT(PAGE_EXCL(newpp)); 5677 ASSERT(newpp->p_szc == pszc); 5678 ppa[i] = newpp; 5679 pp = newpp; 5680 page_sub(&newpp, pp); 5681 page_downgrade(pp); 5682 } 5683 ASSERT(newpp == NULL); 5684 next: 5685 addr += pgsz; 5686 ppa += page_cnt; 5687 npages -= page_cnt; 5688 } 5689 } 5690 5691 uint_t page_reclaim_maxcnt = 60; /* max total iterations */ 5692 uint_t page_reclaim_nofree_maxcnt = 3; /* max iterations without progress */ 5693 /* 5694 * Reclaim/reserve availrmem for npages. 5695 * If there is not enough memory start reaping seg, kmem caches. 5696 * Start pageout scanner (via page_needfree()). 5697 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5698 * Note: There is no guarantee that any availrmem will be freed as 5699 * this memory typically is locked (kernel heap) or reserved for swap. 5700 * Also due to memory fragmentation kmem allocator may not be able 5701 * to free any memory (single user allocated buffer will prevent 5702 * freeing slab or a page). 5703 */ 5704 int 5705 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5706 { 5707 int i = 0; 5708 int i_nofree = 0; 5709 int ret = 0; 5710 pgcnt_t deficit; 5711 pgcnt_t old_availrmem = 0; 5712 5713 mutex_enter(&freemem_lock); 5714 while (availrmem < tune.t_minarmem + npages + epages && 5715 i++ < page_reclaim_maxcnt) { 5716 /* ensure we made some progress in the last few iterations */ 5717 if (old_availrmem < availrmem) { 5718 old_availrmem = availrmem; 5719 i_nofree = 0; 5720 } else if (i_nofree++ >= page_reclaim_nofree_maxcnt) { 5721 break; 5722 } 5723 5724 deficit = tune.t_minarmem + npages + epages - availrmem; 5725 mutex_exit(&freemem_lock); 5726 page_needfree(deficit); 5727 kmem_reap(); 5728 delay(hz); 5729 page_needfree(-(spgcnt_t)deficit); 5730 mutex_enter(&freemem_lock); 5731 } 5732 5733 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5734 availrmem -= npages; 5735 ret = 1; 5736 } 5737 5738 mutex_exit(&freemem_lock); 5739 5740 return (ret); 5741 } 5742 5743 /* 5744 * Search the memory segments to locate the desired page. Within a 5745 * segment, pages increase linearly with one page structure per 5746 * physical page frame (size PAGESIZE). The search begins 5747 * with the segment that was accessed last, to take advantage of locality. 5748 * If the hint misses, we start from the beginning of the sorted memseg list 5749 */ 5750 5751 5752 /* 5753 * Some data structures for pfn to pp lookup. 5754 */ 5755 ulong_t mhash_per_slot; 5756 struct memseg *memseg_hash[N_MEM_SLOTS]; 5757 5758 page_t * 5759 page_numtopp_nolock(pfn_t pfnum) 5760 { 5761 struct memseg *seg; 5762 page_t *pp; 5763 vm_cpu_data_t *vc; 5764 5765 /* 5766 * We need to disable kernel preemption while referencing the 5767 * cpu_vm_data field in order to prevent us from being switched to 5768 * another cpu and trying to reference it after it has been freed. 5769 * This will keep us on cpu and prevent it from being removed while 5770 * we are still on it. 5771 * 5772 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5773 * which is being resued by DR who will flush those references 5774 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5775 */ 5776 kpreempt_disable(); 5777 vc = CPU->cpu_vm_data; 5778 ASSERT(vc != NULL); 5779 5780 MEMSEG_STAT_INCR(nsearch); 5781 5782 /* Try last winner first */ 5783 if (((seg = vc->vc_pnum_memseg) != NULL) && 5784 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5785 MEMSEG_STAT_INCR(nlastwon); 5786 pp = seg->pages + (pfnum - seg->pages_base); 5787 if (pp->p_pagenum == pfnum) { 5788 kpreempt_enable(); 5789 return ((page_t *)pp); 5790 } 5791 } 5792 5793 /* Else Try hash */ 5794 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5795 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5796 MEMSEG_STAT_INCR(nhashwon); 5797 vc->vc_pnum_memseg = seg; 5798 pp = seg->pages + (pfnum - seg->pages_base); 5799 if (pp->p_pagenum == pfnum) { 5800 kpreempt_enable(); 5801 return ((page_t *)pp); 5802 } 5803 } 5804 5805 /* Else Brute force */ 5806 for (seg = memsegs; seg != NULL; seg = seg->next) { 5807 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5808 vc->vc_pnum_memseg = seg; 5809 pp = seg->pages + (pfnum - seg->pages_base); 5810 if (pp->p_pagenum == pfnum) { 5811 kpreempt_enable(); 5812 return ((page_t *)pp); 5813 } 5814 } 5815 } 5816 vc->vc_pnum_memseg = NULL; 5817 kpreempt_enable(); 5818 MEMSEG_STAT_INCR(nnotfound); 5819 return ((page_t *)NULL); 5820 5821 } 5822 5823 struct memseg * 5824 page_numtomemseg_nolock(pfn_t pfnum) 5825 { 5826 struct memseg *seg; 5827 page_t *pp; 5828 5829 /* 5830 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5831 * which is being resued by DR who will flush those references 5832 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5833 */ 5834 kpreempt_disable(); 5835 /* Try hash */ 5836 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5837 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5838 pp = seg->pages + (pfnum - seg->pages_base); 5839 if (pp->p_pagenum == pfnum) { 5840 kpreempt_enable(); 5841 return (seg); 5842 } 5843 } 5844 5845 /* Else Brute force */ 5846 for (seg = memsegs; seg != NULL; seg = seg->next) { 5847 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5848 pp = seg->pages + (pfnum - seg->pages_base); 5849 if (pp->p_pagenum == pfnum) { 5850 kpreempt_enable(); 5851 return (seg); 5852 } 5853 } 5854 } 5855 kpreempt_enable(); 5856 return ((struct memseg *)NULL); 5857 } 5858 5859 /* 5860 * Given a page and a count return the page struct that is 5861 * n structs away from the current one in the global page 5862 * list. 5863 * 5864 * This function wraps to the first page upon 5865 * reaching the end of the memseg list. 5866 */ 5867 page_t * 5868 page_nextn(page_t *pp, ulong_t n) 5869 { 5870 struct memseg *seg; 5871 page_t *ppn; 5872 vm_cpu_data_t *vc; 5873 5874 /* 5875 * We need to disable kernel preemption while referencing the 5876 * cpu_vm_data field in order to prevent us from being switched to 5877 * another cpu and trying to reference it after it has been freed. 5878 * This will keep us on cpu and prevent it from being removed while 5879 * we are still on it. 5880 * 5881 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg 5882 * which is being resued by DR who will flush those references 5883 * before modifying the reused memseg. See memseg_cpu_vm_flush(). 5884 */ 5885 kpreempt_disable(); 5886 vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5887 5888 ASSERT(vc != NULL); 5889 5890 if (((seg = vc->vc_pnext_memseg) == NULL) || 5891 (seg->pages_base == seg->pages_end) || 5892 !(pp >= seg->pages && pp < seg->epages)) { 5893 5894 for (seg = memsegs; seg; seg = seg->next) { 5895 if (pp >= seg->pages && pp < seg->epages) 5896 break; 5897 } 5898 5899 if (seg == NULL) { 5900 /* Memory delete got in, return something valid. */ 5901 /* TODO: fix me. */ 5902 seg = memsegs; 5903 pp = seg->pages; 5904 } 5905 } 5906 5907 /* check for wraparound - possible if n is large */ 5908 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5909 n -= seg->epages - pp; 5910 seg = seg->next; 5911 if (seg == NULL) 5912 seg = memsegs; 5913 pp = seg->pages; 5914 } 5915 vc->vc_pnext_memseg = seg; 5916 kpreempt_enable(); 5917 return (ppn); 5918 } 5919 5920 /* 5921 * Initialize for a loop using page_next_scan_large(). 5922 */ 5923 page_t * 5924 page_next_scan_init(void **cookie) 5925 { 5926 ASSERT(cookie != NULL); 5927 *cookie = (void *)memsegs; 5928 return ((page_t *)memsegs->pages); 5929 } 5930 5931 /* 5932 * Return the next page in a scan of page_t's, assuming we want 5933 * to skip over sub-pages within larger page sizes. 5934 * 5935 * The cookie is used to keep track of the current memseg. 5936 */ 5937 page_t * 5938 page_next_scan_large( 5939 page_t *pp, 5940 ulong_t *n, 5941 void **cookie) 5942 { 5943 struct memseg *seg = (struct memseg *)*cookie; 5944 page_t *new_pp; 5945 ulong_t cnt; 5946 pfn_t pfn; 5947 5948 5949 /* 5950 * get the count of page_t's to skip based on the page size 5951 */ 5952 ASSERT(pp != NULL); 5953 if (pp->p_szc == 0) { 5954 cnt = 1; 5955 } else { 5956 pfn = page_pptonum(pp); 5957 cnt = page_get_pagecnt(pp->p_szc); 5958 cnt -= pfn & (cnt - 1); 5959 } 5960 *n += cnt; 5961 new_pp = pp + cnt; 5962 5963 /* 5964 * Catch if we went past the end of the current memory segment. If so, 5965 * just move to the next segment with pages. 5966 */ 5967 if (new_pp >= seg->epages || seg->pages_base == seg->pages_end) { 5968 do { 5969 seg = seg->next; 5970 if (seg == NULL) 5971 seg = memsegs; 5972 } while (seg->pages_base == seg->pages_end); 5973 new_pp = seg->pages; 5974 *cookie = (void *)seg; 5975 } 5976 5977 return (new_pp); 5978 } 5979 5980 5981 /* 5982 * Returns next page in list. Note: this function wraps 5983 * to the first page in the list upon reaching the end 5984 * of the list. Callers should be aware of this fact. 5985 */ 5986 5987 /* We should change this be a #define */ 5988 5989 page_t * 5990 page_next(page_t *pp) 5991 { 5992 return (page_nextn(pp, 1)); 5993 } 5994 5995 page_t * 5996 page_first() 5997 { 5998 return ((page_t *)memsegs->pages); 5999 } 6000 6001 6002 /* 6003 * This routine is called at boot with the initial memory configuration 6004 * and when memory is added or removed. 6005 */ 6006 void 6007 build_pfn_hash() 6008 { 6009 pfn_t cur; 6010 pgcnt_t index; 6011 struct memseg *pseg; 6012 int i; 6013 6014 /* 6015 * Clear memseg_hash array. 6016 * Since memory add/delete is designed to operate concurrently 6017 * with normal operation, the hash rebuild must be able to run 6018 * concurrently with page_numtopp_nolock(). To support this 6019 * functionality, assignments to memseg_hash array members must 6020 * be done atomically. 6021 * 6022 * NOTE: bzero() does not currently guarantee this for kernel 6023 * threads, and cannot be used here. 6024 */ 6025 for (i = 0; i < N_MEM_SLOTS; i++) 6026 memseg_hash[i] = NULL; 6027 6028 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6029 6030 /* 6031 * Physmax is the last valid pfn. 6032 */ 6033 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6034 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6035 index = MEMSEG_PFN_HASH(pseg->pages_base); 6036 cur = pseg->pages_base; 6037 do { 6038 if (index >= N_MEM_SLOTS) 6039 index = MEMSEG_PFN_HASH(cur); 6040 6041 if (memseg_hash[index] == NULL || 6042 memseg_hash[index]->pages_base > pseg->pages_base) { 6043 memseg_hash[index] = pseg; 6044 hat_kpm_mseghash_update(index, pseg); 6045 } 6046 cur += mhash_per_slot; 6047 index++; 6048 } while (cur < pseg->pages_end); 6049 } 6050 } 6051 6052 /* 6053 * Return the pagenum for the pp 6054 */ 6055 pfn_t 6056 page_pptonum(page_t *pp) 6057 { 6058 return (pp->p_pagenum); 6059 } 6060 6061 /* 6062 * interface to the referenced and modified etc bits 6063 * in the PSM part of the page struct 6064 * when no locking is desired. 6065 */ 6066 void 6067 page_set_props(page_t *pp, uint_t flags) 6068 { 6069 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6070 pp->p_nrm |= (uchar_t)flags; 6071 } 6072 6073 void 6074 page_clr_all_props(page_t *pp) 6075 { 6076 pp->p_nrm = 0; 6077 } 6078 6079 /* 6080 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6081 */ 6082 int 6083 page_clear_lck_cow(page_t *pp, int adjust) 6084 { 6085 int f_amount; 6086 6087 ASSERT(PAGE_EXCL(pp)); 6088 6089 /* 6090 * The page_struct_lock need not be acquired here since 6091 * we require the caller hold the page exclusively locked. 6092 */ 6093 f_amount = 0; 6094 if (pp->p_lckcnt) { 6095 f_amount = 1; 6096 pp->p_lckcnt = 0; 6097 } 6098 if (pp->p_cowcnt) { 6099 f_amount += pp->p_cowcnt; 6100 pp->p_cowcnt = 0; 6101 } 6102 6103 if (adjust && f_amount) { 6104 mutex_enter(&freemem_lock); 6105 availrmem += f_amount; 6106 mutex_exit(&freemem_lock); 6107 } 6108 6109 return (f_amount); 6110 } 6111 6112 /* 6113 * The following functions is called from free_vp_pages() 6114 * for an inexact estimate of a newly free'd page... 6115 */ 6116 ulong_t 6117 page_share_cnt(page_t *pp) 6118 { 6119 return (hat_page_getshare(pp)); 6120 } 6121 6122 int 6123 page_isshared(page_t *pp) 6124 { 6125 return (hat_page_checkshare(pp, 1)); 6126 } 6127 6128 int 6129 page_isfree(page_t *pp) 6130 { 6131 return (PP_ISFREE(pp)); 6132 } 6133 6134 int 6135 page_isref(page_t *pp) 6136 { 6137 return (hat_page_getattr(pp, P_REF)); 6138 } 6139 6140 int 6141 page_ismod(page_t *pp) 6142 { 6143 return (hat_page_getattr(pp, P_MOD)); 6144 } 6145 6146 /* 6147 * The following code all currently relates to the page capture logic: 6148 * 6149 * This logic is used for cases where there is a desire to claim a certain 6150 * physical page in the system for the caller. As it may not be possible 6151 * to capture the page immediately, the p_toxic bits are used in the page 6152 * structure to indicate that someone wants to capture this page. When the 6153 * page gets unlocked, the toxic flag will be noted and an attempt to capture 6154 * the page will be made. If it is successful, the original callers callback 6155 * will be called with the page to do with it what they please. 6156 * 6157 * There is also an async thread which wakes up to attempt to capture 6158 * pages occasionally which have the capture bit set. All of the pages which 6159 * need to be captured asynchronously have been inserted into the 6160 * page_capture_hash and thus this thread walks that hash list. Items in the 6161 * hash have an expiration time so this thread handles that as well by removing 6162 * the item from the hash if it has expired. 6163 * 6164 * Some important things to note are: 6165 * - if the PR_CAPTURE bit is set on a page, then the page is in the 6166 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed 6167 * to set and clear this bit, and while the lock is held is the only time 6168 * you can add or remove an entry from the hash. 6169 * - the PR_CAPTURE bit can only be set and cleared while holding the 6170 * page_capture_hash_head.pchh_mutex 6171 * - the t_flag field of the thread struct is used with the T_CAPTURING 6172 * flag to prevent recursion while dealing with large pages. 6173 * - pages which need to be retired never expire on the page_capture_hash. 6174 */ 6175 6176 static void page_capture_thread(void); 6177 static kthread_t *pc_thread_id; 6178 kcondvar_t pc_cv; 6179 static kmutex_t pc_thread_mutex; 6180 static clock_t pc_thread_shortwait; 6181 static clock_t pc_thread_longwait; 6182 static int pc_thread_retry; 6183 6184 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS]; 6185 6186 /* Note that this is a circular linked list */ 6187 typedef struct page_capture_hash_bucket { 6188 page_t *pp; 6189 uchar_t szc; 6190 uchar_t pri; 6191 uint_t flags; 6192 clock_t expires; /* lbolt at which this request expires. */ 6193 void *datap; /* Cached data passed in for callback */ 6194 struct page_capture_hash_bucket *next; 6195 struct page_capture_hash_bucket *prev; 6196 } page_capture_hash_bucket_t; 6197 6198 #define PC_PRI_HI 0 /* capture now */ 6199 #define PC_PRI_LO 1 /* capture later */ 6200 #define PC_NUM_PRI 2 6201 6202 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI) 6203 6204 6205 /* 6206 * Each hash bucket will have it's own mutex and two lists which are: 6207 * active (0): represents requests which have not been processed by 6208 * the page_capture async thread yet. 6209 * walked (1): represents requests which have been processed by the 6210 * page_capture async thread within it's given walk of this bucket. 6211 * 6212 * These are all needed so that we can synchronize all async page_capture 6213 * events. When the async thread moves to a new bucket, it will append the 6214 * walked list to the active list and walk each item one at a time, moving it 6215 * from the active list to the walked list. Thus if there is an async request 6216 * outstanding for a given page, it will always be in one of the two lists. 6217 * New requests will always be added to the active list. 6218 * If we were not able to capture a page before the request expired, we'd free 6219 * up the request structure which would indicate to page_capture that there is 6220 * no longer a need for the given page, and clear the PR_CAPTURE flag if 6221 * possible. 6222 */ 6223 typedef struct page_capture_hash_head { 6224 kmutex_t pchh_mutex; 6225 uint_t num_pages[PC_NUM_PRI]; 6226 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */ 6227 } page_capture_hash_head_t; 6228 6229 #ifdef DEBUG 6230 #define NUM_PAGE_CAPTURE_BUCKETS 4 6231 #else 6232 #define NUM_PAGE_CAPTURE_BUCKETS 64 6233 #endif 6234 6235 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS]; 6236 6237 /* for now use a very simple hash based upon the size of a page struct */ 6238 #define PAGE_CAPTURE_HASH(pp) \ 6239 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1))) 6240 6241 extern pgcnt_t swapfs_minfree; 6242 6243 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap); 6244 6245 /* 6246 * a callback function is required for page capture requests. 6247 */ 6248 void 6249 page_capture_register_callback(uint_t index, clock_t duration, 6250 int (*cb_func)(page_t *, void *, uint_t)) 6251 { 6252 ASSERT(pc_cb[index].cb_active == 0); 6253 ASSERT(cb_func != NULL); 6254 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6255 pc_cb[index].duration = duration; 6256 pc_cb[index].cb_func = cb_func; 6257 pc_cb[index].cb_active = 1; 6258 rw_exit(&pc_cb[index].cb_rwlock); 6259 } 6260 6261 void 6262 page_capture_unregister_callback(uint_t index) 6263 { 6264 int i, j; 6265 struct page_capture_hash_bucket *bp1; 6266 struct page_capture_hash_bucket *bp2; 6267 struct page_capture_hash_bucket *head = NULL; 6268 uint_t flags = (1 << index); 6269 6270 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6271 ASSERT(pc_cb[index].cb_active == 1); 6272 pc_cb[index].duration = 0; /* Paranoia */ 6273 pc_cb[index].cb_func = NULL; /* Paranoia */ 6274 pc_cb[index].cb_active = 0; 6275 rw_exit(&pc_cb[index].cb_rwlock); 6276 6277 /* 6278 * Just move all the entries to a private list which we can walk 6279 * through without the need to hold any locks. 6280 * No more requests can get added to the hash lists for this consumer 6281 * as the cb_active field for the callback has been cleared. 6282 */ 6283 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 6284 mutex_enter(&page_capture_hash[i].pchh_mutex); 6285 for (j = 0; j < 2; j++) { 6286 bp1 = page_capture_hash[i].lists[j].next; 6287 /* walk through all but first (sentinel) element */ 6288 while (bp1 != &page_capture_hash[i].lists[j]) { 6289 bp2 = bp1; 6290 if (bp2->flags & flags) { 6291 bp1 = bp2->next; 6292 bp1->prev = bp2->prev; 6293 bp2->prev->next = bp1; 6294 bp2->next = head; 6295 head = bp2; 6296 /* 6297 * Clear the PR_CAPTURE bit as we 6298 * hold appropriate locks here. 6299 */ 6300 page_clrtoxic(head->pp, PR_CAPTURE); 6301 page_capture_hash[i]. 6302 num_pages[bp2->pri]--; 6303 continue; 6304 } 6305 bp1 = bp1->next; 6306 } 6307 } 6308 mutex_exit(&page_capture_hash[i].pchh_mutex); 6309 } 6310 6311 while (head != NULL) { 6312 bp1 = head; 6313 head = head->next; 6314 kmem_free(bp1, sizeof (*bp1)); 6315 } 6316 } 6317 6318 6319 /* 6320 * Find pp in the active list and move it to the walked list if it 6321 * exists. 6322 * Note that most often pp should be at the front of the active list 6323 * as it is currently used and thus there is no other sort of optimization 6324 * being done here as this is a linked list data structure. 6325 * Returns 1 on successful move or 0 if page could not be found. 6326 */ 6327 static int 6328 page_capture_move_to_walked(page_t *pp) 6329 { 6330 page_capture_hash_bucket_t *bp; 6331 int index; 6332 6333 index = PAGE_CAPTURE_HASH(pp); 6334 6335 mutex_enter(&page_capture_hash[index].pchh_mutex); 6336 bp = page_capture_hash[index].lists[0].next; 6337 while (bp != &page_capture_hash[index].lists[0]) { 6338 if (bp->pp == pp) { 6339 /* Remove from old list */ 6340 bp->next->prev = bp->prev; 6341 bp->prev->next = bp->next; 6342 6343 /* Add to new list */ 6344 bp->next = page_capture_hash[index].lists[1].next; 6345 bp->prev = &page_capture_hash[index].lists[1]; 6346 page_capture_hash[index].lists[1].next = bp; 6347 bp->next->prev = bp; 6348 6349 /* 6350 * There is a small probability of page on a free 6351 * list being retired while being allocated 6352 * and before P_RAF is set on it. The page may 6353 * end up marked as high priority request instead 6354 * of low priority request. 6355 * If P_RAF page is not marked as low priority request 6356 * change it to low priority request. 6357 */ 6358 page_capture_hash[index].num_pages[bp->pri]--; 6359 bp->pri = PAGE_CAPTURE_PRIO(pp); 6360 page_capture_hash[index].num_pages[bp->pri]++; 6361 mutex_exit(&page_capture_hash[index].pchh_mutex); 6362 return (1); 6363 } 6364 bp = bp->next; 6365 } 6366 mutex_exit(&page_capture_hash[index].pchh_mutex); 6367 return (0); 6368 } 6369 6370 /* 6371 * Add a new entry to the page capture hash. The only case where a new 6372 * entry is not added is when the page capture consumer is no longer registered. 6373 * In this case, we'll silently not add the page to the hash. We know that 6374 * page retire will always be registered for the case where we are currently 6375 * unretiring a page and thus there are no conflicts. 6376 */ 6377 static void 6378 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) 6379 { 6380 page_capture_hash_bucket_t *bp1; 6381 page_capture_hash_bucket_t *bp2; 6382 int index; 6383 int cb_index; 6384 int i; 6385 uchar_t pri; 6386 #ifdef DEBUG 6387 page_capture_hash_bucket_t *tp1; 6388 int l; 6389 #endif 6390 6391 ASSERT(!(flags & CAPTURE_ASYNC)); 6392 6393 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP); 6394 6395 bp1->pp = pp; 6396 bp1->szc = szc; 6397 bp1->flags = flags; 6398 bp1->datap = datap; 6399 6400 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6401 if ((flags >> cb_index) & 1) { 6402 break; 6403 } 6404 } 6405 6406 ASSERT(cb_index != PC_NUM_CALLBACKS); 6407 6408 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6409 if (pc_cb[cb_index].cb_active) { 6410 if (pc_cb[cb_index].duration == -1) { 6411 bp1->expires = (clock_t)-1; 6412 } else { 6413 bp1->expires = ddi_get_lbolt() + 6414 pc_cb[cb_index].duration; 6415 } 6416 } else { 6417 /* There's no callback registered so don't add to the hash */ 6418 rw_exit(&pc_cb[cb_index].cb_rwlock); 6419 kmem_free(bp1, sizeof (*bp1)); 6420 return; 6421 } 6422 6423 index = PAGE_CAPTURE_HASH(pp); 6424 6425 /* 6426 * Only allow capture flag to be modified under this mutex. 6427 * Prevents multiple entries for same page getting added. 6428 */ 6429 mutex_enter(&page_capture_hash[index].pchh_mutex); 6430 6431 /* 6432 * if not already on the hash, set capture bit and add to the hash 6433 */ 6434 if (!(pp->p_toxic & PR_CAPTURE)) { 6435 #ifdef DEBUG 6436 /* Check for duplicate entries */ 6437 for (l = 0; l < 2; l++) { 6438 tp1 = page_capture_hash[index].lists[l].next; 6439 while (tp1 != &page_capture_hash[index].lists[l]) { 6440 if (tp1->pp == pp) { 6441 panic("page pp 0x%p already on hash " 6442 "at 0x%p\n", 6443 (void *)pp, (void *)tp1); 6444 } 6445 tp1 = tp1->next; 6446 } 6447 } 6448 6449 #endif 6450 page_settoxic(pp, PR_CAPTURE); 6451 pri = PAGE_CAPTURE_PRIO(pp); 6452 bp1->pri = pri; 6453 bp1->next = page_capture_hash[index].lists[0].next; 6454 bp1->prev = &page_capture_hash[index].lists[0]; 6455 bp1->next->prev = bp1; 6456 page_capture_hash[index].lists[0].next = bp1; 6457 page_capture_hash[index].num_pages[pri]++; 6458 if (flags & CAPTURE_RETIRE) { 6459 page_retire_incr_pend_count(datap); 6460 } 6461 mutex_exit(&page_capture_hash[index].pchh_mutex); 6462 rw_exit(&pc_cb[cb_index].cb_rwlock); 6463 cv_signal(&pc_cv); 6464 return; 6465 } 6466 6467 /* 6468 * A page retire request will replace any other request. 6469 * A second physmem request which is for a different process than 6470 * the currently registered one will be dropped as there is 6471 * no way to hold the private data for both calls. 6472 * In the future, once there are more callers, this will have to 6473 * be worked out better as there needs to be private storage for 6474 * at least each type of caller (maybe have datap be an array of 6475 * *void's so that we can index based upon callers index). 6476 */ 6477 6478 /* walk hash list to update expire time */ 6479 for (i = 0; i < 2; i++) { 6480 bp2 = page_capture_hash[index].lists[i].next; 6481 while (bp2 != &page_capture_hash[index].lists[i]) { 6482 if (bp2->pp == pp) { 6483 if (flags & CAPTURE_RETIRE) { 6484 if (!(bp2->flags & CAPTURE_RETIRE)) { 6485 page_retire_incr_pend_count( 6486 datap); 6487 bp2->flags = flags; 6488 bp2->expires = bp1->expires; 6489 bp2->datap = datap; 6490 } 6491 } else { 6492 ASSERT(flags & CAPTURE_PHYSMEM); 6493 if (!(bp2->flags & CAPTURE_RETIRE) && 6494 (datap == bp2->datap)) { 6495 bp2->expires = bp1->expires; 6496 } 6497 } 6498 mutex_exit(&page_capture_hash[index]. 6499 pchh_mutex); 6500 rw_exit(&pc_cb[cb_index].cb_rwlock); 6501 kmem_free(bp1, sizeof (*bp1)); 6502 return; 6503 } 6504 bp2 = bp2->next; 6505 } 6506 } 6507 6508 /* 6509 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes 6510 * and thus it either has to be set or not set and can't change 6511 * while holding the mutex above. 6512 */ 6513 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", 6514 (void *)pp); 6515 } 6516 6517 /* 6518 * We have a page in our hands, lets try and make it ours by turning 6519 * it into a clean page like it had just come off the freelists. 6520 * 6521 * Returns 0 on success, with the page still EXCL locked. 6522 * On failure, the page will be unlocked, and returns EAGAIN 6523 */ 6524 static int 6525 page_capture_clean_page(page_t *pp) 6526 { 6527 page_t *newpp; 6528 int skip_unlock = 0; 6529 spgcnt_t count; 6530 page_t *tpp; 6531 int ret = 0; 6532 int extra; 6533 6534 ASSERT(PAGE_EXCL(pp)); 6535 ASSERT(!PP_RETIRED(pp)); 6536 ASSERT(curthread->t_flag & T_CAPTURING); 6537 6538 if (PP_ISFREE(pp)) { 6539 if (!page_reclaim(pp, NULL)) { 6540 skip_unlock = 1; 6541 ret = EAGAIN; 6542 goto cleanup; 6543 } 6544 ASSERT(pp->p_szc == 0); 6545 if (pp->p_vnode != NULL) { 6546 /* 6547 * Since this page came from the 6548 * cachelist, we must destroy the 6549 * old vnode association. 6550 */ 6551 page_hashout(pp, NULL); 6552 } 6553 goto cleanup; 6554 } 6555 6556 /* 6557 * If we know page_relocate will fail, skip it 6558 * It could still fail due to a UE on another page but we 6559 * can't do anything about that. 6560 */ 6561 if (pp->p_toxic & PR_UE) { 6562 goto skip_relocate; 6563 } 6564 6565 /* 6566 * It's possible that pages can not have a vnode as fsflush comes 6567 * through and cleans up these pages. It's ugly but that's how it is. 6568 */ 6569 if (pp->p_vnode == NULL) { 6570 goto skip_relocate; 6571 } 6572 6573 /* 6574 * Page was not free, so lets try to relocate it. 6575 * page_relocate only works with root pages, so if this is not a root 6576 * page, we need to demote it to try and relocate it. 6577 * Unfortunately this is the best we can do right now. 6578 */ 6579 newpp = NULL; 6580 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { 6581 if (page_try_demote_pages(pp) == 0) { 6582 ret = EAGAIN; 6583 goto cleanup; 6584 } 6585 } 6586 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); 6587 if (ret == 0) { 6588 page_t *npp; 6589 /* unlock the new page(s) */ 6590 while (count-- > 0) { 6591 ASSERT(newpp != NULL); 6592 npp = newpp; 6593 page_sub(&newpp, npp); 6594 page_unlock(npp); 6595 } 6596 ASSERT(newpp == NULL); 6597 /* 6598 * Check to see if the page we have is too large. 6599 * If so, demote it freeing up the extra pages. 6600 */ 6601 if (pp->p_szc > 0) { 6602 /* For now demote extra pages to szc == 0 */ 6603 extra = page_get_pagecnt(pp->p_szc) - 1; 6604 while (extra > 0) { 6605 tpp = pp->p_next; 6606 page_sub(&pp, tpp); 6607 tpp->p_szc = 0; 6608 page_free(tpp, 1); 6609 extra--; 6610 } 6611 /* Make sure to set our page to szc 0 as well */ 6612 ASSERT(pp->p_next == pp && pp->p_prev == pp); 6613 pp->p_szc = 0; 6614 } 6615 goto cleanup; 6616 } else if (ret == EIO) { 6617 ret = EAGAIN; 6618 goto cleanup; 6619 } else { 6620 /* 6621 * Need to reset return type as we failed to relocate the page 6622 * but that does not mean that some of the next steps will not 6623 * work. 6624 */ 6625 ret = 0; 6626 } 6627 6628 skip_relocate: 6629 6630 if (pp->p_szc > 0) { 6631 if (page_try_demote_pages(pp) == 0) { 6632 ret = EAGAIN; 6633 goto cleanup; 6634 } 6635 } 6636 6637 ASSERT(pp->p_szc == 0); 6638 6639 if (hat_ismod(pp)) { 6640 ret = EAGAIN; 6641 goto cleanup; 6642 } 6643 if (PP_ISKAS(pp)) { 6644 ret = EAGAIN; 6645 goto cleanup; 6646 } 6647 if (pp->p_lckcnt || pp->p_cowcnt) { 6648 ret = EAGAIN; 6649 goto cleanup; 6650 } 6651 6652 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 6653 ASSERT(!hat_page_is_mapped(pp)); 6654 6655 if (hat_ismod(pp)) { 6656 /* 6657 * This is a semi-odd case as the page is now modified but not 6658 * mapped as we just unloaded the mappings above. 6659 */ 6660 ret = EAGAIN; 6661 goto cleanup; 6662 } 6663 if (pp->p_vnode != NULL) { 6664 page_hashout(pp, NULL); 6665 } 6666 6667 /* 6668 * At this point, the page should be in a clean state and 6669 * we can do whatever we want with it. 6670 */ 6671 6672 cleanup: 6673 if (ret != 0) { 6674 if (!skip_unlock) { 6675 page_unlock(pp); 6676 } 6677 } else { 6678 ASSERT(pp->p_szc == 0); 6679 ASSERT(PAGE_EXCL(pp)); 6680 6681 pp->p_next = pp; 6682 pp->p_prev = pp; 6683 } 6684 return (ret); 6685 } 6686 6687 /* 6688 * Various callers of page_trycapture() can have different restrictions upon 6689 * what memory they have access to. 6690 * Returns 0 on success, with the following error codes on failure: 6691 * EPERM - The requested page is long term locked, and thus repeated 6692 * requests to capture this page will likely fail. 6693 * ENOMEM - There was not enough free memory in the system to safely 6694 * map the requested page. 6695 * ENOENT - The requested page was inside the kernel cage, and the 6696 * PHYSMEM_CAGE flag was not set. 6697 */ 6698 int 6699 page_capture_pre_checks(page_t *pp, uint_t flags) 6700 { 6701 ASSERT(pp != NULL); 6702 6703 #if defined(__sparc) 6704 if (pp->p_vnode == &promvp) { 6705 return (EPERM); 6706 } 6707 6708 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) && 6709 (flags & CAPTURE_PHYSMEM)) { 6710 return (ENOENT); 6711 } 6712 6713 if (PP_ISNORELOCKERNEL(pp)) { 6714 return (EPERM); 6715 } 6716 #else 6717 if (PP_ISKAS(pp)) { 6718 return (EPERM); 6719 } 6720 #endif /* __sparc */ 6721 6722 /* only physmem currently has the restrictions checked below */ 6723 if (!(flags & CAPTURE_PHYSMEM)) { 6724 return (0); 6725 } 6726 6727 if (availrmem < swapfs_minfree) { 6728 /* 6729 * We won't try to capture this page as we are 6730 * running low on memory. 6731 */ 6732 return (ENOMEM); 6733 } 6734 return (0); 6735 } 6736 6737 /* 6738 * Once we have a page in our mits, go ahead and complete the capture 6739 * operation. 6740 * Returns 1 on failure where page is no longer needed 6741 * Returns 0 on success 6742 * Returns -1 if there was a transient failure. 6743 * Failure cases must release the SE_EXCL lock on pp (usually via page_free). 6744 */ 6745 int 6746 page_capture_take_action(page_t *pp, uint_t flags, void *datap) 6747 { 6748 int cb_index; 6749 int ret = 0; 6750 page_capture_hash_bucket_t *bp1; 6751 page_capture_hash_bucket_t *bp2; 6752 int index; 6753 int found = 0; 6754 int i; 6755 6756 ASSERT(PAGE_EXCL(pp)); 6757 ASSERT(curthread->t_flag & T_CAPTURING); 6758 6759 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6760 if ((flags >> cb_index) & 1) { 6761 break; 6762 } 6763 } 6764 ASSERT(cb_index < PC_NUM_CALLBACKS); 6765 6766 /* 6767 * Remove the entry from the page_capture hash, but don't free it yet 6768 * as we may need to put it back. 6769 * Since we own the page at this point in time, we should find it 6770 * in the hash if this is an ASYNC call. If we don't it's likely 6771 * that the page_capture_async() thread decided that this request 6772 * had expired, in which case we just continue on. 6773 */ 6774 if (flags & CAPTURE_ASYNC) { 6775 6776 index = PAGE_CAPTURE_HASH(pp); 6777 6778 mutex_enter(&page_capture_hash[index].pchh_mutex); 6779 for (i = 0; i < 2 && !found; i++) { 6780 bp1 = page_capture_hash[index].lists[i].next; 6781 while (bp1 != &page_capture_hash[index].lists[i]) { 6782 if (bp1->pp == pp) { 6783 bp1->next->prev = bp1->prev; 6784 bp1->prev->next = bp1->next; 6785 page_capture_hash[index]. 6786 num_pages[bp1->pri]--; 6787 page_clrtoxic(pp, PR_CAPTURE); 6788 found = 1; 6789 break; 6790 } 6791 bp1 = bp1->next; 6792 } 6793 } 6794 mutex_exit(&page_capture_hash[index].pchh_mutex); 6795 } 6796 6797 /* Synchronize with the unregister func. */ 6798 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6799 if (!pc_cb[cb_index].cb_active) { 6800 page_free(pp, 1); 6801 rw_exit(&pc_cb[cb_index].cb_rwlock); 6802 if (found) { 6803 kmem_free(bp1, sizeof (*bp1)); 6804 } 6805 return (1); 6806 } 6807 6808 /* 6809 * We need to remove the entry from the page capture hash and turn off 6810 * the PR_CAPTURE bit before calling the callback. We'll need to cache 6811 * the entry here, and then based upon the return value, cleanup 6812 * appropriately or re-add it to the hash, making sure that someone else 6813 * hasn't already done so. 6814 * It should be rare for the callback to fail and thus it's ok for 6815 * the failure path to be a bit complicated as the success path is 6816 * cleaner and the locking rules are easier to follow. 6817 */ 6818 6819 ret = pc_cb[cb_index].cb_func(pp, datap, flags); 6820 6821 rw_exit(&pc_cb[cb_index].cb_rwlock); 6822 6823 /* 6824 * If this was an ASYNC request, we need to cleanup the hash if the 6825 * callback was successful or if the request was no longer valid. 6826 * For non-ASYNC requests, we return failure to map and the caller 6827 * will take care of adding the request to the hash. 6828 * Note also that the callback itself is responsible for the page 6829 * at this point in time in terms of locking ... The most common 6830 * case for the failure path should just be a page_free. 6831 */ 6832 if (ret >= 0) { 6833 if (found) { 6834 if (bp1->flags & CAPTURE_RETIRE) { 6835 page_retire_decr_pend_count(datap); 6836 } 6837 kmem_free(bp1, sizeof (*bp1)); 6838 } 6839 return (ret); 6840 } 6841 if (!found) { 6842 return (ret); 6843 } 6844 6845 ASSERT(flags & CAPTURE_ASYNC); 6846 6847 /* 6848 * Check for expiration time first as we can just free it up if it's 6849 * expired. 6850 */ 6851 if (ddi_get_lbolt() > bp1->expires && bp1->expires != -1) { 6852 kmem_free(bp1, sizeof (*bp1)); 6853 return (ret); 6854 } 6855 6856 /* 6857 * The callback failed and there used to be an entry in the hash for 6858 * this page, so we need to add it back to the hash. 6859 */ 6860 mutex_enter(&page_capture_hash[index].pchh_mutex); 6861 if (!(pp->p_toxic & PR_CAPTURE)) { 6862 /* just add bp1 back to head of walked list */ 6863 page_settoxic(pp, PR_CAPTURE); 6864 bp1->next = page_capture_hash[index].lists[1].next; 6865 bp1->prev = &page_capture_hash[index].lists[1]; 6866 bp1->next->prev = bp1; 6867 bp1->pri = PAGE_CAPTURE_PRIO(pp); 6868 page_capture_hash[index].lists[1].next = bp1; 6869 page_capture_hash[index].num_pages[bp1->pri]++; 6870 mutex_exit(&page_capture_hash[index].pchh_mutex); 6871 return (ret); 6872 } 6873 6874 /* 6875 * Otherwise there was a new capture request added to list 6876 * Need to make sure that our original data is represented if 6877 * appropriate. 6878 */ 6879 for (i = 0; i < 2; i++) { 6880 bp2 = page_capture_hash[index].lists[i].next; 6881 while (bp2 != &page_capture_hash[index].lists[i]) { 6882 if (bp2->pp == pp) { 6883 if (bp1->flags & CAPTURE_RETIRE) { 6884 if (!(bp2->flags & CAPTURE_RETIRE)) { 6885 bp2->szc = bp1->szc; 6886 bp2->flags = bp1->flags; 6887 bp2->expires = bp1->expires; 6888 bp2->datap = bp1->datap; 6889 } 6890 } else { 6891 ASSERT(bp1->flags & CAPTURE_PHYSMEM); 6892 if (!(bp2->flags & CAPTURE_RETIRE)) { 6893 bp2->szc = bp1->szc; 6894 bp2->flags = bp1->flags; 6895 bp2->expires = bp1->expires; 6896 bp2->datap = bp1->datap; 6897 } 6898 } 6899 page_capture_hash[index].num_pages[bp2->pri]--; 6900 bp2->pri = PAGE_CAPTURE_PRIO(pp); 6901 page_capture_hash[index].num_pages[bp2->pri]++; 6902 mutex_exit(&page_capture_hash[index]. 6903 pchh_mutex); 6904 kmem_free(bp1, sizeof (*bp1)); 6905 return (ret); 6906 } 6907 bp2 = bp2->next; 6908 } 6909 } 6910 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp); 6911 /*NOTREACHED*/ 6912 } 6913 6914 /* 6915 * Try to capture the given page for the caller specified in the flags 6916 * parameter. The page will either be captured and handed over to the 6917 * appropriate callback, or will be queued up in the page capture hash 6918 * to be captured asynchronously. 6919 * If the current request is due to an async capture, the page must be 6920 * exclusively locked before calling this function. 6921 * Currently szc must be 0 but in the future this should be expandable to 6922 * other page sizes. 6923 * Returns 0 on success, with the following error codes on failure: 6924 * EPERM - The requested page is long term locked, and thus repeated 6925 * requests to capture this page will likely fail. 6926 * ENOMEM - There was not enough free memory in the system to safely 6927 * map the requested page. 6928 * ENOENT - The requested page was inside the kernel cage, and the 6929 * CAPTURE_GET_CAGE flag was not set. 6930 * EAGAIN - The requested page could not be capturead at this point in 6931 * time but future requests will likely work. 6932 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag 6933 * was not set. 6934 */ 6935 int 6936 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 6937 { 6938 int ret; 6939 int cb_index; 6940 6941 if (flags & CAPTURE_ASYNC) { 6942 ASSERT(PAGE_EXCL(pp)); 6943 goto async; 6944 } 6945 6946 /* Make sure there's enough availrmem ... */ 6947 ret = page_capture_pre_checks(pp, flags); 6948 if (ret != 0) { 6949 return (ret); 6950 } 6951 6952 if (!page_trylock(pp, SE_EXCL)) { 6953 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6954 if ((flags >> cb_index) & 1) { 6955 break; 6956 } 6957 } 6958 ASSERT(cb_index < PC_NUM_CALLBACKS); 6959 ret = EAGAIN; 6960 /* Special case for retired pages */ 6961 if (PP_RETIRED(pp)) { 6962 if (flags & CAPTURE_GET_RETIRED) { 6963 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { 6964 /* 6965 * Need to set capture bit and add to 6966 * hash so that the page will be 6967 * retired when freed. 6968 */ 6969 page_capture_add_hash(pp, szc, 6970 CAPTURE_RETIRE, NULL); 6971 ret = 0; 6972 goto own_page; 6973 } 6974 } else { 6975 return (EBUSY); 6976 } 6977 } 6978 page_capture_add_hash(pp, szc, flags, datap); 6979 return (ret); 6980 } 6981 6982 async: 6983 ASSERT(PAGE_EXCL(pp)); 6984 6985 /* Need to check for physmem async requests that availrmem is sane */ 6986 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) == 6987 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) && 6988 (availrmem < swapfs_minfree)) { 6989 page_unlock(pp); 6990 return (ENOMEM); 6991 } 6992 6993 ret = page_capture_clean_page(pp); 6994 6995 if (ret != 0) { 6996 /* We failed to get the page, so lets add it to the hash */ 6997 if (!(flags & CAPTURE_ASYNC)) { 6998 page_capture_add_hash(pp, szc, flags, datap); 6999 } 7000 return (ret); 7001 } 7002 7003 own_page: 7004 ASSERT(PAGE_EXCL(pp)); 7005 ASSERT(pp->p_szc == 0); 7006 7007 /* Call the callback */ 7008 ret = page_capture_take_action(pp, flags, datap); 7009 7010 if (ret == 0) { 7011 return (0); 7012 } 7013 7014 /* 7015 * Note that in the failure cases from page_capture_take_action, the 7016 * EXCL lock will have already been dropped. 7017 */ 7018 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) { 7019 page_capture_add_hash(pp, szc, flags, datap); 7020 } 7021 return (EAGAIN); 7022 } 7023 7024 int 7025 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 7026 { 7027 int ret; 7028 7029 curthread->t_flag |= T_CAPTURING; 7030 ret = page_itrycapture(pp, szc, flags, datap); 7031 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */ 7032 return (ret); 7033 } 7034 7035 /* 7036 * When unlocking a page which has the PR_CAPTURE bit set, this routine 7037 * gets called to try and capture the page. 7038 */ 7039 void 7040 page_unlock_capture(page_t *pp) 7041 { 7042 page_capture_hash_bucket_t *bp; 7043 int index; 7044 int i; 7045 uint_t szc; 7046 uint_t flags = 0; 7047 void *datap; 7048 kmutex_t *mp; 7049 extern vnode_t retired_pages; 7050 7051 /* 7052 * We need to protect against a possible deadlock here where we own 7053 * the vnode page hash mutex and want to acquire it again as there 7054 * are locations in the code, where we unlock a page while holding 7055 * the mutex which can lead to the page being captured and eventually 7056 * end up here. As we may be hashing out the old page and hashing into 7057 * the retire vnode, we need to make sure we don't own them. 7058 * Other callbacks who do hash operations also need to make sure that 7059 * before they hashin to a vnode that they do not currently own the 7060 * vphm mutex otherwise there will be a panic. 7061 */ 7062 if (mutex_owned(page_vnode_mutex(&retired_pages))) { 7063 page_unlock_nocapture(pp); 7064 return; 7065 } 7066 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { 7067 page_unlock_nocapture(pp); 7068 return; 7069 } 7070 7071 index = PAGE_CAPTURE_HASH(pp); 7072 7073 mp = &page_capture_hash[index].pchh_mutex; 7074 mutex_enter(mp); 7075 for (i = 0; i < 2; i++) { 7076 bp = page_capture_hash[index].lists[i].next; 7077 while (bp != &page_capture_hash[index].lists[i]) { 7078 if (bp->pp == pp) { 7079 szc = bp->szc; 7080 flags = bp->flags | CAPTURE_ASYNC; 7081 datap = bp->datap; 7082 mutex_exit(mp); 7083 (void) page_trycapture(pp, szc, flags, datap); 7084 return; 7085 } 7086 bp = bp->next; 7087 } 7088 } 7089 7090 /* Failed to find page in hash so clear flags and unlock it. */ 7091 page_clrtoxic(pp, PR_CAPTURE); 7092 page_unlock(pp); 7093 7094 mutex_exit(mp); 7095 } 7096 7097 void 7098 page_capture_init() 7099 { 7100 int i; 7101 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7102 page_capture_hash[i].lists[0].next = 7103 &page_capture_hash[i].lists[0]; 7104 page_capture_hash[i].lists[0].prev = 7105 &page_capture_hash[i].lists[0]; 7106 page_capture_hash[i].lists[1].next = 7107 &page_capture_hash[i].lists[1]; 7108 page_capture_hash[i].lists[1].prev = 7109 &page_capture_hash[i].lists[1]; 7110 } 7111 7112 pc_thread_shortwait = 23 * hz; 7113 pc_thread_longwait = 1201 * hz; 7114 pc_thread_retry = 3; 7115 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL); 7116 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL); 7117 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0, 7118 TS_RUN, minclsyspri); 7119 } 7120 7121 /* 7122 * It is necessary to scrub any failing pages prior to reboot in order to 7123 * prevent a latent error trap from occurring on the next boot. 7124 */ 7125 void 7126 page_retire_mdboot() 7127 { 7128 page_t *pp; 7129 int i, j; 7130 page_capture_hash_bucket_t *bp; 7131 uchar_t pri; 7132 7133 /* walk lists looking for pages to scrub */ 7134 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7135 for (pri = 0; pri < PC_NUM_PRI; pri++) { 7136 if (page_capture_hash[i].num_pages[pri] != 0) { 7137 break; 7138 } 7139 } 7140 if (pri == PC_NUM_PRI) 7141 continue; 7142 7143 mutex_enter(&page_capture_hash[i].pchh_mutex); 7144 7145 for (j = 0; j < 2; j++) { 7146 bp = page_capture_hash[i].lists[j].next; 7147 while (bp != &page_capture_hash[i].lists[j]) { 7148 pp = bp->pp; 7149 if (PP_TOXIC(pp)) { 7150 if (page_trylock(pp, SE_EXCL)) { 7151 PP_CLRFREE(pp); 7152 pagescrub(pp, 0, PAGESIZE); 7153 page_unlock(pp); 7154 } 7155 } 7156 bp = bp->next; 7157 } 7158 } 7159 mutex_exit(&page_capture_hash[i].pchh_mutex); 7160 } 7161 } 7162 7163 /* 7164 * Walk the page_capture_hash trying to capture pages and also cleanup old 7165 * entries which have expired. 7166 */ 7167 void 7168 page_capture_async() 7169 { 7170 page_t *pp; 7171 int i; 7172 int ret; 7173 page_capture_hash_bucket_t *bp1, *bp2; 7174 uint_t szc; 7175 uint_t flags; 7176 void *datap; 7177 uchar_t pri; 7178 7179 /* If there are outstanding pages to be captured, get to work */ 7180 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7181 for (pri = 0; pri < PC_NUM_PRI; pri++) { 7182 if (page_capture_hash[i].num_pages[pri] != 0) 7183 break; 7184 } 7185 if (pri == PC_NUM_PRI) 7186 continue; 7187 7188 /* Append list 1 to list 0 and then walk through list 0 */ 7189 mutex_enter(&page_capture_hash[i].pchh_mutex); 7190 bp1 = &page_capture_hash[i].lists[1]; 7191 bp2 = bp1->next; 7192 if (bp1 != bp2) { 7193 bp1->prev->next = page_capture_hash[i].lists[0].next; 7194 bp2->prev = &page_capture_hash[i].lists[0]; 7195 page_capture_hash[i].lists[0].next->prev = bp1->prev; 7196 page_capture_hash[i].lists[0].next = bp2; 7197 bp1->next = bp1; 7198 bp1->prev = bp1; 7199 } 7200 7201 /* list[1] will be empty now */ 7202 7203 bp1 = page_capture_hash[i].lists[0].next; 7204 while (bp1 != &page_capture_hash[i].lists[0]) { 7205 /* Check expiration time */ 7206 if ((ddi_get_lbolt() > bp1->expires && 7207 bp1->expires != -1) || 7208 page_deleted(bp1->pp)) { 7209 page_capture_hash[i].lists[0].next = bp1->next; 7210 bp1->next->prev = 7211 &page_capture_hash[i].lists[0]; 7212 page_capture_hash[i].num_pages[bp1->pri]--; 7213 7214 /* 7215 * We can safely remove the PR_CAPTURE bit 7216 * without holding the EXCL lock on the page 7217 * as the PR_CAPTURE bit requres that the 7218 * page_capture_hash[].pchh_mutex be held 7219 * to modify it. 7220 */ 7221 page_clrtoxic(bp1->pp, PR_CAPTURE); 7222 mutex_exit(&page_capture_hash[i].pchh_mutex); 7223 kmem_free(bp1, sizeof (*bp1)); 7224 mutex_enter(&page_capture_hash[i].pchh_mutex); 7225 bp1 = page_capture_hash[i].lists[0].next; 7226 continue; 7227 } 7228 pp = bp1->pp; 7229 szc = bp1->szc; 7230 flags = bp1->flags; 7231 datap = bp1->datap; 7232 mutex_exit(&page_capture_hash[i].pchh_mutex); 7233 if (page_trylock(pp, SE_EXCL)) { 7234 ret = page_trycapture(pp, szc, 7235 flags | CAPTURE_ASYNC, datap); 7236 } else { 7237 ret = 1; /* move to walked hash */ 7238 } 7239 7240 if (ret != 0) { 7241 /* Move to walked hash */ 7242 (void) page_capture_move_to_walked(pp); 7243 } 7244 mutex_enter(&page_capture_hash[i].pchh_mutex); 7245 bp1 = page_capture_hash[i].lists[0].next; 7246 } 7247 7248 mutex_exit(&page_capture_hash[i].pchh_mutex); 7249 } 7250 } 7251 7252 /* 7253 * This function is called by the page_capture_thread, and is needed in 7254 * in order to initiate aio cleanup, so that pages used in aio 7255 * will be unlocked and subsequently retired by page_capture_thread. 7256 */ 7257 static int 7258 do_aio_cleanup(void) 7259 { 7260 proc_t *procp; 7261 int (*aio_cleanup_dr_delete_memory)(proc_t *); 7262 int cleaned = 0; 7263 7264 if (modload("sys", "kaio") == -1) { 7265 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio"); 7266 return (0); 7267 } 7268 /* 7269 * We use the aio_cleanup_dr_delete_memory function to 7270 * initiate the actual clean up; this function will wake 7271 * up the per-process aio_cleanup_thread. 7272 */ 7273 aio_cleanup_dr_delete_memory = (int (*)(proc_t *)) 7274 modgetsymvalue("aio_cleanup_dr_delete_memory", 0); 7275 if (aio_cleanup_dr_delete_memory == NULL) { 7276 cmn_err(CE_WARN, 7277 "aio_cleanup_dr_delete_memory not found in kaio"); 7278 return (0); 7279 } 7280 mutex_enter(&pidlock); 7281 for (procp = practive; (procp != NULL); procp = procp->p_next) { 7282 mutex_enter(&procp->p_lock); 7283 if (procp->p_aio != NULL) { 7284 /* cleanup proc's outstanding kaio */ 7285 cleaned += (*aio_cleanup_dr_delete_memory)(procp); 7286 } 7287 mutex_exit(&procp->p_lock); 7288 } 7289 mutex_exit(&pidlock); 7290 return (cleaned); 7291 } 7292 7293 /* 7294 * helper function for page_capture_thread 7295 */ 7296 static void 7297 page_capture_handle_outstanding(void) 7298 { 7299 int ntry; 7300 7301 /* Reap pages before attempting capture pages */ 7302 kmem_reap(); 7303 7304 if ((page_retire_pend_count() > page_retire_pend_kas_count()) && 7305 hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 7306 /* 7307 * Note: Purging only for platforms that support 7308 * ISM hat_pageunload() - mainly SPARC. On x86/x64 7309 * platforms ISM pages SE_SHARED locked until destroyed. 7310 */ 7311 7312 /* disable and purge seg_pcache */ 7313 (void) seg_p_disable(); 7314 for (ntry = 0; ntry < pc_thread_retry; ntry++) { 7315 if (!page_retire_pend_count()) 7316 break; 7317 if (do_aio_cleanup()) { 7318 /* 7319 * allow the apps cleanup threads 7320 * to run 7321 */ 7322 delay(pc_thread_shortwait); 7323 } 7324 page_capture_async(); 7325 } 7326 /* reenable seg_pcache */ 7327 seg_p_enable(); 7328 7329 /* completed what can be done. break out */ 7330 return; 7331 } 7332 7333 /* 7334 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap 7335 * and then attempt to capture. 7336 */ 7337 seg_preap(); 7338 page_capture_async(); 7339 } 7340 7341 /* 7342 * The page_capture_thread loops forever, looking to see if there are 7343 * pages still waiting to be captured. 7344 */ 7345 static void 7346 page_capture_thread(void) 7347 { 7348 callb_cpr_t c; 7349 int i; 7350 int high_pri_pages; 7351 int low_pri_pages; 7352 clock_t timeout; 7353 7354 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture"); 7355 7356 mutex_enter(&pc_thread_mutex); 7357 for (;;) { 7358 high_pri_pages = 0; 7359 low_pri_pages = 0; 7360 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7361 high_pri_pages += 7362 page_capture_hash[i].num_pages[PC_PRI_HI]; 7363 low_pri_pages += 7364 page_capture_hash[i].num_pages[PC_PRI_LO]; 7365 } 7366 7367 timeout = pc_thread_longwait; 7368 if (high_pri_pages != 0) { 7369 timeout = pc_thread_shortwait; 7370 page_capture_handle_outstanding(); 7371 } else if (low_pri_pages != 0) { 7372 page_capture_async(); 7373 } 7374 CALLB_CPR_SAFE_BEGIN(&c); 7375 (void) cv_reltimedwait(&pc_cv, &pc_thread_mutex, 7376 timeout, TR_CLOCK_TICK); 7377 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7378 } 7379 /*NOTREACHED*/ 7380 } 7381 /* 7382 * Attempt to locate a bucket that has enough pages to satisfy the request. 7383 * The initial check is done without the lock to avoid unneeded contention. 7384 * The function returns 1 if enough pages were found, else 0 if it could not 7385 * find enough pages in a bucket. 7386 */ 7387 static int 7388 pcf_decrement_bucket(pgcnt_t npages) 7389 { 7390 struct pcf *p; 7391 struct pcf *q; 7392 int i; 7393 7394 p = &pcf[PCF_INDEX()]; 7395 q = &pcf[pcf_fanout]; 7396 for (i = 0; i < pcf_fanout; i++) { 7397 if (p->pcf_count > npages) { 7398 /* 7399 * a good one to try. 7400 */ 7401 mutex_enter(&p->pcf_lock); 7402 if (p->pcf_count > npages) { 7403 p->pcf_count -= (uint_t)npages; 7404 /* 7405 * freemem is not protected by any lock. 7406 * Thus, we cannot have any assertion 7407 * containing freemem here. 7408 */ 7409 freemem -= npages; 7410 mutex_exit(&p->pcf_lock); 7411 return (1); 7412 } 7413 mutex_exit(&p->pcf_lock); 7414 } 7415 p++; 7416 if (p >= q) { 7417 p = pcf; 7418 } 7419 } 7420 return (0); 7421 } 7422 7423 /* 7424 * Arguments: 7425 * pcftotal_ret: If the value is not NULL and we have walked all the 7426 * buckets but did not find enough pages then it will 7427 * be set to the total number of pages in all the pcf 7428 * buckets. 7429 * npages: Is the number of pages we have been requested to 7430 * find. 7431 * unlock: If set to 0 we will leave the buckets locked if the 7432 * requested number of pages are not found. 7433 * 7434 * Go and try to satisfy the page request from any number of buckets. 7435 * This can be a very expensive operation as we have to lock the buckets 7436 * we are checking (and keep them locked), starting at bucket 0. 7437 * 7438 * The function returns 1 if enough pages were found, else 0 if it could not 7439 * find enough pages in the buckets. 7440 * 7441 */ 7442 static int 7443 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock) 7444 { 7445 struct pcf *p; 7446 pgcnt_t pcftotal; 7447 int i; 7448 7449 p = pcf; 7450 /* try to collect pages from several pcf bins */ 7451 for (pcftotal = 0, i = 0; i < pcf_fanout; i++) { 7452 mutex_enter(&p->pcf_lock); 7453 pcftotal += p->pcf_count; 7454 if (pcftotal >= npages) { 7455 /* 7456 * Wow! There are enough pages laying around 7457 * to satisfy the request. Do the accounting, 7458 * drop the locks we acquired, and go back. 7459 * 7460 * freemem is not protected by any lock. So, 7461 * we cannot have any assertion containing 7462 * freemem. 7463 */ 7464 freemem -= npages; 7465 while (p >= pcf) { 7466 if (p->pcf_count <= npages) { 7467 npages -= p->pcf_count; 7468 p->pcf_count = 0; 7469 } else { 7470 p->pcf_count -= (uint_t)npages; 7471 npages = 0; 7472 } 7473 mutex_exit(&p->pcf_lock); 7474 p--; 7475 } 7476 ASSERT(npages == 0); 7477 return (1); 7478 } 7479 p++; 7480 } 7481 if (unlock) { 7482 /* failed to collect pages - release the locks */ 7483 while (--p >= pcf) { 7484 mutex_exit(&p->pcf_lock); 7485 } 7486 } 7487 if (pcftotal_ret != NULL) 7488 *pcftotal_ret = pcftotal; 7489 return (0); 7490 } 7491