1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #pragma ident "%Z%%M% %I% %E% SMI" 40 41 /* 42 * VM - anonymous pages. 43 * 44 * This layer sits immediately above the vm_swap layer. It manages 45 * physical pages that have no permanent identity in the file system 46 * name space, using the services of the vm_swap layer to allocate 47 * backing storage for these pages. Since these pages have no external 48 * identity, they are discarded when the last reference is removed. 49 * 50 * An important function of this layer is to manage low-level sharing 51 * of pages that are logically distinct but that happen to be 52 * physically identical (e.g., the corresponding pages of the processes 53 * resulting from a fork before one process or the other changes their 54 * contents). This pseudo-sharing is present only as an optimization 55 * and is not to be confused with true sharing in which multiple 56 * address spaces deliberately contain references to the same object; 57 * such sharing is managed at a higher level. 58 * 59 * The key data structure here is the anon struct, which contains a 60 * reference count for its associated physical page and a hint about 61 * the identity of that page. Anon structs typically live in arrays, 62 * with an instance's position in its array determining where the 63 * corresponding backing storage is allocated; however, the swap_xlate() 64 * routine abstracts away this representation information so that the 65 * rest of the anon layer need not know it. (See the swap layer for 66 * more details on anon struct layout.) 67 * 68 * In the future versions of the system, the association between an 69 * anon struct and its position on backing store will change so that 70 * we don't require backing store all anonymous pages in the system. 71 * This is important for consideration for large memory systems. 72 * We can also use this technique to delay binding physical locations 73 * to anonymous pages until pageout/swapout time where we can make 74 * smarter allocation decisions to improve anonymous klustering. 75 * 76 * Many of the routines defined here take a (struct anon **) argument, 77 * which allows the code at this level to manage anon pages directly, 78 * so that callers can regard anon structs as opaque objects and not be 79 * concerned with assigning or inspecting their contents. 80 * 81 * Clients of this layer refer to anon pages indirectly. That is, they 82 * maintain arrays of pointers to anon structs rather than maintaining 83 * anon structs themselves. The (struct anon **) arguments mentioned 84 * above are pointers to entries in these arrays. It is these arrays 85 * that capture the mapping between offsets within a given segment and 86 * the corresponding anonymous backing storage address. 87 */ 88 89 #ifdef DEBUG 90 #define ANON_DEBUG 91 #endif 92 93 #include <sys/types.h> 94 #include <sys/t_lock.h> 95 #include <sys/param.h> 96 #include <sys/systm.h> 97 #include <sys/mman.h> 98 #include <sys/cred.h> 99 #include <sys/thread.h> 100 #include <sys/vnode.h> 101 #include <sys/cpuvar.h> 102 #include <sys/swap.h> 103 #include <sys/cmn_err.h> 104 #include <sys/vtrace.h> 105 #include <sys/kmem.h> 106 #include <sys/sysmacros.h> 107 #include <sys/bitmap.h> 108 #include <sys/vmsystm.h> 109 #include <sys/debug.h> 110 #include <sys/fs/swapnode.h> 111 #include <sys/tnf_probe.h> 112 #include <sys/lgrp.h> 113 #include <sys/policy.h> 114 #include <sys/condvar_impl.h> 115 #include <sys/mutex_impl.h> 116 #include <sys/rctl.h> 117 118 #include <vm/as.h> 119 #include <vm/hat.h> 120 #include <vm/anon.h> 121 #include <vm/page.h> 122 #include <vm/vpage.h> 123 #include <vm/seg.h> 124 #include <vm/rm.h> 125 126 #include <fs/fs_subr.h> 127 128 struct vnode *anon_vp; 129 130 int anon_debug; 131 132 kmutex_t anoninfo_lock; 133 struct k_anoninfo k_anoninfo; 134 ani_free_t ani_free_pool[ANI_MAX_POOL]; 135 pad_mutex_t anon_array_lock[ANON_LOCKSIZE]; 136 kcondvar_t anon_array_cv[ANON_LOCKSIZE]; 137 138 /* 139 * Global hash table for (vp, off) -> anon slot 140 */ 141 extern int swap_maxcontig; 142 size_t anon_hash_size; 143 struct anon **anon_hash; 144 145 static struct kmem_cache *anon_cache; 146 static struct kmem_cache *anonmap_cache; 147 148 #ifdef VM_STATS 149 static struct anonvmstats_str { 150 ulong_t getpages[30]; 151 ulong_t privatepages[10]; 152 ulong_t demotepages[9]; 153 ulong_t decrefpages[9]; 154 ulong_t dupfillholes[4]; 155 ulong_t freepages[1]; 156 } anonvmstats; 157 #endif /* VM_STATS */ 158 159 160 /*ARGSUSED*/ 161 static int 162 anonmap_cache_constructor(void *buf, void *cdrarg, int kmflags) 163 { 164 struct anon_map *amp = buf; 165 166 rw_init(&->a_rwlock, NULL, RW_DEFAULT, NULL); 167 return (0); 168 } 169 170 /*ARGSUSED1*/ 171 static void 172 anonmap_cache_destructor(void *buf, void *cdrarg) 173 { 174 struct anon_map *amp = buf; 175 176 rw_destroy(&->a_rwlock); 177 } 178 179 kmutex_t anonhash_lock[AH_LOCK_SIZE]; 180 kmutex_t anonpages_hash_lock[AH_LOCK_SIZE]; 181 182 void 183 anon_init(void) 184 { 185 int i; 186 187 anon_hash_size = 1L << highbit(physmem / ANON_HASHAVELEN); 188 189 for (i = 0; i < AH_LOCK_SIZE; i++) { 190 mutex_init(&anonhash_lock[i], NULL, MUTEX_DEFAULT, NULL); 191 mutex_init(&anonpages_hash_lock[i], NULL, MUTEX_DEFAULT, NULL); 192 } 193 194 for (i = 0; i < ANON_LOCKSIZE; i++) { 195 mutex_init(&anon_array_lock[i].pad_mutex, NULL, 196 MUTEX_DEFAULT, NULL); 197 cv_init(&anon_array_cv[i], NULL, CV_DEFAULT, NULL); 198 } 199 200 anon_hash = (struct anon **) 201 kmem_zalloc(sizeof (struct anon *) * anon_hash_size, KM_SLEEP); 202 anon_cache = kmem_cache_create("anon_cache", sizeof (struct anon), 203 AN_CACHE_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 204 anonmap_cache = kmem_cache_create("anonmap_cache", 205 sizeof (struct anon_map), 0, 206 anonmap_cache_constructor, anonmap_cache_destructor, NULL, 207 NULL, NULL, 0); 208 swap_maxcontig = (1024 * 1024) >> PAGESHIFT; /* 1MB of pages */ 209 210 anon_vp = vn_alloc(KM_SLEEP); 211 vn_setops(anon_vp, swap_vnodeops); 212 anon_vp->v_type = VREG; 213 anon_vp->v_flag |= (VISSWAP|VISSWAPFS); 214 } 215 216 /* 217 * Global anon slot hash table manipulation. 218 */ 219 220 static void 221 anon_addhash(struct anon *ap) 222 { 223 int index; 224 225 ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)])); 226 index = ANON_HASH(ap->an_vp, ap->an_off); 227 ap->an_hash = anon_hash[index]; 228 anon_hash[index] = ap; 229 } 230 231 static void 232 anon_rmhash(struct anon *ap) 233 { 234 struct anon **app; 235 236 ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)])); 237 238 for (app = &anon_hash[ANON_HASH(ap->an_vp, ap->an_off)]; 239 *app; app = &((*app)->an_hash)) { 240 if (*app == ap) { 241 *app = ap->an_hash; 242 break; 243 } 244 } 245 } 246 247 /* 248 * The anon array interfaces. Functions allocating, 249 * freeing array of pointers, and returning/setting 250 * entries in the array of pointers for a given offset. 251 * 252 * Create the list of pointers 253 */ 254 struct anon_hdr * 255 anon_create(pgcnt_t npages, int flags) 256 { 257 struct anon_hdr *ahp; 258 ulong_t nchunks; 259 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 260 261 if ((ahp = kmem_zalloc(sizeof (struct anon_hdr), kmemflags)) == NULL) { 262 return (NULL); 263 } 264 265 mutex_init(&ahp->serial_lock, NULL, MUTEX_DEFAULT, NULL); 266 /* 267 * Single level case. 268 */ 269 ahp->size = npages; 270 if (npages <= ANON_CHUNK_SIZE || (flags & ANON_ALLOC_FORCE)) { 271 272 if (flags & ANON_ALLOC_FORCE) 273 ahp->flags |= ANON_ALLOC_FORCE; 274 275 ahp->array_chunk = kmem_zalloc( 276 ahp->size * sizeof (struct anon *), kmemflags); 277 278 if (ahp->array_chunk == NULL) { 279 kmem_free(ahp, sizeof (struct anon_hdr)); 280 return (NULL); 281 } 282 } else { 283 /* 284 * 2 Level case. 285 * anon hdr size needs to be rounded off to be a multiple 286 * of ANON_CHUNK_SIZE. This is important as various anon 287 * related functions depend on this. 288 * NOTE - 289 * anon_grow() makes anon hdr size a multiple of 290 * ANON_CHUNK_SIZE. 291 * amp size is <= anon hdr size. 292 * anon_index + seg_pgs <= anon hdr size. 293 */ 294 ahp->size = P2ROUNDUP(npages, ANON_CHUNK_SIZE); 295 nchunks = ahp->size >> ANON_CHUNK_SHIFT; 296 297 ahp->array_chunk = kmem_zalloc(nchunks * sizeof (ulong_t *), 298 kmemflags); 299 300 if (ahp->array_chunk == NULL) { 301 kmem_free(ahp, sizeof (struct anon_hdr)); 302 return (NULL); 303 } 304 } 305 return (ahp); 306 } 307 308 /* 309 * Free the array of pointers 310 */ 311 void 312 anon_release(struct anon_hdr *ahp, pgcnt_t npages) 313 { 314 ulong_t i; 315 void **ppp; 316 ulong_t nchunks; 317 318 ASSERT(npages <= ahp->size); 319 320 /* 321 * Single level case. 322 */ 323 if (npages <= ANON_CHUNK_SIZE || (ahp->flags & ANON_ALLOC_FORCE)) { 324 kmem_free(ahp->array_chunk, ahp->size * sizeof (struct anon *)); 325 } else { 326 /* 327 * 2 level case. 328 */ 329 nchunks = ahp->size >> ANON_CHUNK_SHIFT; 330 for (i = 0; i < nchunks; i++) { 331 ppp = &ahp->array_chunk[i]; 332 if (*ppp != NULL) 333 kmem_free(*ppp, PAGESIZE); 334 } 335 kmem_free(ahp->array_chunk, nchunks * sizeof (ulong_t *)); 336 } 337 mutex_destroy(&ahp->serial_lock); 338 kmem_free(ahp, sizeof (struct anon_hdr)); 339 } 340 341 /* 342 * Return the pointer from the list for a 343 * specified anon index. 344 */ 345 struct anon * 346 anon_get_ptr(struct anon_hdr *ahp, ulong_t an_idx) 347 { 348 struct anon **app; 349 350 ASSERT(an_idx < ahp->size); 351 352 /* 353 * Single level case. 354 */ 355 if ((ahp->size <= ANON_CHUNK_SIZE) || (ahp->flags & ANON_ALLOC_FORCE)) { 356 return ((struct anon *) 357 ((uintptr_t)ahp->array_chunk[an_idx] & ANON_PTRMASK)); 358 } else { 359 360 /* 361 * 2 level case. 362 */ 363 app = ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 364 if (app) { 365 return ((struct anon *) 366 ((uintptr_t)app[an_idx & ANON_CHUNK_OFF] & 367 ANON_PTRMASK)); 368 } else { 369 return (NULL); 370 } 371 } 372 } 373 374 /* 375 * Return the anon pointer for the first valid entry in the anon list, 376 * starting from the given index. 377 */ 378 struct anon * 379 anon_get_next_ptr(struct anon_hdr *ahp, ulong_t *index) 380 { 381 struct anon *ap; 382 struct anon **app; 383 ulong_t chunkoff; 384 ulong_t i; 385 ulong_t j; 386 pgcnt_t size; 387 388 i = *index; 389 size = ahp->size; 390 391 ASSERT(i < size); 392 393 if ((size <= ANON_CHUNK_SIZE) || (ahp->flags & ANON_ALLOC_FORCE)) { 394 /* 395 * 1 level case 396 */ 397 while (i < size) { 398 ap = (struct anon *) 399 ((uintptr_t)ahp->array_chunk[i] & ANON_PTRMASK); 400 if (ap) { 401 *index = i; 402 return (ap); 403 } 404 i++; 405 } 406 } else { 407 /* 408 * 2 level case 409 */ 410 chunkoff = i & ANON_CHUNK_OFF; 411 while (i < size) { 412 app = ahp->array_chunk[i >> ANON_CHUNK_SHIFT]; 413 if (app) 414 for (j = chunkoff; j < ANON_CHUNK_SIZE; j++) { 415 ap = (struct anon *) 416 ((uintptr_t)app[j] & ANON_PTRMASK); 417 if (ap) { 418 *index = i + (j - chunkoff); 419 return (ap); 420 } 421 } 422 chunkoff = 0; 423 i = (i + ANON_CHUNK_SIZE) & ~ANON_CHUNK_OFF; 424 } 425 } 426 *index = size; 427 return (NULL); 428 } 429 430 /* 431 * Set list entry with a given pointer for a specified offset 432 */ 433 int 434 anon_set_ptr(struct anon_hdr *ahp, ulong_t an_idx, struct anon *ap, int flags) 435 { 436 void **ppp; 437 struct anon **app; 438 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 439 uintptr_t *ap_addr; 440 441 ASSERT(an_idx < ahp->size); 442 443 /* 444 * Single level case. 445 */ 446 if (ahp->size <= ANON_CHUNK_SIZE || (ahp->flags & ANON_ALLOC_FORCE)) { 447 ap_addr = (uintptr_t *)&ahp->array_chunk[an_idx]; 448 } else { 449 450 /* 451 * 2 level case. 452 */ 453 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 454 455 ASSERT(ppp != NULL); 456 if (*ppp == NULL) { 457 mutex_enter(&ahp->serial_lock); 458 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 459 if (*ppp == NULL) { 460 *ppp = kmem_zalloc(PAGESIZE, kmemflags); 461 if (*ppp == NULL) { 462 mutex_exit(&ahp->serial_lock); 463 return (ENOMEM); 464 } 465 } 466 mutex_exit(&ahp->serial_lock); 467 } 468 app = *ppp; 469 ap_addr = (uintptr_t *)&app[an_idx & ANON_CHUNK_OFF]; 470 } 471 *ap_addr = (*ap_addr & ~ANON_PTRMASK) | (uintptr_t)ap; 472 return (0); 473 } 474 475 /* 476 * Copy anon array into a given new anon array 477 */ 478 int 479 anon_copy_ptr(struct anon_hdr *sahp, ulong_t s_idx, 480 struct anon_hdr *dahp, ulong_t d_idx, 481 pgcnt_t npages, int flags) 482 { 483 void **sapp, **dapp; 484 void *ap; 485 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 486 487 ASSERT((s_idx < sahp->size) && (d_idx < dahp->size)); 488 ASSERT((npages <= sahp->size) && (npages <= dahp->size)); 489 490 /* 491 * Both arrays are 1 level. 492 */ 493 if (((sahp->size <= ANON_CHUNK_SIZE) && 494 (dahp->size <= ANON_CHUNK_SIZE)) || 495 ((sahp->flags & ANON_ALLOC_FORCE) && 496 (dahp->flags & ANON_ALLOC_FORCE))) { 497 498 bcopy(&sahp->array_chunk[s_idx], &dahp->array_chunk[d_idx], 499 npages * sizeof (struct anon *)); 500 return (0); 501 } 502 503 /* 504 * Both arrays are 2 levels. 505 */ 506 if (sahp->size > ANON_CHUNK_SIZE && 507 dahp->size > ANON_CHUNK_SIZE && 508 ((sahp->flags & ANON_ALLOC_FORCE) == 0) && 509 ((dahp->flags & ANON_ALLOC_FORCE) == 0)) { 510 511 ulong_t sapidx, dapidx; 512 ulong_t *sap, *dap; 513 ulong_t chknp; 514 515 while (npages != 0) { 516 517 sapidx = s_idx & ANON_CHUNK_OFF; 518 dapidx = d_idx & ANON_CHUNK_OFF; 519 chknp = ANON_CHUNK_SIZE - MAX(sapidx, dapidx); 520 if (chknp > npages) 521 chknp = npages; 522 523 sapp = &sahp->array_chunk[s_idx >> ANON_CHUNK_SHIFT]; 524 if ((sap = *sapp) != NULL) { 525 dapp = &dahp->array_chunk[d_idx 526 >> ANON_CHUNK_SHIFT]; 527 if ((dap = *dapp) == NULL) { 528 *dapp = kmem_zalloc(PAGESIZE, 529 kmemflags); 530 if ((dap = *dapp) == NULL) 531 return (ENOMEM); 532 } 533 bcopy((sap + sapidx), (dap + dapidx), 534 chknp << ANON_PTRSHIFT); 535 } 536 s_idx += chknp; 537 d_idx += chknp; 538 npages -= chknp; 539 } 540 return (0); 541 } 542 543 /* 544 * At least one of the arrays is 2 level. 545 */ 546 while (npages--) { 547 if ((ap = anon_get_ptr(sahp, s_idx)) != NULL) { 548 ASSERT(!ANON_ISBUSY(anon_get_slot(sahp, s_idx))); 549 if (anon_set_ptr(dahp, d_idx, ap, flags) == ENOMEM) 550 return (ENOMEM); 551 } 552 s_idx++; 553 d_idx++; 554 } 555 return (0); 556 } 557 558 559 /* 560 * ANON_INITBUF is a convenience macro for anon_grow() below. It 561 * takes a buffer dst, which is at least as large as buffer src. It 562 * does a bcopy from src into dst, and then bzeros the extra bytes 563 * of dst. If tail is set, the data in src is tail aligned within 564 * dst instead of head aligned. 565 */ 566 567 #define ANON_INITBUF(src, srclen, dst, dstsize, tail) \ 568 if (tail) { \ 569 bzero((dst), (dstsize) - (srclen)); \ 570 bcopy((src), (char *)(dst) + (dstsize) - (srclen), (srclen)); \ 571 } else { \ 572 bcopy((src), (dst), (srclen)); \ 573 bzero((char *)(dst) + (srclen), (dstsize) - (srclen)); \ 574 } 575 576 #define ANON_1_LEVEL_INC (ANON_CHUNK_SIZE / 8) 577 #define ANON_2_LEVEL_INC (ANON_1_LEVEL_INC * ANON_CHUNK_SIZE) 578 579 /* 580 * anon_grow() is used to efficiently extend an existing anon array. 581 * startidx_p points to the index into the anon array of the first page 582 * that is in use. oldseg_pgs is the number of pages in use, starting at 583 * *startidx_p. newpages is the number of additional pages desired. 584 * 585 * If startidx_p == NULL, startidx is taken to be 0 and cannot be changed. 586 * 587 * The growth is done by creating a new top level of the anon array, 588 * and (if the array is 2-level) reusing the existing second level arrays. 589 * 590 * flags can be used to specify ANON_NOSLEEP and ANON_GROWDOWN. 591 * 592 * Returns the new number of pages in the anon array. 593 */ 594 pgcnt_t 595 anon_grow(struct anon_hdr *ahp, ulong_t *startidx_p, pgcnt_t oldseg_pgs, 596 pgcnt_t newseg_pgs, int flags) 597 { 598 ulong_t startidx = startidx_p ? *startidx_p : 0; 599 pgcnt_t oldamp_pgs = ahp->size, newamp_pgs; 600 pgcnt_t oelems, nelems, totpages; 601 void **level1; 602 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 603 int growdown = (flags & ANON_GROWDOWN); 604 size_t newarrsz, oldarrsz; 605 void *level2; 606 607 ASSERT(!(startidx_p == NULL && growdown)); 608 ASSERT(startidx + oldseg_pgs <= ahp->size); 609 610 /* 611 * Determine the total number of pages needed in the new 612 * anon array. If growing down, totpages is all pages from 613 * startidx through the end of the array, plus <newseg_pgs> 614 * pages. If growing up, keep all pages from page 0 through 615 * the last page currently in use, plus <newseg_pgs> pages. 616 */ 617 if (growdown) 618 totpages = oldamp_pgs - startidx + newseg_pgs; 619 else 620 totpages = startidx + oldseg_pgs + newseg_pgs; 621 622 /* If the array is already large enough, just return. */ 623 624 if (oldamp_pgs >= totpages) { 625 if (growdown) 626 *startidx_p = oldamp_pgs - totpages; 627 return (oldamp_pgs); 628 } 629 630 /* 631 * oldamp_pgs/newamp_pgs are the total numbers of pages represented 632 * by the corresponding arrays. 633 * oelems/nelems are the number of pointers in the top level arrays 634 * which may be either level 1 or level 2. 635 * Will the new anon array be one level or two levels? 636 */ 637 if (totpages <= ANON_CHUNK_SIZE || (ahp->flags & ANON_ALLOC_FORCE)) { 638 newamp_pgs = P2ROUNDUP(totpages, ANON_1_LEVEL_INC); 639 oelems = oldamp_pgs; 640 nelems = newamp_pgs; 641 } else { 642 newamp_pgs = P2ROUNDUP(totpages, ANON_2_LEVEL_INC); 643 oelems = (oldamp_pgs + ANON_CHUNK_OFF) >> ANON_CHUNK_SHIFT; 644 nelems = newamp_pgs >> ANON_CHUNK_SHIFT; 645 } 646 647 newarrsz = nelems * sizeof (void *); 648 level1 = kmem_alloc(newarrsz, kmemflags); 649 if (level1 == NULL) 650 return (0); 651 652 /* Are we converting from a one level to a two level anon array? */ 653 654 if (newamp_pgs > ANON_CHUNK_SIZE && oldamp_pgs <= ANON_CHUNK_SIZE && 655 !(ahp->flags & ANON_ALLOC_FORCE)) { 656 657 /* 658 * Yes, we're converting to a two level. Reuse old level 1 659 * as new level 2 if it is exactly PAGESIZE. Otherwise 660 * alloc a new level 2 and copy the old level 1 data into it. 661 */ 662 if (oldamp_pgs == ANON_CHUNK_SIZE) { 663 level2 = (void *)ahp->array_chunk; 664 } else { 665 level2 = kmem_alloc(PAGESIZE, kmemflags); 666 if (level2 == NULL) { 667 kmem_free(level1, newarrsz); 668 return (0); 669 } 670 oldarrsz = oldamp_pgs * sizeof (void *); 671 672 ANON_INITBUF(ahp->array_chunk, oldarrsz, 673 level2, PAGESIZE, growdown); 674 kmem_free(ahp->array_chunk, oldarrsz); 675 } 676 bzero(level1, newarrsz); 677 if (growdown) 678 level1[nelems - 1] = level2; 679 else 680 level1[0] = level2; 681 } else { 682 oldarrsz = oelems * sizeof (void *); 683 684 ANON_INITBUF(ahp->array_chunk, oldarrsz, 685 level1, newarrsz, growdown); 686 kmem_free(ahp->array_chunk, oldarrsz); 687 } 688 689 ahp->array_chunk = level1; 690 ahp->size = newamp_pgs; 691 if (growdown) 692 *startidx_p = newamp_pgs - totpages; 693 694 return (newamp_pgs); 695 } 696 697 698 /* 699 * Called from clock handler to sync ani_free value. 700 */ 701 702 void 703 set_anoninfo(void) 704 { 705 int ix; 706 pgcnt_t total = 0; 707 708 for (ix = 0; ix < ANI_MAX_POOL; ix++) { 709 total += ani_free_pool[ix].ani_count; 710 } 711 k_anoninfo.ani_free = total; 712 } 713 714 /* 715 * Reserve anon space. 716 * 717 * It's no longer simply a matter of incrementing ani_resv to 718 * reserve swap space, we need to check memory-based as well 719 * as disk-backed (physical) swap. The following algorithm 720 * is used: 721 * Check the space on physical swap 722 * i.e. amount needed < ani_max - ani_phys_resv 723 * If we are swapping on swapfs check 724 * amount needed < (availrmem - swapfs_minfree) 725 * Since the algorithm to check for the quantity of swap space is 726 * almost the same as that for reserving it, we'll just use anon_resvmem 727 * with a flag to decrement availrmem. 728 * 729 * Return non-zero on success. 730 */ 731 int 732 anon_resvmem(size_t size, boolean_t takemem, zone_t *zone, int tryhard) 733 { 734 pgcnt_t npages = btopr(size); 735 pgcnt_t mswap_pages = 0; 736 pgcnt_t pswap_pages = 0; 737 proc_t *p = curproc; 738 739 if (zone != NULL && takemem) { 740 /* test zone.max-swap resource control */ 741 mutex_enter(&p->p_lock); 742 if (rctl_incr_swap(p, zone, ptob(npages)) != 0) { 743 mutex_exit(&p->p_lock); 744 return (0); 745 } 746 mutex_exit(&p->p_lock); 747 } 748 mutex_enter(&anoninfo_lock); 749 750 /* 751 * pswap_pages is the number of pages we can take from 752 * physical (i.e. disk-backed) swap. 753 */ 754 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 755 pswap_pages = k_anoninfo.ani_max - k_anoninfo.ani_phys_resv; 756 757 ANON_PRINT(A_RESV, 758 ("anon_resvmem: npages %lu takemem %u pswap %lu caller %p\n", 759 npages, takemem, pswap_pages, (void *)caller())); 760 761 if (npages <= pswap_pages) { 762 /* 763 * we have enough space on a physical swap 764 */ 765 if (takemem) 766 k_anoninfo.ani_phys_resv += npages; 767 mutex_exit(&anoninfo_lock); 768 return (1); 769 } else if (pswap_pages != 0) { 770 /* 771 * we have some space on a physical swap 772 */ 773 if (takemem) { 774 /* 775 * use up remainder of phys swap 776 */ 777 k_anoninfo.ani_phys_resv += pswap_pages; 778 ASSERT(k_anoninfo.ani_phys_resv == k_anoninfo.ani_max); 779 } 780 } 781 /* 782 * since (npages > pswap_pages) we need mem swap 783 * mswap_pages is the number of pages needed from availrmem 784 */ 785 ASSERT(npages > pswap_pages); 786 mswap_pages = npages - pswap_pages; 787 788 ANON_PRINT(A_RESV, ("anon_resvmem: need %ld pages from memory\n", 789 mswap_pages)); 790 791 /* 792 * priv processes can reserve memory as swap as long as availrmem 793 * remains greater than swapfs_minfree; in the case of non-priv 794 * processes, memory can be reserved as swap only if availrmem 795 * doesn't fall below (swapfs_minfree + swapfs_reserve). Thus, 796 * swapfs_reserve amount of memswap is not available to non-priv 797 * processes. This protects daemons such as automounter dying 798 * as a result of application processes eating away almost entire 799 * membased swap. This safeguard becomes useless if apps are run 800 * with root access. 801 * 802 * swapfs_reserve is minimum of 4Mb or 1/16 of physmem. 803 * 804 */ 805 if (tryhard) { 806 mutex_exit(&anoninfo_lock); 807 (void) page_reclaim_mem(mswap_pages, 808 swapfs_minfree + swapfs_reserve, 0); 809 mutex_enter(&anoninfo_lock); 810 } 811 812 mutex_enter(&freemem_lock); 813 if (availrmem > (swapfs_minfree + swapfs_reserve + mswap_pages) || 814 (availrmem > (swapfs_minfree + mswap_pages) && 815 secpolicy_resource(CRED()) == 0)) { 816 817 if (takemem) { 818 /* 819 * Take the memory from the rest of the system. 820 */ 821 availrmem -= mswap_pages; 822 mutex_exit(&freemem_lock); 823 k_anoninfo.ani_mem_resv += mswap_pages; 824 ANI_ADD(mswap_pages); 825 ANON_PRINT((A_RESV | A_MRESV), 826 ("anon_resvmem: took %ld pages of availrmem\n", 827 mswap_pages)); 828 } else { 829 mutex_exit(&freemem_lock); 830 } 831 832 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 833 mutex_exit(&anoninfo_lock); 834 return (1); 835 836 } else { 837 /* 838 * Fail if not enough memory 839 */ 840 841 if (takemem) { 842 k_anoninfo.ani_phys_resv -= pswap_pages; 843 } 844 845 mutex_exit(&freemem_lock); 846 mutex_exit(&anoninfo_lock); 847 ANON_PRINT(A_RESV, 848 ("anon_resvmem: not enough space from swapfs\n")); 849 if (zone != NULL && takemem) 850 rctl_decr_swap(zone, ptob(npages)); 851 return (0); 852 } 853 } 854 855 /* 856 * Give back an anon reservation. 857 */ 858 void 859 anon_unresvmem(size_t size, zone_t *zone) 860 { 861 pgcnt_t npages = btopr(size); 862 spgcnt_t mem_free_pages = 0; 863 pgcnt_t phys_free_slots; 864 #ifdef ANON_DEBUG 865 pgcnt_t mem_resv; 866 #endif 867 if (zone != NULL) 868 rctl_decr_swap(zone, ptob(npages)); 869 870 mutex_enter(&anoninfo_lock); 871 872 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 873 /* 874 * If some of this reservation belonged to swapfs 875 * give it back to availrmem. 876 * ani_mem_resv is the amount of availrmem swapfs has reserved. 877 * but some of that memory could be locked by segspt so we can only 878 * return non locked ani_mem_resv back to availrmem 879 */ 880 if (k_anoninfo.ani_mem_resv > k_anoninfo.ani_locked_swap) { 881 ANON_PRINT((A_RESV | A_MRESV), 882 ("anon_unresv: growing availrmem by %ld pages\n", 883 MIN(k_anoninfo.ani_mem_resv, npages))); 884 885 mem_free_pages = MIN((spgcnt_t)(k_anoninfo.ani_mem_resv - 886 k_anoninfo.ani_locked_swap), npages); 887 mutex_enter(&freemem_lock); 888 availrmem += mem_free_pages; 889 mutex_exit(&freemem_lock); 890 k_anoninfo.ani_mem_resv -= mem_free_pages; 891 892 ANI_ADD(-mem_free_pages); 893 } 894 /* 895 * The remainder of the pages is returned to phys swap 896 */ 897 ASSERT(npages >= mem_free_pages); 898 phys_free_slots = npages - mem_free_pages; 899 900 if (phys_free_slots) { 901 k_anoninfo.ani_phys_resv -= phys_free_slots; 902 } 903 904 #ifdef ANON_DEBUG 905 mem_resv = k_anoninfo.ani_mem_resv; 906 #endif 907 908 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 909 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 910 911 mutex_exit(&anoninfo_lock); 912 913 ANON_PRINT(A_RESV, ("anon_unresv: %lu, tot %lu, caller %p\n", 914 npages, mem_resv, (void *)caller())); 915 } 916 917 /* 918 * Allocate an anon slot and return it with the lock held. 919 */ 920 struct anon * 921 anon_alloc(struct vnode *vp, anoff_t off) 922 { 923 struct anon *ap; 924 kmutex_t *ahm; 925 926 ap = kmem_cache_alloc(anon_cache, KM_SLEEP); 927 if (vp == NULL) { 928 swap_alloc(ap); 929 } else { 930 ap->an_vp = vp; 931 ap->an_off = off; 932 } 933 ap->an_refcnt = 1; 934 ap->an_pvp = NULL; 935 ap->an_poff = 0; 936 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 937 mutex_enter(ahm); 938 anon_addhash(ap); 939 mutex_exit(ahm); 940 ANI_ADD(-1); 941 ANON_PRINT(A_ANON, ("anon_alloc: returning ap %p, vp %p\n", 942 (void *)ap, (ap ? (void *)ap->an_vp : NULL))); 943 return (ap); 944 } 945 946 /* 947 * Decrement the reference count of an anon page. 948 * If reference count goes to zero, free it and 949 * its associated page (if any). 950 */ 951 void 952 anon_decref(struct anon *ap) 953 { 954 page_t *pp; 955 struct vnode *vp; 956 anoff_t off; 957 kmutex_t *ahm; 958 959 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 960 mutex_enter(ahm); 961 ASSERT(ap->an_refcnt != 0); 962 if (ap->an_refcnt == 0) 963 panic("anon_decref: slot count 0"); 964 if (--ap->an_refcnt == 0) { 965 swap_xlate(ap, &vp, &off); 966 anon_rmhash(ap); 967 if (ap->an_pvp != NULL) 968 swap_phys_free(ap->an_pvp, ap->an_poff, PAGESIZE); 969 mutex_exit(ahm); 970 971 /* 972 * If there is a page for this anon slot we will need to 973 * call VN_DISPOSE to get rid of the vp association and 974 * put the page back on the free list as really free. 975 * Acquire the "exclusive" lock to ensure that any 976 * pending i/o always completes before the swap slot 977 * is freed. 978 */ 979 pp = page_lookup(vp, (u_offset_t)off, SE_EXCL); 980 if (pp != NULL) { 981 /*LINTED: constant in conditional context */ 982 VN_DISPOSE(pp, B_INVAL, 0, kcred); 983 } 984 ANON_PRINT(A_ANON, ("anon_decref: free ap %p, vp %p\n", 985 (void *)ap, (void *)ap->an_vp)); 986 987 kmem_cache_free(anon_cache, ap); 988 989 ANI_ADD(1); 990 } else { 991 mutex_exit(ahm); 992 } 993 } 994 995 996 /* 997 * check an_refcnt of the root anon slot (anon_index argument is aligned at 998 * seg->s_szc level) to determine whether COW processing is required. 999 * anonpages_hash_lock[] held on the root ap ensures that if root's 1000 * refcnt is 1 all other refcnt's are 1 as well (and they can't increase 1001 * later since this process can't fork while its AS lock is held). 1002 * 1003 * returns 1 if the root anon slot has a refcnt > 1 otherwise returns 0. 1004 */ 1005 int 1006 anon_szcshare(struct anon_hdr *ahp, ulong_t anon_index) 1007 { 1008 struct anon *ap; 1009 kmutex_t *ahmpages = NULL; 1010 1011 ap = anon_get_ptr(ahp, anon_index); 1012 if (ap == NULL) 1013 return (0); 1014 1015 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1016 mutex_enter(ahmpages); 1017 ASSERT(ap->an_refcnt >= 1); 1018 if (ap->an_refcnt == 1) { 1019 mutex_exit(ahmpages); 1020 return (0); 1021 } 1022 mutex_exit(ahmpages); 1023 return (1); 1024 } 1025 /* 1026 * Check 'nslots' anon slots for refcnt > 1. 1027 * 1028 * returns 1 if any of the 'nslots' anon slots has a refcnt > 1 otherwise 1029 * returns 0. 1030 */ 1031 static int 1032 anon_share(struct anon_hdr *ahp, ulong_t anon_index, pgcnt_t nslots) 1033 { 1034 struct anon *ap; 1035 1036 while (nslots-- > 0) { 1037 if ((ap = anon_get_ptr(ahp, anon_index)) != NULL && 1038 ap->an_refcnt > 1) 1039 return (1); 1040 anon_index++; 1041 } 1042 1043 return (0); 1044 } 1045 1046 static void 1047 anon_decref_pages( 1048 struct anon_hdr *ahp, 1049 ulong_t an_idx, 1050 uint_t szc) 1051 { 1052 struct anon *ap = anon_get_ptr(ahp, an_idx); 1053 kmutex_t *ahmpages = NULL; 1054 page_t *pp; 1055 pgcnt_t pgcnt = page_get_pagecnt(szc); 1056 pgcnt_t i; 1057 struct vnode *vp; 1058 anoff_t off; 1059 kmutex_t *ahm; 1060 #ifdef DEBUG 1061 int refcnt = 1; 1062 #endif 1063 1064 ASSERT(szc != 0); 1065 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1066 ASSERT(IS_P2ALIGNED(an_idx, pgcnt)); 1067 ASSERT(an_idx < ahp->size); 1068 1069 if (ahp->size - an_idx < pgcnt) { 1070 /* 1071 * In case of shared mappings total anon map size may not be 1072 * the largest page size aligned. 1073 */ 1074 pgcnt = ahp->size - an_idx; 1075 } 1076 1077 VM_STAT_ADD(anonvmstats.decrefpages[0]); 1078 1079 if (ap != NULL) { 1080 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1081 mutex_enter(ahmpages); 1082 ASSERT((refcnt = ap->an_refcnt) != 0); 1083 VM_STAT_ADD(anonvmstats.decrefpages[1]); 1084 if (ap->an_refcnt == 1) { 1085 VM_STAT_ADD(anonvmstats.decrefpages[2]); 1086 ASSERT(!anon_share(ahp, an_idx, pgcnt)); 1087 mutex_exit(ahmpages); 1088 ahmpages = NULL; 1089 } 1090 } 1091 1092 i = 0; 1093 while (i < pgcnt) { 1094 if ((ap = anon_get_ptr(ahp, an_idx + i)) == NULL) { 1095 ASSERT(refcnt == 1 && ahmpages == NULL); 1096 i++; 1097 continue; 1098 } 1099 ASSERT(ap->an_refcnt == refcnt); 1100 ASSERT(ahmpages != NULL || ap->an_refcnt == 1); 1101 ASSERT(ahmpages == NULL || ap->an_refcnt > 1); 1102 1103 if (ahmpages == NULL) { 1104 swap_xlate(ap, &vp, &off); 1105 pp = page_lookup(vp, (u_offset_t)off, SE_EXCL); 1106 if (pp == NULL || pp->p_szc == 0) { 1107 VM_STAT_ADD(anonvmstats.decrefpages[3]); 1108 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, 1109 ap->an_off)]; 1110 (void) anon_set_ptr(ahp, an_idx + i, NULL, 1111 ANON_SLEEP); 1112 mutex_enter(ahm); 1113 ap->an_refcnt--; 1114 ASSERT(ap->an_refcnt == 0); 1115 anon_rmhash(ap); 1116 if (ap->an_pvp) 1117 swap_phys_free(ap->an_pvp, ap->an_poff, 1118 PAGESIZE); 1119 mutex_exit(ahm); 1120 if (pp == NULL) { 1121 pp = page_lookup(vp, (u_offset_t)off, 1122 SE_EXCL); 1123 ASSERT(pp == NULL || pp->p_szc == 0); 1124 } 1125 if (pp != NULL) { 1126 VM_STAT_ADD(anonvmstats.decrefpages[4]); 1127 /*LINTED*/ 1128 VN_DISPOSE(pp, B_INVAL, 0, kcred); 1129 } 1130 kmem_cache_free(anon_cache, ap); 1131 ANI_ADD(1); 1132 i++; 1133 } else { 1134 pgcnt_t j; 1135 pgcnt_t curpgcnt = 1136 page_get_pagecnt(pp->p_szc); 1137 size_t ppasize = curpgcnt * sizeof (page_t *); 1138 page_t **ppa = kmem_alloc(ppasize, KM_SLEEP); 1139 int dispose = 0; 1140 1141 VM_STAT_ADD(anonvmstats.decrefpages[5]); 1142 1143 ASSERT(pp->p_szc <= szc); 1144 ASSERT(IS_P2ALIGNED(curpgcnt, curpgcnt)); 1145 ASSERT(IS_P2ALIGNED(i, curpgcnt)); 1146 ASSERT(i + curpgcnt <= pgcnt); 1147 ASSERT(!(page_pptonum(pp) & (curpgcnt - 1))); 1148 ppa[0] = pp; 1149 for (j = i + 1; j < i + curpgcnt; j++) { 1150 ap = anon_get_ptr(ahp, an_idx + j); 1151 ASSERT(ap != NULL && 1152 ap->an_refcnt == 1); 1153 swap_xlate(ap, &vp, &off); 1154 pp = page_lookup(vp, (u_offset_t)off, 1155 SE_EXCL); 1156 if (pp == NULL) 1157 panic("anon_decref_pages: " 1158 "no page"); 1159 1160 (void) hat_pageunload(pp, 1161 HAT_FORCE_PGUNLOAD); 1162 ASSERT(pp->p_szc == ppa[0]->p_szc); 1163 ASSERT(page_pptonum(pp) - 1 == 1164 page_pptonum(ppa[j - i - 1])); 1165 ppa[j - i] = pp; 1166 if (ap->an_pvp != NULL && 1167 !vn_matchopval(ap->an_pvp, 1168 VOPNAME_DISPOSE, 1169 (fs_generic_func_p)fs_dispose)) 1170 dispose = 1; 1171 } 1172 for (j = i; j < i + curpgcnt; j++) { 1173 ap = anon_get_ptr(ahp, an_idx + j); 1174 ASSERT(ap != NULL && 1175 ap->an_refcnt == 1); 1176 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, 1177 ap->an_off)]; 1178 (void) anon_set_ptr(ahp, an_idx + j, 1179 NULL, ANON_SLEEP); 1180 mutex_enter(ahm); 1181 ap->an_refcnt--; 1182 ASSERT(ap->an_refcnt == 0); 1183 anon_rmhash(ap); 1184 if (ap->an_pvp) 1185 swap_phys_free(ap->an_pvp, 1186 ap->an_poff, PAGESIZE); 1187 mutex_exit(ahm); 1188 kmem_cache_free(anon_cache, ap); 1189 ANI_ADD(1); 1190 } 1191 if (!dispose) { 1192 VM_STAT_ADD(anonvmstats.decrefpages[6]); 1193 page_destroy_pages(ppa[0]); 1194 } else { 1195 VM_STAT_ADD(anonvmstats.decrefpages[7]); 1196 for (j = 0; j < curpgcnt; j++) { 1197 ASSERT(PAGE_EXCL(ppa[j])); 1198 ppa[j]->p_szc = 0; 1199 } 1200 for (j = 0; j < curpgcnt; j++) { 1201 ASSERT(!hat_page_is_mapped( 1202 ppa[j])); 1203 /*LINTED*/ 1204 VN_DISPOSE(ppa[j], B_INVAL, 0, 1205 kcred); 1206 } 1207 } 1208 kmem_free(ppa, ppasize); 1209 i += curpgcnt; 1210 } 1211 } else { 1212 VM_STAT_ADD(anonvmstats.decrefpages[8]); 1213 (void) anon_set_ptr(ahp, an_idx + i, NULL, ANON_SLEEP); 1214 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1215 mutex_enter(ahm); 1216 ap->an_refcnt--; 1217 mutex_exit(ahm); 1218 i++; 1219 } 1220 } 1221 1222 if (ahmpages != NULL) { 1223 mutex_exit(ahmpages); 1224 } 1225 } 1226 1227 /* 1228 * Duplicate references to size bytes worth of anon pages. 1229 * Used when duplicating a segment that contains private anon pages. 1230 * This code assumes that procedure calling this one has already used 1231 * hat_chgprot() to disable write access to the range of addresses that 1232 * that *old actually refers to. 1233 */ 1234 void 1235 anon_dup(struct anon_hdr *old, ulong_t old_idx, struct anon_hdr *new, 1236 ulong_t new_idx, size_t size) 1237 { 1238 spgcnt_t npages; 1239 kmutex_t *ahm; 1240 struct anon *ap; 1241 ulong_t off; 1242 ulong_t index; 1243 1244 npages = btopr(size); 1245 while (npages > 0) { 1246 index = old_idx; 1247 if ((ap = anon_get_next_ptr(old, &index)) == NULL) 1248 break; 1249 1250 ASSERT(!ANON_ISBUSY(anon_get_slot(old, index))); 1251 off = index - old_idx; 1252 npages -= off; 1253 if (npages <= 0) 1254 break; 1255 1256 (void) anon_set_ptr(new, new_idx + off, ap, ANON_SLEEP); 1257 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1258 1259 mutex_enter(ahm); 1260 ap->an_refcnt++; 1261 mutex_exit(ahm); 1262 1263 off++; 1264 new_idx += off; 1265 old_idx += off; 1266 npages--; 1267 } 1268 } 1269 1270 /* 1271 * Just like anon_dup but also guarantees there are no holes (unallocated anon 1272 * slots) within any large page region. That means if a large page region is 1273 * empty in the old array it will skip it. If there are 1 or more valid slots 1274 * in the large page region of the old array it will make sure to fill in any 1275 * unallocated ones and also copy them to the new array. If noalloc is 1 large 1276 * page region should either have no valid anon slots or all slots should be 1277 * valid. 1278 */ 1279 void 1280 anon_dup_fill_holes( 1281 struct anon_hdr *old, 1282 ulong_t old_idx, 1283 struct anon_hdr *new, 1284 ulong_t new_idx, 1285 size_t size, 1286 uint_t szc, 1287 int noalloc) 1288 { 1289 struct anon *ap; 1290 spgcnt_t npages; 1291 kmutex_t *ahm, *ahmpages = NULL; 1292 pgcnt_t pgcnt, i; 1293 ulong_t index, off; 1294 #ifdef DEBUG 1295 int refcnt; 1296 #endif 1297 1298 ASSERT(szc != 0); 1299 pgcnt = page_get_pagecnt(szc); 1300 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1301 npages = btopr(size); 1302 ASSERT(IS_P2ALIGNED(npages, pgcnt)); 1303 ASSERT(IS_P2ALIGNED(old_idx, pgcnt)); 1304 1305 VM_STAT_ADD(anonvmstats.dupfillholes[0]); 1306 1307 while (npages > 0) { 1308 index = old_idx; 1309 1310 /* 1311 * Find the next valid slot. 1312 */ 1313 if (anon_get_next_ptr(old, &index) == NULL) 1314 break; 1315 1316 ASSERT(!ANON_ISBUSY(anon_get_slot(old, index))); 1317 /* 1318 * Now backup index to the beginning of the 1319 * current large page region of the old array. 1320 */ 1321 index = P2ALIGN(index, pgcnt); 1322 off = index - old_idx; 1323 ASSERT(IS_P2ALIGNED(off, pgcnt)); 1324 npages -= off; 1325 if (npages <= 0) 1326 break; 1327 1328 /* 1329 * Fill and copy a large page regions worth 1330 * of anon slots. 1331 */ 1332 for (i = 0; i < pgcnt; i++) { 1333 if ((ap = anon_get_ptr(old, index + i)) == NULL) { 1334 if (noalloc) { 1335 panic("anon_dup_fill_holes: " 1336 "empty anon slot\n"); 1337 } 1338 VM_STAT_ADD(anonvmstats.dupfillholes[1]); 1339 ap = anon_alloc(NULL, 0); 1340 (void) anon_set_ptr(old, index + i, ap, 1341 ANON_SLEEP); 1342 } else if (i == 0) { 1343 /* 1344 * make the increment of all refcnts of all 1345 * anon slots of a large page appear atomic by 1346 * getting an anonpages_hash_lock for the 1347 * first anon slot of a large page. 1348 */ 1349 int hash = AH_LOCK(ap->an_vp, ap->an_off); 1350 1351 VM_STAT_ADD(anonvmstats.dupfillholes[2]); 1352 1353 ahmpages = &anonpages_hash_lock[hash]; 1354 mutex_enter(ahmpages); 1355 /*LINTED*/ 1356 ASSERT(refcnt = ap->an_refcnt); 1357 1358 VM_STAT_COND_ADD(ap->an_refcnt > 1, 1359 anonvmstats.dupfillholes[3]); 1360 } 1361 (void) anon_set_ptr(new, new_idx + off + i, ap, 1362 ANON_SLEEP); 1363 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1364 mutex_enter(ahm); 1365 ASSERT(ahmpages != NULL || ap->an_refcnt == 1); 1366 ASSERT(i == 0 || ahmpages == NULL || 1367 refcnt == ap->an_refcnt); 1368 ap->an_refcnt++; 1369 mutex_exit(ahm); 1370 } 1371 if (ahmpages != NULL) { 1372 mutex_exit(ahmpages); 1373 ahmpages = NULL; 1374 } 1375 off += pgcnt; 1376 new_idx += off; 1377 old_idx += off; 1378 npages -= pgcnt; 1379 } 1380 } 1381 1382 /* 1383 * Used when a segment with a vnode changes szc. similarly to 1384 * anon_dup_fill_holes() makes sure each large page region either has no anon 1385 * slots or all of them. but new slots are created by COWing the file 1386 * pages. on entrance no anon slots should be shared. 1387 */ 1388 int 1389 anon_fill_cow_holes( 1390 struct seg *seg, 1391 caddr_t addr, 1392 struct anon_hdr *ahp, 1393 ulong_t an_idx, 1394 struct vnode *vp, 1395 u_offset_t vp_off, 1396 size_t size, 1397 uint_t szc, 1398 uint_t prot, 1399 struct vpage vpage[], 1400 struct cred *cred) 1401 { 1402 struct anon *ap; 1403 spgcnt_t npages; 1404 pgcnt_t pgcnt, i; 1405 ulong_t index, off; 1406 int err = 0; 1407 int pageflags = 0; 1408 1409 ASSERT(szc != 0); 1410 pgcnt = page_get_pagecnt(szc); 1411 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1412 npages = btopr(size); 1413 ASSERT(IS_P2ALIGNED(npages, pgcnt)); 1414 ASSERT(IS_P2ALIGNED(an_idx, pgcnt)); 1415 1416 while (npages > 0) { 1417 index = an_idx; 1418 1419 /* 1420 * Find the next valid slot. 1421 */ 1422 if (anon_get_next_ptr(ahp, &index) == NULL) { 1423 break; 1424 } 1425 1426 ASSERT(!ANON_ISBUSY(anon_get_slot(ahp, index))); 1427 /* 1428 * Now backup index to the beginning of the 1429 * current large page region of the anon array. 1430 */ 1431 index = P2ALIGN(index, pgcnt); 1432 off = index - an_idx; 1433 ASSERT(IS_P2ALIGNED(off, pgcnt)); 1434 npages -= off; 1435 if (npages <= 0) 1436 break; 1437 an_idx += off; 1438 vp_off += ptob(off); 1439 addr += ptob(off); 1440 if (vpage != NULL) { 1441 vpage += off; 1442 } 1443 1444 for (i = 0; i < pgcnt; i++, an_idx++, vp_off += PAGESIZE) { 1445 if ((ap = anon_get_ptr(ahp, an_idx)) == NULL) { 1446 page_t *pl[1 + 1]; 1447 page_t *pp; 1448 1449 err = VOP_GETPAGE(vp, vp_off, PAGESIZE, NULL, 1450 pl, PAGESIZE, seg, addr, S_READ, cred, 1451 NULL); 1452 if (err) { 1453 break; 1454 } 1455 if (vpage != NULL) { 1456 prot = VPP_PROT(vpage); 1457 pageflags = VPP_ISPPLOCK(vpage) ? 1458 LOCK_PAGE : 0; 1459 } 1460 pp = anon_private(&ap, seg, addr, prot, pl[0], 1461 pageflags, cred); 1462 if (pp == NULL) { 1463 err = ENOMEM; 1464 break; 1465 } 1466 (void) anon_set_ptr(ahp, an_idx, ap, 1467 ANON_SLEEP); 1468 page_unlock(pp); 1469 } 1470 ASSERT(ap->an_refcnt == 1); 1471 addr += PAGESIZE; 1472 if (vpage != NULL) { 1473 vpage++; 1474 } 1475 } 1476 npages -= pgcnt; 1477 } 1478 1479 return (err); 1480 } 1481 1482 /* 1483 * Free a group of "size" anon pages, size in bytes, 1484 * and clear out the pointers to the anon entries. 1485 */ 1486 void 1487 anon_free(struct anon_hdr *ahp, ulong_t index, size_t size) 1488 { 1489 spgcnt_t npages; 1490 struct anon *ap; 1491 ulong_t old; 1492 1493 npages = btopr(size); 1494 1495 while (npages > 0) { 1496 old = index; 1497 if ((ap = anon_get_next_ptr(ahp, &index)) == NULL) 1498 break; 1499 1500 ASSERT(!ANON_ISBUSY(anon_get_slot(ahp, index))); 1501 npages -= index - old; 1502 if (npages <= 0) 1503 break; 1504 1505 (void) anon_set_ptr(ahp, index, NULL, ANON_SLEEP); 1506 anon_decref(ap); 1507 /* 1508 * Bump index and decrement page count 1509 */ 1510 index++; 1511 npages--; 1512 } 1513 } 1514 1515 void 1516 anon_free_pages( 1517 struct anon_hdr *ahp, 1518 ulong_t an_idx, 1519 size_t size, 1520 uint_t szc) 1521 { 1522 spgcnt_t npages; 1523 pgcnt_t pgcnt; 1524 ulong_t index, off; 1525 1526 ASSERT(szc != 0); 1527 pgcnt = page_get_pagecnt(szc); 1528 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1529 npages = btopr(size); 1530 ASSERT(IS_P2ALIGNED(npages, pgcnt)); 1531 ASSERT(IS_P2ALIGNED(an_idx, pgcnt)); 1532 ASSERT(an_idx < ahp->size); 1533 1534 VM_STAT_ADD(anonvmstats.freepages[0]); 1535 1536 while (npages > 0) { 1537 index = an_idx; 1538 1539 /* 1540 * Find the next valid slot. 1541 */ 1542 if (anon_get_next_ptr(ahp, &index) == NULL) 1543 break; 1544 1545 ASSERT(!ANON_ISBUSY(anon_get_slot(ahp, index))); 1546 /* 1547 * Now backup index to the beginning of the 1548 * current large page region of the old array. 1549 */ 1550 index = P2ALIGN(index, pgcnt); 1551 off = index - an_idx; 1552 ASSERT(IS_P2ALIGNED(off, pgcnt)); 1553 npages -= off; 1554 if (npages <= 0) 1555 break; 1556 1557 anon_decref_pages(ahp, index, szc); 1558 1559 off += pgcnt; 1560 an_idx += off; 1561 npages -= pgcnt; 1562 } 1563 } 1564 1565 /* 1566 * Make anonymous pages discardable 1567 */ 1568 void 1569 anon_disclaim(struct anon_map *amp, ulong_t index, size_t size) 1570 { 1571 spgcnt_t npages = btopr(size); 1572 struct anon *ap; 1573 struct vnode *vp; 1574 anoff_t off; 1575 page_t *pp, *root_pp; 1576 kmutex_t *ahm; 1577 pgcnt_t pgcnt; 1578 ulong_t old_idx, idx, i; 1579 struct anon_hdr *ahp = amp->ahp; 1580 anon_sync_obj_t cookie; 1581 1582 ASSERT(RW_READ_HELD(&->a_rwlock)); 1583 pgcnt = 1; 1584 for (; npages > 0; index = (pgcnt == 1) ? index + 1 : 1585 P2ROUNDUP(index + 1, pgcnt), npages -= pgcnt) { 1586 1587 /* 1588 * get anon pointer and index for the first valid entry 1589 * in the anon list, starting from "index" 1590 */ 1591 old_idx = index; 1592 if ((ap = anon_get_next_ptr(ahp, &index)) == NULL) 1593 break; 1594 1595 /* 1596 * decrement npages by number of NULL anon slots we skipped 1597 */ 1598 npages -= index - old_idx; 1599 if (npages <= 0) 1600 break; 1601 1602 anon_array_enter(amp, index, &cookie); 1603 ap = anon_get_ptr(ahp, index); 1604 ASSERT(ap != NULL); 1605 1606 /* 1607 * Get anonymous page and try to lock it SE_EXCL; 1608 * if we couldn't grab the lock we skip to next page. 1609 */ 1610 swap_xlate(ap, &vp, &off); 1611 pp = page_lookup_nowait(vp, (u_offset_t)off, SE_EXCL); 1612 if (pp == NULL) { 1613 segadvstat.MADV_FREE_miss.value.ul++; 1614 pgcnt = 1; 1615 anon_array_exit(&cookie); 1616 continue; 1617 } 1618 pgcnt = page_get_pagecnt(pp->p_szc); 1619 1620 /* 1621 * we cannot free a page which is permanently locked. 1622 * The page_struct_lock need not be acquired to examine 1623 * these fields since the page has an "exclusive" lock. 1624 */ 1625 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 1626 page_unlock(pp); 1627 segadvstat.MADV_FREE_miss.value.ul++; 1628 anon_array_exit(&cookie); 1629 continue; 1630 } 1631 1632 ahm = &anonhash_lock[AH_LOCK(vp, off)]; 1633 mutex_enter(ahm); 1634 ASSERT(ap->an_refcnt != 0); 1635 /* 1636 * skip this one if copy-on-write is not yet broken. 1637 */ 1638 if (ap->an_refcnt > 1) { 1639 mutex_exit(ahm); 1640 page_unlock(pp); 1641 segadvstat.MADV_FREE_miss.value.ul++; 1642 anon_array_exit(&cookie); 1643 continue; 1644 } 1645 1646 if (pp->p_szc == 0) { 1647 pgcnt = 1; 1648 1649 /* 1650 * free swap slot; 1651 */ 1652 if (ap->an_pvp) { 1653 swap_phys_free(ap->an_pvp, ap->an_poff, 1654 PAGESIZE); 1655 ap->an_pvp = NULL; 1656 ap->an_poff = 0; 1657 } 1658 mutex_exit(ahm); 1659 segadvstat.MADV_FREE_hit.value.ul++; 1660 1661 /* 1662 * while we are at it, unload all the translations 1663 * and attempt to free the page. 1664 */ 1665 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1666 /*LINTED: constant in conditional context */ 1667 VN_DISPOSE(pp, B_FREE, 0, kcred); 1668 anon_array_exit(&cookie); 1669 continue; 1670 } 1671 1672 pgcnt = page_get_pagecnt(pp->p_szc); 1673 if (!IS_P2ALIGNED(index, pgcnt) || npages < pgcnt) { 1674 if (!page_try_demote_pages(pp)) { 1675 mutex_exit(ahm); 1676 page_unlock(pp); 1677 segadvstat.MADV_FREE_miss.value.ul++; 1678 anon_array_exit(&cookie); 1679 continue; 1680 } else { 1681 pgcnt = 1; 1682 if (ap->an_pvp) { 1683 swap_phys_free(ap->an_pvp, 1684 ap->an_poff, PAGESIZE); 1685 ap->an_pvp = NULL; 1686 ap->an_poff = 0; 1687 } 1688 mutex_exit(ahm); 1689 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1690 /*LINTED*/ 1691 VN_DISPOSE(pp, B_FREE, 0, kcred); 1692 segadvstat.MADV_FREE_hit.value.ul++; 1693 anon_array_exit(&cookie); 1694 continue; 1695 } 1696 } 1697 mutex_exit(ahm); 1698 root_pp = pp; 1699 1700 /* 1701 * try to lock remaining pages 1702 */ 1703 for (idx = 1; idx < pgcnt; idx++) { 1704 pp++; 1705 if (!page_trylock(pp, SE_EXCL)) 1706 break; 1707 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 1708 page_unlock(pp); 1709 break; 1710 } 1711 } 1712 1713 if (idx == pgcnt) { 1714 for (i = 0; i < pgcnt; i++) { 1715 ap = anon_get_ptr(ahp, index + i); 1716 if (ap == NULL) 1717 break; 1718 swap_xlate(ap, &vp, &off); 1719 ahm = &anonhash_lock[AH_LOCK(vp, off)]; 1720 mutex_enter(ahm); 1721 ASSERT(ap->an_refcnt != 0); 1722 1723 /* 1724 * skip this one if copy-on-write 1725 * is not yet broken. 1726 */ 1727 if (ap->an_refcnt > 1) { 1728 mutex_exit(ahm); 1729 goto skiplp; 1730 } 1731 if (ap->an_pvp) { 1732 swap_phys_free(ap->an_pvp, 1733 ap->an_poff, PAGESIZE); 1734 ap->an_pvp = NULL; 1735 ap->an_poff = 0; 1736 } 1737 mutex_exit(ahm); 1738 } 1739 page_destroy_pages(root_pp); 1740 segadvstat.MADV_FREE_hit.value.ul += pgcnt; 1741 anon_array_exit(&cookie); 1742 continue; 1743 } 1744 skiplp: 1745 segadvstat.MADV_FREE_miss.value.ul += pgcnt; 1746 for (i = 0, pp = root_pp; i < idx; pp++, i++) 1747 page_unlock(pp); 1748 anon_array_exit(&cookie); 1749 } 1750 } 1751 1752 /* 1753 * Return the kept page(s) and protections back to the segment driver. 1754 */ 1755 int 1756 anon_getpage( 1757 struct anon **app, 1758 uint_t *protp, 1759 page_t *pl[], 1760 size_t plsz, 1761 struct seg *seg, 1762 caddr_t addr, 1763 enum seg_rw rw, 1764 struct cred *cred) 1765 { 1766 page_t *pp; 1767 struct anon *ap = *app; 1768 struct vnode *vp; 1769 anoff_t off; 1770 int err; 1771 kmutex_t *ahm; 1772 1773 swap_xlate(ap, &vp, &off); 1774 1775 /* 1776 * Lookup the page. If page is being paged in, 1777 * wait for it to finish as we must return a list of 1778 * pages since this routine acts like the VOP_GETPAGE 1779 * routine does. 1780 */ 1781 if (pl != NULL && (pp = page_lookup(vp, (u_offset_t)off, SE_SHARED))) { 1782 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1783 mutex_enter(ahm); 1784 if (ap->an_refcnt == 1) 1785 *protp = PROT_ALL; 1786 else 1787 *protp = PROT_ALL & ~PROT_WRITE; 1788 mutex_exit(ahm); 1789 pl[0] = pp; 1790 pl[1] = NULL; 1791 return (0); 1792 } 1793 1794 /* 1795 * Simply treat it as a vnode fault on the anon vp. 1796 */ 1797 1798 TRACE_3(TR_FAC_VM, TR_ANON_GETPAGE, 1799 "anon_getpage:seg %x addr %x vp %x", 1800 seg, addr, vp); 1801 1802 err = VOP_GETPAGE(vp, (u_offset_t)off, PAGESIZE, protp, pl, plsz, 1803 seg, addr, rw, cred, NULL); 1804 1805 if (err == 0 && pl != NULL) { 1806 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1807 mutex_enter(ahm); 1808 if (ap->an_refcnt != 1) 1809 *protp &= ~PROT_WRITE; /* make read-only */ 1810 mutex_exit(ahm); 1811 } 1812 return (err); 1813 } 1814 1815 /* 1816 * Creates or returns kept pages to the segment driver. returns -1 if a large 1817 * page cannot be allocated. returns -2 if some other process has allocated a 1818 * larger page. 1819 * 1820 * For cowfault it will allocate any size pages to fill the requested area to 1821 * avoid partially overwriting anon slots (i.e. sharing only some of the anon 1822 * slots within a large page with other processes). This policy greatly 1823 * simplifies large page freeing (which is only freed when all anon slot 1824 * refcnts are 0). 1825 */ 1826 int 1827 anon_map_getpages( 1828 struct anon_map *amp, 1829 ulong_t start_idx, 1830 uint_t szc, 1831 struct seg *seg, 1832 caddr_t addr, 1833 uint_t prot, 1834 uint_t *protp, 1835 page_t *ppa[], 1836 uint_t *ppa_szc, 1837 struct vpage vpage[], 1838 enum seg_rw rw, 1839 int brkcow, 1840 int anypgsz, 1841 int pgflags, 1842 struct cred *cred) 1843 { 1844 pgcnt_t pgcnt; 1845 struct anon *ap; 1846 struct vnode *vp; 1847 anoff_t off; 1848 page_t *pp, *pl[2], *conpp = NULL; 1849 caddr_t vaddr; 1850 ulong_t pg_idx, an_idx, i; 1851 spgcnt_t nreloc = 0; 1852 int prealloc = 1; 1853 int err, slotcreate; 1854 uint_t vpprot; 1855 int upsize = (szc < seg->s_szc); 1856 1857 #if !defined(__i386) && !defined(__amd64) 1858 ASSERT(seg->s_szc != 0); 1859 #endif 1860 ASSERT(szc <= seg->s_szc); 1861 ASSERT(ppa_szc != NULL); 1862 ASSERT(rw != S_CREATE); 1863 1864 *protp = PROT_ALL; 1865 1866 VM_STAT_ADD(anonvmstats.getpages[0]); 1867 1868 if (szc == 0) { 1869 VM_STAT_ADD(anonvmstats.getpages[1]); 1870 if ((ap = anon_get_ptr(amp->ahp, start_idx)) != NULL) { 1871 err = anon_getpage(&ap, protp, pl, PAGESIZE, seg, 1872 addr, rw, cred); 1873 if (err) 1874 return (err); 1875 ppa[0] = pl[0]; 1876 if (brkcow == 0 || (*protp & PROT_WRITE)) { 1877 VM_STAT_ADD(anonvmstats.getpages[2]); 1878 if (ppa[0]->p_szc != 0 && upsize) { 1879 VM_STAT_ADD(anonvmstats.getpages[3]); 1880 *ppa_szc = MIN(ppa[0]->p_szc, 1881 seg->s_szc); 1882 page_unlock(ppa[0]); 1883 return (-2); 1884 } 1885 return (0); 1886 } 1887 panic("anon_map_getpages: cowfault for szc 0"); 1888 } else { 1889 VM_STAT_ADD(anonvmstats.getpages[4]); 1890 ppa[0] = anon_zero(seg, addr, &ap, cred); 1891 if (ppa[0] == NULL) 1892 return (ENOMEM); 1893 (void) anon_set_ptr(amp->ahp, start_idx, ap, 1894 ANON_SLEEP); 1895 return (0); 1896 } 1897 } 1898 1899 pgcnt = page_get_pagecnt(szc); 1900 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1901 ASSERT(IS_P2ALIGNED(start_idx, pgcnt)); 1902 1903 /* 1904 * First we check for the case that the requtested large 1905 * page or larger page already exists in the system. 1906 * Actually we only check if the first constituent page 1907 * exists and only preallocate if it's not found. 1908 */ 1909 ap = anon_get_ptr(amp->ahp, start_idx); 1910 if (ap) { 1911 uint_t pszc; 1912 swap_xlate(ap, &vp, &off); 1913 if (page_exists_forreal(vp, (u_offset_t)off, &pszc)) { 1914 if (pszc > szc && upsize) { 1915 *ppa_szc = MIN(pszc, seg->s_szc); 1916 return (-2); 1917 } 1918 if (pszc >= szc) { 1919 prealloc = 0; 1920 } 1921 } 1922 } 1923 1924 VM_STAT_COND_ADD(prealloc == 0, anonvmstats.getpages[5]); 1925 VM_STAT_COND_ADD(prealloc != 0, anonvmstats.getpages[6]); 1926 1927 top: 1928 /* 1929 * If a smaller page or no page at all was found, 1930 * grab a large page off the freelist. 1931 */ 1932 if (prealloc) { 1933 ASSERT(conpp == NULL); 1934 if (page_alloc_pages(anon_vp, seg, addr, NULL, ppa, 1935 szc, 0, pgflags) != 0) { 1936 VM_STAT_ADD(anonvmstats.getpages[7]); 1937 if (brkcow == 0 || szc < seg->s_szc || 1938 !anon_szcshare(amp->ahp, start_idx)) { 1939 /* 1940 * If the refcnt's of all anon slots are <= 1 1941 * they can't increase since we are holding 1942 * the address space's lock. So segvn can 1943 * safely decrease szc without risking to 1944 * generate a cow fault for the region smaller 1945 * than the segment's largest page size. 1946 */ 1947 VM_STAT_ADD(anonvmstats.getpages[8]); 1948 return (-1); 1949 } 1950 docow: 1951 /* 1952 * This is a cow fault. Copy away the entire 1 large 1953 * page region of this segment. 1954 */ 1955 if (szc != seg->s_szc) 1956 panic("anon_map_getpages: cowfault for szc %d", 1957 szc); 1958 vaddr = addr; 1959 for (pg_idx = 0, an_idx = start_idx; pg_idx < pgcnt; 1960 pg_idx++, an_idx++, vaddr += PAGESIZE) { 1961 if ((ap = anon_get_ptr(amp->ahp, an_idx)) != 1962 NULL) { 1963 err = anon_getpage(&ap, &vpprot, pl, 1964 PAGESIZE, seg, vaddr, rw, cred); 1965 if (err) { 1966 for (i = 0; i < pg_idx; i++) { 1967 if ((pp = ppa[i]) != 1968 NULL) 1969 page_unlock(pp); 1970 } 1971 return (err); 1972 } 1973 ppa[pg_idx] = pl[0]; 1974 } else { 1975 /* 1976 * Since this is a cowfault we know 1977 * that this address space has a 1978 * parent or children which means 1979 * anon_dup_fill_holes() has initialized 1980 * all anon slots within a large page 1981 * region that had at least one anon 1982 * slot at the time of fork(). 1983 */ 1984 panic("anon_map_getpages: " 1985 "cowfault but anon slot is empty"); 1986 } 1987 } 1988 VM_STAT_ADD(anonvmstats.getpages[9]); 1989 *protp = PROT_ALL; 1990 return (anon_map_privatepages(amp, start_idx, szc, seg, 1991 addr, prot, ppa, vpage, anypgsz, pgflags, cred)); 1992 } 1993 } 1994 1995 VM_STAT_ADD(anonvmstats.getpages[10]); 1996 1997 an_idx = start_idx; 1998 pg_idx = 0; 1999 vaddr = addr; 2000 while (pg_idx < pgcnt) { 2001 slotcreate = 0; 2002 if ((ap = anon_get_ptr(amp->ahp, an_idx)) == NULL) { 2003 VM_STAT_ADD(anonvmstats.getpages[11]); 2004 /* 2005 * For us to have decided not to preallocate 2006 * would have meant that a large page 2007 * was found. Which also means that all of the 2008 * anon slots for that page would have been 2009 * already created for us. 2010 */ 2011 if (prealloc == 0) 2012 panic("anon_map_getpages: prealloc = 0"); 2013 2014 slotcreate = 1; 2015 ap = anon_alloc(NULL, 0); 2016 } 2017 swap_xlate(ap, &vp, &off); 2018 2019 /* 2020 * Now setup our preallocated page to pass down 2021 * to swap_getpage(). 2022 */ 2023 if (prealloc) { 2024 ASSERT(ppa[pg_idx]->p_szc == szc); 2025 conpp = ppa[pg_idx]; 2026 } 2027 ASSERT(prealloc || conpp == NULL); 2028 2029 /* 2030 * If we just created this anon slot then call 2031 * with S_CREATE to prevent doing IO on the page. 2032 * Similar to the anon_zero case. 2033 */ 2034 err = swap_getconpage(vp, (u_offset_t)off, PAGESIZE, 2035 NULL, pl, PAGESIZE, conpp, ppa_szc, &nreloc, seg, vaddr, 2036 slotcreate == 1 ? S_CREATE : rw, cred); 2037 2038 if (err) { 2039 ASSERT(err != -2 || upsize); 2040 VM_STAT_ADD(anonvmstats.getpages[12]); 2041 ASSERT(slotcreate == 0); 2042 goto io_err; 2043 } 2044 2045 pp = pl[0]; 2046 2047 if (pp->p_szc < szc || (pp->p_szc > szc && upsize)) { 2048 VM_STAT_ADD(anonvmstats.getpages[13]); 2049 ASSERT(slotcreate == 0); 2050 ASSERT(prealloc == 0); 2051 ASSERT(pg_idx == 0); 2052 if (pp->p_szc > szc) { 2053 ASSERT(upsize); 2054 *ppa_szc = MIN(pp->p_szc, seg->s_szc); 2055 page_unlock(pp); 2056 VM_STAT_ADD(anonvmstats.getpages[14]); 2057 return (-2); 2058 } 2059 page_unlock(pp); 2060 prealloc = 1; 2061 goto top; 2062 } 2063 2064 /* 2065 * If we decided to preallocate but VOP_GETPAGE 2066 * found a page in the system that satisfies our 2067 * request then free up our preallocated large page 2068 * and continue looping accross the existing large 2069 * page via VOP_GETPAGE. 2070 */ 2071 if (prealloc && pp != ppa[pg_idx]) { 2072 VM_STAT_ADD(anonvmstats.getpages[15]); 2073 ASSERT(slotcreate == 0); 2074 ASSERT(pg_idx == 0); 2075 conpp = NULL; 2076 prealloc = 0; 2077 page_free_pages(ppa[0]); 2078 } 2079 2080 if (prealloc && nreloc > 1) { 2081 /* 2082 * we have relocated out of a smaller large page. 2083 * skip npgs - 1 iterations and continue which will 2084 * increment by one the loop indices. 2085 */ 2086 spgcnt_t npgs = nreloc; 2087 2088 VM_STAT_ADD(anonvmstats.getpages[16]); 2089 2090 ASSERT(pp == ppa[pg_idx]); 2091 ASSERT(slotcreate == 0); 2092 ASSERT(pg_idx + npgs <= pgcnt); 2093 if ((*protp & PROT_WRITE) && 2094 anon_share(amp->ahp, an_idx, npgs)) { 2095 *protp &= ~PROT_WRITE; 2096 } 2097 pg_idx += npgs; 2098 an_idx += npgs; 2099 vaddr += PAGESIZE * npgs; 2100 continue; 2101 } 2102 2103 VM_STAT_ADD(anonvmstats.getpages[17]); 2104 2105 /* 2106 * Anon_zero case. 2107 */ 2108 if (slotcreate) { 2109 ASSERT(prealloc); 2110 pagezero(pp, 0, PAGESIZE); 2111 CPU_STATS_ADD_K(vm, zfod, 1); 2112 hat_setrefmod(pp); 2113 } 2114 2115 ASSERT(prealloc == 0 || ppa[pg_idx] == pp); 2116 ASSERT(prealloc != 0 || PAGE_SHARED(pp)); 2117 ASSERT(prealloc == 0 || PAGE_EXCL(pp)); 2118 2119 if (pg_idx > 0 && 2120 ((page_pptonum(pp) != page_pptonum(ppa[pg_idx - 1]) + 1) || 2121 (pp->p_szc != ppa[pg_idx - 1]->p_szc))) { 2122 panic("anon_map_getpages: unexpected page"); 2123 } else if (pg_idx == 0 && (page_pptonum(pp) & (pgcnt - 1))) { 2124 panic("anon_map_getpages: unaligned page"); 2125 } 2126 2127 if (prealloc == 0) { 2128 ppa[pg_idx] = pp; 2129 } 2130 2131 if (ap->an_refcnt > 1) { 2132 VM_STAT_ADD(anonvmstats.getpages[18]); 2133 *protp &= ~PROT_WRITE; 2134 } 2135 2136 /* 2137 * If this is a new anon slot then initialize 2138 * the anon array entry. 2139 */ 2140 if (slotcreate) { 2141 (void) anon_set_ptr(amp->ahp, an_idx, ap, ANON_SLEEP); 2142 } 2143 pg_idx++; 2144 an_idx++; 2145 vaddr += PAGESIZE; 2146 } 2147 2148 /* 2149 * Since preallocated pages come off the freelist 2150 * they are locked SE_EXCL. Simply downgrade and return. 2151 */ 2152 if (prealloc) { 2153 VM_STAT_ADD(anonvmstats.getpages[19]); 2154 conpp = NULL; 2155 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2156 page_downgrade(ppa[pg_idx]); 2157 } 2158 } 2159 ASSERT(conpp == NULL); 2160 2161 if (brkcow == 0 || (*protp & PROT_WRITE)) { 2162 VM_STAT_ADD(anonvmstats.getpages[20]); 2163 return (0); 2164 } 2165 2166 if (szc < seg->s_szc) 2167 panic("anon_map_getpages: cowfault for szc %d", szc); 2168 2169 VM_STAT_ADD(anonvmstats.getpages[21]); 2170 2171 *protp = PROT_ALL; 2172 return (anon_map_privatepages(amp, start_idx, szc, seg, addr, prot, 2173 ppa, vpage, anypgsz, pgflags, cred)); 2174 io_err: 2175 /* 2176 * We got an IO error somewhere in our large page. 2177 * If we were using a preallocated page then just demote 2178 * all the constituent pages that we've succeeded with sofar 2179 * to PAGESIZE pages and leave them in the system 2180 * unlocked. 2181 */ 2182 2183 ASSERT(err != -2 || ((pg_idx == 0) && upsize)); 2184 2185 VM_STAT_COND_ADD(err > 0, anonvmstats.getpages[22]); 2186 VM_STAT_COND_ADD(err == -1, anonvmstats.getpages[23]); 2187 VM_STAT_COND_ADD(err == -2, anonvmstats.getpages[24]); 2188 2189 if (prealloc) { 2190 conpp = NULL; 2191 if (pg_idx > 0) { 2192 VM_STAT_ADD(anonvmstats.getpages[25]); 2193 for (i = 0; i < pgcnt; i++) { 2194 pp = ppa[i]; 2195 ASSERT(PAGE_EXCL(pp)); 2196 ASSERT(pp->p_szc == szc); 2197 pp->p_szc = 0; 2198 } 2199 for (i = 0; i < pg_idx; i++) { 2200 ASSERT(!hat_page_is_mapped(ppa[i])); 2201 page_unlock(ppa[i]); 2202 } 2203 /* 2204 * Now free up the remaining unused constituent 2205 * pages. 2206 */ 2207 while (pg_idx < pgcnt) { 2208 ASSERT(!hat_page_is_mapped(ppa[pg_idx])); 2209 page_free(ppa[pg_idx], 0); 2210 pg_idx++; 2211 } 2212 } else { 2213 VM_STAT_ADD(anonvmstats.getpages[26]); 2214 page_free_pages(ppa[0]); 2215 } 2216 } else { 2217 VM_STAT_ADD(anonvmstats.getpages[27]); 2218 ASSERT(err > 0); 2219 for (i = 0; i < pg_idx; i++) 2220 page_unlock(ppa[i]); 2221 } 2222 ASSERT(conpp == NULL); 2223 if (err != -1) 2224 return (err); 2225 /* 2226 * we are here because we failed to relocate. 2227 */ 2228 ASSERT(prealloc); 2229 if (brkcow == 0 || szc < seg->s_szc || 2230 !anon_szcshare(amp->ahp, start_idx)) { 2231 VM_STAT_ADD(anonvmstats.getpages[28]); 2232 return (-1); 2233 } 2234 VM_STAT_ADD(anonvmstats.getpages[29]); 2235 goto docow; 2236 } 2237 2238 2239 /* 2240 * Turn a reference to an object or shared anon page 2241 * into a private page with a copy of the data from the 2242 * original page which is always locked by the caller. 2243 * This routine unloads the translation and unlocks the 2244 * original page, if it isn't being stolen, before returning 2245 * to the caller. 2246 * 2247 * NOTE: The original anon slot is not freed by this routine 2248 * It must be freed by the caller while holding the 2249 * "anon_map" lock to prevent races which can occur if 2250 * a process has multiple lwps in its address space. 2251 */ 2252 page_t * 2253 anon_private( 2254 struct anon **app, 2255 struct seg *seg, 2256 caddr_t addr, 2257 uint_t prot, 2258 page_t *opp, 2259 int oppflags, 2260 struct cred *cred) 2261 { 2262 struct anon *old = *app; 2263 struct anon *new; 2264 page_t *pp = NULL; 2265 struct vnode *vp; 2266 anoff_t off; 2267 page_t *anon_pl[1 + 1]; 2268 int err; 2269 2270 if (oppflags & STEAL_PAGE) 2271 ASSERT(PAGE_EXCL(opp)); 2272 else 2273 ASSERT(PAGE_LOCKED(opp)); 2274 2275 CPU_STATS_ADD_K(vm, cow_fault, 1); 2276 2277 /* Kernel probe */ 2278 TNF_PROBE_1(anon_private, "vm pagefault", /* CSTYLED */, 2279 tnf_opaque, address, addr); 2280 2281 *app = new = anon_alloc(NULL, 0); 2282 swap_xlate(new, &vp, &off); 2283 2284 if (oppflags & STEAL_PAGE) { 2285 page_rename(opp, vp, (u_offset_t)off); 2286 pp = opp; 2287 TRACE_5(TR_FAC_VM, TR_ANON_PRIVATE, 2288 "anon_private:seg %p addr %x pp %p vp %p off %lx", 2289 seg, addr, pp, vp, off); 2290 hat_setmod(pp); 2291 2292 /* bug 4026339 */ 2293 page_downgrade(pp); 2294 return (pp); 2295 } 2296 2297 /* 2298 * Call the VOP_GETPAGE routine to create the page, thereby 2299 * enabling the vnode driver to allocate any filesystem 2300 * space (e.g., disk block allocation for UFS). This also 2301 * prevents more than one page from being added to the 2302 * vnode at the same time. 2303 */ 2304 err = VOP_GETPAGE(vp, (u_offset_t)off, PAGESIZE, NULL, 2305 anon_pl, PAGESIZE, seg, addr, S_CREATE, cred, NULL); 2306 if (err) 2307 goto out; 2308 2309 pp = anon_pl[0]; 2310 2311 /* 2312 * If the original page was locked, we need to move the lock 2313 * to the new page by transfering 'cowcnt/lckcnt' of the original 2314 * page to 'cowcnt/lckcnt' of the new page. 2315 * 2316 * See Statement at the beginning of segvn_lockop() and 2317 * comments in page_pp_useclaim() regarding the way 2318 * cowcnts/lckcnts are handled. 2319 * 2320 * Also availrmem must be decremented up front for read only mapping 2321 * before calling page_pp_useclaim. page_pp_useclaim will bump it back 2322 * if availrmem did not need to be decremented after all. 2323 */ 2324 if (oppflags & LOCK_PAGE) { 2325 if ((prot & PROT_WRITE) == 0) { 2326 mutex_enter(&freemem_lock); 2327 if (availrmem > pages_pp_maximum) { 2328 availrmem--; 2329 pages_useclaim++; 2330 } else { 2331 mutex_exit(&freemem_lock); 2332 goto out; 2333 } 2334 mutex_exit(&freemem_lock); 2335 } 2336 page_pp_useclaim(opp, pp, prot & PROT_WRITE); 2337 } 2338 2339 /* 2340 * Now copy the contents from the original page, 2341 * which is locked and loaded in the MMU by 2342 * the caller to prevent yet another page fault. 2343 */ 2344 /* XXX - should set mod bit in here */ 2345 if (ppcopy(opp, pp) == 0) { 2346 /* 2347 * Before ppcopy could hanlde UE or other faults, we 2348 * would have panicked here, and still have no option 2349 * but to do so now. 2350 */ 2351 panic("anon_private, ppcopy failed, opp = 0x%p, pp = 0x%p", 2352 opp, pp); 2353 } 2354 2355 hat_setrefmod(pp); /* mark as modified */ 2356 2357 /* 2358 * Unload the old translation. 2359 */ 2360 hat_unload(seg->s_as->a_hat, addr, PAGESIZE, HAT_UNLOAD); 2361 2362 /* 2363 * Free unmapped, unmodified original page. 2364 * or release the lock on the original page, 2365 * otherwise the process will sleep forever in 2366 * anon_decref() waiting for the "exclusive" lock 2367 * on the page. 2368 */ 2369 (void) page_release(opp, 1); 2370 2371 /* 2372 * we are done with page creation so downgrade the new 2373 * page's selock to shared, this helps when multiple 2374 * as_fault(...SOFTLOCK...) are done to the same 2375 * page(aio) 2376 */ 2377 page_downgrade(pp); 2378 2379 /* 2380 * NOTE: The original anon slot must be freed by the 2381 * caller while holding the "anon_map" lock, if we 2382 * copied away from an anonymous page. 2383 */ 2384 return (pp); 2385 2386 out: 2387 *app = old; 2388 if (pp) 2389 page_unlock(pp); 2390 anon_decref(new); 2391 page_unlock(opp); 2392 return ((page_t *)NULL); 2393 } 2394 2395 int 2396 anon_map_privatepages( 2397 struct anon_map *amp, 2398 ulong_t start_idx, 2399 uint_t szc, 2400 struct seg *seg, 2401 caddr_t addr, 2402 uint_t prot, 2403 page_t *ppa[], 2404 struct vpage vpage[], 2405 int anypgsz, 2406 int pgflags, 2407 struct cred *cred) 2408 { 2409 pgcnt_t pgcnt; 2410 struct vnode *vp; 2411 anoff_t off; 2412 page_t *pl[2], *conpp = NULL; 2413 int err; 2414 int prealloc = 1; 2415 struct anon *ap, *oldap; 2416 caddr_t vaddr; 2417 page_t *pplist, *pp; 2418 ulong_t pg_idx, an_idx; 2419 spgcnt_t nreloc = 0; 2420 int pagelock = 0; 2421 kmutex_t *ahmpages = NULL; 2422 #ifdef DEBUG 2423 int refcnt; 2424 #endif 2425 2426 ASSERT(szc != 0); 2427 ASSERT(szc == seg->s_szc); 2428 2429 VM_STAT_ADD(anonvmstats.privatepages[0]); 2430 2431 pgcnt = page_get_pagecnt(szc); 2432 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 2433 ASSERT(IS_P2ALIGNED(start_idx, pgcnt)); 2434 2435 ASSERT(amp != NULL); 2436 ap = anon_get_ptr(amp->ahp, start_idx); 2437 ASSERT(ap == NULL || ap->an_refcnt >= 1); 2438 2439 VM_STAT_COND_ADD(ap == NULL, anonvmstats.privatepages[1]); 2440 2441 /* 2442 * Now try and allocate the large page. If we fail then just 2443 * let VOP_GETPAGE give us PAGESIZE pages. Normally we let 2444 * the caller make this decision but to avoid added complexity 2445 * it's simplier to handle that case here. 2446 */ 2447 if (anypgsz == -1) { 2448 VM_STAT_ADD(anonvmstats.privatepages[2]); 2449 prealloc = 0; 2450 } else if (page_alloc_pages(anon_vp, seg, addr, &pplist, NULL, szc, 2451 anypgsz, pgflags) != 0) { 2452 VM_STAT_ADD(anonvmstats.privatepages[3]); 2453 prealloc = 0; 2454 } 2455 2456 /* 2457 * make the decrement of all refcnts of all 2458 * anon slots of a large page appear atomic by 2459 * getting an anonpages_hash_lock for the 2460 * first anon slot of a large page. 2461 */ 2462 if (ap != NULL) { 2463 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, 2464 ap->an_off)]; 2465 mutex_enter(ahmpages); 2466 if (ap->an_refcnt == 1) { 2467 VM_STAT_ADD(anonvmstats.privatepages[4]); 2468 ASSERT(!anon_share(amp->ahp, start_idx, pgcnt)); 2469 mutex_exit(ahmpages); 2470 2471 if (prealloc) { 2472 page_free_replacement_page(pplist); 2473 page_create_putback(pgcnt); 2474 } 2475 ASSERT(ppa[0]->p_szc <= szc); 2476 if (ppa[0]->p_szc == szc) { 2477 VM_STAT_ADD(anonvmstats.privatepages[5]); 2478 return (0); 2479 } 2480 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2481 ASSERT(ppa[pg_idx] != NULL); 2482 page_unlock(ppa[pg_idx]); 2483 } 2484 return (-1); 2485 } 2486 } 2487 2488 /* 2489 * If we are passed in the vpage array and this is 2490 * not PROT_WRITE then we need to decrement availrmem 2491 * up front before we try anything. If we need to and 2492 * can't decrement availrmem then its better to fail now 2493 * than in the middle of processing the new large page. 2494 * page_pp_usclaim() on behalf of each constituent page 2495 * below will adjust availrmem back for the cases not needed. 2496 */ 2497 if (vpage != NULL && (prot & PROT_WRITE) == 0) { 2498 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2499 if (VPP_ISPPLOCK(&vpage[pg_idx])) { 2500 pagelock = 1; 2501 break; 2502 } 2503 } 2504 if (pagelock) { 2505 VM_STAT_ADD(anonvmstats.privatepages[6]); 2506 mutex_enter(&freemem_lock); 2507 if (availrmem >= pages_pp_maximum + pgcnt) { 2508 availrmem -= pgcnt; 2509 pages_useclaim += pgcnt; 2510 } else { 2511 VM_STAT_ADD(anonvmstats.privatepages[7]); 2512 mutex_exit(&freemem_lock); 2513 if (ahmpages != NULL) { 2514 mutex_exit(ahmpages); 2515 } 2516 if (prealloc) { 2517 page_free_replacement_page(pplist); 2518 page_create_putback(pgcnt); 2519 } 2520 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) 2521 if (ppa[pg_idx] != NULL) 2522 page_unlock(ppa[pg_idx]); 2523 return (ENOMEM); 2524 } 2525 mutex_exit(&freemem_lock); 2526 } 2527 } 2528 2529 CPU_STATS_ADD_K(vm, cow_fault, pgcnt); 2530 2531 VM_STAT_ADD(anonvmstats.privatepages[8]); 2532 2533 an_idx = start_idx; 2534 pg_idx = 0; 2535 vaddr = addr; 2536 for (; pg_idx < pgcnt; pg_idx++, an_idx++, vaddr += PAGESIZE) { 2537 ASSERT(ppa[pg_idx] != NULL); 2538 oldap = anon_get_ptr(amp->ahp, an_idx); 2539 ASSERT(ahmpages != NULL || oldap == NULL); 2540 ASSERT(ahmpages == NULL || oldap != NULL); 2541 ASSERT(ahmpages == NULL || oldap->an_refcnt > 1); 2542 ASSERT(ahmpages == NULL || pg_idx != 0 || 2543 (refcnt = oldap->an_refcnt)); 2544 ASSERT(ahmpages == NULL || pg_idx == 0 || 2545 refcnt == oldap->an_refcnt); 2546 2547 ap = anon_alloc(NULL, 0); 2548 2549 swap_xlate(ap, &vp, &off); 2550 2551 /* 2552 * Now setup our preallocated page to pass down to 2553 * swap_getpage(). 2554 */ 2555 if (prealloc) { 2556 pp = pplist; 2557 page_sub(&pplist, pp); 2558 conpp = pp; 2559 } 2560 2561 err = swap_getconpage(vp, (u_offset_t)off, PAGESIZE, NULL, pl, 2562 PAGESIZE, conpp, NULL, &nreloc, seg, vaddr, 2563 S_CREATE, cred); 2564 2565 /* 2566 * Impossible to fail this is S_CREATE. 2567 */ 2568 if (err) 2569 panic("anon_map_privatepages: VOP_GETPAGE failed"); 2570 2571 ASSERT(prealloc ? pp == pl[0] : pl[0]->p_szc == 0); 2572 ASSERT(prealloc == 0 || nreloc == 1); 2573 2574 pp = pl[0]; 2575 2576 /* 2577 * If the original page was locked, we need to move 2578 * the lock to the new page by transfering 2579 * 'cowcnt/lckcnt' of the original page to 'cowcnt/lckcnt' 2580 * of the new page. pg_idx can be used to index 2581 * into the vpage array since the caller will guarentee 2582 * that vpage struct passed in corresponds to addr 2583 * and forward. 2584 */ 2585 if (vpage != NULL && VPP_ISPPLOCK(&vpage[pg_idx])) { 2586 page_pp_useclaim(ppa[pg_idx], pp, prot & PROT_WRITE); 2587 } else if (pagelock) { 2588 mutex_enter(&freemem_lock); 2589 availrmem++; 2590 pages_useclaim--; 2591 mutex_exit(&freemem_lock); 2592 } 2593 2594 /* 2595 * Now copy the contents from the original page. 2596 */ 2597 if (ppcopy(ppa[pg_idx], pp) == 0) { 2598 /* 2599 * Before ppcopy could hanlde UE or other faults, we 2600 * would have panicked here, and still have no option 2601 * but to do so now. 2602 */ 2603 panic("anon_map_privatepages, ppcopy failed"); 2604 } 2605 2606 hat_setrefmod(pp); /* mark as modified */ 2607 2608 /* 2609 * Release the lock on the original page, 2610 * derement the old slot, and down grade the lock 2611 * on the new copy. 2612 */ 2613 page_unlock(ppa[pg_idx]); 2614 2615 if (!prealloc) 2616 page_downgrade(pp); 2617 2618 ppa[pg_idx] = pp; 2619 2620 /* 2621 * Now reflect the copy in the new anon array. 2622 */ 2623 ASSERT(ahmpages == NULL || oldap->an_refcnt > 1); 2624 if (oldap != NULL) 2625 anon_decref(oldap); 2626 (void) anon_set_ptr(amp->ahp, an_idx, ap, ANON_SLEEP); 2627 } 2628 2629 /* 2630 * Unload the old large page translation. 2631 */ 2632 hat_unload(seg->s_as->a_hat, addr, pgcnt << PAGESHIFT, HAT_UNLOAD); 2633 2634 if (ahmpages != NULL) { 2635 mutex_exit(ahmpages); 2636 } 2637 ASSERT(prealloc == 0 || pplist == NULL); 2638 if (prealloc) { 2639 VM_STAT_ADD(anonvmstats.privatepages[9]); 2640 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2641 page_downgrade(ppa[pg_idx]); 2642 } 2643 } 2644 2645 return (0); 2646 } 2647 2648 /* 2649 * Allocate a private zero-filled anon page. 2650 */ 2651 page_t * 2652 anon_zero(struct seg *seg, caddr_t addr, struct anon **app, struct cred *cred) 2653 { 2654 struct anon *ap; 2655 page_t *pp; 2656 struct vnode *vp; 2657 anoff_t off; 2658 page_t *anon_pl[1 + 1]; 2659 int err; 2660 2661 /* Kernel probe */ 2662 TNF_PROBE_1(anon_zero, "vm pagefault", /* CSTYLED */, 2663 tnf_opaque, address, addr); 2664 2665 *app = ap = anon_alloc(NULL, 0); 2666 swap_xlate(ap, &vp, &off); 2667 2668 /* 2669 * Call the VOP_GETPAGE routine to create the page, thereby 2670 * enabling the vnode driver to allocate any filesystem 2671 * dependent structures (e.g., disk block allocation for UFS). 2672 * This also prevents more than on page from being added to 2673 * the vnode at the same time since it is locked. 2674 */ 2675 err = VOP_GETPAGE(vp, off, PAGESIZE, NULL, 2676 anon_pl, PAGESIZE, seg, addr, S_CREATE, cred, NULL); 2677 if (err) { 2678 *app = NULL; 2679 anon_decref(ap); 2680 return (NULL); 2681 } 2682 pp = anon_pl[0]; 2683 2684 pagezero(pp, 0, PAGESIZE); /* XXX - should set mod bit */ 2685 page_downgrade(pp); 2686 CPU_STATS_ADD_K(vm, zfod, 1); 2687 hat_setrefmod(pp); /* mark as modified so pageout writes back */ 2688 return (pp); 2689 } 2690 2691 2692 /* 2693 * Allocate array of private zero-filled anon pages for empty slots 2694 * and kept pages for non empty slots within given range. 2695 * 2696 * NOTE: This rontine will try and use large pages 2697 * if available and supported by underlying platform. 2698 */ 2699 int 2700 anon_map_createpages( 2701 struct anon_map *amp, 2702 ulong_t start_index, 2703 size_t len, 2704 page_t *ppa[], 2705 struct seg *seg, 2706 caddr_t addr, 2707 enum seg_rw rw, 2708 struct cred *cred) 2709 { 2710 2711 struct anon *ap; 2712 struct vnode *ap_vp; 2713 page_t *pp, *pplist, *anon_pl[1 + 1], *conpp = NULL; 2714 int err = 0; 2715 ulong_t p_index, index; 2716 pgcnt_t npgs, pg_cnt; 2717 spgcnt_t nreloc = 0; 2718 uint_t l_szc, szc, prot; 2719 anoff_t ap_off; 2720 size_t pgsz; 2721 lgrp_t *lgrp; 2722 kmutex_t *ahm; 2723 2724 /* 2725 * XXX For now only handle S_CREATE. 2726 */ 2727 ASSERT(rw == S_CREATE); 2728 2729 index = start_index; 2730 p_index = 0; 2731 npgs = btopr(len); 2732 2733 /* 2734 * If this platform supports multiple page sizes 2735 * then try and allocate directly from the free 2736 * list for pages larger than PAGESIZE. 2737 * 2738 * NOTE:When we have page_create_ru we can stop 2739 * directly allocating from the freelist. 2740 */ 2741 l_szc = seg->s_szc; 2742 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 2743 while (npgs) { 2744 2745 /* 2746 * if anon slot already exists 2747 * (means page has been created) 2748 * so 1) look up the page 2749 * 2) if the page is still in memory, get it. 2750 * 3) if not, create a page and 2751 * page in from physical swap device. 2752 * These are done in anon_getpage(). 2753 */ 2754 ap = anon_get_ptr(amp->ahp, index); 2755 if (ap) { 2756 err = anon_getpage(&ap, &prot, anon_pl, PAGESIZE, 2757 seg, addr, S_READ, cred); 2758 if (err) { 2759 ANON_LOCK_EXIT(&->a_rwlock); 2760 panic("anon_map_createpages: anon_getpage"); 2761 } 2762 pp = anon_pl[0]; 2763 ppa[p_index++] = pp; 2764 2765 /* 2766 * an_pvp can become non-NULL after SysV's page was 2767 * paged out before ISM was attached to this SysV 2768 * shared memory segment. So free swap slot if needed. 2769 */ 2770 if (ap->an_pvp != NULL) { 2771 page_io_lock(pp); 2772 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, 2773 ap->an_off)]; 2774 mutex_enter(ahm); 2775 if (ap->an_pvp != NULL) { 2776 swap_phys_free(ap->an_pvp, 2777 ap->an_poff, PAGESIZE); 2778 ap->an_pvp = NULL; 2779 ap->an_poff = 0; 2780 mutex_exit(ahm); 2781 hat_setmod(pp); 2782 } else { 2783 mutex_exit(ahm); 2784 } 2785 page_io_unlock(pp); 2786 } 2787 2788 addr += PAGESIZE; 2789 index++; 2790 npgs--; 2791 continue; 2792 } 2793 /* 2794 * Now try and allocate the largest page possible 2795 * for the current address and range. 2796 * Keep dropping down in page size until: 2797 * 2798 * 1) Properly aligned 2799 * 2) Does not overlap existing anon pages 2800 * 3) Fits in remaining range. 2801 * 4) able to allocate one. 2802 * 2803 * NOTE: XXX When page_create_ru is completed this code 2804 * will change. 2805 */ 2806 szc = l_szc; 2807 pplist = NULL; 2808 pg_cnt = 0; 2809 while (szc) { 2810 pgsz = page_get_pagesize(szc); 2811 pg_cnt = pgsz >> PAGESHIFT; 2812 if (IS_P2ALIGNED(addr, pgsz) && pg_cnt <= npgs && 2813 anon_pages(amp->ahp, index, pg_cnt) == 0) { 2814 /* 2815 * XXX 2816 * Since we are faking page_create() 2817 * we also need to do the freemem and 2818 * pcf accounting. 2819 */ 2820 (void) page_create_wait(pg_cnt, PG_WAIT); 2821 2822 /* 2823 * Get lgroup to allocate next page of shared 2824 * memory from and use it to specify where to 2825 * allocate the physical memory 2826 */ 2827 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2828 2829 pplist = page_get_freelist( 2830 anon_vp, (u_offset_t)0, seg, 2831 addr, pgsz, 0, lgrp); 2832 2833 if (pplist == NULL) { 2834 page_create_putback(pg_cnt); 2835 } 2836 2837 /* 2838 * If a request for a page of size 2839 * larger than PAGESIZE failed 2840 * then don't try that size anymore. 2841 */ 2842 if (pplist == NULL) { 2843 l_szc = szc - 1; 2844 } else { 2845 break; 2846 } 2847 } 2848 szc--; 2849 } 2850 2851 /* 2852 * If just using PAGESIZE pages then don't 2853 * directly allocate from the free list. 2854 */ 2855 if (pplist == NULL) { 2856 ASSERT(szc == 0); 2857 pp = anon_zero(seg, addr, &ap, cred); 2858 if (pp == NULL) { 2859 ANON_LOCK_EXIT(&->a_rwlock); 2860 panic("anon_map_createpages: anon_zero"); 2861 } 2862 ppa[p_index++] = pp; 2863 2864 ASSERT(anon_get_ptr(amp->ahp, index) == NULL); 2865 (void) anon_set_ptr(amp->ahp, index, ap, ANON_SLEEP); 2866 2867 addr += PAGESIZE; 2868 index++; 2869 npgs--; 2870 continue; 2871 } 2872 2873 /* 2874 * pplist is a list of pg_cnt PAGESIZE pages. 2875 * These pages are locked SE_EXCL since they 2876 * came directly off the free list. 2877 */ 2878 ASSERT(IS_P2ALIGNED(pg_cnt, pg_cnt)); 2879 ASSERT(IS_P2ALIGNED(index, pg_cnt)); 2880 ASSERT(conpp == NULL); 2881 while (pg_cnt--) { 2882 2883 ap = anon_alloc(NULL, 0); 2884 swap_xlate(ap, &ap_vp, &ap_off); 2885 2886 ASSERT(pplist != NULL); 2887 pp = pplist; 2888 page_sub(&pplist, pp); 2889 PP_CLRFREE(pp); 2890 PP_CLRAGED(pp); 2891 conpp = pp; 2892 2893 err = swap_getconpage(ap_vp, ap_off, PAGESIZE, 2894 (uint_t *)NULL, anon_pl, PAGESIZE, conpp, NULL, 2895 &nreloc, seg, addr, S_CREATE, cred); 2896 2897 if (err) { 2898 ANON_LOCK_EXIT(&->a_rwlock); 2899 panic("anon_map_createpages: S_CREATE"); 2900 } 2901 2902 ASSERT(anon_pl[0] == pp); 2903 ASSERT(nreloc == 1); 2904 pagezero(pp, 0, PAGESIZE); 2905 CPU_STATS_ADD_K(vm, zfod, 1); 2906 hat_setrefmod(pp); 2907 2908 ASSERT(anon_get_ptr(amp->ahp, index) == NULL); 2909 (void) anon_set_ptr(amp->ahp, index, ap, ANON_SLEEP); 2910 2911 ppa[p_index++] = pp; 2912 2913 addr += PAGESIZE; 2914 index++; 2915 npgs--; 2916 } 2917 conpp = NULL; 2918 pg_cnt = pgsz >> PAGESHIFT; 2919 p_index = p_index - pg_cnt; 2920 while (pg_cnt--) { 2921 page_downgrade(ppa[p_index++]); 2922 } 2923 } 2924 ANON_LOCK_EXIT(&->a_rwlock); 2925 return (0); 2926 } 2927 2928 static int 2929 anon_try_demote_pages( 2930 struct anon_hdr *ahp, 2931 ulong_t sidx, 2932 uint_t szc, 2933 page_t **ppa, 2934 int private) 2935 { 2936 struct anon *ap; 2937 pgcnt_t pgcnt = page_get_pagecnt(szc); 2938 page_t *pp; 2939 pgcnt_t i; 2940 kmutex_t *ahmpages = NULL; 2941 int root = 0; 2942 pgcnt_t npgs; 2943 pgcnt_t curnpgs = 0; 2944 size_t ppasize = 0; 2945 2946 ASSERT(szc != 0); 2947 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 2948 ASSERT(IS_P2ALIGNED(sidx, pgcnt)); 2949 ASSERT(sidx < ahp->size); 2950 2951 if (ppa == NULL) { 2952 ppasize = pgcnt * sizeof (page_t *); 2953 ppa = kmem_alloc(ppasize, KM_SLEEP); 2954 } 2955 2956 ap = anon_get_ptr(ahp, sidx); 2957 if (ap != NULL && private) { 2958 VM_STAT_ADD(anonvmstats.demotepages[1]); 2959 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 2960 mutex_enter(ahmpages); 2961 } 2962 2963 if (ap != NULL && ap->an_refcnt > 1) { 2964 if (ahmpages != NULL) { 2965 VM_STAT_ADD(anonvmstats.demotepages[2]); 2966 mutex_exit(ahmpages); 2967 } 2968 if (ppasize != 0) { 2969 kmem_free(ppa, ppasize); 2970 } 2971 return (0); 2972 } 2973 if (ahmpages != NULL) { 2974 mutex_exit(ahmpages); 2975 } 2976 if (ahp->size - sidx < pgcnt) { 2977 ASSERT(private == 0); 2978 pgcnt = ahp->size - sidx; 2979 } 2980 for (i = 0; i < pgcnt; i++, sidx++) { 2981 ap = anon_get_ptr(ahp, sidx); 2982 if (ap != NULL) { 2983 if (ap->an_refcnt != 1) { 2984 panic("anon_try_demote_pages: an_refcnt != 1"); 2985 } 2986 pp = ppa[i] = page_lookup(ap->an_vp, ap->an_off, 2987 SE_EXCL); 2988 if (pp != NULL) { 2989 (void) hat_pageunload(pp, 2990 HAT_FORCE_PGUNLOAD); 2991 } 2992 } else { 2993 ppa[i] = NULL; 2994 } 2995 } 2996 for (i = 0; i < pgcnt; i++) { 2997 if ((pp = ppa[i]) != NULL && pp->p_szc != 0) { 2998 ASSERT(pp->p_szc <= szc); 2999 if (!root) { 3000 VM_STAT_ADD(anonvmstats.demotepages[3]); 3001 if (curnpgs != 0) 3002 panic("anon_try_demote_pages: " 3003 "bad large page"); 3004 3005 root = 1; 3006 curnpgs = npgs = 3007 page_get_pagecnt(pp->p_szc); 3008 3009 ASSERT(npgs <= pgcnt); 3010 ASSERT(IS_P2ALIGNED(npgs, npgs)); 3011 ASSERT(!(page_pptonum(pp) & (npgs - 1))); 3012 } else { 3013 ASSERT(i > 0); 3014 ASSERT(page_pptonum(pp) - 1 == 3015 page_pptonum(ppa[i - 1])); 3016 if ((page_pptonum(pp) & (npgs - 1)) == 3017 npgs - 1) 3018 root = 0; 3019 } 3020 ASSERT(PAGE_EXCL(pp)); 3021 pp->p_szc = 0; 3022 ASSERT(curnpgs > 0); 3023 curnpgs--; 3024 } 3025 } 3026 if (root != 0 || curnpgs != 0) 3027 panic("anon_try_demote_pages: bad large page"); 3028 3029 for (i = 0; i < pgcnt; i++) { 3030 if ((pp = ppa[i]) != NULL) { 3031 ASSERT(!hat_page_is_mapped(pp)); 3032 ASSERT(pp->p_szc == 0); 3033 page_unlock(pp); 3034 } 3035 } 3036 if (ppasize != 0) { 3037 kmem_free(ppa, ppasize); 3038 } 3039 return (1); 3040 } 3041 3042 /* 3043 * anon_map_demotepages() can only be called by MAP_PRIVATE segments. 3044 */ 3045 int 3046 anon_map_demotepages( 3047 struct anon_map *amp, 3048 ulong_t start_idx, 3049 struct seg *seg, 3050 caddr_t addr, 3051 uint_t prot, 3052 struct vpage vpage[], 3053 struct cred *cred) 3054 { 3055 struct anon *ap; 3056 uint_t szc = seg->s_szc; 3057 pgcnt_t pgcnt = page_get_pagecnt(szc); 3058 size_t ppasize = pgcnt * sizeof (page_t *); 3059 page_t **ppa = kmem_alloc(ppasize, KM_SLEEP); 3060 page_t *pp; 3061 page_t *pl[2]; 3062 pgcnt_t i, pg_idx; 3063 ulong_t an_idx; 3064 caddr_t vaddr; 3065 int err; 3066 int retry = 0; 3067 uint_t vpprot; 3068 3069 ASSERT(RW_WRITE_HELD(&->a_rwlock)); 3070 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 3071 ASSERT(IS_P2ALIGNED(start_idx, pgcnt)); 3072 ASSERT(ppa != NULL); 3073 ASSERT(szc != 0); 3074 ASSERT(szc == amp->a_szc); 3075 3076 VM_STAT_ADD(anonvmstats.demotepages[0]); 3077 3078 top: 3079 if (anon_try_demote_pages(amp->ahp, start_idx, szc, ppa, 1)) { 3080 kmem_free(ppa, ppasize); 3081 return (0); 3082 } 3083 3084 VM_STAT_ADD(anonvmstats.demotepages[4]); 3085 3086 ASSERT(retry == 0); /* we can be here only once */ 3087 3088 vaddr = addr; 3089 for (pg_idx = 0, an_idx = start_idx; pg_idx < pgcnt; 3090 pg_idx++, an_idx++, vaddr += PAGESIZE) { 3091 ap = anon_get_ptr(amp->ahp, an_idx); 3092 if (ap == NULL) 3093 panic("anon_map_demotepages: no anon slot"); 3094 err = anon_getpage(&ap, &vpprot, pl, PAGESIZE, seg, vaddr, 3095 S_READ, cred); 3096 if (err) { 3097 for (i = 0; i < pg_idx; i++) { 3098 if ((pp = ppa[i]) != NULL) 3099 page_unlock(pp); 3100 } 3101 kmem_free(ppa, ppasize); 3102 return (err); 3103 } 3104 ppa[pg_idx] = pl[0]; 3105 } 3106 3107 err = anon_map_privatepages(amp, start_idx, szc, seg, addr, prot, ppa, 3108 vpage, -1, 0, cred); 3109 if (err > 0) { 3110 VM_STAT_ADD(anonvmstats.demotepages[5]); 3111 kmem_free(ppa, ppasize); 3112 return (err); 3113 } 3114 ASSERT(err == 0 || err == -1); 3115 if (err == -1) { 3116 VM_STAT_ADD(anonvmstats.demotepages[6]); 3117 retry = 1; 3118 goto top; 3119 } 3120 for (i = 0; i < pgcnt; i++) { 3121 ASSERT(ppa[i] != NULL); 3122 if (ppa[i]->p_szc != 0) 3123 retry = 1; 3124 page_unlock(ppa[i]); 3125 } 3126 if (retry) { 3127 VM_STAT_ADD(anonvmstats.demotepages[7]); 3128 goto top; 3129 } 3130 3131 VM_STAT_ADD(anonvmstats.demotepages[8]); 3132 3133 kmem_free(ppa, ppasize); 3134 3135 return (0); 3136 } 3137 3138 /* 3139 * Free pages of shared anon map. It's assumed that anon maps don't share anon 3140 * structures with private anon maps. Therefore all anon structures should 3141 * have at most one reference at this point. This means underlying pages can 3142 * be exclusively locked and demoted or freed. If not freeing the entire 3143 * large pages demote the ends of the region we free to be able to free 3144 * subpages. Page roots correspond to aligned index positions in anon map. 3145 */ 3146 void 3147 anon_shmap_free_pages(struct anon_map *amp, ulong_t sidx, size_t len) 3148 { 3149 ulong_t eidx = sidx + btopr(len); 3150 pgcnt_t pages = page_get_pagecnt(amp->a_szc); 3151 struct anon_hdr *ahp = amp->ahp; 3152 ulong_t tidx; 3153 size_t size; 3154 ulong_t sidx_aligned; 3155 ulong_t eidx_aligned; 3156 3157 ASSERT(RW_WRITE_HELD(&->a_rwlock)); 3158 ASSERT(amp->refcnt <= 1); 3159 ASSERT(amp->a_szc > 0); 3160 ASSERT(eidx <= ahp->size); 3161 ASSERT(!anon_share(ahp, sidx, btopr(len))); 3162 3163 if (len == 0) { /* XXX */ 3164 return; 3165 } 3166 3167 sidx_aligned = P2ALIGN(sidx, pages); 3168 if (sidx_aligned != sidx || 3169 (eidx < sidx_aligned + pages && eidx < ahp->size)) { 3170 if (!anon_try_demote_pages(ahp, sidx_aligned, 3171 amp->a_szc, NULL, 0)) { 3172 panic("anon_shmap_free_pages: demote failed"); 3173 } 3174 size = (eidx <= sidx_aligned + pages) ? (eidx - sidx) : 3175 P2NPHASE(sidx, pages); 3176 size <<= PAGESHIFT; 3177 anon_free(ahp, sidx, size); 3178 sidx = sidx_aligned + pages; 3179 if (eidx <= sidx) { 3180 return; 3181 } 3182 } 3183 eidx_aligned = P2ALIGN(eidx, pages); 3184 if (sidx < eidx_aligned) { 3185 anon_free_pages(ahp, sidx, 3186 (eidx_aligned - sidx) << PAGESHIFT, 3187 amp->a_szc); 3188 sidx = eidx_aligned; 3189 } 3190 ASSERT(sidx == eidx_aligned); 3191 if (eidx == eidx_aligned) { 3192 return; 3193 } 3194 tidx = eidx; 3195 if (eidx != ahp->size && anon_get_next_ptr(ahp, &tidx) != NULL && 3196 tidx - sidx < pages) { 3197 if (!anon_try_demote_pages(ahp, sidx, amp->a_szc, NULL, 0)) { 3198 panic("anon_shmap_free_pages: demote failed"); 3199 } 3200 size = (eidx - sidx) << PAGESHIFT; 3201 anon_free(ahp, sidx, size); 3202 } else { 3203 anon_free_pages(ahp, sidx, pages << PAGESHIFT, amp->a_szc); 3204 } 3205 } 3206 3207 /* 3208 * Allocate and initialize an anon_map structure for seg 3209 * associating the given swap reservation with the new anon_map. 3210 */ 3211 struct anon_map * 3212 anonmap_alloc(size_t size, size_t swresv, int flags) 3213 { 3214 struct anon_map *amp; 3215 int kmflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 3216 3217 amp = kmem_cache_alloc(anonmap_cache, kmflags); 3218 if (amp == NULL) { 3219 ASSERT(kmflags == KM_NOSLEEP); 3220 return (NULL); 3221 } 3222 3223 amp->ahp = anon_create(btopr(size), flags); 3224 if (amp->ahp == NULL) { 3225 ASSERT(flags == ANON_NOSLEEP); 3226 kmem_cache_free(anonmap_cache, amp); 3227 return (NULL); 3228 } 3229 amp->refcnt = 1; 3230 amp->size = size; 3231 amp->swresv = swresv; 3232 amp->locality = 0; 3233 amp->a_szc = 0; 3234 amp->a_sp = NULL; 3235 return (amp); 3236 } 3237 3238 void 3239 anonmap_free(struct anon_map *amp) 3240 { 3241 ASSERT(amp->ahp); 3242 ASSERT(amp->refcnt == 0); 3243 3244 lgrp_shm_policy_fini(amp, NULL); 3245 anon_release(amp->ahp, btopr(amp->size)); 3246 kmem_cache_free(anonmap_cache, amp); 3247 } 3248 3249 /* 3250 * Returns true if the app array has some empty slots. 3251 * The offp and lenp parameters are in/out parameters. On entry 3252 * these values represent the starting offset and length of the 3253 * mapping. When true is returned, these values may be modified 3254 * to be the largest range which includes empty slots. 3255 */ 3256 int 3257 non_anon(struct anon_hdr *ahp, ulong_t anon_idx, u_offset_t *offp, 3258 size_t *lenp) 3259 { 3260 ulong_t i, el; 3261 ssize_t low, high; 3262 struct anon *ap; 3263 3264 low = -1; 3265 for (i = 0, el = *lenp; i < el; i += PAGESIZE, anon_idx++) { 3266 ap = anon_get_ptr(ahp, anon_idx); 3267 if (ap == NULL) { 3268 if (low == -1) 3269 low = i; 3270 high = i; 3271 } 3272 } 3273 if (low != -1) { 3274 /* 3275 * Found at least one non-anon page. 3276 * Set up the off and len return values. 3277 */ 3278 if (low != 0) 3279 *offp += low; 3280 *lenp = high - low + PAGESIZE; 3281 return (1); 3282 } 3283 return (0); 3284 } 3285 3286 /* 3287 * Return a count of the number of existing anon pages in the anon array 3288 * app in the range (off, off+len). The array and slots must be guaranteed 3289 * stable by the caller. 3290 */ 3291 pgcnt_t 3292 anon_pages(struct anon_hdr *ahp, ulong_t anon_index, pgcnt_t nslots) 3293 { 3294 pgcnt_t cnt = 0; 3295 3296 while (nslots-- > 0) { 3297 if ((anon_get_ptr(ahp, anon_index)) != NULL) 3298 cnt++; 3299 anon_index++; 3300 } 3301 return (cnt); 3302 } 3303 3304 /* 3305 * Move reserved phys swap into memory swap (unreserve phys swap 3306 * and reserve mem swap by the same amount). 3307 * Used by segspt when it needs to lock reserved swap npages in memory 3308 */ 3309 int 3310 anon_swap_adjust(pgcnt_t npages) 3311 { 3312 pgcnt_t unlocked_mem_swap; 3313 3314 mutex_enter(&anoninfo_lock); 3315 3316 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 3317 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 3318 3319 unlocked_mem_swap = k_anoninfo.ani_mem_resv 3320 - k_anoninfo.ani_locked_swap; 3321 if (npages > unlocked_mem_swap) { 3322 spgcnt_t adjusted_swap = npages - unlocked_mem_swap; 3323 3324 /* 3325 * if there is not enough unlocked mem swap we take missing 3326 * amount from phys swap and give it to mem swap 3327 */ 3328 if (!page_reclaim_mem(adjusted_swap, segspt_minfree, 1)) { 3329 mutex_exit(&anoninfo_lock); 3330 return (ENOMEM); 3331 } 3332 3333 k_anoninfo.ani_mem_resv += adjusted_swap; 3334 ASSERT(k_anoninfo.ani_phys_resv >= adjusted_swap); 3335 k_anoninfo.ani_phys_resv -= adjusted_swap; 3336 3337 ANI_ADD(adjusted_swap); 3338 } 3339 k_anoninfo.ani_locked_swap += npages; 3340 3341 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 3342 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 3343 3344 mutex_exit(&anoninfo_lock); 3345 3346 return (0); 3347 } 3348 3349 /* 3350 * 'unlocked' reserved mem swap so when it is unreserved it 3351 * can be moved back phys (disk) swap 3352 */ 3353 void 3354 anon_swap_restore(pgcnt_t npages) 3355 { 3356 mutex_enter(&anoninfo_lock); 3357 3358 ASSERT(k_anoninfo.ani_locked_swap <= k_anoninfo.ani_mem_resv); 3359 3360 ASSERT(k_anoninfo.ani_locked_swap >= npages); 3361 k_anoninfo.ani_locked_swap -= npages; 3362 3363 ASSERT(k_anoninfo.ani_locked_swap <= k_anoninfo.ani_mem_resv); 3364 3365 mutex_exit(&anoninfo_lock); 3366 } 3367 3368 /* 3369 * Return the pointer from the list for a 3370 * specified anon index. 3371 */ 3372 ulong_t * 3373 anon_get_slot(struct anon_hdr *ahp, ulong_t an_idx) 3374 { 3375 struct anon **app; 3376 void **ppp; 3377 3378 ASSERT(an_idx < ahp->size); 3379 3380 /* 3381 * Single level case. 3382 */ 3383 if ((ahp->size <= ANON_CHUNK_SIZE) || (ahp->flags & ANON_ALLOC_FORCE)) { 3384 return ((ulong_t *)&ahp->array_chunk[an_idx]); 3385 } else { 3386 3387 /* 3388 * 2 level case. 3389 */ 3390 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 3391 if (*ppp == NULL) { 3392 mutex_enter(&ahp->serial_lock); 3393 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 3394 if (*ppp == NULL) 3395 *ppp = kmem_zalloc(PAGESIZE, KM_SLEEP); 3396 mutex_exit(&ahp->serial_lock); 3397 } 3398 app = *ppp; 3399 return ((ulong_t *)&app[an_idx & ANON_CHUNK_OFF]); 3400 } 3401 } 3402 3403 void 3404 anon_array_enter(struct anon_map *amp, ulong_t an_idx, anon_sync_obj_t *sobj) 3405 { 3406 ulong_t *ap_slot; 3407 kmutex_t *mtx; 3408 kcondvar_t *cv; 3409 int hash; 3410 3411 /* 3412 * Use szc to determine anon slot(s) to appear atomic. 3413 * If szc = 0, then lock the anon slot and mark it busy. 3414 * If szc > 0, then lock the range of slots by getting the 3415 * anon_array_lock for the first anon slot, and mark only the 3416 * first anon slot busy to represent whole range being busy. 3417 */ 3418 3419 ASSERT(RW_READ_HELD(&->a_rwlock)); 3420 an_idx = P2ALIGN(an_idx, page_get_pagecnt(amp->a_szc)); 3421 hash = ANON_ARRAY_HASH(amp, an_idx); 3422 sobj->sync_mutex = mtx = &anon_array_lock[hash].pad_mutex; 3423 sobj->sync_cv = cv = &anon_array_cv[hash]; 3424 mutex_enter(mtx); 3425 ap_slot = anon_get_slot(amp->ahp, an_idx); 3426 while (ANON_ISBUSY(ap_slot)) 3427 cv_wait(cv, mtx); 3428 ANON_SETBUSY(ap_slot); 3429 sobj->sync_data = ap_slot; 3430 mutex_exit(mtx); 3431 } 3432 3433 int 3434 anon_array_try_enter(struct anon_map *amp, ulong_t an_idx, 3435 anon_sync_obj_t *sobj) 3436 { 3437 ulong_t *ap_slot; 3438 kmutex_t *mtx; 3439 int hash; 3440 3441 /* 3442 * Try to lock a range of anon slots. 3443 * Use szc to determine anon slot(s) to appear atomic. 3444 * If szc = 0, then lock the anon slot and mark it busy. 3445 * If szc > 0, then lock the range of slots by getting the 3446 * anon_array_lock for the first anon slot, and mark only the 3447 * first anon slot busy to represent whole range being busy. 3448 * Fail if the mutex or the anon_array are busy. 3449 */ 3450 3451 ASSERT(RW_READ_HELD(&->a_rwlock)); 3452 an_idx = P2ALIGN(an_idx, page_get_pagecnt(amp->a_szc)); 3453 hash = ANON_ARRAY_HASH(amp, an_idx); 3454 sobj->sync_mutex = mtx = &anon_array_lock[hash].pad_mutex; 3455 sobj->sync_cv = &anon_array_cv[hash]; 3456 if (!mutex_tryenter(mtx)) { 3457 return (EWOULDBLOCK); 3458 } 3459 ap_slot = anon_get_slot(amp->ahp, an_idx); 3460 if (ANON_ISBUSY(ap_slot)) { 3461 mutex_exit(mtx); 3462 return (EWOULDBLOCK); 3463 } 3464 ANON_SETBUSY(ap_slot); 3465 sobj->sync_data = ap_slot; 3466 mutex_exit(mtx); 3467 return (0); 3468 } 3469 3470 void 3471 anon_array_exit(anon_sync_obj_t *sobj) 3472 { 3473 mutex_enter(sobj->sync_mutex); 3474 ASSERT(ANON_ISBUSY(sobj->sync_data)); 3475 ANON_CLRBUSY(sobj->sync_data); 3476 if (CV_HAS_WAITERS(sobj->sync_cv)) 3477 cv_broadcast(sobj->sync_cv); 3478 mutex_exit(sobj->sync_mutex); 3479 } 3480