1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #pragma ident "%Z%%M% %I% %E% SMI" 40 41 /* 42 * VM - anonymous pages. 43 * 44 * This layer sits immediately above the vm_swap layer. It manages 45 * physical pages that have no permanent identity in the file system 46 * name space, using the services of the vm_swap layer to allocate 47 * backing storage for these pages. Since these pages have no external 48 * identity, they are discarded when the last reference is removed. 49 * 50 * An important function of this layer is to manage low-level sharing 51 * of pages that are logically distinct but that happen to be 52 * physically identical (e.g., the corresponding pages of the processes 53 * resulting from a fork before one process or the other changes their 54 * contents). This pseudo-sharing is present only as an optimization 55 * and is not to be confused with true sharing in which multiple 56 * address spaces deliberately contain references to the same object; 57 * such sharing is managed at a higher level. 58 * 59 * The key data structure here is the anon struct, which contains a 60 * reference count for its associated physical page and a hint about 61 * the identity of that page. Anon structs typically live in arrays, 62 * with an instance's position in its array determining where the 63 * corresponding backing storage is allocated; however, the swap_xlate() 64 * routine abstracts away this representation information so that the 65 * rest of the anon layer need not know it. (See the swap layer for 66 * more details on anon struct layout.) 67 * 68 * In the future versions of the system, the association between an 69 * anon struct and its position on backing store will change so that 70 * we don't require backing store all anonymous pages in the system. 71 * This is important for consideration for large memory systems. 72 * We can also use this technique to delay binding physical locations 73 * to anonymous pages until pageout/swapout time where we can make 74 * smarter allocation decisions to improve anonymous klustering. 75 * 76 * Many of the routines defined here take a (struct anon **) argument, 77 * which allows the code at this level to manage anon pages directly, 78 * so that callers can regard anon structs as opaque objects and not be 79 * concerned with assigning or inspecting their contents. 80 * 81 * Clients of this layer refer to anon pages indirectly. That is, they 82 * maintain arrays of pointers to anon structs rather than maintaining 83 * anon structs themselves. The (struct anon **) arguments mentioned 84 * above are pointers to entries in these arrays. It is these arrays 85 * that capture the mapping between offsets within a given segment and 86 * the corresponding anonymous backing storage address. 87 */ 88 89 #ifdef DEBUG 90 #define ANON_DEBUG 91 #endif 92 93 #include <sys/types.h> 94 #include <sys/t_lock.h> 95 #include <sys/param.h> 96 #include <sys/systm.h> 97 #include <sys/mman.h> 98 #include <sys/cred.h> 99 #include <sys/thread.h> 100 #include <sys/vnode.h> 101 #include <sys/cpuvar.h> 102 #include <sys/swap.h> 103 #include <sys/cmn_err.h> 104 #include <sys/vtrace.h> 105 #include <sys/kmem.h> 106 #include <sys/sysmacros.h> 107 #include <sys/bitmap.h> 108 #include <sys/vmsystm.h> 109 #include <sys/debug.h> 110 #include <sys/fs/swapnode.h> 111 #include <sys/tnf_probe.h> 112 #include <sys/lgrp.h> 113 #include <sys/policy.h> 114 #include <sys/condvar_impl.h> 115 #include <sys/mutex_impl.h> 116 #include <sys/rctl.h> 117 118 #include <vm/as.h> 119 #include <vm/hat.h> 120 #include <vm/anon.h> 121 #include <vm/page.h> 122 #include <vm/vpage.h> 123 #include <vm/seg.h> 124 #include <vm/rm.h> 125 126 #include <fs/fs_subr.h> 127 128 struct vnode *anon_vp; 129 130 int anon_debug; 131 132 kmutex_t anoninfo_lock; 133 struct k_anoninfo k_anoninfo; 134 ani_free_t ani_free_pool[ANI_MAX_POOL]; 135 pad_mutex_t anon_array_lock[ANON_LOCKSIZE]; 136 kcondvar_t anon_array_cv[ANON_LOCKSIZE]; 137 138 /* 139 * Global hash table for (vp, off) -> anon slot 140 */ 141 extern int swap_maxcontig; 142 size_t anon_hash_size; 143 struct anon **anon_hash; 144 145 static struct kmem_cache *anon_cache; 146 static struct kmem_cache *anonmap_cache; 147 148 #ifdef VM_STATS 149 static struct anonvmstats_str { 150 ulong_t getpages[30]; 151 ulong_t privatepages[10]; 152 ulong_t demotepages[9]; 153 ulong_t decrefpages[9]; 154 ulong_t dupfillholes[4]; 155 ulong_t freepages[1]; 156 } anonvmstats; 157 #endif /* VM_STATS */ 158 159 160 /*ARGSUSED*/ 161 static int 162 anonmap_cache_constructor(void *buf, void *cdrarg, int kmflags) 163 { 164 struct anon_map *amp = buf; 165 166 rw_init(&->a_rwlock, NULL, RW_DEFAULT, NULL); 167 return (0); 168 } 169 170 /*ARGSUSED1*/ 171 static void 172 anonmap_cache_destructor(void *buf, void *cdrarg) 173 { 174 struct anon_map *amp = buf; 175 176 rw_destroy(&->a_rwlock); 177 } 178 179 kmutex_t anonhash_lock[AH_LOCK_SIZE]; 180 kmutex_t anonpages_hash_lock[AH_LOCK_SIZE]; 181 182 void 183 anon_init(void) 184 { 185 int i; 186 187 anon_hash_size = 1L << highbit(physmem / ANON_HASHAVELEN); 188 189 for (i = 0; i < AH_LOCK_SIZE; i++) { 190 mutex_init(&anonhash_lock[i], NULL, MUTEX_DEFAULT, NULL); 191 mutex_init(&anonpages_hash_lock[i], NULL, MUTEX_DEFAULT, NULL); 192 } 193 194 for (i = 0; i < ANON_LOCKSIZE; i++) { 195 mutex_init(&anon_array_lock[i].pad_mutex, NULL, 196 MUTEX_DEFAULT, NULL); 197 cv_init(&anon_array_cv[i], NULL, CV_DEFAULT, NULL); 198 } 199 200 anon_hash = (struct anon **) 201 kmem_zalloc(sizeof (struct anon *) * anon_hash_size, KM_SLEEP); 202 anon_cache = kmem_cache_create("anon_cache", sizeof (struct anon), 203 AN_CACHE_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 204 anonmap_cache = kmem_cache_create("anonmap_cache", 205 sizeof (struct anon_map), 0, 206 anonmap_cache_constructor, anonmap_cache_destructor, NULL, 207 NULL, NULL, 0); 208 swap_maxcontig = (1024 * 1024) >> PAGESHIFT; /* 1MB of pages */ 209 210 anon_vp = vn_alloc(KM_SLEEP); 211 vn_setops(anon_vp, swap_vnodeops); 212 anon_vp->v_type = VREG; 213 anon_vp->v_flag |= (VISSWAP|VISSWAPFS); 214 } 215 216 /* 217 * Global anon slot hash table manipulation. 218 */ 219 220 static void 221 anon_addhash(struct anon *ap) 222 { 223 int index; 224 225 ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)])); 226 index = ANON_HASH(ap->an_vp, ap->an_off); 227 ap->an_hash = anon_hash[index]; 228 anon_hash[index] = ap; 229 } 230 231 static void 232 anon_rmhash(struct anon *ap) 233 { 234 struct anon **app; 235 236 ASSERT(MUTEX_HELD(&anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)])); 237 238 for (app = &anon_hash[ANON_HASH(ap->an_vp, ap->an_off)]; 239 *app; app = &((*app)->an_hash)) { 240 if (*app == ap) { 241 *app = ap->an_hash; 242 break; 243 } 244 } 245 } 246 247 /* 248 * The anon array interfaces. Functions allocating, 249 * freeing array of pointers, and returning/setting 250 * entries in the array of pointers for a given offset. 251 * 252 * Create the list of pointers 253 */ 254 struct anon_hdr * 255 anon_create(pgcnt_t npages, int flags) 256 { 257 struct anon_hdr *ahp; 258 ulong_t nchunks; 259 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 260 261 if ((ahp = kmem_zalloc(sizeof (struct anon_hdr), kmemflags)) == NULL) { 262 return (NULL); 263 } 264 265 mutex_init(&ahp->serial_lock, NULL, MUTEX_DEFAULT, NULL); 266 /* 267 * Single level case. 268 */ 269 ahp->size = npages; 270 if (npages <= ANON_CHUNK_SIZE || (flags & ANON_ALLOC_FORCE)) { 271 272 if (flags & ANON_ALLOC_FORCE) 273 ahp->flags |= ANON_ALLOC_FORCE; 274 275 ahp->array_chunk = kmem_zalloc( 276 ahp->size * sizeof (struct anon *), kmemflags); 277 278 if (ahp->array_chunk == NULL) { 279 kmem_free(ahp, sizeof (struct anon_hdr)); 280 return (NULL); 281 } 282 } else { 283 /* 284 * 2 Level case. 285 * anon hdr size needs to be rounded off to be a multiple 286 * of ANON_CHUNK_SIZE. This is important as various anon 287 * related functions depend on this. 288 * NOTE - 289 * anon_grow() makes anon hdr size a multiple of 290 * ANON_CHUNK_SIZE. 291 * amp size is <= anon hdr size. 292 * anon_index + seg_pgs <= anon hdr size. 293 */ 294 ahp->size = P2ROUNDUP(npages, ANON_CHUNK_SIZE); 295 nchunks = ahp->size >> ANON_CHUNK_SHIFT; 296 297 ahp->array_chunk = kmem_zalloc(nchunks * sizeof (ulong_t *), 298 kmemflags); 299 300 if (ahp->array_chunk == NULL) { 301 kmem_free(ahp, sizeof (struct anon_hdr)); 302 return (NULL); 303 } 304 } 305 return (ahp); 306 } 307 308 /* 309 * Free the array of pointers 310 */ 311 void 312 anon_release(struct anon_hdr *ahp, pgcnt_t npages) 313 { 314 ulong_t i; 315 void **ppp; 316 ulong_t nchunks; 317 318 ASSERT(npages <= ahp->size); 319 320 /* 321 * Single level case. 322 */ 323 if (npages <= ANON_CHUNK_SIZE || (ahp->flags & ANON_ALLOC_FORCE)) { 324 kmem_free(ahp->array_chunk, ahp->size * sizeof (struct anon *)); 325 } else { 326 /* 327 * 2 level case. 328 */ 329 nchunks = ahp->size >> ANON_CHUNK_SHIFT; 330 for (i = 0; i < nchunks; i++) { 331 ppp = &ahp->array_chunk[i]; 332 if (*ppp != NULL) 333 kmem_free(*ppp, PAGESIZE); 334 } 335 kmem_free(ahp->array_chunk, nchunks * sizeof (ulong_t *)); 336 } 337 mutex_destroy(&ahp->serial_lock); 338 kmem_free(ahp, sizeof (struct anon_hdr)); 339 } 340 341 /* 342 * Return the pointer from the list for a 343 * specified anon index. 344 */ 345 struct anon * 346 anon_get_ptr(struct anon_hdr *ahp, ulong_t an_idx) 347 { 348 struct anon **app; 349 350 ASSERT(an_idx < ahp->size); 351 352 /* 353 * Single level case. 354 */ 355 if ((ahp->size <= ANON_CHUNK_SIZE) || (ahp->flags & ANON_ALLOC_FORCE)) { 356 return ((struct anon *) 357 ((uintptr_t)ahp->array_chunk[an_idx] & ANON_PTRMASK)); 358 } else { 359 360 /* 361 * 2 level case. 362 */ 363 app = ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 364 if (app) { 365 return ((struct anon *) 366 ((uintptr_t)app[an_idx & ANON_CHUNK_OFF] & 367 ANON_PTRMASK)); 368 } else { 369 return (NULL); 370 } 371 } 372 } 373 374 /* 375 * Return the anon pointer for the first valid entry in the anon list, 376 * starting from the given index. 377 */ 378 struct anon * 379 anon_get_next_ptr(struct anon_hdr *ahp, ulong_t *index) 380 { 381 struct anon *ap; 382 struct anon **app; 383 ulong_t chunkoff; 384 ulong_t i; 385 ulong_t j; 386 pgcnt_t size; 387 388 i = *index; 389 size = ahp->size; 390 391 ASSERT(i < size); 392 393 if ((size <= ANON_CHUNK_SIZE) || (ahp->flags & ANON_ALLOC_FORCE)) { 394 /* 395 * 1 level case 396 */ 397 while (i < size) { 398 ap = (struct anon *) 399 ((uintptr_t)ahp->array_chunk[i] & ANON_PTRMASK); 400 if (ap) { 401 *index = i; 402 return (ap); 403 } 404 i++; 405 } 406 } else { 407 /* 408 * 2 level case 409 */ 410 chunkoff = i & ANON_CHUNK_OFF; 411 while (i < size) { 412 app = ahp->array_chunk[i >> ANON_CHUNK_SHIFT]; 413 if (app) 414 for (j = chunkoff; j < ANON_CHUNK_SIZE; j++) { 415 ap = (struct anon *) 416 ((uintptr_t)app[j] & 417 ANON_PTRMASK); 418 if (ap) { 419 *index = i + (j - chunkoff); 420 return (ap); 421 } 422 } 423 chunkoff = 0; 424 i = (i + ANON_CHUNK_SIZE) & ~ANON_CHUNK_OFF; 425 } 426 } 427 *index = size; 428 return (NULL); 429 } 430 431 /* 432 * Set list entry with a given pointer for a specified offset 433 */ 434 int 435 anon_set_ptr(struct anon_hdr *ahp, ulong_t an_idx, struct anon *ap, int flags) 436 { 437 void **ppp; 438 struct anon **app; 439 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 440 uintptr_t *ap_addr; 441 442 ASSERT(an_idx < ahp->size); 443 444 /* 445 * Single level case. 446 */ 447 if (ahp->size <= ANON_CHUNK_SIZE || (ahp->flags & ANON_ALLOC_FORCE)) { 448 ap_addr = (uintptr_t *)&ahp->array_chunk[an_idx]; 449 } else { 450 451 /* 452 * 2 level case. 453 */ 454 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 455 456 ASSERT(ppp != NULL); 457 if (*ppp == NULL) { 458 mutex_enter(&ahp->serial_lock); 459 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 460 if (*ppp == NULL) { 461 *ppp = kmem_zalloc(PAGESIZE, kmemflags); 462 if (*ppp == NULL) { 463 mutex_exit(&ahp->serial_lock); 464 return (ENOMEM); 465 } 466 } 467 mutex_exit(&ahp->serial_lock); 468 } 469 app = *ppp; 470 ap_addr = (uintptr_t *)&app[an_idx & ANON_CHUNK_OFF]; 471 } 472 *ap_addr = (*ap_addr & ~ANON_PTRMASK) | (uintptr_t)ap; 473 return (0); 474 } 475 476 /* 477 * Copy anon array into a given new anon array 478 */ 479 int 480 anon_copy_ptr(struct anon_hdr *sahp, ulong_t s_idx, 481 struct anon_hdr *dahp, ulong_t d_idx, 482 pgcnt_t npages, int flags) 483 { 484 void **sapp, **dapp; 485 void *ap; 486 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 487 488 ASSERT((s_idx < sahp->size) && (d_idx < dahp->size)); 489 ASSERT((npages <= sahp->size) && (npages <= dahp->size)); 490 491 /* 492 * Both arrays are 1 level. 493 */ 494 if (((sahp->size <= ANON_CHUNK_SIZE) && 495 (dahp->size <= ANON_CHUNK_SIZE)) || 496 ((sahp->flags & ANON_ALLOC_FORCE) && 497 (dahp->flags & ANON_ALLOC_FORCE))) { 498 499 bcopy(&sahp->array_chunk[s_idx], &dahp->array_chunk[d_idx], 500 npages * sizeof (struct anon *)); 501 return (0); 502 } 503 504 /* 505 * Both arrays are 2 levels. 506 */ 507 if (sahp->size > ANON_CHUNK_SIZE && 508 dahp->size > ANON_CHUNK_SIZE && 509 ((sahp->flags & ANON_ALLOC_FORCE) == 0) && 510 ((dahp->flags & ANON_ALLOC_FORCE) == 0)) { 511 512 ulong_t sapidx, dapidx; 513 ulong_t *sap, *dap; 514 ulong_t chknp; 515 516 while (npages != 0) { 517 518 sapidx = s_idx & ANON_CHUNK_OFF; 519 dapidx = d_idx & ANON_CHUNK_OFF; 520 chknp = ANON_CHUNK_SIZE - MAX(sapidx, dapidx); 521 if (chknp > npages) 522 chknp = npages; 523 524 sapp = &sahp->array_chunk[s_idx >> ANON_CHUNK_SHIFT]; 525 if ((sap = *sapp) != NULL) { 526 dapp = &dahp->array_chunk[d_idx 527 >> ANON_CHUNK_SHIFT]; 528 if ((dap = *dapp) == NULL) { 529 *dapp = kmem_zalloc(PAGESIZE, 530 kmemflags); 531 if ((dap = *dapp) == NULL) 532 return (ENOMEM); 533 } 534 bcopy((sap + sapidx), (dap + dapidx), 535 chknp << ANON_PTRSHIFT); 536 } 537 s_idx += chknp; 538 d_idx += chknp; 539 npages -= chknp; 540 } 541 return (0); 542 } 543 544 /* 545 * At least one of the arrays is 2 level. 546 */ 547 while (npages--) { 548 if ((ap = anon_get_ptr(sahp, s_idx)) != NULL) { 549 ASSERT(!ANON_ISBUSY(anon_get_slot(sahp, s_idx))); 550 if (anon_set_ptr(dahp, d_idx, ap, flags) == ENOMEM) 551 return (ENOMEM); 552 } 553 s_idx++; 554 d_idx++; 555 } 556 return (0); 557 } 558 559 560 /* 561 * ANON_INITBUF is a convenience macro for anon_grow() below. It 562 * takes a buffer dst, which is at least as large as buffer src. It 563 * does a bcopy from src into dst, and then bzeros the extra bytes 564 * of dst. If tail is set, the data in src is tail aligned within 565 * dst instead of head aligned. 566 */ 567 568 #define ANON_INITBUF(src, srclen, dst, dstsize, tail) \ 569 if (tail) { \ 570 bzero((dst), (dstsize) - (srclen)); \ 571 bcopy((src), (char *)(dst) + (dstsize) - (srclen), (srclen)); \ 572 } else { \ 573 bcopy((src), (dst), (srclen)); \ 574 bzero((char *)(dst) + (srclen), (dstsize) - (srclen)); \ 575 } 576 577 #define ANON_1_LEVEL_INC (ANON_CHUNK_SIZE / 8) 578 #define ANON_2_LEVEL_INC (ANON_1_LEVEL_INC * ANON_CHUNK_SIZE) 579 580 /* 581 * anon_grow() is used to efficiently extend an existing anon array. 582 * startidx_p points to the index into the anon array of the first page 583 * that is in use. oldseg_pgs is the number of pages in use, starting at 584 * *startidx_p. newpages is the number of additional pages desired. 585 * 586 * If startidx_p == NULL, startidx is taken to be 0 and cannot be changed. 587 * 588 * The growth is done by creating a new top level of the anon array, 589 * and (if the array is 2-level) reusing the existing second level arrays. 590 * 591 * flags can be used to specify ANON_NOSLEEP and ANON_GROWDOWN. 592 * 593 * Returns the new number of pages in the anon array. 594 */ 595 pgcnt_t 596 anon_grow(struct anon_hdr *ahp, ulong_t *startidx_p, pgcnt_t oldseg_pgs, 597 pgcnt_t newseg_pgs, int flags) 598 { 599 ulong_t startidx = startidx_p ? *startidx_p : 0; 600 pgcnt_t oldamp_pgs = ahp->size, newamp_pgs; 601 pgcnt_t oelems, nelems, totpages; 602 void **level1; 603 int kmemflags = (flags & ANON_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 604 int growdown = (flags & ANON_GROWDOWN); 605 size_t newarrsz, oldarrsz; 606 void *level2; 607 608 ASSERT(!(startidx_p == NULL && growdown)); 609 ASSERT(startidx + oldseg_pgs <= ahp->size); 610 611 /* 612 * Determine the total number of pages needed in the new 613 * anon array. If growing down, totpages is all pages from 614 * startidx through the end of the array, plus <newseg_pgs> 615 * pages. If growing up, keep all pages from page 0 through 616 * the last page currently in use, plus <newseg_pgs> pages. 617 */ 618 if (growdown) 619 totpages = oldamp_pgs - startidx + newseg_pgs; 620 else 621 totpages = startidx + oldseg_pgs + newseg_pgs; 622 623 /* If the array is already large enough, just return. */ 624 625 if (oldamp_pgs >= totpages) { 626 if (growdown) 627 *startidx_p = oldamp_pgs - totpages; 628 return (oldamp_pgs); 629 } 630 631 /* 632 * oldamp_pgs/newamp_pgs are the total numbers of pages represented 633 * by the corresponding arrays. 634 * oelems/nelems are the number of pointers in the top level arrays 635 * which may be either level 1 or level 2. 636 * Will the new anon array be one level or two levels? 637 */ 638 if (totpages <= ANON_CHUNK_SIZE || (ahp->flags & ANON_ALLOC_FORCE)) { 639 newamp_pgs = P2ROUNDUP(totpages, ANON_1_LEVEL_INC); 640 oelems = oldamp_pgs; 641 nelems = newamp_pgs; 642 } else { 643 newamp_pgs = P2ROUNDUP(totpages, ANON_2_LEVEL_INC); 644 oelems = (oldamp_pgs + ANON_CHUNK_OFF) >> ANON_CHUNK_SHIFT; 645 nelems = newamp_pgs >> ANON_CHUNK_SHIFT; 646 } 647 648 newarrsz = nelems * sizeof (void *); 649 level1 = kmem_alloc(newarrsz, kmemflags); 650 if (level1 == NULL) 651 return (0); 652 653 /* Are we converting from a one level to a two level anon array? */ 654 655 if (newamp_pgs > ANON_CHUNK_SIZE && oldamp_pgs <= ANON_CHUNK_SIZE && 656 !(ahp->flags & ANON_ALLOC_FORCE)) { 657 658 /* 659 * Yes, we're converting to a two level. Reuse old level 1 660 * as new level 2 if it is exactly PAGESIZE. Otherwise 661 * alloc a new level 2 and copy the old level 1 data into it. 662 */ 663 if (oldamp_pgs == ANON_CHUNK_SIZE) { 664 level2 = (void *)ahp->array_chunk; 665 } else { 666 level2 = kmem_alloc(PAGESIZE, kmemflags); 667 if (level2 == NULL) { 668 kmem_free(level1, newarrsz); 669 return (0); 670 } 671 oldarrsz = oldamp_pgs * sizeof (void *); 672 673 ANON_INITBUF(ahp->array_chunk, oldarrsz, 674 level2, PAGESIZE, growdown); 675 kmem_free(ahp->array_chunk, oldarrsz); 676 } 677 bzero(level1, newarrsz); 678 if (growdown) 679 level1[nelems - 1] = level2; 680 else 681 level1[0] = level2; 682 } else { 683 oldarrsz = oelems * sizeof (void *); 684 685 ANON_INITBUF(ahp->array_chunk, oldarrsz, 686 level1, newarrsz, growdown); 687 kmem_free(ahp->array_chunk, oldarrsz); 688 } 689 690 ahp->array_chunk = level1; 691 ahp->size = newamp_pgs; 692 if (growdown) 693 *startidx_p = newamp_pgs - totpages; 694 695 return (newamp_pgs); 696 } 697 698 699 /* 700 * Called from clock handler to sync ani_free value. 701 */ 702 703 void 704 set_anoninfo(void) 705 { 706 int ix; 707 pgcnt_t total = 0; 708 709 for (ix = 0; ix < ANI_MAX_POOL; ix++) { 710 total += ani_free_pool[ix].ani_count; 711 } 712 k_anoninfo.ani_free = total; 713 } 714 715 /* 716 * Reserve anon space. 717 * 718 * It's no longer simply a matter of incrementing ani_resv to 719 * reserve swap space, we need to check memory-based as well 720 * as disk-backed (physical) swap. The following algorithm 721 * is used: 722 * Check the space on physical swap 723 * i.e. amount needed < ani_max - ani_phys_resv 724 * If we are swapping on swapfs check 725 * amount needed < (availrmem - swapfs_minfree) 726 * Since the algorithm to check for the quantity of swap space is 727 * almost the same as that for reserving it, we'll just use anon_resvmem 728 * with a flag to decrement availrmem. 729 * 730 * Return non-zero on success. 731 */ 732 int 733 anon_resvmem(size_t size, boolean_t takemem, zone_t *zone) 734 { 735 pgcnt_t npages = btopr(size); 736 pgcnt_t mswap_pages = 0; 737 pgcnt_t pswap_pages = 0; 738 proc_t *p = curproc; 739 740 if (zone != NULL && takemem) { 741 /* test zone.max-swap resource control */ 742 mutex_enter(&p->p_lock); 743 if (rctl_incr_swap(p, zone, ptob(npages)) != 0) { 744 mutex_exit(&p->p_lock); 745 return (0); 746 } 747 mutex_exit(&p->p_lock); 748 } 749 mutex_enter(&anoninfo_lock); 750 751 /* 752 * pswap_pages is the number of pages we can take from 753 * physical (i.e. disk-backed) swap. 754 */ 755 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 756 pswap_pages = k_anoninfo.ani_max - k_anoninfo.ani_phys_resv; 757 758 ANON_PRINT(A_RESV, 759 ("anon_resvmem: npages %lu takemem %u pswap %lu caller %p\n", 760 npages, takemem, pswap_pages, (void *)caller())); 761 762 if (npages <= pswap_pages) { 763 /* 764 * we have enough space on a physical swap 765 */ 766 if (takemem) 767 k_anoninfo.ani_phys_resv += npages; 768 mutex_exit(&anoninfo_lock); 769 return (1); 770 } else if (pswap_pages != 0) { 771 /* 772 * we have some space on a physical swap 773 */ 774 if (takemem) { 775 /* 776 * use up remainder of phys swap 777 */ 778 k_anoninfo.ani_phys_resv += pswap_pages; 779 ASSERT(k_anoninfo.ani_phys_resv == k_anoninfo.ani_max); 780 } 781 } 782 /* 783 * since (npages > pswap_pages) we need mem swap 784 * mswap_pages is the number of pages needed from availrmem 785 */ 786 ASSERT(npages > pswap_pages); 787 mswap_pages = npages - pswap_pages; 788 789 ANON_PRINT(A_RESV, ("anon_resvmem: need %ld pages from memory\n", 790 mswap_pages)); 791 792 /* 793 * priv processes can reserve memory as swap as long as availrmem 794 * remains greater than swapfs_minfree; in the case of non-priv 795 * processes, memory can be reserved as swap only if availrmem 796 * doesn't fall below (swapfs_minfree + swapfs_reserve). Thus, 797 * swapfs_reserve amount of memswap is not available to non-priv 798 * processes. This protects daemons such as automounter dying 799 * as a result of application processes eating away almost entire 800 * membased swap. This safeguard becomes useless if apps are run 801 * with root access. 802 * 803 * swapfs_reserve is minimum of 4Mb or 1/16 of physmem. 804 * 805 */ 806 mutex_exit(&anoninfo_lock); 807 (void) page_reclaim_mem(mswap_pages, 808 swapfs_minfree + swapfs_reserve, 0); 809 mutex_enter(&anoninfo_lock); 810 811 mutex_enter(&freemem_lock); 812 if (availrmem > (swapfs_minfree + swapfs_reserve + mswap_pages) || 813 (availrmem > (swapfs_minfree + mswap_pages) && 814 secpolicy_resource(CRED()) == 0)) { 815 816 if (takemem) { 817 /* 818 * Take the memory from the rest of the system. 819 */ 820 availrmem -= mswap_pages; 821 mutex_exit(&freemem_lock); 822 k_anoninfo.ani_mem_resv += mswap_pages; 823 ANI_ADD(mswap_pages); 824 ANON_PRINT((A_RESV | A_MRESV), 825 ("anon_resvmem: took %ld pages of availrmem\n", 826 mswap_pages)); 827 } else { 828 mutex_exit(&freemem_lock); 829 } 830 831 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 832 mutex_exit(&anoninfo_lock); 833 return (1); 834 835 } else { 836 /* 837 * Fail if not enough memory 838 */ 839 840 if (takemem) { 841 k_anoninfo.ani_phys_resv -= pswap_pages; 842 } 843 844 mutex_exit(&freemem_lock); 845 mutex_exit(&anoninfo_lock); 846 ANON_PRINT(A_RESV, 847 ("anon_resvmem: not enough space from swapfs\n")); 848 if (zone != NULL && takemem) 849 rctl_decr_swap(zone, ptob(npages)); 850 return (0); 851 } 852 } 853 854 /* 855 * Give back an anon reservation. 856 */ 857 void 858 anon_unresvmem(size_t size, zone_t *zone) 859 { 860 pgcnt_t npages = btopr(size); 861 spgcnt_t mem_free_pages = 0; 862 pgcnt_t phys_free_slots; 863 #ifdef ANON_DEBUG 864 pgcnt_t mem_resv; 865 #endif 866 if (zone != NULL) 867 rctl_decr_swap(zone, ptob(npages)); 868 869 mutex_enter(&anoninfo_lock); 870 871 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 872 /* 873 * If some of this reservation belonged to swapfs 874 * give it back to availrmem. 875 * ani_mem_resv is the amount of availrmem swapfs has reserved. 876 * but some of that memory could be locked by segspt so we can only 877 * return non locked ani_mem_resv back to availrmem 878 */ 879 if (k_anoninfo.ani_mem_resv > k_anoninfo.ani_locked_swap) { 880 ANON_PRINT((A_RESV | A_MRESV), 881 ("anon_unresv: growing availrmem by %ld pages\n", 882 MIN(k_anoninfo.ani_mem_resv, npages))); 883 884 mem_free_pages = MIN((spgcnt_t)(k_anoninfo.ani_mem_resv - 885 k_anoninfo.ani_locked_swap), npages); 886 mutex_enter(&freemem_lock); 887 availrmem += mem_free_pages; 888 mutex_exit(&freemem_lock); 889 k_anoninfo.ani_mem_resv -= mem_free_pages; 890 891 ANI_ADD(-mem_free_pages); 892 } 893 /* 894 * The remainder of the pages is returned to phys swap 895 */ 896 ASSERT(npages >= mem_free_pages); 897 phys_free_slots = npages - mem_free_pages; 898 899 if (phys_free_slots) { 900 k_anoninfo.ani_phys_resv -= phys_free_slots; 901 } 902 903 #ifdef ANON_DEBUG 904 mem_resv = k_anoninfo.ani_mem_resv; 905 #endif 906 907 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 908 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 909 910 mutex_exit(&anoninfo_lock); 911 912 ANON_PRINT(A_RESV, ("anon_unresv: %lu, tot %lu, caller %p\n", 913 npages, mem_resv, (void *)caller())); 914 } 915 916 /* 917 * Allocate an anon slot and return it with the lock held. 918 */ 919 struct anon * 920 anon_alloc(struct vnode *vp, anoff_t off) 921 { 922 struct anon *ap; 923 kmutex_t *ahm; 924 925 ap = kmem_cache_alloc(anon_cache, KM_SLEEP); 926 if (vp == NULL) { 927 swap_alloc(ap); 928 } else { 929 ap->an_vp = vp; 930 ap->an_off = off; 931 } 932 ap->an_refcnt = 1; 933 ap->an_pvp = NULL; 934 ap->an_poff = 0; 935 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 936 mutex_enter(ahm); 937 anon_addhash(ap); 938 mutex_exit(ahm); 939 ANI_ADD(-1); 940 ANON_PRINT(A_ANON, ("anon_alloc: returning ap %p, vp %p\n", 941 (void *)ap, (ap ? (void *)ap->an_vp : NULL))); 942 return (ap); 943 } 944 945 /* 946 * Decrement the reference count of an anon page. 947 * If reference count goes to zero, free it and 948 * its associated page (if any). 949 */ 950 void 951 anon_decref(struct anon *ap) 952 { 953 page_t *pp; 954 struct vnode *vp; 955 anoff_t off; 956 kmutex_t *ahm; 957 958 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 959 mutex_enter(ahm); 960 ASSERT(ap->an_refcnt != 0); 961 if (ap->an_refcnt == 0) 962 panic("anon_decref: slot count 0"); 963 if (--ap->an_refcnt == 0) { 964 swap_xlate(ap, &vp, &off); 965 mutex_exit(ahm); 966 967 /* 968 * If there is a page for this anon slot we will need to 969 * call VN_DISPOSE to get rid of the vp association and 970 * put the page back on the free list as really free. 971 * Acquire the "exclusive" lock to ensure that any 972 * pending i/o always completes before the swap slot 973 * is freed. 974 */ 975 pp = page_lookup(vp, (u_offset_t)off, SE_EXCL); 976 977 /* 978 * If there was a page, we've synchronized on it (getting 979 * the exclusive lock is as good as gettting the iolock) 980 * so now we can free the physical backing store. Also, this 981 * is where we would free the name of the anonymous page 982 * (swap_free(ap)), a no-op in the current implementation. 983 */ 984 mutex_enter(ahm); 985 ASSERT(ap->an_refcnt == 0); 986 anon_rmhash(ap); 987 if (ap->an_pvp) 988 swap_phys_free(ap->an_pvp, ap->an_poff, PAGESIZE); 989 mutex_exit(ahm); 990 991 if (pp != NULL) { 992 /*LINTED: constant in conditional context */ 993 VN_DISPOSE(pp, B_INVAL, 0, kcred); 994 } 995 ANON_PRINT(A_ANON, ("anon_decref: free ap %p, vp %p\n", 996 (void *)ap, (void *)ap->an_vp)); 997 kmem_cache_free(anon_cache, ap); 998 999 ANI_ADD(1); 1000 } else { 1001 mutex_exit(ahm); 1002 } 1003 } 1004 1005 static int 1006 anon_share(struct anon_hdr *ahp, ulong_t anon_index, pgcnt_t nslots) 1007 { 1008 struct anon *ap; 1009 1010 while (nslots-- > 0) { 1011 if ((ap = anon_get_ptr(ahp, anon_index)) != NULL && 1012 ap->an_refcnt > 1) 1013 return (1); 1014 anon_index++; 1015 } 1016 1017 return (0); 1018 } 1019 1020 static void 1021 anon_decref_pages( 1022 struct anon_hdr *ahp, 1023 ulong_t an_idx, 1024 uint_t szc) 1025 { 1026 struct anon *ap = anon_get_ptr(ahp, an_idx); 1027 kmutex_t *ahmpages = NULL; 1028 page_t *pp; 1029 pgcnt_t pgcnt = page_get_pagecnt(szc); 1030 pgcnt_t i; 1031 struct vnode *vp; 1032 anoff_t off; 1033 kmutex_t *ahm; 1034 #ifdef DEBUG 1035 int refcnt = 1; 1036 #endif 1037 1038 ASSERT(szc != 0); 1039 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1040 ASSERT(IS_P2ALIGNED(an_idx, pgcnt)); 1041 ASSERT(an_idx < ahp->size); 1042 1043 if (ahp->size - an_idx < pgcnt) { 1044 /* 1045 * In case of shared mappings total anon map size may not be 1046 * the largest page size aligned. 1047 */ 1048 pgcnt = ahp->size - an_idx; 1049 } 1050 1051 VM_STAT_ADD(anonvmstats.decrefpages[0]); 1052 1053 if (ap != NULL) { 1054 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1055 mutex_enter(ahmpages); 1056 ASSERT((refcnt = ap->an_refcnt) != 0); 1057 VM_STAT_ADD(anonvmstats.decrefpages[1]); 1058 if (ap->an_refcnt == 1) { 1059 VM_STAT_ADD(anonvmstats.decrefpages[2]); 1060 ASSERT(!anon_share(ahp, an_idx, pgcnt)); 1061 mutex_exit(ahmpages); 1062 ahmpages = NULL; 1063 } 1064 } 1065 1066 i = 0; 1067 while (i < pgcnt) { 1068 if ((ap = anon_get_ptr(ahp, an_idx + i)) == NULL) { 1069 ASSERT(refcnt == 1 && ahmpages == NULL); 1070 i++; 1071 continue; 1072 } 1073 ASSERT(ap->an_refcnt == refcnt); 1074 ASSERT(ahmpages != NULL || ap->an_refcnt == 1); 1075 ASSERT(ahmpages == NULL || ap->an_refcnt > 1); 1076 1077 if (ahmpages == NULL) { 1078 swap_xlate(ap, &vp, &off); 1079 pp = page_lookup(vp, (u_offset_t)off, SE_EXCL); 1080 if (pp == NULL || pp->p_szc == 0) { 1081 VM_STAT_ADD(anonvmstats.decrefpages[3]); 1082 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, 1083 ap->an_off)]; 1084 (void) anon_set_ptr(ahp, an_idx + i, NULL, 1085 ANON_SLEEP); 1086 mutex_enter(ahm); 1087 ap->an_refcnt--; 1088 ASSERT(ap->an_refcnt == 0); 1089 anon_rmhash(ap); 1090 if (ap->an_pvp) 1091 swap_phys_free(ap->an_pvp, ap->an_poff, 1092 PAGESIZE); 1093 mutex_exit(ahm); 1094 if (pp != NULL) { 1095 VM_STAT_ADD(anonvmstats.decrefpages[4]); 1096 /*LINTED*/ 1097 VN_DISPOSE(pp, B_INVAL, 0, kcred); 1098 } 1099 kmem_cache_free(anon_cache, ap); 1100 ANI_ADD(1); 1101 i++; 1102 } else { 1103 pgcnt_t j; 1104 pgcnt_t curpgcnt = 1105 page_get_pagecnt(pp->p_szc); 1106 size_t ppasize = curpgcnt * sizeof (page_t *); 1107 page_t **ppa = kmem_alloc(ppasize, KM_SLEEP); 1108 int dispose = 0; 1109 1110 VM_STAT_ADD(anonvmstats.decrefpages[5]); 1111 1112 ASSERT(pp->p_szc <= szc); 1113 ASSERT(IS_P2ALIGNED(curpgcnt, curpgcnt)); 1114 ASSERT(IS_P2ALIGNED(i, curpgcnt)); 1115 ASSERT(i + curpgcnt <= pgcnt); 1116 ASSERT(!(page_pptonum(pp) & (curpgcnt - 1))); 1117 ppa[0] = pp; 1118 for (j = i + 1; j < i + curpgcnt; j++) { 1119 ap = anon_get_ptr(ahp, an_idx + j); 1120 ASSERT(ap != NULL && 1121 ap->an_refcnt == 1); 1122 swap_xlate(ap, &vp, &off); 1123 pp = page_lookup(vp, (u_offset_t)off, 1124 SE_EXCL); 1125 if (pp == NULL) 1126 panic("anon_decref_pages: " 1127 "no page"); 1128 1129 (void) hat_pageunload(pp, 1130 HAT_FORCE_PGUNLOAD); 1131 ASSERT(pp->p_szc == ppa[0]->p_szc); 1132 ASSERT(page_pptonum(pp) - 1 == 1133 page_pptonum(ppa[j - i - 1])); 1134 ppa[j - i] = pp; 1135 if (ap->an_pvp != NULL && 1136 !vn_matchopval(ap->an_pvp, 1137 VOPNAME_DISPOSE, 1138 (fs_generic_func_p)fs_dispose)) 1139 dispose = 1; 1140 } 1141 if (!dispose) { 1142 VM_STAT_ADD(anonvmstats.decrefpages[6]); 1143 page_destroy_pages(ppa[0]); 1144 } else { 1145 VM_STAT_ADD(anonvmstats.decrefpages[7]); 1146 for (j = 0; j < curpgcnt; j++) { 1147 ASSERT(PAGE_EXCL(ppa[j])); 1148 ppa[j]->p_szc = 0; 1149 } 1150 for (j = 0; j < curpgcnt; j++) { 1151 ASSERT(!hat_page_is_mapped( 1152 ppa[j])); 1153 /*LINTED*/ 1154 VN_DISPOSE(ppa[j], B_INVAL, 0, 1155 kcred); 1156 } 1157 } 1158 kmem_free(ppa, ppasize); 1159 for (j = i; j < i + curpgcnt; j++) { 1160 ap = anon_get_ptr(ahp, an_idx + j); 1161 ASSERT(ap != NULL && 1162 ap->an_refcnt == 1); 1163 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, 1164 ap->an_off)]; 1165 (void) anon_set_ptr(ahp, an_idx + j, 1166 NULL, ANON_SLEEP); 1167 mutex_enter(ahm); 1168 ap->an_refcnt--; 1169 ASSERT(ap->an_refcnt == 0); 1170 anon_rmhash(ap); 1171 if (ap->an_pvp) 1172 swap_phys_free(ap->an_pvp, 1173 ap->an_poff, PAGESIZE); 1174 mutex_exit(ahm); 1175 kmem_cache_free(anon_cache, ap); 1176 ANI_ADD(1); 1177 } 1178 i += curpgcnt; 1179 } 1180 } else { 1181 VM_STAT_ADD(anonvmstats.decrefpages[8]); 1182 (void) anon_set_ptr(ahp, an_idx + i, NULL, ANON_SLEEP); 1183 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1184 mutex_enter(ahm); 1185 ap->an_refcnt--; 1186 mutex_exit(ahm); 1187 i++; 1188 } 1189 } 1190 1191 if (ahmpages != NULL) { 1192 mutex_exit(ahmpages); 1193 } 1194 } 1195 1196 /* 1197 * Duplicate references to size bytes worth of anon pages. 1198 * Used when duplicating a segment that contains private anon pages. 1199 * This code assumes that procedure calling this one has already used 1200 * hat_chgprot() to disable write access to the range of addresses that 1201 * that *old actually refers to. 1202 */ 1203 void 1204 anon_dup(struct anon_hdr *old, ulong_t old_idx, struct anon_hdr *new, 1205 ulong_t new_idx, size_t size) 1206 { 1207 spgcnt_t npages; 1208 kmutex_t *ahm; 1209 struct anon *ap; 1210 ulong_t off; 1211 ulong_t index; 1212 1213 npages = btopr(size); 1214 while (npages > 0) { 1215 index = old_idx; 1216 if ((ap = anon_get_next_ptr(old, &index)) == NULL) 1217 break; 1218 1219 ASSERT(!ANON_ISBUSY(anon_get_slot(old, index))); 1220 off = index - old_idx; 1221 npages -= off; 1222 if (npages <= 0) 1223 break; 1224 1225 (void) anon_set_ptr(new, new_idx + off, ap, ANON_SLEEP); 1226 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1227 1228 mutex_enter(ahm); 1229 ap->an_refcnt++; 1230 mutex_exit(ahm); 1231 1232 off++; 1233 new_idx += off; 1234 old_idx += off; 1235 npages--; 1236 } 1237 } 1238 1239 /* 1240 * Just like anon_dup but also guarantees there are no holes (unallocated anon 1241 * slots) within any large page region. That means if a large page region is 1242 * empty in the old array it will skip it. If there are 1 or more valid slots 1243 * in the large page region of the old array it will make sure to fill in any 1244 * unallocated ones and also copy them to the new array. If noalloc is 1 large 1245 * page region should either have no valid anon slots or all slots should be 1246 * valid. 1247 */ 1248 void 1249 anon_dup_fill_holes( 1250 struct anon_hdr *old, 1251 ulong_t old_idx, 1252 struct anon_hdr *new, 1253 ulong_t new_idx, 1254 size_t size, 1255 uint_t szc, 1256 int noalloc) 1257 { 1258 struct anon *ap; 1259 spgcnt_t npages; 1260 kmutex_t *ahm, *ahmpages = NULL; 1261 pgcnt_t pgcnt, i; 1262 ulong_t index, off; 1263 #ifdef DEBUG 1264 int refcnt; 1265 #endif 1266 1267 ASSERT(szc != 0); 1268 pgcnt = page_get_pagecnt(szc); 1269 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1270 npages = btopr(size); 1271 ASSERT(IS_P2ALIGNED(npages, pgcnt)); 1272 ASSERT(IS_P2ALIGNED(old_idx, pgcnt)); 1273 1274 VM_STAT_ADD(anonvmstats.dupfillholes[0]); 1275 1276 while (npages > 0) { 1277 index = old_idx; 1278 1279 /* 1280 * Find the next valid slot. 1281 */ 1282 if (anon_get_next_ptr(old, &index) == NULL) 1283 break; 1284 1285 ASSERT(!ANON_ISBUSY(anon_get_slot(old, index))); 1286 /* 1287 * Now backup index to the beginning of the 1288 * current large page region of the old array. 1289 */ 1290 index = P2ALIGN(index, pgcnt); 1291 off = index - old_idx; 1292 ASSERT(IS_P2ALIGNED(off, pgcnt)); 1293 npages -= off; 1294 if (npages <= 0) 1295 break; 1296 1297 /* 1298 * Fill and copy a large page regions worth 1299 * of anon slots. 1300 */ 1301 for (i = 0; i < pgcnt; i++) { 1302 if ((ap = anon_get_ptr(old, index + i)) == NULL) { 1303 if (noalloc) { 1304 panic("anon_dup_fill_holes: " 1305 "empty anon slot\n"); 1306 } 1307 VM_STAT_ADD(anonvmstats.dupfillholes[1]); 1308 ap = anon_alloc(NULL, 0); 1309 (void) anon_set_ptr(old, index + i, ap, 1310 ANON_SLEEP); 1311 } else if (i == 0) { 1312 /* 1313 * make the increment of all refcnts of all 1314 * anon slots of a large page appear atomic by 1315 * getting an anonpages_hash_lock for the 1316 * first anon slot of a large page. 1317 */ 1318 int hash = AH_LOCK(ap->an_vp, ap->an_off); 1319 1320 VM_STAT_ADD(anonvmstats.dupfillholes[2]); 1321 1322 ahmpages = &anonpages_hash_lock[hash]; 1323 mutex_enter(ahmpages); 1324 /*LINTED*/ 1325 ASSERT(refcnt = ap->an_refcnt); 1326 1327 VM_STAT_COND_ADD(ap->an_refcnt > 1, 1328 anonvmstats.dupfillholes[3]); 1329 } 1330 (void) anon_set_ptr(new, new_idx + off + i, ap, 1331 ANON_SLEEP); 1332 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1333 mutex_enter(ahm); 1334 ASSERT(ahmpages != NULL || ap->an_refcnt == 1); 1335 ASSERT(i == 0 || ahmpages == NULL || 1336 refcnt == ap->an_refcnt); 1337 ap->an_refcnt++; 1338 mutex_exit(ahm); 1339 } 1340 if (ahmpages != NULL) { 1341 mutex_exit(ahmpages); 1342 ahmpages = NULL; 1343 } 1344 off += pgcnt; 1345 new_idx += off; 1346 old_idx += off; 1347 npages -= pgcnt; 1348 } 1349 } 1350 1351 /* 1352 * Used when a segment with a vnode changes szc. similarly to 1353 * anon_dup_fill_holes() makes sure each large page region either has no anon 1354 * slots or all of them. but new slots are created by COWing the file 1355 * pages. on entrance no anon slots should be shared. 1356 */ 1357 int 1358 anon_fill_cow_holes( 1359 struct seg *seg, 1360 caddr_t addr, 1361 struct anon_hdr *ahp, 1362 ulong_t an_idx, 1363 struct vnode *vp, 1364 u_offset_t vp_off, 1365 size_t size, 1366 uint_t szc, 1367 uint_t prot, 1368 struct vpage vpage[], 1369 struct cred *cred) 1370 { 1371 struct anon *ap; 1372 spgcnt_t npages; 1373 pgcnt_t pgcnt, i; 1374 ulong_t index, off; 1375 int err = 0; 1376 int pageflags = 0; 1377 1378 ASSERT(szc != 0); 1379 pgcnt = page_get_pagecnt(szc); 1380 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1381 npages = btopr(size); 1382 ASSERT(IS_P2ALIGNED(npages, pgcnt)); 1383 ASSERT(IS_P2ALIGNED(an_idx, pgcnt)); 1384 1385 while (npages > 0) { 1386 index = an_idx; 1387 1388 /* 1389 * Find the next valid slot. 1390 */ 1391 if (anon_get_next_ptr(ahp, &index) == NULL) { 1392 break; 1393 } 1394 1395 ASSERT(!ANON_ISBUSY(anon_get_slot(ahp, index))); 1396 /* 1397 * Now backup index to the beginning of the 1398 * current large page region of the anon array. 1399 */ 1400 index = P2ALIGN(index, pgcnt); 1401 off = index - an_idx; 1402 ASSERT(IS_P2ALIGNED(off, pgcnt)); 1403 npages -= off; 1404 if (npages <= 0) 1405 break; 1406 an_idx += off; 1407 vp_off += ptob(off); 1408 addr += ptob(off); 1409 if (vpage != NULL) { 1410 vpage += off; 1411 } 1412 1413 for (i = 0; i < pgcnt; i++, an_idx++, vp_off += PAGESIZE) { 1414 if ((ap = anon_get_ptr(ahp, an_idx)) == NULL) { 1415 page_t *pl[1 + 1]; 1416 page_t *pp; 1417 1418 err = VOP_GETPAGE(vp, vp_off, PAGESIZE, NULL, 1419 pl, PAGESIZE, seg, addr, S_READ, cred); 1420 if (err) { 1421 break; 1422 } 1423 if (vpage != NULL) { 1424 prot = VPP_PROT(vpage); 1425 pageflags = VPP_ISPPLOCK(vpage) ? 1426 LOCK_PAGE : 0; 1427 } 1428 pp = anon_private(&ap, seg, addr, prot, pl[0], 1429 pageflags, cred); 1430 if (pp == NULL) { 1431 err = ENOMEM; 1432 break; 1433 } 1434 (void) anon_set_ptr(ahp, an_idx, ap, 1435 ANON_SLEEP); 1436 page_unlock(pp); 1437 } 1438 ASSERT(ap->an_refcnt == 1); 1439 addr += PAGESIZE; 1440 if (vpage != NULL) { 1441 vpage++; 1442 } 1443 } 1444 npages -= pgcnt; 1445 } 1446 1447 return (err); 1448 } 1449 1450 /* 1451 * Free a group of "size" anon pages, size in bytes, 1452 * and clear out the pointers to the anon entries. 1453 */ 1454 void 1455 anon_free(struct anon_hdr *ahp, ulong_t index, size_t size) 1456 { 1457 spgcnt_t npages; 1458 struct anon *ap; 1459 ulong_t old; 1460 1461 npages = btopr(size); 1462 1463 while (npages > 0) { 1464 old = index; 1465 if ((ap = anon_get_next_ptr(ahp, &index)) == NULL) 1466 break; 1467 1468 ASSERT(!ANON_ISBUSY(anon_get_slot(ahp, index))); 1469 npages -= index - old; 1470 if (npages <= 0) 1471 break; 1472 1473 (void) anon_set_ptr(ahp, index, NULL, ANON_SLEEP); 1474 anon_decref(ap); 1475 /* 1476 * Bump index and decrement page count 1477 */ 1478 index++; 1479 npages--; 1480 } 1481 } 1482 1483 void 1484 anon_free_pages( 1485 struct anon_hdr *ahp, 1486 ulong_t an_idx, 1487 size_t size, 1488 uint_t szc) 1489 { 1490 spgcnt_t npages; 1491 pgcnt_t pgcnt; 1492 ulong_t index, off; 1493 1494 ASSERT(szc != 0); 1495 pgcnt = page_get_pagecnt(szc); 1496 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1497 npages = btopr(size); 1498 ASSERT(IS_P2ALIGNED(npages, pgcnt)); 1499 ASSERT(IS_P2ALIGNED(an_idx, pgcnt)); 1500 ASSERT(an_idx < ahp->size); 1501 1502 VM_STAT_ADD(anonvmstats.freepages[0]); 1503 1504 while (npages > 0) { 1505 index = an_idx; 1506 1507 /* 1508 * Find the next valid slot. 1509 */ 1510 if (anon_get_next_ptr(ahp, &index) == NULL) 1511 break; 1512 1513 ASSERT(!ANON_ISBUSY(anon_get_slot(ahp, index))); 1514 /* 1515 * Now backup index to the beginning of the 1516 * current large page region of the old array. 1517 */ 1518 index = P2ALIGN(index, pgcnt); 1519 off = index - an_idx; 1520 ASSERT(IS_P2ALIGNED(off, pgcnt)); 1521 npages -= off; 1522 if (npages <= 0) 1523 break; 1524 1525 anon_decref_pages(ahp, index, szc); 1526 1527 off += pgcnt; 1528 an_idx += off; 1529 npages -= pgcnt; 1530 } 1531 } 1532 1533 /* 1534 * Make anonymous pages discardable 1535 */ 1536 void 1537 anon_disclaim(struct anon_map *amp, ulong_t index, size_t size, int flags) 1538 { 1539 spgcnt_t npages = btopr(size); 1540 struct anon *ap; 1541 struct vnode *vp; 1542 anoff_t off; 1543 page_t *pp, *root_pp; 1544 kmutex_t *ahm; 1545 pgcnt_t pgcnt; 1546 ulong_t old_idx, idx, i; 1547 struct anon_hdr *ahp = amp->ahp; 1548 anon_sync_obj_t cookie; 1549 1550 ASSERT(RW_READ_HELD(&->a_rwlock)); 1551 pgcnt = 1; 1552 for (; npages > 0; index = (pgcnt == 1) ? index + 1: 1553 P2ROUNDUP(index + 1, pgcnt), npages -= pgcnt) { 1554 1555 /* 1556 * get anon pointer and index for the first valid entry 1557 * in the anon list, starting from "index" 1558 */ 1559 old_idx = index; 1560 if ((ap = anon_get_next_ptr(ahp, &index)) == NULL) 1561 break; 1562 1563 /* 1564 * decrement npages by number of NULL anon slots we skipped 1565 */ 1566 npages -= index - old_idx; 1567 if (npages <= 0) 1568 break; 1569 1570 anon_array_enter(amp, index, &cookie); 1571 ap = anon_get_ptr(ahp, index); 1572 ASSERT(ap != NULL); 1573 1574 /* 1575 * Get anonymous page and try to lock it SE_EXCL; 1576 * For non blocking case if we couldn't grab the lock 1577 * we skip to next page. 1578 * For blocking case (ANON_PGLOOKUP_BLK) block 1579 * until we grab SE_EXCL lock. 1580 */ 1581 swap_xlate(ap, &vp, &off); 1582 if (flags & ANON_PGLOOKUP_BLK) 1583 pp = page_lookup_create(vp, (u_offset_t)off, 1584 SE_EXCL, NULL, NULL, SE_EXCL_WANTED); 1585 else 1586 pp = page_lookup_nowait(vp, (u_offset_t)off, SE_EXCL); 1587 if (pp == NULL) { 1588 segadvstat.MADV_FREE_miss.value.ul++; 1589 pgcnt = 1; 1590 anon_array_exit(&cookie); 1591 continue; 1592 } 1593 pgcnt = page_get_pagecnt(pp->p_szc); 1594 1595 /* 1596 * we cannot free a page which is permanently locked. 1597 * The page_struct_lock need not be acquired to examine 1598 * these fields since the page has an "exclusive" lock. 1599 */ 1600 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 1601 page_unlock(pp); 1602 segadvstat.MADV_FREE_miss.value.ul++; 1603 anon_array_exit(&cookie); 1604 continue; 1605 } 1606 1607 ahm = &anonhash_lock[AH_LOCK(vp, off)]; 1608 mutex_enter(ahm); 1609 ASSERT(ap->an_refcnt != 0); 1610 /* 1611 * skip this one if copy-on-write is not yet broken. 1612 */ 1613 if (ap->an_refcnt > 1) { 1614 mutex_exit(ahm); 1615 page_unlock(pp); 1616 segadvstat.MADV_FREE_miss.value.ul++; 1617 anon_array_exit(&cookie); 1618 continue; 1619 } 1620 1621 if (pp->p_szc == 0) { 1622 pgcnt = 1; 1623 1624 /* 1625 * free swap slot; 1626 */ 1627 if (ap->an_pvp) { 1628 swap_phys_free(ap->an_pvp, ap->an_poff, 1629 PAGESIZE); 1630 ap->an_pvp = NULL; 1631 ap->an_poff = 0; 1632 } 1633 mutex_exit(ahm); 1634 segadvstat.MADV_FREE_hit.value.ul++; 1635 1636 /* 1637 * while we are at it, unload all the translations 1638 * and attempt to free the page. 1639 */ 1640 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1641 /*LINTED: constant in conditional context */ 1642 VN_DISPOSE(pp, B_FREE, 0, kcred); 1643 anon_array_exit(&cookie); 1644 continue; 1645 } 1646 1647 pgcnt = page_get_pagecnt(pp->p_szc); 1648 if (!IS_P2ALIGNED(index, pgcnt) || npages < pgcnt) { 1649 if (!page_try_demote_pages(pp)) { 1650 mutex_exit(ahm); 1651 page_unlock(pp); 1652 segadvstat.MADV_FREE_miss.value.ul++; 1653 anon_array_exit(&cookie); 1654 continue; 1655 } else { 1656 pgcnt = 1; 1657 if (ap->an_pvp) { 1658 swap_phys_free(ap->an_pvp, 1659 ap->an_poff, PAGESIZE); 1660 ap->an_pvp = NULL; 1661 ap->an_poff = 0; 1662 } 1663 mutex_exit(ahm); 1664 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1665 /*LINTED*/ 1666 VN_DISPOSE(pp, B_FREE, 0, kcred); 1667 segadvstat.MADV_FREE_hit.value.ul++; 1668 anon_array_exit(&cookie); 1669 continue; 1670 } 1671 } 1672 mutex_exit(ahm); 1673 root_pp = pp; 1674 1675 /* 1676 * try to lock remaining pages 1677 */ 1678 for (idx = 1; idx < pgcnt; idx++) { 1679 pp++; 1680 if (!page_trylock(pp, SE_EXCL)) 1681 break; 1682 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 1683 page_unlock(pp); 1684 break; 1685 } 1686 } 1687 1688 if (idx == pgcnt) { 1689 for (i = 0; i < pgcnt; i++) { 1690 ap = anon_get_ptr(ahp, index + i); 1691 if (ap == NULL) 1692 break; 1693 swap_xlate(ap, &vp, &off); 1694 ahm = &anonhash_lock[AH_LOCK(vp, off)]; 1695 mutex_enter(ahm); 1696 ASSERT(ap->an_refcnt != 0); 1697 1698 /* 1699 * skip this one if copy-on-write 1700 * is not yet broken. 1701 */ 1702 if (ap->an_refcnt > 1) { 1703 mutex_exit(ahm); 1704 goto skiplp; 1705 } 1706 if (ap->an_pvp) { 1707 swap_phys_free(ap->an_pvp, 1708 ap->an_poff, PAGESIZE); 1709 ap->an_pvp = NULL; 1710 ap->an_poff = 0; 1711 } 1712 mutex_exit(ahm); 1713 } 1714 page_destroy_pages(root_pp); 1715 segadvstat.MADV_FREE_hit.value.ul += pgcnt; 1716 anon_array_exit(&cookie); 1717 continue; 1718 } 1719 skiplp: 1720 segadvstat.MADV_FREE_miss.value.ul += pgcnt; 1721 for (i = 0, pp = root_pp; i < idx; pp++, i++) 1722 page_unlock(pp); 1723 anon_array_exit(&cookie); 1724 } 1725 } 1726 1727 /* 1728 * Return the kept page(s) and protections back to the segment driver. 1729 */ 1730 int 1731 anon_getpage( 1732 struct anon **app, 1733 uint_t *protp, 1734 page_t *pl[], 1735 size_t plsz, 1736 struct seg *seg, 1737 caddr_t addr, 1738 enum seg_rw rw, 1739 struct cred *cred) 1740 { 1741 page_t *pp; 1742 struct anon *ap = *app; 1743 struct vnode *vp; 1744 anoff_t off; 1745 int err; 1746 kmutex_t *ahm; 1747 1748 swap_xlate(ap, &vp, &off); 1749 1750 /* 1751 * Lookup the page. If page is being paged in, 1752 * wait for it to finish as we must return a list of 1753 * pages since this routine acts like the VOP_GETPAGE 1754 * routine does. 1755 */ 1756 if (pl != NULL && (pp = page_lookup(vp, (u_offset_t)off, SE_SHARED))) { 1757 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1758 mutex_enter(ahm); 1759 if (ap->an_refcnt == 1) 1760 *protp = PROT_ALL; 1761 else 1762 *protp = PROT_ALL & ~PROT_WRITE; 1763 mutex_exit(ahm); 1764 pl[0] = pp; 1765 pl[1] = NULL; 1766 return (0); 1767 } 1768 1769 /* 1770 * Simply treat it as a vnode fault on the anon vp. 1771 */ 1772 1773 TRACE_3(TR_FAC_VM, TR_ANON_GETPAGE, 1774 "anon_getpage:seg %x addr %x vp %x", 1775 seg, addr, vp); 1776 1777 err = VOP_GETPAGE(vp, (u_offset_t)off, PAGESIZE, protp, pl, plsz, 1778 seg, addr, rw, cred); 1779 1780 if (err == 0 && pl != NULL) { 1781 ahm = &anonhash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 1782 mutex_enter(ahm); 1783 if (ap->an_refcnt != 1) 1784 *protp &= ~PROT_WRITE; /* make read-only */ 1785 mutex_exit(ahm); 1786 } 1787 return (err); 1788 } 1789 1790 /* 1791 * Creates or returns kept pages to the segment driver. returns -1 if a large 1792 * page cannot be allocated. returns -2 if some other process has allocated a 1793 * larger page. 1794 * 1795 * For cowfault it will alocate any size pages to fill the requested area to 1796 * avoid partially overwritting anon slots (i.e. sharing only some of the anon 1797 * slots within a large page with other processes). This policy greatly 1798 * simplifies large page freeing (which is only freed when all anon slot 1799 * refcnts are 0). 1800 */ 1801 int 1802 anon_map_getpages( 1803 struct anon_map *amp, 1804 ulong_t start_idx, 1805 uint_t szc, 1806 struct seg *seg, 1807 caddr_t addr, 1808 uint_t prot, 1809 uint_t *protp, 1810 page_t *ppa[], 1811 uint_t *ppa_szc, 1812 struct vpage vpage[], 1813 enum seg_rw rw, 1814 int brkcow, 1815 int anypgsz, 1816 struct cred *cred) 1817 { 1818 pgcnt_t pgcnt; 1819 struct anon *ap; 1820 struct vnode *vp; 1821 anoff_t off; 1822 page_t *pp, *pl[2], *conpp = NULL; 1823 caddr_t vaddr; 1824 ulong_t pg_idx, an_idx, i; 1825 spgcnt_t nreloc = 0; 1826 int prealloc = 1; 1827 int err, slotcreate; 1828 uint_t vpprot; 1829 int upsize = (szc < seg->s_szc); 1830 1831 #if !defined(__i386) && !defined(__amd64) 1832 ASSERT(seg->s_szc != 0); 1833 #endif 1834 ASSERT(szc <= seg->s_szc); 1835 ASSERT(ppa_szc != NULL); 1836 ASSERT(rw != S_CREATE); 1837 1838 *protp = PROT_ALL; 1839 1840 VM_STAT_ADD(anonvmstats.getpages[0]); 1841 1842 if (szc == 0) { 1843 VM_STAT_ADD(anonvmstats.getpages[1]); 1844 if ((ap = anon_get_ptr(amp->ahp, start_idx)) != NULL) { 1845 err = anon_getpage(&ap, protp, pl, PAGESIZE, seg, 1846 addr, rw, cred); 1847 if (err) 1848 return (err); 1849 ppa[0] = pl[0]; 1850 if (brkcow == 0 || (*protp & PROT_WRITE)) { 1851 VM_STAT_ADD(anonvmstats.getpages[2]); 1852 if (ppa[0]->p_szc != 0 && upsize) { 1853 VM_STAT_ADD(anonvmstats.getpages[3]); 1854 *ppa_szc = MIN(ppa[0]->p_szc, 1855 seg->s_szc); 1856 page_unlock(ppa[0]); 1857 return (-2); 1858 } 1859 return (0); 1860 } 1861 panic("anon_map_getpages: cowfault for szc 0"); 1862 } else { 1863 VM_STAT_ADD(anonvmstats.getpages[4]); 1864 ppa[0] = anon_zero(seg, addr, &ap, cred); 1865 if (ppa[0] == NULL) 1866 return (ENOMEM); 1867 (void) anon_set_ptr(amp->ahp, start_idx, ap, 1868 ANON_SLEEP); 1869 return (0); 1870 } 1871 } 1872 1873 pgcnt = page_get_pagecnt(szc); 1874 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 1875 ASSERT(IS_P2ALIGNED(start_idx, pgcnt)); 1876 1877 /* 1878 * First we check for the case that the requtested large 1879 * page or larger page already exists in the system. 1880 * Actually we only check if the first constituent page 1881 * exists and only preallocate if it's not found. 1882 */ 1883 ap = anon_get_ptr(amp->ahp, start_idx); 1884 if (ap) { 1885 uint_t pszc; 1886 swap_xlate(ap, &vp, &off); 1887 if (page_exists_forreal(vp, (u_offset_t)off, &pszc)) { 1888 if (pszc > szc && upsize) { 1889 *ppa_szc = MIN(pszc, seg->s_szc); 1890 return (-2); 1891 } 1892 if (pszc >= szc) { 1893 prealloc = 0; 1894 } 1895 } 1896 } 1897 1898 VM_STAT_COND_ADD(prealloc == 0, anonvmstats.getpages[5]); 1899 VM_STAT_COND_ADD(prealloc != 0, anonvmstats.getpages[6]); 1900 1901 top: 1902 /* 1903 * If a smaller page or no page at all was found, 1904 * grab a large page off the freelist. 1905 */ 1906 if (prealloc) { 1907 ASSERT(conpp == NULL); 1908 if (page_alloc_pages(anon_vp, seg, addr, NULL, ppa, 1909 szc, 0) != 0) { 1910 VM_STAT_ADD(anonvmstats.getpages[7]); 1911 if (brkcow == 0 || 1912 !anon_share(amp->ahp, start_idx, pgcnt)) { 1913 /* 1914 * If the refcnt's of all anon slots are <= 1 1915 * they can't increase since we are holding 1916 * the address space's lock. So segvn can 1917 * safely decrease szc without risking to 1918 * generate a cow fault for the region smaller 1919 * than the segment's largest page size. 1920 */ 1921 VM_STAT_ADD(anonvmstats.getpages[8]); 1922 return (-1); 1923 } 1924 docow: 1925 /* 1926 * This is a cow fault. Copy away the entire 1 large 1927 * page region of this segment. 1928 */ 1929 if (szc != seg->s_szc) 1930 panic("anon_map_getpages: cowfault for szc %d", 1931 szc); 1932 vaddr = addr; 1933 for (pg_idx = 0, an_idx = start_idx; pg_idx < pgcnt; 1934 pg_idx++, an_idx++, vaddr += PAGESIZE) { 1935 if ((ap = anon_get_ptr(amp->ahp, an_idx)) != 1936 NULL) { 1937 err = anon_getpage(&ap, &vpprot, pl, 1938 PAGESIZE, seg, vaddr, rw, cred); 1939 if (err) { 1940 for (i = 0; i < pg_idx; i++) { 1941 if ((pp = ppa[i]) != 1942 NULL) 1943 page_unlock(pp); 1944 } 1945 return (err); 1946 } 1947 ppa[pg_idx] = pl[0]; 1948 } else { 1949 /* 1950 * Since this is a cowfault we know 1951 * that this address space has a 1952 * parent or children which means 1953 * anon_dup_fill_holes() has initialized 1954 * all anon slots within a large page 1955 * region that had at least one anon 1956 * slot at the time of fork(). 1957 */ 1958 panic("anon_map_getpages: " 1959 "cowfault but anon slot is empty"); 1960 } 1961 } 1962 VM_STAT_ADD(anonvmstats.getpages[9]); 1963 *protp = PROT_ALL; 1964 return (anon_map_privatepages(amp, start_idx, szc, seg, 1965 addr, prot, ppa, vpage, anypgsz, cred)); 1966 } 1967 } 1968 1969 VM_STAT_ADD(anonvmstats.getpages[10]); 1970 1971 an_idx = start_idx; 1972 pg_idx = 0; 1973 vaddr = addr; 1974 while (pg_idx < pgcnt) { 1975 slotcreate = 0; 1976 if ((ap = anon_get_ptr(amp->ahp, an_idx)) == NULL) { 1977 VM_STAT_ADD(anonvmstats.getpages[11]); 1978 /* 1979 * For us to have decided not to preallocate 1980 * would have meant that a large page 1981 * was found. Which also means that all of the 1982 * anon slots for that page would have been 1983 * already created for us. 1984 */ 1985 if (prealloc == 0) 1986 panic("anon_map_getpages: prealloc = 0"); 1987 1988 slotcreate = 1; 1989 ap = anon_alloc(NULL, 0); 1990 } 1991 swap_xlate(ap, &vp, &off); 1992 1993 /* 1994 * Now setup our preallocated page to pass down 1995 * to swap_getpage(). 1996 */ 1997 if (prealloc) { 1998 ASSERT(ppa[pg_idx]->p_szc == szc); 1999 conpp = ppa[pg_idx]; 2000 } 2001 ASSERT(prealloc || conpp == NULL); 2002 2003 /* 2004 * If we just created this anon slot then call 2005 * with S_CREATE to prevent doing IO on the page. 2006 * Similar to the anon_zero case. 2007 */ 2008 err = swap_getconpage(vp, (u_offset_t)off, PAGESIZE, 2009 NULL, pl, PAGESIZE, conpp, ppa_szc, &nreloc, seg, vaddr, 2010 slotcreate == 1 ? S_CREATE : rw, cred); 2011 2012 if (err) { 2013 ASSERT(err != -2 || upsize); 2014 VM_STAT_ADD(anonvmstats.getpages[12]); 2015 ASSERT(slotcreate == 0); 2016 goto io_err; 2017 } 2018 2019 pp = pl[0]; 2020 2021 if (pp->p_szc < szc || (pp->p_szc > szc && upsize)) { 2022 VM_STAT_ADD(anonvmstats.getpages[13]); 2023 ASSERT(slotcreate == 0); 2024 ASSERT(prealloc == 0); 2025 ASSERT(pg_idx == 0); 2026 if (pp->p_szc > szc) { 2027 ASSERT(upsize); 2028 *ppa_szc = MIN(pp->p_szc, seg->s_szc); 2029 page_unlock(pp); 2030 VM_STAT_ADD(anonvmstats.getpages[14]); 2031 return (-2); 2032 } 2033 page_unlock(pp); 2034 prealloc = 1; 2035 goto top; 2036 } 2037 2038 /* 2039 * If we decided to preallocate but VOP_GETPAGE 2040 * found a page in the system that satisfies our 2041 * request then free up our preallocated large page 2042 * and continue looping accross the existing large 2043 * page via VOP_GETPAGE. 2044 */ 2045 if (prealloc && pp != ppa[pg_idx]) { 2046 VM_STAT_ADD(anonvmstats.getpages[15]); 2047 ASSERT(slotcreate == 0); 2048 ASSERT(pg_idx == 0); 2049 conpp = NULL; 2050 prealloc = 0; 2051 page_free_pages(ppa[0]); 2052 } 2053 2054 if (prealloc && nreloc > 1) { 2055 /* 2056 * we have relocated out of a smaller large page. 2057 * skip npgs - 1 iterations and continue which will 2058 * increment by one the loop indices. 2059 */ 2060 spgcnt_t npgs = nreloc; 2061 2062 VM_STAT_ADD(anonvmstats.getpages[16]); 2063 2064 ASSERT(pp == ppa[pg_idx]); 2065 ASSERT(slotcreate == 0); 2066 ASSERT(pg_idx + npgs <= pgcnt); 2067 if ((*protp & PROT_WRITE) && 2068 anon_share(amp->ahp, an_idx, npgs)) { 2069 *protp &= ~PROT_WRITE; 2070 } 2071 pg_idx += npgs; 2072 an_idx += npgs; 2073 vaddr += PAGESIZE * npgs; 2074 continue; 2075 } 2076 2077 VM_STAT_ADD(anonvmstats.getpages[17]); 2078 2079 /* 2080 * Anon_zero case. 2081 */ 2082 if (slotcreate) { 2083 ASSERT(prealloc); 2084 pagezero(pp, 0, PAGESIZE); 2085 CPU_STATS_ADD_K(vm, zfod, 1); 2086 hat_setrefmod(pp); 2087 } 2088 2089 ASSERT(prealloc == 0 || ppa[pg_idx] == pp); 2090 ASSERT(prealloc != 0 || PAGE_SHARED(pp)); 2091 ASSERT(prealloc == 0 || PAGE_EXCL(pp)); 2092 2093 if (pg_idx > 0 && 2094 ((page_pptonum(pp) != page_pptonum(ppa[pg_idx - 1]) + 1) || 2095 (pp->p_szc != ppa[pg_idx - 1]->p_szc))) { 2096 panic("anon_map_getpages: unexpected page"); 2097 } else if (pg_idx == 0 && (page_pptonum(pp) & (pgcnt - 1))) { 2098 panic("anon_map_getpages: unaligned page"); 2099 } 2100 2101 if (prealloc == 0) { 2102 ppa[pg_idx] = pp; 2103 } 2104 2105 if (ap->an_refcnt > 1) { 2106 VM_STAT_ADD(anonvmstats.getpages[18]); 2107 *protp &= ~PROT_WRITE; 2108 } 2109 2110 /* 2111 * If this is a new anon slot then initialize 2112 * the anon array entry. 2113 */ 2114 if (slotcreate) { 2115 (void) anon_set_ptr(amp->ahp, an_idx, ap, ANON_SLEEP); 2116 } 2117 pg_idx++; 2118 an_idx++; 2119 vaddr += PAGESIZE; 2120 } 2121 2122 /* 2123 * Since preallocated pages come off the freelist 2124 * they are locked SE_EXCL. Simply downgrade and return. 2125 */ 2126 if (prealloc) { 2127 VM_STAT_ADD(anonvmstats.getpages[19]); 2128 conpp = NULL; 2129 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2130 page_downgrade(ppa[pg_idx]); 2131 } 2132 } 2133 ASSERT(conpp == NULL); 2134 2135 if (brkcow == 0 || (*protp & PROT_WRITE)) { 2136 VM_STAT_ADD(anonvmstats.getpages[20]); 2137 return (0); 2138 } 2139 2140 if (szc < seg->s_szc) 2141 panic("anon_map_getpages: cowfault for szc %d", szc); 2142 2143 VM_STAT_ADD(anonvmstats.getpages[21]); 2144 2145 *protp = PROT_ALL; 2146 return (anon_map_privatepages(amp, start_idx, szc, seg, addr, prot, 2147 ppa, vpage, anypgsz, cred)); 2148 io_err: 2149 /* 2150 * We got an IO error somewhere in our large page. 2151 * If we were using a preallocated page then just demote 2152 * all the constituent pages that we've succeeded with sofar 2153 * to PAGESIZE pages and leave them in the system 2154 * unlocked. 2155 */ 2156 2157 ASSERT(err != -2 || ((pg_idx == 0) && upsize)); 2158 2159 VM_STAT_COND_ADD(err > 0, anonvmstats.getpages[22]); 2160 VM_STAT_COND_ADD(err == -1, anonvmstats.getpages[23]); 2161 VM_STAT_COND_ADD(err == -2, anonvmstats.getpages[24]); 2162 2163 if (prealloc) { 2164 conpp = NULL; 2165 if (pg_idx > 0) { 2166 VM_STAT_ADD(anonvmstats.getpages[25]); 2167 for (i = 0; i < pgcnt; i++) { 2168 pp = ppa[i]; 2169 ASSERT(PAGE_EXCL(pp)); 2170 ASSERT(pp->p_szc == szc); 2171 pp->p_szc = 0; 2172 } 2173 for (i = 0; i < pg_idx; i++) { 2174 ASSERT(!hat_page_is_mapped(ppa[i])); 2175 page_unlock(ppa[i]); 2176 } 2177 /* 2178 * Now free up the remaining unused constituent 2179 * pages. 2180 */ 2181 while (pg_idx < pgcnt) { 2182 ASSERT(!hat_page_is_mapped(ppa[pg_idx])); 2183 page_free(ppa[pg_idx], 0); 2184 pg_idx++; 2185 } 2186 } else { 2187 VM_STAT_ADD(anonvmstats.getpages[26]); 2188 page_free_pages(ppa[0]); 2189 } 2190 } else { 2191 VM_STAT_ADD(anonvmstats.getpages[27]); 2192 ASSERT(err > 0); 2193 for (i = 0; i < pg_idx; i++) 2194 page_unlock(ppa[i]); 2195 } 2196 ASSERT(conpp == NULL); 2197 if (err != -1) 2198 return (err); 2199 /* 2200 * we are here because we failed to relocate. 2201 */ 2202 ASSERT(prealloc); 2203 if (brkcow == 0 || !anon_share(amp->ahp, start_idx, pgcnt)) { 2204 VM_STAT_ADD(anonvmstats.getpages[28]); 2205 return (-1); 2206 } 2207 VM_STAT_ADD(anonvmstats.getpages[29]); 2208 goto docow; 2209 } 2210 2211 2212 /* 2213 * Turn a reference to an object or shared anon page 2214 * into a private page with a copy of the data from the 2215 * original page which is always locked by the caller. 2216 * This routine unloads the translation and unlocks the 2217 * original page, if it isn't being stolen, before returning 2218 * to the caller. 2219 * 2220 * NOTE: The original anon slot is not freed by this routine 2221 * It must be freed by the caller while holding the 2222 * "anon_map" lock to prevent races which can occur if 2223 * a process has multiple lwps in its address space. 2224 */ 2225 page_t * 2226 anon_private( 2227 struct anon **app, 2228 struct seg *seg, 2229 caddr_t addr, 2230 uint_t prot, 2231 page_t *opp, 2232 int oppflags, 2233 struct cred *cred) 2234 { 2235 struct anon *old = *app; 2236 struct anon *new; 2237 page_t *pp = NULL; 2238 struct vnode *vp; 2239 anoff_t off; 2240 page_t *anon_pl[1 + 1]; 2241 int err; 2242 2243 if (oppflags & STEAL_PAGE) 2244 ASSERT(PAGE_EXCL(opp)); 2245 else 2246 ASSERT(PAGE_LOCKED(opp)); 2247 2248 CPU_STATS_ADD_K(vm, cow_fault, 1); 2249 2250 /* Kernel probe */ 2251 TNF_PROBE_1(anon_private, "vm pagefault", /* CSTYLED */, 2252 tnf_opaque, address, addr); 2253 2254 *app = new = anon_alloc(NULL, 0); 2255 swap_xlate(new, &vp, &off); 2256 2257 if (oppflags & STEAL_PAGE) { 2258 page_rename(opp, vp, (u_offset_t)off); 2259 pp = opp; 2260 TRACE_5(TR_FAC_VM, TR_ANON_PRIVATE, 2261 "anon_private:seg %p addr %x pp %p vp %p off %lx", 2262 seg, addr, pp, vp, off); 2263 hat_setmod(pp); 2264 2265 /* bug 4026339 */ 2266 page_downgrade(pp); 2267 return (pp); 2268 } 2269 2270 /* 2271 * Call the VOP_GETPAGE routine to create the page, thereby 2272 * enabling the vnode driver to allocate any filesystem 2273 * space (e.g., disk block allocation for UFS). This also 2274 * prevents more than one page from being added to the 2275 * vnode at the same time. 2276 */ 2277 err = VOP_GETPAGE(vp, (u_offset_t)off, PAGESIZE, NULL, 2278 anon_pl, PAGESIZE, seg, addr, S_CREATE, cred); 2279 if (err) 2280 goto out; 2281 2282 pp = anon_pl[0]; 2283 2284 /* 2285 * If the original page was locked, we need to move the lock 2286 * to the new page by transfering 'cowcnt/lckcnt' of the original 2287 * page to 'cowcnt/lckcnt' of the new page. 2288 * 2289 * See Statement at the beginning of segvn_lockop() and 2290 * comments in page_pp_useclaim() regarding the way 2291 * cowcnts/lckcnts are handled. 2292 * 2293 * Also availrmem must be decremented up front for read only mapping 2294 * before calling page_pp_useclaim. page_pp_useclaim will bump it back 2295 * if availrmem did not need to be decremented after all. 2296 */ 2297 if (oppflags & LOCK_PAGE) { 2298 if ((prot & PROT_WRITE) == 0) { 2299 mutex_enter(&freemem_lock); 2300 if (availrmem > pages_pp_maximum) { 2301 availrmem--; 2302 pages_useclaim++; 2303 } else { 2304 mutex_exit(&freemem_lock); 2305 goto out; 2306 } 2307 mutex_exit(&freemem_lock); 2308 } 2309 page_pp_useclaim(opp, pp, prot & PROT_WRITE); 2310 } 2311 2312 /* 2313 * Now copy the contents from the original page, 2314 * which is locked and loaded in the MMU by 2315 * the caller to prevent yet another page fault. 2316 */ 2317 /* XXX - should set mod bit in here */ 2318 if (ppcopy(opp, pp) == 0) { 2319 /* 2320 * Before ppcopy could hanlde UE or other faults, we 2321 * would have panicked here, and still have no option 2322 * but to do so now. 2323 */ 2324 panic("anon_private, ppcopy failed, opp = 0x%p, pp = 0x%p", 2325 opp, pp); 2326 } 2327 2328 hat_setrefmod(pp); /* mark as modified */ 2329 2330 /* 2331 * Unload the old translation. 2332 */ 2333 hat_unload(seg->s_as->a_hat, addr, PAGESIZE, HAT_UNLOAD); 2334 2335 /* 2336 * Free unmapped, unmodified original page. 2337 * or release the lock on the original page, 2338 * otherwise the process will sleep forever in 2339 * anon_decref() waiting for the "exclusive" lock 2340 * on the page. 2341 */ 2342 (void) page_release(opp, 1); 2343 2344 /* 2345 * we are done with page creation so downgrade the new 2346 * page's selock to shared, this helps when multiple 2347 * as_fault(...SOFTLOCK...) are done to the same 2348 * page(aio) 2349 */ 2350 page_downgrade(pp); 2351 2352 /* 2353 * NOTE: The original anon slot must be freed by the 2354 * caller while holding the "anon_map" lock, if we 2355 * copied away from an anonymous page. 2356 */ 2357 return (pp); 2358 2359 out: 2360 *app = old; 2361 if (pp) 2362 page_unlock(pp); 2363 anon_decref(new); 2364 page_unlock(opp); 2365 return ((page_t *)NULL); 2366 } 2367 2368 int 2369 anon_map_privatepages( 2370 struct anon_map *amp, 2371 ulong_t start_idx, 2372 uint_t szc, 2373 struct seg *seg, 2374 caddr_t addr, 2375 uint_t prot, 2376 page_t *ppa[], 2377 struct vpage vpage[], 2378 int anypgsz, 2379 struct cred *cred) 2380 { 2381 pgcnt_t pgcnt; 2382 struct vnode *vp; 2383 anoff_t off; 2384 page_t *pl[2], *conpp = NULL; 2385 int err; 2386 int prealloc = 1; 2387 struct anon *ap, *oldap; 2388 caddr_t vaddr; 2389 page_t *pplist, *pp; 2390 ulong_t pg_idx, an_idx; 2391 spgcnt_t nreloc = 0; 2392 int pagelock = 0; 2393 kmutex_t *ahmpages = NULL; 2394 #ifdef DEBUG 2395 int refcnt; 2396 #endif 2397 2398 ASSERT(szc != 0); 2399 ASSERT(szc == seg->s_szc); 2400 2401 VM_STAT_ADD(anonvmstats.privatepages[0]); 2402 2403 pgcnt = page_get_pagecnt(szc); 2404 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 2405 ASSERT(IS_P2ALIGNED(start_idx, pgcnt)); 2406 2407 ASSERT(amp != NULL); 2408 ap = anon_get_ptr(amp->ahp, start_idx); 2409 ASSERT(ap == NULL || ap->an_refcnt >= 1); 2410 2411 VM_STAT_COND_ADD(ap == NULL, anonvmstats.privatepages[1]); 2412 2413 /* 2414 * Now try and allocate the large page. If we fail then just 2415 * let VOP_GETPAGE give us PAGESIZE pages. Normally we let 2416 * the caller make this decision but to avoid added complexity 2417 * it's simplier to handle that case here. 2418 */ 2419 if (anypgsz == -1) { 2420 VM_STAT_ADD(anonvmstats.privatepages[2]); 2421 prealloc = 0; 2422 } else if (page_alloc_pages(anon_vp, seg, addr, &pplist, NULL, szc, 2423 anypgsz) != 0) { 2424 VM_STAT_ADD(anonvmstats.privatepages[3]); 2425 prealloc = 0; 2426 } 2427 2428 /* 2429 * make the decrement of all refcnts of all 2430 * anon slots of a large page appear atomic by 2431 * getting an anonpages_hash_lock for the 2432 * first anon slot of a large page. 2433 */ 2434 if (ap != NULL) { 2435 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, 2436 ap->an_off)]; 2437 mutex_enter(ahmpages); 2438 if (ap->an_refcnt == 1) { 2439 VM_STAT_ADD(anonvmstats.privatepages[4]); 2440 ASSERT(!anon_share(amp->ahp, start_idx, pgcnt)); 2441 mutex_exit(ahmpages); 2442 2443 if (prealloc) { 2444 page_free_replacement_page(pplist); 2445 page_create_putback(pgcnt); 2446 } 2447 ASSERT(ppa[0]->p_szc <= szc); 2448 if (ppa[0]->p_szc == szc) { 2449 VM_STAT_ADD(anonvmstats.privatepages[5]); 2450 return (0); 2451 } 2452 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2453 ASSERT(ppa[pg_idx] != NULL); 2454 page_unlock(ppa[pg_idx]); 2455 } 2456 return (-1); 2457 } 2458 } 2459 2460 /* 2461 * If we are passed in the vpage array and this is 2462 * not PROT_WRITE then we need to decrement availrmem 2463 * up front before we try anything. If we need to and 2464 * can't decrement availrmem then its better to fail now 2465 * than in the middle of processing the new large page. 2466 * page_pp_usclaim() on behalf of each constituent page 2467 * below will adjust availrmem back for the cases not needed. 2468 */ 2469 if (vpage != NULL && (prot & PROT_WRITE) == 0) { 2470 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2471 if (VPP_ISPPLOCK(&vpage[pg_idx])) { 2472 pagelock = 1; 2473 break; 2474 } 2475 } 2476 if (pagelock) { 2477 VM_STAT_ADD(anonvmstats.privatepages[6]); 2478 mutex_enter(&freemem_lock); 2479 if (availrmem >= pages_pp_maximum + pgcnt) { 2480 availrmem -= pgcnt; 2481 pages_useclaim += pgcnt; 2482 } else { 2483 VM_STAT_ADD(anonvmstats.privatepages[7]); 2484 mutex_exit(&freemem_lock); 2485 if (ahmpages != NULL) { 2486 mutex_exit(ahmpages); 2487 } 2488 if (prealloc) { 2489 page_free_replacement_page(pplist); 2490 page_create_putback(pgcnt); 2491 } 2492 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) 2493 if (ppa[pg_idx] != NULL) 2494 page_unlock(ppa[pg_idx]); 2495 return (ENOMEM); 2496 } 2497 mutex_exit(&freemem_lock); 2498 } 2499 } 2500 2501 CPU_STATS_ADD_K(vm, cow_fault, pgcnt); 2502 2503 VM_STAT_ADD(anonvmstats.privatepages[8]); 2504 2505 an_idx = start_idx; 2506 pg_idx = 0; 2507 vaddr = addr; 2508 for (; pg_idx < pgcnt; pg_idx++, an_idx++, vaddr += PAGESIZE) { 2509 ASSERT(ppa[pg_idx] != NULL); 2510 oldap = anon_get_ptr(amp->ahp, an_idx); 2511 ASSERT(ahmpages != NULL || oldap == NULL); 2512 ASSERT(ahmpages == NULL || oldap != NULL); 2513 ASSERT(ahmpages == NULL || oldap->an_refcnt > 1); 2514 ASSERT(ahmpages == NULL || pg_idx != 0 || 2515 (refcnt = oldap->an_refcnt)); 2516 ASSERT(ahmpages == NULL || pg_idx == 0 || 2517 refcnt == oldap->an_refcnt); 2518 2519 ap = anon_alloc(NULL, 0); 2520 2521 swap_xlate(ap, &vp, &off); 2522 2523 /* 2524 * Now setup our preallocated page to pass down to 2525 * swap_getpage(). 2526 */ 2527 if (prealloc) { 2528 pp = pplist; 2529 page_sub(&pplist, pp); 2530 conpp = pp; 2531 } 2532 2533 err = swap_getconpage(vp, (u_offset_t)off, PAGESIZE, NULL, pl, 2534 PAGESIZE, conpp, NULL, &nreloc, seg, vaddr, 2535 S_CREATE, cred); 2536 2537 /* 2538 * Impossible to fail this is S_CREATE. 2539 */ 2540 if (err) 2541 panic("anon_map_privatepages: VOP_GETPAGE failed"); 2542 2543 ASSERT(prealloc ? pp == pl[0] : pl[0]->p_szc == 0); 2544 ASSERT(prealloc == 0 || nreloc == 1); 2545 2546 pp = pl[0]; 2547 2548 /* 2549 * If the original page was locked, we need to move 2550 * the lock to the new page by transfering 2551 * 'cowcnt/lckcnt' of the original page to 'cowcnt/lckcnt' 2552 * of the new page. pg_idx can be used to index 2553 * into the vpage array since the caller will guarentee 2554 * that vpage struct passed in corresponds to addr 2555 * and forward. 2556 */ 2557 if (vpage != NULL && VPP_ISPPLOCK(&vpage[pg_idx])) { 2558 page_pp_useclaim(ppa[pg_idx], pp, prot & PROT_WRITE); 2559 } else if (pagelock) { 2560 mutex_enter(&freemem_lock); 2561 availrmem++; 2562 pages_useclaim--; 2563 mutex_exit(&freemem_lock); 2564 } 2565 2566 /* 2567 * Now copy the contents from the original page. 2568 */ 2569 if (ppcopy(ppa[pg_idx], pp) == 0) { 2570 /* 2571 * Before ppcopy could hanlde UE or other faults, we 2572 * would have panicked here, and still have no option 2573 * but to do so now. 2574 */ 2575 panic("anon_map_privatepages, ppcopy failed"); 2576 } 2577 2578 hat_setrefmod(pp); /* mark as modified */ 2579 2580 /* 2581 * Release the lock on the original page, 2582 * derement the old slot, and down grade the lock 2583 * on the new copy. 2584 */ 2585 page_unlock(ppa[pg_idx]); 2586 2587 if (!prealloc) 2588 page_downgrade(pp); 2589 2590 ppa[pg_idx] = pp; 2591 2592 /* 2593 * Now reflect the copy in the new anon array. 2594 */ 2595 ASSERT(ahmpages == NULL || oldap->an_refcnt > 1); 2596 if (oldap != NULL) 2597 anon_decref(oldap); 2598 (void) anon_set_ptr(amp->ahp, an_idx, ap, ANON_SLEEP); 2599 } 2600 if (ahmpages != NULL) { 2601 mutex_exit(ahmpages); 2602 } 2603 ASSERT(prealloc == 0 || pplist == NULL); 2604 if (prealloc) { 2605 VM_STAT_ADD(anonvmstats.privatepages[9]); 2606 for (pg_idx = 0; pg_idx < pgcnt; pg_idx++) { 2607 page_downgrade(ppa[pg_idx]); 2608 } 2609 } 2610 2611 /* 2612 * Unload the old large page translation. 2613 */ 2614 hat_unload(seg->s_as->a_hat, addr, pgcnt << PAGESHIFT, HAT_UNLOAD); 2615 return (0); 2616 } 2617 2618 /* 2619 * Allocate a private zero-filled anon page. 2620 */ 2621 page_t * 2622 anon_zero(struct seg *seg, caddr_t addr, struct anon **app, struct cred *cred) 2623 { 2624 struct anon *ap; 2625 page_t *pp; 2626 struct vnode *vp; 2627 anoff_t off; 2628 page_t *anon_pl[1 + 1]; 2629 int err; 2630 2631 /* Kernel probe */ 2632 TNF_PROBE_1(anon_zero, "vm pagefault", /* CSTYLED */, 2633 tnf_opaque, address, addr); 2634 2635 *app = ap = anon_alloc(NULL, 0); 2636 swap_xlate(ap, &vp, &off); 2637 2638 /* 2639 * Call the VOP_GETPAGE routine to create the page, thereby 2640 * enabling the vnode driver to allocate any filesystem 2641 * dependent structures (e.g., disk block allocation for UFS). 2642 * This also prevents more than on page from being added to 2643 * the vnode at the same time since it is locked. 2644 */ 2645 err = VOP_GETPAGE(vp, off, PAGESIZE, NULL, 2646 anon_pl, PAGESIZE, seg, addr, S_CREATE, cred); 2647 if (err) { 2648 *app = NULL; 2649 anon_decref(ap); 2650 return (NULL); 2651 } 2652 pp = anon_pl[0]; 2653 2654 pagezero(pp, 0, PAGESIZE); /* XXX - should set mod bit */ 2655 page_downgrade(pp); 2656 CPU_STATS_ADD_K(vm, zfod, 1); 2657 hat_setrefmod(pp); /* mark as modified so pageout writes back */ 2658 return (pp); 2659 } 2660 2661 2662 /* 2663 * Allocate array of private zero-filled anon pages for empty slots 2664 * and kept pages for non empty slots within given range. 2665 * 2666 * NOTE: This rontine will try and use large pages 2667 * if available and supported by underlying platform. 2668 */ 2669 int 2670 anon_map_createpages( 2671 struct anon_map *amp, 2672 ulong_t start_index, 2673 size_t len, 2674 page_t *ppa[], 2675 struct seg *seg, 2676 caddr_t addr, 2677 enum seg_rw rw, 2678 struct cred *cred) 2679 { 2680 2681 struct anon *ap; 2682 struct vnode *ap_vp; 2683 page_t *pp, *pplist, *anon_pl[1 + 1], *conpp = NULL; 2684 int err = 0; 2685 ulong_t p_index, index; 2686 pgcnt_t npgs, pg_cnt; 2687 spgcnt_t nreloc = 0; 2688 uint_t l_szc, szc, prot; 2689 anoff_t ap_off; 2690 size_t pgsz; 2691 lgrp_t *lgrp; 2692 2693 /* 2694 * XXX For now only handle S_CREATE. 2695 */ 2696 ASSERT(rw == S_CREATE); 2697 2698 index = start_index; 2699 p_index = 0; 2700 npgs = btopr(len); 2701 2702 /* 2703 * If this platform supports multiple page sizes 2704 * then try and allocate directly from the free 2705 * list for pages larger than PAGESIZE. 2706 * 2707 * NOTE:When we have page_create_ru we can stop 2708 * directly allocating from the freelist. 2709 */ 2710 l_szc = seg->s_szc; 2711 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 2712 while (npgs) { 2713 2714 /* 2715 * if anon slot already exists 2716 * (means page has been created) 2717 * so 1) look up the page 2718 * 2) if the page is still in memory, get it. 2719 * 3) if not, create a page and 2720 * page in from physical swap device. 2721 * These are done in anon_getpage(). 2722 */ 2723 ap = anon_get_ptr(amp->ahp, index); 2724 if (ap) { 2725 err = anon_getpage(&ap, &prot, anon_pl, PAGESIZE, 2726 seg, addr, S_READ, cred); 2727 if (err) { 2728 ANON_LOCK_EXIT(&->a_rwlock); 2729 panic("anon_map_createpages: anon_getpage"); 2730 } 2731 pp = anon_pl[0]; 2732 ppa[p_index++] = pp; 2733 2734 addr += PAGESIZE; 2735 index++; 2736 npgs--; 2737 continue; 2738 } 2739 /* 2740 * Now try and allocate the largest page possible 2741 * for the current address and range. 2742 * Keep dropping down in page size until: 2743 * 2744 * 1) Properly aligned 2745 * 2) Does not overlap existing anon pages 2746 * 3) Fits in remaining range. 2747 * 4) able to allocate one. 2748 * 2749 * NOTE: XXX When page_create_ru is completed this code 2750 * will change. 2751 */ 2752 szc = l_szc; 2753 pplist = NULL; 2754 pg_cnt = 0; 2755 while (szc) { 2756 pgsz = page_get_pagesize(szc); 2757 pg_cnt = pgsz >> PAGESHIFT; 2758 if (IS_P2ALIGNED(addr, pgsz) && pg_cnt <= npgs && 2759 anon_pages(amp->ahp, index, pg_cnt) == 0) { 2760 /* 2761 * XXX 2762 * Since we are faking page_create() 2763 * we also need to do the freemem and 2764 * pcf accounting. 2765 */ 2766 (void) page_create_wait(pg_cnt, PG_WAIT); 2767 2768 /* 2769 * Get lgroup to allocate next page of shared 2770 * memory from and use it to specify where to 2771 * allocate the physical memory 2772 */ 2773 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2774 2775 pplist = page_get_freelist( 2776 anon_vp, (u_offset_t)0, seg, 2777 addr, pgsz, 0, lgrp); 2778 2779 if (pplist == NULL) { 2780 page_create_putback(pg_cnt); 2781 } 2782 2783 /* 2784 * If a request for a page of size 2785 * larger than PAGESIZE failed 2786 * then don't try that size anymore. 2787 */ 2788 if (pplist == NULL) { 2789 l_szc = szc - 1; 2790 } else { 2791 break; 2792 } 2793 } 2794 szc--; 2795 } 2796 2797 /* 2798 * If just using PAGESIZE pages then don't 2799 * directly allocate from the free list. 2800 */ 2801 if (pplist == NULL) { 2802 ASSERT(szc == 0); 2803 pp = anon_zero(seg, addr, &ap, cred); 2804 if (pp == NULL) { 2805 ANON_LOCK_EXIT(&->a_rwlock); 2806 panic("anon_map_createpages: anon_zero"); 2807 } 2808 ppa[p_index++] = pp; 2809 2810 ASSERT(anon_get_ptr(amp->ahp, index) == NULL); 2811 (void) anon_set_ptr(amp->ahp, index, ap, ANON_SLEEP); 2812 2813 addr += PAGESIZE; 2814 index++; 2815 npgs--; 2816 continue; 2817 } 2818 2819 /* 2820 * pplist is a list of pg_cnt PAGESIZE pages. 2821 * These pages are locked SE_EXCL since they 2822 * came directly off the free list. 2823 */ 2824 ASSERT(IS_P2ALIGNED(pg_cnt, pg_cnt)); 2825 ASSERT(IS_P2ALIGNED(index, pg_cnt)); 2826 ASSERT(conpp == NULL); 2827 while (pg_cnt--) { 2828 2829 ap = anon_alloc(NULL, 0); 2830 swap_xlate(ap, &ap_vp, &ap_off); 2831 2832 ASSERT(pplist != NULL); 2833 pp = pplist; 2834 page_sub(&pplist, pp); 2835 PP_CLRFREE(pp); 2836 PP_CLRAGED(pp); 2837 conpp = pp; 2838 2839 err = swap_getconpage(ap_vp, ap_off, PAGESIZE, 2840 (uint_t *)NULL, anon_pl, PAGESIZE, conpp, NULL, 2841 &nreloc, seg, addr, S_CREATE, cred); 2842 2843 if (err) { 2844 ANON_LOCK_EXIT(&->a_rwlock); 2845 panic("anon_map_createpages: S_CREATE"); 2846 } 2847 2848 ASSERT(anon_pl[0] == pp); 2849 ASSERT(nreloc == 1); 2850 pagezero(pp, 0, PAGESIZE); 2851 CPU_STATS_ADD_K(vm, zfod, 1); 2852 hat_setrefmod(pp); 2853 2854 ASSERT(anon_get_ptr(amp->ahp, index) == NULL); 2855 (void) anon_set_ptr(amp->ahp, index, ap, ANON_SLEEP); 2856 2857 ppa[p_index++] = pp; 2858 2859 addr += PAGESIZE; 2860 index++; 2861 npgs--; 2862 } 2863 conpp = NULL; 2864 pg_cnt = pgsz >> PAGESHIFT; 2865 p_index = p_index - pg_cnt; 2866 while (pg_cnt--) { 2867 page_downgrade(ppa[p_index++]); 2868 } 2869 } 2870 ANON_LOCK_EXIT(&->a_rwlock); 2871 return (0); 2872 } 2873 2874 static int 2875 anon_try_demote_pages( 2876 struct anon_hdr *ahp, 2877 ulong_t sidx, 2878 uint_t szc, 2879 page_t **ppa, 2880 int private) 2881 { 2882 struct anon *ap; 2883 pgcnt_t pgcnt = page_get_pagecnt(szc); 2884 page_t *pp; 2885 pgcnt_t i; 2886 kmutex_t *ahmpages = NULL; 2887 int root = 0; 2888 pgcnt_t npgs; 2889 pgcnt_t curnpgs = 0; 2890 size_t ppasize = 0; 2891 2892 ASSERT(szc != 0); 2893 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 2894 ASSERT(IS_P2ALIGNED(sidx, pgcnt)); 2895 ASSERT(sidx < ahp->size); 2896 2897 if (ppa == NULL) { 2898 ppasize = pgcnt * sizeof (page_t *); 2899 ppa = kmem_alloc(ppasize, KM_SLEEP); 2900 } 2901 2902 ap = anon_get_ptr(ahp, sidx); 2903 if (ap != NULL && private) { 2904 VM_STAT_ADD(anonvmstats.demotepages[1]); 2905 ahmpages = &anonpages_hash_lock[AH_LOCK(ap->an_vp, ap->an_off)]; 2906 mutex_enter(ahmpages); 2907 } 2908 2909 if (ap != NULL && ap->an_refcnt > 1) { 2910 if (ahmpages != NULL) { 2911 VM_STAT_ADD(anonvmstats.demotepages[2]); 2912 mutex_exit(ahmpages); 2913 } 2914 if (ppasize != 0) { 2915 kmem_free(ppa, ppasize); 2916 } 2917 return (0); 2918 } 2919 if (ahmpages != NULL) { 2920 mutex_exit(ahmpages); 2921 } 2922 if (ahp->size - sidx < pgcnt) { 2923 ASSERT(private == 0); 2924 pgcnt = ahp->size - sidx; 2925 } 2926 for (i = 0; i < pgcnt; i++, sidx++) { 2927 ap = anon_get_ptr(ahp, sidx); 2928 if (ap != NULL) { 2929 if (ap->an_refcnt != 1) { 2930 panic("anon_try_demote_pages: an_refcnt != 1"); 2931 } 2932 pp = ppa[i] = page_lookup(ap->an_vp, ap->an_off, 2933 SE_EXCL); 2934 if (pp != NULL) { 2935 (void) hat_pageunload(pp, 2936 HAT_FORCE_PGUNLOAD); 2937 } 2938 } else { 2939 ppa[i] = NULL; 2940 } 2941 } 2942 for (i = 0; i < pgcnt; i++) { 2943 if ((pp = ppa[i]) != NULL && pp->p_szc != 0) { 2944 ASSERT(pp->p_szc <= szc); 2945 if (!root) { 2946 VM_STAT_ADD(anonvmstats.demotepages[3]); 2947 if (curnpgs != 0) 2948 panic("anon_try_demote_pages: " 2949 "bad large page"); 2950 2951 root = 1; 2952 curnpgs = npgs = 2953 page_get_pagecnt(pp->p_szc); 2954 2955 ASSERT(npgs <= pgcnt); 2956 ASSERT(IS_P2ALIGNED(npgs, npgs)); 2957 ASSERT(!(page_pptonum(pp) & 2958 (npgs - 1))); 2959 } else { 2960 ASSERT(i > 0); 2961 ASSERT(page_pptonum(pp) - 1 == 2962 page_pptonum(ppa[i - 1])); 2963 if ((page_pptonum(pp) & (npgs - 1)) == 2964 npgs - 1) 2965 root = 0; 2966 } 2967 ASSERT(PAGE_EXCL(pp)); 2968 pp->p_szc = 0; 2969 ASSERT(curnpgs > 0); 2970 curnpgs--; 2971 } 2972 } 2973 if (root != 0 || curnpgs != 0) 2974 panic("anon_try_demote_pages: bad large page"); 2975 2976 for (i = 0; i < pgcnt; i++) { 2977 if ((pp = ppa[i]) != NULL) { 2978 ASSERT(!hat_page_is_mapped(pp)); 2979 ASSERT(pp->p_szc == 0); 2980 page_unlock(pp); 2981 } 2982 } 2983 if (ppasize != 0) { 2984 kmem_free(ppa, ppasize); 2985 } 2986 return (1); 2987 } 2988 2989 /* 2990 * anon_map_demotepages() can only be called by MAP_PRIVATE segments. 2991 */ 2992 int 2993 anon_map_demotepages( 2994 struct anon_map *amp, 2995 ulong_t start_idx, 2996 struct seg *seg, 2997 caddr_t addr, 2998 uint_t prot, 2999 struct vpage vpage[], 3000 struct cred *cred) 3001 { 3002 struct anon *ap; 3003 uint_t szc = seg->s_szc; 3004 pgcnt_t pgcnt = page_get_pagecnt(szc); 3005 size_t ppasize = pgcnt * sizeof (page_t *); 3006 page_t **ppa = kmem_alloc(ppasize, KM_SLEEP); 3007 page_t *pp; 3008 page_t *pl[2]; 3009 pgcnt_t i, pg_idx; 3010 ulong_t an_idx; 3011 caddr_t vaddr; 3012 int err; 3013 int retry = 0; 3014 uint_t vpprot; 3015 3016 ASSERT(RW_WRITE_HELD(&->a_rwlock)); 3017 ASSERT(IS_P2ALIGNED(pgcnt, pgcnt)); 3018 ASSERT(IS_P2ALIGNED(start_idx, pgcnt)); 3019 ASSERT(ppa != NULL); 3020 ASSERT(szc != 0); 3021 ASSERT(szc == amp->a_szc); 3022 3023 VM_STAT_ADD(anonvmstats.demotepages[0]); 3024 3025 top: 3026 if (anon_try_demote_pages(amp->ahp, start_idx, szc, ppa, 1)) { 3027 kmem_free(ppa, ppasize); 3028 return (0); 3029 } 3030 3031 VM_STAT_ADD(anonvmstats.demotepages[4]); 3032 3033 ASSERT(retry == 0); /* we can be here only once */ 3034 3035 vaddr = addr; 3036 for (pg_idx = 0, an_idx = start_idx; pg_idx < pgcnt; 3037 pg_idx++, an_idx++, vaddr += PAGESIZE) { 3038 ap = anon_get_ptr(amp->ahp, an_idx); 3039 if (ap == NULL) 3040 panic("anon_map_demotepages: no anon slot"); 3041 err = anon_getpage(&ap, &vpprot, pl, PAGESIZE, seg, vaddr, 3042 S_READ, cred); 3043 if (err) { 3044 for (i = 0; i < pg_idx; i++) { 3045 if ((pp = ppa[i]) != NULL) 3046 page_unlock(pp); 3047 } 3048 kmem_free(ppa, ppasize); 3049 return (err); 3050 } 3051 ppa[pg_idx] = pl[0]; 3052 } 3053 3054 err = anon_map_privatepages(amp, start_idx, szc, seg, addr, prot, ppa, 3055 vpage, -1, cred); 3056 if (err > 0) { 3057 VM_STAT_ADD(anonvmstats.demotepages[5]); 3058 kmem_free(ppa, ppasize); 3059 return (err); 3060 } 3061 ASSERT(err == 0 || err == -1); 3062 if (err == -1) { 3063 VM_STAT_ADD(anonvmstats.demotepages[6]); 3064 retry = 1; 3065 goto top; 3066 } 3067 for (i = 0; i < pgcnt; i++) { 3068 ASSERT(ppa[i] != NULL); 3069 if (ppa[i]->p_szc != 0) 3070 retry = 1; 3071 page_unlock(ppa[i]); 3072 } 3073 if (retry) { 3074 VM_STAT_ADD(anonvmstats.demotepages[7]); 3075 goto top; 3076 } 3077 3078 VM_STAT_ADD(anonvmstats.demotepages[8]); 3079 3080 kmem_free(ppa, ppasize); 3081 3082 return (0); 3083 } 3084 3085 /* 3086 * Free pages of shared anon map. It's assumed that anon maps don't share anon 3087 * structures with private anon maps. Therefore all anon structures should 3088 * have at most one reference at this point. This means underlying pages can 3089 * be exclusively locked and demoted or freed. If not freeing the entire 3090 * large pages demote the ends of the region we free to be able to free 3091 * subpages. Page roots correspend to aligned index positions in anon map. 3092 */ 3093 void 3094 anon_shmap_free_pages(struct anon_map *amp, ulong_t sidx, size_t len) 3095 { 3096 ulong_t eidx = sidx + btopr(len); 3097 pgcnt_t pages = page_get_pagecnt(amp->a_szc); 3098 struct anon_hdr *ahp = amp->ahp; 3099 ulong_t tidx; 3100 size_t size; 3101 ulong_t sidx_aligned; 3102 ulong_t eidx_aligned; 3103 3104 ASSERT(RW_WRITE_HELD(&->a_rwlock)); 3105 ASSERT(amp->refcnt <= 1); 3106 ASSERT(amp->a_szc > 0); 3107 ASSERT(eidx <= ahp->size); 3108 ASSERT(!anon_share(ahp, sidx, btopr(len))); 3109 3110 if (len == 0) { /* XXX */ 3111 return; 3112 } 3113 3114 sidx_aligned = P2ALIGN(sidx, pages); 3115 if (sidx_aligned != sidx || 3116 (eidx < sidx_aligned + pages && eidx < ahp->size)) { 3117 if (!anon_try_demote_pages(ahp, sidx_aligned, 3118 amp->a_szc, NULL, 0)) { 3119 panic("anon_shmap_free_pages: demote failed"); 3120 } 3121 size = (eidx <= sidx_aligned + pages) ? (eidx - sidx) : 3122 P2NPHASE(sidx, pages); 3123 size <<= PAGESHIFT; 3124 anon_free(ahp, sidx, size); 3125 sidx = sidx_aligned + pages; 3126 if (eidx <= sidx) { 3127 return; 3128 } 3129 } 3130 eidx_aligned = P2ALIGN(eidx, pages); 3131 if (sidx < eidx_aligned) { 3132 anon_free_pages(ahp, sidx, 3133 (eidx_aligned - sidx) << PAGESHIFT, 3134 amp->a_szc); 3135 sidx = eidx_aligned; 3136 } 3137 ASSERT(sidx == eidx_aligned); 3138 if (eidx == eidx_aligned) { 3139 return; 3140 } 3141 tidx = eidx; 3142 if (eidx != ahp->size && anon_get_next_ptr(ahp, &tidx) != NULL && 3143 tidx - sidx < pages) { 3144 if (!anon_try_demote_pages(ahp, sidx, amp->a_szc, NULL, 0)) { 3145 panic("anon_shmap_free_pages: demote failed"); 3146 } 3147 size = (eidx - sidx) << PAGESHIFT; 3148 anon_free(ahp, sidx, size); 3149 } else { 3150 anon_free_pages(ahp, sidx, pages << PAGESHIFT, amp->a_szc); 3151 } 3152 } 3153 3154 /* 3155 * Allocate and initialize an anon_map structure for seg 3156 * associating the given swap reservation with the new anon_map. 3157 */ 3158 struct anon_map * 3159 anonmap_alloc(size_t size, size_t swresv) 3160 { 3161 struct anon_map *amp; 3162 3163 amp = kmem_cache_alloc(anonmap_cache, KM_SLEEP); 3164 3165 amp->refcnt = 1; 3166 amp->size = size; 3167 3168 amp->ahp = anon_create(btopr(size), ANON_SLEEP); 3169 amp->swresv = swresv; 3170 amp->locality = 0; 3171 amp->a_szc = 0; 3172 amp->a_sp = NULL; 3173 return (amp); 3174 } 3175 3176 void 3177 anonmap_free(struct anon_map *amp) 3178 { 3179 ASSERT(amp->ahp); 3180 ASSERT(amp->refcnt == 0); 3181 3182 lgrp_shm_policy_fini(amp, NULL); 3183 anon_release(amp->ahp, btopr(amp->size)); 3184 kmem_cache_free(anonmap_cache, amp); 3185 } 3186 3187 /* 3188 * Returns true if the app array has some empty slots. 3189 * The offp and lenp paramters are in/out paramters. On entry 3190 * these values represent the starting offset and length of the 3191 * mapping. When true is returned, these values may be modified 3192 * to be the largest range which includes empty slots. 3193 */ 3194 int 3195 non_anon(struct anon_hdr *ahp, ulong_t anon_idx, u_offset_t *offp, 3196 size_t *lenp) 3197 { 3198 ulong_t i, el; 3199 ssize_t low, high; 3200 struct anon *ap; 3201 3202 low = -1; 3203 for (i = 0, el = *lenp; i < el; i += PAGESIZE, anon_idx++) { 3204 ap = anon_get_ptr(ahp, anon_idx); 3205 if (ap == NULL) { 3206 if (low == -1) 3207 low = i; 3208 high = i; 3209 } 3210 } 3211 if (low != -1) { 3212 /* 3213 * Found at least one non-anon page. 3214 * Set up the off and len return values. 3215 */ 3216 if (low != 0) 3217 *offp += low; 3218 *lenp = high - low + PAGESIZE; 3219 return (1); 3220 } 3221 return (0); 3222 } 3223 3224 /* 3225 * Return a count of the number of existing anon pages in the anon array 3226 * app in the range (off, off+len). The array and slots must be guaranteed 3227 * stable by the caller. 3228 */ 3229 pgcnt_t 3230 anon_pages(struct anon_hdr *ahp, ulong_t anon_index, pgcnt_t nslots) 3231 { 3232 pgcnt_t cnt = 0; 3233 3234 while (nslots-- > 0) { 3235 if ((anon_get_ptr(ahp, anon_index)) != NULL) 3236 cnt++; 3237 anon_index++; 3238 } 3239 return (cnt); 3240 } 3241 3242 /* 3243 * Move reserved phys swap into memory swap (unreserve phys swap 3244 * and reserve mem swap by the same amount). 3245 * Used by segspt when it needs to lock resrved swap npages in memory 3246 */ 3247 int 3248 anon_swap_adjust(pgcnt_t npages) 3249 { 3250 pgcnt_t unlocked_mem_swap; 3251 3252 mutex_enter(&anoninfo_lock); 3253 3254 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 3255 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 3256 3257 unlocked_mem_swap = k_anoninfo.ani_mem_resv 3258 - k_anoninfo.ani_locked_swap; 3259 if (npages > unlocked_mem_swap) { 3260 spgcnt_t adjusted_swap = npages - unlocked_mem_swap; 3261 3262 /* 3263 * if there is not enough unlocked mem swap we take missing 3264 * amount from phys swap and give it to mem swap 3265 */ 3266 if (!page_reclaim_mem(adjusted_swap, segspt_minfree, 1)) { 3267 mutex_exit(&anoninfo_lock); 3268 return (ENOMEM); 3269 } 3270 3271 k_anoninfo.ani_mem_resv += adjusted_swap; 3272 ASSERT(k_anoninfo.ani_phys_resv >= adjusted_swap); 3273 k_anoninfo.ani_phys_resv -= adjusted_swap; 3274 3275 ANI_ADD(adjusted_swap); 3276 } 3277 k_anoninfo.ani_locked_swap += npages; 3278 3279 ASSERT(k_anoninfo.ani_mem_resv >= k_anoninfo.ani_locked_swap); 3280 ASSERT(k_anoninfo.ani_max >= k_anoninfo.ani_phys_resv); 3281 3282 mutex_exit(&anoninfo_lock); 3283 3284 return (0); 3285 } 3286 3287 /* 3288 * 'unlocked' reserved mem swap so when it is unreserved it 3289 * can be moved back phys (disk) swap 3290 */ 3291 void 3292 anon_swap_restore(pgcnt_t npages) 3293 { 3294 mutex_enter(&anoninfo_lock); 3295 3296 ASSERT(k_anoninfo.ani_locked_swap <= k_anoninfo.ani_mem_resv); 3297 3298 ASSERT(k_anoninfo.ani_locked_swap >= npages); 3299 k_anoninfo.ani_locked_swap -= npages; 3300 3301 ASSERT(k_anoninfo.ani_locked_swap <= k_anoninfo.ani_mem_resv); 3302 3303 mutex_exit(&anoninfo_lock); 3304 } 3305 3306 /* 3307 * Return the pointer from the list for a 3308 * specified anon index. 3309 */ 3310 ulong_t * 3311 anon_get_slot(struct anon_hdr *ahp, ulong_t an_idx) 3312 { 3313 struct anon **app; 3314 void **ppp; 3315 3316 ASSERT(an_idx < ahp->size); 3317 3318 /* 3319 * Single level case. 3320 */ 3321 if ((ahp->size <= ANON_CHUNK_SIZE) || (ahp->flags & ANON_ALLOC_FORCE)) { 3322 return ((ulong_t *)&ahp->array_chunk[an_idx]); 3323 } else { 3324 3325 /* 3326 * 2 level case. 3327 */ 3328 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 3329 if (*ppp == NULL) { 3330 mutex_enter(&ahp->serial_lock); 3331 ppp = &ahp->array_chunk[an_idx >> ANON_CHUNK_SHIFT]; 3332 if (*ppp == NULL) 3333 *ppp = kmem_zalloc(PAGESIZE, KM_SLEEP); 3334 mutex_exit(&ahp->serial_lock); 3335 } 3336 app = *ppp; 3337 return ((ulong_t *)&app[an_idx & ANON_CHUNK_OFF]); 3338 } 3339 } 3340 3341 void 3342 anon_array_enter(struct anon_map *amp, ulong_t an_idx, anon_sync_obj_t *sobj) 3343 { 3344 ulong_t *ap_slot; 3345 kmutex_t *mtx; 3346 kcondvar_t *cv; 3347 int hash; 3348 3349 /* 3350 * Use szc to determine anon slot(s) to appear atomic. 3351 * If szc = 0, then lock the anon slot and mark it busy. 3352 * If szc > 0, then lock the range of slots by getting the 3353 * anon_array_lock for the first anon slot, and mark only the 3354 * first anon slot busy to represent whole range being busy. 3355 */ 3356 3357 ASSERT(RW_READ_HELD(&->a_rwlock)); 3358 an_idx = P2ALIGN(an_idx, page_get_pagecnt(amp->a_szc)); 3359 hash = ANON_ARRAY_HASH(amp, an_idx); 3360 sobj->sync_mutex = mtx = &anon_array_lock[hash].pad_mutex; 3361 sobj->sync_cv = cv = &anon_array_cv[hash]; 3362 mutex_enter(mtx); 3363 ap_slot = anon_get_slot(amp->ahp, an_idx); 3364 while (ANON_ISBUSY(ap_slot)) 3365 cv_wait(cv, mtx); 3366 ANON_SETBUSY(ap_slot); 3367 sobj->sync_data = ap_slot; 3368 mutex_exit(mtx); 3369 } 3370 3371 int 3372 anon_array_try_enter(struct anon_map *amp, ulong_t an_idx, 3373 anon_sync_obj_t *sobj) 3374 { 3375 ulong_t *ap_slot; 3376 kmutex_t *mtx; 3377 int hash; 3378 3379 /* 3380 * Try to lock a range of anon slots. 3381 * Use szc to determine anon slot(s) to appear atomic. 3382 * If szc = 0, then lock the anon slot and mark it busy. 3383 * If szc > 0, then lock the range of slots by getting the 3384 * anon_array_lock for the first anon slot, and mark only the 3385 * first anon slot busy to represent whole range being busy. 3386 * Fail if the mutex or the anon_array are busy. 3387 */ 3388 3389 ASSERT(RW_READ_HELD(&->a_rwlock)); 3390 an_idx = P2ALIGN(an_idx, page_get_pagecnt(amp->a_szc)); 3391 hash = ANON_ARRAY_HASH(amp, an_idx); 3392 sobj->sync_mutex = mtx = &anon_array_lock[hash].pad_mutex; 3393 sobj->sync_cv = &anon_array_cv[hash]; 3394 if (!mutex_tryenter(mtx)) { 3395 return (EWOULDBLOCK); 3396 } 3397 ap_slot = anon_get_slot(amp->ahp, an_idx); 3398 if (ANON_ISBUSY(ap_slot)) { 3399 mutex_exit(mtx); 3400 return (EWOULDBLOCK); 3401 } 3402 ANON_SETBUSY(ap_slot); 3403 sobj->sync_data = ap_slot; 3404 mutex_exit(mtx); 3405 return (0); 3406 } 3407 3408 void 3409 anon_array_exit(anon_sync_obj_t *sobj) 3410 { 3411 mutex_enter(sobj->sync_mutex); 3412 ASSERT(ANON_ISBUSY(sobj->sync_data)); 3413 ANON_CLRBUSY(sobj->sync_data); 3414 if (CV_HAS_WAITERS(sobj->sync_cv)) 3415 cv_broadcast(sobj->sync_cv); 3416 mutex_exit(sobj->sync_mutex); 3417 } 3418