1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/param.h> 26 #include <sys/user.h> 27 #include <sys/mman.h> 28 #include <sys/kmem.h> 29 #include <sys/sysmacros.h> 30 #include <sys/cmn_err.h> 31 #include <sys/systm.h> 32 #include <sys/tuneable.h> 33 #include <vm/hat.h> 34 #include <vm/seg.h> 35 #include <vm/as.h> 36 #include <vm/anon.h> 37 #include <vm/page.h> 38 #include <sys/buf.h> 39 #include <sys/swap.h> 40 #include <sys/atomic.h> 41 #include <vm/seg_spt.h> 42 #include <sys/debug.h> 43 #include <sys/vtrace.h> 44 #include <sys/shm.h> 45 #include <sys/shm_impl.h> 46 #include <sys/lgrp.h> 47 #include <sys/vmsystm.h> 48 #include <sys/policy.h> 49 #include <sys/project.h> 50 #include <sys/tnf_probe.h> 51 #include <sys/zone.h> 52 53 #define SEGSPTADDR (caddr_t)0x0 54 55 /* 56 * # pages used for spt 57 */ 58 size_t spt_used; 59 60 /* 61 * segspt_minfree is the memory left for system after ISM 62 * locked its pages; it is set up to 5% of availrmem in 63 * sptcreate when ISM is created. ISM should not use more 64 * than ~90% of availrmem; if it does, then the performance 65 * of the system may decrease. Machines with large memories may 66 * be able to use up more memory for ISM so we set the default 67 * segspt_minfree to 5% (which gives ISM max 95% of availrmem. 68 * If somebody wants even more memory for ISM (risking hanging 69 * the system) they can patch the segspt_minfree to smaller number. 70 */ 71 pgcnt_t segspt_minfree = 0; 72 73 static int segspt_create(struct seg *seg, caddr_t argsp); 74 static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize); 75 static void segspt_free(struct seg *seg); 76 static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len); 77 static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr); 78 79 static void 80 segspt_badop() 81 { 82 panic("segspt_badop called"); 83 /*NOTREACHED*/ 84 } 85 86 #define SEGSPT_BADOP(t) (t(*)())segspt_badop 87 88 struct seg_ops segspt_ops = { 89 SEGSPT_BADOP(int), /* dup */ 90 segspt_unmap, 91 segspt_free, 92 SEGSPT_BADOP(int), /* fault */ 93 SEGSPT_BADOP(faultcode_t), /* faulta */ 94 SEGSPT_BADOP(int), /* setprot */ 95 SEGSPT_BADOP(int), /* checkprot */ 96 SEGSPT_BADOP(int), /* kluster */ 97 SEGSPT_BADOP(size_t), /* swapout */ 98 SEGSPT_BADOP(int), /* sync */ 99 SEGSPT_BADOP(size_t), /* incore */ 100 SEGSPT_BADOP(int), /* lockop */ 101 SEGSPT_BADOP(int), /* getprot */ 102 SEGSPT_BADOP(u_offset_t), /* getoffset */ 103 SEGSPT_BADOP(int), /* gettype */ 104 SEGSPT_BADOP(int), /* getvp */ 105 SEGSPT_BADOP(int), /* advise */ 106 SEGSPT_BADOP(void), /* dump */ 107 SEGSPT_BADOP(int), /* pagelock */ 108 SEGSPT_BADOP(int), /* setpgsz */ 109 SEGSPT_BADOP(int), /* getmemid */ 110 segspt_getpolicy, /* getpolicy */ 111 SEGSPT_BADOP(int), /* capable */ 112 }; 113 114 static int segspt_shmdup(struct seg *seg, struct seg *newseg); 115 static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize); 116 static void segspt_shmfree(struct seg *seg); 117 static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg, 118 caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); 119 static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr); 120 static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr, 121 register size_t len, register uint_t prot); 122 static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, 123 uint_t prot); 124 static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta); 125 static size_t segspt_shmswapout(struct seg *seg); 126 static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, 127 register char *vec); 128 static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len, 129 int attr, uint_t flags); 130 static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 131 int attr, int op, ulong_t *lockmap, size_t pos); 132 static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, 133 uint_t *protv); 134 static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr); 135 static int segspt_shmgettype(struct seg *seg, caddr_t addr); 136 static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 137 static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, 138 uint_t behav); 139 static void segspt_shmdump(struct seg *seg); 140 static int segspt_shmpagelock(struct seg *, caddr_t, size_t, 141 struct page ***, enum lock_type, enum seg_rw); 142 static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t); 143 static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *); 144 static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t); 145 static int segspt_shmcapable(struct seg *, segcapability_t); 146 147 struct seg_ops segspt_shmops = { 148 segspt_shmdup, 149 segspt_shmunmap, 150 segspt_shmfree, 151 segspt_shmfault, 152 segspt_shmfaulta, 153 segspt_shmsetprot, 154 segspt_shmcheckprot, 155 segspt_shmkluster, 156 segspt_shmswapout, 157 segspt_shmsync, 158 segspt_shmincore, 159 segspt_shmlockop, 160 segspt_shmgetprot, 161 segspt_shmgetoffset, 162 segspt_shmgettype, 163 segspt_shmgetvp, 164 segspt_shmadvise, /* advise */ 165 segspt_shmdump, 166 segspt_shmpagelock, 167 segspt_shmsetpgsz, 168 segspt_shmgetmemid, 169 segspt_shmgetpolicy, 170 segspt_shmcapable, 171 }; 172 173 static void segspt_purge(struct seg *seg); 174 static int segspt_reclaim(void *, caddr_t, size_t, struct page **, 175 enum seg_rw, int); 176 static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len, 177 page_t **ppa); 178 179 180 181 /*ARGSUSED*/ 182 int 183 sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp, 184 uint_t prot, uint_t flags, uint_t share_szc) 185 { 186 int err; 187 struct as *newas; 188 struct segspt_crargs sptcargs; 189 190 #ifdef DEBUG 191 TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */, 192 tnf_ulong, size, size ); 193 #endif 194 if (segspt_minfree == 0) /* leave min 5% of availrmem for */ 195 segspt_minfree = availrmem/20; /* for the system */ 196 197 if (!hat_supported(HAT_SHARED_PT, (void *)0)) 198 return (EINVAL); 199 200 /* 201 * get a new as for this shared memory segment 202 */ 203 newas = as_alloc(); 204 newas->a_proc = NULL; 205 sptcargs.amp = amp; 206 sptcargs.prot = prot; 207 sptcargs.flags = flags; 208 sptcargs.szc = share_szc; 209 /* 210 * create a shared page table (spt) segment 211 */ 212 213 if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) { 214 as_free(newas); 215 return (err); 216 } 217 *sptseg = sptcargs.seg_spt; 218 return (0); 219 } 220 221 void 222 sptdestroy(struct as *as, struct anon_map *amp) 223 { 224 225 #ifdef DEBUG 226 TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */); 227 #endif 228 (void) as_unmap(as, SEGSPTADDR, amp->size); 229 as_free(as); 230 } 231 232 /* 233 * called from seg_free(). 234 * free (i.e., unlock, unmap, return to free list) 235 * all the pages in the given seg. 236 */ 237 void 238 segspt_free(struct seg *seg) 239 { 240 struct spt_data *sptd = (struct spt_data *)seg->s_data; 241 242 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 243 244 if (sptd != NULL) { 245 if (sptd->spt_realsize) 246 segspt_free_pages(seg, seg->s_base, sptd->spt_realsize); 247 248 if (sptd->spt_ppa_lckcnt) 249 kmem_free(sptd->spt_ppa_lckcnt, 250 sizeof (*sptd->spt_ppa_lckcnt) 251 * btopr(sptd->spt_amp->size)); 252 kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp)); 253 cv_destroy(&sptd->spt_cv); 254 mutex_destroy(&sptd->spt_lock); 255 kmem_free(sptd, sizeof (*sptd)); 256 } 257 } 258 259 /*ARGSUSED*/ 260 static int 261 segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr, 262 uint_t flags) 263 { 264 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 265 266 return (0); 267 } 268 269 /*ARGSUSED*/ 270 static size_t 271 segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec) 272 { 273 caddr_t eo_seg; 274 pgcnt_t npages; 275 struct shm_data *shmd = (struct shm_data *)seg->s_data; 276 struct seg *sptseg; 277 struct spt_data *sptd; 278 279 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 280 #ifdef lint 281 seg = seg; 282 #endif 283 sptseg = shmd->shm_sptseg; 284 sptd = sptseg->s_data; 285 286 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 287 eo_seg = addr + len; 288 while (addr < eo_seg) { 289 /* page exists, and it's locked. */ 290 *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED | 291 SEG_PAGE_ANON; 292 addr += PAGESIZE; 293 } 294 return (len); 295 } else { 296 struct anon_map *amp = shmd->shm_amp; 297 struct anon *ap; 298 page_t *pp; 299 pgcnt_t anon_index; 300 struct vnode *vp; 301 u_offset_t off; 302 ulong_t i; 303 int ret; 304 anon_sync_obj_t cookie; 305 306 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 307 anon_index = seg_page(seg, addr); 308 npages = btopr(len); 309 if (anon_index + npages > btopr(shmd->shm_amp->size)) { 310 return (EINVAL); 311 } 312 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 313 for (i = 0; i < npages; i++, anon_index++) { 314 ret = 0; 315 anon_array_enter(amp, anon_index, &cookie); 316 ap = anon_get_ptr(amp->ahp, anon_index); 317 if (ap != NULL) { 318 swap_xlate(ap, &vp, &off); 319 anon_array_exit(&cookie); 320 pp = page_lookup_nowait(vp, off, SE_SHARED); 321 if (pp != NULL) { 322 ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON; 323 page_unlock(pp); 324 } 325 } else { 326 anon_array_exit(&cookie); 327 } 328 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 329 ret |= SEG_PAGE_LOCKED; 330 } 331 *vec++ = (char)ret; 332 } 333 ANON_LOCK_EXIT(&->a_rwlock); 334 return (len); 335 } 336 } 337 338 static int 339 segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize) 340 { 341 size_t share_size; 342 343 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 344 345 /* 346 * seg.s_size may have been rounded up to the largest page size 347 * in shmat(). 348 * XXX This should be cleanedup. sptdestroy should take a length 349 * argument which should be the same as sptcreate. Then 350 * this rounding would not be needed (or is done in shm.c) 351 * Only the check for full segment will be needed. 352 * 353 * XXX -- shouldn't raddr == 0 always? These tests don't seem 354 * to be useful at all. 355 */ 356 share_size = page_get_pagesize(seg->s_szc); 357 ssize = P2ROUNDUP(ssize, share_size); 358 359 if (raddr == seg->s_base && ssize == seg->s_size) { 360 seg_free(seg); 361 return (0); 362 } else 363 return (EINVAL); 364 } 365 366 int 367 segspt_create(struct seg *seg, caddr_t argsp) 368 { 369 int err; 370 caddr_t addr = seg->s_base; 371 struct spt_data *sptd; 372 struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp; 373 struct anon_map *amp = sptcargs->amp; 374 struct kshmid *sp = amp->a_sp; 375 struct cred *cred = CRED(); 376 ulong_t i, j, anon_index = 0; 377 pgcnt_t npages = btopr(amp->size); 378 struct vnode *vp; 379 page_t **ppa; 380 uint_t hat_flags; 381 size_t pgsz; 382 pgcnt_t pgcnt; 383 caddr_t a; 384 pgcnt_t pidx; 385 size_t sz; 386 proc_t *procp = curproc; 387 rctl_qty_t lockedbytes = 0; 388 kproject_t *proj; 389 390 /* 391 * We are holding the a_lock on the underlying dummy as, 392 * so we can make calls to the HAT layer. 393 */ 394 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 395 ASSERT(sp != NULL); 396 397 #ifdef DEBUG 398 TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */, 399 tnf_opaque, addr, addr, tnf_ulong, len, seg->s_size); 400 #endif 401 if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 402 if (err = anon_swap_adjust(npages)) 403 return (err); 404 } 405 err = ENOMEM; 406 407 if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL) 408 goto out1; 409 410 if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 411 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages), 412 KM_NOSLEEP)) == NULL) 413 goto out2; 414 } 415 416 mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL); 417 418 if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL) 419 goto out3; 420 421 seg->s_ops = &segspt_ops; 422 sptd->spt_vp = vp; 423 sptd->spt_amp = amp; 424 sptd->spt_prot = sptcargs->prot; 425 sptd->spt_flags = sptcargs->flags; 426 seg->s_data = (caddr_t)sptd; 427 sptd->spt_ppa = NULL; 428 sptd->spt_ppa_lckcnt = NULL; 429 seg->s_szc = sptcargs->szc; 430 cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL); 431 sptd->spt_gen = 0; 432 433 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 434 if (seg->s_szc > amp->a_szc) { 435 amp->a_szc = seg->s_szc; 436 } 437 ANON_LOCK_EXIT(&->a_rwlock); 438 439 /* 440 * Set policy to affect initial allocation of pages in 441 * anon_map_createpages() 442 */ 443 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index, 444 NULL, 0, ptob(npages)); 445 446 if (sptcargs->flags & SHM_PAGEABLE) { 447 size_t share_sz; 448 pgcnt_t new_npgs, more_pgs; 449 struct anon_hdr *nahp; 450 zone_t *zone; 451 452 share_sz = page_get_pagesize(seg->s_szc); 453 if (!IS_P2ALIGNED(amp->size, share_sz)) { 454 /* 455 * We are rounding up the size of the anon array 456 * on 4 M boundary because we always create 4 M 457 * of page(s) when locking, faulting pages and we 458 * don't have to check for all corner cases e.g. 459 * if there is enough space to allocate 4 M 460 * page. 461 */ 462 new_npgs = btop(P2ROUNDUP(amp->size, share_sz)); 463 more_pgs = new_npgs - npages; 464 465 /* 466 * The zone will never be NULL, as a fully created 467 * shm always has an owning zone. 468 */ 469 zone = sp->shm_perm.ipc_zone; 470 ASSERT(zone != NULL); 471 if (anon_resv_zone(ptob(more_pgs), zone) == 0) { 472 err = ENOMEM; 473 goto out4; 474 } 475 476 nahp = anon_create(new_npgs, ANON_SLEEP); 477 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 478 (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages, 479 ANON_SLEEP); 480 anon_release(amp->ahp, npages); 481 amp->ahp = nahp; 482 ASSERT(amp->swresv == ptob(npages)); 483 amp->swresv = amp->size = ptob(new_npgs); 484 ANON_LOCK_EXIT(&->a_rwlock); 485 npages = new_npgs; 486 } 487 488 sptd->spt_ppa_lckcnt = kmem_zalloc(npages * 489 sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP); 490 sptd->spt_pcachecnt = 0; 491 sptd->spt_realsize = ptob(npages); 492 sptcargs->seg_spt = seg; 493 return (0); 494 } 495 496 /* 497 * get array of pages for each anon slot in amp 498 */ 499 if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa, 500 seg, addr, S_CREATE, cred)) != 0) 501 goto out4; 502 503 mutex_enter(&sp->shm_mlock); 504 505 /* May be partially locked, so, count bytes to charge for locking */ 506 for (i = 0; i < npages; i++) 507 if (ppa[i]->p_lckcnt == 0) 508 lockedbytes += PAGESIZE; 509 510 proj = sp->shm_perm.ipc_proj; 511 512 if (lockedbytes > 0) { 513 mutex_enter(&procp->p_lock); 514 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) { 515 mutex_exit(&procp->p_lock); 516 mutex_exit(&sp->shm_mlock); 517 for (i = 0; i < npages; i++) 518 page_unlock(ppa[i]); 519 err = ENOMEM; 520 goto out4; 521 } 522 mutex_exit(&procp->p_lock); 523 } 524 525 /* 526 * addr is initial address corresponding to the first page on ppa list 527 */ 528 for (i = 0; i < npages; i++) { 529 /* attempt to lock all pages */ 530 if (page_pp_lock(ppa[i], 0, 1) == 0) { 531 /* 532 * if unable to lock any page, unlock all 533 * of them and return error 534 */ 535 for (j = 0; j < i; j++) 536 page_pp_unlock(ppa[j], 0, 1); 537 for (i = 0; i < npages; i++) 538 page_unlock(ppa[i]); 539 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0); 540 mutex_exit(&sp->shm_mlock); 541 err = ENOMEM; 542 goto out4; 543 } 544 } 545 mutex_exit(&sp->shm_mlock); 546 547 /* 548 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 549 * for the entire life of the segment. For example platforms 550 * that do not support Dynamic Reconfiguration. 551 */ 552 hat_flags = HAT_LOAD_SHARE; 553 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) 554 hat_flags |= HAT_LOAD_LOCK; 555 556 /* 557 * Load translations one lare page at a time 558 * to make sure we don't create mappings bigger than 559 * segment's size code in case underlying pages 560 * are shared with segvn's segment that uses bigger 561 * size code than we do. 562 */ 563 pgsz = page_get_pagesize(seg->s_szc); 564 pgcnt = page_get_pagecnt(seg->s_szc); 565 for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) { 566 sz = MIN(pgsz, ptob(npages - pidx)); 567 hat_memload_array(seg->s_as->a_hat, a, sz, 568 &ppa[pidx], sptd->spt_prot, hat_flags); 569 } 570 571 /* 572 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 573 * we will leave the pages locked SE_SHARED for the life 574 * of the ISM segment. This will prevent any calls to 575 * hat_pageunload() on this ISM segment for those platforms. 576 */ 577 if (!(hat_flags & HAT_LOAD_LOCK)) { 578 /* 579 * On platforms that support HAT_DYNAMIC_ISM_UNMAP, 580 * we no longer need to hold the SE_SHARED lock on the pages, 581 * since L_PAGELOCK and F_SOFTLOCK calls will grab the 582 * SE_SHARED lock on the pages as necessary. 583 */ 584 for (i = 0; i < npages; i++) 585 page_unlock(ppa[i]); 586 } 587 sptd->spt_pcachecnt = 0; 588 kmem_free(ppa, ((sizeof (page_t *)) * npages)); 589 sptd->spt_realsize = ptob(npages); 590 atomic_add_long(&spt_used, npages); 591 sptcargs->seg_spt = seg; 592 return (0); 593 594 out4: 595 seg->s_data = NULL; 596 kmem_free(vp, sizeof (*vp)); 597 cv_destroy(&sptd->spt_cv); 598 out3: 599 mutex_destroy(&sptd->spt_lock); 600 if ((sptcargs->flags & SHM_PAGEABLE) == 0) 601 kmem_free(ppa, (sizeof (*ppa) * npages)); 602 out2: 603 kmem_free(sptd, sizeof (*sptd)); 604 out1: 605 if ((sptcargs->flags & SHM_PAGEABLE) == 0) 606 anon_swap_restore(npages); 607 return (err); 608 } 609 610 /*ARGSUSED*/ 611 void 612 segspt_free_pages(struct seg *seg, caddr_t addr, size_t len) 613 { 614 struct page *pp; 615 struct spt_data *sptd = (struct spt_data *)seg->s_data; 616 pgcnt_t npages; 617 ulong_t anon_idx; 618 struct anon_map *amp; 619 struct anon *ap; 620 struct vnode *vp; 621 u_offset_t off; 622 uint_t hat_flags; 623 int root = 0; 624 pgcnt_t pgs, curnpgs = 0; 625 page_t *rootpp; 626 rctl_qty_t unlocked_bytes = 0; 627 kproject_t *proj; 628 kshmid_t *sp; 629 630 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 631 632 len = P2ROUNDUP(len, PAGESIZE); 633 634 npages = btop(len); 635 636 hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP; 637 if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) || 638 (sptd->spt_flags & SHM_PAGEABLE)) { 639 hat_flags = HAT_UNLOAD_UNMAP; 640 } 641 642 hat_unload(seg->s_as->a_hat, addr, len, hat_flags); 643 644 amp = sptd->spt_amp; 645 if (sptd->spt_flags & SHM_PAGEABLE) 646 npages = btop(amp->size); 647 648 ASSERT(amp != NULL); 649 650 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 651 sp = amp->a_sp; 652 proj = sp->shm_perm.ipc_proj; 653 mutex_enter(&sp->shm_mlock); 654 } 655 for (anon_idx = 0; anon_idx < npages; anon_idx++) { 656 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 657 if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) { 658 panic("segspt_free_pages: null app"); 659 /*NOTREACHED*/ 660 } 661 } else { 662 if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx)) 663 == NULL) 664 continue; 665 } 666 ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0); 667 swap_xlate(ap, &vp, &off); 668 669 /* 670 * If this platform supports HAT_DYNAMIC_ISM_UNMAP, 671 * the pages won't be having SE_SHARED lock at this 672 * point. 673 * 674 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 675 * the pages are still held SE_SHARED locked from the 676 * original segspt_create() 677 * 678 * Our goal is to get SE_EXCL lock on each page, remove 679 * permanent lock on it and invalidate the page. 680 */ 681 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 682 if (hat_flags == HAT_UNLOAD_UNMAP) 683 pp = page_lookup(vp, off, SE_EXCL); 684 else { 685 if ((pp = page_find(vp, off)) == NULL) { 686 panic("segspt_free_pages: " 687 "page not locked"); 688 /*NOTREACHED*/ 689 } 690 if (!page_tryupgrade(pp)) { 691 page_unlock(pp); 692 pp = page_lookup(vp, off, SE_EXCL); 693 } 694 } 695 if (pp == NULL) { 696 panic("segspt_free_pages: " 697 "page not in the system"); 698 /*NOTREACHED*/ 699 } 700 ASSERT(pp->p_lckcnt > 0); 701 page_pp_unlock(pp, 0, 1); 702 if (pp->p_lckcnt == 0) 703 unlocked_bytes += PAGESIZE; 704 } else { 705 if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL) 706 continue; 707 } 708 /* 709 * It's logical to invalidate the pages here as in most cases 710 * these were created by segspt. 711 */ 712 if (pp->p_szc != 0) { 713 if (root == 0) { 714 ASSERT(curnpgs == 0); 715 root = 1; 716 rootpp = pp; 717 pgs = curnpgs = page_get_pagecnt(pp->p_szc); 718 ASSERT(pgs > 1); 719 ASSERT(IS_P2ALIGNED(pgs, pgs)); 720 ASSERT(!(page_pptonum(pp) & (pgs - 1))); 721 curnpgs--; 722 } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) { 723 ASSERT(curnpgs == 1); 724 ASSERT(page_pptonum(pp) == 725 page_pptonum(rootpp) + (pgs - 1)); 726 page_destroy_pages(rootpp); 727 root = 0; 728 curnpgs = 0; 729 } else { 730 ASSERT(curnpgs > 1); 731 ASSERT(page_pptonum(pp) == 732 page_pptonum(rootpp) + (pgs - curnpgs)); 733 curnpgs--; 734 } 735 } else { 736 if (root != 0 || curnpgs != 0) { 737 panic("segspt_free_pages: bad large page"); 738 /*NOTREACHED*/ 739 } 740 /* 741 * Before destroying the pages, we need to take care 742 * of the rctl locked memory accounting. For that 743 * we need to calculte the unlocked_bytes. 744 */ 745 if (pp->p_lckcnt > 0) 746 unlocked_bytes += PAGESIZE; 747 /*LINTED: constant in conditional context */ 748 VN_DISPOSE(pp, B_INVAL, 0, kcred); 749 } 750 } 751 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 752 if (unlocked_bytes > 0) 753 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0); 754 mutex_exit(&sp->shm_mlock); 755 } 756 if (root != 0 || curnpgs != 0) { 757 panic("segspt_free_pages: bad large page"); 758 /*NOTREACHED*/ 759 } 760 761 /* 762 * mark that pages have been released 763 */ 764 sptd->spt_realsize = 0; 765 766 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 767 atomic_add_long(&spt_used, -npages); 768 anon_swap_restore(npages); 769 } 770 } 771 772 /* 773 * Get memory allocation policy info for specified address in given segment 774 */ 775 static lgrp_mem_policy_info_t * 776 segspt_getpolicy(struct seg *seg, caddr_t addr) 777 { 778 struct anon_map *amp; 779 ulong_t anon_index; 780 lgrp_mem_policy_info_t *policy_info; 781 struct spt_data *spt_data; 782 783 ASSERT(seg != NULL); 784 785 /* 786 * Get anon_map from segspt 787 * 788 * Assume that no lock needs to be held on anon_map, since 789 * it should be protected by its reference count which must be 790 * nonzero for an existing segment 791 * Need to grab readers lock on policy tree though 792 */ 793 spt_data = (struct spt_data *)seg->s_data; 794 if (spt_data == NULL) 795 return (NULL); 796 amp = spt_data->spt_amp; 797 ASSERT(amp->refcnt != 0); 798 799 /* 800 * Get policy info 801 * 802 * Assume starting anon index of 0 803 */ 804 anon_index = seg_page(seg, addr); 805 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 806 807 return (policy_info); 808 } 809 810 /* 811 * DISM only. 812 * Return locked pages over a given range. 813 * 814 * We will cache all DISM locked pages and save the pplist for the 815 * entire segment in the ppa field of the underlying DISM segment structure. 816 * Later, during a call to segspt_reclaim() we will use this ppa array 817 * to page_unlock() all of the pages and then we will free this ppa list. 818 */ 819 /*ARGSUSED*/ 820 static int 821 segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len, 822 struct page ***ppp, enum lock_type type, enum seg_rw rw) 823 { 824 struct shm_data *shmd = (struct shm_data *)seg->s_data; 825 struct seg *sptseg = shmd->shm_sptseg; 826 struct spt_data *sptd = sptseg->s_data; 827 pgcnt_t pg_idx, npages, tot_npages, npgs; 828 struct page **pplist, **pl, **ppa, *pp; 829 struct anon_map *amp; 830 spgcnt_t an_idx; 831 int ret = ENOTSUP; 832 uint_t pl_built = 0; 833 struct anon *ap; 834 struct vnode *vp; 835 u_offset_t off; 836 pgcnt_t claim_availrmem = 0; 837 uint_t szc; 838 839 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 840 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK); 841 842 /* 843 * We want to lock/unlock the entire ISM segment. Therefore, 844 * we will be using the underlying sptseg and it's base address 845 * and length for the caching arguments. 846 */ 847 ASSERT(sptseg); 848 ASSERT(sptd); 849 850 pg_idx = seg_page(seg, addr); 851 npages = btopr(len); 852 853 /* 854 * check if the request is larger than number of pages covered 855 * by amp 856 */ 857 if (pg_idx + npages > btopr(sptd->spt_amp->size)) { 858 *ppp = NULL; 859 return (ENOTSUP); 860 } 861 862 if (type == L_PAGEUNLOCK) { 863 ASSERT(sptd->spt_ppa != NULL); 864 865 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size, 866 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 867 868 /* 869 * If someone is blocked while unmapping, we purge 870 * segment page cache and thus reclaim pplist synchronously 871 * without waiting for seg_pasync_thread. This speeds up 872 * unmapping in cases where munmap(2) is called, while 873 * raw async i/o is still in progress or where a thread 874 * exits on data fault in a multithreaded application. 875 */ 876 if ((sptd->spt_flags & DISM_PPA_CHANGED) || 877 (AS_ISUNMAPWAIT(seg->s_as) && 878 shmd->shm_softlockcnt > 0)) { 879 segspt_purge(seg); 880 } 881 return (0); 882 } 883 884 /* The L_PAGELOCK case ... */ 885 886 if (sptd->spt_flags & DISM_PPA_CHANGED) { 887 segspt_purge(seg); 888 /* 889 * for DISM ppa needs to be rebuild since 890 * number of locked pages could be changed 891 */ 892 *ppp = NULL; 893 return (ENOTSUP); 894 } 895 896 /* 897 * First try to find pages in segment page cache, without 898 * holding the segment lock. 899 */ 900 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 901 S_WRITE, SEGP_FORCE_WIRED); 902 if (pplist != NULL) { 903 ASSERT(sptd->spt_ppa != NULL); 904 ASSERT(sptd->spt_ppa == pplist); 905 ppa = sptd->spt_ppa; 906 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 907 if (ppa[an_idx] == NULL) { 908 seg_pinactive(seg, NULL, seg->s_base, 909 sptd->spt_amp->size, ppa, 910 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 911 *ppp = NULL; 912 return (ENOTSUP); 913 } 914 if ((szc = ppa[an_idx]->p_szc) != 0) { 915 npgs = page_get_pagecnt(szc); 916 an_idx = P2ROUNDUP(an_idx + 1, npgs); 917 } else { 918 an_idx++; 919 } 920 } 921 /* 922 * Since we cache the entire DISM segment, we want to 923 * set ppp to point to the first slot that corresponds 924 * to the requested addr, i.e. pg_idx. 925 */ 926 *ppp = &(sptd->spt_ppa[pg_idx]); 927 return (0); 928 } 929 930 mutex_enter(&sptd->spt_lock); 931 /* 932 * try to find pages in segment page cache with mutex 933 */ 934 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 935 S_WRITE, SEGP_FORCE_WIRED); 936 if (pplist != NULL) { 937 ASSERT(sptd->spt_ppa != NULL); 938 ASSERT(sptd->spt_ppa == pplist); 939 ppa = sptd->spt_ppa; 940 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 941 if (ppa[an_idx] == NULL) { 942 mutex_exit(&sptd->spt_lock); 943 seg_pinactive(seg, NULL, seg->s_base, 944 sptd->spt_amp->size, ppa, 945 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 946 *ppp = NULL; 947 return (ENOTSUP); 948 } 949 if ((szc = ppa[an_idx]->p_szc) != 0) { 950 npgs = page_get_pagecnt(szc); 951 an_idx = P2ROUNDUP(an_idx + 1, npgs); 952 } else { 953 an_idx++; 954 } 955 } 956 /* 957 * Since we cache the entire DISM segment, we want to 958 * set ppp to point to the first slot that corresponds 959 * to the requested addr, i.e. pg_idx. 960 */ 961 mutex_exit(&sptd->spt_lock); 962 *ppp = &(sptd->spt_ppa[pg_idx]); 963 return (0); 964 } 965 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size, 966 SEGP_FORCE_WIRED) == SEGP_FAIL) { 967 mutex_exit(&sptd->spt_lock); 968 *ppp = NULL; 969 return (ENOTSUP); 970 } 971 972 /* 973 * No need to worry about protections because DISM pages are always rw. 974 */ 975 pl = pplist = NULL; 976 amp = sptd->spt_amp; 977 978 /* 979 * Do we need to build the ppa array? 980 */ 981 if (sptd->spt_ppa == NULL) { 982 pgcnt_t lpg_cnt = 0; 983 984 pl_built = 1; 985 tot_npages = btopr(sptd->spt_amp->size); 986 987 ASSERT(sptd->spt_pcachecnt == 0); 988 pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP); 989 pl = pplist; 990 991 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 992 for (an_idx = 0; an_idx < tot_npages; ) { 993 ap = anon_get_ptr(amp->ahp, an_idx); 994 /* 995 * Cache only mlocked pages. For large pages 996 * if one (constituent) page is mlocked 997 * all pages for that large page 998 * are cached also. This is for quick 999 * lookups of ppa array; 1000 */ 1001 if ((ap != NULL) && (lpg_cnt != 0 || 1002 (sptd->spt_ppa_lckcnt[an_idx] != 0))) { 1003 1004 swap_xlate(ap, &vp, &off); 1005 pp = page_lookup(vp, off, SE_SHARED); 1006 ASSERT(pp != NULL); 1007 if (lpg_cnt == 0) { 1008 lpg_cnt++; 1009 /* 1010 * For a small page, we are done -- 1011 * lpg_count is reset to 0 below. 1012 * 1013 * For a large page, we are guaranteed 1014 * to find the anon structures of all 1015 * constituent pages and a non-zero 1016 * lpg_cnt ensures that we don't test 1017 * for mlock for these. We are done 1018 * when lpg_count reaches (npgs + 1). 1019 * If we are not the first constituent 1020 * page, restart at the first one. 1021 */ 1022 npgs = page_get_pagecnt(pp->p_szc); 1023 if (!IS_P2ALIGNED(an_idx, npgs)) { 1024 an_idx = P2ALIGN(an_idx, npgs); 1025 page_unlock(pp); 1026 continue; 1027 } 1028 } 1029 if (++lpg_cnt > npgs) 1030 lpg_cnt = 0; 1031 1032 /* 1033 * availrmem is decremented only 1034 * for unlocked pages 1035 */ 1036 if (sptd->spt_ppa_lckcnt[an_idx] == 0) 1037 claim_availrmem++; 1038 pplist[an_idx] = pp; 1039 } 1040 an_idx++; 1041 } 1042 ANON_LOCK_EXIT(&->a_rwlock); 1043 1044 if (claim_availrmem) { 1045 mutex_enter(&freemem_lock); 1046 if (availrmem < tune.t_minarmem + claim_availrmem) { 1047 mutex_exit(&freemem_lock); 1048 ret = ENOTSUP; 1049 claim_availrmem = 0; 1050 goto insert_fail; 1051 } else { 1052 availrmem -= claim_availrmem; 1053 } 1054 mutex_exit(&freemem_lock); 1055 } 1056 1057 sptd->spt_ppa = pl; 1058 } else { 1059 /* 1060 * We already have a valid ppa[]. 1061 */ 1062 pl = sptd->spt_ppa; 1063 } 1064 1065 ASSERT(pl != NULL); 1066 1067 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size, 1068 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED, 1069 segspt_reclaim); 1070 if (ret == SEGP_FAIL) { 1071 /* 1072 * seg_pinsert failed. We return 1073 * ENOTSUP, so that the as_pagelock() code will 1074 * then try the slower F_SOFTLOCK path. 1075 */ 1076 if (pl_built) { 1077 /* 1078 * No one else has referenced the ppa[]. 1079 * We created it and we need to destroy it. 1080 */ 1081 sptd->spt_ppa = NULL; 1082 } 1083 ret = ENOTSUP; 1084 goto insert_fail; 1085 } 1086 1087 /* 1088 * In either case, we increment softlockcnt on the 'real' segment. 1089 */ 1090 sptd->spt_pcachecnt++; 1091 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 1092 1093 ppa = sptd->spt_ppa; 1094 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 1095 if (ppa[an_idx] == NULL) { 1096 mutex_exit(&sptd->spt_lock); 1097 seg_pinactive(seg, NULL, seg->s_base, 1098 sptd->spt_amp->size, 1099 pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 1100 *ppp = NULL; 1101 return (ENOTSUP); 1102 } 1103 if ((szc = ppa[an_idx]->p_szc) != 0) { 1104 npgs = page_get_pagecnt(szc); 1105 an_idx = P2ROUNDUP(an_idx + 1, npgs); 1106 } else { 1107 an_idx++; 1108 } 1109 } 1110 /* 1111 * We can now drop the sptd->spt_lock since the ppa[] 1112 * exists and he have incremented pacachecnt. 1113 */ 1114 mutex_exit(&sptd->spt_lock); 1115 1116 /* 1117 * Since we cache the entire segment, we want to 1118 * set ppp to point to the first slot that corresponds 1119 * to the requested addr, i.e. pg_idx. 1120 */ 1121 *ppp = &(sptd->spt_ppa[pg_idx]); 1122 return (0); 1123 1124 insert_fail: 1125 /* 1126 * We will only reach this code if we tried and failed. 1127 * 1128 * And we can drop the lock on the dummy seg, once we've failed 1129 * to set up a new ppa[]. 1130 */ 1131 mutex_exit(&sptd->spt_lock); 1132 1133 if (pl_built) { 1134 if (claim_availrmem) { 1135 mutex_enter(&freemem_lock); 1136 availrmem += claim_availrmem; 1137 mutex_exit(&freemem_lock); 1138 } 1139 1140 /* 1141 * We created pl and we need to destroy it. 1142 */ 1143 pplist = pl; 1144 for (an_idx = 0; an_idx < tot_npages; an_idx++) { 1145 if (pplist[an_idx] != NULL) 1146 page_unlock(pplist[an_idx]); 1147 } 1148 kmem_free(pl, sizeof (page_t *) * tot_npages); 1149 } 1150 1151 if (shmd->shm_softlockcnt <= 0) { 1152 if (AS_ISUNMAPWAIT(seg->s_as)) { 1153 mutex_enter(&seg->s_as->a_contents); 1154 if (AS_ISUNMAPWAIT(seg->s_as)) { 1155 AS_CLRUNMAPWAIT(seg->s_as); 1156 cv_broadcast(&seg->s_as->a_cv); 1157 } 1158 mutex_exit(&seg->s_as->a_contents); 1159 } 1160 } 1161 *ppp = NULL; 1162 return (ret); 1163 } 1164 1165 1166 1167 /* 1168 * return locked pages over a given range. 1169 * 1170 * We will cache the entire ISM segment and save the pplist for the 1171 * entire segment in the ppa field of the underlying ISM segment structure. 1172 * Later, during a call to segspt_reclaim() we will use this ppa array 1173 * to page_unlock() all of the pages and then we will free this ppa list. 1174 */ 1175 /*ARGSUSED*/ 1176 static int 1177 segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len, 1178 struct page ***ppp, enum lock_type type, enum seg_rw rw) 1179 { 1180 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1181 struct seg *sptseg = shmd->shm_sptseg; 1182 struct spt_data *sptd = sptseg->s_data; 1183 pgcnt_t np, page_index, npages; 1184 caddr_t a, spt_base; 1185 struct page **pplist, **pl, *pp; 1186 struct anon_map *amp; 1187 ulong_t anon_index; 1188 int ret = ENOTSUP; 1189 uint_t pl_built = 0; 1190 struct anon *ap; 1191 struct vnode *vp; 1192 u_offset_t off; 1193 1194 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1195 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK); 1196 1197 1198 /* 1199 * We want to lock/unlock the entire ISM segment. Therefore, 1200 * we will be using the underlying sptseg and it's base address 1201 * and length for the caching arguments. 1202 */ 1203 ASSERT(sptseg); 1204 ASSERT(sptd); 1205 1206 if (sptd->spt_flags & SHM_PAGEABLE) { 1207 return (segspt_dismpagelock(seg, addr, len, ppp, type, rw)); 1208 } 1209 1210 page_index = seg_page(seg, addr); 1211 npages = btopr(len); 1212 1213 /* 1214 * check if the request is larger than number of pages covered 1215 * by amp 1216 */ 1217 if (page_index + npages > btopr(sptd->spt_amp->size)) { 1218 *ppp = NULL; 1219 return (ENOTSUP); 1220 } 1221 1222 if (type == L_PAGEUNLOCK) { 1223 1224 ASSERT(sptd->spt_ppa != NULL); 1225 1226 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size, 1227 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 1228 1229 /* 1230 * If someone is blocked while unmapping, we purge 1231 * segment page cache and thus reclaim pplist synchronously 1232 * without waiting for seg_pasync_thread. This speeds up 1233 * unmapping in cases where munmap(2) is called, while 1234 * raw async i/o is still in progress or where a thread 1235 * exits on data fault in a multithreaded application. 1236 */ 1237 if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 1238 segspt_purge(seg); 1239 } 1240 return (0); 1241 } 1242 1243 /* The L_PAGELOCK case... */ 1244 1245 /* 1246 * First try to find pages in segment page cache, without 1247 * holding the segment lock. 1248 */ 1249 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 1250 S_WRITE, SEGP_FORCE_WIRED); 1251 if (pplist != NULL) { 1252 ASSERT(sptd->spt_ppa == pplist); 1253 ASSERT(sptd->spt_ppa[page_index]); 1254 /* 1255 * Since we cache the entire ISM segment, we want to 1256 * set ppp to point to the first slot that corresponds 1257 * to the requested addr, i.e. page_index. 1258 */ 1259 *ppp = &(sptd->spt_ppa[page_index]); 1260 return (0); 1261 } 1262 1263 mutex_enter(&sptd->spt_lock); 1264 1265 /* 1266 * try to find pages in segment page cache 1267 */ 1268 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 1269 S_WRITE, SEGP_FORCE_WIRED); 1270 if (pplist != NULL) { 1271 ASSERT(sptd->spt_ppa == pplist); 1272 /* 1273 * Since we cache the entire segment, we want to 1274 * set ppp to point to the first slot that corresponds 1275 * to the requested addr, i.e. page_index. 1276 */ 1277 mutex_exit(&sptd->spt_lock); 1278 *ppp = &(sptd->spt_ppa[page_index]); 1279 return (0); 1280 } 1281 1282 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size, 1283 SEGP_FORCE_WIRED) == SEGP_FAIL) { 1284 mutex_exit(&sptd->spt_lock); 1285 *ppp = NULL; 1286 return (ENOTSUP); 1287 } 1288 1289 /* 1290 * No need to worry about protections because ISM pages 1291 * are always rw. 1292 */ 1293 pl = pplist = NULL; 1294 1295 /* 1296 * Do we need to build the ppa array? 1297 */ 1298 if (sptd->spt_ppa == NULL) { 1299 ASSERT(sptd->spt_ppa == pplist); 1300 1301 spt_base = sptseg->s_base; 1302 pl_built = 1; 1303 1304 /* 1305 * availrmem is decremented once during anon_swap_adjust() 1306 * and is incremented during the anon_unresv(), which is 1307 * called from shm_rm_amp() when the segment is destroyed. 1308 */ 1309 amp = sptd->spt_amp; 1310 ASSERT(amp != NULL); 1311 1312 /* pcachecnt is protected by sptd->spt_lock */ 1313 ASSERT(sptd->spt_pcachecnt == 0); 1314 pplist = kmem_zalloc(sizeof (page_t *) 1315 * btopr(sptd->spt_amp->size), KM_SLEEP); 1316 pl = pplist; 1317 1318 anon_index = seg_page(sptseg, spt_base); 1319 1320 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 1321 for (a = spt_base; a < (spt_base + sptd->spt_amp->size); 1322 a += PAGESIZE, anon_index++, pplist++) { 1323 ap = anon_get_ptr(amp->ahp, anon_index); 1324 ASSERT(ap != NULL); 1325 swap_xlate(ap, &vp, &off); 1326 pp = page_lookup(vp, off, SE_SHARED); 1327 ASSERT(pp != NULL); 1328 *pplist = pp; 1329 } 1330 ANON_LOCK_EXIT(&->a_rwlock); 1331 1332 if (a < (spt_base + sptd->spt_amp->size)) { 1333 ret = ENOTSUP; 1334 goto insert_fail; 1335 } 1336 sptd->spt_ppa = pl; 1337 } else { 1338 /* 1339 * We already have a valid ppa[]. 1340 */ 1341 pl = sptd->spt_ppa; 1342 } 1343 1344 ASSERT(pl != NULL); 1345 1346 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size, 1347 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED, 1348 segspt_reclaim); 1349 if (ret == SEGP_FAIL) { 1350 /* 1351 * seg_pinsert failed. We return 1352 * ENOTSUP, so that the as_pagelock() code will 1353 * then try the slower F_SOFTLOCK path. 1354 */ 1355 if (pl_built) { 1356 /* 1357 * No one else has referenced the ppa[]. 1358 * We created it and we need to destroy it. 1359 */ 1360 sptd->spt_ppa = NULL; 1361 } 1362 ret = ENOTSUP; 1363 goto insert_fail; 1364 } 1365 1366 /* 1367 * In either case, we increment softlockcnt on the 'real' segment. 1368 */ 1369 sptd->spt_pcachecnt++; 1370 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 1371 1372 /* 1373 * We can now drop the sptd->spt_lock since the ppa[] 1374 * exists and he have incremented pacachecnt. 1375 */ 1376 mutex_exit(&sptd->spt_lock); 1377 1378 /* 1379 * Since we cache the entire segment, we want to 1380 * set ppp to point to the first slot that corresponds 1381 * to the requested addr, i.e. page_index. 1382 */ 1383 *ppp = &(sptd->spt_ppa[page_index]); 1384 return (0); 1385 1386 insert_fail: 1387 /* 1388 * We will only reach this code if we tried and failed. 1389 * 1390 * And we can drop the lock on the dummy seg, once we've failed 1391 * to set up a new ppa[]. 1392 */ 1393 mutex_exit(&sptd->spt_lock); 1394 1395 if (pl_built) { 1396 /* 1397 * We created pl and we need to destroy it. 1398 */ 1399 pplist = pl; 1400 np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT); 1401 while (np) { 1402 page_unlock(*pplist); 1403 np--; 1404 pplist++; 1405 } 1406 kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size)); 1407 } 1408 if (shmd->shm_softlockcnt <= 0) { 1409 if (AS_ISUNMAPWAIT(seg->s_as)) { 1410 mutex_enter(&seg->s_as->a_contents); 1411 if (AS_ISUNMAPWAIT(seg->s_as)) { 1412 AS_CLRUNMAPWAIT(seg->s_as); 1413 cv_broadcast(&seg->s_as->a_cv); 1414 } 1415 mutex_exit(&seg->s_as->a_contents); 1416 } 1417 } 1418 *ppp = NULL; 1419 return (ret); 1420 } 1421 1422 /* 1423 * purge any cached pages in the I/O page cache 1424 */ 1425 static void 1426 segspt_purge(struct seg *seg) 1427 { 1428 seg_ppurge(seg, NULL, SEGP_FORCE_WIRED); 1429 } 1430 1431 static int 1432 segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist, 1433 enum seg_rw rw, int async) 1434 { 1435 struct seg *seg = (struct seg *)ptag; 1436 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1437 struct seg *sptseg; 1438 struct spt_data *sptd; 1439 pgcnt_t npages, i, free_availrmem = 0; 1440 int done = 0; 1441 1442 #ifdef lint 1443 addr = addr; 1444 #endif 1445 sptseg = shmd->shm_sptseg; 1446 sptd = sptseg->s_data; 1447 npages = (len >> PAGESHIFT); 1448 ASSERT(npages); 1449 ASSERT(sptd->spt_pcachecnt != 0); 1450 ASSERT(sptd->spt_ppa == pplist); 1451 ASSERT(npages == btopr(sptd->spt_amp->size)); 1452 ASSERT(async || AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1453 1454 /* 1455 * Acquire the lock on the dummy seg and destroy the 1456 * ppa array IF this is the last pcachecnt. 1457 */ 1458 mutex_enter(&sptd->spt_lock); 1459 if (--sptd->spt_pcachecnt == 0) { 1460 for (i = 0; i < npages; i++) { 1461 if (pplist[i] == NULL) { 1462 continue; 1463 } 1464 if (rw == S_WRITE) { 1465 hat_setrefmod(pplist[i]); 1466 } else { 1467 hat_setref(pplist[i]); 1468 } 1469 if ((sptd->spt_flags & SHM_PAGEABLE) && 1470 (sptd->spt_ppa_lckcnt[i] == 0)) 1471 free_availrmem++; 1472 page_unlock(pplist[i]); 1473 } 1474 if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) { 1475 mutex_enter(&freemem_lock); 1476 availrmem += free_availrmem; 1477 mutex_exit(&freemem_lock); 1478 } 1479 /* 1480 * Since we want to cach/uncache the entire ISM segment, 1481 * we will track the pplist in a segspt specific field 1482 * ppa, that is initialized at the time we add an entry to 1483 * the cache. 1484 */ 1485 ASSERT(sptd->spt_pcachecnt == 0); 1486 kmem_free(pplist, sizeof (page_t *) * npages); 1487 sptd->spt_ppa = NULL; 1488 sptd->spt_flags &= ~DISM_PPA_CHANGED; 1489 sptd->spt_gen++; 1490 cv_broadcast(&sptd->spt_cv); 1491 done = 1; 1492 } 1493 mutex_exit(&sptd->spt_lock); 1494 1495 /* 1496 * If we are pcache async thread or called via seg_ppurge_wiredpp() we 1497 * may not hold AS lock (in this case async argument is not 0). This 1498 * means if softlockcnt drops to 0 after the decrement below address 1499 * space may get freed. We can't allow it since after softlock 1500 * derement to 0 we still need to access as structure for possible 1501 * wakeup of unmap waiters. To prevent the disappearance of as we take 1502 * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes 1503 * this mutex as a barrier to make sure this routine completes before 1504 * segment is freed. 1505 * 1506 * The second complication we have to deal with in async case is a 1507 * possibility of missed wake up of unmap wait thread. When we don't 1508 * hold as lock here we may take a_contents lock before unmap wait 1509 * thread that was first to see softlockcnt was still not 0. As a 1510 * result we'll fail to wake up an unmap wait thread. To avoid this 1511 * race we set nounmapwait flag in as structure if we drop softlockcnt 1512 * to 0 if async is not 0. unmapwait thread 1513 * will not block if this flag is set. 1514 */ 1515 if (async) 1516 mutex_enter(&shmd->shm_segfree_syncmtx); 1517 1518 /* 1519 * Now decrement softlockcnt. 1520 */ 1521 ASSERT(shmd->shm_softlockcnt > 0); 1522 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 1523 1524 if (shmd->shm_softlockcnt <= 0) { 1525 if (async || AS_ISUNMAPWAIT(seg->s_as)) { 1526 mutex_enter(&seg->s_as->a_contents); 1527 if (async) 1528 AS_SETNOUNMAPWAIT(seg->s_as); 1529 if (AS_ISUNMAPWAIT(seg->s_as)) { 1530 AS_CLRUNMAPWAIT(seg->s_as); 1531 cv_broadcast(&seg->s_as->a_cv); 1532 } 1533 mutex_exit(&seg->s_as->a_contents); 1534 } 1535 } 1536 1537 if (async) 1538 mutex_exit(&shmd->shm_segfree_syncmtx); 1539 1540 return (done); 1541 } 1542 1543 /* 1544 * Do a F_SOFTUNLOCK call over the range requested. 1545 * The range must have already been F_SOFTLOCK'ed. 1546 * 1547 * The calls to acquire and release the anon map lock mutex were 1548 * removed in order to avoid a deadly embrace during a DR 1549 * memory delete operation. (Eg. DR blocks while waiting for a 1550 * exclusive lock on a page that is being used for kaio; the 1551 * thread that will complete the kaio and call segspt_softunlock 1552 * blocks on the anon map lock; another thread holding the anon 1553 * map lock blocks on another page lock via the segspt_shmfault 1554 * -> page_lookup -> page_lookup_create -> page_lock_es code flow.) 1555 * 1556 * The appropriateness of the removal is based upon the following: 1557 * 1. If we are holding a segment's reader lock and the page is held 1558 * shared, then the corresponding element in anonmap which points to 1559 * anon struct cannot change and there is no need to acquire the 1560 * anonymous map lock. 1561 * 2. Threads in segspt_softunlock have a reader lock on the segment 1562 * and already have the shared page lock, so we are guaranteed that 1563 * the anon map slot cannot change and therefore can call anon_get_ptr() 1564 * without grabbing the anonymous map lock. 1565 * 3. Threads that softlock a shared page break copy-on-write, even if 1566 * its a read. Thus cow faults can be ignored with respect to soft 1567 * unlocking, since the breaking of cow means that the anon slot(s) will 1568 * not be shared. 1569 */ 1570 static void 1571 segspt_softunlock(struct seg *seg, caddr_t sptseg_addr, 1572 size_t len, enum seg_rw rw) 1573 { 1574 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1575 struct seg *sptseg; 1576 struct spt_data *sptd; 1577 page_t *pp; 1578 caddr_t adr; 1579 struct vnode *vp; 1580 u_offset_t offset; 1581 ulong_t anon_index; 1582 struct anon_map *amp; /* XXX - for locknest */ 1583 struct anon *ap = NULL; 1584 pgcnt_t npages; 1585 1586 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1587 1588 sptseg = shmd->shm_sptseg; 1589 sptd = sptseg->s_data; 1590 1591 /* 1592 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 1593 * and therefore their pages are SE_SHARED locked 1594 * for the entire life of the segment. 1595 */ 1596 if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) && 1597 ((sptd->spt_flags & SHM_PAGEABLE) == 0)) { 1598 goto softlock_decrement; 1599 } 1600 1601 /* 1602 * Any thread is free to do a page_find and 1603 * page_unlock() on the pages within this seg. 1604 * 1605 * We are already holding the as->a_lock on the user's 1606 * real segment, but we need to hold the a_lock on the 1607 * underlying dummy as. This is mostly to satisfy the 1608 * underlying HAT layer. 1609 */ 1610 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 1611 hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len); 1612 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 1613 1614 amp = sptd->spt_amp; 1615 ASSERT(amp != NULL); 1616 anon_index = seg_page(sptseg, sptseg_addr); 1617 1618 for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) { 1619 ap = anon_get_ptr(amp->ahp, anon_index++); 1620 ASSERT(ap != NULL); 1621 swap_xlate(ap, &vp, &offset); 1622 1623 /* 1624 * Use page_find() instead of page_lookup() to 1625 * find the page since we know that it has a 1626 * "shared" lock. 1627 */ 1628 pp = page_find(vp, offset); 1629 ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1)); 1630 if (pp == NULL) { 1631 panic("segspt_softunlock: " 1632 "addr %p, ap %p, vp %p, off %llx", 1633 (void *)adr, (void *)ap, (void *)vp, offset); 1634 /*NOTREACHED*/ 1635 } 1636 1637 if (rw == S_WRITE) { 1638 hat_setrefmod(pp); 1639 } else if (rw != S_OTHER) { 1640 hat_setref(pp); 1641 } 1642 page_unlock(pp); 1643 } 1644 1645 softlock_decrement: 1646 npages = btopr(len); 1647 ASSERT(shmd->shm_softlockcnt >= npages); 1648 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages); 1649 if (shmd->shm_softlockcnt == 0) { 1650 /* 1651 * All SOFTLOCKS are gone. Wakeup any waiting 1652 * unmappers so they can try again to unmap. 1653 * Check for waiters first without the mutex 1654 * held so we don't always grab the mutex on 1655 * softunlocks. 1656 */ 1657 if (AS_ISUNMAPWAIT(seg->s_as)) { 1658 mutex_enter(&seg->s_as->a_contents); 1659 if (AS_ISUNMAPWAIT(seg->s_as)) { 1660 AS_CLRUNMAPWAIT(seg->s_as); 1661 cv_broadcast(&seg->s_as->a_cv); 1662 } 1663 mutex_exit(&seg->s_as->a_contents); 1664 } 1665 } 1666 } 1667 1668 int 1669 segspt_shmattach(struct seg *seg, caddr_t *argsp) 1670 { 1671 struct shm_data *shmd_arg = (struct shm_data *)argsp; 1672 struct shm_data *shmd; 1673 struct anon_map *shm_amp = shmd_arg->shm_amp; 1674 struct spt_data *sptd; 1675 int error = 0; 1676 1677 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 1678 1679 shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP); 1680 if (shmd == NULL) 1681 return (ENOMEM); 1682 1683 shmd->shm_sptas = shmd_arg->shm_sptas; 1684 shmd->shm_amp = shm_amp; 1685 shmd->shm_sptseg = shmd_arg->shm_sptseg; 1686 1687 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0, 1688 NULL, 0, seg->s_size); 1689 1690 mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL); 1691 1692 seg->s_data = (void *)shmd; 1693 seg->s_ops = &segspt_shmops; 1694 seg->s_szc = shmd->shm_sptseg->s_szc; 1695 sptd = shmd->shm_sptseg->s_data; 1696 1697 if (sptd->spt_flags & SHM_PAGEABLE) { 1698 if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size), 1699 KM_NOSLEEP)) == NULL) { 1700 seg->s_data = (void *)NULL; 1701 kmem_free(shmd, (sizeof (*shmd))); 1702 return (ENOMEM); 1703 } 1704 shmd->shm_lckpgs = 0; 1705 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 1706 if ((error = hat_share(seg->s_as->a_hat, seg->s_base, 1707 shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 1708 seg->s_size, seg->s_szc)) != 0) { 1709 kmem_free(shmd->shm_vpage, 1710 btopr(shm_amp->size)); 1711 } 1712 } 1713 } else { 1714 error = hat_share(seg->s_as->a_hat, seg->s_base, 1715 shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 1716 seg->s_size, seg->s_szc); 1717 } 1718 if (error) { 1719 seg->s_szc = 0; 1720 seg->s_data = (void *)NULL; 1721 kmem_free(shmd, (sizeof (*shmd))); 1722 } else { 1723 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 1724 shm_amp->refcnt++; 1725 ANON_LOCK_EXIT(&shm_amp->a_rwlock); 1726 } 1727 return (error); 1728 } 1729 1730 int 1731 segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize) 1732 { 1733 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1734 int reclaim = 1; 1735 1736 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 1737 retry: 1738 if (shmd->shm_softlockcnt > 0) { 1739 if (reclaim == 1) { 1740 segspt_purge(seg); 1741 reclaim = 0; 1742 goto retry; 1743 } 1744 return (EAGAIN); 1745 } 1746 1747 if (ssize != seg->s_size) { 1748 #ifdef DEBUG 1749 cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n", 1750 ssize, seg->s_size); 1751 #endif 1752 return (EINVAL); 1753 } 1754 1755 (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK, 1756 NULL, 0); 1757 hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc); 1758 1759 seg_free(seg); 1760 1761 return (0); 1762 } 1763 1764 void 1765 segspt_shmfree(struct seg *seg) 1766 { 1767 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1768 struct anon_map *shm_amp = shmd->shm_amp; 1769 1770 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 1771 1772 (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0, 1773 MC_UNLOCK, NULL, 0); 1774 1775 /* 1776 * Need to increment refcnt when attaching 1777 * and decrement when detaching because of dup(). 1778 */ 1779 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 1780 shm_amp->refcnt--; 1781 ANON_LOCK_EXIT(&shm_amp->a_rwlock); 1782 1783 if (shmd->shm_vpage) { /* only for DISM */ 1784 kmem_free(shmd->shm_vpage, btopr(shm_amp->size)); 1785 shmd->shm_vpage = NULL; 1786 } 1787 1788 /* 1789 * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's 1790 * still working with this segment without holding as lock. 1791 */ 1792 ASSERT(shmd->shm_softlockcnt == 0); 1793 mutex_enter(&shmd->shm_segfree_syncmtx); 1794 mutex_destroy(&shmd->shm_segfree_syncmtx); 1795 1796 kmem_free(shmd, sizeof (*shmd)); 1797 } 1798 1799 /*ARGSUSED*/ 1800 int 1801 segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 1802 { 1803 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1804 1805 /* 1806 * Shared page table is more than shared mapping. 1807 * Individual process sharing page tables can't change prot 1808 * because there is only one set of page tables. 1809 * This will be allowed after private page table is 1810 * supported. 1811 */ 1812 /* need to return correct status error? */ 1813 return (0); 1814 } 1815 1816 1817 faultcode_t 1818 segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr, 1819 size_t len, enum fault_type type, enum seg_rw rw) 1820 { 1821 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1822 struct seg *sptseg = shmd->shm_sptseg; 1823 struct as *curspt = shmd->shm_sptas; 1824 struct spt_data *sptd = sptseg->s_data; 1825 pgcnt_t npages; 1826 size_t size; 1827 caddr_t segspt_addr, shm_addr; 1828 page_t **ppa; 1829 int i; 1830 ulong_t an_idx = 0; 1831 int err = 0; 1832 int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0); 1833 size_t pgsz; 1834 pgcnt_t pgcnt; 1835 caddr_t a; 1836 pgcnt_t pidx; 1837 1838 #ifdef lint 1839 hat = hat; 1840 #endif 1841 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1842 1843 /* 1844 * Because of the way spt is implemented 1845 * the realsize of the segment does not have to be 1846 * equal to the segment size itself. The segment size is 1847 * often in multiples of a page size larger than PAGESIZE. 1848 * The realsize is rounded up to the nearest PAGESIZE 1849 * based on what the user requested. This is a bit of 1850 * ungliness that is historical but not easily fixed 1851 * without re-designing the higher levels of ISM. 1852 */ 1853 ASSERT(addr >= seg->s_base); 1854 if (((addr + len) - seg->s_base) > sptd->spt_realsize) 1855 return (FC_NOMAP); 1856 /* 1857 * For all of the following cases except F_PROT, we need to 1858 * make any necessary adjustments to addr and len 1859 * and get all of the necessary page_t's into an array called ppa[]. 1860 * 1861 * The code in shmat() forces base addr and len of ISM segment 1862 * to be aligned to largest page size supported. Therefore, 1863 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 1864 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 1865 * in large pagesize chunks, or else we will screw up the HAT 1866 * layer by calling hat_memload_array() with differing page sizes 1867 * over a given virtual range. 1868 */ 1869 pgsz = page_get_pagesize(sptseg->s_szc); 1870 pgcnt = page_get_pagecnt(sptseg->s_szc); 1871 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 1872 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 1873 npages = btopr(size); 1874 1875 /* 1876 * Now we need to convert from addr in segshm to addr in segspt. 1877 */ 1878 an_idx = seg_page(seg, shm_addr); 1879 segspt_addr = sptseg->s_base + ptob(an_idx); 1880 1881 ASSERT((segspt_addr + ptob(npages)) <= 1882 (sptseg->s_base + sptd->spt_realsize)); 1883 ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size)); 1884 1885 switch (type) { 1886 1887 case F_SOFTLOCK: 1888 1889 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 1890 /* 1891 * Fall through to the F_INVAL case to load up the hat layer 1892 * entries with the HAT_LOAD_LOCK flag. 1893 */ 1894 /* FALLTHRU */ 1895 case F_INVAL: 1896 1897 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 1898 return (FC_NOMAP); 1899 1900 ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP); 1901 1902 err = spt_anon_getpages(sptseg, segspt_addr, size, ppa); 1903 if (err != 0) { 1904 if (type == F_SOFTLOCK) { 1905 atomic_add_long((ulong_t *)( 1906 &(shmd->shm_softlockcnt)), -npages); 1907 } 1908 goto dism_err; 1909 } 1910 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 1911 a = segspt_addr; 1912 pidx = 0; 1913 if (type == F_SOFTLOCK) { 1914 1915 /* 1916 * Load up the translation keeping it 1917 * locked and don't unlock the page. 1918 */ 1919 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 1920 hat_memload_array(sptseg->s_as->a_hat, 1921 a, pgsz, &ppa[pidx], sptd->spt_prot, 1922 HAT_LOAD_LOCK | HAT_LOAD_SHARE); 1923 } 1924 } else { 1925 if (hat == seg->s_as->a_hat) { 1926 1927 /* 1928 * Migrate pages marked for migration 1929 */ 1930 if (lgrp_optimizations()) 1931 page_migrate(seg, shm_addr, ppa, 1932 npages); 1933 1934 /* CPU HAT */ 1935 for (; pidx < npages; 1936 a += pgsz, pidx += pgcnt) { 1937 hat_memload_array(sptseg->s_as->a_hat, 1938 a, pgsz, &ppa[pidx], 1939 sptd->spt_prot, 1940 HAT_LOAD_SHARE); 1941 } 1942 } else { 1943 /* XHAT. Pass real address */ 1944 hat_memload_array(hat, shm_addr, 1945 size, ppa, sptd->spt_prot, HAT_LOAD_SHARE); 1946 } 1947 1948 /* 1949 * And now drop the SE_SHARED lock(s). 1950 */ 1951 if (dyn_ism_unmap) { 1952 for (i = 0; i < npages; i++) { 1953 page_unlock(ppa[i]); 1954 } 1955 } 1956 } 1957 1958 if (!dyn_ism_unmap) { 1959 if (hat_share(seg->s_as->a_hat, shm_addr, 1960 curspt->a_hat, segspt_addr, ptob(npages), 1961 seg->s_szc) != 0) { 1962 panic("hat_share err in DISM fault"); 1963 /* NOTREACHED */ 1964 } 1965 if (type == F_INVAL) { 1966 for (i = 0; i < npages; i++) { 1967 page_unlock(ppa[i]); 1968 } 1969 } 1970 } 1971 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 1972 dism_err: 1973 kmem_free(ppa, npages * sizeof (page_t *)); 1974 return (err); 1975 1976 case F_SOFTUNLOCK: 1977 1978 /* 1979 * This is a bit ugly, we pass in the real seg pointer, 1980 * but the segspt_addr is the virtual address within the 1981 * dummy seg. 1982 */ 1983 segspt_softunlock(seg, segspt_addr, size, rw); 1984 return (0); 1985 1986 case F_PROT: 1987 1988 /* 1989 * This takes care of the unusual case where a user 1990 * allocates a stack in shared memory and a register 1991 * window overflow is written to that stack page before 1992 * it is otherwise modified. 1993 * 1994 * We can get away with this because ISM segments are 1995 * always rw. Other than this unusual case, there 1996 * should be no instances of protection violations. 1997 */ 1998 return (0); 1999 2000 default: 2001 #ifdef DEBUG 2002 panic("segspt_dismfault default type?"); 2003 #else 2004 return (FC_NOMAP); 2005 #endif 2006 } 2007 } 2008 2009 2010 faultcode_t 2011 segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr, 2012 size_t len, enum fault_type type, enum seg_rw rw) 2013 { 2014 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2015 struct seg *sptseg = shmd->shm_sptseg; 2016 struct as *curspt = shmd->shm_sptas; 2017 struct spt_data *sptd = sptseg->s_data; 2018 pgcnt_t npages; 2019 size_t size; 2020 caddr_t sptseg_addr, shm_addr; 2021 page_t *pp, **ppa; 2022 int i; 2023 u_offset_t offset; 2024 ulong_t anon_index = 0; 2025 struct vnode *vp; 2026 struct anon_map *amp; /* XXX - for locknest */ 2027 struct anon *ap = NULL; 2028 size_t pgsz; 2029 pgcnt_t pgcnt; 2030 caddr_t a; 2031 pgcnt_t pidx; 2032 size_t sz; 2033 2034 #ifdef lint 2035 hat = hat; 2036 #endif 2037 2038 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2039 2040 if (sptd->spt_flags & SHM_PAGEABLE) { 2041 return (segspt_dismfault(hat, seg, addr, len, type, rw)); 2042 } 2043 2044 /* 2045 * Because of the way spt is implemented 2046 * the realsize of the segment does not have to be 2047 * equal to the segment size itself. The segment size is 2048 * often in multiples of a page size larger than PAGESIZE. 2049 * The realsize is rounded up to the nearest PAGESIZE 2050 * based on what the user requested. This is a bit of 2051 * ungliness that is historical but not easily fixed 2052 * without re-designing the higher levels of ISM. 2053 */ 2054 ASSERT(addr >= seg->s_base); 2055 if (((addr + len) - seg->s_base) > sptd->spt_realsize) 2056 return (FC_NOMAP); 2057 /* 2058 * For all of the following cases except F_PROT, we need to 2059 * make any necessary adjustments to addr and len 2060 * and get all of the necessary page_t's into an array called ppa[]. 2061 * 2062 * The code in shmat() forces base addr and len of ISM segment 2063 * to be aligned to largest page size supported. Therefore, 2064 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 2065 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 2066 * in large pagesize chunks, or else we will screw up the HAT 2067 * layer by calling hat_memload_array() with differing page sizes 2068 * over a given virtual range. 2069 */ 2070 pgsz = page_get_pagesize(sptseg->s_szc); 2071 pgcnt = page_get_pagecnt(sptseg->s_szc); 2072 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 2073 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 2074 npages = btopr(size); 2075 2076 /* 2077 * Now we need to convert from addr in segshm to addr in segspt. 2078 */ 2079 anon_index = seg_page(seg, shm_addr); 2080 sptseg_addr = sptseg->s_base + ptob(anon_index); 2081 2082 /* 2083 * And now we may have to adjust npages downward if we have 2084 * exceeded the realsize of the segment or initial anon 2085 * allocations. 2086 */ 2087 if ((sptseg_addr + ptob(npages)) > 2088 (sptseg->s_base + sptd->spt_realsize)) 2089 size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr; 2090 2091 npages = btopr(size); 2092 2093 ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size)); 2094 ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0); 2095 2096 switch (type) { 2097 2098 case F_SOFTLOCK: 2099 2100 /* 2101 * availrmem is decremented once during anon_swap_adjust() 2102 * and is incremented during the anon_unresv(), which is 2103 * called from shm_rm_amp() when the segment is destroyed. 2104 */ 2105 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 2106 /* 2107 * Some platforms assume that ISM pages are SE_SHARED 2108 * locked for the entire life of the segment. 2109 */ 2110 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) 2111 return (0); 2112 /* 2113 * Fall through to the F_INVAL case to load up the hat layer 2114 * entries with the HAT_LOAD_LOCK flag. 2115 */ 2116 2117 /* FALLTHRU */ 2118 case F_INVAL: 2119 2120 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 2121 return (FC_NOMAP); 2122 2123 /* 2124 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP 2125 * may still rely on this call to hat_share(). That 2126 * would imply that those hat's can fault on a 2127 * HAT_LOAD_LOCK translation, which would seem 2128 * contradictory. 2129 */ 2130 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2131 if (hat_share(seg->s_as->a_hat, seg->s_base, 2132 curspt->a_hat, sptseg->s_base, 2133 sptseg->s_size, sptseg->s_szc) != 0) { 2134 panic("hat_share error in ISM fault"); 2135 /*NOTREACHED*/ 2136 } 2137 return (0); 2138 } 2139 ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP); 2140 2141 /* 2142 * I see no need to lock the real seg, 2143 * here, because all of our work will be on the underlying 2144 * dummy seg. 2145 * 2146 * sptseg_addr and npages now account for large pages. 2147 */ 2148 amp = sptd->spt_amp; 2149 ASSERT(amp != NULL); 2150 anon_index = seg_page(sptseg, sptseg_addr); 2151 2152 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2153 for (i = 0; i < npages; i++) { 2154 ap = anon_get_ptr(amp->ahp, anon_index++); 2155 ASSERT(ap != NULL); 2156 swap_xlate(ap, &vp, &offset); 2157 pp = page_lookup(vp, offset, SE_SHARED); 2158 ASSERT(pp != NULL); 2159 ppa[i] = pp; 2160 } 2161 ANON_LOCK_EXIT(&->a_rwlock); 2162 ASSERT(i == npages); 2163 2164 /* 2165 * We are already holding the as->a_lock on the user's 2166 * real segment, but we need to hold the a_lock on the 2167 * underlying dummy as. This is mostly to satisfy the 2168 * underlying HAT layer. 2169 */ 2170 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 2171 a = sptseg_addr; 2172 pidx = 0; 2173 if (type == F_SOFTLOCK) { 2174 /* 2175 * Load up the translation keeping it 2176 * locked and don't unlock the page. 2177 */ 2178 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 2179 sz = MIN(pgsz, ptob(npages - pidx)); 2180 hat_memload_array(sptseg->s_as->a_hat, a, 2181 sz, &ppa[pidx], sptd->spt_prot, 2182 HAT_LOAD_LOCK | HAT_LOAD_SHARE); 2183 } 2184 } else { 2185 if (hat == seg->s_as->a_hat) { 2186 2187 /* 2188 * Migrate pages marked for migration. 2189 */ 2190 if (lgrp_optimizations()) 2191 page_migrate(seg, shm_addr, ppa, 2192 npages); 2193 2194 /* CPU HAT */ 2195 for (; pidx < npages; 2196 a += pgsz, pidx += pgcnt) { 2197 sz = MIN(pgsz, ptob(npages - pidx)); 2198 hat_memload_array(sptseg->s_as->a_hat, 2199 a, sz, &ppa[pidx], 2200 sptd->spt_prot, HAT_LOAD_SHARE); 2201 } 2202 } else { 2203 /* XHAT. Pass real address */ 2204 hat_memload_array(hat, shm_addr, 2205 ptob(npages), ppa, sptd->spt_prot, 2206 HAT_LOAD_SHARE); 2207 } 2208 2209 /* 2210 * And now drop the SE_SHARED lock(s). 2211 */ 2212 for (i = 0; i < npages; i++) 2213 page_unlock(ppa[i]); 2214 } 2215 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 2216 2217 kmem_free(ppa, sizeof (page_t *) * npages); 2218 return (0); 2219 case F_SOFTUNLOCK: 2220 2221 /* 2222 * This is a bit ugly, we pass in the real seg pointer, 2223 * but the sptseg_addr is the virtual address within the 2224 * dummy seg. 2225 */ 2226 segspt_softunlock(seg, sptseg_addr, ptob(npages), rw); 2227 return (0); 2228 2229 case F_PROT: 2230 2231 /* 2232 * This takes care of the unusual case where a user 2233 * allocates a stack in shared memory and a register 2234 * window overflow is written to that stack page before 2235 * it is otherwise modified. 2236 * 2237 * We can get away with this because ISM segments are 2238 * always rw. Other than this unusual case, there 2239 * should be no instances of protection violations. 2240 */ 2241 return (0); 2242 2243 default: 2244 #ifdef DEBUG 2245 cmn_err(CE_WARN, "segspt_shmfault default type?"); 2246 #endif 2247 return (FC_NOMAP); 2248 } 2249 } 2250 2251 /*ARGSUSED*/ 2252 static faultcode_t 2253 segspt_shmfaulta(struct seg *seg, caddr_t addr) 2254 { 2255 return (0); 2256 } 2257 2258 /*ARGSUSED*/ 2259 static int 2260 segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta) 2261 { 2262 return (0); 2263 } 2264 2265 /*ARGSUSED*/ 2266 static size_t 2267 segspt_shmswapout(struct seg *seg) 2268 { 2269 return (0); 2270 } 2271 2272 /* 2273 * duplicate the shared page tables 2274 */ 2275 int 2276 segspt_shmdup(struct seg *seg, struct seg *newseg) 2277 { 2278 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2279 struct anon_map *amp = shmd->shm_amp; 2280 struct shm_data *shmd_new; 2281 struct seg *spt_seg = shmd->shm_sptseg; 2282 struct spt_data *sptd = spt_seg->s_data; 2283 int error = 0; 2284 2285 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 2286 2287 shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP); 2288 newseg->s_data = (void *)shmd_new; 2289 shmd_new->shm_sptas = shmd->shm_sptas; 2290 shmd_new->shm_amp = amp; 2291 shmd_new->shm_sptseg = shmd->shm_sptseg; 2292 newseg->s_ops = &segspt_shmops; 2293 newseg->s_szc = seg->s_szc; 2294 ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc); 2295 2296 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 2297 amp->refcnt++; 2298 ANON_LOCK_EXIT(&->a_rwlock); 2299 2300 if (sptd->spt_flags & SHM_PAGEABLE) { 2301 shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP); 2302 shmd_new->shm_lckpgs = 0; 2303 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2304 if ((error = hat_share(newseg->s_as->a_hat, 2305 newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR, 2306 seg->s_size, seg->s_szc)) != 0) { 2307 kmem_free(shmd_new->shm_vpage, 2308 btopr(amp->size)); 2309 } 2310 } 2311 return (error); 2312 } else { 2313 return (hat_share(newseg->s_as->a_hat, newseg->s_base, 2314 shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size, 2315 seg->s_szc)); 2316 2317 } 2318 } 2319 2320 /*ARGSUSED*/ 2321 int 2322 segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 2323 { 2324 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2325 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2326 2327 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2328 2329 /* 2330 * ISM segment is always rw. 2331 */ 2332 return (((sptd->spt_prot & prot) != prot) ? EACCES : 0); 2333 } 2334 2335 /* 2336 * Return an array of locked large pages, for empty slots allocate 2337 * private zero-filled anon pages. 2338 */ 2339 static int 2340 spt_anon_getpages( 2341 struct seg *sptseg, 2342 caddr_t sptaddr, 2343 size_t len, 2344 page_t *ppa[]) 2345 { 2346 struct spt_data *sptd = sptseg->s_data; 2347 struct anon_map *amp = sptd->spt_amp; 2348 enum seg_rw rw = sptd->spt_prot; 2349 uint_t szc = sptseg->s_szc; 2350 size_t pg_sz, share_sz = page_get_pagesize(szc); 2351 pgcnt_t lp_npgs; 2352 caddr_t lp_addr, e_sptaddr; 2353 uint_t vpprot, ppa_szc = 0; 2354 struct vpage *vpage = NULL; 2355 ulong_t j, ppa_idx; 2356 int err, ierr = 0; 2357 pgcnt_t an_idx; 2358 anon_sync_obj_t cookie; 2359 int anon_locked = 0; 2360 pgcnt_t amp_pgs; 2361 2362 2363 ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz)); 2364 ASSERT(len != 0); 2365 2366 pg_sz = share_sz; 2367 lp_npgs = btop(pg_sz); 2368 lp_addr = sptaddr; 2369 e_sptaddr = sptaddr + len; 2370 an_idx = seg_page(sptseg, sptaddr); 2371 ppa_idx = 0; 2372 2373 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2374 2375 amp_pgs = page_get_pagecnt(amp->a_szc); 2376 2377 /*CONSTCOND*/ 2378 while (1) { 2379 for (; lp_addr < e_sptaddr; 2380 an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) { 2381 2382 /* 2383 * If we're currently locked, and we get to a new 2384 * page, unlock our current anon chunk. 2385 */ 2386 if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) { 2387 anon_array_exit(&cookie); 2388 anon_locked = 0; 2389 } 2390 if (!anon_locked) { 2391 anon_array_enter(amp, an_idx, &cookie); 2392 anon_locked = 1; 2393 } 2394 ppa_szc = (uint_t)-1; 2395 ierr = anon_map_getpages(amp, an_idx, szc, sptseg, 2396 lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx], 2397 &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred); 2398 2399 if (ierr != 0) { 2400 if (ierr > 0) { 2401 err = FC_MAKE_ERR(ierr); 2402 goto lpgs_err; 2403 } 2404 break; 2405 } 2406 } 2407 if (lp_addr == e_sptaddr) { 2408 break; 2409 } 2410 ASSERT(lp_addr < e_sptaddr); 2411 2412 /* 2413 * ierr == -1 means we failed to allocate a large page. 2414 * so do a size down operation. 2415 * 2416 * ierr == -2 means some other process that privately shares 2417 * pages with this process has allocated a larger page and we 2418 * need to retry with larger pages. So do a size up 2419 * operation. This relies on the fact that large pages are 2420 * never partially shared i.e. if we share any constituent 2421 * page of a large page with another process we must share the 2422 * entire large page. Note this cannot happen for SOFTLOCK 2423 * case, unless current address (lpaddr) is at the beginning 2424 * of the next page size boundary because the other process 2425 * couldn't have relocated locked pages. 2426 */ 2427 ASSERT(ierr == -1 || ierr == -2); 2428 if (segvn_anypgsz) { 2429 ASSERT(ierr == -2 || szc != 0); 2430 ASSERT(ierr == -1 || szc < sptseg->s_szc); 2431 szc = (ierr == -1) ? szc - 1 : szc + 1; 2432 } else { 2433 /* 2434 * For faults and segvn_anypgsz == 0 2435 * we need to be careful not to loop forever 2436 * if existing page is found with szc other 2437 * than 0 or seg->s_szc. This could be due 2438 * to page relocations on behalf of DR or 2439 * more likely large page creation. For this 2440 * case simply re-size to existing page's szc 2441 * if returned by anon_map_getpages(). 2442 */ 2443 if (ppa_szc == (uint_t)-1) { 2444 szc = (ierr == -1) ? 0 : sptseg->s_szc; 2445 } else { 2446 ASSERT(ppa_szc <= sptseg->s_szc); 2447 ASSERT(ierr == -2 || ppa_szc < szc); 2448 ASSERT(ierr == -1 || ppa_szc > szc); 2449 szc = ppa_szc; 2450 } 2451 } 2452 pg_sz = page_get_pagesize(szc); 2453 lp_npgs = btop(pg_sz); 2454 ASSERT(IS_P2ALIGNED(lp_addr, pg_sz)); 2455 } 2456 if (anon_locked) { 2457 anon_array_exit(&cookie); 2458 } 2459 ANON_LOCK_EXIT(&->a_rwlock); 2460 return (0); 2461 2462 lpgs_err: 2463 if (anon_locked) { 2464 anon_array_exit(&cookie); 2465 } 2466 ANON_LOCK_EXIT(&->a_rwlock); 2467 for (j = 0; j < ppa_idx; j++) 2468 page_unlock(ppa[j]); 2469 return (err); 2470 } 2471 2472 /* 2473 * count the number of bytes in a set of spt pages that are currently not 2474 * locked 2475 */ 2476 static rctl_qty_t 2477 spt_unlockedbytes(pgcnt_t npages, page_t **ppa) 2478 { 2479 ulong_t i; 2480 rctl_qty_t unlocked = 0; 2481 2482 for (i = 0; i < npages; i++) { 2483 if (ppa[i]->p_lckcnt == 0) 2484 unlocked += PAGESIZE; 2485 } 2486 return (unlocked); 2487 } 2488 2489 extern u_longlong_t randtick(void); 2490 /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */ 2491 #define NLCK (NCPU_P2) 2492 /* Random number with a range [0, n-1], n must be power of two */ 2493 #define RAND_P2(n) \ 2494 ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1)) 2495 2496 int 2497 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 2498 page_t **ppa, ulong_t *lockmap, size_t pos, 2499 rctl_qty_t *locked) 2500 { 2501 struct shm_data *shmd = seg->s_data; 2502 struct spt_data *sptd = shmd->shm_sptseg->s_data; 2503 ulong_t i; 2504 int kernel; 2505 pgcnt_t nlck = 0; 2506 int rv = 0; 2507 int use_reserved = 1; 2508 2509 /* return the number of bytes actually locked */ 2510 *locked = 0; 2511 2512 /* 2513 * To avoid contention on freemem_lock, availrmem and pages_locked 2514 * global counters are updated only every nlck locked pages instead of 2515 * every time. Reserve nlck locks up front and deduct from this 2516 * reservation for each page that requires a lock. When the reservation 2517 * is consumed, reserve again. nlck is randomized, so the competing 2518 * threads do not fall into a cyclic lock contention pattern. When 2519 * memory is low, the lock ahead is disabled, and instead page_pp_lock() 2520 * is used to lock pages. 2521 */ 2522 for (i = 0; i < npages; anon_index++, pos++, i++) { 2523 if (nlck == 0 && use_reserved == 1) { 2524 nlck = NLCK + RAND_P2(NLCK); 2525 /* if fewer loops left, decrease nlck */ 2526 nlck = MIN(nlck, npages - i); 2527 /* 2528 * Reserve nlck locks up front and deduct from this 2529 * reservation for each page that requires a lock. When 2530 * the reservation is consumed, reserve again. 2531 */ 2532 mutex_enter(&freemem_lock); 2533 if ((availrmem - nlck) < pages_pp_maximum) { 2534 /* Do not do advance memory reserves */ 2535 use_reserved = 0; 2536 } else { 2537 availrmem -= nlck; 2538 pages_locked += nlck; 2539 } 2540 mutex_exit(&freemem_lock); 2541 } 2542 if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) { 2543 if (sptd->spt_ppa_lckcnt[anon_index] < 2544 (ushort_t)DISM_LOCK_MAX) { 2545 if (++sptd->spt_ppa_lckcnt[anon_index] == 2546 (ushort_t)DISM_LOCK_MAX) { 2547 cmn_err(CE_WARN, 2548 "DISM page lock limit " 2549 "reached on DISM offset 0x%lx\n", 2550 anon_index << PAGESHIFT); 2551 } 2552 kernel = (sptd->spt_ppa && 2553 sptd->spt_ppa[anon_index]); 2554 if (!page_pp_lock(ppa[i], 0, kernel || 2555 use_reserved)) { 2556 sptd->spt_ppa_lckcnt[anon_index]--; 2557 rv = EAGAIN; 2558 break; 2559 } 2560 /* if this is a newly locked page, count it */ 2561 if (ppa[i]->p_lckcnt == 1) { 2562 if (kernel == 0 && use_reserved == 1) 2563 nlck--; 2564 *locked += PAGESIZE; 2565 } 2566 shmd->shm_lckpgs++; 2567 shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED; 2568 if (lockmap != NULL) 2569 BT_SET(lockmap, pos); 2570 } 2571 } 2572 } 2573 /* Return unused lock reservation */ 2574 if (nlck != 0 && use_reserved == 1) { 2575 mutex_enter(&freemem_lock); 2576 availrmem += nlck; 2577 pages_locked -= nlck; 2578 mutex_exit(&freemem_lock); 2579 } 2580 2581 return (rv); 2582 } 2583 2584 int 2585 spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 2586 rctl_qty_t *unlocked) 2587 { 2588 struct shm_data *shmd = seg->s_data; 2589 struct spt_data *sptd = shmd->shm_sptseg->s_data; 2590 struct anon_map *amp = sptd->spt_amp; 2591 struct anon *ap; 2592 struct vnode *vp; 2593 u_offset_t off; 2594 struct page *pp; 2595 int kernel; 2596 anon_sync_obj_t cookie; 2597 ulong_t i; 2598 pgcnt_t nlck = 0; 2599 pgcnt_t nlck_limit = NLCK; 2600 2601 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2602 for (i = 0; i < npages; i++, anon_index++) { 2603 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 2604 anon_array_enter(amp, anon_index, &cookie); 2605 ap = anon_get_ptr(amp->ahp, anon_index); 2606 ASSERT(ap); 2607 2608 swap_xlate(ap, &vp, &off); 2609 anon_array_exit(&cookie); 2610 pp = page_lookup(vp, off, SE_SHARED); 2611 ASSERT(pp); 2612 /* 2613 * availrmem is decremented only for pages which are not 2614 * in seg pcache, for pages in seg pcache availrmem was 2615 * decremented in _dismpagelock() 2616 */ 2617 kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]); 2618 ASSERT(pp->p_lckcnt > 0); 2619 2620 /* 2621 * lock page but do not change availrmem, we do it 2622 * ourselves every nlck loops. 2623 */ 2624 page_pp_unlock(pp, 0, 1); 2625 if (pp->p_lckcnt == 0) { 2626 if (kernel == 0) 2627 nlck++; 2628 *unlocked += PAGESIZE; 2629 } 2630 page_unlock(pp); 2631 shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED; 2632 sptd->spt_ppa_lckcnt[anon_index]--; 2633 shmd->shm_lckpgs--; 2634 } 2635 2636 /* 2637 * To reduce freemem_lock contention, do not update availrmem 2638 * until at least NLCK pages have been unlocked. 2639 * 1. No need to update if nlck is zero 2640 * 2. Always update if the last iteration 2641 */ 2642 if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) { 2643 mutex_enter(&freemem_lock); 2644 availrmem += nlck; 2645 pages_locked -= nlck; 2646 mutex_exit(&freemem_lock); 2647 nlck = 0; 2648 nlck_limit = NLCK + RAND_P2(NLCK); 2649 } 2650 } 2651 ANON_LOCK_EXIT(&->a_rwlock); 2652 2653 return (0); 2654 } 2655 2656 /*ARGSUSED*/ 2657 static int 2658 segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 2659 int attr, int op, ulong_t *lockmap, size_t pos) 2660 { 2661 struct shm_data *shmd = seg->s_data; 2662 struct seg *sptseg = shmd->shm_sptseg; 2663 struct spt_data *sptd = sptseg->s_data; 2664 struct kshmid *sp = sptd->spt_amp->a_sp; 2665 pgcnt_t npages, a_npages; 2666 page_t **ppa; 2667 pgcnt_t an_idx, a_an_idx, ppa_idx; 2668 caddr_t spt_addr, a_addr; /* spt and aligned address */ 2669 size_t a_len; /* aligned len */ 2670 size_t share_sz; 2671 ulong_t i; 2672 int sts = 0; 2673 rctl_qty_t unlocked = 0; 2674 rctl_qty_t locked = 0; 2675 struct proc *p = curproc; 2676 kproject_t *proj; 2677 2678 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2679 ASSERT(sp != NULL); 2680 2681 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 2682 return (0); 2683 } 2684 2685 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2686 an_idx = seg_page(seg, addr); 2687 npages = btopr(len); 2688 2689 if (an_idx + npages > btopr(shmd->shm_amp->size)) { 2690 return (ENOMEM); 2691 } 2692 2693 /* 2694 * A shm's project never changes, so no lock needed. 2695 * The shm has a hold on the project, so it will not go away. 2696 * Since we have a mapping to shm within this zone, we know 2697 * that the zone will not go away. 2698 */ 2699 proj = sp->shm_perm.ipc_proj; 2700 2701 if (op == MC_LOCK) { 2702 2703 /* 2704 * Need to align addr and size request if they are not 2705 * aligned so we can always allocate large page(s) however 2706 * we only lock what was requested in initial request. 2707 */ 2708 share_sz = page_get_pagesize(sptseg->s_szc); 2709 a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 2710 a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)), 2711 share_sz); 2712 a_npages = btop(a_len); 2713 a_an_idx = seg_page(seg, a_addr); 2714 spt_addr = sptseg->s_base + ptob(a_an_idx); 2715 ppa_idx = an_idx - a_an_idx; 2716 2717 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages), 2718 KM_NOSLEEP)) == NULL) { 2719 return (ENOMEM); 2720 } 2721 2722 /* 2723 * Don't cache any new pages for IO and 2724 * flush any cached pages. 2725 */ 2726 mutex_enter(&sptd->spt_lock); 2727 if (sptd->spt_ppa != NULL) 2728 sptd->spt_flags |= DISM_PPA_CHANGED; 2729 2730 sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa); 2731 if (sts != 0) { 2732 mutex_exit(&sptd->spt_lock); 2733 kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 2734 return (sts); 2735 } 2736 2737 mutex_enter(&sp->shm_mlock); 2738 /* enforce locked memory rctl */ 2739 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]); 2740 2741 mutex_enter(&p->p_lock); 2742 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) { 2743 mutex_exit(&p->p_lock); 2744 sts = EAGAIN; 2745 } else { 2746 mutex_exit(&p->p_lock); 2747 sts = spt_lockpages(seg, an_idx, npages, 2748 &ppa[ppa_idx], lockmap, pos, &locked); 2749 2750 /* 2751 * correct locked count if not all pages could be 2752 * locked 2753 */ 2754 if ((unlocked - locked) > 0) { 2755 rctl_decr_locked_mem(NULL, proj, 2756 (unlocked - locked), 0); 2757 } 2758 } 2759 /* 2760 * unlock pages 2761 */ 2762 for (i = 0; i < a_npages; i++) 2763 page_unlock(ppa[i]); 2764 if (sptd->spt_ppa != NULL) 2765 sptd->spt_flags |= DISM_PPA_CHANGED; 2766 mutex_exit(&sp->shm_mlock); 2767 mutex_exit(&sptd->spt_lock); 2768 2769 kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 2770 2771 } else if (op == MC_UNLOCK) { /* unlock */ 2772 page_t **ppa; 2773 2774 mutex_enter(&sptd->spt_lock); 2775 if (shmd->shm_lckpgs == 0) { 2776 mutex_exit(&sptd->spt_lock); 2777 return (0); 2778 } 2779 /* 2780 * Don't cache new IO pages. 2781 */ 2782 if (sptd->spt_ppa != NULL) 2783 sptd->spt_flags |= DISM_PPA_CHANGED; 2784 2785 mutex_enter(&sp->shm_mlock); 2786 sts = spt_unlockpages(seg, an_idx, npages, &unlocked); 2787 if ((ppa = sptd->spt_ppa) != NULL) 2788 sptd->spt_flags |= DISM_PPA_CHANGED; 2789 mutex_exit(&sptd->spt_lock); 2790 2791 rctl_decr_locked_mem(NULL, proj, unlocked, 0); 2792 mutex_exit(&sp->shm_mlock); 2793 2794 if (ppa != NULL) 2795 seg_ppurge_wiredpp(ppa); 2796 } 2797 return (sts); 2798 } 2799 2800 /*ARGSUSED*/ 2801 int 2802 segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 2803 { 2804 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2805 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2806 spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1; 2807 2808 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2809 2810 /* 2811 * ISM segment is always rw. 2812 */ 2813 while (--pgno >= 0) 2814 *protv++ = sptd->spt_prot; 2815 return (0); 2816 } 2817 2818 /*ARGSUSED*/ 2819 u_offset_t 2820 segspt_shmgetoffset(struct seg *seg, caddr_t addr) 2821 { 2822 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2823 2824 /* Offset does not matter in ISM memory */ 2825 2826 return ((u_offset_t)0); 2827 } 2828 2829 /* ARGSUSED */ 2830 int 2831 segspt_shmgettype(struct seg *seg, caddr_t addr) 2832 { 2833 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2834 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2835 2836 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2837 2838 /* 2839 * The shared memory mapping is always MAP_SHARED, SWAP is only 2840 * reserved for DISM 2841 */ 2842 return (MAP_SHARED | 2843 ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE)); 2844 } 2845 2846 /*ARGSUSED*/ 2847 int 2848 segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 2849 { 2850 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2851 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2852 2853 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2854 2855 *vpp = sptd->spt_vp; 2856 return (0); 2857 } 2858 2859 /* 2860 * We need to wait for pending IO to complete to a DISM segment in order for 2861 * pages to get kicked out of the seg_pcache. 120 seconds should be more 2862 * than enough time to wait. 2863 */ 2864 static clock_t spt_pcache_wait = 120; 2865 2866 /*ARGSUSED*/ 2867 static int 2868 segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 2869 { 2870 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2871 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2872 struct anon_map *amp; 2873 pgcnt_t pg_idx; 2874 ushort_t gen; 2875 clock_t end_lbolt; 2876 int writer; 2877 page_t **ppa; 2878 2879 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2880 2881 if (behav == MADV_FREE) { 2882 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) 2883 return (0); 2884 2885 amp = sptd->spt_amp; 2886 pg_idx = seg_page(seg, addr); 2887 2888 mutex_enter(&sptd->spt_lock); 2889 if ((ppa = sptd->spt_ppa) == NULL) { 2890 mutex_exit(&sptd->spt_lock); 2891 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2892 anon_disclaim(amp, pg_idx, len); 2893 ANON_LOCK_EXIT(&->a_rwlock); 2894 return (0); 2895 } 2896 2897 sptd->spt_flags |= DISM_PPA_CHANGED; 2898 gen = sptd->spt_gen; 2899 2900 mutex_exit(&sptd->spt_lock); 2901 2902 /* 2903 * Purge all DISM cached pages 2904 */ 2905 seg_ppurge_wiredpp(ppa); 2906 2907 /* 2908 * Drop the AS_LOCK so that other threads can grab it 2909 * in the as_pageunlock path and hopefully get the segment 2910 * kicked out of the seg_pcache. We bump the shm_softlockcnt 2911 * to keep this segment resident. 2912 */ 2913 writer = AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock); 2914 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 2915 AS_LOCK_EXIT(seg->s_as, &seg->s_as->a_lock); 2916 2917 mutex_enter(&sptd->spt_lock); 2918 2919 end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait); 2920 2921 /* 2922 * Try to wait for pages to get kicked out of the seg_pcache. 2923 */ 2924 while (sptd->spt_gen == gen && 2925 (sptd->spt_flags & DISM_PPA_CHANGED) && 2926 ddi_get_lbolt() < end_lbolt) { 2927 if (!cv_timedwait_sig(&sptd->spt_cv, 2928 &sptd->spt_lock, end_lbolt)) { 2929 break; 2930 } 2931 } 2932 2933 mutex_exit(&sptd->spt_lock); 2934 2935 /* Regrab the AS_LOCK and release our hold on the segment */ 2936 AS_LOCK_ENTER(seg->s_as, &seg->s_as->a_lock, 2937 writer ? RW_WRITER : RW_READER); 2938 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 2939 if (shmd->shm_softlockcnt <= 0) { 2940 if (AS_ISUNMAPWAIT(seg->s_as)) { 2941 mutex_enter(&seg->s_as->a_contents); 2942 if (AS_ISUNMAPWAIT(seg->s_as)) { 2943 AS_CLRUNMAPWAIT(seg->s_as); 2944 cv_broadcast(&seg->s_as->a_cv); 2945 } 2946 mutex_exit(&seg->s_as->a_contents); 2947 } 2948 } 2949 2950 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2951 anon_disclaim(amp, pg_idx, len); 2952 ANON_LOCK_EXIT(&->a_rwlock); 2953 } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP || 2954 behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) { 2955 int already_set; 2956 ulong_t anon_index; 2957 lgrp_mem_policy_t policy; 2958 caddr_t shm_addr; 2959 size_t share_size; 2960 size_t size; 2961 struct seg *sptseg = shmd->shm_sptseg; 2962 caddr_t sptseg_addr; 2963 2964 /* 2965 * Align address and length to page size of underlying segment 2966 */ 2967 share_size = page_get_pagesize(shmd->shm_sptseg->s_szc); 2968 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 2969 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), 2970 share_size); 2971 2972 amp = shmd->shm_amp; 2973 anon_index = seg_page(seg, shm_addr); 2974 2975 /* 2976 * And now we may have to adjust size downward if we have 2977 * exceeded the realsize of the segment or initial anon 2978 * allocations. 2979 */ 2980 sptseg_addr = sptseg->s_base + ptob(anon_index); 2981 if ((sptseg_addr + size) > 2982 (sptseg->s_base + sptd->spt_realsize)) 2983 size = (sptseg->s_base + sptd->spt_realsize) - 2984 sptseg_addr; 2985 2986 /* 2987 * Set memory allocation policy for this segment 2988 */ 2989 policy = lgrp_madv_to_policy(behav, len, MAP_SHARED); 2990 already_set = lgrp_shm_policy_set(policy, amp, anon_index, 2991 NULL, 0, len); 2992 2993 /* 2994 * If random memory allocation policy set already, 2995 * don't bother reapplying it. 2996 */ 2997 if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy)) 2998 return (0); 2999 3000 /* 3001 * Mark any existing pages in the given range for 3002 * migration, flushing the I/O page cache, and using 3003 * underlying segment to calculate anon index and get 3004 * anonmap and vnode pointer from 3005 */ 3006 if (shmd->shm_softlockcnt > 0) 3007 segspt_purge(seg); 3008 3009 page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0); 3010 } 3011 3012 return (0); 3013 } 3014 3015 /*ARGSUSED*/ 3016 void 3017 segspt_shmdump(struct seg *seg) 3018 { 3019 /* no-op for ISM segment */ 3020 } 3021 3022 /*ARGSUSED*/ 3023 static faultcode_t 3024 segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc) 3025 { 3026 return (ENOTSUP); 3027 } 3028 3029 /* 3030 * get a memory ID for an addr in a given segment 3031 */ 3032 static int 3033 segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 3034 { 3035 struct shm_data *shmd = (struct shm_data *)seg->s_data; 3036 struct anon *ap; 3037 size_t anon_index; 3038 struct anon_map *amp = shmd->shm_amp; 3039 struct spt_data *sptd = shmd->shm_sptseg->s_data; 3040 struct seg *sptseg = shmd->shm_sptseg; 3041 anon_sync_obj_t cookie; 3042 3043 anon_index = seg_page(seg, addr); 3044 3045 if (addr > (seg->s_base + sptd->spt_realsize)) { 3046 return (EFAULT); 3047 } 3048 3049 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 3050 anon_array_enter(amp, anon_index, &cookie); 3051 ap = anon_get_ptr(amp->ahp, anon_index); 3052 if (ap == NULL) { 3053 struct page *pp; 3054 caddr_t spt_addr = sptseg->s_base + ptob(anon_index); 3055 3056 pp = anon_zero(sptseg, spt_addr, &ap, kcred); 3057 if (pp == NULL) { 3058 anon_array_exit(&cookie); 3059 ANON_LOCK_EXIT(&->a_rwlock); 3060 return (ENOMEM); 3061 } 3062 (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP); 3063 page_unlock(pp); 3064 } 3065 anon_array_exit(&cookie); 3066 ANON_LOCK_EXIT(&->a_rwlock); 3067 memidp->val[0] = (uintptr_t)ap; 3068 memidp->val[1] = (uintptr_t)addr & PAGEOFFSET; 3069 return (0); 3070 } 3071 3072 /* 3073 * Get memory allocation policy info for specified address in given segment 3074 */ 3075 static lgrp_mem_policy_info_t * 3076 segspt_shmgetpolicy(struct seg *seg, caddr_t addr) 3077 { 3078 struct anon_map *amp; 3079 ulong_t anon_index; 3080 lgrp_mem_policy_info_t *policy_info; 3081 struct shm_data *shm_data; 3082 3083 ASSERT(seg != NULL); 3084 3085 /* 3086 * Get anon_map from segshm 3087 * 3088 * Assume that no lock needs to be held on anon_map, since 3089 * it should be protected by its reference count which must be 3090 * nonzero for an existing segment 3091 * Need to grab readers lock on policy tree though 3092 */ 3093 shm_data = (struct shm_data *)seg->s_data; 3094 if (shm_data == NULL) 3095 return (NULL); 3096 amp = shm_data->shm_amp; 3097 ASSERT(amp->refcnt != 0); 3098 3099 /* 3100 * Get policy info 3101 * 3102 * Assume starting anon index of 0 3103 */ 3104 anon_index = seg_page(seg, addr); 3105 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 3106 3107 return (policy_info); 3108 } 3109 3110 /*ARGSUSED*/ 3111 static int 3112 segspt_shmcapable(struct seg *seg, segcapability_t capability) 3113 { 3114 return (0); 3115 } 3116