17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*e67882ffSbs21162 * Common Development and Distribution License (the "License"). 6*e67882ffSbs21162 * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*e67882ffSbs21162 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/param.h> 297c478bd9Sstevel@tonic-gate #include <sys/user.h> 307c478bd9Sstevel@tonic-gate #include <sys/mman.h> 317c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 327c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 337c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 347c478bd9Sstevel@tonic-gate #include <sys/systm.h> 357c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 367c478bd9Sstevel@tonic-gate #include <vm/hat.h> 377c478bd9Sstevel@tonic-gate #include <vm/seg.h> 387c478bd9Sstevel@tonic-gate #include <vm/as.h> 397c478bd9Sstevel@tonic-gate #include <vm/anon.h> 407c478bd9Sstevel@tonic-gate #include <vm/page.h> 417c478bd9Sstevel@tonic-gate #include <sys/buf.h> 427c478bd9Sstevel@tonic-gate #include <sys/swap.h> 437c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 447c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 457c478bd9Sstevel@tonic-gate #include <sys/debug.h> 467c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 477c478bd9Sstevel@tonic-gate #include <sys/shm.h> 487c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 497c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 507c478bd9Sstevel@tonic-gate 517c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h> 527c478bd9Sstevel@tonic-gate 537c478bd9Sstevel@tonic-gate #define SEGSPTADDR (caddr_t)0x0 547c478bd9Sstevel@tonic-gate 557c478bd9Sstevel@tonic-gate /* 567c478bd9Sstevel@tonic-gate * # pages used for spt 577c478bd9Sstevel@tonic-gate */ 587c478bd9Sstevel@tonic-gate static size_t spt_used; 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate /* 617c478bd9Sstevel@tonic-gate * segspt_minfree is the memory left for system after ISM 627c478bd9Sstevel@tonic-gate * locked its pages; it is set up to 5% of availrmem in 637c478bd9Sstevel@tonic-gate * sptcreate when ISM is created. ISM should not use more 647c478bd9Sstevel@tonic-gate * than ~90% of availrmem; if it does, then the performance 657c478bd9Sstevel@tonic-gate * of the system may decrease. Machines with large memories may 667c478bd9Sstevel@tonic-gate * be able to use up more memory for ISM so we set the default 677c478bd9Sstevel@tonic-gate * segspt_minfree to 5% (which gives ISM max 95% of availrmem. 687c478bd9Sstevel@tonic-gate * If somebody wants even more memory for ISM (risking hanging 697c478bd9Sstevel@tonic-gate * the system) they can patch the segspt_minfree to smaller number. 707c478bd9Sstevel@tonic-gate */ 717c478bd9Sstevel@tonic-gate pgcnt_t segspt_minfree = 0; 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate static int segspt_create(struct seg *seg, caddr_t argsp); 747c478bd9Sstevel@tonic-gate static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize); 757c478bd9Sstevel@tonic-gate static void segspt_free(struct seg *seg); 767c478bd9Sstevel@tonic-gate static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len); 777c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr); 787c478bd9Sstevel@tonic-gate 797c478bd9Sstevel@tonic-gate static void 807c478bd9Sstevel@tonic-gate segspt_badop() 817c478bd9Sstevel@tonic-gate { 827c478bd9Sstevel@tonic-gate panic("segspt_badop called"); 837c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 847c478bd9Sstevel@tonic-gate } 857c478bd9Sstevel@tonic-gate 867c478bd9Sstevel@tonic-gate #define SEGSPT_BADOP(t) (t(*)())segspt_badop 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate struct seg_ops segspt_ops = { 897c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* dup */ 907c478bd9Sstevel@tonic-gate segspt_unmap, 917c478bd9Sstevel@tonic-gate segspt_free, 927c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* fault */ 937c478bd9Sstevel@tonic-gate SEGSPT_BADOP(faultcode_t), /* faulta */ 947c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* setprot */ 957c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* checkprot */ 967c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* kluster */ 977c478bd9Sstevel@tonic-gate SEGSPT_BADOP(size_t), /* swapout */ 987c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* sync */ 997c478bd9Sstevel@tonic-gate SEGSPT_BADOP(size_t), /* incore */ 1007c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* lockop */ 1017c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* getprot */ 1027c478bd9Sstevel@tonic-gate SEGSPT_BADOP(u_offset_t), /* getoffset */ 1037c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* gettype */ 1047c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* getvp */ 1057c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* advise */ 1067c478bd9Sstevel@tonic-gate SEGSPT_BADOP(void), /* dump */ 1077c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* pagelock */ 1087c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* setpgsz */ 1097c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* getmemid */ 1107c478bd9Sstevel@tonic-gate segspt_getpolicy, /* getpolicy */ 1111bd5c35fSelowe SEGSPT_BADOP(int), /* capable */ 1127c478bd9Sstevel@tonic-gate }; 1137c478bd9Sstevel@tonic-gate 1147c478bd9Sstevel@tonic-gate static int segspt_shmdup(struct seg *seg, struct seg *newseg); 1157c478bd9Sstevel@tonic-gate static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize); 1167c478bd9Sstevel@tonic-gate static void segspt_shmfree(struct seg *seg); 1177c478bd9Sstevel@tonic-gate static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg, 1187c478bd9Sstevel@tonic-gate caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); 1197c478bd9Sstevel@tonic-gate static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr); 1207c478bd9Sstevel@tonic-gate static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr, 1217c478bd9Sstevel@tonic-gate register size_t len, register uint_t prot); 1227c478bd9Sstevel@tonic-gate static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, 1237c478bd9Sstevel@tonic-gate uint_t prot); 1247c478bd9Sstevel@tonic-gate static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta); 1257c478bd9Sstevel@tonic-gate static size_t segspt_shmswapout(struct seg *seg); 1267c478bd9Sstevel@tonic-gate static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, 1277c478bd9Sstevel@tonic-gate register char *vec); 1287c478bd9Sstevel@tonic-gate static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len, 1297c478bd9Sstevel@tonic-gate int attr, uint_t flags); 1307c478bd9Sstevel@tonic-gate static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 1317c478bd9Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos); 1327c478bd9Sstevel@tonic-gate static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, 1337c478bd9Sstevel@tonic-gate uint_t *protv); 1347c478bd9Sstevel@tonic-gate static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr); 1357c478bd9Sstevel@tonic-gate static int segspt_shmgettype(struct seg *seg, caddr_t addr); 1367c478bd9Sstevel@tonic-gate static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 1377c478bd9Sstevel@tonic-gate static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, 1387c478bd9Sstevel@tonic-gate uint_t behav); 1397c478bd9Sstevel@tonic-gate static void segspt_shmdump(struct seg *seg); 1407c478bd9Sstevel@tonic-gate static int segspt_shmpagelock(struct seg *, caddr_t, size_t, 1417c478bd9Sstevel@tonic-gate struct page ***, enum lock_type, enum seg_rw); 1427c478bd9Sstevel@tonic-gate static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t); 1437c478bd9Sstevel@tonic-gate static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *); 1447c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t); 1451bd5c35fSelowe static int segspt_shmcapable(struct seg *, segcapability_t); 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate struct seg_ops segspt_shmops = { 1487c478bd9Sstevel@tonic-gate segspt_shmdup, 1497c478bd9Sstevel@tonic-gate segspt_shmunmap, 1507c478bd9Sstevel@tonic-gate segspt_shmfree, 1517c478bd9Sstevel@tonic-gate segspt_shmfault, 1527c478bd9Sstevel@tonic-gate segspt_shmfaulta, 1537c478bd9Sstevel@tonic-gate segspt_shmsetprot, 1547c478bd9Sstevel@tonic-gate segspt_shmcheckprot, 1557c478bd9Sstevel@tonic-gate segspt_shmkluster, 1567c478bd9Sstevel@tonic-gate segspt_shmswapout, 1577c478bd9Sstevel@tonic-gate segspt_shmsync, 1587c478bd9Sstevel@tonic-gate segspt_shmincore, 1597c478bd9Sstevel@tonic-gate segspt_shmlockop, 1607c478bd9Sstevel@tonic-gate segspt_shmgetprot, 1617c478bd9Sstevel@tonic-gate segspt_shmgetoffset, 1627c478bd9Sstevel@tonic-gate segspt_shmgettype, 1637c478bd9Sstevel@tonic-gate segspt_shmgetvp, 1647c478bd9Sstevel@tonic-gate segspt_shmadvise, /* advise */ 1657c478bd9Sstevel@tonic-gate segspt_shmdump, 1667c478bd9Sstevel@tonic-gate segspt_shmpagelock, 1677c478bd9Sstevel@tonic-gate segspt_shmsetpgsz, 1687c478bd9Sstevel@tonic-gate segspt_shmgetmemid, 1697c478bd9Sstevel@tonic-gate segspt_shmgetpolicy, 1701bd5c35fSelowe segspt_shmcapable, 1717c478bd9Sstevel@tonic-gate }; 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate static void segspt_purge(struct seg *seg); 1747c478bd9Sstevel@tonic-gate static int segspt_reclaim(struct seg *, caddr_t, size_t, struct page **, 1757c478bd9Sstevel@tonic-gate enum seg_rw); 1767c478bd9Sstevel@tonic-gate static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len, 1777c478bd9Sstevel@tonic-gate page_t **ppa); 1787c478bd9Sstevel@tonic-gate 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate 1817c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1827c478bd9Sstevel@tonic-gate int 1837c478bd9Sstevel@tonic-gate sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp, 1847c478bd9Sstevel@tonic-gate uint_t prot, uint_t flags, uint_t share_szc) 1857c478bd9Sstevel@tonic-gate { 1867c478bd9Sstevel@tonic-gate int err; 1877c478bd9Sstevel@tonic-gate struct as *newas; 1887c478bd9Sstevel@tonic-gate struct segspt_crargs sptcargs; 1897c478bd9Sstevel@tonic-gate 1907c478bd9Sstevel@tonic-gate #ifdef DEBUG 1917c478bd9Sstevel@tonic-gate TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */, 1927c478bd9Sstevel@tonic-gate tnf_ulong, size, size ); 1937c478bd9Sstevel@tonic-gate #endif 1947c478bd9Sstevel@tonic-gate if (segspt_minfree == 0) /* leave min 5% of availrmem for */ 1957c478bd9Sstevel@tonic-gate segspt_minfree = availrmem/20; /* for the system */ 1967c478bd9Sstevel@tonic-gate 1977c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_SHARED_PT, (void *)0)) 1987c478bd9Sstevel@tonic-gate return (EINVAL); 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * get a new as for this shared memory segment 2027c478bd9Sstevel@tonic-gate */ 2037c478bd9Sstevel@tonic-gate newas = as_alloc(); 2047c478bd9Sstevel@tonic-gate sptcargs.amp = amp; 2057c478bd9Sstevel@tonic-gate sptcargs.prot = prot; 2067c478bd9Sstevel@tonic-gate sptcargs.flags = flags; 2077c478bd9Sstevel@tonic-gate sptcargs.szc = share_szc; 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate /* 2107c478bd9Sstevel@tonic-gate * create a shared page table (spt) segment 2117c478bd9Sstevel@tonic-gate */ 2127c478bd9Sstevel@tonic-gate 2137c478bd9Sstevel@tonic-gate if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) { 2147c478bd9Sstevel@tonic-gate as_free(newas); 2157c478bd9Sstevel@tonic-gate return (err); 2167c478bd9Sstevel@tonic-gate } 2177c478bd9Sstevel@tonic-gate *sptseg = sptcargs.seg_spt; 2187c478bd9Sstevel@tonic-gate return (0); 2197c478bd9Sstevel@tonic-gate } 2207c478bd9Sstevel@tonic-gate 2217c478bd9Sstevel@tonic-gate void 2227c478bd9Sstevel@tonic-gate sptdestroy(struct as *as, struct anon_map *amp) 2237c478bd9Sstevel@tonic-gate { 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate #ifdef DEBUG 2267c478bd9Sstevel@tonic-gate TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */); 2277c478bd9Sstevel@tonic-gate #endif 2287c478bd9Sstevel@tonic-gate (void) as_unmap(as, SEGSPTADDR, amp->size); 2297c478bd9Sstevel@tonic-gate as_free(as); 2307c478bd9Sstevel@tonic-gate } 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate /* 2337c478bd9Sstevel@tonic-gate * called from seg_free(). 2347c478bd9Sstevel@tonic-gate * free (i.e., unlock, unmap, return to free list) 2357c478bd9Sstevel@tonic-gate * all the pages in the given seg. 2367c478bd9Sstevel@tonic-gate */ 2377c478bd9Sstevel@tonic-gate void 2387c478bd9Sstevel@tonic-gate segspt_free(struct seg *seg) 2397c478bd9Sstevel@tonic-gate { 2407c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data; 2417c478bd9Sstevel@tonic-gate 2427c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate if (sptd != NULL) { 2457c478bd9Sstevel@tonic-gate if (sptd->spt_realsize) 2467c478bd9Sstevel@tonic-gate segspt_free_pages(seg, seg->s_base, sptd->spt_realsize); 2477c478bd9Sstevel@tonic-gate 2487c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt) 2497c478bd9Sstevel@tonic-gate kmem_free(sptd->spt_ppa_lckcnt, 2507c478bd9Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt) 2517c478bd9Sstevel@tonic-gate * btopr(sptd->spt_amp->size)); 2527c478bd9Sstevel@tonic-gate kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp)); 2537c478bd9Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock); 2547c478bd9Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd)); 2557c478bd9Sstevel@tonic-gate } 2567c478bd9Sstevel@tonic-gate } 2577c478bd9Sstevel@tonic-gate 2587c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2597c478bd9Sstevel@tonic-gate static int 2607c478bd9Sstevel@tonic-gate segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr, 2617c478bd9Sstevel@tonic-gate uint_t flags) 2627c478bd9Sstevel@tonic-gate { 2637c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2647c478bd9Sstevel@tonic-gate 2657c478bd9Sstevel@tonic-gate return (0); 2667c478bd9Sstevel@tonic-gate } 2677c478bd9Sstevel@tonic-gate 2687c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2697c478bd9Sstevel@tonic-gate static size_t 2707c478bd9Sstevel@tonic-gate segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec) 2717c478bd9Sstevel@tonic-gate { 2727c478bd9Sstevel@tonic-gate caddr_t eo_seg; 2737c478bd9Sstevel@tonic-gate pgcnt_t npages; 2747c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 2757c478bd9Sstevel@tonic-gate struct seg *sptseg; 2767c478bd9Sstevel@tonic-gate struct spt_data *sptd; 2777c478bd9Sstevel@tonic-gate 2787c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2797c478bd9Sstevel@tonic-gate #ifdef lint 2807c478bd9Sstevel@tonic-gate seg = seg; 2817c478bd9Sstevel@tonic-gate #endif 2827c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 2837c478bd9Sstevel@tonic-gate sptd = sptseg->s_data; 2847c478bd9Sstevel@tonic-gate 2857c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 2867c478bd9Sstevel@tonic-gate eo_seg = addr + len; 2877c478bd9Sstevel@tonic-gate while (addr < eo_seg) { 2887c478bd9Sstevel@tonic-gate /* page exists, and it's locked. */ 2897c478bd9Sstevel@tonic-gate *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED | 2907c478bd9Sstevel@tonic-gate SEG_PAGE_ANON; 2917c478bd9Sstevel@tonic-gate addr += PAGESIZE; 2927c478bd9Sstevel@tonic-gate } 2937c478bd9Sstevel@tonic-gate return (len); 2947c478bd9Sstevel@tonic-gate } else { 2957c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 2967c478bd9Sstevel@tonic-gate struct anon *ap; 2977c478bd9Sstevel@tonic-gate page_t *pp; 2987c478bd9Sstevel@tonic-gate pgcnt_t anon_index; 2997c478bd9Sstevel@tonic-gate struct vnode *vp; 3007c478bd9Sstevel@tonic-gate u_offset_t off; 3017c478bd9Sstevel@tonic-gate ulong_t i; 3027c478bd9Sstevel@tonic-gate int ret; 3037c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 3047c478bd9Sstevel@tonic-gate 3057c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3067c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 3077c478bd9Sstevel@tonic-gate npages = btopr(len); 3087c478bd9Sstevel@tonic-gate if (anon_index + npages > btopr(shmd->shm_amp->size)) { 3097c478bd9Sstevel@tonic-gate return (EINVAL); 3107c478bd9Sstevel@tonic-gate } 3117c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 3127c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++, anon_index++) { 3137c478bd9Sstevel@tonic-gate ret = 0; 3147c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 3157c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 3167c478bd9Sstevel@tonic-gate if (ap != NULL) { 3177c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 3187c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 3197c478bd9Sstevel@tonic-gate pp = page_lookup_nowait(vp, off, SE_SHARED); 3207c478bd9Sstevel@tonic-gate if (pp != NULL) { 3217c478bd9Sstevel@tonic-gate ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON; 3227c478bd9Sstevel@tonic-gate page_unlock(pp); 3237c478bd9Sstevel@tonic-gate } 3247c478bd9Sstevel@tonic-gate } else { 3257c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 3267c478bd9Sstevel@tonic-gate } 3277c478bd9Sstevel@tonic-gate if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 3287c478bd9Sstevel@tonic-gate ret |= SEG_PAGE_LOCKED; 3297c478bd9Sstevel@tonic-gate } 3307c478bd9Sstevel@tonic-gate *vec++ = (char)ret; 3317c478bd9Sstevel@tonic-gate } 3327c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 3337c478bd9Sstevel@tonic-gate return (len); 3347c478bd9Sstevel@tonic-gate } 3357c478bd9Sstevel@tonic-gate } 3367c478bd9Sstevel@tonic-gate 3377c478bd9Sstevel@tonic-gate static int 3387c478bd9Sstevel@tonic-gate segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize) 3397c478bd9Sstevel@tonic-gate { 3407c478bd9Sstevel@tonic-gate size_t share_size; 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3437c478bd9Sstevel@tonic-gate 3447c478bd9Sstevel@tonic-gate /* 3457c478bd9Sstevel@tonic-gate * seg.s_size may have been rounded up to the largest page size 3467c478bd9Sstevel@tonic-gate * in shmat(). 3477c478bd9Sstevel@tonic-gate * XXX This should be cleanedup. sptdestroy should take a length 3487c478bd9Sstevel@tonic-gate * argument which should be the same as sptcreate. Then 3497c478bd9Sstevel@tonic-gate * this rounding would not be needed (or is done in shm.c) 3507c478bd9Sstevel@tonic-gate * Only the check for full segment will be needed. 3517c478bd9Sstevel@tonic-gate * 3527c478bd9Sstevel@tonic-gate * XXX -- shouldn't raddr == 0 always? These tests don't seem 3537c478bd9Sstevel@tonic-gate * to be useful at all. 3547c478bd9Sstevel@tonic-gate */ 3557c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(seg->s_szc); 3567c478bd9Sstevel@tonic-gate ssize = P2ROUNDUP(ssize, share_size); 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate if (raddr == seg->s_base && ssize == seg->s_size) { 3597c478bd9Sstevel@tonic-gate seg_free(seg); 3607c478bd9Sstevel@tonic-gate return (0); 3617c478bd9Sstevel@tonic-gate } else 3627c478bd9Sstevel@tonic-gate return (EINVAL); 3637c478bd9Sstevel@tonic-gate } 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate int 3667c478bd9Sstevel@tonic-gate segspt_create(struct seg *seg, caddr_t argsp) 3677c478bd9Sstevel@tonic-gate { 3687c478bd9Sstevel@tonic-gate int err; 3697c478bd9Sstevel@tonic-gate caddr_t addr = seg->s_base; 3707c478bd9Sstevel@tonic-gate struct spt_data *sptd; 3717c478bd9Sstevel@tonic-gate struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp; 3727c478bd9Sstevel@tonic-gate struct anon_map *amp = sptcargs->amp; 3737c478bd9Sstevel@tonic-gate struct cred *cred = CRED(); 3747c478bd9Sstevel@tonic-gate ulong_t i, j, anon_index = 0; 3757c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(amp->size); 3767c478bd9Sstevel@tonic-gate struct vnode *vp; 3777c478bd9Sstevel@tonic-gate page_t **ppa; 3787c478bd9Sstevel@tonic-gate uint_t hat_flags; 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate /* 3817c478bd9Sstevel@tonic-gate * We are holding the a_lock on the underlying dummy as, 3827c478bd9Sstevel@tonic-gate * so we can make calls to the HAT layer. 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate #ifdef DEBUG 3877c478bd9Sstevel@tonic-gate TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */, 3887c478bd9Sstevel@tonic-gate tnf_opaque, addr, addr, 3897c478bd9Sstevel@tonic-gate tnf_ulong, len, seg->s_size); 3907c478bd9Sstevel@tonic-gate #endif 3917c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 3927c478bd9Sstevel@tonic-gate if (err = anon_swap_adjust(npages)) 3937c478bd9Sstevel@tonic-gate return (err); 3947c478bd9Sstevel@tonic-gate } 3957c478bd9Sstevel@tonic-gate err = ENOMEM; 3967c478bd9Sstevel@tonic-gate 3977c478bd9Sstevel@tonic-gate if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL) 3987c478bd9Sstevel@tonic-gate goto out1; 3997c478bd9Sstevel@tonic-gate 4007c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 4017c478bd9Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages), 4027c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) 4037c478bd9Sstevel@tonic-gate goto out2; 4047c478bd9Sstevel@tonic-gate } 4057c478bd9Sstevel@tonic-gate 4067c478bd9Sstevel@tonic-gate mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL); 4077c478bd9Sstevel@tonic-gate 4087c478bd9Sstevel@tonic-gate if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL) 4097c478bd9Sstevel@tonic-gate goto out3; 4107c478bd9Sstevel@tonic-gate 4117c478bd9Sstevel@tonic-gate seg->s_ops = &segspt_ops; 4127c478bd9Sstevel@tonic-gate sptd->spt_vp = vp; 4137c478bd9Sstevel@tonic-gate sptd->spt_amp = amp; 4147c478bd9Sstevel@tonic-gate sptd->spt_prot = sptcargs->prot; 4157c478bd9Sstevel@tonic-gate sptd->spt_flags = sptcargs->flags; 4167c478bd9Sstevel@tonic-gate seg->s_data = (caddr_t)sptd; 4177c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 4187c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt = NULL; 4197c478bd9Sstevel@tonic-gate seg->s_szc = sptcargs->szc; 4207c478bd9Sstevel@tonic-gate 4217c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 4227c478bd9Sstevel@tonic-gate amp->a_szc = seg->s_szc; 4237c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate /* 4267c478bd9Sstevel@tonic-gate * Set policy to affect initial allocation of pages in 4277c478bd9Sstevel@tonic-gate * anon_map_createpages() 4287c478bd9Sstevel@tonic-gate */ 4297c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index, 4307c478bd9Sstevel@tonic-gate NULL, 0, ptob(npages)); 4317c478bd9Sstevel@tonic-gate 4327c478bd9Sstevel@tonic-gate if (sptcargs->flags & SHM_PAGEABLE) { 4337c478bd9Sstevel@tonic-gate size_t share_sz; 4347c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, more_pgs; 4357c478bd9Sstevel@tonic-gate struct anon_hdr *nahp; 4367c478bd9Sstevel@tonic-gate 4377c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(seg->s_szc); 4387c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(amp->size, share_sz)) { 4397c478bd9Sstevel@tonic-gate /* 4407c478bd9Sstevel@tonic-gate * We are rounding up the size of the anon array 4417c478bd9Sstevel@tonic-gate * on 4 M boundary because we always create 4 M 4427c478bd9Sstevel@tonic-gate * of page(s) when locking, faulting pages and we 4437c478bd9Sstevel@tonic-gate * don't have to check for all corner cases e.g. 4447c478bd9Sstevel@tonic-gate * if there is enough space to allocate 4 M 4457c478bd9Sstevel@tonic-gate * page. 4467c478bd9Sstevel@tonic-gate */ 4477c478bd9Sstevel@tonic-gate new_npgs = btop(P2ROUNDUP(amp->size, share_sz)); 4487c478bd9Sstevel@tonic-gate more_pgs = new_npgs - npages; 4497c478bd9Sstevel@tonic-gate 4507c478bd9Sstevel@tonic-gate if (anon_resv(ptob(more_pgs)) == 0) { 4517c478bd9Sstevel@tonic-gate err = ENOMEM; 4527c478bd9Sstevel@tonic-gate goto out4; 4537c478bd9Sstevel@tonic-gate } 4547c478bd9Sstevel@tonic-gate nahp = anon_create(new_npgs, ANON_SLEEP); 4557c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 4567c478bd9Sstevel@tonic-gate (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages, 4577c478bd9Sstevel@tonic-gate ANON_SLEEP); 4587c478bd9Sstevel@tonic-gate anon_release(amp->ahp, npages); 4597c478bd9Sstevel@tonic-gate amp->ahp = nahp; 4607c478bd9Sstevel@tonic-gate amp->swresv = amp->size = ptob(new_npgs); 4617c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 4627c478bd9Sstevel@tonic-gate npages = new_npgs; 4637c478bd9Sstevel@tonic-gate } 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt = kmem_zalloc(npages * 4667c478bd9Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP); 4677c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt = 0; 4687c478bd9Sstevel@tonic-gate sptd->spt_realsize = ptob(npages); 4697c478bd9Sstevel@tonic-gate sptcargs->seg_spt = seg; 4707c478bd9Sstevel@tonic-gate return (0); 4717c478bd9Sstevel@tonic-gate } 4727c478bd9Sstevel@tonic-gate 4737c478bd9Sstevel@tonic-gate /* 4747c478bd9Sstevel@tonic-gate * get array of pages for each anon slot in amp 4757c478bd9Sstevel@tonic-gate */ 4767c478bd9Sstevel@tonic-gate if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa, 4777c478bd9Sstevel@tonic-gate seg, addr, S_CREATE, cred)) != 0) 4787c478bd9Sstevel@tonic-gate goto out4; 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate /* 4817c478bd9Sstevel@tonic-gate * addr is initial address corresponding to the first page on ppa list 4827c478bd9Sstevel@tonic-gate */ 4837c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 4847c478bd9Sstevel@tonic-gate /* attempt to lock all pages */ 4857c478bd9Sstevel@tonic-gate if (!page_pp_lock(ppa[i], 0, 1)) { 4867c478bd9Sstevel@tonic-gate /* 4877c478bd9Sstevel@tonic-gate * if unable to lock any page, unlock all 4887c478bd9Sstevel@tonic-gate * of them and return error 4897c478bd9Sstevel@tonic-gate */ 4907c478bd9Sstevel@tonic-gate for (j = 0; j < i; j++) 4917c478bd9Sstevel@tonic-gate page_pp_unlock(ppa[j], 0, 1); 4927c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 4937c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 4947c478bd9Sstevel@tonic-gate } 4957c478bd9Sstevel@tonic-gate err = ENOMEM; 4967c478bd9Sstevel@tonic-gate goto out4; 4977c478bd9Sstevel@tonic-gate } 4987c478bd9Sstevel@tonic-gate } 4997c478bd9Sstevel@tonic-gate 5007c478bd9Sstevel@tonic-gate /* 5017c478bd9Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 5027c478bd9Sstevel@tonic-gate * for the entire life of the segment. For example platforms 5037c478bd9Sstevel@tonic-gate * that do not support Dynamic Reconfiguration. 5047c478bd9Sstevel@tonic-gate */ 5057c478bd9Sstevel@tonic-gate hat_flags = HAT_LOAD_SHARE; 5067c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) 5077c478bd9Sstevel@tonic-gate hat_flags |= HAT_LOAD_LOCK; 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate hat_memload_array(seg->s_as->a_hat, addr, ptob(npages), 5107c478bd9Sstevel@tonic-gate ppa, sptd->spt_prot, hat_flags); 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate /* 5137c478bd9Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 5147c478bd9Sstevel@tonic-gate * we will leave the pages locked SE_SHARED for the life 5157c478bd9Sstevel@tonic-gate * of the ISM segment. This will prevent any calls to 5167c478bd9Sstevel@tonic-gate * hat_pageunload() on this ISM segment for those platforms. 5177c478bd9Sstevel@tonic-gate */ 5187c478bd9Sstevel@tonic-gate if (!(hat_flags & HAT_LOAD_LOCK)) { 5197c478bd9Sstevel@tonic-gate /* 5207c478bd9Sstevel@tonic-gate * On platforms that support HAT_DYNAMIC_ISM_UNMAP, 5217c478bd9Sstevel@tonic-gate * we no longer need to hold the SE_SHARED lock on the pages, 5227c478bd9Sstevel@tonic-gate * since L_PAGELOCK and F_SOFTLOCK calls will grab the 5237c478bd9Sstevel@tonic-gate * SE_SHARED lock on the pages as necessary. 5247c478bd9Sstevel@tonic-gate */ 5257c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) 5267c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 5277c478bd9Sstevel@tonic-gate } 5287c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt = 0; 5297c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * npages)); 5307c478bd9Sstevel@tonic-gate sptd->spt_realsize = ptob(npages); 5317c478bd9Sstevel@tonic-gate atomic_add_long(&spt_used, npages); 5327c478bd9Sstevel@tonic-gate sptcargs->seg_spt = seg; 5337c478bd9Sstevel@tonic-gate return (0); 5347c478bd9Sstevel@tonic-gate 5357c478bd9Sstevel@tonic-gate out4: 5367c478bd9Sstevel@tonic-gate seg->s_data = NULL; 5377c478bd9Sstevel@tonic-gate kmem_free(vp, sizeof (*vp)); 5387c478bd9Sstevel@tonic-gate out3: 5397c478bd9Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock); 5407c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) 5417c478bd9Sstevel@tonic-gate kmem_free(ppa, (sizeof (*ppa) * npages)); 5427c478bd9Sstevel@tonic-gate out2: 5437c478bd9Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd)); 5447c478bd9Sstevel@tonic-gate out1: 5457c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) 5467c478bd9Sstevel@tonic-gate anon_swap_restore(npages); 5477c478bd9Sstevel@tonic-gate return (err); 5487c478bd9Sstevel@tonic-gate } 5497c478bd9Sstevel@tonic-gate 5507c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 5517c478bd9Sstevel@tonic-gate void 5527c478bd9Sstevel@tonic-gate segspt_free_pages(struct seg *seg, caddr_t addr, size_t len) 5537c478bd9Sstevel@tonic-gate { 5547c478bd9Sstevel@tonic-gate struct page *pp; 5557c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data; 5567c478bd9Sstevel@tonic-gate pgcnt_t npages; 5577c478bd9Sstevel@tonic-gate ulong_t anon_idx; 5587c478bd9Sstevel@tonic-gate struct anon_map *amp; 5597c478bd9Sstevel@tonic-gate struct anon *ap; 5607c478bd9Sstevel@tonic-gate struct vnode *vp; 5617c478bd9Sstevel@tonic-gate u_offset_t off; 5627c478bd9Sstevel@tonic-gate uint_t hat_flags; 5637c478bd9Sstevel@tonic-gate int root = 0; 5647c478bd9Sstevel@tonic-gate pgcnt_t pgs, curnpgs = 0; 5657c478bd9Sstevel@tonic-gate page_t *rootpp; 5667c478bd9Sstevel@tonic-gate 5677c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 5687c478bd9Sstevel@tonic-gate 5697c478bd9Sstevel@tonic-gate len = P2ROUNDUP(len, PAGESIZE); 5707c478bd9Sstevel@tonic-gate 5717c478bd9Sstevel@tonic-gate npages = btop(len); 5727c478bd9Sstevel@tonic-gate 5737c478bd9Sstevel@tonic-gate hat_flags = HAT_UNLOAD_UNLOCK; 5747c478bd9Sstevel@tonic-gate if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) || 5757c478bd9Sstevel@tonic-gate (sptd->spt_flags & SHM_PAGEABLE)) { 5767c478bd9Sstevel@tonic-gate hat_flags = HAT_UNLOAD; 5777c478bd9Sstevel@tonic-gate } 5787c478bd9Sstevel@tonic-gate 5797c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, hat_flags); 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 5827c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) 5837c478bd9Sstevel@tonic-gate npages = btop(amp->size); 5847c478bd9Sstevel@tonic-gate 5857c478bd9Sstevel@tonic-gate ASSERT(amp); 5867c478bd9Sstevel@tonic-gate for (anon_idx = 0; anon_idx < npages; anon_idx++) { 5877c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 5887c478bd9Sstevel@tonic-gate if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) { 5897c478bd9Sstevel@tonic-gate panic("segspt_free_pages: null app"); 5907c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 5917c478bd9Sstevel@tonic-gate } 5927c478bd9Sstevel@tonic-gate } else { 5937c478bd9Sstevel@tonic-gate if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx)) 5947c478bd9Sstevel@tonic-gate == NULL) 5957c478bd9Sstevel@tonic-gate continue; 5967c478bd9Sstevel@tonic-gate } 5977c478bd9Sstevel@tonic-gate ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0); 5987c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate /* 6017c478bd9Sstevel@tonic-gate * If this platform supports HAT_DYNAMIC_ISM_UNMAP, 6027c478bd9Sstevel@tonic-gate * the pages won't be having SE_SHARED lock at this 6037c478bd9Sstevel@tonic-gate * point. 6047c478bd9Sstevel@tonic-gate * 6057c478bd9Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 6067c478bd9Sstevel@tonic-gate * the pages are still held SE_SHARED locked from the 6077c478bd9Sstevel@tonic-gate * original segspt_create() 6087c478bd9Sstevel@tonic-gate * 6097c478bd9Sstevel@tonic-gate * Our goal is to get SE_EXCL lock on each page, remove 6107c478bd9Sstevel@tonic-gate * permanent lock on it and invalidate the page. 6117c478bd9Sstevel@tonic-gate */ 6127c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6137c478bd9Sstevel@tonic-gate if (hat_flags == HAT_UNLOAD) 6147c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL); 6157c478bd9Sstevel@tonic-gate else { 6167c478bd9Sstevel@tonic-gate if ((pp = page_find(vp, off)) == NULL) { 6177c478bd9Sstevel@tonic-gate panic("segspt_free_pages: " 6187c478bd9Sstevel@tonic-gate "page not locked"); 6197c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6207c478bd9Sstevel@tonic-gate } 6217c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 6227c478bd9Sstevel@tonic-gate page_unlock(pp); 6237c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL); 6247c478bd9Sstevel@tonic-gate } 6257c478bd9Sstevel@tonic-gate } 6267c478bd9Sstevel@tonic-gate if (pp == NULL) { 6277c478bd9Sstevel@tonic-gate panic("segspt_free_pages: " 6287c478bd9Sstevel@tonic-gate "page not in the system"); 6297c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6307c478bd9Sstevel@tonic-gate } 6317c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, 1); 6327c478bd9Sstevel@tonic-gate } else { 6337c478bd9Sstevel@tonic-gate if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL) 6347c478bd9Sstevel@tonic-gate continue; 6357c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, 0); 6367c478bd9Sstevel@tonic-gate } 6377c478bd9Sstevel@tonic-gate /* 6387c478bd9Sstevel@tonic-gate * It's logical to invalidate the pages here as in most cases 6397c478bd9Sstevel@tonic-gate * these were created by segspt. 6407c478bd9Sstevel@tonic-gate */ 6417c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 6427c478bd9Sstevel@tonic-gate /* 6437c478bd9Sstevel@tonic-gate * For DISM swap is released in shm_rm_amp. 6447c478bd9Sstevel@tonic-gate */ 6457c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0 && 6467c478bd9Sstevel@tonic-gate ap->an_pvp != NULL) { 6477c478bd9Sstevel@tonic-gate panic("segspt_free_pages: pvp non NULL"); 6487c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6497c478bd9Sstevel@tonic-gate } 6507c478bd9Sstevel@tonic-gate if (root == 0) { 6517c478bd9Sstevel@tonic-gate ASSERT(curnpgs == 0); 6527c478bd9Sstevel@tonic-gate root = 1; 6537c478bd9Sstevel@tonic-gate rootpp = pp; 6547c478bd9Sstevel@tonic-gate pgs = curnpgs = page_get_pagecnt(pp->p_szc); 6557c478bd9Sstevel@tonic-gate ASSERT(pgs > 1); 6567c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pgs, pgs)); 6577c478bd9Sstevel@tonic-gate ASSERT(!(page_pptonum(pp) & (pgs - 1))); 6587c478bd9Sstevel@tonic-gate curnpgs--; 6597c478bd9Sstevel@tonic-gate } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) { 6607c478bd9Sstevel@tonic-gate ASSERT(curnpgs == 1); 6617c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(pp) == 6627c478bd9Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - 1)); 6637c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 6647c478bd9Sstevel@tonic-gate root = 0; 6657c478bd9Sstevel@tonic-gate curnpgs = 0; 6667c478bd9Sstevel@tonic-gate } else { 6677c478bd9Sstevel@tonic-gate ASSERT(curnpgs > 1); 6687c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(pp) == 6697c478bd9Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - curnpgs)); 6707c478bd9Sstevel@tonic-gate curnpgs--; 6717c478bd9Sstevel@tonic-gate } 6727c478bd9Sstevel@tonic-gate } else { 6737c478bd9Sstevel@tonic-gate if (root != 0 || curnpgs != 0) { 6747c478bd9Sstevel@tonic-gate panic("segspt_free_pages: bad large page"); 6757c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6767c478bd9Sstevel@tonic-gate } 6777c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional context */ 6787c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred); 6797c478bd9Sstevel@tonic-gate } 6807c478bd9Sstevel@tonic-gate } 6817c478bd9Sstevel@tonic-gate 6827c478bd9Sstevel@tonic-gate if (root != 0 || curnpgs != 0) { 6837c478bd9Sstevel@tonic-gate panic("segspt_free_pages: bad large page"); 6847c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6857c478bd9Sstevel@tonic-gate } 6867c478bd9Sstevel@tonic-gate 6877c478bd9Sstevel@tonic-gate /* 6887c478bd9Sstevel@tonic-gate * mark that pages have been released 6897c478bd9Sstevel@tonic-gate */ 6907c478bd9Sstevel@tonic-gate sptd->spt_realsize = 0; 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6937c478bd9Sstevel@tonic-gate atomic_add_long(&spt_used, -npages); 6947c478bd9Sstevel@tonic-gate anon_swap_restore(npages); 6957c478bd9Sstevel@tonic-gate } 6967c478bd9Sstevel@tonic-gate } 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate /* 6997c478bd9Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment 7007c478bd9Sstevel@tonic-gate */ 7017c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 7027c478bd9Sstevel@tonic-gate segspt_getpolicy(struct seg *seg, caddr_t addr) 7037c478bd9Sstevel@tonic-gate { 7047c478bd9Sstevel@tonic-gate struct anon_map *amp; 7057c478bd9Sstevel@tonic-gate ulong_t anon_index; 7067c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 7077c478bd9Sstevel@tonic-gate struct spt_data *spt_data; 7087c478bd9Sstevel@tonic-gate 7097c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate /* 7127c478bd9Sstevel@tonic-gate * Get anon_map from segspt 7137c478bd9Sstevel@tonic-gate * 7147c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since 7157c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be 7167c478bd9Sstevel@tonic-gate * nonzero for an existing segment 7177c478bd9Sstevel@tonic-gate * Need to grab readers lock on policy tree though 7187c478bd9Sstevel@tonic-gate */ 7197c478bd9Sstevel@tonic-gate spt_data = (struct spt_data *)seg->s_data; 7207c478bd9Sstevel@tonic-gate if (spt_data == NULL) 7217c478bd9Sstevel@tonic-gate return (NULL); 7227c478bd9Sstevel@tonic-gate amp = spt_data->spt_amp; 7237c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 7247c478bd9Sstevel@tonic-gate 7257c478bd9Sstevel@tonic-gate /* 7267c478bd9Sstevel@tonic-gate * Get policy info 7277c478bd9Sstevel@tonic-gate * 7287c478bd9Sstevel@tonic-gate * Assume starting anon index of 0 7297c478bd9Sstevel@tonic-gate */ 7307c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 7317c478bd9Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 7327c478bd9Sstevel@tonic-gate 7337c478bd9Sstevel@tonic-gate return (policy_info); 7347c478bd9Sstevel@tonic-gate } 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate /* 7377c478bd9Sstevel@tonic-gate * DISM only. 7387c478bd9Sstevel@tonic-gate * Return locked pages over a given range. 7397c478bd9Sstevel@tonic-gate * 7407c478bd9Sstevel@tonic-gate * We will cache all DISM locked pages and save the pplist for the 7417c478bd9Sstevel@tonic-gate * entire segment in the ppa field of the underlying DISM segment structure. 7427c478bd9Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array 7437c478bd9Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list. 7447c478bd9Sstevel@tonic-gate */ 7457c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7467c478bd9Sstevel@tonic-gate static int 7477c478bd9Sstevel@tonic-gate segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len, 7487c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 7497c478bd9Sstevel@tonic-gate { 7507c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 7517c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 7527c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 7537c478bd9Sstevel@tonic-gate pgcnt_t pg_idx, npages, tot_npages, npgs; 7547c478bd9Sstevel@tonic-gate struct page **pplist, **pl, **ppa, *pp; 7557c478bd9Sstevel@tonic-gate struct anon_map *amp; 7567c478bd9Sstevel@tonic-gate spgcnt_t an_idx; 7577c478bd9Sstevel@tonic-gate int ret = ENOTSUP; 7587c478bd9Sstevel@tonic-gate uint_t pl_built = 0; 7597c478bd9Sstevel@tonic-gate struct anon *ap; 7607c478bd9Sstevel@tonic-gate struct vnode *vp; 7617c478bd9Sstevel@tonic-gate u_offset_t off; 7627c478bd9Sstevel@tonic-gate pgcnt_t claim_availrmem = 0; 7637c478bd9Sstevel@tonic-gate uint_t szc; 7647c478bd9Sstevel@tonic-gate 7657c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate /* 7687c478bd9Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore, 7697c478bd9Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address 7707c478bd9Sstevel@tonic-gate * and length for the caching arguments. 7717c478bd9Sstevel@tonic-gate */ 7727c478bd9Sstevel@tonic-gate ASSERT(sptseg); 7737c478bd9Sstevel@tonic-gate ASSERT(sptd); 7747c478bd9Sstevel@tonic-gate 7757c478bd9Sstevel@tonic-gate pg_idx = seg_page(seg, addr); 7767c478bd9Sstevel@tonic-gate npages = btopr(len); 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate /* 7797c478bd9Sstevel@tonic-gate * check if the request is larger than number of pages covered 7807c478bd9Sstevel@tonic-gate * by amp 7817c478bd9Sstevel@tonic-gate */ 7827c478bd9Sstevel@tonic-gate if (pg_idx + npages > btopr(sptd->spt_amp->size)) { 7837c478bd9Sstevel@tonic-gate *ppp = NULL; 7847c478bd9Sstevel@tonic-gate return (ENOTSUP); 7857c478bd9Sstevel@tonic-gate } 7867c478bd9Sstevel@tonic-gate 7877c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 7887c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 7897c478bd9Sstevel@tonic-gate 7907c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 7917c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot, segspt_reclaim); 7927c478bd9Sstevel@tonic-gate 7937c478bd9Sstevel@tonic-gate /* 7947c478bd9Sstevel@tonic-gate * If someone is blocked while unmapping, we purge 7957c478bd9Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously 7967c478bd9Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up 7977c478bd9Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while 7987c478bd9Sstevel@tonic-gate * raw async i/o is still in progress or where a thread 7997c478bd9Sstevel@tonic-gate * exits on data fault in a multithreaded application. 8007c478bd9Sstevel@tonic-gate */ 8017c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 8027c478bd9Sstevel@tonic-gate segspt_purge(seg); 8037c478bd9Sstevel@tonic-gate } 8047c478bd9Sstevel@tonic-gate return (0); 8057c478bd9Sstevel@tonic-gate } else if (type == L_PAGERECLAIM) { 8067c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8077c478bd9Sstevel@tonic-gate (void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size, 8087c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot); 8097c478bd9Sstevel@tonic-gate return (0); 8107c478bd9Sstevel@tonic-gate } 8117c478bd9Sstevel@tonic-gate 8127c478bd9Sstevel@tonic-gate if (sptd->spt_flags & DISM_PPA_CHANGED) { 8137c478bd9Sstevel@tonic-gate segspt_purge(seg); 8147c478bd9Sstevel@tonic-gate /* 8157c478bd9Sstevel@tonic-gate * for DISM ppa needs to be rebuild since 8167c478bd9Sstevel@tonic-gate * number of locked pages could be changed 8177c478bd9Sstevel@tonic-gate */ 8187c478bd9Sstevel@tonic-gate *ppp = NULL; 8197c478bd9Sstevel@tonic-gate return (ENOTSUP); 8207c478bd9Sstevel@tonic-gate } 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate /* 8237c478bd9Sstevel@tonic-gate * First try to find pages in segment page cache, without 8247c478bd9Sstevel@tonic-gate * holding the segment lock. 8257c478bd9Sstevel@tonic-gate */ 8267c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 8277c478bd9Sstevel@tonic-gate sptd->spt_prot); 8287c478bd9Sstevel@tonic-gate if (pplist != NULL) { 8297c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8307c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 8317c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa; 8327c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 8337c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 8347c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, 8357c478bd9Sstevel@tonic-gate sptd->spt_amp->size, ppa, 8367c478bd9Sstevel@tonic-gate sptd->spt_prot, segspt_reclaim); 8377c478bd9Sstevel@tonic-gate *ppp = NULL; 8387c478bd9Sstevel@tonic-gate return (ENOTSUP); 8397c478bd9Sstevel@tonic-gate } 8407c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 8417c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 8427c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 8437c478bd9Sstevel@tonic-gate } else { 8447c478bd9Sstevel@tonic-gate an_idx++; 8457c478bd9Sstevel@tonic-gate } 8467c478bd9Sstevel@tonic-gate } 8477c478bd9Sstevel@tonic-gate /* 8487c478bd9Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to 8497c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 8507c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 8517c478bd9Sstevel@tonic-gate */ 8527c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 8537c478bd9Sstevel@tonic-gate return (0); 8547c478bd9Sstevel@tonic-gate } 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate /* The L_PAGELOCK case... */ 8577c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 8587c478bd9Sstevel@tonic-gate /* 8597c478bd9Sstevel@tonic-gate * try to find pages in segment page cache with mutex 8607c478bd9Sstevel@tonic-gate */ 8617c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 8627c478bd9Sstevel@tonic-gate sptd->spt_prot); 8637c478bd9Sstevel@tonic-gate if (pplist != NULL) { 8647c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8657c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 8667c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa; 8677c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 8687c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 8697c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 8707c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, 8717c478bd9Sstevel@tonic-gate sptd->spt_amp->size, ppa, 8727c478bd9Sstevel@tonic-gate sptd->spt_prot, segspt_reclaim); 8737c478bd9Sstevel@tonic-gate *ppp = NULL; 8747c478bd9Sstevel@tonic-gate return (ENOTSUP); 8757c478bd9Sstevel@tonic-gate } 8767c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 8777c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 8787c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 8797c478bd9Sstevel@tonic-gate } else { 8807c478bd9Sstevel@tonic-gate an_idx++; 8817c478bd9Sstevel@tonic-gate } 8827c478bd9Sstevel@tonic-gate } 8837c478bd9Sstevel@tonic-gate /* 8847c478bd9Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to 8857c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 8867c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 8877c478bd9Sstevel@tonic-gate */ 8887c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 8897c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 8907c478bd9Sstevel@tonic-gate return (0); 8917c478bd9Sstevel@tonic-gate } 8927c478bd9Sstevel@tonic-gate if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) == 8937c478bd9Sstevel@tonic-gate SEGP_FAIL) { 8947c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 8957c478bd9Sstevel@tonic-gate *ppp = NULL; 8967c478bd9Sstevel@tonic-gate return (ENOTSUP); 8977c478bd9Sstevel@tonic-gate } 8987c478bd9Sstevel@tonic-gate 8997c478bd9Sstevel@tonic-gate /* 9007c478bd9Sstevel@tonic-gate * No need to worry about protections because DISM pages are always rw. 9017c478bd9Sstevel@tonic-gate */ 9027c478bd9Sstevel@tonic-gate pl = pplist = NULL; 9037c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 9047c478bd9Sstevel@tonic-gate 9057c478bd9Sstevel@tonic-gate /* 9067c478bd9Sstevel@tonic-gate * Do we need to build the ppa array? 9077c478bd9Sstevel@tonic-gate */ 9087c478bd9Sstevel@tonic-gate if (sptd->spt_ppa == NULL) { 9097c478bd9Sstevel@tonic-gate pgcnt_t lpg_cnt = 0; 9107c478bd9Sstevel@tonic-gate 9117c478bd9Sstevel@tonic-gate pl_built = 1; 9127c478bd9Sstevel@tonic-gate tot_npages = btopr(sptd->spt_amp->size); 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 9157c478bd9Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP); 9167c478bd9Sstevel@tonic-gate pl = pplist; 9177c478bd9Sstevel@tonic-gate 9187c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 9197c478bd9Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; ) { 9207c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 9217c478bd9Sstevel@tonic-gate /* 9227c478bd9Sstevel@tonic-gate * Cache only mlocked pages. For large pages 9237c478bd9Sstevel@tonic-gate * if one (constituent) page is mlocked 9247c478bd9Sstevel@tonic-gate * all pages for that large page 9257c478bd9Sstevel@tonic-gate * are cached also. This is for quick 9267c478bd9Sstevel@tonic-gate * lookups of ppa array; 9277c478bd9Sstevel@tonic-gate */ 9287c478bd9Sstevel@tonic-gate if ((ap != NULL) && (lpg_cnt != 0 || 9297c478bd9Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[an_idx] != 0))) { 9307c478bd9Sstevel@tonic-gate 9317c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 9327c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 9337c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 9347c478bd9Sstevel@tonic-gate if (lpg_cnt == 0) { 935*e67882ffSbs21162 lpg_cnt++; 936*e67882ffSbs21162 /* 937*e67882ffSbs21162 * For a small page, we are done -- 938*e67882ffSbs21162 * lpg_count is reset to 0 below. 939*e67882ffSbs21162 * 940*e67882ffSbs21162 * For a large page, we are guaranteed 941*e67882ffSbs21162 * to find the anon structures of all 942*e67882ffSbs21162 * constituent pages and a non-zero 943*e67882ffSbs21162 * lpg_cnt ensures that we don't test 944*e67882ffSbs21162 * for mlock for these. We are done 945*e67882ffSbs21162 * when lpg_count reaches (npgs + 1). 946*e67882ffSbs21162 * If we are not the first constituent 947*e67882ffSbs21162 * page, restart at the first one. 948*e67882ffSbs21162 */ 9497c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 9507c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(an_idx, npgs)) { 9517c478bd9Sstevel@tonic-gate an_idx = P2ALIGN(an_idx, npgs); 9527c478bd9Sstevel@tonic-gate page_unlock(pp); 9537c478bd9Sstevel@tonic-gate continue; 9547c478bd9Sstevel@tonic-gate } 9557c478bd9Sstevel@tonic-gate } 956*e67882ffSbs21162 if (++lpg_cnt > npgs) 9577c478bd9Sstevel@tonic-gate lpg_cnt = 0; 9587c478bd9Sstevel@tonic-gate 9597c478bd9Sstevel@tonic-gate /* 9607c478bd9Sstevel@tonic-gate * availrmem is decremented only 9617c478bd9Sstevel@tonic-gate * for unlocked pages 9627c478bd9Sstevel@tonic-gate */ 9637c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[an_idx] == 0) 9647c478bd9Sstevel@tonic-gate claim_availrmem++; 9657c478bd9Sstevel@tonic-gate pplist[an_idx] = pp; 9667c478bd9Sstevel@tonic-gate } 9677c478bd9Sstevel@tonic-gate an_idx++; 9687c478bd9Sstevel@tonic-gate } 9697c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 9707c478bd9Sstevel@tonic-gate 9717c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 9727c478bd9Sstevel@tonic-gate if (availrmem < tune.t_minarmem + claim_availrmem) { 9737c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 9747c478bd9Sstevel@tonic-gate ret = FC_MAKE_ERR(ENOMEM); 9757c478bd9Sstevel@tonic-gate claim_availrmem = 0; 9767c478bd9Sstevel@tonic-gate goto insert_fail; 9777c478bd9Sstevel@tonic-gate } else { 9787c478bd9Sstevel@tonic-gate availrmem -= claim_availrmem; 9797c478bd9Sstevel@tonic-gate } 9807c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 9817c478bd9Sstevel@tonic-gate 9827c478bd9Sstevel@tonic-gate sptd->spt_ppa = pl; 9837c478bd9Sstevel@tonic-gate } else { 9847c478bd9Sstevel@tonic-gate /* 9857c478bd9Sstevel@tonic-gate * We already have a valid ppa[]. 9867c478bd9Sstevel@tonic-gate */ 9877c478bd9Sstevel@tonic-gate pl = sptd->spt_ppa; 9887c478bd9Sstevel@tonic-gate } 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate ASSERT(pl != NULL); 9917c478bd9Sstevel@tonic-gate 9927c478bd9Sstevel@tonic-gate ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size, 9937c478bd9Sstevel@tonic-gate pl, sptd->spt_prot, SEGP_FORCE_WIRED | SEGP_ASYNC_FLUSH, 9947c478bd9Sstevel@tonic-gate segspt_reclaim); 9957c478bd9Sstevel@tonic-gate if (ret == SEGP_FAIL) { 9967c478bd9Sstevel@tonic-gate /* 9977c478bd9Sstevel@tonic-gate * seg_pinsert failed. We return 9987c478bd9Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will 9997c478bd9Sstevel@tonic-gate * then try the slower F_SOFTLOCK path. 10007c478bd9Sstevel@tonic-gate */ 10010da3d2a8Srd117015 if (pl_built) { 10020da3d2a8Srd117015 /* 10030da3d2a8Srd117015 * No one else has referenced the ppa[]. 10040da3d2a8Srd117015 * We created it and we need to destroy it. 10050da3d2a8Srd117015 */ 10067c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 10070da3d2a8Srd117015 } 10087c478bd9Sstevel@tonic-gate ret = ENOTSUP; 10097c478bd9Sstevel@tonic-gate goto insert_fail; 10107c478bd9Sstevel@tonic-gate } 10117c478bd9Sstevel@tonic-gate 10127c478bd9Sstevel@tonic-gate /* 10137c478bd9Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment. 10147c478bd9Sstevel@tonic-gate */ 10157c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt++; 10167c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 10177c478bd9Sstevel@tonic-gate 10187c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa; 10197c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 10207c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 10217c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 10227c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 10237c478bd9Sstevel@tonic-gate pl, sptd->spt_prot, segspt_reclaim); 10247c478bd9Sstevel@tonic-gate *ppp = NULL; 10257c478bd9Sstevel@tonic-gate return (ENOTSUP); 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 10287c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 10297c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 10307c478bd9Sstevel@tonic-gate } else { 10317c478bd9Sstevel@tonic-gate an_idx++; 10327c478bd9Sstevel@tonic-gate } 10337c478bd9Sstevel@tonic-gate } 10347c478bd9Sstevel@tonic-gate /* 10357c478bd9Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[] 10367c478bd9Sstevel@tonic-gate * exists and he have incremented pacachecnt. 10377c478bd9Sstevel@tonic-gate */ 10387c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 10397c478bd9Sstevel@tonic-gate 10407c478bd9Sstevel@tonic-gate /* 10417c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to 10427c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 10437c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 10447c478bd9Sstevel@tonic-gate */ 10457c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 10467c478bd9Sstevel@tonic-gate return (ret); 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate insert_fail: 10497c478bd9Sstevel@tonic-gate /* 10507c478bd9Sstevel@tonic-gate * We will only reach this code if we tried and failed. 10517c478bd9Sstevel@tonic-gate * 10527c478bd9Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed 10537c478bd9Sstevel@tonic-gate * to set up a new ppa[]. 10547c478bd9Sstevel@tonic-gate */ 10557c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 10567c478bd9Sstevel@tonic-gate 10577c478bd9Sstevel@tonic-gate if (pl_built) { 10587c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 10597c478bd9Sstevel@tonic-gate availrmem += claim_availrmem; 10607c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate /* 10637c478bd9Sstevel@tonic-gate * We created pl and we need to destroy it. 10647c478bd9Sstevel@tonic-gate */ 10657c478bd9Sstevel@tonic-gate pplist = pl; 10667c478bd9Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; an_idx++) { 10677c478bd9Sstevel@tonic-gate if (pplist[an_idx] != NULL) 10687c478bd9Sstevel@tonic-gate page_unlock(pplist[an_idx]); 10697c478bd9Sstevel@tonic-gate } 10707c478bd9Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * tot_npages); 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate 10737c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 10747c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 10757c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 10767c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 10777c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 10787c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 10797c478bd9Sstevel@tonic-gate } 10807c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 10817c478bd9Sstevel@tonic-gate } 10827c478bd9Sstevel@tonic-gate } 10837c478bd9Sstevel@tonic-gate *ppp = NULL; 10847c478bd9Sstevel@tonic-gate return (ret); 10857c478bd9Sstevel@tonic-gate } 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate 10887c478bd9Sstevel@tonic-gate 10897c478bd9Sstevel@tonic-gate /* 10907c478bd9Sstevel@tonic-gate * return locked pages over a given range. 10917c478bd9Sstevel@tonic-gate * 10927c478bd9Sstevel@tonic-gate * We will cache the entire ISM segment and save the pplist for the 10937c478bd9Sstevel@tonic-gate * entire segment in the ppa field of the underlying ISM segment structure. 10947c478bd9Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array 10957c478bd9Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list. 10967c478bd9Sstevel@tonic-gate */ 10977c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10987c478bd9Sstevel@tonic-gate static int 10997c478bd9Sstevel@tonic-gate segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len, 11007c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 11017c478bd9Sstevel@tonic-gate { 11027c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 11037c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 11047c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 11057c478bd9Sstevel@tonic-gate pgcnt_t np, page_index, npages; 11067c478bd9Sstevel@tonic-gate caddr_t a, spt_base; 11077c478bd9Sstevel@tonic-gate struct page **pplist, **pl, *pp; 11087c478bd9Sstevel@tonic-gate struct anon_map *amp; 11097c478bd9Sstevel@tonic-gate ulong_t anon_index; 11107c478bd9Sstevel@tonic-gate int ret = ENOTSUP; 11117c478bd9Sstevel@tonic-gate uint_t pl_built = 0; 11127c478bd9Sstevel@tonic-gate struct anon *ap; 11137c478bd9Sstevel@tonic-gate struct vnode *vp; 11147c478bd9Sstevel@tonic-gate u_offset_t off; 11157c478bd9Sstevel@tonic-gate 11167c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 11177c478bd9Sstevel@tonic-gate 11187c478bd9Sstevel@tonic-gate /* 11197c478bd9Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore, 11207c478bd9Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address 11217c478bd9Sstevel@tonic-gate * and length for the caching arguments. 11227c478bd9Sstevel@tonic-gate */ 11237c478bd9Sstevel@tonic-gate ASSERT(sptseg); 11247c478bd9Sstevel@tonic-gate ASSERT(sptd); 11257c478bd9Sstevel@tonic-gate 11267c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 11277c478bd9Sstevel@tonic-gate return (segspt_dismpagelock(seg, addr, len, ppp, type, rw)); 11287c478bd9Sstevel@tonic-gate } 11297c478bd9Sstevel@tonic-gate 11307c478bd9Sstevel@tonic-gate page_index = seg_page(seg, addr); 11317c478bd9Sstevel@tonic-gate npages = btopr(len); 11327c478bd9Sstevel@tonic-gate 11337c478bd9Sstevel@tonic-gate /* 11347c478bd9Sstevel@tonic-gate * check if the request is larger than number of pages covered 11357c478bd9Sstevel@tonic-gate * by amp 11367c478bd9Sstevel@tonic-gate */ 11377c478bd9Sstevel@tonic-gate if (page_index + npages > btopr(sptd->spt_amp->size)) { 11387c478bd9Sstevel@tonic-gate *ppp = NULL; 11397c478bd9Sstevel@tonic-gate return (ENOTSUP); 11407c478bd9Sstevel@tonic-gate } 11417c478bd9Sstevel@tonic-gate 11427c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 11457c478bd9Sstevel@tonic-gate 11467c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 11477c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot, segspt_reclaim); 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate /* 11507c478bd9Sstevel@tonic-gate * If someone is blocked while unmapping, we purge 11517c478bd9Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously 11527c478bd9Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up 11537c478bd9Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while 11547c478bd9Sstevel@tonic-gate * raw async i/o is still in progress or where a thread 11557c478bd9Sstevel@tonic-gate * exits on data fault in a multithreaded application. 11567c478bd9Sstevel@tonic-gate */ 11577c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 11587c478bd9Sstevel@tonic-gate segspt_purge(seg); 11597c478bd9Sstevel@tonic-gate } 11607c478bd9Sstevel@tonic-gate return (0); 11617c478bd9Sstevel@tonic-gate } else if (type == L_PAGERECLAIM) { 11627c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 11637c478bd9Sstevel@tonic-gate 11647c478bd9Sstevel@tonic-gate (void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size, 11657c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot); 11667c478bd9Sstevel@tonic-gate return (0); 11677c478bd9Sstevel@tonic-gate } 11687c478bd9Sstevel@tonic-gate 11697c478bd9Sstevel@tonic-gate /* 11707c478bd9Sstevel@tonic-gate * First try to find pages in segment page cache, without 11717c478bd9Sstevel@tonic-gate * holding the segment lock. 11727c478bd9Sstevel@tonic-gate */ 11737c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 11747c478bd9Sstevel@tonic-gate sptd->spt_prot); 11757c478bd9Sstevel@tonic-gate if (pplist != NULL) { 11767c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 11777c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa[page_index]); 11787c478bd9Sstevel@tonic-gate /* 11797c478bd9Sstevel@tonic-gate * Since we cache the entire ISM segment, we want to 11807c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 11817c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index. 11827c478bd9Sstevel@tonic-gate */ 11837c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 11847c478bd9Sstevel@tonic-gate return (0); 11857c478bd9Sstevel@tonic-gate } 11867c478bd9Sstevel@tonic-gate 11877c478bd9Sstevel@tonic-gate /* The L_PAGELOCK case... */ 11887c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 11897c478bd9Sstevel@tonic-gate 11907c478bd9Sstevel@tonic-gate /* 11917c478bd9Sstevel@tonic-gate * try to find pages in segment page cache 11927c478bd9Sstevel@tonic-gate */ 11937c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 11947c478bd9Sstevel@tonic-gate sptd->spt_prot); 11957c478bd9Sstevel@tonic-gate if (pplist != NULL) { 11967c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 11977c478bd9Sstevel@tonic-gate /* 11987c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to 11997c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 12007c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index. 12017c478bd9Sstevel@tonic-gate */ 12027c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 12037c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 12047c478bd9Sstevel@tonic-gate return (0); 12057c478bd9Sstevel@tonic-gate } 12067c478bd9Sstevel@tonic-gate 12077c478bd9Sstevel@tonic-gate if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) == 12087c478bd9Sstevel@tonic-gate SEGP_FAIL) { 12097c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 12107c478bd9Sstevel@tonic-gate *ppp = NULL; 12117c478bd9Sstevel@tonic-gate return (ENOTSUP); 12127c478bd9Sstevel@tonic-gate } 12137c478bd9Sstevel@tonic-gate 12147c478bd9Sstevel@tonic-gate /* 12157c478bd9Sstevel@tonic-gate * No need to worry about protections because ISM pages 12167c478bd9Sstevel@tonic-gate * are always rw. 12177c478bd9Sstevel@tonic-gate */ 12187c478bd9Sstevel@tonic-gate pl = pplist = NULL; 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate /* 12217c478bd9Sstevel@tonic-gate * Do we need to build the ppa array? 12227c478bd9Sstevel@tonic-gate */ 12237c478bd9Sstevel@tonic-gate if (sptd->spt_ppa == NULL) { 12247c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 12257c478bd9Sstevel@tonic-gate 12267c478bd9Sstevel@tonic-gate spt_base = sptseg->s_base; 12277c478bd9Sstevel@tonic-gate pl_built = 1; 12287c478bd9Sstevel@tonic-gate 12297c478bd9Sstevel@tonic-gate /* 12307c478bd9Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust() 12317c478bd9Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is 12327c478bd9Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed. 12337c478bd9Sstevel@tonic-gate */ 12347c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 12357c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 12367c478bd9Sstevel@tonic-gate 12377c478bd9Sstevel@tonic-gate /* pcachecnt is protected by sptd->spt_lock */ 12387c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 12397c478bd9Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) 12407c478bd9Sstevel@tonic-gate * btopr(sptd->spt_amp->size), KM_SLEEP); 12417c478bd9Sstevel@tonic-gate pl = pplist; 12427c478bd9Sstevel@tonic-gate 12437c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, spt_base); 12447c478bd9Sstevel@tonic-gate 12457c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 12467c478bd9Sstevel@tonic-gate for (a = spt_base; a < (spt_base + sptd->spt_amp->size); 12477c478bd9Sstevel@tonic-gate a += PAGESIZE, anon_index++, pplist++) { 12487c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 12497c478bd9Sstevel@tonic-gate ASSERT(ap != NULL); 12507c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 12517c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 12527c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 12537c478bd9Sstevel@tonic-gate *pplist = pp; 12547c478bd9Sstevel@tonic-gate } 12557c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 12567c478bd9Sstevel@tonic-gate 12577c478bd9Sstevel@tonic-gate if (a < (spt_base + sptd->spt_amp->size)) { 12587c478bd9Sstevel@tonic-gate ret = ENOTSUP; 12597c478bd9Sstevel@tonic-gate goto insert_fail; 12607c478bd9Sstevel@tonic-gate } 12617c478bd9Sstevel@tonic-gate sptd->spt_ppa = pl; 12627c478bd9Sstevel@tonic-gate } else { 12637c478bd9Sstevel@tonic-gate /* 12647c478bd9Sstevel@tonic-gate * We already have a valid ppa[]. 12657c478bd9Sstevel@tonic-gate */ 12667c478bd9Sstevel@tonic-gate pl = sptd->spt_ppa; 12677c478bd9Sstevel@tonic-gate } 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate ASSERT(pl != NULL); 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size, 12727c478bd9Sstevel@tonic-gate pl, sptd->spt_prot, SEGP_FORCE_WIRED, segspt_reclaim); 12737c478bd9Sstevel@tonic-gate if (ret == SEGP_FAIL) { 12747c478bd9Sstevel@tonic-gate /* 12757c478bd9Sstevel@tonic-gate * seg_pinsert failed. We return 12767c478bd9Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will 12777c478bd9Sstevel@tonic-gate * then try the slower F_SOFTLOCK path. 12787c478bd9Sstevel@tonic-gate */ 12797c478bd9Sstevel@tonic-gate if (pl_built) { 12807c478bd9Sstevel@tonic-gate /* 12817c478bd9Sstevel@tonic-gate * No one else has referenced the ppa[]. 12827c478bd9Sstevel@tonic-gate * We created it and we need to destroy it. 12837c478bd9Sstevel@tonic-gate */ 12847c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 12857c478bd9Sstevel@tonic-gate } 12867c478bd9Sstevel@tonic-gate ret = ENOTSUP; 12877c478bd9Sstevel@tonic-gate goto insert_fail; 12887c478bd9Sstevel@tonic-gate } 12897c478bd9Sstevel@tonic-gate 12907c478bd9Sstevel@tonic-gate /* 12917c478bd9Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment. 12927c478bd9Sstevel@tonic-gate */ 12937c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt++; 12947c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 12957c478bd9Sstevel@tonic-gate 12967c478bd9Sstevel@tonic-gate /* 12977c478bd9Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[] 12987c478bd9Sstevel@tonic-gate * exists and he have incremented pacachecnt. 12997c478bd9Sstevel@tonic-gate */ 13007c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 13017c478bd9Sstevel@tonic-gate 13027c478bd9Sstevel@tonic-gate /* 13037c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to 13047c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 13057c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index. 13067c478bd9Sstevel@tonic-gate */ 13077c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 13087c478bd9Sstevel@tonic-gate return (ret); 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate insert_fail: 13117c478bd9Sstevel@tonic-gate /* 13127c478bd9Sstevel@tonic-gate * We will only reach this code if we tried and failed. 13137c478bd9Sstevel@tonic-gate * 13147c478bd9Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed 13157c478bd9Sstevel@tonic-gate * to set up a new ppa[]. 13167c478bd9Sstevel@tonic-gate */ 13177c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 13187c478bd9Sstevel@tonic-gate 13197c478bd9Sstevel@tonic-gate if (pl_built) { 13207c478bd9Sstevel@tonic-gate /* 13217c478bd9Sstevel@tonic-gate * We created pl and we need to destroy it. 13227c478bd9Sstevel@tonic-gate */ 13237c478bd9Sstevel@tonic-gate pplist = pl; 13247c478bd9Sstevel@tonic-gate np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT); 13257c478bd9Sstevel@tonic-gate while (np) { 13267c478bd9Sstevel@tonic-gate page_unlock(*pplist); 13277c478bd9Sstevel@tonic-gate np--; 13287c478bd9Sstevel@tonic-gate pplist++; 13297c478bd9Sstevel@tonic-gate } 13307c478bd9Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * 13317c478bd9Sstevel@tonic-gate btopr(sptd->spt_amp->size)); 13327c478bd9Sstevel@tonic-gate } 13337c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 13347c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 13357c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 13367c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 13377c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 13387c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 13397c478bd9Sstevel@tonic-gate } 13407c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 13417c478bd9Sstevel@tonic-gate } 13427c478bd9Sstevel@tonic-gate } 13437c478bd9Sstevel@tonic-gate *ppp = NULL; 13447c478bd9Sstevel@tonic-gate return (ret); 13457c478bd9Sstevel@tonic-gate } 13467c478bd9Sstevel@tonic-gate 13477c478bd9Sstevel@tonic-gate /* 13487c478bd9Sstevel@tonic-gate * purge any cached pages in the I/O page cache 13497c478bd9Sstevel@tonic-gate */ 13507c478bd9Sstevel@tonic-gate static void 13517c478bd9Sstevel@tonic-gate segspt_purge(struct seg *seg) 13527c478bd9Sstevel@tonic-gate { 13537c478bd9Sstevel@tonic-gate seg_ppurge(seg); 13547c478bd9Sstevel@tonic-gate } 13557c478bd9Sstevel@tonic-gate 13567c478bd9Sstevel@tonic-gate static int 13577c478bd9Sstevel@tonic-gate segspt_reclaim(struct seg *seg, caddr_t addr, size_t len, struct page **pplist, 13587c478bd9Sstevel@tonic-gate enum seg_rw rw) 13597c478bd9Sstevel@tonic-gate { 13607c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 13617c478bd9Sstevel@tonic-gate struct seg *sptseg; 13627c478bd9Sstevel@tonic-gate struct spt_data *sptd; 13637c478bd9Sstevel@tonic-gate pgcnt_t npages, i, free_availrmem = 0; 13647c478bd9Sstevel@tonic-gate int done = 0; 13657c478bd9Sstevel@tonic-gate 13667c478bd9Sstevel@tonic-gate #ifdef lint 13677c478bd9Sstevel@tonic-gate addr = addr; 13687c478bd9Sstevel@tonic-gate #endif 13697c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 13707c478bd9Sstevel@tonic-gate sptd = sptseg->s_data; 13717c478bd9Sstevel@tonic-gate npages = (len >> PAGESHIFT); 13727c478bd9Sstevel@tonic-gate ASSERT(npages); 13737c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt != 0); 13747c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 13757c478bd9Sstevel@tonic-gate ASSERT(npages == btopr(sptd->spt_amp->size)); 13767c478bd9Sstevel@tonic-gate 13777c478bd9Sstevel@tonic-gate /* 13787c478bd9Sstevel@tonic-gate * Acquire the lock on the dummy seg and destroy the 13797c478bd9Sstevel@tonic-gate * ppa array IF this is the last pcachecnt. 13807c478bd9Sstevel@tonic-gate */ 13817c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 13827c478bd9Sstevel@tonic-gate if (--sptd->spt_pcachecnt == 0) { 13837c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 13847c478bd9Sstevel@tonic-gate if (pplist[i] == NULL) { 13857c478bd9Sstevel@tonic-gate continue; 13867c478bd9Sstevel@tonic-gate } 13877c478bd9Sstevel@tonic-gate if (rw == S_WRITE) { 13887c478bd9Sstevel@tonic-gate hat_setrefmod(pplist[i]); 13897c478bd9Sstevel@tonic-gate } else { 13907c478bd9Sstevel@tonic-gate hat_setref(pplist[i]); 13917c478bd9Sstevel@tonic-gate } 13927c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) && 13937c478bd9Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[i] == 0)) 13947c478bd9Sstevel@tonic-gate free_availrmem++; 13957c478bd9Sstevel@tonic-gate page_unlock(pplist[i]); 13967c478bd9Sstevel@tonic-gate } 13977c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 13987c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 13997c478bd9Sstevel@tonic-gate availrmem += free_availrmem; 14007c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 14017c478bd9Sstevel@tonic-gate } 14027c478bd9Sstevel@tonic-gate /* 14037c478bd9Sstevel@tonic-gate * Since we want to cach/uncache the entire ISM segment, 14047c478bd9Sstevel@tonic-gate * we will track the pplist in a segspt specific field 14057c478bd9Sstevel@tonic-gate * ppa, that is initialized at the time we add an entry to 14067c478bd9Sstevel@tonic-gate * the cache. 14077c478bd9Sstevel@tonic-gate */ 14087c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 14097c478bd9Sstevel@tonic-gate kmem_free(pplist, sizeof (page_t *) * npages); 14107c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 14117c478bd9Sstevel@tonic-gate sptd->spt_flags &= ~DISM_PPA_CHANGED; 14127c478bd9Sstevel@tonic-gate done = 1; 14137c478bd9Sstevel@tonic-gate } 14147c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 14157c478bd9Sstevel@tonic-gate /* 14167c478bd9Sstevel@tonic-gate * Now decrement softlockcnt. 14177c478bd9Sstevel@tonic-gate */ 14187c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 14217c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 14227c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 14237c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 14247c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 14257c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 14267c478bd9Sstevel@tonic-gate } 14277c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 14287c478bd9Sstevel@tonic-gate } 14297c478bd9Sstevel@tonic-gate } 14307c478bd9Sstevel@tonic-gate return (done); 14317c478bd9Sstevel@tonic-gate } 14327c478bd9Sstevel@tonic-gate 14337c478bd9Sstevel@tonic-gate /* 14347c478bd9Sstevel@tonic-gate * Do a F_SOFTUNLOCK call over the range requested. 14357c478bd9Sstevel@tonic-gate * The range must have already been F_SOFTLOCK'ed. 14367c478bd9Sstevel@tonic-gate * 14377c478bd9Sstevel@tonic-gate * The calls to acquire and release the anon map lock mutex were 14387c478bd9Sstevel@tonic-gate * removed in order to avoid a deadly embrace during a DR 14397c478bd9Sstevel@tonic-gate * memory delete operation. (Eg. DR blocks while waiting for a 14407c478bd9Sstevel@tonic-gate * exclusive lock on a page that is being used for kaio; the 14417c478bd9Sstevel@tonic-gate * thread that will complete the kaio and call segspt_softunlock 14427c478bd9Sstevel@tonic-gate * blocks on the anon map lock; another thread holding the anon 14437c478bd9Sstevel@tonic-gate * map lock blocks on another page lock via the segspt_shmfault 14447c478bd9Sstevel@tonic-gate * -> page_lookup -> page_lookup_create -> page_lock_es code flow.) 14457c478bd9Sstevel@tonic-gate * 14467c478bd9Sstevel@tonic-gate * The appropriateness of the removal is based upon the following: 14477c478bd9Sstevel@tonic-gate * 1. If we are holding a segment's reader lock and the page is held 14487c478bd9Sstevel@tonic-gate * shared, then the corresponding element in anonmap which points to 14497c478bd9Sstevel@tonic-gate * anon struct cannot change and there is no need to acquire the 14507c478bd9Sstevel@tonic-gate * anonymous map lock. 14517c478bd9Sstevel@tonic-gate * 2. Threads in segspt_softunlock have a reader lock on the segment 14527c478bd9Sstevel@tonic-gate * and already have the shared page lock, so we are guaranteed that 14537c478bd9Sstevel@tonic-gate * the anon map slot cannot change and therefore can call anon_get_ptr() 14547c478bd9Sstevel@tonic-gate * without grabbing the anonymous map lock. 14557c478bd9Sstevel@tonic-gate * 3. Threads that softlock a shared page break copy-on-write, even if 14567c478bd9Sstevel@tonic-gate * its a read. Thus cow faults can be ignored with respect to soft 14577c478bd9Sstevel@tonic-gate * unlocking, since the breaking of cow means that the anon slot(s) will 14587c478bd9Sstevel@tonic-gate * not be shared. 14597c478bd9Sstevel@tonic-gate */ 14607c478bd9Sstevel@tonic-gate static void 14617c478bd9Sstevel@tonic-gate segspt_softunlock(struct seg *seg, caddr_t sptseg_addr, 14627c478bd9Sstevel@tonic-gate size_t len, enum seg_rw rw) 14637c478bd9Sstevel@tonic-gate { 14647c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 14657c478bd9Sstevel@tonic-gate struct seg *sptseg; 14667c478bd9Sstevel@tonic-gate struct spt_data *sptd; 14677c478bd9Sstevel@tonic-gate page_t *pp; 14687c478bd9Sstevel@tonic-gate caddr_t adr; 14697c478bd9Sstevel@tonic-gate struct vnode *vp; 14707c478bd9Sstevel@tonic-gate u_offset_t offset; 14717c478bd9Sstevel@tonic-gate ulong_t anon_index; 14727c478bd9Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */ 14737c478bd9Sstevel@tonic-gate struct anon *ap = NULL; 14747c478bd9Sstevel@tonic-gate pgcnt_t npages; 14757c478bd9Sstevel@tonic-gate 14767c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 14777c478bd9Sstevel@tonic-gate 14787c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 14797c478bd9Sstevel@tonic-gate sptd = sptseg->s_data; 14807c478bd9Sstevel@tonic-gate 14817c478bd9Sstevel@tonic-gate /* 14827c478bd9Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 14837c478bd9Sstevel@tonic-gate * and therefore their pages are SE_SHARED locked 14847c478bd9Sstevel@tonic-gate * for the entire life of the segment. 14857c478bd9Sstevel@tonic-gate */ 14867c478bd9Sstevel@tonic-gate if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) && 14877c478bd9Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) == 0)) { 14887c478bd9Sstevel@tonic-gate goto softlock_decrement; 14897c478bd9Sstevel@tonic-gate } 14907c478bd9Sstevel@tonic-gate 14917c478bd9Sstevel@tonic-gate /* 14927c478bd9Sstevel@tonic-gate * Any thread is free to do a page_find and 14937c478bd9Sstevel@tonic-gate * page_unlock() on the pages within this seg. 14947c478bd9Sstevel@tonic-gate * 14957c478bd9Sstevel@tonic-gate * We are already holding the as->a_lock on the user's 14967c478bd9Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the 14977c478bd9Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the 14987c478bd9Sstevel@tonic-gate * underlying HAT layer. 14997c478bd9Sstevel@tonic-gate */ 15007c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 15017c478bd9Sstevel@tonic-gate hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len); 15027c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 15037c478bd9Sstevel@tonic-gate 15047c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 15057c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 15067c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr); 15077c478bd9Sstevel@tonic-gate 15087c478bd9Sstevel@tonic-gate for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) { 15097c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++); 15107c478bd9Sstevel@tonic-gate ASSERT(ap != NULL); 15117c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &offset); 15127c478bd9Sstevel@tonic-gate 15137c478bd9Sstevel@tonic-gate /* 15147c478bd9Sstevel@tonic-gate * Use page_find() instead of page_lookup() to 15157c478bd9Sstevel@tonic-gate * find the page since we know that it has a 15167c478bd9Sstevel@tonic-gate * "shared" lock. 15177c478bd9Sstevel@tonic-gate */ 15187c478bd9Sstevel@tonic-gate pp = page_find(vp, offset); 15197c478bd9Sstevel@tonic-gate ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1)); 15207c478bd9Sstevel@tonic-gate if (pp == NULL) { 15217c478bd9Sstevel@tonic-gate panic("segspt_softunlock: " 15227c478bd9Sstevel@tonic-gate "addr %p, ap %p, vp %p, off %llx", 15237c478bd9Sstevel@tonic-gate (void *)adr, (void *)ap, (void *)vp, offset); 15247c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 15257c478bd9Sstevel@tonic-gate } 15267c478bd9Sstevel@tonic-gate 15277c478bd9Sstevel@tonic-gate if (rw == S_WRITE) { 15287c478bd9Sstevel@tonic-gate hat_setrefmod(pp); 15297c478bd9Sstevel@tonic-gate } else if (rw != S_OTHER) { 15307c478bd9Sstevel@tonic-gate hat_setref(pp); 15317c478bd9Sstevel@tonic-gate } 15327c478bd9Sstevel@tonic-gate page_unlock(pp); 15337c478bd9Sstevel@tonic-gate } 15347c478bd9Sstevel@tonic-gate 15357c478bd9Sstevel@tonic-gate softlock_decrement: 15367c478bd9Sstevel@tonic-gate npages = btopr(len); 15377c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages); 15387c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt == 0) { 15397c478bd9Sstevel@tonic-gate /* 15407c478bd9Sstevel@tonic-gate * All SOFTLOCKS are gone. Wakeup any waiting 15417c478bd9Sstevel@tonic-gate * unmappers so they can try again to unmap. 15427c478bd9Sstevel@tonic-gate * Check for waiters first without the mutex 15437c478bd9Sstevel@tonic-gate * held so we don't always grab the mutex on 15447c478bd9Sstevel@tonic-gate * softunlocks. 15457c478bd9Sstevel@tonic-gate */ 15467c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 15477c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 15487c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 15497c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 15507c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 15517c478bd9Sstevel@tonic-gate } 15527c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 15537c478bd9Sstevel@tonic-gate } 15547c478bd9Sstevel@tonic-gate } 15557c478bd9Sstevel@tonic-gate } 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate int 15587c478bd9Sstevel@tonic-gate segspt_shmattach(struct seg *seg, caddr_t *argsp) 15597c478bd9Sstevel@tonic-gate { 15607c478bd9Sstevel@tonic-gate struct shm_data *shmd_arg = (struct shm_data *)argsp; 15617c478bd9Sstevel@tonic-gate struct shm_data *shmd; 15627c478bd9Sstevel@tonic-gate struct anon_map *shm_amp = shmd_arg->shm_amp; 15637c478bd9Sstevel@tonic-gate struct spt_data *sptd; 15647c478bd9Sstevel@tonic-gate int error = 0; 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 15677c478bd9Sstevel@tonic-gate 15687c478bd9Sstevel@tonic-gate shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP); 15697c478bd9Sstevel@tonic-gate if (shmd == NULL) 15707c478bd9Sstevel@tonic-gate return (ENOMEM); 15717c478bd9Sstevel@tonic-gate 15727c478bd9Sstevel@tonic-gate shmd->shm_sptas = shmd_arg->shm_sptas; 15737c478bd9Sstevel@tonic-gate shmd->shm_amp = shm_amp; 15747c478bd9Sstevel@tonic-gate shmd->shm_sptseg = shmd_arg->shm_sptseg; 15757c478bd9Sstevel@tonic-gate 15767c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0, 15777c478bd9Sstevel@tonic-gate NULL, 0, seg->s_size); 15787c478bd9Sstevel@tonic-gate 15797c478bd9Sstevel@tonic-gate seg->s_data = (void *)shmd; 15807c478bd9Sstevel@tonic-gate seg->s_ops = &segspt_shmops; 15817c478bd9Sstevel@tonic-gate seg->s_szc = shmd->shm_sptseg->s_szc; 15827c478bd9Sstevel@tonic-gate sptd = shmd->shm_sptseg->s_data; 15837c478bd9Sstevel@tonic-gate 15847c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 15857c478bd9Sstevel@tonic-gate if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size), 15867c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) { 15877c478bd9Sstevel@tonic-gate seg->s_data = (void *)NULL; 15887c478bd9Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd))); 15897c478bd9Sstevel@tonic-gate return (ENOMEM); 15907c478bd9Sstevel@tonic-gate } 15917c478bd9Sstevel@tonic-gate shmd->shm_lckpgs = 0; 15927c478bd9Sstevel@tonic-gate if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 15937c478bd9Sstevel@tonic-gate if ((error = hat_share(seg->s_as->a_hat, seg->s_base, 15947c478bd9Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 15957c478bd9Sstevel@tonic-gate seg->s_size, seg->s_szc)) != 0) { 15967c478bd9Sstevel@tonic-gate kmem_free(shmd->shm_vpage, 15977c478bd9Sstevel@tonic-gate btopr(shm_amp->size)); 15987c478bd9Sstevel@tonic-gate } 15997c478bd9Sstevel@tonic-gate } 16007c478bd9Sstevel@tonic-gate } else { 16017c478bd9Sstevel@tonic-gate error = hat_share(seg->s_as->a_hat, seg->s_base, 16027c478bd9Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 16037c478bd9Sstevel@tonic-gate seg->s_size, seg->s_szc); 16047c478bd9Sstevel@tonic-gate } 16057c478bd9Sstevel@tonic-gate if (error) { 16067c478bd9Sstevel@tonic-gate seg->s_szc = 0; 16077c478bd9Sstevel@tonic-gate seg->s_data = (void *)NULL; 16087c478bd9Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd))); 16097c478bd9Sstevel@tonic-gate } else { 16107c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 16117c478bd9Sstevel@tonic-gate shm_amp->refcnt++; 16127c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock); 16137c478bd9Sstevel@tonic-gate } 16147c478bd9Sstevel@tonic-gate return (error); 16157c478bd9Sstevel@tonic-gate } 16167c478bd9Sstevel@tonic-gate 16177c478bd9Sstevel@tonic-gate int 16187c478bd9Sstevel@tonic-gate segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize) 16197c478bd9Sstevel@tonic-gate { 16207c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 16217c478bd9Sstevel@tonic-gate int reclaim = 1; 16227c478bd9Sstevel@tonic-gate 16237c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 16247c478bd9Sstevel@tonic-gate retry: 16257c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) { 16267c478bd9Sstevel@tonic-gate if (reclaim == 1) { 16277c478bd9Sstevel@tonic-gate segspt_purge(seg); 16287c478bd9Sstevel@tonic-gate reclaim = 0; 16297c478bd9Sstevel@tonic-gate goto retry; 16307c478bd9Sstevel@tonic-gate } 16317c478bd9Sstevel@tonic-gate return (EAGAIN); 16327c478bd9Sstevel@tonic-gate } 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate if (ssize != seg->s_size) { 16357c478bd9Sstevel@tonic-gate #ifdef DEBUG 16367c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n", 16377c478bd9Sstevel@tonic-gate ssize, seg->s_size); 16387c478bd9Sstevel@tonic-gate #endif 16397c478bd9Sstevel@tonic-gate return (EINVAL); 16407c478bd9Sstevel@tonic-gate } 16417c478bd9Sstevel@tonic-gate 16427c478bd9Sstevel@tonic-gate (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK, 16437c478bd9Sstevel@tonic-gate NULL, 0); 16447c478bd9Sstevel@tonic-gate hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc); 16457c478bd9Sstevel@tonic-gate 16467c478bd9Sstevel@tonic-gate seg_free(seg); 16477c478bd9Sstevel@tonic-gate 16487c478bd9Sstevel@tonic-gate return (0); 16497c478bd9Sstevel@tonic-gate } 16507c478bd9Sstevel@tonic-gate 16517c478bd9Sstevel@tonic-gate void 16527c478bd9Sstevel@tonic-gate segspt_shmfree(struct seg *seg) 16537c478bd9Sstevel@tonic-gate { 16547c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 16557c478bd9Sstevel@tonic-gate struct anon_map *shm_amp = shmd->shm_amp; 16567c478bd9Sstevel@tonic-gate 16577c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 16587c478bd9Sstevel@tonic-gate 16597c478bd9Sstevel@tonic-gate (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0, 16607c478bd9Sstevel@tonic-gate MC_UNLOCK, NULL, 0); 16617c478bd9Sstevel@tonic-gate 16627c478bd9Sstevel@tonic-gate /* 16637c478bd9Sstevel@tonic-gate * Need to increment refcnt when attaching 16647c478bd9Sstevel@tonic-gate * and decrement when detaching because of dup(). 16657c478bd9Sstevel@tonic-gate */ 16667c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 16677c478bd9Sstevel@tonic-gate shm_amp->refcnt--; 16687c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock); 16697c478bd9Sstevel@tonic-gate 16707c478bd9Sstevel@tonic-gate if (shmd->shm_vpage) { /* only for DISM */ 16717c478bd9Sstevel@tonic-gate kmem_free(shmd->shm_vpage, btopr(shm_amp->size)); 16727c478bd9Sstevel@tonic-gate shmd->shm_vpage = NULL; 16737c478bd9Sstevel@tonic-gate } 16747c478bd9Sstevel@tonic-gate kmem_free(shmd, sizeof (*shmd)); 16757c478bd9Sstevel@tonic-gate } 16767c478bd9Sstevel@tonic-gate 16777c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16787c478bd9Sstevel@tonic-gate int 16797c478bd9Sstevel@tonic-gate segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 16807c478bd9Sstevel@tonic-gate { 16817c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 16827c478bd9Sstevel@tonic-gate 16837c478bd9Sstevel@tonic-gate /* 16847c478bd9Sstevel@tonic-gate * Shared page table is more than shared mapping. 16857c478bd9Sstevel@tonic-gate * Individual process sharing page tables can't change prot 16867c478bd9Sstevel@tonic-gate * because there is only one set of page tables. 16877c478bd9Sstevel@tonic-gate * This will be allowed after private page table is 16887c478bd9Sstevel@tonic-gate * supported. 16897c478bd9Sstevel@tonic-gate */ 16907c478bd9Sstevel@tonic-gate /* need to return correct status error? */ 16917c478bd9Sstevel@tonic-gate return (0); 16927c478bd9Sstevel@tonic-gate } 16937c478bd9Sstevel@tonic-gate 16947c478bd9Sstevel@tonic-gate 16957c478bd9Sstevel@tonic-gate faultcode_t 16967c478bd9Sstevel@tonic-gate segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr, 16977c478bd9Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw) 16987c478bd9Sstevel@tonic-gate { 16997c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 17007c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 17017c478bd9Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas; 17027c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 17037c478bd9Sstevel@tonic-gate pgcnt_t npages; 17047c478bd9Sstevel@tonic-gate size_t share_sz, size; 17057c478bd9Sstevel@tonic-gate caddr_t segspt_addr, shm_addr; 17067c478bd9Sstevel@tonic-gate page_t **ppa; 17077c478bd9Sstevel@tonic-gate int i; 17087c478bd9Sstevel@tonic-gate ulong_t an_idx = 0; 17097c478bd9Sstevel@tonic-gate int err = 0; 17101b42782eSmec int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0); 17117c478bd9Sstevel@tonic-gate 17127c478bd9Sstevel@tonic-gate #ifdef lint 17137c478bd9Sstevel@tonic-gate hat = hat; 17147c478bd9Sstevel@tonic-gate #endif 17157c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 17167c478bd9Sstevel@tonic-gate 17177c478bd9Sstevel@tonic-gate /* 17187c478bd9Sstevel@tonic-gate * Because of the way spt is implemented 17197c478bd9Sstevel@tonic-gate * the realsize of the segment does not have to be 17207c478bd9Sstevel@tonic-gate * equal to the segment size itself. The segment size is 17217c478bd9Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE. 17227c478bd9Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE 17237c478bd9Sstevel@tonic-gate * based on what the user requested. This is a bit of 17247c478bd9Sstevel@tonic-gate * ungliness that is historical but not easily fixed 17257c478bd9Sstevel@tonic-gate * without re-designing the higher levels of ISM. 17267c478bd9Sstevel@tonic-gate */ 17277c478bd9Sstevel@tonic-gate ASSERT(addr >= seg->s_base); 17287c478bd9Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize) 17297c478bd9Sstevel@tonic-gate return (FC_NOMAP); 17307c478bd9Sstevel@tonic-gate /* 17317c478bd9Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to 17327c478bd9Sstevel@tonic-gate * make any necessary adjustments to addr and len 17337c478bd9Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[]. 17347c478bd9Sstevel@tonic-gate * 17357c478bd9Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment 17367c478bd9Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore, 17377c478bd9Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 17387c478bd9Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 17397c478bd9Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT 17407c478bd9Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes 17417c478bd9Sstevel@tonic-gate * over a given virtual range. 17427c478bd9Sstevel@tonic-gate */ 17437c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(sptseg->s_szc); 17447c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 17457c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), share_sz); 17467c478bd9Sstevel@tonic-gate npages = btopr(size); 17477c478bd9Sstevel@tonic-gate 17487c478bd9Sstevel@tonic-gate /* 17497c478bd9Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt. 17507c478bd9Sstevel@tonic-gate */ 17517c478bd9Sstevel@tonic-gate an_idx = seg_page(seg, shm_addr); 17527c478bd9Sstevel@tonic-gate segspt_addr = sptseg->s_base + ptob(an_idx); 17537c478bd9Sstevel@tonic-gate 17547c478bd9Sstevel@tonic-gate ASSERT((segspt_addr + ptob(npages)) <= 17557c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)); 17567c478bd9Sstevel@tonic-gate ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size)); 17577c478bd9Sstevel@tonic-gate 17587c478bd9Sstevel@tonic-gate switch (type) { 17597c478bd9Sstevel@tonic-gate 17607c478bd9Sstevel@tonic-gate case F_SOFTLOCK: 17617c478bd9Sstevel@tonic-gate 17627c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 17637c478bd9Sstevel@tonic-gate if (availrmem < tune.t_minarmem + npages) { 17647c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 17657c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(ENOMEM)); 17667c478bd9Sstevel@tonic-gate } else { 17677c478bd9Sstevel@tonic-gate availrmem -= npages; 17687c478bd9Sstevel@tonic-gate } 17697c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 17707c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 17717c478bd9Sstevel@tonic-gate /* 17727c478bd9Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer 17737c478bd9Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag. 17747c478bd9Sstevel@tonic-gate */ 17757c478bd9Sstevel@tonic-gate /* FALLTHRU */ 17767c478bd9Sstevel@tonic-gate case F_INVAL: 17777c478bd9Sstevel@tonic-gate 17787c478bd9Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 17797c478bd9Sstevel@tonic-gate return (FC_NOMAP); 17807c478bd9Sstevel@tonic-gate 17817c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP); 17827c478bd9Sstevel@tonic-gate 17837c478bd9Sstevel@tonic-gate err = spt_anon_getpages(sptseg, segspt_addr, size, ppa); 17847c478bd9Sstevel@tonic-gate if (err != 0) { 17857c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 17867c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 17877c478bd9Sstevel@tonic-gate availrmem += npages; 17887c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 17897c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)( 17907c478bd9Sstevel@tonic-gate &(shmd->shm_softlockcnt)), -npages); 17917c478bd9Sstevel@tonic-gate } 17927c478bd9Sstevel@tonic-gate goto dism_err; 17937c478bd9Sstevel@tonic-gate } 17947c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 17957c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 17967c478bd9Sstevel@tonic-gate 17977c478bd9Sstevel@tonic-gate /* 17987c478bd9Sstevel@tonic-gate * Load up the translation keeping it 17997c478bd9Sstevel@tonic-gate * locked and don't unlock the page. 18007c478bd9Sstevel@tonic-gate */ 18017c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, segspt_addr, 18027c478bd9Sstevel@tonic-gate size, ppa, sptd->spt_prot, 18037c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | HAT_LOAD_SHARE); 18047c478bd9Sstevel@tonic-gate } else { 18057c478bd9Sstevel@tonic-gate if (hat == seg->s_as->a_hat) { 18067c478bd9Sstevel@tonic-gate 18077c478bd9Sstevel@tonic-gate /* 18087c478bd9Sstevel@tonic-gate * Migrate pages marked for migration 18097c478bd9Sstevel@tonic-gate */ 18107c478bd9Sstevel@tonic-gate if (lgrp_optimizations()) 18117c478bd9Sstevel@tonic-gate page_migrate(seg, shm_addr, ppa, 18127c478bd9Sstevel@tonic-gate npages); 18137c478bd9Sstevel@tonic-gate 18147c478bd9Sstevel@tonic-gate /* CPU HAT */ 18157c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, 18167c478bd9Sstevel@tonic-gate segspt_addr, size, ppa, sptd->spt_prot, 18177c478bd9Sstevel@tonic-gate HAT_LOAD_SHARE); 18187c478bd9Sstevel@tonic-gate } else { 18197c478bd9Sstevel@tonic-gate /* XHAT. Pass real address */ 18207c478bd9Sstevel@tonic-gate hat_memload_array(hat, shm_addr, 18217c478bd9Sstevel@tonic-gate size, ppa, sptd->spt_prot, HAT_LOAD_SHARE); 18227c478bd9Sstevel@tonic-gate } 18237c478bd9Sstevel@tonic-gate 18247c478bd9Sstevel@tonic-gate /* 18257c478bd9Sstevel@tonic-gate * And now drop the SE_SHARED lock(s). 18267c478bd9Sstevel@tonic-gate */ 18271b42782eSmec if (dyn_ism_unmap) { 18281b42782eSmec for (i = 0; i < npages; i++) { 18297c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 18307c478bd9Sstevel@tonic-gate } 18311b42782eSmec } 18321b42782eSmec } 18337c478bd9Sstevel@tonic-gate 18341b42782eSmec if (!dyn_ism_unmap) { 18357c478bd9Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, shm_addr, 18367c478bd9Sstevel@tonic-gate curspt->a_hat, segspt_addr, ptob(npages), 18377c478bd9Sstevel@tonic-gate seg->s_szc) != 0) { 18387c478bd9Sstevel@tonic-gate panic("hat_share err in DISM fault"); 18397c478bd9Sstevel@tonic-gate /* NOTREACHED */ 18407c478bd9Sstevel@tonic-gate } 18411b42782eSmec if (type == F_INVAL) { 18421b42782eSmec for (i = 0; i < npages; i++) { 18431b42782eSmec page_unlock(ppa[i]); 18441b42782eSmec } 18451b42782eSmec } 18467c478bd9Sstevel@tonic-gate } 18477c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 18487c478bd9Sstevel@tonic-gate dism_err: 18497c478bd9Sstevel@tonic-gate kmem_free(ppa, npages * sizeof (page_t *)); 18507c478bd9Sstevel@tonic-gate return (err); 18517c478bd9Sstevel@tonic-gate 18527c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 18537c478bd9Sstevel@tonic-gate 18547c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 18557c478bd9Sstevel@tonic-gate availrmem += npages; 18567c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 18577c478bd9Sstevel@tonic-gate 18587c478bd9Sstevel@tonic-gate /* 18597c478bd9Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer, 18607c478bd9Sstevel@tonic-gate * but the segspt_addr is the virtual address within the 18617c478bd9Sstevel@tonic-gate * dummy seg. 18627c478bd9Sstevel@tonic-gate */ 18637c478bd9Sstevel@tonic-gate segspt_softunlock(seg, segspt_addr, size, rw); 18647c478bd9Sstevel@tonic-gate return (0); 18657c478bd9Sstevel@tonic-gate 18667c478bd9Sstevel@tonic-gate case F_PROT: 18677c478bd9Sstevel@tonic-gate 18687c478bd9Sstevel@tonic-gate /* 18697c478bd9Sstevel@tonic-gate * This takes care of the unusual case where a user 18707c478bd9Sstevel@tonic-gate * allocates a stack in shared memory and a register 18717c478bd9Sstevel@tonic-gate * window overflow is written to that stack page before 18727c478bd9Sstevel@tonic-gate * it is otherwise modified. 18737c478bd9Sstevel@tonic-gate * 18747c478bd9Sstevel@tonic-gate * We can get away with this because ISM segments are 18757c478bd9Sstevel@tonic-gate * always rw. Other than this unusual case, there 18767c478bd9Sstevel@tonic-gate * should be no instances of protection violations. 18777c478bd9Sstevel@tonic-gate */ 18787c478bd9Sstevel@tonic-gate return (0); 18797c478bd9Sstevel@tonic-gate 18807c478bd9Sstevel@tonic-gate default: 18817c478bd9Sstevel@tonic-gate #ifdef DEBUG 18827c478bd9Sstevel@tonic-gate panic("segspt_dismfault default type?"); 18837c478bd9Sstevel@tonic-gate #else 18847c478bd9Sstevel@tonic-gate return (FC_NOMAP); 18857c478bd9Sstevel@tonic-gate #endif 18867c478bd9Sstevel@tonic-gate } 18877c478bd9Sstevel@tonic-gate } 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate 18907c478bd9Sstevel@tonic-gate faultcode_t 18917c478bd9Sstevel@tonic-gate segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr, 18927c478bd9Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw) 18937c478bd9Sstevel@tonic-gate { 18947c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 18957c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 18967c478bd9Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas; 18977c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 18987c478bd9Sstevel@tonic-gate pgcnt_t npages; 18997c478bd9Sstevel@tonic-gate size_t share_size, size; 19007c478bd9Sstevel@tonic-gate caddr_t sptseg_addr, shm_addr; 19017c478bd9Sstevel@tonic-gate page_t *pp, **ppa; 19027c478bd9Sstevel@tonic-gate int i; 19037c478bd9Sstevel@tonic-gate u_offset_t offset; 19047c478bd9Sstevel@tonic-gate ulong_t anon_index = 0; 19057c478bd9Sstevel@tonic-gate struct vnode *vp; 19067c478bd9Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */ 19077c478bd9Sstevel@tonic-gate struct anon *ap = NULL; 19087c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 19097c478bd9Sstevel@tonic-gate 19107c478bd9Sstevel@tonic-gate #ifdef lint 19117c478bd9Sstevel@tonic-gate hat = hat; 19127c478bd9Sstevel@tonic-gate #endif 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 19157c478bd9Sstevel@tonic-gate 19167c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 19177c478bd9Sstevel@tonic-gate return (segspt_dismfault(hat, seg, addr, len, type, rw)); 19187c478bd9Sstevel@tonic-gate } 19197c478bd9Sstevel@tonic-gate 19207c478bd9Sstevel@tonic-gate /* 19217c478bd9Sstevel@tonic-gate * Because of the way spt is implemented 19227c478bd9Sstevel@tonic-gate * the realsize of the segment does not have to be 19237c478bd9Sstevel@tonic-gate * equal to the segment size itself. The segment size is 19247c478bd9Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE. 19257c478bd9Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE 19267c478bd9Sstevel@tonic-gate * based on what the user requested. This is a bit of 19277c478bd9Sstevel@tonic-gate * ungliness that is historical but not easily fixed 19287c478bd9Sstevel@tonic-gate * without re-designing the higher levels of ISM. 19297c478bd9Sstevel@tonic-gate */ 19307c478bd9Sstevel@tonic-gate ASSERT(addr >= seg->s_base); 19317c478bd9Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize) 19327c478bd9Sstevel@tonic-gate return (FC_NOMAP); 19337c478bd9Sstevel@tonic-gate /* 19347c478bd9Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to 19357c478bd9Sstevel@tonic-gate * make any necessary adjustments to addr and len 19367c478bd9Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[]. 19377c478bd9Sstevel@tonic-gate * 19387c478bd9Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment 19397c478bd9Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore, 19407c478bd9Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 19417c478bd9Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 19427c478bd9Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT 19437c478bd9Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes 19447c478bd9Sstevel@tonic-gate * over a given virtual range. 19457c478bd9Sstevel@tonic-gate */ 19467c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(sptseg->s_szc); 19477c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 19487c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), share_size); 19497c478bd9Sstevel@tonic-gate npages = btopr(size); 19507c478bd9Sstevel@tonic-gate 19517c478bd9Sstevel@tonic-gate /* 19527c478bd9Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt. 19537c478bd9Sstevel@tonic-gate */ 19547c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr); 19557c478bd9Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index); 19567c478bd9Sstevel@tonic-gate 19577c478bd9Sstevel@tonic-gate /* 19587c478bd9Sstevel@tonic-gate * And now we may have to adjust npages downward if we have 19597c478bd9Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon 19607c478bd9Sstevel@tonic-gate * allocations. 19617c478bd9Sstevel@tonic-gate */ 19627c478bd9Sstevel@tonic-gate if ((sptseg_addr + ptob(npages)) > 19637c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)) 19647c478bd9Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr; 19657c478bd9Sstevel@tonic-gate 19667c478bd9Sstevel@tonic-gate npages = btopr(size); 19677c478bd9Sstevel@tonic-gate 19687c478bd9Sstevel@tonic-gate ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size)); 19697c478bd9Sstevel@tonic-gate ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0); 19707c478bd9Sstevel@tonic-gate 19717c478bd9Sstevel@tonic-gate switch (type) { 19727c478bd9Sstevel@tonic-gate 19737c478bd9Sstevel@tonic-gate case F_SOFTLOCK: 19747c478bd9Sstevel@tonic-gate 19757c478bd9Sstevel@tonic-gate /* 19767c478bd9Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust() 19777c478bd9Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is 19787c478bd9Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed. 19797c478bd9Sstevel@tonic-gate */ 19807c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 19817c478bd9Sstevel@tonic-gate /* 19827c478bd9Sstevel@tonic-gate * Some platforms assume that ISM pages are SE_SHARED 19837c478bd9Sstevel@tonic-gate * locked for the entire life of the segment. 19847c478bd9Sstevel@tonic-gate */ 19857c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) 19867c478bd9Sstevel@tonic-gate return (0); 19877c478bd9Sstevel@tonic-gate /* 19887c478bd9Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer 19897c478bd9Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag. 19907c478bd9Sstevel@tonic-gate */ 19917c478bd9Sstevel@tonic-gate 19927c478bd9Sstevel@tonic-gate /* FALLTHRU */ 19937c478bd9Sstevel@tonic-gate case F_INVAL: 19947c478bd9Sstevel@tonic-gate 19957c478bd9Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 19967c478bd9Sstevel@tonic-gate return (FC_NOMAP); 19977c478bd9Sstevel@tonic-gate 19987c478bd9Sstevel@tonic-gate /* 19997c478bd9Sstevel@tonic-gate * Some platforms that do NOT support DYNAMIC_ISM_UNMAP 20007c478bd9Sstevel@tonic-gate * may still rely on this call to hat_share(). That 20017c478bd9Sstevel@tonic-gate * would imply that those hat's can fault on a 20027c478bd9Sstevel@tonic-gate * HAT_LOAD_LOCK translation, which would seem 20037c478bd9Sstevel@tonic-gate * contradictory. 20047c478bd9Sstevel@tonic-gate */ 20057c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 20067c478bd9Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, seg->s_base, 20077c478bd9Sstevel@tonic-gate curspt->a_hat, sptseg->s_base, 20087c478bd9Sstevel@tonic-gate sptseg->s_size, sptseg->s_szc) != 0) { 20097c478bd9Sstevel@tonic-gate panic("hat_share error in ISM fault"); 20107c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 20117c478bd9Sstevel@tonic-gate } 20127c478bd9Sstevel@tonic-gate return (0); 20137c478bd9Sstevel@tonic-gate } 20147c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP); 20157c478bd9Sstevel@tonic-gate 20167c478bd9Sstevel@tonic-gate /* 20177c478bd9Sstevel@tonic-gate * I see no need to lock the real seg, 20187c478bd9Sstevel@tonic-gate * here, because all of our work will be on the underlying 20197c478bd9Sstevel@tonic-gate * dummy seg. 20207c478bd9Sstevel@tonic-gate * 20217c478bd9Sstevel@tonic-gate * sptseg_addr and npages now account for large pages. 20227c478bd9Sstevel@tonic-gate */ 20237c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 20247c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 20257c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr); 20267c478bd9Sstevel@tonic-gate 20277c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 20287c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 20297c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 20307c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++); 20317c478bd9Sstevel@tonic-gate ASSERT(ap != NULL); 20327c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &offset); 20337c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 20347c478bd9Sstevel@tonic-gate pp = page_lookup(vp, offset, SE_SHARED); 20357c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 20367c478bd9Sstevel@tonic-gate ppa[i] = pp; 20377c478bd9Sstevel@tonic-gate } 20387c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 20397c478bd9Sstevel@tonic-gate ASSERT(i == npages); 20407c478bd9Sstevel@tonic-gate 20417c478bd9Sstevel@tonic-gate /* 20427c478bd9Sstevel@tonic-gate * We are already holding the as->a_lock on the user's 20437c478bd9Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the 20447c478bd9Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the 20457c478bd9Sstevel@tonic-gate * underlying HAT layer. 20467c478bd9Sstevel@tonic-gate */ 20477c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 20487c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 20497c478bd9Sstevel@tonic-gate /* 20507c478bd9Sstevel@tonic-gate * Load up the translation keeping it 20517c478bd9Sstevel@tonic-gate * locked and don't unlock the page. 20527c478bd9Sstevel@tonic-gate */ 20537c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, sptseg_addr, 20547c478bd9Sstevel@tonic-gate ptob(npages), ppa, sptd->spt_prot, 20557c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | HAT_LOAD_SHARE); 20567c478bd9Sstevel@tonic-gate } else { 20577c478bd9Sstevel@tonic-gate if (hat == seg->s_as->a_hat) { 20587c478bd9Sstevel@tonic-gate 20597c478bd9Sstevel@tonic-gate /* 20607c478bd9Sstevel@tonic-gate * Migrate pages marked for migration. 20617c478bd9Sstevel@tonic-gate */ 20627c478bd9Sstevel@tonic-gate if (lgrp_optimizations()) 20637c478bd9Sstevel@tonic-gate page_migrate(seg, shm_addr, ppa, 20647c478bd9Sstevel@tonic-gate npages); 20657c478bd9Sstevel@tonic-gate 20667c478bd9Sstevel@tonic-gate /* CPU HAT */ 20677c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, 20687c478bd9Sstevel@tonic-gate sptseg_addr, ptob(npages), ppa, 20697c478bd9Sstevel@tonic-gate sptd->spt_prot, HAT_LOAD_SHARE); 20707c478bd9Sstevel@tonic-gate } else { 20717c478bd9Sstevel@tonic-gate /* XHAT. Pass real address */ 20727c478bd9Sstevel@tonic-gate hat_memload_array(hat, shm_addr, 20737c478bd9Sstevel@tonic-gate ptob(npages), ppa, sptd->spt_prot, 20747c478bd9Sstevel@tonic-gate HAT_LOAD_SHARE); 20757c478bd9Sstevel@tonic-gate } 20767c478bd9Sstevel@tonic-gate 20777c478bd9Sstevel@tonic-gate /* 20787c478bd9Sstevel@tonic-gate * And now drop the SE_SHARED lock(s). 20797c478bd9Sstevel@tonic-gate */ 20807c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) 20817c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 20827c478bd9Sstevel@tonic-gate } 20837c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 20847c478bd9Sstevel@tonic-gate 20857c478bd9Sstevel@tonic-gate kmem_free(ppa, sizeof (page_t *) * npages); 20867c478bd9Sstevel@tonic-gate return (0); 20877c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 20887c478bd9Sstevel@tonic-gate 20897c478bd9Sstevel@tonic-gate /* 20907c478bd9Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer, 20917c478bd9Sstevel@tonic-gate * but the sptseg_addr is the virtual address within the 20927c478bd9Sstevel@tonic-gate * dummy seg. 20937c478bd9Sstevel@tonic-gate */ 20947c478bd9Sstevel@tonic-gate segspt_softunlock(seg, sptseg_addr, ptob(npages), rw); 20957c478bd9Sstevel@tonic-gate return (0); 20967c478bd9Sstevel@tonic-gate 20977c478bd9Sstevel@tonic-gate case F_PROT: 20987c478bd9Sstevel@tonic-gate 20997c478bd9Sstevel@tonic-gate /* 21007c478bd9Sstevel@tonic-gate * This takes care of the unusual case where a user 21017c478bd9Sstevel@tonic-gate * allocates a stack in shared memory and a register 21027c478bd9Sstevel@tonic-gate * window overflow is written to that stack page before 21037c478bd9Sstevel@tonic-gate * it is otherwise modified. 21047c478bd9Sstevel@tonic-gate * 21057c478bd9Sstevel@tonic-gate * We can get away with this because ISM segments are 21067c478bd9Sstevel@tonic-gate * always rw. Other than this unusual case, there 21077c478bd9Sstevel@tonic-gate * should be no instances of protection violations. 21087c478bd9Sstevel@tonic-gate */ 21097c478bd9Sstevel@tonic-gate return (0); 21107c478bd9Sstevel@tonic-gate 21117c478bd9Sstevel@tonic-gate default: 21127c478bd9Sstevel@tonic-gate #ifdef DEBUG 21137c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "segspt_shmfault default type?"); 21147c478bd9Sstevel@tonic-gate #endif 21157c478bd9Sstevel@tonic-gate return (FC_NOMAP); 21167c478bd9Sstevel@tonic-gate } 21177c478bd9Sstevel@tonic-gate } 21187c478bd9Sstevel@tonic-gate 21197c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 21207c478bd9Sstevel@tonic-gate static faultcode_t 21217c478bd9Sstevel@tonic-gate segspt_shmfaulta(struct seg *seg, caddr_t addr) 21227c478bd9Sstevel@tonic-gate { 21237c478bd9Sstevel@tonic-gate return (0); 21247c478bd9Sstevel@tonic-gate } 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 21277c478bd9Sstevel@tonic-gate static int 21287c478bd9Sstevel@tonic-gate segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta) 21297c478bd9Sstevel@tonic-gate { 21307c478bd9Sstevel@tonic-gate return (0); 21317c478bd9Sstevel@tonic-gate } 21327c478bd9Sstevel@tonic-gate 21337c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 21347c478bd9Sstevel@tonic-gate static size_t 21357c478bd9Sstevel@tonic-gate segspt_shmswapout(struct seg *seg) 21367c478bd9Sstevel@tonic-gate { 21377c478bd9Sstevel@tonic-gate return (0); 21387c478bd9Sstevel@tonic-gate } 21397c478bd9Sstevel@tonic-gate 21407c478bd9Sstevel@tonic-gate /* 21417c478bd9Sstevel@tonic-gate * duplicate the shared page tables 21427c478bd9Sstevel@tonic-gate */ 21437c478bd9Sstevel@tonic-gate int 21447c478bd9Sstevel@tonic-gate segspt_shmdup(struct seg *seg, struct seg *newseg) 21457c478bd9Sstevel@tonic-gate { 21467c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 21477c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 21487c478bd9Sstevel@tonic-gate struct shm_data *shmd_new; 21497c478bd9Sstevel@tonic-gate struct seg *spt_seg = shmd->shm_sptseg; 21507c478bd9Sstevel@tonic-gate struct spt_data *sptd = spt_seg->s_data; 21511b42782eSmec int error = 0; 21527c478bd9Sstevel@tonic-gate 21537c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 21547c478bd9Sstevel@tonic-gate 21557c478bd9Sstevel@tonic-gate shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP); 21567c478bd9Sstevel@tonic-gate newseg->s_data = (void *)shmd_new; 21577c478bd9Sstevel@tonic-gate shmd_new->shm_sptas = shmd->shm_sptas; 21587c478bd9Sstevel@tonic-gate shmd_new->shm_amp = amp; 21597c478bd9Sstevel@tonic-gate shmd_new->shm_sptseg = shmd->shm_sptseg; 21607c478bd9Sstevel@tonic-gate newseg->s_ops = &segspt_shmops; 21617c478bd9Sstevel@tonic-gate newseg->s_szc = seg->s_szc; 21627c478bd9Sstevel@tonic-gate ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc); 21637c478bd9Sstevel@tonic-gate 21647c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 21657c478bd9Sstevel@tonic-gate amp->refcnt++; 21667c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 21677c478bd9Sstevel@tonic-gate 21687c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 21697c478bd9Sstevel@tonic-gate shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP); 21707c478bd9Sstevel@tonic-gate shmd_new->shm_lckpgs = 0; 21711b42782eSmec if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 21721b42782eSmec if ((error = hat_share(newseg->s_as->a_hat, 21731b42782eSmec newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR, 21741b42782eSmec seg->s_size, seg->s_szc)) != 0) { 21751b42782eSmec kmem_free(shmd_new->shm_vpage, 21761b42782eSmec btopr(amp->size)); 21777c478bd9Sstevel@tonic-gate } 21781b42782eSmec } 21791b42782eSmec return (error); 21801b42782eSmec } else { 21817c478bd9Sstevel@tonic-gate return (hat_share(newseg->s_as->a_hat, newseg->s_base, 21821b42782eSmec shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size, 21831b42782eSmec seg->s_szc)); 21841b42782eSmec 21851b42782eSmec } 21867c478bd9Sstevel@tonic-gate } 21877c478bd9Sstevel@tonic-gate 21887c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 21897c478bd9Sstevel@tonic-gate int 21907c478bd9Sstevel@tonic-gate segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 21917c478bd9Sstevel@tonic-gate { 21927c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 21937c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 21947c478bd9Sstevel@tonic-gate 21957c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate /* 21987c478bd9Sstevel@tonic-gate * ISM segment is always rw. 21997c478bd9Sstevel@tonic-gate */ 22007c478bd9Sstevel@tonic-gate return (((sptd->spt_prot & prot) != prot) ? EACCES : 0); 22017c478bd9Sstevel@tonic-gate } 22027c478bd9Sstevel@tonic-gate 22037c478bd9Sstevel@tonic-gate /* 22047c478bd9Sstevel@tonic-gate * Return an array of locked large pages, for empty slots allocate 22057c478bd9Sstevel@tonic-gate * private zero-filled anon pages. 22067c478bd9Sstevel@tonic-gate */ 22077c478bd9Sstevel@tonic-gate static int 22087c478bd9Sstevel@tonic-gate spt_anon_getpages( 22097c478bd9Sstevel@tonic-gate struct seg *sptseg, 22107c478bd9Sstevel@tonic-gate caddr_t sptaddr, 22117c478bd9Sstevel@tonic-gate size_t len, 22127c478bd9Sstevel@tonic-gate page_t *ppa[]) 22137c478bd9Sstevel@tonic-gate { 22147c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 22157c478bd9Sstevel@tonic-gate struct anon_map *amp = sptd->spt_amp; 22167c478bd9Sstevel@tonic-gate enum seg_rw rw = sptd->spt_prot; 22177c478bd9Sstevel@tonic-gate uint_t szc = sptseg->s_szc; 22187c478bd9Sstevel@tonic-gate size_t pg_sz, share_sz = page_get_pagesize(szc); 22197c478bd9Sstevel@tonic-gate pgcnt_t lp_npgs; 22207c478bd9Sstevel@tonic-gate caddr_t lp_addr, e_sptaddr; 22217c478bd9Sstevel@tonic-gate uint_t vpprot, ppa_szc = 0; 22227c478bd9Sstevel@tonic-gate struct vpage *vpage = NULL; 22237c478bd9Sstevel@tonic-gate ulong_t j, ppa_idx; 22247c478bd9Sstevel@tonic-gate int err, ierr = 0; 22257c478bd9Sstevel@tonic-gate pgcnt_t an_idx; 22267c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz)); 22297c478bd9Sstevel@tonic-gate ASSERT(len != 0); 22307c478bd9Sstevel@tonic-gate 22317c478bd9Sstevel@tonic-gate pg_sz = share_sz; 22327c478bd9Sstevel@tonic-gate lp_npgs = btop(pg_sz); 22337c478bd9Sstevel@tonic-gate lp_addr = sptaddr; 22347c478bd9Sstevel@tonic-gate e_sptaddr = sptaddr + len; 22357c478bd9Sstevel@tonic-gate an_idx = seg_page(sptseg, sptaddr); 22367c478bd9Sstevel@tonic-gate ppa_idx = 0; 22377c478bd9Sstevel@tonic-gate 22387c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 22397c478bd9Sstevel@tonic-gate /*CONSTCOND*/ 22407c478bd9Sstevel@tonic-gate while (1) { 22417c478bd9Sstevel@tonic-gate for (; lp_addr < e_sptaddr; 22427c478bd9Sstevel@tonic-gate an_idx += lp_npgs, lp_addr += pg_sz, 22437c478bd9Sstevel@tonic-gate ppa_idx += lp_npgs) { 22447c478bd9Sstevel@tonic-gate 22457c478bd9Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 22467c478bd9Sstevel@tonic-gate ppa_szc = (uint_t)-1; 22477c478bd9Sstevel@tonic-gate ierr = anon_map_getpages(amp, an_idx, szc, sptseg, 22487c478bd9Sstevel@tonic-gate lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx], 22497c478bd9Sstevel@tonic-gate &ppa_szc, vpage, rw, 0, segvn_anypgsz, kcred); 22507c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 22517c478bd9Sstevel@tonic-gate 22527c478bd9Sstevel@tonic-gate if (ierr != 0) { 22537c478bd9Sstevel@tonic-gate if (ierr > 0) { 22547c478bd9Sstevel@tonic-gate err = FC_MAKE_ERR(ierr); 22557c478bd9Sstevel@tonic-gate goto lpgs_err; 22567c478bd9Sstevel@tonic-gate } 22577c478bd9Sstevel@tonic-gate break; 22587c478bd9Sstevel@tonic-gate } 22597c478bd9Sstevel@tonic-gate } 22607c478bd9Sstevel@tonic-gate if (lp_addr == e_sptaddr) { 22617c478bd9Sstevel@tonic-gate break; 22627c478bd9Sstevel@tonic-gate } 22637c478bd9Sstevel@tonic-gate ASSERT(lp_addr < e_sptaddr); 22647c478bd9Sstevel@tonic-gate 22657c478bd9Sstevel@tonic-gate /* 22667c478bd9Sstevel@tonic-gate * ierr == -1 means we failed to allocate a large page. 22677c478bd9Sstevel@tonic-gate * so do a size down operation. 22687c478bd9Sstevel@tonic-gate * 22697c478bd9Sstevel@tonic-gate * ierr == -2 means some other process that privately shares 22707c478bd9Sstevel@tonic-gate * pages with this process has allocated a larger page and we 22717c478bd9Sstevel@tonic-gate * need to retry with larger pages. So do a size up 22727c478bd9Sstevel@tonic-gate * operation. This relies on the fact that large pages are 22737c478bd9Sstevel@tonic-gate * never partially shared i.e. if we share any constituent 22747c478bd9Sstevel@tonic-gate * page of a large page with another process we must share the 22757c478bd9Sstevel@tonic-gate * entire large page. Note this cannot happen for SOFTLOCK 22767c478bd9Sstevel@tonic-gate * case, unless current address (lpaddr) is at the beginning 22777c478bd9Sstevel@tonic-gate * of the next page size boundary because the other process 22787c478bd9Sstevel@tonic-gate * couldn't have relocated locked pages. 22797c478bd9Sstevel@tonic-gate */ 22807c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || ierr == -2); 22817c478bd9Sstevel@tonic-gate if (segvn_anypgsz) { 22827c478bd9Sstevel@tonic-gate ASSERT(ierr == -2 || szc != 0); 22837c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || szc < sptseg->s_szc); 22847c478bd9Sstevel@tonic-gate szc = (ierr == -1) ? szc - 1 : szc + 1; 22857c478bd9Sstevel@tonic-gate } else { 22867c478bd9Sstevel@tonic-gate /* 22877c478bd9Sstevel@tonic-gate * For faults and segvn_anypgsz == 0 22887c478bd9Sstevel@tonic-gate * we need to be careful not to loop forever 22897c478bd9Sstevel@tonic-gate * if existing page is found with szc other 22907c478bd9Sstevel@tonic-gate * than 0 or seg->s_szc. This could be due 22917c478bd9Sstevel@tonic-gate * to page relocations on behalf of DR or 22927c478bd9Sstevel@tonic-gate * more likely large page creation. For this 22937c478bd9Sstevel@tonic-gate * case simply re-size to existing page's szc 22947c478bd9Sstevel@tonic-gate * if returned by anon_map_getpages(). 22957c478bd9Sstevel@tonic-gate */ 22967c478bd9Sstevel@tonic-gate if (ppa_szc == (uint_t)-1) { 22977c478bd9Sstevel@tonic-gate szc = (ierr == -1) ? 0 : sptseg->s_szc; 22987c478bd9Sstevel@tonic-gate } else { 22997c478bd9Sstevel@tonic-gate ASSERT(ppa_szc <= sptseg->s_szc); 23007c478bd9Sstevel@tonic-gate ASSERT(ierr == -2 || ppa_szc < szc); 23017c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || ppa_szc > szc); 23027c478bd9Sstevel@tonic-gate szc = ppa_szc; 23037c478bd9Sstevel@tonic-gate } 23047c478bd9Sstevel@tonic-gate } 23057c478bd9Sstevel@tonic-gate pg_sz = page_get_pagesize(szc); 23067c478bd9Sstevel@tonic-gate lp_npgs = btop(pg_sz); 23077c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(lp_addr, pg_sz)); 23087c478bd9Sstevel@tonic-gate } 23097c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 23107c478bd9Sstevel@tonic-gate return (0); 23117c478bd9Sstevel@tonic-gate 23127c478bd9Sstevel@tonic-gate lpgs_err: 23137c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 23147c478bd9Sstevel@tonic-gate for (j = 0; j < ppa_idx; j++) 23157c478bd9Sstevel@tonic-gate page_unlock(ppa[j]); 23167c478bd9Sstevel@tonic-gate return (err); 23177c478bd9Sstevel@tonic-gate } 23187c478bd9Sstevel@tonic-gate 23197c478bd9Sstevel@tonic-gate int 23207c478bd9Sstevel@tonic-gate spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 23217c478bd9Sstevel@tonic-gate page_t **ppa, ulong_t *lockmap, size_t pos) 23227c478bd9Sstevel@tonic-gate { 23237c478bd9Sstevel@tonic-gate struct shm_data *shmd = seg->s_data; 23247c478bd9Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data; 23257c478bd9Sstevel@tonic-gate ulong_t i; 23267c478bd9Sstevel@tonic-gate int kernel; 23277c478bd9Sstevel@tonic-gate 23287c478bd9Sstevel@tonic-gate for (i = 0; i < npages; anon_index++, pos++, i++) { 23297c478bd9Sstevel@tonic-gate if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) { 23307c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[anon_index] < 23317c478bd9Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) { 23327c478bd9Sstevel@tonic-gate if (++sptd->spt_ppa_lckcnt[anon_index] == 23337c478bd9Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) { 23347c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 23357c478bd9Sstevel@tonic-gate "DISM page lock limit " 23367c478bd9Sstevel@tonic-gate "reached on DISM offset 0x%lx\n", 23377c478bd9Sstevel@tonic-gate anon_index << PAGESHIFT); 23387c478bd9Sstevel@tonic-gate } 23397c478bd9Sstevel@tonic-gate kernel = (sptd->spt_ppa && 23407c478bd9Sstevel@tonic-gate sptd->spt_ppa[anon_index]) ? 1 : 0; 23417c478bd9Sstevel@tonic-gate if (!page_pp_lock(ppa[i], 0, kernel)) { 23427c478bd9Sstevel@tonic-gate /* unlock rest of the pages */ 23437c478bd9Sstevel@tonic-gate for (; i < npages; i++) 23447c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 23457c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt[anon_index]--; 23467c478bd9Sstevel@tonic-gate return (EAGAIN); 23477c478bd9Sstevel@tonic-gate } 23487c478bd9Sstevel@tonic-gate shmd->shm_lckpgs++; 23497c478bd9Sstevel@tonic-gate shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED; 23507c478bd9Sstevel@tonic-gate if (lockmap != NULL) 23517c478bd9Sstevel@tonic-gate BT_SET(lockmap, pos); 23527c478bd9Sstevel@tonic-gate } 23537c478bd9Sstevel@tonic-gate } 23547c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 23557c478bd9Sstevel@tonic-gate } 23567c478bd9Sstevel@tonic-gate return (0); 23577c478bd9Sstevel@tonic-gate } 23587c478bd9Sstevel@tonic-gate 23597c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23607c478bd9Sstevel@tonic-gate static int 23617c478bd9Sstevel@tonic-gate segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 23627c478bd9Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos) 23637c478bd9Sstevel@tonic-gate { 23647c478bd9Sstevel@tonic-gate struct shm_data *shmd = seg->s_data; 23657c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 23667c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 23677c478bd9Sstevel@tonic-gate pgcnt_t npages, a_npages; 23687c478bd9Sstevel@tonic-gate page_t **ppa; 23697c478bd9Sstevel@tonic-gate pgcnt_t an_idx, a_an_idx, ppa_idx; 23707c478bd9Sstevel@tonic-gate caddr_t spt_addr, a_addr; /* spt and aligned address */ 23717c478bd9Sstevel@tonic-gate size_t a_len; /* aligned len */ 23727c478bd9Sstevel@tonic-gate size_t share_sz; 23737c478bd9Sstevel@tonic-gate ulong_t i; 23747c478bd9Sstevel@tonic-gate int sts = 0; 23757c478bd9Sstevel@tonic-gate 23767c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23777c478bd9Sstevel@tonic-gate 23787c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 23797c478bd9Sstevel@tonic-gate return (0); 23807c478bd9Sstevel@tonic-gate } 23817c478bd9Sstevel@tonic-gate 23827c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 23837c478bd9Sstevel@tonic-gate an_idx = seg_page(seg, addr); 23847c478bd9Sstevel@tonic-gate npages = btopr(len); 23857c478bd9Sstevel@tonic-gate 23867c478bd9Sstevel@tonic-gate if (an_idx + npages > btopr(shmd->shm_amp->size)) { 23877c478bd9Sstevel@tonic-gate return (ENOMEM); 23887c478bd9Sstevel@tonic-gate } 23897c478bd9Sstevel@tonic-gate 23907c478bd9Sstevel@tonic-gate if (op == MC_LOCK) { 23917c478bd9Sstevel@tonic-gate /* 23927c478bd9Sstevel@tonic-gate * Need to align addr and size request if they are not 23937c478bd9Sstevel@tonic-gate * aligned so we can always allocate large page(s) however 23947c478bd9Sstevel@tonic-gate * we only lock what was requested in initial request. 23957c478bd9Sstevel@tonic-gate */ 23967c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(sptseg->s_szc); 23977c478bd9Sstevel@tonic-gate a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 23987c478bd9Sstevel@tonic-gate a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)), 23997c478bd9Sstevel@tonic-gate share_sz); 24007c478bd9Sstevel@tonic-gate a_npages = btop(a_len); 24017c478bd9Sstevel@tonic-gate a_an_idx = seg_page(seg, a_addr); 24027c478bd9Sstevel@tonic-gate spt_addr = sptseg->s_base + ptob(a_an_idx); 24037c478bd9Sstevel@tonic-gate ppa_idx = an_idx - a_an_idx; 24047c478bd9Sstevel@tonic-gate 24057c478bd9Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages), 24067c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) { 24077c478bd9Sstevel@tonic-gate return (ENOMEM); 24087c478bd9Sstevel@tonic-gate } 24097c478bd9Sstevel@tonic-gate 24107c478bd9Sstevel@tonic-gate /* 24117c478bd9Sstevel@tonic-gate * Don't cache any new pages for IO and 24127c478bd9Sstevel@tonic-gate * flush any cached pages. 24137c478bd9Sstevel@tonic-gate */ 24147c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 24157c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 24167c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 24177c478bd9Sstevel@tonic-gate 24187c478bd9Sstevel@tonic-gate sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa); 24197c478bd9Sstevel@tonic-gate if (sts != 0) { 24207c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 24217c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 24227c478bd9Sstevel@tonic-gate return (sts); 24237c478bd9Sstevel@tonic-gate } 24247c478bd9Sstevel@tonic-gate 24257c478bd9Sstevel@tonic-gate sts = spt_lockpages(seg, an_idx, npages, 24267c478bd9Sstevel@tonic-gate &ppa[ppa_idx], lockmap, pos); 24277c478bd9Sstevel@tonic-gate /* 24287c478bd9Sstevel@tonic-gate * unlock remaining pages for requests which are not 24297c478bd9Sstevel@tonic-gate * aligned or not in 4 M chunks 24307c478bd9Sstevel@tonic-gate */ 24317c478bd9Sstevel@tonic-gate for (i = 0; i < ppa_idx; i++) 24327c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 24337c478bd9Sstevel@tonic-gate for (i = ppa_idx + npages; i < a_npages; i++) 24347c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 24357c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 24367c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 24377c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 24407c478bd9Sstevel@tonic-gate 24417c478bd9Sstevel@tonic-gate } else if (op == MC_UNLOCK) { /* unlock */ 24427c478bd9Sstevel@tonic-gate struct anon_map *amp; 24437c478bd9Sstevel@tonic-gate struct anon *ap; 24447c478bd9Sstevel@tonic-gate struct vnode *vp; 24457c478bd9Sstevel@tonic-gate u_offset_t off; 24467c478bd9Sstevel@tonic-gate struct page *pp; 24477c478bd9Sstevel@tonic-gate int kernel; 24487c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 24497c478bd9Sstevel@tonic-gate 24507c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 24517c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 24527c478bd9Sstevel@tonic-gate if (shmd->shm_lckpgs == 0) { 24537c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 24547c478bd9Sstevel@tonic-gate return (0); 24557c478bd9Sstevel@tonic-gate } 24567c478bd9Sstevel@tonic-gate /* 24577c478bd9Sstevel@tonic-gate * Don't cache new IO pages. 24587c478bd9Sstevel@tonic-gate */ 24597c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 24607c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 24617c478bd9Sstevel@tonic-gate 24627c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 24637c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++, an_idx++) { 24647c478bd9Sstevel@tonic-gate if (shmd->shm_vpage[an_idx] & DISM_PG_LOCKED) { 24657c478bd9Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 24667c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 24677c478bd9Sstevel@tonic-gate ASSERT(ap); 24687c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa_lckcnt[an_idx] > 0); 24697c478bd9Sstevel@tonic-gate 24707c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 24717c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 24727c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 24737c478bd9Sstevel@tonic-gate ASSERT(pp); 24747c478bd9Sstevel@tonic-gate /* 24757c478bd9Sstevel@tonic-gate * the availrmem is decremented only for 24767c478bd9Sstevel@tonic-gate * pages which are not in seg pcache, 24777c478bd9Sstevel@tonic-gate * for pages in seg pcache availrmem was 24787c478bd9Sstevel@tonic-gate * decremented in _dismpagelock() (if 24797c478bd9Sstevel@tonic-gate * they were not locked here) 24807c478bd9Sstevel@tonic-gate */ 24817c478bd9Sstevel@tonic-gate kernel = (sptd->spt_ppa && 24827c478bd9Sstevel@tonic-gate sptd->spt_ppa[an_idx]) ? 1 : 0; 24837c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, kernel); 24847c478bd9Sstevel@tonic-gate page_unlock(pp); 24857c478bd9Sstevel@tonic-gate shmd->shm_vpage[an_idx] &= ~DISM_PG_LOCKED; 24867c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt[an_idx]--; 24877c478bd9Sstevel@tonic-gate shmd->shm_lckpgs--; 24887c478bd9Sstevel@tonic-gate } 24897c478bd9Sstevel@tonic-gate } 24907c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 24917c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 24927c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 24937c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 24947c478bd9Sstevel@tonic-gate } 24957c478bd9Sstevel@tonic-gate return (sts); 24967c478bd9Sstevel@tonic-gate } 24977c478bd9Sstevel@tonic-gate 24987c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 24997c478bd9Sstevel@tonic-gate int 25007c478bd9Sstevel@tonic-gate segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 25017c478bd9Sstevel@tonic-gate { 25027c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 25037c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 25047c478bd9Sstevel@tonic-gate spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1; 25057c478bd9Sstevel@tonic-gate 25067c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25077c478bd9Sstevel@tonic-gate 25087c478bd9Sstevel@tonic-gate /* 25097c478bd9Sstevel@tonic-gate * ISM segment is always rw. 25107c478bd9Sstevel@tonic-gate */ 25117c478bd9Sstevel@tonic-gate while (--pgno >= 0) 25127c478bd9Sstevel@tonic-gate *protv++ = sptd->spt_prot; 25137c478bd9Sstevel@tonic-gate return (0); 25147c478bd9Sstevel@tonic-gate } 25157c478bd9Sstevel@tonic-gate 25167c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 25177c478bd9Sstevel@tonic-gate u_offset_t 25187c478bd9Sstevel@tonic-gate segspt_shmgetoffset(struct seg *seg, caddr_t addr) 25197c478bd9Sstevel@tonic-gate { 25207c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate /* Offset does not matter in ISM memory */ 25237c478bd9Sstevel@tonic-gate 25247c478bd9Sstevel@tonic-gate return ((u_offset_t)0); 25257c478bd9Sstevel@tonic-gate } 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate /* ARGSUSED */ 25287c478bd9Sstevel@tonic-gate int 25297c478bd9Sstevel@tonic-gate segspt_shmgettype(struct seg *seg, caddr_t addr) 25307c478bd9Sstevel@tonic-gate { 25317c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 25327c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 25337c478bd9Sstevel@tonic-gate 25347c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25357c478bd9Sstevel@tonic-gate 25367c478bd9Sstevel@tonic-gate /* 25377c478bd9Sstevel@tonic-gate * The shared memory mapping is always MAP_SHARED, SWAP is only 25387c478bd9Sstevel@tonic-gate * reserved for DISM 25397c478bd9Sstevel@tonic-gate */ 25407c478bd9Sstevel@tonic-gate return (MAP_SHARED | 25417c478bd9Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE)); 25427c478bd9Sstevel@tonic-gate } 25437c478bd9Sstevel@tonic-gate 25447c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 25457c478bd9Sstevel@tonic-gate int 25467c478bd9Sstevel@tonic-gate segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 25477c478bd9Sstevel@tonic-gate { 25487c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 25497c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 25507c478bd9Sstevel@tonic-gate 25517c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25527c478bd9Sstevel@tonic-gate 25537c478bd9Sstevel@tonic-gate *vpp = sptd->spt_vp; 25547c478bd9Sstevel@tonic-gate return (0); 25557c478bd9Sstevel@tonic-gate } 25567c478bd9Sstevel@tonic-gate 25577c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 25587c478bd9Sstevel@tonic-gate static int 25597c478bd9Sstevel@tonic-gate segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 25607c478bd9Sstevel@tonic-gate { 25617c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 25627c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 25637c478bd9Sstevel@tonic-gate struct anon_map *amp; 25647c478bd9Sstevel@tonic-gate pgcnt_t pg_idx; 25657c478bd9Sstevel@tonic-gate 25667c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25677c478bd9Sstevel@tonic-gate 25687c478bd9Sstevel@tonic-gate if (behav == MADV_FREE) { 25697c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) 25707c478bd9Sstevel@tonic-gate return (0); 25717c478bd9Sstevel@tonic-gate 25727c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 25737c478bd9Sstevel@tonic-gate pg_idx = seg_page(seg, addr); 25747c478bd9Sstevel@tonic-gate 25757c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 25767c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 25777c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 25787c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 25797c478bd9Sstevel@tonic-gate 25807c478bd9Sstevel@tonic-gate /* 25817c478bd9Sstevel@tonic-gate * Purge all DISM cached pages 25827c478bd9Sstevel@tonic-gate */ 25837c478bd9Sstevel@tonic-gate seg_ppurge_seg(segspt_reclaim); 25847c478bd9Sstevel@tonic-gate 25857c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 25867c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 25877c478bd9Sstevel@tonic-gate anon_disclaim(amp, pg_idx, len, ANON_PGLOOKUP_BLK); 25887c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 25897c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 25907c478bd9Sstevel@tonic-gate } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP || 25917c478bd9Sstevel@tonic-gate behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) { 25927c478bd9Sstevel@tonic-gate int already_set; 25937c478bd9Sstevel@tonic-gate ulong_t anon_index; 25947c478bd9Sstevel@tonic-gate lgrp_mem_policy_t policy; 25957c478bd9Sstevel@tonic-gate caddr_t shm_addr; 25967c478bd9Sstevel@tonic-gate size_t share_size; 25977c478bd9Sstevel@tonic-gate size_t size; 25987c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 25997c478bd9Sstevel@tonic-gate caddr_t sptseg_addr; 26007c478bd9Sstevel@tonic-gate 26017c478bd9Sstevel@tonic-gate /* 26027c478bd9Sstevel@tonic-gate * Align address and length to page size of underlying segment 26037c478bd9Sstevel@tonic-gate */ 26047c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(shmd->shm_sptseg->s_szc); 26057c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 26067c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), 26077c478bd9Sstevel@tonic-gate share_size); 26087c478bd9Sstevel@tonic-gate 26097c478bd9Sstevel@tonic-gate amp = shmd->shm_amp; 26107c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr); 26117c478bd9Sstevel@tonic-gate 26127c478bd9Sstevel@tonic-gate /* 26137c478bd9Sstevel@tonic-gate * And now we may have to adjust size downward if we have 26147c478bd9Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon 26157c478bd9Sstevel@tonic-gate * allocations. 26167c478bd9Sstevel@tonic-gate */ 26177c478bd9Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index); 26187c478bd9Sstevel@tonic-gate if ((sptseg_addr + size) > 26197c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)) 26207c478bd9Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - 26217c478bd9Sstevel@tonic-gate sptseg_addr; 26227c478bd9Sstevel@tonic-gate 26237c478bd9Sstevel@tonic-gate /* 26247c478bd9Sstevel@tonic-gate * Set memory allocation policy for this segment 26257c478bd9Sstevel@tonic-gate */ 26267c478bd9Sstevel@tonic-gate policy = lgrp_madv_to_policy(behav, len, MAP_SHARED); 26277c478bd9Sstevel@tonic-gate already_set = lgrp_shm_policy_set(policy, amp, anon_index, 26287c478bd9Sstevel@tonic-gate NULL, 0, len); 26297c478bd9Sstevel@tonic-gate 26307c478bd9Sstevel@tonic-gate /* 26317c478bd9Sstevel@tonic-gate * If random memory allocation policy set already, 26327c478bd9Sstevel@tonic-gate * don't bother reapplying it. 26337c478bd9Sstevel@tonic-gate */ 26347c478bd9Sstevel@tonic-gate if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy)) 26357c478bd9Sstevel@tonic-gate return (0); 26367c478bd9Sstevel@tonic-gate 26377c478bd9Sstevel@tonic-gate /* 26387c478bd9Sstevel@tonic-gate * Mark any existing pages in the given range for 26397c478bd9Sstevel@tonic-gate * migration, flushing the I/O page cache, and using 26407c478bd9Sstevel@tonic-gate * underlying segment to calculate anon index and get 26417c478bd9Sstevel@tonic-gate * anonmap and vnode pointer from 26427c478bd9Sstevel@tonic-gate */ 26437c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) 26447c478bd9Sstevel@tonic-gate segspt_purge(seg); 26457c478bd9Sstevel@tonic-gate 26467c478bd9Sstevel@tonic-gate page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0); 26477c478bd9Sstevel@tonic-gate } 26487c478bd9Sstevel@tonic-gate 26497c478bd9Sstevel@tonic-gate return (0); 26507c478bd9Sstevel@tonic-gate } 26517c478bd9Sstevel@tonic-gate 26527c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 26537c478bd9Sstevel@tonic-gate void 26547c478bd9Sstevel@tonic-gate segspt_shmdump(struct seg *seg) 26557c478bd9Sstevel@tonic-gate { 26567c478bd9Sstevel@tonic-gate /* no-op for ISM segment */ 26577c478bd9Sstevel@tonic-gate } 26587c478bd9Sstevel@tonic-gate 26597c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 26607c478bd9Sstevel@tonic-gate static faultcode_t 26617c478bd9Sstevel@tonic-gate segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc) 26627c478bd9Sstevel@tonic-gate { 26637c478bd9Sstevel@tonic-gate return (ENOTSUP); 26647c478bd9Sstevel@tonic-gate } 26657c478bd9Sstevel@tonic-gate 26667c478bd9Sstevel@tonic-gate /* 26677c478bd9Sstevel@tonic-gate * get a memory ID for an addr in a given segment 26687c478bd9Sstevel@tonic-gate */ 26697c478bd9Sstevel@tonic-gate static int 26707c478bd9Sstevel@tonic-gate segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 26717c478bd9Sstevel@tonic-gate { 26727c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 26737c478bd9Sstevel@tonic-gate struct anon *ap; 26747c478bd9Sstevel@tonic-gate size_t anon_index; 26757c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 26767c478bd9Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data; 26777c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 26787c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 26797c478bd9Sstevel@tonic-gate 26807c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 26817c478bd9Sstevel@tonic-gate 26827c478bd9Sstevel@tonic-gate if (addr > (seg->s_base + sptd->spt_realsize)) { 26837c478bd9Sstevel@tonic-gate return (EFAULT); 26847c478bd9Sstevel@tonic-gate } 26857c478bd9Sstevel@tonic-gate 26867c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 26877c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 26887c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 26897c478bd9Sstevel@tonic-gate if (ap == NULL) { 26907c478bd9Sstevel@tonic-gate struct page *pp; 26917c478bd9Sstevel@tonic-gate caddr_t spt_addr = sptseg->s_base + ptob(anon_index); 26927c478bd9Sstevel@tonic-gate 26937c478bd9Sstevel@tonic-gate pp = anon_zero(sptseg, spt_addr, &ap, kcred); 26947c478bd9Sstevel@tonic-gate if (pp == NULL) { 26957c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 26967c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 26977c478bd9Sstevel@tonic-gate return (ENOMEM); 26987c478bd9Sstevel@tonic-gate } 26997c478bd9Sstevel@tonic-gate (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP); 27007c478bd9Sstevel@tonic-gate page_unlock(pp); 27017c478bd9Sstevel@tonic-gate } 27027c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 27037c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 27047c478bd9Sstevel@tonic-gate memidp->val[0] = (uintptr_t)ap; 27057c478bd9Sstevel@tonic-gate memidp->val[1] = (uintptr_t)addr & PAGEOFFSET; 27067c478bd9Sstevel@tonic-gate return (0); 27077c478bd9Sstevel@tonic-gate } 27087c478bd9Sstevel@tonic-gate 27097c478bd9Sstevel@tonic-gate /* 27107c478bd9Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment 27117c478bd9Sstevel@tonic-gate */ 27127c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 27137c478bd9Sstevel@tonic-gate segspt_shmgetpolicy(struct seg *seg, caddr_t addr) 27147c478bd9Sstevel@tonic-gate { 27157c478bd9Sstevel@tonic-gate struct anon_map *amp; 27167c478bd9Sstevel@tonic-gate ulong_t anon_index; 27177c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 27187c478bd9Sstevel@tonic-gate struct shm_data *shm_data; 27197c478bd9Sstevel@tonic-gate 27207c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 27217c478bd9Sstevel@tonic-gate 27227c478bd9Sstevel@tonic-gate /* 27237c478bd9Sstevel@tonic-gate * Get anon_map from segshm 27247c478bd9Sstevel@tonic-gate * 27257c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since 27267c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be 27277c478bd9Sstevel@tonic-gate * nonzero for an existing segment 27287c478bd9Sstevel@tonic-gate * Need to grab readers lock on policy tree though 27297c478bd9Sstevel@tonic-gate */ 27307c478bd9Sstevel@tonic-gate shm_data = (struct shm_data *)seg->s_data; 27317c478bd9Sstevel@tonic-gate if (shm_data == NULL) 27327c478bd9Sstevel@tonic-gate return (NULL); 27337c478bd9Sstevel@tonic-gate amp = shm_data->shm_amp; 27347c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 27357c478bd9Sstevel@tonic-gate 27367c478bd9Sstevel@tonic-gate /* 27377c478bd9Sstevel@tonic-gate * Get policy info 27387c478bd9Sstevel@tonic-gate * 27397c478bd9Sstevel@tonic-gate * Assume starting anon index of 0 27407c478bd9Sstevel@tonic-gate */ 27417c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 27427c478bd9Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 27437c478bd9Sstevel@tonic-gate 27447c478bd9Sstevel@tonic-gate return (policy_info); 27457c478bd9Sstevel@tonic-gate } 27461bd5c35fSelowe 27471bd5c35fSelowe /*ARGSUSED*/ 27481bd5c35fSelowe static int 27491bd5c35fSelowe segspt_shmcapable(struct seg *seg, segcapability_t capability) 27501bd5c35fSelowe { 27511bd5c35fSelowe return (0); 27521bd5c35fSelowe } 2753