17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 237c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/param.h> 307c478bd9Sstevel@tonic-gate #include <sys/user.h> 317c478bd9Sstevel@tonic-gate #include <sys/mman.h> 327c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 337c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 347c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 357c478bd9Sstevel@tonic-gate #include <sys/systm.h> 367c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 377c478bd9Sstevel@tonic-gate #include <vm/hat.h> 387c478bd9Sstevel@tonic-gate #include <vm/seg.h> 397c478bd9Sstevel@tonic-gate #include <vm/as.h> 407c478bd9Sstevel@tonic-gate #include <vm/anon.h> 417c478bd9Sstevel@tonic-gate #include <vm/page.h> 427c478bd9Sstevel@tonic-gate #include <sys/buf.h> 437c478bd9Sstevel@tonic-gate #include <sys/swap.h> 447c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 457c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 467c478bd9Sstevel@tonic-gate #include <sys/debug.h> 477c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 487c478bd9Sstevel@tonic-gate #include <sys/shm.h> 497c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 507c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 517c478bd9Sstevel@tonic-gate 527c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h> 537c478bd9Sstevel@tonic-gate 547c478bd9Sstevel@tonic-gate #define SEGSPTADDR (caddr_t)0x0 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate /* 577c478bd9Sstevel@tonic-gate * # pages used for spt 587c478bd9Sstevel@tonic-gate */ 597c478bd9Sstevel@tonic-gate static size_t spt_used; 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate /* 627c478bd9Sstevel@tonic-gate * segspt_minfree is the memory left for system after ISM 637c478bd9Sstevel@tonic-gate * locked its pages; it is set up to 5% of availrmem in 647c478bd9Sstevel@tonic-gate * sptcreate when ISM is created. ISM should not use more 657c478bd9Sstevel@tonic-gate * than ~90% of availrmem; if it does, then the performance 667c478bd9Sstevel@tonic-gate * of the system may decrease. Machines with large memories may 677c478bd9Sstevel@tonic-gate * be able to use up more memory for ISM so we set the default 687c478bd9Sstevel@tonic-gate * segspt_minfree to 5% (which gives ISM max 95% of availrmem. 697c478bd9Sstevel@tonic-gate * If somebody wants even more memory for ISM (risking hanging 707c478bd9Sstevel@tonic-gate * the system) they can patch the segspt_minfree to smaller number. 717c478bd9Sstevel@tonic-gate */ 727c478bd9Sstevel@tonic-gate pgcnt_t segspt_minfree = 0; 737c478bd9Sstevel@tonic-gate 747c478bd9Sstevel@tonic-gate static int segspt_create(struct seg *seg, caddr_t argsp); 757c478bd9Sstevel@tonic-gate static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize); 767c478bd9Sstevel@tonic-gate static void segspt_free(struct seg *seg); 777c478bd9Sstevel@tonic-gate static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len); 787c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr); 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate static void 817c478bd9Sstevel@tonic-gate segspt_badop() 827c478bd9Sstevel@tonic-gate { 837c478bd9Sstevel@tonic-gate panic("segspt_badop called"); 847c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 857c478bd9Sstevel@tonic-gate } 867c478bd9Sstevel@tonic-gate 877c478bd9Sstevel@tonic-gate #define SEGSPT_BADOP(t) (t(*)())segspt_badop 887c478bd9Sstevel@tonic-gate 897c478bd9Sstevel@tonic-gate struct seg_ops segspt_ops = { 907c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* dup */ 917c478bd9Sstevel@tonic-gate segspt_unmap, 927c478bd9Sstevel@tonic-gate segspt_free, 937c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* fault */ 947c478bd9Sstevel@tonic-gate SEGSPT_BADOP(faultcode_t), /* faulta */ 957c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* setprot */ 967c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* checkprot */ 977c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* kluster */ 987c478bd9Sstevel@tonic-gate SEGSPT_BADOP(size_t), /* swapout */ 997c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* sync */ 1007c478bd9Sstevel@tonic-gate SEGSPT_BADOP(size_t), /* incore */ 1017c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* lockop */ 1027c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* getprot */ 1037c478bd9Sstevel@tonic-gate SEGSPT_BADOP(u_offset_t), /* getoffset */ 1047c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* gettype */ 1057c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* getvp */ 1067c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* advise */ 1077c478bd9Sstevel@tonic-gate SEGSPT_BADOP(void), /* dump */ 1087c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* pagelock */ 1097c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* setpgsz */ 1107c478bd9Sstevel@tonic-gate SEGSPT_BADOP(int), /* getmemid */ 1117c478bd9Sstevel@tonic-gate segspt_getpolicy, /* getpolicy */ 112*1bd5c35fSelowe SEGSPT_BADOP(int), /* capable */ 1137c478bd9Sstevel@tonic-gate }; 1147c478bd9Sstevel@tonic-gate 1157c478bd9Sstevel@tonic-gate static int segspt_shmdup(struct seg *seg, struct seg *newseg); 1167c478bd9Sstevel@tonic-gate static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize); 1177c478bd9Sstevel@tonic-gate static void segspt_shmfree(struct seg *seg); 1187c478bd9Sstevel@tonic-gate static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg, 1197c478bd9Sstevel@tonic-gate caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); 1207c478bd9Sstevel@tonic-gate static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr); 1217c478bd9Sstevel@tonic-gate static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr, 1227c478bd9Sstevel@tonic-gate register size_t len, register uint_t prot); 1237c478bd9Sstevel@tonic-gate static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, 1247c478bd9Sstevel@tonic-gate uint_t prot); 1257c478bd9Sstevel@tonic-gate static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta); 1267c478bd9Sstevel@tonic-gate static size_t segspt_shmswapout(struct seg *seg); 1277c478bd9Sstevel@tonic-gate static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, 1287c478bd9Sstevel@tonic-gate register char *vec); 1297c478bd9Sstevel@tonic-gate static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len, 1307c478bd9Sstevel@tonic-gate int attr, uint_t flags); 1317c478bd9Sstevel@tonic-gate static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 1327c478bd9Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos); 1337c478bd9Sstevel@tonic-gate static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, 1347c478bd9Sstevel@tonic-gate uint_t *protv); 1357c478bd9Sstevel@tonic-gate static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr); 1367c478bd9Sstevel@tonic-gate static int segspt_shmgettype(struct seg *seg, caddr_t addr); 1377c478bd9Sstevel@tonic-gate static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 1387c478bd9Sstevel@tonic-gate static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, 1397c478bd9Sstevel@tonic-gate uint_t behav); 1407c478bd9Sstevel@tonic-gate static void segspt_shmdump(struct seg *seg); 1417c478bd9Sstevel@tonic-gate static int segspt_shmpagelock(struct seg *, caddr_t, size_t, 1427c478bd9Sstevel@tonic-gate struct page ***, enum lock_type, enum seg_rw); 1437c478bd9Sstevel@tonic-gate static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t); 1447c478bd9Sstevel@tonic-gate static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *); 1457c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t); 146*1bd5c35fSelowe static int segspt_shmcapable(struct seg *, segcapability_t); 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate struct seg_ops segspt_shmops = { 1497c478bd9Sstevel@tonic-gate segspt_shmdup, 1507c478bd9Sstevel@tonic-gate segspt_shmunmap, 1517c478bd9Sstevel@tonic-gate segspt_shmfree, 1527c478bd9Sstevel@tonic-gate segspt_shmfault, 1537c478bd9Sstevel@tonic-gate segspt_shmfaulta, 1547c478bd9Sstevel@tonic-gate segspt_shmsetprot, 1557c478bd9Sstevel@tonic-gate segspt_shmcheckprot, 1567c478bd9Sstevel@tonic-gate segspt_shmkluster, 1577c478bd9Sstevel@tonic-gate segspt_shmswapout, 1587c478bd9Sstevel@tonic-gate segspt_shmsync, 1597c478bd9Sstevel@tonic-gate segspt_shmincore, 1607c478bd9Sstevel@tonic-gate segspt_shmlockop, 1617c478bd9Sstevel@tonic-gate segspt_shmgetprot, 1627c478bd9Sstevel@tonic-gate segspt_shmgetoffset, 1637c478bd9Sstevel@tonic-gate segspt_shmgettype, 1647c478bd9Sstevel@tonic-gate segspt_shmgetvp, 1657c478bd9Sstevel@tonic-gate segspt_shmadvise, /* advise */ 1667c478bd9Sstevel@tonic-gate segspt_shmdump, 1677c478bd9Sstevel@tonic-gate segspt_shmpagelock, 1687c478bd9Sstevel@tonic-gate segspt_shmsetpgsz, 1697c478bd9Sstevel@tonic-gate segspt_shmgetmemid, 1707c478bd9Sstevel@tonic-gate segspt_shmgetpolicy, 171*1bd5c35fSelowe segspt_shmcapable, 1727c478bd9Sstevel@tonic-gate }; 1737c478bd9Sstevel@tonic-gate 1747c478bd9Sstevel@tonic-gate static void segspt_purge(struct seg *seg); 1757c478bd9Sstevel@tonic-gate static int segspt_reclaim(struct seg *, caddr_t, size_t, struct page **, 1767c478bd9Sstevel@tonic-gate enum seg_rw); 1777c478bd9Sstevel@tonic-gate static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len, 1787c478bd9Sstevel@tonic-gate page_t **ppa); 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate 1817c478bd9Sstevel@tonic-gate 1827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1837c478bd9Sstevel@tonic-gate int 1847c478bd9Sstevel@tonic-gate sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp, 1857c478bd9Sstevel@tonic-gate uint_t prot, uint_t flags, uint_t share_szc) 1867c478bd9Sstevel@tonic-gate { 1877c478bd9Sstevel@tonic-gate int err; 1887c478bd9Sstevel@tonic-gate struct as *newas; 1897c478bd9Sstevel@tonic-gate struct segspt_crargs sptcargs; 1907c478bd9Sstevel@tonic-gate 1917c478bd9Sstevel@tonic-gate #ifdef DEBUG 1927c478bd9Sstevel@tonic-gate TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */, 1937c478bd9Sstevel@tonic-gate tnf_ulong, size, size ); 1947c478bd9Sstevel@tonic-gate #endif 1957c478bd9Sstevel@tonic-gate if (segspt_minfree == 0) /* leave min 5% of availrmem for */ 1967c478bd9Sstevel@tonic-gate segspt_minfree = availrmem/20; /* for the system */ 1977c478bd9Sstevel@tonic-gate 1987c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_SHARED_PT, (void *)0)) 1997c478bd9Sstevel@tonic-gate return (EINVAL); 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate /* 2027c478bd9Sstevel@tonic-gate * get a new as for this shared memory segment 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate newas = as_alloc(); 2057c478bd9Sstevel@tonic-gate sptcargs.amp = amp; 2067c478bd9Sstevel@tonic-gate sptcargs.prot = prot; 2077c478bd9Sstevel@tonic-gate sptcargs.flags = flags; 2087c478bd9Sstevel@tonic-gate sptcargs.szc = share_szc; 2097c478bd9Sstevel@tonic-gate 2107c478bd9Sstevel@tonic-gate /* 2117c478bd9Sstevel@tonic-gate * create a shared page table (spt) segment 2127c478bd9Sstevel@tonic-gate */ 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) { 2157c478bd9Sstevel@tonic-gate as_free(newas); 2167c478bd9Sstevel@tonic-gate return (err); 2177c478bd9Sstevel@tonic-gate } 2187c478bd9Sstevel@tonic-gate *sptseg = sptcargs.seg_spt; 2197c478bd9Sstevel@tonic-gate return (0); 2207c478bd9Sstevel@tonic-gate } 2217c478bd9Sstevel@tonic-gate 2227c478bd9Sstevel@tonic-gate void 2237c478bd9Sstevel@tonic-gate sptdestroy(struct as *as, struct anon_map *amp) 2247c478bd9Sstevel@tonic-gate { 2257c478bd9Sstevel@tonic-gate 2267c478bd9Sstevel@tonic-gate #ifdef DEBUG 2277c478bd9Sstevel@tonic-gate TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */); 2287c478bd9Sstevel@tonic-gate #endif 2297c478bd9Sstevel@tonic-gate (void) as_unmap(as, SEGSPTADDR, amp->size); 2307c478bd9Sstevel@tonic-gate as_free(as); 2317c478bd9Sstevel@tonic-gate } 2327c478bd9Sstevel@tonic-gate 2337c478bd9Sstevel@tonic-gate /* 2347c478bd9Sstevel@tonic-gate * called from seg_free(). 2357c478bd9Sstevel@tonic-gate * free (i.e., unlock, unmap, return to free list) 2367c478bd9Sstevel@tonic-gate * all the pages in the given seg. 2377c478bd9Sstevel@tonic-gate */ 2387c478bd9Sstevel@tonic-gate void 2397c478bd9Sstevel@tonic-gate segspt_free(struct seg *seg) 2407c478bd9Sstevel@tonic-gate { 2417c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data; 2427c478bd9Sstevel@tonic-gate 2437c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 2447c478bd9Sstevel@tonic-gate 2457c478bd9Sstevel@tonic-gate if (sptd != NULL) { 2467c478bd9Sstevel@tonic-gate if (sptd->spt_realsize) 2477c478bd9Sstevel@tonic-gate segspt_free_pages(seg, seg->s_base, sptd->spt_realsize); 2487c478bd9Sstevel@tonic-gate 2497c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt) 2507c478bd9Sstevel@tonic-gate kmem_free(sptd->spt_ppa_lckcnt, 2517c478bd9Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt) 2527c478bd9Sstevel@tonic-gate * btopr(sptd->spt_amp->size)); 2537c478bd9Sstevel@tonic-gate kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp)); 2547c478bd9Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock); 2557c478bd9Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd)); 2567c478bd9Sstevel@tonic-gate } 2577c478bd9Sstevel@tonic-gate } 2587c478bd9Sstevel@tonic-gate 2597c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2607c478bd9Sstevel@tonic-gate static int 2617c478bd9Sstevel@tonic-gate segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr, 2627c478bd9Sstevel@tonic-gate uint_t flags) 2637c478bd9Sstevel@tonic-gate { 2647c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2657c478bd9Sstevel@tonic-gate 2667c478bd9Sstevel@tonic-gate return (0); 2677c478bd9Sstevel@tonic-gate } 2687c478bd9Sstevel@tonic-gate 2697c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2707c478bd9Sstevel@tonic-gate static size_t 2717c478bd9Sstevel@tonic-gate segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec) 2727c478bd9Sstevel@tonic-gate { 2737c478bd9Sstevel@tonic-gate caddr_t eo_seg; 2747c478bd9Sstevel@tonic-gate pgcnt_t npages; 2757c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 2767c478bd9Sstevel@tonic-gate struct seg *sptseg; 2777c478bd9Sstevel@tonic-gate struct spt_data *sptd; 2787c478bd9Sstevel@tonic-gate 2797c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2807c478bd9Sstevel@tonic-gate #ifdef lint 2817c478bd9Sstevel@tonic-gate seg = seg; 2827c478bd9Sstevel@tonic-gate #endif 2837c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 2847c478bd9Sstevel@tonic-gate sptd = sptseg->s_data; 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 2877c478bd9Sstevel@tonic-gate eo_seg = addr + len; 2887c478bd9Sstevel@tonic-gate while (addr < eo_seg) { 2897c478bd9Sstevel@tonic-gate /* page exists, and it's locked. */ 2907c478bd9Sstevel@tonic-gate *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED | 2917c478bd9Sstevel@tonic-gate SEG_PAGE_ANON; 2927c478bd9Sstevel@tonic-gate addr += PAGESIZE; 2937c478bd9Sstevel@tonic-gate } 2947c478bd9Sstevel@tonic-gate return (len); 2957c478bd9Sstevel@tonic-gate } else { 2967c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 2977c478bd9Sstevel@tonic-gate struct anon *ap; 2987c478bd9Sstevel@tonic-gate page_t *pp; 2997c478bd9Sstevel@tonic-gate pgcnt_t anon_index; 3007c478bd9Sstevel@tonic-gate struct vnode *vp; 3017c478bd9Sstevel@tonic-gate u_offset_t off; 3027c478bd9Sstevel@tonic-gate ulong_t i; 3037c478bd9Sstevel@tonic-gate int ret; 3047c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 3057c478bd9Sstevel@tonic-gate 3067c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3077c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 3087c478bd9Sstevel@tonic-gate npages = btopr(len); 3097c478bd9Sstevel@tonic-gate if (anon_index + npages > btopr(shmd->shm_amp->size)) { 3107c478bd9Sstevel@tonic-gate return (EINVAL); 3117c478bd9Sstevel@tonic-gate } 3127c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 3137c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++, anon_index++) { 3147c478bd9Sstevel@tonic-gate ret = 0; 3157c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 3167c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 3177c478bd9Sstevel@tonic-gate if (ap != NULL) { 3187c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 3197c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 3207c478bd9Sstevel@tonic-gate pp = page_lookup_nowait(vp, off, SE_SHARED); 3217c478bd9Sstevel@tonic-gate if (pp != NULL) { 3227c478bd9Sstevel@tonic-gate ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON; 3237c478bd9Sstevel@tonic-gate page_unlock(pp); 3247c478bd9Sstevel@tonic-gate } 3257c478bd9Sstevel@tonic-gate } else { 3267c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 3277c478bd9Sstevel@tonic-gate } 3287c478bd9Sstevel@tonic-gate if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 3297c478bd9Sstevel@tonic-gate ret |= SEG_PAGE_LOCKED; 3307c478bd9Sstevel@tonic-gate } 3317c478bd9Sstevel@tonic-gate *vec++ = (char)ret; 3327c478bd9Sstevel@tonic-gate } 3337c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 3347c478bd9Sstevel@tonic-gate return (len); 3357c478bd9Sstevel@tonic-gate } 3367c478bd9Sstevel@tonic-gate } 3377c478bd9Sstevel@tonic-gate 3387c478bd9Sstevel@tonic-gate static int 3397c478bd9Sstevel@tonic-gate segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize) 3407c478bd9Sstevel@tonic-gate { 3417c478bd9Sstevel@tonic-gate size_t share_size; 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3447c478bd9Sstevel@tonic-gate 3457c478bd9Sstevel@tonic-gate /* 3467c478bd9Sstevel@tonic-gate * seg.s_size may have been rounded up to the largest page size 3477c478bd9Sstevel@tonic-gate * in shmat(). 3487c478bd9Sstevel@tonic-gate * XXX This should be cleanedup. sptdestroy should take a length 3497c478bd9Sstevel@tonic-gate * argument which should be the same as sptcreate. Then 3507c478bd9Sstevel@tonic-gate * this rounding would not be needed (or is done in shm.c) 3517c478bd9Sstevel@tonic-gate * Only the check for full segment will be needed. 3527c478bd9Sstevel@tonic-gate * 3537c478bd9Sstevel@tonic-gate * XXX -- shouldn't raddr == 0 always? These tests don't seem 3547c478bd9Sstevel@tonic-gate * to be useful at all. 3557c478bd9Sstevel@tonic-gate */ 3567c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(seg->s_szc); 3577c478bd9Sstevel@tonic-gate ssize = P2ROUNDUP(ssize, share_size); 3587c478bd9Sstevel@tonic-gate 3597c478bd9Sstevel@tonic-gate if (raddr == seg->s_base && ssize == seg->s_size) { 3607c478bd9Sstevel@tonic-gate seg_free(seg); 3617c478bd9Sstevel@tonic-gate return (0); 3627c478bd9Sstevel@tonic-gate } else 3637c478bd9Sstevel@tonic-gate return (EINVAL); 3647c478bd9Sstevel@tonic-gate } 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate int 3677c478bd9Sstevel@tonic-gate segspt_create(struct seg *seg, caddr_t argsp) 3687c478bd9Sstevel@tonic-gate { 3697c478bd9Sstevel@tonic-gate int err; 3707c478bd9Sstevel@tonic-gate caddr_t addr = seg->s_base; 3717c478bd9Sstevel@tonic-gate struct spt_data *sptd; 3727c478bd9Sstevel@tonic-gate struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp; 3737c478bd9Sstevel@tonic-gate struct anon_map *amp = sptcargs->amp; 3747c478bd9Sstevel@tonic-gate struct cred *cred = CRED(); 3757c478bd9Sstevel@tonic-gate ulong_t i, j, anon_index = 0; 3767c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(amp->size); 3777c478bd9Sstevel@tonic-gate struct vnode *vp; 3787c478bd9Sstevel@tonic-gate page_t **ppa; 3797c478bd9Sstevel@tonic-gate uint_t hat_flags; 3807c478bd9Sstevel@tonic-gate 3817c478bd9Sstevel@tonic-gate /* 3827c478bd9Sstevel@tonic-gate * We are holding the a_lock on the underlying dummy as, 3837c478bd9Sstevel@tonic-gate * so we can make calls to the HAT layer. 3847c478bd9Sstevel@tonic-gate */ 3857c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3867c478bd9Sstevel@tonic-gate 3877c478bd9Sstevel@tonic-gate #ifdef DEBUG 3887c478bd9Sstevel@tonic-gate TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */, 3897c478bd9Sstevel@tonic-gate tnf_opaque, addr, addr, 3907c478bd9Sstevel@tonic-gate tnf_ulong, len, seg->s_size); 3917c478bd9Sstevel@tonic-gate #endif 3927c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 3937c478bd9Sstevel@tonic-gate if (err = anon_swap_adjust(npages)) 3947c478bd9Sstevel@tonic-gate return (err); 3957c478bd9Sstevel@tonic-gate } 3967c478bd9Sstevel@tonic-gate err = ENOMEM; 3977c478bd9Sstevel@tonic-gate 3987c478bd9Sstevel@tonic-gate if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL) 3997c478bd9Sstevel@tonic-gate goto out1; 4007c478bd9Sstevel@tonic-gate 4017c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 4027c478bd9Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages), 4037c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) 4047c478bd9Sstevel@tonic-gate goto out2; 4057c478bd9Sstevel@tonic-gate } 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL); 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL) 4107c478bd9Sstevel@tonic-gate goto out3; 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate seg->s_ops = &segspt_ops; 4137c478bd9Sstevel@tonic-gate sptd->spt_vp = vp; 4147c478bd9Sstevel@tonic-gate sptd->spt_amp = amp; 4157c478bd9Sstevel@tonic-gate sptd->spt_prot = sptcargs->prot; 4167c478bd9Sstevel@tonic-gate sptd->spt_flags = sptcargs->flags; 4177c478bd9Sstevel@tonic-gate seg->s_data = (caddr_t)sptd; 4187c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 4197c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt = NULL; 4207c478bd9Sstevel@tonic-gate seg->s_szc = sptcargs->szc; 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 4237c478bd9Sstevel@tonic-gate amp->a_szc = seg->s_szc; 4247c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 4257c478bd9Sstevel@tonic-gate 4267c478bd9Sstevel@tonic-gate /* 4277c478bd9Sstevel@tonic-gate * Set policy to affect initial allocation of pages in 4287c478bd9Sstevel@tonic-gate * anon_map_createpages() 4297c478bd9Sstevel@tonic-gate */ 4307c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index, 4317c478bd9Sstevel@tonic-gate NULL, 0, ptob(npages)); 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate if (sptcargs->flags & SHM_PAGEABLE) { 4347c478bd9Sstevel@tonic-gate size_t share_sz; 4357c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, more_pgs; 4367c478bd9Sstevel@tonic-gate struct anon_hdr *nahp; 4377c478bd9Sstevel@tonic-gate 4387c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(seg->s_szc); 4397c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(amp->size, share_sz)) { 4407c478bd9Sstevel@tonic-gate /* 4417c478bd9Sstevel@tonic-gate * We are rounding up the size of the anon array 4427c478bd9Sstevel@tonic-gate * on 4 M boundary because we always create 4 M 4437c478bd9Sstevel@tonic-gate * of page(s) when locking, faulting pages and we 4447c478bd9Sstevel@tonic-gate * don't have to check for all corner cases e.g. 4457c478bd9Sstevel@tonic-gate * if there is enough space to allocate 4 M 4467c478bd9Sstevel@tonic-gate * page. 4477c478bd9Sstevel@tonic-gate */ 4487c478bd9Sstevel@tonic-gate new_npgs = btop(P2ROUNDUP(amp->size, share_sz)); 4497c478bd9Sstevel@tonic-gate more_pgs = new_npgs - npages; 4507c478bd9Sstevel@tonic-gate 4517c478bd9Sstevel@tonic-gate if (anon_resv(ptob(more_pgs)) == 0) { 4527c478bd9Sstevel@tonic-gate err = ENOMEM; 4537c478bd9Sstevel@tonic-gate goto out4; 4547c478bd9Sstevel@tonic-gate } 4557c478bd9Sstevel@tonic-gate nahp = anon_create(new_npgs, ANON_SLEEP); 4567c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 4577c478bd9Sstevel@tonic-gate (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages, 4587c478bd9Sstevel@tonic-gate ANON_SLEEP); 4597c478bd9Sstevel@tonic-gate anon_release(amp->ahp, npages); 4607c478bd9Sstevel@tonic-gate amp->ahp = nahp; 4617c478bd9Sstevel@tonic-gate amp->swresv = amp->size = ptob(new_npgs); 4627c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 4637c478bd9Sstevel@tonic-gate npages = new_npgs; 4647c478bd9Sstevel@tonic-gate } 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt = kmem_zalloc(npages * 4677c478bd9Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP); 4687c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt = 0; 4697c478bd9Sstevel@tonic-gate sptd->spt_realsize = ptob(npages); 4707c478bd9Sstevel@tonic-gate sptcargs->seg_spt = seg; 4717c478bd9Sstevel@tonic-gate return (0); 4727c478bd9Sstevel@tonic-gate } 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate /* 4757c478bd9Sstevel@tonic-gate * get array of pages for each anon slot in amp 4767c478bd9Sstevel@tonic-gate */ 4777c478bd9Sstevel@tonic-gate if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa, 4787c478bd9Sstevel@tonic-gate seg, addr, S_CREATE, cred)) != 0) 4797c478bd9Sstevel@tonic-gate goto out4; 4807c478bd9Sstevel@tonic-gate 4817c478bd9Sstevel@tonic-gate /* 4827c478bd9Sstevel@tonic-gate * addr is initial address corresponding to the first page on ppa list 4837c478bd9Sstevel@tonic-gate */ 4847c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 4857c478bd9Sstevel@tonic-gate /* attempt to lock all pages */ 4867c478bd9Sstevel@tonic-gate if (!page_pp_lock(ppa[i], 0, 1)) { 4877c478bd9Sstevel@tonic-gate /* 4887c478bd9Sstevel@tonic-gate * if unable to lock any page, unlock all 4897c478bd9Sstevel@tonic-gate * of them and return error 4907c478bd9Sstevel@tonic-gate */ 4917c478bd9Sstevel@tonic-gate for (j = 0; j < i; j++) 4927c478bd9Sstevel@tonic-gate page_pp_unlock(ppa[j], 0, 1); 4937c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 4947c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 4957c478bd9Sstevel@tonic-gate } 4967c478bd9Sstevel@tonic-gate err = ENOMEM; 4977c478bd9Sstevel@tonic-gate goto out4; 4987c478bd9Sstevel@tonic-gate } 4997c478bd9Sstevel@tonic-gate } 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate /* 5027c478bd9Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 5037c478bd9Sstevel@tonic-gate * for the entire life of the segment. For example platforms 5047c478bd9Sstevel@tonic-gate * that do not support Dynamic Reconfiguration. 5057c478bd9Sstevel@tonic-gate */ 5067c478bd9Sstevel@tonic-gate hat_flags = HAT_LOAD_SHARE; 5077c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) 5087c478bd9Sstevel@tonic-gate hat_flags |= HAT_LOAD_LOCK; 5097c478bd9Sstevel@tonic-gate 5107c478bd9Sstevel@tonic-gate hat_memload_array(seg->s_as->a_hat, addr, ptob(npages), 5117c478bd9Sstevel@tonic-gate ppa, sptd->spt_prot, hat_flags); 5127c478bd9Sstevel@tonic-gate 5137c478bd9Sstevel@tonic-gate /* 5147c478bd9Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 5157c478bd9Sstevel@tonic-gate * we will leave the pages locked SE_SHARED for the life 5167c478bd9Sstevel@tonic-gate * of the ISM segment. This will prevent any calls to 5177c478bd9Sstevel@tonic-gate * hat_pageunload() on this ISM segment for those platforms. 5187c478bd9Sstevel@tonic-gate */ 5197c478bd9Sstevel@tonic-gate if (!(hat_flags & HAT_LOAD_LOCK)) { 5207c478bd9Sstevel@tonic-gate /* 5217c478bd9Sstevel@tonic-gate * On platforms that support HAT_DYNAMIC_ISM_UNMAP, 5227c478bd9Sstevel@tonic-gate * we no longer need to hold the SE_SHARED lock on the pages, 5237c478bd9Sstevel@tonic-gate * since L_PAGELOCK and F_SOFTLOCK calls will grab the 5247c478bd9Sstevel@tonic-gate * SE_SHARED lock on the pages as necessary. 5257c478bd9Sstevel@tonic-gate */ 5267c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) 5277c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 5287c478bd9Sstevel@tonic-gate } 5297c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt = 0; 5307c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * npages)); 5317c478bd9Sstevel@tonic-gate sptd->spt_realsize = ptob(npages); 5327c478bd9Sstevel@tonic-gate atomic_add_long(&spt_used, npages); 5337c478bd9Sstevel@tonic-gate sptcargs->seg_spt = seg; 5347c478bd9Sstevel@tonic-gate return (0); 5357c478bd9Sstevel@tonic-gate 5367c478bd9Sstevel@tonic-gate out4: 5377c478bd9Sstevel@tonic-gate seg->s_data = NULL; 5387c478bd9Sstevel@tonic-gate kmem_free(vp, sizeof (*vp)); 5397c478bd9Sstevel@tonic-gate out3: 5407c478bd9Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock); 5417c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) 5427c478bd9Sstevel@tonic-gate kmem_free(ppa, (sizeof (*ppa) * npages)); 5437c478bd9Sstevel@tonic-gate out2: 5447c478bd9Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd)); 5457c478bd9Sstevel@tonic-gate out1: 5467c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) 5477c478bd9Sstevel@tonic-gate anon_swap_restore(npages); 5487c478bd9Sstevel@tonic-gate return (err); 5497c478bd9Sstevel@tonic-gate } 5507c478bd9Sstevel@tonic-gate 5517c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 5527c478bd9Sstevel@tonic-gate void 5537c478bd9Sstevel@tonic-gate segspt_free_pages(struct seg *seg, caddr_t addr, size_t len) 5547c478bd9Sstevel@tonic-gate { 5557c478bd9Sstevel@tonic-gate struct page *pp; 5567c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data; 5577c478bd9Sstevel@tonic-gate pgcnt_t npages; 5587c478bd9Sstevel@tonic-gate ulong_t anon_idx; 5597c478bd9Sstevel@tonic-gate struct anon_map *amp; 5607c478bd9Sstevel@tonic-gate struct anon *ap; 5617c478bd9Sstevel@tonic-gate struct vnode *vp; 5627c478bd9Sstevel@tonic-gate u_offset_t off; 5637c478bd9Sstevel@tonic-gate uint_t hat_flags; 5647c478bd9Sstevel@tonic-gate int root = 0; 5657c478bd9Sstevel@tonic-gate pgcnt_t pgs, curnpgs = 0; 5667c478bd9Sstevel@tonic-gate page_t *rootpp; 5677c478bd9Sstevel@tonic-gate 5687c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 5697c478bd9Sstevel@tonic-gate 5707c478bd9Sstevel@tonic-gate len = P2ROUNDUP(len, PAGESIZE); 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate npages = btop(len); 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate hat_flags = HAT_UNLOAD_UNLOCK; 5757c478bd9Sstevel@tonic-gate if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) || 5767c478bd9Sstevel@tonic-gate (sptd->spt_flags & SHM_PAGEABLE)) { 5777c478bd9Sstevel@tonic-gate hat_flags = HAT_UNLOAD; 5787c478bd9Sstevel@tonic-gate } 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, hat_flags); 5817c478bd9Sstevel@tonic-gate 5827c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 5837c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) 5847c478bd9Sstevel@tonic-gate npages = btop(amp->size); 5857c478bd9Sstevel@tonic-gate 5867c478bd9Sstevel@tonic-gate ASSERT(amp); 5877c478bd9Sstevel@tonic-gate for (anon_idx = 0; anon_idx < npages; anon_idx++) { 5887c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 5897c478bd9Sstevel@tonic-gate if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) { 5907c478bd9Sstevel@tonic-gate panic("segspt_free_pages: null app"); 5917c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 5927c478bd9Sstevel@tonic-gate } 5937c478bd9Sstevel@tonic-gate } else { 5947c478bd9Sstevel@tonic-gate if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx)) 5957c478bd9Sstevel@tonic-gate == NULL) 5967c478bd9Sstevel@tonic-gate continue; 5977c478bd9Sstevel@tonic-gate } 5987c478bd9Sstevel@tonic-gate ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0); 5997c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate /* 6027c478bd9Sstevel@tonic-gate * If this platform supports HAT_DYNAMIC_ISM_UNMAP, 6037c478bd9Sstevel@tonic-gate * the pages won't be having SE_SHARED lock at this 6047c478bd9Sstevel@tonic-gate * point. 6057c478bd9Sstevel@tonic-gate * 6067c478bd9Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 6077c478bd9Sstevel@tonic-gate * the pages are still held SE_SHARED locked from the 6087c478bd9Sstevel@tonic-gate * original segspt_create() 6097c478bd9Sstevel@tonic-gate * 6107c478bd9Sstevel@tonic-gate * Our goal is to get SE_EXCL lock on each page, remove 6117c478bd9Sstevel@tonic-gate * permanent lock on it and invalidate the page. 6127c478bd9Sstevel@tonic-gate */ 6137c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6147c478bd9Sstevel@tonic-gate if (hat_flags == HAT_UNLOAD) 6157c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL); 6167c478bd9Sstevel@tonic-gate else { 6177c478bd9Sstevel@tonic-gate if ((pp = page_find(vp, off)) == NULL) { 6187c478bd9Sstevel@tonic-gate panic("segspt_free_pages: " 6197c478bd9Sstevel@tonic-gate "page not locked"); 6207c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6217c478bd9Sstevel@tonic-gate } 6227c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 6237c478bd9Sstevel@tonic-gate page_unlock(pp); 6247c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL); 6257c478bd9Sstevel@tonic-gate } 6267c478bd9Sstevel@tonic-gate } 6277c478bd9Sstevel@tonic-gate if (pp == NULL) { 6287c478bd9Sstevel@tonic-gate panic("segspt_free_pages: " 6297c478bd9Sstevel@tonic-gate "page not in the system"); 6307c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6317c478bd9Sstevel@tonic-gate } 6327c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, 1); 6337c478bd9Sstevel@tonic-gate } else { 6347c478bd9Sstevel@tonic-gate if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL) 6357c478bd9Sstevel@tonic-gate continue; 6367c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, 0); 6377c478bd9Sstevel@tonic-gate } 6387c478bd9Sstevel@tonic-gate /* 6397c478bd9Sstevel@tonic-gate * It's logical to invalidate the pages here as in most cases 6407c478bd9Sstevel@tonic-gate * these were created by segspt. 6417c478bd9Sstevel@tonic-gate */ 6427c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 6437c478bd9Sstevel@tonic-gate /* 6447c478bd9Sstevel@tonic-gate * For DISM swap is released in shm_rm_amp. 6457c478bd9Sstevel@tonic-gate */ 6467c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0 && 6477c478bd9Sstevel@tonic-gate ap->an_pvp != NULL) { 6487c478bd9Sstevel@tonic-gate panic("segspt_free_pages: pvp non NULL"); 6497c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6507c478bd9Sstevel@tonic-gate } 6517c478bd9Sstevel@tonic-gate if (root == 0) { 6527c478bd9Sstevel@tonic-gate ASSERT(curnpgs == 0); 6537c478bd9Sstevel@tonic-gate root = 1; 6547c478bd9Sstevel@tonic-gate rootpp = pp; 6557c478bd9Sstevel@tonic-gate pgs = curnpgs = page_get_pagecnt(pp->p_szc); 6567c478bd9Sstevel@tonic-gate ASSERT(pgs > 1); 6577c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pgs, pgs)); 6587c478bd9Sstevel@tonic-gate ASSERT(!(page_pptonum(pp) & (pgs - 1))); 6597c478bd9Sstevel@tonic-gate curnpgs--; 6607c478bd9Sstevel@tonic-gate } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) { 6617c478bd9Sstevel@tonic-gate ASSERT(curnpgs == 1); 6627c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(pp) == 6637c478bd9Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - 1)); 6647c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp); 6657c478bd9Sstevel@tonic-gate root = 0; 6667c478bd9Sstevel@tonic-gate curnpgs = 0; 6677c478bd9Sstevel@tonic-gate } else { 6687c478bd9Sstevel@tonic-gate ASSERT(curnpgs > 1); 6697c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(pp) == 6707c478bd9Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - curnpgs)); 6717c478bd9Sstevel@tonic-gate curnpgs--; 6727c478bd9Sstevel@tonic-gate } 6737c478bd9Sstevel@tonic-gate } else { 6747c478bd9Sstevel@tonic-gate if (root != 0 || curnpgs != 0) { 6757c478bd9Sstevel@tonic-gate panic("segspt_free_pages: bad large page"); 6767c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6777c478bd9Sstevel@tonic-gate } 6787c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional context */ 6797c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred); 6807c478bd9Sstevel@tonic-gate } 6817c478bd9Sstevel@tonic-gate } 6827c478bd9Sstevel@tonic-gate 6837c478bd9Sstevel@tonic-gate if (root != 0 || curnpgs != 0) { 6847c478bd9Sstevel@tonic-gate panic("segspt_free_pages: bad large page"); 6857c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6867c478bd9Sstevel@tonic-gate } 6877c478bd9Sstevel@tonic-gate 6887c478bd9Sstevel@tonic-gate /* 6897c478bd9Sstevel@tonic-gate * mark that pages have been released 6907c478bd9Sstevel@tonic-gate */ 6917c478bd9Sstevel@tonic-gate sptd->spt_realsize = 0; 6927c478bd9Sstevel@tonic-gate 6937c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 6947c478bd9Sstevel@tonic-gate atomic_add_long(&spt_used, -npages); 6957c478bd9Sstevel@tonic-gate anon_swap_restore(npages); 6967c478bd9Sstevel@tonic-gate } 6977c478bd9Sstevel@tonic-gate } 6987c478bd9Sstevel@tonic-gate 6997c478bd9Sstevel@tonic-gate /* 7007c478bd9Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment 7017c478bd9Sstevel@tonic-gate */ 7027c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 7037c478bd9Sstevel@tonic-gate segspt_getpolicy(struct seg *seg, caddr_t addr) 7047c478bd9Sstevel@tonic-gate { 7057c478bd9Sstevel@tonic-gate struct anon_map *amp; 7067c478bd9Sstevel@tonic-gate ulong_t anon_index; 7077c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 7087c478bd9Sstevel@tonic-gate struct spt_data *spt_data; 7097c478bd9Sstevel@tonic-gate 7107c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 7117c478bd9Sstevel@tonic-gate 7127c478bd9Sstevel@tonic-gate /* 7137c478bd9Sstevel@tonic-gate * Get anon_map from segspt 7147c478bd9Sstevel@tonic-gate * 7157c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since 7167c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be 7177c478bd9Sstevel@tonic-gate * nonzero for an existing segment 7187c478bd9Sstevel@tonic-gate * Need to grab readers lock on policy tree though 7197c478bd9Sstevel@tonic-gate */ 7207c478bd9Sstevel@tonic-gate spt_data = (struct spt_data *)seg->s_data; 7217c478bd9Sstevel@tonic-gate if (spt_data == NULL) 7227c478bd9Sstevel@tonic-gate return (NULL); 7237c478bd9Sstevel@tonic-gate amp = spt_data->spt_amp; 7247c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 7257c478bd9Sstevel@tonic-gate 7267c478bd9Sstevel@tonic-gate /* 7277c478bd9Sstevel@tonic-gate * Get policy info 7287c478bd9Sstevel@tonic-gate * 7297c478bd9Sstevel@tonic-gate * Assume starting anon index of 0 7307c478bd9Sstevel@tonic-gate */ 7317c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 7327c478bd9Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 7337c478bd9Sstevel@tonic-gate 7347c478bd9Sstevel@tonic-gate return (policy_info); 7357c478bd9Sstevel@tonic-gate } 7367c478bd9Sstevel@tonic-gate 7377c478bd9Sstevel@tonic-gate /* 7387c478bd9Sstevel@tonic-gate * DISM only. 7397c478bd9Sstevel@tonic-gate * Return locked pages over a given range. 7407c478bd9Sstevel@tonic-gate * 7417c478bd9Sstevel@tonic-gate * We will cache all DISM locked pages and save the pplist for the 7427c478bd9Sstevel@tonic-gate * entire segment in the ppa field of the underlying DISM segment structure. 7437c478bd9Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array 7447c478bd9Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list. 7457c478bd9Sstevel@tonic-gate */ 7467c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7477c478bd9Sstevel@tonic-gate static int 7487c478bd9Sstevel@tonic-gate segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len, 7497c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 7507c478bd9Sstevel@tonic-gate { 7517c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 7527c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 7537c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 7547c478bd9Sstevel@tonic-gate pgcnt_t pg_idx, npages, tot_npages, npgs; 7557c478bd9Sstevel@tonic-gate struct page **pplist, **pl, **ppa, *pp; 7567c478bd9Sstevel@tonic-gate struct anon_map *amp; 7577c478bd9Sstevel@tonic-gate spgcnt_t an_idx; 7587c478bd9Sstevel@tonic-gate int ret = ENOTSUP; 7597c478bd9Sstevel@tonic-gate uint_t pl_built = 0; 7607c478bd9Sstevel@tonic-gate struct anon *ap; 7617c478bd9Sstevel@tonic-gate struct vnode *vp; 7627c478bd9Sstevel@tonic-gate u_offset_t off; 7637c478bd9Sstevel@tonic-gate pgcnt_t claim_availrmem = 0; 7647c478bd9Sstevel@tonic-gate uint_t szc; 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 7677c478bd9Sstevel@tonic-gate 7687c478bd9Sstevel@tonic-gate /* 7697c478bd9Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore, 7707c478bd9Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address 7717c478bd9Sstevel@tonic-gate * and length for the caching arguments. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate ASSERT(sptseg); 7747c478bd9Sstevel@tonic-gate ASSERT(sptd); 7757c478bd9Sstevel@tonic-gate 7767c478bd9Sstevel@tonic-gate pg_idx = seg_page(seg, addr); 7777c478bd9Sstevel@tonic-gate npages = btopr(len); 7787c478bd9Sstevel@tonic-gate 7797c478bd9Sstevel@tonic-gate /* 7807c478bd9Sstevel@tonic-gate * check if the request is larger than number of pages covered 7817c478bd9Sstevel@tonic-gate * by amp 7827c478bd9Sstevel@tonic-gate */ 7837c478bd9Sstevel@tonic-gate if (pg_idx + npages > btopr(sptd->spt_amp->size)) { 7847c478bd9Sstevel@tonic-gate *ppp = NULL; 7857c478bd9Sstevel@tonic-gate return (ENOTSUP); 7867c478bd9Sstevel@tonic-gate } 7877c478bd9Sstevel@tonic-gate 7887c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 7897c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 7907c478bd9Sstevel@tonic-gate 7917c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 7927c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot, segspt_reclaim); 7937c478bd9Sstevel@tonic-gate 7947c478bd9Sstevel@tonic-gate /* 7957c478bd9Sstevel@tonic-gate * If someone is blocked while unmapping, we purge 7967c478bd9Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously 7977c478bd9Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up 7987c478bd9Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while 7997c478bd9Sstevel@tonic-gate * raw async i/o is still in progress or where a thread 8007c478bd9Sstevel@tonic-gate * exits on data fault in a multithreaded application. 8017c478bd9Sstevel@tonic-gate */ 8027c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 8037c478bd9Sstevel@tonic-gate segspt_purge(seg); 8047c478bd9Sstevel@tonic-gate } 8057c478bd9Sstevel@tonic-gate return (0); 8067c478bd9Sstevel@tonic-gate } else if (type == L_PAGERECLAIM) { 8077c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8087c478bd9Sstevel@tonic-gate (void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size, 8097c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot); 8107c478bd9Sstevel@tonic-gate return (0); 8117c478bd9Sstevel@tonic-gate } 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate if (sptd->spt_flags & DISM_PPA_CHANGED) { 8147c478bd9Sstevel@tonic-gate segspt_purge(seg); 8157c478bd9Sstevel@tonic-gate /* 8167c478bd9Sstevel@tonic-gate * for DISM ppa needs to be rebuild since 8177c478bd9Sstevel@tonic-gate * number of locked pages could be changed 8187c478bd9Sstevel@tonic-gate */ 8197c478bd9Sstevel@tonic-gate *ppp = NULL; 8207c478bd9Sstevel@tonic-gate return (ENOTSUP); 8217c478bd9Sstevel@tonic-gate } 8227c478bd9Sstevel@tonic-gate 8237c478bd9Sstevel@tonic-gate /* 8247c478bd9Sstevel@tonic-gate * First try to find pages in segment page cache, without 8257c478bd9Sstevel@tonic-gate * holding the segment lock. 8267c478bd9Sstevel@tonic-gate */ 8277c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 8287c478bd9Sstevel@tonic-gate sptd->spt_prot); 8297c478bd9Sstevel@tonic-gate if (pplist != NULL) { 8307c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8317c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 8327c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa; 8337c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 8347c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 8357c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, 8367c478bd9Sstevel@tonic-gate sptd->spt_amp->size, ppa, 8377c478bd9Sstevel@tonic-gate sptd->spt_prot, segspt_reclaim); 8387c478bd9Sstevel@tonic-gate *ppp = NULL; 8397c478bd9Sstevel@tonic-gate return (ENOTSUP); 8407c478bd9Sstevel@tonic-gate } 8417c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 8427c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 8437c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 8447c478bd9Sstevel@tonic-gate } else { 8457c478bd9Sstevel@tonic-gate an_idx++; 8467c478bd9Sstevel@tonic-gate } 8477c478bd9Sstevel@tonic-gate } 8487c478bd9Sstevel@tonic-gate /* 8497c478bd9Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to 8507c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 8517c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 8527c478bd9Sstevel@tonic-gate */ 8537c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 8547c478bd9Sstevel@tonic-gate return (0); 8557c478bd9Sstevel@tonic-gate } 8567c478bd9Sstevel@tonic-gate 8577c478bd9Sstevel@tonic-gate /* The L_PAGELOCK case... */ 8587c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 8597c478bd9Sstevel@tonic-gate /* 8607c478bd9Sstevel@tonic-gate * try to find pages in segment page cache with mutex 8617c478bd9Sstevel@tonic-gate */ 8627c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 8637c478bd9Sstevel@tonic-gate sptd->spt_prot); 8647c478bd9Sstevel@tonic-gate if (pplist != NULL) { 8657c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 8667c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 8677c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa; 8687c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 8697c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 8707c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 8717c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, 8727c478bd9Sstevel@tonic-gate sptd->spt_amp->size, ppa, 8737c478bd9Sstevel@tonic-gate sptd->spt_prot, segspt_reclaim); 8747c478bd9Sstevel@tonic-gate *ppp = NULL; 8757c478bd9Sstevel@tonic-gate return (ENOTSUP); 8767c478bd9Sstevel@tonic-gate } 8777c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 8787c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 8797c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 8807c478bd9Sstevel@tonic-gate } else { 8817c478bd9Sstevel@tonic-gate an_idx++; 8827c478bd9Sstevel@tonic-gate } 8837c478bd9Sstevel@tonic-gate } 8847c478bd9Sstevel@tonic-gate /* 8857c478bd9Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to 8867c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 8877c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 8887c478bd9Sstevel@tonic-gate */ 8897c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 8907c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 8917c478bd9Sstevel@tonic-gate return (0); 8927c478bd9Sstevel@tonic-gate } 8937c478bd9Sstevel@tonic-gate if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) == 8947c478bd9Sstevel@tonic-gate SEGP_FAIL) { 8957c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 8967c478bd9Sstevel@tonic-gate *ppp = NULL; 8977c478bd9Sstevel@tonic-gate return (ENOTSUP); 8987c478bd9Sstevel@tonic-gate } 8997c478bd9Sstevel@tonic-gate 9007c478bd9Sstevel@tonic-gate /* 9017c478bd9Sstevel@tonic-gate * No need to worry about protections because DISM pages are always rw. 9027c478bd9Sstevel@tonic-gate */ 9037c478bd9Sstevel@tonic-gate pl = pplist = NULL; 9047c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 9057c478bd9Sstevel@tonic-gate 9067c478bd9Sstevel@tonic-gate /* 9077c478bd9Sstevel@tonic-gate * Do we need to build the ppa array? 9087c478bd9Sstevel@tonic-gate */ 9097c478bd9Sstevel@tonic-gate if (sptd->spt_ppa == NULL) { 9107c478bd9Sstevel@tonic-gate pgcnt_t lpg_cnt = 0; 9117c478bd9Sstevel@tonic-gate 9127c478bd9Sstevel@tonic-gate pl_built = 1; 9137c478bd9Sstevel@tonic-gate tot_npages = btopr(sptd->spt_amp->size); 9147c478bd9Sstevel@tonic-gate 9157c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 9167c478bd9Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP); 9177c478bd9Sstevel@tonic-gate pl = pplist; 9187c478bd9Sstevel@tonic-gate 9197c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 9207c478bd9Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; ) { 9217c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 9227c478bd9Sstevel@tonic-gate /* 9237c478bd9Sstevel@tonic-gate * Cache only mlocked pages. For large pages 9247c478bd9Sstevel@tonic-gate * if one (constituent) page is mlocked 9257c478bd9Sstevel@tonic-gate * all pages for that large page 9267c478bd9Sstevel@tonic-gate * are cached also. This is for quick 9277c478bd9Sstevel@tonic-gate * lookups of ppa array; 9287c478bd9Sstevel@tonic-gate */ 9297c478bd9Sstevel@tonic-gate if ((ap != NULL) && (lpg_cnt != 0 || 9307c478bd9Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[an_idx] != 0))) { 9317c478bd9Sstevel@tonic-gate 9327c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 9337c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 9347c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 9357c478bd9Sstevel@tonic-gate if (lpg_cnt == 0) { 9367c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 9377c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(an_idx, npgs)) { 9387c478bd9Sstevel@tonic-gate an_idx = P2ALIGN(an_idx, npgs); 9397c478bd9Sstevel@tonic-gate page_unlock(pp); 9407c478bd9Sstevel@tonic-gate continue; 9417c478bd9Sstevel@tonic-gate } 9427c478bd9Sstevel@tonic-gate } 9437c478bd9Sstevel@tonic-gate if (++lpg_cnt == npgs) 9447c478bd9Sstevel@tonic-gate lpg_cnt = 0; 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate /* 9477c478bd9Sstevel@tonic-gate * availrmem is decremented only 9487c478bd9Sstevel@tonic-gate * for unlocked pages 9497c478bd9Sstevel@tonic-gate */ 9507c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[an_idx] == 0) 9517c478bd9Sstevel@tonic-gate claim_availrmem++; 9527c478bd9Sstevel@tonic-gate pplist[an_idx] = pp; 9537c478bd9Sstevel@tonic-gate } 9547c478bd9Sstevel@tonic-gate an_idx++; 9557c478bd9Sstevel@tonic-gate } 9567c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 9577c478bd9Sstevel@tonic-gate 9587c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 9597c478bd9Sstevel@tonic-gate if (availrmem < tune.t_minarmem + claim_availrmem) { 9607c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 9617c478bd9Sstevel@tonic-gate ret = FC_MAKE_ERR(ENOMEM); 9627c478bd9Sstevel@tonic-gate claim_availrmem = 0; 9637c478bd9Sstevel@tonic-gate goto insert_fail; 9647c478bd9Sstevel@tonic-gate } else { 9657c478bd9Sstevel@tonic-gate availrmem -= claim_availrmem; 9667c478bd9Sstevel@tonic-gate } 9677c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate sptd->spt_ppa = pl; 9707c478bd9Sstevel@tonic-gate } else { 9717c478bd9Sstevel@tonic-gate /* 9727c478bd9Sstevel@tonic-gate * We already have a valid ppa[]. 9737c478bd9Sstevel@tonic-gate */ 9747c478bd9Sstevel@tonic-gate pl = sptd->spt_ppa; 9757c478bd9Sstevel@tonic-gate } 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate ASSERT(pl != NULL); 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size, 9807c478bd9Sstevel@tonic-gate pl, sptd->spt_prot, SEGP_FORCE_WIRED | SEGP_ASYNC_FLUSH, 9817c478bd9Sstevel@tonic-gate segspt_reclaim); 9827c478bd9Sstevel@tonic-gate if (ret == SEGP_FAIL) { 9837c478bd9Sstevel@tonic-gate /* 9847c478bd9Sstevel@tonic-gate * seg_pinsert failed. We return 9857c478bd9Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will 9867c478bd9Sstevel@tonic-gate * then try the slower F_SOFTLOCK path. 9877c478bd9Sstevel@tonic-gate */ 9887c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 9897c478bd9Sstevel@tonic-gate ret = ENOTSUP; 9907c478bd9Sstevel@tonic-gate goto insert_fail; 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate /* 9947c478bd9Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment. 9957c478bd9Sstevel@tonic-gate */ 9967c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt++; 9977c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa; 10007c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 10017c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) { 10027c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 10037c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 10047c478bd9Sstevel@tonic-gate pl, sptd->spt_prot, segspt_reclaim); 10057c478bd9Sstevel@tonic-gate *ppp = NULL; 10067c478bd9Sstevel@tonic-gate return (ENOTSUP); 10077c478bd9Sstevel@tonic-gate } 10087c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) { 10097c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 10107c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs); 10117c478bd9Sstevel@tonic-gate } else { 10127c478bd9Sstevel@tonic-gate an_idx++; 10137c478bd9Sstevel@tonic-gate } 10147c478bd9Sstevel@tonic-gate } 10157c478bd9Sstevel@tonic-gate /* 10167c478bd9Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[] 10177c478bd9Sstevel@tonic-gate * exists and he have incremented pacachecnt. 10187c478bd9Sstevel@tonic-gate */ 10197c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 10207c478bd9Sstevel@tonic-gate 10217c478bd9Sstevel@tonic-gate /* 10227c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to 10237c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 10247c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx. 10257c478bd9Sstevel@tonic-gate */ 10267c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]); 10277c478bd9Sstevel@tonic-gate return (ret); 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate insert_fail: 10307c478bd9Sstevel@tonic-gate /* 10317c478bd9Sstevel@tonic-gate * We will only reach this code if we tried and failed. 10327c478bd9Sstevel@tonic-gate * 10337c478bd9Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed 10347c478bd9Sstevel@tonic-gate * to set up a new ppa[]. 10357c478bd9Sstevel@tonic-gate */ 10367c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 10377c478bd9Sstevel@tonic-gate 10387c478bd9Sstevel@tonic-gate if (pl_built) { 10397c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 10407c478bd9Sstevel@tonic-gate availrmem += claim_availrmem; 10417c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 10427c478bd9Sstevel@tonic-gate 10437c478bd9Sstevel@tonic-gate /* 10447c478bd9Sstevel@tonic-gate * We created pl and we need to destroy it. 10457c478bd9Sstevel@tonic-gate */ 10467c478bd9Sstevel@tonic-gate pplist = pl; 10477c478bd9Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; an_idx++) { 10487c478bd9Sstevel@tonic-gate if (pplist[an_idx] != NULL) 10497c478bd9Sstevel@tonic-gate page_unlock(pplist[an_idx]); 10507c478bd9Sstevel@tonic-gate } 10517c478bd9Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * tot_npages); 10527c478bd9Sstevel@tonic-gate } 10537c478bd9Sstevel@tonic-gate 10547c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 10557c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 10567c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 10577c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 10587c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 10597c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 10607c478bd9Sstevel@tonic-gate } 10617c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 10627c478bd9Sstevel@tonic-gate } 10637c478bd9Sstevel@tonic-gate } 10647c478bd9Sstevel@tonic-gate *ppp = NULL; 10657c478bd9Sstevel@tonic-gate return (ret); 10667c478bd9Sstevel@tonic-gate } 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate /* 10717c478bd9Sstevel@tonic-gate * return locked pages over a given range. 10727c478bd9Sstevel@tonic-gate * 10737c478bd9Sstevel@tonic-gate * We will cache the entire ISM segment and save the pplist for the 10747c478bd9Sstevel@tonic-gate * entire segment in the ppa field of the underlying ISM segment structure. 10757c478bd9Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array 10767c478bd9Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list. 10777c478bd9Sstevel@tonic-gate */ 10787c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10797c478bd9Sstevel@tonic-gate static int 10807c478bd9Sstevel@tonic-gate segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len, 10817c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 10827c478bd9Sstevel@tonic-gate { 10837c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 10847c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 10857c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 10867c478bd9Sstevel@tonic-gate pgcnt_t np, page_index, npages; 10877c478bd9Sstevel@tonic-gate caddr_t a, spt_base; 10887c478bd9Sstevel@tonic-gate struct page **pplist, **pl, *pp; 10897c478bd9Sstevel@tonic-gate struct anon_map *amp; 10907c478bd9Sstevel@tonic-gate ulong_t anon_index; 10917c478bd9Sstevel@tonic-gate int ret = ENOTSUP; 10927c478bd9Sstevel@tonic-gate uint_t pl_built = 0; 10937c478bd9Sstevel@tonic-gate struct anon *ap; 10947c478bd9Sstevel@tonic-gate struct vnode *vp; 10957c478bd9Sstevel@tonic-gate u_offset_t off; 10967c478bd9Sstevel@tonic-gate 10977c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 10987c478bd9Sstevel@tonic-gate 10997c478bd9Sstevel@tonic-gate /* 11007c478bd9Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore, 11017c478bd9Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address 11027c478bd9Sstevel@tonic-gate * and length for the caching arguments. 11037c478bd9Sstevel@tonic-gate */ 11047c478bd9Sstevel@tonic-gate ASSERT(sptseg); 11057c478bd9Sstevel@tonic-gate ASSERT(sptd); 11067c478bd9Sstevel@tonic-gate 11077c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 11087c478bd9Sstevel@tonic-gate return (segspt_dismpagelock(seg, addr, len, ppp, type, rw)); 11097c478bd9Sstevel@tonic-gate } 11107c478bd9Sstevel@tonic-gate 11117c478bd9Sstevel@tonic-gate page_index = seg_page(seg, addr); 11127c478bd9Sstevel@tonic-gate npages = btopr(len); 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate /* 11157c478bd9Sstevel@tonic-gate * check if the request is larger than number of pages covered 11167c478bd9Sstevel@tonic-gate * by amp 11177c478bd9Sstevel@tonic-gate */ 11187c478bd9Sstevel@tonic-gate if (page_index + npages > btopr(sptd->spt_amp->size)) { 11197c478bd9Sstevel@tonic-gate *ppp = NULL; 11207c478bd9Sstevel@tonic-gate return (ENOTSUP); 11217c478bd9Sstevel@tonic-gate } 11227c478bd9Sstevel@tonic-gate 11237c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) { 11247c478bd9Sstevel@tonic-gate 11257c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 11267c478bd9Sstevel@tonic-gate 11277c478bd9Sstevel@tonic-gate seg_pinactive(seg, seg->s_base, sptd->spt_amp->size, 11287c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot, segspt_reclaim); 11297c478bd9Sstevel@tonic-gate 11307c478bd9Sstevel@tonic-gate /* 11317c478bd9Sstevel@tonic-gate * If someone is blocked while unmapping, we purge 11327c478bd9Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously 11337c478bd9Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up 11347c478bd9Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while 11357c478bd9Sstevel@tonic-gate * raw async i/o is still in progress or where a thread 11367c478bd9Sstevel@tonic-gate * exits on data fault in a multithreaded application. 11377c478bd9Sstevel@tonic-gate */ 11387c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 11397c478bd9Sstevel@tonic-gate segspt_purge(seg); 11407c478bd9Sstevel@tonic-gate } 11417c478bd9Sstevel@tonic-gate return (0); 11427c478bd9Sstevel@tonic-gate } else if (type == L_PAGERECLAIM) { 11437c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL); 11447c478bd9Sstevel@tonic-gate 11457c478bd9Sstevel@tonic-gate (void) segspt_reclaim(seg, seg->s_base, sptd->spt_amp->size, 11467c478bd9Sstevel@tonic-gate sptd->spt_ppa, sptd->spt_prot); 11477c478bd9Sstevel@tonic-gate return (0); 11487c478bd9Sstevel@tonic-gate } 11497c478bd9Sstevel@tonic-gate 11507c478bd9Sstevel@tonic-gate /* 11517c478bd9Sstevel@tonic-gate * First try to find pages in segment page cache, without 11527c478bd9Sstevel@tonic-gate * holding the segment lock. 11537c478bd9Sstevel@tonic-gate */ 11547c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 11557c478bd9Sstevel@tonic-gate sptd->spt_prot); 11567c478bd9Sstevel@tonic-gate if (pplist != NULL) { 11577c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 11587c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa[page_index]); 11597c478bd9Sstevel@tonic-gate /* 11607c478bd9Sstevel@tonic-gate * Since we cache the entire ISM segment, we want to 11617c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 11627c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index. 11637c478bd9Sstevel@tonic-gate */ 11647c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 11657c478bd9Sstevel@tonic-gate return (0); 11667c478bd9Sstevel@tonic-gate } 11677c478bd9Sstevel@tonic-gate 11687c478bd9Sstevel@tonic-gate /* The L_PAGELOCK case... */ 11697c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 11707c478bd9Sstevel@tonic-gate 11717c478bd9Sstevel@tonic-gate /* 11727c478bd9Sstevel@tonic-gate * try to find pages in segment page cache 11737c478bd9Sstevel@tonic-gate */ 11747c478bd9Sstevel@tonic-gate pplist = seg_plookup(seg, seg->s_base, sptd->spt_amp->size, 11757c478bd9Sstevel@tonic-gate sptd->spt_prot); 11767c478bd9Sstevel@tonic-gate if (pplist != NULL) { 11777c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 11787c478bd9Sstevel@tonic-gate /* 11797c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to 11807c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 11817c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index. 11827c478bd9Sstevel@tonic-gate */ 11837c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 11847c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 11857c478bd9Sstevel@tonic-gate return (0); 11867c478bd9Sstevel@tonic-gate } 11877c478bd9Sstevel@tonic-gate 11887c478bd9Sstevel@tonic-gate if (seg_pinsert_check(seg, sptd->spt_amp->size, SEGP_FORCE_WIRED) == 11897c478bd9Sstevel@tonic-gate SEGP_FAIL) { 11907c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 11917c478bd9Sstevel@tonic-gate *ppp = NULL; 11927c478bd9Sstevel@tonic-gate return (ENOTSUP); 11937c478bd9Sstevel@tonic-gate } 11947c478bd9Sstevel@tonic-gate 11957c478bd9Sstevel@tonic-gate /* 11967c478bd9Sstevel@tonic-gate * No need to worry about protections because ISM pages 11977c478bd9Sstevel@tonic-gate * are always rw. 11987c478bd9Sstevel@tonic-gate */ 11997c478bd9Sstevel@tonic-gate pl = pplist = NULL; 12007c478bd9Sstevel@tonic-gate 12017c478bd9Sstevel@tonic-gate /* 12027c478bd9Sstevel@tonic-gate * Do we need to build the ppa array? 12037c478bd9Sstevel@tonic-gate */ 12047c478bd9Sstevel@tonic-gate if (sptd->spt_ppa == NULL) { 12057c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 12067c478bd9Sstevel@tonic-gate 12077c478bd9Sstevel@tonic-gate spt_base = sptseg->s_base; 12087c478bd9Sstevel@tonic-gate pl_built = 1; 12097c478bd9Sstevel@tonic-gate 12107c478bd9Sstevel@tonic-gate /* 12117c478bd9Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust() 12127c478bd9Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is 12137c478bd9Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed. 12147c478bd9Sstevel@tonic-gate */ 12157c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 12167c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate /* pcachecnt is protected by sptd->spt_lock */ 12197c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 12207c478bd9Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) 12217c478bd9Sstevel@tonic-gate * btopr(sptd->spt_amp->size), KM_SLEEP); 12227c478bd9Sstevel@tonic-gate pl = pplist; 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, spt_base); 12257c478bd9Sstevel@tonic-gate 12267c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 12277c478bd9Sstevel@tonic-gate for (a = spt_base; a < (spt_base + sptd->spt_amp->size); 12287c478bd9Sstevel@tonic-gate a += PAGESIZE, anon_index++, pplist++) { 12297c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 12307c478bd9Sstevel@tonic-gate ASSERT(ap != NULL); 12317c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 12327c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 12337c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 12347c478bd9Sstevel@tonic-gate *pplist = pp; 12357c478bd9Sstevel@tonic-gate } 12367c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 12377c478bd9Sstevel@tonic-gate 12387c478bd9Sstevel@tonic-gate if (a < (spt_base + sptd->spt_amp->size)) { 12397c478bd9Sstevel@tonic-gate ret = ENOTSUP; 12407c478bd9Sstevel@tonic-gate goto insert_fail; 12417c478bd9Sstevel@tonic-gate } 12427c478bd9Sstevel@tonic-gate sptd->spt_ppa = pl; 12437c478bd9Sstevel@tonic-gate } else { 12447c478bd9Sstevel@tonic-gate /* 12457c478bd9Sstevel@tonic-gate * We already have a valid ppa[]. 12467c478bd9Sstevel@tonic-gate */ 12477c478bd9Sstevel@tonic-gate pl = sptd->spt_ppa; 12487c478bd9Sstevel@tonic-gate } 12497c478bd9Sstevel@tonic-gate 12507c478bd9Sstevel@tonic-gate ASSERT(pl != NULL); 12517c478bd9Sstevel@tonic-gate 12527c478bd9Sstevel@tonic-gate ret = seg_pinsert(seg, seg->s_base, sptd->spt_amp->size, 12537c478bd9Sstevel@tonic-gate pl, sptd->spt_prot, SEGP_FORCE_WIRED, segspt_reclaim); 12547c478bd9Sstevel@tonic-gate if (ret == SEGP_FAIL) { 12557c478bd9Sstevel@tonic-gate /* 12567c478bd9Sstevel@tonic-gate * seg_pinsert failed. We return 12577c478bd9Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will 12587c478bd9Sstevel@tonic-gate * then try the slower F_SOFTLOCK path. 12597c478bd9Sstevel@tonic-gate */ 12607c478bd9Sstevel@tonic-gate if (pl_built) { 12617c478bd9Sstevel@tonic-gate /* 12627c478bd9Sstevel@tonic-gate * No one else has referenced the ppa[]. 12637c478bd9Sstevel@tonic-gate * We created it and we need to destroy it. 12647c478bd9Sstevel@tonic-gate */ 12657c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 12667c478bd9Sstevel@tonic-gate } 12677c478bd9Sstevel@tonic-gate ret = ENOTSUP; 12687c478bd9Sstevel@tonic-gate goto insert_fail; 12697c478bd9Sstevel@tonic-gate } 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate /* 12727c478bd9Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment. 12737c478bd9Sstevel@tonic-gate */ 12747c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt++; 12757c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 12767c478bd9Sstevel@tonic-gate 12777c478bd9Sstevel@tonic-gate /* 12787c478bd9Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[] 12797c478bd9Sstevel@tonic-gate * exists and he have incremented pacachecnt. 12807c478bd9Sstevel@tonic-gate */ 12817c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 12827c478bd9Sstevel@tonic-gate 12837c478bd9Sstevel@tonic-gate /* 12847c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to 12857c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds 12867c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index. 12877c478bd9Sstevel@tonic-gate */ 12887c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]); 12897c478bd9Sstevel@tonic-gate return (ret); 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate insert_fail: 12927c478bd9Sstevel@tonic-gate /* 12937c478bd9Sstevel@tonic-gate * We will only reach this code if we tried and failed. 12947c478bd9Sstevel@tonic-gate * 12957c478bd9Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed 12967c478bd9Sstevel@tonic-gate * to set up a new ppa[]. 12977c478bd9Sstevel@tonic-gate */ 12987c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate if (pl_built) { 13017c478bd9Sstevel@tonic-gate /* 13027c478bd9Sstevel@tonic-gate * We created pl and we need to destroy it. 13037c478bd9Sstevel@tonic-gate */ 13047c478bd9Sstevel@tonic-gate pplist = pl; 13057c478bd9Sstevel@tonic-gate np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT); 13067c478bd9Sstevel@tonic-gate while (np) { 13077c478bd9Sstevel@tonic-gate page_unlock(*pplist); 13087c478bd9Sstevel@tonic-gate np--; 13097c478bd9Sstevel@tonic-gate pplist++; 13107c478bd9Sstevel@tonic-gate } 13117c478bd9Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * 13127c478bd9Sstevel@tonic-gate btopr(sptd->spt_amp->size)); 13137c478bd9Sstevel@tonic-gate } 13147c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 13157c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 13167c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 13177c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 13187c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 13197c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 13207c478bd9Sstevel@tonic-gate } 13217c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 13227c478bd9Sstevel@tonic-gate } 13237c478bd9Sstevel@tonic-gate } 13247c478bd9Sstevel@tonic-gate *ppp = NULL; 13257c478bd9Sstevel@tonic-gate return (ret); 13267c478bd9Sstevel@tonic-gate } 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate /* 13297c478bd9Sstevel@tonic-gate * purge any cached pages in the I/O page cache 13307c478bd9Sstevel@tonic-gate */ 13317c478bd9Sstevel@tonic-gate static void 13327c478bd9Sstevel@tonic-gate segspt_purge(struct seg *seg) 13337c478bd9Sstevel@tonic-gate { 13347c478bd9Sstevel@tonic-gate seg_ppurge(seg); 13357c478bd9Sstevel@tonic-gate } 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate static int 13387c478bd9Sstevel@tonic-gate segspt_reclaim(struct seg *seg, caddr_t addr, size_t len, struct page **pplist, 13397c478bd9Sstevel@tonic-gate enum seg_rw rw) 13407c478bd9Sstevel@tonic-gate { 13417c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 13427c478bd9Sstevel@tonic-gate struct seg *sptseg; 13437c478bd9Sstevel@tonic-gate struct spt_data *sptd; 13447c478bd9Sstevel@tonic-gate pgcnt_t npages, i, free_availrmem = 0; 13457c478bd9Sstevel@tonic-gate int done = 0; 13467c478bd9Sstevel@tonic-gate 13477c478bd9Sstevel@tonic-gate #ifdef lint 13487c478bd9Sstevel@tonic-gate addr = addr; 13497c478bd9Sstevel@tonic-gate #endif 13507c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 13517c478bd9Sstevel@tonic-gate sptd = sptseg->s_data; 13527c478bd9Sstevel@tonic-gate npages = (len >> PAGESHIFT); 13537c478bd9Sstevel@tonic-gate ASSERT(npages); 13547c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt != 0); 13557c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist); 13567c478bd9Sstevel@tonic-gate ASSERT(npages == btopr(sptd->spt_amp->size)); 13577c478bd9Sstevel@tonic-gate 13587c478bd9Sstevel@tonic-gate /* 13597c478bd9Sstevel@tonic-gate * Acquire the lock on the dummy seg and destroy the 13607c478bd9Sstevel@tonic-gate * ppa array IF this is the last pcachecnt. 13617c478bd9Sstevel@tonic-gate */ 13627c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 13637c478bd9Sstevel@tonic-gate if (--sptd->spt_pcachecnt == 0) { 13647c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 13657c478bd9Sstevel@tonic-gate if (pplist[i] == NULL) { 13667c478bd9Sstevel@tonic-gate continue; 13677c478bd9Sstevel@tonic-gate } 13687c478bd9Sstevel@tonic-gate if (rw == S_WRITE) { 13697c478bd9Sstevel@tonic-gate hat_setrefmod(pplist[i]); 13707c478bd9Sstevel@tonic-gate } else { 13717c478bd9Sstevel@tonic-gate hat_setref(pplist[i]); 13727c478bd9Sstevel@tonic-gate } 13737c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) && 13747c478bd9Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[i] == 0)) 13757c478bd9Sstevel@tonic-gate free_availrmem++; 13767c478bd9Sstevel@tonic-gate page_unlock(pplist[i]); 13777c478bd9Sstevel@tonic-gate } 13787c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 13797c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 13807c478bd9Sstevel@tonic-gate availrmem += free_availrmem; 13817c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 13827c478bd9Sstevel@tonic-gate } 13837c478bd9Sstevel@tonic-gate /* 13847c478bd9Sstevel@tonic-gate * Since we want to cach/uncache the entire ISM segment, 13857c478bd9Sstevel@tonic-gate * we will track the pplist in a segspt specific field 13867c478bd9Sstevel@tonic-gate * ppa, that is initialized at the time we add an entry to 13877c478bd9Sstevel@tonic-gate * the cache. 13887c478bd9Sstevel@tonic-gate */ 13897c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0); 13907c478bd9Sstevel@tonic-gate kmem_free(pplist, sizeof (page_t *) * npages); 13917c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL; 13927c478bd9Sstevel@tonic-gate sptd->spt_flags &= ~DISM_PPA_CHANGED; 13937c478bd9Sstevel@tonic-gate done = 1; 13947c478bd9Sstevel@tonic-gate } 13957c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 13967c478bd9Sstevel@tonic-gate /* 13977c478bd9Sstevel@tonic-gate * Now decrement softlockcnt. 13987c478bd9Sstevel@tonic-gate */ 13997c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) { 14027c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 14037c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 14047c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 14057c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 14067c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 14077c478bd9Sstevel@tonic-gate } 14087c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 14097c478bd9Sstevel@tonic-gate } 14107c478bd9Sstevel@tonic-gate } 14117c478bd9Sstevel@tonic-gate return (done); 14127c478bd9Sstevel@tonic-gate } 14137c478bd9Sstevel@tonic-gate 14147c478bd9Sstevel@tonic-gate /* 14157c478bd9Sstevel@tonic-gate * Do a F_SOFTUNLOCK call over the range requested. 14167c478bd9Sstevel@tonic-gate * The range must have already been F_SOFTLOCK'ed. 14177c478bd9Sstevel@tonic-gate * 14187c478bd9Sstevel@tonic-gate * The calls to acquire and release the anon map lock mutex were 14197c478bd9Sstevel@tonic-gate * removed in order to avoid a deadly embrace during a DR 14207c478bd9Sstevel@tonic-gate * memory delete operation. (Eg. DR blocks while waiting for a 14217c478bd9Sstevel@tonic-gate * exclusive lock on a page that is being used for kaio; the 14227c478bd9Sstevel@tonic-gate * thread that will complete the kaio and call segspt_softunlock 14237c478bd9Sstevel@tonic-gate * blocks on the anon map lock; another thread holding the anon 14247c478bd9Sstevel@tonic-gate * map lock blocks on another page lock via the segspt_shmfault 14257c478bd9Sstevel@tonic-gate * -> page_lookup -> page_lookup_create -> page_lock_es code flow.) 14267c478bd9Sstevel@tonic-gate * 14277c478bd9Sstevel@tonic-gate * The appropriateness of the removal is based upon the following: 14287c478bd9Sstevel@tonic-gate * 1. If we are holding a segment's reader lock and the page is held 14297c478bd9Sstevel@tonic-gate * shared, then the corresponding element in anonmap which points to 14307c478bd9Sstevel@tonic-gate * anon struct cannot change and there is no need to acquire the 14317c478bd9Sstevel@tonic-gate * anonymous map lock. 14327c478bd9Sstevel@tonic-gate * 2. Threads in segspt_softunlock have a reader lock on the segment 14337c478bd9Sstevel@tonic-gate * and already have the shared page lock, so we are guaranteed that 14347c478bd9Sstevel@tonic-gate * the anon map slot cannot change and therefore can call anon_get_ptr() 14357c478bd9Sstevel@tonic-gate * without grabbing the anonymous map lock. 14367c478bd9Sstevel@tonic-gate * 3. Threads that softlock a shared page break copy-on-write, even if 14377c478bd9Sstevel@tonic-gate * its a read. Thus cow faults can be ignored with respect to soft 14387c478bd9Sstevel@tonic-gate * unlocking, since the breaking of cow means that the anon slot(s) will 14397c478bd9Sstevel@tonic-gate * not be shared. 14407c478bd9Sstevel@tonic-gate */ 14417c478bd9Sstevel@tonic-gate static void 14427c478bd9Sstevel@tonic-gate segspt_softunlock(struct seg *seg, caddr_t sptseg_addr, 14437c478bd9Sstevel@tonic-gate size_t len, enum seg_rw rw) 14447c478bd9Sstevel@tonic-gate { 14457c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 14467c478bd9Sstevel@tonic-gate struct seg *sptseg; 14477c478bd9Sstevel@tonic-gate struct spt_data *sptd; 14487c478bd9Sstevel@tonic-gate page_t *pp; 14497c478bd9Sstevel@tonic-gate caddr_t adr; 14507c478bd9Sstevel@tonic-gate struct vnode *vp; 14517c478bd9Sstevel@tonic-gate u_offset_t offset; 14527c478bd9Sstevel@tonic-gate ulong_t anon_index; 14537c478bd9Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */ 14547c478bd9Sstevel@tonic-gate struct anon *ap = NULL; 14557c478bd9Sstevel@tonic-gate pgcnt_t npages; 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 14587c478bd9Sstevel@tonic-gate 14597c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg; 14607c478bd9Sstevel@tonic-gate sptd = sptseg->s_data; 14617c478bd9Sstevel@tonic-gate 14627c478bd9Sstevel@tonic-gate /* 14637c478bd9Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 14647c478bd9Sstevel@tonic-gate * and therefore their pages are SE_SHARED locked 14657c478bd9Sstevel@tonic-gate * for the entire life of the segment. 14667c478bd9Sstevel@tonic-gate */ 14677c478bd9Sstevel@tonic-gate if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) && 14687c478bd9Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) == 0)) { 14697c478bd9Sstevel@tonic-gate goto softlock_decrement; 14707c478bd9Sstevel@tonic-gate } 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate /* 14737c478bd9Sstevel@tonic-gate * Any thread is free to do a page_find and 14747c478bd9Sstevel@tonic-gate * page_unlock() on the pages within this seg. 14757c478bd9Sstevel@tonic-gate * 14767c478bd9Sstevel@tonic-gate * We are already holding the as->a_lock on the user's 14777c478bd9Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the 14787c478bd9Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the 14797c478bd9Sstevel@tonic-gate * underlying HAT layer. 14807c478bd9Sstevel@tonic-gate */ 14817c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 14827c478bd9Sstevel@tonic-gate hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len); 14837c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 14847c478bd9Sstevel@tonic-gate 14857c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 14867c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 14877c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr); 14887c478bd9Sstevel@tonic-gate 14897c478bd9Sstevel@tonic-gate for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) { 14907c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++); 14917c478bd9Sstevel@tonic-gate ASSERT(ap != NULL); 14927c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &offset); 14937c478bd9Sstevel@tonic-gate 14947c478bd9Sstevel@tonic-gate /* 14957c478bd9Sstevel@tonic-gate * Use page_find() instead of page_lookup() to 14967c478bd9Sstevel@tonic-gate * find the page since we know that it has a 14977c478bd9Sstevel@tonic-gate * "shared" lock. 14987c478bd9Sstevel@tonic-gate */ 14997c478bd9Sstevel@tonic-gate pp = page_find(vp, offset); 15007c478bd9Sstevel@tonic-gate ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1)); 15017c478bd9Sstevel@tonic-gate if (pp == NULL) { 15027c478bd9Sstevel@tonic-gate panic("segspt_softunlock: " 15037c478bd9Sstevel@tonic-gate "addr %p, ap %p, vp %p, off %llx", 15047c478bd9Sstevel@tonic-gate (void *)adr, (void *)ap, (void *)vp, offset); 15057c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 15067c478bd9Sstevel@tonic-gate } 15077c478bd9Sstevel@tonic-gate 15087c478bd9Sstevel@tonic-gate if (rw == S_WRITE) { 15097c478bd9Sstevel@tonic-gate hat_setrefmod(pp); 15107c478bd9Sstevel@tonic-gate } else if (rw != S_OTHER) { 15117c478bd9Sstevel@tonic-gate hat_setref(pp); 15127c478bd9Sstevel@tonic-gate } 15137c478bd9Sstevel@tonic-gate page_unlock(pp); 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate softlock_decrement: 15177c478bd9Sstevel@tonic-gate npages = btopr(len); 15187c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages); 15197c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt == 0) { 15207c478bd9Sstevel@tonic-gate /* 15217c478bd9Sstevel@tonic-gate * All SOFTLOCKS are gone. Wakeup any waiting 15227c478bd9Sstevel@tonic-gate * unmappers so they can try again to unmap. 15237c478bd9Sstevel@tonic-gate * Check for waiters first without the mutex 15247c478bd9Sstevel@tonic-gate * held so we don't always grab the mutex on 15257c478bd9Sstevel@tonic-gate * softunlocks. 15267c478bd9Sstevel@tonic-gate */ 15277c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 15287c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 15297c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 15307c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 15317c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 15327c478bd9Sstevel@tonic-gate } 15337c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 15347c478bd9Sstevel@tonic-gate } 15357c478bd9Sstevel@tonic-gate } 15367c478bd9Sstevel@tonic-gate } 15377c478bd9Sstevel@tonic-gate 15387c478bd9Sstevel@tonic-gate int 15397c478bd9Sstevel@tonic-gate segspt_shmattach(struct seg *seg, caddr_t *argsp) 15407c478bd9Sstevel@tonic-gate { 15417c478bd9Sstevel@tonic-gate struct shm_data *shmd_arg = (struct shm_data *)argsp; 15427c478bd9Sstevel@tonic-gate struct shm_data *shmd; 15437c478bd9Sstevel@tonic-gate struct anon_map *shm_amp = shmd_arg->shm_amp; 15447c478bd9Sstevel@tonic-gate struct spt_data *sptd; 15457c478bd9Sstevel@tonic-gate int error = 0; 15467c478bd9Sstevel@tonic-gate 15477c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 15487c478bd9Sstevel@tonic-gate 15497c478bd9Sstevel@tonic-gate shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP); 15507c478bd9Sstevel@tonic-gate if (shmd == NULL) 15517c478bd9Sstevel@tonic-gate return (ENOMEM); 15527c478bd9Sstevel@tonic-gate 15537c478bd9Sstevel@tonic-gate shmd->shm_sptas = shmd_arg->shm_sptas; 15547c478bd9Sstevel@tonic-gate shmd->shm_amp = shm_amp; 15557c478bd9Sstevel@tonic-gate shmd->shm_sptseg = shmd_arg->shm_sptseg; 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0, 15587c478bd9Sstevel@tonic-gate NULL, 0, seg->s_size); 15597c478bd9Sstevel@tonic-gate 15607c478bd9Sstevel@tonic-gate seg->s_data = (void *)shmd; 15617c478bd9Sstevel@tonic-gate seg->s_ops = &segspt_shmops; 15627c478bd9Sstevel@tonic-gate seg->s_szc = shmd->shm_sptseg->s_szc; 15637c478bd9Sstevel@tonic-gate sptd = shmd->shm_sptseg->s_data; 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 15667c478bd9Sstevel@tonic-gate if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size), 15677c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) { 15687c478bd9Sstevel@tonic-gate seg->s_data = (void *)NULL; 15697c478bd9Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd))); 15707c478bd9Sstevel@tonic-gate return (ENOMEM); 15717c478bd9Sstevel@tonic-gate } 15727c478bd9Sstevel@tonic-gate shmd->shm_lckpgs = 0; 15737c478bd9Sstevel@tonic-gate if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 15747c478bd9Sstevel@tonic-gate if ((error = hat_share(seg->s_as->a_hat, seg->s_base, 15757c478bd9Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 15767c478bd9Sstevel@tonic-gate seg->s_size, seg->s_szc)) != 0) { 15777c478bd9Sstevel@tonic-gate kmem_free(shmd->shm_vpage, 15787c478bd9Sstevel@tonic-gate btopr(shm_amp->size)); 15797c478bd9Sstevel@tonic-gate } 15807c478bd9Sstevel@tonic-gate } 15817c478bd9Sstevel@tonic-gate } else { 15827c478bd9Sstevel@tonic-gate error = hat_share(seg->s_as->a_hat, seg->s_base, 15837c478bd9Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 15847c478bd9Sstevel@tonic-gate seg->s_size, seg->s_szc); 15857c478bd9Sstevel@tonic-gate } 15867c478bd9Sstevel@tonic-gate if (error) { 15877c478bd9Sstevel@tonic-gate seg->s_szc = 0; 15887c478bd9Sstevel@tonic-gate seg->s_data = (void *)NULL; 15897c478bd9Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd))); 15907c478bd9Sstevel@tonic-gate } else { 15917c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 15927c478bd9Sstevel@tonic-gate shm_amp->refcnt++; 15937c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock); 15947c478bd9Sstevel@tonic-gate } 15957c478bd9Sstevel@tonic-gate return (error); 15967c478bd9Sstevel@tonic-gate } 15977c478bd9Sstevel@tonic-gate 15987c478bd9Sstevel@tonic-gate int 15997c478bd9Sstevel@tonic-gate segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize) 16007c478bd9Sstevel@tonic-gate { 16017c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 16027c478bd9Sstevel@tonic-gate int reclaim = 1; 16037c478bd9Sstevel@tonic-gate 16047c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 16057c478bd9Sstevel@tonic-gate retry: 16067c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) { 16077c478bd9Sstevel@tonic-gate if (reclaim == 1) { 16087c478bd9Sstevel@tonic-gate segspt_purge(seg); 16097c478bd9Sstevel@tonic-gate reclaim = 0; 16107c478bd9Sstevel@tonic-gate goto retry; 16117c478bd9Sstevel@tonic-gate } 16127c478bd9Sstevel@tonic-gate return (EAGAIN); 16137c478bd9Sstevel@tonic-gate } 16147c478bd9Sstevel@tonic-gate 16157c478bd9Sstevel@tonic-gate if (ssize != seg->s_size) { 16167c478bd9Sstevel@tonic-gate #ifdef DEBUG 16177c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n", 16187c478bd9Sstevel@tonic-gate ssize, seg->s_size); 16197c478bd9Sstevel@tonic-gate #endif 16207c478bd9Sstevel@tonic-gate return (EINVAL); 16217c478bd9Sstevel@tonic-gate } 16227c478bd9Sstevel@tonic-gate 16237c478bd9Sstevel@tonic-gate (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK, 16247c478bd9Sstevel@tonic-gate NULL, 0); 16257c478bd9Sstevel@tonic-gate hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc); 16267c478bd9Sstevel@tonic-gate 16277c478bd9Sstevel@tonic-gate seg_free(seg); 16287c478bd9Sstevel@tonic-gate 16297c478bd9Sstevel@tonic-gate return (0); 16307c478bd9Sstevel@tonic-gate } 16317c478bd9Sstevel@tonic-gate 16327c478bd9Sstevel@tonic-gate void 16337c478bd9Sstevel@tonic-gate segspt_shmfree(struct seg *seg) 16347c478bd9Sstevel@tonic-gate { 16357c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 16367c478bd9Sstevel@tonic-gate struct anon_map *shm_amp = shmd->shm_amp; 16377c478bd9Sstevel@tonic-gate 16387c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 16397c478bd9Sstevel@tonic-gate 16407c478bd9Sstevel@tonic-gate (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0, 16417c478bd9Sstevel@tonic-gate MC_UNLOCK, NULL, 0); 16427c478bd9Sstevel@tonic-gate 16437c478bd9Sstevel@tonic-gate /* 16447c478bd9Sstevel@tonic-gate * Need to increment refcnt when attaching 16457c478bd9Sstevel@tonic-gate * and decrement when detaching because of dup(). 16467c478bd9Sstevel@tonic-gate */ 16477c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 16487c478bd9Sstevel@tonic-gate shm_amp->refcnt--; 16497c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock); 16507c478bd9Sstevel@tonic-gate 16517c478bd9Sstevel@tonic-gate if (shmd->shm_vpage) { /* only for DISM */ 16527c478bd9Sstevel@tonic-gate kmem_free(shmd->shm_vpage, btopr(shm_amp->size)); 16537c478bd9Sstevel@tonic-gate shmd->shm_vpage = NULL; 16547c478bd9Sstevel@tonic-gate } 16557c478bd9Sstevel@tonic-gate kmem_free(shmd, sizeof (*shmd)); 16567c478bd9Sstevel@tonic-gate } 16577c478bd9Sstevel@tonic-gate 16587c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16597c478bd9Sstevel@tonic-gate int 16607c478bd9Sstevel@tonic-gate segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 16617c478bd9Sstevel@tonic-gate { 16627c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 16637c478bd9Sstevel@tonic-gate 16647c478bd9Sstevel@tonic-gate /* 16657c478bd9Sstevel@tonic-gate * Shared page table is more than shared mapping. 16667c478bd9Sstevel@tonic-gate * Individual process sharing page tables can't change prot 16677c478bd9Sstevel@tonic-gate * because there is only one set of page tables. 16687c478bd9Sstevel@tonic-gate * This will be allowed after private page table is 16697c478bd9Sstevel@tonic-gate * supported. 16707c478bd9Sstevel@tonic-gate */ 16717c478bd9Sstevel@tonic-gate /* need to return correct status error? */ 16727c478bd9Sstevel@tonic-gate return (0); 16737c478bd9Sstevel@tonic-gate } 16747c478bd9Sstevel@tonic-gate 16757c478bd9Sstevel@tonic-gate 16767c478bd9Sstevel@tonic-gate faultcode_t 16777c478bd9Sstevel@tonic-gate segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr, 16787c478bd9Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw) 16797c478bd9Sstevel@tonic-gate { 16807c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 16817c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 16827c478bd9Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas; 16837c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 16847c478bd9Sstevel@tonic-gate pgcnt_t npages; 16857c478bd9Sstevel@tonic-gate size_t share_sz, size; 16867c478bd9Sstevel@tonic-gate caddr_t segspt_addr, shm_addr; 16877c478bd9Sstevel@tonic-gate page_t **ppa; 16887c478bd9Sstevel@tonic-gate int i; 16897c478bd9Sstevel@tonic-gate ulong_t an_idx = 0; 16907c478bd9Sstevel@tonic-gate int err = 0; 16917c478bd9Sstevel@tonic-gate 16927c478bd9Sstevel@tonic-gate #ifdef lint 16937c478bd9Sstevel@tonic-gate hat = hat; 16947c478bd9Sstevel@tonic-gate #endif 16957c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 16967c478bd9Sstevel@tonic-gate 16977c478bd9Sstevel@tonic-gate /* 16987c478bd9Sstevel@tonic-gate * Because of the way spt is implemented 16997c478bd9Sstevel@tonic-gate * the realsize of the segment does not have to be 17007c478bd9Sstevel@tonic-gate * equal to the segment size itself. The segment size is 17017c478bd9Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE. 17027c478bd9Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE 17037c478bd9Sstevel@tonic-gate * based on what the user requested. This is a bit of 17047c478bd9Sstevel@tonic-gate * ungliness that is historical but not easily fixed 17057c478bd9Sstevel@tonic-gate * without re-designing the higher levels of ISM. 17067c478bd9Sstevel@tonic-gate */ 17077c478bd9Sstevel@tonic-gate ASSERT(addr >= seg->s_base); 17087c478bd9Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize) 17097c478bd9Sstevel@tonic-gate return (FC_NOMAP); 17107c478bd9Sstevel@tonic-gate /* 17117c478bd9Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to 17127c478bd9Sstevel@tonic-gate * make any necessary adjustments to addr and len 17137c478bd9Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[]. 17147c478bd9Sstevel@tonic-gate * 17157c478bd9Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment 17167c478bd9Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore, 17177c478bd9Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 17187c478bd9Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 17197c478bd9Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT 17207c478bd9Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes 17217c478bd9Sstevel@tonic-gate * over a given virtual range. 17227c478bd9Sstevel@tonic-gate */ 17237c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(sptseg->s_szc); 17247c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 17257c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), share_sz); 17267c478bd9Sstevel@tonic-gate npages = btopr(size); 17277c478bd9Sstevel@tonic-gate 17287c478bd9Sstevel@tonic-gate /* 17297c478bd9Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt. 17307c478bd9Sstevel@tonic-gate */ 17317c478bd9Sstevel@tonic-gate an_idx = seg_page(seg, shm_addr); 17327c478bd9Sstevel@tonic-gate segspt_addr = sptseg->s_base + ptob(an_idx); 17337c478bd9Sstevel@tonic-gate 17347c478bd9Sstevel@tonic-gate ASSERT((segspt_addr + ptob(npages)) <= 17357c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)); 17367c478bd9Sstevel@tonic-gate ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size)); 17377c478bd9Sstevel@tonic-gate 17387c478bd9Sstevel@tonic-gate switch (type) { 17397c478bd9Sstevel@tonic-gate 17407c478bd9Sstevel@tonic-gate case F_SOFTLOCK: 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 17437c478bd9Sstevel@tonic-gate if (availrmem < tune.t_minarmem + npages) { 17447c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 17457c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(ENOMEM)); 17467c478bd9Sstevel@tonic-gate } else { 17477c478bd9Sstevel@tonic-gate availrmem -= npages; 17487c478bd9Sstevel@tonic-gate } 17497c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 17507c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 17517c478bd9Sstevel@tonic-gate /* 17527c478bd9Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer 17537c478bd9Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag. 17547c478bd9Sstevel@tonic-gate */ 17557c478bd9Sstevel@tonic-gate /* FALLTHRU */ 17567c478bd9Sstevel@tonic-gate case F_INVAL: 17577c478bd9Sstevel@tonic-gate 17587c478bd9Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 17597c478bd9Sstevel@tonic-gate return (FC_NOMAP); 17607c478bd9Sstevel@tonic-gate 17617c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP); 17627c478bd9Sstevel@tonic-gate 17637c478bd9Sstevel@tonic-gate err = spt_anon_getpages(sptseg, segspt_addr, size, ppa); 17647c478bd9Sstevel@tonic-gate if (err != 0) { 17657c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 17667c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 17677c478bd9Sstevel@tonic-gate availrmem += npages; 17687c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 17697c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)( 17707c478bd9Sstevel@tonic-gate &(shmd->shm_softlockcnt)), -npages); 17717c478bd9Sstevel@tonic-gate } 17727c478bd9Sstevel@tonic-gate goto dism_err; 17737c478bd9Sstevel@tonic-gate } 17747c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 17757c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 17767c478bd9Sstevel@tonic-gate 17777c478bd9Sstevel@tonic-gate /* 17787c478bd9Sstevel@tonic-gate * Load up the translation keeping it 17797c478bd9Sstevel@tonic-gate * locked and don't unlock the page. 17807c478bd9Sstevel@tonic-gate */ 17817c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, segspt_addr, 17827c478bd9Sstevel@tonic-gate size, ppa, sptd->spt_prot, 17837c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | HAT_LOAD_SHARE); 17847c478bd9Sstevel@tonic-gate } else { 17857c478bd9Sstevel@tonic-gate if (hat == seg->s_as->a_hat) { 17867c478bd9Sstevel@tonic-gate 17877c478bd9Sstevel@tonic-gate /* 17887c478bd9Sstevel@tonic-gate * Migrate pages marked for migration 17897c478bd9Sstevel@tonic-gate */ 17907c478bd9Sstevel@tonic-gate if (lgrp_optimizations()) 17917c478bd9Sstevel@tonic-gate page_migrate(seg, shm_addr, ppa, 17927c478bd9Sstevel@tonic-gate npages); 17937c478bd9Sstevel@tonic-gate 17947c478bd9Sstevel@tonic-gate /* CPU HAT */ 17957c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, 17967c478bd9Sstevel@tonic-gate segspt_addr, size, ppa, sptd->spt_prot, 17977c478bd9Sstevel@tonic-gate HAT_LOAD_SHARE); 17987c478bd9Sstevel@tonic-gate } else { 17997c478bd9Sstevel@tonic-gate /* XHAT. Pass real address */ 18007c478bd9Sstevel@tonic-gate hat_memload_array(hat, shm_addr, 18017c478bd9Sstevel@tonic-gate size, ppa, sptd->spt_prot, HAT_LOAD_SHARE); 18027c478bd9Sstevel@tonic-gate } 18037c478bd9Sstevel@tonic-gate 18047c478bd9Sstevel@tonic-gate /* 18057c478bd9Sstevel@tonic-gate * And now drop the SE_SHARED lock(s). 18067c478bd9Sstevel@tonic-gate */ 18077c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) 18087c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 18097c478bd9Sstevel@tonic-gate } 18107c478bd9Sstevel@tonic-gate 18117c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 18127c478bd9Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, shm_addr, 18137c478bd9Sstevel@tonic-gate curspt->a_hat, segspt_addr, ptob(npages), 18147c478bd9Sstevel@tonic-gate seg->s_szc) != 0) { 18157c478bd9Sstevel@tonic-gate panic("hat_share err in DISM fault"); 18167c478bd9Sstevel@tonic-gate /* NOTREACHED */ 18177c478bd9Sstevel@tonic-gate } 18187c478bd9Sstevel@tonic-gate } 18197c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 18207c478bd9Sstevel@tonic-gate dism_err: 18217c478bd9Sstevel@tonic-gate kmem_free(ppa, npages * sizeof (page_t *)); 18227c478bd9Sstevel@tonic-gate return (err); 18237c478bd9Sstevel@tonic-gate 18247c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 18257c478bd9Sstevel@tonic-gate 18267c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 18277c478bd9Sstevel@tonic-gate availrmem += npages; 18287c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 18297c478bd9Sstevel@tonic-gate 18307c478bd9Sstevel@tonic-gate /* 18317c478bd9Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer, 18327c478bd9Sstevel@tonic-gate * but the segspt_addr is the virtual address within the 18337c478bd9Sstevel@tonic-gate * dummy seg. 18347c478bd9Sstevel@tonic-gate */ 18357c478bd9Sstevel@tonic-gate segspt_softunlock(seg, segspt_addr, size, rw); 18367c478bd9Sstevel@tonic-gate return (0); 18377c478bd9Sstevel@tonic-gate 18387c478bd9Sstevel@tonic-gate case F_PROT: 18397c478bd9Sstevel@tonic-gate 18407c478bd9Sstevel@tonic-gate /* 18417c478bd9Sstevel@tonic-gate * This takes care of the unusual case where a user 18427c478bd9Sstevel@tonic-gate * allocates a stack in shared memory and a register 18437c478bd9Sstevel@tonic-gate * window overflow is written to that stack page before 18447c478bd9Sstevel@tonic-gate * it is otherwise modified. 18457c478bd9Sstevel@tonic-gate * 18467c478bd9Sstevel@tonic-gate * We can get away with this because ISM segments are 18477c478bd9Sstevel@tonic-gate * always rw. Other than this unusual case, there 18487c478bd9Sstevel@tonic-gate * should be no instances of protection violations. 18497c478bd9Sstevel@tonic-gate */ 18507c478bd9Sstevel@tonic-gate return (0); 18517c478bd9Sstevel@tonic-gate 18527c478bd9Sstevel@tonic-gate default: 18537c478bd9Sstevel@tonic-gate #ifdef DEBUG 18547c478bd9Sstevel@tonic-gate panic("segspt_dismfault default type?"); 18557c478bd9Sstevel@tonic-gate #else 18567c478bd9Sstevel@tonic-gate return (FC_NOMAP); 18577c478bd9Sstevel@tonic-gate #endif 18587c478bd9Sstevel@tonic-gate } 18597c478bd9Sstevel@tonic-gate } 18607c478bd9Sstevel@tonic-gate 18617c478bd9Sstevel@tonic-gate 18627c478bd9Sstevel@tonic-gate faultcode_t 18637c478bd9Sstevel@tonic-gate segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr, 18647c478bd9Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw) 18657c478bd9Sstevel@tonic-gate { 18667c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 18677c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 18687c478bd9Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas; 18697c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 18707c478bd9Sstevel@tonic-gate pgcnt_t npages; 18717c478bd9Sstevel@tonic-gate size_t share_size, size; 18727c478bd9Sstevel@tonic-gate caddr_t sptseg_addr, shm_addr; 18737c478bd9Sstevel@tonic-gate page_t *pp, **ppa; 18747c478bd9Sstevel@tonic-gate int i; 18757c478bd9Sstevel@tonic-gate u_offset_t offset; 18767c478bd9Sstevel@tonic-gate ulong_t anon_index = 0; 18777c478bd9Sstevel@tonic-gate struct vnode *vp; 18787c478bd9Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */ 18797c478bd9Sstevel@tonic-gate struct anon *ap = NULL; 18807c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 18817c478bd9Sstevel@tonic-gate 18827c478bd9Sstevel@tonic-gate #ifdef lint 18837c478bd9Sstevel@tonic-gate hat = hat; 18847c478bd9Sstevel@tonic-gate #endif 18857c478bd9Sstevel@tonic-gate 18867c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 18877c478bd9Sstevel@tonic-gate 18887c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 18897c478bd9Sstevel@tonic-gate return (segspt_dismfault(hat, seg, addr, len, type, rw)); 18907c478bd9Sstevel@tonic-gate } 18917c478bd9Sstevel@tonic-gate 18927c478bd9Sstevel@tonic-gate /* 18937c478bd9Sstevel@tonic-gate * Because of the way spt is implemented 18947c478bd9Sstevel@tonic-gate * the realsize of the segment does not have to be 18957c478bd9Sstevel@tonic-gate * equal to the segment size itself. The segment size is 18967c478bd9Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE. 18977c478bd9Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE 18987c478bd9Sstevel@tonic-gate * based on what the user requested. This is a bit of 18997c478bd9Sstevel@tonic-gate * ungliness that is historical but not easily fixed 19007c478bd9Sstevel@tonic-gate * without re-designing the higher levels of ISM. 19017c478bd9Sstevel@tonic-gate */ 19027c478bd9Sstevel@tonic-gate ASSERT(addr >= seg->s_base); 19037c478bd9Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize) 19047c478bd9Sstevel@tonic-gate return (FC_NOMAP); 19057c478bd9Sstevel@tonic-gate /* 19067c478bd9Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to 19077c478bd9Sstevel@tonic-gate * make any necessary adjustments to addr and len 19087c478bd9Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[]. 19097c478bd9Sstevel@tonic-gate * 19107c478bd9Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment 19117c478bd9Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore, 19127c478bd9Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 19137c478bd9Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 19147c478bd9Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT 19157c478bd9Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes 19167c478bd9Sstevel@tonic-gate * over a given virtual range. 19177c478bd9Sstevel@tonic-gate */ 19187c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(sptseg->s_szc); 19197c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 19207c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), share_size); 19217c478bd9Sstevel@tonic-gate npages = btopr(size); 19227c478bd9Sstevel@tonic-gate 19237c478bd9Sstevel@tonic-gate /* 19247c478bd9Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt. 19257c478bd9Sstevel@tonic-gate */ 19267c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr); 19277c478bd9Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index); 19287c478bd9Sstevel@tonic-gate 19297c478bd9Sstevel@tonic-gate /* 19307c478bd9Sstevel@tonic-gate * And now we may have to adjust npages downward if we have 19317c478bd9Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon 19327c478bd9Sstevel@tonic-gate * allocations. 19337c478bd9Sstevel@tonic-gate */ 19347c478bd9Sstevel@tonic-gate if ((sptseg_addr + ptob(npages)) > 19357c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)) 19367c478bd9Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr; 19377c478bd9Sstevel@tonic-gate 19387c478bd9Sstevel@tonic-gate npages = btopr(size); 19397c478bd9Sstevel@tonic-gate 19407c478bd9Sstevel@tonic-gate ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size)); 19417c478bd9Sstevel@tonic-gate ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0); 19427c478bd9Sstevel@tonic-gate 19437c478bd9Sstevel@tonic-gate switch (type) { 19447c478bd9Sstevel@tonic-gate 19457c478bd9Sstevel@tonic-gate case F_SOFTLOCK: 19467c478bd9Sstevel@tonic-gate 19477c478bd9Sstevel@tonic-gate /* 19487c478bd9Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust() 19497c478bd9Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is 19507c478bd9Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed. 19517c478bd9Sstevel@tonic-gate */ 19527c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 19537c478bd9Sstevel@tonic-gate /* 19547c478bd9Sstevel@tonic-gate * Some platforms assume that ISM pages are SE_SHARED 19557c478bd9Sstevel@tonic-gate * locked for the entire life of the segment. 19567c478bd9Sstevel@tonic-gate */ 19577c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) 19587c478bd9Sstevel@tonic-gate return (0); 19597c478bd9Sstevel@tonic-gate /* 19607c478bd9Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer 19617c478bd9Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag. 19627c478bd9Sstevel@tonic-gate */ 19637c478bd9Sstevel@tonic-gate 19647c478bd9Sstevel@tonic-gate /* FALLTHRU */ 19657c478bd9Sstevel@tonic-gate case F_INVAL: 19667c478bd9Sstevel@tonic-gate 19677c478bd9Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 19687c478bd9Sstevel@tonic-gate return (FC_NOMAP); 19697c478bd9Sstevel@tonic-gate 19707c478bd9Sstevel@tonic-gate /* 19717c478bd9Sstevel@tonic-gate * Some platforms that do NOT support DYNAMIC_ISM_UNMAP 19727c478bd9Sstevel@tonic-gate * may still rely on this call to hat_share(). That 19737c478bd9Sstevel@tonic-gate * would imply that those hat's can fault on a 19747c478bd9Sstevel@tonic-gate * HAT_LOAD_LOCK translation, which would seem 19757c478bd9Sstevel@tonic-gate * contradictory. 19767c478bd9Sstevel@tonic-gate */ 19777c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 19787c478bd9Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, seg->s_base, 19797c478bd9Sstevel@tonic-gate curspt->a_hat, sptseg->s_base, 19807c478bd9Sstevel@tonic-gate sptseg->s_size, sptseg->s_szc) != 0) { 19817c478bd9Sstevel@tonic-gate panic("hat_share error in ISM fault"); 19827c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 19837c478bd9Sstevel@tonic-gate } 19847c478bd9Sstevel@tonic-gate return (0); 19857c478bd9Sstevel@tonic-gate } 19867c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP); 19877c478bd9Sstevel@tonic-gate 19887c478bd9Sstevel@tonic-gate /* 19897c478bd9Sstevel@tonic-gate * I see no need to lock the real seg, 19907c478bd9Sstevel@tonic-gate * here, because all of our work will be on the underlying 19917c478bd9Sstevel@tonic-gate * dummy seg. 19927c478bd9Sstevel@tonic-gate * 19937c478bd9Sstevel@tonic-gate * sptseg_addr and npages now account for large pages. 19947c478bd9Sstevel@tonic-gate */ 19957c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 19967c478bd9Sstevel@tonic-gate ASSERT(amp != NULL); 19977c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr); 19987c478bd9Sstevel@tonic-gate 19997c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 20007c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) { 20017c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 20027c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++); 20037c478bd9Sstevel@tonic-gate ASSERT(ap != NULL); 20047c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &offset); 20057c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 20067c478bd9Sstevel@tonic-gate pp = page_lookup(vp, offset, SE_SHARED); 20077c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 20087c478bd9Sstevel@tonic-gate ppa[i] = pp; 20097c478bd9Sstevel@tonic-gate } 20107c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 20117c478bd9Sstevel@tonic-gate ASSERT(i == npages); 20127c478bd9Sstevel@tonic-gate 20137c478bd9Sstevel@tonic-gate /* 20147c478bd9Sstevel@tonic-gate * We are already holding the as->a_lock on the user's 20157c478bd9Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the 20167c478bd9Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the 20177c478bd9Sstevel@tonic-gate * underlying HAT layer. 20187c478bd9Sstevel@tonic-gate */ 20197c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 20207c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 20217c478bd9Sstevel@tonic-gate /* 20227c478bd9Sstevel@tonic-gate * Load up the translation keeping it 20237c478bd9Sstevel@tonic-gate * locked and don't unlock the page. 20247c478bd9Sstevel@tonic-gate */ 20257c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, sptseg_addr, 20267c478bd9Sstevel@tonic-gate ptob(npages), ppa, sptd->spt_prot, 20277c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | HAT_LOAD_SHARE); 20287c478bd9Sstevel@tonic-gate } else { 20297c478bd9Sstevel@tonic-gate if (hat == seg->s_as->a_hat) { 20307c478bd9Sstevel@tonic-gate 20317c478bd9Sstevel@tonic-gate /* 20327c478bd9Sstevel@tonic-gate * Migrate pages marked for migration. 20337c478bd9Sstevel@tonic-gate */ 20347c478bd9Sstevel@tonic-gate if (lgrp_optimizations()) 20357c478bd9Sstevel@tonic-gate page_migrate(seg, shm_addr, ppa, 20367c478bd9Sstevel@tonic-gate npages); 20377c478bd9Sstevel@tonic-gate 20387c478bd9Sstevel@tonic-gate /* CPU HAT */ 20397c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat, 20407c478bd9Sstevel@tonic-gate sptseg_addr, ptob(npages), ppa, 20417c478bd9Sstevel@tonic-gate sptd->spt_prot, HAT_LOAD_SHARE); 20427c478bd9Sstevel@tonic-gate } else { 20437c478bd9Sstevel@tonic-gate /* XHAT. Pass real address */ 20447c478bd9Sstevel@tonic-gate hat_memload_array(hat, shm_addr, 20457c478bd9Sstevel@tonic-gate ptob(npages), ppa, sptd->spt_prot, 20467c478bd9Sstevel@tonic-gate HAT_LOAD_SHARE); 20477c478bd9Sstevel@tonic-gate } 20487c478bd9Sstevel@tonic-gate 20497c478bd9Sstevel@tonic-gate /* 20507c478bd9Sstevel@tonic-gate * And now drop the SE_SHARED lock(s). 20517c478bd9Sstevel@tonic-gate */ 20527c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) 20537c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 20547c478bd9Sstevel@tonic-gate } 20557c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate kmem_free(ppa, sizeof (page_t *) * npages); 20587c478bd9Sstevel@tonic-gate return (0); 20597c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 20607c478bd9Sstevel@tonic-gate 20617c478bd9Sstevel@tonic-gate /* 20627c478bd9Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer, 20637c478bd9Sstevel@tonic-gate * but the sptseg_addr is the virtual address within the 20647c478bd9Sstevel@tonic-gate * dummy seg. 20657c478bd9Sstevel@tonic-gate */ 20667c478bd9Sstevel@tonic-gate segspt_softunlock(seg, sptseg_addr, ptob(npages), rw); 20677c478bd9Sstevel@tonic-gate return (0); 20687c478bd9Sstevel@tonic-gate 20697c478bd9Sstevel@tonic-gate case F_PROT: 20707c478bd9Sstevel@tonic-gate 20717c478bd9Sstevel@tonic-gate /* 20727c478bd9Sstevel@tonic-gate * This takes care of the unusual case where a user 20737c478bd9Sstevel@tonic-gate * allocates a stack in shared memory and a register 20747c478bd9Sstevel@tonic-gate * window overflow is written to that stack page before 20757c478bd9Sstevel@tonic-gate * it is otherwise modified. 20767c478bd9Sstevel@tonic-gate * 20777c478bd9Sstevel@tonic-gate * We can get away with this because ISM segments are 20787c478bd9Sstevel@tonic-gate * always rw. Other than this unusual case, there 20797c478bd9Sstevel@tonic-gate * should be no instances of protection violations. 20807c478bd9Sstevel@tonic-gate */ 20817c478bd9Sstevel@tonic-gate return (0); 20827c478bd9Sstevel@tonic-gate 20837c478bd9Sstevel@tonic-gate default: 20847c478bd9Sstevel@tonic-gate #ifdef DEBUG 20857c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "segspt_shmfault default type?"); 20867c478bd9Sstevel@tonic-gate #endif 20877c478bd9Sstevel@tonic-gate return (FC_NOMAP); 20887c478bd9Sstevel@tonic-gate } 20897c478bd9Sstevel@tonic-gate } 20907c478bd9Sstevel@tonic-gate 20917c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20927c478bd9Sstevel@tonic-gate static faultcode_t 20937c478bd9Sstevel@tonic-gate segspt_shmfaulta(struct seg *seg, caddr_t addr) 20947c478bd9Sstevel@tonic-gate { 20957c478bd9Sstevel@tonic-gate return (0); 20967c478bd9Sstevel@tonic-gate } 20977c478bd9Sstevel@tonic-gate 20987c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20997c478bd9Sstevel@tonic-gate static int 21007c478bd9Sstevel@tonic-gate segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta) 21017c478bd9Sstevel@tonic-gate { 21027c478bd9Sstevel@tonic-gate return (0); 21037c478bd9Sstevel@tonic-gate } 21047c478bd9Sstevel@tonic-gate 21057c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 21067c478bd9Sstevel@tonic-gate static size_t 21077c478bd9Sstevel@tonic-gate segspt_shmswapout(struct seg *seg) 21087c478bd9Sstevel@tonic-gate { 21097c478bd9Sstevel@tonic-gate return (0); 21107c478bd9Sstevel@tonic-gate } 21117c478bd9Sstevel@tonic-gate 21127c478bd9Sstevel@tonic-gate /* 21137c478bd9Sstevel@tonic-gate * duplicate the shared page tables 21147c478bd9Sstevel@tonic-gate */ 21157c478bd9Sstevel@tonic-gate int 21167c478bd9Sstevel@tonic-gate segspt_shmdup(struct seg *seg, struct seg *newseg) 21177c478bd9Sstevel@tonic-gate { 21187c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 21197c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 21207c478bd9Sstevel@tonic-gate struct shm_data *shmd_new; 21217c478bd9Sstevel@tonic-gate struct seg *spt_seg = shmd->shm_sptseg; 21227c478bd9Sstevel@tonic-gate struct spt_data *sptd = spt_seg->s_data; 21237c478bd9Sstevel@tonic-gate 21247c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP); 21277c478bd9Sstevel@tonic-gate newseg->s_data = (void *)shmd_new; 21287c478bd9Sstevel@tonic-gate shmd_new->shm_sptas = shmd->shm_sptas; 21297c478bd9Sstevel@tonic-gate shmd_new->shm_amp = amp; 21307c478bd9Sstevel@tonic-gate shmd_new->shm_sptseg = shmd->shm_sptseg; 21317c478bd9Sstevel@tonic-gate newseg->s_ops = &segspt_shmops; 21327c478bd9Sstevel@tonic-gate newseg->s_szc = seg->s_szc; 21337c478bd9Sstevel@tonic-gate ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc); 21347c478bd9Sstevel@tonic-gate 21357c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 21367c478bd9Sstevel@tonic-gate amp->refcnt++; 21377c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 21387c478bd9Sstevel@tonic-gate 21397c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) { 21407c478bd9Sstevel@tonic-gate shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP); 21417c478bd9Sstevel@tonic-gate shmd_new->shm_lckpgs = 0; 21427c478bd9Sstevel@tonic-gate } 21437c478bd9Sstevel@tonic-gate return (hat_share(newseg->s_as->a_hat, newseg->s_base, 21447c478bd9Sstevel@tonic-gate shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size, seg->s_szc)); 21457c478bd9Sstevel@tonic-gate } 21467c478bd9Sstevel@tonic-gate 21477c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 21487c478bd9Sstevel@tonic-gate int 21497c478bd9Sstevel@tonic-gate segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 21507c478bd9Sstevel@tonic-gate { 21517c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 21527c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 21537c478bd9Sstevel@tonic-gate 21547c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 21557c478bd9Sstevel@tonic-gate 21567c478bd9Sstevel@tonic-gate /* 21577c478bd9Sstevel@tonic-gate * ISM segment is always rw. 21587c478bd9Sstevel@tonic-gate */ 21597c478bd9Sstevel@tonic-gate return (((sptd->spt_prot & prot) != prot) ? EACCES : 0); 21607c478bd9Sstevel@tonic-gate } 21617c478bd9Sstevel@tonic-gate 21627c478bd9Sstevel@tonic-gate /* 21637c478bd9Sstevel@tonic-gate * Return an array of locked large pages, for empty slots allocate 21647c478bd9Sstevel@tonic-gate * private zero-filled anon pages. 21657c478bd9Sstevel@tonic-gate */ 21667c478bd9Sstevel@tonic-gate static int 21677c478bd9Sstevel@tonic-gate spt_anon_getpages( 21687c478bd9Sstevel@tonic-gate struct seg *sptseg, 21697c478bd9Sstevel@tonic-gate caddr_t sptaddr, 21707c478bd9Sstevel@tonic-gate size_t len, 21717c478bd9Sstevel@tonic-gate page_t *ppa[]) 21727c478bd9Sstevel@tonic-gate { 21737c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 21747c478bd9Sstevel@tonic-gate struct anon_map *amp = sptd->spt_amp; 21757c478bd9Sstevel@tonic-gate enum seg_rw rw = sptd->spt_prot; 21767c478bd9Sstevel@tonic-gate uint_t szc = sptseg->s_szc; 21777c478bd9Sstevel@tonic-gate size_t pg_sz, share_sz = page_get_pagesize(szc); 21787c478bd9Sstevel@tonic-gate pgcnt_t lp_npgs; 21797c478bd9Sstevel@tonic-gate caddr_t lp_addr, e_sptaddr; 21807c478bd9Sstevel@tonic-gate uint_t vpprot, ppa_szc = 0; 21817c478bd9Sstevel@tonic-gate struct vpage *vpage = NULL; 21827c478bd9Sstevel@tonic-gate ulong_t j, ppa_idx; 21837c478bd9Sstevel@tonic-gate int err, ierr = 0; 21847c478bd9Sstevel@tonic-gate pgcnt_t an_idx; 21857c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz)); 21887c478bd9Sstevel@tonic-gate ASSERT(len != 0); 21897c478bd9Sstevel@tonic-gate 21907c478bd9Sstevel@tonic-gate pg_sz = share_sz; 21917c478bd9Sstevel@tonic-gate lp_npgs = btop(pg_sz); 21927c478bd9Sstevel@tonic-gate lp_addr = sptaddr; 21937c478bd9Sstevel@tonic-gate e_sptaddr = sptaddr + len; 21947c478bd9Sstevel@tonic-gate an_idx = seg_page(sptseg, sptaddr); 21957c478bd9Sstevel@tonic-gate ppa_idx = 0; 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 21987c478bd9Sstevel@tonic-gate /*CONSTCOND*/ 21997c478bd9Sstevel@tonic-gate while (1) { 22007c478bd9Sstevel@tonic-gate for (; lp_addr < e_sptaddr; 22017c478bd9Sstevel@tonic-gate an_idx += lp_npgs, lp_addr += pg_sz, 22027c478bd9Sstevel@tonic-gate ppa_idx += lp_npgs) { 22037c478bd9Sstevel@tonic-gate 22047c478bd9Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 22057c478bd9Sstevel@tonic-gate ppa_szc = (uint_t)-1; 22067c478bd9Sstevel@tonic-gate ierr = anon_map_getpages(amp, an_idx, szc, sptseg, 22077c478bd9Sstevel@tonic-gate lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx], 22087c478bd9Sstevel@tonic-gate &ppa_szc, vpage, rw, 0, segvn_anypgsz, kcred); 22097c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 22107c478bd9Sstevel@tonic-gate 22117c478bd9Sstevel@tonic-gate if (ierr != 0) { 22127c478bd9Sstevel@tonic-gate if (ierr > 0) { 22137c478bd9Sstevel@tonic-gate err = FC_MAKE_ERR(ierr); 22147c478bd9Sstevel@tonic-gate goto lpgs_err; 22157c478bd9Sstevel@tonic-gate } 22167c478bd9Sstevel@tonic-gate break; 22177c478bd9Sstevel@tonic-gate } 22187c478bd9Sstevel@tonic-gate } 22197c478bd9Sstevel@tonic-gate if (lp_addr == e_sptaddr) { 22207c478bd9Sstevel@tonic-gate break; 22217c478bd9Sstevel@tonic-gate } 22227c478bd9Sstevel@tonic-gate ASSERT(lp_addr < e_sptaddr); 22237c478bd9Sstevel@tonic-gate 22247c478bd9Sstevel@tonic-gate /* 22257c478bd9Sstevel@tonic-gate * ierr == -1 means we failed to allocate a large page. 22267c478bd9Sstevel@tonic-gate * so do a size down operation. 22277c478bd9Sstevel@tonic-gate * 22287c478bd9Sstevel@tonic-gate * ierr == -2 means some other process that privately shares 22297c478bd9Sstevel@tonic-gate * pages with this process has allocated a larger page and we 22307c478bd9Sstevel@tonic-gate * need to retry with larger pages. So do a size up 22317c478bd9Sstevel@tonic-gate * operation. This relies on the fact that large pages are 22327c478bd9Sstevel@tonic-gate * never partially shared i.e. if we share any constituent 22337c478bd9Sstevel@tonic-gate * page of a large page with another process we must share the 22347c478bd9Sstevel@tonic-gate * entire large page. Note this cannot happen for SOFTLOCK 22357c478bd9Sstevel@tonic-gate * case, unless current address (lpaddr) is at the beginning 22367c478bd9Sstevel@tonic-gate * of the next page size boundary because the other process 22377c478bd9Sstevel@tonic-gate * couldn't have relocated locked pages. 22387c478bd9Sstevel@tonic-gate */ 22397c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || ierr == -2); 22407c478bd9Sstevel@tonic-gate if (segvn_anypgsz) { 22417c478bd9Sstevel@tonic-gate ASSERT(ierr == -2 || szc != 0); 22427c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || szc < sptseg->s_szc); 22437c478bd9Sstevel@tonic-gate szc = (ierr == -1) ? szc - 1 : szc + 1; 22447c478bd9Sstevel@tonic-gate } else { 22457c478bd9Sstevel@tonic-gate /* 22467c478bd9Sstevel@tonic-gate * For faults and segvn_anypgsz == 0 22477c478bd9Sstevel@tonic-gate * we need to be careful not to loop forever 22487c478bd9Sstevel@tonic-gate * if existing page is found with szc other 22497c478bd9Sstevel@tonic-gate * than 0 or seg->s_szc. This could be due 22507c478bd9Sstevel@tonic-gate * to page relocations on behalf of DR or 22517c478bd9Sstevel@tonic-gate * more likely large page creation. For this 22527c478bd9Sstevel@tonic-gate * case simply re-size to existing page's szc 22537c478bd9Sstevel@tonic-gate * if returned by anon_map_getpages(). 22547c478bd9Sstevel@tonic-gate */ 22557c478bd9Sstevel@tonic-gate if (ppa_szc == (uint_t)-1) { 22567c478bd9Sstevel@tonic-gate szc = (ierr == -1) ? 0 : sptseg->s_szc; 22577c478bd9Sstevel@tonic-gate } else { 22587c478bd9Sstevel@tonic-gate ASSERT(ppa_szc <= sptseg->s_szc); 22597c478bd9Sstevel@tonic-gate ASSERT(ierr == -2 || ppa_szc < szc); 22607c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || ppa_szc > szc); 22617c478bd9Sstevel@tonic-gate szc = ppa_szc; 22627c478bd9Sstevel@tonic-gate } 22637c478bd9Sstevel@tonic-gate } 22647c478bd9Sstevel@tonic-gate pg_sz = page_get_pagesize(szc); 22657c478bd9Sstevel@tonic-gate lp_npgs = btop(pg_sz); 22667c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(lp_addr, pg_sz)); 22677c478bd9Sstevel@tonic-gate } 22687c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 22697c478bd9Sstevel@tonic-gate return (0); 22707c478bd9Sstevel@tonic-gate 22717c478bd9Sstevel@tonic-gate lpgs_err: 22727c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 22737c478bd9Sstevel@tonic-gate for (j = 0; j < ppa_idx; j++) 22747c478bd9Sstevel@tonic-gate page_unlock(ppa[j]); 22757c478bd9Sstevel@tonic-gate return (err); 22767c478bd9Sstevel@tonic-gate } 22777c478bd9Sstevel@tonic-gate 22787c478bd9Sstevel@tonic-gate int 22797c478bd9Sstevel@tonic-gate spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 22807c478bd9Sstevel@tonic-gate page_t **ppa, ulong_t *lockmap, size_t pos) 22817c478bd9Sstevel@tonic-gate { 22827c478bd9Sstevel@tonic-gate struct shm_data *shmd = seg->s_data; 22837c478bd9Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data; 22847c478bd9Sstevel@tonic-gate ulong_t i; 22857c478bd9Sstevel@tonic-gate int kernel; 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate for (i = 0; i < npages; anon_index++, pos++, i++) { 22887c478bd9Sstevel@tonic-gate if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) { 22897c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[anon_index] < 22907c478bd9Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) { 22917c478bd9Sstevel@tonic-gate if (++sptd->spt_ppa_lckcnt[anon_index] == 22927c478bd9Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) { 22937c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 22947c478bd9Sstevel@tonic-gate "DISM page lock limit " 22957c478bd9Sstevel@tonic-gate "reached on DISM offset 0x%lx\n", 22967c478bd9Sstevel@tonic-gate anon_index << PAGESHIFT); 22977c478bd9Sstevel@tonic-gate } 22987c478bd9Sstevel@tonic-gate kernel = (sptd->spt_ppa && 22997c478bd9Sstevel@tonic-gate sptd->spt_ppa[anon_index]) ? 1 : 0; 23007c478bd9Sstevel@tonic-gate if (!page_pp_lock(ppa[i], 0, kernel)) { 23017c478bd9Sstevel@tonic-gate /* unlock rest of the pages */ 23027c478bd9Sstevel@tonic-gate for (; i < npages; i++) 23037c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 23047c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt[anon_index]--; 23057c478bd9Sstevel@tonic-gate return (EAGAIN); 23067c478bd9Sstevel@tonic-gate } 23077c478bd9Sstevel@tonic-gate shmd->shm_lckpgs++; 23087c478bd9Sstevel@tonic-gate shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED; 23097c478bd9Sstevel@tonic-gate if (lockmap != NULL) 23107c478bd9Sstevel@tonic-gate BT_SET(lockmap, pos); 23117c478bd9Sstevel@tonic-gate } 23127c478bd9Sstevel@tonic-gate } 23137c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 23147c478bd9Sstevel@tonic-gate } 23157c478bd9Sstevel@tonic-gate return (0); 23167c478bd9Sstevel@tonic-gate } 23177c478bd9Sstevel@tonic-gate 23187c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23197c478bd9Sstevel@tonic-gate static int 23207c478bd9Sstevel@tonic-gate segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 23217c478bd9Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos) 23227c478bd9Sstevel@tonic-gate { 23237c478bd9Sstevel@tonic-gate struct shm_data *shmd = seg->s_data; 23247c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 23257c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data; 23267c478bd9Sstevel@tonic-gate pgcnt_t npages, a_npages; 23277c478bd9Sstevel@tonic-gate page_t **ppa; 23287c478bd9Sstevel@tonic-gate pgcnt_t an_idx, a_an_idx, ppa_idx; 23297c478bd9Sstevel@tonic-gate caddr_t spt_addr, a_addr; /* spt and aligned address */ 23307c478bd9Sstevel@tonic-gate size_t a_len; /* aligned len */ 23317c478bd9Sstevel@tonic-gate size_t share_sz; 23327c478bd9Sstevel@tonic-gate ulong_t i; 23337c478bd9Sstevel@tonic-gate int sts = 0; 23347c478bd9Sstevel@tonic-gate 23357c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23367c478bd9Sstevel@tonic-gate 23377c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 23387c478bd9Sstevel@tonic-gate return (0); 23397c478bd9Sstevel@tonic-gate } 23407c478bd9Sstevel@tonic-gate 23417c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 23427c478bd9Sstevel@tonic-gate an_idx = seg_page(seg, addr); 23437c478bd9Sstevel@tonic-gate npages = btopr(len); 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate if (an_idx + npages > btopr(shmd->shm_amp->size)) { 23467c478bd9Sstevel@tonic-gate return (ENOMEM); 23477c478bd9Sstevel@tonic-gate } 23487c478bd9Sstevel@tonic-gate 23497c478bd9Sstevel@tonic-gate if (op == MC_LOCK) { 23507c478bd9Sstevel@tonic-gate /* 23517c478bd9Sstevel@tonic-gate * Need to align addr and size request if they are not 23527c478bd9Sstevel@tonic-gate * aligned so we can always allocate large page(s) however 23537c478bd9Sstevel@tonic-gate * we only lock what was requested in initial request. 23547c478bd9Sstevel@tonic-gate */ 23557c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(sptseg->s_szc); 23567c478bd9Sstevel@tonic-gate a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 23577c478bd9Sstevel@tonic-gate a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)), 23587c478bd9Sstevel@tonic-gate share_sz); 23597c478bd9Sstevel@tonic-gate a_npages = btop(a_len); 23607c478bd9Sstevel@tonic-gate a_an_idx = seg_page(seg, a_addr); 23617c478bd9Sstevel@tonic-gate spt_addr = sptseg->s_base + ptob(a_an_idx); 23627c478bd9Sstevel@tonic-gate ppa_idx = an_idx - a_an_idx; 23637c478bd9Sstevel@tonic-gate 23647c478bd9Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages), 23657c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) { 23667c478bd9Sstevel@tonic-gate return (ENOMEM); 23677c478bd9Sstevel@tonic-gate } 23687c478bd9Sstevel@tonic-gate 23697c478bd9Sstevel@tonic-gate /* 23707c478bd9Sstevel@tonic-gate * Don't cache any new pages for IO and 23717c478bd9Sstevel@tonic-gate * flush any cached pages. 23727c478bd9Sstevel@tonic-gate */ 23737c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 23747c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 23757c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 23767c478bd9Sstevel@tonic-gate 23777c478bd9Sstevel@tonic-gate sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa); 23787c478bd9Sstevel@tonic-gate if (sts != 0) { 23797c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 23807c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 23817c478bd9Sstevel@tonic-gate return (sts); 23827c478bd9Sstevel@tonic-gate } 23837c478bd9Sstevel@tonic-gate 23847c478bd9Sstevel@tonic-gate sts = spt_lockpages(seg, an_idx, npages, 23857c478bd9Sstevel@tonic-gate &ppa[ppa_idx], lockmap, pos); 23867c478bd9Sstevel@tonic-gate /* 23877c478bd9Sstevel@tonic-gate * unlock remaining pages for requests which are not 23887c478bd9Sstevel@tonic-gate * aligned or not in 4 M chunks 23897c478bd9Sstevel@tonic-gate */ 23907c478bd9Sstevel@tonic-gate for (i = 0; i < ppa_idx; i++) 23917c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 23927c478bd9Sstevel@tonic-gate for (i = ppa_idx + npages; i < a_npages; i++) 23937c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 23947c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 23957c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 23967c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 23977c478bd9Sstevel@tonic-gate 23987c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 23997c478bd9Sstevel@tonic-gate 24007c478bd9Sstevel@tonic-gate } else if (op == MC_UNLOCK) { /* unlock */ 24017c478bd9Sstevel@tonic-gate struct anon_map *amp; 24027c478bd9Sstevel@tonic-gate struct anon *ap; 24037c478bd9Sstevel@tonic-gate struct vnode *vp; 24047c478bd9Sstevel@tonic-gate u_offset_t off; 24057c478bd9Sstevel@tonic-gate struct page *pp; 24067c478bd9Sstevel@tonic-gate int kernel; 24077c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 24087c478bd9Sstevel@tonic-gate 24097c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 24107c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 24117c478bd9Sstevel@tonic-gate if (shmd->shm_lckpgs == 0) { 24127c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 24137c478bd9Sstevel@tonic-gate return (0); 24147c478bd9Sstevel@tonic-gate } 24157c478bd9Sstevel@tonic-gate /* 24167c478bd9Sstevel@tonic-gate * Don't cache new IO pages. 24177c478bd9Sstevel@tonic-gate */ 24187c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 24197c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 24207c478bd9Sstevel@tonic-gate 24217c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 24227c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++, an_idx++) { 24237c478bd9Sstevel@tonic-gate if (shmd->shm_vpage[an_idx] & DISM_PG_LOCKED) { 24247c478bd9Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 24257c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 24267c478bd9Sstevel@tonic-gate ASSERT(ap); 24277c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa_lckcnt[an_idx] > 0); 24287c478bd9Sstevel@tonic-gate 24297c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off); 24307c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 24317c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED); 24327c478bd9Sstevel@tonic-gate ASSERT(pp); 24337c478bd9Sstevel@tonic-gate /* 24347c478bd9Sstevel@tonic-gate * the availrmem is decremented only for 24357c478bd9Sstevel@tonic-gate * pages which are not in seg pcache, 24367c478bd9Sstevel@tonic-gate * for pages in seg pcache availrmem was 24377c478bd9Sstevel@tonic-gate * decremented in _dismpagelock() (if 24387c478bd9Sstevel@tonic-gate * they were not locked here) 24397c478bd9Sstevel@tonic-gate */ 24407c478bd9Sstevel@tonic-gate kernel = (sptd->spt_ppa && 24417c478bd9Sstevel@tonic-gate sptd->spt_ppa[an_idx]) ? 1 : 0; 24427c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, kernel); 24437c478bd9Sstevel@tonic-gate page_unlock(pp); 24447c478bd9Sstevel@tonic-gate shmd->shm_vpage[an_idx] &= ~DISM_PG_LOCKED; 24457c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt[an_idx]--; 24467c478bd9Sstevel@tonic-gate shmd->shm_lckpgs--; 24477c478bd9Sstevel@tonic-gate } 24487c478bd9Sstevel@tonic-gate } 24497c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 24507c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 24517c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 24527c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 24537c478bd9Sstevel@tonic-gate } 24547c478bd9Sstevel@tonic-gate return (sts); 24557c478bd9Sstevel@tonic-gate } 24567c478bd9Sstevel@tonic-gate 24577c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 24587c478bd9Sstevel@tonic-gate int 24597c478bd9Sstevel@tonic-gate segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 24607c478bd9Sstevel@tonic-gate { 24617c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 24627c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 24637c478bd9Sstevel@tonic-gate spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1; 24647c478bd9Sstevel@tonic-gate 24657c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 24667c478bd9Sstevel@tonic-gate 24677c478bd9Sstevel@tonic-gate /* 24687c478bd9Sstevel@tonic-gate * ISM segment is always rw. 24697c478bd9Sstevel@tonic-gate */ 24707c478bd9Sstevel@tonic-gate while (--pgno >= 0) 24717c478bd9Sstevel@tonic-gate *protv++ = sptd->spt_prot; 24727c478bd9Sstevel@tonic-gate return (0); 24737c478bd9Sstevel@tonic-gate } 24747c478bd9Sstevel@tonic-gate 24757c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 24767c478bd9Sstevel@tonic-gate u_offset_t 24777c478bd9Sstevel@tonic-gate segspt_shmgetoffset(struct seg *seg, caddr_t addr) 24787c478bd9Sstevel@tonic-gate { 24797c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 24807c478bd9Sstevel@tonic-gate 24817c478bd9Sstevel@tonic-gate /* Offset does not matter in ISM memory */ 24827c478bd9Sstevel@tonic-gate 24837c478bd9Sstevel@tonic-gate return ((u_offset_t)0); 24847c478bd9Sstevel@tonic-gate } 24857c478bd9Sstevel@tonic-gate 24867c478bd9Sstevel@tonic-gate /* ARGSUSED */ 24877c478bd9Sstevel@tonic-gate int 24887c478bd9Sstevel@tonic-gate segspt_shmgettype(struct seg *seg, caddr_t addr) 24897c478bd9Sstevel@tonic-gate { 24907c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 24917c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 24927c478bd9Sstevel@tonic-gate 24937c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 24947c478bd9Sstevel@tonic-gate 24957c478bd9Sstevel@tonic-gate /* 24967c478bd9Sstevel@tonic-gate * The shared memory mapping is always MAP_SHARED, SWAP is only 24977c478bd9Sstevel@tonic-gate * reserved for DISM 24987c478bd9Sstevel@tonic-gate */ 24997c478bd9Sstevel@tonic-gate return (MAP_SHARED | 25007c478bd9Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE)); 25017c478bd9Sstevel@tonic-gate } 25027c478bd9Sstevel@tonic-gate 25037c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 25047c478bd9Sstevel@tonic-gate int 25057c478bd9Sstevel@tonic-gate segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 25067c478bd9Sstevel@tonic-gate { 25077c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 25087c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 25097c478bd9Sstevel@tonic-gate 25107c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25117c478bd9Sstevel@tonic-gate 25127c478bd9Sstevel@tonic-gate *vpp = sptd->spt_vp; 25137c478bd9Sstevel@tonic-gate return (0); 25147c478bd9Sstevel@tonic-gate } 25157c478bd9Sstevel@tonic-gate 25167c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 25177c478bd9Sstevel@tonic-gate static int 25187c478bd9Sstevel@tonic-gate segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 25197c478bd9Sstevel@tonic-gate { 25207c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 25217c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 25227c478bd9Sstevel@tonic-gate struct anon_map *amp; 25237c478bd9Sstevel@tonic-gate pgcnt_t pg_idx; 25247c478bd9Sstevel@tonic-gate 25257c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate if (behav == MADV_FREE) { 25287c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) 25297c478bd9Sstevel@tonic-gate return (0); 25307c478bd9Sstevel@tonic-gate 25317c478bd9Sstevel@tonic-gate amp = sptd->spt_amp; 25327c478bd9Sstevel@tonic-gate pg_idx = seg_page(seg, addr); 25337c478bd9Sstevel@tonic-gate 25347c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 25357c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL) 25367c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED; 25377c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 25387c478bd9Sstevel@tonic-gate 25397c478bd9Sstevel@tonic-gate /* 25407c478bd9Sstevel@tonic-gate * Purge all DISM cached pages 25417c478bd9Sstevel@tonic-gate */ 25427c478bd9Sstevel@tonic-gate seg_ppurge_seg(segspt_reclaim); 25437c478bd9Sstevel@tonic-gate 25447c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock); 25457c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 25467c478bd9Sstevel@tonic-gate anon_disclaim(amp, pg_idx, len, ANON_PGLOOKUP_BLK); 25477c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 25487c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock); 25497c478bd9Sstevel@tonic-gate } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP || 25507c478bd9Sstevel@tonic-gate behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) { 25517c478bd9Sstevel@tonic-gate int already_set; 25527c478bd9Sstevel@tonic-gate ulong_t anon_index; 25537c478bd9Sstevel@tonic-gate lgrp_mem_policy_t policy; 25547c478bd9Sstevel@tonic-gate caddr_t shm_addr; 25557c478bd9Sstevel@tonic-gate size_t share_size; 25567c478bd9Sstevel@tonic-gate size_t size; 25577c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 25587c478bd9Sstevel@tonic-gate caddr_t sptseg_addr; 25597c478bd9Sstevel@tonic-gate 25607c478bd9Sstevel@tonic-gate /* 25617c478bd9Sstevel@tonic-gate * Align address and length to page size of underlying segment 25627c478bd9Sstevel@tonic-gate */ 25637c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(shmd->shm_sptseg->s_szc); 25647c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 25657c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), 25667c478bd9Sstevel@tonic-gate share_size); 25677c478bd9Sstevel@tonic-gate 25687c478bd9Sstevel@tonic-gate amp = shmd->shm_amp; 25697c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr); 25707c478bd9Sstevel@tonic-gate 25717c478bd9Sstevel@tonic-gate /* 25727c478bd9Sstevel@tonic-gate * And now we may have to adjust size downward if we have 25737c478bd9Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon 25747c478bd9Sstevel@tonic-gate * allocations. 25757c478bd9Sstevel@tonic-gate */ 25767c478bd9Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index); 25777c478bd9Sstevel@tonic-gate if ((sptseg_addr + size) > 25787c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize)) 25797c478bd9Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - 25807c478bd9Sstevel@tonic-gate sptseg_addr; 25817c478bd9Sstevel@tonic-gate 25827c478bd9Sstevel@tonic-gate /* 25837c478bd9Sstevel@tonic-gate * Set memory allocation policy for this segment 25847c478bd9Sstevel@tonic-gate */ 25857c478bd9Sstevel@tonic-gate policy = lgrp_madv_to_policy(behav, len, MAP_SHARED); 25867c478bd9Sstevel@tonic-gate already_set = lgrp_shm_policy_set(policy, amp, anon_index, 25877c478bd9Sstevel@tonic-gate NULL, 0, len); 25887c478bd9Sstevel@tonic-gate 25897c478bd9Sstevel@tonic-gate /* 25907c478bd9Sstevel@tonic-gate * If random memory allocation policy set already, 25917c478bd9Sstevel@tonic-gate * don't bother reapplying it. 25927c478bd9Sstevel@tonic-gate */ 25937c478bd9Sstevel@tonic-gate if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy)) 25947c478bd9Sstevel@tonic-gate return (0); 25957c478bd9Sstevel@tonic-gate 25967c478bd9Sstevel@tonic-gate /* 25977c478bd9Sstevel@tonic-gate * Mark any existing pages in the given range for 25987c478bd9Sstevel@tonic-gate * migration, flushing the I/O page cache, and using 25997c478bd9Sstevel@tonic-gate * underlying segment to calculate anon index and get 26007c478bd9Sstevel@tonic-gate * anonmap and vnode pointer from 26017c478bd9Sstevel@tonic-gate */ 26027c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) 26037c478bd9Sstevel@tonic-gate segspt_purge(seg); 26047c478bd9Sstevel@tonic-gate 26057c478bd9Sstevel@tonic-gate page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0); 26067c478bd9Sstevel@tonic-gate } 26077c478bd9Sstevel@tonic-gate 26087c478bd9Sstevel@tonic-gate return (0); 26097c478bd9Sstevel@tonic-gate } 26107c478bd9Sstevel@tonic-gate 26117c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 26127c478bd9Sstevel@tonic-gate void 26137c478bd9Sstevel@tonic-gate segspt_shmdump(struct seg *seg) 26147c478bd9Sstevel@tonic-gate { 26157c478bd9Sstevel@tonic-gate /* no-op for ISM segment */ 26167c478bd9Sstevel@tonic-gate } 26177c478bd9Sstevel@tonic-gate 26187c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 26197c478bd9Sstevel@tonic-gate static faultcode_t 26207c478bd9Sstevel@tonic-gate segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc) 26217c478bd9Sstevel@tonic-gate { 26227c478bd9Sstevel@tonic-gate return (ENOTSUP); 26237c478bd9Sstevel@tonic-gate } 26247c478bd9Sstevel@tonic-gate 26257c478bd9Sstevel@tonic-gate /* 26267c478bd9Sstevel@tonic-gate * get a memory ID for an addr in a given segment 26277c478bd9Sstevel@tonic-gate */ 26287c478bd9Sstevel@tonic-gate static int 26297c478bd9Sstevel@tonic-gate segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 26307c478bd9Sstevel@tonic-gate { 26317c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data; 26327c478bd9Sstevel@tonic-gate struct anon *ap; 26337c478bd9Sstevel@tonic-gate size_t anon_index; 26347c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp; 26357c478bd9Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data; 26367c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg; 26377c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 26387c478bd9Sstevel@tonic-gate 26397c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 26407c478bd9Sstevel@tonic-gate 26417c478bd9Sstevel@tonic-gate if (addr > (seg->s_base + sptd->spt_realsize)) { 26427c478bd9Sstevel@tonic-gate return (EFAULT); 26437c478bd9Sstevel@tonic-gate } 26447c478bd9Sstevel@tonic-gate 26457c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 26467c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie); 26477c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index); 26487c478bd9Sstevel@tonic-gate if (ap == NULL) { 26497c478bd9Sstevel@tonic-gate struct page *pp; 26507c478bd9Sstevel@tonic-gate caddr_t spt_addr = sptseg->s_base + ptob(anon_index); 26517c478bd9Sstevel@tonic-gate 26527c478bd9Sstevel@tonic-gate pp = anon_zero(sptseg, spt_addr, &ap, kcred); 26537c478bd9Sstevel@tonic-gate if (pp == NULL) { 26547c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 26557c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 26567c478bd9Sstevel@tonic-gate return (ENOMEM); 26577c478bd9Sstevel@tonic-gate } 26587c478bd9Sstevel@tonic-gate (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP); 26597c478bd9Sstevel@tonic-gate page_unlock(pp); 26607c478bd9Sstevel@tonic-gate } 26617c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 26627c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 26637c478bd9Sstevel@tonic-gate memidp->val[0] = (uintptr_t)ap; 26647c478bd9Sstevel@tonic-gate memidp->val[1] = (uintptr_t)addr & PAGEOFFSET; 26657c478bd9Sstevel@tonic-gate return (0); 26667c478bd9Sstevel@tonic-gate } 26677c478bd9Sstevel@tonic-gate 26687c478bd9Sstevel@tonic-gate /* 26697c478bd9Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment 26707c478bd9Sstevel@tonic-gate */ 26717c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 26727c478bd9Sstevel@tonic-gate segspt_shmgetpolicy(struct seg *seg, caddr_t addr) 26737c478bd9Sstevel@tonic-gate { 26747c478bd9Sstevel@tonic-gate struct anon_map *amp; 26757c478bd9Sstevel@tonic-gate ulong_t anon_index; 26767c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 26777c478bd9Sstevel@tonic-gate struct shm_data *shm_data; 26787c478bd9Sstevel@tonic-gate 26797c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 26807c478bd9Sstevel@tonic-gate 26817c478bd9Sstevel@tonic-gate /* 26827c478bd9Sstevel@tonic-gate * Get anon_map from segshm 26837c478bd9Sstevel@tonic-gate * 26847c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since 26857c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be 26867c478bd9Sstevel@tonic-gate * nonzero for an existing segment 26877c478bd9Sstevel@tonic-gate * Need to grab readers lock on policy tree though 26887c478bd9Sstevel@tonic-gate */ 26897c478bd9Sstevel@tonic-gate shm_data = (struct shm_data *)seg->s_data; 26907c478bd9Sstevel@tonic-gate if (shm_data == NULL) 26917c478bd9Sstevel@tonic-gate return (NULL); 26927c478bd9Sstevel@tonic-gate amp = shm_data->shm_amp; 26937c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 26947c478bd9Sstevel@tonic-gate 26957c478bd9Sstevel@tonic-gate /* 26967c478bd9Sstevel@tonic-gate * Get policy info 26977c478bd9Sstevel@tonic-gate * 26987c478bd9Sstevel@tonic-gate * Assume starting anon index of 0 26997c478bd9Sstevel@tonic-gate */ 27007c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr); 27017c478bd9Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 27027c478bd9Sstevel@tonic-gate 27037c478bd9Sstevel@tonic-gate return (policy_info); 27047c478bd9Sstevel@tonic-gate } 2705*1bd5c35fSelowe 2706*1bd5c35fSelowe /*ARGSUSED*/ 2707*1bd5c35fSelowe static int 2708*1bd5c35fSelowe segspt_shmcapable(struct seg *seg, segcapability_t capability) 2709*1bd5c35fSelowe { 2710*1bd5c35fSelowe return (0); 2711*1bd5c35fSelowe } 2712