17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5e67882ffSbs21162 * Common Development and Distribution License (the "License").
6e67882ffSbs21162 * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22b52a336eSPavel Tatashin * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23*6fca3924SJohn Levon * Copyright 2019 Joyent, Inc.
2448bbca81SDaniel Hoffman * Copyright (c) 2016 by Delphix. All rights reserved.
257c478bd9Sstevel@tonic-gate */
267c478bd9Sstevel@tonic-gate
277c478bd9Sstevel@tonic-gate #include <sys/param.h>
287c478bd9Sstevel@tonic-gate #include <sys/user.h>
297c478bd9Sstevel@tonic-gate #include <sys/mman.h>
307c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
317c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
337c478bd9Sstevel@tonic-gate #include <sys/systm.h>
347c478bd9Sstevel@tonic-gate #include <sys/tuneable.h>
357c478bd9Sstevel@tonic-gate #include <vm/hat.h>
367c478bd9Sstevel@tonic-gate #include <vm/seg.h>
377c478bd9Sstevel@tonic-gate #include <vm/as.h>
387c478bd9Sstevel@tonic-gate #include <vm/anon.h>
397c478bd9Sstevel@tonic-gate #include <vm/page.h>
407c478bd9Sstevel@tonic-gate #include <sys/buf.h>
417c478bd9Sstevel@tonic-gate #include <sys/swap.h>
427c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
437c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h>
447c478bd9Sstevel@tonic-gate #include <sys/debug.h>
457c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
467c478bd9Sstevel@tonic-gate #include <sys/shm.h>
47c6939658Ssl108498 #include <sys/shm_impl.h>
487c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
497c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
50c6939658Ssl108498 #include <sys/policy.h>
51c6939658Ssl108498 #include <sys/project.h>
52c6939658Ssl108498 #include <sys/zone.h>
537c478bd9Sstevel@tonic-gate
547c478bd9Sstevel@tonic-gate #define SEGSPTADDR (caddr_t)0x0
557c478bd9Sstevel@tonic-gate
567c478bd9Sstevel@tonic-gate /*
577c478bd9Sstevel@tonic-gate * # pages used for spt
587c478bd9Sstevel@tonic-gate */
59cee1d74bSjfrank size_t spt_used;
607c478bd9Sstevel@tonic-gate
617c478bd9Sstevel@tonic-gate /*
62*6fca3924SJohn Levon * See spt_setminfree().
637c478bd9Sstevel@tonic-gate */
647c478bd9Sstevel@tonic-gate pgcnt_t segspt_minfree = 0;
65*6fca3924SJohn Levon size_t segspt_minfree_clamp = (1UL << 30); /* 1GB in bytes */
667c478bd9Sstevel@tonic-gate
67284ce987SPatrick Mooney static int segspt_create(struct seg **segpp, void *argsp);
687c478bd9Sstevel@tonic-gate static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
697c478bd9Sstevel@tonic-gate static void segspt_free(struct seg *seg);
707c478bd9Sstevel@tonic-gate static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
717c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);
727c478bd9Sstevel@tonic-gate
73e09cef95SToomas Soome /* ARGSUSED */
74e09cef95SToomas Soome __NORETURN static int
segspt_badop_dup(struct seg * seg __unused,struct seg * newseg __unused)75e09cef95SToomas Soome segspt_badop_dup(struct seg *seg __unused, struct seg *newseg __unused)
767c478bd9Sstevel@tonic-gate {
77e09cef95SToomas Soome panic("%s called", __func__);
787c478bd9Sstevel@tonic-gate }
797c478bd9Sstevel@tonic-gate
80e09cef95SToomas Soome /* ARGSUSED */
81e09cef95SToomas Soome __NORETURN static faultcode_t
segspt_badop_fault(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw)82e09cef95SToomas Soome segspt_badop_fault(struct hat *hat, struct seg *seg, caddr_t addr,
83e09cef95SToomas Soome size_t len, enum fault_type type, enum seg_rw rw)
84e09cef95SToomas Soome {
85e09cef95SToomas Soome panic("%s called", __func__);
86e09cef95SToomas Soome }
87e09cef95SToomas Soome
88e09cef95SToomas Soome /* ARGSUSED */
89e09cef95SToomas Soome __NORETURN static faultcode_t
segspt_badop_faulta(struct seg * seg __unused,caddr_t addr __unused)90e09cef95SToomas Soome segspt_badop_faulta(struct seg *seg __unused, caddr_t addr __unused)
91e09cef95SToomas Soome {
92e09cef95SToomas Soome panic("%s called", __func__);
93e09cef95SToomas Soome }
94e09cef95SToomas Soome
95e09cef95SToomas Soome /* ARGSUSED */
96e09cef95SToomas Soome __NORETURN static int
segspt_badop_prot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)97e09cef95SToomas Soome segspt_badop_prot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
98e09cef95SToomas Soome {
99e09cef95SToomas Soome panic("%s called", __func__);
100e09cef95SToomas Soome }
101e09cef95SToomas Soome
102e09cef95SToomas Soome /* ARGSUSED */
103e09cef95SToomas Soome __NORETURN static int
segspt_badop_checkprot(struct seg * seg,caddr_t addr,size_t size,uint_t prot)104e09cef95SToomas Soome segspt_badop_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
105e09cef95SToomas Soome {
106e09cef95SToomas Soome panic("%s called", __func__);
107e09cef95SToomas Soome }
108e09cef95SToomas Soome
109e09cef95SToomas Soome /* ARGSUSED */
110e09cef95SToomas Soome __NORETURN static int
segspt_badop_kluster(struct seg * seg,caddr_t addr,ssize_t delta)111e09cef95SToomas Soome segspt_badop_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
112e09cef95SToomas Soome {
113e09cef95SToomas Soome panic("%s called", __func__);
114e09cef95SToomas Soome }
115e09cef95SToomas Soome
116e09cef95SToomas Soome /* ARGSUSED */
117e09cef95SToomas Soome __NORETURN static size_t
segspt_badop_swapout(struct seg * seg)118e09cef95SToomas Soome segspt_badop_swapout(struct seg *seg)
119e09cef95SToomas Soome {
120e09cef95SToomas Soome panic("%s called", __func__);
121e09cef95SToomas Soome }
122e09cef95SToomas Soome
123e09cef95SToomas Soome /* ARGSUSED */
124e09cef95SToomas Soome __NORETURN static int
segspt_badop_sync(struct seg * seg,caddr_t addr,size_t len,int attr,uint_t flags)125e09cef95SToomas Soome segspt_badop_sync(struct seg *seg, caddr_t addr, size_t len, int attr,
126e09cef95SToomas Soome uint_t flags)
127e09cef95SToomas Soome {
128e09cef95SToomas Soome panic("%s called", __func__);
129e09cef95SToomas Soome }
130e09cef95SToomas Soome
131e09cef95SToomas Soome /* ARGSUSED */
132e09cef95SToomas Soome __NORETURN
133e09cef95SToomas Soome static size_t
segspt_badop_incore(struct seg * seg,caddr_t addr,size_t len,char * vec)134e09cef95SToomas Soome segspt_badop_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
135e09cef95SToomas Soome {
136e09cef95SToomas Soome panic("%s called", __func__);
137e09cef95SToomas Soome }
138e09cef95SToomas Soome
139e09cef95SToomas Soome /* ARGSUSED */
140e09cef95SToomas Soome __NORETURN static int
segspt_badop_lockop(struct seg * seg,caddr_t addr,size_t len,int attr,int op,ulong_t * lockmap,size_t pos)141e09cef95SToomas Soome segspt_badop_lockop(struct seg *seg, caddr_t addr, size_t len, int attr,
142e09cef95SToomas Soome int op, ulong_t *lockmap, size_t pos)
143e09cef95SToomas Soome {
144e09cef95SToomas Soome panic("%s called", __func__);
145e09cef95SToomas Soome }
146e09cef95SToomas Soome
147e09cef95SToomas Soome /* ARGSUSED */
148e09cef95SToomas Soome __NORETURN static int
segspt_badop_getprot(struct seg * seg,caddr_t addr,size_t len,uint_t * protv)149e09cef95SToomas Soome segspt_badop_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
150e09cef95SToomas Soome {
151e09cef95SToomas Soome panic("%s called", __func__);
152e09cef95SToomas Soome }
153e09cef95SToomas Soome
154e09cef95SToomas Soome /* ARGSUSED */
155e09cef95SToomas Soome __NORETURN static u_offset_t
segspt_badop_getoffset(struct seg * seg,caddr_t addr)156e09cef95SToomas Soome segspt_badop_getoffset(struct seg *seg, caddr_t addr)
157e09cef95SToomas Soome {
158e09cef95SToomas Soome panic("%s called", __func__);
159e09cef95SToomas Soome }
160e09cef95SToomas Soome
161e09cef95SToomas Soome /* ARGSUSED */
162e09cef95SToomas Soome __NORETURN static int
segspt_badop_gettype(struct seg * seg,caddr_t addr)163e09cef95SToomas Soome segspt_badop_gettype(struct seg *seg, caddr_t addr)
164e09cef95SToomas Soome {
165e09cef95SToomas Soome panic("%s called", __func__);
166e09cef95SToomas Soome }
167e09cef95SToomas Soome
168e09cef95SToomas Soome /* ARGSUSED */
169e09cef95SToomas Soome __NORETURN static int
segspt_badop_getvp(struct seg * seg,caddr_t addr,struct vnode ** vpp)170e09cef95SToomas Soome segspt_badop_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
171e09cef95SToomas Soome {
172e09cef95SToomas Soome panic("%s called", __func__);
173e09cef95SToomas Soome }
174e09cef95SToomas Soome
175e09cef95SToomas Soome /* ARGSUSED */
176e09cef95SToomas Soome __NORETURN static int
segspt_badop_advise(struct seg * seg,caddr_t addr,size_t len,uint_t behav)177e09cef95SToomas Soome segspt_badop_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
178e09cef95SToomas Soome {
179e09cef95SToomas Soome panic("%s called", __func__);
180e09cef95SToomas Soome }
181e09cef95SToomas Soome
182e09cef95SToomas Soome /* ARGSUSED */
183e09cef95SToomas Soome __NORETURN static void
segspt_badop_dump(struct seg * seg)184e09cef95SToomas Soome segspt_badop_dump(struct seg *seg)
185e09cef95SToomas Soome {
186e09cef95SToomas Soome panic("%s called", __func__);
187e09cef95SToomas Soome }
188e09cef95SToomas Soome
189e09cef95SToomas Soome /* ARGSUSED */
190e09cef95SToomas Soome __NORETURN static int
segspt_badop_pagelock(struct seg * seg,caddr_t addr,size_t len,struct page *** ppp,enum lock_type type,enum seg_rw rw)191e09cef95SToomas Soome segspt_badop_pagelock(struct seg *seg, caddr_t addr, size_t len,
192e09cef95SToomas Soome struct page ***ppp, enum lock_type type, enum seg_rw rw)
193e09cef95SToomas Soome {
194e09cef95SToomas Soome panic("%s called", __func__);
195e09cef95SToomas Soome }
196e09cef95SToomas Soome
197e09cef95SToomas Soome /* ARGSUSED */
198e09cef95SToomas Soome __NORETURN static int
segspt_badop_setpgsz(struct seg * seg,caddr_t addr,size_t len,uint_t szc)199e09cef95SToomas Soome segspt_badop_setpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
200e09cef95SToomas Soome {
201e09cef95SToomas Soome panic("%s called", __func__);
202e09cef95SToomas Soome }
203e09cef95SToomas Soome
204e09cef95SToomas Soome /* ARGSUSED */
205e09cef95SToomas Soome __NORETURN static int
segspt_badop_getmemid(struct seg * seg,caddr_t addr,memid_t * memidp)206e09cef95SToomas Soome segspt_badop_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
207e09cef95SToomas Soome {
208e09cef95SToomas Soome panic("%s called", __func__);
209e09cef95SToomas Soome }
210e09cef95SToomas Soome
211e09cef95SToomas Soome /* ARGSUSED */
212e09cef95SToomas Soome __NORETURN static int
segspt_badop_capable(struct seg * seg,segcapability_t capability)213e09cef95SToomas Soome segspt_badop_capable(struct seg *seg, segcapability_t capability)
214e09cef95SToomas Soome {
215e09cef95SToomas Soome panic("%s called", __func__);
216e09cef95SToomas Soome }
2177c478bd9Sstevel@tonic-gate
2187c478bd9Sstevel@tonic-gate struct seg_ops segspt_ops = {
219e09cef95SToomas Soome segspt_badop_dup, /* dup */
2207c478bd9Sstevel@tonic-gate segspt_unmap,
2217c478bd9Sstevel@tonic-gate segspt_free,
222e09cef95SToomas Soome segspt_badop_fault, /* fault */
223e09cef95SToomas Soome segspt_badop_faulta, /* faulta */
224e09cef95SToomas Soome segspt_badop_prot, /* setprot */
225e09cef95SToomas Soome segspt_badop_checkprot, /* checkprot */
226e09cef95SToomas Soome segspt_badop_kluster, /* kluster */
227e09cef95SToomas Soome segspt_badop_swapout, /* swapout */
228e09cef95SToomas Soome segspt_badop_sync, /* sync */
229e09cef95SToomas Soome segspt_badop_incore, /* incore */
230e09cef95SToomas Soome segspt_badop_lockop, /* lockop */
231e09cef95SToomas Soome segspt_badop_getprot, /* getprot */
232e09cef95SToomas Soome segspt_badop_getoffset, /* getoffset */
233e09cef95SToomas Soome segspt_badop_gettype, /* gettype */
234e09cef95SToomas Soome segspt_badop_getvp, /* getvp */
235e09cef95SToomas Soome segspt_badop_advise, /* advise */
236e09cef95SToomas Soome segspt_badop_dump, /* dump */
237e09cef95SToomas Soome segspt_badop_pagelock, /* pagelock */
238e09cef95SToomas Soome segspt_badop_setpgsz, /* setpgsz */
239e09cef95SToomas Soome segspt_badop_getmemid, /* getmemid */
2407c478bd9Sstevel@tonic-gate segspt_getpolicy, /* getpolicy */
241e09cef95SToomas Soome segspt_badop_capable, /* capable */
2429d12795fSRobert Mustacchi seg_inherit_notsup /* inherit */
2437c478bd9Sstevel@tonic-gate };
2447c478bd9Sstevel@tonic-gate
2457c478bd9Sstevel@tonic-gate static int segspt_shmdup(struct seg *seg, struct seg *newseg);
2467c478bd9Sstevel@tonic-gate static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
2477c478bd9Sstevel@tonic-gate static void segspt_shmfree(struct seg *seg);
2487c478bd9Sstevel@tonic-gate static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
2497c478bd9Sstevel@tonic-gate caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
2507c478bd9Sstevel@tonic-gate static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
251e09cef95SToomas Soome static int segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len,
252e09cef95SToomas Soome uint_t prot);
2537c478bd9Sstevel@tonic-gate static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
2547c478bd9Sstevel@tonic-gate uint_t prot);
2557c478bd9Sstevel@tonic-gate static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
2567c478bd9Sstevel@tonic-gate static size_t segspt_shmswapout(struct seg *seg);
2577c478bd9Sstevel@tonic-gate static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
258e09cef95SToomas Soome char *vec);
259e09cef95SToomas Soome static int segspt_shmsync(struct seg *seg, caddr_t addr, size_t len,
2607c478bd9Sstevel@tonic-gate int attr, uint_t flags);
2617c478bd9Sstevel@tonic-gate static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
2627c478bd9Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos);
2637c478bd9Sstevel@tonic-gate static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
2647c478bd9Sstevel@tonic-gate uint_t *protv);
2657c478bd9Sstevel@tonic-gate static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
2667c478bd9Sstevel@tonic-gate static int segspt_shmgettype(struct seg *seg, caddr_t addr);
2677c478bd9Sstevel@tonic-gate static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
2687c478bd9Sstevel@tonic-gate static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
2697c478bd9Sstevel@tonic-gate uint_t behav);
2707c478bd9Sstevel@tonic-gate static void segspt_shmdump(struct seg *seg);
2717c478bd9Sstevel@tonic-gate static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
2727c478bd9Sstevel@tonic-gate struct page ***, enum lock_type, enum seg_rw);
2737c478bd9Sstevel@tonic-gate static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t);
2747c478bd9Sstevel@tonic-gate static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
2757c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
2761bd5c35fSelowe static int segspt_shmcapable(struct seg *, segcapability_t);
2777c478bd9Sstevel@tonic-gate
2787c478bd9Sstevel@tonic-gate struct seg_ops segspt_shmops = {
2797c478bd9Sstevel@tonic-gate segspt_shmdup,
2807c478bd9Sstevel@tonic-gate segspt_shmunmap,
2817c478bd9Sstevel@tonic-gate segspt_shmfree,
2827c478bd9Sstevel@tonic-gate segspt_shmfault,
2837c478bd9Sstevel@tonic-gate segspt_shmfaulta,
2847c478bd9Sstevel@tonic-gate segspt_shmsetprot,
2857c478bd9Sstevel@tonic-gate segspt_shmcheckprot,
2867c478bd9Sstevel@tonic-gate segspt_shmkluster,
2877c478bd9Sstevel@tonic-gate segspt_shmswapout,
2887c478bd9Sstevel@tonic-gate segspt_shmsync,
2897c478bd9Sstevel@tonic-gate segspt_shmincore,
2907c478bd9Sstevel@tonic-gate segspt_shmlockop,
2917c478bd9Sstevel@tonic-gate segspt_shmgetprot,
2927c478bd9Sstevel@tonic-gate segspt_shmgetoffset,
2937c478bd9Sstevel@tonic-gate segspt_shmgettype,
2947c478bd9Sstevel@tonic-gate segspt_shmgetvp,
2957c478bd9Sstevel@tonic-gate segspt_shmadvise, /* advise */
2967c478bd9Sstevel@tonic-gate segspt_shmdump,
2977c478bd9Sstevel@tonic-gate segspt_shmpagelock,
2987c478bd9Sstevel@tonic-gate segspt_shmsetpgsz,
2997c478bd9Sstevel@tonic-gate segspt_shmgetmemid,
3007c478bd9Sstevel@tonic-gate segspt_shmgetpolicy,
3011bd5c35fSelowe segspt_shmcapable,
3029d12795fSRobert Mustacchi seg_inherit_notsup
3037c478bd9Sstevel@tonic-gate };
3047c478bd9Sstevel@tonic-gate
3057c478bd9Sstevel@tonic-gate static void segspt_purge(struct seg *seg);
306a98e9dbfSaguzovsk static int segspt_reclaim(void *, caddr_t, size_t, struct page **,
307a98e9dbfSaguzovsk enum seg_rw, int);
3087c478bd9Sstevel@tonic-gate static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
3097c478bd9Sstevel@tonic-gate page_t **ppa);
3107c478bd9Sstevel@tonic-gate
311*6fca3924SJohn Levon /*
312*6fca3924SJohn Levon * This value corresponds to headroom in availrmem that ISM can never allocate
313*6fca3924SJohn Levon * (but others can). The original intent here was to prevent ISM from locking
314*6fca3924SJohn Levon * all of the remaining availrmem into memory, making forward progress
315*6fca3924SJohn Levon * difficult. It's not clear how much this matters on modern systems.
316*6fca3924SJohn Levon *
317*6fca3924SJohn Levon * The traditional default value of 5% of total memory is used, except on
318*6fca3924SJohn Levon * systems where that quickly gets ridiculous: in that case we clamp at a rather
319*6fca3924SJohn Levon * arbitrary value of 1GB.
320*6fca3924SJohn Levon *
321*6fca3924SJohn Levon * Note that since this is called lazily on the first sptcreate(), in theory,
322*6fca3924SJohn Levon * this could represent a very small value if the system is heavily loaded
323*6fca3924SJohn Levon * already. In practice, the first ISM user is pretty likely to come along
324*6fca3924SJohn Levon * earlier during the system's operation.
325*6fca3924SJohn Levon *
326*6fca3924SJohn Levon * This never gets re-figured.
327*6fca3924SJohn Levon */
328*6fca3924SJohn Levon static void
spt_setminfree(void)329*6fca3924SJohn Levon spt_setminfree(void)
330*6fca3924SJohn Levon {
331*6fca3924SJohn Levon segspt_minfree = availrmem / 20;
3327c478bd9Sstevel@tonic-gate
333*6fca3924SJohn Levon if (segspt_minfree_clamp != 0 &&
334*6fca3924SJohn Levon segspt_minfree > (segspt_minfree_clamp / PAGESIZE))
335*6fca3924SJohn Levon segspt_minfree = segspt_minfree_clamp / PAGESIZE;
336*6fca3924SJohn Levon }
3377c478bd9Sstevel@tonic-gate
3387c478bd9Sstevel@tonic-gate int
sptcreate(size_t size,struct seg ** sptseg,struct anon_map * amp,uint_t prot,uint_t flags,uint_t share_szc)3397c478bd9Sstevel@tonic-gate sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
3407c478bd9Sstevel@tonic-gate uint_t prot, uint_t flags, uint_t share_szc)
3417c478bd9Sstevel@tonic-gate {
3427c478bd9Sstevel@tonic-gate int err;
3437c478bd9Sstevel@tonic-gate struct as *newas;
3447c478bd9Sstevel@tonic-gate struct segspt_crargs sptcargs;
3457c478bd9Sstevel@tonic-gate
346*6fca3924SJohn Levon if (segspt_minfree == 0)
347*6fca3924SJohn Levon spt_setminfree();
3487c478bd9Sstevel@tonic-gate
3497c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_SHARED_PT, (void *)0))
3507c478bd9Sstevel@tonic-gate return (EINVAL);
3517c478bd9Sstevel@tonic-gate
3527c478bd9Sstevel@tonic-gate /*
3537c478bd9Sstevel@tonic-gate * get a new as for this shared memory segment
3547c478bd9Sstevel@tonic-gate */
3557c478bd9Sstevel@tonic-gate newas = as_alloc();
356c6939658Ssl108498 newas->a_proc = NULL;
3577c478bd9Sstevel@tonic-gate sptcargs.amp = amp;
3587c478bd9Sstevel@tonic-gate sptcargs.prot = prot;
3597c478bd9Sstevel@tonic-gate sptcargs.flags = flags;
3607c478bd9Sstevel@tonic-gate sptcargs.szc = share_szc;
3617c478bd9Sstevel@tonic-gate /*
3627c478bd9Sstevel@tonic-gate * create a shared page table (spt) segment
3637c478bd9Sstevel@tonic-gate */
3647c478bd9Sstevel@tonic-gate
3657c478bd9Sstevel@tonic-gate if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
3667c478bd9Sstevel@tonic-gate as_free(newas);
3677c478bd9Sstevel@tonic-gate return (err);
3687c478bd9Sstevel@tonic-gate }
3697c478bd9Sstevel@tonic-gate *sptseg = sptcargs.seg_spt;
3707c478bd9Sstevel@tonic-gate return (0);
3717c478bd9Sstevel@tonic-gate }
3727c478bd9Sstevel@tonic-gate
3737c478bd9Sstevel@tonic-gate void
sptdestroy(struct as * as,struct anon_map * amp)3747c478bd9Sstevel@tonic-gate sptdestroy(struct as *as, struct anon_map *amp)
3757c478bd9Sstevel@tonic-gate {
3767c478bd9Sstevel@tonic-gate
3777c478bd9Sstevel@tonic-gate (void) as_unmap(as, SEGSPTADDR, amp->size);
3787c478bd9Sstevel@tonic-gate as_free(as);
3797c478bd9Sstevel@tonic-gate }
3807c478bd9Sstevel@tonic-gate
3817c478bd9Sstevel@tonic-gate /*
3827c478bd9Sstevel@tonic-gate * called from seg_free().
3837c478bd9Sstevel@tonic-gate * free (i.e., unlock, unmap, return to free list)
3847c478bd9Sstevel@tonic-gate * all the pages in the given seg.
3857c478bd9Sstevel@tonic-gate */
3867c478bd9Sstevel@tonic-gate void
segspt_free(struct seg * seg)3877c478bd9Sstevel@tonic-gate segspt_free(struct seg *seg)
3887c478bd9Sstevel@tonic-gate {
3897c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data;
3907c478bd9Sstevel@tonic-gate
391dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
3927c478bd9Sstevel@tonic-gate
3937c478bd9Sstevel@tonic-gate if (sptd != NULL) {
3947c478bd9Sstevel@tonic-gate if (sptd->spt_realsize)
3957c478bd9Sstevel@tonic-gate segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);
3967c478bd9Sstevel@tonic-gate
397c99fb8b3SToomas Soome if (sptd->spt_ppa_lckcnt) {
3987c478bd9Sstevel@tonic-gate kmem_free(sptd->spt_ppa_lckcnt,
3997c478bd9Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt)
4007c478bd9Sstevel@tonic-gate * btopr(sptd->spt_amp->size));
401c99fb8b3SToomas Soome }
4027c478bd9Sstevel@tonic-gate kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
4032ba723d8Smec cv_destroy(&sptd->spt_cv);
4047c478bd9Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock);
4057c478bd9Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd));
4067c478bd9Sstevel@tonic-gate }
4077c478bd9Sstevel@tonic-gate }
4087c478bd9Sstevel@tonic-gate
4097c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4107c478bd9Sstevel@tonic-gate static int
segspt_shmsync(struct seg * seg,caddr_t addr,size_t len,int attr,uint_t flags)4117c478bd9Sstevel@tonic-gate segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
4127c478bd9Sstevel@tonic-gate uint_t flags)
4137c478bd9Sstevel@tonic-gate {
414dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
4157c478bd9Sstevel@tonic-gate
4167c478bd9Sstevel@tonic-gate return (0);
4177c478bd9Sstevel@tonic-gate }
4187c478bd9Sstevel@tonic-gate
4197c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4207c478bd9Sstevel@tonic-gate static size_t
segspt_shmincore(struct seg * seg,caddr_t addr,size_t len,char * vec)4217c478bd9Sstevel@tonic-gate segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
4227c478bd9Sstevel@tonic-gate {
4237c478bd9Sstevel@tonic-gate caddr_t eo_seg;
4247c478bd9Sstevel@tonic-gate pgcnt_t npages;
4257c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
4267c478bd9Sstevel@tonic-gate struct seg *sptseg;
4277c478bd9Sstevel@tonic-gate struct spt_data *sptd;
4287c478bd9Sstevel@tonic-gate
429dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
4307c478bd9Sstevel@tonic-gate #ifdef lint
4317c478bd9Sstevel@tonic-gate seg = seg;
4327c478bd9Sstevel@tonic-gate #endif
4337c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg;
4347c478bd9Sstevel@tonic-gate sptd = sptseg->s_data;
4357c478bd9Sstevel@tonic-gate
4367c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
4377c478bd9Sstevel@tonic-gate eo_seg = addr + len;
4387c478bd9Sstevel@tonic-gate while (addr < eo_seg) {
4397c478bd9Sstevel@tonic-gate /* page exists, and it's locked. */
4407c478bd9Sstevel@tonic-gate *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
4417c478bd9Sstevel@tonic-gate SEG_PAGE_ANON;
4427c478bd9Sstevel@tonic-gate addr += PAGESIZE;
4437c478bd9Sstevel@tonic-gate }
4447c478bd9Sstevel@tonic-gate return (len);
4457c478bd9Sstevel@tonic-gate } else {
4467c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp;
4477c478bd9Sstevel@tonic-gate struct anon *ap;
4487c478bd9Sstevel@tonic-gate page_t *pp;
4497c478bd9Sstevel@tonic-gate pgcnt_t anon_index;
4507c478bd9Sstevel@tonic-gate struct vnode *vp;
4517c478bd9Sstevel@tonic-gate u_offset_t off;
4527c478bd9Sstevel@tonic-gate ulong_t i;
4537c478bd9Sstevel@tonic-gate int ret;
4547c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie;
4557c478bd9Sstevel@tonic-gate
4567c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
4577c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr);
4587c478bd9Sstevel@tonic-gate npages = btopr(len);
4597c478bd9Sstevel@tonic-gate if (anon_index + npages > btopr(shmd->shm_amp->size)) {
4607c478bd9Sstevel@tonic-gate return (EINVAL);
4617c478bd9Sstevel@tonic-gate }
4627c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
4637c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++, anon_index++) {
4647c478bd9Sstevel@tonic-gate ret = 0;
4657c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie);
4667c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index);
4677c478bd9Sstevel@tonic-gate if (ap != NULL) {
4687c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off);
4697c478bd9Sstevel@tonic-gate anon_array_exit(&cookie);
4707c478bd9Sstevel@tonic-gate pp = page_lookup_nowait(vp, off, SE_SHARED);
4717c478bd9Sstevel@tonic-gate if (pp != NULL) {
4727c478bd9Sstevel@tonic-gate ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
4737c478bd9Sstevel@tonic-gate page_unlock(pp);
4747c478bd9Sstevel@tonic-gate }
4757c478bd9Sstevel@tonic-gate } else {
4767c478bd9Sstevel@tonic-gate anon_array_exit(&cookie);
4777c478bd9Sstevel@tonic-gate }
4787c478bd9Sstevel@tonic-gate if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
4797c478bd9Sstevel@tonic-gate ret |= SEG_PAGE_LOCKED;
4807c478bd9Sstevel@tonic-gate }
4817c478bd9Sstevel@tonic-gate *vec++ = (char)ret;
4827c478bd9Sstevel@tonic-gate }
4837c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
4847c478bd9Sstevel@tonic-gate return (len);
4857c478bd9Sstevel@tonic-gate }
4867c478bd9Sstevel@tonic-gate }
4877c478bd9Sstevel@tonic-gate
4887c478bd9Sstevel@tonic-gate static int
segspt_unmap(struct seg * seg,caddr_t raddr,size_t ssize)4897c478bd9Sstevel@tonic-gate segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
4907c478bd9Sstevel@tonic-gate {
4917c478bd9Sstevel@tonic-gate size_t share_size;
4927c478bd9Sstevel@tonic-gate
493dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
4947c478bd9Sstevel@tonic-gate
4957c478bd9Sstevel@tonic-gate /*
4967c478bd9Sstevel@tonic-gate * seg.s_size may have been rounded up to the largest page size
4977c478bd9Sstevel@tonic-gate * in shmat().
4987c478bd9Sstevel@tonic-gate * XXX This should be cleanedup. sptdestroy should take a length
4997c478bd9Sstevel@tonic-gate * argument which should be the same as sptcreate. Then
5007c478bd9Sstevel@tonic-gate * this rounding would not be needed (or is done in shm.c)
5017c478bd9Sstevel@tonic-gate * Only the check for full segment will be needed.
5027c478bd9Sstevel@tonic-gate *
5037c478bd9Sstevel@tonic-gate * XXX -- shouldn't raddr == 0 always? These tests don't seem
5047c478bd9Sstevel@tonic-gate * to be useful at all.
5057c478bd9Sstevel@tonic-gate */
5067c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(seg->s_szc);
5077c478bd9Sstevel@tonic-gate ssize = P2ROUNDUP(ssize, share_size);
5087c478bd9Sstevel@tonic-gate
5097c478bd9Sstevel@tonic-gate if (raddr == seg->s_base && ssize == seg->s_size) {
5107c478bd9Sstevel@tonic-gate seg_free(seg);
5117c478bd9Sstevel@tonic-gate return (0);
5127c478bd9Sstevel@tonic-gate } else
5137c478bd9Sstevel@tonic-gate return (EINVAL);
5147c478bd9Sstevel@tonic-gate }
5157c478bd9Sstevel@tonic-gate
5167c478bd9Sstevel@tonic-gate int
segspt_create(struct seg ** segpp,void * argsp)517284ce987SPatrick Mooney segspt_create(struct seg **segpp, void *argsp)
5187c478bd9Sstevel@tonic-gate {
519284ce987SPatrick Mooney struct seg *seg = *segpp;
5207c478bd9Sstevel@tonic-gate int err;
5217c478bd9Sstevel@tonic-gate caddr_t addr = seg->s_base;
5227c478bd9Sstevel@tonic-gate struct spt_data *sptd;
5237c478bd9Sstevel@tonic-gate struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
5247c478bd9Sstevel@tonic-gate struct anon_map *amp = sptcargs->amp;
525c6939658Ssl108498 struct kshmid *sp = amp->a_sp;
5267c478bd9Sstevel@tonic-gate struct cred *cred = CRED();
5277c478bd9Sstevel@tonic-gate ulong_t i, j, anon_index = 0;
5287c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(amp->size);
5297c478bd9Sstevel@tonic-gate struct vnode *vp;
5307c478bd9Sstevel@tonic-gate page_t **ppa;
5317c478bd9Sstevel@tonic-gate uint_t hat_flags;
53207b65a64Saguzovsk size_t pgsz;
53307b65a64Saguzovsk pgcnt_t pgcnt;
53407b65a64Saguzovsk caddr_t a;
53507b65a64Saguzovsk pgcnt_t pidx;
53607b65a64Saguzovsk size_t sz;
537c6939658Ssl108498 proc_t *procp = curproc;
538c6939658Ssl108498 rctl_qty_t lockedbytes = 0;
539c6939658Ssl108498 kproject_t *proj;
5407c478bd9Sstevel@tonic-gate
5417c478bd9Sstevel@tonic-gate /*
5427c478bd9Sstevel@tonic-gate * We are holding the a_lock on the underlying dummy as,
5437c478bd9Sstevel@tonic-gate * so we can make calls to the HAT layer.
5447c478bd9Sstevel@tonic-gate */
545dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
546c6939658Ssl108498 ASSERT(sp != NULL);
5477c478bd9Sstevel@tonic-gate
5487c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
549e0cb4e8dSOndrej Kubecka if (err = anon_swap_adjust(npages))
5507c478bd9Sstevel@tonic-gate return (err);
5517c478bd9Sstevel@tonic-gate }
5527c478bd9Sstevel@tonic-gate err = ENOMEM;
5537c478bd9Sstevel@tonic-gate
5547c478bd9Sstevel@tonic-gate if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
5557c478bd9Sstevel@tonic-gate goto out1;
5567c478bd9Sstevel@tonic-gate
557c6f039c7SToomas Soome ppa = NULL;
5587c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
5597c478bd9Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
5607c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL)
5617c478bd9Sstevel@tonic-gate goto out2;
5627c478bd9Sstevel@tonic-gate }
5637c478bd9Sstevel@tonic-gate
5647c478bd9Sstevel@tonic-gate mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);
5657c478bd9Sstevel@tonic-gate
5667c478bd9Sstevel@tonic-gate if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
5677c478bd9Sstevel@tonic-gate goto out3;
5687c478bd9Sstevel@tonic-gate
5697c478bd9Sstevel@tonic-gate seg->s_ops = &segspt_ops;
5707c478bd9Sstevel@tonic-gate sptd->spt_vp = vp;
5717c478bd9Sstevel@tonic-gate sptd->spt_amp = amp;
5727c478bd9Sstevel@tonic-gate sptd->spt_prot = sptcargs->prot;
5737c478bd9Sstevel@tonic-gate sptd->spt_flags = sptcargs->flags;
5747c478bd9Sstevel@tonic-gate seg->s_data = (caddr_t)sptd;
5757c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL;
5767c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt = NULL;
5777c478bd9Sstevel@tonic-gate seg->s_szc = sptcargs->szc;
5782ba723d8Smec cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL);
5792ba723d8Smec sptd->spt_gen = 0;
5807c478bd9Sstevel@tonic-gate
5817c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
58207b65a64Saguzovsk if (seg->s_szc > amp->a_szc) {
5837c478bd9Sstevel@tonic-gate amp->a_szc = seg->s_szc;
58407b65a64Saguzovsk }
5857c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
5867c478bd9Sstevel@tonic-gate
5877c478bd9Sstevel@tonic-gate /*
5887c478bd9Sstevel@tonic-gate * Set policy to affect initial allocation of pages in
5897c478bd9Sstevel@tonic-gate * anon_map_createpages()
5907c478bd9Sstevel@tonic-gate */
5917c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
5927c478bd9Sstevel@tonic-gate NULL, 0, ptob(npages));
5937c478bd9Sstevel@tonic-gate
5947c478bd9Sstevel@tonic-gate if (sptcargs->flags & SHM_PAGEABLE) {
5957c478bd9Sstevel@tonic-gate size_t share_sz;
5967c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, more_pgs;
5977c478bd9Sstevel@tonic-gate struct anon_hdr *nahp;
59868803f2dSsl108498 zone_t *zone;
5997c478bd9Sstevel@tonic-gate
6007c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(seg->s_szc);
6017c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(amp->size, share_sz)) {
6027c478bd9Sstevel@tonic-gate /*
6037c478bd9Sstevel@tonic-gate * We are rounding up the size of the anon array
6047c478bd9Sstevel@tonic-gate * on 4 M boundary because we always create 4 M
6057c478bd9Sstevel@tonic-gate * of page(s) when locking, faulting pages and we
6067c478bd9Sstevel@tonic-gate * don't have to check for all corner cases e.g.
6077c478bd9Sstevel@tonic-gate * if there is enough space to allocate 4 M
6087c478bd9Sstevel@tonic-gate * page.
6097c478bd9Sstevel@tonic-gate */
6107c478bd9Sstevel@tonic-gate new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
6117c478bd9Sstevel@tonic-gate more_pgs = new_npgs - npages;
6127c478bd9Sstevel@tonic-gate
61368803f2dSsl108498 /*
61467253d2cSsl108498 * The zone will never be NULL, as a fully created
61567253d2cSsl108498 * shm always has an owning zone.
61668803f2dSsl108498 */
617a19609f8Sjv227347 zone = sp->shm_perm.ipc_zone_ref.zref_zone;
61867253d2cSsl108498 ASSERT(zone != NULL);
61968803f2dSsl108498 if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
6207c478bd9Sstevel@tonic-gate err = ENOMEM;
6217c478bd9Sstevel@tonic-gate goto out4;
6227c478bd9Sstevel@tonic-gate }
62368803f2dSsl108498
6247c478bd9Sstevel@tonic-gate nahp = anon_create(new_npgs, ANON_SLEEP);
6257c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
6267c478bd9Sstevel@tonic-gate (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
6277c478bd9Sstevel@tonic-gate ANON_SLEEP);
6287c478bd9Sstevel@tonic-gate anon_release(amp->ahp, npages);
6297c478bd9Sstevel@tonic-gate amp->ahp = nahp;
63068803f2dSsl108498 ASSERT(amp->swresv == ptob(npages));
6317c478bd9Sstevel@tonic-gate amp->swresv = amp->size = ptob(new_npgs);
6327c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
6337c478bd9Sstevel@tonic-gate npages = new_npgs;
6347c478bd9Sstevel@tonic-gate }
6357c478bd9Sstevel@tonic-gate
6367c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
6377c478bd9Sstevel@tonic-gate sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
6387c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt = 0;
6397c478bd9Sstevel@tonic-gate sptd->spt_realsize = ptob(npages);
6407c478bd9Sstevel@tonic-gate sptcargs->seg_spt = seg;
6417c478bd9Sstevel@tonic-gate return (0);
6427c478bd9Sstevel@tonic-gate }
6437c478bd9Sstevel@tonic-gate
6447c478bd9Sstevel@tonic-gate /*
6457c478bd9Sstevel@tonic-gate * get array of pages for each anon slot in amp
6467c478bd9Sstevel@tonic-gate */
6477c478bd9Sstevel@tonic-gate if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
6487c478bd9Sstevel@tonic-gate seg, addr, S_CREATE, cred)) != 0)
6497c478bd9Sstevel@tonic-gate goto out4;
6507c478bd9Sstevel@tonic-gate
651c6939658Ssl108498 mutex_enter(&sp->shm_mlock);
652c6939658Ssl108498
653c6939658Ssl108498 /* May be partially locked, so, count bytes to charge for locking */
654c6939658Ssl108498 for (i = 0; i < npages; i++)
655c6939658Ssl108498 if (ppa[i]->p_lckcnt == 0)
656c6939658Ssl108498 lockedbytes += PAGESIZE;
657c6939658Ssl108498
658c6939658Ssl108498 proj = sp->shm_perm.ipc_proj;
659c6939658Ssl108498
660c6939658Ssl108498 if (lockedbytes > 0) {
661c6939658Ssl108498 mutex_enter(&procp->p_lock);
662c6939658Ssl108498 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
663c6939658Ssl108498 mutex_exit(&procp->p_lock);
664c6939658Ssl108498 mutex_exit(&sp->shm_mlock);
665c6939658Ssl108498 for (i = 0; i < npages; i++)
666c6939658Ssl108498 page_unlock(ppa[i]);
667c6939658Ssl108498 err = ENOMEM;
668c6939658Ssl108498 goto out4;
669c6939658Ssl108498 }
670c6939658Ssl108498 mutex_exit(&procp->p_lock);
671c6939658Ssl108498 }
672c6939658Ssl108498
6737c478bd9Sstevel@tonic-gate /*
6747c478bd9Sstevel@tonic-gate * addr is initial address corresponding to the first page on ppa list
6757c478bd9Sstevel@tonic-gate */
6767c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) {
6777c478bd9Sstevel@tonic-gate /* attempt to lock all pages */
678c6939658Ssl108498 if (page_pp_lock(ppa[i], 0, 1) == 0) {
6797c478bd9Sstevel@tonic-gate /*
6807c478bd9Sstevel@tonic-gate * if unable to lock any page, unlock all
6817c478bd9Sstevel@tonic-gate * of them and return error
6827c478bd9Sstevel@tonic-gate */
6837c478bd9Sstevel@tonic-gate for (j = 0; j < i; j++)
6847c478bd9Sstevel@tonic-gate page_pp_unlock(ppa[j], 0, 1);
685c6939658Ssl108498 for (i = 0; i < npages; i++)
6867c478bd9Sstevel@tonic-gate page_unlock(ppa[i]);
687c6939658Ssl108498 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
688c6939658Ssl108498 mutex_exit(&sp->shm_mlock);
6897c478bd9Sstevel@tonic-gate err = ENOMEM;
6907c478bd9Sstevel@tonic-gate goto out4;
6917c478bd9Sstevel@tonic-gate }
6927c478bd9Sstevel@tonic-gate }
693c6939658Ssl108498 mutex_exit(&sp->shm_mlock);
6947c478bd9Sstevel@tonic-gate
6957c478bd9Sstevel@tonic-gate /*
6967c478bd9Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
6977c478bd9Sstevel@tonic-gate * for the entire life of the segment. For example platforms
6987c478bd9Sstevel@tonic-gate * that do not support Dynamic Reconfiguration.
6997c478bd9Sstevel@tonic-gate */
7007c478bd9Sstevel@tonic-gate hat_flags = HAT_LOAD_SHARE;
7017c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
7027c478bd9Sstevel@tonic-gate hat_flags |= HAT_LOAD_LOCK;
7037c478bd9Sstevel@tonic-gate
70407b65a64Saguzovsk /*
70507b65a64Saguzovsk * Load translations one lare page at a time
70607b65a64Saguzovsk * to make sure we don't create mappings bigger than
70707b65a64Saguzovsk * segment's size code in case underlying pages
70807b65a64Saguzovsk * are shared with segvn's segment that uses bigger
70907b65a64Saguzovsk * size code than we do.
71007b65a64Saguzovsk */
71107b65a64Saguzovsk pgsz = page_get_pagesize(seg->s_szc);
71207b65a64Saguzovsk pgcnt = page_get_pagecnt(seg->s_szc);
71307b65a64Saguzovsk for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
71407b65a64Saguzovsk sz = MIN(pgsz, ptob(npages - pidx));
71507b65a64Saguzovsk hat_memload_array(seg->s_as->a_hat, a, sz,
71607b65a64Saguzovsk &ppa[pidx], sptd->spt_prot, hat_flags);
71707b65a64Saguzovsk }
7187c478bd9Sstevel@tonic-gate
7197c478bd9Sstevel@tonic-gate /*
7207c478bd9Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
7217c478bd9Sstevel@tonic-gate * we will leave the pages locked SE_SHARED for the life
7227c478bd9Sstevel@tonic-gate * of the ISM segment. This will prevent any calls to
7237c478bd9Sstevel@tonic-gate * hat_pageunload() on this ISM segment for those platforms.
7247c478bd9Sstevel@tonic-gate */
7257c478bd9Sstevel@tonic-gate if (!(hat_flags & HAT_LOAD_LOCK)) {
7267c478bd9Sstevel@tonic-gate /*
7277c478bd9Sstevel@tonic-gate * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
7287c478bd9Sstevel@tonic-gate * we no longer need to hold the SE_SHARED lock on the pages,
7297c478bd9Sstevel@tonic-gate * since L_PAGELOCK and F_SOFTLOCK calls will grab the
7307c478bd9Sstevel@tonic-gate * SE_SHARED lock on the pages as necessary.
7317c478bd9Sstevel@tonic-gate */
7327c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++)
7337c478bd9Sstevel@tonic-gate page_unlock(ppa[i]);
7347c478bd9Sstevel@tonic-gate }
7357c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt = 0;
7367c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * npages));
7377c478bd9Sstevel@tonic-gate sptd->spt_realsize = ptob(npages);
7387c478bd9Sstevel@tonic-gate atomic_add_long(&spt_used, npages);
7397c478bd9Sstevel@tonic-gate sptcargs->seg_spt = seg;
7407c478bd9Sstevel@tonic-gate return (0);
7417c478bd9Sstevel@tonic-gate
7427c478bd9Sstevel@tonic-gate out4:
7437c478bd9Sstevel@tonic-gate seg->s_data = NULL;
7447c478bd9Sstevel@tonic-gate kmem_free(vp, sizeof (*vp));
7452ba723d8Smec cv_destroy(&sptd->spt_cv);
7467c478bd9Sstevel@tonic-gate out3:
7477c478bd9Sstevel@tonic-gate mutex_destroy(&sptd->spt_lock);
7487c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0)
7497c478bd9Sstevel@tonic-gate kmem_free(ppa, (sizeof (*ppa) * npages));
7507c478bd9Sstevel@tonic-gate out2:
7517c478bd9Sstevel@tonic-gate kmem_free(sptd, sizeof (*sptd));
7527c478bd9Sstevel@tonic-gate out1:
7537c478bd9Sstevel@tonic-gate if ((sptcargs->flags & SHM_PAGEABLE) == 0)
7547c478bd9Sstevel@tonic-gate anon_swap_restore(npages);
7557c478bd9Sstevel@tonic-gate return (err);
7567c478bd9Sstevel@tonic-gate }
7577c478bd9Sstevel@tonic-gate
7587c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7597c478bd9Sstevel@tonic-gate void
segspt_free_pages(struct seg * seg,caddr_t addr,size_t len)7607c478bd9Sstevel@tonic-gate segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
7617c478bd9Sstevel@tonic-gate {
7627c478bd9Sstevel@tonic-gate struct page *pp;
7637c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)seg->s_data;
7647c478bd9Sstevel@tonic-gate pgcnt_t npages;
7657c478bd9Sstevel@tonic-gate ulong_t anon_idx;
7667c478bd9Sstevel@tonic-gate struct anon_map *amp;
7677c478bd9Sstevel@tonic-gate struct anon *ap;
7687c478bd9Sstevel@tonic-gate struct vnode *vp;
7697c478bd9Sstevel@tonic-gate u_offset_t off;
7707c478bd9Sstevel@tonic-gate uint_t hat_flags;
7717c478bd9Sstevel@tonic-gate int root = 0;
7727c478bd9Sstevel@tonic-gate pgcnt_t pgs, curnpgs = 0;
7737c478bd9Sstevel@tonic-gate page_t *rootpp;
774c6939658Ssl108498 rctl_qty_t unlocked_bytes = 0;
775c6939658Ssl108498 kproject_t *proj;
776c6939658Ssl108498 kshmid_t *sp;
7777c478bd9Sstevel@tonic-gate
778dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
7797c478bd9Sstevel@tonic-gate
7807c478bd9Sstevel@tonic-gate len = P2ROUNDUP(len, PAGESIZE);
7817c478bd9Sstevel@tonic-gate
7827c478bd9Sstevel@tonic-gate npages = btop(len);
7837c478bd9Sstevel@tonic-gate
78405d3dc4bSpaulsan hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP;
7857c478bd9Sstevel@tonic-gate if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) ||
7867c478bd9Sstevel@tonic-gate (sptd->spt_flags & SHM_PAGEABLE)) {
78705d3dc4bSpaulsan hat_flags = HAT_UNLOAD_UNMAP;
7887c478bd9Sstevel@tonic-gate }
7897c478bd9Sstevel@tonic-gate
7907c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, hat_flags);
7917c478bd9Sstevel@tonic-gate
7927c478bd9Sstevel@tonic-gate amp = sptd->spt_amp;
7937c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE)
7947c478bd9Sstevel@tonic-gate npages = btop(amp->size);
7957c478bd9Sstevel@tonic-gate
796c6939658Ssl108498 ASSERT(amp != NULL);
797c6939658Ssl108498
798c6f039c7SToomas Soome proj = NULL;
799c6f039c7SToomas Soome rootpp = NULL;
800c6f039c7SToomas Soome sp = NULL;
801c6939658Ssl108498 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
802c6939658Ssl108498 sp = amp->a_sp;
803c6939658Ssl108498 proj = sp->shm_perm.ipc_proj;
804c6939658Ssl108498 mutex_enter(&sp->shm_mlock);
805c6939658Ssl108498 }
8067c478bd9Sstevel@tonic-gate for (anon_idx = 0; anon_idx < npages; anon_idx++) {
8077c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
8087c478bd9Sstevel@tonic-gate if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
8097c478bd9Sstevel@tonic-gate panic("segspt_free_pages: null app");
8107c478bd9Sstevel@tonic-gate /*NOTREACHED*/
8117c478bd9Sstevel@tonic-gate }
8127c478bd9Sstevel@tonic-gate } else {
8137c478bd9Sstevel@tonic-gate if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
8147c478bd9Sstevel@tonic-gate == NULL)
8157c478bd9Sstevel@tonic-gate continue;
8167c478bd9Sstevel@tonic-gate }
8177c478bd9Sstevel@tonic-gate ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
8187c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off);
8197c478bd9Sstevel@tonic-gate
8207c478bd9Sstevel@tonic-gate /*
8217c478bd9Sstevel@tonic-gate * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
8227c478bd9Sstevel@tonic-gate * the pages won't be having SE_SHARED lock at this
8237c478bd9Sstevel@tonic-gate * point.
8247c478bd9Sstevel@tonic-gate *
8257c478bd9Sstevel@tonic-gate * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
8267c478bd9Sstevel@tonic-gate * the pages are still held SE_SHARED locked from the
8277c478bd9Sstevel@tonic-gate * original segspt_create()
8287c478bd9Sstevel@tonic-gate *
8297c478bd9Sstevel@tonic-gate * Our goal is to get SE_EXCL lock on each page, remove
8307c478bd9Sstevel@tonic-gate * permanent lock on it and invalidate the page.
8317c478bd9Sstevel@tonic-gate */
8327c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
83305d3dc4bSpaulsan if (hat_flags == HAT_UNLOAD_UNMAP)
8347c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL);
8357c478bd9Sstevel@tonic-gate else {
8367c478bd9Sstevel@tonic-gate if ((pp = page_find(vp, off)) == NULL) {
8377c478bd9Sstevel@tonic-gate panic("segspt_free_pages: "
8387c478bd9Sstevel@tonic-gate "page not locked");
8397c478bd9Sstevel@tonic-gate /*NOTREACHED*/
8407c478bd9Sstevel@tonic-gate }
8417c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) {
8427c478bd9Sstevel@tonic-gate page_unlock(pp);
8437c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_EXCL);
8447c478bd9Sstevel@tonic-gate }
8457c478bd9Sstevel@tonic-gate }
8467c478bd9Sstevel@tonic-gate if (pp == NULL) {
8477c478bd9Sstevel@tonic-gate panic("segspt_free_pages: "
8487c478bd9Sstevel@tonic-gate "page not in the system");
8497c478bd9Sstevel@tonic-gate /*NOTREACHED*/
8507c478bd9Sstevel@tonic-gate }
851c6939658Ssl108498 ASSERT(pp->p_lckcnt > 0);
8527c478bd9Sstevel@tonic-gate page_pp_unlock(pp, 0, 1);
853c6939658Ssl108498 if (pp->p_lckcnt == 0)
854c6939658Ssl108498 unlocked_bytes += PAGESIZE;
8557c478bd9Sstevel@tonic-gate } else {
8567c478bd9Sstevel@tonic-gate if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL)
8577c478bd9Sstevel@tonic-gate continue;
8587c478bd9Sstevel@tonic-gate }
8597c478bd9Sstevel@tonic-gate /*
8607c478bd9Sstevel@tonic-gate * It's logical to invalidate the pages here as in most cases
8617c478bd9Sstevel@tonic-gate * these were created by segspt.
8627c478bd9Sstevel@tonic-gate */
8637c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) {
8647c478bd9Sstevel@tonic-gate if (root == 0) {
8657c478bd9Sstevel@tonic-gate ASSERT(curnpgs == 0);
8667c478bd9Sstevel@tonic-gate root = 1;
8677c478bd9Sstevel@tonic-gate rootpp = pp;
8687c478bd9Sstevel@tonic-gate pgs = curnpgs = page_get_pagecnt(pp->p_szc);
8697c478bd9Sstevel@tonic-gate ASSERT(pgs > 1);
8707c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pgs, pgs));
8717c478bd9Sstevel@tonic-gate ASSERT(!(page_pptonum(pp) & (pgs - 1)));
8727c478bd9Sstevel@tonic-gate curnpgs--;
8737c478bd9Sstevel@tonic-gate } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
8747c478bd9Sstevel@tonic-gate ASSERT(curnpgs == 1);
8757c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(pp) ==
8767c478bd9Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - 1));
8777c478bd9Sstevel@tonic-gate page_destroy_pages(rootpp);
8787c478bd9Sstevel@tonic-gate root = 0;
8797c478bd9Sstevel@tonic-gate curnpgs = 0;
8807c478bd9Sstevel@tonic-gate } else {
8817c478bd9Sstevel@tonic-gate ASSERT(curnpgs > 1);
8827c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(pp) ==
8837c478bd9Sstevel@tonic-gate page_pptonum(rootpp) + (pgs - curnpgs));
8847c478bd9Sstevel@tonic-gate curnpgs--;
8857c478bd9Sstevel@tonic-gate }
8867c478bd9Sstevel@tonic-gate } else {
8877c478bd9Sstevel@tonic-gate if (root != 0 || curnpgs != 0) {
8887c478bd9Sstevel@tonic-gate panic("segspt_free_pages: bad large page");
8897c478bd9Sstevel@tonic-gate /*NOTREACHED*/
8907c478bd9Sstevel@tonic-gate }
891552507c5SGangadhar Mylapuram /*
892552507c5SGangadhar Mylapuram * Before destroying the pages, we need to take care
893552507c5SGangadhar Mylapuram * of the rctl locked memory accounting. For that
894552507c5SGangadhar Mylapuram * we need to calculte the unlocked_bytes.
895552507c5SGangadhar Mylapuram */
896552507c5SGangadhar Mylapuram if (pp->p_lckcnt > 0)
897552507c5SGangadhar Mylapuram unlocked_bytes += PAGESIZE;
8987c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional context */
8997c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred);
9007c478bd9Sstevel@tonic-gate }
9017c478bd9Sstevel@tonic-gate }
902c6939658Ssl108498 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
903c6939658Ssl108498 if (unlocked_bytes > 0)
904c6939658Ssl108498 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
905c6939658Ssl108498 mutex_exit(&sp->shm_mlock);
906c6939658Ssl108498 }
9077c478bd9Sstevel@tonic-gate if (root != 0 || curnpgs != 0) {
9087c478bd9Sstevel@tonic-gate panic("segspt_free_pages: bad large page");
9097c478bd9Sstevel@tonic-gate /*NOTREACHED*/
9107c478bd9Sstevel@tonic-gate }
9117c478bd9Sstevel@tonic-gate
9127c478bd9Sstevel@tonic-gate /*
9137c478bd9Sstevel@tonic-gate * mark that pages have been released
9147c478bd9Sstevel@tonic-gate */
9157c478bd9Sstevel@tonic-gate sptd->spt_realsize = 0;
9167c478bd9Sstevel@tonic-gate
9177c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
9187c478bd9Sstevel@tonic-gate atomic_add_long(&spt_used, -npages);
9197c478bd9Sstevel@tonic-gate anon_swap_restore(npages);
9207c478bd9Sstevel@tonic-gate }
9217c478bd9Sstevel@tonic-gate }
9227c478bd9Sstevel@tonic-gate
9237c478bd9Sstevel@tonic-gate /*
9247c478bd9Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment
9257c478bd9Sstevel@tonic-gate */
9267c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *
segspt_getpolicy(struct seg * seg,caddr_t addr)9277c478bd9Sstevel@tonic-gate segspt_getpolicy(struct seg *seg, caddr_t addr)
9287c478bd9Sstevel@tonic-gate {
9297c478bd9Sstevel@tonic-gate struct anon_map *amp;
9307c478bd9Sstevel@tonic-gate ulong_t anon_index;
9317c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info;
9327c478bd9Sstevel@tonic-gate struct spt_data *spt_data;
9337c478bd9Sstevel@tonic-gate
9347c478bd9Sstevel@tonic-gate ASSERT(seg != NULL);
9357c478bd9Sstevel@tonic-gate
9367c478bd9Sstevel@tonic-gate /*
9377c478bd9Sstevel@tonic-gate * Get anon_map from segspt
9387c478bd9Sstevel@tonic-gate *
9397c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since
9407c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be
9417c478bd9Sstevel@tonic-gate * nonzero for an existing segment
9427c478bd9Sstevel@tonic-gate * Need to grab readers lock on policy tree though
9437c478bd9Sstevel@tonic-gate */
9447c478bd9Sstevel@tonic-gate spt_data = (struct spt_data *)seg->s_data;
9457c478bd9Sstevel@tonic-gate if (spt_data == NULL)
9467c478bd9Sstevel@tonic-gate return (NULL);
9477c478bd9Sstevel@tonic-gate amp = spt_data->spt_amp;
9487c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0);
9497c478bd9Sstevel@tonic-gate
9507c478bd9Sstevel@tonic-gate /*
9517c478bd9Sstevel@tonic-gate * Get policy info
9527c478bd9Sstevel@tonic-gate *
9537c478bd9Sstevel@tonic-gate * Assume starting anon index of 0
9547c478bd9Sstevel@tonic-gate */
9557c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr);
9567c478bd9Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
9577c478bd9Sstevel@tonic-gate
9587c478bd9Sstevel@tonic-gate return (policy_info);
9597c478bd9Sstevel@tonic-gate }
9607c478bd9Sstevel@tonic-gate
9617c478bd9Sstevel@tonic-gate /*
9627c478bd9Sstevel@tonic-gate * DISM only.
9637c478bd9Sstevel@tonic-gate * Return locked pages over a given range.
9647c478bd9Sstevel@tonic-gate *
9657c478bd9Sstevel@tonic-gate * We will cache all DISM locked pages and save the pplist for the
9667c478bd9Sstevel@tonic-gate * entire segment in the ppa field of the underlying DISM segment structure.
9677c478bd9Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array
9687c478bd9Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list.
9697c478bd9Sstevel@tonic-gate */
9707c478bd9Sstevel@tonic-gate /*ARGSUSED*/
9717c478bd9Sstevel@tonic-gate static int
segspt_dismpagelock(struct seg * seg,caddr_t addr,size_t len,struct page *** ppp,enum lock_type type,enum seg_rw rw)9727c478bd9Sstevel@tonic-gate segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
9737c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw)
9747c478bd9Sstevel@tonic-gate {
9757c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
9767c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
9777c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data;
9787c478bd9Sstevel@tonic-gate pgcnt_t pg_idx, npages, tot_npages, npgs;
9797c478bd9Sstevel@tonic-gate struct page **pplist, **pl, **ppa, *pp;
9807c478bd9Sstevel@tonic-gate struct anon_map *amp;
9817c478bd9Sstevel@tonic-gate spgcnt_t an_idx;
9827c478bd9Sstevel@tonic-gate int ret = ENOTSUP;
9837c478bd9Sstevel@tonic-gate uint_t pl_built = 0;
9847c478bd9Sstevel@tonic-gate struct anon *ap;
9857c478bd9Sstevel@tonic-gate struct vnode *vp;
9867c478bd9Sstevel@tonic-gate u_offset_t off;
9877c478bd9Sstevel@tonic-gate pgcnt_t claim_availrmem = 0;
9887c478bd9Sstevel@tonic-gate uint_t szc;
9897c478bd9Sstevel@tonic-gate
990dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
991a98e9dbfSaguzovsk ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
9927c478bd9Sstevel@tonic-gate
9937c478bd9Sstevel@tonic-gate /*
9947c478bd9Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore,
9957c478bd9Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address
9967c478bd9Sstevel@tonic-gate * and length for the caching arguments.
9977c478bd9Sstevel@tonic-gate */
9987c478bd9Sstevel@tonic-gate ASSERT(sptseg);
9997c478bd9Sstevel@tonic-gate ASSERT(sptd);
10007c478bd9Sstevel@tonic-gate
10017c478bd9Sstevel@tonic-gate pg_idx = seg_page(seg, addr);
10027c478bd9Sstevel@tonic-gate npages = btopr(len);
10037c478bd9Sstevel@tonic-gate
10047c478bd9Sstevel@tonic-gate /*
10057c478bd9Sstevel@tonic-gate * check if the request is larger than number of pages covered
10067c478bd9Sstevel@tonic-gate * by amp
10077c478bd9Sstevel@tonic-gate */
10087c478bd9Sstevel@tonic-gate if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
10097c478bd9Sstevel@tonic-gate *ppp = NULL;
10107c478bd9Sstevel@tonic-gate return (ENOTSUP);
10117c478bd9Sstevel@tonic-gate }
10127c478bd9Sstevel@tonic-gate
10137c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) {
10147c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL);
10157c478bd9Sstevel@tonic-gate
1016a98e9dbfSaguzovsk seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
1017a98e9dbfSaguzovsk sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
10187c478bd9Sstevel@tonic-gate
10197c478bd9Sstevel@tonic-gate /*
10207c478bd9Sstevel@tonic-gate * If someone is blocked while unmapping, we purge
10217c478bd9Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously
10227c478bd9Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up
10237c478bd9Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while
10247c478bd9Sstevel@tonic-gate * raw async i/o is still in progress or where a thread
10257c478bd9Sstevel@tonic-gate * exits on data fault in a multithreaded application.
10267c478bd9Sstevel@tonic-gate */
1027a98e9dbfSaguzovsk if ((sptd->spt_flags & DISM_PPA_CHANGED) ||
1028a98e9dbfSaguzovsk (AS_ISUNMAPWAIT(seg->s_as) &&
1029a98e9dbfSaguzovsk shmd->shm_softlockcnt > 0)) {
10307c478bd9Sstevel@tonic-gate segspt_purge(seg);
10317c478bd9Sstevel@tonic-gate }
10327c478bd9Sstevel@tonic-gate return (0);
10337c478bd9Sstevel@tonic-gate }
10347c478bd9Sstevel@tonic-gate
1035a98e9dbfSaguzovsk /* The L_PAGELOCK case ... */
1036a98e9dbfSaguzovsk
10377c478bd9Sstevel@tonic-gate if (sptd->spt_flags & DISM_PPA_CHANGED) {
10387c478bd9Sstevel@tonic-gate segspt_purge(seg);
10397c478bd9Sstevel@tonic-gate /*
10407c478bd9Sstevel@tonic-gate * for DISM ppa needs to be rebuild since
10417c478bd9Sstevel@tonic-gate * number of locked pages could be changed
10427c478bd9Sstevel@tonic-gate */
10437c478bd9Sstevel@tonic-gate *ppp = NULL;
10447c478bd9Sstevel@tonic-gate return (ENOTSUP);
10457c478bd9Sstevel@tonic-gate }
10467c478bd9Sstevel@tonic-gate
10477c478bd9Sstevel@tonic-gate /*
10487c478bd9Sstevel@tonic-gate * First try to find pages in segment page cache, without
10497c478bd9Sstevel@tonic-gate * holding the segment lock.
10507c478bd9Sstevel@tonic-gate */
1051a98e9dbfSaguzovsk pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1052a98e9dbfSaguzovsk S_WRITE, SEGP_FORCE_WIRED);
10537c478bd9Sstevel@tonic-gate if (pplist != NULL) {
10547c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL);
10557c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist);
10567c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa;
10577c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
10587c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) {
1059a98e9dbfSaguzovsk seg_pinactive(seg, NULL, seg->s_base,
10607c478bd9Sstevel@tonic-gate sptd->spt_amp->size, ppa,
1061a98e9dbfSaguzovsk S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
10627c478bd9Sstevel@tonic-gate *ppp = NULL;
10637c478bd9Sstevel@tonic-gate return (ENOTSUP);
10647c478bd9Sstevel@tonic-gate }
10657c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) {
10667c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc);
10677c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs);
10687c478bd9Sstevel@tonic-gate } else {
10697c478bd9Sstevel@tonic-gate an_idx++;
10707c478bd9Sstevel@tonic-gate }
10717c478bd9Sstevel@tonic-gate }
10727c478bd9Sstevel@tonic-gate /*
10737c478bd9Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to
10747c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds
10757c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx.
10767c478bd9Sstevel@tonic-gate */
10777c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]);
10787c478bd9Sstevel@tonic-gate return (0);
10797c478bd9Sstevel@tonic-gate }
10807c478bd9Sstevel@tonic-gate
10817c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
10827c478bd9Sstevel@tonic-gate /*
10837c478bd9Sstevel@tonic-gate * try to find pages in segment page cache with mutex
10847c478bd9Sstevel@tonic-gate */
1085a98e9dbfSaguzovsk pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1086a98e9dbfSaguzovsk S_WRITE, SEGP_FORCE_WIRED);
10877c478bd9Sstevel@tonic-gate if (pplist != NULL) {
10887c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL);
10897c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist);
10907c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa;
10917c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
10927c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) {
10937c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
1094a98e9dbfSaguzovsk seg_pinactive(seg, NULL, seg->s_base,
10957c478bd9Sstevel@tonic-gate sptd->spt_amp->size, ppa,
1096a98e9dbfSaguzovsk S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
10977c478bd9Sstevel@tonic-gate *ppp = NULL;
10987c478bd9Sstevel@tonic-gate return (ENOTSUP);
10997c478bd9Sstevel@tonic-gate }
11007c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) {
11017c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc);
11027c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs);
11037c478bd9Sstevel@tonic-gate } else {
11047c478bd9Sstevel@tonic-gate an_idx++;
11057c478bd9Sstevel@tonic-gate }
11067c478bd9Sstevel@tonic-gate }
11077c478bd9Sstevel@tonic-gate /*
11087c478bd9Sstevel@tonic-gate * Since we cache the entire DISM segment, we want to
11097c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds
11107c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx.
11117c478bd9Sstevel@tonic-gate */
11127c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
11137c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]);
11147c478bd9Sstevel@tonic-gate return (0);
11157c478bd9Sstevel@tonic-gate }
1116a98e9dbfSaguzovsk if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
1117a98e9dbfSaguzovsk SEGP_FORCE_WIRED) == SEGP_FAIL) {
11187c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
11197c478bd9Sstevel@tonic-gate *ppp = NULL;
11207c478bd9Sstevel@tonic-gate return (ENOTSUP);
11217c478bd9Sstevel@tonic-gate }
11227c478bd9Sstevel@tonic-gate
11237c478bd9Sstevel@tonic-gate /*
11247c478bd9Sstevel@tonic-gate * No need to worry about protections because DISM pages are always rw.
11257c478bd9Sstevel@tonic-gate */
11267c478bd9Sstevel@tonic-gate pl = pplist = NULL;
11277c478bd9Sstevel@tonic-gate amp = sptd->spt_amp;
11287c478bd9Sstevel@tonic-gate
11297c478bd9Sstevel@tonic-gate /*
11307c478bd9Sstevel@tonic-gate * Do we need to build the ppa array?
11317c478bd9Sstevel@tonic-gate */
11327c478bd9Sstevel@tonic-gate if (sptd->spt_ppa == NULL) {
11337c478bd9Sstevel@tonic-gate pgcnt_t lpg_cnt = 0;
11347c478bd9Sstevel@tonic-gate
11357c478bd9Sstevel@tonic-gate pl_built = 1;
11367c478bd9Sstevel@tonic-gate tot_npages = btopr(sptd->spt_amp->size);
11377c478bd9Sstevel@tonic-gate
11387c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0);
11397c478bd9Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP);
11407c478bd9Sstevel@tonic-gate pl = pplist;
11417c478bd9Sstevel@tonic-gate
11427c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
11437c478bd9Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; ) {
11447c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx);
11457c478bd9Sstevel@tonic-gate /*
11467c478bd9Sstevel@tonic-gate * Cache only mlocked pages. For large pages
11477c478bd9Sstevel@tonic-gate * if one (constituent) page is mlocked
11487c478bd9Sstevel@tonic-gate * all pages for that large page
11497c478bd9Sstevel@tonic-gate * are cached also. This is for quick
11507c478bd9Sstevel@tonic-gate * lookups of ppa array;
11517c478bd9Sstevel@tonic-gate */
11527c478bd9Sstevel@tonic-gate if ((ap != NULL) && (lpg_cnt != 0 ||
11537c478bd9Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[an_idx] != 0))) {
11547c478bd9Sstevel@tonic-gate
11557c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off);
11567c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED);
11577c478bd9Sstevel@tonic-gate ASSERT(pp != NULL);
11587c478bd9Sstevel@tonic-gate if (lpg_cnt == 0) {
1159e67882ffSbs21162 lpg_cnt++;
1160e67882ffSbs21162 /*
1161e67882ffSbs21162 * For a small page, we are done --
1162e67882ffSbs21162 * lpg_count is reset to 0 below.
1163e67882ffSbs21162 *
1164e67882ffSbs21162 * For a large page, we are guaranteed
1165e67882ffSbs21162 * to find the anon structures of all
1166e67882ffSbs21162 * constituent pages and a non-zero
1167e67882ffSbs21162 * lpg_cnt ensures that we don't test
1168e67882ffSbs21162 * for mlock for these. We are done
1169e67882ffSbs21162 * when lpg_count reaches (npgs + 1).
1170e67882ffSbs21162 * If we are not the first constituent
1171e67882ffSbs21162 * page, restart at the first one.
1172e67882ffSbs21162 */
11737c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc);
11747c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(an_idx, npgs)) {
11757c478bd9Sstevel@tonic-gate an_idx = P2ALIGN(an_idx, npgs);
11767c478bd9Sstevel@tonic-gate page_unlock(pp);
11777c478bd9Sstevel@tonic-gate continue;
11787c478bd9Sstevel@tonic-gate }
11797c478bd9Sstevel@tonic-gate }
1180e67882ffSbs21162 if (++lpg_cnt > npgs)
11817c478bd9Sstevel@tonic-gate lpg_cnt = 0;
11827c478bd9Sstevel@tonic-gate
11837c478bd9Sstevel@tonic-gate /*
11847c478bd9Sstevel@tonic-gate * availrmem is decremented only
11857c478bd9Sstevel@tonic-gate * for unlocked pages
11867c478bd9Sstevel@tonic-gate */
11877c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[an_idx] == 0)
11887c478bd9Sstevel@tonic-gate claim_availrmem++;
11897c478bd9Sstevel@tonic-gate pplist[an_idx] = pp;
11907c478bd9Sstevel@tonic-gate }
11917c478bd9Sstevel@tonic-gate an_idx++;
11927c478bd9Sstevel@tonic-gate }
11937c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
11947c478bd9Sstevel@tonic-gate
1195a98e9dbfSaguzovsk if (claim_availrmem) {
11967c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock);
11977c478bd9Sstevel@tonic-gate if (availrmem < tune.t_minarmem + claim_availrmem) {
11987c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock);
1199a98e9dbfSaguzovsk ret = ENOTSUP;
12007c478bd9Sstevel@tonic-gate claim_availrmem = 0;
12017c478bd9Sstevel@tonic-gate goto insert_fail;
12027c478bd9Sstevel@tonic-gate } else {
12037c478bd9Sstevel@tonic-gate availrmem -= claim_availrmem;
12047c478bd9Sstevel@tonic-gate }
12057c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock);
1206a98e9dbfSaguzovsk }
12077c478bd9Sstevel@tonic-gate
12087c478bd9Sstevel@tonic-gate sptd->spt_ppa = pl;
12097c478bd9Sstevel@tonic-gate } else {
12107c478bd9Sstevel@tonic-gate /*
12117c478bd9Sstevel@tonic-gate * We already have a valid ppa[].
12127c478bd9Sstevel@tonic-gate */
12137c478bd9Sstevel@tonic-gate pl = sptd->spt_ppa;
12147c478bd9Sstevel@tonic-gate }
12157c478bd9Sstevel@tonic-gate
12167c478bd9Sstevel@tonic-gate ASSERT(pl != NULL);
12177c478bd9Sstevel@tonic-gate
1218a98e9dbfSaguzovsk ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1219a98e9dbfSaguzovsk sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
12207c478bd9Sstevel@tonic-gate segspt_reclaim);
12217c478bd9Sstevel@tonic-gate if (ret == SEGP_FAIL) {
12227c478bd9Sstevel@tonic-gate /*
12237c478bd9Sstevel@tonic-gate * seg_pinsert failed. We return
12247c478bd9Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will
12257c478bd9Sstevel@tonic-gate * then try the slower F_SOFTLOCK path.
12267c478bd9Sstevel@tonic-gate */
12270da3d2a8Srd117015 if (pl_built) {
12280da3d2a8Srd117015 /*
12290da3d2a8Srd117015 * No one else has referenced the ppa[].
12300da3d2a8Srd117015 * We created it and we need to destroy it.
12310da3d2a8Srd117015 */
12327c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL;
12330da3d2a8Srd117015 }
12347c478bd9Sstevel@tonic-gate ret = ENOTSUP;
12357c478bd9Sstevel@tonic-gate goto insert_fail;
12367c478bd9Sstevel@tonic-gate }
12377c478bd9Sstevel@tonic-gate
12387c478bd9Sstevel@tonic-gate /*
12397c478bd9Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment.
12407c478bd9Sstevel@tonic-gate */
12417c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt++;
12421a5e258fSJosef 'Jeff' Sipek atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
12437c478bd9Sstevel@tonic-gate
12447c478bd9Sstevel@tonic-gate ppa = sptd->spt_ppa;
12457c478bd9Sstevel@tonic-gate for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
12467c478bd9Sstevel@tonic-gate if (ppa[an_idx] == NULL) {
12477c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
1248a98e9dbfSaguzovsk seg_pinactive(seg, NULL, seg->s_base,
1249a98e9dbfSaguzovsk sptd->spt_amp->size,
1250a98e9dbfSaguzovsk pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
12517c478bd9Sstevel@tonic-gate *ppp = NULL;
12527c478bd9Sstevel@tonic-gate return (ENOTSUP);
12537c478bd9Sstevel@tonic-gate }
12547c478bd9Sstevel@tonic-gate if ((szc = ppa[an_idx]->p_szc) != 0) {
12557c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc);
12567c478bd9Sstevel@tonic-gate an_idx = P2ROUNDUP(an_idx + 1, npgs);
12577c478bd9Sstevel@tonic-gate } else {
12587c478bd9Sstevel@tonic-gate an_idx++;
12597c478bd9Sstevel@tonic-gate }
12607c478bd9Sstevel@tonic-gate }
12617c478bd9Sstevel@tonic-gate /*
12627c478bd9Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[]
126348bbca81SDaniel Hoffman * exists and we have incremented pacachecnt.
12647c478bd9Sstevel@tonic-gate */
12657c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
12667c478bd9Sstevel@tonic-gate
12677c478bd9Sstevel@tonic-gate /*
12687c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to
12697c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds
12707c478bd9Sstevel@tonic-gate * to the requested addr, i.e. pg_idx.
12717c478bd9Sstevel@tonic-gate */
12727c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[pg_idx]);
1273a98e9dbfSaguzovsk return (0);
12747c478bd9Sstevel@tonic-gate
12757c478bd9Sstevel@tonic-gate insert_fail:
12767c478bd9Sstevel@tonic-gate /*
12777c478bd9Sstevel@tonic-gate * We will only reach this code if we tried and failed.
12787c478bd9Sstevel@tonic-gate *
12797c478bd9Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed
12807c478bd9Sstevel@tonic-gate * to set up a new ppa[].
12817c478bd9Sstevel@tonic-gate */
12827c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
12837c478bd9Sstevel@tonic-gate
12847c478bd9Sstevel@tonic-gate if (pl_built) {
1285a98e9dbfSaguzovsk if (claim_availrmem) {
12867c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock);
12877c478bd9Sstevel@tonic-gate availrmem += claim_availrmem;
12887c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock);
1289a98e9dbfSaguzovsk }
12907c478bd9Sstevel@tonic-gate
12917c478bd9Sstevel@tonic-gate /*
12927c478bd9Sstevel@tonic-gate * We created pl and we need to destroy it.
12937c478bd9Sstevel@tonic-gate */
12947c478bd9Sstevel@tonic-gate pplist = pl;
12957c478bd9Sstevel@tonic-gate for (an_idx = 0; an_idx < tot_npages; an_idx++) {
12967c478bd9Sstevel@tonic-gate if (pplist[an_idx] != NULL)
12977c478bd9Sstevel@tonic-gate page_unlock(pplist[an_idx]);
12987c478bd9Sstevel@tonic-gate }
12997c478bd9Sstevel@tonic-gate kmem_free(pl, sizeof (page_t *) * tot_npages);
13007c478bd9Sstevel@tonic-gate }
13017c478bd9Sstevel@tonic-gate
13027c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) {
13037c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
13047c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents);
13057c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
13067c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as);
13077c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv);
13087c478bd9Sstevel@tonic-gate }
13097c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents);
13107c478bd9Sstevel@tonic-gate }
13117c478bd9Sstevel@tonic-gate }
13127c478bd9Sstevel@tonic-gate *ppp = NULL;
13137c478bd9Sstevel@tonic-gate return (ret);
13147c478bd9Sstevel@tonic-gate }
13157c478bd9Sstevel@tonic-gate
13167c478bd9Sstevel@tonic-gate
13177c478bd9Sstevel@tonic-gate
13187c478bd9Sstevel@tonic-gate /*
13197c478bd9Sstevel@tonic-gate * return locked pages over a given range.
13207c478bd9Sstevel@tonic-gate *
13217c478bd9Sstevel@tonic-gate * We will cache the entire ISM segment and save the pplist for the
13227c478bd9Sstevel@tonic-gate * entire segment in the ppa field of the underlying ISM segment structure.
13237c478bd9Sstevel@tonic-gate * Later, during a call to segspt_reclaim() we will use this ppa array
13247c478bd9Sstevel@tonic-gate * to page_unlock() all of the pages and then we will free this ppa list.
13257c478bd9Sstevel@tonic-gate */
13267c478bd9Sstevel@tonic-gate /*ARGSUSED*/
13277c478bd9Sstevel@tonic-gate static int
segspt_shmpagelock(struct seg * seg,caddr_t addr,size_t len,struct page *** ppp,enum lock_type type,enum seg_rw rw)13287c478bd9Sstevel@tonic-gate segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len,
13297c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw)
13307c478bd9Sstevel@tonic-gate {
13317c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
13327c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
13337c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data;
13347c478bd9Sstevel@tonic-gate pgcnt_t np, page_index, npages;
13357c478bd9Sstevel@tonic-gate caddr_t a, spt_base;
13367c478bd9Sstevel@tonic-gate struct page **pplist, **pl, *pp;
13377c478bd9Sstevel@tonic-gate struct anon_map *amp;
13387c478bd9Sstevel@tonic-gate ulong_t anon_index;
13397c478bd9Sstevel@tonic-gate int ret = ENOTSUP;
13407c478bd9Sstevel@tonic-gate uint_t pl_built = 0;
13417c478bd9Sstevel@tonic-gate struct anon *ap;
13427c478bd9Sstevel@tonic-gate struct vnode *vp;
13437c478bd9Sstevel@tonic-gate u_offset_t off;
13447c478bd9Sstevel@tonic-gate
1345dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1346a98e9dbfSaguzovsk ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
1347a98e9dbfSaguzovsk
13487c478bd9Sstevel@tonic-gate
13497c478bd9Sstevel@tonic-gate /*
13507c478bd9Sstevel@tonic-gate * We want to lock/unlock the entire ISM segment. Therefore,
13517c478bd9Sstevel@tonic-gate * we will be using the underlying sptseg and it's base address
13527c478bd9Sstevel@tonic-gate * and length for the caching arguments.
13537c478bd9Sstevel@tonic-gate */
13547c478bd9Sstevel@tonic-gate ASSERT(sptseg);
13557c478bd9Sstevel@tonic-gate ASSERT(sptd);
13567c478bd9Sstevel@tonic-gate
13577c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) {
13587c478bd9Sstevel@tonic-gate return (segspt_dismpagelock(seg, addr, len, ppp, type, rw));
13597c478bd9Sstevel@tonic-gate }
13607c478bd9Sstevel@tonic-gate
13617c478bd9Sstevel@tonic-gate page_index = seg_page(seg, addr);
13627c478bd9Sstevel@tonic-gate npages = btopr(len);
13637c478bd9Sstevel@tonic-gate
13647c478bd9Sstevel@tonic-gate /*
13657c478bd9Sstevel@tonic-gate * check if the request is larger than number of pages covered
13667c478bd9Sstevel@tonic-gate * by amp
13677c478bd9Sstevel@tonic-gate */
13687c478bd9Sstevel@tonic-gate if (page_index + npages > btopr(sptd->spt_amp->size)) {
13697c478bd9Sstevel@tonic-gate *ppp = NULL;
13707c478bd9Sstevel@tonic-gate return (ENOTSUP);
13717c478bd9Sstevel@tonic-gate }
13727c478bd9Sstevel@tonic-gate
13737c478bd9Sstevel@tonic-gate if (type == L_PAGEUNLOCK) {
13747c478bd9Sstevel@tonic-gate
13757c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa != NULL);
13767c478bd9Sstevel@tonic-gate
1377a98e9dbfSaguzovsk seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
1378a98e9dbfSaguzovsk sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
13797c478bd9Sstevel@tonic-gate
13807c478bd9Sstevel@tonic-gate /*
13817c478bd9Sstevel@tonic-gate * If someone is blocked while unmapping, we purge
13827c478bd9Sstevel@tonic-gate * segment page cache and thus reclaim pplist synchronously
13837c478bd9Sstevel@tonic-gate * without waiting for seg_pasync_thread. This speeds up
13847c478bd9Sstevel@tonic-gate * unmapping in cases where munmap(2) is called, while
13857c478bd9Sstevel@tonic-gate * raw async i/o is still in progress or where a thread
13867c478bd9Sstevel@tonic-gate * exits on data fault in a multithreaded application.
13877c478bd9Sstevel@tonic-gate */
13887c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
13897c478bd9Sstevel@tonic-gate segspt_purge(seg);
13907c478bd9Sstevel@tonic-gate }
13917c478bd9Sstevel@tonic-gate return (0);
13927c478bd9Sstevel@tonic-gate }
13937c478bd9Sstevel@tonic-gate
1394a98e9dbfSaguzovsk /* The L_PAGELOCK case... */
1395a98e9dbfSaguzovsk
13967c478bd9Sstevel@tonic-gate /*
13977c478bd9Sstevel@tonic-gate * First try to find pages in segment page cache, without
13987c478bd9Sstevel@tonic-gate * holding the segment lock.
13997c478bd9Sstevel@tonic-gate */
1400a98e9dbfSaguzovsk pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1401a98e9dbfSaguzovsk S_WRITE, SEGP_FORCE_WIRED);
14027c478bd9Sstevel@tonic-gate if (pplist != NULL) {
14037c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist);
14047c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa[page_index]);
14057c478bd9Sstevel@tonic-gate /*
14067c478bd9Sstevel@tonic-gate * Since we cache the entire ISM segment, we want to
14077c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds
14087c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index.
14097c478bd9Sstevel@tonic-gate */
14107c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]);
14117c478bd9Sstevel@tonic-gate return (0);
14127c478bd9Sstevel@tonic-gate }
14137c478bd9Sstevel@tonic-gate
14147c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
14157c478bd9Sstevel@tonic-gate
14167c478bd9Sstevel@tonic-gate /*
14177c478bd9Sstevel@tonic-gate * try to find pages in segment page cache
14187c478bd9Sstevel@tonic-gate */
1419a98e9dbfSaguzovsk pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1420a98e9dbfSaguzovsk S_WRITE, SEGP_FORCE_WIRED);
14217c478bd9Sstevel@tonic-gate if (pplist != NULL) {
14227c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist);
14237c478bd9Sstevel@tonic-gate /*
14247c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to
14257c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds
14267c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index.
14277c478bd9Sstevel@tonic-gate */
14287c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
14297c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]);
14307c478bd9Sstevel@tonic-gate return (0);
14317c478bd9Sstevel@tonic-gate }
14327c478bd9Sstevel@tonic-gate
1433a98e9dbfSaguzovsk if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
1434a98e9dbfSaguzovsk SEGP_FORCE_WIRED) == SEGP_FAIL) {
14357c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
14367c478bd9Sstevel@tonic-gate *ppp = NULL;
14377c478bd9Sstevel@tonic-gate return (ENOTSUP);
14387c478bd9Sstevel@tonic-gate }
14397c478bd9Sstevel@tonic-gate
14407c478bd9Sstevel@tonic-gate /*
14417c478bd9Sstevel@tonic-gate * No need to worry about protections because ISM pages
14427c478bd9Sstevel@tonic-gate * are always rw.
14437c478bd9Sstevel@tonic-gate */
14447c478bd9Sstevel@tonic-gate pl = pplist = NULL;
14457c478bd9Sstevel@tonic-gate
14467c478bd9Sstevel@tonic-gate /*
14477c478bd9Sstevel@tonic-gate * Do we need to build the ppa array?
14487c478bd9Sstevel@tonic-gate */
14497c478bd9Sstevel@tonic-gate if (sptd->spt_ppa == NULL) {
14507c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist);
14517c478bd9Sstevel@tonic-gate
14527c478bd9Sstevel@tonic-gate spt_base = sptseg->s_base;
14537c478bd9Sstevel@tonic-gate pl_built = 1;
14547c478bd9Sstevel@tonic-gate
14557c478bd9Sstevel@tonic-gate /*
14567c478bd9Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust()
14577c478bd9Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is
14587c478bd9Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed.
14597c478bd9Sstevel@tonic-gate */
14607c478bd9Sstevel@tonic-gate amp = sptd->spt_amp;
14617c478bd9Sstevel@tonic-gate ASSERT(amp != NULL);
14627c478bd9Sstevel@tonic-gate
14637c478bd9Sstevel@tonic-gate /* pcachecnt is protected by sptd->spt_lock */
14647c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0);
14657c478bd9Sstevel@tonic-gate pplist = kmem_zalloc(sizeof (page_t *)
14667c478bd9Sstevel@tonic-gate * btopr(sptd->spt_amp->size), KM_SLEEP);
14677c478bd9Sstevel@tonic-gate pl = pplist;
14687c478bd9Sstevel@tonic-gate
14697c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, spt_base);
14707c478bd9Sstevel@tonic-gate
14717c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
14727c478bd9Sstevel@tonic-gate for (a = spt_base; a < (spt_base + sptd->spt_amp->size);
14737c478bd9Sstevel@tonic-gate a += PAGESIZE, anon_index++, pplist++) {
14747c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index);
14757c478bd9Sstevel@tonic-gate ASSERT(ap != NULL);
14767c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &off);
14777c478bd9Sstevel@tonic-gate pp = page_lookup(vp, off, SE_SHARED);
14787c478bd9Sstevel@tonic-gate ASSERT(pp != NULL);
14797c478bd9Sstevel@tonic-gate *pplist = pp;
14807c478bd9Sstevel@tonic-gate }
14817c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
14827c478bd9Sstevel@tonic-gate
14837c478bd9Sstevel@tonic-gate if (a < (spt_base + sptd->spt_amp->size)) {
14847c478bd9Sstevel@tonic-gate ret = ENOTSUP;
14857c478bd9Sstevel@tonic-gate goto insert_fail;
14867c478bd9Sstevel@tonic-gate }
14877c478bd9Sstevel@tonic-gate sptd->spt_ppa = pl;
14887c478bd9Sstevel@tonic-gate } else {
14897c478bd9Sstevel@tonic-gate /*
14907c478bd9Sstevel@tonic-gate * We already have a valid ppa[].
14917c478bd9Sstevel@tonic-gate */
14927c478bd9Sstevel@tonic-gate pl = sptd->spt_ppa;
14937c478bd9Sstevel@tonic-gate }
14947c478bd9Sstevel@tonic-gate
14957c478bd9Sstevel@tonic-gate ASSERT(pl != NULL);
14967c478bd9Sstevel@tonic-gate
1497a98e9dbfSaguzovsk ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1498a98e9dbfSaguzovsk sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1499a98e9dbfSaguzovsk segspt_reclaim);
15007c478bd9Sstevel@tonic-gate if (ret == SEGP_FAIL) {
15017c478bd9Sstevel@tonic-gate /*
15027c478bd9Sstevel@tonic-gate * seg_pinsert failed. We return
15037c478bd9Sstevel@tonic-gate * ENOTSUP, so that the as_pagelock() code will
15047c478bd9Sstevel@tonic-gate * then try the slower F_SOFTLOCK path.
15057c478bd9Sstevel@tonic-gate */
15067c478bd9Sstevel@tonic-gate if (pl_built) {
15077c478bd9Sstevel@tonic-gate /*
15087c478bd9Sstevel@tonic-gate * No one else has referenced the ppa[].
15097c478bd9Sstevel@tonic-gate * We created it and we need to destroy it.
15107c478bd9Sstevel@tonic-gate */
15117c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL;
15127c478bd9Sstevel@tonic-gate }
15137c478bd9Sstevel@tonic-gate ret = ENOTSUP;
15147c478bd9Sstevel@tonic-gate goto insert_fail;
15157c478bd9Sstevel@tonic-gate }
15167c478bd9Sstevel@tonic-gate
15177c478bd9Sstevel@tonic-gate /*
15187c478bd9Sstevel@tonic-gate * In either case, we increment softlockcnt on the 'real' segment.
15197c478bd9Sstevel@tonic-gate */
15207c478bd9Sstevel@tonic-gate sptd->spt_pcachecnt++;
15211a5e258fSJosef 'Jeff' Sipek atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
15227c478bd9Sstevel@tonic-gate
15237c478bd9Sstevel@tonic-gate /*
15247c478bd9Sstevel@tonic-gate * We can now drop the sptd->spt_lock since the ppa[]
152548bbca81SDaniel Hoffman * exists and we have incremented pacachecnt.
15267c478bd9Sstevel@tonic-gate */
15277c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
15287c478bd9Sstevel@tonic-gate
15297c478bd9Sstevel@tonic-gate /*
15307c478bd9Sstevel@tonic-gate * Since we cache the entire segment, we want to
15317c478bd9Sstevel@tonic-gate * set ppp to point to the first slot that corresponds
15327c478bd9Sstevel@tonic-gate * to the requested addr, i.e. page_index.
15337c478bd9Sstevel@tonic-gate */
15347c478bd9Sstevel@tonic-gate *ppp = &(sptd->spt_ppa[page_index]);
1535a98e9dbfSaguzovsk return (0);
15367c478bd9Sstevel@tonic-gate
15377c478bd9Sstevel@tonic-gate insert_fail:
15387c478bd9Sstevel@tonic-gate /*
15397c478bd9Sstevel@tonic-gate * We will only reach this code if we tried and failed.
15407c478bd9Sstevel@tonic-gate *
15417c478bd9Sstevel@tonic-gate * And we can drop the lock on the dummy seg, once we've failed
15427c478bd9Sstevel@tonic-gate * to set up a new ppa[].
15437c478bd9Sstevel@tonic-gate */
15447c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
15457c478bd9Sstevel@tonic-gate
15467c478bd9Sstevel@tonic-gate if (pl_built) {
15477c478bd9Sstevel@tonic-gate /*
15487c478bd9Sstevel@tonic-gate * We created pl and we need to destroy it.
15497c478bd9Sstevel@tonic-gate */
15507c478bd9Sstevel@tonic-gate pplist = pl;
15517c478bd9Sstevel@tonic-gate np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT);
15527c478bd9Sstevel@tonic-gate while (np) {
15537c478bd9Sstevel@tonic-gate page_unlock(*pplist);
15547c478bd9Sstevel@tonic-gate np--;
15557c478bd9Sstevel@tonic-gate pplist++;
15567c478bd9Sstevel@tonic-gate }
15572ba723d8Smec kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size));
15587c478bd9Sstevel@tonic-gate }
15597c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) {
15607c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
15617c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents);
15627c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
15637c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as);
15647c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv);
15657c478bd9Sstevel@tonic-gate }
15667c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents);
15677c478bd9Sstevel@tonic-gate }
15687c478bd9Sstevel@tonic-gate }
15697c478bd9Sstevel@tonic-gate *ppp = NULL;
15707c478bd9Sstevel@tonic-gate return (ret);
15717c478bd9Sstevel@tonic-gate }
15727c478bd9Sstevel@tonic-gate
15737c478bd9Sstevel@tonic-gate /*
15747c478bd9Sstevel@tonic-gate * purge any cached pages in the I/O page cache
15757c478bd9Sstevel@tonic-gate */
15767c478bd9Sstevel@tonic-gate static void
segspt_purge(struct seg * seg)15777c478bd9Sstevel@tonic-gate segspt_purge(struct seg *seg)
15787c478bd9Sstevel@tonic-gate {
1579a98e9dbfSaguzovsk seg_ppurge(seg, NULL, SEGP_FORCE_WIRED);
15807c478bd9Sstevel@tonic-gate }
15817c478bd9Sstevel@tonic-gate
15827c478bd9Sstevel@tonic-gate static int
segspt_reclaim(void * ptag,caddr_t addr,size_t len,struct page ** pplist,enum seg_rw rw,int async)1583a98e9dbfSaguzovsk segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist,
1584a98e9dbfSaguzovsk enum seg_rw rw, int async)
15857c478bd9Sstevel@tonic-gate {
1586a98e9dbfSaguzovsk struct seg *seg = (struct seg *)ptag;
15877c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
15887c478bd9Sstevel@tonic-gate struct seg *sptseg;
15897c478bd9Sstevel@tonic-gate struct spt_data *sptd;
15907c478bd9Sstevel@tonic-gate pgcnt_t npages, i, free_availrmem = 0;
15917c478bd9Sstevel@tonic-gate int done = 0;
15927c478bd9Sstevel@tonic-gate
15937c478bd9Sstevel@tonic-gate #ifdef lint
15947c478bd9Sstevel@tonic-gate addr = addr;
15957c478bd9Sstevel@tonic-gate #endif
15967c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg;
15977c478bd9Sstevel@tonic-gate sptd = sptseg->s_data;
15987c478bd9Sstevel@tonic-gate npages = (len >> PAGESHIFT);
15997c478bd9Sstevel@tonic-gate ASSERT(npages);
16007c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt != 0);
16017c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_ppa == pplist);
16027c478bd9Sstevel@tonic-gate ASSERT(npages == btopr(sptd->spt_amp->size));
1603dc32d872SJosef 'Jeff' Sipek ASSERT(async || AS_LOCK_HELD(seg->s_as));
1604a98e9dbfSaguzovsk
16057c478bd9Sstevel@tonic-gate /*
16067c478bd9Sstevel@tonic-gate * Acquire the lock on the dummy seg and destroy the
16077c478bd9Sstevel@tonic-gate * ppa array IF this is the last pcachecnt.
16087c478bd9Sstevel@tonic-gate */
16097c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
16107c478bd9Sstevel@tonic-gate if (--sptd->spt_pcachecnt == 0) {
16117c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) {
16127c478bd9Sstevel@tonic-gate if (pplist[i] == NULL) {
16137c478bd9Sstevel@tonic-gate continue;
16147c478bd9Sstevel@tonic-gate }
16157c478bd9Sstevel@tonic-gate if (rw == S_WRITE) {
16167c478bd9Sstevel@tonic-gate hat_setrefmod(pplist[i]);
16177c478bd9Sstevel@tonic-gate } else {
16187c478bd9Sstevel@tonic-gate hat_setref(pplist[i]);
16197c478bd9Sstevel@tonic-gate }
16207c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) &&
16217c478bd9Sstevel@tonic-gate (sptd->spt_ppa_lckcnt[i] == 0))
16227c478bd9Sstevel@tonic-gate free_availrmem++;
16237c478bd9Sstevel@tonic-gate page_unlock(pplist[i]);
16247c478bd9Sstevel@tonic-gate }
1625a98e9dbfSaguzovsk if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) {
16267c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock);
16277c478bd9Sstevel@tonic-gate availrmem += free_availrmem;
16287c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock);
16297c478bd9Sstevel@tonic-gate }
16307c478bd9Sstevel@tonic-gate /*
16317c478bd9Sstevel@tonic-gate * Since we want to cach/uncache the entire ISM segment,
16327c478bd9Sstevel@tonic-gate * we will track the pplist in a segspt specific field
16337c478bd9Sstevel@tonic-gate * ppa, that is initialized at the time we add an entry to
16347c478bd9Sstevel@tonic-gate * the cache.
16357c478bd9Sstevel@tonic-gate */
16367c478bd9Sstevel@tonic-gate ASSERT(sptd->spt_pcachecnt == 0);
16377c478bd9Sstevel@tonic-gate kmem_free(pplist, sizeof (page_t *) * npages);
16387c478bd9Sstevel@tonic-gate sptd->spt_ppa = NULL;
16397c478bd9Sstevel@tonic-gate sptd->spt_flags &= ~DISM_PPA_CHANGED;
16402ba723d8Smec sptd->spt_gen++;
16412ba723d8Smec cv_broadcast(&sptd->spt_cv);
16427c478bd9Sstevel@tonic-gate done = 1;
16437c478bd9Sstevel@tonic-gate }
16447c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
1645a98e9dbfSaguzovsk
1646a98e9dbfSaguzovsk /*
1647a98e9dbfSaguzovsk * If we are pcache async thread or called via seg_ppurge_wiredpp() we
1648a98e9dbfSaguzovsk * may not hold AS lock (in this case async argument is not 0). This
1649a98e9dbfSaguzovsk * means if softlockcnt drops to 0 after the decrement below address
1650a98e9dbfSaguzovsk * space may get freed. We can't allow it since after softlock
1651a98e9dbfSaguzovsk * derement to 0 we still need to access as structure for possible
1652a98e9dbfSaguzovsk * wakeup of unmap waiters. To prevent the disappearance of as we take
1653a98e9dbfSaguzovsk * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes
1654a98e9dbfSaguzovsk * this mutex as a barrier to make sure this routine completes before
1655a98e9dbfSaguzovsk * segment is freed.
1656a98e9dbfSaguzovsk *
1657a98e9dbfSaguzovsk * The second complication we have to deal with in async case is a
1658a98e9dbfSaguzovsk * possibility of missed wake up of unmap wait thread. When we don't
1659a98e9dbfSaguzovsk * hold as lock here we may take a_contents lock before unmap wait
1660a98e9dbfSaguzovsk * thread that was first to see softlockcnt was still not 0. As a
1661a98e9dbfSaguzovsk * result we'll fail to wake up an unmap wait thread. To avoid this
1662a98e9dbfSaguzovsk * race we set nounmapwait flag in as structure if we drop softlockcnt
1663a98e9dbfSaguzovsk * to 0 if async is not 0. unmapwait thread
1664a98e9dbfSaguzovsk * will not block if this flag is set.
1665a98e9dbfSaguzovsk */
1666a98e9dbfSaguzovsk if (async)
1667a98e9dbfSaguzovsk mutex_enter(&shmd->shm_segfree_syncmtx);
1668a98e9dbfSaguzovsk
16697c478bd9Sstevel@tonic-gate /*
16707c478bd9Sstevel@tonic-gate * Now decrement softlockcnt.
16717c478bd9Sstevel@tonic-gate */
1672a98e9dbfSaguzovsk ASSERT(shmd->shm_softlockcnt > 0);
16731a5e258fSJosef 'Jeff' Sipek atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
16747c478bd9Sstevel@tonic-gate
16757c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt <= 0) {
1676a98e9dbfSaguzovsk if (async || AS_ISUNMAPWAIT(seg->s_as)) {
16777c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents);
1678a98e9dbfSaguzovsk if (async)
1679a98e9dbfSaguzovsk AS_SETNOUNMAPWAIT(seg->s_as);
16807c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
16817c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as);
16827c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv);
16837c478bd9Sstevel@tonic-gate }
16847c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents);
16857c478bd9Sstevel@tonic-gate }
16867c478bd9Sstevel@tonic-gate }
1687a98e9dbfSaguzovsk
1688a98e9dbfSaguzovsk if (async)
1689a98e9dbfSaguzovsk mutex_exit(&shmd->shm_segfree_syncmtx);
1690a98e9dbfSaguzovsk
16917c478bd9Sstevel@tonic-gate return (done);
16927c478bd9Sstevel@tonic-gate }
16937c478bd9Sstevel@tonic-gate
16947c478bd9Sstevel@tonic-gate /*
16957c478bd9Sstevel@tonic-gate * Do a F_SOFTUNLOCK call over the range requested.
16967c478bd9Sstevel@tonic-gate * The range must have already been F_SOFTLOCK'ed.
16977c478bd9Sstevel@tonic-gate *
16987c478bd9Sstevel@tonic-gate * The calls to acquire and release the anon map lock mutex were
16997c478bd9Sstevel@tonic-gate * removed in order to avoid a deadly embrace during a DR
17007c478bd9Sstevel@tonic-gate * memory delete operation. (Eg. DR blocks while waiting for a
17017c478bd9Sstevel@tonic-gate * exclusive lock on a page that is being used for kaio; the
17027c478bd9Sstevel@tonic-gate * thread that will complete the kaio and call segspt_softunlock
17037c478bd9Sstevel@tonic-gate * blocks on the anon map lock; another thread holding the anon
17047c478bd9Sstevel@tonic-gate * map lock blocks on another page lock via the segspt_shmfault
17057c478bd9Sstevel@tonic-gate * -> page_lookup -> page_lookup_create -> page_lock_es code flow.)
17067c478bd9Sstevel@tonic-gate *
17077c478bd9Sstevel@tonic-gate * The appropriateness of the removal is based upon the following:
17087c478bd9Sstevel@tonic-gate * 1. If we are holding a segment's reader lock and the page is held
17097c478bd9Sstevel@tonic-gate * shared, then the corresponding element in anonmap which points to
17107c478bd9Sstevel@tonic-gate * anon struct cannot change and there is no need to acquire the
17117c478bd9Sstevel@tonic-gate * anonymous map lock.
17127c478bd9Sstevel@tonic-gate * 2. Threads in segspt_softunlock have a reader lock on the segment
17137c478bd9Sstevel@tonic-gate * and already have the shared page lock, so we are guaranteed that
17147c478bd9Sstevel@tonic-gate * the anon map slot cannot change and therefore can call anon_get_ptr()
17157c478bd9Sstevel@tonic-gate * without grabbing the anonymous map lock.
17167c478bd9Sstevel@tonic-gate * 3. Threads that softlock a shared page break copy-on-write, even if
17177c478bd9Sstevel@tonic-gate * its a read. Thus cow faults can be ignored with respect to soft
17187c478bd9Sstevel@tonic-gate * unlocking, since the breaking of cow means that the anon slot(s) will
17197c478bd9Sstevel@tonic-gate * not be shared.
17207c478bd9Sstevel@tonic-gate */
17217c478bd9Sstevel@tonic-gate static void
segspt_softunlock(struct seg * seg,caddr_t sptseg_addr,size_t len,enum seg_rw rw)17227c478bd9Sstevel@tonic-gate segspt_softunlock(struct seg *seg, caddr_t sptseg_addr,
17237c478bd9Sstevel@tonic-gate size_t len, enum seg_rw rw)
17247c478bd9Sstevel@tonic-gate {
17257c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
17267c478bd9Sstevel@tonic-gate struct seg *sptseg;
17277c478bd9Sstevel@tonic-gate struct spt_data *sptd;
17287c478bd9Sstevel@tonic-gate page_t *pp;
17297c478bd9Sstevel@tonic-gate caddr_t adr;
17307c478bd9Sstevel@tonic-gate struct vnode *vp;
17317c478bd9Sstevel@tonic-gate u_offset_t offset;
17327c478bd9Sstevel@tonic-gate ulong_t anon_index;
17337c478bd9Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */
17347c478bd9Sstevel@tonic-gate struct anon *ap = NULL;
17357c478bd9Sstevel@tonic-gate pgcnt_t npages;
17367c478bd9Sstevel@tonic-gate
1737dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
17387c478bd9Sstevel@tonic-gate
17397c478bd9Sstevel@tonic-gate sptseg = shmd->shm_sptseg;
17407c478bd9Sstevel@tonic-gate sptd = sptseg->s_data;
17417c478bd9Sstevel@tonic-gate
17427c478bd9Sstevel@tonic-gate /*
17437c478bd9Sstevel@tonic-gate * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
17447c478bd9Sstevel@tonic-gate * and therefore their pages are SE_SHARED locked
17457c478bd9Sstevel@tonic-gate * for the entire life of the segment.
17467c478bd9Sstevel@tonic-gate */
17477c478bd9Sstevel@tonic-gate if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) &&
17487c478bd9Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) == 0)) {
17497c478bd9Sstevel@tonic-gate goto softlock_decrement;
17507c478bd9Sstevel@tonic-gate }
17517c478bd9Sstevel@tonic-gate
17527c478bd9Sstevel@tonic-gate /*
17537c478bd9Sstevel@tonic-gate * Any thread is free to do a page_find and
17547c478bd9Sstevel@tonic-gate * page_unlock() on the pages within this seg.
17557c478bd9Sstevel@tonic-gate *
17567c478bd9Sstevel@tonic-gate * We are already holding the as->a_lock on the user's
17577c478bd9Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the
17587c478bd9Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the
17597c478bd9Sstevel@tonic-gate * underlying HAT layer.
17607c478bd9Sstevel@tonic-gate */
1761dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(sptseg->s_as, RW_READER);
17627c478bd9Sstevel@tonic-gate hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len);
1763dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(sptseg->s_as);
17647c478bd9Sstevel@tonic-gate
17657c478bd9Sstevel@tonic-gate amp = sptd->spt_amp;
17667c478bd9Sstevel@tonic-gate ASSERT(amp != NULL);
17677c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr);
17687c478bd9Sstevel@tonic-gate
17697c478bd9Sstevel@tonic-gate for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) {
17707c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++);
17717c478bd9Sstevel@tonic-gate ASSERT(ap != NULL);
17727c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &offset);
17737c478bd9Sstevel@tonic-gate
17747c478bd9Sstevel@tonic-gate /*
17757c478bd9Sstevel@tonic-gate * Use page_find() instead of page_lookup() to
17767c478bd9Sstevel@tonic-gate * find the page since we know that it has a
17777c478bd9Sstevel@tonic-gate * "shared" lock.
17787c478bd9Sstevel@tonic-gate */
17797c478bd9Sstevel@tonic-gate pp = page_find(vp, offset);
17807c478bd9Sstevel@tonic-gate ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1));
17817c478bd9Sstevel@tonic-gate if (pp == NULL) {
17827c478bd9Sstevel@tonic-gate panic("segspt_softunlock: "
17837c478bd9Sstevel@tonic-gate "addr %p, ap %p, vp %p, off %llx",
17847c478bd9Sstevel@tonic-gate (void *)adr, (void *)ap, (void *)vp, offset);
17857c478bd9Sstevel@tonic-gate /*NOTREACHED*/
17867c478bd9Sstevel@tonic-gate }
17877c478bd9Sstevel@tonic-gate
17887c478bd9Sstevel@tonic-gate if (rw == S_WRITE) {
17897c478bd9Sstevel@tonic-gate hat_setrefmod(pp);
17907c478bd9Sstevel@tonic-gate } else if (rw != S_OTHER) {
17917c478bd9Sstevel@tonic-gate hat_setref(pp);
17927c478bd9Sstevel@tonic-gate }
17937c478bd9Sstevel@tonic-gate page_unlock(pp);
17947c478bd9Sstevel@tonic-gate }
17957c478bd9Sstevel@tonic-gate
17967c478bd9Sstevel@tonic-gate softlock_decrement:
17977c478bd9Sstevel@tonic-gate npages = btopr(len);
1798a98e9dbfSaguzovsk ASSERT(shmd->shm_softlockcnt >= npages);
17997c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages);
18007c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt == 0) {
18017c478bd9Sstevel@tonic-gate /*
18027c478bd9Sstevel@tonic-gate * All SOFTLOCKS are gone. Wakeup any waiting
18037c478bd9Sstevel@tonic-gate * unmappers so they can try again to unmap.
18047c478bd9Sstevel@tonic-gate * Check for waiters first without the mutex
18057c478bd9Sstevel@tonic-gate * held so we don't always grab the mutex on
18067c478bd9Sstevel@tonic-gate * softunlocks.
18077c478bd9Sstevel@tonic-gate */
18087c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
18097c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents);
18107c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) {
18117c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as);
18127c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv);
18137c478bd9Sstevel@tonic-gate }
18147c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents);
18157c478bd9Sstevel@tonic-gate }
18167c478bd9Sstevel@tonic-gate }
18177c478bd9Sstevel@tonic-gate }
18187c478bd9Sstevel@tonic-gate
18197c478bd9Sstevel@tonic-gate int
segspt_shmattach(struct seg ** segpp,void * argsp)1820284ce987SPatrick Mooney segspt_shmattach(struct seg **segpp, void *argsp)
18217c478bd9Sstevel@tonic-gate {
1822284ce987SPatrick Mooney struct seg *seg = *segpp;
18237c478bd9Sstevel@tonic-gate struct shm_data *shmd_arg = (struct shm_data *)argsp;
18247c478bd9Sstevel@tonic-gate struct shm_data *shmd;
18257c478bd9Sstevel@tonic-gate struct anon_map *shm_amp = shmd_arg->shm_amp;
18267c478bd9Sstevel@tonic-gate struct spt_data *sptd;
18277c478bd9Sstevel@tonic-gate int error = 0;
18287c478bd9Sstevel@tonic-gate
1829dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
18307c478bd9Sstevel@tonic-gate
18317c478bd9Sstevel@tonic-gate shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP);
18327c478bd9Sstevel@tonic-gate if (shmd == NULL)
18337c478bd9Sstevel@tonic-gate return (ENOMEM);
18347c478bd9Sstevel@tonic-gate
18357c478bd9Sstevel@tonic-gate shmd->shm_sptas = shmd_arg->shm_sptas;
18367c478bd9Sstevel@tonic-gate shmd->shm_amp = shm_amp;
18377c478bd9Sstevel@tonic-gate shmd->shm_sptseg = shmd_arg->shm_sptseg;
18387c478bd9Sstevel@tonic-gate
18397c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0,
18407c478bd9Sstevel@tonic-gate NULL, 0, seg->s_size);
18417c478bd9Sstevel@tonic-gate
1842a98e9dbfSaguzovsk mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL);
1843a98e9dbfSaguzovsk
18447c478bd9Sstevel@tonic-gate seg->s_data = (void *)shmd;
18457c478bd9Sstevel@tonic-gate seg->s_ops = &segspt_shmops;
18467c478bd9Sstevel@tonic-gate seg->s_szc = shmd->shm_sptseg->s_szc;
18477c478bd9Sstevel@tonic-gate sptd = shmd->shm_sptseg->s_data;
18487c478bd9Sstevel@tonic-gate
18497c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) {
18507c478bd9Sstevel@tonic-gate if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size),
18517c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) {
18527c478bd9Sstevel@tonic-gate seg->s_data = (void *)NULL;
18537c478bd9Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd)));
18547c478bd9Sstevel@tonic-gate return (ENOMEM);
18557c478bd9Sstevel@tonic-gate }
18567c478bd9Sstevel@tonic-gate shmd->shm_lckpgs = 0;
18577c478bd9Sstevel@tonic-gate if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
18587c478bd9Sstevel@tonic-gate if ((error = hat_share(seg->s_as->a_hat, seg->s_base,
18597c478bd9Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
18607c478bd9Sstevel@tonic-gate seg->s_size, seg->s_szc)) != 0) {
18617c478bd9Sstevel@tonic-gate kmem_free(shmd->shm_vpage,
18627c478bd9Sstevel@tonic-gate btopr(shm_amp->size));
18637c478bd9Sstevel@tonic-gate }
18647c478bd9Sstevel@tonic-gate }
18657c478bd9Sstevel@tonic-gate } else {
18667c478bd9Sstevel@tonic-gate error = hat_share(seg->s_as->a_hat, seg->s_base,
18677c478bd9Sstevel@tonic-gate shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
18687c478bd9Sstevel@tonic-gate seg->s_size, seg->s_szc);
18697c478bd9Sstevel@tonic-gate }
18707c478bd9Sstevel@tonic-gate if (error) {
18717c478bd9Sstevel@tonic-gate seg->s_szc = 0;
18727c478bd9Sstevel@tonic-gate seg->s_data = (void *)NULL;
18737c478bd9Sstevel@tonic-gate kmem_free(shmd, (sizeof (*shmd)));
18747c478bd9Sstevel@tonic-gate } else {
18757c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
18767c478bd9Sstevel@tonic-gate shm_amp->refcnt++;
18777c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock);
18787c478bd9Sstevel@tonic-gate }
18797c478bd9Sstevel@tonic-gate return (error);
18807c478bd9Sstevel@tonic-gate }
18817c478bd9Sstevel@tonic-gate
18827c478bd9Sstevel@tonic-gate int
segspt_shmunmap(struct seg * seg,caddr_t raddr,size_t ssize)18837c478bd9Sstevel@tonic-gate segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize)
18847c478bd9Sstevel@tonic-gate {
18857c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
18867c478bd9Sstevel@tonic-gate int reclaim = 1;
18877c478bd9Sstevel@tonic-gate
1888dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
18897c478bd9Sstevel@tonic-gate retry:
18907c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0) {
18917c478bd9Sstevel@tonic-gate if (reclaim == 1) {
18927c478bd9Sstevel@tonic-gate segspt_purge(seg);
18937c478bd9Sstevel@tonic-gate reclaim = 0;
18947c478bd9Sstevel@tonic-gate goto retry;
18957c478bd9Sstevel@tonic-gate }
18967c478bd9Sstevel@tonic-gate return (EAGAIN);
18977c478bd9Sstevel@tonic-gate }
18987c478bd9Sstevel@tonic-gate
18997c478bd9Sstevel@tonic-gate if (ssize != seg->s_size) {
19007c478bd9Sstevel@tonic-gate #ifdef DEBUG
19017c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n",
19027c478bd9Sstevel@tonic-gate ssize, seg->s_size);
19037c478bd9Sstevel@tonic-gate #endif
19047c478bd9Sstevel@tonic-gate return (EINVAL);
19057c478bd9Sstevel@tonic-gate }
19067c478bd9Sstevel@tonic-gate
19077c478bd9Sstevel@tonic-gate (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK,
19087c478bd9Sstevel@tonic-gate NULL, 0);
19097c478bd9Sstevel@tonic-gate hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc);
19107c478bd9Sstevel@tonic-gate
19117c478bd9Sstevel@tonic-gate seg_free(seg);
19127c478bd9Sstevel@tonic-gate
19137c478bd9Sstevel@tonic-gate return (0);
19147c478bd9Sstevel@tonic-gate }
19157c478bd9Sstevel@tonic-gate
19167c478bd9Sstevel@tonic-gate void
segspt_shmfree(struct seg * seg)19177c478bd9Sstevel@tonic-gate segspt_shmfree(struct seg *seg)
19187c478bd9Sstevel@tonic-gate {
19197c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
19207c478bd9Sstevel@tonic-gate struct anon_map *shm_amp = shmd->shm_amp;
19217c478bd9Sstevel@tonic-gate
1922dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
19237c478bd9Sstevel@tonic-gate
19247c478bd9Sstevel@tonic-gate (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0,
19257c478bd9Sstevel@tonic-gate MC_UNLOCK, NULL, 0);
19267c478bd9Sstevel@tonic-gate
19277c478bd9Sstevel@tonic-gate /*
19287c478bd9Sstevel@tonic-gate * Need to increment refcnt when attaching
19297c478bd9Sstevel@tonic-gate * and decrement when detaching because of dup().
19307c478bd9Sstevel@tonic-gate */
19317c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
19327c478bd9Sstevel@tonic-gate shm_amp->refcnt--;
19337c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&shm_amp->a_rwlock);
19347c478bd9Sstevel@tonic-gate
19357c478bd9Sstevel@tonic-gate if (shmd->shm_vpage) { /* only for DISM */
19367c478bd9Sstevel@tonic-gate kmem_free(shmd->shm_vpage, btopr(shm_amp->size));
19377c478bd9Sstevel@tonic-gate shmd->shm_vpage = NULL;
19387c478bd9Sstevel@tonic-gate }
1939a98e9dbfSaguzovsk
1940a98e9dbfSaguzovsk /*
1941a98e9dbfSaguzovsk * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's
1942a98e9dbfSaguzovsk * still working with this segment without holding as lock.
1943a98e9dbfSaguzovsk */
1944a98e9dbfSaguzovsk ASSERT(shmd->shm_softlockcnt == 0);
1945a98e9dbfSaguzovsk mutex_enter(&shmd->shm_segfree_syncmtx);
1946a98e9dbfSaguzovsk mutex_destroy(&shmd->shm_segfree_syncmtx);
1947a98e9dbfSaguzovsk
19487c478bd9Sstevel@tonic-gate kmem_free(shmd, sizeof (*shmd));
19497c478bd9Sstevel@tonic-gate }
19507c478bd9Sstevel@tonic-gate
19517c478bd9Sstevel@tonic-gate /*ARGSUSED*/
19527c478bd9Sstevel@tonic-gate int
segspt_shmsetprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)19537c478bd9Sstevel@tonic-gate segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
19547c478bd9Sstevel@tonic-gate {
1955dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
19567c478bd9Sstevel@tonic-gate
19577c478bd9Sstevel@tonic-gate /*
19587c478bd9Sstevel@tonic-gate * Shared page table is more than shared mapping.
19597c478bd9Sstevel@tonic-gate * Individual process sharing page tables can't change prot
19607c478bd9Sstevel@tonic-gate * because there is only one set of page tables.
19617c478bd9Sstevel@tonic-gate * This will be allowed after private page table is
19627c478bd9Sstevel@tonic-gate * supported.
19637c478bd9Sstevel@tonic-gate */
19647c478bd9Sstevel@tonic-gate /* need to return correct status error? */
19657c478bd9Sstevel@tonic-gate return (0);
19667c478bd9Sstevel@tonic-gate }
19677c478bd9Sstevel@tonic-gate
19687c478bd9Sstevel@tonic-gate
19697c478bd9Sstevel@tonic-gate faultcode_t
segspt_dismfault(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw)19707c478bd9Sstevel@tonic-gate segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr,
19717c478bd9Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw)
19727c478bd9Sstevel@tonic-gate {
19737c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
19747c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
19757c478bd9Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas;
19767c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data;
19777c478bd9Sstevel@tonic-gate pgcnt_t npages;
197807b65a64Saguzovsk size_t size;
19797c478bd9Sstevel@tonic-gate caddr_t segspt_addr, shm_addr;
19807c478bd9Sstevel@tonic-gate page_t **ppa;
19817c478bd9Sstevel@tonic-gate int i;
19827c478bd9Sstevel@tonic-gate ulong_t an_idx = 0;
19837c478bd9Sstevel@tonic-gate int err = 0;
19841b42782eSmec int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0);
198507b65a64Saguzovsk size_t pgsz;
198607b65a64Saguzovsk pgcnt_t pgcnt;
198707b65a64Saguzovsk caddr_t a;
198807b65a64Saguzovsk pgcnt_t pidx;
19897c478bd9Sstevel@tonic-gate
19907c478bd9Sstevel@tonic-gate #ifdef lint
19917c478bd9Sstevel@tonic-gate hat = hat;
19927c478bd9Sstevel@tonic-gate #endif
1993dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
19947c478bd9Sstevel@tonic-gate
19957c478bd9Sstevel@tonic-gate /*
19967c478bd9Sstevel@tonic-gate * Because of the way spt is implemented
19977c478bd9Sstevel@tonic-gate * the realsize of the segment does not have to be
19987c478bd9Sstevel@tonic-gate * equal to the segment size itself. The segment size is
19997c478bd9Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE.
20007c478bd9Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE
20017c478bd9Sstevel@tonic-gate * based on what the user requested. This is a bit of
20027c478bd9Sstevel@tonic-gate * ungliness that is historical but not easily fixed
20037c478bd9Sstevel@tonic-gate * without re-designing the higher levels of ISM.
20047c478bd9Sstevel@tonic-gate */
20057c478bd9Sstevel@tonic-gate ASSERT(addr >= seg->s_base);
20067c478bd9Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize)
20077c478bd9Sstevel@tonic-gate return (FC_NOMAP);
20087c478bd9Sstevel@tonic-gate /*
20097c478bd9Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to
20107c478bd9Sstevel@tonic-gate * make any necessary adjustments to addr and len
20117c478bd9Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[].
20127c478bd9Sstevel@tonic-gate *
20137c478bd9Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment
20147c478bd9Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore,
20157c478bd9Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
20167c478bd9Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
20177c478bd9Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT
20187c478bd9Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes
20197c478bd9Sstevel@tonic-gate * over a given virtual range.
20207c478bd9Sstevel@tonic-gate */
202107b65a64Saguzovsk pgsz = page_get_pagesize(sptseg->s_szc);
202207b65a64Saguzovsk pgcnt = page_get_pagecnt(sptseg->s_szc);
202307b65a64Saguzovsk shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
202407b65a64Saguzovsk size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
20257c478bd9Sstevel@tonic-gate npages = btopr(size);
20267c478bd9Sstevel@tonic-gate
20277c478bd9Sstevel@tonic-gate /*
20287c478bd9Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt.
20297c478bd9Sstevel@tonic-gate */
20307c478bd9Sstevel@tonic-gate an_idx = seg_page(seg, shm_addr);
20317c478bd9Sstevel@tonic-gate segspt_addr = sptseg->s_base + ptob(an_idx);
20327c478bd9Sstevel@tonic-gate
20337c478bd9Sstevel@tonic-gate ASSERT((segspt_addr + ptob(npages)) <=
20347c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize));
20357c478bd9Sstevel@tonic-gate ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size));
20367c478bd9Sstevel@tonic-gate
20377c478bd9Sstevel@tonic-gate switch (type) {
20387c478bd9Sstevel@tonic-gate
20397c478bd9Sstevel@tonic-gate case F_SOFTLOCK:
20407c478bd9Sstevel@tonic-gate
20417c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
20427c478bd9Sstevel@tonic-gate /*
20437c478bd9Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer
20447c478bd9Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag.
20457c478bd9Sstevel@tonic-gate */
20467c478bd9Sstevel@tonic-gate /* FALLTHRU */
20477c478bd9Sstevel@tonic-gate case F_INVAL:
20487c478bd9Sstevel@tonic-gate
20497c478bd9Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
20507c478bd9Sstevel@tonic-gate return (FC_NOMAP);
20517c478bd9Sstevel@tonic-gate
20527c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
20537c478bd9Sstevel@tonic-gate
20547c478bd9Sstevel@tonic-gate err = spt_anon_getpages(sptseg, segspt_addr, size, ppa);
20557c478bd9Sstevel@tonic-gate if (err != 0) {
20567c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) {
20577c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(
20587c478bd9Sstevel@tonic-gate &(shmd->shm_softlockcnt)), -npages);
20597c478bd9Sstevel@tonic-gate }
20607c478bd9Sstevel@tonic-gate goto dism_err;
20617c478bd9Sstevel@tonic-gate }
2062dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(sptseg->s_as, RW_READER);
206307b65a64Saguzovsk a = segspt_addr;
206407b65a64Saguzovsk pidx = 0;
20657c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) {
20667c478bd9Sstevel@tonic-gate
20677c478bd9Sstevel@tonic-gate /*
20687c478bd9Sstevel@tonic-gate * Load up the translation keeping it
20697c478bd9Sstevel@tonic-gate * locked and don't unlock the page.
20707c478bd9Sstevel@tonic-gate */
207107b65a64Saguzovsk for (; pidx < npages; a += pgsz, pidx += pgcnt) {
207207b65a64Saguzovsk hat_memload_array(sptseg->s_as->a_hat,
207307b65a64Saguzovsk a, pgsz, &ppa[pidx], sptd->spt_prot,
20747c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | HAT_LOAD_SHARE);
207507b65a64Saguzovsk }
20767c478bd9Sstevel@tonic-gate } else {
20777c478bd9Sstevel@tonic-gate /*
20787c478bd9Sstevel@tonic-gate * Migrate pages marked for migration
20797c478bd9Sstevel@tonic-gate */
20807c478bd9Sstevel@tonic-gate if (lgrp_optimizations())
20810d5ae8c1SJosef 'Jeff' Sipek page_migrate(seg, shm_addr, ppa, npages);
20827c478bd9Sstevel@tonic-gate
20830d5ae8c1SJosef 'Jeff' Sipek for (; pidx < npages; a += pgsz, pidx += pgcnt) {
20847c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat,
208507b65a64Saguzovsk a, pgsz, &ppa[pidx],
208607b65a64Saguzovsk sptd->spt_prot,
20877c478bd9Sstevel@tonic-gate HAT_LOAD_SHARE);
208807b65a64Saguzovsk }
20897c478bd9Sstevel@tonic-gate
20907c478bd9Sstevel@tonic-gate /*
20917c478bd9Sstevel@tonic-gate * And now drop the SE_SHARED lock(s).
20927c478bd9Sstevel@tonic-gate */
20931b42782eSmec if (dyn_ism_unmap) {
20941b42782eSmec for (i = 0; i < npages; i++) {
20957c478bd9Sstevel@tonic-gate page_unlock(ppa[i]);
20967c478bd9Sstevel@tonic-gate }
20971b42782eSmec }
20981b42782eSmec }
20997c478bd9Sstevel@tonic-gate
21001b42782eSmec if (!dyn_ism_unmap) {
21017c478bd9Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, shm_addr,
21027c478bd9Sstevel@tonic-gate curspt->a_hat, segspt_addr, ptob(npages),
21037c478bd9Sstevel@tonic-gate seg->s_szc) != 0) {
21047c478bd9Sstevel@tonic-gate panic("hat_share err in DISM fault");
21057c478bd9Sstevel@tonic-gate /* NOTREACHED */
21067c478bd9Sstevel@tonic-gate }
21071b42782eSmec if (type == F_INVAL) {
21081b42782eSmec for (i = 0; i < npages; i++) {
21091b42782eSmec page_unlock(ppa[i]);
21101b42782eSmec }
21111b42782eSmec }
21127c478bd9Sstevel@tonic-gate }
2113dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(sptseg->s_as);
21147c478bd9Sstevel@tonic-gate dism_err:
21157c478bd9Sstevel@tonic-gate kmem_free(ppa, npages * sizeof (page_t *));
21167c478bd9Sstevel@tonic-gate return (err);
21177c478bd9Sstevel@tonic-gate
21187c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK:
21197c478bd9Sstevel@tonic-gate
21207c478bd9Sstevel@tonic-gate /*
21217c478bd9Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer,
21227c478bd9Sstevel@tonic-gate * but the segspt_addr is the virtual address within the
21237c478bd9Sstevel@tonic-gate * dummy seg.
21247c478bd9Sstevel@tonic-gate */
21257c478bd9Sstevel@tonic-gate segspt_softunlock(seg, segspt_addr, size, rw);
21267c478bd9Sstevel@tonic-gate return (0);
21277c478bd9Sstevel@tonic-gate
21287c478bd9Sstevel@tonic-gate case F_PROT:
21297c478bd9Sstevel@tonic-gate
21307c478bd9Sstevel@tonic-gate /*
21317c478bd9Sstevel@tonic-gate * This takes care of the unusual case where a user
21327c478bd9Sstevel@tonic-gate * allocates a stack in shared memory and a register
21337c478bd9Sstevel@tonic-gate * window overflow is written to that stack page before
21347c478bd9Sstevel@tonic-gate * it is otherwise modified.
21357c478bd9Sstevel@tonic-gate *
21367c478bd9Sstevel@tonic-gate * We can get away with this because ISM segments are
21377c478bd9Sstevel@tonic-gate * always rw. Other than this unusual case, there
21387c478bd9Sstevel@tonic-gate * should be no instances of protection violations.
21397c478bd9Sstevel@tonic-gate */
21407c478bd9Sstevel@tonic-gate return (0);
21417c478bd9Sstevel@tonic-gate
21427c478bd9Sstevel@tonic-gate default:
21437c478bd9Sstevel@tonic-gate #ifdef DEBUG
21447c478bd9Sstevel@tonic-gate panic("segspt_dismfault default type?");
21457c478bd9Sstevel@tonic-gate #else
21467c478bd9Sstevel@tonic-gate return (FC_NOMAP);
21477c478bd9Sstevel@tonic-gate #endif
21487c478bd9Sstevel@tonic-gate }
21497c478bd9Sstevel@tonic-gate }
21507c478bd9Sstevel@tonic-gate
21517c478bd9Sstevel@tonic-gate
21527c478bd9Sstevel@tonic-gate faultcode_t
segspt_shmfault(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw)21537c478bd9Sstevel@tonic-gate segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr,
21547c478bd9Sstevel@tonic-gate size_t len, enum fault_type type, enum seg_rw rw)
21557c478bd9Sstevel@tonic-gate {
21567c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
21577c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
21587c478bd9Sstevel@tonic-gate struct as *curspt = shmd->shm_sptas;
21597c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data;
21607c478bd9Sstevel@tonic-gate pgcnt_t npages;
216107b65a64Saguzovsk size_t size;
21627c478bd9Sstevel@tonic-gate caddr_t sptseg_addr, shm_addr;
21637c478bd9Sstevel@tonic-gate page_t *pp, **ppa;
21647c478bd9Sstevel@tonic-gate int i;
21657c478bd9Sstevel@tonic-gate u_offset_t offset;
21667c478bd9Sstevel@tonic-gate ulong_t anon_index = 0;
21677c478bd9Sstevel@tonic-gate struct vnode *vp;
21687c478bd9Sstevel@tonic-gate struct anon_map *amp; /* XXX - for locknest */
21697c478bd9Sstevel@tonic-gate struct anon *ap = NULL;
217007b65a64Saguzovsk size_t pgsz;
217107b65a64Saguzovsk pgcnt_t pgcnt;
217207b65a64Saguzovsk caddr_t a;
217307b65a64Saguzovsk pgcnt_t pidx;
217407b65a64Saguzovsk size_t sz;
21757c478bd9Sstevel@tonic-gate
21767c478bd9Sstevel@tonic-gate #ifdef lint
21777c478bd9Sstevel@tonic-gate hat = hat;
21787c478bd9Sstevel@tonic-gate #endif
21797c478bd9Sstevel@tonic-gate
2180dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
21817c478bd9Sstevel@tonic-gate
21827c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) {
21837c478bd9Sstevel@tonic-gate return (segspt_dismfault(hat, seg, addr, len, type, rw));
21847c478bd9Sstevel@tonic-gate }
21857c478bd9Sstevel@tonic-gate
21867c478bd9Sstevel@tonic-gate /*
21877c478bd9Sstevel@tonic-gate * Because of the way spt is implemented
21887c478bd9Sstevel@tonic-gate * the realsize of the segment does not have to be
21897c478bd9Sstevel@tonic-gate * equal to the segment size itself. The segment size is
21907c478bd9Sstevel@tonic-gate * often in multiples of a page size larger than PAGESIZE.
21917c478bd9Sstevel@tonic-gate * The realsize is rounded up to the nearest PAGESIZE
21927c478bd9Sstevel@tonic-gate * based on what the user requested. This is a bit of
21937c478bd9Sstevel@tonic-gate * ungliness that is historical but not easily fixed
21947c478bd9Sstevel@tonic-gate * without re-designing the higher levels of ISM.
21957c478bd9Sstevel@tonic-gate */
21967c478bd9Sstevel@tonic-gate ASSERT(addr >= seg->s_base);
21977c478bd9Sstevel@tonic-gate if (((addr + len) - seg->s_base) > sptd->spt_realsize)
21987c478bd9Sstevel@tonic-gate return (FC_NOMAP);
21997c478bd9Sstevel@tonic-gate /*
22007c478bd9Sstevel@tonic-gate * For all of the following cases except F_PROT, we need to
22017c478bd9Sstevel@tonic-gate * make any necessary adjustments to addr and len
22027c478bd9Sstevel@tonic-gate * and get all of the necessary page_t's into an array called ppa[].
22037c478bd9Sstevel@tonic-gate *
22047c478bd9Sstevel@tonic-gate * The code in shmat() forces base addr and len of ISM segment
22057c478bd9Sstevel@tonic-gate * to be aligned to largest page size supported. Therefore,
22067c478bd9Sstevel@tonic-gate * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
22077c478bd9Sstevel@tonic-gate * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
22087c478bd9Sstevel@tonic-gate * in large pagesize chunks, or else we will screw up the HAT
22097c478bd9Sstevel@tonic-gate * layer by calling hat_memload_array() with differing page sizes
22107c478bd9Sstevel@tonic-gate * over a given virtual range.
22117c478bd9Sstevel@tonic-gate */
221207b65a64Saguzovsk pgsz = page_get_pagesize(sptseg->s_szc);
221307b65a64Saguzovsk pgcnt = page_get_pagecnt(sptseg->s_szc);
221407b65a64Saguzovsk shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
221507b65a64Saguzovsk size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
22167c478bd9Sstevel@tonic-gate npages = btopr(size);
22177c478bd9Sstevel@tonic-gate
22187c478bd9Sstevel@tonic-gate /*
22197c478bd9Sstevel@tonic-gate * Now we need to convert from addr in segshm to addr in segspt.
22207c478bd9Sstevel@tonic-gate */
22217c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr);
22227c478bd9Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index);
22237c478bd9Sstevel@tonic-gate
22247c478bd9Sstevel@tonic-gate /*
22257c478bd9Sstevel@tonic-gate * And now we may have to adjust npages downward if we have
22267c478bd9Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon
22277c478bd9Sstevel@tonic-gate * allocations.
22287c478bd9Sstevel@tonic-gate */
22297c478bd9Sstevel@tonic-gate if ((sptseg_addr + ptob(npages)) >
22307c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize))
22317c478bd9Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr;
22327c478bd9Sstevel@tonic-gate
22337c478bd9Sstevel@tonic-gate npages = btopr(size);
22347c478bd9Sstevel@tonic-gate
22357c478bd9Sstevel@tonic-gate ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size));
22367c478bd9Sstevel@tonic-gate ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0);
22377c478bd9Sstevel@tonic-gate
22387c478bd9Sstevel@tonic-gate switch (type) {
22397c478bd9Sstevel@tonic-gate
22407c478bd9Sstevel@tonic-gate case F_SOFTLOCK:
22417c478bd9Sstevel@tonic-gate
22427c478bd9Sstevel@tonic-gate /*
22437c478bd9Sstevel@tonic-gate * availrmem is decremented once during anon_swap_adjust()
22447c478bd9Sstevel@tonic-gate * and is incremented during the anon_unresv(), which is
22457c478bd9Sstevel@tonic-gate * called from shm_rm_amp() when the segment is destroyed.
22467c478bd9Sstevel@tonic-gate */
22477c478bd9Sstevel@tonic-gate atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
22487c478bd9Sstevel@tonic-gate /*
22497c478bd9Sstevel@tonic-gate * Some platforms assume that ISM pages are SE_SHARED
22507c478bd9Sstevel@tonic-gate * locked for the entire life of the segment.
22517c478bd9Sstevel@tonic-gate */
22527c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0))
22537c478bd9Sstevel@tonic-gate return (0);
22547c478bd9Sstevel@tonic-gate /*
22557c478bd9Sstevel@tonic-gate * Fall through to the F_INVAL case to load up the hat layer
22567c478bd9Sstevel@tonic-gate * entries with the HAT_LOAD_LOCK flag.
22577c478bd9Sstevel@tonic-gate */
22587c478bd9Sstevel@tonic-gate
22597c478bd9Sstevel@tonic-gate /* FALLTHRU */
22607c478bd9Sstevel@tonic-gate case F_INVAL:
22617c478bd9Sstevel@tonic-gate
22627c478bd9Sstevel@tonic-gate if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
22637c478bd9Sstevel@tonic-gate return (FC_NOMAP);
22647c478bd9Sstevel@tonic-gate
22657c478bd9Sstevel@tonic-gate /*
22667c478bd9Sstevel@tonic-gate * Some platforms that do NOT support DYNAMIC_ISM_UNMAP
22677c478bd9Sstevel@tonic-gate * may still rely on this call to hat_share(). That
22687c478bd9Sstevel@tonic-gate * would imply that those hat's can fault on a
22697c478bd9Sstevel@tonic-gate * HAT_LOAD_LOCK translation, which would seem
22707c478bd9Sstevel@tonic-gate * contradictory.
22717c478bd9Sstevel@tonic-gate */
22727c478bd9Sstevel@tonic-gate if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
22737c478bd9Sstevel@tonic-gate if (hat_share(seg->s_as->a_hat, seg->s_base,
22747c478bd9Sstevel@tonic-gate curspt->a_hat, sptseg->s_base,
22757c478bd9Sstevel@tonic-gate sptseg->s_size, sptseg->s_szc) != 0) {
22767c478bd9Sstevel@tonic-gate panic("hat_share error in ISM fault");
22777c478bd9Sstevel@tonic-gate /*NOTREACHED*/
22787c478bd9Sstevel@tonic-gate }
22797c478bd9Sstevel@tonic-gate return (0);
22807c478bd9Sstevel@tonic-gate }
22817c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP);
22827c478bd9Sstevel@tonic-gate
22837c478bd9Sstevel@tonic-gate /*
22847c478bd9Sstevel@tonic-gate * I see no need to lock the real seg,
22857c478bd9Sstevel@tonic-gate * here, because all of our work will be on the underlying
22867c478bd9Sstevel@tonic-gate * dummy seg.
22877c478bd9Sstevel@tonic-gate *
22887c478bd9Sstevel@tonic-gate * sptseg_addr and npages now account for large pages.
22897c478bd9Sstevel@tonic-gate */
22907c478bd9Sstevel@tonic-gate amp = sptd->spt_amp;
22917c478bd9Sstevel@tonic-gate ASSERT(amp != NULL);
22927c478bd9Sstevel@tonic-gate anon_index = seg_page(sptseg, sptseg_addr);
22937c478bd9Sstevel@tonic-gate
22947c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
22957c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++) {
22967c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index++);
22977c478bd9Sstevel@tonic-gate ASSERT(ap != NULL);
22987c478bd9Sstevel@tonic-gate swap_xlate(ap, &vp, &offset);
22997c478bd9Sstevel@tonic-gate pp = page_lookup(vp, offset, SE_SHARED);
23007c478bd9Sstevel@tonic-gate ASSERT(pp != NULL);
23017c478bd9Sstevel@tonic-gate ppa[i] = pp;
23027c478bd9Sstevel@tonic-gate }
23037c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
23047c478bd9Sstevel@tonic-gate ASSERT(i == npages);
23057c478bd9Sstevel@tonic-gate
23067c478bd9Sstevel@tonic-gate /*
23077c478bd9Sstevel@tonic-gate * We are already holding the as->a_lock on the user's
23087c478bd9Sstevel@tonic-gate * real segment, but we need to hold the a_lock on the
23097c478bd9Sstevel@tonic-gate * underlying dummy as. This is mostly to satisfy the
23107c478bd9Sstevel@tonic-gate * underlying HAT layer.
23117c478bd9Sstevel@tonic-gate */
2312dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(sptseg->s_as, RW_READER);
231307b65a64Saguzovsk a = sptseg_addr;
231407b65a64Saguzovsk pidx = 0;
23157c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) {
23167c478bd9Sstevel@tonic-gate /*
23177c478bd9Sstevel@tonic-gate * Load up the translation keeping it
23187c478bd9Sstevel@tonic-gate * locked and don't unlock the page.
23197c478bd9Sstevel@tonic-gate */
232007b65a64Saguzovsk for (; pidx < npages; a += pgsz, pidx += pgcnt) {
232107b65a64Saguzovsk sz = MIN(pgsz, ptob(npages - pidx));
232207b65a64Saguzovsk hat_memload_array(sptseg->s_as->a_hat, a,
232307b65a64Saguzovsk sz, &ppa[pidx], sptd->spt_prot,
23247c478bd9Sstevel@tonic-gate HAT_LOAD_LOCK | HAT_LOAD_SHARE);
232507b65a64Saguzovsk }
23267c478bd9Sstevel@tonic-gate } else {
23277c478bd9Sstevel@tonic-gate /*
23287c478bd9Sstevel@tonic-gate * Migrate pages marked for migration.
23297c478bd9Sstevel@tonic-gate */
23307c478bd9Sstevel@tonic-gate if (lgrp_optimizations())
23310d5ae8c1SJosef 'Jeff' Sipek page_migrate(seg, shm_addr, ppa, npages);
23327c478bd9Sstevel@tonic-gate
23330d5ae8c1SJosef 'Jeff' Sipek for (; pidx < npages; a += pgsz, pidx += pgcnt) {
233407b65a64Saguzovsk sz = MIN(pgsz, ptob(npages - pidx));
23357c478bd9Sstevel@tonic-gate hat_memload_array(sptseg->s_as->a_hat,
233607b65a64Saguzovsk a, sz, &ppa[pidx],
23377c478bd9Sstevel@tonic-gate sptd->spt_prot, HAT_LOAD_SHARE);
233807b65a64Saguzovsk }
23397c478bd9Sstevel@tonic-gate
23407c478bd9Sstevel@tonic-gate /*
23417c478bd9Sstevel@tonic-gate * And now drop the SE_SHARED lock(s).
23427c478bd9Sstevel@tonic-gate */
23437c478bd9Sstevel@tonic-gate for (i = 0; i < npages; i++)
23447c478bd9Sstevel@tonic-gate page_unlock(ppa[i]);
23457c478bd9Sstevel@tonic-gate }
2346dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(sptseg->s_as);
23477c478bd9Sstevel@tonic-gate
23487c478bd9Sstevel@tonic-gate kmem_free(ppa, sizeof (page_t *) * npages);
23497c478bd9Sstevel@tonic-gate return (0);
23507c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK:
23517c478bd9Sstevel@tonic-gate
23527c478bd9Sstevel@tonic-gate /*
23537c478bd9Sstevel@tonic-gate * This is a bit ugly, we pass in the real seg pointer,
23547c478bd9Sstevel@tonic-gate * but the sptseg_addr is the virtual address within the
23557c478bd9Sstevel@tonic-gate * dummy seg.
23567c478bd9Sstevel@tonic-gate */
23577c478bd9Sstevel@tonic-gate segspt_softunlock(seg, sptseg_addr, ptob(npages), rw);
23587c478bd9Sstevel@tonic-gate return (0);
23597c478bd9Sstevel@tonic-gate
23607c478bd9Sstevel@tonic-gate case F_PROT:
23617c478bd9Sstevel@tonic-gate
23627c478bd9Sstevel@tonic-gate /*
23637c478bd9Sstevel@tonic-gate * This takes care of the unusual case where a user
23647c478bd9Sstevel@tonic-gate * allocates a stack in shared memory and a register
23657c478bd9Sstevel@tonic-gate * window overflow is written to that stack page before
23667c478bd9Sstevel@tonic-gate * it is otherwise modified.
23677c478bd9Sstevel@tonic-gate *
23687c478bd9Sstevel@tonic-gate * We can get away with this because ISM segments are
23697c478bd9Sstevel@tonic-gate * always rw. Other than this unusual case, there
23707c478bd9Sstevel@tonic-gate * should be no instances of protection violations.
23717c478bd9Sstevel@tonic-gate */
23727c478bd9Sstevel@tonic-gate return (0);
23737c478bd9Sstevel@tonic-gate
23747c478bd9Sstevel@tonic-gate default:
23757c478bd9Sstevel@tonic-gate #ifdef DEBUG
23767c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "segspt_shmfault default type?");
23777c478bd9Sstevel@tonic-gate #endif
23787c478bd9Sstevel@tonic-gate return (FC_NOMAP);
23797c478bd9Sstevel@tonic-gate }
23807c478bd9Sstevel@tonic-gate }
23817c478bd9Sstevel@tonic-gate
23827c478bd9Sstevel@tonic-gate /*ARGSUSED*/
23837c478bd9Sstevel@tonic-gate static faultcode_t
segspt_shmfaulta(struct seg * seg,caddr_t addr)23847c478bd9Sstevel@tonic-gate segspt_shmfaulta(struct seg *seg, caddr_t addr)
23857c478bd9Sstevel@tonic-gate {
23867c478bd9Sstevel@tonic-gate return (0);
23877c478bd9Sstevel@tonic-gate }
23887c478bd9Sstevel@tonic-gate
23897c478bd9Sstevel@tonic-gate /*ARGSUSED*/
23907c478bd9Sstevel@tonic-gate static int
segspt_shmkluster(struct seg * seg,caddr_t addr,ssize_t delta)23917c478bd9Sstevel@tonic-gate segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta)
23927c478bd9Sstevel@tonic-gate {
23937c478bd9Sstevel@tonic-gate return (0);
23947c478bd9Sstevel@tonic-gate }
23957c478bd9Sstevel@tonic-gate
23967c478bd9Sstevel@tonic-gate /*ARGSUSED*/
23977c478bd9Sstevel@tonic-gate static size_t
segspt_shmswapout(struct seg * seg)23987c478bd9Sstevel@tonic-gate segspt_shmswapout(struct seg *seg)
23997c478bd9Sstevel@tonic-gate {
24007c478bd9Sstevel@tonic-gate return (0);
24017c478bd9Sstevel@tonic-gate }
24027c478bd9Sstevel@tonic-gate
24037c478bd9Sstevel@tonic-gate /*
24047c478bd9Sstevel@tonic-gate * duplicate the shared page tables
24057c478bd9Sstevel@tonic-gate */
24067c478bd9Sstevel@tonic-gate int
segspt_shmdup(struct seg * seg,struct seg * newseg)24077c478bd9Sstevel@tonic-gate segspt_shmdup(struct seg *seg, struct seg *newseg)
24087c478bd9Sstevel@tonic-gate {
24097c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
24107c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp;
24117c478bd9Sstevel@tonic-gate struct shm_data *shmd_new;
24127c478bd9Sstevel@tonic-gate struct seg *spt_seg = shmd->shm_sptseg;
24137c478bd9Sstevel@tonic-gate struct spt_data *sptd = spt_seg->s_data;
24141b42782eSmec int error = 0;
24157c478bd9Sstevel@tonic-gate
2416dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
24177c478bd9Sstevel@tonic-gate
24187c478bd9Sstevel@tonic-gate shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP);
24197c478bd9Sstevel@tonic-gate newseg->s_data = (void *)shmd_new;
24207c478bd9Sstevel@tonic-gate shmd_new->shm_sptas = shmd->shm_sptas;
24217c478bd9Sstevel@tonic-gate shmd_new->shm_amp = amp;
24227c478bd9Sstevel@tonic-gate shmd_new->shm_sptseg = shmd->shm_sptseg;
24237c478bd9Sstevel@tonic-gate newseg->s_ops = &segspt_shmops;
24247c478bd9Sstevel@tonic-gate newseg->s_szc = seg->s_szc;
24257c478bd9Sstevel@tonic-gate ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc);
24267c478bd9Sstevel@tonic-gate
24277c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
24287c478bd9Sstevel@tonic-gate amp->refcnt++;
24297c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
24307c478bd9Sstevel@tonic-gate
24317c478bd9Sstevel@tonic-gate if (sptd->spt_flags & SHM_PAGEABLE) {
24327c478bd9Sstevel@tonic-gate shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP);
24337c478bd9Sstevel@tonic-gate shmd_new->shm_lckpgs = 0;
24341b42782eSmec if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
24351b42782eSmec if ((error = hat_share(newseg->s_as->a_hat,
24361b42782eSmec newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR,
24371b42782eSmec seg->s_size, seg->s_szc)) != 0) {
24381b42782eSmec kmem_free(shmd_new->shm_vpage,
24391b42782eSmec btopr(amp->size));
24407c478bd9Sstevel@tonic-gate }
24411b42782eSmec }
24421b42782eSmec return (error);
24431b42782eSmec } else {
24447c478bd9Sstevel@tonic-gate return (hat_share(newseg->s_as->a_hat, newseg->s_base,
24451b42782eSmec shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size,
24461b42782eSmec seg->s_szc));
24471b42782eSmec
24481b42782eSmec }
24497c478bd9Sstevel@tonic-gate }
24507c478bd9Sstevel@tonic-gate
24517c478bd9Sstevel@tonic-gate /*ARGSUSED*/
24527c478bd9Sstevel@tonic-gate int
segspt_shmcheckprot(struct seg * seg,caddr_t addr,size_t size,uint_t prot)24537c478bd9Sstevel@tonic-gate segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
24547c478bd9Sstevel@tonic-gate {
24557c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
24567c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
24577c478bd9Sstevel@tonic-gate
2458dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
24597c478bd9Sstevel@tonic-gate
24607c478bd9Sstevel@tonic-gate /*
24617c478bd9Sstevel@tonic-gate * ISM segment is always rw.
24627c478bd9Sstevel@tonic-gate */
24637c478bd9Sstevel@tonic-gate return (((sptd->spt_prot & prot) != prot) ? EACCES : 0);
24647c478bd9Sstevel@tonic-gate }
24657c478bd9Sstevel@tonic-gate
24667c478bd9Sstevel@tonic-gate /*
24677c478bd9Sstevel@tonic-gate * Return an array of locked large pages, for empty slots allocate
24687c478bd9Sstevel@tonic-gate * private zero-filled anon pages.
24697c478bd9Sstevel@tonic-gate */
24707c478bd9Sstevel@tonic-gate static int
spt_anon_getpages(struct seg * sptseg,caddr_t sptaddr,size_t len,page_t * ppa[])24717c478bd9Sstevel@tonic-gate spt_anon_getpages(
24727c478bd9Sstevel@tonic-gate struct seg *sptseg,
24737c478bd9Sstevel@tonic-gate caddr_t sptaddr,
24747c478bd9Sstevel@tonic-gate size_t len,
24757c478bd9Sstevel@tonic-gate page_t *ppa[])
24767c478bd9Sstevel@tonic-gate {
24777c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data;
24787c478bd9Sstevel@tonic-gate struct anon_map *amp = sptd->spt_amp;
24797c478bd9Sstevel@tonic-gate enum seg_rw rw = sptd->spt_prot;
24807c478bd9Sstevel@tonic-gate uint_t szc = sptseg->s_szc;
24817c478bd9Sstevel@tonic-gate size_t pg_sz, share_sz = page_get_pagesize(szc);
24827c478bd9Sstevel@tonic-gate pgcnt_t lp_npgs;
24837c478bd9Sstevel@tonic-gate caddr_t lp_addr, e_sptaddr;
24847c478bd9Sstevel@tonic-gate uint_t vpprot, ppa_szc = 0;
24857c478bd9Sstevel@tonic-gate struct vpage *vpage = NULL;
24867c478bd9Sstevel@tonic-gate ulong_t j, ppa_idx;
24877c478bd9Sstevel@tonic-gate int err, ierr = 0;
24887c478bd9Sstevel@tonic-gate pgcnt_t an_idx;
24897c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie;
24902ba723d8Smec int anon_locked = 0;
24912ba723d8Smec pgcnt_t amp_pgs;
24922ba723d8Smec
24937c478bd9Sstevel@tonic-gate
24947c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz));
24957c478bd9Sstevel@tonic-gate ASSERT(len != 0);
24967c478bd9Sstevel@tonic-gate
24977c478bd9Sstevel@tonic-gate pg_sz = share_sz;
24987c478bd9Sstevel@tonic-gate lp_npgs = btop(pg_sz);
24997c478bd9Sstevel@tonic-gate lp_addr = sptaddr;
25007c478bd9Sstevel@tonic-gate e_sptaddr = sptaddr + len;
25017c478bd9Sstevel@tonic-gate an_idx = seg_page(sptseg, sptaddr);
25027c478bd9Sstevel@tonic-gate ppa_idx = 0;
25037c478bd9Sstevel@tonic-gate
25047c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
25052ba723d8Smec
25062ba723d8Smec amp_pgs = page_get_pagecnt(amp->a_szc);
25072ba723d8Smec
25087c478bd9Sstevel@tonic-gate /*CONSTCOND*/
25097c478bd9Sstevel@tonic-gate while (1) {
25107c478bd9Sstevel@tonic-gate for (; lp_addr < e_sptaddr;
25112ba723d8Smec an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) {
25127c478bd9Sstevel@tonic-gate
25132ba723d8Smec /*
25142ba723d8Smec * If we're currently locked, and we get to a new
25152ba723d8Smec * page, unlock our current anon chunk.
25162ba723d8Smec */
25172ba723d8Smec if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) {
25182ba723d8Smec anon_array_exit(&cookie);
25192ba723d8Smec anon_locked = 0;
25202ba723d8Smec }
25212ba723d8Smec if (!anon_locked) {
25227c478bd9Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie);
25232ba723d8Smec anon_locked = 1;
25242ba723d8Smec }
25257c478bd9Sstevel@tonic-gate ppa_szc = (uint_t)-1;
25267c478bd9Sstevel@tonic-gate ierr = anon_map_getpages(amp, an_idx, szc, sptseg,
25277c478bd9Sstevel@tonic-gate lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx],
25282cb27123Saguzovsk &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred);
25297c478bd9Sstevel@tonic-gate
25307c478bd9Sstevel@tonic-gate if (ierr != 0) {
25317c478bd9Sstevel@tonic-gate if (ierr > 0) {
25327c478bd9Sstevel@tonic-gate err = FC_MAKE_ERR(ierr);
25337c478bd9Sstevel@tonic-gate goto lpgs_err;
25347c478bd9Sstevel@tonic-gate }
25357c478bd9Sstevel@tonic-gate break;
25367c478bd9Sstevel@tonic-gate }
25377c478bd9Sstevel@tonic-gate }
25387c478bd9Sstevel@tonic-gate if (lp_addr == e_sptaddr) {
25397c478bd9Sstevel@tonic-gate break;
25407c478bd9Sstevel@tonic-gate }
25417c478bd9Sstevel@tonic-gate ASSERT(lp_addr < e_sptaddr);
25427c478bd9Sstevel@tonic-gate
25437c478bd9Sstevel@tonic-gate /*
25447c478bd9Sstevel@tonic-gate * ierr == -1 means we failed to allocate a large page.
25457c478bd9Sstevel@tonic-gate * so do a size down operation.
25467c478bd9Sstevel@tonic-gate *
25477c478bd9Sstevel@tonic-gate * ierr == -2 means some other process that privately shares
25487c478bd9Sstevel@tonic-gate * pages with this process has allocated a larger page and we
25497c478bd9Sstevel@tonic-gate * need to retry with larger pages. So do a size up
25507c478bd9Sstevel@tonic-gate * operation. This relies on the fact that large pages are
25517c478bd9Sstevel@tonic-gate * never partially shared i.e. if we share any constituent
25527c478bd9Sstevel@tonic-gate * page of a large page with another process we must share the
25537c478bd9Sstevel@tonic-gate * entire large page. Note this cannot happen for SOFTLOCK
25547c478bd9Sstevel@tonic-gate * case, unless current address (lpaddr) is at the beginning
25557c478bd9Sstevel@tonic-gate * of the next page size boundary because the other process
25567c478bd9Sstevel@tonic-gate * couldn't have relocated locked pages.
25577c478bd9Sstevel@tonic-gate */
25587c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || ierr == -2);
25597c478bd9Sstevel@tonic-gate if (segvn_anypgsz) {
25607c478bd9Sstevel@tonic-gate ASSERT(ierr == -2 || szc != 0);
25617c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || szc < sptseg->s_szc);
25627c478bd9Sstevel@tonic-gate szc = (ierr == -1) ? szc - 1 : szc + 1;
25637c478bd9Sstevel@tonic-gate } else {
25647c478bd9Sstevel@tonic-gate /*
25657c478bd9Sstevel@tonic-gate * For faults and segvn_anypgsz == 0
25667c478bd9Sstevel@tonic-gate * we need to be careful not to loop forever
25677c478bd9Sstevel@tonic-gate * if existing page is found with szc other
25687c478bd9Sstevel@tonic-gate * than 0 or seg->s_szc. This could be due
25697c478bd9Sstevel@tonic-gate * to page relocations on behalf of DR or
25707c478bd9Sstevel@tonic-gate * more likely large page creation. For this
25717c478bd9Sstevel@tonic-gate * case simply re-size to existing page's szc
25727c478bd9Sstevel@tonic-gate * if returned by anon_map_getpages().
25737c478bd9Sstevel@tonic-gate */
25747c478bd9Sstevel@tonic-gate if (ppa_szc == (uint_t)-1) {
25757c478bd9Sstevel@tonic-gate szc = (ierr == -1) ? 0 : sptseg->s_szc;
25767c478bd9Sstevel@tonic-gate } else {
25777c478bd9Sstevel@tonic-gate ASSERT(ppa_szc <= sptseg->s_szc);
25787c478bd9Sstevel@tonic-gate ASSERT(ierr == -2 || ppa_szc < szc);
25797c478bd9Sstevel@tonic-gate ASSERT(ierr == -1 || ppa_szc > szc);
25807c478bd9Sstevel@tonic-gate szc = ppa_szc;
25817c478bd9Sstevel@tonic-gate }
25827c478bd9Sstevel@tonic-gate }
25837c478bd9Sstevel@tonic-gate pg_sz = page_get_pagesize(szc);
25847c478bd9Sstevel@tonic-gate lp_npgs = btop(pg_sz);
25857c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(lp_addr, pg_sz));
25867c478bd9Sstevel@tonic-gate }
25872ba723d8Smec if (anon_locked) {
25882ba723d8Smec anon_array_exit(&cookie);
25892ba723d8Smec }
25907c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
25917c478bd9Sstevel@tonic-gate return (0);
25927c478bd9Sstevel@tonic-gate
25937c478bd9Sstevel@tonic-gate lpgs_err:
25942ba723d8Smec if (anon_locked) {
25952ba723d8Smec anon_array_exit(&cookie);
25962ba723d8Smec }
25977c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
25987c478bd9Sstevel@tonic-gate for (j = 0; j < ppa_idx; j++)
25997c478bd9Sstevel@tonic-gate page_unlock(ppa[j]);
26007c478bd9Sstevel@tonic-gate return (err);
26017c478bd9Sstevel@tonic-gate }
26027c478bd9Sstevel@tonic-gate
2603c6939658Ssl108498 /*
2604c6939658Ssl108498 * count the number of bytes in a set of spt pages that are currently not
2605c6939658Ssl108498 * locked
2606c6939658Ssl108498 */
2607c6939658Ssl108498 static rctl_qty_t
spt_unlockedbytes(pgcnt_t npages,page_t ** ppa)2608c6939658Ssl108498 spt_unlockedbytes(pgcnt_t npages, page_t **ppa)
2609c6939658Ssl108498 {
2610c6939658Ssl108498 ulong_t i;
2611c6939658Ssl108498 rctl_qty_t unlocked = 0;
2612c6939658Ssl108498
2613c6939658Ssl108498 for (i = 0; i < npages; i++) {
2614c6939658Ssl108498 if (ppa[i]->p_lckcnt == 0)
2615c6939658Ssl108498 unlocked += PAGESIZE;
2616c6939658Ssl108498 }
2617c6939658Ssl108498 return (unlocked);
2618c6939658Ssl108498 }
2619c6939658Ssl108498
2620b52a336eSPavel Tatashin extern u_longlong_t randtick(void);
2621b52a336eSPavel Tatashin /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */
2622b52a336eSPavel Tatashin #define NLCK (NCPU_P2)
2623b52a336eSPavel Tatashin /* Random number with a range [0, n-1], n must be power of two */
2624b52a336eSPavel Tatashin #define RAND_P2(n) \
2625b52a336eSPavel Tatashin ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1))
2626b52a336eSPavel Tatashin
26277c478bd9Sstevel@tonic-gate int
spt_lockpages(struct seg * seg,pgcnt_t anon_index,pgcnt_t npages,page_t ** ppa,ulong_t * lockmap,size_t pos,rctl_qty_t * locked)26287c478bd9Sstevel@tonic-gate spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2629c6939658Ssl108498 page_t **ppa, ulong_t *lockmap, size_t pos,
2630c6939658Ssl108498 rctl_qty_t *locked)
26317c478bd9Sstevel@tonic-gate {
26327c478bd9Sstevel@tonic-gate struct shm_data *shmd = seg->s_data;
26337c478bd9Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data;
26347c478bd9Sstevel@tonic-gate ulong_t i;
26357c478bd9Sstevel@tonic-gate int kernel;
2636b52a336eSPavel Tatashin pgcnt_t nlck = 0;
2637b52a336eSPavel Tatashin int rv = 0;
2638b52a336eSPavel Tatashin int use_reserved = 1;
26397c478bd9Sstevel@tonic-gate
2640c6939658Ssl108498 /* return the number of bytes actually locked */
2641c6939658Ssl108498 *locked = 0;
2642b52a336eSPavel Tatashin
2643b52a336eSPavel Tatashin /*
2644b52a336eSPavel Tatashin * To avoid contention on freemem_lock, availrmem and pages_locked
2645b52a336eSPavel Tatashin * global counters are updated only every nlck locked pages instead of
2646b52a336eSPavel Tatashin * every time. Reserve nlck locks up front and deduct from this
2647b52a336eSPavel Tatashin * reservation for each page that requires a lock. When the reservation
2648b52a336eSPavel Tatashin * is consumed, reserve again. nlck is randomized, so the competing
2649b52a336eSPavel Tatashin * threads do not fall into a cyclic lock contention pattern. When
2650b52a336eSPavel Tatashin * memory is low, the lock ahead is disabled, and instead page_pp_lock()
2651b52a336eSPavel Tatashin * is used to lock pages.
2652b52a336eSPavel Tatashin */
26537c478bd9Sstevel@tonic-gate for (i = 0; i < npages; anon_index++, pos++, i++) {
2654b52a336eSPavel Tatashin if (nlck == 0 && use_reserved == 1) {
2655b52a336eSPavel Tatashin nlck = NLCK + RAND_P2(NLCK);
2656b52a336eSPavel Tatashin /* if fewer loops left, decrease nlck */
2657b52a336eSPavel Tatashin nlck = MIN(nlck, npages - i);
2658b52a336eSPavel Tatashin /*
2659b52a336eSPavel Tatashin * Reserve nlck locks up front and deduct from this
2660b52a336eSPavel Tatashin * reservation for each page that requires a lock. When
2661b52a336eSPavel Tatashin * the reservation is consumed, reserve again.
2662b52a336eSPavel Tatashin */
2663b52a336eSPavel Tatashin mutex_enter(&freemem_lock);
2664b52a336eSPavel Tatashin if ((availrmem - nlck) < pages_pp_maximum) {
2665b52a336eSPavel Tatashin /* Do not do advance memory reserves */
2666b52a336eSPavel Tatashin use_reserved = 0;
2667b52a336eSPavel Tatashin } else {
2668b52a336eSPavel Tatashin availrmem -= nlck;
2669b52a336eSPavel Tatashin pages_locked += nlck;
2670b52a336eSPavel Tatashin }
2671b52a336eSPavel Tatashin mutex_exit(&freemem_lock);
2672b52a336eSPavel Tatashin }
26737c478bd9Sstevel@tonic-gate if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
26747c478bd9Sstevel@tonic-gate if (sptd->spt_ppa_lckcnt[anon_index] <
26757c478bd9Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) {
26767c478bd9Sstevel@tonic-gate if (++sptd->spt_ppa_lckcnt[anon_index] ==
26777c478bd9Sstevel@tonic-gate (ushort_t)DISM_LOCK_MAX) {
26787c478bd9Sstevel@tonic-gate cmn_err(CE_WARN,
26797c478bd9Sstevel@tonic-gate "DISM page lock limit "
26807c478bd9Sstevel@tonic-gate "reached on DISM offset 0x%lx\n",
26817c478bd9Sstevel@tonic-gate anon_index << PAGESHIFT);
26827c478bd9Sstevel@tonic-gate }
26837c478bd9Sstevel@tonic-gate kernel = (sptd->spt_ppa &&
2684b52a336eSPavel Tatashin sptd->spt_ppa[anon_index]);
2685b52a336eSPavel Tatashin if (!page_pp_lock(ppa[i], 0, kernel ||
2686b52a336eSPavel Tatashin use_reserved)) {
26877c478bd9Sstevel@tonic-gate sptd->spt_ppa_lckcnt[anon_index]--;
2688b52a336eSPavel Tatashin rv = EAGAIN;
2689b52a336eSPavel Tatashin break;
26907c478bd9Sstevel@tonic-gate }
2691c6939658Ssl108498 /* if this is a newly locked page, count it */
2692c6939658Ssl108498 if (ppa[i]->p_lckcnt == 1) {
2693b52a336eSPavel Tatashin if (kernel == 0 && use_reserved == 1)
2694b52a336eSPavel Tatashin nlck--;
2695c6939658Ssl108498 *locked += PAGESIZE;
2696c6939658Ssl108498 }
26977c478bd9Sstevel@tonic-gate shmd->shm_lckpgs++;
26987c478bd9Sstevel@tonic-gate shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED;
26997c478bd9Sstevel@tonic-gate if (lockmap != NULL)
27007c478bd9Sstevel@tonic-gate BT_SET(lockmap, pos);
27017c478bd9Sstevel@tonic-gate }
27027c478bd9Sstevel@tonic-gate }
27037c478bd9Sstevel@tonic-gate }
2704b52a336eSPavel Tatashin /* Return unused lock reservation */
2705b52a336eSPavel Tatashin if (nlck != 0 && use_reserved == 1) {
2706b52a336eSPavel Tatashin mutex_enter(&freemem_lock);
2707b52a336eSPavel Tatashin availrmem += nlck;
2708b52a336eSPavel Tatashin pages_locked -= nlck;
2709b52a336eSPavel Tatashin mutex_exit(&freemem_lock);
2710b52a336eSPavel Tatashin }
2711b52a336eSPavel Tatashin
2712b52a336eSPavel Tatashin return (rv);
2713b52a336eSPavel Tatashin }
2714b52a336eSPavel Tatashin
2715b52a336eSPavel Tatashin int
spt_unlockpages(struct seg * seg,pgcnt_t anon_index,pgcnt_t npages,rctl_qty_t * unlocked)2716b52a336eSPavel Tatashin spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2717b52a336eSPavel Tatashin rctl_qty_t *unlocked)
2718b52a336eSPavel Tatashin {
2719b52a336eSPavel Tatashin struct shm_data *shmd = seg->s_data;
2720b52a336eSPavel Tatashin struct spt_data *sptd = shmd->shm_sptseg->s_data;
2721b52a336eSPavel Tatashin struct anon_map *amp = sptd->spt_amp;
2722b52a336eSPavel Tatashin struct anon *ap;
2723b52a336eSPavel Tatashin struct vnode *vp;
2724b52a336eSPavel Tatashin u_offset_t off;
2725b52a336eSPavel Tatashin struct page *pp;
2726b52a336eSPavel Tatashin int kernel;
2727b52a336eSPavel Tatashin anon_sync_obj_t cookie;
2728b52a336eSPavel Tatashin ulong_t i;
2729b52a336eSPavel Tatashin pgcnt_t nlck = 0;
2730b52a336eSPavel Tatashin pgcnt_t nlck_limit = NLCK;
2731b52a336eSPavel Tatashin
2732b52a336eSPavel Tatashin ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
2733b52a336eSPavel Tatashin for (i = 0; i < npages; i++, anon_index++) {
2734b52a336eSPavel Tatashin if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
2735b52a336eSPavel Tatashin anon_array_enter(amp, anon_index, &cookie);
2736b52a336eSPavel Tatashin ap = anon_get_ptr(amp->ahp, anon_index);
2737b52a336eSPavel Tatashin ASSERT(ap);
2738b52a336eSPavel Tatashin
2739b52a336eSPavel Tatashin swap_xlate(ap, &vp, &off);
2740b52a336eSPavel Tatashin anon_array_exit(&cookie);
2741b52a336eSPavel Tatashin pp = page_lookup(vp, off, SE_SHARED);
2742b52a336eSPavel Tatashin ASSERT(pp);
2743b52a336eSPavel Tatashin /*
2744b52a336eSPavel Tatashin * availrmem is decremented only for pages which are not
2745b52a336eSPavel Tatashin * in seg pcache, for pages in seg pcache availrmem was
2746b52a336eSPavel Tatashin * decremented in _dismpagelock()
2747b52a336eSPavel Tatashin */
2748b52a336eSPavel Tatashin kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]);
2749b52a336eSPavel Tatashin ASSERT(pp->p_lckcnt > 0);
2750b52a336eSPavel Tatashin
2751b52a336eSPavel Tatashin /*
2752b52a336eSPavel Tatashin * lock page but do not change availrmem, we do it
2753b52a336eSPavel Tatashin * ourselves every nlck loops.
2754b52a336eSPavel Tatashin */
2755b52a336eSPavel Tatashin page_pp_unlock(pp, 0, 1);
2756b52a336eSPavel Tatashin if (pp->p_lckcnt == 0) {
2757b52a336eSPavel Tatashin if (kernel == 0)
2758b52a336eSPavel Tatashin nlck++;
2759b52a336eSPavel Tatashin *unlocked += PAGESIZE;
2760b52a336eSPavel Tatashin }
2761b52a336eSPavel Tatashin page_unlock(pp);
2762b52a336eSPavel Tatashin shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED;
2763b52a336eSPavel Tatashin sptd->spt_ppa_lckcnt[anon_index]--;
2764b52a336eSPavel Tatashin shmd->shm_lckpgs--;
2765b52a336eSPavel Tatashin }
2766b52a336eSPavel Tatashin
2767b52a336eSPavel Tatashin /*
2768b52a336eSPavel Tatashin * To reduce freemem_lock contention, do not update availrmem
2769b52a336eSPavel Tatashin * until at least NLCK pages have been unlocked.
2770b52a336eSPavel Tatashin * 1. No need to update if nlck is zero
2771b52a336eSPavel Tatashin * 2. Always update if the last iteration
2772b52a336eSPavel Tatashin */
2773b52a336eSPavel Tatashin if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) {
2774b52a336eSPavel Tatashin mutex_enter(&freemem_lock);
2775b52a336eSPavel Tatashin availrmem += nlck;
2776b52a336eSPavel Tatashin pages_locked -= nlck;
2777b52a336eSPavel Tatashin mutex_exit(&freemem_lock);
2778b52a336eSPavel Tatashin nlck = 0;
2779b52a336eSPavel Tatashin nlck_limit = NLCK + RAND_P2(NLCK);
2780b52a336eSPavel Tatashin }
2781b52a336eSPavel Tatashin }
2782b52a336eSPavel Tatashin ANON_LOCK_EXIT(&->a_rwlock);
2783b52a336eSPavel Tatashin
27847c478bd9Sstevel@tonic-gate return (0);
27857c478bd9Sstevel@tonic-gate }
27867c478bd9Sstevel@tonic-gate
27877c478bd9Sstevel@tonic-gate /*ARGSUSED*/
27887c478bd9Sstevel@tonic-gate static int
segspt_shmlockop(struct seg * seg,caddr_t addr,size_t len,int attr,int op,ulong_t * lockmap,size_t pos)27897c478bd9Sstevel@tonic-gate segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
27907c478bd9Sstevel@tonic-gate int attr, int op, ulong_t *lockmap, size_t pos)
27917c478bd9Sstevel@tonic-gate {
27927c478bd9Sstevel@tonic-gate struct shm_data *shmd = seg->s_data;
27937c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
27947c478bd9Sstevel@tonic-gate struct spt_data *sptd = sptseg->s_data;
2795c6939658Ssl108498 struct kshmid *sp = sptd->spt_amp->a_sp;
27967c478bd9Sstevel@tonic-gate pgcnt_t npages, a_npages;
27977c478bd9Sstevel@tonic-gate page_t **ppa;
27987c478bd9Sstevel@tonic-gate pgcnt_t an_idx, a_an_idx, ppa_idx;
27997c478bd9Sstevel@tonic-gate caddr_t spt_addr, a_addr; /* spt and aligned address */
28007c478bd9Sstevel@tonic-gate size_t a_len; /* aligned len */
28017c478bd9Sstevel@tonic-gate size_t share_sz;
28027c478bd9Sstevel@tonic-gate ulong_t i;
28037c478bd9Sstevel@tonic-gate int sts = 0;
2804c6939658Ssl108498 rctl_qty_t unlocked = 0;
2805c6939658Ssl108498 rctl_qty_t locked = 0;
2806c6939658Ssl108498 struct proc *p = curproc;
2807c6939658Ssl108498 kproject_t *proj;
28087c478bd9Sstevel@tonic-gate
2809dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2810c6939658Ssl108498 ASSERT(sp != NULL);
28117c478bd9Sstevel@tonic-gate
28127c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
28137c478bd9Sstevel@tonic-gate return (0);
28147c478bd9Sstevel@tonic-gate }
28157c478bd9Sstevel@tonic-gate
28167c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
28177c478bd9Sstevel@tonic-gate an_idx = seg_page(seg, addr);
28187c478bd9Sstevel@tonic-gate npages = btopr(len);
28197c478bd9Sstevel@tonic-gate
28207c478bd9Sstevel@tonic-gate if (an_idx + npages > btopr(shmd->shm_amp->size)) {
28217c478bd9Sstevel@tonic-gate return (ENOMEM);
28227c478bd9Sstevel@tonic-gate }
28237c478bd9Sstevel@tonic-gate
2824c6939658Ssl108498 /*
2825c6939658Ssl108498 * A shm's project never changes, so no lock needed.
2826c6939658Ssl108498 * The shm has a hold on the project, so it will not go away.
2827c6939658Ssl108498 * Since we have a mapping to shm within this zone, we know
2828c6939658Ssl108498 * that the zone will not go away.
2829c6939658Ssl108498 */
2830c6939658Ssl108498 proj = sp->shm_perm.ipc_proj;
2831c6939658Ssl108498
28327c478bd9Sstevel@tonic-gate if (op == MC_LOCK) {
2833c6939658Ssl108498
28347c478bd9Sstevel@tonic-gate /*
28357c478bd9Sstevel@tonic-gate * Need to align addr and size request if they are not
28367c478bd9Sstevel@tonic-gate * aligned so we can always allocate large page(s) however
28377c478bd9Sstevel@tonic-gate * we only lock what was requested in initial request.
28387c478bd9Sstevel@tonic-gate */
28397c478bd9Sstevel@tonic-gate share_sz = page_get_pagesize(sptseg->s_szc);
28407c478bd9Sstevel@tonic-gate a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz);
28417c478bd9Sstevel@tonic-gate a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)),
28427c478bd9Sstevel@tonic-gate share_sz);
28437c478bd9Sstevel@tonic-gate a_npages = btop(a_len);
28447c478bd9Sstevel@tonic-gate a_an_idx = seg_page(seg, a_addr);
28457c478bd9Sstevel@tonic-gate spt_addr = sptseg->s_base + ptob(a_an_idx);
28467c478bd9Sstevel@tonic-gate ppa_idx = an_idx - a_an_idx;
28477c478bd9Sstevel@tonic-gate
28487c478bd9Sstevel@tonic-gate if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages),
28497c478bd9Sstevel@tonic-gate KM_NOSLEEP)) == NULL) {
28507c478bd9Sstevel@tonic-gate return (ENOMEM);
28517c478bd9Sstevel@tonic-gate }
28527c478bd9Sstevel@tonic-gate
28537c478bd9Sstevel@tonic-gate /*
28547c478bd9Sstevel@tonic-gate * Don't cache any new pages for IO and
28557c478bd9Sstevel@tonic-gate * flush any cached pages.
28567c478bd9Sstevel@tonic-gate */
28577c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
28587c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL)
28597c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED;
28607c478bd9Sstevel@tonic-gate
28617c478bd9Sstevel@tonic-gate sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa);
28627c478bd9Sstevel@tonic-gate if (sts != 0) {
28637c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
28647c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
28657c478bd9Sstevel@tonic-gate return (sts);
28667c478bd9Sstevel@tonic-gate }
28677c478bd9Sstevel@tonic-gate
2868c6939658Ssl108498 mutex_enter(&sp->shm_mlock);
2869c6939658Ssl108498 /* enforce locked memory rctl */
2870c6939658Ssl108498 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]);
2871c6939658Ssl108498
2872c6939658Ssl108498 mutex_enter(&p->p_lock);
2873c6939658Ssl108498 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) {
2874c6939658Ssl108498 mutex_exit(&p->p_lock);
2875c6939658Ssl108498 sts = EAGAIN;
2876c6939658Ssl108498 } else {
2877c6939658Ssl108498 mutex_exit(&p->p_lock);
28787c478bd9Sstevel@tonic-gate sts = spt_lockpages(seg, an_idx, npages,
2879c6939658Ssl108498 &ppa[ppa_idx], lockmap, pos, &locked);
2880c6939658Ssl108498
28817c478bd9Sstevel@tonic-gate /*
2882c6939658Ssl108498 * correct locked count if not all pages could be
2883c6939658Ssl108498 * locked
28847c478bd9Sstevel@tonic-gate */
2885c6939658Ssl108498 if ((unlocked - locked) > 0) {
2886c6939658Ssl108498 rctl_decr_locked_mem(NULL, proj,
2887c6939658Ssl108498 (unlocked - locked), 0);
2888c6939658Ssl108498 }
2889c6939658Ssl108498 }
2890c6939658Ssl108498 /*
2891c6939658Ssl108498 * unlock pages
2892c6939658Ssl108498 */
2893c6939658Ssl108498 for (i = 0; i < a_npages; i++)
28947c478bd9Sstevel@tonic-gate page_unlock(ppa[i]);
28957c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL)
28967c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED;
2897c6939658Ssl108498 mutex_exit(&sp->shm_mlock);
28987c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
28997c478bd9Sstevel@tonic-gate
29007c478bd9Sstevel@tonic-gate kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
29017c478bd9Sstevel@tonic-gate
29027c478bd9Sstevel@tonic-gate } else if (op == MC_UNLOCK) { /* unlock */
2903a98e9dbfSaguzovsk page_t **ppa;
29047c478bd9Sstevel@tonic-gate
29057c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
29067c478bd9Sstevel@tonic-gate if (shmd->shm_lckpgs == 0) {
29077c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
29087c478bd9Sstevel@tonic-gate return (0);
29097c478bd9Sstevel@tonic-gate }
29107c478bd9Sstevel@tonic-gate /*
29117c478bd9Sstevel@tonic-gate * Don't cache new IO pages.
29127c478bd9Sstevel@tonic-gate */
29137c478bd9Sstevel@tonic-gate if (sptd->spt_ppa != NULL)
29147c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED;
29157c478bd9Sstevel@tonic-gate
2916c6939658Ssl108498 mutex_enter(&sp->shm_mlock);
2917b52a336eSPavel Tatashin sts = spt_unlockpages(seg, an_idx, npages, &unlocked);
2918a98e9dbfSaguzovsk if ((ppa = sptd->spt_ppa) != NULL)
29197c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED;
29207c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
2921c6939658Ssl108498
2922c6939658Ssl108498 rctl_decr_locked_mem(NULL, proj, unlocked, 0);
2923c6939658Ssl108498 mutex_exit(&sp->shm_mlock);
2924a98e9dbfSaguzovsk
2925a98e9dbfSaguzovsk if (ppa != NULL)
2926a98e9dbfSaguzovsk seg_ppurge_wiredpp(ppa);
29277c478bd9Sstevel@tonic-gate }
29287c478bd9Sstevel@tonic-gate return (sts);
29297c478bd9Sstevel@tonic-gate }
29307c478bd9Sstevel@tonic-gate
29317c478bd9Sstevel@tonic-gate /*ARGSUSED*/
29327c478bd9Sstevel@tonic-gate int
segspt_shmgetprot(struct seg * seg,caddr_t addr,size_t len,uint_t * protv)29337c478bd9Sstevel@tonic-gate segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
29347c478bd9Sstevel@tonic-gate {
29357c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
29367c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
29377c478bd9Sstevel@tonic-gate spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1;
29387c478bd9Sstevel@tonic-gate
2939dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
29407c478bd9Sstevel@tonic-gate
29417c478bd9Sstevel@tonic-gate /*
29427c478bd9Sstevel@tonic-gate * ISM segment is always rw.
29437c478bd9Sstevel@tonic-gate */
29447c478bd9Sstevel@tonic-gate while (--pgno >= 0)
29457c478bd9Sstevel@tonic-gate *protv++ = sptd->spt_prot;
29467c478bd9Sstevel@tonic-gate return (0);
29477c478bd9Sstevel@tonic-gate }
29487c478bd9Sstevel@tonic-gate
29497c478bd9Sstevel@tonic-gate /*ARGSUSED*/
29507c478bd9Sstevel@tonic-gate u_offset_t
segspt_shmgetoffset(struct seg * seg,caddr_t addr)29517c478bd9Sstevel@tonic-gate segspt_shmgetoffset(struct seg *seg, caddr_t addr)
29527c478bd9Sstevel@tonic-gate {
2953dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
29547c478bd9Sstevel@tonic-gate
29557c478bd9Sstevel@tonic-gate /* Offset does not matter in ISM memory */
29567c478bd9Sstevel@tonic-gate
29577c478bd9Sstevel@tonic-gate return ((u_offset_t)0);
29587c478bd9Sstevel@tonic-gate }
29597c478bd9Sstevel@tonic-gate
29607c478bd9Sstevel@tonic-gate /* ARGSUSED */
29617c478bd9Sstevel@tonic-gate int
segspt_shmgettype(struct seg * seg,caddr_t addr)29627c478bd9Sstevel@tonic-gate segspt_shmgettype(struct seg *seg, caddr_t addr)
29637c478bd9Sstevel@tonic-gate {
29647c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
29657c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
29667c478bd9Sstevel@tonic-gate
2967dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
29687c478bd9Sstevel@tonic-gate
29697c478bd9Sstevel@tonic-gate /*
29707c478bd9Sstevel@tonic-gate * The shared memory mapping is always MAP_SHARED, SWAP is only
29717c478bd9Sstevel@tonic-gate * reserved for DISM
29727c478bd9Sstevel@tonic-gate */
29737c478bd9Sstevel@tonic-gate return (MAP_SHARED |
29747c478bd9Sstevel@tonic-gate ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE));
29757c478bd9Sstevel@tonic-gate }
29767c478bd9Sstevel@tonic-gate
29777c478bd9Sstevel@tonic-gate /*ARGSUSED*/
29787c478bd9Sstevel@tonic-gate int
segspt_shmgetvp(struct seg * seg,caddr_t addr,struct vnode ** vpp)29797c478bd9Sstevel@tonic-gate segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
29807c478bd9Sstevel@tonic-gate {
29817c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
29827c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
29837c478bd9Sstevel@tonic-gate
2984dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
29857c478bd9Sstevel@tonic-gate
29867c478bd9Sstevel@tonic-gate *vpp = sptd->spt_vp;
29877c478bd9Sstevel@tonic-gate return (0);
29887c478bd9Sstevel@tonic-gate }
29897c478bd9Sstevel@tonic-gate
29902ba723d8Smec /*
29912ba723d8Smec * We need to wait for pending IO to complete to a DISM segment in order for
29922ba723d8Smec * pages to get kicked out of the seg_pcache. 120 seconds should be more
29932ba723d8Smec * than enough time to wait.
29942ba723d8Smec */
29952ba723d8Smec static clock_t spt_pcache_wait = 120;
29962ba723d8Smec
29977c478bd9Sstevel@tonic-gate /*ARGSUSED*/
29987c478bd9Sstevel@tonic-gate static int
segspt_shmadvise(struct seg * seg,caddr_t addr,size_t len,uint_t behav)29997c478bd9Sstevel@tonic-gate segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
30007c478bd9Sstevel@tonic-gate {
30017c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
30027c478bd9Sstevel@tonic-gate struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
30037c478bd9Sstevel@tonic-gate struct anon_map *amp;
30047c478bd9Sstevel@tonic-gate pgcnt_t pg_idx;
30052ba723d8Smec ushort_t gen;
30062ba723d8Smec clock_t end_lbolt;
30072ba723d8Smec int writer;
3008a98e9dbfSaguzovsk page_t **ppa;
30097c478bd9Sstevel@tonic-gate
3010dc32d872SJosef 'Jeff' Sipek ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
30117c478bd9Sstevel@tonic-gate
30128905f42cSBryan Cantrill if (behav == MADV_FREE || behav == MADV_PURGE) {
30137c478bd9Sstevel@tonic-gate if ((sptd->spt_flags & SHM_PAGEABLE) == 0)
30147c478bd9Sstevel@tonic-gate return (0);
30157c478bd9Sstevel@tonic-gate
30167c478bd9Sstevel@tonic-gate amp = sptd->spt_amp;
30177c478bd9Sstevel@tonic-gate pg_idx = seg_page(seg, addr);
30187c478bd9Sstevel@tonic-gate
30197c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
3020a98e9dbfSaguzovsk if ((ppa = sptd->spt_ppa) == NULL) {
30212ba723d8Smec mutex_exit(&sptd->spt_lock);
30222ba723d8Smec ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
30238905f42cSBryan Cantrill (void) anon_disclaim(amp, pg_idx, len, behav, NULL);
30242ba723d8Smec ANON_LOCK_EXIT(&->a_rwlock);
30252ba723d8Smec return (0);
30262ba723d8Smec }
30272ba723d8Smec
30287c478bd9Sstevel@tonic-gate sptd->spt_flags |= DISM_PPA_CHANGED;
30292ba723d8Smec gen = sptd->spt_gen;
30302ba723d8Smec
30317c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
30327c478bd9Sstevel@tonic-gate
30337c478bd9Sstevel@tonic-gate /*
30347c478bd9Sstevel@tonic-gate * Purge all DISM cached pages
30357c478bd9Sstevel@tonic-gate */
3036a98e9dbfSaguzovsk seg_ppurge_wiredpp(ppa);
30377c478bd9Sstevel@tonic-gate
30382ba723d8Smec /*
30392ba723d8Smec * Drop the AS_LOCK so that other threads can grab it
30402ba723d8Smec * in the as_pageunlock path and hopefully get the segment
30412ba723d8Smec * kicked out of the seg_pcache. We bump the shm_softlockcnt
30422ba723d8Smec * to keep this segment resident.
30432ba723d8Smec */
3044dc32d872SJosef 'Jeff' Sipek writer = AS_WRITE_HELD(seg->s_as);
30451a5e258fSJosef 'Jeff' Sipek atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
3046dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(seg->s_as);
30472ba723d8Smec
30487c478bd9Sstevel@tonic-gate mutex_enter(&sptd->spt_lock);
30492ba723d8Smec
3050d3d50737SRafael Vanoni end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait);
30512ba723d8Smec
30522ba723d8Smec /*
30532ba723d8Smec * Try to wait for pages to get kicked out of the seg_pcache.
30542ba723d8Smec */
30552ba723d8Smec while (sptd->spt_gen == gen &&
30562ba723d8Smec (sptd->spt_flags & DISM_PPA_CHANGED) &&
3057d3d50737SRafael Vanoni ddi_get_lbolt() < end_lbolt) {
30582ba723d8Smec if (!cv_timedwait_sig(&sptd->spt_cv,
30592ba723d8Smec &sptd->spt_lock, end_lbolt)) {
30602ba723d8Smec break;
30612ba723d8Smec }
30622ba723d8Smec }
30632ba723d8Smec
30647c478bd9Sstevel@tonic-gate mutex_exit(&sptd->spt_lock);
30652ba723d8Smec
30662ba723d8Smec /* Regrab the AS_LOCK and release our hold on the segment */
3067dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(seg->s_as, writer ? RW_WRITER : RW_READER);
30681a5e258fSJosef 'Jeff' Sipek atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
30692ba723d8Smec if (shmd->shm_softlockcnt <= 0) {
30702ba723d8Smec if (AS_ISUNMAPWAIT(seg->s_as)) {
30712ba723d8Smec mutex_enter(&seg->s_as->a_contents);
30722ba723d8Smec if (AS_ISUNMAPWAIT(seg->s_as)) {
30732ba723d8Smec AS_CLRUNMAPWAIT(seg->s_as);
30742ba723d8Smec cv_broadcast(&seg->s_as->a_cv);
30752ba723d8Smec }
30762ba723d8Smec mutex_exit(&seg->s_as->a_contents);
30772ba723d8Smec }
30782ba723d8Smec }
30792ba723d8Smec
30802ba723d8Smec ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
30818905f42cSBryan Cantrill (void) anon_disclaim(amp, pg_idx, len, behav, NULL);
30822ba723d8Smec ANON_LOCK_EXIT(&->a_rwlock);
30837c478bd9Sstevel@tonic-gate } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP ||
30847c478bd9Sstevel@tonic-gate behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) {
30857c478bd9Sstevel@tonic-gate int already_set;
30867c478bd9Sstevel@tonic-gate ulong_t anon_index;
30877c478bd9Sstevel@tonic-gate lgrp_mem_policy_t policy;
30887c478bd9Sstevel@tonic-gate caddr_t shm_addr;
30897c478bd9Sstevel@tonic-gate size_t share_size;
30907c478bd9Sstevel@tonic-gate size_t size;
30917c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
30927c478bd9Sstevel@tonic-gate caddr_t sptseg_addr;
30937c478bd9Sstevel@tonic-gate
30947c478bd9Sstevel@tonic-gate /*
30957c478bd9Sstevel@tonic-gate * Align address and length to page size of underlying segment
30967c478bd9Sstevel@tonic-gate */
30977c478bd9Sstevel@tonic-gate share_size = page_get_pagesize(shmd->shm_sptseg->s_szc);
30987c478bd9Sstevel@tonic-gate shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size);
30997c478bd9Sstevel@tonic-gate size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)),
31007c478bd9Sstevel@tonic-gate share_size);
31017c478bd9Sstevel@tonic-gate
31027c478bd9Sstevel@tonic-gate amp = shmd->shm_amp;
31037c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, shm_addr);
31047c478bd9Sstevel@tonic-gate
31057c478bd9Sstevel@tonic-gate /*
31067c478bd9Sstevel@tonic-gate * And now we may have to adjust size downward if we have
31077c478bd9Sstevel@tonic-gate * exceeded the realsize of the segment or initial anon
31087c478bd9Sstevel@tonic-gate * allocations.
31097c478bd9Sstevel@tonic-gate */
31107c478bd9Sstevel@tonic-gate sptseg_addr = sptseg->s_base + ptob(anon_index);
31117c478bd9Sstevel@tonic-gate if ((sptseg_addr + size) >
31127c478bd9Sstevel@tonic-gate (sptseg->s_base + sptd->spt_realsize))
31137c478bd9Sstevel@tonic-gate size = (sptseg->s_base + sptd->spt_realsize) -
31147c478bd9Sstevel@tonic-gate sptseg_addr;
31157c478bd9Sstevel@tonic-gate
31167c478bd9Sstevel@tonic-gate /*
31177c478bd9Sstevel@tonic-gate * Set memory allocation policy for this segment
31187c478bd9Sstevel@tonic-gate */
31197c478bd9Sstevel@tonic-gate policy = lgrp_madv_to_policy(behav, len, MAP_SHARED);
31207c478bd9Sstevel@tonic-gate already_set = lgrp_shm_policy_set(policy, amp, anon_index,
31217c478bd9Sstevel@tonic-gate NULL, 0, len);
31227c478bd9Sstevel@tonic-gate
31237c478bd9Sstevel@tonic-gate /*
31247c478bd9Sstevel@tonic-gate * If random memory allocation policy set already,
31257c478bd9Sstevel@tonic-gate * don't bother reapplying it.
31267c478bd9Sstevel@tonic-gate */
31277c478bd9Sstevel@tonic-gate if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy))
31287c478bd9Sstevel@tonic-gate return (0);
31297c478bd9Sstevel@tonic-gate
31307c478bd9Sstevel@tonic-gate /*
31317c478bd9Sstevel@tonic-gate * Mark any existing pages in the given range for
31327c478bd9Sstevel@tonic-gate * migration, flushing the I/O page cache, and using
31337c478bd9Sstevel@tonic-gate * underlying segment to calculate anon index and get
31347c478bd9Sstevel@tonic-gate * anonmap and vnode pointer from
31357c478bd9Sstevel@tonic-gate */
31367c478bd9Sstevel@tonic-gate if (shmd->shm_softlockcnt > 0)
31377c478bd9Sstevel@tonic-gate segspt_purge(seg);
31387c478bd9Sstevel@tonic-gate
31397c478bd9Sstevel@tonic-gate page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0);
31407c478bd9Sstevel@tonic-gate }
31417c478bd9Sstevel@tonic-gate
31427c478bd9Sstevel@tonic-gate return (0);
31437c478bd9Sstevel@tonic-gate }
31447c478bd9Sstevel@tonic-gate
31457c478bd9Sstevel@tonic-gate /*ARGSUSED*/
31467c478bd9Sstevel@tonic-gate void
segspt_shmdump(struct seg * seg)31477c478bd9Sstevel@tonic-gate segspt_shmdump(struct seg *seg)
31487c478bd9Sstevel@tonic-gate {
31497c478bd9Sstevel@tonic-gate /* no-op for ISM segment */
31507c478bd9Sstevel@tonic-gate }
31517c478bd9Sstevel@tonic-gate
31527c478bd9Sstevel@tonic-gate /*ARGSUSED*/
3153e09cef95SToomas Soome static int
segspt_shmsetpgsz(struct seg * seg,caddr_t addr,size_t len,uint_t szc)31547c478bd9Sstevel@tonic-gate segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
31557c478bd9Sstevel@tonic-gate {
31567c478bd9Sstevel@tonic-gate return (ENOTSUP);
31577c478bd9Sstevel@tonic-gate }
31587c478bd9Sstevel@tonic-gate
31597c478bd9Sstevel@tonic-gate /*
31607c478bd9Sstevel@tonic-gate * get a memory ID for an addr in a given segment
31617c478bd9Sstevel@tonic-gate */
31627c478bd9Sstevel@tonic-gate static int
segspt_shmgetmemid(struct seg * seg,caddr_t addr,memid_t * memidp)31637c478bd9Sstevel@tonic-gate segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
31647c478bd9Sstevel@tonic-gate {
31657c478bd9Sstevel@tonic-gate struct shm_data *shmd = (struct shm_data *)seg->s_data;
31667c478bd9Sstevel@tonic-gate struct anon *ap;
31677c478bd9Sstevel@tonic-gate size_t anon_index;
31687c478bd9Sstevel@tonic-gate struct anon_map *amp = shmd->shm_amp;
31697c478bd9Sstevel@tonic-gate struct spt_data *sptd = shmd->shm_sptseg->s_data;
31707c478bd9Sstevel@tonic-gate struct seg *sptseg = shmd->shm_sptseg;
31717c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie;
31727c478bd9Sstevel@tonic-gate
31737c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr);
31747c478bd9Sstevel@tonic-gate
31757c478bd9Sstevel@tonic-gate if (addr > (seg->s_base + sptd->spt_realsize)) {
31767c478bd9Sstevel@tonic-gate return (EFAULT);
31777c478bd9Sstevel@tonic-gate }
31787c478bd9Sstevel@tonic-gate
31797c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
31807c478bd9Sstevel@tonic-gate anon_array_enter(amp, anon_index, &cookie);
31817c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, anon_index);
31827c478bd9Sstevel@tonic-gate if (ap == NULL) {
31837c478bd9Sstevel@tonic-gate struct page *pp;
31847c478bd9Sstevel@tonic-gate caddr_t spt_addr = sptseg->s_base + ptob(anon_index);
31857c478bd9Sstevel@tonic-gate
31867c478bd9Sstevel@tonic-gate pp = anon_zero(sptseg, spt_addr, &ap, kcred);
31877c478bd9Sstevel@tonic-gate if (pp == NULL) {
31887c478bd9Sstevel@tonic-gate anon_array_exit(&cookie);
31897c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
31907c478bd9Sstevel@tonic-gate return (ENOMEM);
31917c478bd9Sstevel@tonic-gate }
31927c478bd9Sstevel@tonic-gate (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP);
31937c478bd9Sstevel@tonic-gate page_unlock(pp);
31947c478bd9Sstevel@tonic-gate }
31957c478bd9Sstevel@tonic-gate anon_array_exit(&cookie);
31967c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock);
31977c478bd9Sstevel@tonic-gate memidp->val[0] = (uintptr_t)ap;
31987c478bd9Sstevel@tonic-gate memidp->val[1] = (uintptr_t)addr & PAGEOFFSET;
31997c478bd9Sstevel@tonic-gate return (0);
32007c478bd9Sstevel@tonic-gate }
32017c478bd9Sstevel@tonic-gate
32027c478bd9Sstevel@tonic-gate /*
32037c478bd9Sstevel@tonic-gate * Get memory allocation policy info for specified address in given segment
32047c478bd9Sstevel@tonic-gate */
32057c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *
segspt_shmgetpolicy(struct seg * seg,caddr_t addr)32067c478bd9Sstevel@tonic-gate segspt_shmgetpolicy(struct seg *seg, caddr_t addr)
32077c478bd9Sstevel@tonic-gate {
32087c478bd9Sstevel@tonic-gate struct anon_map *amp;
32097c478bd9Sstevel@tonic-gate ulong_t anon_index;
32107c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info;
32117c478bd9Sstevel@tonic-gate struct shm_data *shm_data;
32127c478bd9Sstevel@tonic-gate
32137c478bd9Sstevel@tonic-gate ASSERT(seg != NULL);
32147c478bd9Sstevel@tonic-gate
32157c478bd9Sstevel@tonic-gate /*
32167c478bd9Sstevel@tonic-gate * Get anon_map from segshm
32177c478bd9Sstevel@tonic-gate *
32187c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map, since
32197c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be
32207c478bd9Sstevel@tonic-gate * nonzero for an existing segment
32217c478bd9Sstevel@tonic-gate * Need to grab readers lock on policy tree though
32227c478bd9Sstevel@tonic-gate */
32237c478bd9Sstevel@tonic-gate shm_data = (struct shm_data *)seg->s_data;
32247c478bd9Sstevel@tonic-gate if (shm_data == NULL)
32257c478bd9Sstevel@tonic-gate return (NULL);
32267c478bd9Sstevel@tonic-gate amp = shm_data->shm_amp;
32277c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0);
32287c478bd9Sstevel@tonic-gate
32297c478bd9Sstevel@tonic-gate /*
32307c478bd9Sstevel@tonic-gate * Get policy info
32317c478bd9Sstevel@tonic-gate *
32327c478bd9Sstevel@tonic-gate * Assume starting anon index of 0
32337c478bd9Sstevel@tonic-gate */
32347c478bd9Sstevel@tonic-gate anon_index = seg_page(seg, addr);
32357c478bd9Sstevel@tonic-gate policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
32367c478bd9Sstevel@tonic-gate
32377c478bd9Sstevel@tonic-gate return (policy_info);
32387c478bd9Sstevel@tonic-gate }
32391bd5c35fSelowe
32401bd5c35fSelowe /*ARGSUSED*/
32411bd5c35fSelowe static int
segspt_shmcapable(struct seg * seg,segcapability_t capability)32421bd5c35fSelowe segspt_shmcapable(struct seg *seg, segcapability_t capability)
32431bd5c35fSelowe {
32441bd5c35fSelowe return (0);
32451bd5c35fSelowe }
3246