xref: /titanic_41/usr/src/uts/common/os/grow.c (revision 0b42f15ac52b077791d4ba079e8c163c592c3fda)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /* Copyright 2013 OmniTI Computer Consulting, Inc. All rights reserved. */
23 
24 /*
25  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
26  * Use is subject to license terms.
27  */
28 
29 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
30 /*	  All Rights Reserved  	*/
31 
32 #include <sys/types.h>
33 #include <sys/inttypes.h>
34 #include <sys/param.h>
35 #include <sys/sysmacros.h>
36 #include <sys/systm.h>
37 #include <sys/signal.h>
38 #include <sys/user.h>
39 #include <sys/errno.h>
40 #include <sys/var.h>
41 #include <sys/proc.h>
42 #include <sys/tuneable.h>
43 #include <sys/debug.h>
44 #include <sys/cmn_err.h>
45 #include <sys/cred.h>
46 #include <sys/vnode.h>
47 #include <sys/vfs.h>
48 #include <sys/vm.h>
49 #include <sys/file.h>
50 #include <sys/mman.h>
51 #include <sys/vmparam.h>
52 #include <sys/fcntl.h>
53 #include <sys/lwpchan_impl.h>
54 #include <sys/nbmlock.h>
55 
56 #include <vm/hat.h>
57 #include <vm/as.h>
58 #include <vm/seg.h>
59 #include <vm/seg_dev.h>
60 #include <vm/seg_vn.h>
61 
62 int use_brk_lpg = 1;
63 int use_stk_lpg = 1;
64 
65 static int brk_lpg(caddr_t nva);
66 static int grow_lpg(caddr_t sp);
67 
68 int
brk(caddr_t nva)69 brk(caddr_t nva)
70 {
71 	int error;
72 	proc_t *p = curproc;
73 
74 	/*
75 	 * Serialize brk operations on an address space.
76 	 * This also serves as the lock protecting p_brksize
77 	 * and p_brkpageszc.
78 	 */
79 	as_rangelock(p->p_as);
80 	if (use_brk_lpg && (p->p_flag & SAUTOLPG) != 0) {
81 		error = brk_lpg(nva);
82 	} else {
83 		error = brk_internal(nva, p->p_brkpageszc);
84 	}
85 	as_rangeunlock(p->p_as);
86 	return ((error != 0 ? set_errno(error) : 0));
87 }
88 
89 /*
90  * Algorithm: call arch-specific map_pgsz to get best page size to use,
91  * then call brk_internal().
92  * Returns 0 on success.
93  */
94 static int
brk_lpg(caddr_t nva)95 brk_lpg(caddr_t nva)
96 {
97 	struct proc *p = curproc;
98 	size_t pgsz, len;
99 	caddr_t addr, brkend;
100 	caddr_t bssbase = p->p_bssbase;
101 	caddr_t brkbase = p->p_brkbase;
102 	int oszc, szc;
103 	int err;
104 
105 	oszc = p->p_brkpageszc;
106 
107 	/*
108 	 * If p_brkbase has not yet been set, the first call
109 	 * to brk_internal() will initialize it.
110 	 */
111 	if (brkbase == 0) {
112 		return (brk_internal(nva, oszc));
113 	}
114 
115 	len = nva - bssbase;
116 
117 	pgsz = map_pgsz(MAPPGSZ_HEAP, p, bssbase, len, 0);
118 	szc = page_szc(pgsz);
119 
120 	/*
121 	 * Covers two cases:
122 	 * 1. page_szc() returns -1 for invalid page size, so we want to
123 	 * ignore it in that case.
124 	 * 2. By design we never decrease page size, as it is more stable.
125 	 */
126 	if (szc <= oszc) {
127 		err = brk_internal(nva, oszc);
128 		/* If failed, back off to base page size. */
129 		if (err != 0 && oszc != 0) {
130 			err = brk_internal(nva, 0);
131 		}
132 		return (err);
133 	}
134 
135 	err = brk_internal(nva, szc);
136 	/* If using szc failed, map with base page size and return. */
137 	if (err != 0) {
138 		if (szc != 0) {
139 			err = brk_internal(nva, 0);
140 		}
141 		return (err);
142 	}
143 
144 	/*
145 	 * Round up brk base to a large page boundary and remap
146 	 * anything in the segment already faulted in beyond that
147 	 * point.
148 	 */
149 	addr = (caddr_t)P2ROUNDUP((uintptr_t)p->p_bssbase, pgsz);
150 	brkend = brkbase + p->p_brksize;
151 	len = brkend - addr;
152 	/* Check that len is not negative. Update page size code for heap. */
153 	if (addr >= p->p_bssbase && brkend > addr && IS_P2ALIGNED(len, pgsz)) {
154 		(void) as_setpagesize(p->p_as, addr, len, szc, B_FALSE);
155 		p->p_brkpageszc = szc;
156 	}
157 
158 	ASSERT(err == 0);
159 	return (err);		/* should always be 0 */
160 }
161 
162 /*
163  * Returns 0 on success.
164  */
165 int
brk_internal(caddr_t nva,uint_t brkszc)166 brk_internal(caddr_t nva, uint_t brkszc)
167 {
168 	caddr_t ova;			/* current break address */
169 	size_t size;
170 	int	error;
171 	struct proc *p = curproc;
172 	struct as *as = p->p_as;
173 	size_t pgsz;
174 	uint_t szc;
175 	rctl_qty_t as_rctl;
176 
177 	/*
178 	 * extend heap to brkszc alignment but use current p->p_brkpageszc
179 	 * for the newly created segment. This allows the new extension
180 	 * segment to be concatenated successfully with the existing brk
181 	 * segment.
182 	 */
183 	if ((szc = brkszc) != 0) {
184 		pgsz = page_get_pagesize(szc);
185 		ASSERT(pgsz > PAGESIZE);
186 	} else {
187 		pgsz = PAGESIZE;
188 	}
189 
190 	mutex_enter(&p->p_lock);
191 	as_rctl = rctl_enforced_value(rctlproc_legacy[RLIMIT_DATA],
192 	    p->p_rctls, p);
193 	mutex_exit(&p->p_lock);
194 
195 	/*
196 	 * If p_brkbase has not yet been set, the first call
197 	 * to brk() will initialize it.
198 	 */
199 	if (p->p_brkbase == 0)
200 		p->p_brkbase = nva;
201 
202 	/*
203 	 * Before multiple page size support existed p_brksize was the value
204 	 * not rounded to the pagesize (i.e. it stored the exact user request
205 	 * for heap size). If pgsz is greater than PAGESIZE calculate the
206 	 * heap size as the real new heap size by rounding it up to pgsz.
207 	 * This is useful since we may want to know where the heap ends
208 	 * without knowing heap pagesize (e.g. some old code) and also if
209 	 * heap pagesize changes we can update p_brkpageszc but delay adding
210 	 * new mapping yet still know from p_brksize where the heap really
211 	 * ends. The user requested heap end is stored in libc variable.
212 	 */
213 	if (pgsz > PAGESIZE) {
214 		caddr_t tnva = (caddr_t)P2ROUNDUP((uintptr_t)nva, pgsz);
215 		size = tnva - p->p_brkbase;
216 		if (tnva < p->p_brkbase || (size > p->p_brksize &&
217 		    size > (size_t)as_rctl)) {
218 			szc = 0;
219 			pgsz = PAGESIZE;
220 			size = nva - p->p_brkbase;
221 		}
222 	} else {
223 		size = nva - p->p_brkbase;
224 	}
225 
226 	/*
227 	 * use PAGESIZE to roundup ova because we want to know the real value
228 	 * of the current heap end in case p_brkpageszc changes since the last
229 	 * p_brksize was computed.
230 	 */
231 	nva = (caddr_t)P2ROUNDUP((uintptr_t)nva, pgsz);
232 	ova = (caddr_t)P2ROUNDUP((uintptr_t)(p->p_brkbase + p->p_brksize),
233 	    PAGESIZE);
234 
235 	if ((nva < p->p_brkbase) || (size > p->p_brksize &&
236 	    size > as_rctl)) {
237 		mutex_enter(&p->p_lock);
238 		(void) rctl_action(rctlproc_legacy[RLIMIT_DATA], p->p_rctls, p,
239 		    RCA_SAFE);
240 		mutex_exit(&p->p_lock);
241 		return (ENOMEM);
242 	}
243 
244 	if (nva > ova) {
245 		struct segvn_crargs crargs =
246 		    SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
247 
248 		if (!(p->p_datprot & PROT_EXEC)) {
249 			crargs.prot &= ~PROT_EXEC;
250 		}
251 
252 		/*
253 		 * Add new zfod mapping to extend UNIX data segment
254 		 * AS_MAP_NO_LPOOB means use 0, and don't reapply OOB policies
255 		 * via map_pgszcvec(). Use AS_MAP_HEAP to get intermediate
256 		 * page sizes if ova is not aligned to szc's pgsz.
257 		 */
258 		if (szc > 0) {
259 			caddr_t rbss;
260 
261 			rbss = (caddr_t)P2ROUNDUP((uintptr_t)p->p_bssbase,
262 			    pgsz);
263 			if (IS_P2ALIGNED(p->p_bssbase, pgsz) || ova > rbss) {
264 				crargs.szc = p->p_brkpageszc ? p->p_brkpageszc :
265 				    AS_MAP_NO_LPOOB;
266 			} else if (ova == rbss) {
267 				crargs.szc = szc;
268 			} else {
269 				crargs.szc = AS_MAP_HEAP;
270 			}
271 		} else {
272 			crargs.szc = AS_MAP_NO_LPOOB;
273 		}
274 		crargs.lgrp_mem_policy_flags = LGRP_MP_FLAG_EXTEND_UP;
275 		error = as_map(as, ova, (size_t)(nva - ova), segvn_create,
276 		    &crargs);
277 		if (error) {
278 			return (error);
279 		}
280 
281 	} else if (nva < ova) {
282 		/*
283 		 * Release mapping to shrink UNIX data segment.
284 		 */
285 		(void) as_unmap(as, nva, (size_t)(ova - nva));
286 	}
287 	p->p_brksize = size;
288 	return (0);
289 }
290 
291 /*
292  * Grow the stack to include sp.  Return 1 if successful, 0 otherwise.
293  * This routine assumes that the stack grows downward.
294  */
295 int
grow(caddr_t sp)296 grow(caddr_t sp)
297 {
298 	struct proc *p = curproc;
299 	struct as *as = p->p_as;
300 	size_t oldsize = p->p_stksize;
301 	size_t newsize;
302 	int err;
303 
304 	/*
305 	 * Serialize grow operations on an address space.
306 	 * This also serves as the lock protecting p_stksize
307 	 * and p_stkpageszc.
308 	 */
309 	as_rangelock(as);
310 	if (use_stk_lpg && (p->p_flag & SAUTOLPG) != 0) {
311 		err = grow_lpg(sp);
312 	} else {
313 		err = grow_internal(sp, p->p_stkpageszc);
314 	}
315 	as_rangeunlock(as);
316 
317 	if (err == 0 && (newsize = p->p_stksize) > oldsize) {
318 		ASSERT(IS_P2ALIGNED(oldsize, PAGESIZE));
319 		ASSERT(IS_P2ALIGNED(newsize, PAGESIZE));
320 		/*
321 		 * Set up translations so the process doesn't have to fault in
322 		 * the stack pages we just gave it.
323 		 */
324 		(void) as_fault(as->a_hat, as, p->p_usrstack - newsize,
325 		    newsize - oldsize, F_INVAL, S_WRITE);
326 	}
327 	return ((err == 0 ? 1 : 0));
328 }
329 
330 /*
331  * Algorithm: call arch-specific map_pgsz to get best page size to use,
332  * then call grow_internal().
333  * Returns 0 on success.
334  */
335 static int
grow_lpg(caddr_t sp)336 grow_lpg(caddr_t sp)
337 {
338 	struct proc *p = curproc;
339 	size_t pgsz;
340 	size_t len, newsize;
341 	caddr_t addr, saddr;
342 	caddr_t growend;
343 	int oszc, szc;
344 	int err;
345 
346 	newsize = p->p_usrstack - sp;
347 
348 	oszc = p->p_stkpageszc;
349 	pgsz = map_pgsz(MAPPGSZ_STK, p, sp, newsize, 0);
350 	szc = page_szc(pgsz);
351 
352 	/*
353 	 * Covers two cases:
354 	 * 1. page_szc() returns -1 for invalid page size, so we want to
355 	 * ignore it in that case.
356 	 * 2. By design we never decrease page size, as it is more stable.
357 	 * This shouldn't happen as the stack never shrinks.
358 	 */
359 	if (szc <= oszc) {
360 		err = grow_internal(sp, oszc);
361 		/* failed, fall back to base page size */
362 		if (err != 0 && oszc != 0) {
363 			err = grow_internal(sp, 0);
364 		}
365 		return (err);
366 	}
367 
368 	/*
369 	 * We've grown sufficiently to switch to a new page size.
370 	 * So we are going to remap the whole segment with the new page size.
371 	 */
372 	err = grow_internal(sp, szc);
373 	/* The grow with szc failed, so fall back to base page size. */
374 	if (err != 0) {
375 		if (szc != 0) {
376 			err = grow_internal(sp, 0);
377 		}
378 		return (err);
379 	}
380 
381 	/*
382 	 * Round up stack pointer to a large page boundary and remap
383 	 * any pgsz pages in the segment already faulted in beyond that
384 	 * point.
385 	 */
386 	saddr = p->p_usrstack - p->p_stksize;
387 	addr = (caddr_t)P2ROUNDUP((uintptr_t)saddr, pgsz);
388 	growend = (caddr_t)P2ALIGN((uintptr_t)p->p_usrstack, pgsz);
389 	len = growend - addr;
390 	/* Check that len is not negative. Update page size code for stack. */
391 	if (addr >= saddr && growend > addr && IS_P2ALIGNED(len, pgsz)) {
392 		(void) as_setpagesize(p->p_as, addr, len, szc, B_FALSE);
393 		p->p_stkpageszc = szc;
394 	}
395 
396 	ASSERT(err == 0);
397 	return (err);		/* should always be 0 */
398 }
399 
400 /*
401  * This routine assumes that the stack grows downward.
402  * Returns 0 on success, errno on failure.
403  */
404 int
grow_internal(caddr_t sp,uint_t growszc)405 grow_internal(caddr_t sp, uint_t growszc)
406 {
407 	struct proc *p = curproc;
408 	size_t newsize;
409 	size_t oldsize;
410 	int    error;
411 	size_t pgsz;
412 	uint_t szc;
413 	struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
414 
415 	ASSERT(sp < p->p_usrstack);
416 	sp = (caddr_t)P2ALIGN((uintptr_t)sp, PAGESIZE);
417 
418 	/*
419 	 * grow to growszc alignment but use current p->p_stkpageszc for
420 	 * the segvn_crargs szc passed to segvn_create. For memcntl to
421 	 * increase the szc, this allows the new extension segment to be
422 	 * concatenated successfully with the existing stack segment.
423 	 */
424 	if ((szc = growszc) != 0) {
425 		pgsz = page_get_pagesize(szc);
426 		ASSERT(pgsz > PAGESIZE);
427 		newsize = p->p_usrstack - (caddr_t)P2ALIGN((uintptr_t)sp, pgsz);
428 		if (newsize > (size_t)p->p_stk_ctl) {
429 			szc = 0;
430 			pgsz = PAGESIZE;
431 			newsize = p->p_usrstack - sp;
432 		}
433 	} else {
434 		pgsz = PAGESIZE;
435 		newsize = p->p_usrstack - sp;
436 	}
437 
438 	if (newsize > (size_t)p->p_stk_ctl) {
439 		(void) rctl_action(rctlproc_legacy[RLIMIT_STACK], p->p_rctls, p,
440 		    RCA_UNSAFE_ALL);
441 
442 		return (ENOMEM);
443 	}
444 
445 	oldsize = p->p_stksize;
446 	ASSERT(P2PHASE(oldsize, PAGESIZE) == 0);
447 
448 	if (newsize <= oldsize) {	/* prevent the stack from shrinking */
449 		return (0);
450 	}
451 
452 	if (!(p->p_stkprot & PROT_EXEC)) {
453 		crargs.prot &= ~PROT_EXEC;
454 	}
455 	/*
456 	 * extend stack with the proposed new growszc, which is different
457 	 * than p_stkpageszc only on a memcntl to increase the stack pagesize.
458 	 * AS_MAP_NO_LPOOB means use 0, and don't reapply OOB policies via
459 	 * map_pgszcvec(). Use AS_MAP_STACK to get intermediate page sizes
460 	 * if not aligned to szc's pgsz.
461 	 */
462 	if (szc > 0) {
463 		caddr_t oldsp = p->p_usrstack - oldsize;
464 		caddr_t austk = (caddr_t)P2ALIGN((uintptr_t)p->p_usrstack,
465 		    pgsz);
466 
467 		if (IS_P2ALIGNED(p->p_usrstack, pgsz) || oldsp < austk) {
468 			crargs.szc = p->p_stkpageszc ? p->p_stkpageszc :
469 			    AS_MAP_NO_LPOOB;
470 		} else if (oldsp == austk) {
471 			crargs.szc = szc;
472 		} else {
473 			crargs.szc = AS_MAP_STACK;
474 		}
475 	} else {
476 		crargs.szc = AS_MAP_NO_LPOOB;
477 	}
478 	crargs.lgrp_mem_policy_flags = LGRP_MP_FLAG_EXTEND_DOWN;
479 
480 	if ((error = as_map(p->p_as, p->p_usrstack - newsize, newsize - oldsize,
481 	    segvn_create, &crargs)) != 0) {
482 		if (error == EAGAIN) {
483 			cmn_err(CE_WARN, "Sorry, no swap space to grow stack "
484 			    "for pid %d (%s)", p->p_pid, PTOU(p)->u_comm);
485 		}
486 		return (error);
487 	}
488 	p->p_stksize = newsize;
489 	return (0);
490 }
491 
492 /*
493  * Find address for user to map.
494  * If MAP_FIXED is not specified, we can pick any address we want, but we will
495  * first try the value in *addrp if it is non-NULL.  Thus this is implementing
496  * a way to try and get a preferred address.
497  */
498 int
choose_addr(struct as * as,caddr_t * addrp,size_t len,offset_t off,int vacalign,uint_t flags)499 choose_addr(struct as *as, caddr_t *addrp, size_t len, offset_t off,
500     int vacalign, uint_t flags)
501 {
502 	caddr_t basep = (caddr_t)(uintptr_t)((uintptr_t)*addrp & PAGEMASK);
503 	size_t lenp = len;
504 
505 	ASSERT(AS_ISCLAIMGAP(as));	/* searches should be serialized */
506 	if (flags & MAP_FIXED) {
507 		(void) as_unmap(as, *addrp, len);
508 		return (0);
509 	} else if (basep != NULL && ((flags & MAP_ALIGN) == 0) &&
510 	    !as_gap(as, len, &basep, &lenp, 0, *addrp)) {
511 		/* User supplied address was available */
512 		*addrp = basep;
513 	} else {
514 		/*
515 		 * No user supplied address or the address supplied was not
516 		 * available.
517 		 */
518 		map_addr(addrp, len, off, vacalign, flags);
519 	}
520 	if (*addrp == NULL)
521 		return (ENOMEM);
522 	return (0);
523 }
524 
525 
526 /*
527  * Used for MAP_ANON - fast way to get anonymous pages
528  */
529 static int
zmap(struct as * as,caddr_t * addrp,size_t len,uint_t uprot,int flags,offset_t pos)530 zmap(struct as *as, caddr_t *addrp, size_t len, uint_t uprot, int flags,
531     offset_t pos)
532 {
533 	struct segvn_crargs vn_a;
534 	int error;
535 
536 	if (((PROT_ALL & uprot) != uprot))
537 		return (EACCES);
538 
539 	if ((flags & MAP_FIXED) != 0) {
540 		caddr_t userlimit;
541 
542 		/*
543 		 * Use the user address.  First verify that
544 		 * the address to be used is page aligned.
545 		 * Then make some simple bounds checks.
546 		 */
547 		if (((uintptr_t)*addrp & PAGEOFFSET) != 0)
548 			return (EINVAL);
549 
550 		userlimit = flags & _MAP_LOW32 ?
551 		    (caddr_t)USERLIMIT32 : as->a_userlimit;
552 		switch (valid_usr_range(*addrp, len, uprot, as, userlimit)) {
553 		case RANGE_OKAY:
554 			break;
555 		case RANGE_BADPROT:
556 			return (ENOTSUP);
557 		case RANGE_BADADDR:
558 		default:
559 			return (ENOMEM);
560 		}
561 	}
562 	/*
563 	 * No need to worry about vac alignment for anonymous
564 	 * pages since this is a "clone" object that doesn't
565 	 * yet exist.
566 	 */
567 	error = choose_addr(as, addrp, len, pos, ADDR_NOVACALIGN, flags);
568 	if (error != 0) {
569 		return (error);
570 	}
571 
572 	/*
573 	 * Use the seg_vn segment driver; passing in the NULL amp
574 	 * gives the desired "cloning" effect.
575 	 */
576 	vn_a.vp = NULL;
577 	vn_a.offset = 0;
578 	vn_a.type = flags & MAP_TYPE;
579 	vn_a.prot = uprot;
580 	vn_a.maxprot = PROT_ALL;
581 	vn_a.flags = flags & ~MAP_TYPE;
582 	vn_a.cred = CRED();
583 	vn_a.amp = NULL;
584 	vn_a.szc = 0;
585 	vn_a.lgrp_mem_policy_flags = 0;
586 
587 	return (as_map(as, *addrp, len, segvn_create, &vn_a));
588 }
589 
590 static int
smmap_common(caddr_t * addrp,size_t len,int prot,int flags,struct file * fp,offset_t pos)591 smmap_common(caddr_t *addrp, size_t len,
592     int prot, int flags, struct file *fp, offset_t pos)
593 {
594 	struct vnode *vp;
595 	struct as *as = curproc->p_as;
596 	uint_t uprot, maxprot, type;
597 	int error;
598 	int in_crit = 0;
599 
600 	if ((flags & ~(MAP_SHARED | MAP_PRIVATE | MAP_FIXED | _MAP_NEW |
601 	    _MAP_LOW32 | MAP_NORESERVE | MAP_ANON | MAP_ALIGN |
602 	    MAP_TEXT | MAP_INITDATA)) != 0) {
603 		/* | MAP_RENAME */	/* not implemented, let user know */
604 		return (EINVAL);
605 	}
606 
607 	if ((flags & MAP_TEXT) && !(prot & PROT_EXEC)) {
608 		return (EINVAL);
609 	}
610 
611 	if ((flags & (MAP_TEXT | MAP_INITDATA)) == (MAP_TEXT | MAP_INITDATA)) {
612 		return (EINVAL);
613 	}
614 
615 #if defined(__sparc)
616 	/*
617 	 * See if this is an "old mmap call".  If so, remember this
618 	 * fact and convert the flags value given to mmap to indicate
619 	 * the specified address in the system call must be used.
620 	 * _MAP_NEW is turned set by all new uses of mmap.
621 	 */
622 	if ((flags & _MAP_NEW) == 0)
623 		flags |= MAP_FIXED;
624 #endif
625 	flags &= ~_MAP_NEW;
626 
627 	type = flags & MAP_TYPE;
628 	if (type != MAP_PRIVATE && type != MAP_SHARED)
629 		return (EINVAL);
630 
631 
632 	if (flags & MAP_ALIGN) {
633 
634 		if (flags & MAP_FIXED)
635 			return (EINVAL);
636 
637 		/* alignment needs to be a power of 2 >= page size */
638 		if (((uintptr_t)*addrp < PAGESIZE && (uintptr_t)*addrp != 0) ||
639 		    !ISP2((uintptr_t)*addrp))
640 			return (EINVAL);
641 	}
642 	/*
643 	 * Check for bad lengths and file position.
644 	 * We let the VOP_MAP routine check for negative lengths
645 	 * since on some vnode types this might be appropriate.
646 	 */
647 	if (len == 0 || (pos & (u_offset_t)PAGEOFFSET) != 0)
648 		return (EINVAL);
649 
650 	maxprot = PROT_ALL;		/* start out allowing all accesses */
651 	uprot = prot | PROT_USER;
652 
653 	if (fp == NULL) {
654 		ASSERT(flags & MAP_ANON);
655 		/* discard lwpchan mappings, like munmap() */
656 		if ((flags & MAP_FIXED) && curproc->p_lcp != NULL)
657 			lwpchan_delete_mapping(curproc, *addrp, *addrp + len);
658 		as_rangelock(as);
659 		error = zmap(as, addrp, len, uprot, flags, pos);
660 		as_rangeunlock(as);
661 		/*
662 		 * Tell machine specific code that lwp has mapped shared memory
663 		 */
664 		if (error == 0 && (flags & MAP_SHARED)) {
665 			/* EMPTY */
666 			LWP_MMODEL_SHARED_AS(*addrp, len);
667 		}
668 		return (error);
669 	} else if ((flags & MAP_ANON) != 0)
670 		return (EINVAL);
671 
672 	vp = fp->f_vnode;
673 
674 	/* Can't execute code from "noexec" mounted filesystem. */
675 	if ((vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0)
676 		maxprot &= ~PROT_EXEC;
677 
678 	/*
679 	 * These checks were added as part of large files.
680 	 *
681 	 * Return ENXIO if the initial position is negative; return EOVERFLOW
682 	 * if (offset + len) would overflow the maximum allowed offset for the
683 	 * type of file descriptor being used.
684 	 */
685 	if (vp->v_type == VREG) {
686 		if (pos < 0)
687 			return (ENXIO);
688 		if ((offset_t)len > (OFFSET_MAX(fp) - pos))
689 			return (EOVERFLOW);
690 	}
691 
692 	if (type == MAP_SHARED && (fp->f_flag & FWRITE) == 0) {
693 		/* no write access allowed */
694 		maxprot &= ~PROT_WRITE;
695 	}
696 
697 	/*
698 	 * XXX - Do we also adjust maxprot based on protections
699 	 * of the vnode?  E.g. if no execute permission is given
700 	 * on the vnode for the current user, maxprot probably
701 	 * should disallow PROT_EXEC also?  This is different
702 	 * from the write access as this would be a per vnode
703 	 * test as opposed to a per fd test for writability.
704 	 */
705 
706 	/*
707 	 * Verify that the specified protections are not greater than
708 	 * the maximum allowable protections.  Also test to make sure
709 	 * that the file descriptor does allows for read access since
710 	 * "write only" mappings are hard to do since normally we do
711 	 * the read from the file before the page can be written.
712 	 */
713 	if (((maxprot & uprot) != uprot) || (fp->f_flag & FREAD) == 0)
714 		return (EACCES);
715 
716 	/*
717 	 * If the user specified an address, do some simple checks here
718 	 */
719 	if ((flags & MAP_FIXED) != 0) {
720 		caddr_t userlimit;
721 
722 		/*
723 		 * Use the user address.  First verify that
724 		 * the address to be used is page aligned.
725 		 * Then make some simple bounds checks.
726 		 */
727 		if (((uintptr_t)*addrp & PAGEOFFSET) != 0)
728 			return (EINVAL);
729 
730 		userlimit = flags & _MAP_LOW32 ?
731 		    (caddr_t)USERLIMIT32 : as->a_userlimit;
732 		switch (valid_usr_range(*addrp, len, uprot, as, userlimit)) {
733 		case RANGE_OKAY:
734 			break;
735 		case RANGE_BADPROT:
736 			return (ENOTSUP);
737 		case RANGE_BADADDR:
738 		default:
739 			return (ENOMEM);
740 		}
741 	}
742 
743 	if ((prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) &&
744 	    nbl_need_check(vp)) {
745 		int svmand;
746 		nbl_op_t nop;
747 
748 		nbl_start_crit(vp, RW_READER);
749 		in_crit = 1;
750 		error = nbl_svmand(vp, fp->f_cred, &svmand);
751 		if (error != 0)
752 			goto done;
753 		if ((prot & PROT_WRITE) && (type == MAP_SHARED)) {
754 			if (prot & (PROT_READ | PROT_EXEC)) {
755 				nop = NBL_READWRITE;
756 			} else {
757 				nop = NBL_WRITE;
758 			}
759 		} else {
760 			nop = NBL_READ;
761 		}
762 		if (nbl_conflict(vp, nop, 0, LONG_MAX, svmand, NULL)) {
763 			error = EACCES;
764 			goto done;
765 		}
766 	}
767 
768 	/* discard lwpchan mappings, like munmap() */
769 	if ((flags & MAP_FIXED) && curproc->p_lcp != NULL)
770 		lwpchan_delete_mapping(curproc, *addrp, *addrp + len);
771 
772 	/*
773 	 * Ok, now let the vnode map routine do its thing to set things up.
774 	 */
775 	error = VOP_MAP(vp, pos, as,
776 	    addrp, len, uprot, maxprot, flags, fp->f_cred, NULL);
777 
778 	if (error == 0) {
779 		/*
780 		 * Tell machine specific code that lwp has mapped shared memory
781 		 */
782 		if (flags & MAP_SHARED) {
783 			/* EMPTY */
784 			LWP_MMODEL_SHARED_AS(*addrp, len);
785 		}
786 		if (vp->v_type == VREG &&
787 		    (flags & (MAP_TEXT | MAP_INITDATA)) != 0) {
788 			/*
789 			 * Mark this as an executable vnode
790 			 */
791 			mutex_enter(&vp->v_lock);
792 			vp->v_flag |= VVMEXEC;
793 			mutex_exit(&vp->v_lock);
794 		}
795 	}
796 
797 done:
798 	if (in_crit)
799 		nbl_end_crit(vp);
800 	return (error);
801 }
802 
803 #ifdef _LP64
804 /*
805  * LP64 mmap(2) system call: 64-bit offset, 64-bit address.
806  *
807  * The "large file" mmap routine mmap64(2) is also mapped to this routine
808  * by the 64-bit version of libc.
809  *
810  * Eventually, this should be the only version, and have smmap_common()
811  * folded back into it again.  Some day.
812  */
813 caddr_t
smmap64(caddr_t addr,size_t len,int prot,int flags,int fd,off_t pos)814 smmap64(caddr_t addr, size_t len, int prot, int flags, int fd, off_t pos)
815 {
816 	struct file *fp;
817 	int error;
818 
819 	if (fd == -1 && (flags & MAP_ANON) != 0)
820 		error = smmap_common(&addr, len, prot, flags,
821 		    NULL, (offset_t)pos);
822 	else if ((fp = getf(fd)) != NULL) {
823 		error = smmap_common(&addr, len, prot, flags,
824 		    fp, (offset_t)pos);
825 		releasef(fd);
826 	} else
827 		error = EBADF;
828 
829 	return (error ? (caddr_t)(uintptr_t)set_errno(error) : addr);
830 }
831 #endif	/* _LP64 */
832 
833 #if defined(_SYSCALL32_IMPL) || defined(_ILP32)
834 
835 /*
836  * ILP32 mmap(2) system call: 32-bit offset, 32-bit address.
837  */
838 caddr_t
smmap32(caddr32_t addr,size32_t len,int prot,int flags,int fd,off32_t pos)839 smmap32(caddr32_t addr, size32_t len, int prot, int flags, int fd, off32_t pos)
840 {
841 	struct file *fp;
842 	int error;
843 	caddr_t a = (caddr_t)(uintptr_t)addr;
844 
845 	if (flags & _MAP_LOW32)
846 		error = EINVAL;
847 	else if (fd == -1 && (flags & MAP_ANON) != 0)
848 		error = smmap_common(&a, (size_t)len, prot,
849 		    flags | _MAP_LOW32, NULL, (offset_t)pos);
850 	else if ((fp = getf(fd)) != NULL) {
851 		error = smmap_common(&a, (size_t)len, prot,
852 		    flags | _MAP_LOW32, fp, (offset_t)pos);
853 		releasef(fd);
854 	} else
855 		error = EBADF;
856 
857 	ASSERT(error != 0 || (uintptr_t)(a + len) < (uintptr_t)UINT32_MAX);
858 
859 	return (error ? (caddr_t)(uintptr_t)set_errno(error) : a);
860 }
861 
862 /*
863  * ILP32 mmap64(2) system call: 64-bit offset, 32-bit address.
864  *
865  * Now things really get ugly because we can't use the C-style
866  * calling convention for more than 6 args, and 64-bit parameter
867  * passing on 32-bit systems is less than clean.
868  */
869 
870 struct mmaplf32a {
871 	caddr_t addr;
872 	size_t len;
873 #ifdef _LP64
874 	/*
875 	 * 32-bit contents, 64-bit cells
876 	 */
877 	uint64_t prot;
878 	uint64_t flags;
879 	uint64_t fd;
880 	uint64_t offhi;
881 	uint64_t offlo;
882 #else
883 	/*
884 	 * 32-bit contents, 32-bit cells
885 	 */
886 	uint32_t prot;
887 	uint32_t flags;
888 	uint32_t fd;
889 	uint32_t offhi;
890 	uint32_t offlo;
891 #endif
892 };
893 
894 int
smmaplf32(struct mmaplf32a * uap,rval_t * rvp)895 smmaplf32(struct mmaplf32a *uap, rval_t *rvp)
896 {
897 	struct file *fp;
898 	int error;
899 	caddr_t a = uap->addr;
900 	int flags = (int)uap->flags;
901 	int fd = (int)uap->fd;
902 #ifdef _BIG_ENDIAN
903 	offset_t off = ((u_offset_t)uap->offhi << 32) | (u_offset_t)uap->offlo;
904 #else
905 	offset_t off = ((u_offset_t)uap->offlo << 32) | (u_offset_t)uap->offhi;
906 #endif
907 
908 	if (flags & _MAP_LOW32)
909 		error = EINVAL;
910 	else if (fd == -1 && (flags & MAP_ANON) != 0)
911 		error = smmap_common(&a, uap->len, (int)uap->prot,
912 		    flags | _MAP_LOW32, NULL, off);
913 	else if ((fp = getf(fd)) != NULL) {
914 		error = smmap_common(&a, uap->len, (int)uap->prot,
915 		    flags | _MAP_LOW32, fp, off);
916 		releasef(fd);
917 	} else
918 		error = EBADF;
919 
920 	if (error == 0)
921 		rvp->r_val1 = (uintptr_t)a;
922 	return (error);
923 }
924 
925 #endif	/* _SYSCALL32_IMPL || _ILP32 */
926 
927 int
munmap(caddr_t addr,size_t len)928 munmap(caddr_t addr, size_t len)
929 {
930 	struct proc *p = curproc;
931 	struct as *as = p->p_as;
932 
933 	if (((uintptr_t)addr & PAGEOFFSET) != 0 || len == 0)
934 		return (set_errno(EINVAL));
935 
936 	if (valid_usr_range(addr, len, 0, as, as->a_userlimit) != RANGE_OKAY)
937 		return (set_errno(EINVAL));
938 
939 	/*
940 	 * Discard lwpchan mappings.
941 	 */
942 	if (p->p_lcp != NULL)
943 		lwpchan_delete_mapping(p, addr, addr + len);
944 	if (as_unmap(as, addr, len) != 0)
945 		return (set_errno(EINVAL));
946 
947 	return (0);
948 }
949 
950 int
mprotect(caddr_t addr,size_t len,int prot)951 mprotect(caddr_t addr, size_t len, int prot)
952 {
953 	struct as *as = curproc->p_as;
954 	uint_t uprot = prot | PROT_USER;
955 	int error;
956 
957 	if (((uintptr_t)addr & PAGEOFFSET) != 0 || len == 0)
958 		return (set_errno(EINVAL));
959 
960 	switch (valid_usr_range(addr, len, prot, as, as->a_userlimit)) {
961 	case RANGE_OKAY:
962 		break;
963 	case RANGE_BADPROT:
964 		return (set_errno(ENOTSUP));
965 	case RANGE_BADADDR:
966 	default:
967 		return (set_errno(ENOMEM));
968 	}
969 
970 	error = as_setprot(as, addr, len, uprot);
971 	if (error)
972 		return (set_errno(error));
973 	return (0);
974 }
975 
976 #define	MC_CACHE	128			/* internal result buffer */
977 #define	MC_QUANTUM	(MC_CACHE * PAGESIZE)	/* addresses covered in loop */
978 
979 int
mincore(caddr_t addr,size_t len,char * vecp)980 mincore(caddr_t addr, size_t len, char *vecp)
981 {
982 	struct as *as = curproc->p_as;
983 	caddr_t ea;			/* end address of loop */
984 	size_t rl;			/* inner result length */
985 	char vec[MC_CACHE];		/* local vector cache */
986 	int error;
987 	model_t model;
988 	long	llen;
989 
990 	model = get_udatamodel();
991 	/*
992 	 * Validate form of address parameters.
993 	 */
994 	if (model == DATAMODEL_NATIVE) {
995 		llen = (long)len;
996 	} else {
997 		llen = (int32_t)(size32_t)len;
998 	}
999 	if (((uintptr_t)addr & PAGEOFFSET) != 0 || llen <= 0)
1000 		return (set_errno(EINVAL));
1001 
1002 	if (valid_usr_range(addr, len, 0, as, as->a_userlimit) != RANGE_OKAY)
1003 		return (set_errno(ENOMEM));
1004 
1005 	/*
1006 	 * Loop over subranges of interval [addr : addr + len), recovering
1007 	 * results internally and then copying them out to caller.  Subrange
1008 	 * is based on the size of MC_CACHE, defined above.
1009 	 */
1010 	for (ea = addr + len; addr < ea; addr += MC_QUANTUM) {
1011 		error = as_incore(as, addr,
1012 		    (size_t)MIN(MC_QUANTUM, ea - addr), vec, &rl);
1013 		if (rl != 0) {
1014 			rl = (rl + PAGESIZE - 1) / PAGESIZE;
1015 			if (copyout(vec, vecp, rl) != 0)
1016 				return (set_errno(EFAULT));
1017 			vecp += rl;
1018 		}
1019 		if (error != 0)
1020 			return (set_errno(ENOMEM));
1021 	}
1022 	return (0);
1023 }
1024