xref: /titanic_41/usr/src/uts/common/vm/vm_as.c (revision 2a3221a4fea0ec9c322f85a262eab79e762d32f2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 /*
30  * University Copyright- Copyright (c) 1982, 1986, 1988
31  * The Regents of the University of California
32  * All Rights Reserved
33  *
34  * University Acknowledgment- Portions of this document are derived from
35  * software developed by the University of California, Berkeley, and its
36  * contributors.
37  */
38 
39 #pragma ident	"%Z%%M%	%I%	%E% SMI"
40 
41 /*
42  * VM - address spaces.
43  */
44 
45 #include <sys/types.h>
46 #include <sys/t_lock.h>
47 #include <sys/param.h>
48 #include <sys/errno.h>
49 #include <sys/systm.h>
50 #include <sys/mman.h>
51 #include <sys/sysmacros.h>
52 #include <sys/cpuvar.h>
53 #include <sys/sysinfo.h>
54 #include <sys/kmem.h>
55 #include <sys/vnode.h>
56 #include <sys/vmsystm.h>
57 #include <sys/cmn_err.h>
58 #include <sys/debug.h>
59 #include <sys/tnf_probe.h>
60 #include <sys/vtrace.h>
61 
62 #include <vm/hat.h>
63 #include <vm/xhat.h>
64 #include <vm/as.h>
65 #include <vm/seg.h>
66 #include <vm/seg_vn.h>
67 #include <vm/seg_dev.h>
68 #include <vm/seg_kmem.h>
69 #include <vm/seg_map.h>
70 #include <vm/seg_spt.h>
71 #include <vm/page.h>
72 
73 clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
74 
75 static struct kmem_cache *as_cache;
76 
77 static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
78 static void as_clearwatchprot(struct as *, caddr_t, size_t);
79 int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *);
80 
81 
82 /*
83  * Verifying the segment lists is very time-consuming; it may not be
84  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
85  */
86 #ifdef DEBUG
87 #define	VERIFY_SEGLIST
88 int do_as_verify = 0;
89 #endif
90 
91 /*
92  * Allocate a new callback data structure entry and fill in the events of
93  * interest, the address range of interest, and the callback argument.
94  * Link the entry on the as->a_callbacks list. A callback entry for the
95  * entire address space may be specified with vaddr = 0 and size = -1.
96  *
97  * CALLERS RESPONSIBILITY: If not calling from within the process context for
98  * the specified as, the caller must guarantee persistence of the specified as
99  * for the duration of this function (eg. pages being locked within the as
100  * will guarantee persistence).
101  */
102 int
103 as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
104 		caddr_t vaddr, size_t size, int sleepflag)
105 {
106 	struct as_callback 	*current_head, *cb;
107 	caddr_t 		saddr;
108 	size_t 			rsize;
109 
110 	/* callback function and an event are mandatory */
111 	if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
112 		return (EINVAL);
113 
114 	/* Adding a callback after as_free has been called is not allowed */
115 	if (as == &kas)
116 		return (ENOMEM);
117 
118 	/*
119 	 * vaddr = 0 and size = -1 is used to indicate that the callback range
120 	 * is the entire address space so no rounding is done in that case.
121 	 */
122 	if (size != -1) {
123 		saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
124 		rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
125 			(size_t)saddr;
126 		/* check for wraparound */
127 		if (saddr + rsize < saddr)
128 			return (ENOMEM);
129 	} else {
130 		if (vaddr != 0)
131 			return (EINVAL);
132 		saddr = vaddr;
133 		rsize = size;
134 	}
135 
136 	/* Allocate and initialize a callback entry */
137 	cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
138 	if (cb == NULL)
139 		return (EAGAIN);
140 
141 	cb->ascb_func = cb_func;
142 	cb->ascb_arg = arg;
143 	cb->ascb_events = events;
144 	cb->ascb_saddr = saddr;
145 	cb->ascb_len = rsize;
146 
147 	/* Add the entry to the list */
148 	mutex_enter(&as->a_contents);
149 	current_head = as->a_callbacks;
150 	as->a_callbacks = cb;
151 	cb->ascb_next = current_head;
152 
153 	/*
154 	 * The call to this function may lose in a race with
155 	 * a pertinent event - eg. a thread does long term memory locking
156 	 * but before the callback is added another thread executes as_unmap.
157 	 * A broadcast here resolves that.
158 	 */
159 	if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
160 		AS_CLRUNMAPWAIT(as);
161 		cv_broadcast(&as->a_cv);
162 	}
163 
164 	mutex_exit(&as->a_contents);
165 	return (0);
166 }
167 
168 /*
169  * Search the callback list for an entry which pertains to arg.
170  *
171  * This is called from within the client upon completion of the callback.
172  * RETURN VALUES:
173  *	AS_CALLBACK_DELETED  (callback entry found and deleted)
174  *	AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
175  *	AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
176  *			entry will be made in as_do_callbacks)
177  *
178  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
179  * set, it indicates that as_do_callbacks is processing this entry.  The
180  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
181  * to unblock as_do_callbacks, in case it is blocked.
182  *
183  * CALLERS RESPONSIBILITY: If not calling from within the process context for
184  * the specified as, the caller must guarantee persistence of the specified as
185  * for the duration of this function (eg. pages being locked within the as
186  * will guarantee persistence).
187  */
188 uint_t
189 as_delete_callback(struct as *as, void *arg)
190 {
191 	struct as_callback **prevcb = &as->a_callbacks;
192 	struct as_callback *cb;
193 	uint_t rc = AS_CALLBACK_NOTFOUND;
194 
195 	mutex_enter(&as->a_contents);
196 	for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
197 		if (cb->ascb_arg != arg)
198 			continue;
199 
200 		/*
201 		 * If the events indicate AS_CALLBACK_CALLED, just clear
202 		 * AS_ALL_EVENT in the events field and wakeup the thread
203 		 * that may be waiting in as_do_callbacks.  as_do_callbacks
204 		 * will take care of removing this entry from the list.  In
205 		 * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
206 		 * (AS_CALLBACK_CALLED not set), just remove it from the
207 		 * list, return the memory and return AS_CALLBACK_DELETED.
208 		 */
209 		if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
210 			/* leave AS_CALLBACK_CALLED */
211 			cb->ascb_events &= ~AS_ALL_EVENT;
212 			rc = AS_CALLBACK_DELETE_DEFERRED;
213 			cv_broadcast(&as->a_cv);
214 		} else {
215 			*prevcb = cb->ascb_next;
216 			kmem_free(cb, sizeof (struct as_callback));
217 			rc = AS_CALLBACK_DELETED;
218 		}
219 		break;
220 	}
221 	mutex_exit(&as->a_contents);
222 	return (rc);
223 }
224 
225 /*
226  * Searches the as callback list for a matching entry.
227  * Returns a pointer to the first matching callback, or NULL if
228  * nothing is found.
229  * This function never sleeps so it is ok to call it with more
230  * locks held but the (required) a_contents mutex.
231  *
232  * See also comment on as_do_callbacks below.
233  */
234 static struct as_callback *
235 as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
236 			size_t event_len)
237 {
238 	struct as_callback	*cb;
239 
240 	ASSERT(MUTEX_HELD(&as->a_contents));
241 	for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
242 		/*
243 		 * If the callback has not already been called, then
244 		 * check if events or address range pertains.  An event_len
245 		 * of zero means do an unconditional callback.
246 		 */
247 		if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
248 		    ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
249 		    (event_addr + event_len < cb->ascb_saddr) ||
250 		    (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
251 			continue;
252 		}
253 		break;
254 	}
255 	return (cb);
256 }
257 
258 /*
259  * Executes a given callback and removes it from the callback list for
260  * this address space.
261  * This function may sleep so the caller must drop all locks except
262  * a_contents before calling this func.
263  *
264  * See also comments on as_do_callbacks below.
265  */
266 static void
267 as_execute_callback(struct as *as, struct as_callback *cb,
268 				uint_t events)
269 {
270 	struct as_callback **prevcb;
271 	void	*cb_arg;
272 
273 	ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
274 	cb->ascb_events |= AS_CALLBACK_CALLED;
275 	mutex_exit(&as->a_contents);
276 	(*cb->ascb_func)(as, cb->ascb_arg, events);
277 	mutex_enter(&as->a_contents);
278 	/*
279 	 * the callback function is required to delete the callback
280 	 * when the callback function determines it is OK for
281 	 * this thread to continue. as_delete_callback will clear
282 	 * the AS_ALL_EVENT in the events field when it is deleted.
283 	 * If the callback function called as_delete_callback,
284 	 * events will already be cleared and there will be no blocking.
285 	 */
286 	while ((cb->ascb_events & events) != 0) {
287 		cv_wait(&as->a_cv, &as->a_contents);
288 	}
289 	/*
290 	 * This entry needs to be taken off the list. Normally, the
291 	 * callback func itself does that, but unfortunately the list
292 	 * may have changed while the callback was running because the
293 	 * a_contents mutex was dropped and someone else other than the
294 	 * callback func itself could have called as_delete_callback,
295 	 * so we have to search to find this entry again.  The entry
296 	 * must have AS_CALLBACK_CALLED, and have the same 'arg'.
297 	 */
298 	cb_arg = cb->ascb_arg;
299 	prevcb = &as->a_callbacks;
300 	for (cb = as->a_callbacks; cb != NULL;
301 	    prevcb = &cb->ascb_next, cb = *prevcb) {
302 		if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
303 		    (cb_arg != cb->ascb_arg)) {
304 			continue;
305 		}
306 		*prevcb = cb->ascb_next;
307 		kmem_free(cb, sizeof (struct as_callback));
308 		break;
309 	}
310 }
311 
312 /*
313  * Check the callback list for a matching event and intersection of
314  * address range. If there is a match invoke the callback.  Skip an entry if:
315  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
316  *    - not event of interest
317  *    - not address range of interest
318  *
319  * An event_len of zero indicates a request for an unconditional callback
320  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
321  * a_contents lock must be dropped before a callback, so only one callback
322  * can be done before returning. Return -1 (true) if a callback was
323  * executed and removed from the list, else return 0 (false).
324  *
325  * The logically separate parts, i.e. finding a matching callback and
326  * executing a given callback have been separated into two functions
327  * so that they can be called with different sets of locks held beyond
328  * the always-required a_contents. as_find_callback does not sleep so
329  * it is ok to call it if more locks than a_contents (i.e. the a_lock
330  * rwlock) are held. as_execute_callback on the other hand may sleep
331  * so all locks beyond a_contents must be dropped by the caller if one
332  * does not want to end comatose.
333  */
334 static int
335 as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
336 			size_t event_len)
337 {
338 	struct as_callback *cb;
339 
340 	if ((cb = as_find_callback(as, events, event_addr, event_len))) {
341 		as_execute_callback(as, cb, events);
342 		return (-1);
343 	}
344 	return (0);
345 }
346 
347 /*
348  * Search for the segment containing addr. If a segment containing addr
349  * exists, that segment is returned.  If no such segment exists, and
350  * the list spans addresses greater than addr, then the first segment
351  * whose base is greater than addr is returned; otherwise, NULL is
352  * returned unless tail is true, in which case the last element of the
353  * list is returned.
354  *
355  * a_seglast is used to cache the last found segment for repeated
356  * searches to the same addr (which happens frequently).
357  */
358 struct seg *
359 as_findseg(struct as *as, caddr_t addr, int tail)
360 {
361 	struct seg *seg = as->a_seglast;
362 	avl_index_t where;
363 
364 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
365 
366 	if (seg != NULL &&
367 	    seg->s_base <= addr &&
368 	    addr < seg->s_base + seg->s_size)
369 		return (seg);
370 
371 	seg = avl_find(&as->a_segtree, &addr, &where);
372 	if (seg != NULL)
373 		return (as->a_seglast = seg);
374 
375 	seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
376 	if (seg == NULL && tail)
377 		seg = avl_last(&as->a_segtree);
378 	return (as->a_seglast = seg);
379 }
380 
381 #ifdef VERIFY_SEGLIST
382 /*
383  * verify that the linked list is coherent
384  */
385 static void
386 as_verify(struct as *as)
387 {
388 	struct seg *seg, *seglast, *p, *n;
389 	uint_t nsegs = 0;
390 
391 	if (do_as_verify == 0)
392 		return;
393 
394 	seglast = as->a_seglast;
395 
396 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
397 		ASSERT(seg->s_as == as);
398 		p = AS_SEGPREV(as, seg);
399 		n = AS_SEGNEXT(as, seg);
400 		ASSERT(p == NULL || p->s_as == as);
401 		ASSERT(p == NULL || p->s_base < seg->s_base);
402 		ASSERT(n == NULL || n->s_base > seg->s_base);
403 		ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
404 		if (seg == seglast)
405 			seglast = NULL;
406 		nsegs++;
407 	}
408 	ASSERT(seglast == NULL);
409 	ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
410 }
411 #endif /* VERIFY_SEGLIST */
412 
413 /*
414  * Add a new segment to the address space. The avl_find()
415  * may be expensive so we attempt to use last segment accessed
416  * in as_gap() as an insertion point.
417  */
418 int
419 as_addseg(struct as  *as, struct seg *newseg)
420 {
421 	struct seg *seg;
422 	caddr_t addr;
423 	caddr_t eaddr;
424 	avl_index_t where;
425 
426 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
427 
428 	as->a_updatedir = 1;	/* inform /proc */
429 	gethrestime(&as->a_updatetime);
430 
431 	if (as->a_lastgaphl != NULL) {
432 		struct seg *hseg = NULL;
433 		struct seg *lseg = NULL;
434 
435 		if (as->a_lastgaphl->s_base > newseg->s_base) {
436 			hseg = as->a_lastgaphl;
437 			lseg = AVL_PREV(&as->a_segtree, hseg);
438 		} else {
439 			lseg = as->a_lastgaphl;
440 			hseg = AVL_NEXT(&as->a_segtree, lseg);
441 		}
442 
443 		if (hseg && lseg && lseg->s_base < newseg->s_base &&
444 		    hseg->s_base > newseg->s_base) {
445 			avl_insert_here(&as->a_segtree, newseg, lseg,
446 			    AVL_AFTER);
447 			as->a_lastgaphl = NULL;
448 			as->a_seglast = newseg;
449 			return (0);
450 		}
451 		as->a_lastgaphl = NULL;
452 	}
453 
454 	addr = newseg->s_base;
455 	eaddr = addr + newseg->s_size;
456 again:
457 
458 	seg = avl_find(&as->a_segtree, &addr, &where);
459 
460 	if (seg == NULL)
461 		seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
462 
463 	if (seg == NULL)
464 		seg = avl_last(&as->a_segtree);
465 
466 	if (seg != NULL) {
467 		caddr_t base = seg->s_base;
468 
469 		/*
470 		 * If top of seg is below the requested address, then
471 		 * the insertion point is at the end of the linked list,
472 		 * and seg points to the tail of the list.  Otherwise,
473 		 * the insertion point is immediately before seg.
474 		 */
475 		if (base + seg->s_size > addr) {
476 			if (addr >= base || eaddr > base) {
477 #ifdef __sparc
478 				extern struct seg_ops segnf_ops;
479 
480 				/*
481 				 * no-fault segs must disappear if overlaid.
482 				 * XXX need new segment type so
483 				 * we don't have to check s_ops
484 				 */
485 				if (seg->s_ops == &segnf_ops) {
486 					seg_unmap(seg);
487 					goto again;
488 				}
489 #endif
490 				return (-1);	/* overlapping segment */
491 			}
492 		}
493 	}
494 	as->a_seglast = newseg;
495 	avl_insert(&as->a_segtree, newseg, where);
496 
497 #ifdef VERIFY_SEGLIST
498 	as_verify(as);
499 #endif
500 	return (0);
501 }
502 
503 struct seg *
504 as_removeseg(struct as *as, struct seg *seg)
505 {
506 	avl_tree_t *t;
507 
508 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
509 
510 	as->a_updatedir = 1;	/* inform /proc */
511 	gethrestime(&as->a_updatetime);
512 
513 	if (seg == NULL)
514 		return (NULL);
515 
516 	t = &as->a_segtree;
517 	if (as->a_seglast == seg)
518 		as->a_seglast = NULL;
519 	as->a_lastgaphl = NULL;
520 
521 	/*
522 	 * if this segment is at an address higher than
523 	 * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
524 	 */
525 	if (as->a_lastgap &&
526 	    (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
527 		as->a_lastgap = AVL_NEXT(t, seg);
528 
529 	/*
530 	 * remove the segment from the seg tree
531 	 */
532 	avl_remove(t, seg);
533 
534 #ifdef VERIFY_SEGLIST
535 	as_verify(as);
536 #endif
537 	return (seg);
538 }
539 
540 /*
541  * Find a segment containing addr.
542  */
543 struct seg *
544 as_segat(struct as *as, caddr_t addr)
545 {
546 	struct seg *seg = as->a_seglast;
547 
548 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
549 
550 	if (seg != NULL && seg->s_base <= addr &&
551 	    addr < seg->s_base + seg->s_size)
552 		return (seg);
553 
554 	seg = avl_find(&as->a_segtree, &addr, NULL);
555 	return (seg);
556 }
557 
558 /*
559  * Serialize all searches for holes in an address space to
560  * prevent two or more threads from allocating the same virtual
561  * address range.  The address space must not be "read/write"
562  * locked by the caller since we may block.
563  */
564 void
565 as_rangelock(struct as *as)
566 {
567 	mutex_enter(&as->a_contents);
568 	while (AS_ISCLAIMGAP(as))
569 		cv_wait(&as->a_cv, &as->a_contents);
570 	AS_SETCLAIMGAP(as);
571 	mutex_exit(&as->a_contents);
572 }
573 
574 /*
575  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
576  */
577 void
578 as_rangeunlock(struct as *as)
579 {
580 	mutex_enter(&as->a_contents);
581 	AS_CLRCLAIMGAP(as);
582 	cv_signal(&as->a_cv);
583 	mutex_exit(&as->a_contents);
584 }
585 
586 /*
587  * compar segments (or just an address) by segment address range
588  */
589 static int
590 as_segcompar(const void *x, const void *y)
591 {
592 	struct seg *a = (struct seg *)x;
593 	struct seg *b = (struct seg *)y;
594 
595 	if (a->s_base < b->s_base)
596 		return (-1);
597 	if (a->s_base >= b->s_base + b->s_size)
598 		return (1);
599 	return (0);
600 }
601 
602 
603 void
604 as_avlinit(struct as *as)
605 {
606 	avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
607 	    offsetof(struct seg, s_tree));
608 	avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
609 	    offsetof(struct watched_page, wp_link));
610 }
611 
612 /*ARGSUSED*/
613 static int
614 as_constructor(void *buf, void *cdrarg, int kmflags)
615 {
616 	struct as *as = buf;
617 
618 	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
619 	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
620 	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
621 	as_avlinit(as);
622 	return (0);
623 }
624 
625 /*ARGSUSED1*/
626 static void
627 as_destructor(void *buf, void *cdrarg)
628 {
629 	struct as *as = buf;
630 
631 	avl_destroy(&as->a_segtree);
632 	mutex_destroy(&as->a_contents);
633 	cv_destroy(&as->a_cv);
634 	rw_destroy(&as->a_lock);
635 }
636 
637 void
638 as_init(void)
639 {
640 	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
641 		as_constructor, as_destructor, NULL, NULL, NULL, 0);
642 }
643 
644 /*
645  * Allocate and initialize an address space data structure.
646  * We call hat_alloc to allow any machine dependent
647  * information in the hat structure to be initialized.
648  */
649 struct as *
650 as_alloc(void)
651 {
652 	struct as *as;
653 
654 	as = kmem_cache_alloc(as_cache, KM_SLEEP);
655 
656 	as->a_flags		= 0;
657 	as->a_vbits		= 0;
658 	as->a_hrm		= NULL;
659 	as->a_seglast		= NULL;
660 	as->a_size		= 0;
661 	as->a_updatedir		= 0;
662 	gethrestime(&as->a_updatetime);
663 	as->a_objectdir		= NULL;
664 	as->a_sizedir		= 0;
665 	as->a_userlimit		= (caddr_t)USERLIMIT;
666 	as->a_lastgap		= NULL;
667 	as->a_lastgaphl		= NULL;
668 	as->a_callbacks		= NULL;
669 
670 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
671 	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
672 	AS_LOCK_EXIT(as, &as->a_lock);
673 
674 	as->a_xhat = NULL;
675 
676 	return (as);
677 }
678 
679 /*
680  * Free an address space data structure.
681  * Need to free the hat first and then
682  * all the segments on this as and finally
683  * the space for the as struct itself.
684  */
685 void
686 as_free(struct as *as)
687 {
688 	struct hat *hat = as->a_hat;
689 	struct seg *seg, *next;
690 	int called = 0;
691 
692 top:
693 	/*
694 	 * Invoke ALL callbacks. as_do_callbacks will do one callback
695 	 * per call, and not return (-1) until the callback has completed.
696 	 * When as_do_callbacks returns zero, all callbacks have completed.
697 	 */
698 	mutex_enter(&as->a_contents);
699 	while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0));
700 
701 	/* This will prevent new XHATs from attaching to as */
702 	if (!called)
703 		AS_SETBUSY(as);
704 	mutex_exit(&as->a_contents);
705 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
706 
707 	if (!called) {
708 		called = 1;
709 		hat_free_start(hat);
710 		if (as->a_xhat != NULL)
711 			xhat_free_start_all(as);
712 	}
713 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
714 		int err;
715 
716 		next = AS_SEGNEXT(as, seg);
717 		err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
718 		if (err == EAGAIN) {
719 			mutex_enter(&as->a_contents);
720 			if (as->a_callbacks) {
721 				AS_LOCK_EXIT(as, &as->a_lock);
722 			} else {
723 				/*
724 				 * Memory is currently locked. Wait for a
725 				 * cv_signal that it has been unlocked, then
726 				 * try the operation again.
727 				 */
728 				if (AS_ISUNMAPWAIT(as) == 0)
729 					cv_broadcast(&as->a_cv);
730 				AS_SETUNMAPWAIT(as);
731 				AS_LOCK_EXIT(as, &as->a_lock);
732 				while (AS_ISUNMAPWAIT(as))
733 					cv_wait(&as->a_cv, &as->a_contents);
734 			}
735 			mutex_exit(&as->a_contents);
736 			goto top;
737 		} else {
738 			/*
739 			 * We do not expect any other error return at this
740 			 * time. This is similar to an ASSERT in seg_unmap()
741 			 */
742 			ASSERT(err == 0);
743 		}
744 	}
745 	hat_free_end(hat);
746 	if (as->a_xhat != NULL)
747 		xhat_free_end_all(as);
748 	AS_LOCK_EXIT(as, &as->a_lock);
749 
750 	/* /proc stuff */
751 	ASSERT(avl_numnodes(&as->a_wpage) == 0);
752 	if (as->a_objectdir) {
753 		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
754 		as->a_objectdir = NULL;
755 		as->a_sizedir = 0;
756 	}
757 
758 	/*
759 	 * Free the struct as back to kmem.  Assert it has no segments.
760 	 */
761 	ASSERT(avl_numnodes(&as->a_segtree) == 0);
762 	kmem_cache_free(as_cache, as);
763 }
764 
765 int
766 as_dup(struct as *as, struct as **outas)
767 {
768 	struct as *newas;
769 	struct seg *seg, *newseg;
770 	int error;
771 
772 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
773 	as_clearwatch(as);
774 	newas = as_alloc();
775 	newas->a_userlimit = as->a_userlimit;
776 	AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER);
777 
778 	/* This will prevent new XHATs from attaching */
779 	mutex_enter(&as->a_contents);
780 	AS_SETBUSY(as);
781 	mutex_exit(&as->a_contents);
782 	mutex_enter(&newas->a_contents);
783 	AS_SETBUSY(newas);
784 	mutex_exit(&newas->a_contents);
785 
786 
787 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
788 
789 		if (seg->s_flags & S_PURGE)
790 			continue;
791 
792 		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
793 		if (newseg == NULL) {
794 			AS_LOCK_EXIT(newas, &newas->a_lock);
795 			as_setwatch(as);
796 			mutex_enter(&as->a_contents);
797 			AS_CLRBUSY(as);
798 			mutex_exit(&as->a_contents);
799 			AS_LOCK_EXIT(as, &as->a_lock);
800 			as_free(newas);
801 			return (-1);
802 		}
803 		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
804 			/*
805 			 * We call seg_free() on the new seg
806 			 * because the segment is not set up
807 			 * completely; i.e. it has no ops.
808 			 */
809 			as_setwatch(as);
810 			mutex_enter(&as->a_contents);
811 			AS_CLRBUSY(as);
812 			mutex_exit(&as->a_contents);
813 			AS_LOCK_EXIT(as, &as->a_lock);
814 			seg_free(newseg);
815 			AS_LOCK_EXIT(newas, &newas->a_lock);
816 			as_free(newas);
817 			return (error);
818 		}
819 		newas->a_size += seg->s_size;
820 	}
821 
822 	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
823 	if (as->a_xhat != NULL)
824 		error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL);
825 
826 	mutex_enter(&newas->a_contents);
827 	AS_CLRBUSY(newas);
828 	mutex_exit(&newas->a_contents);
829 	AS_LOCK_EXIT(newas, &newas->a_lock);
830 
831 	as_setwatch(as);
832 	mutex_enter(&as->a_contents);
833 	AS_CLRBUSY(as);
834 	mutex_exit(&as->a_contents);
835 	AS_LOCK_EXIT(as, &as->a_lock);
836 	if (error != 0) {
837 		as_free(newas);
838 		return (error);
839 	}
840 	*outas = newas;
841 	return (0);
842 }
843 
844 /*
845  * Handle a ``fault'' at addr for size bytes.
846  */
847 faultcode_t
848 as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
849 	enum fault_type type, enum seg_rw rw)
850 {
851 	struct seg *seg;
852 	caddr_t raddr;			/* rounded down addr */
853 	size_t rsize;			/* rounded up size */
854 	size_t ssize;
855 	faultcode_t res = 0;
856 	caddr_t addrsav;
857 	struct seg *segsav;
858 	int as_lock_held;
859 	klwp_t *lwp = ttolwp(curthread);
860 	int is_xhat = 0;
861 	int holding_wpage = 0;
862 	extern struct seg_ops   segdev_ops;
863 
864 
865 
866 	if (as->a_hat != hat) {
867 		/* This must be an XHAT then */
868 		is_xhat = 1;
869 
870 		if ((type != F_INVAL) || (as == &kas))
871 			return (FC_NOSUPPORT);
872 	}
873 
874 retry:
875 	if (!is_xhat) {
876 		/*
877 		 * Indicate that the lwp is not to be stopped while waiting
878 		 * for a pagefault.  This is to avoid deadlock while debugging
879 		 * a process via /proc over NFS (in particular).
880 		 */
881 		if (lwp != NULL) {
882 			lwp->lwp_nostop++;
883 			lwp->lwp_nostop_r++;
884 		}
885 
886 		/*
887 		 * same length must be used when we softlock and softunlock.
888 		 * We don't support softunlocking lengths less than
889 		 * the original length when there is largepage support.
890 		 * See seg_dev.c for more comments.
891 		 */
892 		switch (type) {
893 
894 		case F_SOFTLOCK:
895 			CPU_STATS_ADD_K(vm, softlock, 1);
896 			break;
897 
898 		case F_SOFTUNLOCK:
899 			break;
900 
901 		case F_PROT:
902 			CPU_STATS_ADD_K(vm, prot_fault, 1);
903 			break;
904 
905 		case F_INVAL:
906 			CPU_STATS_ENTER_K();
907 			CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
908 			if (as == &kas)
909 				CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
910 			CPU_STATS_EXIT_K();
911 			break;
912 		}
913 	}
914 
915 	/* Kernel probe */
916 	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
917 		tnf_opaque,	address,	addr,
918 		tnf_fault_type,	fault_type,	type,
919 		tnf_seg_access,	access,		rw);
920 
921 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
922 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
923 		(size_t)raddr;
924 
925 	/*
926 	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
927 	 * correctness, but then we could be stuck holding this lock for
928 	 * a LONG time if the fault needs to be resolved on a slow
929 	 * filesystem, and then no-one will be able to exec new commands,
930 	 * as exec'ing requires the write lock on the as.
931 	 */
932 	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
933 	    raddr + size < segkmap->s_base + segkmap->s_size) {
934 		/*
935 		 * if (as==&kas), this can't be XHAT: we've already returned
936 		 * FC_NOSUPPORT.
937 		 */
938 		seg = segkmap;
939 		as_lock_held = 0;
940 	} else {
941 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
942 		if (is_xhat && avl_numnodes(&as->a_wpage) != 0) {
943 			/*
944 			 * Grab and hold the writers' lock on the as
945 			 * if the fault is to a watched page.
946 			 * This will keep CPUs from "peeking" at the
947 			 * address range while we're temporarily boosting
948 			 * the permissions for the XHAT device to
949 			 * resolve the fault in the segment layer.
950 			 *
951 			 * We could check whether faulted address
952 			 * is within a watched page and only then grab
953 			 * the writer lock, but this is simpler.
954 			 */
955 			AS_LOCK_EXIT(as, &as->a_lock);
956 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
957 		}
958 
959 		seg = as_segat(as, raddr);
960 		if (seg == NULL) {
961 			AS_LOCK_EXIT(as, &as->a_lock);
962 			if ((lwp != NULL) && (!is_xhat)) {
963 				lwp->lwp_nostop--;
964 				lwp->lwp_nostop_r--;
965 			}
966 			return (FC_NOMAP);
967 		}
968 
969 		as_lock_held = 1;
970 	}
971 
972 	addrsav = raddr;
973 	segsav = seg;
974 
975 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
976 		if (raddr >= seg->s_base + seg->s_size) {
977 			seg = AS_SEGNEXT(as, seg);
978 			if (seg == NULL || raddr != seg->s_base) {
979 				res = FC_NOMAP;
980 				break;
981 			}
982 		}
983 		if (raddr + rsize > seg->s_base + seg->s_size)
984 			ssize = seg->s_base + seg->s_size - raddr;
985 		else
986 			ssize = rsize;
987 
988 		if (!is_xhat || (seg->s_ops != &segdev_ops)) {
989 
990 			if (is_xhat && avl_numnodes(&as->a_wpage) != 0 &&
991 			    pr_is_watchpage_as(raddr, rw, as)) {
992 				/*
993 				 * Handle watch pages.  If we're faulting on a
994 				 * watched page from an X-hat, we have to
995 				 * restore the original permissions while we
996 				 * handle the fault.
997 				 */
998 				as_clearwatch(as);
999 				holding_wpage = 1;
1000 			}
1001 
1002 			res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
1003 
1004 			/* Restore watchpoints */
1005 			if (holding_wpage) {
1006 				as_setwatch(as);
1007 				holding_wpage = 0;
1008 			}
1009 
1010 			if (res != 0)
1011 				break;
1012 		} else {
1013 			/* XHAT does not support seg_dev */
1014 			res = FC_NOSUPPORT;
1015 			break;
1016 		}
1017 	}
1018 
1019 	/*
1020 	 * If we were SOFTLOCKing and encountered a failure,
1021 	 * we must SOFTUNLOCK the range we already did. (Maybe we
1022 	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
1023 	 * right here...)
1024 	 */
1025 	if (res != 0 && type == F_SOFTLOCK) {
1026 		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
1027 			if (addrsav >= seg->s_base + seg->s_size)
1028 				seg = AS_SEGNEXT(as, seg);
1029 			ASSERT(seg != NULL);
1030 			/*
1031 			 * Now call the fault routine again to perform the
1032 			 * unlock using S_OTHER instead of the rw variable
1033 			 * since we never got a chance to touch the pages.
1034 			 */
1035 			if (raddr > seg->s_base + seg->s_size)
1036 				ssize = seg->s_base + seg->s_size - addrsav;
1037 			else
1038 				ssize = raddr - addrsav;
1039 			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
1040 			    F_SOFTUNLOCK, S_OTHER);
1041 		}
1042 	}
1043 	if (as_lock_held)
1044 		AS_LOCK_EXIT(as, &as->a_lock);
1045 	if ((lwp != NULL) && (!is_xhat)) {
1046 		lwp->lwp_nostop--;
1047 		lwp->lwp_nostop_r--;
1048 	}
1049 	/*
1050 	 * If the lower levels returned EDEADLK for a fault,
1051 	 * It means that we should retry the fault.  Let's wait
1052 	 * a bit also to let the deadlock causing condition clear.
1053 	 * This is part of a gross hack to work around a design flaw
1054 	 * in the ufs/sds logging code and should go away when the
1055 	 * logging code is re-designed to fix the problem. See bug
1056 	 * 4125102 for details of the problem.
1057 	 */
1058 	if (FC_ERRNO(res) == EDEADLK) {
1059 		delay(deadlk_wait);
1060 		res = 0;
1061 		goto retry;
1062 	}
1063 	return (res);
1064 }
1065 
1066 
1067 
1068 /*
1069  * Asynchronous ``fault'' at addr for size bytes.
1070  */
1071 faultcode_t
1072 as_faulta(struct as *as, caddr_t addr, size_t size)
1073 {
1074 	struct seg *seg;
1075 	caddr_t raddr;			/* rounded down addr */
1076 	size_t rsize;			/* rounded up size */
1077 	faultcode_t res = 0;
1078 	klwp_t *lwp = ttolwp(curthread);
1079 
1080 retry:
1081 	/*
1082 	 * Indicate that the lwp is not to be stopped while waiting
1083 	 * for a pagefault.  This is to avoid deadlock while debugging
1084 	 * a process via /proc over NFS (in particular).
1085 	 */
1086 	if (lwp != NULL) {
1087 		lwp->lwp_nostop++;
1088 		lwp->lwp_nostop_r++;
1089 	}
1090 
1091 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1092 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1093 		(size_t)raddr;
1094 
1095 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1096 	seg = as_segat(as, raddr);
1097 	if (seg == NULL) {
1098 		AS_LOCK_EXIT(as, &as->a_lock);
1099 		if (lwp != NULL) {
1100 			lwp->lwp_nostop--;
1101 			lwp->lwp_nostop_r--;
1102 		}
1103 		return (FC_NOMAP);
1104 	}
1105 
1106 	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
1107 		if (raddr >= seg->s_base + seg->s_size) {
1108 			seg = AS_SEGNEXT(as, seg);
1109 			if (seg == NULL || raddr != seg->s_base) {
1110 				res = FC_NOMAP;
1111 				break;
1112 			}
1113 		}
1114 		res = SEGOP_FAULTA(seg, raddr);
1115 		if (res != 0)
1116 			break;
1117 	}
1118 	AS_LOCK_EXIT(as, &as->a_lock);
1119 	if (lwp != NULL) {
1120 		lwp->lwp_nostop--;
1121 		lwp->lwp_nostop_r--;
1122 	}
1123 	/*
1124 	 * If the lower levels returned EDEADLK for a fault,
1125 	 * It means that we should retry the fault.  Let's wait
1126 	 * a bit also to let the deadlock causing condition clear.
1127 	 * This is part of a gross hack to work around a design flaw
1128 	 * in the ufs/sds logging code and should go away when the
1129 	 * logging code is re-designed to fix the problem. See bug
1130 	 * 4125102 for details of the problem.
1131 	 */
1132 	if (FC_ERRNO(res) == EDEADLK) {
1133 		delay(deadlk_wait);
1134 		res = 0;
1135 		goto retry;
1136 	}
1137 	return (res);
1138 }
1139 
1140 /*
1141  * Set the virtual mapping for the interval from [addr : addr + size)
1142  * in address space `as' to have the specified protection.
1143  * It is ok for the range to cross over several segments,
1144  * as long as they are contiguous.
1145  */
1146 int
1147 as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1148 {
1149 	struct seg *seg;
1150 	struct as_callback *cb;
1151 	size_t ssize;
1152 	caddr_t raddr;			/* rounded down addr */
1153 	size_t rsize;			/* rounded up size */
1154 	int error = 0, writer = 0;
1155 	caddr_t saveraddr;
1156 	size_t saversize;
1157 
1158 setprot_top:
1159 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1160 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1161 		(size_t)raddr;
1162 
1163 	if (raddr + rsize < raddr)		/* check for wraparound */
1164 		return (ENOMEM);
1165 
1166 	saveraddr = raddr;
1167 	saversize = rsize;
1168 
1169 	/*
1170 	 * Normally we only lock the as as a reader. But
1171 	 * if due to setprot the segment driver needs to split
1172 	 * a segment it will return IE_RETRY. Therefore we re-aquire
1173 	 * the as lock as a writer so the segment driver can change
1174 	 * the seg list. Also the segment driver will return IE_RETRY
1175 	 * after it has changed the segment list so we therefore keep
1176 	 * locking as a writer. Since these opeartions should be rare
1177 	 * want to only lock as a writer when necessary.
1178 	 */
1179 	if (writer || avl_numnodes(&as->a_wpage) != 0) {
1180 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1181 	} else {
1182 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1183 	}
1184 
1185 	as_clearwatchprot(as, raddr, rsize);
1186 	seg = as_segat(as, raddr);
1187 	if (seg == NULL) {
1188 		as_setwatch(as);
1189 		AS_LOCK_EXIT(as, &as->a_lock);
1190 		return (ENOMEM);
1191 	}
1192 
1193 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1194 		if (raddr >= seg->s_base + seg->s_size) {
1195 			seg = AS_SEGNEXT(as, seg);
1196 			if (seg == NULL || raddr != seg->s_base) {
1197 				error = ENOMEM;
1198 				break;
1199 			}
1200 		}
1201 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1202 			ssize = seg->s_base + seg->s_size - raddr;
1203 		else
1204 			ssize = rsize;
1205 		error = SEGOP_SETPROT(seg, raddr, ssize, prot);
1206 
1207 		if (error == IE_NOMEM) {
1208 			error = EAGAIN;
1209 			break;
1210 		}
1211 
1212 		if (error == IE_RETRY) {
1213 			AS_LOCK_EXIT(as, &as->a_lock);
1214 			writer = 1;
1215 			goto setprot_top;
1216 		}
1217 
1218 		if (error == EAGAIN) {
1219 			/*
1220 			 * Make sure we have a_lock as writer.
1221 			 */
1222 			if (writer == 0) {
1223 				AS_LOCK_EXIT(as, &as->a_lock);
1224 				writer = 1;
1225 				goto setprot_top;
1226 			}
1227 
1228 			/*
1229 			 * Memory is currently locked.  It must be unlocked
1230 			 * before this operation can succeed through a retry.
1231 			 * The possible reasons for locked memory and
1232 			 * corresponding strategies for unlocking are:
1233 			 * (1) Normal I/O
1234 			 *	wait for a signal that the I/O operation
1235 			 *	has completed and the memory is unlocked.
1236 			 * (2) Asynchronous I/O
1237 			 *	The aio subsystem does not unlock pages when
1238 			 *	the I/O is completed. Those pages are unlocked
1239 			 *	when the application calls aiowait/aioerror.
1240 			 *	So, to prevent blocking forever, cv_broadcast()
1241 			 *	is done to wake up aio_cleanup_thread.
1242 			 *	Subsequently, segvn_reclaim will be called, and
1243 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1244 			 * (3) Long term page locking:
1245 			 *	Drivers intending to have pages locked for a
1246 			 *	period considerably longer than for normal I/O
1247 			 *	(essentially forever) may have registered for a
1248 			 *	callback so they may unlock these pages on
1249 			 *	request. This is needed to allow this operation
1250 			 *	to succeed. Each entry on the callback list is
1251 			 *	examined. If the event or address range pertains
1252 			 *	the callback is invoked (unless it already is in
1253 			 *	progress). The a_contents lock must be dropped
1254 			 *	before the callback, so only one callback can
1255 			 *	be done at a time. Go to the top and do more
1256 			 *	until zero is returned. If zero is returned,
1257 			 *	either there were no callbacks for this event
1258 			 *	or they were already in progress.
1259 			 */
1260 			mutex_enter(&as->a_contents);
1261 			if (as->a_callbacks &&
1262 				(cb = as_find_callback(as, AS_SETPROT_EVENT,
1263 						seg->s_base, seg->s_size))) {
1264 				AS_LOCK_EXIT(as, &as->a_lock);
1265 				as_execute_callback(as, cb, AS_SETPROT_EVENT);
1266 			} else {
1267 				if (AS_ISUNMAPWAIT(as) == 0)
1268 					cv_broadcast(&as->a_cv);
1269 				AS_SETUNMAPWAIT(as);
1270 				AS_LOCK_EXIT(as, &as->a_lock);
1271 				while (AS_ISUNMAPWAIT(as))
1272 					cv_wait(&as->a_cv, &as->a_contents);
1273 			}
1274 			mutex_exit(&as->a_contents);
1275 			goto setprot_top;
1276 		} else if (error != 0)
1277 			break;
1278 	}
1279 	if (error != 0) {
1280 		as_setwatch(as);
1281 	} else {
1282 		as_setwatchprot(as, saveraddr, saversize, prot);
1283 	}
1284 	AS_LOCK_EXIT(as, &as->a_lock);
1285 	return (error);
1286 }
1287 
1288 /*
1289  * Check to make sure that the interval [addr, addr + size)
1290  * in address space `as' has at least the specified protection.
1291  * It is ok for the range to cross over several segments, as long
1292  * as they are contiguous.
1293  */
1294 int
1295 as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1296 {
1297 	struct seg *seg;
1298 	size_t ssize;
1299 	caddr_t raddr;			/* rounded down addr */
1300 	size_t rsize;			/* rounded up size */
1301 	int error = 0;
1302 
1303 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1304 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1305 		(size_t)raddr;
1306 
1307 	if (raddr + rsize < raddr)		/* check for wraparound */
1308 		return (ENOMEM);
1309 
1310 	/*
1311 	 * This is ugly as sin...
1312 	 * Normally, we only acquire the address space readers lock.
1313 	 * However, if the address space has watchpoints present,
1314 	 * we must acquire the writer lock on the address space for
1315 	 * the benefit of as_clearwatchprot() and as_setwatchprot().
1316 	 */
1317 	if (avl_numnodes(&as->a_wpage) != 0)
1318 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1319 	else
1320 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1321 	as_clearwatchprot(as, raddr, rsize);
1322 	seg = as_segat(as, raddr);
1323 	if (seg == NULL) {
1324 		as_setwatch(as);
1325 		AS_LOCK_EXIT(as, &as->a_lock);
1326 		return (ENOMEM);
1327 	}
1328 
1329 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1330 		if (raddr >= seg->s_base + seg->s_size) {
1331 			seg = AS_SEGNEXT(as, seg);
1332 			if (seg == NULL || raddr != seg->s_base) {
1333 				error = ENOMEM;
1334 				break;
1335 			}
1336 		}
1337 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1338 			ssize = seg->s_base + seg->s_size - raddr;
1339 		else
1340 			ssize = rsize;
1341 
1342 		error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
1343 		if (error != 0)
1344 			break;
1345 	}
1346 	as_setwatch(as);
1347 	AS_LOCK_EXIT(as, &as->a_lock);
1348 	return (error);
1349 }
1350 
1351 int
1352 as_unmap(struct as *as, caddr_t addr, size_t size)
1353 {
1354 	struct seg *seg, *seg_next;
1355 	struct as_callback *cb;
1356 	caddr_t raddr, eaddr;
1357 	size_t ssize;
1358 	int err;
1359 
1360 top:
1361 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1362 	eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) &
1363 	    (uintptr_t)PAGEMASK);
1364 
1365 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1366 
1367 	as->a_updatedir = 1;	/* inform /proc */
1368 	gethrestime(&as->a_updatetime);
1369 
1370 	/*
1371 	 * Use as_findseg to find the first segment in the range, then
1372 	 * step through the segments in order, following s_next.
1373 	 */
1374 	as_clearwatchprot(as, raddr, eaddr - raddr);
1375 
1376 	for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
1377 		if (eaddr <= seg->s_base)
1378 			break;		/* eaddr was in a gap; all done */
1379 
1380 		/* this is implied by the test above */
1381 		ASSERT(raddr < eaddr);
1382 
1383 		if (raddr < seg->s_base)
1384 			raddr = seg->s_base; 	/* raddr was in a gap */
1385 
1386 		if (eaddr > (seg->s_base + seg->s_size))
1387 			ssize = seg->s_base + seg->s_size - raddr;
1388 		else
1389 			ssize = eaddr - raddr;
1390 
1391 		/*
1392 		 * Save next segment pointer since seg can be
1393 		 * destroyed during the segment unmap operation.
1394 		 */
1395 		seg_next = AS_SEGNEXT(as, seg);
1396 
1397 		err = SEGOP_UNMAP(seg, raddr, ssize);
1398 		if (err == EAGAIN) {
1399 			/*
1400 			 * Memory is currently locked.  It must be unlocked
1401 			 * before this operation can succeed through a retry.
1402 			 * The possible reasons for locked memory and
1403 			 * corresponding strategies for unlocking are:
1404 			 * (1) Normal I/O
1405 			 *	wait for a signal that the I/O operation
1406 			 *	has completed and the memory is unlocked.
1407 			 * (2) Asynchronous I/O
1408 			 *	The aio subsystem does not unlock pages when
1409 			 *	the I/O is completed. Those pages are unlocked
1410 			 *	when the application calls aiowait/aioerror.
1411 			 *	So, to prevent blocking forever, cv_broadcast()
1412 			 *	is done to wake up aio_cleanup_thread.
1413 			 *	Subsequently, segvn_reclaim will be called, and
1414 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1415 			 * (3) Long term page locking:
1416 			 *	Drivers intending to have pages locked for a
1417 			 *	period considerably longer than for normal I/O
1418 			 *	(essentially forever) may have registered for a
1419 			 *	callback so they may unlock these pages on
1420 			 *	request. This is needed to allow this operation
1421 			 *	to succeed. Each entry on the callback list is
1422 			 *	examined. If the event or address range pertains
1423 			 *	the callback is invoked (unless it already is in
1424 			 *	progress). The a_contents lock must be dropped
1425 			 *	before the callback, so only one callback can
1426 			 *	be done at a time. Go to the top and do more
1427 			 *	until zero is returned. If zero is returned,
1428 			 *	either there were no callbacks for this event
1429 			 *	or they were already in progress.
1430 			 */
1431 			as_setwatch(as);
1432 			mutex_enter(&as->a_contents);
1433 			if (as->a_callbacks &&
1434 				(cb = as_find_callback(as, AS_UNMAP_EVENT,
1435 						seg->s_base, seg->s_size))) {
1436 				AS_LOCK_EXIT(as, &as->a_lock);
1437 				as_execute_callback(as, cb, AS_UNMAP_EVENT);
1438 			} else {
1439 				if (AS_ISUNMAPWAIT(as) == 0)
1440 					cv_broadcast(&as->a_cv);
1441 				AS_SETUNMAPWAIT(as);
1442 				AS_LOCK_EXIT(as, &as->a_lock);
1443 				while (AS_ISUNMAPWAIT(as))
1444 					cv_wait(&as->a_cv, &as->a_contents);
1445 			}
1446 			mutex_exit(&as->a_contents);
1447 			goto top;
1448 		} else if (err == IE_RETRY) {
1449 			as_setwatch(as);
1450 			AS_LOCK_EXIT(as, &as->a_lock);
1451 			goto top;
1452 		} else if (err) {
1453 			as_setwatch(as);
1454 			AS_LOCK_EXIT(as, &as->a_lock);
1455 			return (-1);
1456 		}
1457 
1458 		as->a_size -= ssize;
1459 		raddr += ssize;
1460 	}
1461 	AS_LOCK_EXIT(as, &as->a_lock);
1462 	return (0);
1463 }
1464 
1465 static int
1466 as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
1467     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1468 {
1469 	uint_t szc;
1470 	uint_t nszc;
1471 	int error;
1472 	caddr_t a;
1473 	caddr_t eaddr;
1474 	size_t segsize;
1475 	struct seg *seg;
1476 	size_t pgsz;
1477 	int do_off = (vn_a->vp != NULL || vn_a->amp != NULL);
1478 	uint_t save_szcvec;
1479 
1480 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1481 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1482 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1483 	ASSERT(vn_a->vp == NULL || vn_a->amp == NULL);
1484 	if (!do_off) {
1485 		vn_a->offset = 0;
1486 	}
1487 
1488 	if (szcvec <= 1) {
1489 		seg = seg_alloc(as, addr, size);
1490 		if (seg == NULL) {
1491 			return (ENOMEM);
1492 		}
1493 		vn_a->szc = 0;
1494 		error = (*crfp)(seg, vn_a);
1495 		if (error != 0) {
1496 			seg_free(seg);
1497 		}
1498 		return (error);
1499 	}
1500 
1501 	eaddr = addr + size;
1502 	save_szcvec = szcvec;
1503 	szcvec >>= 1;
1504 	szc = 0;
1505 	nszc = 0;
1506 	while (szcvec) {
1507 		if ((szcvec & 0x1) == 0) {
1508 			nszc++;
1509 			szcvec >>= 1;
1510 			continue;
1511 		}
1512 		nszc++;
1513 		pgsz = page_get_pagesize(nszc);
1514 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
1515 		if (a != addr) {
1516 			ASSERT(a < eaddr);
1517 			segsize = a - addr;
1518 			seg = seg_alloc(as, addr, segsize);
1519 			if (seg == NULL) {
1520 				return (ENOMEM);
1521 			}
1522 			vn_a->szc = szc;
1523 			error = (*crfp)(seg, vn_a);
1524 			if (error != 0) {
1525 				seg_free(seg);
1526 				return (error);
1527 			}
1528 			*segcreated = 1;
1529 			if (do_off) {
1530 				vn_a->offset += segsize;
1531 			}
1532 			addr = a;
1533 		}
1534 		szc = nszc;
1535 		szcvec >>= 1;
1536 	}
1537 
1538 	ASSERT(addr < eaddr);
1539 	szcvec = save_szcvec | 1; /* add 8K pages */
1540 	while (szcvec) {
1541 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
1542 		ASSERT(a >= addr);
1543 		if (a != addr) {
1544 			segsize = a - addr;
1545 			seg = seg_alloc(as, addr, segsize);
1546 			if (seg == NULL) {
1547 				return (ENOMEM);
1548 			}
1549 			vn_a->szc = szc;
1550 			error = (*crfp)(seg, vn_a);
1551 			if (error != 0) {
1552 				seg_free(seg);
1553 				return (error);
1554 			}
1555 			*segcreated = 1;
1556 			if (do_off) {
1557 				vn_a->offset += segsize;
1558 			}
1559 			addr = a;
1560 		}
1561 		szcvec &= ~(1 << szc);
1562 		if (szcvec) {
1563 			szc = highbit(szcvec) - 1;
1564 			pgsz = page_get_pagesize(szc);
1565 		}
1566 	}
1567 	ASSERT(addr == eaddr);
1568 
1569 	return (0);
1570 }
1571 
1572 static int
1573 as_map_vnsegs(struct as *as, caddr_t addr, size_t size,
1574     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1575 {
1576 	uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA);
1577 	int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
1578 	uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1579 	    type, 0);
1580 	int error;
1581 	struct seg *seg;
1582 	struct vattr va;
1583 	u_offset_t eoff;
1584 	size_t save_size = 0;
1585 
1586 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1587 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1588 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1589 	ASSERT(vn_a->vp != NULL);
1590 	ASSERT(vn_a->amp == NULL);
1591 
1592 again:
1593 	if (szcvec <= 1) {
1594 		seg = seg_alloc(as, addr, size);
1595 		if (seg == NULL) {
1596 			return (ENOMEM);
1597 		}
1598 		vn_a->szc = 0;
1599 		error = (*crfp)(seg, vn_a);
1600 		if (error != 0) {
1601 			seg_free(seg);
1602 		}
1603 		return (error);
1604 	}
1605 
1606 	va.va_mask = AT_SIZE;
1607 	if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred) != 0) {
1608 		szcvec = 0;
1609 		goto again;
1610 	}
1611 	eoff = vn_a->offset & PAGEMASK;
1612 	if (eoff >= va.va_size) {
1613 		szcvec = 0;
1614 		goto again;
1615 	}
1616 	eoff += size;
1617 	if (btopr(va.va_size) < btopr(eoff)) {
1618 		save_size = size;
1619 		size = va.va_size - (vn_a->offset & PAGEMASK);
1620 		size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t);
1621 		szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1622 		    type, 0);
1623 		if (szcvec <= 1) {
1624 			size = save_size;
1625 			goto again;
1626 		}
1627 	}
1628 
1629 	error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a,
1630 	    segcreated);
1631 	if (error != 0) {
1632 		return (error);
1633 	}
1634 	if (save_size) {
1635 		addr += size;
1636 		size = save_size - size;
1637 		szcvec = 0;
1638 		goto again;
1639 	}
1640 	return (0);
1641 }
1642 
1643 /*
1644  * as_map_ansegs: shared or private anonymous memory.  Note that the flags
1645  * passed to map_pgszvec cannot be MAP_INITDATA, for anon.
1646  */
1647 static int
1648 as_map_ansegs(struct as *as, caddr_t addr, size_t size,
1649     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1650 {
1651 	uint_t szcvec;
1652 	uchar_t type;
1653 
1654 	ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE);
1655 	if (vn_a->type == MAP_SHARED) {
1656 		type = MAPPGSZC_SHM;
1657 	} else if (vn_a->type == MAP_PRIVATE) {
1658 		if (vn_a->szc == AS_MAP_HEAP) {
1659 			type = MAPPGSZC_HEAP;
1660 		} else if (vn_a->szc == AS_MAP_STACK) {
1661 			type = MAPPGSZC_STACK;
1662 		} else {
1663 			type = MAPPGSZC_PRIVM;
1664 		}
1665 	}
1666 	szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ?
1667 	    (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE),
1668 	    (vn_a->flags & MAP_TEXT), type, 0);
1669 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1670 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1671 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1672 	ASSERT(vn_a->vp == NULL);
1673 
1674 	return (as_map_segvn_segs(as, addr, size, szcvec,
1675 	    crfp, vn_a, segcreated));
1676 }
1677 
1678 int
1679 as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp)
1680 {
1681 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1682 	return (as_map_locked(as, addr, size, crfp, argsp));
1683 }
1684 
1685 int
1686 as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(),
1687 		void *argsp)
1688 {
1689 	struct seg *seg = NULL;
1690 	caddr_t raddr;			/* rounded down addr */
1691 	size_t rsize;			/* rounded up size */
1692 	int error;
1693 	int unmap = 0;
1694 	struct proc *p = curproc;
1695 	struct segvn_crargs crargs;
1696 
1697 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1698 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1699 		(size_t)raddr;
1700 
1701 	/*
1702 	 * check for wrap around
1703 	 */
1704 	if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
1705 		AS_LOCK_EXIT(as, &as->a_lock);
1706 		return (ENOMEM);
1707 	}
1708 
1709 	as->a_updatedir = 1;	/* inform /proc */
1710 	gethrestime(&as->a_updatetime);
1711 
1712 	if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) {
1713 		AS_LOCK_EXIT(as, &as->a_lock);
1714 
1715 		(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p,
1716 		    RCA_UNSAFE_ALL);
1717 
1718 		return (ENOMEM);
1719 	}
1720 
1721 	if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) {
1722 		crargs = *(struct segvn_crargs *)argsp;
1723 		error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap);
1724 		if (error != 0) {
1725 			AS_LOCK_EXIT(as, &as->a_lock);
1726 			if (unmap) {
1727 				(void) as_unmap(as, addr, size);
1728 			}
1729 			return (error);
1730 		}
1731 	} else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) {
1732 		crargs = *(struct segvn_crargs *)argsp;
1733 		error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap);
1734 		if (error != 0) {
1735 			AS_LOCK_EXIT(as, &as->a_lock);
1736 			if (unmap) {
1737 				(void) as_unmap(as, addr, size);
1738 			}
1739 			return (error);
1740 		}
1741 	} else {
1742 		seg = seg_alloc(as, addr, size);
1743 		if (seg == NULL) {
1744 			AS_LOCK_EXIT(as, &as->a_lock);
1745 			return (ENOMEM);
1746 		}
1747 
1748 		error = (*crfp)(seg, argsp);
1749 		if (error != 0) {
1750 			seg_free(seg);
1751 			AS_LOCK_EXIT(as, &as->a_lock);
1752 			return (error);
1753 		}
1754 	}
1755 
1756 	/*
1757 	 * Add size now so as_unmap will work if as_ctl fails.
1758 	 */
1759 	as->a_size += rsize;
1760 
1761 	as_setwatch(as);
1762 
1763 	/*
1764 	 * If the address space is locked,
1765 	 * establish memory locks for the new segment.
1766 	 */
1767 	mutex_enter(&as->a_contents);
1768 	if (AS_ISPGLCK(as)) {
1769 		mutex_exit(&as->a_contents);
1770 		AS_LOCK_EXIT(as, &as->a_lock);
1771 		error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
1772 		if (error != 0)
1773 			(void) as_unmap(as, addr, size);
1774 	} else {
1775 		mutex_exit(&as->a_contents);
1776 		AS_LOCK_EXIT(as, &as->a_lock);
1777 	}
1778 	return (error);
1779 }
1780 
1781 
1782 /*
1783  * Delete all segments in the address space marked with S_PURGE.
1784  * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c).
1785  * These segments are deleted as a first step before calls to as_gap(), so
1786  * that they don't affect mmap() or shmat().
1787  */
1788 void
1789 as_purge(struct as *as)
1790 {
1791 	struct seg *seg;
1792 	struct seg *next_seg;
1793 
1794 	/*
1795 	 * the setting of NEEDSPURGE is protect by as_rangelock(), so
1796 	 * no need to grab a_contents mutex for this check
1797 	 */
1798 	if ((as->a_flags & AS_NEEDSPURGE) == 0)
1799 		return;
1800 
1801 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1802 	next_seg = NULL;
1803 	seg = AS_SEGFIRST(as);
1804 	while (seg != NULL) {
1805 		next_seg = AS_SEGNEXT(as, seg);
1806 		if (seg->s_flags & S_PURGE)
1807 			SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
1808 		seg = next_seg;
1809 	}
1810 	AS_LOCK_EXIT(as, &as->a_lock);
1811 
1812 	mutex_enter(&as->a_contents);
1813 	as->a_flags &= ~AS_NEEDSPURGE;
1814 	mutex_exit(&as->a_contents);
1815 }
1816 
1817 /*
1818  * Find a hole of at least size minlen within [base, base + len).
1819  *
1820  * If flags specifies AH_HI, the hole will have the highest possible address
1821  * in the range.  We use the as->a_lastgap field to figure out where to
1822  * start looking for a gap.
1823  *
1824  * Otherwise, the gap will have the lowest possible address.
1825  *
1826  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
1827  *
1828  * If an adequate hole is found, base and len are set to reflect the part of
1829  * the hole that is within range, and 0 is returned, otherwise,
1830  * -1 is returned.
1831  *
1832  * NOTE: This routine is not correct when base+len overflows caddr_t.
1833  */
1834 int
1835 as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags,
1836     caddr_t addr)
1837 {
1838 	caddr_t lobound = *basep;
1839 	caddr_t hibound = lobound + *lenp;
1840 	struct seg *lseg, *hseg;
1841 	caddr_t lo, hi;
1842 	int forward;
1843 	caddr_t save_base;
1844 	size_t save_len;
1845 
1846 	save_base = *basep;
1847 	save_len = *lenp;
1848 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1849 	if (AS_SEGFIRST(as) == NULL) {
1850 		if (valid_va_range(basep, lenp, minlen, flags & AH_DIR)) {
1851 			AS_LOCK_EXIT(as, &as->a_lock);
1852 			return (0);
1853 		} else {
1854 			AS_LOCK_EXIT(as, &as->a_lock);
1855 			*basep = save_base;
1856 			*lenp = save_len;
1857 			return (-1);
1858 		}
1859 	}
1860 
1861 	/*
1862 	 * Set up to iterate over all the inter-segment holes in the given
1863 	 * direction.  lseg is NULL for the lowest-addressed hole and hseg is
1864 	 * NULL for the highest-addressed hole.  If moving backwards, we reset
1865 	 * sseg to denote the highest-addressed segment.
1866 	 */
1867 	forward = (flags & AH_DIR) == AH_LO;
1868 	if (forward) {
1869 		hseg = as_findseg(as, lobound, 1);
1870 		lseg = AS_SEGPREV(as, hseg);
1871 	} else {
1872 
1873 		/*
1874 		 * If allocating at least as much as the last allocation,
1875 		 * use a_lastgap's base as a better estimate of hibound.
1876 		 */
1877 		if (as->a_lastgap &&
1878 		    minlen >= as->a_lastgap->s_size &&
1879 		    hibound >= as->a_lastgap->s_base)
1880 			hibound = as->a_lastgap->s_base;
1881 
1882 		hseg = as_findseg(as, hibound, 1);
1883 		if (hseg->s_base + hseg->s_size < hibound) {
1884 			lseg = hseg;
1885 			hseg = NULL;
1886 		} else {
1887 			lseg = AS_SEGPREV(as, hseg);
1888 		}
1889 	}
1890 
1891 	for (;;) {
1892 		/*
1893 		 * Set lo and hi to the hole's boundaries.  (We should really
1894 		 * use MAXADDR in place of hibound in the expression below,
1895 		 * but can't express it easily; using hibound in its place is
1896 		 * harmless.)
1897 		 */
1898 		lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
1899 		hi = (hseg == NULL) ? hibound : hseg->s_base;
1900 		/*
1901 		 * If the iteration has moved past the interval from lobound
1902 		 * to hibound it's pointless to continue.
1903 		 */
1904 		if ((forward && lo > hibound) || (!forward && hi < lobound))
1905 			break;
1906 		else if (lo > hibound || hi < lobound)
1907 			goto cont;
1908 		/*
1909 		 * Candidate hole lies at least partially within the allowable
1910 		 * range.  Restrict it to fall completely within that range,
1911 		 * i.e., to [max(lo, lobound), min(hi, hibound)].
1912 		 */
1913 		if (lo < lobound)
1914 			lo = lobound;
1915 		if (hi > hibound)
1916 			hi = hibound;
1917 		/*
1918 		 * Verify that the candidate hole is big enough and meets
1919 		 * hardware constraints.
1920 		 */
1921 		*basep = lo;
1922 		*lenp = hi - lo;
1923 		if (valid_va_range(basep, lenp, minlen,
1924 		    forward ? AH_LO : AH_HI) &&
1925 		    ((flags & AH_CONTAIN) == 0 ||
1926 		    (*basep <= addr && *basep + *lenp > addr))) {
1927 			if (!forward)
1928 				as->a_lastgap = hseg;
1929 			if (hseg != NULL)
1930 				as->a_lastgaphl = hseg;
1931 			else
1932 				as->a_lastgaphl = lseg;
1933 			AS_LOCK_EXIT(as, &as->a_lock);
1934 			return (0);
1935 		}
1936 	cont:
1937 		/*
1938 		 * Move to the next hole.
1939 		 */
1940 		if (forward) {
1941 			lseg = hseg;
1942 			if (lseg == NULL)
1943 				break;
1944 			hseg = AS_SEGNEXT(as, hseg);
1945 		} else {
1946 			hseg = lseg;
1947 			if (hseg == NULL)
1948 				break;
1949 			lseg = AS_SEGPREV(as, lseg);
1950 		}
1951 	}
1952 	*basep = save_base;
1953 	*lenp = save_len;
1954 	AS_LOCK_EXIT(as, &as->a_lock);
1955 	return (-1);
1956 }
1957 
1958 /*
1959  * Return the next range within [base, base + len) that is backed
1960  * with "real memory".  Skip holes and non-seg_vn segments.
1961  * We're lazy and only return one segment at a time.
1962  */
1963 int
1964 as_memory(struct as *as, caddr_t *basep, size_t *lenp)
1965 {
1966 	extern struct seg_ops segspt_shmops;	/* needs a header file */
1967 	struct seg *seg;
1968 	caddr_t addr, eaddr;
1969 	caddr_t segend;
1970 
1971 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1972 
1973 	addr = *basep;
1974 	eaddr = addr + *lenp;
1975 
1976 	seg = as_findseg(as, addr, 0);
1977 	if (seg != NULL)
1978 		addr = MAX(seg->s_base, addr);
1979 
1980 	for (;;) {
1981 		if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
1982 			AS_LOCK_EXIT(as, &as->a_lock);
1983 			return (EINVAL);
1984 		}
1985 
1986 		if (seg->s_ops == &segvn_ops) {
1987 			segend = seg->s_base + seg->s_size;
1988 			break;
1989 		}
1990 
1991 		/*
1992 		 * We do ISM by looking into the private data
1993 		 * to determine the real size of the segment.
1994 		 */
1995 		if (seg->s_ops == &segspt_shmops) {
1996 			segend = seg->s_base + spt_realsize(seg);
1997 			if (addr < segend)
1998 				break;
1999 		}
2000 
2001 		seg = AS_SEGNEXT(as, seg);
2002 
2003 		if (seg != NULL)
2004 			addr = seg->s_base;
2005 	}
2006 
2007 	*basep = addr;
2008 
2009 	if (segend > eaddr)
2010 		*lenp = eaddr - addr;
2011 	else
2012 		*lenp = segend - addr;
2013 
2014 	AS_LOCK_EXIT(as, &as->a_lock);
2015 	return (0);
2016 }
2017 
2018 /*
2019  * Swap the pages associated with the address space as out to
2020  * secondary storage, returning the number of bytes actually
2021  * swapped.
2022  *
2023  * The value returned is intended to correlate well with the process's
2024  * memory requirements.  Its usefulness for this purpose depends on
2025  * how well the segment-level routines do at returning accurate
2026  * information.
2027  */
2028 size_t
2029 as_swapout(struct as *as)
2030 {
2031 	struct seg *seg;
2032 	size_t swpcnt = 0;
2033 
2034 	/*
2035 	 * Kernel-only processes have given up their address
2036 	 * spaces.  Of course, we shouldn't be attempting to
2037 	 * swap out such processes in the first place...
2038 	 */
2039 	if (as == NULL)
2040 		return (0);
2041 
2042 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2043 
2044 	/* Prevent XHATs from attaching */
2045 	mutex_enter(&as->a_contents);
2046 	AS_SETBUSY(as);
2047 	mutex_exit(&as->a_contents);
2048 
2049 
2050 	/*
2051 	 * Free all mapping resources associated with the address
2052 	 * space.  The segment-level swapout routines capitalize
2053 	 * on this unmapping by scavanging pages that have become
2054 	 * unmapped here.
2055 	 */
2056 	hat_swapout(as->a_hat);
2057 	if (as->a_xhat != NULL)
2058 		xhat_swapout_all(as);
2059 
2060 	mutex_enter(&as->a_contents);
2061 	AS_CLRBUSY(as);
2062 	mutex_exit(&as->a_contents);
2063 
2064 	/*
2065 	 * Call the swapout routines of all segments in the address
2066 	 * space to do the actual work, accumulating the amount of
2067 	 * space reclaimed.
2068 	 */
2069 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
2070 		struct seg_ops *ov = seg->s_ops;
2071 
2072 		/*
2073 		 * We have to check to see if the seg has
2074 		 * an ops vector because the seg may have
2075 		 * been in the middle of being set up when
2076 		 * the process was picked for swapout.
2077 		 */
2078 		if ((ov != NULL) && (ov->swapout != NULL))
2079 			swpcnt += SEGOP_SWAPOUT(seg);
2080 	}
2081 	AS_LOCK_EXIT(as, &as->a_lock);
2082 	return (swpcnt);
2083 }
2084 
2085 /*
2086  * Determine whether data from the mappings in interval [addr, addr + size)
2087  * are in the primary memory (core) cache.
2088  */
2089 int
2090 as_incore(struct as *as, caddr_t addr,
2091     size_t size, char *vec, size_t *sizep)
2092 {
2093 	struct seg *seg;
2094 	size_t ssize;
2095 	caddr_t raddr;		/* rounded down addr */
2096 	size_t rsize;		/* rounded up size */
2097 	size_t isize;			/* iteration size */
2098 	int error = 0;		/* result, assume success */
2099 
2100 	*sizep = 0;
2101 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2102 	rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
2103 		(size_t)raddr;
2104 
2105 	if (raddr + rsize < raddr)		/* check for wraparound */
2106 		return (ENOMEM);
2107 
2108 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2109 	seg = as_segat(as, raddr);
2110 	if (seg == NULL) {
2111 		AS_LOCK_EXIT(as, &as->a_lock);
2112 		return (-1);
2113 	}
2114 
2115 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2116 		if (raddr >= seg->s_base + seg->s_size) {
2117 			seg = AS_SEGNEXT(as, seg);
2118 			if (seg == NULL || raddr != seg->s_base) {
2119 				error = -1;
2120 				break;
2121 			}
2122 		}
2123 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2124 			ssize = seg->s_base + seg->s_size - raddr;
2125 		else
2126 			ssize = rsize;
2127 		*sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
2128 		if (isize != ssize) {
2129 			error = -1;
2130 			break;
2131 		}
2132 		vec += btopr(ssize);
2133 	}
2134 	AS_LOCK_EXIT(as, &as->a_lock);
2135 	return (error);
2136 }
2137 
2138 static void
2139 as_segunlock(struct seg *seg, caddr_t addr, int attr,
2140 	ulong_t *bitmap, size_t position, size_t npages)
2141 {
2142 	caddr_t	range_start;
2143 	size_t	pos1 = position;
2144 	size_t	pos2;
2145 	size_t	size;
2146 	size_t  end_pos = npages + position;
2147 
2148 	while (bt_range(bitmap, &pos1, &pos2, end_pos)) {
2149 		size = ptob((pos2 - pos1));
2150 		range_start = (caddr_t)((uintptr_t)addr +
2151 			ptob(pos1 - position));
2152 
2153 		(void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK,
2154 			(ulong_t *)NULL, (size_t)NULL);
2155 		pos1 = pos2;
2156 	}
2157 }
2158 
2159 static void
2160 as_unlockerr(struct as *as, int attr, ulong_t *mlock_map,
2161 	caddr_t raddr, size_t rsize)
2162 {
2163 	struct seg *seg = as_segat(as, raddr);
2164 	size_t ssize;
2165 
2166 	while (rsize != 0) {
2167 		if (raddr >= seg->s_base + seg->s_size)
2168 			seg = AS_SEGNEXT(as, seg);
2169 
2170 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2171 			ssize = seg->s_base + seg->s_size - raddr;
2172 		else
2173 			ssize = rsize;
2174 
2175 		as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize));
2176 
2177 		rsize -= ssize;
2178 		raddr += ssize;
2179 	}
2180 }
2181 
2182 /*
2183  * Cache control operations over the interval [addr, addr + size) in
2184  * address space "as".
2185  */
2186 /*ARGSUSED*/
2187 int
2188 as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
2189     uintptr_t arg, ulong_t *lock_map, size_t pos)
2190 {
2191 	struct seg *seg;	/* working segment */
2192 	caddr_t raddr;		/* rounded down addr */
2193 	caddr_t initraddr;	/* saved initial rounded down addr */
2194 	size_t rsize;		/* rounded up size */
2195 	size_t initrsize;	/* saved initial rounded up size */
2196 	size_t ssize;		/* size of seg */
2197 	int error = 0;			/* result */
2198 	size_t mlock_size;	/* size of bitmap */
2199 	ulong_t *mlock_map;	/* pointer to bitmap used */
2200 				/* to represent the locked */
2201 				/* pages. */
2202 retry:
2203 	if (error == IE_RETRY)
2204 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2205 	else
2206 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2207 
2208 	/*
2209 	 * If these are address space lock/unlock operations, loop over
2210 	 * all segments in the address space, as appropriate.
2211 	 */
2212 	if (func == MC_LOCKAS) {
2213 		size_t npages, idx;
2214 		size_t rlen = 0;	/* rounded as length */
2215 
2216 		idx = pos;
2217 
2218 		if (arg & MCL_FUTURE) {
2219 			mutex_enter(&as->a_contents);
2220 			AS_SETPGLCK(as);
2221 			mutex_exit(&as->a_contents);
2222 		}
2223 		if ((arg & MCL_CURRENT) == 0) {
2224 			AS_LOCK_EXIT(as, &as->a_lock);
2225 			return (0);
2226 		}
2227 
2228 		seg = AS_SEGFIRST(as);
2229 		if (seg == NULL) {
2230 			AS_LOCK_EXIT(as, &as->a_lock);
2231 			return (0);
2232 		}
2233 
2234 		do {
2235 			raddr = (caddr_t)((uintptr_t)seg->s_base &
2236 			    (uintptr_t)PAGEMASK);
2237 			rlen += (((uintptr_t)(seg->s_base + seg->s_size) +
2238 				PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr;
2239 		} while ((seg = AS_SEGNEXT(as, seg)) != NULL);
2240 
2241 		mlock_size = BT_BITOUL(btopr(rlen));
2242 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2243 			sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2244 				AS_LOCK_EXIT(as, &as->a_lock);
2245 				return (EAGAIN);
2246 		}
2247 
2248 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2249 			error = SEGOP_LOCKOP(seg, seg->s_base,
2250 			    seg->s_size, attr, MC_LOCK, mlock_map, pos);
2251 			if (error != 0)
2252 				break;
2253 			pos += seg_pages(seg);
2254 		}
2255 
2256 		if (error) {
2257 			for (seg = AS_SEGFIRST(as); seg != NULL;
2258 				seg = AS_SEGNEXT(as, seg)) {
2259 
2260 				raddr = (caddr_t)((uintptr_t)seg->s_base &
2261 					(uintptr_t)PAGEMASK);
2262 				npages = seg_pages(seg);
2263 				as_segunlock(seg, raddr, attr, mlock_map,
2264 					idx, npages);
2265 				idx += npages;
2266 			}
2267 		}
2268 
2269 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2270 		AS_LOCK_EXIT(as, &as->a_lock);
2271 		goto lockerr;
2272 	} else if (func == MC_UNLOCKAS) {
2273 		mutex_enter(&as->a_contents);
2274 		AS_CLRPGLCK(as);
2275 		mutex_exit(&as->a_contents);
2276 
2277 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2278 			error = SEGOP_LOCKOP(seg, seg->s_base,
2279 			    seg->s_size, attr, MC_UNLOCK, NULL, 0);
2280 			if (error != 0)
2281 				break;
2282 		}
2283 
2284 		AS_LOCK_EXIT(as, &as->a_lock);
2285 		goto lockerr;
2286 	}
2287 
2288 	/*
2289 	 * Normalize addresses and sizes.
2290 	 */
2291 	initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2292 	initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2293 		(size_t)raddr;
2294 
2295 	if (raddr + rsize < raddr) {		/* check for wraparound */
2296 		AS_LOCK_EXIT(as, &as->a_lock);
2297 		return (ENOMEM);
2298 	}
2299 
2300 	/*
2301 	 * Get initial segment.
2302 	 */
2303 	if ((seg = as_segat(as, raddr)) == NULL) {
2304 		AS_LOCK_EXIT(as, &as->a_lock);
2305 		return (ENOMEM);
2306 	}
2307 
2308 	if (func == MC_LOCK) {
2309 		mlock_size = BT_BITOUL(btopr(rsize));
2310 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2311 			sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2312 				AS_LOCK_EXIT(as, &as->a_lock);
2313 				return (EAGAIN);
2314 		}
2315 	}
2316 
2317 	/*
2318 	 * Loop over all segments.  If a hole in the address range is
2319 	 * discovered, then fail.  For each segment, perform the appropriate
2320 	 * control operation.
2321 	 */
2322 	while (rsize != 0) {
2323 
2324 		/*
2325 		 * Make sure there's no hole, calculate the portion
2326 		 * of the next segment to be operated over.
2327 		 */
2328 		if (raddr >= seg->s_base + seg->s_size) {
2329 			seg = AS_SEGNEXT(as, seg);
2330 			if (seg == NULL || raddr != seg->s_base) {
2331 				if (func == MC_LOCK) {
2332 					as_unlockerr(as, attr, mlock_map,
2333 						initraddr, initrsize - rsize);
2334 					kmem_free(mlock_map,
2335 						mlock_size * sizeof (ulong_t));
2336 				}
2337 				AS_LOCK_EXIT(as, &as->a_lock);
2338 				return (ENOMEM);
2339 			}
2340 		}
2341 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2342 			ssize = seg->s_base + seg->s_size - raddr;
2343 		else
2344 			ssize = rsize;
2345 
2346 		/*
2347 		 * Dispatch on specific function.
2348 		 */
2349 		switch (func) {
2350 
2351 		/*
2352 		 * Synchronize cached data from mappings with backing
2353 		 * objects.
2354 		 */
2355 		case MC_SYNC:
2356 			if (error = SEGOP_SYNC(seg, raddr, ssize,
2357 			    attr, (uint_t)arg)) {
2358 				AS_LOCK_EXIT(as, &as->a_lock);
2359 				return (error);
2360 			}
2361 			break;
2362 
2363 		/*
2364 		 * Lock pages in memory.
2365 		 */
2366 		case MC_LOCK:
2367 			if (error = SEGOP_LOCKOP(seg, raddr, ssize,
2368 				attr, func, mlock_map, pos)) {
2369 				as_unlockerr(as, attr, mlock_map, initraddr,
2370 					initrsize - rsize + ssize);
2371 				kmem_free(mlock_map, mlock_size *
2372 					sizeof (ulong_t));
2373 				AS_LOCK_EXIT(as, &as->a_lock);
2374 				goto lockerr;
2375 			}
2376 			break;
2377 
2378 		/*
2379 		 * Unlock mapped pages.
2380 		 */
2381 		case MC_UNLOCK:
2382 			(void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
2383 				(ulong_t *)NULL, (size_t)NULL);
2384 			break;
2385 
2386 		/*
2387 		 * Store VM advise for mapped pages in segment layer.
2388 		 */
2389 		case MC_ADVISE:
2390 			error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg);
2391 
2392 			/*
2393 			 * Check for regular errors and special retry error
2394 			 */
2395 			if (error) {
2396 				if (error == IE_RETRY) {
2397 					/*
2398 					 * Need to acquire writers lock, so
2399 					 * have to drop readers lock and start
2400 					 * all over again
2401 					 */
2402 					AS_LOCK_EXIT(as, &as->a_lock);
2403 					goto retry;
2404 				} else if (error == IE_REATTACH) {
2405 					/*
2406 					 * Find segment for current address
2407 					 * because current segment just got
2408 					 * split or concatenated
2409 					 */
2410 					seg = as_segat(as, raddr);
2411 					if (seg == NULL) {
2412 						AS_LOCK_EXIT(as, &as->a_lock);
2413 						return (ENOMEM);
2414 					}
2415 				} else {
2416 					/*
2417 					 * Regular error
2418 					 */
2419 					AS_LOCK_EXIT(as, &as->a_lock);
2420 					return (error);
2421 				}
2422 			}
2423 			break;
2424 
2425 		/*
2426 		 * Can't happen.
2427 		 */
2428 		default:
2429 			panic("as_ctl: bad operation %d", func);
2430 			/*NOTREACHED*/
2431 		}
2432 
2433 		rsize -= ssize;
2434 		raddr += ssize;
2435 	}
2436 
2437 	if (func == MC_LOCK)
2438 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2439 	AS_LOCK_EXIT(as, &as->a_lock);
2440 	return (0);
2441 lockerr:
2442 
2443 	/*
2444 	 * If the lower levels returned EDEADLK for a segment lockop,
2445 	 * it means that we should retry the operation.  Let's wait
2446 	 * a bit also to let the deadlock causing condition clear.
2447 	 * This is part of a gross hack to work around a design flaw
2448 	 * in the ufs/sds logging code and should go away when the
2449 	 * logging code is re-designed to fix the problem. See bug
2450 	 * 4125102 for details of the problem.
2451 	 */
2452 	if (error == EDEADLK) {
2453 		delay(deadlk_wait);
2454 		error = 0;
2455 		goto retry;
2456 	}
2457 	return (error);
2458 }
2459 
2460 /*
2461  * Special code for exec to move the stack segment from its interim
2462  * place in the old address to the right place in the new address space.
2463  */
2464 /*ARGSUSED*/
2465 int
2466 as_exec(struct as *oas, caddr_t ostka, size_t stksz,
2467     struct as *nas, caddr_t nstka, uint_t hatflag)
2468 {
2469 	struct seg *stkseg;
2470 
2471 	AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER);
2472 	stkseg = as_segat(oas, ostka);
2473 	stkseg = as_removeseg(oas, stkseg);
2474 	ASSERT(stkseg != NULL);
2475 	ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz);
2476 	stkseg->s_as = nas;
2477 	stkseg->s_base = nstka;
2478 
2479 	/*
2480 	 * It's ok to lock the address space we are about to exec to.
2481 	 */
2482 	AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER);
2483 	ASSERT(avl_numnodes(&nas->a_wpage) == 0);
2484 	nas->a_size += stkseg->s_size;
2485 	oas->a_size -= stkseg->s_size;
2486 	(void) as_addseg(nas, stkseg);
2487 	AS_LOCK_EXIT(nas, &nas->a_lock);
2488 	AS_LOCK_EXIT(oas, &oas->a_lock);
2489 	return (0);
2490 }
2491 
2492 static int
2493 f_decode(faultcode_t fault_err)
2494 {
2495 	int error = 0;
2496 
2497 	switch (FC_CODE(fault_err)) {
2498 	case FC_OBJERR:
2499 		error = FC_ERRNO(fault_err);
2500 		break;
2501 	case FC_PROT:
2502 		error = EACCES;
2503 		break;
2504 	default:
2505 		error = EFAULT;
2506 		break;
2507 	}
2508 	return (error);
2509 }
2510 
2511 /*
2512  * lock pages in a given address space. Return shadow list. If
2513  * the list is NULL, the MMU mapping is also locked.
2514  */
2515 int
2516 as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
2517     size_t size, enum seg_rw rw)
2518 {
2519 	size_t rsize;
2520 	caddr_t base;
2521 	caddr_t raddr;
2522 	faultcode_t fault_err;
2523 	struct seg *seg;
2524 	int res;
2525 	int prefaulted = 0;
2526 
2527 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
2528 	    "as_pagelock_start: addr %p size %ld", addr, size);
2529 
2530 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2531 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2532 		(size_t)raddr;
2533 top:
2534 	/*
2535 	 * if the request crosses two segments let
2536 	 * as_fault handle it.
2537 	 */
2538 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2539 	seg = as_findseg(as, addr, 0);
2540 	if ((seg == NULL) || ((base = seg->s_base) > addr) ||
2541 	    (addr + size) > base + seg->s_size) {
2542 		AS_LOCK_EXIT(as, &as->a_lock);
2543 		goto slow;
2544 	}
2545 
2546 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
2547 	    "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
2548 
2549 	/*
2550 	 * try to lock pages and pass back shadow list
2551 	 */
2552 	res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
2553 
2554 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
2555 	AS_LOCK_EXIT(as, &as->a_lock);
2556 	if (res == 0) {
2557 		return (0);
2558 	} else if (res == ENOTSUP || prefaulted) {
2559 		/*
2560 		 * (1) segment driver doesn't support PAGELOCK fastpath, or
2561 		 * (2) we've already tried fast path unsuccessfully after
2562 		 *    faulting in the addr range below; system might be
2563 		 *    thrashing or there may not be enough availrmem.
2564 		 */
2565 		goto slow;
2566 	}
2567 
2568 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_FAULT_START,
2569 	    "as_fault_start: addr %p size %ld", addr, size);
2570 
2571 	/*
2572 	 * we might get here because of some COW fault or non
2573 	 * existing page. Let as_fault deal with it. Just load
2574 	 * the page, don't lock the MMU mapping.
2575 	 */
2576 	fault_err = as_fault(as->a_hat, as, addr, size, F_INVAL, rw);
2577 	if (fault_err != 0) {
2578 		return (f_decode(fault_err));
2579 	}
2580 
2581 	prefaulted = 1;
2582 
2583 	/*
2584 	 * try fast path again; since we've dropped a_lock,
2585 	 * we need to try the dance from the start to see if
2586 	 * the addr range is still valid.
2587 	 */
2588 	goto top;
2589 slow:
2590 	/*
2591 	 * load the page and lock the MMU mapping.
2592 	 */
2593 	fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
2594 	if (fault_err != 0) {
2595 		return (f_decode(fault_err));
2596 	}
2597 	*ppp = NULL;
2598 
2599 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
2600 	return (0);
2601 }
2602 
2603 /*
2604  * unlock pages in a given address range
2605  */
2606 void
2607 as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
2608     enum seg_rw rw)
2609 {
2610 	struct seg *seg;
2611 	size_t rsize;
2612 	caddr_t raddr;
2613 
2614 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
2615 	    "as_pageunlock_start: addr %p size %ld", addr, size);
2616 
2617 	/*
2618 	 * if the shadow list is NULL, as_pagelock was
2619 	 * falling back to as_fault
2620 	 */
2621 	if (pp == NULL) {
2622 		(void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
2623 		return;
2624 	}
2625 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2626 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2627 		(size_t)raddr;
2628 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2629 	seg = as_findseg(as, addr, 0);
2630 	ASSERT(seg);
2631 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
2632 	    "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
2633 	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
2634 	AS_LOCK_EXIT(as, &as->a_lock);
2635 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
2636 }
2637 
2638 /*
2639  * reclaim cached pages in a given address range
2640  */
2641 void
2642 as_pagereclaim(struct as *as, struct page **pp, caddr_t addr,
2643     size_t size, enum seg_rw rw)
2644 {
2645 	struct seg *seg;
2646 	size_t rsize;
2647 	caddr_t raddr;
2648 
2649 	ASSERT(AS_READ_HELD(as, &as->a_lock));
2650 	ASSERT(pp != NULL);
2651 
2652 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2653 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2654 		(size_t)raddr;
2655 	seg = as_findseg(as, addr, 0);
2656 	ASSERT(seg);
2657 	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGERECLAIM, rw);
2658 }
2659 
2660 #define	MAXPAGEFLIP	4
2661 #define	MAXPAGEFLIPSIZ	MAXPAGEFLIP*PAGESIZE
2662 
2663 int
2664 as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2665     boolean_t wait)
2666 {
2667 	struct seg *seg;
2668 	size_t ssize;
2669 	caddr_t raddr;			/* rounded down addr */
2670 	size_t rsize;			/* rounded up size */
2671 	int error = 0;
2672 	size_t pgsz = page_get_pagesize(szc);
2673 
2674 setpgsz_top:
2675 	if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) {
2676 		return (EINVAL);
2677 	}
2678 
2679 	raddr = addr;
2680 	rsize = size;
2681 
2682 	if (raddr + rsize < raddr)		/* check for wraparound */
2683 		return (ENOMEM);
2684 
2685 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2686 	as_clearwatchprot(as, raddr, rsize);
2687 	seg = as_segat(as, raddr);
2688 	if (seg == NULL) {
2689 		as_setwatch(as);
2690 		AS_LOCK_EXIT(as, &as->a_lock);
2691 		return (ENOMEM);
2692 	}
2693 
2694 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2695 		if (raddr >= seg->s_base + seg->s_size) {
2696 			seg = AS_SEGNEXT(as, seg);
2697 			if (seg == NULL || raddr != seg->s_base) {
2698 				error = ENOMEM;
2699 				break;
2700 			}
2701 		}
2702 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2703 			ssize = seg->s_base + seg->s_size - raddr;
2704 		} else {
2705 			ssize = rsize;
2706 		}
2707 
2708 		error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2709 
2710 		if (error == IE_NOMEM) {
2711 			error = EAGAIN;
2712 			break;
2713 		}
2714 
2715 		if (error == IE_RETRY) {
2716 			AS_LOCK_EXIT(as, &as->a_lock);
2717 			goto setpgsz_top;
2718 		}
2719 
2720 		if (error == ENOTSUP) {
2721 			error = EINVAL;
2722 			break;
2723 		}
2724 
2725 		if (wait && (error == EAGAIN)) {
2726 			/*
2727 			 * Memory is currently locked.  It must be unlocked
2728 			 * before this operation can succeed through a retry.
2729 			 * The possible reasons for locked memory and
2730 			 * corresponding strategies for unlocking are:
2731 			 * (1) Normal I/O
2732 			 *	wait for a signal that the I/O operation
2733 			 *	has completed and the memory is unlocked.
2734 			 * (2) Asynchronous I/O
2735 			 *	The aio subsystem does not unlock pages when
2736 			 *	the I/O is completed. Those pages are unlocked
2737 			 *	when the application calls aiowait/aioerror.
2738 			 *	So, to prevent blocking forever, cv_broadcast()
2739 			 *	is done to wake up aio_cleanup_thread.
2740 			 *	Subsequently, segvn_reclaim will be called, and
2741 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
2742 			 * (3) Long term page locking:
2743 			 *	This is not relevant for as_setpagesize()
2744 			 *	because we cannot change the page size for
2745 			 *	driver memory. The attempt to do so will
2746 			 *	fail with a different error than EAGAIN so
2747 			 *	there's no need to trigger as callbacks like
2748 			 *	as_unmap, as_setprot or as_free would do.
2749 			 */
2750 			mutex_enter(&as->a_contents);
2751 			if (AS_ISUNMAPWAIT(as) == 0) {
2752 				cv_broadcast(&as->a_cv);
2753 			}
2754 			AS_SETUNMAPWAIT(as);
2755 			AS_LOCK_EXIT(as, &as->a_lock);
2756 			while (AS_ISUNMAPWAIT(as)) {
2757 				cv_wait(&as->a_cv, &as->a_contents);
2758 			}
2759 			mutex_exit(&as->a_contents);
2760 			goto setpgsz_top;
2761 		} else if (error != 0) {
2762 			break;
2763 		}
2764 	}
2765 	as_setwatch(as);
2766 	AS_LOCK_EXIT(as, &as->a_lock);
2767 	return (error);
2768 }
2769 
2770 /*
2771  * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments
2772  * in its chunk where s_szc is less than the szc we want to set.
2773  */
2774 static int
2775 as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
2776     int *retry)
2777 {
2778 	struct seg *seg;
2779 	size_t ssize;
2780 	int error;
2781 
2782 	seg = as_segat(as, raddr);
2783 	if (seg == NULL) {
2784 		panic("as_iset3_default_lpsize: no seg");
2785 	}
2786 
2787 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2788 		if (raddr >= seg->s_base + seg->s_size) {
2789 			seg = AS_SEGNEXT(as, seg);
2790 			if (seg == NULL || raddr != seg->s_base) {
2791 				panic("as_iset3_default_lpsize: as changed");
2792 			}
2793 		}
2794 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2795 			ssize = seg->s_base + seg->s_size - raddr;
2796 		} else {
2797 			ssize = rsize;
2798 		}
2799 
2800 		if (szc > seg->s_szc) {
2801 			error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2802 			/* Only retry on EINVAL segments that have no vnode. */
2803 			if (error == EINVAL) {
2804 				vnode_t *vp = NULL;
2805 				if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) &&
2806 				    (SEGOP_GETVP(seg, raddr, &vp) != 0 ||
2807 				    vp == NULL)) {
2808 					*retry = 1;
2809 				} else {
2810 					*retry = 0;
2811 				}
2812 			}
2813 			if (error) {
2814 				return (error);
2815 			}
2816 		}
2817 	}
2818 	return (0);
2819 }
2820 
2821 /*
2822  * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the
2823  * pagesize on each segment in its range, but if any fails with EINVAL,
2824  * then it reduces the pagesizes to the next size in the bitmap and
2825  * retries as_iset3_default_lpsize(). The reason why the code retries
2826  * smaller allowed sizes on EINVAL is because (a) the anon offset may not
2827  * match the bigger sizes, and (b) it's hard to get this offset (to begin
2828  * with) to pass to map_pgszcvec().
2829  */
2830 static int
2831 as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2832     uint_t szcvec)
2833 {
2834 	int error;
2835 	int retry;
2836 
2837 	for (;;) {
2838 		error = as_iset3_default_lpsize(as, addr, size, szc, &retry);
2839 		if (error == EINVAL && retry) {
2840 			szcvec &= ~(1 << szc);
2841 			if (szcvec <= 1) {
2842 				return (EINVAL);
2843 			}
2844 			szc = highbit(szcvec) - 1;
2845 		} else {
2846 			return (error);
2847 		}
2848 	}
2849 }
2850 
2851 /*
2852  * as_iset1_default_lpsize() breaks its chunk into areas where existing
2853  * segments have a smaller szc than we want to set. For each such area,
2854  * it calls as_iset2_default_lpsize()
2855  */
2856 static int
2857 as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
2858     uint_t szcvec)
2859 {
2860 	struct seg *seg;
2861 	size_t ssize;
2862 	caddr_t setaddr = raddr;
2863 	size_t setsize = 0;
2864 	int set;
2865 	int error;
2866 
2867 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2868 
2869 	seg = as_segat(as, raddr);
2870 	if (seg == NULL) {
2871 		panic("as_iset1_default_lpsize: no seg");
2872 	}
2873 	if (seg->s_szc < szc) {
2874 		set = 1;
2875 	} else {
2876 		set = 0;
2877 	}
2878 
2879 	for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
2880 		if (raddr >= seg->s_base + seg->s_size) {
2881 			seg = AS_SEGNEXT(as, seg);
2882 			if (seg == NULL || raddr != seg->s_base) {
2883 				panic("as_iset1_default_lpsize: as changed");
2884 			}
2885 			if (seg->s_szc >= szc && set) {
2886 				ASSERT(setsize != 0);
2887 				error = as_iset2_default_lpsize(as,
2888 				    setaddr, setsize, szc, szcvec);
2889 				if (error) {
2890 					return (error);
2891 				}
2892 				set = 0;
2893 			} else if (seg->s_szc < szc && !set) {
2894 				setaddr = raddr;
2895 				setsize = 0;
2896 				set = 1;
2897 			}
2898 		}
2899 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2900 			ssize = seg->s_base + seg->s_size - raddr;
2901 		} else {
2902 			ssize = rsize;
2903 		}
2904 	}
2905 	error = 0;
2906 	if (set) {
2907 		ASSERT(setsize != 0);
2908 		error = as_iset2_default_lpsize(as, setaddr, setsize,
2909 		    szc, szcvec);
2910 	}
2911 	return (error);
2912 }
2913 
2914 /*
2915  * as_iset_default_lpsize() breaks its chunk according to the size code bitmap
2916  * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each
2917  * chunk to as_iset1_default_lpsize().
2918  */
2919 static int
2920 as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags,
2921     int type)
2922 {
2923 	int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
2924 	uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr,
2925 					flags, rtype, 1);
2926 	uint_t szc;
2927 	uint_t nszc;
2928 	int error;
2929 	caddr_t a;
2930 	caddr_t eaddr;
2931 	size_t segsize;
2932 	size_t pgsz;
2933 	uint_t save_szcvec;
2934 
2935 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2936 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2937 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2938 
2939 	szcvec &= ~1;
2940 	if (szcvec <= 1) {	/* skip if base page size */
2941 		return (0);
2942 	}
2943 
2944 	/* Get the pagesize of the first larger page size. */
2945 	szc = lowbit(szcvec) - 1;
2946 	pgsz = page_get_pagesize(szc);
2947 	eaddr = addr + size;
2948 	addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
2949 	eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
2950 
2951 	save_szcvec = szcvec;
2952 	szcvec >>= (szc + 1);
2953 	nszc = szc;
2954 	while (szcvec) {
2955 		if ((szcvec & 0x1) == 0) {
2956 			nszc++;
2957 			szcvec >>= 1;
2958 			continue;
2959 		}
2960 		nszc++;
2961 		pgsz = page_get_pagesize(nszc);
2962 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
2963 		if (a != addr) {
2964 			ASSERT(szc > 0);
2965 			ASSERT(a < eaddr);
2966 			segsize = a - addr;
2967 			error = as_iset1_default_lpsize(as, addr, segsize, szc,
2968 			    save_szcvec);
2969 			if (error) {
2970 				return (error);
2971 			}
2972 			addr = a;
2973 		}
2974 		szc = nszc;
2975 		szcvec >>= 1;
2976 	}
2977 
2978 	ASSERT(addr < eaddr);
2979 	szcvec = save_szcvec;
2980 	while (szcvec) {
2981 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
2982 		ASSERT(a >= addr);
2983 		if (a != addr) {
2984 			ASSERT(szc > 0);
2985 			segsize = a - addr;
2986 			error = as_iset1_default_lpsize(as, addr, segsize, szc,
2987 			    save_szcvec);
2988 			if (error) {
2989 				return (error);
2990 			}
2991 			addr = a;
2992 		}
2993 		szcvec &= ~(1 << szc);
2994 		if (szcvec) {
2995 			szc = highbit(szcvec) - 1;
2996 			pgsz = page_get_pagesize(szc);
2997 		}
2998 	}
2999 	ASSERT(addr == eaddr);
3000 
3001 	return (0);
3002 }
3003 
3004 /*
3005  * Set the default large page size for the range. Called via memcntl with
3006  * page size set to 0. as_set_default_lpsize breaks the range down into
3007  * chunks with the same type/flags, ignores-non segvn segments, and passes
3008  * each chunk to as_iset_default_lpsize().
3009  */
3010 int
3011 as_set_default_lpsize(struct as *as, caddr_t addr, size_t size)
3012 {
3013 	struct seg *seg;
3014 	caddr_t raddr;
3015 	size_t rsize;
3016 	size_t ssize;
3017 	int rtype, rflags;
3018 	int stype, sflags;
3019 	int error;
3020 	caddr_t	setaddr;
3021 	size_t setsize;
3022 	int segvn;
3023 
3024 	if (size == 0)
3025 		return (0);
3026 
3027 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
3028 again:
3029 	error = 0;
3030 
3031 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3032 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
3033 	    (size_t)raddr;
3034 
3035 	if (raddr + rsize < raddr) {		/* check for wraparound */
3036 		AS_LOCK_EXIT(as, &as->a_lock);
3037 		return (ENOMEM);
3038 	}
3039 	as_clearwatchprot(as, raddr, rsize);
3040 	seg = as_segat(as, raddr);
3041 	if (seg == NULL) {
3042 		as_setwatch(as);
3043 		AS_LOCK_EXIT(as, &as->a_lock);
3044 		return (ENOMEM);
3045 	}
3046 	if (seg->s_ops == &segvn_ops) {
3047 		rtype = SEGOP_GETTYPE(seg, addr);
3048 		rflags = rtype & (MAP_TEXT | MAP_INITDATA);
3049 		rtype = rtype & (MAP_SHARED | MAP_PRIVATE);
3050 		segvn = 1;
3051 	} else {
3052 		segvn = 0;
3053 	}
3054 	setaddr = raddr;
3055 	setsize = 0;
3056 
3057 	for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3058 		if (raddr >= (seg->s_base + seg->s_size)) {
3059 			seg = AS_SEGNEXT(as, seg);
3060 			if (seg == NULL || raddr != seg->s_base) {
3061 				error = ENOMEM;
3062 				break;
3063 			}
3064 			if (seg->s_ops == &segvn_ops) {
3065 				stype = SEGOP_GETTYPE(seg, raddr);
3066 				sflags = stype & (MAP_TEXT | MAP_INITDATA);
3067 				stype &= (MAP_SHARED | MAP_PRIVATE);
3068 				if (segvn && (rflags != sflags ||
3069 				    rtype != stype)) {
3070 					/*
3071 					 * The next segment is also segvn but
3072 					 * has different flags and/or type.
3073 					 */
3074 					ASSERT(setsize != 0);
3075 					error = as_iset_default_lpsize(as,
3076 					    setaddr, setsize, rflags, rtype);
3077 					if (error) {
3078 						break;
3079 					}
3080 					rflags = sflags;
3081 					rtype = stype;
3082 					setaddr = raddr;
3083 					setsize = 0;
3084 				} else if (!segvn) {
3085 					rflags = sflags;
3086 					rtype = stype;
3087 					setaddr = raddr;
3088 					setsize = 0;
3089 					segvn = 1;
3090 				}
3091 			} else if (segvn) {
3092 				/* The next segment is not segvn. */
3093 				ASSERT(setsize != 0);
3094 				error = as_iset_default_lpsize(as,
3095 				    setaddr, setsize, rflags, rtype);
3096 				if (error) {
3097 					break;
3098 				}
3099 				segvn = 0;
3100 			}
3101 		}
3102 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3103 			ssize = seg->s_base + seg->s_size - raddr;
3104 		} else {
3105 			ssize = rsize;
3106 		}
3107 	}
3108 	if (error == 0 && segvn) {
3109 		/* The last chunk when rsize == 0. */
3110 		ASSERT(setsize != 0);
3111 		error = as_iset_default_lpsize(as, setaddr, setsize,
3112 		    rflags, rtype);
3113 	}
3114 
3115 	if (error == IE_RETRY) {
3116 		goto again;
3117 	} else if (error == IE_NOMEM) {
3118 		error = EAGAIN;
3119 	} else if (error == ENOTSUP) {
3120 		error = EINVAL;
3121 	} else if (error == EAGAIN) {
3122 		mutex_enter(&as->a_contents);
3123 		if (AS_ISUNMAPWAIT(as) == 0) {
3124 			cv_broadcast(&as->a_cv);
3125 		}
3126 		AS_SETUNMAPWAIT(as);
3127 		AS_LOCK_EXIT(as, &as->a_lock);
3128 		while (AS_ISUNMAPWAIT(as)) {
3129 			cv_wait(&as->a_cv, &as->a_contents);
3130 		}
3131 		mutex_exit(&as->a_contents);
3132 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
3133 		goto again;
3134 	}
3135 
3136 	as_setwatch(as);
3137 	AS_LOCK_EXIT(as, &as->a_lock);
3138 	return (error);
3139 }
3140 
3141 /*
3142  * Setup all of the uninitialized watched pages that we can.
3143  */
3144 void
3145 as_setwatch(struct as *as)
3146 {
3147 	struct watched_page *pwp;
3148 	struct seg *seg;
3149 	caddr_t vaddr;
3150 	uint_t prot;
3151 	int  err, retrycnt;
3152 
3153 	if (avl_numnodes(&as->a_wpage) == 0)
3154 		return;
3155 
3156 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3157 
3158 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
3159 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
3160 		retrycnt = 0;
3161 	retry:
3162 		vaddr = pwp->wp_vaddr;
3163 		if (pwp->wp_oprot != 0 ||	/* already set up */
3164 		    (seg = as_segat(as, vaddr)) == NULL ||
3165 		    SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
3166 			continue;
3167 
3168 		pwp->wp_oprot = prot;
3169 		if (pwp->wp_read)
3170 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3171 		if (pwp->wp_write)
3172 			prot &= ~PROT_WRITE;
3173 		if (pwp->wp_exec)
3174 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3175 		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) {
3176 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
3177 			if (err == IE_RETRY) {
3178 				pwp->wp_oprot = 0;
3179 				ASSERT(retrycnt == 0);
3180 				retrycnt++;
3181 				goto retry;
3182 			}
3183 		}
3184 		pwp->wp_prot = prot;
3185 	}
3186 }
3187 
3188 /*
3189  * Clear all of the watched pages in the address space.
3190  */
3191 void
3192 as_clearwatch(struct as *as)
3193 {
3194 	struct watched_page *pwp;
3195 	struct seg *seg;
3196 	caddr_t vaddr;
3197 	uint_t prot;
3198 	int err, retrycnt;
3199 
3200 	if (avl_numnodes(&as->a_wpage) == 0)
3201 		return;
3202 
3203 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3204 
3205 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
3206 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
3207 		retrycnt = 0;
3208 	retry:
3209 		vaddr = pwp->wp_vaddr;
3210 		if (pwp->wp_oprot == 0 ||	/* not set up */
3211 		    (seg = as_segat(as, vaddr)) == NULL)
3212 			continue;
3213 
3214 		if ((prot = pwp->wp_oprot) != pwp->wp_prot) {
3215 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
3216 			if (err == IE_RETRY) {
3217 				ASSERT(retrycnt == 0);
3218 				retrycnt++;
3219 				goto retry;
3220 			}
3221 		}
3222 		pwp->wp_oprot = 0;
3223 		pwp->wp_prot = 0;
3224 	}
3225 }
3226 
3227 /*
3228  * Force a new setup for all the watched pages in the range.
3229  */
3230 static void
3231 as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
3232 {
3233 	struct watched_page *pwp;
3234 	struct watched_page tpw;
3235 	caddr_t eaddr = addr + size;
3236 	caddr_t vaddr;
3237 	struct seg *seg;
3238 	int err, retrycnt;
3239 	uint_t	wprot;
3240 	avl_index_t where;
3241 
3242 	if (avl_numnodes(&as->a_wpage) == 0)
3243 		return;
3244 
3245 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3246 
3247 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3248 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
3249 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
3250 
3251 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
3252 		retrycnt = 0;
3253 		vaddr = pwp->wp_vaddr;
3254 
3255 		wprot = prot;
3256 		if (pwp->wp_read)
3257 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3258 		if (pwp->wp_write)
3259 			wprot &= ~PROT_WRITE;
3260 		if (pwp->wp_exec)
3261 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3262 		if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) {
3263 		retry:
3264 			seg = as_segat(as, vaddr);
3265 			if (seg == NULL) {
3266 				panic("as_setwatchprot: no seg");
3267 				/*NOTREACHED*/
3268 			}
3269 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot);
3270 			if (err == IE_RETRY) {
3271 				ASSERT(retrycnt == 0);
3272 				retrycnt++;
3273 				goto retry;
3274 			}
3275 		}
3276 		pwp->wp_oprot = prot;
3277 		pwp->wp_prot = wprot;
3278 
3279 		pwp = AVL_NEXT(&as->a_wpage, pwp);
3280 	}
3281 }
3282 
3283 /*
3284  * Clear all of the watched pages in the range.
3285  */
3286 static void
3287 as_clearwatchprot(struct as *as, caddr_t addr, size_t size)
3288 {
3289 	caddr_t eaddr = addr + size;
3290 	struct watched_page *pwp;
3291 	struct watched_page tpw;
3292 	uint_t prot;
3293 	struct seg *seg;
3294 	int err, retrycnt;
3295 	avl_index_t where;
3296 
3297 	if (avl_numnodes(&as->a_wpage) == 0)
3298 		return;
3299 
3300 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3301 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
3302 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
3303 
3304 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3305 
3306 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
3307 		ASSERT(addr >= pwp->wp_vaddr);
3308 
3309 		if ((prot = pwp->wp_oprot) != 0) {
3310 			retrycnt = 0;
3311 
3312 			if (prot != pwp->wp_prot) {
3313 			retry:
3314 				seg = as_segat(as, pwp->wp_vaddr);
3315 				if (seg == NULL)
3316 					continue;
3317 				err = SEGOP_SETPROT(seg, pwp->wp_vaddr,
3318 				    PAGESIZE, prot);
3319 				if (err == IE_RETRY) {
3320 					ASSERT(retrycnt == 0);
3321 					retrycnt++;
3322 					goto retry;
3323 
3324 				}
3325 			}
3326 			pwp->wp_oprot = 0;
3327 			pwp->wp_prot = 0;
3328 		}
3329 
3330 		pwp = AVL_NEXT(&as->a_wpage, pwp);
3331 	}
3332 }
3333 
3334 void
3335 as_signal_proc(struct as *as, k_siginfo_t *siginfo)
3336 {
3337 	struct proc *p;
3338 
3339 	mutex_enter(&pidlock);
3340 	for (p = practive; p; p = p->p_next) {
3341 		if (p->p_as == as) {
3342 			mutex_enter(&p->p_lock);
3343 			if (p->p_as == as)
3344 				sigaddq(p, NULL, siginfo, KM_NOSLEEP);
3345 			mutex_exit(&p->p_lock);
3346 		}
3347 	}
3348 	mutex_exit(&pidlock);
3349 }
3350 
3351 /*
3352  * return memory object ID
3353  */
3354 int
3355 as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
3356 {
3357 	struct seg	*seg;
3358 	int		sts;
3359 
3360 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
3361 	seg = as_segat(as, addr);
3362 	if (seg == NULL) {
3363 		AS_LOCK_EXIT(as, &as->a_lock);
3364 		return (EFAULT);
3365 	}
3366 	/*
3367 	 * catch old drivers which may not support getmemid
3368 	 */
3369 	if (seg->s_ops->getmemid == NULL) {
3370 		AS_LOCK_EXIT(as, &as->a_lock);
3371 		return (ENODEV);
3372 	}
3373 
3374 	sts = SEGOP_GETMEMID(seg, addr, memidp);
3375 
3376 	AS_LOCK_EXIT(as, &as->a_lock);
3377 	return (sts);
3378 }
3379