xref: /titanic_52/usr/src/uts/common/vm/vm_as.c (revision ba4e3c84e6b9390bbf7df80b5f1d11dec34cc525)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 /*
30  * University Copyright- Copyright (c) 1982, 1986, 1988
31  * The Regents of the University of California
32  * All Rights Reserved
33  *
34  * University Acknowledgment- Portions of this document are derived from
35  * software developed by the University of California, Berkeley, and its
36  * contributors.
37  */
38 
39 #pragma ident	"%Z%%M%	%I%	%E% SMI"
40 
41 /*
42  * VM - address spaces.
43  */
44 
45 #include <sys/types.h>
46 #include <sys/t_lock.h>
47 #include <sys/param.h>
48 #include <sys/errno.h>
49 #include <sys/systm.h>
50 #include <sys/mman.h>
51 #include <sys/sysmacros.h>
52 #include <sys/cpuvar.h>
53 #include <sys/sysinfo.h>
54 #include <sys/kmem.h>
55 #include <sys/vnode.h>
56 #include <sys/vmsystm.h>
57 #include <sys/cmn_err.h>
58 #include <sys/debug.h>
59 #include <sys/tnf_probe.h>
60 #include <sys/vtrace.h>
61 
62 #include <vm/hat.h>
63 #include <vm/xhat.h>
64 #include <vm/as.h>
65 #include <vm/seg.h>
66 #include <vm/seg_vn.h>
67 #include <vm/seg_dev.h>
68 #include <vm/seg_kmem.h>
69 #include <vm/seg_map.h>
70 #include <vm/seg_spt.h>
71 #include <vm/page.h>
72 
73 clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
74 
75 static struct kmem_cache *as_cache;
76 
77 static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
78 static void as_clearwatchprot(struct as *, caddr_t, size_t);
79 int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *);
80 
81 
82 /*
83  * Verifying the segment lists is very time-consuming; it may not be
84  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
85  */
86 #ifdef DEBUG
87 #define	VERIFY_SEGLIST
88 int do_as_verify = 0;
89 #endif
90 
91 /*
92  * Allocate a new callback data structure entry and fill in the events of
93  * interest, the address range of interest, and the callback argument.
94  * Link the entry on the as->a_callbacks list. A callback entry for the
95  * entire address space may be specified with vaddr = 0 and size = -1.
96  *
97  * CALLERS RESPONSIBILITY: If not calling from within the process context for
98  * the specified as, the caller must guarantee persistence of the specified as
99  * for the duration of this function (eg. pages being locked within the as
100  * will guarantee persistence).
101  */
102 int
103 as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
104 		caddr_t vaddr, size_t size, int sleepflag)
105 {
106 	struct as_callback 	*current_head, *cb;
107 	caddr_t 		saddr;
108 	size_t 			rsize;
109 
110 	/* callback function and an event are mandatory */
111 	if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
112 		return (EINVAL);
113 
114 	/* Adding a callback after as_free has been called is not allowed */
115 	if (as == &kas)
116 		return (ENOMEM);
117 
118 	/*
119 	 * vaddr = 0 and size = -1 is used to indicate that the callback range
120 	 * is the entire address space so no rounding is done in that case.
121 	 */
122 	if (size != -1) {
123 		saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
124 		rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
125 			(size_t)saddr;
126 		/* check for wraparound */
127 		if (saddr + rsize < saddr)
128 			return (ENOMEM);
129 	} else {
130 		if (vaddr != 0)
131 			return (EINVAL);
132 		saddr = vaddr;
133 		rsize = size;
134 	}
135 
136 	/* Allocate and initialize a callback entry */
137 	cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
138 	if (cb == NULL)
139 		return (EAGAIN);
140 
141 	cb->ascb_func = cb_func;
142 	cb->ascb_arg = arg;
143 	cb->ascb_events = events;
144 	cb->ascb_saddr = saddr;
145 	cb->ascb_len = rsize;
146 
147 	/* Add the entry to the list */
148 	mutex_enter(&as->a_contents);
149 	current_head = as->a_callbacks;
150 	as->a_callbacks = cb;
151 	cb->ascb_next = current_head;
152 
153 	/*
154 	 * The call to this function may lose in a race with
155 	 * a pertinent event - eg. a thread does long term memory locking
156 	 * but before the callback is added another thread executes as_unmap.
157 	 * A broadcast here resolves that.
158 	 */
159 	if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
160 		AS_CLRUNMAPWAIT(as);
161 		cv_broadcast(&as->a_cv);
162 	}
163 
164 	mutex_exit(&as->a_contents);
165 	return (0);
166 }
167 
168 /*
169  * Search the callback list for an entry which pertains to arg.
170  *
171  * This is called from within the client upon completion of the callback.
172  * RETURN VALUES:
173  *	AS_CALLBACK_DELETED  (callback entry found and deleted)
174  *	AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
175  *	AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
176  *			entry will be made in as_do_callbacks)
177  *
178  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
179  * set, it indicates that as_do_callbacks is processing this entry.  The
180  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
181  * to unblock as_do_callbacks, in case it is blocked.
182  *
183  * CALLERS RESPONSIBILITY: If not calling from within the process context for
184  * the specified as, the caller must guarantee persistence of the specified as
185  * for the duration of this function (eg. pages being locked within the as
186  * will guarantee persistence).
187  */
188 uint_t
189 as_delete_callback(struct as *as, void *arg)
190 {
191 	struct as_callback **prevcb = &as->a_callbacks;
192 	struct as_callback *cb;
193 	uint_t rc = AS_CALLBACK_NOTFOUND;
194 
195 	mutex_enter(&as->a_contents);
196 	for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
197 		if (cb->ascb_arg != arg)
198 			continue;
199 
200 		/*
201 		 * If the events indicate AS_CALLBACK_CALLED, just clear
202 		 * AS_ALL_EVENT in the events field and wakeup the thread
203 		 * that may be waiting in as_do_callbacks.  as_do_callbacks
204 		 * will take care of removing this entry from the list.  In
205 		 * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
206 		 * (AS_CALLBACK_CALLED not set), just remove it from the
207 		 * list, return the memory and return AS_CALLBACK_DELETED.
208 		 */
209 		if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
210 			/* leave AS_CALLBACK_CALLED */
211 			cb->ascb_events &= ~AS_ALL_EVENT;
212 			rc = AS_CALLBACK_DELETE_DEFERRED;
213 			cv_broadcast(&as->a_cv);
214 		} else {
215 			*prevcb = cb->ascb_next;
216 			kmem_free(cb, sizeof (struct as_callback));
217 			rc = AS_CALLBACK_DELETED;
218 		}
219 		break;
220 	}
221 	mutex_exit(&as->a_contents);
222 	return (rc);
223 }
224 
225 /*
226  * Searches the as callback list for a matching entry.
227  * Returns a pointer to the first matching callback, or NULL if
228  * nothing is found.
229  * This function never sleeps so it is ok to call it with more
230  * locks held but the (required) a_contents mutex.
231  *
232  * See also comment on as_do_callbacks below.
233  */
234 static struct as_callback *
235 as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
236 			size_t event_len)
237 {
238 	struct as_callback	*cb;
239 
240 	ASSERT(MUTEX_HELD(&as->a_contents));
241 	for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
242 		/*
243 		 * If the callback has not already been called, then
244 		 * check if events or address range pertains.  An event_len
245 		 * of zero means do an unconditional callback.
246 		 */
247 		if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
248 		    ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
249 		    (event_addr + event_len < cb->ascb_saddr) ||
250 		    (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
251 			continue;
252 		}
253 		break;
254 	}
255 	return (cb);
256 }
257 
258 /*
259  * Executes a given callback and removes it from the callback list for
260  * this address space.
261  * This function may sleep so the caller must drop all locks except
262  * a_contents before calling this func.
263  *
264  * See also comments on as_do_callbacks below.
265  */
266 static void
267 as_execute_callback(struct as *as, struct as_callback *cb,
268 				uint_t events)
269 {
270 	struct as_callback **prevcb;
271 	void	*cb_arg;
272 
273 	ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
274 	cb->ascb_events |= AS_CALLBACK_CALLED;
275 	mutex_exit(&as->a_contents);
276 	(*cb->ascb_func)(as, cb->ascb_arg, events);
277 	mutex_enter(&as->a_contents);
278 	/*
279 	 * the callback function is required to delete the callback
280 	 * when the callback function determines it is OK for
281 	 * this thread to continue. as_delete_callback will clear
282 	 * the AS_ALL_EVENT in the events field when it is deleted.
283 	 * If the callback function called as_delete_callback,
284 	 * events will already be cleared and there will be no blocking.
285 	 */
286 	while ((cb->ascb_events & events) != 0) {
287 		cv_wait(&as->a_cv, &as->a_contents);
288 	}
289 	/*
290 	 * This entry needs to be taken off the list. Normally, the
291 	 * callback func itself does that, but unfortunately the list
292 	 * may have changed while the callback was running because the
293 	 * a_contents mutex was dropped and someone else other than the
294 	 * callback func itself could have called as_delete_callback,
295 	 * so we have to search to find this entry again.  The entry
296 	 * must have AS_CALLBACK_CALLED, and have the same 'arg'.
297 	 */
298 	cb_arg = cb->ascb_arg;
299 	prevcb = &as->a_callbacks;
300 	for (cb = as->a_callbacks; cb != NULL;
301 	    prevcb = &cb->ascb_next, cb = *prevcb) {
302 		if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
303 		    (cb_arg != cb->ascb_arg)) {
304 			continue;
305 		}
306 		*prevcb = cb->ascb_next;
307 		kmem_free(cb, sizeof (struct as_callback));
308 		break;
309 	}
310 }
311 
312 /*
313  * Check the callback list for a matching event and intersection of
314  * address range. If there is a match invoke the callback.  Skip an entry if:
315  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
316  *    - not event of interest
317  *    - not address range of interest
318  *
319  * An event_len of zero indicates a request for an unconditional callback
320  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
321  * a_contents lock must be dropped before a callback, so only one callback
322  * can be done before returning. Return -1 (true) if a callback was
323  * executed and removed from the list, else return 0 (false).
324  *
325  * The logically separate parts, i.e. finding a matching callback and
326  * executing a given callback have been separated into two functions
327  * so that they can be called with different sets of locks held beyond
328  * the always-required a_contents. as_find_callback does not sleep so
329  * it is ok to call it if more locks than a_contents (i.e. the a_lock
330  * rwlock) are held. as_execute_callback on the other hand may sleep
331  * so all locks beyond a_contents must be dropped by the caller if one
332  * does not want to end comatose.
333  */
334 static int
335 as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
336 			size_t event_len)
337 {
338 	struct as_callback *cb;
339 
340 	if ((cb = as_find_callback(as, events, event_addr, event_len))) {
341 		as_execute_callback(as, cb, events);
342 		return (-1);
343 	}
344 	return (0);
345 }
346 
347 /*
348  * Search for the segment containing addr. If a segment containing addr
349  * exists, that segment is returned.  If no such segment exists, and
350  * the list spans addresses greater than addr, then the first segment
351  * whose base is greater than addr is returned; otherwise, NULL is
352  * returned unless tail is true, in which case the last element of the
353  * list is returned.
354  *
355  * a_seglast is used to cache the last found segment for repeated
356  * searches to the same addr (which happens frequently).
357  */
358 struct seg *
359 as_findseg(struct as *as, caddr_t addr, int tail)
360 {
361 	struct seg *seg = as->a_seglast;
362 	avl_index_t where;
363 
364 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
365 
366 	if (seg != NULL &&
367 	    seg->s_base <= addr &&
368 	    addr < seg->s_base + seg->s_size)
369 		return (seg);
370 
371 	seg = avl_find(&as->a_segtree, &addr, &where);
372 	if (seg != NULL)
373 		return (as->a_seglast = seg);
374 
375 	seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
376 	if (seg == NULL && tail)
377 		seg = avl_last(&as->a_segtree);
378 	return (as->a_seglast = seg);
379 }
380 
381 #ifdef VERIFY_SEGLIST
382 /*
383  * verify that the linked list is coherent
384  */
385 static void
386 as_verify(struct as *as)
387 {
388 	struct seg *seg, *seglast, *p, *n;
389 	uint_t nsegs = 0;
390 
391 	if (do_as_verify == 0)
392 		return;
393 
394 	seglast = as->a_seglast;
395 
396 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
397 		ASSERT(seg->s_as == as);
398 		p = AS_SEGPREV(as, seg);
399 		n = AS_SEGNEXT(as, seg);
400 		ASSERT(p == NULL || p->s_as == as);
401 		ASSERT(p == NULL || p->s_base < seg->s_base);
402 		ASSERT(n == NULL || n->s_base > seg->s_base);
403 		ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
404 		if (seg == seglast)
405 			seglast = NULL;
406 		nsegs++;
407 	}
408 	ASSERT(seglast == NULL);
409 	ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
410 }
411 #endif /* VERIFY_SEGLIST */
412 
413 /*
414  * Add a new segment to the address space. The avl_find()
415  * may be expensive so we attempt to use last segment accessed
416  * in as_gap() as an insertion point.
417  */
418 int
419 as_addseg(struct as  *as, struct seg *newseg)
420 {
421 	struct seg *seg;
422 	caddr_t addr;
423 	caddr_t eaddr;
424 	avl_index_t where;
425 
426 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
427 
428 	as->a_updatedir = 1;	/* inform /proc */
429 	gethrestime(&as->a_updatetime);
430 
431 	if (as->a_lastgaphl != NULL) {
432 		struct seg *hseg = NULL;
433 		struct seg *lseg = NULL;
434 
435 		if (as->a_lastgaphl->s_base > newseg->s_base) {
436 			hseg = as->a_lastgaphl;
437 			lseg = AVL_PREV(&as->a_segtree, hseg);
438 		} else {
439 			lseg = as->a_lastgaphl;
440 			hseg = AVL_NEXT(&as->a_segtree, lseg);
441 		}
442 
443 		if (hseg && lseg && lseg->s_base < newseg->s_base &&
444 		    hseg->s_base > newseg->s_base) {
445 			avl_insert_here(&as->a_segtree, newseg, lseg,
446 			    AVL_AFTER);
447 			as->a_lastgaphl = NULL;
448 			as->a_seglast = newseg;
449 			return (0);
450 		}
451 		as->a_lastgaphl = NULL;
452 	}
453 
454 	addr = newseg->s_base;
455 	eaddr = addr + newseg->s_size;
456 again:
457 
458 	seg = avl_find(&as->a_segtree, &addr, &where);
459 
460 	if (seg == NULL)
461 		seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
462 
463 	if (seg == NULL)
464 		seg = avl_last(&as->a_segtree);
465 
466 	if (seg != NULL) {
467 		caddr_t base = seg->s_base;
468 
469 		/*
470 		 * If top of seg is below the requested address, then
471 		 * the insertion point is at the end of the linked list,
472 		 * and seg points to the tail of the list.  Otherwise,
473 		 * the insertion point is immediately before seg.
474 		 */
475 		if (base + seg->s_size > addr) {
476 			if (addr >= base || eaddr > base) {
477 #ifdef __sparc
478 				extern struct seg_ops segnf_ops;
479 
480 				/*
481 				 * no-fault segs must disappear if overlaid.
482 				 * XXX need new segment type so
483 				 * we don't have to check s_ops
484 				 */
485 				if (seg->s_ops == &segnf_ops) {
486 					seg_unmap(seg);
487 					goto again;
488 				}
489 #endif
490 				return (-1);	/* overlapping segment */
491 			}
492 		}
493 	}
494 	as->a_seglast = newseg;
495 	avl_insert(&as->a_segtree, newseg, where);
496 
497 #ifdef VERIFY_SEGLIST
498 	as_verify(as);
499 #endif
500 	return (0);
501 }
502 
503 struct seg *
504 as_removeseg(struct as *as, struct seg *seg)
505 {
506 	avl_tree_t *t;
507 
508 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
509 
510 	as->a_updatedir = 1;	/* inform /proc */
511 	gethrestime(&as->a_updatetime);
512 
513 	if (seg == NULL)
514 		return (NULL);
515 
516 	t = &as->a_segtree;
517 	if (as->a_seglast == seg)
518 		as->a_seglast = NULL;
519 	as->a_lastgaphl = NULL;
520 
521 	/*
522 	 * if this segment is at an address higher than
523 	 * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
524 	 */
525 	if (as->a_lastgap &&
526 	    (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
527 		as->a_lastgap = AVL_NEXT(t, seg);
528 
529 	/*
530 	 * remove the segment from the seg tree
531 	 */
532 	avl_remove(t, seg);
533 
534 #ifdef VERIFY_SEGLIST
535 	as_verify(as);
536 #endif
537 	return (seg);
538 }
539 
540 /*
541  * Find a segment containing addr.
542  */
543 struct seg *
544 as_segat(struct as *as, caddr_t addr)
545 {
546 	struct seg *seg = as->a_seglast;
547 
548 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
549 
550 	if (seg != NULL && seg->s_base <= addr &&
551 	    addr < seg->s_base + seg->s_size)
552 		return (seg);
553 
554 	seg = avl_find(&as->a_segtree, &addr, NULL);
555 	return (seg);
556 }
557 
558 /*
559  * Serialize all searches for holes in an address space to
560  * prevent two or more threads from allocating the same virtual
561  * address range.  The address space must not be "read/write"
562  * locked by the caller since we may block.
563  */
564 void
565 as_rangelock(struct as *as)
566 {
567 	mutex_enter(&as->a_contents);
568 	while (AS_ISCLAIMGAP(as))
569 		cv_wait(&as->a_cv, &as->a_contents);
570 	AS_SETCLAIMGAP(as);
571 	mutex_exit(&as->a_contents);
572 }
573 
574 /*
575  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
576  */
577 void
578 as_rangeunlock(struct as *as)
579 {
580 	mutex_enter(&as->a_contents);
581 	AS_CLRCLAIMGAP(as);
582 	cv_signal(&as->a_cv);
583 	mutex_exit(&as->a_contents);
584 }
585 
586 /*
587  * compar segments (or just an address) by segment address range
588  */
589 static int
590 as_segcompar(const void *x, const void *y)
591 {
592 	struct seg *a = (struct seg *)x;
593 	struct seg *b = (struct seg *)y;
594 
595 	if (a->s_base < b->s_base)
596 		return (-1);
597 	if (a->s_base >= b->s_base + b->s_size)
598 		return (1);
599 	return (0);
600 }
601 
602 
603 void
604 as_avlinit(struct as *as)
605 {
606 	avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
607 	    offsetof(struct seg, s_tree));
608 	avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
609 	    offsetof(struct watched_page, wp_link));
610 }
611 
612 /*ARGSUSED*/
613 static int
614 as_constructor(void *buf, void *cdrarg, int kmflags)
615 {
616 	struct as *as = buf;
617 
618 	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
619 	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
620 	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
621 	as_avlinit(as);
622 	return (0);
623 }
624 
625 /*ARGSUSED1*/
626 static void
627 as_destructor(void *buf, void *cdrarg)
628 {
629 	struct as *as = buf;
630 
631 	avl_destroy(&as->a_segtree);
632 	mutex_destroy(&as->a_contents);
633 	cv_destroy(&as->a_cv);
634 	rw_destroy(&as->a_lock);
635 }
636 
637 void
638 as_init(void)
639 {
640 	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
641 		as_constructor, as_destructor, NULL, NULL, NULL, 0);
642 }
643 
644 /*
645  * Allocate and initialize an address space data structure.
646  * We call hat_alloc to allow any machine dependent
647  * information in the hat structure to be initialized.
648  */
649 struct as *
650 as_alloc(void)
651 {
652 	struct as *as;
653 
654 	as = kmem_cache_alloc(as_cache, KM_SLEEP);
655 
656 	as->a_flags		= 0;
657 	as->a_vbits		= 0;
658 	as->a_hrm		= NULL;
659 	as->a_seglast		= NULL;
660 	as->a_size		= 0;
661 	as->a_updatedir		= 0;
662 	gethrestime(&as->a_updatetime);
663 	as->a_objectdir		= NULL;
664 	as->a_sizedir		= 0;
665 	as->a_userlimit		= (caddr_t)USERLIMIT;
666 	as->a_lastgap		= NULL;
667 	as->a_lastgaphl		= NULL;
668 	as->a_callbacks		= NULL;
669 
670 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
671 	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
672 	AS_LOCK_EXIT(as, &as->a_lock);
673 
674 	as->a_xhat = NULL;
675 
676 	return (as);
677 }
678 
679 /*
680  * Free an address space data structure.
681  * Need to free the hat first and then
682  * all the segments on this as and finally
683  * the space for the as struct itself.
684  */
685 void
686 as_free(struct as *as)
687 {
688 	struct hat *hat = as->a_hat;
689 	struct seg *seg, *next;
690 	int called = 0;
691 
692 top:
693 	/*
694 	 * Invoke ALL callbacks. as_do_callbacks will do one callback
695 	 * per call, and not return (-1) until the callback has completed.
696 	 * When as_do_callbacks returns zero, all callbacks have completed.
697 	 */
698 	mutex_enter(&as->a_contents);
699 	while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0));
700 
701 	/* This will prevent new XHATs from attaching to as */
702 	if (!called)
703 		AS_SETBUSY(as);
704 	mutex_exit(&as->a_contents);
705 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
706 
707 	if (!called) {
708 		called = 1;
709 		hat_free_start(hat);
710 		if (as->a_xhat != NULL)
711 			xhat_free_start_all(as);
712 	}
713 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
714 		int err;
715 
716 		next = AS_SEGNEXT(as, seg);
717 		err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
718 		if (err == EAGAIN) {
719 			mutex_enter(&as->a_contents);
720 			if (as->a_callbacks) {
721 				AS_LOCK_EXIT(as, &as->a_lock);
722 			} else {
723 				/*
724 				 * Memory is currently locked. Wait for a
725 				 * cv_signal that it has been unlocked, then
726 				 * try the operation again.
727 				 */
728 				if (AS_ISUNMAPWAIT(as) == 0)
729 					cv_broadcast(&as->a_cv);
730 				AS_SETUNMAPWAIT(as);
731 				AS_LOCK_EXIT(as, &as->a_lock);
732 				while (AS_ISUNMAPWAIT(as))
733 					cv_wait(&as->a_cv, &as->a_contents);
734 			}
735 			mutex_exit(&as->a_contents);
736 			goto top;
737 		} else {
738 			/*
739 			 * We do not expect any other error return at this
740 			 * time. This is similar to an ASSERT in seg_unmap()
741 			 */
742 			ASSERT(err == 0);
743 		}
744 	}
745 	hat_free_end(hat);
746 	if (as->a_xhat != NULL)
747 		xhat_free_end_all(as);
748 	AS_LOCK_EXIT(as, &as->a_lock);
749 
750 	/* /proc stuff */
751 	ASSERT(avl_numnodes(&as->a_wpage) == 0);
752 	if (as->a_objectdir) {
753 		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
754 		as->a_objectdir = NULL;
755 		as->a_sizedir = 0;
756 	}
757 
758 	/*
759 	 * Free the struct as back to kmem.  Assert it has no segments.
760 	 */
761 	ASSERT(avl_numnodes(&as->a_segtree) == 0);
762 	kmem_cache_free(as_cache, as);
763 }
764 
765 int
766 as_dup(struct as *as, struct as **outas)
767 {
768 	struct as *newas;
769 	struct seg *seg, *newseg;
770 	int error;
771 
772 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
773 	as_clearwatch(as);
774 	newas = as_alloc();
775 	newas->a_userlimit = as->a_userlimit;
776 	AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER);
777 
778 	/* This will prevent new XHATs from attaching */
779 	mutex_enter(&as->a_contents);
780 	AS_SETBUSY(as);
781 	mutex_exit(&as->a_contents);
782 	mutex_enter(&newas->a_contents);
783 	AS_SETBUSY(newas);
784 	mutex_exit(&newas->a_contents);
785 
786 
787 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
788 
789 		if (seg->s_flags & S_PURGE)
790 			continue;
791 
792 		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
793 		if (newseg == NULL) {
794 			AS_LOCK_EXIT(newas, &newas->a_lock);
795 			as_setwatch(as);
796 			mutex_enter(&as->a_contents);
797 			AS_CLRBUSY(as);
798 			mutex_exit(&as->a_contents);
799 			AS_LOCK_EXIT(as, &as->a_lock);
800 			as_free(newas);
801 			return (-1);
802 		}
803 		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
804 			/*
805 			 * We call seg_free() on the new seg
806 			 * because the segment is not set up
807 			 * completely; i.e. it has no ops.
808 			 */
809 			as_setwatch(as);
810 			mutex_enter(&as->a_contents);
811 			AS_CLRBUSY(as);
812 			mutex_exit(&as->a_contents);
813 			AS_LOCK_EXIT(as, &as->a_lock);
814 			seg_free(newseg);
815 			AS_LOCK_EXIT(newas, &newas->a_lock);
816 			as_free(newas);
817 			return (error);
818 		}
819 		newas->a_size += seg->s_size;
820 	}
821 
822 	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
823 	if (as->a_xhat != NULL)
824 		error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL);
825 
826 	mutex_enter(&newas->a_contents);
827 	AS_CLRBUSY(newas);
828 	mutex_exit(&newas->a_contents);
829 	AS_LOCK_EXIT(newas, &newas->a_lock);
830 
831 	as_setwatch(as);
832 	mutex_enter(&as->a_contents);
833 	AS_CLRBUSY(as);
834 	mutex_exit(&as->a_contents);
835 	AS_LOCK_EXIT(as, &as->a_lock);
836 	if (error != 0) {
837 		as_free(newas);
838 		return (error);
839 	}
840 	*outas = newas;
841 	return (0);
842 }
843 
844 /*
845  * Handle a ``fault'' at addr for size bytes.
846  */
847 faultcode_t
848 as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
849 	enum fault_type type, enum seg_rw rw)
850 {
851 	struct seg *seg;
852 	caddr_t raddr;			/* rounded down addr */
853 	size_t rsize;			/* rounded up size */
854 	size_t ssize;
855 	faultcode_t res = 0;
856 	caddr_t addrsav;
857 	struct seg *segsav;
858 	int as_lock_held;
859 	klwp_t *lwp = ttolwp(curthread);
860 	int is_xhat = 0;
861 	int holding_wpage = 0;
862 	extern struct seg_ops   segdev_ops;
863 
864 
865 
866 	if (as->a_hat != hat) {
867 		/* This must be an XHAT then */
868 		is_xhat = 1;
869 
870 		if ((type != F_INVAL) || (as == &kas))
871 			return (FC_NOSUPPORT);
872 	}
873 
874 retry:
875 	if (!is_xhat) {
876 		/*
877 		 * Indicate that the lwp is not to be stopped while waiting
878 		 * for a pagefault.  This is to avoid deadlock while debugging
879 		 * a process via /proc over NFS (in particular).
880 		 */
881 		if (lwp != NULL) {
882 			lwp->lwp_nostop++;
883 			lwp->lwp_nostop_r++;
884 		}
885 
886 		/*
887 		 * same length must be used when we softlock and softunlock.
888 		 * We don't support softunlocking lengths less than
889 		 * the original length when there is largepage support.
890 		 * See seg_dev.c for more comments.
891 		 */
892 		switch (type) {
893 
894 		case F_SOFTLOCK:
895 			CPU_STATS_ADD_K(vm, softlock, 1);
896 			break;
897 
898 		case F_SOFTUNLOCK:
899 			break;
900 
901 		case F_PROT:
902 			CPU_STATS_ADD_K(vm, prot_fault, 1);
903 			break;
904 
905 		case F_INVAL:
906 			CPU_STATS_ENTER_K();
907 			CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
908 			if (as == &kas)
909 				CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
910 			CPU_STATS_EXIT_K();
911 			break;
912 		}
913 	}
914 
915 	/* Kernel probe */
916 	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
917 		tnf_opaque,	address,	addr,
918 		tnf_fault_type,	fault_type,	type,
919 		tnf_seg_access,	access,		rw);
920 
921 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
922 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
923 		(size_t)raddr;
924 
925 	/*
926 	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
927 	 * correctness, but then we could be stuck holding this lock for
928 	 * a LONG time if the fault needs to be resolved on a slow
929 	 * filesystem, and then no-one will be able to exec new commands,
930 	 * as exec'ing requires the write lock on the as.
931 	 */
932 	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
933 	    raddr + size < segkmap->s_base + segkmap->s_size) {
934 		/*
935 		 * if (as==&kas), this can't be XHAT: we've already returned
936 		 * FC_NOSUPPORT.
937 		 */
938 		seg = segkmap;
939 		as_lock_held = 0;
940 	} else {
941 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
942 		if (is_xhat && avl_numnodes(&as->a_wpage) != 0) {
943 			/*
944 			 * Grab and hold the writers' lock on the as
945 			 * if the fault is to a watched page.
946 			 * This will keep CPUs from "peeking" at the
947 			 * address range while we're temporarily boosting
948 			 * the permissions for the XHAT device to
949 			 * resolve the fault in the segment layer.
950 			 *
951 			 * We could check whether faulted address
952 			 * is within a watched page and only then grab
953 			 * the writer lock, but this is simpler.
954 			 */
955 			AS_LOCK_EXIT(as, &as->a_lock);
956 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
957 		}
958 
959 		seg = as_segat(as, raddr);
960 		if (seg == NULL) {
961 			AS_LOCK_EXIT(as, &as->a_lock);
962 			if ((lwp != NULL) && (!is_xhat)) {
963 				lwp->lwp_nostop--;
964 				lwp->lwp_nostop_r--;
965 			}
966 			return (FC_NOMAP);
967 		}
968 
969 		as_lock_held = 1;
970 	}
971 
972 	addrsav = raddr;
973 	segsav = seg;
974 
975 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
976 		if (raddr >= seg->s_base + seg->s_size) {
977 			seg = AS_SEGNEXT(as, seg);
978 			if (seg == NULL || raddr != seg->s_base) {
979 				res = FC_NOMAP;
980 				break;
981 			}
982 		}
983 		if (raddr + rsize > seg->s_base + seg->s_size)
984 			ssize = seg->s_base + seg->s_size - raddr;
985 		else
986 			ssize = rsize;
987 
988 		if (!is_xhat || (seg->s_ops != &segdev_ops)) {
989 
990 			if (is_xhat && avl_numnodes(&as->a_wpage) != 0 &&
991 			    pr_is_watchpage_as(raddr, rw, as)) {
992 				/*
993 				 * Handle watch pages.  If we're faulting on a
994 				 * watched page from an X-hat, we have to
995 				 * restore the original permissions while we
996 				 * handle the fault.
997 				 */
998 				as_clearwatch(as);
999 				holding_wpage = 1;
1000 			}
1001 
1002 			res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
1003 
1004 			/* Restore watchpoints */
1005 			if (holding_wpage) {
1006 				as_setwatch(as);
1007 				holding_wpage = 0;
1008 			}
1009 
1010 			if (res != 0)
1011 				break;
1012 		} else {
1013 			/* XHAT does not support seg_dev */
1014 			res = FC_NOSUPPORT;
1015 			break;
1016 		}
1017 	}
1018 
1019 	/*
1020 	 * If we were SOFTLOCKing and encountered a failure,
1021 	 * we must SOFTUNLOCK the range we already did. (Maybe we
1022 	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
1023 	 * right here...)
1024 	 */
1025 	if (res != 0 && type == F_SOFTLOCK) {
1026 		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
1027 			if (addrsav >= seg->s_base + seg->s_size)
1028 				seg = AS_SEGNEXT(as, seg);
1029 			ASSERT(seg != NULL);
1030 			/*
1031 			 * Now call the fault routine again to perform the
1032 			 * unlock using S_OTHER instead of the rw variable
1033 			 * since we never got a chance to touch the pages.
1034 			 */
1035 			if (raddr > seg->s_base + seg->s_size)
1036 				ssize = seg->s_base + seg->s_size - addrsav;
1037 			else
1038 				ssize = raddr - addrsav;
1039 			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
1040 			    F_SOFTUNLOCK, S_OTHER);
1041 		}
1042 	}
1043 	if (as_lock_held)
1044 		AS_LOCK_EXIT(as, &as->a_lock);
1045 	if ((lwp != NULL) && (!is_xhat)) {
1046 		lwp->lwp_nostop--;
1047 		lwp->lwp_nostop_r--;
1048 	}
1049 	/*
1050 	 * If the lower levels returned EDEADLK for a fault,
1051 	 * It means that we should retry the fault.  Let's wait
1052 	 * a bit also to let the deadlock causing condition clear.
1053 	 * This is part of a gross hack to work around a design flaw
1054 	 * in the ufs/sds logging code and should go away when the
1055 	 * logging code is re-designed to fix the problem. See bug
1056 	 * 4125102 for details of the problem.
1057 	 */
1058 	if (FC_ERRNO(res) == EDEADLK) {
1059 		delay(deadlk_wait);
1060 		res = 0;
1061 		goto retry;
1062 	}
1063 	return (res);
1064 }
1065 
1066 
1067 
1068 /*
1069  * Asynchronous ``fault'' at addr for size bytes.
1070  */
1071 faultcode_t
1072 as_faulta(struct as *as, caddr_t addr, size_t size)
1073 {
1074 	struct seg *seg;
1075 	caddr_t raddr;			/* rounded down addr */
1076 	size_t rsize;			/* rounded up size */
1077 	faultcode_t res = 0;
1078 	klwp_t *lwp = ttolwp(curthread);
1079 
1080 retry:
1081 	/*
1082 	 * Indicate that the lwp is not to be stopped while waiting
1083 	 * for a pagefault.  This is to avoid deadlock while debugging
1084 	 * a process via /proc over NFS (in particular).
1085 	 */
1086 	if (lwp != NULL) {
1087 		lwp->lwp_nostop++;
1088 		lwp->lwp_nostop_r++;
1089 	}
1090 
1091 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1092 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1093 		(size_t)raddr;
1094 
1095 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1096 	seg = as_segat(as, raddr);
1097 	if (seg == NULL) {
1098 		AS_LOCK_EXIT(as, &as->a_lock);
1099 		if (lwp != NULL) {
1100 			lwp->lwp_nostop--;
1101 			lwp->lwp_nostop_r--;
1102 		}
1103 		return (FC_NOMAP);
1104 	}
1105 
1106 	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
1107 		if (raddr >= seg->s_base + seg->s_size) {
1108 			seg = AS_SEGNEXT(as, seg);
1109 			if (seg == NULL || raddr != seg->s_base) {
1110 				res = FC_NOMAP;
1111 				break;
1112 			}
1113 		}
1114 		res = SEGOP_FAULTA(seg, raddr);
1115 		if (res != 0)
1116 			break;
1117 	}
1118 	AS_LOCK_EXIT(as, &as->a_lock);
1119 	if (lwp != NULL) {
1120 		lwp->lwp_nostop--;
1121 		lwp->lwp_nostop_r--;
1122 	}
1123 	/*
1124 	 * If the lower levels returned EDEADLK for a fault,
1125 	 * It means that we should retry the fault.  Let's wait
1126 	 * a bit also to let the deadlock causing condition clear.
1127 	 * This is part of a gross hack to work around a design flaw
1128 	 * in the ufs/sds logging code and should go away when the
1129 	 * logging code is re-designed to fix the problem. See bug
1130 	 * 4125102 for details of the problem.
1131 	 */
1132 	if (FC_ERRNO(res) == EDEADLK) {
1133 		delay(deadlk_wait);
1134 		res = 0;
1135 		goto retry;
1136 	}
1137 	return (res);
1138 }
1139 
1140 /*
1141  * Set the virtual mapping for the interval from [addr : addr + size)
1142  * in address space `as' to have the specified protection.
1143  * It is ok for the range to cross over several segments,
1144  * as long as they are contiguous.
1145  */
1146 int
1147 as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1148 {
1149 	struct seg *seg;
1150 	struct as_callback *cb;
1151 	size_t ssize;
1152 	caddr_t raddr;			/* rounded down addr */
1153 	size_t rsize;			/* rounded up size */
1154 	int error = 0, writer = 0;
1155 	caddr_t saveraddr;
1156 	size_t saversize;
1157 
1158 setprot_top:
1159 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1160 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1161 		(size_t)raddr;
1162 
1163 	if (raddr + rsize < raddr)		/* check for wraparound */
1164 		return (ENOMEM);
1165 
1166 	saveraddr = raddr;
1167 	saversize = rsize;
1168 
1169 	/*
1170 	 * Normally we only lock the as as a reader. But
1171 	 * if due to setprot the segment driver needs to split
1172 	 * a segment it will return IE_RETRY. Therefore we re-aquire
1173 	 * the as lock as a writer so the segment driver can change
1174 	 * the seg list. Also the segment driver will return IE_RETRY
1175 	 * after it has changed the segment list so we therefore keep
1176 	 * locking as a writer. Since these opeartions should be rare
1177 	 * want to only lock as a writer when necessary.
1178 	 */
1179 	if (writer || avl_numnodes(&as->a_wpage) != 0) {
1180 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1181 	} else {
1182 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1183 	}
1184 
1185 	as_clearwatchprot(as, raddr, rsize);
1186 	seg = as_segat(as, raddr);
1187 	if (seg == NULL) {
1188 		as_setwatch(as);
1189 		AS_LOCK_EXIT(as, &as->a_lock);
1190 		return (ENOMEM);
1191 	}
1192 
1193 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1194 		if (raddr >= seg->s_base + seg->s_size) {
1195 			seg = AS_SEGNEXT(as, seg);
1196 			if (seg == NULL || raddr != seg->s_base) {
1197 				error = ENOMEM;
1198 				break;
1199 			}
1200 		}
1201 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1202 			ssize = seg->s_base + seg->s_size - raddr;
1203 		else
1204 			ssize = rsize;
1205 		error = SEGOP_SETPROT(seg, raddr, ssize, prot);
1206 
1207 		if (error == IE_NOMEM) {
1208 			error = EAGAIN;
1209 			break;
1210 		}
1211 
1212 		if (error == IE_RETRY) {
1213 			AS_LOCK_EXIT(as, &as->a_lock);
1214 			writer = 1;
1215 			goto setprot_top;
1216 		}
1217 
1218 		if (error == EAGAIN) {
1219 			/*
1220 			 * Make sure we have a_lock as writer.
1221 			 */
1222 			if (writer == 0) {
1223 				AS_LOCK_EXIT(as, &as->a_lock);
1224 				writer = 1;
1225 				goto setprot_top;
1226 			}
1227 
1228 			/*
1229 			 * Memory is currently locked.  It must be unlocked
1230 			 * before this operation can succeed through a retry.
1231 			 * The possible reasons for locked memory and
1232 			 * corresponding strategies for unlocking are:
1233 			 * (1) Normal I/O
1234 			 *	wait for a signal that the I/O operation
1235 			 *	has completed and the memory is unlocked.
1236 			 * (2) Asynchronous I/O
1237 			 *	The aio subsystem does not unlock pages when
1238 			 *	the I/O is completed. Those pages are unlocked
1239 			 *	when the application calls aiowait/aioerror.
1240 			 *	So, to prevent blocking forever, cv_broadcast()
1241 			 *	is done to wake up aio_cleanup_thread.
1242 			 *	Subsequently, segvn_reclaim will be called, and
1243 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1244 			 * (3) Long term page locking:
1245 			 *	Drivers intending to have pages locked for a
1246 			 *	period considerably longer than for normal I/O
1247 			 *	(essentially forever) may have registered for a
1248 			 *	callback so they may unlock these pages on
1249 			 *	request. This is needed to allow this operation
1250 			 *	to succeed. Each entry on the callback list is
1251 			 *	examined. If the event or address range pertains
1252 			 *	the callback is invoked (unless it already is in
1253 			 *	progress). The a_contents lock must be dropped
1254 			 *	before the callback, so only one callback can
1255 			 *	be done at a time. Go to the top and do more
1256 			 *	until zero is returned. If zero is returned,
1257 			 *	either there were no callbacks for this event
1258 			 *	or they were already in progress.
1259 			 */
1260 			mutex_enter(&as->a_contents);
1261 			if (as->a_callbacks &&
1262 				(cb = as_find_callback(as, AS_SETPROT_EVENT,
1263 						seg->s_base, seg->s_size))) {
1264 				AS_LOCK_EXIT(as, &as->a_lock);
1265 				as_execute_callback(as, cb, AS_SETPROT_EVENT);
1266 			} else {
1267 				if (AS_ISUNMAPWAIT(as) == 0)
1268 					cv_broadcast(&as->a_cv);
1269 				AS_SETUNMAPWAIT(as);
1270 				AS_LOCK_EXIT(as, &as->a_lock);
1271 				while (AS_ISUNMAPWAIT(as))
1272 					cv_wait(&as->a_cv, &as->a_contents);
1273 			}
1274 			mutex_exit(&as->a_contents);
1275 			goto setprot_top;
1276 		} else if (error != 0)
1277 			break;
1278 	}
1279 	if (error != 0) {
1280 		as_setwatch(as);
1281 	} else {
1282 		as_setwatchprot(as, saveraddr, saversize, prot);
1283 	}
1284 	AS_LOCK_EXIT(as, &as->a_lock);
1285 	return (error);
1286 }
1287 
1288 /*
1289  * Check to make sure that the interval [addr, addr + size)
1290  * in address space `as' has at least the specified protection.
1291  * It is ok for the range to cross over several segments, as long
1292  * as they are contiguous.
1293  */
1294 int
1295 as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1296 {
1297 	struct seg *seg;
1298 	size_t ssize;
1299 	caddr_t raddr;			/* rounded down addr */
1300 	size_t rsize;			/* rounded up size */
1301 	int error = 0;
1302 
1303 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1304 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1305 		(size_t)raddr;
1306 
1307 	if (raddr + rsize < raddr)		/* check for wraparound */
1308 		return (ENOMEM);
1309 
1310 	/*
1311 	 * This is ugly as sin...
1312 	 * Normally, we only acquire the address space readers lock.
1313 	 * However, if the address space has watchpoints present,
1314 	 * we must acquire the writer lock on the address space for
1315 	 * the benefit of as_clearwatchprot() and as_setwatchprot().
1316 	 */
1317 	if (avl_numnodes(&as->a_wpage) != 0)
1318 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1319 	else
1320 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1321 	as_clearwatchprot(as, raddr, rsize);
1322 	seg = as_segat(as, raddr);
1323 	if (seg == NULL) {
1324 		as_setwatch(as);
1325 		AS_LOCK_EXIT(as, &as->a_lock);
1326 		return (ENOMEM);
1327 	}
1328 
1329 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1330 		if (raddr >= seg->s_base + seg->s_size) {
1331 			seg = AS_SEGNEXT(as, seg);
1332 			if (seg == NULL || raddr != seg->s_base) {
1333 				error = ENOMEM;
1334 				break;
1335 			}
1336 		}
1337 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1338 			ssize = seg->s_base + seg->s_size - raddr;
1339 		else
1340 			ssize = rsize;
1341 
1342 		error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
1343 		if (error != 0)
1344 			break;
1345 	}
1346 	as_setwatch(as);
1347 	AS_LOCK_EXIT(as, &as->a_lock);
1348 	return (error);
1349 }
1350 
1351 int
1352 as_unmap(struct as *as, caddr_t addr, size_t size)
1353 {
1354 	struct seg *seg, *seg_next;
1355 	struct as_callback *cb;
1356 	caddr_t raddr, eaddr;
1357 	size_t ssize;
1358 	int err;
1359 
1360 top:
1361 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1362 	eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) &
1363 	    (uintptr_t)PAGEMASK);
1364 
1365 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1366 
1367 	as->a_updatedir = 1;	/* inform /proc */
1368 	gethrestime(&as->a_updatetime);
1369 
1370 	/*
1371 	 * Use as_findseg to find the first segment in the range, then
1372 	 * step through the segments in order, following s_next.
1373 	 */
1374 	as_clearwatchprot(as, raddr, eaddr - raddr);
1375 
1376 	for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
1377 		if (eaddr <= seg->s_base)
1378 			break;		/* eaddr was in a gap; all done */
1379 
1380 		/* this is implied by the test above */
1381 		ASSERT(raddr < eaddr);
1382 
1383 		if (raddr < seg->s_base)
1384 			raddr = seg->s_base; 	/* raddr was in a gap */
1385 
1386 		if (eaddr > (seg->s_base + seg->s_size))
1387 			ssize = seg->s_base + seg->s_size - raddr;
1388 		else
1389 			ssize = eaddr - raddr;
1390 
1391 		/*
1392 		 * Save next segment pointer since seg can be
1393 		 * destroyed during the segment unmap operation.
1394 		 */
1395 		seg_next = AS_SEGNEXT(as, seg);
1396 
1397 		err = SEGOP_UNMAP(seg, raddr, ssize);
1398 		if (err == EAGAIN) {
1399 			/*
1400 			 * Memory is currently locked.  It must be unlocked
1401 			 * before this operation can succeed through a retry.
1402 			 * The possible reasons for locked memory and
1403 			 * corresponding strategies for unlocking are:
1404 			 * (1) Normal I/O
1405 			 *	wait for a signal that the I/O operation
1406 			 *	has completed and the memory is unlocked.
1407 			 * (2) Asynchronous I/O
1408 			 *	The aio subsystem does not unlock pages when
1409 			 *	the I/O is completed. Those pages are unlocked
1410 			 *	when the application calls aiowait/aioerror.
1411 			 *	So, to prevent blocking forever, cv_broadcast()
1412 			 *	is done to wake up aio_cleanup_thread.
1413 			 *	Subsequently, segvn_reclaim will be called, and
1414 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1415 			 * (3) Long term page locking:
1416 			 *	Drivers intending to have pages locked for a
1417 			 *	period considerably longer than for normal I/O
1418 			 *	(essentially forever) may have registered for a
1419 			 *	callback so they may unlock these pages on
1420 			 *	request. This is needed to allow this operation
1421 			 *	to succeed. Each entry on the callback list is
1422 			 *	examined. If the event or address range pertains
1423 			 *	the callback is invoked (unless it already is in
1424 			 *	progress). The a_contents lock must be dropped
1425 			 *	before the callback, so only one callback can
1426 			 *	be done at a time. Go to the top and do more
1427 			 *	until zero is returned. If zero is returned,
1428 			 *	either there were no callbacks for this event
1429 			 *	or they were already in progress.
1430 			 */
1431 			as_setwatch(as);
1432 			mutex_enter(&as->a_contents);
1433 			if (as->a_callbacks &&
1434 				(cb = as_find_callback(as, AS_UNMAP_EVENT,
1435 						seg->s_base, seg->s_size))) {
1436 				AS_LOCK_EXIT(as, &as->a_lock);
1437 				as_execute_callback(as, cb, AS_UNMAP_EVENT);
1438 			} else {
1439 				if (AS_ISUNMAPWAIT(as) == 0)
1440 					cv_broadcast(&as->a_cv);
1441 				AS_SETUNMAPWAIT(as);
1442 				AS_LOCK_EXIT(as, &as->a_lock);
1443 				while (AS_ISUNMAPWAIT(as))
1444 					cv_wait(&as->a_cv, &as->a_contents);
1445 			}
1446 			mutex_exit(&as->a_contents);
1447 			goto top;
1448 		} else if (err == IE_RETRY) {
1449 			as_setwatch(as);
1450 			AS_LOCK_EXIT(as, &as->a_lock);
1451 			goto top;
1452 		} else if (err) {
1453 			as_setwatch(as);
1454 			AS_LOCK_EXIT(as, &as->a_lock);
1455 			return (-1);
1456 		}
1457 
1458 		as->a_size -= ssize;
1459 		raddr += ssize;
1460 	}
1461 	AS_LOCK_EXIT(as, &as->a_lock);
1462 	return (0);
1463 }
1464 
1465 static int
1466 as_map_vnsegs(struct as *as, caddr_t addr, size_t size,
1467     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1468 {
1469 	int text = vn_a->flags & MAP_TEXT;
1470 	uint_t szcvec = map_execseg_pgszcvec(text, addr, size);
1471 	uint_t szc;
1472 	uint_t nszc;
1473 	int error;
1474 	caddr_t a;
1475 	caddr_t eaddr;
1476 	size_t segsize;
1477 	struct seg *seg;
1478 	uint_t save_szcvec;
1479 	size_t pgsz;
1480 	struct vattr va;
1481 	u_offset_t eoff;
1482 	size_t save_size = 0;
1483 
1484 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1485 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1486 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1487 	ASSERT(vn_a->vp != NULL);
1488 	ASSERT(vn_a->amp == NULL);
1489 
1490 again:
1491 	if (szcvec <= 1) {
1492 		seg = seg_alloc(as, addr, size);
1493 		if (seg == NULL) {
1494 			return (ENOMEM);
1495 		}
1496 		vn_a->szc = 0;
1497 		error = (*crfp)(seg, vn_a);
1498 		if (error != 0) {
1499 			seg_free(seg);
1500 		}
1501 		return (error);
1502 	}
1503 
1504 	va.va_mask = AT_SIZE;
1505 	if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred) != 0) {
1506 		szcvec = 0;
1507 		goto again;
1508 	}
1509 	eoff = vn_a->offset & PAGEMASK;
1510 	if (eoff >= va.va_size) {
1511 		szcvec = 0;
1512 		goto again;
1513 	}
1514 	eoff += size;
1515 	if (btopr(va.va_size) < btopr(eoff)) {
1516 		save_size = size;
1517 		size = va.va_size - (vn_a->offset & PAGEMASK);
1518 		size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t);
1519 		szcvec = map_execseg_pgszcvec(text, addr, size);
1520 		if (szcvec <= 1) {
1521 			size = save_size;
1522 			goto again;
1523 		}
1524 	}
1525 
1526 	eaddr = addr + size;
1527 	save_szcvec = szcvec;
1528 	szcvec >>= 1;
1529 	szc = 0;
1530 	nszc = 0;
1531 	while (szcvec) {
1532 		if ((szcvec & 0x1) == 0) {
1533 			nszc++;
1534 			szcvec >>= 1;
1535 			continue;
1536 		}
1537 		nszc++;
1538 		pgsz = page_get_pagesize(nszc);
1539 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
1540 		if (a != addr) {
1541 			ASSERT(a < eaddr);
1542 			segsize = a - addr;
1543 			seg = seg_alloc(as, addr, segsize);
1544 			if (seg == NULL) {
1545 				return (ENOMEM);
1546 			}
1547 			vn_a->szc = szc;
1548 			error = (*crfp)(seg, vn_a);
1549 			if (error != 0) {
1550 				seg_free(seg);
1551 				return (error);
1552 			}
1553 			*segcreated = 1;
1554 			vn_a->offset += segsize;
1555 			addr = a;
1556 		}
1557 		szc = nszc;
1558 		szcvec >>= 1;
1559 	}
1560 
1561 	ASSERT(addr < eaddr);
1562 	szcvec = save_szcvec | 1; /* add 8K pages */
1563 	while (szcvec) {
1564 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
1565 		ASSERT(a >= addr);
1566 		if (a != addr) {
1567 			segsize = a - addr;
1568 			seg = seg_alloc(as, addr, segsize);
1569 			if (seg == NULL) {
1570 				return (ENOMEM);
1571 			}
1572 			vn_a->szc = szc;
1573 			error = (*crfp)(seg, vn_a);
1574 			if (error != 0) {
1575 				seg_free(seg);
1576 				return (error);
1577 			}
1578 			*segcreated = 1;
1579 			vn_a->offset += segsize;
1580 			addr = a;
1581 		}
1582 		szcvec &= ~(1 << szc);
1583 		if (szcvec) {
1584 			szc = highbit(szcvec) - 1;
1585 			pgsz = page_get_pagesize(szc);
1586 		}
1587 	}
1588 	ASSERT(addr == eaddr);
1589 
1590 	if (save_size) {
1591 		size = save_size - size;
1592 		goto again;
1593 	}
1594 
1595 	return (0);
1596 }
1597 
1598 int
1599 as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp)
1600 {
1601 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1602 	return (as_map_locked(as, addr, size, crfp, argsp));
1603 }
1604 
1605 int
1606 as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(),
1607 		void *argsp)
1608 {
1609 	struct seg *seg = NULL;
1610 	caddr_t raddr;			/* rounded down addr */
1611 	size_t rsize;			/* rounded up size */
1612 	int error;
1613 	struct proc *p = curproc;
1614 
1615 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1616 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1617 		(size_t)raddr;
1618 
1619 	/*
1620 	 * check for wrap around
1621 	 */
1622 	if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
1623 		AS_LOCK_EXIT(as, &as->a_lock);
1624 		return (ENOMEM);
1625 	}
1626 
1627 	as->a_updatedir = 1;	/* inform /proc */
1628 	gethrestime(&as->a_updatetime);
1629 
1630 	if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) {
1631 		AS_LOCK_EXIT(as, &as->a_lock);
1632 
1633 		(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p,
1634 		    RCA_UNSAFE_ALL);
1635 
1636 		return (ENOMEM);
1637 	}
1638 
1639 	if (AS_MAP_VNSEGS_USELPGS(crfp, argsp)) {
1640 		int unmap = 0;
1641 		error = as_map_vnsegs(as, raddr, rsize, crfp,
1642 		    (struct segvn_crargs *)argsp, &unmap);
1643 		if (error != 0) {
1644 			AS_LOCK_EXIT(as, &as->a_lock);
1645 			if (unmap) {
1646 				(void) as_unmap(as, addr, size);
1647 			}
1648 			return (error);
1649 		}
1650 	} else {
1651 		seg = seg_alloc(as, addr, size);
1652 		if (seg == NULL) {
1653 			AS_LOCK_EXIT(as, &as->a_lock);
1654 			return (ENOMEM);
1655 		}
1656 
1657 		error = (*crfp)(seg, argsp);
1658 		if (error != 0) {
1659 			seg_free(seg);
1660 			AS_LOCK_EXIT(as, &as->a_lock);
1661 			return (error);
1662 		}
1663 	}
1664 
1665 	/*
1666 	 * Add size now so as_unmap will work if as_ctl fails.
1667 	 */
1668 	as->a_size += rsize;
1669 
1670 	as_setwatch(as);
1671 
1672 	/*
1673 	 * If the address space is locked,
1674 	 * establish memory locks for the new segment.
1675 	 */
1676 	mutex_enter(&as->a_contents);
1677 	if (AS_ISPGLCK(as)) {
1678 		mutex_exit(&as->a_contents);
1679 		AS_LOCK_EXIT(as, &as->a_lock);
1680 		error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
1681 		if (error != 0)
1682 			(void) as_unmap(as, addr, size);
1683 	} else {
1684 		mutex_exit(&as->a_contents);
1685 		AS_LOCK_EXIT(as, &as->a_lock);
1686 	}
1687 	return (error);
1688 }
1689 
1690 
1691 /*
1692  * Delete all segments in the address space marked with S_PURGE.
1693  * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c).
1694  * These segments are deleted as a first step before calls to as_gap(), so
1695  * that they don't affect mmap() or shmat().
1696  */
1697 void
1698 as_purge(struct as *as)
1699 {
1700 	struct seg *seg;
1701 	struct seg *next_seg;
1702 
1703 	/*
1704 	 * the setting of NEEDSPURGE is protect by as_rangelock(), so
1705 	 * no need to grab a_contents mutex for this check
1706 	 */
1707 	if ((as->a_flags & AS_NEEDSPURGE) == 0)
1708 		return;
1709 
1710 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1711 	next_seg = NULL;
1712 	seg = AS_SEGFIRST(as);
1713 	while (seg != NULL) {
1714 		next_seg = AS_SEGNEXT(as, seg);
1715 		if (seg->s_flags & S_PURGE)
1716 			SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
1717 		seg = next_seg;
1718 	}
1719 	AS_LOCK_EXIT(as, &as->a_lock);
1720 
1721 	mutex_enter(&as->a_contents);
1722 	as->a_flags &= ~AS_NEEDSPURGE;
1723 	mutex_exit(&as->a_contents);
1724 }
1725 
1726 /*
1727  * Find a hole of at least size minlen within [base, base + len).
1728  *
1729  * If flags specifies AH_HI, the hole will have the highest possible address
1730  * in the range.  We use the as->a_lastgap field to figure out where to
1731  * start looking for a gap.
1732  *
1733  * Otherwise, the gap will have the lowest possible address.
1734  *
1735  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
1736  *
1737  * If an adequate hole is found, base and len are set to reflect the part of
1738  * the hole that is within range, and 0 is returned, otherwise,
1739  * -1 is returned.
1740  *
1741  * NOTE: This routine is not correct when base+len overflows caddr_t.
1742  */
1743 int
1744 as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags,
1745     caddr_t addr)
1746 {
1747 	caddr_t lobound = *basep;
1748 	caddr_t hibound = lobound + *lenp;
1749 	struct seg *lseg, *hseg;
1750 	caddr_t lo, hi;
1751 	int forward;
1752 	caddr_t save_base;
1753 	size_t save_len;
1754 
1755 	save_base = *basep;
1756 	save_len = *lenp;
1757 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1758 	if (AS_SEGFIRST(as) == NULL) {
1759 		if (valid_va_range(basep, lenp, minlen, flags & AH_DIR)) {
1760 			AS_LOCK_EXIT(as, &as->a_lock);
1761 			return (0);
1762 		} else {
1763 			AS_LOCK_EXIT(as, &as->a_lock);
1764 			*basep = save_base;
1765 			*lenp = save_len;
1766 			return (-1);
1767 		}
1768 	}
1769 
1770 	/*
1771 	 * Set up to iterate over all the inter-segment holes in the given
1772 	 * direction.  lseg is NULL for the lowest-addressed hole and hseg is
1773 	 * NULL for the highest-addressed hole.  If moving backwards, we reset
1774 	 * sseg to denote the highest-addressed segment.
1775 	 */
1776 	forward = (flags & AH_DIR) == AH_LO;
1777 	if (forward) {
1778 		hseg = as_findseg(as, lobound, 1);
1779 		lseg = AS_SEGPREV(as, hseg);
1780 	} else {
1781 
1782 		/*
1783 		 * If allocating at least as much as the last allocation,
1784 		 * use a_lastgap's base as a better estimate of hibound.
1785 		 */
1786 		if (as->a_lastgap &&
1787 		    minlen >= as->a_lastgap->s_size &&
1788 		    hibound >= as->a_lastgap->s_base)
1789 			hibound = as->a_lastgap->s_base;
1790 
1791 		hseg = as_findseg(as, hibound, 1);
1792 		if (hseg->s_base + hseg->s_size < hibound) {
1793 			lseg = hseg;
1794 			hseg = NULL;
1795 		} else {
1796 			lseg = AS_SEGPREV(as, hseg);
1797 		}
1798 	}
1799 
1800 	for (;;) {
1801 		/*
1802 		 * Set lo and hi to the hole's boundaries.  (We should really
1803 		 * use MAXADDR in place of hibound in the expression below,
1804 		 * but can't express it easily; using hibound in its place is
1805 		 * harmless.)
1806 		 */
1807 		lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
1808 		hi = (hseg == NULL) ? hibound : hseg->s_base;
1809 		/*
1810 		 * If the iteration has moved past the interval from lobound
1811 		 * to hibound it's pointless to continue.
1812 		 */
1813 		if ((forward && lo > hibound) || (!forward && hi < lobound))
1814 			break;
1815 		else if (lo > hibound || hi < lobound)
1816 			goto cont;
1817 		/*
1818 		 * Candidate hole lies at least partially within the allowable
1819 		 * range.  Restrict it to fall completely within that range,
1820 		 * i.e., to [max(lo, lobound), min(hi, hibound)].
1821 		 */
1822 		if (lo < lobound)
1823 			lo = lobound;
1824 		if (hi > hibound)
1825 			hi = hibound;
1826 		/*
1827 		 * Verify that the candidate hole is big enough and meets
1828 		 * hardware constraints.
1829 		 */
1830 		*basep = lo;
1831 		*lenp = hi - lo;
1832 		if (valid_va_range(basep, lenp, minlen,
1833 		    forward ? AH_LO : AH_HI) &&
1834 		    ((flags & AH_CONTAIN) == 0 ||
1835 		    (*basep <= addr && *basep + *lenp > addr))) {
1836 			if (!forward)
1837 				as->a_lastgap = hseg;
1838 			if (hseg != NULL)
1839 				as->a_lastgaphl = hseg;
1840 			else
1841 				as->a_lastgaphl = lseg;
1842 			AS_LOCK_EXIT(as, &as->a_lock);
1843 			return (0);
1844 		}
1845 	cont:
1846 		/*
1847 		 * Move to the next hole.
1848 		 */
1849 		if (forward) {
1850 			lseg = hseg;
1851 			if (lseg == NULL)
1852 				break;
1853 			hseg = AS_SEGNEXT(as, hseg);
1854 		} else {
1855 			hseg = lseg;
1856 			if (hseg == NULL)
1857 				break;
1858 			lseg = AS_SEGPREV(as, lseg);
1859 		}
1860 	}
1861 	*basep = save_base;
1862 	*lenp = save_len;
1863 	AS_LOCK_EXIT(as, &as->a_lock);
1864 	return (-1);
1865 }
1866 
1867 /*
1868  * Return the next range within [base, base + len) that is backed
1869  * with "real memory".  Skip holes and non-seg_vn segments.
1870  * We're lazy and only return one segment at a time.
1871  */
1872 int
1873 as_memory(struct as *as, caddr_t *basep, size_t *lenp)
1874 {
1875 	extern struct seg_ops segspt_shmops;	/* needs a header file */
1876 	struct seg *seg;
1877 	caddr_t addr, eaddr;
1878 	caddr_t segend;
1879 
1880 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1881 
1882 	addr = *basep;
1883 	eaddr = addr + *lenp;
1884 
1885 	seg = as_findseg(as, addr, 0);
1886 	if (seg != NULL)
1887 		addr = MAX(seg->s_base, addr);
1888 
1889 	for (;;) {
1890 		if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
1891 			AS_LOCK_EXIT(as, &as->a_lock);
1892 			return (EINVAL);
1893 		}
1894 
1895 		if (seg->s_ops == &segvn_ops) {
1896 			segend = seg->s_base + seg->s_size;
1897 			break;
1898 		}
1899 
1900 		/*
1901 		 * We do ISM by looking into the private data
1902 		 * to determine the real size of the segment.
1903 		 */
1904 		if (seg->s_ops == &segspt_shmops) {
1905 			segend = seg->s_base + spt_realsize(seg);
1906 			if (addr < segend)
1907 				break;
1908 		}
1909 
1910 		seg = AS_SEGNEXT(as, seg);
1911 
1912 		if (seg != NULL)
1913 			addr = seg->s_base;
1914 	}
1915 
1916 	*basep = addr;
1917 
1918 	if (segend > eaddr)
1919 		*lenp = eaddr - addr;
1920 	else
1921 		*lenp = segend - addr;
1922 
1923 	AS_LOCK_EXIT(as, &as->a_lock);
1924 	return (0);
1925 }
1926 
1927 /*
1928  * Swap the pages associated with the address space as out to
1929  * secondary storage, returning the number of bytes actually
1930  * swapped.
1931  *
1932  * The value returned is intended to correlate well with the process's
1933  * memory requirements.  Its usefulness for this purpose depends on
1934  * how well the segment-level routines do at returning accurate
1935  * information.
1936  */
1937 size_t
1938 as_swapout(struct as *as)
1939 {
1940 	struct seg *seg;
1941 	size_t swpcnt = 0;
1942 
1943 	/*
1944 	 * Kernel-only processes have given up their address
1945 	 * spaces.  Of course, we shouldn't be attempting to
1946 	 * swap out such processes in the first place...
1947 	 */
1948 	if (as == NULL)
1949 		return (0);
1950 
1951 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1952 
1953 	/* Prevent XHATs from attaching */
1954 	mutex_enter(&as->a_contents);
1955 	AS_SETBUSY(as);
1956 	mutex_exit(&as->a_contents);
1957 
1958 
1959 	/*
1960 	 * Free all mapping resources associated with the address
1961 	 * space.  The segment-level swapout routines capitalize
1962 	 * on this unmapping by scavanging pages that have become
1963 	 * unmapped here.
1964 	 */
1965 	hat_swapout(as->a_hat);
1966 	if (as->a_xhat != NULL)
1967 		xhat_swapout_all(as);
1968 
1969 	mutex_enter(&as->a_contents);
1970 	AS_CLRBUSY(as);
1971 	mutex_exit(&as->a_contents);
1972 
1973 	/*
1974 	 * Call the swapout routines of all segments in the address
1975 	 * space to do the actual work, accumulating the amount of
1976 	 * space reclaimed.
1977 	 */
1978 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
1979 		struct seg_ops *ov = seg->s_ops;
1980 
1981 		/*
1982 		 * We have to check to see if the seg has
1983 		 * an ops vector because the seg may have
1984 		 * been in the middle of being set up when
1985 		 * the process was picked for swapout.
1986 		 */
1987 		if ((ov != NULL) && (ov->swapout != NULL))
1988 			swpcnt += SEGOP_SWAPOUT(seg);
1989 	}
1990 	AS_LOCK_EXIT(as, &as->a_lock);
1991 	return (swpcnt);
1992 }
1993 
1994 /*
1995  * Determine whether data from the mappings in interval [addr, addr + size)
1996  * are in the primary memory (core) cache.
1997  */
1998 int
1999 as_incore(struct as *as, caddr_t addr,
2000     size_t size, char *vec, size_t *sizep)
2001 {
2002 	struct seg *seg;
2003 	size_t ssize;
2004 	caddr_t raddr;		/* rounded down addr */
2005 	size_t rsize;		/* rounded up size */
2006 	size_t isize;			/* iteration size */
2007 	int error = 0;		/* result, assume success */
2008 
2009 	*sizep = 0;
2010 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2011 	rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
2012 		(size_t)raddr;
2013 
2014 	if (raddr + rsize < raddr)		/* check for wraparound */
2015 		return (ENOMEM);
2016 
2017 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2018 	seg = as_segat(as, raddr);
2019 	if (seg == NULL) {
2020 		AS_LOCK_EXIT(as, &as->a_lock);
2021 		return (-1);
2022 	}
2023 
2024 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2025 		if (raddr >= seg->s_base + seg->s_size) {
2026 			seg = AS_SEGNEXT(as, seg);
2027 			if (seg == NULL || raddr != seg->s_base) {
2028 				error = -1;
2029 				break;
2030 			}
2031 		}
2032 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2033 			ssize = seg->s_base + seg->s_size - raddr;
2034 		else
2035 			ssize = rsize;
2036 		*sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
2037 		if (isize != ssize) {
2038 			error = -1;
2039 			break;
2040 		}
2041 		vec += btopr(ssize);
2042 	}
2043 	AS_LOCK_EXIT(as, &as->a_lock);
2044 	return (error);
2045 }
2046 
2047 static void
2048 as_segunlock(struct seg *seg, caddr_t addr, int attr,
2049 	ulong_t *bitmap, size_t position, size_t npages)
2050 {
2051 	caddr_t	range_start;
2052 	size_t	pos1 = position;
2053 	size_t	pos2;
2054 	size_t	size;
2055 	size_t  end_pos = npages + position;
2056 
2057 	while (bt_range(bitmap, &pos1, &pos2, end_pos)) {
2058 		size = ptob((pos2 - pos1));
2059 		range_start = (caddr_t)((uintptr_t)addr +
2060 			ptob(pos1 - position));
2061 
2062 		(void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK,
2063 			(ulong_t *)NULL, (size_t)NULL);
2064 		pos1 = pos2;
2065 	}
2066 }
2067 
2068 static void
2069 as_unlockerr(struct as *as, int attr, ulong_t *mlock_map,
2070 	caddr_t raddr, size_t rsize)
2071 {
2072 	struct seg *seg = as_segat(as, raddr);
2073 	size_t ssize;
2074 
2075 	while (rsize != 0) {
2076 		if (raddr >= seg->s_base + seg->s_size)
2077 			seg = AS_SEGNEXT(as, seg);
2078 
2079 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2080 			ssize = seg->s_base + seg->s_size - raddr;
2081 		else
2082 			ssize = rsize;
2083 
2084 		as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize));
2085 
2086 		rsize -= ssize;
2087 		raddr += ssize;
2088 	}
2089 }
2090 
2091 /*
2092  * Cache control operations over the interval [addr, addr + size) in
2093  * address space "as".
2094  */
2095 /*ARGSUSED*/
2096 int
2097 as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
2098     uintptr_t arg, ulong_t *lock_map, size_t pos)
2099 {
2100 	struct seg *seg;	/* working segment */
2101 	caddr_t raddr;		/* rounded down addr */
2102 	caddr_t initraddr;	/* saved initial rounded down addr */
2103 	size_t rsize;		/* rounded up size */
2104 	size_t initrsize;	/* saved initial rounded up size */
2105 	size_t ssize;		/* size of seg */
2106 	int error = 0;			/* result */
2107 	size_t mlock_size;	/* size of bitmap */
2108 	ulong_t *mlock_map;	/* pointer to bitmap used */
2109 				/* to represent the locked */
2110 				/* pages. */
2111 retry:
2112 	if (error == IE_RETRY)
2113 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2114 	else
2115 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2116 
2117 	/*
2118 	 * If these are address space lock/unlock operations, loop over
2119 	 * all segments in the address space, as appropriate.
2120 	 */
2121 	if (func == MC_LOCKAS) {
2122 		size_t npages, idx;
2123 		size_t rlen = 0;	/* rounded as length */
2124 
2125 		idx = pos;
2126 
2127 		if (arg & MCL_FUTURE) {
2128 			mutex_enter(&as->a_contents);
2129 			AS_SETPGLCK(as);
2130 			mutex_exit(&as->a_contents);
2131 		}
2132 		if ((arg & MCL_CURRENT) == 0) {
2133 			AS_LOCK_EXIT(as, &as->a_lock);
2134 			return (0);
2135 		}
2136 
2137 		seg = AS_SEGFIRST(as);
2138 		if (seg == NULL) {
2139 			AS_LOCK_EXIT(as, &as->a_lock);
2140 			return (0);
2141 		}
2142 
2143 		do {
2144 			raddr = (caddr_t)((uintptr_t)seg->s_base &
2145 			    (uintptr_t)PAGEMASK);
2146 			rlen += (((uintptr_t)(seg->s_base + seg->s_size) +
2147 				PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr;
2148 		} while ((seg = AS_SEGNEXT(as, seg)) != NULL);
2149 
2150 		mlock_size = BT_BITOUL(btopr(rlen));
2151 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2152 			sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2153 				AS_LOCK_EXIT(as, &as->a_lock);
2154 				return (EAGAIN);
2155 		}
2156 
2157 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2158 			error = SEGOP_LOCKOP(seg, seg->s_base,
2159 			    seg->s_size, attr, MC_LOCK, mlock_map, pos);
2160 			if (error != 0)
2161 				break;
2162 			pos += seg_pages(seg);
2163 		}
2164 
2165 		if (error) {
2166 			for (seg = AS_SEGFIRST(as); seg != NULL;
2167 				seg = AS_SEGNEXT(as, seg)) {
2168 
2169 				raddr = (caddr_t)((uintptr_t)seg->s_base &
2170 					(uintptr_t)PAGEMASK);
2171 				npages = seg_pages(seg);
2172 				as_segunlock(seg, raddr, attr, mlock_map,
2173 					idx, npages);
2174 				idx += npages;
2175 			}
2176 		}
2177 
2178 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2179 		AS_LOCK_EXIT(as, &as->a_lock);
2180 		goto lockerr;
2181 	} else if (func == MC_UNLOCKAS) {
2182 		mutex_enter(&as->a_contents);
2183 		AS_CLRPGLCK(as);
2184 		mutex_exit(&as->a_contents);
2185 
2186 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2187 			error = SEGOP_LOCKOP(seg, seg->s_base,
2188 			    seg->s_size, attr, MC_UNLOCK, NULL, 0);
2189 			if (error != 0)
2190 				break;
2191 		}
2192 
2193 		AS_LOCK_EXIT(as, &as->a_lock);
2194 		goto lockerr;
2195 	}
2196 
2197 	/*
2198 	 * Normalize addresses and sizes.
2199 	 */
2200 	initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2201 	initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2202 		(size_t)raddr;
2203 
2204 	if (raddr + rsize < raddr) {		/* check for wraparound */
2205 		AS_LOCK_EXIT(as, &as->a_lock);
2206 		return (ENOMEM);
2207 	}
2208 
2209 	/*
2210 	 * Get initial segment.
2211 	 */
2212 	if ((seg = as_segat(as, raddr)) == NULL) {
2213 		AS_LOCK_EXIT(as, &as->a_lock);
2214 		return (ENOMEM);
2215 	}
2216 
2217 	if (func == MC_LOCK) {
2218 		mlock_size = BT_BITOUL(btopr(rsize));
2219 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2220 			sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2221 				AS_LOCK_EXIT(as, &as->a_lock);
2222 				return (EAGAIN);
2223 		}
2224 	}
2225 
2226 	/*
2227 	 * Loop over all segments.  If a hole in the address range is
2228 	 * discovered, then fail.  For each segment, perform the appropriate
2229 	 * control operation.
2230 	 */
2231 	while (rsize != 0) {
2232 
2233 		/*
2234 		 * Make sure there's no hole, calculate the portion
2235 		 * of the next segment to be operated over.
2236 		 */
2237 		if (raddr >= seg->s_base + seg->s_size) {
2238 			seg = AS_SEGNEXT(as, seg);
2239 			if (seg == NULL || raddr != seg->s_base) {
2240 				if (func == MC_LOCK) {
2241 					as_unlockerr(as, attr, mlock_map,
2242 						initraddr, initrsize - rsize);
2243 					kmem_free(mlock_map,
2244 						mlock_size * sizeof (ulong_t));
2245 				}
2246 				AS_LOCK_EXIT(as, &as->a_lock);
2247 				return (ENOMEM);
2248 			}
2249 		}
2250 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2251 			ssize = seg->s_base + seg->s_size - raddr;
2252 		else
2253 			ssize = rsize;
2254 
2255 		/*
2256 		 * Dispatch on specific function.
2257 		 */
2258 		switch (func) {
2259 
2260 		/*
2261 		 * Synchronize cached data from mappings with backing
2262 		 * objects.
2263 		 */
2264 		case MC_SYNC:
2265 			if (error = SEGOP_SYNC(seg, raddr, ssize,
2266 			    attr, (uint_t)arg)) {
2267 				AS_LOCK_EXIT(as, &as->a_lock);
2268 				return (error);
2269 			}
2270 			break;
2271 
2272 		/*
2273 		 * Lock pages in memory.
2274 		 */
2275 		case MC_LOCK:
2276 			if (error = SEGOP_LOCKOP(seg, raddr, ssize,
2277 				attr, func, mlock_map, pos)) {
2278 				as_unlockerr(as, attr, mlock_map, initraddr,
2279 					initrsize - rsize + ssize);
2280 				kmem_free(mlock_map, mlock_size *
2281 					sizeof (ulong_t));
2282 				AS_LOCK_EXIT(as, &as->a_lock);
2283 				goto lockerr;
2284 			}
2285 			break;
2286 
2287 		/*
2288 		 * Unlock mapped pages.
2289 		 */
2290 		case MC_UNLOCK:
2291 			(void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
2292 				(ulong_t *)NULL, (size_t)NULL);
2293 			break;
2294 
2295 		/*
2296 		 * Store VM advise for mapped pages in segment layer.
2297 		 */
2298 		case MC_ADVISE:
2299 			error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg);
2300 
2301 			/*
2302 			 * Check for regular errors and special retry error
2303 			 */
2304 			if (error) {
2305 				if (error == IE_RETRY) {
2306 					/*
2307 					 * Need to acquire writers lock, so
2308 					 * have to drop readers lock and start
2309 					 * all over again
2310 					 */
2311 					AS_LOCK_EXIT(as, &as->a_lock);
2312 					goto retry;
2313 				} else if (error == IE_REATTACH) {
2314 					/*
2315 					 * Find segment for current address
2316 					 * because current segment just got
2317 					 * split or concatenated
2318 					 */
2319 					seg = as_segat(as, raddr);
2320 					if (seg == NULL) {
2321 						AS_LOCK_EXIT(as, &as->a_lock);
2322 						return (ENOMEM);
2323 					}
2324 				} else {
2325 					/*
2326 					 * Regular error
2327 					 */
2328 					AS_LOCK_EXIT(as, &as->a_lock);
2329 					return (error);
2330 				}
2331 			}
2332 			break;
2333 
2334 		/*
2335 		 * Can't happen.
2336 		 */
2337 		default:
2338 			panic("as_ctl: bad operation %d", func);
2339 			/*NOTREACHED*/
2340 		}
2341 
2342 		rsize -= ssize;
2343 		raddr += ssize;
2344 	}
2345 
2346 	if (func == MC_LOCK)
2347 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2348 	AS_LOCK_EXIT(as, &as->a_lock);
2349 	return (0);
2350 lockerr:
2351 
2352 	/*
2353 	 * If the lower levels returned EDEADLK for a segment lockop,
2354 	 * it means that we should retry the operation.  Let's wait
2355 	 * a bit also to let the deadlock causing condition clear.
2356 	 * This is part of a gross hack to work around a design flaw
2357 	 * in the ufs/sds logging code and should go away when the
2358 	 * logging code is re-designed to fix the problem. See bug
2359 	 * 4125102 for details of the problem.
2360 	 */
2361 	if (error == EDEADLK) {
2362 		delay(deadlk_wait);
2363 		error = 0;
2364 		goto retry;
2365 	}
2366 	return (error);
2367 }
2368 
2369 /*
2370  * Special code for exec to move the stack segment from its interim
2371  * place in the old address to the right place in the new address space.
2372  */
2373 /*ARGSUSED*/
2374 int
2375 as_exec(struct as *oas, caddr_t ostka, size_t stksz,
2376     struct as *nas, caddr_t nstka, uint_t hatflag)
2377 {
2378 	struct seg *stkseg;
2379 
2380 	AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER);
2381 	stkseg = as_segat(oas, ostka);
2382 	stkseg = as_removeseg(oas, stkseg);
2383 	ASSERT(stkseg != NULL);
2384 	ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz);
2385 	stkseg->s_as = nas;
2386 	stkseg->s_base = nstka;
2387 
2388 	/*
2389 	 * It's ok to lock the address space we are about to exec to.
2390 	 */
2391 	AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER);
2392 	ASSERT(avl_numnodes(&nas->a_wpage) == 0);
2393 	nas->a_size += stkseg->s_size;
2394 	oas->a_size -= stkseg->s_size;
2395 	(void) as_addseg(nas, stkseg);
2396 	AS_LOCK_EXIT(nas, &nas->a_lock);
2397 	AS_LOCK_EXIT(oas, &oas->a_lock);
2398 	return (0);
2399 }
2400 
2401 static int
2402 f_decode(faultcode_t fault_err)
2403 {
2404 	int error = 0;
2405 
2406 	switch (FC_CODE(fault_err)) {
2407 	case FC_OBJERR:
2408 		error = FC_ERRNO(fault_err);
2409 		break;
2410 	case FC_PROT:
2411 		error = EACCES;
2412 		break;
2413 	default:
2414 		error = EFAULT;
2415 		break;
2416 	}
2417 	return (error);
2418 }
2419 
2420 /*
2421  * lock pages in a given address space. Return shadow list. If
2422  * the list is NULL, the MMU mapping is also locked.
2423  */
2424 int
2425 as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
2426     size_t size, enum seg_rw rw)
2427 {
2428 	size_t rsize;
2429 	caddr_t base;
2430 	caddr_t raddr;
2431 	faultcode_t fault_err;
2432 	struct seg *seg;
2433 	int res;
2434 	int prefaulted = 0;
2435 
2436 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
2437 	    "as_pagelock_start: addr %p size %ld", addr, size);
2438 
2439 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2440 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2441 		(size_t)raddr;
2442 top:
2443 	/*
2444 	 * if the request crosses two segments let
2445 	 * as_fault handle it.
2446 	 */
2447 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2448 	seg = as_findseg(as, addr, 0);
2449 	if ((seg == NULL) || ((base = seg->s_base) > addr) ||
2450 	    (addr + size) > base + seg->s_size) {
2451 		AS_LOCK_EXIT(as, &as->a_lock);
2452 		goto slow;
2453 	}
2454 
2455 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
2456 	    "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
2457 
2458 	/*
2459 	 * try to lock pages and pass back shadow list
2460 	 */
2461 	res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
2462 
2463 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
2464 	AS_LOCK_EXIT(as, &as->a_lock);
2465 	if (res == 0) {
2466 		return (0);
2467 	} else if (res == ENOTSUP || prefaulted) {
2468 		/*
2469 		 * (1) segment driver doesn't support PAGELOCK fastpath, or
2470 		 * (2) we've already tried fast path unsuccessfully after
2471 		 *    faulting in the addr range below; system might be
2472 		 *    thrashing or there may not be enough availrmem.
2473 		 */
2474 		goto slow;
2475 	}
2476 
2477 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_FAULT_START,
2478 	    "as_fault_start: addr %p size %ld", addr, size);
2479 
2480 	/*
2481 	 * we might get here because of some COW fault or non
2482 	 * existing page. Let as_fault deal with it. Just load
2483 	 * the page, don't lock the MMU mapping.
2484 	 */
2485 	fault_err = as_fault(as->a_hat, as, addr, size, F_INVAL, rw);
2486 	if (fault_err != 0) {
2487 		return (f_decode(fault_err));
2488 	}
2489 
2490 	prefaulted = 1;
2491 
2492 	/*
2493 	 * try fast path again; since we've dropped a_lock,
2494 	 * we need to try the dance from the start to see if
2495 	 * the addr range is still valid.
2496 	 */
2497 	goto top;
2498 slow:
2499 	/*
2500 	 * load the page and lock the MMU mapping.
2501 	 */
2502 	fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
2503 	if (fault_err != 0) {
2504 		return (f_decode(fault_err));
2505 	}
2506 	*ppp = NULL;
2507 
2508 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
2509 	return (0);
2510 }
2511 
2512 /*
2513  * unlock pages in a given address range
2514  */
2515 void
2516 as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
2517     enum seg_rw rw)
2518 {
2519 	struct seg *seg;
2520 	size_t rsize;
2521 	caddr_t raddr;
2522 
2523 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
2524 	    "as_pageunlock_start: addr %p size %ld", addr, size);
2525 
2526 	/*
2527 	 * if the shadow list is NULL, as_pagelock was
2528 	 * falling back to as_fault
2529 	 */
2530 	if (pp == NULL) {
2531 		(void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
2532 		return;
2533 	}
2534 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2535 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2536 		(size_t)raddr;
2537 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2538 	seg = as_findseg(as, addr, 0);
2539 	ASSERT(seg);
2540 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
2541 	    "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
2542 	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
2543 	AS_LOCK_EXIT(as, &as->a_lock);
2544 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
2545 }
2546 
2547 /*
2548  * reclaim cached pages in a given address range
2549  */
2550 void
2551 as_pagereclaim(struct as *as, struct page **pp, caddr_t addr,
2552     size_t size, enum seg_rw rw)
2553 {
2554 	struct seg *seg;
2555 	size_t rsize;
2556 	caddr_t raddr;
2557 
2558 	ASSERT(AS_READ_HELD(as, &as->a_lock));
2559 	ASSERT(pp != NULL);
2560 
2561 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2562 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2563 		(size_t)raddr;
2564 	seg = as_findseg(as, addr, 0);
2565 	ASSERT(seg);
2566 	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGERECLAIM, rw);
2567 }
2568 
2569 #define	MAXPAGEFLIP	4
2570 #define	MAXPAGEFLIPSIZ	MAXPAGEFLIP*PAGESIZE
2571 
2572 int
2573 as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2574     boolean_t wait)
2575 {
2576 	struct seg *seg;
2577 	size_t ssize;
2578 	caddr_t raddr;			/* rounded down addr */
2579 	size_t rsize;			/* rounded up size */
2580 	int error = 0;
2581 	size_t pgsz = page_get_pagesize(szc);
2582 
2583 setpgsz_top:
2584 	if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) {
2585 		return (EINVAL);
2586 	}
2587 
2588 	raddr = addr;
2589 	rsize = size;
2590 
2591 	if (raddr + rsize < raddr)		/* check for wraparound */
2592 		return (ENOMEM);
2593 
2594 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2595 	as_clearwatchprot(as, raddr, rsize);
2596 	seg = as_segat(as, raddr);
2597 	if (seg == NULL) {
2598 		as_setwatch(as);
2599 		AS_LOCK_EXIT(as, &as->a_lock);
2600 		return (ENOMEM);
2601 	}
2602 
2603 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2604 		if (raddr >= seg->s_base + seg->s_size) {
2605 			seg = AS_SEGNEXT(as, seg);
2606 			if (seg == NULL || raddr != seg->s_base) {
2607 				error = ENOMEM;
2608 				break;
2609 			}
2610 		}
2611 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2612 			ssize = seg->s_base + seg->s_size - raddr;
2613 		} else {
2614 			ssize = rsize;
2615 		}
2616 
2617 		error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2618 
2619 		if (error == IE_NOMEM) {
2620 			error = EAGAIN;
2621 			break;
2622 		}
2623 
2624 		if (error == IE_RETRY) {
2625 			AS_LOCK_EXIT(as, &as->a_lock);
2626 			goto setpgsz_top;
2627 		}
2628 
2629 		if (error == ENOTSUP) {
2630 			error = EINVAL;
2631 			break;
2632 		}
2633 
2634 		if (wait && (error == EAGAIN)) {
2635 			/*
2636 			 * Memory is currently locked.  It must be unlocked
2637 			 * before this operation can succeed through a retry.
2638 			 * The possible reasons for locked memory and
2639 			 * corresponding strategies for unlocking are:
2640 			 * (1) Normal I/O
2641 			 *	wait for a signal that the I/O operation
2642 			 *	has completed and the memory is unlocked.
2643 			 * (2) Asynchronous I/O
2644 			 *	The aio subsystem does not unlock pages when
2645 			 *	the I/O is completed. Those pages are unlocked
2646 			 *	when the application calls aiowait/aioerror.
2647 			 *	So, to prevent blocking forever, cv_broadcast()
2648 			 *	is done to wake up aio_cleanup_thread.
2649 			 *	Subsequently, segvn_reclaim will be called, and
2650 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
2651 			 * (3) Long term page locking:
2652 			 *	This is not relevant for as_setpagesize()
2653 			 *	because we cannot change the page size for
2654 			 *	driver memory. The attempt to do so will
2655 			 *	fail with a different error than EAGAIN so
2656 			 *	there's no need to trigger as callbacks like
2657 			 *	as_unmap, as_setprot or as_free would do.
2658 			 */
2659 			mutex_enter(&as->a_contents);
2660 			if (AS_ISUNMAPWAIT(as) == 0) {
2661 				cv_broadcast(&as->a_cv);
2662 			}
2663 			AS_SETUNMAPWAIT(as);
2664 			AS_LOCK_EXIT(as, &as->a_lock);
2665 			while (AS_ISUNMAPWAIT(as)) {
2666 				cv_wait(&as->a_cv, &as->a_contents);
2667 			}
2668 			mutex_exit(&as->a_contents);
2669 			goto setpgsz_top;
2670 		} else if (error != 0) {
2671 			break;
2672 		}
2673 	}
2674 	as_setwatch(as);
2675 	AS_LOCK_EXIT(as, &as->a_lock);
2676 	return (error);
2677 }
2678 
2679 /*
2680  * Setup all of the uninitialized watched pages that we can.
2681  */
2682 void
2683 as_setwatch(struct as *as)
2684 {
2685 	struct watched_page *pwp;
2686 	struct seg *seg;
2687 	caddr_t vaddr;
2688 	uint_t prot;
2689 	int  err, retrycnt;
2690 
2691 	if (avl_numnodes(&as->a_wpage) == 0)
2692 		return;
2693 
2694 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2695 
2696 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
2697 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
2698 		retrycnt = 0;
2699 	retry:
2700 		vaddr = pwp->wp_vaddr;
2701 		if (pwp->wp_oprot != 0 ||	/* already set up */
2702 		    (seg = as_segat(as, vaddr)) == NULL ||
2703 		    SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
2704 			continue;
2705 
2706 		pwp->wp_oprot = prot;
2707 		if (pwp->wp_read)
2708 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2709 		if (pwp->wp_write)
2710 			prot &= ~PROT_WRITE;
2711 		if (pwp->wp_exec)
2712 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2713 		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) {
2714 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
2715 			if (err == IE_RETRY) {
2716 				pwp->wp_oprot = 0;
2717 				ASSERT(retrycnt == 0);
2718 				retrycnt++;
2719 				goto retry;
2720 			}
2721 		}
2722 		pwp->wp_prot = prot;
2723 	}
2724 }
2725 
2726 /*
2727  * Clear all of the watched pages in the address space.
2728  */
2729 void
2730 as_clearwatch(struct as *as)
2731 {
2732 	struct watched_page *pwp;
2733 	struct seg *seg;
2734 	caddr_t vaddr;
2735 	uint_t prot;
2736 	int err, retrycnt;
2737 
2738 	if (avl_numnodes(&as->a_wpage) == 0)
2739 		return;
2740 
2741 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2742 
2743 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
2744 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
2745 		retrycnt = 0;
2746 	retry:
2747 		vaddr = pwp->wp_vaddr;
2748 		if (pwp->wp_oprot == 0 ||	/* not set up */
2749 		    (seg = as_segat(as, vaddr)) == NULL)
2750 			continue;
2751 
2752 		if ((prot = pwp->wp_oprot) != pwp->wp_prot) {
2753 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
2754 			if (err == IE_RETRY) {
2755 				ASSERT(retrycnt == 0);
2756 				retrycnt++;
2757 				goto retry;
2758 			}
2759 		}
2760 		pwp->wp_oprot = 0;
2761 		pwp->wp_prot = 0;
2762 	}
2763 }
2764 
2765 /*
2766  * Force a new setup for all the watched pages in the range.
2767  */
2768 static void
2769 as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
2770 {
2771 	struct watched_page *pwp;
2772 	struct watched_page tpw;
2773 	caddr_t eaddr = addr + size;
2774 	caddr_t vaddr;
2775 	struct seg *seg;
2776 	int err, retrycnt;
2777 	uint_t	wprot;
2778 	avl_index_t where;
2779 
2780 	if (avl_numnodes(&as->a_wpage) == 0)
2781 		return;
2782 
2783 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2784 
2785 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2786 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
2787 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
2788 
2789 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
2790 		retrycnt = 0;
2791 		vaddr = pwp->wp_vaddr;
2792 
2793 		wprot = prot;
2794 		if (pwp->wp_read)
2795 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2796 		if (pwp->wp_write)
2797 			wprot &= ~PROT_WRITE;
2798 		if (pwp->wp_exec)
2799 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2800 		if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) {
2801 		retry:
2802 			seg = as_segat(as, vaddr);
2803 			if (seg == NULL) {
2804 				panic("as_setwatchprot: no seg");
2805 				/*NOTREACHED*/
2806 			}
2807 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot);
2808 			if (err == IE_RETRY) {
2809 				ASSERT(retrycnt == 0);
2810 				retrycnt++;
2811 				goto retry;
2812 			}
2813 		}
2814 		pwp->wp_oprot = prot;
2815 		pwp->wp_prot = wprot;
2816 
2817 		pwp = AVL_NEXT(&as->a_wpage, pwp);
2818 	}
2819 }
2820 
2821 /*
2822  * Clear all of the watched pages in the range.
2823  */
2824 static void
2825 as_clearwatchprot(struct as *as, caddr_t addr, size_t size)
2826 {
2827 	caddr_t eaddr = addr + size;
2828 	struct watched_page *pwp;
2829 	struct watched_page tpw;
2830 	uint_t prot;
2831 	struct seg *seg;
2832 	int err, retrycnt;
2833 	avl_index_t where;
2834 
2835 	if (avl_numnodes(&as->a_wpage) == 0)
2836 		return;
2837 
2838 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2839 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
2840 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
2841 
2842 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2843 
2844 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
2845 		ASSERT(addr >= pwp->wp_vaddr);
2846 
2847 		if ((prot = pwp->wp_oprot) != 0) {
2848 			retrycnt = 0;
2849 
2850 			if (prot != pwp->wp_prot) {
2851 			retry:
2852 				seg = as_segat(as, pwp->wp_vaddr);
2853 				if (seg == NULL)
2854 					continue;
2855 				err = SEGOP_SETPROT(seg, pwp->wp_vaddr,
2856 				    PAGESIZE, prot);
2857 				if (err == IE_RETRY) {
2858 					ASSERT(retrycnt == 0);
2859 					retrycnt++;
2860 					goto retry;
2861 
2862 				}
2863 			}
2864 			pwp->wp_oprot = 0;
2865 			pwp->wp_prot = 0;
2866 		}
2867 
2868 		pwp = AVL_NEXT(&as->a_wpage, pwp);
2869 	}
2870 }
2871 
2872 void
2873 as_signal_proc(struct as *as, k_siginfo_t *siginfo)
2874 {
2875 	struct proc *p;
2876 
2877 	mutex_enter(&pidlock);
2878 	for (p = practive; p; p = p->p_next) {
2879 		if (p->p_as == as) {
2880 			mutex_enter(&p->p_lock);
2881 			if (p->p_as == as)
2882 				sigaddq(p, NULL, siginfo, KM_NOSLEEP);
2883 			mutex_exit(&p->p_lock);
2884 		}
2885 	}
2886 	mutex_exit(&pidlock);
2887 }
2888 
2889 /*
2890  * return memory object ID
2891  */
2892 int
2893 as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
2894 {
2895 	struct seg	*seg;
2896 	int		sts;
2897 
2898 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2899 	seg = as_segat(as, addr);
2900 	if (seg == NULL) {
2901 		AS_LOCK_EXIT(as, &as->a_lock);
2902 		return (EFAULT);
2903 	}
2904 	/*
2905 	 * catch old drivers which may not support getmemid
2906 	 */
2907 	if (seg->s_ops->getmemid == NULL) {
2908 		AS_LOCK_EXIT(as, &as->a_lock);
2909 		return (ENODEV);
2910 	}
2911 
2912 	sts = SEGOP_GETMEMID(seg, addr, memidp);
2913 
2914 	AS_LOCK_EXIT(as, &as->a_lock);
2915 	return (sts);
2916 }
2917