xref: /titanic_51/usr/src/uts/common/vm/vm_as.c (revision ace1a5f11236a072fca1b5e0ea1416a083a9f2aa)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
28 /*	  All Rights Reserved  	*/
29 
30 /*
31  * University Copyright- Copyright (c) 1982, 1986, 1988
32  * The Regents of the University of California
33  * All Rights Reserved
34  *
35  * University Acknowledgment- Portions of this document are derived from
36  * software developed by the University of California, Berkeley, and its
37  * contributors.
38  */
39 
40 #pragma ident	"%Z%%M%	%I%	%E% SMI"
41 
42 /*
43  * VM - address spaces.
44  */
45 
46 #include <sys/types.h>
47 #include <sys/t_lock.h>
48 #include <sys/param.h>
49 #include <sys/errno.h>
50 #include <sys/systm.h>
51 #include <sys/mman.h>
52 #include <sys/sysmacros.h>
53 #include <sys/cpuvar.h>
54 #include <sys/sysinfo.h>
55 #include <sys/kmem.h>
56 #include <sys/vnode.h>
57 #include <sys/vmsystm.h>
58 #include <sys/cmn_err.h>
59 #include <sys/debug.h>
60 #include <sys/tnf_probe.h>
61 #include <sys/vtrace.h>
62 
63 #include <vm/hat.h>
64 #include <vm/xhat.h>
65 #include <vm/as.h>
66 #include <vm/seg.h>
67 #include <vm/seg_vn.h>
68 #include <vm/seg_dev.h>
69 #include <vm/seg_kmem.h>
70 #include <vm/seg_map.h>
71 #include <vm/seg_spt.h>
72 #include <vm/page.h>
73 
74 clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
75 
76 static struct kmem_cache *as_cache;
77 
78 static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
79 static void as_clearwatchprot(struct as *, caddr_t, size_t);
80 
81 
82 /*
83  * Verifying the segment lists is very time-consuming; it may not be
84  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
85  */
86 #ifdef DEBUG
87 #define	VERIFY_SEGLIST
88 int do_as_verify = 0;
89 #endif
90 
91 /*
92  * Allocate a new callback data structure entry and fill in the events of
93  * interest, the address range of interest, and the callback argument.
94  * Link the entry on the as->a_callbacks list. A callback entry for the
95  * entire address space may be specified with vaddr = 0 and size = -1.
96  *
97  * CALLERS RESPONSIBILITY: If not calling from within the process context for
98  * the specified as, the caller must guarantee persistence of the specified as
99  * for the duration of this function (eg. pages being locked within the as
100  * will guarantee persistence).
101  */
102 int
103 as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
104 		caddr_t vaddr, size_t size, int sleepflag)
105 {
106 	struct as_callback 	*current_head, *cb;
107 	caddr_t 		saddr;
108 	size_t 			rsize;
109 
110 	/* callback function and an event are mandatory */
111 	if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
112 		return (EINVAL);
113 
114 	/* Adding a callback after as_free has been called is not allowed */
115 	if (as == &kas)
116 		return (ENOMEM);
117 
118 	/*
119 	 * vaddr = 0 and size = -1 is used to indicate that the callback range
120 	 * is the entire address space so no rounding is done in that case.
121 	 */
122 	if (size != -1) {
123 		saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
124 		rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
125 			(size_t)saddr;
126 		/* check for wraparound */
127 		if (saddr + rsize < saddr)
128 			return (ENOMEM);
129 	} else {
130 		if (vaddr != 0)
131 			return (EINVAL);
132 		saddr = vaddr;
133 		rsize = size;
134 	}
135 
136 	/* Allocate and initialize a callback entry */
137 	cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
138 	if (cb == NULL)
139 		return (EAGAIN);
140 
141 	cb->ascb_func = cb_func;
142 	cb->ascb_arg = arg;
143 	cb->ascb_events = events;
144 	cb->ascb_saddr = saddr;
145 	cb->ascb_len = rsize;
146 
147 	/* Add the entry to the list */
148 	mutex_enter(&as->a_contents);
149 	current_head = as->a_callbacks;
150 	as->a_callbacks = cb;
151 	cb->ascb_next = current_head;
152 
153 	/*
154 	 * The call to this function may lose in a race with
155 	 * a pertinent event - eg. a thread does long term memory locking
156 	 * but before the callback is added another thread executes as_unmap.
157 	 * A broadcast here resolves that.
158 	 */
159 	if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
160 		AS_CLRUNMAPWAIT(as);
161 		cv_broadcast(&as->a_cv);
162 	}
163 
164 	mutex_exit(&as->a_contents);
165 	return (0);
166 }
167 
168 /*
169  * Search the callback list for an entry which pertains to arg.
170  *
171  * This is called from within the client upon completion of the callback.
172  * RETURN VALUES:
173  *	AS_CALLBACK_DELETED  (callback entry found and deleted)
174  *	AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
175  *	AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
176  *			entry will be made in as_do_callbacks)
177  *
178  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
179  * set, it indicates that as_do_callbacks is processing this entry.  The
180  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
181  * to unblock as_do_callbacks, in case it is blocked.
182  *
183  * CALLERS RESPONSIBILITY: If not calling from within the process context for
184  * the specified as, the caller must guarantee persistence of the specified as
185  * for the duration of this function (eg. pages being locked within the as
186  * will guarantee persistence).
187  */
188 uint_t
189 as_delete_callback(struct as *as, void *arg)
190 {
191 	struct as_callback **prevcb = &as->a_callbacks;
192 	struct as_callback *cb;
193 	uint_t rc = AS_CALLBACK_NOTFOUND;
194 
195 	mutex_enter(&as->a_contents);
196 	for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
197 		if (cb->ascb_arg != arg)
198 			continue;
199 
200 		/*
201 		 * If the events indicate AS_CALLBACK_CALLED, just clear
202 		 * AS_ALL_EVENT in the events field and wakeup the thread
203 		 * that may be waiting in as_do_callbacks.  as_do_callbacks
204 		 * will take care of removing this entry from the list.  In
205 		 * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
206 		 * (AS_CALLBACK_CALLED not set), just remove it from the
207 		 * list, return the memory and return AS_CALLBACK_DELETED.
208 		 */
209 		if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
210 			/* leave AS_CALLBACK_CALLED */
211 			cb->ascb_events &= ~AS_ALL_EVENT;
212 			rc = AS_CALLBACK_DELETE_DEFERRED;
213 			cv_broadcast(&as->a_cv);
214 		} else {
215 			*prevcb = cb->ascb_next;
216 			kmem_free(cb, sizeof (struct as_callback));
217 			rc = AS_CALLBACK_DELETED;
218 		}
219 		break;
220 	}
221 	mutex_exit(&as->a_contents);
222 	return (rc);
223 }
224 
225 /*
226  * Searches the as callback list for a matching entry.
227  * Returns a pointer to the first matching callback, or NULL if
228  * nothing is found.
229  * This function never sleeps so it is ok to call it with more
230  * locks held but the (required) a_contents mutex.
231  *
232  * See also comment on as_do_callbacks below.
233  */
234 static struct as_callback *
235 as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
236 			size_t event_len)
237 {
238 	struct as_callback	*cb;
239 
240 	ASSERT(MUTEX_HELD(&as->a_contents));
241 	for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
242 		/*
243 		 * If the callback has not already been called, then
244 		 * check if events or address range pertains.  An event_len
245 		 * of zero means do an unconditional callback.
246 		 */
247 		if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
248 		    ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
249 		    (event_addr + event_len < cb->ascb_saddr) ||
250 		    (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
251 			continue;
252 		}
253 		break;
254 	}
255 	return (cb);
256 }
257 
258 /*
259  * Executes a given callback and removes it from the callback list for
260  * this address space.
261  * This function may sleep so the caller must drop all locks except
262  * a_contents before calling this func.
263  *
264  * See also comments on as_do_callbacks below.
265  */
266 static void
267 as_execute_callback(struct as *as, struct as_callback *cb,
268 				uint_t events)
269 {
270 	struct as_callback **prevcb;
271 	void	*cb_arg;
272 
273 	ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
274 	cb->ascb_events |= AS_CALLBACK_CALLED;
275 	mutex_exit(&as->a_contents);
276 	(*cb->ascb_func)(as, cb->ascb_arg, events);
277 	mutex_enter(&as->a_contents);
278 	/*
279 	 * the callback function is required to delete the callback
280 	 * when the callback function determines it is OK for
281 	 * this thread to continue. as_delete_callback will clear
282 	 * the AS_ALL_EVENT in the events field when it is deleted.
283 	 * If the callback function called as_delete_callback,
284 	 * events will already be cleared and there will be no blocking.
285 	 */
286 	while ((cb->ascb_events & events) != 0) {
287 		cv_wait(&as->a_cv, &as->a_contents);
288 	}
289 	/*
290 	 * This entry needs to be taken off the list. Normally, the
291 	 * callback func itself does that, but unfortunately the list
292 	 * may have changed while the callback was running because the
293 	 * a_contents mutex was dropped and someone else other than the
294 	 * callback func itself could have called as_delete_callback,
295 	 * so we have to search to find this entry again.  The entry
296 	 * must have AS_CALLBACK_CALLED, and have the same 'arg'.
297 	 */
298 	cb_arg = cb->ascb_arg;
299 	prevcb = &as->a_callbacks;
300 	for (cb = as->a_callbacks; cb != NULL;
301 	    prevcb = &cb->ascb_next, cb = *prevcb) {
302 		if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
303 		    (cb_arg != cb->ascb_arg)) {
304 			continue;
305 		}
306 		*prevcb = cb->ascb_next;
307 		kmem_free(cb, sizeof (struct as_callback));
308 		break;
309 	}
310 }
311 
312 /*
313  * Check the callback list for a matching event and intersection of
314  * address range. If there is a match invoke the callback.  Skip an entry if:
315  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
316  *    - not event of interest
317  *    - not address range of interest
318  *
319  * An event_len of zero indicates a request for an unconditional callback
320  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
321  * a_contents lock must be dropped before a callback, so only one callback
322  * can be done before returning. Return -1 (true) if a callback was
323  * executed and removed from the list, else return 0 (false).
324  *
325  * The logically separate parts, i.e. finding a matching callback and
326  * executing a given callback have been separated into two functions
327  * so that they can be called with different sets of locks held beyond
328  * the always-required a_contents. as_find_callback does not sleep so
329  * it is ok to call it if more locks than a_contents (i.e. the a_lock
330  * rwlock) are held. as_execute_callback on the other hand may sleep
331  * so all locks beyond a_contents must be dropped by the caller if one
332  * does not want to end comatose.
333  */
334 static int
335 as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
336 			size_t event_len)
337 {
338 	struct as_callback *cb;
339 
340 	if ((cb = as_find_callback(as, events, event_addr, event_len))) {
341 		as_execute_callback(as, cb, events);
342 		return (-1);
343 	}
344 	return (0);
345 }
346 
347 /*
348  * Search for the segment containing addr. If a segment containing addr
349  * exists, that segment is returned.  If no such segment exists, and
350  * the list spans addresses greater than addr, then the first segment
351  * whose base is greater than addr is returned; otherwise, NULL is
352  * returned unless tail is true, in which case the last element of the
353  * list is returned.
354  *
355  * a_seglast is used to cache the last found segment for repeated
356  * searches to the same addr (which happens frequently).
357  */
358 struct seg *
359 as_findseg(struct as *as, caddr_t addr, int tail)
360 {
361 	struct seg *seg = as->a_seglast;
362 	avl_index_t where;
363 
364 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
365 
366 	if (seg != NULL &&
367 	    seg->s_base <= addr &&
368 	    addr < seg->s_base + seg->s_size)
369 		return (seg);
370 
371 	seg = avl_find(&as->a_segtree, &addr, &where);
372 	if (seg != NULL)
373 		return (as->a_seglast = seg);
374 
375 	seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
376 	if (seg == NULL && tail)
377 		seg = avl_last(&as->a_segtree);
378 	return (as->a_seglast = seg);
379 }
380 
381 #ifdef VERIFY_SEGLIST
382 /*
383  * verify that the linked list is coherent
384  */
385 static void
386 as_verify(struct as *as)
387 {
388 	struct seg *seg, *seglast, *p, *n;
389 	uint_t nsegs = 0;
390 
391 	if (do_as_verify == 0)
392 		return;
393 
394 	seglast = as->a_seglast;
395 
396 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
397 		ASSERT(seg->s_as == as);
398 		p = AS_SEGPREV(as, seg);
399 		n = AS_SEGNEXT(as, seg);
400 		ASSERT(p == NULL || p->s_as == as);
401 		ASSERT(p == NULL || p->s_base < seg->s_base);
402 		ASSERT(n == NULL || n->s_base > seg->s_base);
403 		ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
404 		if (seg == seglast)
405 			seglast = NULL;
406 		nsegs++;
407 	}
408 	ASSERT(seglast == NULL);
409 	ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
410 }
411 #endif /* VERIFY_SEGLIST */
412 
413 /*
414  * Add a new segment to the address space. The avl_find()
415  * may be expensive so we attempt to use last segment accessed
416  * in as_gap() as an insertion point.
417  */
418 int
419 as_addseg(struct as  *as, struct seg *newseg)
420 {
421 	struct seg *seg;
422 	caddr_t addr;
423 	caddr_t eaddr;
424 	avl_index_t where;
425 
426 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
427 
428 	as->a_updatedir = 1;	/* inform /proc */
429 	gethrestime(&as->a_updatetime);
430 
431 	if (as->a_lastgaphl != NULL) {
432 		struct seg *hseg = NULL;
433 		struct seg *lseg = NULL;
434 
435 		if (as->a_lastgaphl->s_base > newseg->s_base) {
436 			hseg = as->a_lastgaphl;
437 			lseg = AVL_PREV(&as->a_segtree, hseg);
438 		} else {
439 			lseg = as->a_lastgaphl;
440 			hseg = AVL_NEXT(&as->a_segtree, lseg);
441 		}
442 
443 		if (hseg && lseg && lseg->s_base < newseg->s_base &&
444 		    hseg->s_base > newseg->s_base) {
445 			avl_insert_here(&as->a_segtree, newseg, lseg,
446 			    AVL_AFTER);
447 			as->a_lastgaphl = NULL;
448 			as->a_seglast = newseg;
449 			return (0);
450 		}
451 		as->a_lastgaphl = NULL;
452 	}
453 
454 	addr = newseg->s_base;
455 	eaddr = addr + newseg->s_size;
456 again:
457 
458 	seg = avl_find(&as->a_segtree, &addr, &where);
459 
460 	if (seg == NULL)
461 		seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
462 
463 	if (seg == NULL)
464 		seg = avl_last(&as->a_segtree);
465 
466 	if (seg != NULL) {
467 		caddr_t base = seg->s_base;
468 
469 		/*
470 		 * If top of seg is below the requested address, then
471 		 * the insertion point is at the end of the linked list,
472 		 * and seg points to the tail of the list.  Otherwise,
473 		 * the insertion point is immediately before seg.
474 		 */
475 		if (base + seg->s_size > addr) {
476 			if (addr >= base || eaddr > base) {
477 #ifdef __sparc
478 				extern struct seg_ops segnf_ops;
479 
480 				/*
481 				 * no-fault segs must disappear if overlaid.
482 				 * XXX need new segment type so
483 				 * we don't have to check s_ops
484 				 */
485 				if (seg->s_ops == &segnf_ops) {
486 					seg_unmap(seg);
487 					goto again;
488 				}
489 #endif
490 				return (-1);	/* overlapping segment */
491 			}
492 		}
493 	}
494 	as->a_seglast = newseg;
495 	avl_insert(&as->a_segtree, newseg, where);
496 
497 #ifdef VERIFY_SEGLIST
498 	as_verify(as);
499 #endif
500 	return (0);
501 }
502 
503 struct seg *
504 as_removeseg(struct as *as, struct seg *seg)
505 {
506 	avl_tree_t *t;
507 
508 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
509 
510 	as->a_updatedir = 1;	/* inform /proc */
511 	gethrestime(&as->a_updatetime);
512 
513 	if (seg == NULL)
514 		return (NULL);
515 
516 	t = &as->a_segtree;
517 	if (as->a_seglast == seg)
518 		as->a_seglast = NULL;
519 	as->a_lastgaphl = NULL;
520 
521 	/*
522 	 * if this segment is at an address higher than
523 	 * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
524 	 */
525 	if (as->a_lastgap &&
526 	    (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
527 		as->a_lastgap = AVL_NEXT(t, seg);
528 
529 	/*
530 	 * remove the segment from the seg tree
531 	 */
532 	avl_remove(t, seg);
533 
534 #ifdef VERIFY_SEGLIST
535 	as_verify(as);
536 #endif
537 	return (seg);
538 }
539 
540 /*
541  * Find a segment containing addr.
542  */
543 struct seg *
544 as_segat(struct as *as, caddr_t addr)
545 {
546 	struct seg *seg = as->a_seglast;
547 
548 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
549 
550 	if (seg != NULL && seg->s_base <= addr &&
551 	    addr < seg->s_base + seg->s_size)
552 		return (seg);
553 
554 	seg = avl_find(&as->a_segtree, &addr, NULL);
555 	return (seg);
556 }
557 
558 /*
559  * Serialize all searches for holes in an address space to
560  * prevent two or more threads from allocating the same virtual
561  * address range.  The address space must not be "read/write"
562  * locked by the caller since we may block.
563  */
564 void
565 as_rangelock(struct as *as)
566 {
567 	mutex_enter(&as->a_contents);
568 	while (AS_ISCLAIMGAP(as))
569 		cv_wait(&as->a_cv, &as->a_contents);
570 	AS_SETCLAIMGAP(as);
571 	mutex_exit(&as->a_contents);
572 }
573 
574 /*
575  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
576  */
577 void
578 as_rangeunlock(struct as *as)
579 {
580 	mutex_enter(&as->a_contents);
581 	AS_CLRCLAIMGAP(as);
582 	cv_signal(&as->a_cv);
583 	mutex_exit(&as->a_contents);
584 }
585 
586 /*
587  * compar segments (or just an address) by segment address range
588  */
589 static int
590 as_segcompar(const void *x, const void *y)
591 {
592 	struct seg *a = (struct seg *)x;
593 	struct seg *b = (struct seg *)y;
594 
595 	if (a->s_base < b->s_base)
596 		return (-1);
597 	if (a->s_base >= b->s_base + b->s_size)
598 		return (1);
599 	return (0);
600 }
601 
602 
603 void
604 as_avlinit(struct as *as)
605 {
606 	avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
607 	    offsetof(struct seg, s_tree));
608 	avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
609 	    offsetof(struct watched_page, wp_link));
610 }
611 
612 /*ARGSUSED*/
613 static int
614 as_constructor(void *buf, void *cdrarg, int kmflags)
615 {
616 	struct as *as = buf;
617 
618 	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
619 	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
620 	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
621 	as_avlinit(as);
622 	return (0);
623 }
624 
625 /*ARGSUSED1*/
626 static void
627 as_destructor(void *buf, void *cdrarg)
628 {
629 	struct as *as = buf;
630 
631 	avl_destroy(&as->a_segtree);
632 	mutex_destroy(&as->a_contents);
633 	cv_destroy(&as->a_cv);
634 	rw_destroy(&as->a_lock);
635 }
636 
637 void
638 as_init(void)
639 {
640 	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
641 		as_constructor, as_destructor, NULL, NULL, NULL, 0);
642 }
643 
644 /*
645  * Allocate and initialize an address space data structure.
646  * We call hat_alloc to allow any machine dependent
647  * information in the hat structure to be initialized.
648  */
649 struct as *
650 as_alloc(void)
651 {
652 	struct as *as;
653 
654 	as = kmem_cache_alloc(as_cache, KM_SLEEP);
655 
656 	as->a_flags		= 0;
657 	as->a_vbits		= 0;
658 	as->a_hrm		= NULL;
659 	as->a_seglast		= NULL;
660 	as->a_size		= 0;
661 	as->a_updatedir		= 0;
662 	gethrestime(&as->a_updatetime);
663 	as->a_objectdir		= NULL;
664 	as->a_sizedir		= 0;
665 	as->a_userlimit		= (caddr_t)USERLIMIT;
666 	as->a_lastgap		= NULL;
667 	as->a_lastgaphl		= NULL;
668 	as->a_callbacks		= NULL;
669 
670 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
671 	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
672 	AS_LOCK_EXIT(as, &as->a_lock);
673 
674 	as->a_xhat = NULL;
675 
676 	return (as);
677 }
678 
679 /*
680  * Free an address space data structure.
681  * Need to free the hat first and then
682  * all the segments on this as and finally
683  * the space for the as struct itself.
684  */
685 void
686 as_free(struct as *as)
687 {
688 	struct hat *hat = as->a_hat;
689 	struct seg *seg, *next;
690 	int called = 0;
691 
692 top:
693 	/*
694 	 * Invoke ALL callbacks. as_do_callbacks will do one callback
695 	 * per call, and not return (-1) until the callback has completed.
696 	 * When as_do_callbacks returns zero, all callbacks have completed.
697 	 */
698 	mutex_enter(&as->a_contents);
699 	while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0));
700 
701 	/* This will prevent new XHATs from attaching to as */
702 	if (!called)
703 		AS_SETBUSY(as);
704 	mutex_exit(&as->a_contents);
705 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
706 
707 	if (!called) {
708 		called = 1;
709 		hat_free_start(hat);
710 		if (as->a_xhat != NULL)
711 			xhat_free_start_all(as);
712 	}
713 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
714 		int err;
715 
716 		next = AS_SEGNEXT(as, seg);
717 		err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
718 		if (err == EAGAIN) {
719 			mutex_enter(&as->a_contents);
720 			if (as->a_callbacks) {
721 				AS_LOCK_EXIT(as, &as->a_lock);
722 			} else {
723 				/*
724 				 * Memory is currently locked. Wait for a
725 				 * cv_signal that it has been unlocked, then
726 				 * try the operation again.
727 				 */
728 				if (AS_ISUNMAPWAIT(as) == 0)
729 					cv_broadcast(&as->a_cv);
730 				AS_SETUNMAPWAIT(as);
731 				AS_LOCK_EXIT(as, &as->a_lock);
732 				while (AS_ISUNMAPWAIT(as))
733 					cv_wait(&as->a_cv, &as->a_contents);
734 			}
735 			mutex_exit(&as->a_contents);
736 			goto top;
737 		} else {
738 			/*
739 			 * We do not expect any other error return at this
740 			 * time. This is similar to an ASSERT in seg_unmap()
741 			 */
742 			ASSERT(err == 0);
743 		}
744 	}
745 	hat_free_end(hat);
746 	if (as->a_xhat != NULL)
747 		xhat_free_end_all(as);
748 	AS_LOCK_EXIT(as, &as->a_lock);
749 
750 	/* /proc stuff */
751 	ASSERT(avl_numnodes(&as->a_wpage) == 0);
752 	if (as->a_objectdir) {
753 		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
754 		as->a_objectdir = NULL;
755 		as->a_sizedir = 0;
756 	}
757 
758 	/*
759 	 * Free the struct as back to kmem.  Assert it has no segments.
760 	 */
761 	ASSERT(avl_numnodes(&as->a_segtree) == 0);
762 	kmem_cache_free(as_cache, as);
763 }
764 
765 int
766 as_dup(struct as *as, struct as **outas)
767 {
768 	struct as *newas;
769 	struct seg *seg, *newseg;
770 	int error;
771 
772 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
773 	as_clearwatch(as);
774 	newas = as_alloc();
775 	newas->a_userlimit = as->a_userlimit;
776 	AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER);
777 
778 	/* This will prevent new XHATs from attaching */
779 	mutex_enter(&as->a_contents);
780 	AS_SETBUSY(as);
781 	mutex_exit(&as->a_contents);
782 	mutex_enter(&newas->a_contents);
783 	AS_SETBUSY(newas);
784 	mutex_exit(&newas->a_contents);
785 
786 
787 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
788 
789 		if (seg->s_flags & S_PURGE)
790 			continue;
791 
792 		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
793 		if (newseg == NULL) {
794 			AS_LOCK_EXIT(newas, &newas->a_lock);
795 			as_setwatch(as);
796 			mutex_enter(&as->a_contents);
797 			AS_CLRBUSY(as);
798 			mutex_exit(&as->a_contents);
799 			AS_LOCK_EXIT(as, &as->a_lock);
800 			as_free(newas);
801 			return (-1);
802 		}
803 		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
804 			/*
805 			 * We call seg_free() on the new seg
806 			 * because the segment is not set up
807 			 * completely; i.e. it has no ops.
808 			 */
809 			as_setwatch(as);
810 			mutex_enter(&as->a_contents);
811 			AS_CLRBUSY(as);
812 			mutex_exit(&as->a_contents);
813 			AS_LOCK_EXIT(as, &as->a_lock);
814 			seg_free(newseg);
815 			AS_LOCK_EXIT(newas, &newas->a_lock);
816 			as_free(newas);
817 			return (error);
818 		}
819 		newas->a_size += seg->s_size;
820 	}
821 
822 	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
823 	if (as->a_xhat != NULL)
824 		error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL);
825 
826 	mutex_enter(&newas->a_contents);
827 	AS_CLRBUSY(newas);
828 	mutex_exit(&newas->a_contents);
829 	AS_LOCK_EXIT(newas, &newas->a_lock);
830 
831 	as_setwatch(as);
832 	mutex_enter(&as->a_contents);
833 	AS_CLRBUSY(as);
834 	mutex_exit(&as->a_contents);
835 	AS_LOCK_EXIT(as, &as->a_lock);
836 	if (error != 0) {
837 		as_free(newas);
838 		return (error);
839 	}
840 	*outas = newas;
841 	return (0);
842 }
843 
844 /*
845  * Handle a ``fault'' at addr for size bytes.
846  */
847 faultcode_t
848 as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
849 	enum fault_type type, enum seg_rw rw)
850 {
851 	struct seg *seg;
852 	caddr_t raddr;			/* rounded down addr */
853 	size_t rsize;			/* rounded up size */
854 	size_t ssize;
855 	faultcode_t res = 0;
856 	caddr_t addrsav;
857 	struct seg *segsav;
858 	int as_lock_held;
859 	klwp_t *lwp = ttolwp(curthread);
860 	int is_xhat = 0;
861 	int holding_wpage = 0;
862 	extern struct seg_ops   segdev_ops;
863 
864 
865 
866 	if (as->a_hat != hat) {
867 		/* This must be an XHAT then */
868 		is_xhat = 1;
869 
870 		if ((type != F_INVAL) || (as == &kas))
871 			return (FC_NOSUPPORT);
872 	}
873 
874 retry:
875 	if (!is_xhat) {
876 		/*
877 		 * Indicate that the lwp is not to be stopped while waiting
878 		 * for a pagefault.  This is to avoid deadlock while debugging
879 		 * a process via /proc over NFS (in particular).
880 		 */
881 		if (lwp != NULL) {
882 			lwp->lwp_nostop++;
883 			lwp->lwp_nostop_r++;
884 		}
885 
886 		/*
887 		 * same length must be used when we softlock and softunlock.
888 		 * We don't support softunlocking lengths less than
889 		 * the original length when there is largepage support.
890 		 * See seg_dev.c for more comments.
891 		 */
892 		switch (type) {
893 
894 		case F_SOFTLOCK:
895 			CPU_STATS_ADD_K(vm, softlock, 1);
896 			break;
897 
898 		case F_SOFTUNLOCK:
899 			break;
900 
901 		case F_PROT:
902 			CPU_STATS_ADD_K(vm, prot_fault, 1);
903 			break;
904 
905 		case F_INVAL:
906 			CPU_STATS_ENTER_K();
907 			CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
908 			if (as == &kas)
909 				CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
910 			CPU_STATS_EXIT_K();
911 			break;
912 		}
913 	}
914 
915 	/* Kernel probe */
916 	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
917 		tnf_opaque,	address,	addr,
918 		tnf_fault_type,	fault_type,	type,
919 		tnf_seg_access,	access,		rw);
920 
921 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
922 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
923 		(size_t)raddr;
924 
925 	/*
926 	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
927 	 * correctness, but then we could be stuck holding this lock for
928 	 * a LONG time if the fault needs to be resolved on a slow
929 	 * filesystem, and then no-one will be able to exec new commands,
930 	 * as exec'ing requires the write lock on the as.
931 	 */
932 	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
933 	    raddr + size < segkmap->s_base + segkmap->s_size) {
934 		/*
935 		 * if (as==&kas), this can't be XHAT: we've already returned
936 		 * FC_NOSUPPORT.
937 		 */
938 		seg = segkmap;
939 		as_lock_held = 0;
940 	} else {
941 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
942 		if (is_xhat && avl_numnodes(&as->a_wpage) != 0) {
943 			/*
944 			 * Grab and hold the writers' lock on the as
945 			 * if the fault is to a watched page.
946 			 * This will keep CPUs from "peeking" at the
947 			 * address range while we're temporarily boosting
948 			 * the permissions for the XHAT device to
949 			 * resolve the fault in the segment layer.
950 			 *
951 			 * We could check whether faulted address
952 			 * is within a watched page and only then grab
953 			 * the writer lock, but this is simpler.
954 			 */
955 			AS_LOCK_EXIT(as, &as->a_lock);
956 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
957 		}
958 
959 		seg = as_segat(as, raddr);
960 		if (seg == NULL) {
961 			AS_LOCK_EXIT(as, &as->a_lock);
962 			if ((lwp != NULL) && (!is_xhat)) {
963 				lwp->lwp_nostop--;
964 				lwp->lwp_nostop_r--;
965 			}
966 			return (FC_NOMAP);
967 		}
968 
969 		as_lock_held = 1;
970 	}
971 
972 	addrsav = raddr;
973 	segsav = seg;
974 
975 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
976 		if (raddr >= seg->s_base + seg->s_size) {
977 			seg = AS_SEGNEXT(as, seg);
978 			if (seg == NULL || raddr != seg->s_base) {
979 				res = FC_NOMAP;
980 				break;
981 			}
982 		}
983 		if (raddr + rsize > seg->s_base + seg->s_size)
984 			ssize = seg->s_base + seg->s_size - raddr;
985 		else
986 			ssize = rsize;
987 
988 		if (!is_xhat || (seg->s_ops != &segdev_ops)) {
989 
990 			if (is_xhat && avl_numnodes(&as->a_wpage) != 0 &&
991 			    pr_is_watchpage_as(raddr, rw, as)) {
992 				/*
993 				 * Handle watch pages.  If we're faulting on a
994 				 * watched page from an X-hat, we have to
995 				 * restore the original permissions while we
996 				 * handle the fault.
997 				 */
998 				as_clearwatch(as);
999 				holding_wpage = 1;
1000 			}
1001 
1002 			res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
1003 
1004 			/* Restore watchpoints */
1005 			if (holding_wpage) {
1006 				as_setwatch(as);
1007 				holding_wpage = 0;
1008 			}
1009 
1010 			if (res != 0)
1011 				break;
1012 		} else {
1013 			/* XHAT does not support seg_dev */
1014 			res = FC_NOSUPPORT;
1015 			break;
1016 		}
1017 	}
1018 
1019 	/*
1020 	 * If we were SOFTLOCKing and encountered a failure,
1021 	 * we must SOFTUNLOCK the range we already did. (Maybe we
1022 	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
1023 	 * right here...)
1024 	 */
1025 	if (res != 0 && type == F_SOFTLOCK) {
1026 		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
1027 			if (addrsav >= seg->s_base + seg->s_size)
1028 				seg = AS_SEGNEXT(as, seg);
1029 			ASSERT(seg != NULL);
1030 			/*
1031 			 * Now call the fault routine again to perform the
1032 			 * unlock using S_OTHER instead of the rw variable
1033 			 * since we never got a chance to touch the pages.
1034 			 */
1035 			if (raddr > seg->s_base + seg->s_size)
1036 				ssize = seg->s_base + seg->s_size - addrsav;
1037 			else
1038 				ssize = raddr - addrsav;
1039 			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
1040 			    F_SOFTUNLOCK, S_OTHER);
1041 		}
1042 	}
1043 	if (as_lock_held)
1044 		AS_LOCK_EXIT(as, &as->a_lock);
1045 	if ((lwp != NULL) && (!is_xhat)) {
1046 		lwp->lwp_nostop--;
1047 		lwp->lwp_nostop_r--;
1048 	}
1049 	/*
1050 	 * If the lower levels returned EDEADLK for a fault,
1051 	 * It means that we should retry the fault.  Let's wait
1052 	 * a bit also to let the deadlock causing condition clear.
1053 	 * This is part of a gross hack to work around a design flaw
1054 	 * in the ufs/sds logging code and should go away when the
1055 	 * logging code is re-designed to fix the problem. See bug
1056 	 * 4125102 for details of the problem.
1057 	 */
1058 	if (FC_ERRNO(res) == EDEADLK) {
1059 		delay(deadlk_wait);
1060 		res = 0;
1061 		goto retry;
1062 	}
1063 	return (res);
1064 }
1065 
1066 
1067 
1068 /*
1069  * Asynchronous ``fault'' at addr for size bytes.
1070  */
1071 faultcode_t
1072 as_faulta(struct as *as, caddr_t addr, size_t size)
1073 {
1074 	struct seg *seg;
1075 	caddr_t raddr;			/* rounded down addr */
1076 	size_t rsize;			/* rounded up size */
1077 	faultcode_t res = 0;
1078 	klwp_t *lwp = ttolwp(curthread);
1079 
1080 retry:
1081 	/*
1082 	 * Indicate that the lwp is not to be stopped while waiting
1083 	 * for a pagefault.  This is to avoid deadlock while debugging
1084 	 * a process via /proc over NFS (in particular).
1085 	 */
1086 	if (lwp != NULL) {
1087 		lwp->lwp_nostop++;
1088 		lwp->lwp_nostop_r++;
1089 	}
1090 
1091 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1092 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1093 		(size_t)raddr;
1094 
1095 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1096 	seg = as_segat(as, raddr);
1097 	if (seg == NULL) {
1098 		AS_LOCK_EXIT(as, &as->a_lock);
1099 		if (lwp != NULL) {
1100 			lwp->lwp_nostop--;
1101 			lwp->lwp_nostop_r--;
1102 		}
1103 		return (FC_NOMAP);
1104 	}
1105 
1106 	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
1107 		if (raddr >= seg->s_base + seg->s_size) {
1108 			seg = AS_SEGNEXT(as, seg);
1109 			if (seg == NULL || raddr != seg->s_base) {
1110 				res = FC_NOMAP;
1111 				break;
1112 			}
1113 		}
1114 		res = SEGOP_FAULTA(seg, raddr);
1115 		if (res != 0)
1116 			break;
1117 	}
1118 	AS_LOCK_EXIT(as, &as->a_lock);
1119 	if (lwp != NULL) {
1120 		lwp->lwp_nostop--;
1121 		lwp->lwp_nostop_r--;
1122 	}
1123 	/*
1124 	 * If the lower levels returned EDEADLK for a fault,
1125 	 * It means that we should retry the fault.  Let's wait
1126 	 * a bit also to let the deadlock causing condition clear.
1127 	 * This is part of a gross hack to work around a design flaw
1128 	 * in the ufs/sds logging code and should go away when the
1129 	 * logging code is re-designed to fix the problem. See bug
1130 	 * 4125102 for details of the problem.
1131 	 */
1132 	if (FC_ERRNO(res) == EDEADLK) {
1133 		delay(deadlk_wait);
1134 		res = 0;
1135 		goto retry;
1136 	}
1137 	return (res);
1138 }
1139 
1140 /*
1141  * Set the virtual mapping for the interval from [addr : addr + size)
1142  * in address space `as' to have the specified protection.
1143  * It is ok for the range to cross over several segments,
1144  * as long as they are contiguous.
1145  */
1146 int
1147 as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1148 {
1149 	struct seg *seg;
1150 	struct as_callback *cb;
1151 	size_t ssize;
1152 	caddr_t raddr;			/* rounded down addr */
1153 	size_t rsize;			/* rounded up size */
1154 	int error = 0, writer = 0;
1155 	caddr_t saveraddr;
1156 	size_t saversize;
1157 
1158 setprot_top:
1159 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1160 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1161 		(size_t)raddr;
1162 
1163 	if (raddr + rsize < raddr)		/* check for wraparound */
1164 		return (ENOMEM);
1165 
1166 	saveraddr = raddr;
1167 	saversize = rsize;
1168 
1169 	/*
1170 	 * Normally we only lock the as as a reader. But
1171 	 * if due to setprot the segment driver needs to split
1172 	 * a segment it will return IE_RETRY. Therefore we re-aquire
1173 	 * the as lock as a writer so the segment driver can change
1174 	 * the seg list. Also the segment driver will return IE_RETRY
1175 	 * after it has changed the segment list so we therefore keep
1176 	 * locking as a writer. Since these opeartions should be rare
1177 	 * want to only lock as a writer when necessary.
1178 	 */
1179 	if (writer || avl_numnodes(&as->a_wpage) != 0) {
1180 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1181 	} else {
1182 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1183 	}
1184 
1185 	as_clearwatchprot(as, raddr, rsize);
1186 	seg = as_segat(as, raddr);
1187 	if (seg == NULL) {
1188 		as_setwatch(as);
1189 		AS_LOCK_EXIT(as, &as->a_lock);
1190 		return (ENOMEM);
1191 	}
1192 
1193 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1194 		if (raddr >= seg->s_base + seg->s_size) {
1195 			seg = AS_SEGNEXT(as, seg);
1196 			if (seg == NULL || raddr != seg->s_base) {
1197 				error = ENOMEM;
1198 				break;
1199 			}
1200 		}
1201 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1202 			ssize = seg->s_base + seg->s_size - raddr;
1203 		else
1204 			ssize = rsize;
1205 		error = SEGOP_SETPROT(seg, raddr, ssize, prot);
1206 
1207 		if (error == IE_NOMEM) {
1208 			error = EAGAIN;
1209 			break;
1210 		}
1211 
1212 		if (error == IE_RETRY) {
1213 			AS_LOCK_EXIT(as, &as->a_lock);
1214 			writer = 1;
1215 			goto setprot_top;
1216 		}
1217 
1218 		if (error == EAGAIN) {
1219 			/*
1220 			 * Make sure we have a_lock as writer.
1221 			 */
1222 			if (writer == 0) {
1223 				AS_LOCK_EXIT(as, &as->a_lock);
1224 				writer = 1;
1225 				goto setprot_top;
1226 			}
1227 
1228 			/*
1229 			 * Memory is currently locked.  It must be unlocked
1230 			 * before this operation can succeed through a retry.
1231 			 * The possible reasons for locked memory and
1232 			 * corresponding strategies for unlocking are:
1233 			 * (1) Normal I/O
1234 			 *	wait for a signal that the I/O operation
1235 			 *	has completed and the memory is unlocked.
1236 			 * (2) Asynchronous I/O
1237 			 *	The aio subsystem does not unlock pages when
1238 			 *	the I/O is completed. Those pages are unlocked
1239 			 *	when the application calls aiowait/aioerror.
1240 			 *	So, to prevent blocking forever, cv_broadcast()
1241 			 *	is done to wake up aio_cleanup_thread.
1242 			 *	Subsequently, segvn_reclaim will be called, and
1243 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1244 			 * (3) Long term page locking:
1245 			 *	Drivers intending to have pages locked for a
1246 			 *	period considerably longer than for normal I/O
1247 			 *	(essentially forever) may have registered for a
1248 			 *	callback so they may unlock these pages on
1249 			 *	request. This is needed to allow this operation
1250 			 *	to succeed. Each entry on the callback list is
1251 			 *	examined. If the event or address range pertains
1252 			 *	the callback is invoked (unless it already is in
1253 			 *	progress). The a_contents lock must be dropped
1254 			 *	before the callback, so only one callback can
1255 			 *	be done at a time. Go to the top and do more
1256 			 *	until zero is returned. If zero is returned,
1257 			 *	either there were no callbacks for this event
1258 			 *	or they were already in progress.
1259 			 */
1260 			mutex_enter(&as->a_contents);
1261 			if (as->a_callbacks &&
1262 				(cb = as_find_callback(as, AS_SETPROT_EVENT,
1263 						seg->s_base, seg->s_size))) {
1264 				AS_LOCK_EXIT(as, &as->a_lock);
1265 				as_execute_callback(as, cb, AS_SETPROT_EVENT);
1266 			} else {
1267 				if (AS_ISUNMAPWAIT(as) == 0)
1268 					cv_broadcast(&as->a_cv);
1269 				AS_SETUNMAPWAIT(as);
1270 				AS_LOCK_EXIT(as, &as->a_lock);
1271 				while (AS_ISUNMAPWAIT(as))
1272 					cv_wait(&as->a_cv, &as->a_contents);
1273 			}
1274 			mutex_exit(&as->a_contents);
1275 			goto setprot_top;
1276 		} else if (error != 0)
1277 			break;
1278 	}
1279 	if (error != 0) {
1280 		as_setwatch(as);
1281 	} else {
1282 		as_setwatchprot(as, saveraddr, saversize, prot);
1283 	}
1284 	AS_LOCK_EXIT(as, &as->a_lock);
1285 	return (error);
1286 }
1287 
1288 /*
1289  * Check to make sure that the interval [addr, addr + size)
1290  * in address space `as' has at least the specified protection.
1291  * It is ok for the range to cross over several segments, as long
1292  * as they are contiguous.
1293  */
1294 int
1295 as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1296 {
1297 	struct seg *seg;
1298 	size_t ssize;
1299 	caddr_t raddr;			/* rounded down addr */
1300 	size_t rsize;			/* rounded up size */
1301 	int error = 0;
1302 
1303 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1304 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1305 		(size_t)raddr;
1306 
1307 	if (raddr + rsize < raddr)		/* check for wraparound */
1308 		return (ENOMEM);
1309 
1310 	/*
1311 	 * This is ugly as sin...
1312 	 * Normally, we only acquire the address space readers lock.
1313 	 * However, if the address space has watchpoints present,
1314 	 * we must acquire the writer lock on the address space for
1315 	 * the benefit of as_clearwatchprot() and as_setwatchprot().
1316 	 */
1317 	if (avl_numnodes(&as->a_wpage) != 0)
1318 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1319 	else
1320 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1321 	as_clearwatchprot(as, raddr, rsize);
1322 	seg = as_segat(as, raddr);
1323 	if (seg == NULL) {
1324 		as_setwatch(as);
1325 		AS_LOCK_EXIT(as, &as->a_lock);
1326 		return (ENOMEM);
1327 	}
1328 
1329 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1330 		if (raddr >= seg->s_base + seg->s_size) {
1331 			seg = AS_SEGNEXT(as, seg);
1332 			if (seg == NULL || raddr != seg->s_base) {
1333 				error = ENOMEM;
1334 				break;
1335 			}
1336 		}
1337 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1338 			ssize = seg->s_base + seg->s_size - raddr;
1339 		else
1340 			ssize = rsize;
1341 
1342 		error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
1343 		if (error != 0)
1344 			break;
1345 	}
1346 	as_setwatch(as);
1347 	AS_LOCK_EXIT(as, &as->a_lock);
1348 	return (error);
1349 }
1350 
1351 int
1352 as_unmap(struct as *as, caddr_t addr, size_t size)
1353 {
1354 	struct seg *seg, *seg_next;
1355 	struct as_callback *cb;
1356 	caddr_t raddr, eaddr;
1357 	size_t ssize;
1358 	int err;
1359 
1360 top:
1361 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1362 	eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) &
1363 	    (uintptr_t)PAGEMASK);
1364 
1365 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1366 
1367 	as->a_updatedir = 1;	/* inform /proc */
1368 	gethrestime(&as->a_updatetime);
1369 
1370 	/*
1371 	 * Use as_findseg to find the first segment in the range, then
1372 	 * step through the segments in order, following s_next.
1373 	 */
1374 	as_clearwatchprot(as, raddr, eaddr - raddr);
1375 
1376 	for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
1377 		if (eaddr <= seg->s_base)
1378 			break;		/* eaddr was in a gap; all done */
1379 
1380 		/* this is implied by the test above */
1381 		ASSERT(raddr < eaddr);
1382 
1383 		if (raddr < seg->s_base)
1384 			raddr = seg->s_base; 	/* raddr was in a gap */
1385 
1386 		if (eaddr > (seg->s_base + seg->s_size))
1387 			ssize = seg->s_base + seg->s_size - raddr;
1388 		else
1389 			ssize = eaddr - raddr;
1390 
1391 		/*
1392 		 * Save next segment pointer since seg can be
1393 		 * destroyed during the segment unmap operation.
1394 		 */
1395 		seg_next = AS_SEGNEXT(as, seg);
1396 
1397 		err = SEGOP_UNMAP(seg, raddr, ssize);
1398 		if (err == EAGAIN) {
1399 			/*
1400 			 * Memory is currently locked.  It must be unlocked
1401 			 * before this operation can succeed through a retry.
1402 			 * The possible reasons for locked memory and
1403 			 * corresponding strategies for unlocking are:
1404 			 * (1) Normal I/O
1405 			 *	wait for a signal that the I/O operation
1406 			 *	has completed and the memory is unlocked.
1407 			 * (2) Asynchronous I/O
1408 			 *	The aio subsystem does not unlock pages when
1409 			 *	the I/O is completed. Those pages are unlocked
1410 			 *	when the application calls aiowait/aioerror.
1411 			 *	So, to prevent blocking forever, cv_broadcast()
1412 			 *	is done to wake up aio_cleanup_thread.
1413 			 *	Subsequently, segvn_reclaim will be called, and
1414 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1415 			 * (3) Long term page locking:
1416 			 *	Drivers intending to have pages locked for a
1417 			 *	period considerably longer than for normal I/O
1418 			 *	(essentially forever) may have registered for a
1419 			 *	callback so they may unlock these pages on
1420 			 *	request. This is needed to allow this operation
1421 			 *	to succeed. Each entry on the callback list is
1422 			 *	examined. If the event or address range pertains
1423 			 *	the callback is invoked (unless it already is in
1424 			 *	progress). The a_contents lock must be dropped
1425 			 *	before the callback, so only one callback can
1426 			 *	be done at a time. Go to the top and do more
1427 			 *	until zero is returned. If zero is returned,
1428 			 *	either there were no callbacks for this event
1429 			 *	or they were already in progress.
1430 			 */
1431 			as_setwatch(as);
1432 			mutex_enter(&as->a_contents);
1433 			if (as->a_callbacks &&
1434 				(cb = as_find_callback(as, AS_UNMAP_EVENT,
1435 						seg->s_base, seg->s_size))) {
1436 				AS_LOCK_EXIT(as, &as->a_lock);
1437 				as_execute_callback(as, cb, AS_UNMAP_EVENT);
1438 			} else {
1439 				if (AS_ISUNMAPWAIT(as) == 0)
1440 					cv_broadcast(&as->a_cv);
1441 				AS_SETUNMAPWAIT(as);
1442 				AS_LOCK_EXIT(as, &as->a_lock);
1443 				while (AS_ISUNMAPWAIT(as))
1444 					cv_wait(&as->a_cv, &as->a_contents);
1445 			}
1446 			mutex_exit(&as->a_contents);
1447 			goto top;
1448 		} else if (err == IE_RETRY) {
1449 			as_setwatch(as);
1450 			AS_LOCK_EXIT(as, &as->a_lock);
1451 			goto top;
1452 		} else if (err) {
1453 			as_setwatch(as);
1454 			AS_LOCK_EXIT(as, &as->a_lock);
1455 			return (-1);
1456 		}
1457 
1458 		as->a_size -= ssize;
1459 		raddr += ssize;
1460 	}
1461 	AS_LOCK_EXIT(as, &as->a_lock);
1462 	return (0);
1463 }
1464 
1465 static int
1466 as_map_vnsegs(struct as *as, caddr_t addr, size_t size,
1467     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1468 {
1469 	int text = vn_a->flags & MAP_TEXT;
1470 	uint_t szcvec = map_execseg_pgszcvec(text, addr, size);
1471 	uint_t szc;
1472 	uint_t nszc;
1473 	int error;
1474 	caddr_t a;
1475 	caddr_t eaddr;
1476 	size_t segsize;
1477 	struct seg *seg;
1478 	uint_t save_szcvec;
1479 	size_t pgsz;
1480 	struct vattr va;
1481 	u_offset_t eoff;
1482 	size_t save_size = 0;
1483 
1484 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1485 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1486 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1487 	ASSERT(vn_a->vp != NULL);
1488 	ASSERT(vn_a->amp == NULL);
1489 
1490 again:
1491 	if (szcvec <= 1) {
1492 		seg = seg_alloc(as, addr, size);
1493 		if (seg == NULL) {
1494 			return (ENOMEM);
1495 		}
1496 		vn_a->szc = 0;
1497 		error = (*crfp)(seg, vn_a);
1498 		if (error != 0) {
1499 			seg_free(seg);
1500 		}
1501 		return (error);
1502 	}
1503 
1504 	va.va_mask = AT_SIZE;
1505 	if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred) != 0) {
1506 		szcvec = 0;
1507 		goto again;
1508 	}
1509 	eoff = vn_a->offset & PAGEMASK;
1510 	if (eoff >= va.va_size) {
1511 		szcvec = 0;
1512 		goto again;
1513 	}
1514 	eoff += size;
1515 	if (btopr(va.va_size) < btopr(eoff)) {
1516 		save_size = size;
1517 		size = va.va_size - (vn_a->offset & PAGEMASK);
1518 		size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t);
1519 		szcvec = map_execseg_pgszcvec(text, addr, size);
1520 		if (szcvec <= 1) {
1521 			size = save_size;
1522 			goto again;
1523 		}
1524 	}
1525 
1526 	eaddr = addr + size;
1527 	save_szcvec = szcvec;
1528 	szcvec >>= 1;
1529 	szc = 0;
1530 	nszc = 0;
1531 	while (szcvec) {
1532 		if ((szcvec & 0x1) == 0) {
1533 			nszc++;
1534 			szcvec >>= 1;
1535 			continue;
1536 		}
1537 		nszc++;
1538 		pgsz = page_get_pagesize(nszc);
1539 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
1540 		if (a != addr) {
1541 			ASSERT(a < eaddr);
1542 			segsize = a - addr;
1543 			seg = seg_alloc(as, addr, segsize);
1544 			if (seg == NULL) {
1545 				return (ENOMEM);
1546 			}
1547 			vn_a->szc = szc;
1548 			error = (*crfp)(seg, vn_a);
1549 			if (error != 0) {
1550 				seg_free(seg);
1551 				return (error);
1552 			}
1553 			*segcreated = 1;
1554 			vn_a->offset += segsize;
1555 			addr = a;
1556 		}
1557 		szc = nszc;
1558 		szcvec >>= 1;
1559 	}
1560 
1561 	ASSERT(addr < eaddr);
1562 	szcvec = save_szcvec | 1; /* add 8K pages */
1563 	while (szcvec) {
1564 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
1565 		ASSERT(a >= addr);
1566 		if (a != addr) {
1567 			segsize = a - addr;
1568 			seg = seg_alloc(as, addr, segsize);
1569 			if (seg == NULL) {
1570 				return (ENOMEM);
1571 			}
1572 			vn_a->szc = szc;
1573 			error = (*crfp)(seg, vn_a);
1574 			if (error != 0) {
1575 				seg_free(seg);
1576 				return (error);
1577 			}
1578 			*segcreated = 1;
1579 			vn_a->offset += segsize;
1580 			addr = a;
1581 		}
1582 		szcvec &= ~(1 << szc);
1583 		if (szcvec) {
1584 			szc = highbit(szcvec) - 1;
1585 			pgsz = page_get_pagesize(szc);
1586 		}
1587 	}
1588 	ASSERT(addr == eaddr);
1589 
1590 	if (save_size) {
1591 		size = save_size - size;
1592 		goto again;
1593 	}
1594 
1595 	return (0);
1596 }
1597 
1598 int
1599 as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp)
1600 {
1601 	struct seg *seg = NULL;
1602 	caddr_t raddr;			/* rounded down addr */
1603 	size_t rsize;			/* rounded up size */
1604 	int error;
1605 	struct proc *p = curproc;
1606 
1607 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1608 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1609 		(size_t)raddr;
1610 
1611 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1612 
1613 	/*
1614 	 * check for wrap around
1615 	 */
1616 	if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
1617 		AS_LOCK_EXIT(as, &as->a_lock);
1618 		return (ENOMEM);
1619 	}
1620 
1621 	as->a_updatedir = 1;	/* inform /proc */
1622 	gethrestime(&as->a_updatetime);
1623 
1624 	if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) {
1625 		AS_LOCK_EXIT(as, &as->a_lock);
1626 
1627 		(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p,
1628 		    RCA_UNSAFE_ALL);
1629 
1630 		return (ENOMEM);
1631 	}
1632 
1633 	if (AS_MAP_VNSEGS_USELPGS(crfp, argsp)) {
1634 		int unmap = 0;
1635 		error = as_map_vnsegs(as, raddr, rsize, crfp,
1636 		    (struct segvn_crargs *)argsp, &unmap);
1637 		if (error != 0) {
1638 			AS_LOCK_EXIT(as, &as->a_lock);
1639 			if (unmap) {
1640 				(void) as_unmap(as, addr, size);
1641 			}
1642 			return (error);
1643 		}
1644 	} else {
1645 		seg = seg_alloc(as, addr, size);
1646 		if (seg == NULL) {
1647 			AS_LOCK_EXIT(as, &as->a_lock);
1648 			return (ENOMEM);
1649 		}
1650 
1651 		error = (*crfp)(seg, argsp);
1652 		if (error != 0) {
1653 			seg_free(seg);
1654 			AS_LOCK_EXIT(as, &as->a_lock);
1655 			return (error);
1656 		}
1657 	}
1658 
1659 	/*
1660 	 * Add size now so as_unmap will work if as_ctl fails.
1661 	 */
1662 	as->a_size += rsize;
1663 
1664 	as_setwatch(as);
1665 
1666 	/*
1667 	 * If the address space is locked,
1668 	 * establish memory locks for the new segment.
1669 	 */
1670 	mutex_enter(&as->a_contents);
1671 	if (AS_ISPGLCK(as)) {
1672 		mutex_exit(&as->a_contents);
1673 		AS_LOCK_EXIT(as, &as->a_lock);
1674 		error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
1675 		if (error != 0)
1676 			(void) as_unmap(as, addr, size);
1677 	} else {
1678 		mutex_exit(&as->a_contents);
1679 		AS_LOCK_EXIT(as, &as->a_lock);
1680 	}
1681 	return (error);
1682 }
1683 
1684 
1685 /*
1686  * Delete all segments in the address space marked with S_PURGE.
1687  * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c).
1688  * These segments are deleted as a first step before calls to as_gap(), so
1689  * that they don't affect mmap() or shmat().
1690  */
1691 void
1692 as_purge(struct as *as)
1693 {
1694 	struct seg *seg;
1695 	struct seg *next_seg;
1696 
1697 	/*
1698 	 * the setting of NEEDSPURGE is protect by as_rangelock(), so
1699 	 * no need to grab a_contents mutex for this check
1700 	 */
1701 	if ((as->a_flags & AS_NEEDSPURGE) == 0)
1702 		return;
1703 
1704 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1705 	next_seg = NULL;
1706 	seg = AS_SEGFIRST(as);
1707 	while (seg != NULL) {
1708 		next_seg = AS_SEGNEXT(as, seg);
1709 		if (seg->s_flags & S_PURGE)
1710 			SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
1711 		seg = next_seg;
1712 	}
1713 	AS_LOCK_EXIT(as, &as->a_lock);
1714 
1715 	mutex_enter(&as->a_contents);
1716 	as->a_flags &= ~AS_NEEDSPURGE;
1717 	mutex_exit(&as->a_contents);
1718 }
1719 
1720 /*
1721  * Find a hole of at least size minlen within [base, base + len).
1722  *
1723  * If flags specifies AH_HI, the hole will have the highest possible address
1724  * in the range.  We use the as->a_lastgap field to figure out where to
1725  * start looking for a gap.
1726  *
1727  * Otherwise, the gap will have the lowest possible address.
1728  *
1729  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
1730  *
1731  * If an adequate hole is found, base and len are set to reflect the part of
1732  * the hole that is within range, and 0 is returned, otherwise,
1733  * -1 is returned.
1734  *
1735  * NOTE: This routine is not correct when base+len overflows caddr_t.
1736  */
1737 int
1738 as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags,
1739     caddr_t addr)
1740 {
1741 	caddr_t lobound = *basep;
1742 	caddr_t hibound = lobound + *lenp;
1743 	struct seg *lseg, *hseg;
1744 	caddr_t lo, hi;
1745 	int forward;
1746 	caddr_t save_base;
1747 	size_t save_len;
1748 
1749 	save_base = *basep;
1750 	save_len = *lenp;
1751 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1752 	if (AS_SEGFIRST(as) == NULL) {
1753 		if (valid_va_range(basep, lenp, minlen, flags & AH_DIR)) {
1754 			AS_LOCK_EXIT(as, &as->a_lock);
1755 			return (0);
1756 		} else {
1757 			AS_LOCK_EXIT(as, &as->a_lock);
1758 			*basep = save_base;
1759 			*lenp = save_len;
1760 			return (-1);
1761 		}
1762 	}
1763 
1764 	/*
1765 	 * Set up to iterate over all the inter-segment holes in the given
1766 	 * direction.  lseg is NULL for the lowest-addressed hole and hseg is
1767 	 * NULL for the highest-addressed hole.  If moving backwards, we reset
1768 	 * sseg to denote the highest-addressed segment.
1769 	 */
1770 	forward = (flags & AH_DIR) == AH_LO;
1771 	if (forward) {
1772 		hseg = as_findseg(as, lobound, 1);
1773 		lseg = AS_SEGPREV(as, hseg);
1774 	} else {
1775 
1776 		/*
1777 		 * If allocating at least as much as the last allocation,
1778 		 * use a_lastgap's base as a better estimate of hibound.
1779 		 */
1780 		if (as->a_lastgap &&
1781 		    minlen >= as->a_lastgap->s_size &&
1782 		    hibound >= as->a_lastgap->s_base)
1783 			hibound = as->a_lastgap->s_base;
1784 
1785 		hseg = as_findseg(as, hibound, 1);
1786 		if (hseg->s_base + hseg->s_size < hibound) {
1787 			lseg = hseg;
1788 			hseg = NULL;
1789 		} else {
1790 			lseg = AS_SEGPREV(as, hseg);
1791 		}
1792 	}
1793 
1794 	for (;;) {
1795 		/*
1796 		 * Set lo and hi to the hole's boundaries.  (We should really
1797 		 * use MAXADDR in place of hibound in the expression below,
1798 		 * but can't express it easily; using hibound in its place is
1799 		 * harmless.)
1800 		 */
1801 		lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
1802 		hi = (hseg == NULL) ? hibound : hseg->s_base;
1803 		/*
1804 		 * If the iteration has moved past the interval from lobound
1805 		 * to hibound it's pointless to continue.
1806 		 */
1807 		if ((forward && lo > hibound) || (!forward && hi < lobound))
1808 			break;
1809 		else if (lo > hibound || hi < lobound)
1810 			goto cont;
1811 		/*
1812 		 * Candidate hole lies at least partially within the allowable
1813 		 * range.  Restrict it to fall completely within that range,
1814 		 * i.e., to [max(lo, lobound), min(hi, hibound)].
1815 		 */
1816 		if (lo < lobound)
1817 			lo = lobound;
1818 		if (hi > hibound)
1819 			hi = hibound;
1820 		/*
1821 		 * Verify that the candidate hole is big enough and meets
1822 		 * hardware constraints.
1823 		 */
1824 		*basep = lo;
1825 		*lenp = hi - lo;
1826 		if (valid_va_range(basep, lenp, minlen,
1827 		    forward ? AH_LO : AH_HI) &&
1828 		    ((flags & AH_CONTAIN) == 0 ||
1829 		    (*basep <= addr && *basep + *lenp > addr))) {
1830 			if (!forward)
1831 				as->a_lastgap = hseg;
1832 			if (hseg != NULL)
1833 				as->a_lastgaphl = hseg;
1834 			else
1835 				as->a_lastgaphl = lseg;
1836 			AS_LOCK_EXIT(as, &as->a_lock);
1837 			return (0);
1838 		}
1839 	cont:
1840 		/*
1841 		 * Move to the next hole.
1842 		 */
1843 		if (forward) {
1844 			lseg = hseg;
1845 			if (lseg == NULL)
1846 				break;
1847 			hseg = AS_SEGNEXT(as, hseg);
1848 		} else {
1849 			hseg = lseg;
1850 			if (hseg == NULL)
1851 				break;
1852 			lseg = AS_SEGPREV(as, lseg);
1853 		}
1854 	}
1855 	*basep = save_base;
1856 	*lenp = save_len;
1857 	AS_LOCK_EXIT(as, &as->a_lock);
1858 	return (-1);
1859 }
1860 
1861 /*
1862  * Return the next range within [base, base + len) that is backed
1863  * with "real memory".  Skip holes and non-seg_vn segments.
1864  * We're lazy and only return one segment at a time.
1865  */
1866 int
1867 as_memory(struct as *as, caddr_t *basep, size_t *lenp)
1868 {
1869 	extern struct seg_ops segspt_shmops;	/* needs a header file */
1870 	struct seg *seg;
1871 	caddr_t addr, eaddr;
1872 	caddr_t segend;
1873 
1874 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1875 
1876 	addr = *basep;
1877 	eaddr = addr + *lenp;
1878 
1879 	seg = as_findseg(as, addr, 0);
1880 	if (seg != NULL)
1881 		addr = MAX(seg->s_base, addr);
1882 
1883 	for (;;) {
1884 		if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
1885 			AS_LOCK_EXIT(as, &as->a_lock);
1886 			return (EINVAL);
1887 		}
1888 
1889 		if (seg->s_ops == &segvn_ops) {
1890 			segend = seg->s_base + seg->s_size;
1891 			break;
1892 		}
1893 
1894 		/*
1895 		 * We do ISM by looking into the private data
1896 		 * to determine the real size of the segment.
1897 		 */
1898 		if (seg->s_ops == &segspt_shmops) {
1899 			segend = seg->s_base + spt_realsize(seg);
1900 			if (addr < segend)
1901 				break;
1902 		}
1903 
1904 		seg = AS_SEGNEXT(as, seg);
1905 
1906 		if (seg != NULL)
1907 			addr = seg->s_base;
1908 	}
1909 
1910 	*basep = addr;
1911 
1912 	if (segend > eaddr)
1913 		*lenp = eaddr - addr;
1914 	else
1915 		*lenp = segend - addr;
1916 
1917 	AS_LOCK_EXIT(as, &as->a_lock);
1918 	return (0);
1919 }
1920 
1921 /*
1922  * Swap the pages associated with the address space as out to
1923  * secondary storage, returning the number of bytes actually
1924  * swapped.
1925  *
1926  * The value returned is intended to correlate well with the process's
1927  * memory requirements.  Its usefulness for this purpose depends on
1928  * how well the segment-level routines do at returning accurate
1929  * information.
1930  */
1931 size_t
1932 as_swapout(struct as *as)
1933 {
1934 	struct seg *seg;
1935 	size_t swpcnt = 0;
1936 
1937 	/*
1938 	 * Kernel-only processes have given up their address
1939 	 * spaces.  Of course, we shouldn't be attempting to
1940 	 * swap out such processes in the first place...
1941 	 */
1942 	if (as == NULL)
1943 		return (0);
1944 
1945 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1946 
1947 	/* Prevent XHATs from attaching */
1948 	mutex_enter(&as->a_contents);
1949 	AS_SETBUSY(as);
1950 	mutex_exit(&as->a_contents);
1951 
1952 
1953 	/*
1954 	 * Free all mapping resources associated with the address
1955 	 * space.  The segment-level swapout routines capitalize
1956 	 * on this unmapping by scavanging pages that have become
1957 	 * unmapped here.
1958 	 */
1959 	hat_swapout(as->a_hat);
1960 	if (as->a_xhat != NULL)
1961 		xhat_swapout_all(as);
1962 
1963 	mutex_enter(&as->a_contents);
1964 	AS_CLRBUSY(as);
1965 	mutex_exit(&as->a_contents);
1966 
1967 	/*
1968 	 * Call the swapout routines of all segments in the address
1969 	 * space to do the actual work, accumulating the amount of
1970 	 * space reclaimed.
1971 	 */
1972 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
1973 		struct seg_ops *ov = seg->s_ops;
1974 
1975 		/*
1976 		 * We have to check to see if the seg has
1977 		 * an ops vector because the seg may have
1978 		 * been in the middle of being set up when
1979 		 * the process was picked for swapout.
1980 		 */
1981 		if ((ov != NULL) && (ov->swapout != NULL))
1982 			swpcnt += SEGOP_SWAPOUT(seg);
1983 	}
1984 	AS_LOCK_EXIT(as, &as->a_lock);
1985 	return (swpcnt);
1986 }
1987 
1988 /*
1989  * Determine whether data from the mappings in interval [addr, addr + size)
1990  * are in the primary memory (core) cache.
1991  */
1992 int
1993 as_incore(struct as *as, caddr_t addr,
1994     size_t size, char *vec, size_t *sizep)
1995 {
1996 	struct seg *seg;
1997 	size_t ssize;
1998 	caddr_t raddr;		/* rounded down addr */
1999 	size_t rsize;		/* rounded up size */
2000 	size_t isize;			/* iteration size */
2001 	int error = 0;		/* result, assume success */
2002 
2003 	*sizep = 0;
2004 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2005 	rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
2006 		(size_t)raddr;
2007 
2008 	if (raddr + rsize < raddr)		/* check for wraparound */
2009 		return (ENOMEM);
2010 
2011 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2012 	seg = as_segat(as, raddr);
2013 	if (seg == NULL) {
2014 		AS_LOCK_EXIT(as, &as->a_lock);
2015 		return (-1);
2016 	}
2017 
2018 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2019 		if (raddr >= seg->s_base + seg->s_size) {
2020 			seg = AS_SEGNEXT(as, seg);
2021 			if (seg == NULL || raddr != seg->s_base) {
2022 				error = -1;
2023 				break;
2024 			}
2025 		}
2026 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2027 			ssize = seg->s_base + seg->s_size - raddr;
2028 		else
2029 			ssize = rsize;
2030 		*sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
2031 		if (isize != ssize) {
2032 			error = -1;
2033 			break;
2034 		}
2035 		vec += btopr(ssize);
2036 	}
2037 	AS_LOCK_EXIT(as, &as->a_lock);
2038 	return (error);
2039 }
2040 
2041 static void
2042 as_segunlock(struct seg *seg, caddr_t addr, int attr,
2043 	ulong_t *bitmap, size_t position, size_t npages)
2044 {
2045 	caddr_t	range_start;
2046 	size_t	pos1 = position;
2047 	size_t	pos2;
2048 	size_t	size;
2049 	size_t  end_pos = npages + position;
2050 
2051 	while (bt_range(bitmap, &pos1, &pos2, end_pos)) {
2052 		size = ptob((pos2 - pos1));
2053 		range_start = (caddr_t)((uintptr_t)addr +
2054 			ptob(pos1 - position));
2055 
2056 		(void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK,
2057 			(ulong_t *)NULL, (size_t)NULL);
2058 		pos1 = pos2;
2059 	}
2060 }
2061 
2062 static void
2063 as_unlockerr(struct as *as, int attr, ulong_t *mlock_map,
2064 	caddr_t raddr, size_t rsize)
2065 {
2066 	struct seg *seg = as_segat(as, raddr);
2067 	size_t ssize;
2068 
2069 	while (rsize != 0) {
2070 		if (raddr >= seg->s_base + seg->s_size)
2071 			seg = AS_SEGNEXT(as, seg);
2072 
2073 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2074 			ssize = seg->s_base + seg->s_size - raddr;
2075 		else
2076 			ssize = rsize;
2077 
2078 		as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize));
2079 
2080 		rsize -= ssize;
2081 		raddr += ssize;
2082 	}
2083 }
2084 
2085 /*
2086  * Cache control operations over the interval [addr, addr + size) in
2087  * address space "as".
2088  */
2089 /*ARGSUSED*/
2090 int
2091 as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
2092     uintptr_t arg, ulong_t *lock_map, size_t pos)
2093 {
2094 	struct seg *seg;	/* working segment */
2095 	caddr_t raddr;		/* rounded down addr */
2096 	caddr_t initraddr;	/* saved initial rounded down addr */
2097 	size_t rsize;		/* rounded up size */
2098 	size_t initrsize;	/* saved initial rounded up size */
2099 	size_t ssize;		/* size of seg */
2100 	int error = 0;			/* result */
2101 	size_t mlock_size;	/* size of bitmap */
2102 	ulong_t *mlock_map;	/* pointer to bitmap used */
2103 				/* to represent the locked */
2104 				/* pages. */
2105 retry:
2106 	if (error == IE_RETRY)
2107 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2108 	else
2109 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2110 
2111 	/*
2112 	 * If these are address space lock/unlock operations, loop over
2113 	 * all segments in the address space, as appropriate.
2114 	 */
2115 	if (func == MC_LOCKAS) {
2116 		size_t npages, idx;
2117 		size_t rlen = 0;	/* rounded as length */
2118 
2119 		idx = pos;
2120 
2121 		if (arg & MCL_FUTURE) {
2122 			mutex_enter(&as->a_contents);
2123 			AS_SETPGLCK(as);
2124 			mutex_exit(&as->a_contents);
2125 		}
2126 		if ((arg & MCL_CURRENT) == 0) {
2127 			AS_LOCK_EXIT(as, &as->a_lock);
2128 			return (0);
2129 		}
2130 
2131 		seg = AS_SEGFIRST(as);
2132 		if (seg == NULL) {
2133 			AS_LOCK_EXIT(as, &as->a_lock);
2134 			return (0);
2135 		}
2136 
2137 		do {
2138 			raddr = (caddr_t)((uintptr_t)seg->s_base &
2139 			    (uintptr_t)PAGEMASK);
2140 			rlen += (((uintptr_t)(seg->s_base + seg->s_size) +
2141 				PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr;
2142 		} while ((seg = AS_SEGNEXT(as, seg)) != NULL);
2143 
2144 		mlock_size = BT_BITOUL(btopr(rlen));
2145 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2146 			sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2147 				AS_LOCK_EXIT(as, &as->a_lock);
2148 				return (EAGAIN);
2149 		}
2150 
2151 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2152 			error = SEGOP_LOCKOP(seg, seg->s_base,
2153 			    seg->s_size, attr, MC_LOCK, mlock_map, pos);
2154 			if (error != 0)
2155 				break;
2156 			pos += seg_pages(seg);
2157 		}
2158 
2159 		if (error) {
2160 			for (seg = AS_SEGFIRST(as); seg != NULL;
2161 				seg = AS_SEGNEXT(as, seg)) {
2162 
2163 				raddr = (caddr_t)((uintptr_t)seg->s_base &
2164 					(uintptr_t)PAGEMASK);
2165 				npages = seg_pages(seg);
2166 				as_segunlock(seg, raddr, attr, mlock_map,
2167 					idx, npages);
2168 				idx += npages;
2169 			}
2170 		}
2171 
2172 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2173 		AS_LOCK_EXIT(as, &as->a_lock);
2174 		goto lockerr;
2175 	} else if (func == MC_UNLOCKAS) {
2176 		mutex_enter(&as->a_contents);
2177 		AS_CLRPGLCK(as);
2178 		mutex_exit(&as->a_contents);
2179 
2180 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2181 			error = SEGOP_LOCKOP(seg, seg->s_base,
2182 			    seg->s_size, attr, MC_UNLOCK, NULL, 0);
2183 			if (error != 0)
2184 				break;
2185 		}
2186 
2187 		AS_LOCK_EXIT(as, &as->a_lock);
2188 		goto lockerr;
2189 	}
2190 
2191 	/*
2192 	 * Normalize addresses and sizes.
2193 	 */
2194 	initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2195 	initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2196 		(size_t)raddr;
2197 
2198 	if (raddr + rsize < raddr) {		/* check for wraparound */
2199 		AS_LOCK_EXIT(as, &as->a_lock);
2200 		return (ENOMEM);
2201 	}
2202 
2203 	/*
2204 	 * Get initial segment.
2205 	 */
2206 	if ((seg = as_segat(as, raddr)) == NULL) {
2207 		AS_LOCK_EXIT(as, &as->a_lock);
2208 		return (ENOMEM);
2209 	}
2210 
2211 	if (func == MC_LOCK) {
2212 		mlock_size = BT_BITOUL(btopr(rsize));
2213 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2214 			sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2215 				AS_LOCK_EXIT(as, &as->a_lock);
2216 				return (EAGAIN);
2217 		}
2218 	}
2219 
2220 	/*
2221 	 * Loop over all segments.  If a hole in the address range is
2222 	 * discovered, then fail.  For each segment, perform the appropriate
2223 	 * control operation.
2224 	 */
2225 	while (rsize != 0) {
2226 
2227 		/*
2228 		 * Make sure there's no hole, calculate the portion
2229 		 * of the next segment to be operated over.
2230 		 */
2231 		if (raddr >= seg->s_base + seg->s_size) {
2232 			seg = AS_SEGNEXT(as, seg);
2233 			if (seg == NULL || raddr != seg->s_base) {
2234 				if (func == MC_LOCK) {
2235 					as_unlockerr(as, attr, mlock_map,
2236 						initraddr, initrsize - rsize);
2237 					kmem_free(mlock_map,
2238 						mlock_size * sizeof (ulong_t));
2239 				}
2240 				AS_LOCK_EXIT(as, &as->a_lock);
2241 				return (ENOMEM);
2242 			}
2243 		}
2244 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2245 			ssize = seg->s_base + seg->s_size - raddr;
2246 		else
2247 			ssize = rsize;
2248 
2249 		/*
2250 		 * Dispatch on specific function.
2251 		 */
2252 		switch (func) {
2253 
2254 		/*
2255 		 * Synchronize cached data from mappings with backing
2256 		 * objects.
2257 		 */
2258 		case MC_SYNC:
2259 			if (error = SEGOP_SYNC(seg, raddr, ssize,
2260 			    attr, (uint_t)arg)) {
2261 				AS_LOCK_EXIT(as, &as->a_lock);
2262 				return (error);
2263 			}
2264 			break;
2265 
2266 		/*
2267 		 * Lock pages in memory.
2268 		 */
2269 		case MC_LOCK:
2270 			if (error = SEGOP_LOCKOP(seg, raddr, ssize,
2271 				attr, func, mlock_map, pos)) {
2272 				as_unlockerr(as, attr, mlock_map, initraddr,
2273 					initrsize - rsize + ssize);
2274 				kmem_free(mlock_map, mlock_size *
2275 					sizeof (ulong_t));
2276 				AS_LOCK_EXIT(as, &as->a_lock);
2277 				goto lockerr;
2278 			}
2279 			break;
2280 
2281 		/*
2282 		 * Unlock mapped pages.
2283 		 */
2284 		case MC_UNLOCK:
2285 			(void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
2286 				(ulong_t *)NULL, (size_t)NULL);
2287 			break;
2288 
2289 		/*
2290 		 * Store VM advise for mapped pages in segment layer.
2291 		 */
2292 		case MC_ADVISE:
2293 			error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg);
2294 
2295 			/*
2296 			 * Check for regular errors and special retry error
2297 			 */
2298 			if (error) {
2299 				if (error == IE_RETRY) {
2300 					/*
2301 					 * Need to acquire writers lock, so
2302 					 * have to drop readers lock and start
2303 					 * all over again
2304 					 */
2305 					AS_LOCK_EXIT(as, &as->a_lock);
2306 					goto retry;
2307 				} else if (error == IE_REATTACH) {
2308 					/*
2309 					 * Find segment for current address
2310 					 * because current segment just got
2311 					 * split or concatenated
2312 					 */
2313 					seg = as_segat(as, raddr);
2314 					if (seg == NULL) {
2315 						AS_LOCK_EXIT(as, &as->a_lock);
2316 						return (ENOMEM);
2317 					}
2318 				} else {
2319 					/*
2320 					 * Regular error
2321 					 */
2322 					AS_LOCK_EXIT(as, &as->a_lock);
2323 					return (error);
2324 				}
2325 			}
2326 			break;
2327 
2328 		/*
2329 		 * Can't happen.
2330 		 */
2331 		default:
2332 			panic("as_ctl: bad operation %d", func);
2333 			/*NOTREACHED*/
2334 		}
2335 
2336 		rsize -= ssize;
2337 		raddr += ssize;
2338 	}
2339 
2340 	if (func == MC_LOCK)
2341 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2342 	AS_LOCK_EXIT(as, &as->a_lock);
2343 	return (0);
2344 lockerr:
2345 
2346 	/*
2347 	 * If the lower levels returned EDEADLK for a segment lockop,
2348 	 * it means that we should retry the operation.  Let's wait
2349 	 * a bit also to let the deadlock causing condition clear.
2350 	 * This is part of a gross hack to work around a design flaw
2351 	 * in the ufs/sds logging code and should go away when the
2352 	 * logging code is re-designed to fix the problem. See bug
2353 	 * 4125102 for details of the problem.
2354 	 */
2355 	if (error == EDEADLK) {
2356 		delay(deadlk_wait);
2357 		error = 0;
2358 		goto retry;
2359 	}
2360 	return (error);
2361 }
2362 
2363 /*
2364  * Special code for exec to move the stack segment from its interim
2365  * place in the old address to the right place in the new address space.
2366  */
2367 /*ARGSUSED*/
2368 int
2369 as_exec(struct as *oas, caddr_t ostka, size_t stksz,
2370     struct as *nas, caddr_t nstka, uint_t hatflag)
2371 {
2372 	struct seg *stkseg;
2373 
2374 	AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER);
2375 	stkseg = as_segat(oas, ostka);
2376 	stkseg = as_removeseg(oas, stkseg);
2377 	ASSERT(stkseg != NULL);
2378 	ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz);
2379 	stkseg->s_as = nas;
2380 	stkseg->s_base = nstka;
2381 
2382 	/*
2383 	 * It's ok to lock the address space we are about to exec to.
2384 	 */
2385 	AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER);
2386 	ASSERT(avl_numnodes(&nas->a_wpage) == 0);
2387 	nas->a_size += stkseg->s_size;
2388 	oas->a_size -= stkseg->s_size;
2389 	(void) as_addseg(nas, stkseg);
2390 	AS_LOCK_EXIT(nas, &nas->a_lock);
2391 	AS_LOCK_EXIT(oas, &oas->a_lock);
2392 	return (0);
2393 }
2394 
2395 static int
2396 f_decode(faultcode_t fault_err)
2397 {
2398 	int error = 0;
2399 
2400 	switch (FC_CODE(fault_err)) {
2401 	case FC_OBJERR:
2402 		error = FC_ERRNO(fault_err);
2403 		break;
2404 	case FC_PROT:
2405 		error = EACCES;
2406 		break;
2407 	default:
2408 		error = EFAULT;
2409 		break;
2410 	}
2411 	return (error);
2412 }
2413 
2414 /*
2415  * lock pages in a given address space. Return shadow list. If
2416  * the list is NULL, the MMU mapping is also locked.
2417  */
2418 int
2419 as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
2420     size_t size, enum seg_rw rw)
2421 {
2422 	size_t rsize;
2423 	caddr_t base;
2424 	caddr_t raddr;
2425 	faultcode_t fault_err;
2426 	struct seg *seg;
2427 	int res;
2428 	int prefaulted = 0;
2429 
2430 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
2431 	    "as_pagelock_start: addr %p size %ld", addr, size);
2432 
2433 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2434 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2435 		(size_t)raddr;
2436 top:
2437 	/*
2438 	 * if the request crosses two segments let
2439 	 * as_fault handle it.
2440 	 */
2441 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2442 	seg = as_findseg(as, addr, 0);
2443 	if ((seg == NULL) || ((base = seg->s_base) > addr) ||
2444 	    (addr + size) > base + seg->s_size) {
2445 		AS_LOCK_EXIT(as, &as->a_lock);
2446 		goto slow;
2447 	}
2448 
2449 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
2450 	    "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
2451 
2452 	/*
2453 	 * try to lock pages and pass back shadow list
2454 	 */
2455 	res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
2456 
2457 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
2458 	AS_LOCK_EXIT(as, &as->a_lock);
2459 	if (res == 0) {
2460 		return (0);
2461 	} else if (res == ENOTSUP || prefaulted) {
2462 		/*
2463 		 * (1) segment driver doesn't support PAGELOCK fastpath, or
2464 		 * (2) we've already tried fast path unsuccessfully after
2465 		 *    faulting in the addr range below; system might be
2466 		 *    thrashing or there may not be enough availrmem.
2467 		 */
2468 		goto slow;
2469 	}
2470 
2471 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_FAULT_START,
2472 	    "as_fault_start: addr %p size %ld", addr, size);
2473 
2474 	/*
2475 	 * we might get here because of some COW fault or non
2476 	 * existing page. Let as_fault deal with it. Just load
2477 	 * the page, don't lock the MMU mapping.
2478 	 */
2479 	fault_err = as_fault(as->a_hat, as, addr, size, F_INVAL, rw);
2480 	if (fault_err != 0) {
2481 		return (f_decode(fault_err));
2482 	}
2483 
2484 	prefaulted = 1;
2485 
2486 	/*
2487 	 * try fast path again; since we've dropped a_lock,
2488 	 * we need to try the dance from the start to see if
2489 	 * the addr range is still valid.
2490 	 */
2491 	goto top;
2492 slow:
2493 	/*
2494 	 * load the page and lock the MMU mapping.
2495 	 */
2496 	fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
2497 	if (fault_err != 0) {
2498 		return (f_decode(fault_err));
2499 	}
2500 	*ppp = NULL;
2501 
2502 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
2503 	return (0);
2504 }
2505 
2506 /*
2507  * unlock pages in a given address range
2508  */
2509 void
2510 as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
2511     enum seg_rw rw)
2512 {
2513 	struct seg *seg;
2514 	size_t rsize;
2515 	caddr_t raddr;
2516 
2517 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
2518 	    "as_pageunlock_start: addr %p size %ld", addr, size);
2519 
2520 	/*
2521 	 * if the shadow list is NULL, as_pagelock was
2522 	 * falling back to as_fault
2523 	 */
2524 	if (pp == NULL) {
2525 		(void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
2526 		return;
2527 	}
2528 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2529 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2530 		(size_t)raddr;
2531 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2532 	seg = as_findseg(as, addr, 0);
2533 	ASSERT(seg);
2534 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
2535 	    "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
2536 	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
2537 	AS_LOCK_EXIT(as, &as->a_lock);
2538 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
2539 }
2540 
2541 /*
2542  * reclaim cached pages in a given address range
2543  */
2544 void
2545 as_pagereclaim(struct as *as, struct page **pp, caddr_t addr,
2546     size_t size, enum seg_rw rw)
2547 {
2548 	struct seg *seg;
2549 	size_t rsize;
2550 	caddr_t raddr;
2551 
2552 	ASSERT(AS_READ_HELD(as, &as->a_lock));
2553 	ASSERT(pp != NULL);
2554 
2555 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2556 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2557 		(size_t)raddr;
2558 	seg = as_findseg(as, addr, 0);
2559 	ASSERT(seg);
2560 	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGERECLAIM, rw);
2561 }
2562 
2563 #define	MAXPAGEFLIP	4
2564 #define	MAXPAGEFLIPSIZ	MAXPAGEFLIP*PAGESIZE
2565 
2566 int
2567 as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2568     boolean_t wait)
2569 {
2570 	struct seg *seg;
2571 	size_t ssize;
2572 	caddr_t raddr;			/* rounded down addr */
2573 	size_t rsize;			/* rounded up size */
2574 	int error = 0;
2575 	size_t pgsz = page_get_pagesize(szc);
2576 
2577 setpgsz_top:
2578 	if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) {
2579 		return (EINVAL);
2580 	}
2581 
2582 	raddr = addr;
2583 	rsize = size;
2584 
2585 	if (raddr + rsize < raddr)		/* check for wraparound */
2586 		return (ENOMEM);
2587 
2588 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2589 	as_clearwatchprot(as, raddr, rsize);
2590 	seg = as_segat(as, raddr);
2591 	if (seg == NULL) {
2592 		as_setwatch(as);
2593 		AS_LOCK_EXIT(as, &as->a_lock);
2594 		return (ENOMEM);
2595 	}
2596 
2597 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2598 		if (raddr >= seg->s_base + seg->s_size) {
2599 			seg = AS_SEGNEXT(as, seg);
2600 			if (seg == NULL || raddr != seg->s_base) {
2601 				error = ENOMEM;
2602 				break;
2603 			}
2604 		}
2605 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2606 			ssize = seg->s_base + seg->s_size - raddr;
2607 		} else {
2608 			ssize = rsize;
2609 		}
2610 
2611 		error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2612 
2613 		if (error == IE_NOMEM) {
2614 			error = EAGAIN;
2615 			break;
2616 		}
2617 
2618 		if (error == IE_RETRY) {
2619 			AS_LOCK_EXIT(as, &as->a_lock);
2620 			goto setpgsz_top;
2621 		}
2622 
2623 		if (error == ENOTSUP) {
2624 			error = EINVAL;
2625 			break;
2626 		}
2627 
2628 		if (wait && (error == EAGAIN)) {
2629 			/*
2630 			 * Memory is currently locked.  It must be unlocked
2631 			 * before this operation can succeed through a retry.
2632 			 * The possible reasons for locked memory and
2633 			 * corresponding strategies for unlocking are:
2634 			 * (1) Normal I/O
2635 			 *	wait for a signal that the I/O operation
2636 			 *	has completed and the memory is unlocked.
2637 			 * (2) Asynchronous I/O
2638 			 *	The aio subsystem does not unlock pages when
2639 			 *	the I/O is completed. Those pages are unlocked
2640 			 *	when the application calls aiowait/aioerror.
2641 			 *	So, to prevent blocking forever, cv_broadcast()
2642 			 *	is done to wake up aio_cleanup_thread.
2643 			 *	Subsequently, segvn_reclaim will be called, and
2644 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
2645 			 * (3) Long term page locking:
2646 			 *	This is not relevant for as_setpagesize()
2647 			 *	because we cannot change the page size for
2648 			 *	driver memory. The attempt to do so will
2649 			 *	fail with a different error than EAGAIN so
2650 			 *	there's no need to trigger as callbacks like
2651 			 *	as_unmap, as_setprot or as_free would do.
2652 			 */
2653 			mutex_enter(&as->a_contents);
2654 			if (AS_ISUNMAPWAIT(as) == 0) {
2655 				cv_broadcast(&as->a_cv);
2656 			}
2657 			AS_SETUNMAPWAIT(as);
2658 			AS_LOCK_EXIT(as, &as->a_lock);
2659 			while (AS_ISUNMAPWAIT(as)) {
2660 				cv_wait(&as->a_cv, &as->a_contents);
2661 			}
2662 			mutex_exit(&as->a_contents);
2663 			goto setpgsz_top;
2664 		} else if (error != 0) {
2665 			break;
2666 		}
2667 	}
2668 	as_setwatch(as);
2669 	AS_LOCK_EXIT(as, &as->a_lock);
2670 	return (error);
2671 }
2672 
2673 /*
2674  * Setup all of the uninitialized watched pages that we can.
2675  */
2676 void
2677 as_setwatch(struct as *as)
2678 {
2679 	struct watched_page *pwp;
2680 	struct seg *seg;
2681 	caddr_t vaddr;
2682 	uint_t prot;
2683 	int  err, retrycnt;
2684 
2685 	if (avl_numnodes(&as->a_wpage) == 0)
2686 		return;
2687 
2688 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2689 
2690 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
2691 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
2692 		retrycnt = 0;
2693 	retry:
2694 		vaddr = pwp->wp_vaddr;
2695 		if (pwp->wp_oprot != 0 ||	/* already set up */
2696 		    (seg = as_segat(as, vaddr)) == NULL ||
2697 		    SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
2698 			continue;
2699 
2700 		pwp->wp_oprot = prot;
2701 		if (pwp->wp_read)
2702 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2703 		if (pwp->wp_write)
2704 			prot &= ~PROT_WRITE;
2705 		if (pwp->wp_exec)
2706 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2707 		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) {
2708 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
2709 			if (err == IE_RETRY) {
2710 				pwp->wp_oprot = 0;
2711 				ASSERT(retrycnt == 0);
2712 				retrycnt++;
2713 				goto retry;
2714 			}
2715 		}
2716 		pwp->wp_prot = prot;
2717 	}
2718 }
2719 
2720 /*
2721  * Clear all of the watched pages in the address space.
2722  */
2723 void
2724 as_clearwatch(struct as *as)
2725 {
2726 	struct watched_page *pwp;
2727 	struct seg *seg;
2728 	caddr_t vaddr;
2729 	uint_t prot;
2730 	int err, retrycnt;
2731 
2732 	if (avl_numnodes(&as->a_wpage) == 0)
2733 		return;
2734 
2735 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2736 
2737 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
2738 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
2739 		retrycnt = 0;
2740 	retry:
2741 		vaddr = pwp->wp_vaddr;
2742 		if (pwp->wp_oprot == 0 ||	/* not set up */
2743 		    (seg = as_segat(as, vaddr)) == NULL)
2744 			continue;
2745 
2746 		if ((prot = pwp->wp_oprot) != pwp->wp_prot) {
2747 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
2748 			if (err == IE_RETRY) {
2749 				ASSERT(retrycnt == 0);
2750 				retrycnt++;
2751 				goto retry;
2752 			}
2753 		}
2754 		pwp->wp_oprot = 0;
2755 		pwp->wp_prot = 0;
2756 	}
2757 }
2758 
2759 /*
2760  * Force a new setup for all the watched pages in the range.
2761  */
2762 static void
2763 as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
2764 {
2765 	struct watched_page *pwp;
2766 	struct watched_page tpw;
2767 	caddr_t eaddr = addr + size;
2768 	caddr_t vaddr;
2769 	struct seg *seg;
2770 	int err, retrycnt;
2771 	uint_t	wprot;
2772 	avl_index_t where;
2773 
2774 	if (avl_numnodes(&as->a_wpage) == 0)
2775 		return;
2776 
2777 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2778 
2779 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2780 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
2781 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
2782 
2783 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
2784 		retrycnt = 0;
2785 		vaddr = pwp->wp_vaddr;
2786 
2787 		wprot = prot;
2788 		if (pwp->wp_read)
2789 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2790 		if (pwp->wp_write)
2791 			wprot &= ~PROT_WRITE;
2792 		if (pwp->wp_exec)
2793 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
2794 		if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) {
2795 		retry:
2796 			seg = as_segat(as, vaddr);
2797 			if (seg == NULL) {
2798 				panic("as_setwatchprot: no seg");
2799 				/*NOTREACHED*/
2800 			}
2801 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot);
2802 			if (err == IE_RETRY) {
2803 				ASSERT(retrycnt == 0);
2804 				retrycnt++;
2805 				goto retry;
2806 			}
2807 		}
2808 		pwp->wp_oprot = prot;
2809 		pwp->wp_prot = wprot;
2810 
2811 		pwp = AVL_NEXT(&as->a_wpage, pwp);
2812 	}
2813 }
2814 
2815 /*
2816  * Clear all of the watched pages in the range.
2817  */
2818 static void
2819 as_clearwatchprot(struct as *as, caddr_t addr, size_t size)
2820 {
2821 	caddr_t eaddr = addr + size;
2822 	struct watched_page *pwp;
2823 	struct watched_page tpw;
2824 	uint_t prot;
2825 	struct seg *seg;
2826 	int err, retrycnt;
2827 	avl_index_t where;
2828 
2829 	if (avl_numnodes(&as->a_wpage) == 0)
2830 		return;
2831 
2832 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2833 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
2834 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
2835 
2836 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
2837 
2838 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
2839 		ASSERT(addr >= pwp->wp_vaddr);
2840 
2841 		if ((prot = pwp->wp_oprot) != 0) {
2842 			retrycnt = 0;
2843 
2844 			if (prot != pwp->wp_prot) {
2845 			retry:
2846 				seg = as_segat(as, pwp->wp_vaddr);
2847 				if (seg == NULL)
2848 					continue;
2849 				err = SEGOP_SETPROT(seg, pwp->wp_vaddr,
2850 				    PAGESIZE, prot);
2851 				if (err == IE_RETRY) {
2852 					ASSERT(retrycnt == 0);
2853 					retrycnt++;
2854 					goto retry;
2855 
2856 				}
2857 			}
2858 			pwp->wp_oprot = 0;
2859 			pwp->wp_prot = 0;
2860 		}
2861 
2862 		pwp = AVL_NEXT(&as->a_wpage, pwp);
2863 	}
2864 }
2865 
2866 void
2867 as_signal_proc(struct as *as, k_siginfo_t *siginfo)
2868 {
2869 	struct proc *p;
2870 
2871 	mutex_enter(&pidlock);
2872 	for (p = practive; p; p = p->p_next) {
2873 		if (p->p_as == as) {
2874 			mutex_enter(&p->p_lock);
2875 			if (p->p_as == as)
2876 				sigaddq(p, NULL, siginfo, KM_NOSLEEP);
2877 			mutex_exit(&p->p_lock);
2878 		}
2879 	}
2880 	mutex_exit(&pidlock);
2881 }
2882 
2883 /*
2884  * return memory object ID
2885  */
2886 int
2887 as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
2888 {
2889 	struct seg	*seg;
2890 	int		sts;
2891 
2892 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2893 	seg = as_segat(as, addr);
2894 	if (seg == NULL) {
2895 		AS_LOCK_EXIT(as, &as->a_lock);
2896 		return (EFAULT);
2897 	}
2898 	/*
2899 	 * catch old drivers which may not support getmemid
2900 	 */
2901 	if (seg->s_ops->getmemid == NULL) {
2902 		AS_LOCK_EXIT(as, &as->a_lock);
2903 		return (ENODEV);
2904 	}
2905 
2906 	sts = SEGOP_GETMEMID(seg, addr, memidp);
2907 
2908 	AS_LOCK_EXIT(as, &as->a_lock);
2909 	return (sts);
2910 }
2911