xref: /titanic_51/usr/src/uts/common/vm/vm_page.c (revision 672986541be54a7a471bb088e60780c37e371d7e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989  AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 /*
30  * University Copyright- Copyright (c) 1982, 1986, 1988
31  * The Regents of the University of California
32  * All Rights Reserved
33  *
34  * University Acknowledgment- Portions of this document are derived from
35  * software developed by the University of California, Berkeley, and its
36  * contributors.
37  */
38 
39 #pragma ident	"%Z%%M%	%I%	%E% SMI"
40 
41 /*
42  * VM - physical page management.
43  */
44 
45 #include <sys/types.h>
46 #include <sys/t_lock.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/errno.h>
50 #include <sys/time.h>
51 #include <sys/vnode.h>
52 #include <sys/vm.h>
53 #include <sys/vtrace.h>
54 #include <sys/swap.h>
55 #include <sys/cmn_err.h>
56 #include <sys/tuneable.h>
57 #include <sys/sysmacros.h>
58 #include <sys/cpuvar.h>
59 #include <sys/callb.h>
60 #include <sys/debug.h>
61 #include <sys/tnf_probe.h>
62 #include <sys/condvar_impl.h>
63 #include <sys/mem_config.h>
64 #include <sys/mem_cage.h>
65 #include <sys/kmem.h>
66 #include <sys/atomic.h>
67 #include <sys/strlog.h>
68 #include <sys/mman.h>
69 #include <sys/ontrap.h>
70 #include <sys/lgrp.h>
71 #include <sys/vfs.h>
72 
73 #include <vm/hat.h>
74 #include <vm/anon.h>
75 #include <vm/page.h>
76 #include <vm/seg.h>
77 #include <vm/pvn.h>
78 #include <vm/seg_kmem.h>
79 #include <vm/vm_dep.h>
80 #include <sys/vm_usage.h>
81 #include <fs/fs_subr.h>
82 #include <sys/ddi.h>
83 #include <sys/modctl.h>
84 
85 static int nopageage = 0;
86 
87 static pgcnt_t max_page_get;	/* max page_get request size in pages */
88 pgcnt_t total_pages = 0;	/* total number of pages (used by /proc) */
89 
90 /*
91  * freemem_lock protects all freemem variables:
92  * availrmem. Also this lock protects the globals which track the
93  * availrmem changes for accurate kernel footprint calculation.
94  * See below for an explanation of these
95  * globals.
96  */
97 kmutex_t freemem_lock;
98 pgcnt_t availrmem;
99 pgcnt_t availrmem_initial;
100 
101 /*
102  * These globals track availrmem changes to get a more accurate
103  * estimate of tke kernel size. Historically pp_kernel is used for
104  * kernel size and is based on availrmem. But availrmem is adjusted for
105  * locked pages in the system not just for kernel locked pages.
106  * These new counters will track the pages locked through segvn and
107  * by explicit user locking.
108  *
109  * segvn_pages_locked : This keeps track on a global basis how many pages
110  * are currently locked because of I/O.
111  *
112  * pages_locked : How many pages are locked becuase of user specified
113  * locking through mlock or plock.
114  *
115  * pages_useclaim,pages_claimed : These two variables track the
116  * cliam adjustments because of the protection changes on a segvn segment.
117  *
118  * All these globals are protected by the same lock which protects availrmem.
119  */
120 pgcnt_t segvn_pages_locked;
121 pgcnt_t pages_locked;
122 pgcnt_t pages_useclaim;
123 pgcnt_t pages_claimed;
124 
125 
126 /*
127  * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
128  */
129 static kmutex_t	new_freemem_lock;
130 static uint_t	freemem_wait;	/* someone waiting for freemem */
131 static kcondvar_t freemem_cv;
132 
133 /*
134  * The logical page free list is maintained as two lists, the 'free'
135  * and the 'cache' lists.
136  * The free list contains those pages that should be reused first.
137  *
138  * The implementation of the lists is machine dependent.
139  * page_get_freelist(), page_get_cachelist(),
140  * page_list_sub(), and page_list_add()
141  * form the interface to the machine dependent implementation.
142  *
143  * Pages with p_free set are on the cache list.
144  * Pages with p_free and p_age set are on the free list,
145  *
146  * A page may be locked while on either list.
147  */
148 
149 /*
150  * free list accounting stuff.
151  *
152  *
153  * Spread out the value for the number of pages on the
154  * page free and page cache lists.  If there is just one
155  * value, then it must be under just one lock.
156  * The lock contention and cache traffic are a real bother.
157  *
158  * When we acquire and then drop a single pcf lock
159  * we can start in the middle of the array of pcf structures.
160  * If we acquire more than one pcf lock at a time, we need to
161  * start at the front to avoid deadlocking.
162  *
163  * pcf_count holds the number of pages in each pool.
164  *
165  * pcf_block is set when page_create_get_something() has asked the
166  * PSM page freelist and page cachelist routines without specifying
167  * a color and nothing came back.  This is used to block anything
168  * else from moving pages from one list to the other while the
169  * lists are searched again.  If a page is freeed while pcf_block is
170  * set, then pcf_reserve is incremented.  pcgs_unblock() takes care
171  * of clearning pcf_block, doing the wakeups, etc.
172  */
173 
174 #if NCPU <= 4
175 #define	PAD	2
176 #define	PCF_FANOUT	4
177 static	uint_t	pcf_mask = PCF_FANOUT - 1;
178 #else
179 #define	PAD	10
180 #ifdef sun4v
181 #define	PCF_FANOUT	32
182 #else
183 #define	PCF_FANOUT	128
184 #endif
185 static	uint_t	pcf_mask = PCF_FANOUT - 1;
186 #endif
187 
188 struct pcf {
189 	kmutex_t	pcf_lock;	/* protects the structure */
190 	uint_t		pcf_count;	/* page count */
191 	uint_t		pcf_wait;	/* number of waiters */
192 	uint_t		pcf_block; 	/* pcgs flag to page_free() */
193 	uint_t		pcf_reserve; 	/* pages freed after pcf_block set */
194 	uint_t		pcf_fill[PAD];	/* to line up on the caches */
195 };
196 
197 static struct	pcf	pcf[PCF_FANOUT];
198 #define	PCF_INDEX()	((CPU->cpu_id) & (pcf_mask))
199 
200 kmutex_t	pcgs_lock;		/* serializes page_create_get_ */
201 kmutex_t	pcgs_cagelock;		/* serializes NOSLEEP cage allocs */
202 kmutex_t	pcgs_wait_lock;		/* used for delay in pcgs */
203 static kcondvar_t	pcgs_cv;	/* cv for delay in pcgs */
204 
205 #ifdef VM_STATS
206 
207 /*
208  * No locks, but so what, they are only statistics.
209  */
210 
211 static struct page_tcnt {
212 	int	pc_free_cache;		/* free's into cache list */
213 	int	pc_free_dontneed;	/* free's with dontneed */
214 	int	pc_free_pageout;	/* free's from pageout */
215 	int	pc_free_free;		/* free's into free list */
216 	int	pc_free_pages;		/* free's into large page free list */
217 	int	pc_destroy_pages;	/* large page destroy's */
218 	int	pc_get_cache;		/* get's from cache list */
219 	int	pc_get_free;		/* get's from free list */
220 	int	pc_reclaim;		/* reclaim's */
221 	int	pc_abortfree;		/* abort's of free pages */
222 	int	pc_find_hit;		/* find's that find page */
223 	int	pc_find_miss;		/* find's that don't find page */
224 	int	pc_destroy_free;	/* # of free pages destroyed */
225 #define	PC_HASH_CNT	(4*PAGE_HASHAVELEN)
226 	int	pc_find_hashlen[PC_HASH_CNT+1];
227 	int	pc_addclaim_pages;
228 	int	pc_subclaim_pages;
229 	int	pc_free_replacement_page[2];
230 	int	pc_try_demote_pages[6];
231 	int	pc_demote_pages[2];
232 } pagecnt;
233 
234 uint_t	hashin_count;
235 uint_t	hashin_not_held;
236 uint_t	hashin_already;
237 
238 uint_t	hashout_count;
239 uint_t	hashout_not_held;
240 
241 uint_t	page_create_count;
242 uint_t	page_create_not_enough;
243 uint_t	page_create_not_enough_again;
244 uint_t	page_create_zero;
245 uint_t	page_create_hashout;
246 uint_t	page_create_page_lock_failed;
247 uint_t	page_create_trylock_failed;
248 uint_t	page_create_found_one;
249 uint_t	page_create_hashin_failed;
250 uint_t	page_create_dropped_phm;
251 
252 uint_t	page_create_new;
253 uint_t	page_create_exists;
254 uint_t	page_create_putbacks;
255 uint_t	page_create_overshoot;
256 
257 uint_t	page_reclaim_zero;
258 uint_t	page_reclaim_zero_locked;
259 
260 uint_t	page_rename_exists;
261 uint_t	page_rename_count;
262 
263 uint_t	page_lookup_cnt[20];
264 uint_t	page_lookup_nowait_cnt[10];
265 uint_t	page_find_cnt;
266 uint_t	page_exists_cnt;
267 uint_t	page_exists_forreal_cnt;
268 uint_t	page_lookup_dev_cnt;
269 uint_t	get_cachelist_cnt;
270 uint_t	page_create_cnt[10];
271 uint_t	alloc_pages[8];
272 uint_t	page_exphcontg[19];
273 uint_t  page_create_large_cnt[10];
274 
275 /*
276  * Collects statistics.
277  */
278 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
279 	uint_t	mylen = 0; \
280 			\
281 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \
282 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
283 			break; \
284 	} \
285 	if ((pp) != NULL) \
286 		pagecnt.pc_find_hit++; \
287 	else \
288 		pagecnt.pc_find_miss++; \
289 	if (mylen > PC_HASH_CNT) \
290 		mylen = PC_HASH_CNT; \
291 	pagecnt.pc_find_hashlen[mylen]++; \
292 }
293 
294 #else	/* VM_STATS */
295 
296 /*
297  * Don't collect statistics
298  */
299 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
300 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
301 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
302 			break; \
303 	} \
304 }
305 
306 #endif	/* VM_STATS */
307 
308 
309 
310 #ifdef DEBUG
311 #define	MEMSEG_SEARCH_STATS
312 #endif
313 
314 #ifdef MEMSEG_SEARCH_STATS
315 struct memseg_stats {
316     uint_t nsearch;
317     uint_t nlastwon;
318     uint_t nhashwon;
319     uint_t nnotfound;
320 } memseg_stats;
321 
322 #define	MEMSEG_STAT_INCR(v) \
323 	atomic_add_32(&memseg_stats.v, 1)
324 #else
325 #define	MEMSEG_STAT_INCR(x)
326 #endif
327 
328 struct memseg *memsegs;		/* list of memory segments */
329 
330 
331 static void page_init_mem_config(void);
332 static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
333 static void page_do_hashout(page_t *);
334 static void page_capture_init();
335 int page_capture_take_action(page_t *, uint_t, void *);
336 
337 static void page_demote_vp_pages(page_t *);
338 
339 /*
340  * vm subsystem related initialization
341  */
342 void
343 vm_init(void)
344 {
345 	boolean_t callb_vm_cpr(void *, int);
346 
347 	(void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
348 	page_init_mem_config();
349 	page_retire_init();
350 	vm_usage_init();
351 	page_capture_init();
352 }
353 
354 /*
355  * This function is called at startup and when memory is added or deleted.
356  */
357 void
358 init_pages_pp_maximum()
359 {
360 	static pgcnt_t p_min;
361 	static pgcnt_t pages_pp_maximum_startup;
362 	static pgcnt_t avrmem_delta;
363 	static int init_done;
364 	static int user_set;	/* true if set in /etc/system */
365 
366 	if (init_done == 0) {
367 
368 		/* If the user specified a value, save it */
369 		if (pages_pp_maximum != 0) {
370 			user_set = 1;
371 			pages_pp_maximum_startup = pages_pp_maximum;
372 		}
373 
374 		/*
375 		 * Setting of pages_pp_maximum is based first time
376 		 * on the value of availrmem just after the start-up
377 		 * allocations. To preserve this relationship at run
378 		 * time, use a delta from availrmem_initial.
379 		 */
380 		ASSERT(availrmem_initial >= availrmem);
381 		avrmem_delta = availrmem_initial - availrmem;
382 
383 		/* The allowable floor of pages_pp_maximum */
384 		p_min = tune.t_minarmem + 100;
385 
386 		/* Make sure we don't come through here again. */
387 		init_done = 1;
388 	}
389 	/*
390 	 * Determine pages_pp_maximum, the number of currently available
391 	 * pages (availrmem) that can't be `locked'. If not set by
392 	 * the user, we set it to 4% of the currently available memory
393 	 * plus 4MB.
394 	 * But we also insist that it be greater than tune.t_minarmem;
395 	 * otherwise a process could lock down a lot of memory, get swapped
396 	 * out, and never have enough to get swapped back in.
397 	 */
398 	if (user_set)
399 		pages_pp_maximum = pages_pp_maximum_startup;
400 	else
401 		pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
402 		    + btop(4 * 1024 * 1024);
403 
404 	if (pages_pp_maximum <= p_min) {
405 		pages_pp_maximum = p_min;
406 	}
407 }
408 
409 void
410 set_max_page_get(pgcnt_t target_total_pages)
411 {
412 	max_page_get = target_total_pages / 2;
413 }
414 
415 static pgcnt_t pending_delete;
416 
417 /*ARGSUSED*/
418 static void
419 page_mem_config_post_add(
420 	void *arg,
421 	pgcnt_t delta_pages)
422 {
423 	set_max_page_get(total_pages - pending_delete);
424 	init_pages_pp_maximum();
425 }
426 
427 /*ARGSUSED*/
428 static int
429 page_mem_config_pre_del(
430 	void *arg,
431 	pgcnt_t delta_pages)
432 {
433 	pgcnt_t nv;
434 
435 	nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
436 	set_max_page_get(total_pages - nv);
437 	return (0);
438 }
439 
440 /*ARGSUSED*/
441 static void
442 page_mem_config_post_del(
443 	void *arg,
444 	pgcnt_t delta_pages,
445 	int cancelled)
446 {
447 	pgcnt_t nv;
448 
449 	nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
450 	set_max_page_get(total_pages - nv);
451 	if (!cancelled)
452 		init_pages_pp_maximum();
453 }
454 
455 static kphysm_setup_vector_t page_mem_config_vec = {
456 	KPHYSM_SETUP_VECTOR_VERSION,
457 	page_mem_config_post_add,
458 	page_mem_config_pre_del,
459 	page_mem_config_post_del,
460 };
461 
462 static void
463 page_init_mem_config(void)
464 {
465 	int ret;
466 
467 	ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
468 	ASSERT(ret == 0);
469 }
470 
471 /*
472  * Evenly spread out the PCF counters for large free pages
473  */
474 static void
475 page_free_large_ctr(pgcnt_t npages)
476 {
477 	static struct pcf	*p = pcf;
478 	pgcnt_t			lump;
479 
480 	freemem += npages;
481 
482 	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;
483 
484 	while (npages > 0) {
485 
486 		ASSERT(!p->pcf_block);
487 
488 		if (lump < npages) {
489 			p->pcf_count += (uint_t)lump;
490 			npages -= lump;
491 		} else {
492 			p->pcf_count += (uint_t)npages;
493 			npages = 0;
494 		}
495 
496 		ASSERT(!p->pcf_wait);
497 
498 		if (++p > &pcf[PCF_FANOUT - 1])
499 			p = pcf;
500 	}
501 
502 	ASSERT(npages == 0);
503 }
504 
505 /*
506  * Add a physical chunk of memory to the system freee lists during startup.
507  * Platform specific startup() allocates the memory for the page structs.
508  *
509  * num	- number of page structures
510  * base - page number (pfn) to be associated with the first page.
511  *
512  * Since we are doing this during startup (ie. single threaded), we will
513  * use shortcut routines to avoid any locking overhead while putting all
514  * these pages on the freelists.
515  *
516  * NOTE: Any changes performed to page_free(), must also be performed to
517  *	 add_physmem() since this is how we initialize all page_t's at
518  *	 boot time.
519  */
520 void
521 add_physmem(
522 	page_t	*pp,
523 	pgcnt_t	num,
524 	pfn_t	pnum)
525 {
526 	page_t	*root = NULL;
527 	uint_t	szc = page_num_pagesizes() - 1;
528 	pgcnt_t	large = page_get_pagecnt(szc);
529 	pgcnt_t	cnt = 0;
530 
531 	TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
532 	    "add_physmem:pp %p num %lu", pp, num);
533 
534 	/*
535 	 * Arbitrarily limit the max page_get request
536 	 * to 1/2 of the page structs we have.
537 	 */
538 	total_pages += num;
539 	set_max_page_get(total_pages);
540 
541 	PLCNT_MODIFY_MAX(pnum, (long)num);
542 
543 	/*
544 	 * The physical space for the pages array
545 	 * representing ram pages has already been
546 	 * allocated.  Here we initialize each lock
547 	 * in the page structure, and put each on
548 	 * the free list
549 	 */
550 	for (; num; pp++, pnum++, num--) {
551 
552 		/*
553 		 * this needs to fill in the page number
554 		 * and do any other arch specific initialization
555 		 */
556 		add_physmem_cb(pp, pnum);
557 
558 		pp->p_lckcnt = 0;
559 		pp->p_cowcnt = 0;
560 		pp->p_slckcnt = 0;
561 
562 		/*
563 		 * Initialize the page lock as unlocked, since nobody
564 		 * can see or access this page yet.
565 		 */
566 		pp->p_selock = 0;
567 
568 		/*
569 		 * Initialize IO lock
570 		 */
571 		page_iolock_init(pp);
572 
573 		/*
574 		 * initialize other fields in the page_t
575 		 */
576 		PP_SETFREE(pp);
577 		page_clr_all_props(pp);
578 		PP_SETAGED(pp);
579 		pp->p_offset = (u_offset_t)-1;
580 		pp->p_next = pp;
581 		pp->p_prev = pp;
582 
583 		/*
584 		 * Simple case: System doesn't support large pages.
585 		 */
586 		if (szc == 0) {
587 			pp->p_szc = 0;
588 			page_free_at_startup(pp);
589 			continue;
590 		}
591 
592 		/*
593 		 * Handle unaligned pages, we collect them up onto
594 		 * the root page until we have a full large page.
595 		 */
596 		if (!IS_P2ALIGNED(pnum, large)) {
597 
598 			/*
599 			 * If not in a large page,
600 			 * just free as small page.
601 			 */
602 			if (root == NULL) {
603 				pp->p_szc = 0;
604 				page_free_at_startup(pp);
605 				continue;
606 			}
607 
608 			/*
609 			 * Link a constituent page into the large page.
610 			 */
611 			pp->p_szc = szc;
612 			page_list_concat(&root, &pp);
613 
614 			/*
615 			 * When large page is fully formed, free it.
616 			 */
617 			if (++cnt == large) {
618 				page_free_large_ctr(cnt);
619 				page_list_add_pages(root, PG_LIST_ISINIT);
620 				root = NULL;
621 				cnt = 0;
622 			}
623 			continue;
624 		}
625 
626 		/*
627 		 * At this point we have a page number which
628 		 * is aligned. We assert that we aren't already
629 		 * in a different large page.
630 		 */
631 		ASSERT(IS_P2ALIGNED(pnum, large));
632 		ASSERT(root == NULL && cnt == 0);
633 
634 		/*
635 		 * If insufficient number of pages left to form
636 		 * a large page, just free the small page.
637 		 */
638 		if (num < large) {
639 			pp->p_szc = 0;
640 			page_free_at_startup(pp);
641 			continue;
642 		}
643 
644 		/*
645 		 * Otherwise start a new large page.
646 		 */
647 		pp->p_szc = szc;
648 		cnt++;
649 		root = pp;
650 	}
651 	ASSERT(root == NULL && cnt == 0);
652 }
653 
654 /*
655  * Find a page representing the specified [vp, offset].
656  * If we find the page but it is intransit coming in,
657  * it will have an "exclusive" lock and we wait for
658  * the i/o to complete.  A page found on the free list
659  * is always reclaimed and then locked.  On success, the page
660  * is locked, its data is valid and it isn't on the free
661  * list, while a NULL is returned if the page doesn't exist.
662  */
663 page_t *
664 page_lookup(vnode_t *vp, u_offset_t off, se_t se)
665 {
666 	return (page_lookup_create(vp, off, se, NULL, NULL, 0));
667 }
668 
669 /*
670  * Find a page representing the specified [vp, offset].
671  * We either return the one we found or, if passed in,
672  * create one with identity of [vp, offset] of the
673  * pre-allocated page. If we find exsisting page but it is
674  * intransit coming in, it will have an "exclusive" lock
675  * and we wait for the i/o to complete.  A page found on
676  * the free list is always reclaimed and then locked.
677  * On success, the page is locked, its data is valid and
678  * it isn't on the free list, while a NULL is returned
679  * if the page doesn't exist and newpp is NULL;
680  */
681 page_t *
682 page_lookup_create(
683 	vnode_t *vp,
684 	u_offset_t off,
685 	se_t se,
686 	page_t *newpp,
687 	spgcnt_t *nrelocp,
688 	int flags)
689 {
690 	page_t		*pp;
691 	kmutex_t	*phm;
692 	ulong_t		index;
693 	uint_t		hash_locked;
694 	uint_t		es;
695 
696 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
697 	VM_STAT_ADD(page_lookup_cnt[0]);
698 	ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
699 
700 	/*
701 	 * Acquire the appropriate page hash lock since
702 	 * we have to search the hash list.  Pages that
703 	 * hash to this list can't change identity while
704 	 * this lock is held.
705 	 */
706 	hash_locked = 0;
707 	index = PAGE_HASH_FUNC(vp, off);
708 	phm = NULL;
709 top:
710 	PAGE_HASH_SEARCH(index, pp, vp, off);
711 	if (pp != NULL) {
712 		VM_STAT_ADD(page_lookup_cnt[1]);
713 		es = (newpp != NULL) ? 1 : 0;
714 		es |= flags;
715 		if (!hash_locked) {
716 			VM_STAT_ADD(page_lookup_cnt[2]);
717 			if (!page_try_reclaim_lock(pp, se, es)) {
718 				/*
719 				 * On a miss, acquire the phm.  Then
720 				 * next time, page_lock() will be called,
721 				 * causing a wait if the page is busy.
722 				 * just looping with page_trylock() would
723 				 * get pretty boring.
724 				 */
725 				VM_STAT_ADD(page_lookup_cnt[3]);
726 				phm = PAGE_HASH_MUTEX(index);
727 				mutex_enter(phm);
728 				hash_locked = 1;
729 				goto top;
730 			}
731 		} else {
732 			VM_STAT_ADD(page_lookup_cnt[4]);
733 			if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
734 				VM_STAT_ADD(page_lookup_cnt[5]);
735 				goto top;
736 			}
737 		}
738 
739 		/*
740 		 * Since `pp' is locked it can not change identity now.
741 		 * Reconfirm we locked the correct page.
742 		 *
743 		 * Both the p_vnode and p_offset *must* be cast volatile
744 		 * to force a reload of their values: The PAGE_HASH_SEARCH
745 		 * macro will have stuffed p_vnode and p_offset into
746 		 * registers before calling page_trylock(); another thread,
747 		 * actually holding the hash lock, could have changed the
748 		 * page's identity in memory, but our registers would not
749 		 * be changed, fooling the reconfirmation.  If the hash
750 		 * lock was held during the search, the casting would
751 		 * not be needed.
752 		 */
753 		VM_STAT_ADD(page_lookup_cnt[6]);
754 		if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
755 		    ((volatile u_offset_t)(pp->p_offset) != off)) {
756 			VM_STAT_ADD(page_lookup_cnt[7]);
757 			if (hash_locked) {
758 				panic("page_lookup_create: lost page %p",
759 				    (void *)pp);
760 				/*NOTREACHED*/
761 			}
762 			page_unlock(pp);
763 			phm = PAGE_HASH_MUTEX(index);
764 			mutex_enter(phm);
765 			hash_locked = 1;
766 			goto top;
767 		}
768 
769 		/*
770 		 * If page_trylock() was called, then pp may still be on
771 		 * the cachelist (can't be on the free list, it would not
772 		 * have been found in the search).  If it is on the
773 		 * cachelist it must be pulled now. To pull the page from
774 		 * the cachelist, it must be exclusively locked.
775 		 *
776 		 * The other big difference between page_trylock() and
777 		 * page_lock(), is that page_lock() will pull the
778 		 * page from whatever free list (the cache list in this
779 		 * case) the page is on.  If page_trylock() was used
780 		 * above, then we have to do the reclaim ourselves.
781 		 */
782 		if ((!hash_locked) && (PP_ISFREE(pp))) {
783 			ASSERT(PP_ISAGED(pp) == 0);
784 			VM_STAT_ADD(page_lookup_cnt[8]);
785 
786 			/*
787 			 * page_relcaim will insure that we
788 			 * have this page exclusively
789 			 */
790 
791 			if (!page_reclaim(pp, NULL)) {
792 				/*
793 				 * Page_reclaim dropped whatever lock
794 				 * we held.
795 				 */
796 				VM_STAT_ADD(page_lookup_cnt[9]);
797 				phm = PAGE_HASH_MUTEX(index);
798 				mutex_enter(phm);
799 				hash_locked = 1;
800 				goto top;
801 			} else if (se == SE_SHARED && newpp == NULL) {
802 				VM_STAT_ADD(page_lookup_cnt[10]);
803 				page_downgrade(pp);
804 			}
805 		}
806 
807 		if (hash_locked) {
808 			mutex_exit(phm);
809 		}
810 
811 		if (newpp != NULL && pp->p_szc < newpp->p_szc &&
812 		    PAGE_EXCL(pp) && nrelocp != NULL) {
813 			ASSERT(nrelocp != NULL);
814 			(void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
815 			    NULL);
816 			if (*nrelocp > 0) {
817 				VM_STAT_COND_ADD(*nrelocp == 1,
818 				    page_lookup_cnt[11]);
819 				VM_STAT_COND_ADD(*nrelocp > 1,
820 				    page_lookup_cnt[12]);
821 				pp = newpp;
822 				se = SE_EXCL;
823 			} else {
824 				if (se == SE_SHARED) {
825 					page_downgrade(pp);
826 				}
827 				VM_STAT_ADD(page_lookup_cnt[13]);
828 			}
829 		} else if (newpp != NULL && nrelocp != NULL) {
830 			if (PAGE_EXCL(pp) && se == SE_SHARED) {
831 				page_downgrade(pp);
832 			}
833 			VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
834 			    page_lookup_cnt[14]);
835 			VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
836 			    page_lookup_cnt[15]);
837 			VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
838 			    page_lookup_cnt[16]);
839 		} else if (newpp != NULL && PAGE_EXCL(pp)) {
840 			se = SE_EXCL;
841 		}
842 	} else if (!hash_locked) {
843 		VM_STAT_ADD(page_lookup_cnt[17]);
844 		phm = PAGE_HASH_MUTEX(index);
845 		mutex_enter(phm);
846 		hash_locked = 1;
847 		goto top;
848 	} else if (newpp != NULL) {
849 		/*
850 		 * If we have a preallocated page then
851 		 * insert it now and basically behave like
852 		 * page_create.
853 		 */
854 		VM_STAT_ADD(page_lookup_cnt[18]);
855 		/*
856 		 * Since we hold the page hash mutex and
857 		 * just searched for this page, page_hashin
858 		 * had better not fail.  If it does, that
859 		 * means some thread did not follow the
860 		 * page hash mutex rules.  Panic now and
861 		 * get it over with.  As usual, go down
862 		 * holding all the locks.
863 		 */
864 		ASSERT(MUTEX_HELD(phm));
865 		if (!page_hashin(newpp, vp, off, phm)) {
866 			ASSERT(MUTEX_HELD(phm));
867 			panic("page_lookup_create: hashin failed %p %p %llx %p",
868 			    (void *)newpp, (void *)vp, off, (void *)phm);
869 			/*NOTREACHED*/
870 		}
871 		ASSERT(MUTEX_HELD(phm));
872 		mutex_exit(phm);
873 		phm = NULL;
874 		page_set_props(newpp, P_REF);
875 		page_io_lock(newpp);
876 		pp = newpp;
877 		se = SE_EXCL;
878 	} else {
879 		VM_STAT_ADD(page_lookup_cnt[19]);
880 		mutex_exit(phm);
881 	}
882 
883 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
884 
885 	ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);
886 
887 	return (pp);
888 }
889 
890 /*
891  * Search the hash list for the page representing the
892  * specified [vp, offset] and return it locked.  Skip
893  * free pages and pages that cannot be locked as requested.
894  * Used while attempting to kluster pages.
895  */
896 page_t *
897 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
898 {
899 	page_t		*pp;
900 	kmutex_t	*phm;
901 	ulong_t		index;
902 	uint_t		locked;
903 
904 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
905 	VM_STAT_ADD(page_lookup_nowait_cnt[0]);
906 
907 	index = PAGE_HASH_FUNC(vp, off);
908 	PAGE_HASH_SEARCH(index, pp, vp, off);
909 	locked = 0;
910 	if (pp == NULL) {
911 top:
912 		VM_STAT_ADD(page_lookup_nowait_cnt[1]);
913 		locked = 1;
914 		phm = PAGE_HASH_MUTEX(index);
915 		mutex_enter(phm);
916 		PAGE_HASH_SEARCH(index, pp, vp, off);
917 	}
918 
919 	if (pp == NULL || PP_ISFREE(pp)) {
920 		VM_STAT_ADD(page_lookup_nowait_cnt[2]);
921 		pp = NULL;
922 	} else {
923 		if (!page_trylock(pp, se)) {
924 			VM_STAT_ADD(page_lookup_nowait_cnt[3]);
925 			pp = NULL;
926 		} else {
927 			VM_STAT_ADD(page_lookup_nowait_cnt[4]);
928 			/*
929 			 * See the comment in page_lookup()
930 			 */
931 			if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
932 			    ((u_offset_t)(pp->p_offset) != off)) {
933 				VM_STAT_ADD(page_lookup_nowait_cnt[5]);
934 				if (locked) {
935 					panic("page_lookup_nowait %p",
936 					    (void *)pp);
937 					/*NOTREACHED*/
938 				}
939 				page_unlock(pp);
940 				goto top;
941 			}
942 			if (PP_ISFREE(pp)) {
943 				VM_STAT_ADD(page_lookup_nowait_cnt[6]);
944 				page_unlock(pp);
945 				pp = NULL;
946 			}
947 		}
948 	}
949 	if (locked) {
950 		VM_STAT_ADD(page_lookup_nowait_cnt[7]);
951 		mutex_exit(phm);
952 	}
953 
954 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
955 
956 	return (pp);
957 }
958 
959 /*
960  * Search the hash list for a page with the specified [vp, off]
961  * that is known to exist and is already locked.  This routine
962  * is typically used by segment SOFTUNLOCK routines.
963  */
964 page_t *
965 page_find(vnode_t *vp, u_offset_t off)
966 {
967 	page_t		*pp;
968 	kmutex_t	*phm;
969 	ulong_t		index;
970 
971 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
972 	VM_STAT_ADD(page_find_cnt);
973 
974 	index = PAGE_HASH_FUNC(vp, off);
975 	phm = PAGE_HASH_MUTEX(index);
976 
977 	mutex_enter(phm);
978 	PAGE_HASH_SEARCH(index, pp, vp, off);
979 	mutex_exit(phm);
980 
981 	ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
982 	return (pp);
983 }
984 
985 /*
986  * Determine whether a page with the specified [vp, off]
987  * currently exists in the system.  Obviously this should
988  * only be considered as a hint since nothing prevents the
989  * page from disappearing or appearing immediately after
990  * the return from this routine. Subsequently, we don't
991  * even bother to lock the list.
992  */
993 page_t *
994 page_exists(vnode_t *vp, u_offset_t off)
995 {
996 	page_t	*pp;
997 	ulong_t		index;
998 
999 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1000 	VM_STAT_ADD(page_exists_cnt);
1001 
1002 	index = PAGE_HASH_FUNC(vp, off);
1003 	PAGE_HASH_SEARCH(index, pp, vp, off);
1004 
1005 	return (pp);
1006 }
1007 
1008 /*
1009  * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1010  * page_size(szc)) range.  if they exist and ppa is not NULL fill ppa array
1011  * with these pages locked SHARED. If necessary reclaim pages from
1012  * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1013  *
1014  * If we fail to lock pages still return 1 if pages exist and contiguous.
1015  * But in this case return value is just a hint. ppa array won't be filled.
1016  * Caller should initialize ppa[0] as NULL to distinguish return value.
1017  *
1018  * Returns 0 if pages don't exist or not physically contiguous.
1019  *
1020  * This routine doesn't work for anonymous(swapfs) pages.
1021  */
1022 int
1023 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1024 {
1025 	pgcnt_t pages;
1026 	pfn_t pfn;
1027 	page_t *rootpp;
1028 	pgcnt_t i;
1029 	pgcnt_t j;
1030 	u_offset_t save_off = off;
1031 	ulong_t index;
1032 	kmutex_t *phm;
1033 	page_t *pp;
1034 	uint_t pszc;
1035 	int loopcnt = 0;
1036 
1037 	ASSERT(szc != 0);
1038 	ASSERT(vp != NULL);
1039 	ASSERT(!IS_SWAPFSVP(vp));
1040 	ASSERT(!VN_ISKAS(vp));
1041 
1042 again:
1043 	if (++loopcnt > 3) {
1044 		VM_STAT_ADD(page_exphcontg[0]);
1045 		return (0);
1046 	}
1047 
1048 	index = PAGE_HASH_FUNC(vp, off);
1049 	phm = PAGE_HASH_MUTEX(index);
1050 
1051 	mutex_enter(phm);
1052 	PAGE_HASH_SEARCH(index, pp, vp, off);
1053 	mutex_exit(phm);
1054 
1055 	VM_STAT_ADD(page_exphcontg[1]);
1056 
1057 	if (pp == NULL) {
1058 		VM_STAT_ADD(page_exphcontg[2]);
1059 		return (0);
1060 	}
1061 
1062 	pages = page_get_pagecnt(szc);
1063 	rootpp = pp;
1064 	pfn = rootpp->p_pagenum;
1065 
1066 	if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1067 		VM_STAT_ADD(page_exphcontg[3]);
1068 		if (!page_trylock(pp, SE_SHARED)) {
1069 			VM_STAT_ADD(page_exphcontg[4]);
1070 			return (1);
1071 		}
1072 		if (pp->p_szc != pszc || pp->p_vnode != vp ||
1073 		    pp->p_offset != off) {
1074 			VM_STAT_ADD(page_exphcontg[5]);
1075 			page_unlock(pp);
1076 			off = save_off;
1077 			goto again;
1078 		}
1079 		/*
1080 		 * szc was non zero and vnode and offset matched after we
1081 		 * locked the page it means it can't become free on us.
1082 		 */
1083 		ASSERT(!PP_ISFREE(pp));
1084 		if (!IS_P2ALIGNED(pfn, pages)) {
1085 			page_unlock(pp);
1086 			return (0);
1087 		}
1088 		ppa[0] = pp;
1089 		pp++;
1090 		off += PAGESIZE;
1091 		pfn++;
1092 		for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1093 			if (!page_trylock(pp, SE_SHARED)) {
1094 				VM_STAT_ADD(page_exphcontg[6]);
1095 				pp--;
1096 				while (i-- > 0) {
1097 					page_unlock(pp);
1098 					pp--;
1099 				}
1100 				ppa[0] = NULL;
1101 				return (1);
1102 			}
1103 			if (pp->p_szc != pszc) {
1104 				VM_STAT_ADD(page_exphcontg[7]);
1105 				page_unlock(pp);
1106 				pp--;
1107 				while (i-- > 0) {
1108 					page_unlock(pp);
1109 					pp--;
1110 				}
1111 				ppa[0] = NULL;
1112 				off = save_off;
1113 				goto again;
1114 			}
1115 			/*
1116 			 * szc the same as for previous already locked pages
1117 			 * with right identity. Since this page had correct
1118 			 * szc after we locked it can't get freed or destroyed
1119 			 * and therefore must have the expected identity.
1120 			 */
1121 			ASSERT(!PP_ISFREE(pp));
1122 			if (pp->p_vnode != vp ||
1123 			    pp->p_offset != off) {
1124 				panic("page_exists_physcontig: "
1125 				    "large page identity doesn't match");
1126 			}
1127 			ppa[i] = pp;
1128 			ASSERT(pp->p_pagenum == pfn);
1129 		}
1130 		VM_STAT_ADD(page_exphcontg[8]);
1131 		ppa[pages] = NULL;
1132 		return (1);
1133 	} else if (pszc >= szc) {
1134 		VM_STAT_ADD(page_exphcontg[9]);
1135 		if (!IS_P2ALIGNED(pfn, pages)) {
1136 			return (0);
1137 		}
1138 		return (1);
1139 	}
1140 
1141 	if (!IS_P2ALIGNED(pfn, pages)) {
1142 		VM_STAT_ADD(page_exphcontg[10]);
1143 		return (0);
1144 	}
1145 
1146 	if (page_numtomemseg_nolock(pfn) !=
1147 	    page_numtomemseg_nolock(pfn + pages - 1)) {
1148 		VM_STAT_ADD(page_exphcontg[11]);
1149 		return (0);
1150 	}
1151 
1152 	/*
1153 	 * We loop up 4 times across pages to promote page size.
1154 	 * We're extra cautious to promote page size atomically with respect
1155 	 * to everybody else.  But we can probably optimize into 1 loop if
1156 	 * this becomes an issue.
1157 	 */
1158 
1159 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1160 		ASSERT(pp->p_pagenum == pfn);
1161 		if (!page_trylock(pp, SE_EXCL)) {
1162 			VM_STAT_ADD(page_exphcontg[12]);
1163 			break;
1164 		}
1165 		if (pp->p_vnode != vp ||
1166 		    pp->p_offset != off) {
1167 			VM_STAT_ADD(page_exphcontg[13]);
1168 			page_unlock(pp);
1169 			break;
1170 		}
1171 		if (pp->p_szc >= szc) {
1172 			ASSERT(i == 0);
1173 			page_unlock(pp);
1174 			off = save_off;
1175 			goto again;
1176 		}
1177 	}
1178 
1179 	if (i != pages) {
1180 		VM_STAT_ADD(page_exphcontg[14]);
1181 		--pp;
1182 		while (i-- > 0) {
1183 			page_unlock(pp);
1184 			--pp;
1185 		}
1186 		return (0);
1187 	}
1188 
1189 	pp = rootpp;
1190 	for (i = 0; i < pages; i++, pp++) {
1191 		if (PP_ISFREE(pp)) {
1192 			VM_STAT_ADD(page_exphcontg[15]);
1193 			ASSERT(!PP_ISAGED(pp));
1194 			ASSERT(pp->p_szc == 0);
1195 			if (!page_reclaim(pp, NULL)) {
1196 				break;
1197 			}
1198 		} else {
1199 			ASSERT(pp->p_szc < szc);
1200 			VM_STAT_ADD(page_exphcontg[16]);
1201 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
1202 		}
1203 	}
1204 	if (i < pages) {
1205 		VM_STAT_ADD(page_exphcontg[17]);
1206 		/*
1207 		 * page_reclaim failed because we were out of memory.
1208 		 * drop the rest of the locks and return because this page
1209 		 * must be already reallocated anyway.
1210 		 */
1211 		pp = rootpp;
1212 		for (j = 0; j < pages; j++, pp++) {
1213 			if (j != i) {
1214 				page_unlock(pp);
1215 			}
1216 		}
1217 		return (0);
1218 	}
1219 
1220 	off = save_off;
1221 	pp = rootpp;
1222 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE) {
1223 		ASSERT(PAGE_EXCL(pp));
1224 		ASSERT(!PP_ISFREE(pp));
1225 		ASSERT(!hat_page_is_mapped(pp));
1226 		ASSERT(pp->p_vnode == vp);
1227 		ASSERT(pp->p_offset == off);
1228 		pp->p_szc = szc;
1229 	}
1230 	pp = rootpp;
1231 	for (i = 0; i < pages; i++, pp++) {
1232 		if (ppa == NULL) {
1233 			page_unlock(pp);
1234 		} else {
1235 			ppa[i] = pp;
1236 			page_downgrade(ppa[i]);
1237 		}
1238 	}
1239 	if (ppa != NULL) {
1240 		ppa[pages] = NULL;
1241 	}
1242 	VM_STAT_ADD(page_exphcontg[18]);
1243 	ASSERT(vp->v_pages != NULL);
1244 	return (1);
1245 }
1246 
1247 /*
1248  * Determine whether a page with the specified [vp, off]
1249  * currently exists in the system and if so return its
1250  * size code. Obviously this should only be considered as
1251  * a hint since nothing prevents the page from disappearing
1252  * or appearing immediately after the return from this routine.
1253  */
1254 int
1255 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1256 {
1257 	page_t		*pp;
1258 	kmutex_t	*phm;
1259 	ulong_t		index;
1260 	int		rc = 0;
1261 
1262 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1263 	ASSERT(szc != NULL);
1264 	VM_STAT_ADD(page_exists_forreal_cnt);
1265 
1266 	index = PAGE_HASH_FUNC(vp, off);
1267 	phm = PAGE_HASH_MUTEX(index);
1268 
1269 	mutex_enter(phm);
1270 	PAGE_HASH_SEARCH(index, pp, vp, off);
1271 	if (pp != NULL) {
1272 		*szc = pp->p_szc;
1273 		rc = 1;
1274 	}
1275 	mutex_exit(phm);
1276 	return (rc);
1277 }
1278 
1279 /* wakeup threads waiting for pages in page_create_get_something() */
1280 void
1281 wakeup_pcgs(void)
1282 {
1283 	if (!CV_HAS_WAITERS(&pcgs_cv))
1284 		return;
1285 	cv_broadcast(&pcgs_cv);
1286 }
1287 
1288 /*
1289  * 'freemem' is used all over the kernel as an indication of how many
1290  * pages are free (either on the cache list or on the free page list)
1291  * in the system.  In very few places is a really accurate 'freemem'
1292  * needed.  To avoid contention of the lock protecting a the
1293  * single freemem, it was spread out into NCPU buckets.  Set_freemem
1294  * sets freemem to the total of all NCPU buckets.  It is called from
1295  * clock() on each TICK.
1296  */
1297 void
1298 set_freemem()
1299 {
1300 	struct pcf	*p;
1301 	ulong_t		t;
1302 	uint_t		i;
1303 
1304 	t = 0;
1305 	p = pcf;
1306 	for (i = 0;  i < PCF_FANOUT; i++) {
1307 		t += p->pcf_count;
1308 		p++;
1309 	}
1310 	freemem = t;
1311 
1312 	/*
1313 	 * Don't worry about grabbing mutex.  It's not that
1314 	 * critical if we miss a tick or two.  This is
1315 	 * where we wakeup possible delayers in
1316 	 * page_create_get_something().
1317 	 */
1318 	wakeup_pcgs();
1319 }
1320 
1321 ulong_t
1322 get_freemem()
1323 {
1324 	struct pcf	*p;
1325 	ulong_t		t;
1326 	uint_t		i;
1327 
1328 	t = 0;
1329 	p = pcf;
1330 	for (i = 0; i < PCF_FANOUT; i++) {
1331 		t += p->pcf_count;
1332 		p++;
1333 	}
1334 	/*
1335 	 * We just calculated it, might as well set it.
1336 	 */
1337 	freemem = t;
1338 	return (t);
1339 }
1340 
1341 /*
1342  * Acquire all of the page cache & free (pcf) locks.
1343  */
1344 void
1345 pcf_acquire_all()
1346 {
1347 	struct pcf	*p;
1348 	uint_t		i;
1349 
1350 	p = pcf;
1351 	for (i = 0; i < PCF_FANOUT; i++) {
1352 		mutex_enter(&p->pcf_lock);
1353 		p++;
1354 	}
1355 }
1356 
1357 /*
1358  * Release all the pcf_locks.
1359  */
1360 void
1361 pcf_release_all()
1362 {
1363 	struct pcf	*p;
1364 	uint_t		i;
1365 
1366 	p = pcf;
1367 	for (i = 0; i < PCF_FANOUT; i++) {
1368 		mutex_exit(&p->pcf_lock);
1369 		p++;
1370 	}
1371 }
1372 
1373 /*
1374  * Inform the VM system that we need some pages freed up.
1375  * Calls must be symmetric, e.g.:
1376  *
1377  *	page_needfree(100);
1378  *	wait a bit;
1379  *	page_needfree(-100);
1380  */
1381 void
1382 page_needfree(spgcnt_t npages)
1383 {
1384 	mutex_enter(&new_freemem_lock);
1385 	needfree += npages;
1386 	mutex_exit(&new_freemem_lock);
1387 }
1388 
1389 /*
1390  * Throttle for page_create(): try to prevent freemem from dropping
1391  * below throttlefree.  We can't provide a 100% guarantee because
1392  * KM_NOSLEEP allocations, page_reclaim(), and various other things
1393  * nibble away at the freelist.  However, we can block all PG_WAIT
1394  * allocations until memory becomes available.  The motivation is
1395  * that several things can fall apart when there's no free memory:
1396  *
1397  * (1) If pageout() needs memory to push a page, the system deadlocks.
1398  *
1399  * (2) By (broken) specification, timeout(9F) can neither fail nor
1400  *     block, so it has no choice but to panic the system if it
1401  *     cannot allocate a callout structure.
1402  *
1403  * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
1404  *     it panics if it cannot allocate a callback structure.
1405  *
1406  * (4) Untold numbers of third-party drivers have not yet been hardened
1407  *     against KM_NOSLEEP and/or allocb() failures; they simply assume
1408  *     success and panic the system with a data fault on failure.
1409  *     (The long-term solution to this particular problem is to ship
1410  *     hostile fault-injecting DEBUG kernels with the DDK.)
1411  *
1412  * It is theoretically impossible to guarantee success of non-blocking
1413  * allocations, but in practice, this throttle is very hard to break.
1414  */
1415 static int
1416 page_create_throttle(pgcnt_t npages, int flags)
1417 {
1418 	ulong_t	fm;
1419 	uint_t	i;
1420 	pgcnt_t tf;	/* effective value of throttlefree */
1421 
1422 	/*
1423 	 * Never deny pages when:
1424 	 * - it's a thread that cannot block [NOMEMWAIT()]
1425 	 * - the allocation cannot block and must not fail
1426 	 * - the allocation cannot block and is pageout dispensated
1427 	 */
1428 	if (NOMEMWAIT() ||
1429 	    ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) ||
1430 	    ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE))
1431 		return (1);
1432 
1433 	/*
1434 	 * If the allocation can't block, we look favorably upon it
1435 	 * unless we're below pageout_reserve.  In that case we fail
1436 	 * the allocation because we want to make sure there are a few
1437 	 * pages available for pageout.
1438 	 */
1439 	if ((flags & PG_WAIT) == 0)
1440 		return (freemem >= npages + pageout_reserve);
1441 
1442 	/* Calculate the effective throttlefree value */
1443 	tf = throttlefree -
1444 	    ((flags & PG_PUSHPAGE) ? pageout_reserve : 0);
1445 
1446 	cv_signal(&proc_pageout->p_cv);
1447 
1448 	while (freemem < npages + tf) {
1449 		pcf_acquire_all();
1450 		mutex_enter(&new_freemem_lock);
1451 		fm = 0;
1452 		for (i = 0; i < PCF_FANOUT; i++) {
1453 			fm += pcf[i].pcf_count;
1454 			pcf[i].pcf_wait++;
1455 			mutex_exit(&pcf[i].pcf_lock);
1456 		}
1457 		freemem = fm;
1458 		needfree += npages;
1459 		freemem_wait++;
1460 		cv_wait(&freemem_cv, &new_freemem_lock);
1461 		freemem_wait--;
1462 		needfree -= npages;
1463 		mutex_exit(&new_freemem_lock);
1464 	}
1465 	return (1);
1466 }
1467 
1468 /*
1469  * page_create_wait() is called to either coalecse pages from the
1470  * different pcf buckets or to wait because there simply are not
1471  * enough pages to satisfy the caller's request.
1472  *
1473  * Sadly, this is called from platform/vm/vm_machdep.c
1474  */
1475 int
1476 page_create_wait(size_t npages, uint_t flags)
1477 {
1478 	pgcnt_t		total;
1479 	uint_t		i;
1480 	struct pcf	*p;
1481 
1482 	/*
1483 	 * Wait until there are enough free pages to satisfy our
1484 	 * entire request.
1485 	 * We set needfree += npages before prodding pageout, to make sure
1486 	 * it does real work when npages > lotsfree > freemem.
1487 	 */
1488 	VM_STAT_ADD(page_create_not_enough);
1489 
1490 	ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1);
1491 checkagain:
1492 	if ((flags & PG_NORELOC) &&
1493 	    kcage_freemem < kcage_throttlefree + npages)
1494 		(void) kcage_create_throttle(npages, flags);
1495 
1496 	if (freemem < npages + throttlefree)
1497 		if (!page_create_throttle(npages, flags))
1498 			return (0);
1499 
1500 	/*
1501 	 * Since page_create_va() looked at every
1502 	 * bucket, assume we are going to have to wait.
1503 	 * Get all of the pcf locks.
1504 	 */
1505 	total = 0;
1506 	p = pcf;
1507 	for (i = 0; i < PCF_FANOUT; i++) {
1508 		mutex_enter(&p->pcf_lock);
1509 		total += p->pcf_count;
1510 		if (total >= npages) {
1511 			/*
1512 			 * Wow!  There are enough pages laying around
1513 			 * to satisfy the request.  Do the accounting,
1514 			 * drop the locks we acquired, and go back.
1515 			 *
1516 			 * freemem is not protected by any lock. So,
1517 			 * we cannot have any assertion containing
1518 			 * freemem.
1519 			 */
1520 			freemem -= npages;
1521 
1522 			while (p >= pcf) {
1523 				if (p->pcf_count <= npages) {
1524 					npages -= p->pcf_count;
1525 					p->pcf_count = 0;
1526 				} else {
1527 					p->pcf_count -= (uint_t)npages;
1528 					npages = 0;
1529 				}
1530 				mutex_exit(&p->pcf_lock);
1531 				p--;
1532 			}
1533 			ASSERT(npages == 0);
1534 			return (1);
1535 		}
1536 		p++;
1537 	}
1538 
1539 	/*
1540 	 * All of the pcf locks are held, there are not enough pages
1541 	 * to satisfy the request (npages < total).
1542 	 * Be sure to acquire the new_freemem_lock before dropping
1543 	 * the pcf locks.  This prevents dropping wakeups in page_free().
1544 	 * The order is always pcf_lock then new_freemem_lock.
1545 	 *
1546 	 * Since we hold all the pcf locks, it is a good time to set freemem.
1547 	 *
1548 	 * If the caller does not want to wait, return now.
1549 	 * Else turn the pageout daemon loose to find something
1550 	 * and wait till it does.
1551 	 *
1552 	 */
1553 	freemem = total;
1554 
1555 	if ((flags & PG_WAIT) == 0) {
1556 		pcf_release_all();
1557 
1558 		TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM,
1559 		"page_create_nomem:npages %ld freemem %ld", npages, freemem);
1560 		return (0);
1561 	}
1562 
1563 	ASSERT(proc_pageout != NULL);
1564 	cv_signal(&proc_pageout->p_cv);
1565 
1566 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START,
1567 	    "page_create_sleep_start: freemem %ld needfree %ld",
1568 	    freemem, needfree);
1569 
1570 	/*
1571 	 * We are going to wait.
1572 	 * We currently hold all of the pcf_locks,
1573 	 * get the new_freemem_lock (it protects freemem_wait),
1574 	 * before dropping the pcf_locks.
1575 	 */
1576 	mutex_enter(&new_freemem_lock);
1577 
1578 	p = pcf;
1579 	for (i = 0; i < PCF_FANOUT; i++) {
1580 		p->pcf_wait++;
1581 		mutex_exit(&p->pcf_lock);
1582 		p++;
1583 	}
1584 
1585 	needfree += npages;
1586 	freemem_wait++;
1587 
1588 	cv_wait(&freemem_cv, &new_freemem_lock);
1589 
1590 	freemem_wait--;
1591 	needfree -= npages;
1592 
1593 	mutex_exit(&new_freemem_lock);
1594 
1595 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END,
1596 	    "page_create_sleep_end: freemem %ld needfree %ld",
1597 	    freemem, needfree);
1598 
1599 	VM_STAT_ADD(page_create_not_enough_again);
1600 	goto checkagain;
1601 }
1602 
1603 /*
1604  * A routine to do the opposite of page_create_wait().
1605  */
1606 void
1607 page_create_putback(spgcnt_t npages)
1608 {
1609 	struct pcf	*p;
1610 	pgcnt_t		lump;
1611 	uint_t		*which;
1612 
1613 	/*
1614 	 * When a contiguous lump is broken up, we have to
1615 	 * deal with lots of pages (min 64) so lets spread
1616 	 * the wealth around.
1617 	 */
1618 	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;
1619 	freemem += npages;
1620 
1621 	for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) {
1622 		which = &p->pcf_count;
1623 
1624 		mutex_enter(&p->pcf_lock);
1625 
1626 		if (p->pcf_block) {
1627 			which = &p->pcf_reserve;
1628 		}
1629 
1630 		if (lump < npages) {
1631 			*which += (uint_t)lump;
1632 			npages -= lump;
1633 		} else {
1634 			*which += (uint_t)npages;
1635 			npages = 0;
1636 		}
1637 
1638 		if (p->pcf_wait) {
1639 			mutex_enter(&new_freemem_lock);
1640 			/*
1641 			 * Check to see if some other thread
1642 			 * is actually waiting.  Another bucket
1643 			 * may have woken it up by now.  If there
1644 			 * are no waiters, then set our pcf_wait
1645 			 * count to zero to avoid coming in here
1646 			 * next time.
1647 			 */
1648 			if (freemem_wait) {
1649 				if (npages > 1) {
1650 					cv_broadcast(&freemem_cv);
1651 				} else {
1652 					cv_signal(&freemem_cv);
1653 				}
1654 				p->pcf_wait--;
1655 			} else {
1656 				p->pcf_wait = 0;
1657 			}
1658 			mutex_exit(&new_freemem_lock);
1659 		}
1660 		mutex_exit(&p->pcf_lock);
1661 	}
1662 	ASSERT(npages == 0);
1663 }
1664 
1665 /*
1666  * A helper routine for page_create_get_something.
1667  * The indenting got to deep down there.
1668  * Unblock the pcf counters.  Any pages freed after
1669  * pcf_block got set are moved to pcf_count and
1670  * wakeups (cv_broadcast() or cv_signal()) are done as needed.
1671  */
1672 static void
1673 pcgs_unblock(void)
1674 {
1675 	int		i;
1676 	struct pcf	*p;
1677 
1678 	/* Update freemem while we're here. */
1679 	freemem = 0;
1680 	p = pcf;
1681 	for (i = 0; i < PCF_FANOUT; i++) {
1682 		mutex_enter(&p->pcf_lock);
1683 		ASSERT(p->pcf_count == 0);
1684 		p->pcf_count = p->pcf_reserve;
1685 		p->pcf_block = 0;
1686 		freemem += p->pcf_count;
1687 		if (p->pcf_wait) {
1688 			mutex_enter(&new_freemem_lock);
1689 			if (freemem_wait) {
1690 				if (p->pcf_reserve > 1) {
1691 					cv_broadcast(&freemem_cv);
1692 					p->pcf_wait = 0;
1693 				} else {
1694 					cv_signal(&freemem_cv);
1695 					p->pcf_wait--;
1696 				}
1697 			} else {
1698 				p->pcf_wait = 0;
1699 			}
1700 			mutex_exit(&new_freemem_lock);
1701 		}
1702 		p->pcf_reserve = 0;
1703 		mutex_exit(&p->pcf_lock);
1704 		p++;
1705 	}
1706 }
1707 
1708 /*
1709  * Called from page_create_va() when both the cache and free lists
1710  * have been checked once.
1711  *
1712  * Either returns a page or panics since the accounting was done
1713  * way before we got here.
1714  *
1715  * We don't come here often, so leave the accounting on permanently.
1716  */
1717 
1718 #define	MAX_PCGS	100
1719 
1720 #ifdef	DEBUG
1721 #define	PCGS_TRIES	100
1722 #else	/* DEBUG */
1723 #define	PCGS_TRIES	10
1724 #endif	/* DEBUG */
1725 
1726 #ifdef	VM_STATS
1727 uint_t	pcgs_counts[PCGS_TRIES];
1728 uint_t	pcgs_too_many;
1729 uint_t	pcgs_entered;
1730 uint_t	pcgs_entered_noreloc;
1731 uint_t	pcgs_locked;
1732 uint_t	pcgs_cagelocked;
1733 #endif	/* VM_STATS */
1734 
1735 static page_t *
1736 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg,
1737     caddr_t vaddr, uint_t flags)
1738 {
1739 	uint_t		count;
1740 	page_t		*pp;
1741 	uint_t		locked, i;
1742 	struct	pcf	*p;
1743 	lgrp_t		*lgrp;
1744 	int		cagelocked = 0;
1745 
1746 	VM_STAT_ADD(pcgs_entered);
1747 
1748 	/*
1749 	 * Tap any reserve freelists: if we fail now, we'll die
1750 	 * since the page(s) we're looking for have already been
1751 	 * accounted for.
1752 	 */
1753 	flags |= PG_PANIC;
1754 
1755 	if ((flags & PG_NORELOC) != 0) {
1756 		VM_STAT_ADD(pcgs_entered_noreloc);
1757 		/*
1758 		 * Requests for free pages from critical threads
1759 		 * such as pageout still won't throttle here, but
1760 		 * we must try again, to give the cageout thread
1761 		 * another chance to catch up. Since we already
1762 		 * accounted for the pages, we had better get them
1763 		 * this time.
1764 		 *
1765 		 * N.B. All non-critical threads acquire the pcgs_cagelock
1766 		 * to serialize access to the freelists. This implements a
1767 		 * turnstile-type synchornization to avoid starvation of
1768 		 * critical requests for PG_NORELOC memory by non-critical
1769 		 * threads: all non-critical threads must acquire a 'ticket'
1770 		 * before passing through, which entails making sure
1771 		 * kcage_freemem won't fall below minfree prior to grabbing
1772 		 * pages from the freelists.
1773 		 */
1774 		if (kcage_create_throttle(1, flags) == KCT_NONCRIT) {
1775 			mutex_enter(&pcgs_cagelock);
1776 			cagelocked = 1;
1777 			VM_STAT_ADD(pcgs_cagelocked);
1778 		}
1779 	}
1780 
1781 	/*
1782 	 * Time to get serious.
1783 	 * We failed to get a `correctly colored' page from both the
1784 	 * free and cache lists.
1785 	 * We escalate in stage.
1786 	 *
1787 	 * First try both lists without worring about color.
1788 	 *
1789 	 * Then, grab all page accounting locks (ie. pcf[]) and
1790 	 * steal any pages that they have and set the pcf_block flag to
1791 	 * stop deletions from the lists.  This will help because
1792 	 * a page can get added to the free list while we are looking
1793 	 * at the cache list, then another page could be added to the cache
1794 	 * list allowing the page on the free list to be removed as we
1795 	 * move from looking at the cache list to the free list. This
1796 	 * could happen over and over. We would never find the page
1797 	 * we have accounted for.
1798 	 *
1799 	 * Noreloc pages are a subset of the global (relocatable) page pool.
1800 	 * They are not tracked separately in the pcf bins, so it is
1801 	 * impossible to know when doing pcf accounting if the available
1802 	 * page(s) are noreloc pages or not. When looking for a noreloc page
1803 	 * it is quite easy to end up here even if the global (relocatable)
1804 	 * page pool has plenty of free pages but the noreloc pool is empty.
1805 	 *
1806 	 * When the noreloc pool is empty (or low), additional noreloc pages
1807 	 * are created by converting pages from the global page pool. This
1808 	 * process will stall during pcf accounting if the pcf bins are
1809 	 * already locked. Such is the case when a noreloc allocation is
1810 	 * looping here in page_create_get_something waiting for more noreloc
1811 	 * pages to appear.
1812 	 *
1813 	 * Short of adding a new field to the pcf bins to accurately track
1814 	 * the number of free noreloc pages, we instead do not grab the
1815 	 * pcgs_lock, do not set the pcf blocks and do not timeout when
1816 	 * allocating a noreloc page. This allows noreloc allocations to
1817 	 * loop without blocking global page pool allocations.
1818 	 *
1819 	 * NOTE: the behaviour of page_create_get_something has not changed
1820 	 * for the case of global page pool allocations.
1821 	 */
1822 
1823 	flags &= ~PG_MATCH_COLOR;
1824 	locked = 0;
1825 #if defined(__i386) || defined(__amd64)
1826 	/*
1827 	 * page_create_get_something may be called because 4g memory may be
1828 	 * depleted. Set flags to allow for relocation of base page below
1829 	 * 4g if necessary.
1830 	 */
1831 	if (physmax4g)
1832 		flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1833 #endif
1834 
1835 	lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
1836 
1837 	for (count = 0; kcage_on || count < MAX_PCGS; count++) {
1838 		pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
1839 		    flags, lgrp);
1840 		if (pp == NULL) {
1841 			pp = page_get_cachelist(vp, off, seg, vaddr,
1842 			    flags, lgrp);
1843 		}
1844 		if (pp == NULL) {
1845 			/*
1846 			 * Serialize.  Don't fight with other pcgs().
1847 			 */
1848 			if (!locked && (!kcage_on || !(flags & PG_NORELOC))) {
1849 				mutex_enter(&pcgs_lock);
1850 				VM_STAT_ADD(pcgs_locked);
1851 				locked = 1;
1852 				p = pcf;
1853 				for (i = 0; i < PCF_FANOUT; i++) {
1854 					mutex_enter(&p->pcf_lock);
1855 					ASSERT(p->pcf_block == 0);
1856 					p->pcf_block = 1;
1857 					p->pcf_reserve = p->pcf_count;
1858 					p->pcf_count = 0;
1859 					mutex_exit(&p->pcf_lock);
1860 					p++;
1861 				}
1862 				freemem = 0;
1863 			}
1864 
1865 			if (count) {
1866 				/*
1867 				 * Since page_free() puts pages on
1868 				 * a list then accounts for it, we
1869 				 * just have to wait for page_free()
1870 				 * to unlock any page it was working
1871 				 * with. The page_lock()-page_reclaim()
1872 				 * path falls in the same boat.
1873 				 *
1874 				 * We don't need to check on the
1875 				 * PG_WAIT flag, we have already
1876 				 * accounted for the page we are
1877 				 * looking for in page_create_va().
1878 				 *
1879 				 * We just wait a moment to let any
1880 				 * locked pages on the lists free up,
1881 				 * then continue around and try again.
1882 				 *
1883 				 * Will be awakened by set_freemem().
1884 				 */
1885 				mutex_enter(&pcgs_wait_lock);
1886 				cv_wait(&pcgs_cv, &pcgs_wait_lock);
1887 				mutex_exit(&pcgs_wait_lock);
1888 			}
1889 		} else {
1890 #ifdef VM_STATS
1891 			if (count >= PCGS_TRIES) {
1892 				VM_STAT_ADD(pcgs_too_many);
1893 			} else {
1894 				VM_STAT_ADD(pcgs_counts[count]);
1895 			}
1896 #endif
1897 			if (locked) {
1898 				pcgs_unblock();
1899 				mutex_exit(&pcgs_lock);
1900 			}
1901 			if (cagelocked)
1902 				mutex_exit(&pcgs_cagelock);
1903 			return (pp);
1904 		}
1905 	}
1906 	/*
1907 	 * we go down holding the pcf locks.
1908 	 */
1909 	panic("no %spage found %d",
1910 	    ((flags & PG_NORELOC) ? "non-reloc " : ""), count);
1911 	/*NOTREACHED*/
1912 }
1913 
1914 /*
1915  * Create enough pages for "bytes" worth of data starting at
1916  * "off" in "vp".
1917  *
1918  *	Where flag must be one of:
1919  *
1920  *		PG_EXCL:	Exclusive create (fail if any page already
1921  *				exists in the page cache) which does not
1922  *				wait for memory to become available.
1923  *
1924  *		PG_WAIT:	Non-exclusive create which can wait for
1925  *				memory to become available.
1926  *
1927  *		PG_PHYSCONTIG:	Allocate physically contiguous pages.
1928  *				(Not Supported)
1929  *
1930  * A doubly linked list of pages is returned to the caller.  Each page
1931  * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
1932  * lock.
1933  *
1934  * Unable to change the parameters to page_create() in a minor release,
1935  * we renamed page_create() to page_create_va(), changed all known calls
1936  * from page_create() to page_create_va(), and created this wrapper.
1937  *
1938  * Upon a major release, we should break compatibility by deleting this
1939  * wrapper, and replacing all the strings "page_create_va", with "page_create".
1940  *
1941  * NOTE: There is a copy of this interface as page_create_io() in
1942  *	 i86/vm/vm_machdep.c. Any bugs fixed here should be applied
1943  *	 there.
1944  */
1945 page_t *
1946 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags)
1947 {
1948 	caddr_t random_vaddr;
1949 	struct seg kseg;
1950 
1951 #ifdef DEBUG
1952 	cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p",
1953 	    (void *)caller());
1954 #endif
1955 
1956 	random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^
1957 	    (uintptr_t)(off >> PAGESHIFT));
1958 	kseg.s_as = &kas;
1959 
1960 	return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr));
1961 }
1962 
1963 #ifdef DEBUG
1964 uint32_t pg_alloc_pgs_mtbf = 0;
1965 #endif
1966 
1967 /*
1968  * Used for large page support. It will attempt to allocate
1969  * a large page(s) off the freelist.
1970  *
1971  * Returns non zero on failure.
1972  */
1973 int
1974 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr,
1975     page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags)
1976 {
1977 	pgcnt_t		npgs, curnpgs, totpgs;
1978 	size_t		pgsz;
1979 	page_t		*pplist = NULL, *pp;
1980 	int		err = 0;
1981 	lgrp_t		*lgrp;
1982 
1983 	ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1));
1984 	ASSERT(pgflags == 0 || pgflags == PG_LOCAL);
1985 
1986 	VM_STAT_ADD(alloc_pages[0]);
1987 
1988 #ifdef DEBUG
1989 	if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) {
1990 		return (ENOMEM);
1991 	}
1992 #endif
1993 
1994 	pgsz = page_get_pagesize(szc);
1995 	totpgs = curnpgs = npgs = pgsz >> PAGESHIFT;
1996 
1997 	ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0);
1998 	/*
1999 	 * One must be NULL but not both.
2000 	 * And one must be non NULL but not both.
2001 	 */
2002 	ASSERT(basepp != NULL || ppa != NULL);
2003 	ASSERT(basepp == NULL || ppa == NULL);
2004 
2005 	(void) page_create_wait(npgs, PG_WAIT);
2006 
2007 	while (npgs && szc) {
2008 		lgrp = lgrp_mem_choose(seg, addr, pgsz);
2009 		if (pgflags == PG_LOCAL) {
2010 			pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2011 			    pgflags, lgrp);
2012 			if (pp == NULL) {
2013 				pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2014 				    0, lgrp);
2015 			}
2016 		} else {
2017 			pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2018 			    0, lgrp);
2019 		}
2020 		if (pp != NULL) {
2021 			VM_STAT_ADD(alloc_pages[1]);
2022 			page_list_concat(&pplist, &pp);
2023 			ASSERT(npgs >= curnpgs);
2024 			npgs -= curnpgs;
2025 		} else if (anypgsz) {
2026 			VM_STAT_ADD(alloc_pages[2]);
2027 			szc--;
2028 			pgsz = page_get_pagesize(szc);
2029 			curnpgs = pgsz >> PAGESHIFT;
2030 		} else {
2031 			VM_STAT_ADD(alloc_pages[3]);
2032 			ASSERT(npgs == totpgs);
2033 			page_create_putback(npgs);
2034 			return (ENOMEM);
2035 		}
2036 	}
2037 	if (szc == 0) {
2038 		VM_STAT_ADD(alloc_pages[4]);
2039 		ASSERT(npgs != 0);
2040 		page_create_putback(npgs);
2041 		err = ENOMEM;
2042 	} else if (basepp != NULL) {
2043 		ASSERT(npgs == 0);
2044 		ASSERT(ppa == NULL);
2045 		*basepp = pplist;
2046 	}
2047 
2048 	npgs = totpgs - npgs;
2049 	pp = pplist;
2050 
2051 	/*
2052 	 * Clear the free and age bits. Also if we were passed in a ppa then
2053 	 * fill it in with all the constituent pages from the large page. But
2054 	 * if we failed to allocate all the pages just free what we got.
2055 	 */
2056 	while (npgs != 0) {
2057 		ASSERT(PP_ISFREE(pp));
2058 		ASSERT(PP_ISAGED(pp));
2059 		if (ppa != NULL || err != 0) {
2060 			if (err == 0) {
2061 				VM_STAT_ADD(alloc_pages[5]);
2062 				PP_CLRFREE(pp);
2063 				PP_CLRAGED(pp);
2064 				page_sub(&pplist, pp);
2065 				*ppa++ = pp;
2066 				npgs--;
2067 			} else {
2068 				VM_STAT_ADD(alloc_pages[6]);
2069 				ASSERT(pp->p_szc != 0);
2070 				curnpgs = page_get_pagecnt(pp->p_szc);
2071 				page_list_break(&pp, &pplist, curnpgs);
2072 				page_list_add_pages(pp, 0);
2073 				page_create_putback(curnpgs);
2074 				ASSERT(npgs >= curnpgs);
2075 				npgs -= curnpgs;
2076 			}
2077 			pp = pplist;
2078 		} else {
2079 			VM_STAT_ADD(alloc_pages[7]);
2080 			PP_CLRFREE(pp);
2081 			PP_CLRAGED(pp);
2082 			pp = pp->p_next;
2083 			npgs--;
2084 		}
2085 	}
2086 	return (err);
2087 }
2088 
2089 /*
2090  * Get a single large page off of the freelists, and set it up for use.
2091  * Number of bytes requested must be a supported page size.
2092  *
2093  * Note that this call may fail even if there is sufficient
2094  * memory available or PG_WAIT is set, so the caller must
2095  * be willing to fallback on page_create_va(), block and retry,
2096  * or fail the requester.
2097  */
2098 page_t *
2099 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2100     struct seg *seg, caddr_t vaddr, void *arg)
2101 {
2102 	pgcnt_t		npages, pcftotal;
2103 	page_t		*pp;
2104 	page_t		*rootpp;
2105 	lgrp_t		*lgrp;
2106 	uint_t		enough;
2107 	uint_t		pcf_index;
2108 	uint_t		i;
2109 	struct pcf	*p;
2110 	struct pcf	*q;
2111 	lgrp_id_t	*lgrpid = (lgrp_id_t *)arg;
2112 
2113 	ASSERT(vp != NULL);
2114 
2115 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2116 	    PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0);
2117 	/* but no others */
2118 
2119 	ASSERT((flags & PG_EXCL) == PG_EXCL);
2120 
2121 	npages = btop(bytes);
2122 
2123 	if (!kcage_on || panicstr) {
2124 		/*
2125 		 * Cage is OFF, or we are single threaded in
2126 		 * panic, so make everything a RELOC request.
2127 		 */
2128 		flags &= ~PG_NORELOC;
2129 	}
2130 
2131 	/*
2132 	 * Make sure there's adequate physical memory available.
2133 	 * Note: PG_WAIT is ignored here.
2134 	 */
2135 	if (freemem <= throttlefree + npages) {
2136 		VM_STAT_ADD(page_create_large_cnt[1]);
2137 		return (NULL);
2138 	}
2139 
2140 	/*
2141 	 * If cage is on, dampen draw from cage when available
2142 	 * cage space is low.
2143 	 */
2144 	if ((flags & (PG_NORELOC | PG_WAIT)) ==  (PG_NORELOC | PG_WAIT) &&
2145 	    kcage_freemem < kcage_throttlefree + npages) {
2146 
2147 		/*
2148 		 * The cage is on, the caller wants PG_NORELOC
2149 		 * pages and available cage memory is very low.
2150 		 * Call kcage_create_throttle() to attempt to
2151 		 * control demand on the cage.
2152 		 */
2153 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE) {
2154 			VM_STAT_ADD(page_create_large_cnt[2]);
2155 			return (NULL);
2156 		}
2157 	}
2158 
2159 	enough = 0;
2160 	pcf_index = PCF_INDEX();
2161 	p = &pcf[pcf_index];
2162 	q = &pcf[PCF_FANOUT];
2163 	for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
2164 		if (p->pcf_count > npages) {
2165 			/*
2166 			 * a good one to try.
2167 			 */
2168 			mutex_enter(&p->pcf_lock);
2169 			if (p->pcf_count > npages) {
2170 				p->pcf_count -= (uint_t)npages;
2171 				/*
2172 				 * freemem is not protected by any lock.
2173 				 * Thus, we cannot have any assertion
2174 				 * containing freemem here.
2175 				 */
2176 				freemem -= npages;
2177 				enough = 1;
2178 				mutex_exit(&p->pcf_lock);
2179 				break;
2180 			}
2181 			mutex_exit(&p->pcf_lock);
2182 		}
2183 		pcftotal += p->pcf_count;
2184 		p++;
2185 		if (p >= q) {
2186 			p = pcf;
2187 		}
2188 	}
2189 
2190 	if (!enough) {
2191 		/* If there isn't enough memory available, give up. */
2192 		if (pcftotal < npages) {
2193 			VM_STAT_ADD(page_create_large_cnt[3]);
2194 			return (NULL);
2195 		}
2196 
2197 		/* try to collect pages from several pcf bins */
2198 		for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
2199 			mutex_enter(&p->pcf_lock);
2200 			pcftotal += p->pcf_count;
2201 			if (pcftotal >= npages) {
2202 				/*
2203 				 * Wow!  There are enough pages laying around
2204 				 * to satisfy the request.  Do the accounting,
2205 				 * drop the locks we acquired, and go back.
2206 				 *
2207 				 * freemem is not protected by any lock. So,
2208 				 * we cannot have any assertion containing
2209 				 * freemem.
2210 				 */
2211 				pgcnt_t	tpages = npages;
2212 				freemem -= npages;
2213 				while (p >= pcf) {
2214 					if (p->pcf_count <= tpages) {
2215 						tpages -= p->pcf_count;
2216 						p->pcf_count = 0;
2217 					} else {
2218 						p->pcf_count -= (uint_t)tpages;
2219 						tpages = 0;
2220 					}
2221 					mutex_exit(&p->pcf_lock);
2222 					p--;
2223 				}
2224 				ASSERT(tpages == 0);
2225 				break;
2226 			}
2227 			p++;
2228 		}
2229 		if (i == PCF_FANOUT) {
2230 			/* failed to collect pages - release the locks */
2231 			while (--p >= pcf) {
2232 				mutex_exit(&p->pcf_lock);
2233 			}
2234 			VM_STAT_ADD(page_create_large_cnt[4]);
2235 			return (NULL);
2236 		}
2237 	}
2238 
2239 	/*
2240 	 * This is where this function behaves fundamentally differently
2241 	 * than page_create_va(); since we're intending to map the page
2242 	 * with a single TTE, we have to get it as a physically contiguous
2243 	 * hardware pagesize chunk.  If we can't, we fail.
2244 	 */
2245 	if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max &&
2246 	    LGRP_EXISTS(lgrp_table[*lgrpid]))
2247 		lgrp = lgrp_table[*lgrpid];
2248 	else
2249 		lgrp = lgrp_mem_choose(seg, vaddr, bytes);
2250 
2251 	if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr,
2252 	    bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) {
2253 		page_create_putback(npages);
2254 		VM_STAT_ADD(page_create_large_cnt[5]);
2255 		return (NULL);
2256 	}
2257 
2258 	/*
2259 	 * if we got the page with the wrong mtype give it back this is a
2260 	 * workaround for CR 6249718. When CR 6249718 is fixed we never get
2261 	 * inside "if" and the workaround becomes just a nop
2262 	 */
2263 	if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) {
2264 		page_list_add_pages(rootpp, 0);
2265 		page_create_putback(npages);
2266 		VM_STAT_ADD(page_create_large_cnt[6]);
2267 		return (NULL);
2268 	}
2269 
2270 	/*
2271 	 * If satisfying this request has left us with too little
2272 	 * memory, start the wheels turning to get some back.  The
2273 	 * first clause of the test prevents waking up the pageout
2274 	 * daemon in situations where it would decide that there's
2275 	 * nothing to do.
2276 	 */
2277 	if (nscan < desscan && freemem < minfree) {
2278 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2279 		    "pageout_cv_signal:freemem %ld", freemem);
2280 		cv_signal(&proc_pageout->p_cv);
2281 	}
2282 
2283 	pp = rootpp;
2284 	while (npages--) {
2285 		ASSERT(PAGE_EXCL(pp));
2286 		ASSERT(pp->p_vnode == NULL);
2287 		ASSERT(!hat_page_is_mapped(pp));
2288 		PP_CLRFREE(pp);
2289 		PP_CLRAGED(pp);
2290 		if (!page_hashin(pp, vp, off, NULL))
2291 			panic("page_create_large: hashin failed: page %p",
2292 			    (void *)pp);
2293 		page_io_lock(pp);
2294 		off += PAGESIZE;
2295 		pp = pp->p_next;
2296 	}
2297 
2298 	VM_STAT_ADD(page_create_large_cnt[0]);
2299 	return (rootpp);
2300 }
2301 
2302 page_t *
2303 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2304     struct seg *seg, caddr_t vaddr)
2305 {
2306 	page_t		*plist = NULL;
2307 	pgcnt_t		npages;
2308 	pgcnt_t		found_on_free = 0;
2309 	pgcnt_t		pages_req;
2310 	page_t		*npp = NULL;
2311 	uint_t		enough;
2312 	uint_t		i;
2313 	uint_t		pcf_index;
2314 	struct pcf	*p;
2315 	struct pcf	*q;
2316 	lgrp_t		*lgrp;
2317 
2318 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
2319 	    "page_create_start:vp %p off %llx bytes %lu flags %x",
2320 	    vp, off, bytes, flags);
2321 
2322 	ASSERT(bytes != 0 && vp != NULL);
2323 
2324 	if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) {
2325 		panic("page_create: invalid flags");
2326 		/*NOTREACHED*/
2327 	}
2328 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2329 	    PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0);
2330 	    /* but no others */
2331 
2332 	pages_req = npages = btopr(bytes);
2333 	/*
2334 	 * Try to see whether request is too large to *ever* be
2335 	 * satisfied, in order to prevent deadlock.  We arbitrarily
2336 	 * decide to limit maximum size requests to max_page_get.
2337 	 */
2338 	if (npages >= max_page_get) {
2339 		if ((flags & PG_WAIT) == 0) {
2340 			TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG,
2341 			    "page_create_toobig:vp %p off %llx npages "
2342 			    "%lu max_page_get %lu",
2343 			    vp, off, npages, max_page_get);
2344 			return (NULL);
2345 		} else {
2346 			cmn_err(CE_WARN,
2347 			    "Request for too much kernel memory "
2348 			    "(%lu bytes), will hang forever", bytes);
2349 			for (;;)
2350 				delay(1000000000);
2351 		}
2352 	}
2353 
2354 	if (!kcage_on || panicstr) {
2355 		/*
2356 		 * Cage is OFF, or we are single threaded in
2357 		 * panic, so make everything a RELOC request.
2358 		 */
2359 		flags &= ~PG_NORELOC;
2360 	}
2361 
2362 	if (freemem <= throttlefree + npages)
2363 		if (!page_create_throttle(npages, flags))
2364 			return (NULL);
2365 
2366 	/*
2367 	 * If cage is on, dampen draw from cage when available
2368 	 * cage space is low.
2369 	 */
2370 	if ((flags & PG_NORELOC) &&
2371 	    kcage_freemem < kcage_throttlefree + npages) {
2372 
2373 		/*
2374 		 * The cage is on, the caller wants PG_NORELOC
2375 		 * pages and available cage memory is very low.
2376 		 * Call kcage_create_throttle() to attempt to
2377 		 * control demand on the cage.
2378 		 */
2379 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE)
2380 			return (NULL);
2381 	}
2382 
2383 	VM_STAT_ADD(page_create_cnt[0]);
2384 
2385 	enough = 0;
2386 	pcf_index = PCF_INDEX();
2387 
2388 	p = &pcf[pcf_index];
2389 	q = &pcf[PCF_FANOUT];
2390 	for (i = 0; i < PCF_FANOUT; i++) {
2391 		if (p->pcf_count > npages) {
2392 			/*
2393 			 * a good one to try.
2394 			 */
2395 			mutex_enter(&p->pcf_lock);
2396 			if (p->pcf_count > npages) {
2397 				p->pcf_count -= (uint_t)npages;
2398 				/*
2399 				 * freemem is not protected by any lock.
2400 				 * Thus, we cannot have any assertion
2401 				 * containing freemem here.
2402 				 */
2403 				freemem -= npages;
2404 				enough = 1;
2405 				mutex_exit(&p->pcf_lock);
2406 				break;
2407 			}
2408 			mutex_exit(&p->pcf_lock);
2409 		}
2410 		p++;
2411 		if (p >= q) {
2412 			p = pcf;
2413 		}
2414 	}
2415 
2416 	if (!enough) {
2417 		/*
2418 		 * Have to look harder.  If npages is greater than
2419 		 * one, then we might have to coalecse the counters.
2420 		 *
2421 		 * Go wait.  We come back having accounted
2422 		 * for the memory.
2423 		 */
2424 		VM_STAT_ADD(page_create_cnt[1]);
2425 		if (!page_create_wait(npages, flags)) {
2426 			VM_STAT_ADD(page_create_cnt[2]);
2427 			return (NULL);
2428 		}
2429 	}
2430 
2431 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
2432 	    "page_create_success:vp %p off %llx", vp, off);
2433 
2434 	/*
2435 	 * If satisfying this request has left us with too little
2436 	 * memory, start the wheels turning to get some back.  The
2437 	 * first clause of the test prevents waking up the pageout
2438 	 * daemon in situations where it would decide that there's
2439 	 * nothing to do.
2440 	 */
2441 	if (nscan < desscan && freemem < minfree) {
2442 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2443 		    "pageout_cv_signal:freemem %ld", freemem);
2444 		cv_signal(&proc_pageout->p_cv);
2445 	}
2446 
2447 	/*
2448 	 * Loop around collecting the requested number of pages.
2449 	 * Most of the time, we have to `create' a new page. With
2450 	 * this in mind, pull the page off the free list before
2451 	 * getting the hash lock.  This will minimize the hash
2452 	 * lock hold time, nesting, and the like.  If it turns
2453 	 * out we don't need the page, we put it back at the end.
2454 	 */
2455 	while (npages--) {
2456 		page_t		*pp;
2457 		kmutex_t	*phm = NULL;
2458 		ulong_t		index;
2459 
2460 		index = PAGE_HASH_FUNC(vp, off);
2461 top:
2462 		ASSERT(phm == NULL);
2463 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
2464 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
2465 
2466 		if (npp == NULL) {
2467 			/*
2468 			 * Try to get a page from the freelist (ie,
2469 			 * a page with no [vp, off] tag).  If that
2470 			 * fails, use the cachelist.
2471 			 *
2472 			 * During the first attempt at both the free
2473 			 * and cache lists we try for the correct color.
2474 			 */
2475 			/*
2476 			 * XXXX-how do we deal with virtual indexed
2477 			 * caches and and colors?
2478 			 */
2479 			VM_STAT_ADD(page_create_cnt[4]);
2480 			/*
2481 			 * Get lgroup to allocate next page of shared memory
2482 			 * from and use it to specify where to allocate
2483 			 * the physical memory
2484 			 */
2485 			lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
2486 			npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
2487 			    flags | PG_MATCH_COLOR, lgrp);
2488 			if (npp == NULL) {
2489 				npp = page_get_cachelist(vp, off, seg,
2490 				    vaddr, flags | PG_MATCH_COLOR, lgrp);
2491 				if (npp == NULL) {
2492 					npp = page_create_get_something(vp,
2493 					    off, seg, vaddr,
2494 					    flags & ~PG_MATCH_COLOR);
2495 				}
2496 
2497 				if (PP_ISAGED(npp) == 0) {
2498 					/*
2499 					 * Since this page came from the
2500 					 * cachelist, we must destroy the
2501 					 * old vnode association.
2502 					 */
2503 					page_hashout(npp, NULL);
2504 				}
2505 			}
2506 		}
2507 
2508 		/*
2509 		 * We own this page!
2510 		 */
2511 		ASSERT(PAGE_EXCL(npp));
2512 		ASSERT(npp->p_vnode == NULL);
2513 		ASSERT(!hat_page_is_mapped(npp));
2514 		PP_CLRFREE(npp);
2515 		PP_CLRAGED(npp);
2516 
2517 		/*
2518 		 * Here we have a page in our hot little mits and are
2519 		 * just waiting to stuff it on the appropriate lists.
2520 		 * Get the mutex and check to see if it really does
2521 		 * not exist.
2522 		 */
2523 		phm = PAGE_HASH_MUTEX(index);
2524 		mutex_enter(phm);
2525 		PAGE_HASH_SEARCH(index, pp, vp, off);
2526 		if (pp == NULL) {
2527 			VM_STAT_ADD(page_create_new);
2528 			pp = npp;
2529 			npp = NULL;
2530 			if (!page_hashin(pp, vp, off, phm)) {
2531 				/*
2532 				 * Since we hold the page hash mutex and
2533 				 * just searched for this page, page_hashin
2534 				 * had better not fail.  If it does, that
2535 				 * means somethread did not follow the
2536 				 * page hash mutex rules.  Panic now and
2537 				 * get it over with.  As usual, go down
2538 				 * holding all the locks.
2539 				 */
2540 				ASSERT(MUTEX_HELD(phm));
2541 				panic("page_create: "
2542 				    "hashin failed %p %p %llx %p",
2543 				    (void *)pp, (void *)vp, off, (void *)phm);
2544 				/*NOTREACHED*/
2545 			}
2546 			ASSERT(MUTEX_HELD(phm));
2547 			mutex_exit(phm);
2548 			phm = NULL;
2549 
2550 			/*
2551 			 * Hat layer locking need not be done to set
2552 			 * the following bits since the page is not hashed
2553 			 * and was on the free list (i.e., had no mappings).
2554 			 *
2555 			 * Set the reference bit to protect
2556 			 * against immediate pageout
2557 			 *
2558 			 * XXXmh modify freelist code to set reference
2559 			 * bit so we don't have to do it here.
2560 			 */
2561 			page_set_props(pp, P_REF);
2562 			found_on_free++;
2563 		} else {
2564 			VM_STAT_ADD(page_create_exists);
2565 			if (flags & PG_EXCL) {
2566 				/*
2567 				 * Found an existing page, and the caller
2568 				 * wanted all new pages.  Undo all of the work
2569 				 * we have done.
2570 				 */
2571 				mutex_exit(phm);
2572 				phm = NULL;
2573 				while (plist != NULL) {
2574 					pp = plist;
2575 					page_sub(&plist, pp);
2576 					page_io_unlock(pp);
2577 					/* large pages should not end up here */
2578 					ASSERT(pp->p_szc == 0);
2579 					/*LINTED: constant in conditional ctx*/
2580 					VN_DISPOSE(pp, B_INVAL, 0, kcred);
2581 				}
2582 				VM_STAT_ADD(page_create_found_one);
2583 				goto fail;
2584 			}
2585 			ASSERT(flags & PG_WAIT);
2586 			if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) {
2587 				/*
2588 				 * Start all over again if we blocked trying
2589 				 * to lock the page.
2590 				 */
2591 				mutex_exit(phm);
2592 				VM_STAT_ADD(page_create_page_lock_failed);
2593 				phm = NULL;
2594 				goto top;
2595 			}
2596 			mutex_exit(phm);
2597 			phm = NULL;
2598 
2599 			if (PP_ISFREE(pp)) {
2600 				ASSERT(PP_ISAGED(pp) == 0);
2601 				VM_STAT_ADD(pagecnt.pc_get_cache);
2602 				page_list_sub(pp, PG_CACHE_LIST);
2603 				PP_CLRFREE(pp);
2604 				found_on_free++;
2605 			}
2606 		}
2607 
2608 		/*
2609 		 * Got a page!  It is locked.  Acquire the i/o
2610 		 * lock since we are going to use the p_next and
2611 		 * p_prev fields to link the requested pages together.
2612 		 */
2613 		page_io_lock(pp);
2614 		page_add(&plist, pp);
2615 		plist = plist->p_next;
2616 		off += PAGESIZE;
2617 		vaddr += PAGESIZE;
2618 	}
2619 
2620 	ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1);
2621 fail:
2622 	if (npp != NULL) {
2623 		/*
2624 		 * Did not need this page after all.
2625 		 * Put it back on the free list.
2626 		 */
2627 		VM_STAT_ADD(page_create_putbacks);
2628 		PP_SETFREE(npp);
2629 		PP_SETAGED(npp);
2630 		npp->p_offset = (u_offset_t)-1;
2631 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
2632 		page_unlock(npp);
2633 
2634 	}
2635 
2636 	ASSERT(pages_req >= found_on_free);
2637 
2638 	{
2639 		uint_t overshoot = (uint_t)(pages_req - found_on_free);
2640 
2641 		if (overshoot) {
2642 			VM_STAT_ADD(page_create_overshoot);
2643 			p = &pcf[pcf_index];
2644 			mutex_enter(&p->pcf_lock);
2645 			if (p->pcf_block) {
2646 				p->pcf_reserve += overshoot;
2647 			} else {
2648 				p->pcf_count += overshoot;
2649 				if (p->pcf_wait) {
2650 					mutex_enter(&new_freemem_lock);
2651 					if (freemem_wait) {
2652 						cv_signal(&freemem_cv);
2653 						p->pcf_wait--;
2654 					} else {
2655 						p->pcf_wait = 0;
2656 					}
2657 					mutex_exit(&new_freemem_lock);
2658 				}
2659 			}
2660 			mutex_exit(&p->pcf_lock);
2661 			/* freemem is approximate, so this test OK */
2662 			if (!p->pcf_block)
2663 				freemem += overshoot;
2664 		}
2665 	}
2666 
2667 	return (plist);
2668 }
2669 
2670 /*
2671  * One or more constituent pages of this large page has been marked
2672  * toxic. Simply demote the large page to PAGESIZE pages and let
2673  * page_free() handle it. This routine should only be called by
2674  * large page free routines (page_free_pages() and page_destroy_pages().
2675  * All pages are locked SE_EXCL and have already been marked free.
2676  */
2677 static void
2678 page_free_toxic_pages(page_t *rootpp)
2679 {
2680 	page_t	*tpp;
2681 	pgcnt_t	i, pgcnt = page_get_pagecnt(rootpp->p_szc);
2682 	uint_t	szc = rootpp->p_szc;
2683 
2684 	for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) {
2685 		ASSERT(tpp->p_szc == szc);
2686 		ASSERT((PAGE_EXCL(tpp) &&
2687 		    !page_iolock_assert(tpp)) || panicstr);
2688 		tpp->p_szc = 0;
2689 	}
2690 
2691 	while (rootpp != NULL) {
2692 		tpp = rootpp;
2693 		page_sub(&rootpp, tpp);
2694 		ASSERT(PP_ISFREE(tpp));
2695 		PP_CLRFREE(tpp);
2696 		page_free(tpp, 1);
2697 	}
2698 }
2699 
2700 /*
2701  * Put page on the "free" list.
2702  * The free list is really two lists maintained by
2703  * the PSM of whatever machine we happen to be on.
2704  */
2705 void
2706 page_free(page_t *pp, int dontneed)
2707 {
2708 	struct pcf	*p;
2709 	uint_t		pcf_index;
2710 
2711 	ASSERT((PAGE_EXCL(pp) &&
2712 	    !page_iolock_assert(pp)) || panicstr);
2713 
2714 	if (PP_ISFREE(pp)) {
2715 		panic("page_free: page %p is free", (void *)pp);
2716 	}
2717 
2718 	if (pp->p_szc != 0) {
2719 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
2720 		    PP_ISKAS(pp)) {
2721 			panic("page_free: anon or kernel "
2722 			    "or no vnode large page %p", (void *)pp);
2723 		}
2724 		page_demote_vp_pages(pp);
2725 		ASSERT(pp->p_szc == 0);
2726 	}
2727 
2728 	/*
2729 	 * The page_struct_lock need not be acquired to examine these
2730 	 * fields since the page has an "exclusive" lock.
2731 	 */
2732 	if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
2733 	    pp->p_slckcnt != 0) {
2734 		panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
2735 		    "slckcnt = %d", pp, page_pptonum(pp), pp->p_lckcnt,
2736 		    pp->p_cowcnt, pp->p_slckcnt);
2737 		/*NOTREACHED*/
2738 	}
2739 
2740 	ASSERT(!hat_page_getshare(pp));
2741 
2742 	PP_SETFREE(pp);
2743 	ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) ||
2744 	    !hat_ismod(pp));
2745 	page_clr_all_props(pp);
2746 	ASSERT(!hat_page_getshare(pp));
2747 
2748 	/*
2749 	 * Now we add the page to the head of the free list.
2750 	 * But if this page is associated with a paged vnode
2751 	 * then we adjust the head forward so that the page is
2752 	 * effectively at the end of the list.
2753 	 */
2754 	if (pp->p_vnode == NULL) {
2755 		/*
2756 		 * Page has no identity, put it on the free list.
2757 		 */
2758 		PP_SETAGED(pp);
2759 		pp->p_offset = (u_offset_t)-1;
2760 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
2761 		VM_STAT_ADD(pagecnt.pc_free_free);
2762 		TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2763 		    "page_free_free:pp %p", pp);
2764 	} else {
2765 		PP_CLRAGED(pp);
2766 
2767 		if (!dontneed || nopageage) {
2768 			/* move it to the tail of the list */
2769 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL);
2770 
2771 			VM_STAT_ADD(pagecnt.pc_free_cache);
2772 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL,
2773 			    "page_free_cache_tail:pp %p", pp);
2774 		} else {
2775 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD);
2776 
2777 			VM_STAT_ADD(pagecnt.pc_free_dontneed);
2778 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD,
2779 			    "page_free_cache_head:pp %p", pp);
2780 		}
2781 	}
2782 	page_unlock(pp);
2783 
2784 	/*
2785 	 * Now do the `freemem' accounting.
2786 	 */
2787 	pcf_index = PCF_INDEX();
2788 	p = &pcf[pcf_index];
2789 
2790 	mutex_enter(&p->pcf_lock);
2791 	if (p->pcf_block) {
2792 		p->pcf_reserve += 1;
2793 	} else {
2794 		p->pcf_count += 1;
2795 		if (p->pcf_wait) {
2796 			mutex_enter(&new_freemem_lock);
2797 			/*
2798 			 * Check to see if some other thread
2799 			 * is actually waiting.  Another bucket
2800 			 * may have woken it up by now.  If there
2801 			 * are no waiters, then set our pcf_wait
2802 			 * count to zero to avoid coming in here
2803 			 * next time.  Also, since only one page
2804 			 * was put on the free list, just wake
2805 			 * up one waiter.
2806 			 */
2807 			if (freemem_wait) {
2808 				cv_signal(&freemem_cv);
2809 				p->pcf_wait--;
2810 			} else {
2811 				p->pcf_wait = 0;
2812 			}
2813 			mutex_exit(&new_freemem_lock);
2814 		}
2815 	}
2816 	mutex_exit(&p->pcf_lock);
2817 
2818 	/* freemem is approximate, so this test OK */
2819 	if (!p->pcf_block)
2820 		freemem += 1;
2821 }
2822 
2823 /*
2824  * Put page on the "free" list during intial startup.
2825  * This happens during initial single threaded execution.
2826  */
2827 void
2828 page_free_at_startup(page_t *pp)
2829 {
2830 	struct pcf	*p;
2831 	uint_t		pcf_index;
2832 
2833 	page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT);
2834 	VM_STAT_ADD(pagecnt.pc_free_free);
2835 
2836 	/*
2837 	 * Now do the `freemem' accounting.
2838 	 */
2839 	pcf_index = PCF_INDEX();
2840 	p = &pcf[pcf_index];
2841 
2842 	ASSERT(p->pcf_block == 0);
2843 	ASSERT(p->pcf_wait == 0);
2844 	p->pcf_count += 1;
2845 
2846 	/* freemem is approximate, so this is OK */
2847 	freemem += 1;
2848 }
2849 
2850 void
2851 page_free_pages(page_t *pp)
2852 {
2853 	page_t	*tpp, *rootpp = NULL;
2854 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
2855 	pgcnt_t	i;
2856 	uint_t	szc = pp->p_szc;
2857 
2858 	VM_STAT_ADD(pagecnt.pc_free_pages);
2859 	TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2860 	    "page_free_free:pp %p", pp);
2861 
2862 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
2863 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
2864 		panic("page_free_pages: not root page %p", (void *)pp);
2865 		/*NOTREACHED*/
2866 	}
2867 
2868 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
2869 		ASSERT((PAGE_EXCL(tpp) &&
2870 		    !page_iolock_assert(tpp)) || panicstr);
2871 		if (PP_ISFREE(tpp)) {
2872 			panic("page_free_pages: page %p is free", (void *)tpp);
2873 			/*NOTREACHED*/
2874 		}
2875 		if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 ||
2876 		    tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) {
2877 			panic("page_free_pages %p", (void *)tpp);
2878 			/*NOTREACHED*/
2879 		}
2880 
2881 		ASSERT(!hat_page_getshare(tpp));
2882 		ASSERT(tpp->p_vnode == NULL);
2883 		ASSERT(tpp->p_szc == szc);
2884 
2885 		PP_SETFREE(tpp);
2886 		page_clr_all_props(tpp);
2887 		PP_SETAGED(tpp);
2888 		tpp->p_offset = (u_offset_t)-1;
2889 		ASSERT(tpp->p_next == tpp);
2890 		ASSERT(tpp->p_prev == tpp);
2891 		page_list_concat(&rootpp, &tpp);
2892 	}
2893 	ASSERT(rootpp == pp);
2894 
2895 	page_list_add_pages(rootpp, 0);
2896 	page_create_putback(pgcnt);
2897 }
2898 
2899 int free_pages = 1;
2900 
2901 /*
2902  * This routine attempts to return pages to the cachelist via page_release().
2903  * It does not *have* to be successful in all cases, since the pageout scanner
2904  * will catch any pages it misses.  It does need to be fast and not introduce
2905  * too much overhead.
2906  *
2907  * If a page isn't found on the unlocked sweep of the page_hash bucket, we
2908  * don't lock and retry.  This is ok, since the page scanner will eventually
2909  * find any page we miss in free_vp_pages().
2910  */
2911 void
2912 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len)
2913 {
2914 	page_t *pp;
2915 	u_offset_t eoff;
2916 	extern int swap_in_range(vnode_t *, u_offset_t, size_t);
2917 
2918 	eoff = off + len;
2919 
2920 	if (free_pages == 0)
2921 		return;
2922 	if (swap_in_range(vp, off, len))
2923 		return;
2924 
2925 	for (; off < eoff; off += PAGESIZE) {
2926 
2927 		/*
2928 		 * find the page using a fast, but inexact search. It'll be OK
2929 		 * if a few pages slip through the cracks here.
2930 		 */
2931 		pp = page_exists(vp, off);
2932 
2933 		/*
2934 		 * If we didn't find the page (it may not exist), the page
2935 		 * is free, looks still in use (shared), or we can't lock it,
2936 		 * just give up.
2937 		 */
2938 		if (pp == NULL ||
2939 		    PP_ISFREE(pp) ||
2940 		    page_share_cnt(pp) > 0 ||
2941 		    !page_trylock(pp, SE_EXCL))
2942 			continue;
2943 
2944 		/*
2945 		 * Once we have locked pp, verify that it's still the
2946 		 * correct page and not already free
2947 		 */
2948 		ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL));
2949 		if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) {
2950 			page_unlock(pp);
2951 			continue;
2952 		}
2953 
2954 		/*
2955 		 * try to release the page...
2956 		 */
2957 		(void) page_release(pp, 1);
2958 	}
2959 }
2960 
2961 /*
2962  * Reclaim the given page from the free list.
2963  * If pp is part of a large pages, only the given constituent page is reclaimed
2964  * and the large page it belonged to will be demoted.  This can only happen
2965  * if the page is not on the cachelist.
2966  *
2967  * Returns 1 on success or 0 on failure.
2968  *
2969  * The page is unlocked if it can't be reclaimed (when freemem == 0).
2970  * If `lock' is non-null, it will be dropped and re-acquired if
2971  * the routine must wait while freemem is 0.
2972  *
2973  * As it turns out, boot_getpages() does this.  It picks a page,
2974  * based on where OBP mapped in some address, gets its pfn, searches
2975  * the memsegs, locks the page, then pulls it off the free list!
2976  */
2977 int
2978 page_reclaim(page_t *pp, kmutex_t *lock)
2979 {
2980 	struct pcf	*p;
2981 	uint_t		pcf_index;
2982 	struct cpu	*cpup;
2983 	int		enough;
2984 	uint_t		i;
2985 
2986 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
2987 	ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
2988 
2989 	/*
2990 	 * If `freemem' is 0, we cannot reclaim this page from the
2991 	 * freelist, so release every lock we might hold: the page,
2992 	 * and the `lock' before blocking.
2993 	 *
2994 	 * The only way `freemem' can become 0 while there are pages
2995 	 * marked free (have their p->p_free bit set) is when the
2996 	 * system is low on memory and doing a page_create().  In
2997 	 * order to guarantee that once page_create() starts acquiring
2998 	 * pages it will be able to get all that it needs since `freemem'
2999 	 * was decreased by the requested amount.  So, we need to release
3000 	 * this page, and let page_create() have it.
3001 	 *
3002 	 * Since `freemem' being zero is not supposed to happen, just
3003 	 * use the usual hash stuff as a starting point.  If that bucket
3004 	 * is empty, then assume the worst, and start at the beginning
3005 	 * of the pcf array.  If we always start at the beginning
3006 	 * when acquiring more than one pcf lock, there won't be any
3007 	 * deadlock problems.
3008 	 */
3009 
3010 	/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
3011 
3012 	if (freemem <= throttlefree && !page_create_throttle(1l, 0)) {
3013 		pcf_acquire_all();
3014 		goto page_reclaim_nomem;
3015 	}
3016 
3017 	enough = 0;
3018 	pcf_index = PCF_INDEX();
3019 	p = &pcf[pcf_index];
3020 	mutex_enter(&p->pcf_lock);
3021 	if (p->pcf_count >= 1) {
3022 		enough = 1;
3023 		p->pcf_count--;
3024 	}
3025 	mutex_exit(&p->pcf_lock);
3026 
3027 	if (!enough) {
3028 		VM_STAT_ADD(page_reclaim_zero);
3029 		/*
3030 		 * Check again. Its possible that some other thread
3031 		 * could have been right behind us, and added one
3032 		 * to a list somewhere.  Acquire each of the pcf locks
3033 		 * until we find a page.
3034 		 */
3035 		p = pcf;
3036 		for (i = 0; i < PCF_FANOUT; i++) {
3037 			mutex_enter(&p->pcf_lock);
3038 			if (p->pcf_count >= 1) {
3039 				p->pcf_count -= 1;
3040 				enough = 1;
3041 				break;
3042 			}
3043 			p++;
3044 		}
3045 
3046 		if (!enough) {
3047 page_reclaim_nomem:
3048 			/*
3049 			 * We really can't have page `pp'.
3050 			 * Time for the no-memory dance with
3051 			 * page_free().  This is just like
3052 			 * page_create_wait().  Plus the added
3053 			 * attraction of releasing whatever mutex
3054 			 * we held when we were called with in `lock'.
3055 			 * Page_unlock() will wakeup any thread
3056 			 * waiting around for this page.
3057 			 */
3058 			if (lock) {
3059 				VM_STAT_ADD(page_reclaim_zero_locked);
3060 				mutex_exit(lock);
3061 			}
3062 			page_unlock(pp);
3063 
3064 			/*
3065 			 * get this before we drop all the pcf locks.
3066 			 */
3067 			mutex_enter(&new_freemem_lock);
3068 
3069 			p = pcf;
3070 			for (i = 0; i < PCF_FANOUT; i++) {
3071 				p->pcf_wait++;
3072 				mutex_exit(&p->pcf_lock);
3073 				p++;
3074 			}
3075 
3076 			freemem_wait++;
3077 			cv_wait(&freemem_cv, &new_freemem_lock);
3078 			freemem_wait--;
3079 
3080 			mutex_exit(&new_freemem_lock);
3081 
3082 			if (lock) {
3083 				mutex_enter(lock);
3084 			}
3085 			return (0);
3086 		}
3087 
3088 		/*
3089 		 * The pcf accounting has been done,
3090 		 * though none of the pcf_wait flags have been set,
3091 		 * drop the locks and continue on.
3092 		 */
3093 		while (p >= pcf) {
3094 			mutex_exit(&p->pcf_lock);
3095 			p--;
3096 		}
3097 	}
3098 
3099 	/*
3100 	 * freemem is not protected by any lock. Thus, we cannot
3101 	 * have any assertion containing freemem here.
3102 	 */
3103 	freemem -= 1;
3104 
3105 	VM_STAT_ADD(pagecnt.pc_reclaim);
3106 
3107 	/*
3108 	 * page_list_sub will handle the case where pp is a large page.
3109 	 * It's possible that the page was promoted while on the freelist
3110 	 */
3111 	if (PP_ISAGED(pp)) {
3112 		page_list_sub(pp, PG_FREE_LIST);
3113 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
3114 		    "page_reclaim_free:pp %p", pp);
3115 	} else {
3116 		page_list_sub(pp, PG_CACHE_LIST);
3117 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
3118 		    "page_reclaim_cache:pp %p", pp);
3119 	}
3120 
3121 	/*
3122 	 * clear the p_free & p_age bits since this page is no longer
3123 	 * on the free list.  Notice that there was a brief time where
3124 	 * a page is marked as free, but is not on the list.
3125 	 *
3126 	 * Set the reference bit to protect against immediate pageout.
3127 	 */
3128 	PP_CLRFREE(pp);
3129 	PP_CLRAGED(pp);
3130 	page_set_props(pp, P_REF);
3131 
3132 	CPU_STATS_ENTER_K();
3133 	cpup = CPU;	/* get cpup now that CPU cannot change */
3134 	CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
3135 	CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
3136 	CPU_STATS_EXIT_K();
3137 	ASSERT(pp->p_szc == 0);
3138 
3139 	return (1);
3140 }
3141 
3142 /*
3143  * Destroy identity of the page and put it back on
3144  * the page free list.  Assumes that the caller has
3145  * acquired the "exclusive" lock on the page.
3146  */
3147 void
3148 page_destroy(page_t *pp, int dontfree)
3149 {
3150 	ASSERT((PAGE_EXCL(pp) &&
3151 	    !page_iolock_assert(pp)) || panicstr);
3152 	ASSERT(pp->p_slckcnt == 0 || panicstr);
3153 
3154 	if (pp->p_szc != 0) {
3155 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
3156 		    PP_ISKAS(pp)) {
3157 			panic("page_destroy: anon or kernel or no vnode "
3158 			    "large page %p", (void *)pp);
3159 		}
3160 		page_demote_vp_pages(pp);
3161 		ASSERT(pp->p_szc == 0);
3162 	}
3163 
3164 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp);
3165 
3166 	/*
3167 	 * Unload translations, if any, then hash out the
3168 	 * page to erase its identity.
3169 	 */
3170 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3171 	page_hashout(pp, NULL);
3172 
3173 	if (!dontfree) {
3174 		/*
3175 		 * Acquire the "freemem_lock" for availrmem.
3176 		 * The page_struct_lock need not be acquired for lckcnt
3177 		 * and cowcnt since the page has an "exclusive" lock.
3178 		 */
3179 		if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
3180 			mutex_enter(&freemem_lock);
3181 			if (pp->p_lckcnt != 0) {
3182 				availrmem++;
3183 				pp->p_lckcnt = 0;
3184 			}
3185 			if (pp->p_cowcnt != 0) {
3186 				availrmem += pp->p_cowcnt;
3187 				pp->p_cowcnt = 0;
3188 			}
3189 			mutex_exit(&freemem_lock);
3190 		}
3191 		/*
3192 		 * Put the page on the "free" list.
3193 		 */
3194 		page_free(pp, 0);
3195 	}
3196 }
3197 
3198 void
3199 page_destroy_pages(page_t *pp)
3200 {
3201 
3202 	page_t	*tpp, *rootpp = NULL;
3203 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
3204 	pgcnt_t	i, pglcks = 0;
3205 	uint_t	szc = pp->p_szc;
3206 
3207 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
3208 
3209 	VM_STAT_ADD(pagecnt.pc_destroy_pages);
3210 
3211 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp);
3212 
3213 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
3214 		panic("page_destroy_pages: not root page %p", (void *)pp);
3215 		/*NOTREACHED*/
3216 	}
3217 
3218 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
3219 		ASSERT((PAGE_EXCL(tpp) &&
3220 		    !page_iolock_assert(tpp)) || panicstr);
3221 		ASSERT(tpp->p_slckcnt == 0 || panicstr);
3222 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
3223 		page_hashout(tpp, NULL);
3224 		ASSERT(tpp->p_offset == (u_offset_t)-1);
3225 		if (tpp->p_lckcnt != 0) {
3226 			pglcks++;
3227 			tpp->p_lckcnt = 0;
3228 		} else if (tpp->p_cowcnt != 0) {
3229 			pglcks += tpp->p_cowcnt;
3230 			tpp->p_cowcnt = 0;
3231 		}
3232 		ASSERT(!hat_page_getshare(tpp));
3233 		ASSERT(tpp->p_vnode == NULL);
3234 		ASSERT(tpp->p_szc == szc);
3235 
3236 		PP_SETFREE(tpp);
3237 		page_clr_all_props(tpp);
3238 		PP_SETAGED(tpp);
3239 		ASSERT(tpp->p_next == tpp);
3240 		ASSERT(tpp->p_prev == tpp);
3241 		page_list_concat(&rootpp, &tpp);
3242 	}
3243 
3244 	ASSERT(rootpp == pp);
3245 	if (pglcks != 0) {
3246 		mutex_enter(&freemem_lock);
3247 		availrmem += pglcks;
3248 		mutex_exit(&freemem_lock);
3249 	}
3250 
3251 	page_list_add_pages(rootpp, 0);
3252 	page_create_putback(pgcnt);
3253 }
3254 
3255 /*
3256  * Similar to page_destroy(), but destroys pages which are
3257  * locked and known to be on the page free list.  Since
3258  * the page is known to be free and locked, no one can access
3259  * it.
3260  *
3261  * Also, the number of free pages does not change.
3262  */
3263 void
3264 page_destroy_free(page_t *pp)
3265 {
3266 	ASSERT(PAGE_EXCL(pp));
3267 	ASSERT(PP_ISFREE(pp));
3268 	ASSERT(pp->p_vnode);
3269 	ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0);
3270 	ASSERT(!hat_page_is_mapped(pp));
3271 	ASSERT(PP_ISAGED(pp) == 0);
3272 	ASSERT(pp->p_szc == 0);
3273 
3274 	VM_STAT_ADD(pagecnt.pc_destroy_free);
3275 	page_list_sub(pp, PG_CACHE_LIST);
3276 
3277 	page_hashout(pp, NULL);
3278 	ASSERT(pp->p_vnode == NULL);
3279 	ASSERT(pp->p_offset == (u_offset_t)-1);
3280 	ASSERT(pp->p_hash == NULL);
3281 
3282 	PP_SETAGED(pp);
3283 	page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
3284 	page_unlock(pp);
3285 
3286 	mutex_enter(&new_freemem_lock);
3287 	if (freemem_wait) {
3288 		cv_signal(&freemem_cv);
3289 	}
3290 	mutex_exit(&new_freemem_lock);
3291 }
3292 
3293 /*
3294  * Rename the page "opp" to have an identity specified
3295  * by [vp, off].  If a page already exists with this name
3296  * it is locked and destroyed.  Note that the page's
3297  * translations are not unloaded during the rename.
3298  *
3299  * This routine is used by the anon layer to "steal" the
3300  * original page and is not unlike destroying a page and
3301  * creating a new page using the same page frame.
3302  *
3303  * XXX -- Could deadlock if caller 1 tries to rename A to B while
3304  * caller 2 tries to rename B to A.
3305  */
3306 void
3307 page_rename(page_t *opp, vnode_t *vp, u_offset_t off)
3308 {
3309 	page_t		*pp;
3310 	int		olckcnt = 0;
3311 	int		ocowcnt = 0;
3312 	kmutex_t	*phm;
3313 	ulong_t		index;
3314 
3315 	ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp));
3316 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3317 	ASSERT(PP_ISFREE(opp) == 0);
3318 
3319 	VM_STAT_ADD(page_rename_count);
3320 
3321 	TRACE_3(TR_FAC_VM, TR_PAGE_RENAME,
3322 	    "page rename:pp %p vp %p off %llx", opp, vp, off);
3323 
3324 	/*
3325 	 * CacheFS may call page_rename for a large NFS page
3326 	 * when both CacheFS and NFS mount points are used
3327 	 * by applications. Demote this large page before
3328 	 * renaming it, to ensure that there are no "partial"
3329 	 * large pages left lying around.
3330 	 */
3331 	if (opp->p_szc != 0) {
3332 		vnode_t *ovp = opp->p_vnode;
3333 		ASSERT(ovp != NULL);
3334 		ASSERT(!IS_SWAPFSVP(ovp));
3335 		ASSERT(!VN_ISKAS(ovp));
3336 		page_demote_vp_pages(opp);
3337 		ASSERT(opp->p_szc == 0);
3338 	}
3339 
3340 	page_hashout(opp, NULL);
3341 	PP_CLRAGED(opp);
3342 
3343 	/*
3344 	 * Acquire the appropriate page hash lock, since
3345 	 * we're going to rename the page.
3346 	 */
3347 	index = PAGE_HASH_FUNC(vp, off);
3348 	phm = PAGE_HASH_MUTEX(index);
3349 	mutex_enter(phm);
3350 top:
3351 	/*
3352 	 * Look for an existing page with this name and destroy it if found.
3353 	 * By holding the page hash lock all the way to the page_hashin()
3354 	 * call, we are assured that no page can be created with this
3355 	 * identity.  In the case when the phm lock is dropped to undo any
3356 	 * hat layer mappings, the existing page is held with an "exclusive"
3357 	 * lock, again preventing another page from being created with
3358 	 * this identity.
3359 	 */
3360 	PAGE_HASH_SEARCH(index, pp, vp, off);
3361 	if (pp != NULL) {
3362 		VM_STAT_ADD(page_rename_exists);
3363 
3364 		/*
3365 		 * As it turns out, this is one of only two places where
3366 		 * page_lock() needs to hold the passed in lock in the
3367 		 * successful case.  In all of the others, the lock could
3368 		 * be dropped as soon as the attempt is made to lock
3369 		 * the page.  It is tempting to add yet another arguement,
3370 		 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3371 		 */
3372 		if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3373 			/*
3374 			 * Went to sleep because the page could not
3375 			 * be locked.  We were woken up when the page
3376 			 * was unlocked, or when the page was destroyed.
3377 			 * In either case, `phm' was dropped while we
3378 			 * slept.  Hence we should not just roar through
3379 			 * this loop.
3380 			 */
3381 			goto top;
3382 		}
3383 
3384 		/*
3385 		 * If an existing page is a large page, then demote
3386 		 * it to ensure that no "partial" large pages are
3387 		 * "created" after page_rename. An existing page
3388 		 * can be a CacheFS page, and can't belong to swapfs.
3389 		 */
3390 		if (hat_page_is_mapped(pp)) {
3391 			/*
3392 			 * Unload translations.  Since we hold the
3393 			 * exclusive lock on this page, the page
3394 			 * can not be changed while we drop phm.
3395 			 * This is also not a lock protocol violation,
3396 			 * but rather the proper way to do things.
3397 			 */
3398 			mutex_exit(phm);
3399 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3400 			if (pp->p_szc != 0) {
3401 				ASSERT(!IS_SWAPFSVP(vp));
3402 				ASSERT(!VN_ISKAS(vp));
3403 				page_demote_vp_pages(pp);
3404 				ASSERT(pp->p_szc == 0);
3405 			}
3406 			mutex_enter(phm);
3407 		} else if (pp->p_szc != 0) {
3408 			ASSERT(!IS_SWAPFSVP(vp));
3409 			ASSERT(!VN_ISKAS(vp));
3410 			mutex_exit(phm);
3411 			page_demote_vp_pages(pp);
3412 			ASSERT(pp->p_szc == 0);
3413 			mutex_enter(phm);
3414 		}
3415 		page_hashout(pp, phm);
3416 	}
3417 	/*
3418 	 * Hash in the page with the new identity.
3419 	 */
3420 	if (!page_hashin(opp, vp, off, phm)) {
3421 		/*
3422 		 * We were holding phm while we searched for [vp, off]
3423 		 * and only dropped phm if we found and locked a page.
3424 		 * If we can't create this page now, then some thing
3425 		 * is really broken.
3426 		 */
3427 		panic("page_rename: Can't hash in page: %p", (void *)pp);
3428 		/*NOTREACHED*/
3429 	}
3430 
3431 	ASSERT(MUTEX_HELD(phm));
3432 	mutex_exit(phm);
3433 
3434 	/*
3435 	 * Now that we have dropped phm, lets get around to finishing up
3436 	 * with pp.
3437 	 */
3438 	if (pp != NULL) {
3439 		ASSERT(!hat_page_is_mapped(pp));
3440 		/* for now large pages should not end up here */
3441 		ASSERT(pp->p_szc == 0);
3442 		/*
3443 		 * Save the locks for transfer to the new page and then
3444 		 * clear them so page_free doesn't think they're important.
3445 		 * The page_struct_lock need not be acquired for lckcnt and
3446 		 * cowcnt since the page has an "exclusive" lock.
3447 		 */
3448 		olckcnt = pp->p_lckcnt;
3449 		ocowcnt = pp->p_cowcnt;
3450 		pp->p_lckcnt = pp->p_cowcnt = 0;
3451 
3452 		/*
3453 		 * Put the page on the "free" list after we drop
3454 		 * the lock.  The less work under the lock the better.
3455 		 */
3456 		/*LINTED: constant in conditional context*/
3457 		VN_DISPOSE(pp, B_FREE, 0, kcred);
3458 	}
3459 
3460 	/*
3461 	 * Transfer the lock count from the old page (if any).
3462 	 * The page_struct_lock need not be acquired for lckcnt and
3463 	 * cowcnt since the page has an "exclusive" lock.
3464 	 */
3465 	opp->p_lckcnt += olckcnt;
3466 	opp->p_cowcnt += ocowcnt;
3467 }
3468 
3469 /*
3470  * low level routine to add page `pp' to the hash and vp chains for [vp, offset]
3471  *
3472  * Pages are normally inserted at the start of a vnode's v_pages list.
3473  * If the vnode is VMODSORT and the page is modified, it goes at the end.
3474  * This can happen when a modified page is relocated for DR.
3475  *
3476  * Returns 1 on success and 0 on failure.
3477  */
3478 static int
3479 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset)
3480 {
3481 	page_t		**listp;
3482 	page_t		*tp;
3483 	ulong_t		index;
3484 
3485 	ASSERT(PAGE_EXCL(pp));
3486 	ASSERT(vp != NULL);
3487 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3488 
3489 	/*
3490 	 * Be sure to set these up before the page is inserted on the hash
3491 	 * list.  As soon as the page is placed on the list some other
3492 	 * thread might get confused and wonder how this page could
3493 	 * possibly hash to this list.
3494 	 */
3495 	pp->p_vnode = vp;
3496 	pp->p_offset = offset;
3497 
3498 	/*
3499 	 * record if this page is on a swap vnode
3500 	 */
3501 	if ((vp->v_flag & VISSWAP) != 0)
3502 		PP_SETSWAP(pp);
3503 
3504 	index = PAGE_HASH_FUNC(vp, offset);
3505 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index)));
3506 	listp = &page_hash[index];
3507 
3508 	/*
3509 	 * If this page is already hashed in, fail this attempt to add it.
3510 	 */
3511 	for (tp = *listp; tp != NULL; tp = tp->p_hash) {
3512 		if (tp->p_vnode == vp && tp->p_offset == offset) {
3513 			pp->p_vnode = NULL;
3514 			pp->p_offset = (u_offset_t)(-1);
3515 			return (0);
3516 		}
3517 	}
3518 	pp->p_hash = *listp;
3519 	*listp = pp;
3520 
3521 	/*
3522 	 * Add the page to the vnode's list of pages
3523 	 */
3524 	if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp))
3525 		listp = &vp->v_pages->p_vpprev->p_vpnext;
3526 	else
3527 		listp = &vp->v_pages;
3528 
3529 	page_vpadd(listp, pp);
3530 
3531 	return (1);
3532 }
3533 
3534 /*
3535  * Add page `pp' to both the hash and vp chains for [vp, offset].
3536  *
3537  * Returns 1 on success and 0 on failure.
3538  * If hold is passed in, it is not dropped.
3539  */
3540 int
3541 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold)
3542 {
3543 	kmutex_t	*phm = NULL;
3544 	kmutex_t	*vphm;
3545 	int		rc;
3546 
3547 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3548 
3549 	TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN,
3550 	    "page_hashin:pp %p vp %p offset %llx",
3551 	    pp, vp, offset);
3552 
3553 	VM_STAT_ADD(hashin_count);
3554 
3555 	if (hold != NULL)
3556 		phm = hold;
3557 	else {
3558 		VM_STAT_ADD(hashin_not_held);
3559 		phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset));
3560 		mutex_enter(phm);
3561 	}
3562 
3563 	vphm = page_vnode_mutex(vp);
3564 	mutex_enter(vphm);
3565 	rc = page_do_hashin(pp, vp, offset);
3566 	mutex_exit(vphm);
3567 	if (hold == NULL)
3568 		mutex_exit(phm);
3569 	if (rc == 0)
3570 		VM_STAT_ADD(hashin_already);
3571 	return (rc);
3572 }
3573 
3574 /*
3575  * Remove page ``pp'' from the hash and vp chains and remove vp association.
3576  * All mutexes must be held
3577  */
3578 static void
3579 page_do_hashout(page_t *pp)
3580 {
3581 	page_t	**hpp;
3582 	page_t	*hp;
3583 	vnode_t	*vp = pp->p_vnode;
3584 
3585 	ASSERT(vp != NULL);
3586 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3587 
3588 	/*
3589 	 * First, take pp off of its hash chain.
3590 	 */
3591 	hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)];
3592 
3593 	for (;;) {
3594 		hp = *hpp;
3595 		if (hp == pp)
3596 			break;
3597 		if (hp == NULL) {
3598 			panic("page_do_hashout");
3599 			/*NOTREACHED*/
3600 		}
3601 		hpp = &hp->p_hash;
3602 	}
3603 	*hpp = pp->p_hash;
3604 
3605 	/*
3606 	 * Now remove it from its associated vnode.
3607 	 */
3608 	if (vp->v_pages)
3609 		page_vpsub(&vp->v_pages, pp);
3610 
3611 	pp->p_hash = NULL;
3612 	page_clr_all_props(pp);
3613 	PP_CLRSWAP(pp);
3614 	pp->p_vnode = NULL;
3615 	pp->p_offset = (u_offset_t)-1;
3616 }
3617 
3618 /*
3619  * Remove page ``pp'' from the hash and vp chains and remove vp association.
3620  *
3621  * When `phm' is non-NULL it contains the address of the mutex protecting the
3622  * hash list pp is on.  It is not dropped.
3623  */
3624 void
3625 page_hashout(page_t *pp, kmutex_t *phm)
3626 {
3627 	vnode_t		*vp;
3628 	ulong_t		index;
3629 	kmutex_t	*nphm;
3630 	kmutex_t	*vphm;
3631 	kmutex_t	*sep;
3632 
3633 	ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1);
3634 	ASSERT(pp->p_vnode != NULL);
3635 	ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr);
3636 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode)));
3637 
3638 	vp = pp->p_vnode;
3639 
3640 	TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT,
3641 	    "page_hashout:pp %p vp %p", pp, vp);
3642 
3643 	/* Kernel probe */
3644 	TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */,
3645 	    tnf_opaque, vnode, vp,
3646 	    tnf_offset, offset, pp->p_offset);
3647 
3648 	/*
3649 	 *
3650 	 */
3651 	VM_STAT_ADD(hashout_count);
3652 	index = PAGE_HASH_FUNC(vp, pp->p_offset);
3653 	if (phm == NULL) {
3654 		VM_STAT_ADD(hashout_not_held);
3655 		nphm = PAGE_HASH_MUTEX(index);
3656 		mutex_enter(nphm);
3657 	}
3658 	ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1);
3659 
3660 
3661 	/*
3662 	 * grab page vnode mutex and remove it...
3663 	 */
3664 	vphm = page_vnode_mutex(vp);
3665 	mutex_enter(vphm);
3666 
3667 	page_do_hashout(pp);
3668 
3669 	mutex_exit(vphm);
3670 	if (phm == NULL)
3671 		mutex_exit(nphm);
3672 
3673 	/*
3674 	 * Wake up processes waiting for this page.  The page's
3675 	 * identity has been changed, and is probably not the
3676 	 * desired page any longer.
3677 	 */
3678 	sep = page_se_mutex(pp);
3679 	mutex_enter(sep);
3680 	pp->p_selock &= ~SE_EWANTED;
3681 	if (CV_HAS_WAITERS(&pp->p_cv))
3682 		cv_broadcast(&pp->p_cv);
3683 	mutex_exit(sep);
3684 }
3685 
3686 /*
3687  * Add the page to the front of a linked list of pages
3688  * using the p_next & p_prev pointers for the list.
3689  * The caller is responsible for protecting the list pointers.
3690  */
3691 void
3692 page_add(page_t **ppp, page_t *pp)
3693 {
3694 	ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3695 
3696 	page_add_common(ppp, pp);
3697 }
3698 
3699 
3700 
3701 /*
3702  *  Common code for page_add() and mach_page_add()
3703  */
3704 void
3705 page_add_common(page_t **ppp, page_t *pp)
3706 {
3707 	if (*ppp == NULL) {
3708 		pp->p_next = pp->p_prev = pp;
3709 	} else {
3710 		pp->p_next = *ppp;
3711 		pp->p_prev = (*ppp)->p_prev;
3712 		(*ppp)->p_prev = pp;
3713 		pp->p_prev->p_next = pp;
3714 	}
3715 	*ppp = pp;
3716 }
3717 
3718 
3719 /*
3720  * Remove this page from a linked list of pages
3721  * using the p_next & p_prev pointers for the list.
3722  *
3723  * The caller is responsible for protecting the list pointers.
3724  */
3725 void
3726 page_sub(page_t **ppp, page_t *pp)
3727 {
3728 	ASSERT((PP_ISFREE(pp)) ? 1 :
3729 	    (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3730 
3731 	if (*ppp == NULL || pp == NULL) {
3732 		panic("page_sub: bad arg(s): pp %p, *ppp %p",
3733 		    (void *)pp, (void *)(*ppp));
3734 		/*NOTREACHED*/
3735 	}
3736 
3737 	page_sub_common(ppp, pp);
3738 }
3739 
3740 
3741 /*
3742  *  Common code for page_sub() and mach_page_sub()
3743  */
3744 void
3745 page_sub_common(page_t **ppp, page_t *pp)
3746 {
3747 	if (*ppp == pp)
3748 		*ppp = pp->p_next;		/* go to next page */
3749 
3750 	if (*ppp == pp)
3751 		*ppp = NULL;			/* page list is gone */
3752 	else {
3753 		pp->p_prev->p_next = pp->p_next;
3754 		pp->p_next->p_prev = pp->p_prev;
3755 	}
3756 	pp->p_prev = pp->p_next = pp;		/* make pp a list of one */
3757 }
3758 
3759 
3760 /*
3761  * Break page list cppp into two lists with npages in the first list.
3762  * The tail is returned in nppp.
3763  */
3764 void
3765 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages)
3766 {
3767 	page_t *s1pp = *oppp;
3768 	page_t *s2pp;
3769 	page_t *e1pp, *e2pp;
3770 	long n = 0;
3771 
3772 	if (s1pp == NULL) {
3773 		*nppp = NULL;
3774 		return;
3775 	}
3776 	if (npages == 0) {
3777 		*nppp = s1pp;
3778 		*oppp = NULL;
3779 		return;
3780 	}
3781 	for (n = 0, s2pp = *oppp; n < npages; n++) {
3782 		s2pp = s2pp->p_next;
3783 	}
3784 	/* Fix head and tail of new lists */
3785 	e1pp = s2pp->p_prev;
3786 	e2pp = s1pp->p_prev;
3787 	s1pp->p_prev = e1pp;
3788 	e1pp->p_next = s1pp;
3789 	s2pp->p_prev = e2pp;
3790 	e2pp->p_next = s2pp;
3791 
3792 	/* second list empty */
3793 	if (s2pp == s1pp) {
3794 		*oppp = s1pp;
3795 		*nppp = NULL;
3796 	} else {
3797 		*oppp = s1pp;
3798 		*nppp = s2pp;
3799 	}
3800 }
3801 
3802 /*
3803  * Concatenate page list nppp onto the end of list ppp.
3804  */
3805 void
3806 page_list_concat(page_t **ppp, page_t **nppp)
3807 {
3808 	page_t *s1pp, *s2pp, *e1pp, *e2pp;
3809 
3810 	if (*nppp == NULL) {
3811 		return;
3812 	}
3813 	if (*ppp == NULL) {
3814 		*ppp = *nppp;
3815 		return;
3816 	}
3817 	s1pp = *ppp;
3818 	e1pp =  s1pp->p_prev;
3819 	s2pp = *nppp;
3820 	e2pp = s2pp->p_prev;
3821 	s1pp->p_prev = e2pp;
3822 	e2pp->p_next = s1pp;
3823 	e1pp->p_next = s2pp;
3824 	s2pp->p_prev = e1pp;
3825 }
3826 
3827 /*
3828  * return the next page in the page list
3829  */
3830 page_t *
3831 page_list_next(page_t *pp)
3832 {
3833 	return (pp->p_next);
3834 }
3835 
3836 
3837 /*
3838  * Add the page to the front of the linked list of pages
3839  * using p_vpnext/p_vpprev pointers for the list.
3840  *
3841  * The caller is responsible for protecting the lists.
3842  */
3843 void
3844 page_vpadd(page_t **ppp, page_t *pp)
3845 {
3846 	if (*ppp == NULL) {
3847 		pp->p_vpnext = pp->p_vpprev = pp;
3848 	} else {
3849 		pp->p_vpnext = *ppp;
3850 		pp->p_vpprev = (*ppp)->p_vpprev;
3851 		(*ppp)->p_vpprev = pp;
3852 		pp->p_vpprev->p_vpnext = pp;
3853 	}
3854 	*ppp = pp;
3855 }
3856 
3857 /*
3858  * Remove this page from the linked list of pages
3859  * using p_vpnext/p_vpprev pointers for the list.
3860  *
3861  * The caller is responsible for protecting the lists.
3862  */
3863 void
3864 page_vpsub(page_t **ppp, page_t *pp)
3865 {
3866 	if (*ppp == NULL || pp == NULL) {
3867 		panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
3868 		    (void *)pp, (void *)(*ppp));
3869 		/*NOTREACHED*/
3870 	}
3871 
3872 	if (*ppp == pp)
3873 		*ppp = pp->p_vpnext;		/* go to next page */
3874 
3875 	if (*ppp == pp)
3876 		*ppp = NULL;			/* page list is gone */
3877 	else {
3878 		pp->p_vpprev->p_vpnext = pp->p_vpnext;
3879 		pp->p_vpnext->p_vpprev = pp->p_vpprev;
3880 	}
3881 	pp->p_vpprev = pp->p_vpnext = pp;	/* make pp a list of one */
3882 }
3883 
3884 /*
3885  * Lock a physical page into memory "long term".  Used to support "lock
3886  * in memory" functions.  Accepts the page to be locked, and a cow variable
3887  * to indicate whether a the lock will travel to the new page during
3888  * a potential copy-on-write.
3889  */
3890 int
3891 page_pp_lock(
3892 	page_t *pp,			/* page to be locked */
3893 	int cow,			/* cow lock */
3894 	int kernel)			/* must succeed -- ignore checking */
3895 {
3896 	int r = 0;			/* result -- assume failure */
3897 
3898 	ASSERT(PAGE_LOCKED(pp));
3899 
3900 	page_struct_lock(pp);
3901 	/*
3902 	 * Acquire the "freemem_lock" for availrmem.
3903 	 */
3904 	if (cow) {
3905 		mutex_enter(&freemem_lock);
3906 		if ((availrmem > pages_pp_maximum) &&
3907 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
3908 			availrmem--;
3909 			pages_locked++;
3910 			mutex_exit(&freemem_lock);
3911 			r = 1;
3912 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
3913 				cmn_err(CE_WARN,
3914 				    "COW lock limit reached on pfn 0x%lx",
3915 				    page_pptonum(pp));
3916 			}
3917 		} else
3918 			mutex_exit(&freemem_lock);
3919 	} else {
3920 		if (pp->p_lckcnt) {
3921 			if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
3922 				r = 1;
3923 				if (++pp->p_lckcnt ==
3924 				    (ushort_t)PAGE_LOCK_MAXIMUM) {
3925 					cmn_err(CE_WARN, "Page lock limit "
3926 					    "reached on pfn 0x%lx",
3927 					    page_pptonum(pp));
3928 				}
3929 			}
3930 		} else {
3931 			if (kernel) {
3932 				/* availrmem accounting done by caller */
3933 				++pp->p_lckcnt;
3934 				r = 1;
3935 			} else {
3936 				mutex_enter(&freemem_lock);
3937 				if (availrmem > pages_pp_maximum) {
3938 					availrmem--;
3939 					pages_locked++;
3940 					++pp->p_lckcnt;
3941 					r = 1;
3942 				}
3943 				mutex_exit(&freemem_lock);
3944 			}
3945 		}
3946 	}
3947 	page_struct_unlock(pp);
3948 	return (r);
3949 }
3950 
3951 /*
3952  * Decommit a lock on a physical page frame.  Account for cow locks if
3953  * appropriate.
3954  */
3955 void
3956 page_pp_unlock(
3957 	page_t *pp,			/* page to be unlocked */
3958 	int cow,			/* expect cow lock */
3959 	int kernel)			/* this was a kernel lock */
3960 {
3961 	ASSERT(PAGE_LOCKED(pp));
3962 
3963 	page_struct_lock(pp);
3964 	/*
3965 	 * Acquire the "freemem_lock" for availrmem.
3966 	 * If cowcnt or lcknt is already 0 do nothing; i.e., we
3967 	 * could be called to unlock even if nothing is locked. This could
3968 	 * happen if locked file pages were truncated (removing the lock)
3969 	 * and the file was grown again and new pages faulted in; the new
3970 	 * pages are unlocked but the segment still thinks they're locked.
3971 	 */
3972 	if (cow) {
3973 		if (pp->p_cowcnt) {
3974 			mutex_enter(&freemem_lock);
3975 			pp->p_cowcnt--;
3976 			availrmem++;
3977 			pages_locked--;
3978 			mutex_exit(&freemem_lock);
3979 		}
3980 	} else {
3981 		if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
3982 			if (!kernel) {
3983 				mutex_enter(&freemem_lock);
3984 				availrmem++;
3985 				pages_locked--;
3986 				mutex_exit(&freemem_lock);
3987 			}
3988 		}
3989 	}
3990 	page_struct_unlock(pp);
3991 }
3992 
3993 /*
3994  * This routine reserves availrmem for npages;
3995  * 	flags: KM_NOSLEEP or KM_SLEEP
3996  * 	returns 1 on success or 0 on failure
3997  */
3998 int
3999 page_resv(pgcnt_t npages, uint_t flags)
4000 {
4001 	mutex_enter(&freemem_lock);
4002 	while (availrmem < tune.t_minarmem + npages) {
4003 		if (flags & KM_NOSLEEP) {
4004 			mutex_exit(&freemem_lock);
4005 			return (0);
4006 		}
4007 		mutex_exit(&freemem_lock);
4008 		page_needfree(npages);
4009 		kmem_reap();
4010 		delay(hz >> 2);
4011 		page_needfree(-(spgcnt_t)npages);
4012 		mutex_enter(&freemem_lock);
4013 	}
4014 	availrmem -= npages;
4015 	mutex_exit(&freemem_lock);
4016 	return (1);
4017 }
4018 
4019 /*
4020  * This routine unreserves availrmem for npages;
4021  */
4022 void
4023 page_unresv(pgcnt_t npages)
4024 {
4025 	mutex_enter(&freemem_lock);
4026 	availrmem += npages;
4027 	mutex_exit(&freemem_lock);
4028 }
4029 
4030 /*
4031  * See Statement at the beginning of segvn_lockop() regarding
4032  * the way we handle cowcnts and lckcnts.
4033  *
4034  * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
4035  * that breaks COW has PROT_WRITE.
4036  *
4037  * Note that, we may also break COW in case we are softlocking
4038  * on read access during physio;
4039  * in this softlock case, the vpage may not have PROT_WRITE.
4040  * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
4041  * if the vpage doesn't have PROT_WRITE.
4042  *
4043  * This routine is never called if we are stealing a page
4044  * in anon_private.
4045  *
4046  * The caller subtracted from availrmem for read only mapping.
4047  * if lckcnt is 1 increment availrmem.
4048  */
4049 void
4050 page_pp_useclaim(
4051 	page_t *opp,		/* original page frame losing lock */
4052 	page_t *npp,		/* new page frame gaining lock */
4053 	uint_t	write_perm) 	/* set if vpage has PROT_WRITE */
4054 {
4055 	int payback = 0;
4056 
4057 	ASSERT(PAGE_LOCKED(opp));
4058 	ASSERT(PAGE_LOCKED(npp));
4059 
4060 	page_struct_lock(opp);
4061 
4062 	ASSERT(npp->p_cowcnt == 0);
4063 	ASSERT(npp->p_lckcnt == 0);
4064 
4065 	/* Don't use claim if nothing is locked (see page_pp_unlock above) */
4066 	if ((write_perm && opp->p_cowcnt != 0) ||
4067 	    (!write_perm && opp->p_lckcnt != 0)) {
4068 
4069 		if (write_perm) {
4070 			npp->p_cowcnt++;
4071 			ASSERT(opp->p_cowcnt != 0);
4072 			opp->p_cowcnt--;
4073 		} else {
4074 
4075 			ASSERT(opp->p_lckcnt != 0);
4076 
4077 			/*
4078 			 * We didn't need availrmem decremented if p_lckcnt on
4079 			 * original page is 1. Here, we are unlocking
4080 			 * read-only copy belonging to original page and
4081 			 * are locking a copy belonging to new page.
4082 			 */
4083 			if (opp->p_lckcnt == 1)
4084 				payback = 1;
4085 
4086 			npp->p_lckcnt++;
4087 			opp->p_lckcnt--;
4088 		}
4089 	}
4090 	if (payback) {
4091 		mutex_enter(&freemem_lock);
4092 		availrmem++;
4093 		pages_useclaim--;
4094 		mutex_exit(&freemem_lock);
4095 	}
4096 	page_struct_unlock(opp);
4097 }
4098 
4099 /*
4100  * Simple claim adjust functions -- used to support changes in
4101  * claims due to changes in access permissions.  Used by segvn_setprot().
4102  */
4103 int
4104 page_addclaim(page_t *pp)
4105 {
4106 	int r = 0;			/* result */
4107 
4108 	ASSERT(PAGE_LOCKED(pp));
4109 
4110 	page_struct_lock(pp);
4111 	ASSERT(pp->p_lckcnt != 0);
4112 
4113 	if (pp->p_lckcnt == 1) {
4114 		if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4115 			--pp->p_lckcnt;
4116 			r = 1;
4117 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4118 				cmn_err(CE_WARN,
4119 				    "COW lock limit reached on pfn 0x%lx",
4120 				    page_pptonum(pp));
4121 			}
4122 		}
4123 	} else {
4124 		mutex_enter(&freemem_lock);
4125 		if ((availrmem > pages_pp_maximum) &&
4126 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
4127 			--availrmem;
4128 			++pages_claimed;
4129 			mutex_exit(&freemem_lock);
4130 			--pp->p_lckcnt;
4131 			r = 1;
4132 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4133 				cmn_err(CE_WARN,
4134 				    "COW lock limit reached on pfn 0x%lx",
4135 				    page_pptonum(pp));
4136 			}
4137 		} else
4138 			mutex_exit(&freemem_lock);
4139 	}
4140 	page_struct_unlock(pp);
4141 	return (r);
4142 }
4143 
4144 int
4145 page_subclaim(page_t *pp)
4146 {
4147 	int r = 0;
4148 
4149 	ASSERT(PAGE_LOCKED(pp));
4150 
4151 	page_struct_lock(pp);
4152 	ASSERT(pp->p_cowcnt != 0);
4153 
4154 	if (pp->p_lckcnt) {
4155 		if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4156 			r = 1;
4157 			/*
4158 			 * for availrmem
4159 			 */
4160 			mutex_enter(&freemem_lock);
4161 			availrmem++;
4162 			pages_claimed--;
4163 			mutex_exit(&freemem_lock);
4164 
4165 			pp->p_cowcnt--;
4166 
4167 			if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4168 				cmn_err(CE_WARN,
4169 				    "Page lock limit reached on pfn 0x%lx",
4170 				    page_pptonum(pp));
4171 			}
4172 		}
4173 	} else {
4174 		r = 1;
4175 		pp->p_cowcnt--;
4176 		pp->p_lckcnt++;
4177 	}
4178 	page_struct_unlock(pp);
4179 	return (r);
4180 }
4181 
4182 int
4183 page_addclaim_pages(page_t  **ppa)
4184 {
4185 
4186 	pgcnt_t	lckpgs = 0, pg_idx;
4187 
4188 	VM_STAT_ADD(pagecnt.pc_addclaim_pages);
4189 
4190 	mutex_enter(&page_llock);
4191 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4192 
4193 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4194 		ASSERT(ppa[pg_idx]->p_lckcnt != 0);
4195 		if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4196 			mutex_exit(&page_llock);
4197 			return (0);
4198 		}
4199 		if (ppa[pg_idx]->p_lckcnt > 1)
4200 			lckpgs++;
4201 	}
4202 
4203 	if (lckpgs != 0) {
4204 		mutex_enter(&freemem_lock);
4205 		if (availrmem >= pages_pp_maximum + lckpgs) {
4206 			availrmem -= lckpgs;
4207 			pages_claimed += lckpgs;
4208 		} else {
4209 			mutex_exit(&freemem_lock);
4210 			mutex_exit(&page_llock);
4211 			return (0);
4212 		}
4213 		mutex_exit(&freemem_lock);
4214 	}
4215 
4216 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4217 		ppa[pg_idx]->p_lckcnt--;
4218 		ppa[pg_idx]->p_cowcnt++;
4219 	}
4220 	mutex_exit(&page_llock);
4221 	return (1);
4222 }
4223 
4224 int
4225 page_subclaim_pages(page_t  **ppa)
4226 {
4227 	pgcnt_t	ulckpgs = 0, pg_idx;
4228 
4229 	VM_STAT_ADD(pagecnt.pc_subclaim_pages);
4230 
4231 	mutex_enter(&page_llock);
4232 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4233 
4234 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4235 		ASSERT(ppa[pg_idx]->p_cowcnt != 0);
4236 		if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4237 			mutex_exit(&page_llock);
4238 			return (0);
4239 		}
4240 		if (ppa[pg_idx]->p_lckcnt != 0)
4241 			ulckpgs++;
4242 	}
4243 
4244 	if (ulckpgs != 0) {
4245 		mutex_enter(&freemem_lock);
4246 		availrmem += ulckpgs;
4247 		pages_claimed -= ulckpgs;
4248 		mutex_exit(&freemem_lock);
4249 	}
4250 
4251 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4252 		ppa[pg_idx]->p_cowcnt--;
4253 		ppa[pg_idx]->p_lckcnt++;
4254 
4255 	}
4256 	mutex_exit(&page_llock);
4257 	return (1);
4258 }
4259 
4260 page_t *
4261 page_numtopp(pfn_t pfnum, se_t se)
4262 {
4263 	page_t *pp;
4264 
4265 retry:
4266 	pp = page_numtopp_nolock(pfnum);
4267 	if (pp == NULL) {
4268 		return ((page_t *)NULL);
4269 	}
4270 
4271 	/*
4272 	 * Acquire the appropriate lock on the page.
4273 	 */
4274 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) {
4275 		if (page_pptonum(pp) != pfnum)
4276 			goto retry;
4277 		continue;
4278 	}
4279 
4280 	if (page_pptonum(pp) != pfnum) {
4281 		page_unlock(pp);
4282 		goto retry;
4283 	}
4284 
4285 	return (pp);
4286 }
4287 
4288 page_t *
4289 page_numtopp_noreclaim(pfn_t pfnum, se_t se)
4290 {
4291 	page_t *pp;
4292 
4293 retry:
4294 	pp = page_numtopp_nolock(pfnum);
4295 	if (pp == NULL) {
4296 		return ((page_t *)NULL);
4297 	}
4298 
4299 	/*
4300 	 * Acquire the appropriate lock on the page.
4301 	 */
4302 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) {
4303 		if (page_pptonum(pp) != pfnum)
4304 			goto retry;
4305 		continue;
4306 	}
4307 
4308 	if (page_pptonum(pp) != pfnum) {
4309 		page_unlock(pp);
4310 		goto retry;
4311 	}
4312 
4313 	return (pp);
4314 }
4315 
4316 /*
4317  * This routine is like page_numtopp, but will only return page structs
4318  * for pages which are ok for loading into hardware using the page struct.
4319  */
4320 page_t *
4321 page_numtopp_nowait(pfn_t pfnum, se_t se)
4322 {
4323 	page_t *pp;
4324 
4325 retry:
4326 	pp = page_numtopp_nolock(pfnum);
4327 	if (pp == NULL) {
4328 		return ((page_t *)NULL);
4329 	}
4330 
4331 	/*
4332 	 * Try to acquire the appropriate lock on the page.
4333 	 */
4334 	if (PP_ISFREE(pp))
4335 		pp = NULL;
4336 	else {
4337 		if (!page_trylock(pp, se))
4338 			pp = NULL;
4339 		else {
4340 			if (page_pptonum(pp) != pfnum) {
4341 				page_unlock(pp);
4342 				goto retry;
4343 			}
4344 			if (PP_ISFREE(pp)) {
4345 				page_unlock(pp);
4346 				pp = NULL;
4347 			}
4348 		}
4349 	}
4350 	return (pp);
4351 }
4352 
4353 /*
4354  * Returns a count of dirty pages that are in the process
4355  * of being written out.  If 'cleanit' is set, try to push the page.
4356  */
4357 pgcnt_t
4358 page_busy(int cleanit)
4359 {
4360 	page_t *page0 = page_first();
4361 	page_t *pp = page0;
4362 	pgcnt_t nppbusy = 0;
4363 	u_offset_t off;
4364 
4365 	do {
4366 		vnode_t *vp = pp->p_vnode;
4367 
4368 		/*
4369 		 * A page is a candidate for syncing if it is:
4370 		 *
4371 		 * (a)	On neither the freelist nor the cachelist
4372 		 * (b)	Hashed onto a vnode
4373 		 * (c)	Not a kernel page
4374 		 * (d)	Dirty
4375 		 * (e)	Not part of a swapfile
4376 		 * (f)	a page which belongs to a real vnode; eg has a non-null
4377 		 *	v_vfsp pointer.
4378 		 * (g)	Backed by a filesystem which doesn't have a
4379 		 *	stubbed-out sync operation
4380 		 */
4381 		if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) &&
4382 		    hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL &&
4383 		    vfs_can_sync(vp->v_vfsp)) {
4384 			nppbusy++;
4385 			vfs_syncprogress();
4386 
4387 			if (!cleanit)
4388 				continue;
4389 			if (!page_trylock(pp, SE_EXCL))
4390 				continue;
4391 
4392 			if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) ||
4393 			    pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
4394 			    !(hat_pagesync(pp,
4395 			    HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) {
4396 				page_unlock(pp);
4397 				continue;
4398 			}
4399 			off = pp->p_offset;
4400 			VN_HOLD(vp);
4401 			page_unlock(pp);
4402 			(void) VOP_PUTPAGE(vp, off, PAGESIZE,
4403 			    B_ASYNC | B_FREE, kcred);
4404 			VN_RELE(vp);
4405 		}
4406 	} while ((pp = page_next(pp)) != page0);
4407 
4408 	return (nppbusy);
4409 }
4410 
4411 void page_invalidate_pages(void);
4412 
4413 /*
4414  * callback handler to vm sub-system
4415  *
4416  * callers make sure no recursive entries to this func.
4417  */
4418 /*ARGSUSED*/
4419 boolean_t
4420 callb_vm_cpr(void *arg, int code)
4421 {
4422 	if (code == CB_CODE_CPR_CHKPT)
4423 		page_invalidate_pages();
4424 	return (B_TRUE);
4425 }
4426 
4427 /*
4428  * Invalidate all pages of the system.
4429  * It shouldn't be called until all user page activities are all stopped.
4430  */
4431 void
4432 page_invalidate_pages()
4433 {
4434 	page_t *pp;
4435 	page_t *page0;
4436 	pgcnt_t nbusypages;
4437 	int retry = 0;
4438 	const int MAXRETRIES = 4;
4439 #if defined(__sparc)
4440 	extern struct vnode prom_ppages;
4441 #endif /* __sparc */
4442 
4443 top:
4444 	/*
4445 	 * Flush dirty pages and destroy the clean ones.
4446 	 */
4447 	nbusypages = 0;
4448 
4449 	pp = page0 = page_first();
4450 	do {
4451 		struct vnode	*vp;
4452 		u_offset_t	offset;
4453 		int		mod;
4454 
4455 		/*
4456 		 * skip the page if it has no vnode or the page associated
4457 		 * with the kernel vnode or prom allocated kernel mem.
4458 		 */
4459 #if defined(__sparc)
4460 		if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp) ||
4461 		    vp == &prom_ppages)
4462 #else /* x86 doesn't have prom or prom_ppage */
4463 		if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp))
4464 #endif /* __sparc */
4465 			continue;
4466 
4467 		/*
4468 		 * skip the page which is already free invalidated.
4469 		 */
4470 		if (PP_ISFREE(pp) && PP_ISAGED(pp))
4471 			continue;
4472 
4473 		/*
4474 		 * skip pages that are already locked or can't be "exclusively"
4475 		 * locked or are already free.  After we lock the page, check
4476 		 * the free and age bits again to be sure it's not destroied
4477 		 * yet.
4478 		 * To achieve max. parallelization, we use page_trylock instead
4479 		 * of page_lock so that we don't get block on individual pages
4480 		 * while we have thousands of other pages to process.
4481 		 */
4482 		if (!page_trylock(pp, SE_EXCL)) {
4483 			nbusypages++;
4484 			continue;
4485 		} else if (PP_ISFREE(pp)) {
4486 			if (!PP_ISAGED(pp)) {
4487 				page_destroy_free(pp);
4488 			} else {
4489 				page_unlock(pp);
4490 			}
4491 			continue;
4492 		}
4493 		/*
4494 		 * Is this page involved in some I/O? shared?
4495 		 *
4496 		 * The page_struct_lock need not be acquired to
4497 		 * examine these fields since the page has an
4498 		 * "exclusive" lock.
4499 		 */
4500 		if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
4501 			page_unlock(pp);
4502 			continue;
4503 		}
4504 
4505 		if (vp->v_type == VCHR) {
4506 			panic("vp->v_type == VCHR");
4507 			/*NOTREACHED*/
4508 		}
4509 
4510 		if (!page_try_demote_pages(pp)) {
4511 			page_unlock(pp);
4512 			continue;
4513 		}
4514 
4515 		/*
4516 		 * Check the modified bit. Leave the bits alone in hardware
4517 		 * (they will be modified if we do the putpage).
4518 		 */
4519 		mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD)
4520 		    & P_MOD);
4521 		if (mod) {
4522 			offset = pp->p_offset;
4523 			/*
4524 			 * Hold the vnode before releasing the page lock
4525 			 * to prevent it from being freed and re-used by
4526 			 * some other thread.
4527 			 */
4528 			VN_HOLD(vp);
4529 			page_unlock(pp);
4530 			/*
4531 			 * No error return is checked here. Callers such as
4532 			 * cpr deals with the dirty pages at the dump time
4533 			 * if this putpage fails.
4534 			 */
4535 			(void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL,
4536 			    kcred);
4537 			VN_RELE(vp);
4538 		} else {
4539 			page_destroy(pp, 0);
4540 		}
4541 	} while ((pp = page_next(pp)) != page0);
4542 	if (nbusypages && retry++ < MAXRETRIES) {
4543 		delay(1);
4544 		goto top;
4545 	}
4546 }
4547 
4548 /*
4549  * Replace the page "old" with the page "new" on the page hash and vnode lists
4550  *
4551  * the replacemnt must be done in place, ie the equivalent sequence:
4552  *
4553  *	vp = old->p_vnode;
4554  *	off = old->p_offset;
4555  *	page_do_hashout(old)
4556  *	page_do_hashin(new, vp, off)
4557  *
4558  * doesn't work, since
4559  *  1) if old is the only page on the vnode, the v_pages list has a window
4560  *     where it looks empty. This will break file system assumptions.
4561  * and
4562  *  2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
4563  */
4564 static void
4565 page_do_relocate_hash(page_t *new, page_t *old)
4566 {
4567 	page_t	**hash_list;
4568 	vnode_t	*vp = old->p_vnode;
4569 	kmutex_t *sep;
4570 
4571 	ASSERT(PAGE_EXCL(old));
4572 	ASSERT(PAGE_EXCL(new));
4573 	ASSERT(vp != NULL);
4574 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
4575 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset))));
4576 
4577 	/*
4578 	 * First find old page on the page hash list
4579 	 */
4580 	hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)];
4581 
4582 	for (;;) {
4583 		if (*hash_list == old)
4584 			break;
4585 		if (*hash_list == NULL) {
4586 			panic("page_do_hashout");
4587 			/*NOTREACHED*/
4588 		}
4589 		hash_list = &(*hash_list)->p_hash;
4590 	}
4591 
4592 	/*
4593 	 * update new and replace old with new on the page hash list
4594 	 */
4595 	new->p_vnode = old->p_vnode;
4596 	new->p_offset = old->p_offset;
4597 	new->p_hash = old->p_hash;
4598 	*hash_list = new;
4599 
4600 	if ((new->p_vnode->v_flag & VISSWAP) != 0)
4601 		PP_SETSWAP(new);
4602 
4603 	/*
4604 	 * replace old with new on the vnode's page list
4605 	 */
4606 	if (old->p_vpnext == old) {
4607 		new->p_vpnext = new;
4608 		new->p_vpprev = new;
4609 	} else {
4610 		new->p_vpnext = old->p_vpnext;
4611 		new->p_vpprev = old->p_vpprev;
4612 		new->p_vpnext->p_vpprev = new;
4613 		new->p_vpprev->p_vpnext = new;
4614 	}
4615 	if (vp->v_pages == old)
4616 		vp->v_pages = new;
4617 
4618 	/*
4619 	 * clear out the old page
4620 	 */
4621 	old->p_hash = NULL;
4622 	old->p_vpnext = NULL;
4623 	old->p_vpprev = NULL;
4624 	old->p_vnode = NULL;
4625 	PP_CLRSWAP(old);
4626 	old->p_offset = (u_offset_t)-1;
4627 	page_clr_all_props(old);
4628 
4629 	/*
4630 	 * Wake up processes waiting for this page.  The page's
4631 	 * identity has been changed, and is probably not the
4632 	 * desired page any longer.
4633 	 */
4634 	sep = page_se_mutex(old);
4635 	mutex_enter(sep);
4636 	old->p_selock &= ~SE_EWANTED;
4637 	if (CV_HAS_WAITERS(&old->p_cv))
4638 		cv_broadcast(&old->p_cv);
4639 	mutex_exit(sep);
4640 }
4641 
4642 /*
4643  * This function moves the identity of page "pp_old" to page "pp_new".
4644  * Both pages must be locked on entry.  "pp_new" is free, has no identity,
4645  * and need not be hashed out from anywhere.
4646  */
4647 void
4648 page_relocate_hash(page_t *pp_new, page_t *pp_old)
4649 {
4650 	vnode_t *vp = pp_old->p_vnode;
4651 	u_offset_t off = pp_old->p_offset;
4652 	kmutex_t *phm, *vphm;
4653 
4654 	/*
4655 	 * Rehash two pages
4656 	 */
4657 	ASSERT(PAGE_EXCL(pp_old));
4658 	ASSERT(PAGE_EXCL(pp_new));
4659 	ASSERT(vp != NULL);
4660 	ASSERT(pp_new->p_vnode == NULL);
4661 
4662 	/*
4663 	 * hashout then hashin while holding the mutexes
4664 	 */
4665 	phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off));
4666 	mutex_enter(phm);
4667 	vphm = page_vnode_mutex(vp);
4668 	mutex_enter(vphm);
4669 
4670 	page_do_relocate_hash(pp_new, pp_old);
4671 
4672 	mutex_exit(vphm);
4673 	mutex_exit(phm);
4674 
4675 	/*
4676 	 * The page_struct_lock need not be acquired for lckcnt and
4677 	 * cowcnt since the page has an "exclusive" lock.
4678 	 */
4679 	ASSERT(pp_new->p_lckcnt == 0);
4680 	ASSERT(pp_new->p_cowcnt == 0);
4681 	pp_new->p_lckcnt = pp_old->p_lckcnt;
4682 	pp_new->p_cowcnt = pp_old->p_cowcnt;
4683 	pp_old->p_lckcnt = pp_old->p_cowcnt = 0;
4684 
4685 	/* The following comment preserved from page_flip(). */
4686 	/* XXX - Do we need to protect fsdata? */
4687 	pp_new->p_fsdata = pp_old->p_fsdata;
4688 }
4689 
4690 /*
4691  * Helper routine used to lock all remaining members of a
4692  * large page. The caller is responsible for passing in a locked
4693  * pp. If pp is a large page, then it succeeds in locking all the
4694  * remaining constituent pages or it returns with only the
4695  * original page locked.
4696  *
4697  * Returns 1 on success, 0 on failure.
4698  *
4699  * If success is returned this routine gurantees p_szc for all constituent
4700  * pages of a large page pp belongs to can't change. To achieve this we
4701  * recheck szc of pp after locking all constituent pages and retry if szc
4702  * changed (it could only decrease). Since hat_page_demote() needs an EXCL
4703  * lock on one of constituent pages it can't be running after all constituent
4704  * pages are locked.  hat_page_demote() with a lock on a constituent page
4705  * outside of this large page (i.e. pp belonged to a larger large page) is
4706  * already done with all constituent pages of pp since the root's p_szc is
4707  * changed last. Thefore no need to synchronize with hat_page_demote() that
4708  * locked a constituent page outside of pp's current large page.
4709  */
4710 #ifdef DEBUG
4711 uint32_t gpg_trylock_mtbf = 0;
4712 #endif
4713 
4714 int
4715 group_page_trylock(page_t *pp, se_t se)
4716 {
4717 	page_t  *tpp;
4718 	pgcnt_t	npgs, i, j;
4719 	uint_t pszc = pp->p_szc;
4720 
4721 #ifdef DEBUG
4722 	if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) {
4723 		return (0);
4724 	}
4725 #endif
4726 
4727 	if (pp != PP_GROUPLEADER(pp, pszc)) {
4728 		return (0);
4729 	}
4730 
4731 retry:
4732 	ASSERT(PAGE_LOCKED_SE(pp, se));
4733 	ASSERT(!PP_ISFREE(pp));
4734 	if (pszc == 0) {
4735 		return (1);
4736 	}
4737 	npgs = page_get_pagecnt(pszc);
4738 	tpp = pp + 1;
4739 	for (i = 1; i < npgs; i++, tpp++) {
4740 		if (!page_trylock(tpp, se)) {
4741 			tpp = pp + 1;
4742 			for (j = 1; j < i; j++, tpp++) {
4743 				page_unlock(tpp);
4744 			}
4745 			return (0);
4746 		}
4747 	}
4748 	if (pp->p_szc != pszc) {
4749 		ASSERT(pp->p_szc < pszc);
4750 		ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) &&
4751 		    !IS_SWAPFSVP(pp->p_vnode));
4752 		tpp = pp + 1;
4753 		for (i = 1; i < npgs; i++, tpp++) {
4754 			page_unlock(tpp);
4755 		}
4756 		pszc = pp->p_szc;
4757 		goto retry;
4758 	}
4759 	return (1);
4760 }
4761 
4762 void
4763 group_page_unlock(page_t *pp)
4764 {
4765 	page_t *tpp;
4766 	pgcnt_t	npgs, i;
4767 
4768 	ASSERT(PAGE_LOCKED(pp));
4769 	ASSERT(!PP_ISFREE(pp));
4770 	ASSERT(pp == PP_PAGEROOT(pp));
4771 	npgs = page_get_pagecnt(pp->p_szc);
4772 	for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) {
4773 		page_unlock(tpp);
4774 	}
4775 }
4776 
4777 /*
4778  * returns
4779  * 0 		: on success and *nrelocp is number of relocated PAGESIZE pages
4780  * ERANGE	: this is not a base page
4781  * EBUSY	: failure to get locks on the page/pages
4782  * ENOMEM	: failure to obtain replacement pages
4783  * EAGAIN	: OBP has not yet completed its boot-time handoff to the kernel
4784  * EIO		: An error occurred while trying to copy the page data
4785  *
4786  * Return with all constituent members of target and replacement
4787  * SE_EXCL locked. It is the callers responsibility to drop the
4788  * locks.
4789  */
4790 int
4791 do_page_relocate(
4792 	page_t **target,
4793 	page_t **replacement,
4794 	int grouplock,
4795 	spgcnt_t *nrelocp,
4796 	lgrp_t *lgrp)
4797 {
4798 	page_t *first_repl;
4799 	page_t *repl;
4800 	page_t *targ;
4801 	page_t *pl = NULL;
4802 	uint_t ppattr;
4803 	pfn_t   pfn, repl_pfn;
4804 	uint_t	szc;
4805 	spgcnt_t npgs, i;
4806 	int repl_contig = 0;
4807 	uint_t flags = 0;
4808 	spgcnt_t dofree = 0;
4809 
4810 	*nrelocp = 0;
4811 
4812 #if defined(__sparc)
4813 	/*
4814 	 * We need to wait till OBP has completed
4815 	 * its boot-time handoff of its resources to the kernel
4816 	 * before we allow page relocation
4817 	 */
4818 	if (page_relocate_ready == 0) {
4819 		return (EAGAIN);
4820 	}
4821 #endif
4822 
4823 	/*
4824 	 * If this is not a base page,
4825 	 * just return with 0x0 pages relocated.
4826 	 */
4827 	targ = *target;
4828 	ASSERT(PAGE_EXCL(targ));
4829 	ASSERT(!PP_ISFREE(targ));
4830 	szc = targ->p_szc;
4831 	ASSERT(szc < mmu_page_sizes);
4832 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4833 	pfn = targ->p_pagenum;
4834 	if (pfn != PFN_BASE(pfn, szc)) {
4835 		VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]);
4836 		return (ERANGE);
4837 	}
4838 
4839 	if ((repl = *replacement) != NULL && repl->p_szc >= szc) {
4840 		repl_pfn = repl->p_pagenum;
4841 		if (repl_pfn != PFN_BASE(repl_pfn, szc)) {
4842 			VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]);
4843 			return (ERANGE);
4844 		}
4845 		repl_contig = 1;
4846 	}
4847 
4848 	/*
4849 	 * We must lock all members of this large page or we cannot
4850 	 * relocate any part of it.
4851 	 */
4852 	if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) {
4853 		VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]);
4854 		return (EBUSY);
4855 	}
4856 
4857 	/*
4858 	 * reread szc it could have been decreased before
4859 	 * group_page_trylock() was done.
4860 	 */
4861 	szc = targ->p_szc;
4862 	ASSERT(szc < mmu_page_sizes);
4863 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4864 	ASSERT(pfn == PFN_BASE(pfn, szc));
4865 
4866 	npgs = page_get_pagecnt(targ->p_szc);
4867 
4868 	if (repl == NULL) {
4869 		dofree = npgs;		/* Size of target page in MMU pages */
4870 		if (!page_create_wait(dofree, 0)) {
4871 			if (grouplock != 0) {
4872 				group_page_unlock(targ);
4873 			}
4874 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4875 			return (ENOMEM);
4876 		}
4877 
4878 		/*
4879 		 * seg kmem pages require that the target and replacement
4880 		 * page be the same pagesize.
4881 		 */
4882 		flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0;
4883 		repl = page_get_replacement_page(targ, lgrp, flags);
4884 		if (repl == NULL) {
4885 			if (grouplock != 0) {
4886 				group_page_unlock(targ);
4887 			}
4888 			page_create_putback(dofree);
4889 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4890 			return (ENOMEM);
4891 		}
4892 	}
4893 #ifdef DEBUG
4894 	else {
4895 		ASSERT(PAGE_LOCKED(repl));
4896 	}
4897 #endif /* DEBUG */
4898 
4899 #if defined(__sparc)
4900 	/*
4901 	 * Let hat_page_relocate() complete the relocation if it's kernel page
4902 	 */
4903 	if (VN_ISKAS(targ->p_vnode)) {
4904 		*replacement = repl;
4905 		if (hat_page_relocate(target, replacement, nrelocp) != 0) {
4906 			if (grouplock != 0) {
4907 				group_page_unlock(targ);
4908 			}
4909 			if (dofree) {
4910 				*replacement = NULL;
4911 				page_free_replacement_page(repl);
4912 				page_create_putback(dofree);
4913 			}
4914 			VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]);
4915 			return (EAGAIN);
4916 		}
4917 		VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4918 		return (0);
4919 	}
4920 #else
4921 #if defined(lint)
4922 	dofree = dofree;
4923 #endif
4924 #endif
4925 
4926 	first_repl = repl;
4927 
4928 	for (i = 0; i < npgs; i++) {
4929 		ASSERT(PAGE_EXCL(targ));
4930 		ASSERT(targ->p_slckcnt == 0);
4931 		ASSERT(repl->p_slckcnt == 0);
4932 
4933 		(void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD);
4934 
4935 		ASSERT(hat_page_getshare(targ) == 0);
4936 		ASSERT(!PP_ISFREE(targ));
4937 		ASSERT(targ->p_pagenum == (pfn + i));
4938 		ASSERT(repl_contig == 0 ||
4939 		    repl->p_pagenum == (repl_pfn + i));
4940 
4941 		/*
4942 		 * Copy the page contents and attributes then
4943 		 * relocate the page in the page hash.
4944 		 */
4945 		if (ppcopy(targ, repl) == 0) {
4946 			targ = *target;
4947 			repl = first_repl;
4948 			VM_STAT_ADD(vmm_vmstats.ppr_copyfail);
4949 			if (grouplock != 0) {
4950 				group_page_unlock(targ);
4951 			}
4952 			if (dofree) {
4953 				*replacement = NULL;
4954 				page_free_replacement_page(repl);
4955 				page_create_putback(dofree);
4956 			}
4957 			return (EIO);
4958 		}
4959 
4960 		targ++;
4961 		if (repl_contig != 0) {
4962 			repl++;
4963 		} else {
4964 			repl = repl->p_next;
4965 		}
4966 	}
4967 
4968 	repl = first_repl;
4969 	targ = *target;
4970 
4971 	for (i = 0; i < npgs; i++) {
4972 		ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO));
4973 		page_clr_all_props(repl);
4974 		page_set_props(repl, ppattr);
4975 		page_relocate_hash(repl, targ);
4976 
4977 		ASSERT(hat_page_getshare(targ) == 0);
4978 		ASSERT(hat_page_getshare(repl) == 0);
4979 		/*
4980 		 * Now clear the props on targ, after the
4981 		 * page_relocate_hash(), they no longer
4982 		 * have any meaning.
4983 		 */
4984 		page_clr_all_props(targ);
4985 		ASSERT(targ->p_next == targ);
4986 		ASSERT(targ->p_prev == targ);
4987 		page_list_concat(&pl, &targ);
4988 
4989 		targ++;
4990 		if (repl_contig != 0) {
4991 			repl++;
4992 		} else {
4993 			repl = repl->p_next;
4994 		}
4995 	}
4996 	/* assert that we have come full circle with repl */
4997 	ASSERT(repl_contig == 1 || first_repl == repl);
4998 
4999 	*target = pl;
5000 	if (*replacement == NULL) {
5001 		ASSERT(first_repl == repl);
5002 		*replacement = repl;
5003 	}
5004 	VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
5005 	*nrelocp = npgs;
5006 	return (0);
5007 }
5008 /*
5009  * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
5010  */
5011 int
5012 page_relocate(
5013 	page_t **target,
5014 	page_t **replacement,
5015 	int grouplock,
5016 	int freetarget,
5017 	spgcnt_t *nrelocp,
5018 	lgrp_t *lgrp)
5019 {
5020 	spgcnt_t ret;
5021 
5022 	/* do_page_relocate returns 0 on success or errno value */
5023 	ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp);
5024 
5025 	if (ret != 0 || freetarget == 0) {
5026 		return (ret);
5027 	}
5028 	if (*nrelocp == 1) {
5029 		ASSERT(*target != NULL);
5030 		page_free(*target, 1);
5031 	} else {
5032 		page_t *tpp = *target;
5033 		uint_t szc = tpp->p_szc;
5034 		pgcnt_t npgs = page_get_pagecnt(szc);
5035 		ASSERT(npgs > 1);
5036 		ASSERT(szc != 0);
5037 		do {
5038 			ASSERT(PAGE_EXCL(tpp));
5039 			ASSERT(!hat_page_is_mapped(tpp));
5040 			ASSERT(tpp->p_szc == szc);
5041 			PP_SETFREE(tpp);
5042 			PP_SETAGED(tpp);
5043 			npgs--;
5044 		} while ((tpp = tpp->p_next) != *target);
5045 		ASSERT(npgs == 0);
5046 		page_list_add_pages(*target, 0);
5047 		npgs = page_get_pagecnt(szc);
5048 		page_create_putback(npgs);
5049 	}
5050 	return (ret);
5051 }
5052 
5053 /*
5054  * it is up to the caller to deal with pcf accounting.
5055  */
5056 void
5057 page_free_replacement_page(page_t *pplist)
5058 {
5059 	page_t *pp;
5060 
5061 	while (pplist != NULL) {
5062 		/*
5063 		 * pp_targ is a linked list.
5064 		 */
5065 		pp = pplist;
5066 		if (pp->p_szc == 0) {
5067 			page_sub(&pplist, pp);
5068 			page_clr_all_props(pp);
5069 			PP_SETFREE(pp);
5070 			PP_SETAGED(pp);
5071 			page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
5072 			page_unlock(pp);
5073 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]);
5074 		} else {
5075 			spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc);
5076 			page_t *tpp;
5077 			page_list_break(&pp, &pplist, curnpgs);
5078 			tpp = pp;
5079 			do {
5080 				ASSERT(PAGE_EXCL(tpp));
5081 				ASSERT(!hat_page_is_mapped(tpp));
5082 				page_clr_all_props(pp);
5083 				PP_SETFREE(tpp);
5084 				PP_SETAGED(tpp);
5085 			} while ((tpp = tpp->p_next) != pp);
5086 			page_list_add_pages(pp, 0);
5087 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]);
5088 		}
5089 	}
5090 }
5091 
5092 /*
5093  * Relocate target to non-relocatable replacement page.
5094  */
5095 int
5096 page_relocate_cage(page_t **target, page_t **replacement)
5097 {
5098 	page_t *tpp, *rpp;
5099 	spgcnt_t pgcnt, npgs;
5100 	int result;
5101 
5102 	tpp = *target;
5103 
5104 	ASSERT(PAGE_EXCL(tpp));
5105 	ASSERT(tpp->p_szc == 0);
5106 
5107 	pgcnt = btop(page_get_pagesize(tpp->p_szc));
5108 
5109 	do {
5110 		(void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC);
5111 		rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC);
5112 		if (rpp == NULL) {
5113 			page_create_putback(pgcnt);
5114 			kcage_cageout_wakeup();
5115 		}
5116 	} while (rpp == NULL);
5117 
5118 	ASSERT(PP_ISNORELOC(rpp));
5119 
5120 	result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL);
5121 
5122 	if (result == 0) {
5123 		*replacement = rpp;
5124 		if (pgcnt != npgs)
5125 			panic("page_relocate_cage: partial relocation");
5126 	}
5127 
5128 	return (result);
5129 }
5130 
5131 /*
5132  * Release the page lock on a page, place on cachelist
5133  * tail if no longer mapped. Caller can let us know if
5134  * the page is known to be clean.
5135  */
5136 int
5137 page_release(page_t *pp, int checkmod)
5138 {
5139 	int status;
5140 
5141 	ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) &&
5142 	    (pp->p_vnode != NULL));
5143 
5144 	if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) &&
5145 	    ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) &&
5146 	    pp->p_lckcnt == 0 && pp->p_cowcnt == 0 &&
5147 	    !hat_page_is_mapped(pp)) {
5148 
5149 		/*
5150 		 * If page is modified, unlock it
5151 		 *
5152 		 * (p_nrm & P_MOD) bit has the latest stuff because:
5153 		 * (1) We found that this page doesn't have any mappings
5154 		 *	_after_ holding SE_EXCL and
5155 		 * (2) We didn't drop SE_EXCL lock after the check in (1)
5156 		 */
5157 		if (checkmod && hat_ismod(pp)) {
5158 			page_unlock(pp);
5159 			status = PGREL_MOD;
5160 		} else {
5161 			/*LINTED: constant in conditional context*/
5162 			VN_DISPOSE(pp, B_FREE, 0, kcred);
5163 			status = PGREL_CLEAN;
5164 		}
5165 	} else {
5166 		page_unlock(pp);
5167 		status = PGREL_NOTREL;
5168 	}
5169 	return (status);
5170 }
5171 
5172 /*
5173  * Given a constituent page, try to demote the large page on the freelist.
5174  *
5175  * Returns nonzero if the page could be demoted successfully. Returns with
5176  * the constituent page still locked.
5177  */
5178 int
5179 page_try_demote_free_pages(page_t *pp)
5180 {
5181 	page_t *rootpp = pp;
5182 	pfn_t	pfn = page_pptonum(pp);
5183 	spgcnt_t npgs;
5184 	uint_t	szc = pp->p_szc;
5185 
5186 	ASSERT(PP_ISFREE(pp));
5187 	ASSERT(PAGE_EXCL(pp));
5188 
5189 	/*
5190 	 * Adjust rootpp and lock it, if `pp' is not the base
5191 	 * constituent page.
5192 	 */
5193 	npgs = page_get_pagecnt(pp->p_szc);
5194 	if (npgs == 1) {
5195 		return (0);
5196 	}
5197 
5198 	if (!IS_P2ALIGNED(pfn, npgs)) {
5199 		pfn = P2ALIGN(pfn, npgs);
5200 		rootpp = page_numtopp_nolock(pfn);
5201 	}
5202 
5203 	if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) {
5204 		return (0);
5205 	}
5206 
5207 	if (rootpp->p_szc != szc) {
5208 		if (pp != rootpp)
5209 			page_unlock(rootpp);
5210 		return (0);
5211 	}
5212 
5213 	page_demote_free_pages(rootpp);
5214 
5215 	if (pp != rootpp)
5216 		page_unlock(rootpp);
5217 
5218 	ASSERT(PP_ISFREE(pp));
5219 	ASSERT(PAGE_EXCL(pp));
5220 	return (1);
5221 }
5222 
5223 /*
5224  * Given a constituent page, try to demote the large page.
5225  *
5226  * Returns nonzero if the page could be demoted successfully. Returns with
5227  * the constituent page still locked.
5228  */
5229 int
5230 page_try_demote_pages(page_t *pp)
5231 {
5232 	page_t *tpp, *rootpp = pp;
5233 	pfn_t	pfn = page_pptonum(pp);
5234 	spgcnt_t i, npgs;
5235 	uint_t	szc = pp->p_szc;
5236 	vnode_t *vp = pp->p_vnode;
5237 
5238 	ASSERT(PAGE_EXCL(pp));
5239 
5240 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]);
5241 
5242 	if (pp->p_szc == 0) {
5243 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]);
5244 		return (1);
5245 	}
5246 
5247 	if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) {
5248 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]);
5249 		page_demote_vp_pages(pp);
5250 		ASSERT(pp->p_szc == 0);
5251 		return (1);
5252 	}
5253 
5254 	/*
5255 	 * Adjust rootpp if passed in is not the base
5256 	 * constituent page.
5257 	 */
5258 	npgs = page_get_pagecnt(pp->p_szc);
5259 	ASSERT(npgs > 1);
5260 	if (!IS_P2ALIGNED(pfn, npgs)) {
5261 		pfn = P2ALIGN(pfn, npgs);
5262 		rootpp = page_numtopp_nolock(pfn);
5263 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]);
5264 		ASSERT(rootpp->p_vnode != NULL);
5265 		ASSERT(rootpp->p_szc == szc);
5266 	}
5267 
5268 	/*
5269 	 * We can't demote kernel pages since we can't hat_unload()
5270 	 * the mappings.
5271 	 */
5272 	if (VN_ISKAS(rootpp->p_vnode))
5273 		return (0);
5274 
5275 	/*
5276 	 * Attempt to lock all constituent pages except the page passed
5277 	 * in since it's already locked.
5278 	 */
5279 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5280 		ASSERT(!PP_ISFREE(tpp));
5281 		ASSERT(tpp->p_vnode != NULL);
5282 
5283 		if (tpp != pp && !page_trylock(tpp, SE_EXCL))
5284 			break;
5285 		ASSERT(tpp->p_szc == rootpp->p_szc);
5286 		ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i);
5287 	}
5288 
5289 	/*
5290 	 * If we failed to lock them all then unlock what we have
5291 	 * locked so far and bail.
5292 	 */
5293 	if (i < npgs) {
5294 		tpp = rootpp;
5295 		while (i-- > 0) {
5296 			if (tpp != pp)
5297 				page_unlock(tpp);
5298 			tpp++;
5299 		}
5300 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]);
5301 		return (0);
5302 	}
5303 
5304 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5305 		ASSERT(PAGE_EXCL(tpp));
5306 		ASSERT(tpp->p_slckcnt == 0);
5307 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
5308 		tpp->p_szc = 0;
5309 	}
5310 
5311 	/*
5312 	 * Unlock all pages except the page passed in.
5313 	 */
5314 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5315 		ASSERT(!hat_page_is_mapped(tpp));
5316 		if (tpp != pp)
5317 			page_unlock(tpp);
5318 	}
5319 
5320 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]);
5321 	return (1);
5322 }
5323 
5324 /*
5325  * Called by page_free() and page_destroy() to demote the page size code
5326  * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
5327  * p_szc on free list, neither can we just clear p_szc of a single page_t
5328  * within a large page since it will break other code that relies on p_szc
5329  * being the same for all page_t's of a large page). Anonymous pages should
5330  * never end up here because anon_map_getpages() cannot deal with p_szc
5331  * changes after a single constituent page is locked.  While anonymous or
5332  * kernel large pages are demoted or freed the entire large page at a time
5333  * with all constituent pages locked EXCL for the file system pages we
5334  * have to be able to demote a large page (i.e. decrease all constituent pages
5335  * p_szc) with only just an EXCL lock on one of constituent pages. The reason
5336  * we can easily deal with anonymous page demotion the entire large page at a
5337  * time is that those operation originate at address space level and concern
5338  * the entire large page region with actual demotion only done when pages are
5339  * not shared with any other processes (therefore we can always get EXCL lock
5340  * on all anonymous constituent pages after clearing segment page
5341  * cache). However file system pages can be truncated or invalidated at a
5342  * PAGESIZE level from the file system side and end up in page_free() or
5343  * page_destroy() (we also allow only part of the large page to be SOFTLOCKed
5344  * and therfore pageout should be able to demote a large page by EXCL locking
5345  * any constituent page that is not under SOFTLOCK). In those cases we cannot
5346  * rely on being able to lock EXCL all constituent pages.
5347  *
5348  * To prevent szc changes on file system pages one has to lock all constituent
5349  * pages at least SHARED (or call page_szc_lock()). The only subsystem that
5350  * doesn't rely on locking all constituent pages (or using page_szc_lock()) to
5351  * prevent szc changes is hat layer that uses its own page level mlist
5352  * locks. hat assumes that szc doesn't change after mlist lock for a page is
5353  * taken. Therefore we need to change szc under hat level locks if we only
5354  * have an EXCL lock on a single constituent page and hat still references any
5355  * of constituent pages.  (Note we can't "ignore" hat layer by simply
5356  * hat_pageunload() all constituent pages without having EXCL locks on all of
5357  * constituent pages). We use hat_page_demote() call to safely demote szc of
5358  * all constituent pages under hat locks when we only have an EXCL lock on one
5359  * of constituent pages.
5360  *
5361  * This routine calls page_szc_lock() before calling hat_page_demote() to
5362  * allow segvn in one special case not to lock all constituent pages SHARED
5363  * before calling hat_memload_array() that relies on p_szc not changeing even
5364  * before hat level mlist lock is taken.  In that case segvn uses
5365  * page_szc_lock() to prevent hat_page_demote() changeing p_szc values.
5366  *
5367  * Anonymous or kernel page demotion still has to lock all pages exclusively
5368  * and do hat_pageunload() on all constituent pages before demoting the page
5369  * therefore there's no need for anonymous or kernel page demotion to use
5370  * hat_page_demote() mechanism.
5371  *
5372  * hat_page_demote() removes all large mappings that map pp and then decreases
5373  * p_szc starting from the last constituent page of the large page. By working
5374  * from the tail of a large page in pfn decreasing order allows one looking at
5375  * the root page to know that hat_page_demote() is done for root's szc area.
5376  * e.g. if a root page has szc 1 one knows it only has to lock all constituent
5377  * pages within szc 1 area to prevent szc changes because hat_page_demote()
5378  * that started on this page when it had szc > 1 is done for this szc 1 area.
5379  *
5380  * We are guranteed that all constituent pages of pp's large page belong to
5381  * the same vnode with the consecutive offsets increasing in the direction of
5382  * the pfn i.e. the identity of constituent pages can't change until their
5383  * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
5384  * large mappings to pp even though we don't lock any constituent page except
5385  * pp (i.e. we won't unload e.g. kernel locked page).
5386  */
5387 static void
5388 page_demote_vp_pages(page_t *pp)
5389 {
5390 	kmutex_t *mtx;
5391 
5392 	ASSERT(PAGE_EXCL(pp));
5393 	ASSERT(!PP_ISFREE(pp));
5394 	ASSERT(pp->p_vnode != NULL);
5395 	ASSERT(!IS_SWAPFSVP(pp->p_vnode));
5396 	ASSERT(!PP_ISKAS(pp));
5397 
5398 	VM_STAT_ADD(pagecnt.pc_demote_pages[0]);
5399 
5400 	mtx = page_szc_lock(pp);
5401 	if (mtx != NULL) {
5402 		hat_page_demote(pp);
5403 		mutex_exit(mtx);
5404 	}
5405 	ASSERT(pp->p_szc == 0);
5406 }
5407 
5408 /*
5409  * Mark any existing pages for migration in the given range
5410  */
5411 void
5412 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len,
5413     struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
5414     u_offset_t vnoff, int rflag)
5415 {
5416 	struct anon	*ap;
5417 	vnode_t		*curvp;
5418 	lgrp_t		*from;
5419 	pgcnt_t		i;
5420 	pgcnt_t		nlocked;
5421 	u_offset_t	off;
5422 	pfn_t		pfn;
5423 	size_t		pgsz;
5424 	size_t		segpgsz;
5425 	pgcnt_t		pages;
5426 	uint_t		pszc;
5427 	page_t		**ppa;
5428 	pgcnt_t		ppa_nentries;
5429 	page_t		*pp;
5430 	caddr_t		va;
5431 	ulong_t		an_idx;
5432 	anon_sync_obj_t	cookie;
5433 
5434 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
5435 
5436 	/*
5437 	 * Don't do anything if don't need to do lgroup optimizations
5438 	 * on this system
5439 	 */
5440 	if (!lgrp_optimizations())
5441 		return;
5442 
5443 	/*
5444 	 * Align address and length to (potentially large) page boundary
5445 	 */
5446 	segpgsz = page_get_pagesize(seg->s_szc);
5447 	addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz);
5448 	if (rflag)
5449 		len = P2ROUNDUP(len, segpgsz);
5450 
5451 	/*
5452 	 * Allocate page array to accomodate largest page size
5453 	 */
5454 	pgsz = page_get_pagesize(page_num_pagesizes() - 1);
5455 	ppa_nentries = btop(pgsz);
5456 	ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP);
5457 
5458 	/*
5459 	 * Do one (large) page at a time
5460 	 */
5461 	va = addr;
5462 	while (va < addr + len) {
5463 		/*
5464 		 * Lookup (root) page for vnode and offset corresponding to
5465 		 * this virtual address
5466 		 * Try anonmap first since there may be copy-on-write
5467 		 * pages, but initialize vnode pointer and offset using
5468 		 * vnode arguments just in case there isn't an amp.
5469 		 */
5470 		curvp = vp;
5471 		off = vnoff + va - seg->s_base;
5472 		if (amp) {
5473 			ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
5474 			an_idx = anon_index + seg_page(seg, va);
5475 			anon_array_enter(amp, an_idx, &cookie);
5476 			ap = anon_get_ptr(amp->ahp, an_idx);
5477 			if (ap)
5478 				swap_xlate(ap, &curvp, &off);
5479 			anon_array_exit(&cookie);
5480 			ANON_LOCK_EXIT(&amp->a_rwlock);
5481 		}
5482 
5483 		pp = NULL;
5484 		if (curvp)
5485 			pp = page_lookup(curvp, off, SE_SHARED);
5486 
5487 		/*
5488 		 * If there isn't a page at this virtual address,
5489 		 * skip to next page
5490 		 */
5491 		if (pp == NULL) {
5492 			va += PAGESIZE;
5493 			continue;
5494 		}
5495 
5496 		/*
5497 		 * Figure out which lgroup this page is in for kstats
5498 		 */
5499 		pfn = page_pptonum(pp);
5500 		from = lgrp_pfn_to_lgrp(pfn);
5501 
5502 		/*
5503 		 * Get page size, and round up and skip to next page boundary
5504 		 * if unaligned address
5505 		 */
5506 		pszc = pp->p_szc;
5507 		pgsz = page_get_pagesize(pszc);
5508 		pages = btop(pgsz);
5509 		if (!IS_P2ALIGNED(va, pgsz) ||
5510 		    !IS_P2ALIGNED(pfn, pages) ||
5511 		    pgsz > segpgsz) {
5512 			pgsz = MIN(pgsz, segpgsz);
5513 			page_unlock(pp);
5514 			i = btop(P2END((uintptr_t)va, pgsz) -
5515 			    (uintptr_t)va);
5516 			va = (caddr_t)P2END((uintptr_t)va, pgsz);
5517 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i);
5518 			continue;
5519 		}
5520 
5521 		/*
5522 		 * Upgrade to exclusive lock on page
5523 		 */
5524 		if (!page_tryupgrade(pp)) {
5525 			page_unlock(pp);
5526 			va += pgsz;
5527 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5528 			    btop(pgsz));
5529 			continue;
5530 		}
5531 
5532 		/*
5533 		 * Remember pages locked exclusively and how many
5534 		 */
5535 		ppa[0] = pp;
5536 		nlocked = 1;
5537 
5538 		/*
5539 		 * Lock constituent pages if this is large page
5540 		 */
5541 		if (pages > 1) {
5542 			/*
5543 			 * Lock all constituents except root page, since it
5544 			 * should be locked already.
5545 			 */
5546 			for (i = 1; i < pages; i++) {
5547 				pp++;
5548 				if (!page_trylock(pp, SE_EXCL)) {
5549 					break;
5550 				}
5551 				if (PP_ISFREE(pp) ||
5552 				    pp->p_szc != pszc) {
5553 					/*
5554 					 * hat_page_demote() raced in with us.
5555 					 */
5556 					ASSERT(!IS_SWAPFSVP(curvp));
5557 					page_unlock(pp);
5558 					break;
5559 				}
5560 				ppa[nlocked] = pp;
5561 				nlocked++;
5562 			}
5563 		}
5564 
5565 		/*
5566 		 * If all constituent pages couldn't be locked,
5567 		 * unlock pages locked so far and skip to next page.
5568 		 */
5569 		if (nlocked != pages) {
5570 			for (i = 0; i < nlocked; i++)
5571 				page_unlock(ppa[i]);
5572 			va += pgsz;
5573 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5574 			    btop(pgsz));
5575 			continue;
5576 		}
5577 
5578 		/*
5579 		 * hat_page_demote() can no longer happen
5580 		 * since last cons page had the right p_szc after
5581 		 * all cons pages were locked. all cons pages
5582 		 * should now have the same p_szc.
5583 		 */
5584 
5585 		/*
5586 		 * All constituent pages locked successfully, so mark
5587 		 * large page for migration and unload the mappings of
5588 		 * constituent pages, so a fault will occur on any part of the
5589 		 * large page
5590 		 */
5591 		PP_SETMIGRATE(ppa[0]);
5592 		for (i = 0; i < nlocked; i++) {
5593 			pp = ppa[i];
5594 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
5595 			ASSERT(hat_page_getshare(pp) == 0);
5596 			page_unlock(pp);
5597 		}
5598 		lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked);
5599 
5600 		va += pgsz;
5601 	}
5602 	kmem_free(ppa, ppa_nentries * sizeof (page_t *));
5603 }
5604 
5605 /*
5606  * Migrate any pages that have been marked for migration in the given range
5607  */
5608 void
5609 page_migrate(
5610 	struct seg	*seg,
5611 	caddr_t		addr,
5612 	page_t		**ppa,
5613 	pgcnt_t		npages)
5614 {
5615 	lgrp_t		*from;
5616 	lgrp_t		*to;
5617 	page_t		*newpp;
5618 	page_t		*pp;
5619 	pfn_t		pfn;
5620 	size_t		pgsz;
5621 	spgcnt_t	page_cnt;
5622 	spgcnt_t	i;
5623 	uint_t		pszc;
5624 
5625 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
5626 
5627 	while (npages > 0) {
5628 		pp = *ppa;
5629 		pszc = pp->p_szc;
5630 		pgsz = page_get_pagesize(pszc);
5631 		page_cnt = btop(pgsz);
5632 
5633 		/*
5634 		 * Check to see whether this page is marked for migration
5635 		 *
5636 		 * Assume that root page of large page is marked for
5637 		 * migration and none of the other constituent pages
5638 		 * are marked.  This really simplifies clearing the
5639 		 * migrate bit by not having to clear it from each
5640 		 * constituent page.
5641 		 *
5642 		 * note we don't want to relocate an entire large page if
5643 		 * someone is only using one subpage.
5644 		 */
5645 		if (npages < page_cnt)
5646 			break;
5647 
5648 		/*
5649 		 * Is it marked for migration?
5650 		 */
5651 		if (!PP_ISMIGRATE(pp))
5652 			goto next;
5653 
5654 		/*
5655 		 * Determine lgroups that page is being migrated between
5656 		 */
5657 		pfn = page_pptonum(pp);
5658 		if (!IS_P2ALIGNED(pfn, page_cnt)) {
5659 			break;
5660 		}
5661 		from = lgrp_pfn_to_lgrp(pfn);
5662 		to = lgrp_mem_choose(seg, addr, pgsz);
5663 
5664 		/*
5665 		 * Check to see whether we are trying to migrate page to lgroup
5666 		 * where it is allocated already
5667 		 */
5668 		if (to == from) {
5669 			PP_CLRMIGRATE(pp);
5670 			goto next;
5671 		}
5672 
5673 		/*
5674 		 * Need to get exclusive lock's to migrate
5675 		 */
5676 		for (i = 0; i < page_cnt; i++) {
5677 			ASSERT(PAGE_LOCKED(ppa[i]));
5678 			if (page_pptonum(ppa[i]) != pfn + i ||
5679 			    ppa[i]->p_szc != pszc) {
5680 				break;
5681 			}
5682 			if (!page_tryupgrade(ppa[i])) {
5683 				lgrp_stat_add(from->lgrp_id,
5684 				    LGRP_PM_FAIL_LOCK_PGS,
5685 				    page_cnt);
5686 				break;
5687 			}
5688 		}
5689 		if (i != page_cnt) {
5690 			while (--i != -1) {
5691 				page_downgrade(ppa[i]);
5692 			}
5693 			goto next;
5694 		}
5695 
5696 		(void) page_create_wait(page_cnt, PG_WAIT);
5697 		newpp = page_get_replacement_page(pp, to, PGR_SAMESZC);
5698 		if (newpp == NULL) {
5699 			page_create_putback(page_cnt);
5700 			for (i = 0; i < page_cnt; i++) {
5701 				page_downgrade(ppa[i]);
5702 			}
5703 			lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS,
5704 			    page_cnt);
5705 			goto next;
5706 		}
5707 		ASSERT(newpp->p_szc == pszc);
5708 		/*
5709 		 * Clear migrate bit and relocate page
5710 		 */
5711 		PP_CLRMIGRATE(pp);
5712 		if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) {
5713 			panic("page_migrate: page_relocate failed");
5714 		}
5715 		ASSERT(page_cnt * PAGESIZE == pgsz);
5716 
5717 		/*
5718 		 * Keep stats for number of pages migrated from and to
5719 		 * each lgroup
5720 		 */
5721 		lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt);
5722 		lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt);
5723 		/*
5724 		 * update the page_t array we were passed in and
5725 		 * unlink constituent pages of a large page.
5726 		 */
5727 		for (i = 0; i < page_cnt; ++i, ++pp) {
5728 			ASSERT(PAGE_EXCL(newpp));
5729 			ASSERT(newpp->p_szc == pszc);
5730 			ppa[i] = newpp;
5731 			pp = newpp;
5732 			page_sub(&newpp, pp);
5733 			page_downgrade(pp);
5734 		}
5735 		ASSERT(newpp == NULL);
5736 next:
5737 		addr += pgsz;
5738 		ppa += page_cnt;
5739 		npages -= page_cnt;
5740 	}
5741 }
5742 
5743 ulong_t mem_waiters 	= 0;
5744 ulong_t	max_count 	= 20;
5745 #define	MAX_DELAY	0x1ff
5746 
5747 /*
5748  * Check if enough memory is available to proceed.
5749  * Depending on system configuration and how much memory is
5750  * reserved for swap we need to check against two variables.
5751  * e.g. on systems with little physical swap availrmem can be
5752  * more reliable indicator of how much memory is available.
5753  * On systems with large phys swap freemem can be better indicator.
5754  * If freemem drops below threshold level don't return an error
5755  * immediately but wake up pageout to free memory and block.
5756  * This is done number of times. If pageout is not able to free
5757  * memory within certain time return an error.
5758  * The same applies for availrmem but kmem_reap is used to
5759  * free memory.
5760  */
5761 int
5762 page_mem_avail(pgcnt_t npages)
5763 {
5764 	ulong_t count;
5765 
5766 #if defined(__i386)
5767 	if (freemem > desfree + npages &&
5768 	    availrmem > swapfs_reserve + npages &&
5769 	    btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem +
5770 	    npages)
5771 		return (1);
5772 #else
5773 	if (freemem > desfree + npages &&
5774 	    availrmem > swapfs_reserve + npages)
5775 		return (1);
5776 #endif
5777 
5778 	count = max_count;
5779 	atomic_add_long(&mem_waiters, 1);
5780 
5781 	while (freemem < desfree + npages && --count) {
5782 		cv_signal(&proc_pageout->p_cv);
5783 		if (delay_sig(hz + (mem_waiters & MAX_DELAY))) {
5784 			atomic_add_long(&mem_waiters, -1);
5785 			return (0);
5786 		}
5787 	}
5788 	if (count == 0) {
5789 		atomic_add_long(&mem_waiters, -1);
5790 		return (0);
5791 	}
5792 
5793 	count = max_count;
5794 	while (availrmem < swapfs_reserve + npages && --count) {
5795 		kmem_reap();
5796 		if (delay_sig(hz + (mem_waiters & MAX_DELAY))) {
5797 			atomic_add_long(&mem_waiters, -1);
5798 			return (0);
5799 		}
5800 	}
5801 	atomic_add_long(&mem_waiters, -1);
5802 	if (count == 0)
5803 		return (0);
5804 
5805 #if defined(__i386)
5806 	if (btop(vmem_size(heap_arena, VMEM_FREE)) <
5807 	    tune.t_minarmem + npages)
5808 		return (0);
5809 #endif
5810 	return (1);
5811 }
5812 
5813 #define	MAX_CNT	60	/* max num of iterations */
5814 /*
5815  * Reclaim/reserve availrmem for npages.
5816  * If there is not enough memory start reaping seg, kmem caches.
5817  * Start pageout scanner (via page_needfree()).
5818  * Exit after ~ MAX_CNT s regardless of how much memory has been released.
5819  * Note: There is no guarantee that any availrmem will be freed as
5820  * this memory typically is locked (kernel heap) or reserved for swap.
5821  * Also due to memory fragmentation kmem allocator may not be able
5822  * to free any memory (single user allocated buffer will prevent
5823  * freeing slab or a page).
5824  */
5825 int
5826 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
5827 {
5828 	int	i = 0;
5829 	int	ret = 0;
5830 	pgcnt_t	deficit;
5831 	pgcnt_t old_availrmem;
5832 
5833 	mutex_enter(&freemem_lock);
5834 	old_availrmem = availrmem - 1;
5835 	while ((availrmem < tune.t_minarmem + npages + epages) &&
5836 	    (old_availrmem < availrmem) && (i++ < MAX_CNT)) {
5837 		old_availrmem = availrmem;
5838 		deficit = tune.t_minarmem + npages + epages - availrmem;
5839 		mutex_exit(&freemem_lock);
5840 		page_needfree(deficit);
5841 		seg_preap();
5842 		kmem_reap();
5843 		delay(hz);
5844 		page_needfree(-(spgcnt_t)deficit);
5845 		mutex_enter(&freemem_lock);
5846 	}
5847 
5848 	if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) {
5849 		availrmem -= npages;
5850 		ret = 1;
5851 	}
5852 
5853 	mutex_exit(&freemem_lock);
5854 
5855 	return (ret);
5856 }
5857 
5858 /*
5859  * Search the memory segments to locate the desired page.  Within a
5860  * segment, pages increase linearly with one page structure per
5861  * physical page frame (size PAGESIZE).  The search begins
5862  * with the segment that was accessed last, to take advantage of locality.
5863  * If the hint misses, we start from the beginning of the sorted memseg list
5864  */
5865 
5866 
5867 /*
5868  * Some data structures for pfn to pp lookup.
5869  */
5870 ulong_t mhash_per_slot;
5871 struct memseg *memseg_hash[N_MEM_SLOTS];
5872 
5873 page_t *
5874 page_numtopp_nolock(pfn_t pfnum)
5875 {
5876 	struct memseg *seg;
5877 	page_t *pp;
5878 	vm_cpu_data_t *vc = CPU->cpu_vm_data;
5879 
5880 	ASSERT(vc != NULL);
5881 
5882 	MEMSEG_STAT_INCR(nsearch);
5883 
5884 	/* Try last winner first */
5885 	if (((seg = vc->vc_pnum_memseg) != NULL) &&
5886 	    (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5887 		MEMSEG_STAT_INCR(nlastwon);
5888 		pp = seg->pages + (pfnum - seg->pages_base);
5889 		if (pp->p_pagenum == pfnum)
5890 			return ((page_t *)pp);
5891 	}
5892 
5893 	/* Else Try hash */
5894 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5895 	    (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5896 		MEMSEG_STAT_INCR(nhashwon);
5897 		vc->vc_pnum_memseg = seg;
5898 		pp = seg->pages + (pfnum - seg->pages_base);
5899 		if (pp->p_pagenum == pfnum)
5900 			return ((page_t *)pp);
5901 	}
5902 
5903 	/* Else Brute force */
5904 	for (seg = memsegs; seg != NULL; seg = seg->next) {
5905 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5906 			vc->vc_pnum_memseg = seg;
5907 			pp = seg->pages + (pfnum - seg->pages_base);
5908 			return ((page_t *)pp);
5909 		}
5910 	}
5911 	vc->vc_pnum_memseg = NULL;
5912 	MEMSEG_STAT_INCR(nnotfound);
5913 	return ((page_t *)NULL);
5914 
5915 }
5916 
5917 struct memseg *
5918 page_numtomemseg_nolock(pfn_t pfnum)
5919 {
5920 	struct memseg *seg;
5921 	page_t *pp;
5922 
5923 	/* Try hash */
5924 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5925 	    (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5926 		pp = seg->pages + (pfnum - seg->pages_base);
5927 		if (pp->p_pagenum == pfnum)
5928 			return (seg);
5929 	}
5930 
5931 	/* Else Brute force */
5932 	for (seg = memsegs; seg != NULL; seg = seg->next) {
5933 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5934 			return (seg);
5935 		}
5936 	}
5937 	return ((struct memseg *)NULL);
5938 }
5939 
5940 /*
5941  * Given a page and a count return the page struct that is
5942  * n structs away from the current one in the global page
5943  * list.
5944  *
5945  * This function wraps to the first page upon
5946  * reaching the end of the memseg list.
5947  */
5948 page_t *
5949 page_nextn(page_t *pp, ulong_t n)
5950 {
5951 	struct memseg *seg;
5952 	page_t *ppn;
5953 	vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data;
5954 
5955 	ASSERT(vc != NULL);
5956 
5957 	if (((seg = vc->vc_pnext_memseg) == NULL) ||
5958 	    (seg->pages_base == seg->pages_end) ||
5959 	    !(pp >= seg->pages && pp < seg->epages)) {
5960 
5961 		for (seg = memsegs; seg; seg = seg->next) {
5962 			if (pp >= seg->pages && pp < seg->epages)
5963 				break;
5964 		}
5965 
5966 		if (seg == NULL) {
5967 			/* Memory delete got in, return something valid. */
5968 			/* TODO: fix me. */
5969 			seg = memsegs;
5970 			pp = seg->pages;
5971 		}
5972 	}
5973 
5974 	/* check for wraparound - possible if n is large */
5975 	while ((ppn = (pp + n)) >= seg->epages || ppn < pp) {
5976 		n -= seg->epages - pp;
5977 		seg = seg->next;
5978 		if (seg == NULL)
5979 			seg = memsegs;
5980 		pp = seg->pages;
5981 	}
5982 	vc->vc_pnext_memseg = seg;
5983 	return (ppn);
5984 }
5985 
5986 /*
5987  * Initialize for a loop using page_next_scan_large().
5988  */
5989 page_t *
5990 page_next_scan_init(void **cookie)
5991 {
5992 	ASSERT(cookie != NULL);
5993 	*cookie = (void *)memsegs;
5994 	return ((page_t *)memsegs->pages);
5995 }
5996 
5997 /*
5998  * Return the next page in a scan of page_t's, assuming we want
5999  * to skip over sub-pages within larger page sizes.
6000  *
6001  * The cookie is used to keep track of the current memseg.
6002  */
6003 page_t *
6004 page_next_scan_large(
6005 	page_t		*pp,
6006 	ulong_t		*n,
6007 	void		**cookie)
6008 {
6009 	struct memseg	*seg = (struct memseg *)*cookie;
6010 	page_t		*new_pp;
6011 	ulong_t		cnt;
6012 	pfn_t		pfn;
6013 
6014 
6015 	/*
6016 	 * get the count of page_t's to skip based on the page size
6017 	 */
6018 	ASSERT(pp != NULL);
6019 	if (pp->p_szc == 0) {
6020 		cnt = 1;
6021 	} else {
6022 		pfn = page_pptonum(pp);
6023 		cnt = page_get_pagecnt(pp->p_szc);
6024 		cnt -= pfn & (cnt - 1);
6025 	}
6026 	*n += cnt;
6027 	new_pp = pp + cnt;
6028 
6029 	/*
6030 	 * Catch if we went past the end of the current memory segment. If so,
6031 	 * just move to the next segment with pages.
6032 	 */
6033 	if (new_pp >= seg->epages) {
6034 		do {
6035 			seg = seg->next;
6036 			if (seg == NULL)
6037 				seg = memsegs;
6038 		} while (seg->pages == seg->epages);
6039 		new_pp = seg->pages;
6040 		*cookie = (void *)seg;
6041 	}
6042 
6043 	return (new_pp);
6044 }
6045 
6046 
6047 /*
6048  * Returns next page in list. Note: this function wraps
6049  * to the first page in the list upon reaching the end
6050  * of the list. Callers should be aware of this fact.
6051  */
6052 
6053 /* We should change this be a #define */
6054 
6055 page_t *
6056 page_next(page_t *pp)
6057 {
6058 	return (page_nextn(pp, 1));
6059 }
6060 
6061 page_t *
6062 page_first()
6063 {
6064 	return ((page_t *)memsegs->pages);
6065 }
6066 
6067 
6068 /*
6069  * This routine is called at boot with the initial memory configuration
6070  * and when memory is added or removed.
6071  */
6072 void
6073 build_pfn_hash()
6074 {
6075 	pfn_t cur;
6076 	pgcnt_t index;
6077 	struct memseg *pseg;
6078 	int	i;
6079 
6080 	/*
6081 	 * Clear memseg_hash array.
6082 	 * Since memory add/delete is designed to operate concurrently
6083 	 * with normal operation, the hash rebuild must be able to run
6084 	 * concurrently with page_numtopp_nolock(). To support this
6085 	 * functionality, assignments to memseg_hash array members must
6086 	 * be done atomically.
6087 	 *
6088 	 * NOTE: bzero() does not currently guarantee this for kernel
6089 	 * threads, and cannot be used here.
6090 	 */
6091 	for (i = 0; i < N_MEM_SLOTS; i++)
6092 		memseg_hash[i] = NULL;
6093 
6094 	hat_kpm_mseghash_clear(N_MEM_SLOTS);
6095 
6096 	/*
6097 	 * Physmax is the last valid pfn.
6098 	 */
6099 	mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT;
6100 	for (pseg = memsegs; pseg != NULL; pseg = pseg->next) {
6101 		index = MEMSEG_PFN_HASH(pseg->pages_base);
6102 		cur = pseg->pages_base;
6103 		do {
6104 			if (index >= N_MEM_SLOTS)
6105 				index = MEMSEG_PFN_HASH(cur);
6106 
6107 			if (memseg_hash[index] == NULL ||
6108 			    memseg_hash[index]->pages_base > pseg->pages_base) {
6109 				memseg_hash[index] = pseg;
6110 				hat_kpm_mseghash_update(index, pseg);
6111 			}
6112 			cur += mhash_per_slot;
6113 			index++;
6114 		} while (cur < pseg->pages_end);
6115 	}
6116 }
6117 
6118 /*
6119  * Return the pagenum for the pp
6120  */
6121 pfn_t
6122 page_pptonum(page_t *pp)
6123 {
6124 	return (pp->p_pagenum);
6125 }
6126 
6127 /*
6128  * interface to the referenced and modified etc bits
6129  * in the PSM part of the page struct
6130  * when no locking is desired.
6131  */
6132 void
6133 page_set_props(page_t *pp, uint_t flags)
6134 {
6135 	ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0);
6136 	pp->p_nrm |= (uchar_t)flags;
6137 }
6138 
6139 void
6140 page_clr_all_props(page_t *pp)
6141 {
6142 	pp->p_nrm = 0;
6143 }
6144 
6145 /*
6146  * Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
6147  */
6148 int
6149 page_clear_lck_cow(page_t *pp, int adjust)
6150 {
6151 	int	f_amount;
6152 
6153 	ASSERT(PAGE_EXCL(pp));
6154 
6155 	/*
6156 	 * The page_struct_lock need not be acquired here since
6157 	 * we require the caller hold the page exclusively locked.
6158 	 */
6159 	f_amount = 0;
6160 	if (pp->p_lckcnt) {
6161 		f_amount = 1;
6162 		pp->p_lckcnt = 0;
6163 	}
6164 	if (pp->p_cowcnt) {
6165 		f_amount += pp->p_cowcnt;
6166 		pp->p_cowcnt = 0;
6167 	}
6168 
6169 	if (adjust && f_amount) {
6170 		mutex_enter(&freemem_lock);
6171 		availrmem += f_amount;
6172 		mutex_exit(&freemem_lock);
6173 	}
6174 
6175 	return (f_amount);
6176 }
6177 
6178 /*
6179  * The following functions is called from free_vp_pages()
6180  * for an inexact estimate of a newly free'd page...
6181  */
6182 ulong_t
6183 page_share_cnt(page_t *pp)
6184 {
6185 	return (hat_page_getshare(pp));
6186 }
6187 
6188 int
6189 page_isshared(page_t *pp)
6190 {
6191 	return (hat_page_checkshare(pp, 1));
6192 }
6193 
6194 int
6195 page_isfree(page_t *pp)
6196 {
6197 	return (PP_ISFREE(pp));
6198 }
6199 
6200 int
6201 page_isref(page_t *pp)
6202 {
6203 	return (hat_page_getattr(pp, P_REF));
6204 }
6205 
6206 int
6207 page_ismod(page_t *pp)
6208 {
6209 	return (hat_page_getattr(pp, P_MOD));
6210 }
6211 
6212 /*
6213  * The following code all currently relates to the page capture logic:
6214  *
6215  * This logic is used for cases where there is a desire to claim a certain
6216  * physical page in the system for the caller.  As it may not be possible
6217  * to capture the page immediately, the p_toxic bits are used in the page
6218  * structure to indicate that someone wants to capture this page.  When the
6219  * page gets unlocked, the toxic flag will be noted and an attempt to capture
6220  * the page will be made.  If it is successful, the original callers callback
6221  * will be called with the page to do with it what they please.
6222  *
6223  * There is also an async thread which wakes up to attempt to capture
6224  * pages occasionally which have the capture bit set.  All of the pages which
6225  * need to be captured asynchronously have been inserted into the
6226  * page_capture_hash and thus this thread walks that hash list.  Items in the
6227  * hash have an expiration time so this thread handles that as well by removing
6228  * the item from the hash if it has expired.
6229  *
6230  * Some important things to note are:
6231  * - if the PR_CAPTURE bit is set on a page, then the page is in the
6232  *   page_capture_hash.  The page_capture_hash_head.pchh_mutex is needed
6233  *   to set and clear this bit, and while the lock is held is the only time
6234  *   you can add or remove an entry from the hash.
6235  * - the PR_CAPTURE bit can only be set and cleared while holding the
6236  *   page_capture_hash_head.pchh_mutex
6237  * - the t_flag field of the thread struct is used with the T_CAPTURING
6238  *   flag to prevent recursion while dealing with large pages.
6239  * - pages which need to be retired never expire on the page_capture_hash.
6240  */
6241 
6242 static void page_capture_thread(void);
6243 static kthread_t *pc_thread_id;
6244 kcondvar_t pc_cv;
6245 static kmutex_t pc_thread_mutex;
6246 static clock_t pc_thread_shortwait;
6247 static clock_t pc_thread_longwait;
6248 static int pc_thread_ism_retry;
6249 
6250 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS];
6251 
6252 /* Note that this is a circular linked list */
6253 typedef struct page_capture_hash_bucket {
6254 	page_t *pp;
6255 	uint_t szc;
6256 	uint_t flags;
6257 	clock_t expires;	/* lbolt at which this request expires. */
6258 	void *datap;		/* Cached data passed in for callback */
6259 	struct page_capture_hash_bucket *next;
6260 	struct page_capture_hash_bucket *prev;
6261 } page_capture_hash_bucket_t;
6262 
6263 /*
6264  * Each hash bucket will have it's own mutex and two lists which are:
6265  * active (0):	represents requests which have not been processed by
6266  *		the page_capture async thread yet.
6267  * walked (1):	represents requests which have been processed by the
6268  *		page_capture async thread within it's given walk of this bucket.
6269  *
6270  * These are all needed so that we can synchronize all async page_capture
6271  * events.  When the async thread moves to a new bucket, it will append the
6272  * walked list to the active list and walk each item one at a time, moving it
6273  * from the active list to the walked list.  Thus if there is an async request
6274  * outstanding for a given page, it will always be in one of the two lists.
6275  * New requests will always be added to the active list.
6276  * If we were not able to capture a page before the request expired, we'd free
6277  * up the request structure which would indicate to page_capture that there is
6278  * no longer a need for the given page, and clear the PR_CAPTURE flag if
6279  * possible.
6280  */
6281 typedef struct page_capture_hash_head {
6282 	kmutex_t pchh_mutex;
6283 	uint_t num_pages;
6284 	page_capture_hash_bucket_t lists[2]; /* sentinel nodes */
6285 } page_capture_hash_head_t;
6286 
6287 #ifdef DEBUG
6288 #define	NUM_PAGE_CAPTURE_BUCKETS 4
6289 #else
6290 #define	NUM_PAGE_CAPTURE_BUCKETS 64
6291 #endif
6292 
6293 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS];
6294 
6295 /* for now use a very simple hash based upon the size of a page struct */
6296 #define	PAGE_CAPTURE_HASH(pp)	\
6297 	((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
6298 
6299 extern pgcnt_t swapfs_minfree;
6300 
6301 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
6302 
6303 /*
6304  * a callback function is required for page capture requests.
6305  */
6306 void
6307 page_capture_register_callback(uint_t index, clock_t duration,
6308     int (*cb_func)(page_t *, void *, uint_t))
6309 {
6310 	ASSERT(pc_cb[index].cb_active == 0);
6311 	ASSERT(cb_func != NULL);
6312 	rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6313 	pc_cb[index].duration = duration;
6314 	pc_cb[index].cb_func = cb_func;
6315 	pc_cb[index].cb_active = 1;
6316 	rw_exit(&pc_cb[index].cb_rwlock);
6317 }
6318 
6319 void
6320 page_capture_unregister_callback(uint_t index)
6321 {
6322 	int i, j;
6323 	struct page_capture_hash_bucket *bp1;
6324 	struct page_capture_hash_bucket *bp2;
6325 	struct page_capture_hash_bucket *head = NULL;
6326 	uint_t flags = (1 << index);
6327 
6328 	rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6329 	ASSERT(pc_cb[index].cb_active == 1);
6330 	pc_cb[index].duration = 0;	/* Paranoia */
6331 	pc_cb[index].cb_func = NULL;	/* Paranoia */
6332 	pc_cb[index].cb_active = 0;
6333 	rw_exit(&pc_cb[index].cb_rwlock);
6334 
6335 	/*
6336 	 * Just move all the entries to a private list which we can walk
6337 	 * through without the need to hold any locks.
6338 	 * No more requests can get added to the hash lists for this consumer
6339 	 * as the cb_active field for the callback has been cleared.
6340 	 */
6341 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
6342 		mutex_enter(&page_capture_hash[i].pchh_mutex);
6343 		for (j = 0; j < 2; j++) {
6344 			bp1 = page_capture_hash[i].lists[j].next;
6345 			/* walk through all but first (sentinel) element */
6346 			while (bp1 != &page_capture_hash[i].lists[j]) {
6347 				bp2 = bp1;
6348 				if (bp2->flags & flags) {
6349 					bp1 = bp2->next;
6350 					bp1->prev = bp2->prev;
6351 					bp2->prev->next = bp1;
6352 					bp2->next = head;
6353 					head = bp2;
6354 					/*
6355 					 * Clear the PR_CAPTURE bit as we
6356 					 * hold appropriate locks here.
6357 					 */
6358 					page_clrtoxic(head->pp, PR_CAPTURE);
6359 					page_capture_hash[i].num_pages--;
6360 					continue;
6361 				}
6362 				bp1 = bp1->next;
6363 			}
6364 		}
6365 		mutex_exit(&page_capture_hash[i].pchh_mutex);
6366 	}
6367 
6368 	while (head != NULL) {
6369 		bp1 = head;
6370 		head = head->next;
6371 		kmem_free(bp1, sizeof (*bp1));
6372 	}
6373 }
6374 
6375 
6376 /*
6377  * Find pp in the active list and move it to the walked list if it
6378  * exists.
6379  * Note that most often pp should be at the front of the active list
6380  * as it is currently used and thus there is no other sort of optimization
6381  * being done here as this is a linked list data structure.
6382  * Returns 1 on successful move or 0 if page could not be found.
6383  */
6384 static int
6385 page_capture_move_to_walked(page_t *pp)
6386 {
6387 	page_capture_hash_bucket_t *bp;
6388 	int index;
6389 
6390 	index = PAGE_CAPTURE_HASH(pp);
6391 
6392 	mutex_enter(&page_capture_hash[index].pchh_mutex);
6393 	bp = page_capture_hash[index].lists[0].next;
6394 	while (bp != &page_capture_hash[index].lists[0]) {
6395 		if (bp->pp == pp) {
6396 			/* Remove from old list */
6397 			bp->next->prev = bp->prev;
6398 			bp->prev->next = bp->next;
6399 
6400 			/* Add to new list */
6401 			bp->next = page_capture_hash[index].lists[1].next;
6402 			bp->prev = &page_capture_hash[index].lists[1];
6403 			page_capture_hash[index].lists[1].next = bp;
6404 			bp->next->prev = bp;
6405 			mutex_exit(&page_capture_hash[index].pchh_mutex);
6406 
6407 			return (1);
6408 		}
6409 		bp = bp->next;
6410 	}
6411 	mutex_exit(&page_capture_hash[index].pchh_mutex);
6412 	return (0);
6413 }
6414 
6415 /*
6416  * Add a new entry to the page capture hash.  The only case where a new
6417  * entry is not added is when the page capture consumer is no longer registered.
6418  * In this case, we'll silently not add the page to the hash.  We know that
6419  * page retire will always be registered for the case where we are currently
6420  * unretiring a page and thus there are no conflicts.
6421  */
6422 static void
6423 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap)
6424 {
6425 	page_capture_hash_bucket_t *bp1;
6426 	page_capture_hash_bucket_t *bp2;
6427 	int index;
6428 	int cb_index;
6429 	int i;
6430 #ifdef DEBUG
6431 	page_capture_hash_bucket_t *tp1;
6432 	int l;
6433 #endif
6434 
6435 	ASSERT(!(flags & CAPTURE_ASYNC));
6436 
6437 	bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP);
6438 
6439 	bp1->pp = pp;
6440 	bp1->szc = szc;
6441 	bp1->flags = flags;
6442 	bp1->datap = datap;
6443 
6444 	for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6445 		if ((flags >> cb_index) & 1) {
6446 			break;
6447 		}
6448 	}
6449 
6450 	ASSERT(cb_index != PC_NUM_CALLBACKS);
6451 
6452 	rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6453 	if (pc_cb[cb_index].cb_active) {
6454 		if (pc_cb[cb_index].duration == -1) {
6455 			bp1->expires = (clock_t)-1;
6456 		} else {
6457 			bp1->expires = lbolt + pc_cb[cb_index].duration;
6458 		}
6459 	} else {
6460 		/* There's no callback registered so don't add to the hash */
6461 		rw_exit(&pc_cb[cb_index].cb_rwlock);
6462 		kmem_free(bp1, sizeof (*bp1));
6463 		return;
6464 	}
6465 
6466 	index = PAGE_CAPTURE_HASH(pp);
6467 
6468 	/*
6469 	 * Only allow capture flag to be modified under this mutex.
6470 	 * Prevents multiple entries for same page getting added.
6471 	 */
6472 	mutex_enter(&page_capture_hash[index].pchh_mutex);
6473 
6474 	/*
6475 	 * if not already on the hash, set capture bit and add to the hash
6476 	 */
6477 	if (!(pp->p_toxic & PR_CAPTURE)) {
6478 #ifdef DEBUG
6479 		/* Check for duplicate entries */
6480 		for (l = 0; l < 2; l++) {
6481 			tp1 = page_capture_hash[index].lists[l].next;
6482 			while (tp1 != &page_capture_hash[index].lists[l]) {
6483 				if (tp1->pp == pp) {
6484 					panic("page pp 0x%p already on hash "
6485 					    "at 0x%p\n", pp, tp1);
6486 				}
6487 				tp1 = tp1->next;
6488 			}
6489 		}
6490 
6491 #endif
6492 		page_settoxic(pp, PR_CAPTURE);
6493 		bp1->next = page_capture_hash[index].lists[0].next;
6494 		bp1->prev = &page_capture_hash[index].lists[0];
6495 		bp1->next->prev = bp1;
6496 		page_capture_hash[index].lists[0].next = bp1;
6497 		page_capture_hash[index].num_pages++;
6498 		if (flags & CAPTURE_RETIRE) {
6499 			page_retire_incr_pend_count();
6500 		}
6501 		mutex_exit(&page_capture_hash[index].pchh_mutex);
6502 		rw_exit(&pc_cb[cb_index].cb_rwlock);
6503 		cv_signal(&pc_cv);
6504 		return;
6505 	}
6506 
6507 	/*
6508 	 * A page retire request will replace any other request.
6509 	 * A second physmem request which is for a different process than
6510 	 * the currently registered one will be dropped as there is
6511 	 * no way to hold the private data for both calls.
6512 	 * In the future, once there are more callers, this will have to
6513 	 * be worked out better as there needs to be private storage for
6514 	 * at least each type of caller (maybe have datap be an array of
6515 	 * *void's so that we can index based upon callers index).
6516 	 */
6517 
6518 	/* walk hash list to update expire time */
6519 	for (i = 0; i < 2; i++) {
6520 		bp2 = page_capture_hash[index].lists[i].next;
6521 		while (bp2 != &page_capture_hash[index].lists[i]) {
6522 			if (bp2->pp == pp) {
6523 				if (flags & CAPTURE_RETIRE) {
6524 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6525 						page_retire_incr_pend_count();
6526 						bp2->flags = flags;
6527 						bp2->expires = bp1->expires;
6528 						bp2->datap = datap;
6529 					}
6530 				} else {
6531 					ASSERT(flags & CAPTURE_PHYSMEM);
6532 					if (!(bp2->flags & CAPTURE_RETIRE) &&
6533 					    (datap == bp2->datap)) {
6534 						bp2->expires = bp1->expires;
6535 					}
6536 				}
6537 				mutex_exit(&page_capture_hash[index].
6538 				    pchh_mutex);
6539 				rw_exit(&pc_cb[cb_index].cb_rwlock);
6540 				kmem_free(bp1, sizeof (*bp1));
6541 				return;
6542 			}
6543 			bp2 = bp2->next;
6544 		}
6545 	}
6546 
6547 	/*
6548 	 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes
6549 	 * and thus it either has to be set or not set and can't change
6550 	 * while holding the mutex above.
6551 	 */
6552 	panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", pp);
6553 }
6554 
6555 /*
6556  * We have a page in our hands, lets try and make it ours by turning
6557  * it into a clean page like it had just come off the freelists.
6558  *
6559  * Returns 0 on success, with the page still EXCL locked.
6560  * On failure, the page will be unlocked, and returns EAGAIN
6561  */
6562 static int
6563 page_capture_clean_page(page_t *pp)
6564 {
6565 	page_t *newpp;
6566 	int skip_unlock = 0;
6567 	spgcnt_t count;
6568 	page_t *tpp;
6569 	int ret = 0;
6570 	int extra;
6571 
6572 	ASSERT(PAGE_EXCL(pp));
6573 	ASSERT(!PP_RETIRED(pp));
6574 	ASSERT(curthread->t_flag & T_CAPTURING);
6575 
6576 	if (PP_ISFREE(pp)) {
6577 		if (!page_reclaim(pp, NULL)) {
6578 			skip_unlock = 1;
6579 			ret = EAGAIN;
6580 			goto cleanup;
6581 		}
6582 		ASSERT(pp->p_szc == 0);
6583 		if (pp->p_vnode != NULL) {
6584 			/*
6585 			 * Since this page came from the
6586 			 * cachelist, we must destroy the
6587 			 * old vnode association.
6588 			 */
6589 			page_hashout(pp, NULL);
6590 		}
6591 		goto cleanup;
6592 	}
6593 
6594 	/*
6595 	 * If we know page_relocate will fail, skip it
6596 	 * It could still fail due to a UE on another page but we
6597 	 * can't do anything about that.
6598 	 */
6599 	if (pp->p_toxic & PR_UE) {
6600 		goto skip_relocate;
6601 	}
6602 
6603 	/*
6604 	 * It's possible that pages can not have a vnode as fsflush comes
6605 	 * through and cleans up these pages.  It's ugly but that's how it is.
6606 	 */
6607 	if (pp->p_vnode == NULL) {
6608 		goto skip_relocate;
6609 	}
6610 
6611 	/*
6612 	 * Page was not free, so lets try to relocate it.
6613 	 * page_relocate only works with root pages, so if this is not a root
6614 	 * page, we need to demote it to try and relocate it.
6615 	 * Unfortunately this is the best we can do right now.
6616 	 */
6617 	newpp = NULL;
6618 	if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) {
6619 		if (page_try_demote_pages(pp) == 0) {
6620 			ret = EAGAIN;
6621 			goto cleanup;
6622 		}
6623 	}
6624 	ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL);
6625 	if (ret == 0) {
6626 		page_t *npp;
6627 		/* unlock the new page(s) */
6628 		while (count-- > 0) {
6629 			ASSERT(newpp != NULL);
6630 			npp = newpp;
6631 			page_sub(&newpp, npp);
6632 			page_unlock(npp);
6633 		}
6634 		ASSERT(newpp == NULL);
6635 		/*
6636 		 * Check to see if the page we have is too large.
6637 		 * If so, demote it freeing up the extra pages.
6638 		 */
6639 		if (pp->p_szc > 0) {
6640 			/* For now demote extra pages to szc == 0 */
6641 			extra = page_get_pagecnt(pp->p_szc) - 1;
6642 			while (extra > 0) {
6643 				tpp = pp->p_next;
6644 				page_sub(&pp, tpp);
6645 				tpp->p_szc = 0;
6646 				page_free(tpp, 1);
6647 				extra--;
6648 			}
6649 			/* Make sure to set our page to szc 0 as well */
6650 			ASSERT(pp->p_next == pp && pp->p_prev == pp);
6651 			pp->p_szc = 0;
6652 		}
6653 		goto cleanup;
6654 	} else if (ret == EIO) {
6655 		ret = EAGAIN;
6656 		goto cleanup;
6657 	} else {
6658 		/*
6659 		 * Need to reset return type as we failed to relocate the page
6660 		 * but that does not mean that some of the next steps will not
6661 		 * work.
6662 		 */
6663 		ret = 0;
6664 	}
6665 
6666 skip_relocate:
6667 
6668 	if (pp->p_szc > 0) {
6669 		if (page_try_demote_pages(pp) == 0) {
6670 			ret = EAGAIN;
6671 			goto cleanup;
6672 		}
6673 	}
6674 
6675 	ASSERT(pp->p_szc == 0);
6676 
6677 	if (hat_ismod(pp)) {
6678 		ret = EAGAIN;
6679 		goto cleanup;
6680 	}
6681 	if (PP_ISKAS(pp)) {
6682 		ret = EAGAIN;
6683 		goto cleanup;
6684 	}
6685 	if (pp->p_lckcnt || pp->p_cowcnt) {
6686 		ret = EAGAIN;
6687 		goto cleanup;
6688 	}
6689 
6690 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
6691 	ASSERT(!hat_page_is_mapped(pp));
6692 
6693 	if (hat_ismod(pp)) {
6694 		/*
6695 		 * This is a semi-odd case as the page is now modified but not
6696 		 * mapped as we just unloaded the mappings above.
6697 		 */
6698 		ret = EAGAIN;
6699 		goto cleanup;
6700 	}
6701 	if (pp->p_vnode != NULL) {
6702 		page_hashout(pp, NULL);
6703 	}
6704 
6705 	/*
6706 	 * At this point, the page should be in a clean state and
6707 	 * we can do whatever we want with it.
6708 	 */
6709 
6710 cleanup:
6711 	if (ret != 0) {
6712 		if (!skip_unlock) {
6713 			page_unlock(pp);
6714 		}
6715 	} else {
6716 		ASSERT(pp->p_szc == 0);
6717 		ASSERT(PAGE_EXCL(pp));
6718 
6719 		pp->p_next = pp;
6720 		pp->p_prev = pp;
6721 	}
6722 	return (ret);
6723 }
6724 
6725 /*
6726  * Various callers of page_trycapture() can have different restrictions upon
6727  * what memory they have access to.
6728  * Returns 0 on success, with the following error codes on failure:
6729  *      EPERM - The requested page is long term locked, and thus repeated
6730  *              requests to capture this page will likely fail.
6731  *      ENOMEM - There was not enough free memory in the system to safely
6732  *              map the requested page.
6733  *      ENOENT - The requested page was inside the kernel cage, and the
6734  *              PHYSMEM_CAGE flag was not set.
6735  */
6736 int
6737 page_capture_pre_checks(page_t *pp, uint_t flags)
6738 {
6739 #if defined(__sparc)
6740 	extern struct vnode prom_ppages;
6741 #endif /* __sparc */
6742 
6743 	ASSERT(pp != NULL);
6744 
6745 	/* only physmem currently has restrictions */
6746 	if (!(flags & CAPTURE_PHYSMEM)) {
6747 		return (0);
6748 	}
6749 
6750 #if defined(__sparc)
6751 	if (pp->p_vnode == &prom_ppages) {
6752 		return (EPERM);
6753 	}
6754 
6755 	if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE)) {
6756 		return (ENOENT);
6757 	}
6758 
6759 	if (PP_ISNORELOCKERNEL(pp)) {
6760 		return (EPERM);
6761 	}
6762 #else
6763 	if (PP_ISKAS(pp)) {
6764 		return (EPERM);
6765 	}
6766 #endif /* __sparc */
6767 
6768 	if (availrmem < swapfs_minfree) {
6769 		/*
6770 		 * We won't try to capture this page as we are
6771 		 * running low on memory.
6772 		 */
6773 		return (ENOMEM);
6774 	}
6775 	return (0);
6776 }
6777 
6778 /*
6779  * Once we have a page in our mits, go ahead and complete the capture
6780  * operation.
6781  * Returns 1 on failure where page is no longer needed
6782  * Returns 0 on success
6783  * Returns -1 if there was a transient failure.
6784  * Failure cases must release the SE_EXCL lock on pp (usually via page_free).
6785  */
6786 int
6787 page_capture_take_action(page_t *pp, uint_t flags, void *datap)
6788 {
6789 	int cb_index;
6790 	int ret = 0;
6791 	page_capture_hash_bucket_t *bp1;
6792 	page_capture_hash_bucket_t *bp2;
6793 	int index;
6794 	int found = 0;
6795 	int i;
6796 
6797 	ASSERT(PAGE_EXCL(pp));
6798 	ASSERT(curthread->t_flag & T_CAPTURING);
6799 
6800 	for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6801 		if ((flags >> cb_index) & 1) {
6802 			break;
6803 		}
6804 	}
6805 	ASSERT(cb_index < PC_NUM_CALLBACKS);
6806 
6807 	/*
6808 	 * Remove the entry from the page_capture hash, but don't free it yet
6809 	 * as we may need to put it back.
6810 	 * Since we own the page at this point in time, we should find it
6811 	 * in the hash if this is an ASYNC call.  If we don't it's likely
6812 	 * that the page_capture_async() thread decided that this request
6813 	 * had expired, in which case we just continue on.
6814 	 */
6815 	if (flags & CAPTURE_ASYNC) {
6816 
6817 		index = PAGE_CAPTURE_HASH(pp);
6818 
6819 		mutex_enter(&page_capture_hash[index].pchh_mutex);
6820 		for (i = 0; i < 2 && !found; i++) {
6821 			bp1 = page_capture_hash[index].lists[i].next;
6822 			while (bp1 != &page_capture_hash[index].lists[i]) {
6823 				if (bp1->pp == pp) {
6824 					bp1->next->prev = bp1->prev;
6825 					bp1->prev->next = bp1->next;
6826 					page_capture_hash[index].num_pages--;
6827 					page_clrtoxic(pp, PR_CAPTURE);
6828 					found = 1;
6829 					break;
6830 				}
6831 				bp1 = bp1->next;
6832 			}
6833 		}
6834 		mutex_exit(&page_capture_hash[index].pchh_mutex);
6835 	}
6836 
6837 	/* Synchronize with the unregister func. */
6838 	rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6839 	if (!pc_cb[cb_index].cb_active) {
6840 		page_free(pp, 1);
6841 		rw_exit(&pc_cb[cb_index].cb_rwlock);
6842 		if (found) {
6843 			kmem_free(bp1, sizeof (*bp1));
6844 		}
6845 		return (1);
6846 	}
6847 
6848 	/*
6849 	 * We need to remove the entry from the page capture hash and turn off
6850 	 * the PR_CAPTURE bit before calling the callback.  We'll need to cache
6851 	 * the entry here, and then based upon the return value, cleanup
6852 	 * appropriately or re-add it to the hash, making sure that someone else
6853 	 * hasn't already done so.
6854 	 * It should be rare for the callback to fail and thus it's ok for
6855 	 * the failure path to be a bit complicated as the success path is
6856 	 * cleaner and the locking rules are easier to follow.
6857 	 */
6858 
6859 	ret = pc_cb[cb_index].cb_func(pp, datap, flags);
6860 
6861 	rw_exit(&pc_cb[cb_index].cb_rwlock);
6862 
6863 	/*
6864 	 * If this was an ASYNC request, we need to cleanup the hash if the
6865 	 * callback was successful or if the request was no longer valid.
6866 	 * For non-ASYNC requests, we return failure to map and the caller
6867 	 * will take care of adding the request to the hash.
6868 	 * Note also that the callback itself is responsible for the page
6869 	 * at this point in time in terms of locking ...  The most common
6870 	 * case for the failure path should just be a page_free.
6871 	 */
6872 	if (ret >= 0) {
6873 		if (found) {
6874 			if (bp1->flags & CAPTURE_RETIRE) {
6875 				page_retire_decr_pend_count();
6876 			}
6877 			kmem_free(bp1, sizeof (*bp1));
6878 		}
6879 		return (ret);
6880 	}
6881 	if (!found) {
6882 		return (ret);
6883 	}
6884 
6885 	ASSERT(flags & CAPTURE_ASYNC);
6886 
6887 	/*
6888 	 * Check for expiration time first as we can just free it up if it's
6889 	 * expired.
6890 	 */
6891 	if (lbolt > bp1->expires && bp1->expires != -1) {
6892 		kmem_free(bp1, sizeof (*bp1));
6893 		return (ret);
6894 	}
6895 
6896 	/*
6897 	 * The callback failed and there used to be an entry in the hash for
6898 	 * this page, so we need to add it back to the hash.
6899 	 */
6900 	mutex_enter(&page_capture_hash[index].pchh_mutex);
6901 	if (!(pp->p_toxic & PR_CAPTURE)) {
6902 		/* just add bp1 back to head of walked list */
6903 		page_settoxic(pp, PR_CAPTURE);
6904 		bp1->next = page_capture_hash[index].lists[1].next;
6905 		bp1->prev = &page_capture_hash[index].lists[1];
6906 		bp1->next->prev = bp1;
6907 		page_capture_hash[index].lists[1].next = bp1;
6908 		page_capture_hash[index].num_pages++;
6909 		mutex_exit(&page_capture_hash[index].pchh_mutex);
6910 		return (ret);
6911 	}
6912 
6913 	/*
6914 	 * Otherwise there was a new capture request added to list
6915 	 * Need to make sure that our original data is represented if
6916 	 * appropriate.
6917 	 */
6918 	for (i = 0; i < 2; i++) {
6919 		bp2 = page_capture_hash[index].lists[i].next;
6920 		while (bp2 != &page_capture_hash[index].lists[i]) {
6921 			if (bp2->pp == pp) {
6922 				if (bp1->flags & CAPTURE_RETIRE) {
6923 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6924 						bp2->szc = bp1->szc;
6925 						bp2->flags = bp1->flags;
6926 						bp2->expires = bp1->expires;
6927 						bp2->datap = bp1->datap;
6928 					}
6929 				} else {
6930 					ASSERT(bp1->flags & CAPTURE_PHYSMEM);
6931 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6932 						bp2->szc = bp1->szc;
6933 						bp2->flags = bp1->flags;
6934 						bp2->expires = bp1->expires;
6935 						bp2->datap = bp1->datap;
6936 					}
6937 				}
6938 				mutex_exit(&page_capture_hash[index].
6939 				    pchh_mutex);
6940 				kmem_free(bp1, sizeof (*bp1));
6941 				return (ret);
6942 			}
6943 			bp2 = bp2->next;
6944 		}
6945 	}
6946 	panic("PR_CAPTURE set but not on hash for pp 0x%p\n", pp);
6947 	/*NOTREACHED*/
6948 }
6949 
6950 /*
6951  * Try to capture the given page for the caller specified in the flags
6952  * parameter.  The page will either be captured and handed over to the
6953  * appropriate callback, or will be queued up in the page capture hash
6954  * to be captured asynchronously.
6955  * If the current request is due to an async capture, the page must be
6956  * exclusively locked before calling this function.
6957  * Currently szc must be 0 but in the future this should be expandable to
6958  * other page sizes.
6959  * Returns 0 on success, with the following error codes on failure:
6960  *      EPERM - The requested page is long term locked, and thus repeated
6961  *              requests to capture this page will likely fail.
6962  *      ENOMEM - There was not enough free memory in the system to safely
6963  *              map the requested page.
6964  *      ENOENT - The requested page was inside the kernel cage, and the
6965  *              CAPTURE_GET_CAGE flag was not set.
6966  *	EAGAIN - The requested page could not be capturead at this point in
6967  *		time but future requests will likely work.
6968  *	EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag
6969  *		was not set.
6970  */
6971 int
6972 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
6973 {
6974 	int ret;
6975 	int cb_index;
6976 
6977 	if (flags & CAPTURE_ASYNC) {
6978 		ASSERT(PAGE_EXCL(pp));
6979 		goto async;
6980 	}
6981 
6982 	/* Make sure there's enough availrmem ... */
6983 	ret = page_capture_pre_checks(pp, flags);
6984 	if (ret != 0) {
6985 		return (ret);
6986 	}
6987 
6988 	if (!page_trylock(pp, SE_EXCL)) {
6989 		for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6990 			if ((flags >> cb_index) & 1) {
6991 				break;
6992 			}
6993 		}
6994 		ASSERT(cb_index < PC_NUM_CALLBACKS);
6995 		ret = EAGAIN;
6996 		/* Special case for retired pages */
6997 		if (PP_RETIRED(pp)) {
6998 			if (flags & CAPTURE_GET_RETIRED) {
6999 				if (!page_unretire_pp(pp, PR_UNR_TEMP)) {
7000 					/*
7001 					 * Need to set capture bit and add to
7002 					 * hash so that the page will be
7003 					 * retired when freed.
7004 					 */
7005 					page_capture_add_hash(pp, szc,
7006 					    CAPTURE_RETIRE, NULL);
7007 					ret = 0;
7008 					goto own_page;
7009 				}
7010 			} else {
7011 				return (EBUSY);
7012 			}
7013 		}
7014 		page_capture_add_hash(pp, szc, flags, datap);
7015 		return (ret);
7016 	}
7017 
7018 async:
7019 	ASSERT(PAGE_EXCL(pp));
7020 
7021 	/* Need to check for physmem async requests that availrmem is sane */
7022 	if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) ==
7023 	    (CAPTURE_ASYNC | CAPTURE_PHYSMEM) &&
7024 	    (availrmem < swapfs_minfree)) {
7025 		page_unlock(pp);
7026 		return (ENOMEM);
7027 	}
7028 
7029 	ret = page_capture_clean_page(pp);
7030 
7031 	if (ret != 0) {
7032 		/* We failed to get the page, so lets add it to the hash */
7033 		if (!(flags & CAPTURE_ASYNC)) {
7034 			page_capture_add_hash(pp, szc, flags, datap);
7035 		}
7036 		return (ret);
7037 	}
7038 
7039 own_page:
7040 	ASSERT(PAGE_EXCL(pp));
7041 	ASSERT(pp->p_szc == 0);
7042 
7043 	/* Call the callback */
7044 	ret = page_capture_take_action(pp, flags, datap);
7045 
7046 	if (ret == 0) {
7047 		return (0);
7048 	}
7049 
7050 	/*
7051 	 * Note that in the failure cases from page_capture_take_action, the
7052 	 * EXCL lock will have already been dropped.
7053 	 */
7054 	if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) {
7055 		page_capture_add_hash(pp, szc, flags, datap);
7056 	}
7057 	return (EAGAIN);
7058 }
7059 
7060 int
7061 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
7062 {
7063 	int ret;
7064 
7065 	curthread->t_flag |= T_CAPTURING;
7066 	ret = page_itrycapture(pp, szc, flags, datap);
7067 	curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */
7068 	return (ret);
7069 }
7070 
7071 /*
7072  * When unlocking a page which has the PR_CAPTURE bit set, this routine
7073  * gets called to try and capture the page.
7074  */
7075 void
7076 page_unlock_capture(page_t *pp)
7077 {
7078 	page_capture_hash_bucket_t *bp;
7079 	int index;
7080 	int i;
7081 	uint_t szc;
7082 	uint_t flags = 0;
7083 	void *datap;
7084 	kmutex_t *mp;
7085 	extern vnode_t retired_pages;
7086 
7087 	/*
7088 	 * We need to protect against a possible deadlock here where we own
7089 	 * the vnode page hash mutex and want to acquire it again as there
7090 	 * are locations in the code, where we unlock a page while holding
7091 	 * the mutex which can lead to the page being captured and eventually
7092 	 * end up here.  As we may be hashing out the old page and hashing into
7093 	 * the retire vnode, we need to make sure we don't own them.
7094 	 * Other callbacks who do hash operations also need to make sure that
7095 	 * before they hashin to a vnode that they do not currently own the
7096 	 * vphm mutex otherwise there will be a panic.
7097 	 */
7098 	if (mutex_owned(page_vnode_mutex(&retired_pages))) {
7099 		page_unlock_nocapture(pp);
7100 		return;
7101 	}
7102 	if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) {
7103 		page_unlock_nocapture(pp);
7104 		return;
7105 	}
7106 
7107 	index = PAGE_CAPTURE_HASH(pp);
7108 
7109 	mp = &page_capture_hash[index].pchh_mutex;
7110 	mutex_enter(mp);
7111 	for (i = 0; i < 2; i++) {
7112 		bp = page_capture_hash[index].lists[i].next;
7113 		while (bp != &page_capture_hash[index].lists[i]) {
7114 			if (bp->pp == pp) {
7115 				szc = bp->szc;
7116 				flags = bp->flags | CAPTURE_ASYNC;
7117 				datap = bp->datap;
7118 				mutex_exit(mp);
7119 				(void) page_trycapture(pp, szc, flags, datap);
7120 				return;
7121 			}
7122 			bp = bp->next;
7123 		}
7124 	}
7125 
7126 	/* Failed to find page in hash so clear flags and unlock it. */
7127 	page_clrtoxic(pp, PR_CAPTURE);
7128 	page_unlock(pp);
7129 
7130 	mutex_exit(mp);
7131 }
7132 
7133 void
7134 page_capture_init()
7135 {
7136 	int i;
7137 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7138 		page_capture_hash[i].lists[0].next =
7139 		    &page_capture_hash[i].lists[0];
7140 		page_capture_hash[i].lists[0].prev =
7141 		    &page_capture_hash[i].lists[0];
7142 		page_capture_hash[i].lists[1].next =
7143 		    &page_capture_hash[i].lists[1];
7144 		page_capture_hash[i].lists[1].prev =
7145 		    &page_capture_hash[i].lists[1];
7146 	}
7147 
7148 	pc_thread_shortwait = 23 * hz;
7149 	pc_thread_longwait = 1201 * hz;
7150 	pc_thread_ism_retry = 3;
7151 	mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL);
7152 	cv_init(&pc_cv, NULL, CV_DEFAULT, NULL);
7153 	pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0,
7154 	    TS_RUN, minclsyspri);
7155 }
7156 
7157 /*
7158  * It is necessary to scrub any failing pages prior to reboot in order to
7159  * prevent a latent error trap from occurring on the next boot.
7160  */
7161 void
7162 page_retire_mdboot()
7163 {
7164 	page_t *pp;
7165 	int i, j;
7166 	page_capture_hash_bucket_t *bp;
7167 
7168 	/* walk lists looking for pages to scrub */
7169 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7170 		if (page_capture_hash[i].num_pages == 0)
7171 			continue;
7172 
7173 		mutex_enter(&page_capture_hash[i].pchh_mutex);
7174 
7175 		for (j = 0; j < 2; j++) {
7176 			bp = page_capture_hash[i].lists[j].next;
7177 			while (bp != &page_capture_hash[i].lists[j]) {
7178 				pp = bp->pp;
7179 				if (!PP_ISKAS(pp) && PP_TOXIC(pp)) {
7180 					pp->p_selock = -1;  /* pacify ASSERTs */
7181 					PP_CLRFREE(pp);
7182 					pagescrub(pp, 0, PAGESIZE);
7183 					pp->p_selock = 0;
7184 				}
7185 				bp = bp->next;
7186 			}
7187 		}
7188 		mutex_exit(&page_capture_hash[i].pchh_mutex);
7189 	}
7190 }
7191 
7192 /*
7193  * Walk the page_capture_hash trying to capture pages and also cleanup old
7194  * entries which have expired.
7195  */
7196 void
7197 page_capture_async()
7198 {
7199 	page_t *pp;
7200 	int i;
7201 	int ret;
7202 	page_capture_hash_bucket_t *bp1, *bp2;
7203 	uint_t szc;
7204 	uint_t flags;
7205 	void *datap;
7206 
7207 	/* If there are outstanding pages to be captured, get to work */
7208 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7209 		if (page_capture_hash[i].num_pages == 0)
7210 			continue;
7211 		/* Append list 1 to list 0 and then walk through list 0 */
7212 		mutex_enter(&page_capture_hash[i].pchh_mutex);
7213 		bp1 = &page_capture_hash[i].lists[1];
7214 		bp2 = bp1->next;
7215 		if (bp1 != bp2) {
7216 			bp1->prev->next = page_capture_hash[i].lists[0].next;
7217 			bp2->prev = &page_capture_hash[i].lists[0];
7218 			page_capture_hash[i].lists[0].next->prev = bp1->prev;
7219 			page_capture_hash[i].lists[0].next = bp2;
7220 			bp1->next = bp1;
7221 			bp1->prev = bp1;
7222 		}
7223 
7224 		/* list[1] will be empty now */
7225 
7226 		bp1 = page_capture_hash[i].lists[0].next;
7227 		while (bp1 != &page_capture_hash[i].lists[0]) {
7228 			/* Check expiration time */
7229 			if ((lbolt > bp1->expires && bp1->expires != -1) ||
7230 			    page_deleted(bp1->pp)) {
7231 				page_capture_hash[i].lists[0].next = bp1->next;
7232 				bp1->next->prev =
7233 				    &page_capture_hash[i].lists[0];
7234 				page_capture_hash[i].num_pages--;
7235 
7236 				/*
7237 				 * We can safely remove the PR_CAPTURE bit
7238 				 * without holding the EXCL lock on the page
7239 				 * as the PR_CAPTURE bit requres that the
7240 				 * page_capture_hash[].pchh_mutex be held
7241 				 * to modify it.
7242 				 */
7243 				page_clrtoxic(bp1->pp, PR_CAPTURE);
7244 				mutex_exit(&page_capture_hash[i].pchh_mutex);
7245 				kmem_free(bp1, sizeof (*bp1));
7246 				mutex_enter(&page_capture_hash[i].pchh_mutex);
7247 				bp1 = page_capture_hash[i].lists[0].next;
7248 				continue;
7249 			}
7250 			pp = bp1->pp;
7251 			szc = bp1->szc;
7252 			flags = bp1->flags;
7253 			datap = bp1->datap;
7254 			mutex_exit(&page_capture_hash[i].pchh_mutex);
7255 			if (page_trylock(pp, SE_EXCL)) {
7256 				ret = page_trycapture(pp, szc,
7257 				    flags | CAPTURE_ASYNC, datap);
7258 			} else {
7259 				ret = 1;	/* move to walked hash */
7260 			}
7261 
7262 			if (ret != 0) {
7263 				/* Move to walked hash */
7264 				(void) page_capture_move_to_walked(pp);
7265 			}
7266 			mutex_enter(&page_capture_hash[i].pchh_mutex);
7267 			bp1 = page_capture_hash[i].lists[0].next;
7268 		}
7269 
7270 		mutex_exit(&page_capture_hash[i].pchh_mutex);
7271 	}
7272 }
7273 
7274 /*
7275  * This function is called by the page_capture_thread, and is needed in
7276  * in order to initiate aio cleanup, so that pages used in aio
7277  * will be unlocked and subsequently retired by page_capture_thread.
7278  */
7279 static int
7280 do_aio_cleanup(void)
7281 {
7282 	proc_t *procp;
7283 	int (*aio_cleanup_dr_delete_memory)(proc_t *);
7284 	int cleaned = 0;
7285 
7286 	if (modload("sys", "kaio") == -1) {
7287 		cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio");
7288 		return (0);
7289 	}
7290 	/*
7291 	 * We use the aio_cleanup_dr_delete_memory function to
7292 	 * initiate the actual clean up; this function will wake
7293 	 * up the per-process aio_cleanup_thread.
7294 	 */
7295 	aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
7296 	    modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
7297 	if (aio_cleanup_dr_delete_memory == NULL) {
7298 		cmn_err(CE_WARN,
7299 	    "aio_cleanup_dr_delete_memory not found in kaio");
7300 		return (0);
7301 	}
7302 	mutex_enter(&pidlock);
7303 	for (procp = practive; (procp != NULL); procp = procp->p_next) {
7304 		mutex_enter(&procp->p_lock);
7305 		if (procp->p_aio != NULL) {
7306 			/* cleanup proc's outstanding kaio */
7307 			cleaned += (*aio_cleanup_dr_delete_memory)(procp);
7308 		}
7309 		mutex_exit(&procp->p_lock);
7310 	}
7311 	mutex_exit(&pidlock);
7312 	return (cleaned);
7313 }
7314 
7315 /*
7316  * helper function for page_capture_thread
7317  */
7318 static void
7319 page_capture_handle_outstanding(void)
7320 {
7321 	extern size_t spt_used;
7322 	int ntry;
7323 
7324 	if (!page_retire_pend_count()) {
7325 		/*
7326 		 * Do we really want to be this aggressive
7327 		 * for things other than page_retire?
7328 		 * Maybe have a counter for each callback
7329 		 * type to guide how aggressive we should
7330 		 * be here.  Thus if there's at least one
7331 		 * page for page_retire we go ahead and reap
7332 		 * like this.
7333 		 */
7334 		kmem_reap();
7335 		seg_preap();
7336 		page_capture_async();
7337 	} else {
7338 		/*
7339 		 * There are pages pending retirement, so
7340 		 * we reap prior to attempting to capture.
7341 		 */
7342 		kmem_reap();
7343 		/*
7344 		 * When ISM is in use, we need to disable and
7345 		 * purge the seg_pcache, and initiate aio
7346 		 * cleanup in order to release page locks and
7347 		 * subsquently retire pages in need of
7348 		 * retirement.
7349 		 */
7350 		if (spt_used) {
7351 			/* disable and purge seg_pcache */
7352 			(void) seg_p_disable();
7353 			for (ntry = 0; ntry < pc_thread_ism_retry; ntry++) {
7354 				if (!page_retire_pend_count())
7355 					break;
7356 				if (do_aio_cleanup()) {
7357 					/*
7358 					 * allow the apps cleanup threads
7359 					 * to run
7360 					 */
7361 					delay(pc_thread_shortwait);
7362 				}
7363 				page_capture_async();
7364 			}
7365 			/* reenable seg_pcache */
7366 			seg_p_enable();
7367 		} else {
7368 			seg_preap();
7369 			page_capture_async();
7370 		}
7371 	}
7372 }
7373 
7374 /*
7375  * The page_capture_thread loops forever, looking to see if there are
7376  * pages still waiting to be captured.
7377  */
7378 static void
7379 page_capture_thread(void)
7380 {
7381 	callb_cpr_t c;
7382 	int outstanding;
7383 	int i;
7384 
7385 	CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture");
7386 
7387 	mutex_enter(&pc_thread_mutex);
7388 	for (;;) {
7389 		outstanding = 0;
7390 		for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++)
7391 			outstanding += page_capture_hash[i].num_pages;
7392 		if (outstanding) {
7393 			page_capture_handle_outstanding();
7394 			CALLB_CPR_SAFE_BEGIN(&c);
7395 			(void) cv_timedwait(&pc_cv, &pc_thread_mutex,
7396 			    lbolt + pc_thread_shortwait);
7397 			CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
7398 		} else {
7399 			CALLB_CPR_SAFE_BEGIN(&c);
7400 			(void) cv_timedwait(&pc_cv, &pc_thread_mutex,
7401 			    lbolt + pc_thread_longwait);
7402 			CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
7403 		}
7404 	}
7405 	/*NOTREACHED*/
7406 }
7407